2a3,61
> C+
> C*
> C*   This file contains the following routines in the nodal part 
> C*   of the host-node program :
> C*   
> C*   1) Main program (hnbill)
> C*   2) subroutine get_bill_params - obtains the inputs for the nodal part
> C*   3) subroutine send_bill_params - returns the interpolated results for
> C*                                    either the bilinear or bicubic 
> C*                                    interpolation.
> C*   4) subroutine print_time       - computes the timings of the nodal part.
> C*   5) subroutine nbill            - performs the lasagnia decomposition
> C*                                    of the inputs among the various nodes
> C-
> 
> C+
> C  The main program initializes the nodal part of the host-node and then
> C  enters a loop waiting for a request from the host part of the program. 
> C  
> C  Current requests are :-
> C
> C  1) MPBILL for bilinear interpolation
> C  2) MPBIC  for bicubic interpolation (as in bic1nu.f)
> C
> C  The values are defined in slgcm.mp.  Note slgcm.mpp is no longer used.
> C  Once the nodal program receives a request for performing an 
> C  interpolation,  Node 0 alone issues a series of "gets" to get the
> C  input parameters from the host. Then all nodes execute a call to
> C  nbill to perform the interpolation.  Finally node 0 issues two sends
> C  to return the two interpolated sets of values.  Finally print_time
> C  is called to update timings.  The nodal program terminates when the
> C  host program terminates.  The host program does not currently issue
> C  MPQUIT to cause the nodal program to quit the loop.  Even if the loop
> C  were exited the nodal program does not terminate till the host does.
> C
> C  Irrespective of the interpolation, the same parameters are inputs and
> C  the same parameters are outputs.  [The bicubic interpolation performed
> C  in bicpnu.f is still done by the host (serial code).]
> C
> C
> C  Inputs : wina corresponds to ust (yuupt)
> C           winb    "        "  vst (yvvpt)
> C           rlatitude "      "  spuh (sphi)
> C           rlongitude "     "  sluh (slam)
> C           faclama   
> C           faclamb    
> C           facphia
> C           facphib 
> C           ja, jb      - range of indices
> C           interpol    - bilinear (bicubic)
> C  
> C  Outputs : woua corresponds to c1a (yuupts or yvvpts)
> C            woub     "       "  c2a (yvupts or yuvpts)
> C  
> C  The items in parantheses refers to the bicubic interpolation. For
> C  further details about the quantities one is referred to Yong Li's 
> C  notes.
> C-
> 
6c65
< c1      include "slgcm.mpp"
---
> c1      include "slgcm.mpp"  - Not used any more !!
34,36c93,118
< CT
<          real*8 gcbtime, lcbtime, cabtime
<          common /ntime/ cabtime, gcbtime, lcbtime
---
> C+
> C            cabtime - accumulated total cpu (busy) time
> C            gcbtime - accumulated cpu (busy) time for global communication
> C            lcbtime - accumulated cpu (busy) time for local communication
> C            llitime -     "        "          "    "  calc. ii(i,j), jj(i,j)
> C            brtime  -     "        "          "    "  calc. degree of circ.
> C            cstime  -     "        "          "    "  creating southern part
> C                                                       of segment
> C            cntime  -     "        "          "    "  creating northern part
> C                                                       of segment
> C            reatime -     "        "          "    "  calc. local indices
> C            fbtime  -     "        "          "    "   find_bc_jj 
> C                                                      subroutine overhead
> C            intime  -     "        "          "    "  cubit subroutine
> C                                                      overhead
> C            loitime - NOT USED
> C-
>          real*8 gcbtime, lcbtime, cabtime, llitime, loitime,
>      &          brtime, cstime, cntime, reatime, fbtime, intime
>          common /ntime/ cabtime, gcbtime, lcbtime, llitime, 
>      &          loitime, brtime, cstime, cntime, reatime, fbtime,
>      &          intime
> 
>          real niter, siter
>          common /citer/ niter, siter
> 
42c124,126
< 
---
> C
> C       function prototypes
> C
47c131,133
<         
---
> C 
> C       Initialize the host-node mode        
> C
55,71c141,142
< 	tmp = mp_timer_clear(0)
<         if (tmp .eq. -1) then
<            print *, 'Failure in clearing timer'
<            stop
<         end if
< 
< 	tmp = mp_timer_clear(1)
<         if (tmp .eq. -1) then
<            print *, 'Failure in clearing timer'
<            stop
<         end if
< 
< 	tmp = mp_timer_clear(2)
<         if (tmp .eq. -1) then
<            print *, 'Failure in clearing timer'
<            stop
<         end if
---
> CT      Initialize timing variables
> CT 
74a146,153
>         llitime = 0.0 
>         loitime = 0.0
>         brtime  = 0.0
>         cstime  = 0.0
>         cntime  = 0.0 
>         reatime = 0.0 
>         fbtime  = 0.0
>         intime  = 0.0
77a157,159
> C
> C       Receive request from host and enter loop.
> C
79a162,164
> C
> C          Keep a count of the number of interpolations
> C
81a167,171
> C
> C         Request host to stop serving as a server for serial i/o
> C         Communication with host can only take place after this 
> C         is done.
> C
83a174,176
> C
> C          Get the input parameters
> C
88a182,183
> C           Debug code
> C 
102c197,199
<       
---
> C
> C          Perform the lasagnia decomposition and interpolate
> C      
105c202
<            print *, "I am in nodal main with ", status
---
> D           print *, "I am in nodal main with ", status
110,112c207,209
< C!
< C!        Uncomment if nodes are using UNIX system calls
<               
---
> C
> C          Update timings for the interpolation
> C              
114c211
<              call print_time(cabtime, gcbtime, lcbtime, count)
---
>              call print_time(count)
117a215,217
> C
> C         Return the interpolated results to the host.
> C
122,125d221
< C1           do while (mp_msg_pending(host,0) .eq. 0)
< C1              count = count + 1
< C1           end do
< 
129,170d224
< CT
< CT        call mp_sync()
< CT
< CT        gcbtime = mp_busy_time(0)
< CT        lcbtime  = mp_busy_time(1)
< CT        cabtime  = mp_busy_time(2)
< CT        cabtime  = cabtime - lcbtime - gcbtime
< CT
< CT        ttime = cmmd_reduce_double(gcbtime, cmmd_combiner_add)
< CT        gcbtime = ttime /  CMMD_partition_size()
< CT        gcbtime = ttime /  (CMMD_partition_size() * 28.0)
< CT          ttime = cmmd_reduce_double(lcbtime, cmmd_combiner_add)
< CT    lcbtime = ttime /  (CMMD_partition_size() * 28.0)
< CT          lcbtime = ttime /  CMMD_partition_size()
< CT          ttime = cmmd_reduce_double(cabtime, cmmd_combiner_add)
< CT        cabtime = ttime /  (CMMD_partition_size() * 28.0)
< CT          cabtime = ttime /  CMMD_partition_size()
< CT        cobtime = lcbtime + gcbtime
< CT        ttime = cobtime + cabtime
< CT        
< CT
< CT 200    format (A, E22.15.3)
< CT 201    format (A, F12.7)
< CT        call cmmd_sync_with_host()
< CT
< CT        if (node .eq. 0) then
< CT           print *, "Writing Timings"
< CT           open (unit = 20, file = 'timings.dat', status = 'new' )
< CT           write (20,*), "IM : ", im, " JM : ", jm
< CT           write (20, 200), " Global Comm. Time : ", gcbtime
< CT           write (20, 200), " Other Comm. Time : ",  lcbtime
< CT           write (20, 201), " Total Comm. Time : ",  cobtime
< CT           write (20, 201), " Calculation Time : ",  cabtime
< CT           write (20, 201), " Total Time : ", ttime
< CT           write (20, 201), " Ratio : ", (cabtime / cobtime)
< CT           close(20)
< CT        endif
< CT
< CT        call mp_suspend_server()
< CT        
< CT        call cmmd_sync_with_host()
< CT
172a227,249
> 
> C+
> C  Subroutine get_bill_params
> C  It issues a "get" or "receive" for every input from the host.
> C  But conventionally these are the output parameters of this 
> C  subroutine.
> C
> C     Input Parameters : None
> C  
> C     Output Parameters :
> C           wina corresponds to ust (yuupt)
> C           winb    "        "  vst (yvvpt)
> C           rlatitude "      "  spuh (sphi)
> C           rlongitude "     "  sluh (slam)
> C           faclama   
> C           faclamb    
> C           facphia
> C           facphib 
> C           ja, jb      - range of indices
> C           interpol    - bilinear (bicubic)
> C  
> C-
> 
200d276
< C1         host = cmmd_host_node()
203,213d278
< C1         status = mp_rcv_rblock(host, 0, wina, imp3 * jmp3 * 4)
< C1         status = mp_rcv_rblock(host, 0, winb, imp3 * jmp3 * 4)
< C1         status = mp_rcv_rblock(host, 0, rlatitude, imp3 * jm * 4)
< C1         status = mp_rcv_rblock(host, 0, rlongitude, imp3 * jm * 4)
< C1         status = mp_rcv_rblock(host, 0, faclama, 4)
< C1         status = mp_rcv_rblock(host, 0, faclamb, 4)
< C1         status = mp_rcv_rblock(host, 0, facphia, 4)
< C1         status = mp_rcv_rblock(host, 0, facphib, 4)
< C1         status = mp_rcv_rblock(host, 0, ja, 4)
< C1         status = mp_rcv_rblock(host, 0, jb, 4)
< 
226a292,305
> 
> C+
> C  Subroutine send_bill_params
> C  It issues a "send" for the two interpolated results from the 
> C  nodes. By convention these are the input parameters for this 
> C  subroutine.
> C
> C     Input Parameters : 
> C            woua corresponds to c1a (yuupts or yvvpts)
> C            woub     "       "  c2a (yvupts or yuvpts)
> C  
> C     Output Parameters : None
> C  
> C-
229d307
< c1      include "slgcm.mpp"
241d318
< C1         host = cmmd_host_node()
244,245d320
< C1         status = cmmd_send(host, 0, woua, imp3 * jm * 4)
< C1         status = cmmd_send(host, 0, woub, imp3 * jm * 4)
253d327
<       subroutine print_time(cabtime, gcbtime, lcbtime, count)
255c329,345
<       real*8 cabtime, gcbtime, lcbtime
---
> C+
> C  Subroutine print_time
> C  This routine prints the updated timings for each request for 
> C  bilinear (bicubic) interpolation.  The variable count
> C  keeps track of the number of times the interpolation is
> C  performed.  The latest timings are written to the file
> C  'timings.dat'. The older timings are wiped out in the process.
> C
> C     Input Parameters : 
> C            count   - number of times the interpolations was 
> C                      performed.
> C  
> C     Output Parameters : None
> C  
> C-
>       subroutine print_time(count)
> 
257a348,356
>          real*8 gcbtime, lcbtime, cabtime, llitime, loitime,
>      &          brtime, cstime, cntime, reatime, fbtime, intime
>          common /ntime/ cabtime, gcbtime, lcbtime, llitime, 
>      &          loitime, brtime, cstime, cntime, reatime, fbtime,
>      &          intime
> 
>          real niter, siter
>          common /citer/ niter, siter
> 
262a362
>         real titer
264c364,368
<         print *, "Total time : ", cabtime
---
> D        print *, "Total time : ", cabtime
> C
> C      Find the average calculation and communication 
> C      times.
> C
269c373,377
< 
---
>         
> C
> C      Find averaged total commn. time and
> C      avgd. total commn. + calcn. time
> C
275c383
<            print *, "+Writing Timings"
---
> D           print *, "+Writing Timings"
276a385,387
> C
> C          Overwrite contents of timings.dat with new timings.
> C
287a399,423
>            ttime = llitime / count
>            write (20, 201), "Lat Long Calcns : ", ttime
>            ttime = brtime / count
>            write (20,201), "Degree of Circ. : ", ttime
>            ttime = cstime / count
>            write (20,201), "Comm. South : ", ttime
>            ttime = cntime / count
>            write (20,201), "Comm. North : ", ttime
>            ttime = reatime / count
>            write (20,201), "Local Indices : ", ttime
>            ttime = fbtime / count
>            write (20,201), "Find BC JJ overhead : ", ttime
>            ttime = intime / count
>            write (20,201), "Interpolation overhead : ", ttime
>            ttime = llitime + brtime + cstime + cntime + reatime
>      &             + fbtime + intime
>            ttime = ttime / count
>            write (20, 201), " Total Time (no fn. overhead) : ", ttime
> 
> C
> C          Find the ratio of calculation to communication and 
> C          write it to the file
> C
>            ttime = ttime / cobtime
>            write (20, 201), " Adjusted Ratio : ", ttime
289c425,432
<            write (20, 201), " Ratio : ", ttime
---
>            write (20, 201), " Ratio with overhead : ", ttime
> 
>            titer = niter / count
>            write (20, 201), "North Iter : ", titer
>            titer = siter / count
>            write (20, 201), "South Iter : ", titer
>            titer = (siter + niter) / count
>            write (20, 201), "Total Iter : ", titer
295,296c438,469
< c
< c
---
> C+
> C  Subroutine nbill
> C  It performs a lasagnia decomposition on the input arrays      
> C  wina, winb, woua, woub, rlongitude, rlatitude on Node 0.  
> C  The width of lasagnia strip is precomputed and entered
> C  into the header file slgcm.mp.  This is because it is
> C  not possible to define the size of an array at run time
> C  in f77.  The width depends on the resolution of the 
> C  input arrays wina (or winb) and the 
> C  number of nodes being used on a particular machine.
> C  It then ships the partitions to the other nodes.  It 
> C  calls the subroutine bill_mp or bic_mp depending on 
> C  whether it is a bilinear or bicubic interpolation.
> C  The other nodes then return the interpolated results
> C  to node 0 after performing the required interpolation.
> C
> C
> C  Inputs : wina corresponds to ust (yuupt)
> C           winb    "        "  vst (yvvpt)
> C           rlatitude "      "  spuh (sphi)
> C           rlongitude "     "  sluh (slam)
> C           faclama   
> C           faclamb    
> C           facphia
> C           facphib 
> C           ja, jb      - range of indices
> C           interpol    - bilinear (bicubic)
> C  
> C  Outputs : woua corresponds to c1a (yuupts or yvvpts)
> C            woub     "       "  c2a (yvupts or yuvpts)
> C  
> C-
318a492,495
> 
> C
> C     variables used for subscripting arrays in debug mode
> C
320a498,502
> C
> C     Common block which stores the address of neighbors.  The number
> C     of nodes in a particular dimension.  See also the routine
> C     set_lsd_commons
> C
324,325c506,512
<          real*8 gcbtime, lcbtime, cabtime
<          common /ntime/ cabtime, gcbtime, lcbtime
---
> CT       Timing code
> CT 
>          real*8 gcbtime, lcbtime, cabtime, llitime, loitime,
>      &          brtime, cstime, cntime, reatime, fbtime, intime
>          common /ntime/ cabtime, gcbtime, lcbtime, llitime, 
>      &          loitime, brtime, cstime, cntime, reatime, fbtime,
>      &          intime
328,333c515,521
<       integer count,
<      &        lower,
<      &        upper,
<      &        i, tmp,
<      &        range,
<      &        j, k, dny0
---
>       integer count,               ! not used anymore
>      &        lower,               ! lower bound of partition
>      &        upper,               ! upper bound of partition
>      &        i, tmp,              ! temporary variables
>      &        range,               ! width of partition
>      &        j, k,                ! counters
>      &        dny0                 ! not used anymore
335c523,526
<       integer mp_rcv_int
---
> C
> C     Function prototypes
> C
>       integer mp_rcv_int           
341d531
<       parameter (count = imp3)
343c533,539
<        dimension wina_d(imp3, max_col_p3)
---
>       parameter (count = imp3)    ! not used anymore
> 
> C
> C      buffers to hold partitions. Array sections are not
> C      allowed in f77.  
> C
>        dimension wina_d(imp3, max_col_p3)        
351c547
< C                      		          ! the fields wina and winb
---
> C                      		         
352a549,550
> CT    function prototypes for the timing code.
> CT  
359,360d556
< C     make the IO independent
< 
362a559,560
> 
> C     make the IO independent
363a562
> D      print *, "In subrt. nbill"
365a565,568
> C
> C      Set the common block variables such as the address of
> C      northern, southern neighbors etc.
> C
372a576,578
> C
> C       Initialize with the bounds of the first partition (on node 0)
> C
376,377c582,591
<         do k = 2, ny -1
<            lower = upper + 1
---
> C
> C     Copy the partition into temporary buffers.  
> C     This is done as array sections cannot be
> C     passed in f77 as an argument.  The last 
> C     node is dealt with separately, as the
> C     input arrays cannot be split into 32
> C     equal partitions.  
> C
>         do k = 2, ny -1                 ! for node 1 to 30
>            lower = upper + 1            ! increment bounds for next partition
379c593
<            range = upper - lower + 1
---
>            range = upper - lower + 1    
381c595,598
<            do j = 1, range
---
> C
> C          Copy into temporary buffers
> C
>            do j = 1, range              
387d603
< C1                 print 107, k, lower, wina(i, lower + j-1), wina_d(i,j)    
391a608,611
> C
> C   Ship the bounds of the partition followed by the
> C   buffers themselves.
> C
403,404d622
< C3
< C3        Debug
406,407c624,630
< C3        Debug
< C3          
---
> 
> 
> C
> C        For the last node the partitions of rlatitude
> C        and rlongitude are of the same size, but differ
> C        from that of the wina, winb partitions.
> C
426,427d648
< C1                 print 107, ny-1, lower, 
< C1     &            wina(i, lower + j-1), wina_d(i,j)                 
431,433c652,655
< c
< c     Need only the bounds of the decomposed wina, winb arrays
< c
---
> C
> C     Need only the bounds of the decomposed wina, winb arrays
> C     Ship the bounds and partitions to node 31 (last node)
> C
444,445d665
< C3
< C3        Debug
447,448c667,672
< C3        Debug
< C3          
---
> 
> C
> C           Copy the first partition into a buffer 
> C           (for node 0) before calling the 
> C           interpolation routine bill_mp (bic_mp) on node 0
> C
460d683
< C1                 print 107, wina(i,j), wina_d(i,j)
464,465c687,688
< C3
< C3        Debug
---
> 
> 
467,468d689
< C3        Debug
< C3          
469a691,694
> C
> C        Send the other inputs (scalar quantities) to
> C        all the remianing nodes.
> C
479a705,708
> C
> C     If not node 0 receive the various inputs.
> C
> 
500,501d728
< C3
< C3        Debug
503,504d729
< C3        Debug
< C3          
506c731,734
< C     Interpolate
---
> C
> C     Perform a bilinear (or bicubic) interpolation depending
> C     on the request.
> C
508,511c736,740
<          if (ny0 .eq. 0) then
<             print *, "I am going to call bill_mp"
<          endif 
< C***    Timing Code 
---
> D         if (ny0 .eq. 0) then
> D            print *, "I am going to call bill_mp"
> D         endif 
> CT
> CT    Timing Code for measuring total time for interpolation.
522c751,752
< C***    Timing Code 
---
> CT    Timing Code 
> CT
528c758
< C***    Timing Code 
---
> CT    Timing Code 
534c764,766
< C***    Timing Code 
---
> C
> C       Accumulate total times for bilinear interpolations.
> C
535a768
> CT    Timing Code 
538,541c771,775
<          if (ny0 .eq. 0) then
<             print *, "I am going to call bic_mp"
<          endif 
< C***    Timing Code 
---
> D         if (ny0 .eq. 0) then
> D            print *, "I am going to call bic_mp"
> D         endif 
> CT
> CT    Timing Code for measuring total times in bicubic interpolation.
552,553c786,787
< C***    Timing Code 
< 
---
> CT    Timing Code 
> CT
559c793
< C***    Timing Code 
---
> CT    Timing Code 
565c799,802
< C***    Timing Code 
---
> CT   Timing Code 
> C
> C     Accumulate total times for all bicubic interpolations
> C
568c805,808
<        
---
>   
> C
> C     If not node 0, return interpolated results.
> C     
574,575c814
< C3
< C3        Debug
---
> 
577,578c816
< C3        Debug
< C3          
---
> 
579a818,822
> C
> C         If node 0, copy the results into buffer woua, woub of 
> C         size (imp3.jm). Receive the results from other nodes 
> C         and copy them into the buffer at the respective places.
> C
584,609d826
< CC	  do k = 1, ny
< CC	    do j = 1, range
< CC	      do i = 1, imp3
< CC	        woua(i, lower + j -1) = woua_d(i,j)
< CC	        woub(i, lower + j -1) = woub_d(i,j)
< CC	      end do
< CC            end do
< CC	    
< CC	    lower = upper + 1
< CC	    if (k .eq. ny) then 
< CC            if (k .eq. (ny-1)) then
< CC	      upper = jm
< CC	    else
< CC              upper = upper + max_col
< CC	    endif
< CC	
< CC	    range = upper - lower + 1
< CC	    
< CC	    if (k .ne. 1) then
< CC	      tmp = mp_rcv_rblock(k -1, k -1, woua_d,
< CC     &                            imp3, max_col_p3)
< CC      	      tmp = mp_rcv_rblock(k -1, k -1, woub_d,
< CC     &                            imp3, max_col_p3)
< CC	    end if
< CC	 end do
< 
617c834
< 	  do k = 1, (ny - 1)
---
> 	  do k = 1, (ny - 1)             ! for nodes 1 to 31
634a852,855
> C
> C           Copy all interpolated results into the original output
> C           buffer, before returning to host program.
> C
644c865
<  101  format( 1X, 8('+', 4F16.8 //))
---
> C1 101  format( 1X, 8('+', 4F16.8 //))
646,649c867,870
< CC       print *, 'Count = ', count
< CC       print *, '+WOUA'
< CC       print 101, ((woua(dk, dj), dj = 1, jm), dk = 1, imp3)
< CC       print *, '+WOUA OVER'
---
> C1       print *, 'Count = ', count
> C1       print *, '+WOUA'
> C1       print 101, ((woua(dk, dj), dj = 1, jm), dk = 1, imp3)
> C1       print *, '+WOUA OVER'
684a906,909
> 
> 
> 
> 
