%% LATEX TWICE TO GET THE REFERENCES RIGHT
%%\documentstyle[12pt]{article}
\documentstyle[12pt,epsf,psfig]{article}

\newcommand{\postscript}[2]
 {\setlength{\epsfxsize}{#2\hsize}
  \centerline{\epsfbox{#1}}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newtheorem{theorem}{Theorem}
\newtheorem{itlemma}{Lemma}[section] %number by section (set in \em by default)
%%%same numbering counter as itlemmas:
\newtheorem{itproposition}[itlemma]{Proposition}
\newtheorem{itcorollary}[itlemma]{Corollary}
\newtheorem{itremark}[itlemma]{Remark}
\newtheorem{itnote}[itlemma]{Note}
\newtheorem{itobservation}[itlemma]{Observation}
\newtheorem{itdefinition}[itlemma]{Definition}
\newtheorem{itexercise}[itlemma]{Exercise}
\newtheorem{itexample}[itlemma]{Example}

\newenvironment{lemma}{\begin{itlemma}\rm}{\end{itlemma}} %no-italics
\newenvironment{remark}{\begin{itremark}\rm}{\end{itremark}} %no-italics
\newenvironment{observation}{\begin{itobservation}\rm}{\end{itobservation}} %no-italics
\newenvironment{corollary}{\begin{itcorollary}\rm}{\end{itcorollary}}
\newenvironment{proposition}{\begin{itproposition}\rm}{\end{itproposition}}
\newenvironment{definition}{\begin{itdefinition}\rm}{\end{itdefinition}}
\newenvironment{exercise}{\begin{itexercise}\rm}{\end{itexercise}}
\newenvironment{example}{\begin{itexample}\rm}{\end{itexample}}

\tolerance=1600 %allow some tolerance in extending after line.
\parskip=3pt plus 1pt minus 1pt
\overfullrule=0pt %no dark lines if overfull

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% This are for choices.  They take 4 or 6 args
%% (carriage returns allowed in between}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand{\twoif}[4]{
\left\{ \begin{array}{ll}#1&#2\\#3&#4\end{array}\right.
}
\newcommand{\threeif}[6]{
\left\{ \begin{array}{ll}#1&#2\\#3&#4\\#5&#6\end{array}\right.
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\def\partmlab#1#2#3#4#5#6#7#8{\bordermatrix{&#1&#2\cr#3&#5&#6\cr
					\noalign{\medskip}#4&#7&#8\cr}}
%% For 2by2 partitioned matrices; arguments are: two labels of top, side,
%% 4 entries (row by row).  If entries are themselves matrix or pmatrix,
%% better for readability of source to define first: \def\matrix1{...} etc.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\bigframebox}[1]{\framebox{\parbox{\smallerwidth}{#1}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Warning: use \noindent before!
% AND IN TEXT, after begin doc, do:
% \newlength{\smallerwidth}  %for bigboxes
% \setlength{\smallerwidth}{\textwidth}
% \addtolength{\smallerwidth}{-8pt}        %MUST BE DONE AFTER begin doc
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\def \beq {\begin{eqnarray}}
\def \eeq {\end{eqnarray}}
\def \beqn {\begin{eqnarray*}}
\def \eeqn {\end{eqnarray*}}

\newcommand{\rf}[1]{[\ref{#1}]}            %bibliographical references

\newcommand{\text}[1]{\hbox{\rm \ #1\ \/}}
\newcommand{\be}[1]{\begin{equation}\label{#1}}
\newcommand{\ee}{\end{equation}}
\newcommand{\bl}[1]{\begin{lemma}\label{#1}}
\newcommand{\ble}[1]{\begin{lemmaex}\label{#1}}
\newcommand{\br}[1]{\begin{remark}\label{#1}}
\newcommand{\bn}[1]{\begin{note}\label{#1}}
\newcommand{\bo}[1]{\begin{observation}\label{#1}}
\newcommand{\bt}[1]{\begin{theorem}\label{#1}}
\newcommand{\bd}[1]{\begin{definition}\label{#1}}
\newcommand{\bp}[1]{\begin{proposition}\label{#1}}
\newcommand{\bc}[1]{\begin{corollary}\label{#1}}
\newcommand{\bfact}[1]{\begin{fact}\label{#1}}
\newcommand{\ber}[1]{\begin{exercise}\label{#1}}
\newcommand{\bex}[1]{\begin{example}\label{#1}}
\newcommand{\bem}[1]{\begin{example}\label{#1}}    %Yes, 2 different ones...
\newcommand{\ec}{\mybox\end{corollary}}
\newcommand{\efact}{\mybox\end{fact}}
\newcommand{\eer}{\mybox\end{exercise}}
\newcommand{\eex}{\mybox\end{example}}
\newcommand{\eem}{\mybox\end{example}}
\newcommand{\el}{\mybox\end{lemma}}
\newcommand{\ele}{\mybox\end{lemmaex}}
\newcommand{\er}{\mybox\end{remark}}
\newcommand{\en}{\mybox\end{note}}
\newcommand{\eo}{\mybox\end{observation}}
\newcommand{\et}{\qed\end{theorem}}
\newcommand{\ed}{\mybox\end{definition}}
\newcommand{\ep}{\mybox\end{proposition}}
\newcommand{\epr}{\end{proof}}
\newcommand{\bpr}{\begin{proof}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%  These are just for statements, no "qed" box is put in.
%%  (Also for when I have to finish something after a displayed equation
%%        or itemized.)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand{\ecs}{\end{corollary}}
\newcommand{\eers}{\end{exercise}}
\newcommand{\eexs}{\end{example}}
\newcommand{\eems}{\end{example}}
\newcommand{\els}{\end{lemma}}
\newcommand{\eles}{\end{lemmaex}}
\newcommand{\ers}{\end{remark}}
\newcommand{\ens}{\end{note}}
\newcommand{\eos}{\end{observation}}
\newcommand{\ets}{\end{theorem}}
\newcommand{\eds}{\end{definition}}
\newcommand{\eps}{\end{proposition}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\qed}{\hfill \halmos} %put \qed at right margin
\newcommand{\mybox}{\hfill $\Box$} %put \qed at right margin (white square)

%%%\newcommand{\pmat}[2]{\left( \begin{array}{#1} #2 \end{array} \right) }
%%%use as e.g. \pmat{cc}{ a&b\\ a &c }  Note 'cc' for 2 Centered cols
%%%can also use \pmatrix{ a&b\cr a &c \cr} for plainTeX standard construct
%%%\setcounter{page}{27} should set the CURRENT page to 27

\def \R {\bb R} % reals
\def \C {\bb C} % complexes
\def \Z {\bb Z} % integers
\def \X {\bb X} % for this paper only!
\def \inter {{\rm int \,}}
\def \rank {{\rm rank \,}}
%%%\def \inf {{\rm inf \,}}  %better in Latex -- subscripts OK!
%%%\def \min {{\rm min \,}}
\def \im {{\rm im \,}}
\def \ker {{\rm ker \,}}
\def \norm #1{\Vert #1 \Vert}
\def \ip #1\by #2{\langle #1 , #2 \rangle}
\def \Sone {{\bf S}^1} % circle
\def \wigglearrow {\sim \mkern -6mu >}

\def \today {\ifcase \month \or January\or February\or  
  March\or April\or May\or June\or July\or August\or 
  September\or October\or November\or December\fi\   
  \number \day, \number \year}                       
\newcommand{\comment}[1]{}
\newcommand{\halmos}{\rule{1ex}{1.4ex}}
%%%alternative to \qed (see below)
\newenvironment{proof}{\noindent {\em Proof}.\ }{\hspace*{\fill}$\halmos$\medskip}

%%%\def\qed{\kern 1em \vrule height .5em width .5em depth 0em}
%%%(I now use new qed: in right margin)

\def\vbar{\mathchoice{\vrule height6.3ptdepth-.5ptwidth.8pt\kern-.8pt}
   {\vrule height6.3ptdepth-.5ptwidth.8pt\kern-.8pt}
   {\vrule height4.1ptdepth-.35ptwidth.6pt\kern-.6pt}
   {\vrule height3.1ptdepth-.25ptwidth.5pt\kern-.5pt}}
\def\fudge{\mathchoice{}{}{\mkern.5mu}{\mkern.8mu}}
\def\bbc#1#2{{\rm \mkern#2mu\vbar\mkern-#2mu#1}}
\def\bbb#1{{\rm I\mkern-3.5mu #1}}
\def\bba#1#2{{\rm #1\mkern-#2mu\fudge #1}}
\def\bb#1{{\count4=`#1 \advance\count4by-64 \ifcase\count4\or\bba A{11.5}\or
   \bbb B\or\bbc C{5}\or\bbb D\or\bbb E\or\bbb F \or\bbc G{5}\or\bbb H\or
   \bbb I\or\bbc J{3}\or\bbb K\or\bbb L \or\bbb M\or\bbb N\or\bbc O{5} \or
   \bbb P\or\bbc Q{5}\or\bbb R\or\bbc S{4.2}\or\bba T{10.5}\or\bbc U{5}\or
%%%   \bbb P\or\bbc Q{5}\or\bbb R\or\bba S{8}\or\bba T{10.5}\or\bbc U{5}\or
   \bba V{12}\or\bba W{16.5}\or\bba X{11}\or\bba Y{11.7}\or\bba Z{7.5}\fi}}
%
%
\renewcommand{\thefootnote}{\fnsymbol{footnote}}
%%% read as: theFOOTNOTE (i.e. the command  executed when doing a footnote) is
%%% now to be set as a FOOTNOTESYMBOL (type of number) value of counter
%%% FOOTNOTE (Instead of Roman).  See LaTeX manual pp.91-92 as well as 175.
\def\bi{\begin{itemize}}
\def\ei{\end{itemize}}
\def\ben{\begin{enumerate}}
\def\een{\end{enumerate}}
\def\st{\, | \,}

\newcommand{\myboxineq}{\eqno\Box} %put \qed at margin (white square) in eqn
\newcommand{\qedineq}{\eqno\halmos} %put \qed at margin in eqn

\topmargin -0.5in %default is 0.0, i.e. 1 inch from top.  Now it is 0.5
\textheight 9in
\oddsidemargin 0.25in %more reasonable margins!  (This is 1.25; can use <0 too)
\evensidemargin 0.25in %(same as above)
\textwidth 6.25in %(could use also -.5,-.5,7.5 for instance)
\tolerance=1600 %allow some tolerance in extending after line.
\parskip=3pt plus 1pt minus 1pt
\overfullrule=0pt %no dark lines if overfull

%%%% FROM HERE, macros special for paper
\def\mm{m}
\def\pp{p}
\def\proj{\pi _1} % projection on first group of coordinates
\def\x{u}  % vector argument
\def\SEC{(SEC)}
\def\SECo{(SEC$^0$)}
\def\INV{(INV)}
\def\Fp{{\cal F}_\pp}
\def\Fpm{{\cal F}_\pp^\mm}  %there is a 3,3 and a 2,1 in text

\def\Rm{\R^m}
\def\Rn{\R^n}
\newcommand{\ball}[2]{{\bb B}(#1,#2)}
\newcommand{\cball}[2]{\overline {\bb B} (#1,#2)}
\def\Fnm{{\cal F}_n^\mm}
\def\clos{{\rm clos\,}}
\def\inter{{\rm int\,}}

\def\CCF{compatible class of functions}
\begin{document}

\pagestyle{empty}
\begin{flushright}
{\sl Example in HPF}
\end{flushright}

\vspace{1cm}
\vspace{1cm}
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\vspace{.05cm}
\hrule
\vspace{.25cm}
{\bf 
HPF Program Solving a Laplace Equation Over a Rectangle by the
Red/Black SOR technique.
}
\vspace{.25cm}
\hrule
\vspace{.05cm}
\hrule
\hrule
\hrule
\hrule
\hrule
\vspace{1cm}

\begin{center}
Saleh Elmohamed\\
Northeast Parallel Architectures Center,\\
Syracuse University\\
Syracuse, NY 13244-4100\\
saleh@npac.syr.edu\\
\end{center}
\newpage
\pagestyle{empty}
\tableofcontents

\newpage

\pagestyle{plain}
\pagestyle{myheadings}
\markright{\sl Example in HPF}
\section{The Problem}
\setcounter{page}{1}

\subsection{Part 1}
Is to solve a Laplace equation over a rectangle by the Red/Black SOR
method. An HPF program is written to carry out the solution. Also, the
program was ran on the SP2 (minimum of 4 nodes) and uses array grid
sizes on the order of $N$ by $N$, where $N = 100$.

\subsection{Part 2}
The second part of the problem is the investigation of the optimum
value of the over-relaxation parameter $w$ by making a minimum of $10$
runs for values of $w$ between $1$ and $2$.

\section{Brief Description of Our Approach}

Basically, our method is to implement Gauss-Seidel (GS) algorithm in
parallel which is a bit tricky to do, nonetheless, it is doable.
One possible implementation is that of what is commonly referred to
as the odd-even (or maybe the even-odd) Gauss-Seidel method. We
start with a two-dimensional grid of $M$ points, then partition it
into two sets of points. The next step is do the updating as follows:
\begin{enumerate}
\item Update the even set yielding, for example, values $X$.
\item Using the $X$ values, carry an update to the odd set, yielding $Y$.
\end{enumerate}

So, essentially what constitute the GS algorithm is this
semi-multi-stage approach of update/compute or update then compute.

On the aspect of parallelism, we could say that parallelism of the SOR
method is pretty much similar to parallelism of the GS algorithm
except for the over relaxation parameter $\omega$ which is used in the
the SOR update as follows (see Professor Fox's notes):

\be{SOR-update}
z^{k+1}_{SOR} = \omega z^{k+1}_{GS} + (1 - \omega)z^k
\ee

Where $1 < \omega < 2$ mainly for convergence purposes. 
\begin{itemize}

\item A good choice of $\omega$ would be a value close to 2, something
like $1.85$ or $1.9$. 

\item The chosen criterion for convergence is that the difference
between two consecutive iterations equal to some very small value,
equal to $0.0000001$.

\item For the Laplace heat equation, we have used the following format
to deal with the heat distribution:

\be{boundary}
U(x,y) = -4.7 + 0.2 x - 0.3 y + 0.09 (x^2 - y^2) + 
ln(\sqrt((x + 0.2)^2 + (y + 0.2)^2))
\ee
Equation \ref{boundary} is used here to derive the boundary values of
the grid.
\end{itemize}

\section{The HPF Code and Some Analysis}

To handle aspects of parallelism, we will be using {\em forall}
constructs as well as the HPF compiler directives {\em DISTRIBUTE} and
{\em INHERIT} for the updates. Also we will use runtime {\em stat} to
display system timings, etc.


\subsection{The Program}

\begin{verbatim}

!===================================================================
!Author: Saleh Elmohamed
!Purpose: an HPF to solve a Laplace equation over a rectangle by the 
!         Red/Black SOR method.
!
!Hardware platform: nodes (merlin1, etc. ) of the IBM SP2.
!===================================================================

      program Laplace_SOR

      interface
         subroutine SOR_iteration(Grid,oldGrid,int_x,int_y,error,omega)
            double precision, dimension(1:int_x,1:int_y)::Grid,oldGrid
            integer int_x,int_y
            double precision error,omega
         end subroutine

         subroutine compute_val_of_U(Uval,x,y)
            double precision Uval,x,y
         end subroutine

         subroutine assign_boundary_values(U,int_x,int_y)
            double precision, dimension(1:int_x,1:int_y) :: U
            integer int_x,int_y
         end subroutine
      end interface

!!! Set the array grid sizes ....
      parameter(int_x = 100)
      parameter(int_y = 100)
      parameter(thresh = 1.0d-7)   !! or the level of tolerance ...

      double precision error,omega

      double precision, dimension(1:int_x,1:int_y) :: Grid, oldGrid
!HPF$ DISTRIBUTE Grid (BLOCK,BLOCK)
!HPF$ DISTRIBUTE oldGrid (BLOCK,BLOCK)

      integer iteration_num

!!! Set the overrelaxation parameter, omega

      omega = 1.995d0

      write(*,*) '>>>>   The Mesh Resolution is .... '
      write(*,*) '>>>>      x-interval: ',1.0d0/dble(int_x-1)
      write(*,*) '>>>>      y-interval: ',1.0d0/dble(int_y-1)
      write(*,*) '>>>>      and the level of tolerance is  ',thresh

      call assign_boundary_values(Grid,int_x,int_y)
      error = 1.0d0
      iteration_num = 0
      do while (error .gt. thresh)
         call SOR_iteration(Grid, oldGrid, int_x, int_y, error, omega)
         iteration_num = iteration_num + 1
      enddo

      write(*,*) '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
      write(*,*) 'Number of Iterations:                 ',iteration_num
      write(*,*) 'The overRelaxation Parameter (omega): ',omega
      write(*,*) 'The relaxation Error:                 ',error
      write(*,*) 'The tolerance value:                  ',thresh
      write(*,*) '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'

      end
!===================================================================
! This routine implements the Gauss-Seidel overrelaxation scheme
! using the odd-even algorithm. Computing the SOR update afterward.
!===================================================================
      subroutine SOR_iteration(Grid,oldGrid,int_x,int_y,error,omega)
         double precision, dimension(:,:) :: Grid, oldGrid
!HPF$ INHERIT Grid
!HPF$ INHERIT oldGrid
         integer int_x, int_y
         double precision error, omega

!!! local vars ...
      integer k, m

      oldGrid = Grid

!!! Compute Gauss-Seidel update for even points ...
!!! A point is the average of its adjacent north-south and east-west
!!! points ...

      Forall(k=2:int_x-2:2, m=2:int_y-2:2) Grid(k,m) = 
     c   (Grid(k+1,m) + Grid(k-1,m) + Grid(k,m+1) + Grid(k,m-1))/4.0
      Forall(k=3:int_x-1:2, m=3:int_y-1:2) Grid(k,m) = 
     c   (Grid(k+1,m) + Grid(k-1,m) + Grid(k,m+1) + Grid(k,m-1))/4.0

!!! Compute Gauss-Seidel update for odd points ...
!!! A point is the average of its adjacent north-south and east-west
!!! points ...

      Forall(k=3:int_x-1:2, m=2:int_y-2:2) Grid(k,m) = 
     c   (Grid(k+1,m) + Grid(k-1,m) + Grid(k,m+1) + Grid(k,m-1))/4.0
      Forall(k=2:int_x-2:2, m=3:int_y-1:2) Grid(k,m) = 
     c   (Grid(k+1,m) + Grid(k-1,m) + Grid(k,m+1) + Grid(k,m-1))/4.0

!!! Compute the error as the difference between the G-S update and the
!!!  old Grid ...

      error = maxval(abs(Grid - oldGrid))

!!! Now compute the new overRelaxed values then put them back in Grid ...

      Grid = oldGrid + omega * (Grid - oldGrid)

      end subroutine

!=================================================================
! This routine calculates an analytic function, and returns it
! in Uval
!=================================================================
      subroutine compute_val_of_U(Uval, x, y)
      double precision Uval,x,y

!!! local var ...
      double precision part1

        part1 = -4.7 + 0.2 * x - 0.3 * y + 0.09 * (x * x - y * y)
        Uval = part1 + log(sqrt( ((x + 0.2)**2.0) + ((y + 0.2)**2.0) ))

!!!  write(*,*) Uval,x,y
      end subroutine

!===================================================================
! This sub puts the boundary values on the boundary of U
!===================================================================
      subroutine assign_boundary_values(U,int_x,int_y)
         interface
            subroutine compute_val_of_U( Uval,x,y)
                double precision Uval,x,y
            end subroutine
         end interface

      double precision, dimension(:,:) :: U
      integer int_x,int_y
      double precision, dimension(1:int_x) :: x_list
      double precision, dimension(1:int_y) :: y_list

!HPF$ INHERIT U

!!! Reset the grid ...

      U = 0.0d0                !! "d" here is for double precision ...

!!! Compute the x's and y's of the grid ...
      do i=1,int_x
         x_list(i) = dble(i-1)/dble(int_x-1)
      enddo
      do i=1,int_y
         y_list(i) = dble(i-1)/dble(int_y-1)
      enddo

!!! Compute those points forming the boundary region ...
!!! write(*,*) '==== Value of U ====', '==== X value ====', '=== Y value ==='

      do i=1,int_x
         call compute_val_of_U( U(i,1) ,x_list(i), 0.0d0)
         call compute_val_of_U( U(i,int_y), x_list(i), 1.0d0)
      enddo
      do i=1,int_y
         call compute_val_of_U( U(1,i), 0.0d0, y_list(i))
         call compute_val_of_U( U(int_x,i), 1.0d0, y_list(i))
      enddo
      end subroutine
\end{verbatim}

\subsection{The Program Output}

After a setup of the local environment (via SETENV) and then compiling
the above code, as follows:

\begin{verbatim}
saleh:merlin3[86] pghpf -o Laplace_SOR -O Laplace_SOR.hpf
** laplace_sor   === End of Compilation 1 ===
** sor_iteration   === End of Compilation 2 ===
** compute_val_of_u   === End of Compilation 3 ===
** assign_boundary_values   === End of Compilation 4 ===
1501-510  Compilation successful for file pghpfEVAaWdXAAB.f.
Linking:
\end{verbatim}

We run it varying the value of $\omega$ as mentioned above between $1$
and $2$ while comparing the resulting number of SOR iterations. To do
that we start with $\omega$ taking a value just above $1$ and then
slowly incrementing it till a value slightly less than $2$. Results
follow ({\em all the runs are done on $6$ nodes of the SP2}):
\begin{itemize}

\item For $\omega$ = $1.1$, we obtain:

\begin{verbatim}
saleh:merlin3[88] Laplace_SOR -pghpf -host merlin3,merlin4,
                   merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                        10165         <<<<<<<< (1)
 The overRelaxation Parameter (omega):    1.100000000000000     
 The relaxation Error:                   9.9954505117239023E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cpu        real      user       sys     ratio   node
   0*    341.87     88.40     17.41       31%   merlin3
   1     341.57     89.35     13.00       30%   merlin4
   2     341.48     86.53     17.38       30%   merlin5
   3     341.44     86.89     19.25       31%   merlin6
   4     341.37     90.58     12.59       30%   merlin7
   5     341.43     79.76     12.27       27%   merlin8
 min     341.37     79.76     12.27
 avg     341.53     86.92     15.32
 max     341.87     90.58     19.25
total    341.87    521.51     91.90     1.79x
\end{verbatim}

\item For $\omega$ = $1.3$,  we obtain:

\begin{verbatim}
saleh:merlin3[90] Laplace_SOR -pghpf -host merlin3,merlin4,
                   merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         8601         <<<<<<<< (2)
 The overRelaxation Parameter (omega):    1.300000000000000     
 The relaxation Error:                   9.9881376947053013E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    310.34     78.88     14.55       30%   merlin3
   1     306.61     72.33     18.13       30%   merlin4
   2     305.55     69.03     16.84       28%   merlin5
   3     303.47     75.43     16.45       30%   merlin6
   4     304.52     59.61     10.95       23%   merlin7
   5     302.48     78.30     11.90       30%   merlin8
 min     302.48     59.61     10.95
 avg     305.49     72.26     14.80
 max     310.34     78.88     18.13
total    310.34    433.58     88.82     1.68x
\end{verbatim}

\item For $\omega$ = $1.5$,  we obtain:

\begin{verbatim}
saleh:merlin3[92] Laplace_SOR -pghpf -host merlin3,merlin4,
                   merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         7453         <<<<<<<< (3)
 The overRelaxation Parameter (omega):    1.500000000000000     
 The relaxation Error:                   9.9969288847034932E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    275.70     67.16      9.91       28%   merlin3
   1     275.55     72.64     15.56       32%   merlin4
   2     275.47     70.72     11.33       30%   merlin5
   3     267.94     68.30     14.78       31%   merlin6
   4     267.89     54.78      8.90       24%   merlin7
   5     267.90     65.12      9.45       28%   merlin8
 min     267.89     54.78      8.90
 avg     271.74     66.45     11.65
 max     275.70     72.64     15.56
total    275.70    398.72     69.93     1.70x
\end{verbatim}

\item For $\omega$ = $1.7$,  we obtain:

\begin{verbatim}
saleh:merlin3[94] Laplace_SOR -pghpf -host merlin3,merlin4,
                   merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         6576          <<<<<<<< (4)
 The overRelaxation Parameter (omega):    1.700000000000000     
 The relaxation Error:                   9.9906185546672077E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    249.67     49.45     12.27       25%   merlin3
   1     249.41     57.22     13.30       28%   merlin4
   2     245.62     64.12     12.91       31%   merlin5
   3     244.59     68.07     12.36       33%   merlin6
   4     244.47     48.64      8.86       24%   merlin7
   5     243.47     54.65      9.77       26%   merlin8
 min     243.47     48.64      8.86
 avg     246.21     57.02     11.58
 max     249.67     68.07     13.30
total    249.67    342.15     69.47     1.65x
\end{verbatim}

\item For $\omega$ = $1.79$,  we obtain:

\begin{verbatim}
saleh:merlin3[96] Laplace_SOR -pghpf -host merlin3,merlin4,
                   merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         6245      <<<<<<<< (5)
 The overRelaxation Parameter (omega):    1.790000000000000     
 The relaxation Error:                   9.9929635233309000E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    226.57     56.89      9.75       29%   merlin3
   1     224.12     56.37     10.74       30%   merlin4
   2     226.07     58.99     11.05       31%   merlin5
   3     226.01     55.55     12.81       30%   merlin6
   4     225.99     44.90      9.20       24%   merlin7
   5     226.00     47.81      7.29       24%   merlin8
 min     224.12     44.90      7.29
 avg     225.79     53.42     10.14
 max     226.57     58.99     12.81
total    226.57    320.51     60.84     1.68x
\end{verbatim}

\item For $\omega$ = $1.88$,  we obtain:

\begin{verbatim}
saleh:merlin3[98] Laplace_SOR -pghpf -host merlin3,merlin4,
                   merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         5946        <<<<<<<< (6)
 The overRelaxation Parameter (omega):    1.880000000000000     
 The relaxation Error:                   9.9894696958813256E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    230.45     48.23     10.01       25%   merlin3
   1     226.15     59.88      9.68       31%   merlin4
   2     226.07     48.34     11.68       27%   merlin5
   3     225.06     56.31     14.88       32%   merlin6
   4     226.00     43.75      9.62       24%   merlin7
   5     226.01     38.10      6.91       20%   merlin8
 min     225.06     38.10      6.91
 avg     226.62     49.10     10.46
 max     230.45     59.88     14.88
total    230.45    294.61     62.78     1.55x
\end{verbatim}

\item For $\omega$ = $1.92$,  we obtain:

\begin{verbatim}
saleh:merlin3[100] Laplace_SOR -pghpf -host merlin3,merlin4,
                    merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         5822       <<<<<<<< (7)
 The overRelaxation Parameter (omega):    1.920000000000000     
 The relaxation Error:                   9.9900193895052780E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    244.91     55.96      8.31       26%   merlin3
   1     242.77     46.63     10.86       24%   merlin4
   2     244.68     58.40     10.97       28%   merlin5
   3     244.45     63.20     13.75       31%   merlin6
   4     244.40     50.37      7.12       24%   merlin7
   5     244.41     33.56      4.38       16%   merlin8
 min     242.77     33.56      4.38
 avg     244.27     51.35      9.23
 max     244.91     63.20     13.75
total    244.91    308.12     55.39     1.48x
\end{verbatim}

\item For $\omega$ = $1.97$,  we obtain:

\begin{verbatim}
saleh:merlin3[102]  Laplace_SOR -pghpf -host merlin3,merlin4,
                     merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         5674        <<<<<<<< (8)
 The overRelaxation Parameter (omega):    1.970000000000000     
 The relaxation Error:                   9.9923187057981977E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    211.86     52.75     10.08       30%   merlin3
   1     211.79     49.92      9.04       28%   merlin4
   2     207.74     54.55     10.16       31%   merlin5
   3     207.38     52.30     14.42       32%   merlin6
   4     207.35     45.09      7.66       25%   merlin7
   5     207.35     44.19      7.47       25%   merlin8
 min     207.35     44.19      7.47
 avg     208.91     49.80      9.80
 max     211.86     54.55     14.42
total    211.86    298.80     58.83     1.69x
\end{verbatim}

\item For $\omega$ = $1.99$,  we obtain:

\begin{verbatim}
saleh:merlin3[104] Laplace_SOR -pghpf -host merlin3,merlin4,
                    merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                         5617       <<<<<<<< (9)
 The overRelaxation Parameter (omega):    1.990000000000000     
 The relaxation Error:                   9.9908837647433302E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*    207.94     54.56      9.44       31%   merlin3
   1     203.87     54.28      9.05       31%   merlin4
   2     203.82     46.39      9.94       28%   merlin5
   3     203.77     53.08     11.59       32%   merlin6
   4     203.63     39.43      6.77       23%   merlin7
   5     203.65     47.64      6.74       27%   merlin8
 min     203.63     39.43      6.74
 avg     204.44     49.23      8.92
 max     207.94     54.56     11.59
total    207.94    295.38     53.53     1.68x
\end{verbatim}

\item For $\omega$ = $1.9995$,  we obtain:

\begin{verbatim}
saleh:merlin3[106] Laplace_SOR -pghpf -host merlin3,merlin4,
                    merlin5,merlin6,merlin7,merlin8 -np 6 -stat cpus

 >>>>   The Mesh Resolution is .... 
 >>>>      x-interval:   1.0101010101010102E-002
 >>>>      y-interval:   1.0101010101010102E-002
 >>>>      and the level of tolerance is    1.0000000E-07

 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of Iterations:                        35751          <<<<<<<< (10)
 The overRelaxation Parameter (omega):    1.999500000000000     
 The relaxation Error:                   9.9992979230023593E-008
 The tolerance value:                    1.0000000E-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cpu        real      user       sys     ratio   node
   0*   1324.88    309.73     62.71       28%   merlin3
   1    1324.76    334.55     60.74       30%   merlin4
   2    1324.63    331.16     69.01       30%   merlin5
   3    1324.59    322.10     76.08       30%   merlin6
   4    1324.45    270.67     46.20       24%   merlin7
   5    1324.45    358.69     52.22       31%   merlin8
 min    1324.45    270.67     46.20
 avg    1324.62    321.15     61.16
 max    1324.88    358.69     76.08
total   1324.88   1926.90    366.96     1.73x
\end{verbatim}
\end{itemize}

Well, {\bf from the above $10$ runs}, we certainly could see that the
error is the lowest when $\omega$ is closest to $2$, that is {\bf the
optimum value of $\omega$ is $1.9995$}.  The number of the SOR
iterations drops noticeably as $\omega$ increases then jumps up with
the last value giving and improved overall error.

\end{document}


