%% LATEX TWICE TO GET THE REFERENCES RIGHT
%%
%% \documentstyle[11pt,psfig]{article}        
%%
\documentstyle[12pt,epsf,psfig]{article}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% macro for postscript figures - the easy way.
%% See directory ftp.cis.ufl.edu/tech report by fishwick@cis.ufl.edu 
%% This will help to control the size of the figure.
%%
\newcommand{\postscript}[2]
 {\setlength{\epsfxsize}{#2\hsize}
  \centerline{\epsfbox{#1}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newtheorem{theorem}{Theorem}
\newtheorem{itlemma}{Lemma}[section] %number by section (set in \em by default)
%%%same numbering counter as itlemmas:
\newtheorem{itproposition}[itlemma]{Proposition}
\newtheorem{itcorollary}[itlemma]{Corollary}
\newtheorem{itremark}[itlemma]{Remark}
\newtheorem{itnote}[itlemma]{Note}
\newtheorem{itobservation}[itlemma]{Observation}
\newtheorem{itdefinition}[itlemma]{Definition}
\newtheorem{itexercise}[itlemma]{Exercise}
\newtheorem{itexample}[itlemma]{Example}

\newenvironment{lemma}{\begin{itlemma}\rm}{\end{itlemma}} %no-italics
\newenvironment{remark}{\begin{itremark}\rm}{\end{itremark}} %no-italics
\newenvironment{observation}{\begin{itobservation}\rm}{\end{itobservation}} %no-italics
\newenvironment{corollary}{\begin{itcorollary}\rm}{\end{itcorollary}}
\newenvironment{proposition}{\begin{itproposition}\rm}{\end{itproposition}}
\newenvironment{definition}{\begin{itdefinition}\rm}{\end{itdefinition}}
\newenvironment{exercise}{\begin{itexercise}\rm}{\end{itexercise}}
\newenvironment{example}{\begin{itexample}\rm}{\end{itexample}}

\tolerance=1600 %allow some tolerance in extending after line.
\parskip=3pt plus 1pt minus 1pt
\overfullrule=0pt %no dark lines if overfull

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% This are for choices.  They take 4 or 6 args
%% (carriage returns allowed in between}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand{\twoif}[4]{
\left\{ \begin{array}{ll}#1&#2\\#3&#4\end{array}\right.
}
\newcommand{\threeif}[6]{
\left\{ \begin{array}{ll}#1&#2\\#3&#4\\#5&#6\end{array}\right.
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\def\partmlab#1#2#3#4#5#6#7#8{\bordermatrix{&#1&#2\cr#3&#5&#6\cr
					\noalign{\medskip}#4&#7&#8\cr}}
%% For 2by2 partitioned matrices; arguments are: two labels of top, side,
%% 4 entries (row by row).  If entries are themselves matrix or pmatrix,
%% better for readability of source to define first: \def\matrix1{...} etc.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\bigframebox}[1]{\framebox{\parbox{\smallerwidth}{#1}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Warning: use \noindent before!
% AND IN TEXT, after begin doc, do:
% \newlength{\smallerwidth}  %for bigboxes
% \setlength{\smallerwidth}{\textwidth}
% \addtolength{\smallerwidth}{-8pt}        %MUST BE DONE AFTER begin doc
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\def \beq {\begin{eqnarray}}
\def \eeq {\end{eqnarray}}
\def \beqn {\begin{eqnarray*}}
\def \eeqn {\end{eqnarray*}}

\newcommand{\rf}[1]{[\ref{#1}]}            %bibliographical references

\newcommand{\text}[1]{\hbox{\rm \ #1\ \/}}
\newcommand{\be}[1]{\begin{equation}\label{#1}}
\newcommand{\ee}{\end{equation}}
\newcommand{\bl}[1]{\begin{lemma}\label{#1}}
\newcommand{\ble}[1]{\begin{lemmaex}\label{#1}}
\newcommand{\br}[1]{\begin{remark}\label{#1}}
\newcommand{\bn}[1]{\begin{note}\label{#1}}
\newcommand{\bo}[1]{\begin{observation}\label{#1}}
\newcommand{\bt}[1]{\begin{theorem}\label{#1}}
\newcommand{\bd}[1]{\begin{definition}\label{#1}}
\newcommand{\bp}[1]{\begin{proposition}\label{#1}}
\newcommand{\bc}[1]{\begin{corollary}\label{#1}}
\newcommand{\bfact}[1]{\begin{fact}\label{#1}}
\newcommand{\ber}[1]{\begin{exercise}\label{#1}}
\newcommand{\bex}[1]{\begin{example}\label{#1}}
\newcommand{\bem}[1]{\begin{example}\label{#1}}    %Yes, 2 different ones...
\newcommand{\ec}{\mybox\end{corollary}}
\newcommand{\efact}{\mybox\end{fact}}
\newcommand{\eer}{\mybox\end{exercise}}
\newcommand{\eex}{\mybox\end{example}}
\newcommand{\eem}{\mybox\end{example}}
\newcommand{\el}{\mybox\end{lemma}}
\newcommand{\ele}{\mybox\end{lemmaex}}
\newcommand{\er}{\mybox\end{remark}}
\newcommand{\en}{\mybox\end{note}}
\newcommand{\eo}{\mybox\end{observation}}
\newcommand{\et}{\qed\end{theorem}}
\newcommand{\ed}{\mybox\end{definition}}
\newcommand{\ep}{\mybox\end{proposition}}
\newcommand{\epr}{\end{proof}}
\newcommand{\bpr}{\begin{proof}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%  These are just for statements, no "qed" box is put in.
%%  (Also for when I have to finish something after a displayed equation
%%        or itemized.)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand{\ecs}{\end{corollary}}
\newcommand{\eers}{\end{exercise}}
\newcommand{\eexs}{\end{example}}
\newcommand{\eems}{\end{example}}
\newcommand{\els}{\end{lemma}}
\newcommand{\eles}{\end{lemmaex}}
\newcommand{\ers}{\end{remark}}
\newcommand{\ens}{\end{note}}
\newcommand{\eos}{\end{observation}}
\newcommand{\ets}{\end{theorem}}
\newcommand{\eds}{\end{definition}}
\newcommand{\eps}{\end{proposition}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\qed}{\hfill \halmos} %put \qed at right margin
\newcommand{\mybox}{\hfill $\Box$} %put \qed at right margin (white square)

%%%\newcommand{\pmat}[2]{\left( \begin{array}{#1} #2 \end{array} \right) }
%%%use as e.g. \pmat{cc}{ a&b\\ a &c }  Note 'cc' for 2 Centered cols
%%%can also use \pmatrix{ a&b\cr a &c \cr} for plainTeX standard construct
%%%\setcounter{page}{27} should set the CURRENT page to 27

\def \R {\bb R} % reals
\def \C {\bb C} % complexes
\def \Z {\bb Z} % integers
\def \X {\bb X} % for this paper only!
\def \inter {{\rm int \,}}
\def \rank {{\rm rank \,}}

%%%\def \inf {{\rm inf \,}}  %better in Latex -- subscripts OK!
%%%\def \min {{\rm min \,}}

\def \im {{\rm im \,}}
\def \ker {{\rm ker \,}}
\def \norm #1{\Vert #1 \Vert}
\def \ip #1\by #2{\langle #1 , #2 \rangle}
\def \Sone {{\bf S}^1} % circle
\def \wigglearrow {\sim \mkern -6mu >}

\def \today {\ifcase \month \or January\or February\or  
  March\or April\or May\or June\or July\or August\or 
  September\or October\or November\or December\fi\   
  \number \day, \number \year}                       
\newcommand{\comment}[1]{}
\newcommand{\halmos}{\rule{1ex}{1.4ex}}
%%%alternative to \qed (see below)
\newenvironment{proof}{\noindent {\em Proof}.\ }{\hspace*{\fill}$\halmos$\medskip}

%%%\def\qed{\kern 1em \vrule height .5em width .5em depth 0em}
%%%(I now use new qed: in right margin)

\def\vbar{\mathchoice{\vrule height6.3ptdepth-.5ptwidth.8pt\kern-.8pt}
   {\vrule height6.3ptdepth-.5ptwidth.8pt\kern-.8pt}
   {\vrule height4.1ptdepth-.35ptwidth.6pt\kern-.6pt}
   {\vrule height3.1ptdepth-.25ptwidth.5pt\kern-.5pt}}
\def\fudge{\mathchoice{}{}{\mkern.5mu}{\mkern.8mu}}
\def\bbc#1#2{{\rm \mkern#2mu\vbar\mkern-#2mu#1}}
\def\bbb#1{{\rm I\mkern-3.5mu #1}}
\def\bba#1#2{{\rm #1\mkern-#2mu\fudge #1}}
\def\bb#1{{\count4=`#1 \advance\count4by-64 \ifcase\count4\or\bba A{11.5}\or
   \bbb B\or\bbc C{5}\or\bbb D\or\bbb E\or\bbb F \or\bbc G{5}\or\bbb H\or
   \bbb I\or\bbc J{3}\or\bbb K\or\bbb L \or\bbb M\or\bbb N\or\bbc O{5} \or
   \bbb P\or\bbc Q{5}\or\bbb R\or\bbc S{4.2}\or\bba T{10.5}\or\bbc U{5}\or

%%%   \bbb P\or\bbc Q{5}\or\bbb R\or\bba S{8}\or\bba T{10.5}\or\bbc
%%%   U{5}\or

   \bba V{12}\or\bba W{16.5}\or\bba X{11}\or\bba Y{11.7}\or\bba Z{7.5}\fi}}
%
%
\renewcommand{\thefootnote}{\fnsymbol{footnote}}

%%% read as: theFOOTNOTE (i.e. the command  executed when doing a footnote) is
%%% now to be set as a FOOTNOTESYMBOL (type of number) value of counter
%%% FOOTNOTE (Instead of roman).  See LaTeX manual pp.91-92 as well as
%%% 175.

\def\bi{\begin{itemize}}
\def\ei{\end{itemize}}
\def\ben{\begin{enumerate}}
\def\een{\end{enumerate}}
\def\st{\, | \,}

\newcommand{\myboxineq}{\eqno\Box} %put \qed at margin (white square) in eqn
\newcommand{\qedineq}{\eqno\halmos} %put \qed at margin in eqn

\topmargin -0.5in %default is 0.0, i.e. 1 inch from top.  Now it is 0.5
\textheight 9in
\oddsidemargin 0.25in %more reasonable margins!  (This is 1.25; can use <0 too)
\evensidemargin 0.25in %(same as above)
\textwidth 6.25in %(could use also -.5,-.5,7.5 for instance)
\tolerance=1600 %allow some tolerance in extending after line.
\parskip=3pt plus 1pt minus 1pt
\overfullrule=0pt %no dark lines if overfull

%%%%%%%%%%%%%%% FROM HERE, macros special for paper %%%%%%%%%%%%%%%
\def\mm{m}
\def\pp{p}
\def\proj{\pi _1}     %%%% projection on first group of coordinates
\def\x{u}  % vector argument
\def\SEC{(SEC)}
\def\SECo{(SEC$^0$)}
\def\INV{(INV)}
\def\Fp{{\cal F}_\pp}
\def\Fpm{{\cal F}_\pp^\mm}  %%%% there is a 3,3 and a 2,1 in text

\def\Rm{\R^m}
\def\Rn{\R^n}
\newcommand{\ball}[2]{{\bb B}(#1,#2)}
\newcommand{\cball}[2]{\overline {\bb B} (#1,#2)}
\def\Fnm{{\cal F}_n^\mm}
\def\clos{{\rm clos\,}}
\def\inter{{\rm int\,}}

\def\CCF{compatible class of functions}
\begin{document}

\pagestyle{empty}
\begin{flushright}
{\sl Report: CIS-690/CIS-651 (1988-89)}
\end{flushright}
\vfill
\begin{center} 
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\hrule
\vspace{.05cm}
\hrule
\vspace{.25cm}
{\bf 
Some Experimental Results on the Performance of Feedforward Neural Networks
}
\vspace{.25cm}
\hrule
\vspace{.05cm}
\hrule
\hrule
\hrule
\hrule
\hrule
\vspace{1cm}
Saleh Elmohammed\\
School of Computer and Information Science,\\
Syracuse University,\\
Syracuse, NY 13244 , USA .\\
e-mails: saleh@top.cis.syr.edu ,\\
         saleh@npac.syr.edu .\\   
\end{center}

\vspace{1.5cm}
\centerline{ABSTRACT}

We show some experimental results demonstrating the evaluation time 
of two piecewise rational and the $\tanh()$ activation functions on
the two spiral problem. These activation functions are used in a
multi-layer feedforward neural network.

\vfill
\noindent{\bf Key words : neural net, epoch, tanh function, squashing
function, sigmoid function, piecewise rational function, spiral.}

\vfill
\centerline{\today}
\vfill

\newpage
\pagestyle{empty}
\tableofcontents

\newpage

\pagestyle{plain}

\section{Introduction}
\setcounter{page}{1}

In \cite{El-94}, we reported on the issues of network reducibility and
equivalence when using two piecewise rational functions and the widely
used activation function $tanh$. Also, to measure the evaluation time
of each of the aforementioned activation functions, some experimental
results were stated. This report is an extension to those
experiments on the same test problem. Analysis of network performance
and various timing figures of training and testing sessions are reported.

\section{Reasons for using piecewise rational functions}
The main reasons behind our use of feedforward networks with piecewise
rational activation functions (${\cal PWR}$) are:

\begin{enumerate}
\item 
When properly set over intervals such as $(-1,1)$, ${\cal PWR}$
can approximate the function $tanh()$ over the same interval. 
\item 
Evaluation of ${\cal PWR}$ may be much faster than evaluation of
$tanh()$.
\item 
If training and convergence are not significantly worse for
nets using ${\cal PWR}$ then their overall training will be faster. 
\item 
For sequential implementations, the time to evaluate $\sigma$ (the
activation function) and its first derivative, $\sigma^{'}$, may not
be very important in large networks; but parallel implementations can
be more sensitive to the effort required to evaluate $\sigma$ and
$\sigma^{'}$. Thus, we expect a faster parallel implementation when
${\cal PWR}$ functions are used as activation functions.
\end{enumerate}

\section{The Activation Functions}

As reported in \cite{El-94}, we have looked at two piecewise rational
functions and the hyperbolic tangent function. Briefly, we would like
to state what these functions are.

\subsection{Piecewise Rational Activation Functions}

\subsubsection{Definitions and Observations}

\bd{pwr-function}
For an integer $k > 0$, and real value $x_0 =  {k \over {k+1}}$, we define:
\be{pwr-form-1}
pwr_k(x) = 
 1.0 - {(1.0-x_0)x_0^k \over  {x^k}}\;,\;\;\;{\text{for} x \neq 0}\;.
\ee

The first piecewise rational activation function (${\cal PWR}$) used is:

\be{pwr-form-2}
\sigma_k(x) =
\threeif {-pwr_k(-x)} {\text{if} x  < -x_0}
         {pwr_k(x)} {\text{if} x > x_0}
         {x} {\text{otherwise.}}
\ee
\ed

\bo{pwr1.deriv}
The first derivative of $\sigma_k(x)$ is:
\be{pwr1-deriv-first}
\sigma^{'}_k(x) =
\twoif {pwr^{'}_k(|x|)} {\text{if} x < -x_0 \text{or} x_0 < x}
       {1} {\text{otherwise.}}
\ee
\eo

The second ${\cal PWR}$ function, due to Elliott (\cite{elliott-93.1}),
is defined next. 

\bd{pwr-function-type2}
\be{pwr.type2.1}
\sigma_p(x) = {x \over {1+|x|}}\; .
\ee
\ed

\bo{pwr2.deriv}
The activation function $\sigma_p(x)$ is differentiable everywhere to:
\be{observe-pwr2-deriv.1}
\sigma^{'}_p(x) = {1 \over {(1+|x|)^2}} \;\;,
\ee
and satisfies a ``general'' logistic differential equation as follows:
\be{pwr.type2.2}
\sigma^{'}_p = {(1.0 - |\sigma_p|)}^2\; .
\ee
\eo

\subsection{Tanh() and other Related Activation Function}

\subsubsection{Definitions and Observations}

\bd{on-sigmoid}
The function $\sigma$ is any function satisfying the property that
both  $\lim_{u\rightarrow -\infty}\sigma(u)$ and $\lim_{u\rightarrow
+\infty}\sigma(u)$ exist and are distinct. In this context, $\sigma$
is often called a squashing function \cite{Hertz-91}. 
\ed

Typically, $\sigma$ is one of the following:

\begin{enumerate}
\item 
For a $(-1,+1)$ output range, the activation function is:
\be{hyperbolic-tan}
\sigma_h(x) = \tanh(x)\;\;.
\ee
\bo{diff-of-tanh}
$Tanh()$'s first derivative satisfies:
\be{tanh-derv-1}
\sigma_h'(x) =  (1.0 - (\sigma_h(x))^2)\;\;.
\ee
\eo

\item 
For the output range of $(0,1)$, the activation function is:
\be{sigmoid}
\sigma_s(x) = {1 \over {1+ e^{-x}}}\;\;.
\ee
\bo{diff-of-sig}
The first derivative of $\sigma_s(x)$ satisfies:
\be{sig-der-1}
\sigma_s'(x) = \sigma_s(x) (1.0 - \sigma_s(x))\;\;.
\ee
\eo

\end{enumerate}

\bo{hyper-sigmoid-ob}
$\sigma_s(x)$ is equivalent to $\sigma_h(x)$, up to
translations and change of coordinates. The relation between these two
functions is:
\be{relation}
\sigma_h(x) = 2 \sigma_s(x) - 1.0\;.
\ee
\eo

A number of useful observations on these functions are reported in
\cite{El-94}.

\section{Experimental Results}

Using the Aspirin/MIGRAINES \cite{Leighton-92} software system, a
neural net example is used to show the evaluation time of $\sigma_k$
and $\sigma_p$ versus that of $\sigma_h$. The focus of this
experimental work is on the first three reasons stated above; in
particular, the second and third points.

First, given the feedforward network, we substituted the activation
function $\sigma_k$ for $\sigma_h$ (the net's default activation
function), then ran it on the problem and recorded the results.
Second, $\sigma_p$ is used in place of $\sigma_k$ and the results were
also recorded. These two sets of results are compared with those
obtained from running the network on the same problem using
$\sigma_h$. 

Next, we look at the test problem and describe our simulations on it.

\subsection{The Spiral Problem}

\subsubsection{Summary}

Briefly, this is an example of a very hard pattern recognition
problem. The data are the $x$ and $y$ coordinates of two spirals. In
the actual data file the coordinates for each spiral have an
additional number associated with them which denotes to which spiral
the coordinate belongs to (i.e., 0.5 for one spiral and -0.5 for the
other). The two spirals coil three times around the origin and around
one another. The goal of this example is to train the network to map
an $(x,y)$ coordinate into the proper spiral. The network is trained
by giving it $x$ and $y$ and the target classification for some of the
points along these intertwined spirals. This appears to be a very
difficult task for back-propagation networks and their relatives.
Problems like this one, whose inputs are points on the 2-D plane, are
interesting because we can display the 2-D ``receptive field'' of any
node in the network. For a detailed description of this problem, see
Lang and Withbrock \cite{LW-88}.  


\subsubsection{Problem Description}

Two sets of points are generated, each with 97 members (three complete
revolutions at 32 points per revolution, plus endpoints). Each point
is represented by two floating-point numbers, the $x$ and $y$
coordinates, plus an output value of $-0.5$ for one set, $+0.5$ for
the other set. (Some algorithms may require different output values,
such as $1.0$ and $0.0$ or $+0.9$ and $-0.9$ or $0.8$ and $0.2$). 

Here is a fragment of $C$ code to generate these two sets of points:

\begin{verbatim}
    main()
    {
      int i;
      double x, y, angle, radius;

      /* write spiral of data */

      for (i=0; i<=96; i++) {
        angle = i * M_PI / 16.0;
        radius = 6.5 * (104 - i) / 104.0;
        x = radius * sin(angle);
        y = radius * cos(angle);
        printf("((%8.5f  %8.5f)   (%3.1f))\n",  x,  y,  0.5);
        printf("((%8.5f  %8.5f)   (%3.1f))\n", -x, -y, -0.5);
      }
    }
\end{verbatim}


\subsubsection{Methodology}

The task is to train on the 194 (97 x 2) I/O pairs until the
learning system can produce the correct output for all of the inputs.
The time required for learning is then recorded.  For back-propagation
systems, the time is typically recorded in ``epochs''.  
Each epoch includes one training presentation for each of the 194
points in the training set.

The choice of output values for the two sets is up to the
experimenter. For uniformity in reporting results, we suggest that the
40-20-40 criterion be used: an output is considered to be a logical
-1 if it is in the lower $40\%$ of the output range, a logical +1 if
it is in the upper $40\%$, and indeterminate (and therefore incorrect)
if it is in the middle $20\%$ of the range. 

This task can obviously be solved quickly by methods that
essentially do table-lookup, recording the desired result for each
point; on the other hand, the problem appears hard for
back-propagation networks and is impossible for a linear separator.
The interesting question is how learning time varies with the
algorithm chosen, the total number of weights (or equivalent) in
the network, and the type of activation function(s) used. In this
report we will be looking at the last two criteria, specifically, most
of the analysis will be done for the last one.

\subsubsection{Variations}

It is possible to vary the density of points along the spiral
by incrementing the angle by a smaller amount, while still making
three complete revolutions in each set.  Lang and Witbrock
\cite{LW-88} report some results using another version of the
problem, with 770 points in all. 


\subsubsection{Results}

\cite{LW-88} report results obtained on this
problem using $2\rightarrow 5\rightarrow 5\rightarrow 5\rightarrow 1$ 
(a two-node input layer, five-node first hidden layer, five-node
second hidden layer, five-node third hidden layer, and one-node output
layer) network with short-cut connections (each unit is connected to
every unit in all earlier layers, not just to units in the previous
layer). Also, each of the hidden and output units carries a sigmoid
activation function ($\sigma_s$). Counting the unit thresholds, there
are a total of 138 trainable weights in the network.  

Three trials were run using standard back-propagation on this
network, with a uniform distribution of random initial weights in the
range -0.1 to +0.1. Learning rate was initially 0.001 and momentum
was initially 0.5; these values were "gradually" increased over 1000
epochs to final values of 0.002 and 0.95.  The learning times on the
three runs were 18,900 epochs, 22,300 epochs, and 19,000 epochs.
(Average = 20,000 epochs.) Using the same network and starting values,
but with a nonlinear "cross entropy" error function, learning times
were 16,200 epochs, 8600 epochs, and 7600 epochs, respectively.
(Average = 11,000 epochs.) 

In our experiment, we use a $2\rightarrow 5\rightarrow 10\rightarrow
15\rightarrow 1$ network with also short-cut connections. The total
number of weights is 398. Also, three trials were run on this network
using standard back-propagation, with a uniform distribution of random
initial weights in the range -0.1 to +0.1. Each of the activation
functions ($\sigma_h$, $\sigma_k$ for $k = 1$, and $\sigma_p$) is used
for three runs , totalling nine trials altogether.


\subsubsection{Summaries and Analysis}

Each trial involves two operations on the network, training then
testing. Using the C code fragment shown above, two test data sets
(one for each spiral and of size 97) will be created:

[1] The first set:
\begin{verbatim}
    X           Y        ID
 -------     --------  ------
-0.00000     -6.50000   -.5
-1.25589     -6.31381   -.5
-2.43961     -5.88973   -.5
-3.50704     -5.24865   -.5
-4.41942     -4.41942   -.5
-5.14472     -3.43759   -.5
-5.65876     -2.34394   -.5
-5.94601     -1.18274   -.5
-6.00000     -0.00000   -.5
-5.82341      1.15835   -.5
-5.42779      2.24827   -.5
-4.83292      3.22925   -.5
-4.06586      4.06586   -.5
-3.15981      4.72898   -.5
-2.15259      5.19682   -.5
-1.08519      5.45562   -.5
-0.00000      5.50000   -.5
1.06080       5.33302   -.5
2.05692       4.96585   -.5
2.95147       4.41718   -.5
        ...
 etc.
\end{verbatim}

[2] The second set:
\begin{verbatim}
    X         Y        ID
 -------   -------   ------
0.00000     6.50000   0.5 
1.25589     6.31381   0.5
2.43961     5.88973   0.5
3.50704     5.24865   0.5
4.41942     4.41942   0.5
5.14472     3.43759   0.5
5.65876     2.34394   0.5
5.94601     1.18274   0.5
6.00000     0.00000   0.5
5.82341     -1.15835  0.5
5.42779     -2.24827  0.5
4.83292     -3.22925  0.5
4.06586     -4.06586  0.5
3.15981     -4.72898  0.5
2.15259     -5.19682  0.5
1.08519     -5.45562  0.5
0.00000     -5.50000  0.5
-1.06080    -5.33302  0.5
-2.05692    -4.96585  0.5
-2.95147    -4.41718  0.5
        ...
 etc.
\end{verbatim}

The {\it training} data set is of the form:

\begin{verbatim}
   X           Y      ID
-------    --------  ------
 0.00000   6.50000   0.5 
-0.00000  -6.50000   -0.5 
 1.25589   6.31381   0.5 
-1.25589  -6.31381   -0.5 
 2.43961   5.88973   0.5 
-2.43961  -5.88973   -0.5 
 3.50704   5.24865   0.5 
-3.50704  -5.24865   -0.5 
 4.41942   4.41942   0.5 
-4.41942  -4.41942   -0.5 
 5.14472   3.43759   0.5 
-5.14472  -3.43759   -0.5 
 5.65876   2.34394   0.5 
-5.65876  -2.34394   -0.5 
 5.94601   1.18274   0.5 
-5.94601  -1.18274   -0.5 
 6.00000   0.00000   0.5 
-6.00000  -0.00000   -0.5 
 5.82341  -1.15835   0.5 
-5.82341   1.15835   -0.5 
        ...
 etc.
\end{verbatim}       

\subsubsection{The Training Step}
\begin{itemize}
\item 
Training of the feedforward net is done in epochs over the 194 I/O
pairs in the training set using the back-propagation algorithm.
Each single epoch includes one training presentation for each pair till
it is identified to belong to either spiral.
The $40-20-40$ criterion, mentioned above, is used for such an
identification. 

\item 
The training algorithm attempts to minimize the mean
squared error on the output by adjusting the feedforward weights
(including all the short-cuts). The output error (over all output
nodes) measure is defined as:

\bd{out-error-measure}
\be{error}
E = {1 \over 2}\sum_i(d_i - o_i)^2\;\;,
\ee
\ed

where $o_i$ is the value of the $i^{th}$ output node and $d_i$ is the
desired value of the $i^{th}$ output node. The convergence test uses
the maximum absolute difference as the error calculation. If the
absolute value of the difference between each output node and its
associated target during all of the specified iterations is less than
some number (the error bound) then the network is said to have
converged. {\it In all the experiments reported here, this error (also
called BPthreshold) is set to $0.000050$}. So, no back-propagation or
back-iteration will be done if {\it Error $<$ BPthreshold}.
After convergence (or termination without convergence), the 
total number of epochs are computed by dividing the total iterations
by the size of the training set. An iteration consists of one forward
and one backward pass.

\item 
The learning rate and the inertia required by the learning
algorithm are defined in a user-specified C routine, along with the
node(s) activation function(s). See the {\it Appendix A} for a listing of
this file. The error bound, number of iterations desired, when to
pause to check for convergence, number of training pairs (or size of
the training set), and the stopping point (or after how many
iterations to terminate training) are all specified in a Learning and
Testing Shell.

\item
{\it Appendices B, C, and D} are scripts of sample runs of the
Learning and Testing Shell (outlined next) for a $2\rightarrow
5\rightarrow 10\rightarrow 15\rightarrow 1$ network (with short-cuts)
using $\sigma_h$ , $\sigma_k$ (for $k=1$), and $\sigma_p$ ,
respectively.
\end{itemize}

{\bf \underline{Example}}:
\begin{verbatim}

###########  Learning and Testing  Shell ########### 
#!/bin/sh

  /bin/rm -f Network.save Network.Finished

#### learning.............................
## line (1):

     time spiral -l -t 50000 194 0.4 -d spiral.df -s 100000 -I 2000000

  if test ! -s Network.Finished
  then
     cp Network.save Network.Finished
  fi

#### testing..............................
## line (2):

     spiral Network.Finished -d spiral.df -E -P 0.49

## line (3):
    
     Output < spiral1.data > spiral1.pdat
     Output < spiral2.data > spiral2.pdat

#### plot the results.........................

  gnuplot spiral.gnu

\end{verbatim}

As a result of the specification in line (1) of the above shell, the
simulation periodically pauses and checks to see if the net has
converged. How often it checks is controlled by the {\bf -t} option.
The net is periodically saved to the file {\bf Network.save}. The {\bf
-s} option controls the pausing and then saving period. For example,
the argument immediately after {\bf -s} in the above shell will causes
the network to be dumped after every 100000 iterations. The {\bf -t}
option has three arguments. The first argument is the number of iterations
between pauses to test. If this number is set too low, the simulation
will spend the most of the time testing, not learning. If it is set
very high, the user has to wait longer before the simulation stops.
The second argument is the number of iterations the simulation must
pass the convergence test before being allowed to stop. Usually this
number equals the number of training patterns in the data file (here,
it is 194). The third argument is a real number denoting the error
bound or tolerance (here, it is 0.4). {\bf Note: this tolerance is not the
same as the error bound used in the error function of the
back-propagation algorithm}. If the convergence test is passed for all
of these iterations then the network is said to have converged. 
The {\bf -I} option tells the simulation process to exit after doing ,
for example, 1000000 iterations. If there was convergence at the last
pause then another 100000 iterations are needed to exit, without
testing again, of course. The {\bf -l} option is  for learning rather
than testing.


\subsubsection{The Testing Step}

\begin{itemize}

\item 
Testing of the net is done in one epoch over the two sets of test
data, each of size 97. As shown above, each set is for a different
spiral. To do the testing, the data patterns are presented to the net
sequentially , one pair at a time. 

\item
As part of the testing, the output of the net is a listing of the
maximum, minimum, mean and variance of all outputs for all test pairs
of the two data files. Also the probability of detection (Pd), which is
those test pairs that are correctly detected over the total number. 
The network outputs as well the probability of false alarm (Pfa),
which is computed as the difference between the total number and those
correctly detected, let say, the outcome is ${\alpha}$. Pfa is
${\alpha}$ divided by the total numbe of test patterns.

\item
From the above {\it learning and testing shell}, a number of options can be
used in line (2) to carry out testing. The {\bf -d} option indicates
that a test file need to be read. The {\bf -E} option is for (Epoch),
telling the net to go forward through all the data patterns in all the
test files one epoch. The {\bf -P} option calculate an estimate of the
probability of detection (Pd) and probability of false alarm (Pfa) for
option {\bf -f} $<iterations>$ using $<threshold>$ (here it is 0.49) 
for detection threshold or tolerance. 
The program counts the number of times the
Euclidean distance between the output vector and the target vector is
less than some {\it BPthreshold}. Perhaps, the proper way to use the
{\bf -P} option is to run all of the classes of data separately.
\end{itemize}


\subsubsection{Performance and Timing Figures}

 --- See attached notes (tables printed in text) ---


\section{Related Results}                            

      Stephen A. Frostrom \cite{SF-92} of SAIC reports the following
unpublished result, obtained by Dennis Walker \cite{DW-92} also of
SAIC:

Standard back-propagation network, using $tanh()$ activation function,
$2\rightarrow 20\rightarrow 10\rightarrow 1$ with no short-cut
connections (281 weights in all).
Learning rate 0.1, momentum 0.7, unit activation running from -0.5 to
0.5, error was set to 0.0 if the output was within 0.15 of target. 
The task was learned (to within 0.3 tolerance) in 13,900 epochs.  With
a tighter $2\rightarrow 16\rightarrow 8\rightarrow 1$ network, the
task was learned, but it required 300,000 epochs. 

      Fahlman and Lebiere \cite{Fahlman-90} report that the Cascade-Correlation
algorithm is able to solve this problem in 1700 epochs (average over
100 trials, all successful).  This algorithm builds up a network (with
tanh() or its related sigmoid activation function) as it learns.  In
these 100 trials, the networks constructed ranged from 12 to 19 hidden
units, with an average of 15.2 and a median of 15.  Thus, the networks
are roughly the same size as the 15-hidden-unit network investigated
by Lang and Witbrock.

      The figure of 1700 epochs for Cascade-Correlation is somewhat
misleading, since the Cascade-Correlation epochs require less
computation than backpropagation or quickprop \cite{Fahlman-88} epochs.
Scott Fahlman proposes that instead of counting epochs of very
different kinds, we measure these algorithms in terms of connection
crossings: the number of times a value or error is multiplied by a
weight and added to an accumulated value.  By this measure,
Cascade-Correlation requires about 19 million crossings, while the
8000 quickprop epochs of Lang and Witbrock require about 438 million
crossings --- larger by a factor of 23.

      Lang and Witbrock \cite{LW-88} report that the variation with $4X$
density was learned in 64,000 epochs using the same standard
back-propagation setup that they used for the single density case.
Fahlman \cite{Fahlman-92} reports the following results on eight trials (all
successful) of the $4X$ density problem using the Cascade-Correlation
algorithm:  

      The average number of epochs required was 2262. The nets constructed
      ranged from 13 to 18 hidden units (for all hidden layers), with
      an average of 15.5 and a median of 15 units. The training
      parameters used were the same as for the $1X$ version (the
      version we are handling here) of this same problem.

\section{Conclusion}
From the performance and timing figures we state the following:
\begin{enumerate}
\item 
The size of the network has no effect on the evaluation of the
activation function(s).

\item
The overall evaluation of $\sigma_k$ is faster than either the overall
evaluation of $\sigma_h$ or $\sigma_p$ .

\item
Varying the size of the network per an activation function does
affect the training time of the network on the given task. Of course,
this will also have a direct affect on the number of iterations
necessary before convergence. 
\end{enumerate}

I am very grateful to Professor Ernie Sibert, for his guidance while
writing this paper.


\clearpage
%\newpage
\begin{thebibliography}{MASE}

\bibitem[PDP-86]{Rumelhart-86}
Rumelhart, D , J McClelland, and the PDP Research Group.
Parallel Distributed Processing: Explorations in the Microstructure of
Cognition. Vol I, 1986. MIT Press, Cambridge.

\bibitem[A/M.v6-92]{Leighton-92}
Leighton, Russell,
The Aspirin/MIGRAINES Neural Network Software system. Release V6.0,
MP-91W00050, the MITRE Corp. October 1992.

\bibitem[DLE-93.1]{elliott-93.1}
Elliot, David L.
``A Better Activation Function for Artificial Neural Networks'', 
TR 93-8, Institute for Systems Research, University of
Maryland, College Park, MD 20742, January 1993.

\bibitem[HKP-91]{Hertz-91}
Hertz, John, Andres Krogh, and Richard Palmer,
{\it Intro. to the theory of neural computation}, Addison-Wesley,
Redwood City, 1991. 

\bibitem[Min-88]{Minsky}
Minsky, M and S. Papert, {\it Perceptron} , 1988 edition.

\bibitem[Lang-Withbrock.88]{LW-88}
Kevin J. Lang and Michael J, Witbrock, 
``Learning to Tell Two Spirals Apart'', in Proceedings of the 1988
Connectionist Models Summer School, Morgan Kaufmann, 1988.

\bibitem[Fahlman-88]{Fahlman-88} 
Scott E. Fahlman, ``Faster-Learning Variations on Back-Propagation: An
Empirical Study'', in Proceedings of the 1988 Connectionist Models
Summer School, Morgan Kaufmann, 1988.

\bibitem[Fahlman-Lebiere.90]{Fahlman-90} 
Scott E. Fahlman and Christian Lebiere, ``The Cascade-Correlation
Learning Architecture'', in Touretzky (ed.) Advances in Neural
Information Processing Systems 2, Morgan-Kaufmann, 1990.  A slightly
more detailed version of the paper is available as CMU tech report
CMU-CS-90-100.

\bibitem[Saleh-94]{El-94} 
Saleh Elmohamed, ``Some Issues of Reducibility and Equivalence in
Feedforward Neural Networks'', tech. paper to CIS, Syracuse University,
Feb 1994.

\bibitem[Fahlman-92]{Fahlman-92} 
Scott E. Fahlman, private communication to Russ Leighton, 1992.

\bibitem[SF-92]{SF-92} 
Stephen Frostrom, private communication to Russ Leighton, 1992.

\bibitem[SF-92]{DW-92} 
Dennis Walker, private communication to Russ Leighton, 1992.

\bibitem[DLE-93.2]{elliott-93.2}
David Elliot, private communication, 1993.


\end{thebibliography}



\newpage
%%% \renewcommand{\thesection}{Appendix A}
%%% \renewcommand{\theequation}{\mbox{A}\arabic{equation}}
\pagestyle{myheadings}
\markright{Program Listing}
%%% \setcounter{equation}{0}

\section{Appendix}
\appendix

\begin{verbatim}

/*-----------------------------------------------------------------*/
#include "spiral.h"

#include <stdio.h>
#include <stdlib.h>
#include <math.h> 

#define fac_2 2
#define fac_3 6
#define fac_4 24
#define fac_5 120
#define fac_6 720
#define fac_7 5040

extern double sqrt(), log(), log10();


/***--- The following function will work on
        a range of (-0.5,+0.5)                    ---***/

/***--- float Sigmoid(float x) 
        {
          return ( (1.0 / (1.0 + AM_EXP((double)-x))) - 0.5);
        }

        float SigmoidPrime(float x)
        {
          x = Sigmoid(x);
          return ( 0.25 - (x * x));
        }
---***/
/*---------------------------------------------------*/
/* The function considered here is tanh()            */
/*---------------------------------------------------*/

float Sigmoid(float x)               /*  (-1.0,1.0)  */
{
  return(tanh((double) x)); 

/*--or-- return ( (2.0 / (1.0 + AM_EXP((double)-x))) - 1.0); 
-----*/
}
/*--------------------------------------------------*/
/* The tanh'() .                                    */
/*--------------------------------------------------*/

float SigmoidPrime(float x)
{
  x = Sigmoid(x);
  return(1.0 - (x * x));
}

float TaylorSeries(float x)
{
  return((float)(x + (x*x*x)/fac_3 + (x*x*x*x*x)/fac_5 + 
                 (x*x*x*x*x*x*x)/fac_7)               
         /(1.0 + (x*x)/fac_2 + (x*x*x*x)/fac_4 + (x*x*x*x*x*x)/fac_6)); 
}

/*-----------------------------------------------------------*/
#define alpha 1                         /*  alpha = k        */
#define x_0  (1.0/2.0)                  /*  = k/(1+k)        */
#define top ((1.0 - x_0) * x_0)         /*  =(1 - x_0) x_0^k */
/*-----------------------------------------------------------*/
/*-----------------------------------------------------------*/
/* The following is a piecewise-rational function:           */
/* f(x) = 1.0 - ((1-x_0) x_0^k)/ x^k                         */
/* alpha = k = 1,2,...                                       */
/*-----------------------------------------------------------*/

float absol(float value)
{
  if (value < 0.0)
    return(-1.0 * value);
  else
    return(value);
}

float Pwrfunc(float x)
{
  float func;
  if (x == 0)
    return 0.0;
  if (x <= x_0 && x >= -x_0)      
    return (x);
  if (absol(x) > x_0)
    {
      func = 1.0 - (top / absol(x));
      if (x > x_0)               
        return (func);
      else
        {
          if (x < -x_0)       
            return (-func);
        }
    }
}

/*----------------------------------------------------------- */
/* The 1st derivative (pwr'()) is:                            */
/* f'(x) = k((1 - x_0)x_0^k)/ x^(k+1)                         */
/* where k=1,2,... and x_0 = k/(1+k)                          */
/*----------------------------------------------------------- */

float PwrPrimefunc(float x)
{
  if (x == 0.0)
    return 0.0;
  if (x <= x_0 && x >= -x_0)      
    return (1.0);
  return(top / absol(x * x));     /*............ k=1 ........ */
}

/*============================================================*/
/* Another piecewise-rational function takes on the following */
/* form  g(x) = x / (1 + |x|) .  Due to Dave Elliott's .      */
/*------------------------------------------------------------*/

float PWR_2(float x)
{
  return(x / (1.0 + absol(x)));
}

/*-----------------------------------------------------------*/
/* The above function is differentiable everywhere with      */
/*  a derivative:                                            */
/*       g'(x) = 1 / (1+ |x|)^2 .                            */
/*-----------------------------------------------------------*/

float PWR_2_prime(float x)
{
  return(1.0 / ((1.0 + absol(x)) * (1.0 + absol(x))));
}

/*=================================================================*/
/*     The following 2 functions uses Taylor Series to approximate */
/*           Tanh(x) and Tanh'(x) , f(x)                           */
/*-----------------------------------------------------------------*/

float SigmoidApprox(float x)
{
  float temp_var=0.0, func;
  temp_var = ((float)(x + (x*x*x)/fac_3 + (x*x*x*x*x)/fac_5 + 
                      (x*x*x*x*x*x*x)/fac_7)               
      /(1.0 + (x*x)/fac_2 + (x*x*x*x)/fac_4 + (x*x*x*x*x*x)/fac_6));

/***------------------------------------------------------***/
/*** Testing at the far end edges of the function ....... ***/
/***         (Optional)                                   ***/
/***------------------------------------------------------***/

  if (temp_var >= 0.97 || temp_var <= -0.97)         
    {                                                
      func = 1.0 - (top / x * x);          
      if (x >= 0.0)                                  
        return (func);                               
      else                                           
        return (-func);                              
    }                                                
  else                                               
    return(temp_var);
}

/*---------------f'(x)-------------*/
float SigmoidApproxPrime(float x)
{
  x = SigmoidApprox (x);
  return(1.0 - (x * x));
}
/*---------------------------------*/

float signum(float x)
{
  if (x > 0.0)
    return 1.0;
  else
    {
      if (x < 0.0)
        return -1.0;
      else
        {
          if (x == 0)
            return 0.0;
        }
    }
} 

float LinearAND(float x)
{
 return (log(absol(x) + 1.0) * signum(x));
}

float LinearANDprime(float x)
{
  return ((1.0 /(absol(x) + 1.0)) * signum(x));
}

/*-------------------------------------------------------------*/

float L2Error (float *target, float *output, float *credit, int n)
{
  float mag_squared = 0.0;
  int i;

  for(i=0;i<n;i++) {
    credit[i] = target[i] - output[i];
    mag_squared += credit[i] * credit[i];
  }/* end for */

  return( mag_squared / 2.0 );
}

float L1Error (float *target, float *output, float *credit, int n)
{
  float mag = 0.0;
  int i;

  for(i=0;i<n;i++) {
    float val, diff;

    diff = target[i] - output[i] ;

    if (diff == 0.0)
      val = 0.0 ;
    else
      val = ( diff < 0.0 ) ? -1.0 : 1.0 ;

    credit[i] = val;
    mag += AM_FABS(diff);
  }/* end for */

  return( mag );
}

/*
**---------------------------------------------------------------------*
**                                                                     *
**  Asymmetric thresholded squared error                               *
**                                                                     *
**---------------------------------------------------------------------*
*/

#define DELTA 0.150

#define GI 1.384   /* Credit multiplier for an Inhibited node */
#define GA 1.384   /* Credit multiplier for an Active node */

#define GIdb2 0.692
#define GAdb2 0.692

float asymmError(float *target, float *output, float *credit, int nx)
{
  float e;
  double t, total_error = 0.0;
  unsigned int k;
  
  for (k=0; k<nx; ++k) {
    e = output[k] - target[k];    
    if ( e < -DELTA ) {
      t = e + DELTA;
      credit[k] = -GA * t;
      total_error += GAdb2*t*t;
    } 
    else if ( e <= DELTA )
           credit[k] = 0.0;
    else {
      t = e - DELTA;
      credit[k] = -GI * t;
      total_error += GIdb2*t*t;
    }
  }
  return( total_error );
}

/*-----------Initialize ------------------*/
void user_init()
{
  spiral_set_random_init_range(0.5);
  spiral_set_learning_rate(0.1); 
}

/*===============================================================*/
/*---------------------------- spiral.h -------------------------*/ 
/* backpropagation simulator header file                         */
/* From: spiral.aspirin                                          */
/*---------------------------------------------------------------*/
#include "aspirin_bp.h"

/* NETWORK FUNCTIONS */

extern void spiral_set_learning_rate(float x);
extern void spiral_set_inertia(float x);
extern float spiral_get_learning_rate();
extern float spiral_get_inertia();
extern int spiral_init_network();
extern void spiral_set_random_init_seed(long x);
extern void spiral_set_random_init_range(float x);
extern int spiral_load_network(char *filename);
extern int spiral_dump_network();
extern int spiral_ascii_dump_network(char *filename);
extern char *spiral_error_string();
extern LB_PTR spiral_query_network(int bbindex, int lindex);

/* BLACK BOX FUNCTIONS */

extern void spiral_propagate_forward();
extern float spiral_calc_error();
extern void spiral_calc_grad();
extern void spiral_update_weights(float scalar);
extern void spiral_set_backprop_counter(int x);
extern int spiral_get_backprop_counter();
extern float *spiral_get_output();
extern void spiral_set_target_output(float *target);
extern float *spiral_get_target_output();
extern void spiral_set_input(float *input);
extern float *spiral_get_input();

/* GENERIC INTERFACE FUNCTIONS (begin with network_) */

extern void network_initialize(char *filename, int verbose);
extern void network_forward(int iterations, void (*generator)(void) );
extern void network_learn(int iterations, void (*generator)(void) );
extern void network_clear_delays();
extern void network_dump(char *filename);
extern void network_load(char *filename);
extern void network_ascii_dump(int formatted);
extern void network_load_ascii();
extern LB_PTR network_query(int bbindex, int lindex);
extern void network_forward_print(int iterations, void
(*generator)(void) );
extern void network_forward_pdpfa(int iterations, void
(*generator)(void), float threshold); 

\end{verbatim}

\newpage
%%%\renewcommand{\thesection}{Appendix \Alph{section}}

%%% \renewcommand{\thesection}{Appendix B}
%%% \renewcommand{\theequation}{\mbox{A}\arabic{equation}}
\pagestyle{myheadings}
\markright{Learn and Test: activation function $\sigma_h$}
%%% \setcounter{equation}{0}

\section{Appendix}
%%%%\appendix

\begin{verbatim}
Backpropagation Learning ....

Black Boxes:

spiral (Saved at Backprop Iteration 0)
Loading Data Files...Done (194 patterns)

Learning...

Learning Rate: 0.100000
Inertia: 0.950000
Your learning rate is a bit high,
if this does not converge try lowering it.
Testing every 50000 iterations.
Must pass 194 patterns with 0.400000 error bound.
Saving every 100000 iterations.
Truncating Learning after 1000000 iterations.
Dumping Network at 100000.
Dumping Network at 200000.
Success!! Dumping Network.Finished.

Learning Rate: 0.100000
Inertia: 0.950000

Elapsed compute time: 925.500000 seconds
Elapsed system time: 3.530000 seconds
Average performance on this task = 86007.6 connections per second

Total Iterations: 200000
Total Epochs: 1030.927856
Total number of connections: 398
      991.3 real       925.5 user         3.8 sys  
~~~~~~~~~~~~~~~~~~~~~~~~~~
Testing....

Loading Saved Network: Network.Finished
Backpropagation Learning
Black Boxes:

spiral (Saved at Backprop Iteration 191502)
Loading Data Files...Done (194 patterns)

Forward...

spiral:
Maxima: 0.660885 
Minima: -0.608330 
Means: -0.007824 
Variances: 0.248432 

Pd = 194/194 = 1.000000
Pfa = 0/194 = 0.000000

Elapsed compute time: 1.250000 seconds
Elapsed system time: 0.240000 seconds
Average performance on this task = 61769.6 connections per second

Total Iterations: 194
Total Epochs: 1.000000
Total number of connections: 398

\end{verbatim}

\newpage
%%%\renewcommand{\thesection}{Appendix C}
%%%\renewcommand{\theequation}{\mbox{A}\arabic{equation}}
\pagestyle{myheadings}
\markright{Learn and Test: activation function $\sigma_k$}
%%%\setcounter{equation}{0}

\section{Appendix}
%%%\appendix

\begin{verbatim}
Backpropagation Learning

Black Boxes:
spiral (Saved at Backprop Iteration 0)
Loading Data Files...Done (194 patterns)

Learning...

Learning Rate: 0.100000
Inertia: 0.950000
Your learning rate is a bit high,
if this does not converge try lowering it.
Testing every 50000 iterations.
Must pass 194 patterns with 0.400000 error bound.
Saving every 100000 iterations.
Truncating Learning after 1000000 iterations.
Dumping Network at 100000.
Dumping Network at 200000.
Success!! Dumping Network.Finished.

Learning Rate: 0.100000
Inertia: 0.950000

Elapsed compute time: 582.290000 seconds
Elapsed system time: 2.50000 seconds
Average performance on this task = 170877 connections per second

Total Iterations: 250000
Total Epochs: 1288.659790
Total number of connections: 398
      607.1 real       582.3 user         2.4 sys  
~~~~~~~~~~~~~~~~~~~~~~~~~~
Testing...
spiral (Saved at Backprop Iteration 240739)
Loading Data Files...Done (194 patterns)
Forward...

spiral:
Maxima: 0.773346 
Minima: -0.791178 
Means: -0.003651 
Variances: 0.267820 

Pd = 194/194 = 1.000000
Pfa = 0/194 = 0.000000

Elapsed compute time: 0.870000 seconds
Elapsed system time: 0.200000 seconds
Average performance on this task = 88749.4 connections per second

Total Iterations: 194
Total Epochs: 1.000000
Total number of connections: 398
\end{verbatim}

\newpage
%%% \renewcommand{\thesection}{Appendix D}
%%% \renewcommand{\theequation}{\mbox{A}\arabic{equation}}
\pagestyle{myheadings}
\markright{Learn and Test: activation function $\sigma_p$}
%%% \setcounter{equation}{0}

\section{Appendix}
%%% \appendix

\begin{verbatim}
Backpropagation Learning ....
Black Boxes:

spiral (Saved at Backprop Iteration 0)
Loading Data Files...Done (194 patterns)

Learning...

Learning Rate: 0.100000
Inertia: 0.950000
Your learning rate is a bit high,
if this does not converge try lowering it.
Testing every 50000 iterations.
Must pass 194 patterns with 0.400000 error bound.
Saving every 100000 iterations.
Truncating Learning after 1000000 iterations.
Dumping Network at 100000.
Dumping Network at 200000.
Dumping Network at 300000.
Dumping Network at 400000.
Dumping Network at 500000.
Dumping Network at 600000.
Dumping Network at 700000.
Dumping Network at 800000.
Success!! Dumping Network.Finished.

Learning Rate: 0.100000
Inertia: 0.950000
Elapsed compute time: 2034.250000 seconds
Elapsed system time: 8.160000 seconds
Average performance on this task = 166302 connections per second

Total Iterations: 850000
Total Epochs: 4381.443359
Total number of connections: 398
     2211.2 real      2034.3 user         8.4 sys  
~~~~~~~~~~~~~~~~~~~~~~~~~~
Test...
Loading Saved Network: Network.Finished
Black Boxes:
spiral (Saved at Backprop Iteration 791954)
Loading Data Files...Done (194 patterns)
Forward...

spiral:
Maxima: 0.665886 
Minima: -0.680417 
Means: 0.001432 
Variances: 0.237889 

Pd = 194/194 = 1.000000
Pfa = 0/194 = 0.000000

Elapsed compute time: 1.30000 seconds
Elapsed system time: 0.240000 seconds
Average performance on this task = 74963.1 connections per second

Total Iterations: 194
Total Epochs: 1.000000
Total number of connections: 398
\end{verbatim}

\end{document}








