\documentstyle[twoside]{article}
\hfuzz1pc
\begin{document}
\title{A Better Activation Function for Artificial Neural Networks
\protect\footnote{Research supported in part 
 by Washington University and by NeuroDyne, Inc.}}
\author{David L. Elliott\\
Institute for Systems Research\\
University of Maryland\\
College Park, MD 20742\\
\\
\\
\protect{\Large ISR Technical Report TR 93-8}}
\date{January 29, 1993}
\maketitle
\vspace{1.0in}
\newpage
\begin{abstract}
An activation function, possibly new, is proposed for
use in digital simulation of artificial neural networks, on the ground
that the computational operation count for this function is much smaller than 
for those employing exponentials and it satisfies a simple  differential
equation generalizing the logistic equation.

\end{abstract}
\newtheorem{prop}{Proposition}
\section*{Introduction: activation functions}
In the digital simulation of neural networks 
 a feedforward memoryless neuron is represented
by the input-output relation
$ y = \sigma (\sum_1^n w_k x_k) $
where $\sigma(\cdot)$, the sigmoidal 
activation or ``squash'' function, should
have the property that it is positive monotone between the values $-1$ and $1$
(or between $0$ and $1$)
for $u \in (-\infty, \infty)$. 

For use in finding optimal weights $w_k$ to
 minimize $||y - y_{desired}||^2 $
 by back-propagation (gradient) algorithms, 
it is also a requirement that $\sigma(\cdot)$ be differentiable
and that it satisfy a simple differential equation, thus permitting
the evaluation of increments of weights
via the chain rule for partial derivatives.
In the seminal paper~\cite{white} on functional approximation with neural 
networks,  the above properties are the only
ones invoked.

For these reasons, $\sigma(\cdot)$  has 
almost always been one of the
two closely related  functions 
\begin{eqnarray}
\sigma_1(u)&=& \frac{1}{1 + \exp(-u)}; \label{1} \\
\sigma_2(u)&=& \tanh(u).  \label{2}
\end{eqnarray}
although other optimization schemes (not using gradients)
may permit the use of the signum function or the saturation function
\begin{eqnarray*}
 \sigma_3(u)& =& u \mbox{~~if~~} |u| < 1,\\
 			~&=& \mbox{sgn}(u) \mbox{~~if~~} |u| > 1. 
\end{eqnarray*}
Noting that the derivatives are given by
\[\sigma_1'(u) = \frac{\exp(-u)}{(1 + \exp(-u))^2} \]
and
\[\sigma_2'(u) = \mbox{sech}^2(u)); \]
substituting from (\ref{1}) and (\ref{2}) respectively, 
 the differential equations for $\sigma_1$ and $\sigma_2$ are
\begin{eqnarray}
\sigma_1' &=& \sigma_1(1 - \sigma_1),  \label{3} \\
\sigma_2' &=& 1 - \sigma_2^2. \label{4}
\end{eqnarray}
The simplicity of these ``Logistic Equations''~(\ref{3}),(\ref{4})
 permits easy computation 
of partial derivatives in the optimization algorithm. 

\section*{The new squash function}
Our proposed activation function is
\begin{equation} \sigma_e(u) = \frac{u}{1 + |u|}
\end{equation}
and we have 
\begin{prop}
The real function $\sigma_e$ is differentiable everywhere 
with derivative
\[ \sigma_e'(u) = \frac{1}{(1 + |u|)^2} \]
and satisfies a generalized logistic differential equation
\begin{equation}
 \sigma_e' = (1 - |\sigma_e|)^2.  \label{ode}
\end{equation}
\end{prop}
{\bf Proof:} Note that $\sigma_e(u)$ is
 differentiable for $u \neq 0$ and
that its left and right derivatives at $u = 0$ are both equal
to $1$. Since 
\[|x| = \frac{|\sigma_e|}{1 -|\sigma_e|},\]
 the differential equation~(\ref{ode}) is satisfied.$\Box$

Equation~(\ref{ode}) is no more complex than Eq.~(\ref{3}) or~(\ref{4})
and permits simple backpropagation code.
The operation count for $\sigma_e$ is an order of magnitude less
than for the exponential functions used in $\sigma_1$ or $\sigma_2$.
In production backpropagation code at NeuroDyne, Inc.
(as well as the author's experiments with his own code and with
 Scott Fahlman's 1989 Quickprop code)
the CPU time saved in net training is sufficiently  substantial to
warrant calling attention to $\sigma_e$.

It also nicely satisfies the properties needed by Barron~\cite{barron}  
in proving that sigmoids approximate at a rate independent of the
dimension of the input vector. Any previous use of  $\sigma_e$ 
and Eq.~(\ref{ode}) in
the backpropagation literature is unknown to the author. 

 \begin{thebibliography}{1}
\bibitem{white} K. Hornik, H. White, M. Stinchcombe, ``Multilayer
feedforward networks are universal approximators,'' {\it Neural Networks}
{\bf 2}359-366, 1989.

\bibitem{barron} Andrew R. Barron, ``Neural net approximation,'' {\it
Proc. 7th Yale Workshop on Adaptive \& Learning Systems}, New Haven, 69-72,
1992.
\end{thebibliography}
\end{document}
