% \chapter*{Appendix B: The Combined Filtering Method}
% \addcontentsline{toc}{chapter}{Appendix B:The Combined Filtering Method}

\chapter{The Combined Filtering Method}

In general, suppose that we are given $K$ linear transforms $T_1,
\ldots, T_K$ and let $\alpha_k$ be the coefficient sequence of an
object $x$ after applying the transform $T_k$, i.e. $\alpha_k = T_k
x$. We will assume that for each transform $T_k$ we have available a
reconstruction rule that we will denote by $T^{-1}_k$ although this is
clearly an abuse of notation.  Finally, $T$ will denote the block
diagonal matrix with the $T_k$'s as building blocks and $\alpha$ the
amalgamation of the $\alpha_k$'s.

A hard thresholding rule associated with the transform $T_k$ synthesizes
an estimate $\tilde{s}_k$ via the formula
\begin{equation}
\label{eq:ht}
\tilde{s}_k = T_k^{-1} \delta(\alpha_k)
\end{equation}
where $\delta$ is a rule that sets to zero all the coordinates of
$\alpha_k$ whose absolute value falls below a given sequence of
thresholds (such coordinates are said to be non-significant).

Given data $y$ of the form $y = s + \sigma z$, where $s$ is the image
we wish to recover and $z$ is standard white noise, we propose solving
the following optimization problem \cite{starck:spie01a}:
\begin{equation}
  \label{eq:l1-min}
  \min \|T\tilde{s}\|_{\ell_1}, \quad \mbox{subject to} \quad s \in C,
\end{equation}
where $C$ is the set of vectors $\tilde{s}$
which obey the linear constraints
\begin{equation}
\label{eq:constraints}
\left\{  \begin{array}{ll}
  \tilde{s} \ge 0, \\
  |T\tilde{s} - Ty| \le e;
  \end{array}
  \right.
\end{equation}
here, the second inequality constraint
only concerns the set of significant coefficients,
i.e. those indices $\mu$ such that $\alpha_\mu =
(Ty)_\mu$ exceeds (in absolute value) a threshold $t_\mu$. Given a
vector of tolerance $(e_\mu)$, we seek a solution whose coefficients
  $(T\tilde{s})_\mu$ are within $e_\mu$ of the noisy
empirical $\alpha_\mu$'s.  Think of $\alpha_\mu$ as being given by
\[
y = \langle y, \varphi_\mu \rangle,
\]
so that $\alpha_\mu$ is normally distributed with mean $\langle f,
\varphi_\mu \rangle$ and variance $\sigma^2_\mu = \sigma^2
\|\varphi_\mu\|^2_2$. In practice, the threshold values range
typically between three and four times the noise level $\sigma_\mu$
and in our experiments we will put $e_\mu = \sigma_\mu/2$. In short,
our constraints guarantee that the reconstruction will take into
account any pattern which is detected as significant by  any of the
$K$ transforms.

\subsubsection*{The Minimization Method}

We propose solving (\ref{eq:l1-min}) using the method of hybrid
steepest descent (HSD) \cite{wave:yamada01}. HSD consists of building
the sequence
\begin{eqnarray}
 s^{n+1} = P(s^{n}) - \lambda_{n+1} \nabla_J(P(s^{n}));
\end{eqnarray}
Here, $P$ is the $\ell_2$ projection operator onto the feasible set
$C$, $\nabla_J$ is the gradient of equation~\ref{eq:l1-min}, and
$(\lambda_{n})_{n \ge 1}$ is a sequence obeying $(\lambda_{n})_{n\ge
  1} \in [0,1] $ and $\lim_{ n \rightarrow + \infty } \lambda_{n} = 0$.

The combined filtering algorithm is:
\begin{enumerate}
\baselineskip=0.4truecm
\itemsep=0.1truecm
\item Initialize $L_{\max} = 1$, the number of iterations $N_i$, and
  $\delta_{\lambda} = \frac{L_{\max}}{N_i}$.
\item Estimate the noise standard deviation $\sigma$, and set $e_k =
  \frac{\sigma}{2}$.
\item For k = 1, .., $K$ calculate the transform: $\alpha^{(s)}_k
  = T_k s$.
\item Set $\lambda = L_{\max}$, $n = 0$, and $\tilde s^{n}$ to 0.
\item While $\lambda >= 0$ do
\begin{itemize}
\item $u = \tilde s^{n}$.
\item For k = 1, .., $K$ do
  \begin{itemize}
  \item Calculate the transform $\alpha_{k} = T_k u$.
  \item For all coefficients $\alpha_{k,l}$ do
     \begin{itemize}
     \item Calculate the residual $r_{k,l} = \alpha^{(s)}_{k,l} -
       \alpha_{k,l}$

     \item if $\alpha^{(s)}_{k,l}$ is significant and $ \mid r_{k,l}
       \mid > e_{k,l}$ then $\alpha_{k,l} = \alpha^{(s)}_{k,l}$
     \item $\alpha_{k,l} = sgn(\alpha_{k,l}) ( \mid \alpha_{k,l} \mid - \lambda)_{+}$.
     \end{itemize}
   \item $u = T_k^{-1} \alpha_{k}$
  \end{itemize}
\item Threshold negative values in $u$ and $\tilde s^{n+1} = u$.
\item $n = n + 1$, $\lambda = \lambda - \delta_{\lambda} $, and goto 5.
\end{itemize}
\end{enumerate}


\section{Creating a Gaussian process}\label{sub:inst}

This section demonstrates creation of a covariance function, a mean function, and finally several random functions drawn from the Gaussian process distribution defined by those objects.

\subsection{Creating a mean function}\label{subsub:mean}

\begin{figure}
    \centering
        \epsfig{file=figs/mean.pdf,width=8cm}
    \caption{The mean function generated by {\sffamily `examples/mean.py'}.}
    \label{fig:mean}
\end{figure}

The mean function of a univariate Gaussian process can be interpreted as a prior guess for the GP, so it is also a univariate function. Mean functions are represented by class \code{Mean}, which is a wrapper for an ordinary \proglang{Python} function. The following code (from \code{pymc/examples/gp/Mean.py}) will produce an instance of class \code{Mean} called $M$:
\begin{CodeChunk}
\begin{CodeInput}
from pymc.gp import *
def quadfun(x, a, b, c):
    return (a * x ** 2 + b * x + c)
M = Mean(quadfun, a = 1., b = .5, c = 2.)
\end{CodeInput}
\end{CodeChunk}

The first argument of \code{Mean}'s init method is the underlying \proglang{Python} function, in this case \code{quadfun}. The extra arguments $a$, $b$  and $c$ will be memorized and passed to \code{quadfun} whenever $M$ is called; the call $M(x)$ in the plotting portion of the script does not need to pass them in.

Mean functions broadcast over their arguments in the same way as \href{http://docs.scipy.org/doc/numpy/reference/ufuncs.html}{\pkg{NumPy} universal functions} \citep{numpybook}, which means that the call $M(x)$, where $x$ is a vector, returns the vector
\begin{eqnarray*}
    [M(x_0),\ldots, M(x_{N-1})].
\end{eqnarray*}

The last part of the code plots $M(x)$ on $-1<x<1$, and its output is shown in figure \ref{fig:mean}. As expected, the plot is a parabola.

\subsection{Creating a covariance function}\label{subsub:cov}
\begin{figure}
    \centering
        \epsfig{file=figs/cov.pdf,width=12cm}
    \caption{The covariance function generated by {\sffamily `examples/cov.py'}. On the left is the covariance function $C(x,y)$ evaluated over a square: $-1\le x\le 1,\ -1\le y\le 1$. On the right is a slice of the covariance: $C(x,0)$ for $0\le x \le 1$}
    \label{fig:cov}
\end{figure}

Covariance functions are represented by the class \code{Covariance}, which like \code{Mean} is essentially a wrapper for ordinary \proglang{Python} functions. The example in \code{pymc/examples/gp/cov.py} uses the popular Mat\`ern function \citep{banerjee}, which is provided in module \code{cov_funs}. In addition to the two arguments $x$ and $y$, the Mat\`ern function takes three parameters: \code{amp} controls the amount by which realizations may deviate from their mean, \code{diff_degree} controls the roughness of realizations (the degree of differentiability), and \code{scale} controls the lengthscale over which realizations change.

The user is free to write functions to wrap in \code{Covariance} objects. See \href{http://code.google.com/p/pymc}{the package documentation} for more information.

The code in \code{pymc/examples/gp/cov.py} will produce an instance of class \code{Covariance} called $C$:
\begin{CodeChunk}
\begin{CodeInput}
from pymc.gp import *
from pymc.gp.cov_funs import matern

C = Covariance(eval_fun = matern.euclidean, diff_degree = 1.4, amp = .4, scale = 1.)
\end{CodeInput}
\end{CodeChunk}

The first argument to \code{Covariance}'s init method is the \proglang{Python} function from which the covariance function will be made. In this case, \code{eval_fun} is \code{matern.euclidean}. Covariance functions' calling conventions are slightly different from ordinary \pkg{NumPy} universal functions' \citep{numpybook} in two ways. First, broadcasting works differently. If $C$ were a \pkg{NumPy} universal function, $C(x,y)$ would return the following array:
    \begin{eqnarray*}
        \begin{array}{ccc}
            [C(x_0,y_0)& \ldots& C(x_{N-1},y_{N-1})],
        \end{array}
    \end{eqnarray*}
    where $x$ and $y$ would need to be vectors of the same length. In fact $C(x,y)$ returns a matrix:
    \begin{eqnarray*}
        \left[\begin{array}{ccc}
            C(x_0,y_0)& \ldots& C(x_0,y_{N_y-1})\\
            \vdots&\ddots&\vdots\\
            C(x_{N_x-1},y_0)& \ldots& C(x_{N_x-1},y_{N_y-1})
        \end{array}\right],
    \end{eqnarray*}
    and input arguments $x$ and $y$ don't need to be the same length. Second, covariance functions can be called with just one argument. $C(x)$ returns
    \begin{eqnarray*}
         [C(x_0,x_0)& \ldots& C(x_{N_x-1},x_{N_x-1})] = \textup{diag}(C(x,x)),
    \end{eqnarray*}
    but is computed much faster than diag$(C(x,x))$ would be.
The extra arguments \code{diff_degree, amp} and \code{scale}, which are required by \code{matern.euclidean}, will be passed to \code{matern.euclidean} by $C$ every time is called.

The output of \code{examples/cov.py} is shown in figure \ref{fig:cov}.

\subsubsection{Cholesky algorithms}

The numerical `heavy lifting' done by this package is primarily handled by \code{Covariance} and its subclasses. \texttt{Covariance} itself bases all its computations on the incomplete Cholesky decomposition algorithm used by the \proglang{Matlab} package \pkg{chol_incomplete} \citep{seeger}. \code{Covariance} computes rows of covariance matrices as they are needed, so if the function it wraps tends to produce covariance matrices with only a few large eigenvalues it can approximate the Cholesky decomposition in less than $O(n^2)$ arithmetic operations \citep{predictivechol}.

\code{Covariance} calls back to \proglang{Python} from \proglang{Fortran} every time it needs a new row. If the function it wraps tends to produce full-rank covariance matrices (for which all rows are required), this is inefficient. \code{FullRankCovariance} is a drop-in replacement for \code{Covariance} that is much faster, but fails (with a helpful error message) if it attempts to factor a matrix that is not full rank. \code{NearlyFullRankCovariance} provides a compromise between the two: it computes covariance matrices in full in \proglang{Fortran}, then factors them using the robust algorithm of \pkg{chol_incomplete}.

\subsection{Drawing realizations}\label{subsub:realizations}
\begin{figure}
    \centering
        \epsfig{file=figs/realizations.pdf,width=8cm}
    \caption{Three realizations from a Gaussian process displayed with mean $\pm$ 1 sd envelope. Generated by {\sffamily `examples/realizations.py'}.}
    \label{fig:realizations}
\end{figure}

The code in \texttt{pymc/examples/gp/realizations.py} generates a list of \code{Realization} objects, which represent realizations (draws) from the Gaussian process defined by $M$ and $C$:
\begin{CodeChunk}
\begin{CodeInput}
from mean import M
from cov import C
from pymc.gp import *

f_list = [Realization(M,C) for i in range(3)]
\end{CodeInput}
\end{CodeChunk}

The init method of \code{Realization} takes only two required arguments, a \code{Mean} object and a \code{Covariance} object. Each element of \code{f_list} is a Gaussian process realization, which is essentially a randomly-generated \proglang{Python} function. Like \code{Mean} objects, \code{Realization} objects use the same broadcasting rules as \pkg{NumPy} universal functions. The call $f(x)$ returns the vector
\begin{eqnarray*}
    [f(x_0)\ldots f(x_{N-1})].
\end{eqnarray*}

Each of the three realizations in \code{f_list} is plotted in figure \ref{fig:realizations}, superimposed on a $\pm$ 1 standard deviation envelope.

\section{Nonparametric regression: observing Gaussian processes}\label{sec:observing}

\begin{figure}
    \centering
        \epsfig{file=figs/obs.pdf,width=5cm}
        \epsfig{file=figs/cond.pdf,width=5cm}
    \caption{The output of {\sffamily `examples/observations.py'}: the observed GP with \code{obs_V = .002} (left) and \code{obs_V = 0} (right). Note that in the conditioned case, the $\pm$ 1 SD envelope shrinks to zero at the points where the observations were made, and all realizations pass through the observed values. Compare these plots to those in figure \ref{fig:realizations}.}
    \label{fig:obs}
\end{figure}

Consider the following common statistical situation: A Gaussian process prior for an unknown function $f$ is chosen, then the value of $f$ is observed at $N$ input points $[o_0\ldots o_{N-1}]$, possibly with uncertainty. If the observation error is normally distributed, it turns out that $f$'s posterior distribution given the new information is another Gaussian process, with new mean and covariance functions.

The probability model that represents this situation is as follows:
\begin{equation}
    \label{regprior}
    \left.\begin{array}{l}
        \textup{data}_i \stackrel{\tiny{\textup{ind}}}{\sim} \textup{N}(f(o_i), V_i)\\
        f \sim \textup{GP}(M,C)\\
    \end{array}\right\}\Rightarrow f|\textup{data} \sim \textup{GP}(M_o, C_o).
\end{equation}
Function \code{observe} imposes normally-distributed observations on Gaussian process distributions. This function converts $f$'s prior to its posterior by transforming $M$ and $C$ in equation \ref{regprior} to $M_o$ and $C_o$:

The code in \code{pymc/examples/gp/observation.py} imposes the observations
\begin{eqnarray*}
    f(-.5) = 3.1\\
    f(.5) = 2.9
\end{eqnarray*}
with observation variance $V=.002$ on the GP distribution defined in \code{mean.py} and \code{cov.py}:
\begin{CodeChunk}
\begin{CodeInput}
from mean import M
from cov import C
from pymc.gp import *
from numpy import *

obs_x = array([-.5,.5])
V = array([.002,.002])
data = array([3.1, 2.9])
observe(M=M, C=C, obs_mesh=obs_x, obs_V=V, obs_vals=data)

f_list = [Realization(M,C) for i in range(3)]
\end{CodeInput}
\end{CodeChunk}

The function \code{observe} takes a covariance $C$ and a mean $M$ as arguments, and tells them that their `true' realization's value on \code{obs_mesh} has been observed to be \code{obs_vals} with variance \code{obs_V}.

The output of \code{observation.py}  is shown in figure \ref{fig:obs}, along with the output with \code{obs_V=0}. Compare these to the analogous figure for the unobserved GP, figure \ref{fig:realizations}. The covariance after observation is visualized in figure \ref{fig:obscov}. The covariance `tent' has been pressed down at points where $x\approx \pm .5$ and/or $y\approx\pm .5$, which are the values where the observations were made.

\begin{figure}
    \centering
        \epsfig{file=figs/obscov.pdf,width=5cm}
    \caption{The covariance function from {\sffamily `observation.py'} after observation. Compare this with the covariance function before observation, visualized in figure \ref{fig:cov} }
    \label{fig:obscov}
\end{figure}

\section{Higher-dimensional GPs}\label{sec:highdim}

In addition to functions of one variable such as $f(x)$, this package supports Gaussian process priors for functions of many variables such as $f(\mathbf{x})$, where $\mathbf{x}=[x_0\ldots x_{n-1}]$. This is useful for modeling dynamical or biological functions of many variables as well as for spatial statistics.

When array is passed into a \code{Mean}, \code{Covariance} or \code{Realization}'s init method or one of these objects is evaluated on an array, the array's last index is understood to iterate over spatial dimension. To evaluate a covariance $C$ on the ordered pairs $(0,1)$, $(2,3)$, $(4,5)$ and $(6,7)$, the user could pass in the following two-dimensional \pkg{NumPy} array:
\begin{verbatim}
[[0,1]
 [2,3]
 [4,5]
 [6,7]]
\end{verbatim}
or the following three-dimensional array:
\begin{verbatim}
[[[0,1]
  [2,3]],

  [4,5]
  [6,7]]]
\end{verbatim}
Either is fine, since in both the last index iterates over elements of the ordered pairs.

The exception to this rule is one-dimensional input arrays. The array
\begin{verbatim}
[0, 1, 2, 3, 4, 5, 6, 7]
\end{verbatim}
is interpreted as an array of eight one-dimensional values, whereas the array
\begin{verbatim}
[[0, 1, 2, 3, 4, 5, 6, 7]]
\end{verbatim}
is interpreted as a single eight-dimensional value according to the convention above.

Means and covariances learn their spatial dimension the first time they are called or observed. Some covariances, such as those specified in geographic coordinates, have an intrinsic spatial dimension. Realizations inherit their spatial dimension from their means and covariances when possible, otherwise they infer it the first time they are called. If one of these objects is subsequently called with an input of a different dimension, it raises an error.

\subsection{Covariance function bundles and coordinate systems}
The examples so far, starting with \code{examples/cov.py}, have used the covariance function \code{matern.euclidean}. This function is an attribute of the \code{matern} object, which is an instance of class \code{covariance_function_bundle}.

Instances of \code{covariance_function_bundle} have three attributes, \code{euclidean}, \code{geo_deg} and \code{geo_rad}, which correspond to standard coordinate systems:
\begin{itemize}
    \item \code{euclidean}: $n$-dimensional Euclidean coordinates.
    \item \code{geo_deg}: Geographic coordinates (longitude, latitude) in degrees, with unit radius.
    \item \code{geo_rad}: Geographic coordinates (longitude, latitude) in radians, with unit radius.
\end{itemize}

See \href{http://code.google.com/p/pymc}{the package documentation} for information regarding creation and extension of covariance function bundles.

\section{Basis covariances}\label{sec:basis}

\begin{figure}[htbp]
    \centering
        \epsfig{file=figs/basiscov.pdf,width=8cm}
        \caption{Three realizations of an observed Gaussian process whose covariance is an instance of \code{BasisCovariance}. The basis in this case is function \code{fourier_basis} from module \code{cov_funs}. 25 basis functions are used.}
    \label{fig:basiscov}
\end{figure}

It is possible to create random functions from linear combinations of finite sets of basis functions $\{e\}$ with random coefficients $\{c\}$:
\begin{eqnarray*}
    f(x) = M(x) + \sum_{i_0=0}^{n_0-1}\ldots \sum_{i_{N-1}=0}^{n_{N-1}-1} c_{i_1\ldots i_{N-1}} e_{i_1\ldots i_{N-1}}(x), \\
    \{c\}\sim \textup{N}(0,K).
\end{eqnarray*}
It follows that $f$ is a Gaussian process with mean $M$ and covariance defined by
\begin{eqnarray*}
    C(x,y)=\sum_{i_0=0}^{n_0-1}\ldots \sum_{i_{N-1}=0}^{n_{N-1}-1} \sum_{j_0=0}^{n_0-1}\ldots \sum_{j_{N-1}=0}^{n_{N-1}-1} e_{i_0\ldots i_{N-1}}(x) e_{j_0\ldots j_{N-1}}(x) K_{i_0\ldots i_{N-1}, j_0\ldots j_{N-1}},
\end{eqnarray*}
where $K$ is the covariance of the coefficients $c$.

Particularly successful applications of this general idea are:
\begin{description}
    \item[Random Fourier series:] $e_i(x) = \sin(i\pi x/L)$ or $\cos(i\pi x/L)$. See \cite{spanos}.
    \item[Gaussian process convolutions:] $e_i(x) = \exp(-(x-\mu_n)^2)$. See \cite{convolution}.
    \item[B-splines:] $e_i(x) = $ a polynomial times an interval indicator. See \href{http://en.wikipedia.org/wiki/Basis_B-spline}{Wikipedia}'s article.
\end{description}
Such representations can be very efficient when there are many observations in a low-dimensional space, but are relatively inflexible in that they generally produce realizations that are infinitely differentiable. In some applications, this tradeoff makes sense.

This package supports basis representations via the \code{BasisCovariance} class:
\begin{verbatim}
    C = BasisCovariance(basis, cov, **basis_params)
\end{verbatim}
The arguments are:
\begin{description}
    \item[\code{basis}:] Must be an array of functions, of any shape. Each basis function will be evaluated at $x$ with the extra parameters. The basis functions should obey the same calling conventions as mean functions: return values should have shape \code{x.shape[:-1]} unless $x$ is one-dimensional, in which case return values should be of the same shape as \code{x}. Note that each function should take the entire input array as an argument.
    \item[\code{cov}:] An array whose shape is either:
        \begin{itemize}
            \item Of the same shape as \code{basis}. In this case the coefficients are assumed independent, and \code{cov[i[0],...,i[N-1]]} (an $N$-dimensional index) simply gives the prior variance of the corresponding coefficient.
            \item Of shape \code{basis.shape * 2}, using \proglang{Python}'s convention for tuple multiplication. In this case \code{cov[i[0],...,i[N-1], j[0],...,j[N-1]]} (a $2N$-dimensional index) gives the covariance of $c_{i_0\ldots i_{N-1}}$ and $c_{j_1\ldots j_{N-1}}$.
        \end{itemize}
        Internally, the basis array is ravelled and this covariance tensor is reshaped into a matrix. This input convention makes it easier to keep track of which covariance value corresponds to which coefficients. The covariance tensor must be symmetric (\code{cov[i[0],...,i[N-1], j[0],...,j[N-1]]} $=$ \code{cov[j[0],...,j[N-1], i[0],...,i[N-1]]}), and positive semidefinite when reshaped to a matrix.
    \item[\code{basis_params}:] Any extra parameters required by the basis functions.
\end{description}

\section{Separable bases}

Many bases, such as Fourier series, can be decomposed into products of functions as follows:
\begin{eqnarray*}
    e_{i_0\ldots i_{N-1}}(x) = \prod_{j=0}^{N-1}e_{i_j}^j(x)
\end{eqnarray*}
Basis covariances constructed using such bases can be represented more efficiently using \code{SeparableBasisCovariance} objects. These objects are constructed just like \code{BasisCovariance} objects, but instead of an $n_0\times \ldots \times n_{N-1}$ array of basis functions they take a nested lists of functions as follows:
\begin{verbatim}
    basis = [ [e[0][0], ... ,e[0][n[0]-1]]
                       ...
              [e[N-1][0], ... ,e[N-1][n[N-1]-1]] ].
\end{verbatim}
For an $N$-dimensional Fourier basis, each of the \code{e}'s would be a sine or cosine; frequency would increase with the second index. As with \code{BasisCovariance}, each basis needs to take the entire input array \code{x} and \code{basis_params} as arguments. See \code{fourier_basis} in \code{examples/gp/basiscov.py} for an example.

\subsection{Example}

Once created, a \code{BasisCovariance} or \code{SeparableBasisCovariance} object behaves just like a \code{Covariance} object, but it and any \code{Mean} and \code{Realization} objects associated with it will take advantage of the efficient basis representation in their internal computations. An example of \code{SeparableBasisCovariance} usage is given in \code{pymc/examples/gp/basis_cov.py}. Compare its output in figure \ref{fig:basiscov} to that in figure \ref{fig:obs}.


\subsection{Broyden–Fletcher–Goldfarb–Shanno (BFGS) algorithm}


\input{settings}

\begin{document}

\lhead{Hyperparameter tuning, Regularization \\ and Optimization}
\rhead{ Deep Learning specialization}
\cfoot{\thepage \ of \pageref{LastPage}}

Since deep learning is a highly empirical and iterative process, it's really helpfull
to train models quickly. Having fast optimization algorithms is important and can
really speed up the efficiency of the process.

\section*{Mini-batch gradient descent}

Instead of processing the entire dataset all at the same time in mini-batch gradient we
partition the dataset in $N$ smaller subsets, then each subset $(X^{[t]}, Y^{[t]})$ of
the dataset are processed.

When every subset of the dataset is processed we say that we did "1 epoch" or one single
pass through the training set. Usually several epochs are made.

Mini-batch gradient descent is considerably faster than normal gradient descent. A
comparison between the cost functions of batch and mini-batch gradient descent is given
in the following image:

\begin{figure}[H]
    \begin{center}
            \includegraphics[width=0.9\textwidth]{img/minibatch.png}
            \caption{Batch vs Mini-batch gradient descent cost function}
        \end{center}
\end{figure}

\textbf{Differente mini-batch sizes}
\begin{itemize}
    \item mini-batch size $m$ : Batch gradient descent.
    \item mini-batch size $1$ : Stochastic gradient descent.
\end{itemize}

In practice the best mini-batch size should be between $1$ and $m$; if you take mini-batch
size $m$ then it takes too long to train the neural network, in the other case taking
size = $1$ is really noisy and losses almost all the speed of doing vectorization.

\begin{figure}[H]
    \begin{center}
            \includegraphics[width=0.9\textwidth]{img/descent.png}
            \caption{Comparison between mini-batch sizes}
        \end{center}
\end{figure}

\textbf{Which mini-batch size to choose?}
\begin{itemize}
    \item If the training set is small use batch gradient descent
    \item In any other case, typical mini-batch sizes are: $64,128,256,512,1024$. They're
    all powers of two because of the way computer memory is layed out and accessed,
    sometimes the code runs faster if the mini-batch size is a power of 2.
\end{itemize}

\section*{Exponentially weighted averages}

Exponentially weighted averages is a technique for smoothing time series data using the
exponential window function. Let's say we have a sequence $\{x_t\}_{t=0}^n$, the output
of the exponentially weighted averages is $\{v_t\}_{t=0}^n$ where:
\begin{align*}
    v_0 &= x_0 \\
    v_i &= \beta v_{i-1} + (1-\beta) x_i  && \forall i>0
\end{align*}

Where $\beta$ is the smoothing factor, $0 < \beta < 1$
\begin{figure}[H]
    \begin{center}
            \includegraphics[width=0.9\textwidth]{img/es.png}
            \caption{Comparison between different smoothing factors}
        \end{center}
\end{figure}

The green line is for a greater $\beta$ and the red line for a smaller $\beta$, in general
the bigger the value of the smoothing factor the longer it takes for the exponentially
weighted average to adapt.

Notice that in general $v_i$ can be expressed as follows:

\begin{align*}
    v_i &= \beta v_{i-1} + (1-\beta) x_i \\
        &= (1-\beta) x_i + \beta(1-\beta) x_{i-1} + \beta^2 v_{i-2} \\
        & \quad \vdots \\
        &= (1-\beta) x_i + \beta(1-\beta) x_{i-1} + \beta^2(1-\beta)x_{i-2} + \dots + \beta^i(1-\beta)x_0\\
        &= \sum_{k=0}^i \beta^{i-k} (1-\beta) x_i
\end{align*}

That means that we have an exponentially decaying function.

\textbf{Bias correction in exponentially weighted averages}

While computing the value og the exponentially weighted averages for small values of $t$
there's a bias because we have few observations and for big value of $\beta$ the
algorithm may take a while to react to the inicialization. A way to correct this is by
changing $v_t$ with the following expression:
\begin{align*}
    v_t^* := \frac{v_t}{1-\beta^t}
\end{align*}
Notice that as $t$ grows $v_t^*$ approaches $v_t$.

\section*{Gradient descent with momentum}

Usually this algorithm is faster than the standard gradient descent procedure. The idea
is to compute an exponentially weighted average of the gradients and then use that
to update the weights instead. Formally, Compute $dW, db$ normally with the current
mini-batch, then:
\begin{align*}
    V_{dW} &= \beta V_{dW} + (1 - \beta)dW \\
    V_{db} &= \beta V_{db} + (1 - \beta)db \\
    W &= W - \alpha V_{dW} \\
    b &= b - \alpha V_{db}
\end{align*}
Supose that when using the standard gradient descent you get a lot of oscilations
when converging to the optimal value, using momentum can fix this because you are taking
into consideration the former results of the gradient therefore the oscilation starts
to decrease and the covergence is faster.

\textbf{Intuition:} in the convex combination of $V_{dW}$ and $dW$ the derivative term
represents the acceleration in the current point but the momentum term let's you take
into consideration the path traveled before.

Notice that now you have another Hyperparameter, $\beta$. Usually $\beta = .9$ works
pretty well.

\section*{RMSprop}

RMSprop, which stands for root mean square prop is another algorithm that can speed up
gradient descent, on iteration $t$ we would compute $dW, db$ normally with the current
mini-batch and then do the following calculation:
\begin{align*}
    S_{dW} &= \beta S_{dW} + (1 - \beta)dW^2 \\
    S_{db} &= \beta S_{db} + (1 - \beta)db^2 \\
    W &= W - \alpha \frac{dW}{\sqrt{S_{dW}}} \\
    b &= b - \alpha \frac{db}{\sqrt{S_{db}}}
\end{align*}

Observation: Using RMSprop makes it posible to use a higher learning rate and perform
faster learning without diverging

\section*{Adam}
Adam algorithm has been chown to work wll across a wide range of deep learning architectures,
the basic idea is to combine momentum and RMSprop. On iteration $t$ we would compute $dW, db$ normally with the current
mini-batch and then do the following:
\begin{align*}
    & \text{Compute momentum and RMSprop:} \\
    & V_{dW} = \beta_1 V_{dW} + (1 - \beta_1)dW  && V_{db} = \beta_1 V_{db} + (1 - \beta_1)db \\
    & S_{dW} = \beta_2 S_{dW} + (1 - \beta_2)dW^2 && S_{db} = \beta_2 S_{db} + (1 - \beta_2)db^2 \\ \\
    & \text{Perform bias correction:} \\
    & V_{dW}^{\text{corr}} = \frac{V_{dW}}{1-\beta_1^t} && V_{db}^{\text{corr}} = \frac{V_{db}}{1-\beta_1^t} \\
    & S_{dW}^{\text{corr}} = \frac{S_{dW}}{1-\beta_2^t} && S_{db}^{\text{corr}} = \frac{S_{db}}{1-\beta_2^t} \\ \\
    & \text{Perform the update:} \\
    & W = W - \alpha \frac{V_{dW}^{\text{corr}}}{\sqrt{S_{dW}^{\text{corr}}}} \\
    & b = b - \alpha \frac{V_{db}^{\text{corr}}}{\sqrt{S_{db}^{\text{corr}}}} \\
\end{align*}

Hyperparameters choice:
\begin{itemize}
    \item $\alpha : \text{ needs to be tuned}$
    \item $\beta_1 : .9$
    \item $\beta_2 : .999$
\end{itemize}
The betas can be tuned but usually those default values are used.

\section*{Learning rate decay}
One of the things that might help speed up the learning algorithm, is to slowly reduce
the learning rate over time. The idea is that during the initial phases while the learning rate
alpha is still large, the algorithm shows a relatively fast learning. But then as alpha
gets smaller, the steps it takes will be slower and smaller. And so it ends up oscillating
in a tighter region around the minimum. If you never reduce the learning rate you might
oscilate a lot around the minimum without converging at all.

The most used formula to compute the learning rate is the following:
\begin{align*}
    \alpha = \frac{1}{1 + \text{decay\_rate}*\text{epoch}}
\end{align*}

Other learning rate decay methods:
\begin{align*}
    \alpha = .95^{\text{epoch}}\alpha_0 && \text{exponentially decay} \\
    \alpha = \frac{k}{\sqrt{\text{epoch}}}\alpha_0 && \text{constant decay}
\end{align*}

\end{document}


%!TEX root = main.tex
\paragraph{An introduction to Bempp}
Bempp \cite{Betcke2021} is a Python based boundary element library for the Galerkin discretization of boundary integral operators in electrostatics, acoustics and electromagnetics.
Bempp originally started as mixed Python/C++ library.
Recently, Bempp underwent a complete redevelopment and the current version Bempp-cl is written completely in Python with OpenCL kernels for the low-level computational routines that are just-in-time compiled for the underlying architecture during runtime.
To understand Bempp consider the simple boundary integral equation
$$
\int_{\Gamma} g(\mathbf{r}, \mathbf{r'}) \phi(\mathbf{r'})ds(\mathbf{r'}) = f(\mathbf{r})
$$
where $\Gamma\subset\mathbb{R}^3$ is the surface of a bounded domain $\Omega\subset\mathbb{R}^3$, and $g(\mathbf{r}, \mathbf{r'}) = \frac{1}{4\pi|\mathbf{r}-\mathbf{r'}|}$ is the electrostatic Green's function.
A Galerkin discretization of this equation takes the form
$$
A\mathbf{x} = b
$$
with $A_{ij} = \int_{\Gamma}\Psi_i(\mathbf{r})\int_{\Gamma}g(\mathbf{r}, \mathbf{r'})\phi_j(\mathbf{r'})ds(\mathbf{r'})ds(\mathbf{r})$ and $b_i = \int_{\Gamma}\psi_i(\mathbf{r})f(\mathbf{r})ds(\mathbf{r})$.
Here, the functions $\Psi_j$ are a finite dimensional basis of $n$ test functions and the $\phi_j$ are a finite dimensional basis of $n$ trial functions with the Galerkin solution being defined as $\phi=\sum_{i}\mathbf{x}_j\phi_j$.
Typical choices for the test and trial functions are either piecewise constant functions or continuous, piecewise linear functions over a surface triangulation of $\Gamma$.
By default, Bempp explicitly computes the matrix $A$ by applying quadrature rules to the arising integrals.
The singularity of the Green's function needs to be accounted for in the quadrature rules for integration over adjacent or identical test/trial triangles \cite{ERICHSEN1998215}.
For well separated triangles standard triangle Gauss rules can be used for the quadrature.
The memory and computational complexity of this discretization is $\mathcal{O}(n^2)$, which is practical for problems of size up to twenty or thirty thousand elements on a single workstation, depending on the available memory and number of CPU cores.
Bempp-cl evaluates the quadrature routines with highly optimized OpenCL kernels that make use of explicit AVX2/AVX-512 acceleration on CPUs.

\paragraph{FMM-accelerated evaluation of integral operators}
We can split up the action of the discretized integral operator $A$ onto a vector $\mathbf{x}$ in the following way.
\begin{equation}
\label{eq:bempp_fmm_matvec}
A\mathbf{x} = P_2^T (G - C)P_1 \mathbf{x} + S \mathbf{x}.
\end{equation}
The matrices $P_1$ and $P_2$ are sparse matrices that convert the action of trial and test functions onto weighted sums over the quadrature points.
The matrix $G$ is a large dense matrix that contains the Green's function evaluation $g(\mathbf{r}_i, \mathbf{r}_j')$ over all quadrature points $\mathbf{r}_i$ and $\mathbf{r}_j'$ across all triangles.
The matrix $C$ is a sparse correction matrix that subtracts out the Green's function values over quadrature points associated with  adjacent triangles.
This is done since these triangles require a singularity adapted quadrature rule.
By explicitly subtracting out these contributions through the matrix $C$, we can use any code for the fast evaluation of particle sums of the type appearing in the $G$ matrix without the requirement to communicate to the summation code the geometry and singularity structure induced by the triangles of the surface mesh, a functionality that most such codes do not offer in any case.
Finally, the matrix $S$ contains the contributions of $A$ arising from singularity adapted quadrature rules across adjacent or identical test/trial triangles.
This matrix is also highly sparse.

We explicitly compute the matrices $P_1$, $P_2$ and $S$, and keep them in memory using sparse storage.
The matrix $C$ is evaluated on the fly for each vector $\mathbf{x}$ through a fast OpenCL kernel.
This leaves the matrix $G$.
The action of $G$ on the vector $\mathbf{y}=P_1 \mathbf{x}$ can be considered as a black-box to evaluate sums of the form
%
\begin{align}\label{eq:nbody_sum}
s(\mathbf{x}_i) = \sum_j g(\mathbf{r}_i, \mathbf{r}_j')q_j.
\end{align}
%
To evaluate this sum we use the C++ Exafmm library, a highly performant library that implements the kernel-independent fast multipole method (\kifmm) to approximately evaluate sums of the above form.
The complexity of this evaluation is $\mathcal{O}(N)$, where $N$ is the product of the number of surface triangles and the number of regular quadrature points per triangle.
The linear complexity means that we can scale the evaluation of the discretized integral operator from tens of thousands to millions of elements, allowing us to solve large electrostatic simulations on a single workstation. Details of the \fmm implementation are discussed in the following section.


\lab{Data Structures II: Trees}{Data Structures II}
\label{lab:Python_DataStructures2}

\objective{A \emph{Tree} is a linked list where each node in the list may refer to more than one other node. This structural flexibility makes trees more useful and efficient than regular linked lists in many applications. Many trees are most easily constructed recursively, so we begin with an overview of recursion. We then implement a recursively structured doubly linked Binary Search Tree. Finally, we compare the standard linked list, our Binary Search Tree, and an AVL tree to illustrate the relative strengths and weaknesses of each structure.}

\section*{Recursion} % ========================================================

A \emph{recursive} function is one that calls itself.
When the function is executed, it continues calling itself until it reaches a specified \emph{base case} where the solution to the problem is known.
The function then exits without calling itself again, and each previous function call is resolved.

As a simple example, consider the function that sums all positive integers from $1$ to some integer $n$.
This function may be represented recursively:
\[f(n) = \sum_{i=1}^ni = n + \sum_{i=1}^{n-1}i = n + f(n-1)\]
$f(n)$ may be calculated by recursively calculating $f(n-1)$, which calculates $f(n-2)$, and so on.
The recursion halts with the base case $f(1) = 1$.

\begin{lstlisting}
def recursive_sum(n):
	"""Calculate the sum of all positive integers in [1, n] recursively."""
    # Base Case: the sum of all positive integers in [1, 1] is 1.
	if n == 1:
		return 1

	# If the base case hasn't been reached, the function recurses by calling
    # itself on the next smallest integer. The result of that call, plus the
    # particular 'n' from this call, gives the result.
	else:
		return n + recursive_sum(n-1)
\end{lstlisting}

The computer calculates \li{recursive_sum(5)} with a sequence of function calls.

\begin{lstlisting}
# To find the recursive_sum(5), calculate recursive_sum(4).
# But to find recursive_sum(4), calculate recursive_sum(3).
# This continues until the base case is reached.

recursive_sum(5)		# return 5 + recursive_sum(4)
	recursive_sum(4)		# return 4 + recursive_sum(3)
		recursive_sum(3)		# return 3 + recursive_sum(2)
			recursive_sum(2)		# return 2 + recursive_sum(1)
				recursive_sum(1)		# Base case: return 1.
\end{lstlisting}

Substituting the values that resulted from each call unwinds the recursion.

\begin{lstlisting}
recursive_sum(5)		# return 5 + 10
	recursive_sum(4)		# return 4 + 6
		recursive_sum(3)		# return 3 + 3
			recursive_sum(2)		# return 2 + 1
				recursive_sum(1)		# Base case: return 1.
\end{lstlisting}

So \li{recursive_sum(5)} returns 15 (which is correct, since $1 + 2 + 3 + 4 + 5 = 15$).

Many problems that can be solved by iterative methods can also be solved (often more intuitively) with a recursive approach.
Consider the function $g:\mathbb{N}\rightarrow\mathbb{N}$ that calculates the $n^{th}$ Fibonacci number:
\[g(n) = g(n-1) + g(n-2),\quad g(0)=0,\quad g(1)=1\]
The mathematical function itself is defined recusively, so it makes sense for an implementation to use recursion.
Compare the following iterative method implementing $g$ to its recursive equivalent.

\begin{lstlisting}
def iterative_fib(n):
	"""Calculate the nth Fibonacci number iteratively."""
	fibonacci = []          # Initialize an empty list.
	fibonacci.append(0)		# Append 0 (the 0th Fibonacci number).
	fibonacci.append(1)		# Append 1 (the 1st Fibonacci number).
	for i in range(1, n):
		# Starting at the third entry, calculate the next number
		# by adding the last two entries in the list.
		fibonacci.append(fibonacci[-1] + fibonacci[-2])
	# When the entire list has been loaded, return the nth entry.
	return fibonacci[n]

def recursive_fib(n):
	"""Calculate the nth Fibonacci number recursively."""
	# The base cases are the first two Fibonacci numbers.
	if n == 0:				# Base case 1: the 0th Fibonacci number is 0.
		return 0
	elif n == 1:			# Base case 2: the 1st Fibonacci number is 1.
		return 1
	# If this call isn't a base case, the function recurses by calling
	# itself to calculate the previous two Fibonacci numbers.
	else:
		return recursive_fib(n-1) + recursive_fib(n-2)
\end{lstlisting}

This time, the sequence of function calls is slightly more complicated because \li{recursive_fib()} calls itself twice at each step.

\begin{lstlisting}
recursive_fib(5)		# The original call makes two additional calls:
	recursive_fib(4)		# this one...
		recursive_fib(3)
			recursive_fib(2)
				recursive_fib(1)		# Base case 2: return 1
				recursive_fib(0)		# Base case 1: return 0
			recursive_fib(1)		# Base case 2: return 1
		recursive_fib(2)
			recursive_fib(1)		# Base case 2: return 1
			recursive_fib(0)		# Base case 1: return 0
	recursive_fib(3)		# ...and this one.
		recursive_fib(2)
			recursive_fib(1)		# Base case 2: return 1
			recursive_fib(0)		# Base case 1: return 0
		recursive_fib(1)		# Base case 2: return 1
\end{lstlisting}

The sum of all of the base case results, from top to bottom, is $1 + 0 + 1 + 1 + 0 + 1 + 0 + 1 = 5$, so \li{recursive_fib(5)} returns 5 (correctly).
The key to recursion is understanding the base cases correctly and making correct recursive calls.

\begin{problem} % Simple recursion for linked lists traversal
The following code defines a simple class for singly linked lists.
\begin{lstlisting}
class SinglyLinkedListNode(object):
    """Simple singly linked list node."""
    def __init__(self, data):
        self.value, self.<<next>> = data, None

class SinglyLinkedList(object):
    """A very simple singly linked list with a head and a tail."""
    def __init__(self):
        self.head, self.tail = None, None
    def append(self, data):
        """Add a Node containing 'data' to the end of the list."""
        n = SinglyLinkedListNode(data)
        if self.head is None:
            self.head, self.tail = n, n
        else:
            self.tail.<<next>> = n
            self.tail = n
\end{lstlisting}
Rewrite the following iterative function for finding data in a linked list using recursion.
Use instances of the \li{SinglyLinkedList} class to test your function.
\begin{lstlisting}
def iterative_search(linkedlist, data):
    """Search 'linkedlist' iteratively for a node containing 'data'."""
	current = linkedlist.head
	while current is not None:
		if current.value == data:
			return current
		current = current.<<next>>
	raise ValueError(str(data) + " is not in the list.")
\end{lstlisting}
(Hint: define an inner function to perform the actual recursion.)
\label{prob:recursion}
\end{problem}

\begin{warn} % Iterative methods > Recursive methods, usually.
It is \textbf{not} usually better to rewrite an iterative method recursively.
In Python, a function may only call itself 999 times.
On the $1000^{th}$ call, a \li{RuntimeError} is raised to prevent a stack overflow.
Whether or not recursion is appropriate depends on the problem to be solved and the algorithm used to solve it.
\end{warn}

\section*{Trees} % ============================================================

A \emph{tree} data structure is a specialized linked list.
Trees are more difficult to build than standard linked lists, but they are almost always more efficient.
While the computational complexity of finding a node in a linked list is $O(n)$, a well-built, balanced tree will find a node with a complexity of $O(\log{n})$.
Some types of trees can be constructed quickly but take longer to retrieve data, while others take more time to build and less time to retrieve data.

The first node in a tree is called the \emph{root}.
The root node points to other nodes, called children.
Each child node in turn points to its children.
This continues on each branch until its end is reached.
A node with no children is called a \emph{leaf node}.

Mathematically, a tree is a directed graph with no cycles.
Therefore a linked lists as a graph qualifies as a tree, albeit a boring one.
The head node is the root node, and it has one child node.
That child node also has one child node, which in turn has one child.
The last node in the list is the only leaf node.

Other kinds of trees may be more complicated.

\section*{Binary Search Trees} % ==============================================

A \emph{binary search tree} (BST) data structure is a tree that allows each node to have up to two children, usually called \li{left} and \li{right}.
The left child of a node contains data that is less than its parent node's data.
The right child's data is greater.

The tree on the right in Figure \ref{fig:trees} is an example of a of binary search tree.
In practice, binary search tree nodes have attributes that keep track of their data, their children, and (in doubly linked trees) their parent.

\begin{figure}[H]
\begin{tikzpicture}[
    level 1/.style={sibling distance=2cm},
    level 2/.style={sibling distance=20mm}]

    \node [circle,draw] (4){4}
        child {node[draw, circle] (5) {5} edge from parent[draw=none]}
        child {node[circle,draw] (3) {3}edge from parent[draw=none]
        child{node[draw,circle](2){2}edge from parent[draw=none]}
        child{node[circle, draw](7){7}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (4a)[above of=4]{}
        child {node[draw=none] (5a) {} edge from parent[draw=none]}
        child {node[draw=none] (3a) {}edge from parent[draw=none]
        child{node[draw=none](2a){}edge from parent[draw=none]}
        child{node[draw=none](7a){}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (4b)[below of=4]{}
        child {node[draw=none] (5b) {} edge from parent[draw=none]}
        child {node[draw=none] (3b) {}edge from parent[draw=none]
        child{node[draw=none](2b){}edge from parent[draw=none]}
        child{node[draw=none](7b){}edge from parent[draw=none]}
        };
\foreach \s/\t in {4a/5a, 4a/3a, 5a/2a, 3a/7a}
    {\draw[->, >=stealth', shorten <=.23cm, shorten >=.1cm](\s)--(\t);}

\end{tikzpicture}
\qquad
\begin{tikzpicture}[
    auto,
    level 1/.style={sibling distance=2cm},
    level 2/.style={sibling distance=20mm}]

    \node [circle,draw] (5a){5}
        child {node[circle,draw] (2a) {2}edge from parent[draw=none]
            child { node[circle,draw] (1a) {1}edge from parent[draw=none]}
            child {node[draw = none] (invisble){} edge from parent[draw=none]}
        }
        child {node[circle,draw] (7a) {7}edge from parent[draw=none]
        child{node[circle, draw](6a){6}edge from parent[draw=none]}
        child{node[circle, draw](8a){8}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (i5a)[above of=5a]{}
        child {node[draw=none] (i2a) {}edge from parent[draw=none]
            child { node[draw=none] (i1a) {}edge from parent[draw=none]}
            child {node[draw = none] (invisbleA){} edge from parent[draw=none]}
        }
        child {node[draw=none] (i7a) {}edge from parent[draw=none]
        child{node[draw=none](i6a){}edge from parent[draw=none]}
        child{node[draw=none](i8a){}edge from parent[draw=none]}
        };
\foreach \s/\t in {i5a/i2a, i5a/i7a}
    {\draw[->, >=stealth', shorten <=.23cm, shorten >=.1cm](\s)--(\t);}
\foreach \s/\t in {i2a/i1a, i7a/i6a, i7a/i8a}
    {\draw[->, >=stealth', shorten <=.27cm, shorten >=.1cm](\s)--(\t);}
\end{tikzpicture}
\caption{Both of these graphs are trees, but only the tree on the right is a binary search tree. How could the graph on the left be altered to make it a BST?}
\label{fig:trees}
\end{figure}

\begin{lstlisting}
class BSTNode(object):
    """A Node class for Binary Search Trees. Contains some data, a
    reference to the parent node, and references to two child nodes.
    """
    def __init__(self, data):
        """Construct a new node and set the data attribute. The other
        attributes will be set when the node is added to a tree.
        """
        self.value = data
        self.prev = None      # A reference to this node's parent node.
        self.left = None      # This node's value will be less than self.value
        self.right = None     # This node's value will be greater than self.value
\end{lstlisting}

The actual binary search tree class has an attribute pointing to its root.

\begin{lstlisting}
class BST(object):
    """Binary Search Tree data structure class.
    The 'root' attribute references the first node in the tree.
    """
    def __init__(self):
        """Initialize the root attribute."""
        self.root = None
\end{lstlisting}

\subsection*{find()} % --------------------------------------------------------

Finding a node in a binary search tree can be done recursively.
Starting at the root, check if the target data matches the current node.
If it does not, then if the data is less than the current node's value, search again on the left child.
If the data is greater, search on the right child.
Continue the process until the data is found or, if the data is not in the tree, an empty child is searched.

\begin{lstlisting}
class BST(object):
    # ...
    def find(self, data):
        """Return the node containing 'data'. If there is no such node
        in the tree, or if the tree is empty, raise a ValueError.
        """

        # Define a recursive function to traverse the tree.
        def _step(current):
            """Recursively step through the tree until the node containing
            'data' is found. If there is no such node, raise a Value Error.
            """
            if current is None:                     # Base case 1: dead end.
                raise ValueError(str(data) + " is not in the tree.")
            if data == current.value:               # Base case 2: data found!
                return current
            if data < current.value:                # Recursively search left.
                return _step(current.left)
            else:                                   # Recursively search right.
                return _step(current.right)

        # Start the recursion on the root of the tree.
        return _step(self.root)
\end{lstlisting}

\begin{info}
Conceptually, each node of a BST partitions the data of its subtree into two halves: the data that is less than the parent, and the data that is greater.
We will extend this concept to higher dimensions in the next lab.
\end{info}

\subsection*{insert()} % ------------------------------------------------------

To insert new data into a binary search tree, add a leaf node at the correct location.
First, find the node that should be the parent of the new node.
This parent node is found recursively, using a similar approach to the \li{find()} method.
Then the new node is added as the left or right child of the parent.
See Figure \ref{fig:BST.insertion}.

\begin{figure}[H] % BST.insert()
\begin{tikzpicture}[
    auto,
    level 1/.style={sibling distance=2cm},
    level 2/.style={sibling distance=20mm}]

    \node [circle,draw] (5a){5}
        child {node[circle,draw] (2a) {2}edge from parent[draw=none]
            child { node[circle,draw] (1a) {1}edge from parent[draw=none]}
            child {node[draw = none] (invisble){} edge from parent[draw=none]}
        }
        child {node[circle,draw] (7a) {7}edge from parent[draw=none]
        child{node[circle, draw](3a){3}edge from parent[draw=none]}
        child{node[circle, draw](8a){8}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (i5a)[above of=5a]{}
        child {node[draw=none] (i2a) {}edge from parent[draw=none]
            child { node[draw=none] (i1a) {}edge from parent[draw=none]}
            child {node[draw = none] (invisbleA){} edge from parent[draw=none]}
        }
        child {node[draw=none] (i7a) {}edge from parent[draw=none]
        child{node[draw=none](i3a){}edge from parent[draw=none]}
        child{node[draw=none](i8a){}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (i5b)[below of=5a]{}
        child {node[draw=none] (i2b) {}edge from parent[draw=none]
            child { node[draw=none] (i1b) {}edge from parent[draw=none]}
            child {node[draw = none] (invisbleB){} edge from parent[draw=none]}
        }
        child {node[draw=none] (i7b) {}edge from parent[draw=none]
        child{node[draw=none](i3b){}edge from parent[draw=none]}
        child{node[draw=none](i8b){}edge from parent[draw=none]}
        };
\node [draw=none, black!20!blue, node distance=1.5cm](root)[above right of=5a]{root};
\draw[black!20!blue, ->, >=stealth', shorten >= .1cm](root)--(5a);
\foreach \s/\t in {i5a/i2a, i2b/i5b, i5a/i7a, i7b/i5b}
    {\draw[->, >=stealth', shorten <=.23cm, shorten >=.1cm](\s)--(\t);}
\foreach \s/\t in {i2a/i1a, i1b/i2b, i7a/i8a, i8b/i7b}
    {\draw[->, >=stealth', shorten <=.27cm, shorten >=.1cm](\s)--(\t);}
\end{tikzpicture}
\qquad
\begin{tikzpicture}[
    auto,
    level 1/.style={sibling distance=2cm},
    level 2/.style={sibling distance=20mm}]

    \node [circle,draw] (5a){5}
        child {node[circle,draw] (2a) {2}edge from parent[draw=none]
            child { node[circle,draw] (1a) {1}edge from parent[draw=none]}
            child {node[draw = none] (invisble){} edge from parent[draw=none]}
        }
        child {node[circle,draw] (7a) {7}edge from parent[draw=none]
        child{node[circle, draw](3a){3}edge from parent[draw=none]}
        child{node[circle, draw](8a){8}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (i5a)[above of=5a]{}
        child {node[draw=none] (i2a) {}edge from parent[draw=none]
            child { node[draw=none] (i1a) {}edge from parent[draw=none]}
            child {node[draw = none] (invisbleA){} edge from parent[draw=none]}
        }
        child {node[draw=none] (i7a) {}edge from parent[draw=none]
        child{node[draw=none](i3a){}edge from parent[draw=none]}
        child{node[draw=none](i8a){}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (i5b)[below of=5a]{}
        child {node[draw=none] (i2b) {}edge from parent[draw=none]
            child { node[draw=none] (i1b) {}edge from parent[draw=none]}
            child {node[draw = none] (invisbleB){} edge from parent[draw=none]}
        }
        child {node[draw=none] (i7b) {}edge from parent[draw=none]
        child{node[draw=none](i3b){}edge from parent[draw=none]}
        child{node[draw=none](i8b){}edge from parent[draw=none]}
        };
\node [draw=none, black!20!blue, node distance=1.5cm](root)[above right of=5a]{root};
\draw[black!20!blue, ->, >=stealth', shorten >= .1cm](root)--(5a);
\node [draw=none, black!20!blue, node distance=1.5cm](parent)[above left of=2a]{parent};
\draw[black!20!blue, ->, >=stealth', shorten >= .1cm](parent)--(2a);
\foreach \s/\t in {i5a/i2a, i2b/i5b, i5a/i7a, i7b/i5b}
    {\draw[->, >=stealth', shorten <=.23cm, shorten >=.1cm](\s)--(\t);}
\foreach \s/\t in {i2a/i1a, i1b/i2b, i2a/i3a, i3b/i2b, i7a/i8a, i8b/i7b}
    {\draw[->, >=stealth', shorten <=.27cm, shorten >=.1cm](\s)--(\t);}
\foreach \s/\t in {i2a/i3a, i3b/i2b}
    {\draw[red, ->, >=stealth', shorten <=.27cm, shorten >=.1cm](\s)--(\t);}
\end{tikzpicture}
\caption{To insert a node containing 3 to the BST on the left, start at the root and recurse down the tree to find the node that should be 3's parent. Connect that parent to the child, then the child to its new parent.}
\label{fig:BST.insertion}
\end{figure}

\begin{problem} % BST.insert()
Implement the \li{insert()} method in the \li{BST} class.
%
\begin{enumerate}
\item Find the parent of the new node.
Consider writing a recursive method, similar to \li{find()}, to do this.
Determine whether the new node will be the parent's left or right child, then double-link the parent and the new child.
\item Do not allow for duplicates in the tree. Raise a \li{ValueError} if there is already a node in the tree containing the input data.
\end{enumerate}

Be sure to consider the special case of inserting to an empty tree.
To test your tree, use (but do not modify) the provided \li{BST.__str__()} method.
\end{problem}

\subsection*{remove()} % ------------------------------------------------------

Deleting nodes from a binary search tree is more difficult than finding or inserting.
Insertion always creates a new leaf node, but removal may delete any kind of node.
This leads to several different cases to consider.

\subsubsection*{Removing a Leaf Node} % - - - - - - - - - - - - - - - - - - - -

In Python, an object is automatically deleted if there are no references to it.
Call the node to be removed the \emph{target node}, and suppose it has no children.
To remove the target, find the target's parent, then delete the parent's reference to the target.
Then there are no references to the target, so the target node is deleted.
Since the target is a leaf node, removing it does not affect the rest of the tree structure.

\subsubsection*{Removing a Node with One Child} % - - - - - - - - - - - - - - -

If the target node has one or more children, be careful not to delete the children when the target is removed.
Simply removing the target as if it were a leaf node would delete the entire subtree originating from the target.

To avoid deleting all of the target's descendents, point the target's parent to an appropriate successor.
If the target has only one child, then that child is the successor.
Connect the target's parent to the successor, and double-link by setting the successor's parent to be the target node's parent.
Then, since the target has no references pointing to it, it is deleted.
The target's successor, however, is pointed to by the target's parent, and so it remains in the tree.
% TODO: Perhaps a figure for this one, as it is the hardest one to implement.

\subsubsection*{Removing a Node with Two Children} % - - - - - - - - - - - - -

Removal is more complicated if the target node has two children.
To delete this kind of node, first find its immediate in-order successor.
This successor is the node with the smallest value that is larger than the target's value.
It may be found by moving to the right child of the target (so that it's value is greater than the target's value), and then to the left for as long as possible (so that it has the smallest such value).
Note that because of how the successor is chosen, any in-order successor can only have at most one child.

Once the successor is found, the target and its successor must switch places in the graph, and then the target must be removed.
This can be done by simply switching the values for the target and its successor.
Then the node with the target data has at most one child, and may be deleted accordingly.
If the successor was chosen appropriately, then the binary search tree structure and ordering will be maintained once the deletion is finished.

The easiest way to implement this is to use recursion.
First, because the successor has at most one child, remove the successor node recursively by calling \li{remove()} on the successor's value.
Then set the data stored in the target node as the successor's value.
See Figure \ref{fig:BST.remove_twoChild}.

\begin{figure} % BST.remove() for removing a node with two children.
\begin{tikzpicture}[
    auto,
    level 1/.style={sibling distance=2cm},
    level 2/.style={sibling distance=20mm}]

    \node [circle,draw] (5){5}
        child {node[circle,draw] (2) {2}edge from parent[draw=none]
            child { node[circle,draw] (1) {1}edge from parent[draw=none]
            child{node[draw=none](invisible){} edge from parent[draw=none]}
            child{node[draw,circle](3){3} edge from parent[draw=none]}
       }
            child {node[draw, circle] (4){4} edge from parent[draw=none]}
        }
    child {node[circle,draw] (9a) {9}edge from parent[draw=none]
        child{node[draw=none](8){}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (5a)[above of=5]{}
        child {node[draw=none] (2a) {}edge from parent[draw=none]
            child { node[draw=none] (1a) {}edge from parent[draw=none]
            child{node[draw=none](invisibleA){} edge from parent[draw=none]}
            child{node[draw=none](3a){} edge from parent[draw=none]}
        }
            child {node[draw = none] (4a){} edge from parent[draw=none]}
        }
        child {node[draw=none] (9a) {}edge from parent[draw=none]
        child{node[draw=none](8a){}edge from parent[draw=none]}
        };

    \node [draw=none, node distance=.1cm] (5b)[below of=5]{}
        child {node[draw=none] (2b) {}edge from parent[draw=none]
            child { node[draw=none] (1b) {}edge from parent[draw=none]
            child{node[draw=none](invisibleB){} edge from parent[draw=none]}
            child{node[draw=none](3b){} edge from parent[draw=none]}
        }
            child {node[draw = none] (4b){} edge from parent[draw=none]}
        }
        child {node[draw=none] (9b) {}edge from parent[draw=none]
        child{node[draw=none](8b){}edge from parent[draw=none]}
        };
\node [draw=none, black!20!blue, node distance=2cm](target)[left of=2a]{target};
\draw[black!20!blue, ->, >=stealth', shorten >= .2cm](target)--(2a);
\node [draw=none, black!20!blue, node distance=2cm](successor)[right of=3a]{successor};
\draw[black!20!blue, ->, >=stealth', shorten >= .2cm](successor)--(3a);
\foreach \s/\t in {2a/1a, 1b/2b, 2a/4a, 4b/2b, 5a/2a, 2b/5b, 5a/9a, 9b/5b, 4a/3a, 3b/4b}
    {\draw[->, >=stealth', shorten <=.23cm, shorten >=.1cm](\s)--(\t);}
\end{tikzpicture}
\qquad
\begin{tikzpicture}[
    auto,
    level 1/.style={sibling distance=2cm},
    level 2/.style={sibling distance=20mm}]

    \node [circle,draw] (5){5}
        child {node[circle,draw] (2) {3}edge from parent[draw=none]
            child { node[circle,draw] (1) {1}edge from parent[draw=none]
            child{node[draw=none](invisible){} edge from parent[draw=none]}
            child{node[draw,circle](3){2} edge from parent[draw=none]}
       }
            child {node[draw, circle] (4){4} edge from parent[draw=none]}
        }
    child {node[circle,draw] (9a) {9}edge from parent[draw=none]
        child{node[draw=none](8){}edge from parent[draw=none]}
        };
    \node [draw=none, node distance=.1cm] (5a)[above of=5]{}
        child {node[draw=none] (2a) {}edge from parent[draw=none]
            child { node[draw=none] (1a) {}edge from parent[draw=none]
            child{node[draw=none](invisibleA){} edge from parent[draw=none]}
            child{node[draw=none](3a){} edge from parent[draw=none]}
        }
            child {node[draw = none] (4a){} edge from parent[draw=none]}
        }
        child {node[draw=none] (9a) {}edge from parent[draw=none]
        child{node[draw=none](8a){}edge from parent[draw=none]}
        };

    \node [draw=none, node distance=.1cm] (5b)[below of=5]{}
        child {node[draw=none] (2b) {}edge from parent[draw=none]
            child { node[draw=none] (1b) {}edge from parent[draw=none]
            child{node[draw=none](invisibleB){} edge from parent[draw=none]}
            child{node[draw=none](3b){} edge from parent[draw=none]}
        }
            child {node[draw = none] (4b){} edge from parent[draw=none]}
        }
        child {node[draw=none] (9b) {}edge from parent[draw=none]
        child{node[draw=none](8b){}edge from parent[draw=none]}
        };
\node [draw=none, black!20!blue, node distance=2cm](target)[left of=2a]{target};
\draw[black!20!blue, ->, >=stealth', shorten >= .2cm](target)--(2a);
\node [draw=none, black!20!blue, node distance=2cm](successor)[right of=3a]{successor};
\draw[black!20!blue, ->, >=stealth', shorten >= .2cm](successor)--(3a);
\foreach \s/\t in {2a/1a, 1b/2b, 2a/4a, 4b/2b, 5a/2a, 2b/5b, 5a/9a, 9b/5b}
    {\draw[->, >=stealth', shorten <=.23cm, shorten >=.1cm](\s)--(\t);}
\end{tikzpicture}
\caption{To remove the node containing 2 from the top left BST, locate the target and its in-order successor. Delete the successor, recording its value. Finally, replace the data in the target with the data that was in the successor.}
\label{fig:BST.remove_twoChild}
\end{figure}

\subsubsection*{Removing the Root Node} % - - - - - - - - - - - - - - - - - - -

In each of the above cases, we must also consider the subcase where the target is the root node.
If the root has no children, resetting the root or calling the constructor will do.
If the root has one child, that child becomes the new root of the tree.
If the root has two children, the successor becomes the new root of the tree.

\begin{problem} % BST.remove()
Implement the \li{remove()} method in the \li{BST} class.
If the tree is empty, or if the target node is not in the tree, raise a \li{ValueError}.
Test your solutions thoroughly, accounting for all possible cases:
\begin{enumerate}
\item The tree is empty (\li{ValueError}).
\item The target is not in the tree (\li{ValueError}).
\item The target is the root node:
	\begin{enumerate}
	\item the root is a leaf node, hence the only node in the tree.
	\item the root has one child.
	\item the root has two children.
	\end{enumerate}
\item The target is in the tree but is not the root:
	\begin{enumerate}
	\item the target is a leaf node.
	\item the target has one child.
	\item the target has two children.
	\end{enumerate}
\end{enumerate}
(Hints: \textbf{Before coding anything}, outline the entire function with comments and \li{if}-\li{else} blocks.
Use the \li{find()} method wherever appropriate.)
\end{problem}

\section*{AVL Trees} % ========================================================

Binary search trees are a good way of organizing data so that it is quickly accessible.
However, pathologies may arise when certain data sets are stored using a basic binary serch tree.
This is best demonstrated by inserting ordered data into a binary search tree.
Since the data is already ordered, each node will only have one child, and the result is essentially a linked list.

\begin{lstlisting}
# Sequentially adding ordered integers destroys the efficiency of a BST.
>>> unbalanced_tree = BST()
>>> for i in range(10):
...     unbalanced_tree.insert(i)
...
# The tree is perfectly flat, so it loses its search efficiency.
>>> print(unbalanced_tree)
[0]
[1]
[2]
[3]
[4]
[5]
[6]
[7]
[8]
[9]
\end{lstlisting}

Problems also arise when one branch of the tree becomes much longer than the others, leading to longer search times.

An \emph{AVL tree} (named after Georgy Adelson-Velsky and Evgenii Landis) is a tree that prevents any one branch from getting longer than the others.
It accomplishes this by recursively ``balancing'' the branches as nodes are added.
See Figure \ref{fig:avl_balance}.
The AVL's balancing algorithm is beyond the scope of this project, but details and exercises on the algorithm can be found in Chapter 2 of the Volume II text.

\begin{figure}[H] % AVL Tree
\begin{tikzpicture}[
    auto,
    level 1/.style={sibling distance=2cm},
    level 2/.style={sibling distance=20mm}]

    \node [circle,draw] (4){4}
    child{node[draw, circle](2){2}edge from parent[draw=none]
        child{node[draw, circle](1){1}edge from parent[draw=none]}
        child{node[draw, circle](3){3}edge from parent[draw=none]}
    }
    child{node[draw,circle](5){5}edge from parent[draw=none]
        child{node[draw=none](invisible){}edge from parent[draw=none]}
        child{node[draw, circle](6){6}edge from parent[draw=none]}
    };

    \node [draw=none, node distance=.1cm] (4a)[above of=4]{}
    child{node[draw=none](2a){}edge from parent[draw=none]
        child{node[draw=none](1a){}edge from parent[draw=none]}
        child{node[draw=none](3a){}edge from parent[draw=none]}
    }
    child{node[draw=none](5a){}edge from parent[draw=none]
        child{node[draw=none](invisibleA){}edge from parent[draw=none]}
        child{node[draw=none](6a){}edge from parent[draw=none]}
    };

    \node [draw=none, node distance=.1cm] (4b)[below of=4]{}
    child{node[draw=none](2b){}edge from parent[draw=none]
        child{node[draw=none](1b){}edge from parent[draw=none]}
        child{node[draw=none](3b){}edge from parent[draw=none]}
    }
    child{node[draw=none](5b){}edge from parent[draw=none]
        child{node[draw=none](invisibleB){}edge from parent[draw=none]}
        child{node[draw=none](6b){}edge from parent[draw=none]}
    };

\foreach \s/\t in {4a/2a, 2a/1a, 2a/3a, 4a/5a, 5a/6a}
    {\draw[->, >=stealth', shorten <=.23cm, shorten >=.1cm](\s)--(\t);}
\node [draw=none, black!20!blue, node distance=1.5cm](root)[above right of=4a]{root};
\draw[black!20!blue, ->, >=stealth', shorten >= .1cm](root)--(4);

\end{tikzpicture}
\caption{The balanced AVL tree resulting from inserting 1, 2, 3, 4, 5, and 6, in that order. After each insertion the tree rebalances if necessary.}
\label{fig:avl_balance}
\end{figure}

\newpage

\begin{lstlisting}
>>> balanced_tree = AVL()
>>> for i in range(10):
...     balanced_tree.insert(i)
...
# The AVL tree is balanced, so it retains (and optimizes) its search efficiency.
>>> print(balanced_tree)
[3]
[1, 7]
[0, 2, 5, 8]
[4, 6, 9]
\end{lstlisting}

\begin{problem} % Compare build and search times.
Write a function to compare the build and search times of the data structures we have implemented so far.

Read the file \texttt{english.txt}, adding the contents of each line to a list of data.
For various values of $n$, repeat the following:
%
\begin{enumerate}
\item Get a subset of $n$ \textbf{random} items from the data set.
\\(Hint: use a function from the \li{random} or \li{np.random} modules.)
\item Time (separately) how long it takes to load a new \li{SinglyLinkedList}, a \li{BST}, and an \li{AVL} with the $n$ items.
\item Choose 5 \textbf{random} items from the subset, and time how long it takes to find all 5 items in each data structure.
Use the \li{find()} method for the trees, but to avoid exceeding the maximum recursion depth, use the provided \li{iterative_search()} function from Problem \ref{prob:recursion} to search the \li{SinglyLinkedList}.
\end{enumerate}

Report your findings in a single figure with two subplots: one for build times, and one for search times.
Use log scales if appropriate.

\begin{comment} % TODO: Decide whether or not to show them these plots.
Your figure should resemble the following plots, though your results may be less smooth.
%
\begin{figure}[H] % Solution to problem 4.
    \centering
    \begin{subfigure}[b]{.5\textwidth}
        \centering
        \includegraphics[width=\textwidth]{BuildTimes.pdf}
    \end{subfigure}%
    \begin{subfigure}[b]{.5\textwidth}
        \centering
        \includegraphics[width=\textwidth]{SearchTimes.pdf}
    \end{subfigure}
    % \caption{The \li{SinglyLinkedList} has the fastest build times, but the \li{AVL} has the fastest search times. How would the graphs be different if the data had been sorted to begin with?}
\end{figure}
\end{comment}
\end{problem}

\section*{Conclusion} % =======================================================

Every data structure has advantages and disadvantages.
Recognizing when an application may take advantage of a certain structure, especially when that structure is more complicated than a Python list or set, is an important skill.
Choosing structures wisely often results in huge speedups and easier data maintenance.

\newpage

\section*{Additional Material} % ==============================================

\subsection*{Improvements to the BST} % ---------------------------------------

The following are a few ideas for improving the \li{BST} class.

\begin{enumerate}
\item Add a keyword argument to the constructor so that if an iterable is input, each element of the iterable is immediately added to the tree.
This makes it possible to cast other iterables as a \li{BST}, like
Python's standard data structures.
\item Add an attribute that keeps track of the number of items in the tree.
Use this attribute to implement the \li{__len__()} magic method.
\item Add a method for translating the \li{BST} into a sorted Python list using a depth-first search.
\\(Hint: examine the provided \li{__str__()} method carefully.)
\end{enumerate}

\subsection*{Other Trees} % ---------------------------------------------------

There are many variations on the Binary Search Tree, each with its own advantages and disadvantages.
Consider writing classes for the following structures.
%
\begin{enumerate}
\item A \href{https://en.wikipedia.org/wiki/B-tree}{\emph{B-Tree}} is a tree whose nodes can contain more than one piece of data and point to more than one other node.
See Chapter 2 of the Volume II text for details.

\item The nodes of a \href{https://en.wikipedia.org/wiki/Red%E2%80%93black_tree}{\emph{Red-Black Tree}} are labeled either red or black.
The tree satisfies the following rules.
%
\begin{enumerate}
    \item Every leaf node is black.
    \item Red nodes only have black children.
    \item Every (directed) path from a node to any of its descendent leaf nodes contains the same number of black nodes.
\end{enumerate}
%
When a node is added that violates one of these constraints, the tree is rebalanced and recolored.

\item A \href{https://en.wikipedia.org/wiki/Splay_tree}{\emph{Splay Tree}} includes an additional operation, called splaying, that makes a specified node the root of the tree.
Splaying several nodes of interest makes them easier to access because they will be close to the root.
\end{enumerate}


\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{color}
\usepackage[breaklinks,colorlinks=true]{hyperref}

\definecolor{Blue}{RGB}{0,122,255}
\hypersetup{colorlinks,breaklinks,urlcolor=Blue,linkcolor=Blue,citecolor=Blue,urlcolor=Blue}

\def\Pr{\mathop{\mathbb{P}}}
\newcommand\Ex{\mathop{\mathbb{E}}}
\newcommand\br[1]{\left(#1\right)}
\newcommand\bbr[1]{\left[#1\right]}
\newcommand\cbr[1]{\left\{#1\right\}}
\newcommand\pd[2]{\frac{\partial #1}{\partial #2}}

\begin{document}

\title{Price and greeks of European binary option}
\date{}

\maketitle

Consider an asset with spot price $S = \{S_t; 0 \leq t \leq T\}$ following geometric Brownian motion of volatility $\sigma$.

A European binary call option with strike $K$ and maturity $T$ pays off
\begin{align}
    \text{Payoff}_\text{Call}
        = 1_{S_T \geq K} ,
\end{align}
where
$1_{S_T \geq K}$ is an indicator function.
A European binary put option with the same strike and maturity pays off
\begin{align}
    \text{Payoff}_\text{Put}
        = 1_{S_T \leq K} .
\end{align}

\section*{Price}

The price of the European binary call option is given by
\begin{align}
    \text{Price}_\text{Call}
        = \Ex[1_{S_T \geq K}] = N(d_2) ,
\end{align}
where
$N$ is the cumulative distribution function of the normal distribution and
\begin{align}
    d_2
        = \frac{\log (S_0 / K)}{\sigma \sqrt{T}} - \frac12 \sigma \sqrt{T} .
\end{align}

The price of a European binary put option is given by $1 - \text{Price}_\text{Call}$
because a relation $\text{Payoff}_\text{Call} + \text{Payoff}_\text{Put} = 1$ holds almost surely.

\section*{Delta}

Delta is given by
\begin{align}
    \text{Delta}_\text{Call}
        = \frac{N^\prime(d_2)}{S_0 \sigma \sqrt{T}} ,
\end{align}
where
we used a derivative $\partial d_2 / \partial S_0 = 1 / (S_0 \sigma \sqrt{T})$.

Delta of a European binary put option is $\text{Delta}_\text{Put} = - \text{Delta}_\text{Call}$.

\section*{Gamma}

Gamma of the European binary option is given by
\begin{align}
    \text{Gamma}
        = \frac{N^{\prime\prime}(d_2)}{S_0^2 \sigma^2 T}
            - \frac{N^{\prime}(d_2)}{S_0^2 \sigma \sqrt{T}}
        = - \frac{N^{\prime}(d_2)}{S_0^2 \sigma \sqrt{T}}
            \br{\frac{d_2}{\sigma \sqrt{T}} + 1} ,
    \label{eq:gamma}
\end{align}
where we used a relation $N^{\prime\prime}(x) = - x N^\prime(x)$ to show the second equality.

Gamma of a European binary put option is $\text{Gamma}_\text{Put} = - \text{Gamma}_\text{Call}$.

\end{document}


\chapter{Code Examples}

\section{Colour Conversion}

This code enables conversion between RGB channel intensities into greyscale luminance using different conversion methods based on their respective colourspace recommendations in a variety of modes. A starting discussion on the effects of different methods can be found \hyperlink{http://cadik.posvete.cz/color_to_gray_evaluation/}{here}

\begin{lstlisting}
import numpy as np
from math import sqrt

def convert_set_to_greyscale(cifar_set, method=0, gamma=1.0):
	converted_set = np.empty((cifar_set.shape[0], 1, cifar_set.shape[2], cifar_set.shape[3]), 'float32')
	for image_index, image in enumerate(cifar_set):
		for row_index, row in enumerate(image[0]):
			for pixel_index, pixel in enumerate(row):
			grey = 0.0
			if method == 0:  # Rec.709 luminance
				grey = (0.2126 * cifar_set[image_index, 0, row_index, pixel_index]) + \
				(0.7152 * cifar_set[image_index, 1, row_index, pixel_index]) + \
				(0.0722 * cifar_set[image_index, 2, row_index, pixel_index])
			elif method == 1:  # NTSC/W3C luminance
				grey = (0.299 * cifar_set[image_index, 0, row_index, pixel_index]) + \
				(0.587 * cifar_set[image_index, 1, row_index, pixel_index]) + \
				(0.114 * cifar_set[image_index, 2, row_index, pixel_index])
			elif method == 2:
				grey = sqrt(((0.299 * cifar_set[image_index, 0, row_index, pixel_index]) ** 2) +
				((0.587 * cifar_set[image_index, 1, row_index, pixel_index]) ** 2) +
				((0.114 * cifar_set[image_index, 2, row_index, pixel_index]) ** 2))
			elif method == 3:
				grey = sqrt(((0.2126 * cifar_set[image_index, 0, row_index, pixel_index]) ** 2) +
				((0.7152 * cifar_set[image_index, 1, row_index, pixel_index]) ** 2) +
				((0.0722 * cifar_set[image_index, 2, row_index, pixel_index]) ** 2))
			elif method == 4:  # Simple mean of RGB
				grey = ((cifar_set[image_index, 0, row_index, pixel_index]) +
				(cifar_set[image_index, 1, row_index, pixel_index]) +
				(cifar_set[image_index, 2, row_index, pixel_index])) / 3
			else:
				print 'Error: This is not a valid conversion mode.\n Reverting to colour.'
			return cifar_set.astype('float32') / 255

		converted_set[image_index, 0, row_index, pixel_index] = np.float32(grey/255)
	print 'Converted ', len(converted_set), ' images to greyscale.'
	return converted_set

\end{lstlisting}


In this section, we present the formal model for the synthesis problem.
We model glycan molecules and production rules as labeled trees. The glycan molecules are assembled by applying the production rules repeatedly.
Our synthesis problem reduces into finding the pieces of trees that represent the production rules.

Let $S$ be the set of sugar monomers that builds glycans,
the oligomer molecules. Each $s \in S$ is associated with arity $m$
(written $arity(s) = m$).
% , {\em i.e.}, the maximum number of children for $s$.
The children of the monomers are indexed. We refer to the $k$th child of $s$ for some $k \leq arity(s)$.
They correspond to bonds at specific positions in the monomers where children are connected.
Now we define the glycan molecules as labeled trees. Now onward we refer to the glycans simply as molecules.

\begin{df}
A {\em molecule} $m = (V,M,C,v_0)$ is a labeled tree, where
$V$ is a set of nodes in the tree,
$M : V \maps S$ maps nodes to their label,
$C : V \times \naturals \pmaps V$ maps the indexed children of nodes, and
$v_0 \in V$ is the root of $m$.
A molecule must respect the arity of monomers, i.e., if $M(v) = s$ and $C(v,n) = v'$ then $n \leq arity(s)$.
\end{df}
% \todo{Rewrite the following paragraph}
% \todo{definitions mixed with algorithms seems strange I think you should define all the formal stuff first and then present the algorithms}
Let us define notations related to the tree structure.
Let $\mathit{m = (V,M,C,v_0)}$ and $m' = (V',M',C',v_0')$ be molecules.
With an abuse of notation, we write $v \in m$ to denote $v \in V$.
For each $v \in m$, if $(v,n)$ is not in the domain of $C$, we write $C(v,n) = \bot$.
We assume that $C(v,0) = \bot$.
%and if $n > n' > 0$ and $C(v,n) \neq \bot$, then $C(v,n') \neq \bot$.
Let $NumberOfChildren(v)$ be equal to the number of $n$s such that $C(v,n) \neq \bot$.
A node $v \in V$ is a {\em leaf} of $m$ if $C(v,n) = \bot$ for each $n$.
Let $depth(v)$ be the length of the path from $v_0$ to $v$.
A {\em branch} of $m$ is a path from $v_0$ to some leaf of $m$.
Let $height(m)$ be the length of the longest branch in $m$.
We define ancestor relation recursively as follows. Let $ancestor(m,v,0) = v$.
For for $d > 0$, let $ancestor(m,v,d) = ancestor(m,v', d-1)$ if $C(v',i) = v$ for some $i$.

Since we will be matching the parts of the trees and applying rules to expand them,
let us introduce notations for matching.
Let recursively-defined predicate $Match(m,v,m',v')$ state that
$v \in m$, $v' \in m'$, $s = M(v) = M(v')$, and $Match( m, C(v,n), m', C(v',n) )$ for each $n \leq arity(s)$ such that $C(v,n) \neq \bot$.
In other words, the subtree in $m$ rooted at $v$ is embedded in $m'$ at node $v'$.
Let $subtree(m)$ be the set of molecules such that
$m' = (\_,\_,\_,v_0') \in subtree(m) \lequiv Match(m,v_0,m',v_0')$.
Let us also define a utility to copy a subtree of a molecule into another molecule.
Let $Copy(m,v)$ return a molecule $m'' = (V'',M'',C'',v_0'')$ such that
$V''$ is a set of fresh nodes, $Match(m,v,m'',v_0'')$, and $Match(m'',v_0'',m,v)$.
Both the $Match$ conditions say that the trees rooted at
$v$ and $v_0''$ are identical.

Now we define the model of the production process of the molecules.
A production rule expands a molecule $m$ by attaching a new piece of tree at a node that has a vacant spot among its children if the surroundings of the
node satisfy some condition.
The rule is modeled as a tree that has two parts.
One part should already be there in $m$ and the other part will be appended to $m$.

\begin{df}
  A {\em production rule} $r = (V, M, C, v_0, v_e)$ is a labeled tree, where
  $V$ is a set of nodes,
  $M : V \maps S$ maps nodes to labels,
  $C : V \times \naturals \pmaps V$ maps the indexed children of nodes,
  $v_0 \in V$ is the root, and
  $v_e \in V$ is the root of expanding part of the rule.
\end{df}

If we {\em apply}, a rule $r$ on a molecule $m$, then it is extended at some node
$v \in m$. A copy of the descendants of $v_e$ will be attached to
$v$ in $m$, and the rest of the nodes in the rule have to match $v$ and above.
We call the descendants of $v_e$ as {\em expanding nodes}
and all the other nodes as {\em matching nodes}.

\begin{wrapfigure}{r}{0.37\textwidth}
  \vspace{-12mm}
    % \begin{minipage}{0.48\linewidth}
      \small
    % \center
  \begin{tikzpicture}[shorten >=1pt,thick,node distance=1cm,on grid]
    \node[loc] (v0) {$A$};
    \node[loc, below right of=v0] (v1) {$B$};
    \node[sqloc, below left of=v0] (v2) {$A$};
    \path[->] (v0) edge (v1);
    \path[->] (v0) edge (v2);
  \end{tikzpicture}
  % \par
  % (a)
  % \end{minipage}
  % \begin{minipage}{0.48\linewidth}
    % \small
    % \center
  \quad
  \begin{tikzpicture}[shorten >=1pt,thick,node distance=1cm,on grid]
    \node[loc] (v0) {$A$};
    \node[loc, below right of=v0] (v1) {$A$};
    \node[loc, below right of=v1] (v3) {$B$};
    \node[loc, below left of=v1] (v2) {$A$};
    \path[->] (v0) edge (v1);
    \path[->,dashed] (v1) edge (v2);
    \path[->] (v1) edge (v3);
  \end{tikzpicture}
  % \par
  \hfill (a) \hfill (b)\hfill\mbox{}
  % (b)
  % \end{minipage}
  \caption{(a) A rule. (b) An application of the rule.}
  \label{fig:exrule}
  \vspace{-6mm}
\end{wrapfigure}
\paragraph{Example:} In Figure~\ref{fig:exrule}(a), we present a rule. It has
  two kinds of nodes.
  The rule adds the square node ($v_e$).
  The circular nodes %$A$ and its right child $B$
  are the pattern, which must be present in the molecule to apply the rule.
  In Figure~\ref{fig:exrule}(b), we present an application of the rule.
  The solid tree with three nodes is the initial molecule.
  The middle node $A$ and its right child $B$ form a pattern, where the rule is applicable.
  %Upon applying the rule,
  The rule adds a left child with the label $A$ to the middle node.
  The rule is not applicable at the root $A$ due to pattern mismatch.
  % , since it has a right child $A$
  % and the combination does not match the pattern.

We naturally extend the definitions related to molecules, including $Match$ and $Copy$,
to the production rules.
Let us formally define the molecule productions using the rules.
Let $m = (V,M,C,v_0)$ be a molecule and $r = (V_r, M_r, C_r, v_{0r}, v_e)$ be a production rule.
Let $d$ be such that $v_{0r} = ancestor(r,v_e,d)$, i.e., $v_e$ is at the depth $d$ in $r$.
Let $i$ be such that $C_r(v',i) = v_{0r}$ for some $v' \in r$.
We apply $r$ on $m$ at node $v \in m$ such that $C(v,i) = \bot$.
We obtain an expanded molecule as follows.
Let $(V',M',C',v_0') = Copy(r,v_e)$.
The expanded molecule is
$
m' = (V \uplus V', M \uplus M', C \uplus C' \uplus \{(v,i) \mapsto v_0'\}, v_0)
$ if
$Match( r, v_{0r}, m', ancestor(m', v_0', d) )$ where $\uplus$ is the disjoint union.
The match condition states that after attaching the new nodes $V'$
the rule tree must be embedded in $m'$ at the $d$th ancestor of $v'_0$.
We write $m' = Apply(m, v, r)$ to indicate the application of $r$
on molecule $m$ at node $v$ that results in $m'$.
If $r$ is not applicable at $v$, we write $Apply(m, v, r) = \bot$.
We write $m' = Apply(m, r)$ if there is a $v \in m$ such that
$m' = Apply(m,v,r)$.

Let $R$ be a set of rules.
A molecule $m$ is {\em producible} by $R$ from a set of molecules $Q$
if there is sequence of molecules $m_0,...,m_k$
such that $m_0 \in Q$, $m_k=m$, and
for each $0<i\leq k$, $m_{i} = Apply(m_{i-1},r)$ for some $r \in R$.
Let $P(Q,R)$ denote the set of molecules that are producible from
rules $R$ from a set of molecules $Q$.
We have discussed in Section~\ref{sec:bio} that all the production rules
are not applied at the same time.
The rules may live in compartments and
the rule sets of the compartments are applied one after another.
To model compartments for the rules,
let us suppose we have a sequence $R_1,...,R_k$ of set of rules.
Let $P(Q, R_1,..,R_k) = P(..P(P(Q,R_1),R_2),..,R_k)$ denoting the
trees obtained after applying the rule sets one after another.

% \subsection{Synthesis problem}
In nature, we observe a set of glycan molecules $\mu$ present in a cell.
%
However, we may not know the production rules to produce the molecules.
%
We will be developing a method to find the rules.
%
The {\em synthesis problem} is to find a set $R$ of production rules
such that $\mu = P(S,R)$,
where $S$ is the set of monomers.

% %
% We can define a more general form of the synthesis problem.
% Let us suppose we are also given $k$ compartments with unknown rules.
% The goal of synthesis is to find  $R_1,...,R_k$ such that
% $\mu = P(S, R_1,..,R_k)$.
% We may relax the requirement of exactly producing $\mu$.
% We may say that molecules in $\mu$ have to be produced, but we are ok
% if subtrees of the molecules of $\mu$ are also produced.
% Formally, we may weaken the requirement to $\mu \subseteq P(Q, R_1,..,R_k) \subseteq subtree(\mu)$.

% The above are a few simplified versions of the biological problem.
% We may generalize the problem further where
% %rules are partitioned and the partitions are applied in phases one after another,
% rules are not applied exhaustively due to time constraints,
% the given $\mu$ is finite and may not be exhaustive,
% and we only know the weights of the parts of molecules in $\mu$.
% We will first present a method for solving a simplified version of the problem.
% However, our tool handles some of the above variations.
% We will discuss the variations in section~\ref{sec:variations}.

%--------------------- DO NOT ERASE BELOW THIS LINE --------------------------

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: