\begin{intro}
  The following results can be found in any book on linear
  algeba. Thus, we will just keep the arguments short. There will be a
  focus on normal matrices justified by results on conditioning of
  eigenvalue problems later on.

  Thus, spectral theory based on module theory will not be needed in
  this class. The spectral theorem for normal matrices on the other
  hand is fairly simple and can be proved without too much overhead.
\end{intro}

\begin{Definition}{eigenvalue}
  An \define{eigenvalue} of a matrix $\mata\in \C^{n\times n}$ is a
  complex number $\lambda$ such that the matrix
  \begin{gather}
   \mata-\lambda\id
  \end{gather}
  is singular.

  The set of all eigenvalues of $\mata$ is called the
  \define{spectrum} $\sigma(\mata)$.

  The \define{eigenspace} for $\lambda$ is the kernel of
  $A-\lambda\id$, that is, the set
\begin{gather}
    \esp{\lambda} = \bigl\{
    \vv \in \C^n \;\big\vert\;
    \mata\vv = \lambda\vv \bigr\}.
\end{gather}
The \define{geometric multiplicity} of $\lambda$ is the dimension of
$\esp{\lambda}$.

An \define{eigenvector} for $\lambda$ is a (normed) vector in
$\esp\lambda$. We refer to an eigenvector $\lambda$ and a
corresponding eigenvector $\vv$ as \define{eigenpair}.
\end{Definition}

\begin{Definition}{eigenvalue-algebraic}
  An \define{eigenvalue} of a matrix $\mata\in \C^{n\times n}$ is a root of the characteristic polynomial $\chi(\lambda) = \det(\mata-\lambda\id)$.

  The \define{algebraic multiplicity} of an eigenvalue is the multiplicity of the corresponding root of the characteristic polynomial.
\end{Definition}

\begin{Lemma}{eigenvalue-equivalent}
  The two definitions of an eigenvalue are consistent.
\end{Lemma}

\begin{Theorem}{eigenvalue-count}
  Every matrix in $\C^{n\times n}$ has at most $n$ eigenvalues. The algebraic multiplicities of all eigenvalues add up to $n$.
\end{Theorem}

\begin{proof}
  The ``at most'' follows from the fact that a polynomial contains
  linear factors $x-\lambda_i$ for each of its roots
  $\lambda_i$. Thus, if the characteristic polynomial has $k$ roots it
  has at least degree $k$. On the other hand, the characteristic
  polynomial has degree $n$, such that $k\le n$.

  The second statement is due to the fact that every polynomial over
  $\C$ is a product of linear factors.
\end{proof}

\begin{remark}
  The last theorem is not true in $\R$, as it is not algebraically
  closed. Thus, even a real matrix may have complex eigenvalues and
  eigenvectors. Therefore, all results in this chapter will be on
  complex matrices, but some simplifications for real matrices will be
  pointed out.
\end{remark}

\begin{Definition}{eigenvalue-simple}
  An eigenvalue is \define{simple}, if its algebraic and geometric multiplicity are one. It is \define{semi-simple}, if its algebraic and geometric multiplicities are equal.
\end{Definition}

\begin{remark}
  It is possible to refine the concept of eigenvalues and eigenvectors by distinguishing between right eigenvalues and vectors solving
  \begin{gather}
      \mata \vv = \lambda \vv,
  \end{gather}
  and left eigenvalues and vectors solving
  \begin{gather}
    \vu \mata = \lambda \vu,
  \end{gather}
  where $\vu$ is now a row vector. By taking the transpose of this equation\footnote{Here, we refer to the real transpose obtained by simply exchanging indices, not the complex conjugate transpose.},
  \begin{gather}
    \label{eq:evp:1}
    \mata^T \vu^T = \lambda \vu^T,
  \end{gather}
  we see that $\vu$ is a left eigenvector of $\mata$ if and only if
  $\vu^T$ is a right eigenvector of $\mata^T$.
\end{remark}

\begin{Lemma}{eigenvalues-conjugate}
  Every eigenvalue $\lambda$ of $\mata\in\C^{n\times n}$ is also an eigenvalue of $\mata^T$.
\end{Lemma}

\begin{proof}
  The determinant does not change when the matrix is transposed, therefore
  \begin{gather}
    \chi(\mata^T)
    = \det(\mata^T-\lambda \id)
    = \det(\mata-\lambda \id)
    = \chi(\mata).
  \end{gather}
  Thus, the eigenvalues of $\mata$ and of $\mata^T$ coincide.
\end{proof}

\subsection{Normal and Hermitian matrices}

\begin{Definition}{normal-Hermitian}
  A matrix $\mata\in\C^{n\times n}$ is called \define{normal} if there holds
  \begin{gather}
      A^*A = AA^*.
  \end{gather}
  It is called \define{Hermitian} or \define{complex symmetric}, if there holds
  \begin{gather}
      A=A^*.
  \end{gather}
\end{Definition}

\begin{Lemma}{Hermitian-eigenvalues-real}
  All eigenvalues of a Hermitian matrix are real.
\end{Lemma}

\begin{Theorem*}{Hermitian-diagonalizable}{Spectral theorem for Hermitian matrices}
  A Hermitian matrix $\mata\in\C^{n\times n}$ is diagonalizable with
  an orthogonal basis of eigenvectors and real eigenvalues. That is,
  there is a real, diagonal matrix $\matlambda$ and a unitary matrix
  $\matq$ such that
  \begin{gather}
    \mata = \matq^T \matlambda\matq.
  \end{gather}
\end{Theorem*}

%\begin{proof}
%\end{proof}

\begin{Corollary}{symmetric-diagonalizable}
  A symmetric matrix $\mata\in\R^{n\times n}$ is diagonalizable with
  an orthogonal basis of eigenvectors and real eigenvalues.
\end{Corollary}

\begin{Theorem*}{normal-diagonalizable}{Spectral theorem for normal matrices}
  A matrix $\mata\in\C^{n\times n}$ is normal if and only if it is diagonalizable by a unitary matrix.

  It is normal if and only if there exists an orthonormal basis of eigenvectors.
\end{Theorem*}

\begin{proof}

\end{proof}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\lab{Mayavi}{Mayavi}

\section*{3-D Plotting with Mayavi} % =========================================

Although Matplotlib is capable of creating 3-D plots, Mayavi\footnote{If Mayavi is not installed on your machine, run \li{conda install mayavi} from the command line. See Appendix \ref{updateinstall} for more info.} does it much faster and with better visuals.
Here we introduce methods for plotting space curves, scatter plots, and surfaces in 3-D.
The \li{mlab} submodule within the \li{mayavi} package contains the functions for creating these plots.
% We will use Mayavi for all 3-D plots in these labs. % O RLY?

\begin{warn}
Mayavi must be imported before Matplotlib.
\begin{lstlisting}
from mayavi import mlab
from matplotlib import pyplot as plt
\end{lstlisting}
\end{warn}

\begin{comment}
\begin{info} % Installation note. Make into a footnote?
If you do not have the \li{Mayavi} package installed on your system, you may download it by running the following commands from the command line:
\begin{lstlisting}
$ conda install conda               # Download the most recent installer.
$ conda install anaconda            # Update all packages.
$ conda install mayavi              # Installs mayavi.
\end{lstlisting}

% For more information regarding installing Python packages, see Appendix \ref{updateinstall}.
\end{info}

\begin{table} % Mayavi plotting functions. Probably should put this back in.
\begin{center}
\begin{tabular}
{|c|l|}
\hline
Function & Description \\
\hline
\li{barchart} & Produces 3D histogram-like plots\\
\li{contour3d} & Plots level surfaces of functions of three variables\\
\li{flow} & Creates a trajectory of particles following the flow of a vector field\\
\li{imshow} & Use a colormap to view a 2D array as an image\\
\li{mesh} & Plot a surface using \li{(x,y,z)} coordinates supplied as three 2D arrays\\
\li{plot3d} & Draws lines between points\\
\li{points3d} & Plots glyphs (like points) at the coordinates supplied\\
\li{quiver3d} & Generate 3D vector fields\\
\li{surf} & Plot a surface with a 2D array as elevation data\\
\hline
\end{tabular}
\end{center}
\caption{Some plotting functions in \li{mlab}.}
\label{table:mlab_functions}
\end{table}
\end{comment}

\subsection*{Lines} % ---------------------------------------------------------

The function \li{mlab.plot3d()} is the 3-D Mayavi equivalent for Matplotlib's \li{plt.plot()}.
Because the plot is 3-D, we must provide $x$, $y$, and $z$ coordinates, each contained in 1-D arrays of the same length.
The points \li{(x[i], y[i], z[i])} are graphed in $\mathbb{R}^3$ and connected with straight lines.

Consider the following curve, parametrized by time:
\begin{align*}
x(t) &= \cos(t)(1+\cos(6t))\\
y(t) &= \sin(t)(1+\sin(6t)\\
z(t) &= \sin\left(\frac{6}{11}t\right)
\end{align*}
The following code plots the curve over the time domain $t \in [0,2\pi]$.
The resulting plot is shown in Figure \ref{fig:plot3d}.

\begin{lstlisting}
>>> from mayavi import mlab

# Calculate the coordinates of a curve parametrized by time.
>>> t = np.linspace(0, 2*np.pi, 100)
>>> x = np.cos(t) * (1 + np.cos(t*6))
>>> y = np.sin(t) * (1 + np.cos(t*6))
>>> z = np.sin(t*6/11.)

# Plot and show the figure.
>>> mlab.plot3d(x, y, z)
>>> mlab.show()
\end{lstlisting}

\begin{figure} % Mayavi line and point plots.
\captionsetup[subfigure]{justification=centering}
\centering
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{plot3d.png}
    \caption{A 3-D curve generated by \li{mlab.plot3d()}.}
    \label{fig:plot3d}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{points3d.png}
    \caption{A 3-D scatter plot generated by \li{mlab.points3d()}.}
    \label{fig:points3d}
\end{subfigure}
\end{figure}

\subsection*{Points} % --------------------------------------------------------

The function \li{mlab.points3d()} is the 3-D Mayavi equivalent for Matplotlib's \li{plt.scatter()}.
Each point is plotted, but not connected with lines.
In the code below, the optional input array \li{s} defines a scalar for each point that modifies the color and size of the point.
The output is in Figure \ref{fig:points3d}.

\begin{lstlisting}
>>> t = np.linspace(0, 4*np.pi, 30)
>>> x = np.sin(2*t)
>>> y = np.cos(t)
>>> z = np.cos(2*t)
>>> s = 2 + np.sin(t)

# Adjust the keyword argument 'scale_factor' so all points are visible.
>>> mlab.points3d(x, y, z, s, scale_factor=.15)
>>> mlab.show()
\end{lstlisting}

\subsection*{Surfaces} % ------------------------------------------------------

The function \li{mlab.surf()} renders a 3-D surface.
Because the surface is over a 2-D domain, we create a coordinate grid with \li{np.mgrid()} (similar to \li{np.meshgrid()}).
This function uses the slicing syntax \li{[start:stop:step]}, similar to \li{range()} and \li{np.arange()}, but is accessed with brackets instead of parentheses.

The following code produces the hyperbolic paraboloid $f(x,y) = \frac{x^2}{4} - \frac{y^2}{4}$ over the domain $[-4,4]\times[-4,4]$.
The result is displayed in Figure \ref{fig:surf_example}.

\begin{lstlisting}
>>> X, Y = np.mgrid[-4:4:.025, -4:4:.025]
>>> Z = (X**2)/4. - (Y**2)/4.
>>> mlab.surf(X, Y, Z, colormap='RdYlGn')
>>> mlab.show()
\end{lstlisting}

\begin{figure}
\includegraphics[width=.7\textwidth]{mesh_example.png}
\caption{Sample output of \li{mlab.surf()}.}
\label{fig:surf_example}
\end{figure}

Like Matplotlib, Mayavi supports various color schemes, either as a solid color or with a varying colormap.
For example, the plot in Figure \ref{fig:surf_example} uses the colormap \li{'RdYlGn'}.
For a list of all colormaps in Mayavi, see \url{http://docs.enthought.com/mayavi/mayavi/mlab_changing_object_looks.html}.

% TODO: More exercises with Mayavi!!! Don't have to be too hard either...
\begin{problem}
Plot the function $z = \frac{1}{10}\sin(10(x^2+y^2))$ on $[-1,1] \times [-1,1]$ using Mayavi.
\end{problem}


\section{Introduction}

The key to convolutional neural networks (CNNs) lies in the way they employ convolution as a local and shift-invariant operation on Euclidean spaces, e.g.\ $\RR$ for audio or $\RR^2$ for images.
Recently, the concept of CNNs has been extended to more general spaces to exploit different structures that may underlie the data:
This includes spherical convolutions for rotationally invariant data~\cite{cohen2018sphericalcnn,esteves2018sphericalcnn,defferrard2020deepsphere}, more general convolutions on homogeneous spaces~\cite{cohen2016groupnn,kondor2018groupnn,worrall2017harmonicnn}, or convolutions on graphs~\cite{bruna2014graphnn,defferrard2016convolutional}.

Graph neural networks (GNNs) have proven to be an effective tool that can take into account irregular graphs to better learn interactions in the data~\cite{bronstein2017geometric,wu2020survey}.
Although graphs are useful in describing complex systems of irregular relations in a variety of settings, they are intrinsically limited to modeling pairwise relationships. The advance of topological methods in machine learning~\cite{Gabrielsson2020topological, Hofer2019LearningRO, rieck2018neural}, and the earlier establishment of \emph{topological data analysis (TDA)}~\cite{carlsson2008,chazal2017,edelsbrunner2010computational,ghrist2008barcodes} as a field in its own right, have confirmed the usefulness of viewing data as topological spaces in general, or in particular as simplicial complexes. The latter can be thought of as a higher-dimensional analog of graphs~\cite{moore2012,patania2017}. We here take the view that structure is encoded in \emph{simplicial complexes}, and that these represent $n$-fold interactions. In this setting, we present \emph{simplicial neural networks (SNNs)}, a neural network framework that take into account locality of data living over a simplicial complex in the same way a GNN does for graphs or a conventional CNN does for grids.

Higher-order relational learning methods, of which hypergraph neural networks~\cite{feng2018hypergraphs} and motif-based GNNs~\cite{monti2018motif} are examples, have already proven useful in some applications, e.g.\ protein interactions~\cite{ze2020graph}. However the mathematical theory underneath the notion of convolution in these approaches does not have clear connections with the global topological structure of the space in question. This leads us to believe that our method, motivated by Hodge--de Rham theory, is far better suited for situations where topological structure is relevant, such as perhaps in the processing of data that exists naturally as vector fields or data that is sensitive to the space's global structure.


\chapter{Bernoulli-Euler Beam equation}\label{Bernoulli-Euler Beam equation}

Assumptions used to derive the Bernoulli-Euler beam equation are (Complete derivation in \cite{craig2006fundamentals}):
\begin{enumerate}
	\item Beam is bending in a plane, in this case in the y-direction, where the x-direction is along the length of the beam.
	\item The  neutral axis undergoes no deformation in the longitudinal direction.
	\item Cross sections remain plane and perpendicular to the nuetral axis.
	\item The material is linear-elastic.
	\item Stresses in the y and z direction are negligible compared to those in the x direction.
	\item Rotary inertial effects are not considered.
	\item Mass density is constant at each cross section, so that each mass center is coincident with the centroid of that section.
\end{enumerate}
Using kinematics and assumptions 2 \& 3, the strain in the x direction may be related to the curvature of the beam, $\mu(x,t)$, and the distance from the neutral axis by
\begin{equation} \label{eq:curvature}
\epsilon=-\dfrac{y}{\mu}
\end{equation}
then, with assumption 4 \& 7 the relation from curvature to moment is
\begin{equation} \label{eq:curv_moment}
M(x,t)=\dfrac{EI}{\mu}
\end{equation}
\begin{figure}
	\centering
	\def\svgwidth{300pt}
	\import{figures/}{BeamFBD.pdf_tex}
	\caption{Free body diagram of a beam section in planar bending.}
	\label{fig:BeamFBD}
\end{figure}
where $ E $, Young's modulus, and $ I $, area moment of inertia are constant in cross sections. By using Newton's laws and the free body diagram of a single beam element, see Figure \ref{fig:BeamFBD}, the equations of motion are summarized as:
\begin{equation} \label{eq:Newton}
\sum{F_y}=\Delta m \ddot{v} \quad \textrm{\&} \quad \sum{M_G}=0
\end{equation}
Moment equation is represented as moments summarized at the center of mass, $ G $. The right hand side of moment equation of EOM \eqref{eq:Newton} is know to be null due to assumption 6. Applying Newton's equations \eqref{eq:Newton} to the FBD of Figure \ref{fig:BeamFBD} results in the force equation
\begin{equation} \label{eq:force_EOM}
F(x,t)-F(x+\Delta x,t)=\rho A\Delta x \frac{\partial^2 v}{\partial t^2}
\end{equation}
and moment equation
\begin{equation} \label{eq:moment_EOM}
-M(x,t) + M(x+\Delta x,t) + F(x,t)\left( \frac{-\Delta x}{2}\right)  + \left[-F(x+\Delta x,t)\right] \left(\frac{\Delta x}{2}\right) = 0
\end{equation}
Taking the limit of equations \eqref{eq:force_EOM} \& \eqref{eq:moment_EOM} as $ \Delta x \rightarrow 0 $ results in equations \eqref{eq:force_EOM_partial} \& \eqref{eq:moment_EOM_partial} respectively.
\begin{equation} \label{eq:force_EOM_partial}
\frac{\partial F}{\partial x}=-\rho A \frac{\partial^2 v}{\partial t^2}
\end{equation}
\begin{equation} \label{eq:moment_EOM_partial}
\frac{\partial M}{\partial x} - F=0
\end{equation}
Assuming the beam slope, $ \frac{\partial v}{\partial x} $, remains relatively small, then linearized curvature of the beam is inversely related to $ \frac{\partial^2 v}{\partial x^2} $. Substituting this linearized curvature in \eqref{eq:curv_moment} produces
\begin{equation} \label{eq:moment_curvature_partial}
M(x,t)=EI\frac{\partial^2 v}{\partial x^2}
\end{equation}
Using linearized moment equation \eqref{eq:moment_curvature_partial}, combined with \eqref{eq:force_EOM_partial} \&  \eqref{eq:moment_EOM_partial} lends the Euler beam equation
\begin{equation} \label{eq:euler_beam_equation}
\frac{\partial^2}{\partial x^2} \left(EI\frac{\partial^2 v}{\partial x^2}\right) = -\rho A\frac{\partial^2 v}{\partial t^2}
\end{equation}
This is the governing differential equation for transverse motion of a slender beam. This equation is not suitable for an application involving lengths that are not much greater than the width of the beam \cite{genta2007dynamics}.


\documentclass{article}

\usepackage{style/conference}
\usepackage{opensans}
\usepackage{graphicx}
\usepackage{biblatex}
\usepackage{fontawesome}
\usepackage[hidelinks]{hyperref}

\addbibresource{references.bib}
\input{style/math_commands.tex}

\title{Winged Horses with a Deep Convolutional Generative Adversarial Network}

\begin{document}
\maketitle
\begin{abstract}
    This paper proposes using a deep convolutional generative adversarial network (DCGAN) to generate images that look like a Pegasus. We give an overview of the underpinning mathematical theory behind GANs, display the architecture that was applied for the task, suited for use with the CIFAR-10 dataset, and describe and explain our pre-processing and implementation steps. The best result obtained is shown, a Pegasus with a dark body colour, alongside the unique batch, and limitations of the approach as well as improvements unaccomplished due to time constraints are discussed.
\end{abstract}

\section{Methodology}
\subsection{Underpinning mathematical theory}
The method is to train a deep convolutional generative adversarial network (DCGAN) \cite{article}, by having two networks, $D$ and $G$, play the following two-player minimax game with value function $V(D, G)$:
\begin{equation}
    \underset{G}{\text{min}} \ \underset{D}{\text{max}} \ V(D,G) = \mathbb{E}_{x\sim p_{data}(x)}\big[logD(x)\big] + \mathbb{E}_{z\sim p_{z}(z)}\big[log(1-D(G(z)))\big].
\end{equation}
The discriminator network, $D$, discriminates between real and fake images. $D$ takes as input an image, $x$, and outputs the scalar probability, $D(x)$, that $x$ came from training data (real) rather than the generator (fake).

The generator network, $G$, generates fake images. $G$ takes as input a latent space vector, $z$, sampled from a standard normal distribution, $p_z$, and outputs the mapped vector, $D(z)$, in data space (e.g. a space of dimension $3 \times 32 \times 32$, which corresponds to the number of channels, height, and width of an image).

GANs proceed by simultaneously training both $G$ and $D$. We train $D$ to maximise $logD(x)$, the probability of $D$ assigning the correct label to both real and fake images. At the same time, we train $G$ to minimise $log(1-D(G(z)))$, the probability of $D$ assigning the correct label to a fake image. Alternatively, we can train $G$ to maximise $logD(G(z))$, the probability of $D$ assigning the incorrect label to a fake image (i.e. making a mistake), which results in the same dynamics of $G$ and $D$. We decide to train $G$ with the latter objective function, since it is supposed to provide much stronger gradients early in learning \cite{NIPS2014_5ca3e9b1}.

There exists a unique solution (global optimum) to the minimax game. This is achieved when $G$'s estimate of the training data distribution, $p_g$, matches exactly the true training data distribution, $p_{data}$. When $p_g = p_{data}$, the fake images $G$ generates (by sampling from $p_g$) are indistinguishable from the real images (sampled from $p_{data}$), so $D(x)$ equals $\frac{1}{2}$ for all $x$, since the discriminator can do no better than guess whether $x$ is real or fake.

\subsection{Architecture}
An architectural diagram of the method is included in Figure \ref{fig:architecture}. More detailed architectural diagrams of the generator and discriminator networks are also included, in Figure \ref{fig:generator} and Figure \ref{fig:discriminator} respectively. Figures \ref{fig:generator} and \ref{fig:discriminator} were made using the online tool `\textit{NN-SVG}' \footnote{https://github.com/alexlenail/NN-SVG}.

\begin{figure}[h]
    \begin{center}
        \includegraphics[width=0.75\textwidth]{figures/architecture.pdf}
    \end{center}
    \caption{Method architectural diagram. Both generator and discriminator networks process data with a batch size of 64. The image data used to train the discriminator is a manually identified subset of the CIFAR-10 dataset. The motivation and method for doing this is explained in Section \ref{implementation}.}
    \label{fig:architecture}
\end{figure}

\begin{figure}[h]
    \begin{center}
        \includegraphics[width=0.75\textwidth]{figures/generator.png}
    \end{center}
    \caption{Generator architectural diagram. Transformations happen from left to right.}
    \label{fig:generator}
\end{figure}

The generator architecture consists of four layers. The first layer takes as input a $100 \times 1 \times 1$ vector of noise (random numbers) sampled from the standard normal distribution, and consists of a convolutional transpose layer, with a $4 \times 4$ kernel, stride 1, and 0 padding, paired with a batch normalisation layer and ReLU activation function. The second and third layers are like the first, except with stride 2 and 1 padding. The fourth and final layer consists of a convolutional transpose layer, with a $4 \times 4$ kernel, stride 2, and 1 padding, with $tanh$ activation function, and outputs a $3 \times 32 \times 32$ image.

\begin{figure}[h]
    \begin{center}
        \includegraphics[width=0.75\textwidth]{figures/discriminator.png}
    \end{center}
    \caption{Discriminator architectural diagram. Transformations happen from right to left.}
    \label{fig:discriminator}
\end{figure}

The discriminator architecture also consists of four layers. The first layer takes as input a $3 \times 32 \times 32$ image, and consists of a convolutional transpose layer, with a $4 \times 4$ kernel, stride 2, and 1 padding, with Leaky ReLU activation function. The second and third layers are like the first, each paired with an additional batch normalisation layer. The fourth and final layer consists of a convolutional transpose layer, with a $4 \times 4$ kernel, stride 1, and 0 padding, with Sigmoid activation function, and outputs a probability.

\subsection{Implementation} \label{implementation}
Our goal was to create unique and diverse images of winged horses that all looked like a Pegasus. We chose to use the CIFAR-10 dataset to train our DCGAN model.

In order to create an image that looked like a Pegasus, we had to combine a horse's body with a bird's wings. The first step would be to identify good examples of images that exhibited these features clearly. Therefore, we manually identified birds and horses using the annotated class labels available for the CIFAR-10 images. However, upon inspecting the subset, it was noted that the required features were not often displayed clearly: not all horse images showed the horse's body --- instead, showing just the head --- and not all bird images showed birds with their wings outstretched --- instead, showing their beak or neck. (See Appendix \ref{appendix:A}.)

To get around this issue, we trained a convolutional neural network classifier by Chris Willcocks\footnote{https://colab.research.google.com/gist/cwkx/3a6eba039aa9f68d0b9d37a02216d385/convnet.ipynb} for 20 epochs, and used it to choose our model training data. By selecting only horse images that the classifier predicted correctly and with over 80\% confidence, it was almost guaranteed that the resulting horse images would be of the horse's entire body. Similarly, by selecting only bird images that the classifier misclassified as airplanes with over 80\% confidence, the resulting bird images would be of birds in mid-flight with outstretched wings. (See Appendix \ref{appendix:B}.)

From this, we made 10 batches of bird images and 10 batches of horse images, with a batch size of 64. Using a weighted random sampler, we trained the DCGAN on this data, and varied the number of epochs the model trained for (from 100 to 1000), as well as the probability of the discriminator training on horse data versus bird data (from odds of 1:1 to 1:20). Our best results were when the odds of the discriminator being given a batch of horses versus a batch of birds to train on was 1:5.

\section{Results}
The generated images are low resolution, but they are not blurry since there are clear-defined edges which outline the objects in them. The objects have realistic shapes because one can easily identify the body of a horse and distinguish between its features, including the torso, head, legs, and, in some cases, tail. The objects in them also have some texture, in particular where the shading highlights the mane and musculature. Although there is some noise, the generated images look real enough to classify as horse-like.

The images are not that different to their nearest neighbours in the dataset, indicating that the network has learned well the data distribution of horse images. Although the objects in the generated images all hold the same side-view pose, there is diversity between the samples within the batch of 64 provided, in the orientation of pose (left-facing and right-facing), colour (various shades of brown, black, and white), as well as stance (standing and moving). Unfortunately, few clearly identifiable winged horses were made --- the closest were horses with humps instead of wings. There is definitely mode collapse, since we observed the generator rotate through outputting horse-like objects against a green, grassy backdrop and outputting bird-like objects against a blue, cloudless backdrop. (See Appendix \ref{appendix:C}.)

The best batch of images looks like this:
\begin{center}
    \includegraphics[width=0.5\textwidth]{figures/best-batch.png}
\end{center}

From this batch, the most Pegasus-like image is:
\begin{center}
    \includegraphics[width=0.075\textwidth]{figures/best-pegasus.png}
\end{center}

\section{Limitations}
It's very difficult to see anything that looks like a Pegasus. This was anticipated since GANs are notoriously difficult to train due to lacking convergence, having diminishing gradients, and being susceptible to mode collapse. We demonstrate this by plotting the running average of generator and discriminator loss over the previous 100 iterations during a training run, shown below:
\begin{center}
    \includegraphics[width=0.6\textwidth]{figures/avgloss.png}
\end{center}

In the future, mode collapse in the GAN could be reduced by lowering the Lipschitz constant for the discriminator function, as was achieve by Wasserstein GANs \cite{arjovsky2017wasserstein} or by spectral normalisation \cite{DBLP:journals/corr/abs-1802-05957}, which normalises the weights for each layer using the spectral norm $\sigma(\mathbf{W}$). Furthermore the diversity and quantity of training data could have been improved by incorporating data from the STL-10 dataset. Training with STL-10 at the full 96x96 pixels would also have improved issues regarding resolution and realism. Regrettably, these steps could not be accomplished due to the time constraints.

\section*{Bonuses}
This submission has a total bonus of -8 marks (a penalty), as it used an adversarial training method, is trained only on CIFAR-10, and the Pegasus has a dark body colour.

\appendix
\section{Images with the `horse' and `bird' labels} \label{appendix:A}
\begin{center}
    \includegraphics[width=0.5\textwidth]{figures/birds.png}
\end{center}
\begin{center}
    \includegraphics[width=0.5\textwidth]{figures/horses.png}
\end{center}

\section{Images filtered by the classifier} \label{appendix:B}
\begin{center}
    \includegraphics[width=1\textwidth]{figures/c-birds.png}
\end{center}
\begin{center}
    \includegraphics[width=1\textwidth]{figures/c-horses.png}
\end{center}

\section{Demonstrating mode collapse} \label{appendix:C}
\begin{center}
    \includegraphics[width=0.5\textwidth]{figures/best-batch-0.png}
\end{center}
\begin{center}
    \includegraphics[width=0.5\textwidth]{figures/best-batch-2.png}
\end{center}

% you can have an unlimited number of references (they can go on the 5th page and span many additional pages without any penalty)
\printbibliography
\end{document}


\documentclass{article}
\usepackage{amssymb}
\begin{document}
\section{Filters}
Source: Daniel Ch. von Grueningen, "Digitale Signalverarbeitung", Carl Hanser Verlag 2014, page 251.\\

Apparently, the filter coefficients can be calculted like so
\begin{eqnarray*}
h_{lowpass}[n]&=& \frac{\Omega_u}{\pi}sinc \left(\frac{n\Omega_u}{pi}\right)\\
h_{highpass}[n]&=&-\frac{\Omega_l}{\pi}sinc \left(\frac{n\Omega_l}{pi}\right) \\
h_{bandpass}[n]&=& \frac{\Omega_u}{\pi}sinc \left(\frac{n\Omega_u}{pi}\right)-\frac{\Omega_l}{\pi}sinc \left(\frac{n\Omega_l}{\pi}\right){pi}\\
h_{bandstop}[n]&=& \frac{\Omega_l}{\pi}sinc \left(\frac{n\Omega_l}{pi}\right)-\frac{\Omega_u}{\pi}sinc \left(\frac{n\Omega_u}{\pi}\right){pi}\\
\end{eqnarray*}
With
\begin{eqnarray*}
h_{lowpass}[0]&=&\frac{\Omega_u}{\pi}\\
h_{highpass}[0]&=&1-\frac{\Omega_u}{\pi}\\
h_{bandpass}[0]&=&\frac{\Omega_u-\Omega_l}{\pi}\\
h_{bandstop}[0]&=&1-\frac{\Omega_u-\Omega_l}{\pi}\\
\end{eqnarray*}
\section{Windows}
\begin{tabular}{l|c}
Hamming&$w_{Hamm}[n]=\left\{\begin{array}{l@{:}l}c\cdot\left( 0.54+0.46\cos\left(\frac{2\pi n}{N+1}\right)\right)&|n|\leqslant N/2\\0&otherwise\\\end{array}\right.$\\\hline  %% }
\end{tabular}
\end{document}


\section{Matrix Element Analysis Technique}

\subsection{Introduction}
\label{me_intro}
The matrix element analysis uses leading order matrix elements to separate single top quark events from W+jets and $\ttbar$ background events. The matrix element analysis calculates a probability for an event to be a signal top event, either s-channel or t-channel, and a probability for one of several W+jets backgrounds. This probability is defined in equation ~\ref{prob}.

\begin{equation}
\label{prob}
P(\vec{x}) = \frac{1}{\sigma} \sum_{i,j} \int f_{i}(x_{1}, Q^{2})dx_{1}
\times
f_{j}(x_{2}, Q^{2})dx_{2} \times \pderiv{\sigma_{hs,ij}(\vec{y})}{\vec{y}} \times W(\vec{x},\vec{y})d\vec{y}
\end{equation}

where $f(x_{i},Q^{2})$ is the parton distribution function for parton i,  $\pderiv{\sigma_{hs,ij}(\vec{y})}{\vec{y}}$ is the differential cross section for the hard scatter interaction, and $W(\vec{x},\vec{y})$ is the probability that final state $\vec{y}$ is reconstructed as $\vec{x}$ in the detector, and finally, $\sigma$ is an overall normalization constant defined as the integral of the differential cross section over the entire detector phase space, $d\vec{x}$ as shown in equation ~\ref{prob2}.
\begin{equation}
\label{prob2}
\sigma = \sum_{i,j} \int f_{i}(x_{1}, Q^{2})dx_{1}
\times
f_{j}(x_{2}, Q^{2})dx_{2} \times \pderiv{\sigma_{hs,ij}(\vec{y})}{\vec{y}} \times W(\vec{x},\vec{y})d\vec{y}d\vec{x}
\end{equation}

This analysis uses CTEQ 6.1 LO parton distribution functions accessed via
LHAPDF C++ wrapper~\footnote{http://hepforge.cedar.ac.uk/lhapdf/}. The matrix elements are calculated using the MadGraph leading order
matrix element generator.

\subsection{Jet-Parton Assignment}
The event probabilities calculated in section ~\ref{me_intro} assume a particle assignment of a jet to a parton in the final state. In practice, this assignment is not known and we must sum over all possible assignments to ensure the correct one is chosen.
For two jet events, the event probability is then re-written as a sum over assignment possibilities as shown in equation ~\ref{assign}

\begin{eqnarray}
\label{assign}
\nonumber
P(\ell, \nu, \vec{\rm{jets}}) = \alpha_{1 \rightarrow 1;2 \rightarrow 2} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{1}, j_{2} \rightarrow p_{2}) + \\
\alpha_{1 \rightarrow 2; 2 \rightarrow 1} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{2}, j_{2} \rightarrow p_{1})
\end{eqnarray}

where the coefficients, $\alpha$, are free parameters that depend on the jet-parton assignment probability. Determination of these parameters is described in appendix ~\ref{jpassign}. For three jet events, the event probability is defined as

\begin{eqnarray}
\label{assign}
\nonumber
P(\ell, \nu, \vec{\rm{jets}}) =
\alpha_{1 \rightarrow 1;2 \rightarrow 2; 3 \rightarrow 3} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{1}, j_{2} \rightarrow p_{2}, j_{3} \rightarrow p_{3}) + \\
\nonumber
\alpha_{1 \rightarrow 1;2 \rightarrow 3; 3 \rightarrow 2} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{1}, j_{2} \rightarrow p_{3}, j_{3} \rightarrow p_{2}) + \\
\nonumber
\alpha_{1 \rightarrow 2;2 \rightarrow 1; 3 \rightarrow 3} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{2}, j_{2} \rightarrow p_{1}, j_{3} \rightarrow p_{3}) + \\
\nonumber
\alpha_{1 \rightarrow 2;2 \rightarrow 3; 3 \rightarrow 1} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{2}, j_{2} \rightarrow p_{3}, j_{3} \rightarrow p_{1}) + \\
\nonumber
\alpha_{1 \rightarrow 3;2 \rightarrow 2; 3 \rightarrow 1} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{3}, j_{2} \rightarrow p_{2}, j_{3} \rightarrow p_{1}) + \\
\alpha_{1 \rightarrow 3;2 \rightarrow 1; 3 \rightarrow 2} \times P(\ell, \nu, \rm{j}_{1} \rightarrow p_{3}, j_{2} \rightarrow p_{1}, j_{3} \rightarrow p_{2})
\end{eqnarray}

\subsection{Matrix Element Analysis Output}
The matrix element analysis is named so because the differential cross section for the hard scatter interaction is proportional to the leading order matrix element as shown in equation ~\ref{me}

\begin{equation}
\label{me}
d\sigma_{hs} = \frac{(2\pi)^4}{4} \frac{{|\cal M|}^{2}}{\sqrt{(q_{1}q_{2})^2 - m_{1}^2
m_{2}^2}} d\Phi_{n}(y)
\end{equation}

For this analysis, we consider four probabilities for two jet events and and three probabilities for three jet events. For two jet events events, we calculate the probability for s-channel single top ($u\bar{d}$ $\rightarrow$ $t\bar{b}$), t-channel single top ($ub$ $\rightarrow$ $td$), Wbb production ($u\bar{d}$ $\rightarrow$ $Wb\bar{b}$), and Wcg production ($sg$ $\rightarrow$ $W\bar{c}g$). The leading order diagrams for these channels are shown in figure ~\ref{2jets}.

\vspace{0.1in}
\begin{figure}[!h!tbp]
\begin{center}
\subfigure[]{
	\includegraphics[width=0.22\textwidth]{figures/tb.eps}
}
\subfigure[]{
	\includegraphics[width=0.22\textwidth]{figures/tq.eps}
}
\subfigure[]{
	\includegraphics[width=0.22\textwidth]{figures/wbb.eps}
}
\subfigure[]{
	\includegraphics[width=0.22\textwidth]{figures/wcg.eps}
}
\end{center}
\vspace{-0.1in}
\caption[2jets]{The leading order matrix elements used for event probability calculation for events with two jets. (a) $u\bar{d}$ $\rightarrow$ $t\bar{b}$, (b) $ub$ $\rightarrow$ $td$, (c) $u\bar{d}$ $\rightarrow$ $Wb\bar{b}$, (d) $sg$ $\rightarrow$ $W\bar{c}g$}
\label{2jets}
\end{figure}

For events with three jets, we use three matrix elements, s-channel with gluon radiation ($u\bar{d}$ $\rightarrow$ $t\bar{b}g$), t-channel with associated b quark ($ug$ $\rightarrow$ $t\bar{b}d$), and Wbbg ($u\bar{d}$ $\rightarrow$ $Wb\bar{b}g$). The leading order Feynman diagrams are shown in figure ~\ref{3jets}.

\vspace{0.1in}
\begin{figure}[!h!tbp]
\begin{center}
\subfigure[]{
	\includegraphics[width=0.22\textwidth]{figures/tbg.eps}
}
\subfigure[]{
	\includegraphics[width=0.22\textwidth]{figures/tqb.eps}
}
\subfigure[]{
	\includegraphics[width=0.22\textwidth]{figures/wbbg.eps}
}
\end{center}
\vspace{-0.1in}
\caption[3jets]{The leading order matrix elements used for event probability calculation for events with two jets.  (a) $u\bar{d}$ $\rightarrow$ $t\bar{b}g$, (b) $ug$ $\rightarrow$ $t\bar{b}d$, (c) $u\bar{d}$ $\rightarrow$ $Wb\bar{b}g$}
\label{3jets}
\end{figure}

After calculating event probabilities we combine them into a discriminant output variable, D, defined in equation ~\ref{disc}

\begin{equation}
\label{disc}
D(\vec{x}) = \frac{P_{\rm{signal}}(\vec{x})}{P_{\rm{signal}}(\vec{x}) + P_{\rm{background}}(\vec{x})}
\end{equation}

where $c_{wbb}$ and $c_{wcg}$ are, in principle, the relative background fractions for each background in the data; however, these background fractions are optimizable for the analysis.

For events with two jets the discriminant for either s-channel or t-channel as signal is defined as

\begin{equation}
D(\vec{x})_{\rm{s|t}} = \frac{P_{\rm{s|t}}(\vec{x})}{P_{\rm{st}}(\vec{x}) + c_{wbb}P_{\rm{wbb}}(\vec{x}) + c_{wcg}P_{\rm{wcg}}(\vec{x})}
\end{equation}

and events with three jets the discriminant for s-channel and t-channel is defined as

\begin{equation}
D(\vec{x})_{\rm{s|t}} = \frac{P_{\rm{s|t}}(\vec{x})}{P_{\rm{st}}(\vec{x}) + P_{\rm{wbbg}}(\vec{x})}
\end{equation}


\section{The long exact sequence of homotopy groups}
\sectionmark{The long exact sequence}

\subsection{The long exact sequence}

\begin{defn}
A fiber sequence $F\hookrightarrow E \twoheadrightarrow B$ consists of:
\begin{enumerate}
\item Pointed types $F$, $E$, and $B$, with base points $x_0$, $y_0$, and $b_0$ respectively,
\item Base point preserving maps $i:F\to_\ast E$ and $p:E\to_\ast B$, with $\alpha:i(x_0)=y_0$ and $\beta:p(y_0)=b_0$,
\item A pointed homotopy $H:\mathsf{const}_{b_0}\htpy_\ast p\circ_\ast i$ witnessing that the square
\begin{equation*}
\begin{tikzcd}
F \arrow[r,"i"] \arrow[d] & E \arrow[d,"p"] \\
\unit \arrow[r,swap,"\mathsf{const}_{b_0}"] & B,
\end{tikzcd}
\end{equation*}
commutes and is a pullback square.
\end{enumerate}
\end{defn}

\begin{lem}
Any fiber sequence $F\hookrightarrow E\twoheadrightarrow B$ induces a sequence of pointed maps
\begin{equation*}
\begin{tikzcd}
\loopspace{F} \arrow[r,"\loopspace{i}"] & \loopspace{E} \arrow[r,"\loopspace{p}"] & \loopspace{B} \arrow[r,"\partial"] & F \arrow[r,"i"] & E \arrow[r,"p"] & B,
\end{tikzcd}
\end{equation*}
in which every two consecutive maps form a fiber sequence.
\end{lem}

\begin{proof}
By taking pullback squares repeatedly, we obtain the diagram
\begin{equation*}
\begin{gathered}[b]
\begin{tikzcd}[column sep=large]
\loopspace{F} \arrow[d,swap,"\loopspace{i}"] \arrow[r] & \unit \arrow[d,"\mathsf{const}_{\refl{b_0}}"] \\
\loopspace{E} \arrow[r,"\loopspace{p}"] \arrow[d] & \loopspace{B} \arrow[r] \arrow[d,swap,"\partial"] & \unit \arrow[d,"\mathsf{const}_{y_0}"] \\
\unit \arrow[r,swap,"\mathsf{const}_{x_0}"] & F \arrow[r,"i"] \arrow[d] & E \arrow[d,"p"] \\
& \unit \arrow[r,swap,"\mathsf{const}_{b_0}"] & B.
\end{tikzcd}\\[-\dp\strutbox]
\end{gathered}\qedhere
\end{equation*}
\end{proof}

\begin{defn}
We say that a consecutive pair of pointed maps between pointed sets
\begin{equation*}
\begin{tikzcd}
A \arrow[r,"f"] & B \arrow[r,"g"] & C
\end{tikzcd}
\end{equation*}
is \define{exact} at $B$ if we have
\begin{equation*}
\Big(\exis{a:A}f(a)=b\Big)\leftrightarrow (g(b)=c)
\end{equation*}
for any $b:B$.
\end{defn}

\begin{rmk}
If a pair of consecutive pointed maps between pointed sets
\begin{equation*}
\begin{tikzcd}
A \arrow[r,"f"] & B \arrow[r,"g"] & C
\end{tikzcd}
\end{equation*}
is exact at $B$, it directly that $\im(f)=\fib{g}{c}$. Indeed, such a pair of pointed maps is exact at $B$ if and only if there is an equivalence $e:\im(f)\eqvsym \fib{g}{c}$ such that the triangle
\begin{equation*}
\begin{tikzcd}[column sep=tiny]
\im(f) \arrow[dr] \arrow[rr,"e"] & & \fib{g}{c} \arrow[dl] \\
& B
\end{tikzcd}
\end{equation*}
commutes. In other words, $\im(f)$ and $\fib{g}{c}$ are equal \emph{as subsets of $B$}.
\end{rmk}

\begin{lem}
Suppose $F\hookrightarrow E \twoheadrightarrow B$ is a fiber sequence. Then the sequence
\begin{equation*}
\begin{tikzcd}
\trunc{0}{F} \arrow[r,"\trunc{0}{i}"] & \trunc{0}{E} \arrow[r,"\trunc{0}{p}"] & \trunc{0}{B}
\end{tikzcd}
\end{equation*}
is exact at $\trunc{0}{E}$.
\end{lem}

\begin{proof}
To show that the image $\im\trunc{0}{i}$ is the fiber $\fib{\trunc{0}{p}}{\tproj{0}{b_0}}$, it suffices to construct a fiberwise equivalence
\begin{equation*}
\prd{x:\trunc{0}{E}} \trunc{-1}{\fib{\trunc{0}{i}}{x}} \eqvsym \trunc{0}{p}(x)=\tproj{0}{b_0}.
\end{equation*}
By the universal property of $0$-truncation it suffices to show that
\begin{equation*}
\prd{x:E} \trunc{-1}{\fib{\trunc{0}{i}}{\tproj{0}{x}}} \eqvsym \trunc{0}{p}(\tproj{0}{x})=\tproj{0}{b_0}.
\end{equation*}
First we note that
\begin{align*}
\trunc{0}{p}(\tproj{0}{x})=\tproj{0}{b_0} & \eqvsym \tproj{0}{p(x)} = \tproj{0}{b_0} \\
& \eqvsym \trunc{-1}{p(x)=b_0}.
\end{align*}
Next, we note that
\begin{align*}
\fib{\trunc{0}{i}}{\tproj{0}{x}} & \eqvsym \sm{y:\trunc{0}{F}}\trunc{0}{i}(y)=\tproj{0}{x} \\
& \eqvsym \trunc{0}{\sm{y:F}\trunc{0}{i}(\tproj{0}{y})=\tproj{0}{x}} \\
& \eqvsym \trunc{0}{\sm{y:F}\tproj{0}{i(y)}=\tproj{0}{x}} \\
& \eqvsym \trunc{0}{\sm{y:F}\trunc{-1}{i(y)=x}}.
\end{align*}
Therefore it follows that
\begin{align*}
\trunc{-1}{\fib{\trunc{0}{i}}{\tproj{0}{x}}} & \eqvsym \trunc{-1}{\sm{y:F}\trunc{-1}{i(y)=x}} \\
& \eqvsym \trunc{-1}{\sm{y:F}i(y)=x} \\
\end{align*}
Now it suffices to show that $\eqv{\big(\sm{y:F}i(y)=x\big)}{p(x)=b_0}$. This follows by the pasting lemma of pullbacks
\begin{equation*}
\begin{tikzcd}
(p(x)=b_0) \arrow[r] \arrow[d] & \unit \arrow[d] \\
F \arrow[r] \arrow[d] & E \arrow[d] \\
\unit \arrow[r] & B
\end{tikzcd}
\end{equation*}
\end{proof}

\begin{thm}
Any fiber sequence $F\hookrightarrow E\twoheadrightarrow B$ induces a long exact sequence on homotopy groups
\begin{equation*}
\begin{tikzcd}
& & \cdots \arrow[out=355,in=175,dll] \\
\pi_n(F) \arrow[r,"\pi_n(i)"] & \pi_n(E) \arrow[r,"\pi_n(p)"] & \pi_n(B) \arrow[out=355,in=175,dll,densely dotted] \\
\pi_1(F) \arrow[r,"\pi_1(i)"] & \pi_1(E) \arrow[r,"\pi_1(p)"] & \pi_1(B) \arrow[out=355,in=175,dll] \\
\pi_0(F) \arrow[r,"\pi_0(i)"] & \pi_0(E) \arrow[r,"\pi_0(p)"] & \pi_0(B)
\end{tikzcd}
\end{equation*}
\end{thm}

\subsection{The Hopf fibration}
Our goal in this section is to construct the Hopf fibration, i.e.~a fiber sequence
\begin{equation*}
\sphere{1}\hookrightarrow\sphere{3}\twoheadrightarrow\sphere{2}.
\end{equation*}
This fiber sequence involves the complex multiplication of the unit sphere in the complex number, which is a circle. Viewing the circle as a subspace of the complex numbers, we write $1$ for the base point of the circle.

\begin{defn}
We define the \define{complex multiplication} operation
\begin{equation*}
\mu_{\mathbb{C}}:\sphere{1}\to(\sphere{1}\to\sphere{1}).
\end{equation*}
\end{defn}

\begin{constr}
By the universal property of the circle, it is equivalent to define
\begin{align*}
\mu_{\mathbb{C}}(1) & : \sphere{1}\to\sphere{1} \\
\ap{\mu_{\mathbb{C}}}{\lloop} & : \mu_{\mathbb{C}}(1)=\mu_{\mathbb{C}}(1).
\end{align*}
The function $\mu_{\mathbb{C}}(1)$ is multiplication by $1$, which is the identity function. The type of $\ap{\mu_{\mathbb{C}}}{\lloop}$ is equivalent to the type of homotopies
\begin{equation*}
\idfunc[\sphere{1}] \htpy \idfunc[\sphere{1}].
\end{equation*}
We construct this homotopy by induction on $\sphere{1}$. Therefore it suffices to construct
\begin{align*}
p & : 1=1 \\
q & : \mathsf{tr}_{L}(\lloop,p)=p
\end{align*}
\end{constr}

\begin{lem}
The complex multiplication operation $\mu_{\mathbb{C}}$ on the circle satisfies the unit laws
\begin{align*}
\mathsf{left\usc{}unit}_{\mathbb{C}}(x) & : \mu_{\mathbb{C}}(1,x) = x \\
\mathsf{right\usc{}unit}_{\mathbb{C}}(x) & : \mu_{\mathbb{C}}(x,1) = x \\
\mathsf{coh\usc{}unit}_{\mathbb{C}} & : \mathsf{left\usc{}unit}_{\mathbb{C}}(1)=\mathsf{right\usc{}unit}_{\mathbb{C}}(1),
\end{align*}
and the functions $\mu_{\mathbb{C}}(x,\blank)$ and $\mu_{\mathbb{C}}(\blank,y)$ are equivalences for each $x:\sphere{1}$ and $y:\sphere{1}$, respectively.
\end{lem}

\begin{lem}
Both commuting squares in the diagram
\begin{equation*}
\begin{tikzcd}
\sphere{1} \arrow[d] & \sphere{1}\times\sphere{1} \arrow[d,swap,"\mu_{\mathbb{C}}"] \arrow[l,swap,"\proj 1"] \arrow[r,"\proj 2"] & \sphere{1} \arrow[d] \\
\unit & \sphere{1} \arrow[l] \arrow[r] & \unit
\end{tikzcd}
\end{equation*}
are pullback squares.
\end{lem}

\begin{cor}
There is a fiber sequence
\begin{equation*}
\sphere{1}\hookrightarrow \join{\sphere{1}}{\sphere{1}} \twoheadrightarrow \sphere{2}.
\end{equation*}
\end{cor}

\begin{lem}
The join operation is associative
\end{lem}

\begin{proof}
\begin{equation*}
\begin{tikzcd}
A & A\times C \arrow[l] \arrow[r] & A\times C \\
A\times B \arrow[u] \arrow[d] & A\times B \times C \arrow[r] \arrow[d] \arrow[l] \arrow[u] & A\times C \arrow[u] \arrow[d] \\
B & B\times C \arrow[l] \arrow[r] & C
\end{tikzcd}
\end{equation*}
\end{proof}

\begin{cor}
There is an equivalence $\eqv{\join{\sphere{1}}{\sphere{1}}}{\sphere{3}}$.
\end{cor}

\begin{thm}
There is a fiber sequence $\sphere{1}\hookrightarrow\sphere{3}\twoheadrightarrow\sphere{2}$.
\end{thm}

\begin{lem}
Suppose $f:G\to H$ is a group homomorphism, such that the sequence
\begin{equation*}
\begin{tikzcd}
0 \arrow[r] & G \arrow[r,"f"] & H \arrow[r] & 0
\end{tikzcd}
\end{equation*}
is exact at $G$ and $H$, where we write $0$ for the trivial group consisting of just the unit element. Then $f$ is a group isomorphism.
\end{lem}

\begin{cor}
We have $\pi_2(\sphere{2})=\Z$, and for $k>2$ we have $\pi_k(\sphere{2})=\pi_k(\sphere{3})$.
\end{cor}

\begin{exercises}
\item Give the $0$-sphere $\sphere{0}$ the structure of an H-space.
\item For any pointed type $A$, give $\loopspace{A}$ the structure of an H-space.
\item Show that the type of (small) fiber sequences is equivalent to the type of quadruples $(B,P,b_0,x_0)$, consisting of
\begin{align*}
B & : \UU \\
P & : B \to \UU \\
b_0 & : B \\
x_0 & : P(b_0).
\end{align*}
\end{exercises}


%---------------------------Edge Ratio-----------------------------
\section{Edge Ratio}

The edge ratio of a quadrilateral is the ratio of its longest and shortest edge lengths:
\[
  q = \frac{L_{\max}}{L_{\min}}.
\]

\quadmetrictable{edge ratio}%
{$1$}%                                      Dimension
{$[1,1.3]$}%                                Acceptable range
{$[1,DBL\_MAX]$}%                           Normal range
{$[1,DBL\_MAX]$}%                           Full range
{$1$}%                                      Square
{\cite{pebay:04}}%                          Citation
{v\_quad\_edge\_ratio}%                     Verdict function name


\documentclass[12pt]{article} \usepackage{amsmath,amssymb,amsthm}
\usepackage[margin=1in]{geometry} \title{A Naive Bayes Primer / Naive Bayes for
  Text Categorization} \date{November 14, 2016} \author{Nelson Liu\\
  nfliu@uw.edu \and Jonathan Lee\\ jlee27@uw.edu}
\newtheorem{theorem}{Theorem}[section] \newtheorem*{remark}{Remark}

\begin{document}
\maketitle
\section{Motivating Problem}
Consider the following (hypothetical) problem:
\newline

\textit{In our dataset, Hillary Clinton has tweeted 4190 times and Donald Trump
  has tweeted 4246 times. 30\% of Clinton tweets contain the word "climate",
  while only 15\% of Trump tweets contain the word "climate". What is the
  probability that a tweet is written by Trump or Clinton, given that it
  contains the word "climate"?}
\newline

% Let $S$ be the event that a given email is spam, and let $V$ be the event that the email contains the word "viagra". We would like to know $\mathbb{P}(S|V)$, the probability that an email is spam given that it contains "viagra". A straightforward application of Bayes' theorem tells us that

We like to assume that our dataset is fairly accurate sample of the population,
so we'll say that the numbers of tweets we have for each person is an accurate
representation of their tweeting frequency. Let $H$ be the event that a tweet is
written by Hillary Clinton, $T$ be the event that a tweet is written by Donald
Trump. Thus, we can estimate the probability that an arbitrary tweet is written
by Clinton or by Trump from our dataset.

$$\mathbb{P}(H) = \frac{\text{\# Hillary tweets}}{\text{\# total tweets}} =
\frac{4190}{4190+4246} \approx 0.497$$
$$\mathbb{P}(T) = \frac{\text{\# Trump tweets}}{\text{\# total tweets}} =
\frac{4246}{4190+4246} \approx 0.503$$

Let $C$ be the event that the tweet contains the word "climate". We want to
calculate $\mathbb{P}(H|C)$, the probability that a tweet is written by Clinton
given that we know that it contains the word "climate". We can calculate this
probability (called the posterior probability) by applying Bayes' Theorem
(presented in the next section, so don't worry if you don't know what this is).

\begin{equation}
\begin{split}
  \mathbb{P}(H|C) &= \frac{\mathbb{P}(C|H)\mathbb{P}(H)}{\mathbb{P}(C|H)\mathbb{P}(H) + \mathbb{P}(C|T)\mathbb{P}(T)}\\
  &= \frac{0.3 \times 0.497}{0.3 \times 0.497 + 0.15 \times 0.503}\\
  &\approx 0.66
\end{split}
\end{equation}

It's pretty simple to calculate this probability when we're only examining one
word. We would like to extend this sort of thinking to classify entire tweets as
either being written by Hillary Clinton or Donald Trump by examining every word
in the tweet. This is where naive bayes classifiers come in.

\section{Derivation of Bayes' Theorem}
This section seeks to give the reader a theoretical understanding of Bayes'
Theorem. It's difficult to understand how Bayes' theorem works without knowing
what conditional probability is, so we'll begin by giving a recap on that.

\subsection{Conditional Probability: A Recap}
Given two events $A$ and $B$, the probability of $A$ happening given that we
know that $B$ happens (denoted $\mathbb{P}(A|B)$) is defined by:

$$\mathbb{P}(A|B) = \frac{\mathbb{P}(AB)}{\mathbb{P}(B)}$$
$\mathbb{P}(A|B)$ is read ``the probability of $A$ given $B$''. This equation
makes intuitive sense; since you are trying to calculate the probability that
$A$ happens and you know that $B$ has happened, you put the probability that $B$
happens in the denominator (this is your ``sample space'') and the probability
that $A$ and $B$ happen in the numerator. Dividing these thus yields the
probability that $A$ occurs given that $B$ occurs.

\begin{remark}
Notation may shift occasionally; note that $\mathbb{P}(AB) = \mathbb{P}(A,B)$
\end{remark}

\subsection{Derivation}
Before we can derive Bayes' theorem, we need a preliminary result, the law of
total probability. We present it without proof for the sake of brevity, the the
proof can be found online quite easily.
\begin{theorem}[Law of Total Probability]
  Let $A_1, \dots, A_n$ represent events that span the entire sample space (that
  is, the possible outcome events are $A_1, \dots, A_n$). For any event B,
  $$\mathbb{P}(B) = \sum \limits_{i=1}^{k} \mathbb{P}(B|A_i)\mathbb{P}(A_i)$$
\end{theorem}

In our derivation of Bayes' Theorem, let's continue considering the events
$A_1, \dots, A_n$ where $\mathbb{P}(A_i) > 0$ for each $i$. With the formula for
conditional probability, we rearrange the terms a bit and see that:
$$\mathbb{P}(A_i B) = \mathbb{P}(B) \times \mathbb{P}(A_i|B)$$
This signifies that the probability of $A_i$ and $B$ occurring is the
probability of $B$ occurring multiplied by the probability of $A_i$ occurring
given that $B$ occurs.

We can similarly rearrange this the other way around:
$$\mathbb{P}(A_iB) = \mathbb{P}(A_i) \times \mathbb{P}(B|A_i)$$
This signifies that the probability of $A_i$ and $B$ occurring is the
probability of $A_i$ occurring multiplied by the probability of $B$ occurring
given that $A_i$ occurs, which is also a reasonable thing to say.

Because the two equations above equal $\mathbb{P}(A_iB)$, it follows that:
$$\mathbb{P}(B) \times \mathbb{P}(A_i|B) = \mathbb{P}(A_i) \times \mathbb{P}(B|A_i)$$

Now, by dividing $\mathbb{P}(B)$ on both sides, we get the ``simple
formulation'' of Bayes' Theorem:
$$\mathbb{P}(A_i|B) = \frac{\mathbb{P}(A_i) \times \mathbb{P}(B|A_i)}{\mathbb{P}(B)}$$
The simple formulation shows us that the probability of $A$ given $B$ is equal
to the probability of $A$ happening multiplied by the probability of $B$ given
$A$, all divided by the probability of $B$. In reality however, the probability
of $B$ is calculated by applying the aforementioned law of total probability.
Thus, we can apply the law of total probability to the ``simple formulation'' to
yield the formulation of Bayes' Theorem that is commonly found in literature
(simply ``Bayes' Theorem'' from here on out).
\begin{theorem}[Bayes' Theorem]
  Let $A_1, \dots, A_k$ represent the events that span the entire sample space
  such that $\mathbb{P}(A_i) > 0$ for each i. If $\mathbb{P}(B) > 0$ then, for
  each $i = 1, \dots, k$,
  $$\mathbb{P}(A_i|B) = \frac{\mathbb{P}(A_i) \times \mathbb{P}(B|A_i)}{\sum \limits_{i=1}^{k} \mathbb{P}(B|A_i)\mathbb{P}(A_i)}$$
\end{theorem}

\begin{remark}
We call $\mathbb{P}(A_i)$ the \textbf{prior probability of $A$} and
$\mathbb{P}(A_i|B)$ the \textbf{posterior probability of $A$}.
\end{remark}
\section{Theory behind Naive Bayes Classifiers}
Let's represent a tweet as the set of distinct words $(x_1, \dots, x_n)$
contained in the text. We are interested in computing

$$\mathbb{P}(H|x_1, \dots, x_n)$$
where $H$ is the event that the tweet is written by Hillary, and $T$ is the
event that the tweet is written by Trump. Note that the probability of $T$ is
equivalent to $H^c$, or the probability that the tweet is not written by Hillary
(since we're only deciding between two classes, a task called binary
classification). By Bayes' theorem, this is equivalent to
\begin{equation}
\begin{split}
  \mathbb{P}(H|x_1, \dots, x_n) &= \frac{\mathbb{P}(H) \times
                                  \mathbb{P}(x_1, \dots, x_n|H)}{\mathbb{P}(x_1, \dots, x_n)}\\
                                &= \frac{\mathbb{P}(H)\mathbb{P}(x_1, \dots, x_n|H)}{\mathbb{P}(x_1, \dots,
                                  x_n)}\\
                                &= \frac{\mathbb{P}(H)\mathbb{P}(x_1, \dots, x_n|H)}{\mathbb{P}(x_1,
                                  \dots, x_n|H)\mathbb{P}(H)+\mathbb{P}(x_1, \dots, x_n|T)\mathbb{P}(T)}
\end{split}
\end{equation}
Ignoring the denominator for a minute, by definition of conditional probability,
$$\mathbb{P}(H)\mathbb{P}(x_1, \dots, x_n|H) = \mathbb{P}(x_1, \dots, x_n, H)$$

To simplify this equation, we need the chain rule of probability. As a brief
recap, the chain rule is a rearrangement of the generalization of conditional
probability formula to $n$ terms, and not just $2$. We can get the ``product
rule'' by rearranging the formula for conditional probability:
$$\mathbb{P}(A,B) = \mathbb{P}(A|B) \mathbb{P}(B)$$

Extending this for three variables, we can see that:
$$\mathbb{P}(A,B,C) = \mathbb{P}(A|B,C) \mathbb{P}(B,C) = \mathbb{P}(A|B,C)
\mathbb{P}(B|C) \mathbb{P}(C)$$

Applying this to $n$ variables leads to the chain rule.
\begin{theorem}[The Chain Rule of Probability]
$$\mathbb{P}(A_1, A_2, \dots, A_n) = \mathbb{P}(A_1| A_2, \dots,
A_n)\mathbb{P}(A_2| A_3, \dots, A_n) \dots \mathbb{P}(A_{n-1}|A_n) \mathbb{P}(A_n)$$
\end{theorem}

Applying the chain rule to decompose the probability equation we had above,
\begin{equation}
\begin{split}
  \mathbb{P}(x_1, \dots, x_n, H) & = \mathbb{P}(x_1 | x_2, \dots, x_n, H) \mathbb{P}(x_2, \dots, x_n, H) \\
                                 & = \mathbb{P}(x_1 | x_2, \dots, x_n, H) \mathbb{P}(x_2 | x_3, \dots, x_n, H) \mathbb{P}(x_3, \dots, x_n, H) \\
                                 & = \dots \\
                                 & = \mathbb{P}(x_1 | x_2, \dots, x_n, H) \mathbb{P}(x_2 | x_3, \dots, x_n, H) \dots   \mathbb{P}(x_{n-1} | x_n, H) \mathbb{P}(x_n | H) \mathbb{P}(H) \\
\end{split}
\end{equation}
This is a correct formula, but it's not a particularly useful one for us! The
problem is that terms like $\mathbb{P}(x_1 | x_2, \dots, x_n, H)$ have so many
conditions, that it's generally not possible to accurately calculate their
probability. \newline

Think about what that term is really trying to calculate, though: what's the
probability that the word $x_1$ occurs in a Hillary tweet, given that
($x_2, \dots x_n$) also occur? Unless there is another Hillary tweet with all of
those words ($x_2, \dots x_n$) in it that you've already looked at (which is
unlikely unless you have absurd amounts of data), there's no way to know that
probability already. We need to simplify the problem! \newline

Naive Bayes "fixes" this by making the assumption that the words in an tweet
(and more generally, features in data) are \textbf{conditionally independent of
  each other, given that we know whether or not the tweet was written by
  Hillary}. In other words, it assumes that knowing that an tweet has the word
"climate" in it, doesn't tell us anything about whether it has the word "change"
in it, \textbf{if we already factor in whether the tweet was written by Hillary
  or not}. This is why it's called Naive Bayes: it naively assumes independence
where it might not actually (and likely doesn't!) exist. \newline

This assumption, of course, is \textbf{not true} in reality: words often appear
together, and English is not just a random walk through the dictionary. But it's
a useful simplification of the problem that can lead to practical results.
\newline

Let's rewrite those chain rule probabilities using the conditional independence
assumptions.
\begin{equation}
\begin{split}
  \mathbb{P}(x_1, \dots, x_n, H) & = \mathbb{P}(x_1 |  H) \mathbb{P}(x_2 | x_3, \dots, x_n, H) \dots   \mathbb{P}(x_{n-1} | x_n, H) \mathbb{P}(x_n | H) \mathbb{P}(H) \\
                                 & \approx \mathbb{P}(x_1 | H) \mathbb{P}(x_2 | H) \dots   \mathbb{P}(x_{n-1} | H) \mathbb{P}(x_n | H) \mathbb{P}(H) \\
                                 & = \mathbb{P}(H)\prod_{i=1}^n \mathbb{P}(x_i | H)
\end{split}
\end{equation}

By a similar argument,

$$\mathbb{P}(x_1, \dots, x_n, T) \approx \mathbb{P}(T)\prod_{i=1}^n \mathbb{P}(x_i | T)$$
Putting it all together,
$$\mathbb{P}(H|x_1, \dots, x_n) \approx \frac{\mathbb{P}(H)\prod\limits_{i=1}^n \mathbb{P}(x_i | H)}{\mathbb{P}(H)\prod\limits_{i=1}^n \mathbb{P}(x_i | H) + \mathbb{P}(T)\prod\limits_{i=1}^n \mathbb{P}(x_i | T)}$$

Thus, we can calculate the probability of a tweet being authored by Hillary if
we know the underlying probability of tweet being a Hillary-tweet, and the
probabilities of seeing a particular word in either a Hillary or Trump tweet!

\section{How Trump-ish is a word?}
We have a way to compute the probability of an tweet being from Hillary Clinton
or Donald Trump using relatively simple terms, but it's not yet clear how to
compute those conditional probabilities. This classifier works by taking a large
number of tweets that have already been hand-labelled as \textit{Hillary} or
\textit{Trump} and uses that data to compute word-Trump (or word-Hillary)
probabilities, by counting the frequency of each word. \newline

Imagine we're given a labelled training set of $2000$ Hillary tweets and $3000$
Trump tweets (so $\mathbb{P}(H) = 0.4$, and $\mathbb{P}(T) = 0.6$). We'd like to
calculate $\mathbb{P}("woman"|H)$ and $\mathbb{P}("woman"|T)$. The easiest thing
to do would be to count how many Hillary tweets have ``woman'' and divide that by
the total number of Hillary tweets (and do the same thing for Trump). So, if
there were 216 Hillary tweets with ``woman'' and 12 Trump tweets with ``woman'',
we'd say
$$\mathbb{P}("woman"|H) = \frac{216}{2000} \approx 11\%$$
$$\mathbb{P}("woman"|T) = \frac{12}{3000} \approx 0.4\%$$

\section{Important Practical / Engineering Considerations}
While the theory is all dandy and fine, there are some problems that we may run
into when actually implementing the classifier. We detail a few of them below,
and how they are solved.
\subsection{Smoothing}
Our theoretical framework and formulation of the solution has the right idea,
but there's a small problem that we have yet to address: what if there's a word
(say, "Pokemon") that we've only ever seen before in Trump tweets, and not
Hillary tweets? In that case, $\mathbb{P}("Pokemon"|H) = 0$, and the entire
Hillary probability will go to zero, because we're multiplying all of the word
probabilities together and we've never seen "Pokemon" in a Hillary tweet before.
Thus, any tweet that we get that has ``Pokemon'' in it would be classified as a
Trump tweet for sure! We would like to be robust to words we haven't seen
before, or at least words we've only seen in one setting. \newline

The solution is to never let any word probabilities be zero, by smoothing them
upwards. \textbf{Instead of starting each word count at 0, start it at 1}. This
way none of the counts will ever have a numerator of 0. This overestimates the
word probability, so we need to \textbf{add 2 to the denominator}. (We add 2
because we're implicitly keeping track of 2 things: the number of tweets that
contain that word, and the number that don't. The sum of those two things should
be in the denominator, and the 2 accounts for starting both the counters at 1.)
Essentially, you're hallucinating that you've seen the novel word at least once
in your train set. \newline

The smoothed word probabilities for the previous example are now
$$\mathbb{P}("woman"|S) = \frac{217}{2002} \approx 11\%$$
$$\mathbb{P}("woman"|H) = \frac{13}{3002} \approx 0.43\%$$
And our estimate for "Pokemon" in Hillary tweets would now be $\frac{1}{2002}$,
instead of $0$. \newline

This technique is called \textbf{Laplacian smoothing} and was used in the 18th
century to estimate the probability that the sun will rise tomorrow!

\subsection{The Naive Bayes Classification algorithm}
\begin{enumerate}
  \item Iterate over the labelled Hillary tweets, and for each word $w$ seen,
  count how many of the Hillary tweets contain $w$. Compute
  $\mathbb{P}(w | H) = \frac{|\text{\# Hillary tweets containing w}| +
    1}{|\text{\# Hillary Tweets}| + 2}$.
\item Compute $\mathbb{P}(w | T)$ the same way for Trump tweets.
\item Compute $\mathbb{P}(H) = \frac{|\text{\# Hillary tweets}|}{|\text{\#
    Hillary tweets}| + |\text{\# Trump tweets}|}$
\item $\mathbb{P}(T) = \frac{|\text{\# Trump tweets}|}{|\text{\# Hillary
    tweets}| + |\text{\# Trump tweets}|}$
\item Iterate over the unlabelled test tweets:
\begin{enumerate}
  \item Create a set $(x_1, \dots, x_n)$ of the distinct words in the tweet.
  Ignore the words that you haven't seen in the labelled training data.
\item Compute
$$\mathbb{P}(H|x_1, \dots, x_n) \approx \frac{\mathbb{P}(H)\prod\limits_{i=1}^n \mathbb{P}(x_i | H)}{\mathbb{P}(H)\prod\limits_{i=1}^n \mathbb{P}(x_i | H) + \mathbb{P}(T)\prod\limits_{i=1}^n \mathbb{P}(x_i | T)}$$
\item If $\mathbb{P}(H|x_1, \dots, x_n) > 0.5$, output "Hillary", else output
"Trump"
\end{enumerate}
\end{enumerate}
\subsection{Avoiding floating point underflow}
Multiplying a bunch of small probabilities together will probably result in
floating point underflow, where the numbers will become too small to represent
and will go to $0$. Instead of calculating
$$\mathbb{P}(H)\prod\limits_{i=1}^n \mathbb{P}(x_i | H)$$
(which may underflow), consider computing the logarithm of this,
$$\log\Big({\mathbb{P}(H)\prod\limits_{i=1}^n \mathbb{P}(x_i | H)}\Big)$$
which can be written equivalently as
$$\log(\mathbb{P}(H))+\sum\limits_{i=1}^n \log(\mathbb{P}(x_i | H))$$
Which will certainly not underflow. Then, realize that if
$$\log(\mathbb{P}(H))+\sum\limits_{i=1}^n \log(\mathbb{P}(x_i | H)) > \log(\mathbb{P}(T))+\sum\limits_{i=1}^n \log(\mathbb{P}(x_i | T))$$
then, because in general $\log(x) > \log(y)$ means $x > y$,
$$\mathbb{P}(H)\prod\limits_{i=1}^n \mathbb{P}(x_i | H) > \mathbb{P}(T)\prod\limits_{i=1}^n \mathbb{P}(x_i | T)$$
and therefore
$$\mathbb{P}(H|x_1, \dots, x_n) > 0.5$$
and thus the tweet should be classified as ``Hillary'' (or as ``Trump'', if the
sum of the logarithms for ``Trump'' was greater).
\end{document}
%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\subsection{Cointegration}

If we have multiple variables, we can explore the order of integration of linear combinations.

If two series have time trends, a linear combination of them could remove this.


\section{Lagrange Multipliers}\label{sec:LagrangeMultipliers}

Many applied max/min problems take the following form:
we want to find an extreme value of a function, like $V=xyz$, subject
to a constraint, like $\ds1=\sqrt{x^2+y^2+z^2}$. Often this can be
done, as we have, by explicitly combining the equations and then
finding critical points. There is another approach that is often
convenient, the method of \dfont{Lagrange multipliers}\index{Lagrange multipliers}.

It is somewhat easier to understand two variable problems, so we begin
with one as an example. Suppose the perimeter of a rectangle is to be
100 units. Find the rectangle with largest area. This is a fairly
straightforward problem from single variable calculus. We write down
the two equations: $A=xy$, $P=100=2x+2y$, solve the second of these
for $y$ (or $x$), substitute into the first, and end up with a
one-variable maximization problem. Let's now think of it differently:
the equation $A=xy$ defines a surface, and the equation $100=2x+2y$
defines a curve (a line, in this case) in the $x$-$y$ plane. If we
graph both of these in the three-dimensional coordinate system, we can
phrase the problem like this: what is the highest point on the surface
above the line? The solution we already understand effectively
produces the equation of the cross-section of the surface above the
line and then treats it as a single variable problem. Instead, imagine
that we draw the level curves (the contour lines) for the surface in
the $x$-$y$ plane, along with the line.

\begin{figure}[H]
%\texonly
\centerline{
\vbox{\beginpicture
\normalgraphs
%\ninepoint
\setcoordinatesystem units <3truecm,3truecm>
\setplotarea x from -1 to 1, y from 0 to 1
\put {\hbox{\epsfxsize6cm\epsfbox{images/lagrange.eps}}} at 0 0
\endpicture}}
%\endtexonly
\caption{Constraint line with contour plot of the surface $xy$.}
\label{fig:lagrange}
\end{figure}

Imagine that the line represents a hiking trail and the contour lines
are, as on a topographic map, the lines of constant altitude. How
could you estimate, based on the graph, the high (or low) points on
the path? As the path crosses contour lines, you know the path must be
increasing or decreasing in elevation. At some point you will see the
path just touch a contour line (tangent to it), and then begin to
cross contours in the opposite order---that point of tangency must be
a maximum or minimum point. If we can identify all such points, we can
then check them to see which gives the maximum and which the minimum
value. As usual, we also need to check boundary points; in this
problem, we know that $x$ and $y$ are positive, so we are interested
in just the portion of the line in the first quadrant, as shown. The
endpoints of the path, the two points on the axes, are not points of
tangency, but they are the two places that the function $xy$ is a
minimum in the first quadrant.

How can we actually make use of this? At the points of tangency that
we seek, the constraint curve (in this case the line) and the level
curve have the same slope---their tangent lines are parallel. This also
means that the constraint curve is perpendicular to the gradient
vector of the function; going a bit further, if we can express the
constraint curve itself as a level curve, then we seek the points at
which the two level curves have parallel gradients.
The curve $100=2x+2y$ can be thought of as a level curve of the
function $2x+2y$; Figure~\ref{fig:lagrange two} shows both sets of
level curves on a single graph. We are interested in those points
where two level curves are tangent---but there are many such points,
in fact an infinite number, as we've only shown a few of the level
curves. All along the line $y=x$ are points at which two level curves
are tangent. While this might seem to be a show-stopper, it is
not.

\begin{figure}[H]
%\texonly
\centerline{
\vbox{\beginpicture
\normalgraphs
%\ninepoint
\setcoordinatesystem units <3truecm,3truecm>
\setplotarea x from -1 to 1, y from 0 to 1
\put {\hbox{\epsfxsize6cm\epsfbox{images/lagrange2.eps}}} at 0 0
\endpicture}}
%\endtexonly
\caption{Contour plots for $2x+2y$ and $xy$.}
\label{fig:lagrange two}
\end{figure}

The gradient of $2x+2y$ is $\langle 2,2\rangle$, and the gradient of
$xy$ is $\langle y,x\rangle$. They are parallel when
$\langle 2,2\rangle=\lambda\langle y,x\rangle$, that is, when
$2=\lambda y$ and $2=\lambda x$. We have two equations in three
unknowns, which typically results in many solutions (as we
expected). A third equation will reduce the number of solutions; the
third equation is the original constraint, $100=2x+2y$. So we have the
following system to solve:
$$2=\lambda y \qquad 2=\lambda x\qquad 100=2x+2y.$$
In the first two equations, $\lambda$ can't be 0, so we may divide by
it to get $x=y=2/\lambda$. Substituting into the third equation we get
\begin{align*}
2{2\over \lambda}+2{2\over \lambda}&=100	\\
{8\over100}&=\lambda	\\
\end{align*}
so $x=y=25$. Note that we are not really interested in the value of
$\lambda$---it is a clever tool, the Lagrange multiplier, introduced
to solve the problem. In many cases, as here, it is easier to find
$\lambda$ than to find everything else without using $\lambda$.

The same method works for functions of three variables, except of
course everything is one dimension higher: the function to be
optimized is a function of three variables and the constraint represents
a surface---for example, the function may represent temperature, and
we may be interested in the maximum temperature on some surface, like
a sphere.
The points we seek are those at which the constraint
surface is tangent to a level surface of the function. Once again, we
consider the constraint surface to be a level surface of some
function, and we look for points at which the two gradients are
parallel, giving us three equations in four unknowns. The constraint
provides a fourth equation.

\begin{example}{Optimization with Constraints}{}
Maximize the function $xyz$ given the constraint $\ds
1=\sqrt{x^2+y^2+z^2}$.
\end{example}
\begin{solution}
%Recall Example~\ref{exa:box diagonal}: the diagonal of a box is 1, we seek to maximize the volume.
The constraint is $\ds
1=\sqrt{x^2+y^2+z^2}$, which is the same as
$1=x^2+y^2+z^2$. The function to maximize is $xyz$. The two gradient
vectors are $\langle 2x,2y,2z\rangle$ and $\langle yz,xz,xy\rangle$,
so the equations to be solved are
\begin{align*}
yz&=2x\lambda	\\
xz&=2y\lambda	\\
xy&=2z\lambda	\\
1&=x^2+y^2+z^2	\\
\end{align*}
If $\lambda=0$ then at least two of $x$, $y$, $z$ must be 0, giving a
volume of 0, which will not be the maximum. If we multiply the first
two equations by $x$ and $y$ respectively, we get
\begin{align*}
xyz&=2x^2\lambda	\\
xyz&=2y^2\lambda	\\
\end{align*}
so $2x^2\lambda=2y^2\lambda$ or $x^2=y^2$; in the same way we can show
$x^2=z^2$. Hence the fourth equation becomes
$1=x^2+x^2+x^2$ or $x=1/\sqrt3$, and so $x=y=z=1/\sqrt3$ gives the
maximum volume. This is of course the same answer we obtained
previously.
\end{solution}

Another possibility is that we have a function of three variables, and
we want to find a maximum or minimum value not on a surface but on a
curve; often the curve is the intersection of two surfaces, so that we
really have two constraint equations, say $g(x,y,z)=c_1$ and
$h(x,y,z)=c_2$. It turns out that at points on the intersection of the
surfaces where $f$ has a maximum or minimum value,
$$\nabla f=\lambda\nabla g+\mu \nabla h.$$
As before, this gives us three equations, one for each component of
the vectors, but now in five unknowns, $x$, $y$, $z$, $\lambda$, and
$\mu$. Since there are two constraint functions, we have a total of
five equations in five unknowns, and so can usually find the solutions
we need.

\begin{example}{Intersection of a Plane with a Cylinder}{}
The plane $x+y-z=1$ intersects the cylinder $x^2+y^2=1$ in an
ellipse. Find the points on the ellipse closest to and farthest from
the origin.
\end{example}
\begin{solution}
We want the extreme values of $f=\sqrt{x^2+y^2+z^2}$ subject to the
constraints  $g=x^2+y^2=1$ and $h=x+y-z=1$. To simplify the algebra,
we may use instead $f=x^2+y^2+z^2$, since this has a maximum or
minimum value at exactly the points at which $\sqrt{x^2+y^2+z^2}$ does.
The gradients are
$$\nabla f =\langle 2x,2y,2z\rangle\qquad
\nabla g = \langle 2x,2y,0\rangle\qquad
\nabla h = \langle 1,1,-1\rangle,$$
so the equations we need to solve are
\begin{align*}
2x&=\lambda 2x+\mu	\\
2y&=\lambda 2y+\mu	\\
2z&=0-\mu	\\
1&=x^2+y^2	\\
1&=x+y-z.	\\
\end{align*}
Subtracting the first two we get
$2y-2x=\lambda(2y-2x)$, so either $\lambda=1$ or $x=y$. If $\lambda=1$
then $\mu=0$, so $z=0$ and the last two equations are
$$1=x^2+y^2\qquad\hbox{and}\qquad 1=x+y.$$
Solving these gives $x=1$, $y=0$, or $x=0$, $y=1$, so the points of
interest are $(1,0,0)$ and $(0,1,0)$, which are both distance 1 from
the origin. If $x=y$, the fourth equation is $2x^2=1$, giving
$x=y=\pm1/\sqrt2$, and from the fifth equation we get
$z=-1\pm\sqrt2$. The distance from the origin to
$(1/\sqrt2,1/\sqrt2,-1+\sqrt2)$ is $\sqrt{4-2\sqrt2}\approx 1.08$ and
the distance from the origin to
$(-1/\sqrt2,-1/\sqrt2,-1-\sqrt2)$ is $\sqrt{4+2\sqrt2}\approx 2.6$.
Thus, the points $(1,0,0)$ and $(0,1,0)$ are closest to the origin and
$(-1/\sqrt2,-1/\sqrt2,-1-\sqrt2)$ is farthest from the origin.
%The Java
%\expandafter\url\expandafter{\liveurl lagrange_two_constraints.html}%
%applet \endurl shows the cylinder, the plane, the four points of
%interest, and the origin.
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:LagrangeMultipliers}}

\begin{enumialphparenastyle}

\begin{ex}
A six-sided rectangular box is to hold $1/2$ cubic meter;
what shape should the box be to minimize surface area?
\begin{sol}
a cube, $\root 3 \of {1/2}\times\root 3 \of {1/2}\times\root 3 \of {1/2}$
\end{sol}
\end{ex}

\begin{ex}
The post office will accept packages whose combined length
and girth are at most 130 inches (girth is the maximum distance around
the package perpendicular to the length). What is the largest volume
that can be sent in a rectangular box?
\begin{sol}
$65/3\cdot 65/3\cdot 130/3=2\cdot 65^3/27$
\end{sol}
\end{ex}

\begin{ex}
The bottom of a rectangular box costs twice as much per unit
area as the sides and top. Find the shape for a given volume that will
minimize cost.
\begin{sol}
It has a square base, and is one and one half times as tall as wide.
If the volume is $V$ the dimensions are $\root 3 \of {2V/3}\times
\root 3 \of {2V/3}\times \root 3\of {9V/4}$.
\end{sol}
\end{ex}

\begin{ex}
Using Lagrange multipliers, find the shortest
distance from the point $(x_0,y_0,z_0)$ to the plane $ax+by+cz=d$.
\begin{sol}
$|ax_0+by_0+cz_0-d|/\sqrt{a^2+b^2+c^2}$
\end{sol}
\end{ex}

\begin{ex}
Find all points on the surface $xy-z^2+1=0$ that are closest
to the origin.
\begin{sol}
$(0,0,1)$, $(0,0,-1)$
\end{sol}
\end{ex}

\begin{ex}
The material for the bottom of an aquarium costs half as
much as the high strength glass for the four sides. Find the shape of
the cheapest aquarium that holds a given volume $V$.
\begin{sol}
$\root 3\of{4V}\times\root 3\of{4V}\times\root 3\of{V/16}$
\end{sol}
\end{ex}

\begin{ex}
The plane $x-y+z=2$ intersects the cylinder $x^2+y^2=4$ in an
ellipse. Find the points on the ellipse closest to and farthest from
the origin.
\begin{sol}
Farthest: $(-\sqrt2,\sqrt2,2+2\sqrt2)$; closest:
$(2,0,0)$, $(0,-2,0)$
\end{sol}
\end{ex}

\begin{ex}
Find three positive numbers whose sum is 48 and whose
product is as large as possible.
\begin{sol}
$x=y=z=16$
\end{sol}
\end{ex}

\begin{ex}
Find all points on the plane $x+y+z = 5$ in the first octant at
which $\ds f(x,y,z) = xy^2z^2$ has a maximum value.
\begin{sol}
$(1,2,2)$
\end{sol}
\end{ex}

\begin{ex}
Find the points on the surface $x^2 -yz = 5$ that are closest to the
origin.
\begin{sol}
$\ds (\sqrt{5},0,0)$, $\ds (-\sqrt{5},0,0)$
\end{sol}
\end{ex}

\begin{ex}
A manufacturer makes two models of an item, standard and deluxe.  It
costs \$40 to manufacture the standard model and \$60 for the deluxe.  A
market research firm estimates that if the standard model is priced at $x$
dollars and the deluxe at $y$ dollars, then the manufacturer will sell
$500(y-x)$ of the standard items and $45,000+500(x-2y)$ of the deluxe each
year.  How should the items be priced to maximize profit?
\begin{sol}
standard \$65, deluxe \$75
\end{sol}
\end{ex}

\begin{ex}
A length of sheet metal is to be made into a
water trough by bending up two sides as shown in
Figure~\ref{fig:trough two}.  Find $x$ and $\phi$ so that the
trapezoid--shaped cross section has maximum area, when
the width of the metal sheet is 27 inches (that is, $2x+y=27$).
\begin{sol}
$x=9$, $\phi=\pi/3$
\end{sol}

\begin{figure}[H]
%\texonly
\centerline{
\vbox{\beginpicture
\normalgraphs
%\ninepoint
\setcoordinatesystem units <1truecm,1truecm>
\setplotarea x from 0 to 5, y from -0.5 to 1
\plot 0 1 1 0 4 0 5 1 /
\put {$x$} [bl] <2pt,2pt> at 0.5 0.5
\put {$x$} [br] <-2pt,2pt> at 4.5 0.5
\put {$y$} [t] <0pt,-3pt> at 2.5 0
\put {$\phi$} at 4.7 0.35
\put {$\phi$} at 0.3 0.35
\circulararc 45 degrees from 4.5 0 center at 4 0
\circulararc -45 degrees from 0.5 0 center at 1 0
\setdashes
\putrule from 4 0 to 5.3 0
\putrule from -0.3 0 to 1 0
\endpicture}}
%\endtexonly
\caption{Cross-section of a trough.}
\label{fig:trough two}
\end{figure}
\end{ex}

\begin{ex}
Find the maximum and minimum values of $f(x,y,z)=6x+3y+2z$ subject
to the constraint $\ds g(x,y,z) = 4x^2+2y^2 + z^2 - 70 = 0$.
\begin{sol}
$35$, $-35$
\end{sol}
\end{ex}

\begin{ex}
Find the maximum and minimum values of $f(x,y)=e^{xy}$ subject
to the constraint $g(x,y) = x^3+y^3 - 16 = 0$.
\begin{sol}
maximum $e^4$, no minimum
\end{sol}
\end{ex}

\begin{ex}
Find the maximum and minimum values of $\ds f(x,y) = xy +
\sqrt{9-x^2-y^2}$ when $\ds x^2+y^2 \leq 9$.
\begin{sol}
$5$, $-9/2$
\end{sol}
\end{ex}

\begin{ex}
Find three real numbers whose sum is 9 and the sum of whose squares
is a small as possible.
\begin{sol}
$3$, $3$, $3$
\end{sol}
\end{ex}

\begin{ex}
Find the dimensions of the closed rectangular box with maximum volume
that can be inscribed in the unit sphere.
\begin{sol}
a cube of side length $\ds 2/\sqrt{3}$
\end{sol}
\end{ex}

\end{enumialphparenastyle}


%================================================
\section{Iris Dataset}
%================================================
	%----------------------------------------------
	\subsection{Problem and Approach}
	%----------------------------------------------
		\begin{frame}
		\frametitle{Iris Data Classification}

		\begin{itemize}
			\item Very small dataset (150 points with 4 features, 3 labels, balanced).
			\item 4 features: sepal length/width (cm), petal length/width (cm)
			\item 3 classes: setosa, versicolor, and virginica
			\item Very simple and computationally easy, which makes it ideal for quick sanity checks.
			\item Analsis Tools Considered:
				\begin{itemize}
					\item PCA Clustering (qualitative)
					\item K Nearest Neighbor Classifier Accuracy (quantitative)
					\item K Nearest Neighbor Confusion Matrix (quantitative)
					\item Persistence Diagrams (qualitative)
					\item Bottleneck Distance Matrix (quantitative)
				\end{itemize}
		\end{itemize}

		\end{frame}

	%----------------------------------------------
	\subsection{Results}
	%----------------------------------------------

		\begin{frame}
		\frametitle{PCA Clustering}

		\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/iris_PCA.png}
				\caption{PCA Clustering of the Iris Dataset. Visually, we can see that the setosa label has high separability from versicolor and virginica, which are much closer after PCA.}
		\end{figure}

		\end{frame}

		\begin{frame}
			\frametitle{K Nearest Neighbors Classifier}

			\begin{itemize}
				\item Pre-processing:
				\begin{itemize}
					\item We split the data into a 30\% train and 70\% test sets and keep the data balanced.
					\item We also shuffle the data.
					\item We scale the data such that it is within $(0,1)$ using a min-max scaler.
				\end{itemize}
				\item The function class we consider is trivial since it is dependent only on the training data given.
				\item We can view this as a weighting function that weights the $k$-nearest points to the input with $\frac{1}{k}$ and all other points $0$.
				\item Classification Accuracy: 83 percent
				\item Training Accuracy: 91 percent
			\end{itemize}
		\end{frame}

		\begin{frame}
			\frametitle{Confusion Matrix for Iris Data}

			\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/confusion_matrix_iris.png}
				\caption{Confusion Matrix of Iris Data. We see that setosa is classified perfectly, while versicolor and virginica have some errors in classification.}
		\end{figure}
		\end{frame}

		\begin{frame}
		\frametitle{Persistence Diagrams of Iris Features: Setosa}

		\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/PD_H0_setosa.png}
				\caption{Persistence Diagram for Setosa}
		\end{figure}
		\end{frame}

		\begin{frame}
		\frametitle{Persistence Diagrams of Iris Features: Versicolor}

		\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/PD_H0_versicolor.png}
				\caption{Persistence Diagram for Versicolor.}
		\end{figure}
		\end{frame}

				\begin{frame}
		\frametitle{Persistence Diagrams of Iris Features: Virginica}

		\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/PD_H0_virginica.png}
				\caption{Persistence Diagram for Virginica.}
		\end{figure}
		\end{frame}

		\begin{frame}
		\frametitle{Pairwise Bottleneck Distances}

		\begin{itemize}
			\item The bottleneck distances for each of the iris varieties:
				\begin{itemize}
					\item With themselves: 0 (control)
					\item Setosa, Virginica: 0.04335675
					\item Setosa, Versicolor: 0.04335675
					\item Virginica, Versicolor: 0.04331252
				\end{itemize}
		\end{itemize}

		A downside to the Bottleneck Distance as a measure is that quantitatively it can be hard to distinguish for simple data what a good distance is.
		\end{frame}