\documentclass{article}
\usepackage{amsmath, amsfonts, graphicx}
\title{Synthesizing neural-network stabilizing controller for learned dynamics model}
\begin{document}
\section{Problem statement}
Assume that we are given dynamics model represented as neural networks
\begin{align}
	\text{discrete time } x[n+1] = f(x[n], u[n]) , u[n]\in\mathcal{U}\\
	\text{continuous time } \dot{x} = f(x) + G(x)u, u\in\mathcal{U}
\end{align}
where $f, G$ contain neural networks which we will describe later, our goal is to synthesize a controller $u=\pi(x)$, also represented by a neural network, and a Lyapunov function $V(x)$, represented by a third neural network, such that we can prove that the closed-loop system is Lyapunov stable. Namely we want to find a controller, such that the closed loop system is Lyapunov exponentially (or asymptotically) stable within a set $\mathcal{S}$, i.e., for the equilibrium state/control $x^*, u^*$, all the states start within $\mathcal{S}$ would eventually converge to equilibrium state.

We assume the set of admissible inputs $\mathcal{U}$ is a box in the input space, namely we have input bounds $u_{min}(i)\leq u(i)\leq u_{max}(i)$ for each dimension of the input.

Notice that we assume the continuous-time system is control affine. We will exploit this property later.

\section{Approach}
We consider the discrete-time and continuous-time system separately. Let's first consider the discrete-time case. In this project, all the neural networks are feed-forward neural network with leaky Relu units, hence the output of the network is a piecewise affine function of the input.
\subsection{Discrete time system}
First we assume that our forward dynamical system is represented by a neural network
\begin{align}
	x[n+1] = \phi_{dyn}(x[n], u[n]) - \phi_{dyn}(x^*, u^*) + x^* \label{eq:discrete_forward_dyn}
\end{align}
where $\phi_{dyn}$ is a feed-forward neural network with (leaky) ReLU activation units. Notice that by construction \eqref{eq:discrete_forward_dyn} guarantees that with $x[n]=x^*, u[n]=u^*$ the next state is still the equilibrium state $x^*$. This neural network $\phi_{dyn}$ is given and fixed.

The Lyapunov function for exponetial stability is
\begin{subequations}
\begin{align}
	V(x) > 0 \;\forall x\neq x^*, V(x^*) = 0\\
	V(x[n+1]) - V(x[n]) \le -\epsilon_2 V(x[n])\\
	x \rightarrow \infty \Rightarrow V(x)\rightarrow \infty
\end{align}
\end{subequations}

Since we will certify the Lyapunov condition through MILP, which cannot handle strict inequality constraint $V(x) > 0$, we consider the following necessary and sufficient condition
\begin{subequations}
\begin{align}
	V(x) \ge \epsilon_1 |R(x-x^*)|_1\\
	V(x[n+1]) - V(x[n]) \le -\epsilon_2V(x[n])
\end{align}
\label{eq:lyapunov_discrete}
\end{subequations}
where $R$ is a matrix with full column rank, $|R(x-x^*)|_1$ is the 1-norm of the vector $R(x-x^*)$.

We design our Lyapunov function as
\begin{align}
	V(x) = \phi_{V, \theta}(x) -\phi_{V, \theta}(x^*) + \lambda|R(x-x^*)|_1 \label{eq:lyapunov}
\end{align}
where $\phi_{V, \theta}$ is a feedforward neural network with (leaky) ReLU activation functions. $\lambda$ is a given positive constant (with $\lambda > \epsilon_1$). The reason to add the term  $\lambda|R(x-x^*)|_1$ to the Lyapunov function \eqref{eq:lyapunov}, is that it is very hard for the neural network $\phi_{V, \theta}$ to attain its minimum at $x^*$ (Since the neural network $\phi_{V, \theta}$ is a piecewise affine function of $x$. If it were to attain minimal at $x^*$, it implies that $x^*$ is the common vertex of all the neighbouring linear pieces, which is almost impossible to satisfy by gradient descent approach. See fig. \ref{fig:lyapunov_add_l1_3} as a visual explanation.) By adding the function $\lambda|R(x-x^*)|_1$ which has its global minimal at $x^*$, it is much easier to make the Lyapunov function to attain global minimal at $x^*$. Also by construction of \eqref{eq:lyapunov} we have $V(x^*) = 0$.
\begin{figure}
	\includegraphics[width=0.8\textwidth]{/home/hongkaidai/Dropbox/talks/pictures/neural_network_controller/lyapunov_add_l1_3.pdf}
	\caption{Adding the term $\lambda |x-x^*|_1$ helps the neural network to attain (local) minimal at $x^*$.}
	\label{fig:lyapunov_add_l1_3}
\end{figure}

Our controller is also represented by a neural network as
\begin{align}
	u[n] = saturate(\phi_{u, \eta}(x[n]) - \phi_{u, \eta}(x^*) + u^*)\label{eq:controller}
\end{align}
where $\phi_{u, \eta}$ is a feedforward neural network with (leaky) ReLU activation functions. The weights/biases of this network is denoted by $\eta$. $saturate$ is the saturation function that clamp the control within the input limits. Again by construction of \eqref{eq:controller}, the control action at the equilibrium state $x^*$ is $u^*$.

We could solve the following two optimization problem as MILP
\begin{align}
	\max_{x} \epsilon_1|R(x-x^*)|_1 - V(x)\\
	\max_{x[n]} V(x[n+1]) - V(x[n]) + \epsilon_2V(x[n])
\end{align}
When the maximal cost of either the function is larger than 0, we find a counter example that violates the Lyapunov condition \eqref{eq:lyapunov_discrete}. Our goal is to find the neural network for controller and Lyapunov function, such that the violation is 0. Namely we solve the following min-max problem.
\begin{align}
	\min_{\theta, \eta, R} \left(\max_{x}\epsilon_1|R(x-x^*)|_1 - V(x) + \max_{x[n]} V(x[n+1]) - V(x[n]) + \epsilon_2V(x[n])\right)
\end{align}
After solving the inner maximization problem, we then compute the gradient of the maximal cost w.r.t $\theta, \eta, R$, and then use gradient descent to minimize the loss.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Continuous-time system}
We assume that the continuous time system forward dynamics is
\begin{align}
	\dot{x} = f(x) + G(x)u
\end{align}
Without loss of generality we can assume that the bounds on the control input $u$ is
\begin{align}
	-\mathbf{1}\le u \le \mathbf{1}
\end{align}

Due to the control-affine property of the continuous-time system, we will show that we can search for a control Lyapunov function, instead of a controller and a Lyapunov function as in the discrete-time case. The control Lyapunov function satisfies
\begin{subequations}
\begin{align}
	V(x) \ge \epsilon_1|R(x-x^*)|_1\\
	\min_{-\mathbf{1}\le u\le \mathbf{1}} \dot{V}\le -\epsilon_2V(x)\label{eq:lyapunov_condition_Vdot_continuous}
\end{align}
\label{eq:lyapunov_condition_continuous}
\end{subequations}

Note that the left-hand side of the condition \eqref{eq:lyapunov_condition_Vdot_continuous} can be rewritten as
\begin{subequations}
\begin{align}
	&\min_{-\mathbf{1}\le u \le \mathbf{1}} \dot{V}\\
	=&\min_{-\mathbf{1}\le u \le\mathbf{1}}\frac{\partial V}{\partial x}(f(x) + G(x)u)\\
	=&\frac{\partial V}{\partial x}f(x) + \min_{-\mathbf{1}\le u \le \mathbf{1}} \frac{\partial V}{\partial x}G(x)u\label{eq:control_lyapunov3}\\
	=&\frac{\partial V}{\partial x}f(x) - \left|\frac{\partial V}{\partial x}G(x)\right|_1\label{eq:control_lyapunov4}
\end{align}
\end{subequations}
From \eqref{eq:control_lyapunov3} to \eqref{eq:control_lyapunov4} we use the fact that $\min_{|x|_{\infty}\le 1} a^Tx = -|a|_1$, namely 1-norm is the \textit{dual norm} of $\infty$-norm.

We represent the control-Lyapunov function through a neural network as
\begin{align}
	V(x) = \phi_{V, \theta}(x) - \phi_{V, \theta}(x^*) + \lambda|R(x-x^*)|_1
\end{align}

And we solve the following two MILPs to either certify the control-Lyapunov condition, or find the counter-examples.
\begin{align}
	\max_{x} \epsilon_1|R(x-x^*)|_1 - V(x)\\
	\max_{x} \underbrace{\frac{\partial V}{\partial x}f(x) - \left|\frac{\partial V}{\partial x}G(x)\right|_1}_{\min_{-\mathbf{1}\le u \le \mathbf{1}} \dot{V}} + \epsilon_2V(x)
\end{align}

We then compute the gradient of each MILP cost w.r.t $\theta, R$, and then use gradient descent on $\theta, R$ to minimize the loss.

\subsubsection{subgradient in control Lyapunov function}
One tricky thing is that when we compute $\dot{V}$ in the control Lyapunov function, it requires the gradient $\frac{\partial V}{\partial x}$. But as we use the (leaky) ReLU unit (and $l_1$ norm), the Lyapunov function $V$ is not differentiable everywhere. Specifically both the leaky ReLU unit and the $l_1$ norm function have kinks at input equal to 0. Hence we will need to think about the subgradient of the Lyapunov function. We denote the set of subgradient at $x$ as $\mathcal{D}_V(x)$.

Our goal is that for each state, there exists a control action $u$, such that $\dot{V} < 0$ for all subgradient. Namely
\begin{align}
	\max_x \min_{-\mathbf{1}\le u \le \mathbf{1}} \max_{d\in\mathcal{D}_V(x)} d^T(f(x) + G(x)u) < 0\label{eq:max_min_max_subgradient}
\end{align}
This max-min-max problem is really hard. The problem would be a lot easier if we could switch the inner minimization and maximization, such that we end up with the following max-min problem
\begin{align}
	\max_{x, d\in\mathcal{D}_V(x)} \min_{-\mathbf{1}\le u \le \mathbf{1}} d^T(f(x)+G(x)u) < 0 \label{eq:max_min_subgradient}
\end{align}
which we know is equivalent to the following condition with only maximization.
\begin{align}
	\max_{x, d\in\mathcal{D}_V(x)}d^Tf(x) - |d^TG(x)|_1 < 0 \label{eq:max_subgradient}
\end{align}
So the question is, what is the condition such that we can switch the inner minimization and maximization from \eqref{eq:max_min_max_subgradient} to \eqref{eq:max_min_subgradient}?

From \textit{mini-max} theorem we know that if the set of subgradient $\mathcal{D}_V(x)$ is a convex compact set, then we can switch the order of the min-max. We will next describe some conditions such that the subgradient set $\mathcal{D}_V(x)$ is a convex compact set.

Note that
\begin{align}
	V(x) = \phi_V(x) - \phi_V(x^*) + \lambda |R(x-x^*)|_1
\end{align}
Hence the subgradient set $\mathcal{D}_V(x)$ can be written as the Minkowski sum $\mathcal{D}_\phi(x) \oplus \mathcal{D}_l(x)$, where $\mathcal{D}_\phi(x)$ is the set of subgradient for the neural-network $\phi_V(x)$, and $\mathcal{D}_l(x)$ is the set of subgradient for the $l_1$ norm function $l(x) = \lambda |R(x-x^*)|_1$. Let's first analyze the subgradient for the $l_1$ norm function. The set $\mathcal{D}_l(x)$ can be described as
\begin{align}
	\mathcal{D}_l(x) = \{\lambda y^TR\}
\end{align}
where the i'th entry of $y$ has the following form
\begin{align}
	y(i)\begin{cases}
		= 1 & \text{ if } e_i^TR(x - x^*) > 0\\
		= -1 & \text{ if }e_i^TR(x - x^*) < 0\\
		\in [-1, 1] &\text { if } e_i^TR(x - x^*) = 0
	\end{cases}
\end{align}
Hence for a fixed $x$, the set $\mathcal{Y} = \{y\}$ is a convex compact set. As a result, $\mathcal{D}_l(x)$, which is a linear transformation of $\mathcal{Y}$, also becomes a convex compact set.

Similarly if we consider the ReLU network. The set of subgradient for each individual ReLU unit is also a convex set. Hence if only one layer of neurons has non-unique subgradient, while all other layers have unique gradient, then the set of subgradient $\mathcal{D}_V(x)$ for the entire neural network is also a convex compact set.

To summarize, a sufficient condition for the subgradient set $\mathcal{D}_V(x)$ to be convex and compact, is that for the given state $x$, the (leaky) ReLU units with input being 0 are all in the same layer of the neural network.

\subsubsection{Computing subgradient}
We use the $l_1$ norm function $|R(x-x^*)|_1$ to denote how to use the subgradient. We will have the terms $\frac{\partial |R(x-x^*)|_1}{\partial x}f(x)$ and $\frac{\partial |R(x-x^*)|_1}{\partial x}G(x)$ when computing $\dot{V}$. If we denote a valid subgradient as $g$, then these two terms are written as $g^TRf, g^TRG$, where the value of $g$ is
\begin{align}
	g_i  \begin{cases}
		=1 \text{ if } R[i, :](x-x^*) > 0\\
		=-1 \text{ if } R[i, :](x-x^*) < 0\\
		\in[-1, 1] \text{ if } R[i, :](x-x^*)=0
	\end{cases}
\end{align}
Notice that when the gradient is unique, then we only need to consider the discrete value $\pm1$ multiplying the continuous value $Rf, RG$, and this product can be captured by mixed-integer linear constraints. But when we have subgradient, then we need to consider the continuous variable in the range $[-1, 1]$ multiplying continuous value $Rf, RG$. This product between continuous variables can't be captured by mixed-integer linear constraints.

We propose two approaches to remedy this
\subsubsection{sampled subgradient}
In the first approach, we only consider a finite number of sampled subgradients. For example if we only consider the sub-gradient $-1, 0, 1$, then we can write the subgradient as
\begin{align}
	g_i = \beta_0*-1+\beta_1 * 0 + \beta_2 * 1, \beta_i \text{ is binary}, \beta_0 + \beta_1 + \beta_2 = 1 \text{ if } R[i, :] (x-x^*)=0
\end{align}
\subsubsection{all subgradients}
In the second approach, we do consider all subgradients. The key thing is to note that when the subgradient $g_i \in [-1, 1]$,  the term $g_i z$ can be replaced by a new term $t$ with the constraint $-|z| \le t \le |z|$ if $z$ is a scalar. It is very important to aggregate anything that multiplies $g_i$ to a single slack variable $z$ when we use this trick.

We will have the term $g^TRf - |g^TRG|_1$. We denote $\bar{f}=Rf, \bar{G}=RG$
\begin{align}
	g^TRf - |g^TRG|_1 = g^T\bar{f} - |g^T\bar{G}|_1\\
	= \sum_{i=1}^{n_x} g_i \bar{f}_i - \sum_{j=1}^{n_u}|\sum_{i=1}^{n_x}g_i \bar{G}_{ij}|
\end{align}

\section{Control barrier function}
We aim at finding a control barrier function $h(x)$ satisfying
\begin{align}
	\forall x\;\sup_{-\mathbf{1}\le u\le\mathbf{1}} \frac{\partial h}{\partial x}(f(x) + G(x)u) \ge -\epsilon h(x)
\end{align}
with $\epsilon > 0$ a small positive constant. This control barrier function certifies that the superlevel set $\{x | h(x) \ge 0\}$ is an invariant set.

We suppose $x^*$ is a safe state, and the set of unsafe state is denoted as $\mathcal{C}_u$. Then if $h(x)$ satisfies
\begin{subequations}
\begin{align}
	h(x^*) > 0\\
	h(x) < 0 \forall x\in\mathcal{C}_u\\
	\forall x\;\sup_{-\mathbf{1}\le u\le\mathbf{1}} \frac{\partial h}{\partial x}(f(x) + G(x)u) \ge -\epsilon h(x)
\end{align}
\end{subequations}
Then we certify that there exists a safe set $\mathcal{C}$ as the super-level set of $h(x)$, $x^*\in\mathcal{C}$ and $\mathcal{C} \cup \mathcal{C}_u=\emptyset$.

We design our control barrier function as
\begin{align}
	h(x) = \phi_h(x) - \phi_h(x^*) + 1
\end{align}
which trivially satisfies the condition $h(x^*) > 0$.

By formulating the verification as MILP, we only verify the barrier function condition on a bounded set of state $\mathcal{B}$. To ensure that the invariant safe set is contained within the verified region $\mathcal{B}$, we need to add another condition
\begin{align}
	h(x) < 0 \forall x \in \partial\mathcal{B}
\end{align}
This condition can also be checked through an MILP.

\end{document}


\documentclass[10pt]{article}
\usepackage[utf8]{inputenc}
\usepackage{xcolor}
\usepackage[T1]{fontenc}
\usepackage{pagecolor}
\usepackage{amssymb}
\usepackage{lmodern}
\usepackage{mathtools, nccmath}
\usepackage{courier}
\usepackage[dvipsnames]{xcolor}

\definecolor{myyellow}{RGB}{225,225,100}
\definecolor{myred}{RGB}{220,100,100}
\definecolor{mygreen}{RGB}{120,225,120}
\definecolor{myblue}{RGB}{100,200,255}
\definecolor{mypurple}{RGB}{200,50,255}
\color{white}
\title{\Huge{Solution to Calculus \#1}}
\author{@all.about.mathematics}

\begin{document}
\large
\maketitle
\pagecolor{black}
\section{Problem}
Evaluate the integral
$$\int_{0}^{\infty}\frac{1}{{\sqrt[\leftroot{-1}\uproot{1}4]{x}}\:(1+x^3)}dx$$
\newpage
\section{Solution}
Consider the generalized integral where $\frac m n \not \in \mathbb{Z}$
$$I(m,n)=\int_{0}^{\infty}\frac{x^{m-1}}{1+x^n}dx$$
Therefore our desired integral is $I(\frac{3}{4},3)$. Let
$$u=\frac{1}{1+x^n}\implies x=\left(\frac{1-u}{u}\right)^\frac{1}{n}\implies dx=-\left(\dfrac{1-u}{u}\right)^{\large\frac1n-1}\ \dfrac{du}{nu^2}$$
Besides, $x=\infty\implies u=0$ and $x=0\implies u=1$. Therefore,
$$I(m,n)=\int_{1}^{0}\left({\left(\frac{1-u}{u}\right)^\frac{1}{n}}\right)^{m-1}\left(-u\left(\dfrac{1-u}{u}\right)^{\large\frac1n-1}\right) \dfrac{du}{nu^2}$$
\begin{equation}
=\frac1n \int_{0}^{1}u{\left(\frac{1-u}{u}\right)^{\frac{m}{n}-1}}{du}=\frac1n\int_0^1u^{1-\frac{m}{n}-1}(1-u)^{\frac{m}{n}-1} du
\end{equation}
Recall the definition of the Beta function and its special property
$$\text{B}(a,b)=\int_0^1t^{a-1}(1-t)^{b-1} dt=\frac{\Gamma(a)\cdot\Gamma(b)}{\Gamma(a+b)}$$
Hence the last integral in $(1)$ becomes
\begin{equation}
\frac{ B\left(1-\frac mn , \frac m n\right)}{n}= \frac 1 n \frac{\Gamma(1-\frac mn)\cdot\Gamma(\frac m n)}{\Gamma(1-\frac mn + \frac m n)} = \frac 1 n\: \Gamma\left(1-\frac mn\right)\cdot\Gamma\left(\frac m n\right)
\end{equation}
Recall the Euler's reflection formula
$$\Gamma(z)\cdot\Gamma(1-z)=\frac{\pi}{\sin(\pi z)}\:\:\:\:\: z\not\in\mathbb{Z}$$
Given that $\frac m n \not\in \mathbb{Z}$, we use the Euler's reflection formula for $(2)$
$$I(m,n)= \frac{\pi}{n \sin(\frac {m\pi}{n} )}$$
Therefore, our desired integral
$$I\left(\frac{3}{4},3\right)= \frac{\pi}{3 \sin \left(\frac {\frac 3 4\pi}{3}\right)}= \frac{\sqrt{2}\pi}{3}$$
\newpage
\section{Method 2}

\end{document}


\lab{Iterative Solvers}{Iterative Solvers}
\label{lab:iter_methods}

\objective{In this lab, we will talk about the benefits to using iterative solvers
and also implement three popular algorithms: 1) Jacobi, 2) Gauss-Seidel, 3)
 Successive Over-Relaxation.}

In introductory linear algebra classes, we solve linear systems of the form
$A\mathbf{x} = \mathbf{b}$ by using the inverse, $\mathbf{x} = A^{-1}\mathbf{b}$.
While this is a simple analytic solution, computing $A^{-1}$ is expensive for
large systems.
In linear algebra classes, we solve linear systems that are no bigger than 4 or 5
dimensions. However in real-world applications, linear systems can sometimes have
tens of thousands of parameters. We will present such a problem in
Problem \ref{prob:application}.

With problems of this size, it is no longer feasible to calculate the inverse of
a matrix. Calculating the inverse of a matrix is a $O(n^3)$ problem.
That means it requires somewhere on the order
of 1 trillion floating point operations (FLOPS) to calculate the inverse of a
$10000 \times 10000$ matrix. Performing such a large number of operations also
introduces a high probability of accumulating floating point errors. For these
reasons, large linear systems are never solved directly.

Luckily, additional methods have been developed to solve systems of this size.
Though finding an exact solution is either very time intensive or unstable,
iterative methods can give us sufficiently close approximations while taking
much less time.
% TODO talk about LU-decomposition a bit and why we don't want to use it in this case.

\section*{Iterative Methods} % ================================================

% TODO use some of the description from V1 Krylov section.
The general idea behind any iterative method is make an initial guess, apply some
easy computations to give you a better approximate answer,
and repeat this process until convergence.

In this lab, we will discuss three different methods used to solve large linear
systems of equations. These are the Jacobi Method, the Gauss-Seidel method, and
Successive Over-Relaxation.

\section*{The Jacobi Method} % ================================================

To explain the Jacobi Method, we will examine a $ 3 \times 3 $ system.
$$
\begin{matrix}
2x_1 &   &      & - & x_3  & = & 3 \\
-x_1 & + & 3x_2 & + & 2x_3 & = & 3 \\
     & + & x_2  & + & 3x_3 & = & -1 \\
\end{matrix}
$$

\begin{comment}
$$
\begin{matrix}
a_{11}x_1 + a_{12}x_2 + a_{13}x_3 = b_1 \\
a_{21}x_1 + a_{22}x_2 + a_{23}x_3 = b_2 \\
a_{31}x_1 + a_{32}x_2 + a_{33}x_3 = b_3 \\
\end{matrix}
$$
\end{comment}

To complete the first iteration of Jacobi Method, we begin with an initial guess
$(x^{(0)}_1, x^{(0)}_2, x^{(0)}_3) = (0,0,0)$. Throughout this lab, we will use
 the notation $x^{(k)}_i$ to denote the $k^{th}$ iteration of the algorithm and
 the $i^{th}$ element of $\mathbf{x}$.

We then compute the next approximation $(x^{(1)}_1, x^{(1)}_2, x^{(1)}_3)$.
For the first step, we take each of the three equations and solve for $x_1$,
$x_2$, and $x_3$, respectively. This gives us the following.

$$
\begin{matrix}
x_1 & = & \frac{1}{2} ( 3 + x_3) \\
x_2 & = & \frac{1}{3} ( 3 + x_1 - 2x_3) \\
x_3 & = & \frac{1}{3} ( -1 - x_2) \\
\end{matrix}
$$
We then use these equations for our approximation of $(x^{(1)}_1, x^{(1)}_2, x^{(1)}_3)$.

$$
\begin{matrix}
x^{(1)}_1 & = & \frac{1}{2} ( 3 + x^{(0)}_3)  & = & \frac{1}{2} (3 + 0)     & = & \frac{3}{2} \\
x^{(1)}_2 & = & \frac{1}{3} ( 3 + x^{(0)}_1 - 2x^{(0)}_3) & = & \frac{1}{3} (3 + 0 - 0) & = & 1 \\
x^{(1)}_3 & = & \frac{1}{3} ( -1 - x^{(0)}_2)       & = & \frac{1}{3} (-1 - 0)    & = & -\frac{1}{3} \\
\end{matrix}
$$
So $\mathbf{x}^{(1)} = (\frac{3}{2}, 1, -\frac{1}{3})$. This concludes one iteration of
the Jacobi method. Computing $\mathbf{x}^{(2)}$ follows similarly.

$$
\begin{matrix}
x^{(2)}_1 & = & \frac{1}{2} ( 3 + x^{(1)}_3)  & = & \frac{1}{2} (3 - \frac{1}{3})     & = & \frac{4}{3} \\
x^{(2)}_2 & = & \frac{1}{3} ( 3 + x^{(1)}_1 - 2x^{(1)}_3) & = & \frac{1}{3} (3 + \frac{3}{2} + \frac{2}{3}) & = &  \frac{31}{18} \\
x^{(2)}_3 & = & \frac{1}{3} ( -1 - x^{(1)}_2)       & = & \frac{1}{3} (-1 - 1)    & = & -\frac{2}{3} \\
\end{matrix}
$$

\begin{comment}
$$
\begin{matrix}
x^{(1)}_1 = ( b_1 - a_{12}x^{(0)}_2 - a_{13}x^{(0)}_3) / a_{11} \\
x^{(1)}_2 = ( b_1 - a_{21}x^{(0)}_1 - a_{23}x^{(0)}_3) / a_{22} \\
x^{(1)}_3 = ( b_1 - a_{31}x^{(0)}_1 - a_{32}x^{(0)}_2) / a_{33} \\
\end{matrix}
$$
\end{comment}

To find $\mathbf{x}^{(k)}$, we repeat the same process of solving for each
$x^{(k)}_i$ until convergence is reached. For this particular problem,
convergence to 8 decimal places is reached in 29 iterations.

\begin{center}
\begin{tabular}{ c|c c c  }
          & $x^{(k)}_1$ & $x^{(k)}_2$ & $x^{(k)}_3$ \\
    \hline
    $x^{(0)}$ & 0 & 0 & 0 \\
    %\hline
    $x^{(1)}$ & 1.5 & 1 & -0.33333 \\
    %\hline
    $x^{(2)}$ & 1.33333333 & 1.72222222 & -0.66666667 \\
    $x^{(3)}$ & 1.16666667 & 1.88888889 & -0.90740741 \\
    $x^{(4)}$ & 1.0462963 & 1.99382716 & -0.96296296 \\
    \vdots    & \vdots    & \vdots     & \vdots     \\
    $x^{(28)}$ & 0.99999999 & 2.00000001 & -0.99999999 \\
    $x^{(29)}$ & 1 & 2 & -1 \\
\end{tabular}
\end{center}

\subsection*{Matrix Representation of Jacobi Method} % ------------------------

Though it may not be immediately obvious, the iterative steps performed above
can be expressed in matrix form. First, we use the fact that
$A = D + L + U$ where,

$$
D = \begin{bmatrix}
a_{11} & 0 & \ldots & 0 \\
0 & a_{22} & \ldots & 0 \\
 \vdots & \vdots & \ddots & \vdots \\
0 & 0 & \ldots & a_{nn} \\
\end{bmatrix}
$$
$$
L = \begin{bmatrix}
0 & 0 & \ldots & 0 \\
a_{21} &  0 & \ldots & 0\\
 \vdots & \ddots & \ddots & \vdots \\
a_{n1} & \ldots & a_{n,n-1} & 0 \\
\end{bmatrix}
$$
$$
U = \begin{bmatrix}
0 & a_{12} & \ldots & a_{1n} \\
0 & 0 & \ddots & \vdots \\
 \vdots & \vdots & \ddots & a_{n-1,n} \\
0 & 0 & \ldots & 0 \\
\end{bmatrix}.
$$

Now we can make substitutions to get,
$$ A\mathbf{x} = \mathbf{b} $$
$$ (D + L + U)\mathbf{x} = \mathbf{b} $$
$$ D\mathbf{x} = \mathbf{b} - (L+U)\mathbf{x} $$
$$ \mathbf{x} = D^{-1}\mathbf{b} - D^{-1}(L+U)\mathbf{x} $$

With this, we can express our algorithm for iteratively updating $\mathbf{x}$ as,

\begin{equation} \label{eq:trad_jacobi}
\mathbf{x}^{(k+1)} = D^{-1}b - D^{-1}(L+U)\mathbf{x}^{(k)}
\end{equation}

At this point, notice that we have $D^{-1}$ in Equation \ref{eq:trad_jacobi}. However, in
this case, it is okay that we are calculating an inverse. Since $D$ is a diagonal
matrix, $D^{-1}$ is very easy to compute. It is easily verifiable that $D^{-1}$
is just,

$$
\begin{bmatrix}
\frac{1}{a_{11}} & 0 & \ldots & 0 \\
0 & \frac{1}{a_{22}} & \ldots & 0 \\
 \vdots & \vdots & \ddots & \vdots \\
0 & 0 & \ldots & \frac{1}{a_{nn}} \\
\end{bmatrix}.
$$

It is common to see the matrix representation of the Jacobi Method as expressed in
Equation \ref{eq:trad_jacobi}.
However, we can manipulate this equation slightly to make it easier to compute
in Python. Since $A = D + L + U$,

$$\mathbf{x}^{(k+1)} = D^{-1}b - D^{-1}(A-D)\mathbf{x}^{(k)} $$

\begin{equation} \label{eq:jacobi}
    \mathbf{x}^{(k+1)} = D^{-1}(b - (A-D)\mathbf{x}^{(k)})
\end{equation}

Notice that for this algorithm to
work, the matrix $A$ must have nonzero diagonal entries. Additionally, with the
help of array broadcasting we can make a few other adjustments that make it even
more efficient. Note that this algorithm will not always converge. We will talk
more about the requirements for convergence in the next section.

\begin{problem} % Implement the Jacobi Method.
Using Equation \ref{eq:jacobi}, implement the Jacobi Method. Avoid using
\li{la.inv}. Your function should accept a matrix \li{A}, a vector \li{b}, and
an optional parameter \li{tol} with a default value of \li{1e-8}. Additionally,
accept an optional parameter \li{maxiters} to help you catch nonconvergent
systems. Set \li{maxiters} to $100$ as default. Your function should return the
solution vector \li{x} as well as a list of all the approximations of \li{x}
at each iteration. Stop iterating when \li{maxiters} is reached or
$||\mathbf{x}^{(k)} - \mathbf{x}^{(k+1)}||_{\infty} < $ \li{tol}. In Python, you
can calculate $||x||_{\infty}$ in two ways.
\begin{lstlisting}
>>> la.norm(x, ord=np.inf)
<<or>>
>>> np.max(np.abs(x))
\end{lstlisting}
For more information about the $\infty$-norm see Chapter ?? of Volume 1.

If \li{maxiters} is hit, return the current solution. Your function should flexible enought
to be able to handle systems of any size. If your
system is non-convergent, you will find that your approximations have increasingly
large entries. Test your function on the $3 \times 3$ linear system above. Then
test your function on a random $3 \times 3$ matrix. Chances are the random matrix
you generate will not converge, i.e. \li{maxiters} will be hit.
We will discuss why in the next section.

\begin{info}
Remember that NumPy arrays are mutable. Therefore, to check convergence, you
will need to have two copies of the array you are testing,
i.e. a ``before'' version and an ``after'' version.
\end{info}
\label{prob:jacobi}
\end{problem}

\subsection*{Convergence of the Jacobi Method} % ------------------------------

\begin{definition}
    A matrix $A \in M_n(\mathbb{R})$ is \emph{strictly diagonally dominant} if
    $|a_{ii}| > \sum_{j \neq i} |a_{ij}|$ for all $i = 1,2,\hdots,n$.
\end{definition}

\begin{theorem}
The Jacobi method converges for a matrix $A$ if it is strictly diagonally
dominant.
\end{theorem}

Although this seems like a strong requirement, most real-world linear systems
are represented by strictly diagonally dominant matrices.
We will discuss a class of problems that satisfy this condition at the end of
this lab.

\begin{problem}
To visualize the speed of convergence of the Jacobi method, plot the error at each
iteration. Use the list of approximations that you returned in Problem \ref{prob:jacobi}
to calculate the errors. The error can be computed by calculating
$||A\mathbf{x}^{(k)} - \mathbf{b}||$
for each approximation, $x^{(k)}$. Your plot should be similar to the figure below.
To plot with a logrithmic scale on the $y$-axis, run \li{plt.semilogy()}.
\end{problem}

\begin{figure} \label{fig:jacobi_convergence}
\includegraphics[width=.7\textwidth]{jacobi_convergence.pdf}
\label{fig:jacobi_convergence}
\end{figure}

\section*{The Gauss-Seidel Method} % ==========================================

The Gauss-Seidel Method is quite similar to the Jacobi Method. We will examine
the same system as before. The main difference between Gauss-Seidel and Jacobi
is that in Gauss-Seidel, any new information is used immediately. As with Jacobi,

$$
\begin{matrix}
x^{(1)}_1 & = & \frac{1}{2} ( 3 + x^{(0)}_3)  & = & \frac{1}{2} (3 + 0)     & = & \frac{3}{2} \\
\end{matrix}
$$

But now, we use this updated value of $x^{(1)}_1$ in our calculation of
$x^{(1)}_2$ as follows
$$
\begin{matrix}
x^{(1)}_2 & = & \frac{1}{3} ( 3 + x^{(1)}_1 - 2x^{(0)}_3) & = & \frac{1}{3} (3 + \frac{3}{2} - 0) & = & \frac{3}{2} \\
\end{matrix}
$$

We now can use the updated values of $x^{(1)}_1$ and $x^{(1)}_2$ in our
calculation of $x^{(1)}_3$,
$$
\begin{matrix}
x^{(1)}_3 & = & \frac{1}{3} ( -1 - x^{(1)}_2)       & = & \frac{1}{3} (-1 - \frac{3}{2})    & = & -\frac{5}{6}
\end{matrix}.
$$

This process of using what we have calculated immediately is called
\emph{forward substitution}. Using forward substitution causes the algorithm to
converge much faster.

\begin{center}
    \begin{tabular} {c | c c c}
        & $x^{(k)}_1$ & $x^{(k)}_2$ & $x^{(k)}_3$ \\
        \hline
          $x^{(0)}$ & 0 & 0 & 0 \\
          %\hline
          $x^{(1)}$ & 1.5 & 1.5 & -0.833333 \\
          %\hline
          $x^{(2)}$ & 1.08333333 & 1.91666667 & -0.97222222 \\
          $x^{(3)}$ & 1.01388889 & 1.98611111 & -0.99537037 \\
          $x^{(4)}$ & 1.00231481 & 1.99768519 & -0.9992284 \\
          \vdots    & \vdots    & \vdots     & \vdots     \\
          $x^{(11)}$ & 1.00000001 & 1.99999999 & -1 \\
          $x^{(12)}$ & 1 & 2 & -1 \\
        \end{tabular}
\end{center}
Notice that Gauss-Seidel converged in less than half as many iterations.

As shown above, Gauss-Seidel updates one element of the solution vector at a time.
This process is more generally described by the following equation.

\begin{equation} \label{eq:gauss_seidel_full}
x^{(k+1)}_i = \frac{1}{a_{ii}} \left (b_i - \sum_{j < i}a_{ij}x^{(k+1)}_j - \sum_{j > i}a_{ij}x^{(k)}_j \right )
\end{equation}

Notice that the two sums closely resemble an inner product without the
$i^{\text{th}}$ term. If we update the entries of the vector $\mathbf{x}$ in place,
this equation can be represented as,

$$
x^{(k+1)}_i = \frac{1}{a_{ii}} \left ( b_i - \left < A_i, \mathbf{x}^{(k)} \right > + a_{ii}x^{(k)}_i \right )
$$
where $A_i$ is the $i^{th}$ row of $A$. This can be simplified further to

\begin{equation} \label{eq:gauss_seidel}
x^{(k+1)}_i = x^{(k)}_i + \frac{1}{a_{ii}} \left ( b_i - \left < A_i, \mathbf{x}^{(k)}\right >\right)
\end{equation}

Making one sweep through all the entries of $x$ completes one iteration. We
continue this same process until convergence is attained.

\begin{problem} \label{prob:gauss_seidel}
Implement the Gauss-Seidel Method using Equation \ref{eq:gauss_seidel}. Your
function should accept a matrix \li{A}, a vector \li{b}, an optional parameter
\li{tol} that defaults to \li{1e-8} and an optional parameter \li{maxiters} that
defaults to $100$. Your function should return the solution vector \li{x} as
well as a list of all the approximations of \li{x} at each iteration. HINT: It is
easiest to implement this algorithm if you update the approximation vector, $x^{(k)}$, in-place.
\end{problem}

\subsection*{Convergence of Gauss-Seidel} % -----------------------------------

\begin{definition}
    A matrix $A \in M_n(\mathbb{R})$ is \emph{positive definite} if all its
    eigenvalues are positive.
\end{definition}

\begin{theorem}
    The Gauss-Seidel method converges for a matrix $A$ if it is strictly
    diagonally dominant or positive definite.
\end{theorem}

For a treatment on the theory of positive definite matrices, see Chapter ?? of
Volume 1 of the textbook.

\begin{problem}
As we have described up to this point in the lab, we can take advantage of the
sparsity of a linear system by using iterative methods like Jacobi and Gauss-Seidel.
For this problem, compare the runtimes between your \li{gauss_seidel()} function and
\li{la.solve()} for a strictly diagonally dominant system of 5000 parameters. Use the following
code to generate a strictly diagonal dominant matrix:
\begin{lstlisting}
def diag_dom(n, num_entries=None):
    """Generate a strictly diagonally dominant nxn matrix.

    Inputs:
        n (int) - dimension.
        vals (list) - range of values for off-diagonal entries.
        num_entries (int) - number of nonzero values. If None, num_entries
                    defaults to n^(1.5) - n.

    Returns:
        A (array) - nxn strictly diagonally dominant matrix.
    """
    if num_entries is None:
        num_entries = int(n**1.5) - n
    A = np.zeros((n,n))
    i = np.random.choice(np.arange(0,n), size=num_entries)
    j = np.random.choice(np.arange(0,n), size=num_entries)
    data = np.random.randint(-4,4,size=num_entries)
    for c in xrange(num_entries):
        A[ i[c], j[c] ] = data[c]
    for i in xrange(n):
        A[i,i] = np.sum(np.abs(A[i,:])) + 1
    return A
\end{lstlisting}
What difference do you see between these two functions? Explain why there is such a big difference.
\end{problem}

\section*{Sparse Systems} % ===================================================

Some linear systems may have tens of thousands of parameters.
However, in applications where systems of this size arise, it is also common
that such matrices are sparse.
The tools in SciPy's \li{sparse} library make it possible to work with even these very large systems.

\begin{comment}
The following is a quick example of how the different types of sparse matrices
can be used to optimize performance.

\begin{lstlisting}
# Initialize the matrix using a coo_matrix.
>>> rows = np.array([0,1,2,3,0,2])
>>> cols = np.array([0,1,2,3,1,0])
>>> values = np.array([3,5,4,1,2,1])
>>> A = spar.coo_matrix((values, (rows,cols)), shape=(4,4))

# To visualize the matrix, use .todense(). Keep in mind doing
#   operations on this dense matrix forfeits all sparse-related
#   optimizations.
>>> print A.todense()
matrix([[3, 2, 0, 0],
        [0, 5, 0, 0],
        [1, 0, 4, 0],
        [0, 0, 0, 1]])

# perform matrix multiplicaton after converting to a csr_matrix.
>>> Acsr = A.tocsr()
>>> x = np.array([1,0,-1,2])
>>> b = Acsr.dot(x)
array([3,0,-3,2])

# access rows of a csr_matrix. The syntax for slicing a csr_matrix
#  (and csc_matrix) is identical to slicing NumPy arrays. For row slicing,
#  use a csr_matrix. For column slicing, use a csc_matrix.
>>> Acsr[0,:].todense()
matrix([[3, 2, 0, 0]])

>>> Acsc = A.tocsc()
>>> Acsc[:,2].todense()
matrix([[0],
        [0],
        [4],
        [0]])

# access individual elements of the matrix after converting to a dok_matrix.
>>> Adok = A.todok()
>>> print Adok[2,0]
1
>>> print Adok[1,1]
5
\end{lstlisting}

Since the \li{coo_matrix} type allows for fast sparse matrix type conversion, it is usually worth the time and memory to convert to the sparse matrix type that corresponds with the operations you are performing.
\end{comment}

\begin{problem} % Gauss-Seidel with Sparse matrices.
To be able to use the Gauss-Seidel method on sparse matrices, you must translate
the code you wrote in Problem \ref{prob:gauss_seidel} from \li{numpy} to \li{scipy.sparse}.
The algorithm is the same, but there are some functions that are named differently
between these two packages.

Write a new function that accepts a sparse matrix \li{A}, a NumPy array \li{b},
an optional parameter \li{tol} that defauls to \li{1e-8}, and an optional parameter
\li{maxiters} that defaults to 100. Your function should return the solution
vector \li{x} as well as a list of all the approximations of \li{x} at each
iteration.

We want this function to run as quickly as possible. The Gauss-Seidel method calls for us to extract rows from the matrix $A$ then multiply by the vector $x$. There are many ways to do this that cause some fairly serious runtime issues, so we will provide the code for this specific portion of the algorithm.

\begin{lstlisting}
# Slice the i-th row of A and dot product the vector x
rowstart = A.indptr[i]
rowend = A.indptr[i+1]
Aix = np.dot(A.data[rowstart:rowend], x[A.indices[rowstart:rowend]])
\end{lstlisting}

To test your function, you can create a strictly diagonally dominant matrix with
the following code. In this code, also note how different types of sparse matrices
have been used to optimize performance.

\begin{lstlisting}
def spar_diag_dom(n, num_entries=None):
    """Generate a strictly diagonally dominant sparse nxn matrix.

    Inputs:
        n (int) - dimension
        num_entries (int) - number of nonzero values. If None, num_entries
                    defaults to n^(1.5) - n.

    Returns:
        A (spar.csr_matrix) - strictly diagonally dominant sparse nxn matrix.
    """
    return spar.csr_matrix(diag_dom(n, num_entries=num_entries))
\end{lstlisting}

\end{problem}

\subsection*{Successive Over-Relaxation (SOR)} % ------------------------------------

There are some systems that meet the requirements for convergence in the
Gauss-Seidel method that do not converge very quickly. A slightly altered version
of the Gauss-Seidel method has been developed that can result in faster convergence.
This is achieved by introducting a relaxation factor, $\omega$. This method is
called Successive Over-Relaxation. In fact, for many slowly converging iterative processes, convergence can be improved by introducing a relaxation factor.

The iterative equation for Gauss-Seidel, Equation \ref{eq:gauss_seidel_full},
with the introduction of a relaxing factor $\omega$ becomes,

$$
x_i^{(k+1)} = (1 - \omega)x_i^{(k)} + \frac{\omega}{a_{ii}} \left (b_i - \sum_{j < i}a_{ij}x^{(k+1)}_j - \sum_{j > i}a_{ij}x^{(k)}_j \right )
$$

Updating the solution vector $x$ inplace as we did with Gauss-Seidel, this
equation becomes,

\begin{equation} \label{eq:sor}
x^{(k+1)}_i = x^{(k)}_i + \frac{\omega}{a_{ii}} \left ( b_i - \left < A_i, \mathbf{x}^{(k)} \right > \right )
\end{equation}

Equation \ref{eq:sor} follows by simplifying the full analytic equation in the
same way we simplified the Gauss-Seidel method. Notice that when $\omega = 1$,
Successive Over-Relaxation is equivalent to Gauss-Seidel.

\begin{problem}
Implement Successive Over-Relaxation. You should be able to write
this function by changing one line of your code from your Gauss Seidel
function. Your function should be able to handle sparse matrices.

We will examine how varying the values of $\omega$ affects convergence in Problem \ref{prob:application}
\end{problem}

\section*{Application: Finite Difference Method} % ======================================================

Throughout this lab, we have said there are sparse linear systems that arise in applications that have tens of thousands of parameters. Systems of this size
often arise in Finite Difference problems. The underlying theory behind such problems will be addressed in Volume 4 of the textbook. We now present one such problem.

\begin{problem} \label{prob:application}

A common equation that occurs in both pure and applied mathematics is Laplace's equation.

\[ \frac{\partial^2 u}{\partial x^2}+ \frac{\partial^2 u}{\partial y^2}= 0.\]

We can use Laplace's Equation to model steady-state heat flow. Imagine we have a square metal plate where the top and bottom sides are fixed at 0 degrees Celcius and the left and right sides are fixed at 100 degrees Celcius. We can use Laplace's Equation to approximate the temperature of the interior of the metal plate at any given point.

For this problem we will solve Laplace's equation numerically using a technique called \textit{the finite difference method}.  The derivation of the finite difference method will be saved until the lab on Numerical Differentiation, but the relevant equations will be given here.

The finite difference method can be reduced to solving a sparse system of linear equations. We will describe how to initialize this linear system, but again, all the derivation will be saved for the lab on Numerical Differentiation.

In Figure \ref{fig:application} we have the basic set up of the problem described above. The metal plate being represented is $10 in \times 10 in$. The points $u_{i,j}$ are the discretized grid points we will use to approximate the temperature across the plate.

Then at an interior point of the grid, Laplace's equation can be approximated by
\begin{equation} \label{eq:laplace}
\begin{split}
    \frac{\partial^2 u_{i,j}}{\partial x^2}+ \frac{\partial^2 u_{i,y}}{\partial y^2} \approx \frac{1}{h^2}\left[-4u_{i,j} + u_{i+1,j} + u_{i-1,j} + u_{i,j+1} +  u_{i,j-1}\right]
\end{split}
\end{equation}
where $h$ is the distance between gridlines.

Writing Equation \ref{eq:laplace} for the $9$ interior points of the grid in Figure \ref{fig:application}result in a $9 \times 9$ system, $A \mathbf{x} = b$.

$$
\begin{bmatrix}
-4 & 1  &  0 &  1 &  0 &  0 &  0 &  0 & 0 \\
 1 & -4 &  1 &  0 &  1 &  0 &  0 &  0 & 0 \\
 0 & 1  & -4 &  0 &  0 &  1 &  0 &  0 & 0 \\
 1 & 0  &  0 & -4 &  1 &  0 &  1 &  0 & 0 \\
 0 & 1  &  0 &  1 & -4 &  1 &  0 &  1 & 0 \\
 0 & 0  &  1 &  0 &  1 & -4 &  0 &  0 & 0 \\
 0 & 0  &  0 &  1 &  0 &  0 & -4 &  1 & 0 \\
 0 & 0  &  0 &  0 &  1 &  0 &  1 & -4 & 1 \\
 0 & 0  &  0 &  0 &  0 &  1 &  0 &  1 & -4 \\
\end{bmatrix}
\begin{bmatrix}
u_{1,1} \\
u_{1,2} \\
u_{1,3} \\
u_{2,1} \\
u_{2,2} \\
u_{2,3} \\
u_{3,1} \\
u_{3,2} \\
u_{3,3} \\
\end{bmatrix}
=
\begin{bmatrix}
-100 \\
 0  \\
-100 \\
-100 \\
 0  \\
-100 \\
-100 \\
 0  \\
-100 \\
\end{bmatrix}
$$
For any positive integer n, the corresponding system $A \mathbf{x} = b$ can be expressed as,
$$
A = \begin{bmatrix}
B_1 & I &      &        & \\
I & B_2 &  I   &        & \\
  & I & \ddots & \ddots & \\
  &   & \ddots & \ddots & I \\
  &   &        &    I   & B_n
\end{bmatrix}
$$
where $B_i$, is an $n \times n$ matrix,
$$
B_i = \begin{bmatrix}
-4 &  1 &      &        & \\
 1 & -4 &  1   &        & \\
   &  1 & \ddots & \ddots & \\
   &    & \ddots & \ddots & 1 \\
   &    &        &    1   & -4
\end{bmatrix}.
$$
So the resulting matrix $A$ ends up being of size $n^2 \times n^2$.

For the vector $b$, all nonzero entries correspond to interior points that touch the left or right boundaries. The vector $b$ will be $n^2 \times 1$.

For this problem, do the following:
\begin{enumerate}
    \item Write a function \li{finite_difference()} that accepts an integer $n$ for the number of interior gridlines. Your function should return the corresponding sparse matrix $A$ and NumPy array $b$. For example, a value of $n = 3$ should result in the matrix $A$ and vector $b$ described above. HINT: Consider using \li{scipy.sparse.block_diag} and the \li{setdiag()} method of scipy sparse matrices for dynamically creating the matrix $A$.
    \item To demonstrate how convergence is affected by the value of $\omega$, time your \li{sparse_sor()} function for values of $\omega = [1, 1.05, 1.1, \dots , 1.9, 1.95 ]$. Plot your timings as a function of $\omega$. Generate the system $A \mathbf{x} = b$ using your \li{finite_difference()} function and a value of $n = 20$. Remember that $\omega = 1$ corresponds to the Gauss-Seidel method. How does Gauss-Seidel compare to SOR with the optimal $\omega$?

    NOTE: Notice that the matrix $A$ is not strictly diagonally dominant. However, this matrix is positive definite, so the algorithm will converge. Unfortunately, convergence for these kinds of systems require many more iterations. When using the \li{sparse_sor()} function in this problem, set \li{maxiters = 10000} and \li{tol=10**-2}.
    \item Using \li{plt.pcolormesh}, create a heatmap to visualize your solution. The solution vector you get from your \li{sparse_sor()} function will need to be reshaped to properly visualize the result.
\end{enumerate}

\begin{comment}
The following code generates a heatmap to help you visualize the steady-state heat flow across the interior of the plate:

\begin{lstlisting}
from matplotlib import pyplot as plt
import numpy as np

n = 50
A,b = finite_difference(n)
x = sparse_sor(A,b,1.9,maxiters=10000,tol=10**-2)[0]
U = x.reshape(n,n)
x,y = np.linspace(0,10,n), np.linspace(0,10,n)
X,Y = np.meshgrid(x,y)

plt.pcolormesh(X,Y,U,cmap='RdBu_r')
plt.show()
\end{lstlisting}
\end{comment}

\end{problem}

%\newpage

%\section*{Additional Material} % ==============================================

\begin{comment} % Migrated from NumPy lab. vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
\begin{algorithm} % Laplace's equation procedure.
\begin{algorithmic}[1]
\Procedure{Jacobi}{$n$, tol}
\State $U \gets$ an $n\times n$ array of zeros.
\State Set the boundaries of $U$.
\State $U^\prime \gets$ a copy of $U$.
\State diff $\gets$ tol
\While {diff $\ge$ tol}
    \State Set all interior points of $U^\prime$ equal to the average of their neighbors.
    \State diff $\gets$ the maximum of the absolute value of $U - U^\prime$.
    \State Interior of $U \gets$ interior of $U^\prime $.
\EndWhile
\State \pseudoli{return} $U$
\EndProcedure
\end{algorithmic}
\caption{The Jacobi method for solving Laplace's equation.}
\label{alg:jacobi}
\end{algorithm}

\begin{problem}
Laplace's equation is used to model steady-state heat flow on a square plate.
The plate can be approximated by a matrix, where each entry of the matrix represents the average temperature over a small square portion of the plate.
Suppose the plate starts at $0^\circ$ Celsius everywhere except on the east and west boundaries, which are held at a constant temperature of $100^\circ$.
In addition, the north and south boundaries are held constant at $0^\circ$.

\[
U = \left[\begin{array}{ccccc}
\textcolor[rgb]{0.,0.,1.}0 & \textcolor[rgb]{0.,0.,1.}0 & \textcolor[rgb]{0.,0.,1.}\cdots & \textcolor[rgb]{0.,0.,1.}0 & \textcolor[rgb]{0.,0.,1.}0\\
\textcolor[rgb]{1.,0.,0.}{100} & 0 & \cdots & 0 & \textcolor[rgb]{1.,0.,0.}{100}\\
\textcolor[rgb]{1.,0.,0.}\vdots & \vdots & \ddots & \vdots & \textcolor[rgb]{1.,0.,0.}\vdots \\
\textcolor[rgb]{1.,0.,0.}{100} & 0 & \cdots & 0 & \textcolor[rgb]{1.,0.,0.}{100}\\
\textcolor[rgb]{0.,0.,1.}0 & \textcolor[rgb]{0.,0.,1.}0 & \textcolor[rgb]{0.,0.,1.}\cdots & \textcolor[rgb]{0.,0.,1.}0 & \textcolor[rgb]{0.,0.,1.}0\\

\end{array}\right]
\]

The non-boundary, interior portion of the matrix (with black text) can be accessed with the slice \li{U[1:-1,1:-1]}.
To find the steady state of the hot plate, set each entry of the interior of $U$ equal to the average of its 4 immediate neighbors (the entries above, below, and to the left and right).
This step should take only \emph{one line} and should be based entirely on array slicing.

Continue updating the interior entries of $U$ until they stop changing significantly.
The entire procedure is summarized in Algorithm \ref{alg:jacobi}.

(Hint: The slice \li{U[:-2,1:-1]} references the upper neighbors of the interior points of $U$ and \li{U[1:-1,2:]} references the right neighbors. How can you reference the lower and left neighbors?)

Use the following code to visualize your results.
%% This does 3D visualization, but it's easier (and better) as a heat map.
% from mpl_toolkits.mplot3d import Axes 3D

%     x, y = np.linspace(0, 1, n), np.linspace(0, 1, n)
%     X, Y = np.meshgrid(x, y)
%     fig = plt.figure()
%     ax = fig.gca(projection='3d')
%     ax.plot_surface(X, Y, U, rstride=5)
%     plt.show()

\begin{lstlisting}
from matplotlib import pyplot as plt

def jacobi(n=100, tol=1e-8):

    # Perform the algorithm, storing the result in the array 'U'.

    # Visualize the results.
    plt.imshow(U)
    plt.show()
\end{lstlisting}

This Jacobi iteration is a \emph{finite difference method}, and is similar to many of the methods that we will use to find numerical solutions to differential equations in Volume IV.
\end{problem}

\begin{figure}[H]
\centering
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{jacobi_small.pdf}
    \caption{$10\times 10$ approximation.}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{jacobi_big.pdf}
    \caption{$100\times 100$ approximation.}
\end{subfigure}
\caption{Hot plates in steady state with different resolutions.}
\end{figure}
\end{comment} % Used to be in NumPy ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


% !TeX spellcheck = en_US
\section{Preliminaries}\label{sec:prelim}
In this section, the algorithms used for solving each problem and the basis criteria for the theoretical comparison between them will be formally introduced.

\subsection{The Davis-Putnam-Logemann-Loveland algorithm (DPLL)}
One technique to solve the satisfiability problem is the Davis-Putnam-Logemann-Loveland (DPLL) algorithm, which is a refinement of the original resolution-based Davis-Putnam (DP) algorithm. DPLL is based on four rules:
\begin{enumerate}
	\item Tautology elimination
	\item One Literal (Unit clause)
	\item Pure Literal
	\item Branching or splitting
\end{enumerate}
For our purposes, just the one literal and the branching rules will be considered as they are sufficient for the completeness and soundness of the algorithm, and none of the other rules affect the efficiency.

%%% Important rules (2+4) quick explanation
\textit{One Literal:} This rule eliminates unit clauses (any clause contains a single unassigned literal) by assigning the only possible value to make this literal true. Thus, no choice is necessary. By applying this rule, a large part of the entire search space can be pruned, leading to a more efficient SAT instance that is equivalent to the original problem.

\textit{Branching:} When no other rule is applicable, branching is used to assign a truth value to a variable. This assignment is non-deterministic and depending on the used branching-heuristics a variable is chosen.

An unsatisfiable SAT problem will lead to apply branching and unit propagation to all branches until no variable is left. For this reason, it is usually harder to detect unsatisfiability, since all branches must be traversed.

\subsection{Arc consistency and the forward checking algorithm (FC)}
A standard approach to solve the constraint satisfaction problems is maintaining arc consistency (MAC) or forward checking (FC). Both algorithms are built around the idea of enforcing some level of arc consistency (AC) at each step and branch when needed. The main difference between FC and MAC is the fact that MAC tries to maintain AC on each node at every level while FC tries to avoid the extra work by maintaining AC only between the most recently instantiated variables and those that are still not instantiated.

While FC do in general much less work at each node, it needs to branch much more to exclude all wrong paths. On the other hand, MAC usually have smaller number of branches and spends in return more time at each node. This trade-off leads to different characteristics of each algorithm, which in turn prioritize one over another depending on the problem at hand.
% TODO where to cont. ....one can better than the other

\subsection{Analysis approach}
To determinate that an encoding is good enough or rather better than another (see below \ref{subsec:branching_for_comparing}), we will compare solving a given problem with its standard algorithm to solving the encoded version of the same problem with its respective standard algorithm. For example: A CSP problem can be solved directly using MAC or it can be encoded as SAT problem (using order encoding for instance) and solved using DPLL. Note that the encoding process cannot take longer time than solving the problem itself. In concrete terms, the encoding procedure (which can be considered as a type of reduction) can be done in polynomial time since both problems are NP complete.

\subsubsection{Branching for comparing}\label{subsec:branching_for_comparing}
Since both MAC/FC and DPLL are backtracking based approaches, we will use the number of visited branches as indicator for the efficiency of each algorithm and in this way compare the impact of achieving arc-consistency from FC on the CSP with unit propagation from DPLL on the SAT problem. So, one algorithm outperforms (dominates) another if and only if it visits less number of branches assuming equivalent branching heuristics \cite{walsh2000sat}.

This criterion has been chosen because it serves as a good indicator for the performance of these algorithms in real world applications. This comparison will be aided with examples and comparisons at the end of the paper.
Notice that FC is always dominated by MAC regarding this criterion, which does not always reflect the real performance but gives a good clue about the efficiency of each algorithm without diving too deep into the exact implementation.


\section{Example EBNF}
Formulae of kind \texttt{U a, b E c pred(a, b) and not (pred(a) or pred(c))} with precedence order.

\newcommand{\nont}[1]{\;\langle \text{#1} \rangle\;}
\begin{align*}
    \nont{LANG} &\rarr \nont{FORM} (; \nont{FORM})^+ . \\
    \nont{FORM} &\rarr \nont{QUANT} \nont{EXPR} \\
    \nont{QUANT} &\rarr (U \nont{VARL} |\; E \nont{VARL})^* \\
    \nont{EXPR} &\rarr \nont{TERM} (or \nont{TERM})^* \\
    \nont{TERM} &\rarr \nont{FACT} (and \nont{FACT})^* \\
    \nont{FACT} &\rarr [not] \nont{OBJ} \\
    \nont{OBJ} &\rarr \nont{PRED} |\; '(' \nont{EXPR} ')' \\
    \nont{PRED} &\rarr id\; '(' \nont{VARL} ')' \\
    \nont{VARL} &\rarr id (, id)^*
\end{align*}


\def\baselinestretch{1}

\chapter{Appendix Numerical Linear Algebra}
$M_{m,n}(\field{F})$ denotes the vector space of matrices over
the field $\field{F}$.

\[A \in M_{n,n}(\dblf) \; b \in \dblf^n,  \exts x  \ni \; Ax=b \,
\;\texttt{iff}\; det(A)=0\]

$\mathbf{X} \in M_{mn}(\field{R})$ is positive definite if $
v^{t} X v > 0 \fall v \in \field{R}^n$.  We can construct
positive definite symmetric matrices by by forming
$\mathbf{X}^{t} \mathbf{X}$ where $\mathbf{X}$ is an orthogonal
(full rank) matrix.

Horner's rule is a method for evaluating a polynomial at a
point in $O(n)$ time. Straightforward evaluation of a n degree
polynomial is done in $O(n^{2})$ time. Simply rewrite the
function $f(x)= \sum\limits_{i=0}^{n-1} {a}_i x^{i}$ as
$f(x)=(\ldots (a_{n-1}x + a_{n-2} )x +  \ldots + a_1)x+a_0$

\section*{The min max characterization of eigenvalues}
This is a variational characterization of the eigenvalues of compact operators on a Hilbert space. Let $H \st H=H^\dag$ The Rayleigh quotient is defined by $R(x) =  \frac{<Hx,x>}{\norm{x}^2}$ Let $\sigma(H) ={\lambda_i}$, then $\forall S_k \in \dblr^n$ we have
\begin{equation*}
  \underset{x \in S_k \norm{x}=1}{max} <Hx,x> \geq \lambda_k
\end{equation*}
which implies
\begin{equation*}
\underset{S_k}{inf}  \underset{x \in S_k \norm{x}=1}{max} <Hx,x> \geq \lambda_k
\end{equation*}
Equality above is achieved when $S_k = spn{\mu_k}$ where $\mu_k$ is the kth eigenvector of $H$.

The min max theorem is that
\begin{equation*}
\lambda_1 \leq R(x) \leq \lambda_n
\end{equation*}

\section*{The Discrete Fourier Transform on $\ell^2(\dblz_{N_1})$} Let $z \in \dblz_{N_1} ,\!
z=(z(0),z(1),\ldots,z(N_1-1))$.  We index from 0 instead of 1
for convenience of presenting the FFT. Define
\begin{equation*}
\widehat{z(m)}=\sum\limits_{k=0}^{N_1-1} z(k)e^{ \frac{-2 \pi
\imath k m}{N_1}}
\end{equation*}
The map $\hat \!: \ell^2(\dblz_{N_1}) \rightarrow
\ell^2(\dblz_{N_1})$ is the Fourier Transform. The vectors
\begin{equation*}
E_0,E_1, \ldots,E_{N_1 -1}  :\!  E_m(n)=\frac{e^\frac{-2 \pi
\imath m n}{N_1}}{\sqrt{N_1}}
\end{equation*}
form an orthonormal basis for $\ell^2(\dblz_{N_1})$. The
vectors $ \frac{E_0}{N_1},\frac{E_1}{N_1}, \ldots,\frac{E_{N_1
-1}}{N_1}$ form an orthogonal basis called the Fourier Basis.

Extend the indices over $\dblz_{N_1}$ to $\dblz$ by considering
$\dblz_{N_1}$ to be the algebraic group $\dblz mod N_1$.  Then
we can define the translation operator
\begin{equation*}
(R_l z)(n)=z(n-l).
\end{equation*}
We can also define the convolution operator with this extended
notion of $\dblz_{N_1}$;
\begin{equation*}
z * w = \sum \limits_{k=0}^{N_1-1} z(m-n)W(n)
\end{equation*}
The Fourier Multiplier Operator $T_{(m)}$ where $m \in
\ell_2{\dblz_{N_1}}$ is given by
\begin{equation*}
T_{(m)}=(m\hat z)\check{}
\end{equation*}

Fourier Inversion Formula:
\begin{equation*}
z(m)=\frac {1}{N_1}\sum\limits_{k=0}^{N_1-1} \hat{z(k)} e^{
\frac{2 \pi \imath k m}{N_1}}
\end{equation*}

Parsevall's Relation:
\begin{equation*}
<z,w>=\frac{1}{N_1}<\hat z,\hat w>
\end{equation*}

Plancherel's Formula: Parsevall's relation with $w=z$.

Representation in the Fourier Basis:
\begin{equation*}
z=\sum \limits_{k=0}^{N_1-1} \hat{z(k)} F_k
\end{equation*}

The effect of the translation operator is to rotate the phase
of the Fourier Transform:
\begin{equation*}
(R_l z)\hat(k)=e^{ \frac{2 \pi \imath k l}{N_1}}\widehat{z(k)}
\end{equation*}

The effect of conjugation is to reflect the Fourier Transform:
\begin{equation*}
(\overline{z})\hat(k)=\overline{\hat z(-k)}
\end{equation*}

The Convolution Operator is equivalent to a Fourier Multiplier
Operator:
\begin{equation*}
b * z=(m \hat z )\check{} \! : m=\hat b
\end{equation*}

\section*{Multiresolution analysis} Basis functions of a linear
subspace $V_j \subset L^2(\Omega)$ are
defined by a scaling function $\phi$

%bbcrevisit - insert the class of admissable scaling fn's for prefect reconstruction, then address frames
via the following procedure;
\begin{gather*}
\phi_{ij}(x) = \phi(2^{-j} x-i)  \\
V_j = span \{ \phi_{ij} \} \\
W_{j+1}= V_j \ V_{j+1}^\bot \\
\hdots V_{j+1} \subset V_j \subset \hdots \subset V_0 \subset \hdots V_{-j}  \subset \hdots \\
V_j=V_{j+1} \oplus W_{j+1} x \in W_{j+1} \Rightarrow \exts
{a_l} \;\; x=\sum_l \{a_l\} \phi_{jl}
\end{gather*}
A basis for $W_{ij}$ is constructed from a mother wavelet
$\psi$.

\section*{Voroni Tesselations}
A centroidal Voronoi tessellation is a Voronoi tessellation where the
generating points are the centroids of the corresponding regions.
Applications Voronoi tessellations can be found in image compression,
clustering, quadrature, and finite difference methods. distribution of
resources.  The dual of the Voroni tessellation in $\dblr^2$ is the Delaunay
triangulation.

The example below is a simulated example of resource allocation in $\dblr^2$
A partition of a Random walk in $\dblr^2$ obtained by calculating the Voroni
tessellation and associated Delaunay triangulation on the k-means centroids.

\includegraphics[width=8.0cm,height=8.0cm]{Figures/ClusteringRW/rw_10000_delauney_kmens_convex_hull.pdf}

\section*{The Matrix Exponential}
The matrix exponential of a matrix $\mat{A}$ is defined as
\begin{align*}
  e^{\mat{A}}
  &= \mat{I} + \mat{A} + \frac{\mat{A}^2}{2!} + \dots \\
  &= \sum_{k = 0}^\infty \frac{\mat{A}^k}{k!}.
\end{align*}

The Pade approximation to
$e^{\mat{A}}$ is
\begin{displaymath}
  e^{\mat{A}} \approx R(\mat{A}),
\end{displaymath}
with
\begin{align*}
  R_{pq} (\mat{A})
  &= (D_{pq}(\mat{A}))^{-1} N_{pq}(\mat{A}) \\
  \intertext{where}
  D_{pq}(\mat{A})
  &= \sum_{j=1}^p \frac{(p+q-j)! p!}{ (p+q)!j!(p-j)!}\, \mat{A}^j \\
  \intertext{and}
  N_{pq}(\mat{A})
  &= \sum_{j=1}^q \frac{(p+q-j)! q!}{ (p+q)!j!(q-j)!}\, \mat{A}^j.
\end{align*}
See \cite{Moler78nineteendubious} for a detailed accounting of this and other matters regarding the calculation of the matrix exponential.
%\citet{Moler78nineteendubious} \citep*{Moler78nineteendubious} \citep{Moler78nineteendubious} \citet*{Moler78nineteendubious}


\documentclass[12pt]{article}
\input{preamble.tex}
\addbibresource{bibliography.bib}

\renewcommand{\DocTitle}{Barycenters vs. Model Averages in Gaussian Models}
\renewcommand{\DocAuthors}{Gabriel Arpino}

\setcounter{tocdepth}{4} % Show subsections in the TOC
\setcounter{secnumdepth}{4}

\begin{document}
\title
\tableofcontents
\section{Problem Description}

It is of importance in machine learning to be able to combine models, be it as a
means of reducing uncertainty in predictions or hedging risk during prediction time.
The machine learning community has been exploring this topic in various forms,
from ensembling techniques (\cite{NIPS2017_7219}) to stochastic optimal transport
(\cite{DBLP:journals/corr/abs-1802-05757}). A recent paper, \cite{NIPS2017_7149},
explores the idea of averaging Gaussian Process models using Wasserstein Barycenters,
and show that it produces clearer and more representative results than the "naive average",
which is equivalent to calculating the average and standard deviation of the mean of
these gaussian processes. They propose a tractable fixed point iteration method for
calculating the barycenter distribution, and display experimental results. A question
to ask is whether this is required for Gaussian Processes at all, because averaging
finite dimensional representations of these is equivalent to adding up Gaussian measures.
Identifying similarities and differences between these two approaches can help
the community understand when the $O(n^3)$ complexity of calculating the barycenter
is a requirement, or when a simple sum of gaussian moments could be used.

The probability of a predicted function $f$ from a model within a set of models
$M$ of size $N$ is as follows:

\begin{equation} \label{eq:1}
    p(f) = \int p(f | M)p(M) dM
\end{equation}

which follows from marginalization. This method of reasoning about the "average"
of model predictions will be termed \textit{Model Averaging}. For the case of
gaussian process models, the integral in \ref{eq:1} is tractable, as the sum becomes
discrete and gaussians are closed under addition:

\begin{equation} \label{eq:2}
    p(f) = \sum_{i=1}^N \xi_i p(f | M_i)
\end{equation}

where $\xi$ are the discrete probabilities of each model (assuming $p(M)$ is a
discrete measure). For gaussian finite dimensional distributions (the stochastic processes
evaluated at discrete input points) $X_i$, $p(f)$ has the following moments:

\begin{align*}
    \mathbb{E}[X^*] &= \sum_{i=1}^N \xi_i \mathbb{E}[X_i], &
    \mathbb{E}[X^*X^{*T}] &= \sum_{i=1}^N \xi_i \mathbb{E}[X_iX_i^T],
\end{align*}

Where $X^*$ is the random variable denoting $p(f)$ in \ref{eq:2}. It is not always
the case, however, that \ref{eq:1} is analytically tractable, and in such cases
the integral and moments of the distribution can be approximated using Monte Carlo.

The barycenter $\mu^*$ of $N$ gaussian distributions $\mu _i$ is defined as follows:

\begin{align*} \label{barycenter:1}
    \mu^* = \inf_{\mu \in P_2(H)} \sum_{i=1}^{N} \xi_i W_2^2(\mu_i, \mu)
\end{align*}

where $W_2^2(\mu_1, \mu_2)$ denotes the 2-Wasserstein metric between gaussian
measures $\mu_1$, $\mu_2$ \cite{NIPS2017_7149}.

It is unclear whether, for their use in GPs, barycenters provide a reduced uncertainty
ensemble of models compared to standard model averaging. The text explores this idea,
and states certain scenarios where these two methods of averaging are equal, and where
they diverge.

\section{Theoretical Observations}

\subsection{Equivalent Covariance Matrices}

For the finite dimensional distribution representation of Gaussian Processes, it
can be shown that the barycenter distribution is a Gaussian distribution with
mean $\bar{m}$ and covariance matrix $\bar{K}$ denoted as follows \cite{NIPS2017_7149}:

\begin{align*}
    \bar{m} &= \sum_{i=1}^{N} \xi_i m_i, &
    \bar{K} &= \sum_{i=1}^{N} \xi_i (\bar{K}^{\frac{1}{2}} K_i \bar{K}^{\frac{1}{2}}),
\end{align*}

Assume there is some data $\textbf{y}$ and some time points we would like to predict
$t_*$ with function values $y_*$. This can be modelled with a GP:

\begin{align}
  \begin{bmatrix}
    \textbf{y} \\
    y_*
\end{bmatrix} \sim \mathcal{N} \bigg( 0,\begin{bmatrix}
      K & K_*^T \\
      K_* & K_{**}
  \end{bmatrix} \bigg)
\end{align}

Conditioning this GP on a training set $\textbf{y}$ and evaluating on the finite set $T^*$ produces
a finite dimensional gaussian distribution with moments \cite{1505.02965}:

\begin{align*}
    \mathbb{E}[\bar{y}_*] &= K_* K^{-1} \textbf{y}, &
    \hat{K} &= var(\bar{y}_*) = K_{**} - K_{*} K^{-1} K_*^T
\end{align*}

Let us consider the case where we are averaging GPs with equivalent
kernel functions $k(t, t^\prime)$ and means $\mu_i$. Assume that our prior
distribution over these models is uniform, so $\xi_i = 1/N$. The model average $p(f)$ can
be calculated from \ref{eq:2} and the fact that gaussians are closed under addition:

\begin{equation} \label{eq:3}
    p(f) \sim \mathcal{N}(\frac{1}{2} \mu_1 + \frac{1}{2} \mu_2, \frac{1}{2} K_1 + \frac{1}{2} K_2)
\end{equation}

Where $K_i$ denotes the covariance matrix that is the gram matrix produced from a GP's
respective kernel function $k(t, t^\prime)$. According to our assumption that both GPs
contain equivalent kernel functions, their covariance matrices $K_i$ are equivalent,
so \ref{eq:3} reduces to:

\begin{equation} \label{eq:4}
    p(f) \sim \mathcal{N}(\frac{1}{2} \mu_1 + \frac{1}{2} \mu_2, K)
\end{equation}

where $K = K_1 = K_2$. Now, since we currently care about averaging the predictions
of GPs that have been conditioned on data (they are not as informative otherwise),
we can note that the posterior kernels of these two GPs are also equivalent, which
follows from the equivalence of their kernel functions and the equivalent formulations
for $\hat{K}$, the posterior covariance matrix. Let us denote this common posterior covariance matrix
$\hat{K}$, and consider the following proposition:

\begin{proposition} \label{prop:1}
    The barycenter of gaussian processes with equivalent kernel functions $k(t, t^{\prime})$
    is equivalent to their model average.
\end{proposition}

\begin{proof}
    In order to show this is true, we have to show that both the moments of both
    distributions (the barycenter and the model average) are equivalent, since gaussian
    measures are entirely defined by their first two moments. We start with the
    means which follow trivially from \ref{eq:2} and \ref{barycenter:1}:

    \begin{equation}
        \mathbb{E}[p(f)] = \sum \xi_i \mathbb{E}[p(f | M)] = \sum \xi_i \mu_i = \bar{m}
    \end{equation}

    We now work to show that $\bar{K} = \hat{K}$. This can be shown by proving that
    $\hat{K}$ is a solution to the fixed point iteration equation \ref{eq:2}. This
    equation is shown to be convex in \cite{NIPS2017_7149}, so finding a solution
    ensures that it is optimal:

    \begin{equation} \label{OTkern}
        \bar{K} = \sum_{i=1}^{N} \xi_i (\bar{K}^{\frac{1}{2}} K_i \bar{K}^{\frac{1}{2}})
    \end{equation}

    Substitute $\hat{K}$ into the right hand side:

    \begin{equation}
        \sum_{i=1}^{N} \xi_i (\hat{K}^{\frac{1}{2}} K_i \hat{K}^{\frac{1}{2}})
    \end{equation}

    but keeping in mind that, since all gaussian processes possess the same kernel
    function, $\hat{K} = K_i$, so:

    \begin{equation} \label{lolwow}
        = \sum_{i=1}^{N} \xi_i (\hat{K}^{\frac{1}{2}} \hat{K} \hat{K}^{\frac{1}{2}})^(\frac{1}{2})
    \end{equation}

    We have to show that \ref{lolwow} equals $\hat{K}$ for the implicit equation
    to be satisfied. Keeping in mind that covariance matrices are symmetric, hence
    positive semidefinite, diagonalizable, and possess orthogonal eigenvectors, we apply
    the diagonal decomposition to each matrix:

    \begin{equation}
        \hat{K} = U \Lambda U^T
    \end{equation}

    which leads to

    \begin{equation}
        &= \sum_{i=1}^{N} \xi_i ((U \Lambda U^T)^{\frac{1}{2}} (U \Lambda U^T) (U \Lambda U^T)^{\frac{1}{2}}).
    \end{equation}

    Recall that, for orthogonal matrices $U$, $U U^T = I$ and

    \begin{equation}
        (U \Lambda ^{\frac{1}{2}} U^T) (U \Lambda ^{\frac{1}{2}} U^T) = U \Lambda U^T
    \end{equation}

    so, $(U \Lambda ^{\frac{1}{2}} U^T) = (U \Lambda U^T)^{\frac{1}{2}}$. Plugging this into \ref{lolwow}:

    \begin{align*} \label{k_deriv}
        &= \sum_{i=1}^{N} \xi_i ((U \Lambda^{\frac{1}{2}} U^T) (U \Lambda U^T) (U \Lambda^{\frac{1}{2}} U^T))^{\frac{1}{2}} \\
        &= \sum_{i=1}^{N} \xi_i (U \Lambda^{\frac{1}{2}} \Lambda \Lambda^{\frac{1}{2}} U^T)^{\frac{1}{2}} \\
        &= \sum_{i=1}^{N} \xi_i (U \Lambda^2 U^T)^{\frac{1}{2}} \\
        &= U \Lambda U^T \\
        &= \hat{K}
    \end{align*}

    Therefore, $\hat{K}$ is the unique solution to the fixed point iteration equation and we
    have shown that the gaussian distributions created by model averaging and calculating
    the barycenter of gaussian processes with prior probabilities $x_i$ are equivalent
    given that they possess the same kernel function $k(t, t^\prime)$.
\end{proof}

This result entails that conclusions achieved in \cite{NIPS2017_7149} could have
been achieved through simple model averaging, as the paper used the same kernels
for fitting the gaussian processes, but different samples of data.

\subsection{Covariance Matrices differing by a constant}

For the case of covariance matrices differing by a constant, we analyze the following finite dimensional GPs:

\begin{align}
    \textbf{y} \sim \mathcal{N} \big(0, \Sigma \big), &
    \textbf{y} \sim \mathcal{N} \big(0, \alpha \Sigma \big)
\end{align}

where $\alpha$ is a positive constant. Analyzing these two gaussians for now, we can attempt to derive what a solution to the
fixed point iteration barycenter equation could be, suppose it is of the form $\beta \Sigma$, $\beta$ a positive constant. Then
from \ref{OTkern} and assuming a uniform average of distributions ($\xi = 1/N$) we expand:

\begin{align}
    \beta \Sigma = \frac{1}{2} ((\beta \Sigma)^{\frac{1}{2}}  \Sigma (\beta \Sigma)^{\frac{1}{2}})^{\frac{1}{2}} +
    \frac{1}{2} ((\beta \Sigma)^{\frac{1}{2}} \alpha \Sigma (\beta \Sigma)^{\frac{1}{2}})^{\frac{1}{2}}
\end{align}

And, taking into account that $\Sigma$ is a positive definite matrix, we use orthogonality to simplify:

\begin{align}
    \beta \Sigma &= \frac{1}{2} ((\beta \Sigma)^{\frac{1}{2}}  \Sigma (\beta \Sigma)^{\frac{1}{2}})^{\frac{1}{2}} +
    \frac{1}{2} ((\beta \Sigma)^{\frac{1}{2}} \alpha \Sigma (\beta \Sigma)^{\frac{1}{2}})^{\frac{1}{2}} \\
    \beta \Sigma &= \frac{1}{2} \beta^{\frac{1}{2}}((\Sigma)^{\frac{1}{2}}  \Sigma (\Sigma)^{\frac{1}{2}})^{\frac{1}{2}} +
    \frac{1}{2} \beta^{\frac{1}{2}} \alpha^{\frac{1}{2}} ((\Sigma)^{\frac{1}{2}} \Sigma (\Sigma)^{\frac{1}{2}})^{\frac{1}{2}} \\
    \beta \Sigma &= ((\Sigma)^{\frac{1}{2}}  \Sigma (\Sigma)^{\frac{1}{2}})^{\frac{1}{2}} * (\frac{1}{2} \beta^{\frac{1}{2}} +
    \frac{1}{2} \beta^{\frac{1}{2}} \alpha^{\frac{1}{2}}) \\
    \frac{\beta}{(\frac{1}{2} \beta^{\frac{1}{2}} + \frac{1}{2} \beta^{\frac{1}{2}} \alpha^{\frac{1}{2}})} \Sigma &=
    ((\Sigma)^{\frac{1}{2}}  \Sigma (\Sigma)^{\frac{1}{2}})^{\frac{1}{2}} \\
    \frac{\beta}{(\frac{1}{2} \beta^{\frac{1}{2}} + \frac{1}{2} \beta^{\frac{1}{2}} \alpha^{\frac{1}{2}})} &= 1 \\
    \beta^{\frac{1}{2}} (\beta^{\frac{1}{2}} - (\frac{1}{2} + \frac{1}{2} \alpha^{\frac{1}{2}})) &= 0 \\
\end{align}

simplifying taking into account the relation shown earlier, $K = (K^{\frac{1}{2}} K K^{\frac{1}{2}})^{frac{1}{2}}$ for positive definite
matrices. Ignoring the trivial solution $\beta^{frac{1}{2}} = 0$, we get that:

\begin{equation} \label{OT_scalar}
    \beta^{\frac{1}{2}} &=  (\frac{1}{2} + \frac{1}{2} \alpha^{\frac{1}{2}})
\end{equation}

And if we do the same procedure with the euclidean average, calculating $\beta_{EU}$ for $\Sigma_{EU}$ (also called the euclidean barycenter):

\begin{align}
    \beta_{EU} \Sigma_{EU} = \frac{1}{2} \Sigma_{EU}  +\frac{1}{2}  \alpha \Sigma_{EU}
\end{align}
\begin{equation} \label{EU_scalar}
    \beta_{EU} &= \frac{1}{2} + \frac{1}{2} \alpha
\end{equation}

Now, if we analyze equations \label{OT_scalar}, and \label{EU_scalar}, we see that the first grows sublinearly with $\alpha$,
while the second grows linearly with $\alpha$. This makes sense, and it shows that, for gaussians whose covariance matrices differ
by a constant, the barycenter always has lower variance. The two $\beta$'s are equal when $\alpha = 1$, which is the first case we
analyzed, in all other cases,  $(\frac{1}{2} + \frac{1}{2} \alpha^{\frac{1}{2}}) < \frac{1}{2} + \frac{1}{2} \alpha$.

\subsection{Entropy}

Now, we can formulate the same problem using the notion of entropy, and finding which type of barycenter (euclidean or wassserstein)
will yield averages with less entropy (more certainty). The entropy of a gaussian distribution of dimension $k$ is:

\begin{equation} \label{ENTROPY}
    H = \frac{k}{2} + \frac{k}{2} log(2\pi) + \frac{1}{2} log{|\Sigma|}
\end{equation}

Now we can analyze the entropy of the euclidean and wasserstein barycenters, denoted $H_{EU}$ and $H_{OT}$ respectively. Consider the simple
case of computing the barycenter of two normal distributions of dimension $k$ with $\xi_i = \frac{1}{2}$:

\begin{align}
    H_{EU} &= \frac{k}{2} + \frac{k}{2} log(2\pi) + \frac{1}{2} log{|K_{EU}|} \\
    H_{OT} &= \frac{k}{2} + \frac{k}{2} log(2\pi) + \frac{1}{2} log{|K_{OT}|} \\
\end{align}

The only terms that differ at the covariance matrix determinants, so let's analyze those.

% \begin{align}
\begin{eqnarray*}
    |K_{OT}| = |\frac{1}{2} (K_{OT}^{\frac{1}{2}}K_1K_{OT}^{\frac{1}{2}})^{\frac{1}{2}} +
                    \frac{1}{2} (K_{OT}^{\frac{1}{2}}K_1K_{OT}^{\frac{1}{2}})^{\frac{1}{2}}| \\
    |K_{OT}| \geq |\frac{1}{2} (K_{OT}^{\frac{1}{2}}K_1K_{OT}^{\frac{1}{2}})^{\frac{1}{2}}|
                    + |\frac{1}{2} (K_{OT}^{\frac{1}{2}}K_1K_{OT}^{\frac{1}{2}})^{\frac{1}{2}}| \\
    |K_{OT}| \geq \frac{1}{2}^{k} |K_{OT}|^{\frac{1}{2}}|K_1|^{\frac{1}{2}}
                        + \frac{1}{2}^{k} |K_{OT}|^{\frac{1}{2}}|K_2|^{\frac{1}{2}} \\
    (|K_{OT}|^{\frac{1}{2}})(|K_{OT}|^{\frac{1}{2}} - (\frac{1}{2}^{k} |K_1|^{\frac{1}{2}} + \frac{1}{2}^{k} |K_2|^{\frac{1}{2}})) \geq 0 \\
    (|K_{OT}|^{\frac{1}{2}} - (\frac{1}{2}^{k} |K_1|^{\frac{1}{2}} + \frac{1}{2}^{k} |K_2|^{\frac{1}{2}})) \geq 0 \\
\end{eqnarray*}
% \end{align}

Using the fact that $K_{OT}$ is positive definite and $|A + B| \geq |A| + |B|$ for matrices $A$ and $B$. Now if we analyze
the euclidean barycenter covariance matrix:

\begin{eqnarray*}
    |K_{EU}| = |\frac{1}{2} K_1 + \frac{1}{2} K_2| \\
    |K_{EU}| \geq \frac{1}{2}^{k} |K_1| + \frac{1}{2}^{k} |K_2| \\
\end{eqnarray*}

It can also be shown through KKT optimization that both matrix determinants can be upper
bounded by $|K_j|$ where $K$

\begin{todo}

    \begin{itemize}
        \item Loosen the restriction a little bit, and can start looking at covariance matrices that are simultaneously diagonalizable
        (same eigenvectors, but different eigenvalues)

        \item Start looking at the entropy of $K$, and how that changes. This has been started, and calculating the entropy actually
        gives you a lower bound on the $K$ derived from both methods. This is yet to be typed into Latex.

        \item Properly calculate the computational complexity of barycentering vs
                calculating the model average

        \item Comparing with the case where kernels produce covariance functions with the same eigenvectors,
                but potentially different eigenvalues and what that means

        \item Is there any case where the variance of the barycenter is larger?

        \item Any proofs for non-gaussian models?
    \end{itemize}

\end{todo}

%
% \begin{align*}
%     \MEC_i \cond \hat{\MEC}_i &= f^{-1} \circ \hat{\MEC}_i, &
%     \hat{\MEC}_i &\sim p(\hat{\MEC} \cond \mathcal{D}),
% \end{align*}

\appendix

\printbibliography

\end{document}


%
% Copyright 2018 Parakram Majumdar
%
% Licensed under the Apache License, Version 2.0 (the "License");
% you may not use this file except in compliance with the License.
% You may obtain a copy of the License at
%
%     http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS,
% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
% See the License for the specific language governing permissions and
% limitations under the License.
%

\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}
\title{Common Functions on Tensors}
\author{Parakram Majumdar}

\begin{document}
  \maketitle

\newcommand{\R}{\mathbb{R}}
\newcommand{\Prob}[2]{\mathbb{#1}\left[ #2 \right]}
\newenvironment{where}{\noindent{}where\begin{itemize}}{\end{itemize}}

\section{Introduction}
  This document describes some of the standard tensor operations
  that are relevant for machine learning.
  The rigorous mathematician may note
  that this document uses the term \emph{tensor}
  merely as a substitute for \emph{multi-dimensional arrays}.

\section{Definitions}
  \begin{itemize}

  \item
  A \emph{tensor} $T$ of \emph{dimensionality} $d_1, d_2, ..., d_o$ is
  a multi-dimensional array of real numbers in
  $R^{d_1 \times d_2 \times ... \times d_o}$.

  \item
  In the above definition,
  the number of dimensions $o$ is also called
  the \emph{order} of $T$.

  \item
  The number of elements in $T$, i.e.,
  $d_1 \times d_2 \times ... \times d_o$
  is called the size of T.

  \item
  For brevity, we introduce the following notations:
  \begin{eqnarray*}
    \overrightarrow{d_n} & \equiv & d_1, d_2, ..., d_n \\
    \Pi d_n              & \equiv & d_1 \times d_2 \times ... \times d_n
  \end{eqnarray*}

  \item
  A single argument \emph{tensor function}
  \[f: \R^{\Pi a_m}
       \rightarrow
       \R^{\Pi b_n}
  \]
  is a function that maps a tensor to another tensor.

  \item
  The \emph{gradient} of the above mentioned $f$
  at some point $x \in \R^{\Pi a_m}$
  is a tensor
  \[\nabla f(x) \in \R^{\Pi a_m \times \Pi b_n}
  \]
  that gives the rate of change of each element of $f(x)$
  with respect to each element of $x$.
  \[ \frac{\partial f(x)[\overrightarrow{j_n}]}
          {\partial x[\overrightarrow{i_m}]}
     = \nabla f(x)[\overrightarrow{i_m}, \overrightarrow{j_n}]
  \]

  \item
  A multiple argument tensor function
  takes multiple input tensors, and hence,
  has a separate gradient tensor with respect to each input.
  Note that, in general,
  it is impossible to combine all these gradients into a single tensor,
  since the various inputs may have different dimensionalities.

  \end{itemize}

\section{Grouping of co-ordinates}
  The tensors are implemented as a row-major one dimentional array
  with zero based indexing.
  For example, the three dimensional tensor
  $X \in \R^{2 \times 3 \times 4}$
  is stored as a one dimensional array of size
  $2 \times 3 \times 4 = 24$.
  The elements may be accessed by the formula:
  \[ X[i, j, k] = X[i \times 3 \times 4 + j \times 4 + k] \]
  where $i \in \{0, 1\}$,
        $j \in \{0, 1, 2\}$ and
        $k \in \{0, 1, 2, 3\}$.

  An interesting consequence of this
  is that consecutive dimensions may be squashed together
  without affecting the contents of the tensor.
  For example,
  $X \in \R^{2 \times 3 \times 4}$
  can be seen to be in
  $\R^{2 \times 12}$,
  $\R^{6 \times 4}$ and $\R^{24}$,
  as long as the correct formulae are used to interconvert
  the coordinates between these various forms.

  To further clarify,
  we temporarily introduce the notation of left superscripting
  the dimensionality of $X$ to denote which format it is in.
  Thus:
  \begin{eqnarray*}
    X[i, j, k] & = & {}^{2 \times 3 \times 4}X[i, j, k]\\
               & = & {}^{2 \times 12}X[i, 4j+k] \\
               & = & {}^{6 \times 4}X[3i + j, k] \\
               & = & {}^{24}X[12i + 4j+k]
  \end{eqnarray*}

  We now state again the above equivalence
  in even crisper notation by introducing
  the grouping of co-ordinates:
  \begin{eqnarray*}
    X[i, j, k] & = & X[[i], [j], [k]] \\
               & = & X[[i, j], k] \\
               & = & X[i, [j, k]] \\
               & = & X[[i, j, k]]
  \end{eqnarray*}

\section{Identity}
  The identity tensor function $\mathbb{I}$ maps each tensor to itself,
  and hence, is usually not useful in an efficient computation graph.
  However, it is mentioned here as an example
  of how the gradient of simple tensor functions
  can be remarkably sparse.

  Consider an input
  $x \in \R^{\Pi a_n}$.
  The gradient of $\mathbb{I}$ would then be
  \begin{eqnarray*}
    \nabla \mathbb{I}(x)[i_1, i_2, ..., i_n, j_1, j_2, ..., j_n]
    & = & \frac{\partial (\mathbb{I}(x)[j_1, j_2, ..., j_n])}
               {\partial x[i_1, i_2, ..., i_n]} \\
    & = & \frac{\partial (\mathbb{I}(x)[\overrightarrow{j}])}
               {\partial x[\overrightarrow{i}]} \\
    & = & \frac{\partial x[\overrightarrow{j}]}
               {\partial x[\overrightarrow{i}]} \\
    & = & \begin{cases}
            1, & \text{if } \overrightarrow{i} == \overrightarrow{j}\\
            0, & \text{otherwise}
          \end{cases}
  \end{eqnarray*}

  Thus, $\nabla \mathbb{I}(x)$ is a \emph{diagonal} tensor with
  very few non-zero elements.

\section{Back Propagation}
  A common goal in machine learning is
  to minimize an objective with respect to some parameters.
  The back propagation algorithm is a \emph{gradient descent} algorithm
  implemented on a computation graph, i.e.,
  it computes the gradient of the objective
  with respect to each parameter,
  and then shifts the parameters to the direction
  where the objective seems to be \emph{descending}.

  The algorithm starts off by computing the value of the each node
  in the computation graph,
  starting with the values of the nodes directly defined on the parameters,
  and ultimately the value of the final objective.

  Then, to compute the gradient of the objective
  with respect to the parameters,
  it starts applying the \emph{backward propagation} step
  on each computation node, starting from the objective,
  and ending at the parameters.

  Thus, the algorithm uses \emph{forward propagation}
  for computing the value of the objective,
  and then a \emph{backward propagation} to compute the gradient.

  As an example, suppose a given node $f$ has an input tensor $x$.
  $x$ might be a parameter to the overall graph,
  or it might be the output of some other node.
  Also, suppose $f$ feeds into some other nodes $g$ and $h$.
  We will demonstrate the application of the backward propagation step
  to $f$.

  Assume that the objective of the graph is $o$.
  The backward propagation step on $f$ assumes
  that $g^*(f(x))$ and $h^*(f(x))$ are given, and computes $f^*(x)$,
  where
  \[ g^* = \frac{\partial o}{\partial g},
     h^* = \frac{\partial o}{\partial h},
     f^* = \frac{\partial o}{\partial f}
  \]
  by using the simple relationship:
  \[ f^* = \frac{\partial o}{\partial f}
         =   \frac{\partial o}{\partial g} \cdot \frac{\partial g}{\partial f}
           + \frac{\partial o}{\partial h} \cdot \frac{\partial h}{\partial f}
         = g^* \cdot g' + h^* \cdot h'
  \]
  The operator $\cdot$ in the above equation is a generalization
  of the \emph{vector dot product} to tensors.
  We discuss this \emph{tensor dot product} operator
  in a later section.

  Also, note that in general, $f$ could have more than one inputs,
  and could have any number of outputs,
  some of which might be repeated,
  in case $f$ is a parameter to them more than once.
  These generalisations, being somewhat simple,
  are left to the reader.

\section{Tensor Dot Product}
  Tensor dot product is a generalization of vector dot product
  and matrix multiplication.
  Given two tensors
  \[ L \in \R^{\Pi a_m \times \Pi b_n} \]
  \[ R \in \R^{\Pi b_n \times \Pi c_o} \]
  the \emph{tensor dot product of $L$ with $R$ on $n$ dimensions}
  \[ L \cdot_n R = M \in \R^{\Pi a_m \times \Pi c_o} \]
  is computed as \footnote{
   The astute reader will note how
   the bounds of the \emph{loop variable} $\overrightarrow i$
   are element-wise from
   $[0, 0 ... (n \text{ times})]$ to
   $[b_1 - 1, b_2 - 1, ... b_n - 1]$.
   This is consistent with the low level storage details
   discussed in an earlier section.
  }:
  \[ M[\overrightarrow{a}, \overrightarrow{c}]
     = \sum_{\overrightarrow{i} = \overrightarrow{0}_n}
           ^{\overrightarrow{i} < \overrightarrow{b}_n}
            {L[\overrightarrow{a}, \overrightarrow{i}]
             \times R[\overrightarrow{i}, \overrightarrow{c}]
            }
  \]
  Thus, matrix multiplication and vector dot product,
  can be seen as tensor dot products,
  of matrices and vectors, respectively,
  on 1 dimension.

  To revisit how the tensor dot product is connected
  with the backward propagation algorithm,
  suppose we have two tensor functions
  \[ f: \R^{\Pi a_m} \rightarrow \R^{\Pi b_n} \]
  \[ g: \R^{\Pi b_n} \rightarrow \R^{\Pi c_o} \]
  with gradients
  \[ \nabla f: \R^{\Pi a_m} \rightarrow \R^{\Pi a_m \times \Pi b_n} \]
  \[ \nabla g: \R^{\Pi b_n} \rightarrow \R^{\Pi b_n \times \Pi c_o} \]
  then the gradient of
  \[ g \circ f = g(f(\cdot)): \R^{\Pi a_m} \rightarrow \R^{\Pi c_o} \]
  denoted by
  \[ \nabla (g \circ f):
               \R^{\Pi a_m} \rightarrow \R^{\Pi a_m \times \Pi c_o} \]
  is computed by the \emph{chain rule}:
  \[ \nabla (g \circ f) (x) = \nabla g(f(x)) \cdot_n \nabla f(x)
     \text{, }\forall x \in \R^{\Pi a_m}
  \]
  Or in short:
  \[ \nabla(g \circ f) = \nabla g \cdot \nabla f \]

  We now look at the gradients of the dot product product,
  which can of course be seen as a binary tensor,
  with a dimensionality in the above example of:
  \[ (\cdot_n): \R^{\Pi a_m \times \Pi b_n}
                \times
                \R^{\Pi b_n \times \Pi c_o}
                \rightarrow
                \R^{\Pi a_m \times \Pi c_o}
  \]
  There are of course two partial gradients,
  one with respect to each of the two arguments:
  \[ \nabla_L M = \frac{\partial M}
                       {\partial L}
                \in \R^{\Pi a_m \times \Pi b_n \times
                        \Pi a_m \times c_o           }
  \]
  \[ \nabla_R M = \frac{\partial M}
                       {\partial R}
                \in \R^{\Pi b_n \times \Pi c_o \times
                        \Pi a_m \times \Pi c_o       }
  \]
  These are defined as:
  \begin{eqnarray*}
    \nabla_L M[\overrightarrow{a},
               \overrightarrow{b},
               \overrightarrow{a'},
               \overrightarrow{c}]
    & = & \frac{\partial M[\overrightarrow{a'},
                           \overrightarrow{c}]}
               {\partial L[\overrightarrow{a},
                           \overrightarrow{b}]} \\
    & = & \frac{\partial
                \sum_{\overrightarrow{i} \leftarrow \overrightarrow{b}}
                     {L[\overrightarrow{a'}, \overrightarrow{i}]
                      \times R[\overrightarrow{i}, \overrightarrow{c}]
                     }}
               {\partial L[\overrightarrow{a},
                           \overrightarrow{b}]} \\
    & = & \begin{cases}
            R[\overrightarrow{b}, \overrightarrow{c}],
               & \text{if } \overrightarrow{a'} == \overrightarrow{a} \\
            0  & \text{otherwise}
          \end{cases}
  \end{eqnarray*}
  \begin{eqnarray*}
    \nabla_R M[\overrightarrow{b},
               \overrightarrow{c},
               \overrightarrow{a},
               \overrightarrow{c'}]
    & = & \frac{\partial M[\overrightarrow{a},
                           \overrightarrow{c'}]}
               {\partial R[\overrightarrow{b},
                           \overrightarrow{c}]} \\
    & = & \frac{\partial
                \sum_{\overrightarrow{i} \leftarrow \overrightarrow{b}}
                     {L[\overrightarrow{a}, \overrightarrow{i}]
                      \times R[\overrightarrow{i}, \overrightarrow{c'}]
                     }}
               {\partial R[\overrightarrow{b},
                           \overrightarrow{c}]} \\
    & = & \begin{cases}
            L[\overrightarrow{a}, \overrightarrow{b}],
               & \text{if } \overrightarrow{c'} == \overrightarrow{c} \\
            0  & \text{otherwise}
          \end{cases}
  \end{eqnarray*}

\section {Add scalar}
  The core of a perceptron revolves around the formula:
  \[
      p = w \cdot x + b \overrightarrow u
  \]
  \begin{where}
    \item $p$ is the \emph{output}
    \item $w$ are the \emph{weights} of the perceptron
    \item $x$ is the \emph{input}
    \item $b$ is the \emph{bias}
    \item $u$ is a unit tensor created to add $b$
              to every element of $p$
  \end{where}

  We define the scalar addition function $\boxplus$ to encapsulate
  the addition of $b$ to each element in the output
  \[
      z \boxplus b = z + b.\overrightarrow{u}
  \]
  Thus,
  \begin{eqnarray*}
    \frac{\partial (z \boxplus b) }{\partial z}
      & = & \nabla \mathbb{I}(z)
    \\
    \frac{\partial (z \boxplus b) }{\partial b}
      & = & \nabla \mathbb{I}(z)
  \end{eqnarray*}

\section {Sigmoid}
  The sigmoid function on a scalar is defined as:
  \[
    \text{sigmoid}(x) = \frac{e^x}{1 + e^x}
  \]
  The derivative is therefore given as:
  \[
    \frac{\partial \text{sigmoid}(x)}
         {\partial x}
    =
    \frac{e^x}{(1 + e^x)^2}
    =
    \text{sigmoid}(x)(1 - \text{sigmoid}(x))
  \]
  We extend this to tensors as
  an element wise application of the scalar version
  \[
    \text{sigmoid}(z) = \text{map}(z, \text{sigmoid}(.))
  \]
  And therefore, for an input $z \in \R^{\Pi a_n}$,
  the gradient
  $\nabla \text{sigmoid}(z) \in \R^{\Pi a_n \times \Pi a_n}$
  is defined as:
  \[
    \nabla \text{sigmoid}(z)[\overrightarrow{i_n}, \overrightarrow{j_n}]
    =
    \begin{cases}
      \text{sigmoid}(z[\overrightarrow{i_n}])
      \cdot
      (1 - \text{sigmoid}(z[\overrightarrow{i_n}]))
         & \text{if } \overrightarrow{i_n} == \overrightarrow{j_n}
      \\
      0  & \text{otherwise}
    \end{cases}
  \]

\end{document}