\chapter{ELEC 403 CheatSheet}
\begin{multicols}{3}

\textbf{Algorithm 1.1 General optimization algorithm} \newline
\textbf{Step 1:} \newline
(a) Set $k=0$ and initialize $x_0$ \newline
(b) Compute $F_0=f(x_0)$ \newline
\textbf{Step 2:} \newline
(a) Set $k=k+1$ \newline
(b) Compute the changes in $x_k$ given by column vector $\nabla x_k$ %where
\[ \text{where} \quad
\nabla x_k^T = \begin{bmatrix}\nabla x_1 & \nabla x_2 & \cdots & \nabla x_n\end{bmatrix}
\]
by using an appropriate procedure. \newline
(c) Set $x_k=x_{k-1}+\nabla x_k$ \newline
(d) Compute $F_k=f(x_k)$ and $\nabla F_k=F_{k-1}-F_k$. \newline
\textbf{Step 3:} \newline
Check if convergence has been achieved by using an appropriate criterion, e.g., by checking $\nabla F_k$ and/or $\nabla x_k$. If this is the case, continue to
Step 4; otherwise, go to Step 2. \newline
\textbf{Step 4:} \newline
(a) Output $x^* = x_k$ and $F^* = f(x^*)$. \newline
(b) Stop
\section{Ch.2}
Gradient: $g(x)=\nabla f(x)=\begin{bmatrix}\frac{\partial f}{\partial x_1} & \frac{\partial f}{\partial x_2} & \cdots & \frac{\partial f}{\partial x_n} \end{bmatrix}^T$ \newline
Hessian Matrix: $H(x)=\nabla g(x)=\nabla \{ \nabla^T f(x)\}$. \newline
\[
H(x)=
\begin{bmatrix}
\frac{\partial^2 f}{\partial^2 x_1} & \frac{\partial^2 f}{\partial x_1 \partial x_2} & \cdots & \frac{\partial^2 f}{\partial x_1 \partial x_n} \\
\frac{\partial^2 f(x)}{\partial x_2 \partial x_1} & \frac{\partial^2 f(x)}{\partial x_2^2} & \cdots & \frac{\partial f}{\partial x_2 \partial x_n} \\
\vdots & \vdots & \ddots & \vdots \\
\frac{\partial  f}{\partial x_n \partial x_1}& \frac{\partial f}{\partial x_n \partial x_2}& \cdots & \frac{\partial f}{\partial x_n \partial x_n}
\end{bmatrix}
\]
Taylor Series: (quad approx, linear approx): $\delta =\begin{bmatrix} \delta_1 &\delta_2 \end{bmatrix}^T$
$f(x + \delta) = f(x) + g(x)^T\delta + 12\delta^TH(x)\delta + o(||\delta||^2)$ \newline
Linear approximation: $f(x + \delta) \approx f(x) + g(x)^T\delta $
\begin{figure}
	\centering
	\includegraphics[width=\linewidth]{Images/min.jpg}
	\caption{Minimization points plotted}
\end{figure}
$\tilde{x}=x+\delta \quad \delta =\tilde{x}-x$ $\hat{f}(\tilde{x})=f(x)+g(x)(\tilde{x}-x)^T+0.5(\tilde{x}-x)^T H(x)(\tilde{x}-x)^T$
The gradient $g(x)$ and the Hessian $H(x)$ must satisfy certain conditions at a
local minimizer $x^*$.  \newline
1. Conditions which are satisfied at a local minimizer $x^*$.  \newline
2. Conditions which guarantee that $x^*$ is a local minimizer.

\textbf{Definition 2.1} A point $x^* \in R,$ where R is the feasible region, is said to be a
weak local minimizer of f(x) if there exists a distance $\epsilon > 0$ such that
$f(x) \geq f(x^*)$ (2.5)
if
$x \in R$ and
$||x-x^*|| < \epsilon$ \newline

\textbf{Definition 2.2}
A point $x^* \in R $ is said to be a weak global minimizer of f(x) if
$f(x) \geq f(x^*)$ (2.6)
for all $x \in R.$  \newline
\textbf{Definition 2.3}

If Eq. (2.5) in Def. 2.1 or Eq. (2.6) in Def. 2.2 is replaced by
$f(x) > f(x^*)$ (2.7)
$x^*$ is said to be a strong local (or global) minimizer. d

\textbf{Definition 2.4} Let $\delta = \alpha \mathbf{d}$ be a change in x where $\alpha$ is a positive constant and d is a direction vector. If R is the feasible region and a constant $\hat{\alpha} > 0$ exists such that
$x + \alpha d \in R$ for all $\alpha$ in the range $0 \leq \alpha \leq \hat{\alpha}$ , then d is said to be a feasible direction at
point x. \newline

\textbf{Definition 2.5} \newline
(a) Let $d$ be an arbitrary direction vector at point $x$. The quadratic form
$d^TH(x)d$ is said to be \textit{positive definite, positive semidefinite, negative
	semidefinite, negative definite} if $d^TH(x)d > 0, \geq 0, \leq 0, < 0,$ respectively, for all $d \neq 0$ at $x$. If $d^TH(x)d$ can assume positive as well
as negative values, it is said to be indefinite. \newline
(b) If $d^TH(x)d$ is positive definite, positive semidefinite, etc., then matrix
$H(x)$ is said to be positive definite, positive semidefinite, etc. \newline

The objective function must satisfy two sets of conditions in order to have
a minimum, namely, first- and second-order conditions.  \newline
\textbf{First-order necessary conditions for a minimum} \newline %Theorem 2.2
(a) If $f(x) \in C^1$ and $x^*$ is a local minimizer, then
$g(x^*)^Td \geq 0$ for every feasible direction $d$ at $x^*$. \newline
(b) If $x^*$ is located in the interior of $\mathcal{R}$ then
$g(x^*) = 0$ \newline \newline
\textbf{Second-order necessary conditions for a minimum} \newline
%Theorem 2.2
(a) If $f(x) \in C_2$ and $x^*$ is a local minimizer, then for every feasible direction
d at $x^*$. \hfill \break
\indent     $(i) \ g(x^*)Td \geq 0$ \newline
\indent    $(ii)$ If $g(x^*)^Td = 0$, then $d^TH(x^*)d \geq 0$ \newline
(b) If $x^*$ is a local minimizer in the interior of R, then \newline
\indent $ (i)$ $g(x^*) = 0$ \newline
\indent $ (ii)$ $d^TH(x^*)d \geq 0$ for all $d \neq 0$

\textbf{Second-order sufficient conditions for a minimum} \newline %Theorem 2.5
If $f(x) \in C_2$
and $x^*$ is located in the interior of $\mathcal{R}$, then the conditions
(a) $g(x^*) = 0$
(b) $H(x^*)$ is positive definite
are sufficient for $x^*$ to be a strong local minimizer. \newline

\textbf{Definition 2.6} \newline
A point $\bar{x} \in \mathcal{R},$ where $\mathcal{R}$ is the feasible region, is said to be a saddle point if \newline
(a) $g(\bar{x}) = 0$ \newline
(b) point $\bar{x}$ is neither a maximizer nor a minimizer. \hfill \break \newline
Stationary points can be located and classified as follows: \newline
1. Find the points $x_i$ at which $g(x_i) = 0$. \newline
2. Obtain the Hessian $H(x_i)$. \newline
3. Determine the character of $H(x_i)$ for each point $x_i$. \newline
If $H(x_i)$ is positive (or negative) definite, $x_i$ is a minimizer (or maximizer);
if $H(x_i)$ is indefinite, $x_i$ is a saddle point.

\textbf{Techniques to compute Hessian P.D. , N.D. } \newline
Eigenvalues: det $ (\lambda I -A) = 0$ Multiplying all eigenvalues is equal to the determinant. \newline
The leading principal minors of a matrix A or its negative $-A$ can be used to
establish whether the matrix is positive or negative definite whereas the principal
minors of A or $-A$ can be used to establish whether the matrix is positive or
negative semidefinite. \newline

\textbf{Theorem 2.9 Properties of matrices} \newline
(a) If \textbf{H} is positive semidefinite or positive definite, then
det $\mathbf{H} \geq 0  \ \text{or} > 0$ \newline
(b) \textbf{H} is positive definite if and only if all its leading principal minors are
positive, i.e., det $\mathbf{H_i} > 0$ for $ i = 1, 2, \cdots , n.$ \newline
(c) \textbf{H} is positive semidefinite if and only if all its principal minors are nonnegative, i.e., det $(H_i^{(l)}) \geq 0$ for all possible selections of $\{l_1, l_2, \cdots , l_i \}$
for $i = 1, 2, \cdots, n$. \newline
(d) \textbf{H} is negative definite if and only if all the leading principal minors of
$-\mathbf{H}$ are positive, i.e., $det ( -H_i) > 0$ for $i = 1, 2, \cdots, n$. \newline
(e) \textbf{H} is negative semidefinite if and only if all the principal minors of -$\mathbf{H}$
are nonnegative, i.e., det $(-H_i^{(l)}) \geq 0$ for all possible selections of
$\{l_1, l_2, \cdots , l_i \}$ for $i = 1, 2, \cdots , n$. \newline
(f) \textbf{H} is indefinite if neither (c) nor (e) holds.

\textbf{Definition 2.7} \newline
A set $\mathcal{R}_c
\subset E_n $ is said to be convex if for every pair of points $x_1, x_2 \subset R_c$
and for every real number $\alpha$ in the range $0 < \alpha < 1$, the point
$x = \alpha x_1 + (1 - \alpha)x_2$
is located in $\mathcal{R}_c
$, i.e., $x \in \mathcal{R}_c$. \newline

\textbf{Definition 2.8} \newline
(a) A function $f(x)$ defined over a convex set $\mathcal{R}_c$  is said to be convex if for
every pair of points $x_1, x_2 \in \mathcal{R}_c $ and every real number $\alpha$ in the range $0 < \alpha < 1$, the inequality
$f[\alpha x_1 + (1 - \alpha)x_2] \leq \alpha f(x_1) + (1 - \alpha)f(x_2)$
holds. If $x_1 \neq x_2$ and
$f[\alpha x_1 + (1 - \alpha)x_2] < \alpha f(x_1) + (1 - \alpha)f(x_2)$
then f(x) is said to be strictly convex. \newline
(b) If $\phi(x)$ is defined over a convex set $\mathcal{R}_c$ and f(x) = -$\phi(x)$ is convex, then $\phi(x)$ is said to be concave. If f(x) is strictly convex, $\phi(x)$ is strictly concave. \newline

\textbf{Property of convex functions relating to the Hessian} A function $f(x) \in C^2$is convex over a convex set
$\mathcal{R}_c$ if and only if the Hessian H(x) of
f(x) is positive semidefinite for $x \in \mathcal{R}_c.$  \newline

\textbf{Theorem 2.15 Relation between local and global minimizers in convex functions} \newline
If $f(x)$ is a convex function defined on a convex set $\mathcal{R}_c$, then  \newline
(a) the set of points $S_c$ where $f(x)$ is minimum is convex;  \newline
(b) any local minimizer of $f(x)$ is a global minimizer.

\section{Ch. 4}
\textbf{Dichotomous Search} \newline
Two function evaluations per iteration. \newline
A \textbf{unimodal function} on an interval has exactly
one point where a maximum or minimum
occurs in the interval. \newline
Consider a unimodal function which is known to have a minimum in the interval $[x_L, \ x_U]$. This interval is said to be the range of uncertainty. \newline
In this method, f(x) is evaluated at two points $x_a =
x_1 - \epsilon/2$ and $x_b = x_1 +\epsilon/2$ where $\epsilon$ is a small positive number. Then depending
on whether $f(x_a) < f(x_b)$ or $f(x_a) > f(x_b)$, range $x_L$ to $x_1 + \epsilon/2$ or $x_1 - \epsilon/2$
to $x_U$ can be selected and if $f(x_a) = f(x_b)$ either will do fine. If we assume
that $x_1 - x_L = x_U - x_1$, i.e., $x_1 = (x_L + x_U)/2$, the region of uncertainty
is immediately reduced by half. The same procedure can be repeated for the
reduced range, that is, f(x) can be evaluated at $x_2 - \epsilon/2$ and $x2 + \epsilon/2$ where $x_2$ is located at the center of the reduced range, and so on. \newline \newline
%\begin{Figure}
%	\centering
%	\includegraphics[width=\linewidth]{disearch}
%\end{Figure}
\textbf{Algorithm 4.1 Fibonacci} \newline
Computing n =$I_n=\frac{I_1}{F_n}$, function evaluations = n-1 \newline
\textbf{Step 1} \newline
Input $x_{L,1}, x_{U,1}$, and n. \newline
\textbf{Step 2} \newline
Compute $F_1, F_2, \cdots, F_n$ using Eq. (4.4). \newline
\textbf{Step 3} \newline
Assign $I_1 = x_{U,1} - x_{L,1}$ and compute
\begin{align*}
& I_2 = \frac{F_{n -1}}{F_n}I_1 (\text{see Eq. (4.6)})\\
& x_{a,1} = x_{U,1} - I_2, \quad x_{b,1} = x_{L,1} + I_2\\
& f_{a,1} = f(x_{a,1}), \quad f_{b,1} = f(x_{b,1})
\end{align*}
Set k = 1. \newline
\textbf{Step 4} \newline
Compute $I_{k+2}$ using Eq. (4.6).
If $f_{a,k} \geq f_{b,k}$, then update Eqs. (4.7) to (4.12) using
\begin{align*}
& x_{L,k+1}=x_{a,k} \\
& x_{U,k+1}=x_{U,k} \\
& x_{a,k+1}=x_{b,k} \\
& x_{b,k+1}=x_{L,k+1}+I_{k+1} \\
& f_{a,k+1}=f_{b,k} \\
& f_{b,k+1}=f(x_{b,k+1})
\end{align*}
using  Otherwise, if $f_{a,k} < f_{b,k}$, update
information using Eqs. (4.13) to (4.18) using \newline
\begin{align*}
& x_{L,k+1}=x_{L,k} \\
& x_{U,k+1}=x_{b,k} \\
& x_{a,k+1}=x_{U,k+1}-I_{k+2} \\
& x_{b,k+1}=x_{a,k} \\
& f_{a,k+1}=f(x_{a,k+1}) \\
& f_{b,k+1}=f_{a,k}
\end{align*}

\textbf{Step 5} \newline
If $k = n - 2$ or $x_{a,k+1} > x_{b,k+1}$, output $x^* = x_{a,k+1}$ and $f^* = f(x^*)$,
and stop. Otherwise, set $k = k + 1$ and repeat from Step 4.
The condition $x_{a,k+1} > x_{b,k+1}$ implies that $x_{a,k+1} \approx x_{b,k+1}$ within the precision of the computer used, as was stated earlier, or that there is an error in the algorithm. It is thus used as an alternative stopping criterion. \newline

\textbf{Algorithm 4.2 Golden-section search} \newline
(function evaluations = k+1) and Golden Ratio: $K=\cfrac{1+\sqrt{5}}{2}$
$\Lambda_{GS} = I_n = \frac{I_1}
{K_{n-1}}$ \quad $\Lambda_{F} = I_n = \frac{I_1}
{F_n} \approx \frac{\sqrt{5}}
{K^{n+1}}I_1$
$\frac{I_k}{I_{k+1}}=\frac{I_{k+1}}{I_{k+2}}
= \frac{I_{k+2}}{I_{k+3}}
= \cdots = K$ \newline
\textbf{Step 1} \newline
Input $x_{L,1}, x_{U,1},$ and $\epsilon$. \newline
\textbf{Step 2} \newline
Assign $I_1 = x_{U,1} - x_{L,1}, K = 1.618034$ and compute
\begin{align*}
&I_2 = I_1/K \\
&x_{a,1} = x_{U,1} - I_2, \quad x_{b,1} = x_{L,1} + I_2 \\
&f_{a,1} = f(x_{a,1}), \quad f_{b,1} = f(x_{b,1})
\end{align*}
Set $k = 1$. \newline
\textbf{Step 3} \newline
Compute $I_{k+2} = I_{k+1}/K$ \newline
If $f_{a,k} \geq f_{b,k}$, then update $x_{L,k+1}, x_{U,k+1}$, $x_{a,k+1}$, $x_{b,k+1}$, $f_{a,k+1}$,
and $f_{b,k+1}$ as
\begin{align*}
& x_{L,k+1}=x_{a,k} \\
& x_{U,k+1}=x_{U,k} \\
& x_{a,k+1}=x_{b,k} \\
& x_{b,k+1}=x_{L,k+1}+I_{k+1} \\
& f_{a,k+1}=f_{b,k} \\
& f_{b,k+1}=f(x_{b,k+1})
\end{align*}
Or use using Eqs. (4.7) to (4.12).
Otherwise if $f_{a,k} < f_{b,k}$, then update $x_{L,k+1}, x_{U,k+1}$, $x_{a,k+1}$, $x_{b,k+1}$, $f_{a,k+1}$, and $f_{b,k+1}$ as
\begin{align*}
& x_{L,k+1}=x_{L,k} \\
& x_{U,k+1}=x_{b,k} \\
& x_{a,k+1}=x_{U,k+1}-I_{k+2} \\
& x_{b,k+1}=x_{a,k} \\
& f_{a,k+1}=f(x_{a,k+1}) \\
& f_{b,k+1}=f_{a,k}
\end{align*}
Otherwise, if $f_{a,k} < f_{b,k}$, update
information using Eqs. (4.13) to (4.18). \newline
\textbf{Step 4} \newline
If $I_k < \epsilon$ or $x_{a,k+1} > x_{b,k+1}$, then do: \newline
If $f_{a,k+1} > f_{b,k+1}$, compute
$x^* = 0.5(x_{b,k+1} + x_{U,k+1})$ \newline
If $f_{a,k+1} = f_{b,k+1}$, compute
$x^* = 0.5(x_{a,k+1} + x_{b,k+1})$ \newline
If $f_{a,k+1} < f_{b,k+1}$, compute
$x^* = 0.5(x_{L,k+1} + x_{a,k+1})$
Compute $f^* = f(x^*).$ \newline
Output $x^*$ and $f^*$, and stop. \newline
\textbf{Step 5} \newline
Set $k = k + 1$ and repeat from Step 3. \newline \newline
\textbf{Equations 4.4, 4.6, (4.7-4.12) and (4.13-4.18) } \newline
$F_k = F_{k-1} + F_{k-2} \quad$ for $k \geq 2$ (4.4) \newline
$I_{k+2} = \frac{F_{n-k-1}}{F_{n-k}}I_{k+1} (4.6)$

If $f_{a,k} > f_{b,k}$, then $x^*$ is in interval $[x_{a,k}, x_{U,k}]$ and so the new bounds of $x^* \rightarrow$  %\newline
$x_{L,k+1} = x_{a,k} (4.7) \quad x_{U,k+1} = x_{U,k} (4.8)$
Similarly, the two interior points of the new interval, namely, $x_{a,k+1}$ and $x_{b,k+1}$
will be $x_{b,k}$ and $x_{L,k+1} + I_{k+2}$, respectively. We can thus assign
$x_{a,k+1} = x_{b,k}$ (4.9) $x_{b,k+1} = x_{L,k+1} + I_{k+2}$ (4.10)
as illustrated in Fig. 4.5. \newline
The value $f_{b,k}$ is retained as the value of f(x) at
$x_{a,k+1}$, and the value of f(x) at $x_{b,k+1}$ is calculated, i.e.,
$f_{a,k+1} = f_{b,k}$ (4.11)
$f_{b,k+1} = f(x_{b,k+1})$ (4.12) \newline
%\begin{Figure}
%	\centering
%	\includegraphics[width=\linewidth]{fibfirst}
%\end{Figure}
On the other hand, if $f_{a,k} < f_{b,k}$, then $x^*$ is in interval $[x_{L,k}, x_{b,k}]$. In this
case, we assign
$x_{L,k+1} = x_{L,k}$ (4.13)
$x_{U,k+1} = x_{b,k}$ (4.14)
$x_{a,k+1} = x_{U,k+1} - I_{k+2}$ (4.15)
$x_{b,k+1} = x_{a,k}$ (4.16)
$f_{b,k+1} = f_{a,k}$ (4.17)
and calculate
$f_{a,k+1} = f(x_{a,k+1})$ (4.18) \newline  \newline
%\begin{Figure}
%	\centering
%	\includegraphics[width=\linewidth]{fibsecond}
%\end{Figure}
%\begin{Figure}
%	\centering
%	\includegraphics[width=\linewidth]{golden}
%\end{Figure}
%\begin{Figure}
%	\centering
%	\includegraphics[width=\linewidth]{fib}
%\end{Figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Unlikely to be tested %%
%% INCLUDE FOR FINAL %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\textbf{Algorithm 4.6 Inexact line search}  \newline
\textbf{Step 1:} \newline
Input $x_k$, $d_k$, and compute $g_k$. \newline
Initialize algorithm parameters $\rho, \sigma, \tau$, and $\chi$. \newline
Set $\alpha_L = 0$ and $\alpha_U = 10^{99}$. \newline
\textbf{Step 2:} \newline
Compute $f_L = f(x_k + \alpha_L dk)$. \newline
Compute $f_L^\prime = g(x_k + \alpha_L dk)^Tdk$. \newline
\textbf{Step 3:} \newline
Estimate $\alpha_0$. \newline
\textbf{Step 4:} \newline
Compute $f_0 = f(x_k + \alpha_0dk)$. \newline
\textbf{Step 5 (Interpolation)} \newline
If $f_0 > f_L + \rho(\alpha_0 - \alpha_L)f_L^\prime$ , then do: \newline
a. If $\alpha_0 < \alpha_U$, then set $\alpha_U = \alpha_0$.\newline
b. Compute $\breve{\alpha}_0$ using the interpolation formula Eq. (4.57). %\newline
\[
\breve{\alpha}_0 =\alpha_L + \frac{(\alpha_0-\alpha_L)^2f_L^\prime}{2[f_L-f_0+(\alpha_0-\alpha_L)f_L^\prime]}
\]
c. If $\breve{\alpha}_0 < \alpha_L + \tau(\alpha_U - \alpha_L)$ then set $\breve{\alpha}_0 = \alpha_L + \tau(\alpha_U - \alpha_L)$. \newline
d. If $\breve{\alpha}_0 > \alpha_U - \tau(\alpha_U - \alpha_L)$ then set $\breve{\alpha}_0  = \alpha_U - \tau(\alpha_U - \alpha_L)$. \newline
e. Set $\alpha_0 = \breve{\alpha}_0$ and go to Step 4. \newline
\textbf{Step 6} \newline
Compute $f_0^\prime = g(x_k + \alpha_0dk)^Tdk$. \newline
\textbf{Step 7 (Extrapolation)} \newline
If $f_0^\prime < \alpha f_L^\prime$ , then do: \newline
a. Compute $\nabla \alpha_0 = (\alpha_0 - \alpha_L)f_0^\prime /(f_L^\prime - f_0^\prime)$ (see Eq. (4.58)).
\[
\breve{\alpha}_0 = \alpha_0 + (\alpha_0 - \alpha_L)f_0^\prime
(f_L^\prime - f_0^\prime) \quad (Eq. (4.58))
\]
b. If $\nabla \alpha_0 < \tau(\alpha_0 - \alpha_L)$, then set $\nabla \alpha_0 = \tau(\alpha_0 - \alpha_L)$. \newline
c. If $\nabla \alpha_0 > \chi(\alpha_0 - \alpha_L)$, then set $\nabla \alpha_0 = \chi(\alpha_0 - \alpha_L)$. \newline
d. Compute $\breve{\alpha}_0 = \alpha_0 + \nabla \alpha_0$. \newline
e. Set $ \alpha_L =  \alpha_0, \alpha_0 = \breve{\alpha}_0, f_L = f_0, f_L^\prime = f_0^\prime$, and go to Step 4. \newline
\textbf{Step 8} \newline
Output $\alpha_0$ and $f_0 = f(x_k + \alpha_0dk)$, and stop.

\section{Ch. 5}
Standard form: $f(x)=\frac{1}{2}x^T H x + x^T g(x) + C$ \newline
Rate of Convergence $\beta = (1-r^2)/(1+r^2)$, where r is the smallest eigenvalue divided by the biggest eigenvalue. \newline
\begin{align*}
& H^{-1}=\begin{bmatrix}
a & c \\
c & b
\end{bmatrix}^{-1}
=
\frac{1}{ab-c^2}
\begin{bmatrix}
b & -c \\
-c & a
\end{bmatrix} \quad ab-c^2 \neq 0
\end{align*}
$[f(x_k)-f(x^*)]\leq \left(\frac{1-r}{1+r}\right)^2[f(x_k)-f(x^*)]$ \newline
\textbf{Algorithm 5.1 Steepest-descent algorithm} \newline
\textbf{Step 1}  \newline
Input $x_0$ and initialize the tolerance $\epsilon$. \newline
Set $k = 0$. \newline
\textbf{Step 2} \newline
Calculate gradient $g_k$ and set $d_k=-g_k$. \newline
\textbf{Step 3} \newline
Find $\alpha_k$, the value of $\alpha$ that minimizes $f(x_k + \alpha d_k)$, using a line search (Algorithm Inexact Line Search. 4.6). \newline
\textbf{Step 4} \newline
Set $x_{k+1} = x_k + \alpha_k d_k$ and calculate $f_{k+1} = f(x_{k+1})$. \newline
\textbf{Step 5}  \newline
If $||\alpha_k d_k|| < \epsilon$, then do: \newline
Output $x^* = x_{k+1}$ and $f(x^*) = f_{k+1}$, and stop. \newline
Otherwise, set $k = k + 1$ and repeat from Step 2. \newline

\textbf{Algorithm 5.3 Basic Newton algorithm} \newline
\textbf{Step 1} \newline
Input $x_0$ and initialize the tolerance $\epsilon$ \newline
Set $k = 0$. \newline
\textbf{Step 2} \newline
Compute $g_k$ and $H_k$. \newline
If $H_k$ is not positive definite, force it to become positive definite. \newline
\textbf{Step 3} \newline
Compute $H_k^{-1}$ and $d_k=-H_k^{-1}g_k$ \newline
\textbf{Step 4} \newline
Find $\alpha_k$ , the value of $\alpha$ that minimizes $f(x+\alpha d_k)$, using a line search. \newline
\textbf{Step 5} \newline
Set $x_{k+1}=x_k+\alpha_k d_k$ \newline
Compute $f_{k+1}=f(x_{k+1})$. \newline
\textbf{Step 6} \newline
If $||\alpha_k d_k|| < \epsilon$, then do:  \newline
Output $x^*=x_{k+1}$ and $f(x^*)=f(x_{k+1})$, and stop \newline
Otherwise, set $k = k + 1$ and repeat from Step 2. \newline
\textbf{Algorithm 5.5 Gauss --- Newton Algorithm} \newline
$f=\begin{bmatrix}
f_1(x)  & f_2(x) & \cdots f_m(x)
\end{bmatrix}^T$, J = Jacobian \newline
$F(x)=\sum_{p=1}^{m}f_p(x)^2=f^Tf$ \newline
$J=\begin{bmatrix}
\frac{ \partial f_1}{\partial  x_1} & \frac{ \partial  f_1}{ \partial  x_2} & \cdots & \frac{\partial  f_1}{\partial  x_n} \\
\frac{\partial  f_2}{\partial x_1} & \frac{\partial  f_2}{\partial  x_2} & \cdots & \frac{\partial  f_2}{\partial x_n} \\
\vdots & \vdots & \ddots & \vdots \\
\frac{\partial f_m}{\partial  x_1}& \frac{\partial f_m}{\partial  x_2}& \cdots & \frac{\partial f_m}{\partial x_n}
\end{bmatrix}$ \newline
\textbf{Step 1} \newline
Input $x_0$ and initialize the tolerance $\epsilon$.  \newline
Set $k = 0$. \newline
\textbf{Step 2} \newline
Compute $f_{pk} = f_p(x_k)$ for $p = 1, 2, \cdots, m$ and $F_k$.
\newline
\textbf{Step 3}  \newline
Compute $J_k, g_k = 2J^T_k f_k$, and $H_k = 2J^T_k J_k$. \newline
\textbf{Step 4}  \newline
$d_k=-H_k^{-1}g_k$ \newline
\textbf{Step 5} \newline
Find $\alpha_k$, the value of $\alpha$ that minimizes $f(x_k + \alpha d_k)$. \newline
\textbf{Step 6} \newline
Set $x_{k+1} = x_k + \alpha_kd_k$. \newline
Compute $f_{p,k+1}$ for $p = 1, 2,\cdots, m$ and $F_{k+1}$. \newline
\textbf{Step 7} \newline
If $|F_{k+1}-F_k | < \epsilon$, then do: \newline
Output $x^*= x_{k+1}$, $f_{p,k+1}(x^*)$ for $p = 1, 2,\cdots, m$, and $F_{k+1}$. \newline
Stop. \newline
Otherwise, set $k = k + 1$ and repeat from Step 3.

\section{Ch. 7}
\textbf{Problems with Rank-one Method}
\begin{enumerate}
	\item positive definite $S_k$ may not yield positive definite $S_{k+1}$
	\item denominator in correction formula may approach zero
\end{enumerate}
THE DFP and BFGS are implementing of the basic algorithms Quasi Newton (7.2) with changes to the updating function. $d_k=-S_kg_k$ and $f(x_k + \alpha d_k)$  $\rightarrow$  $\alpha_k=\frac{g_k^TS_kg_k}
{g_k^TS_kHS_kg_k}$ \newline
Convergence equation: $\beta = \left(\frac{1-r}{1+r}\right)^2$ $f(x_{k+1})-f(x^*) \leq \left(\frac{1-r}{1+r}\right)^2[f(x_k)-f(x^*)]$ \newline
\textbf{BFGS and then DFP properties} \newline
For convex quadratic functions (BFGS)
\begin{enumerate}
	\item[---] $Sk+1$ becomes identical to $H^{-1}$ for $k = n-1$.
	\item[---] Directions $\delta_0,\delta_1,\cdots,\delta_{n-1}$ form a conjugate set.
	\item[---] $S_{k+1}$ is positive definite if $S_k$ is positive definite.
	\item[---] $\delta^T_k \gamma_k = \delta^T_k g_{k+1}-\delta_T^k g_k > 0$ applies. \newline
\end{enumerate}
\textbf{For DFP (from textbook)}
\begin{enumerate}
	\item If $S_k$ is PD, then
	the matrix $S_{k+1}$ generated by DFP is also PD.
	\item Directions $\delta_0,\delta_1,\cdots,\delta_{n-1}$ form a conjugate set. \newline
\end{enumerate}
\textbf{Algorithm 7.2 adjusted for DFP/BFGS} \newline
\textbf{Step 1} \newline
Input $x_0$  and initialize the tolerance $\epsilon$. \newline
Set $k = 0$ and $S_0 = I_n$. \newline
Compute $g_0$. \newline
\textbf{Step 2} \newline
Set $d_k=-S_kg_k$ \newline
Find $\alpha_k$, the value of $\alpha$ that minimizes $f(x_k+\alpha d_k)$, using a line search \newline
Set $\delta_k=\alpha_k d_k$ and $x_{k+1}=x_k+\delta_k$ \newline
\textbf{Step 3} \newline
If $||\delta_k|| < \epsilon$, output $x^* = x_{k+1}$ and $f(x^*) = f(x_{k+1})$, and stop \\
\textbf{Step 4} \newline
Compute $g_{k+1}$ and set $\gamma_k=g_{k+1}-g_k$ \newline
Compute $S_{k+1}$ using appropriate formula.
\begin{align*}
&\text{Basic/ Rank One:} \quad S_{k+1}=S_k+\frac{(\delta_k-S_k \gamma_k)(\delta_k-S_k \gamma_k)^T}{\gamma_k^T(\delta_k-S_k\gamma_k)}\\
&\text{DFP:}\quad S_{k+1}=S_k+\frac{\delta_k \delta_k^T}{\delta_k^T \gamma_k}-\frac{S_k\gamma_k\gamma_k^TS_k}{\gamma_k^TS_k\gamma_k}\\
&\text{BFGS:} \quad S_{k+1}=S_k+\left(1+\frac{\gamma_k^TS_k\gamma_k}{\gamma_k^T\delta_k}\right)\frac{\delta_k\delta_k^T}{\gamma_k^T\delta_k}-\frac{(\delta_k\gamma_k^TS_k+S_k\gamma_k\delta_k^T)}{\gamma_k^T\delta_k}
\end{align*}
Set $k=k+1$ and repeat from Step 2. \newline
\begin{itemize}
	\item[---] remember that $\left(1+\frac{\gamma_k^TS_k\gamma_k}{\gamma_k^T\delta_k}\right)$ is a single number
	\item[---] $\delta_k\delta_k^T$ is a matrix.
	\item[---] Focus on minimization, $\max[f(x)] =-\min[-f(x)]$
	\item[---] Hessian is positive semidefinite for concave functions.
\end{itemize}
\end{multicols}


\section{What is machine learning?} \label{ch:what_is_ml}

\subsection{Machine learning in general}

Usually the task of programming is to apply rules to certain problems in order to obtain solutions.
With the help of machine learning we want to create a model that uses solutions for certain problems to find out the corresponding rules.
The relationship is shown in figure~\ref{fig:cp_vs_ml}.

\begin{figure}
    \centering
    \includegraphics[width=0.5\textwidth]{images/classical_prog_vs_ml.png}
    \caption[ML, a new programming paradigm]{Machine learning, a new programming paradigm \cite[p.5]{Chollet2017}}
    \label{fig:cp_vs_ml}
\end{figure}

The promise of machine learning is that the rules learned can be much more complex than it is possible to program by hand.
A good example is image recognition (which will be the focus of this project), where the content of images is to be classified.
The relations between the pixels have to be found out, which leads to a rather complex model.
This cannot be programmed in the traditional way even for the simplest tasks.

The universal approximation theorem \cite{Cybenko1989, Hornik1989} states that for any input $x$ there is a function $h$ which approximates the mapping $y$, while the mapping is only the input to the correct output (often labeled by a human).
This relationship is described in equation~\eqref{eq:universal_approx}.

% TODO define sets X and Y of \theta and x
\begin{equation}
    \forall \epsilon > 0 :
    \exists h(\theta, x) : \forall x \in I : | h(\theta, x) - y(x) | < \epsilon
    \label{eq:universal_approx}
\end{equation}
% TODO the text here can be changed with a class of functions (see UAT paper)

The mathematical description of $h$ is called a model (e.g. $h(x) = \sum_{i=1}^n{\lambda (\theta^T * x)}$).
$\lambda \in \mathbb{R}$ is a constant and $\theta$ is a tensor that defines the parameters of $h$, which are made up of so-called weights and biases, which will be explained later.

The universal approximation theorem makes no statements about the form of $\theta$; therefore $h$ is called a hypothesis because it states that for a certain composition of $\theta$ this equation is true.
This term is summarized in the equation~\eqref{eq:hypothesis}.

\begin{equation}
    \forall \epsilon > 0 : \exists \theta \in X : \forall x \in I : |
    h_\theta(x) - y(x) | < \epsilon
    \label{eq:hypothesis}
\end{equation}

A loss function is used for model valuation; it compares the result of the hypothesis with the data provided.
Probably the most prominent loss function is the \name{mean square error}.
It takes the sum of the deviation of $n$ examples squared and divides it by $2*n$.

\begin{equation}
    L_{mse}(\theta, x) = \frac{1}{2 n} \sum_{i=1}^n (h_\theta(x) - y(x))^2
    \label{eq:mse}
\end{equation}

The task of machine learning is to determine a $\theta$ for a model that confirms the equations~\ref{eq:universal_approx} and \ref{eq:hypothesis}, which is done by iterative updating of $\theta$.
Before training, $\theta$ is randomly initiated and then iteratively adjusted to make $\epsilon$ converge toward zero. This minimizes the value of $L$.

In order for the value of the loss function to gradually decrease, $\theta$ is adjusted using an optimization function.
This is usually done with a gradient descent algorithm or a variant of it.
The gradient descent is performed by calculating the gradient of the loss function and subtracting it from the corresponding parameters shown in equation~\eqref{eq:gradient_descent}.

\begin{equation}
    \theta_{i+1} := \theta_i - \eta \nabla_\theta L(\theta, x)
    \label{eq:gradient_descent}
\end{equation}

Where $\eta$ is a constant called the learning rate, which helps the loss function converge to 0 using the gradient descent algorithm.
It is one of many hyperparameters that are not automatically adjusted during training, but are determined beforehand.
Optimal adjustment of hyperparameters is an important task that is difficult to automate and is therefore the focus of much research\footnote{There are algorithms for adjusting the learning rate, namely \name{AdaGrad} \cite{Duchi2010} and derived algorithms and is the main topic of the chapter \ref{ch:hyper_parameter_tuning}}.
If the equation \eqref{eq:hypothesis} applies to the model in question, it is then sufficiently suitable for the given input.

These concepts are presented now using a simple example.

\subsection{Simple linear example} \label{ch:simple_linear_example}

As an illustrative example, a model is created that translates Fahrenheit into Celsius\footnote{Please note that using a learning algorithm here is very inefficient and is in contrast to Maslow's hammer (\aka{https://en.wikipedia.org/wiki/Law_of_the_instrument}).}.
The original equation is given by the linear function $y = mx + b$ as $F = C * 1.8 + 32$, with $F$ as degrees Fahrenheit and $C$ as degrees Celsius.

A model to learn this relationship is defined in the listing~\ref{lst:c_to_f} \footnote{Also available at \aka{https://github.com/klawr/deepmech/tree/master/reports/srp/code/c_to_f.js}} .

% TODO add JavaScript as language
\lstinputlisting[label={lst:c_to_f}, caption={Celsius to Fahrenheit}]{code/c_to_f.js}

The described model is implemented by \code{w} and the hypothesis function by \code{h}.
Since it is known that the relationship is linear, the model represents a one-dimensional polynomial, with the first index being dimensionless (and, as mentioned above, emulating the bias) and the second being used as input x.
We will add a dimension later to demonstrate the behavior with polynomials that do not directly represent the target function.

\code{y} provides the correct, predefined solution.
Please note that \code{y} is only used during training and is omitted if the model parameters are correctly set after training.

In this example the loss function is described by the \textit{mean square error } \footnote{ Abbreviated as \textit{mse}, which is shown in equation \eqref{eq:mse}}.

To minimize the loss function gradient descent described in equation~\eqref{eq:gradient_descent} is used.

In listing~\ref{lst:c_to_f} equation~\eqref{eq:gradient_descent} is implemented as \code{sgd} with $\theta_0 = \code{b}$ and $\theta_1= \code{m}$ as shown in equation~\eqref{eq:sgd_mse_here}.

\code{sgd} is short for \textit{stochastic gradient descent}\footnote{A distinction is made between batch, mini-batch and stochastic gradient descent; using a complete data set, a defined subset or individual data on the individual iterations of the training process}.
It represents the optimization function that updates the parameters of the model by calculating the gradient of loss for each parameter and then updating it.

\begin{equation}
    \begin{split}
        \theta_{0} & := \theta_{0} - \frac{\partial L}{\partial \theta_{0}} =
        \theta_{0} - (h(x) - y(x))  \\
        \theta_{1} & := \theta_{1} - \frac{\partial L}{\partial \theta_{1}} =
        \theta_{0} - (h(x) - y(x)) * x
    \end{split}
    \label{eq:sgd_mse_here}
\end{equation}

\begin{lstlisting}[caption={Output of C to F converter.}]
loss:  343041.666256673
loss:  16242.7707476548
loss:  88.27595280422442
loss:  10.74432403839251
loss:  1.1607137779951606
loss:  0.252186283500194
loss:  0.00040450626910586374
loss:  0.00004247033303206121
loss:  7.158429555712516e-7
loss:  1.6948019668728757e-10
weights:  [ 31.99999973758735, 1.8000003977458756 ]
\end{lstlisting}

The expected behavior for the loss function is to decrease with each iteration.
This is the case in this example, showing that the deviation of the hypothesis from the actual result decreases.
After 200 iterations, the parameters \code{m} and \code{b} of the \code{model} are displayed and show that they are actually close to the expected result.
By increasing the number of iterations, the result can be increased up to the point where they are rounded by the JavaScript compiler to represent exact results (using Node.js\footnote{As far as can be assumed when calculating exact results using floating point numbers.}).

\begin{SCfigure}
    \centering
    \caption[Simplest model]{This example can be visualized as shown here. The bias can be interpreted as additional input, which always has the value 1. The input is multiplied by the parameters marked with the corresponding arrows, which in turn are summed up (resulting in $y = mx + b$).}
    \includegraphics[width=0.45\textwidth]{images/1_simplest_nn.png}
\end{SCfigure}

Provided that this hypothesis is initially well suited to the task, it is important to note that by modifying the hypothesis to represent a polynomial of higher degree (e.g. $y = nx^2 + mx + b$) the additional parameters converge towards 0, which effectively gives the same result as before \footnote{But to achieve similar results, the iteration number must be increased by at least a factor of 20}.

The experiment can be followed at \aka{https://github.com/klawr/deepmech/tree/master/reports/srp/code/c_to_f_adv.js}.


\chapter{Lebesgue integration}
On any measure space $(\Omega, \SA, \mu)$ we can then,
for a function $f \colon \Omega \to [0,\infty]$
define an integral
\[ \int_\Omega f \; d\mu. \]
This integral may be $+\infty$ (even if $f$ is finite).
As the details of the construction won't matter for us later on,
we will state the relevant definitions,
skip all the proofs,
and also state all the properties that we actually care about.
Consequently, this chapter will be quite short.

\section{The definition}
The construction is done in four steps.
\begin{definition}
	If $A$ is a measurable set of $\Omega$,
	then the \vocab{indicator function}
	$\mathbf{1}_A \colon \Omega \to \RR$ is defined by
	\[ \mathbf{1}_A(\omega) = \begin{cases}
			1 & \omega \in A \\
			0 & \omega \notin A.
		\end{cases} \]
\end{definition}

\begin{step}
	[Indicator functions]
	For an indicator function, we require
	\[ \int_\Omega \mathbf{1}_A \; d\mu \defeq \mu(A) \]
	(which may be infinite).
\end{step}
We extend this linearly now for nonnegative functions
which are sums of indicators:
these functions are called \vocab{simple functions}.
\begin{step}
	[Simple functions]
	Let $A_1$, \dots, $A_n$ be a finite collection of measurable sets.
	Let $c_1$, \dots, $c_n$ be either nonnegative real numbers or $+\infty$.
	Then we define
	\[ \int_\Omega \left( \sum_{i=1}^n c_i \mathbf{1}_{A_i} \right) \; d\mu
		\defeq \sum_{i=1}^n c_i \mu(A_i). \]
	If $c_i = \infty$ and $\mu(A_i) = 0$, we treat $c_i \mu(A_i) = 0$.
\end{step}
One can check the resulting sum does not depend
on the representation of the simple function as $\sum c_i \mathbf{1}_{A_i}$.
In particular, it is compatible with the previous step.

Conveniently, this is already enough to define the integral
for $f \colon \Omega \to [0, +\infty]$.
Note that $[0,+\infty]$ can be thought of as a topological space
where we add new open sets $(a,+\infty]$ %chktex 9
for each real number $a$ to our usual basis of open intervals.
Thus we can equip it with the Borel sigma-algebra.\footnote{We
	\emph{could} also try to define a measure on it,
	but we will not: it is a good enough for us
	that it is a measurable space.}
\begin{step}
	[Nonnegative functions]
	For each measurable function $f \colon \Omega \to [0, +\infty]$, let
	\[ \int_\Omega f \; d\mu \defeq
		\sup_{0 \le s \le f} \left( \int_\Omega s \; d\mu \right) \]
	where the supremum is taken over all \emph{simple} $s$
	such that $0 \le s \le f$.
	As before, this integral may be $+\infty$.
\end{step}
One can check this is compatible with the previous definitions.
At this point, we introduce an important term.
\begin{definition}
	A measurable (nonnegative) function $f \colon \Omega \to [0, +\infty]$
	is \vocab{absolutely integrable}
	or just \vocab{integrable} if $\int_\Omega f \; d\mu < \infty$.
\end{definition}
Warning: I find ``integrable'' to be \emph{really} confusing terminology.
Indeed, \emph{every} measurable function from $\Omega$ to $[0,+\infty]$
can be assigned a Lebesgue integral, it's just that
this integral may be $+\infty$.
So the definition is far more stringent than the name suggests.
Even constant functions can fail to be integrable:
\begin{example}
	[We really should call it ``finitely integrable'']
	The constant function $1$ is \emph{not} integrable on $\RR$,
	since $\int_\RR 1 \; d\mu = \mu(\RR) = +\infty$.
\end{example}
For this reason, I will usually prefer the term ``integrable''.
(If it were up to me, I would call it ``finitely integrable'',
and usually do so privately.)

Finally, this lets us integrate general functions.
\begin{definition}
	In general, a measurable function $f \colon \Omega \to [-\infty, \infty]$
	is \vocab{absolutely integrable} or just \vocab{integrable} if $|f|$ is.
\end{definition}
Since we'll be using the first word, this is easy to remember:
``absolutely integrable'' requires taking absolute values.

\begin{step}
	[Absolutely integrable functions]
	If $f \colon \Omega \to [-\infty, \infty]$ is absolutely integrable,
	then we define
	\begin{align*}
		f^+(x) &= \max\left\{ f(x), 0 \right\} \\
		f^-(x) &= \min\left\{ f(x), 0 \right\} \\
	\end{align*}
	and set
	\[ \int_\Omega f \; d\mu = \int_\Omega |f^+| \; d\mu
		- \int_\Omega |f^-| \; d\mu \]
	which in particular is finite.
\end{step}
You may already start to see that we really like nonnegative functions:
with the theory of measures, it is possible to integrate them,
and it's even okay to throw in $+\infty$'s everywhere.
But once we start dealing with functions that can be either positive or negative,
we have to start adding finiteness restrictions ---
actually essentially what we're doing is splitting
the function into its positive and negative part,
requiring both are finite, and then integrating.

To finish this section, we state for completeness
some results that you probably could have guessed were true.
Fix $\Omega = (\Omega, \SA, \mu)$, and
let $f$ and $g$ be measurable real-valued functions
such that $f(x) = g(x)$ almost everywhere.
\begin{itemize}
	\ii (Almost-everywhere preservation)
	The function $f$ is absolutely integrable if and only if $g$ is,
	and if so, their Lebesgue integrals match.
	\ii (Additivity)
	If $f$ and $g$ are absolutely integrable then
	\[ \int_\Omega f+g \; d\mu
		= \int_\Omega f \; d\mu
		+ \int_\Omega g \; d\mu. \]
	The ``absolutely integrable'' hypothesis can be dropped
	if $f$ and $g$ are nonnegative.
	\ii (Scaling) If $f$ is absolutely integrable and $c \in \RR$
	then $cf$ is absolutely integrable and
	\[ \int_\Omega cf \; d\mu = c \int_\Omega f \; d\mu. \]
	The ``absolutely integrable'' hypothesis can be dropped
	if $f$ is nonnegative and $c > 0$.
	\ii (Monotoncity)
	If $f$ and $g$ are absolutely integrable and $f \le g$, then
	\[ \int_\Omega f \; d\mu \le \int_\Omega g \; d\mu. \]
	The ``absolutely integrable'' hypothesis can be dropped
	if $f$ and $g$ are nonnegative.
\end{itemize}
There are more famous results like monotone/dominated convergence
that are also true, but we won't state them here
as we won't really have a use for them in the context of probability.
(They appear later on in a bonus chapter.)

\section{Relation to Riemann integrals (or: actually computing Lebesgue integrals)}
For closed intervals, this actually just works out of the box.
\begin{theorem}
	[Lebesgue integral generalizes Riemann integral]
	Let $f \colon [a,b] \to \RR$ be a Riemann integrable function
	(where $[a,b]$ is equipped with the Borel measure).
	Then $f$ is also Lebesgue integrable and the integrals agree:
	\[ \int_a^b f(x) \; dx = \int_{[a,b]} f \; d\mu. \]
\end{theorem}

Thus in practice, we do all theory with Lebesgue integrals (they're nicer),
but when we actually need to compute $\int_{[1,4]} x^2 \; d\mu$
we just revert back to our usual antics with the
Fundamental Theorem of Calculus.
\begin{example}
	[Integrating $x^2$ over {$[1,4]$}]
	Reprising our old example:
	\[ \int_{[1,4]} x^2 \; d\mu
		= \int_1^4 x^2 \; dx
		= \frac13 \cdot 4^3 - \frac13 \cdot 1^3 = 21.  \]
\end{example}

This even works for \emph{improper} integrals,
if the functions are nonnegative.
The statement is a bit cumbersome to write down, but here it is.
\begin{theorem}
	[Improper integrals are nice Lebesgue ones]
	Let $f \ge 0$ be a \emph{nonnegative}
	continuous function defined on $(a,b) \subseteq \RR$,
	possibly allowing $a = -\infty$ or $b = \infty$.
	Then
	\[ \int_{(a,b)} f \; d\mu
		= \lim_{\substack{a' \to a^+ \\ b' \to b^-}}
		\int_{a'}^{b'} f(x) \; dx \]
	where we allow both sides to be $+\infty$
	if $f$ is not absolutely integrable.
\end{theorem}
The right-hand side makes sense since $[a',b'] \subsetneq (a,b)$
is a compact interval on which $f$ is continuous.
This means that improper Riemann integrals of nonnegative
functions can just be regarded as Lebesgue ones
over the corresponding open intervals.

It's probably better to just look at an example though.
\begin{example}
	[Integrating $1/\sqrt{x}$ on $(0,1)$]
	For example, you might be familiar with improper integrals like
	\[ \int_0^1 \frac{1}{\sqrt x} \; dx
		\defeq \lim_{\eps \to 0^+}
		\int_\eps^1 \frac{1}{\sqrt x} \; dx
		= \lim_{\eps \to 0^+} \left( 2\sqrt{1} - 2\sqrt{\eps} \right) = 2.
	\]
	(Note this appeared before as \Cref{prob:improper}.)
	In the Riemann integration situation, we needed the limit as $\eps \to 0^+$
	since otherwise $\frac{1}{\sqrt x}$ is not defined as a function $[0,1] \to \RR$.
	However, it is a \emph{measurable nonnegative}
	function $(0,1) \to [0,+\infty]$, and hence
	\[ \int_{(0,1)} \frac{1}{\sqrt x} \; d\mu = 2. \]
\end{example}

If $f$ is not nonnegative, then all bets are off.
Indeed \Cref{prob:sin_improper} is the famous counterexample.

\section{\problemhead}

\begin{sproblem}
	[The indicator of the rationals]
	\label{prob:1QQ}
	Take the indicator function
	$\mathbf 1_{\QQ} \colon \RR \to \{0,1\} \subseteq \RR$
	for the rational numbers.
		\begin{enumerate}[(a)]
	   \ii Prove that $\mathbf{1}_\QQ$ is not Riemann integrable.
	   \ii Show that $\int_\RR \mathbf{1}_\QQ$ exists
	   and determine its value --- the one you expect!
   \end{enumerate}
\end{sproblem}

\begin{dproblem}
	[An improper Riemann integral with sign changes]
	\label{prob:sin_improper}
	Define $f \colon (1,\infty) \to \RR$ by $f(x) = \frac{\sin(x)}{x}$.
	Show that $f$ is not absolutely integrable,
	but that the improper Riemann integral
	\[ \int_1^\infty f(x) \; dx \defeq
		\lim_{b \to \infty}
		\int_a^b f(x) \; dx \]
	nonetheless exists.
\end{dproblem}


\documentclass[12pt]{article}

% PACKAGES
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{float}
\usepackage{graphicx}
\usepackage{enumerate}
%\usepackage{mathtools}
\usepackage{hyperref}

% CUSTOM STYLES
%\providecommand{\e}[1]{\ensuremath{\times 10^{#1}}}
%\DeclarePairedDelimiter\abs{\lvert}{\rvert}%

\usepackage{listings}
\usepackage{color}

\definecolor{dkgreen}{rgb}{0,0.6,0}
\definecolor{gray}{rgb}{0.5,0.5,0.5}
\definecolor{mauve}{rgb}{0.58,0,0.82}

\lstset{frame=tb,
  language=Mathematica,
  aboveskip=3mm,
  belowskip=3mm,
  showstringspaces=false,
  columns=flexible,
  basicstyle={\small\ttfamily},
  numbers=none,
  numberstyle=\tiny\color{gray},
  keywordstyle=\color{blue},
  commentstyle=\color{dkgreen},
  stringstyle=\color{mauve},
  breaklines=true,
  breakatwhitespace=true
  tabsize=3
}

\restylefloat{table}

% METADATA
\title{\textbf{FPGA Implementation of the Fast Fourier Transform}}
\date{December 2, 2014}
\author{Garrett Massman and Cory Walker}

\begin{document}

  \maketitle
  \clearpage

  \section*{Introduction}
    In this project we implemented a fast Fourier transform algorithm on a Xilinx FPGA using VHDL. The primary motivation of our project was to digitally analyze musical signals, but the use cases for an FFT chip far surpass that specific use case. \\

    As a very general overview, our completed device functions by sampling an analog signal for a set amount of time and storing the data into an input buffer in BRAM. Next, the signal is is processed using an FT controller and a complex ALU and stored in an output buffer. Finally, a microcontroller can then read the output buffer over a standard SPI protocol. From there the processed frequency domain data can be sent to a computer for further processing or any other device.

  \section*{Theory}
    To understand the discrete Fourier transform, one must first analyze its analogous continuous    time form.
    This is written most simply as
    \begin{align*}
    &X(j\omega) = \int_{-\infty}^{\infty} x(t) e^{-j \omega t}\,dt
    \end{align*}
    which defines the transform of a continuous time signal $f(t)$.
    Without getting into too many details, it defines the signal as a combination of sinusoids,      making it a very useful tool for real world applications.\\

    The discrete Fourier transform (DFT) is simply a reduction of the continuous Fourier transform   into a discrete sample space.
    In other words, if we let $x_n$ represent a sampled version of the continuous time function      $x(t)$ with a total of $N$ samples, we can replace the integral with a summation over the series, as shown below.
    \begin{align*}
        &X_k = \sum\limits_{n=0}^{N-1} x_n e^{\frac{-j2\pi kn}{N}}
    \end{align*}
    As with the continuous time case, this series gives us a glimpse into the component elements of  our original signal.
    However, it is rather costly to compute, requiring a time complexity of $O(N^2)$.
    For each value $X_k$, a series of values from $n = 0$ to $N - 1$ must be generated and summed,   using up valuable computer resources.
    Thus, calculating the DFT in this way is very inefficient, so we instead turn to the Fast  Fourier transform.\\

    In order to perform this operation more quickly, we utilized the Cooley-Tukey FFT algorithm implemented through the Xilinx CoreGen FFT module.
    One requirement is that our input is strictly a power of 2.
    That is because the algorithm works by recursively finding the FFT of smaller and smaller sample sizes of $x_n$, which arises from the fact that the transform itself is periodic.
    The transform function can be broken into the sum of its even and odd components,
    \begin{align*}
        &X_k = \sum \limits_{m=0}^{N/2-1} x_{2m}e^{-\frac{2\pi i}{N} (2m)k} + \sum \limits_{m=0}^{N/ 2-1} x_{2m+1} e^{-\frac{2\pi i}{N} (2m+1)k}
    \end{align*}
    This equation can further be simplified by making the substitutions
    \begin{align*}
        &E_k = \sum \limits_{m=0}^{N/2-1} x_{2m}e^{-\frac{2\pi i}{N} (2m)k}
        &O_k = \sum \limits_{m=0}^{N/2-1} x_{2m+1} e^{-\frac{2\pi i}{N/2} mk}
    \end{align*}
    giving us
    \begin{align*}
        &X_k = E_k + e^{-\frac{2\pi i}{N} k} O_k
    \end{align*}
    The expression $e^{-\frac{2\pi i}{N}k}$ is commonly called the \textit{twiddle factor}.
    Furthermore, because of the periodicity of the transform, we can calculate respective even and   odd components simultaneously
    \begin{align*}
        &E_k = E_{k+\frac{N}{2}}
        &O_k = O_{k+\frac{N}{2}}
    \end{align*}

  \section*{Motivation}
    The FFT is a very common operation used in many digital signal processing tasks.
    Digital signal processors are usually very highly tuned to perform a series of dedicated tasks quickly, and have very little flexability in the way of general purpose usage.
    This is both a blessing and a curse, as they rarely need to be reprogrammed, but are also limited to the range of tasks they can accomplish.
    High end DSPs utilize parallelism to complete mathematical calculations quickly.
    Because of their dedicated nature, DSPs also tend to be rather costly, upwards of \$32 for a SHARC model chip from Analog Electronics.\\

    FPGAs are primarily a large collection of configurable logic blocks that can be wired together to run in parallel, so why not try and use one in place of an expensive DSP?
    By taking the heavy computations off of a typical CPU and putting the burden onto an FPGA, the processor could be freed up to execute any number of instructions until the heavy computations are complete.
    This hybrid model has actually become quite popular in the past several years, and several products such as some of Intel's Xeon chips have FPGAs integrated onboard.
    This can offset the costs for companies who formerly may have been inclined to build expensive ASICs and instead shift their focus onto a new platform.
    A parallel CPU/FPGA integrated system is beyond the scope of this project, but instead a simple model using a Digilent Basys board and an Arduino are built to illustrate the principles.

  \section*{System Overview}
    A critical task in building our FFT device was converting an arbitrary waveform into the SPI signal input that the FPGA expected. This requires the use of a fast analog to digital converter. The Xilinx Spartan 3E FPGA does not come with an onboard ADC. Because of this, a large percentage of the work was actually electronics work done outside of the FPGA:

    \begin{figure}[H]
      \centering
      \includegraphics[width=100mm]{full_system.JPG}
      \caption{Full system including ADC PMOD and audio signal module.}
      \label{overflow}
    \end{figure}

    These external components condition the signal and then send the digital signal into the FPGA at the right voltage level. In addition, the FPGA's output buffer must be read out by an Arduino microcrontroller and sent to a computer with the right software to parse and display the output. We call these external components macro-components and we call the FPGA internal blocks micro-components.

  \section*{Macro-Component Descriptions}
    The required components external to the FPGA are as follows:
    \subsection*{Audio signal module}
      \begin{figure}[H]
        \centering
        \includegraphics[width=60mm]{audio_sig_mod.JPG}
        \caption{Audio signal circuit board with audio connector}
        \label{overflow}
      \end{figure}
      The audio signal module tailors the input audio signal for conversion with the ADC. Traditional audio signals are centered around 0 V and are in the negative voltage range half the time. This module biases the signal by 2.5 V, allowing for a 5 V variation peak to peak in the audio signal without any clipping. This module also provides filtering capacitors that remove most of the supply rail noise that may be present.

    \subsection*{ADC PMOD}
      \begin{figure}[H]
        \centering
        \includegraphics[width=75mm]{adc_pmod.JPG}
        \caption{ADC PMOD circuit board}
        \label{overflow}
      \end{figure}
      The ADC PMOD accepts any input analog input and converts it to a SPI bus that the FPGA can read. The analog signal is fed into the onboard ADS7818 chip. Unfortunately, this chip operates at 5 V but the FPGA operates at 3.3 V. Because of this, there is a voltage divider to drop the voltage down to the FPGA. For FPGA output to be read by the ADS7818, we added a Schmitt inverter IC wired as a Schmitt trigger to act as a level converter.
      \begin{figure}[H]
        \centering
        \includegraphics[width=100mm]{adc_logic.png}
        \caption{ADC interfacing}
        \label{overflow}
      \end{figure}
      We started interfacing with the ADS7818 using an Arduino. Once we understood the protocol completely, we moved the interfacing code from the Arduino to the FPGA and started using real signal data.

    \subsection*{Arduino}
      The Arduino functions as a means of getting the processed information out of the FPGA's BRAM. The Arduino turned out to be the main bottleneck in terms of framerate to the computer's display. We had to push the SPI and serial write functions outside of recommended bounds to get a fluid framerate on the computer display. The full software is available in the \texttt{arduino/osc\_binary\_spi} directory of the source code repository.

    \subsection*{Computer software}
      The computer software for this project is divided into two components. The first component is the Python module that facilitates the reading and decoding of the data from the serial port. It will read out 512 complex numbers and, optionally, compute the magnitude of each of those numbers. The second component is the actual display. It will use the Python module to read the device and it uses Tkinter to display the plot on the screen.

  \section*{Micro-Component Descriptions}
    The final block diagram of our device is as follows:
    \begin{figure}[H]
      \centering
      \includegraphics[trim=0 500 0 30,clip,width=140mm]{vhdl_fft.pdf}
      \caption{Block diagram of the FFT chip}
      \label{overflow}
    \end{figure}
    All of the blocks inside the FPGA device are referred to as micro-components.

    \begin{itemize}
      \item Deterministic ADC: Provides a completely predictable ADC signal to test with and substituted for a real ADC while it was shipping.
      \item SPI Mux: A simple multiplexer that selects between the external ADC and the deterministic ADC.
      \item SPI Master: An OpenCore that implements an SPI master interface. This handles the control and reading of the SPI bus from either the deterministic ADC or the external ADC.
      \item Capture Controller: This block reads samples from the SPI Master and stores the readings as complex numbers in the input buffer. The sample size is 512. It also communicates with the FT controller about when it finishes.
      \item FT controller: This block actually executes the fast Fourier transfer algorithm whenever the capture controller indicates that the input buffer is valid.
      \item Complex ALU: This block handles complex multiplications for the FT controller.
      \item OUTBUF interpreter: This block facilitates the reading of the OUTBUF by feeding words into the SPI slave module.
      \item SPI Slave: An OpenCore that implements an SPI slave interface. This handles the control and writing of the SPI bus, and it connects to an external microcontroller.
    \end{itemize}

  \section*{Simulation Results}

  \section*{Results}
    He seemed really excited about timing results so we should probably discuss this here.

  \section*{Roadblocks}

  \section*{Future Improvements}

  \section*{Conclusion}

  \clearpage
  \section*{Appendix}

  \begin{thebibliography}{9}
    \bibitem{doin}
      Doin, Jonny. \emph{SPI Master/Slave Interface.} OpenCores, 16 May 2011. Web. 13 Sept. 2014. \textless\url{http://opencores.org/project,spi_master_slave}\textgreater.
    \bibitem{reynwar}
      Reynwar, Ben. \emph{FFT on an FPGA.} FFT on an FPGA. N.p., n.d. Web. 13 Sept. 2014. \textless\url{http://www.reynwar.net/ben/docs/fft_dit/index.html}\textgreater.
    \bibitem{roberts}
      Roberts, Michael J. \emph{Signals and Systems: Analysis Using Transform Methods and MATLAB.} New York: McGraw Hill, 2012. Print.
    \bibitem{satoh}
      Satoh, Keiichi, Jubee Tada, Kenta Yamaguchi, and Yasutaka Tamura. \emph{Complex Multiplier Suited for FPGA Structure.} Computers and Communications (2008): 341-44. Web. 13 Sept. 2014.
    \bibitem{tukey}
      Wikipedia contributors. \emph{Cooley–Tukey FFT algorithm.} Wikipedia, The Free Encyclopedia. Wikipedia, The Free Encyclopedia, 27 Jun. 2014. Web. 13 Sep. 2014.
    \bibitem{dft}
      Wikipedia contributors. \emph{Discrete Fourier transform.} Wikipedia, The Free Encyclopedia. Wikipedia, The Free Encyclopedia, 2 Sep. 2014. Web. 13 Sep. 2014.
  \end{thebibliography}

\end{document}
%  \subsection*{Overview}
%    In this project we will implement the Cooley-Tukey Fast Fourier Transform algorithm on a Xilinx FPGA using VHDL.
%    The primary motivation for this is to digitally analyze musical signals, regardless of the instruments they come from.
%    Using the Cooley-Tukey algorithm, we can calculate the FFT in $O(nlog_2(n))$ time.
%    As far as implementation, the project will be split into several main components.
%    First, we will need to sample our audio for a set amount of time and store the data into a buffer in memory.
%    Then, the digital signal will need to be processed using a complex ALU.
%    Finally, the data will need to be sent to an output buffer for application specific usage.
%    This can be in the form of a 7-segment hex display (for instance if we want to print out the fundamental frequency), or a microcontroller (if we want to plot the entire sampled spectrum by passing it to a more capable platform such as Matlab).
%
%  \subsection*{Theory}
%    To understand the discrete Fourier Transform, one must first analyze its analogous continuous time form.
%    This is written most simply as
%    \begin{align*}
%    &X(\omega) = \int_{-\infty}^{\infty} x(t) e^{-j \omega t}\,dt
%    \end{align*}
%    which defines the transform of a continuous time signal $f(t)$.
%    Without getting into too many details, it defines the signal as a combination of sinusoids, making it a very useful tool for real world applications.\\
%
%    The discrete Fourier Transform (DFT) is simply a reduction of the continuous Fourier Transform into a discrete sample space.
%    In other words, if we let $x_n$ represent a sampled version of the continuous time function $x(t)$ with a total of $N$ samples, we can replace the integral with a summation over the series, as shown below.
%    \begin{align*}
%        &X_k = \sum\limits_{n=0}^{N-1} x_n e^{\frac{-j2\pi kn}{N}}
%    \end{align*}
%    As with the continuous time case, this series gives us a glimpse into the component elements of our original signal.
%    However, it is rather costly to compute, requiring a time complexity of $O(N^2)$.
%    For each value $X_k$, a series of values from $n = 0$ to $N - 1$ must be generated and summed, using up valuable computer resources.
%    Thus, calculating the DFT in this way is very inefficient, and we must instead turn to the Fast Fourier Transform.\\
%
%    In order to perform this operation more quickly, we can utilize the Cooley-Tukey FFT algorithm.
%    However, the one requirement is that our input is strictly a power of 2.
%    That is because the algorithm works by recursively finding the FFT of smaller and smaller sample sizes of $x_n$, which arises from the fact that the transform itself is periodic.
%    The transform function can be broken into the sum of its even and odd components,
%    \begin{align*}
%        &X_k = \sum \limits_{m=0}^{N/2-1} x_{2m}e^{-\frac{2\pi i}{N} (2m)k} + \sum \limits_{m=0}^{N/2-1} x_{2m+1} e^{-\frac{2\pi i}{N} (2m+1)k}
%    \end{align*}
%    This equation can further be simplified by making the substitutions
%    \begin{align*}
%        &E_k = \sum \limits_{m=0}^{N/2-1} x_{2m}e^{-\frac{2\pi i}{N} (2m)k}
%        &O_k = \sum \limits_{m=0}^{N/2-1} x_{2m+1} e^{-\frac{2\pi i}{N/2} mk}
%    \end{align*}
%    giving us
%    \begin{align*}
%        &X_k = E_k + e^{-\frac{2\pi i}{N} k} O_k
%    \end{align*}
%    The expression $e^{-\frac{2\pi i}{N}k}$ is commonly called the \textit{twiddle factor}.
%    Furthermore, because of the periodicity of the transform, we can calculate respective even and odd components simultaneously
%    \begin{align*}
%        &E_k = E_{k+\frac{N}{2}}
%        &O_k = O_{k+\frac{N}{2}}
%    \end{align*}
%
%
%
%  \subsection*{In Mathematica}
%  We implemented a candidate version of the Cooley-Tukey algorithm in Mathematica. This code shows how it is possible to build a look up table of calculated results against the variables $N$ and $k$ for the calculation $e^{-\frac{2\pi i}{N} k}$. This table will not only save us time, but also complexity in the design by allowing us to simply look up this value.
%  \begin{lstlisting}
%myfasterft[x_, capn_, s_, precomputed_] := (
%   result = {};
%   If[capn == 1,
%    result = Append[result, x[[1]]],
%    result = Join[result, myfasterft[x, capn/2, 2 s, precomputed]];
%    result =
%     Join[result, myfasterft[x[[s + 1 ;;]], capn/2, 2 s, precomputed]];
%    Do[
%     t = result[[k + 1]];
%     result[[k + 1]] =
%      t + precomputed[[Log2[capn]]][[k + 1]]*
%        result[[k + capn/2 + 1]];
%     result[[k + capn/2 + 1]] =
%      t - precomputed[[Log2[capn]]][[k + 1]]*
%        result[[k + capn/2 + 1]];
%     , {k, 0, capn/2 - 1}];
%    ];
%   result
%   );
%
%precomputedmatrix[datalen_] := (
%  Table[
%   Table[
%    E^(-I*2*\[Pi]*k/capn),
%    {k, 0., datalen/2 - 1}
%    ],
%   {capn, Table[2^x, {x, 1, Log2[datalen]}]}
%   ]
%  )
%
%data = Table[N[Sin[30 2 Pi n/200] + (RandomReal[] - 1/2)], {n, 512}];
%precomputed = precomputedmatrix[Length[data]];
%ListLinePlot[Abs[myfasterft[data, Length[data], 1, precomputed]], PlotRange -> All]
%  \end{lstlisting}
%  \subsection*{FPGA considerations}
%    %Describe how we plan to utilize SPI for communication, what type of logical blocks we will use and how they will work together.
%    Because we are working with a very specialized type of hardware, we must take into consideration some limiting parameters.
%    In particular, the Spartan3Es loaded on each BASYS board only have 72K RAM.
%    This means that if we want a real time FFT, we will need to optimize the space we have available, and only write to it when it is absolutely necessary.
%    We must also consider the quality of our digital input signal.
%    There are various ADC PMODs available for the BASYS boards, which range in quality from 4.8 KHz to 1 MHz in speed.
%    If we are only sampling audio, we probably won't need to gather any frequencies above 1200 Hz, so the lower end model should be fine.\\
%
%    Once our data has been collected and stored inside the FPGA, it needs to be processed.
%    This is perhaps the most challenging part of the project, as we need to implement what will be called an FTC (Fourier Transform Controller) as well as a CALU (Complex Arithmetic Logic Unit).
%    The FTC's primary function will be to keep track of the data as it moves from RAM, into the CALU, and back out into the real world.
%    This will require some heavily synchronized logic and very tight timing control to be carried out properly.
%    It will work in sync with the SPI input master to tell the CALU when data is ready to be read, and then clear the input buffer once the CALU has finished.
%    It will also need to monitor the movement of data from the output buffer into whatever output stream requires it.
%    The data may be sent directly to a microcontroller or into another component for post processing.\\
%
%    The CALU will have one primary job, which is to actually perform the FFT.
%    From its perspective, it will only the addresses of an input buffer and an output buffer, as well as a flag that tells it when to start and stop.
%    Because all of our calculations are being done inside of an FPGA, the CALU will be most composed of combinational logic at its core.
%    In theory (i.e., provided enough gates), all that needs to be built is the base case of the FFT, which takes in two numbers and outputs their sum and difference.
%    \begin{align*}
%        &y_0 = x_0 + e^{-\frac{2\pi i}{N}k}x_1
%        &y_1 = x_0 - e^{-\frac{2\pi i}{N}k}x_1
%    \end{align*}
%    This portion of the computation is commonly called a \textit{butterfly}, because the system's diagram looks vaguely like a butterfly.
%    Each of the twiddle factors included above can be calculated beforehand and stored in a lookup table for quick access before each iteration.
%    Because we will be performing the same type of computation with them each time, they are essentially constants in our system.\\
%
%  \subsection*{Block diagram}
%    Over the past few weeks we have taken what we know and outlined a basic block diagram of our device:
%    \begin{figure}[H]
%      \centering
%      \includegraphics[trim=0 400 0 400,clip,width=140mm]{vhdl_fft.pdf}
%      \caption{Block diagram of the FFT chip}
%      \label{overflow}
%    \end{figure}
%    Now that we have divided the device into several logical blocks, we can now split up the work accordingly.
%  \subsection*{Roadmap}
%    In this section, we will describe the steps, in order, that need to be done. Our steps will essentially follow the data pipeline of the module:
%    \begin{itemize}
%      \item We will begin by implementing a deterministic "ADC" module that will emulate the interface of the real ADC but generate a fixed signal every time. We will use this signal in our own calculations so that we can then verify the outputs of the chip at each stage in the data pipeline. When we are ready to convert to real ADC signals, we can simply switch to an external ADC module using an SPI mux. This step has already been completed.
%      \item We will then implement the capture controller. This controller will have an SPI master unit that can read from the ADC. It will be also connected to the block RAM to write the input buffer.
%      \item Next, we will work on the heart of the chip, the FT controller. This controller will first simply implement the DFT algorithm. It will utilize the complex number ALU to make calculations.
%      \item After the FT controller, we will implement an OUTBUF interpreter that is responsible for outputting the results of the FT controller to the rest of the world using SPI.
%      \item Once all of this is done, we will finish converting our FT controller from using the simple DFT algorithm to the more complex Cooley-Tukey FFT algorithm.
%      \item If we have further time still, we will build interesting demonstrations of this device.
%    \end{itemize}
%  \subsection*{Resources}
%    While our aim is to implement the entire Fast Fourier Transform with our own code, there are some blocks of our design that are common practice to reuse. Those blocks are the SPI modules. While SPI is not a complicated communication protocol, it requires a fair amount of time to implement. There is a great resource online called OpenCores. This resource is essentially a collection of open source HDL modules that can be freely used in designs. We will be using the "SPI Master/Slave Interface" module a few times in our design to implement SPI.
%  \subsection*{Possible extensions}
%    The Fast Fourier Transform algorithm applied to an arbitrary voltage signal is very useful. Because our final chip will provide such capability, we have the option of using our newly-designed chip with multiple different applications. \\
%
%    One of our ideas is to attach a microphone and a preamp and use the frequency-domain signal to determine the musical note being played. We could also build a tuner for a guitar. \\
%
%    Additionally, we could connect an audio signal directly from any music player and display a graphic equalizer effect using either a computer or microcontroller that will interface with our chip. \\
%
%    Finally, provided we are able to tune our chip and ADC to operate fast enough, we might be able to build a radio wave analyzer for very long wavelength signals. This device would almost certainly be too slow for any serious RF work, but it would still be an interesting extension. \\
%
%    These extensions are not crucial to our design but might serve as a great demonstration when we present the device to the class.


\section{Summary and Outlook}\label{se:Summary}

We have developed IMEX schemes suitable for a two-moment model of neutrino transport that obey Fermi-Dirac statistics.
The scheme employs algebraic closure based on Fermi-Dirac statistics, high-order discontinuous Galerkin methods for spatial discretization, and convex-invariant time integration to maintain realizability of the moments.
Since the realizable domain is convex and its convexity can be inherited by a convex combination, a scheme having convex combinations as its stages can preserve the realizable domain.
This encouraged us to construct realizability-preserving time integrators, realizability-preserving IMEX schemes, and a method with a realizability-preserving IMEX time integrator and high-order DG method.

In the applications that motivate this work, the neutrino distribution function can vary from 0 to 1.
Hence, we have considered algebraic closures based on Fermi-Dirac statistics for both low and high occupancy.
Among the seven algebraic closures we considered -- Kershaw~\cite{kershaw_1976}, Wilson~\cite{wilson_1975,leblancWilson_1970}, Levermore~\cite{levermore_1984}, Minerbo~\cite{minerbo_1978}, Janka 1~\cite{janka_1991}, Janka 2~\cite{janka_1992}, and Cernohorsky \& Bludman~\cite{cernohorskyBludman_1994} -- only the Cernohorsky \& Bludman closure obeys Fermi-Dirac statistics for all occupancies.
As a result, we employed the Cernohorsky \& Bludman closure for the neutrino stationary state test in Section~\ref{se: Neutrino Stationary State Test}.
We also ran our code with Minerbo closure, IMEX PC2, IMEX SSP2332 and IMEXRKCB2 schemes, and the results show that only PD-ARS schemes have stability.
In addition, closures have impact on the simulation result.
As we observed, both using PD-ARS scheme, there are $\sim30\%$ difference in the neutrino number densities (and relaxation time) between the results obtained with Minerbo closure and that with CB closure.
Even though RKCB2 with Minerbo closure luckily survived our test, the results it gave were compromised: they were closer to the results of the PD-ARS scheme with Minerbo closure than to the results of the PD-ARS scheme with CB closure.
In what way and to what degree the results are in fact compromised either by the closure or by a particular correction step for unrealizable moments are difficult to determine fully and is left for further study.

Two PD-ARS schemes are proposed.
The one with SSPRK2 has second-order accuracy while the other with SSPRK3 has third-order accuracy, and both have the strong-stability preserving property in the streaming limit.
Their accuracy was demonstrated on problems with known smooth solutions in streaming, absorption, and scattering-dominated regimes.
The neutrino transport test with emission, absorption, and isoenergetic scattering through a stationary background, was designed to test the convex-invariance of our PD-ARS schemes.
The neutrino stationary state test shows that a method combining an algebraic closure based on Fermi-Dirac statistics and convex-invariant time integration is promising for robust CCSN simulation.

In this work, we adopted Cartesian coordinates, a linear collision term, and a fixed material background.
More realistic problems of scientific interest, such as with energy-exchanging scattering and relativistic effects, are left for future research.


 \chapter{Linear Solver}

In many applications one of the main problems is to find the solution of a linear system. This system may arise form an optimisation or, like here, a discretised PDE. In this chapter we will go through the two methods which this is done.

\section{Direct Methods}

Suppose you are given a linear system of the form
$$Ax = b$$
where $A$ is a real $n\times n$ non-singular matrix and right hand side vector $b \in \mathcal{R}^n$, where the aim is to solve for the vector $x \in \mathcal{R}^n$. All direct methods are based on Gaussian elimination.

\subsection{Software}

UMFPACK \cite{Davis:2004:CPS:992200.992205,Davis:2004:AUV:992200.992206,Davis:1999:CUM:305658.287640,davis1997unsymmetric}, PASTIX \cite{henon2002pastix}, SuperLU \cite{superlu_ug99,li05} and MUMPS \cite{amestoy2000multifrontal,amestoy2001fully,amestoy2006hybrid}

\section{Iterative Methods}


\section{Strength Reduction}
\label{strengthreduction}

Strength reduction has been a common compiler optimization
technique for decades. Strength reduction replaces
one operation by an equivalent one that is, typically, faster
to execute. For example, a multiply of an integer by a
power of two could be replaced with a left shift by the base-two logarithm of
the power of two, if the shift operation was
faster than the multiply.~\cite{DFBacon:transformations2}

Strength reduction has been used in APL interpreters for
decades. In the case of Boolean arguments to verbs,
some common strength reductions are, where {\apl B1 and B2} are
conformable Boolean arrays:

\medskip
\begin{tabular}{|l|l|}
\hline
Unoptimized & Optimized\\
\hline
{\apl B1\qtimes\0B2} & {\apl B1\qand\0B2}\\
{\apl B1\qdstile\0B2} & {\apl B1\qand\0B2}\\
{\apl B1\qustile\0B2} & {\apl B1\qor\0B2}\\
{\apl B1\qstar\0B2} & {\apl B1\qge\0B2}\\
{\apl B1\qstile\0B2} & {\apl B1\qlt\0B2}\\
{\apl B1\qbang\0B2} & {\apl B1\qle\0B2}\\
{\apl \qtimes\0B2} & {\apl B2}\\
\hline
\end{tabular}
\medskip

\noindent The reduction of expensive integer (or real) operations
to fast Boolean operations provides a significant performance boost.
Furthermore, operating in the Boolean domain provides further
performance benefits, by eliminating many conditionals
within the element-wise computation, and permitting word-at-a-time
SIMD execution, even in architectures without vector extensions.


\section{Randomized Algorithms}
\subsection{Sampling based randomized algorithm (SRA)}

Instead of solving the original problem is
\begin{equation}\label{ORG}
\mathcal{Z} = min_{x \in \mathbb{R}^n} \| Ax - b\|_2
\quad \text{with solution} \quad
x_{opt} = (A^T A)^{-1} A^T b = A^{\dagger} b,
\end{equation}
actually we solve
\begin{equation}\label{ACT}
\tilde{\mathcal{Z}} = min_{x \in \mathbb{R}^n} \| X Ax - X b\|_2
\quad \text{with solution} \quad
\tilde{x}_{opt} = (XA)^{\dagger} X b.
\end{equation}

%------------------------------------------------
\subsubsection{Theory}
\begin{lemma}[Drineas, Mahoney, Muthukrishnan and Sarlos(2011)]
	Consider the overconstrained least squares problem of (\ref{ORG}) and let the matrix $U_A \in \mathbb{R}^{m\times n}$ contain the top $n$ left singular vectors of $A$. Assume $X$ satisfies the two conditions
	\begin{enumerate}
		\item $\sigma_{min}^2(X U_A) \geq 1/\sqrt{2}; $ \\
		\item $\| U_A^T X^T X b^{\perp}\|_2^2 \leq \epsilon \mathcal{Z}^2 /2, \quad (b^{\perp} = b - A A^T b), $
	\end{enumerate}
	for some $\epsilon \in (0,1).$

	Then the solution $\tilde{x}_{opt}$ to the least squares approximation problem (\ref{ACT}) satisfies:
	\begin{enumerate}
		\item $\| A \tilde{x}_{opt} - b\|_2 \leq (1+ \epsilon) \mathcal{Z} $ , and \\
		\item $\| x_{opt} - \tilde{x}_{opt} \|_2 \leq \frac{1}{\sigma_{min}(A)} \sqrt{\epsilon} \mathcal{Z}. $
	\end{enumerate}
\end{lemma}

%------------------------------------------------
\subsubsection{Implementation: A sampling-based randomized algorithm}
A sampling-based randomized algorithm (SRA) is to construct
$$
X = S^T H D,
$$

\begin{itemize}
	\item $S \in \mathbb{R}^{r \times m}$ is the uniform sampling matrix, where $S_{r} = (\sqrt{m/r}) e_{ir},$ where $ir$ is uniformly chosen from $[m],$
	and $e_{ir}$ is the standard basis. \\
	\item $H \in \mathbb{R}^{m \times m}$ is the Hadamard transform matrix defined recursively by
	$$
	H_m = \begin{bmatrix} H_{m/2} & H_{m/2} \\ H_{m/2} & -H_{m/2} \end{bmatrix}, \quad \text{with} \quad H_2 = \begin{bmatrix} +1 & +1 \\ +1 & -1 \end{bmatrix}.
	$$\\
	\item $D \in \mathbb{R}^{m \times m}$ is the diagonal matrix with
	$$
	D_{ii} = \begin{cases} &+1, \quad \text{with probability} \quad 1/2;\\ &-1 \quad \text{with probability} \quad 1/2. \end{cases}
	$$
\end{itemize}

$$
\tilde{x}_{opt} = (S^T H D A)^{\dagger} S^T H D b.
$$

%------------------------------------------------
\subsubsection{Effect of the Randomized Hadamard Transform}
{\color{red} HD approximately 'uniformizes' information in the left singular subspace of the matrix $A$.}
\begin{lemma}
	Let $U$ be an $m \times n$ orthogonal matrix and let the product $HD$ be the $m \times m$ Randomized Hadamard Transform. Then with probability at least $0.95$,
	$$
	\| (HDU)_{i} \|_2^2 \leq \frac{2n \ln(40 m n)}{m}, \quad \forall i \in [m].
	$$
\end{lemma}

%------------------------------------------------

\begin{theorem}[Convergence and Complexity of SRA]
	Suppose $A,b$ and $\epsilon$ satisfy the input requirement of SRA. Run SRA with
	\begin{equation}\label{eq22}
	r = max \left( 48^2 n \ln(40 mn) \ln(100^2 n \ln(40 mn)) , 40n \ln(40 mn) /\epsilon \right)
	\approx O(n/\epsilon).
	\end{equation}
	and return $\tilde{x}_{opt}.$ Then, with probability at least $0.8$, the following claims hold:
	\begin{enumerate}
		\item $\tilde{x}_{opt}$ satisfies $\| A \tilde{x}_{opt} - b \|_2  \leq (1 + \epsilon) \mathcal{Z}; $ \\
		\item If we assume that $\| U_A  U_A^T b \|_2 \geq \gamma \|b\|_2 $ for some $0< \gamma \leq 1$ then
		$$
		\| x_{opt} - \tilde{x}_{opt}\|_2 \leq \sqrt{\epsilon} \left( \kappa(A) \sqrt{\gamma^{-2} -1} \right) \| x_{opt}\|_2.
		$$\\
		\item $m(n+1) + 2m(n+1)log_2(r+1) + O(rn^2)$ time suffices to compute the solution $\tilde{x}_{opt}. $
	\end{enumerate}
\end{theorem}
The cost is reduced since $XA \in \mathbb{R}^{r \times n}$ while $A \in \mathbb{R}^{m \times n}.$

%------------------------------------------------
\subsubsection{Projection Based Randomized Algorithm (PRA)}
Implementation: A projection-based randomized algorithm
Projection-based randomized algorithm (PRA) is to construct a smaller problem by performing a 'sparse projection' on the pre-processed problem.
$$
X = THD,
$$
where
\begin{itemize}
	\item $H \in \mathbb{R}^{m \times m}$ is the Hadamard Transform and $D \in \mathbb{R}^{m \times m }$ is the randomized diagonal matrix as defined before. \\
	\item $T \in \mathbb{R}^{k \times m}$ is a randomized matrix given by
	$$
	T_{ij} = \begin{cases} &+ \sqrt{1/kq} \quad \text{with probability} \quad q/2, \\
	&-  \sqrt{1/kq} \quad \text{with probability} \quad q/2, \\
	& 0 \quad \text{with probability} \quad 1-q.
	\end{cases}
	$$
\end{itemize}
$$\tilde{x}_{opt} = (THDA)^{\dagger} THDb. $$

%------------------------------------------------

\begin{theorem}[Convergence and Complexity of PRA]
	Suppose $A,b$ and $\epsilon$ satisfy the input requirement of FRA. Run FRA with
	\begin{equation}
	q \geq \frac{C_q n \ln(40mn)}{m} (2 \ln(m) + 16n + 16);
	\end{equation}
	\begin{equation}
	k \geq max \left( C_k (118^2 n + 98^2), \frac{60n}{\epsilon} \right)
	\end{equation}
	and return $\tilde{x}_{opt}.$ Then, with probability at least $0.8$, the following claims hold:
	\begin{enumerate}
		\item $\tilde{x}_{opt}$ satisfies $\| A \tilde{x}_{opt} - b \|_2  \leq (1 + \epsilon) \mathcal{Z}; $ \\
		\item If we assume that $\| U_A  U_A^T b \|_2 \geq \gamma \|b\|_2 $ for some $0< \gamma \leq 1$ then
		$$
		\| x_{opt} - \tilde{x}_{opt}\|_2 \leq \sqrt{\epsilon} \left( \kappa(A) \sqrt{\gamma^{-2} -1} \right) \| x_{opt}\|_2.
		$$\\
		\item $m(n+1) + 2m(n+1)log_2(mkq+1) + O(kn^2)$ time suffices to compute the solution $\tilde{x}_{opt}. $
	\end{enumerate}
\end{theorem}

%------------------------------------------------
\subsection{A Fast Randomized Algorithm (FAR)}
The fast randomized algorithm (FAR) is from Rokhlin and Tygert's paper \cite{SRFT2008}. The algorithm is given as follows.
	\begin{enumerate}
		\item Compute $E = TA$, where $T$ is the $r \times m$ SRFT defined as follows, with $m \geq r \geq n. $ \\
		\item Form a pivoted QR-decomposition of $E = Q R \Pi$,  where $Q_{r \times n}$ has orthonormal columns, $R_{n \times n}$ is upper triangular and $\Pi_{n \times n}$ is the permutation matrix.  \\
		\item Solve a preconditioned Least Squares Problem
		$$
		\| AP^{-1} y - b\|
		$$
		using PCG where $P = R \Pi$ is the preconditioning matrix.
	\end{enumerate}

%------------------------------------------------
\subsubsection{Complexity for A Fast Randomized Algorithm (FAR)}
	\begin{enumerate}
		\item Applying $T$ to every column of $A$ : $O(mnlog(r))$.  \\
		\item Computing the pivoted QR decomposition of $E$ : $O(n^2r) . $ \\
		\item Applying $T$ to $b$: O(mlog(r)). Applying $Q^*$ to $Tb$: $O(nr)$.

		Applying $P^{-1} = \Pi^{-1} R^{-1}$ to $Q^* Tb$ : $O(n^2). $ \\
		\item Applying $A,A^T$ a total of $O(\kappa(AP^{-1}) |log(\epsilon)|)$ times: {\color{red}$O(mn\kappa(AP^{-1}) |log(\epsilon)|).$} \\
		\item Applying $P^{-1},(P^{-1})^*$ to a total of $O(\kappa(AP^{-1}) |log(\epsilon)|)$ times: {\color{red}$O(n^2 \kappa(AP^{-1}) |log(\epsilon)|).$} \\
		\item Applying $P^{-1}$ to $y$ : $O(n^2). $
	\end{enumerate}
Thus
{\color{red}
	$$
	C_{theoretical} = O((log(r) + \kappa(AP^{-1}) |log(\epsilon)|) mn +n^2r).
	$$}

%------------------------------------------------
\subsubsection{SRFT Matrix}
T is the sampled randomized Fourier transform (SRFT) Matrix defined by
$$T _{r \times m} = G_{r \times m} H_{m \times m}, \quad  r \leq m. $$
G is the random matrix given by
$$ G_{r \times m} = S_{r \times m} F_{m \times m} D_{m \times m}, $$

\begin{itemize}
	\item $S$ is a random permutation matrix in each row j there is one column $s_j$ such that $S_{j,s_j} = 1$ and $S_{j,k} =0$ if $k \neq s_j$. $s_j$'s are i.i.d. random variables distributed uniformly over $\{ 1,\cdots,m\}. $ \\
	\item $F$ is the $m \times m$ discrete Fourier transform. \\
	\item $D = diag(d_1,d_2,\cdots,d_m)$
	where $d_1,\cdots,d_m$ are i.i.d. complex random variables distributed uniformly over the unit circle.
\end{itemize}

%------------------------------------------------
$$
H_{m \times m} =\Theta_{m \times m} \Pi_{m \times m} Z_{m \times m} \tilde{\Theta}_{m \times m}\tilde{\Pi}_{m \times m} \tilde{Z}_{m \times m},
$$
where

\begin{itemize}
	\item $\Pi$ and $\tilde{\Pi}$ are permutation matrices chosen independently and uniformly at random. \\
	\item $Z$ and $\tilde{Z}$ are diagonal matrices whose diagonal entries are i.i.d. complex random variables distributed uniformly over the unit circle. \\
	\item
	$\Theta_{m \times m} = \begin{bmatrix} cos(\theta_1)& sin(\theta_1) & 0& \cdots &0\\
	- sin(\theta_1)  & cos(\theta_1) & 0 & \cdots & 0 \\
	0 & 0 & 1 & \cdots & 0 \\
	\cdots & \cdots & \cdots & \cdots & \cdots  \\
	0 & 0 &0 & \cdots & 1
	\end{bmatrix}
	\cdots
	\begin{bmatrix} 1& \cdots & 0& 0  &0\\
	\cdots  & \cdots  & \cdots & \cdots & \cdots \\
	0 & \cdots & 1 & 0 & 0 \\
	0 & \cdots & 0 & cos(\theta_{m-1})& sin(\theta_{m-1}) \\
	0 & \cdots &0 &  -sin(\theta_{m-1}) & cos(\theta_{m-1})
	\end{bmatrix} ,$

	where $\theta_k, \quad k =1 ,\cdots m-1$ are i.i.d. real random variables drawn uniformly from $[0,2\pi]$.
	So is $\tilde{\Theta}$, but defined with different $\tilde{\theta}_k$.

\end{itemize}

%------------------------------------------------

\subsubsection{Why SRFT works?}
\begin{corollary}
	Suppose that $\alpha$ and $\beta$ are real numbers greater than 1 and $r,m$ and $n$ are positive integers such that $m \geq r \geq (\frac{\alpha^2 +1}{\alpha^2-1})^2 \beta n^2. $ Suppose further that $T$ is the $r \times m$ SRFT matrix. Suppose in addition that $U$ is an $m \times n$ matrix whose columns are orthonormal.
	Then, the condition number of $TU$ is at most $\alpha$ with probability at least $1 - \frac{1}{\beta}. $
\end{corollary}
$\kappa(TU)$ can be made arbitrarily close to 1 when $r$ is large enough.

%------------------------------------------------

\begin{theorem}[Rokhlin and Tygert (2008)]
	Suppose that $r,m$ and $n$ are positive integers such that $m \geq r \geq n$. Suppose further that $A$ is a full rank $m \times n$ matrix and the SVD of $A$ is
	$$
	A_{m \times n} = U_{m \times n} \Sigma_{n \times n} V^*_{n \times n}.
	$$
	Suppose in addition that $T$ is an $r \times m$ matrix such that the $r \times n$ matrix $TU$ has full rank.
	Then
	$$
	T_{r \times m} A_{m \times n} = Q_{r \times n} P_{n \times n},
	$$
	where $Q$ has orthonormal columns.
	And
	$$
	\kappa(AP^{-1}) = \kappa(TU).
	$$
\end{theorem}
\small{$r \geq 4n^2$ guarantees $\kappa(TU) \leq 3.$}