% !TEX root = main.tex

%-------------------------------------------------
\section{Most powerful tests}\label{sec:most_powerful_tests}

\begin{definition}
Let $C_1$ and $C_2$ be two critical regions of size $\alpha$ for testing $H_0:\theta\in\Theta_0$ against $H_1:\theta\in\Theta_1$ and consider the power functions $\gamma_1(\theta)$ and $\gamma_2(\theta)$ of the associated tests,
\[
\gamma_1(\theta) = \prob_{\theta}(X\in C_1) \quad\text{and}\quad \gamma_2(\theta) = \prob_{\theta}(X\in C_2) \quad\text{for $\theta\in\Theta_1$.}
\]
If $\gamma_1(\theta) > \gamma_2(\theta)$ for all $\theta\in\Theta_1$, we say that $C_1$ is a \emph{more powerful test} than $C_2$.
\end{definition}

\begin{definition}
Let $C$ be a critical region of size $\alpha$ for testing $H_0:\theta\in\Theta_0$ against $H_1:\theta\in\Theta_1$. Then the associated test is called a \emph{most powerful test} of size $\alpha$ if for every subset $A\subset D$ of size $\alpha$,
\[
\prob_{\theta}(\mathbf{X}\in C) \geq \prob_{\theta}(\mathbf{X}\in A) \quad\text{for all $\theta\in\Theta_1$.}
\]
This means that the test is at least as powerful as any other test of size $\alpha$.
\end{definition}

%-----------------------------
\subsection{The Neyman-Pearson lemma}

\begin{theorem}[The Neyman-Pearson lemma]
The likelihood ratio test is a most powerful test of a simple null hypothesis against a simple alternative.
\end{theorem}
\begin{proof}
Let $H_0:\theta=\theta_0$ and  $H_1:\theta=\theta_1$. The SLRT is given by the critical region
\[
C = \left\{\mathbf{x}:\lambda(\mathbf{x}) \leq k\right\}
\quad\text{where}\quad
\lambda(\mathbf{x}) = \frac{L(\theta_0;\mathbf{x})}{L(\theta_1;\mathbf{x})}
\]
Let $A$ be another critical region of size $\alpha$. We need to show that $\prob_{\theta_1}(C) \geq \prob_{\theta_1}(A)$.

\bit
\it If $\mathbf{x}\in C\setminus A$, then $\lambda(\mathbf{x})\leq k$, so $L(\theta_1;\mathbf{x}) \geq \displaystyle\frac{1}{k}L(\theta_0;\mathbf{x})$, or equivalently  $f(\mathbf{x};\theta_1) \geq \displaystyle\frac{1}{k}f(\mathbf{x};\theta_0)$
\it If $\mathbf{x}\in A\setminus C$, then $\lambda(\mathbf{x})> k$, so $L(\theta_1;\mathbf{x}) \leq \displaystyle\frac{1}{k}L(\theta_0;\mathbf{x})$, or equivalently $f(\mathbf{x};\theta_1) \leq \displaystyle\frac{1}{k}f(\mathbf{x};\theta_0)$
\eit

Hence, for continuous distributions (the discrete case is similar),
\begin{align*}
\prob_{\theta_1}(C) - \prob_{\theta_1}(A)
	& = \int_{C} L(\theta_1;\mathbf{x})\,d\mathbf{x} - \int_{A} L(\theta_1;\mathbf{x})\,d\mathbf{x} \\
	& = \int_{C\setminus A} L(\theta_1;\mathbf{x})\,d\mathbf{x} - \int_{A\setminus C} L(\theta_1;\mathbf{x})\,d\mathbf{x} \\
	& \geq \frac{1}{k}\left(\int_{C\setminus A} L(\theta_0;\mathbf{x})\,dx - \int_{A\setminus C} L(\theta_0;\mathbf{x})\,dx\right) \\
	& = \frac{1}{k}\left(\int_{C} L(\theta_0;\mathbf{x})\,dx - \int_{A} L(\theta_0;\mathbf{x})\,dx\right) \\
	& = \frac{1}{k}(\alpha-\alpha) \\
	& = 0.
\end{align*}

Thus $\prob_{\theta_1}(C) \geq \prob_{\theta_1}(A)$ and because this holds for any critical region $A$ of size $\alpha$, we conclude that the SLRT is a most powerful test of $H_0:\theta=\theta_0$ against $H_1:\theta=\theta_1$.
\end{proof}

% example: SLRT for Poisson
\begin{example}%[SLRT for the mean of a Poisson distribution]
Let $X_1,\ldots,X_n$ be a random sample from the $\text{Poisson}(\theta)$ distribution. Find a most powerful test of the simple hypothesis $H_0:\theta=2$ against the simple alternative $H_1:\theta = 1/2$.
\end{example}

\begin{solution}
The PMF of a $\text{Poisson}(\theta)$ random variable is
\[
f(x) = \begin{cases}
\displaystyle\frac{\theta^{x} e^{-\theta}}{x!}		& x=0,1,2,\ldots \\
0														& \text{otherwise.}
\end{cases}
\]

The likelihood function is
\[
L(\theta;\mathbf{x}) = \frac{\theta^{\sum_i x_i } e^{-n\theta}}{\prod_{i=1}^n x_i!}.
\]

For the null value $\theta_0=2$ and the alternative value $\theta_1=1/2$,
\[
L(\theta_0;\mathbf{x}) = \frac{2^{\sum_i x_i} e^{-2n}}{\prod_{i=1}^n x_i!}
\text{\quad and\quad}
L(\theta_1;\mathbf{x}) = \frac{(1/2)^{\sum_i x_i} e^{-n/2}}{\prod_{i=1}^n x_i!}
\]
so the likelihood ratio is
\[
\lambda(\mathbf{x})
	= \frac{L(\theta_0|\mathbf{x})}{L(\theta_1|\mathbf{x})}
	= \frac{2^{\sum_i x_i} e^{-2n}}{(1/2)^{\sum_i x_i} e^{-n/2}}
	= 4^{\sum_i x_i} e^{-3n/2}.
\]
By the Neyman-Pearson lemma, a most powerful test is given by the critical region
\[
C = \{\mathbf{x}:\lambda(\mathbf{x})\leq k\} = \{\mathbf{x}:4^{\sum_i x_i} e^{-3n/2}\leq k\}
\]
where $k$ is chosen according to the required size of the test.
%
To clarify the decision rule, note that
\begin{align*}
C
	& = \left\{\mathbf{x}: \sum_{i=1}^n x_i\log 4 - \frac{3n}{2} \leq \log k\right\}
	= \left\{\mathbf{x}: \sum_{i=1}^n x_i \leq \frac{3n/2 + \log k}{\log 4}\right\}
	= \left\{\mathbf{x}: \sum_{i=1}^n x_i \leq k'\right\}
\end{align*}
where $k'$ is chosen according to the required size of the test.
\end{solution}

\begin{example}
Let $X_1,X_2,\ldots,X_{10}$ be a random sample from the $\text{Bernoulli}(\theta)$ distribution, where $\theta$ is unknown. The null hypothesis $H_0:\theta=1/2$ is rejected in favour of the alternative $H_1:\theta < 1/2$ whenever the sum of the observations satisfies $\sum_{i=1}^{10}X_i \leq 2$.
\ben
\it Find the size of the test.
\it Find the power of the test at $\theta=1/4$ and the power of the test at $\theta=1/5$.
\it Show that this is a most powerful test of $H_0:\theta=1/2$ against the simple alternative $H_1:\theta=1/4$.
\een
\begin{solution}
\ben
\it % <<<<
Let $S = \sum_{i=1}^{10}X_i \leq 2$. From tables,
\[
\alpha = \prob\big[S\leq 2 \text{ when } S\sim\text{Binomial}(10,1/2)\big] = 0.0547.% \text{\quad (from tables).}
\]
\it % <<<<
From tables,
\begin{align*}
\gamma(1/4) & = \prob\big[S\leq 2 \text{ when } S\sim\text{Binomial}(10,1/4)\big] = 0.5256. \\
\gamma(1/5) & = \prob\big[S\leq 2 \text{ when } S\sim\text{Binomial}(10,1/5)\big] = 0.6778.
\end{align*}
\it % <<<<
Likelihood function:
\[
L(\theta;\mathbf{x}) = \prod_{i=1}^{10}f(x_i;\theta) = \prod_{i=1}^{10}\theta^{x_i}(1-\theta)^{1-x_i}
\]
Likelihood ratio:
\[
\lambda(\mathbf{x})
	= \frac{L(1/2;\mathbf{x})}{L(1/4;\mathbf{x})}
	= \frac{\prod_{i=1}^{10}(1/2)^{x_i}(1/2)^{1-x_i}}{\prod_{i=1}^{10}(1/4)^{x_i}(3/4)^{1-x_i}}
%	= \frac{(1/2)^{10}}{(1/3)^{\sum_i x_i}(3/4)^{10}}
	= 3^{\sum_i x_i}(2/3)^{10}.
\]
Simple likelihood ratio test:
\begin{align*}
C
	& = \{\mathbf{x}:\lambda(\mathbf{x})\leq k\} \\
%	& = \{\mathbf{x}:3^{\sum_i x_i}(2/3)^{10}\leq k\} \\
	& = \big\{\mathbf{x}:\sum_i x_i \leq (\log k + 10\log(3/2))/\log 3 \big\} \\
	& = \{\mathbf{x}:\sum_i x_i \leq k' \}
\end{align*}
where $k'$ is chosen appropriately. By the Neyman-Pearson lemma, this is a most powerful test of $H_0:\theta=1/2$ against $H_1:\theta=1/4$.
\een
\end{solution}
\end{example}

%==========================================================================
\begin{exercise}
\begin{questions}
\question  % 8.1.5
Let $\mathbf{X}=(X_1,X_2,\ldots,X_n)$ be a random sample from a distribution whose PDF is $f(x;\theta) = \theta x^{\theta-1}$ for $0 < x < 1$, and zero otherwise.
%\[
%f(x;\theta) = \begin{cases}
%\theta x^{\theta-1}	& 0 < x < 1, \\
%0					& \text{otherwise.}
%\end{cases}
%\]
Show that a most powerful test of $H_0:\theta=1$ against $H_1:\theta=2$ is given by the critical region
\[
C = \left\{\mathbf{x}: \displaystyle\prod_{i=1}^n x_i \geq c\right\}.
\]
%\[
%C = \left\{\mathbf{x}: T(\mathbf{x})\geq c\right\}\quad\text{where } T(\mathbf{x})=\displaystyle\prod_{i=1}^n x_i.
%\]
%defines a most powerful test of $H_0:\theta=1$ against $H_1:\theta=2$.
\begin{answer}
The likelihood function is
\[
L(\theta|\mathbf{x})
	= \prod_{i=1}^{n} f(x_i;\theta)
	= \prod_{i=1}^{n} \theta x^{\theta-1}
	= \theta^n\prod_{i=1}^{n} x_i^{\theta-1}.
\]
In particular, $L(1|\mathbf{x})=1$ and $L(2|\mathbf{x})=2^n\prod_{i=1}^{n} x_i$, so the likelihood ratio for testing $H_0$ against $H_1$ is given by
\[
\lambda(\mathbf{x})
	= \frac{L(1|\mathbf{x})}{L(2|\mathbf{x})}
	= \frac{1}{2^n\prod_{i=1}^{n} x_i}.
\]
By the Neymann-Pearson lemma, a most powerful test is given by
\[
C 	= \{\mathbf{x}:\lambda(\mathbf{x})\leq k\}
	= \left\{\mathbf{x}:\frac{1}{2^n\prod_{i=1}^{n} x_i} \leq k\right\}
	= \left\{\mathbf{x}:\prod_{i=1}^{n} x_i \geq \frac{1}{k2^n}\right\}.
\]
Hence a most powerful test is given by a set of the form $C = \big\{\mathbf{x}: \prod_{i=1}^n x_i \geq c\big\}$, where $c$ is chosen to fix the size of the test.
\end{answer}

%==========================================================================
\question  % 8.1.7
Let $\mathbf{X}=(X_1,X_2,\ldots,X_n)$ be a random sample from the $N(\theta,100)$ distribution, and let
\[
C = \left\{\mathbf{x}:\frac{1}{n}\sum_{i=1}^n x_i \geq k\right\}
\]
where $k$ is a constant.
\ben
\it % (i)
Show that $C$ defines a most powerful test of $H_0:\theta=75$ against $H_1:\theta=78$.
\item % (ii)
Find values for $n$ and $k$ such that $\prob_{H_0}\left(\frac{1}{n}\sum_{i=1}^n X_i\geq k\right)=0.05$ and $\prob_{H_1}\left(\frac{1}{n}\sum_{i=1}^n X_i\geq k\right)=0.90$, approximately.
\een

\begin{answer}
\ben
\it % << (i)
The density function is $f(x;\theta,\sigma^2) = \displaystyle\frac{1}{\sigma\sqrt{2\pi}}\exp\left(-\frac{1}{2\sigma^2}(x-\theta)^2\right)$.

The likelihood ratio for $H_0:\theta=\theta_0$ against $H_1:\theta=\theta_1$ is therefore
\begin{align*}
\lambda(\mathbf{x})
	& = \frac{L(\theta_0;\mathbf{x})}{L(\theta_1;\mathbf{x})} \\
	& = \frac{\exp\left(-\frac{1}{2\sigma^2}\sum_i (x_i-\theta_0)^2\right)}{\exp\left(-\frac{1}{2\sigma^2}\sum_i(x_i-\theta_1)^2\right)} \\
	& = \exp\left(-\frac{1}{2\sigma^2}\sum_i\big[(x_i-\theta_0)^2 -(x_i-\theta_1)^2\big]\right) \\
	& = \exp\left(-\frac{1}{2\sigma^2}\big[-2(\theta_0-\theta_1)\sum_i x_i + n(\theta_0^2-\theta_1^2)\big]\right) \\
%	& = \exp\left(-\frac{1}{2\sigma^2}\big[-2n(\theta_0-\theta_1)\bar{x} + n(\theta_0^2-\theta_1^2)\big]\right) \\
\end{align*}

By the Neyman-Pearson lemma, a most powerful test is given by
\begin{align*}
\{\mathbf{x}:\lambda(\mathbf{x})\leq k\}
	& = \left\{\mathbf{x}:\exp\left(-\frac{1}{2\sigma^2}\big[-2(\theta_0-\theta_1)\sum_i x_i + n(\theta_0^2-\theta_1^2)\big]\right)\leq k\right\} \\
	& = \left\{\mathbf{x}\,:\,\frac{1}{n}\sum_i x_i \geq \frac{1}{2}(\theta_0+\theta_1) - \frac{\sigma^2\log k}{n(\theta_0-\theta_1)}\right\}
\end{align*}
where we have used the fact that $\theta_0 < \theta_1$.

\bigskip
Thus if $\theta_0<\theta_1$, the set $\displaystyle C=\left\{\mathbf{x}\,:\,\frac{1}{n}\sum_i x_i\geq k\right\}$ defines a most powerful test of $H_0:\theta=\theta_0$ against $H_1:\theta=\theta_1$.

\it % << (i)
Under $H_0:\theta=75$, each $X_i\sim N(75,100)$ so the sample mean is $\bar{X}\sim N(75,100/n)$. For a test of size $\alpha=0.05$, the critical value $c$ must be the $95$th percentile of the $N(75,100/n)$ distribution, so $c = 75 + 1.645(10/\sqrt{n})$ where $1.645$ is the $95$th percentile of the standard normal distribution $N(0,1)$.

Under $H_1:\theta=78$, each $X_i\sim N(78,100)$ so the sample mean is $\bar{X}\sim N(78,100/n)$. For a test of $\gamma = 0.9$, the critical value $c$ must be the $10$th percentile of the $N(78,100/n)$ distribution, so $c = 78 - 1.280(10/\sqrt{n})$ where $-1.280$ is the $10$th percentile of the standard normal distribution $N(0,1)$.

Equating these expressions for $c$, we obtain $\sqrt{n} = 10(1.645+1.280)/(78-75) = 9.75$ and therefore $n=95.0625$, which means that we need a sample size $n=96$ to ensure that the significance level does not exceed 0.05. Substituting for $\sqrt{n}$ in one of the above expression then yields the critical value $c = 76.6872$, which defines the critical region
\[
\displaystyle C=\left\{\mathbf{x}\,:\,\frac{1}{n}\sum_i x_i\geq 76.6872\right\}.
\]
As shown above, this defines a most powerful test of $H_0:\theta=75$ against $H_1:\theta=78$.
\een
\end{answer}

%==========================================================================
\question  % 8.1.4
Let $X_1,X_2,\ldots,X_{10}$ be a random sample from the $N(0,\sigma^2)$ distribution where $\sigma^2$ is unknown.
\ben
\it Find a most powerful test of size $\alpha=0.05$ for testing $H_0:\sigma^2=1$ against $H_1:\sigma^2=2$.
\it Is this also a most powerful test of $H_0:\sigma^2=1$ against $H_1:\sigma^2=4$?
\it Is this a most powerful test of $H_0:\sigma^2=1$ against the composite alternative $H_1:\sigma^2>1$?
\een

\begin{answer}
\ben
\it % << (i)
The density function is $f(x,\sigma^2) = \displaystyle\frac{1}{\sqrt{2\pi\sigma^2}}e^{-x^2/2\sigma^2}$, the likelihood function is
\[
L(\sigma^2;\mathbf{x})
	= \prod_{i=1}^{10} f(x_i,\sigma^2)
	= \left(\frac{1}{2\pi\sigma^2}\right)^{5}\exp\left(-\frac{1}{2\sigma^2}\sum_i x_i^2\right)
\]
and the likelihood ratio for $H_0:\sigma^2=1$ against $H_1:\sigma^2=2$ is therefore
\[
\lambda(\mathbf{x})
	= \frac{L(1;\mathbf{x})}{L(2;\mathbf{x})}
	= 2^{5} \exp\left(-\frac{1}{4}\sum_i x_i^2\right).
\]
Let $\mathbf{X}=(X_1,X_2,\ldots,X_{10})$ denote the random sample. By the Neymann-Pearson lemma, a best critical region for the test is
\begin{align*}
C 	= \{\mathbf{x}:\lambda(\mathbf{x})\leq k\}
	& = \left\{\mathbf{x}:2^{5} \exp\left(-\frac{1}{4}\sum_{i=1}^{10} x_i^2\right) \leq k\right\} \\
	& = \left\{\mathbf{x}:\sum_{i=1}^{10} x_i^2 \geq 4\log\left(\frac{k}{2^{5}}\right)\right\} \\
	& = \left\{\mathbf{x}:\sum_{i=1}^{10} x_i^2 \geq k'\right\} \\
\end{align*}
where $k'$ is chosen to ensure that $\prob(\mathbf{X}\in C;H_0)=0.05$.
\it % << (ii)
Under the null hypothesis we have $X_i\sim N(0,1)$, so the sum-of-squares $\sum_{i=1}^{10} X_i^2$ has chi-squared distribution with $10$ degrees-of-freedom. From tables, the critical value at $\alpha=0.05$ for this distribution is $18.307$, so the critical region is given by
\[
C = \left\{\mathbf{x}\,:\,\sum_{i=1}^{10} x_i^2 \geq 18.307\right\}
\]
This is also a most powerful test of $H_0:\sigma^2=1$ against $H_1:\sigma^2=4$.
\it
The argument of part (b) holds for any simple alternative hypthesis $H_1:\sigma^2=\sigma^2_1$ provided $\sigma^2_1 > 1$. This is therefore a \emph{uniformly most powerful test} of size $\alpha$ for testing $H_0:\sigma^2=1$ against every simple alternative in the composite hypothesis $H_1:\sigma^2>1$.
\een
\end{answer}

\end{questions}
\end{exercise}


\section{Value Function Approximation}

In function approximation, the value function $v_\pi$ becomes:
\begin{equation}
	v_\pi (s) \approx \widehat{v}(s, \mathbf{W}) \text{  ,  } \mathbf{W} \in \mathbb{R}^d
\end{equation}

It is a supervised learning with data pair:
\begin{equation}
	\begin{aligned}
		\langle S_1, R_1 &+ \gamma \widehat{v}(S_2, \mathbf{W}) \rangle \\
		\langle S_2, R_2 &+ \gamma \widehat{v}(S_3, \mathbf{W}) \rangle \\
		&\dots
	\end{aligned}
\end{equation}

\subsection{On-policy Prediction}

\subsubsection{Requirement}

The approximate function $\widehat{v}(s, \mathbf{W})$ has these requirements:
\begin{itemize}
	\item learning needs to be online
	\item the learning target is non-stationary and can change over time
	\item differentiable of $\mathbf{W}$ for all $s \in \mathcal{S}$
\end{itemize}

\subsubsection{Prediction Objective}

In tabular case there is no estimation of value function quality because it will converge to the end goal. However in function approximation there is no exact value function and the quality need to be estimated.

Assume the state is distributed under $\mu(s)>0, \sum\limits_s \mu(s) = 1$, the \cindex{mean squared value error} $\overline{\text{VE}}$ is defined as:
\begin{equation}
	\overline{\text{VE}} = \sum_{s \in \mathcal{S}} \mu (s) \Big ( v_\pi (s) - \widehat{v}(s,\mathbf{W}) \Big )^2
\end{equation}

$\mu(s)$ can be chosen as the fraction of time for episodes, and stationary distribution for continuous tasks.

$\widehat{v}$ is chosen to be differentiable function, which could be:
\begin{itemize}
	\item linear combination of features
	\item neural network
\end{itemize}

\subsubsection{Stochastic Gradient Descent}

Assume $\mu$ is uniform distribution, the $\mathbf{W}$ is updated as:
\begin{equation}\label{wstogradesc}
	\begin{aligned}
		\mathbf{W}_{t+1} &= \mathbf{W}_t - \frac{1}{2} \alpha \nabla_{\mathbf{W}} \Big ( v_\pi(S_t) - \widehat{v}(S_t, \mathbf{W}_t) \Big )^2 \\
		&= \mathbf{W}_t+ \alpha \Big ( v_\pi(S_t) - \widehat{v}(S_t, \mathbf{W}_t) \Big ) \nabla_{\mathbf{W}}  \widehat{v}(S_t, \mathbf{W}_t)
	\end{aligned}
\end{equation}

where $\nabla_{\mathbf{W}} f(\mathbf{W})$ is defined as:
\begin{equation}
	\nabla_{\mathbf{W}} f(\mathbf{W}) = \left( \frac{\partial f(\mathbf{W})}{\partial \mathbf{W}_1}, \frac{\partial f(\mathbf{W})}{\partial \mathbf{W}_2}, \dots, \frac{\partial f(\mathbf{W})}{\partial \mathbf{W}_d}  \right)^{\top}
\end{equation}

 Formula (\ref{wstogradesc}) need to follow these in order to converge to a local minimum:
\begin{itemize}
	\item $\alpha$ follow equation (\ref{convergenceofsequence}).
	\item $v_\pi(S_t) $ is an unbiased estimate of $v$.
\end{itemize}

See Algorithm (\ref{algo:gradmcapprox}) for detail.

In practice, the $v_\pi$ in formula (\ref{wstogradesc}) is chosen as:
\begin{itemize}
	\item for MC, the target is the return $G_t$:
	\item for TD(0), the target is $R_{t+1} + \gamma \widehat{v}(S_{t+1}, \mathbf{W} )$
	\item for TD($\lambda$), the target is $G_t^\lambda$ which could be forward or backward view
\end{itemize}

\begin{algorithm}
	\caption{gradient MC, estimate $\widehat{v} \approx v_\pi$ }\label{algo:gradmcapprox}

	\begin{algorithmic}[1]
		\State $\mathbf{W} \gets $ random

		\Statex

		\Loop
			\State generate $S_0,A_0,R_1,\dots,R_T,S_T$ using $\pi$
			\For{$t \gets \Big[0, 1, \dots, T-1 \Big]$}
				\State $\mathbf{W} \gets \mathbf{W} + \alpha \Big( G_t - \widehat{v}(S_t, \mathbf{W}) \Big) \nabla_{\mathbf{W}} \widehat{v}(S_t, \mathbf{W})$
			\EndFor
		\EndLoop
	\end{algorithmic}
\end{algorithm}

 \subsection{Semi-gradient methods}

 In formula (\ref{wstogradesc}) if $v_\pi(S_t)$ depends on $\mathbf{W}$, the formula is biased and will not converge as the true gradient descent methods. It is called \cindex{semi-gradient methods}.

 Bootstrapping estimate belongs to this category.

\subsection{Linear Methods}

Suppose $\widehat{v}$ is linear: $\widehat{v}(s,\mathbf{W})= \mathbf{W}^\top \mathbf{X}(s) = \sum\limits_{i=1}^d w_i x_i(s)$. $\mathbf{X}(s)$ is called feature vector represents state $s$.  Formula (\ref{wstogradesc}) now becomes:

\begin{equation}\label{lineargraddesc}
	\mathbf{W}_{t+1} = \mathbf{W}_t+ \alpha \Big ( v_\pi(S_t) - \widehat{v}(S_t, \mathbf{W}_t) \Big ) \mathbf{X}(S_t)
\end{equation}

In linear case all local optimum is global optimum.

\subsubsection{Semi-gradient Linear Methods TD(0)}

semi-gradient TD(0) algorithms also converges under linear function. But it converges to a point near the local optimum, rather than global minimum.

\begin{equation}\label{wstosemigradesc}
	\begin{aligned}
		\mathbf{W}_{t+1} &= \mathbf{W}_t+ \alpha \Big ( R_{t+1} + \gamma \mathbf{W}_t^\top \mathbf{X}_{t+1} - \mathbf{W}_t^\top \mathbf{X}_t \Big ) \mathbf{X} \\
		&= \mathbf{W}_t+ \alpha \Big ( R_{t+1}\mathbf{X}_t + \mathbf{X}_t ( \mathbf{X}_t - \gamma \mathbf{X}_{t+1})^\top \mathbf{W}_t  \Big )
	\end{aligned}
\end{equation}

The expected next weight vector could be written as:

\begin{equation}\label{convergesequence}
	\mathbb{E}[\mathbf{W}_{t+1} | \mathbf{W}_t] = \mathbf{W}_t + \alpha (\textbf{b} - \mathbf{A} \mathbf{W}_t)
\end{equation}

where

\begin{equation}
	\textbf{b} = \mathbb{E}[R_{t+1} \mathbf{X}_t] \in \mathcal{R}^d
\end{equation}

and

\begin{equation}
	\mathbf{A} =  \mathbb{E}[\mathbf{X}_t ( \mathbf{X}_t - \gamma \mathbf{X}_{t+1})^\top]
\end{equation}

If formula (\ref{convergesequence}) converges and is unbiased, it will converge to $\mathbf{W}_{TD}$ at which:

\begin{equation}\label{solvesemilr}
	\begin{aligned}
		\textbf{b} - \mathbf{A} \mathbf{W}_{TD} &= 0\\
		\textbf{b} &= \mathbf{A} \mathbf{W}_{TD} \\
		\mathbf{W}_{TD} &= \mathbf{A}^{-1} \textbf{b}
	\end{aligned}
\end{equation}

The solution of formula (\ref{solvesemilr}) is around global minimum:

\begin{equation}\label{semigradientlrerror}
	\overline{\text{VE}}(\mathbf{W}_{TD}) \leq \frac{1}{1-\gamma} \underset{\mathbf{W}}{\min}\ \overline{\text{VE}}(\mathbf{W})
\end{equation}

formula (\ref{semigradientlrerror}) applies to other on-policy bootstrapping methods as well, such as semi-gradient DP, semi-gradient action value methods.

\subsubsection{Least-Squares TD}

In LSTD, the $A$ and $b$ in formula (\ref{solvesemilr}) is defined as:
\begin{equation}
	\widehat{A}_t = \sum_{k=0}^{t-1} \mathbf{X}_k (\mathbf{X}_k - \gamma \mathbf{X}_{k+1} )^\top + \varepsilon \mathbf{I}
\end{equation}
and
\begin{equation}
	\widehat{b}_t=\sum_{k=0}^{t-1}R_{t+1}\mathbf{X}_k
\end{equation}

A small $\varepsilon > 0$ is added to ensure $\widehat{A}_t$ is always invertible.

$w_t$ is now defined as $w_t=\widehat{A}_t^{-1} \widehat{b}_t$.

There is a \cindex{Sherman-Morrison formula} that simplify the calculation of $\widehat{A}$:
\begin{equation}
	\begin{aligned}
		\widehat{A}_t^{-1} &= \Big (\widehat{A}_{t-1} + \mathbf{X}_t (\mathbf{X}_t - \gamma \mathbf{X}_{t+1})^\top \Big)^{-1}\\
		&= \widehat{A}_{t-1}^{-1} - \frac{\widehat{A}_{t-1}^{-1} \mathbf{X}_t (\mathbf{X}_t - \gamma \mathbf{X}_{t+1})^\top \widehat{A}_{t-1}^{-1} }{1 + (\mathbf{X}_t - \gamma \mathbf{X}_{t+1})^\top \widehat{A}_{t-1}^{-1} \mathbf{X}_t }
	\end{aligned}
\end{equation}

with $\widehat{A}_0 = \varepsilon \mathbf{I}$

LSTD does not require $\alpha$, but it needs $\varepsilon$ which has these problems:
\begin{itemize}
	\item small $\varepsilon$: the inverse calculation will vary widly
	\item big $\varepsilon$: the learning is slow
	\item no $\alpha$: it never forgets
\end{itemize}

\subsection{On-policy Control}

In approximate control, the $v$ in formula (\ref{wstogradesc}) is changed to $q$:
\begin{equation}
	\mathbf{W}_{t+1} = \mathbf{W}_t+ \alpha \Big ( U_t - \widehat{q}(S_t, A_t, \mathbf{W}_t) \Big ) \nabla_{\mathbf{W}}  \widehat{q}(S_t, A_t, \mathbf{W}_t)
\end{equation}

As before, the $U_t$ could be :
\begin{itemize}
	\item for MC, the target is the return $G_t$:
	\item for TD(0), the target is $R_{t+1} + \gamma \widehat{q}(S_{t+1}, A_{t+1}, \mathbf{W} )$
	\item for TD($\lambda$), the target is $G_t^\lambda$ with forward and backward view
\end{itemize}


% !TEX root = main.tex

%------------------------------------------------
\chapter{First Order Differentials}

A first order (ordinary differential equations) is an equation of the form : $$\frac{dy}{dt} = f(t,y).$$

Here any solution $y$ is a real or complex valued function of a single real variable and $f$ is a function of 2 variables.

In Newtonian notation the same equation would be written as : $$y'(t) = f(t,y(t)).$$
\section{Separable Equations}

A separable equation is an equation of the form : $$\frac{dy}{dt} = g(t)h(y).$$
Suppose that $h$ never takes the value 0. Then we can write the equation as : $$\frac{1}{h(y)}\frac{dy}{dt}  = g(t).$$
We can also (at least on principle) find a function $F$ whose derivative is $\frac{1}{h}$ : $$F' = \frac{1}{h},$$
hence : $F'(y(t))\frac{dy}{dt} = g(t) \implies \frac{d}{dt}[F(y(t))] = g(t).$

\bigskip

\begin{example}  Solve $$\frac{dy}{dt} = (1+y^2)(1+\alpha{y})$$ where $\alpha \geq 0$ is constant. \newline
\begin{solution} Write the equation as $$\frac{1}{1+y^2}\frac{dy}{dt} = 1 + \alpha{t}.$$
Here we need $F'(y) = \frac{1}{1+y^2}$ so we choose : $$F(y) = tan^{-1}(y)$$ and get : $$\frac{d}{dt}(tan^{-1}(y(t))) = 1 + \alpha{t},$$
and so by integration $$tan^{-1}(y(t)) = t +\alpha{\frac{t^2}{2}} + \beta$$ where $\beta$ is constant.
Hence : $y(t) = tan( t +\alpha\frac{t^2}{2} + \beta).$
Remark : The solution of this equation always blows up in finite time even when $\alpha = \beta = 0 $. In this case the solution blows up at $t = \frac{\pi}{2}$.
\end{solution}
\end{example}
\bigskip

\begin{example}  Find the solution of $$\frac{dy}{dt} = 2\sqrt[]{y}.$$
\begin{solution} Divide by $2\sqrt[]{y}$ to get $$ \frac{1}{2\sqrt[]{y}}\frac{dy}{dt} = 1.$$
For this case : $$F'(y) = \frac{1}{2\sqrt[]{y}}$$ and so we choose $F(y) + \sqrt[]{y}$ and get $$\frac{d}{dt}(\sqrt[]{y(t)} = 1.$$
So, on integration $$\sqrt[]{y(t)} = t + c $$ where $c$ is a constant.
This gives $y(t) = (t + c)^2.$
Suppose we want $y(0) = 0$ we get : $$0 = y(0) = (0 + c)^2 = c^2.$$
Giving $c = 0$. Thus $y(t) = t^2.$

Warning : This is not the only solution of $\frac{dy}{dt} = 2\sqrt[]{y}$ which satisfies $y(0) = 0$, as we said previously. We could also have $y(t) = 0$ for all $t$.
\end{solution}
\end{example}

\bigskip

\begin{example}  Solve the equation : $$\frac{dy}{dt} = m(1 - m).$$
\begin{solution} Suppose $ m \neq 0$, $ m \neq 1$ Then $$\frac{1}{m(1-m)}\frac{dm}{dt} = 1.$$
We need to find a function $F$ such that :
$$F'(m) = \frac{1}{m(1-m)} = \frac{1}{m} + \frac{1}{1-m}.$$
A suitable $F$ is: $$F(m) = \log(m) - \log(1-m) = \log(\frac{m}{1-m})$$ and so our equation is : $$\frac{d}{dt}[\log(\frac{m}{1-m})] = 1 \implies \log(\frac{m}{1-m}) = t + c$$ where $c$ is a constant, thus $$\frac{m(t)}{1-m(t)} = \exp(t + c) = \alpha \exp(t)$$ where $\alpha = \exp(c)$ is also a constant. Suppose that $m(0) = \mu$ then : $$\frac{\mu}{1-\mu} = \alpha.$$
The equation for $m(t)$ can be rearranged as $$\mu(t) = \frac{\alpha \exp(t)}{1 + \alpha \exp(t)} = \frac{1}{\frac{1}{\alpha} \exp(-t) + 1} = \frac{1}{(\frac{1 - \mu}{\mu}) \exp(-t) + 1} = \frac{\mu}{(1-\mu) \exp(-t) = \mu}.$$
\end{solution}
\end{example}

\bigskip

\begin{remark}
\begin{itemize}
\item If we take $m(0) = \mu = 0$ we get $m(t) = 0$ for all $t$ then : \newline
If $0 < \mu , 1$ then $(1 - \mu)\exp(-t) > 0$ and hence $$ 0 , m(t) < \frac{\mu}{0 + \mu} = 1$$ Thus $$\frac{dm}{dt} = m(1-m) > 0 \mbox{ also } \lim m(t) = 1.$$

\item If we take $m(0) = \mu = 1$ we get $m(t) = 1$ for all $t$ then :
\newline
If $\mu > 1$ then the expression $$m(t) = \frac{\mu}{\mu(1-\exp(-t))+ \exp(-t)}$$ tells us that $m(t) > 0$ whilst the expression $$m(t) = \frac{\mu}{(1-m)\exp(-t) + \mu}$$ tells us that $m(t) > 1$ for all $t \geq 0$. Also $\frac{dm}{dt} = m(1-m) < 0$, so $m$ is strictly decreasing. Also $\lim m(t) = 1$.

\end{itemize}
\end{remark}
\section{Homogeneous Equations }

A homogeneous equation is an equation of the form : $$\frac{dy}{dt} = f(\frac{y}{t}).$$
We introduce a new function $z = \frac{y}{t}$; more precisely $z(t) = \frac{y(t)}{t}$. Hence $y(t) = tz(t)$ which gives $$f(z) = \frac{dy}{dt} = z(t) + t\frac{dz}{dt}.$$
Rearranging yields: $$\frac{dz}{dt} = \frac{f(z) - z}{t} = \frac{1}{z}(f(z) - z)$$ which is now separable.

\bigskip

\begin{example} Solve the equation : $$t^2\frac{dy}{dt} = ty + t^2 + y^2$$ and examine the behavior of the solution as $t$ tends to 0.

\begin{solution} Divide by $t^2$ to get $$\frac{dy}{dt} = \frac{y}{t} + 1 +(\frac{y}{t})^2 = f(\frac{y}{t})$$ where $f(z) = z + 1 + z^2$. We let $y = tz$ and get $$\frac{dz}{dt} = \frac{1}{t}(f(z)-z) = \frac{1}{t}(1+z^2)\implies \frac{1}{1+z^2}\frac{dz}{dt} = \frac{1}{z},$$
so $$\frac{1}{1 +z^2}dz = \frac{1}{t}dt.$$
Integrate both sides to obtain $tan^{-1} = \log(t) + c$ where $c$ is a constant of integration. This yields $$z = tan(c + \log(t)) \implies y - t*tan(c+\log(t)).$$
Put $c = \log\alpha$ where $\alpha = \exp(c)$ so that $$y = t*tan(\log\alpha + \log t) = t*tan(\log(\alpha t )).$$ This blows up whenever $\log(\alpha t) = (2k + 1)\frac{\pi}{2}$, $k\in \mathbb{Z}$ i.e $$\alpha t = \exp((2k + 1)\frac{\pi}{2}) \mbox{,} \mbox{ or } t = \frac{1}{\alpha}\exp((2k + 1)\frac{\pi}{2}).$$
Letting $k$ decrease to $-\infty$ through the negative integers we see that these are infinitely many points at which $y$ blows up, in any neighborhood of zero.
\end{solution}
\end{example}

\bigskip

\begin{example}  Solve the homogeneous equation : $$\frac{dy}{dt} = \frac{y}{t}^{\frac{1}{3}}.$$
\begin{solution} Put $y = tz, f(z) = z^{\frac{1}{3}}$, and obtain $$\frac{dz}{dt} = \frac{1}{t}(f(z) - z) = \frac{1}{t}(z^{\frac{1}{3}} - z).$$ This yields $$\frac{1}{z^\frac{1}{3} - z}dz = \frac{1}{t}dt.$$
Alternatively, the original equation is already separable, as $y^{\frac{-1}{3}}dy = t^\frac{-1}{3}dt$. By integration : $$y^\frac{2}{3} = t^\frac{2}{3} + c$$ where $c$ is a constant of integration, thus $$ y = (c + t^\frac{2}{3})^\frac{3}{2}.$$
\end{solution}
\end{example}

\section{Linear Equations}
A first-order differential linear equation is an equation of the form : $$\frac{dy}{dt} = a(t)y+b(t)$$ where $a(t)$ and $b(t)$ are given functions of the dependant variable $t$. For linear equations we can always write the solution $y$ explicitly as a function.

\subsection*{Solution Procedure}
\begin{enumerate}
\item Choose a function $p(t)$ such that $$\frac{dp}{dt} = a(t)$$
\item Observe that $$\frac{d}{dt}(ye^{-p(t)}) = \frac{dy}{dt}e^{-p(t)} + ye^{-p(t)} = (\frac{dy}{dt} - a(t)y)e^{-p(t)} = b(t)e{-p(t)}$$
\item Integrate the equation $$\frac{d}{dt}(ye^{-p(t)}) = b(t)e^{-p(t)}$$ This integrates to $$ye^{-p(t)} = \int b(t)e{-p(s)}ds + c$$ where $c$ is a constant
By doing this we get : $$y = e^p(t)(c+ \int b(s)e^{-p(s)}ds$$
\end{enumerate}
\begin{example} Solve the differential equation $$\frac{dy}{dt} = ky +\sin(t)$$ subject to the condition $y(0)=0$.
\begin{solution} Choose $p = \int kdt = kt$ where $a(t) = k$ and $b(t) = \sin(t)$.
Therefore $$\frac{d}{dt}(e^{-kt}y) = e^{-kt}\sin(t).$$ There are at least two ways of integrating the right hand side of the equation. One is to integrate by parts and the other is to use complex exponentials, therefore : $$\sin(t) = Im(e^{it}).$$ Hence : $$\int e^{-kt}\sin(t) dt \implies \int e^{-kt} Im(e^{-it}dt = Im(\int e^{-kt + it}dt) \implies Im(\frac{e^{(i-k)t}}{(i-k)}$$ $$= Im(\frac{(-i-k)(\cos(t) + \sin(t))*e^{-kt}}{1+k^2} = Im(\frac{(-k\cos(t) +\sin(t) +i(-\cos(t) - k\sin(t)))e^{-kt}}{1+k^2})$$ $$= \frac{-(\cos(t) + k\sin(t))e^{-kt}}{1+k^2}.$$
Therefore :
$$e^{-kt}y =   -(\frac{-(\cos(t) + k\sin(t))e^{-kt}}{1+k^2})$$ where $c$ is a constant and when $t=0$ and $y=0$ so : $$c - \frac{1}{1+k^2} = 0 \implies c = \frac{1}{1+k^2}$$
Thus the solution is $$y = \frac{e^{kt} - (\cos(t) + k\sin(t))}{1+k^2}.$$
\end{solution}
\end{example}
\bigskip

\begin{example} Solve the linear differential equation $$t^2\frac{dy}{dt} + 2ty = e^t$$ subject to $y(1) = 1.$ Is there a solution subject to the condition $y(0)=1$?
\begin{solution}
We can write the equation in the form $$\frac{dy}{dt}= \frac{-2}{t}y +\frac{e^t}{t^2}.$$ Then we find the integrating factors as before however in this case we can simplify the equation by inspection : $$t^2\frac{dy}{dt} + 2ty = \frac{d}{dt}(t^2y) = e^t.$$
Then by integration : $$t^2y = e^t + c \implies y = \frac{e^t + c}{t^2}$$ where $c$ is a constant.
The initial condition $y(1) = 1$ gives you $$ 1 = \frac{e + c}{1} \implies c = 1-e.$$ Thus $$y = \frac{e^t + 1 - e}{t^2}.$$ There is no solution which takes the value 1 when $t=0$! This is because when we write the equation as $\frac{dy}{dt}= f(t,y)$ we have $$\frac{dy}{dt} = \frac{-2y}{t} + \frac{e^t}{t^2}$$ and this function is badly behaved as $t \rightarrow 0$. The initial conditions can not be imposed at a point where the right hand side of the differential equation blows up.
\end{solution}
\end{example}

\bigskip

\begin{example}  Solve the equation $$(t\log(t))\frac{dy}{dt} + y = 3t^3.$$
\begin{solution}
First we must find an integrating factor which is $$\exp( \int \frac{1}{t\log(t)} dt).$$ Then by integration by substitution where $u = \log(t)$ and $\frac{du}{dt} = \frac{1}{t}$ we have that $$\int \frac{1}{t\log(t)} dt \implies \int \frac{1}{u}du = \log(u) \implies \log(\log(t)).$$
Now the integrating factor is $\exp(\log(\log(t))$ which is $\log(t)$ and therefore : $$\log(t)\frac{dy}{dt} + \frac{y}{t} = 3t^2 \implies \frac{d}{dt}[\log(t)y] = 3t^2.$$ Then by integration you get $$\log(t)y = \int3t^2 dt = t^3 + c$$ where $c$ is a constant. Thus by rearranging $$y = \frac{t^3 + c}{\log(t)}.$$
\end{solution}
\end{example}

\bigskip

\section{Bernoulli Equations}
A Bernoulli equation is an equation in the form $$\frac{dy}{dt} + a(t)y = b(t)y^n.$$
Where when $n = 0$ we have a linear equation and when $n = 1$ we have  linear and separable equation. We choose an integration factor $p(t)$ such that $$\frac{p'(t)}{p(t)} = a(t) \mbox{ so that } \frac{dy}{dt} + \frac{p'(t)}{p(t)}y = b(t)y^n.$$
Now we multiply both sides by $p(t)$ to get $$p(t)\frac{dy}{dt} + \frac{dp}{dt}y = p(t)b(t)y^n.$$ Then by differentiation $$\frac{d}{dt}(p(t)y(t0 = \frac{b(t)}{p(t)^{n-1}}(p(t)y(t))^n.$$ We introduce a new function $z$ by the formula $z(t) = p(t)y(t)$ which satisfies $$\frac{dz}{dt} = q(t)z^n \mbox{ where } q(t) = \frac{b(t)}{p(t)^{n-1}}.$$

\begin{example}
Solve the equation $$\frac{dy}{dt} + \frac{2}{t}y = \exp(t)y^2.$$
\begin{solution}
Multiply both sides by $t^2$ to obtain $$t^2\frac{dy}{dt} + 2ty = t^2\exp(t)y^2 \implies \frac{d}{dt}(t^2y) = \frac{1}{t^2}\exp(t)(t^2y)^2.$$ Let $z = t^2y$ such that $$\frac{dz}{dt}=\frac{1}{t^2}\exp(t)z^2$$ which is now a separable equation. This separates as $$\frac{1}{z^2}\frac{dz}{dt} = \frac{1}{t^2}\exp(t) \mbox{ or } \frac{d}{dt}(\frac{-1}{z}) = \frac{1}{t^2}\exp(t)$$ Any further progress depends on integrating the right hand side, probably with an incomplete gamma-function.
\end{solution}
\end{example}
\bigskip

\begin{example}
Solve the equation $$\frac{dy}{dt} = t\exp(t^2-y).$$
\begin{solution}
Thanks to the property $\exp(t^2-y) = \frac{\exp(t^2)}{\exp(y)}$ this is a separable equation. We rearrange it as $$\exp(y)\frac{dy}{dt} = t\exp(t^2) \implies \frac{d}{dt}\exp(y)) = \frac{d}{dt}(\frac{1}{2} \exp(t^2.)$$
Thus $$\exp(y) = \frac{1}{2} \exp(t^2) + c$$ where $c$ is a constant. Finally $$y = \log(c + \frac{1}{2} \exp(t^2)).$$ For the special case when $c = 0$ we get $$y = \log(\frac{1}{2} \exp(t^2)) \implies \log(\frac{1}{2}+t^2 = t^2 - \log(2).$$
\end{solution}
\end{example}

%------------------------------------------------
\endinput


\documentclass[letterpaper,options]{article}
\usepackage[]{amsmath,amssymb}
\usepackage[margin=1in]{geometry}
%preamble
\title{Asteroid Data Hunter MM2: Description of Solution}
\author{Kushal Agarwal (TopCoder Handle: Kushal1)\\ Computer Science\\California Institute of Technology\\ \texttt{kushal@caltech.edu}}
\date{\today}

\begin{document}
\maketitle

\section{Description of Algorithm}
\subsection{Source Extraction}
The first step in the algorithm is completed by the \texttt{findSources()} function. For each pixel $i,j$ in the image, let $g_{i,j}$,  be a two dimensional Gaussian function centered on that pixel with standard deviation in each axis of 1.05 pixels. Then, linear regression can be used to find the values of $\texttt{scaleFac}$ and $\texttt{offset}$ for which the function $(g_{i,j}\cdot\texttt{scaleFac}+\texttt{offset})$ best fits the values of the 5x5 grid of pixels centered on $i,j$.

This results in an $\texttt{offset}$ value and a $\texttt{scaleFac}$ value for each pixel. Noise in an image follows a Poisson Distribution, and the standard deviation of a Poisson process with mean $\lambda$ is $\sqrt{\lambda}$. So the expected amount of noise for a specific pixel is proportional to $\sqrt{\texttt{offset}}$ (the proportionality constant depends on the gain of the image, but I have ignored this by assuming all image have the same gain). Therefore I can compute a signal to noise ratio as $\texttt{SNR} =\texttt{scaleFac}/\sqrt{\texttt{offset}}$ for each pixel.

On this new image where each pixel's value is based on its $\texttt{SNR}$, I take two different thresholds and then apply a flood-fill to individually process each contiguous white region. (Contiguous means connected along the x or y directions; diagonally connected is not enough. Taking a threshold of an image means setting pixels above the threshold to white and those below it to black).

This is first done using a low threshold. Then any regions that are either very non square ($|length-width|>3$ pixels) or very large (area $ >16$ pixels), are kept white and the rest of the white pixels are made black. After this is complete, the pixels that are remaining white form a mask of points from which we will not make any detections.

Then I repeat the process of thresholding and flood-filling, this time with a higher threshold. This time I keep sources which are close to square ($|length-width|\le2$ pixels) and not too large (area $\le16$ pixels). For each region that meets these criteria I create a detection centered at the pixel from the region that has the the largest $\texttt{SNR}$ (as long as the mask at that pixel is black).
\subsection{Subpixel Source Centering}
The constructor for the \texttt{Detection} class next fine tunes the the position of these sources to subpixel accuracy. For each of the detections found above, I use Gauss-Newton minimization to minimize the sum of square error between a Gaussian function  given by $(g_{y,x}\cdot\texttt{scaleFac}+\texttt{offset})$ and the values of the 5x5 grid of pixels centered on the original detection. (Unlike the linear regression done to this same effect earlier, when using Gauss-Newton minimization I also allow $x$ and $y$ to vary and to do so continuously. Allowing $x$ and $y$ to vary break the linearity so non-linear tools must be used in place of linear regression). The result of Gauss-Newton minimization then yields four values describing the fit ($x$, $y$, $\texttt{scaleFac}$, and $\texttt{offset}$), two of which are the new more accurate center.

Finally, this coordinate, $(x,y)$, (in pixel space) is transformed into an RA/Dec space, and then it is transformed into the pixel space of the fourth image. This way all detections from all four images are in the same coordinate space.

\subsection{Removal of Stars}
From the set of aligned detections generated above, the function \texttt{removeStars()} attempts to remove any detections that were created by stationary stars. This helps reduce the total number of detections, reducing the chance of false detections, especially in images with a high density of stars. A set of four detections are labeled as stars if a detection is found in each of the three latter images within 1 pixel of a detection in the first image.

Finally, this process gives me a good opportunity to do a sanity check by checking to see that at least some of the sources were stars. If less than 15\% of detections were labeled stars and removed, that is an indication something has gone terribly wrong (the image has really bad artifacts or the WCS data is inaccurate), and so I do not process it to avoid the risk of accidentally seeing thousands of false detections. However, once mid-contest changes were made, I never saw this check ever taking effect.

\subsection{Finding Asteroids}
This is done by the \texttt{findAsteroids()} function. 	Let the images be taken at times $t_0,\ldots,t_3$. For each detection in the first image (call this detection $d_0$), I look for all detections within  5 pixels of that one, and for each one of those (call it $d_1$) I do the following:

The current estimate for the velocity is now
$$v=\frac{\text{position}(d_1 )-\text{position}(d_0)}{t_1-t_0}$$

So I expect $d_2$ to be at $d_1+v(t_2-t_1)$. For each detection within 1 pixel of this expected center, I repeat the process. The new velocity is
$$v=\frac{\text{position}(d_2 )-\text{position}(d_0 )}{t_2-t_0}$$

And so the new expected location for $d_3$ will be $d_2+v(t_3-t_2)$. And again I do the following for each detection within 1 pixel of this expected center.

For each tuple of four detections $(d_0,d_1,d_2,d_3)$ found in this manner, I will next check how linear its path is. First, the velocity during each of the three legs can be computed as $$\text{$v_i=\frac{\text{position}(d_{i+1} )-\text{position}(d_i)}{t_{i+1}-t_i}$ for $i=1..3.$}$$

Let $\texttt{avgLen}=(|v_0 |+|v_1 |+|v_2 |)/3$. This tells us about how fast the object is moving. Then I compute the linearity of motion by first computing $\texttt{error}=(|v_0-v_1 |+|v_0-v_2 |+|v_1-v_2 |)/\texttt{avgLen}$. The numerator indicates how closely the three velocity vectors were pointing in the same direction, the denominator normalizes this for objects that are moving at different speeds. Finally, any tuple of detections with $\texttt{error}<2.5$ and $\texttt{avgLen}>70$ (meaning the path is fairly linear and the object is moving faster than 70 pixels per day), is called an asteroid.

\subsection{Output Results}
The function \texttt{getAnswer()} returns all asteroids that were found. Each is output twice because most ground truths are also duplicated, and detections are sorted according to the $\texttt{error}$ parameter (described in the previous section) so that the more linear the path, the more confident I am in the detection.

Further, asteroids whose velocity lies outside some very rough bounds (velocity in dec direction is between -0.6 and 0.25 degrees/day, and velocity in the RA direction is between -0.5 and 0.5 degrees/day) are skipped (this really is not recommended, as it leads to a very marginal score increase at the cost of potentially strongly biasing the sample of asteroids detected).

Finally, sources with velocity in the RA direction less than -0.4 degrees/day are considered NEO. Obviously this categorization isn't great, but it's better than nothing.

\section{Overview of Implementation}
The implementation of such an algorithm is fairly straightforward because it is just a bunch of sequential steps, and most are fairly easy to code in a way that uses time and space efficiently. I have included an explanation of the job of each class I used in the next section (which would be helpful if you want to start reading through code). These are mostly helper classes because I feel the main part of the code has been documented above. Note that a good amount of code is obsolete, meaning it is included in my solution but is not being used because it was made to solve the alignment problems that were fixed mid-contest. So if you are puzzled by a particular function, first check to see that it is actually used.

Next, Section 2.2 has some important assorted implementation details for less obvious parts of the code.
\subsection{Classes}
\paragraph{\texttt{Vector} Class}
Represents a fixed size vector in $\mathbb{R}^n$ (for doing math) or $\mathbb{Z}^n$ (for indexing pixels in images) and allows you to add, subtract, multiply, and return these vectors. Be careful not to confuse my \texttt{\underline{V}ector} class (which is a vector in the mathematical sense) with the lowercase C++ STL \texttt{\underline{v}ector} class which I also use frequently, but which represents an expandable array.
\paragraph{\texttt{Matrix} Class}
Represents a standard matrix.
\paragraph{\texttt{Transformation} Class}
This was left over from older versions of the code and I never bothered to completely get rid of it, so it is unnecessary and rarely used. It represents a transformation (linear and translation) using homogeneous coordinates and is based on the \texttt{Matrix} class. It can be applied to \texttt{Vectors} using multiplication just like matrices.
\paragraph{\texttt{Image} Class}
Stores an image as a \texttt{vector<int>} and provides some 2D accessors into the 1D array. For debugging purposes it also includes a \texttt{saveAs} feature to save images in a raw byte format readable by Photoshop.
\paragraph{\texttt{Detection} Class}
Represents a detection in an image and the parameters associated with that detection. It is also responsible for doing the Gauss-Newton minimization described in the previous section to fine tune the position of detections
\paragraph{\texttt{DetectionSet} Class}
Stores a set of 4 \texttt{Detection} objects (one from each image) and stores the parameters related to the set of all 4 detections, such as the $\texttt{error}$.
\paragraph{\texttt{Header} Class}
Reads and parses the provided header format so that any field can be accessed by tag (used to read the \texttt{"MJD"} of each image to determine the time it was taken)
\paragraph{\texttt{WCSTransformation} Class} Stores the WCS data for a single image internally and provides functions to convert between pixel coordinates and RA/Dec coordinates using the code provided in the Java Tester.
\subsection{Assorted Implementation Details}
\paragraph{}
The linear regression fitting was done by setting the derivative of [the sum of square error between the function and the pixel values of the 5x5 grid] to equal 0 and solving the general case. The results in a system of two equations in two unknowns, which can be solved for a formula that was hardcoded into my code. However, a standard linear regression formula would yield the same result.
\paragraph{}
In many cases it was important to switch from \texttt{Vector} based image indexing to raw \texttt{int} based indexing because of performance issues. The point being that it is best to avoid using a wrapper class in inner loops as the compiler is unable to ``optimize the warper out.'' This change led to an approximately 10x speed up, which meant my entire solution runs in 20 sec per set of images, which makes debugging and trial and error much more practical.
\paragraph{}
Finally, you may have seen that many times while describing my algorithm, I reference finding detections in a certain pixel window. Instead of searching through all the detections each time, I do this more efficiently by creating an \texttt{Image} (ie. 2D array) of \texttt{Detection}\** variables. Then each “pixel” or “element of the array” (which is a \texttt{Detection}\**) is set to the address of a detection if one resides somewhere in that pixel, or otherwise it is set to \texttt{null} to indicate no detection there. This makes searches for detections within certain rectangular bounds very fast by just searching through all pixels in that rectangle rather than every single detection. (While this technique limits me to  one detection per pixel, this is not any issue because I should not have multiple detections in a pixel because of the way detections are generated)
\section{Parameter Derivation}
Most parameters were set by hand, using methods that are not worth discussing in detail (such as trial and error, and guessing and being close enough). The few exceptions to this include the following parameters
\paragraph{Asteroid Min and Max Movement} Were tuned by looking at images that had been aligned by eye and noticing that most asteroids moved at least 2 pixels over all 4 frames, and at most 5 pixels per frame. This lead to the 5 pixel search radius and the 70 pixels/day minimum motion limits
\paragraph{Gaussian Standard Deviation} The standard deviation 1.05 was found using the attached Mathematica file (\texttt{PSF.nb}) by fitting a Gaussian to a small 9x9 cropped image of a handpicked sample star from one of the training image (\texttt{PSF.png}). (However, I bet NASA knows the true point spread function function of their telescopes and can therefore use this function instead of a Gaussian)
\paragraph{Velocity Bounds}
These were computed by plotting the velocity (in RA/Dec space) of all the training objects using Mathematica (code: \texttt{NEO.nb}) (data: \texttt{detections.csv}). The plot allowed me to create the rough velocity bounds for both rejection and NEO classification by looking at the graphs by eye. (Again, I do not recommend this because it was a hack with positive---but very small---benefit)
\end{document}

Algorithms/Algorithm #5/Kushal1 - Algorithm.tex


\chapter{Regularization}
\label{ch:regularization}

There has to be some cure for overfitting. Something that helps us control it. To find it, let's check what are the values of the parameters $\theta$ under different degrees of polynomials.

\begin{figure}[h]
    \centering
    \includegraphics[scale=0.6]{workflow-overfitting.png}
    \caption{$\;$}
\end{figure}

With smaller degree polynomials values of $\theta$ stay small, but then as the degree goes up, the numbers get really large.

\begin{figure*}[h]
    \centering
    \newcommand{\third}{\includegraphics[scale=0.55]{data-table-third.png}}
    \newcommand{\eigth}{\includegraphics[scale=0.55]{data-table-eigth.png}}
    \infinitewidthbox{
    \stackinset{r}{-0.32\linewidth}{t}{+0.00\linewidth}{\eigth}{\third}\hspace{11cm}
    }
\end{figure*}

\marginnote{Which inference of linear model would overfit more, the one with high $\lambda$ or the one with low $\lambda$? What should the value of $\lambda$ be to cancel regularization? What if the value of $\lambda$ is really high, say 1000?}

More complex models can fit the training data better. The fitted curve can wiggle sharply. The derivatives of such functions are high, and so need be the coefficients $\theta$. If only we could force the linear regression to infer models with a small value of coefficients. Oh, but we can. Remember, we have started with the optimization function the linear regression minimizes — the sum of squared errors. We could simply add to this a sum of all $\theta$ squared. And ask the linear regression to minimize both terms. Perhaps we should weigh the part with $\theta$ squared, say, with some coefficient $\lambda$, just to control the level of regularization.

Here we go: we just reinvented regularization, a procedure that helps machine learning models not to overfit the training data. To observe the effects of regularization, we can give Polynomial Regression our own learner, which supports these kind of settings.

\newpage

The Linear Regression widget provides two types of regularization. \marginnote{Internally, if no learner is present on its input, the Polynomial Regression widget would use just ordinary, non-regularized linear regression.}Ridge regression is the one we have talked about and minimizes the sum of squared coefficients $\theta$. Lasso regression minimizes the sum of absolute value of coefficients. Although the difference may seem negligible, the consequences are that lasso regression may result in a large proportion of coefficients $\theta$ being zero, in this way performing feature subset selection.

% should it be so small?
\begin{marginfigure}
    \centering
    \includegraphics[scale=0.4]{lin-reg-ridge.png}
    \caption{$\;$}
\end{marginfigure}

\begin{figure}[h]
    \centering
    \includegraphics[scale=0.6]{workflow-ridge.png}
    \caption{$\;$}
\end{figure}

Now for the test. Increase the degree of polynomial to the max. Use Ridge Regression. Does the inferred model overfit the data? How does the degree of overfitting depend on regularization strength?


\section{Berry curvature}

The Berry curvature in the non-interacting crystal
for left and right circularly polarized
($\vc{ϵ}_±$) optical excitations for a given $\vK$
is $± 2 Ω_{+ ↑}^+ \of{k}$, where
\begin{subequations}
  \begin{align}
    Ω_{τ {\s}}^n \of{k}
    & = \vc{\hat{z}} · \vc{Ω}_{τ {\s}}^n \ofK, \\
    & = - n τ
        \left[ \frac{1}{2 k} \pderiv{}{k} \fnTheta{n} \right]
        \sin{\fnTheta{n}}, \\
    & = - n τ
        \frac{2 {\left( a t \right)}^2 \left( E_g - \tau s E_{\text{soc}} \right)}
        {{\left[{\left( 2 a t k \right)}^2
      + {\left( E_g - \tau s E_{\text{soc}} \right)}^2 \right]}^{3/2}}.
  \end{align}
\end{subequations}

The BCS ground state %
\footnote{%
  Note that the full ground state
  also contains the two lower filled bands,
  but those contribute zero net Berry curvature and may be ignored
  in this section and the next.}
is
\begin{subequations}
  \begin{align}
    \Ket{Ω}
    & = ∏_{\vK} \csc{β_{\vK}} γ_{\vK ↑} γ_{-\vK ↓} \Ket{0}, \\
    & = ∏_{\vK} \left( \cos{β_{\vK}} - \sin{β_{\vK}}
        c_{\vK ↑}^† c_{-\vK ↓}^† \right) \Ket{0}.
  \end{align}
\end{subequations}
This superconducting state is built up
from the quasiparticle eigenstates,
$\Ket{\vK}
= \csc{β_{\vK}} γ_{\vK ↑} γ_{-\vK ↓} \Ket{0}$,
of the $\vK$-dependent Hamiltonian
$λ_{\vK} \left( γ_{\vK ↑}^† γ_{\vK ↑}
+ γ_{-\vK ↓}^† γ_{-\vK ↓} \right)$.
The $z$-component of the Berry curvature of
the correlated state is zero,
\begin{equation}
  \vc{\hat{z}} · i ∇_{\vK} ⨯
  \Braket{\vK | ∇_{\vK} | \vK}
  = Ω_{+ ↑}^- \of{k} + Ω_{- ↓}^- \of{-k} = 0.
\end{equation}
A single optically excited state in the left valley
for a given $\vK$ is
${c_{+ ↑}^+}^† \ofK c_{+ ↑}^- \Ket{\vK}$,
which has a Berry curvature
$+2 \sin^6 {β_{\vK}} Ω_{+ ↑}^+ \of{k}$.
The corresponding excitation in the right valley
has a Berry curvature of the same magnitude but opposite sign.


\section{Implementation}
\label{sec:implementation}

\subsection{Training and Test Set}
In different experiments we use a different number of training examples.
Given the original dataset $\mathbb{D} = \{ \varepsilon_\mu, \tau(\varepsilon_\mu)\}_{\mu=1}^M$ (with $M = 5000$), we create the training $\mathbb{D}_{train}$ and test set $\mathbb{D}_{test}$ as follows:
\begin{equation*}
    \begin{split}
        \mathbb{D}_{train} &= \{ \varepsilon_\mu, \tau(\varepsilon_\mu)\}_{\mu=1}^P, \\
        \mathbb{D}_{test} &= \{ \varepsilon_\mu, \tau(\varepsilon_\mu)\}_{\mu=Q}^M,
    \end{split}
\end{equation*}
with $Q > P$.
Since we choose $P \in [1, 2000]$ in different experiments, we fix $Q = 2001$ for all experiments, in order to always test on the same dataset.

\subsection{Stochastic Gradient Descent}
The first step of our implementation of Stochastic Gradient Descent is to initialize the weights' vectors with random values and then normalize them to have unit norm $||w_j|| = 1$.
It is important to initialize the weights randomly in order to avoid symmetry problems:
since the weights gradient and the update rule is the same for all hidden unit, the updates at each epoch are also the same, which causes all units to learn the same final weights and reduces the representation power of the network.
Then, we perform a given number of updates:
at each iteration, we randomly select an example $\nu$ from the training dataset, compute the gradient with respect to the weights (see \cref{sub:gradients}) and update them according to \cref{eq:weights-update}.

At regular intervals (i.e. after a fixed number of iterations), we compute the error on both the entire training and the test sets, as shown in \cref{eq:cost-total}.


\section{Limits, colimits, and adjunctions}\label{906}
% Some organizational stuff, unnecessary for the book:
% Hi! This is 18.906. I'm glad to see familiar faces, and new faces. Let me introduce the course. It's a pset based course, with 6 psets. The first one is due Feb 22. It's due every two weeks, so I can't formulate all porblems right at the beginning of the two weeks. I'll try to have the psets done a week before the psets are due. So the first couple problems are up for this pset. You'll be glad to hear that there's no final. I don't think I'll do it again, not this term.

% I'll have office hours every week, but I don't know when yet. Hood's the grader and he'll have office hours. There's a course website which can be easily found. What's the course about? Really homotopy theory. 905 was homology and cohomology.

%Any questions? I try to correspond problems in the psets and lectures.
\subsection{Limits and colimits}
%I'm interested in ``construction''. In 905, I began by talking about category theory. I just won't introduce basic concepts again.
We will freely use the theory developed in the first part of this book (see \S \ref{categories}).
Suppose $\cI$ is a small category (so that it has a \emph{set} of objects),
and let $\cc$ be another category.
\begin{definition}
    Let $X:\cI\to\cc$ be a functor.
    A \emph{cone under $X$} is a natural transformation $\eta$ from $X$ to a constant functor;
    explicitly, this means that for every object $i$ of $\cI$, we must have a map $\eta_i: X_i\to Y$,
    such that for every $f:i\to j$ in $\cI$, the following diagram commutes:
    \begin{equation*}
	\xymatrix{
	    X_i\ar[d]^{f_\ast}\ar[dr]^{\eta_i} & \\
	    X_j\ar[r]^{\eta_j} & Y.
	    }
    \end{equation*}
    A \emph{colimit} of $X$ is an initial cone $(L,\tau_i)$ under $X$;
    explicitly, this means that for all cones $(Y,\eta_i)$ under $X$,
    there exists a unique natural transformation $h:L\to Y$ such that $h\circ \tau_i = \eta_i$.
\end{definition}
As always for category theoretic concepts, some examples are in order.
\begin{example}\label{coproductsarecolimits}
    If $\cI$ is a discrete category (i.e., only a set, with identity maps), the colimit of any functor $\cI\to\cc$
    is the coproduct. This already illustrates an important point about colimits: they need not exist in general
    (since, for example, coproducts need not exist in a general category).
    Examples of categories $\cc$ where the colimit of a functor $\cI\to\cc$ exists:
    if $\cc$ is sets, or spaces, the colimit is the disjoint union.
    If $\cc=\mathbf{Ab}$, a candidate for the colimit would be the product:
    but this only works if $\cI$ is finite; in general, the correct thing is to take the (possibly infinite) direct sum.
\end{example}
\begin{example}
    Let $\cI = \mathbf{N}$, considered as a category via its natural poset structure;
    then a functor $\cI\to \cc$ is simply a linear system of objects and morphisms in $\cc$.
    As a specific example, suppose $\cc = \mathbf{Ab}$, and
    consider the diagram $X:\cI \to \cc$ defined by the system
    $$\Z\xrightarrow{2}\Z\xrightarrow{3}\Z\to\cdots$$
    The colimit of this diagram is $\QQ$, where the maps are:
    \begin{equation*}
	\xymatrix{
	    \Z\ar[r]^2\ar[dr]^1 & \Z\ar[r]^3\ar[d]^{1/2} & \Z\ar[r]^4\ar[dl]^{1/3!} & \cdots\\
	    & \QQ & &
	    }
    \end{equation*}
\end{example}
\begin{example}\label{colimitgroupaction}
    Let $G$ be a group; we can view this as a category with one object, where the morphisms are the elements of the group
    (composition is given by the group structure).
    If $\cc = \Top$ is the category of topological spaces, a functor $G\to \cc$ is simply a
    group action on a topological space $X$.
    The colimit of this functor is the orbit space of the $G$-action on $X$.
\end{example}
\begin{example}
    Let $\cI$ be the category whose objects and morphisms are determined by the following graph:
    \begin{equation*}
	\xymatrix{ & a\ar[dl]\ar[dr] & \\
	b & & c.}
    \end{equation*}
    The colimit of a diagram $\cI\to \cc$ is called a \emph{pushout}.

    If $\cc=\Top$, again, a functor $\cI\to \cc$ is determined by a diagram of spaces:
    \begin{equation*}
	\xymatrix{
	    & A\ar[dl]^f\ar[dr]^g & \\
	    B & & C.
	    }
    \end{equation*}
    The colimit of such a functor is just the pushout $B\cup_A C:= B\sqcup C/\sim$, where $f(a)\sim g(a)$ for all $a\in A$.
    We have already seen this in action before: the same construction appears in the process of attaching cells to CW-complexes.

    If $\cc$ is the category of groups, instead, the colimit of such a functor is the free product quotiented out
    by a certain relation (the same as for topological spaces); this is called the \emph{amalgamated free product}.
\end{example}
\begin{example}
    Suppose $\cI$ is the category defined by the following graph:
\begin{equation*}
    \begin{tikzcd}
	a\ar[r,shift left=.75ex]\ar[r,shift right=.75ex] & b.
    \end{tikzcd}
\end{equation*}
    The colimit of a diagram $\cI\to\cc$ is called the \emph{coequalizer} of the diagram.
\end{example}

    One can also consider cones \emph{over} a diagram $X:\cI\to\cc$: this is simply a cone in the opposite category.
\begin{definition}
    With notation as above, the \emph{limit} of a diagram $X:\cI\to\cc$ is a terminal object in cones over $X$.
\end{definition}
For instance, products are limits, just like in Example \ref{coproductsarecolimits}.
(This example also shows that abelian groups satisfy an interesting property: finite products are the same as finite coproducts!)
\begin{exercise}
    Revisit the examples provided above: what is the limit of each diagram?
    For instance, the limit of the diagram described in Example \ref{colimitgroupaction} is just the fixed points!
\end{exercise}
\subsection{Adjoint functors}
Adjoint functors are very useful --- and very natural --- objects.
We already have an example: let $\cc^\cI$ be the functor category $\Fun(\cI,\cc)$.
(We've been working in this category this whole time!)
Let's make an additional assumption on $\cc$, namely that all $\cI$-indexed colimits exist.
All examples considered above satisfy this assumption.

There is a functor $\cc\to \cc^\cI$, given by sending any object to the constant functor taking that value.
The process of taking the colimit of a diagram supplies us with a functor $\cc^\cI\to \cc$.
We can characterize this functor via a formula\footnote{There is an analogous formula for the limit of a diagram:
$$\cc(W,\lim_{i\in \cI} X_i) = \cc^{\cI}(\mathrm{const}_W,X).$$}:
$$\cc(\colim_{i\in \cI} X_i,Y) = \cc^\cI(X,\mathrm{const}_Y),$$
where $X$ is some functor from $\cI$ to $\cc$.
This formula is reminiscent of the adjunction operator in linear algebra,
and is in fact our first example of an adjunction.

\begin{definition}
    Let $\cc,\cd$ be categories,
    with specified functors $F:\cc\to \cd$ and $G:\cd\to\cc$.
    An \emph{adjunction between $F$ and $G$} is an isomorphism:
    $$\cd(FX,Y) = \cc(X,GY),$$
    which is natural in $X$ and $Y$.
    In this situation, we say that $F$ is a \emph{left adjoint} of $G$ and $G$ is a \emph{right adjoint} of $X$.
    %People typically write left adjoints on the top.
\end{definition}
This notion was invented by Dan Kan, who worked in the MIT mathematics department until he passed away in 2013.

We've already seen an example above, but here is another one:
\begin{definition}[Free groups]
    There is a forgetful functor $u:\mathrm{Grp}\to\mathrm{Set}$.
    Any set $X$ gives rise to a group $FX$, namely the free group on $X$ elements.
    This is determined by a universal property:
    set maps $X\to u\Gamma$ are the same as group maps $FX\to \Gamma$,
    where $\Gamma$ is any group.
    This is exactly saying that the free group functor the left adjoint to the forgetful functor $u$.
\end{definition}
In general, ``free objects'' come from left adjoints to forgetful functors.

\begin{definition}
    A category $\cc$ is said to be \emph{cocomplete} if all (small) colimits exist in $\cc$.
    Similarly, one says that $\cc$ is \emph{complete} if all (small) limits exist in $\cc$.
\end{definition}
\subsection{The Yoneda lemma}%This belongs to the next lecture, but fits in better here.
One of the many important concepts in category theory is that an object is determined by
the collection of all maps out of it.
The Yoneda lemma is a way of making this precise.
An important reason to even bother thinking about objects in this fashion comes from our
discussion of colimits.
Namely, how do we even know that the notion is well-defined?

The colimit of an object is characterized by maps out of it; precisely:
$$\cc(\colim_{j\in\cJ}X_j,Y) = \cc^\cJ(X_\bullet,\mathrm{const}_Y).$$
The two sides are naturally isomorphic, but if the colimit exists, how do we know that it
is unique?
This is solved by Yoneda lemma\footnote{Sometimes ``you-need-a-lemma''!}:
\begin{theorem}[Yoneda lemma]
    Consider the functor $\cc(X,-):\cc\to\set$. Suppose $G:\cc\to\set$ is another functor. It turns out that:
    $$\mathrm{nt}(\cc(X,-),G)\simeq G(X).$$
\end{theorem}
\begin{proof}
    Let $x\in G(X)$.
    Define a natural transformation that sends a map $f:X\to Y$ to $f_\ast(x)\in G(Y)$.
    On the other hand, we can send a natural transformation $\theta:C(X,-)\to G$
    to $\theta_X(1_X)$.
    Proving that these are inverses is left as an exercise --- largely in notation --- to the reader.
\end{proof}
In particular, if $G=\cc(Y,-)$
--- these are called \emph{corepresentable} functors ---
then $\mathrm{nt}(\cc(X,-),\cc(Y,-))\simeq \cc(Y,X)$.
Simply put, natural isomorphisms $\cc(X,-)\to \cc(Y,-)$ are the same as isomorphisms $Y\to X$.
As a consequence, the object that a corepresentable functor corepresents is unique
(at least up to isomorphism).

From the Yoneda lemma, we can obtain some pretty miraculous conclusions.
For instance, functors with left and/or right adjoints are very well-behaved
(the ``constant functor'' functor is an example where both adjoints exist),
as the following theorem tells us.
\begin{theorem}\label{adjointslimits}
    Let $F:\cc\to\cd$ be a functor.
    If $F$ admits a right adjoint, it preserves colimits.
    Dually, if $F$ admits a left adjoint, it preserves limits.
\end{theorem}
\begin{proof}
    We'll prove the first statement, and leave the other as an (easy) exercise.
    Let $F:\cc\to\cd$ be a functor that admits a right adjoint $G$, and let $X:\cI\to\cc$ be a
    small $\cI$-indexed diagram in $\cc$.
    For any object $Y$ of $\cc$, there is an isomorphism
    $$\Hom(\colim_{\cI} X, Y) \simeq \lim_{\cI}\Hom(X, Y).$$
    This follows easily from the definition of a colimit.
    Let $Y$ be any object of $\cd$; then, we have:
    \begin{align*}
	\cd(F(\colim_{\cI} X), Y) & \simeq \cc(\colim_{\cI} X, G(Y))\\
	& \simeq \lim_{\cI}\cc(X, G(Y))\\
	& \simeq \lim_{\cI}\cd(F(X), Y)\\
	& \simeq \cd(\colim_{\cI}F(X), Y).
    \end{align*}
    The Yoneda lemma now finishes the job.
\end{proof}


\lab{Metropolis Algorithm}{Metropolis Algorithm}
\objective{Understand the basic principles of the Metropolis algorithm and apply these ideas to the
Ising Model.}

\section*{The Metropolis Algorithm}
Sampling from a given probability distribution is an important task in many different applications found throughout the sciences.
When these distributions are complicated, as is often the case when modeling real-world problems, direct sampling methods
can become difficult, as they might involve computing high-dimensional integrals.
The Metropolis algorithm is an effective method to sample from many distributions, requiring only that we
be able to evaluate the probability density function up to a constant of proportionality. In particular,
the Metropolis algorithm does not require us to compute difficult high-dimensional integrals, such as those that are found
in the denominator of Bayesian posterior distributions.

The Metropolis algorithm is an MCMC sampling method which generates a sequence of random variables, similar to Gibbs sampling.
These random variables form a Markov Chain whose invariant distribution is equal to the distribution from which we wish
to sample. Suppose that $h : \mathbb{R}^n \rightarrow \mathbb{R}$ is the probability density function of distribution,
and suppose that $f(\boldsymbol{\theta}) = c \cdot h(\boldsymbol{\theta})$ for some nonzero constant $c$ (in practice, we assume that $f$ is an easy
function to evaluate, while $h$ is difficult). Let $Q : \mathbb{R}^n \times \mathbb{R}^n \rightarrow \mathbb{R}$ be
a symmetric \emph{proposal function}
(so that $Q(\cdot, \y)$ is a probability density function for all $\y \in \mathbb{R}^n$,
 and $Q(\x,\y) = Q(\y,\x)$ for all $\x,\y \in \mathbb{R}^n$) and let
 $A : \mathbb{R}^n \times \mathbb{R}^n \rightarrow \mathbb{R}$ be an \emph{acceptance function} defined by
\[
A(\x,\y) = \min\left(1, \frac{f(\x)}{f(\y)}\right).
\]
We can combine these functions in such a way so as to sample from the aforementioned Markov Chain by following Algorithm \ref{alg:metropolis}.
\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{Metropolis Algorithm}{}
    \State \textrm{Choose initial point } $\y_0$.
    \For{$t=1,2,\ldots$}
        \State \textrm{Draw } $\x \sim Q(\cdot, \y_{t-1})$
        \State \textrm{Draw } $a \sim \text{unif}(0,1)$
        \If{$a \leq A(\x,\y_{t-1})$}
            \State $\y_t = \x$
        \Else
            \State $\y_t = \y_{t-1}$
        \EndIf
    \EndFor
    \State \textrm{Return } $\y_1,\y_2,\y_3,\ldots$
\EndProcedure
\end{algorithmic}
\caption{Metropolis Algorithm}
\label{alg:metropolis}
\end{algorithm}
The Metropolis algorithm can be interpreted as follows:
given our current state $\y$, we propose a new state according to the distribution $Q(\cdot, \y)$. We then accept or reject it according to $A$.  We continue by repeating the process. So long as $Q$ defines an irreducible, aperiodic, and non-null recurrent Markov chain, we will have a Markov chain whose unique invariant distribution will have density $h$. Furthermore, given any initial state, the chain will converge to this invariant distribution.
Note that for numerical reasons, it is often wise to make calculations of the acceptance functions in log space:
\[
\log A(\x,\y) = \min(0, \log f(\x) - \log f(\y)).
\]

Let's apply the Metropolis algorithm to a simple example of Bayesian analysis.
Consider the problem of computing the posterior distribution over the mean $\mu$ and variance $\sigma^2$ of a normal distribution for which we have $n$ data points $y_1,\ldots,y_n$.
For concreteness, we use the data in \texttt{examscores.csv} and we assume
the prior distributions
\begin{align*}
\mu &\sim \mathcal{N}(m=80,\ s^2=16)\\
\sigma^2 &\sim IG(\alpha=3,\beta=50).
\end{align*}
In this situation, we wish to sample from the posterior distribution
\[
p(\mu,\sigma^2 \,|\,y_1,\ldots,y_N) = \frac{p(\mu)p(\sigma^2)\prod_{i=1}^n \mathcal{N}(y_i \, | \, \mu, \sigma^2)}
{\int_{-\infty}^\infty\int_{0}^\infty p(\mu)p(\sigma^2)\prod_{i=1}^n \mathcal{N}(y_i \, | \, \mu, \sigma^2)\,d\sigma^2d\mu}.
\]
However, we can conveniently calculate only the numerator of this expression.
Since the denominator is simply a constant with respect to $\mu$ and $\sigma^2$, the numerator can serve as the function $f$ in the Metropolis algorithm, and the denominator can serve as the constant $c$.

We choose our proposal function to be based on a bivariate Normal distribution:
\[
Q(x,y) = \mathcal{N}(x\, | \, y, sI),
\]
where $I$ is the $2\times 2$ identity matrix and $s$ is some positive scalar.
\begin{lstlisting}
>>> def proposal(y, s):
...     """The proposal function Q(x,y) = N(x|y,sI)."""
...     return stats.multivariate_normal.rvs(mean=y, cov=s*np.eye(len(y)))
...
>>> def propLogDensity(x):
...     """Calculate the log of the proportional density."""
...     logprob = muprior.logpdf(x[0]) + sig2prior.logpdf(x[1])
...     logprob += stats.norm.logpdf(scores, loc=x[0], scale=sqrt(x[1])).<<sum>>()
...     return logprob    # ^this is where the scores are used.
...
>>> def acceptance(x, y):
...     return min(0, propLogDensity(x) - propLogDensity(y))
\end{lstlisting}

We are now ready to code up the Metropolis algorithm using these functions.
We will keep track of the samples generated by the algorithm, along with the proportional log densities of the samples and the proportion of proposed samples that were accepted.

We can evaluate the quality of our results by plotting the log probabilities, the $\mu$ samples, the $\sigma^2$ samples, and kernel density estimators for the marginal posterior distributions of $\mu$ and $\sigma^2$. The kernel density estimator is the posterior distribution for a parameter. It measures the frequency of each draw. In this example, the kernel density estimator for $\mu$ should be approximately normal, and the kernel density estimator for $\sigma^2$ should be approximately an inverse gamma.
\begin{figure}[H]
    \centering
    \includegraphics[width=.7\textwidth]{figures/logprobs.pdf}
    \caption{Log densities of the first 500 Metropolis samples.}
    \label{fig:logprobs}
\end{figure}

\begin{figure}[H]
    \begin{subfigure}[b]{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/mu_traces.pdf}
    \end{subfigure}
    \begin{subfigure}[b]{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/mu_kernel.pdf}
    \end{subfigure}
    \begin{subfigure}[b]{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/sig_traces.pdf}
    \end{subfigure}
    \begin{subfigure}[b]{.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/sig_kernel.pdf}
    \end{subfigure}
\caption{Metropolis samples and KDEs for the marginal posterior distribution of $\mu$ (top row) and $\sigma^2$ (bottom row).}
\label{fig:metropolis_results}
\end{figure}

\begin{comment}
We will use the Metropolis algorithm to obtain samples from a multivariate normal distribution to demonstrate this process.
Suppose also that we desire to obtain samples from a multivariate normal distribution with arbitrary covariance matrix $\Sigma$, and that this is difficult (obviously we can do this directly in Python, but this is merely a tutorial to see how the Metropolis algorithm works). Suppose further that we are able to easily compute the ratio of the density of this distribution at two points $\mathbf{x}$ and $\mathbf{y}$ of length $K$, i.e.
\begin{align*}
\frac{N(\mathbf{x} \; ; \; \mu, \Sigma)}{N(\mathbf{y} \; ; \; \mu, \Sigma)} & = \frac{\frac{1}{(2\pi)^{K/2}|\Sigma|^{1/2}} e^{-\frac{1}{2}(\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}}{\frac{1}{(2\pi)^{K/2}|\Sigma|^{1/2}} e^{-\frac{1}{2}(\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}} \\
& = \frac{e^{-\frac{1}{2}(\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}}{e^{-\frac{1}{2}(\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu)}} \\
& = e^{-\frac{1}{2}\left((\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu) - (\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{y} - \mu)\right)}
\end{align*}

\begin{problem}
\label{problem1}
Write an acceptance function that computes
\begin{equation*}
p = \min \{1, e^{-\frac{1}{2}\left((\mathbf{x} - \mu)^{T} \Sigma^{-1} (\mathbf{x} - \mu) - (\mathbf{y} - \mu)^{T} \Sigma^{-1} (\mathbf{y} - \mu)\right)}\}
\end{equation*}
given $\mathbf{x}, \mathbf{y}, \mu,$ and $\Sigma$, and then draws from a Bernoulli distribution with parameter $p$. It should return a $1$ if it accepts the new state, and a $0$ if it rejects it.
\end{problem}

Specifically, we will try to sample from the distribution centered at the origin, with covariance matrix
\begin{equation*}
\Sigma = \left[ \begin{array}{cc} 12 & 4 \\ 4 & 16 \end{array} \right]
\end{equation*}

\begin{lstlisting}
>>> mu = np.zeros(2)
>>> sigma = np.array([[12., 10.], [10., 16.]])
\end{lstlisting}

We will let $Q(\mathbf{x} | \mathbf{y}) = N(\mathbf{x} \; ; \; \mathbf{y}, I)$ be our proposal distribution, given that we are currently in state $\mathbf{y}$, i.e. we propose a new state by drawing from the multivariate normal distribution centered at $\mathbf{y}$ with identity covariance. We then accept according to our acceptance probability, computed in Problem \ref{problem1}.

\begin{problem}
Write a function that accepts a current state, the mean and covariance from the distribution we desire to sample from, and returns the next state. We should propose according to $Q$ described above, and accept according to the function in Problem \ref{problem1}.
\end{problem}

We now have a way to sample a new state from an old state.
As we've stated before, this method creates a Markov chain that \emph{converges} to the desired distribution; at the beginning, however, if our initial guess is highly unlikely for the desired distribution, it may take a while before we get there.
We would like to measure our progress.

\begin{problem}
Write a function that computes the log of the multivariate normal density of a point $\mathbf{x}$ given a mean $\mu$ and covariance matrix $\Sigma$. Be intelligent about how you implement this, that is, do not simply compute the multivariate normal density and then take the log of it, as this may lead to numerical issues. The whole purpose of looking at the multivariate log is to make this more stable.
\end{problem}

We will finally put everything together.

\begin{problem}
Write a function that accepts an initial point $\mathbf{x}$, a mean $\mu$ and covariance $\Sigma$ for the desired sampling distribution, and which performs the Metropolis algorithm for a number of iterations, $n\_samples$. Save each sample $\mathbf{x}$ as produced by the algorithm. Also compute the log of the multivariate normal density of each point, and return both the samples and the logprobs.
\end{problem}

We would like to see how long it takes for our algorithm to converge to the right distribution. We can do this by plotting the log-probs returned by our function. Here we use an initial state $\mathbf{x} = \left[\begin{array}{cc} 100 & 100 \end{array}\right]$.

\begin{figure}[h]
\centering
\includegraphics[width=\textwidth]{figures/logprobs.pdf}
\caption{Log probabilities of our samples.}
\end{figure}

From this we can see that after between $300$ and $500$ iterations, we had converged to the correct distribution. We can visualize the path of our sampler by plotting the samples themselves:
\begin{figure}[h]
\centering
\includegraphics[width=\textwidth]{figures/samples.pdf}
\caption{Samples from the Metropolis algorithm.}
\end{figure}

\begin{problem}
Using $\mu$ and $\Sigma$ as defined previously and using an initial state $\mathbf{x} = \left[ \begin{array}{cc} 1000 & -1000 \end{array} \right]$ run your Metropolis sampler for $10000$ iterations. Plot the log probs as well as the samples. How long did it take to converge?
\end{problem}
\end{comment}

\begin{problem}
Write a function that uses the Metropolis Hastings algorithm to draw from the posterior distribution over the mean $\mu$ and variance $\sigma^2$. Use the given functions and algorithm \ref{alg:metropolis} to complete the problem.

	Your function should return an array of draws, an array of the log probabilities, and an acceptance rate. Use the following code to check your work. Using the seaborn.kdeplot function, plot the first 500 log probabilities, the $\mu$ samples and posterior distribution, and the $\sigma^2$ samples and posterior distribution. The results should be \textit{similar} to Figures \ref{fig:logprobs} and \ref{fig:metropolis_results}.

When comparing \texttt{a} to the acceptance, remember to use \texttt{log(a)} as we are in log space.

\begin{lstlisting}
# Load in the data and initialize hyperparameters.
>>> scores = np.load("examscores.npy")

# Prior sigma^2 ~ IG(alpha, beta)
>>> alpha = 3
>>> beta = 50

#Prior mu ~ N(m, s)
>>> m = 80
>>> s = 4

# Initialize the prior distributions.
>>> muprior = stats.norm(loc=m, scale=sqrt(s**2))
>>> sig2prior = stats.invgamma(alpha, scale=beta)
\end{lstlisting}

\textit{Hint}: The seaborn package is very useful in plotting kernel densities, with the distplot method. See the \href{https://seaborn.pydata.org/generated/seaborn.distplot.html}{documentation.}
\end{problem}

\section*{The Ising Model}
In statistical mechanics, the Ising model describes how atoms interact in ferromagnetic material. Assume we have some lattice $\Lambda$ of sites. We say $i \sim j$ if $i$ and $j$ are adjacent sites. Each site $i$ in our lattice is assigned an associated \emph{spin} $\sigma_{i} \in \{\pm 1\}$. A \emph{state} in our Ising model is a particular spin configuration $\sigma = (\sigma_{k})_{k \in \Lambda}$. If $L = |\Lambda|$, then there are $2^{L}$ possible states in our model. If $L$ is large, the state space becomes huge, which is why MCMC sampling methods (in particular the Metropolis algorithm) are so useful in calculating model estimations.

With any spin configuration $\sigma$, there is an associated energy
\[
H(\sigma) = -J \sum_{i \sim j} \sigma_{i} \sigma_{j}
\]
 where $J > 0$ for ferromagnetic materials, and $J < 0$ for antiferromagnetic materials. Throughout this lab, we will assume $J = 1$, leaving the energy equation to be $H(\sigma) = -\sum_{i \sim j} \sigma_{i}\sigma_{j}$ where the interaction from each pair is added only once.

We will consider a lattice that is a $100 \times 100$ square grid.
The adjacent sites for a given site are those directly above, below, to the left, and to the right of the site, so to speak.
For sites on the edge of the grid, we assume it wraps around.
In other words, a site at the farthest left side of the grid is adjacent to the corresponding site on the farthest right side.
Thus, a single spin configuration can be represented as a $100 \times 100$ array, with entries of $\pm 1$.

The following code will construct a random spin configuration of size n:

\begin{lstlisting}
def random_lattice(n):
    """Constructs a random spin configuration for an nxn lattice."""
    random_spin = np.zeros((n,n))
    for k in range(n):
        random_spin[k,:] = 2*np.random.binomial(1,.5, n) -1
    return random_spin
\end{lstlisting}

\begin{figure}[H]
\centering
\includegraphics[width=.5\textwidth]{figures/initial_config.pdf}
\caption{Spin configuration from random initialization.}
\label{fig:random_spin}
\end{figure}

\begin{problem} % Calculate the energy of the spin configuration.
\label{problem2}
Write a function that accepts a spin configuration $\sigma$ for a lattice as a NumPy array.
Compute the energy $H(\sigma)$ of the spin configuration.
Be careful to not double count site pair interactions!
\\(Hint: \li{np.roll()} may be helpful.)
\end{problem}

Different spin configurations occur with different probabilities, depending on the energy of the spin configuration and $\beta > 0$, a quantity inversely proportional to the temperature.
More specifically, for a given $\beta$, we have
\begin{equation*}
\mathbb{P}_{\beta}(\sigma) = \frac{e^{-\beta H(\sigma)}}{Z_{\beta}}
\end{equation*}
where $Z_{\beta} = \sum_{\sigma} e^{-\beta H(\sigma)}$.
Because there are $2^{100 \cdot 100} = 2^{10000}$ possible spin configurations for our particular lattice, computing this sum is infeasible.
However, the numerator is quite simple, provided we can efficiently compute the energy $H(\sigma)$ of a spin configuration.
Thus the ratio of the probability densities of two spin configurations is simple:
\begin{equation*}
\frac{\mathbb{P}_{\beta}(\sigma^{*})}{\mathbb{P}_{\beta}(\sigma)}
= \frac{e^{-\beta H(\sigma^{*})}}{e^{-\beta H(\sigma)}}
= e^{\beta (H(\sigma) - H(\sigma^{*}))}
\end{equation*}

The simplicity of this ratio should lead us to think that a Metropolis algorithm might be an appropriate way by which to sample from the spin configuration probability distribution, in which case the acceptance probability would be
\begin{equation}
A(\sigma^{*}, \sigma) = \begin{cases} 1 & \mbox{if } H(\sigma^{*}) < H(\sigma) \\ e^{\beta (H(\sigma) - H(\sigma^{*}))} & \mbox{ otherwise.} \end{cases}
\label{eq:ising-acceptance}
\end{equation}

By choosing our transition matrix $Q$ cleverly, we can also make it easy to compute the energy for any proposed spin configuration.
We restrict our possible proposals to only those spin configurations in which we have flipped the spin at exactly one lattice site, i.e. we choose a lattice site $i$ and flip its spin.
Thus, there are only $L$ possible proposal spin configurations $\sigma^{*}$ given $\sigma$, each being proposed with probability $\frac{1}{L}$, and such that $\sigma_{j}^{*} = \sigma_{j}$ for all $j \neq i$, and $\sigma_{i}^{*} = - \sigma_{i}$.
Note that we would never actually write out this matrix (it would be $2^{10000} \times 2^{10000}$).
Computing the proposed site's energy is simple: if the spin flip site is $i$, then we have
\begin{equation}
H(\sigma^{*}) = H(\sigma) + 2\sum_{j: j \sim i} \sigma_{i}\sigma_{j}.
\label{eq:ising-new-spin-energy}
\end{equation}

\begin{problem} % Choose somewhere to flip a bit.
Write a function that accepts an integer $n$ and chooses a pair of indices $(i,j)$ where $0 \le i,j \le n-1$.
Each possible pair should have an equal probability $\frac{1}{n^2}$ of being chosen.
\label{prob:ising-flip-site}
\end{problem}

\begin{problem} % Compute the energy of the proposed configuration.
Write a function that accepts a spin configuration $\sigma$, its energy $H(\sigma)$, and integer indices $i$ and $j$.
Use \eqref{eq:ising-new-spin-energy} to compute the energy of the new spin configuration $\sigma^*$, which is $\sigma$ but with the spin flipped at the $(i,j)$th entry of the corresponding lattice.
Do not explicitly construct the new lattice for $\sigma^*$.
\label{prob:ising-new-energy}
\end{problem}

\begin{problem} % Accept / reject the new configuration.
Write a function that accepts a float $\beta$ and spin configuration energies $H(\sigma)$ and $H(\sigma^*)$.
Using \eqref{eq:ising-acceptance}, calculate whether or not the new spin configuration $\sigma^*$ should be accepted (return \li{True} or \li{False}).
Consider doing the calculations in log space.
(Hint: np.random.binomial() might be useful)
\label{prob:ising-acceptance}
\end{problem}

To track the convergence of the Markov chain, we would like to look at the probabilities of each sample at each time. However, this would require us to compute the denominator $Z_{\beta}$, which is generally the reason we have to use a Metropolis algorithm to begin with.
We can get away with examining only $-\beta H(\sigma)$.
We should see this value increase as the algorithm proceeds, and it should converge once we are sampling from the correct distribution.
Note that we don't expect these values to converge to a specific value, but rather to a restricted range of values.

\begin{problem}
Write a function that accepts a float $\beta>0$ and integers $n$, \li{n_samples}, and \li{burn_in}.
Initialize an $n\times n$ lattice for a spin configuration $\sigma$ using Problem \ref{problem2}.
Use the Metropolis algorithm to (potentially) update the lattice \li{burn_in} times.
\begin{enumerate}
    \item Use Problem \ref{prob:ising-flip-site} to choose a site for possibly flipping the spin, thus defining a potential new configuration $\sigma^*$.
    \item Use Problem \ref{prob:ising-new-energy} to calculate the energy $H(\sigma^*)$ of the proposed configuration.
    \item Use Problem \ref{prob:ising-acceptance} to accept or reject the proposed configuration.
    If it is accepted, set $\sigma = \sigma^*$ by flipping the spin at the indicated site.
    \item Track $-\beta H(\sigma)$ at each iteration (independent of acceptance).
\end{enumerate}
After the burn-in period, continue the iteration \li{n_samples} times, also recording every $100$th sample (to prevent memory failure).
The acceptance rate is counted after the burn-in period.
Return the samples, the sequence of weighted energies $-\beta H(\sigma)$, and the acceptance rate.

Test your sampler on a $100 \times 100$ grid with $200000$ total iterations, with \li{n_samples} large enough so that you will keep $50$ samples, for $\beta = 0.2, 0.4, 1$.
Plot the proportional log probabilities, as well as a late sample from each test.
How does the ferromagnetic material behave differently with differing temperatures?
Recall that $\beta$ is an inverse function of temperature.
You should see more structure with lower temperature, as illustrated in Figure \ref{fig:ising-results}.

To show the spin configuration, use \li{plt.imshow(L,cmap='gray')}.
\end{problem}

\begin{figure}
\begin{subfigure}{.49\textwidth}
    \centering
    \includegraphics[width=\textwidth]{figures/beta0_2_logprobs.pdf}
    \caption{Proportional log probs when $\beta = 0.2$.}
\end{subfigure}
%
\begin{subfigure}{.49\textwidth}
    \centering
    \includegraphics[width=.62\textwidth]{figures/beta0_2.pdf}
    \caption{Spin configuration sample when $\beta = 0.2$.}
\end{subfigure}
\\
\begin{subfigure}{.49\textwidth}
    \centering
    \includegraphics[width=\textwidth]{figures/beta0_4_logprobs.pdf}
    \caption{Proportional log probs when $\beta = 0.4$.}
\end{subfigure}
%
\begin{subfigure}{.49\textwidth}
    \centering
    \includegraphics[width=.62\textwidth]{figures/beta0_4.pdf}
        \caption{Spin configuration sample when $\beta = 0.4$.}
\end{subfigure}
\\
\begin{subfigure}{.49\textwidth}
    \centering
    \includegraphics[width=\textwidth]{figures/beta1_logprobs.pdf}
    \caption{Proportional log probs when $\beta = 1$.}
\end{subfigure}
%
\begin{subfigure}{.49\textwidth}
    \centering
    \includegraphics[width=.62\textwidth]{figures/beta1.pdf}
    \caption{Spin configuration sample when $\beta = 1$.}
\end{subfigure}
\caption{}
\label{fig:ising-results}
\end{figure}


% Appendix A

\chapter{Quaternions}
\label{ch:app:quaternions}
Quaternions are a number systems that extends complex numbers introduced by William Rowan Hamilton in 1843. They are commonly represented in the form
\[ q = w + x\mathbf{i} + y\mathbf{j} + z\mathbf{k} \]
where $ \mathbf{i},\ \mathbf{j}\ and\ \mathbf{k} $ are the fundamental quaternion units.
Multiplications between quaternions are non-commutative, with
\begin{table}[]
	\centering
	\caption{My caption}
	\label{my-label}
	\begin{tabular}{|l|l|}
		\hline
		\[ \mathbf{j} x \mathbf{k} = \mathbf{i} \] & \[ \mathbf{j} x \mathbf{k} = \mathbf{i} \] \\ \hline
	\end{tabular}
\end{table}

\[ \mathbf{i} x \mathbf{1} = \mathbf{i} \]
\[ \mathbf{i} x \mathbf{i} = \mathbf{-1} \]
\[ \mathbf{i} x \mathbf{j} = \mathbf{k} \]
\[ \mathbf{i} x \mathbf{k} = \mathbf{-j} \]

\[ \mathbf{k} x \mathbf{1} = \mathbf{i} \]
\[ \mathbf{k} x \mathbf{i} = \mathbf{i} \]
\[ \mathbf{k} x \mathbf{j} = \mathbf{i} \]
\[ \mathbf{k} x \mathbf{k} = \mathbf{i} \]

\section{Rotations}
Normal quaternions are


\documentclass[12pt]{article}
\usepackage[usenames]{color} %used for font color
\usepackage{amssymb} %maths
\usepackage{amsmath} %maths
\usepackage[utf8]{inputenc} %useful to type directly diacritic characters
\usepackage{graphicx}
\usepackage [english]{babel}
\usepackage [autostyle, english = american]{csquotes}
\MakeOuterQuote{"}
\graphicspath{ {./} }
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\prob}{\mathbb{P}}

\author{Tianshuang (Ethan) Qiu}
\begin{document}
\title{Math 74, Week 3}
\maketitle

\section{Wed Lec, 1a}
Prove that $\binom{n}{k} + \binom{n}{k+1} = \binom{n+1}{k+1}$
\subsection{algebraic}
We first expand the expression:
$$LHS = \frac{n!}{(n-k)!k!} + \frac{n!}{(n-k-1)!(k+1)!}$$
Then we simplify:
$$LHS = \frac{n!}{(n-k)(n-k-1)!k!} + \frac{n!}{(k+1)(n-k-1)!k!}$$
$$LHS = \frac{(n!(k+1))+(n!(n-k))}{(n-k)(k+1)(n-k-1)!k!} $$
$$LHS = \frac{n!(n+1)}{(k+1)!(n-k)!} $$
$$LHS = \frac{(n+1)!}{(k+1)!(n-k)!}$$
Now we expand the right side:
$$RHS = \frac{(n+1)!}{(k+1)!(n-k)!}$$
We see thatt RHS = LHS.
\newline
Q.E.D.

\subsection{Combinatorial}
The right hand side calculates the number of bitstrings of length $n+1$ with $k+1$ 0's. Since each bit can either end in $0$ or $1$, we can split them into different cases.
\newline
If the the last is $0$, there are $k$ 0's left in the substring of length $n$. So to calculate this, we use $\binom{n}{k}$.
\newline
If the last bit is $1$, there are still $k+1$ 0's left in the substring before it. Using the formula, we get $\binom {n}{k+1}$
\newline
Adding them together equals the right hand side. They are evaluating the same thing.
\newline
Q.E.D.

\section{Wed Lec, 4b}
Let people be the pigeons and each letter is a different hole.
\newline
Number of pigeons = number of names = 33
\newline
Number of holes = number of letters = 30
\newline
Since the amount of pigeons is greater than the amount of holes, some hole must have at least 2 pigeons by PHP. Therefore some ending letter must be shared by at least 2 names.

\section{Wed Lec, 5}

\subsection{a}
We can form a segment from any two points. So the total amount of segments is $\binom{50}{2} = \frac{50!}{2!48!}=1225$

\subsection{b}
Since no three points are co-linear, we can choose any 3 points to form a triangle. So the total amount is $\binom{50}{3} = \frac{50!}{3!47!} = 19600$

\subsection{c}
Similar to part (b), we choose any 4 points to form a quadrilateral. The total amount is $\binom {50} {4} = \frac{50!}{46!4!} = 230300$.

\newpage

\section{Wed Dis, 2}
Since the three books needs to be next to each other in the specified order, we can think of it as if the three are "glued together" into one book. So essentially we are looking for the amount of ways to arrange 5 books.
\newline
There are 5 ways to choose the first one, 4 for the second, ..., for a total of $5! = 120$ ways to arrange it.

\section{Wed Dis, 4}
We are looking for the amount of ways to rearrange "GAUSS". Since there are 5 letters, we have $5!=120$. Then we remove the ways that we over count the repeat "S". $\frac{5!}{2!} = 60$
\newline
Similarly, for "RAMANUJAN", we take all possible permutations and divide by repeating letters "A" and "N"$: \frac{9!}{3!2!} = 30240$

\newpage

\section{Fri Lec, 1b}
\subsection{Combinatorial}
RHS is the number of ways to choose a team and a captain out of $n$ people: choose the captain first, then choose the rest of the team (each member can either be chosen or not): $n \times 2^{n-1}$
\newline
LHS chooses the team first, then chooses the captain. There are $\binom{n}{k}$ ways to choose $k$ people from $n$ to form a team. Then, from that team there are $k$ ways to pick a captain. Finally we sum up all possible number of team size (from 0 to n): $\sum_{k=0}^{n}k\binom{n}{k}$
\newline
RHS = LHS, Q.E.D.

\section{Fri Lec, 4}
Let the $n$ students be our pigeons, and the friends they have be $f$, so $f$ would be the holes. $0 \leq f \leq n-1$
\newline
Case 1: if someone has $0$ friends, then there cannot be anyone with $n-1$ friends. If there is a person who is not friends with anyone, and since friendships are mutual, then there cannot be anyone who is friends with everyone. In this case $0 \leq f < n-1$. There are $n-1$ holes, $n > n-1$. By PHP there must be 2 people with the same amount of friends.
\newline
Case 2: otherwise, $0 < f \leq n-1$. Once again there are $n-1$ holes, by PHP there must be 2 people with the same amount of friends.
\newline
Q.E.D.

\end{document}


\chapter{GPU programs for Fourier pseudospectral simulations of the Navier-Stokes, Cubic Nonlinear Schr\"{o}dinger and sine Gordon equations}

This section includes the programs taken from a conference paper by Cloutier, Muite and Rigge~\cite{CloMuiRig12}. The main purpose is to give example programs which show how to use graphics processing units (GPUs) to solve partial differential equations using Fourier methods. For further background on GPUs and programming models for GPUs see Cloutier, Muite and Rigge~\cite{CloMuiRig12}. It should be noted that the algorithms used for the sine Gordon equation are very similar to those for the Klein Gordon equation discussed elsewhere in this tutorial. For consistency with the rest of the tutorial, programs using CUDA Fortran and OpenACC extensions to Fortran are included. GPUs enable acceleration of Fourier pseudospectral codes by factors of 10 compared to OpenMP parallelizations on a single 8 core node.

\section{2D Navier Stokes Equations}

These programs use the Crank-Nicolson method.

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dNsCUF,caption={A CUDA Fortran program to solve the 2D Navier-Stokes equations.}]{./SingleGPU/Programs/NS/GPUcuf/navierstokes.cuf}

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dNsACC,caption={An OpenACC  Fortran program to solve the 2D Navier-Stokes equations.}]{./SingleGPU/Programs/NS/GPUacc/navierstokes.f90}

\section{2D Cubic Nonlinear Schr\"{o}dinger Equations}

These programs use splitting.

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dNlsCUF,caption={A CUDA Fortran program to solve the 2D Nonlinear Schr\"{o}dinger equation.}]{./SingleGPU/Programs/NLS/GPUcuf/cubicNLS.cuf}

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dNlsACC,caption={An OpenACC  Fortran program to solve the  2D Nonlinear Schr\"{o}dinger equation.}]{./SingleGPU/Programs/NLS/GPUacc/cubicNLS.f90}

\section{2D sine-Gordon Equations}

These programs use a semi-explicit method that is similar to that used for the Klein-Gordon equation. Only the main program is included here, and the auxiliary subroutines can be downloaded from Cloutier, Muite and Rigge~\cite{CloMuiRig12}

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dSgCUF,caption={A CUDA Fortran program to solve the 2D sine-Gordon equation.}]{./SingleGPU/Programs/sineGordon/GPUcuf/sgsemiimp2d.cuf}

\lstinputlisting[style=fortran_style,language=Fortran,label=lst:For2dSgACC,caption={An OpenACC  Fortran program to solve the  2D sine-Gordon equation.}]{./SingleGPU/Programs/sineGordon/GPUacc/sgsemiimp2d.f90}


\section{Sylvester's Law, Sesquilinear Forms}
We start by looking at some immediate corollaries of \ref{bilinear_diag}.
\begin{corollary}
    For a finite dimensional complex vector space $V$ and a symmetric bilinear form $\phi$ on $V$, there is a basis $B$ of $V$ such that
    $$[\phi]_B=\begin{pmatrix}
        I_r&0\\
        0&0
    \end{pmatrix}$$
    where $r=r(\phi)$.
\end{corollary}
\begin{proof}
    Square roots always exist as $F=\mathbb C$.
\end{proof}
\begin{corollary}
    Every symmetric matrix in $\mathbb C$ is congruent to a unique matrix of the form
    $$\begin{pmatrix}
        I_r&0\\
        0&0
    \end{pmatrix}$$
\end{corollary}
\begin{proof}
    Immediate.
\end{proof}
\begin{corollary}
    If $F=\mathbb R$, $\dim V=n<\infty$ and $\phi$ a symmetric bilinear form of $V$, then there exists a basis $\{v_1,\ldots,v_n\}$ of $V$ such that
    $$\begin{pmatrix}
        I_p&&\\
        &-I_q&\\
        &&0
    \end{pmatrix}$$
\end{corollary}
\begin{proof}
    Every positive number in $\mathbb R$ has a square root.
\end{proof}
\begin{definition}
    $s(\phi)=p-q$ is called the signature of the real symmetric bilinear form $\phi$.
\end{definition}
To see it is well-defined,
\begin{theorem}[Sylvester's Law of Inertia].
    If a real symmetric bilinear form $\phi$ has
    $$[\phi]_B=\begin{pmatrix}
        I_p&&\\
        &-I_q&\\
        &&0
    \end{pmatrix},[\phi]_{B'}=\begin{pmatrix}
        I_{p'}&&\\
        &-I_{q'}&\\
        &&0
    \end{pmatrix}$$
    then $p=p',q=q'$.
\end{theorem}
\begin{definition}
    Let $\phi$ be a real symmetric bilinear form.
    We say that $\phi$is positive semidefinite if $\phi(u,u)\ge 0$ for any $u\in V$, and is positive definite if $\phi(u,u)>0$ for any $u\in V\setminus\{0\}$.
    Similarly, $\phi$ is positive semidefinite if $\forall u\in V,\phi(u,u)\le 0$ for any $u\in V$, and negative definite if $\forall u\in V\setminus\{0\},\phi(u,u)<0$.
\end{definition}
\begin{example}
    The matrix
    $$\begin{pmatrix}
        I_p&0\\
        0&0
    \end{pmatrix}\in M_n(\mathbb R)$$
    is always positive semidefinite, and is positive definite iff $p=n$.
\end{example}
\begin{proof}
    Indeed $p$ is the largest dimension of subspace of $V$ in which $\phi$ is positive definite.
    Similarly $q$ is the largest dimension of a subspace in which $q$ is negative definite.
    These descriptions are independent of the choice of basis, so we are done
\end{proof}
\begin{definition}
    The kernel of the bilinear form $\phi:V\times V\to F$ is the set $K(\phi)=\{v\in V:\forall u\in V,\phi(u,v)=0\}$.
\end{definition}
\begin{remark}
    1. $\dim K+r(\phi)=0$.\\
    2. For $F=\mathbb R$, we now know from the preceding theorem that there is a subspace $T$ of dimension $n-(p+q)+\min{p,q}$ such that $\phi|_T=0$.
    More over, this can easily be shown to be the largest dimension such that such a subspace $T$ exists.
\end{remark}
Recall that the standard inner product on $\mathbb C^n$, that is
$$\langle x,y\rangle=\sum_{i=1}^nx_i\bar{y}_i$$
is not a bilinear form.
\begin{definition}
    Let $V,W$ be vector spaces over $\mathbb C$.
    A map $\phi:V\times W\to\mathbb C$ is a sesquilinear form if for any $w\in W$, $\phi(\cdot,w)$ is linear and for any $v\in V,\lambda_1,\lambda_2\in \mathbb C,w_1,w_2\in W$,
    $$\phi(v,\lambda_1w_1+\lambda_2w_2)=\bar{\lambda}_1\phi(v,w_1)+\bar\lambda_2\phi(v,w_2)$$
\end{definition}
\begin{definition}
    With notation as above, for bases $B=\{v_1,\ldots,v_m\}$ of $V$ and $C=\{w_1,\ldots,w_n\}$ of $W$, the matrix of $\phi$ is $(\phi]_{B,C})_{ij}=(\phi(v_i,w_j))$.
\end{definition}
\begin{lemma}
    $\phi(v,w)=[u]_B^\top [\phi]_{B,C}\overline{[v]}_C$.
\end{lemma}
\begin{proof}
    Expand.
\end{proof}
\begin{lemma}
    If $B,B'$ are bases of $V$ and $C,C'$ of $W$ and $P=[\operatorname{id}_V]_{B',B},Q=[\operatorname{id}_W]_{C',C}$, then $[\phi]_{B',C'}=P^\top[\phi]_{B,C}\bar{Q}$
\end{lemma}
\begin{proof}
    Analogous to the bilinear case.
\end{proof}