\documentclass[11pt]{article}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Packages
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\usepackage{oke-header-math}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Mathematics
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\vbf}{\boldsymbol{v}}
\newcommand{\hbf}{\boldsymbol{h}}
\newcommand{\abf}{\boldsymbol{a}}
\newcommand{\bbf}{\boldsymbol{b}}
\newcommand{\Vbf}{\boldsymbol{V}}
\newcommand{\Hbf}{\boldsymbol{H}}
\newcommand{\qt}{\tilde{q}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Title
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\title{RBMs}
\author{Oliver K. Ernst}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Begin document
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Theory}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The objective function is the KL divergence:
%---------------
\begin{equation}
\dkl (p || \pt) = \int dx \; p(x) \ln \frac{p(x)}{\pt(x, \nu)}
\end{equation}
%---------------
where $p(x)$ is the true data distribution and $\pt(x,\nu)$ is the model distribution:
%---------------
\begin{equation}
\begin{split}
\pt(x,\nu) &= \frac{1}{Z(\nu)} \exp [ - E(x, \nu) ] \\
Z(\nu) &= \int dy \; \exp [ - E(y, \nu) ]
\end{split}
\end{equation}
%---------------
for some energy function $E(x,\nu)$ with interactions $\nu$. The gradients are
%---------------
\begin{equation}
\begin{split}
\frac{\partial \dkl}{\partial \nu}
&=
- \int dx \; p(x) \left ( \frac{\pt(x,\nu)}{p(x)} \right ) p(x) \left ( \pt(x,\nu) \right )^{-2} \frac{\partial \pt(x,\nu)}{\partial \nu}
=
- \int dx \; \frac{p(x)}{\pt(x,\nu)} \frac{\partial \pt(x,\nu)}{\partial \nu} \\
%%%%
\frac{\partial \pt(x,\nu)}{\partial \nu}
&=
- \frac{\partial E(x,\nu)}{\partial \nu} \pt(x, \nu) - \frac{1}{Z(\nu)^2} \exp[-E(x,\nu)] \frac{\partial Z(\nu)}{\partial \nu} \\
&=
- \frac{\partial E(x,\nu)}{\partial \nu} \pt(x, \nu) + \frac{1}{Z(\nu)} \pt(x,\nu) \int dy \; \frac{\partial E(y,\nu)}{\partial \nu} \exp[-E(y,\nu)] \\
&=
- \frac{\partial E(x,\nu)}{\partial \nu} \pt(x, \nu) + \pt(x,\nu) \int dy \; \frac{\partial E(y,\nu)}{\partial \nu} \pt(y,\nu) \\
&=
- \frac{\partial E(x,\nu)}{\partial \nu} \pt(x, \nu) + \pt(x,\nu) \left \langle \frac{\partial E}{\partial \nu} \right \rangle_{\pt} \\
%%%%
\frac{\partial \dkl}{\partial \nu}
&=
\int dx \; \frac{p(x)}{\pt(x,\nu)} \frac{\partial E(x,\nu)}{\partial \nu} \pt(x, \nu)
- \int dx \; \frac{p(x)}{\pt(x,\nu)} \pt(x,\nu) \left \langle \frac{\partial E}{\partial \nu} \right \rangle_{\pt} \\
&=
\int dx \; p(x) \frac{\partial E(x,\nu)}{\partial \nu}
- \int dx \; p(x) \left \langle \frac{\partial E}{\partial \nu} \right \rangle_{\pt} \\
&=
\left \langle \frac{\partial E}{\partial \nu} \right \rangle_p
- \left \langle \frac{\partial E}{\partial \nu} \right \rangle_{\pt}
\end{split}
\end{equation}
%---------------
where the second integral is unity because $p$ is normalized by definition.
%---------------
\begin{equation}
\boxed{
\frac{\partial \dkl}{\partial \nu}
=
\left \langle \frac{\partial E}{\partial \nu} \right \rangle_p
- \left \langle \frac{\partial E}{\partial \nu} \right \rangle_{\pt}
}
\end{equation}
%---------------
The first term is often called the \textit{awake phase} moment, or moment under the \textit{data distribution} $p$; the second term is often called the \textit{asleep phase} moment, or moment under the \textit{model distribution} $\pt$.

For the discrete case on a lattice with visible units $\vbf$ and hidden units $\hbf$:
%---------------
\begin{equation}
\dkl (p || \pt) = \sum_{\vbf} \sum_{\hbf} p(\vbf, \hbf) \ln \frac{p(\vbf, \hbf)}{\pt(\vbf,\hbf)}
\end{equation}
%---------------
and a common energy function is:
%---------------
\begin{equation}
\begin{split}
E(\vbf, \hbf) = - \abf^\intercal \vbf - \bbf^\intercal \hbf - \vbf^\intercal W \hbf
\end{split}
\end{equation}
%---------------
for biases $\abf,\bbf$ and weight matrix $W$, which play the role of $\nu$. The gradients are:
%---------------
\begin{equation}
\begin{split}
\frac{\partial E}{\partial \abf} &= - \vbf \\
\frac{\partial E}{\partial \bbf} &= - \hbf \\
\frac{\partial E}{\partial W} &= - \vbf \otimes \hbf
\end{split}
\end{equation}
%---------------
leading to the gradients:
%---------------
\begin{equation}
\boxed{
\begin{split}
\frac{\partial \dkl}{\partial \abf}
&=
\left \langle \vbf \right \rangle_{\pt}
- \left \langle \vbf \right \rangle_p \\
%%%
\frac{\partial \dkl}{\partial \bbf}
&=
\left \langle \hbf \right \rangle_{\pt}
- \left \langle \hbf \right \rangle_p \\
%%%
\frac{\partial \dkl}{\partial W}
&=
\left \langle \vbf \otimes \hbf \right \rangle_{\pt}
- \left \langle \vbf \otimes \hbf \right \rangle_p
\end{split}
}
\end{equation}
%---------------

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Implementation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%

\subsection{Gradients}

%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%

In practice, concerning the moments $\langle \dots \rangle$:
\begin{itemize}
\item In the continuous case, we usually cannot analytically perform the integral $\int dx$.
\item In the discrete case, we usually cannot enumerate all possible states appearing in the sum $\sum_x$.
\end{itemize}
Therefore, these moments are estimated using batches. In the continuous case, let the batch be $X_{\pt}$ of size $N$, typically small ($N\sim 5 -10$). For some observable $\chi(x)$:
%---------------
\begin{equation}
\langle \chi(x) \rangle_{\pt} = \int dx \; \pt(x, \nu) \chi(x) \sim \frac{1}{N} \sum_{i=1}^N \chi(X_{\pt,i})
\end{equation}
%---------------
and similarly for moments with respect to $p$. In the discrete case, with the batch represented as $\Vbf_{\pt}, \Hbf_{\pt}$:
%---------------
\begin{equation}
\langle \chi(\vbf, \hbf) \rangle_{\pt} = \sum_{\vbf} \sum_{\hbf} \; \pt(\vbf, \hbf) \chi(\vbf, \hbf) \sim \frac{1}{N} \sum_{i=1}^N \chi(\Vbf_{\pt,i}, \Hbf_{\pt,i} )
\end{equation}
%---------------

The gradients are therefore estimated as:
%---------------
\begin{equation}
\boxed{
\begin{split}
\frac{\partial \dkl}{\partial \abf}
&=
\frac{1}{N} \sum_{i=1}^N \Vbf_{\pt, i}
- \frac{1}{N} \sum_{i=1}^N \Vbf_{p, i} \\
%%%
\frac{\partial \dkl}{\partial \bbf}
&=
\frac{1}{N} \sum_{i=1}^N \Hbf_{\pt, i}
- \frac{1}{N} \sum_{i=1}^N \Hbf_{p, i} \\
%%%
\frac{\partial \dkl}{\partial W}
&=
\frac{1}{N} \sum_{i=1}^N \Vbf_{\pt, i} \otimes \Hbf_{\pt, i}
- \frac{1}{N} \sum_{i=1}^N \Vbf_{p, i} \otimes \Hbf_{p, i}
\end{split}
}
\label{eq:grads}
\end{equation}
%---------------

%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%

\subsection{Sampling}

%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%

Where do such batches come from? They should be samples of the distributions $p$ or $\pt$ as appropriate.
\begin{itemize}
\item For sampling the model distribution $\pt$, we have several options:
\begin{itemize}
\item In the continuous case, without knowing further about $\pt(x)$, we can always perform Markov Chain Monte Carlo (MCMC) to sample the distributions (other sampling methods \textit{may} be possible).
\item In the discrete case, we can use \textit{Gibbs sampling}, which works by iteratively sampling $\pt(v_i | \hbf)$ and $\pt(h_i | \vbf)$. These are derived as follows:
%---------------
\begin{equation}
\begin{split}
\pt(h_i | \vbf) &= \frac{\pt(h_i, \vbf)}{\pt(\vbf)} \\
\pt(\vbf) &= \sum_{\hbf} \pt(\vbf,\hbf) \propto \exp [ \abf^\intercal \vbf ] \\
\pt(\hbf, \vbf) &= \prod_i \pt(h_i, \vbf)
\end{split}
\end{equation}
%---------------
where the last line follows because in an RBM, the hidden variables are conditionally independent of all other hidden variables, and similarly for visible-visibles, then:
%---------------
\begin{equation}
\begin{split}
\pt(h_i, \vbf) \propto \exp [ \abf^\intercal \vbf + b_i h_i + \vbf^\intercal \text{col}_i(W) h_i ]
\end{split}
\end{equation}
%---------------
then it follows:
%---------------
\begin{equation}
\boxed{
\pt(h_i | \vbf) \propto \exp [ b_i h_i + \vbf^\intercal \text{col}_i(W) h_i ]
}
\end{equation}
%---------------
and similarly
%---------------
\begin{equation}
\boxed{
\pt(v_i | \hbf) \propto \exp [ a_i v_i + v_i \text{row}_i(W) \hbf ]
}
\end{equation}
%---------------
In \textit{contrastive divergence}, this procedure of iteratively sampling $\pt(v_i | \hbf)$ and $\pt(h_i | \vbf)$ is performed only a few times (or even only once!), starting from an initial data vector $\vbf$. This greatly improves computational efficiency; alternatively, you can run this sampler for a long time to let the \textit{chain converge}.

After sampling, we obtain the desired samples for the batch $\Vbf_{\pt,i}, \Hbf_{\pt,i}$. The sampling can be performed in \textit{parallel} for higher efficiency, evaluating all $N$ items in the batch $\Vbf_{\pt}, \Hbf_{\pt}$.

In \textit{persistent contrastive divergence}, we do not throw out the hidden states $\Vbf_{\pt}, \Hbf_{\pt}$ after one gradient step and restart from new data vectors $\vbf$. Instead, we keep these states, and use them in the next gradient step again as the starting point for sampling $\pt(v_i | \hbf)$ and $\pt(h_i | \vbf)$ for a few more steps.

\item The samples $\Vbf_p$ from the data distribution $p$ are obvious; they are provided as training data. The samples $\Hbf_p$ are obtained by \textit{clamping the visible units to the data vectors $\Vbf_p$} and sampling \textit{only} $\pt(h_i | \vbf)$. Note that this is similarly possible with MCMC for the general continuous case; assuming we partition $x$ into observed and latent variables, we can sample only the latent variables, keeping the observed variables clamped.

\end{itemize}
\end{itemize}

%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%

\subsection{Objective function}

%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%

It would be great if we could code up the objective function $\dkl$ onto a computer, but this is \textbf{not trivial}. Instead, we can make an important restriction:
%---------------
\begin{equation*}
\text{Assume that our \textbf{optimizer only uses first-order gradients.}}
\end{equation*}
%---------------
\textbf{If} we make this restriction, we can consider the following objective function
%---------------
\begin{equation}
\begin{split}
S
=&
\abf^\intercal \left (
\frac{1}{N} \sum_{i=1}^N \Vbf_{\pt, i}
- \frac{1}{N} \sum_{i=1}^N \Vbf_{p, i}
\right ) \\
%%%
& +
\bbf^\intercal \left (
\frac{1}{N} \sum_{i=1}^N \Hbf_{\pt, i}
- \frac{1}{N} \sum_{i=1}^N \Hbf_{p, i}
\right ) \\
%%%
& +
\left (
\frac{1}{N} \sum_{i=1}^N \Vbf_{\pt, i}^\intercal W \Hbf_{\pt, i}
- \frac{1}{N} \sum_{i=1}^N \Vbf_{p, i}^\intercal W \Hbf_{p, i}
\right )
\end{split}
\end{equation}
%---------------
which has the same \textbf{first-order} gradients as~(\ref{eq:grads}).

\textbf{Note that the second-order gradients will obviously be incorrect!}

This trick allows us to easily implement an RBM in TensorFlow.

\end{document}


\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}

\title{MATH 542 Homework 11}
\author{Saket Choudhary\\skchoudh@usc.edu}

\begin{document}
\maketitle

\section*{Problem 4b.1}
\begin{align*}
F &= \frac{n-p}{q}\frac{(Y-c1_n)'(P-P_H)(Y-c1_n)}{(Y-c1_n)'(I_n-P)(Y-c1_n)}\\
\end{align*}

Also, $X = \begin{pmatrix} 1 & x_{1,1} & x_{1,2} & \dots & x_{1,p-1}\\
1 & x_{2,1} & x_{2,2} & \dots & x_{2,p-1}\\
\vdots & \vdots & \vdots \\
1 & x_{n,1} & x_{n,2} & \dots & x_{n,p-1}\\
\end{pmatrix}$

Thus, $1_n \in \mathcal{C}(X)$ and hence $(I-P)1_n = 1_n'(I-P) = (P-P_H)1_n = 1_n'(P-P_H) = 0$
and hence $(Y-c1_n)'(P-P_H)(Y-c1_n) = Y'(P-P_H)Y-c1_n'(P-P_H)Y+cY'(P-P_H)1_n+c^21_n'(P-P_H)1_n = Y'(P-P_H)Y$

Similarly $(Y-c1_n)'(I-P)(Y-c1_n) = Y'(I-P)Y$
and hence $F$ statistic is the same as $F = \frac{n-p}{q}\frac{Y'(P-P_H)Y}{Y'(I_n-P)Y}$

\section*{Problem 4b.4}

\begin{align*}
\begin{pmatrix}
Y_1\\Y_2\\Y_3
\end{pmatrix} &= \begin{pmatrix}
1 & 1 \\
0 & 2 \\
-1 & 1\\
\end{pmatrix} \begin{pmatrix}\theta_1 \\ \theta_2 \end{pmatrix}+\begin{pmatrix}\epsilon_1\\ \epsilon_2\end{pmatrix}\\
&= X\beta+\epsilon\\
H:\ \theta_1 &= 2\theta_2\\
\implies \begin{pmatrix}1 & -2 \end{pmatrix}\\
&= A\beta = 0\\
F &= \frac{\frac{RSS_H-RSS}{q}}{\frac{RSS}{n-p}}\\
\end{align*}

Now,
\begin{align*}
\hat{\beta} &= (X'X)^{-1}X'Y\\
X'X = \begin{pmatrix} 2 & 0\\
0 & 6
\end{pmatrix}\\
(X'X)^{-1} &= \begin{pmatrix} 1/2 & 0\\
0 & 1/6
\end{pmatrix}\\
\hat{\beta} &= \begin{pmatrix} 1/2 & 0\\ 0 & 1/6 \end{pmatrix}\begin{pmatrix}\frac{Y_1-Y_3}{2}\\
\frac{Y_1+2Y_2+Y_3}{6}
\end{pmatrix}\\
\end{align*}

\begin{align*}
\hat{Y} &= X\hat{\beta}\\
&= \begin{pmatrix}
1 & 1 \\
0 & 2 \\
-1 & 1\\
\end{pmatrix}\begin{pmatrix}\frac{3Y_1-3Y_3}{6}\\
\frac{Y_1+2Y_2+Y_3}{6}
\end{pmatrix}\\
&= \begin{pmatrix}
\frac{4Y_1+2Y_2-2Y_3}{6}\\
\frac{Y_1+2Y_2+Y_3}{3}\\
\frac{-2Y_1+2Y_2+4Y_3}{6}\\
\end{pmatrix}
\end{align*}

\begin{align*}
RSS &= (Y-\hat{Y})'(Y-\hat{Y})\\
&= \begin{pmatrix}\frac{2Y_1-2Y_2+2Y_3}{6} &
\frac{-Y_1+Y_2-Y_3}{3} & \frac{2Y_1-2Y_2+2Y_3}{6}
\end{pmatrix}'.\begin{pmatrix}\frac{2Y_1-2Y_2+2Y_3}{6} \\
\frac{4y_1-4Y_2-2Y_3}{6} \\ \frac{2Y_1-2Y_2+2Y_3}{6}
\end{pmatrix}\\
&= \frac{1}{9}(Y_1-Y_2+Y_3)^2 + \frac{1}{9}(-Y_1+Y_2-Y_3)^2 + \frac{1}{9}(Y_1-Y_2+Y_3)^2\\
&= \frac{1}{3}(Y_1-Y_2+Y_3)^2
\end{align*}

\begin{align*}
A(X'X)^{-1}A' &= \begin{pmatrix}1/2 & -1/3 \end{pmatrix}\begin{pmatrix}
1 \\ -2
\end{pmatrix}\\
&= \frac{7}{6}\\
RSS_H-RSS &= [A\hat{\beta}]'[A(X'X)^{-1}A']^{-1}[A'\hat{\beta}]\\
&= \frac{6}{7}(\hat{\beta_1}-2\hat{\beta_2})^2
\end{align*}

Thus,
\begin{align*}
F &= \frac{\frac{6}{7}(\hat{\beta_1}-2\hat{\beta_2})^2}{\frac{1}{3}(Y_1-Y_2+Y_3)^2}
\end{align*}

\section*{Problem 4b.5}
$Y=I\theta+\epsilon$

$H: \theta_1 =\theta_3 $ or $\begin{pmatrix} 1 & 0 & -1 & 0 \end{pmatrix}\theta=0$

\begin{align*}
A &= \begin{pmatrix}
1 & 0 & -1 & 0
\end{pmatrix}\\
\hat{\beta} = \begin{pmatrix}
Y_1\\
Y_2\\
Y_3\\
Y_4
\end{pmatrix}\\
A\beta &= \begin{pmatrix} 1 & 0 & -1 & 0 \end{pmatrix} \begin{pmatrix}
Y_1\\Y_2\\Y_3\\Y_4
\end{pmatrix}\\
&= Y_1-Y_3\\
RSS_H-RSS &= \begin{pmatrix} Y_1 & -Y_3 \end{pmatrix}\frac{1}{2}\begin{pmatrix} Y_1 \\
-Y_3
\end{pmatrix}\\
&= \frac{1}{2}(Y_1-Y_3)^2\\
RSS = 3S^2\\
&= 3(Y_1+Y_2+Y_3+Y_4-0)^2\\
F &= \frac{RSS_H-RSS}{3S^2}\frac{3}{1}\\
&= \frac{(Y_1-Y_3)^2}{2(Y_1+Y_2+Y_3+Y_4)^2}
\end{align*}

\section*{Problem 4MISC.2}
\begin{align*}
\begin{pmatrix}
Y_{11}\\
Y_{12}\\
\vdots\\
Y_{1n}\\
Y_{21}\\
Y_{22}
\vdots\\
Y_{2n}
\end{pmatrix}&= \begin{pmatrix}x_1 & 0\\
x_2 & 0 \\
\vdots \\
x_n & 0\\
0 & x_1\\
0 & x_2\\
\vdots\\
0 & x_n
\end{pmatrix}\begin{pmatrix}\beta_1 \\ \beta_2\end{pmatrix} + \epsilon\\
H: A\hat{\beta} &=0 \implies \begin{pmatrix} 1 & -1 \end{pmatrix} = 0\\
A &= \begin{pmatrix} 1 & -1 \end{pmatrix}\\
X'X^{-1} &= \begin{pmatrix}\frac{1}{\sum_i x_i^2} & 0\\
0 & \frac{1}{\sum_i x_i^2}\end{pmatrix}\\
\hat{\beta} &= (X'X)^{-1}X'Y\\
&= \begin{pmatrix}\frac{1}{\sum_i x_i^2} & 0\\
0 & \frac{1}{\sum_i x_i^2}\end{pmatrix}  \begin{pmatrix}\sum_ix_i y_{i1}\\ \sum_i x_iy_{i2}\end{pmatrix}\\
&= \begin{pmatrix}\frac{\sum_ix_i y_{i1}}{\sum_i x_i^2}\\ \frac{\sum_i x_iy_{i2}}{\sum x_i^2}\end{pmatrix}\\
&= \begin{pmatrix}\hat{\beta_1} \\ \hat{\beta_2}\end{pmatrix}\\
RSS &= \sum_j\sum_i(y_{ji}-x_{ji}\beta_i)^2\\
&= \sum_i (y_{i1}^2-2y_{i1}x_i\hat{\beta_1} + x_{i1}^2\beta_1^2 + y_{i2}^2-2y_{2i}x_i + x_{2i}\beta_2^2)\\
RSS_H-RSS &= (A\beta)'[A'(X'X)^{-1}A]^{-1}(A\beta)\\
&= (\hat{\beta_1}-\hat{\beta_2}) [\begin{pmatrix} \frac{1}{\sum_i x_i^2} & \frac{-1}{\sum_i{x_i^2}} \end{pmatrix} \begin{pmatrix} 1\\ -1\end{pmatrix}]^{-1}\begin{pmatrix}\hat{\beta_1} \\ -\hat{\beta_2}\end{pmatrix}\\
&= (n-1)S^2
RSS_H-RSS &= (A\hat{\beta})'[A'(X'X)^{-1}A]^{-1}A\hat{\beta}\\
&= \frac{n}{n+1}(\bar{Y_n}-Y_{n+1})^2\\
&= \sum_i x_i^2 \frac{(\hat{\beta_1}-\hat{\beta_2})^2}{2}\\
F &= \frac{\frac{RSS_H-RSS}{1}}{\frac{(n-1)S^2}{n-1}}\\
&= \frac{\sum_i x_i^2\frac{(\beta_1-\beta_2)^2}{2}}{S^2}\\
&= \frac{(\beta_1-\beta_2)^2}{2S^2(\sum_x x_i^2)^{-1}}
\end{align*}

\section*{Problem 4MISC.4}
\begin{align*}
\begin{pmatrix}
Y_1\\
Y_2\\
\vdots\\
Y_n\\
Y_{n+1}
\end{pmatrix}&= \begin{pmatrix}
1 & 0\\
1 & 0\\
\vdots & \vdots\\
1 & 0\\
0 & 1
\end{pmatrix}\begin{pmatrix}\mu_1 \\ \mu_2
\end{pmatrix}+ \begin{pmatrix} \epsilon_1\\
\epsilon_2\\
\vdots\\
\epsilon_n\\
\epsilon_{n+1}
\end{pmatrix}\\
H &:  A\hat{\beta} = 0 \implies \begin{pmatrix}1 & -1\end{pmatrix}\begin{pmatrix}\mu_1\\
\mu_2
\end{pmatrix} = 0 \\
X'X &= \begin{pmatrix}n & 0 \\ 0 & 1 \end{pmatrix}\\
(X'X)^{-1} &= \begin{pmatrix}1/n & 0 \\ 0 & 1 \end{pmatrix}\\
\begin{pmatrix}\hat{\mu_1}\\
\hat{\mu_2}
\end{pmatrix} &= \begin{pmatrix}\frac{1}{n}\sum_{i=1}^nY_i\\
Y_{n+1}
\end{pmatrix} = \begin{pmatrix} \bar{Y_n} \\ Y_{n+1}\end{pmatrix}\\
\begin{pmatrix}
\hat{Y_1}\\
\hat{Y_2}\\
\vdots\\
\hat{Y_n}\\
\hat{Y_{n+1}}
\end{pmatrix}&= \begin{pmatrix}
\bar{Y_n}\\
\bar{Y_2}\\
\vdots\\
\bar{Y_n}\\
\bar{Y_{n+1}}
\end{pmatrix}\\
RSS &= (Y-\hat{Y})'(Y-\hat{Y})\\
&= \begin{pmatrix}Y_1-\bar{Y_n} & Y_2-\bar{Y_n} & \dots Y_n-\bar{Y_n}& 0 \end{pmatrix}\begin{pmatrix}Y_1-\bar{Y_n} \\ Y_2-\bar{Y_n} \\ \vdots\\ Y_n-\bar{Y_n} \\ 0 \end{pmatrix}\\
&= \sum_{i=1}^n(Y_i-\bar{Y_n})^2\\
&= (n-1)S_n^2\\
F &= \frac{\frac{RSS_H-RSS}{1}}{\frac{RSS}{n+1-2}}\\
&= \frac{n}{n+1}\frac{(Y_{n+1}-\bar{Y_n})^2}{S_n^2}
\end{align*}

\end{document}


\documentclass[12pt]{article}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{fullpage}
\usepackage{systeme}
\usepackage{blindtext}
\usepackage{graphicx}
\graphicspath{{./fig/}}
\setlength{\parskip}{1em}
% This gives us a full line break when we write a new paragraph

\title{MATH 308 Conceptual Problems, Week 4 \vspace{1em}\\\normalsize{SUPPLEMENTARY CONCEPTUAL PROBLEMS, 8d \& 8e}}
\author{Jai Broome}
\date{28 January 2020}

\begin{document}

\maketitle

\section{Question prompt}

In each of the following, $\bar{x}$ is the vector whose components are the coefficients
in $f(t)$, and $\bar{y}$ is the vector whose components are the coefficients in $f'(t)$. Taking
the derivative of $f(t)$ in the usual way, you can find formulas for the $y$'s in terms
of the $x$'s. Then find a matrix $A$ such that $\bar{y}=A\bar{x}$.

\section{8d}
\textit{Question prompt}: $f(t) = x_1\sin(t) + x_2\cos(t) + x_3t\sin(t) + x_4t\cos(t)$,
and $f'(t)$ has the same form with coefficients $y_1$, $y_2$, $y_3$, $y_4$.

The reason we can approach this problem this way is because
\begin{itemize}
  \item we can differentiate $f(t)$ in its general form,
    % that is, we don't need to know specific values for \bar{x}
  \item and because the prompt tells us that $f'(t)$ will have a $\sin(t)$ term,
    a $\cos(t)$ term, a $t\sin(t)$ term and a $t\cos(t)$ term.
\end{itemize}
% could be any function g_i(t) that appears in the function and derivative
% could be zeroes
% and because we can tell by looking at each term

This second point means that the plot of $f'(t)$ will have some visual
similarity to the plot of $f(t)$, since they're both composed of a series of
the same functions functions multiplied by constants. \\

The tricky thing about this type of problem is that $f(t)$ and it's derivatives are 1-dimensional
mappings $\mathbb{R} \rightarrow \mathbb{R}$; \textit{however}, our goal here is to find a mapping
$\mathbb{R}^4 \rightarrow \mathbb{R}^4$ that takes our 4-dimensional $\bar{x}$ and produces a
4-dimensional $\bar{y}$
% the function takes one value $t$ and returns one value $f(t)$.

Find the derivative the usual way, treating the $x_i$s as constants, and making use of the product
rule:

\[
f'(t) = (-x_2 + x_3)\sin(t) + (x_1 + x_4)\cos(t) + (-x_4)t\sin(t) + (x_3)t\cos(t)
\]

Let's pick a vector $\bar{x}=(2,4,6,8)$ to visualize $f(t)$ and $f'(t)$, and to later check our
work. Inserting our values for the $x_i$s:

\includegraphics[width=8cm]{plot_8d_f} \includegraphics[width=8cm]{plot_8d_derivative} \\
\includegraphics[width=17cm]{plot_8d_both} \\
\newpage
Define the $y_i$s in $f'(t)$ so that they correspond to same positions as the $x_i$s in $f(t)$:
% that is, y_1 is in front sin(t) etc

\[
\systeme{-x_2 + x_3 = y_1, x_1 + x_4 = y_2, -x_4 = y_3,x_3 = y_4}
\]

In matrix form, this is our solution $A$:

\[
\begin{bmatrix}
  0 & -1 & 1 & 0 \\
  1 & 0 & 0 & 1 \\
  0 & 0 & 0 & -1 \\
  0 & 0 & 1 & 0
\end{bmatrix}
\]

We can check our work by seeing if we get the same solution by plugging
$\bar{x}$ into $f(t)$ and differentiating; and computing $A\bar{x}$.
\[
  \frac{d}{dt}(2\sin(t)+4\cos(t)+6t\sin(t)+8t\cos(t)) =
\]
\[
  2\sin(t)+10\cos(t)-8t\sin(t)+6t\cos(t)
\]
\bigbreak{}
\[
  \bar{y} = A\bar{x} =
  \begin{bmatrix}
    0 & -1 & 1 & 0 \\
    1 & 0 & 0 & 1 \\
    0 & 0 & 0 & -1 \\
    0 & 0 & 1 & 0
  \end{bmatrix}
  \begin{bmatrix} 2 \\ 4 \\ 6 \\ 8\end{bmatrix} =
  \begin{bmatrix} 2 \\ 10 \\ -8 \\ 6 \end{bmatrix}
\]

\newpage

Briefly, this lets us quickly calculate $f''(t)$. Let $\bar{z}$ be the coefficients in $f''(t)$.

\[
  \bar{z} = A\bar{y} =
  \begin{bmatrix}
    0 & -1 & 1 & 0 \\
    1 & 0 & 0 & 1 \\
    0 & 0 & 0 & -1 \\
    0 & 0 & 1 & 0
  \end{bmatrix}
  \begin{bmatrix} 2 \\ 10 \\ -8 \\ 6 \end{bmatrix} =
  \begin{bmatrix} -18 \\ 8 \\ -6 \\ -8 \end{bmatrix}
\]
\[
  \rightarrow f''(t) = -18\sin(t) + 8\cos(t) - 6t\sin(t) - 8t\cos(t)
  % all that entailed was multiplying a 4x4 matrix by a vector
\]

\section{8e}

\textit{Question prompt}: $f(t) = (x_1 + x_2t + x_3t^2)e^{-kt}$, and $f'(t)$ has the same
form with coefficients $y_1$, $y_2$, $y_3$.

As before, our first step is to differentiate the general form of $f(t)$.

\[
  f'(t) = (-kx_1 + x_2 + (-kx_2+2x_3)t - kx_3t^2)e^{-kt}
\]

We have $k$ terms in here, but that's okay. $k$ is a parameter in our equation, not an unkown and
is \textit{not} in $\bar{x}$. It appears in our solution, but that's alright. We can do matrix
multiplication that includes constant terms.

Next, define the $y_i$s so that they correspond to the constant, $t$ and $t^2$ terms

\[
  \systeme{-kx_1 + x_2 = y_1, -kx_2+2x_3 = y_2, -kx_3 = y_3}
\]

In matrix form, this is our solution $A$:

\[
  \begin{bmatrix}
    -k & 1 & 0 \\
    0 & -k & 2 \\
    0 & 0 & -k
  \end{bmatrix}
\]

\newpage

Using $\bar{x} = (5, 10, 15)$, we can compare our solution differentiating $f(t)$ and computing $A\bar{x}$.

\[
  \frac{d}{dt}((5+10t+15t^2)e^{-kt}) =
\]
\[
  (10-5k + (30-10k)t -15kt^2)e^{-kt}
\]
\medbreak{}
\[
  \bar{y} = A\bar{x} =
  \begin{bmatrix}
    -k & 1 & 0 \\
    0 & -k & 2 \\
    0 & 0 & -k
  \end{bmatrix}
  \begin{bmatrix} 5 \\ 10 \\ 15 \end{bmatrix} =
  \begin{bmatrix} 10-5k \\ 30-10k \\ -15k \end{bmatrix}
\]
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}
\bigbreak{}

\textit{presentation materials avalable at} github.com/broomej/m308\_w4\_cp
\end{document}


\subsection{Definitions}
\begin{itemize}
\item $\lambda$ : intrinsic coordinate, including masses and spins.

\item $\theta$ : extrinsic coordinate, including $d,RA,DEC,\iota,\psi_L,t,\phi_{\rm orb}$

\item $p_s(\theta)$: (joint) sampling prior in extrinsic dimensions

\item $p(\theta)$ : prior on extrinsic parameters

\item $\Like(\lambda,\theta)$ : likelihood.  In terms of individual detector strains $H_k$ and power spectra, provided by
\begin{eqnarray}
\ln L &\equiv \sum_k \ln L_k  = \ln L_{\rm model} + \ln L_{\rm data} \\
\ln L_{\rm model} &\equiv -\frac{1}{2} \sum_k \qmstateproduct{H_k}{H_k}_k  \\
\ln L_{\rm data} &\equiv  \sum_k \text{Re} \qmstateproduct{H_k}{\hat{H}_k}_k
\end{eqnarray}

\item $Z(\lambda) \equiv L_{\rm red}(\lambda,\theta)$ : reduced or integrated likelihood, derived from $L$ via
\begin{eqnarray}
Z(\lambda) = L_{\rm red}(\lambda) = \int d\theta \; p(\theta) L(\lambda,\theta)
\end{eqnarray}

\item
$w=Lp/p_s$ : weight

\item
$n_{\rm eff}$ : ``effective number of samples''
\begin{eqnarray}
n_{\rm eff} \equiv  \frac{\sum_k w_k}{\text{max}_k w_k}
\end{eqnarray}

\item
$h(t|\lambda,x)=h_+-i h_\times$ : complex gravitational wave strain

\item
$h_{lm}(t)$: coefficients of a spin-weighted spherical harmonic decomposition
\begin{eqnarray}
\label{eq:def:hSpinWeightEmissionDirection}
h(t|\lambda,\theta) = \sum_{lm} h_{lm}(t|\lambda) e^{-2i\psi}\Y{-2}_{lm}(\theta_{JN}\phi_{JN})
\end{eqnarray}

\item
$\tilde{h}(f)$ : two-sided Fourier transform of the complex function $h(t)$
\begin{eqnarray}
h(t) = \int_{-\infty}^{\infty} \frac{d \omega}{2\pi} \; e^{-i\omega t} \tilde{h}(\omega)
\end{eqnarray}

%% \item
%% ${\cal I}$ : complex conjugation in time.  Provided to avoid confusion with $\tilde{h}^*$.  \textbf{Hopefully we won't
%%   need it.}

\item
$\vec{x}_k$ : Position of the $k$th detector

\item
$F_{+}$, $F_{\times},F$ : detector response function  to the $+,\times$ polarizations for sources visible in the
  $\hat{n}$ direction relative to detector
\begin{eqnarray}
F(\hat{n}) = F_+(\hat{n}) +i F_\times(\hat{n})
\end{eqnarray}

\item
$\hat{H}_k$ : measured strain in  the $k$th detector

\item
$H_k$ : strain response of the $k$th detector to an incident strain $h$
\begin{align}
H_k(t) &=F_{+,k}(t) h_+(t-\vec{x}_k(t)\cdot \hat{k}) + F_\times(t) h_\times(t-\vec{x}_k(t)\cdot \hat{k}) \\
 &=  \frac{F h(t-\vec{x}_k\cdot \hat{k}) }{2} + \frac{F^*h^*(t-\vec{x}_k\cdot \hat{k})}{2}
\end{align}

\item
$S_k$ : noise power spectrum for the $k$th detector

\item
$\qmstateproduct{a}{b}_k$ : complex-valued inner product defined by the $k$th detector's noise power spectrum:
\begin{eqnarray}
\qmstateproduct{a}{b}_k \equiv 2 \int_{-\infty}^{\infty} df \frac{[\tilde{a}(f)]^*\tilde{b}(f)}{S_h(|f|) }
\end{eqnarray}

\end{itemize}


\chapter{Calculating Amplitudes}
There are a number of possible approaches to calculating an amplitude for a given interaction. These include relativistic generalisations of the Born approximation (Feynman-Stueckelberg) or perturbation theory (Halzen and Martin), and the mathematically heavy canonical field theory or the path integral approach. The idea of the propagator is common to some of these approaches and this is what we will consider in our treatment of amplitude calculations.

\section{Single scattering potential}
\begin{figure}[hb]
\centering
\include{figures/singleScatter}
\caption{Scattering of a plane wave off a localised potential $V$.\label{fig:singleScatter}}
\end{figure}

Consider a particle $\psi(\vec{r},t)$ that scatters off some potential $V(\vec{r},t)$, where $V$ acts only at a position $\vec{r_1}$ and for a short time $\Delta t$. In the interaction picture the Schr{\"o}dinger equation is
\begin{equation}
\left(\hat{H}_0 + V\right)\psi(\vec{r},t) = i \pdv{\psi}{t}
\end{equation}
where the interaction is large, $V \gg \hat{H}_0$. Ignoring the free $\hat{H}_0$ term and solving the separable differential equation,
\begin{equation}
i\int\limits_{\phi}^{\phi+\Delta\psi} \dd{\psi} = \int\limits_{-\infty}^{+\infty} V(\vec{r},t)\, \psi(\vec{r},t) \, \dd{t}
\end{equation}
where $\phi$ is the original, unscattered plane wavefunction and $\psi=\phi+\Delta\psi$ is the scattered wavefunction. Since the potential $V$ only acts at position $\vec{r_1}$ and has constant value $V$ for time $\Delta t$, this evaluates to give
\begin{align}
\Delta\psi(\vec{r_1},t_1) &= -i V(\vec{r_1},t_1) \, \psi(\vec{r_1},t_1) \, \Delta{t} \\
&= -i V({x}_1) \left[ \phi({x}_1) + \Delta\psi({x}_1) \right] \Delta{t} \\
&\approx -iV(x_1) \, \phi(x_1) \, \Delta t\label{eq:scatter}
\end{align}
where ${x}_1$ is the event at $(\vec{r_1}, t_1)$.

We wish to know the wavefunction after scattering at some event $x^\prime$. Using the formalism of Green's functions,
\begin{equation}\boxed{
\Psi(x^\prime) = i \int G(x^\prime, x) \, \Psi(x) \, \dd[3]{\vec{r}} \quad \text{for $t^\prime > t$}
}\end{equation}
where $G(x^\prime, x)$ is the Green's function of the Schr{\"o}dinger equation, and is called the propagator from $x$ to $x^\prime$. Applying this to the change in wavefunction caused by the scattering potential, \eqref{eq:scatter},
\begin{align}
\Delta\psi(x^\prime) &= i \int G(x^\prime, x_1) \, \Delta\psi(x_1) \, \dd[3]{\vec{r}_1} \\
&= \int G(x^\prime, x_1) \, V(x_1) \, \phi(x_1) \, \Delta{t} \, \dd[3]{\vec{r}_1}.
\end{align}
So overall, the particle's wavefunction at the event $x^\prime$ is, to first order,
\begin{equation}
\psi(x^\prime) = \phi(x^\prime) +  \int G(x^\prime, x_1) \, V(x_1) \, \phi(x_1) \, \Delta{t} \, \dd[3]{\vec{r}_1}
.\end{equation}
In the limit of a continuous interaction, we can integrate over the time interval $\Delta t$,
\begin{equation}\boxed{
\psi(x^\prime) = \phi(x^\prime) +  \int G(x^\prime, x_1) \, V(x_1) \, \phi(x_1) \, \dd[4]{x_1}
}.\end{equation}

\section{Two scattering potentials}
The above result may be easily generalised to multiple scattering potentials. A long-range force can be modelled, to first order, by a particle scattering from two localised potentials.
\begin{figure}[hb]
\centering
\include{figures/doubleScatter}
\caption{A particle undergoing double scattering in the propagator approach.\label{eq:doubleScatter}}
\end{figure}

Consider two localised scattering potentials, each with strength $V$, at events $x_1$ and $x_2$. Then extending the above result gives that the post-scattering wavefunction is
\begin{align}
\psi(x^\prime) = \phi(x^\prime) &+ \int G(x^\prime, x_1) \, V(x_1) \, \phi(x_1)  \dd[4]{x_1} \nonumber \\
&+ \int G(x^\prime, x_2) \, V(x_2) \, \phi(x_2) \, \dd[4]{x_2} \nonumber \\
&+ \int \int G(x^\prime, x_2) \, V(x_2) \, G (x_2, x_1) \, V(x_1) \, \phi(x_1) \, \dd[4]{x_1} \, \dd[4]{x_2}\label{eq:doubleScatter}
\end{align}

\section{Free particle propagator}
Start from
\begin{equation}
\psi(x^\prime) = i \int \dd[3]{\vec{r}} \, G(x^\prime; x) \, \psi(x)
\end{equation}
for $t^\prime > t$. This condition can be encapsulated mathematically by the Heaviside function,
\begin{equation}
\theta(t^\prime - t)\, \psi(x^\prime) = i \int \dd[3]{\vec{r}} \, G(x^\prime, x) \, \psi(x). \label{eq:start}
\end{equation}
Now multiply both sides by $\left[ i \pdv{t} - \hat{H} \right]$. The left-hand side becomes,
\begin{align}
\left[ i \pdv{t} - \hat{H} \right]\theta(t^\prime - t)\, \psi(x^\prime) &= i \pdv{\theta}{t} \psi(x^\prime) + \theta(t^\prime-t) \underbrace{\left( i\pdv{\psi}{t} - \hat{H}\psi(x^\prime) \right)}_{0\text{ by TDSE}} \nonumber \\
&= i \delta(t^\prime - t) \, \psi(x^\prime) \\
&= i \int \dd[3]{\vec{r}} \, \delta(x^\prime - x) \, \psi(x) \\
&= i \int \dd[3]{\vec{r}} \int \frac{\dd[4]{p}}{(2\pi)^4} \, e^{i(x^\prime-x)p} \,\psi(x) \label{eq:LHS73}
\end{align}
Now consider the right-hand side of \eqref{eq:start}. Upon multiplication by $\left[ i \pdv{t} - \hat{H} \right]$ it becomes
\begin{equation}
i \int \dd[3]{\vec{r}} \, \left[ i \pdv{t} - \hat{H} \right] \, G(x^\prime, x) \, \psi(x) = i \int \dd[3]{\vec{r}} \, \left[ E - \frac{p^2}{2m} \right] \, G(x^\prime, x) \, \psi(x)
\end{equation}
where the operators have been applied to the free plane wavefunction. Taking the Fourier transform to 4-momentum space, the right hand side becomes
\begin{equation}
i \int \dd[3]{\vec{r}} \int \frac{\dd[4]{p}}{(2\pi)^4} \, \left[ E - \frac{p^2}{2m} \right] \, \mathcal{G}(p^\prime, p) \, e^{i(x^\prime-x)p} \, \psi(x).\label{eq:RHS73}
\end{equation}
where $\mathcal{G}$ is the momentum-space propagator. Now we can compare the integrands of \eqref{eq:LHS73} and \eqref{eq:RHS73} to give an expression for the momentum-space free particle propagator:
\begin{equation}\boxed{
\mathcal{G}(p^\prime, p) = \frac{1}{E - \frac{p^2}{2m}}
}.\end{equation}

This is used as the propagator for off-mass shell virtual particles. As the particle approaches its mass shell, this expression diverges. To allow for integration over momentum, a small complex factor may be added to the denominator,
\begin{equation}
\mathcal{G}(p^\prime, p) = \frac{1}{E - \frac{p^2}{2m} + i\epsilon}
\end{equation}
and the result taken in the limit where $\epsilon \rightarrow 0$.


\chapter{Linear Algebra}
\addcontentsline{toc}{section}{Introduction}
\section*{Introduction}
VSIPL specifies support for standard matrix operations such as matrix products,
methods to solve the standard matrix equation $A \vec{x} = \vec{b}$, and methods to solve least squares problems. VSIPL hides the decomposition of matrices in objects. So in addition to standard matrix products, special functions for doing matrix products with decomposition matrices are provided.

We note that although vectors are treated as column vectors in equations, VSIPL vector views have only one stride and so the action of the vector within the function is defined only by the function definition.

In general all matrix views passed into a function are defined as type const. This means that the area of the block mapped by the view does not change inside of the function call. For some of the defined in place operations where the input and output are defined by the same view the input matrix size may be different than that required by the output data. For these cases the strides of the input view define where the output data is placed. The first element of the output data replaces the first element of the input data. The author recommends defining a view of the output data space for convenience. For a couple of cases the output data space may be bigger than the input data space. Defining an output data view will ensure that the strides of the input view and the size of the block are sufficient to hold the output data.
\addcontentsline{toc}{section}{Simple Matrix-Matrix and Vector-Matrix Operations}
\section*{Simple Matrix-Matrix and Vector-Matrix Operations}
\section*{Simple Solvers}\addcontentsline{toc}{section}{Simple Solvers}
\section*{LU Decomposition}\addcontentsline{toc}{section}{LU Decomposition}
\section*{Cholesky Decompostion}\addcontentsline{toc}{section}{Cholesky Decompostion}
\section*{QR Decompostion}\addcontentsline{toc}{section}{QR Decompostion}
\section*{Singular Value Decomposition}\addcontentsline{toc}{section}{Singular Value Decomposition}


\section{Introduction}\label{sec:intro}
Gravitational Lensing provides us a way to see how dark matter along with
visible matter is distributed in universe. This theory of gravitational lensing is supported by Einstein's general theory of relativity which predicts the deflection of light in a gravitational field if any massive object is present there (\cite{hartle03}).
%
%
%```````````````````````````````````````````````````````````````````````````````
%         Subsection : Einstein's Deflection Angle
%```````````````````````````````````````````````````````````````````````````````
%
\subsection{Einstein's Deflection Angle}
  % aug 14, sch06  page 64
  \begin{figure}[ht!]
      \centering
      \includegraphics[width=0.5\textwidth]{grav_lensing}
      \caption[Simple sketch to gravitational lensing.] {Simple sketch to gravitational lensing ~\protect\cite{barSch_01}} %{~\protect\cite{barSch_01} } % \protect ~\cite{barSch_01}
      \label{[fig:lensing] }% \protect \cite{barSch_01}
  \end{figure} % (Bartelmann and Schneider 2001)}

  General Relativity predicts that when the beam of light passes through near
  the massive objects, the rays of light undergo deflection from their original path.
  This angle of deflection was predicted by Einstein's theory of General Relativity.
  According to this theory if the massive object of mass $M$ is located at a
  perpendicular distance $\xi$ (called \textbf{impact parameter}) from the line of sight
  of source and the observer then the deflection caused by that mass is given by
  \cite{sch07}
  \begin{equation}
    \hat{\alpha} = \frac{4GM}{c^2\xi}.
    \label{[eq:deflection]}
  \end{equation}

  This equation (\ref{[eq:deflection]} ) is valid only when the angle
  $\hat{\alpha} <<1  $ . In  case of gravitational lensing, the product of mass of the
  deflector and the Gravitational Constant is always the much smaller than the
  squared of velocity of light, thus making the deflection angle very small.

  For quantitative purpose,
  we can calculate the value of deflection angle for distant stars appearing near to the
  solar limb by setting the mass $M = M_{\odot}$ and radius $ R = R_{\odot}$ in the above equation to obtain the angle of deflection $\hat{\alpha} = 1.74''$. This value was tested
  in famous solar eclipse experiment in May 29, 1991 which was conceived by Sir Frank Watson Dyson, Astronomer Royal of Britain in 1917 and led by Sir Arthur Stanley Eddington
  two years later in 1919 (\cite{eclipse19}). The experiment was designed to test following hypotheses:

  \begin{itemize}

    \item{The light path is uninfluenced by gravitations.}
    \item{The law of gravitation will follow Newtonian law and will produce
            0''.87 apparent displacement}
    \item{The course of ray of light will follow Einsteins generalized
            relativity and lead to apparent displacement of 1.74''.}
  \end{itemize}
  The experiment concluded that the results were close to the Einsteins predicted
  deflection angle and thus supporting the gravitational lensing theory.

  %capranico 63 and schBar 327
  This was the deflection due to point source. We can also define the delfection
  for the extended mass with certain surface mass density $\Sigma$ as
  \begin{equation}\label{[eq:alphaHat]}
    \hat{\boldsymbol{\alpha}} (\boldsymbol{\xi} ) = \frac{4G}{c^2} \int d^2 \boldsymbol{\xi\prime}\quad  \Sigma (\boldsymbol{\xi\prime}) \quad  \frac{\boldsymbol{\xi} - \boldsymbol{\xi\prime}}{|\boldsymbol{\xi} - \boldsymbol{\xi\prime} |^2}
  \end{equation}
  Here, the \textit{surface mass density} $\boldsymbol{\Sigma}$ is defined as,
  \begin{equation}\label{[eq:surface_mass_density]}
    \boxed{
    \Sigma(\boldsymbol{\xi}) \equiv \int dr_3 \ \rho(\xi_1, \xi_2, r_3)} \quad (\text{Surface Mass Density})
  \end{equation}
  which is the mass density projected onto the lens plane, perpendicular to the
  light ray, and $r_3$ is the coordinate along the line of sight, and
  $\xi_1$, $\xi_2$ the other two perpendicular coordinates.

  %
  %
  %```````````````````````````````````````````````````````````````````````````````
  %         Subsubsection : Lens equation
  %```````````````````````````````````````````````````````````````````````````````
  %
\subsection{Lens equation}
  We consider a simplistic lensing model in which lens and the source objects are
  point objects. The observer observes the rays of light from the source at a
  distance $D_s$ which pass through the gravitational influence field of a massive
  object of mass M and distance $D_d$ located perpendicularly $\boldsymbol{\xi}$ distance away
  from line of sight (also called impact parameter).

  Let  $\boldsymbol{\eta}$  denotes the true, two-dimensional position of the
  source in the source plane and $\boldsymbol{\beta}$ is the true angular position
  of the source. This means in absence of light deflection we would have,
  \begin{equation}\label{[eq:beta]}
    \boldsymbol{\beta} = \frac{\boldsymbol{\eta} }{D_s}.
  \end{equation}
  The relation between position $\boldsymbol{\xi}$ and $\boldsymbol{\theta}$  is
  given by,
  \begin{equation}\label{[eq:theta]}
    \boldsymbol{\theta} = \frac{\boldsymbol{\xi} }{D_d}.
  \end{equation}
  This means $\boldsymbol{\theta}$ is the observed position of the source on the
  sphere relative to the position of center of the lens which is the origin of
  the coordinate system with $\boldsymbol{\xi} = 0$. Where, $D_{ds}$ is the distance of
  the source plane from the lens plane.

  Here we adopt the relation,
  \begin{equation}\label{[eq:D_ds]}
    D_{ds} = D_s - D_d.
  \end{equation}
  This relation holds true as long as the relevant distances are much smaller
  than the radius of the universe ($c/H_0$) and this is always the case for
  distances within our Galaxy and in Local Group. However, this relation no longer
  holds true for cosmological distances between source and lenses.

  From the figure (\ref{[fig:lensing]}), we can relate $\boldsymbol{\eta}$ with
  deflection angle $\boldsymbol{\alpha}$ as
  \begin{equation}\label{[eq:eta]}
    \boldsymbol{\eta} = \frac{D_s}{D_d} \boldsymbol{\xi} - D_{ds} \hat{\boldsymbol{\alpha} } (\boldsymbol{\xi} ).
  \end{equation}

  Using equation (\ref{[eq:beta]}) we can write $\boldsymbol{\beta}$ as
  \begin{equation}\label{[eq:beta2]}
    \boldsymbol{\beta} = \boldsymbol{\theta} - \frac{D_{ds}}{D_s}\  \hat{\boldsymbol{\alpha} } (D_d \boldsymbol{\theta} ).
  \end{equation}

  In this equation (\ref{[eq:beta2]}) we have some factor multiplying the
  deflection angle, so we define \textit{reduced deflection angle}
  \begin{equation}\label{[eq:alphaReduced]}
    \boxed{\boldsymbol{\alpha} (\boldsymbol{\theta} ) = \frac{D_{ds}}{D_s}\  \hat{\boldsymbol{\alpha}} (D_d \boldsymbol{\theta} )} \quad (\text{Reduced Deflection Angle})
  \end{equation}

  and then rewrite the lens equation (\ref{[eq:beta2]}) as
  \begin{equation}\label{[eq:beta3]}
     \boldsymbol{\beta} = \boldsymbol{\theta} - \boldsymbol{\alpha} (\boldsymbol{\theta} ).
  \end{equation}

  For a point mass object, using the equations (\ref{[eq:alphaHat]}) and (\ref{[eq:theta]}) the equation of reduced deflection angle (\ref{[eq:alphaReduced]})
  becomes
  \begin{equation}\label{[eq:alphaAbs]}
    \abs{\boldsymbol{\alpha} (\boldsymbol{\theta} )} = \frac{D_{ds}}{D_s}\  \frac{4GM}{c^2 D_d \abs{\boldsymbol{\theta}  }}.
  \end{equation}

%
%
%```````````````````````````````````````````````````````````````````````````````
%         Subsubsection : Convergence and Deflection Potential
%```````````````````````````````````````````````````````````````````````````````
% \texorpdfstring{$math$}{alternative} is to avoid Hyperref warning
%                                      Token not allowed in a PDF string
\subsection{Convergence \texorpdfstring{$\kappa$ and deflection potential $\psi$}{kappaDefletionPotential}}
  % sch bar 328 Aug 22, 2017 Tue
  Then convergence is defined as the ratio of the surface mass density of the lens
  and the critical surface mass density as like

  \begin{equation}\label{[eq:kappa]}
    \boxed{\kappa (\boldsymbol{\theta} ) \equiv \frac{\Sigma (D_d \boldsymbol{\theta} )}{\Sigma_{cr}}} \quad (\text{Convergence}).
   \end{equation}
  Where, the critical surface mass density is given by
  \begin{equation}\label{[eq:crit_surf_mass_density]}
    \Sigma_{cr} = \frac{c^2}{4\pi G} \frac{D_s}{D_d D_{ds}}.
  \end{equation}
  The critical density depends on the redshift of source and lens.
  If the convergence $\kappa \geq 1$ i.e. surface density is less than critical
  surface density then we can see the multiple images of the source. If the
  value of $\kappa$ is very large it is called "strong gravitational lensing"
  and if it is only slightly greater than one, it is called "weak gravitational lensing".
  So, the value of critical mass density plays the role in distinguishing weak
  vs. strong gravitational lensing.

  Moreover, we can define scaled deflection angle in terms of convergence as
  \begin{equation}\label{[eq:alpah_kappa]}
    \boldsymbol{\alpha}(\boldsymbol{\theta} ) = \frac{1}{\pi}\ \int d^2 \theta\prime \ \kappa(\boldsymbol{\theta}\prime) \ \frac{\boldsymbol{\theta} - \boldsymbol{\theta\prime} }{\abs{\boldsymbol{\theta} - \boldsymbol{\theta\prime} }^2}\  .
  \end{equation}
  and the \textit{deflection potential} can be defined as
  \begin{equation}\label{[eq:psi]}
    \psi(\boldsymbol{\theta}) = \frac{1}{\pi}\  \int d^2 \theta\prime \ \kappa(\boldsymbol{\theta\prime} ) \ ln\abs{\boldsymbol{\theta} - \boldsymbol{\theta\prime}}.
  \end{equation}

%
%
%```````````````````````````````````````````````````````````````````````````````
%         Subsubsection : Multiple Images
%```````````````````````````````````````````````````````````````````````````````
%
\subsection{Multiple Images}
  We can see the multiple images of the source at different places
  $\boldsymbol{\theta}_i $ if the equation (\ref{[eq:beta3]})
  holds true for different values of the deflection angles.

  \begin{figure}[ht!]
      \centering
      \includegraphics[width=0.5\textwidth]{multiple_images}
      \caption[Multiple images of single source]{Multiple images of single source ~\protect\cite{sch07}}
      \label{[fig:multiple_images]}
  \end{figure}

  This figure illustrates a typical situation in which there are two images $S_1$
  and $S_2$ of the single source $S$ which is lensed by the point massive object
  $L$.

  Here, if we take direction of deflection angle as pointing towards the source,
  we can write the deflection angle for the point mass (\ref{[eq:alphaAbs]}) as
  \begin{equation}\label{[eq:alpha_point_mass]}
    \boxed{\boldsymbol{\alpha} (\boldsymbol{\theta}  ) \equiv \frac{4GM}{c^2}\  \frac{D_{ds}}{D_s D_d}\  \frac{\boldsymbol{\theta} }{\abs{\boldsymbol{\theta}}^2}} \quad (\text{Reduced Deflection Angle})
  \end{equation}

  Now, we define \textit{Einstein angle} as
  \begin{equation}\label{[eq:einstein_angle]}
    \boxed{\theta_E = \sqrt{\frac{4GM}{c^2}\  \frac{D_{ds}}{D_s D_d}}} \quad (\text{Einstein Angle}).
  \end{equation}
  Then, we can rewrite the equation (\ref{[eq:alpha_point_mass]}) as
  \begin{equation}\label{[eq:beta_einstein]}
    \boldsymbol{\beta} = \boldsymbol{\theta} - {\theta_E}^2 \ \frac{\boldsymbol{\theta} }{\abs{\boldsymbol{\theta}  }^2 }
  \end{equation}

  To solve this equation (\ref{[eq:beta_einstein]}) we define two scaling factors
  \begin{equation}\label{[eq:x_y_lens]}
    y = \frac{\boldsymbol{\beta} }{\theta_E}\  ; \  x = \frac{\boldsymbol{\theta} }{\theta_E}
  \end{equation}

  Then we get
  \begin{equation}\label{[eq:y_lens]}
    \textbf{y} = \textbf{x} - \frac{\textbf{x}  }{\abs{\textbf{x}  }^2}.
  \end{equation}

  This is a quadratic equation and the solutions are given by,
  \begin{equation}\label{[eq:x_lens]}
    \textbf{x} = \frac{1}{2} (\abs{\textbf{y}} \pm \sqrt{4+ \abs{\textbf{y} }^2}) \
    \frac{\textbf{y} }{\abs{\textbf{y} }}.
  \end{equation}

  Following information can be drawn from the solution of lens equation:
  \begin{easylist}

    & {Except for the divergence condition $\theta \rightarrow 0$ any source at position $y$ has two images.}
    & {The two images are on the opposite sides of the source position when viewed from observer position.}
    & {If the source is exactly behind the lens ($\textbf{y} = 0$), we see the circular \textit{Einstein ring}. }
    & { \textit{Einstein ring} has the angular diameter $2 \theta_E $ and it gives the characteristic images separation.}
  \end{easylist}

  %
  %
  %```````````````````````````````````````````````````````````````````````````````
  %         Subsubsection : Magnification and Shear
  %```````````````````````````````````````````````````````````````````````````````
  %
\subsection{Magnification $\mu$ and shear $\gamma$ }
  The light rays in gravitational are not bent uniformly, the ones near to the
  lens are deviated more and the ones that are farther are bent in smaller
  proportion. This differential deflection gives rise to the distorted and
  magnified images of the source object.

  Let $\textbf{I}^s(\boldsymbol{\beta} )$  be the surface-brightness distribution
  of the source, then following the conservation of total surface brightness
  the observed surface-brightness distribution in the lens
  plane is given by
  \begin{equation}\label{[eq:I_theta]}
    \boldsymbol{I(\theta)} =  \textbf{I}^s \ [\boldsymbol{\beta}(\boldsymbol{\theta})].
  \end{equation}

  Now we expand the \textit{true angular position} of the source $\boldsymbol{\beta}$
  in terms of \textit{observed angular position} of the source $\boldsymbol{\theta}$
  using Taylor expansion around the central observed position $\boldsymbol{\theta_0}$
  we get

  \begin{eqnarray}\label{[eq:beta_taylor]}
    \boldsymbol{\beta}(\boldsymbol{\theta} ) = \boldsymbol{\beta}_0 + ( \boldsymbol{\theta} - \boldsymbol{\theta}_0 ) \frac{\partial\boldsymbol{\beta} }{\partial\boldsymbol{\theta}  }
  \end{eqnarray}

  Here, the term differential of $\boldsymbol{\beta}$ w.r.t. $\boldsymbol{\theta}$ is called \textit{distortion matrix}

  \begin{eqnarray}\label{[eq:dist_matrix]}
    \boxed{ \mathscr{A}(\boldsymbol{\theta} ) \equiv \frac{\partial \boldsymbol{\beta} }{ \partial \boldsymbol{\theta} }} \quad (\text{Distortion Matrix}).
  \end{eqnarray}

  Now we can write the observed surface brightness $\boldsymbol{I(\theta)}$ in terms of
  distortion matrix as
  \begin{eqnarray}\label{[eq:I_taylor]}
    \boxed{\boldsymbol{I}(\boldsymbol{\theta}) = \boldsymbol{I}^s[\beta_0 + \mathscr{A}(\boldsymbol{\theta}) (\theta - \theta_0 ) ]} \quad (\text{Observed Surface Brightness})
  \end{eqnarray}

  In terms of deflection potential $\psi$  \textit{Jacobian matrix} $\mathscr{A}$
  can be written as
  \begin{align}\label{[eq:jacobian]}
    \mathscr{A}(\boldsymbol{\theta} ) & = \frac{\partial \boldsymbol{\beta} }{ \partial \boldsymbol{\theta} } \\
      & = (\delta_{ij} - \frac{\partial^2 \psi(\boldsymbol{\theta})}{\partial \theta_i \partial \theta_j}) \nonumber \\
      & = {  \begin{pmatrix}
               1 - \psi{,11}  &-\psi_{,12}  \\
               -\psi{,21}  & 1 - \psi_{,22} \ .
            \end{pmatrix}
          }
  \end{align}

  From these deflection potential terms, we define shear components
  \begin{eqnarray}\label{[eq:shear_comps]}
    \gamma_1 &=& \frac{1}{2} (\psi_{,11} + \psi_{,22} ) \\
    \gamma_2 &=& \psi_{,12}
  \end{eqnarray}

  Here, the two components tensor $\gamma$ is called \textit{shear} and is
  given by
  \begin{eqnarray}\label{[eq:shear]}
    \boxed{\gamma \equiv \gamma_1 + i \gamma_2
           = \lvert\gamma\rvert e^{2i \phi}} \quad (\text{Shear})
  \end{eqnarray}

  Here, $\gamma_1$ and $\gamma_2$ are two components of shear as given in equation
  (\ref{[eq:shear_comps]}) and $\phi$ is the phase angle.

  The shear has two components $\gamma_1$ and $\gamma_1$ which can be expressed as
  \begin{equation}\label{[eq:shear]}
    \gamma \equiv \gamma_1 + i \gamma_2 = \lvert\gamma\rvert e^{2i \phi}.
  \end{equation}
  Also, in terms of deflection potential the shear components can be expressed as
  \begin{eqnarray}\label{[eq:shear_comp]}
    \gamma_1 &=& \frac{1}{2} (\psi_{,11} - \psi_{,22} ) \\
    \gamma_2 &=& \psi_{,12} \nonumber
  \end{eqnarray}

  Also, the convergence $\kappa$ is related to the deflection potential through
  Poisson equation
  \begin{eqnarray}\label{[eq:kappa_psi]}
    \nabla^2 \psi(\theta) = 2 \kappa(\theta).
  \end{eqnarray}

  In terms of matrix elements of deflection potential $\psi$ we can write $\kappa$ as
  \begin{eqnarray}\label{[eq:kappa_psi]}
    \boxed{\kappa = \frac{1}{2} (\psi_{,11} + \psi_{,22})} \quad (\text{Convergence})
  \end{eqnarray}

  Now we can write the distortion matrix $\mathscr{A}$ in terms of
  shear components $\gamma_1$ and $\gamma_2$ and convergence $\kappa$ as
  \begin{eqnarray}\label{[eq:A_theta]}
    \mathscr{A}(\boldsymbol{\theta}) =
    \begin{pmatrix}
      1 - \kappa - \gamma_1 & - \gamma_2  \\
      - \gamma_2 & 1 - \kappa + \gamma_1
    \end{pmatrix}
  \end{eqnarray}

  If we solve this matrix we get the eigenvalues
  \begin{eqnarray}\label{[eq:eigen]}
    eig(\mathscr{A}) = 1 - \kappa \pm \lvert g \rvert .
  \end{eqnarray}

  The equation \ref{[eq:I_taylor]} has the shape of an ellipse. This means that
  if a circular galaxy is lensed then the resulting observed image will be an
  ellipse. The semi-major and semi-minor axes of the ellipse are given by
  \begin{eqnarray}\label{[eq:semi_axes]}
    a &=& \frac{R}{1 - \kappa  - \lvert g \rvert } \\
    b &=& \frac{R}{1 - \kappa  + \lvert g \rvert }
  \end{eqnarray}

  Here, $R$ is the radius of circular source, $a$ is the semi-major axis, $b$ is semi-minor axis, and $g$ is the reduced shear
  defined as
  \begin{eqnarray}\label{[eq:g]}
    \boxed{g(\boldsymbol{\theta} ) \equiv \frac{\gamma(\boldsymbol{\theta} )}{1 - \kappa(\boldsymbol{\theta} )}} \quad (\text{Reduced Shear}).
  \end{eqnarray}

  The inverse of the determinant of the matrix $\mathscr{\alpha}$ gives the
  magnification tensor. Then we define magnification as
  \begin{eqnarray}\label{[eq:magnification]}
    \boxed{\mu(\boldsymbol{\theta}) \equiv \frac{1}{det \lvert \mathscr{A} \rvert }} \quad (\text{Magnification}).
  \end{eqnarray}


\documentclass[12pt]{cdblatex}
\usepackage{exercises}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% --------------------------------------------------------------------------------------------
\section*{Exercise 6.2 Evaluate on an expression (not a rule)}

\begin{cadabra}
   {\theta, \varphi}::Coordinate.
   {a,b,c,d,e,f,g,h#}::Indices(values={\theta, \varphi}, position=independent).

   \partial{#}::PartialDerivative.

   V  := { V_{\theta} = f(\theta,\varphi), V_{\varphi} = g(\theta,\varphi) }.  # cdb(ex-0602.100,V)
   dV := \partial_{b}{V_{a}} + \partial_{a}{V_{b}}.                            # cdb(ex-0602.101,dV)

   evaluate (dV, V)                                                            # cdb(ex-0602.102,dV)
\end{cadabra}

\begin{align*}
   V_{a} &= \Cdb{ex-0602.100}\\[10pt]
   & \Cdb{ex-0602.101}\\[10pt]
   & \Cdb{ex-0602.102}
\end{align*}

\end{document}