\section{Modular Arithmetic}

% From prime numbers to simple criptography

\frame{
{Part 4: Modular Arithmetic}

\tableofcontents[currentsection,hideallsubsections, firstsection=1, sections={1-5}]
}

% Modular arithmetic and congruence

\subsection{Congruence}
\begin{frame}{Modular Arithmetic and Congruence}

  Using \structure{modular arithmetic}, we can make the Turing Algorithm stronger.\bigskip

  Modular arithmetic is centred around the concept of \structure{{\bf Congruence}}:

  \begin{equation*}
    a\equiv b (\text{modulo } n) \iff n|(a-b)
  \end{equation*}\bigskip

  \structure{Examples:}
  \begin{itemize}
\item $30\equiv12 (\text{modulo } 9)$,\hspace{4.3cm} because $9|(30-12)$
\item $66666663 \equiv 788253 (\text{modulo } 10)$,\hspace{2cm} because $10|66666663 - 788253$
  \end{itemize}
\end{frame}

\begin{frame}{Modular Arithmetic and Congruence}{Remainder Theorem}

  One important implication of the definition of equivalence is the remainder theorem:
  \begin{equation*}
    a \equiv b (\text{mod } n) \iff \text{ rem}(a,n) = \text{ rem}(b,n)
  \end{equation*}

  \begin{proof}
    Proof by two way implication. Let the remainder of $a$ and $b$ be $r_{a,b}$:

    \begin{itemize}
    \item Let $a = q_an + r_{a,n}$, and $b = q_bn + r_{b,n}$
    \item (right side) {\bf if} $r_{a,n} = r_{b,n}$ then $a-b = (q_a-q_b)n \implies n|(a-b)$
    \item (left side) {\bf if} $n|(a-b)$ then $n|((q_a-q_b)n+(r_{a,n}-r_{b,n}))$
    \begin{itemize}
      \item but $0 \leq r_{*,n} < n$ so $r_{a,n}-r_{b,n}$ must be 0
    \end{itemize}
    \end{itemize}
  \end{proof}
\end{frame}

% Congruence is preserved by addition and multiplication
\begin{frame}{Modular Arithmetic and Congruence}{Consequences of the Remainder Theorem}

  Remainder Theorem:
  \begin{equation*}
    a \equiv b (\text{mod } n) \iff \text{ rem}(a,n) = \text{ rem}(b,n)
  \end{equation*}

  \begin{itemize}
  \item $a \equiv b$ (mod n) {\bf and} $b \equiv c$ (mod n) {\bf implies}
    $a \equiv c$ (mod n)
  \item $a \equiv $ rem(a,n) (mod n) \hspace{1cm} (\alert{important!})\bigskip
  \item $a \equiv b$ (mod n) {\bf implies} $a+c \equiv b+c$ (mod n)
  \item $a \equiv b$ (mod n) {\bf implies} $ac \equiv bc$ (mod n)
  \item $a \equiv b$ (mod n) {\bf and} $c\equiv d$ (mod n)\\
    \hspace{2cm}{\bf implies} $a+c \equiv b+d$ (mod n) {\bf and} $ac \equiv bd$ (mod n)
  \end{itemize}\bigskip

  The last three consequences show that we can freely use \structure{addition} and \structure{multiplication} in modular arithmetic. Try to prove some of these consequences!
\end{frame}

% Remainder Arithmetic
\begin{frame}{Using Modular Arithmetic}{General Principle of Modular Arithmetic}

  \begin{block}{}
    You can simplify modulo operations composed of additions and multiplications by replacing integer operands by their remaiders.
  \end{block}

  Example: What is $287^9 \equiv$ ? (mod 4)

  \begin{itemize}
    \item Simplify $287^9$ (mod 4) to $3^9$ (mod 4), becase $r_{287,4} = 3$
    \item $3^9 \to 3^8\times 3 \to 9^4 \times 3$
    \item Simplify $9^4 \times 3$ (mod 4) to $1^4 \times 3$, because $r_{9,4} = 1$
    \item $289^9 \equiv 1^4 \times 3 \equiv 3$ (mod 4)
  \end{itemize}\bigskip

  We calculated a large exponent without actually calculating the exponentiation!
\end{frame}

% Remainder Inverse
% Remainder inverse and relatively prime numbers

\subsection{Modular Inverses}

\begin{frame}{Modular Arithmetic and Division}

  We so that modular arithmetic works for addition and multiplication, so what about \structure{division}?\bigskip

  \begin{itemize}
    \item $8\times 2 \equiv 3\times 2$ (mod 10)
    \item $8\times \not{2} \equiv 3\times \not{2}$ (mod 10)
    \item $8 \equiv 3$ (mod 10) \hspace{2cm} \alert{\bf FALSE!}
  \end{itemize}\bigskip

  We cannot cancel out multiplications arbitrarily!
\end{frame}

\begin{frame}{Modular Arithmetic and Division}{Modular Inverses}

  When we cancel the multiplication of a real number, it is equivalent of multiplying it by its {\bf multiplicative inverse}:

  \begin{equation*}
    an = bn \to a\not{n} = b\not{n} \to a(n\times \frac{1}{n}) = b(n\times \frac{1}{n})
  \end{equation*}

  So a multiplicative inverse of $n$ is the number $k$ so that $n\times k =1$.\bigskip

  In modulo arithmetic, we have: $n\times k \equiv 1$ (mod m). But, it turns out that it is not possible to find a $k$ that solves this equation for every $n$ and $m$!
\end{frame}

\begin{frame}{Modular Inverses}{When does $n$ have an inverse, and how can we find it?}

  It turns out that $n$ only has a modular inverse (module $m$) if $gcd(n,m) = 1$.\bigskip

  It is relatively easy to find the modular inverse:
  \begin{itemize}
    \item First we calculate gcd(n,m) using the pulverizer, and obtaining $s$ and $t$ so that $sm+tn=1$
    \item The multiplicative inverse of $n$ modulo $m$ is given by: $r_{m,t}$
    \item So $n\times r_{m,t} \equiv 1$ (mod $m$)
  \end{itemize}
\end{frame}

\subsection{Turing Code 2.0}

\begin{frame}{Turing Code 2.0}
  Using Modular Arithmetic, we can solve some of the problems of the cryptography algorithm described in the last section:

  \begin{enumerate}
    \item The sender and the receiver agree on a large prime number $n$, which will be the module, as well as the secret key $k < n$
    \item {\bf Encription:} The message $m$ is a prime number ($0 < m < n$). The sender calculates the secret message $\hat{m}$ using modulo multiplication:
    \begin{equation*}
      \hat{m} = m\times k \text{ mod }n
    \end{equation*}
    \item {\bf Decription:} The receiver multiplies the encrypted message $\hat{m}$ by the inverse of the key: $k^{-1}$ (mod $n$):
    \begin{equation*}
      m = \hat{m}0\times k^{-1} \text{ mod }n
    \end{equation*}
  \end{enumerate}
  We cannot find $k$ anymore by having two encrypted messages, so is this algorithm safe?
\end{frame}

\begin{frame}{Turing Code 2.0}{Breaking the Turing Code with a Plaintext Attack}

  The new algorithm can be broken by what is called a \structure{{\bf Plaintext Attack}}.\bigskip

  Imagine that an attacker is able to acquire an encrypted message $\hat{m}$ and its respective clear message $m$. To break this code:\bigskip

  \begin{itemize}
    \item Calculate the multiplicative inverse of $m$ modulo $n$: $m^{-1}$, using the Pulverizer.
    \item Multiply the result by the encrypted message:
    \begin{itemize}
      \item $m^{-1}\times \hat{m} = m^{-1}\times (m \times k) = (m^{-1} \times m) \times k = 1\times k$ (mod $n$)
    \end{itemize}
    \item This allows us to recover the secret key $k$.
  \end{itemize}\bigskip

  So this algorithm is not so good after all! :-(
\end{frame}


\chapter{Massless Spin-1 Particles}
We approach the quantum description of photons via classical electromagnetism, since we know via the correspondence principle that Maxwell's equations describe their dynamics.

\section{Maxwell's equation and the classical potential}\label{sec:maxwell}
For fields $\vec{E}$ and $\vec{B}$ in a vacuum (i.e.~not in material), Maxwell's equations are:
\begin{enumerate}
\item $\div\vec{E} = \rho$, the charge density (Poisson's equation);
\item $\div\vec{B} = 0$, no magnetic monopoles;
\item $\curl\vec{E} = -\pdv{\vec{B}}{t}$, Faraday's law;
\item $\curl\vec{B} = \vec{j} + \pdv{\vec{E}}{t}$ where $\vec{j}$ is the charge current density (Amp{\'e}re's law).
\end{enumerate}

We define the vector potential such that
\begin{equation}\boxed{
\vec{B} = \curl \vec{A}
}\end{equation}
so Maxwell~II is automatically satisfied. Substituting into Maxwell~III,
\begin{align}
\curl\vec{E} &= -\pdv{t} \left( \curl \vec{A} \right) \nonumber \\
&= -\curl \pdv{\vec{A}}{t}
\end{align}
\begin{equation}
\Rightarrow \quad \curl \left( \vec{E} + \pdv{\vec{A}}{t} \right) = 0.
\end{equation}
The solution to this equation defines the scalar potential,
\begin{equation}
\vec{E} + \pdv{\vec{A}}{t} = -\grad \phi.
\end{equation}
Substituting into Maxwell~I,
\begin{align}
\rho &= -\laplacian{\phi} - \pdv{t} {\div\vec{A}} \label{eq:MaxwellDensity}
\end{align}
Now consider Maxwell~IV. The left-hand side is
\begin{align}
\curl \vec{B} &= \curl(\curl{\vec{A}}) \\
&= \grad(\div\vec{A}) - \laplacian{\vec{A}} \label{eq:Maxwell4LHS}
\end{align}
and the right-hand side is
\begin{align}
\vec{j} + \pdv{\vec{E}}{t} &= \vec{j} - \pdv{t}\left(\grad{\phi} + \pdv{\vec{A}}{t}\right) \nonumber \\
&= \vec{j} - \grad{\pdv{\phi}{t}} - \pdv[2]{\vec{A}}{t} \label{eq:Maxwell4RHS}.
\end{align}
Equating \eqref{eq:Maxwell4LHS} and \eqref{eq:Maxwell4RHS} gives an expression for the 3-current,
\begin{equation}
\vec{j} = \pdv[2]{\vec{A}}{t} - \laplacian{\vec{A}} + \grad(\pdv{\phi}{t} + \div\vec{A}).
\end{equation}
Also, \eqref{eq:MaxwellDensity} can be written
\begin{equation}
\rho = \pdv[2]{\phi}{t} - \laplacian{\phi} - \pdv{t}(\pdv{\phi}{t} + \div\vec{A})
\end{equation}
so we can write Maxwell's equations in covariant form:
\begin{equation}\boxed{
j^\mu = \partial^2 A^\mu - \partial^\mu \partial_\nu A^\nu \label{eq:MaxwellCovariant}
}\end{equation}
where the 4-current is $j^\mu = \mqty(\rho \\ \vec{j})$ and the 4-potential is $A^\mu = \mqty(\phi \\ \vec{A})$.

\section{Gauge transformations and the Lorenz condition}
Equation \eqref{eq:MaxwellCovariant} is invariant under gauge transformations that have the form
\begin{equation}
A^\mu \rightarrow A^\mu + \partial^\mu f
\end{equation}
for some scalar $f$; this is easily verified by direct substitution. For a global gauge transformation $f$ is constant but for local gauge transformations $f$ can be a function of the coordinates.

This degree of freedom may be frozen out by the choice of a particular `gauge'. The Lorenz\footnote{The Lorenz gauge is named after Ludwig Lorenz and is Lorentz invariant, named after the work of Hendrik Lorentz.} gauge is the most natural choice with the condition
\begin{equation}
\partial_\mu A^\mu = 0 \label{eq:LorentzGauge}
\end{equation}
such that all electrodynamics (Maxwell's equations) are simply described by
\begin{equation}\boxed{
\partial^2 A^\mu = j^\mu.
}\end{equation}

\subsection{Polarisation states of a free photon}
Since the 4-potential $A^\mu$ describes the photon, there are 4 possible orthogonal polarisations available to it. However, the Lorenz gauge removes one degree of freedom.

For a free photon, we have that the charge current $j^\mu = 0$. Therefore,
\begin{equation}
\partial^2 A^\mu = 0
\end{equation}
with polarised plane wave solutions $A^\mu = \epsilon^\mu_i \, e^{-ipx}$ where $\epsilon^\mu_i$ are the polarisation 4-vectors. Therefore, the Lorenz condition becomes
\begin{equation}
p_\mu \epsilon^\mu_i = 0.
\end{equation}
This has removed one degree of freedom from the available polarisation states.

Now consider the gauge transformation
\begin{equation}
A^\mu \rightarrow A^\mu + \partial^\mu\left( i\alpha e^{-ipx} \right).
\end{equation}
Then
\begin{equation}
\partial_\mu \partial^\mu \left( \alpha e^{-ipx} \right) = -i \alpha p^2 e^{-ipx} = 0
\end{equation}
since $p^2 = 0$ for a free photon (it's massless). Therefore the photon is unchanged by such a gauge transformation, which can be written
\begin{equation}
\epsilon_i^\mu \rightarrow \epsilon_i^\mu + \alpha p^\mu.
\end{equation}

We are therefore free to make the gauge choice $\epsilon_i^0 = \epsilon_i^3 = 0$ (with the spatial axes chosen such that $p^0 = p^3 = E$) which corresponds to $\alpha = -\epsilon_\mu^i p^\mu / \epsilon_\nu^i \epsilon_i^\nu$.

For a free photon with 4-momentum $p = (E,\,0,\,0,\,E)^T$, the surviving polarisation states are
\begin{equation*}
\epsilon_1^\mu = \mqty(0 \\ 1 \\ 0 \\ 0) \quad \text{and} \quad \epsilon_2^\mu = \mqty(0 \\ 0 \\ 1 \\ 0)
\end{equation*}
which can be combined to give circular polarisations
\begin{equation*}
\epsilon_R^\mu = \frac{1}{\sqrt{2}} (\epsilon_1^\mu + i\epsilon_2^\mu) \quad \text{and} \quad \epsilon_L^\mu = \frac{1}{\sqrt{2}} (\epsilon_1^\mu - i\epsilon_2^\mu)
\end{equation*}
which correspond to the $\pm 1$ helicity photon.

In summary, a free spin-1 particle loses one degree of freedom under gauge fixing in electrodynamics. A second gauge fixing removes another degree of freedom, but this relies on the fact that the particle is both massless and free. Therefore, the free photon has only two possible transverse polarisations. In contrast, a virtual photon may also have timelike or longitudinal polarisation.

\section{Virtual photons and the photon propagator}
We saw when discussing the electrodynamics of scalars that the potential \eqref{eq:potential}, $A^\mu = -g^{\mu\nu}j_\nu / q^2$, is a solution of Maxwell's equations in the Lorenz gauge, $\partial^2 A^\mu = j^\mu$. There, this was verified by substitution but now it will be derived via the propagator approach.

As a functional of the spatial propagator, $G(x^\prime; x)$, the potential is given by
\begin{equation}
A^\mu(x^\prime) = \int G(x^\prime; x) \, j^\mu(x) \, \dd[4]{x}.
\end{equation}
Applying the d'Alambertian differential operator, the left-hand side of the Lorenz condition may be expressed
\begin{align}
\partial^2 A^\mu(x^\prime) &= \int  \left[ \partial^2 G(x^\prime; x) \right] \, j^\mu(x) \, \dd[4]{x}
\end{align}
where we have used that $\partial_\mu j^\mu = 0$ since the 4-current should be conserved (not explicitly shown for spin-1). The right hand side is
\begin{equation}
j^\mu = \int \delta^{(4)}(x^\prime - x) \, j^\mu(x) \, \dd[4]{x}.
\end{equation}
Therefore, we have that
\begin{equation}
\partial^2 G(x^\prime; x) = \delta^{(4)}(x^\prime - x).
\end{equation}
Now Fourier transform into momentum-space using $q^\mu = -i\partial^\mu$,
\begin{equation}
\frac{1}{(2\pi)^4} \int (iq)^2 \, \mathcal{G}(q) \, e^{-iq(x^\prime - x)} \, \dd[4]{q}  = \frac{1}{(2\pi)^4} \int e^{-iq(x^\prime - x)} \, \dd[4]{q}
\end{equation}
from which we see that the momentum-space propagator is $\mathcal{G}(q) = -1/q^2$. Then the potential is
\begin{equation}
A^\mu(x) = \frac{-j^\mu(x)}{q^2} = \frac{-g^{\mu\nu} j_\nu(x)}{q^2}
\end{equation}
By convention, we actually define $\boxed{\mathcal{G}(q) = -i g^{\mu\nu}/q^2}$.

\subsection{Momentum-space propagators}
Generalising the above, the momentum-space propagator may be obtained by inverting the equation of motion for a free particle and multiplying by $-i$.

The Klein-Gordon equation gives $(\partial^2 + m^2)\psi = 0$ for free scalars. Therefore, the propagator is
\begin{align}
\frac{-i}{(iq)^2 + m^2} = \frac{i}{q^2-m^2}.
\end{align}

Now consider the Dirac equation for free spin-$\frac{1}{2}$ particles. From \eqref{eq:covDirac}, the equation of motion is
\begin{equation*}
(i \gamma^\mu \partial_\mu - m)\psi = 0.
\end{equation*}
Then, using that $i\partial_\mu = p_\mu$ (note the lower-index form), the propagator is
\begin{align}
\frac{-i}{\gamma^\mu q_\mu - m} &= \frac{i}{\gamma^\mu q_\mu - m} \nonumber \\
&= \frac{i\,(\fsl{q}+m)}{(\fsl{q}-m)(\fsl{q}-m)} \nonumber \\
&= \frac{i \, (\fsl{q}+m)}{q^2 - m^2} \nonumber \\
&= i \frac{\sum u_S \overline{u}_{S^\prime}}{q^2 - m^2}
\end{align}
where on the third line we have used that, from the properties of the $\gamma$-matrices, $\fsl{q}^2 = \gamma^\mu q_\mu \gamma^\nu q_\nu = q^2$ and the spinor completeness relation has been used to obtain the last line. The sum is over all possible spins, $S$ and $S^\prime$.

\section{Longitudinal and timelike virtual photons}
Since we cannot imply the Lorenz gauge condition, virtual photons have four available polarisation states: two transverse, longitudinal, and timelike.

\subsection{Coulomb's law}

The amplitude for photon exchange between two currents is given by
\begin{align}
A &= j_\mu^A \left( \frac{-ig^{\mu\nu}}{q^2} \right) j^\nu_B \nonumber \\
&= j_\mu^A \left( \frac{-i}{q^2} \right) j^\mu_B.
\end{align}
Now the momentum exchange is $q^\mu$. But we know that the four-current is conserved by QED and therefore $q_\mu j^\mu = 0$.

We are free to choose that the photon travels along the $z$- or $t$-axes, so $q_1 j^1 = q_2 j^2 = 0$. Then the continuity equation becomes
\begin{equation}
q_\mu j^\mu = q_0 j^0 - q_3 j^3 = 0.
\end{equation}
Substituting back into the amplitude,
\begin{align}
A &= \frac{-i}{q^2} \left[j_0^A j^0_B - j_3^A j^3_B\right] \nonumber \\
&= \frac{-i}{q^2} \left[j_0^A j^0_B - \frac{q_0^2}{q_3^2}(j_0^A j^0_B)\right] \nonumber \\
&= \frac{-i}{q_0^2 - q_3^2} \left[ \frac{q_3^2 - q_0^2}{q_3^2} j_0^A j^0_B \right] \nonumber \\
&= i \, \frac{j_0^A j^0_B}{q_3^2}.
\end{align}
This is Coulomb's law for electromagnetism in 3-momentum space.

\subsection{Completeness relation}
For free photons, the two allowed transverse polarisations give the completeness relation
\begin{equation}
\sum_{P, Q} \epsilon^P_\mu \epsilon_Q^\nu = \mqty(0,\,1,\,0,\,0) \mqty(0\\1\\0\\0) + \mqty(0,\,0,\,1,\,0) \mqty(0\\0\\1\\0) = \mqty(1&0\\0&1) = I_2.
\end{equation}

In the case of virtual photons with four allowed polarisations, this becomes
\begin{equation}
\sum_{P, Q} \epsilon^P_\mu \epsilon_Q^\nu = \mqty(\dmat[0]{-1,1,1,1}) = -g^{\mu\nu}
\end{equation}
which embeds the completeness relation for real photons.


\newcommand\chapternumber{3}
\input{../tex_header/header.tex}
\usepackage{enumerate}
\usepackage{float}

\begin{document}
Collaborators: Julia Zimmerman, Jordan Donovan, Sam Rosenblatt, Milo Trujillo, Phil Nguyen, Nicholas Vartanian,Connor Klopfer, Brett Meyer...
We might have discussed problems related to the assignments via Slack.
\section{Manual Gradient Descent}
\begin{align*}
    \frac{\partial \ell_i}{\partial \theta_j} = -2 x_{ij} (y_i - \sum_{j'=1}^{p} x_{ij'} \theta_{j'}) \\
\end{align*}
\begin{align*}
    \nabla_{\theta} \ell & = \sum_{i=1}^{N} \begin{bmatrix}
        \frac{\partial \ell_i}{\partial \theta_1} \\
        \frac{\partial \ell_i}{\partial \theta_2} \\
        \vdots                                    \\
        \frac{\partial \ell_i}{\partial \theta_p} \\
    \end{bmatrix} \\\\
                         & = -2 x^T (y-x\theta)
\end{align*}

I used the dataset this \href{https://www.kaggle.com/sulianova/cardiovascular-disease-dataset}{Cardiovascular Disease dataset}.

The are 70k records in this dataset.
The task is to predict whether a person has cardiovascular disease or not base on his age, height, etc.
I split the dataset into 80\% training and 20\% testing.

The source code is on \href{https://github.com/liusida/ds2/blob/main/assignment3/code/q1.py}{GitHub q1.py}.

I discovered that larger batch size and larger learning rate will result in faster convergence,
however, if learning rate is too large, the learning might diverge,
and if batch size and learning rate is too large will cause overflow.
So I normalized the input to reduce the possibility of overflow, and choose a batch size of 7k, and a small learning rate.
Now the learning process with 1k epochs takes 1.5 seconds. (If I use a batch size of 100, it will take 9.8 seconds.)

The final test accuracy is 0.643.
I also report the final confusion matrix:
Sensitivity 0.574, Specificity 0.712,
Precision 0.667, Negative Predictive Value 0.625.

\begin{figure}[h]
    \centering
    \includegraphics[width=.8\textwidth]{./q1.pdf}
    \caption{Learning curve and two test points.}
\end{figure}

Why would I choose to manually do gradient descent?
Because I want to learn how it works, and also, as David mentioned in class, sometimes we have constraints in practice, for example there's possibility that PyTorch is not supported.

\newpage
\section{2D Rosenbrock function}
\begin{figure}[h]
    \includegraphics[width=.99\textwidth]{./rosenbrock.pdf}
    \caption{Visualization of 2D Rosenbrock function. $x \in (-30,30)$, $y \in (-300,800)$}
\end{figure}

It is hard for SGD because the low area is not a dot but a long belt.
If the learning rate is large, it will diverge.
If the learning rate is low, it will be very slow in finding the minima on that flat belt.

Analytically, we need to solve the equations: $\frac{\partial f}{\partial x} = 0$ and $\frac{\partial f}{\partial y} = 0$.

\begin{align*}[left = \empheqlbrace]
    \frac{\partial}{\partial x} (1-x)^2+100(y-x^2)^2 & = 0 \\
    \frac{\partial}{\partial y} (1-x)^2+100(y-x^2)^2 & = 0 \\
\end{align*}

\begin{align*}[left = \empheqlbrace]
    400x^3-400xy+2x-2 & = 0  \\
    y                 & =x^2 \\
\end{align*}

\begin{align*}
    400x^3-400x^3+2x-2 & = 0 \\
    x                  & =1  \\
    y                  & =1  \\
\end{align*}

Using routine gradient descent, start from $x_0=0, y_0=0$, learning rate $\gamma=10^{-3}$, run for 10k steps.
The optimization successfully converged to the minimum.

This is because the initialization is quite close to the minimum,
and because it is smooth in $x \in (0,1)$ and $ y \in (0,1)$.
Figure \ref{fig:routine_gd} shows the trajectory of the optimization.

\begin{figure}[h]
    \includegraphics[width=.8\textwidth]{./routine_gd.png}
    \caption{Visualization of the optimization trajectory. $x \in (0,1)$, $y \in (0,1)$.}
    \label{fig:routine_gd}
\end{figure}

If we are not that lucky and use a bad initialization, such as $x_0 = 3$, $y_0 = -2$,
then the routine gradient descent will be harder.
The learning rate need to be smaller, otherwise it will diverge.
Then momentum is needed to speed the optimization up.
I set momentum to 0.99, learning rate $\gamma=10^{-5}$, and run for 10k steps.
The optimization successfully converged to the minimum.
Figure \ref{fig:gd_m} shows the trajectory.

\begin{figure}[h]
    \includegraphics[width=.8\textwidth]{./gd_m.png}
    \caption{Visualization of the optimization trajectory with momentum. $x \in (0,3)$, $y \in (-2,5)$.}
    \label{fig:gd_m}
\end{figure}

The source code is on \href{https://github.com/liusida/ds2/blob/main/assignment3/code/q2.py}{GitHub q2.py}.

\newpage
\section{Math notations and plate notations}
\subsection{Basic Parametric Density Estimation}
Mathematically, let $l=\text{loc}$, $s=\text{inv scale}$, we have:
\begin{align*}
     & p(l, s, d_i)                                                                                \\
     & \propto p(l)p(s)\prod_{i=1}^N p(d_i|l,s)                                                                                \\
     & = \text{Normal}(0,1) \text{Inv-Gamma}(3,2) \prod_{i=1}^N \text{Normal}(l, s)                                                                                \\
     & = \frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}l^2} \frac{2^3}{2!}s^{-3-1} e^{-\frac{2}{s}} \prod_{i=1}^N \frac{1}{s\sqrt{2\pi}} e^{-\frac{1}{2}(\frac{d_i-l}{s})^2} \\
     & = 4 (\frac{1}{\sqrt{2\pi}})^{N+1} s^{-4-N} e^{-\frac{l^2}{2} -\frac{2}{s} - \frac{1}{2}\sum_{i=1}^N (\frac{d_i-l}{s})^2}                          \\
\end{align*}
It doesn't seem to have conjugate prior to me.

Plate notation:
\begin{figure}[h]
    \centering
    \tikz{
        % nodes
        \node[obs] (data) {$d_i$};%
        \node[latent,above=of data] (loc) {loc};
        %    \node[latent,above=of loc,xshift=-1cm] (mu) {$\mu$}; %
        %    \node[latent,above=of loc,xshift=1cm] (sigma) {$\sigma$}; %
        \node[latent,left=of data] (scale) {scale};
        \node[latent,left=of scale] (inv_scale) {inv\_scale};
        %    \node[latent,left=of inv_scale,yshift=1cm] (alpha) {$\alpha$};
        %    \node[latent,left=of inv_scale,yshift=-1cm] (beta) {$\beta$};
        % plate
        \plate [inner sep=.3cm,xshift=.02cm,yshift=.2cm] {plate1} {(data)} {N}; %
        % edges
        %    \edge {sigma,mu} {loc}
        %    \edge {alpha,beta} {inv_scale}
        \edge {inv_scale} {scale}
        \edge {scale,loc} {data}
    }
    \caption{Plate notation}
\end{figure}

\newpage
\subsection{Continuous, nonstationary hidden Markov}

Mathematically, let $l=\text{loc}$, $s=e^{\text{log scale}}$, $o=\text{obs scale}$, we have:
\begin{align*}
     & p(l,s,o,x_{n,t},y_{n,t})                                                                                \\
     & =p(l)p(s)p(o)\prod_{n=1}^N p(x_{n,0}|l,s) \prod_{t=1}^T p(x_{n,t}|l,x_{n,t-1},s) p(y_{n,t}|x_{n,t}, o)                                                           \\
     & =\text{Normal}(0,1) \text{LogNormal}(0,1) \text{Gamma}(2,2) \prod_{n=1}^N \text{Normal}(l,s) \prod_{t=1}^T \text{Normal}(l+x_{n,t-1},s) \text{Normal}(x_{n,t},o) \\
     & =\frac{1}{\sqrt{2\pi}}e^{-\frac{l^2}{2}} \cdot
    \frac{1}{s\sqrt{2\pi}}e^{-\frac{\log(s)^2}{2}} \cdot
    \frac{1}{2!2^2}o^{2-1}e^{-\frac{o}{2}}  \cdot
    \prod_{n=1}^N
    \frac{1}{s\sqrt{2\pi}}e^{-\frac{(x_{n,0}-l)^2}{2s^2}}
    \prod_{t=1}^T
    \frac{1}{s\sqrt{2\pi}}e^{-\frac{(x_{n,t}-l-x_{n,t-1})^2}{2s^2}}
    \frac{1}{o\sqrt{2\pi}}e^{-\frac{(y_{n,t}-x_{n,t})^2}{2o^2}}                                                                                \\
\end{align*}

Plate notation (Reference: \href{https://davidrushingdewhurst.com/blog/2020-07-28keep-using-plate-notation.html}{this blog post}):
\begin{figure}[h]
    \centering
    \tikz{
        % \draw[lightgray,ultra thin] (-6,-6) grid (6,6);
        % nodes
        \node[obs] (y_t) {$y_{n,t}$};%
        \node[latent,left=of y_t] (x_t) {$x_{n,t}$};%
        \node[latent,above=of x_t] (x_0) {$x_{n,0}$};%
        \node[latent,above=of x_0,xshift=1.5cm] (loc) {loc};%
        \node[latent,right=of loc] (scale) {scale};%
        \node[latent,above=of scale] (log_scale) {log scale};%
        \node[latent,right=of y_t] (obs_scale) {obs scale};%
        % plate
        \plate [inner sep=.3cm,xshift=.02cm,yshift=.2cm] {episode} {(x_t)(y_t)} {T}; %
        \plate [inner sep=.3cm,xshift=.02cm,yshift=.2cm] {plate1} {(x_0)(episode)} {N}; %
        % edges
        \edge {obs_scale,x_t} {y_t}
        \edge {loc,scale} {x_t}
        \edge {loc,scale} {x_0}
        \edge {log_scale} {scale}
        % hmm...
        \draw[blue, thick, ->,shorten >=1pt] (x_0) to [out=-90,in=90] node[left,yshift=.2cm,xshift=-.2cm] {$t=1$} (x_t);
        \Cycle[blue, ->]{x_t}{180}{7mm}[{node[anchor=0,pos=0.5]{$t=2,...,T (t-1\rightarrow t)$}}]{7mm}
    }
    \caption{Plate notation}
\end{figure}

\newpage
\subsection{Switching model}
Mathematically, let $s'=\text{log scale}$, $s=e^{s'}$, $w=\text{switch}$, $m_1=\text{model1}$, $m_2=\text{model2}$, we have:
\begin{align*}
     & p(s,z_0,z_t,p_t,m_{1,t},m_{2,t},w_{t,n},y_{t,n},x_{t,n})                                                                                \\\\
     & =p(s)p(z_0|s) \times                                                                                \\
     & \prod_{t=1}^T p(z_t|z_{t-1}) p(p_t|z_t)
     \prod_{n=1}^N p(w_{t,n}|p_t)
     p(m_{1,t}|t) p(m_{2,t}|t) p(y_{t,n}|w_{t,n},m_{1,t},m_{2,t})
     p(x_{t,n},y_{t,n}) \\\\
     & =\text{LogNormal}(0,1)
    \text{Normal}(0,s) \times                                                                                \\
     & \prod_{t=1}^T \text{LogitNormal}(z_{t-1},1)
    \prod_{n=1}^N \text{Bernoulli}(p_t)
    p(m_{1,t}|t) p(m_{2,t}|t)
    p(y_{t,n}|w_{t,n},m_{1,t},m_{2,t})
    \text{Poisson}(y_{t,n})                                                                                \\\\
     & =\frac{1}{s\sqrt{2\pi}}e^{-\frac{\log(x)^2}{2}} \cdot
    \frac{1}{s\sqrt{2\pi}}e^{-\frac{z_0^2}{2s^2}} \cdot \\
    &\prod_{t=1}^T \frac{1}{\sqrt{2\pi}} e^{-\frac{\text{logit}(z_t-z_{t-1})^2}{2}} \cdot
    \prod_{n=1}^N p_t^{w_{t,n}} (1-p_t)^{1-w_{t,n}}
    p(m_{1,t}|t)^{w_{t,n}} p(m_{2,t}|t)^{1-w_{t,n}}
    \frac{y_{t,n}^{x_{t,n}} e^{-y_{t,n}}}{x_{t,n}!}
\end{align*}

Plate notation (next page):
\begin{figure}[!t]
    \centering
    \tikz{
        % \draw[lightgray,ultra thin] (-6,-6) grid (6,6);
        % nodes
        \node[obs] (x_t) {$x_{n,t}$};%
        \node[latent, left=of x_t] (y_t) {$y_{n,t}$};
        \node[latent, below=of y_t, xshift=-2cm] (model1) {model1$_t$};
        \node[latent, below=of y_t] (model2) {model2$_t$};
        \node[latent, left=of y_t, yshift=2cm] (switch) {switch$_{t,n}$};
        \node[latent, left=of switch] (p) {$p_t$};
        \node[latent, left=of p] (z) {$z_t$};
        \node[latent, above=of z] (z0) {$z_0$};
        \node[latent, above=of z0] (scale) {scale};
        \node[latent, left=of scale] (log_scale) {log scale};
        % plate
        \plate [inner sep=.3cm] {plate_switch} {(x_t)(y_t)(switch)} {N};
        \plate [inner sep=.3cm] {episode} {(plate_switch)(model1)(model2)(p)(z)} {T};
        % edges
        \edge {scale} {z0};
        \edge {log_scale} {scale};
        \edge {z} {p};
        \edge {p} {switch};
        \edge {switch, model1, model2} {y_t};
        \edge {y_t} {x_t};
        \draw[blue, thick, ->,shorten >=1pt] (z0) to [out=-90,in=90] node[left,yshift=.2cm,xshift=-.2cm] {$t=1$} (z);
        \Cycle[blue, ->]{z}{180}{7mm}[{node[anchor=0,pos=0.5]{$t=2,...,T (t-1\rightarrow t)$}}]{7mm}
    }
    \caption{Plate notation}
\end{figure}

\newpage
\section{PPL via Pyro}

The source code is on GitHub:
\begin{enumerate}
    \item \href{https://github.com/liusida/ds2/blob/main/assignment3/code/q4.1.py}{q4.1.py}.
    \item \href{https://github.com/liusida/ds2/blob/main/assignment3/code/q4.2.py}{q4.2.py}.
    \item \href{https://github.com/liusida/ds2/blob/main/assignment3/code/q4.3.py}{q4.3.py}.
\end{enumerate}

\newpage
\section{Project Thoughts}
\subsection{Bayesian Learning Signal}
I like Deep Learning.

In Supervised Learning, the learning signal comes from the difference between the truth label $y$ and the prediction $\hat{y}$.

To analyze its assumption, I perform a derivation using MAP:

Suppose we observe a image $\cal{I}$, we want to know a vector representation of the image $h$ that can tell us $y=\text{argmax}(h)$.

Assuming the prior to be $h_0 \sim \textbf{Uniform}(a,b)$, here $a$ and $b$ are constants.
The observation $\hat{h}$ is the representation produced by the network after seeing the image,
Assuming $\hat{h} \sim \textbf{Normal}(h, \sigma)$, here we don't know $\sigma$ yet.
This Normal distribution is saying the network can produce a prediction that is centered at $h$ but very noisy.
These two are quite strong assumptions.

Now I can use MAP to compute the point estimate for $h_0$ given $\hat{h}$:
\begin{align*}
    p(h_0|\hat{h}) &\propto p(\hat{h}|h_0) p(h_0)\\
    &= \textbf{Normal}(h,\sigma) \cdot \textbf{Uniform}(a,b) \\
    &= \frac{1}{\sigma \sqrt{2 \pi}} e^{-\frac{(h-\hat{h})^2}{2\sigma^2}} \cdot 1\\
\end{align*}

Solve $\frac{d}{dh}p(h_0|\hat{h})=0$ to find the maximum:
\begin{align*}
    \frac{h_0-\hat{h}}{\sigma^3} e^{-\cdots} &= 0\\
    h_0-\hat{h} = 0
\end{align*}

So we want to update our belief from $h_0$ to $h_1$: $h_1 = \hat{h}$.

This is how we find the update target, I think?
And we apply L2 loss to it, it'll just like a normal supervised learning signal.

Now if the agent has the chance to observe a very similar image $\cal{I}'$, here we don't use the i.i.d. assumption,
so a better prior should be $h_1$ after the update of the first step.
The problem here is that by using MAP, we only have a point estimator,
we don't have a distribution for $h_1$, how can I discover a good $h_1$?

Why I am doing this?

Imagine there is a robot with a camera on its face.
It will observe the world, frame by frame.
I assume the current frame of the world is very much similar to the previous frame.
If I can derive a learning signal from this assumption,
the agent will be able to learn without human providing labels.

Once there is a good way to produce the representation $h$ of the image,
we can use whatever existing supervised learning or reinforcement learning to utilize this representation,
and the learning will be faster, I hope.

I read \cite{wang_survey_2020}. It focus of this paper is to connect Bayesian Inference and Deep Learning at an application level, which is not my focus.
But I learned some basic concepts from it.
The so called Bayesian treatments might give the distribution of $h$.

\subsection{Thought two}

With a second thought, the previous idea turns out to be a Recurrent Neural Network.

The representation $h_t$ is just the hidden state.

\begin{figure}[h]
    \centering
    \includegraphics[width=.8\textwidth]{./rnn.pdf}
    \caption{A RNN with Bayesian updated hidden states.}
\end{figure}

The purpose of the observation is to build a good $h_t$,
so that if other modules want to use this representation,
it is readily accessible.

So the job for the network module $N$ is to update $h_t$ properly using Bayesian Theorem.

This is different from the Bayesian Neural Networks (BNN).
In BNN, people tries to make the weights and biases to be r.v.'s.

In the Long Short Term Memory (LSTM) model, there is a cell state, which is similar to this $h$.
Maybe replacing this gating and addition with a Bayesian update will give what I want.

\begin{figure}[h]
    \centering
    \includegraphics[width=.8\textwidth]{./LSTM3-C-line.png}
    \caption{Cell States are updated by Gating and Addition.}
\end{figure}

Later,
I realized it is related to the World Model  \cite{ha_world_2018},
where they use a variational autoencoder to get a latent $z$ (we call that $h$ earlier), and pass that into a RNN.
However, the $h$ produced by the RNN in the World Model is not parameters of distributions, maybe I can improve this.

\begin{figure}[h]
    \centering
    \includegraphics[width=.5\textwidth]{./world_model.png}
    \caption{In the World Model, the $z$ is produced by VAE, so $z$ is the parameters of some distributions.}
\end{figure}

\subsection{Automated Model Search?}

Another thought for the course project is from the idea of automated architecture search.
The architectures are analogous to the generative models in Bayesian Inference.
And people use Evolutionary Algorithms to search for better architectures rather than design it manually \cite{real_regularized_2019}.

So a typically case could be, the EA produces a population of DAGs,
and BI get all the parameters, and we have a set of final performances,
and EA produces a new population of DAGs by mutating the best DAGs,
and BI get all the parameters again, and we get another set of the performances.

But I am still not familiar with the whole Bayesian Inference process, so I'm not sure if the idea of automated design can be applied to Bayesian Inference.

\bibliographystyle{apalike}
\bibliography{b}

\end{document}


The scattering matrix, or S-matrix (also called the scattering function matrix, \cite{tsang2000scattering}), embeds the scattering behavior of an object as a mapping between incident and scattered plane-waves. It transforms between the incident/scattered directions as well as incident/scattered polarizations. This chapter gives the basic definition of the S-matrix, its relation to radar cross sections, routines for the S-matrix of simple objects, and the derivation of an object S-matrix under the Born approximation.

\section{Definition}

From \cite{tsang2000scattering}, let an incident wave be a plane wave with wave vector $\bb{k}_i$, the polarization of which is decomposed into two linearly independent vectors that are perpendicular to $\hat{k}_i$, as
\begin{equation}
\bb{E}_i = \left( E_{pi} \hat{p}_i+  E_{qi}\hat{q}_i\right) e^{i\bb{k}_i \cdot \br}
\end{equation}

\noindent where $\bb{r}$ is the position vector, ${\bb{k}}_i = k\hat{{k}}_i$ and $\hat{p}_i$, $\hat{q}_i$ and $\hat{{k}}_i$ form an orthonormal system.  The scattered field is then defined
\begin{equation}
\bb{E}_s = \left( E_{ps}\hat{p}_s +  E_{qs}\hat{q}_s\right) \dfrac{ e^{i k r}}{r}
\end{equation}

\noindent where the scattered field polarization vectors $\hat{p}_s$ and $\hat{q}_s$ form a right-handed orthonormal system with the scattered field direction $\hat{{k}}_s$. The scattered field components are linearly related to the incident components through the scattering matrix, or S-matrix,
\begin{equation}
\twobyone{E_{ps}(\hat{k}_s)}{E_{qs}(\hat{k}_s)} = \twobytwo{S_{pp}(\hat{k}_s,\hat{k}_i)   }{S_{pq}(\hat{k}_s,\hat{k}_i) }{S_{qp}(\hat{k}_s,\hat{k}_i) }{S_{qq}(\hat{k}_s,\hat{k}_i) }   \twobyone{E_{pi}(\hat{k}_i)}{E_{qi}(\hat{k}_i)}
\end{equation}

These definitions for the fields and S-matrix give them the following units: the incident field amplitudes $E_{pi}$ and $E_{qi}$ have units of electric field (V/m), the S-matrix elements have units of length (m), and the scattering field amplitudes $E_{ps}$ and $E_{qs}$ have units of (Vm/m = V) because the length dimension of the factor of $1/r$ needs to be included. This depends on convention, because some definitions of the S-matrix use $1/kr$ outside the S-matrix, which makes the S-matrix unitless.

We use the wave vector and polarization convention of \cite{tsang2000scattering}. The polarizations are taken as $\hat{p} = \hat{v}$ and $\hat{q} = \hat{h}$ with wave vector directions
\ea{\hat{k}_i &=&  \sin\theta_i \cos\phi_i \hat{x}  + \sin\theta_i \sin\phi_i \hat{y} + \cos\theta_i \hat{z} \label{khati} \\
\hat{v}_i &=& \cos\theta_i \cos\phi_i \hat{x} + \cos\theta_i\sin\phi_i \hat{y} - \sin\theta_i\hat{z} \\
\hat{h}_i &=& -\sin\phi_i\hat{x} + \cos\phi_i \hat{y} \\
\hat{k}_s &=& \sin\theta_s \cos\phi_s \hat{x}  + \sin\theta_s \sin\phi_s \hat{y} + \cos\theta_s \hat{z} \label{ss1} \\
\hat{v}_s &=& \cos\theta_s \cos\phi_s \hat{x} + \cos\theta_s\sin\phi_s \hat{y} - \sin\theta_s\hat{z} \\
\hat{h}_s &=& -\sin\phi_s\hat{x} + \cos\phi_s \hat{y} \label{hhats}}

\noindent where $\theta$, $\phi$ are spherical directions of the propagation waves. Here, the wave vectors are treated as radial vectors, and $\hat{v}$ and $\hat{h}$ are equivalent to the $\hat{\theta}$ and $\hat{\phi}$ spherical unit vectors. The polarizations for $\hat{z}$ propagating waves ($\theta = 0$ or $\theta = \pi$) are well-defined by using $\phi=0$ as a reference
 \ea{\hat{k}(0,0) &=&  \hat{z} \\
\hat{v}(0,0) &=& \hat{x} \\
\hat{h}(0,0) &=& \hat{y} \\
\hat{k}(\pi,0) &=&  -\hat{z} \\
\hat{v}(\pi,0) &=& -\hat{x}  \\
\hat{h}(\pi,0) &=& - \hat{y} }

These special cases can be used as definitions of the $\hat{\theta}$ and $\hat{\phi}$ unit vectors at the poles.

\section{Radar Cross Sections from S-matrix}

We list different radar cross sections and their relations to the S-matrix. Most of these can be found in \cite{tsang2000scattering}.

\paragraph{Radar Cross Section} The bistatic radar cross section is defined
\ea{\sigma_{pq}(\hat{k}_s,\hat{k}_i) &=& \lim_{r \rightarrow \infty} 4\pi r^2 \dfrac{\left\vert \hat{p} \cdot \bb{E}_s \right\vert^2}{\left\vert \hat{q} \cdot \bb{E}_i \right\vert^2} \\
\ &=& 4\pi \vert S_{pq} \vert^2 \label{rcsfromSpq} }

\noindent where $p$ and $q$ are any two orthogonal polarizations. The units of radar cross section are area or length-squared, showing again that $S_{pq}$ has units of length.

\paragraph{Scattering Cross Section} The total scattering cross section for incident direction $\hat{k}_i$ and arbitrary incident polarization $\hat{\beta}$, such that $\hat{\beta} \cdot \hat{k}_i = 0$, is computed as the integral over scattered field directions $\hat{k}_s$ as
\eq{\sigma_{s\beta}(\hat{k}_i) = \int \left( \vert S_{p\beta}(\hat{k}_s,\hat{k}_i) \vert^2 + \vert S_{q\beta}(\hat{k}_s,\hat{k}_i) \vert^2 \right) d\Omega_s }

\noindent where
\eq{
\twobyone{S_{p\beta}  }{S_{q\beta}  } =
\twobytwo{S_{pp}}{S_{pq}}{S_{qp}}{S_{qq}} \twobyone{\hat{p}\cdot \hat{\beta}}{\hat{q}\cdot \hat{\beta}} \label{projectedsmatrix}
}

\noindent and $\beta$ is the angle of the incident polarization relative to the two orthogonal polarizations of the S-matrix, which is
\eq{\hat{\beta} = \cos\beta \hat{p} + \sin\beta \hat{q}}

\paragraph{Extinction Cross Section} The extinction cross section (or total cross section) for a given incident direction is given by the optical theorem, \cite{zhang2019generalized}, as the imaginary part of the co-polarized scattered field in the forward direction
\eq{\sigma_{ext,\beta}(\hat{k}_i) = \dfrac{4\pi}{k}\textrm{Im}\left[S_{\beta\beta}(\hat{k}_i,\hat{k}_i)\right]}

\paragraph{Absorption Cross Section} The absorption cross section is defined as the power absorbed by the scattering object divided by the power scattered. It can be written in terms of the extinction and total cross sections as
\eq{\sigma_a = \sigma_{ext} - \sigma_s}

This subtraction is sometimes numerically inaccurate and so the absorption cross section can also be defined directly in terms of the internal field, \cite{yurkin2007discrete}.

\paragraph{Polarization and Orientation Averaged Scattering Cross Section}

The polarization and orientation averaged scattered cross section is given by the integral of the scattering cross section over all possible incident directions and polarizations including normalization:
%\eq{\left< \sigma \right> = \left<  \sigma_{s\beta}(\hat{k}_i)  \right>_{\beta,\hat{k}_i} = \dfrac{1}{2\pi} \int_0^{2\pi}\left(  \dfrac{1}{4\pi} \int \sigma_{s\beta}(\hat{k}_i) d\Omega_i \right) d\beta }
\eq{\left< \sigma \right> =  \dfrac{1}{2\pi} \int_0^{2\pi}\left(  \dfrac{1}{4\pi} \int \sigma_{s\beta}(\hat{k}_i) d\Omega_i \right) d\beta \label{polorientavescs} }

This requires projecting a given incident polarization $\beta$ onto $\hat{p}$ and $\hat{q}$, computing the projected S-matrix elements \eqref{projectedsmatrix}, computing the scattering cross section for all incident directions, then integrating over incident direction and incident angle $\beta$. Computing this does not require one to explicitly rotate or recompute the S-matrix once it is known.

\paragraph{Scattering Efficiencies} The quantities above can be reinterpreted as efficiencies by dividing by the cross sectional area of the target.  For example, the scattering efficiency for a given incident direction is given by the scattering cross section divided by cross sectional area of the target, $A$,

\eq{Q_{sca} = \dfrac{\sigma_{s}}{A} }

The same can be applied to extinction and absorption cross sections.

\section{S-matrix Rotation}

An S-matrix will often be computed in a reference frame aligned naturally with the geometry of the object. If an object is rotated relative to a global frame, then the incident and scattered directions and polarization in the global frame will appear to rotate in the frame of the object. It can be efficient to transform the global wave vectors and polarizations such that the S-matrix is evaluated in the frame of the object, rather than try to obtain the S-matrix of the rotated object in the global frame. Both \cite{tsang2000scattering} and \cite{van2011synthetic} give equations and procedures for rotating an object with cylindrical symmetry. We give a general procedure next.

%For the purposes of rotation, both the wave vectors and polarization vectors can be treated as Cartesian unit vectors with their tail at the origin. An extrinsic rotation is first applied to the incident and scattered directions to
%
%The polarization vectors are then decomposed into the local $\hat{v}$ and $\hat{h}$ polarizations of the object frame. Then the S-matrix is evaluated in its frame. In this procedure, there is no need to 'rotate back', because it is sufficient to have transformed both the incident and scattered quantities to the object frame.

Let an object rotation be described by ZXZ Euler angles $(\alpha,\beta,\gamma)$ with corresponding rotation matrix $\bb{R}$.  $\bb{R}$ describes a forward rotation, i.e., a point rides with the rotated frame. The inverse rotation is $\bb{R}^t$, where points are fixed in the global frame and we view them as though we ride with the rotating frame. The inverse rotation is first applied to the incident and scattered directions of the global frame so that they are viewed from the rotated object frame. Let these angles be $(\theta_{os},\phi_{os};\theta_{oi},\phi_{oi})$. The polarization vectors are then evaluated in the object frame at these points giving $(\hat{v}_{os},\hat{h}_{os};\hat{v}_{oi},\hat{h}_{oi})$. The object-frame polarization vectors now need to be viewed from the global frame. This is done by rotating the object-frame polarization vectors with an forward rotation, so that they appear as vectors in the global frame:
\eq{\hat{v}_{o}' = \bb{R}(\hat{v}_o)}

In matrix notation, S-matrix of the rotated object, as viewed in the global frame, is given by
\ea{
\twobytwo{S_{vv}(\theta_s,\phi_s;\theta_i,\phi_i)}{S_{vh}(\theta_s,\phi_s;\theta_i,\phi_i)}{S_{hv}(\theta_s,\phi_s;\theta_i,\phi_i)}{S_{hh}(\theta_s,\phi_s;\theta_i,\phi_i)}
&=&
\twobytwo{\hat{v}_s \cdot \hat{v}_{os}' }{\hat{v}_s \cdot \hat{h}_{os}'}{\hat{h}_s \cdot \hat{v}_{os}'}{\hat{h}_s \cdot \hat{h}_{os}'} \cdot \nonumber \\
\ & \ &
\twobytwo{S_{vv}(\theta_{os},\phi_{os};\theta_{oi},\phi_{oi})}{S_{vh}(\theta_{os},\phi_{os};\theta_{oi},\phi_{oi})}{S_{hv}(\theta_{os},\phi_{os};\theta_{oi},\phi_{oi})}{S_{hh}(\theta_{os},\phi_{os};\theta_{oi},\phi_{oi})} \cdot \nonumber \\
\ & \ &
\twobytwo{\hat{v}_{oi}' \cdot \hat{v}_{i} }{\hat{v}_{oi}' \cdot \hat{h}_{i} }{\hat{h}_{oi}' \cdot \hat{v}_{i} }{\hat{h}_{oi}' \cdot \hat{h}_{i} }
}

The inner matrix is the object S-matrix evaluated in its native frame using points rotated from the global frame. The outer two matrices decompose and project the polarization vectors between the two frames.

%\subsection{Euler Rotation}

%\subsection{Cylindrical Symmetry}

\section{Object S-matrix}

\subsection{Thin Circular Cylinder}

From \cite{sarabandi1990low, stiles1996scattering} the far field scattering solution for a thin dielectric cylinder with cross sectional area much smaller than a wavelength and oriented along the $z$ axis is given by
\eq{\bb{E}_s = -E_o\dfrac{e^{ikr}}{4\pi r} k^2 L A \left[ \hat{k}_s \times \hat{k}_s \times (\bb{P} \cdot \hat{e}_i ) \right] \sinc{(U)}}
\eq{U =  \dfrac{kL}{2} \left(\cos\theta_s - \cos\theta_i\right) }

\noindent where $L$ is the length, $A$ is the cross sectional area, $\hat{e}_i$ is the incident polarization, $k$ is the background wavenumber, $\theta_i$ and $\theta_s$ are the incident and scattered angles from the $z$ axis, and $\bb{P}$ is the polarization tensor. Also, $\textrm{sinc}(x) = \sin(x)/x$. The polarization tensor for a homogenous dielectric cylinder with circular cross section is diagonal and given by
\ea{P_{xx} &=&  2 \dfrac{\epsilon_r - 1}{\epsilon_r + 1} \\
P_{yy} &=& 2 \dfrac{\epsilon_r - 1}{\epsilon_r + 1}\\
P_{zz} &=& \epsilon_r - 1}

\noindent where $\epsilon_r$ is the relative permittivity. For a cylinder with radius $a$, the $\hat{v}$ and $\hat{h}$ S-matrix elements are
\ea{\twobytwo{S_{vv}}{S_{vh}}{S_{hv}}{S_{hh}} &=& \dfrac{k^2 L  a^2 }{4} \sinc{(U)} \bb{C} }

%\ea{\twobytwo{S_{vv}}{S_{vh}}{S_{hv}}{S_{hh}} &=& -\dfrac{k^2 L  a^2 }{4} \sinc{(U)} \onebytwo{\hat{v}_s}{\hat{h}_s} \cdot \twobyone{ \hat{k}_s \times \hat{k}_s \times \left(\bb{P} \cdot \hat{v}_i \right)}{ \hat{k}_s \times \hat{k}_s \times \left(\bb{P} \cdot \hat{h}_i \right) } \\
%\ &=&  \dfrac{k^2 L  a^2 }{4} \sinc{(U)} \bb{C} }

\eq{\bb{C}  = \twobytwo
{(\hat{v}_s)\cdot(\hat{k}_s \times \hat{k}_s \times \left(\bb{P} \cdot \hat{v}_i \right))}
{(\hat{v}_s)\cdot\left(\hat{k}_s \times \hat{k}_s \times \left(\bb{P} \cdot \hat{h}_i \right) \right)}
{(\hat{h}_s)\cdot(\hat{k}_s \times \hat{k}_s \times \left(\bb{P} \cdot \hat{v}_i \right))}
{(\hat{h}_s)\cdot\left(\hat{k}_s \times \hat{k}_s \times \left(\bb{P} \cdot \hat{h}_i \right) \right)} \label{matrixCforS}}

Using \eqref{khati}-\eqref{hhats} and simplifying the elements  of $\bb{C}$ are
\ea{C_{vv} &=&  P_{zz}\sin\theta_s\sin\theta_i + P_{xx}\cos\theta_s\cos\phi_s\cos\theta_i\cos\phi_i + P_{yy}\cos\theta_s\cos\theta_i\sin\phi_s\sin\phi_i \\
C_{vh} &=& \cos\theta_s(P_{yy}\sin\phi_s\cos\phi_i - P_{xx}\cos\phi_s\sin\phi_i ) \\
C_{hv} &=& \cos\theta_i(P_{yy}\cos\phi_s\sin\phi_i - P_{xx}\sin\phi_s\cos\phi_i) \\
C_{hh} &=&  P_{yy}\cos\phi_s\cos\phi_i + P_{xx}\sin\phi_s\sin\phi_i}

Applying symmetry $P_{xx} = P_{yy}$ and taking $\phi_i = 0$, these can be simplified to just
\ea{C_{vv} &=&  P_{zz}\sin\theta_s\sin\theta_i + P_{xx}\cos\theta_s\cos\phi_s\cos\theta_i \\
C_{vh} &=& P_{xx} \cos\theta_s\sin\phi_s\\
C_{hv} &=& -P_{xx} \cos\theta_i \sin\phi_s\\
C_{hh} &=&  P_{xx}\cos\phi_s}

This solution is valid for edge-on incidence, and the $hh$ and $vv$ fields become equal, which we expect.  This solution is valid for $\vert n \vert ka \ll 1$ and $a \ll L$, where $n = \sqrt{\epsilon_r}$, \cite{schiffer1979light}.

The routine \texttt{smatrix\char`_thin\char`_circular\char`_cylinder} returns the S-matrix of a thin circular cylinder. It takes the parameters of a single cylinder and computes the four block S-matrices for any number of incident and scattered directions. The block matrices have scattered directions in rows and incident directions in columns. The polarizations $\hat{v}$ and $\hat{h}$ are treated as spherical $\hat{\theta}$ and $\hat{\phi}$ in the forward scattering convention.

{\footnotesize
\VerbatimInput{\code/Smatrix/smatrix_thin_circular_cylinder.m}
}

\clearpage
\newpage

\subsection{Thin Circular Disk}

Scattering from both needles and disks is derived in \cite{schiffer1979light} and the results for needles match those in the previous section. The far field scattered field for a thin disk is
\ea{\bb{E}_s &=& -E_o\dfrac{e^{ikr}}{2\pi r} k^2 L A \left[ \hat{k}_s \times \hat{k}_s \times (\bb{P} \cdot \hat{e}_i ) \right] \textrm{jinc}(U)\\
U &=&  kL \Omega  \\
\Omega^2 &=& (\sin\theta_i \cos\phi_i- \sin\theta_s\cos\phi_s)^2 + (\sin\theta_i \sin\phi_i- \sin\theta_s\sin\phi_s )^2  }

\noindent where $L$ is the disk thickness, $A$ is cross sectional area of the disk, and $\textrm{jinc}(x) = J_1(x)/x$ where $J_1(x)$ is the Bessel function of degree 1. The polarization tensor of the circular disk is
\ea{P_{xx} &=& \epsilon_r - 1 \\
P_{yy} &=& \epsilon_r - 1 \\
P_{zz} &=& \dfrac{\epsilon_r - 1}{\epsilon_r }}

%(In comparing \cite{schiffer1979light} to \cite{sarabandi1990low, stiles1996scattering}, the the factor of 2 between the cylinder and disk scattered field appears correct.)% For the disk, we take the small argument limit of the Bessel function (small radius) which kicks out a factor of 2, while for the cylinder, we take the small argument limit of the sinc (small length) which is equal to 1. The factors in the polarization tensor are otherwise the same (going from \cite{schiffer1979light} to \cite{sarabandi1990low, stiles1996scattering} the factor of $(\epsilon_r-1)$ is brought into $\bb{P}$ and we swap the $x$ and $z$ tensor components).

Then the S-matrix elements are
\eq{\twobytwo{S_{vv}}{S_{vh}}{S_{hv}}{S_{hh}} = \dfrac{k^2 L  a^2 }{2} \dfrac{J_1\left(U\right)}{ U}\bb{C} }

\noindent where $\bb{C}$ is given by \eqref{matrixCforS}. In comparing \cite{schiffer1979light} to \cite{sarabandi1990low, stiles1996scattering}, the factor of 2 appears correct between the cylinder and disk scattered field. This solution is valid for $\vert n \vert kL \ll 1$ and $L \ll a$, where $n = \sqrt{\epsilon_r}$, \cite{schiffer1979light}.

The routine \texttt{smatrix\char`_thin\char`_circular\char`_disk} returns the S-matrix of a thin circular disk. It takes the parameters of a single disk and computes the four block S-matrices for any number of incident and scattered directions. The block matrices have scattered directions in rows and incident directions in columns. The polarizations $\hat{v}$ and $\hat{h}$ are treated as spherical $\hat{\theta}$ and $\hat{\phi}$ in the forward scattering convention.

{\footnotesize
\VerbatimInput{\code/Smatrix/smatrix_thin_circular_disk.m}
}

\section{S-matrix Under the Born Approximation}
\label{smatrixborn}

The far-field Born approximation is explained in Section \ref{bornapprox}. Because the approximation is written in terms of plane waves, it can be cast as an S-matrix.  Recalling \eqref{baesca}, using $\overline{\bb{I}}  - \hat{r}\hat{r} = \hat{\theta}_s\hat{\theta}_s + \hat{\phi}_s\hat{\phi}_s$, and equating the spherical unit vectors with $\hat{v}$ and $\hat{h}$, we obtain the S-matrix
\eq{\twobytwo{S_{vv}}{S_{vh}}{S_{hv}}{S_{hh}} = \dfrac{1}{4\pi}  \twobytwo{\hat{v}_{s} \cdot \hat{v}_{i} }{\hat{v}_{s}  \cdot \hat{h}_{i} }{\hat{h}_{s} \cdot \hat{v}_{i} }{\hat{h}_{s} \cdot \hat{h}_{i}  }  \int O(\br) e^{i (\bb{k}_i-\bb{k}_s) \cdot \br}  dV \label{eqbasmat1} }

%\eq{\bb{E}_{sca}(\br)=  \dfrac{e^{ikr}}{4\pi r} \left[ \hat{\theta}_s\hat{\theta}_s + \hat{\phi}_s\hat{\phi}_s \right] \cdot  \bb{E}_i  \int O(\br') \exp(i (\bb{k}_i-\bb{k}_s) \cdot \br')  dV'  }

\noindent where the object function is given by \eqref{objectfunction} and the wave vectors and polarizations are given by \eqref{khati}-\eqref{hhats}. This shows again that under the Born approximation, the object does not influence the polarization. The depolarization is merely the projection between incident and scattered polarizations.

When the object is homogenous \eqref{eqbasmat1} becomes
\eq{\twobytwo{S_{vv}}{S_{vh}}{S_{hv}}{S_{hh}} = \dfrac{1 }{4\pi}\twobytwo{\hat{v}_{s} \cdot \hat{v}_{i} }{\hat{v}_{s}  \cdot \hat{h}_{i} }{\hat{h}_{s} \cdot \hat{v}_{i} }{\hat{h}_{s} \cdot \hat{h}_{i}  } k^2(\epsilon_r - 1) I(\theta_{s},\phi_{s};\theta_{i},\phi_{i})   }
\eq{I(\theta_{s},\phi_{s};\theta_{i},\phi_{i})   =  \int e^{i (\bb{k}_i-\bb{k}_s) \cdot \br}  dV \label{volphaseint}}

\noindent where $k$ is the wavenumber of the background medium which is also used for the wave vectors, $\epsilon_r$ is complex relative permittivity of the object, and $I$ is the volume phase integral. The volume phase integral is the 3D Fourier transform over the domain of the object.

\clearpage

\section{Volume Phase Integral}
\label{sec:volumephaseint}
The volume phase integral occurs in the S-matrix formulation of the  Born approximation for homogeneous dielectric objects, Section \ref{smatrixborn}.  The volume phase integral for several simple objects is derived below and summarized in Table \ref{tablevolphaseint}. These are just the 3D Fourier transform over the object domain. The volume phase integral can be expressed as a product of the object volume and a function that has maximum value of one that depends on the incident/scattered directions. To facilitate derivations, \eqref{volphaseint} is written in terms of the wave vector difference, $\bb{K}$, as
\eq{I(\theta_{s},\phi_{s};\theta_{i},\phi_{i})   =  \int e^{i \bb{K} \cdot \br}  dV \label{volphaseint2}}
\noindent where
\ea{\bb{K} &=& \bb{k}_i-\bb{k}_s \label{Kvector}  = k(\hat{k}_i-\hat{k}_s) \\
 \hat{k}_i &=& \sin\theta_i \cos\phi_i \hat{x}  + \sin\theta_i \sin\phi_i \hat{y} + \cos\theta_i \hat{z} \\
\hat{k}_r &=& \sin\theta_r \cos\phi_r \hat{x}  + \sin\theta_r \sin\phi_r \hat{y} + \cos\theta_r \hat{z} \\
\bb{r}  &=& x \hat{x} + y \hat{y} + z \hat{z} }
\vspace{-7mm}

\begin{table}[H]
\caption{Table of Volume Phase Integrals}
\vspace{-6mm}
\begin{center}
\begin{tabular}{|p{1.4cm}|p{3cm}|c|c|p{2.7cm}|}
\hhline{|=====|}
%\multicolumn{5}{|c|}{ \ } \\
\multicolumn{5}{|c|}{ $\displaystyle I(\theta_{s},\phi_{s};\theta_{i},\phi_{i})  =  \int e^{i \bb{K} \cdot \br}  dV = V f(\cdot) $} \\
\multicolumn{5}{|c|}{$\displaystyle \bb{K} = \bb{k}_i-\bb{k}_s$} \\
%\multicolumn{5}{|c|}{ \ } \\
\hhline{|=====|}
Object & Geometry & $V$ & $f(\cdot)$ & Notes \\
\hline

Cuboid & \parbox[c]{1em}{\includegraphics[width=1.3in]{Smatrix/Figures/Cuboid}}  & $L_xL_yL_z$ & $ \textrm{sinc}\left(\dfrac{L_x K_x}{2}\right)\textrm{sinc}\left(\dfrac{L_y K_y}{2}\right) \textrm{sinc}\left(\dfrac{L_z K_z}{2}\right)$  & $\textrm{sinc}(x) = \dfrac{\sin(x)}{x}$ \\   \hline

Circular Cylinder & \quad \parbox[c]{1em}{\includegraphics[width=0.9in]{Smatrix/Figures/Cylinder}}  & $\pi a^2 L_z$ & $ 2 \dfrac{ J_1(K_{\rho} a)}{ K_{\rho} a}  \textrm{sinc}\left(\dfrac{L_z K_z}{2}\right)$  & $K_{\rho} = \sqrt{K_x^2 + K_y^2} $ \\   \hline

Sphere & \parbox[c]{1em}{\includegraphics[width=1.2in]{Smatrix/Figures/Sphere}}  & $\dfrac{4}{3}\pi a^3$ & $ \dfrac{3(\sin(K a) - K a \cos(K a))}{(Ka)^3} $  & $K = \vert \bb{K} \vert $ \\   \hline

Ellipsoid & \parbox[c]{1em}{\includegraphics[width=1.3in]{Smatrix/Figures/Ellipsoid}}  & $\dfrac{4}{3}\pi abc$ & $ \dfrac{3(\sin(K') - K' \cos(K'))}{K'^3}   $  & $\begin{array}{c} K' = \vert \bb{K'} \vert \\ (K_x',K_y',K_z') = \\ (a K_x, b K_y, c K_z)\end{array}$ \\   \hline
\end{tabular}
\end{center}
\label{tablevolphaseint}
\end{table}%

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Cuboid}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

Let a cuboid volume be centered at the origin with side lengths $L_x$, $L_y$, $L_z$ and let it be aligned with the Cartesian axes. In Cartesian coordinates, the volume phase integral, \eqref{volphaseint2}, becomes
\ea{
I &=& \int_{-L_x/2}^{L_x/2}  \int_{-L_y/2}^{L_y/2}  \int_{-L_z/2}^{L_z/2}  e^{i(K_x x + K_y y + K_z z)} dx dy dz }
%\ &=& \int_{-L_x/2}^{L_x/2} e^{i K_x x} dx  \int_{-L_y/2}^{L_y/2} e^{i K_y y} dy  \int_{-L_z/2}^{L_z/2}  e^{i K_z z} dz \\

The integral separates, then using the fact that
\eq{\int_{-a/2}^{a/2} e^{i b z} dz = \dfrac{2}{b} \sin\left(\dfrac{ab}{2}\right) \label{sincinz}}

\noindent and multiplying top and bottom by $L_xL_yL_z$ to convert the sines to sinc functions the volume phase intergral over a cuboid is
%\ea{I &=& \dfrac{4}{K_x K_y} \sin\left(\dfrac{L_x K_x}{2}\right)\sin\left(\dfrac{L_y K_y}{2}\right) \\
%\ & = & L_x L_y \textrm{sinc}\left(\dfrac{L_x K_x}{2}\right)\textrm{sinc}\left(\dfrac{L_y K_y}{2}\right) }

\eq{I  = L_x L_y L_z \textrm{sinc}\left(\dfrac{L_x K_x}{2}\right)\textrm{sinc}\left(\dfrac{L_y K_y}{2}\right) \textrm{sinc}\left(\dfrac{L_z K_z}{2}\right)}

\noindent where $\textrm{sinc} = \sin(x)/x$. This is equal to the volume of the cuboid times a product of directionally dependent $\textrm{sinc}$ functions.

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Circular Cylinder}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

Let a circular cylinder be centered at the origin and aligned with the $z$ axis with radius $a$ and length $L_z$. The volume phase integral, \eqref{volphaseint2}, in cylindrical coordinates is written
\ea{I  &=&  \int_{-L_z/2}^{L_z/2}  \int_0^{2\pi} \int_0^{a} e^{i (K_x \rho \cos\phi + K_y \rho \sin\phi + K_z z)} \rho d\rho d\phi dz }

\noindent where we have used the position vector $\br = \rho \cos\phi \hat{x} + \rho \sin\phi \hat{y} + \hat{z}$. The integral separates as
\ea{I  &=& \int_0^{2\pi} \int_0^{a} e^{i (K_x \rho \cos\phi + K_y \rho \sin\phi)} \rho d\rho d\phi   \int_{-L_z/2}^{L_z/2} e^{i K_z z} dz }

The last integral is a sinc function in $z$ given by \eqref{sincinz}. Using the identity
\eq{\int_0^{2\pi} e^{ u \cos t +v \sin t} d t = 2 \pi I_0 \left( \sqrt{u^2 + v^2} \right)}

\noindent the integral over $\phi$ evaluates to
\eq{\int_0^{2\pi}  e^{i K_x \rho \cos\phi + K_y \rho \sin\phi} d\phi = 2\pi I_0 \left( i K_{\rho}  \rho \right)}

\noindent where $K_{\rho} = \sqrt{K_x^2 + K_y^2}$. The quantity $K_{\rho}$ is the component of the wave vector difference in the X-Y plane. Last, the integral in $\rho$ is evaluated using
\eq{\int_0^a I_0(i c x) x dx = \dfrac{a J_1(a c)}{c}}

Putting these together, the volume phase integral over a circular cylinder is
\ea{I  &=&   2\pi \dfrac{a J_1(K_{\rho} a)}{ K_{\rho} }  L_z \textrm{sinc}\left(\dfrac{L_z K_z}{2}\right)  }

Using the volume of the cylinder, $V = \pi a^2 L_z$, this can be written
\ea{I  &=&  V   2 \dfrac{ J_1(K_{\rho} a)}{ K_{\rho} a}  \textrm{sinc}\left(\dfrac{L_z K_z}{2}\right)  }

Similar to the other shapes, this is equal to the volume of the cylinder multiplied by a directionally dependent term that has maximum value of 1.

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Sphere}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\label{sectionvolintsphere}
Let a spherical volume with radius $a$ be centered at the origin. The volume phase integral, \eqref{volphaseint2}, can be written in spherical coordinates as
\vspace{-2mm}
\eq{ I  =  \int_0^{2\pi} \int_0^{\pi} \int_0^a e^{i \bb{K} \cdot \br}  r^2 \sin\theta dr d\theta d\phi }

From symmetry, the dot product is evaluated relative to a common fixed axis such that $\bb{K} \cdot \br = K r \cos t$  where $t$ is the angle between $\bb{K}$ and $\br$. Using this, the integral becomes
\eq{ I  =  \int_0^{2\pi} \int_0^{\pi} \int_0^a e^{i K r \cos t}  r^2 \sin t dr dt d\phi \label{eqtemp21}}
\vspace{-2mm}

Next we use the identity
\eq{j_n(z) = \dfrac{(-i)^n}{2} \int_0^{\pi} e^{i z \cos u} P_n(\cos u) \sin u du}

\noindent where $j_n(z)$ is the spherical Bessel function and $P_n(\cos u)$ is the Legendre polynomial. Applying this and integrating in $\phi$ \eqref{eqtemp21} becomes
\eq{ I  =  4\pi   \int_0^a j_0(K r) r^2 dr   \label{eqtmp22}}

Note that $j_0(x) = \sin(x)/x $.  This integral is given generally as
\eq{\int_0^a j_0 (c x) x^2 dx = \dfrac{\sin(a c) - a c \cos(a c)}{c^3}}

The volume phase integral over a sphere is given by
\eq{ I  =  4\pi \dfrac{\sin(K a) - K a \cos(K a)}{K^3}  \label{volphasphere1}   }

\noindent where $K$ is the magnitude of \eqref{Kvector}.  Using the volume of the sphere, $V = 4/3 \pi a^3$, this can be written
\ea{ I  &=&  V \dfrac{3(\sin(K a) - K a \cos(K a))}{(Ka)^3}}

%
%
%The volume phase integral over a sphere is given by
%\eq{ I  =  4\pi \dfrac{\sin(K a) - K a \cos(K a)}{K^3}  \label{volphasphere1}   }
%
%\noindent where $K$ is the magnitude of \eqref{Kvector}.  This can be written in terms of the volume of the sphere $V = 4/3 \pi a^3$ as
%\ea{ I  &=&  V \dfrac{3(\sin(K a) - K a \cos(K a))}{(Ka)^3}}

The multiplying function acts like a directionally dependent weighting function that has a maximum value of 1 when $K a \rightarrow 0$.

\addtocontents{toc}{\protect\setcounter{tocdepth}{1}}
\subsection{Ellipsoid}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

Let an ellipsoid be centered on the origin with axes $(a, b, c)$ along XYZ, respectively. We define a new position vector, $\br'$, in spherical coordinates using the change of variables
\ea{\br' &=& r a \sin\theta\cos\phi \hat{x} + r b \sin\theta\sin\phi \hat{y} +  r c\cos\theta \hat{z} \\
dV &=& abc r^2 \sin\theta dr d\theta d\phi}

The volume phase integral, \eqref{volphaseint2}, can then be written as an integral over the unit sphere
\eq{ I  =  abc \int_0^{2\pi} \int_0^{\pi} \int_0^1 e^{i \bb{K} \cdot \br'}  r^2 \sin\theta dr d\theta d\phi }

Next, transfer the coefficients $(a,b,c)$ that are in $\br'$ to a new wave vector difference, $\bb{K}'$, so that, $(K_x',K_y',K_z') = (a K_x, b K_y, c K_z)$. The position vector returns to its unstretched form and we have
\eq{ I  =  abc \int_0^{2\pi} \int_0^{\pi} \int_0^1 e^{i \bb{K}' \cdot \br}  r^2 \sin\theta dr d\theta d\phi }

Using the results of Section \ref{sectionvolintsphere}, \eqref{volphasphere1}, the volume phase integral over an ellipsoid is
\eq{ I  =  4\pi abc \dfrac{\sin(K') - K' \cos(K')}{K'^3}     }

\noindent where $K'$ is the magnitude of $\bb{K}'$.  The volume of an ellipsoid is $V = 4/3\pi abc$, and this can be written
\eq{ I  =  V \dfrac{3(\sin(K') - K' \cos(K'))}{K'^3}     }

\clearpage

\clearpage
\newpage


\section{Non-parametric DF Estimation}\label{S:NonparamDFEstimation}
So far, we have been interested in some estimation problems involved in parametric experiments.  In parametric experiments, the parameter space $\BB{\Theta}$ can have many dimensions, but these are finite.  For example, in the $n$ IID $\bernoulli(\theta^*)$ and the $n$ IID $\exponential(\lambda^*)$ experiments:
\begin{eqnarray*}
X_1,\ldots,X_n \overset{IID}{\sim} \bernoulli(\theta^*), & \qquad & \theta^* \in \BB{\Theta}=[0,1] \subset \Rz^1 \enspace , \\
X_1,\ldots,X_n \overset{IID}{\sim} \exponential(\lambda^*), & \qquad & \lambda^* \in \BB{\Lambda}=(0,\infty) \subset \Rz^1 \enspace ,
\end{eqnarray*}
the parameter spaces $\BB{\Theta}$ and $\BB{\Lambda}$ are of dimension $1$.  Similarly, in the $n$ IID $\normal(\mu,{\sigma}^2)$ and the $n$ IID $\lognormal(\lambda,\zeta)$, experiments:
\begin{eqnarray*}
X_1,\ldots,X_n \overset{IID}{\sim} \normal(\mu,\sigma^2), &\qquad & (\mu,\sigma^2) \in \BB{\Theta}=(-\infty,+\infty) \times (0,+\infty) \subset \Rz^2 \\
X_1,\ldots,X_n \overset{IID}{\sim} \lognormal(\lambda,\zeta), &\qquad & (\lambda,\zeta) \in \BB{\Theta}=(0,+\infty) \times (0,+\infty) \subset \Rz^2
\end{eqnarray*}
the parameter space is of dimension $2$.

An experiment with an infinite dimensional parameter space $\BB{\Theta}$ is said to be {\bf non-parametric} .  Next we consider a non-parametric experiment in which $n$ IID samples are drawn according to some fixed and possibly unknown DF $F^*$ from the space of {\bf All Distribution Functions}:
\[
\boxed{
X_1,X_2,\ldots,X_n \overset{IID}{\sim} F^*, \qquad F^* \in \BB{\Theta} = \{ \text{All DFs}\} := \{ \,  F(x; F) :  F~is~a~DF \, \}
}
\]
where the DF $F(x; F)$ is indexed or parameterised by itself. Thus, the parameter space $\BB{\Theta}=\{ \text{All DFs}\}$ is the {\bf infinite dimensional} space of {\bf All DFs}.  In this section, we look at estimation problems in non-parametric experiments with an infinite dimensional parameter space.  That is, we want to estimate the DF $F^*$ from which our IID data are drawn.

The next proposition is often referred to as the {\bf fundamental theorem of statistics} and is at the heart of non-parametric inference, empirical processes, and computationally intensive bootstrap techniques.  Recall  \hyperref[D:ECDF]{Definition~\ref*{D:ECDF}} of the $n$-sample empirical distribution function (EDF or ECDF) $\widehat{F}_n$ that assigns a probability mass of $1/n$ at each data point $x_i$:
\begin{eqnarray*} %\label{E:ECDF}
\widehat{F}_n(x) = \frac{ \sum_{i=1}^n \BB{1}(X_i \leq x) }{n} \ ,  & \quad where \qquad
\BB{1}(X_i \leq x) :=
\begin{cases}
& 1  \quad \text{if $x_i \leq x$} \\
& 0  \quad \text{if $x_i > x$}
\end{cases}
\end{eqnarray*}

\begin{figure}[htpb]
\caption{Plots of ten distinct ECDFs $\widehat{F}_n$ based on $10$ sets of $n$ IID samples from $\uniform(0,1)$ RV $X$, as $n$ increases from $10$ to $100$ to $1000$.  The DF $F(x)=x$ over $[0,1]$ is shown in red.  The script of \hyperref[Mf:GilvenkoCantelliUnif01n10n100n100ECDFs]{Labwork \ref*{Mf:GilvenkoCantelliUnif01n10n100n100ECDFs}} was used to generate this plot.   \label{F:GilvenkoCantelliUnif01n10n100n100ECDFs}}
\centering   \makebox{\includegraphics[width=6.50in]{figures/GilvenkoCantelliUnif01n10n100n100ECDFs}}
\end{figure}
\begin{prop}[Gilvenko-Cantelli Theorem]\label{P:Gilvenko-Cantelli}
Let $X_1,X_2,\ldots,X_n \overset{\IID}{\sim} F^*$.  Then:
\[
\sup_x { | \widehat{F}_n(x) - F^*(x) | } \overset{P}{\longrightarrow} 0 \ .
\]
%{\scriptsize
%\begin{proof}
%Proof to be seen in STAT 318 or another advanced Statistics course.
%\end{proof}
%}
\end{prop}
{\bf Heuristic Interpretation of the Gilvenko-Cantelli Theorem}:  As the sample size $n$ increases, the empirical distribution function $\widehat{F}_n$ converges to the true DF $F^*$ in probability, as shown in \hyperref[F:GilvenkoCantelliUnif01n10n100n100ECDFs]{Figure~\ref*{F:GilvenkoCantelliUnif01n10n100n100ECDFs}}.

\begin{prop}[The Dvoretzky-Kiefer-Wolfowitz (DKW) Inequality]
Let $X_1,X_2,\ldots,X_n \overset{\IID}{\sim} F^*$.  Then, for any $\epsilon>0$:
\begin{equation}\label{E:DKWNeq}
P \left( \sup_x | \widehat{F}_n(x) - F^*(x) | > \epsilon  \right) \leq 2 \exp {(-2 n \epsilon^2)}
\end{equation}
Recall that $\sup(A)$ or supremum of a set $A \subset \Rz$ is the least upper bound of every element in $A$.
\end{prop}

\subsection{Estimating DF}\label{S:EstimDF}
Let $X_1,X_2,\ldots,X_n \overset{\IID}{\sim} F^*$, where $F^*$ is some particular DF in the space of all possible DFs, i.e.~the experiment is non-parametric.  Then, based on the data sequence $X_1,X_2,\ldots,X_n$ we want to estimate $F^*$.

For any fixed value of $x$, the expectation and variance of the empirical DF \eqref{E:ECDF} are:
\begin{eqnarray}
\E \left( \widehat{F}_n(x) \right) &=& F^*(x) \implies \mathsf{bias}_n \left( \widehat{F}_n(x) \right) = 0 \label{E:FnhatUnbiased} \\
\V \left( \widehat{F}_n(x) \right) &=& \frac{F^*(x) (1-F^*(x))}{n} \implies \lim_{n \to \infty}\mathsf{se}_n \left( \widehat{F}_n(x) \right) = 0  \label{E:Fnhatseto0}
\end{eqnarray}
Therefore, by \hyperref[P:AsympConsistencyUnbiasedSE0]{Proposition~\ref* {P:AsympConsistencyUnbiasedSE0}}, the empirical DF evaluated at $x$, i.e.~$\widehat{F}_n(x)$ is an asymptotically consistent estimator of the DF evaluated at $x$, i.e.~$F^*(x)$.  More formally,
\eqref{E:FnhatUnbiased} and \eqref{E:Fnhatseto0}, by \hyperref[P:AsympConsistencyUnbiasedSE0]{Proposition~\ref* {P:AsympConsistencyUnbiasedSE0}}, imply that for any fixed value of $x$:
\[
\widehat{F}_n(x) \overset{P}{\longrightarrow} F^*(x) \ .
\]
We are interested in a point estimate of the entire DF $F^*$, i.e.~$F^*(x)$ over all $x$.  A point estimator $T_n=T_n(X_1,X_2,\ldots,X_n)$ of a fixed and possibly unknown $F \in \{ \text{All DFs} \}$ is the empirical DF $\widehat{F}_n$.  This estimator has an asymptotically desirable property:
\[
\sup_x { | \widehat{F}_n(x) - F^*(x) | } \overset{P}{\longrightarrow} 0
\]
because of the Gilvenko-Cantelli theorem in Proposition~\ref*{P:Gilvenko-Cantelli}.  Thus, we can simply use $\widehat{F}_n$, based on the realized data $(x_1,x_2,\ldots,x_n)$, as a point estimate of $F^*$.

On the basis of the DKW inequality \eqref{E:DKWNeq}, we can obtain a $1-\alpha$ confidence set or {\bf confidence band} $C_n(x) := [\underline{C}_{\, n}(x), \overline{C}_{\, n}(x)]$ about our point estimate of $F^*$:
\begin{eqnarray}
\underline{C}_{\, n}(x) &=& \max \{ \widehat{F}_n(x)-\epsilon_n, 0 \}, \notag \\
\overline{C}_{\, n}(x)  &=& \min \{ \widehat{F}_n(x)+\epsilon_n, 1 \}, \notag \\
\epsilon_n &=& \sqrt{ \frac{1}{2n} \log \left( \frac{2}{\alpha}\right)} \ .
\end{eqnarray}
It follows from \eqref{E:DKWNeq} that for any fixed and possibly unknown $F^*$:
\[
P \left( \underline{C}_{\, n}(x) \leq F^*(x) \leq \overline{C}_{\, n}(x) \right) \geq 1-\alpha \ .
\]
Let us look at a simple example next.
\begin{labwork}[Estimating the DF of $\uniform(0,1)$ RV]
Consider the problem of estimating the DF of $\uniform(0,1)$ RV $U$ on the basis of $n_=10$ samples.  We use the function {\tt ECDF} of Labwork~\ref*{Mf:ECDF} and {\sc Matlab}'s built-in function {\tt stairs} to render the plots.  Figure~\ref*{F:UniformECDFsBands} was generated by {\tt PlotUniformECDFsConfBands.m} given below.
{\VrbMf[label=PlotUniformECDFsConfBands.m]{scripts/PlotUniformECDFsConfBands.m}}
\begin{figure}[htpb]
\caption{The empirical DFs $\widehat{F}^{(1)}_{n}$ from sample size $n=10, 100, 1000$ (black), is the point estimate of the fixed and known DF $F(x)=x, x \in[0,1]$ of $\uniform(0,1)$ RV (red).  The $95\%$ confidence band for each $\widehat{F}_n$ are depicted by green lines.  \label{F:UniformECDFsBands}}
\centering   \makebox{\includegraphics [width=5.5in]{figures/UniformECDFsBands}}
\end{figure}
\end{labwork}
Next we look at a more interesting example involving real-world data.
\begin{labwork}[Non-parametric Estimation of the DF of Times Between Earth Quakes]\label{LW:NZSIEQTimesECDFsConfBands}
Suppose that the $6,128$ observed times between Earth quakes in  NZ between 18-Jan-2008 02:23:44 and 18-Aug-2008 19:29:29 are:
$$X_1,\ldots,X_{6128} \overset{IID}{\sim} F^*, \qquad F^* \in \{ \text{all DFs} \} \enspace .$$
Then the non-parametric point estimate of the unknown $F^*$ is $\widehat{F}_{6128}$, the ECDF of the inter earth quake times.  We plot the non-parametric point estimate as well as the 95\% confidence bands for $F^*$ in  \hyperref[F:NZSIEQTimesECDFsConfBands]{Figure~\ref*{F:NZSIEQTimesECDFsConfBands}}.
\begin{figure}[htpb]
\caption{The empirical DF $\widehat{F}_{6128}$ for the inter earth quake times and the $95\%$ confidence bands for the non-parametric experiment.  \label{F:NZSIEQTimesECDFsConfBands}}
\centering   \makebox{\includegraphics[width=6.5in]{figures/NZSIEQTimesECDFsConfBands}}
\end{figure}
\VrbMf[label=NZSIEQTimesECDFsConfBands.m]{scripts/NZSIEQTimesECDFsConfBands.m}
\end{labwork}

Recall the poor fit of the Exponential PDF at the MLE for the Orbiter waiting time data.  We can attribute the poor fit to coarse resolution of the waiting time measurements in minutes and the rigid decaying form of the exponential PDFs.  Let us revisit the Orbiter waiting time problem with our non-parametric estimator.

\begin{labwork}[Non-parametric Estimation of Orbiter Waiting Times DF]\label{LW:OrbiterECDFsConfBands}
Suppose that the waiting times at the Orbiter bus stop are:
$$X_1,\ldots,X_{132} \overset{IID}{\sim} F^*, \qquad F^* \in \{ \text{all DFs} \} \enspace .$$
Then the non-parametric point estimate of $F^*$ is $\widehat{F}_{132}$, the ECDF of the $132$ Orbiter waiting times.
\begin{figure}[htpb]
\caption{The empirical DF $\widehat{F}_{132}$ for the Orbiter waiting times and the $95\%$ confidence bands for the non-parametric experiment.  \label{F:OrbiterECDFsConfBands}}
\centering   \makebox{\includegraphics[width=5.5in]{figures/OrbiterECDFsConfBands}}
\end{figure}
We compute and plot the non-parametric point estimate as well as the 95\% confidence bands for the unknown DF $F^*$ beside the parametric estimate and $95\%$ confidence bands from \hyperref[LW:ExponentialMLECIOrbiter]{Labwork~\ref*{LW:ExponentialMLECIOrbiter}}.  Clearly, the non-parametric estimate is preferable to the parametric one for this example.  Notice how the non-parametric confidence bands do not contain the parametric estimate of the DF.
\VrbMf[label=OrbiterECDFsConfBands.m]{scripts/OrbiterECDFsConfBands.m}
\end{labwork}

\begin{example}
First take a look at \hyperref[DA:WebLogs]{Data~\ref*{DA:WebLogs}} to understand how the web login times to our Maths \& Stats Department's web server (or requests to our WWW server) were generated. \hyperref[F:WebLogTimesECDFs]{Figure~\ref*{F:WebLogTimesECDFs}} shows the login times in units of seconds over a 24 hour period starting at 0357 hours and 30 seconds (just before 4:00AM) on October 1st, 2007 (red line) and on October 2nd, 2007 (magenta).
\begin{figure}[htpb]
\caption{The empirical DFs $\widehat{F}^{(1)}_{n_1}$ with $n_1= 56485$, for the web log times starting October 1, and $\widehat{F}^{(2)}_{n_2}$ with $n_2=53966 $, for the web log times starting October 2.  Their $95\%$ confidence bands are indicated by the green.  \label{F:WebLogTimesECDFs}}
\centering   \makebox{\includegraphics [width=4.5in]{figures/WebLogTimesECDFsBands}}
%WebLogTimesECDFs.eps}}
\end{figure}
If we assume that some fixed and unknown DF $F^{(1)}$ specifies the distribution of login times for October 1st data and another DF $F^{(2)}$ for October 2nd data, then the non-parametric point estimates of  $F^{(1)}$ and $F^{(2)}$ are simply the empirical DFs $\widehat{F}^{(1)}_{n_1}$ with $n_1= 56485$ and $\widehat{F}^{(2)}_{n_2}$ with $n_2=53966 $, respectively, as depicted in  \hyperref[F:WebLogTimesECDFs]{Figure~\ref*{F:WebLogTimesECDFs}}.  See the script of {\tt WebLogDataProc.m} in \hyperref[DA:WebLogs]{Data~\ref*{DA:WebLogs}} to appreciate how the ECDF plots in \hyperref[F:WebLogTimesECDFs]{Figure~\ref*{F:WebLogTimesECDFs}} were made.
\end{example}

\section{Plug-in Estimators of Statistical Functionals}\label{S:PlugIn}
Recall from \hyperref[S:Statistics]{Chapter~\ref*{S:Statistics}} that a {\bf statistical functional} is simply any function of the DF $F$.  For example, the median $T(F) = F^{[-1]}(1/2)$ is a statistical functional.  Thus, $T(F): \{ \text{All DFs }\} \to \Tz$, being a map or function from the space of DFs to its range $\Tz$, is a functional.
The idea behind the plug-in estimator for a statistical functional is simple: just plug-in the point estimate $\widehat{F}_n$ instead of the unknown DF $F^*$ to estimate the statistical functional of interest.
\begin{definition}[Plug-in estimator]
Suppose, $X_1,\ldots,X_n \overset{IID}{\sim} F^*$.  The plug-in estimator of a statistical functional of interest, namely, $T(F^*)$, is defined by:
\[
\widehat{T}_n := \widehat{T}_n (X_1,\ldots,X_n) = T(\widehat{F}_n) \ .
\]
\end{definition}

\begin{definition}[Linear functional]
If $T(F) = \int r(x) dF(x)$ for some function $r(x):\Xz\to \Rz$, then $T$ is called a {\bf linear functional}.  Thus, $T$ is linear in its arguments:
\[
T(a F + a' F') = a T(F) + a' T(F') \enspace .
\]
\end{definition}

\begin{prop}[Plug-in Estimator of a linear functional]
The plug-in estimator for a linear functional $T = \int r(x) dF(x)$ is:
\[
\boxed{
T(\widehat{F}_n) = \int r(x) d \widehat{F}_n(x)=\frac{1}{n}\sum_{i=1}^n r(X_i)
} \enspace .
\]
\end{prop}

Some specific examples of statistical linear functionals we have already seen include:
\begin{enumerate}
\item The {\bf mean} of RV $X \sim F$ is a function of the DF $F$:
\[
T(F) = \E(X) = \int x\,dF(x) \ .
\]
\item The {\bf variance} of RV $X \sim F$ is a function of the DF $F$:
\[
T(F) = \E(X-\E(X))^2 = \int (x-\E(X))^2\,dF(x) \ .
\]
\item The {\bf value of DF at a given $x \in \Rz$} of RV $X \sim F$ is also a function of DF $F$:
\[
T(F) = F(x) \  .
\]
\item The {\bf $q^{\text{th}}$ quantile} of RV $X \sim F$:
\[
T(F) = F^{[-1]}(q) \ \text{ where } q \in [0,1] \ .
\]
\item The {\bf first quartile} or the {\bf $0.25^{\text{th}}$ quantile} of the RV $X \sim F$:
\[
T(F) = F^{[-1]}(0.25) \ .
\]
\item The {\bf median} or the {\bf second quartile} or the {\bf $0.50^{\text{th}}$ quantile} of the RV $X \sim F$:
\[
T(F) = F^{[-1]}(0.50) \  .
\]
\item The {\bf third quartile} or the {\bf $0.75^{\text{th}}$ quantile} of the RV $X \sim F$:
\[
T(F) = F^{[-1]}(0.75) \ .
\]
\end{enumerate}

\begin{labwork}[Plug-in Estimate for Median of Web Login Data]
Compute the plug-in estimates for the median for each of the data arrays:
\[
\text{{\tt WebLogSeconds20071001035730} and {\tt WebLogSeconds20071002035730} }
\]
that can be loaded into memory by following the commands in the first $13$ lines of the script file {\tt WebLogDataProc.m} of \hyperref[DA:WebLogs]{Data~\ref*{DA:WebLogs}}.
\end{labwork}

\begin{labwork}[Plug-in Estimates of Times Between Earth Quakes]\label{LW:PlugInEstimatesEarthQuakes}
Compute the plug-in estimates for the median and mean time in minutes between earth quakes in NZ using the data in {\tt earthquakes.csv}.
\VrbMf[label=NZSIEQTimesPlugInEstimates.m]{scripts/NZSIEQTimesPlugInEstimates.m}
\begin{VrbM}
>> NZSIEQTimesPlugInEstimates
PlugInMedianEstimate =    0.0177
PlugInMedianEstimateMinutes =   25.5092
PlugInMeanEstimate =    0.0349
PlugInMeanEstimateMinutes =   50.2278
\end{VrbM}
\end{labwork}

Note that any statistical functional can be estimated using the plug-in estimator.  However, to produce a $1-\alpha$ confidence set for the plug-in point estimate, we need bootstrap methods.  The subject of next chapter.