\chapter{Model and Estimation} \label{chapter2:Procedure}

Throughout this chapter, assume that the data is observed at $n$ (not necessarily evenly spaced) spatial locations $\{\bm{s}_1, \dots, \bm{s}_n\}$. These locations exist on some bounded domain $\mathcal{D} \subseteq  \mathbb{R}^d$. The data, $\bm{y} =(y_1, \dots, y_n)^T$, is a realization of the random vector $\bm{Y} = (Y(\bm{s}_1), \dots, Y(\bm{s}_n))^T$, which constitute observations of single instance of a Gaussian process with mean 0 and stationary isotropic covariance function $C(h)$. %Then
% \[
% 	\bm{Y} \sim \mathcal{N}_n(\bm{0}, \bm{\Sigma}),
% \]
% where $\Sigma_{ij} = C(|| \bm{s}_i - \bm{s}_j ||)$.
We aim to estimate the covariance function.

\section{Estimating the Covariance Function} % (fold)
\label{sec:estimating_the_covariance_function}

Under the assumptions of stationarity and isotropy, the random vector $\bm{Y} \sim \mathcal{N}_n(\bm{0}, \bm{\Sigma})$, where the $ij$th element of $\bm{\Sigma}$ is
\[
	\Sigma_{ij} = \textrm{Cov}(y_i, y_j) = C(||\bm{s}_i - \bm{s}_j||) = C(h_{ij}).
\]
The scaler $h_{ij}$ is the Euclidean distance between locations $\bm{s}_i$ and $\bm{s}_j$. We can use \eqref{eq:bochner2} to obtain an integral representation for each element of $\bm{\Sigma}$,
\begin{equation} \label{eq:cov-elements-spectral}
	\Sigma_{ij} = \int \cos(h_{ij}\omega) \; f(\omega) \; d\omega.
\end{equation}
Rather than model the covariance function directly, we will model the spectral density with a semiparametric Bayesian model and use \eqref{eq:cov-elements-spectral} to construct the data likelihood.

% Were we able to sample from the spectral density $f(\omega)$, we could estimate $\Sigma_{ij}$ using Monte Carlo integration:
% \begin{equation}
% 	\Sigma_{ij} \approx \widehat{\Sigma}_{ij} = \frac{1}{M} \sum_{m=1}^M \cos(h_{ij}\widetilde{\omega}_m)
% \end{equation}
% for $M$ samples $\{\widetilde{\omega}_1, \dots, \widetilde{\omega}_M\}$ from $f(\omega)$. The next section discusses an approach for approximating $f(\omega)$.

% section estimating_the_covariance_function (end)

\subsection{Calculating the Likelihood} % (fold)
\label{sec:calculating_the_likelihood}

Consider a rich family of spectral densities $f_{\bm{\theta}}(\omega)$ indexed by a parameter vector $\bm{\theta}$.  Since $\bm{Y} \sim \mathcal{N}_n(\bm{0}, \bm{\Sigma})$, the likelihood of $\bm{\theta}$ given the data $\bm{y}$ is
\begin{equation} \label{eq:loglik}
	\ell(\bm{\theta}; \bm{y}) = -\frac{n}{2} \log(2\pi) - \frac{1}{2} \log |\bm{\Sigma}(\bm{\theta})| - \frac{1}{2} \bm{y}^T \bm{\Sigma}(\bm{\theta})^{-1} \bm{y},
\end{equation}
where $\bm{\Sigma}(\bm{\theta})$ is defined as in \eqref{eq:cov-elements-spectral}.  In general, the solution to the Fourier transform integral in \eqref{eq:cov-elements-spectral} will not be available analytically.

Our strategy is to replace each element $\Sigma_{ij}(\bm{\theta})$ with a simple Monte Carlo approximation.  Suppressing dependence on $\bm{\theta}$ for notational convenience, we will use
\begin{equation}
	\Sigma_{ij} = \int \cos(h_{ij}\omega) \; f(\omega) \; d\omega \approx  \frac{1}{M} \sum_{m=1}^M \cos(h_{ij}\widetilde{\omega}_m) = \widehat{\Sigma}_{ij},
\end{equation}
where $\widetilde{\omega}_1, \dots, \widetilde{\omega}_M$ are $M$ samples from $f_{\bm{\theta}}(\omega)$. We then plug $\widehat{\bm{\Sigma}}$ into the likelihood \eqref{eq:loglik} to estimate $\bm{\theta}$.

It is natural to ask how closely the likelihood evaluated using \eqref{eq:cov-elements-spectral} approximates the true likelihood.  The following theoretical result shows that the estimated likelihood $\hat{\ell}(\bm{\theta}; \bm{y})$ converges to the true likelihood $\ell(\bm{\theta}; \bm{y})$ almost surely as the number of Monte Carlo samples approaches infinity, even when the spectral density has heavy tails. A proof is given in the Appendix.

\begin{theorem}
  \label{thm:conssitency}
	Let $\bm{y} = (y_1, \dots, y_n)^T$ be a vector of observations from a mean-zero stationary, isotropic Gaussian process with covariance function $C(h)$, taken at locations $\bm{s}_1, \dots, \bm{s}_n \in \mathcal{D} \subseteq \mathbb{R}^d$ in some bounded domain $\mathcal{D}$.  For some symmetric density $f(\omega)$, let $\ell(\bm{y})$ be the likelihood
	\[
		\ell(\bm{y}) = -\frac{n}{2} \log(2\pi) - \frac{1}{2} \log(|\bm{\Sigma}|) - \frac{1}{2} \bm{y}^T \bm{\Sigma}^{-1} \bm{y},
	\]
where $\Sigma_{ij} = \int \cos(h_{ij}\omega)f(\omega)d\omega$, and $\hat{\ell}(\bm{y})$ be the Monte Carlo approximated likelihood
   	\[
 		\hat{\ell}(\bm{y}) = -\frac{n}{2} \log(2\pi) - \frac{1}{2} \log (| \widehat{\bm{\Sigma}}|) - \frac{1}{2} \bm{y}^T \widehat{\bm{\Sigma}}^{-1} \bm{y}
	\]
where $\widehat{\Sigma}_{ij} = \frac{1}{M} \sum_{m=1}^M \cos(\widetilde{\omega}_m h_{ij})$, and $\widehat{\omega}_1, \dots, \widehat{\omega}_M$ are and iid sample from $f(\omega)$.  Then as $M \to \infty$,
\[
\hat{\ell}(\bm{y}) \to \ell(\bm{y}) \text{ a.s.-}f_\omega.
\]

%     Then the distribution of $\bm{Y}$ is
% 	\[
% 		\bm{Y} \sim \mathcal{N}_n(0, \bm{\Sigma})
% 	\]
% 	where $\Sigma_{ij} = C(||\bm{s}_i - \bm{s}_j||) = C(h_{ij})$. The true likelihood function is
% 	\[
% 		\ell(\bm{\Sigma}; \bm{y}) = -\frac{n}{2} \log(2\pi) - \frac{1}{2} \log \det (\bm{\Sigma}) - \frac{1}{2} \bm{y}^T \bm{\Sigma}^{-1} \bm{y}.
% 	\]
% 	Suppose we observe data $\bm{y} = (y_1, \dots, y_n)^T$ from this Gaussian process. Let
% 	\[
% 		\hat{\ell}(\widehat{\bm{\Sigma}}; \bm{y}) = -\frac{n}{2} \log(2\pi) - \frac{1}{2} \log \det (\widehat{\bm{\Sigma}}(\bm{\beta})) - \frac{1}{2} \bm{y}^T \widehat{\bm{\Sigma}}^{-1}(\bm{\beta}) \bm{y}
% 	\]
% 	be the approximated likelihood from our proposed method. The $(i,j)$th element of the approximated covariance matrix $\widehat{\bm{\Sigma}}$ is defined as
% 	\[
% 		\widehat{\Sigma}_{ij} = \frac{1}{M} \sum_{m=1}^M \cos(\widetilde{\omega}_m h_{ij})
% 	\]
% 	where $\{\widehat{\omega}_1, \dots, \widehat{\omega}_M\}$ are random samples distributed according to the spectral density $f_\omega$. Then as $M \to \infty$, $\hat{\ell}(\bm{\beta}; \bm{y}) \to \ell(\bm{\Sigma}; \bm{y})$ a.s. $f_\omega$.
\end{theorem}

Theorem \ref{thm:conssitency} says that as long as we are willing to draw a large enough collection of Monte Carlo samples from a spectral density $f_{\bm{\theta}}(\omega)$, the Monte Carlo estimated likelihood of $\bm{\theta}$ will get arbitrarily close to the exact likelihood.  Furthermore, this remains true even when $f_{\bm{\theta}}(\omega)$ has heavy tails, as we expect for realistic models of spatial phenomena.

% To be explicit, in \eqref{eq:loglik} the estimated covariance matrix $\widehat{\bm{\Sigma}}$ is expressed as a function of $\bm{\beta}$ because
% \[
% 	\widehat{\Sigma}_{ij} = \frac{1}{M} \sum_{m=1}^M \cos(h_{ij} \widetilde{\omega}_m),
% \]
% \[
% 	\widetilde{\omega}_m \overset{\textrm{i.i.d.}}{\sim} \hat{f}(\omega),
% \]
% and
% \[
% 	\log \hat{f}(\log \omega) = \sum_{k=1}^K \beta_k B_k(\log \omega).
% \]
% The only free parameters in this procedure are the values of $\bm{\beta}$. This is summarized in Algorithm~\ref{alg:lik}.

% From \eqref{eq:spline}, we can transform back to $\hat{f}(\omega)$, integrate numerically, and sample via the inverse CDF method.

% \begin{algorithm}[!htb]
% 	\caption{\small Calculating the log likelihood} \label{alg:lik}
% 	\begin{algorithmic}[1]
% 		\Procedure{Likelihood}{$\bm{\beta}, \bm{y}$}
% 		\State $\log \hat{f}(\log \omega) = \sum_{k=1}^K \beta_k B_k(\log \omega)$\Comment{$k$ preselected knot locations}
% 		\State Sample $\widetilde{\omega}_1, \dots, \widetilde{\omega}_M$ from $\hat{f}(\omega)$\Comment{Inverse CDF method; $M$ large}
% 		\For{$1 \leq i,j \leq n$}
% 			\State $\widehat{\Sigma}_{ij} = \frac{1}{M} \sum_{m=1}^M \cos(h_{ij} \widetilde{\omega}_m)$\Comment{$h_{ij}$: distance between locations $\bm{s}_i$ and $\bm{s}_j$}
% 		\EndFor
% 		\State $\hat{\ell}(\bm{\beta}; \bm{y}) = -\frac{n}{2} \log(2\pi) - \frac{1}{2} \log |\widehat{\bm{\Sigma}}| - \frac{1}{2} \bm{y}^T \widehat{\bm{\Sigma}}^{-1} \bm{y}$
% 		\EndProcedure
% 	\end{algorithmic}
% \end{algorithm}

\section{Semiparametric Modeling of the Spectral \\ Density} % (fold)
\label{sec:semiparametric_modeling_of_the_spectral_density}

Since we don't want to impose a parametric form for $f(\omega)$, we will model it semiparametrically using splines. According to Bochner's theorem, $f(\omega)$ is a symmetric density, so it suffices to restrict our attention to modeling $f(\omega)$ for $\omega > 0$.

Rather than modeling $f(\omega)$ directly, we apply a straightforward variable transformation and model $\log f(\log \omega)$ instead. Working on the log-log scale is a natural way to flexibly specify densities with heavy tails.  It is easy to show that any density with power law tails has linear tails on the log-log scale. Suppose $f_X(x) = cx^{-b}$, $b > 1$, and let $Y = \log X$. Then
\begin{align*}
	f_Y(y) &= f_X(e^y) \frac{d}{dy}(e^y) \\
	&= ce^{-by}e^y \\
	&= ce^{-(b-1)y},
\end{align*}
and so
\[
	\log f_Y(y) \propto -(b-1)y.
\]
This fact is illustrated further in Figure~\ref{fig:logdens_ex}.

\begin{figure}[!htb]
	\centering
	\includegraphics[width=0.48\textwidth]{dens_ex.png}
	\includegraphics[width=0.48\textwidth]{logdens_ex.png}
	\caption{\small A density $f(\omega)$ with power law tails (left) and the same density transformed to the log-log scale (right). Both the left and right tails of the transformed density are linear.}
	\label{fig:logdens_ex}
\end{figure}

An important benefit to working in this log-transformed space is that linear tails are ideal for modeling the curve using a natural spline basis, which is constructed to ensure linearity beyond the outermost (\emph{boundary}) knots. If we are willing to assume that the tails of $\log f(\log \omega)$ are truly linear, then fitting it with natural splines will result in a good approximation over then entire domain $\log \omega \in (-\infty, \infty)$ without needing to use a large number of basis functions.

It might appear that requiring that $f(\omega)$ have power law tails is more restrictive than we would like. After all, the goal is to avoid the need to restrict $C(h)$ to a particular parametric family. However, the class of covariance functions that result from the Fourier transform of power law spectral densities is significantly broader than any of the parametric families, yet Bochner's theorem still guarantees that they are positive definite. In fact, this class covers most covariance models of practical interest, include the entire Mat\'ern class with finite $\nu$. Gaussian processes with spectral densities that decay quickly, e.g. with exponential tails, yield realizations that are generally regarded as unrealistically smooth~\cite{Stein1999}. %Two spatial locations a moderate distance apart would have nearly zero covariance, which is not an acceptably accurate model for real spatial data. By exclusively working with spectral densities that decay more slowly, we are restricting ourselves to the more realistic scenario where two locations can be far apart but still have a non-negligible covariance between them.

To model $\log f(\log \omega)$ using natural cubic splines, let
\begin{equation} \label{eq:spline}
	\log f(\log \omega) = \sum_{k=1}^K \beta_k B_k(\log \omega),
\end{equation}
where $\bm{\beta} = (\beta_1, \dots, \beta_K)^T$ are coefficients associated with $K$ known cubic spline basis functions $B_1(\cdot), \dots, B_K(\cdot)$. The $K$ basis functions are centered around a collection of $K$ locations referred to as knots.

% section semiparametric_modeling_of_the_spectral_density (end)

% section calculating_the_likelihood (end)

\section{Estimating the Spline Coefficients} % (fold)
\label{sec:estimating_the_spline_coefficients}

For a given collection of basis functions, the likelihood of the parameter vector $\bm{\theta} = \bm{\beta} = (\beta_1, \dots, \beta_K)^T$ can be approximated by plugging \eqref{eq:cov-elements-spectral} into \eqref{eq:loglik}. To be explicit, in \eqref{eq:loglik} the estimated covariance matrix $\bm{\Sigma}$ is expressed as a function of $\bm{\beta}$ because
\[
	\Sigma_{ij} = \int \cos(h_{ij}\omega) \, f_{\bm{\beta}}(\omega) \, d\omega \approx  \frac{1}{M} \sum_{m=1}^M \cos(h_{ij} \widetilde{\omega}_m) = \widehat{\Sigma}_{ij},
\]
\[
	\widetilde{\omega}_m \overset{\textrm{i.i.d.}}{\sim} f_{\bm{\beta}}(\omega),
\]
and
\[
	\log f_{\bm{\beta}}(\log \omega) = \sum_{k=1}^K \beta_k B_k(\log \omega).
\]
The only free parameters in this procedure are the elements of $\bm{\beta}$. This is summarized in Algorithm~\ref{alg:lik}.

To quickly generate the samples $\widetilde{\omega}_1, \ldots, \widetilde{\omega}_M$, we transform $\log f_{\bm{\beta}}(\log \omega)$ back to $f_{\bm{\beta}}(\omega)$, integrate numerically, and sample via the inverse CDF method.

\begin{algorithm}[!htb]
	\caption{\small Calculating the log likelihood $\hat{\ell}(\bm{\beta}; \bm{y})$} \label{alg:lik}
	\begin{algorithmic}[1]
		\Procedure{Likelihood}{$\bm{\beta}, \bm{y}$}
		\State $\log f_{\bm{\beta}}(\log \omega) = \sum_{k=1}^K \beta_k B_k(\log \omega)$\Comment{$k$ preselected knot locations}
		\State Sample $\widetilde{\omega}_1, \dots, \widetilde{\omega}_M$ from $f_{\bm{\beta}}(\omega)$\Comment{Inverse CDF method; $M$ large}
		\For{$1 \leq i,j \leq n$}
			\State $\widehat{\Sigma}_{ij} = \frac{1}{M} \sum_{m=1}^M \cos(h_{ij} \widetilde{\omega}_m)$\Comment{$h_{ij}$: distance between locations $\bm{s}_i$ and $\bm{s}_j$}
		\EndFor
		\State $\hat{\ell}(\bm{\beta}; \bm{y}) = -\frac{n}{2} \log(2\pi) - \frac{1}{2} \log |\widehat{\bm{\Sigma}}| - \frac{1}{2} \bm{y}^T \widehat{\bm{\Sigma}}^{-1} \bm{y}$
		\EndProcedure
	\end{algorithmic}
\end{algorithm}

Since we can calculate the likelihood as a function of the data and $\bm{\beta}$, we can estimate $\bm{\beta}$ using a Markov chain Monte Carlo (MCMC) algorithm. %However, the problem of choosing how many knots to use and where they should be located still remains.

To complete the model, we specify priors for $\bm{\beta}$ according to the Bayesian formulation of penalized splines introduced in~\cite{lang2004bayesian}.  The penalization corresponds to priors that specify to a second-order random walk, with
\[
	\beta_k \;|\; \tau^2 \sim \mathcal{N}(2\beta_{k-1} - \beta_{k-2}, \; \tau^2)
\]
where
\[
	\beta_2 \;|\; \tau^2 \sim \mathcal{N}(\beta_1, \; \tau^2)
\]
and
\[
	\beta_1 \;|\; \tau^2 \sim \mathcal{N}(0, \; \tau^2).
\]
The variance parameter $\tau^2$ controls the smoothness of the spline fit, and must also be estimated. Its prior is specified as
\[
	\tau^2 \sim \mathcal{N}^+(0, \; \sigma^2_\tau),
\]
the half-normal distribution with some variance hyperparameter $\sigma^2_\tau$.

With the priors specified, we can proceed with the MCMC algorithm in Algorithm~\ref{alg:mcmc}. We use a Metropolis-Hastings update to determine whether or not to accept the joint proposal $\bm{\beta}$. The result is random draws from the posterior distribution of $\bm{\beta} \;|\; \bm{y}$.

\begin{algorithm}[!htb]
	\caption{\small Metropolis-Hastings Sampler} \label{alg:mcmc}
	\begin{algorithmic}[1]
		\Procedure{MCMC}{$\bm{\beta}, \bm{y}$}
		\State $\bm{\beta}_1 \gets \bm{\beta}$\Comment{Initialization}
		\For{$2 \leq i \leq N$}\Comment{$N$ large}
		\State Propose new $\bm{\beta}^* \sim q(\cdot)$
		\State $\ell_0 = \hat{\ell}(\bm{\beta}_{i-1}, \bm{y})$\Comment{See Algorithm~\ref{alg:lik}}
		\State $\ell_1 = \hat{\ell}(\bm{\beta}^*, \bm{y})$
		\State $a = \Big[\pi(\bm{\beta}^*) \cdot \ell_1 \cdot q(\bm{\beta}_{i-1}|\bm{\beta}^*)\Big]/\Big[\pi(\bm{\beta}_{i-1}) \cdot \ell_0 \cdot q(\bm{\beta}^*|\bm{\beta}_{i-1})\Big]$
		\State Generate $u \sim Unif(0, 1)$
		\If{$u < \min(a, 1)$}
		\State $\bm{\beta}_i \gets \bm{\beta}^*$\Comment{Accept the proposal}
		\Else
		\State $\bm{\beta}_i \gets \bm{\beta}_{i-1}$\Comment{Reject the proposal}
		\EndIf
		\EndFor
		\EndProcedure
	\end{algorithmic}
\end{algorithm}

% section estimating_the_spline_coefficients (end)

\section{Computational Challenges} % (fold)
\label{sec:computational_challenges}

The primary difficulty inherent in this method is that it is extraordinarily computationally intensive, requiring a Monte Carlo approximation for every element of the covariance matrix. For a moderately large number of observations, say $n = 400$, we need to estimate $\frac{n(n-1)}{2} = 79800$ separate elements. For each of these elements, we need to perform a Monte Carlo integration with a large number of samples. This all happens within one likelihood calculation, and we need to compute the likelihood at every iteration of the MCMC algorithm. The computational costs would be prohibitive for a na\"{i}ve implementation of this method.

Fortunately, most of these difficulties can be assuaged by taking advantage of parallel computing. Algorithm~\ref{alg:mcmc} requires previous information $\bm{\beta}_{i-1}$ to compute $\bm{\beta}_i$, so unfortunately it would be extremely difficult to restructure it in a way where multiple updates are being performed simultaneously. However, it is possible to parallelize aspects of the likelihood calculation in Algorithm~\ref{alg:lik}.

Consider the \textbf{for} loop in Algorithm~\ref{alg:lik}. For each value of $i$ and $j$, the computation of $\widehat{\Sigma}_{ij}$
\[
	\frac{1}{M} \sum_{m=1}^M \cos(h_{ij} \widetilde{\omega}_m)
\]
depends on two values: $h_{ij}$, the distance between $\bm{s}_i$ and $\bm{s}_j$, and $\widetilde{\bm{\omega}}$, the vector of $M$ samples from $f(\omega)$. It does not depend on any previously computed element of $\widehat{\bm{\Sigma}}$. Therefore, performance would improve dramatically if we could calculate the estimates simultaneously, in parallel.

There are several options for speeding up a parallelizable problem. The simplest, and least reliant on expensive additional hardware, is \emph{parallel processing}~\cite{suchard2010understanding}. Today, most personal computers ship with CPUs that contain 2, 4, or even 8 computing cores. Each core is capable of running a single set of sequential instructions. By using tools such as OpenMP, users can write code that will assign a set of instructions to every available core. The instructions will execute simultaneously, and the results are gathered together upon completion. Of course, the potential speedup is limited by the number of cores available in the CPU---if there are 4 cores, the instructions will be executed in about one fourth of the time.

This type of parallel processing works well for problems like obtaining multiple MCMC chains at the same time. For the estimation of $\bm{\Sigma}$ in this context, however, it is not optimal. Recall that for $n = 400$, we need to estimate $79800$ elements of $\bm{\Sigma}$. Reducing this by a factor of 4 is helpful, but ideally we would like to perform all $79800$ at once, not just take them four at a time.

Another option is to utilize a computing cluster. Clusters usually are large rooms filled with racks of computers, connected in such a way that one process can be spread uniformly across them and use hundreds or thousands of cores at once~\cite{suchard2010understanding}. One downside to this approach is that access to such a cluster is difficult to get, outside of a large company or academic institution. Beyond that, though, latency starts to become a problem when dealing with clusters. Because the cores are spread over many physical computers, the increases in computational speed becomes overwhelmed by the large amount of time it takes to share data between the cores. As a result, the overall performance increase may not be as large as we might expect.

A third option is \emph{GPU computing}. Originally created to run video games, graphics processing units, or GPUs, have become increasingly popular for parallel computing. CPUs contain a small number of cores that are optimized to handle highly complex instructions quickly. By contrast, GPUs are made up of thousands of simple, highly efficient cores that are optimized for simpler instructions and designed to work in parallel with each other. Because the GPU cores lack many of the sophisticated features of CPU cores (a sacrifice made in exchange for speed and efficient exchange of data), the programmer must have a more intricate knowledge of the architecture of their particular GPU in order to get the best possible speedup. Fortunately, directives such as OpenACC let the compiler make most of the architecture-related decisions instead of leaving them in the hands of the user.

To take full advantage of the highly parallelizable nature of the covariance estimation problem, I have chosen the GPU option. I wrote the code to carry out the Monte Carlo integrations in CUDA C. CUDA is an API provided by Nvidia that allows users to write programs in C or Fortran that interact with Nvidia brand GPUs. Computations were done on three different hardware setups:

\begin{itemize}
	\item Nvidia GeForce GTX 1060 GPU, on a personal computer running Ubuntu 16.04;
	\item Nvidia Tesla P100 GPU, provided by Penn State University's Institute for CyberScience Advanced CyberInfrastructure (ICS-ACI);
	\item Nvidia Tesla K80 GPU, running on an Amazon EC2 instance.
\end{itemize}

With these tools, the computations can be performed fairly quickly. On the Tesla P100, one entire likelihood calculation described by Algorithm~\ref{alg:lik} with $M = 50000$ and $N = 400$ completes in just under one second.

To illustrate the power of parallel computing, and to show that this element-wise approximation to the covariance matrix is a feasible approach, Figure~\ref{fig:timings-mc} shows the results of an experiment comparing the runtime of the Monte Carlo approximation from Algorithm~\ref{alg:lik} to the runtime of the exact covariance calculation. That is, for a fixed $N$, generate 100 sets of points $\bm{s}_1, \dots, \bm{s}_N$ and their distances $h_{ij} = ||\bm{s}_i - \bm{s}_j||$. Also generate $M = 20000$ samples $\widetilde{\omega}_1, \dots, \widetilde{\omega}_M$ from the spectral density $f(\omega)$ corresponding to a Mat\`ern covariance function. Since the computational cost of the random sample generation is constant regardless of $N$, it is not included when timing.
\[
	\Sigma_{ij} = C(h_{ij})
\]
and
\[
	\widehat{\Sigma}_{ij} = \frac{1}{M}\sum_{m=1}^M \cos(h_{ij} \widetilde{\omega}_{m})
\]
100 times each, and compare the median runtimes as a function of $N$.

Notice from Figure~\ref{fig:timings-mc} that for large enough $N$, it actually becomes faster to perform a Monte Carlo approximation to the elements of $\Sigma$ than to directly plug in the distances to the covariance function $C$. In this experiment $N$ needed to be greater than about 2000, but this is of course hardware dependent.

This is not a surprising result considering the parallelized nature of the Monte Carlo approximation. As $N$ increases, the exact calculation must handle all additional computations sequentially, whereas the Monte Carlo approximation can simply allocate more cores that execute their tasks simultaneously. As a result, the Monte Carlo runtime increases more slowly with $N$ than the exact covariance runtime.

However, there are nontrivial computational costs associated with GPU computing that cause it to be significantly slower than the exact alternative when $N$ is not large. In Figure~\ref{fig:timings-mc}, it appears that the Monte Carlo runtime is more or less unchanged from $N = 40$ to $N = 400$. The bottleneck here is data transfer and allocation. For small $N$, the time required to do the calculations themselves is negligible compared to the time required to allocate memory on the GPU, transfer the spectral density samples from the CPU to the GPU, combine the results, and transfer them back to the CPU. This is the reason that sometimes, seemingly paradoxically, parallelizing a routine can cause it to execute more slowly than the equivalent sequential routine. But for large enough $N$, we expect the speedup to be substantial.

\begin{figure}[htbp]
	\centering
	\includegraphics[width=0.95\textwidth]{timings_mc.pdf}
	\caption{Elapsed time taken to calculate the element-wise Monte Carlo approximation to the covariance matrix corresponding to a Mat\'ern covariance function, versus calculating it directly using $C(h)$. The covariance matrix is $n \times n$, where $n$ is the number of observations. Each method was run 100 times for every value of $n$. The medians are plotted with dots and connected with a line. The error bars represent the range from the first to third quartiles---some times for the exact calculation were misleadingly high due to CPU wakeup. GPU computations were done on a Tesla P100. Note the logarithmic axes.}
	\label{fig:timings-mc}
\end{figure}

% section computational_challenges (end)


\section{Multivariate Distributions}
\subsection{Joint and Marginal Distributions}
Let $X$ be a continuous random variable has density $f$, then we know that
$$\mathbb P(X\le x)=\int_{-\infty}^xf(y)\,\mathrm dy$$
It can be proved that this can generalize to an arbitrary (measurable) subsets $B\subset\mathbb R$, that is,
$$\mathbb P(X\in B)=\int_Bf(x)\,\mathrm dx$$
\begin{definition}
    For a tuple of random variables $X=(X_1,\ldots,X_n)\in\mathbb R^n$ where $X_i$ are continuous, $X$ is said to have density $f$ if
    \begin{align*}
        F(x_1,\ldots,x_n)&=\mathbb P(X_1\le x_1,\ldots,X_n\le x_n)\\
        &=\int_{-\infty}^{x_1}\cdots\int_{-\infty}^{x_n}f(y_1,\ldots,y_n)\,\mathrm dy_n\cdots\mathrm dy_1
    \end{align*}
    So $f(x_1,\ldots,x_n)=\partial^n F/(\partial x_1\cdots\partial x_n)$.
    We can also generalize this to (measurable) subsets $B\subset\mathbb R^n$ by saying
    $$\mathbb P(X\in B)=\int_Bf(x_1,\ldots,x_n)\,\mathrm dx_1\cdots\mathrm dx_n$$
    Let $g:\mathbb R^n\to\mathbb R^+$, we define
    $$\mathbb E[g(X)]=\int_{\mathbb R^n}g(x_1,\ldots,x_n)f(x_1,\ldots,x_n)\,\mathrm dx_1\cdots\mathrm dx_n$$
\end{definition}
\begin{definition}
    We say $X_1,\ldots,X_n$ are independent if $\mathbb P(X_1\le x_1,\ldots,X_n\le x_n)=\mathbb P(X_1\le x_1)\cdots\mathbb P(X_n\le x_n)$.
\end{definition}
\begin{theorem}
    Let $X=(X_1,\ldots,X_n)$ have density $f$, then\\
    1. If $X_1,\ldots,X_n$ are independent and $X_i$ has density $f_i$, then
    $$f(x_1,\ldots,x_n)=\prod_{i=1}^nf_i(x_i)$$
    2. Conversely, suppose we have the above formula for some nonnegative functions $f_i$, then $X_1\ldots,X_n$ are independent and have density functions proportional to $f_i$.
\end{theorem}
\begin{proof}
    1. We know
    \begin{align*}
        \mathbb P(X_1\le x_1,\ldots,X_n\le x_n)&=\prod_{i=1}^n\int_{-\infty}^xf_i(y_i)\,\mathrm dy_i\\
        &=\int_{-\infty}^{x_1}\cdots\int_{-\infty}^{x_n}\prod_{i=1}^nf(y_i)\,\mathrm dy_n\cdots\mathrm dy_1
    \end{align*}
    2. By multiplying constant, we may assume that
    $$\int_{-\infty}^\infty f_i(x)\,\mathrm dx=1$$
    for all $i$.
    So for $B_i\subset\mathbb R$ (measurable),
    $$\mathbb P(X\in B_1\times\cdots\times B_n)=\int_{B_1}\cdots\int_{B_n}\prod_{i=1}^nf(y_i)\,\mathrm dy_n\cdots\mathrm dy_1$$
    Now fix $i$ and let $B_j=\mathbb R$ for any $j\neq i$, expanding the integral then gives
    $$\mathbb P(X_i\in B_i)=\mathbb P(X_i\in B_i,X_j\in B_j,j\neq i)=\int_{B_i}f_i(y_i)\,\mathrm dy_i$$
    So $f_i$ is the density of $X_i$.
    Independence follows.
\end{proof}
\begin{proposition}
    Let $X=(X_1,\ldots,X_n)$ have density $f$, we have
    $$\mathbb P(X_i\le x)=\int_{-\infty}^x\int_{-\infty}^\infty\cdots\int_{-\infty}^\infty f(x_1,\ldots,x_n)\,\mathrm dx_1\cdots\mathrm dx_{i-1}\,\mathrm dx_{i+1}\cdots\mathrm dx_n\,\mathrm dx_i$$
    So the density of $X_i$ is
    $$f_{X_i}(x)=\int_{-\infty}^\infty\cdots\int_{-\infty}^\infty f(x_1,\ldots,x_n)\,\mathrm dx_1\cdots\mathrm dx_{i-1}\,\mathrm dx_{i+1}\cdots\mathrm dx_n$$
\end{proposition}
\begin{proof}
    Obvious.
\end{proof}
\begin{definition}
    This is called the marginal density.
\end{definition}
\begin{definition}
    Let $f,g$ be two densities on $\mathbb R$, their convolution is defined as
    $$(f\ast g)(x)=\int_{-\infty}^\infty f(x-y)g(y)\,\mathrm dy$$
\end{definition}
Let $X,Y$ be two independent random variables with densities $f_X,f_Y$, then
\begin{align*}
    \mathbb P(X+Y\le z)&=\int_{\mathbb R^2}1_{x+y\le z}f_{X,Y}(x,y)\,\mathrm dx\,\mathrm dy\\
    &=\int_{x+y\le z}f_X(x)f_Y(y)\,\mathrm dx\,\mathrm dy\\
    &=\int_{-\infty}^\infty\int_{-\infty}^{z-x}f_X(x)f_Y(y)\,\mathrm dy\,\mathrm dx\\
    &=\int_{-\infty}^\infty\int_{-\infty}^zf_X(x)f_Y(y-x)\,\mathrm dy\,\mathrm dx\\
    &=\int_{-\infty}^z\int_{-\infty}^\infty f_X(x)f_Y(y-x)\,\mathrm dx\,\mathrm dy\\
    &=\int_{-\infty}^z(f\ast g)(y)\,\mathrm dy
\end{align*}
So basically the convolution of $f_X$ and $f_Y$ is the density of $X+Y$.
There is another way to obtain the same result, but it is not rigorous at all.
\begin{align*}
    \mathbb P(X+Y\le z)&=\int_{-\infty}^\infty\mathbb P(X+Y\le z,y\in\mathrm dy)\\
    &=\int_{-\infty}^\infty\mathbb P(X+y\le z,y\in\mathrm dy)\\
    &=\int_{-\infty}^\infty\mathbb P(X\le z-y)\mathbb P(y\in\mathrm dy)\\
    &=\int_{-\infty}^\infty F_X(z-y)f_Y(y)\,\mathrm dy
\end{align*}
Just differentiating and changing the order gives the result.
\subsection{Conditionals}
\begin{definition}
    Let $X,Y$ be continuous random variables with joint density $f_{X,Y}$, then the conditional density of $X$ given $Y=y$ is
    $$f_{X|Y}(x|y)=\frac{f_{X,Y}(x,y)}{f_Y(y)}$$
    Given that $f_Y$ is nonzero.
\end{definition}
\begin{definition}
    The conditional expectation of $X$ given $Y$ is $g(Y)$ where
    $$g(y)=\int_{-\infty}^\infty xf_{X|Y}(x|y)\,\mathrm dx(=\mathbb E(X|Y=y))$$
    We write $\mathbb E[X|Y]=g(Y)$.
\end{definition}
\subsection{Law of Total Probability}
\begin{proposition}[Law of Total Probability]
    $$f_X(x)=\int_{-\infty}^\infty f_{X|Y}(x|y)f_Y(y)\,\mathrm dy$$
\end{proposition}
\begin{proof}
    Plug in definition.
\end{proof}
\begin{example}
    Let $X\sim\operatorname{Exp}(\lambda),Y\sim\operatorname{Exp}(\mu)$ be independent.
    Set $Z=\min(X,Y)$.
    So
    \begin{align*}
        \mathbb P(Z\le z)&=1-\mathbb P(Z>z)=1-\mathbb P(X>z,Y>z)\\
        &=1-\mathbb P(X>z)\mathbb P(Y>z)=1-e^{-\lambda z}e^{-\mu z}=1-e^{-(\lambda+\mu)z}
    \end{align*}
    Therefore $Z\sim\operatorname{Exp}(\lambda+\mu)$.
    So by the same way, for $X_i\sim\operatorname{Exp}(\lambda_i)$, then $\min(X_1,\ldots,X_n)\sim \operatorname{Exp}(\sum_i\lambda_i)$
\end{example}
\subsection{Transformation of a Multidimensional Random Variable}
\begin{theorem}
    Let $X$ be a continuous random variable in $D\subset\mathbb R^d$ with density $f_X$.
    Let $g$ be a bijection $D\to g(D)$ with continuous derivative on $D$ and $\forall x\in D,\det g^\prime\neq 0$.
    Set $y=g(x)$ and $Y=g(X)$, then the density of $Y$ is given by
    $$f_Y(y)=f_X(x)|J|=f_X(x)\left\|\left(\frac{\partial x_i}{\partial y_j}\right)_{i,j=1}^d\right\|$$
\end{theorem}
\begin{proof}
    Omitted.
\end{proof}
\begin{example}
    Let $X,Y\sim\mathcal N(0,1)$ be independent, and $R=\sqrt{X^2+Y^2},\Theta=\arg(X,Y)$, so $X=R\cos\Theta,Y=R\sin\Theta$.
    So we want the joint density of $(R,\Theta)$.
    We have
    $$f_{R,\Theta}(r,\theta)=f_{X,Y}(x,y)|J|=rf_{X,Y}(x,y)=rf_X(x)f_Y(y)=\frac{r}{2\pi}e^{-r^2/2}$$
    So $R,\Theta$ are independent with $\Theta\sim\operatorname{Unif}(0,2\pi)$ and $R$ has density $f_R(r)=re^{-r^2/2}$.
\end{example}
\subsection{Order Statistics of a Random Sample}
Let $X_1,\ldots,X_n$ be i.i.d. with distribution $F$ and density $f$.
If we put them in order from smallest to biggest, $X_{(1)}\le \cdots X_{(n)}$, then $Y_i=X_{(i)}$ is called the order statistics.
So $\mathbb P(Y_1\le x)=1-\mathbb P(Y_1>x)=1-(1-F(x))^n$ is the distribution of $Y_1$.
The density then is $nf(x)(1-F(x))^{n-1}$.
Also $\mathbb P(Y_n\le x)=(F(x))^n$, so it has density $nf(x)F(x)^{n-1}$.
Now we want to find the joint density of $(Y_1,\ldots,Y_n)$.
Let $x_1<\cdots<x_n$, we have
\begin{align*}
    \mathbb P(Y_1\le x_1,\ldots,Y_n\le x_n)&=\sum_{\sigma\in S_n}\mathbb P(X_1\le x_{\sigma(1)},\ldots X_n\le x_{\sigma(n)})\\
    &=n!\mathbb P(X_1\le x_1,\ldots X_n\le x_n)\\
    &=n!\int_{-\infty}^{x_1}f(u_1)\int_{-\infty}^{x_2}f(u_2)\cdots\int_{-\infty}^{x_n}f(u_n)\,\mathrm du_n\cdots\mathrm  du_1
\end{align*}
So by differentiating the density $f_Y$ of $Y_i$'s is $f_Y(y_1,\ldots,y_n)=n!f(y_1)\cdots f(y_n)$ for $y_1<y_2<\cdots<y_n$ and $0$ otherwise.
\begin{example}
    Let $X_1,\ldots,X_n$ be i.i.d. $\operatorname{Exp}(\lambda)$ and $Y_i$ be the order statistics and $Z_1=Y_1,Z_i=Z_i-Z_{i-1}$ for $i=2,\ldots,n$.
    Then
    $$Z=\begin{pmatrix}
        Z_1\\
        \vdots\\
        Z_n
    \end{pmatrix}=A\begin{pmatrix}
        Y_1\\
        \vdots\\
        Y_n
    \end{pmatrix},A=\begin{pmatrix}
        1&0&0&\dots&0\\
        -1&1&0&\dots&0\\
        0&-1&1&\dots&0\\
        \vdots&\vdots&\vdots&\ddots&\vdots\\
        0&0&0&\dots&1
    \end{pmatrix}$$
    where we have $\det A=1$ and $Y_j=\sum_{i=1}^jZ_i$, so for $y_j=\sum_{i=1}^jz_i$, we have
    \begin{align*}
        f_{Z_1,\ldots,Z_n}(z_1,\ldots,z_n)&=f_{Y_1,\ldots,Y_n}(y_1,\ldots,y_n)|J|\\
        &=n!e^{-\lambda y_1}\cdots e^{-\lambda y_n}\\
        &=n!\lambda^ne^{-\lambda(nz_1+(n-1)z_2+\cdots +2z_{n-1}+z_n)}\\
        &=\prod_{i=1}^n(n-i+1)\lambda e^{-\lambda(n-i+1)z_1}
    \end{align*}
    So $(Z_1,\ldots,Z_n)$ are independent exponentials with $Z_i\sim\operatorname{Exp}(\lambda(n-i+1))$.
\end{example}


\lab{Algorithm}{Invertible Affine Transformations and Linear Systems}{Invertible Affine Transformations and Linear Systems}
\label{lab:ChangeBasis}
\objective{Understand how to apply various affine transformations to a set of vectors and how to solve linear systems.}

\section*{Basis}
A \emph{basis} for a vector space is a linearly independent set of vectors that spans the entire space. Given a basis, it is possible to express every vector in the space as a unique linear combination of the basis vectors (indeed, this property can be taken as the definition for a basis).

Bases themselves are not necessarily unique, however, and it can often be very useful to change the basis representation of vectors, both in theory and practice. Changing bases is an example of an important class of mappings known as \emph{linear transformations}.

In finite dimensional vector spaces like $\mathbb{R}^n$, any linear
transformation can be implemented as a single left matrix multiplication.
In other words, a mapping $T : \mathbb{R}^n \to \mathbb{R}^n$ is linear if and only if there exists an $n \times n$ matrix $A$ such that $T\left(X\right) = AX$ for all vectors $X \in \mathbb{R}^n$. If $A$ is the $n \times n$ matrix representation of a linear transformation and $P$ is an $n \times m$ matrix in $\mathbb{R}^n$, then $AP$ is the matrix of transformed points.

Different linear transformations can be represented by different types of matrices. In this lab we will take vectors in $\mathbb{R}^2$ and apply various linear and affine transformations.

We have represented each point as a column of the array \li{pts}.
It is also common to represent points as rows. In that case, the transformations described below can still be performed by transposing the arrays in the appropriate places.

\section*{Dilation}
Dilation, or scaling, is a type of linear transformation in which, from a geometrical perspective, points are stretched or compressed.
The matrix representation, $A$, of a dilation is a diagonal matrix. The values on the main diagonal indicate the amount of stretching in their respective directions.

\begin{figure}
\includegraphics[width=\textwidth]{stretch.pdf}
\caption{An example of a dilation. The top image is the original image and the bottom image is the modified image. The figure was stretched by a factor of $1.5$ in all directions.}
\end{figure}

\begin{problem}
Write a function that accepts an array of points and an array giving the stretching factors in each direction, and returns the dilated points.

Plot the original points and their image under the transformation. This can be done with the following code:
\begin{lstlisting}
from matplotlib import pyplot as plt
# Assume the arrays 'old' and 'new' have x values along the first row and y values along the second row.

plt.subplot(2, 1, 1)
plt.scatter(old[0], old[1])
plt.axis('equal')
plt.subplot(2, 1, 2)
plt.scatter(new[0], new[1])
plt.show()
\end{lstlisting}
\end{problem}

\section*{Rotation}
Rotating points around the origin is another type of linear transformation. To perform a rotation of $\theta$ radians counterclockwise, let
\[
A = \begin{pmatrix}
\cos(\theta) & -\sin(\theta) \\
\sin(\theta) & \cos(\theta)
\end{pmatrix}
\]

\begin{figure}
\includegraphics[width=\textwidth]{rotate.pdf}
\caption{An example of a rotation.
The top image is the original and the bottom is the rotated image.
The rotation angle is $\frac{\pi}{3}$.}
\label{basis:rotate}
\end{figure}

\begin{problem}
Write a function that accepts an array of points and the angle of rotation (in radians). Have it return the rotated points.
Plot the original points and their image under the transformation.
\end{problem}

\section*{Shear}

\begin{figure}
\includegraphics[width=\textwidth]{shear.pdf}
\caption{An example of a shear.
The top image is the original and the bottom is the sheared image.}
\label{basis:shear}
\end{figure}

A shear is a linear transformation that displaces a vector in a fixed direction. In $\mathbb{R}^2$, a shear can either be horizontal or vertical, and has one of the two following forms, respectively:

\[
A = \begin{pmatrix}
1 & c \\
0 & 1
\end{pmatrix}
\]

\[
A = \begin{pmatrix}
1 & 0 \\
c & 1
\end{pmatrix},
\]

where $c$ indicates the amount of displacement.

Note that these shearing matrices are instances of a type III elementary matrix. Horizontal shears leave the y-coordinates fixed, while vertical shears leave the x-coordinate fixed.

\begin{comment}
In physics, shears are examples of what are known as Galilean Transformations, and are used to switch between reference frames that differ only by constant relative motion.
\end{comment}

\begin{problem}
Write a function that accepts an array of points, a floating point argument that indicates the shearing amount, and an integer argument
that indicates the direction of shearing (0 for horizontal, 1 for vertical). Have it return the sheared points.
Plot the original points and their image under the transformation.
\end{problem}

\section*{Reflection}

\begin{figure}
\includegraphics[width=\textwidth]{reflect.pdf}
\caption{An example of a reflection. The top image is the original and the bottom is the image after being reflected about the line $y = \sqrt{3}x$.}
\label{basis:reflect}
\end{figure}

Reflection about a line (or in higher-dimensional spaces, a hyperplane) can also be accomplished by a linear transformation. Such a transformation is called a \emph{Householder Transformation}, and the general form of the matrix representation in
$\mathbb{R}^2$ is

\[
A = \frac{1}{l_1^2 + l_2^2}
\begin{pmatrix}
l_1^2 - l_2^2 & 2l_1l_2 \\
2l_1l_2 & l_2^2 - l_1^2
\end{pmatrix},
\]

where $(l_1, l_2)$ is a vector in the direction of the axis of reflection. In the simple case where the axis of reflection is the line $y=x$, the matrix is just a Type I elementary matrix. See Figure \ref{basis:reflection} for an example.

\begin{problem}
Write a function that accepts an array of points and an array giving the axis of reflection (in the notation above, this argument is $(l_1, l_2)$. Have it return the reflected points.
Plot the original points and their image under the transformation.
\end{problem}

\section*{Translation}

\begin{figure}
\includegraphics[width=\textwidth]{translate.pdf}
\caption{
An example of a translation.
The top image is the original and the bottom is the translated image.}
\label{basis:translate}
\end{figure}

Translations are not linear transformations, but they can easily be performed with array operations. Together, with linear transformations, they make up the broader class of transformations called
``affine transformations." These are transformations of the form $T: \mathbb{R}^n \to \mathbb{R}^n$, $T(X) = AX + b$ where $A$ is an $n\times n$ matrix and $b \in \mathbb{R}^n$. Affine transformations include all compositions of scalings, rotations, and translations.

Let $b$ be a vector that represents the desired translation.
In order to shift a set of points, you add to the row how much you would like it to shift in that direction. Thus, to shift the set of points up by 2, simply add \li{b = np.array([[0], [2]])}. This particular shape for $b$ allows for proper array broadcasting.
See Figure \ref{basis:translation} for an example.

\begin{problem}
Write a function that takes an array of points and an array indicating how much to shift them in each direction. The function should return the translated points.
Plot the original points and their image under the transformation.
\end{problem}

\section*{Compositions of Transformations}

\begin{figure}
\centering
\includegraphics[width=\textwidth]{combo.pdf}
\caption{
A composition of affine transformations: shear, reflection, and translation.}
\label{basis:combo}
\end{figure}

All the various transformations we have discussed can be combined into a single affine transformation through function composition.
For example, if you want to apply a shear and then a reflection, you can represent the shear as a matrix $S$ and the reflection as a matrix $R$. The combined transformation is then $RS$. The image of $X$ under both transformations and then a translation that moves the origin to point $P$ would be $RSX+P$. An example of this is shown in Figure \ref{basis:combo}.

\begin{problem}
We can use affine transforms to track the trajectory of a particle $p_1$ that rotates about another particle $p_2$ at a constant angular speed $\omega$ (a positive value means counterclockwise rotation), while $p_2$ moves in a particular direction at a constant speed $v$.

Suppose that $p_2$ has a starting position at the origin and moves in the direction of the vector $(1, 1)$, and suppose that $p_1$ has a starting position at the point $(0,1)$. Write a function that takes three parameters giving the time, angular velocity, and directional speed, respectively, and returns the position of $p_1$ at the given  time.

Visualize a trajectory by plotting the position of the particle at each time (in seconds) in the array \li{times = numpy.arange(0,10,.1)} for an angular velocity $\pi$ radians per second and a directional speed of 3 meters per second (assume the plot is on a meters scale).
\end{problem}

\section*{Linear systems}
Matrices are powerful for many reasons. Once such reason is that they can be used to represent linear systems of equations. This next section focuses on better ways to solve systems of linear equations by manipulating matrices.

\section*{Elementry row operations}
In linear algebra there are three elementary row operations on matrices: switching two rows, multiplying a row by a constant, and adding a multiple of one row to another row. Each of these operations can, in theory, be done through left multiplication by the appropriate elementary matrix. This approach is \emph{extremely} slow in practice.
It is much faster to perform these operations directly by modifying only the portions of an array that change as a result of the row operation.

The following code shows how these modifications can be made in-place to an array.
\lstinputlisting[style=fromfile]{row_opers.py}

\section*{Programming Row Reduction}
Solving a linear system can be done most efficiently by using elementary row operations to reduce a matrix to \emph{row echelon form} (REF), as opposed to \emph{reduced row echelon form} (RREF).
Consider the following matrix:

\[
\begin{pmatrix}
4&5&6&3 \\
2&4&6&4 \\
7&8&0&5
\end{pmatrix}
\]

Using elementary row operations, we can reduce $A$ to REF as follows:

\begin{lstlisting}
>>> import numpy as np
>>> A = np.array([[4., 5., 6., 3.],[2., 4., 6., 4.],[7., 8., 0., 5.]])
array([[ 4.,  5.,  6.,  3.],
       [ 2.,  4.,  6.,  4.],
       [ 7.,  8.,  0.,  5.]])
>>> A[1] -= (A[1,0]/A[0,0]) * A[0]
>>> A[2] -= (A[2,0]/A[0,0]) * A[0]
>>> A[2,1:] -= (A[2,1]/A[1,1]) * A[1,1:]
>>> A
array([[ 4. ,  5. ,  6. ,  3. ],
       [ 0. ,  1.5,  3. ,  2.5],
       [ 0. ,  0. , -9. ,  1. ]])
\end{lstlisting}

The additional requirement is often added that the first nonzero entry of each row be 1. Do not worry about that requirement here.
Notice that in our third row operation we were able to operate on only a portion of the third row because we knew that the first value would still be 0. In this case it made little difference, but it is good to watch for things like this because they can save a great deal of time when working with larger matrices.

A brief discussion of some potential pitfalls is in order.
Round-off error can lead to serious numerical issues. Consider the
following variation on the above example.

\begin{lstlisting}
>>> A = np.array([[4., 5., 6., 3.],[2., 2.5, 6., 4.],[7., 8., 0., 5.]])
array([[ 4.,  5.,  6.,  3.],
       [ 2.,  2.5,  6.,  4.],
       [ 7.,  8.,  0.,  5.]])
>>> A[1] -= (A[1,0]/A[0,0]) * A[0]
>>> A[2] -= (A[2,0]/A[0,0]) * A[0]
\end{lstlisting}

If we work this out by hand, we should currently have

\[
\begin{pmatrix}
4&5&6&3 \\
0&0&3&2.5 \\
0&-7.5&-10.5&-.25
\end{pmatrix}.
\]

All that is left is to swap the second and third rows, and the matrix is in row echelon form. However, suppose that due to numerical round-off error, the machine instead computes:

\[
\begin{pmatrix}
4&5&6&3 \\
0&10^{-15}&3&2.5 \\
0&-7.5&-10.5&-.25
\end{pmatrix}.
\]

The algorithm would then attempt to pivot on the \li{A[1,1]} entry:

\begin{lstlisting}
>>> A[2,1:] -= (A[2,1]/A[1,1]) * A[1,1:]
>>> A
array([[ 4. ,  5.0e+00 , 6.00e+00 ,  3.000e-00 ],
       [ 0. ,  1.0e-14 , 3.00e+00 ,  2.500e-00 ],
       [ 0. ,  0.0e+00 , 2.25e+14 ,  1.875e+14 ]])
\end{lstlisting}

The round-off error in the \li{A[1,1]} entry has affected the third
row, and the matrix is now much different than our first calculation. In larger matrices, such an error could potentially propagate through many steps in the calculation, resulting in garbage output.

This example also illustrates another issue to be aware of. Even if there were no round-off error in the \li{A[1,1]} entry, a naive implementation of the algorithm might still attempt to pivot on that entry, which would result in division by zero. As noted above, we would first need to swap rows 2 and 3 in order to proceed. Dealing with leading zeros by permuting the rows is another consideration when designing a fool-proof REF solver.

\begin{problem}
\label{prob:REF}
Write a Python function which takes a matrix and reduces it to REF.
Assume that the matrix is invertible and ignore the possibility that a zero may appear on the main diagonal during row reduction. You are not
responsible for dealing with potential round-off errors.
\end{problem}

\section*{LU Decomposition}
LU Decomposition refers to a process for factoring a square matrix into the product of a lower triangular matrix and an upper triangular matrix. Such a factorization exists only for certain classes of matrices. In the case of an invertible $n \times n$ matrix $A$, the LU decomposition exists if and only if all of the leading principle minors are non-zero (that is, all of the submatrices \li{A[:k,:k]} for $k = 1,\ldots,n-1$ are invertible). Even in this case, however, it may still be necessary to permute the rows of the matrix in order to prevent zeros from appearing on the main diagonal. Thus, row swap operations are, in general, necessary for the LU decomposition.

Using row reduction we can reduce the matrix $A$ to upper triangular form. Say this can be done in $k$ row operations.
Let $U$ be the upper triangular form of $A$, so we have:

\[
U = E_k \dots E_2 E_1 A.
\]

Since the elementary matrices are invertible, we also have

\[
(E_k \dots E_2 E_1)^{-1} U =  A.
\]

Then we define $L$ to be

\[
L = (E_k \dots E_2 E_1)^{-1}
\]

which is the same as

\[
L = E_1^{-1} E_2^{-1} \dots E_k^{-1}
\]

In either case we have $L U = A$.

The inverses of elementary matrices are also elementary matrices. Thus, $L$ can be computed by applying a series of simple operations to an identity matrix.
Note when we are only doing type 3 row operations, each of the operations represented by right multiplication by these inverse matrices results in the change of a single entry in $L$.

In practice, the LU decomposition of an array $A$ can be computed like this:

\begin{itemize}
\item Make a copy, $U$, of $A$.
\item Make an identity matrix $L$ that is the same shape as $A$.
\item Iterate through the entries below the diagonal of $U$.

Now for each entry below the main diagonal of $U$ do the following:
	\begin{itemize}
	\item Set the corresponding entry of $L$ to the quotient of the current entry of $U$ and the entry of the main diagonal of $U$ located above the current entry.
	\item Perform the type 3 row operation to set the current entry of $U$ to 0.
		Remember to avoid computation involving columns that have already been processed.
	\end{itemize}
\item Return $L$ and $U$
\end{itemize}

In this case, we have ignored the possibility that a 0 may appear along the main diagonal during computation. A full implementation of the LU decomposition would have to account for this possibility as well by swapping rows appropriately.

\section*{Why This Matters}
The LU decomposition is more efficient for solving linear systems than traditional row reduction and also allows for quick computation of inverses and determinants. For very large matrices, the LU decomposition can be performed without using any extra space as follows: $L$ is stored above the main diagonal of the array and $U$ is  stored below it.
Note there is no need to store the main diagonal of $L$ since all its entries are ones.

\begin{problem}
\label{prob:LU}
Write a function takes in an $n\times n$ matrix, performs the LU decomposition, and returns $L$ and $U$.
To verify that it works, multiply $L$ and $U$ together and compare to $A$.
Assume that the matrix is invertible and ignore the possibility that a zero may appear on the main diagonal during row reduction.

Write another version of the function that modifies its input in place, storing $L$ below the main diagonal and $U$ in the rest of the array.
\end{problem}

As noted earlier, in general one needs to consider row-swapping when
calculating the LU decomposition. This means that it is not always possible to simply have

\[
A = LU,
\]
but rather

\[
PA = LU,
\]

where $P$ is a permutation matrix representing the necessary row-swaps. Given such a decomposition, we can solve the linear system $Ax = b$ by
first solving $Ly = Pb$ and then $Ux = y$. Since $L$ and $U$ are triangular, these systems can be solved easily with backward and forward substitution. We can use this technique to calculate $A^{-1}$ by solving the matrix equation $LUX = P$ column-by-column. Finally, we can calculate the determinant of $A$ via the formula

\[
\det(A) = (-1)^S\left(\displaystyle\prod_{i=1}^nu_{ii}\right),
\]

where $S$ is the number of row-swaps.

SciPy includes a complete implementation of the LU decomposition in the \li{linalg} module. Furthermore, several other methods in the \li{linalg} module are based on the LU decomposition, including \li{linalg.solve}, \li{linalg.inv}, and \li{linalg.det}.

\begin{problem}
\label{prob:Solve}
Suppose there is a need to solve the system $Ax = b$ for fixed $A$ and many different values of $b$. It would be unwise to perform row-reduction for every single new value of $b$, and the LU decomposition helps us avoid this and save time.

Create a random $1000 \times 1000$ array $A$, and a random $1000 \times 500$ array $B$. Use the \li{scipy.linalg.lu_factor} function to compute the LU factorization of $A$. Then use the \li{scipy.linalg.lu_solve} function to solve the system $AX = B$, and time this operation.

Now use the \li{scipy.linalg.inv} function to compute the inverse of $A$, and then use it to solve the system $AX = B$. Again time the operation.

\end{problem}

%\begin{problem}
%\label{prob:lusolve}
%Write a function that takes the LU decomposition computed by the second function you made in Problem \ref{prob:LU} and another array representing the right hand side of a linear system and modifies the second array in place so that it represents the solution to the linear system.
%No changes to the array storing the LU decomposition are necessary.
%\end{problem}
%

\begin{problem}
\label{prob:det}
Write a function which takes as input a square matrix $A$ and uses \li{linalg.lu_factor} to find the determinant of $A$. Note that \li{linalg.lu_factor} returns a square array \li{lu} that contains $U$ in its upper triangle, as well as an array \li{piv} that represents the
permutation matrix $P$. Use \li{piv} to compute the number of row swaps, and use the \li{diagonal} method on \li{lu} to compute the product of the diagonal terms, as needed in the formula for the determinant.

Hint: if the $i^{th}$ entry of \li{piv} does not equal $i$, this means that a row swap occurred.
\end{problem}

\section*{The Cholesky Decomposition (Optional)}

Under certain conditions, the Cholesky decomposition offers a more efficient alternative to the LU decomposition.
It requires half the number of calculations and half the memory that the standard LU decomposition needs.
Furthermore, it is a \emph{numerically stable} decomposition, which means that round-off and truncation errors are kept suitably under control, rather than growing and propagating throughout the computation.
Because of the efficiency and numerical stability, Cholesky decomposition is used in solving least squares, optimization, and state estimation problems.
The Cholesky decomposition, however, is only applicable to Hermitian (for real matrices, this means symmetric) positive definite matrices.
It can be thought of as the matrix equivalent to taking the square root of a positive real number.

The Cholesky decomposition of a $A$ is a lower-triangular matrix, $L$, such that

\begin{equation*}
 A = LL^*
\end{equation*}

Where $L^*$ is the conjugate transpose of $L$.
For real valued matrices, this is equivalent to $L^T$.

The entries of $L$ are calculated as follows.

\begin{align*}
&L_{i,j} = \frac{1}{L_{j,j}}\left(A_{i,j} -\sum_{k=1}^{j-1}{L_{i,k}L_{j,k}^*}\right) \mbox{ for $i>j$} \\ \\
&L_{i,i} = \sqrt{A_{i,i} - \sum_{k=1}^{i-1}{L_{i,k}L_{i,k}^*}}
\end{align*}

where $L^*$ denotes the conjugate transpose of $L$.

Notice that in this computation, current calculation will depend on previous calculations. To calculate $L$ properly, you must start in the upper left corner and iterate down.

Note: when testing positive definite systems, an easy way to generate a random symmetric positive definite matrix is by generating a random array \li{A} and then computing \li{A.dot(A.T)}.

\begin{problem}
Write your own implementation of the Cholesky decomposition.
Test it using a random symmetric matrix (build a random square matrix $A$, then $A^TA$ will be positive definite).
Check the output of your function to ensure that it is functioning properly.
\end{problem}

%\begin{problem}
%Modify your previous answer so that it computes the Cholesky decomposition by modifying the array in place.
%Make sure you set the portion of the array above the main diagonal to 0.
%Then write a function that takes this reduced form of the array and uses it to solve a linear system by back substitution.
%This should be nearly the same as Problem \ref{prob:lusolve}.
%\end{problem}

The linalg module of SciPy also includes a Cholesky decomposition that should be much faster than the one you just implemented. It works much like the LU decomposition, providing the methods \li{cho_factor} and \li{cho_solve}.

\begin{problem}
Repeat the steps in problem \ref{prob:loopSolve}, this time using the
Cholesky decomposition. Make sure the input matrix to be factored is
positive definite.
\end{problem}


\begin{tabular}{lcll}\hline
Variate    & $x$         & \ccode{double} &  $\mu \leq x < \infty$ \\
Location   & $\mu$       & \ccode{double} &  $-\infty < \mu < \infty$\\
Scale      & $\lambda$   & \ccode{double} &  $\lambda > 0$ \\
Shape      & $\tau$      & \ccode{double} &  $\tau > 0$ \\ \hline
\end{tabular}

The probability density function (PDF) is:

\begin{equation}
P(X=x) =  \frac{\lambda^{\tau}}{\Gamma(\tau)}  (x-\mu)^{\tau-1}  e^{-\lambda (x - \mu)}
\label{eqn:gamma_pdf}
\end{equation}

The cumulative distribution function (CDF) does not have an analytical
expression. It is calculated numerically, using the incomplete Gamma
function (\ccode{esl\_stats\_IncompleteGamma()}).

The ``standard Gamma distribution'' has $\mu = 0$, $\lambda = 1$.

\subsection{Sampling}

\subsection{Parameter estimation}

\subsubsection{Complete data; known location}

We usually know the location $\mu$. It is often 0, or in the case of
fitting a gamma density to a right tail, we know the threshold $\mu$
at which we truncated the tail.

Given a complete dataset of $N$ observed samples $x_i$ ($i=1..N$) and
a \emph{known} location parameter $\mu$, maximum likelihood estimation
of $\lambda$ and $\tau$ is performed by first solving this rootfinding
equation for $\hat{\tau}$ by binary search:

\begin{equation}
  \log \hat{\tau}
  - \Psi(\hat{\tau})
  - \log \left[ \frac{1}{N} \sum_{i=1}^{N} (x_i - \mu) \right]
  + \frac{1}{N} \sum_{i=1}^N \log (x_i - \mu)
\label{eqn:gamma_tau_root}
\end{equation}

then using that to obtain $\hat{\lambda}$:

\begin{equation}
\hat{\lambda} = \frac{N \hat{\tau}} {\sum_{i=1}^{N} (x_i - \mu)}
\end{equation}

Equation~\ref{eqn:gamma_tau_root} decreases as $\tau$ increases.


\chapter{Generating Function}
\label{chapter:generating-functions}
In this chapter we discuss the basics of one of the most general methods we have
in combinatorics, the method is called ``generating functions''. The core idea
of this method is to use knowledge we have about mathematical analysis in
combinatorics.

\section{Easy Two Term Recurrences}
Let us start from the following problem.
Sasha took an insane credit in a bank: he took $100$\$ at the beginning and
his debt is growing twofold every year. At the beginning of each year John is
paying $100$\$ to the bank. How big will be his debt in 5 years?

It is easy to see that the answer for this and similar questions can be answered
using a recurrence relation. Indeed, if $a_i$ denotes his debt on $i$th year,
then $a_0 = 100$, and $a_{n + 1} = 2 a_n - 100$. Using this, one may compute
all the values of $a_i$. However, the question became tricky if we want to
find an explicit formula for $a_i$.

To solve this kind of questions we can use beforementioned generating functions.
\begin{definition}
  Let $\set{c_n}_{n \ge 0}$ be a sequence of real numbers. Then
  the generating function for this sequence is the power series
  $F(x) = \sum_{n \ge 0} c_n x^n$.
\end{definition}
Note that these power series may not converge for $x \neq 0$.
In this chapter, we will not discuss this problem and always pretend that
they are converging, for a formal explanation of how to deal with this issue see
Appendix~\ref{chapter:formal-power-series}.

Let us use the definition of $a_i$ to find the generating function $G(x)$
for this sequence. Note that
$a_{n + 1} x^{n + 1} = 2 a_n x^{n + 1} - 100 x^{n + 1}$. Thus
\[
  \sum_{n \ge 0} a_{n + 1} x^{n + 1} =
    \sum_{n \ge 0} 2 a_n x^{n + 1} - 100 \sum_{n \ge 0} x^{n + 1}.
\]
The left-hand side is equal to $G(x) - a_0$ and the right-hand side is
equal to $2xG(x) - \frac{100 x}{1 - x}$. So we can derive the equality
\[
  G(x) - 100 = 2xG(x) - \frac{100 x}{1 - x}.
\]
Using this equality we can find explicitly a formula for $G(x)$,
\[
  G(x) = \frac{100}{1 - 2x} - \frac{100x}{(1 - x)(1 - 2x)}.
\]
Let us simplify the formula a bit.
\[
  G(x) = \frac{100}{1 - 2x} + \frac{100}{1 - x} - \frac{100}{1 - 2x} =
  \frac{100}{1 - x}.
\]
Thus $G(x) = \sum_{n \ge 0} 100 x^n$. As a result, $a_n = 100$.

\begin{exercise}
  Find a formula for $a_n$ in the case when $a_0 = 200$.
\end{exercise}

Let us consider another, more complicated, example. Consider a sequence
$\set{a_n}_{n \ge 0}$ such that $a_{n + 1} = 2a_n + n$ for $n \ge 0$ and
$a_0 = 1$. As in the previous case let us write an equation for the
generating function $G(x)$.
\[
  G(x) - a_0 = 2xG(x) + \sum_{n \ge 0} n x^{n + 1}.
\]
First, we find a formula for $\sum_{n \ge 0} n x^n$,
\[
  \sum_{n \ge 0} n x^{n + 1} = \sum_{n \ge 0} x^2 \cdot \frac{d x^n}{dx} =
  x^2 \cdot \frac{d \sum_{n \ge 0} x^n}{dx} = x^2 \left(\frac{1}{1 - x}\right)'
  = \frac{x^2}{(1 - x)^2}.
\]
Therefore,
\[
  G(x) = \frac{1 - 2x + 2x^2}{(1 - x)^2 (1 - 2x)}.
\]
So we need to find a more appropriate formula for $G(x)$.
Let us try to find a formula in the form
\[
  \frac{1 - 2x + 2x^2}{(1 - x)^2 (1 - 2x)} =
    \frac{A}{(1 - x)^2} + \frac{B}{1 - x} + \frac{C}{1 - 2x}.
\]
To find $A$, $B$, and $C$ we multiply both sides by $(1 - x)^2$ and set $x = 1$.
We get that $A = -1$. We can also multiply by $1 - 2x$ and substitute
$x = 1 / 2$ and derive that $C = 2$. Now we need to find $B$, we substitute $x =
0$ to the equation and get $B = 0$. As a result,
$G(x) = \frac{-1}{(1 - x)^2} + \frac{2}{1 - 2x}$. Using simple equalities from
calculus we can derive $G(x) = \sum_{n \ge 0} -(n + 1) + 2^{n + 1} x^n$.
So $a_n = -(n + 1) 2^{n + 1}$.

\section{Recurrences With Two Variables}
To illustrate how to deal with recurrence relations in cases when we have
more than one variable, we prove a version of the binomial theorem and
derive a formula for binomial coefficients. In order to do it,
we consider the recurrence relation
\[
  \binom{n + 1}{k} = \binom{n}{k} + \binom{n}{k - 1}.
\]
Let us denote $\sum_{k \ge 0} \binom{n}{k} x^k$ by $B_n(x)$.
It is clear that
\[
  B_{n + 1}(x) - 1 = (B_n(x) - 1) + x B_n(x).
\]
Therefore, $B_{n + 1}(x) = (1 + x) B_n(x)$. As a result, $B_n(x) = (1 + x)^n$;
i.e. $\sum_{k \ge 0} \binom{n}{k} x^k = (1 + x)^n$. To find a
formula for binomial coefficients, we just need to use Taylor's formula,
$\binom{n}{k} = \frac{d^k}{dx^k} B_n(x)\big\rvert_{x = 0}  / k!$. So
$\binom{n}{k} = n (n - 1) \dots (n - k + 1) / k!$.

\section{Products of Generating Functions}
Let us consider a new problem, how many ways to design a class consisting of
$n$ lectures with theoretical part and laboratory part (the first $k$ days of
the quarter form the theoretical part, note that $k$ is not fixed) such that
there are two midterms during the theoretical part and one exam during the
laboratory part.

Let $a_n$ be the answer. It is easy to see that
\[
  a_n = \sum_{k = 1}^{n - 2} k \binom{n - k}{2}.
\]
However, this formula does not suggest an explicit formula.
Let us write an equation for the generating function for $a_n$,
\[
  G(x) = \sum_{n \ge 0} \sum_{k = 1}^{n - 2} k \binom{n - k}{2} x^n.
\]
It is easy to see that this formula implies that
\[
  G(x) = \left( \sum_{k \ge 0} k x^k \right)
    \left( \sum_{k' \ge 0} \binom{k'}{2} x^{k'} \right).
\]
Thus
\[
  G(x) = \frac{x}{(1 - x)^2} \cdot \frac{x^2}{(1 - x)^3} =
  \frac{x^4}{(1 - x)^5} = x^3\sum_{n \ge 0} \binom{n + 4}{4} x^n.
\]
As a result, $a_n = \binom{n + 1}{4}$.

Using this example, we can formulate a general rule.
\begin{theorem}
  Let $a_n$ be the number of ways to build a certain structure on an $n$-element
  set, and let $b_n$ be the number of way to build another structure on an
  $n$-element set. Let $c_n$ be the number of ways to separate $[n]$ into two
  parts consisting of numbers $\set{1, \dots, k}$ and $\set{k + 1, \dots, n}$
  ($k \ge 0$), and then to build a structure of the first type on the first set,
  and a structure of the second type on the second set.

  Then $H(x) = F(x)G(x)$, where $F(x)$, $G(x)$, and $H(x)$ are generating
  functions for $\set{a_n}_{n \ge 0}$, $\set{b_n}_{n \ge 0}$, and
  $\set{c_n}_{n \ge 0}$, respectively.
\end{theorem}

To illustrate this theorem, let us solve another problem. A company
``bolshoy brat'' needs to finish two projects. To do this, a manager of the
company splits all the employees into two projects and in each project she selects
product team and marketing team. How many ways to do this.
Let $c_n$ be the number of ways the manager can complete this task. Again, let
us split the problem into two parts. Let $A(x)$ be the generating function for
the number of ways to split people in the first project into marketing and
product teams. It is clear that $A(x) = \sum_{k \ge 0} 2^k x^k = 1 / (1 - 2x)$
since any $k$ element set has $2^k$ subsets. It is easy to see that the second
project has the same generating function. Thus the generating function for
$\set{c_n}_{n \ge 0}$, $C(x) = A(x) A(x) = 1 / (1 - 2x)^2$.
As a result,
\[
  C(x) = \frac{1}{2} \sum_{n \ge 1} n 2^n x^{n - 1} =
  \frac{1}{2} \sum_{n \ge 0} (n + 1) 2^{n + 1} x^n
\]
and $c_n = (n + 1) 2^{n + 1}$.

\begin{exercise}
  Find the number of ways to split an $n$-day semester into three parts, choose
  any number of holidays in the first part, an odd number of holidays in the
  second part, and an even number of holidays in the third part.
\end{exercise}

\section{Compositions of Generating Functions}
As usual, we start the section from a problem.
All $n$ soldiers of a military squadron stand in a line. The officer in charge
splits the line at several places, forming (non-empty) squads. Then she names
one person in each unit to be the commander of that unit. Let $c_n$ be the
number of ways she can do this. Find an explicit formula for $c_n$.

If the officer splits the soldiers into $k$ squads, then there
are
\[
  \sum_{n_1, \dots, n_k : n = n_1 + \dots + n_k} n_1 \cdot n_2 \dots n_2
\]
ways to do this. Hence, the generating function for splitting into squads and
selecting commanders in all $k$ squads is equal to $A^k(x)$, where
$A(x) = \sum_{n \ge 0} n x^n = \frac{x}{(1 - x)^2}$. Therefore,
the generating function $C(x)$ for $\set{c_n}_{n \ge 0}$ is equal to
$\sum_{k \ge 1} A^k(x)$. As a result,
\[
  C(x) = \frac{1}{1 - A(x)} = 1 + \frac{x}{1 - 3x + x^2}.
\]
It is possible to note that the roots $\alpha$ and $\beta$ of $x^2 - 3x + 1$ are
equal to $(3 \pm \sqrt{5}) / 2$, respectively. We want to find $A$ and $B$ such
that
\[
  \frac{1}{1 - 3x + x^2} = \frac{A}{x - \alpha} - \frac{B}{x - \beta}.
\]
Thus $1 = (A - B) x - A\beta + B\alpha$. Therefore, we have
$A = B$ and $A(\alpha - \beta) = A\sqrt{5} = 1$; i.e.
$A = B = \frac{1}{\sqrt{5}}$.
By some simple calculations we may conclude that
\[
  \frac{1}{1 - 3x + x^2} =
  \frac{1}{\sqrt{5}}(\frac{\alpha}{1 - \alpha x} - \frac{\beta}{1 - \beta x}).
\]
Therefore
$C(x) = 1 + \frac{1}{\sqrt{5}}
  \sum_{n \ge 0} (\alpha^{n + 1} - \beta^{n + 1}) x^{n + 1}$.
Hence, $c_0 = 1$ and $c_n = \frac{1}{\sqrt{5}}(\alpha^n - \beta^n)$ for $n > 0$.

The following theorem generalises this observation.
\begin{theorem}
  Let $a_n$ be the number of ways to build a certain structure on an $n$-element
  set, and let us assume that $a_0 = 0$. Let $c_n$ be the number of ways to
  split the set $[n]$ into an unspecified number of disjoint non-empty
  intervals, then build a structure of the given type on each of these
  intervals. Set $h_0 = 1$. Denote $F(x) = \sum_{n \ge 0} a_n x^n$ and
  $G(x) = \sum_{n \ge 0} c_n x^n$. Then $G(x) = \frac{1}{1 - A(x)}$.
\end{theorem}

\begin{chapterendexercises}
  \exercise[recommended] Find the generating functions of each of the following sequences
    (in the simplest form):
    \begin{enumerate}[nolistsep]
      \begin{multicols}{2}
        \item $a_n = n$;
        \item $a_n = \alpha n + \beta$;
        \item $a_n = n^2$;
        \item $a_n = \alpha n^2 + \beta n + \gamma$;
        \item $a_n = 3^n$.
      \end{multicols}
    \end{enumerate}
  \exercise[recommended] Let $F(x)$ be a generating function for the sequence
    $\set{a_n}_{n \ge 0}$. Write, in terms of $F(x)$, the generating functions
    of the following sequences:
    \begin{enumerate}[nolistsep]
      \begin{multicols}{2}
        \item $\set{a_n + \alpha}_{n \ge 0}$;
        \item $\set{\alpha a_n + \beta}_{n \ge 0}$;
        \item $\set{n a_n}_{n \ge 0}$;
        \item $0$, $a_1$, \dots, $a_n$, \dots;
        \item $a_1$, \dots, $a_n$, \dots;
        \item $\set{a_{n + m}}_{n \ge 0}$ ($m$ is a constant).
      \end{multicols}
    \end{enumerate}
  \exercise Let $k$ be a positive integer. Find an explicit formula for the
    generating function for the sequence $\set{\binom{n}{k}}_{n \ge 0}$.
    \begin{solution}
      Let us consider a generating function $F(x) = \sum_{n \ge 0} x^n$. Note that
      $F^{(k)}(x) = \sum_{n \ge k} n (n - 1) (n - 2) \dots (n - k + 1) x^{n - k}$.
      Therefore, $\frac{x^k}{k!} F^{(k)}(x) = \sum_{n \ge 0} \binom{n}{k} x^n$. It
      is also easy to see that $F^{(k)}(x) = \frac{k!}{(1 - x)^{k + 1}}$. As a
      result, $\sum_{n \ge 0} \binom{n}{k} x^n = \frac{x^k}{(1 - x)^{k + 1}}$.
    \end{solution}
  \exercise Let $f(n)$ be the number of subsets of $\range{n}$ that contain no two
    consecutive elements, for integer $n$. Find the recurrence that is satisfied
    by these numbers, and then find an explicit formula for these numbers.
  \exercise Find an explicit formula for $a_n$ if $a_0 = 0$ and for any
    $n \ge 0$, $a_{n + 1} = a_n + 2^n$.
  \exercise[recommended] Let $a_n$ be the number of ways to pay $n$ dollars using ten-dollar
    bills, five-dollar bills, and one-dollar bills only. Find the generating
    function for $a_n$.
  \exercise[recommended] Let $x_1$ and $x_2$ be two different solutions of the equation
    $1 - bx - cx^2 = 0$. Show that a sequence $\set{f_n}_{n \ge}$ satisfies the
    recurrence relation $f_{n + 2} = b f_{n + 1} + c f_n$ iff
    $t_n = \alpha x_1^{-n} + \beta x_2^{-n}$ for some $\alpha, \beta \in \R$.
  \exercise[recommended] Let $\set{a_n}_{n \ge 0}$, $\set{b_n}_{n \ge 0}$ be two sequences
    such that $b_n = \sum_{k = 0}^n a_n$ and $F(x)$ be the generating function
    for $\set{a_n}_{n \ge 0}$. Find the generating function for
    $\set{b_n}_{n \ge 0}$ in terms of $F(x)$.
  \exercise Let $\set{a_n}_{n \ge 0}$, $\set{b_n}_{n \ge 0}$ be two sequences
    such that $b_n = a_{2n}$ and $F(x)$ be the generating function
    for $\set{a_n}_{n \ge 0}$. Find the generating function for
    $\set{b_n}_{n \ge 0}$ in terms of $F(x)$.
\end{chapterendexercises}


%!TEX root = ./writeup.tex

\subsection{Diamond Difference Discrete Ordinates}
	The \gls{dd} \SN equations corresponding to Eq. \ref{eq:sn} are
		\begin{equation} \label{eq:dd}
			\frac{\mu_n}{h_i}\left(\psi_{n,i+1/2} - \psi_{n,i-1/2}\right)
				+ \Sigma_{t,i} \psi_{n,i} = \frac{\Sigma_{s,i}}{2}\sum_{n'=1}^N \psi_{n',i}w_{n'}
				+ \frac{Q_i}{2} , \ 1 \leq n \leq N, \ 1 \leq i \leq I
		\end{equation}
	where $\psi_{n,i\pm1/2} = \psi_n(x_{i\pm1/2})$ is the cell edge angular flux and $\Sigma_{t,i} = \Sigma_t(x_i)$, $\Sigma_{s,i} = \Sigma_s(x_i)$ and $Q_i = Q(x_i)$ the cell averaged total cross section, scattering cross section and fixed source. The $x_{i\pm1/2}$ are the cell edge locations of cell $i$ of cell width $h = x_{i+1/2} - x_{i-1/2}$. In DD, the cell centered angular flux is taken to be the average of the adjacent cell edge angular fluxes:
		\begin{equation} \label{eq:auxDD}
			\psi_{n,i} = \frac{1}{2} \left(\psi_{n,i+1/2} + \psi_{n,i-1/2}\right).
		\end{equation}
	Using this result for the scattering term yields
		\begin{equation}
		\begin{aligned}
			\frac{\Sigma_{s,i}}{2}\sum_{n'=1}^N \psi_{n',i}w_{n'} &=
			\frac{\Sigma_{s,i}}{2}\sum_{n'=1}^N
				\frac{1}{2} \left(\psi_{n,i+1/2} + \psi_{n,i-1/2}\right) w_{n'} \\
			&= \frac{\Sigma_{s,i}}{4} \left(\phi_{i-1/2} + \phi_{i+1/2}\right)
		\end{aligned}
		\end{equation}
	In SI, the scattering term is lagged:
		\begin{equation} \label{eq:DDSN}
			\frac{\mu_n}{h_i}\left(\psi_{n,i+1/2}^{\ell+1} - \psi_{n,i-1/2}^{\ell+1}\right)
				+ \Sigma_{t,i} \psi_{n,i}^{\ell+1} =
				\frac{\Sigma_{s,i}}{4} \left(\phi_{i-1/2}^\ell + \phi_{i+1/2}^\ell\right)
				+ \frac{Q_i}{2}, \
			1 \leq n \leq N, \ 1 \leq i \leq I
		\end{equation}
	Solving for $\psi_{n,j\pm1/2}^{\ell+1}$ yields
		\begin{subequations}
			\begin{equation} \label{eq:psiplus}
				\psi_{n,i+1/2}^{\ell+1} =
				\frac{
				\frac{\Sigma_s}{2}h_i \left(\phi_{i-1/2}^\ell + \phi_{i+1/2}^\ell\right)
				+ Q_i h_i - \left(\Sigma_t h_i - 2\mu_n\right) \psi_{n,i-1/2}^{\ell+1}
				}
				{
				\Sigma_t h_i + 2\mu_n
				}, \ \mu_n > 0
			\end{equation}
			\begin{equation} \label{eq:psiminus}
				\psi_{n,i-1/2}^{\ell+1} =
				\frac{
				\frac{\Sigma_s}{2}h_i \left(\phi_{i-1/2}^\ell + \phi_{i+1/2}^\ell\right)
				+ Q_i h_i -
					\left(\Sigma_t h_i - 2|\mu_n|\right) \psi_{n,i+1/2}^{\ell+1}
				}
				{
				\Sigma_t h_i + 2|\mu_n|
				}, \ \mu_n < 0
			\end{equation}
		\end{subequations}
	Equation \ref{eq:psiplus} specifies the flux exiting the right side of cell $i$ given the flux that entered through the left side while Eq. \ref{eq:psiminus} specifies the flux exiting the left side of cell $i$ given the flux that entered through the right side.

	By specifying boundary conditions for $\psi_{n,1/2}^{\ell+1}$ for $\mu_n>0$ and $\psi_{n,I+1/2}^{\ell+1}$ for $\mu<0$, Eqs. \ref{eq:psiplus} and \ref{eq:psiminus} can be solved non-iteratively. The boundary conditions for a vacuum left boundary and reflecting right boundary are
		\begin{subequations}
		\begin{equation} \label{eq:leftBC}
			\psi_{n,1/2}^{\ell+1} = 0, \ \mu_n > 0
		\end{equation}
		\begin{equation} \label{eq:rightBC}
			\psi_{n,I+1/2}^{\ell+1} = \psi_{m,I+1/2}^{\ell+1}, \ \mu_n = -\mu_m.
		\end{equation}
		\end{subequations}

	Using Eq. \ref{eq:leftBC}, the flux exiting the right side of cell $i=1$, $\psi_{n,3/2}^{\ell+1}$, can be found through Eq. \ref{eq:psiplus}. This exiting flux is then the flux entering cell $i=2$ allowing for the determination of $\psi_{n,5/2}^{\ell+1}$. This process of using the result from the previous cell is repeated until $i=I$. At this point all rightward ($\mu>0$) moving flux has been determined for all cells $1 \leq i \leq I$.

	The reflecting boundary condition, Eq. \ref{eq:rightBC}, can now be applied. This sets the incoming flux on the right side of cell $i=I$. Equation \ref{eq:psiminus} then determines the exiting flux through the left side, $\psi_{n,I-1/2}^{\ell+1}$. Working backward from cell $i=I$, $\psi_{n,I-3/2}^{\ell+1}, \psi_{n,I-5/2}, \dots, \psi_{n,1/2}^{\ell+1}$ for $\mu_n < 0$ can be found.

	This process of propagating the solution from left to right for $\mu_n > 0$ and then from right to left for $\mu_n < 0$ is known as a transport sweep. At the end of the sweep, new cell edge scalar flux values $\phi_{i\pm1/2}^{\ell+1}$ are generated through
		\begin{equation}
			\phi_{i\pm1/2}^{\ell+1} = \sum_{n=1}^N \psi_{n,i\pm1/2}^{\ell+1} w_n.
		\end{equation}
	A new sweep is then conducted using $\phi_{i\pm1/2}^{\ell+1}$. This process is repeated until the stop criterion of
		\begin{equation}
			\frac{
			\sum_{i=0}^{I} \left(\phi_{i+1/2}^{\ell+1} - \phi_{i+1/2}^\ell\right)^2
			}{
			\sum_{i=0}^{I} \left(\phi_{i+1/2}^{\ell+1}\right)^2
			}
			< \epsilon
		\end{equation}
	is met.


\chapter{Random variables (TO DO)}
\todo{write chapter}
Having properly developed the Lebesgue measure
and the integral on it,
we can now proceed to develop random variables.

\section{Random variables}
With all this set-up, random variables are going to be really quick to define.
\begin{definition}
	A (real) \vocab{random variable} $X$ on a probability space
	$\Omega = (\Omega, \SA, \mu)$
	is a measurable function $X \colon \Omega \to \RR$,
	where $\RR$ is equipped with the Borel $\sigma$-algebra.
\end{definition}
In particular, addition of random variables, etc.\
all makes sense, as we can just add.
Also, we can integrate $X$ over $\Omega$, by previous chapter.

\begin{definition}
	[First properties of random variables]
	Given a random variable $X$,
	the \vocab{expected value} of $X$ is defined by
	the Lebesgue integral
	\[ \EE[X] = \int_{\Omega} X(\omega) \; d\mu. \]
	Confusingly, the letter $\mu$ is often used for expected values.

	The \vocab{$k$th moment} of $X$ is defined as $\EE[X^k]$,
	for each positive integer $k \ge 1$.
	The \vocab{variance} of $X$ is then defined as
	\[ \Var(X) = \EE\left[ (X-\EE[X])^2 \right]. \]
\end{definition}
\begin{ques}
	Show that $\mathbf{1}_A$ is a random variable
	(just check that it is Borel measurable),
	and its expected value is $\mu(A)$.
\end{ques}

An important property of expected value you probably already know:
\begin{theorem}
	[Linearity of expectation]
	If $X$ and $Y$ are random variables on $\Omega$ then
	\[ \EE[X+Y] = \EE[X] + \EE[Y]. \]
\end{theorem}
\begin{proof}
	$\EE[X+Y] = \int_\Omega X(\omega) + Y(\omega) \; d\mu
	= \int_\Omega X(\omega) \; d\mu + \int_\Omega Y(\omega) \; d\mu
	= \EE[X] + \EE[Y]$.
\end{proof}
Note that $X$ and $Y$ do not have to be ``independent'' here:
a notion we will define shortly.

\section{Distribution functions}

\section{Examples of random variables}

\section{Characteristic functions}

\section{Independent random variables}

\section{\problemhead}
\begin{problem}
	[Equidistribution]
	Let $X_1$, $X_2$, \dots be i.i.d.\ uniform random variables on $[0,1]$.
	Show that almost surely the $X_i$ are equidistributed,
	meaning that
	\[ \lim_{N \to \infty} \frac{ \# \{1 \le i \le N \mid a \le X_i(\omega) \le b \}}{N}
		= b-a \qquad \forall 0 \le a < b \le 1 \]
	holds for almost all choices of $\omega$.
\end{problem}

\begin{problem}
	[Side length of triangle independent from median]
	Let $X_1$, $Y_1$, $X_2$, $Y_2$, $X_3$, $Y_3$
	be six independent standard Gaussians.
	Define triangle $ABC$ in the Cartesian plane
	by $A = (X_1,Y_1)$, $B = (X_2,Y_2)$, $C = (X_3,Y_3)$.
	Prove that the length of side $BC$
	is independent from the length of the $A$-median.
\end{problem}

% 18.175 has some other good pset problems that could go here


\newpage
\part{Unsupervised Learning}
\section{Clustering}
\begin{itemize}
	\item Definition: given a database $D = {t_1, \dots, t_n}$ of tuples and an integer value $k$, define a mappping $f: D \rightarrow {1, \dots, k}$ where each tuple $t_i$ is assigned to a cluster $K_j$.
	\item Input: a dataset with $n$ p-dimensional data instances.

	Output: a \textbf{natural partitioning} of the dataset into $k$ clusters and noise
	\item Clustering VS. Classification
	\begin{table}[H]
		\begin{center}
			\begin{tabular}{|l|p {0.3\linewidth}|p {0.35\linewidth}|}
				\hline
				Characteristics        & \textbf{Classification}  & \textbf{Clustering}   \\ \hline
				Learning     & supervised  & unsupervised \\ \hline
				Target       & known & unknown, no dependent variables \\ \hline
				Training  	 & training data and training phase exists  & no training data/training phase, no labels/true classes  \\ \hline
			\end{tabular}
		\end{center}
	\end{table}
	\item key questions:
	\begin{itemize}
		\item the right number of clusters k
		\item identification of class membership between instances
	\end{itemize}
	\item Issues:
	\begin{itemize}
		\item interpreting results
		\item evaluating results: high intra-similarity within cluster, low inter-similarity across cluster?
		\item outlier
		\item number of clusters k
		\item scalability of algorithms
	\end{itemize}
\end{itemize}

\subsection{Hierarchical Clustering : Minimum Spanning Tree}
\begin{itemize}
	\item Input: a database D with tuples, \textbf{adjacency matrix} based on distances

	Output: \textbf{dendogram}
	\item Methods of building a dendogram:
	\begin{itemize}
		\item top-down
		\item bottom-up
	\end{itemize}
	\item Algorithms: with adjacency matrix, \textbf{each instance} can be seen as \textbf{node}, \textbf{distance} to other instances can be seen as \textbf{weighted edges}

	$\rightarrow$ graph problem

	$\rightarrow$ compute \textbf{Minimum Spanning Tree}, bottom-up method.
	\begin{itemize}
		\item Kruskal's algorithm: $\mathcal{O}(d\log(d))$
		\item Prim's algorithm: $\mathcal{O}(d\log(n))$
	\end{itemize}
	$\rightarrow$ each hierarchical level shares the same distance/weight.

	\item Distance measures in adjacency matrix:
	\begin{itemize}
		\item \textbf{Euclidean distance} between instance $p_1$ and $p_2$
		$$d_E(p_1,p_2) = \sqrt{(x_{p1} - x_{p2})^2 + (y_{p1} - y_{p2})^2 +\dots }$$
		\item \textbf{Manhattan distance} between instance $p_1$ and $p_2$
		$$d_M(p_1,p_2) = |x_{p1} - x_{p2}| + |y_{p1} - y_{p2}| + \dots$$
	\end{itemize}
\end{itemize}

\subsection{Partitional Clustering for Numeric Data: K-Means}
\begin{itemize}
	\item Input:
	\begin{itemize}
		\item a database D with tuples
		\item $k$ number of clusters
	\end{itemize}

	Output: k partitioned clusters
	\item Process:
	\begin{itemize}
		\item Initialization: randomly picked k centers
		\item compute the distance between each instance to the centers, assign instance to the \textbf{nearest center}.
		\item \textbf{update} the center of the clusters: \textbf{mean of assigned instances}
		\item repeat step 2-3 until convergence.
	\end{itemize}

	\item Advantages:
	\begin{itemize}
		\item simple
		\item items automatically assigned to clusters
	\end{itemize}
	Disadvantages:
	\begin{itemize}
		\item number of clusters $k$ must be predefined
		\item result significantly depends on \textbf{initial choice of centers}

		$\rightarrow$ traps in \textbf{local minimum}

		$\rightarrow$ repeat algorithm by starting from different random centers (eg: Iterative Improvement)
		\item sensitive to \textbf{outliers}
	\end{itemize}
\end{itemize}

\subsection{Probabilistic Clustering: Expectation Maximization}
We only discuss the simplified case here: instance with single \textbf{numeric} attribute and 2 clusters A \& B.
\begin{itemize}
	\item Input: random assigned parameters for cluster A \& B, assume \textbf{normal distribution}
	\begin{itemize}
		\item A: $\mu_A, \sigma_A$, prior probability of instance in cluster A $Pr(A)$
		\item B: $\mu_B, \sigma_B$, prior probability of instance in cluster B $Pr(B) = 1 - Pr(A)$
	\end{itemize}
	Output: 2 clusters with assigned instances

	\item Process:
	\begin{itemize}
		\item \textbf{Expectation} step: calculate the probability for \textbf{all instances} in \textbf{each cluster}: \textbf{Bayes Theorem}
			$$Pr(A|x) = \frac{Pr(x|A) \cdot Pr(A)}{Pr(x)} ,\quad Pr(x|A) = \frac{1}{\sqrt{2\pi} \cdot \sigma_A} e^{-\frac{(x - \mu_A)^2}{2\sigma_A^2}}$$
			$$Pr(B|x) = \frac{Pr(x|B) \cdot Pr(B)}{Pr(x)} ,\quad Pr(x|B) = \frac{1}{\sqrt{2\pi} \cdot \sigma_B} e^{-\frac{(x - \mu_B)^2}{2\sigma_B^2}}$$
		\textbf{no need to pick cluster here!}
		\item \textbf{Maximization} step: update the parameters for cluster A \& B. calculate the weighted mean and weighted variance using \textbf{all instances}.
		$$w_{iA} = Pr(A|x), \quad  w_{iB} = Pr(B|x)$$
		$$\mu_A = \frac{w_{1A}x_1 + w_{2A}x_2 + \dots + w_{nA}x_n}{w_{1A} + w_{2A} + \dots + w_{nA}}$$
		$$\sigma_A = \sqrt{\frac{w_{1A} (x_1 - \mu_A)^2 + \dots + w_{nA} (x_n - \mu_A)^2}{w_{1A} + w_{2A} + \dots + w_{nA}}}$$
		analog to $\mu_B$ and $\sigma_B$

		$$Pr(A) = \frac{\Sigma w_A}{\Sigma w_A + \Sigma w_B}, \quad Pr(B) = 1 - Pr(A)$$
		\item repeat expectation and maximization step until convergence.
	\end{itemize}
	\item Limitation: can stuck in \textbf{local optimum}.

	$\rightarrow$ repeat algorithm by starting with \textbf{different initial parameters}.

	\item Extension of model:
	\begin{itemize}
		\item multiple clusters: calculate k normal distributions
		\item multiple attributes:
		\begin{itemize}
			\item independent: multiply probabilities of all attributes
			\item correlated: multivariate normal distribution
		\end{itemize}
		\item nominal attributes: create probability distribution
	\end{itemize}
\end{itemize}


\documentclass{article}
\usepackage{graphicx}
\usepackage{verbatim}
\usepackage[margin=1in]{geometry}
\usepackage{enumitem}
\usepackage{amsmath,amssymb}
\usepackage{bm}
\usepackage{stmaryrd}
\title{Understanding Reflux}
\author{B. Runnels and V. Agrawal}
\begin{document}
\maketitle
\setlength\parindent{0pt}
\def\flux{\operatorname{Flux}}
\def\Int{\operatorname{int}}

\section{Interpretation as integral of flux jump}
Let $\Omega\subset\mathbb{R}^n$ be the problem domain, and let $A:C^2(\Omega)\to C^0(\Omega)$ be a linear operator.
In the implementation of \texttt{amrex::MLNodeLaplacian}, $A$ has the form
\begin{align}
  A[\phi](\bm{x}) = \nabla\cdot\bm{\sigma}(\bm{x})\nabla\phi,
\end{align}
where $\bm{\sigma}:\Omega\to\mathbb{R}^{n\times n}$ is a coefficient matrix\footnote{Appears to be implemented in \texttt{amrex} as a vector, corresponding to a diagonal matrix}.
The corresponding {\it flux} of the operator $\bm{f}_A:C^1(\Omega)\to C^0(\Omega,\mathbb{R}^n)$ is
\begin{align}
  \bm{f}_A[\phi](\bm{x}) = \bm{\sigma}(\bm{x})\nabla\phi
\end{align}
The pointwise residual is defined as
\begin{align}
  r(\bm{x}) = b(\bm{x}) - A[\phi](\bm{x})
\end{align}
where $b:\Omega\to\mathbb{R}$ is the right hand side.
The {\it total residual} is the integral
\begin{align}
  R = \int_\Omega r(\bm{x})\,d\bm{x}
\end{align}
Let $B\subset\Omega$ represent the {\it refined region} and $\Omega\setminus B$ the {\it coarse region}.
By nature of the discrete algorithm, we assume that $\nabla\phi$ is discontinuous over the boundary $\partial B$.
\begin{align}\label{eq:resorig}
  R = \int_\Omega b(\bm{x})\,d\bm{x} - \int_{B} A[\phi](\bm{x})\,d\bm{x} - \int_{\Omega\setminus B} A[\phi](\bm{x})\,d\bm{x}
\end{align}
Using the divergence theorem this reduces to
\begin{align}
  R = \int_\Omega b(\bm{x})\,d\bm{x} - \int_{\partial \Omega} \bm{n}\cdot\bm{f}_A[\phi](\bm{x})\,d\bm{x} - \int_{\partial B} \bm{n}\cdot\bm{f}^{B}_A[\phi](\bm{x})\,d\bm{x} + \int_{\partial B} \bm{n}\cdot\bm{f}^{\Omega\setminus B}_A[\phi](\bm{x})\,d\bm{x}
\end{align}
where $\bm{n}$ is the outward-facing normal for $B$.
The superscripts on $\bm{f}$ indicate where the flux is computed: $\bm{f}_A^{B}$ is evaluated on $B$, $\bm{f}_A^{\Omega\setminus B}$ on $\Omega\setminus B$.
This is alternatively expressed using jump-bracket notation,
\begin{align}
  R =
  \int_\Omega b(\bm{x})\,d\bm{x}
  - \int_{\partial \Omega} \bm{n}\cdot\bm{f}_A[\phi](\bm{x})\,d\bm{x}
  - \int_{\partial B} \bm{n}\cdot\Big\llbracket\bm{f}_A[\phi](\bm{x})\Big\rrbracket\,d\bm{x}.
\end{align}
An additional application of the divergence theorem to the second term yelds the result
\begin{align}\label{eq:rhscfreflux}
  R =
  \underbrace{\int_\Omega b(\bm{x})\,d\bm{x} }_{\text{r.h.s.}}
  - \underbrace{\int_{\Int(\Omega\setminus B)} A[\phi](\bm{x})\,d\bm{x}}_{\text{coarse}}
  - \underbrace{\int_{\Int(B)} A[\phi](\bm{x})\,d\bm{x}}_{\text{fine}}
  - \underbrace{\int_{\partial B} \bm{n}\cdot\Big\llbracket\bm{f}_A[\phi](\bm{x})\Big\rrbracket\,d\bm{x}}_{\text{reflux}}.
\end{align}
We note that (\ref{eq:rhscfreflux}) is nearly identical to (\ref{eq:resorig}) except for the additional boundary integral.
This is the result of the discretization and subsequent loss of continuity at the boundary.
\begin{enumerate}
\item In the continuous case, the jump term $\llbracket\bm{f}_A\rrbracket$ would be zero and would consequently vanish.
  Including it explicitly accounts for the reflux contribution at the coarse/fine boundary.
\item Terms 2 and 3 are now over the {\it interiors} of $\Omega\setminus B,B$.
  This is a trivial distinction in the continuous case, but is meaningful in the discrete case as it indicates that the boundary nodes are not included in the coarse and fine contributions.
\end{enumerate}

\section{Multi-level stencil}

Consider a multi-level stencil in which the solution on the coarse level is $pc$ and on the fine level is $pf$.
We wish to estimate the second derivative in the $x$ direction at coarse/fine boundary node $(i,j)$ using both coarse and fine nodes.
We assume that $pc(i,j) = pf(2i,2j)$.

\begin{center}
  \includegraphics{stencil1}
\end{center}

We first compute finite difference first derivatives on the coarse and fine levels, denoting them as $dpc,dpf$, respectively.
The values ``live'' half way between the two stencil points, as shown below:

\begin{center}
  \includegraphics{stencil2}
\end{center}

The coarse derivative is just
\begin{align}
  pc = \frac{pc(i,j) - pc(i-1,j)}{dxc}.
\end{align}
The fine derivative is a weighted average:
\begin{align}
  pf =
  \frac{1}{2}\frac{pf(2i+1,j) - pc(i,j)}{dxf}
  + \frac{1}{4}\frac{pf(2i+1,j+1) - pc(i,j+1)}{dxf}
  + \frac{1}{4}\frac{pf(2i+1,j-1) - pc(i,j-1)}{dxf}
\end{align}
Then the second derivative at $(i,j)$ is computed by finite difference between the two first derivatives
\begin{align}
  ddpc(i,j) = \frac{pf - fc}{(dxf/2) + (dxc/2)}
\end{align}
and lives at the coarse fine node
\begin{center}
  \includegraphics{stencil3}
\end{center}
The location of this node means that the derivative is a combination of central and backward differencing.

\end{document}