%!TEX root = ../main.tex
%-------------------------------------------------------------------------------
\section{Improvements}\label{Computation}
%-------------------------------------------------------------------------------
The implementation of EKW models poses several computational challenges. Among them are numerical integration, global optimization, function approximation, and efficient parallelization. We now describe some of our efforts to align \verb+respy+ and \verb+estimagic+ with the state-of-the-art in computational methods. We have concluded our preparatory work and actively seek input from domain experts for further improvements and joint publication.
%-------------------------------------------------------------------------------
\subsection{Numerical integration}
%-------------------------------------------------------------------------------
The solution of EKW models requires the evaluation of millions of integrals to determine the future value of each action in each state. In \citet{Eisenhauer.2020c}, we draw on the extensive literature on numerical integration \citep{Davis.2007, Gerstner.1998} to improve the precision and reliability of their solution. The current practice in economics is to implement a random Monte Carlo integration which introduces considerable numerical error and computational instabilities \citep{Judd.2011}.\\

\noindent We consider the optimality equation in a generic time period $t$ to clarify the structure of the integral. Let $v^{\pi}_{t}(s_t, a_t)$ denote the action-specific value function of choosing action $a_t$ in state $s_t$ while continuing with the optimal policy going forward.
%
\begin{align*}
v^{\pi}_{t}(s_t, a_t) & = u_t(s_t, a_t) + \delta\,\E_{s_t} \left[\left.v^{\pi^*}_{t + 1}(s_{t + 1})\,\right\vert\,\mathcal{I}_t\,\right] \\
& =  u_t(s_t, a_t) + \delta\, \int_S v^{\pi^*}_{t + 1}(s_{t + 1})\, \diff p_t(a_t, s_t)\\
& =  u_t(s_t, a_t) + \delta\, \int_S \max_{a_{t + 1} \in A}\bigg\{v^{\pi^*}_{t + 1}(s_{t + 1}, a_{t + 1})\bigg\}\diff p_t(a_t, s_t).
\end{align*}

\noindent Let's consider an atemporal version of the typical integral from \citet{Keane.1997} as an example. As outlined earlier, individuals can choose among five alternatives. Each of the alternative-specific utilities is, in part, determined by a stochastic continuous state variable $\epsilon$. The transition of all other state variables $x$ is deterministic. This results in a five-dimensional integral of the following form:
%
\begin{align*}
   \int_{\epsilon}\, \max_{a\in A} \bigg\{v^{\pi^*}(x, \epsilon, a)\bigg\} \phi_{\mu, \Sigma}(\epsilon) \diff\epsilon \quad\forall\, x \in X,
\end{align*}

\noindent where $\epsilon$ follows a multivariate normal distribution with mean $\mu$, covariance matrix $ \Sigma$, and probability density function $\phi_{\mu, \Sigma}$.
%-------------------------------------------------------------------------------
\subsection{Global optimization}
%-------------------------------------------------------------------------------
The calibration of EKW models is challenging due to a large number of parameters and multiplicity of local minima. In \citet{Eisenhauer.2020b}, we draw on the literature on global optimization to assess and improve the reliability of the calibrations \citep{Locatelli.2013, Nocedal.2006}.\\

\noindent We conduct a benchmarking exercise using \citet{Keane.1994, Keane.1997} as a well-known and empirically-grounded test case. Depending on the calibration procedure, particular challenges arise. For example, while likelihood-based calibration requires smoothing of the choice probabilities, simulation-based calibration involves the optimization of a noisy function. We provide guidelines for selecting the appropriate algorithm in each setting and showcase diagnostics to assess the reliability of the calibration results.


\documentclass[letterpaper, 10pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amsthm,amssymb,scrextend}
\usepackage{fancyhdr}
\pagestyle{fancy}
\usepackage{silence}
\WarningFilter{latex}{You have requested package}
\input{ltx/pkg/preamble}

\begin{document}

\lhead{MAT224 Linear Algebra II}
\chead{Linear Transformations, Image \& Kernel}
\rhead{Week 04}

\title{Linear Algebra II \\ \Large{MAT224}}
\author{Lennart Döppenschmitt}
% \maketitle
% \tableofcontents

\section*{Linear Transformations Part II}%
\label{sec:title}

\textbf{Textbook:} Section 2.2

\lb
\textbf{Announcements}
\begin{itemize}
\item
The last hour before the lecture next week will be a review session!
Please collect your questions and email me if you would like to discuss anything
particular. (include MAT224 in subject, thanks)

\item
All vector spaces from now on, unless stated otherwise,
will be assumed to be finite dimensional.
\end{itemize}

\lb
Remember that a basis encodes a vector $\vec v ∈ V$ as an $n$-tuple.
We can use the same idea to encode linear transformations

\lb
\textbf{Example 2.2.2}
\lb
Let $V = W = \R^2$ with the standard basis $ \cb{\vec e_1, \vec e_2}$.
Define $ \map{V}[T]{W}$ by
\[ T(\vec e_1) = \vec e_1 + \vec e_2 \]
\[ T(\vec e_2) = \vec 2 e_1 - 2 \vec e_2 \]
\vspace{50pt}

\newpage
\lb
\textbf{Algorithm}
\lb
Given a linear transformation $ \map{V}[T]{W}$ given as a `formula',
this is how to compute its matrix in two chosen bases
\[ α = \cb{\vec{α}_1, \ldots \vec{α}_m} \]
of $V$ and
\[ β = \cb{\vec{β}_1, \ldots \vec{β}_n} \]
of $W$
\lb
\begin{enumerate}
    \item For each basis element $\vec{α}_i$ in $V$, compute $T(\vec{α}_i)$.
    \item Find the coorindate vector $γ^β(T(\vec{α}_i)) = [ T(\vec{α}_i) ]_β$.
    \item Assemble these coordinate vectors as columns in a matrix
\end{enumerate}

\lb
\textbf{Discussion}
\lb
Apply the above algorithm to find the matrix representing the derivative
$ \frac{d}{dx}$ from $\pol{2}{\R}$ to itself.
Choose the basis on $\pol{2}{\R}$ consisting of monomials $α = \cb{1, x, x^2}$.

\newpage
\lb
\textbf{Definition 2.2.6}
\lb
Let $T$ be a linear transformation between finite dimensional vector spaces $V$ and $W$
with bases $α$ and $β$ respectively.
The \emph{matrix of the linear transformation} $T$ with respect to bases $α$ and $β$ is the
matrix $[T]_α^β$ satisfying
\[ [T]_α^β \cdot [\vec v]_α = [T(\vec v) ]_β \]

\newpage
\lb
\textbf{Discussion}
\begin{enumerate}
    \item
        What does he `size' of the matrix $[T]_α^β$ depend on?
    \item
        What is the matrix of the identity transformation $ \map{V}[\tx{id}_V]{V}$?
\end{enumerate}

\lb
\textbf{Exmaple}
\lb
We compute the matrix of the linear transformation $ \map{\pol{3}{\R}}[\tx{ev}_2]{\R}$

\newpage
\lb
\textbf{Discussion}
\lb
On the contrary, given a matrix $A ∈ \mat{n, m}{\R}$, does this give us a
linear transformation? What are the domain and codomain?

\vspace{300pt}
\lb
\textbf{Summary}
\begin{enumerate}
    \item
        The upshot of this section is that linear transformations are completely interchangable
        with matrices!
        The identification depends on the choice of bases for the domain and codomain.
    \item
        The operations
        \begin{itemize}
            \item matrix of a linear transformation $[T]_α^β$
            \item linear transformation of a matrix $T_A$
        \end{itemize}
        are inverse to each other.
\end{enumerate}

\newpage
\lb
We now rephrase the \emph{algorithm} from before in more mathematical terms.
(Remember, abstraction is a powerful tool!)
\lb
\textbf{Proposition}
\lb
In the context of the above definition, the matrix of $T$ can be computed as
\[ [T]_α^β = γ^β \circ T \circ (γ^α)^{-1}\]
\begin{proof}
\end{proof}

\vspace{200pt}
\lb
\textbf{Discussion}
\lb
Without doing a lot of work, can you argue what the matrix
representing the composition $F \circ T$ is assuming you know $[F]$ and $[T]$?

\newpage

\section*{Imge and Kernel}%
\label{sec:Imge and Kernel}

\textbf{Textbook:} Section 2.3

\lb
\textbf{Definition (2.3.1 \& 2.3.10)}
\lb
For a linear transformation $ \map{V}[T]{W}$, we define
\begin{enumerate}
    \item
        the \emph{preimage} $T^{-1}(S)$ of $S \subseteq W$ under $T$ as all $\vec v ∈ V$ that map into $S$.
    \item
        the \emph{kernel} $\ker(T)$ of $T$  as all $\vec v ∈ V$ that map to $\vec 0$ under $T$,
    \item
        the \emph{image} $\tx{im}(T)$ of $T$ as all $\vec w ∈ W$ such that $\vec w = T(\vec v)$ for some $\vec v ∈ V$,
\end{enumerate}

\lb
\textbf{Example}
\begin{itemize}
    \item
    The kernel of $\map{\pol{n}{\R}}[\frac{d}{dx}]{\pol{n}{\R}}$ are all constant polynomials,
    while the image consists of polynomials of degree $n-1$.

    \item
    The kernel of the evaluation map $\map{\pol{n}{\R}}[\tx{ev}_2]{\pol{n}{\R}}$ are all
    polynomials that have a root at $x = 2$. What is the image?

    \item
    What is the image of the linear transformation defined in example 2.2.2 $\map{\R^2}[T]{\R^2}$?
    \[ T(\vec e_1) = \vec e_1 + \vec e_2 \]
    \[ T(\vec e_2) = \vec 2 e_1 - 2 \vec e_2 \]
\end{itemize}

\newpage
\lb
\textbf{Proposition 2.3.2 \& 2.3.11}
\lb
For every linear transformation $\map{V}[T]{W}$
\begin{enumerate}
    \item $\ker(T)$ is a subspace in $V$
    \item $\tx{im}(T)$ is a subspace in $W$.
\end{enumerate}
\begin{proof}
\end{proof}

\vspace{300pt}
\lb
\textbf{Proposition 2.3.7}
\lb
The subspace $\ker(T)$ is the solution space to the homogeneous system of $[T]_α^β$.
\begin{proof}
\end{proof}

\vspace{200pt}
\lb
\textbf{Example}
\lb
Example of computation to find $\ker(T)$

\newpage
\lb
\textbf{Observation}
\lb
The subspace $\tx{im}(T)$ is the space of all $ \vec b ∈ \R^n$ such that the system
$[T]_α^β \vec x = \vec b$ has a solution.

\vspace{200pt}
\lb
\textbf{Proposition 2.3.12}
\lb
If $ \cb{\vec v_1, \ldots, \vec v_k}$ spans $V$, then $ \cb{T(\vec v_1), \ldots, T(\vec v_k)}$
spans $\tx{im}(T)$.
\begin{proof}
\end{proof}

\vspace{200pt}
\lb
\textbf{Definition}
\lb
For a matrix $A = [ a_1, a_2, \ldots, a_m] ∈ \mat{n, m}{\R}$ we denote
the span of the columns of $A$ by
\[ \tx{col}(A) = \spn{a_1, \ldots, a_m} \]

\lb
\textbf{Proposition}
\lb
For every linear transformation $\map{V}[T]{W}$
\[ \tx{im}(T) = \tx{col}([T]_α^β) \]
\begin{proof}
\end{proof}

\newpage
\lb
\textbf{Example}
\lb
Example computation to find $\tx{im}(T)$

\vspace{300pt}
\lb
Notice that the columns might not be independent,
in which case the columns are a spanning set of the image, but not a basis.

\lb
\textbf{Theorem}
\lb
Given a linear transformation $\map{V}[T]{W}$ with matrix $[T]_α^β$ for some bases $α$ and $β$.
Let $R = \tx{RREF}([T]_α^β)$ be the reduced row echolon form of $[T]_α^β$.
\lb
Then if the leading 1s in are $R$ lie in columns
$j_1, j_2,\ldots, j_r$, the columns $j_1, j_2,\ldots, j_r$ of $[T]_α^β$ are a basis for
$ \tx{col}([T]_α^β)$
\begin{proof}
\end{proof}

\vspace{100pt}
\lb
\textbf{Discussion}
\lb
Suppose a linear transformation $\map{V}[T]{W}$ is given in a some bases $α$ and $β$ by
\[ [T]_α^β = \begin{pmatrix} 1 & 2 & 0 & 1 \\ 1 & 2 & 1 & 0 \end{pmatrix} \]
Find a basis for $\tx{im}(T)$ and $\ker(T)$.

\newpage
\lb
\textbf{Theorem 2.3.17 (Rank-Nullity)}
\lb
For any linear transformation $\map{V}[T]{W}$
\[ \dim(V) = \dim(\ker(T)) + \dim( \tx{im}(T)) \]

\lb
\textbf{Remark}
\begin{itemize}
    \item
        $\dim( \tx{im}(T))$ is the same as the rank of $[T]_α^β$ and by abuse of
        notation also referred to as $\tx{rank}(T)$.
    \item
        Some books refer to $\dim(\ker(T))$ as the \emph{nullity} of $T$.
\end{itemize}

\begin{proof}
\end{proof}

\vspace{300pt}
\lb
\textbf{Theorem}
\lb
A linear transformation $T$ is injective if and only if $\ker(T) = \cb{\vec 0}$
\begin{proof}
\end{proof}

\newpage
\lb
\textbf{True or False}
Let $\map{V}[T]{W}$ be a linear transformation
\begin{enumerate}
    \item[$\square$]
        If $T$ is an isomorphism, then $\dim(V) = \dim(W)$.
    \item[$\square$]
        If $\dim(V) > \dim(W)$, $T$ has to be injective.
    \item[$\square$]
\end{enumerate}

\end{document}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{tikz}
\setlength{\parindent}{0pt}

\newtheorem*{theorem}{Theorem}
\newtheorem*{definition}{Definition}
\newtheorem*{lemma}{Lemma}
\newtheorem*{corollary}{Corollary}
\newtheorem{example}{Example}
\newtheorem*{trick}{Trick}
\newtheorem*{question}{Question}

\title{Lecture 16: Double Integrals}
\author{}
\date{}

\begin{document}

\maketitle

\section{Definition of Double Integrals}

For single-variable functions, we use integral to calculate the area under the
function curve. For a multivariable function with two independent variables
$z = f(x, y)$, we define double integrals to calculate the volume under the
function graph over a region $R$ in the $x$$y$-plane. The notation is
$\iint_R f(x, y)dA$.

\bigskip

Definition of double integrals:

If we cut the region $R$ into small pieces of area $\Delta A_i$, then the volume
can be calculated as
\begin{equation*}
  V \approx \sum_i f(x_i, y_i) \Delta A_i
\end{equation*}
If we take the limit of $\Delta A_i$ to 0, then it becomes the double integral
\begin{equation*}
  \iint_R f(x, y) dA = \lim_{\Delta A_i \to 0} \sum_i f(x_i, y_i) \Delta A_i
\end{equation*}

\section{Calculation of Double Integrals}

To compute a double integral $\iint_R f(x, y) dA$, we can take slices of the
function graph along one dimension and use integral to get the area of slices.
In this way, the area of all slices becomes a function of another dimension,
then we can use the integral again to sum up those slices to get the volume.

More concretely, let $S(x)$ denote the area of slices by plane parallel to
$y$$z$-plane. Then
\begin{equation*}
  \iint_R f(x, y) dA = \int_{x_min}^{x_max} S(x) dx
\end{equation*}
For a given $x_0$, the area of the slice
\begin{equation*}
  S(x_0) = \int_{y_min(x_0)}^{y_max(x_0)} f(x, y) dy
\end{equation*}
Therefore, the double integral
\begin{equation*}
  \begin{split}
    \iint_R f(x, y) dA &= \int_{x_min}^{x_max} S(x) dx \\
                       &= \int_{x_min}^{x_max} \int_{y_min(x_0)}^{y_max(x_0)} f(x, y) dy dx
  \end{split}
\end{equation*}
This calculation method is called \textbf{iterated integral}, because we iterate
twice through both dimensions to get the integral.

Notice that the bounds of the outer integral are numbers, and the bounds of the
inner integral are numbers or functions depending on the variable of the outer
integral. The bounds of both outer and inner integrals are determined by the
region of the double integral.

\section{Examples of Double Integral}

\begin{example}
  Find the value of the double integral of the function $z = 1 - x^2 - y^2$ over
  the region $R$ as $0 \leq x \leq 1$, $0 \leq y \leq 1$.

  Solution:

  By using iterated integrals,
  \begin{equation*}
    \iint_R f(x, y) dA = \int_0^1 \int_0^1 1 - x^2 - y^2 dy dx
  \end{equation*}
  For the inner integral,
  \begin{equation*}
    \begin{split}
      \int_0^1 1 - x^2 - y^2 dy &= (y - x^2y - \frac{y^3}{3})|_0^1 \\
                                &= 1 - x^2 - \frac{1}{3} \\
                                &= \frac{2}{3} - x^2 \\
    \end{split}
  \end{equation*}
  For the outer integral,
  \begin{equation*}
    \begin{split}
      \int_0^1 \frac{2}{3} - x^2 dx &= (\frac{2x}{3} - \frac{x^3}{3})|_0^1 \\
                                    &= \frac{2}{3} - \frac{1}{3} \\
                                    &= \frac{1}{3} \\
    \end{split}
  \end{equation*}
  Therefore, the value of the double integral is $\frac{1}{3}$.
\end{example}

\begin{example}
  Find the value of the double integral of the function $z = 1 - x^2 - y^2$ over
  the region $R$ as
  \begin{equation*}
    \begin{cases}
      x^2 + y^2 \leq 1 \\
      x \geq 0 \\
      y \geq 0 \\
    \end{cases}
  \end{equation*}

  By the equation of the region $R$, we can get that for any given $x_0$, the
  range of $y$ in the region $R$ is $[0, \sqrt{1 - x^2}]$.

  By using iterated integrals,
  \begin{equation*}
    \iint_R f(x, y) dA = \int_0^1 \int_0^{\sqrt{1 - x^2}} 1 - x^2 - y^2 dy dx
  \end{equation*}
  For the inner integral,
  \begin{equation*}
    \begin{split}
      \int_0^{\sqrt{1 - x^2}} 1 - x^2 - y^2 dy &= (y - x^2y - \frac{y^3}{3})|_0^{\sqrt{1 - x^2}} \\
                                               &= \sqrt{1 - x^2} - x^2\sqrt{1 - x^2} - \frac{(1 - x^2)^{\frac{3}{2}}}{3}\\
                                               &= \frac{2}{3}(1 - x^2)^{\frac{3}{2}} \\
    \end{split}
  \end{equation*}
  For the outer integral,
  \begin{equation*}
    \begin{split}
      \int_0^1 \frac{2}{3}(1 - x^2)^{\frac{3}{2}} dx &= \int_0^{\frac{\pi}{2}} \frac{2}{3}(1 - \sin^2\theta)^{\frac{3}{2}} d(\sin\theta) \\
                                                     &= \frac{2}{3} \int_0^{\frac{\pi}{2}} \cos^4\theta d\theta \\
                                                     &= \frac{2}{3} \int_0^{\frac{\pi}{2}} (\frac{1 + \cos 2\theta}{2})^2 d\theta \\
                                                     &= \frac{1}{6} \int_0^{\frac{\pi}{2}} (1 + 2\cos2\theta + \cos^2 2\theta) d\theta \\
                                                     &= \frac{1}{6} \int_0^{\frac{\pi}{2}} (1 + 2\cos2\theta + \frac{1 + \cos 4\theta}{2}) d\theta \\
                                                     &= \frac{1}{12} \int_0^{\frac{\pi}{2}} (3 + 4\cos2\theta + \cos 4\theta) d\theta \\
                                                     &= \frac{1}{12} (3\theta + 2\sin2\theta + \frac{1}{4}\cos 4\theta)|_0^{\frac{\pi}{2}} \\
                                                     &= \frac{1}{12} \cdot \frac{3\pi}{2} \\
                                                     &= \frac{\pi}{8} \\
    \end{split}
  \end{equation*}
  Therefore, the value of the double integral is $\frac{\pi}{8}$.
\end{example}

\section{Exchange the Order of Iterated Integrals}

In theory, no matter in which order we calculate the iterated integral, the
values always exist and are the same. However, in practice an order requires
much more complex computation than the other; sometimes we might not even be
able to compute the double integral with a specific order, but able to do it
with the other order. Therefore, we need to be mindful about the order of
iterated integral we use.

In some cases, we can exchange the order of an iterated integral, with some
adaption to the bounds of integrals.

\begin{example}
  \begin{equation*}
    \int_0^1 \int_0^2 dx dy = \int_0^2 \int_0^1 dy dx
  \end{equation*}
  For rectangle regions, the bounds of the inner integral don't depend on the
  variable of the outer integral, so we can exchange the inner integral and
  outer integral freely.

  From the perspective of the definition of iterated integrals, for rectangle
  regions, slices of either direction don't change the set of small pieces of
  area $dA_i$ to sum up.
\end{example}

\begin{example}
  Find the value of $\int_0^1 \int_x^{\sqrt{x}} \frac{e^y}{y} dy dx$.

  It is too difficult to calculate $\int_x^{\sqrt{x}} \frac{e^y}{y} dy$, so we
  need to consider exchanging the order of integrals.

  By looking at the region, we derive that
  \begin{equation*}
    \int_0^1 \int_x^{\sqrt{x}} \frac{e^y}{y} dy dx = \int_0^1 \int_{y^2}^y \frac{e^y}{y} dx dy
  \end{equation*}
  and the one on the right-hand side is doable.

  For the inner integral,
  \begin{equation*}
    \begin{split}
      \int_{y^2}^y \frac{e^y}{y} dx &= \frac{xe^y}{y}|_{y^2}^y \\
                                    &= e^y - ye^y \\
    \end{split}
  \end{equation*}
  For the outer integral,
  \begin{equation*}
    \begin{split}
      \int_0^1 e^y - ye^y dy &= (2e^y - ye^y)|_0^1 \\
                             &= 2e - e - 2 \\
                             &= e - 2 \\
    \end{split}
  \end{equation*}
  Therefore, the value of $\int_0^1 \int_x^{\sqrt{x}} \frac{e^y}{y} dy dx$ is
  $e - 2$.
\end{example}

\end{document}


\chapter{Bonus: Let's solve Pell's equation!}
This is an optional aside, and can be safely ignored.
(On the other hand, it's pretty short.)

\section{Units}
\prototype{$\pm 1$, roots of unity, $3-2\sqrt2$ and its powers.}
Recall according to \Cref{prob:OK_unit_norm} that $\alpha \in \OO_K$ is invertible
if and only if \[ \NK(\alpha) = \pm 1. \]
We let $\OO_K^\times$ denote the set of units of $\OO_K$.

\begin{ques}
	Show that $\OO_K^\times$ is a group under multiplication.
	Hence we name it the \vocab{unit group} of $\OO_K$.
\end{ques}

What are some examples of units?
\begin{example}
	[Examples of units in a number field]
	\listhack
	\begin{enumerate}
		\ii $\pm 1$ are certainly units, present in any number field.

		\ii If $\OO_K$ contains a root of unity $\omega$ (i.e.\ $\omega^n=1$),
		then $\omega$ is a unit.
		(In fact, $\pm 1$ are special cases of this.)

		\ii Of course, not all units of $\OO_K$ are roots of unity.
		For example, if $\OO_K = \ZZ[\sqrt3]$ (from $K = \QQ(\sqrt3)$) then
		the number $2+\sqrt3$ is a unit, as its norm is
		\[ \NK(2+\sqrt3) = 2^2 - 3 \cdot 1^2 = 1. \]
		Alternatively, just note that the inverse $2-\sqrt3 \in \OO_K$ as well:
		\[ \left( 2-\sqrt3 \right)\left( 2+\sqrt3 \right) = 1. \]
		Either way, $2-\sqrt3$ is a unit.

		\ii Given any unit $u \in \OO_K^\times$, all its powers are also units.
		So for example, $(3-2\sqrt2)^n$ is always a unit of $\ZZ[\sqrt2]$, for any $n$.
		If $u$ is not a root of unity, then this generates infinitely many new units in $\OO_K^\times$.
	\end{enumerate}
\end{example}

\begin{ques}
	Verify the claims above that
	\begin{enumerate}[(a)]
		\ii Roots of unity are units, and
		\ii Powers of units are units.
	\end{enumerate}
	One can either proceed from the definition
	or use the characterization $\NK(\alpha) = \pm 1$.
	If one definition seems more natural to you, use the other.
\end{ques}

\section{Dirichlet's unit theorem}
\prototype{The units of $\ZZ[\sqrt3]$ are $\pm(2+\sqrt3)^n$.}

\begin{definition}
	Let $\mu(\OO_K)$ denote the set of roots of unity
	contained in a number field $K$ (equivalently, in $\OO_K$).
\end{definition}
\begin{example}[Examples of $\mu(\OO_K)$]
	\listhack
	\begin{enumerate}[(a)]
		\ii If $K = \QQ(i)$, then $\OO_K = \ZZ[i]$. So
		\[ \mu(\OO_K) = \{\pm1, \pm i\} \quad\text{where } K = \QQ(i). \]
		\ii If $K = \QQ(\sqrt3)$, then $\OO_K = \ZZ[\sqrt 3]$. So
		\[ \mu(\OO_K) = \{\pm 1\} \quad\text{where } K = \QQ(\sqrt 3). \]
		\ii If $K = \QQ(\sqrt{-3})$, then $\OO_K = \ZZ[\half(1+\sqrt{-3})]$.
		So
		\[ \mu(\OO_K)
			= \left\{ \pm 1, \frac{\pm 1 \pm \sqrt{-3}}{2} \right\}
			\quad\text{where } K = \QQ(\sqrt{-3})
		\]
		where the $\pm$'s in the second term need not depend on each other;
		in other words $\mu(\OO_K) = \left\{ z \mid z^6=1 \right\}$.
	\end{enumerate}
\end{example}
\begin{exercise}
	Show that we always have that $\mu(\OO_K)$
	comprises the roots to $x^n-1$ for some integer $n$.
	(First, show it is a finite group under multiplication.)
\end{exercise}

We now quote, without proof, the so-called Dirichlet's unit theorem,
which gives us a much more complete picture of what the units in $\OO_K$ are.
Legend says that Dirichlet found the proof of this theorem
during an Easter concert in the Sistine Chapel.
\begin{theorem}
	[Dirichlet's unit theorem]
	Let $K$ be a number field with signature $(r_1, r_2)$ and set
	\[ s = r_1 + r_2 - 1. \]
	Then there exist units $u_1$, \dots, $u_s$ such that every unit
	$\alpha \in \OO_K^\times$ can be written \emph{uniquely} in the form
	\[ \alpha = \omega \cdot u_1^{n_1} \dots u_s^{n_s} \]
	for $\omega \in \mu(\OO_K)$ is a root of unity,
	and $n_1, \dots, n_s \in \ZZ$.
\end{theorem}
More succinctly:
\begin{moral}
We have $\OO_K^\times \cong \ZZ^{r_1+r_2-1} \times \mu(\OO_K)$.
\end{moral}
A choice of $u_1$, \dots, $u_s$ is called a choice of \vocab{fundamental units}.

Here are some example applications.
\begin{example}
	[Some unit groups]
	\listhack
	\begin{enumerate}[(a)]
		\ii Let $K = \QQ(i)$ with signature $(0,1)$.
		Then we obtain $s = 0$, so Dirichlet's Unit theorem says that there are no
		units other than the roots of unity.
		Thus
		\[ \OO_K^\times = \{\pm 1, \pm i\} \quad\text{where } K = \QQ(i). \]
		This is not surprising,
		since $a+bi \in \ZZ[i]$ is a unit if and only if $a^2+b^2 = 1$.

		\ii Let $K = \QQ(\sqrt 3)$, which has signature $(2,0)$.
		Then $s=1$, so we expect exactly one fundamental unit.
		A fundamental unit is $2+\sqrt3$ (or $2-\sqrt3$, its inverse) with norm $1$, and so we find
		\[ \OO_K^\times = \left\{ \pm (2+\sqrt3)^n \mid n \in \ZZ \right\}.  \]

		\ii Let $K = \QQ(\sqrt[3]{2})$ with signature $(1,1)$.
		Then $s=1$, so we expect exactly one fundamental unit.
		The choice $1 + \sqrt[3]{2} + \sqrt[3]{4}$. So
		\[ \OO_K^\times
			= \left\{ \pm \left( 1+\sqrt[3]{2}+\sqrt[3]{4} \right)^n \mid n \in \ZZ \right\}. \]
	\end{enumerate}
\end{example}

I haven't actually shown you that these are fundamental units,
and indeed computing fundamental units is in general hard.

\section{Finding fundamental units}
Here is a table with some fundamental units.
\[
	\begin{array}{rl}
		d & \text{Unit} \\ \hline
		d=2 & 1+\sqrt 2 \\
		d=3 & 2+\sqrt3 \\
		d=5 & \half(1+\sqrt5) \\
		d=6 & 5+2\sqrt6 \\
		d=7 & 8+3\sqrt7 \\
		d=10 & 3+\sqrt{10} \\
		d=11 & 10+3\sqrt11
	\end{array}
\]

In general, determining fundamental units is computationally hard.

However, once I tell you what the fundamental unit is, it's not too bad
(at least in the case $s=1$) to verify it.
For example,
suppose we want to show that $10 + 3\sqrt{11}$ is a fundamental unit of $K = \QQ(\sqrt 11)$,
which has ring of integers $\ZZ[\sqrt{11}]$.
If not, then for some $n > 1$, we would have to have
\[ 10 + 3 \sqrt{11} = \pm \left( x+y\sqrt{11} \right)^n. \]
For this to happen, at the very least we would need $\left\lvert y \right\rvert < 3$.
We would also have $x^2-11y^2 = \pm 1$.
So one can just verify (using $y= 1,2$) that this fails.

The point is that: Since $(10,3)$ is the \emph{smallest}
(in the sense of $\left\lvert y \right\rvert$)
integer solution to $x^2-11y^2 = \pm 1$, it must be the fundamental unit.
This holds more generally, although in the case that $d \equiv 1 \pmod 4$
a modification must be made as $x$, $y$ might be half-integers (like $\half(1+\sqrt5)$).
\begin{theorem}
	[Fundamental units of pell equations]
	Assume $d$ is a squarefree integer.
	\begin{enumerate}[(a)]
		\ii If $d \equiv 2,3 \pmod 4$,
		and $(x,y)$ is a minimal integer solution to $x^2-dy^2 = \pm 1$,
		then $x + y \sqrt d$ is a fundamental unit.
		\ii If $d \equiv 1 \pmod 4$,
		and $(x,y)$ is a minimal \emph{half-integer} solution to $x^2-dy^2 = \pm 1$,
		then $x + y \sqrt d$ is a fundamental unit.
		(Equivalently, the minimal integer solution to $a^2 - db^2 = \pm 4$
		gives $\half (a + b \sqrt d)$.)
	\end{enumerate}
	(Any reasonable definition of ``minimal'' will work, such as sorting by $\left\lvert y \right\rvert$.)
\end{theorem}

\section{Pell's equation}
This class of results completely eradicates Pell's Equation.
After all, solving
\[ a^2 - d \cdot b^2 = \pm 1 \]
amounts to finding elements of $\ZZ[\sqrt d]$ with norm $\pm 1$.
It's a bit weirder in the $d \equiv 1 \pmod 4$ case, since in that case $K = \QQ(\sqrt d)$
gives $\OO_K = \ZZ[\half(1+\sqrt d)]$, and so the fundamental unit may not actually be a solution.
(For example, when $d = 5$, we get the solution $(\half, \half)$.)
Nonetheless, all \emph{integer} solutions are eventually generated.

To make this all concrete, here's a simple example.
\begin{example}[$x^2-5y^2 = \pm 1$]
	Set $K = \QQ(\sqrt 5)$, so $\OO_K = \ZZ[\half(1+\sqrt 5)]$.
	By Dirichlet's unit theorem, $\OO_K^\times$ is generated by a single element $u$.
	The choice
	\[ u = \frac 12 + \frac 12 \sqrt 5 \]
	serves as a fundamental unit,
	as there are no smaller integer solutions to $a^2-5b^2=\pm 4$.

	The first several powers of $u$ are
	\[
	\begin{array}{rrr}
		\renewcommand{\arraystretch}{1.4}
		n & \multicolumn{1}{c}{u^n} & \text{Norm} \\ \hline
		-2 & \half(3-\sqrt5) & 1 \\
		-1 & \half (1-\sqrt5) & -1 \\
		0 & 1 & 1 \\
		1 & \half(1+\sqrt5) & -1 \\
		2 & \half(3+\sqrt5) & 1 \\
		3 & 2 + \sqrt 5 & -1 \\
		4 & \half(7+3\sqrt5) & 1 \\
		5 & \half(11+5\sqrt5) & -1 \\
		6 & 9 + 4\sqrt 5 & 1
	\end{array}
	\]
	One can see that the first integer solution is $(2,1)$, which gives $-1$.
	The first solution with $+1$ is $(9,4)$.
	Continuing the pattern, we find that every third power of $u$ gives an integer solution
	(see also \Cref{prob:unit_cubed}),
	with the odd ones giving a solution to $x^2-5y^2=-1$ and
	the even ones a solution to $x^2-5y^2=+1$.
	All solutions are generated this way, up to $\pm$ signs
	(by considering $\pm u^{\pm n}$).
\end{example}

\section\problemhead
\begin{problem}[Fictitious account of the battle of Hastings]
	Determine the number of soldiers in the following battle:
	\begin{quote}
		The men of Harold stood well together,
		as their wont was, and formed thirteen squares,
		with a like number of men in every square thereof, and woe
		to the hardy Norman who ventured to enter their redoubts;
		for a single blow of Saxon war-hatched would break his lance
		and cut through his coat of mail . . .
		when Harold threw himself into the fray the Saxons
		were one might square of men, shouting the battle-cries,
		``Ut!'', ``Olicrosse!'', ``Godemite!''
	\end{quote}
	% The answer can be found by hand.
	% You may assume that the army has size at least $1$ and
	% at most one billion.
\end{problem}
\begin{problem}
	\label{prob:unit_cubed}
	Let $d > 0$ be a squarefree integer,
	and let $u$ denote the fundamental unit of $\QQ(\sqrt d)$.
	Show that either $u \in \ZZ[\sqrt d]$,
	or $u^n \in \ZZ[\sqrt d] \iff 3 \mid n$.
\end{problem}
\begin{problem}
	Show that there are no integer solutions to
	\[ x^2 - 34y^2 = -1 \]
	despite the fact that $-1$ is a quadratic residue mod $34$.
\end{problem}


\lab{Profiling Python Code}{Profiling}
\objective{Learn how to efficiently optimize Python code.}
\label{lab:ProfilingCode}

The best code goes through multiple drafts.
In a first draft, you should focus on writing code that does what it is supposed to and is easy to read.
After writing a first draft, you may find that your code does not run as quickly as you need it to.
Then it is time to \emph{optimize} the most time consuming parts of your code so that they run as quickly as possible.

In this lab we will optimize the function \texttt{qr1()} that computes the QR decomposition of a matrix via the modified Gram-Schmidt algorithm (see Lab \ref{lab:QRdecomp}).
\begin{lstlisting}
import numpy as np
from scipy import linalg as la

def qr1(A):
    ncols = A.shape[1]
    Q = A.copy()
    R = np.zeros((ncols, ncols))
    for i in range(ncols):
        R[i, i] = la.norm(Q[:, i])
        Q[:, i] = Q[:, i]/la.norm(Q[:, i])
        for j in range(i+1, ncols):
            R[i, j] = Q[:, j].dot(Q[:, i])
            Q[:,j] = Q[:,j]-Q[:, j].dot(Q[:, i])*Q[:,i]
    return Q, R
\end{lstlisting}

\section*{What to Optimize}
Python provides a \emph{profiler} that can identify where code spends most of its runtime.
The output of the profiler will tell you where to begin your optimization efforts.

In IPython\footnote{If you are not using IPython, you will need to use the \li{cProfile} module documented here: \url{https://docs.python.org/2/library/profile.html}.},
you can profile a function from the command line with \texttt{\%prun}.
Here we profile \texttt{qr1()} on a random $300 \times 300$ array.
\begin{lstlisting}
>>> A = np.random.rand(300, 300)
>>> \%prun qr1(A)
\end{lstlisting}

On this computer, we get the following output.

{\scriptsize
\begin{verbatim}
         97206 function calls in 1.343 seconds

   Ordered by: internal time

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
        1    0.998    0.998    1.342    1.342 profiling_hw.py:4(qr1)
    89700    0.319    0.000    0.319    0.000 {method 'dot' of 'numpy.ndarray' objects}
      600    0.006    0.000    0.012    0.000 function_base.py:526(asarray_chkfinite)
      600    0.006    0.000    0.009    0.000 linalg.py:1840(norm)
     1200    0.005    0.000    0.005    0.000 {method 'any' of 'numpy.ndarray' objects}
      600    0.002    0.000    0.002    0.000 {method 'reduce' of 'numpy.ufunc' objects}
     1200    0.001    0.000    0.001    0.000 {numpy.core.multiarray.array}
     1200    0.001    0.000    0.002    0.000 numeric.py:167(asarray)
        1    0.001    0.001    0.001    0.001 {method 'copy' of 'numpy.ndarray' objects}
      600    0.001    0.000    0.022    0.000 misc.py:7(norm)
      301    0.001    0.000    0.001    0.000 {range}
        1    0.001    0.001    0.001    0.001 {numpy.core.multiarray.zeros}
      600    0.001    0.000    0.001    0.000 {method 'ravel' of 'numpy.ndarray' objects}
      600    0.000    0.000    0.000    0.000 {method 'conj' of 'numpy.ndarray' objects}
        1    0.000    0.000    1.343    1.343 <string>:1(<module>)
        1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
\end{verbatim}
}

The first line of the output tells us that executing \texttt{qr1(A)} results in almost 100,000 function calls.
Then we see a table listing these functions along with data telling us how much time each takes.
Here, \texttt{ncalls} is the number of calls to the function, \texttt{tottime} is the total time spent in the function, and \texttt{cumtime} is the amount of time spent in the function including calls to other functions.

For example, the first line of the table is the function \texttt{qr1(A)} itself.
This function was called once, it took 1.342s to run, and 0.344s of that was spent in calls to other functions.
Of that 0.344s, there were 0.319s spent on 89,700 calls to \texttt{np.dot()}.

With this output, we see that most time is spent in multiplying matrices.
Since we cannot write a faster method to do this multiplication, we may want to try to reduce the number of matrix multiplications we perform.

\section*{How to Optimize}
Once you have identified those parts of your code that take the most time, how do you make them run faster?
This section lists a few ideas.
Always, you should use the profiling and timing functions to help you decide when an optimization is actually useful.

\subsection*{Avoid Recomputing Values}
In our function \texttt{qr1()}, we can avoid recomputing \texttt{R[i,i]} in the outer loop and \texttt{R[i,j]} in the inner loop.
The rewritten function is as follows:
\begin{lstlisting}
def qr2(A):
    ncols = A.shape[1]
    Q = A.copy()
    R = np.zeros((ncols, ncols))
    for i in range(ncols):
        R[i, i] = la.norm(Q[:, i])
        Q[:, i] = Q[:, i]/R[i, i]
        for j in range(i+1, ncols):
            R[i, j] = Q[:, j].dot(Q[:, i])
            Q[:,j] = Q[:,j]-R[i, j]*Q[:,i]
    return Q, R
\end{lstlisting}

Profiling \texttt{qr2()} on a $300 \times 300$ matrix produces the following output.

{\scriptsize
\begin{verbatim}
         48756 function calls in 1.047 seconds

   Ordered by: internal time

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
        1    0.863    0.863    1.047    1.047 profiling_hw.py:16(qr2)
    44850    0.171    0.000    0.171    0.000 {method 'dot' of 'numpy.ndarray' objects}
      300    0.003    0.000    0.006    0.000 function_base.py:526(asarray_chkfinite)
      300    0.003    0.000    0.005    0.000 linalg.py:1840(norm)
      600    0.002    0.000    0.002    0.000 {method 'any' of 'numpy.ndarray' objects}
      300    0.001    0.000    0.001    0.000 {method 'reduce' of 'numpy.ufunc' objects}
      301    0.001    0.000    0.001    0.000 {range}
      600    0.001    0.000    0.001    0.000 {numpy.core.multiarray.array}
      600    0.001    0.000    0.001    0.000 numeric.py:167(asarray)
      300    0.000    0.000    0.012    0.000 misc.py:7(norm)
        1    0.000    0.000    0.000    0.000 {method 'copy' of 'numpy.ndarray' objects}
      300    0.000    0.000    0.000    0.000 {method 'ravel' of 'numpy.ndarray' objects}
        1    0.000    0.000    1.047    1.047 <string>:1(<module>)
      300    0.000    0.000    0.000    0.000 {method 'conj' of 'numpy.ndarray' objects}
        1    0.000    0.000    0.000    0.000 {numpy.core.multiarray.zeros}
        1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
\end{verbatim}
}

Our optimization reduced almost every kind of function call by half, and reduced the total run time by 0.295s.

Some less obvious ways to eliminate excess computations include moving computations out of loops, not copying large data structures, and simplifying mathematical expressions.

\subsection*{Avoid Nested Loops}
The best way to do this is to use NumPy array operations instead of iterating through arrays.
If you must use nested loops, focus your optimization efforts on the innermost loop, which gets called the most times.

\subsection*{Use Existing Functions Instead of Writing Your Own}
If there is an intuitive operation you would like to perform on an array, chances are that NumPy or another library already has a function that does it.
Python and NumPy functions have already been optimized, and are usually many times faster than the equivalent you might write.
We saw an example of this in Lab \ref{lab:NumPyArrays} where we compared NumPy array multiplication with our own matrix multiplication implemented in Python.

\subsection*{Use Generators When Possible}
When you are iterating through a list, you can often replace the list with a \emph{generator}.
Instead of storing the entire list in memory, a generator computes each item as it is needed.
For example, the code
\begin{lstlisting}
>>> for i in range(100):
>>>     print i
\end{lstlisting}
stores the numbers 0 to 99 in memory, looks up each one in turn, and prints it.
On the other hand, the code
\begin{lstlisting}
>>> for i in xrange(100):
>>>     print i
\end{lstlisting}
uses a generator instead of a list.
This code computes the first number in the specified range (which is 0), and prints it.
Then it computes the next number (which is 1) and prints that.

It is also possible to write your own generators.
See \url{https://docs.python.org/2/tutorial/classes.html#generators} and \url{https://wiki.python.org/moin/Generators} for more information.

In our example, replacing each \texttt{range} with \texttt{xrange} does not speed up \texttt{qr2()} by a noticeable amount.

\subsection*{Avoid Excessive Function Calls}
Function calls take time.
Moreover, looking up methods associated with objects takes time.
Removing ``dots'' can significantly speed up execution time.

For example, we could rewrite our function to reduce the number of times we need to look up the function \texttt{la.norm()}.

\begin{lstlisting}
def qr2(A):
    norm = la.norm
    ncols = A.shape[1]
    Q = A.copy()
    R = np.zeros((ncols, ncols))
    for i in range(ncols):
        R[i, i] = norm(Q[:, i])
        Q[:, i] = Q[:, i]/R[i, i]
        for j in range(i+1, ncols):
            R[i, j] = Q[:, j].dot(Q[:, i])
            Q[:,j] = Q[:,j]-R[i, j]*Q[:,i]
    return Q, R
\end{lstlisting}
Once again, an analysis with \texttt{\%prun} reveals that this optimization does not help significantly in this case.

\subsection*{Write Pythonic Code}
Several special features of Python allow you to write fast code easily.
First, list comprehensions are much faster than for loops.
For example, replace
\begin{lstlisting}
>>> mylist = []
>>> for i in xrange(100):
>>>     mylist.append(math.sqrt(i))
\end{lstlisting}
with
\begin{lstlisting}
>>> mylist = [math.sqrt(i) for i in xrange(100)]
\end{lstlisting}
When it can be used, the function \texttt{map()} is even faster.
\begin{lstlisting}
>>> mylist = map(math.sqrt, xrange(100))
\end{lstlisting}
The analog of a list comprehension also exists for generators, dictionaries, and sets.

Second, swap values with a single assignment.
\begin{lstlisting}
>>> a, b = 1, 2
>>> a, b = b, a
>>> print a, b
2 1
\end{lstlisting}

Third, many non-Boolean objects in Python have truth values.
For example, numbers are \texttt{False} when equal to zero and \texttt{True} otherwise.
Similarly, lists and strings are \texttt{False} when they are empty and \texttt{True} otherwise.
So when \texttt{a} is a number, instead of
\begin{lstlisting}
>>> if a != 0:
\end{lstlisting}
use
\begin{lstlisting}
>>> if a:
\end{lstlisting}
\subsection*{Use Cython}
We will discuss Cython at the end of this lab.

\subsection*{Use a More Efficient Algorithm}
The optimizations discussed thus far will speed up your code at most by a constant.
They will not change the complexity of your code.
In order to reduce the complexity (say from $O(n^2)$ to $O(n \log(n))$), you typically need to change your algorithm.

\section*{When to Stop}
You don't need to apply every possible optimization to your code.
When your code runs acceptably fast, stop optimizing.

\section*{Cython}
Cython code is basically Python with extra type declarations.
This code is then compiled into C, which---depending on the details---can run much faster than the Python equivalent.
In this lab we will introduce Cython as a language and discuss how it can be used to speed up Python code.

\subsection*{Compiling Cython}
With a few exceptions, every Python program is also a Cython program.
For example, suppose you save the following script as \texttt{cymodule.pyx}.
\begin{lstlisting}
import numpy as np
from scipy import linalg as la

def qr(A):
    norm = la.norm
    ncols = A.shape[1]
    Q = A.copy()
    R = np.zeros((ncols, ncols))
    for i in range(ncols):
        R[i, i] = norm(Q[:, i])
        Q[:, i] = Q[:, i]/R[i, i]
        for j in range(i+1, ncols):
            R[i, j] = Q[:, j].dot(Q[:, i])
            Q[:,j] = Q[:,j]-R[i, j]*Q[:,i]
    return Q, R
\end{lstlisting}
To compile this code as Cython, we need to write another script, call it \texttt{setup.py}.
\lstinputlisting[style=fromfile, language=Python]{setup.py}
To run the setup script, type the following in the command line.
\footnote{Note that \texttt{python} in this line will need to refer to your version of Python that comes with Cython.
Often, you will need to replace \texttt{python} with a path pointing to a specific distribution of Python.}
\begin{lstlisting}
>>> python setup.py build_ext --inplace
\end{lstlisting}
Now start up IPython in the same directory where you ran the setup script.
You can import \texttt{cymodule} just as if it were a Python module.
\begin{lstlisting}
>>> import cymodule
>>> cymodule.qr(A)
\end{lstlisting}

\subsection*{Speeding up Cython with Type Declarations}
So far, our Cythonized \texttt{qr()} function does not run any faster than the Python version.
\footnote{Sometimes you will see speedup immediately after Cythonizing a Python function.}
The simplest way to take advantage of the C-compilation is to declare data types, as you would in a C program.
All Cython data types and their NumPy equivalents are listed in Table \ref{table:cython_types}.

\begin{table}
\begin{center}
\begin{tabular}{|p{2.8cm}|p{2.5cm}|p{6cm}|}
\hline
Cython Type & NumPy Type& Description \\
\hline
float & float32 & 32 bit floating point number \\
double & float64 & 64 bit floating point number \\
float complex & complex64 & 64 bit floating point complex number \\
double complex & complex128 & 128 bit floating point complex number \\
char & int8 & 8 bit signed integer \\
unsigned char & uint8 & 8 bit unsigned integer \\
short & int16 & 16 bit signed integer \\
unsigned short & uint16 & 16 bit unsigned integer \\
int & int32 & 32 bit signed integer \\
unsigned int & uint32 & 32 bit unsigned integer \\
long & int32 or int64 & 32 or 64 bit signed integer (depends on platform) \\
unsigned long & uint32 or uint64 & 32 or 64 bit unsigned integer (depends on platform ) \\
long long & int64 & 64 bit signed integer \\
unsigned long long & uint64 & 64 bit unsigned integer \\
\hline
\end{tabular}
\end{center}
\caption{Numeric types available in Cython.}
\label{table:cython_types}
\end{table}

In our example, when we declare the index \texttt{j} to be an \texttt{int}, the inner for loop is pushed into C (instead of Python), speeding up our code immensely.
We do this by modifying \texttt{cymodule.pyx} as follows.
\begin{lstlisting}
...
def qr():
    cdef int j
    ncols = A.shape[1]
    Q = A.copy()
    ...
\end{lstlisting}

We could also declare \texttt{i} to be an int, but the effect of doing so is negligible.

After modifying \texttt{cymodule.pyx}, we must recompile it by running the script \texttt{setup.py}, and then re-import it into Python.
When we profile this function, we see that it is indeed a good deal faster than its Python cousin, and all the speed up is in the function itself (where the for loops are).
{\scriptsize
\begin{verbatim}
   ...
   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
        1    0.949    0.949    0.961    0.961 {cymodule.qr}
   ...
       \end{verbatim}
       }
Similarly, you can speed up function calls in Cython by declaring the types of some or all of the arguments.
\begin{lstlisting}
def myfunction(double arg1, int arg2, arg3):
    ...
\end{lstlisting}

\subsection*{A Caution}
NumPy's array methods are usually faster than a Cython equivalent you could code yourself.
If you are unsure which method is fastest, time them.

Moreover, a good algorithm written with a slow language (like Python) is faster than a bad algorithm written in a fast language (like C).
Hence, focus on writing fast algorithms with good Python code, and only use Cython when and where it is necessary.

\begin{problem}
Practice profiling and optimizing functions you have already written.
Some suggested functions are listed below.
\begin{itemize}
\item LU decomposition (Lab \ref{lab:ChangeBasis}).
\item Householder triangularization or Hessenburg decomposition (Lab \ref{lab:QRdecomp}).
\item Givens triangularization (Lab \ref{lab:givens})
\item Image Segmentation (Lab \ref{lab:ImgSeg_eigenvalues})
\item Eigenvalue Solvers (Lab \ref{lab:EigSolve})
\end{itemize}
Your solution should include a before and after profiling of the function.
It should also include a list of changes, the reasoning behind the changes, and the effect of the changes on runtime.

It is possible that you will not be able to speed up the function significantly.
Remember it is more important for code to be readable than to execute quickly.
\end{problem}

\section*{More on Cython (Optional)}
This section has a more complete introduction to Cython.
For another reference, see \url{http://docs.cython.org/src/tutorial/cython_tutorial.html}.

\subsection*{Compilation}

Cython code is usually written in a \texttt{.pyx} file, which is then compiled to C.
Next the C is compiled to a Python extension written in machine code.
Figure \ref{cython:compilation} shows how Cython code is compiled and called.

\begin{figure}
\centering
\includegraphics[width=.7\textwidth]{compilation.pdf}
\caption{A diagram showing how Cython code is compiled and called.
The path from calling a Cython function in a Python file to its evaluation is shown in red.}
\label{cython:compilation}
\end{figure}

These two compilations (first to C and then to a Python extension) are accomplished by the script \texttt{setup.py} discussed earlier in this lab.
\lstinputlisting[style=fromfile, language=Python]{setup.py}
The call to \texttt{cythonize()} compiles \texttt{cymodule.pyx} to C, and the call to \texttt{setup()} converts the C code to a Python extension.

\begin{info}
You can learn more about the Python/C balance in your Cython file with the following line.
\begin{lstlisting}
>>> cython -a cymodule.pyx
\end{lstlisting}
This will generate a \li{html} file that you can open with your web browser.
It will show the lines that use Python in bright yellow and lines that use C in white.
\end{info}

Cython code can be compiled and imported to IPython with the Cython magic function.
Load this function with the following command.
\footnote{In older versions of IPython and Cython, you may need to use the command \li{\%load_ext cythonmagic}.}
\begin{lstlisting}
>>> %load_ext Cython
\end{lstlisting}
Now you can define any Cython function in the command line by prefacing it with \li{\%\%cython}.
\begin{lstlisting}
>>> %%cython
>>> import numpy as np
>>> from scipy import linalg as la
>>> def qr():
>>>     cdef int j
>>> ...
\end{lstlisting}

\subsection*{Type Declarations}
Type declarations make Cython faster than Python because a computer can process numbers faster when it knows ahead of time what type they are.
Declaring a variable type in Cython makes the variable a native machine type instead of a Python object.

You can initialize a variable when it is declared or afterwards.
It is possible to initialize several variables at once, as demonstrated below.

\begin{lstlisting}
# Declare integer variables i, j, and k.
# Set k equal to 2.
cdef int i, j, k=2

cdef:
    # Declare and initialize m equal to 4 and n equal to 5.
    int m=4, n=5
    # Declare and initialize e equal to 2.71.
    double e = 2.71
    # Declare a double precision complex number a.
    double complex a
\end{lstlisting}

\begin{warn}
Unlike Python integers, which can be arbitrarily large, Cython integers can overflow.
\end{warn}

\subsection*{Arrays in Cython}
You can also declare types for arrays.
Doing so produces a \emph{typed memoryview}, or Cython array.
As with ordinary variables, typed memoryviews can be initialized when they are declared or later.
\begin{lstlisting}
# Define a NumPy array
A = np.linspace(0,1,6)

# Create typed memoryviews on A and B
cdef double[:] cA = A
cdef int[:,:] cB

# Initialize cB
cB = np.array([[1,2],[3,4],[5,6]], dtype=np.dtype("i"))
\end{lstlisting}

Accessing single entries from a memoryview is faster than accessing entries of a NumPy array.
Also, passing slices of a memoryview to a Cython function is fast.
However, memoryviews do not support mathematical operations.
These must be performed on NumPy objects, or by looping through the arrays.

\begin{info}
Memoryviews can be passed as arguments to most NumPy functions.
Numpy includes functions for all common arithmetic operations.
For example, you can add typed memoryviews with \li{c = np.add(a, b)}.
This may or may not be faster than looping through the array.
\end{info}

\subsection*{Compiler Directives}
When you access elements of an array, Cython checks that the indices are within bounds.
Cython also allows negative indexing the same way Python does.
These features slow down code execution.
After a program has been carefully debugged, they may be removed via \emph{compiler directives}.

Compiler directives in Cython can be included as comments or function decorators.
Directives included in comments will apply to the whole file, while function decorators will only apply to the function immediately following.
The comments to turn off bounds checking and negative indices are
\begin{lstlisting}
# cython: boundscheck=False
# cython: wraparound=False
\end{lstlisting}
To use the function decorators, first import the \li{cython} module by including the line \li{cimport cython} in your import statements.
The decorators are
\begin{lstlisting}
cimport cython
@cython.boundscheck(False)
@cython.wraparound(False)
\end{lstlisting}

Cython has many other compiler directives, including \li{cdivision}.
When \li{cdivision} is set to \li{True}, the \li{\%} operator returns a number with the sign of the first argument (like in C).
Also, division by 0 will no longer raise a \li{ZeroDivisionError}, which will increase the speed of your program.

\subsection*{More on Functions in Cython}
Speed up function calls in Cython by declaring the types of some or all of the arguments.
\begin{lstlisting}
def myfunction(double[:] X, int n, double h, items):
    ...
\end{lstlisting}
If we pass in a NumPy array for the argument \li{X}, Cython will convert it to a typed memoryview before it enters the function.
However, if we pass in a NumPy array for \li{items}, it will remain a NumPy array in the function.
Both typed and untyped arguments can be made into keyword arguments.

Cython also allows you to make C functions that are only callable within the C extension you are currently building.
They are not ported into the Python namespace.
These functions are declared using the same syntax as in Python, except the keyword \li{def} is replaced with \li{cdef}.
The keyword \li{cpdef} combines \li{def} and \li{cdef} by creasting two versions of the function: one for Python and one for C.

You can also specify the return type for functions declared using \li{cdef} and \li{cpdef}, as in the code below.
\begin{lstlisting}
cpdef int myfunction(double[:] X, int n, double h, items):
    ...
\end{lstlisting}

\subsection*{Some Examples}
The following Python function computes the dot product of two 1-D arrays.
\begin{lstlisting}
def pydot(A, B):
    tot = 0.
    for i in xrange(A.shape[0]):
        tot += A[i] * B[i]
    return tot
\end{lstlisting}
A C equivalent can be compiled from the following Cython code.
\begin{lstlisting}
cimport cython

@cython.boundscheck(False)
@cython.wraparound(False)
def cydot(double[:] A, double[:] B):
    cdef double tot=0.
    cdef int i
    for i in xrange(A.shape[0]):
        tot += A[i] * B[i]
    return tot
\end{lstlisting}

Figure \ref{cython:dot} compares the speed of \li{pydot()}, \li{cydot()}, and the \li{dot()} method of a NumPy array.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{dot.pdf}
\caption{The running times of \li{pydot()}, \li{cydot()}, and the \li{dot()} method of a NumPy array on vectors of length ``Array Size.''
The Cython version runs almost as fast as the version built into NumPy.}
\label{cython:dot}
\end{figure}

For a second example, we will write functions which compute $AA^T$ from $A$.
That is, given \li{A}, these functions compute \li{B} where
\li{B[i,j] = dot(A[i],A[j])}.

Here is the Python solution.
\begin{lstlisting}
def pyrowdot(A):
    B = np.empty((A.shape[0], A.shape[0]))
    for i in xrange(A.shape[0]):
        for j in xrange(i):
            B[i,j] = pydot(A[i], A[j])
        B[i,i] = pydot(A[i], A[i])
    for i in xrange(A.shape[0]):
        for j in xrange(i+1, A.shape[0]):
            B[i,j] = B[j,i]
    return B
\end{lstlisting}

Here is the Cython solution.
We changed the function \li{cydot()} to a C function since it will only be called by \li{cyrowdot()}.
Also, the Cython solution uses a typed memoryview of its input.

\begin{lstlisting}
import numpy as np
cimport cython
# cython: boundscheck=False
# cython: wraparound=False

cdef double cydot(double[:] A, double[:] B):
    cdef double tot=0.
    cdef int i, n=A.shape[0]
    for i in xrange(n):
        tot += A[i] * B[i]
    return tot

def cyrowdot(double[:,:] A):
    cdef double[:,:] B = np.empty((A.shape[0], A.shape[0]))
    cdef int i, j, n=A.shape[0]
    for i in xrange(n):
        for j in xrange(i):
           B[i,j] = cydot(A[i], A[j])
        B[i,i] = cydot(A[i], A[i])
    for i in xrange(n):
        for j in xrange(i+1, n):
            B[i,j] = B[j,i]
    return np.array(B)
\end{lstlisting}

This can also be done in NumPy by running \li{A.dot(A.T)}.
The timings of \li{pyrowdot()}, \li{cyrowdot()}, and the NumPy command \li{A.dot(A.T)} are shown in Figure \ref{cython:rowdot}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{rowdot.pdf}
\caption{The timings of \li{pyrowdot()}, \li{cyrowdot()}, and the NumPy command \li{A.dot(A.T)}.
The arrays used for testing were $n\times 3$ where $n$ is shown along the horizontal axis.}
\label{cython:rowdot}
\end{figure}

In both of these examples, NumPy's implementation was faster than the version we wrote in Cython.

\begin{problem}
\leavevmode
\begin{enumerate}
\item Write the following function in Python.
\begin{lstlisting}
def pysum(X):
    ''' Return the sum of the elements of X.

    INPUTS:
    X - a 1-D NumPy array
    ''''
\end{lstlisting}

\item Rewrite the same function in Cython using a typed for-loop, a typed memoryview of \li{X}, and appropriate compiler directives.

\item Compare the speed of the functions you just wrote, the builtin \li{sum()} function, and NumPy's \li{sum()} function.
\end{enumerate}
\end{problem}

\begin{problem}
The code below defines a Python function which takes a matrix to the $n$th power.
\begin{lstlisting}
def pymatpow(X, power):
    ''' Return X^{power}.

    INPUTS:
    X		- A square 2-D NumPy array
    power	- An integer
    '''
    prod = X.copy()
    temparr = np.empty_like(X[0])
    size = X.shape[0]
    for n in xrange(1, power):
        for i in xrange(size):
            for j in xrange(size):
                tot = 0.
                for k in xrange(size):
                    tot += prod[i,k] * X[k,j]
                temparr[j] = tot
            prod[i] = temparr
    return prod
\end{lstlisting}

\begin{enumerate}
\item Port \li{pymatpow()} to Cython using typed for-loops, typed arrays, and appropriate compiler directives.
\item Compare the speed of \li{pymatpow()}, the function you just wrote, and the \li{np.dot()} function.
\end{enumerate}
NumPy takes products of matrices by calling BLAS and LAPACK, which are heavily optimized linear algebra libraries written in C, assembly, and Fortran.
\end{problem}

\begin{problem}
In Lab \ref{lab:ChangeBasis} you wrote a function to compute the LU decomposition of a matrix.
\begin{enumerate}
\item Rewrite this function so it performs every operation element-by-element instead of using NumPy array operations.
\item Port the function from part (a) to Cython.
Use typed for-loops, typed memoryviews, and appropriate compiler directives.
You may assume that you are only dealing with real arrays of double precision floating point numbers.
\item Compare the speed of your new solutions to the speed of the NumPy version you wrote earlier.
\end{enumerate}
\end{problem}

The correct choice of algorithm is more important than a fast implementation.
For example, suppose you wish to solve the following tridiagonal system.
\[\begin{bmatrix}
b_0 & c_0 & 0 & 0 & 0 & \cdots & \cdots & 0 \\
a_0 & b_1 & c_1 & 0 & 0 & \cdots & \cdots & 0 \\
0 & a_1 & b_2 & c_2 & 0 & \cdots & \cdots & 0 \\
0 & 0 & a_2 & b_3 & c_3 & \cdots & \cdots & 0 \\
\vdots & \vdots & \vdots & \vdots & \vdots & \ddots & \ddots & \vdots \\
\vdots & \vdots & \vdots & \vdots & \vdots & \ddots & \ddots & c_{n-1} \\
0 & 0 & 0 & 0 & 0 & \cdots & a_{n-1} & b_n
\end{bmatrix}
\begin{bmatrix}
d_0\\
d_1\\
d_2\\
d_3\\
\vdots\\
\vdots\\
d_n
\end{bmatrix}
=
\begin{bmatrix}
x_0\\
x_1\\
x_2\\
x_3\\
\vdots\\
\vdots\\
x_n
\end{bmatrix}\]
One way to do this is with the general solve method in SciPy's \li{linalg} module.
Alternatively, you could use an algorithm optimized for tridiagonal matrices.
The code below implements one such algorithm in Python.

The final result is stored in \li{x}, and \li{c} is used to store temporary values.
\begin{lstlisting}
def pytridiag(a, b, c, x):
    '''Solve the tridiagonal system Ad = x where A has diagonals a, b, and c.

    INPUTS:
    a, b, c, x - All 1-D NumPy arrays.

    NOTE:
    The final result is stored in `x` and `c` is used to store temporary values.
    '''
    n = x.size
    temp = 0.
    c[0] /= b[0]
    x[0] /= b[0]
    for i in xrange(n-2):
        temp = 1. / (b[i+1] - a[i] * c[i])
        c[i+1] *= temp
        x[i+1] = (x[i+1] - a[i] * x[i]) * temp
    x[n-1] = (x[n-1] - a[n-2] * x[n-2]) / (b[n-1] - a[n-2] * c[n-2])
    for i in xrange(n-2, -1, -1):
        x[i] = x[i] - c[i] * x[i+1]
\end{lstlisting}

\begin{problem}
\leavevmode
\begin{enumerate}
\item Port the above code to Cython using typed for-loops, typed memoryviews and appropriate compiler directives.
\item Compare the speed of your new function with \li{pytridiag()} and \li{scipy.linalg.solve()}.
To compare the first two functions, start with $1000000 \times 1000000$ sized systems.
When testing the SciPy algorithm, start with $1000 \times 1000$ systems.
\item What do you learn about good implementation versus proper choice of algorithm?
\end{enumerate}
Note that an efficient tridiagonal matrix solver is implemented by \li{scipy.sparse.linalg.spsolve()}.
\end{problem}


\section{Scope and Motivation}
\label{sec:toyMC}

In this section, we discuss the crucialness of waveform analysis for incident light measurements in PMT-based experiments.

\subsection{Light curve}
A particle interacts with matter and deposits energy when passing through the detector. Part of such energy converts into visible Cherenkov or scintillation photons. The \textit{light curve} is the time evolution of light intensity at a PMT,
\begin{equation}
  \label{eq:light-curve}
  \mu\phi(t-t_0)
\end{equation}
where $\mu$ is the intensity factor, $t_0$ is the time shift factor and $\phi(\cdot)$ is the normalized shape function. We parameterize the scintillation light curve as several exponentials with different decay rates~\cite{rebber_particle_2021} and instantaneous Cherenkov light curve by a Dirac delta function.  It is convienent to model the PMT transit time spread~(TTS) in $\phi(t)$ as a Gaussian smear, giving an \textit{ex-Gaussian} or \textit{exponentially modified Gaussian}~\cite{li_separation_2016},
\begin{align}
    \phi(t) = \frac{1}{2\tau_\ell} \exp\left(\frac{\sigma_\ell^2}{2\tau_\ell^2}-\frac{t}{\tau_\ell}\right) \left[1 - \erf\left( \frac{\sigma_\ell}{\sqrt{2}\tau_\ell} - \frac{t}{\sqrt{2}\sigma_\ell} \right)\right],
    \label{eq:time-pro}
\end{align}
where subscript $\ell$ stands for ``light curve'' and $\sigma_\ell$ encodes the timing uncertainty mainly from TTS. $\phi(t)$ of Cherenkov light is a pure Gaussian by taking $\tau_\ell \rightarrow 0$. Figure~\ref{fig:time-pro} illustrates 3 examples of $\phi(t)$.

\begin{figure}[!htb]
  \centering
  \resizebox{0.5\textwidth}{!}{\input{figures/profile.pgf}}
  \caption{\label{fig:time-pro} Effective light curves in 3 settings.  A scintillator paired with ultra-fast photo-sensors gives the green curve with $\tau_\ell \gg \sigma_\ell$.  A fast Cherenkov detector by the red curve has $\tau_\ell \ll \sigma_\ell$.  The blue curve combining $\tau_\ell=\SI{20}{ns}$ and $\sigma_\ell=\SI{5}{ns}$ represents a typical scintillation detector.  We can regard $\phi(t)\mathrm{d}t$ as a probability density function~(PDF at the vertical axis) of PE times.}
\end{figure}

\subsection{Single PE response}
\label{subsec:spe}

A PE induced by a photon at the PMT photocathode is then accelerated, collected, and amplified into $\num[retain-unity-mantissa=false]{\sim 1e7}$ electrons, forming a voltage pulse $V_\mathrm{PE}(t)$ in the PMT output.
\begin{figure}[H]
  \begin{subfigure}{.49\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/spe.pgf}}
    \caption{\label{fig:spe} Single PE response $V_\mathrm{PE}(t)$ in eq.~\eqref{eq:dayaspe}.}
  \end{subfigure}
  \begin{subfigure}{.49\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/wave.pgf}}
    \caption{\label{fig:pile} PE pile-up and white noise in an PMT waveform.}
  \end{subfigure}
  \caption{A single PE from a PMT induces a voltage pulse in \subref{fig:spe}. Multiple PEs piling up at a PMT form an input waveform $w(t)$ in \subref{fig:pile}, when PEs are barely visually separable from each other. Output charge $\hat{\bm{t}}, \hat{\bm{q}}$ also in \subref{fig:pile}, which we will discuss in~\ref{sec:algorithm}. }
\end{figure}

S.~Jetter~et al.~\cite{jetter_pmt_2012} parameterizes $V_\mathrm{PE}(t)$ as,
\begin{equation}
  V_\mathrm{PE}(t) = V_{0}\exp\left[-\frac{1}{2}\left(\frac{\log(t/\tau_\mathrm{PE})}{\sigma_\mathrm{PE}}\right)^{2}\right],
  \label{eq:dayaspe}
\end{equation}
where shape parameters $\tau_\mathrm{PE}=\SI{8}{ns}$, $\sigma_\mathrm{PE}=\SI{0.5}{ns}$ and $V_{0}=\SI{14.08}{mV}$, see figure~\ref{fig:spe}.  A sample of PEs from the light curve $\phi(t)$ in eq.~\eqref{eq:time-pro} can be formulated as several delta functions, also known as sparse spike train~\cite{levy_reconstruction_1981},
\begin{equation}
  \label{eq:lc-sample}
  \tilde{\phi}(t) = \sum_{i=1}^{N_{\mathrm{PE}}} q_i \delta(t-t_i),
\end{equation}
where $N_\mathrm{PE}$ is the number of PEs, following Poisson distribution with parameter $\mu$.  $t_i$ is the hit time of the $i$-th PE, $q_i$ is the relative charge of the $i$-th PE from charge distribution $\mathcal{N}(1,\sigma_\mathrm{q}^2)$.  $\sigma_\mathrm{q}$ is set to $0.4$ in our simulation.

A noise-free waveform $\tilde{w}(t)$ is convolution of $\tilde{\phi}(t)$ and $V_\mathrm{PE}(t)$, and the PMT voltage output waveform $w(t)$ is a time series modeled by the sum of $\tilde{w}(t)$ and a Gaussian white noise $\epsilon(t)$,
\begin{equation}
  \label{eq:1}
  \begin{aligned}
    \tilde{w}(t) &= \tilde{\phi}(t) \otimes V_\mathrm{PE}(t) \\
    w(t) &= \tilde{w}(t) + \epsilon(t) = \sum_{i=1}^{N_\mathrm{PE}} q_i V_\mathrm{PE}(t-t_i) + \epsilon(t).
  \end{aligned}
\end{equation}
See figure~\ref{fig:pile} for an example.

\subsection{Measurement of incident light}
\label{sec:time}
We see in figure~\ref{fig:pile} that pile-ups and noises hinder the time $t_i$ and charge $q_i$ of the PEs. Fortunately event reconstruction only takes the time shift $t_0$ and the intensity $\mu$ in eq.~\eqref{eq:light-curve} as inputs, where $t_0$ carries the time of flight information and $\mu$ is the expected $N_\mathrm{PE}$ in a real detector.  All the uncertainties of $\hat{t}_i$, $\hat{q}_i$ and $\hat{N}_\mathrm{PE}$ are reflected in $\hat{t}_0$ and $\hat{\mu}$. In this article, we denote $\hat{x}$ as an estimator for the corresponding variable $x$ if not otherwise specified.  Classical TDC extracts the waveform's threshold crossing time $\hat{t}_\mathrm{1st}$ to approximate hit time of the first PE, while QDC extracts total charge $Q$ from waveform integration to estimate $\mu$ by $\hat{\mu}_Q$.

\subsubsection{Time $t_0$}
\label{sec:time-shift-t_0}

$\hat{t}_\mathrm{1st}$ is a biased estimator of $t_0$.  It is affected by the light intensity $\mu$, as will be shown in figure~\ref{fig:biasmethods} and discussed in section~\ref{subsec:timeresolution}. We define the resolution $\sigma_\mathrm{1st}$ for a batch of waveforms as the standard deviation of $\Delta t_0 = \hat{t}_\mathrm{1st} - t_0$. From a hypothetical perfect measurement of $t_i$, we define an ideal maximum likelihood estimator~(MLE) $\hat{t}_\mathrm{ALL}$ to capture time information of all the PEs,
\begin{equation}
  \label{eq:2}
  \hat{t}_\mathrm{ALL} = \arg\underset{t_0}{\max} \prod_{i=1}^{N_\mathrm{PE}} \phi(t_i-t_0).
\end{equation}
The corresponding $\sigma_\mathrm{ALL}$ is the lower bound of time resolution.

To characterize $\hat{t}_\mathrm{1st}$ and $\hat{t}_\mathrm{ALL}$, we scan $\mu$ from \numrange{0}{30} for each light curve in figure~\ref{fig:time-pro}. We generate a sample of $\num[retain-unity-mantissa=false]{1e4}$ waveforms having at least 1 PE for every triplet of $(\tau_\ell, \sigma_\ell, \mu)$.  Figure~\ref{fig:reso-diff} shows that $\sigma_{\mathrm{1st}}$ equals $\sigma_{\mathrm{ALL}}$ when $\tau_\ell \gg \sigma_\ell$, because $\hat{t}_\mathrm{ALL}$ reduces to $\hat{t}_\mathrm{1st}$(=$\min_i t_i$) for an exponential light curve. For $\mu \to 0$, $\sigma_{\mathrm{1st}}$ and $\sigma_{\mathrm{ALL}}$ are also equal because at most 1 PE is available.  TDC is effective in those cases.

\begin{figure}[H]
  \centering
  \resizebox{0.8\textwidth}{!}{\input{figures/vs-deltadiv.pgf}}
  \caption{\label{fig:reso-diff} Time resolution comparisons between $\hat{t}_{\mathrm{ALL}}$~(solid lines, when using information of all PEs) and $\hat{t}_\mathrm{1st}$~(dashed lines, using the true first PE instead of the threshold crossing time to eliminate the influence from TDC, while their resolutions are in comparison).  The difference is manifested especially when $\sigma_\ell$ and $\mu$ are large. The three cases of $(\tau_\ell, \sigma_\ell)/\si{ns}$ represent the scintillation with ultra-fast PMTs~$(20, 0)$, Cherenkov~$(0, 5)$ and scintilation coupled with PMT TTS~$(20, 5)$. The performance of waveform analysis lies between $\sigma_{\mathrm{1st}}$ and $\sigma_{\mathrm{ALL}}$. }
\end{figure}

Nevertheless, for $\sigma_\ell > 0$ and $\mu >0$, the $\sigma_{\mathrm{1st}}$ significantly deteriorates from $\sigma_{\mathrm{ALL}}$.  For Cherenkov and scintillation experiments with non-negligible PMT TTS and occupancy, we shall explore more sophisticated waveform analysis algorithms to go beyond $\hat{t}_{\mathrm{1st}}$ and recover the accuracy of $\hat{t}_\mathrm{ALL}$ in eq.~\eqref{eq:2} from waveform in eq.~\eqref{eq:1}.

\subsubsection{Intensity $\mu$}
\label{sec:intensity-mu}

The charge estimator $\hat{\mu}_Q$
\begin{equation}
  \begin{aligned}
  \label{eq:mu-q}
  \hat{\mu}_Q = \frac{\int w(t) \mathrm{d} t}{\int V_\mathrm{PE}(t) \mathrm{d} t} &= \frac{\sum_{i=1}^{N_\mathrm{PE}} \int \left[ q_i V_\mathrm{PE}(t-t_i) + \epsilon(t) \right] \mathrm{d} t}{\int V_\mathrm{PE}(t) \mathrm{d} t} \\
  & = \sum_{i=1}^{N_\mathrm{PE}} q_i + \frac{\int \epsilon(t) \mathrm{d} t}{\int V_\mathrm{PE}(t) \mathrm{d} t}
  \end{aligned}
\end{equation}
is not only affected by Poisson fluctuation of $N_\mathrm{PE}$, but also by charge resolution of a PE $\sigma_\mathrm{q}$.  Sometimes we mitigate the impact of $\sigma_\mathrm{q}$ by rounding $\hat{\mu}_Q$ to integers.  It works well for $N_\mathrm{PE} \le 1$, which is equivalently a hit-based 0-1 $\mu$ estimator $\hat{\mu}_\mathrm{hit}$.  But for $N_\mathrm{PE} \ge 2$, it is hard to interpret $\hat{\mu}_Q$ rounding by physics principles and $\hat{\mu}_\mathrm{hit}$ does not gain any additional information from the extra PEs.

The goal of waveform analysis is to eliminate the impact of $\sigma_\mathrm{q}$ as much as possible.  The resolution of any $\mu$ estimator is therefore lower bounded by the pure Poisson fluctuation of $N_\mathrm{PE}$.

\subsubsection{Shape $\phi(\cdot)$}
\label{sec:shape-phicdot}

The shape of a light curve is determined by light emission time profile, PMT timing and light propagation including refraction, reflection, dispersion and scattering.  $\phi(\cdot)$ thus depends on event locations.  In this article, we model $\phi(\cdot)$ by eq.~\eqref{eq:time-pro} for simplicity and leave the $\phi(\cdot)$ variations to event reconstruction in the future publications.


\title{\bf Gravitational Lensing}

\section{Basics}

Under general relativity, in the presence of mass light is bent by the
curvature of spacetime. On astronomical scales this can cause the
phenomenon of {\it gravitational lensing}.

\subsection{Point mass lensing}

Understanding lensing begins with the point mass case. It can be shown
that a photon traveling by a point mass, with an impact parameter $r$,
is in the small deflection limit deflected by an angle:
\begin{equation}
\theta_D = \frac{4GM}{rc^2}
\end{equation}
This differs by a factor of two from the equivalent Newtonian
calculation. An important feature of lensing is that it is achromatic;
i.e., independent of wavelength.

Figure \ref{fig:symmetric} describes the symmetric point lens case and
defines the distances involved. In an analog to the optical thin lens
approximation, we define the {\it source plane} and the {\it lens
  plane}. In the perfectly aligned case the observer sees the source
as a ring surrounding the lens; perfect alignment means an offset
substantially than the source size.  A characteristic quantity of a
lens is radius of this ring, which is the {\it Einstein angle}:
\begin{equation}
\theta_E =  \sqrt{\frac{4GM}{c^2}} \sqrt{\frac{D_{LS}}{D_L D_S}}
\end{equation}
which can be related to the {\it Einstein radius} in the lens plane
$r_E = D_L \theta_E$.

Figure \ref{fig:offset} describes the offset point lens case. If the
source is a point source, this will result in two magnified (and one
highly demagnified) images for the observer. The condition on the
source angle:
\begin{equation}
\beta < \theta_E
\end{equation}
defines the {\it strong lensing} regime. In this regime, the two
images appear near the Einstein ring location. If the source is
extended instead of point-like, it can appear highly distorted in the
strong lensing case.

The opposite case is known as the {\it weak lensing} regime.

In either case, the distortion of lensing has an effect on the
apparent brightness of the object. The total magnification can be
defined as the increase in the solid angle of the image. This solid
angle increase occurs even if our instrumentation still cannot detect
the extended nature of the image. Because surface brightness (more
technically specific intensity) is conserved in general relativity
this magnification leads to an increase in total flux density.

For multiply imaged sources, the images formed follow different paths
of different distances. This fact leads to a relative delay between
photons that travel different paths. In addition, a different general
relativistic delay is associated with each path, known as the {\it
Shapiro delay}. Fluctuations in the source will appear to the observer
at different times. A measured delay yields a measurement of physical
distance that can in principle be used to determine the distances of
the source and lens.

\subsection{Lensing from extended mass sheets}

On cosmological scales, weak lensing outside the Einstein radius of
individual groups and clusters occurs, but is not well described by
single point mass lensing. We will instead here describe the lensing
as due to a sheet of mass in the lens plane of varying surface
density.

Let us consider a point in the source plane that (undeflected) would
be at angle $\vec{\beta}$. Let $\vec{x}$ represent the physical
position in the lens plane that the undeflected ray would have passed
through.  The deflection angle is the sum of the contributions of all
the mass in the lens plans:
\begin{equation}
\vec{\theta_D}\left(\vec{x}\right) = \frac{4G}{c^2
D_L}  \int \dd^2 \vec{x}' \Sigma\left(\vec{x}\right) \frac{\vec{x}
- \vec{x}'}{\left|\vec{x} - \vec{x}'\right|^2}
\end{equation}
We can relate the source plane position $\vec{\beta}$ to the observed
angle $\theta$ with the {\it lens equation}:
\begin{equation}
\vec{\beta} = \vec{\theta}
- \frac{D_{LS}}{D_S} \vec{\theta_D}\left(D_L \vec{\theta}\right)
  = \vec{\theta} - \vec{\alpha}
\end{equation}
We can use the above relations to show:
\begin{equation}
\vec{\alpha}
= \frac{1}{\pi} \int \dd^2\vec{\theta}' \kappa\left(\vec{\theta}'\right)
\frac{\vec{\theta} - \vec{\theta}'}
{\left|\vec{\theta} - \vec{\theta}'\right|^2}.
\end{equation}
where we define:
\begin{equation}
\kappa = \frac{\Sigma}{\Sigma_{\rm cr}},
\end{equation}
and:
\begin{equation}
\Sigma_{\rm cr} = \frac{c^2 D_S}{4\pi G D_{LS} D_L}
\end{equation}
The condition $\kappa>1$ leads to multiply imaged sources.

The form of $\vec{\alpha}$ suggests that it can be written as the
gradient of a potential,
\begin{equation}
\vec{\beta} = \vec{\theta} - \vec{\nabla}\psi,
\end{equation}
where
\begin{equation}
\psi
= \frac{1}{\pi} \int \dd^2\vec{\theta}' \kappa\left(\vec{\theta}'\right)
\ln \left|\vec{\theta} - \vec{\theta}'\right|.
\end{equation}
We can also show:
\begin{equation}
\label{eq:poissonlike}
\nabla^2\psi = 2 \kappa
\end{equation}
We can define the Fermat time delay potential as
\begin{equation}
\tau\left(\vec{\theta}; \vec{\beta}\right) =
\frac{1}{2} \left(\vec{\theta} - \vec{\beta}\right)^2
- \psi\left(\vec{\theta}\right),
\end{equation}
and the lens equation can be rewritten as.
\begin{equation}
\vec{\nabla}\tau = 0.
\end{equation}
This result is an expression of the general relativistic version of
Fermat's principle.

\subsection{Weak lensing}

The lens equation can be locally linearized  around $\vec{\beta_0}$:
\begin{equation}
\vec{\beta} = \vec{\beta}_0
+ \frac{\partial \vec{\beta}}{\partial \vec{\theta}} \cdot \left(\vec{\theta}
- \vec{\theta_D} \right),
\end{equation}
The vector $\vec{\theta}$ and the Jacobian can be written in index form:
\begin{eqnarray}
\vec{\theta} &=& \theta_i {\hat e}_i = \theta_1 {\hat e}_1
+ \theta_2 {\hat e}_2 \cr
{\bf A}\left(\vec{\theta}\right) &=&
\frac{\partial \vec{\beta}}{\partial \vec{\theta}} = \left(\delta_{ij}
- \frac{\partial^2 \psi}{\partial\theta_i \partial\theta_j}\right)
{\hat e}_i {\hat e}_j.
\end{eqnarray}

The Jacobian is symmetric, so has only three independent parameters,
and we can define three parameters to characterize it, $\kappa$,
$\gamma_1$, and $\gamma_2$:
\begin{equation}
{\bf A} = \left(\begin{array}{cc}
1-\kappa - \gamma_1 & -\gamma_2 \cr
- \gamma_2 & 1-\kappa + \gamma_1
\end{array}\right).
\end{equation}
We do this because, as we are about to show, $\kappa$ (called the {\it
convergence}) is basically the isotropic component of the distortion,
and $\gamma = \gamma_1 + i\gamma_2 = |\gamma| \exp(2i\phi)$ is a shear
term expressing the nonisotropic component, including the rotation of
the image $\phi$.

If you have a circularly symmetric source on sky, this linear
transformation will convert it into an ellipse. You can determine the
parameters of the ellipse from the eigenspace of the of the distortion
matrix. The two eigenvalues are
\begin{equation}
\lambda_{\pm} = (1-\kappa) \pm |\gamma|
\end{equation}
The eigenvector associated with $\lambda_+$ is rotated from ${\hat
e}_1$ towards ${\hat e}_2$ by the angle $\phi$, which is defined by:
\begin{equation}
\cos 2\phi = \frac{\gamma_1}{|\gamma|}
\end{equation}
Based on the eigenvalues, the magnification is:
\begin{equation}
M = \left(\lambda_+ \lambda_-\right)^{-1} = \left[(1-\kappa)^2 -
|\gamma|^2\right]^{-1}
\end{equation}

In order for $\kappa$ to express an isotropic distortion, then we must
define:
\begin{eqnarray}
\kappa &=& \frac{1}{2}\left(
\frac{\partial^2\psi}{\partial\theta_1^2} +
\frac{\partial^2\psi}{\partial\theta_2^2}\right) \cr
\gamma_1 &=& \frac{1}{2} \left(
\frac{\partial^2\psi}{\partial\theta_1^2} -
\frac{\partial^2\psi}{\partial\theta_2^2}\right),
\end{eqnarray}
and we can further see:
\begin{eqnarray}
\gamma_2 &=& \frac{\partial^2\psi}{\partial\theta_1 \partial\theta_2}.
\end{eqnarray}
Therefore, we can relate the lensing potential curvature to the shear
and convergence of distortions.

We also see why we defined the convergence with $\kappa$, because it
is related to $\psi$ in the same way as the $\kappa$ in the previous
subsection (Equation \ref{eq:poissonlike}).

In reality, the lensing is integrated through a series of mass sheets
that comprise the three-dimensional density field. In addition, the
convergence due to the mean density is already accounted for in the
cosmological comoving transerve distance, luminosity distance, and
angular diameter distances. The cosmological comoving transverse
distance is that which is relevant to lensing. A rigorous derivation
is beyond our scope here, but the consequence is that the convergence
in some direction for sources at $D=D_S$ can be written as:
\begin{equation}
\kappa(\vec{\theta}, D_S)
= \frac{3 \Omega_m}{2} \int_0^{D_S} \frac{\dd{D_L}}{a(D_L)}
\frac{D_{LS} D_L}{D_S} \delta(\vec{\theta}, D_L)
\end{equation}
where is it understood that $D_L$, $D_{LS}$, and $z_L$ are the
appropriate quantities for a giving lens distance $D_L$. Two important
aspects of this formula are the dependence on $\Omega_m$ and the
weighting factor $D_{LS} D_L$, which tells us which distances do the
most lensing. For a flat universe, $D_{LS}D_L = (D_S - D_L)D_L$, which
has a maximum at $D_L = D_S/2$, so most of the lensing effect is at
from about half the distance of the source. The convergence is also
dependent on the distance of both the lenses and sources, which
therefore need to be known to interpret lensing data accurately.

Weak lensing can be observed through its effects on the size and
brightness of sources (the magnification) or due to the induced change
in ellipticity. Magnification causes a {\it magnification bias} in
flux-limited samples with steep flux counts, because the magnification
will bring a large number of sources from below the (unlensed) flux
limit.  Magnification can also be detected by correlating the number
of background sources against foreground lens galaxies.

The changes in ellipticity can only be observed statistically, because
galaxies at best have random ellipticities (in fact, it is worse as
their ellipticities tend to be aligned, an effect known as {\it
intrinsic alignment}). If you observe an individual galaxy, its
typical ellipticity is $\sim 0.3$ whereas the lensing induced
ellipticity is $<10^{-3}$. One can average over many galaxies along
some line of site to detect a net ellipticity that can be associated
with lensing, which is the ratio of eigenvalues:
\begin{equation}
\frac{\lambda_+}{\lambda_-} = \frac{1-\kappa +  |\gamma|}{1-\kappa -
|\gamma|}
\end{equation}
Since one cannot nearly as easily determine the magnification and
therefore $\kappa$, the ratio in effect one is measuring the {\it
reduced shear}:
\begin{equation}
g = \frac{\gamma}{1-\kappa},
\end{equation}
for which
\begin{equation}
\frac{\lambda_+}{\lambda_-} = \frac{1 +  |g|}{1- |g|}.
\end{equation}

There are two general ways the shear is used. The first is the {\it
cosmic shear}, which looks at shear-shear correlations. This method
yields a fairly direct constraint on the statistics of the total mass
density fluctuations. The second is {\it galaxy-galaxy lensing}
correlations, which cross-correlates known foreground sources with
background shear patterns. In general galaxy-galaxy lensing is usually
easier, because the cross-correlation increases the signal-to-noise
and because it tends to average over systematics.

The shear-shear correlation is a bit subtle, because there is a
component of the shear transverse to the separation vector, and
cross-wise from it. The treatment of this is beyond our scope
here. The key points are that the shear is interrelated with the
surface mass density. In particular:
\begin{eqnarray}
\gamma &=& \frac{1}{2}\left(\partial\psi_{,11}
- \partial\psi_{,22}\right) + i \partial\psi_{,12} \cr
&=& \frac{1}{\pi} \int \dd^2\vec{\theta}' \kappa\left(\vec{\theta}'\right)
\left[\frac{1}{2}\left(\partial_1\partial_1 - \partial_2 \partial_2\right)
+ i \partial_1 \partial_2\right]
\ln \left|\vec{\theta} - \vec{\theta}'\right] \cr
&=& \frac{1}{\pi} \int \dd^2\vec{\theta}' \kappa\left(\vec{\theta}'\right)
\left( \frac{\theta_2^2 - \theta_1^2 -
2i \theta_1\theta_2}{\theta^4} \right)
\end{eqnarray}
So the shear is related through an integral with the surface mass
density. This integral may be inverted from shear data to yield the
mass density; it is a bit tricky because the kernel is $1/\theta^2$ so
edge and finite volume effects are important (though remember it is in
two-dimensions so this scaling is not as bad as for three dimensions).
It can be further shown that the correlation function and power
spectrum of the shear field can be directly related to the convergence
field, meaning that the statistics of the shear field can be used
directly (without explicitly building a map of $\kappa$).

Galaxy-galaxy lensing typically is used by measuring the mean
tangential shear $\gamma_t$ around identified foreground sources
(usually galaxies or clusters of galaxies).

\subsection{Microlensing}

A phenomenon called {\it microlensing} occurs when the lensing mass
and background source have a relative angular motion. The background
source increases as it moves through the Einstein radius of the
lens. This increase has a distinctive, achromatic signature, that can
be seen for individual stars in our Galaxy through monitoring.

A related phenomenon also known as microlensing occurs when viewing a
background source through a galactic system. The stars create a
lensing potential surface with distinct cusps that cause fluctuations
in the flux of the background source.

These phenomena can only occur if the background source is physically
smaller than the Einstein radius. Otherwise even if the center of the
source is aligned with the lens, most of the light is well outside the
Einstein radius in the lens plane and is not deflected. This fact
makes it possible to constrain the relative sizes of the background
source in different wavelengths (e.g. radio vs. optical) through
observations of its lensing.

\section{Important numbers}

\section{Key References}

\begin{itemize}
  \item
    {\it Strong Lensing by Galaxies, \href{https://ui.adsabs.harvard.edu/abs/2010ARA%26A..48...87T/abstract}
    {\citet{treu10a}}}
  \item
    {\it Weak lensing for precision cosmology, \href{https://ui.adsabs.harvard.edu/abs/2018ARA%26A..56..393M/abstract}
    {\citet{mandelbaum18a}}}
\end{itemize}

\section{Order-of-magnitude Exercises}

\begin{enumerate}
\item Calculate a typical Einstein angles for:
\begin{enumerate}
\item A stellar mass lens between the Sun and the Galactic Center.
\item A galaxy lens at $z\sim 0.1$, lensing objects at $z\sim 1$.
\item A galaxy cluster lens at $z\sim 0.1$, lensing objects at $z\sim
    1$.
\end{enumerate}
\item For a Milky Way-mass galaxy lens at cosmological distances (100s
    of Mpc) and a background source about twice that distance,
    estimate the typical time delay for a multiply-lensed system
    Assume a point mass lens and that the source is at
    $\beta\sim \theta_E/2$, and ignore the General Relativistic
    Shapiro delay contribution.
\item Assuming a constant density of stars in the Galactic disk,
    estimate the probability of lensing of any given star near the
    Galactic center.
\item Typical shear values
\end{enumerate}

\section{Analytic Exercises}

\begin{enumerate}
\item GR calculation of lensing offset
\item For a symmetric, point mass lens, derive the angular radius of the image
    that is formed, called the Einstein angle.
\begin{answer}
Using the notation in Figure \ref{fig:symmetric}, geometrically it
must be that:
\begin{equation}
\theta_D = \theta_S + \theta_E
\end{equation}
Under the small angle approximation, therefore:
\begin{equation}
\theta_D = r_E \left(\frac{1}{D_L} + \frac{1}{D_{LS}}\right) =
r_E \frac{D_S}{D_{LS} D_L}
\end{equation}
Therefore:
\begin{equation}
r_E = \frac{D_{LS} D_L}{D_S} \theta_D.
\end{equation}
Again using the small angle approximation, the impact parameter is
$r_E$, so we can write:
\begin{equation}
r_E = \frac{D_{LS} D_L}{D_S} \frac{4GM}{r_Ec^2}
\end{equation}
and solve for:
\begin{equation}
r_E = \sqrt{\frac{4GM}{c^2} \frac{D_{LS} D_L}{D_S}}
\end{equation}
The Einstein angle can then be calculated (again using small angles):
\begin{equation}
\theta_E = \sqrt{\frac{4GM}{c^2} \frac{D_{LS}}{D_S D_L}}
\end{equation}
\end{answer}

\item Calculate the location of the two magnified images that form when the
source is offset from the point lens.
\begin{answer}
Using the notation in Figure \ref{fig:offset}, we have:
\begin{equation}
r_I = r_S + r_D,
\end{equation}
and therefore:
\begin{eqnarray}
\theta_\pm D_S &=& \beta D_S + D_{LS} \theta_D \cr
\theta_\pm &=& \beta + \frac{D_{LS}}{D_S} \frac{4GM}{r_\pm c^2} \cr
&=& \beta + \frac{D_{LS}}{D_S D_L} \frac{4GM}{\theta_\pm c^2} \cr
&=& \beta + \frac{\theta_E^2}{\theta_\pm}
\end{eqnarray}
Then we can rearrange this to:
\begin{equation}
\theta_\pm^2 - \beta \theta_\pm - \theta_E^2 = 0
\end{equation}
with the solutions:
\begin{equation}
\theta_\pm = \frac{\beta \pm \sqrt{\beta^2 + 4\theta_E^2}}{2}
\end{equation}
For the strong-lensing limit $\beta \ll \theta_E$, this leads to:
\begin{equation}
\theta_\pm = \pm \theta_E + \frac{\beta}{2}
\end{equation}
For the weak-lensing limit $\beta \gg \theta_E$:
\begin{equation}
\theta_\pm
= \beta \left(\frac{1}{2} \pm \frac{1}{2} \sqrt{1+
\frac{4\theta_E^2}{\beta^2}}\right)
\end{equation}
and we find:
\begin{eqnarray}
\theta_+ &\approx& \beta + \frac{\theta_E^2}{\beta} \cr
\theta_- &\approx& - \frac{\theta_E^2}{\beta}
\end{eqnarray}
\end{answer}
\item In the weak lensing limit for a point mass lens,  what is the
magnification?
\begin{answer}
Consider a source at $\beta$, with a size in the radial and tangential
directions of $\dd\beta$ and $\dd\phi$ (for the polar coordinate
$\phi$). Its unlensed area is:
\begin{equation}
A_S = \beta \dd\beta \dd\phi
\end{equation}
The area of each lensed image is:
\begin{equation}
A_\pm = \left| \theta_\pm\right| \dd\theta_\pm \dd\phi
= \left| \theta_\pm\right| \left|\frac{\dd\theta_\pm}{\dd\beta}\right| \dd\beta\dd\phi
= \left|\frac{\theta_\pm}{\beta}\right| \left|\frac{\dd\theta_\pm}{\dd\beta}\right| A_S
\end{equation}
Then:
\begin{equation}
M = \frac{A_+ + A_-}{A_S} = \frac{1+
2\theta_E^2/\beta^2}{\sqrt{1+4\theta_E^2/\beta^2} }
\end{equation}
\end{answer}
\item Critical surface density case
\item Derive shear and magnification properties
\end{enumerate}

\section{Numerics and Data Exercises}

\begin{enumerate}
\item Modeling of lens system
\item Specific strong lenses
\item Measurements of shear
\end{enumerate}

\bibliographystyle{apj}
\bibliography{exex}

\begin{figure}
\includegraphics[width=0.9\textwidth]{figures/symmetric_lens.jpg}
\caption{\label{fig:symmetric} Geometry for symmetric point mass lens.}
\end{figure}

\begin{figure}
\includegraphics[width=0.9\textwidth]{figures/offset_lens.jpg}
\caption{\label{fig:offset} Geometry for offset point mass lens.}
\end{figure}