\documentclass[a4paper]{article}
%\documentclass[11pt]{article}
%\documentclass[12pt]{article}
%LATEX HEADER

\ifx\pdfoutput\undefined
% we are running LaTeX, not pdflatex
\usepackage{graphicx}
\usepackage{overpic}
\else
% we are running pdflatex, so convert .eps files to .pdf
%\usepackage[pdftex]{graphicx}
\usepackage{graphicx}
\usepackage{overpic}
\usepackage{epstopdf}
\fi

\usepackage{mathrsfs}
\usepackage{color}
\usepackage{verbatim}
\usepackage{epsfig}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{float}
\usepackage{setspace}
\usepackage{moreverb}
\usepackage{alltt}
\usepackage{ulem}
\usepackage{amsthm}
\usepackage{cancel}
\usepackage{verbatim}

\setlength{\textwidth}{17cm}
% for US letter
%\setlength{\textheight}{22.5cm}
% for A4

\setlength{\textheight}{24cm}
\setlength{\parskip}{0.3cm plus 0.1 cm minus 0.1 cm}
\setlength{\oddsidemargin}{-0.5cm}
\setlength{\topmargin}{0.5cm}
\setlength{\headsep}{0cm}
\setlength{\headheight}{0cm}
\setlength{\parindent}{0.5cm}
% remove extra space at top and bottom of lists
%\setlength{\partopsep}{-2ex}

% section,subsection and subsubsection
% numbering enable
\setcounter{secnumdepth}{3}
\setcounter{tocdepth}{3}

\newcommand{\bit}{\begin{itemize}}
\newcommand{\eit}{\end{itemize}}
\newcommand{\bnum}{\begin{enumerate}}
\newcommand{\enum}{\end{enumerate}}
\newcommand{\bt}{\begin{tabular}}
\newcommand{\et}{\end{tabular}}
\newcommand{\be}{\begin{equation}}
\newcommand{\ee}{\end{equation}}
\newcommand{\pdv}[2]{\frac{\partial #1}{\partial #2}}
\newcommand{\dv}[2]{\frac{d #1}{d #2}}
\newcommand{\ds}{\displaystyle}
\newcommand{\vect}[1]{\mbox{\boldmath $#1$}}
\newcommand{\subs}[1]{$_{#1}$}
\newcommand{\sups}[1]{$^{#1}$}
%\pagestyle{empty}% no page numbering

%in order to obtain doublspacing put following 2 lines in the
%preamble and the last line just above the place where the
%double-spacing should commence.
\def\doublespaced{\baselineskip=1.5\normalbaselineskip}
\let\doublespace=\doublespaced

\begin{document}

% for double spacing
\title{\Huge A Wrapper Class For LAPACK and BLAS }
% \Large
\date{\today}
\author{Minh Tran\\Email: minh.t.tran@adelaide.edu.au}
\maketitle
% %\tableofcontents
% %\clearpage

\section{Introduction} \label{Introduction}
The Matrix class is a wrapper class for LAPACK and BLAS. The class implements matrix solvers and operators typically used in mathematical modelling of physical systems. It is the hope of the author that the class will be useful for anyone working with C++ projects that require matrix operations.

\section{Examples} \label{Examples}
The use of class will be demonstrated through examples.

\subsection{Example 1}

$\boldsymbol{A}_{1}$ = $\begin{bmatrix}
5 &  7 &  6 & 5 \\
7 & 10 &  8 & 7 \\
6 &  8 & 10 & 9 \\
5 &  7 &  9 & 10
 \end{bmatrix}$ \hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{b}_{1}$ = $\begin{bmatrix}
23 \\
32 \\
33 \\
31
\end{bmatrix}$

\vspace{0.5cm}

Consider the system,
\begin{equation*}
\begin{aligned}
\left(5*\boldsymbol{A}_{1}+\boldsymbol{A}_{1}*15\right)  * \boldsymbol{x}_{1}  = \boldsymbol{b}_{1}
\end{aligned}
\end{equation*}

\begin{equation*}
\begin{aligned}
\boldsymbol{x}_{1}  &= \left(5*\boldsymbol{A}_{1}+\boldsymbol{A}_{1}*15\right)^{-1} * \boldsymbol{b}_{1} \\
\boldsymbol{x}_{1}  &=
\begin{bmatrix}
		0.050 \\
		0.050 \\
		0.050 \\
		0.050
\end{bmatrix}
\end{aligned}
\end{equation*}

Below are codes to compute $\boldsymbol{x}_{1}$. The result is stored in the variable \texttt{example1}.
\begin{verbatim}
double A1[] = { 5 ,  7 ,  6 , 5 ,
                7 , 10 ,  8 , 7 ,
                6 ,  8 , 10 , 9 ,
                5 ,  7 ,  9 , 10 };
double b1[] = { 23 , 32 , 33 , 31 };
Matrix P1(A1,4);
Matrix example1 = (5*P1+P1*15) | b1;
\end{verbatim}

\subsection{Example 2}

$\boldsymbol{A}_{2}$ = $\begin{bmatrix}
0 & 1 & 2 \\
3 & 4 & 5 \\
6 & 7 & 0 \\
\end{bmatrix}$ \hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{b}_{2}$ = $\begin{bmatrix}
1 \\
2 \\
3 \\
\end{bmatrix}$

\vspace{0.5cm}

Consider the system,
\begin{equation*}
\begin{aligned}
\left( \boldsymbol{A}_{2}*\boldsymbol{A}_{2}*\boldsymbol{A}_{2} \right)  * \boldsymbol{x}_{2}  = \boldsymbol{b}_{2}
\end{aligned}
\end{equation*}

\begin{equation*}
\begin{aligned}
\boldsymbol{x}_{2}  &= \left(\boldsymbol{A}_{2}*\boldsymbol{A}_{2}*\boldsymbol{A}_{2} \right)^{-1} * \boldsymbol{b}_{2} \\
\boldsymbol{x}_{2}  &\approx
\begin{bmatrix}
-3.088 \\
2.694 \\
-0.569 \\
\end{bmatrix}
\end{aligned}
\end{equation*}

Below are codes to compute $\boldsymbol{x}_{2}$. The result is stored in the variable \texttt{example2}.

\begin{verbatim}
double A2[] = { 0 , 1 , 2 ,
                3 , 4 , 5 ,
                6 , 7 , 0  };
double B2[] = { 1 , 2 , 3 };
Matrix P2(A2, 3);
Matrix example2 = (P2*P2*P2) | B2;
\end{verbatim}

\subsection{Example 3 \& 4}

$\boldsymbol{A}_{3}$ = $\begin{bmatrix}
 0 &  1 &  0 & 0\\
-1 &  0 &  1 & 0\\
 0 & -1 &  0 & 1\\
 0 &  0 & -1 & 0\\
\end{bmatrix}$ \hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{d}$ = $\begin{bmatrix}
1 \\
2 \\
3 \\
4 \\
\end{bmatrix}$

\vspace{0.5cm}

Consider the system,
\begin{equation*}
\begin{aligned}
\boldsymbol{A}_{3} * \boldsymbol{x}_{3}  = \boldsymbol{d}
\end{aligned}
\end{equation*}

\begin{equation*}
\begin{aligned}
\boldsymbol{x}_{3}  &=  \boldsymbol{A}_{3}  ^{-1} * \boldsymbol{d}  \\
\boldsymbol{x}_{3}  &\approx
\begin{bmatrix}
-6 \\
1 \\
-4 \\
4\\
\end{bmatrix}
\end{aligned}
\end{equation*}

Below are codes to compute $\boldsymbol{x}_{3}$. The result is stored in the variables \texttt{example3}. The system for \texttt{example4} is a similar to  \texttt{example3}. The main difference is $\boldsymbol{d}$ is an array and $\boldsymbol{Q}_{3}$ is a matrix object.

\begin{verbatim}
double a[] = { -1, -1, -1 };
double b[] = { 0, 0, 0, 0};
double c[] = { 1, 1, 1 };
double d[] = { 1, 2, 3, 4 };
Matrix Q3(d, 4,1);
Matrix P3(a,b,c,4);
Matrix example3 = P3 | d;
Matrix example4 = P3 | Q3;
\end{verbatim}

\subsection{Example 5 \& 6}

This example demonstrates the use of pseudo inverse to compute a solution of the system below.

$\boldsymbol{A}_{5}$ = $\begin{bmatrix}
 -74 &  80 &  18 & -11 & -4 \\
  14 & -69 &  21 &  28 & 0 \\
  66 & -72 &  -5 &   7 & 1 \\
 -12 &  66 & -30 & -23 & 3 \\
   3 &   8 &  -7 &  -4 & 1 \\
   4 & -12 &   4 &   4 & 0\\
\end{bmatrix}$
\hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{d}$ = $\begin{bmatrix}
51 \\
-61 \\
-56\\
 69\\
 10\\
 -12\\
\end{bmatrix}$

\vspace{0.5cm}

Consider the system,
\begin{equation*}
\begin{aligned}
\boldsymbol{A}_{5} * \boldsymbol{x}_{5}  = \boldsymbol{d}
\end{aligned}
\end{equation*}

\begin{equation*}
\begin{aligned}
\boldsymbol{x}_{5}  &=  \boldsymbol{A}_{5}  ^{-1} * \boldsymbol{d}  \\
\boldsymbol{x}_{5}  &=
\begin{bmatrix}
1 \\
2 \\
-1 \\
3 \\
-4 \\
\end{bmatrix}
\end{aligned}
\end{equation*}

Below are codes to compute $\boldsymbol{x}_{5}$. The result is stored in the variables \texttt{example5}. The system for \texttt{example6} is a similar to  \texttt{example5}. The main difference is $\boldsymbol{b}_{5}$ is an array and $\boldsymbol{Q}_{5}$ is a matrix object.

\begin{verbatim}
double A5[30] = {  -74 ,  80 ,  18 , -11 , -4 ,
                    14 , -69 ,  21 ,  28 , 0 ,
                    66 , -72 ,  -5 ,   7 , 1 ,
                   -12 ,  66 , -30 , -23 , 3 ,
                     3 ,   8 ,  -7 ,  -4 , 1 ,
                     4 , -12 ,   4 ,   4 , 0};
double b5[6] = { 51 , -61 , -56 , 69 , 10 , -12 };
Matrix P5(A5,6,5);
Matrix example5 = P5 | b5;
Matrix Q5(b5, 6, 1);
Matrix example6 = P5 | Q5;
\end{verbatim}

\subsection{Example 7}

Consider solving the system below,

\begin{equation*}
\begin{aligned}
\underset{\boldsymbol{x},\boldsymbol{y}}{\min} \hspace{0.2cm} || \boldsymbol{y} ||_{2}  \hspace{0.5cm} \text{ subject to } \hspace{0.5cm}  \boldsymbol{d}_{7} = \boldsymbol{A}_{7}\boldsymbol{x} + \boldsymbol{B}_{7}\boldsymbol{y}
\end{aligned}
\end{equation*}

$\boldsymbol{A}_{7}$ = $\begin{bmatrix}
1 &  2 &  1 & 4  \\
-1 &  1 &  1 & 1  \\
-1 & -2 & -1 & 1  \\
-1 &  2 & -1 & -1 \\
1 &  1 &  1 & 2 \\
\end{bmatrix}$
\hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{B}_{7}$ = $\begin{bmatrix}
1 &  2 & 2 \\
-1 &  1 & -2 \\
3 &  1 & 6 \\
2 & -2 & 4 \\
1 & -1 & 2\\
\end{bmatrix}$
\hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{d}_{7}$ = $\begin{bmatrix}
7.99 \\
0.98 \\
-2.98 \\
3.04 \\
4.02 \\
\end{bmatrix}$

Solution:

$\boldsymbol{x}$ = $\begin{bmatrix}
1.002951 \\
2.001436 \\
-0.987798 \\
0.990908 \\
\end{bmatrix}$
\hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{y}$ = $\begin{bmatrix}
0.003436 \\
-0.004417 \\
0.006871 \\
\end{bmatrix}$

Below are codes to compute $\boldsymbol{x}$ and $\boldsymbol{y}$. The variable \texttt{info} is used to flag whether the algorithm is able to obtain a solution to the system.

\begin{verbatim}
double *x7 = new double[4];
double *y7 = new double[3];
double A7[] = { 1 ,  2 ,  1 ,  4 ,
               -1 ,  1 ,  1 ,  1 ,
               -1 , -2 , -1 ,  1 ,
               -1 ,  2 , -1 , -1 ,
                1 ,  1 ,  1 ,  2} ;
double B7[] = { 1 ,  2 ,  2 ,
               -1 ,  1 , -2 ,
                3 ,  1 ,  6 ,
                2 , -2 ,  4 ,
                1 , -1 , 2 };
double d7[] = { 7.99 , 0.98 , -2.98 , 3.04 , 4.02 };
Matrix P7(A7, B7, d7, 5, 4, 3);
int info7 = P7.solve(x7,y7);
\end{verbatim}

\subsection{Example 8}

Consider solving the system below,

\begin{equation*}
\begin{aligned}
\underset{\boldsymbol{x}_{8}}{\min} \hspace{0.2cm} ||\boldsymbol{d}_{8} - \boldsymbol{A}_{8}\boldsymbol{x}_{8} ||_{2}  \hspace{0.5cm} \text{ subject to } \hspace{0.5cm}   \boldsymbol{H}_{8}\boldsymbol{x}_{8} = \boldsymbol{f}_{8}
\end{aligned}
\end{equation*}

$\boldsymbol{A}_{8}$ = $\begin{bmatrix}
1 &  1 & 1 \\
1 &  3 & 1 \\
1 & -1 & 1 \\
1 &  1 & 1 \\
\end{bmatrix}$
\hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{d}_{8}$ = $\begin{bmatrix}
1 \\
2 \\
3 \\
4\\
\end{bmatrix}$
\hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{H}_{8}$ = $\begin{bmatrix}
1 & 1 & 1 \\
1 & 1 & -1 \\
\end{bmatrix}$
\hspace{0.5cm} , \hspace{0.5cm}
$\boldsymbol{f}_{8}$ = $\begin{bmatrix}
7 \\
4 \\
\end{bmatrix}$

Solution:

$\boldsymbol{x}_{8}$ = $\begin{bmatrix}
5.75 \\
-0.25 \\
1.5  \\
\end{bmatrix}$

Below are codes to compute a solution to the system. The variable \texttt{info} is used to flag whether the algorithm is able to obtain a solution to the system.

\begin{verbatim}
double A8[] = { 1 ,  1 , 1 ,
                1 ,  3 , 1 ,
                1 , -1 , 1 ,
                1 ,  1 , 1};
double H8[] = { 1 , 1 , 1 ,
                1 , 1 , -1 };
double D8[] = { 1 ,  2 , 3 , 4 };
double F8[] = { 7 ,
4 };
double *x8 = new double[3];
Matrix P8(A8, D8, H8, F8, 4, 3, 2);
int info8 = P8.solve(x8);
printf("\n X8 = \n ");
Matrix::printMatrix(x8,3,1);
\end{verbatim}

\section{Constructor}
Below is a description of the syntax used to create a single matrix or a matrix system.

\subsection{Create a Single Matrix Object}

\subsubsection{\texttt{Matrix( double Ain[], int nd )}}

This is used to create a square matrix with dimension of $n_{d}*n_{d}$.
The variable $A_{in}$ is an array storing entries of the matrix. The entries are read in row by row.

\subsubsection{\texttt{Matrix(double Ain[], int nd , int md)}}

This is used to create a general matrix with dimension of $n_{d}*m_{d}$.
The variable $A_{in}$ is an array storing entries of the matrix. The entries are read in row by row.

\subsubsection{\texttt{Matrix(double ain[], double bin[], double cin[], int nd)}}

Create a tridiagonal matrix with dimension of $n_{d}*n_{d}$.
The array $b_{in}$ is the main diagonal with $n_{d}$ number of entries. The array $a_{in}$ is the subdiagonal with $n_{d}-1$ number of entries. The array $c_{in}$ is the superdiagonal  with $n_{d}-1$ number of entries.

\subsubsection{\texttt{Matrix(double Ain[], int nd, int md, bool isTranspose)}}

This is used to create a general matrix with dimension of $n_{d}*m_{d}$. The variable $A_{in}$ is an array storing entries of the matrix. The entries are read in row by row if \texttt{isTranspose} is \texttt{false}. On the other hand, if  \texttt{isTranspose} is \texttt{true} entries are read in column by column.

\subsubsection{\texttt{Matrix(double Ain[], int nd, int md, bool isTranspose, int flag)}}

This is used to create a general matrix with dimension of $n_{d}*m_{d}$. The variable $A_{in}$ is an array storing entries of the matrix. The entries are read in row by row if \texttt{isTranspose} is \texttt{false}. On the other hand, if  \texttt{isTranspose} is \texttt{true} entries are read in column by column. The parameter input \texttt{flag} is used to set the default \texttt{info}  about the matrix. That is, if \texttt{flag} = 0, then the matrix is non-singular. It should be noted that \texttt{info}  is an instance field of the Matrix class.

\subsection{Create a Matrix System}

\subsubsection{\texttt{Matrix(double Ain[], double din[], double Hin[], double fin[], int nd , int md, int pd)}}

This is used to construct the following matrix system.

\begin{equation*}
\begin{aligned}
\underset{\boldsymbol{x}}{\min} \hspace{0.2cm} ||\boldsymbol{d} - \boldsymbol{A}\boldsymbol{x} ||_{2}  \hspace{0.5cm} \text{ subject to } \hspace{0.5cm}   \boldsymbol{H}\boldsymbol{x} = \boldsymbol{f}
\end{aligned}
\end{equation*}

Dimension of the matrix:
\begin{equation*}
\begin{aligned}
\boldsymbol{A} &\hspace{0.7cm}&& \text{dimension } n_{d}*m_{d} \\
\boldsymbol{H} &\hspace{0.7cm}&& \text{dimension } p_{d}*m_{d}\\
\boldsymbol{d} &\hspace{0.7cm}&& \text{dimension } n_{d}*1\\
\boldsymbol{f} &\hspace{0.7cm}&& \text{dimension } p_{d}*1\\
\boldsymbol{x} &\hspace{0.7cm}&& \text{dimension } m_{d}*1\\
\end{aligned}
\end{equation*}

The method ``\texttt{int Matrix::solve(double xinout[])}" is used to solve the system.

\subsubsection{\texttt{Matrix(double Ain[], double Bin[], double din[], int nd, int md, int pd)}}

This is used to construct the following matrix system.

\begin{equation*}
\begin{aligned}
\underset{\boldsymbol{x},\boldsymbol{y}}{\min} \hspace{0.2cm} || \boldsymbol{y} ||_{2}  \hspace{0.5cm} \text{ subject to } \hspace{0.5cm}  \boldsymbol{d}  = \boldsymbol{A} \boldsymbol{x} + \boldsymbol{B} \boldsymbol{y}
\end{aligned}
\end{equation*}

Dimension of the matrix:
\begin{equation*}
\begin{aligned}
\boldsymbol{A} &\hspace{0.7cm}&& \text{dimension } n_{d}*m_{d} \\
\boldsymbol{B} &\hspace{0.7cm}&& \text{dimension } n_{d}*p_{d} \\
\boldsymbol{d} &\hspace{0.7cm}&& \text{dimension } n_{d}*1 \\
\boldsymbol{x} &\hspace{0.7cm}&& \text{dimension } m_{d}*1 \\
\boldsymbol{y} &\hspace{0.7cm}&& \text{dimension } p_{d}*1 \\
\end{aligned}
\end{equation*}

The method  ``\texttt{int Matrix::solve(double xinout[], double yinout[])}'' is used to solve the system.

\section{Methods}

\subsubsection{\texttt{int Matrix::solve(double xinout[])}}

This method is used to solve the system
\begin{equation*}
\begin{aligned}
\underset{\boldsymbol{x}}{\min} \hspace{0.2cm} ||\boldsymbol{d} - \boldsymbol{A}\boldsymbol{x} ||_{2}  \hspace{0.5cm} \text{ subject to } \hspace{0.5cm}   \boldsymbol{H}\boldsymbol{x} = \boldsymbol{f}
\end{aligned}
\end{equation*}

Solution of the system is stored in the array $x_{inout}$.

\subsubsection{\texttt{int Matrix::solve(double xinout[], double yinout[])}}

This method is used to solve the system

\begin{equation*}
\begin{aligned}
\underset{\boldsymbol{x},\boldsymbol{y}}{\min} \hspace{0.2cm} || \boldsymbol{y} ||_{2}  \hspace{0.5cm} \text{ subject to } \hspace{0.5cm}  \boldsymbol{d}  = \boldsymbol{A} \boldsymbol{x} + \boldsymbol{B} \boldsymbol{y}
\end{aligned}
\end{equation*}

 Solution of the system is stored in the array $x_{inout}$ and $y_{inout}$ .

\subsubsection{\texttt{ void Matrix::printMatrix()}}

This method is used to print to console all the matrix entries stored in the Matrix object.

\subsubsection{\texttt{static void Matrix::printMatrix(double input[], int nd, int md)}}

This method is used to print to console all the matrix entries stored in an array \texttt{input}. The dimension of the matrix is $n_{d}*m_{d}$.

\section{Operators}
	\begin{tabular}{|c|c|l|}
		\hline
		Operator & Example & Description    \\
		\hline
		+& $\boldsymbol{A} + \boldsymbol{B} $ & Adding two matrix object.  \\
		-& $\boldsymbol{A} - \boldsymbol{B}$ & Substract two matrix object.  \\
		*&   $c*\boldsymbol{A}$  &  multiply matrix $\boldsymbol{A}$ with a scalar $c$  \\
		*& $\boldsymbol{A}*c$ & multiply matrix $\boldsymbol{A}$ with a scalar $c$  \\
		*& $\boldsymbol{A}*\boldsymbol{B}$ &  multiply matrix two matrix together  \\
		$|$& $\boldsymbol{A} \hspace{0.1cm} | \hspace{0.1cm} \boldsymbol{b}$ & $\boldsymbol{b}$ is an array, compute $\boldsymbol{A}^{-1} \boldsymbol{b}$.  If  $\boldsymbol{A}$ is not a square, matrix pseudo inverse is used. \\
		$|$& $\boldsymbol{A} \hspace{0.1cm} | \hspace{0.1cm} \boldsymbol{B}$  &  $\boldsymbol{B}$ is matrix, compute $\boldsymbol{A}^{-1} \boldsymbol{B}$. If  $\boldsymbol{A}$ is not a square matrix, pseudo inverse is used. \\
		\hline
	\end{tabular}

\end{document}

%\font size
%   1.  \tiny
%   2. \scriptsize
%   3. \footnotesize
%   4. \small
%   5. \normalsize
%   6. \large
%   7. \Large
%   8. \LARGE
%   9. \huge
%  10. \Huge


\documentclass{ximera}
\input{../preamble}
\title{Exercises: Substitution and Tables}
%%%%%\author{Philip T. Gressman}

\begin{document}
\begin{abstract}
Various exercises relating to substitution and the use of integral tables.
\end{abstract}
\maketitle

Compute the indefinite integrals below. Since there are many possible answers (which differ by constant values), use the given instructions if needed to choose which possible answer to use.

\begin{exercise}%[APEX0601SUBS06]
\[ \int (12 x+14) \left(3 x^2+7 x-1\right)^5 dx = \answer{\frac{1}{3}(3 x^2+7 x-1)^6}+C \]
(Add a constant to your answer if needed so that it equals $1/3$ at $x = 0$.)
%
%
\end{exercise}

\begin{exercise}%[APEX0601SUBS11]
\[ \int \frac{e^{\sqrt{x}}}{\sqrt{x}} dx  = \answer{2 e^{\sqrt{x}}}+C \]
(Add a constant to your answer if needed so that it equals $2$ at $x = 0$.)
%
%
\end{exercise}

\begin{exercise}%[APEX0601SUBS09]
\[ \int \frac{x}{\sqrt{x+3}} dx  = \answer{\frac{2}{3} (x-6) \sqrt{x+3}}+C\]
(Add a constant to your answer if needed so that it equals $0$ at $x=6$.)
%
%
\end{exercise}

\begin{exercise}%[APEX0601SUBS33]
\[ \int \frac{\ln |x|}{x} dx  = \answer{\frac{1}{2}\ln^2 |x|}+C\]
Remember absolute value in your logarithm.
(Add a constant to your answer if needed so that it equals $0$ at $x = 1$.)
%
%
\end{exercise}

\begin{exercise}%[APEX0601SUBS57]
\[ \int \sin x \sqrt{\cos x} dx =  \answer{-\frac{2}{3} \cos ^{\frac{3}{2}}(x)}+C\]
(Add a constant to your answer if needed so that it equals $-2/3$ at $x = 0$.)
%
%
\end{exercise}

\begin{exercise}%[APEX0601SUBS63]
\[  \int \frac{9 (2 x+3)}{3 x^2+9 x+7} dx = \answer{3 \ln \left|3 x^2+9 x+7\right|}+C \]
Use absolute values as needed in logarithms. (Add a constant to your answer if needed so that it equals $3 \ln 7$ at $x = 0$.)
%
%
\end{exercise}

\begin{exercise}%[APEX0601SUBS65]
\[ \int \frac{x}{x^4+81} dx = \answer{\frac{1}{18} \arctan \left(\frac{x^2}{9}\right)}+C \]
(Add a constant to your answer if needed so that it equals $0$ at $x = 0$.)
\begin{hint}
Make a substitution $x^2 = 9 u$.
\end{hint}
%
%
\end{exercise}

\begin{exercise}%[APEX0601SUBS83]
Evaluate the definite integral \(\displaystyle \int_{-2}^{-1} (x+1)e^{x^2+2x+1}\ dx. \)
\begin{center}
\begin{prompt}
Value = \(\answer{(1-e)/2}\)
\end{prompt}
\end{center}
%
%
\end{exercise}

\section*{Sample Exam Questions}

\begin{question}%[2016.13]
Evaluate the integral $\displaystyle \int_1^3 \left( x - \sqrt{4 x^2 - 8 x + 13} \right)  dx$ using the fact that $\displaystyle \int_0^4 \sqrt{x^2 + 9} ~ dx =  \frac{20 + 9 \ln 3}{2}$.
(Hints will not be displayed until you have chosen a response.)
\begin{multiplechoice}
\choice{\(\displaystyle -\frac{4 + 9 \ln 3}{6}\)}
\choice[correct]{\(\displaystyle -\frac{4 + 9 \ln 3}{4}\)}
\choice{\(\displaystyle -\frac{4 + 9 \ln 3}{2}\)}
\choice{\(\displaystyle -4 - 9 \ln 3\)}
\choice{\(\displaystyle -8 - 18 \ln 3\)}
\choice{\(\displaystyle -12 - 27 \ln 3\)}
\end{multiplechoice}
\begin{feedback}
First complete the square:
\[ 4 x^2 - 8 x + 13 = 4 x^2 - 8 x + 4 + (13 - 4) = (2x - 2)^2 + 9. \] \begin{hint}
Next substitute $u = 2x-2$ (i.e., $x = \frac{u}{2} + 1$). Then $du = 2 dx$ and $x=1 \leftrightarrow u = 0$, $x=3 \leftrightarrow u = 4$, so
\[ \int_1^3 \left( x - \sqrt{4x^2 - 8x + 13} \right) dx = \int_0^4 \left( \frac{u}{2} + 1 - \sqrt{u^2 + 9} \right) \frac{du}{2}. \] \begin{hint}
Now use linearity of the integral to finish:
\[ \int_0^4 \frac{u}{4} ~ du + \int_0^4 \frac{1}{2} ~ du - \int_0^4 \frac{1}{2} \sqrt{u^2+9} ~ du = \left. \frac{u^2}{8} \right|_0^4 + \left. \frac{u}{2} \right|_0^4 - \frac{20 + 9 \ln 3}{4} = 4 - \frac{20+9 \ln 3}{4} = - \frac{4 + 9 \ln 3}{4}.\] \end{hint}\end{hint}
\end{feedback}
\end{question}

\end{document}


\section{Model}
\label{sec:model}

\subsection{Model components}

The core of this term project is to demonstrate the working of the human capital accumulation mechanism in light of demographic transitions. For this, I use a standard overlapping generations model with household heterogeneity by age. Household's maximize discounted lifetime utility from consumption by choosing between consumption, saving and effort invested in human capital accumulation in every model period subject to a budget constraint, borrowing constraint, time constraint and the human capital formation technology.

Human capital is produced with a technology commonly used in the literature and first introduced by \cite{Ben-Porath1967}. Human capital next period is a function of the current level of human capital, $h_t$, and the effort exerted to produce additional human capital, $e_t$,
$$ h_{t+1} = (1-\delta_h) h_t + \zeta (e_t h_t)^\psi$$
where $\delta_h$ is the depreciation rate on human capital, $\zeta$ is a scaling factor capturing average learning ability and $\psi$ is the curvature parameter of the production technology.

The model features neither idiosyncratic income risk, nor aggregate risk of any kind, i.e. the only risk households are facing is mortality risk. Households are assumed to have perfect foresight regarding survival and idiosyncratic survival risk washes out in the aggregate. Accidental bequests, i.e. asset holdings of deceased households are taxed 100\% and spent on non-productive government expenditures. The latter is a deviation from the framework in \citet{LudwigSchelkleVogel2012}, where accidental bequests are redistributed to households via transfers. While altering the budget constraint of households, the  assumption taken for this term paper simplifies computation. In the original framework, income from transfers affects the consumption/ savings/ investment decision and, therefore, the level of accidental bequests. In effect, transfers act as an additional variable (next to aggregate capital, aggregate labor and pension benefits), that needs to be first guessed as input and then verified as results of aggregation. Thus, (wasteful) spending of tax revenue from accidental bequests removes one equlibrium condition.

In contrast to the original paper by \cite{LudwigSchelkleVogel2012}, pension benefits are in this paper not linked to income histories. In other words, every retired agent qualifies for the same level of pension benefits. While the level of pension benefits remains an aggregate variable to be solved for by guess and verify, this assumption simplifies computation, as it is no longer necessary to keep track of a households earnings history to calculate pension benefit claims, effectively eliminating a state variable from the household problem. In the specific model used here, the difference is not too material, as in the absence of idiosyncratic risk other than mortality risk, all households of a given age are identical in their earnings history. However, this simplifying assumption becomes truly relevent when introducing idiosyncratic income risk to the model. Pension benefits are thus given by
$$ b_j = \begin{cases}
				0 \;\text{if}\; j<J^r \\
                \bar{b} \;\text{if}\; J^r \leq j \leq J \\
			\end{cases} $$

In line with \citet{LudwigSchelkleVogel2012}, households derive utility from consumption and do not have a bequest motive. Contrasting the original framework, I assume that households do not derive utility from leisure. This, in turn, means that households will use their entire time endowment to either earn income in the labor market or invest in accumulating human capital. While abstracting from the labor-leisure choice, one key margin of adjustment for the household in light of a changing environment is disabled. The remaining trade-off in the labor supply decision solely consists of investing time in earning labor income versus investing time in accumulating / maintaining human capital for higher future productivity. On one hand, this substantially simplifies the computational problem and on the other hand, the labor-leisure trade-off is removed from the analysis, thus making clearer the role of endogenous human capital accumulation.

Households offer capital and labor at competitive markets and earn interest rate $\bar{r}$ on physical capital and wage rate $\bar{w}$ on effective labor supply $(1-e_j) z_j$, where $z_j$ is the fixed and exogenous age-dependent labor efficiency multiplier. Moreover, households face a zero borrowing limit and the minimum level of human capital is assumed to be zero (the latter will not be binding during working age due to the assumption on the human capital formation technology). Finally, households take factor prices for capital and labor, as well as the level of pension benefits as given

The model is closed by a continuum of firms that rent out capital and labor on perfectly competitive input markets and produce the final output good using an identical Cobb-Douglas production function.

The household optimization problem in the stationary economy is given by
\begin{align*}
    \max_{\{c_j, e_j, a_{j+1}, h_{j+1}\}_{j=0}^J} & \sum_{j=0}^J \beta^j \pi_{j} u(c_j) \\
    \text{s.t.} \;  & c_j + a_{j+1} = (1+\bar{r}) a_j + (1-\tau) (1-e_j) z_j h_j + b_j \quad\text{for}\; j=0,\dots,J \\
                    & h_{j+1} = (1-\delta_h) h_j + \zeta (e_j h_j)^\psi \quad\text{for}\; j=0,\dots,J \\
                    & a_{j+1} \leq 0 \quad \quad\text{for}\; j=0,\dots,J \\
                    & h_{j+1} \leq 0 \quad \quad\text{for}\; j=0,\dots,J \\
                    & e_j \in [0, 1] \quad \quad\text{for}\; j=0,\dots,J \\
                    & a_0, h_0 \;\text{given}
\end{align*}

The household problem can be rewritten recursively, which yields an equivalent optimal programming problem given by a set of value functions $\{V_j(a, h)\}_{j=0}^J$ defined by
\begin{align*}
    V_j (a, h) = \max_{ \{c, e, a', h' \} } & [ u(c) + \beta \pi_{j, j+1} V_{j+1} (a', h') ]  \\
    \text{s.t.} \;  & c + a' = (1+\bar{r}) a + (1-\tau) (1-e) z_j h + b_j \\
                    & h' = (1-\delta_h) h + \zeta (e h)^\psi \\
                    & a' \leq 0 \\
                    & h' \leq 0 \\
                    & e \in [0, 1]
\end{align*}
for all $j=0, \dots, J$. Note that two of the four choice variables $\{c, e, a', h' \}$ can be substituted out using the budget constraint and the human capital accumulation equation.

\subsection{Equilibrium definition}

Using the reduced recursive characterization of the household problem, a stationary equilibrium in this economy is described by
\begin{enumerate}
    \item Sets of policy functions for physical and human capital by household age $\{a'_{j}(a, h), h'_{j}(a, h) \}_{j=0}^{J}$
    \item Factor prices $\bar{r}, \bar{w}$ and pension benefit level $\bar{b}$
\end{enumerate}
such that the following conditions hold:
\begin{enumerate}
    \item The policy functions solve the functional equations of the household problem for all ages $j=0, \dots, J$
    \item Firms behave optimally in every period
    \item Government budget balances
        $$ \tau \bar{w} \sum_{j=0}^{J^r-1} (1 - e_j) h_{j} N_j = \rho \sum_{j=J^r}^J N_{j} b_{j} $$
    \item Factor markets clear
        $$ K^D = \sum_{j=0}^{J} a_j \quad\text{and}\quad L^D = \sum_{j=0}^{J^r-1} (1 - e_j) z_j $$
\end{enumerate}

To assess the transition from one stationary equilibrium to another, the time dimension is introduced to the model. Population dynamics are no longer assumed to be time invariant and, as a consequence, neither will be factor prices and household policy functions. For further analysis, I first provide an equilibrium definition.

%An equilibrium in the economy with transitional dynamics is given by:
%\begin{enumerate}
%    \item Sequences of sets of policy functions for physical and human capital by household age and time $\{ \{c_{j, t}, a_{j+1, t+1}, h_{j+1, t+1} \}_{j=0}^{J} \}_{t=0}^{\infty}$
%    \item Sequences of aggregate variables $\{L_t, K_{t+1}, Y_t \}_{t=0}^{\infty}$
%    \item Sequences of factor prices $\{r_t, w_t \}_{t=0}^{\infty}$ and pension benefit levels $\{b_t\}_{t=0}^{\infty}$
%\end{enumerate}
%such that the following conditions hold:
%\begin{enumerate}
%    \item The policy functions solve the functional equations of the household problem for all ages $j=0, \dots, J$ and all times $t=0, \dots$
%    \item Firms behave optimally in every period
%    \item Government budget balances in every period
%    \item Factor markets (labor, capital, and output) clear in every period
%    \item The initial state of the economy constitutes a stationary equilibrium
%    \item The system converges to a stationary equilibrium, i.e. the limits w.r.t. time $t$ of factor prices and policy functions exist and constitute a stationary equilibrium
%        constitute
%\end{enumerate}
%
%The economy with transitional dynamics described above does not have a closed form solution, which means that numerical methods are required to solve it. Thus, for computational reasons, it is not feasible to %solve for the exact equilibrium, which motivates the following definition.

An equilibrium in the economy with transitional dynamics is given by some finite time $T$ and:
\begin{enumerate}
    \item Sequences of sets of policy functions for physical and human capital by household age and time $\{ \{c_{j, t}, a_{j+1, t+1}, h_{j+1, t+1} \}_{j=0}^{J} \}_{t=0}^{T}$
    \item Sequences of aggregate variables $\{L_t, K_{t+1}, Y_t \}_{t=0}^{T}$
    \item Sequences of factor prices $\{r_t, w_t \}_{t=0}^{T}$ and pension benefit levels $\{b_t\}_{t=0}^{T}$
\end{enumerate}
such that the following conditions hold:
\begin{enumerate}
    \item The policy functions solve the functional equations of the household problem for all ages $j=0, \dots, J$ and all times $t=0, \dots$
    \item Firms behave optimally in every period
    \item Government budget balances in every period
    \item Factor markets (labor, capital, and output) clear in every period
    \item The initial state (at $t=0$) and the final state (at $t=T$) of the economy approximately constitute a stationary equilibrium
\end{enumerate}
where the latter statement is interpreted in a numerical manner, i.e. up to a pre-specified tolerance level for the market clearing conditions.


\section{Stochastic Particle Tracing using Particle Filtering}

As presented above, the particle tracing result of a given seed position $x_0$ for the distribution-based data $\mathcal{H}$ can be represented as a pdf $p({v_{0:n}}|{\lambda_{0:n}})$ over the trace domain $\Omega_{x_0}$, which is given by equation (8). However, $p({v_{0:n}}|{\lambda_{0:n}})$ is high-dimensional, non-standard, and only known up to a proportionality constant, which makes it infeasible to be evaluated in closed-form. Therefore, a Monte Carlo based method needs to be used to approximate the target distribution. In this case, particle filtering as one of Sequential Monte Carlo methods is suitable to approximate the target distribution in iterations.

\subsection{Posterior Approximation with Properly Weighted Traces}

\begin{figure}[htb]
  \centering
  \includegraphics[width=3in]{../figures/importance_sampling.eps}
  \caption{Non-weighted and weighted samples representing the probability density function $f(x)$. Fewer samples is needed to represent the target pdf by using weighted samples.}
  \label{importance_sampling}
\end{figure}

In order to work with the complicated target posterior density $p({v_{0:n}}|{\lambda_{0:n}})$, a more convenient representation is needed to approximate it. By applying the particle filtering method, the target posterior density can be represented by a set of properly weighted traces, which are denoted as $\{ v_{0:n}^i,w_n^i\} _{i = 1}^{{N_s}}$, where $N_s$ is the number of samples and $ w_n^i,i = 0,...,{N_s} $ are the associated weights. Therefore, we can estimate some statistical properties, for example the mean, variance, and maximum likelihood of the target posterior density from the sample traces. Compared to the non-weighted samples used by the conventional Monte Carlo method, weighted samples can represent the target distribution more efficiently. A comparison between weighted and non-weighted samples is illustrated in Figure~\ref{importance_sampling}. A common way of drawing properly weighted samples from the target distribution is to use importance sampling, which samples from a trivial distribution $q({v_{0:n}}|{\lambda_{0:n}})$ called the importance function and assign a weight to each of the random samples according to
\begin{equation}
  w_n = \frac{{p({v_{0:n}}|{\lambda_{0:n}})}}{{q({v_{0:n}}|{\lambda_{0:n}})}}
\end{equation}

\subsection{Sequential Build-Up}

Since the dimension of the target posterior density increases as the number of propagation steps in the particle tracing grows, which makes it difficult to directly obtain the final sample traces with a given step number $n$. By applying particle filtering we smoothly approach the final probability density by building up the sample traces sequentially. To do so, we put ${N_s}$ particles at the seed position $x_0$ and update them iteratively. At each iteration $t$, we have a set of properly weighted samples $\{ v_{0:t-1}^i,w_{t-1}^i\} _{i = 1}^{{N_s}}$ which represents the probability density $p({v_{0:t-1}}|{\lambda_{0:t-1}})$, then each sample can be updated to step $t$ according to the recursive form of the posterior density $p({v_{0:t}}|{\lambda_{0:t}})$, which can be written as:
\begin{equation}
  p({v_{0:t}}|{\lambda_{0:t}}) = p({v_{0:t - 1}}|{\lambda_{0:t - 1}})p(v_t|v_{t-1},\lambda_t)
\end{equation}
where
\begin{equation}
  p(v_t|v_{t-1},\lambda_t) = \frac{{p({\lambda_t}|{v_t})p({v_t}|{v_{t - 1}})}}{{p({\lambda_t}|{\lambda_{0:t - 1}})}}
\end{equation}
Since directly sampling vectors from $p(v_t|v_{t-1},\lambda_t)$ is difficult, we make use of an alternative density function $q({v_t}|{v_{t - 1}},{\lambda_t})$ which is so-call the importance function presented above. From the importance function, vector directions $v_t$ can be obtained, therefore the weights can be updated according to equation (13). Theoretically, the importance function can be any distribution for the vector direction $v_t$. However, a bad choice of the importance function may produce vector directions which have low probability in the target distribution. Hence, the choice of the importance function will influence the performance of the particle filtering algorithm significantly, which will be detailed in the next section. In order to iteratively update the weights on-line, the importance function is chosen to factorize such that:
\begin{equation}
  q({v_{0:t}}|{\lambda_{0:t}}) = q({v_{0:t - 1}}|{\lambda_{0:t - 1}})q({v_t}|{v_{t - 1}},{\lambda_t})
\end{equation}
After getting random vector directions according to the importance density $q({v_t}|{v_{t - 1}},{\lambda_t})$, the weights $\{w_t^i\} _{i = 1}^{{N_s}}$ of traces is updated based on $\{w_{t-1}^i\} _{i = 1}^{{N_s}}$. By substituting (14), (16) into (13), we can get the equation to update the weight:
\begin{equation}
  w_t^i = w_{t - 1}^i\frac{{p({\lambda_t}|v_t^i)p(v_t^i|v_{t - 1}^i)}}{{q(v_t^i|v_{t - 1}^i,{\lambda_t})p({\lambda_t}|{\lambda_{0:t - 1}})}}
\end{equation}
Since the weights can be normalized by:
\begin{equation}
  w_t^i = \frac{{w_t^i}}{{\sum\limits_{j = 1}^{{N_s}} {w_t^j} }}
\end{equation}
the normalizing constant ${p({\lambda_t}|{\lambda_{0:t - 1}})}$ can be ignored. Therefore, the weight update equation can be simplified as
\begin{equation}
  w_t^i \propto w_{t - 1}^i\frac{{p({\lambda_t}|v_t^i)p(v_t^i|v_{t - 1}^i)}}{{q(v_t^i|v_{t - 1}^i,{\lambda_t})}}
\end{equation}

The results provided by the particle filtering algorithm for a given seed position is a set of weighted traces which approximate the posterior probability distribution of the possible traces. We directly render them to see the uncertainty of the particle tracing process. With respect to the goal of visualizing the global phenomenon of the vector fields, streamlines originating from different seed positions need to be visualized together, which make it inappropriate to show all the sample traces for a given seed position all the time. To solve this problem, we extract the trace with highest probability based on the posterior density function and visualize all the most likely traces together. The most likely trace is chosen by maximum a posteriori probability (MAP) estimation, which is the trace with the maximal importance weight.

\subsection{Resampling and Choice of Importance Function}

A common issue with the particle filtering algorithm is the degeneracy problem, which is that after several iterations, some samples will have extremely small weights. This means that the contribution of those samples to the posterior distribution is negligible. A reasonable measurement of degeneracy is the effective sample size $N_{eff}$ introduced in~\cite{Liu98sequentialmonte}, which can be estimated by
\begin{equation}
  {N_{eff}} = \frac{1}{{\sum\limits_{i = 1}^{{N_s}} {{{(w_t^i)}^2}} }}
\end{equation}
where $w_t^i$ is the normalized weight. The value of $N_{eff}$ is between $1$ and $N_s$ and the smaller $N_{eff}$ is, the weights degenerate more. The degeneracy problem is an undesirable effect in particle filters. The brute force approach to reduce this effect is to use a very large sample size, which is often impractical. Hence, we focus on two other methods.

\textbf{Resampling} The first method is to perform resampling~\cite{doucet2001sequential, gordon:107} whenever the degeneracy becomes significant (i.e., when $N_{eff}$ is smaller than some hard threshold $N_t$). The key idea of resampling is to remove the traces that have small weights and duplicate the traces which have large weights. To do so, we generate a new set of weighted traces by resampling (with repalcement) $N_s$ times from the given set $\{ v_{0:t}^i,w_t^i\} _{i = 1}^{{N_s}}$, making use of the weights $\{w_t^i\} _{i = 1}^{{N_s}}$ as the probabilities for a sample to be resampled, then reset the weights to $w_t^i = \frac{1}{{{N_s}}}$.

\textbf{Good Choice of Importance Function} The second method is choosing a good importance density $q({v_t}|{v_{t - 1}},{\lambda_t})$. The optimal importance density function that minimizes the variance of the weights $w_t^i$ conditioned on $v_{t-1}$ and $\lambda_t$, pointed out by Doucet et al. in~\cite{Doucet00onsequential}, is $p(v_t|v_{t-1}, \lambda_t)$. However, this optimal importance density requires the evaluation of the integral over the new state, which makes it difficult to sample efficiently from. Hence, we focus on designing a suboptimal importance density which can be easily sampled from and represents $p(v_t|v_{t-1}, \lambda_t)$ well. A usual approach is to use the same distribution as the prior density as the importance function, which is called bootstrap filter or condensation algorithm. However, such importance function may not be always effective, since no observation information is used. As a result, the resulting particles are often outliers of the posterior distribution. Therefore, the observation density is used to determine the $v_t$. Since it is expensive to evaluate the observation density function and there is no need to use the actual observation density as the importance density for the particle filtering method, we choose the observed distribution $\lambda_t$ as the importance density function.

\subsection{Algorithm Summary}

Our stochastic particle tracing algorithm is summarized in Algorithm~\ref{algo:pf}.

\begin{algorithm}[h]
\caption{Streamline Estimation with Particle Filtering} \label{algo:pf}
\begin{algorithmic} [1]

\State Let $x_0$ be a given seed position, $n$ be the number of steps, $N_s$ be the number of samples.
\For {$t=0:n$}
\For {$i=1:N_s$}
\State Sample $v_t^i$ at position $x_t$ according to ${\lambda_t}$
\State Compute weight $w_t^i$ according to (11), (12), and (19)
\EndFor
\State Normalize the weights $\{w_t^i\}_{i=1}^{N_s}$ according to (18)
\State Calculate $N_{eff}$ using (20)
\If {$N_{eff} < N_t$}
\State Resample $\{ v_{0:t}^i,w_t^i\} _{i = 1}^{{N_s}}$ to obtain $N_s$ equally-weighted particles $\{ \hat{v}_{0:t}^i,\frac{1}{N_s}\} _{i = 1}^{{N_s}}$
\EndIf
\State Propagate $x_t$ to $x_{t+1}$ according to (1)
\EndFor

\end{algorithmic}
\end{algorithm}


\subsection{Banach spaces}\label{subsec:banach_spaces}

\begin{definition}\label{def:banach_space}
  A \term{Banach space} is a \hyperref[def:norm]{normed} \hyperref[def:vector_space]{vector space} which is also a \hyperref[def:complete_metric_space]{complete metric spaces} with the metric induced by the \hyperref[def:norm_induced_metric]{norm}.
\end{definition}

\begin{definition}\label{def:topological_duality_pairing}
  Let \( M \) and \( N \) be left \( R \)-modules. A \term{duality pairing} \( \inprod \cdot \cdot: M \times N \to R \) is a \hyperref[def:nondegenerate_bilinear_form]{nondegenerate} bilinear form.

  The \term{canonical duality pairing} of a vector space \( V \) over \( F \) is
  \begin{balign*}
     & \inprod \cdot \cdot: V^* \times V \to F \\
     & \inprod {x^*} x \mapsto x^*(x).
  \end{balign*}
\end{definition}

\begin{example}\label{ex:noncomplete_normed_space}\mcite{MathCounterExamples:noncomplete_normed_space}
  Consider the polynomial \hyperref[def:algebra_of_polynomials]{algebra} \( \BbbR[x] \) as a vector space with the supremum norm. We will show that it is not complete. Define the sequence
  \begin{equation*}
    p_n(x) \coloneqq \sum_{k=0}^n \frac{x^k} {2^k}, n = 1, 2, \ldots
  \end{equation*}

  Then the limit of the sequence in \( C([0, 1]) \) is the power series
  \begin{equation*}
    \lim_{n \to \infty} p_n(x)
    =
    \sum_{k=0}^n \frac{x^k} {2^k}
    =
    \frac 2 {2 - x}.
  \end{equation*}

  Since \( \BbbR[x] \) is a subspace of \( C([0, 1]) \), we conclude that \( \BbbR[x] \) has fundamental sequence, but we just demonstrated that its limit is not in \( \BbbR[x] \).
\end{example}

\begin{definition}\label{def:dual_norm}
  Fix two nonempty Banach spaces \( (X, \norm{\cdot}_X) \) and \( (Y, \norm{\cdot}_Y) \). We define the \term{operator norm} \( \norm{\cdot}_{\hom(X, Y)} \) on \( \hom(X, Y) \) equivalently as
  \begin{thmenum}
    \thmitem{def:dual_norm/sup_unit_sphere}
    \begin{equation*}
      \norm{L}_{\hom(X, Y)} \coloneqq \sup_{\norm{x}_X = 1} \norm{Lx}_Y.
    \end{equation*}

    \thmitem{def:dual_norm/sup_unit_ball}
    \begin{equation*}
      \norm{L}_{\hom(X, Y)} \coloneqq \sup_{\norm{x}_X < 1} \norm{Lx}_Y.
    \end{equation*}

    \thmitem{def:dual_norm/sup_nonzero}
    \begin{equation*}
      \norm{L}_{\hom(X, Y)} \coloneqq \sup_{x \neq 0_X} \frac {\norm{Lx}_Y} {\norm{x}_X}.
    \end{equation*}

    \thmitem{def:dual_norm/inf}
    \begin{equation*}
      \norm{L}_{\hom(X, Y)} \coloneqq \inf \left\{ c \geq 0 \colon \norm{Lx}_Y \leq c \norm{x}_X \right\}.
    \end{equation*}
  \end{thmenum}

  In particular, this induces a norm on \( X^* \).
\end{definition}

\begin{definition}\label{def:banach_space_support_function}\mcite[exmpl. 3.2(a)]{Phelps1993})
  Let \( X \) be a Banach space.

  We define the \term{support function \( \sigma_{A^*} \) for the set of functionals \( A^* \subseteq X^* \)} by
  \begin{balign*}
     & \sigma_{A^*}: X \to \BbbR \cup \{ \infty \}                             \\
     & \sigma_{A^*}(x) \coloneqq \sup \{ \inprod {x^*} x \colon x^* \in A^* \}
  \end{balign*}

  and the \term{weak* support function \( \sigma^*_A \) for the set of points \( A \subseteq X \)} by
  \begin{balign*}
     & \sigma^*_A: X^* \to \BbbR \cup \{ \infty \}                          \\
     & \sigma^*_A(x^*) \coloneqq \sup \{ \inprod {x^*} x \colon x \in A \}.
  \end{balign*}
\end{definition}

\begin{definition}\label{def:banach_space_slice}\mcite[def. 2.17]{Phelps1993}
  Given a linear functional \( x^* \), a nonempty subset \( A \) of \( X \) and a \term{diameter} \( \alpha > 0 \), the value \( S(x^*, A, \alpha) \) is called a \term{slice} of \( A \), where
  \begin{balign*}
     & S: X^* \times \pow(X) \times \BbbR_{>0} \mapsto \pow(A)                                      \\
     & S(x^*, A, \alpha) \coloneqq \{ x \in A \colon \inprod {x^*} x > \sigma_A^*(x^*) - \alpha \}.
  \end{balign*}

  We define a weak* slice of \( A^* \subseteq X^* \) as \( S^*(x, A^*, \alpha) \), where
  \begin{balign*}
     & S^*: X \times \pow(X) \times \BbbR_{>0} \mapsto \pow(A)                                            \\
     & S^*(x, A^*, \alpha) \coloneqq \{ x^* \in A^* \colon \inprod {x^*} x > \sigma_{A^*}(x) - \alpha \}.
  \end{balign*}

  If we need to make the underlying space explicit, we will use \( S_X(x^*, A, \alpha) \) and \( S_X^*(x, A^*, \alpha) \).
\end{definition}

\begin{proposition}
  If \( \{ a_k \}_{k=1}^\infty \) and \( \{ b_k \}_{k=1}^\infty \) are sequences a in a \hyperref[def:banach_space]{Banach} \hyperref[def:algebra_over_ring]{algebra} \( X \), that converge to \( a \) and \( b \), correspondingly, then \( a_k b_k \to a b \).
\end{proposition}
\begin{proof}
  Let \( \delta > 0 \) and let \( k_0 \) be an index such that for \( k \geq k_0 \) we have both \( \norm{a - a_k} < \delta \) and \( \norm{b - b_k} < \delta \). Then
  \begin{balign*}
    ab - a_k b_k
     & =
    (ab - a b_k) + (a b_k - a_k b) + (a_k b - a_k b_k)
    =    \\ &=
    a (b - b_k) + (a b_k - ab + ab - a_k b) + (-a_k)(b_k - b)
    =    \\ &=
    a (b - b_k) + a \underbrace{(b_k - b)} + (a - a_k) b + (-a_k)\underbrace{(b_k - b)}
    =    \\ &=
    a \underbrace{(b - b_k)}_{\in B(0, \delta)} + \underbrace{(a - a_k)}_{\in B(0, \delta)} \underbrace{(b_k - b)}_{\in B(0, \delta)} + \underbrace{(a - a_k)}_{\in B(0, \delta)} b.
  \end{balign*}

  Therefore, \( \norm{ab - a_k b_k} < \delta^2 + \norm{a + b} \delta \). If we require \( \delta \) to be strictly less than \( 1 \), we obtain \( \delta^2 < \delta \) and \( \norm{ab - a_k b_k} < (1 + \norm{a + b}) \delta \).

  Given an arbitrary \( \varepsilon > 0 \), we can choose \( \delta = \tfrac {\min \{\varepsilon, 1 \}} {1 + \norm{a + b}} \) in order to have \( \norm{ab - a_k b_k} < \varepsilon \) for some large enough \( k \).

  Therefore, \( a_k b_k \to a b \).
\end{proof}


\subsection{SMT-LIB}

\begin{frame}[fragile]
  \frametitle{SMT-LIB (v2) \url{http://www.smtlib.org}}

  The SMT-LIB initiative
  \begin{itemize}
    \item defines a standard input language for \smtsolvers
    \item defines theories and logics in which \formulae can be written
    \item collects benchmarks
  \end{itemize}

  \vfill
  \pause
  The SMT-LIB language allows to write \formulae in a lisp-like format. E.g.:
  \begin{verbatim}
    (< (+ x y) 0)
    (= (f x y) (g z))
  \end{verbatim}
  stand for $x + y < 0$ and $f(x,y) = g(z)$ respectively

  \vfill
  \pause
  An SMT-LIB file looks more similar to a {\bf set of commands} for
  an \smtsolver, rather then a logic formula

\end{frame}

\begin{frame}[fragile]
  \frametitle{SMT-LIB Theories}

  \scriptsize

  An SMT-LIB theory consists of some {\bf sorts}, (e.g., \SInt)
  and of some functions (e.g., $-,+$).
  Predicates are also considered functions, with codomain in \SBoo
  (e.g., $<,\leq$). \pause For instance

  \vfill
  \begin{tabular}{cc}
  \begin{minipage}{.4\textwidth}
  \tiny
  \begin{verbatim}
     (theory Ints

      :sorts ((Int 0))

      :funs ((NUMERAL Int)
             (- Int Int)
             (- Int Int Int :left-assoc)
             (+ Int Int Int :left-assoc)
             (* Int Int Int :left-assoc)
             (div Int Int Int :left-assoc)
             (mod Int Int Int)
             (abs Int Int)
             (<= Int Int Bool :chainable)
             (<  Int Int Bool :chainable)
             (>= Int Int Bool :chainable)
             (>  Int Int Bool :chainable)
            )

       [...]
     )
   \end{verbatim}
   \end{minipage}
   &
   \begin{minipage}{.4\textwidth}
     \tiny
      \begin{verbatim}
      (theory Core

       :sorts ((Bool 0))

       :funs ((true Bool)
              (false Bool)
              (not Bool Bool)
              (=> Bool Bool Bool :right-assoc)
              (and Bool Bool Bool :left-assoc)
              (or Bool Bool Bool :left-assoc)
              (xor Bool Bool Bool :left-assoc)
              (par (A) (= A A Bool :chainable))
              (par (A) (distinct A A Bool :pairwise))
              (par (A) (ite Bool A A A))
             )

       [...]
      )
      \end{verbatim}
      \end{minipage} \\
    \multicolumn{2}{c}{\tiny These definitions can be found at \url{www.smtlib.org}}
  \end{tabular}
  \vfill
  \pause

  The sorts and the function symbols declared in a theory are always {\bf interpreted}.
  This means that a to specify a model for a formula $\varphi$, we just need to specify
  the assignment of the variables to the concrete values in the sorts.

\end{frame}

\begin{frame}[fragile]
  \frametitle{SMT-LIB Logics}

  \scriptsize
  The difference between ``logic'' and ``theory'' might look very subtle.
  An SMT-LIB logic includes a theory definition, plus it describes some
  restrictions on how \formulae can be built. \pause

  \vfill
  \hspace{-25pt}
  \begin{tabular}{ccc}
  \begin{minipage}{.4\textwidth}
    \tiny
    \begin{verbatim}
    (logic QF_LIA

     :theories (Ints)

     :language
     "Closed quantifier-free formulas built
     over an arbitrary expansion of the
     Ints signature with free constant symbols,
     but whose terms of sort Int are all linear,
     that is, have no occurrences of the function
     symbols *, /, div, mod, and abs, except as
     specified the :extensions attribute.
     "

     :extensions
     "Terms with _concrete_ coefficients are also
     allowed, that is, terms of the form c, (* c x),
     or (* x c)  where x is a free constant and c
     is a term of the form n or (- n) for some numeral n.
     "
    )
    \end{verbatim}
  \end{minipage}
  & ~~~~ & \pause
  \begin{minipage}{.4\textwidth}
    \tiny
    \begin{verbatim}
    (logic QF_IDL

     :theories (Ints)

     :language
     "Closed quantifier-free formulas with
     atoms of the form:
     - q
     - (op (- x y) n),
     - (op (- x y) (- n)), or
     - (op x y)
     where
     - q is a variable or free constant symbol of sort Bool,
     - op is <, <=, >, >=, =, or distinct,
     - x, y are free constant symbols of sort Int,
     - n is a numeral.
     "
    )
    \end{verbatim}
  \end{minipage}
  \end{tabular}

  \vfill
  \pause

  In the following we will not be so strict, and we will not make any distinction
  between ``theories'' and ``logics'', calling both ``theories''. \pause
  For instance when we will say that we reason modulo the theory \Lia we mean
  that we are working with {\tt QF\_LIA} \formulae

\end{frame}

\begin{frame}[fragile]
  \frametitle{Writing an SMT-LIB file}
  \scriptsize

  The logic can be specified with the command
  \begin{verbatim}
    (set-logic QF_LIA)
  \end{verbatim}
  \vfill
  \pause
  Variables are declared with
  \begin{verbatim}
    (declare-fun x ( ) Int)
  \end{verbatim}
  \vfill
  \pause
  A formula is specified with
  \begin{verbatim}
    (assert (<= (+ x y) 0))
  \end{verbatim}
  \vfill
  \pause
  Asks the tool to compute satisfiability of assertions
  \begin{verbatim}
    (check-sat)
  \end{verbatim}
  \vfill
  \pause
  Asks the tool to return a model (in case of sat result)
  \begin{verbatim}
    (set-option :produce-models true)
    ...
    (get-value (x y))
  \end{verbatim}
  \vfill
  \pause
  Disable annoying printouts
  \begin{verbatim}
    (set-option :print-success false)
  \end{verbatim}

\end{frame}

\begin{frame}[fragile]
  \frametitle{Example}

  \begin{verbatim}
    (set-logic QF_LIA)
    (declare-fun x ( ) Int)
    (declare-fun y ( ) Int)
    (declare-fun a ( ) Bool)
    (assert (<= (+ x y) 0))
    (assert (= x 0))
    (assert (or (not a) (= x 1) (>= y 0)))
    (assert (not (= (+ y 1) 0)))
    (check-sat)
    (exit)
  \end{verbatim}

  which stands for the \Lia formula
  $$
  (x + y \leq 0) \wedge (x = 0) \wedge ((\neg a \vee (x = 1) \vee (y \geq 0)) \wedge \neg(y + 1 = 0)
  $$

\end{frame}


\section{Multi-armed Bandits}

\subsection{Exercise 1}
$1 - \epsilon = \frac{1}{2}$.

\subsection{Exercise 2}
Exploration definitely happened at $t = 2$, since we switched from action 1 to action 2 despite
$Q_2(1) = 1 > Q_2(2) = 0$. Similarly, exploration also definitely happened at $t = 5$, since at that
point action 2 would have been the greedy choice. Depending on how ties are broken during greedy selection,
it is possible that exploration also happened at  $t = 3$, since at that point actions 1 and 2 would have
been tied.

\subsection{Exercise 3}
Intuitively, it seems clear that the $\epsilon = 0.01$ strategy will perform the best both in terms of
cumulative reward and probability of selecting the best action over the long run, as it will eventually
find the optimal action and choose it 99\% of the time.

To put this more quantitatively, we can consider the performance of each of the 3 methods after $N$ trials,
such that $N$ is large enough for the two $\epsilon > 0$ approaches to have found the optimal action with a
high degree of certainty (we can choose $N$ for any degree of certainty we like, courtesy of the law of
large numbers). After $N$ trials, the probability that the $\epsilon = 0.1$ strategy selects the best action is
0.9, whereas for $\epsilon = 0.01$ the probability is 0.99. For greedy, the probability is either 1 or 0 after
$N$ trials, since there is no guarantee that it found the optimal action. Since it is unlikely (\textbf{TODO:}
make this more precise) that greedy converged to the optimal action, the expected difference in cumulative
reward after $N$ trials will quickly grow to favor the $\epsilon = 0.01$ strategy.


\chapter{Background}\label{ch:background}
In this chapter we provide the background needed for different concepts that are discussed in this thesis. In section \ref{sec:cnn} we introduce convolutional neural networks and how they are trained. Moreover we define the necessary terminology and provide the mathematical background required to navigate this thesis. Section \ref{sec:synth} provides the terminology that we use throughout this thesis to describe small parts, synthetic images and real images.

\section{Convolutional Neural Networks}\label{sec:cnn}
A \textbf{neural network} is a massively parallel distributed processor made up of single processing units, which has a natural propensity for storing experiential knowledge and making it available for use \cite{haykin1994neural}. Convolutional neural networks are a special type of neural network. They are typically used to solve computer vision problems like image classification and object recognition.

Regular neural networks consist of an \textbf{input layer}, \textbf{hidden layers} and an \textbf{output layer}. Every layer is made up of a set of \textbf{neurons}, where each neuron is fully connected to all neurons in the layer before. A neuron which has an input $x$ of size $n$ calculates the weighted sum $z$ as follows: \[ z = \sum_{i=1}^n w_ix_i + b_i \] where $w$ is called \textbf{weight} and $b$ is called \textbf{bias}. The weights and biases of all the neurons in a network are called the network \textbf{parameters}. Each neuron calculates its output $y$ by applying a differentiable non-linear function $\varphi(\cdot)$, called the \textbf{activation function}, to the weighted sum of its input signals $z$. \[y = \varphi(z)\] A neural networks relays the output of its neurons through a series of hidden layers. Finally, the last fully-connected layer, called the output layer, computes the predictions of the neural network. Figure \ref{fig:NN} shows an example structure of a regular neural network.

\begin{figure}[H]
\centering
\includegraphics[width=0.75\textwidth]{NN}
\caption[A neural network with an input layer, two hidden layers and an output layer. Each neuron is fully connected to all the neurons in the preceding layer.]{A neural network with an input layer, two hidden layers and an output layer. Each neuron is fully connected to all the neurons in the preceding layer.\footnotemark}
\label{fig:NN}
\end{figure}
\footnotetext{Image: http://cs231n.github.io/convolutional-networks/}

Convolutional neural networks (CNNs) are structured differently. CNNs have 3 types of layers: \textbf{convolution layers}, \textbf{pooling layers} and \textbf{fully connected} layers. Convolution layers slide a weighted matrix (called \textbf{filter} or \textbf{kernel}) over the previous layer, and calculate the sum of products. The size of the step that the filter takes while sliding is called a \textbf{stride}. Figure \ref{fig:CONV} shows how a convolution layer applies a 3x3 filter to the previous layer. A non-linear activation function is applied to the output of the convolution operation to create a \textbf{feature map}. It is common to add a pooling layer after the convolution layer for subsampling, which reduces the convolution layer dimensionality. A frequently used type of pooling is \textbf{max pooling} \cite{weng1992cresceptron}. Max pooling divides the input layer into sections and computes the maximum activation of each section. Figure \ref{fig:MP} shows how a max pooling layer applies a 2x2 filter to the preceding layer. Fully connected layers are similar to layers in a regular neural network. Each neuron in a fully connected layer is connected to all neurons in the previous layer.

Figure \ref{fig:lenet5} shows the LeNet-5 convolutional neural network, which was designed by LeCun et al. \cite{lecun1998gradient} for handwritten digit recognition. LeNet-5 is an example of a CNN \textbf{architecture}. A CNN architecture is a set up of different types of layers which are combined to predict the class of the input image.

\begin{figure}[H]
\centering
\includegraphics[width=\textwidth]{CONV}
\caption[A 3x3 filter is applied to the source pixel. Each weight in the filter is multiplied by the source's neighboring pixel in the corresponding position. The sum of products is used to calculate the destination pixel's value in the output feature map.]{A 3x3 filter is applied to the source pixel. Each weight in the filter is multiplied by the source's neighboring pixel in the corresponding position. The sum of products is used to calculate the destination pixel's value in the output feature map.\footnotemark}
\label{fig:CONV}
\end{figure}
\footnotetext{Image: https://towardsdatascience.com/applied-deep-learning-part-4-convolutional-neural-networks-584bc134c1e2}

\begin{figure}[H]
\centering
\includegraphics[width=0.7\textwidth]{MP}
\caption[A max-pooling filter of size 2x2 and stride 2 is applied to the source feature map. The maximum value of each filter is used in the destination feature map.]{A max-pooling filter of size 2x2 and stride 2 is applied to the source feature map. The maximum value of each filter is used in the destination feature map.\footnotemark}
\label{fig:MP}
\end{figure}
\footnotetext{Image: http://cs231n.github.io/convolutional-networks/}

\begin{figure}[H]
\centering
\makebox[\textwidth][c]{\includegraphics[width=1.3\textwidth]{lenet5}}
\caption[Architecture of the LeNet-5 convolutional neural network.]{Architecture of the LeNet-5 convolutional neural network.\cite{lecun1989backpropagation}}
\label{fig:lenet5}
\end{figure}

The LeNet-5 network uses 3x3 filters with a stride of 1 for convolution layers and 2x2 filters with a stride of 2 for subsampling layers. LeNet-5 uses \textbf{average pooling}, which calculates the average of the values in the filter.

The network takes a 32x32 image as input. The first layer is a convolution layer (C1), which applies 6 filters, followed by a subsampling layer (S2). Next comes another convolution layer (C3) which applies 16 filter, followed by a subsampling layer (S4). A final convolution layer (C5) that applies 120 filters follows S4. Afterwards comes a fully connected (F6) layer with 84 neurons. Finally, an output layer of size 10 is connected to F6. Each neuron in the output layer corresponds to the probability of the input image to be one of the 10 digits (0-9).

\subsection{Training a Convolutional Neural Network}
We train a CNN using supervised learning. Supervised learning entails that our network learns from examples. The network is presented with a \textbf{dataset} of images. The dataset is divided into a \textbf{training set}, a \textbf{validation set} and a \textbf{testing set}. Each set contains images and their corresponding \textbf{labels} (also referred to as \textbf{classes}). We train a CNN to predict the labels of images as seen in the training set. The validation set is used for intermediate assessment of the CNN performance during the training phase, while the testing set is used to assess the performance of the CNN after the training phase is done.

CNNs process the training set in \textbf{mini-batches}. While the use of large mini-batches increases the available computational parallelism, small batch training has been shown to provide improved generalization performance \cite{masters2018revisiting}. It is also common for CNNs to process the whole training set multiple times. Each single iteration through the training set is called an \textbf{epoch}. The batch size and number of epochs are examples of CNN \textbf{hyperparameters}. Hyperparameters are network parameters that are not learnable, but rather optimized manually.

Training a convolutional neural network is the process of learning the right parameters (filter weights) to compute the correct labels from the input images in the training set. To do so, each CNN has a \textbf{loss function}. A loss function calculates the error of the predictions made by the CNN. The loss of a CNN which predicts an output \textit{y} on a training set that has the correct labels $\hat{y}$ is calculated as follows: \[L(y, \hat{y}) = \dfrac{1}{m}\sum_{i=1}^m \mathscr{L}(y_i, \hat{y}_i)\] where \textit{m} is the number of examples in the training set, and $\mathscr{L}(\cdot)$ is a distance function that calculates the difference between pairwise instances in $y$ and $\hat{y}$.

An \textbf{optimization function} (or \textbf{optimizer}) uses \textbf{back propagation} \cite{lecun1989backpropagation} to minimize the loss of a CNN during the training phase. The optimizer propagates the output of the loss function backwards by calculating the partial derivative of the loss function with respect to weights and using it to update the values of the weights.

The \textbf{stochastic gradient descent (SGD)} optimizer updates the weights of a CNN that uses the loss function $L$ thusly: \[w_{new}=w_{old}-\alpha\dfrac{\partial L}{\partial w_{old}}\] where $\alpha$ is called the \textbf{learning rate}. The learning rate controls how fast the weights are updated during training. The learning rate is a tricky hyperparameter to tune. If it is too large, the optimizer might overshoot the optimum parameter values. If it is too small, the optimizer might never reach the optimal value.

\textbf{Transfer learning} \cite{pan2010survey} is sometimes employed to speed up the training of convolutional neural networks. Transfer learning involves pre-training a CNN on images from a different domain, and then retraining the network on an image set that is specific to the desired classification task. The advantage of transfer learning is the ability to learn features from a different dataset. This is useful when we have a classification task in one domain of interest, but we only have sufficient training data in another domain. A common technique is to freeze the weights of earlier layers while training a CNN, meaning that the weights are set to be untrainable. The number of \textbf{frozen layers} is a network hyperparameter.

\section{Small Part Classification}\label{sec:synth}
We describe a system for the automatic classification of \textbf{small parts} in aircraft engines. Small parts refer to the fasteners used to build the engine like screws, bolts and nuts.

There is a set of classification challenges that are specific to small parts. A number of small parts can only be separated by subtle distinctions. This difference can be a slight change in size, which means that small parts, unlike scale invariant categories, are sensitive to differences in size, length and width. Figure \ref{fig:similar_small_parts} shows 2 screws that are identical except for a 6mm difference in length.

\begin{figure}[h]
\centering
  \includegraphics[width=0.5\textwidth]{similar_small_parts}
\caption{Screw (a) is identical to screw (b) except that screw (a) is 6mm shorter than screw (b). An overhead light shines on the 2 screws. The reflection changes the natural color of their surface.}
\label{fig:similar_small_parts}
\end{figure}

Aircraft fasteners are metallic and have a shiny surface. The light reflection off the fastener surface disturbs the natural surface color as shown in figure \ref{fig:similar_small_parts}, and may hide important features. Furthermore, the effect of the light reflection depends on the direction of the light, which may cause variance between images of a single small part.

Unlike cases where the background can provide information, the background is non-informative to the classification of small parts. Additionally, exposure to a regular lighting condition, like an overhead light in a room, causes the fasteners to cast a shadow on their background. A fastener shadow is not a discriminative feature. Both the background and the fastener shadow are a source of noise for the classifier.

\subsection{Real and Synthetic Images}
To automatically classify small parts, we build a convolutional neural network that is trained on both \textbf{real images} and \textbf{synthetic images}. We define real images as the natural images of the small parts, taken using a camera. Synthetic images, on the other hand, are artificially created images. They are 2-dimensional renditions of \textbf{3D models} of small parts. To generate synthetic images, we must create a \textbf{synthetic scene}: an artificial environment created in 3D modeling software. Figure \ref{fig:synthetic_scene} shows an example of a synthetic scene. A 3D model is placed in a synthetic scene in 3D modeling software. The software renders a 2D image of the environment to create a synthetic scene.

The real and synthetic images should capture the small parts and their corresponding 3D models from different angles and in different positions. To do so, we apply \textbf{transformations} to the small parts and 3D models. There are two types of transformations: \textbf{translation} to change the position of the target object, and \textbf{rotation} to change the angle. Each transformation has a range. This prevents the target objects from being translated or rotated away from the camera view.

\begin{figure}[h]
\centering
  \includegraphics[width=0.75\textwidth]{synthetic_scene}
\caption{A synthetic scene where a screw lies on a horizontal plane. (a) shows the isometric view of the scene while (b) shows the side view.}
\label{fig:synthetic_scene}
\end{figure}


    \filetitle{!transition\_equations}{Block of transition equations}{modellang/transitionequations}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
!transition_equations
    Equation1;
    Equation2;
    Equation2;
    ...
\end{verbatim}

\paragraph{Abbreviated syntax}\label{abbreviated-syntax}

\begin{verbatim}
!equations
    Equation1;
    Equation2;
    Equation3;
    ...
\end{verbatim}

\paragraph{Syntax with equation
labels}\label{syntax-with-equation-labels}

\begin{verbatim}
!transition_equations
    Equation1;
    'Equation label' Equation2;
    Equation3;
    ...
\end{verbatim}

\paragraph{Description}\label{description}

The \texttt{!transition\_equations} keyword starts a new block of
transition equations (i.e.~endogenous equations); the eqautions can
stretch over multiple lines and must be separated by semi-colons. You
can have as many equation blocks as you wish in any order in your model
file: They all get combined together when you read the model file in.

You can add descriptive labels to the equations (in single or double
quotes, preceding the equation); these will be stored in, and accessible
from, the model object.

\paragraph{Example}\label{example}

\begin{verbatim}
!transition_equations
    'Euler equation' C{1}/C = R*beta;
\end{verbatim}


\subsection{Breaking monoalphabetic substitution ciphers with frequency analysis}

(need to identify algorithm and needs to identify key)

finding substitution cyphers

Search space is larger, \(26!=4*10^26\). need alternative to brute force.

Letter popularity. Compare against popularity for corpus.
Monogram (ie letters); ngrams(ie n letter in a row frequency); common words.

Single letter words are I or A. More generally. corpus smaller for fewer letters

Can test substitution cypher by matching each word against a corpus


\chapter{Inequalities}

\begin{ex}
  Let $X\sim\text{Exponential}(\beta)$, and recall that $\mu_X=\beta$ and
  $\sigma^2_X=\beta^2$. Then, for any $k>1$,
  \begin{align*}
    \P{|X-\mu_X|\geq k\sigma_X}
     & =\P{|X-\beta|\geq k\beta}                                    \\
     & =\P{X-\beta\geq k\beta}+\P{X-\beta\leq -k\beta}              \\
     & =\P{X\geq (k+1)\beta}+\P{X\leq -(k+1)\beta}                  \\
     & =\P{X\geq (k+1)\beta}                                        \\
     & =\int_{(k+1)\beta}^\infty \frac{1}{\beta}e^{-x/\beta}\,\d{x} \\
     & =\int_{k+1}^\infty e^{-u}\,\d{u}                             \\
     & =e^{-(k+1)}.
  \end{align*}

  The comparable bound obtained from Chebyshev's inequality is
  \begin{align*}
    \P{|X-\mu_X|\geq k\sigma_X}<\frac{\beta^2}{k^2\beta^2}=\frac{1}{k^2}.
  \end{align*}
  This is a weaker bound, since it decays quadratically in $k$, instead of
  exponentially.
\end{ex}

\begin{ex}
  Let $X\sim\text{Poisson}(\lambda)$. Recall that $\mu_X=\lambda$ and
  $\sigma_X^2=\lambda$. Note that
  \begin{align*}
    \P{|X-\mu_X|\geq \lambda}
     & =\P{X-\lambda\geq \lambda}+\P{X-\lambda\leq -\lambda}     \\
     & =\P{X\geq \lambda + \lambda}+\P{X\leq -\lambda + \lambda} \\
     & =\P{X\geq 2\lambda}+\P{X\leq 0}                           \\
     & =\P{X\geq 2\lambda}.
  \end{align*}
  However, by Chebyshev's inequality for $t=\lambda$,
  \[
    \P{X\geq 2\lambda}
    =\P{|X-\mu_X|\geq \lambda}
    \leq \frac{1}{\lambda}.
  \]
\end{ex}

\begin{ex}
  Let $X_1,\ldots,X_n\sim\text{Bernoulli}(p)$. Then
  \[
    \E{\overline{X}_n}=\frac{np}{n}=p,\quad
    \var{\overline{X}_n}=\frac{np(p-1)}{n^2}=\frac{p(p-1)}{n}.
  \]
  Thus, by Chebyshev's inequality,
  \[
    \P{|\overline{X}_n-p|>\epsilon}
    \leq \frac{\frac{p(p-1)}{n}}{\epsilon^2}
    =\frac{p(p-1)}{n\epsilon^2}.
  \]

  By Theorem 4.5, we also have
  \[
    \P{|\overline{X}_n-p|>\epsilon}
    \leq 2e^{-2n\epsilon^2}.
  \]

  Note that the bound from Chebyshev's inequality decays at the rate of $1/n$
  as $n\to\infty$, while the bound from Hoeffding's inequality does so at an
  exponential rate. Hence, for $n$ sufficiently large, Hoeffding's inequality
  will provide a tighter bound.
\end{ex}

\begin{ex}
  Let $X_1,\ldots X_n\sim\text{Bernoulli}(p)$. For any $\alpha>0$ define
  \[
    \epsilon_n=\sqrt{\frac{1}{2n}\log\left(\frac{2}{\alpha}\right)},
  \]
  and let $C_n=(\widehat{p}_n-\epsilon_n, \widehat{p}_n+\epsilon_n)$.
  \begin{enumerate}[(a)]
    \item Note that
          \begin{align*}
            \P{C_n\text{ contains } p}
             & =\P{|\widehat{p}_n-p|\leq \epsilon_n} \\
             & =1-\P{|\widehat{p}_n-p|> \epsilon_n}  \\
             & \geq 1-2e^{-2n\epsilon_n^2}           \\
             & =1-2e^{-\log(2/\alpha)}               \\
             & =1 - \alpha.
          \end{align*}
    \item
          \inputminted{python}{../code/04-04b.py}

          \begin{figure}[H]
            \centering
            \includegraphics[scale=0.9]{../images/04-04b}
            \caption{Graph of simulated coverage versus $n$ for $p=0.4$ and
              $\alpha=0.05$, with 50,000 simulations per $n$.}
          \end{figure}
    \item
          \inputminted{python}{../code/04-04c.py}
          \inputminted{text}{../output/04-04c.txt}

          \begin{figure}[H]
            \centering
            \includegraphics[scale=0.9]{../images/04-04c}
            \caption{Graph of interval width versus $n$ for $\alpha=0.05$.}
          \end{figure}
  \end{enumerate}
\end{ex}

\begin{ex}
  Let $Z\sim N(0, 1)$. Then
  \[
    \P{|Z|>t}
    =2\P{Z>t}
    =\sqrt{\frac{2}{\pi}}\frac{e^{-t^2/2}}{t},
  \]
  since
  \begin{align*}
    t\P{Z>t}
     & =t\int_t^\infty \frac{1}{\sqrt{2\pi}}e^{-x^2/2}\,\d{x}    \\
     & \leq \int_t^\infty \frac{x}{\sqrt{2\pi}}e^{-x^2/2}\,\d{x} \\
     & =-\frac{1}{\sqrt{2\pi}}e^{-x^2/2}\bigg\rvert_{x=t}^\infty \\
     & =\frac{1}{\sqrt{2\pi}}e^{-t^2/2}.
  \end{align*}
\end{ex}

\begin{ex}
  Let $k\geq 3$ be an integer. Note that
  \begin{align*}
    \E{|Z|}
     & =\frac{2}{\sqrt{2\pi}}\int_{0}^\infty\!xe^{-\frac{1}{2}x^2}\,\d{x}
    =-\frac{2}{\sqrt{2\pi}}\int_{0}^\infty\!e^{-u}\,\d{u}
    =\sqrt{\frac{2}{\pi}},                                                \\
    \E{|Z|^2}
     & =\E{Z^2}=\var{Z}+\E{Z}^2=1,
  \end{align*}
  \begin{align*}
    \E{|Z|^k}
     & =\frac{2}{\sqrt{2\pi}}\int_{0}^\infty\!x^ke^{-\frac{1}{2}x^2}\,\d{x}                  \\
     & =\frac{2}{\sqrt{2\pi}}\frac{1}{2}\int_0^\infty\!u^{(k-1)/2}e^{-u/2}\,\d{u}            \\
     & =\frac{2}{\sqrt{2\pi}}\frac{1}{2}\left[-2e^{-u/2}u^{(k-1)/2}\bigg\rvert_{u=0}^\infty+
    (k-1)\int_0^\infty e^{-u/2}u^{(k-3)/2}\right]                                            \\
     & =(k-1)\frac{2}{\sqrt{2\pi}}\frac{1}{2}\int_0^\infty e^{-u/2}u^{(k-3)/2}\,\d{u}        \\
     & =(k-1)\E{|Z|^{k-2}},
  \end{align*}
  and that therefore
  \[
    \E{|Z|^k}=\begin{cases}
      \sqrt{\frac{2}{\pi}}(k-1)!! & \text{if $k$ is odd},  \\
      (k-1)!!                     & \text{if $k$ is even}.
    \end{cases}
  \]

  Hence,
  \[
    \P{|Z|>t}
    =\P{|Z|^k>t^k}
    \leq \frac{\E{|Z|^k}}{t^k}
    =\begin{cases}
      \sqrt{\frac{2}{\pi}}\frac{(k-1)!!}{t^k} & \text{if $k$ is odd},  \\
      \frac{(k-1)!!}{t^k}                     & \text{if $k$ is even},
    \end{cases}
  \]
  or, in particular, $\P{|Z|>t}<t_0$ for $t_0\in\left\{
    \sqrt{\frac{2}{\pi}}\frac{1}{t}, \frac{1}{t^2}, 2\sqrt{\frac{2}{\pi}}\frac{1}{t^3},
    \frac{3}{t^4}, 8\sqrt{\frac{2}{\pi}}\frac{1}{t^5}
    \right\}$.

  Likewise, by Mill's inequality,
  \[
    \P{|Z|>t}
    \leq \sqrt{\frac{2}{\pi}}\frac{e^{-t^2/2}}{t}.
  \]

  Finally,
  \begin{align*}
    \P{|Z|>t}
    =\frac{2}{\sqrt{2\pi}}\int_{t}^\infty\!e^{-\frac{1}{2}x^2}\,\d{x}
    =\erfc\left(\frac{t}{\sqrt{2}}\right).
  \end{align*}

  \inputminted{python}{../code/04-06.py}

  \begin{figure}[H]
    \centering
    \includegraphics[scale=0.6]{../images/04-06}
    \caption{Graph of various lower bounds for $\P{|Z|>t}$.}
  \end{figure}
\end{ex}

\begin{ex}
  Let $X_1,\ldots,X_n\sim N(0,1)$. Then $X_1+X_2+\cdots+X_n\sim N(0, n)$ and
  therefore
  \[
    \sqrt{n}\cdot \overline{X}_n=\frac{\sqrt{n}}{n}\left(X_1+X_2+\cdots+X_n\right)
  \]
  is a standard normal random variable. Thus, by Mill's inequality,
  \[
    \P{|\overline{X}_n|>t}
    =\P{|\sqrt{n}\cdot \overline{X}_n|>t\sqrt{n}}\leq \sqrt{\frac{2}{\pi}}\frac{\exp(-t^2n/2)}{t\sqrt{n}},
  \]
  while, since $\overline{X}_n\sim N(0,1/n)$, by the Chebyshev bound,
  \[
    \P{|\overline{X}_n|>t}\leq \frac{1}{nt^2}.
  \]
\end{ex}


\input{../newcommands}

Although full PIC codes are powerful tools, which capture a wide range
of physical phenomena, they also require large computational ressources.
This is partly due to the use of a 3D Cartesian grid, which
leads to a very large number of grid cells. (Typical 3D simulations of
laser-wakefield acceleration require $\sim 10^6$--$ 10^8$ grid
cells.) For this reason, these algorithms need to be highly parallelized, and
high-resolution simulations can only be run on costly large-scale
computer facilities. However, when the driver is
cylindrically-symmetric, it is possible to take advantage of the
symmetry of the problem to reduce the computational cost of the algorithm \cite{godfrey1985iprop,LifschitzJCP2009,DavidsonJCP2015,Lehe2016}.

\subsection{Azimuthal decomposition}
Let us consider the fields $\vec{E}$, $\vec{B}$, $\vec{J}$ and $\rho$
 in cylindral coordinates $(r,\theta,z)$, expressed as a Fourier series in $\theta$:
%
\begin{equation}
F(r,\theta,z) = \mathrm{Re}\left[ \sum_{\ell=0}^\infty
  \tilde{F}_{\ell}(r,z) e^{-i\ell\theta} \right]
\label{eq:chap2:azimuthal}
\end{equation}
%
\begin{equation}
\mathrm{with} \qquad \tilde{F}_{\ell} = C_\ell \int_0^{2\pi} d\theta
\,F(r,\theta,z)e^{i\ell\theta} \qquad
\label{eq:chap2:Fourier-coeffs}
\end{equation}
\begin{equation}
\mathrm{and} \;
\left \{ \begin{array}{l l}
C_{0} = 1/2\pi &\\
C_\ell = 1/\pi &\mathrm{for}\,\ell > 0
\end{array} \right.
\end{equation}
%
where $F$ represents any of the quantities $E_r$,
$E_\theta$, $E_z$, $B_r$, $B_\theta$, $B_z$, $J_r$, $J_\theta$, $J_z$
are $\rho$, and where the
$\tilde{F}_\ell$ are the associated Fourier components ($\ell$ is the
index of the corresponding azimuthal mode). In the general case, this
azimuthal decomposition does not simplify the problem, since an
infinity of modes have to be considered in (\ref{eq:chap2:azimuthal}). However, in the case of a
cylindrically-symmetric laser pulse, only the very first modes have
non-zero components. For instance, the wakefield is represented
exclusively by the mode $\ell = 0$. (This is because the quantities $E_r$,
$E_\theta$, $E_z$, $B_r$, $B_\theta$, $B_z$, $J_r$, $J_\theta$, $J_z$
and $\rho$ associated with the
wakefield are independent of $\theta$.) On the other hand, the field
of the laser pulse \emph{does}
depend on $\theta$, in cylindrical coordinates. For example, for a
cylindrically-symmetric pulse propagating along $z$ and polarized along $\vec{e}_\alpha = \cos(\alpha)\vec{e}_x + \sin(\alpha)\vec{e}_y$:
%
\begin{align}
\vec{E} &= E_0(r,z)\vec{e}_\alpha \\
& = E_0(r,z) [\; \cos(\alpha)(\cos(\theta)\vec{e}_r - \sin(\theta)\vec{e}_\theta) \; \nonumber \\
& + \; \sin(\alpha)(\sin(\theta)\vec{e}_r + \cos(\theta)\vec{e}_\theta) \; ]\\
& = \mathrm{Re}[ \; E_0(r,z) e^{i\alpha} e^{-i\theta} \; ]\vec{e}_r \; \nonumber \\
& + \; \mathrm{Re}[ \; -i E_0(r,z) e^{i\alpha} e^{-i\theta} \; ]\vec{e}_\theta.
\end{align}
%
Here the amplitude $E_0$ does not depend on $\theta$ because the pulse was assumed
to be cylindrically symmetric. In this case, the above relation shows
that the fields $E_r$ and $E_\theta$ of the laser are represented
exclusively by the mode $\ell = 1$. A similar calculation shows that
the same holds for $B_r$ and $B_\theta$. On the whole, only the modes
$\ell = 0$ and $\ell = 1$ are a priori necessary to model
laser-wakefield acceleration.
Under those conditions, the infinite sum in
(\ref{eq:chap2:azimuthal}) is truncated at a chosen $\ell_{max}$. In
principle, $\ell_{max} = 1$ is sufficient for laser-wakefield
acceleration. However, $\ell_{max}$ is kept as a free parameter in the algorithm, in order to verify that
higher modes are negligible, as well as to allow for less-symmetric configurations.
Because codes based on this algorithm are able to take into account the modes with $\ell > 0$, they are said to be
``quasi-cylindrical'' (or ``quasi-3D'' by some authors \cite{DavidsonJCP2015}), in contrast to cylindrical codes, which
assume that all fields are independent of $\theta$, and thus only
consider the mode $\ell = 0$.

\subsection{Discretized Maxwell equations} When the Fourier expressions
of the fields are injected into the Maxwell equations (written in
cylindrical coordinates), the different azimuthal modes
decouple. In this case, the Maxwell-Amp\`ere and Maxwell-Faraday equations
-- which are needed to update the fields in the PIC cycle -- can be written separately
for each azimuthal mode $\ell$:
\begin{subequations}
\begin{align}
\frac{\partial \tilde{B}_{r,\ell} }{\partial t} &=
\frac{i\ell}{r}\tilde{E}_{z,\ell} + \frac{\partial
  \tilde{E}_{\theta,\ell}}{\partial z} \\[3mm]
\frac{\partial \tilde{B}_{\theta,\ell} }{\partial t} &=
 - \frac{\partial \tilde{E}_{r,\ell}}{\partial z} + \frac{\partial
  \tilde{E}_{z,\ell}}{\partial r} \\[3mm]
\frac{\partial \tilde{B}_{z,\ell} }{\partial t} &=
- \frac{1}{r} \frac{\partial (r\tilde{E}_{\theta,\ell})}{\partial r} - \frac{i\ell}{r}\tilde{E}_{r,\ell} \\[3mm]
\frac{1}{c^2} \frac{\partial \tilde{E}_{r,\ell} }{\partial t} &=
-\frac{i\ell}{r}\tilde{B}_{z,\ell} - \frac{\partial
  \tilde{B}_{\theta,\ell}}{\partial z} - \mu_0 \tilde{J}_{r,\ell} \\[3mm]
\frac{1}{c^2}\frac{\partial \tilde{E}_{\theta,\ell} }{\partial t} &=
 \frac{\partial \tilde{B}_{r,\ell}}{\partial z} - \frac{\partial
  \tilde{B}_{z,\ell}}{\partial r} - \mu_0 \tilde{J}_{\theta,\ell} \\[3mm]
\frac{1}{c^2}\frac{\partial \tilde{E}_{z,\ell} }{\partial t} &=
 \frac{1}{r} \frac{\partial (r\tilde{B}_{\theta,\ell})}{\partial r} +
 \frac{i\ell}{r}\tilde{B}_{r,\ell} - \mu_0 \tilde{J}_{z,\ell}
\end{align}
\end{subequations}
%\begin{figure}
%\input{./Chap2/Circ_lattice.tex}
%\caption{Representation of the lattice in \CCirc. The
%  table shows at which
%  position each component of the fields is defined ($j$,$k$ and
%  $n$ are integers ; $\Delta r$ and $\Delta z$ are the
%  spatial steps of the grid). The above sketch represents one grid
%  cell, and the positions of the fields within it.}
%\label{fig:chap2:Circ_lattice}
%\end{figure}
In order to discretize these equations, each azimuthal mode is
represented on a two-dimensional grid,
%(The two dimensions correspond
%to $r$ and $z$.) \Cref{fig:chap2:Circ_lattice} summarizes the
%positions of the different fields within one grid cell, as well as the
%corresponding notations for these fields. Using these notations,
on which the discretized
Maxwell-Amp\`ere and Maxwell-Faraday equations are given by
%\begin{strip}
%\begin{align*}
%
%\frac{ \tBr{n+\hf}{j,\ell,k+\hf}- \tBr{n-\hf}{j,\ell,k+\hf}
%}{\Delta t} =& \frac{i\,\ell}{j\Delta r}\tEz{n}{j,\ell,k+\hf} + (D_z \tilde{E}_{\theta}^n)_{j,\ell,k+\hf} \\
%
%\frac{ \tBt{n+\hf}{j+\hf,\ell,k+\hf}- \tBt{n-\hf}{j+\hf,\ell,k+\hf} }{\Delta t} =& -(D_z \tilde{E}_r^n)_{j+\hf,\ell,k+\hf} + (D_r \tilde{E}_z^{n})_{j+\hf,\ell,k+\hf} \\
%
%\frac{ \tBz{n+\hf}{j+\hf,\ell,k}- \tBz{n-\hf}{j+\hf,\ell,k} }{\Delta t} =&
% -\frac{(j+1)\tEt{n}{j+1,\ell,k} - j\tEt{n}{j,\ell,k}}{(j+\hf)\Delta r} -
%\frac{i\,\ell}{(j+\hf) \Delta r}\tEr{n}{j+\hf,\ell,k} \\
%
%\frac{ \tEr{n+1}{j+\hf,\ell,k}- \tEr{n}{j+\hf,\ell,k}}{c^2 \Delta t}
%=& -\frac{i\,\ell}{(j+\hf)\Delta r}\tBz{n+\hf}{j+\hf,\ell,k} - (D_z \tilde{B}_{\theta}^{n+\hf})_{j+\hf,\ell,k} - \mu_0\tJr{n+\hf}{j+\hf,\ell,k} \\
%
%\frac{ \tEt{n+1}{j,\ell,k}- \tEt{n}{j,\ell,k}}{c^2 \Delta t}
%=& (D_z \tilde{B}_r^{n+\hf})_{j,\ell,k} - (D_r \tilde{B}_z^{n+\hf})_{j,\ell,k} - \mu_0\tJt{n+\hf}{j,\ell,k} \\
%
%\frac{ \tEz{n+1}{j,\ell,k+\hf}- \tEz{n}{j,\ell,k+\hf}}{c^2 \Delta t}
%=&   \frac{\left(j+\hf\right)\tBt{n+\hf}{j+\hf,\ell,k+\hf}  - \left(j-\hf\right)\tBt{n+\hf}{j-\hf,\ell,k+\hf}}{j\Delta r} \\
%& \qquad  \qquad + \frac{i\,\ell}{j\Delta r}\tBr{n+\hf}{j,\ell,k+\hf} - \mu_0\tJz{n+\hf}{j,\ell,k+\hf}
%\end{align*}
%\end{strip}

\begin{subequations}
\begin{align}
%
D_{t}\tilde{B}_r|_{j,\ell,k+\hf}^{n} \nonumber
=& \frac{i\,\ell}{j\Delta r}\tEz{n}{j,\ell,k+\hf} \\
& + D_z \tilde{E}_{\theta}|^n_{j,\ell,k+\hf} \\
%
D_{t}\tilde{B}_\theta|_{j+\hf,\ell,k+\hf}^{n} \nonumber
=& -D_z \tilde{E}_r|^n_{j+\hf,\ell,k+\hf} \\
& + D_r \tilde{E}_z|^{n}_{j+\hf,\ell,k+\hf} \\
%
D_{t}\tilde{B}_z|_{j+\hf,\ell,k}^{n} =& \nonumber
 -\frac{(j+1)\tEt{n}{j+1,\ell,k} }{(j+\hf)\Delta r} \\ \nonumber
 & +\frac{ j\tEt{n}{j,\ell,k}}{(j+\hf)\Delta r} \\
 & - \frac{i\,\ell}{(j+\hf) \Delta r}\tEr{n}{j+\hf,\ell,k}
\end{align}
\end{subequations}
for the magnetic field components, and
\begin{subequations}
\begin{align}
%
\frac{1}{c^2}D_{t}\tilde{E}_r|_{j+\hf,\ell,k}^{n+\hf} \nonumber
=& -\frac{i\,\ell}{(j+\hf)\Delta r}\tBz{n+\hf}{j+\hf,\ell,k} \\
& - D_z \tilde{B}_{\theta}|^{n+\hf}_{j+\hf,\ell,k} \nonumber\\
& - \mu_0\tJr{n+\hf}{j+\hf,\ell,k} \\
%
\frac{1}{c^2}D_{t}\tilde{E}_\theta|_{j,\ell,k}^{n+\hf} \nonumber
=& D_z \tilde{B}_r|^{n+\hf}_{j,\ell,k} - D_r \tilde{B}_z|^{n+\hf}_{j,\ell,k} \\
& - \mu_0\tJt{n+\hf}{j,\ell,k} \\
%
\frac{1}{c^2}D_{t}\tilde{E}_z|_{j,\ell,k+\hf}^{n+\hf} \nonumber
=&   \frac{\left(j+\hf\right)\tBt{n+\hf}{j+\hf,\ell,k+\hf} }{j\Delta r} \\
=&   -\frac{\left(j-\hf\right)\tBt{n+\hf}{j-\hf,\ell,k+\hf}}{j\Delta r} \nonumber\\
& + \frac{i\,\ell}{j\Delta r}\tBr{n+\hf}{j,\ell,k+\hf} \nonumber\\
& - \mu_0\tJz{n+\hf}{j,\ell,k+\hf}
\end{align}
\end{subequations}
for the electric field components.

The numerical operator $D_r$ and $D_z$ are defined by
\begin{align*}
(D_r F)_{j',\ell,k'} = \frac{F_{j'+\hf,\ell,k'}-F_{j'-\hf,\ell,k'} }{\Delta r} \\
(D_z F)_{j',\ell,k'} = \frac{F_{j',\ell,k'+\hf}-F_{j',\ell,k'-\hf} }{\Delta z} \\
\end{align*}
where $j'$ and $k'$ can be integers or half-integers. Notice
that these discretized Maxwell equations are not valid on-axis (i.e. for $j=0$), due to
singularities in some of the terms. Therefore, on the axis, these equations are replaced by specific boundary conditions, which are based on the symmetry properties of the fields (see \cite{LifschitzJCP2009} for details).

Compared to a 3D Cartesian calculation with $n_x\times n_y \times n_z$
grid cells, a quasi-cylindrical calculation with two modes ($l=0$ and $l=1$)
will require only $3 \,n_r \times n_z$ grid cells. Assuming $n_x=n_y=n_r=100$
as a typical transverse resolution, a quasi-cylindrical calculation is typically
over an order of magnitude less computationally demanding than its 3D Cartesian
equivalent.