\documentclass[12pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath}
\usepackage{hyperref}

\title{Piecewise Polynomial Path Planner (P4)}
\author{Tucker Haydon}
\date{Spring 2019}

\begin{document}

\maketitle

\section{Introduction}
This document describes the underlying theory for the piecewise polynomial path
planner.

\section{Polynomial Theory}
\subsection{Problem Statement}
Given a set of waypoints and corresponding arrival times, construct a
piecewise-continuous polynomial trajectory through these waypoints that minimizes
the norm of a specified derivative. Furthermore, the trajectory --- or any
derivative of the trajectory --- may be subject to linear inequality
constraints.
\bigbreak
\noindent Mathematically, the problem may be stated so: given a set of $m$ waypoints
%
\begin{equation}
  \mathbf{W} =
  \begin{bmatrix}
    \vec{w}_{0} & \vec{w}_{1} & \vec{w}_{2} & \hdots & \vec{w}_{m}
  \end{bmatrix}
  \in \mathcal{R}^a
\end{equation}
%
and a set of times to arrive at these waypoints
\begin{equation}
  \vec{T} =
  \begin{bmatrix}
    t_{0} & t_{1} & t_{2} & \hdots & t_{m}
  \end{bmatrix}
\end{equation}
%
determine a set of $n$'th-order piecewise-continuous polynomials that smoothly connect these waypoints
%
\begin{equation}
  \vec{p}(t) =
  \begin{cases}
    p_{0}(t) = \sum_{k=0}^{n} p_{0,k} \cdot (\frac{1}{k!} t^{k}) & t_{0} \leq t < t_{1} \\
    p_{1}(t) = \sum_{k=0}^{n} p_{1,k} \cdot (\frac{1}{k!} t^{k}) & t_{1} \leq t < t_{2} \\
    \vdots \\
    p_{m}(t) = \sum_{k=0}^{n} p_{m,k} \cdot (\frac{1}{k!} t^{k}) & t_{m-1} \leq t < t_{m}
  \end{cases}
\end{equation}
%
while minimizing the squared norm of the $r$'th derivative
%
\begin{equation}
  \min
  \int_{t_{0}}^{t_{m}} \frac{d^{r}}{dt^{r}}|| \vec{p}(t) ||^{2} dt
\end{equation}
%
subject to continuity and path constrains (see sections \ref{} \& \ref{}).
\bigbreak
\noindent This problem construction and the following solution follow those of Mellinger
and Kumar \cite{mellinger2011minimum}.

\subsection{Polynomial Basis}
Trajectories for quadcopters may be specified with no more than four independent
dimensions: x, y, z, and yaw. Any more dimensions (roll, pitch) introduce
constraints that may render certain specified trajectories infeasible. For
example, a quadcopter cannot be simultaneously stationary and have a non-zero
roll angle \cite{mellinger2011minimum}.

Importantly, the x, y, z, and yaw dimensions may be specified
\textit{independently}: each dimension of the polynomial basis functions may be
considered alone, permitting scalar analysis. The following section uses a
3rd-order polynomial, but the analysis generalizes to any order polynomial.

Consider a 3rd-order polynomial representing the first section of trajectory.
\begin{equation*}
  p_{0}(t) = \frac{1}{0!}t^{0}x_{0} + \frac{1}{1!}t^{1}v_{0}
           + \frac{1}{2!}t^{2}a_{0} + \frac{1}{3!}t^{3}j_{0}
\end{equation*}
%
This expression may be divided into two components: a vector of coefficients and
a vector of basis functions.
\begin{equation*}
  p_{0}(t) =
  \underbrace{
  \begin{bmatrix}
    x_{0} & v_{0} & a_{0} & j_{0}
  \end{bmatrix}
  }_{\text{coefficients}}
  \underbrace{
  \begin{bmatrix}
    \frac{t^{0}}{0!} \\[6pt]
    \frac{t^{1}}{1!} \\[6pt]
    \frac{t^{2}}{2!} \\[6pt]
    \frac{t^{3}}{3!}
  \end{bmatrix}
  }_{\text{basis functions}}
\end{equation*}
%
Or in vector form
\begin{equation*}
  p_{0}(t) = \vec{x}^{T} \vec{t}
\end{equation*}

\subsection{Polynomial Optimization}
For each section in the trajectory, the $r$'th derivative must be minimized.
\begin{align*}
    x_{i} &= \text{arg} \min_{\vec{x}} \int_{t_{i}}^{t_{i+1}} \bigg[ \frac{d^{r}
    p(\tau)}{d \tau^{r}} \bigg]^2 dt \\
        &= \text{arg} \min_{\vec{x}} \int_{t_{i}}^{t_{i+1}} \bigg[ \frac{d^{r}
    p(\tau)}{d \tau^{r}}\bigg]^T \bigg[ \frac{d^{r} p(\tau)}{d \tau^{r}}\bigg]
    dt \\
        &= \text{arg} \min_{\vec{x}} \int_{t_{i}}^{t_{i+1}} \vec{x}^T \frac{d^{r}
    \vec{t}(\tau)}{d \tau^{r}}^{T} \frac{d^{r} \vec{t}(\tau)}{d \tau^{r}}
    \vec{x} dt \\
        &= \text{arg} \min_{\vec{x}} \vec{x}^{T} \int_{t_{i}}^{t_{i+1}} \frac{d^{r}
    \vec{t}(\tau)}{d \tau^{r}}^{T} \frac{d^{r} \vec{t}(\tau)}{d \tau^{r}} dt
    \vec{x}
\end{align*}
%
By defining
\begin{equation*}
  \mathbf{Q} = \int_{t_{i}}^{t_{i+1}} \frac{d^{r} \vec{t}(\tau)}{d
  \tau^{r}}^{T} \frac{d^{r} \vec{t}(\tau)}{d \tau^{r}} dt
\end{equation*}
%
and noting that the time-varying components will be integrated out by the
definite integral, the optimization problem may be written as
\begin{equation*}
  x_{i} = \text{arg} \min_{\vec{x}} \vec{x}^{T} \mathbf{Q} \vec{x}
\end{equation*}
%
The structure of the integrand of $\mathbf{Q}$ is revealed by first computing the initial couple
derivatives of $\vec{t}(\tau)$.
\begin{align*}
  \vec{t(\tau)} &=
  \begin{bmatrix}
    \frac{\tau^{0}}{0!} & \frac{\tau^{1}}{1!} & \frac{\tau^{2}}{2!} & \hdots
  \end{bmatrix} \\
  \frac{d \vec{t(\tau)}}{d\tau} &=
  \begin{bmatrix}
    0 & \frac{\tau^{0}}{0!} & \frac{\tau^{1}}{1!} & \hdots
  \end{bmatrix} \\
  \frac{d^{2} \vec{t(\tau)}}{d\tau^{2}} &=
  \begin{bmatrix}
    0 & 0 & \frac{\tau^{0}}{0!} & \hdots
  \end{bmatrix} \\
\end{align*}
%
Notice that successive derivatives pre-pad zeros and shift the vector
to the right.

The integrand of $\mathbf{Q}$ is found by taking the outer product of the
derivative of the time vector with itself.
\begin{align*}
  \vec{t(\tau)}^{T} \vec{t(\tau)} &=
  \begin{bmatrix}
    \frac{t^{0}}{0!} \frac{t^{0}}{0!} & \frac{t^{0}}{0!} \frac{t^{1}}{1!} &
    \frac{t^{0}}{0!} \frac{t^{2}}{2!}  & \hdots \\[6pt]
    \frac{t^{1}}{1!} \frac{t^{0}}{0!} & \frac{t^{1}}{1!} \frac{t^{1}}{1!} &
    \frac{t^{1}}{1!} \frac{t^{2}}{2!}  & \hdots \\[6pt]
    \frac{t^{2}}{2!} \frac{t^{0}}{0!} & \frac{t^{2}}{2!} \frac{t^{1}}{1!} &
    \frac{t^{2}}{2!} \frac{t^{2}}{2!}  & \hdots \\[6pt]
    \vdots & \vdots & \vdots & \ddots
  \end{bmatrix} \\
  \frac{d\vec{t(\tau)}}{d\tau}^{T} \frac{d\vec{t(\tau)}}{d\tau} &=
  \begin{bmatrix}
    0 & 0 & 0 & \hdots \\[6pt]
    0 & \frac{t^{0}}{0!} \frac{t^{0}}{0!} & \frac{t^{0}}{0!} \frac{t^{1}}{1!} & \hdots \\[6pt]
    0 & \frac{t^{1}}{1!} \frac{t^{0}}{0!} & \frac{t^{1}}{1!} \frac{t^{1}}{1!} & \hdots \\[6pt]
    \vdots & \vdots & \vdots & \ddots
  \end{bmatrix} \\
  \frac{d^{2}\vec{t(\tau)}}{d\tau^{2}}^{T} \frac{d^{2}\vec{t(\tau)}}{d\tau^{2}} &=
  \begin{bmatrix}
    0 & 0 & 0 & \hdots \\[6pt]
    0 & 0 & 0 & \hdots \\[6pt]
    0 & 0 & \frac{t^{0}}{0!} \frac{t^{0}}{0!} & \hdots \\[6pt]
    \vdots & \vdots & \vdots & \ddots
  \end{bmatrix} \\
\end{align*}
%
Notice that the outer product of successive derivatives pre-pads columns and rows
with zeros and shift rows down and columns to the right. This admits a
convenient way to compute the integrand of $\mathbf{Q}$: take the outer product of
$\vec{t}(\tau)$ with itself and shift the rows and columns by the power of the
derivative.
\begin{equation*}
  \frac{d^{r}\vec{t(\tau)}}{d\tau^{r}}^{T} \frac{d^{r}\vec{t(\tau)}}{d\tau^{r}}
  = \text{shift}(\vec{t}(\tau)^{T} \vec{t}(\tau), r)
\end{equation*}
\\
To compute $\mathbf{Q}$, the matrix integrand must be integrated. Integrating a
matrix is as easy as integrating its components element-wise. The polynomial
structure of the components of $\mathbf{Q}$ admit and easy-to-compute integral
formula:
\begin{equation*}
  \int \frac{\tau^{i}}{i!} \frac{\tau^{j}}{j!} d\tau =
  \frac{\tau^{i+j+1}}{i!j!(i+j+1)!}
\end{equation*}
\\
Finally, a careful observer might note that the structure of problem permits one
to swap the shift and integrate operations. Instead of shifting all elements and
then integrating, one could integrate all elements and then shift. This
simplifies implementation in a for-loop-based subroutine.
\bigbreak
\noindent In summary, to compute $\mathbf{Q}$:
\begin{enumerate}
  \item Compute the outer product of $\vec{t}(\tau)$ with itself
  \item Integrate the outer product
  \item Shift the rows and columns of the integrated matrix by the order of the
    derivative
\end{enumerate}

\subsection{Relaxing Arrival Time}
Until this point, one very important topic has been overlooked: timing. Users
may specify arbitrary arrival times for waypoints. When times are too large, too
small, or a combination of the both, numerical accuracy problems occur and many
optimizers struggle to find sufficiently accurate solutions. To prevent this
from occurring, segment time is normalized to $t \in [0,1]$. Stated otherwise,
every segment in a trajectory is assumed to start at $t=0$ and end at $t=1$.

This new timing scheme is referred to as the \textit{computational time} while
the original timing scheme is referred to as the \textit{physical time}.
Transforming the problem from physical time to computational time requires one
to determine the segment index a particular time is, subtracting the start time,
and normalized by the total segment time. The inverse transform is defined in a
similar manner.

\subsection{Time Scaling}
Let $t \in [0, t_{f}]$ represent the physical time along a segment and $\tau \in
[0, 1]$ represent the computational time along a segment. The two are related by
a scaling factor.
%
\begin{equation}
  t = \alpha \tau
\end{equation}
%
Path constraints are specified using physical time, but the polynomial functions
and optimization use computational time. These two timing schemes must be
reconciled. Let a constraint be defined as:
%
\begin{equation*}
  \frac{dp(\tau)}{dt} < \kappa
\end{equation*}
%
Change of basis using $\tau = t / \alpha$ and $d\tau = dt / \alpha$. Substitute
and rearrange:
%
\begin{equation}
  \frac{dp(\tau)}{d\tau} < \alpha \kappa
\end{equation}
%
Constraints on higher derivatives are derived using the same process.
\begin{equation}
  \frac{dp^{n}(\tau)}{dt^{n}} < \kappa \rightarrow
  \frac{dp^{n}(\tau)}{d\tau^{n}} < \alpha^{n} \kappa
\end{equation}
%
Continuity constraints (discussed in the next section) merit special
consideration. These constraints require that the $n$th derivative of the end of
a segment match the $n$th derivative of the following segment. Since the time
scaling of each segment may be different, the constraint manifests so:
\begin{equation}
  \frac{dp_{k}^{n}(\tau)}{d\tau^{n}} \frac{1}{\alpha_{k}^{n}} =
  \frac{dp_{k+1}^{n}(\tau)}{d\tau^{n}} \frac{1}{\alpha_{k+1}^{n}}
\end{equation}

\subsection{Continuity Constraints}
Most piecewise-continuous problems require some sort of continuity --- the
position, velocity, or acceleration must not instantaneously change at endpoints
of the piecewise trajectory. Typically this is motivated by real-life
constraints: a quadcopter cannot instantaneously teleport or change its
velocity from 1 m/s to -3 m/s.

Consider the first two segments of a scaler third-order piecewise polynomial
trajectory. Let the optimization vector contain the coefficients for both
segments of the trajectory.
\begin{equation*}
  \vec{x} =
  \begin{bmatrix}
    x_{0} & v_{0} & a_{0} & j_{0} & x_{1} & v_{1} & a_{1} & j_{1}
  \end{bmatrix}^{T}
\end{equation*}
%
The new quadratic matrix is a block-diagonal of the segment quadratic matrices.
\begin{equation*}
  \mathbf{Q} =
  \begin{bmatrix}
    \mathbf{Q}_{0} & \mathbf{0} \\
    \mathbf{0} & \mathbf{Q}_{1}
  \end{bmatrix}
\end{equation*}
%
The new optimization function is:
\begin{equation*}
  J = \vec{x}^T \mathbf{Q} \vec{x}
\end{equation*}

For the function to be continuous, the first segment must be equal to the second
segment at the boundary point.
\begin{equation*}
  p_{0}(t=1) - p_{1}(t=0) = 0 \\
\end{equation*}
%
Written in matrix form:
\begin{equation*}
  \underbrace{
  \begin{bmatrix}
    1/0! & 1/1! & 1/2! & 1/3! & -1/0! & 0 & 0 & 0 \\
  \end{bmatrix}
  }_{\mathbf{A}}
  \underbrace{
  \begin{bmatrix}
    x_{0} \\
    v_{0} \\
    a_{0} \\
    j_{0} \\
    x_{1} \\
    v_{1} \\
    a_{1} \\
    j_{1}
  \end{bmatrix}
  }_{\vec{x}}
  =
  \vec{0}
\end{equation*}
%
Or equivalently
\begin{equation*}
  \vec{l} \leq \mathbf{A} \vec{x} \leq \vec{u}
\end{equation*}
%
where both $\vec{l}$ and $\vec{u}$ are $\vec{0}$ in this case. Continuity
constraints for arbitrary derivatives may be specified in a similar manner by
leveraging the fact that the derivative of $\vec{t}$ is a shifted version of
itself. The derivatives may be formulated in the same $\vec{l} \leq \mathbf{A}
\vec{x} \leq \vec{u}$. Without further justification, all continuity and
waypoint constraints may be formulated as bounded linear constraints.

\subsection{Path Constraints}
Path constraints are necessary for any non-trivial path. For example, quadcopters
often have input constraints and can't fly faster than or accelerate faster than
a certain limit. Alternatively, if obstacles are present, the position of the
path should be constrained to be outside of obstacles.

In coordination with the continuity constraints, path constraints can be
formulated in the form:
\begin{equation*}
  \vec{l} \leq \mathbf{A} \vec{x} \leq \vec{u}
\end{equation*}
Unfortunately, these constraints are linear --- they cannot be used to describe
non-linear constraints. For example, if a quadcopter is limited to a total
acceleration of 0.5 m/$s^2$, then the two-norm magnitude of its acceleration
should be bounded. The two-norm magnitude is a non-linear constraint and cannot
be expressed. However, one could approximate it by specifying an arbitrary number of planes
tangent-to or inscribed-in the hypersphere that delimits the two-norm. It's not
perfect, but it enables the quadratic programming solver to find a solution
fast.

Finally, path constraints introduce one additional complication: violation of
dimensional independence. Previously, I claimed that the dimensions are
independent and thus can be considered standalone as scalar polynomials. In the
presence of path constraints, this is not necessarily true. Two dimensions can
be linked together by a path constraint. For example, consider approximating a
two-norm magnitude by requiring:
\begin{align*}
  x + y &< 1 \\
  x - y &< 1 \\
  -x + y &< 1 \\
  -x - y &< 1
\end{align*}
Clearly these two dimensions are now linked. Despite this dependency, the scalar
analysis still holds. The dependency is managed by the quadratic programming
solver.

One last note: if there were no interdimensional dependence, a multi-dimensional
problem could be broken down into multiple scalar dimensional problems and
solved simultaneously by multiple solvers distributed over multiple threads.
However, it's usually the case that at least one path constraint introduces
interdimensional dependence and this technique cannot be leveraged.

\subsection{Quadratic Programming}
The problem of finding these piecewise polynomials can be cast as a quadratic
programming problem:
%
\begin{equation}
  \begin{split}
    \min_{\vec{x}} \quad& \frac{1}{2} \vec{x}^T \mathbf{Q} \vec{x}  \\
    \text{s.t.} \quad& \vec{l} \leq \mathbf{A} \vec{x} \leq \vec{u}
  \end{split}
\end{equation}
%
\href{https://osqp.org}{OSQP} is used to solve the QP problem \cite{osqp}.

\bibliography{main}
\bibliographystyle{IEEEtran}

\end{document}


%---------------------------------------------------------
%	PACKAGES AND OTHER DOCUMENT CONFIGURATIONS
%---------------------------------------------------------

\documentclass[]{amsart}

% Template-specific packages
\usepackage[utf8]{inputenc} % Required for inputting international characters
\usepackage[T1]{fontenc} % Output font encoding for international characters
\usepackage{mathpazo} % Use the Palatino font
\usepackage{graphicx} % Required for including images
\usepackage{booktabs} % Required for better horizontal rules in tables
\usepackage{listings} % Required for insertion of code
\usepackage{enumerate} % To modify the enumerate environment

%bonus essentials
%\usepackage{amsmath} % To display symbols like hookrightarrow. . . Presently unused
\usepackage{amssymb} % to display \varnothing
\usepackage{xcolor} % for \textcolor
\usepackage{soul} % for \hl
\usepackage{hyperref} % for links

%---------------------------------------------------------

\def\n{\newline \text}
\newcommand{\bt}[1]{\textbf{#1}}
\newcommand{\ans}[1]{{\huge \hl{#1}}}
\newcommand{\anstwo}[1]{{\huge #1}}
%consider also \usepackage{color} and \sethlcolor
\newcommand{\vfrac}[2]{\ensuremath{\frac{#1}{#2}}}

\setlength{\textheight}{8.5in}
\setlength{\textwidth}{6.5in}
\setlength{\oddsidemargin}{0in}
\setlength{\evensidemargin}{0in}
\voffset0.0in

%---------------------------------------------------------
\begin{document}

\centerline{\bf \Large Open-Source Software - Lab 3}
\centerline{Kevin Khaghani}
\thispagestyle{empty}

%---------------------------------------------------------
%	ASSIGNMENT CONTENT
%---------------------------------------------------------

\section*{Part 1, Question 3}

\text Use \href{http://www.artofproblemsolving.com/texer}{latex} to generate the formulae depicted in \href{https://github.com/rcos/CSCI-4961-01-Summer-2018/blob/master/Labs/latex_formulae.png}{this image}.

\medskip
\begin{enumerate}[a.]
	\item $\sqrt{1 + 2\sqrt{1 + 3\sqrt{1 + 4\sqrt{1 + . . .}}}}$

	\medskip\item $\sqrt{2}^{\sqrt{2}^{\sqrt{2}^{\sqrt{2}}}}$
\end{enumerate}

%\includegraphics[width=0.5\columnwidth]{ex1.png}

%------------------------------------------------
%\bigskip
%\begin{enumerate}[a.]
%	\item answers aren't separate this time
%\end{enumerate}
%---------------------------------------------------------

\newpage
\thispagestyle{empty}

\section*{Part 1, Question 4}

\text Use latex to display a \href{https://mathworld.wolfram.com/HadamardMatrix.html}{Hadamard Matrix} of size 4 (equation 5). You can use just 1 and -1; although, if you want to make a matrix using the white and black boxes you are welcome to.

\medskip
\begin{enumerate}[a.]
	%\item H$_{4}$ =
	%\begin{equation}
	%	\begin{bmatrix}
	%		 1 &  1 &  1 &  1 \\
	%		-1 &  1 & -1 &  1 \\
	%		-1 & -1 &  1 &  1 \\
	%		 1 & -1 & -1 &  1
	%	\end{bmatrix}
	%\end{equation}

	\item H$_{4}$ =
	$
	\begin{bmatrix}
		1 &  1 &  1 &  1 \\
		-1 &  1 & -1 &  1 \\
		-1 & -1 &  1 &  1 \\
		1 & -1 & -1 &  1
	\end{bmatrix}
	$
\end{enumerate}

%------------------------------------------------
%\bigskip
%\begin{enumerate}[a.]
%	\item answers aren't separate this time
%\end{enumerate}
%---------------------------------------------------------

\end{document}


\chapter*{Part One: Count Based Language Modeling}
\addcontentsline{toc}{chapter}{Part One}

    % \begin{lstlisting}[style=latexFrameTB, caption={Example of Sleek Template packages usage.}, gobble=8]
    %     \usepackage[english]{babel}
    %     \usepackage[noheader]{packages/sleek}
    %     \usepackage{packages/sleek-title}
    % \end{lstlisting}

\subsection*{Theoretical Problem A}
The assignment is to solve for the maximum likelihood estimate (MLE) of a bigram language model, where the likelihood function is given by

\begin{equation*}
    \prod_{t=1}^{T} p_{\theta_{u,v}}(w_t | w_{t-1})
\end{equation*}

For brevity, I will refer to  $p_{\theta_{u,v}}$ as simply \theta.

If we define our corpus to consist of many two word pairs, or bigrams, we can re-write the likelihood as

\begin{equation*}
    \prod_{t=1}^{T} \theta^{c(u,v)}
\end{equation*}

where $c(u,v)$ represents the count of how many times the bigram $u,v$ appears in the corpus.

It is convenient to rewrite our function in terms of the log likelihood:
\begin{equation*}
    \sum_{t=1}^{T} c(u,v) * \ln{\theta}
\end{equation*}

We want to maximize the log likelihood in terms subject to the constraint that the probabilities $\theta$ sum to 1:
\begin{equation*}
    \sum_{u,v\in \mathcal{V}} \theta = 1
\end{equation*}

We introduce a Lagrangian multiplier to solve the problem in the form
\begin{equation*}
    \mathcal{L}(\theta,\lambda) = L(\theta) - \lambda f(\theta)
\end{equation*},
yielding
\begin{equation*}
    \mathcal{L}(\theta,\lambda) = \sum_{t=1}^{T} c(u,v) * \ln(\theta) - \lambda (\sum_{u,v\in \mathcal{V}}\theta -1)
\end{equation*}

Taking the partial derivatives gives the following system of equations:

\begin{equation*}
    \frac{\partial}{\partial \theta} = \frac{c(u,v)}{\theta} - \lambda = 0
\end{equation*}

\begin{equation*}
    \frac{\partial}{\partial \lambda} = \sum \theta - 1 = 0
\end{equation*}

Solve the system of linear equations:

\begin{equation*}
    \theta = \frac{c(u,v)}{\lambda}
\end{equation*}

\begin{equation*}
    \sum \frac{c(u,v)}{\lambda} = 1
\end{equation*}

\begin{equation*}
    \lambda = \frac{1}{\sum c(u,v)}
\end{equation*}

The final solution to the system of equations is the MLE of the bigram model:
\begin{equation*}
    {\hat \theta_{u,v}} = \frac{c(u,v)}{\sum_{u,v\in \mathcal{V}}c(u,v)}
\end{equation*}

\subsection*{Theoretical Problem B}

The assignment is to show that for the case of a unigram language model, given a prior distribution of a Dirichlet distribution, setting all values in parameter vector $\beta$ results in a posterior distribution with mean equal to the Laplace smoothing estimate:

\begin{equation*}
    \hat{\theta}_u = \frac{c(u) + \alpha}{T + \alpha|\mathcal{V}'}
\end{equation*}

The posterior is defined as:
\begin{equation*}
    p(\pi|D;\beta) = \frac{p(D|\pi)p(\pi;\beta)}{p(D)}
\end{equation*}

The density function for the prior is:
\begin{equation*}
    p(\pi;\beta) = \frac{1}{B(\beta)}\prod_{u \in \mathcal{V}}\pi_u^{\beta_u - 1}
\end{equation*}

The likelihood model of $p(D|\pi)$ is given by:
\begin{equation*}
    \prod \pi_u^{c(u)}
\end{equation*}
where we multiply over probabilities over all words in the corpus and account for duplicates with the count term in the exponent.

Plugging the density function (ignoring the normalizing constant $B(\beta)$) and likelihood model into the posterior gives
\begin{equation*}
    p(\pi|D;\beta) = \frac{\prod \pi_u^{c(u)} * \frac{1}{B(\beta)}\prod_{u \in \mathcal{V}}\pi_u^{\beta_u - 1}}{p(D)}
\end{equation*}

\begin{equation*}
    = \frac{\prod \pi_u^{c(u) + \alpha - 1}}{p(D)}
\end{equation*}

The numerator of this equation is itself a Dirichlet distribution with parameter vector $\beta' = c(u) + \alpha -1$. Because we are changing the observed data by applying a smoothing constant, p(D) becomes trivial. Our result is proportional and the $p(D)$ term does not impact the result. Observe that the expected value of the $\beta'$ distribution is

\begin{equation*}
    \mathbb{E_{p(\pi|D;\beta')}} = \frac{c(u) + \alpha}{\sum_u^{\mathcal{V}} c(u) + \alpha}
\end{equation*}

We have increased the count of every word in our corpus by $\alpha$, meaning the size of the corpus has increased by $\alpha$ times the number of unique words. This yields

\begin{equation*}
    \mathbb{E_{p(\pi|D;\beta')}} = \frac{c(u) + \alpha}{T + \alpha|\mathcal{V}|} = \theta_u
\end{equation*}
where $T$ is the size of the original corpus.

\chapter*{Part Two: RNN Language Modeling}
\addcontentsline{toc}{chapter}{Part Two}

\subsection*{Computational Problem A}

In order to determine if a substring is part of BPL, I observe that one must know three quantities: the number of "(", the number of ")", and if an illegal ")" has ever been observed. A ")" is illegal if there is not already a matching "(".

My hidden state is a column vector with elements \begin{bmatrix}
    "(" count \\ ")" count \\ Illegal \\
\end{bmatrix}. The initial hidden state is given by
\begin{equation*}
    h_0 = \begin{bmatrix}
        -1 \\ -1 \\ 0 \\
    \end{bmatrix}
\end{equation*}

$x_t$ is a vector representing either "(" or ")", where I map \begin{bmatrix}
    1 && 0 && 0
\end{bmatrix} to  "(" and \begin{bmatrix}
    0 && 1 && 0
\end{bmatrix} to ")".

W and U are unnecessary in my BPL model, and are set to 1. The update function is given by the piecewise function

\begin{equation*}
    f (x) =
     \begin{cases}
        x & x * \begin{bmatrix}
            1 && -1 && 0
        \end{bmatrix}\geq 0 \\
        x + \begin{bmatrix}
            0 \\ 0 \\ 1 \\
        \end{bmatrix} & x * \begin{bmatrix}
            1 && -1 && 0
        \end{bmatrix}< 0 \\
     \end{cases}

\end{equation*}

If $h_t \lbrack 0 \rbrack - h_t \lbrack 1 \rbrack $ is 0 and $h_t \lbrack 2 \rbrack $ is 0 then $x_t$ is in BPL.

\subsection*{Vanilla Recursive Neural Network Implementation}

Note to the teaching assistant: I apologize for some weak analysis in the implementation sections. I finished all my code several days ago, but forgot to check the notebook into Github and lost all of my code on Monday night when Chrome crashed.

My vanilla RNN is essentially a wrapper around either an RNNCell or an LSTMCell. The forward function takes a matrix of input ids representing a batch of sentences, and maps them to an embedding. We then initialize a hidden state to zeros, and iterate over the embeddings symbol by symbol for the 50 symbol sequence. At every step the previous hidden state $h_{t-1}$ is fed back into the model, as well as the next embedding column.

The perplexity of just my basic model is shown below in Figure \label{firsttry}.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.4\textwidth]{resources/png/no_bias_1e3.png}
    \noskipcaption{Perplexity of a vanilla RNN over 10 epochs with learning rate 1e-3}
    \label{firsttry}
\end{figure}

The fact that the perplexity gets worse with more training implies that perhaps my optimizer is descending too far, and passing over the maximum for a local maxima. My perplexity rating is far away from the target somewhere in the 70s, so I decided to use a new architecture. Long Short Term Memory or LSTM incorporate a feedback mechanism in addition to the feed-forward method of an RNN. This allows the network to develop connections over longer sequences, potentially allowing for context a vanilla RNN can't achieve. My LSTM has perplexity ~78 as shown in Figure \label{secondtry}.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.4\textwidth]{resources/png/LSTM_05e3.png}
    \noskipcaption{Perplexity of a LSTM RNN over 10 epochs with learning rate 5e-4}
    \label{secondtry}
\end{figure}

\subsection*{Sequence to Sequence Model}
A sequence to sequence model interprets an input sequence through an input neural network called an encoder, and then feeds the final layer of the encoder through a decoder to produce the output sequence.

My decoder contains a Gated Recurrent Unit, which functions similar to an LSTM but it also can "forget". I chose it because some research indicates GRUs function well on smaller vocabularies ("Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling", Chung et al), although I did not have time to compare to other architectures.

Additionally I feed the output of my GRU through a linear layer. Empirically this improved my BLEU score by about 0.5, although I do not have great intuition for why.

The first attempt at a decoder with a single layer GRU and no linear layer achieved a perplexity score in the mid 40s (Figure \label{seq1}).

\begin{figure}[h]
    \centering
    \includegraphics[width=0.4\textwidth]{resources/png/seq2seq1layer.png}
    \noskipcaption{Single layer GRU Decoder perplexity}
    \label{seq1}
\end{figure}

Adding a second GRU layer and the hidden layer resulted in a perplexity in the low 30s, with a BLEU score of 6.83 (Figure \label{seq2}).

\begin{figure}[h]
    \centering
    \includegraphics[width=0.4\textwidth]{resources/png/seq.png}
    \noskipcaption{Double layer GRU with linear output Decoder perplexity}
    \label{seq1}
\end{figure}

One thing I noticed about my decoder is the output produces some seemingly unrelated predictions sometimes; the following example is taken from the training data:

    Src :  Tôi đoán các bạn đều đã biết về lịch sử của " đàn quay " trong tiếng Pháp là " <unk> à <unk> "  \hline
    Trg :  I see you 're all up on the history of hurdy-gurdy -- " <unk> <unk> <unk> . " \hline
    Pred:  I think you know what most of the most popular TEDTalks " is " <unk> . " \hline

I looked it up, and there is a sentence about TEDTalks right next to a sentence relating to hurdy-gurdies in the training data. I suspect my batch data is getting entangled. This could be fixed by duplicating some of the training data but with sentences rearranged more randomly, or by playing with the batch size parameters.

Another error is that the prediction seems to sometimes follow the most likely path, regardless of sentence context.

Src :  Tôi muốn cho các bạn biết về sự to lớn của những nỗ lực khoa học đã góp phần làm nên các dòng tít bạn thường thấy trên báo . \hline
Trg :  I 'd like to talk to you today about the scale of the scientific effort that goes into making the headlines you see in the paper . \hline
Pred:  I want to tell you about the most important thing that you 've learned in the first place of the brain that works in the front of the job . \hline

In this example it starts out alright, but I suspect addressing a second person after stating "I want to tell you" is more common than "scientific effort...". If my model had beam search, it could explore multiple paths instead of just taking the likely followup to a shorter sequence.

\chapter*{Appendix}
\subsection*{Vanilla RNN}
\begin{python}
    class RNN(nn.Module):

    def __init__(self, input_size, hidden_size, src_embed, generator, LSTM=True):
      """
      Inputs:
        - `input_size`: a positive integer corresponding to the size of the
            word embeddings
        - `hidden_size`: a positive integer representing the dimensionality of
            the RNN's hidden state vector
        - `src_embed`: an nn.Embedding object representing the lookup table for
            input (source) sentences
        - `generator`: a `Generator` object. Essentially a linear mapping
            followed by a softmax. You should not call it within this class; it
            is called in the SimpleLossCompute class above
      """
      super(RNN, self).__init__()
      # `input_size`, `hidden_size`, and `output_size` are all int.

      self.hidden_size = hidden_size
      self.src_embed = src_embed
      self.generator = generator
      self.use_lstm = LSTM
      # hint: unless you choose to implement the RNN update equations yourself
      #       you will want a `self.rnn` module that does that for you, which
      #       is where the RNNCell/LSTMCell/GRUCell modules come in handy

      self.rnn = nn.RNNCell(input_size, hidden_size, bias=False, device=device)
      self.lstm = nn.LSTMCell(input_size, hidden_size, device=device)

    def forward(self, input_ids):
      """
      Given a sequence of words (represented as IDs), compute and return the
      hidden state at each timestep (equivalently, for each input word).
      Input:
        - `input_ids`: a 2d-tensor of shape
           (batch_size, MAX_SENT_LENGTH_PLUS_SOS_EOS) representing a batch of
           sentences from the dataset (with IDs instead of words)

      Returns:
        - `hiddens`: a 3d-tensor of shape
            (batch_size, MAX_SENT_LENGTH_PLUS_SOS_EOS, hidden_size) representing
            the hidden state of the model at each timestep
      """
      # hint: pay close attention to the shapes of your tensors; you may find
      #       pytorch's `permute()` method for tensors useful

      ### Your code here!
      bsize, fsize = input_ids.size()

      hsize = self.hidden_size

      embedding = self.src_embed(input_ids)
      embedding = embedding.transpose(0,1).contiguous()

      hidden = self.init_hidden(bsize)
      output = []
      if self.use_lstm:
        cell_state = self.init_hidden(bsize)
        for i in range(fsize):
          hidden, cell_state = self.lstm(embedding[i], (hidden, cell_state))
          output.append(hidden)
      else:
        for index in range(fsize):
          hidden = self.rnn(embedding[index], hidden)
          output.append(hidden)

      output = torch.stack(output, dim=1)
      return output

    def init_hidden(self, batch_size):
      """
      Input:
        - `batch_size`: a positive integer

      Returns:
        - `hidden`: a 2d-tensor of shape (batch_size, hidden_size) representing
            the initial hidden state of the RNN
      """
      # Use to initialize hidden state everytime before running a sentence.
      hidden = torch.zeros(batch_size, self.hidden_size).to(device)
      return hidden
\end{python}

\subsection*{Seq2Seq}
\begin{python}
    class Decoder(nn.Module):
  """An RNN decoder without attention."""

  def __init__(self, input_size, hidden_size, dropout=0.):
    """
      Inputs:
        - `input_size`, `hidden_size`, and `dropout` the same as in Encoder.
    """
    super(Decoder, self).__init__()

    # hint 1: while the encoder just needed a single RNN layer, more will be
    #         required for the decoder.
    # hint 2: think about what you'll need in init_hidden(self, encoder_finals)
    # hint 3: what will you use to compute the pre_output (see docstring of
    #         forward function)?

    ### Your code here!
    print("Creating Decoder with input size {} and hidden size {}".format(input_size, hidden_size))
    self.bsize = input_size
    self.hsize = hidden_size
    self.nlayers = 2

    self.gru = nn.GRU(input_size=input_size,
                      hidden_size=hidden_size,
                      num_layers=self.nlayers,
                      batch_first=True, dropout=dropout,
                      bidirectional=False, device=device)

    self.lin = nn.Linear(input_size, hidden_size, device=device)

  def forward_step(self, prev_embed, hidden):
    """Helper function for forward below:
       Perform a single decoder step (1 word).

       Inputs:
      - `prev_embed`: a 3d-tensor of shape (batch_size, 1, embed_size)
          representing the padded embedded word vectors at this step in training
      - `hidden`: a 3d-tensor of shape (1, batch_size, hidden_size) representing
          the current hidden state.

      Returns:
      - `hidden`: a 3d-tensor of shape (1, batch_size, hidden_size)
          representing the current decoder hidden state.
      - `pre_output`: a 3d-tensor of shape (batch_size, 1, hidden_size)
          representing the total decoder output for one step
    """
    # hint: you'll want to do more here than just run self.rnn (think about
    #       what you should do to the output of the self.rnn in order to
    #       compute the `pre_output`)

    pre_output, hidden = self.gru(prev_embed, hidden)
    x = pre_output.to(device)
    pre_output = self.lin(x)
    return hidden, pre_output

  def forward(self, inputs, encoder_finals, hidden=None, max_len=None):
    """Unroll the decoder one step at a time.

    Inputs:
      - `inputs`: a 3d-tensor of shape (batch_size, max_seq_length, embed_size)
          representing a batch of padded embedded word vectors of target
          sentences (for teacher-forcing during training).
      - `encoder_finals`: a 3d-tensor of shape
          (num_enc_layers, batch_size, hidden_size) representing the final
          encoder hidden states used to initialize the initial decoder hidden
          states.
      - `hidden`: a 3d-tensor of shape (1, batch_size, hidden_size) representing
          the value to be used to initialize the initial decoder hidden states.
          If None, then use `encoder_finals`.
      - `max_len`: an int representing the maximum decoding length.

    Returns:
      - `hidden`: a 3d-tensor of shape
          (num_layers, batch_size, hidden_size) representing the final hidden
          state for each element in the batch.
      - `pre_output_vectors`: a 3d-tensor of shape
          (batch_size, max_len, hidden_size) representing the raw decoder
          outputs (before mapping to a `trg_vocab_size`-dim vector).
    """

    # The maximum number of steps to unroll the RNN.
    if max_len is None:
      max_len = inputs.size(1)

    # Initialize decoder hidden state.
    if hidden is None:
      hidden = self.init_hidden(encoder_finals)

    # hint: you'll want to keep track of the `pre_output` for each timestep,
    #       but you only need the final `hidden` state

    ### Your code here!
    bsize,fsize,hsize = inputs.size()
    pre_output_vectors = []
    for i in range(fsize):
      input = torch.unsqueeze(inputs[:,i,:], 1)
      hidden, pre_output = self.forward_step(input, hidden)
      pre_output_vectors.append(pre_output)

    pre_output_vectors = torch.stack(pre_output_vectors, dim=1).squeeze(2)
    return hidden, pre_output_vectors

  def init_hidden(self, encoder_finals):
    """Use encoder final hidden state to initialize decoder's first hidden
       state.

       Input: `encoder_finals` is same as in forward()

       Returns:
         - `decoder_init_hiddens`: a 3d-tensor of shape
              (num_layers, batch_size, hidden_size) representing the initial
              hidden state of the decoder for each element in the batch
    """
    # hint: think about whether or not an activation function is needed here

    ### Your code here!

    decoder_init_hiddens = torch.ones(self.nlayers,1,1,device=device) * encoder_finals

    return decoder_init_hiddens
\end{python}


\documentclass[11pt]{article}
\usepackage[top=1.5cm,bottom=1.5cm]{geometry}
\usepackage[T1]{fontenc}
\usepackage{url}
\usepackage{hyperref}
\usepackage{xcolor}
%
\usepackage{amsthm}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{ifthen}
\usepackage{mathpartir}
\usepackage{mathtools}
\input{macros}
%
\begin{document}

\title{CO663: Language $\lggeT$ --- double}

\date{\vspace{-7ex}}
\maketitle

% \pagenumbering{gobble}

\newcommand{\lazyeval}[1]{\framebox{\parbox[c][#1]{\textwidth}{
      \color{white}{h}%
    }}}

\section{Correct implementation}

A function that doubles its argument can be defined as follows:
%
\[
\text{double} \defi
\clam{\tynat}{x}
{
  \citer{\ez}{v}{\esucc{\esucc{\var{v}}}}{\var{x}}
}
\]

We can convince ourselves that it works as expected by applying it to
some parameter (${\esucc{\ez}}$ or 1)  and evaluate it.
\begin{enumerate}
\item Instantiate the function: \[
  \semrule
  {\tsemrulename{lav}}
  {\,}
  {\jtrans
    {\capp{\text{double}}{\esucc{\ez}}}
    {
      \citer{\ez}{v}{\esucc{\esucc{\var{v}}}}{{\esucc{\ez}}}
    }}
  \]
\item Iterate once (and using $\alpha$-renaming, for safety):
  \[
  \semrule
  {\tsemrulename{rs}}
  {\,}
  {
    \jtrans
    {
      \citer{\ez}{v}{\esucc{\esucc{\var{v}}}}{{\esucc{\ez}}}
    }
    {
      \subs
      {\esucc{\esucc{\var{v}}}}
      {
        \citer{\ez}{v'}{\esucc{\esucc{\var{v'}}}}{{{\ez}}}
      }
      {v}
    }
  }
  \]
\item Applying the substitution, we obtain
  \[
  {\esucc{\esucc{
        \citer{\ez}{v'}{\esucc{\esucc{\var{v'}}}}{{{\ez}}}
      }}}
  \]
\item Reducing one more time:
  \[
  \semrule
  {\tsemrulename{ss}}
  {
    \semrule
    {\tsemrulename{ss}}
    {
      \semrule
      {\tsemrulename{r0}}
      {\,}
      { \jtrans{{{
            \citer{\ez}{v'}{\esucc{\esucc{\var{v'}}}}{{{\ez}}}
          }}}
      {{{\ez}}}}
    }
    { \jtrans{{\esucc{
            \citer{\ez}{v'}{\esucc{\esucc{\var{v'}}}}{{{\ez}}}
          }}}
      {{\esucc{\ez}}}}
  }
  {
    \jtrans{\esucc{\esucc{
          \citer{\ez}{v'}{\esucc{\esucc{\var{v'}}}}{{{\ez}}}
        }}}
    {\esucc{\esucc{\ez}}}
  }
  \]
\end{enumerate}

\section{Wrong implementation}

If we use $\var{x}$ instead of $\var{v}$ in the inductive case, we
get:
\[
\text{baddouble} \defi
\clam{\tynat}{x}
{
  \citer{\ez}{v}{\esucc{\esucc{\var{\underline{x}}}}}{\var{{x}}}
}
\]

We can convince ourselves that it \emph{does not} work as expected by
applying it to some parameter (${\esucc{\ez}}$ or 1) and evaluate it.
%
\begin{enumerate}
\item Instantiate the function:
  %
  \[
  \semrule
  {\tsemrulename{lav}}
  {\,}
  {\jtrans
    {\capp{\text{baddouble}}{\esucc{\ez}}}
    {
      \citer{\ez}{v}{\esucc{\esucc{\esucc{\ez}}}}{{\esucc{\ez}}}
    }}
  \]
\item   Iterate once (and using $\alpha$-renaming, for safety):
  \[
  \semrule
  {\tsemrulename{rs}}
  {\,}
  {
    \jtrans
    {
      \citer{\ez}{v}{\esucc{\esucc{\esucc{\ez}}}}{{\esucc{\ez}}}
    }
    {
      \subs
      {\esucc{\esucc{\esucc{\ez}}}}
      {
        \citer{\ez}{v'}{\esucc{\esucc{\esucc{\ez}}}}{\ez}
      }
      {v}
    }
  }
  \]
\item Applying the substitution, we obtain (note that $\var{v}$ does
  not appear in the term, so there is nothing to replace).
  \[
  {\esucc{\esucc{\esucc{\ez}}}}
  \]
\item ${\esucc{\esucc{\esucc{\ez}}}}$ is a value, so we stop
  here. Notice that the final result is $3$!
\end{enumerate}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\documentclass{article}
\usepackage{eecstex}
\usepackage{pgfplots}

\title{EE 123 HW 04}
\author{Bryan Ngo}
\date{2022-02-11}

\begin{document}

\maketitle

\setcounter{section}{2}

\section{}

\subsection{}

Each circular convolution in the overlap-save method will result in a length \(2^v - P + 1\) signal.
Then, the cost of the FFT and IFFT is \(\frac{2^v}{2} \log_2(2^v) = v 2^{v - 1}\).
Then, there is a \(2^v\)-pointwise multiplication.
The total number of multiplications is \(2^v (v + 1)\).
Thus, the FFT for each sample will require
\begin{equation}
    \frac{2^v (v + 1)}{2^v - P + 1}.
\end{equation}
complex multiplications per output sample.

\subsection{}

\begin{center}
    \begin{tikzpicture}
        \begin{axis}[
            xlabel=\(v\), ylabel={Cost},
            title={Complex Multiplications},
            axis lines=middle,
            width=0.4\textwidth
        ]
        \addplot[ycomb, mark=*, color=blue] table[
            col sep=comma,
            x=v, y=cost
        ]{q3.csv};
        \end{axis}
    \end{tikzpicture}
\end{center}
with a minimum cost of \(v = 12\).
The direct evaluation would cost \(500\) complex multiplications per output sample, since that is the length of a given sample.

\subsection{}

\begin{equation}
    \lim_{v \to \infty} \frac{2^v (v + 1)}{2^v - P + 1} = \lim_{v \to \infty} \frac{v + 1}{1 - \left(\frac{P - 1}{2^v}\right)} = v
\end{equation}
Thus, for \(P = 500\), the direct method will be more efficient for \(v > 500\).

\newpage
\section{Fun with FFT}

\begin{enumerate}
    \item For \(0 \leqslant k < N\), \(H[k] = X_r[k] + W_{2N}^k X_i[k]\).
    \item For \(0 \leqslant k < N\), \(H[k + N] = X_r[k] - W_{2N}^k X_i[k]\).
    \item \(X[k] = \frac{1}{2} (H[k] + H[k + N]) + \frac{j}{2} W_{2N}^{-k} (H[k] - H[k + N])\), which takes 3 multiplications and 3 additions.
\end{enumerate}

\newpage
\section{Hadamard Transform}

\subsection{}

\begin{equation}
    H_3 =
    \begin{bmatrix}
        1 & 1 & 1 & 1 & 1 & 1 & 1 & 1 \\
        1 & -1 & 1 & -1 & 1 & -1 & 1 & -1 \\
        1 & 1 & -1 & -1 & 1 & 1 & -1 & -1 \\
        1 & -1 & -1 & 1 & 1 & -1 & -1 & 1 \\
        1 & 1 & 1 & 1 & -1 & -1 & -1 & -1 \\
        1 & -1 & 1 & -1 & -1 & 1 & -1 & 1 \\
        1 & 1 & -1 & -1 & -1 & -1 & 1 & 1 \\
        1 & -1 & -1 & 1 & -1 & 1 & 1 & -1
    \end{bmatrix}
\end{equation}
The order that represents increasing frequency content is the sequency ordering.

\subsection{}

\begin{center}
    \includegraphics[width=0.8\textwidth]{q5b.png}
\end{center}

\newpage
\section{}

\subsection{}

\begin{align}
    X[3k] &= \sum_{n = 0}^{N - 1} x[n] W_N^{3kn} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} x[n] W_N^{3kn} + \sum_{n = \frac{N}{3}}^{\frac{2N}{3} - 1} x[n] W_N^{3kn} + \sum_{n = \frac{2N}{3}}^{N - 1} x[n] W_N^{3kn} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} x[n] W_N^{3kn} + \sum_{n = 0}^{\frac{N}{3} - 1} x\left[n + \frac{N}{3}\right] W_N^{3kn} + \sum_{n = 0}^{\frac{N}{3} - 1} x\left[n + \frac{2N}{3}\right] W_N^{3kn} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} \left(x[n] + x\left[n + \frac{N}{3}\right] + x\left[n + \frac{2N}{3}\right]\right) W_N^{3kn} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} \underbrace{\left(x[n] + x\left[n + \frac{N}{3}\right] + x\left[n + \frac{2N}{3}\right]\right)}_{x_1[n]} W_{\frac{N}{3}}^{kn}
\end{align}

\subsection{}

\begin{align}
    X[3k + 1] &= \sum_{n = 0}^{N - 1} x[n] W_N^{n(3k + 1)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} x[n] W_N^{n(3k + 1)} + \sum_{n = \frac{N}{3}}^{\frac{2N}{3} - 1} x[n] W_N^{n(3k + 1)} + \sum_{n = \frac{2N}{3}}^{N - 1} x[n] W_N^{n(3k + 1)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} x[n] W_N^{n(3k + 1)} + \sum_{n = 0}^{\frac{N}{3} - 1} x\left[n + \frac{N}{3}\right] W_N^{\frac{N}{3}} W_N^{n(3k + 1)} + \sum_{n = 0}^{\frac{N}{3} - 1} x\left[n + \frac{2N}{3}\right] W_N^{\frac{2N}{3}} W_N^{n(3k + 1)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} \left(x[n] + x\left[n + \frac{N}{3}\right] W_N^{\frac{N}{3}} + x\left[n + \frac{2N}{3}\right] W_N^{\frac{2N}{3}}\right) W_N^{n(3k + 1)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} \underbrace{\left(x[n] + x\left[n + \frac{N}{3}\right] W_N^{\frac{N}{3}} + x\left[n + \frac{2N}{3}\right] W_N^{\frac{2N}{3}}\right) W_N^n}_{x_2[n]} W_{\frac{N}{3}}^{kn} \\
    X[3k + 2] &= \sum_{n = 0}^{N - 1} x[n] W_N^{n(3k + 2)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} x[n] W_N^{n(3k + 2)} + \sum_{n = \frac{N}{3}}^{\frac{2N}{3} - 1} x[n] W_N^{n(3k + 2)} + \sum_{n = \frac{2N}{3}}^{N - 1} x[n] W_N^{n(3k + 2)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} x[n] W_N^{n(3k + 2)} + \sum_{n = 0}^{\frac{N}{3} - 1} x\left[n + \frac{N}{3}\right] W_N^{\frac{2N}{3}} W_N^{n(3k + 2)} + \sum_{n = 0}^{\frac{N}{3} - 1} x\left[n + \frac{2N}{3}\right] W_N^{\frac{4N}{3}} W_N^{n(3k + 2)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} \left(x[n] + x\left[n + \frac{N}{3}\right] W_N^{\frac{2N}{3}} + x\left[n + \frac{2N}{3}\right] W_N^{\frac{4N}{3}}\right) W_N^{n(3k + 2)} \\
    &= \sum_{n = 0}^{\frac{N}{3} - 1} \underbrace{\left(x[n] + x\left[n + \frac{N}{3}\right] W_N^{\frac{N}{3}} + x\left[n + \frac{2N}{3}\right] W_N^{\frac{2N}{3}}\right) W_N^{2n}}_{x_3[n]} W_{\frac{N}{3}}^{kn}
\end{align}

\subsection{}

\begin{center}
    \includegraphics[width=0.7\textwidth]{q6c.png}
\end{center}

\subsection{}

\begin{center}
    \includegraphics[width=0.8\textwidth]{q6d.png}
\end{center}

\newpage
\section{}

\begin{enumerate}
    \item \(|X[k]| \leqslant N\) for \(k = 0\).
    \item We want \(x[n]\) to be a constant under the DFT, so we can cancel out the complex exponential terms to obtain \(x[n] = e^{j \theta} W_N^{-kn}\) for all \(\theta \in \R\), and \(k, n \in \Z\).
\end{enumerate}

\newpage
\section{}

\subsection{}

\begin{equation}
    H_4[k] = \sum_{k = 0}^3 h[n] W_4^{kn} = 1 - W_4^k = 1 - (-j)^k = \{0, 1 + j, 2, 1 - j\}
\end{equation}
\begin{center}
    \begin{tikzpicture}
        \begin{axis}[
            xlabel=\(k\), ylabel={\(|H_4[k]|\)},
            title={Magnitude of 4-point DFT},
            axis lines=middle
        ]
        \addplot[
            ycomb,
            color=blue,
            mark=*
        ]
        coordinates {
            (0, 0)
            (1, 2^0.5)
            (2, 2)
            (3, 2^0.5)
        };
        \end{axis}
    \end{tikzpicture}
\end{center}
The DFT is not even, not odd, is conjugate symmetric.
The DFT is a high-pass filter since it lets in \(\omega = \pi\), which is the highest frequency, and blocks out \(\omega = 0\), the DC frequency.

\subsection{}

It is not possible to uniquely identify \(x[n]\) since the expression
\begin{equation}
    X[k] = \frac{Y[k]}{H[k]}
\end{equation}
involves \(H[0] = 0\), so the expression is undefined at \(k = 0\), meaning that
\begin{equation}
    X[k] =
    \begin{cases}
        C & k \equiv 0 \pmod{4} \\
        \frac{Y[k]}{1 - (-j)^k} & k \in [1, 3] \pmod{4}
    \end{cases}
\end{equation}
for some \(C \in \C\).

\subsection{}

Using Parseval's theorem for the DFT,
\begin{align}
    \sum_{n = 0}^3 |x[n]|^2 &= \frac{1}{4} \sum_{k = 0}^3 |X[k]|^2 = D \\
    &= \frac{1}{4} \left(|X[0]|^2 + |X[1]|^2 + |X[2]|^2 + |X[3]|^2\right) \\
    \implies X[k] &=
    \begin{cases}
        \pm \sqrt{4D - \frac{1}{\sqrt{2}} |Y[1]|^2 - \frac{1}{2} |Y[2]|^2 - \frac{1}{\sqrt{2}} |Y[3]|^2} & k \equiv 0 \pmod{4} \\
        \frac{Y[k]}{1 - (-j)^k} & k \in [1, 3] \pmod{4}
    \end{cases}
\end{align}

\subsection{}

Using the frequency shift property of the DFT,
\begin{equation}
    \tilde{Y}[k] = X[k] \tilde{H}[(k + 1)_N]
\end{equation}
Assuming nothing else about \(x[n]\),
\begin{equation}
    X[k] =
    \begin{cases}
        \frac{Y[k]}{1 - (-j)^{k + 1}} & k \in [0, 2] \pmod{4} \\
        C & k \equiv 3 \pmod{4}
    \end{cases}
\end{equation}
for \(C \in \C\).
Assuming the sum holds from the previous part,
\begin{equation}
    X[k] =
    \begin{cases}
        \frac{Y[k]}{1 - (-j)^{k + 1}} & k \in [0, 2] \pmod{4} \\
        \pm \sqrt{4D - \frac{1}{\sqrt{2}} |Y[0]|^2 - \frac{1}{2} |Y[1]|^2 - \frac{1}{\sqrt{2}} |Y[2]|^2} & k \equiv 3 \pmod{4} \\
    \end{cases}
\end{equation}

\end{document}


\section{Error Analysis\label{sec:error}}
As with other approximation-based quadrature methods, \qbkix has two primary sources of error: the quadrature error $e_Q$ incurred as a result of evaluating potential at the check points and the extrapolation error $e_E$ due to evaluating the polynomial approximation of the potential at the target point, assuming $\Pcoarse$ is admissible. Let
%This is assumes that the approximation error of geometry and boundary data is less than $\etrg$.
\begin{align}
e_Q(\vx) &=\left|\sum_{s=0}^p (u(c_s) - \hat{u}(c_s,\Pfine) )\ell_s(t_\vx)\right|,
  \label{eq:err-quad}\\
e_E(\vx) &= \left|u(\vx) - \sum_{s=0}^p u(c_s)\ell_s(t_\vx)\right|,
  \label{eq:err-ext} \\
e_\lbl{hedgehog}(\vx) &\leq e_Q(\vx) + e_E(\vx),
  \label{eq:err-hedgehog} \\
\end{align}
where $u(\vx)$ and $\hat{u}(\vx, \Pfine)$ are defined in \cref{eq:double_layer,eq:double_layer_disc} and $\ell_s(t)$ is the $s$-th Lagrange polynomial defined on the points $\{0,1,\hdots, p\}$.
We define $t_\vx$ such that $\vx = -\vn(\vy)(R + t_\vx r)$, so $t_\vx = \frac{\|\vx - \vy\| - R}{r}$.
In this section, we first prove that we achieve high-order accuracy with our singular/near-singular evaluation scheme in \cref{sec:singular-eval} with respect to extrapolation order $p$ and quadrature order $q$.
%We then derive a heuristic inspired by the approaches taken in \cite{aT2, elliott2011estimates} to estimate the quadrature error at a point $\vx$ in the domain.
%Our error heuristic applies an estimate similar to \cite{aT2} in the principal curvature directions, specialized for our algorithmic framework.
%Using a low-order approximation of the surface closest to $\vx$, we can accurately estimate the quadrature accuracy due to local surface curvature without performing Newton iterations, which become costly in \threed.
We then detail the impact of surface approximation on overall solution accuracy.

%In this section, we describe how to ensure that $\err{Q}$ and $\err{E}$ are $O(\err{target})$.
%In practice, we actually want $ \err{hedgehog} \leq \err{target}$; we achieve this by following the discussions in \cref{sec:algo,sec:extrap_error,sec:quad_error_heuristic} with $\err{target}/2$ in place $\err{target}$.
%We will drop the factors of two for clarity of exposition.

\subsection{Quadrature error\label{sec:quad_error}}
We briefly state a tensor-product variation of known Clenshaw-Curtis quadrature error results as applied to smooth functions in \threed.
This estimate is derived based on assumptions detailed in \cref{app:proof_of_error_quad_high_order} that, in general, are difficult to verify in practice and may not hold for all functions we consider.
For this reason, we refer to it as a heuristic.

\begin{heuristic}
    Let the boundary $\Gammah$ be discretized by quadrature patches over the domains $[-h,h]$ and the boundary condition $f$ in \cref{eq:pde} be at least $C^k$.
  Apply the $q$-th order Clenshaw-Curtis quadrature rule to the double-layer potential $u(\vx)$ given in \cref{eq:double_layer_patches} and let $\vx$ be in the interior of $\Omega$.
  Then for all sufficiently large $q$:
\begin{align}
  e_\lbl{Q}(\vx) &%= \sum_{i=1}^N R^2_n\left[\frac{\partial G(\vx,P_i(u,v))}{\partial \vn}\phi(P_i(u,v)) J_{P_i}(u,v)\right]
  \lesssim  \frac{128h^{k+1}}{15\pi k(2q + 1 - k)^{k}} \tilde{V}, \\
  \shortintertext{where}
  \tilde{V} &= \max_{i=1,\hdots,N} \max_{\alpha,\beta\leq k}\left\| \frac{\partial^{\alpha+\beta}}{\partial u^\alpha\partial v^\beta}\left(\frac{\partial G(\vx,\vP_i(s,t))}{\partial \vn}\phi(\vP_i(s,t)) g_{\vP_i}(s,t)\right)\right\|_T,
  \label{eq:error_quad_gen_target}
\end{align}
 $g_\vP$ is the determinant of the metric tensor of a patch $\vP$ implicit in \cref{eq:double_layer_patches}, $\lesssim$ means "approximately less than or equal to," and $\|\zeta\|_T = \|\zeta^\prime/\sqrt{1-x^2}\|_1$.
  %\begin{equation}
  %e_Q \leq \frac{128N}{15\pi(2q + 1 - k)^{k}} \max_{i} \tilde{V}_{k_i}\left( \frac{\partial G(\vx,P_i)}{\partial \vn(\vy)}\phi(P_i) J_{P_i}\right)
  %\end{equation}
  \label{heuristic:error_quad_high_order}
\end{heuristic}

This heuristic captures the qualitative behavior of the error.
We present the derivation of \Cref{heuristic:error_quad_high_order} in \cref{app:proof_of_error_quad_high_order}.
\edit{}{The presence of the derivatives of $\phi$ in \cref{heuristic:error_quad_high_order} largely captures two troublesome cases for integral equations: (a) when the underlying singularities of $f$ are close to $\Gamma$; and (b) when pieces of non-local are nearly touching.
Both of these cases, if not properly resolved with adaptive refinement, can induce singularities in the harmonic extension of $\phi$ and cause \qbkix to accumulate error.
These sources of error are handled by \cref{criteria:2,criteria:3} in \cref{sec:geom_criteria} and the algorithms to enforce them.
}

As $\vx \to \Gammah$, the value of $k$ required in \Cref{heuristic:error_quad_high_order} grows rapidly due to growing higher order derivatives of the integrand.
Such large values of $q$ and $k$ imply that smooth quadrature rules are cost-prohibitive; this is the problem that singular/near-singular quadrature schemes like \qbkix aim to address.
This means that \cref{heuristic:error_quad_high_order} is insufficient for direct application to \cref{eq:double_layer_patches}, as we know.
Moreover, this estimate is too loose to determine whether \qbkix or smooth quadrature is required to evaluate the potential.
The assumption in \cref{sec:adaptive_upsampling} addresses this problem by providing a cheap, reasonably robust criterion for refinement that is motivated by existing analyses \cite{aT2,barnett2014evaluation} instead of relying on \Cref{heuristic:error_quad_high_order}.

\subsection{Extrapolation error\label{sec:extrap_error}}
%Before beginning a discussion of the extrapolation error of \qbkix, it is important to keep in mind the well-known relationship between polynomial interpolation of order $p$ and a local expansion of order $p$.
%For a fixed $p$ in exact arithmetic, letting the interpolation interval size tend to zero produces a Taylor expansion of order $p$ centered at the interval's origin.
A reasonable critique of \qbkix is its reliance on an equispaced polynomial interpolant to extrapolate values of $u$ to the target point.
Despite using the first-kind barycentric interpolation formula \cite{webb2012stability}, polynomial interpolation and extrapolation in equispaced points is well-known for an exponentially growing Lebesgue constant and poor stability properties as the number of points $p$ increases \cite{trefethen1991two,platte2011impossibility}.
Recently \cite{DT} demonstrated stable extrapolation in equispaced $p+1$ points
using least-sqaures polynomials of degree $\sqrt{p}$.
However, these results are asymptotic in nature and don't tell the full story for small to moderate values of $p$, as in the \qbkix context.

\begin{figure}[!htb]
      \centering
      \includegraphics[width=.45\linewidth]{figs/extrapolation_error_schematic.pdf}
  \mcaption{fig:extrap-err-setup}{Diagram of extrapolation setup}{ The toy setup used to study the extrapolation error of a singular function. We choose a simple point singularity $\mu(t) = \frac{1}{\|t - q\|}$ where $q = (\rho, 0, 0)$ (black star) with $\rho =-.1$.
  We choose samples at the points $t_i = (R+ir, 0,0)$ for $i=0,\hdots, p$ (black dots) and extrapolate the values $\mu(t_0),\hdots, \mu(t_p)$ to $t=0$ (green dot).
    }
\end{figure}

We begin our discussion with a simple representative experiment in equispaced extrapolation.
\Cref{fig:extrap-err-setup} depicts a minimal extrapolation setup in \threed of a simple singular function $\mu(t) = 1/\|t-q\|$ along a line, with $q = (\rho, 0, 0)$ and $\rho = -.1$.
We extrapolate exact values of $\mu$ from $p$ points, located at $t_i = (R + ir,0,0)$, to the origin.
This closely mimics the worse-case extrapolation error in \oned of a function analytic in a Bernstein ellipse with a real axis intercept of $\rho+R+ rp/2$.
We repeat this for a large range of values of $r$ and $R$ for various values of $p$.
The log of the relative error is plotted in \Cref{fig:extrap-err-p6,fig:extrap-err-p8,fig:extrap-err-p10,fig:extrap-err-p12,fig:extrap-err-p14} as a function of the relative extrapolation interval size $rp/R$ and the scaled extrapolation distance $R/\rho$.

As mentioned in \cite[Section 3.4]{RBZ}, the adaptive refinement of $\Pcoarse$ resolves the boundary data $f$, and therefore $u$ and $\phi$, on the length scale $L$ of the patch.
This means we can reasonably assume that the distance of the nearest singularity is $O(L)$ from $\Gammah$, i.e., $\rho = \lambda L$ for some $\lambda$.
In the context of \qbkix, we know that $R=bL(P)$ and $r=aL(P)$.
\Cref{fig:extrap-err-p6,fig:extrap-err-p8,fig:extrap-err-p10,fig:extrap-err-p12,fig:extrap-err-p14}
are a study of extrapolation error as a function of $a/b$, $b/\lambda$ and $p$.

\begin{figure}[!htb]
  \centering
  \hfill

  %\captionsetup[subfigure]{labelformat=empty}
%  \begin{subfigure}{.4\textwidth}
%      \centering
%      \includegraphics[width=\linewidth]{extrapolation_error_schematic.pdf}
%    \caption{\label{fig:extrap-err-setup}}
%  \end{subfigure}%
  \begin{subfigure}{.33\textwidth}
      \centering
      \includegraphics[width=\linewidth]{figs/extrapolation_error_plot_p6.pdf}
    \caption{\label{fig:extrap-err-p6}}
  \end{subfigure}
  \begin{subfigure}{.33\textwidth}
      \centering
      \includegraphics[width=\linewidth]{figs/extrapolation_error_plot_p8.pdf}
    \caption{\label{fig:extrap-err-p8}}
  \end{subfigure}%
    \begin{subfigure}{.33\textwidth}
      \centering
      \includegraphics[width=\linewidth]{figs/extrapolation_error_plot_p10.pdf}
    \caption{\label{fig:extrap-err-p10}}
  \end{subfigure}
  \begin{subfigure}{.33\textwidth}
      \centering
      \includegraphics[width=\linewidth]{figs/extrapolation_error_plot_p12.pdf}
    \caption{\label{fig:extrap-err-p12}}
  \end{subfigure}%
  \begin{subfigure}{.33\textwidth}
      \centering
      \includegraphics[width=\linewidth]{figs/extrapolation_error_plot_p14.pdf}
    \caption{\label{fig:extrap-err-p14}}
  \end{subfigure}%
  \mcaption{fig:extrap-experiment}{Empirical extrapolation error behavior}{We sweep over a range of $R$ and $r$ values to vary \Cref{fig:extrap-err-setup} and plot the log of the relative error in
    \Cref{fig:extrap-err-p6,fig:extrap-err-p8,fig:extrap-err-p10,fig:extrap-err-p12,fig:extrap-err-p14}, for values $p=6,8,10,12,14$, in increasing order, from (a) to (e).
In these figures, the $x$-axis is the extrapolation distance $R$ normalized by $\rho$ and the $y$-axis is the ratio $rp/R$.%: the total size of the approximation interval ($rp$) relative to the extrapolation distance $R$.
The top of the $y$-axis corresponds to $r=R$; $rp/R = 1$ corresponds to our choice of the parameter $a$.
Assuming that $\rho = O(L)$, $r/R = a/b$ and $R/\rho = b/\lambda$ for some constant $\lambda$.
}
\end{figure}
There are several important observations to make from these plots:
\begin{itemize}
  \item Extrapolation error decreases as $R/\rho$ decreases, as expected.
  \item For a fixed value of $R/\rho$, the extrapolation error \textit{decreases} rapidly as $rp$ decreases, up to a certain value $r^*p$.
This is somewhat counterintuitive, since this means placing points closer together and extrapolating a further distance relative to $rp$.
For a fixed $p$ in exact arithmetic, letting the interpolation interval size tend to zero produces an order $p$ Taylor expansion of the solution $u$ centered at the interval's origin, which accounts for this phenomenon.

%\note[MJM]{Another interpretation of this result is from the perspective of Bernstein ellipses.
%Since $u$ is $C^\infty$ in a large neighborhood around the interval of interest, we can reason that the Bernstein parameter of the corresponding \oned approximation problem is growing as we decrease $rp$, thus increasing the convergence rate with respect to $R/\rho$.}

  \item Beyond $r^*p$, the extrapolation error \textit{increases}.
    The effects of finite precision eventually pollutes the convergence behavior described above.
    Moreover, the spacing $r^*$ appears to be a function of $p$.
    For $p=6$, $r$ can be reduced to $1/p$ without any numerical issues, but
    by $p=14$, only $r>\frac{1}{2}$ is a safe choice for extrapolation.
\end{itemize}
We do not aim to rigorously analyze these phenomena in this work.
We highlight them to provide empirical evidence that equispaced extrapolation is a reasonable, but not optimal, choice for our problem of singular/near-singular integration and to provide some intuition for our parameter choices.

The following simple result describes the behavior of the extrapolation error in \cref{eq:err-ext}.
\begin{theorem}
    Let $u(\vc(t))$ be the solution to \cref{eq:pde} given by \cref{eq:double_layer}, restricted to the line $\vc(t)$ in \threed intersecting $\vx$, let $\vc(t)$ be given by
  \begin{equation}
    \vc(t) = \vsx - (R + tr)\vn(\vsx),
    \label{eq:check_point_param}
  \end{equation}
  where $\vsx$ is the closest point on $\Gammah$ to $\vx$, $R=bL_{\vsx}$, $r=aL_{\vsx}$, $\vn(\vsx)$ is the outward surface normal at $\vsx$, and let $|u^{(p)}(\vc(t))|$ be bounded above by $C_p$ on the interval $[-R, R+pr]$.
  Let $\mathfrak{P}(t)$ be the $p$-th order polynomial interpolant of $u(\vc(t))$ constructed from the check points $\vc_0, \hdots, \vc_p$, where $\vc_i = \vc(i)$. Then the extrapolation error associated with \qbkix behaves according to:
\begin{equation}
  |u(\vc(t_\vx)) - \mathfrak{P}(t_\vx)| \leq \frac{C_p}{(p+1)!}|R + rp|^p = \frac{C_p}{(p+1)!}|b+ap|^p\cdot | L|^p,
  \label{eq:extrap_err_init}
\end{equation}
where $t_\vx = \frac{\|\vx - \vsx\| - R}{r}$.
\label{thm:extrap_error}
\end{theorem}

\begin{proof}

    We know that for a smooth function $f$ and points $x_0, \hdots x_p$ in a \oned interval $I_0$, for some $\xi \in I_0$, the following relation holds for all $x \in I_0$:
\begin{equation}
    f(x) - \mathfrak{P}(x) = \frac{f^{(p)}(\xi)}{(p+1)!}\prod_{i=0}^p(x-x_i).
  \label{eq:exterp_err_init}
\end{equation}
Let $\mathfrak{P}$ be the $p$th order polynomial interpolating the points $x_0,\hdots x_p$.
In the \qbkix setup, since $R+rp$ is the distance of the furthest check point to $\vy$, we know that $x - x_i < R +rp$ for each $i$.
Since $f(t) = u(\vc(t))$ is harmonic, and therefore $C^\infty$, in $\Omega$, $|f^{(p)}(\xi)|$ can be uniformly bounded on $I_0$ by some constant $C_p$,
Noting that $R = bL$ and $r=aL$ yields our result.
\end{proof}

For fixed values of $a$ and $b$, as we let $L\to 0$, the extrapolation error is bounded by $O(L^p)$.
In practice, however, this means that we can choose $a$ and $b$ to minimize the constant factor $|b+ap|^p$ in \cref{thm:extrap_error}.
Since $p>1$, $a$ must be chosen to balance out the contribution of $p$, yet our extrapolation study shows that we can't simply set $a=0$.
We therefore choose $a \leq 1/p$ for $p=6$ and 8, motivated by \cref{fig:extrap-err-p6,fig:extrap-err-p8}.
Moreover, since $b < 1$, we can choose $a \leq b/p$, which allows $a$ and $b$ to decay at the same rate.
%In exact arithmetic, $a$ controls how close the convergence order is to $p$.
%By holding $b$ fixed and letting $a\to 0$, we recover a $p$th order Taylor expansion of $u$ centered at $c_0$.
%Convergence to an approximate Taylor series is critical to the success of \qbkix and can be seen in \cref{fig:extrap-experiment}:
%we can improve extrapolation accuracy for a fixed value of $b$ by decreasing $a$, until numerical errors take over.
The advantage of choosing $a \leq b/p$ is that $b$ is a single parameter that controls the accuracy of \qbkix.
Since we have fixed the quadrature order $q=20$ to satisfy the assumption in \cref{sec:adaptive_upsampling}, a smaller value of $b$ will trigger more upsampling in \cref{alg:adaptive_upsampling}, keeping quadrature error fixed while reducing extrapolation error.

It is important to keep in mind that \cref{thm:extrap_error} only provides insight for moderate values of $p$; our conclusions are largely irrelevant for large $p$.
We use  $p = 6$ and $a = b/6$, leaving the construction of an optimal extrapolation extrapolation scheme to future work.

\subsection{Geometry approximation error\label{sec:error_geom}}
Let $\theta$ be a \edit{}{smooth} scalar function defined on the surface of $\partial \Omega$ with $|\theta| \leq 1$ and let $\delta$ be a small real constant.
Suppose the boundary of the domain $\Omega$ is perturbed by $\delta$ along the normal field of $\partial\Omega$, scaled by $\theta$, to produce the perturbed domain $\Omega_\delta$ with boundary $\partial \Omega_\delta$.
More concretely, for $\vy \in \partial\Omega$ and $\vy_\delta\in \partial \Omega_\delta$, $\vy_\delta= \vy + \delta\theta\vn(\vy)$.
We can define the \textit{Eulerian shape derivative} of $u$ with respect to $\theta$, denoted $u_\theta$, at a point $\vx \in \Omega_\delta \cap \Omega$ as the rate of change in $u$ at $\vx$ as $\delta \rightarrow 0$.
This quantity is of interest to us because the solution to \cite[Equation 2]{morse2020robust} on $\Omega_\delta \cap \Omega$ can be written as $u + \delta u_\theta$, where $u$ is the solution to \cite[Equation 2]{morse2020robust} on $\Omega$.
Moreover, we can compute the shape derivative by solving a Laplace problem on the unperturbed domain \cite{pironneau1982optimal}:
\begin{equation}
\Delta u_\theta= 0\;\mbox{in $\Omega$, } u_\theta = -  \theta \frac{\partial u}{\partial n} \mbox{on $\partial\Omega$.}
\label{eq:shape-deriv}
\end{equation}
where $u$ is the solution of the \cite[Equation 2]{morse2020robust} on $\Omega$.
For small $\delta$, this means that the error in the solution introduced by a boundary perturbation along the field $\theta$ can be estimated by  $\delta \sup_{\Omega} \| u_\theta \|$.
Assuming the boundary is smooth and the gradient of the solution $u$ is bounded, then
\begin{equation}
\| u_\theta \| \leq C_g \sup_{\partial\Omega} \left| \theta  \frac{\partial u}{\partial n} \right| \leq
C_g \sup_{\partial\Omega} \left| \frac{\partial u}{\partial n}\right|
\label{eq:shape-deriv-bound}
\end{equation}
for some real constant $C_g$.
The right-hand side of \cref{eq:shape-deriv-bound} yields a constant $C_g^\prime$, such that if $\err{g} <  \zeta\etrg/C_g^\prime$ for some $\zeta < 1$,  the change in the solution is less than $\etrg$ for a sufficiently small $\err{g}$.
%\note[DZ]{Need to find (again) a reference saying this, as this is not true for general boundaries}
The constant depends implicitly on the surface geometry: for example, if an area element of $\partial\Omega$ is close to a sharp, concave corner, then  $\frac{\partial u}{\partial n}$ can be arbitrarily large.

\subsection{Limitations \label{sec:limitations}}
Our error discussion reveals several limitations of our method.
The first and most apparent shortcoming is that extrapolation instability fundamentally limits convergence order.
However, for reasonable orders of convergence, up to 14, we have discussed an empirical scheme to choose parameters to maximize the available convergence behavior.
Moreover, low-order surface geometries used in engineering applications will likely limit the convergence rate before it is limited by the extrapolation order, making this a non-issue in practical scenarios.

Another downside of the chosen extrapolation approach is lack of direct extension of \linebreak \qbkix to oscillatory problems like the Helmholtz equation.
Due to the limitation on the values of $p$, we can't guarantee the ability to resolve high-frequency oscillations in the solution.
A new extrapolation procedure is required to do so robustly without compromising efficiency.

In \cite{wala20193d}, the authors demonstrate a relationship between the truncation error of a \qbx expansion and the local curvature of $\Gammah$.
Our scheme also is susceptible to this form of error and we do not address nor analyze this in this work.
This is a subtle problem that requires a detailed analysis of the surface geometry with respect to the chosen extrapolation scheme.
%We leave this to a future work that produces an optimal extrapolation approach in the boundary integral context.
Another  limitation is the lack of an accurate error estimate to serve as an upsampling criteria in place of the criteria in \cref{sec:adaptive_upsampling}, such as \cite{klinteberg2019accurate}.
Extending \cite{klinteberg2019accurate} to \threed surfaces is non-trivial and whether the size of $\Pfine$ would be reduced enough to outweigh the added cost of the additional Newton iterations required by their scheme remains to be seen.

Finally, for certain accuracy targets and geometries, the algorithm above may lead to an impractically high number of patches in $\Pcoarse$ and $\Pfine$.
Geometries with nearly-touching non-local regions, as shown in \cref{fig:torii}, will see large amounts of refinement.
If the nearly-touching embeddings $\gamma_r$ are close enough, i.e., less than $10^{-10}$ apart, there is little hope of an accurate solution with a fixed computational budget.
We allow the user to enforce a minimal patch size $L_\lbl{min}$, limiting the time and memory consumption at the expense of not reaching the requested target accuracy.


\documentclass{report}

\usepackage{fullpage}

\begin{document}

\chapter{Background}

\section{Graph Theory}

An {\bf undirected graph} $G = (V, E)$ over a set of vertices $V$ and edges $E\subseteq V\times V$.
Note that $E$ should be a multisubset of $V\times V$.

\section{Group Theory}

\subsection{Permutation Groups}

Given a set $X$ of points, the {\bf symmetric group} on $X$, denoted $S_X$, is the group of all permutations on $X$.

\subsection{Coxeter Groups}

A group is a {\bf Coxeter group} if it is generated by a set of involutions.

Some examples of Coxeter twiztors include AAA, AFD, and AFG.

\subsection{Cayley Graphs}

Given a group $G$ and a set of generators $S$ for G, the {\bf Cayley graph} $C(G, S)$ is the directed graph with vertices the elements of $G$ and an edge from $g$ to $h$ if and only if $h = gs$ for some $s\in S$.
The {\bf expanded Cayley graph} is the undirected graph with edge $gh$ if and only if $g = sh$ or $h = sg$ for some generator $s\in S$.

For twiztors, the Cayley graph is not necessarily the same as the twiztor graph.

The puzzle ACG is based on the Cayley graph of $S_4$ with generators $S = \{(1 2 3), (1 4)\}$.

\chapter{Twiztor Properties}

Define a {\bf twiztor} $T = (X, C, P)$ to be a pair of a set $X$ of {\bf points}, a partition $C$ of $X$, and a set $P\subseteq S_X$ of permutation generators on $X$.

In general, we will only look at twiztors for which $X$ is finite.

\chapter{Puzzle Families}

Some subsets of twiztors in the database form interesting families.

\section{Graphs}

Every undirected graph can be converted to a Coxeter twiztor by way of an edge coloring of the graph.

If unique colors are used for every edge, then isomorphic to a sub-generator of $S_n$ with all possible inversions.

\subsection{Trees}

A ton of puzzles.
Number of non-isomorphic tree twiztors?

\subsubsection{Binary Trees}

Puzzles ADT, AFI, ACS.
Assumes edges map to distinct left/right partitions.

\section{Bipartite Graphs}
Two generators.
Includes trees.

\section{Isometric Graphs}
Coxeter twiztors.

\subsubsection{Hex Graphs}
Bunch of them.

\section{Cycle Gluing}
Twiztors $C_m\cdot C_n$ and $C_m\circ C_n$.  Can be joined as a graph.

\section{Two Generators}

Arbitrary complexity of two generators.

Twiztors with generators which are disjoint cycles of length $3$ $\Rightarrow $ even permutations.
Cayley graph is isomorphic to line graph of a cubic graph.

\end{document}