\section{Transforms}

Transforms are invertible functions that can be applied to a
random variable to change the distribution.
\subsection{Transform (Base Class)}

\subsection{Inverse Transform}

\subsection{Chain}

\subsection{Affine}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item Location $\mu \in \mathbb{R}^n$
    \item Scale $\sigma > 0$
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = \mu + \sigma \cdot x
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \frac{y - \mu}{\sigma}
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = \log \vert \sigma \vert
  \end{equation}
\end{itemize}

\subsection{Exp}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item None
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = e^x
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \log y
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = x
  \end{equation}
\end{itemize}

\subsection{Expm1}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item None
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = e^x - 1
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \log ( 1 + y )
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = x
  \end{equation}
\end{itemize}

\subsection{Gumbel}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item Location $\mu \in \mathbb{R}^n$
    \item Scale $\sigma > 0$
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = \exp \left( - \exp \left( - \frac{x - \mu}{\sigma} \right) \right)
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \mu - \sigma \cdot \log \left( - \log \left( y \right) \right)
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = - \log \left( \frac{\sigma}{-\log(y) \cdot y} \right)
  \end{equation}
\end{itemize}

\subsection{Identity}

\subsection{Kumaraswamy}

\subsection{Log}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item None
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = \log x
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \exp y
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = - y
  \end{equation}
\end{itemize}

\subsection{Logit}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item None
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = \log \left( \frac{x}{1 - x} \right)
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \frac{1}{1 + e^{-y}}
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = \log\left( 1 + e^{-y} \right) + \log\left( 1 + e^{y} \right)
  \end{equation}
\end{itemize}

\subsection{NICE}

\subsection{Planar}

\subsection{Power}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item Power $p$
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = \begin{cases} e^x & p = 0 \\ \left( 1 + x \cdot p \right) ^ {1 / p} & \text{otherwise} \end{cases}
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \begin{cases} \log y & p = 0 \\ y^{p - 1} / p & \text{otherwise} \end{cases}
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = \begin{cases} x & p = 0 \\ \left(\frac{1}{p} - 1 \right) \cdot \log \left(x \cdot p + 1 \right) & \text{otherwise} \end{cases}
  \end{equation}
\end{itemize}

\subsection{Radial}

\subsection{Reciprocal}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item None
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = 1 / x
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = 1 / y
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = -2 \cdot \log \vert x \vert
  \end{equation}
\end{itemize}

\subsection{Sigmoid}
\begin{itemize}
  \item \textbf{Parameters}
  \begin{itemize}
    \item None
  \end{itemize}
  \item \textbf{Forward}
  \begin{equation}
    f(x) = \frac{1}{1 + e^{-x}}
  \end{equation}
  \item \textbf{Inverse}
  \begin{equation}
    f^{-1}(y) = \log \left( \frac{y}{1 - y} \right)
  \end{equation}
  \item \textbf{Log Absolute Determinant Jacobian}
  \begin{equation}
    \log \vert \text{det} \, \mathbf{J} \vert (x, y) = -\log\left( 1 + e^{-x} \right) - \log\left( 1 + e^{x} \right)
  \end{equation}
\end{itemize}

\subsection{SinhArcsinh}

\subsection{Softplus}

\subsection{Softsign}

\subsection{Square}

\subsection{Tanh}

\subsection{Weibull}


%########
% Hvad er totality?
% Hvorfor totality?
% Totalitet skal bevises
%% Hvordan beviser man det?
% Hvorfor er totalitet ikke standard?
% Partiality
% Coverage
% Termination
% Produktivitet

% Finite prefix af coinduktiv data ved endeligt antal unfolds
% Partielle funktioner defineres ved subset-intuition

% Ny viden: Hvorfor totality?, Hvad består totalitet af? Produktivitet vs. terminering, total vs. partiel funktion
%########

\section{Totality and Total Functional Programming}
\label{sec:totality}
Totality is a property of functions. In the context of functional programming,
totality is thus a property of programs, assuming that our programming language
models mathematical functions.

\begin{definition}[\textit{Total function}]
\label{def:total_function}
A function $f$ with domain $A$ and codomain $B$ is total if for \emph{every}
element $x\in A$, $f$ assigns a value
$f x\in B$\,\citep{Turner04totalfunctional}.
\end{definition}

Within Definition \ref{def:total_function} is the implicit assumption that each
such assignment happens in finite time; if $f$ requires infinite time to compute
$f y$ for some input $y\in A$, then $f$ does not assign a value $f y\in B$, and is
therefore \emph{undefined} for $y$. In this case, $f$ is not total. Programming
solely with total functions has attracted more and more attention in recent
years, and has even given rise to the aptly named discipline of ``total
functional programming''. But why should we care whether the programs we write
are total?

\subsection{Partiality and Partial Functional Programming}
An appreciation of totality is perhaps best achieved by understanding its
counterpart, \emph{partiality}. Like totality, partiality is also a property of
functions.

\begin{definition}[\textit{Partial function}]
  A function $f$ with domain $A$ and codomain $B$ is partial if $f$ is undefined
  for some $x\in A$. Specifically, if $f$ is a partial function, then there
  exists a total function $g$ with domain $A'$ and codomain $B$, such that
  $A' \subset A$.
  % inhabitant $x\in A$, $g$ does not assign a value $g x\in B$. More precisely, there exists
  % an element $x\in A$ for which $g$ is undefined, i.e. either $g$
  % requires infinite time to compute $g x$, or the definition of $g$ has no
  % reduction rule for $x$.
  \centering
  \includegraphics[scale=0.8]{figures/partialfunc}
\end{definition}

For input  where a partial function is undefined, we say that
the function evaluates to $\bot$, here denoting either an exceptional state (a
run-time error) or infinite recursion\,\citep{Turner04totalfunctional}. The
implications of programming with partial functions (``partial functional
programming'') is greater than it may seem: Whenever we have no guarantee that a
function is total, $\bot$ is a possible result, which should be handled
properly. When $\bot$ signifies that program evaluation has reached an
exceptional state, an error or exception handling machinery can be set in motion
to recover from the situation, if possible. But when program evaluation reaches
an infinite recursion, meaningful recovery becomes impossible. First, due to the
undecidability of the halting problem, it is impossible to determine whether an
infinite recursion has, in fact, been reached. Secondly, it might not be
possible for the runtime system to simply stop evaluating the infinitely
recursive function and move on, since later computations could depend on the lost
output value. With this in mind, handling $\bot$ cases for all partial functions
is clearly undesirable, and simply doing nothing is not necessarily a sensible
solution. The main selling point of total functional programming is therefore
that $\bot$ is never a possible result of program evaluation.

\subsection{A Total Program is a Proof}
Another advantage of programming with total functions is that our programs can
act as proofs. The Curry-Howard isomorphism identifies a deep connection
between logic and computer
programs\,\citep{Curry1934,Howard80,Wadler2014}. Within this connection, types
are identified as propositions, (total) programs as proofs, and evaluation of
programs as simplification of proofs. Consequently, a program can be viewed as a
proof script in a logic (which, in this case, is our programming language),
showing that a given proposition, specified by the program's type, holds. This
is not merely an academic curiosity, but leads to a situation where a program
can be \emph{correct by construction}\,\citep{Pierce:2002:TPL:509043} if its
type sufficiently describes the program's behaviour. The correctness of the
program can then be established statically by the type checker.

A partial function can never be correct by construction, since evaluating it
might lead to infinite recursion. Infinite recursion is logically equivalent to
Russell's paradox (the equivalent type theoretic formulation is given by
Girard's paradox\,\citep{Girard1972}), which is known to introduce an inconsistency into an
intuitionistic logic, and therefore also into the logic that is our programming
language. Any proposition can be deduced in an inconsistent logic, and as a
consequence, a partial function can never act as a valid proof.

%  which is the general notion that
% programs are proofs and types are propositions, a well-typed program $p$ of type
% $T$ constitutes a constructive proof of the proposition $T$. The proof is
% constructed by providing an inhabitant of the output type. If a total program $p$ has a domain $A$
% and a codomain $B$, and there is an $x\in A$, $p$ can construct a proof of $B$ by
% providing an inbitant of $B$ for $p x$. The implication of
% having programs as proofs is that given a type specification which is sufficiently
% strong, we can implement programs which are \emph{correct by construction},
% i.e. statically proven correct by the compiler. Partial functions cannot act as
% proofs, since $\bot$ is a possible result. If the aforementioned $p$ is a
% partial program, no inhabitant for $B$ can be provided for input where $p$ is
% undefined.

\subsection{Ensuring Totality}
If have no guarantee that a function is total, then it must be assumed to be partial. To
obtain such a guarantee for an arbitrary function $f$, we must be able to
construct a proof showing that $f$ is total. A totality proof is twofold,
consisting of a proof of coverage and a proof that all invocations result in an
output value in finite time.

\begin{definition}[\textit{Covering function}]
  \label{def:covering_function}
  A function $f$ with domain $A$ and codomain $B$ is covering if its definition
  has a reduction rule for every inhabitant $x\in A$.
\end{definition}

Depending on whether the codomain of a covering function $f$ is an inductive or
a coinductive type, $f$ must be shown to be either \emph{terminating} or
\emph{productive} in order to ensure that all invocations of $f$ lead to a
result in finite time.

\begin{definition}[\textit{Terminating function}]
\label{def:terminating_function}
  A function $f$ with domain $A$ and codomain $B$, where $B$ is an inductive
  type, is terminating if for every inhabitant $x\in A$, the output value
  $f x\in B$ is \emph{fully} constructed in finite time.
\end{definition}

Since any inductive data has finite size by definition, a terminating function
terminates in the literal sense of the word: When the output has been
constructed, no further computations can be done for the given
input. Termination proofs are often constructed by showing that all recursive
invocations of a function happen on structurally smaller input. For
productive functions, this is may not the case.

\begin{definition}[\textit{Productive function}]
\label{def:productive_function}
  A function $f$ with domain $A$ and codomain $B$, where $B$ is a coinductive
  type, is productive if for every inhabitant $x\in A$, a finite number of
  unfoldings of the output value $f x\in B$ can be constructed in finite time.
\end{definition}

Because coinductive data is possibly infinite, a productive function continually
unfolds its output, providing each unfolding in finite time. Notably, it might
never terminate, since constructing an infinite term (e.g. a list of all prime
numbers) never comes to a natural end. Productive functions are therefore often
evaluated using a call-by-name strategy, computing exactly the number of
unfoldings needed, deferring the computation of the remaining term.  A proof
of productivity can be constructed by showing that the function definition in
question exhibits some continuity property, e.g. causality: that any output only
depends on previously computed values.

\subsection{Automated Totality Proofs}
Manually writing proofs of totality is clearly not feasible in practice, as it
must be done for every function in a program individually. Consequently,
automated construction of coverage, termination, and productivity proofs has
received much attention in the community in recent years, a development which
will be outlined in Chapter~\ref{cha:related-work}. Most techniques are either
based on a purely syntactical analysis of terms or on type checking, where the
type of a program is decorated with auxiliary information. A popular and purely
syntactical technique for automating termination proofs is to analyze whether a
program is size-change terminating\,\citep{LeeJones01SizeChange} by tracing
(mutually) recursive calls in a statically constructed call graph. Syntactic
guardedness, which will be the subject of Section~\ref{sec:synt-guard-1}, is a
purely syntactical method for approximating productivity, whereas guarded
recursion, covered in Section~\ref{sec:guarded-recursion}, is a type-based
approach to constructing productivity proofs. A full automation of guarded recursion
proofs has yet to be described.

% Equational reasoning?
% partiality and partial functions
% recursive and corecursive functions
% Turing-completeness and corecursion
% Curry-Howard

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../../copatterns-thesis"
%%% End:


\documentclass{article}

\begin{document}
\title{interest rate}
\begin{itemize}
	\item Interest(I) is the price of money. It is the fee paid for borrowing or investing money.
	\item Principal(P) is the amount of money being borrowed or deposited.
	\item Interest Rate(R) is the percentage of the principal that is paid as a fee over a period of time'
	\item Accumulated Balance(A) is the total amount to be repaid or the total value of money invested.
\end{itemize}

\paragraph{}
\begin{equation}
simple interest formula I = P*R*T
\end{equation}
\begin{equation}
Accumulated balance A = P + I
\end{equation}
\begin{equation}
A = P(1 + R*T)
\end{equation}
\end{document}


% !TEX root = thesis.tex

\chapter{Mathematical foundations of spatial modelling}
\label{ch:modelling-mathematics}

Spatial modelling has its origins in the geographical notion of space, which is in turn based on our own observations of the world and empirical experience \citep{Couclelis99}.
However, these informal notions are error-prone and differ from person to person.
In order to describe space unambiguously, people have thus turned to models that still describe geographical phenomena, but do so using formal notions derived from mathematics.
These formal models make it possible to create and store digital representations of the world in a computer, and thus to use the power of a computer to easily solve spatial problems \citep{Burrough86,Bailey95}.

The current chapter describes some of these formal notions and their relevant context, which are used to study the spatial modelling approaches presented in the upcoming chapters.
\refse{se:settheory} introduces some concepts of elementary set theory and mathematical logic, which are later used in definitions in this thesis.
\refse{se:geometry} introduces the basic concepts of geometry, which are used to describe the position, shape and orientation of objects.
\refse{se:topology} builds on these to present topology, which formalises notions such as the boundary and interior of an object or the relationships between multiple objects.

\section{Elementary set theory and mathematical logic}
\label{se:settheory}

Set theory is the branch of mathematics that studies \emph{sets}, which are collections of abstract objects.
While the study of set theory only formally started with \citet{Cantor74}, its intuitive and minimal concepts were later used in order to give a foundation to almost all areas of mathematics\footnote{Even as some mathematicians and philosophers have argued against set theory as a foundation for all of mathematics.}.
Since the basic concepts of set theory are used in this thesis in order to describe many other concepts, this section gives a very short primer using the same notation that is used in this thesis.
However, it is worth noting that the descriptions used here are reflect the concepts generally used in GIS, and so are meant to be intuitive and not very formal.
Perhaps more importantly, these definitions do not reflect modern mathematical thought on the topic\footnote{In short, these intuitive definitions pretty much assume that anything can be put into a set without leading to paradoxes, which formally is not the case.}, which is much more precise but also less accessible, \eg\ axiomatic set theory.

Set theory starts by considering the existence of a given domain of objects from which one may build sets, which is known as the \emph{universe set} and denoted as $\mathbb{U}$.
These objects can be anything, including other sets.
Set theory allows sets to be regarded as single entities and operated upon \citep{Devlin93}.
If an object $a$ is part of a set $\mathbb{X}$, it is denoted as $a \in \mathbb{X}$, read as `$a$ is an \emph{element} of $\mathbb{X}$'.
If $a$ is not part of a set $\mathbb{X}$, it is denoted as $a \notin \mathbb{X}$, read as `$a$ is not an element of $\mathbb{X}$'.

There are two broad ways to describe the elements in a set, both using curly braces, \ie\ \{ and \}.
One way to do so is to enumerate all the elements of the set one by one.
For instance, the set $\left\{ 1,2,3 \right\}$ is the set containing $1$, $2$ and $3$ as elements (and no others).
The other way to do so is to specify one or more rules that the elements of the set need to fulfil.
For instance, the set $\{ x \mid x$~\emph{is~a~prime~number}$\}$ consists of all prime numbers.
It is read as `$x$, such that $x$ is a prime number'.

Sets are by definition unordered and contain unique elements---duplicate items are ignored by convention.
A set may contain an infinite number of elements (\eg\ as the prime number example above), or no elements at all, in which case it is a special set known as the \emph{null set} and denoted as $\{ \}$ or $\emptyset$.
Other commonly used sets with a special notation and name are: the natural numbers ($\mathbb{N}$), the real numbers ($\mathbb{R}$), the rational numbers ($\mathbb{Q}$) and the integers ($\mathbb{Z}$).

In order to build more complex sets, the concepts and notation from mathematical logic are used, in particular \emph{propositional logic}.
Propositional logic works with \emph{propositions}, which are sentences that are either true or false, but not both.
These propositions might be altered and combined using various symbols expressing various notions, such as: \emph{and} ($\wedge$), \emph{or} ($\vee$), \emph{not} ($\neg$), \emph{implies} ($\Rightarrow$), \emph{is implied by} ($\Leftarrow$), \emph{if and only if} ($\Leftrightarrow$), \emph{for all} ($\forall$) and \emph{exists} ($\exists$).

Using these concepts it becomes possible to state relationships between sets.
For instance, $\mathbb{A}$ and $\mathbb{B}$ are then equal ($\mathbb{A} = \mathbb{B}$) when an element is in $\mathbb{A}$ if and only if it is also in $\mathbb{B}$, which can be denoted as $\forall x : x \in \mathbb{A} \Leftrightarrow x \in \mathbb{B}$.
A set $\mathbb{A}$ is called a subset of a set $\mathbb{B}$ ($\mathbb{A} \subseteq \mathbb{B}$), or $\mathbb{B}$ is a superset of $\mathbb{A}$ ($\mathbb{B} \supseteq \mathbb{A}$), when if an element is in $\mathbb{A}$ then it is also in $\mathbb{B}$, denoted as $\forall x : x \in \mathbb{A} \Rightarrow x \in \mathbb{B}$.
If $\mathbb{A} \subseteq \mathbb{B}$ but $\mathbb{A} \neq \mathbb{B}$, \ie\ there is at least one extra element in $\mathbb{B}$, then $\mathbb{A}$ is a proper subset of $\mathbb{B}$ ($\mathbb{A} \subset \mathbb{B}$), or alternatively $\mathbb{B}$ is a proper superset of $\mathbb{A}$ ($\mathbb{B} \supset \mathbb{A}$).

It is also possible to use propositional logic to create new sets by defining certain operations between sets, in particular \emph{Boolean set operations}, consisting of intersection, union, difference and complement\footnote{These are the most commonly described basic operations. However, it is possible to define other operations that are equally useful as a base. Either can be used to form other operations by composition.}.
The intersection of the sets $\mathbb{A}$ and $\mathbb{B}$, denoted as $\mathbb{A} \cap \mathbb{B}$, consists of all the elements that are both in $\mathbb{A}$ and in $\mathbb{B}$, \ie\ $\mathbb{A} \cap \mathbb{B} = \left\{ x \mid x \in \mathbb{A} \wedge x \in \mathbb{B} \right\}$.
The union of the sets $\mathbb{A}$ and $\mathbb{B}$, denoted as $\mathbb{A} \cup \mathbb{B}$, consists of all the elements that are either in $\mathbb{A}$ or in $\mathbb{B}$, \ie\ $\mathbb{A} \cup \mathbb{B} = \left\{ x \mid x \in \mathbb{A} \vee x \in \mathbb{B} \right\}$.
The difference between sets $\mathbb{A}$ and $\mathbb{B}$, denoted as $\mathbb{A} - \mathbb{B}$, consists of all the elements that are in $\mathbb{A}$ but not in $\mathbb{B}$, \ie\ $\mathbb{A} - \mathbb{B} = \left\{ x \mid x \in \mathbb{A} \wedge x \notin \mathbb{B} \right\}$.
The complement of a set $\mathbb{A}$, denoted as $\neg \mathbb{A}$, consists of all the elements that are in the universe set but are not in $\mathbb{A}$, \ie\ $\neg \mathbb{A} = \left\{ x \mid x \in \mathbb{U} \wedge x \notin \mathbb{A} \right\}$.

Apart from sets, it is also possible to consider \emph{tuples} of elements, which unlike sets are sequences of ordered elements.
A tuple containing exactly two elements is known as a \emph{pair}, one containing three elements is a \emph{treble} and one containing $n$ elements is an $n$-tuple.
Tuples are denoted using parenthesis, \ie\ (\ and~).

A common operation that generates tuples is the Cartesian product.
The Cartesian product of sets $\mathbb{A}$ and $\mathbb{B}$, denoted as $\mathbb{A} \times \mathbb{B}$, is defined as $\left\{ (a,b) \mid a \in \mathbb{A} \wedge b \in \mathbb{B} \right\}$.
In other words, it is a set of pairs, where the first element of a pair is an element of $\mathbb{A}$ and the second element of the pair is an element of $\mathbb{B}$.
This can be generalised to more than two sets, such that the $n$-fold Cartesian product of $n$ sets is an $n$-tuple.
The $n$-fold Cartesian product of a set $\mathbb{A}$ with itself, \ie\ $\mathbb{A} \times \mathbb{A} \times \cdots \mathbb{A}$, is denoted as $\mathbb{A}^n$.

\section{Geometry}
\label{se:geometry}

Geometry is the branch of mathematics concerned with the position of objects in space, a topic that was already formalised by the ancient Greek mathematician Euclid in his textbook \emph{the Elements} around 300 BCE \citep{Fitzpatrick08}.
Euclidean geometry consists of a small set of geometric axioms considered to be intuitively obvious, such as the fact that it is possible to draw exactly one line that passes through two points (\reffig{fig:line}), as well as a long series of postulates derived from these and which describe more complex constructions\footnote{Non-Euclidean geometry does away with some of these axioms while remaining self-consistent \citep{Bolyai32,Lobachevsky40}. However, it is much less relevant in the context of spatial modelling.}.
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/line}
\caption[A single line passes through two points]{There is exactly one line that passes through any pair of points.}
\label{fig:line}
}

However, even as Euclidean geometry has the notions of relative distances, angles and areas, objects in Euclidean geometry do not have an absolute position in space.
Analytic or Cartesian geometry, developed by \citet{Descartes37} and \citet{de-Fermat79}, significantly changed this by introducing the concept of coordinates.
A coordinate system makes it possible to uniquely describe the absolute location of a point as a tuple of real numbers.
In particular, the Cartesian coordinate system uses a tuple of perpendicular directed lines as axes, with a positive direction and a negative direction, all of which intersect at a common point known as the origin.
A point's coordinates in the system are then given by signed distances to the respective axes\footnote{A more rigorous and correct explanation would be based on a set of linearly independent vectors, but this creates a recursive definition.}.

$n$-dimensional Euclidean space, which can be described by the set of points $\mathbb{R}^n$, has $n$ perpendicular axes intersecting at the origin $O$, defined by the $n$-tuple $(0, 0, \ldots, 0)$, and a point $p$ in $n$D space is thus described by an $n$-tuple $(p_1, p_2, \ldots, p_n)$, where $p_i$ is the signed distance to the $i$-th axis.
For example, as shown in \reffig{fig:point}, three-dimensional Euclidean space ($\mathbb{R}^3$), has three axes, usually named $X$, $Y$ and $Z$, such that a given point $p$ in 3D can be described by a treble $(p_x,p_y,p_z)$, where $p_x$ is the signed distance to the $X$ axis, $p_y$ to the $Y$ axis and $p_z$ to the $Z$ axis.
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/point}
\caption{A point $p$ in 3D described by a treble $(p_x,p_y,p_z)$}
\label{fig:point}
}

A point $a = (a_1, a_2, \ldots, a_n)$ can also be used to define a vector $\vec{a}$, which goes from the origin to $a$.
% Why is            this vvvv necessary for spacing!?!?
The norm, or magnitude of\ \ $\vec{a}$, denoted as $\| \vec{a} \|$, gives the length of the line segment between $a$ and the origin and is computed as:
\begin{equation*}
\| \vec{a} \| = \sqrt{{a_1}^{2} + {a_2}^{2} + \cdots + {a_n}^{2}}
\end{equation*}

This analytic description of objects also enables using algebra to compute properties, such as the Euclidean distance between two points $a = (a_1, a_2, \ldots, a_n)$ and $b = (b_1, b_2, \ldots, b_n)$, also known as the Euclidean metric.
This is given by:
\begin{equation*}
\mathrm{distance}(a,b) = \sqrt{{(a_1 - b_1)}^{2} + {(a_2 - b_2)}^{2} + \cdots + {(a_n - b_n)}^{2}}
\end{equation*}

Some other objects can be described as a linear combination of linearly independent points (\ie\ two different points, three non-collinear points, four non-coplanar points, etc.).
Considering the points $p_1,p_2,\ldots,p_n$, a linear combination of them takes the form $a_1 p_1 + a_2 p_2 + \cdots + a_n p_n$, where $\sum_{i=1}^n a_i = 1$.
For every point $p_i$, $a_i$ is thus a scalar coefficient that determines its \emph{weight}.

If negative weights are allowed, the linear combination of $n+1$ linearly independent points forms an $n$-dimensional unbounded linear object, \eg\ a line using two points or a plane using three points.
All of these points lie exactly on the object.
When the weights are instead restricted to the interval $[0,1]$, the linear combination of $n+1$ linearly independent points forms an $n$-dimensional simplex (called an $n$-simplex)---a convex shape with $n+1$ vertices.
A \emph{0-simplex} is thus a point, a \emph{1-simplex} is a line segment, a \emph{2-simplex} is a triangle, a \emph{3-simplex} is a tetrahedron, and so on.

Other, more complex objects can be described using equations, which describe particular subsets of $\mathbb{R}^n$.
A hyperplane in $\mathbb{R}^n$, \ie\ a space of dimension $\mathbb{R}^{n-1}$ in $\mathbb{R}^n$, can be described by a linear equation of the form $a_1 x_1 + a_2 x_2 + \cdots + a_n x_n = b$, where $a_1, a_2, \ldots, a_n$ are the coefficients of the linear equation.
Apart from the points exactly on the hyperplane, as shown in \reffig{fig:halfspaces}, such a hyperplane separates $\mathbb{R}^n$ into two parts on either side of it.
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/halfspaces}
\caption{A plane separates $\mathbb{R}^3$ into two parts on either side of it.}
\label{fig:halfspaces}
}
These are known as open half-spaces and can be obtained by transforming the linear equation into the strict linear inequalities: $a_1 x_1 + a_2 x_2 + \cdots + a_n x_n < b$ for the half-space below the hyperplane and $a_1 x_1 + a_2 x_2 + \cdots + a_n x_n > b$ for the one above it.
If non-strict linear inequalities are used instead (\ie\ using $\leq$ and $\geq$ instead of $<$ and $>$), these \emph{closed half-spaces} also contain the points on the hyperplane.

Considering that a point can be described as a tuple of its coordinates, a hyperplane as a tuple of its coefficients, and similar constructions are possible for many other objects (\eg\ a sphere based on a centre point and radius), it becomes possible to have a \emph{computer representation} of these objects simply by storing tuples of numbers in a data structure\footnote{This hides the fact that using real numbers in a computer is very difficult in practice, thus floating-point approximations are generally used instead \citep{Goldberg91}.
The main consequences of this in spatial modelling are discussed in \refse{se:computerarithmetic}.}.
Moreover, it becomes possible to use them as a basis to describe other, more complex objects by using them as building blocks, either directly or using some of the topological concepts described in the next section, \eg\ them forming the boundary of another object.

Since analytic geometry allows the description of objects as sets of points in $\mathbb{R}^n$, as shown in \reffig{fig:boolean}, it is also possible to define objects based on Boolean set operations of their point sets.
\begin{figure}[hb]
\centering
\subfloat[$\mathbb{A}$ (purple) and $\mathbb{B}$ (blue)]{
\includegraphics[width=0.5\linewidth]{figs/boolean}
\label{subfig:boolean}}
% \quad
\subfloat[Intersection: $\mathbb{A} \cap \mathbb{B}$]{
\includegraphics[width=0.5\linewidth]{figs/boolean-intersection}
\label{subfig:boolean-intersection}}
\\
\subfloat[Union: $\mathbb{A} \cup \mathbb{B}$]{
\includegraphics[width=0.5\linewidth]{figs/boolean-union}
\label{subfig:boolean-union}}
% \quad
\subfloat[Difference: $\mathbb{A} - \mathbb{B}$]{
\includegraphics[width=0.5\linewidth]{figs/boolean-difference}
\label{subfig:boolean-difference}}
\caption[Objects can be defined using Boolean set operations]{Based on two balls $\mathbb{A}$ and $\mathbb{B}$, other objects that can be defined using Boolean set operations.}
\label{fig:boolean}
\end{figure}

\section{Topology}
\label{se:topology}

Topology is the mathematical study of the shape of objects, growing out of the analysis of certain problems in geometry, such as the boundaries of objects and the different possible notions of connectedness.
In particular, it studies the properties of certain objects that are preserved under so-called \emph{topological transformations} or \emph{continuous maps}, which include stretching and bending but exclude tearing or gluing.

There are two branches of topology that are most relevant in the context of spatial modelling, \emph{point-set topology} and \emph{algebraic topology}, respectively presented in \refse{ss:point-set-topology} and \refse{ss:algebraic-topology}.
Point-set topology describes space using concepts derived mainly from set theory, representing objects as continuous sets of points.
The properties of these sets and the relationships between multiple sets can then be analysed and described.
Algebraic topology adds concepts from abstract algebra as well, representing objects as structured sets of discrete elements, such as points, edges and faces.
As these elements and the relationships between them are both discrete, it is possible to use a wide variety of algorithmic methods on them, including graph theory, combinatorics, algorithmic algebra, and computational geometry and topology.

\subsection{Point-set topology}
\label{ss:point-set-topology}

Point-set topology, also known as general topology, describes objects as sets of points satisfying certain conditions, such as those in the construction in \reffig{fig:pointset}.
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/pointset}
\caption[Two rectangles and two points defined as point sets]{The rectangle $\mathbb{A}$ is represented by the set of points where $1 \leq x \leq 4$ and $1 \leq y \leq 3$. In more compact (set builder) notation, $\mathbb{A} = \left\{ (x,y) \middle| x \in [1,4] \wedge y \in [1,3] \right\}$.
For the other objects, rectangle $\mathbb{B} = \left\{ (x,y) \middle| x \in [2,3] \wedge y \in [1,2] \right\}$, point $a \in \mathbb{A}$, point $b \in \mathbb{B}$ and point $b \in \mathbb{A}$.
$\mathbb{B}$ is a subset of $\mathbb{A}$ (\ie\ $\mathbb{B} \subset \mathbb{A}$).
}
\label{fig:pointset}
}
These objects can then be analysed based on the properties of the sets that describe them, such as whether a set is bounded or unbounded, has a certain number of holes, or is orientable or unorientable.
When multiple objects are present, the relationships between their corresponding sets can be analysed as well, such as whether they are touching or overlapping, or whether it is possible to define a function that maps between these sets.

Point-set topology works with \emph{topological spaces}, a much more general notion than that of Euclidean space.
This allows the description of different types of space with different properties.
A topological space consists of a set of points and a \emph{topology} on them satisfying a series of axioms.
\citet{Edelsbrunner14} provides the following simple formulation.
Given a set of points $\mathbb{X}$, a topology of $\mathbb{X}$ is a collection of subsets, which are called open sets\footnote{Note that there are alternative definitions based on the concepts of closed sets or of neighbourhoods \citep{Hausdorff14}. For a simple definition using neighbourhoods in a GIS context see \citet[\S{}3.2.2]{Worboys04}.}, such that:
\begin{itemize}
\item $\mathbb{X}$ is open and the empty set is open;
\item the intersection of any two open sets is open;
\item the union of any family of open sets is open.
\end{itemize}

While the definition of an open set for general topological spaces is rather complex, in the context of spatial modelling we are generally interested in Euclidean space, which has a straightforward definition analogous to the concept of an open interval in 1D (\reffig{fig:intervals}).
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/intervals}
\caption[Open and closed intervals]{An open interval $\mathbb{C} = (0,3) = \left\{x \middle| 0 < x < 3 \right\}$ does not include its endpoints. By contrast, a closed interval $\mathbb{D} = [0,3] = \left\{x \middle| 0 \leq x \leq 3 \right\}$ includes its endpoints.}
\label{fig:intervals}
}
A point set $\mathbb{S}$ in Euclidean space is open if, given any point $p \in \mathbb{S}$, there exists a real number $\epsilon > 0$ such that, given any point $q$ whose Euclidean distance to $p$ is smaller than $\epsilon$, then $q \in \mathbb{S}$ as well.
A point set is closed when the point set formed by its complement is open.
Any point on an open interval fulfils these conditions, but the endpoints of a closed interval do not.
Note that it is possible for a set to be open \emph{and} closed (\eg\ an interval containing only one of its endpoints).

For example, in 2D, the plane together with the topology generated by the Euclidean metric is the topological space known as the \emph{Euclidean topology of the plane}, which can be defined based on \emph{open disks}, which are analogous to 1D open intervals.
An open disk is the set of points closer to a point $p \in \mathbb{R}^2$ than a non-zero distance $r$, such as the unit open disk shown in \reffig{fig:unitopendisk}.
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/unitopendisk}
\caption[The unit open disk]{The unit open disk, \ie\ an open disk of radius 1 centred at the origin can be defined as $\mathbb{D} = \left\{ x \in \mathbb{R}^2 \middle| \| x \| < 1 \right\}$.}
\label{fig:unitopendisk}
}
It is easy to see that as these 2D disks do not contain their boundaries, the intersection of any two open disks and the union of any number of disks are both open.

Based on the concepts of open intervals in 1D, open disks in 2D, or open balls when talking about any dimension, it is possible to partition a Euclidean space into three parts: its \emph{interior}, \emph{boundary} and \emph{exterior}.
An example of these is shown in \reffig{fig:annulus}.
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/annulus}
\caption[An annulus partitions the Euclidean plane into three parts]{An annulus with boundary partitions the Euclidean plane into three parts: its interior (yellow), its boundary (black) and its exterior (the rest of this page).
Note that none of these necessarily have to be connected.}
\label{fig:annulus}
}
The \emph{interior} of a point set $\mathbb{S}$ consists of all points where there exists an open ball centred at them such that all the points in the ball are in $\mathbb{S}$, the boundary of $\mathbb{S}$ consists of the points where all possible open balls centred at them have points in $\mathbb{S}$ and out of $\mathbb{S}$, and the exterior of $\mathbb{S}$ consists of all points where there exists an open ball centred at them such that all the points in the ball are out of $\mathbb{S}$.
The \emph{closure} of $\mathbb{S}$ is the union of its interior and its boundary.
The \emph{regularisation} of $\mathbb{S}$ is the closure of its interior and a point set is thus \emph{regular} when it is equal to its regularisation.

Once objects are defined as sets of points, point-set topology works with functions that relate these sets to each other.
A function from one point set to another is said to be \emph{continuous} if the preimage (\ie\ the inverse image) of every open set is open.
If a function is continuous and its inverse function is also continuous, it is known as a \emph{homeomorphism}.
When such a function exists between two point sets, they are said to be \emph{homeomorphic} or, more informally, \emph{topologically equivalent}, such as the two objects shown in \reffig{fig:homeomorphism}.

\begin{figure}[b]
\centering
\subfloat[A coffee mug]{\includegraphics[width=\marginparwidth]{figs/mug}} \quad
\subfloat[A donut]{\includegraphics[width=\marginparwidth]{figs/donut}}
\caption[A coffee mug and a donut are homeomorphic]{A coffee mug and a donut are homeomorphic\protect\footnotemark.
Intuitively, this can be known as it is possible to deform one into the other.
The mug was rendered from the model at \url{http://www.thingiverse.com/thing:7953}.}
\label{fig:homeomorphism}
\end{figure}
\footnotetext{Related to the joke: `A topologist is a mathematician who can't tell the difference between a coffee mug and a donut'.}

\begin{figure*}[tbp]
\centering
\includegraphics[width=\linewidth]{figs/konigsberg}
\caption[The Seven Bridges of K\"onigsberg]{The problem of the Seven Bridges of K\"onigsberg asks whether it is possible to find a route through the city that would cross each bridge (highlighted in red) exactly once. \citet{Euler41} proved that there is no such route in terms of a graph. Whenever one enters a piece of land by a bridge, one has to leave it by another bridge except at the beginning or end of the route. Thus, there must be an \emph{even} number of bridges connected to all but (at most) two pieces of land. Since all pieces of land have an \emph{odd} number of bridges, the problem has no solution. Based on an image from a 1613 engraving by Joachim Bering.}
\label{fig:konigsberg}
\end{figure*}

Another important topological concept is that of a manifold.
A manifold is a topological space that is homeomorphic to the Euclidean space of a certain dimension.
Intuitively, this means that a manifold locally resembles Euclidean space, even if globally it does not.
For example, a line and a circle are 1-manifolds, while a plane, a sphere and a torus are 2-manifolds.
Generally, when the term manifold is used in GIS it refers to a 2-manifold.

\subsection{Algebraic topology}
\label{ss:algebraic-topology}

Conceptually based on point-set topology, \emph{algebraic topology}, also known as combinatorial topology, uses concepts from abstract algebra in order to analyse topological spaces.
A famous early application involved the answer to the problem of the Seven Bridges of K\"onigsberg by \citet{Euler41}, explained in \reffig{fig:konigsberg}.
However, the real foundations of the field were set when many of its concepts were formalised in algebraic form by \citet{Poincare95}.

Algebraic topology works by relating topological spaces to groups with specific properties, often by creating combinatorial analogues of such spaces, from which their properties can be extracted using algebraic methods \citep{Henle94}, which can be applied algorithmically.
As it uses discrete structures rather than continuous point sets, it is often more suited to computer implementations of topological concepts than point-set topology \citep[\S{}3.3.5]{Worboys04}.

Two constructions of algebraic topology are widely used as the basis of GIS:\ \emph{simplicial complexes} and \emph{cell complexes}.
An $n$-dimensional simplicial complex is a structure made of connected \emph{simplices}, the simplest objects that can be built in any dimension.
As shown in \reffig{fig:simplex}, an $n$-dimensional simplex ($n$-simplex) is a combinatorial primitive made from a set of $n+1$ vertices.
\marginpar{
\captionsetup{type=figure}
\includegraphics[width=\marginparwidth]{figs/simplex}
\caption[0-, 1-, 2- and 3-simplices]{An $n$-dimensional simplex is a combinatorial primitive made from a set of $n+1$ vertices. A 0-simplex is thus a point, a 1-simplex is a line segment, a 2-simplex is a triangle, and a 3-simplex is a tetrahedron. Here they are shown as if embedded in $\mathbb{R}^3$.}
\label{fig:simplex}
}
\reffig{fig:simplicescampus} shows a group of buildings represented as a 3D simplicial complex.
\begin{figure*}[b]
\centering
\includegraphics[width=\linewidth]{figs/simplicescampus}
\caption[The TU Delft campus as a 3D simplicial complex]{The buildings in the TU Delft campus are represented as a 3D simplicial complex, such that each separate building is a set of adjacent tetrahedra. Note that only the tetrahedra's edges are shown here.}
\label{fig:simplicescampus}
\end{figure*}
A 0D simplicial complex consists of a set of discrete points (\ie\ a point cloud) and a 1D simplicial complex is a plane graph.
A 2D simplicial complex is known as a \emph{triangulation} and a 3D simplicial complex as a \emph{tetrahedralisation}.

A $j$-dimensional face ($j$-face) of an $i$-simplex, $j < i$, is a $j$-simplex made from a proper subset of its vertices.
Sometimes the dimension of the face is omitted and it can be deduced from the context, but in GIS it generally refers to each of the $i+1$ $(i-1)$-faces of an $i$-simplex.
In the context of an $i$-dimensional simplicial complex, a face refers to each of the $i$-simplices in the complex, such as the triangles in a triangular mesh.

More formally, a simplicial complex can be defined as a collection of simplices such that:
\begin{itemize}
\item
every face of a simplex is also in the simplicial complex;
\item
the intersection of any two simplices is either empty or is a common face of both of them\footnote{Note that this implies a definition where a simplex contains its boundary.}.
\end{itemize}

Based on the set of common vertices shared by two simplices, it is possible to define certain \emph{topological relationships} between them.
Two $i$-simplices are said to be adjacent if they have a common $(i-1)$-face.
An $i$-simplex and a $j$-simplex, $i \neq j$, are said to be incident if either is a face of the other.

A cell complex is a structure made of connected \emph{cells}, where an $i$-dimensional cell ($i$-cell) is an object homeomorphic to an open $i$-ball (\ie\ point, open arc, open disk and open ball).
0-cells are known as vertices, 1-cells as edges, 2-cells as faces and 3-cells as volumes.
Considering only linear geometries, 1-cells are thus line segments, 2-cells polygons and 3-cells polyhedra.
\reffig{fig:cellscampus} shows a group of buildings represented as a 3D cell complex.
\begin{figure*}[tbp]
\centering
\includegraphics[width=\linewidth]{figs/cellscampus}
\caption[The TU Delft campus as a 3D cell complex]{The buildings in the TU Delft campus are represented as a 3D cell complex. All the 2-cells of a 3-cell are shown in the same colour, the 1-cells are shown as black lines.}
\label{fig:cellscampus}
\end{figure*}

A $j$-dimensional face ($j$-face) of an $i$-cell is a $j$-cell, $j \leq i$, that lies on the boundary of the $i$-cell.
A facet of an $i$-cell is an $(i-1)$-face of the cell.
As in a simplicial complex, two $i$-cells are said to be adjacent if they have a common facet, and an $i$-cell and a $j$-cell, $i \neq j$, are said to be incident if either is a face of the other.
In the context of an $i$-dimensional cell complex, a face refers to each of the $i$-cells in the complex, such as the polygons in a polygonal mesh.

More formally, a cell complex can be defined inductively as in \citet{Hatcher02}.
An $n$-dimensional cell complex is built by starting from a set of isolated vertices, and $\forall 0 < i \leq n$ an $i$-cell is built by attaching itself to the $(i-1)$-faces (facets) on its boundary, these facets having been previously added to the complex.
That is, an edge is built by linking the vertices on its boundary, a surface by linking the edges on its boundary, a volume by linking the surfaces on its boundary, and so on.
Like in a simplicial complex, a facet of a $n$-cell in an $n$-dimensional cell complex lies between it and an adjacent $n$-cell, unless it is on the boundary of the complex.

Apart from the concepts of adjacency, incidence and other relationships between between individual simplices and cells in a complex, it is also possible to define relations and transformations between entire simplicial/cell complexes.
The \emph{Poincar\'e duality} theorem \citep{Poincare93}\footnote{It was only formulated as an observation without proof in \citet{Poincare93}. \citet{Poincare95} describes it in more detail but contains a flawed proof.
Valid proofs would have to wait until \citet{Poincare99,Poincare00}.} states that for every $n$-dimensional simplicial/cell complex, there exists a \emph{dual simplicial/cell complex} of the same dimension, where for every dimension $i$, the $i$-simplices/cells in the original complex are mapped one-to-one to $(n-i)$-simplices/cells in the dual complex.
The \emph{duality transformation} is an operation that creates the dual of a simplicial/cell complex.
This can be seen as a generalisation of the concept of a dual graph in 2D, where a graph $G$ has a dual $G^\ast$, such that vertices in $G$ correspond to faces in $G^\ast$, edges in $G$ correspond to edges in $G^\ast$ and faces in $G$ correspond to vertices in $G^\ast$.

For example, considering the Platonic solids in \reffig{fig:ps}, the tetrahedron is self-dual (\ie\ it is dual to itself), the octahedron is dual to the cube (and vice versa), and the dodecahedron is dual to the icosahedron (and vice versa).
\begin{figure}[tbp]
\centering
\subfloat[]{
\includegraphics[width=0.15\textwidth]{figs/ps_tetrahedron}
\label{subfig:ps-tetrahedron}}
\quad
\subfloat[]{
\includegraphics[width=0.15\textwidth]{figs/ps_octahedron}
\label{subfig:ps-octahedron}}
\quad
\subfloat[]{
\includegraphics[width=0.15\textwidth]{figs/ps_hexahedron}
\label{subfig:ps-hexahedron}}
\quad
\subfloat[]{
\includegraphics[width=0.15\textwidth]{figs/ps_dodecahedron}
\label{subfig:ps-dodecahedron}}
\quad
\subfloat[]{
\includegraphics[width=0.15\textwidth]{figs/ps_icosahedron}
\label{subfig:ps-icosahedron}}
\caption[The Platonic solids]{The Platonic solids are the five regular polyhedra that have regular polygonal faces: (a) tetrahedron, (b) octahedron, (c) cube or hexahedron, (d) dodecahedron, and (e) icosahedron. From Wikimedia Commons.}
\label{fig:ps}
\end{figure}
This transformation can be seen by creating a new vertex at the centre point of the face of the Platonic solid, connecting these vertices when their dual faces are adjacent.
The original vertices become faces whose number of vertices is equal to the number of originally incident faces.


\section{Convergence Analysis}

Given that the result of \hogwild, in the absence of noise generated by the
asynchronous updates of $x$, henceforth denoted {\it asynchronous noise}, is
equivalent to a stochastic gradient method, we should expect convergence rates
similar to those of stochastic gradient, should noise be small. Take for example
the typical linear least squares loss function $f(x) = \frac{1}{2n}
\norm{Dx-b}_2^2$, where $x \in \mathbb{R}^n$ and $D \in \mathbb{R}^{n \times n}$
a diagonal matrix. Writing this as a sum of each data entry:
\[
  f(x)
  = \frac{1}{2n} \sum_{k=1}^n (D_{ii}x_i - b_i)^2
  = \frac{1}{n} \sum_{k=1}^n f_i(x)
  \implies
  (\nabla f_i(x))_k
  =
  \begin{cases}
    D_{ii}(D_{ii}x_i - b_i), & k = i \\
    0, & k = 0
  \end{cases}
\]
Because our $\nabla f_i(x)$'s only have a single entry in their own component, we
can see that as long as no other thread is working on component $i$
simultaneously, no asynchronous noise will be generated; should we use \hogwild\
without replacement then this is guaranteed. In the original paper
\cite{2011NRRW}, sparsity of the vector $\nabla f_i(x)$ was required in order to
guarantee convergence, but as we'll see later, this isn't always necessary.

As a baseline of comparison, we first state a result on the convergence of the
stochastic gradient method:
\begin{theorem} \label{thm:sgd}
  (Convergence of the Stochastic Gradient Method \cite{2016BCN}) Let $F:
  \mathbb{R}^d \to \mathbb{R}$ be an objective function we're seeking to
  minimize. We can write this as either an expected risk $F(x)
  = \mathbb{E}_\xi{f(x, \xi)}$, or an empirical risk $F(x) = \frac{1}{n}
  \sum_{k=1}^n f_k(x)$.  Under the assumptions:
  \begin{enumerate}[(1)]
    \item $F$ is continuously differentiable and $\nabla F$ is Lipschitz
      continuous with Lipshitz constant $L$, i.e.
      \[
        \norm{\nabla F(x) - \nabla F(y)}
        \leq
        L\norm{x-y}, \forall x,y \in \mathbb{R}^d
      \]
    \item $F$ is strongly convex with constant $c$, i.e.
      \[
        F(x) \geq F(y) + \nabla F(y)^T (x-y) + \frac{1}{2}c\norm{x-y}^2,
        \forall x,y \in \mathbb{R}^d
      \]
    \item $F$ is bounded below over the region explored by stochastic gradient
      method.
    \item In expectation, the vector $-\nabla f(x_k, \xi_k)$ is a descent
      direction for $F$ with norm bounded by it's own norm. That is: $\exists
      \mu_G \geq \mu > 0$ such that $\forall k \in \mathbb{N}$:
      \[
        \nabla F(x_k)^T \E{\nabla f(x_k,\xi_k)} \geq \mu \norm{\nabla F(x_k)}^2
        \text{ and }
        \norm{ \E{\nabla f(x_k,\xi_k)} } \geq \mu_G \norm{\nabla F(x_k)}
      \]
    \item $\exists M, M_V \geq 0$ such that $\forall k\in \mathbb{N}$:
      \[
        \Var{\nabla f(x_k, \xi_k)} \leq M + M_V \norm{\nabla F(x_k)}^2
      \]
  \end{enumerate}
  Then, assuming a fixed stepsize $\alpha$ (a.k.a. learning rate), satisfying
  $\alpha \in (0, \mu/ LM_g]$, we have:
  \[
    \E{F(x_k) - F_*}
    \leq
    \frac{\alpha LM}{2c\mu} + (1 - \alpha c \mu)^{k-1}
    \left(
      F(x_1) - F_* - \frac{\alpha LM}{2c\mu}
    \right)
  \]
  and if we instead choose $\alpha$ diminishing, i.e. let $\alpha_k
  = \frac{\beta}{\gamma + k}$ where $\beta > 1/c\mu, \gamma > 0$ are chose such
  that $\alpha_1 \leq \mu/LM_G$, then:
  \[
    \E{F(x_k) - F_*}
    \leq
    \frac{1}{\gamma + k} \max
    \left\{
      \frac{\beta^2 LM}{2(\beta c \mu -1)},
      (\gamma + 1)(F(x) - F_*)
    \right\}
  \]
\end{theorem}
The result is technical, and very long to prove, so I just refer to the article
\cite{2016BCN} for it. Regardless, the last result in the above theorem is what
we strive for in $\hogwild$: convergence in $\mathcal{O}(1/k)$ time.

\input{./src/convergence/theory.tex}
\input{./src/convergence/numerical.tex}


\section[Simplex Algorithm of Nelder and Mead]{Simplex Algorithm of Nelder and Mead with the Extension of O'Neill}
\lab{sec:simAlgNelMea}

The Simplex algorithm of Nelder and Mead is a derivative free optimization algorithm.
It can be used to seek a solution of problem $\mathbf P_c$ defined in~\eqref{sub:Proc}
and problem $\mathbf P_{cg}$ defined in~\eqref{sub:Procg},
with constraints on the dependent parameters implemented as
described in Section~\ref{cha:conGen}.
The number of independent parameters $n$ must be larger than 1.\\

The Simplex algorithm constructs an $n$-dimensional simplex
in the space that is spanned by the independent parameters.
At each of the $(n+1)$ vertices of the simplex, the value of the cost function is evaluated. In each iteration step, the point with the highest value of the cost function is replaced by another point. The algorithm consists of three main operations: (a) \emph{point reflection}, (b) \emph{contraction of the simplex} and (c) \emph{expansion of the simplex}.\\

Despite the well known fact
that the Simplex algorithm can fail to converge to a stationary
point~\cite{Kelley1999:2,Torczon1989,Kelley1999:1,Wright1996,McKinnon1998,Lagarias1998}, both in practice and theory,
particularly if the dimension of independent variables is large,
say bigger than $10$~\cite{Torczon1989}, it is an often used algorithm.
Several improvements to the Simplex algorithm
or algorithms that were motivated by the Simplex algorithm exist,
see for example~\cite{Kelley1999:2,Torczon1989,Kelley1999:1,Tseng1999}.
However, in GenOpt, we use the original Nelder-Mead algorithm~\cite{NelderMea1965}
with the extension of O'Neill~\cite{ONeill1971}.
Optionally, the here implemented algorithm allows using
a modified stopping criteria.\\

We will now explain the different steps of the Simplex algorithm.

\subsection{Main Operations}
\begin{figure}
  \centering
  \mbox{ \subfigure[Reflection.]{\epsfig
      {file=img/nel_mea_ref.eps, bb=35 265 225 420, scale=0.9, clip=}}
    \subfigure[Expansion.]{\epsfig
      {file=img/nel_mea_exp.eps, bb=35 265 225 420, scale=0.9, clip=}} }
  \mbox{
    \subfigure[Partial inside contraction.]{\epsfig
      {file=img/nel_mea_par_ins.eps, bb=35 265 225 420, scale=0.9, clip=}}
    \subfigure[Partial outside contraction.]{\epsfig
      {file=img/nel_mea_par_out.eps, bb=35 265 225 420, scale=0.9, clip=}} }
  \mbox{
    \subfigure[Total contraction.]{\epsfig
      {file=img/nel_mea_tot.eps, bb=35 265 225 420, scale=0.9, clip=}} }
  \caption{Simplex operations.}
  \label{fig:simOpeAll}
\end{figure}
The notation defined below is used in describing the main operations. The operations are illustrated in Fig.~\ref{fig:simOpeAll} where for simplicity
a two-dimensional simplex is illustrated.\\

We now introduce some notation and definitions.
\begin{subequations}
\begin{enumerate}
\item We will denote by $\mathbf{I} \triangleq \{1, \, \ldots \, , \, n+1\}$
the set of all vertex indices.
\item
We will denote by $l \in \mathbf I$ the smallest index in $\mathbf I$
such that
\begin{equation}
l = \argmin_{i \in \mathbf{I} }  f(x_i).
\label{eq:simIndL}
\end{equation}
Hence, $f(x_l) \le f(x_i)$, for all $i \in \mathbf I$.
\item
We will denote by $h \in \mathbf I$ the smallest index in $\mathbf I$
such that
\begin{equation}
h = \argmax_{i \in \mathbf{I} }  f(x_i).
\label{eq:simIndH}
\end{equation}
Hence, $f(x_h) \ge f(x_i)$, for all $i \in \mathbf I$.
\item
Let $x_i$, for $i \in \mathbf I$, denote the simplex vertices, and let $h$ be as in~\eqref{eq:simIndH}.
We will denote by $x_c \in \Re^n$ the {\em centroid} of the simplex, defined as
\begin{equation}
x_c \triangleq \frac{1}{n} \,
\mathop{\sum_{i=1}}_{i \neq h}^{n+1} x_i
\label{eq:simCenDef}
\end{equation}
\end{enumerate}
\end{subequations}

Next, we introduce the three main operations.
\begin{subequations}
\begin{description}
\item[Reflection]
Let $h \in \mathbf I$ be as in~\eqref{eq:simIndH} and let
$x_c$ be as in~\eqref{eq:simCenDef}.
The reflection of $x_h \in \Re^n$ to a point denoted as
$x^* \in \Re^n$ is defined as
\begin{equation}
   x^* \triangleq (1+\alpha) \, x_c - \alpha \, x_h,
\label{eq:simRefOpe}
\end{equation}
where $\alpha \in \Re$, with $\alpha > 0$, is
called the \emph{reflection coefficient}.

\item[Expansion of the simplex]
Let $x^* \in \Re^n$ be as in~\eqref{eq:simRefOpe} and
$x_c$ be as in~\eqref{eq:simCenDef}.
The expansion of $x^* \in \Re^n$ to a point denoted as
$x^{**} \in \Re^n$ is defined as
\begin{equation}
   x^{**} \triangleq \gamma \, x^* + (1-\gamma) \, x_c,
\label{eq:simExp}
\end{equation}
where $\gamma \in \Re$, with $\gamma > 1$, is
called the \emph{expansion coefficient}.

\item[Contraction of the simplex]
Let $h \in \mathbf I$ be as in~\eqref{eq:simIndH} and
$x_c$ be as in~\eqref{eq:simCenDef}.
The contraction of $x_h \in \Re^n$ to a point denoted as
$x^{**} \in \Re^n$ is defined as
\begin{equation}
   x^{**} \triangleq \beta \, x_h + (1 - \beta) \, x_c,
  \label{eq:simAlgCon}
\end{equation}
where $\beta \in \Re$, with $0 < \beta < 1$, is
called the \emph{contraction coefficient}.
\end{description}
\end{subequations}

% ---------------------------------
\subsection{Basic Algorithm}
In this section, we describe the basic Nelder and Mead algorithm~\cite{NelderMea1965}.
The extension of O'Neill and the modified restart criterion are discussed later.
The algorithm is as follows:
\begin{enumerate}

\item Initialization:
Given an initial iterate $x_1 \in \Re^n$,
a scalar $c$, with $c=1$ in the initialization,
a vector $s \in \Re^n$ with user-specified step sizes for each independent parameter,
and the set of unit coordinate vectors $\{ e_i \}_{i=1}^n$,
construct an initial simplex with vertices,
for $i \in \{1, \ldots , n \}$,
\begin{equation}
  x_{i+1} = x_1 + c \, s^i \, e_i.
\label{eq:simAlgIni}
\end{equation}
Compute $f(x_i)$, for $i \in \mathbf I$.
\item \label{des:simAlgRef} Reflection:
Reflect the worst point, that is, compute $x^*$ as in~\eqref{eq:simRefOpe}.
\item \label{des:simAlgCheBes} Test whether we got the best point:
If $f(x^*) < f(x_l)$, expand the simplex using~\eqref{eq:simExp}
since further improvement in this direction is likely.
If $f(x^{**})< f(x_l)$, then
$x_h$ is replaced by $x^{**}$,
otherwise $x_h$ is replaced by $x^*$, and
the procedure is restarted from \ref{des:simAlgRef}.

\item
If it turned out under \ref{des:simAlgCheBes} that $f(x^*) \ge f(x_l)$,
then we check if the new point $x^*$ is the worst of all points:
If $f(x^*) > f(x_i)$, for all $i \in \mathbf I$, with $i \ne h$, we contract
the simplex (see \ref{des:simAlgCon});
otherwise we replace $x_h$ by $x^*$ and
go to \ref{des:simAlgRef}.

\item \label{des:simAlgCon}
For the contraction, we first check if we should try a partial outside
contraction or a partial inside contraction: If $f(x^*) \ge f(x_h)$,
then we try a partial inside contraction.
To do so, we leave our indices as is and
apply (\ref{eq:simAlgCon}).
Otherwise, we try a partial outside contraction.
This is done
by replacing $x_h$ by $x^*$ and applying (\ref{eq:simAlgCon}).
After the partial inside or the partial outside contraction,
we continue at \ref{des:simAlgCheWor}.

\item \label{des:simAlgCheWor}
If $f(x^{**}) \ge f(x_h)$\footnote{ Nelder and Mead~\cite{NelderMea1965} use the strict
inequality $f(x^{**}) > f(x_h)$.
However, if the user writes the cost function value with only a few
representative digits to a text file,
then the function looks like a step function if slow convergence is
achieved. In such cases, $f(x^{**})$ might sometimes be equal to $f(x_h)$.
Experimentally, it has been shown
advantageous to perform a total contraction rather than continuing with a reflection.
Therefore, the strict inequality has been changed to a weak inequality.},
we do a total contraction of the
simplex by replacing $x_i \leftarrow (x_i + x_l)/2$, for all $i \in \mathbf I$.
Otherwise, we replace $x_h$ by $x^{**}$.
In both cases, we continue from \ref{des:simAlgRef}.
\end{enumerate}

\begin{figure}
\centering
\epsfig{file=img/nel_mea_mov_tra.eps, bb=98 47 640 400, width=\headwidth, clip=}
\caption{Sequence of iterates generated by the Simplex algorithm.}
\label{fig:simSeqAlg}
\end{figure}

Fig. \ref{fig:simSeqAlg} shows a contour plot of a cost function $f \colon \Re^n \to \Re$ with a sequence of iterates
generated by the Simplex algorithm.
The sequence starts with
constructing an initial simplex $x_1$, $x_2$, $x_3$. $x_1$ has the highest function value and is therefore
reflected, which generates $x_4$. $x_4$ is the best point in the set $\{x_1, x_2, x_3, x_4\}$. Thus, it is further
expanded, which generates $x_5$. $x_2$, $x_3$ and $x_5$ now span the new simplex. In this simplex, $x_3$ is the
vertex with the highest function value and hence goes over to $x_6$ and further to $x_7$. The process of reflection
and expansion is continued again two times, which leads to the simplex spanned by $x_7$, $x_9$ and
$x_{11}$. $x_7$ goes over to $x_{12}$ which turns out to be the worst point. Hence, we do a partial inside
contraction, which generates $x_{13}$. $x_{13}$ is better than $x_7$ so we use the simplex spanned by $x_9$,
$x_{11}$ and $x_{13}$ for the next reflection.
The last steps of the optimization are for clarity not shown.

% ---------------------------------

\subsection{Stopping Criteria}
The first criterion is a test of the variance of the function values at the vertices of the simplex
\begin{equation}
\frac{1}{n} \, \left(
\sum_{i=1}^{n+1} \bigl( f(x_i) \bigr)^2 -
   \frac{1}{n+1} \, \left(\sum_{i=1}^{n+1} f(x_i)    \right)^2 \,
\right) < \epsilon^2,
 \label{eq:neaMeaVar}
\end{equation}
then the original implementation of the algorithm stops.
Nelder and Mead have chosen this stopping criterion based on the statistical problem
of finding the minimum of a sum of squares surface.
In this problem, the curvature near the minimum yields information about
the unknown parameters.
A slight curvature indicates a high sampling variance of the estimate.
Nelder and Mead argue that in such cases, there is no reason for finding
the minimum point with high accuracy.
However, if the curvature is marked,
then the sampling variance is low and a higher accuracy in determining the optimal parameter set is desirable.

Note that the stopping criterion~\eqref{eq:neaMeaVar} requires the variance
of the function values at the simplex vertices to be smaller
than a prescribed limit.
However, if $f(\cdot)$ has large discontinuities, which has been observed
in building energy optimization problems~\cite{WetterWright2003:1},
then the test~\eqref{eq:neaMeaVar} may never be satisfied.
For this reason, among others,
we do not recommend using this algorithm if the cost function
has large discontinuities.

\pagebreak[4]
% -------------------
\subsection{O'Neill's Modification}
O'Neill modified the termination criterion by adding a further condition~\cite{ONeill1971}. He checks
whether any orthogonal step, each starting from the best vertex of the current simplex, leads to a further
improvement of the cost function.
He therefore sets $c = 0.001$ and tests if
\begin{subequations}
\label{eq:optCheNelMeaOri}
\begin{equation}
  f(x_l) < f(x)
\label{eq:optCheNelMeaOriCon}
\end{equation}
for all $x$ defined by
\begin{equation}
   x \triangleq x_l + c \, s^i \, e_i, \qquad i \in \{1, \ldots , n \},
  \label{eq:optCheNelMeaOriCoo}
\end{equation}
where $x_l$ denotes the best known point,
and $s^i$ and $e_i$ are as in \eqref{eq:simAlgIni}.\\

% ----------------------------
\subsection{Modification of Stopping Criteria}

In GenOpt, (\ref{eq:optCheNelMeaOri}) has been modified. It has been observed that users sometimes
write the cost function value with only few representative digits to the output file.
In such cases,
(\ref{eq:optCheNelMeaOriCon}) is not satisfied if the write statement
in the simulation program truncates
digits so that the difference $f(x_l)-f(x)$, where $f\depd$ denotes the value that is
read from the simulation output file, is zero.
To overcome this numerical problem,
(\ref{eq:optCheNelMeaOriCoo}) has been modified to
\begin{equation}
   x = x_l + \exp(j) \, c \, s^i \, e_i, \qquad i \in \{1, \ldots , n \}
\label{eq:optCheNelMeaOriMod}
\end{equation}
\end{subequations}
where for each direction $i \in \{1, \ldots , n \}$,
the counter $j \in \Na$ is set to zero for the first trial and increased by one as long as
$f(x_l) = f(x)$.

If (\ref{eq:optCheNelMeaOriCon}) fails for any direction, then $x$ computed by
(\ref{eq:optCheNelMeaOriMod}) is the new starting point and a new simplex with side lengths $(c\, s^i)$, $i \in \{1, \ldots, n\}$, is constructed.
The point $x$ that failed (\ref{eq:optCheNelMeaOriCon}) is then used as the initial point $x_l$ in (\ref{eq:simAlgIni}).\\

\begin{figure}
  \mbox{ \subfigure[Sequence of iterates in the neighborhood of the minimum.]{
  \epsfig{file=img/nel_mea_res_tra.eps, bb=25 290 360 575, clip=, width=0.5\headwidth} \label{fig:neaMeaResTra} } \quad
\subfigure[2-dimensional test function ``2D1''.]{
  \epsfig{file=img/fun_f2d1.eps, bb=120 95 740 500, width=0.5\headwidth}
\label{fig:neaMeaTesFun} } }
\caption{Nelder Mead trajectory.}
\end{figure}
Numerical experiments showed that during slow convergence the algorithm was restarted too frequently.

Fig.~\ref{fig:neaMeaResTra} shows a sequence of iterates where the algorithm was restarted too frequently.
The iterates in the figure are part of the iteration sequence near the
minimum of the test function shown in Fig.~\ref{fig:neaMeaTesFun}.
The algorithm gets close to the minimum with appropriately large steps.
The last of these steps can be seen at the right of the figure.
After this step, the stopping criterion (\ref{eq:neaMeaVar}) was satisfied
which led to a restart check, followed by a new construction of the simplex. From there on, the
convergence was very slow due to the small step size. After each step, the stopping criterion was satisfied
again which led to a new test of the optimality condition (\ref{eq:optCheNelMeaOriCon}), followed by a
reconstruction of the simplex. This check is very costly in terms of function evaluations and, furthermore,
the restart with a new simplex does not allow increasing the step size, though we are heading locally in
the right direction.\\

O'Neill's modification prevents both excessive checking of the optimality condition as well as excessive
reconstruction of the initial simplex. This is done by checking for convergence only after a predetermined
number of steps (e.g., after five iterations).
However, the performance of the algorithm depends
strongly on this number. As an extreme case, a few test runs were done where convergence was checked
after each step as in Fig. \ref{fig:neaMeaResTra}. It turned out that in some cases no convergence was
reached within a moderate number of function evaluations if $\epsilon$ in (\ref{eq:neaMeaVar}) is chosen
too large, e.g., $\epsilon = 10^{-3}$ (see Tab. \ref{tab:nelMeaBenMarRes}).\\

\pagebreak[2]
To make the algorithm more robust, it is modified based on the following arguments:
\begin{enumerate}
\item
If the simplex is moving in the same direction in the last two steps, then the search is not
interrupted by checking for optimality since we are making steady progress in the moving direction.
\item
If we do \emph{not} have a partial inside or total contraction immediately beyond us, then it is likely
that the minimum lies in the direction currently being explored.
Hence, we do not interrupt the search with a restart.
\end{enumerate}

These considerations have led to two criteria that both have to be satisfied to permit the convergence check according to (\ref{eq:neaMeaVar}), which might be followed by a check for optimality.\\

First, it is checked if we have done a partial inside contraction or a total contraction. If so, we
check if the direction of the latest two steps in which the simplex is moving has changed by an angle of at
least $(\pi / 2)$.
To do so, we introduce the center of the simplex, defined by
\begin{equation}
  x_m \triangleq \frac{1}{n+1} \, \sum_{i=1}^{n+1} x_i,
\end{equation}
where $x_i$, $i \in \{ 1, \ldots, n\}$, are the simplex vertices.
We also introduce the normalized direction of the simplex between two steps,
\begin{equation}
  d_k \triangleq \frac{ x_{m, k} - x_{m, k-1} }
{ \| x_{m, k} - x_{m, k-1} \| },
\end{equation}
where $k \in \Na$ is the current iteration number.

We determine how much the simplex has changed its direction $d_k$ between two steps by
computing the inner product $\langle d_{k-1}, d_k \rangle$.
The inner product is equal to the cosine of the angle $d_{k-1}$ and $d_k$.
If
\begin{equation}
  \cos \phi_k = \langle d_{k-1}, \, d_k \rangle \le 0,
  \label{eq:nelMeaCosMov}
\end{equation}
then the moving direction of the simplex has changed by at least $\pi / 2$.
Hence, the simplex has changed the exploration direction.
Therefore, a minimum might be achieved and we need to test the variance of the
vertices (\ref{eq:neaMeaVar}), possibly followed by a test of (\ref{eq:optCheNelMeaOriCon}).\\

%---
Besides the above modification, a further modification was tested:
In some cases, a reconstruction of the simplex after a failed check (\ref{eq:optCheNelMeaOriCon})
yields to slow convergence.
Therefore, the algorithm was modified so that it continues at
point \ref{des:simAlgRef} on page~\pageref{des:simAlgRef} without reconstructing the simplex after
failing the test (\ref{eq:optCheNelMeaOriCon}).
However, reconstructing the simplex led in most of the benchmark tests to faster convergence. Therefore,
this modification is no longer used in the algorithm.

\subsection{Benchmark Tests}
\label{sec:algSimBenTes}

\begin{table}
\newcolumntype{C}{>{\centering\arraybackslash}X}
\begin{tabularx}{\headwidth}{|p{2cm}|C|C|C|C|C|C|C|C|}
\hline
 & \multicolumn{8}{|c|}{Accuracy} \\ \cline{2-9}
& \multicolumn{4}{|c|}{$\epsilon = 10^{-3}$} & \multicolumn{4}{|c|}{$\epsilon = 10^{-5}$} \\ \hline
\raggedright Test function & Rosen-brock & 2D1 & Quad with I matrix & Quad with Q matrix & Rosen-brock & 2D1 & Quad with I matrix & Quad with Q matrix \\ \hline
\raggedright Original, with reconstruction &  137  &  120  &  3061  &  1075  &  139  &  109  &  1066  &  1165 \\ \hline
\raggedright Original, no reconstruction &  136  &  110  &  1436  &  1356  &  139  &  109  &  1433  &  1253 \\ \hline
\raggedright Modified, with reconstruction &  145  &  112  &  1296  &  1015  &  152  &  111  &  1060  &  1185 \\ \hline
\raggedright Modified, no reconstruction &  155  &  120  &  1371  &  1347  &  152  &  109  &  1359  &  1312 \\ \hline
\end{tabularx}
\caption{Comparison of the number of function evaluations for different implementations of the simplex algorithm. See Appendix for the definition of the function.}
\label{tab:nelMeaBenMarRes}
\end{table}

\begin{figure}
  \centering
  \epsfig{file=img/nel_mea_ben.eps, bb=60 255 420 520, clip=}
  \caption{Comparison of the benchmark tests.}
  \label{fig:nelMeaBenMarRes}
\end{figure}

Tab.~\ref{tab:nelMeaBenMarRes} shows the number of function evaluations and
Fig.~\ref{fig:nelMeaBenMarRes} shows the relative number of function evaluations compared to the original
implementation for several test cases. The different functions and the parameter settings are given in the
Appendix.
The only numerical parameter that was changed for the different optimizations is the accuracy, $\epsilon$.\\

It turned out that modifying the stopping criterion is effective in most cases,
particularly if a new simplex is constructed after the check~\eqref{eq:optCheNelMeaOriCon} failed.
Therefore, the following two versions of the simplex algorithm are implemented in GenOpt:
\begin{enumerate}
\item
The base algorithm of Nelder and Mead, including the extension of O'Neill.
After failing~\eqref{eq:optCheNelMeaOriCon},
the simplex is \emph{always} reconstructed with the new step size.
\item
The base algorithm of Nelder and Mead, including the extension of O'Neill,
but with the modified stopping criterion as explained above.
That is, the simplex is only reconstructed if its moving direction changed,
and if we have an inside or total construction beyond us.
\end{enumerate}

\subsection{Keywords}
For the Simplex algorithm, the command file (see page~\pageref{par:comFil}) must only contain continuous parameters.\\

To invoke the Simplex algorithm, the \texttt{Algorithm} section of the GenOpt command file must
have following form:
\begin{lstlisting}
Algorithm{
   Main                    = NelderMeadONeill;
   Accuracy                = Double;   // 0 <  Accuracy
   StepSizeFactor          = Double;   // 0 <  StepSizeFactor
   BlockRestartCheck       = Integer;  // 0 <= BlockRestartCheck
   ModifyStoppingCriterion = Boolean;
}
\end{lstlisting}

\noindent The key words have following meaning:
\begin{codedescription}
\item[Main]
   The name of the main algorithm.
\item[Accuracy]
The accuracy that has to be reached before the optimality condition is checked. \texttt{Accuracy} is
defined as equal to $\epsilon$ of (\ref{eq:neaMeaVar}), page~\pageref{eq:neaMeaVar}.

\item[StepSizeFactor]
A factor that multiplies the step size of each parameter for
(a) testing the optimality condition and
(b) reconstructing the simplex.
\texttt{StepSizeFactor} is equal to $c$ in
(\ref{eq:simAlgIni}) and (\ref{eq:optCheNelMeaOriMod}).

\item[BlockRestartCheck]
Number that indicates for how many main iterations the restart criterion is not checked. If zero, restart
might be checked after each main iteration.

\item[ModifyStoppingCriterion]
Flag indicating whether the stopping criterion should be modified.
If \texttt{true}, then the optimality check
(\ref{eq:neaMeaVar}) is done only if both of the following conditions are satisfied:
(a) in the last step, either a partial inside contraction or total contraction was done, and
(b) the moving direction of the simplex has changed by an angle $\phi_k$ of at least $(\pi/2)$,
where $\phi_k$ is computed using (\ref{eq:nelMeaCosMov}).
\end{codedescription}


\section{A Visual Approach to Operations}\label{sec:rightmost}
\epigraph{
``The fact that the carry chain allows a single bit
to affect all the bits to its left
makes addition a peculiarly powerful data manipulation operation''
}{
\emph{Hacker's Delight}, page \texttt{xiv}
\cite{Warren:2012:HD:2462741}
}

As mentioned in \autoref{sec:book},
the goal is to complement \emph{Hacker's Delight}:
chapter 2 of the book simply lists formulas.
While it is possible to understand why they work as they do,
it is a completely different task to create these on your own
-- this needs a systematic approach.

In order to develop the interpretations described by Henry Warren,
a \emph{visual approach} to operations has to be obtained first.
Consider the \lstinline$INC$ operator as an example:
Similar to performing addition by hand,
the value \lstinline$1$ gets added onto the least significant bit.
In the case of performing \lstinline$1+1$,
the overflow gets carried over to the next bit.

This repeats until the first \lstinline$0$ is reached.
So starting from the right, every bit gets inverted
until the first \lstinline$0$-bit is found.
This latter description is considered
the visual interpretation of \lstinline$INC$.
\lstinline$DEC$ follows an analog pattern
to the presented \lstinline$INC$ operator.
This allows for the visual interpretations
shown in \autoref{table:inc} and \autoref{table:dec}.

\begin{table}[H]
\centering
\begin{tabular}{c|ccc}
\lstinline$x$ & left of rightmost \lstinline$0$
    & rightmost \lstinline$0$ & trailings \lstinline$1$s\\
\hline
\lstinline$(x+1)$ & unchanged & \lstinline$1$ & \lstinline$0...0$\\
& \multicolumn{3}{c}{
    \fbox{invert all bits up to the rightmost \lstinline$0$}}
\end{tabular}
\caption{\lstinline$INC$ (visual interpretation)}
\label{table:inc}
\end{table}

\begin{table}[H]
\centering
\begin{tabular}{c|ccc}
\lstinline$x$ & left of rightmost \lstinline$1$
    & rightmost \lstinline$1$ & trailings \lstinline$0$s\\
\hline
\lstinline$(x-1)$ & unchanged & \lstinline$0$ & \lstinline$1...1$\\
& \multicolumn{3}{c}{
    \fbox{invert all bits up to the rightmost \lstinline$1$}}
\end{tabular}
\caption{\lstinline$DEC$ (visual interpretation)}
\label{table:dec}
\end{table}

After the now covered mathematical operators from
\autoref{sec:introduction},
the bitwise logical operators are of interest.
\lstinline$NOT$ is the trivial one to visually interpret,
because it simply inverts all bits of a given bitword.

Regarding \lstinline$OR$ and \lstinline$AND$,
the approach is to reduce the original definition of calculating
\lstinline$r$$_i$\lstinline$ = x$$_i \lor$\lstinline$y$$_i$ and
\lstinline$r$$_i$\lstinline$ = x$$_i \land$\lstinline$y$$_i$ respectively
for each bit $i$.

Instead, one might think of the second argument
manipulating the first one to become the result:
Set \lstinline$r$ to \lstinline$x$, then
for every \lstinline$1$ in \lstinline$y$
set the corresponding bit in \lstinline$r$ to \lstinline$1$.
Set to \lstinline$0$ for every \lstinline$0$ in \lstinline$y$
when executing \lstinline$AND$.
This is summarised in \autoref{table:or} and \autoref{table:and}.

\begin{table}[H]
\centering
\begin{tabular}{c|c|c}
\lstinline$y$ & bits that are \lstinline$0$
    & bits that are \lstinline$1$\\
\hline
\lstinline$(x|y)$ & \lstinline$x$ (unchanged) & \lstinline$1$\\
& \multicolumn{2}{c}{\fbox{for each \lstinline$1$ in \lstinline$y$,
    set to \lstinline$1$ in \lstinline$x$}}
\end{tabular}
\caption{\lstinline$OR$ (visual interpretation)}
\label{table:or}
\end{table}

\begin{table}[H]
\centering
\begin{tabular}{c|c|c}
\lstinline$y$ & bits that are \lstinline$0$
    & bits that are \lstinline$1$\\
\hline
\lstinline$(x&y)$ & \lstinline$0$ & \lstinline$x$ (unchanged)\\
& \multicolumn{2}{c}{\fbox{for each \lstinline$0$ in \lstinline$y$,
    set to \lstinline$0$ in \lstinline$x$}}
\end{tabular}
\caption{\lstinline$AND$ (visual interpretation)}
\label{table:and}
\end{table}

Notice that there was made a specific choice
in \autoref{table:or} and \autoref{table:and}
to let the second argument \lstinline$y$
manipulate the first argument \lstinline$x$
instead of the other way round.
This will enhance readability in formulas
presented in \autoref{sec:combining}.


% jam 2004-08-27

\subsection{Functions of the edges}
\label{sec:edges}

%------------------------------------------------------------------

\begin{figure}[!htp]
\centering
\begin{verbatim}
          p1
          o
         /|\
        / | \
       /  |  \e31
   e12/   |   \
     /    |e01 \
    /     |     \
p2 o f012 | f031 o p3
    \     |     /
     \    |    /
   e20\   |   /e03
       \  |  /
        \ | /
         \|/
          o
          p0

\end{verbatim}
\caption{Edge face pair labeling.
\label{diagram:EdgeFaces}}
\end{figure}

Notation in this section is based on figure \ref{diagram:EdgeFaces}.
We are discussing functions defined on a neighborhood of edge $e_{01}$.

We assume that, for each edge, an arbitrary order is assigned to
its two vertices, which are then at the positions $\p_0,\p_1$ in the diagram.

An interior edge has 2 adjacent faces, $f_{012}$ and $f_{031}$.
We assume that these 2 faces are oriented consistently, with the labels
taken counterclockwise, so that the normal vectors point out of the page.
Each face is represented by an ordered triple of vertices,
but the order is only determined up to a circular permutation;
for example, $f_{012}$ may be represented by the ordered triples
$(\p_0,\p_1,\p_2)$, $(\p_2,\p_0,\p_1)$, or $(\p_1,\p_2,\p_0)$,
but not by
$(\p_0,\p_2,\p_1)$, $(\p_1,\p_0,\p_2)$, or $(\p_2,\p_1,\p_0)$.

For the given ordering $(\p_0,\p_1)$ of the edge,
$f_{120}$ is the edge's {\it left face}
and $f_{031}$ is the {\it right face}.

Note that we cannot assume any consistent ordering of the 4 neighboring edges;
for example, $e_{12}$ may be represented by either ordered pair
$(\p_1,\p_2)$ or $(\p_2,\p_1)$.

%------------------------------------------------------------------

\subsubsection{Edge length}
\label{sec:edge_length}

The edge tangent vector is $\p_1 - \p_0$.

The gradient of its squared length is:
\begin{equation}
\Gc{\p_i}{\| \p_1 - \p_0 \|^2}{\q} = 2 \left( \p_i - \p_{(i+1) \bmod 1} \right)
\end{equation}

The gradient of the edge length, $\|\p_1 - \p_0\|$ is:
\begin{equation}
\Gc{\p_i}{\| \p_1 - \p_0 \|}{\q} =
\frac{\left( \p_i - \p_{(i+1) \bmod 1} \right)}
{\|\p_1 - \p_0\|}
\end{equation}

%------------------------------------------------------------------

\paragraph{Difference in face normals}
\label{sec:normal_difference}

One measure of the change in surface normal across an edge
is simply the vector difference of the two normals:

\begin{equation}
\label{eq:deltan}
{\mathbf \dn} (\p_0, \p_1, \p_2, \p_3)
=
\n (\p_{012}) - \n (\p_{031})
\end{equation}

The (total) derivative of the squared distance between adjacent face normals is:
\begin{eqnarray}
\Db{\|\dn(\p)\|^2}{\q}
& =
2 \ \dn ( \q )^\dagger &
\left( \Db{ ( \dn ) }{\q} \right)
\\
& =
2 \ \dn(\q)^\dagger &
\left( \Db{\n(\p_{012})}{\q} - \Db{\n(\p_{031})}{\q} \right)
\nonumber \\
& =
2 \dn(\q)^\dagger &
\{ \; \left[ \Identity_{\Reals^3} - \left( \n( \q_{012} ) \otimes \n( \q_{012} ) \right)
\right]
\ast \Db{\a ( \p_{012} ) }{\q}
\nonumber \\
\label{eq:deltan_derivative}
&
& - \left[ \Identity_{\Reals^3} - \left( \n( \q_{031} ) \otimes \n ( \q_{031} ) \right)
\right]
\ast \Db{\a ( \p_{031} ) }{\q}
\; \}
\nonumber
\end{eqnarray}

The partial derivatives, with respect to one of the vertices,
like $\Dd{\p_0}{\a ( \p_{012} ) }{\q}{\r_0}$,
all have a similar form:
\begin{equation}
\Dd{\p_0}{\a ( \p_{012} ) }{\q}{\r_0}  = (\q_1 - \q_3) \times \r_0
\end{equation}
Using this, equation \ref{eq:deltan_derivative}, equation \ref{eq:dot_cross},
and the facts that
$\dn(\q)  \perp  \n(\q_{012}) = - \left( \n(\q_{031})  \perp  \n(\q_{012}) \right)$
and
$\dn(\q)  \perp  \n(\q_{031}) = \n(\q_{012})  \perp  \n(\q_{031})$,
we can write the partial gradients without reference to the
derivative's argument $\r$:
\begin{eqnarray}
\label{eq:normal-difference-gradient}
\Gc{\p_0}{\|\dn\|^2}{\q}
& = &
\left[
\frac{{ \n(\q_{031})  \perp  \n(\q_{012}) }
{A(\q_{012})}}
\times (\q_1 - \q_2)
\right]
\; + \;
\left[
\frac{{ \n(\q_{012})  \perp  \n(\q_{031}) }
{A(\q_{031})}}
\times (\q_3 - \q_1)
\right]
\\
\Gc{\p_1}{\|\dn\|^2}{\q}
& = &
\left[
\frac{{ \n(\q_{031})  \perp  \n(\q_{012}) }
{A(\q_{012})}}
\times (\q_2 - \q_0)
\right]
\; + \;
\left[
\frac{{ \n(\q_{012})  \perp  \n(\q_{031}) }
{A(\q_{031})}}
\times (\q_0 - \q_3)
\right]
\nonumber
\\
\Gc{\p_2}{\|\dn\|^2}{\q}
& = &
\left[
\frac{{ \n(\q_{031})  \perp  \n(\q_{012}) }
{A(\q_{012})}}
\times (\q_0 - \q_1)
\right]
\nonumber
\\
\Gc{\p_3}{\|\dn\|^2}{\q}
& = &
\left[
\frac{{ \n(\q_{012})  \perp  \n(\q_{031}) }
{A(\q_{031})}}
\times (\q_1 - \q_0)
\right]
\nonumber
\end{eqnarray}

%------------------------------------------------------------------

\paragraph{Inner product between face normals}
\label{sec:normal_dot}

The inner product $\left( \n_{012} \bullet \n_{031} \right)$
is another important measure of edge curvature.
It is closely related to the squared distance between adjacent normals:
\begin{equation}
\label{eq:normal-distance-dot}
\| \n_{012} - \n_{031} \|^2
= \| \n_{012} \|^2
+ \| \n_{031} \|^2
- 2 \left( \n_{012} \bullet \n_{031} \right)
= 2 \left[ 1 - \left( \n_{012} \bullet \n_{031} \right) \right]
\end{equation}

The function $f(\p) = 1 - \left( \n_{012} \bullet \n_{031} \right)$
achieves its minimum, $0$, on flat face pairs,
and its maximum, $2$, on face pairs that are folded back on themselves.
It's a reasonable choice the total bending or curvature of a surface.
And $\Da{f} = - \Da{\left( \n_{012} \bullet \n_{031} \right)}$.

The derivative of
$\left( \n_{012} \bullet \n_{031} \right)$
can be calculated using equations \ref{eq:dot_derivative} and
\ref{eq:unit_normal_derivative}:
\begin{eqnarray}
\label{normal_dot_derivative}
\Db{\left( \n_{012} \bullet \n_{031} \right)}{\q}
& = & \n(\q_{031}) \bullet \Db{\n_{012}}{\q} + \n(\q_{012}) \bullet \Db{\n_{031}}{\q}
\\
\nonumber \\
& = &
\n(\q_{031}) \bullet
\frac{\Identity - \left(\n(\q_{012}) \otimes \n(\q_{012}) \right)}{\| \a(\q_{012}) \|}
\; \Db{\a_{012}}{\q}
\nonumber \\
& + &
\n(\q_{012}) \bullet
\frac{\Identity - \left(\n(\q_{031}) \otimes \n(\q_{031}) \right)}{\| \a(\q_{031}) \|}
\; \Db{\a_{031}}{\q}
\nonumber
\end{eqnarray}

As in \autoref{sec:normal_difference}, we can write the partial gradients
without reference to an argument:
\begin{eqnarray}
\label{eq:normal_dot_gradient}
\Gc{\p_0}{(\n_{012} \bullet \n_{031})}{\q}
& = \; &
\frac{ \n(\q_{031}) - \left[ \n(\q_{012}) \bullet \n(\q_{031}) \right] \n(\q_{012}) }
{\| \a (\q_{012}) \| }
\times (\q_2 - \q_1)
\\
& \; + &
\frac{ \n(\q_{012}) - \left[ \n(\q_{012}) \bullet \n(\q_{031}) \right] \n(\q_{031})  }
{\| \a (\q_{031}) \| }
\times (\q_1 - \q_3)
\nonumber \\
& & \nonumber \\
\Gc{\p_1}{(\n_{012} \bullet \n_{031})}{\q}
& = \; &
\frac{ \n(\q_{031}) - \left[ \n(\q_{012}) \bullet \n(\q_{031}) \right] \n(\q_{012})  }
{\| \a (\q_{012}) \| }
\times (\q_0 - \q_2)
\nonumber \\
& \; + &
\frac{ \n(\q_{012}) - \left[ \n(\q_{012}) \bullet \n(\q_{031}) \right] \n(\q_{031})   }
{\| \a (\q_{031}) \| }
\times (\q_3 - \q_0)
\nonumber \\
& & \nonumber \\
\Gc{\p_2}{(\n_{012} \bullet \n_{031})}{\q}
& = \; &
\frac{ \n(\q_{031}) - \left[ \n(\q_{012}) \bullet \n(\q_{031}) \right] \n(\q_{012})  }
{\| \a (\q_{012}) \| }
\times (\q_1 - \q_0)
\nonumber \\
& & \nonumber \\
\Gc{\p_3}{(\n_{012} \bullet \n_{031})}{\q}
& = \; &
\frac{ \n(\q_{012}) - \left[ \n(\q_{012}) \bullet \n(\q_{031}) \right] \n(\q_{031}) }
{\| \a (\q_{031}) \| }
\times (\q_0 - \q_1)
\nonumber
\end{eqnarray}

This can be simplified using the fact that
\(\n_i \perp \n_j = \n_i - \left[ \n_i \bullet \n_j \right] \n_j\), for unit vectors,
and the face area \(A(\q) = \frac{1}{2} \| \a(\q) \|\):
\begin{eqnarray}
\label{eq:simplified_normal_dot_gradient}
\Gc{\p_0}{(\n_{012} \bullet \n_{031})}{\q}
& = \;\;\; &
\frac{\left[ \n(\q_{031}) \perp \n(\q_{012}) \right]}{2A(\q_{012})}
\times (\q_2 - \q_1)
\\
& \;\;\; + &
\frac{\left[ \n(\q_{012}) \perp \n(\q_{031}) \right]}{2A(\q_{031})}
\times (\q_1 - \q_3)
\nonumber \\
& & \nonumber \\
\Gc{\p_1}{(\n_{012} \bullet \n_{031})}{\q}
& = \;\;\; &
\frac{\left[ \n(\q_{031}) \perp \n(\q_{012}) \right]}{2A(\q_{012})}
\times (\q_0 - \q_2)
\nonumber \\
& \;\;\; + &
\frac{\left[ \n(\q_{012}) \perp \n(\q_{031}) \right]}{2A(\q_{031})}
\times (\q_3 - \q_0)
\nonumber \\
& & \nonumber \\
\Gc{\p_2}{(\n_{012} \bullet \n_{031})}{\q}
& = \;\;\; &
\frac{\left[ \n(\q_{031}) \perp \n(\q_{012}) \right]}{2A(\q_{012})}
\times (\q_1 - \q_0)
\nonumber \\
& & \nonumber \\
\Gc{\p_3}{(\n_{012} \bullet \n_{031})}{\q}
& = \;\;\; &
\frac{\left[ \n(\q_{012}) \perp \n(\q_{031}) \right]}{2A(\q_{031})}
\times (\q_0 - \q_1)
\nonumber
\end{eqnarray}

%------------------------------------------------------------------

\paragraph{Squared inner product between face normals}
\label{sec:squared_normal_dot}

We can get a more even distribution of bending by giving
a higher weight to sharper edge bends.
A simple way to do that is to square some existing function,
for example: $\left(1 - \n_{012} \bullet \n_{031}\right)^2$.
The derivative is simply:
\begin{equation}
\Da{\left(1 - \n_{012} \bullet \n_{031}\right)^2}
= -2 \left( 1 - \n_{012} \bullet \n_{031} \right)
\Da{(\n_{012} \bullet \n_{031})}
\end{equation}

It follows from equation \ref{eq:simplified_normal_dot_gradient}
that the partial gradients are:
\begin{eqnarray}
\label{eq:squared_normal_dot_gradient}
\Gc{\p_0}{\left(1 - \n_{012} \bullet \n_{031}\right)^2}{\q}
& = \;\;\; &
\frac{\left( \n(\q_{012}) \bullet \n(\q_{031}) - 1\right)
}
{A(\q_{012}) }
\left[ \n(\q_{031}) \perp \n(\q_{012}) \right]
\times (\q_2 - \q_1)
\\
& \;\;\; + &
\frac{\left( \n(\q_{012}) \bullet \n(\q_{031}) - 1\right)
}{A(\q_{031})}
\left[ \n(\q_{012}) \perp \n(\q_{031}) \right]
\times (\q_1 - \q_3)
\nonumber \\
& & \nonumber \\
\Gc{\p_1}{\left(1 - \n_{012} \bullet \n_{031}\right)^2}{\q}
& = \;\;\; &
\frac{\left( \n(\q_{012}) \bullet \n(\q_{031}) - 1\right)
}{A(\q_{012})}
\left[ \n(\q_{031}) \perp \n(\q_{012}) \right]
\times (\q_0 - \q_2)
\nonumber \\
& \;\;\; + &
\frac{\left( \n(\q_{012}) \bullet \n(\q_{031}) - 1\right)
}{A(\q_{031})}
\left[ \n(\q_{012}) \perp \n(\q_{031}) \right]
\times (\q_3 - \q_0)
\nonumber \\
& & \nonumber \\
\Gc{\p_2}{\left(1 - \n_{012} \bullet \n_{031}\right)^2}{\q}
& = \;\;\; &
\frac{\left( \n(\q_{012}) \bullet \n(\q_{031}) - 1\right)
}{A(\q_{012})}
\left[ \n(\q_{031}) \perp \n(\q_{012}) \right]
\times (\q_1 - \q_0)
\nonumber \\
& & \nonumber \\
\Gc{\p_3}{\left(1 - \n_{012} \bullet \n_{031}\right)^2}{\q}
& = \;\;\; &
\frac{\left( \n(\q_{012}) \bullet \n(\q_{031}) - 1\right)
}{A(\q_{031})}
\left[ \n(\q_{012}) \perp \n(\q_{031}) \right]
\times (\q_0 - \q_1)
\nonumber
\end{eqnarray}


%!TeX root = nagurka_math
%\documentclass{article}
\documentclass[paper.tex]{subfiles}

\begin{document}

(coordinates that are determined by the fourier-type series are expressed in capital letters, like X or U).

In standard form, the differential equation translates one

$$ \ddot{U} + \frac{\beta}{\alpha} \dot{U} + \frac{\gamma}{\alpha}U =  \frac{1}{\alpha} \ddot{X} + \frac{\phi}{\alpha}\dot{X} + \frac{\xi}{\alpha}X $$

$$ P(t)= \sum_{j=0}^{5} t^{j} {p}_{j} $$

$$L(t) = \sum_{m=1}^{M} \sin{\left(\frac{2 \pi m t}{t_{f}} \right)} {b}_{m} + \sum_{m=1}^{M} \cos{\left(\frac{2 \pi m t}{t_{f}} \right)} {a}_{m}$$

$$X = P + L$$

Term-by-term derivatives (most symbolic operations were performed via SymPy; small sections offloaded to Maxima):

$$\dot{X}=\sum_{j=0}^{5} \frac{j t^{j} {p}_{j}}{t} + \sum_{m=1}^{M} - \frac{2 \pi m \sin{\left(\frac{2 \pi m t}{t_{f}} \right)} {a}_{m}}{t_{f}} + \sum_{m=1}^{M} \frac{2 \pi m \cos{\left(\frac{2 \pi m t}{t_{f}} \right)} {b}_{m}}{t_{f}}$$
$$\ddot{X} = \frac{\sum_{j=0}^{5} j t^{j} \left(j - 1\right) {p}_{j}}{t^{2}} - \frac{4 \pi^{2} \sum_{m=1}^{M} m^{2} \sin{\left(\frac{2 \pi m t}{t_{f}} \right)} {b}_{m}}{t_{f}^{2}} - \frac{4 \pi^{2} \sum_{m=1}^{M} m^{2} \cos{\left(\frac{2 \pi m t}{t_{f}} \right)} {a}_{m}}{t_{f}^{2}}$$\\

When numerically evaluating these symbolic derivatives are undefined at t=0; a small epsilon was always added to the beginning time vector.

"The solution is of form $U = U_p(x) + U_c(x)$ where $U_p(x)$ is a particular solution and $U_c(x)$ is
the solution of the associated homogeneous ODE."\\

Homogenous solution:

$$\ddot{U} + \beta /\alpha \dot{U} + \gamma/\alpha U = 0$$

$$(\beta/\alpha)^2 - 4(\gamma/\alpha) \approx 10^{17}$$

$$y_c(t) = c_1e^{(m_1t)} + c_2e^{(m_2t)}$$

\section{ODEINT}

$$F(t) = \frac{1}{\alpha} \ddot{X} + \frac{\phi}{\alpha}\dot{X} + \frac{\xi}{\alpha}X$$

$$ \ddot{U} + \frac{\beta}{\alpha} \dot{U} + \frac{\gamma}{\alpha}U =  F(t) $$

$$ U1 = \dot{U} $$

\section{}

Despite the simplicity of the technique, difficulties were encountered trying to implement it.

\begin{toolchain}
\textbf{Lesson learned:} For software architecture, testing, and readability reasons, it may be useful to ensure that intermediate coefficients are physically meaningful, not simply amorphous many-term operations.
\end{toolchain}

\begin{toolchain}
	\textbf{Lesson learned:} Autodiff is an excellent way to cross-check analytic derivatives.
\end{toolchain}

Many Yen and Nagurka's work. One of their formulations used explicit polynomial boundary conditions, but the derivation was not easily at hand (small derivative dots appear to be removed by scanners) so we quickly re-derive the coefficients.\\

This uses a technique very similar to a spline fit. We first wish to create a polynomial with specified boundary conditions $P(0), P(t_f), \dot P(0), \dot P(t_f), \ddot P(0), \ddot P(t_f)$.

$$P(t) = p_0 + p_1 t + p_2 t^2 + p_3 t^3 + p_4 t^4 + p_5 t^5$$

$$\dot P(t) = p_1 + 2 p_2 t + 3 p_3 t^2 + 4 p_4 t^3 + 5 p_5 t^4$$

$$\ddot P(t) = 2 p_2 + 6 p_3 t + 12 p_4 t^2 + 20 p_5 t^3$$

$$P(0) = p_0$$

$$P(t_f) = p_0 + p_1 t_f + p_2 t_f^2 + p_3 t_f^3 + p_4 t_f^4 + p_5 t_f^5$$

$$\dot P(0) = p_1$$

$$\dot P(t_f) = p_1 + 2 p_2 t_f + 3 p_3 t_f^2 + 4 p_4 t_f^3 + 5 p_5 t_f^4$$

$$\ddot P(0) = 2 p_2$$

$$\ddot P(t_f) = 2 p_2 + 6 p_3 t_f + 12 p_4 t_f^2 + 20 p_5 t_f^3$$

Substituting,

$$P(t_f) = P(0) + \dot P(0)t_f + (\ddot P(0) / 2)t_f^2 + p_3 t_f^3 + p_4 t_f^4 + p_5 t_f^5$$

$$\dot P(t_f) = \dot P(0) + \ddot P(0)t_f + 3 p_3 t_f^2 + 4 p_4 t_f^3 + 5 p_5 t_f^4 $$

$$\ddot P(t_f) = \ddot P(0) + 6 p_3 t_f + 12 p_4 t_f^2 + 20 p_5 t_f^3$$

Running through SymPy, \url{polynomial_system_of_equations.py}, % PAPERID 10

$$p_{0} = {P(0)}$$
$$p_{1} = {\dot{P}(0)}$$
$$p_{2} = \frac{{\ddot{P}(0)}}{2}$$
$$p_{3} = \frac{t_{f}^{2} \left(- 3 {\ddot{P}(0)} + {\ddot{P}(t_f)}\right) - 4 t_{f} \left(3 {\dot{P}(0)} + 2 {\dot{P}(t_f)}\right) - 20 {P(0)} + 20 {P(t_f)}}{2 t_{f}^{3}}$$
$$p_{4} = \frac{\frac{t_{f}^{2} \left(3 {\ddot{P}(0)} - 2 {\ddot{P}(t_f)}\right)}{2} + t_{f} \left(8 {\dot{P}(0)} + 7 {\dot{P}(t_f)}\right) + 15 {P(0)} - 15 {P(t_f)}}{t_{f}^{4}}$$
$$p_{5} = \frac{t_{f}^{2} \left(- {\ddot{P}(0)} + {\ddot{P}(t_f)}\right) - 6 t_{f} \left({\dot{P}(0)} + {\dot{P}(t_f)}\right) - 12 {P(0)} + 12 {P(t_f)}}{2 t_{f}^{5}}$$

Given the 'physical' boundary conditions $\{X(0), X(t_f), \dot X(0), \dot X(t_f), \ddot X(0), \ddot X(t_f)\}$ and the Fourier-type series boundary conditions $\{L(0), L(t_f), \dot L(0), \dot L(t_f), \ddot L(0), \ddot L(t_f)\}$, $\{P(0) = X(0) - L(0), ...\}$.

\section{}

One could also generalize the parametrization to arbitrary integer order of derivative, where $z$ is the derivative order and $J = 2(z+1)$ is the necessary polynomial order, $v = \frac{2\pi m}{t_f}$, by applying the trig functions

$$\frac{d^z}{dx^z}\cos x=\cos\left(x+z\frac{\pi}2\right)$$

$$\frac{d^z}{dx^z}\sin x=\sin\left(x+z\frac{\pi}2\right)$$

we get

$$\frac{d^z}{dt^z}=  v^z \sin(v t+ z\frac{\pi}{2}) $$

$$\frac{d^6}{dt^6} = \frac{\sum_{j=0}^{J} j t^{j} \left(???\sum_{j=0}^{J-1} j t^{j} ??? \right) {p}_{j}}{t^{6}}$$

%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}


%!TEX root = paper.tex
\subsection{Propagating solitary wave}
The propagating solitary wave tests the balancing of dispersion effects and nonlinearity for an appropriate initial condition.
On constant bathymetry, the Serre equations \cite{Serre.1953} have an analytic solitary wave solution
\begin{align}
\label{eq:Serre_sol_surface}
\xi(\bx,t)&=a \ \text{cosh}^{-2}(K(x-ct-x_0)), \\
u(\bx,t)&=c\frac{\xi(\bx,t)}{d+\xi(\bx,t)},
\end{align}
with the ratio $\frac{a}{d}=0.2$, propagation velocity $c=\sqrt{g(d+a)}$ on a constant depth $d=10 \, \text{m}$, scale factor $K=\sqrt{\left(\frac{3a}{4d^2(d+a)}\right)}$ and displacement $x_0=l/2$ on a domain of length $l=800 \, \text{m}$.
Again, we impose double periodic boundary conditions solitary wave propagating in the positive $x$-direction. The simulation time is $30$ seconds.

\subsubsection{Results of \nh\ model}
The equations \eqref{eq:nh_conti}--\eqref{eq:nh_closure} using the quadratic vertical pressure profile are equivalent to the Serre equations \cite{Serre.1953}, which is shown in  \cite{Jeschke.2016}. Using the linear vertical pressure profile, there is no equivalence to the Serre equations.
For the \nh\ model, we also need to initialize the vertical velocity, which directly follows from equation Equation \eqref{eq:nh_closure}, s.t.
\begin{align}
w&=-0.5h\partial_x u. \label{eq:nh_solit_init_w}
\end{align}
Figure \ref{fig:nh_solitarywave} compares our numerical computations with the analytical solution at the end of the simulation time.
The numerical result using the quadratic pressure profile shows a very good agreement with the analytical solution. In contrast, the application of the linear pressure profile yields a threefold mismatch arising from the inconsistency in initial conditions combined with the underlying equation system: Small amplitude waves propagate to the opposite direction, the wave height increases {because of weaker dispersion of the linear profile and trailing waves start to establish.
In \cite{StellingZijlema.2003, Walters.2005, Yamazaki.2008}, different solitary waves are computed with \nh\ models using the traditional linear vertical pressure profile. Therein, these mismatches are also visible, except that their amplitudes tend to diminish than to amplify. The reason is the application of another vertical velocity, namely $W(z)=-z\partial_x u$, so $w=-0.5(\xi-d)\partial_x u$. If the term $-d\partial_x u$ is added, the vertical velocity \eqref{eq:nh_solit_init_w} results. Hence, in \cite{StellingZijlema.2003, Walters.2005, Yamazaki.2008}, the vertical velocity is too large on the left hand side of the maximum amplitude and too small on the right hand side, which decreases the nonlinear steepening of the wave.
Their aim to compute the solitary waves was merely to show that \nh\ models produce similar results as \Bt\ models.

\begin{figure}[htbp]
\includegraphics[width=\textwidth]{solitary_nh}
\caption{Comparison of the analytical (black) sea surface height of the
solitary wave with the simulation results of the quadratic (yellow) and linear (blue) vertical profile after a propagation time of 10, 20 and 30 seconds to the right.}
\label{fig:nh_solitarywave}
\end{figure}