\columnbreak
\section{Model Validation}

% ===
\subsection{Stability-based Validation}

\emph{Stability:}\enspace
Solutions on two data sets drawn from the same source should be similar.

%\emph{(In-)Stability:} Expected (dis-)agreement of solutions from two data sets from the same source.

% ===
\subsection{Information-theoretic Validation}

\subsubsection{Shannon's Channel Coding Thm.}

\begin{itemize}
    \item \textbf{Channel:}\enspace
        $(\mathcal S, \{p(\cdot\mid s)\}_{s\in\mathcal S} )$,\enspace
        $\mathcal S$: alphabet
        \begin{itemize}
            \item $\epsilon$-noisy binary channel:
                $p(\hat s\mid s) = \big\{ \substack{
                    1-\epsilon \:\:\text{if}\:\: \hat s=s \\
                    \epsilon \phantom{1-} \:\:\text{if}\:\: \hat s\neq s}$
        \end{itemize}
        \vspace{-3pt}
    \item \textbf{Capacity:}\enspace
        \highlight{$\mathrm{cap} = \max_p I(S;\hat S)$}
        $\rightsquigarrow p_S(s)$
    \item \textbf{$\bm{(M,n)}$-code:}\enspace
        is a pair $(Enc,Dec)$\hfill
        $\color{gray} \leftarrow\text{scr. p.87}$\\\quad
        where $M$: \#messages, $n$: code-length
        \begin{itemize}
            \item \textbf{Rate:}\enspace
                $r = \frac{\log_2 M}{n}$
                $\Leftrightarrow$ $M=\floor{2^{nr}}$
            \item \textbf{Commu. err.:}\enspace
                $p\ped{err} \coloneqq \max\limits_{i\leq M}
                \P{ Dec(\widehat{ Enc(i) }) \neq i }$
        \end{itemize}
\end{itemize}

Goal / \textbf{Best code:}\enspace
\highlight*{$\lim\limits_{n\to\infty} \frac{\log M}{n}$\enspace s.t.\enspace $\lim\limits_{n\to\infty} p\ped{err} \to 0$}

\emph{Asymptotic equiparition property (AEP):}
\vspace{-2pt}
\begin{itemize}
    \item $A_\epsilon^{(n)}$:
        Typical set of sequences $(s_1,\ldots,s_n) \in \mathcal S^n$\\\qquad
        $\abs*{-\frac1n \log p_{S^n} (\bm s^n) - H[S]} < \epsilon$\hfill
        $\color{gray} \leftarrow \text{scr. p.89}$
    \item $\P*{(\bm S^n, \bm{\hat S}^n) \in A_\epsilon^{(n)}} \overset{n\to\infty}{\to} 1$\hfill
        $\color{gray} \leftarrow \text{scr. p.90}$
    \item \highlight{$p\ped{err} \leq 2^{-n(\mathrm{cap} - 3\epsilon -r)}$}
        $\overset{n\to\infty}{\to} 0$\enspace if\enspace \highlight{$r<\mathrm{cap}$}
\end{itemize}

\subsubsection{Algorithm Validation}

\textbf{Assumptions:}
\begin{itemize}
    \item Exponential solution space, i.e. $\log\abs{\mathcal C} = \mathcal O(n)$
    \item $\mathcal A$'s output is probabilistic, i.e. $p(\cdot \mid X')$
\end{itemize}

\emph{Ideal variant:}

\textbf{Messages:}\enspace
$\mathcal M = \brace{ X'_1, \ldots, X'_m }$
\vspace{-2pt}

\textbf{Code:}\enspace
$X'_i \xrightarrow{Enc_{\mathcal A}} p(\cdot\mid X'_i)
\xrightarrow{\mathcal{C_A}} p(\cdot\mid X''_i)
\xrightarrow{Dec_{\mathcal A}} \hat X$
\iffalse
    \begin{itemize}
        \item $Enc_{\mathcal A}$:\enspace
            encodes $X'_i \in \mathcal M$ as $p(\cdot\mid X'_i)$.
        \item $Dec_{\mathcal A}$:\enspace
            selects $\hat X = \arg\max\limits_X \kappa(X''_i, X)$\\
            whereby $\kappa(X'', X') \coloneqq \sum_c p(c\mid X'') p(c\mid X')$
    \end{itemize}
\fi

\emph{Empirical variant:}

\textbf{Messages:}\enspace
$\mathcal M = \brace{ \tau_1, \ldots, \tau_m }$
drawn u.a.r. from $\mathbb T$
\begin{itemize}
    \item Require\enspace $\sum_\tau p(c\mid \tau\circ X') \approx \frac{\abs{\mathbb T}}{\abs{\mathcal C}} \pm \rho$\hfill
        $\color{gray} \leftarrow \text{scr. p.95}$
\end{itemize}
\vspace{-2pt}

\textbf{Code:}\enspace
$\tau_i \xrightarrow{Enc} p(\cdot\mid \tau_i \circ X')
\xrightarrow{\mathcal{C_A}} p(\cdot\mid \tau_i \circ X'')
\xrightarrow{Dec} \hat\tau$
\begin{itemize}
    \item $Enc_{\mathcal A}$:\enspace
        encodes $\tau_i \in \mathcal M$ as $p(\cdot\mid \tau_i \circ X')$
    \item $Dec_{\mathcal A}$:\enspace
        selects $\hat\tau = \arg\max\limits_{\color{red}\tau} \kappa(\tau_i \circ X'', {\color{red}\tau} \circ X')$\\
        whereby \highlight{$\kappa(X'', X') \coloneqq \sum_c p(c\mid X'') p(c\mid X')$}
\end{itemize}

\emph{Asymptotic Equipartition Property (AEP):}

\textit{AEP fulfilled}\enspace if\enspace \highlight*{$\log \kappa(X',X'') \overset{n\to\infty}{\to} \mathcal E$}\\
\enspace whereby $\mathcal E \coloneqq \E[X',X'']{\log\kappa(X',X'')}$
\begin{itemize}
    \item $A^{(n)}_\epsilon$: set of $(\epsilon,n)$-typical pairs $X', X''$\\\qquad
        $\abs*{ \log\kappa(X',X'') - \mathcal E } < \epsilon$
    \item \highlight{$p\ped{err} \leq P_{(n)} \color{gray} \text{ c.f. scr. (6.19)}$}
        $\overset{n\to\infty}{\to} 0$ \enspace if\enspace \highlight{$\frac{\log m}{\log\abs{\mathcal C}} < I$}\\
        where $I \coloneqq \frac{1}{\log\abs{\mathcal C}} \E[X',X'']{\log(\abs{\mathcal C} \kappa(X',X''))}$
\end{itemize}

% ===
\subsection{Applications of PA}

\emph{PA:}\enskip
\textit{quantifies the amount of information that algorithms extract from phenomena.}
$\to$ quantified by \textbf{capacity} (max. \# distinguishable messages that can be communicated)

\emph{Temperature:}\enspace
$T^\ast = \arg\max_T \kappa(X',X'')$

\emph{Cost functions:}\enspace
Given $R_1(\cdot,\cdot),\ldots,R_s(\cdot,\cdot)$\\
$\max\limits_{\ell\leq s} \kappa_\ell(X',X'') = \max\limits_{\ell\leq s} \frac{1}{Z_{X'} Z_{X''}} \sum\limits_c \eu^{-\frac1T R_\ell(c,X')} \eu^{-\frac1T R_\ell(c,X'')}$

\emph{Algorithms:}\enspace
%\textit{here for MST (min. span. tree)}
Many MST (min. spanning tree) algo's are \textbf{contractive} ($\to$ sequence of candidate sol's).
\\
\textbf{Approximation Set Coding (ASC):}\\
\quad $p\ap{ASC}(c\mid X') = \begin{cases}
    1 / \abs*{G_\gamma(X')}  & \text{if}\enspace c\in G_\gamma(X') \\[-2pt]
    0   & \text{otw.}
\end{cases}$\\
\quad $G_\gamma(X') \coloneqq \brace*{ c\in\mathcal C : R(c,X') - \min\limits_{c\in\mathcal C} R(c,X') \leq \gamma }$
\begin{enumerate}
    \item Run $\mathcal A$ to compute $G_t^{\mathcal A}(X')$ and $G_t^{\mathcal A}(X'')$,\enspace for all $t$
    \vspace{2pt}
    \item $t^\ast = \arg\max\limits_t \kappa(X',X'')
        = \arg\max\limits_t \frac{\abs*{G_t^{\mathcal A}(X') \:\cap\: G_t^{\mathcal A}(X'')}}{\abs*{G_t^{\mathcal A}(X')} \:\cdot\: \abs*{G_t^{\mathcal A}(X'')}}$
    \item $c^\ast \xleftarrow{\$\:\text{sample}} \mathrm{Unif}\paren*{G_{t^\ast}^{\mathcal A}(X') \cap G_{t^\ast}^{\mathcal A}(X'')}$
\end{enumerate}

% ===


\chapter{Milky Way diffuse background study: denoising and inpainting}
\label{ch_inpainting}

% \markright{Milky Way diffuse background study: denoising and inpainting}

In order to extract a  diffuse emission, we want to remove the  point sources from the data.
As our HSD algorithm is very close to the MCA algorithm~\citep{starck:sta04}, an idea is to mask the most intense sources and to modify our algorithm in order to interpolate through the gaps exactly as in the MCA-Inpainting algorithm~\citep{inpainting:abrial06}.
This modified algorithm can be called MS-VSTS-Inpainting algorithm.

The problem can be reformulated as a convex constrained minimization problem:

\begin{equation}
\label{inp_eq34}
\begin{split}
\text{Arg} \min_{\mathbf{X}} \| \mathbf{ \Phi}^{T}\mathbf{X}\|_1,
\text{s.t.} \\ \: \left\{\begin{array}{c}\mathbf{X} \geqslant 0 , \\\forall (j,k)\in \mathcal{M},      (\mathbf{ \Phi}^{T}\Pi \mathbf{X})_j[k]=(\mathbf{ \Phi}^{T} \mathbf{Y})_j[k] , \end{array}\right.
\end{split}
\end{equation}
where $\Pi$ is a binary mask ($1$ on valid data and $0$ on invalid data).

The iterative scheme can be adapted to cope with a binary mask, which gives:
\begin{eqnarray}
\tilde{\mathbf{X}} = P_{+}[\mathbf{ X}^{(n)} + \mathbf{ \Phi} P_{\mathcal{M}} \mathbf{ \Phi}^{T} \Pi (\mathbf{ Y} - \mathbf{ X}^{(n)})] , \\
\mathbf{X}^{(n+1)} = \mathbf{ \Phi} \text{ST}_{\lambda_n}[\mathbf{ \Phi}\tilde{\mathbf{X}}] .
\end{eqnarray}

The thresholding strategy has to be adapted. Indeed, for the impainting task we need to have a very large initial threshold in order to have a very smooth image in the beginning and to refine the details progressively. We chose an exponentially decreasing threshold:
\begin{equation}
\label{eq42}
\lambda_{n} = \lambda_{\max}  (2^{(\frac{N_{\max} - n}{N_{\max} - 1})} -1),n=1,2,\cdots,N_{\max} ,
\end{equation}
where $\lambda_{\max} = \max (\mathbf{\Phi}^{T}\mathbf{X})$.

\begin{algorithm}
\caption{MS-VST + IUWT Denoising + Inpainting}
\label{alg2}

\begin{algorithmic}[1]
\REQUIRE $\quad$ data $a_0:=\mathbf{Y}$, mask $\Pi$, number of iterations $N_{\max}$, threshold $\kappa$.\\
\underline{\emph{\textbf{Detection}}} \\
\FOR{$j=1$ to $J$}
\STATE Compute $a_j$ and $d_j$ using (\ref{eq27}).
\STATE Hard threshold $|d_j[k]|$ with threshold $\kappa \sigma_j$ and update $\mathcal{M}$.
\ENDFOR \\
\underline{\emph{\textbf{Estimation}}} \\
\STATE Initialize $\mathbf{X}^{(0)}=0$, $\lambda_{0} = \lambda_{\max}$.
\FOR{$n=0$ to $N_{\max}-1$}
\STATE $\tilde{\mathbf{X}}= P_{+}[\mathbf{ X}^{(n)} + \mathbf{ \Phi} P_{\mathcal{M}} \mathbf{ \Phi}^{T} \Pi(\mathbf{ Y} - \mathbf{ X}^{(n)})]$.
\STATE $\mathbf{X}^{(n+1)} = \mathbf{ \Phi}^\text{ST}_{\lambda_n}[\mathbf{ \Phi}^{T}\tilde{\mathbf{X}}]$.
\STATE $\lambda_{n+1} = \lambda_{\max}  (2^{(\frac{N_{\max} - (n+1)}{N_{\max} - 1})} -1)$
\ENDFOR
\STATE Get the estimate $\hat{\mathbf{\Lambda}} = \mathbf{X}^{(N_{\max})}$.

\end{algorithmic}
\end{algorithm}

\section*{Experiment}

We applied this method on simulated Fermi data where we masked the most luminous sources.

The results are on Figure~\ref{impainting}. The MS-VST + IUWT + Inpainting method (Algorithm~\ref{alg2}) interpolates the missing data very well. Indeed, the missing part can not be seen anymore in the inpainted map, which shows that the diffuse emission component  has been correctly reconstructed.

\begin{figure}[htb]
\centering{
\includegraphics[width=5.5in]{13822fg22.pdf}
\includegraphics[width=5.5in]{13822fg23.pdf}
}
\caption{MS-VSTS - Inpainting.
\emph{Top}: Fermi simulated map with Poisson noise and the most luminous sources masked.
\emph{Bottomt}: Fermi simulated map denoised and inpainted with wavelets (Algorithm~\ref{alg2}).
Pictures are in logarithmic scale.
}
\label{impainting}
\end{figure}


\documentclass[12pt]{amsart}
\usepackage{geometry} % see geometry.pdf on how to lay out the page. There's lots.
\usepackage{calculational}
\usepackage{unitb}
\usepackage{amsmath}
\geometry{a4paper} % or letter or a5paper or ... etc
% \geometry{landscape} % rotated page geometry

% See the ``Article customise'' template for come common customisations

\title{A Sample Unit-B Toolkit proof}
\author{Simon Hudon}
\date{} % delete this line to display the current date

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}

%%% BEGIN DOCUMENT
\begin{document}

\maketitle
%\tableofcontents

%\section{}

%\begin{theorem}
%Any conjunctive function is monotonic
%\begin{assumption}
%f.(x \land y) ~ \equiv~ f.x \land f.y
%\end{assumption}
%\end{theorem}
%\begin{dummy}
\begin{calculation}
	f.x \le f.y
\hint{=}{ $\le$ to $=$ and $\uparrow$ }
	f.x \uparrow f.y \,=\, f.y
\hint{=}{ $f$ over $\uparrow$ }
	f.(x \uparrow y) \,=\, f.y
\hint{\follows}{ Leibniz }
	x \uparrow y \,=\, y
\hint{=}{ $\le$ from $=$ and $\uparrow$ }
	x \le y
\end{calculation}

\begin{calculation}
	f.x
\hint{\le}{ $x \le x\uparrow y$ }
	f.x \uparrow f.y
\hint{=}{ $f$ over $\uparrow$ }
	f.(x \uparrow y)
\hint{=}{ $x \le y$ cast as $x\uparrow y = y$ }
	f.y
\end{calculation}

%\subsection{}

\end{document}


\documentclass[12pt]{article}

\usepackage{amsmath}

\begin{document}

\section{Geometry definition of \texttt{pyFAI}}

All dimentions are in meter and all rotation are in radians.

\texttt{pyFAI} describes the position and orientation of the detector
by six variables, collectively called the PONI, for point of normal
incidence. In addition, a detector calibration is provided in the
PONI-file to convert pixel coordinates into real-space coordinates. In
the simplest case this is a pixel size.

\texttt{pyFAI} uses a coordinate system where the first axis (1)  is
vertically up (or $y$), the second axis (2) is horizontal ($x$) towards the ring
center (starboard), and the third axis (3) along the beam $z$. Note that
in this order (1,2,3) is a right-handed coordinate system, which makes
$xyz$ in the usual order a left-handed coordinate system!

The PONI parameters are: a distance $L$, the $y$ and $x$ coordinates
of the point of normal incidence in meters, $\mathrm{poni}_{1,2}$, and
3 detector rotations, $\theta_{1,2,3}$ about the $y$, $x$ and $z$
axes. The inversion of the $x$ and $y$ axes is due to the arrangement
of the detector data, with $x$-rows being the ``slow'' axis and
$y$-columns the ``fast'' axis. Extra care has to be taken with the
signs of the rotations when converting form this coordinate system to
another.

The detector rotations are taken about the origin of the coordinate
system (sample position). They are represented by the rotation
matrices:

\begin{align}
  \mathrm{R}_1(\theta_1)
  & =
  \begin{bmatrix}
    1 & 0 & 0 \\
    0 & \cos(\theta_1) & \sin(\theta_1) \\
    0 & -\sin(\theta_1) & \cos(\theta_1)
  \end{bmatrix}
  \\
  \mathrm{R}_2(\theta_2)
  & =
  \begin{bmatrix}
    \cos(\theta_2) & 0 & -\sin(\theta_2) \\
    0 & 1 & 0 \\
    \sin(\theta_2) & 0 & \cos(\theta_2)
  \end{bmatrix}
  \\
  \mathrm{R}_3(\theta_3)
  & =
  \begin{bmatrix}
    \cos(\theta_3) & -\sin(\theta_3) & 0\\
    \sin(\theta_3) & \cos(\theta_3) & 0\\
    0 & 0 & 1
  \end{bmatrix}
  \\
  R(\theta_1, \theta_2, \theta_3)
  & =
  R_3(\theta_3) \cdot R_2(\theta_2) \cdot R_1(\theta_1)
\end{align}

Let $P$ be a point on the detector at pixel coordinates $(d_1, d_2)$
in units $\mathrm{px}$. In the most simple case (in the absence of
distortions) these are converted into coordinates in units of meters
by multiplication with a scaling factor ``pixel size''
($\mathrm{pxsize}_{1,2}$m in units of $\mathrm{m}/\mathrm{px}$) and
subsequent subtraction of an offset ($\mathrm{poni}_{1,2}$, in units of
meters and yielding the coordinates $(p_1, p_2)$ in meters.

\begin{align}
  p_1 & = \mathrm{pxsize}_1 d_1 - \mathrm{poni}_1 \\
  p_2 & = \mathrm{pxsize}_2 d_2 - \mathrm{poni}_2
\end{align}

Let $L$ be the distance from the
origin/sample/diffractometer center of rotation. In the absence of any
detector rotations, $L$ is taken along $z$, $p_1$ along the $x$-axis
(horizontal) and $p_2$ along the $y$ axis (vertical). Then, in the
``detector coordinate system''

\begin{align}
  P & = \begin{bmatrix} p_1 \\ p_2 \\ L \end{bmatrix}
\end{align}

The coordinates in the laboratory system are obtained by
multiplication with the rotation matrix $R$

\begin{align}
  \begin{bmatrix} t_1 \\ t_2 \\ t_3 \end{bmatrix}
  = &
  R_{\mathtt{pyFAI}}(\theta_1, \theta_2, \theta_3) \cdot P
  \\
  = &
  R_3(\theta_3) \cdot R_2(\theta_2) \cdot R_1(\theta_1) \cdot
  \begin{bmatrix}
    \mathrm{pxsize}_1 d_1 - \mathrm{poni}_1 \\
    \mathrm{pxsize}_2 d_2 - \mathrm{poni}_2 \\
    L
  \end{bmatrix}
  \\
  = &
  R_3(\theta_3) \cdot R_2(\theta_2) \cdot R_1(\theta_1)
  \nonumber \\ &
  \cdot
  \left(
  \begin{bmatrix}
    \mathrm{pxsize}_1 & 0 & 0 \\
    0 & \mathrm{pxsize}_2 & 0 \\
    0 & 0 & 1
  \end{bmatrix}
  \cdot
  \begin{bmatrix}
    d_1 \\
    d_2 \\
    0
  \end{bmatrix}
  +
  \begin{bmatrix}
    - \mathrm{poni}_1 \\
    - \mathrm{poni}_2 \\
    L
  \end{bmatrix}
  \right)
  \label{eq-tpyFAI}
\end{align}

The inversion is straight-forward:

\begin{align}
  R_1(-\theta_1)\cdot R_2(-\theta_2) \cdot R_3(-\theta_3) \cdot
  \begin{bmatrix} t_1 \\ t_2 \\ t_3
  \end{bmatrix}
  & =
  \begin{bmatrix} p_1 \\ p_2 \\ L \end{bmatrix}
  \label{eq-find-alpha}
  \\
  \begin{bmatrix}
    t_1 \\ t_2 \\ t_3
  \end{bmatrix}
  & =
  \begin{bmatrix}
    A_1  \\ A_2 \\ A_3
  \end{bmatrix}
  + \alpha
  \begin{bmatrix}
    k_1 \\ k_2 \\ k_3
  \end{bmatrix},
\end{align}
where $A_{1,2,3}$ are the coordinates, in the laboratory system, of
the scattering center (grain, sub-grain or voxel), and $k_{1,2,3}$ is
the direction of the scattered beam. The third line ($\ldots = L$) of
eq.~\ref{eq-find-alpha} is then used to determine the free parameter
$\alpha$, which in turn is used in the first and second lines to find
$p_{1,2}$ and thus $d_{1,2}$.

As the most trivial example we consider the case of no rotations,
$\theta_1 = \theta_2 = \theta_3 = 0$. Then

\begin{align}
  A_3 + \alpha k_3 & = L \\
  \alpha & = \frac{L-A_3}{k_3} \\
  p_1 & = A_1 + (L-A_3) \frac{k_1}{k_3} \\
  p_2 & = A_2 + (L-A_3) \frac{k_2}{k_3}.
\end{align}

We see also that when all rotations are zero, $(\mathrm{poni}_1,
\mathrm{poni_2})$ are the real space coordinates of the direct beam
($A_{1,2,3}=k_{1,2}=0$) .

\section{Geometry definition of \texttt{ImageD11}}

For maximum convenience, \texttt{ImageD11} defines almost everything
differently than \texttt{pyFAI}.

Coordinate system: \texttt{ImageD11} uses the ID06 coordinate system
with $x$ along the beam, $y$ to port (away from the ring center), and
$z$ up.

Units: As the system is somewhat scale-invariant, \texttt{ImageD11}
allows a free choice of the unit of length, which we will call $X$
here. The same unit has to be used for all translations, and for the
pixel size of the detector. The default used in the code appears to be
$X = 1\,\mathrm{\mu m}$, but it might as well be Planck lengths,
millimeters, inches, meters, tlalcuahuitl, furlongs, nautical miles,
light years, kparsec, or whatever else floats your boat. The only
requirement is that you can actually measure and express the detector
pixel size and COR-to-detector distance in your units of choice.

Parameters: \texttt{ImageD11} defines the detector geometry via the
following parameters:

\begin{itemize}
\item Beam center $y_{\mathrm{center}}$ and $z_{\mathrm{center}}$:
  Position of the direct beam on the detector, in $\mathrm{px}$.
\item Pixel size $y_{\mathrm{size}}$ and $z_{\mathrm{size}}$ in
  ${X}/{\mathrm{px}}$.
\item Detector tilts $t_x$, $t_y$, and $t_z$, in $\mathrm{rad}$. The
  center of rotation is the point where the direct beam intersects the
  detector.
\item Distance $D$, in units $X$, between the origin to the point
  where the direct beam intersects the detector.
\item Detector flip matrix $O = \begin{bmatrix} o_{11} & o_{12}
  \\ o_{21} & o_{22} \end{bmatrix}$. This matrix takes care of
  correcting typical problems with the way pixel coordinates are
  arranged on the detector. If, e.g., the detector is rotated by
  $90^{\circ}$, then $O=\begin{bmatrix} 0 & 1 \\ -1 &
  0\end{bmatrix}$. If left and right (or up and down) are inverted on the
  detector, then $o_{22} = -1$ ($o_{11}=-1$).
\end{itemize}

It appears that these conventions where defined under the assumption
that the detector is more or less centered in the direct beam, and
that the detector tilts are small.

The implementation in the code \texttt{transform.py} is using the
following equations:

\begin{align}
  R_{\mathtt{ImageD11}}(\theta_x, \theta_y, \theta_z)
  & =
  R_1(\theta_x) \cdot R_2(\theta_y) \cdot R_3(\theta_z)
  \\
  \begin{bmatrix}
    p_z \\ p_y
  \end{bmatrix}
  & =
  \begin{bmatrix}
    o_{11} & o_{12}
    \\ o_{21} & o_{22}
  \end{bmatrix}
  \cdot
  \begin{bmatrix}
    (d_z - z_{\mathrm{center}}) z_{\mathrm{size}} \\
    (d_y - y_{\mathrm{center}}) y_{\mathrm{size}}
  \end{bmatrix}
  \label{eq-p}
  \\
  \begin{bmatrix}
    t_x \\ t_y \\ t_z
  \end{bmatrix}
  & =
  R_{\mathtt{ImageD11}}(\theta_x, \theta_y, \theta_z)
  \cdot
  \begin{bmatrix}
    0 \\ p_y \\ p_z
  \end{bmatrix}
  +
  \begin{bmatrix}
    D \\ 0 \\ 0
  \end{bmatrix}
  \label{eq-t}
\end{align}
Note that the order of $y$ and $z$ is not the same in eqs.~\ref{eq-p} and \ref{eq-t}.

The same could be achieved more easily by

\begin{align}
  \begin{bmatrix}
    t_x \\ t_y \\ t_z
  \end{bmatrix}
  & =
  R_{\mathtt{ImageD11}}(\theta_x, \theta_y, \theta_z)
  \cdot
  \begin{bmatrix}
    1 & 0 & 0 \\
    0 & y_{\mathrm{size}} o_{22} & z_{\mathrm{size}} o_{21} \\
    0 & y_{\mathrm{size}} o_{12} & z_{\mathrm{size}} o_{11}
  \end{bmatrix}
  \cdot
  \begin{bmatrix}
    D \\ p_y - y_{\mathrm{center}} \\ p_z - z_{\mathrm{center}}
  \end{bmatrix}.
  \label{eq-tImageD11}
\end{align}

\section{Conversion}

The transformation between the different coordinate systems is
achieved by
\begin{align}
  t_{\mathtt{ImageD11}}
  =
  \begin{bmatrix}
    0 & 0 & 1 \\ 0 & -1 & 0 \\ 1 & 0 & 0
  \end{bmatrix}
  t_{\mathtt{pyFAI}},
  \label{eq-coordconv}
\end{align}
where $t_{\mathtt{ImageD11}}$ is given by eq.~\ref{eq-tImageD11}, and
$t_{\mathtt{pyFAI}}$ is given by eq.~\ref{eq-tpyFAI}. Substituting
these equations into eq.~\ref{eq-coordconv}, one can them attempt to
convert \texttt{pyFAI} parameters into \texttt{ImageD11} parameters
and vice versa. As \texttt{pyFAI} does not allow for detector
flipping, $o_{11}=o_{22}=1$ and $o_{12}=o_{21}=0$. Furthermore,
$\mathrm{pxsize}_1 = z_{\mathrm{size}}$ and $\mathrm{pxsize}_2 =
y_{\mathrm{size}}$. Because we assume that both equations describe the
same pixel, twice in detector coordinates and twice in real-space
coordinates, $p_y = p_2$ and $p_z = p_1$.

CHECK IF $p_y = p_2$ or $p_y = - p_2$/$o_{22}=-1$ DUE TO
\texttt{pyFAI} LEFT HANDED COORDINATE SYSTEM!!!

For the translations, the equivalence is tractable by setting $p_1 =
p_2 = p_y = p_z = 0$. One obtains 3 equations that are coupled via the
rotation matrices --- as the center of rotations are different, there
should be a noticeable influence of the distance on the offsets. In
general, $D \geq L$ with $D=L$ only for $\theta_1=\theta_2=0$
($\theta_y = \theta_z = 0$. Because the rotations are applied in
different order one can not assume, \emph{a priori}, that the angles
are the same! (If at least two tilt angles are small, this may be a
good approximation)

\end{document}


\documentclass{article}
\usepackage{amsmath,amssymb}
\usepackage{hyperref}
\usepackage{graphicx}
\usepackage{listings}
\title{\bf{Turbine Formulation}}
\author{Nicholas Malaya, Roy Stogner, Robert D. Moser \\ Institute for
Computational Engineering and Sciences \\ University of Texas at Austin}
\date{}

\begin{document}
\maketitle
\newpage
\section{Turbine Definitions}

The normal in the blade's velocity direction is,
\begin{equation}
n_B = \frac{u_B}{||u_B||}.
\end{equation}
Where $u_B$ is the blade velocity and is specified.
The normal in the fan vertical direction is also set, and is typically,
\begin{equation}
n_v = \left(0,0,1\right)
\end{equation}
e.g. pointing ``up''. Then the normal in the radial direction must be,
\begin{equation}
n_r = n_B \times n_v.
\end{equation}
% The fan-wing-plane bit means that we're only looking at the projection
% of velocity into the plane that's defined by the base velocity and
% vertical direction.

% (01:03:44 PM) Roy Stogner: The "local relative velocity" means that
% we're taking the velocity not in the reference frame of the domain, but
% in the reference frame of the wing.  So if the base velocity is U_B and
% the air velocity is U, then the local relative velocity is U - U_B.
% (01:04:25 PM) Roy Stogner: Note that we simplify that equation a tiny
% bit by using the fact that U_B and N_R are perpendicular.

Then, the fan-wing-plane component (e.g. the plane perpendicular to the
radius) of local relative velocity is
\begin{equation}
u_p = u - (u\cdot n_r)\cdot n_r - u_B.
\end{equation}
We can now define the lift and drag normals, where the direction
opposing drag is,
\begin{equation}
n_{\text{drag}} = \frac{u_p}{||u_p||}
\end{equation}
and the direction opposing lift orthogonal to the drag and the radial direction,
\begin{equation}
n_{\text{lift}}= n_{\text{drag}} \times n_r.
\end{equation}
Now the ``forward velocity'' in the reference frame of the turbine is,
\begin{equation}
u_{\text{fwd}}= -u_p \cdot n_B
\end{equation}
and the ``upward'' velocity in this frame is,
\begin{equation}
u_{\text{up}} = u_p \cdot n_v.
\end{equation}
Finally, we can specify the angle with respect to the fan velocity
direction as,
\begin{equation}
 \theta_f = \text{atan2}\left(\frac{u_{\text{up}}}{u_{\text{fwd}}}\right)
 %\theta_f = \text{tan}^{-1}\left(\frac{u_{\text{up}}}{u_{\text{fwd}}}\right)
\end{equation}
while the angle with respect to the chord is this with the addition of
the angle of attack of the blade,
\begin{equation}
 \theta = \theta_f + \alpha(r).
\end{equation}
Now, we only need the drag polars in order to fully specify the force
on the blades.

\newpage
\section{Drag Polars}

We have been provided lift and drag as a function of angle of attack for
two cases, the flat plate and semi-circle turbine blades.
For the flat plate, the drag polars are specified as,

\begin{lstlisting}
    theta := ((t+pi/2)%pi)-pi/2;
    lift   = 'if(abs(theta)<pi/24,theta*9,sin(2*theta))'
    drag   = 'if(abs(theta)<pi/24,0.005+theta*theta*81/25,1-0.8*cos(2*theta))'
\end{lstlisting}
for example.

\begin{figure}[!htb]
  \begin{center}
    \includegraphics[width = 12 cm]{figs/flat}
    \caption{The flat plate interpolation.}
    \label{flat}
  \end{center}
\end{figure}
As shown in Figure \ref{flat}, the fit is largely accurate.

The semi-circular plots are more complicated. Here, we use a high order
polynomial fit to continuously interpolate between drag polars. This
results in a
\begin{figure}[!htb]
  \begin{center}
    \includegraphics[width = 12 cm]{figs/semi}
    \caption{The semi-circular plate interpolation.}
    \label{semi}
  \end{center}
\end{figure}
As shown in Figure \ref{semi}, the fit is decent, but not remarkably accurate.

Then, the force on the turbine is,
\begin{equation}
 \boxed{F = \frac{1}{2}\frac{\rho u_p^2 C}{A}\left(C_l \cdot
					      n_\text{lift} + C_d \cdot n_\text{drag}  \right)}
\end{equation}

Where C is the chord length (specified as input), and A is the area,
which is also specified. For instance, the area swept might be,
\begin{lstlisting}
     area_swept = '{r:=sqrt(x^2+y^2); 2*pi*r*(.6-.4)/4}'
\end{lstlisting}
for a 4 blade fan. The chord is specified similarly, for instance,
\begin{lstlisting}
     chord_length = '.2*sqrt(2)'
\end{lstlisting}
would set a 0.28 meter chord length.
%
% ---------------------------------------------------
% old turbine discussion:
% ---------------------------------------------------

% The penalty function will be added to the navier-stokes as,
% % \begin{align}
% %  \partial_t(\rho u) = \text{NS} &+ \frac{1}{\epsilon}(u-u_t)\cdot n_p \\
% %                                 &+ \frac{1}{\epsilon}(u-\omega u_t)\cdot n_p \\
% % \end{align}

% \begin{align}
%  \partial_t(\rho u) = \text{NS} + \frac{1}{\epsilon}(u-\omega u_t)\cdot n_p
% \end{align}

% We will start by formulating a constant rotation speed
% turbine. The rotor will have counter-clockwise spin with angular
% velocity omega.

% \begin{verbatim}
%   base_velocity=
%                 '{(r<r_max)*(z<zmax) * r * -sin(theta)*omega}
%                  {(r<r_max)*(z<zmax) * r * cos(theta)*omega}
%                  {0}'
% \end{verbatim}
% There are a few problems with this formulation. One, in a turbine, the
% fluid velocity is not necessarily identical to the blade velocity.
% The turbine speed can take a range of values, and is often quoted as the
% `tip-speed ratio'', $\lambda$, which is,
%  \begin{equation}
%   \lambda = \frac{\omega R}{v}.
%  \end{equation}
% Here, R is the radius of the turbine blades, omega the angular velocity,
%  and v is the velocity of the fluid. Values for $\lambda$ generate power
%  in the range of 0-16, and are typically between 4 and 14.

% Zmax will be the height of the vanes, which is approximately 0.84 meters in the laboratory,
% and 1.0795 meters for the two-meter SoV configuration. Rmax will be the inner diameter of the vanes.

%
% ---------------------------------------------------
% old discussion on determining the rotation speed:
% ---------------------------------------------------
% We now need to choose $\omega$. At the Betz limit, the velocities will be
% $V_{\text{out}}/V_{\text{in}} = 1/3$. Furthermore, the control volume
% analysis implies that,
% \begin{equation}
%  V_{\text{turbine}} = \frac{1}{2}\left(V_{\text{in}} + V_{\text{out}} \right)
% \end{equation}
% then,
% \begin{align}
%  V_{\text{turbine}} &= \left(\frac{1}{2}\right) \left(\frac{4}{3}\right) V_{\text{in}} \\
%  &= \frac{2}{3} V_{\text{in}}.
% \end{align}
% Note this is not the velocity of the turbine, but the velocity of the
% fluid around the turbine. We can estimate the turbine speed from the
% ``tip-speed ratio'', $\lambda$, which is,
% \begin{equation}
%  \lambda = \frac{\omega R}{v}.
% \end{equation}
% Here, R is the radius of the turbine blades, omega the angular velocity,
% and v is the velocity of the fluid. Values for $\lambda$ generate power
% in the range of 0-16, and are typically between 4 and 14. Thus, our
% angular velocity of the turbine is,
% \begin{equation}
%  \omega = \frac{ 2 V_{\text{in}}\lambda}{3 R_{\text{max}}}.
% \end{equation}

% ---------------------------------------------------
% refs
% ---------------------------------------------------

%
% https://github.com/grinsfem/grins/tree/master/src/physics/src
% https://github.com/grinsfem/grins/blob/master/src/physics/src/averaged_fan.C
% https://github.com/grinsfem/grins/blob/master/src/physics/src/averaged_fan_base.C#L120
% https://github.com/grinsfem/grins/blob/3f86653ab5ba01981193ccce142ea196d0a192de/src/physics/src/averaged_fan_base.C
% https://github.com/grinsfem/grins/blob/34e2fc1e7d7143e0080f81c67bd584ce1236e2c1/src/physics/include/grins/averaged_fan.h
% https://github.com/grinsfem/grins/blob/34e2fc1e7d7143e0080f81c67bd584ce1236e2c1/src/physics/include/grins/averaged_fan_base.h
%

\end{document}


\newpage

\captionsetup{labelformat=empty}

\section*{\texorpdfstring{Appendix C: VOI for conservation auctions
heuristic solution \emph{n} =
2}{Appendix C: VOI for conservation auctions heuristic solution n = 2}}\label{appendix-c-voi-for-conservation-auctions-heuristic-solution-n-2}
\addcontentsline{toc}{section}{Appendix C: VOI for conservation auctions
heuristic solution \emph{n} = 2}

\textbf{Priors}

Let \(A_1\) and \(A_2\) be two assets. Each has some cost efficiency
\(c_1\) and \(c_2\), both of which are uncertain with prior probability
distributions,

\begin{equation}
c_1\sim\mathcal{N}(\mu_1, \sigma_1)
(\#eq:c1)
\end{equation}

and

\begin{equation}
c_2\sim\mathcal{N}(\mu_2, \sigma_2)
(\#eq:c2)
\end{equation}

where \(\mu_1\) and \(\mu_2\) are the prior means and \(\sigma_1\) and
\(\sigma_2\) are the prior standard deviations.

Let \(\pi=\Pr(c_1 > c_2)\) and therefore,

\begin{equation}
\pi = \Phi\left(\frac{\mu_1-\mu_2}{\sqrt{\sigma^2_1+\sigma^2_2}}\right)
(\#eq:c2)
\end{equation}

\textbf{Utilities}

If \(\pi > 1 - \pi\) then the optimal action is to purchase asset
\(A_1\), otherwise it is to purchase \(A_2\).

We can assign utilities to each combination of the condition of \(c_1\)
and \(c_2\), and each action such that,

\begin{equation}
\begin{aligned}
u(c_1 > c_2, A_1)&=1\\
u(c_1 < c_2, A_1)&=0\\
u(c_1 > c_2, A_2)&=0\\
u(c_1 < c_2, A_2)&=1\\
\end{aligned}
(\#eq:utilitiesapen)
\end{equation}

Therefore the expected values of taking each action are
\(\mathrm{E}[u(A_1)]=\pi\) and \(\mathrm{E}[u(A_2)]=1-\pi\).

\textbf{Value of perfect information}

If we could reduce the uncertainty in the prior probabilites of \(c_1\)
and \(c_2\) such that \(\pi\) approached either limit, then the expected
value of perfect information is,

\begin{equation}
\mathrm{EVPI}=1-\max(\pi, 1-\pi)=\min(\pi, 1-\pi)
(\#eq:evpiaucapen)
\end{equation}

\textbf{Preposterior analysis}

Now let's assume we can sample from some process and learn about \(c_1\)
and \(c_2\).\\
Let \(X_1\) and \(X_2\) be observations from a process described by the
following sampling distributions,

\begin{equation}
X_1\sim\mathcal{N}\left(c_1, 1\right)
(\#eq:x1)
\end{equation}

and

\begin{equation}
X_2\sim\mathcal{N}\left(c_2, 1\right)
(\#eq:x2)
\end{equation}

where for simplicity the standard deviation is one.

Furthermore, we can make \(Mp\) and \(M(p-1)\) observations of \(X_1\)
and \(X_2\) respectively. Where \(M\) is the total number of samples the
budget allows and \(p\) and \(p-1\) are proportions of that budget
allocated to each assets. Given the observations of \(X_1\) and \(X_1\)
with sample sizes \(Mp\) and \(M(p-1)\) we can update the prior beliefs
in \(c_1\) and \(c_2\) and arrive at posterior distributions,

\begin{equation}
\begin{aligned}
c_1^\prime &\sim \mathcal{N}(\mu^\prime_1, \sigma^\prime_1)\\
c_2^\prime &\sim \mathcal{N}(\mu^\prime_2, \sigma^\prime_2)
\end{aligned}
(\#eq:posteriorcs)
\end{equation}

where,

\begin{equation}
\begin{aligned}
\mu^\prime_1 &= \frac{\mu_1 + X_1Mp\sigma^2_1}{Mp\sigma^2_1 + 1}\\
\sigma^\prime_1 &= \sqrt{\frac{\sigma^2_1}{Mp\sigma^2_1 + 1}}\\
\mu^\prime_2 &= \frac{\mu_2 + X_2M(p-1)\sigma^2_2}{M(p-1)\sigma^2_2 + 1}\\
\sigma^\prime_2 &= \sqrt{\frac{\sigma^2_2}{M(p-1)\sigma^2_2 + 1}}
\end{aligned}
(\#eq:posteriormusig)
\end{equation}

\textbf{Value of sample information}

Now we make the simplifying assumptions that
\(\mathrm{E}[X_1]=\mathrm{E}[c_1]=\mu_1\) and
\(\mathrm{E}[X_1]=\mathrm{E}[c_1]=\mu_1=0\).

In this scenario the prior probability of \(c_1>c_2\) is,

\begin{equation}
\pi=\Phi\left(\frac{\mu_1}{\sqrt{\sigma^2_1+\sigma^2_2}}\right)
(\#eq:priorpi)
\end{equation}

, and the posterior is

\begin{equation}
\pi^\prime=\Phi\left(\frac{\mu_1}{\sqrt{\frac{\sigma^2_1}{Mp\sigma^2_1 + 1}+\frac{\sigma^2_2}{M(p-1)\sigma^2_2 + 1}}}\right)
(\#eq:posteriorpi)
\end{equation}

We can now calculate the expected value of sample information,

\begin{equation}
\mathrm{EVSI}=\max(\pi^{\prime}, 1-\pi^{\prime})-\max(\pi, 1-\pi)
(\#eq:evsiaucapen)
\end{equation}

\textbf{Optimal sampling allocation}

With equations 10-12, for any given set of \(\mu_1\), \(\sigma_1\),
\(\mu_2\), and \(M\) we can find the optimal allocation to sample for
asset \(A_1\), \(p\) to maximise the value of sample information.

\textbf{Summary}

\begin{itemize}
\tightlist
\item
  Optimal allocation insensitive to ratio of \(\mu_a\) to \(\mu_b\).
\item
  Optimal allocation sensitive to ratio of \(\sigma_1\) to \(\sigma_2\).
\item
  Always preferential to sample asset with greater uncertainty.
\item
  Solution is symmetrical.
\item
  The \(M\) at which you start to allocate sampling to both assets is
  proportional to the ratio of \(\sigma_1\) to \(\sigma_2\).
\item
  The \(M\) at which you start to allocate sampling to both assets is
  insensitive to whether \(\sigma_1 < \sigma_2\) or
  \(\sigma_1 > \sigma_2\).
\item
  Optimal allocation always has greater EVSI when
  \(\sigma_1 \neq \sigma_2\).
\end{itemize}

\textbf{\(\mu_1 > \mu_2, \sigma_1 \gg \sigma_2\)}

First let's examine the case where our prior belief is that \(c_1\) is
somewhat greater than \(c_2\), where \(\mu_1=.2\) and \(\mu_2 = 0\) but
the uncertainty in \(\theta_1\) is far greater than in \(\theta_2\),
\(\sigma_1 = 2\) and \(\sigma_2 = .2\).

\includegraphics{figure/unnamed-chunk-2-1.png} \clearpage

In this case, until we can take more than a total of 24 samples
(\(N=24\)) the optimal allocation is to allocate all sampling effort to
asset \(A_1\) (a). With a total budget greater than 24 samples, we start
allocating sampling to asset \(A_2\) at a diminishing rate with total
sample size and asymptoting at \(p=0.5\). The expected value of sample
information (EVSI) increases with sample size with diminishing returns
asymptoting below the EVPI (c). At all sample sizes, the optimal
allocation has greater expected value than naive assumption of constant
allocation of \(p=0.5\). The greatest benefit of allocating optimally is
when the sample size is \(N=10\). As sample size increases the
additional benefit of allocating optimally declines asymptotically as
the optimal allocation approaches \(p=0.5\) (d).

\textbf{\(\mu_1 \simeq \mu_2, \sigma_1 \gg \sigma_2\)}

Holding all the other parameters constant, let's examine a scenario
where the prior expection of \(c_1\) is only marginally better than
\(c_2\) (i.e., reduce \(\mu_1\) to \(.01\)).

\includegraphics{figure/unnamed-chunk-3-1.png} \clearpage

We still allocate in the same way as before (a). The EVPI approaches its
theoretical maxmimum of 0.5. But the value of sample information
achievable for anything less than 200 samples is reduced to near zero
(c). The shape of the additional benefit from optimal allocation is the
same but the scale has reduced and the optimal value of N has increased
meaning more samples must be taken to maxmise the additional gain in
EVSI by using the optimal allocation vs the naive allocation of
\(p=0.5\) (d).

\textbf{\(\mu_1 \gg \mu_2, \sigma_1 \gg \sigma_2\)}

All other parameters still the same, but now with \(\mu_1=1\) much
greater than \(\mu_2\).

\includegraphics{figure/unnamed-chunk-4-1.png} \clearpage

Still same optimal allocation (a). EVPI reduced as we start already more
certain that \(A_1\) is a better asset than \(A_2\). The return on
investing in each additional sample diminishes quicker and sooner as
EVSI asymptotes at nearer EVPI (c). The additional benefit in EVSI seen
by sampling optimally has increased by peaks earlier at \(N=2\) (d).

\textbf{\(\mu_1 > \mu_2, \sigma_1 > \sigma_2\)}

Resetting \(\mu_1\) now we examine what happens when \(\sigma_1\) is
double \(\sigma_2\).

\includegraphics{figure/unnamed-chunk-5-1.png} \clearpage

The optimal allocation has the same shape as before but now we allocate
sampling to asset \(A_2\) sooner than before. Now if we take more than 3
samples we will allocate an increasing amount of them to asset \(A_2\)
(a). The EVPI has been reduced as we have started off more certain that
\(A_1\) is greater \(A_2\) (c). There is less advantage to sampling
optimally rather than the naive allocation but the point at which the
additional benefit in EVSI peaks is at higher \(N\) (\(N=2\)) than when
\(\sigma_1\) was much more than \(\sigma_2\) (d).

\textbf{\(\mu_1 > \mu_2, \sigma_1 = \sigma_2\)}

Now increase the precision of \(c_1\) so that \(\sigma_1 = \sigma_2\).

\includegraphics{figure/unnamed-chunk-6-1.png} \clearpage

Now the optimal allocation is to just allocate evenly between the assets
(i.e., the naive allocation is now optimal and there is effectively no
advantage to optimize) (a). The allocation is still insentive to the
value of \(\mu_1\) and the value of \(\sigma\)'s. Decreasing the prior
precisions increases EVPI, and with it the point at which EVSI vs \(N\)
asymptotes, but without changing the rate EVPI increases with \(N\).
Increaseing the value of \(\mu_1\) decreases EVPI (but less sensitively
than changing the precision) and also increases the rate at which EVPI
vs \(N\) approaches EVPI (c).

\textbf{\(\mu_1 > \mu_2, \sigma_1 \ll \sigma_2\)}

Returning to the original paramterisation now reverse the prior
precisions so that \(\tau^\prime_a\) is much greater than
\(\tau^\prime_b\).

\includegraphics{figure/unnamed-chunk-7-1.png} \clearpage

The allocation is the same but now reflected so that we still allocate
to the asset with more uncertainty (a). This applies no matter what the
other parameter values are (i.e., the problem is symetrical).

\captionsetup{labelformat=default}


\documentclass[./\jobname.tex]{subfiles}
\begin{document}

\chapter{State of the Art}
\label{chap:state_of_the_art}
This chapter provides an overview of the current state of the art in solving \gls{pde}. Included are the widely used \gls{fem} as well as heuristic optimisation methods. Further, an introduction to the \gls{de} framework is given, which provides the basis for the algorithms described in this thesis.

\section{Finite Element Method}
Currently, the Finite Element Method is the go-to approach to solve partial differential equations. The domain $\Omega$ on which the \gls{pde} is posed, is discretised into multiple smaller elements - as the name suggests. Thus, \gls{fem} counts to the category of meshed methods. The underlying solution function $u(\mathbf{x})$ to the PDE is then approximated by so called ``basis-functions'' $\Phi(\mathbf{x})$ limited to these finite elements. This thesis uses the open-source Netgen/NGSolve \gls{fem} package (\cite{schoberl_ngsolvengsolve_2020}).

The general steps taken to solve a PDE with an FEM solver are:
\begin{enumerate}
	\item \underline{Step: Strong Form} \\
		  This is the standard formulation of the linear \gls{pde}. $\mathbf{L}$ and $\mathbf{B}$ are linear differential operators that include the derivatives. \\
		  \begin{equation}
		  \label{eq: strong form}
			  \begin{split}
			  	\mathbf{x} \in \mathbb{R}^2 \\
			  	u(\mathbf{x}), f(\mathbf{x}), g(\mathbf{x}): \Omega \rightarrow \mathbb{R} \\
				\mathbf{L} u(\mathbf{x}) = f(\mathbf{x}) \text{ on $\Omega$} \\
				\mathbf{B} u(\mathbf{x}) = g(\mathbf{x}) \text{ on $\partial \Omega$}
			  \end{split}
		  \end{equation}
		  Further, only Dirichlet boundary conditions are considered, thus the boundary operator is always the identity matrix $\mathbf{B} = \mathbb{I}$. Therefore, the linear operator on the boundary $\mathbf{B}$ can be disregarded, resulting in  \\
		  \begin{equation}
		  	u(\mathbf{x})|_{\partial \Omega} = g(\mathbf{x}) .
		  \end{equation}

	\item \underline{Step: Weak Form} \\
		  The next step is to reformulate the strong form into a usable weak form. This is equivalent to the strong form but written in an integral notation. In this equation, the $A$, $b$ and $c$ correspond to the constant factors of the derivatives in strong form. For the sake of completeness, this is kept abstract. In the \gls{pde}s considered in this work, $A = \mathbb{I}$ and $b=\mathbf{0}$, $c = 0$. Currently, the so-called test-function $v(\mathbf{x})$ is an arbitrary function, but it has to be 0 on the boundary $v(\mathbf{x})|_{\Omega} = 0$. The choice of the test-function correspond to different \gls{fem} types (\cite[p. 6f]{shen_spectral_2011}).\\
		  \begin{equation}
		  \label{eq: weak form}
			  \begin{split}
			      \underbrace{\int_{\Omega} - (\nabla^T A \nabla) u(\mathbf{x}) v(\mathbf{x}) dV - \int_{\Omega} b^T \nabla u(\mathbf{x}) v(\mathbf{x}) dV + \int_{\Omega} c u(\mathbf{x}) v(\mathbf{x}) dV}_{a(u,v)} \\ = \underbrace{\int_{\Omega} f(\mathbf{x}) v(\mathbf{x}) dV}_{F(v)}
			  \end{split}
		  \end{equation}
	\item \underline{Step: Discretisation of $\Omega$} \\
		  Create a mesh of finite elements that span the whole domain. Usually these are triangles. Thus, this step is sometimes called ``triangulation''.
	\item \underline{Step: Basis functions} \\
		  Choose a basis function $\Phi(\mathbf{x})$ that can be used to approximate the solution $u(\mathbf{x}) \approx u_{h}(\mathbf{x}) = \sum_{i = 1}^{N} u_i \Phi_i(\mathbf{x})$. A common choice are Lagrange or Chebyshev polynomials. In the Galerkin type \gls{fem}, the test-function $v(\mathbf{x})$ is the same as the trail-function, thus $v(\mathbf{x}) = \sum_{j = 1}^{N} v_j \Phi_j(\mathbf{x})$. The choice of the basis function $\Phi(\mathbf{x})$ largely influences the computational effort.  $\Phi(\mathbf{x})$ should have a small support, to produce a thinly populated matrix $A$ in the linear system of equations \eqref{eq:linear_system_of_equations} below.
	\item \underline{Step: Solution} \\
		  In the weak form, as seen in equation \eqref{eq: weak form}, $a(u,v)$ is a continuous bilinear form and $F(v)$ is a continuous linear functional. Substituting $u$ and $v$ with their corresponding approximation from \mbox{step 4} results in
		  \begin{equation}
		  \sum_{j=1}^{N} v_j \sum_{i=1}^{N} u_i a(\Phi_i, \Phi_j) = \sum_{j=1}^{N} v_j F(\Phi_j).
		  \end{equation}
		  Dividing by the $v_j$ values on both sides results in a linear system of equations, where the constant factors $u_i$ need to be determined.
		  \begin{equation}
		  \label{eq:linear_system_of_equations}
		  \underbrace{\sum_{i=1}^{N} u_i a(\Phi_i, \Phi_j)}_{\mathbf{A u}} = \underbrace{F(\Phi_j)}_{\mathbf{b}} \text{ for $j=1,...N$}
		  \end{equation}
\end{enumerate}

Modern solvers include more complex and advanced techniques to further improve the solution error and the computation time. Some of the most important concepts that are also available in NGSolve are listed here.

\begin{itemize}
	\item \underline{Static Condensation}: \\
		  Depending on the number of discrete elements, the $\mathbf{A}$ matrix can be very large. Inverting large matrices is very time consuming. Static condensation, also called Guyan reduction (\cite{guyan_reduction_1965}), reduces this dimensionality by exploiting the structure of $\mathbf{A}$.
	\item \underline{Preconditioner}: \\
		  Instead of solving the $\mathbf{A}^{-1}$ exactly, this can also be approximated by a matrix that is similar to $\mathbf{A}^{-1}$. The actual inverse can be iteratively approximated. NGSolve implements multiple different preconditioners and it even allows to create your own method.
	\item \underline{Adaptive Mesh Refinement}: \\
		The accuracy of a FEM-approximated solution mainly depends on the density of the mesh. Typically, finer meshes tend to produce more accurate solutions, but the computation time is longer. This trade-off can be overcome by a self-adaptive mesh. NGSolve implements that in an adaptive loop that executes:
		\begin{itemize}
			\item Solve PDE (with coarse mesh)
			\item Estimate Error (for every element)
			\item Mark Elements (that have the greatest error)
			\item Refine Elements (that were previously marked)
			\item Repeat until degrees of freedom exceed a specified $N$
		\end{itemize}
\end{itemize}

\section{Computational Intelligence Methods}
\label{chap:literature_overview}

The research community interested in computational intelligence solvers for differential equations has been steadily growing over the past 20 years. This chapter summarises the most important works done in the general field of development and application of such statistical numerical solvers. The following table \ref{tab:literature_research} gives a brief overview of these papers and sorts them historically.

In general, all of these papers from the table use the \gls{wrm}, or some variant of that concept, to transform their differential equation into an optimisation problem. This serves as the fitness function and is necessary to evaluate a possible candidate solution and perform the evolutionary selection. The fitness function is the function to be optimised. It is also called objective function and these terms are used interchangeably in this thesis. In short, the residual $R$ is defined through the differential equation itself and can be calculated by $R(u(\mathbf{x})) = \mathbf{L}u(\mathbf{x}) - f(\mathbf{x})$. The residual can be thought of as a functional that substitutes $u(\mathbf{x})$ with an approximate solution $u_{apx}(\mathbf{x})$ and returns a numerical score. The \gls{wrm} method is further described in chapter \ref{chap:opt_problem}.

\cite{howard_genetic_2001} is one of the first advances in this field. They approximate a subset of the convection-diffusion equations with \gls{gp} (\cite{koza_genetic_1992}). Their main idea is to use a polynomial of variable length as the candidate solution that is forced to satisfy the boundary condition. Their fitness value, as seen in equation \eqref{eq:howard_fitness_2001}, is calculated by squaring the residual $R$ and integrating it over the domain. Since the polynomials are known, and the problems are restricted to a specific differential equation, the integral can be evaluated analytically.

\begin{equation}
\label{eq:howard_fitness_2001}
	F(u_{apx}(\mathbf{x})) = -\int_{\Omega} R(u_{apx}(\mathbf{x}))^2 dx
\end{equation}

\cite{kirstukas_hybrid_2005} proposes a three-step procedure. The first step is time consuming and employs \gls{gp} techniques to find basis functions that span the solution space. The second step is faster and uses a Gram–Schmidt algorithm to compute the basis function multipliers to develop a complete solution for a given set of boundary conditions. Using linear solver methods, a set of coefficients is found that produces a single function that both satisfies the differential equation and the boundary or initial conditions at distinct points over the domain. These points are further called collocation points.

\cite{tsoulos_solving_2006} use \gls{ge} (\cite{ryan_grammatical_1998}) to find solutions to various differential equations. In contrary to \gls{gp}, \gls{ge} uses vectors instead of trees to represent the candidate string. The solution is evaluated as an analytical string, constructed of the functions $sin$, $cos$, $exp$ and $log$, as well as all digits and all four basic arithmetic operations. Because the \gls{ge} step could result in virtually any function, the fitness integral from equation \eqref{eq:howard_fitness_2001} can not be calculated analytically. Thus, the integral is approximated by evaluating the residual at collocation points within the domain. This is seen in equation \eqref{eq:fit_func_tsoulos}. The algorithm was tested on multiple problems of \gls{ode}, system of ODEs and \gls{pde}. Only the results for ODEs were promising.

\begin{equation}
\label{eq:fit_func_tsoulos}
F(u_{apx}(\mathbf{x})) = \sum_{i=1}^{n_C} ||R(u_{apx}(\mathbf{x}_i))||^2
\end{equation}

\cite{mastorakis_unstable_2006} couples a \gls{ga} (\cite{holland_outline_1962}) with a \gls{ds} method (\cite{nelder_simplex_1965}) for the local solution refinement. The candidates are represented as polynomials of the order 5 where the coefficients are optimised. The boundary condition is directly incorporated into the candidate, thus simplifying the objective function to equation \eqref{eq:fit_func_tsoulos}. The focus here is on unstable ODEs that can not be solved with finite difference methods.

\cite{sobester_genetic_2008} tried a radical different approach to incorporate the boundary condition into the solution. They found that using \gls{gp} for the inner domain is only effective if the algorithm does not have to consider the boundary. They split the solution $u_{apx}(\mathbf{x})$ into two parts where $u_{GP}(\mathbf{x})$ represents the solution for the inner domain and $u_{RBF}(\mathbf{x})$ ensures the boundary condition
\begin{equation}
\label{eq:solution_sobester}
u(\mathbf{x})_{apx} = u_{GP}(\mathbf{x}) + u(\mathbf{x})_{RBF}.
\end{equation}
At first, the \gls{gp} step produced a trial solution according to the objective function \eqref{eq:fit_func_tsoulos}. After the \gls{gp} procedure, a linear combination of radial basis functions $u(\mathbf{x})_{RBF} = \sum_{j=1}^{n_B} \alpha_j \Phi (||\mathbf{x}-\mathbf{x}_{j}||)$ is specifically tailored to $u_{GP}(\mathbf{x})$ that ensures the boundary condition at all $\mathbf{x}_{j}$ points on $\partial \Omega$. Finding the parameters $\alpha_j$ can be formulated as a least squares problem.

\cite{howard_genetic_2011} use a \gls{gp} scheme to find the solution to a specific set of simplified convection-diffusion equations. They represent a candidate as discrete function value points over the domain. The function between these points is interpolated. The fitness function is similar to equation \eqref{eq:fit_func_tsoulos} with the exception that the $n_C$ points are not predetermined. These points are sampled randomly in the domain, thus allowing the algorithm to approximate the solution aside from fixed base points.

\cite{chaquet_solving_2012} use a simple self-adaptive \gls{es} (as developed by \cite{schwefel_evolutionsstrategien_1977} and \cite{rechenberg_evolutionsstrategien_1978}) to evolve the coefficients of a partial Fourier series. The fitness function is expressed in equation \eqref{eq:fit_func_chaquet}. This is similar to the fitness function \ref{eq:fit_func_tsoulos}, but it extends the definition of the boundary to also include Neumann conditions by introducing the linear differential operator $\mathbf{B}$. The limit $n_C$ denotes the number of inner collocation points $\mathbf{x}_i$ within the domain $\Omega$, whereas $n_B$ is the number of discrete points $\mathbf{x}_j$ on the boundary $\partial \Omega$. Further, a penalty factor $\phi$ shifts the focus of the fitness to the boundary. Additionally, this objective function can also represent systems of differential equations, where the number of equations is denoted by $m$. To reduce the search dimension (represented by the number of harmonics), they developed a scheme that only optimises one harmonic at a time and freezes the other coefficients. This scheme is based on the often observed principle that lower frequencies are more important in reconstructing a signal than higher ones. Albeit this concept might not be valid for all possible functions, it worked on all differential equations of their testbed.

\begin{equation}
\label{eq:fit_func_chaquet_2012}
F(u_{apx}(\mathbf{x})) = \frac{\sum_{i=1}^{n_C} || \mathbf{L}u_{apx}(\mathbf{x}_i) - f(\mathbf{x}_i)||^2 + \phi \sum_{j=1}^{n_B} || \mathbf{B}u_{apx}(\mathbf{x}_j) - g(\mathbf{x}_j)||^2}{m (n_C + n_B)}
\end{equation}

\cite{babaei_general_2013} takes a similar approach. They approximate a solution using a partial Fourier series. The optimal parameters for the candidates are found using a \gls{pso} algorithm (\cite{kennedy_particle_1995}). The fitness function consists of two parts, one for the inner area (equation \eqref{eq:inner_WRF}) and one for the boundary (equation \eqref{eq:boundary_penalty}). These are added together resulting in equation \eqref{eq:inner_and_boundary_fitness}.

The weighted residual integral WRF is exactly the formulation of the \gls{wrm} from chapter \ref{chap:opt_problem}. $W$ is an arbitrary weighting function. The absolute values of $W$ and $R$ ensure that only positive values count towards the fitness. Instead of using a sum over collocation points, the integral is evaluated using a numerical integration scheme.

\begin{equation}
\label{eq:inner_WRF}
WRF(u_{apx}(\mathbf{x})) = \int_{\Omega} |W(\mathbf{x)}| |R(u_{apx}(\mathbf{x}))| dx
\end{equation}

The boundary condition is incorporated by summing up its normed violations at distinct points $\mathbf{x}_i$. $K_j$ are penality multipliers that shift the focus to different points of the boundary. The concept of this penalty function originates from \cite{rajeev_discrete_1992}.

\begin{equation}
\label{eq:boundary_penalty}
PFV(u_{apx}(\mathbf{x})) = WRF(u_{apx}(\mathbf{x})) \cdot \sum_{j=1}^{n_B} K_j \left(\frac{u_{apx}(\mathbf{x}_i)}{g(\mathbf{x}_i)} - 1\right)
\end{equation}

\begin{equation}
\label{eq:inner_and_boundary_fitness}
F(u_{apx}(\mathbf{x})) = WRF(u_{apx}(\mathbf{x})) + PFV(u_{apx}(\mathbf{x}))
\end{equation}

\cite{panagant_solving_2014} use polynomials as a candidate representation. They do not specify the order or the type of the polynomial. They test five different simple versions of the optimisation algorithm \gls{de} (\cite{storn_differential_1997}). Further, they introduce a so called DE-New that increases the population size after every generation. Their proposition is that greater population sizes are better at finding good solutions.

\cite{sadollah_metaheuristic_2017} compares three different optimisation algorithms to approximate differential equations: \gls{pso}, \gls{hs} (\cite{geem_new_2001}) and \gls{wca} (\cite{eskandar_water_2012}). They use the formulation in equation \eqref{eq:inner_WRF}, where the weighting function is the same as the residual $|W(\mathbf{x})| = |R(u_{apx}(\mathbf{x}))| \rightarrow WRF = \int_{\Omega} |R(u_{apx}(\mathbf{x}))|^2 dx$. The integral is again approximated using a numerical integration scheme. They find that the \gls{pso} is slightly better at producing low error solutions, however \gls{wca} is better at satisfying the boundary condition.

In their paper \cite{chaquet_using_2019} describe an algorithm that approximates a solution with a linear combination of Gaussian \gls{rbf} as kernels:
\begin{equation}
u(\mathbf{x})_{apx} = \sum_{i=1}^{N} \omega_i e^{\gamma_i (\left||\mathbf{x} - \mathbf{c}_i\right||^2)}
\end{equation}
The approximated function $u(\mathbf{x})_{apx}$ can be fully determined by a finite number of parameters: $\omega_i, \gamma_i, \mathbf{c}_i$. These are stacked together into a vector $\mathbf{p_{apx}}$ and called the decision variables which are optimised by the algorithm.
The objective function can be seen in equation \eqref{eq:fit_func_chaquet}. This is an update of the objective function in \ref{eq:fit_func_chaquet_2012} where the inner collocation points also get scaled by a weighting function $\xi(\mathbf{x}_i)$.
\begin{equation}
\label{eq:fit_func_chaquet}
F(u_{apx}(\mathbf{x})) = \frac{\sum_{i=1}^{n_C} \xi (\mathbf{x}_i) || \mathbf{L}u_{apx}(\mathbf{x}_i) - f(\mathbf{x}_i)||^2 + \phi \sum_{j=1}^{n_B} || \mathbf{B}u_{apx}(\mathbf{x}_j) - g(\mathbf{x}_j)||^2}{m (n_C + n_B)}
\end{equation}
The multipliers $\xi(\mathbf{x}_i)$ and $\phi$ are weighting factors for either the inner or the boundary term. The whole term is normalised with the number of collocation points.
The parameters of the kernels are determined via a \gls{cma_es} (\cite{hansen_reducing_2003}). To further improve the solution, the evolutionary algorithm is coupled with a \gls{ds} method to carry out the local search. The authors show empirically that the local search significantly improves the performance by testing the algorithm on a set of 32 differential equations.

\cite{fateh_differential_2019} use a simple variant of \gls{de} where candidates are represented as discrete function value points within the domain. The function values between the grid points are linearly interpolated. This is a radical brute force approach that results in a massive search space dimension. Yet, the main advantage is that the solution is not limited to a decomposition of kernel functions and thus, even non-smooth functions can be approximated. Since this approach does not produce an analytical solution, the differential equation and the boundary condition is incorporated into the fitness function by taking the sum of squared residuals at every grid point, as seen in equation \eqref{eq:fit_fateh}. The derivatives within the residual are calculated between two neighbouring points by the difference quotient.
\begin{equation}
\label{eq:fit_fateh}
F(\mathbf{x}) = \sqrt{\sum_{i=0}^{n} R(\mathbf{x}_i)^2}
\end{equation}

\begin{table}[H]
	\centering
	\noindent\adjustbox{max width=\linewidth}{
		\begin{tabular}{|c|c|c|c|}

			\hline
			\rowcolor[HTML]{\farbeTabA}

			Paper & Algorithm & Representation & Problems \\ \hline

			\multilinecell{\cite{howard_genetic_2001}} & \multilinecell{\gls{gp}} & \multilinecell{polynomial of \\ arbitrary length} & \multilinecell{one-dimensional \\ steady-state \\ model of \\ convection-diffusion \\ equation} \\ \hline

			\multilinecell{\cite{kirstukas_hybrid_2005}} & \multilinecell{\gls{gp}} & \multilinecell{algebraic \\ expression} & \multilinecell{heating of thin rod \\ heating by current} \\ \hline

			\multilinecell{\cite{tsoulos_solving_2006}} & \multilinecell{\gls{ge}} & \multilinecell{algebraic term} & \multilinecell{set of ODEs \\ system of ODEs \\ and PDEs} \\ \hline

			\multilinecell{\cite{mastorakis_unstable_2006}} & \multilinecell{\gls{ga}\\(global); \\ \gls{ds}\\(local)} & \multilinecell{5th order \\ polynomial}& \multilinecell{unstable \\ ODEs} \\ \hline

			\multilinecell{\cite{sobester_genetic_2008}} & \multilinecell{\gls{gp} \\ and \\ RBF-NN} & \multilinecell{algebraic term \\ for inner; \\ RBF for boundary} & elliptic PDEs \\ \hline

			\multilinecell{\cite{howard_genetic_2011}} & \multilinecell{\gls{gp}} & function value grid & \multilinecell{convection–diffusion \\ equation \\ at different \\ Peclet numbers } \\ \hline

			\multilinecell{\cite{chaquet_solving_2012}} & \multilinecell{\gls{es}} & \multilinecell{partial sum \\ of Fourier series} & \multilinecell{testbench of \\ ODEs \\ system of ODEs \\ and PDEs} \\ \hline

			\multilinecell{\cite{babaei_general_2013}} & \multilinecell{\gls{pso}} & \multilinecell{partial sum\\of Fourier series} & \multilinecell{integro-differential equation\\system of linear ODEs \\ Brachistochrone \\ nonlinear Bernoulli} \\ \hline

			\multilinecell{\cite{panagant_solving_2014}} & \multilinecell{\gls{de}} & \multilinecell{polynomial of \\ unspecified order} & \multilinecell{set of 6 \\ different PDEs}  \\ \hline

			\multilinecell{\cite{sadollah_metaheuristic_2017}} & \multilinecell{\gls{pso}\\\gls{hs}\\\gls{wca}} & \multilinecell{partial sum\\of Fourier series} & \multilinecell{singular BVP} \\ \hline

			\multilinecell{\cite{chaquet_using_2019}} & \multilinecell{\gls{cma_es}\\(global); \\ \gls{ds}\\(local)} & \multilinecell{linear combination \\ of Gaussian kernels} & \multilinecell{testbench of \\ ODEs \\ system of ODEs \\ and PDEs}\\ \hline

			\multilinecell{\cite{fateh_differential_2019}} & \multilinecell{\gls{de}} & \multilinecell{function value\\grid} & elliptic PDEs \\ \hline

		\end{tabular}
	}
	\unterschrift{Literature research on the general topic of stochastic solver and their application. The papers are sorted by date of release. }{}{}
	\label{tab:literature_research}
\end{table}

\section{Differential Evolution}

The differential evolution framework was first introduced in \cite{storn_differential_1997}. Due to its simple and flexible structure, it quickly became one of the most successful evolutionary algorithm. Over the years, several adaptations to the original framework have been proposed and some of them currently count to the best performing algorithms, as the 100-Digit Challenge at GECCO 2019 (\cite{suganthan_suganthancec2019_2020}) shows.

The main \gls{de} framework consists of three necessary steps that continuously update a population of possible solutions. The population can be interpreted as a matrix, where each row-vector $\mathbf{x}_i$, also called individual, represents a point within the search domain and has a fitness value corresponding to the fitness function $f(\mathbf{x}_i): \mathbb{R}^n \rightarrow \mathbb{R}$. The goal is to minimise the fitness function. These steps are performed in a loop until a predefined termination condition is reached. Each individual step is controlled by a user-defined parameter:
\begin{itemize}
	\item \underline{Mutation}: \\
		  Mutation strength parameter F;\\
		  The mutation uses the information from within the population to create a trial vector $v_i$. This is done by scaling the difference between some vectors in the population - hence the name \textit{differential} evolution. The \textit{/current-to-pbest/1} mutation operator can be seen in equation \eqref{eq:mut_rand_1} where $x_i$ is the current individual, $x_{best}^p$ is one random vector of the p\% top vectors, $x_{r1}$ is a random vector from the population while $\tilde{x}_{r2}$ is randomly chosen from the population and the archive. $x_{r1}$ and $\tilde{x}_{r2}$ must not describe the same individual.
		  \begin{equation}
		  \label{eq:mut_rand_1}
		  v_i = x_{i} + F_i(x_{best}^p - x_{i}) + F_i(x_{r1} - \tilde{x}_{r2})
		  \end{equation}
	\item \underline{Crossover}: \\
		  Crossover probability parameter CR;\\
		  The crossover procedure randomly mixes the information between the trial vector $v_i$ and a random candidate from the population $x_{i}$ to create a new trial vector $u_i$. The binomial crossover from equation \eqref{eq:crs_bin} randomly takes elements from both vectors, where $K$ is a random index to ensure that at least one element from the trial vector $v_i$ is taken.
		  \begin{equation}
		  \label{eq:crs_bin}
		  u_{ij}=\begin{cases}
		  v_{ij}, &\text{if $j = K \lor rand[0,1] \leq CR$}\\
		  x_{ij}, &\text{otherwise}
		  \end{cases}
		  \end{equation}
	\item \underline{Selection}: \\
		  Population size N;\\
		  The selection replaces the old candidate $x_i$ if the trial candidate $u_i$ is better as measured by the fitness function. This is performed for every individual in the population, then the next generation is started.
\end{itemize}

In modern \gls{de} variants, these parameters are self-adapted during the evolutionary process. This means that the algorithms can balance out between exploration of the search-space and exploitation of promising locations.

A prominent example of a modern \gls{de} with self-adaption is JADE, which was developed by \cite{zhang_jade_2009}. The adaption is performed by taking successful F and CR values of the last generation into account. If a certain setting is successful in generating better candidates, newly selected F and CR gravitate towards that setting. The pseudocode is presented in the appendix \ref{chap:pscode_jade}.

This idea was later refined by \cite{tanabe_success-history_2013}. They propose a similar self-adaptive scheme but extend the ``memory'' for good F and CR parameters over multiple generations. This idea improves the robustness as compared to JADE. The pseudocode in appendix \ref{chap:pscode_shade} shows the outline of this so called SHADE algorithm.

The latest iteration of SHADE is called L-SHADE (\cite{tanabe_improving_2014}), which improves the performance by including a deterministic adaptive concept for the population size. At first, L-SHADE starts with a big population size, and reduces the number of individuals in a linear fashion by deleting bad candidates. This has the effect of reducing the number of unnecessary function evaluations. The code is displayed in the appendix \ref{chap:pscode_lshade}.

\end{document}