\input{../common/common.tex}

\title{Math notes - Airplane seating}
\author{Uwe Hoffmann}
\hypersetup{colorlinks, pdftitle={Math notes - Airplane seating}}

\begin{document}

\setcounter{chapter}{1}
\section*{Airplane seating}

\vspace{10 mm}
\begin{problem}
 A line of $n$ airline passengers is waiting to board a plane. They each hold a ticket to one of the $n$ seats on that flight. (For convenience, let's say that the $i$th passenger in line has a ticket for the seat number $i$.) Unfortunately, the first person in line is crazy, and will ignore the seat number on their ticket, picking a random seat to occupy. All of the other passengers are quite normal, and will go to their proper seat unless it is already occupied. If it is occupied they will then find a free seat to sit in, at random. What is the probability that the last ($n$th)
person to board the plane will sit in their proper seat (\#$n$)?
\end{problem}

Any seat arrangement under the rules of the problem is a permutation $\pi$ from the set $S_n$ of permutations of size $n$. Let's define $A_n \subseteq S_n$ the subset of permutations of size $n$ that are valid seat arrangements.

Let $B_n := \{\pi \in A_n: \pi(n) = n\}$ be the subset of $A_n$ where the last person gets their proper seat. A strategy to solve the problem would be to count $|A_n|$ and $|B_n|$ and then divide them up to get the probability.

We will use the permutation cycle notation $(i_1,i_2,\ldots,i_k)$ for a cycle of length $k$ that maps $i_1 \mapsto i_2 \mapsto \ldots i_k\mapsto i_1$. Also let $\iota_n$ be the identity permutation in $S_n$ and let $A_n^* = A_n \setminus \{\iota_n\}$ and $B_n^* = B_n \setminus \{\iota_n\}$.

Let's characterize permutations in $A_n^*$.

\begin{lem}\label{a_n}
A permutation $\pi \in A_n^*$ is a cycle of the form
\begin{equation*}
\pi = (1, i_1, i_2,\ldots, i_k)  \text{ with } 2 \leq i_1 < i_2 < \ldots < i_k \leq n
\end{equation*}
\end{lem}

\begin{proof}

Consider $\pi \in A_n^*$. Suppose $\pi(1) = 1$ then under the rules of the problem all other passengers can occupy their seat and $\pi =\iota_n$ which is a contradiction because $A_n^*$ doesn't have the identity permutation. So there exists a $i_1 \in \{2,\ldots,n\}$ with $\pi(1) = i_1$.  $i_1$ cannot map to any $j  <  i_1$ because under the rules of the problem every $j  < i_1$ maps to itself (every $j < i_1$ finds their seat unoccupied so they take it). So there exists a $i_2 \in \{2,\ldots,n\}$ with $i_2 >i_1$ and $i_1 \mapsto i_2$. And so on. This means that $\pi$ has at least the cycle $(1, i_1, i_2,\ldots, i_k)$ with $2 \leq i_1 < i_2 < \ldots < i_k \leq n$. It cannot have any other cycles that don't have $1$ in them because under the rules of the problem only passenger $1$ can start a seat rearrangement and all passengers not affected by that rearrangement will occupy their seat.

\end{proof}

\begin{defn}\label{func}
Let $2^{\{2,\ldots,n\}}$ be the set of all subsets of $\{2,\ldots,n\}$. The function $\varphi: 2^{\{2,\ldots,n\}} \rightarrow S_n$ is defined as:
\begin{equation*}
    \begin{split}
        \varphi(\oslash) & = \iota_n \\
        \varphi(\{i_1, i_2,\ldots, i_k\}) & =  (1, i_1, i_2,\ldots, i_k) \\
             \text{assuming } & 2 \leq i_1 < i_2 < \ldots < i_k \leq n
    \end{split}
\end{equation*}
\end{defn}

$\varphi$ is a valid function because for each subset there is only one cycle possible with the monotonically increasing ordering. From lemma \ref{a_n} it then follows that  $\varphi(2^{\{2,\ldots,n\}}) = A_n$, so $|A_n| = 2^{n -1}$.

For $B_n$ we apply the same arguments, except we take out the n-th passenger. A permutation $\pi' \in B_n^*$ is a cycle of the form

\begin{equation*}
\pi' = (1, i_1, i_2,\ldots, i_k)  \text{ with } 2 \leq i_1 < i_2 < \ldots < i_k \leq n -1
\end{equation*}

and there is a function $\varphi'$ defined as

\begin{equation*}
    \begin{split}
        \varphi'(\oslash) & = \iota_n \\
        \varphi'(\{i_1, i_2,\ldots, i_k\}) & =  (1, i_1, i_2,\ldots, i_k) \\
             \text{assuming } & 2 \leq i_1 < i_2 < \ldots < i_k \leq n -1
    \end{split}
\end{equation*}

that defines a bijection from $2^{\{2,\ldots,n - 1\}}$ to $B_n$. It means that $|B_n| = 2^{n - 2}$ for $n \geq 2$.

So the probability that the last ($n$th) person to board the plane will sit in their proper seat is $\frac{|B_n|}{|A_n|} = 0.5$ for $n \geq 2$.

\end{document}


% JuliaCon proceedings template
\documentclass{juliacon}
\setcounter{page}{1}
\usepackage{amsmath}

\begin{document}

\input{header}

\maketitle

\begin{abstract}
Pairwise learning is a machine learning paradigm where the goal is to predict properties of pairs of objects.
Applications include recommender systems, molecular network inference, and ecological interaction prediction.
Kronecker-based learning systems provide a simple yet elegant method to learn from such pairs.
Using tricks from linear algebra, these models can be trained, tuned, and validated on large datasets.
Our Julia package \texttt{Kronecker.jl} aggregates these shortcuts and efficient algorithms using a lazily-evaluated Kronecker product `$\otimes$', such that it is easy to experiment with learning algorithms using the Kronecker product.

\end{abstract}

\section{Background}

The Kronecker product, denoted by $\otimes$, between an $(n\times m)$ matrix $A=[A_{ij}]$ and an $(p\times q)$ matrix $B=[B_{kl}]$ is computed as
\begin{equation}\label{eq:kron}
  {A} \otimes  {B} ={\begin{bmatrix}A_{1,1} {B} &\cdots &A_{1,m} {B} \\\vdots &\ddots &\vdots \\A_{n,1} {B} &\cdots &A_{n,m} {B} \end{bmatrix}}\,.
\end{equation}
Simply put, the Kronecker product creates a new $(np\times mq)$ matrix containing all element-wise products between the respective elements of the two matrices.

Though conceptually simple, the Kronecker product gives rise to some elegant mathematics which allows performing many important computations, such as the eigenvalue decomposition, determinant or trace, in an efficient way~\cite{Sch2013,VanLoan2000}.
The Kronecker product has numerous applications in applied mathematics, for example in defining the matrix normal distribution, modeling complex networks~\cite{Leskovec2008} and pairwise learning~\cite{Stock2017tskrr}.
The reason that one can use the Kronecker product in large numerical problems is that (\ref{eq:kron}) often does not have to be computed explicitly, but it can be circumvented using various computational shortcuts.

\section{Basic use}

Our package aims to be a toolkit to effortless build Kronecker-based applications, where the focus is on the mathematics, and computational efficiency is taken care of under the hood. Essentially, it provides a lazily-evaluated Kronecker product of a \texttt{Kronecker} type.

%\begin{verbatim}
\begin{lstlisting}[language = Julia]
(n, m), (p, q) = (20, 20), (30, 30);  # A and B do not have to be square
A = rand(n, m); B = randn(p, q);
K = kronecker(A, B)  # lazy Kronecker product
\end{lstlisting}
%\end{verbatim}

Alternatively, one can make use of Unicode, i.e.  $\text{\texttt{K = A $\otimes$ B}}\,.$

The elementary functions of \texttt{LinearAlgebra} are overloaded to work with the respective subtypes of \texttt{GeneralizedKroneckerProduct} and provide the most efficient implementation.

\begin{lstlisting}[language = Julia]
tr(K)  # computed as tr(A) * tr(B)
det(K)  # computed as det(A)^n * det(B)^q
eigen(K)  # kronecker(eigen(A), eigen(B))
inv(K)  # yields a Kronecker instance
v = randn(600);
K * v  # computed using the vec trick
\end{lstlisting}

For example, the last line is evaluated using the so-called "vec trick"~\cite{VanLoan2000} with a time complexity of $\mathcal{O}(nm+pq)$ instead of $\mathcal{O}(nmpq)$ naively.
Similarly, efficiently solving large shifted Kronecker systems can be done directly as \texttt{eigen(A $\otimes$ B +$\lambda$I) $\backslash$ v}, exploiting the fast eigenvalue decomposition for Kronecker products.

Our package fully supports higher-order Kronecker products, e.g. \texttt{A $\otimes$ B $\otimes$ C}.
The structure \texttt{KroneckerPower} (for example constructed as \texttt{kronecker(A, 4)} for $A \otimes A \otimes A \otimes A$) and its methods provide efficient storage and manipulation of repeated Kronecker multiplications of the same matrix. We also provide the functionality to generate Kronecker graphs.

We provide support for dealing with submatrices of a Kronecker product through the sampled vec trick~\cite{Airola2017genvectric}.

\begin{lstlisting}[language = Julia]
# subsample a 200 x 100 submatrix of K
i, j = rand(1:n, 200), rand(1:m, 200);
k, l = rand(1:p, 100), rand(1:q, 100);
Ksubset = K[i,j,k,l];
u = randn(100);
Ksubset * u  # computed using sampled vec trick
\end{lstlisting}

\section{Prospects}

\texttt{Kronecker.jl} is a package in development. The developers are continuously adding new features.
To fully make use of the power of Julia, we will explore three directions.
Firstly, we will integrate libraries for automatic differentiation, such as \texttt{Zygote.jl}~\cite{Innes2019}.
This will allow for developing pairwise learning methods with complex loss and regularization functions. Secondly, we want to leverage the GPU support to make these methods scalable to large datasets using \texttt{CuArrays.jl}~\cite{Besard2019}.
Finally, we want to explore how symmetries and anti-symmetries can be incorporated, for example, when switching the order of two matrices would not change the result or only influences the sign of the result.
\input{bib.tex}

\end{document}

% Inspired by the International Journal of Computer Applications template


%---------------------------Minimum Angle-----------------------------
\section{Minimum Angle}

In order to properly compute the included angle, we'll need to
correct for incorrectly oriented elements.
Let
\[
s_i = \left\{ \begin{array}{ll}
  1\rule{2em}{0pt} & \alpha_i < 0\\
  0                & \alpha_i \geq 0
  \end{array}\right.
\]
The included angle between two neighboring edges is
\[
\theta_i = (-1)^{s_i} \arccos{ \left( - \frac {\vec L_{i} \cdot \vec L_{i+1} }
                                {\normvec{L_{i}} \normvec{L_{i+1}}} \right) }
                                  \left( \frac {180} {\pi} \right)
           + 360\dgr s_i
\]
where $i\in\{0,1,2,3\}$ and $\vec L_4 = \vec L_0$.
We take the minimum of this quantity as the value of the metric:
\[
q = \min_{i\in\{0,1,2,3\}}\left\{ \theta_i \right\}
\]

Note that if $\normvec{L_i} \leq DBL\_MIN$ or $\normvec{L_{i+1}} \leq DBL\_MIN$,
\verd\ returns $q = 360\dgr$.

\quadmetrictable{minimum included angle}%
{$A^1$}%                                    Dimension
{$[45\dgr,90\dgr]$}%                        Acceptable range
{$[0\dgr,90\dgr]$}%                         Normal range
{$[0\dgr,360\dgr]$}%                        Full range
{$90\dgr$}%                                 Unit square
{--}%                                       Citation
{v\_quad\_minimum\_angle}%                  Verdict function name


\subsection{part b}
I use a zero and a far pole to make controller feasible.

Controller:
$$
C(s) = \dfrac{2.2368\times10^5(s+11.91)}{s+10^4}
$$
Phase margin is above 40 degree.
\begin{figure}[H]
    \caption{Phase margin with controller}
    \centering
    \includegraphics[width=12cm]{../Figure/Q1/Q1_b/margin.png}
\end{figure}
Maximum closed loop is below than 3 decibels.
\begin{figure}[H]
    \caption{Nichols chart with controller}
    \centering
    \includegraphics[width=16cm]{../Figure/Q1/Q1_b/nichols.png}
\end{figure}
Setteing time and overshoot for step responde in closed loop system are shown in figure.
\begin{figure}[H]
    \caption{Step responde}
    \centering
    \includegraphics[width=12cm]{../Figure/Q1/Q1_b/step.png}
\end{figure}


In this section I explore four methods of dimensionality reduction on both of my datasets.

\subsection{Principal Components Analysis}\label{subsec:principal-components-analysis}
For principal component analysis I chose to vary the number of components to explain variance, and collected their resulting eigenvalues to better understand this DR technique.
Below is a table of results for both data sets.
\begin{center}
    \begin{tabular}{|c| c |c|}
        \hline
        & Minimum Components to Explain 99.9\% Variance & Eigenvalues                               \\
        \hline
        \hline
        Dataset 1 & 2                                             & [11320901200.524, 54484006.504]           \\
        \hline
        Dataset 2 & 4                                             & [65088.565, 37527.175, 4525.986, 106.278] \\
        \hline
    \end{tabular}
\end{center}
Two features explain 100\% of the variance in the data in the case of PCA for dimensionality reduction, and four features explain
99.9\% of the variance in the case of the second dataset.
In the case of dataset one the eigenvalues are very large which would seem to imply that the data are close together and
the are being multiplied by a large number to separate.
By contrast, data set two seems to have more components needed to explain the variance and each of those as a slightly smaller eigenvalue.
Figures~\ref{Fig:PCA DS1} and~\ref{Fig:PCA DS2} show the projection of the transformed data according to their principal
components for the first two features (to allow for plotting).
\begin{figure}
    \begin{minipage}{0.5\textwidth}
        \centering
        \includegraphics[width=.9\linewidth]{pcads1.png}
        \caption{DS1 Projection via PCA}\label{Fig:PCA DS1}
    \end{minipage}\hfill
    \begin{minipage}{0.5\textwidth}
        \centering
        \includegraphics[width=.9\linewidth]{pcads2.png}
        \caption{DS1 Projection via PCA}\label{Fig:PCA DS2}
    \end{minipage}
\end{figure}

\subsection{Independent Components Analysis}\label{subsec:independent-components-analysis}
Similar to the previous I also attempted to search for the best number of independent "source components" for each dataset.
According to Carsten Klein\cite{klein_2019}
"An interesting thing about two independent, non-Gaussian signals is that their sum is more Gaussian than any of the source signals.
Therefore we need to optimize W in a way that the resulting signals of Wx are as non-Gaussian as possible."
In other words, using a measurement like Kurtosis we can compare the "Gaussianity" of different ICA runs.
Since the normal distribution (Gaussian) has a kurtosis of 3, we are searching for ICA components with kurtosis of $<$ 3.
In order to reduce the covariance of the feature vectors as much as possible (to in effect ensure they were independent
components, I elected to whiten the feature space before processing.)
Below is a table depicting the n-components selected for each dataset and the resulting Kurtosis.
\begin{figure}
    \begin{minipage}{0.5\textwidth}
        \centering
        \includegraphics[width=.9\linewidth]{icads1.png}
        \caption{ICA Kurtosis vs RCError DS1}\label{Fig:ICA DS1}
    \end{minipage}\hfill
    \begin{minipage}{0.5\textwidth}
        \centering
        \includegraphics[width=.9\linewidth]{icads2.png}
        \caption{ICA Kurtosis vs RCError DS2}\label{Fig:ICA DS2}
    \end{minipage}
\end{figure}
Looking at figures~\ref{Fig:ICA DS1} and~\ref{Fig:ICA DS2} we can see the relationship between the number of components (the dimensions reduced to)
and both Kurtosis and Reconstruction Error (as measured by mean squared error).
Similar to PCA, for small values of n\_components reconstruction error quickly approaches zero where the kurtosis gradually
rises for each additional component.
This would seem to suggest that for $n=2$ (Dataset 1) and $n=4$ (Dataset 2) we reach the optimal tradeoff between treating
each source attribute "as a seperate mixed signal" and reducing RC error (the obvious bias in DR being that we want n to be as
small as possible.)
With respect to each dataset the meaningfulness of this result is that it is possible to reconstruct the distribution with
significantly less data than the many more attributes which generated the original.

\subsection{Randomized Projection}\label{subsec:randomized-projection}
Randomized projections allowed me to generate random dimensionality reduction transformations from the data.
I chose to repeat this process 10 times for each dataset and to utilize a Gaussian Randomized projection such that each
feature was itself equally likely to be a part of the final dimensionally reduced output.
Along with repeating each experiment I chose to vary the number of components as I did in other experiments to search
for an optimal reconstruction error.
Perhaps unsurprisingly looking at figures~\ref{Fig:RP DS1} and~\ref{Fig:RP DS2} we can see that for larger values of
\texttt{n\_components} the reconstruction error was lower.
The oscillations in the graph can be explained by the fact that the random projections can increase or decrease the RCError
by small amounts.
There was a significant amount of variance between runs as shown in the table~\ref{RP Var}.
Dataset 1 had much more variance than dataset 2 during randomized projection runs.
This is most likely due to the 30+ additional features in the space for which a random selection/transformation needed
to be made.
\begin{center}\label{RP Var}
    \begin{tabular}{|c| c |c|}
        \hline
        & Variance               & Standard Deviation \\
        \hline
        \hline
        Dataset 1 & 5.4456097845336664e+16 & 233358303.57057506 \\
        \hline
        Dataset 2 & 34540925361.18495      & 185851.89092711688 \\
        \hline
    \end{tabular}
\end{center}
\begin{figure}
    \begin{minipage}{0.5\textwidth}
        \centering
        \includegraphics[width=.9\linewidth]{rpds1.png}
        \caption{RCError RP DS1}\label{Fig:RP DS1}
    \end{minipage}\hfill
    \begin{minipage}{0.5\textwidth}
        \centering
        \includegraphics[width=.9\linewidth]{rpds2.png}
        \caption{RCError RP DS1}\label{Fig:RP DS2}
    \end{minipage}
\end{figure}

\subsection{Linear Discriminant Analysis}\label{subsec:linear-discriminant-analysis}
For my final dimensionality reduction technique, I chose to use linear discriminant analysis.
Due to the fact that each of my problems are binary classification, and while not linearly seperable (otherwise accuracy would be better,)
a linear decision boundary with Bayesian conditional density would allow me to reduce the dimensionality of the dataset similar
to PCA in the direction of the largest discriminant.
However the difference here is the introduction of a priori information in the form of class labels.
The number of components had to be $min(num\_features, unique\_labels - 1)$ which in my case was $1$ for both datasets given
that they were both binary classification problems.
Since LDA takes into account class labels I chose (similar to classifiers) to split the data between a test/training set (25/75\% respectively.)
After the transformation I opted to classify the remainder of instances, below is the result on test data.
\begin{center}
    \begin{tabular}{|c| c |}
        \hline
        & Accuracy \\
        \hline
        \hline
        Dataset 1 & 82.86\%  \\
        \hline
        Dataset 2 & 91.17\%  \\
        \hline
    \end{tabular}
\end{center}
As you can see these results are comparable to the accuracy scores of some of the better trained classifiers from assignment 1.
This I believe speaks to the power of LDA being able to consider apriori information in constructing the DR projection.
Unfortunately given that there was only one value for \texttt{n\_components} I cannot compare reconstruction error over time,
but given the positive results, I don't think that the reconstruction error would be very high.


% !TeX root = thoughts.tex
\section{Simple linear systems}

\subsection{Introduction with a 2 parameter system}
In a first analysis a simple linear system is considered, which allows us to get a grip on what is happening behind the scenes. A first analysis will be done on a two dimensional system given by the following linear system;
\begin{gather}\label{eq:linear_system.system}
	\begin{pmatrix}
	d_1\\
	d_2
	\end{pmatrix}
	=
	\begin{pmatrix}
	1 & 0\\
	0 & 2\\
	\end{pmatrix}
	\begin{pmatrix}
	q_1\\q_2
	\end{pmatrix}.
\end{gather}
For our synthetic data I choose $q_1 = 1$ and $q_2 = 3$. Prior information and measurement uncertainty all have an impact on posterior exploration, but for now I will set some arbitrary values. Our prior I synthetically set to have all means of 2 and standard deviations of 1. Measurement errors are chosen to be uncorrelated and having a standard deviation of 0.5. The influence of the two covariance matrices on the misfit functional are given in Figure~\ref{fig:linear_system.prior_influence}.

\begin{figure}
	\centering
	\begin{subfigure}{.5\textwidth}
		\centering
		\includegraphics[width=\textwidth]{figures/linear_systems/gradient_2d_narrow_prior}
		\caption{Gradient with standard deviation $\sigma = 1$}
		\label{fig:linear_system.prior_influence.narrow}
	\end{subfigure}%
	\begin{subfigure}{.5\textwidth}
		\centering
		\includegraphics[width=\textwidth]{figures/linear_systems/gradient_2d_wide_prior}
		\caption{Gradient with standard deviation $\sigma = 5$}
		\label{fig:linear_system.prior_influence.wide}
	\end{subfigure}
	\caption{Normalized gradient of 2D misfit functionals with differently confined prior information. The means of the prior are for both parameters 2. It is obvious that decreasing the standard deviation for all parameters means that we increase the importance of the prior over the data, `pulling' the minimal gradient more towards the prior mean. Similar effects can be attained with increasing the magnitude of the data covariance matrix. Together, these matrices finely tune the minimum of the misfit function. Choosing well informed data and parameter covariace matrices is essential to meaningful \gls{HMC} sampling. In this example, the data covariance was uniformly 0.5.}
	\label{fig:linear_system.prior_influence}
\end{figure}

\index{Trajectories}For illustration purposes, 50 time steps were taken with a stepsize of 0.05 in time units (actual units of time might be a bit meaningless, without defining the units of length and mass). Using the unit mass matrix now generates trajectories like the one described in Figure~\ref{fig:linear_system.trajectory_simple}.

\begin{figure}
	\centering

	\includegraphics[width=0.5\textwidth]{figures/linear_systems/trajectory_simple}

	\caption{Untuned trajectory of the simple linear system described in Equation~\eqref{eq:linear_system.system}. The color of the trajectory points represents the normalized misfit, with red being the highest value, and dark blue being the lowest. Note that the direction of the trajectory can not be determined upon inspection of the trajectory itself; Hamilton's equation are invariant under time reversal. A particle would traverse exactly the same trajectory in reverse if at the end it's momentum would be reversed. The trajectory is superimposed on the gradient of the misfit functional $\chi$, which acts as the direction of the largest increase in gravitational potential. A particle starting without momentum would start to roll in opposite direction of these vectors, eventually orbiting the point of zero gradient (without energy loss it can never reach a steady state in the point of minimum energy).}
	\label{fig:linear_system.trajectory_simple}
\end{figure}

\paragraph{Mass matrix optimization}\index{Mass matrix}\index{Trajectories!Tuning the}There are some undesirable characteristics of this trajectory. Due to the mass matrix being a diagonal unit matrix, but the misfit functional being elongated along the dimension of parameter $q_1$ the oscillations in either dimension do not have the same period. Using a mass matrix based on Equation~\eqref{eq:massMatrixForward} (which is still diagonal with the current forward model) will equalize oscillations. The reason this behavior is desired is that this way we see as many different energy levels in every dimension. The result of assigning the mass matrix based on the trace of $\MatrixVariable{G}^T\MatrixVariable{G}$ is seen in Figure~\ref{fig:linear_system.trajectory_massTuned}. The resulting mass matrix is ~
\begin{gather}\label{eq:linear_system.massMatrix}
\MatrixVariable{M} =
\begin{pmatrix}
1 & 0\\
0 & 4\\
\end{pmatrix}.
\end{gather}

\begin{figure}
	\centering

	\includegraphics[width=0.5\textwidth]{figures/linear_systems/trajectory_massTuned}

	\caption{Trajectory tuned with mass matrix, according to the simple linear system described in Equation~\eqref{eq:linear_system.system}. The color of the trajectory points represents the normalized misfit, with red being the highest value, and dark blue being the lowest. With this augmented mass matrix, oscillations are equal in duration for each dimension.}
	\label{fig:linear_system.trajectory_massTuned}
\end{figure}

\paragraph{No U-Turn Criterion}\index{Trajectories!Trajectory length}\index{Trajectories!Tuning the}\index{No U-Turn Criterion}Visible now in Figure~\ref{fig:linear_system.trajectory_massTuned} is another characteristic of untuned trajectories. What would be ideal is that the algorithm explores the model space as efficiently as possible. The trajectory depicted in the figure traverses the model space around the minimum fully, but also start to come back to the original position. This so called `U-Turn' behavior can be mitigated by terminating the trajectory as soon as one detects the propagated model coming closer to the initial model. This point is reached as
\begin{gather}
	  \mathbf{v}(t) \cdot \left[ \mathbf{m}(t) - \mathbf{m}_0 \right] < 0, \quad
	 \text{and} \quad
	 \mathbf{v}_0\cdot \left[ \mathbf{m}_0 - \mathbf{m}(t) \right] < 0.
\end{gather}
This means that as soon as the momenta vectors for both the beginning and end of the trajectory both make an angle of less than 90 degrees with the vector connecting the points, so if the two models are `moving towards' each other, the trajectory is terminated. An illustration of a terminated trajectory is given in Figure~\ref{fig:linear_system.trajectory_uTurn}. By limiting the trajectories using this criterion, and additionally increasing the step size so only a few samples are needed to traverse to model space will effectively reduce computational costs of proposing new models.

\begin{figure}
	\centering

	\includegraphics[width=0.5\textwidth]{figures/linear_systems/trajectory_uTurn_29_samples}

	\caption{Trajectory tuned with the \textbf{No U-Turn Criterion} and forward model based mass matrix. The color of the trajectory points represents the normalized misfit, with red being the highest value, and dark blue being the lowest. As soon as the momentum of the initial point and the last point of the trajectory point `towards' eachother the trajectory is terminated. In this case, 29 samples were made before the trajectory was terminated.}
	\label{fig:linear_system.trajectory_uTurn}
\end{figure}

\paragraph{Stepsize of the trajectories}\index{Trajectories!Tuning the}\index{Trajectories!Stepsize} The step size is another important tuning parameter. Wasting many computation on many steps is wasteful if the `particle' will end up in the same place, as was done in previous illustrations. According to \cite{neal2011mcmc} the maximum step size is defined by the minimum standard deviation of momentum, or square root of the mass matrix.

\paragraph{Results}
Using optimized parameters, given in Table~\ref{tab:naive_params}, multiple inversions were performed using \gls{HMC}. The used stepsize is well below the upper limit of stability predicted by the mass matrix, according to \cite{neal2011mcmc}. As can be seen in Figure~\ref{fig:linear_system.accepted}, over many iterations the amount of accepted models evens out to approximately 66\%. I'll call this method `naive' for now, for reasons which will become apparent later.

\begin{table}[]
\centering
\begin{tabular}{l|l}
 Number of timesteps & 50 \\
 Length of timestep & 0.05 \\
 Predicted stability &  0.5
\end{tabular}
\caption{Inversion parameters for `naive' trajectory settings.}
\label{tab:naive_params}
\end{table}

In Figure~\ref{fig:linear_system.histq2} one sees what we'd expect to see with a linear model with Gaussian uncertainties. After more and more iterations the marginal posterior probability density functions approaches a normal distribution. It's actual mean in the last iteration is $\mu_1 = 2.941$, close to the original parameter. This value is reached to within 1\% as soon as 500 iterations. The actual development of the value is given in Figure~\ref{fig:linear_system.evolution_q2}. This all seems like ideal behaviour, our sampler is able to get meaningful statistic from our dataset after only 500 iterations, and after 50,000 iterations we have a very nice looking posterior.

When one however examines not only the marginal posterior of parameter 2, but also the marginal posterior of parameter 1 (Figure~\ref{fig:linear_system.histq1}) and the full two dimensional posterior (Figure~\ref{fig:linear_system.hist2d}) unexpected characteristics are found. It seems that parameter 1 has a bimodal distribution, while none of our input prior information nor forward model suggests such behaviour.

\begin{figure}
	\centering
	\includegraphics[width=\textwidth]{figures/linear_systems/2d_naive/accepted.pdf}
	\caption{Acceptance rates of the 'naive' sampling.}
	\label{fig:linear_system.accepted}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[width=\textwidth]{figures/linear_systems/2d_naive/histogram_p2.pdf}
	\caption{Histogram for marginal probability density of parameter 2 after `naive' HMC sampling using different amount of
	samples.}
	\label{fig:linear_system.histq2}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[width=0.75\textwidth]{figures/linear_systems/2d_naive/means2.pdf}
	\caption{Evolution of mean of parameter 2 as the number of samples increases during the `naive' sampling.}
	\label{fig:linear_system.evolution_q2}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[width=\textwidth]{figures/linear_systems/2d_naive/histogram_p1.pdf}
	\caption{Histogram for marginal probability density of parameter 1 after `naive' HMC sampling using different amount of
	samples.}
	\label{fig:linear_system.histq1}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[width=\textwidth]{figures/linear_systems/2d_naive/histogram_2d.pdf}
	\caption{Full 2D marginal probability density after `naive' HMC sampling using different amount of
	samples.}
	\label{fig:linear_system.hist2d}
\end{figure}

\afterpage{\clearpage}

Upon closer inspection of the random walk trajectory done after 50,000 iterations (Figure~\ref{fig:linear_system.randomWalk50000naive}), there seems to be some bias in proposing new models. What happened in this case which spefically leads to the bimodal distribution is theorized as follows; the prior information always leads to propose the first model close to $\mathbf{q} = (2,2)^T$. This starting point coupled with the rather long trajectory length of 50 samples and still rather large stepsize, result in every iteration traversing almost the complete model space.

This, in addition with the No U-Turn Criterion, makes these trajectories `resonate', always exploring the same portion of the model space. If a model is launched away from the misfit minimum, it wil return towards its original position and then be killed by the No U-Turn Criterion. If it is launched in a relatively beneficial direction. it will traverse a lot of space and turn at the opposite side of the model space. I would not go as far as to call it \index{Hysteresis}hysteresis, but rather a fundamental result of the \index{Tuning parameters}tuning parameters.

There's two ways to test this hypothesis. First, the random walk using a different prior is analysed. Now the prior means are chosen at exactly the values of the parameters used for the synthetics. The result, given in Figure~\ref{fig:linear_system.improvedmean}, is that sampling is a little bit improved. The sampling is more uniform in the 2D space, but there is still some kind of `orbiting' behaviour, the sampler never reaches the minimum potential. This is because all trajectories passing through this point are not yet terminated. Making trajectories even longer won't work, because the No U-Turn Criterion will terminate them anyway. This leads us to testing another method of avoiding \index{Biased sampling}biased sampling; shorter trajectories.

\index{Trajectories!Trajectory length}I think that this is a better solution. By decreasing total trajectory length, one does not need to tune actual physical intuition or information (the prior) to the needs of the algorithm. Recognizing this behaviour in non-linear or high dimensional systems might prove very challenging upon inspection of marginal probability distributions. One quality check one could do, even during sampling, is to monitor how often the No U-Turn Criterion triggers the termination of a trajectory, relative to total models ran. The promising result is given in Figure~\ref{fig:linear_system.improvedtrajectory}.

\begin{figure}
	\centering
	\includegraphics[width=0.66\textwidth]{figures/linear_systems/2d_naive/randomWalk_50000.png}
	\caption{Random walk during 50,000 `naive' \gls{HMC} samples.}
	\label{fig:linear_system.randomWalk50000naive}
\end{figure}

\begin{figure}
	\centering
	\begin{subfigure}{.43\textwidth}
		\centering
		\includegraphics[width=\textwidth]{figures/linear_systems/2d_improved/mean_histogram_2d}
		\caption{Full 2D posterior}
		\label{}
	\end{subfigure}%
	\begin{subfigure}{.57\textwidth}
		\centering
		\includegraphics[width=\textwidth]{figures/linear_systems/2d_improved/mean_randomWalk}
		\caption{Random walk sequence}
		\label{}
	\end{subfigure}
	\caption{50,000 samples of the same 2D posterior as Figure~\ref{fig:linear_system.hist2d}, tuned with a different starting prior. Note that the random walk still traverse most of the model space in each proposal, leading again to `orbiting' of the minimum.}
	\label{fig:linear_system.improvedmean}
\end{figure}

\begin{figure}
	\centering
	\begin{subfigure}{.43\textwidth}
		\centering
		\includegraphics[width=\textwidth]{figures/linear_systems/2d_improved/short_histogram_2d}
		\caption{Full 2D posterior}
		\label{}
	\end{subfigure}%
	\begin{subfigure}{.57\textwidth}
		\centering
		\includegraphics[width=\textwidth]{figures/linear_systems/2d_improved/short_randomWalk}
		\caption{Random walk sequence}
		\label{}
	\end{subfigure}
	\caption{50,000 samples of the same 2D posterior as Figure~\ref{fig:linear_system.hist2d}, tuned with a much smaller trajectory length. Note that now each new model only has a small distance from the previous, but that the model space is much more evenly sampled.}
	\label{fig:linear_system.improvedtrajectory}
\end{figure}

\subsection{Performance of higher dimensional systems}

In this section I want to look at the influence of dimensionality; how it affects computation time and acceptance rates. A general forward model is presented for $n$-dimensions;

\begin{gather}
	\MatrixVariable{G} =
	\begin{bmatrix}
		1\cdot 1 &   &        & \\
		  & 2\cdot 2 &        & \\
		  &   & \ddots & \\
		  &   &        & n \cdot n
	\end{bmatrix}
\end{gather}
\paragraph{A note on the mass matrix}\index{Mass matrix!revised}The mass matrix was initially again based upon the trace of $\MatrixVariable{G}^T\MatrixVariable{G}$. This \textbf{did not} produce equally oscillating trajectories with models larger than 2 dimension and varying forward models. After some trial and error, and revisiting the theory, an extension upon the mass matrix is made. In the aforementioned reader, the covariance matrices were not analyzed in the general solution. By substituting the `new' $\MatrixVariable{A}$ and $\mathbf{b}$ from equations \eqref{eq:linear_system.misfit_A} and \eqref{eq:linear_system.misfit_b}, we see that a more optimized mass matrix could be constructed from (the trace of) $\MatrixVariable{A}$ like so;
\begin{align}\label{eq:extendedMassMatrix}
	\MatrixVariable{M} = \MatrixVariable{C}_M^{-1} + \MatrixVariable{G}^T \MatrixVariable{C}_D^{-1}\MatrixVariable{G}.
\end{align}
This deficit was not apparent in the analysis of the 2 dimensional system probably due to the small difference in forward matrix components and parameters.

One important note is that this matrix is required to be positive definite to allow us to sample the full model space. If the $n\times n$ matrix $\MatrixVariable{M}$ is not of full rank, the column space is not completely $\mathbb{R}^n$, and one does not explore the `interesting' part of our model space \textbf{I want to refine this wording, be a bit more specific}. Now, one does not generally now much about the properties $\MatrixVariable{G}^T$ if the system is very large. We'll also see later that if the mass matrix is not positive definite, the suggested code won't run since the required matrix decomposition does not exist.

Usually, we can see that a system is under-determined or mixed-determined, but it is not easily verifiable if the system is over-determined (i.e. full rank). In this sense, the prior information in parameter space acts as a stabilizer.

Positive definiteness is defined by;

\begin{align}
	\mathbf{x}^T \MatrixVariable{A}\; \mathbf{x} > 0, \quad \forall\, \mathbf{x} \neq \mathbf{0}
\end{align}

We first assume that our prior covariance for both the data and parameters are positive definite. I think for data covariances this describes independence of measured values, and the same goes for parameter covariances. Positive definiteness ensures that matrices can be inverted, and that their inverse is also positive definite.

Now, we can easily show that the second term of the right hand side of Equation~\eqref{eq:extendedMassMatrix} is at least positive semi-definite;

\begin{align}
	\mathbf{x}^T \MatrixVariable{G}^T \MatrixVariable{C}_D^{-1}\MatrixVariable{G} \mathbf{x} =\;
	\left( \MatrixVariable{G} \mathbf{x} \right)^T \MatrixVariable{C}_D^{-1}\MatrixVariable{G} \mathbf{x} \geq
	0, \quad \forall\, \mathbf{x} \neq \mathbf{0}
\end{align}
The reason this semi-definiteness creeps in is that one can't be sure that $\MatrixVariable{G}$ doesn't map $\mathbf{x}$ to $\mathbf{0}$, since we don't know the dimension of it's null space. This is directly related to a system being under- or mixed determined, as $\text{rank}(\MatrixVariable{G})+\text{nullity}(\MatrixVariable{G}) = n$. If a system is under-determined, the second term is positive semi-definite. If one knows that $\MatrixVariable{G}$ is of full rank, then the resulting second term is positive definite.

\index{Mass matrix!positive definite}This is also where our prior knowledge comes in, and acts as a stabilizer when the data prior is not sufficient for an inversion. If given two matrices for which hold $\mathbf{x}^T \MatrixVariable{A}\, \mathbf{x} > 0, \quad \forall\, \mathbf{x} \neq \mathbf{0}$ and $\mathbf{x}^T \MatrixVariable{B}\, \mathbf{x} \geq 0, \quad \forall\, \mathbf{x} \neq \mathbf{0}$ then the following holds:

\begin{align}
	\mathbf{x}^T \left( \MatrixVariable{A} + \MatrixVariable{B} \right) \mathbf{x} =\;
	\mathbf{x}^T  \MatrixVariable{A}  \mathbf{x} + \mathbf{x}^T \MatrixVariable{B}  \mathbf{x} >
	0, \quad \forall\, \mathbf{x} \neq \mathbf{0},
\end{align}
thus, proving that the mass matrix is always positive definite if $C_M^{-1}$ and $C_D^{-1}$ are too.

\paragraph{Performance increase}\index{Algorithm performance}At this stage I also realized that there are two main points on which we can increase performance. The code at this stage was sped up threefold by compiling using any of the -O\# g++ flags. Any specific compiler flag (-O1 through -O4) didn't noticeably alter performance, mostly invisible due to the random nature of the sampling. I advise therefore to use -O2, the most tested mode, which alters your code the least. Also, RAM usage can be easily decreased by limiting the amount of std::vector$<>$ copies. Every time one of those objects is passed and used only once, (near) perfect forwarding can be achieved using std::move().

\paragraph{Inversion characteristics}The step size of any model is fixed at 0.05, since the first dimension provides the upper bound to the stability. For the trajectory, 10 different iterations are done, or whenever the U-Turn criterion is met. In total, 50,000 samples are drawn. Prior information for every parameter is set at 10, with a standard deviation of 5. This means that as the dimensionality increases, the prior becomes worse with respect to the actual parameter. Data covariance is fixed at 10 percent of the observed value, as to negate it's influence on the algorithm as much as possible.

\paragraph{Results}\index{Algorithm performance!runtime}The computation time versus number of parameters exhibits an almost linear relationship, as seen in Figure~\ref{fig:linear_system.high_dim.time}. The acceptance rate seems to suffer from the increasing dimensions, appearing inversely proportional to the amount of parameters (Figure~\ref{fig:linear_system.high_dim.acceptance}). The results are rather good, but decreasing in quality as the number of accepted models decreases. This performance might be improved by actually drawing correlated samples from the mass matrix (I advise Cholesky decomposition).

\begin{figure}
	\centering
	\includegraphics[width=0.8\textwidth]{figures/linear_systems/higher_dim/performance}
	\caption{Computation time of 50,000 samples for a varying amount of parameters.}
	\label{fig:linear_system.high_dim.time}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[width=0.8\textwidth]{figures/linear_systems/higher_dim/acceptance_rate}
	\caption{Acceptance rate of 50,000 samples for a varying amount of parameters.}
	\label{fig:linear_system.high_dim.acceptance}
\end{figure}


\documentclass[11pt]{article}
\title{Math 20D: Ordinary Differential Equations}
\author{Suhas Arehalli}

\usepackage[margin=1.0in]{geometry}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{mathrsfs}

\begin{document}
\maketitle

\section{1st Order ODEs}
\subsection{Integrating Factor}
\subsubsection{Given}
A 1st order linear differential equation of the form
    \[ \frac{dy}{dt} + p(t)y = g(t) \]
\subsubsection{Technique}
Begin by calculating the integrating factor (feel free to ignore the constant of integration - it's
irrelevant),
    \[ \mu (t) = e^{\int p(t)dt} \]
Then multiply both sides by it
\begin{align*}
    \frac{dy}{dt} e^{\int p(t)dt} + p(t)e^{\int p(t)}y &= g(t)e^{\int p(t)}  \\
    \frac{dy}{dt} \mu (t) + \frac{d\mu}{dt}y &= g(t)\mu (t)
\end{align*}
Note that the right-hand side is just $\frac{d}{dt}[y \mu (t)]$
\begin{align*}
    \frac{d}{dt}[y \mu (t)] &= g(t)\mu (t) \\
    y \mu(t) &= \int g(t)\mu (t)dt \\
    y &= \frac{\int g(t) \mu (t)dt}{\mu (t)}
\end{align*}

\subsection{Separable Equations}
\subsubsection{Given}
A 1st order differential equation of the form
    \[ M(x) + N(y)\frac{dy}{dx} = 0 \]
These are called \textbf{separable}.
\subsubsection{Technique}
Separate your variables
    \[ N(y)\frac{dy}{dx} = M(x) \]
And integrate
    \[ \int N(y)dy = \int M(x)dx \]
And then solve for y.

\subsection{Exact Equations}
\subsubsection{Given}
A 1st order differential equation of the form
    \[ M(x,y) + N(x,y)\frac{dy}{dx} = 0 \]
Such that
\begin{align*}
    \frac{d}{dy}[M(x,y)] = \frac{d}{dx}[ N(x,y)]
\end{align*}
\subsubsection{Technique}
Calculate $\psi (x,y)$ from
\begin{align*}
    \psi (x,y) = \int M(x,y)dx + C_1(y) \\
    \psi (x,y) = \int N(x,y)dy + C_2(x)
\end{align*}
There should be one function that satisfies both of these equations with varying functions $C_1(x)$ and
$C_2(y)$: That is $\psi (x,y)$. Then we can rewrite the differential equation as
\begin{align*}
\frac{d\psi}{dx} + \frac{d\psi}{dy}\frac{dy}{dx} &= 0 \\
\frac{d}{dx}[\psi (x,y)] &= 0\\
\psi (x,y) &= C
\end{align*}
Which we can proceed to algebraically solve for y.

\section{Second Order ODEs}
\subsection{The Wronskian}
In first order ODEs, we know there is only one solution to each Differential Equation (plus all of it's
scalar multiples). With 2nd Order ODEs, we expect 2 linearly independent solutions. How do we determine
if these solutions are independent? By calculating the Wronskian:
\[
    W = det\begin{bmatrix}
            y_1  & y_2 \\
            y_1' & y_2'\\
            \end{bmatrix}
\]
If $ W \neq 0 $, then the 2 solutions are linearly independent enough.
\subsection{Characteristic Equation }
\subsubsection{Given}
A homogenous 2nd order differential equation of the form
\[ y'' + ay' + b = 0 \]
\subsubsection{Technique}
Assume our solution is of the form $y = e^{rt}$. Substituting it in gives us
\begin{align*}
    r^2e^{rt} + are^{rt} + be^{rt} &= 0 \\
    (r^2 + ar + b)e^{rt} &= 0
\end{align*}
However, since $e^{rt} \neq 0, \forall r \in \mathbb{R}$, we can divide by
$e^{rt}$ and obtain the characteristic equation
    \[ r^2 + ar + b = 0 \]
and solve for the $r_1, r_2$ that satisfy the equation (the roots of the polynomial on the LHS). Depending
on the kind of roots obtained, follow one of the techniques below:
\subsubsection{Technique: Real and Unique Roots}
In this case, your solution is simply
    \[ y = C_1e^{r_1t} + C_2e^{r_2t} \]
Why? If you followed along above, you see that we assumed that solutions would be of the form $y = e^{rt}$.
However, since there are 2 solutions for that equations, by the Law of Superposition, every linear
combination of those solutions is also a solution.
\subsubsection{Technique: Complex Roots}
Here we can say that out solution is simply the same as above. However, we don't care about imaginary
solutions: The DE was real-valued, so so should our solution. Thus, we will inspect one solution (which
is complete, as we could show with the Wronskian) and remove the imaginary terms. Consider
\begin{align*}
    y = Ce^{rt} &= Ce^{(a+bi)t} \\
    &= Ce^{at}e^{(bt)i} \\
    &= Ce^{at}(cos(bt) + isin(bt)) \\
    &= (C_1 + C_2i)e^{at}(cos(bt) + isin(bt)) \\
    &= e^{at}(C_1cos(bt) - C_2sin(bt)) + ie^{at}(C_1sin(bt) + C_2cos(bt))\\
\end{align*}
If we swap the sign on $C_2$ and drop the imaginary part, we obtain our real solution
\[ y = e^{at}(C_1cos(bt) + C_2sin(bt)) \]
\subsubsection{Technique: Repeated Roots}
If we only have a single root, we can simply say the answer is
\[ y = C_1e^{rt} + C_2te^{rt} \]
Why? This is a special case of the next technique, Reduction of Order.
\subsection{Reduction of Order}
\subsubsection{Given}
A 2nd order linear DE of the form
    \[ y'' + p(t)y' + q(t)y = 0 \]
and a single solution $y_1(t)$.
\subsubsection{Technique}
Assume that
    \[ y_2(t) = v(t)y_1(t) \]
for some function $v(t)$. Our goal is to find this function $v(t)$, and thus find $y_2(t)$. Now
simply substitute into the original equation, first finding that
\begin{align*}
    y_2(t)   &= v(t)y_1(t) \\
    y_2'(t)  &= v'(t)y_1(t) + v(t)y_1'(t) \\
    y_2''(t) &= v''(t)y_1(t) + 2v'(t)y_1'(t) + v(t)y_1''(t)
\end{align*}
And then substiting fully, getting
\begin{align*}
    \left[v''(t)y_1(t) + 2v'(t)y_1'(t) + v(t)y_1''(t)\right] + p(t)\left[v'(t)y_1(t) + v(t)y_1'(t)\right] + q(t)\left[v(t)y_1(t)\right]  = 0 \\
    \left[v''(t)y_1(t) + 2v'(t)y_1'(t) \right] + p(t)\left[v'(t)y_1(t)\right] + v(t)\left[y_2''(t) + p(t)y_1'(t) + q(t)y_1(t)\right]  = 0 \\
    \left[v''(t)y_1(t) + 2v'(t)y_1'(t) \right] + p(t)\left[v'(t)y_1(t)\right] = 0 \\
\end{align*}
At this point, since all terms with v(t) were eliminated, we can define a new variable $w(t) = v'(t)$. Now
We're left with
    \[ \left[w'(t)y_1(t) + 2w(t)y_1'(t) \right] + p(t)\left[w(t)y_1(t)\right] = 0 \]
And since $y_1(t)$ is known, we have a 1st order DE that we can solve using the previous methods, finding
$w = v'(t)$, from which we can find $v(t) = \int w(t)dt$, which in turn lets us find $y_2(t) = v(t)y_1(t)$.
\subsection{Method of Undetermined Coefficients}
\subsubsection{Given}
A second order nonhomogenous ODE of the form
    \[ y'' + p(t)y' + q(t)y = g(t) \]
with a g(t) that is a sum/product of sin/cos, polynomials, or exponentials,
plus homogenous solutions $y_1(t)$ and $y_2(t)$ that satisfy
    \[ y'' + p(t)y' + q(t)y = 0 \]
We could obtain these with the other homogenous 2nd order techniques discussed above.
\subsubsection{Technique}
We need to add an additional term, $Y(t)$ to our homogenous solution to account for the function $g(t)$
on the RHS. To find $Y(t)$, first, guess a generalized version of $g(t)$: Replace an nth order polynomial
with $\sum_{i=0}^n A_it^i$, sine or cosine with $Acos(Bt) + Csin(Dt)$, and any exponential with
$Ae^{Bt}$.

Then find $Y'(t), Y''(t)$ and substitute into the differential equation. The equate all of the coefficients
of equal terms, obtaining a system of linear equations which you can proceed to solve for the undetermined
coefficients you got from "generalizing" when guessing $Y(t)$. Then our final solution will be
    \[ y = y_1(t) + y_2(t) + Y(t) \]
Sometimes the system we find will have no solutions. In this case, we can multiply any $Ae^{Bt}$ terms
by $t$ and try again, and if that doesn't work repeat. Don't ask me why this works. \textbf{TIP}: If something
of the form $Ae^{Bt}$ appears in the homogenous solution, multiply by t. If something of the form $Ate^{Bt}$
appears in the homogenous solution, multiply by t again.
To recap:
\begin{enumerate}
    \item Construct a guess Y(t)
    \item Differentiate Y(t) and substitute into the DE
    \item Equate coefficients of similar terms, getting a system of linear equations for the coefficients
    \item Solve the system to get the coefficients
    \item If the system has no solutions, multiply $Ae^{Bt}$ terms by t and go back to 2.
    \item Write the final solution
\end{enumerate}
\subsection{Variation of Parameters}
\subsubsection{Given}
A second order nonhomogenous ODE of the form
    \[ y'' + p(t)y' + q(t)y = g(t) \]
with homogenous solutions $y_1(t)$ and $y_2(t)$ that satisfy
    \[ y'' + p(t)y' + q(t)y = 0 \]
We could obtain these with the other homogenous 2nd order techniques discussed above.
\subsubsection{Technique}
Again we begin by making an assumption about $Y(t)$:
    \[ Y(t) = u_1(t)y_1(t) + u_2(t)y_2(t) \]
From this, we again differentiate
\begin{align*}
    Y(t) &= u_1(t)y_1(t) + u_2(t)y_2(t) \\
    Y'(t) &= u_1(t)y_1'(t) + u_1'(t)y_1(t) + u_2(t)y_2'(t) + u_2'(t)y_2(t) \\
\end{align*}
And now, since that looks disgusting, and because we have the freedom to, let's impose an additional constraint:
\begin{equation}
u_1'(t)y_1(t) + u_2'(t)y_1(t) = 0  \\
\end{equation}
which gets us
\begin{align*}
    Y'(t) &= u_1(t)y_1'(t) + u_2(t)y_2'(t)\\
    Y''(t) &= u_1'(t)y_1'(t) + u_1(t)y_1''(t) + u_2'(t)y_2'(t) + u_2(t)y_2''(t) \\
\end{align*}
Now substitute into the DE, getting
\begin{eqnarray*}
\begin{split}
    [ u_1'(t)y_1'(t) + u_1(t)y_1''(t) + u_2'(t)&y_2'(t) + u_2(t)y_2''(t)] \\
    + p(t)[ u_1(t)y_1'(t) &+ u_2(t)y_2'(t)]  \\
    + &q(t)\left[ u_1(t)y_1(t) + u_2(t)y_2(t) \right] = g(t)
\end{split}
\end{eqnarray*}
\begin{eqnarray*}
\begin{split}
   u_1(t)[y_1''(t) + p(t)y_1'(t) + q(t)y_1(t)&] \\
   + u_2(t)[y_2''(t) + p(t)&y_2'(t) + q(t)y_2(t)] \\
   &+ u_1'(t)y_1'(t) + u_2'(t)y_2'(t) = g(t)
\end{split}
\end{eqnarray*}
\begin{equation}
    u_1'(t)y_1'(t) + u_2'(t)y_2'(t) = g(t) \\
\end{equation}
This paired with (1) gives us a system of equations we can use to solve for $u_1'(t)$ and $u_2'(t)$,
giving us
\begin{align*}
    u_1'(t) &= -\frac{y_2(t)g(t)}{W(y_1, y_2)(t)} \\
    u_2'(t) &= \frac{y_1(t)g(t)}{W(y_1, y_2)(t)}
\end{align*}
Which, of course leads to
\begin{align}
    u_1(t) &= -\int \frac{y_2(t)g(t)}{W(y_1, y_2)(t)} \\
    u_2(t) &= \int \frac{y_1(t)g(t)}{W(y_1, y_2)(t)}
\end{align}
As part of
\[ Y(t) = u_1(t)y_1(t) + u_2(t)y_2(t) \]
Which gives us our final answer
\[ y = y_1(t) + y_2(t) + Y(t) \]
For normal use of this technique:
\begin{enumerate}
    \item Find $u_1(t)$ and $u_2(t)$ using equations (3) and (4)
    \item Write the final solution
    \end{enumerate}
Not actually that bad.
\section{Systems of ODEs}
\subsection{The Wronskian for Systems}
Again we must ask ourselves the question: How do we know if we've found all of the solutions.
Again we answer by checking is $W \neq 0, \forall t$. How do we compute the Wronskian for
a system of equations? Since our solutions $y_1, ..., y_n$ are vector valued functions
$y_i: \mathbb{R} \rightarrow \mathbb{R}^n$, we say that
\[ W(y_1, ..., y_n) = det \left[y_1, ..., y_n \right] \]
In this case, we literally mean that our solutions are linearly independent over all $t$.
\subsection{Characteristic Equation}
\subsubsection{Given}
Given a system of $n$ linear ODEs of order $n$ with coefficients that can be written in the form
\[ x' = Ax \]
With A being an $n$ x $n$ matrix and $x$ being a vector-valued functions $x: \mathbb{R}
\rightarrow \mathbb{R}^n$
\subsubsection{Technique}
Assume that solutions are of the form
\[ y = \xi e^{\lambda t} \]
Then we get
\begin{align*}
    \lambda \xi e^{\lambda t} &= A \xi e^{\lambda t} \\
    \lambda \xi &= A \xi
\end{align*}
Thus we discover that $\xi$ must be an eigenvector of $A$ with eigenvalue $\lambda$. To find the
eigenvalue, we must first find the roots of the characteristic polynomial of $A$, since
\begin{align*}
    A \xi - \lambda \xi &= 0 \\
    (A - \lambda I) \xi &= 0
\end{align*}
And in order for this system to have a solution for $\xi$ other than the 0 vector, we must have that
\begin{align*}
    det(A - \lambda I) &= 0
\end{align*}
Depending on the $\lambda$'s we find, we can use different techniques to find the individual solutions
for each eigenvalue, and use the Principle of Superposition (the linear combination of all solutions is
the general solution) to give the general solution.

Note that these directly parallel the 2nd order techniques discussed. If we formulate a 2nd order
equation
\begin{equation*}
    y'' + ay' + by = 0
\end{equation*}
as
\begin{align*}
    x_1' &= x_2 \\
    x_2' &= -ax_2 - bx_1
\end{align*}
With $y = x_1$.
\subsubsection{Technique: Real and Unique Eigenvalues}
The solution is simply
\[ x(t) = \sum_{i = 1}^n C_i \xi_i e^{\lambda_i t}  \]
for each real eigenvalue $\lambda_i$ and associated eigenvector $\xi_i$.
\subsection{Technique: Imaginary Eigenvalues}
For reasons similar to the 2nd order ODEs, we'll drop the imaginary portion. Let $\lambda = a + bi$
be one of the conjugate pair of imaginary eigenvalues and $\xi$ be the corresponding eigenvector. Their
\begin{align*}
    x(t) &= C\xi e^{(a + bi)t} \\
         &= C\xi e^{at} (cos(bt) + isin(bt)) \\
        \dots \\
         &= y_1 + iy_2
\end{align*}
Since $\xi$ can have imaginary parts, you must multiply through and separate imaginary and real
parts. Then, since the coefficient $C \in \mathbb{C}$, you may take both the real and imaginary parts
of this solution as your $x_1$ and $x_2$.

Then your solution will be

\[ x(t) = C_1y_1 + C_2y_2 \]

\subsubsection{Technique: Repeated Eigenvalues}
If there is a repeated eigenvalue $\lambda$ with corresponding eigenvector $\xi$, it is tempting to
follow our logic from 2nd order DEs and simply multiply our solution by t. However, that is \textbf{wrong}.
Instead, we must assume our second solution is of the form
\[ x_2(t) = \xi te^{\lambda t} + \eta e^{\lambda t} \]
With vector $\eta$. Derive, getting
\[ x_2'(t) = (\lambda t + 1)\xi e^{\lambda t} + \lambda \eta e^{\lambda t} \]
And substitute into our system
\begin{align*}
    x' &= Ax \\
    (\lambda t + 1)\xi e^{\lambda t} + \lambda \eta e^{\lambda t} &= A(t\xi e^{\lambda t} + \eta e^{\lambda t})  \\
    \lambda t \xi e^{\lambda t} + \xi e^{\lambda t} + \lambda \eta e^{\lambda t} &= \lambda \xi t e^{\lambda t} + A\eta e^{\lambda t} \\
     \xi e^{\lambda t} + \lambda \eta e^{\lambda t} &= A\eta e^{\lambda t} \\
     \xi  + \lambda \eta &= A\eta  \\
    (A - \lambda I)\eta &= \xi
\end{align*}
This will create a linear system, which we can solve for $\eta$. Then we have that
\begin{equation}
x(t) = C_1\xi e^{\lambda t} + C_2(\xi te^{\lambda t} + \eta e^{\lambda t})
\end{equation}
To summarize:
\begin{enumerate}
    \item Find eigenvalues of the matrix A, and determine that one of the roots is repeated.
    \item Solve $(A - \lambda I)\eta = \xi$ to find $\eta$.
    \item The solution for that repeated eigenvalue is as given in (5).
\end{enumerate}
\subsection{Undetermined Coefficents}
\subsubsection{Given}
A non-homogenous 2nd order system of ODEs of the form
\begin{align*}
    x' = At + g(t) \\
\end{align*}
And a homogenous solution, $x_h(t)$ presumably obtained using the characteristic equation method above.

\subsubsection{Technique}
Using the same generalization techniques as for 2nd order ODEs, craft a guess for an additional
term we'll call $v(t)$, except instead of multiplying a term by t, add the higher order term. e.g.
\[ ae^{Bt} \rightarrow ate^{Bt} + be^{Bt} \]

The process is then pretty much the same:
\begin{enumerate}
    \item Derive and substitute into the original equation
    \item Equate coeffients of similar terms to get systems of systems of equations,
          some corresponding to eigenvalue equations, others solvable by row reduction,
          and so on.
    \item Solve them. If you decide to go down this painful road and the system of systems is
          unsolvable, then go back and do the "add a higher order" thing mentioned above. The
          tips on that above are relevant here too.
    \item Now you know $v(t)$, so write the final solution: $x(t) = x_h(t) + v(t)$
\end{enumerate}
\subsection{Diagonalization}
\subsubsection{Given}
A system of ODEs of the form
\[ x' = Ax + g(t) \]

No homogenous solutions needed!

\subsubsection{Technique}
Remember that given a matrix $A$ and an eigenbasis $\{\xi_1, ..., \xi_n\}$, we can construct a
Change of Basis matrix $T = \left[ \xi_1, ..., \xi_n \right]$, with
\begin{align}
    T^{-1}AT &= D = \begin{bmatrix}
                        \lambda_1 & 0 & \dots & \\
                        0 & \lambda_2 & \dots \\
                        \dots & \dots & \dots \\
                    \end{bmatrix}
\end{align}
Given these properties, we can reduce a system of ODEs into a series of 1st order DEs. How? First
let
    \[ x = Ty \]
For some $y$. We then get
    \[ Ty' = ATy + g(t) \]
Then we simply multiply on the left (because matrix multiplication order matters!) by $T^{-1}$
    \[ y = T^{-1}ATy + T^{-1}g(t) \]
And, using (6) we get
    \[ y = Dy + T^{-1}g(t) \]
Now, since $D$ is diagonal, we have a series of 1st order ODEs! Split the one matrix equation into
it's component system, and the $i$th equation/row will only have
    \[y_i' = \lambda_i y_i + \left[ T^{-1}g(t) \right]_i \]
Using a method like the Integrating factor to solve each of these, we can obtain $y$, and can now reverse
the transformation to get back the solution to our first equation, $x$. Our final solution will be
\begin{align*}
    x(t) = Ty
\end{align*}
To summarize
\begin{enumerate}
    \item Find the eigenvectors and the eigenbasis for $A$ (solving the characteristic equation and doing
          row reduction is the simplest method)
    \item Construct $T$ by normalizing each eigenvector and making them the columns of $T$.
    \item Let $x = Ty$, and multiply to the left by $T^{-1}$. Remember $T^{-1}AT = D$, the matrix
          with $D_{i,i} = \lambda_i$ and all else $0$.
    \item Separate this into $n$ 1st order linear equations with the form
            $y_i' = \lambda_i y_i + \left[ T^{-1}g(t) \right]_i $. Solve them using whatever technique (Integrating
          factor always works).
    \item Reconstruct the vector function $y$. The final solution will be $x(t) = Ty$, as we used before
          in 2.
\end{enumerate}
\section{More Powerful Techniques}
\subsection{Power Series}
\subsubsection{Given}
Anything. Seriously. Any ODE. We can get \textit{a} solution. Possibly a disgusting one. For the
sake of making the example easy, we'll assume a 2nd order equation of the form
    \[ y'' + p(t)y' + q(t)y = g(t) \]
\subsubsection{Technique}
Assume our solution is a \textbf{Power Series}. This is what makes this technique so \textbf{power}ful.
Get it? No? Ok.
    \[ y = \sum_{n=0}^{\infty}a_n(x - x_0)^n \]
Note that
\begin{align*}
    y' &= \sum_{n=0}^{\infty}na_n(x - x_0)^{n-1} \\
    y'' &= \sum_{n=0}^{\infty}n(n-1)a_n(x - x_0)^{n-2}
\end{align*}
Then note these 2 techniques to shift indices of summations and the power of the $(x-x_0)^n$ term.
\begin{align}
    \sum_{n=k}^{\infty} a_n (x - x_0)^{n-k} &= \sum_{n=k}^{\infty} a_{n+k} (x - x_0)^n \\
    \sum_{n=0}^{\infty} a_n (x - x_0)^n = a_0 + a_1(x - x_0) + &... + a_k(x - x_0)^k + \sum_{n=k+1}^{\infty} a_n (x - x_0)^n
\end{align}
With these techniques and derivatives, we have one goal: rearrange the differential equation into the
form
\[ a_0 + ... + a_k(x - x_0)^k + \sum_{n=k+1}^{\infty} F(a_n, a_{n+1}, ..., a_{n+l}) = 0 \]
From which we learn that $a_0 = ... = a_k = 0$ and $F(a_n,..., a_{n+l}) = 0, \forall n > k$. From this
second equation, we can derive a \textbf{Recurrence Relation} that relates $a_{n+l}$ to the previous
$l$ terms.
We can always let $a_0$ and $a_1$ vary (since it's a 2nd order ODE), analagous to our $C_1$ and $C_2$
in previous techniques, and solve for all higher $a_n$ through these. Sometimes we will be able to
find a closed form solution for $a_n$, and in even rarer circumstances, we may be able to identify that
as a taylor series of a function, in which case we can remove summations from our answer. But in general,
our solution will look like what we originally guessed,
    \[ y = \sum_{n=0}^{\infty}a_n(x - x_0)^n \]
With some sort of recurrance relation for $a_n$. Yeah, this is pretty ugly. To sum up
\begin{enumerate}
    \item Substitute in a general power series.
    \item Manipulate sums and powers until we get one large sum equal to zero.
    \item Set all coefficients equal to 0. Obtain a recurrence relation for $a_n$
    \item Attempt to find a closed source form for $a_n$ (this is tricky - have fun). Further, see
          if this is a taylor series for an elementary function. Even if not, we "solved" it, to
          an arbitrary precision.
\end{enumerate}
\subsection{Laplace Transform}
\subsubsection{Given}
An ODE. That's really about it. One caveat is that the functions involved must have Laplace and
Inverse Laplace transforms that we know of. We'll just say it looks like
    \[ \sum_{i=0}^n a_i(t)y^{(i)}= g(t) \]
Yeah. That general.
\subsubsection{Technique}
We define the Laplace Transform of a function as
\[ \mathscr{L}\{f(x)\} = \int_{0}^{\infty} e^{-st}f(t)dt \]
Note that it's linear: that
\[ \mathscr{L}\{af(x) + bg(x)\} = a\mathscr{L}\{f(x)\} + b\mathscr{L}\{g(x)\} \]
Most importantly, if we evaluate this for a derivative,
\begin{align*}
    \mathscr{L}\{f'(x)\} &= \int_{0}^{\infty} e^{st}f'(t)dt  \\
    \mathscr{L}\{f'(x)\} &=  \left[ e^{-st}f(t)\right]_0^{\infty} -\left(- s\int_0^{\infty} e^{-st}f(t)dt \right)  \\
    \mathscr{L}\{f'(x)\} &=   -f(0) + s\int_0^{\infty} e^{-st}f(t)dt  \\
    \mathscr{L}\{f'(x)\} &=   s\mathscr{L}\{f(x)\} - f(0)
\end{align*}
With $f(0)$ and $f'(0)$ either being arbitrary constants of the general solution, or initial conditions.

Applying this further
\[ \mathscr{L}\{f^{(n)}\} = s^n\mathscr{L}\{f(x)\} - \sum_{i=0}^{n} s^{n-i-1}f^{(i)}(0) \]
But we probably don't want to use this since it's so ugly.

Now the amazing thing is that this transforms differential equations into algebraic equations. That means that
given any differential equation, we can get some algebraic equation we can solve for $\mathscr{L}\{y\}$.
Transforming something merely requires an indefinite integral. however, the inverse transform $\mathscr{L}^{-1}$
is a lot more difficult to compute. For that reason, we typically have look-up tables to transform
$\mathscr{L}\{y\}$ back into $y$.

To summarize the technique:
\begin{enumerate}
    \item Apply the Laplace Transform to both sides of the DE.
    \item Solve for $\mathscr{L}\{y\}$ algebraically.
    \item Manipulate the RHS until we can use a lookup table to apply the inverse transform (typically partial
          fractions).
\end{enumerate}
AND WE'RE DONE HERE FAM.
\end{document}


\subsection{ROOT\_VAL Operator}

The {\tt ROOT\_VAL} operator takes a single univariate polynomial as
argument, and returns a list of root values at system precision (or
greater if required to separate roots).  It is used with the syntax
\begin{verbatim}
	ROOT_VAL(EXPRN:univariate polynomial):list.
\end{verbatim}
For example, the sequence
\begin{verbatim}
        on rounded; root_val(x^3-x-1);
\end{verbatim}
gives the result
\begin{verbatim}
        {0.562279512062*I - 0.662358978622, - 0.562279512062*I

          - 0.662358978622,1.32471795724}
\end{verbatim}


\section{Theory}
\label{sec:theory}

The following is a short summary of the used equations, which were adopted
from chapter 3 of the course book~\cite{Bonet2008}.
Unless stated otherwise, the uppercase letters refer to the initial (reference)
configuration of the problem, while the lowercase ones refer to the current
configuration. The boldface letters denote vectors, matrices and tensors.

For a given load, solving the problem implies finding a configuration that
simultaneously satisfies both the global (system) equilibrium equations and
the constitutive equations.
The equilibrium equations are expressed in terms of the residual (out-of-balance)
vector \(\bm{R} (\bm{x})\) as the balance between internal and external forces:
\begin{equation}
  \bm{R} (\bm{x}) = \bm{T} (\bm{x}) - \bm{F} = \bm{0},
\end{equation}
where \(\bm{x} =\left[ \bm{x}_{1}, \bm{x}_{2}, \cdots, \bm{x}_{N} \right]^{\text{T}}\)
is the vector of current nodal positions;
\(\bm{T} =\left[ \bm{T}_{1}, \bm{T}_{2}, \cdots, \bm{T}_{N} \right]^{\text{T}}\) is
the vector of internal nodal forces;
\(\bm{F} =\left[ \bm{F}_{1}, \bm{F}_{2}, \cdots, \bm{F}_{N} \right]^{\text{T}}\) is
the vector of external nodal forces, where it is assumed to be independent
of the current nodal positions \(x\) (generally this is not true);
and \(N\) is the number of nodes.

\subsection{Hyperelasticity}
\label{sec:hyperelasticity}

In the case of hyperelastic material behaviour of a rod, i.e.\ material whose
strain energy per unit volume \(V\) does not depend on the path taken by the rod as
it moved from initial length \(L\) to the current length \(l\), the internal
truss forces \(\bm{T}_{a}\) and \(\bm{T}_{b}\) can be computed as
\begin{equation}
  \bm{T}_{b} = \frac{V E}{l} \ln \left( \frac{l}{L} \right) \bm{n}
             = \tau \frac{V}{l} \bm{n}, \quad
  \bm{T}_{a} = - \bm{T}_{b}.
\end{equation}
Here, a Young's modulus like constant \(E\) has been used to relate Kirchhoff
stress \(\tau = E \varepsilon = \sigma v / V\) to logarithmic strain
\(\varepsilon = \ln(l/L)\).

Finding equilibrium position is carried out using Newton-Raphson method, which
involves linearisation of the equilibrium equations. The linearisation yields
the directional derivative \(D \bm{T}^{(e)} (\bm{x}^{(e)}) [\bm{u}^{(e)}]\),
which gives the
expression for the tangent stiffness matrix:
\begin{equation}
  D \bm{T}^{(e)} (\bm{x}^{(e)}) [\bm{u}^{(e)}] = \bm{K}^{(e)} \bm{u}^{(e)} =
  \begin{bmatrix}
    \bm{K}_{aa}^{(e)} & \bm{K}_{ab}^{(e)} \\
    \bm{K}_{ba}^{(e)} & \bm{K}_{bb}^{(e)}
  \end{bmatrix}
  \begin{bmatrix}
    \bm{u}_{a} \\
    \bm{u}_{b}
  \end{bmatrix}
\end{equation}
with
\begin{align}
  \bm{K}_{aa}^{(e)} &= \bm{K}_{bb}^{(e)} =\left( \frac{V}{v}
                      \frac{d \tau}{d \varepsilon} \frac{a}{l} -
                      \frac{2 \sigma a}{l} \right)
                      \bm{n} \otimes \bm{n} + \frac{\sigma a}{l} \bm{I} \\
  \bm{K}_{ab}^{(e)} &= \bm{K}_{ba}^{(e)} = - \bm{K}_{aa}^{(e)},
\end{align}
where \(d \tau / d \varepsilon = E\) is the elastic material tangent modulus.

\subsection{Rate-independent plasticity}
\label{sec:plasticity}

In the case of rate-independent finite strain plasticity with isotropic hardening,
the stress is defined via elastic strain \(\varepsilon_{e}\):
\begin{align}
  \tau &= E \varepsilon_{e} = E \left( \varepsilon - \varepsilon_{p} \right) \\
  \varepsilon_{p} &= \int_{0}^{t} \dot{\varepsilon}_{p} dt \label{eq:plastic-strain}
\end{align}

The onset of plastic deformation is governed by the yield condition, which for the
problem at issue is
\begin{equation}
  f (\tau, \bar{\varepsilon}_{p}) = \left| \tau \right| - \left(
    \tau_{y}^{0} + H \bar{\varepsilon}_{p} \right) \leq 0, \quad
  \bar{\varepsilon}_{p} \geq 0,
\end{equation}
where \(\tau_{y}^{0}\) is the initial yield stress,
\(\bar{\varepsilon}_{p}\) is the hardening parameter and \(H\) is a material
property called plastic modulus.
At its simplest, the hardening parameter is defined as the accumulated absolute
plastic strain occurring over time:
\begin{equation}
  \bar{\varepsilon}_{p} = \int_{0}^{t} \dot{\bar{\varepsilon}}_{p} dt,
  \quad \dot{\bar{\varepsilon}}_{p} = \left| \dot{\varepsilon}_{p} \right|,
  \quad \dot{\varepsilon}_{p} = \dot{\gamma} \frac{\partial f}{\partial \tau},
\end{equation}
where \(\dot{\gamma}\) is plastic multiplier.

In a computational setting the time integration of \eqref{eq:plastic-strain} can
only be performed approximately from a finite sequence of values determined at
different time steps.
In order to satisfy the yield condition exactly at each incremental time step,
a return-mapping algorithm described in figure~\ref{fig:return-mapping} is used,
which employs incremental kinematics (see figure~\ref{fig:incr-kinematics}).
\begin{figure}[th]
  \begin{subfigure}[t]{0.38\textwidth}
    \centering
    \includegraphics[width=\textwidth]{return_mapping.png}
    \caption{Return-mapping algorithm.}
    \label{fig:return-mapping}
  \end{subfigure}
  ~
  \begin{subfigure}[t]{0.56\textwidth}
    \centering
    \includegraphics[width=\textwidth]{incremental_kinematics.png}
    \caption{Incremental kinematics. \(\lambda = l / L\).}
    \label{fig:incr-kinematics}
  \end{subfigure}
  \caption{Two figures from \cite{Bonet2008}.}
\end{figure}

The last matter to be addressed in the presence of plasticity is the material
tangent modulus \(d \tau / d \varepsilon\).
The tangent modulus derived from incremental considerations is generally not
the same as the one obtained from the rate equations.
The reason is that the incremental change in stress imposed by the chosen
return-mapping algorithm is different from the continuous change in stress
stemming from the rate equations.
That is why the algorithmic tangent modulus is used:
\begin{equation}
  \frac{d \tau_{n+1}}{d \varepsilon_{n+1}} = \frac{E H}{E + H}
\end{equation}
When plasticity occurs, this replaces the elastic tangent stiffness
\(d \tau / d \varepsilon = E \).

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../main"
%%% End:


%intro
\section{Introduction}

We briefly dicuss the prerequisit of category theoretic basics. We implicitly assume for each object $X$ an Grothendieck universe $\calu(X)$ such that $X \in \calu(X)$.
\\
\subsection{Basic definitions}

A category $\calc$ consists of the following four data -
\bd
\item[Objects] the class of objects - similar to elements in sets. The class of objects are denoted as
$$\objc,$$
however, for brevity and if no ambiguity can arise, $A \in \calc$ (reading - $A$ an object in category $\calc$).
\item[Morphisms] for two objects $A, B$ in $\calc$ we call an arrow or simply a map $f : A \longrightarrow B$ a morphism and its class is denoted by
$$\homcab.$$
Irrespective of objects one writes $\mrm{Hom}^{\calc}$.
\item[Identity] is always an object in the class of morphisms for any object $A \in \calc$:
$$id_A = [a \longmapsto a] \in \mrm{Hom}^{\calc}(A,A).$$
\item[Composition] For any triplet $A, B, C \in \calc$ and arrows
$$f \in  \mrm{Hom}^{\calc}(A, B),\ g \in \mrm{Hom}^{\calc}(B, C)$$
there exists a unique arrow
$$h := g \circ f \in \mrm{Hom}^{\calc}(A,C).$$
As a commutative diagram:
$$\xymatrix{A \ar[r]^{f}\ar[rd]_{\exists!h}&B\ar[d]^g\\
&C.}$$
This is usually combined in the quadruplet $(\objc,\mrm{Hom}^{\calc},id,\circ)$.
\ed
Furthermore, we call $\calc$
\bd
\item[small] category if its class of objects is small - i.e. the class of its objects is set-like.
\item[dual] given $\calc$ its opposite category $\mrm{C^{\mrm{op}}}$ has the same class of objects just with its arrows reverted:
$$\op{f} \in \homcopab \Leftrightarrow f \in \homc{B}{A}.$$
Its composition is for all $A, B, C \in \calcop$ and $\op{f} \in \homcopab$ and $\op{g} \in \homcop{B}{C}$:
$$\op{g} \op{\circ} \op{f} \in \homcop{A}{C} \Leftrightarrow f \circ g \in \homc{C}{A}.$$
\item[monomorphims] we call a morphism $f \in \mrm{Hom}(A,B)$ a monomorphism if for all $g_1, g_2 \in \mrm{Hom}(B,C)$
$f \circ g_1 = f \circ g_2$ implies $g_1 = g_2$.
\item[epimorphims] we call a morphism $f \in \mrm{Hom}(A,B)$ a epimorphism if for all $g_1, g_2 \in \mrm{Hom}(C,A)$
$g_1 \circ f = g_2 \circ f$ implies $g_1 = g_2$ (aquivalently, $\op{f}$ is a monomorphism).
\item[isomorphisms] $f \in \homcab$ is an isomorphism if it is an epimorphism and a monomorphism within the same category. An automorphism is an isomorphism in $\homc{A}{A}$.
\item[initial] we call $\calc$ a category with initial object $\ast$ if for each object $A \in \calc$ we get
$$\exists ! f \in \homc{\ast}{A}.$$
$\ast$ is its initial object
\item[terminal] we call $\calc$ a category with terminal object $\ast$ if for each object $A \in \calc$ we
get
$$\exists ! f \in \homc{A}{\ast}.$$
\item[null] a null object is terminal and inital.
\item[equalizer] given two objects $A, B \in \calc$ and two morphisms $f, g \in \homcab$, we call an object $X \in \calc$ an equalizer if for any object $O \in \calc$ and arrow $h \in \homc{O}{A}$ we get two unique morphisms $o \in \homc{O}{X}, e \in \homc{X}{A}$
$$\xymatrix{
&O\ar[rd]^h\ar[ld]_h\ar[d]_o&\\
A\ar[rd]_f&X\ar[l]_e\ar[r]^e&A\ar[ld]^g\\
&B&\\
}$$
commutes. Not all category have equalizers for all pairs of morphisms. But, if they exist they are unique up to isomorhism. Coequalizers are equalizers in $\op{\calc}$.
\item[products] we call $\calc$ a category with finite products if
\bn
\item for each $A, B$ in $\calc$ there exists an object $P$ and two morphisms $\pi_A : P \longrightarrow A, \pi_B : P \longrightarrow B$ such that for each $O \in \calc$ with $f \in \homc{O}{A}$ and $g \in \homc{O}{B}$ there is a unique $h \in \homc{O}{P}$ so that the following diagram:
$$\xymatrix{
&O\ar[d]^h\ar[dl]_f\ar[dr]^g&\\
A&P\ar[l]^{\pi_A}\ar[r]_{\pi_B}&B\\
}$$
commutes. $P$ is usually denoted by the cartesian product notation $P = A \times B$. This, in turn, enables us to define the $\call{J}$ products: given a subclass of objects $A_J$ indexed by $\call{J}$ then its product is defined to be an object $P \in \calc$ such that
$$\xymatrix{
O \ar[dd]_{h}\ar[rd]_{f_J}\ar[rrrd]_{f_{J'}}\ar[rrrrrd]_{f_{J''}}\\
&A_J && A_{J'} && A_{J''}&\ldots\\
\prod_{J \in \call{J}} A_J =: P\ar[ur]^{\pi_J} \ar[urrr]^{\pi_{J'}}\ar[urrrrr]^{\pi_{J''}}\\
}$$
\en
Coproducts are products in $\op{\calc}$.
\item[pullback] Given three objects $A, B, C \in \calc$ and two morphisms $f: A \longrightarrow C, g : B \longrightarrow C$ we call an object $P \in \calc$ the pullback of $(f,g)$ if $P$ is the equalizer of $(\pi_A \circ f, \pi_B \circ g)$ in $\homc{A \times B}{C}$ and projections $\pi_A : A \times B \longrightarrow A, \pi_B : A \times B \longrightarrow B$:
$$\xymatrix{
&O\ar[rd]^h\ar[ld]_h\ar[d]_o&\\
A \times B\ar[d]_{\pi_A}&P\ar[l]_e\ar[r]^e&A \times B\ar[d]^{\pi_B}\\
A\ar[rd]_f&&B \ar[ld]^g\\
&C&\\
}$$
$P$ is denoted by $A \times_{C} B$ and can be thought off as
\ed
Given two categories $\calc$ and $\call{D}$ - we call
\begin{defi}[Functors]
a pairing $F : \calc \longrightarrow \call{D}$ a functor for any two objects $A, B \in \calc$ with arrow  $f \in \homcab$ - if one of the following holds:
\bd
\item[Covariant] $$\xymatrix{
A \ar[d]_F\ar[r]^f &B\ar[d]^F\\
F(A) \ar[r]_{F(f)}&F(B).\\
}$$ and $F(id_A) = id_{F(A)}$. In particular, $F(f) \in \homo{\call{D}}{F(A)}{F(B)}$.
\item[Contravariant] $$\xymatrix{
A \ar[d]_F\ar[r]^f &B\ar[d]^F\\
F(A) &F(B)\ar[l]_{F(f)}\\
}$$ and $F(id_A) = id^{-1}_{F(A)}$. In particular, $F(f) \in \homo{\op{\call{D}}}{F(A)}{F(B)}$.
\ed
\end{defi}
\subsection{Examples}
We will be discussing some promiment examples:
\subsubsection{General categories}
\paragraph{Category $\mrm{Set}$}
The category of sets is denoted by $\mrm{Set}$ and consists of all objects the behave "set-like" (the most basic property is the choice property - $\chi_{x} : \{A \subset X\} \longrightarrow \{Y: Y \subset X,\ |Y| = 0,1 \}$ mapping singleton subsets to itself or empty set
$$\chi_{x} = \left[A \longmapsto \begin{cases}
\emptyset, & x \notin A\\
\{x\},& \mrm{else}\\\end{cases}\right]$$
). With these classes of maps we may define the equality property - elements are equal if and only if the choice function collide. Its class of arrows are simply all maps among sets.
The category of pointed spaces is denoted by $\mrm{pspc}$ and consists of objects $X$ with a "choice function":
$ \chi : X \longrightarrow \mrm{sing}, x' \longmapsto \ast$. With $\ast$ the base point. This definitions is equivalent to $\mrm{Set}$ except that no subspace (sub object) may ever be empty (empty set is not an object in $\mrm{pspc}$. Its morphisms are the base point preserving maps in $\mrm{Set}$.

\paragraph{Power set as functor} Given an arbitrary set $X$, its power set $\call{P}(X)$ defines a functor
$$\calp : \mrm{set} \longrightarrow \mrm{set},\ X \longmapsto \calp(X).$$
which is contravariant and covariant at the same time. Consider a map $f : A \longrightarrow B$:
\bn
\item is covariant as  $\calp(f) : \calp(A) \longrightarrow \calp(B)$ simply maps each subset of $A$ to the subsets of $B$ containing images of each subset mapped:

$$f(A') \cap \calp(B) = \{f(A') \cap B' : B' \subset B\},\ \forall A' \subset A.$$
\item is contravariant as with the same we get
$$\calp(f) : \calp(B) \longrightarrow \calp(A),\ B' \longmapsto f^{-1}(B') \subset A.$$
\en
Thus, within $\mrm{set}$ both concepts (Co/Contravariance), with respect to $\calp$, are isomorphic.
\subsection{Algebraic catories}
There a some prominent examples we will be discussing algebraic categories as we will be revisiting some of them later on.

\subsubsection{Semi groups} We call $(S, m) \in \mrm{SGrp}$ - if $m : S \times S \longrightarrow S$ is a closed associative binary operation:
\bn
\item Closedness:
$$m(s,t) \in S\ \forall (s,t) \in S^2,$$
$$\Leftrightarrow m^{-1}(S) \supset S^2.$$
\item Associativity:
$$\xymatrix{
S^3 \ar[r]^{m \times id_S}\ar[d]_{id_S \times m}& S^2 \ar[d]^m\\
S^2 \ar[r]_m &S\\
}
$$
\en
We remark that the empty set is a semi group as clearly the diagonal projection defines a map $\emptyset \times \emptyset \longrightarrow \emptyset$.
\subsubsection{Monoids} A semi group is a monoid or, $(M, m, e) \in \mrm{Mon}$, if there is an initial object $\ast \in \mrm{Mon}$ and an arrow $e : \ast \longrightarrow M \in \homo{\mrm{Mon}}{\ast}{M}$, such that
$$\xymatrix{
\ast \times M \ar[rr]^{e \times id_M}\ar[rrd]_\simeq&&M^2 \ar[d]_m&&M \times \ast\ar[ll]_{id_M \times e} \ar[lld]^{\simeq}\\
&&M&&\\
}$$
commutes. The map $e$ is is called the unit.
\subsubsection{Groups} Monoids are the quatuple $(G,m,e,S)$ with map $S : G \longrightarrow G$ such that
$$\xymatrix{
&G \ar[rd]^\Delta\ar[ld]_\Delta\ar[dd]&\\
G^2\ar[dd]_{S \times id_G}&&G^2\ar[dd]^{id_G \times S}\\
&\ast\ar[dd]_e&\\
G^2\ar[rd]_m&&G^2\ar[ld]^m\\
&G&\\
}$$
with $\Delta : G \longrightarrow G^2$, the diagonal map:
$$\xymatrix{
G \ar[r]^\Delta\ar[d]_\Delta\ar[rd]_\simeq&G^2\ar[d]^{\pi_1}\\
G^2\ar[r]_{\pi_2}&G\\
}$$
for the two projections $\pi_1, \pi_2$ in the first and second factor, respectively. The map $S$, the so called antipode, is ań antiisomorphism - that is it is an isomorphism in $\grp$, but its image object is in general not $G$ itself. Rather, it is the so called opposite group:
$$\op{G} \in \grp: \op{G} := G \in \obj{\grp},\ \op{m} := m_G \circ \tau.$$
The object is $G$ itself but its multiplication is is the flipped version of $m_G = m$. We remark that the object $G$ need not to be an object in $\grp$ - it suffice for $G$ to be a semi group. Inversion and unity are of no consequence.
\subsubsection{Abelian groups} We call an object $(A,m,e,S,\tau) \in \abel \subset \grp$ abelian (group) if $\op{A} = A$, that is $\op{m} = m$. Aquivalently, $S$ is a group automorphism. The map $\tau$ is the flip isomorphism:
$$\tau = \tau_{A\times B}: A \times B \longrightarrow B \times A$$
such that
$$\xymatrix{
&A \times B \ar[ld]_{\pi_A}\ar[rd]^{\pi_B}\ar[dd]_\tau&\\
A & & B\\
&B \times A\ar[lu]^{\pi_A}\ar[ru]_{\pi_B}\\
}$$
commutes.
\subsubsection{Rings} Rings $(R,m_+,e_+,S_+,m_*,\tau) \in \mrm{Rng}$ are subobjects $(R,m_+,e_+,S_+,\tau) \in \mrm{Abel}$ such that $(R,m_*)$ is a semi group and the following diagrams commute:
$$\bao{cc}
\xymatrix{
R^4\ar[rrr]^{id_R \times \tau \times id_R}&&&R^4\ar[d]^{m_* \times m_*}\\
R^3\ar[d]_{id_R \times m_+}\ar[u]^{\Delta \times id_{R^2}}&&&R^2\ar[d]^{m_+}\\
R^2 \ar[rrr]_{m_*}&&
&R\\
} & \xymatrix{
R^4\ar[rrr]^{id_R \times \tau \times id_R}&&&R^4\ar[d]^{m_* \times m_*}\\
R^3\ar[d]_{m_+ \times id_R}\ar[u]^{id_{R^2} \times \Delta}&&&R^2\ar[d]^{m_+}\\
R^2 \ar[rrr]_{m_*}&&
&R\\
}
\ea$$
The left hand side represents the left dissociativity and the right hand side the right dissociativity. We call $R \in \mrm{Rng}$ unital or $R \in \mrm{URng}$ is there is an arrow $e_* : \ast \longrightarrow R$ such that $(R,m_*,e_*)$ is a monoid and commutative or $R \in \mrm{CRng}$ if $\op{(R,m_*)} = (R,m_*)$.
\newcommand{\htalpha}{\hat{\alpha}}
\subsubsection{Modules} Given a ring $(R,m_+,e_+,S_+,m_*)$ we call an abelian group $(M,m_M,e_M,S_M) \in \abel$ an $R$ left module if there is an arrow $$\alpha \in \homo{\grp}{R}{S(M) := \mrm{Aut}^{\grp}(M)}$$ such that for $\htalpha : R \times M \longrightarrow M, \htalpha = eval \circ (\alpha \times id_M)$ and $eval: S(M) \times M \longrightarrow M,$:
$$\bao{cc}
 \xymatrix{
R^2 \times M^2\ar[rr]^{id_R \times \tau_{R \times M} \times id_M}&&(R \times M)^2\ar[d]^{\htalpha \times \htalpha}\\
R \times M^2\ar[d]_{id_R \times m_M}\ar[u]^{\Delta_R \times id_{M^2}}&&M^2\ar[d]^{m_M}\\
R \times M \ar[rr]_{\htalpha}&&M\\
}
& \xymatrix{
R^2 \times M \ar[d]_{m_* \times id_M}\ar[r]^{id_R \times \htalpha}&R \times M\ar[d]^\htalpha\\
R \times M \ar[r]_\htalpha &M\\
}
\\ \xymatrix{
R^2 \times M^2\ar[rr]^{id_R \times \tau_{R \times M} \times id_M}&&(R \times M)^2\ar[d]^{\htalpha \times \htalpha}\\
R^2 \times M\ar[d]_{m_+ \times id_M}\ar[u]^{id_{R^2} \times \Delta_M}&&M^2\ar[d]^{m_M}\\
R \times M \ar[rr]_{\htalpha}&&M\\
}
\ea$$
commute. The first column represents the ring and module dissociativities, respectively. The second column is the semi group torsor property - $\htalpha$ commutes with ring multiplication $m_*$. $R$ right modules are $\op{R}$ left modules and aquivalently, an $R$ left module is an $\op{R}$ right module. Here, $\op{m_+} = m_+$ and $\op{m_*} = m_* \circ \tau_{R^2}$. Therefore,
$$\op{\alpha} = \alpha,\ \op{eval} = eval \circ \tau_{S(M) \times R} \ \mrm{and}\ \op{\htalpha} = \op{eval} \circ (id_M \times \alpha).$$
A two sided module is left and right sided $R$ module. Two sided modules are of specifial interest as they do have products: given two ts-$R$ modules $M$ and $N$ for any bi-linear map $f : M \times N \longrightarrow P$ and module $P$ there is the following universal property:
$$\xymatrix{
M \times N \ar[r]^{\otimes_R}\ar[rd]_{f}&M \otimes_R N\ar[d]^{\exists! g \in \mrm{Hom}_R(M\otimes N,P)}\\
&P\\
}$$
We call $M \otimes_R N$ the tensor product of $M$ and $N$ being also a two sided module. In general, a right module $M$ can have (tensor) products $M \otimes_R N$ with left modules $N$. However, these objects are no longer modules. They are merely objects in $\abel$.
\subsubsection{Algebras} A two sided $R$ module $A$ is an algebra if there is an $R$ linear map $\mu : A \otimes_R A \longrightarrow A$ making $A$ a two-sided $A$ module.


\subsection{Bonus: Lebesgue Integral}
If we want to find the area under a curve, we choose to slice up the horizontal region (domain) of the function (Riemannian integration).
But can we choose to slice up the vertical region that is the range of the function?
This is known as the Lebesgue integration.
Sometimes this is better.
For example if we want to integrate $f(x)=1_{\mathbb Q}$, Riemannian integration would not work if we want to evaluate
$$\int_0^1f(x)\,\mathrm dx$$
But if we do it by Lebesgue intgeral, but since $\mathbb Q$ is countable
\footnote{Hence of Lebesgue measure $0$.}
this integral is obviously $0$.


\chapter{Deconvolution}
\section{Wavelet and Deconvolution}
\label{wdec}
Consider an image characterized by its intensity
distribution $I$, corresponding to the observation of a
``real image'' $O$ through an optical system. If the
imaging system is linear and shift-invariant, the relation between
the data and the image in the same coordinate frame is a
convolution:
$I(x,y) = (P * O)(x,y) + N(x,y)$,
where
$P$ is the point spread function (PSF) of the imaging system, and $N$
is additive noise. We want to determine $O(x,y)$ knowing $I$ and $P$. This
inverse problem has led to a large amount of work, the main difficulties
being the existence of: (i) a cut-off frequency of the
PSF,
and (ii) the additive noise (see for example \cite{ima:bertero98}).

The wavelet based non-iterative algorithm,
the wavelet-vaguelette decomposition \cite{rest:donoho95b},
consists of first applying an inverse filtering
($F = P^{-1} * I  +  P^{-1} * N = O + Z$
where $\hat{P}^{-1}(\nu) = \frac{1}{\hat{P}(\nu)}$).
The noise $Z =  P^{-1} * N$ is not white but remains
Gaussian. It is amplified when the deconvolution
problem is unstable.
Then, a wavelet transform is applied on $F$, the wavelet coefficients
are soft or hard thresholded \cite{rest:donoho93_2},
and the inverse wavelet transform
furnishes the solution.

The method has been refined by
adapting the wavelet basis to the frequency response of the inverse of $P$
\cite{rest:kalifa99}. This leads to a special basis,
the {\em Mirror Wavelet Basis}.  This basis has a
time-frequency tiling structure different from the conventional wavelets one.
It isolates the frequency $\nu_s$ where $\hat{P}$ is close to zero, because
a singularity in $\hat{P}^{-1}(\nu_s)$ influences the noise variance in
the wavelet scale corresponding to the frequency band which includes $\nu_s$.
Because it may not be possible to isolate all singularities, Neelamani
\cite{rest:neelamani99} has advocated a hybrid approach,
and proposes to still use the Fourier domain to restrict excessive noise
amplification. These approaches are fast and competitive compared to linear methods, and
the wavelet thresholding removes the Gibbs oscillations.
This  presents however several drawbacks:
(i) the first step (division in the Fourier space by the PSF)
cannot always be done properly,
(ii) the positivity a priori is not used, and
(iii) it is not trivial to consider non-Gaussian noise.

As an alternative,
several wavelet-based iterative algorithms have been proposed
 \cite{starck:book98}, especially in the astronomical
domain where the positivity a priori is known to improve
significantly the result. The simplest method consists of first
estimating the multiresolution support $M$ (i.e. $M(j,x,y)= 1$ if
the wavelet transform of the data presents a significant coefficient
at band $j$ and at pixel position $(x,y)$, and $0$ otherwise), and to
apply the following iterative scheme:
\begin{equation}
O^{n+1} = O^{n} + P^* * \cW^{-1}[M.\cW (I - P * O^n)]
\end{equation}
where $\cW$ is the wavelet transform operator.
At each iteration, information is extracted from the
residual only at scales and positions defined by the multiresolution support.
$M$ is estimated from the input data and the correct noise modeling
can easily be considered.

\section{The Combined Deconvolution Method}
\label{cbdec}
 Similar to the filtering, we expect that the combination
of different transforms can improve the quality of the result.
The combined approach for the deconvolution leads to two different
methods.

If the noise is Gaussian and if the division by the PSF in the Fourier space
can be carried out properly, then the deconvolution problem becomes a
filtering problem where the noise is still Gaussian, but not white.
The Combined Filtering Algorithm can then be applied using the curvelet
transform and the wavelet transform, but by estimating first the
correct thresholds in the different bands of both transforms.
Since the mirror wavelet basis is known to produce better results than
the wavelet basis, it is recommended to use it instead of the standard
undecimated wavelet transform.

An iterative deconvolution method
 is more general and can always be applied.
Furthermore,
the correct noise modeling can much more easily be taken into account.
This approach consists of detecting, first, all the significant coefficients
with all multiscale transforms used. If we use $K$ transforms
$T_1, \dots, T_K$, we derive $K$ multiresolution supports $M_1, \dots, M_K$
from the input image $I$ using noise modeling.

For instance, in the case of Poisson noise, we apply the Anscombe transform
to the data (i.e. $\cA(I) = 2 \sqrt{I + \frac{3}{8}}$). Then
we detect the significant coefficients with the kth transform $T_k$,
assuming Gaussian noise with standard deviation equal to 1,
in $T_k \cA(I)$ instead of $T_k I$.  $M_k(j,x,y) = 1$ if a coefficient
in band $j$ at pixel position $(x,y)$ is detected , and  $M_k(j,x,y) = 0$
otherwise. For the band $J$ which corresponds to the smooth array
in transforms such as the wavelet or the curvelet transform,
we force $M_k(J,x,y) = 1$ for all $(x,y)$.

Following determination of a set of multiresolution supports,
we propose to solve
the following optimization problem:
\begin{equation}
  \label{eq:dec-min}
  \min \cS(\tilde O), \quad \mbox{subject to} \quad \tilde O \in C,
\end{equation}
where $\cS$ is an edge preservation penalization term defined by:
\[ \cS(\tilde O) = \int \parallel \nabla \tilde O \parallel_p, \]
with $p=1.1$.
 $C$ is the set of images $\tilde{O}$
which obey the two constraints:
\begin{enumerate}
\item $\tilde{O} \ge 0$ (positivity).
\item $ M_k T_k I = M_k T_k [P *\tilde{O}]$, for all $k$.
\end{enumerate}
The second constraint
imposes fidelity to the data, or more exactly,
to the significant coefficients of the data,
obtained by the different transforms.
Non-significant (i.e. noisy) coefficients are not taken into account,
preventing any noise amplification in the final algorithm.

The solution is computed by using the
projected Landweber method \cite{ima:bertero98}:
\begin{eqnarray}
\tilde O^{n+1} = {\cP}_c \left[
\tilde O^n + \alpha( P^* * {\bar R}^n - \lambda \frac{\partial \cS(\tilde O)}{\partial O}) \right]
\end{eqnarray}
where ${\cP}_c$ is the projection operator which enforces the positivity
(i.e. set to 0 all negative values).
${\bar R}^n$ is the significant residual which
 is obtained using the following algorithm:
\begin{itemize}
\item Set $I^n_{0} = I^n = P * \tilde O^n$.
\item For $k=1,\dots,K$ do
$
I^n_k = I^n_{k-1} + T_k^{-1} \left[ M_k (T_k I - T_k I^n_{k-1}) \right] \nonumber
$
\item The significant residual ${\bar R}^n$ is obtained by:
  $ {\bar R}^n = I^n_K - I^n $.
\end{itemize}

$\alpha$ is a convergence parameter and $\lambda$ is the
regularization hyperparameter. Since the noise is controlled by the
multiscale transforms, the regularization parameter does not
have the same importance as in standard deconvolution methods.
A much lower value is enough to remove the artifacts relative to the
use of the wavelets and the curvelets. The positivity constraint can
be applied at each iteration.

\begin{figure}[htb]
\vbox{
\centerline{
\hbox{
\psfig{figure=phantom.ps,bbllx=8.2cm,bblly=12.6cm,bburx=12.7cm,bbury=17.1cm,width=7cm,height=7cm,clip=}
\psfig{figure=phantom_p.ps,bbllx=8.2cm,bblly=12.6cm,bburx=12.7cm,bbury=17.1cm,width=7cm,height=7cm,clip=}
}}
\centerline{
\hbox{
\psfig{figure=dec_mr_t24_p_n5_i400_s5.ps,bbllx=8.2cm,bblly=12.6cm,bburx=12.7cm,bbury=17.1cm,width=7cm,height=7cm,clip=}
\psfig{figure=dec_cb_s5_i400.ps,bbllx=8.2cm,bblly=12.6cm,bburx=12.7cm,bbury=17.1cm,width=7cm,height=7cm,clip=}
}}
}
\caption{Top, original image (phantom) and simulated data
(i.e. convolved image plus Poisson noise). Bottom, deconvolved image
by the wavelet based method and the combined approach.}
\label{fig_dec_phantom}
\end{figure}
Figure~\ref{fig_dec_phantom}, top, shows the Logan-Shepp Phantom
and the simulated data, i.e. original image convolved by a Gaussian
PSF (full width at half maximum,
FWHM=3.2) and Poisson noise. Figure~\ref{fig_dec_phantom},
bottom, shows the deconvolution with  (left) a pure wavelet deconvolution
method (no penalization term) and (right) the combined deconvolution method
(parameter $\lambda = 0.4$).

\clearpage
\newpage


\section{Constant Coefficients}
The general form of an nth order linear equation is
\begin{equation*}
	a_n(x)y^{(n)} + a_{n-1}y^{(n-1)} + \ldots + a_1(x)y' + a_0y = b(x)
\end{equation*}
If each $a_i(x)$ is a constant, then the equation has constant coefficients.\\

\noindent
We already know how to solve linear first order differential equations using an integrating factor, but let's see if we can develop a method that can solve any order linear, homogeneous differential equation with constant coefficients.

\ifodd\includeHigherOrderExamples\input{./higherOrder/constCoeffs/constCoeffs_example.tex}\fi

% Auxillary Equation
\input{./higherOrder/constCoeffs/auxillaryEquation.tex}


\documentclass[ag.tex]{subfiles}

\begin{document}

\chapter{Preliminaries}

\section{Commutative algebra}

We begin with some algebraic prerequisites.  For more see \cite{atiyah2018introduction}.

\begin{definition}
A \textit{commutative ring} $(R, +, \cdot, 0,1)$ is an abelian group $(R,+,0)$ along with an associative binary operator $\cdot$ and a unit element $1$ such that for all $x,y,z \in R$

\begin{enumerate}
\item $x \cdot y = y \cdot x$,
\item $x \cdot 1 = 1$, and
\item $x\cdot(y+z) = x\cdot y+x\cdot z$

\end{enumerate}

\end{definition}

We call a commutative ring $(R, +, \cdot , 0,1)$ simply a \textit{ring} and write it as $R$ when there is no ambiguity. Also we denote $x \cdot y$ by $xy$.

\begin{definition}
An \textit{ideal} of a ring $R$ is a subset $I \subset R$ that is closed under addition ($+$) and "absorbs" multiplication from elements of $R$: for all $x \in R$ and $y \in I$,  $xy \in I$.
\end{definition}

\begin{proposition}\label{primes_map_to_primes}
The images and preimages of a prime ideal under ring homomorphisms are prime.
\end{proposition}

\section{Categories and functors}

\begin{definition}
A \textit{category} $\mathcal{C}$ consists of a collection of \textit{objects}, a set $\mathcal{C}(A,B)$ of \textit{morphisms} between any two objects, an \textit{identity} morphism $id_A \in \mathcal{C}(A,A)$ for each object $A$, and a composition law

\begin{equation}
\circ : \mathcal{C} ( B , C ) \times \mathcal{C} ( A , B ) \to \mathcal{C} ( A , C )
\end{equation}

for each triple of objects $A, B, C$.  Composition must be associative, and identity
morphisms must behave as their names indicate: $h \circ (g\circ f)=(h\circ g)\circ f, id\circ f=f$, and $f\circ id=f$
whenever the the composites are defined.
\end{definition}

\begin{definition}\label{terminal object}
A \textit{terminal object} of a category $\mathcal{C}$ is an object $T$ to which there is a unique morphism from each object of $\mathcal{C}$.
\end{definition}

\begin{definition}
A \textit{functor} $F : \mathcal{C} \to \mathcal{D}$ assigns an object $F(A)$ of $\mathcal{D}$ to each object $A$ of $\mathcal{C}$ and a morphism $F(f) : F(A) \to F(B)$ of $\mathcal{D}$ to each morphism $f:A\to B$ of $\mathcal{C}$ such that

\begin{equation}
F(id_A) = id_{F(A)} \text{ and } F(g \circ f) = F(g) \circ F(f).
\end{equation}

\begin{definition}
A \textit{natural transformation} $\alpha : F \to G$ between functors $F, G: \mathcal{C} \to \mathcal{D}$ consists of a morphism $\alpha_A : F(A) \to G(A)$ for each object $A$ of $\mathcal{C}$ such that the following diagram commutes for each morphism $f : A \to B$ of $\mathcal{C}$:

\begin{center}
\begin{tikzcd}
F(A) \arrow[r, "F(f)"] \arrow[d, "\alpha_A"]
& F(B) \arrow[d, "\alpha_B"] \\
G(A) \arrow[r, "G(f)"]
& |[]| G(B)
\end{tikzcd}
\end{center}
\end{definition}

\end{definition}

\end{document}