%!TEX root=ClassNotes.tex

\section{Integrals}
\subsection{Integrals as Areas}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.5\textwidth]{integral.png}
	\caption*{Signed area under the curve is given by $\int_a^b f(t)\:dt$}
\end{figure}
The integral of a function captures the concept of an area.
\begin{align*}
	\int \limits_{a}^b f(t) dt = \mbox{ signed area between the curve $y=f(x)$ and the x-axis \strut}
\end{align*}
where by signed area we mean that the integral is negative if $f(x) < 0$ and hence equals negative of the area in this case (as area is always positive). This is a good {\it interpretation} but not a good {\it definition} as we have not rigorously defined what area is.
Instead we'll define the integral in terms of a limit and then define the (signed) area as the integral.

\begin{exercise}
	Express the following areas as (sums of) integrals. For each problem, draw pictures to justify your answer.
	\begin{enumerate}
		\item Area between the line $x + y = 1$, the x-axis and the y-axis.
		\item Area enclosed between the graph $y = \sin x$ and the x-axis for $x \in [0,2\pi]$.
		\item Area enclosed between the parabola $y=x^2$ and the line $y=1$.
		\item Area of a circle of radius 1.
	\end{enumerate}
\end{exercise}
Several physical quantities can be expressed as integrals, for example,
if $t$ denotes time and $f(t)$ is the speed of a particle then the integral measures the distance traveled, if $t$ denotes length and $f(t)$ is the linear density of a one dimensional object then the integral measures the total mass, etc.

\subsection{Definition of Integral}
We need several preliminary definitions to define the integral.
\begin{definition}$ $
	\begin{enumerate}
		\item For an interval $[a,b]$ a {\bf partition} $P$ is a sequence of real numbers
		      \begin{align*}
			      a = x_0 < x_1 < \dots < x_n = b
		      \end{align*}
		\item A {\bf uniform partition} $P_n$ of $[a,b]$ is the {\it uniform length} partition
		      \begin{align*}
			      x_0 & = a                          \\
			      x_1 & = a + \dfrac{b-a}{n}         \\
			          & \:\:\: \vdots                \\
			      x_i & = a + i \cdot \dfrac{b-a}{n} \\
			          & \:\:\: \vdots                \\
			      x_n & = b
		      \end{align*}
		\item For a partition $P$ define
		      \begin{align*}
			      M_i & = \sup \{ f(x) : x_i \le x \le x_{i+1}\} \\
			      m_i & = \inf \{ f(x) : x_i \le x \le x_{i+1}\}
		      \end{align*}
		      Define the {\bf upper Riemann sum} $U(f,P)$ and the {\bf lower Riemann sum} $L(f,P)$ as
		      \begin{align*}
			      U(f,P) & = \sum \limits_{i=0}^{n-1} M_i \cdot (x_{i+1} - x_i) \\
			      L(f,P) & = \sum \limits_{i=0}^{n-1} m_i \cdot (x_{i+1} - x_i)
		      \end{align*}
	\end{enumerate}
\end{definition}
\begin{figure}[H]
	\centering
	\includegraphics[width=0.8\textwidth]{RiemannSums2.png}
	\includegraphics[width=0.8\textwidth]{RiemannSums1.png}
	\caption*{For finer partitions, the Riemann sums provide better approximations of the integral, hence in the limiting case as $n\rightarrow \infty$ we obtain the exact integral.}
\end{figure}
\begin{definition}
	\label{def:definition_Integral}
	We say that a function $f(x)$ is (Riemann) {\bf integrable} on an interval $[a,b]$ if there exists a real number $I$ such that
	\begin{align*}
		\lim \limits_{ n \rightarrow \infty} U(f,P_n) = I = \lim \limits_{ n \rightarrow \infty} L(f,P_n)
	\end{align*}
	In this case, we define the integral of $f$ on $[a,b]$ as
	\begin{align*}
		\int \limits_{a}^{b} f(t) \: dt & = I
	\end{align*}
\end{definition}
\noindent Thus when a function $f(x)$ is integrable it suffices to compute either $\lim \limits_{ n \rightarrow \infty} U(f,P_n)$ or $\lim \limits_{ n \rightarrow \infty} L(f,P_n)$ to compute the integral. \\

We'll assume the following theorem without proof. The proof requires the notion of {\bf uniform continuity} which is beyond the scope of this class.

\begin{theorem}
	\label{theorem:continuous_are_integrable}
	If $f$ is a continuous function on the interval $[a,b]$ then $f$ is integrable on $[a,b]$.
\end{theorem}

We'll do a few examples to better understand the definition of integral.
\begin{exercise}
	Let $f(x) = -x$, let $a=0$ and $b = 1$.
	\begin{enumerate}
		\item Use basic geometry to find $\int_{0}^1 f(t) \: dt$.
		\item Using pictures describe and compute $L(f,P_1)$, $L(f,P_2)$, and $L(f,P_3)$.
		\item Compute $L(f,P_n)$ where $n$ is a positive integer. You'll need to use
		      \begin{align*}
			      1 + 2 + 3 + \dots + n = \dfrac{n(n+1)}{2}
		      \end{align*}
		\item Find $\lim \limits_{n \rightarrow \infty} L(f,P_n)$, which equals $\int_{0}^1 f(t) \: dt$ by Theorem \ref{theorem:continuous_are_integrable}, and verify that it agrees with part 1.
	\end{enumerate}
\end{exercise}

\begin{exercise}
	Let $f(x) = x^2$, let $a=0$ and $b = 1$.
	\begin{enumerate}
		\item Compute $U(f,P_n)$ where $n$ is a positive integer. You'll need to use the identity
		      \begin{align*}
			      1^2 + 2^2 + 3^2 + \dots + n^2 = \dfrac{n(n+1)(2n+1)}{6}
		      \end{align*}
		\item Find $\lim \limits_{n \rightarrow \infty} U(f,P_n)$, which equals $\int_{0}^1 f(t) \: dt$ by Theorem \ref{theorem:continuous_are_integrable}.
	\end{enumerate}
\end{exercise}

\begin{exercise}
	Let $a = 0$, $b = 1$, and let
	\begin{align*}
		f(x) = \begin{cases}
			1 & \mbox{ if $x$ is rational}    \\
			0 & \mbox{ if $x$ is irrational.}
		\end{cases}
	\end{align*}
	\begin{enumerate}
		\item For $n$ a positive integer, compute $U(f,P_n)$ and $L(f,P_n)$.
		\item Find $\lim \limits_{n \rightarrow \infty} U(f,P_n)$ and $\lim \limits_{n \rightarrow \infty} L(f,P_n)$ and show that the $f(x)$ is not integrable.
	\end{enumerate}
\end{exercise}

\subsubsection{Indefinite Integrals}
We'll assume the following theorem without proof.
\begin{theorem}
	\label{theorem:integral_sum}
	For real numbers $a < b < c$, if $f$ is integrable on $[a,c]$ then
	\begin{align*}
		\int \limits_a ^c f(t) \: dt = \int \limits_a ^b f(t) \: dt  + \int \limits_b ^c f(t) \: dt
	\end{align*}
\end{theorem}
\begin{exercise}
	Draw a picture to explain the statement of the above theorem.
\end{exercise}

So far, we've only defined $\int_a^b f(t) \: dt$ for $a < b$. We now extend this to all real numbers $a$, $b$ by defining
\begin{align*}
	\int \limits_a^a f(t) \:dt
	= 0
	 &  & \mbox{ and } &  &
	\int \limits_a^b f(t) \:dt
	= -\int \limits_b^a f(t) \:dt
	\mbox{ \quad if } a > b
\end{align*}
With this extended notation one can check that Theorem \ref{theorem:integral_sum} is true for all real numbers $a$, $b$, $c$. For example, for $ a < b < c$
\begin{align*}
	         &  &
	\int \limits_a ^c f(t) \: dt = \int \limits_a ^b f(t) \: dt  + \int \limits_b ^c f(t) \: dt \\
	\implies &  &
	\int \limits_a ^c f(t) \: dt = \int \limits_a ^b f(t) \: dt  - \int \limits_c ^b f(t) \: dt \\
	\implies &  &
	\int \limits_a ^c f(t) \: dt + \int \limits_c ^b f(t) \: dt = \int \limits_a ^b f(t) \: dt
\end{align*}

\begin{definition}
	For an integrable function $f$ and a real number $a$, an {\bf antiderivative} is a function on the real numbers defined as
	\begin{align*}
		F_a(x) = \int \limits_a^x f(t) \: dt
	\end{align*}
\end{definition}

\begin{exercise}
	\label{q:difference_antiderivatives}
	For real numbers $a$, $b$ show that
	\begin{align*}
		F_a(x) - F_b(x)
	\end{align*}
	is a constant that does not depend on $x$.
\end{exercise}

Thus for any integrable function $f$ any two antiderivatives differ by a constant. This allows us to {\it define} the {\bf indefinite integral} denoted
\begin{align*}
	\int f(x) \: dx
\end{align*}
as $\int_a^x f(t) \: dt$ for some real number $a$. Changing the constant $a$ changes $F_a(x)$ by a constant, hence the indefinite integral is only defined up to a constant.


\section{Tolerance of Equipment for 1-D PDE Example}\label{ext:pde-tolerance}
Similar to \ref{fig:ode-convergence-std}, we demonstrate that using more sensitive measurement equipment improves the estimation of the MUD point by considering the same choices for $\tau$, the precision of the sensors.
In Figure~\ref{fig:pde-convergence-std}, we study the impact of more precise measurement equipment on the absolute error's mean and variance.

\begin{figure}[htbp]
  \centering
  \includegraphics[width=0.475\linewidth]{figures/pde/pde_convergence_mud_std_mean}
  \includegraphics[width=0.475\linewidth]{figures/pde/pde_convergence_mud_std_var}
  \caption{Convergence of the MUD point given $N=1E4$ model evaluations for different measurement precision for randomly placed sensors, incorporating $S=100$ measurements.
  We note that the convergence rates are the same but the overall accuracy and precision improve when sensors are placed in regions of $u$ that exhibit higher sensitivity to changes in $\param$.
  }
  \label{fig:pde-convergence-std}
\end{figure}


\section{Selection}

%%%%%%%%%%%%%%%%%%%%
\begin{frame}{The $3$rd largest element (Problem 3.1)}
  \[
	V_k(n): \text{$\min$ \#comparisons to find the $k$-th largest element of $n$ elements.}
  \]

  \pause
  \begin{align*}
	V_1(n) &= n - 1 \\
	V_2(n) &= (n - 1) + (\lceil \log n \rceil - 1) \\
  \end{align*}

  \pause
  \[
	V_3(n) = \;?
  \]

  \pause
  \[
	V_3(n) \le (n - 1) + (\lceil \log n \rceil - 1) + \textcolor{red}{(n - 3)}
  \]

  \pause
  \[
	V_3(n) \le (n - 1) + (\lceil \log n \rceil - 1) + \textcolor{red}{(\lceil \log n \rceil - 1)}
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{The $3$rd largest element (Problem 3.1)}
  \centerline{``$Q_1$: What is the exact value of $V_3(n)$?''}

  \begin{theorem}[$V_3(n)$]
	$n \ge 6, n = 2^k + r (0 \le r < 2^k)$:
	\begin{equation*}
	  V_3(n) = \begin{cases}
		(n - 3) + 2 k 	  & r = 0, 1 \\
		(n - 3) + 2 k + 1 & 2 \le r \le 2^{k -2} + 1 \\
		(n - 3) + 2 k + 2 & \text{o.w.} \\
	  \end{cases}
	\end{equation*}
  \end{theorem}

  \begin{alertblock}{References}
	``Selecting the Top Three Elements'' by Aigner, 1982.
  \end{alertblock}

  \pause
  \begin{alertblock}{Reference}
	``The Art of Computer Programming, Vol 3: Sorting and Searching (Section 5.3.3)'' by Donald E. Knuth.
  \end{alertblock}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{The $3$rd largest element (Problem 3.1)}
  \begin{center}
	``$Q_2$: Does your algorithm need to find the $1$st and the $2$nd elements?'' \\[0.30cm] \pause

	``YES!''
  \end{center}

  \pause
  \begin{center}
	``$Q_3$: Do all algorithms have to find the $1$st and the $2$nd elements?'' \\[0.30cm] \pause

	``NO!''
  \end{center}

  \pause
  \begin{alertblock}{References}
	``Selecting the Top Three Elements'' by Aigner, 1982.
  \end{alertblock}

  % \pause
  % \begin{description}
  %   \item[$V_t(n)$:]
  %   \item[$W_t(n)$:]
  %   \item[$U_t(n)$:]
  % \end{description}

  % \pause
  % \begin{align*}
  %   U_1(n) &= V_1(n) = W_1(n) = n - 1 \\
  %   W_2(n) &= V_2(n) = n + \lceil \log n \rceil - 2
  % \end{align*}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Selection with minimum \#comparisons (Problem 3.2)}
  Selecting the median of $5$ elements using $6$ comparisons.

  \fignocaption{width = 0.60\textwidth}{figs/median5.jpg}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Sorting with minimum \#comparisons (Problem 2.4)}
  Sorting $5$ elements using $7$ comparisons.

  \[
    S(5) = 7
  \]

  \pause
  \begin{alertblock}{Reference}
	``The Art of Computer Programming, Vol 3: Sorting and Searching (Section 5.3.1)'' by Donald E. Knuth.
  \end{alertblock}

  \[
	S(21) = 66
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Sorting with minimum \#comparisons (Problem 2.4)}
  Sorting $5$ elements using $7$ comparisons.
  \fignocaption{width = 0.60\textwidth}{figs/median5.jpg}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Medians of sorted arrays (Problem 3.7)}
  \centerline{\url{http://cs.stackexchange.com/a/33129/4911}}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
% \begin{frame}{The largest $k$ elements (Problem 3.5)}
% \end{frame}
%%%%%%%%%%%%%%%%%%%%
% \begin{frame}{Close to median (Problem 3.6)}
% \end{frame}
% %%%%%%%%%%%%%%%%%%%%
% \begin{frame}{Dynamic median (Problem 3.8)}
% \end{frame}
% %%%%%%%%%%%%%%%%%%%%
% \begin{frame}{Weighted median (Problem 3.9)}
% \end{frame}
%%%%%%%%%%%%%%%%%%%%


\SecDef{lindiff}{Resistance against Linear and Differential Cryptanalysis}

Linear and differential cryptanalysis are powerful methods of attacking symmetric cryptographic primitives.

In most block ciphers, S-Boxes are usually the only source of nonlinearity. The resistance of a cipher depends largely on the cryptographic strength of the S-Boxes it uses. Due to typically small sizes of S-Boxes, the linear and differential propagations through them may be analyzed in an exhaustive manner. For this purpose, the \emph{Linear Approximation Table (LAT)} and the \emph{Difference Distribution Table (DDT)} are used. Even though these objects are motivated by the analysis of S-Boxes, they are also useful theoretical tools in the analysis of larger cryptographic functions.

\begin{definition}[Walsh Transform]
The \emph{Walsh transform} $\walsh{f}$ of a Boolean function $f\colon \field{n} \to \field{}$ is defined as:
\begin{align*}
    & \walsh{f}\colon \field{n} \to \ZZ,\\
    & \walsh{f}(a) \eqdef \sum_{x \in \field{n}} (-1)^{f(x)\oplus \inprod{a,x}} = -2^n \cor(f \oplus \varphi_a),
\end{align*}
where $\varphi_a(x) \eqdef \inprod{a,x}$. It can be seen as a multidimensional Fourier transform of the function $x \mapsto (-1)^{f(x)}$. The multiset of all values of the Walsh transform of $f$ is called the \emph{Walsh spectrum} of $f$.
\end{definition}

\begin{definition}[Linear Approximation Table (LAT)]
Let $S: \field{n} \to \field{m}$. The linear approximation table (LAT) of $S$ is the mapping
\begin{align*}
& \LAT{S}: \field{n}\times\field{n} \to \ZZ,\\
& \LAT{S}(a,b) \eqdef \walsh{S_b}(a) = 2\psize{\pset{x \in \field{n} \mid \inprod{a,x} = \inprod{b,S(x)} }} - 2^{n}
= \sum_{x \in \field{n}} (-1)^{\inprod{a,x} \oplus \inprod{b,S(x)}}
.
\end{align*}
$\LAT{S}$ naturally defines a $2^n \times 2^n$ matrix over $\ZZ$ (where the inputs $a,b$ are ordered in the lexicographic order). The columns of $\LAT{S}$ correspond to \emph{Walsh transforms} of the components of $S$.
\end{definition}

I remark that in several papers the LAT is defined with a coefficient $1/2$ or $-1/2$, e.g. in~\cite{OurFeistel}.

\begin{definition}[Difference Distribution Table (DDT)]
Let $S: \field{n} \to \field{m}$. The difference distribution table (DDT) of $S$ is the mapping
\begin{align*}
& \DDT{S}: \field{n}\times\field{n} \to \ZZzeroplus, \\
& \DDT{S}(a,b) =
\psize{\pset{x \in \field{n} \mid S(x\oplus a) \oplus S(x) = b }}
.
\end{align*}
$\DDT{S}$ naturally defines a $2^n \times 2^n$ matrix over $\ZZzeroplus$ (where the inputs $a,b$ are ordered in the lexicographic order).
\end{definition}

The maximum absolute values of the LAT and the DDT of an S-Box are used to measure the cryptographic strength of the S-Box. For this purpose, the \emph{linearity} and the \emph{differential uniformity} of an function are defined.

\begin{definition}[Linearity]
Let $f\colon \field{n} \to \field{}$ be a Boolean function. The \emph{linearity} of $f$ is denoted by $\LIN(f)$ and is defined to be the maximum absolute value in the Walsh spectrum of $f$:
$$
\LIN(f) \eqdef \max_{a \in \field{n}} \pabs{\walsh{f}(a)}
= \max_{a \in \field{n}} \abs{\sum_{x \in \field{n}}(-1)^{f(x) \oplus \inprod{a,x}}}
.$$

Let $S: \field{n} \to \field{m}$ be a Vectorial Boolean function. The \emph{linearity} of $S$ is denoted by $\LIN(S)$ and is equal to the maximum linearity among the components of $S$:
$$
\LIN(S) \eqdef \max_{b \in \field{n}, b \ne 0} \LIN(S_b)
= \max_{a \in \field{n},b\in \field{n},b\ne 0} \abs{\sum_{x \in \field{n}} (-1)^{\inprod{b,f(x)} \oplus \inprod{a,x}}}
.
$$
\end{definition}

\begin{definition}[Differential Uniformity]
Let $f: \field{n} \to \field{m}$. The differential uniformity of $f$ is denoted by $\DU(f)$ and is given by:
$$
\DU(f) = \max_{a\in \field{n}, b\in \field{m}, a\ne 0} \DDT{f}(a,b).
$$
\end{definition}

The entries of the DDT of any S-Box are always even. It follows that the differential uniformity can never be smaller than 2. The functions achieving this lower bound are called \emph{Almost Perfect Nonlinear (APN)}. For example, the cube function over the finite field is always APN~\cite{Nyb94}: $x \mapsto x^3, x \in \fielde{n}$. When $n$ is odd, the cube function is a \emph{permutation} of $\fielde{n}$ and thus is an \emph{APN permutation}. However, it is not bijective when $n$ is even. The question of existence of APN permutations in even dimensions is a long-standing problem. For $n=4$ the answer is known to be negative, and for $n=6$ the positive answer was given by Dillon~\etal{}~\cite{DillonAPN} who explicitly provided a 6-bit APN permutation as a look-up table. In \ChapRef{apn} I describe an interesting decomposition of this function which we found together with my colleagues using S-Box reverse-engineering methods~\cite{OurAPN}. For even $n\ge 8$ the question is still a big open problem.

\subsubsection{Effect of Affine Encodings on the LAT}

Compositions of a function with affine mappings have a simple effect on the function's LAT. The following propositions describe the effect separately for addition of constants and composition with linear maps. The constant addition only affects the signs of the LAT coefficients, and the linear encodings shuffle the LAT coefficients in a linear way.

\begin{proposition}
Let $S\colon \field{n} \to \field{m}$ and let $S'\colon \field{n} \to \field{m}, S'(x) = S(x \oplus c_x) \oplus c_y$ for some $c_x \in \field{n}, c_y \in \field{m}$. Then for any $a \in \field{n}, b \in \field{m}$
$$
\LAT{S'}(a,b) = \LAT{S}(a,b) (-1)^{\inprod{a,c_x}\oplus {\inprod{b, c_y}}}.
$$
\end{proposition}
% \begin{proof}
% \begin{multline*}
% \LAT{S'}(a,b) =
% \sum_{x \in \field{n}} (-1)^{\inprod{a,x} \oplus \inprod{b,S(x \oplus c_x)\oplus  c_y}} =
% \sum_{z \in \field{n}} (-1)^{\inprod{a,z \oplus c_x} \oplus \inprod{b,S(z)\oplus  c_y}} = \\
% = \sum_{z \in \field{n}} (-1)^{\inprod{a,z} \oplus \inprod{b,S(z)}} (-1)^{\inprod{a, c_x} \oplus \inprod{b, c_y}} = \LAT{S}(a,b) (-1)^{\inprod{a, c_x} \oplus \inprod{b, c_y}}.
% \end{multline*}
% \end{proof}

\begin{proposition}
\Label{prop:linear-lat}
Let $S\colon \field{n} \to \field{m}$ and let $S' \eqdef B \circ S \circ A$ for some $A \in \linbij{n}, B \in \linbij{m}$. Then for any $a \in \field{n}, b \in \field{m}$
$$
\LAT{S'}(a,b) = \LAT{S}(\invtop{A} \times a, B \times b).
$$
\end{proposition}
% \begin{proof}
% \begin{multline*}
% \LAT{S'}(a,b) =
% \sum_{x \in \field{n}} (-1)^{\inprod{a,x} \oplus \inprod{b,B(S(A(x)))}} =
% \sum_{z \in \field{n}} (-1)^{\inprod{a,A^{-1}(z)} \oplus \inprod{b,B(S(z))}} = \\
% \sum_{z \in \field{n}} (-1)^{\inprod{\invtop{A} (a),(z)} \oplus \inprod{B^{\top} (b), S(z)}} = \LAT{S}(\invtop{A} \times a, B^{\top} \times b).
% \end{multline*}
% \end{proof}


\section{The \adjacentdifference algorithm}
\Label{sec:adjacentdifference}

The \adjacentdifference algorithm in the \cxx Standard Library \cite[\S 29.8.11]{cxx-17-draft}
computes the differences of adjacent elements in a range.
%
Our version of the original signature reads:

\begin{lstlisting}[style=acsl-block]

size_type
adjacent_difference(const value_type* a, size_type n, value_type* b);
\end{lstlisting}

After executing the function \adjacentdifference the array \inl{b[0..n-1]} holds the following values
\begin{align}
   \mathtt{b}[0] &= \mathtt{a}[0] \nonumber \\
   \mathtt{b}[1] &= \mathtt{a}[1] - \mathtt{a}[0] \nonumber \\
                 &\vdotswithin{=} \nonumber \\
   \mathtt{b}[n-1] &= \mathtt{a}[n-1] - \mathtt{a}[n-2] \nonumber \\
\end{align}

\subsection{The predicate \AdjacentDifference}

We start with the definition of the logic function \Difference whose definition
is shown in the following listing.

\input{Listings/Difference.acsl.tex}

\clearpage

Building on top of \Difference we now introduce the predicate \AdjacentDifference.
We also provide the predicate \AdjacentDifferenceBounds
that captures conditions that prevent numeric overflows
while computing differences of the form \inl{a[i] - a[i-1]}.

\input{Listings/AdjacentDifference.acsl.tex}

Lemmas \logicref{AdjacentDifferenceStep} and \logicref{AdjacentDifferenceSection}
will help us later in the verification of \implref{adjacentdifferenceinv}.

\clearpage

\subsection{Formal specification of \adjacentdifference}

Using the predicates \logicref{AdjacentDifference} and \logicref{AdjacentDifferenceBounds}
we can provide in the following listing a concise formal specification of \adjacentdifference.
As in the case of the specification of \specref{partialsum}
we require that the arrays \inl{a[0..n-1]} and \inl{b[0..n-1]}
are separated.

\input{Listings/adjacent_difference.h.tex}

\clearpage

\subsection{Implementation of \adjacentdifference}

The following listing shows an implementation of \adjacentdifference
with corresponding loop annotations.
In order to achieve the verification of the loop invariant \inl{difference} we
rely on
\begin{itemize}
\item the assertions \inl{bound} and \inl{difference}
\item the lemmas \logicref{AdjacentDifferenceStep} and \logicref{AdjacentDifferenceSection}
\item a statement contract with the two postconditions labeled as \inl{step}
\end{itemize}

\input{Listings/adjacent_difference.c.tex}

\clearpage


\subsection{Optimization problem}
\label{subsec:learningprocess:optimizationproblem}

% ----------------------- paths to graphics ------------------------

% ----------------------- contents from here ------------------------
%

We can define the learning process as follows: given a sample from a dataset, its schema and a set of pattern detectors as input, output a compression tree that, when applied to the dataset, produces a compressed representation of it of minimum disk size.

The schema is a list of columns and their data types. The pattern detectors are implementations of the \nameref{subsec:genericpd}---receives the columns as input, evaluates the sample and returns a list of \textit{(expression node, evaluation result)} tuples. Adding an \textit{expression node} to the compression tree means: 1) altering the schema by deleting existing columns and creating new ones; 2) altering the sample by applying the \textit{compression operator} to the input columns and generating new data. The learning process can go on by recursively feeding the new schema and sample data to the pattern detectors, resulting in new \textit{(expression node, evaluation result)} tuples. This recursive process stops when no pattern detector outputs any result anymore---no pattern matches on the current schema and data.

Each decision of adding or not adding an \textit{expression node} to the compression tree generates a new solution. This leads to a total number of \(2^n\) possible solutions (different compression trees), where \(n\) is the total number of \textit{expression nodes} generated by the recursive process (\(n\) binary decisions: \(1\) means adding a node and \(0\) not adding it). The total number of \textit{expression nodes} (\(n\)) depends on how well pattern detectors match on the initial columns and the newly generated ones. This is entirely dependent on the characteristics of the data and the patterns that were evaluated on it. In the worst case, all pattern detectors will match on any column, leading to the following expression for \(n\):
\begin{equation}
\label{eq:optimizationproblem:n}
    n = c_{in} \times (p \times \mathit{avg}(n_{p}) \times b) ^ h
\end{equation}
\begin{equation}
\label{eq:optimizationproblem:b}
    b = \mathit{avg}(c_{out})
\end{equation}
where:
\begin{itemize}
    \item[] \(n\) = total number of expression nodes
    \item[] \(c_{in}\) = number of input columns
    \item[] \(p\) = number of pattern detectors
    \item[] \(n_{p}\) = number of expression nodes returned by a pattern detector
    \item[] \(b\) = branching factor of the compression tree
    \item[] \(h\) = height of the compression tree
    \item[] \(c_{out}\) = number of output columns of an expression node
\end{itemize}

The score of each solution is given by the size of the compressed data that resulted after applying the compression tree to the dataset. The goal of the learning process is to choose the one that gives the smallest size.

The computational effort needed for each individual \textit{expression node} consists of: 1) applying the compression operator on its input columns to generate the new data (feeding each tuple in the sample data to the operator); 2) evaluating the pattern detectors on all the new columns (feeding each tuple in the sample data to each pattern detector). Moreover, some pattern detectors may need to evaluate combinations of columns instead of individual columns, which requires all the existing columns to be reevaluated for every newly generated column (e.g. \nameref{subsec:pd:columncorrelation} evaluates all pairs of 2 columns to determine the correlation coefficient between them).

\iffalse
TODO:
better formalize problem
https://en.wikipedia.org/wiki/Optimization\_problem
\fi

% ---------------------------------------------------------------------------
% ----------------------- end of thesis sub-document ------------------------
% ---------------------------------------------------------------------------


% !TEX root = index.tex

\section{Dimension}

\begin{qbox}
  Consider the line $L = \{ c \vec{v}: c \in \bbr \} $ where $\vec{v}$ is a non-zero vector in $\bbr^n$. One basis for $L$ is $\calb = \{ \vec{v}\}$.
  \begin{enumerate}
    \item Argue that every basis of $L$ has exactly one element.
  \end{enumerate}
\end{qbox}

\begin{qbox}
  Consider the plane $P = \{ c_1 \vec{v}_1 + c_2 \vec{v}_2: c_1, c_2 \in \bbr \} $ where $\vec{v}_1, \vec{v}_2$ are non-zero vectors in $\bbr^n$ which are not linear multiples of each other.
  One basis for $P$ is $\{ \vec{v}_1, \vec{v}_2 \}$.
  \begin{enumerate}
    \item Argue that every basis of $P$ has exactly two elements.
  \end{enumerate}
\end{qbox}

This generalizes to arbitrary vector spaces.

\begin{theorem}
  \label{theorem:dimensionVectorSpace}
  If a subspace $V$ of $\bbr^n$ has a basis $\calb$ of size $k$ then every basis $\calb'$ of $V$ has size $k$.
\end{theorem}

This number is then defined to be the dimension of the subspace $V$.

\begin{mdframed}
  The proof of Theorem \ref{theorem:dimensionVectorSpace} is a bit long and complicated.
  If you are seeing this for the first time, you can assume this theorem without proof, so the the sections below are optional.
  You should at least skim through them to get an idea of how the proof goes.
\end{mdframed}

We start by analysing arbitrary linearly independent sets inside a vector space.

\begin{proposition}
  \label{proposition:sizeOfLinIndependentSets}
  Let $V$ be a subspace of $\bbr^n$ with a finite basis $\calb$.
  Let $\cals$ be an arbitrary linearly independent subset of $V$.
  Then $\abs{\cals} \le \abs{\calb}$.
\end{proposition}
\begin{proof}
  Let $\calb = \set{\vec{v}_1, \dots, \vec{v}_k}$ be a basis of $V$ and let $\cals$ be a linearly independent subset of $V$. We want to show that $\ell \le k$.
  We will instead prove the contrapositive:
  {\it If  $\ell > k$ then there exist scalars $c_1$, \dots, $c_{\ell}$, not all 0, such that \begin{align}
  \label{equation:linearDependence}
    c_1\vec{w}_1 + \dots + c_\ell \vec{w}_\ell= 0.
  \end{align}}
  In the basis $\calb$, we can express the vectors $\vec{w_i}$'s as coordinate vectors.
  \begin{align*}
    [w_i]_{\calb}
    &=
    \begin{bmatrix}
      A_{1i} \\
      A_{2i} \\
      \vdots \\
      A_{ki}
    \end{bmatrix}
  \end{align*}
  Equation \eqref{equation:linearDependence} then simplifies as
  \begin{align*}
    %&&
    c_1\vec{w}_1 + \dots + c_\ell \vec{w}_\ell&= 0 \\
    \implies %&&
    c_1[\vec{w}_1]_\calb + \dots + c_\ell [\vec{w}_\ell]_\calb&= 0 \\
    \implies %&&
    c_1 \begin{bmatrix}
      A_{11} \\
      A_{21} \\
      \vdots \\
      A_{k1}
    \end{bmatrix}
    +
    \dots
    +
    c_\ell \begin{bmatrix}
      A_{1\ell} \\
      A_{2\ell} \\
      \vdots \\
      A_{k\ell}
    \end{bmatrix}
    &= 0 \\
    \implies
    %&&
    \begin{bmatrix}
      A_{11}c_1 \\
      A_{21}c_1 \\
      \vdots \\
      A_{k1}c_1
    \end{bmatrix}
    +
    \dots
    +
    \begin{bmatrix}
       A_{1\ell} c_\ell\\
       A_{2\ell} c_\ell\\
      \vdots \\
       A_{k\ell}c_\ell
    \end{bmatrix}
    &= 0 \\
    \implies
    %&&
    \begin{bmatrix}
      A_{11} c_1 + \dots + A_{1\ell} c_\ell\\
      A_{21} c_1 + \dots + A_{2\ell} c_\ell\\
      \vdots \\
      A_{k1} c_1 + \dots + A_{k\ell}c_\ell
    \end{bmatrix}
    &= 0 \\\\
    \implies
    %&&
      A_{11} c_1 + \dots + A_{1\ell} c_\ell &= 0 \\
      %&&
      A_{21} c_1 + \dots + A_{2\ell} c_\ell &= 0 \\
      %&&
      \vdots \\
      %&&
      A_{k1} c_1 + \dots +  A_{k\ell}c_\ell &= 0
  \end{align*}
  These are $k$ equations in $\ell$ variables and $\ell > k$, so we have more variables than equations. Such a system is called an \emph{under-determined} linear system. Because the right-hand is zero, such a system always has a solution (Lemma \ref{theorem:underDeterminedLinearSystem}) with not all $c_i$ equal to 0.
\end{proof}

Because every basis is also a linearly independent set, we can use Proposition \ref{proposition:sizeOfLinIndependentSets} to say several things about bases.

\begin{corollary}
  Let $V$ be a subspace of $\bbr^n$.
  \begin{enumerate}
    \item If $\calb$ is a basis of $V$ then $|\calb| \le n$.
    \item If $\calb$ and $\calb'$ are two bases of $V$, then $|\calb| = |\calb'|$.
    \item Every basis of $\bbr^n$ has $n$ elements.
  \end{enumerate}
\end{corollary}

\begin{qbox}
  Prove the above corollaries using Proposition \ref{proposition:sizeOfLinIndependentSets}.
\end{qbox}

Notice that the above theorems compare the sizes of linearly independent sets to the size of a basis. But we have not shown that a basis exists in the first place.

\begin{theorem}
  \label{theorem:existenceOfBasis}
  Let $V$ be a subspace of $\bbr^n$. Then $V$ has a basis.
\end{theorem}
\begin{proof}
  We provide an algorithm for constructing a basis $\calb$.\\\\
  Set $i=0$ and let $S_0 = \varnothing$.
  \begin{enumerate}
    \item If $\spn(S_i) = V$ we are done.
    \item If $\spn(S_i) \neq V$ then there exists a vector $\vec{v}_i$ such that $\vec{v}_i \in V \setminus \spn(S_i)$.
    \item Let $S_{i+1} = S_i \cup \set{\vec{v}_i}$ and increment $i$ by 1. Go back to Step 1.
  \end{enumerate}
  Once this process terminates the final set $S_i$ that we obtain is a basis for $V$.
  \begin{qbox}
    \begin{enumerate}
      \item Show that the sets $S_i$ constructed above are all linearly independent.
      \item Using Proposition \ref{proposition:sizeOfLinIndependentSets} argue that the above algorithm always terminates?
      \item Prove that the final $S_i$ is a basis for $V$?
    \end{enumerate}
  \end{qbox}
\end{proof}

This allows to make the following definition.
\begin{definition}
  For a subspace $V$ of $\bbr^n$, the \emph{dimension} of $V$ is defined to be the size of any basis $\calb$.
  \begin{align*}
    \dim V := \abs{\calb} \mbox{ where } \calb \mbox{ is any basis of } V
  \end{align*}
\end{definition}

\begin{qbox}(Optional)
  Using a similar algorithm it is possible to prove a slightly stronger statement than Theorem \ref{theorem:existenceOfBasis}.
  Let $V \subsetneq V'$ be subspaces of $\bbr^n$. Let $\calb$ be a basis of $V$. Show that $\calb$ can be extended to a basis $\calb'$ of $V'$.
\end{qbox}

\begin{definition}
  If a vector space $V$ has a finite basis $\calb$ then we define
  \begin{align*}
    \dim V = \abs{\calb}
  \end{align*}
  Otherwise we say that $V$ is infinite dimensional.
\end{definition}

\begin{remark}
  Our proofs about subspaces of $\bbr^n$ carry over to arbitrary vector spaces verbatim, except the one about existence of basis. For arbitrary vector spaces, what do you think might go wrong with the proof of Theorem \ref{theorem:existenceOfBasis}?\footnote{The proof of existence of basis of an arbitrary vector space relies on the Axiom of Choice.}
\end{remark}

\begin{lemma}[Under-determined linear system]
  \label{theorem:underDeterminedLinearSystem}
  Consider the following system with $k$ equations and $\ell$ variables. ($A_{ij}$ are scalar constants and we are solving for the $c_i$'s.)
  \begin{align*}
    A_{11} c_1 + \dots + A_{1\ell}c_\ell  &= 0 \\
    A_{21} c_1 + \dots + A_{2\ell}c_\ell  &= 0 \\
    \vdots \\
    A_{k1} c_1 + \dots +  A_{k\ell}c_\ell &= 0
  \end{align*}
  If $k < \ell$, then this system of equations always has a solution such that at least one of the $c_i$'s is non-zero.
\end{lemma}
\begin{proof}
  Proof is induction on $k$.
  \begin{qbox}
    \emph{Base case: }Prove Lemma \ref{theorem:underDeterminedLinearSystem} for $k = 1$.
  \end{qbox}
  \emph{Induction hypothesis: }
  Assume that we know Lemma \ref{theorem:underDeterminedLinearSystem} to be true when $k = m$.

  \emph{Induction step: }
  Consider now $k = m+1$.
  \begin{qbox}
    Argue that if $A_{11} = \dots = A_{1\ell} = 0$ then we are done by the induction hypothesis.
  \end{qbox}
  Without any loss of generality, assume that $A_{11} \neq 0$. Then the first equation gives us
  \begin{align*}
    c_1 = -\dfrac{c_2 A_{12} + \dots + c_l A_{1l}}{A_{11}}
  \end{align*}
  \begin{qbox}
    Plug this in the other equations and conclude the proof using the induction hypothesis.
  \end{qbox}
\end{proof}

\iffalse

\subsection{Optional section: Direct sums and direct products}
Given a collection of vector spaces $\set{V_i}_{i \in I}$, it is possible to construct two vector\todo{Put this in the linear transformations section.}
\begin{definition}
  The \emph{direct sum} $\bigoplus_{i \in I} V_i$ of the vector spaces $\set{V_i}_{i \in I}$ is defined as the collection of finite sums $\vec{v}_1 + \dots + \vec{v}_k$ for $\vec{v}_j$ in some $V_i$.
\end{definition}

\begin{definition}
  The \emph{direct product} $\prod_{i \in I} V_i$ of the vector spaces $\set{V_i}_{i \in I}$ is defined as the collection of tuples $(\vec{v}_i)_{i \in I}$ where $\vec{v}_i \in V_i$.
\end{definition}

\begin{qbox}
  Show that there is a natural linear transformation
  \begin{equation*}
    \varphi: \bigoplus_{i \in I} V_i \longrightarrow \prod_{i \in I} V_i
  \end{equation*}
  which is injective.
  Further, show that $\varphi$ is an isomorphism if and only if $I$ is finite.
\end{qbox}

\subsection{Optional section: Infinite dimensional vector spaces}
The following theorem requires the axiom of choice to prove.
\begin{theorem}
  Every vector space $V$ has a basis.
\end{theorem}
\begin{qbox}
  Try to mimic the proof of Theorem \ref{theorem:existenceOfBasis}. What goes wrong? Can you think of a way to fix it?
\end{qbox}

This innocent theorem has surprising conquences.
\begin{corollary}
  The space of continuous functions $f: \bbr \rightarrow \bbr$ has a basis.
\end{corollary}
\begin{corollary}
  The set of real numbers thought of as a vector space over the rational numbers has a basis. This is called the Hamel basis.
\end{corollary}
% For most of this class you can assume that $V$ is as subspace of $\bbr^n$.
%
% \begin{qbox}
%   direct sums
%   quotients
% \end{qbox}
\fi


\chapter{Source detection: denoising and background modeling}
\label{ch_background}

% \markright{Source detection: denoising and background modeling}

\section{Method}
In some cases such as for Fermi data, the diffuse emission from the Milky Way  makes a relatively intense background. We have to extract this background in order to detect point sources. This diffuse interstellar emission may be modeled, and we want to use such a background model and incorporate a background removal in our denoising algorithm.

We note $\mathbf{Y}$ the data, $\mathbf{B}$ the background we want to remove, and $d^{(b)}_{j}[k]$ the MS-VSTS coefficients of $\mathbf{B}$ at scale $j$ and position $k$. We determine the multi-resolution support by comparing $|d_j[k]-d^{(b)}_{j}[k]|$ with $\kappa \sigma_j$.

We formulate the reconstruction problem as a convex constrained minimization problem:

\begin{equation}
\label{bgr_eq34}
\begin{split}
\text{Arg} \min_{\mathbf{X}} \| \mathbf{ \Phi}^{T}\mathbf{X}\|_1,
\text{s.t.} \\ \: \left\{\begin{array}{c}\mathbf{X} \geqslant 0 , \\\forall (j,k)\in \mathcal{M},      (\mathbf{ \Phi}^{T}\mathbf{X})_j[k]=(\mathbf{ \Phi}^{T}(\mathbf{Y} - \mathbf{B}))_j[k] , \end{array}\right.
\end{split}
\end{equation}

Then, the reconstruction algorithm scheme becomes:
\begin{eqnarray}
\tilde{\mathbf{X}} = P_{+}[\mathbf{ X}^{(n)} + \mathbf{ \Phi} P_{\mathcal{M}} \mathbf{ \Phi}^{T} (\mathbf{ Y} - \mathbf{B} - \mathbf{ X}^{(n)})] , \\
\mathbf{X}^{(n+1)} = \mathbf{ \Phi}\text{ST}_{\lambda_n}[\mathbf{ \Phi}^{T}\tilde{\mathbf{X}}].
\end{eqnarray}

\begin{figure}[htb]
\begin{center}
\includegraphics[width=2.9in]{13822fg24.pdf} \hfill
\includegraphics[width=2.9in]{13822fg25.pdf} \hfill
\includegraphics[width=2.9in]{13822fg26.pdf} \hfill
\includegraphics[width=2.9in]{13822fg27.pdf}
\caption{Theoretical testing for MS-VSTS + IUWT denoising + background removal algorithm (Algorithm~\ref{alg3}). View on a single HEALPix face.
\emph{Top Left}: Simulated background : sum of two Gaussians of standard deviation equal to 0.1 and 0.01 respectively.
\emph{Top Right}: Simulated source: Gaussian of standard deviation equal to 0.01.
\emph{Bottom Left}: Simulated poisson data.
\emph{Bottom Right}: Image denoised with MS-VSTS + IUWT and background removal.
}
\label{background}
\end{center}
\end{figure}
The algorithm is illustrated by the theoretical study in Figure~\ref{background}. We denoise Poisson data while separating a single source, which is a Gaussian of standard deviation equal to 0.01, from a background, which is a sum of two Gaussians of standard deviation equal to 0.1 and 0.01 respectively.

\begin{algorithm}
\caption{MS-VSTS + IUWT Denoising + Background extraction}
\label{alg3}

\begin{algorithmic}[1]
\REQUIRE $\quad$ data $a_0:=\mathbf{Y}$, background $B$, number of iterations $N_{\max}$, threshold $\kappa$. \\
\underline{\emph{\textbf{Detection}}} \\
\FOR{$j=1$ to $J$}
\STATE Compute $a_j$ and $d_j$ using (\ref{eq27}).
\STATE Hard threshold $(d_j[k] - d^{(b)}_{j}[k])$ with threshold $\kappa \sigma_j$ and update $\mathcal{M}$.
\ENDFOR \\
\underline{\emph{\textbf{Estimation}}} \\
\STATE Initialize $\mathbf{X}^{(0)}=0$, $\lambda_0 = 1$.
\FOR{$n=0$ to $N_{\max}-1$}
\STATE $\tilde{\mathbf{X}}= P_{+}[\mathbf{ X}^{(n)} + \mathbf{ \Phi} P_{\mathcal{M}} \mathbf{ \Phi}^{T} (\mathbf{ Y} - \mathbf{B} - \mathbf{ X}^{(n)})]$.
\STATE $\mathbf{X}^{(n+1)} = \mathbf{ \Phi}\text{ST}_{\lambda_n}[\mathbf{ \Phi}^{T}\tilde{\mathbf{X}}]$.
\STATE $\lambda_{n+1} = \frac{N_{\max} - (n+1)}{N_{\max} - 1}$.
\ENDFOR
\STATE Get the estimate $\hat{\mathbf{\Lambda}} = \mathbf{X}^{(N_{\max})}$.
\end{algorithmic}
\end{algorithm}

Like Algorithm~\ref{alg1}, Algorithm~\ref{alg3} can be adapted to make multiresolution support adaptation.

\section{Experiment}

We applied Algorithms~\ref{alg3} on simulated Fermi data. To test the efficiency of our method, we detect the sources with the SExtractor routine~\citep{astro:bertin96}, and compare the detected sources with the theoretical sources catalog to get the number of true and false detections. Results are shown on Figures~\ref{sources} and~\ref{sourcesreest}. The SExtractor method was applied on the first wavelet scale of the reconstructed map, with a detection threshold equal to 1. It has been chosen to optimise the number of true detections. SExtractor makes $593$ true detections and $71$ false detections on the Fermi simulated map restored with Algorithm~\ref{alg4} among the $1000$ sources of the simulation. On noisy data, many fluctuations due to Poisson noise are detected as sources by SExtractor, which leads to a big number of false detections (more than 2000 in the case of Fermi data).

\begin{figure}[htb]
\begin{center}
\includegraphics[width=2.9in]{13822fg28.pdf} \hfill
\includegraphics[width=2.9in]{13822fg29.pdf} \hfill
\includegraphics[width=2.9in]{13822fg30.pdf} \hfill
\includegraphics[width=2.9in]{13822fg31.pdf} \hfill
\includegraphics[width=2.9in]{13822fg32.pdf}
\caption{\emph{Top Left}: Simulated background model.
\emph{Top Right}: Simulated Gamma Ray sources.
\emph{Middle Left}: Simulated Fermi data with Poisson noise.
\emph{Middle Right}: Reconstructed Gamma Ray Sources with MS-VSTS + IUWT + background removal (Algorithm~\ref{alg3}) with threshold $5\sigma_j$.
\emph{Bottom}: Reconstructed Gamma Ray Sources with MS-VSTS + IUWT + background removal (Algorithm~\ref{alg3}) with threshold $3\sigma_j$.
Pictures are in logarithmic scale.}
\label{sources}
\end{center}
\end{figure}

\begin{figure}[htb]
\begin{center}
\includegraphics[width=2.5in]{13822fg33.pdf} \hfill
\includegraphics[width=2.5in]{13822fg34.pdf} \hfill
\includegraphics[width=2.5in]{13822fg35.pdf} \hfill
\includegraphics[width=2.5in]{13822fg36.pdf} \hfill
\includegraphics[width=2.5in]{13822fg37.pdf}
\caption{View of a single HEALPix face from the results of Figure~\ref{sources}.\emph{Top Left}: Simulated background model.
\emph{Top Right}: Simulated Gamma Ray sources.
\emph{Middle Left}: Simulated Fermi data with Poisson noise.
\emph{Middle Right}: Reconstructed Gamma Ray Sources with MS-VSTS + IUWT + background removal (Algorithm~\ref{alg3}) with threshold $5\sigma_j$.
\emph{Bottom}: Reconstructed Gamma Ray Sources with MS-VSTS + IUWT + background removal (Algorithm~\ref{alg3}) with threshold $3\sigma_j$.
Pictures are in logarithmic scale.
}
\label{sourcesreest}
\end{center}
\end{figure}

\subsection{Sensitivity to model errors}
\begin{table}
  \centering
  \caption{Percent of true and false detection and signal-noise ratio versus the standard deviation of the Gaussian noise on the background model.
  }
  \begin{tabular}{|c|c|c|c|}
\hline
Model error std dev & $\%$ of true detect & $\%$ of false detect & SNR (dB) \\
\hline
  0 & $59.3\%$ & $7.1\%$ & 23.8 \\
  10 & $57.0\%$ & $11.0\%$ & 23.2 \\
  20 & $53.2\%$ & $18.9\%$ & 22.6 \\
  30 & $49.1\%$ & $43.5\%$ & 21.7 \\
  40 & $42.3\%$ & $44.3\%$ & 21.0 \\
  50 & $34.9\%$ & $39.0\%$ & 20.3 \\
  60 & $30.3\%$ & $37.5\%$ & 19.5 \\
  70 & $25.0\%$ & $34.6\%$ & 18.9 \\
  80 & $23.0\%$ & $28.5\%$ & 18.7 \\
  90 & $23.6\%$ & $27.1\%$ & 18.3 \\
\hline
\end{tabular}

  \label{table1}
\end{table}

As it is difficult to model the background precisely, it is important to study the sensitivity of the method to model errors. We add a stationary Gaussian noise to the background model, we compute the MS-VSTS + IUWT with threshold $3\sigma_j$ on the simulated Fermi Poisson data with extraction of the noisy background, and we study the percent of true and false detections with respect to the total number of sources of the simulation and the signal-noise ratio ($\text{SNR} (dB) = 20 \log (\sigma_{signal} / \sigma_{noise})$) versus the standard deviation of the Gaussian perturbation. Table~\ref{table1} shows that, when the standard deviation of the noise on the background model becomes of the same range as the mean of the Poisson intensity distribution ($\lambda_{\text{mean}} = 68.764$), the number of false detections increases, the number of true detections decreases and the signal noise ratio decreases. While the perturbation is not too strong (standard deviation $< 10$), the effect of the model error remains low.


%!TEX root = forallxyyc.tex
\chapter[Quick reference]{Quick reference}
%\pagestyle{plain}
\section{Characteristic Truth Tables}
\label{app.CharacteristicTTs}

\begin{tabular}{c|c}
\meta{A} & \enot\meta{A}\\
\hline
T & F\\
F & T \\
\phantom{.}\\
\phantom{.}
\end{tabular}
\hfill
\begin{tabular}{c c|c|c|c|c}
\meta{A} & \meta{B} & $\meta{A}\eand\meta{B}$ & $\meta{A}\eor\meta{B}$ & $\meta{A}\eif\meta{B}$ & $\meta{A}\eiff\meta{B}$\\
\hline
T & T & T & T & T & T\\
T & F & F & T & F & F\\
F & T & F & T & T & F\\
F & F & F & F & T & T
\end{tabular}

\vfill

\section{Symbolization}
\begin{center}
\label{app.symbolization}
\begin{tabular*}{\textwidth}{rl}
\multicolumn{2}{c}{\textsc{Sentential Connectives}}\\ \\
It is not the case that P & $\enot P$\\
Either P, or Q & $(P \eor Q)$\\
Neither P, nor Q & $\enot(P \eor Q)$\ or \ $(\enot P \eand \enot Q)$\\
Both P, and Q & $(P \eand Q)$\\
If P, then Q & $(P \eif Q)$\\
P only if Q & $(P \eif Q)$\\
P if and only if Q & $(P \eiff Q)$\\
P unless Q & $(P \eor Q)$\\
\\
\multicolumn{2}{c}{\label{SymbolizingPredicates}\textsc{Predicates}}\\ \\
All Fs are Gs & $\forall x(Fx \eif Gx)$\\
Some Fs are Gs & $\exists x(Fx \eand Gx)$\\
Not all Fs are Gs & $\enot\forall x(Fx \eif Gx)$\ or\ $\exists x(Fx \eand \enot Gx)$\\
No Fs are Gs & $\forall x(Fx \eif\enot Gx)$\ or\ $\enot\exists x(Fx \eand Gx)$\\
\\
\multicolumn{2}{c}{\textsc{Identity}}\\ \\
Only c is G & $\forall x(Gx \eiff x=c)$\\
Everything besides c is G & $\forall x(\enot x = c \eif Gx)$\\
%$j$ is more $R$ than anyone else. & $\forall x(x\neq j \eif Rjx)$\\
The F is G & $\exists x(Fx \eand \forall y(Fy \eif x=y) \eand Gx)$\\
It is not the case that the F is G & $\enot\exists x(Fx \eand \forall y(Fy \eif x=y) \eand Gx)$\\
The F is non-G & $\exists x(Fx \eand \forall y(Fy \eif x=y) \eand \enot Gx)$
\end{tabular*}
\end{center}

% BEGIN: symbolizing cardinality

\newpage
\section{Using identity to symbolize quantities}

\subsection*{There are at least \blank\ Fs.}
\label{summary.atleast}

\begin{ekey}
\item[\text{one}] $\exists xFx$
\item[\text{two}] $\exists x_1\exists x_2(Fx_1 \eand Fx_2 \eand \enot x_1  = x_2)$
\item[\text{three}] $\exists x_1\exists x_2\exists x_3(Fx_1 \eand Fx_2 \eand Fx_3 \eand \enot x_1 = x_2 \eand\enot x_1 = x_3 \eand \enot x_2 = x_3)$
\item[\text{four}] $\exists x_1\exists x_2\exists x_3\exists x_4 (Fx_1 \eand Fx_2 \eand Fx_3 \eand Fx_4 \eand \phantom{x}$\\
\phantom{$\exists x_1\exists x_2$}$\enot x_1 = x_2 \eand \enot x_1 = x_3 \eand \enot x_1 = x_4 \eand \enot x_2 = x_3 \eand \enot x_2 = x_4 \eand \enot x_3 = x_4)$
\item[n] $\exists x_1\ldots\exists x_n(Fx_1 \eand\ldots\eand Fx_n \eand \enot x_1 = x_2 \eand\ldots\eand \enot x_{n-1} = x_n)$
\end{ekey}

\subsection*{There are at most \blank\ Fs.}
\label{summary.atmost}

One way to say `there are at most $n$ Fs' is to put a negation sign in front of the symbolization for `there are at least $n+1$ Fs'. Equivalently, we can offer:
\begin{ekey}
\item[\text{one}] $\forall x_1\forall x_2\bigl[(Fx_1 \eand Fx_2) \eif x_1=x_2\bigr]$
\item[\text{two}] $\forall x_1\forall x_2\forall x_3\bigl[(Fx_1 \eand Fx_2 \eand Fx_3) \eif (x_1=x_2 \eor x_1=x_3 \eor x_2=x_3)\bigr]$
\item[\text{three}] $\forall x_1\forall x_2\forall x_3\forall x_4\bigl[(Fx_1 \eand Fx_2 \eand Fx_3 \eand Fx_4) \eif \phantom{.}$\\
\phantom{$\exists x_1 \exists x_2$}$(x_1=x_2 \eor x_1=x_3 \eor x_1=x_4 \eor x_2=x_3 \eor x_2=x_4 \eor x_3=x_4)\bigr]$
\item[n]$\forall x_1\ldots\forall x_{n+1}
\bigl[(Fx_1\eand \ldots \eand Fx_{n+1}) \eif (x_1=x_2 \eor \ldots \eor x_n=x_{n+1})\bigr]$
\end{ekey}

\subsection*{There are exactly \blank\ Fs.}
\label{summary.exactly}

One way to say `there are exactly $n$ Fs' is to conjoin two of the symbolizations above and say `there are at least $n$ Fs and there are at most $n$ Fs.' The following equivalent formulae are shorter:
\begin{ekey}
\item[\text{zero}] $\forall x\enot Fx$
\item[\text{one}] $\exists x\bigl[Fx \eand \forall y(Fy \eif x= y)\bigr]$
\item[\text{two}] $\exists x_1\exists x_2\bigl[Fx_1 \eand Fx_2 \eand \enot x_1 = x_2 \eand \forall y\bigl(Fy \eif (y= x_1 \eor y = x_2)\bigr) \bigr]$
\item[\text{three}] $\exists x_1\exists x_2\exists x_3\bigl[Fx_1 \eand Fx_2 \eand Fx_3 \eand \enot x_1 =  x_2 \eand \enot  x_1 = x_3 \eand \enot x_2 = x_3 \eand \phantom{.}$\\
\phantom{$\exists x_1 \exists x_2$}$\forall y\bigl(Fy \eif (y = x_1 \eor y = x_2 \eor y =  x_3)\bigr) \bigr]$
\item[n] $\exists x_1\ldots\exists x_n\bigl[Fx_1 \eand\ldots\eand Fx_n  \eand \enot x_1 = x_2 \eand\ldots\eand \enot x_{n-1}= x_n \eand \phantom{.}$\\
\phantom{$\exists x_1\exists x_2$}$\forall y\bigl(Fy \eif (y= x_1 \eor \ldots \eor y= x_n)\bigr)\bigr]$
%\item[one] $\exists x\forall y\bigl[Fx \eand (Fy \eif y = x)\bigr]$
%\item[two] $\exists x\exists y\forall z\Bigl(Fx \eand Fy \eand \bigl[Fz \eif (z=x \eor z=y)\bigr] \eand x \neq y\Bigr)$
%\item[three] $\exists x_1\exists x_2\exists x_3\forall y\Bigl(Fx_1 \eand Fx_2 \eand Fx_3 \eand [Fy \eif (y=x_1 \eor y=x_2 \eor y=x_3)] \eand x_1 \neq x_2 \eand x_1 \neq x_3 \eand x_2 \neq x_3\Bigr)$
%\item[n] $\exists x_1\cdots\exists x_n\forall y\Bigl(Fx_1 \eand\cdots\eand Fx_n \eand \bigl[Fy \eif (y=x_1 \eor \cdots \eor y=x_n)\bigr] \eand x_1 \neq x_2 \eand\cdots\eand x_{n-1}\neq x_n\Bigr)$
\end{ekey}

\label{ProofRules}
\newpage\section{Basic deduction rules for TFL}
\renewenvironment{proof}
	{\noindent\par\noindent\small$\begin{nd}}
	{\end{nd}$\noindent\normalsize\ignorespacesafterend}

%{\LARGE \textbf{Basic Rules of Proof}}
\begin{multicols}{2}

\subsection*{Conjunction}

\begin{proof}
	\have[m]{a}{\meta{A}}
	\have[n]{b}{\meta{B}}
	\have[\ ]{c}{\meta{A}\eand\meta{B}} \ai{a, b}

	\have[m]{ab}{\meta{A}\eand\meta{B}}
\\	\have[\ ]{a}{\meta{A}} \ae{ab}

	\have[m]{ab}{\meta{A}\eand\meta{B}}
\\	\have[\ ]{b}{\meta{B}} \ae{ab}
\end{proof}

\subsection*{Conditional}

\begin{proof}
	\open
		\hypo[i]{a}{\meta{A}}
		\have[j]{b}{\meta{B}}
	\close
	\have[\ ]{ab}{\meta{A}\eif\meta{B}}\ci{a-b}

	\have[m]{ab}{\meta{A}\eif\meta{B}}
\\	\have[n]{a}{\meta{A}}
	\have[\ ]{b}{\meta{B}} \ce{ab,a}
\end{proof}

\subsection*{Negation}

\begin{proof}
\open
	\hypo[i]{a}{\meta{A}}
	\have[j]{nb}{\ered}
\close
\have[\ ]{na}{\enot\meta{A}}\ni{a-nb}

\have[m]{na}{\enot\meta{A}}
\\ \have[n]{a}{\meta{A}}
\have[ ]{bot}{\ered}\ri{na, a}
\end{proof}

\subsection*{Indirect proof}

\begin{proof}
\open
	\hypo[i]{a}{\enot\meta{A}}
	\have[j]{nb}{\ered}
\close
\have[\ ]{na}{\meta{A}}\ip{a-nb}
\end{proof}

\subsection*{Explosion}

\begin{proof}
\have[m]{bot}{\ered}
\\\have[ ]{}{\meta{A}}\re{bot}
\end{proof}

\subsection*{Disjunction}

\begin{proof}
	\have[m]{a}{\meta{A}}
	\have[\ ]{ab}{\meta{A}\eor\meta{B}}\oi{a}

	\have[m]{a}{\meta{A}}
\\	\have[\ ]{ba}{\meta{B}\eor\meta{A}}\oi{a}

	\have[m]{ab}{\meta{A}\eor\meta{B}}
\\	\open
		\hypo[i]{a}{\meta{A}}
		\have[j]{c1}{\meta{C}}
	\close
	\open
		\hypo[k]{b}{\meta{B}}
		\have[l]{c2}{\meta{C}}
	\close
	\have[\ ]{c}{\meta{C}} \oe{ab,a-c1, b-c2}
\end{proof}

\subsection*{Biconditional}

\begin{proof}
	\open
		\hypo[i]{a1}{\meta{A}}
		\have[j]{b1}{\meta{B}}
	\close
	\open
		\hypo[k]{b2}{\meta{B}}
		\have[l]{a2}{\meta{A}}
	\close
	\have[\ ]{ab}{\meta{A}\eiff\meta{B}}\bi{a1-b1,b2-a2}

	\have[m]{ab}{\meta{A}\eiff\meta{B}}
\\	\have[n]{a}{\meta{A}}
	\have[\ ]{b}{\meta{B}} \be{ab,a}

	\have[m]{ab}{\meta{A}\eiff\meta{B}}
\\	\have[n]{a}{\meta{B}}
	\have[\ ]{b}{\meta{A}} \be{ab,a}
\end{proof}

\end{multicols}

\newpage
\section{Derived rules for TFL}
\begin{multicols}{2}
\subsection*{Disjunctive syllogism}
\begin{proof}
	\have[m]{ab}{\meta{A} \eor \meta{B}}
	\have[n]{nb}{\enot \meta{A}}
	\have[\ ]{con}{\meta{B}}\by{DS}{ab, nb}

	\have[m]{ab}{\meta{A} \eor \meta{B}}
\\	\have[n]{nb}{\enot \meta{B}}
	\have[\ ]{con}{\meta{A}}\by{DS}{ab, nb}
\end{proof}

\subsection*{Reiteration}

\begin{proof}
	\have[m]{a}{\meta{A}}
	\have[\ ]{c}{\meta{A}} \by{R}{a}
\end{proof}

\subsection*{Modus Tollens}

\begin{proof}
	\have[m]{ab}{\meta{A}\eif\meta{B}}
	\have[n]{a}{\enot\meta{B}}
	\have[\ ]{b}{\enot\meta{A}} \by{MT}{ab,a}
\end{proof}

\subsection*{Double-negation elimination}
	\begin{proof}
		\have[m]{dna}{\enot \enot \meta{A}}
		\have[ ]{a}{\meta{A}}\dne{dna}
	\end{proof}

\subsection*{Excluded middle}
	\begin{proof}
		\open
			\hypo[i]{a}{\meta{A}}
			\have[j]{c1}{\meta{B}}
		\close
		\open
			\hypo[k]{b}{\enot\meta{A}}
			\have[l]{c2}{\meta{B}}
		\close
		\have[\ ]{ab}{\meta{B}}\tnd{a-c1,b-c2}
	\end{proof}

%
%\subsection*{Hypothetical Syllogism}
%
%\begin{proof}
%	\have[m]{ab}{\meta{A}\eif\meta{B}}
%	\have[n]{bc}{\meta{B}\eif\meta{C}}
%	\have[\ ]{ac}{\meta{A}\eif\meta{C}}\by{HS}{ab,bc}
%\end{proof}

\subsection*{De Morgan Rules}
\begin{proof}
	\have[m]{ab}{\enot (\meta{A} \eor \meta{B})}
	\have[\ ]{dm}{\enot \meta{A} \eand \enot \meta{B}}\dem{ab}

	\have[m]{ab}{\enot \meta{A} \eand \enot \meta{B}}
\\	\have[\ ]{dm}{\enot (\meta{A} \eor \meta{B})}\dem{ab}

	\have[m]{ab}{\enot (\meta{A} \eand \meta{B})}
\\	\have[\ ]{dm}{\enot \meta{A} \eor \enot \meta{B}}\dem{ab}

	\have[m]{ab}{\enot \meta{A} \eor \enot \meta{B}}
\\	\have[\ ]{dm}{\enot (\meta{A} \eand \meta{B})}\dem{ab}
\end{proof}
\end{multicols}

\newpage

\section{Basic deduction rules for FOL}

\begin{multicols}{2}
\subsection*{Universal elimination}

\begin{proof}
	\have[m]{a}{\forall \meta{x}\meta{A}(\ldots \meta{x} \ldots \meta{x}\ldots)}
	\have[\ ]{c}{\meta{A}(\ldots \meta{c} \ldots \meta{c}\ldots)} \Ae{a}
\end{proof}

\subsection*{Universal introduction}

\begin{proof}
	\have[m]{a}{\meta{A}(\ldots \meta{c} \ldots \meta{c}\ldots)}
	\have[\ ]{c}{\forall \meta{x}\meta{A}(\ldots \meta{x} \ldots \meta{x}\ldots)} \Ai{a}
\end{proof}

\medskip\begin{raggedright}
\meta{c} must not occur in any undischarged assumption

\meta{x} must not occur in\\ $\meta{A}(\ldots \meta{c} \ldots \meta{c}\ldots)$
\end{raggedright}

\subsection*{Existential introduction}

\begin{proof}
	\have[m]{a}{\meta{A}(\ldots \meta{c} \ldots \meta{c}\ldots)}
	\have[\ ]{c}{\exists \meta{x}\meta{A}(\ldots \meta{x} \ldots \meta{c}\ldots)}\Ei{a}
\end{proof}

\medskip\begin{raggedright}
\noindent \meta{x} must not occur in\\ $\meta{A}(\ldots \meta{c} \ldots \meta{c}\ldots)$
\end{raggedright}
%\noindent You can replace one or more instance of \meta{c} with \meta{x}.

\subsection*{Existential elimination}

\begin{proof}
	\have[m]{a}{\exists \meta{x}\meta{A}(\ldots \meta{x} \ldots \meta{x}\ldots)}
	\open
		\hypo[i]{b}{\meta{A}(\ldots \meta{c} \ldots \meta{c}\ldots)}
		\have[j]{c}{\meta{B}}
	\close
	\have[\ ]{d}{\meta{B}}\Ee{a,b-c}
\end{proof}

\medskip\begin{raggedright}
\noindent \meta{c} must not occur in any undischarged assumption, in $\exists \meta{x}\meta{A}(\ldots \meta{x} \ldots \meta{x}\ldots)$, or in \meta{B}\end{raggedright}\vfill\columnbreak

\end{multicols}

\subsection*{Identity introduction}

\begin{proof}
	\have[\ \,\,\,]{x}{\meta{c}=\meta{c}} \by{=I}{}
\end{proof}

\subsection*{Identity elimination}

\begin{multicols}{2}
\begin{proof}
	\have[m]{e}{\meta{a}=\meta{b}}
	\have[n]{a}{\meta{A}(\ldots \meta{a} \ldots \meta{a}\ldots)}
	\have[\ ]{ea1}{\meta{A}(\ldots \meta{b} \ldots \meta{a}\ldots)} \by{=E}{e,a}
\end{proof}
\begin{proof}
	\have[m]{e}{\meta{a}=\meta{b}}
	\have[n]{a}{\meta{A}(\ldots \meta{b} \ldots \meta{b}\ldots)}
	\have[\ ]{ea2}{\meta{A}(\ldots \meta{a} \ldots \meta{b}\ldots)} \by{=E}{e,a}
\end{proof}
\end{multicols}

\begin{minipage}{\textwidth} % hack to keep section header with table
\section{Derived rules for FOL}

\begin{multicols}{2}
\begin{proof}
	\have[m]{ab}{\forall \meta{x}\enot \meta{A}}
	\have[\ ]{ac}{\enot \exists \meta{x} \meta{A}}\cq{m}

	\have[m]{ab}{\enot \exists \meta{x}  \meta{A}}
\\	\have[\ ]{ac}{\forall \meta{x}\enot\meta{A}}\cq{m}
\end{proof}
\begin{proof}
	\have[m]{ab}{\exists \meta{x}\enot\meta{A}}
	\have[\ ]{ac}{\enot \forall \meta{x} \meta{A}}\cq{m}

	\have[m]{ab}{\enot \forall \meta{x}  \meta{A}}
\\	\have[\ ]{ac}{\exists \meta{x}\enot \meta{A}}\cq{m}
\end{proof}
\end{multicols}
\end{minipage}


\section{M-estimation}

Generalization of maximum likelihood estimation. No statistical model needs to be assumed to perform M-estimation.\\

Median

\section{Hubert loss}

$h_\delta (x) = \begin{cases}  \frac{x^2}{2} \quad \text {if} \,  \,  \left| x \right| < \delta \\ \delta ( \left| x \right| - \delta /2 ) \quad \text {if} \,  \,  \left| x \right| > \delta \end{cases}.$

the derivative of Huber's loss is the clip function :

$\text {clip}_\delta (x) := \frac{d}{dx} h_\delta (x) = \begin{cases}  \delta \quad \text {if} \,  \,  x > \delta \\ x \quad \text {if} \,  \,  -\delta \leq x \leq \delta \\ -\delta \quad \text {if} \,  \,  x < -\delta \\ \end{cases}$


\documentclass[10pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{makeidx}
\usepackage{graphicx}
\newtheorem{defi}{Definition}
\author{moi}
\begin{document}
\section{Introduction}
Let $R$ be a unital commutative ring, $(A,\mu,\eta)$ an $R$ algebra and $\alpha \in \mathrm{Aut}_{R-\mathrm{alg}}(A)$. We are interested in certain Ore extension over polynomial rings in multiple variables.
\subsection{$\alpha$ derivations and Ore extensions}
\begin{defi}
We call a map $\delta \in \mathrm{End}_R(A)$ an $\alpha$ derivation if
$$\delta(a b) = \alpha(a) \delta(b) + \delta(a) b$$
for all $a, b \in A$. Let $\alpha$ and $\delta$ be as above, an Ore extension $A[X,\alpha,\delta]$ over $A$ is an $A$ algebra with
$$X a = \alpha(a) X + \delta(a)$$
defining the $A$ right module structure.
\end{defi}
Our question can be reformulated as given $A = R[x]$, we are interested in all $\alpha$ derivations on $R[x]$.
\subsection{Generator action}
As $A$ is spanned by all finite sums of powers over $R$, we get:
$$\delta_\alpha(x^i) = \alpha(x) \delta_\alpha(x^{i-1}) + \delta_\alpha(x) x^{i-1}.$$
Therefore:
$$\delta_\alpha(x^2) = \alpha(x) \delta_\alpha(x) + \delta_\alpha(x) x$$
and in case $A$ is commutative nicely:
$$(\alpha + id_A)(x) \delta_\alpha(x) = \delta_\alpha(x^2)$$
and generally in a tensorial manner:
$$(\alpha \otimes \delta_\alpha + \delta_\alpha \otimes id_A) (x \otimes x) = \delta_\alpha(\mu_A(x \otimes x)).$$
This shows that $\delta_\alpha$ is a $(\alpha,id_A)$ skew-primitive element in the coalgebra generated by group-likes (algebra automorphisms) and skew-primitive elements (generalized derivations).
\paragraph{Example} Let us consider
$$\alpha = [x \longmapsto x - 1] \in \mathrm{Aut}_{R-\mathrm{alg}}(A)$$
as automorphism. We get
$$\begin{array}{rclcl}
\delta_\alpha(x^2) &=& (x - 1) \delta_\alpha(x) + x \delta_\alpha(x) &=& (2 x - 1) \delta_\alpha(x)\\
&&&&\\
\delta(x^3) &=& (x - 1) \delta_\alpha(x^2) + \delta_\alpha(x) x^2 &=& (x - 1)(2 x - 1) \delta_\alpha(x) + x^2 \delta_\alpha(x)\\
&&&&\\
&=& (2 x^2 - 3 x + 1 + x^2) \delta_\alpha(x) &=& (3 x^2 - 3 x + 1) \delta_\alpha(x)\\
&\vdots&&\vdots&\\
\delta_\alpha(x^i) &=& (x - 1) \delta_\alpha(x^{i-1}) + \delta_\alpha(x) x^{i-1} &=& (x^i - (x - 1)^i) \delta_\alpha(x^i)\\
\end{array}$$
We want to show that any $R$ derivation (in sense of Ore extensions, an $id_A$ derivation) can be/can not be extended to $R[x]$ as an $\alpha$ derivation
\end{document}


\section{Additional Background} \label{preliminaries}
We denote the security parameter by $\lambda$ and use ``$||$'' as concatenation operator
(i.e., if $a$ and $b$ are two strings then by $a||b$ we denote the concatenation of $a$ and $b$).
For a finite set $Q$, $x\from Q$ denotes a sampling of $x$ from $Q$
with uniform distribution.
We use the abbreviation \ppt\ that stands for probabilistic polynomial time. We use $\poly(\cdot)$
 to indicate a generic polynomial function.
When it is necessary to refer to the randomness $r$ used by and algorithm $A$ we use the following notation: $A(\cdot;r)$.
We say a function $\negligible$ is {\em negligible} if for every positive integer $c$ there is an integer $N_c$ such that for all $x > N_c$, $|\negligible(x)|< 1/x^c$.
We say $a \stackrel{\text{negl}}{\leq} b$ for real values $a, b$ to mean that $a \leq b + \negligible(\lambda)$ for negligible function $\negligible$.
We denote with $[n]$ the set  $\{1,\dots,n\}$, with $\mathbb{F}$ an arbitrary (but fixed) finite field and with $\mathbb{N}$ the set of non-negative integer.


\input{../header.tex}
\title{\vspace{-2cm}INF3490/INF4490 Exercise Solutions - Representations}
\author{Ole Herman S. Elgesem\input{../author_footnote.tex}}
\date{}

% Removing paragraph indents is sometimes useful:
\setlength\parindent{0pt}
% ==============================================================================

% ================================= DOCUMENT ===================================
\begin{document}
    \renewcommand\marginsymbol[1][0pt]{%
  \tabto*{0cm}\makebox[-1cm][c]{$\mathbb{P}$}\tabto*{\TabPrevPos}}

\maketitle
\input{../intro.tex}

\section{Representations}
Recall all the representations that have been presented.
Which mutation and recombination operators are compatible with which representations?\\

\textit{Answer:}

\begin{itemize}
    \item Binary representation
    \begin{itemize}
        \item Bit-flip mutation
        \item N-point and uniform crossover
    \end{itemize}
    \item Integer representation
    \begin{itemize}
        \item Random reset and creep mutation
        \item N-point and uniform crossover
    \end{itemize}
    \item Cardinal/enumerated/symbolic representations
    \begin{itemize}
        \item Random reset mutation
        \item N-point and uniform crossover
    \end{itemize}
    \item Real-valued/Continuous representation
    \begin{itemize}
        \item Uniform and Gaussian mutation
        \item N-point, discrete uniform and arithmetic crossover
    \end{itemize}
    \item Permutation representation
    \begin{itemize}
        \item Swap, insert, scramble and invert mutation
        \item Partially mapped, order, cycle and edge crossover
    \end{itemize}
    \item Tree representation
    \begin{itemize}
        \item Mutation by random replacement
        \item Subtree swap mutation
    \end{itemize}
\end{itemize}

\section{Bit flip mutation}
Given the binary chromosome with length 4, calculate the probability that no bits,
one bit and more than one bit will be flipped in a bit-flip mutation with \(p_m = \frac{1}{4}\).\\

\textit{Answer:}

Probability of no mutation:
\begin{equation*}
    P(0) = \frac{3}{4} \frac{3}{4} \frac{3}{4} \frac{3}{4} = \frac{3^4}{4^4} \approx 32\%
\end{equation*}
\href{https://en.wikipedia.org/wiki/Binomial_distribution}{Binomial probability}:
\begin{equation}
    P = {n \choose k} p^k (1-p)^{n-k}
\end{equation}
\href{https://en.wikipedia.org/wiki/Binomial_coefficient}{Binomial coefficient}:
\begin{equation}
    {n \choose k} = \frac{n!}{k!(n-k)!}
\end{equation}
Where \(n\) is number of events,
\(k\) is number of occurences wanted and \(p\) is the probability of a single event.\\

In this case we want exactly 1 mutation in 4 events, and the probability is 0.25:
\begin{align*}
    n &= 4\\
    k &= 1\\
    p &= p_m = 0.25\\
    P(1) &= {4 \choose 1} 0.25^1 (1-0.25)^{4-1}\\
    P(1) &= \frac{4!}{1!(4-1)!} 0.25^1 (1-0.25)^{4-1}\\
    P(1) &= \frac{4*3*2}{3*2} 0.25^1 (0.75)^{3}\\
    P(1) &= 4 * 0.25^1 (0.75)^{3}\\
    P(1) &= 0.75^3 \approx 42\%
\end{align*}
This makes intuitive sense and we could also arrive at this result without using the general formula.
For example, the probability of mutation (yes,no,no,no) is:
\begin{align*}
    P(1) &= \frac{1}{4} \frac{3}{4} \frac{3}{4} \frac{3}{4}\\
    P(1) &= 0.25 * 0.75^3
\end{align*}
And there are 4 variants, 4 places where the mutation can happen, so:
\begin{align*}
    P(1) &= 4 * (0.25 * 0.75^3)\\
    P(1) &= 0.75^3 \approx 42\%\\
\end{align*}
Finally, the probability of more than one mutation:
\begin{align*}
    P(2+) &= 1 - P(0) - P(1)\\
    P(2+) &\approx 1 - 32\% - 42\%\\
    P(2+) &\approx 26\%
\end{align*}

\section{Crossover \marginsymbol}
Given the sequences (2,4,7,1,3,6,8,9,5) and (5,9,8,6,2,4,1,3,7).
Implement these algorithms to create a new pair of solutions:
\renewcommand{\theenumi}{\alph{enumi}}
\begin{enumerate}
  \item Partially mapped crossover (PMX).
  \item Order crossover.
  \item Cycle crossover.
\end{enumerate}

\textit{Answer:}

\subsection{Partially mapped crossover}
\subsubsection{Output}
\begin{verbatim}
Parents:
[2, 4, 7, 1, 3, 6, 8, 9, 5]
[5, 9, 8, 6, 2, 4, 1, 3, 7]
Children:
[5, 9, 7, 1, 3, 6, 4, 2, 8]
[3, 1, 8, 6, 2, 4, 7, 9, 5]
\end{verbatim}
\subsubsection{Source code}
\inputminted{Python}{pmx.py}
\subsection{Order Crossover}
\subsubsection{Output}
\begin{verbatim}
Parents:
[2, 4, 7, 1, 3, 6, 8, 9, 5]
[5, 9, 8, 6, 2, 4, 1, 3, 7]
Children:
[9, 2, 4, 1, 3, 6, 8, 7, 5]
[7, 3, 8, 6, 2, 4, 1, 9, 5]
\end{verbatim}
\subsubsection{Source code}
\inputminted{Python}{order.py}
\subsection{Cycle Crossover}
\subsubsection{Output}
\begin{verbatim}
Parents:
[2, 4, 7, 1, 3, 6, 8, 9, 5]
[5, 9, 8, 6, 2, 4, 1, 3, 7]
Children:
[2, 4, 7, 1, 3, 6, 8, 9, 5]
[5, 9, 8, 6, 2, 4, 1, 3, 7]
\end{verbatim}
(These 2 parents have just one cycle, so the child will be the same as parent 1).
\subsubsection{Source code}
\inputminted{Python}{cycle.py}

\input{../contact.tex}
\end{document}
% ==============================================================================


\chapter{Combinatorial}

\section{The Twelvefold Way}
	\import{twelvefoldway.tex}

\section{Permutations}
	\subsection{Factorial}
		\import{factorial.tex}
		\kactlimport{intperm.h}

	\subsection{Cycles}
		Let the number of $n$-permutations whose cycle lengths all belong to the set $S$ be denoted by $g_S(n)$. Then
		$$\sum_{n=0} ^\infty g_S(n) \frac{x^n}{n!} = \exp\left(\sum_{n\in S} \frac{x^n} {n} \right)$$

	\subsection{Derangements}
		Permutations of a set such that none of the elements appear in their original position.
		\[ D(n) = (n-1)(D(n-1)+D(n-2)) = n D(n-1)+(-1)^n = \left\lfloor\frac{n!}{e}\right\rceil \]

		\kactlimport{derangements.h}

	\subsection{Involutions}
		An involution is a permutation with maximum cycle length 2, and it is its own inverse.
		$$a(n) = a(n-1) + (n-1)a(n-2)$$
		$$a(0) = a(1) = 1$$
		1, 1, 2, 4, 10, 26, 76, 232, 764, 2620, 9496, 35696, 140152

	\subsection{Stirling numbers of the first kind}
		$$s(n,k) = (-1)^{n-k}c(n,k)$$
		$c(n,k)$ is the unsigned Stirling numbers of the first kind, and they count the number of permutations on $n$ items with $k$ cycles.
		$$s(n,k) = s(n-1,k-1) - (n-1) s(n-1,k)$$
		$$s(0,0) = 1, s(n,0) = s(0,n) = 0$$
		$$c(n,k) = c(n-1,k-1) + (n-1) c(n-1,k)$$
		$$c(0,0) = 1, c(n,0)=c(0,n)=0$$

	\subsection{Eulerian numbers}
		Number of permutations $\pi \in S_n$ in which exactly $k$ elements are greater than the previous element. $k$ $j$:s s.t. $\pi(j)>\pi(j+1)$, $k+1$ $j$:s s.t. $\pi(j)\geq j$, $k$ $j$:s s.t. $\pi(j)>j$.
		$$E(n,k) = (n-k)E(n-1,k-1) + (k+1)E(n-1,k)$$
		$$E(n,0) = E(n,n-1) = 1$$
		$$E(n,k) = \sum_{j=0}^k(-1)^j\binom{n+1}{j}(k+1-j)^n$$

	\subsection{Burnside's lemma}
		Given a group $G$ of symmetries and a set $X$, the number of elements of $X$ \emph{up to symmetry} equals
		 \[ {\frac {1}{|G|}}\sum _{{g\in G}}|X^{g}|, \]
		 where $X^{g}$ are the elements fixed by $g$ ($g.x = x$).

		 If $f(n)$ counts "configurations" (of some sort) of length $n$, we can ignore rotational symmetry using $G = \mathbb Z_n$ to get
		 \[ g(n) = \frac 1 n \sum_{k=0}^{n-1}{f(\text{gcd}(n, k))} = \frac 1 n \sum_{k|n}{f(k)\phi(n/k)}. \]

\section{Partitions and subsets}
	\subsection{Partition function}
		Partitions of $n$ with exactly $k$ parts, $p(n,k)$, i.e., writing $n$ as a sum of $k$ positive integers, disregarding the order of the summands.
		$$p(n,k) = p(n-1,k-1)+p(n-k,k)$$
		$$p(0,0)=p(1,n)=p(n,n)=p(n,n-1)=1$$

		For partitions with any number of parts, $p(n)$ obeys
		\[ p(0) = 1,\ p(n) = \sum_{k \in \mathbb Z \setminus \{0\}}{(-1)^{k+1} p(n - k(3k-1) / 2)} \]
		\[ p(n) \sim 0.145 / n \cdot \exp(2.56 \sqrt{n}) \]

		\begin{center}
		\begin{tabular}{c|c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c@{\ }c}
			$n$    & 0 & 1 & 2 & 3 & 4 & 5 & 6  & 7  & 8  & 9  & 20  & 50  & 100 \\ \hline
			$p(n)$ & 1 & 1 & 2 & 3 & 5 & 7 & 11 & 15 & 22 & 30 & 627 & $\mathtt{\sim}$2e5 & $\mathtt{\sim}$2e8 \\
		\end{tabular}
		\end{center}

	\subsection{Binomials}
		\kactlimport{binomial.h}
		\kactlimport{binomialModPrime.h}
		\kactlimport{RollingBinomial.h}
		\kactlimport{multinomial.h}

	\subsection{Stirling numbers of the second kind}
		Partitions of $n$ distinct elements into exactly $k$ groups.
		$$S(n,k) = S(n-1,k-1) + k S(n-1,k)$$
		$$S(n,1) = S(n,n) = 1$$
		$$S(n,k) = \frac{1}{k!}\sum_{j=0}^k (-1)^{k-j}\binom{k}{j}j^n$$

	\subsection{Bell numbers}
		Total number of partitions of $n$ distinct elements.
		$$B(n) = \sum_{k=1}^n \binom{n-1}{k-1}B(n-k) = \sum_{k=1}^n S(n,k)$$
		$$B(0) = B(1) = 1$$
		The first are 1, 1, 2, 5, 15, 52, 203, 877, 4140, 21147, 115975, 678570, 4213597.
		For a prime $p$
		$$B(p^m+n)\equiv mB(n)+B(n+1) \pmod{p}$$

	\subsection{Triangles}
		Given rods of length $1,\ldots,n$,
		$$T(n) = \frac{1}{24} \left\{\begin{array}{ll}n(n-2)(2n-5) & n \text{ even}\\(n-1)(n-3)(2n-1) & n \text{ odd}\end{array}\right.$$
		is the number of distinct triangles (positive are) that can be constructed, i.e., the \# of 3-subsets of $[n]$ s.t. $x\leq y\leq z$ and $z\neq x+y$.

\section{General purpose numbers}
	\subsection{Catalan numbers}
		$$C_n=\frac{1}{n+1}\binom{2n}{n}= \binom{2n}{n}-\binom{2n}{n+1} = \frac{(2n)!}{(n+1)!n!}$$
		$$C_{n+1} = \frac{2(2n+1)}{n+2}C_n$$
		$$C_0=1, C_{n+1}=\sum C_iC_{n-i}$$
		First few are 1, 1, 2, 5, 14, 42, 132, 429, 1430, 4862, 16796, 58786, 208012, 742900.
		\begin{itemize}
			\setlength\itemsep{0pt}
			\item \# of monotonic lattice paths of a $n\times n$-grid which do not pass above the diagonal.
			\item \# of expressions containing $n$ pairs of parenthesis which are correctly matched.
			\item \# of full binary trees with with $n+1$ leaves (0 or 2 children).
			\item \# of non-isomorphic ordered trees with $n+1$ vertices.
			\item \# of ways a convex polygon with $n+2$ sides can be cut into triangles by connecting vertices with straight lines.
			\item \# of permutations of $[n]$ with no three-term increasing subsequence.
		\end{itemize}

	\subsection{Super Catalan numbers}
		The number of monotonic lattice paths of a $n\times n$-grid that do not touch the diagonal.
		$$S(n) = \frac{3(2n-3)S(n-1)-(n-3)S(n-2)}{n}$$
		$$S(1)=S(2)=1$$
		1, 1, 3, 11, 45, 197, 903, 4279, 20793, 103049, 518859

	\subsection{Motzkin numbers}
		Number of ways of drawing any number of nonintersecting chords among $n$ points on a circle. Number of lattice paths from $(0,0)$ to $(n,0)$ never going below the $x$-axis, using only steps NE, E, SE.
		$$M(n) = \frac{3(n-1)M(n-2)+(2n+1)M(n-1)}{n+2}$$
		$$M(0) = M(1) = 1$$
		1, 1, 2, 4, 9, 21, 51, 127, 323, 835, 2188, 5798, 15511, 41835, 113634

	\subsection{Narayana numbers}
		Number of lattice paths from $(0,0)$ to $(2n,0)$ never going below the $x$-axis, using only steps NE and SE, and with $k$ peaks.
		$$N(n,k) = \frac{1}{n}\binom{n}{k}\binom{n}{k-1}$$
		$$N(n,1) = N(n,n) = 1$$
		$$\sum_{k=1}^n N(n,k) = C_n$$
		1, 1, 1, 1, 3, 1, 1, 6, 6, 1, 1, 10, 20, 10, 1, 1, 15, 50

	\subsection{Schröder numbers}
		Number of lattice paths from $(0,0)$ to $(n,n)$ using only steps N,NE,E, never going above the diagonal. Number of lattice paths from $(0,0)$ to $(2n,0)$ using only steps NE, SE and double east EE, never going below the $x$-axis. Twice the Super Catalan number, except for the first term.
		1, 2, 6, 22, 90, 394, 1806, 8558, 41586, 206098


\section{Component segment coordinate transform}
%
\begin{figure}[htb]
  \centering
  % trim left, bottom, right , top
  \includegraphics[trim=1cm 1cm 1cm 0.5cm, clip=true,width = 8cm]{gfx/compSegTheo}
	\caption{Illustration for the calculation of the component segment coordinate transformation}
	\label{fig:cs_surf}
\end{figure}
%
The coordinate transform of component segment coordinates is somewhat more complicated than the segment coordinate transform. To simplify the procedure, the component segment should consist of only one segment for now. The segment is again defined by its corner points $\vec p_1 \dots \vec p_4$. Figure \ref{fig:cs_surf} should help understanding the forthcoming calculations. \par
First we need some definitions:
\begin{itemize}
	\item Leading edge:  $ \vec s_v = \vec p_2 - \vec p_1 $
	\item Trailing edge: $ \vec s_h = \vec p_4 - \vec p_3 $
	\item Projected leading edge:   $\vec n = -\vec s_v$, with $n_x = 0$
\end{itemize}

\subsection{Extending leading and trailing edges}
Plane through $\vec p_4$ with normal vector $\vec n$. Calculate inersection with leading edge. Plane equation is:
\begin{equation}
(\vec p - \vec p_4) \cdot \vec n = 0
\label{eq:plane}
\end{equation}
 Linear equation for the leading edge:

\begin{equation}
\vec p = \vec p_1 + \alpha (\vec p_2 - \vec p_1)
\label{eq:lin_eq1}
\end{equation}

Inserting (\ref{eq:lin_eq1}) into (\ref{eq:plane}) yields:
\begin{equation}
\alpha_v = \frac {(\vec p_4 - \vec p_1) \cdot \vec n }{(\vec p_2 - \vec p_1) \cdot \vec n}
\label{eq:nothing}
\end{equation}
%
If $\alpha_v > 1$, the leading edge has to be extended, else the trailing edge must be extended. In the first case, we calculate the extended leading edge point $\vec p_2^\prime$:
\begin{equation}
\vec p_2^\prime = \vec p_1 + \alpha_v (\vec p_2 - \vec p_1)
\label{eq:}
\end{equation}
%
In the other case ($\alpha_v < 1$), the intersection point with the trailing edge can be calculated in the same fashion.  \par
Now, we apply the same method also to the inner section, getting the extended points $\vec p_1^\prime$ and $\vec p_2^\prime$.

\subsection{Calculating $\eta$ values of the corners}
For the following calculations, we need to know the eta coordinates of the corner points $\vec p_1 \dots \vec p_4$. Lets image a plane with the previously defined normal vector $\vec n$ that goes through one of these points. Without loss of generality, let this point be $\vec p_3$. This plane is then defined by the equation
%
\begin{equation}
(\vec p - \vec p_3) \cdot \vec n = 0
\label{eq:plane_p3}
\end{equation}
%
Now we should find out, at which eta coordinate the plane intersects the leading edge, which is now parametrized as follows:
\begin{equation}
p = \vec p_1^ \prime + \eta ({\vec p_2}^\prime - {\vec p_1}^\prime).
\end{equation}
Combining both equations leads to $\eta_3 = \frac {(\vec p_3 - {\vec p_1}^\prime) \cdot \vec n }{({\vec p_2}^\prime - {\vec p_1}^\prime) \cdot \vec n}$, or in general:
\begin{equation}
\eta_i = \frac {(\vec p_i - {\vec p_1}^\prime) \cdot \vec n }{({\vec p_2}^\prime - {\vec p_1}^\prime)\cdot \vec n}.
\end{equation}

After doing these calculations (which have to be done only once), we can finally proceed to the coordinate transform.

\subsection{Calculating 3D coordinates of the $(\eta, \xi)$ pair}
The easiest way to do the transformation is to move first in $\xi$ direction. As the component segment is straight between the points $(\vec p_1, \vec p_3)$ and $(\vec p_2, \vec p_4)$ by definition, we move first along these lines:
\begin{align}
\vec p_{beg}(\xi) &= (1-\xi)\vec p_1 + \xi \vec p_3 \\
\vec p_{end}(\xi) &= (1-\xi)\vec p_2 + \xi \vec p_4
\end{align}

The corresponding eta coordinates of the points are
\begin{align}
\eta_{beg}(\xi) &= (1-\xi)\eta_1 + \xi \eta_3 \\
\eta_{end}(\xi) &= (1-\xi)\eta_2 + \xi \eta_4
\end{align}

Finally, we walk along the line defined by the new point $\vec p_{beg}$ and $\vec p_{end}$ (depicted green in figure \ref{fig:cs_surf}). We have to keep in mind their eta coordinates (which are not 0 and 1)! We get our final result

\begin{equation}
\vec p(\eta, \xi) = \vec p_{beg} (\xi) + \frac {\eta - \eta_{beg}(\xi)}{\eta_{end}(\xi) - \eta_{beg}(\xi)} \cdot \left( \vec p_{end}(\xi) - \vec p_{beg}(\xi) \right).
\end{equation}


\section{Class Structure}
\begin{definition}
    For $i,j\in I$, we say $i$ leads to $j$ (or sometimes $i\rightarrow j$) if $\mathbb P_i[\exists n,X_n=j]>0$ for some $n$.\\
    We say $i$ communicates with $j$ (or sometimes $i\leftrightarrow j$) if $i\rightarrow j$ and $j\rightarrow i$.
\end{definition}
This definition, as were most definitions in maths, is motivated by a theorem.
\begin{theorem}
    For $i\neq j$ the followings are equivalent:\\
    (a) $i\rightarrow j$.\\
    (b) $p_{i_1i_2}\cdots p_{i_{n-1}i_n}>0$ for some $i_1,\ldots,i_n$ with $i_1=i$ and $i_n=j$.\\
    (c) $p_{ij}^{(n)}>0$ for some $n$.
\end{theorem}
\begin{proof}
    Quite obvious.
\end{proof}
\begin{proposition}
    The relation $\leftrightarrow$ is an equivalence relation.
\end{proposition}
\begin{proof}
    Reflexivity and symmetry are straight from definition.
    Transitivity follows from the preceding theorem.
\end{proof}
\begin{definition}
    The equivalence classes of $\leftrightarrow$ is called the communicating classes of the Markov chain.\\
    A Markov chain is irreducible if there is only one communicating class in it.
\end{definition}
\begin{definition}
    A subset $C\subset I$ of the state space is closed if $i\in C$ and $i\rightarrow j$ implies $j\in C$.\\
    A state $i\in I$ is absorbing if $\{i\}$ is closed.
\end{definition}
\begin{example}
    Take the Markov chain with transition matrix
    $$\begin{pmatrix}
        1/2&1/2&&&\\
        &&1&&&\\
        1/3&&&1/3&1/3&\\
        &&&1/2&1/2&\\
        &&&&&1\\
        &&&&1&
    \end{pmatrix}$$
    By observation, the communicating classes are $\{1,2,3\},\{4\},\{5,6\}$ and only $\{5,6\}$ is closed.
\end{example}


\chapter{Refining Heuristics}
\label{chapter:refining}
In this chapter are discussed the heuristic that starting from an initial solution of the TSP, return a better solution in term of cost. This kind of heuristic will be called Refining Heuristics because of the solution cost improvement.

\section{2-opt} \label{sec:best_2_opt}

This heuristic is a simple and effective local search algorithm for TSP problem.\\
A pseudo-code is proposed in alg. \ref{alg:2opt} to describe how it works. The input parameter $ T $ is an ordered list of edges that define a tour and each edge is represented with an ordered pair of nodes. The \texttt{swap\_edges} method remove the two input edges from the tour $ T $ and add the new edges as show in fig. \ref{fig:2_opt_graph}.

\begin{algorithm}
	\caption{}\label{alg:2opt}
	\begin{algorithmic}[1]
	\Procedure{Procedure 2\_opt}{T}
		\State{\textit{// Variable initialization}}
		\State{$ (n_1, n_2), (m_1, m_2) = T[1], T[3] $}
		\State{$ (n_1^*, n_2^*), (m_1^*, m_2^*) = (n_1, n_2), (m_1, m_2) $}
		\State{$ \delta_{cost}^* = c_{n_1m_1} + c_{n_2m_2} - c_{n_1n_2} - c_{m_1m_2}  $}
		\For{$ (n_1,n_2) \in T $}
			\For{$ (m_1,m_2) \in T $ } \textit{ // For each pair of edge in the tour}
				\If{$ n_1 == m_1 \lor n_1 == m_2 \lor n_2 == m_1 \lor n_2 == m_2 $}
					\State{\textbf{continue}}
				\EndIf
				\State{$ \delta_{cost} = c_{n_1m_1} + c_{n_2m_2} - c_{n_1n_2} - c_{m_1m_2} $}
				\If{$ \delta_{cost} < \delta_{cost}^* $} \textit{ // Find 2-opt candidates}
					\State{$ \delta_{cost}^* = \delta_{cost} $}
					\State{$ (n_1^*, n_2^*), (m_1^*, m_2^*) = (n_1, n_2), (m_1, m_2) $}
				\EndIf
			\EndFor
		\EndFor
		\State{\texttt{swap\_edges($ T, (n_1^*, n_2^*), (m_1^*, m_2^*) $)}}
	\EndProcedure
	\end{algorithmic}
\end{algorithm}

\begin{figure}[h]
	\centering
	\includegraphics[width=.3\columnwidth]{img/2_opt_graph.png}
	\caption{Example graph to explain \texttt{2\_opt} method. The input graph is that with continuous edges. \texttt{2\_opt} would remove $ (1,4) $ and $ (2,5) $ from the tour and replace them with $ (1,5) $ and $  (2,4) $.}
	\label{fig:2_opt_graph}
\end{figure}
As can be seen in fig. \ref{fig:lb_time_grasp_best_two_opt_d2103} multiple execution of \texttt{2\_opt} can improve more than 50\% the cost of the solution before to find a local minimum. This multiple execution of \texttt{2\_opt} in the report is called \texttt{best\_two\_opt}.
\begin{figure}[!h]
\centering
\includegraphics[width=.6\columnwidth]{../res/lb_time_grasp_best_two_opt_d2103.png}
\caption{Solution cost profile of \texttt{best\_two\_opt}. \texttt{GRASP} solution is used as warm start.}
\label{fig:lb_time_grasp_best_two_opt_d2103}
\end{figure}

It is interesting to see the solution cost profile of \texttt{best\_two\_opt} (fig. \ref{fig:a280_25}) applied to the best of the Greedy solutions (called \texttt{greedy\_best\_two\_opt}). The improve of the cost solution is considerable even if it stop on a local minimum.  Note that, the best tour is calculated in about $ 8 $s with \texttt{subtour\_callback\_general}, \texttt{n\_greedy} take about $ 0.0045 $s to find the best tour and \texttt{best\_two\_opt} find a local minimum in $0.07$s. Moreover is important to consider that greedy method can be easily modify to execute each greedy tour in parallel and than apply \texttt{best\_two\_opt}.

In fig.s \ref{fig:a280_10} and \ref{fig:a280_25} can be compared the \texttt{Greedy}, \texttt{greedy\_best\_two\_opt} and \texttt{subtour\_callback\_gen \\ eral} tours, solutions cost and execution time.

\begin{figure}[!h]
	\begin{subfigure}{.5\columnwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/a280_28.png}
		\caption{\texttt{n\_greedy\_10}: cost=3078, time=0.0045s}
		\label{fig:a280_10}
	\end{subfigure}
	\begin{subfigure}{.5\columnwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/a280_25.png}
		\caption{\texttt{n\_greedy\_best\_two\_opt}: cost=2683, time=0.06s}
		\label{fig:a280_25}
	\end{subfigure}
	\begin{subfigure}{.5\columnwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/lb_greedy_best_two_opt_a280.png}
		\caption{\centering{The profile of the solution cost over time of the \texttt{best\_two\_opt} optimization phase.}}
		\label{fig:lb_greedy_best_two_opt_a280}
	\end{subfigure}
	\begin{subfigure}{.5\columnwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/a280_0.png}
		\caption{\texttt{subtour\_callback\_general}: cost=2579, time=8s}
		\label{fig:a280_0}
	\end{subfigure}
\end{figure}
\begin{figure}[!h]
	\centering
	\begin{subfigure}{.95\textwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/Lconstructives_refining_LA_time_new.png}
		\caption{Solution time domain.}
		\label{fig:Lconstructives_refining_time}
	\end{subfigure}
	\begin{subfigure}{.95\textwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/Lconstructives_refining_LA_lb_new.png}
		\caption{Solution cost domain.}
		\label{fig:Lconstructives_refining_lb}
	\end{subfigure}
\caption{Performance profiles of refining heuristics.}
\label{fig:pp_Lconstructives_refining}
\end{figure}

The performance profile in fig. \ref{fig:pp_Lgreedy_refining} compare greedy algorithms with their \texttt{best\_two\_opt} optimization. It is clear that most of the execution time is required for the constructive heuristic, however the difference in term of cost optimization is done by the \texttt{best\_two\_opt}. In fig. \ref{fig:Lgreedy_refining_LA_lb} \texttt{n\_greedy\_best\_two\_opt} gain the solution of \texttt{greedy\_best\_two\_opt}, proving that it is not necessary to find the best tour created from the greedy, but is enough to compute 10 random tour and than apply \texttt{best\_two\_opt} to find similar solutions.

\begin{figure}
	\centering
	\begin{subfigure}{0.8\textwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/Lgreedy_refining_LA_time.png}
		\caption{Performance profile in solution cost domain.}
		\label{fig:Lgreedy_refining_LA_time}
	\end{subfigure}
	\begin{subfigure}{0.8\textwidth}
		\centering
		\includegraphics[width=\columnwidth]{../res/Lgreedy_refining_LA_lb.png}
		\caption{Performance profile in solution cost domain.}
		\label{fig:Lgreedy_refining_LA_lb}
	\end{subfigure}
	\caption{Comparison of constructive (\texttt{Greedy n\_greedy}) heuristics and their refining versions of greedy methods.}
	\label{fig:pp_Lgreedy_refining}
\end{figure}

The effectiveness of the refining heuristic depend from the warm start solution. In fig. \ref{fig:Lgrasp_insertion_refining_LA_lb} it is show that, even if \texttt{heuristic\_insertion} generate shorter tour than \texttt{n\_grasp}, \texttt{n\_grasp\_best\_two\_opt} is better than \texttt{insertion\_best\_two\_opt}, even over time (fig. \ref{fig:Lgrasp_insertion_refining_LA_time}).

\begin{figure}
\centering
\begin{subfigure}{0.9\textwidth}
	\centering
	\includegraphics[width=\columnwidth]{../res/Lgrasp_insertion_refining_LA_time_new.png}
	\caption{Performance profile in solution cost domain.}
	\label{fig:Lgrasp_insertion_refining_LA_time}
\end{subfigure}
\begin{subfigure}{0.9\textwidth}
	\centering
	\includegraphics[width=\columnwidth]{../res/Lgrasp_insertion_refining_LA_lb_new.png}
	\caption{Performance profile in solution cost domain.}
	\label{fig:Lgrasp_insertion_refining_LA_lb}
\end{subfigure}
\caption{Comparison of \texttt{n\_grasp} and \texttt{heuristic\_insertion} with and without \texttt{best\_two\_opt}.}
\label{fig:pp_Lgrasp_insertion_refining}
\end{figure}

In conclusion, \texttt{best\_two\_opt} is a really useful refining heuristic for its time efficiency and cost effectiveness, however the second one depends from how many cross edges has the warm start solution.


\documentclass[11pt,twoside]{article}

\usepackage{amsmath,amsthm}
\usepackage[headings]{fullpage}
\usepackage[utopia]{mathdesign}
\usepackage{color}
\usepackage{graphicx}

\pagestyle{myheadings}
\markboth{Eye See You}{Eye See You}

\input{../../fncextra}

\begin{document}

\begin{center}
  \bf Eye See You
\end{center}

We have used least-squares fitting to create functions of a single variable, $y=f(t)$, which can be plotted as a curve to represent data. Some curves, however, cannot be represented as a single function. A more flexible representation is a parametric curve:
\begin{equation}
  \label{eq:1}
  x = f(t), \qquad y = g(t).
\end{equation}
Given points in the plane as $(x_i,y_i)$, we can separately fit them as functions of a third parametric variable $t$ and use the curve $(f(t),g(t))$ to pass near the points.

\subsection*{Preparation}

Read section 3.1.

\subsection*{Goals}

You will capture an image of an eye and find points along the top and bottom eyelids, then do two least-squares fits to represent the eyelids as curves. Because both $x$ and $y$ are periodic as you go around the eye once, you will use periodic functions for the least-squares fitting implied in equation~(1):
\begin{align}
  f(t) &= b_1 + b_2 \cos(2\pi t) + b_3 \cos(4\pi t) + b_4 \cos(6\pi t)  + b_5 \sin(2\pi t) + b_6 \sin(4\pi t) + b_7 \sin(6\pi t),\label{eq:2x} \\
  g(t) &= c_1 + c_2 \cos(2\pi t) + c_3 \cos(4\pi t) + c_4 \cos(6\pi t)  + c_5 \sin(2\pi t) + c_6 \sin(4\pi t) + c_7 \sin(6\pi t). \label{eq:2y}
\end{align}

\subsection*{Procedure}

Download the template script and edit it to perform the following steps.

\begin{enumerate}
\item Using a phone, take a picture of an open eye (your own or someone else's). Load the image into MATLAB using \texttt{imread} and display it using \texttt{image}.
\item Enter the command
\begin{verbatim}
[xup,yup] = ginput(10);
\end{verbatim}
This will create a crosshair in the image window. Click at ten roughly
evenly spaced points along the upper eyelid \textbf{from right to left}. Get close to the corners of the eye, but don't put points on the corners. Afterward both \texttt{xup} and \texttt{yup} will be $10\times 1$ vectors representing the selected points.
\item Repeat step 2 using \verb+[xlo,ylo] = ginput(10)+ and clicking
  along the lower eyelid \textbf{from left to right}.
\item Stack \texttt{xup} and \texttt{xlo} into a vector \texttt{x},
  and stack \texttt{yup} and \texttt{ylo} into a vector \texttt{y}. Both of these should be $20\times 1$. On top of your eye image, plot the points $(x_i,y_i)$ using \texttt{'o'} markers. (If the points don't lie close to the eyelids, you have done something wrong.)
\item Now let \texttt{t} be a $20\times 1$ vector where $t_i=(i-1)/20$ for $i=1,\ldots,20$. Referring back to equations~\eqref{eq:2x} and~\eqref{eq:2y}, create a $20\times 7$ matrix \texttt{A} whose columns are the values of the functions $1$, $\cos(2\pi t)$, and so on, through $\sin(6\pi t)$.
\item Apply linear least squares (using backslash) to solve for the coefficients $b_j$ in~(2) using the \texttt{x} data, and to solve for the coefficients $c_j$ in~(3) using the \texttt{y} data.
\item Evaluate the functions in~\eqref{eq:2x} and~\eqref{eq:2y} at 500 equally spaced values of $t$ between 0 and 1. On top of the axes showing the eye image and the selected points, and using the coefficients from the previous step, plot the curve defined by equation~\eqref{eq:1}.
\end{enumerate}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


% Created 2016-04-06 ons 13:25
% Intended LaTeX compiler: pdflatex
\documentclass{scrartcl}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble}
\author{Kjartan Halvorsen}
\date{Due 2016-03-21}
\title{Computerized control - homework 3}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={Computerized control - homework 3},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 24.5.1 (Org mode 8.3.4)},
 pdflang={English}}
\begin{document}

\maketitle

\section*{The system}
\label{sec:orgheadline1}
A feedback system for level control of a tank is shown in the figure below.
\begin{center}
\includegraphics[width=\linewidth]{tank-system-crop}
\caption{Feedback system for level control. Alla variables are variations from a working point. Water is pumped from the water tank as needed by some industrial process downstream.}
\end{center}
The flow of water, \(x(t)\) does not change immediately with a change of input \(u(t)\) to the valve, but reacts as a first order system
\[ \frac{dx}{dt} = -2x + 4u. \]
The flows \(x(t)\) and \(v(t)\) has the dimension \unit{}{\meter\cubed\per\second}.

The PID controller has transfer function
\[ F(s) = K_P + sK_D + \frac{K_I}{s}. \]

\section*{Exercises}
\label{sec:orgheadline7}
\subsection*{Problem 1}
\label{sec:orgheadline2}
Draw a block diagram of the system, and derive the transfer function for the closed-loop system
\[ Y(s) = G_c(s)Y_{ref}(s) - S(s)V(s) = \frac{G_1(s)G_2(s)F(s)}{1 + G_1(s)G_2(s)F(s)} Y_{ref}(s) - \frac{G_1(s)}{1 + G_1(s)G_2(s)F(s)}V(s), \]
where
\begin{align}
G_1(s) &= \frac{2}{s}\\
G_2(s) &= \frac{4}{s+2}.
\end{align}

\subsection*{Problem 2}
\label{sec:orgheadline3}
Show that the transfer function from the outlet \(-v(t)\) to the variation in the water level \(y(t)\) is given by
\[ S(s) = \frac{2s(s+2)}{s^3 + (2+8K_D)s^2  + 8K_Ps + 8K_I}. \]

\subsection*{Problem 3}
\label{sec:orgheadline4}
Consider the three different settings of the PID-controller in the table below. Pair each of the settings with the correct pole-zero plot of \(S(s)\) and the correct response of the water level to a step in \(v(t)\).

\begin{center}
\begin{tabular}{lrrr}
 & \(K_P\) & \(K_I\) & \(K_D\)\\
\hline
I & 2 & 0 & 0\\
\hline
II & 4 & 1 & 0\\
\hline
III & 2 & 2 & 1\\
\hline
\end{tabular}
\end{center}

\includegraphics[width=\linewidth]{tank-system-step-response}

\subsection*{Problem 4}
\label{sec:orgheadline5}
Assume that there is a small delay of \(T_d=\unit{0.1}{\second}\) in the feedback path of the controller. Tune the PID controller using the Ziegler-Nicholls ultimate sensitivity method (Table 8.3 in Å\&W). To do this, you need to implement the feedback system in matlab using a P-controller. Do step-responses, and crank up the gain of the P-controller until you get sustained oscillations. The corresponding gain is the so-called \emph{ultimate gain} \(K_u\). Find the period of the oscillations, \(T_u\), at this gain. From these two values you can determine the PID controller parameters from Table 8.3.

Here is some matlab code to get you started
\begin{verbatim}
% The plant model
Td = 0.1;
s = tf('s');
G1 = 2/s;
G2 = 4/(s+2);
G = G1*G2

% Set D and I gain to zero and try different values of the proportional
% gain of the controller
Ki = 0;
Kd = 0;
Kp = 2
F = Kp + s*Kd + Ki/s;

% The loop gain and closed loop system
Go = G*F;
Gc = feedback(Go,exp(-Td*s)); % Note the delay in the feedback path

% Step response
figure(1)
clf
step(Gc, 10)
\end{verbatim}

\textbf{Include a step response with the PID parameters you found!}

\subsection*{Problem 5}
\label{sec:orgheadline6}
\begin{enumerate}
\item Discretize your PID controller for general sampling period \(h\) using Tustin's formula.
\item Simulate the discretized model in matlab using a sampling period that is equal to the time delay \(T_d\). Here is some code to help:
\begin{verbatim}
Gd = c2d(G, Td, 'zoh'); % Discretize plant model to be able to simulate
Fd = c2d(F, Td, 'tustin')
God = Gd*Fd;
Gcd = feedback(God,tf([1],[1 0], Td)); % Note the delay in the feedback path

% Step response
figure(1)
clf
step(Gc, Gcd, 10)
\end{verbatim}
\textbf{Include the step response in your report.}
\item Discuss in 2-4 sentences the difference between the response using the continuous PID controller and the discretized PID controller.
\end{enumerate}
\section*{Solutions}
\label{sec:orgheadline16}
\subsection*{Problem 1}
\label{sec:orgheadline8}
\begin{tikzpicture}[node distance=2.6cm, block/.style={rectangle, draw, minimum height=15mm, minimum width=20mm}, sumnode/.style={circle, draw, inner sep=1pt}]

  \node[coordinate] (input) {};
  \node[sumnode, right of=input] (sum) {$\sum$};
  \node[block, right of=sum] (control) {PID};
  \node[block, right of=control, node distance=3.1cm] (valve) {$G_2$};
  \node[sumnode, right of=valve] (sumdist) {$\sum$};
  \node[block, right of=sumdist] (tank) {$G_1$};
  \node[coordinate, right of=tank] (output) {};
  \draw[->] (tank) -- node[coordinate] (measure) {} node[above, pos=0.9] {$y$} (output);

  \node[coordinate, above of=sumdist, node distance=2cm] (dist) {};

  \draw[->] (input) -- node[above, pos=0.2] {$y_{ref}$} (sum);
  \draw[->] (sum) -- node[above] {$e$} (control);
  \draw[->] (control) -- node[above] {$u$} (valve);
  \draw[->] (valve) -- node[above] {$x$} (sumdist);
  \draw[->] (dist) -- node[left]{$v$} node[right, pos=0.9] {$-$} (sumdist);
  \draw[->] (sumdist)  to (tank);
  \draw[->] (measure) |- ++(-2cm, -2cm) -| node[pos=0.97, right] {$-$} (sum);
\end{tikzpicture}

Transform the signals to the Laplace-domain and write the PID controller as the transfer function \(F(s)\).
\begin{equation*}
 \begin{split}
  Y &= G_1(X-V) = G_1(G_2FE-V) = G_1\big(G_2F(Y_{ref}-Y)-V\big)\\
  (I + G_1G_2F) Y &= G_1G_2FY_{ref} - G_1V\\
   Y &= \underbrace{\frac{G_1G_2F}{1 + G_1G_2F}}_{G_c(s)} Y_{ref} - \underbrace{\frac{G_1}{1 + G_1G_2F}}_{S(s)}V
 \end{split}
\end{equation*}
\subsection*{Problem 2}
\label{sec:orgheadline9}
Substituting \(F(s) = K_P + sK_D + K_I/s\), \(G_1(s)= 2/s\) and \(G_2(s) = 4/(s+2)\) into the expression for \(S(s)\) gives
\begin{equation*}
 \begin{split}
 S(s) &= \frac{2/s}{1 + \frac{2}{s}\frac{4}{s+2}(K_P + sK_D + K_I/s)}\\
      &= \frac{2(s+2)}{s(s+2) + 8(K_P + sK_D + K_I/s)}\\
      &= \frac{2s(s+2)}{s^2(s+2) + 8K_Ds^2 + 8K_Ps + 8K_I}\\
      &= \frac{2s(s+2)}{s^3 + (2 + 8K_D)s^2 + 8K_Ps + 8K_I}
 \end{split}
\end{equation*}

\subsection*{Problem 3}
\label{sec:orgheadline13}
\subsubsection*{Setting I - P control}
\label{sec:orgheadline10}
Setting I is pure proportional control, so \(K_D=K_I=0\). This gives
\[ S(s) = \frac{2s(s+2)}{s^3 + 2s^2 + 8K_Ps} = \frac{2(s+2)}{s^2 + 2s + 16} \]
with one zero at \(s=-2\) and two complex-conjugated poles in \(s = -1 \pm 3.873i\). This must correspond to the zero-pole plot \textbf{B}. The step response will have a stationary error, since there is no zero in the origin. This corresponds to the \textbf{dotted line}.
\subsubsection*{Setting II - PI control}
\label{sec:orgheadline11}
Setting II gives the transfer function
\[ S(s) = \frac{2s(s+2)}{s^3 + 2s^2 + 8K_Ps + 8K_I} = \frac{2s(s+2)}{s^3 + 2s^2 + 32s + 8}\]
with zeros at -2 and in the origin and poles in \(s=-0.2535\) and \(s = -0.8732 \pm 5.549i\). This must correspond to the zero-pole plot \textbf{A}. The PI controller has higher gain compared to the P controller with setting I, and an integrating term, which gives the two fast poles with small damping. We also have a dominating pole close to the origin, which gives a slow response. Hence, the corresponding step-response must be the \textbf{dashed line}.
\subsubsection*{Setting III - PID control}
\label{sec:orgheadline12}
By elimination, we now have that this corresponds to zero-pole plot \textbf{C} and the \textbf{solid line} in the step-response. This can also be seen by calculating the poles of the transfer function
\[ S(s) = \frac{2s(s+2)}{s^3 + 10s^2 + 16s + 16}.\]
The poles are \(s=-8.3055\) and \(s = -0.8472 \pm 1.0994i\). The response has reasonable damping.
\subsection*{Problem 4}
\label{sec:orgheadline14}
Trying a few different values for \(K_P\) leads to \(K_u = 2.58\). A step-response of the closed-loop system with this ultimate gain is shown below
\begin{center}
\includegraphics[width=0.6\linewidth]{ultimate_period_hw3_spring16}
\end{center}

Using table 8.3 in Å\&W, we get the PID controller
\[F(s) = K(1 + \frac{1}{T_is} + T_ds) = 0.6K_u(1 + \frac{1}{0.5T_us} + 0.125T_us) = 1.548\big(1 + \frac{1}{0.725s} +  0.1812 s\big). \]

The step response of the closed-loop system (also with discretized controller) is shown below.
\begin{center}
\includegraphics[width=0.6\linewidth]{tuned_response_hw3_spring16}
\end{center}

\subsection*{Problem 5}
\label{sec:orgheadline15}
\begin{enumerate}
\item Discretizing the controller using Tustin's approximation gives
\begin{equation*}
\begin{split}
 F_d(z) &= F(s)|_{s=\frac{2}{h}\frac{z-1}{z+1}}\\
       &= 1.548 \frac{0.1812s^2 + s +1/0.725}{s}|_{s=\frac{2}{h}\frac{z-1}{z+1}}\\
       &= 1.548 \frac{0.1812 \left( \frac{2}{h}\frac{z-1}{z+1} \right)^2 + \frac{2}{h}\frac{z-1}{z+1} + 1.3793}{\frac{2}{h}\frac{z-1}{z+1}}\\
       &= 1.548 \frac{0.1812 \frac{4(z-1)^2}{h(z+1)} + 2(z-1) + 1.3793 h(z+1)}{2(z-1)}\\
       &= 1.548 \frac{0.7248 (z-1)^2 + 2h(z-1)(z+1) + 1.3793 h^2(z+1)^2}{2h(z-1)(z+1)}\\
       \end{split}
\end{equation*}
\item Discretizing the controller and doing a step-response test gives the figure already shown above under Problem 4.
\item The discretized controller gives a system with less damping. The discretization involves a sample-and-hold, which gives a time-delay of approximately \(h/2\). This explains the smaller damping. The discretization of the controller is an approximation, so we cannot expect better performance with the discrete controller, but as the sampling period dicreases, the performance will be closer to that of the contninuous controller. This is illustrated in the figure below.
\begin{center}
\includegraphics[width=0.6\linewidth]{tuned_response_half_h_hw3_spring16}
\end{center}
\end{enumerate}
\end{document}