\documentclass{article}

\usepackage[margin=.25in, letterpaper]{geometry}

\usepackage{siunitx}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{mathtools}
\usepackage{multicol}
\usepackage{enumitem}
\usepackage{scrextend}
\usepackage{microtype}
\usepackage{mathrsfs}
\usepackage{booktabs}

\theoremstyle{remark}
\newtheorem{example}{Example}[section]
\newtheorem*{solution}{Solution}

\begin{document}

\section{Probability}
\begin{align*}
    P[A|B] & = \frac{P[B|A] P[A]}{P[B]} \\
           & = \frac{P[A \cap B]}{P[B]}
\end{align*}

\begin{align*}
    P[A \cap B]                      & =P[A] P[B|A]  \\
                                     & = P[B] P[A|B] \\
    \text{(A and B are independent)} & = P[A] P[B]
\end{align*}

\[
    P[A \cup B] = P[A] + P[B] - P[A \cap B]
\]

\[
    P[A \cup B \cup C] = P[A] + P[B] + P[C] - P[A \cap B] - P[B \cap C] - P[A \cap C] + P[A \cap B \cap C]
\]

\begin{align*}
    P[A] & = P[A \cap B_1] + P[A \cap B_2] + \cdots + P[A\cap B_n]       \\
         & = P[A|B_1] P[B_1]+ P[A|B_2] P[B_2] + \cdots + P[A|B_n] P[B_n]
\end{align*}

\[
    P[A^C | B] = 1 - P[A|B]
\]

\(k\) samples, \(n\) objects

\begin{table}[!htbp]
    \centering
    \begin{tabular}{ c c }
        \toprule
        Action                         & Equation                                      \\
        \midrule
        With replacement               & \(nk\)                                        \\
        Without Replacement            & \(\frac{n!}{(n-k)!} = nPk = {(n)}_k\)         \\
        Without Replacement, unordered & \(\frac{n!}{(n-k)!k!} = nCk = {n \choose k}\) \\
        \bottomrule
    \end{tabular}
\end{table}

\section{Random Values}

\begin{multicols*}{2}
    \subsection{PMF CDF PDF}

    PMF (Discrete):\@
    \begin{itemize}
        \item \(P_X(x) = P[X = x]\)
        \item \(P_X(x) \geq 0, \forall x\)
        \item \(\sum_{x\in S_X} P_X(x) = 1\)
        \item \(P[B] = \sum_{x\in B} P_X(x)\)
    \end{itemize}

    CDF (Discrete):\@
    \begin{itemize}
        \item \(F_X(a) = P[X \leq a]\)
        \item \(F_X(-\infty) = 0,  F_X(\infty) = 1\)
        \item \(a_i \geq a_j \rightarrow F_X(a_i) \geq F_X(a_j)\)
        \item \(\lim_{\epsilon \rightarrow 0} F_X(a_i) - F_X(a_i - \epsilon) = P_X(a_i)\)
    \end{itemize}

    CDF (Continuous):\@
    \begin{itemize}
        \item \(F_X(x) = P[X \leq x]\)
        \item \(F_X(-\infty) = 0,  F_X(\infty) = 1\)
        \item \(x_1 < x_2 \rightarrow F_X(x_1) \leq F_X(x_2)\)
        \item \(P_[x_1 < X \leq x_2] = F_X(x_2) - F_X(x_1)\)
        \item \(P[X\leq x_1] = P[X < x_1]\)
    \end{itemize}

    PDF (Continuous):\@
    \begin{itemize}
        \item \(f_X(x) = \frac{d}{dx} F_X(x)\)
        \item \(f_X(x) \geq 0\)
        \item \(F_X(x) = \int_{-\infty}^x f_X(u) du\)
        \item \(\int_{-\infty}^{\infty} f_X(x) = 1\)
        \item \(P[x_1 < X \leq x_2] = \int_{x_1}^{x_2} f_X(x)dx\)
    \end{itemize}

    \subsection{Expected Value and Variance}
    \(E[X]\)
    \begin{itemize}
        \item \(E[X] = \sum_{x\in S_X} xP_X(x) = \mu_X\)
        \item \(E[g(X)] = \sum_{x\in S_X} g(x) P_X(x) \)
        \item \(E[aX + b] = a E[X] + b\)
    \end{itemize}
    \(Var[X]\)
    \begin{itemize}
        \item \(Var[X] = \sigma^2_X = E[{(X-\mu_X)}^2] = E[X^2] - {(E[X])}^2\)
        \item \(Var[aX+b] = a^2 Var[X]\)
    \end{itemize}
\end{multicols*}

Discrete RV:\@
\begin{table}[!htbp]
    \centering
    \begin{tabular}{ c c c c c }
        \toprule
        Name                & Use                                  & PMF \(P_X(i)\)                      & E[X]        & Var[X]                      \\
        \midrule
        Bernoulli\((p)\)    & Two Outcomes                         & \((p, x=1),(1-p, x=0)\)             & \(p\)       & \(p(1-p)\)                  \\
        Geometric\((p)\)    & Bernoulli until first success        & \({(1-p)}^{i-1}p\)                  & \(1/p\)     & \((1-p)/p^2\)               \\
        Binomial\((n, p)\)  & Successes in sequence of N Bernoulli & \((nCi) p^i {(1-p)}^{n-i}\)         & \(np\)      & \(np(1-p)\)                 \\
        Pascal\((k,p)\)     & Trials until n success               & \((i - 1 C k-1) p^k {(1-p)}^{i-k}\) & \(k/p\)     & \(k(1-p)/p^2\)              \\
        Poisson\((\alpha)\) & Events in time                       & \( (\alpha ^i e^{-\alpha})/i! \)    & \(\alpha \) & \(\alpha \)                 \\
        Uniform\((l,k)\)    & Always same                          & \(1/(k-l + 1)\)                     & \((k+l)/2\) & \(\frac{(k-l)(k-1+2)}{12}\) \\
        \bottomrule
    \end{tabular}
\end{table}

Continuous RV:\@
\begin{table}[!htbp]
    \centering
    \begin{tabular}{ c c c c c }
        \toprule
        Name                        & PDF \(f_X(x)\)                                                        & CDF  \(F_X(x)\)      & E[X]                  & Var[X]                   \\
        \midrule
        Uniform\((a,b)\)            & \(1/(b-a)\)                                                           & \(\frac{x-a}{b-a}\)  & \((b+a)/2\)           & \(\frac{{(b-a)}^2}{12}\) \\
        Exponential\((\lambda)\)    & \(\lambda e^{-\lambda x}\)                                            & \(1-e^{-\lambda a}\) & \(\frac{1}{\lambda}\) & \(\frac{1}{\lambda^2}\)  \\
        Gaussian\((\mu, \sigma^2)\) & \(\frac{1}{\sqrt{2\pi \sigma^2}}e^{-\frac{{(x-\mu )}^2}{2\sigma^2}}\) & \(\phi(a)\)          & \(\mu \)              & \(\sigma ^2\)            \\
        \bottomrule
    \end{tabular}
\end{table}

Normalization:
\[
    Z = \frac{X-\mu}{\sigma}
\]
\end{document}


\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\begmath 15.2 Cumulative Distribution Function and Percentage Points
\hbox{for Normal Probability Distribution}

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

The procedures described in this chapter compute the Cumulative Distribution
Function (CDF) and the percentage points of the Normal or Gaussian
distribution. The CDF is sometimes called the lower tail. The lower tail,
or CDF, $g(x;\mu ,\sigma )$, and the upper
tail, $h(x;\mu ,\sigma )$ for the Normal probability distribution with mean $%
\mu $ and standard deviation $\sigma $ are defined by%
\begin{gather*}
\hspace{-15pt}g(x;\mu ,\sigma )=\frac 1{\sigma \sqrt{2\pi }}\int_{-\infty }^x\!\!\exp
\!\left( -\frac{(t-\mu )^2}{2\sigma ^2}\right) dt,\rule{50pt}{0pt}\\
\hspace{-15pt}h(x;\mu ,\sigma )=\frac 1{\sigma \sqrt{2\pi }}\int_x^\infty \!\!\exp
\!\left( -\frac{(t-\mu )^2}{2\sigma ^2}\right) dt=g(-x;\mu ,\sigma )
\end{gather*}
The percentage point of a distribution is the value of $x$ that gives the
lower tail a specified value. In this case, the problem is to compute $x$
given $u = g(x;\mu ,\sigma )$, $\mu $ and $\sigma $, that is, compute $x =
g^{-1}(u;\mu ,\sigma ).$

\subsection{Usage}

\subsubsection{Cumulative Distribution Function}
\paragraph{Program Prototype, Single Precision}

\begin{description}
\item[REAL]  \ {\bf U, X, MU, SIGMA, SCDNML}

\item[EXTERNAL]  \ {\bf SCDNML}
\end{description}

Assign values to X, MU and SIGMA and obtain U $= g(x;\mu ,\sigma )$ by using
$$
\fbox{{\bf U = SCDNML(X, MU, SIGMA)}}
$$

\paragraph{Argument Definitions}

\begin{description}
\item[X]  \ [in] Argument $x$ of the function $g(x;\mu ,\sigma ).$

\item[MU]  \ [in] Parameter $\mu $ of the function $g(x;\mu ,\sigma ).$

\item[SIGMA]  \ [in] Parameter $\sigma $ of the function $g(x;\mu ,\sigma ).$
\end{description}

\subsubsection{Percentage Points}
\paragraph{Program Prototype, Single Precision}

\begin{description}
\item[REAL]  \ {\bf U, X, MU, SIGMA, SPPNML}

\item[EXTERNAL]  \ {\bf SPPNML}
\end{description}

Assign values to U, MU and SIGMA and obtain X $= g^{-1}(u;\mu ,\sigma )$ by
using
$$
\fbox{{\bf X = SPPNML(U, MU, SIGMA)}}
$$

\paragraph{Argument Definitions}

\begin{description}
\item[U]  \ [in] Argument $u$ of the function $g^{-1}(u;\mu ,\sigma ).$
Require $0.0 \leq \text{U} \leq 1.0.$

\item[MU]  \ [in] Parameter $\mu $ of the function $g^{-1}(u;\mu ,\sigma ).$

\item[SIGMA]  \ [in] Parameter $\sigma $ of the function $g^{-1}(u;\mu
,\sigma ).$
\end{description}

\subsubsection{Modifications for Double Precision}

For double precision computation, change the REAL type statement to
DOUBLE PRECISION and change the initial letter of the function names to D.
Since these functions are not generic intrinsic functions, it is important
to declare them explicitly to be DOUBLE PRECISION, because the default
implicit type would be REAL.

\subsection{Example and Remarks}

See DRDCDNML and ODDCDNML for an example of the usage of these subprograms.

\subsection{Functional Description}

\subsubsection{Method}

To avoid cancellation error when $x-\mu << 0$, the identity $g(x;\mu
,\sigma) = \frac 12 \erfc ((\mu - x)/\sigma \sqrt 2)$ is used.  This
expression never causes more cancellation error than mathematically
equivalent alternatives, so it is used for all allowed values of $x$,
$\mu $, and $\sigma $ (see Section E for restrictions). The procedure
SERFC described in Chapter~2.2 is used to
evaluate $\erfc ((\mu - x)/\sigma \sqrt 2).$

To compute the percentage points, invert the last expression to compute $x =
g^{-1}(u;\mu ,\sigma ) = \mu - \sigma \sqrt 2$\ erfc$^{-1}(2u)$. The
procedure SERFCI described in Chapter~2.13 is used to evaluate erfc$%
^{-1}(2u).$

\subsubsection{Accuracy Tests}

See Sections 2.2.D and~2.13.D.

\subsection{Error Procedures and Restrictions}

The procedure SERFC issues a warning message by way of the error message
processor described in Chapter~19.2 if (X $-$ MU)/($\sqrt{2.0}\ \times $
SIGMA) $< -xmax$. The value of $xmax$ depends on the system and the precision.
Let $t=\sqrt {-\log(\sqrt \pi f)}$ where $f$ is the underflow limit
provided by R1MACH(1) or D1MACH(1) of Chapter~19.1.  Then $xmax = t
- ((\log \,t)/t) - 0.01.$ For example, $xmax \approx 9.18$ (26.5) for single
(double) precision IEEE arithmetic.  The procedure SERFCI issues an error
message at level 2 by way of the error message processor described in
Chapter~19.2 if U $< 0.0 $ or U $> 1.0.$
\subsection{Supporting Information}

Designed and programmed by W. V. Snyder, JPL, 1993.

\pagebreak
%\rule[-50pt]{0pt}{10pt}

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
DCDNML & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
AMACH, DCSEVL, DERF, DERM1, DERV1, DINITS, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
DPPNML & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
AMACH, DERFI, DERM1, DERV1, DPPNML, ERFIN, ERMSG\rule[-5pt]{0pt}{8pt}}\\
\end{tabular}

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
SCDNML & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
AMACH, ERFIN, ERMSG, IERM1, IERV1, SCSEVL, SERF, SERM1, SERV1, SINITS\rule[-5pt]{0pt}{8pt}}\\
SPPNML & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
AMACH, ERFIN, ERMSG, SERFI, SERM1, SERV1, SPPNML}\\
\end{tabular}

\begcode

\medskip\
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRDCDNML}\vspace{5pt}
\lstinputlisting{\codeloc{dcdnml}}

\vspace{15pt}\centerline{\bf \large ODDCDNML}\vspace{5pt}
\lstset{language={}}
\lstinputlisting{\outputloc{dcdnml}}
\end{document}


\section{表达式处理}

\begin{minted}{c++}
bool shouldPopFromStack(char current, char fromStack) {
    int p1 = (current == '*' || current == '/') ? 1 : 0;
    int p2 = (fromStack == '*' || fromStack == '/') ? 1 : 0;
    return p1 <= p2;
}
string convert(string raw) {
    int length = raw.length();
    stack<char> s;
    string res = "";
    for (int i = 0; i < length; i++) {
        if (raw[i] >= '0' && raw[i] <= '9') {
            res += raw[i];
            continue;
        }
        else
            res += ' ';
        if (raw[i] == '(')
            s.push('(');
        else if (raw[i] == ')')  {
            while (true) {
                char now = s.top();
                s.pop();
                if (now == '(')
                    break;
                else
                    res += now;
            }
        }
        else {
            if (s.empty())
                s.push(raw[i]);
            else {
                while (!s.empty()) {
                    char now = s.top();
                    if (now == '(' || !shouldPopFromStack(raw[i], now))
                        break;
                    else {
                        s.pop();
                        res += now;
                    }
                }
                s.push(raw[i]);
            }
        }
    }
    res += ' ';
    while (!s.empty()) {
        char now = s.top();
        s.pop();
        res += now;
    }
    return res;
}
int main() {
    /* ... */
    stack<long long> s;
    int length = res.length();
    long long cur = 0;
    for (int i = 0; i < length; i++) {
        if (res[i] >= '0' && res[i] <= '9') {
            cur = cur * 10 + res[i] - '0';
        } else if (res[i] == ' ') {
            s.push(cur);
            cur = 0;
        }
        else {
            long long s2 = s.top();
            s.pop();
            long long s1 = s.top();
            s.pop();
            switch (res[i]) {
                case '+':
                    s.push(s1 + s2);
                    break;
                case '-':
                    s.push(s1 - s2);
                    break;
                case '*':
                    s.push(s1 * s2);
                    break;
                case '/':
                    s.push(s1 / s2);
                    break;
            }
        }
    }
    /* ... */
}
\end{minted}


\chapter{Local polynomial regression}
\section{Different bandwidths and kernels}
This exercise is about a method for non-parametric regression called local polynomial regression (LPR). It is a generalisation of the kernel based Nadaraya-Watson estimator, similar to the kernel density estimation in Exercise 6. The Nadaraya-Watson estimator is a local weighted mean estimation. The local polynomial regression estimation for $(X_1,Y_1),...,(X_n,Y_n)\stackrel{iid}{\sim}(X,Y)$ and the model $$Y_i=f(X_i)+\varepsilon_i, \; \varepsilon_i \stackrel{iid}{\sim}\left\langle0,\sigma^2\right\rangle$$ is derived by a Taylor approximation of the function $f$.
\begin{figure}[!htb]
\centering
\includegraphics[width=0.8\textwidth, keepaspectratio]{ex9/polyfits.png}
\caption{First: Age against Z-score in the Kenyan children data. Second row: LPR fit with different bandwidths and Epanechnikov kernel. Third: LPR fit with different kernels and fixed bandwidth $h=10$.}
\label{7fits}
\end{figure}

We apply this method to the data of Kenyan children already used in Exercise 1. We want to fit a curve for the Z-score for wasting (\texttt{zwast}) against age. This Z-score is a transformation of a child's weight standardised by the median and standard deviation of the weight of healthy children with the same height. We wrote our own function for fitting a local polynomial curve with data, bandwidth, kernel and polynomial order as arguments. To examine the influence of argument choice we again fitted a curve for different bandwidths and the same kernels as in Exercise 6 with a fixed order 1 polynomial regression (see Figure \ref{7fits}).

We see similar results as in Exercise 6. With increasing bandwidth the curve gets smoother. With a bandwidth of 1 the fit represents the data rather locally. With $h=2$ some of the fluctuations are flattened out and with increasing bandwidth the curve converges to the constant mean of the data (see blue line). I took 40 as the highest bandwidth here to demonstrate this fact. A reasonable choice seems to be $h=10$ since there the functions looks pretty smooth but still represents the first drop of the Z-score and the slight increase and convergence afterwards that we see in the scatter plot in Figure \ref{7fits}. Then using this bandwidth we were asked to fit the curve for different kernels. Like in Exercise 6 again we see that the Gaussian kernel produces the most flat curve. Indeed it looks quite similar to the fit for the Epanechnikov kernel with an astonishing high bandwidth compared to the one used here. The fits for the triangular and Epanechnikov kernel don't differ obviously. The red line representing the fit for the rectangular kernel seems piecewise linear which is quite natural as the kernel is piecewise constant and we used a LPR of order 1 (i.e. using linear polynomials) in all these plots.

\section{Generalised cross-validation}
Similar to the cross-validation method in Exercise 6 there is a generalised version for LPR we want to use here to find the best bandwidth for different polynomial degrees. The generalised cross-validation criterion (GCV) is given by $$GCV(h)=\frac{\sum_{i=1}^{n}{\left[Y_i-\hat{f}_{n,h}(X_i)\right]^2}}{1-n^{-1}\sum_{i=1}^{n} W_{i}(X_i,h)}$$ where $W_i$ are the weights of the different observations for the linear generalised least square estimation used to compute the parameters. This time we used only the Epanechnikov kernel to search for the minimal GCV. Again we have the same problem - like in Exercise 6 - that the GCV curve is not unimodal and thus optimize won't find the global minimum (see Figure \ref{7opth}).
\begin{figure}[!bht]
\centering
\includegraphics[width=0.8\textwidth, keepaspectratio]{ex9/opth.png}
\caption{GCV against $h$. Points show optimal $h$ for the scan over 30 points (blue) and the \texttt{optimize} function (red).}
\label{7opth}
\end{figure}

We therefore use a scanning strategy only to get the optimal bandwidths. The results are shown in Table \ref{7tableh}. We see that the results for degree 2 and 3 are identical (rounded to two digits) and similar to the results for degree 4 whereas for degree 1 a lower bandwidths seems the best option. The degree 4 fit gives the highest GCV indicating a worse fit.
\begin{table}[!ht]
\centering
\begin{tabular}{lrr}
  \hline
deg & h & GCV \\
  \hline
1 & 1.44 & 6517.30 \\
  2 & 3.19 & 6518.00 \\
  3 & 3.19 & 6518.00 \\
  4 & 3.06 & 6522.78 \\
   \hline
\end{tabular}
\caption{Optimal bandwidth $h$ together with minimal GCV for different polynmial degrees (deg).}
\label{7tableh}
\end{table}

Now have a look at the fitted curves (see Figure \ref{7fitsh}). The curves don't differ but look very similar. This is especially surprising since the optimal bandwidth for degree 1 is much lower than for the other curves. But with higher polynomial degree the regression as more degrees of freedom (i.e. more parameters) to adapt to the data. This would explain why all three curves look like a very local fit. On the left border all three curves decrease as age tends to 0. This does not seem reasonable since the shape of the data suggests that the Z-score is very high at birth and decreases rapidly in the first months. This is one disadvantage of local non-parametric regressions, that in general at the border and outside of the observed data range the prediction is not valid and thus generalisations difficult. Finally note, that the curves in Figure \ref{7fitsh} look piecewise linear. Indeed in this plot they are drawn since we approximated the regression curves at the data points. One could have generalised the function fitting the curve with an argument $x$ to draw a better approximation with more points but note that one has to fit a linear model for every point then and this would lead to higher computational effort. Still one would recognizes difference in shape in the way the data is presented here. Summing up and looking back to the second plot in Figure \ref{7fits} using the GCV may not be the best method to find a suitable bandwidth in every case but can result in a - in my opinion - too low bandwidth that results in a more local representation than a reasonable and useful trend analysis of the data.

\begin{figure}[!t]
\centering
\includegraphics[width=0.8\textwidth, keepaspectratio]{ex9/fitsh.png}
\caption{LPR fit for different polynomial degrees each with optimal bandwidth according to Table \ref{7tableh} with Epanechnikov kernel.}
\label{7fitsh}
\end{figure}

\section{Estimating derivatives}
When considering the change of a variable over time it may be useful to have a look at the derivative of the curve. The method of LPR provides also an on-the-fly method to estimate the derivatives since it is based on a Taylor expansion of the function. For a regression of the derivatives of the curve we use the R function \texttt{localpoly.reg} in the package \texttt{NonpModelCheck}.
\begin{figure}[p]
\centering
\includegraphics[width=\textwidth, height=0.87\textheight, keepaspectratio]{ex9/deriv.png}
\caption{LPR fits with function \texttt{localpoly.reg} for different polynomial degrees. First plot: first derivative with optimal bandwidths according to \ref{7tableh}; Second plot: regression curve with fixed bandwidth $h=10$; Third plot: first derivative with fixed bandwidth $h=10$.}
\label{7deriv}
\end{figure}

Indeed this function provides all functionality we need for this exercise. The result in the first row Figure \ref{7deriv} show the derivatives with the optimal bandwidths with respect to GCV we found in the last section. We recognize that the strong fluctuation of the regression function itself, shown in the previous section, is also visible for the derivative with is also highly fluctuating and thus in my opinion not really helpful for a reasonable examination of the change in the Z-score. Especially the values at the borders are very high and don't fit at all to the apparently converging behaviour of the data we see in Figure \ref{7fits}. Here we would maybe guess that the Z-score is improving since the derivative is positive but because of the high fluctuation we would expect it to decrease shortly afterwards. I decided to use a higher bandwidth again and fit the curve again (see second and third plot in Figure \ref{7deriv}). I used $h=10$ because it seemed already a reasonable bandwidth in the first section. I included the regression for the actual function and their derivative. Obviously the fluctuation is lower as expected. The fits for degree 3 and 4 still show a slight overfit, especially at the left border. The green and the red line are hard do distinct, so the fit for degree 1 and 2 are very similar. Both start with a negative derivative that tends to 0 and keeps a slightly positive value then. All in all I would conclude that there is a very small improvement of the Z-score at the age of 2 since the red and green curve are slightly above 0 on the right corner of Figure \ref{7deriv} in the last plot.


% !TEX root = ../master/master.tex

%% Dexter Barrows, 2016
%% dbarrows.github.io

	Markov Chain Monte Carlo (MCMC) is a general class of methods designed to sample from the posterior distribution of model parameters \cite{Andrieu2003}. It is an algorithm used when we wish to fit a model $M$ that depends on some parameter (or more typically vector of parameters) $\theta$ to observed data $D$. MCMC works by constructing a Markov chain whose stationary distribution converges to desired posterior distribution. The samples drawn using MCMC are used to numerically approximate the stationary distribution, and in turn the posterior \cite{Andrieu2003}.

\section{Markov Chains}

    Figure [\ref{fsm}] shows a finite state machine with 3 states $S = \{x_1, x_2, x_3\}$.

    \begin{figure}
        \centering
        \captionsetup{width=0.8\linewidth}
        \includegraphics[width=0.5\textwidth]{./images/finitemachine.pdf}
        \caption{A finite state machine. States are shown as graph nodes, and the probability of transitioning from one particular state to another is shown as a weighted graph edge. \cite{Andrieu2003} \label{fsm}}
    \end{figure}

    The transition probabilities can be summarized as a matrix as

    \begin{equation}
    	T =
	    \begin{bmatrix}
	        0 & 1 & 0 \\
	        0 & 0.1 & 0.9 \\
	        0.6 & 0.4 & 0
	    \end{bmatrix}.
    \end{equation}

    The probability vector $\mu(x^{(1)})$ for a state $x^{(1)}$ can be evolved using $T$ by evaluating $\mu(x^{(1)})T$, then again by evaluating $\mu(x^{(1)})T^2$, and so on. If we take the limit as the number of transitions approaches infinity, we find

    \begin{equation}
    	\lim_{t \to \infty} \mu(x^{(1)})T^t = (27/122, 50/122, 45/122).
    \end{equation}

    This indicates that no matter what we pick for the initial probability distribution $\mu(x^{(1)})$, the chain will always stabilize at the equilibrium distribution.

    This property holds when the chain satisfies the following conditions

    \begin{itemize}
        \item \textit{Irreducible} Any state A can be reached from any other state B with non-zero probability
        \item \textit{Positive Recurrent} The number of steps required for the chain to reach state A from state B must be finite
        \item \textit{Aperiodic} The chain must be able to explore the parameter space without becoming trapped in a cycle
    \end{itemize}

    Note that MCMC sampling generates a Markov chain $(\theta^{(1)}, \theta^{(2)},..., \theta^{(N)})$ that does indeed satisfy these conditions, and uses the chain's equilibrium distribution to approximate the posterior distribution of the parameter space \cite{Andrieu2003}.

\section{Likelihood}

    MCMC and similar methods hinge on the idea that the weight or support bestowed upon a particular set of parameters $\theta$ should be proportional to the probability of observing the data $D$ given the model output using that set of parameters $M(\theta)$. In order to do this we need a way to evaluate whether or not $M(\theta)$ is a good fit for $D$; this is done by specifying a likelihood function $\mathcal{L}(\theta)$ such that

    \begin{equation}
    	\mathcal{L}(\theta) \propto P(D|\theta).
    \end{equation}

    In frequentist Maximum Likelihood approaches, $\mathcal{L}(\theta)$ is searched to find a value of $\theta$ that maximizes $\mathcal{L}(\theta)$, then this $\theta$ is taken to be the most likely true value. Bayesian approaches take this further by aiming to generate a posterior distribution of likelihood values conditioned on prior information about the parameters and the data -- to not just maximize the likelihood but to also explore the area around it \cite{Andrieu2003}.

\section{Prior distribution}

    Another significant component of MCMC is the user-specified prior distribution for $\theta$ or distributions for the individual components of $\theta$ (priors). Priors serve as a way for us to tell the MCMC algorithm what we think consist of good values for the parameters. Note that if very little is known about the parameters, or we are worried about biasing our estimate of the posterior, we can simply use a a wide uniform distribution. We cannot, however, avoid this problem entirely. Bayesian frameworks, such as MCMC, \textit{require} priors to be specified; what the user must decide is how strong to make priors.

   	Exceedingly weak priors can prove problematic in some circumstances. In the case of MCMC, weak priors handicap the algorithm in two ways: convergence of the chain may become exceedingly slow, and more pressure is put on the likelihood function to be as good as possible -- it will now be the only thing informing the algorithm of what constitutes a ``good'' set of parameters, and what should be considered poor. In the majority of cases this does not pose as much as a problem as it would appear; if enough samples are drawn, we should still obtain a good posterior estimate. We will only really run into problems if an exceedingly weak prior, such as an unbounded uniform distribution, or another unbounded distribution with a high standard deviation, is specified -- in those cases we may obtain poor posterior estimates if the data are weak \cite{Andrieu2003}.

\section{Proposal distribution}

    As part of the MCMC algorithm, when we find a state in the parameter space that is accepted as part of the Markov chain construction process, we need a good way of generating a good next step to try. Unlike basic rejection sampling in which we would just randomly sample from our prior distribution, MCMC attempts to optimise our choices by choosing a step that is close enough to the last accepted step so as to stand a decent chance of also being accepted, but far enough away that it doesn't get ``trapped'' in a particular region of the parameter space.

    This is done through the use of a proposal or candidate distribution. This will usually be a distribution centred around our last accepted step and with a dispersion potential narrower than that of our prior distribution.

    The choice of this distribution is theoretically not of the utmost importance, but in practice becomes important so as to not waste computer time \cite{Andrieu2003}.

\section{Algorithm}

    Now that we have all the pieces necessary, we can discuss the details of the MCMC algorithm.

    We will denote the previously discussed quantities as

    \begin{itemize}
        \item $p(\cdot)$ - the prior distribution
        \item $q(\cdot|\cdot)$ - the proposal distribution
        \item $\mathcal{L}(\cdot)$ - the Likelihood function
        \item $\mathcal{U}(\cdot,\cdot)$ - the uniform distribution
    \end{itemize}

    and the define the acceptance ratio, $r$, as

    \begin{equation}
    	r = \frac{\mathcal{L}(\theta^*)p(\theta^*)q(\theta^*|\theta)}{\mathcal{L}(\theta)p(\theta)q(\theta|\theta^*)},
    \end{equation}

    where $\theta^*$ is the proposed sample to draw from the posterior, and $\theta$ is the last accepted sample. This is known as the Metropolis-Hastings rule.

    In the special case of the Metropolis variation of MCMC, the proposal distribution is symmetric, meaning $q(\theta^*|\theta) = q(\theta|\theta^*)$, and so the acceptance ratio simplifies to

    \begin{equation}
    	r = \frac{\mathcal{L}(\theta^*)p(\theta^*)}{\mathcal{L}(\theta)p(\theta)}.
    \end{equation}

    Algorithm [\ref{mhmcmc}] shows the Metropolis MCMC algorithm.

    \begin{algorithm}

        \BlankLine

        \SetKwInOut{Input}{Input}
        \SetKwInOut{Output}{Output}
        \DontPrintSemicolon

        \tcc{Select a starting point}
        \Input{Initialize $\theta^{(1)}$}

        \BlankLine

        \For{$i = 2:N$}{

            \BlankLine

            \tcc{Sample}
            $\theta^* \sim q(\cdot|\theta^{(i-1)})$ \;
            $u \sim \mathcal{U}(0,1)$

            \BlankLine

            \tcc{Evaluate acceptance ratio}
            $r$ $\gets$ $\frac{\mathcal{L}(\theta^*)p(\theta^*)}{\mathcal{L}(\theta)p(\theta)}$

            \BlankLine

            \tcc{Step acceptance criterion}
            \eIf{ $u < \min\left\{ 1 , r \right\}$ }{
                $\theta^{(i)} = \theta^*$\;
            }{
                $\theta^{(i)} = \theta^{(i-1)}$\;
            }
        }

        \BlankLine

        \tcc{Samples from approximated posterior distribution}
        \Output{Chain of samples $(\theta^{(1)},\theta^{(2)},...,\theta^{(N)})$}

        \BlankLine

        \caption{Metropolis MCMC \label{mhmcmc}}

    \end{algorithm}

    In this way we are ensuring that steps that lead to better likelihood outcomes are likely to be accepted, but steps that do not will not be accepted as frequently. Note that these less ``advantageous'' moves will still occur but that this is by design -- it ensures that as much of the parameter space as possible will be explored but more efficiently than using pure brute force \cite{Andrieu2003}.

\section{Burn-in}

    One critical aspect of MCMC-based algorithms has yet to be discussed. The algorithm requires an initial starting point $\theta$ to be selected, but as the proposal distribution is supposed to restrict moves to an area close to the last accepted state, then the posterior distribution will be biased towards this starting point. This issue is avoided through the use of a Burn-in period.

    Burning in a chain is the act of running the MCMC algorithm normally without saving first $M$ samples. As we are seeking a chain of length $N$, the total computation will be equivalent to generating a chain of length $M+N$ \cite{Andrieu2003}.

\section{Thinning}

    Some models will require very long chains to get a good approximation of the posterior, which will consequently require a non-trivial amount of computer storage. One way to reduce the burden of storing so many samples is by thinning. This involves saving only every $n^{\text{th}}$ step, which should still give a decent approximate of the posterior (since the chain has time to explore a large portion of the parameter space), but requires less room to store \cite{Link2012}.

\section{Hamiltonian Monte Carlo}

    The Metropolis-Hastings algorithm has a primary drawback in that the parameter space may not be explored efficiently in some circumstances -- a consequence of the rudimentary proposal mechanism. Instead, smarter moves can be proposed through the use of Hamiltonian dynamics, leading to a better exploration of the target distribution and a potential decrease in overall computational complexity. This algorithm is coined Hamiltonian MCMC (HMC) \cite{Neal2011}. Prior to the advent of HMC, some work was conducted exploring adaptive step-sizing using MCMC-based methods, but found they lack strong theoretical justification, and can lead to some samples being drawn from an incorrect distribution \cite{Neal2011}. HMC has in fact existed for nearly the same amount of time as MCMC -- both methods having been developed to model molecular dynamics, with MCMC taking a probabilistic approach and HMC taking a more deterministic one -- but had not received much attention outside its native discipline until recently.

    In the HMC formulation, the parameter estimates are treated as a physical particle exploring a sloped likelihood surface. From physics, we will borrow the ideas of potential and kinetic energy. Here potential energy, or gravity, is analogous to the negative log likelihood of the parameter selection given the data, formally

    \begin{equation}
        U(\theta) = -\log(\mathcal{L}(\theta)p(\theta)).
    \end{equation}

    Kinetic energy will serve as a way to ``nudge'' the parameters along a different moment for each component of $\theta$. We introduce $n$ auxiliary variables $r = (r_1, r_1,...,r_n)$, where $n$ is the number of components in $\theta$. Note that the samples drawn for $r$ are not of interest, they are only used to inform the evolution of the Hamiltonian dynamics of the system. We can now define the kinetic energy as

    \begin{equation}
        K(r) = \frac{1}{2} r^T M^{-1} r,
    \end{equation}

    where $M$ is an $n \times n$ matrix. In practice $M$ can simply be chosen as the identity matrix of size $n$, however it can also be used to account for correlation between components of $\theta$.

    The Hamiltonian of the system is defined as

    \begin{equation}
        H(\theta,r) = U(\theta) + K(r),
    \end{equation}

    where the Hamiltonian dynamics of the combined system can be simulated using the following system of ODEs:

    \begin{equation}
        \begin{array}{rl}
        \displaystyle
            \dfrac{d\theta}{dt} & = M^{-1} r \\
            \dfrac{dr}{dt} & = - \nabla U(\theta) .
        \end{array}
    \end{equation}

    It is tempting to try to integrate this system using the standard Euler evolution scheme, but in practice this leads to instability as it will not preserve the volume of the system. Instead the ``Leapfrog'' scheme is used. This scheme is very similar to Euler scheme, except instead of using a fixed step size $h$ for all evolutions, a step size of $\epsilon$ is used for most evolutions, with a half step size of $\epsilon / 2$ for evolutions of $\frac{dr}{dt}$ at the first step, and last step $L$. In this way the evolution steps ``leapfrog'' over each other while using future values from the other set of steps, leading to the scheme's name.

    The end product of the Leapfrog steps are the new proposed parameters $(\theta^*,r^*)$. These are either accepted or rejected using a mechanism similar to that of standard Metropolis-Hastings MCMC. Now, however, the acceptance ratio $r$ is defined as

    \begin{equation}\label{hmcratio}
        r = \exp \left[ H(\theta,r) - H(\theta^*,r^*) \right],
    \end{equation}

    where $(\theta,r)$ are the last values in the chain. This form of the acceptance ratio comes from the definition of the Hamiltonian as an energy function. If we define the distribution of the total potential energy in the system (known as the canonical distribution) as a function of the Hamiltonian as

    \begin{equation}
    	P(\theta,r) = \frac{1}{Z} \exp (-H(\theta,r))
    \end{equation}

    where $Z$ is a normalizing constant, then taking the ratio of the total potential energy of the proposed step $P(\theta^*,r^*)$ to the total potential energy in the last accepted step $P(\theta, r)$, we obtain Equation (\ref{hmcratio}).

    Together, we have Algorithm [\ref{hmcmc}].

    \begin{algorithm}

        \BlankLine

        \SetKwInOut{Input}{Input}
        \SetKwInOut{Output}{Output}
        \DontPrintSemicolon

        \tcc{Select a starting point}
        \Input{Initialize $\theta^{(1)}$}

        \BlankLine

        \For{$i = 2:N$}{

            \BlankLine

            \tcc{Resample moments}
            \For{$i = 1:n$}{
                r(i) $\gets$ $\mathcal{N}(0,1)$
            }

            \BlankLine

            \tcc{Leapfrog initialization}
            $\theta_0$ $\gets$ $\theta^{(i-1)}$ \;
            $r_0$ $\gets$ $r - \nabla U(\theta_0) \cdot \epsilon / 2$

            \BlankLine

            \tcc{Leapfrog intermediate steps}
            \For{$j = 1:L-1$}{
                $\theta_j$ $\gets$ $\theta_{j-1} + M^{-1} r_{j-1} \cdot \epsilon$ \;
                $r_j$ $\gets$ $r_{j-1} - \nabla U(\theta_j) \cdot \epsilon$
            }

            \BlankLine

            \tcc{Leapfrog last steps}
            $\theta^*$ $\gets$ $\theta_{L-1} + M^{-1} r_{L-1} \cdot \epsilon$ \;
            $r^*$ $\gets$ $\nabla U(\theta_L) \cdot \epsilon / 2 - r_{L-1}$
            \BlankLine

            \tcc{Evaluate acceptance ratio}
            $r = \exp \left[ H(\theta^{(i-1)},r) - H(\theta^*,r^*) \right]$

            \BlankLine

            \tcc{Sample}
            $u \sim \mathcal{U}(0,1)$

            \BlankLine

            \tcc{Step acceptance criterion}
            \eIf{ $u < \min\left\{ 1 , r \right\}$ }{
                $\theta^{(i)} = \theta^*$\;
            }{
                $\theta^{(i)} = \theta^{(i-1)}$\;
            }
        }

        \BlankLine

        \tcc{Samples from approximated posterior distribution}
        \Output{Chain of samples $(\theta^{(1)},\theta^{(2)},...,\theta^{(N)})$}

        \BlankLine

        \caption{Hamiltonian MCMC \label{hmcmc}}

    \end{algorithm}

    Note that the parameters $\epsilon$ and $L$ have to be tuned in order to maintain stability and maximize efficiency, a sometimes non-trivial process utilising trial fitting with candidate values of $\epsilon$ and $L$ \cite{Neal2011}. However, some recent algorithms, such as the No U-Turn sampler implemented in RStan, and adaptively select appropriate values automatically during the sampling process \cite{Hoffman2014}.

\section{RStan Fitting}

    Here we will examine a test case in which Hamiltonian MCMC will be used to fit a Susceptible-Infected-Removed (SIR) epidemic model to mock infectious count data.

    The synthetic data was produced by taking the solution to a basic SIR ODE model, sampling it at regular intervals, and perturbing those values by adding in observation noise. The SIR model used was outlined in the introduction in Equation [\ref{sirode}].

    The solution to this system was obtained using the \verb|ode()| function from the \verb|deSolve| package. The required derivative array function in the format required by \verb|ode()| was specified as the gradient in Equation [\ref{sirode}].

    The true parameter values were set to $\mathcal{R}_0 = 3.0$, $\gamma = 0.1$ recoveries/week, $N = 500$ individuals. The initial conditions were set to 5 infectious individuals, 495 people susceptible to infection, and no one had yet recovered from infection and been removed. The system was integrated over $[0,100]$ weeks with infected counts drawn at each integer time step.

    The observation error was taken to be $\varepsilon_{obs} \sim \mathcal{N}(0,\sigma)$, where individual values were drawn for each synthetic data point.

    Figure [\ref{mcmcdataplot}] shows the system simulation results.

    \begin{figure}
        \centering
        \includegraphics[width=\textwidth]{./images/dataplot.pdf}
        \caption{True SIR ODE solution infected counts, and with added observation noise. \label{mcmcdataplot}}
    \end{figure}

    The Hamiltonian MCMC model fitting was done using Stan (\url{http://mc-stan.org/}), a program written in \verb|C++| that does Baysian statistical inference using Hamiltonian MCMC. Stan's R interface (\url{http://mc-stan.org/interfaces/rstan.html}) was used to ease implementation.

    Throughout this paper, the explicit Euler integration scheme was used to obtain solutions to our ODE-based models. While this scheme is not the most accurate or efficient one available, it as chosen for its ease of implementation in the required languages and transparency with regards to stochastic processes, which have been added into later models. Using a more advanced integrator such as Runge-Kutta makes it harder to properly specify how stochastic process evolution should be handled, and would have required significantly more implementation work to boot. Hence, we have opted for the lo-fi solution we know will function the way we require.

    In order to use an Explicit Euler-like stepping method, with a step size of one per day, in the later Stan model, the synthetic observation counts were treated as weekly observations in which the counts on the other six days of the week were unobserved.

    Figure [\ref{traceplot}] shows the traceplot for the the post-burn-in chain data returned by the RStan fitting. We see that the chains are mixing well and convergence has likely been reached.

    \begin{figure}
        \centering
        \captionsetup{width=0.8\linewidth}
        \includegraphics[width=\textwidth]{./images/traceplotR0.pdf}
        \caption{Traceplot of samples drawn for parameter $\mathcal{R}_0$, excluding burn-in. \label{traceplot}}
    \end{figure}

	Figure [\ref{traceplot2}] shows the chain data including the burn-in samples. We can see why it is wise to discard these samples (note the scale).

    \begin{figure}
        \centering
        \captionsetup{width=0.8\linewidth}
        \includegraphics[width=\textwidth]{./images/traceplotR0_inc.pdf}
        \caption{Traceplot of samples drawn for parameter $\mathcal{R}_0$, including burn-in. \label{traceplot2}}
    \end{figure}

    Figure [\ref{hmckernels}] shows the the kernel density estimates for each of the model parameters and the initial number of cases. We see that while the estimates are not perfect, they are more than satisfactory.

    \begin{figure}
    	\centering
    	\captionsetup{width=0.8\linewidth}
        \begin{subfigure}[tl]{0.4\textwidth}
            \includegraphics[width=\textwidth]{./images/kernelR0.pdf}
        \end{subfigure}
        \begin{subfigure}[tr]{0.4\textwidth}
            \includegraphics[width=\textwidth]{./images/kernelr.pdf}
        \end{subfigure}
        \begin{subfigure}[bl]{0.4\textwidth}
            \includegraphics[width=\textwidth]{./images/kernelsigma.pdf}
        \end{subfigure}
        \begin{subfigure}[br]{0.4\textwidth}
            \includegraphics[width=\textwidth]{./images/kernelinfec.pdf}
        \end{subfigure}
        \caption{Kernel density estimates produced by Stan. Dashed lines show true parameter values. \label{hmckernels}}
    \end{figure}


\section{The \makeheap algorithm}
\Label{sec:makeheap}

Whereas in the \cxx Standard Library \cite[\S 28.7.7.3]{cxx-17-draft}
\makeheap works on a pair of generic random access
iterators,
our version operators on a range of \valuetype.
Thus the signature of \makeheap reads

\begin{lstlisting}[style = acsl-block]

    void make_heap(value_type* a, size_type n);
\end{lstlisting}

The function \makeheap rearranges the elements of the given
array \inl{a[0..n-1]} such that they form a heap.

As an examples we look at the array in Figure~\ref{fig:makeheap-array-pre}.
The elements of this array do not form a heap, as indicated by the grey colouring.
Executing the \makeheap algorithm on this array rearranges its elements
so that they form a heap as shown in Figure~\ref{fig:heap-array}.

\begin{figure}[hbt]
\centering
\includegraphics[width=0.65\linewidth]{Figures/make_heap_array_pre.pdf}
\caption{\Label{fig:makeheap-array-pre}Array before the call of \makeheap}
\end{figure}

\FloatBarrier

\subsection{Formal specification of \makeheap}

The following listing shows the specification of \makeheap.

\input{Listings/make_heap.h.tex}

Like with \pushheap the formal specification of \makeheap
must ensure that the resulting array is a heap of size \inl{n}
and contains the same multiset of elements as in the pre-state of the function.
These properties are expressed by the \inl{heap} and \inl{reorder}
postconditions respectively.
The \inl{reorder} postcondition uses the predicate \logicref{MultisetReorder}
to ensure that \makeheap only rearranges the array elements.

\clearpage

\subsection{Implementation of \makeheap}

The implementation of \makeheap, shown in the next listing, is straightforward.
%
From low to high the array's elements are pushed to the growing heap.
%
We used \inl{i < n} as loop condition, rather than the more tempting
\inl{i <= n}, in order to admit also \inl{n == SIZE_TYPE_MAX};
as a consequence, we had to call \specref{pushheap} with \inl{i+1}.
%
The iteration starts at \inl{i+1 == 2}, because an array with length one is
a heap already.

\input{Listings/make_heap.c.tex}

Since the loop statement consists just of a call to \specref{pushheap}
we obtain the both loop invariants \inl{heap} and \inl{reorder} by simply
lifting them from the contract of \pushheap.

The postcondition of \pushheap only specifies the multiset
of elements from index 0 to \inl{i}.
%
We therefore also have to specify
that the elements from index \inl{i+1} to \inl{n-1}  are only reordered.
%
This property can be derived from the \inl{unchanged} property of \pushheap.

\clearpage


\section{Camera and Image formation process}
\label{imageFormation}

To understand how a camera perceives the environment, it is necessary we understand the image formation process. For us humans, we “see” things when a light originating from a light source is reflected on an object and enters our eyes. A camera acts very much similar to the human eye. The earliest and first model of an optical camera is the pinhole camera which is a simple and highly accurate representation of our eye model. This is the simplest device to form an image of a 3D scene on a 2D surface. As seen in figure 1 rays of light enter the pinhole and forms an inverted image of the object. This is called perspective projection. As the image formed in the image place is inverted, we consider a virtual plane in front of the pinhole that acts as the image plane.

% For one-column wide figures use
\begin{figure}[H]
% Use the relevant command to insert your figure file.
% For example, with the graphicx package use
  \includegraphics[width=\textwidth]{./figures/imageFormation.png}
% figure caption is below the figure
\caption{Pinhole Image formation~\cite{imgFormBoomgaard}}
\label{fig:1}       % Give a unique label
\end{figure}

Although the pinhole model is quite accurate, modern cameras have lenses. They help gather more light from a source leading to higher quality sharper images. Ignoring the internal diffraction, we assume thin lens equations for the imaging process. Every camera has a region of depths over which the scene is sharp. Modern cameras have variable apertures which help in focusing on objects at varying distances in the scene.

Every camera has an intrinsic, extrinsic and distortion parameters that are important to understand for every application. These parameters are used to correct for lens distortion, measure objects in physical world and to determine the location of the camera in the real world. Camera calibration is the process of estimating these parameters specific to a given camera and application setup. Figure~\ref{fig:transform} demonstrates how a real world 3D coordinate is related to a 2D pixel coordinate using the parameters mentioned above. The intrinsic camera matrix \textbf{K} is given by
\begin{equation}
K =
\begin{pmatrix}
  f_x & s & c_x \\
  0 & f_y & c_y \\
  0 & 0 & 1 \\
 \end{pmatrix}
\end{equation}
where $(f_x, f_y)$ is the focal length, $(c_x, c_y)$ is the optical center
and $s$ is skew factor. The extrinsic camera matrix $\textbf{[R|t]}$ is given by
\begin{equation}
R =
\begin{pmatrix}
  r_1 & r_2 & r_3 \\
  r_4 & r_5 & r_6 \\
  r_7 & r_8 & r_9 \\
 \end{pmatrix}
t =
\begin{pmatrix}
  t_x \\
  t_y \\
  t_z \\
 \end{pmatrix}
\end{equation}
where $R$ is the rotation matrix and $t$ is the translation vector between the camera center and world co-ordinates. The camera matrix $P$ is given by
\begin{equation}
P = K[R t]
\end{equation}

% For one-column wide figures use
\begin{figure}[H]
% Use the relevant command to insert your figure file.
% For example, with the graphicx package use
  \includegraphics[width=\textwidth]{./figures/imageParams.png}
% figure caption is below the figure
\caption{World co-ordinates to pixel co-ordinates~\cite{camMATLAB}}
\label{fig:transform}       % Give a unique label
\end{figure}

The extrinsic parameters relate the rotation $R$ and translation $t$ of the camera origin at the optical center to the world frame. The intrinsic parameters consists of the focal length given by $f_x$ and $f_y$, the optical center given by $c_x$ and $c_y$ and the skew coefficient $s$. The distortion parameters consists of radial and tangential distortion along the $x$ and $y$ direction respectively. The number of coefficients are decided based on the lens in consideration. Camera calibration techniques presented in~\cite{zhang,bouguet2004camera,heikkila1997four} are widely used in commercial and open source camera calibration tool boxes.


\documentclass{article}
\usepackage{amsmath,amssymb,fouriernc,parskip,amsthm}
\usepackage{hyperref}
\hypersetup{
  citecolor=red,
  colorlinks=true
}

\begin{document}
\theoremstyle{definition}
\newtheorem{thm}{Theorem}[section]
\newtheorem{lem}[thm]{Lemma}

\section{Normalized Fractions/Exponents}

\begin{lem}
\label{pow}
Let
\begin{equation*}
S = \left \{ \; i \in \mathbb{Z}^{\geq 0} \; | \; r^{p - 1} \leq f \cdot r^i
\; \right \}
\end{equation*}
Then
\begin{equation*}
\forall r, f, p \in \mathbb{N} \; . f > 0 \; \wedge \; p > 0 \;
\Longrightarrow \; \exists i^* \in S \; . \forall i \in S \; . \; i^* \leq i
\end{equation*}
\begin{proof} Assume the antecedent. Since $f > 0$, $f \geq 1$. Moreover,
$p > 0$, so $p - 1 \geq 0$ and $r^{p - 1} \leq f \cdot r^{p - 1}$. It follows
that $p - 1 \in S$. From the Well-Ordering Principle, $S$ has a smallest
element $i^*$.
\end{proof}
\end{lem}

\begin{lem}
\label{posfrac}
\begin{equation*}
\forall x \in \mathbb{R}, f \in \mathbb{N}, e \in \mathbb{Z} \; . \;
x \neq 0 \; \wedge \; frep(f,e,x) \; \Longrightarrow \; f > 0
\end{equation*}
\begin{proof} Assume the antecedent. Expanding the defn of $frep$, we know
\begin{equation*}
|x| = f \cdot r^{(e - p + 1)}
\end{equation*}
Since $x \neq 0$, $|x| > 0$. $r^{(e - p + 1)}$ is always positive, so $f > 0$.
\end{proof}
\end{lem}

\begin{thm}
\label{normexists}
\begin{align*}
&\forall x \in \mathbb{R} \; . \; x \neq 0 \; \wedge \; is\_float(x) \;
\Longrightarrow \\
& \qquad \exists f' \in \mathbb{N}, e' \in \mathbb{Z} \; . \;
is\_norm\_frac(f', x) \; \wedge \; is\_norm\_exp(e', x)
\end{align*}
\begin{proof} Assume $is\_float(x)$ and $x \neq 0$. From the defn of
$is\_float$, we know there exists $f \in \mathbb{N}$ and $e \in \mathbb{Z}$
such that $frep(f, e, x)$. From ~\ref{posfrac}, we know $f > 0$.

Also, from the defn of $frep$, we know
\begin{equation*}
f < r^p \; \wedge \; |x| = f \cdot r^{(e - p + 1)}
\end{equation*}
From ~\ref{pow}, we know there is a smallest integer $i^*$ such that
$r^{p - 1} \leq f \cdot r^{i^*}$. We also know that $f \cdot r^{i^*} < r^p$
(if it wasn't, we could cancel r on both sides and obtain a smaller
$i$, contradicting the definition of $i^*$). Take $f' = f \cdot r^{i^*}$.
It immediately follows that $f' \geq r^{p - 1}$. If we also take
$e' = e - i^*$, then
\begin{align*}
f' \cdot r^{e' - p + 1} &= f \cdot r^{i^*} \cdot r^{e - i^* - p + 1}\\
&= f \cdot r^{e - p + 1}\\
&= |x|
\end{align*}
and so $is\_norm\_frac(f', x)$ is true and $is\_norm\_exp(e', x)$ is true.
\end{proof}
\end{thm}

\begin{lem}
\label{normbound}
\begin{equation*}
\forall x \in \mathbb{R} \; . \; x \neq 0 \; \wedge \; is\_float(x) \;
\Longrightarrow \; r^{ne(x)} \leq |x| < r^{ne(x) + 1}
\end{equation*}
(where $ne(x)$ is an abbreviation for $decode\_norm\_exp(x)$).
\begin{proof} Assume the antecedent. Because $x$ is a non-zero float, we
know from ~\ref{normexists} that $x$ has a normalized fraction and exponent
$f$ and $e$ with $r^{p - 1} \leq f < r^p$. Multiply all three by the positive
value $r^{e - p + 1}$ to get $r^e \leq |x| < r^{e + 1}$.
\end{proof}
\end{lem}

\begin{lem}
\label{expupper}
\begin{equation*}
\forall x \in \mathbb{R}, i \in \mathbb{Z} \; .
\; x \neq 0 \; \wedge \; is\_float(x) \; \wedge \; |x| \leq r^i
\Longrightarrow \; ne(x) < i
\end{equation*}
\begin{proof} Assume the antecedent. Because $x$ is a non-zero float,
from ~\ref{normbound} we know
\begin{equation*}
r^{ne(x)} \leq |x| < r^i
\end{equation*}
From the monotonicty of pow, we can conlude $ne(x) < i$.
\end{proof}
\end{lem}

\section{largest/smallest}

\begin{lem}
\label{realpow1}
\begin{equation*}
\forall n, r \in \mathbb{N} \; . \; r > 1 \; \Longrightarrow \;
\exists i \in \mathbb{Z} \; . \; r^i > n
\end{equation*}
\begin{proof} Let $r > 1$. The proof is by induction on $n$.

\textsc{Base case.} For $n = 0, 1$, take $i = 1$.

\textsc{Inductive step.} Assume there is an $i$ such that $r^i > n$, and
consider $n + 1$:
\begin{align*}
n + 1 &< r^i + 1 \tag{induction hyp}\\
&< r^i + r \tag{$r > 1$}\\
&\leq r^{i'} + r^{i'} \tag{$i' = max(i, 1)$, monotonicity of pow}\\
&= 2r^{i'}\\
&\leq r \cdot r^{i'} \tag{$r \geq 2$}\\
&= r^{i' + 1}
\end{align*}
So, take $i = i' + 1$, and we're done.
\end{proof}
\end{lem}

\begin{lem}
\label{realpow2}
\begin{equation*}
\forall r \in \mathbb{N}, x \in \mathbb{R}^{\geq 0} \; . \; r > 1 \;
\Longrightarrow \; \exists i \in \mathbb{Z} \; . \; r^i > x
\end{equation*}
\begin{proof} Let $x$ and $r > 1$. From the archimedean property (available
in HOL light), we know
there is an $n$ such that $n > x$. From ~\ref{realpow1}, there is an $i$
such that $r^i > n > x$.
\end{proof}
\end{lem}

\begin{lem}
\label{realpow3}
\begin{equation*}
\forall r \in \mathbb{N}, x \in \mathbb{R^+} \; . \; r > 1 \; \Longrightarrow \;
\exists i \in \mathbb{Z} \; . \; r^i < x
\end{equation*}
\begin{proof} Let $x$ and $r > 1$. Since $x > 0$,
$inv(x) > 0$, and from ~\ref{realpow2}, there is an $i'$ such that
$r^{i'} > inv(x)$. Then, multiplying by $x$ and $r^{-i'}$ on both sides (both are
positive):
\begin{equation*}
x > r^{-i'}
\end{equation*}
Take $i = i'$.
\end{proof}
\end{lem}

\begin{lem}
\label{realbetw}
\begin{equation*}
\forall x \in \mathbb{R} \; . \; \exists u, v \in \mathbb{Z} \; . \;
v = u + 1 \; \wedge \; u \leq x < v
\end{equation*}
\begin{proof} Let $x \in \mathbb{R}$.
\begin{itemize}
\item If $x > 0$, let $S = \{ \; n \in \mathbb{N} \; | \; x < n \; \}$.
From the archimedean property (REAL\_ARCH), we know $S$ is non-empty,
and so from the well-ordering property (num\_WOP), $S$ has a least element
$n^*$. Since $n^* > x > 0$, $n^* - 1 < n^*$, and hence $n^* - 1 \leq x$.
Take $u = n^* - 1$ and $v = n^*$.
\item If $x < 0$, get $u', v'$ for $-x > 0$ such that $u' \leq -x < v'$. Then
$-v' < x \leq -u'$. If $x = -u'$, take $u = -u'$ and $v = -u' + 1$. If
$x < -u'$, take $v = -u'$ and $u = -u' - 1 \geq -v'$. (I'm guessing the
real arithmetic decision procedures could take over from here ...).
\item If $x = 0$, take $u = 0$ and $v = 1$.
\end{itemize}
\end{proof}
\end{lem}

\begin{lem}
\label{intmax}
\begin{equation*}
\forall S \subset \mathbb{Z}, b \in \mathbb{R} \; . \;
 S \neq \emptyset \; \wedge \; \big [ \; \forall s \in S \; . \; s \leq b \;
\big ] \; \Longrightarrow \; \exists s^* \in S \; . \; \forall s \in S \; . \;
s \leq s^*
\end{equation*}
\begin{proof} Assume the antecedent. Because $S \neq \emptyset$ and
there is an upper bound $b$ for $S$, $sup(S)$ exists.
\begin{itemize}
\item If $sup(S) \in S$, take $s^* = sup(S)$ (from the HOL-light defn of
$sup$, the result will follow).
\item Otherwise, $sup(S) \notin S$. From ~\ref{realbetw}, there are
$u, v \in \mathbb{Z}$ with $u \leq sup(S) < v$.
\begin{itemize}
\item If $u = sup(S)$, then
\begin{itemize}
\item If $u \in S$, then $sup(S) \in S$, contradiction.
\item If $u \notin S$, then consider $u - 1$. If $u - 1 \geq s$ for all
$s \in S$, then we get a contradiction in the defn of sup. Otherwise, if
there is some $s \in S$ with $s > u - 1$, then $s \geq u = sup(S)$
(INT\_GT\_DISCRETE). If $s = sup(S)$, then $sup(S) \in S$, contradiction.
Otherwise, $s > sup(S)$, also a contradiction.
\end{itemize}
\item If $u < sup(S)$, then if for all $s \in S$, $s \leq u$, we get a
contradiction in defn of sup. Otherwise, there is some $s \in S$ with
$s > u$, so $s \geq u + 1 = v > sup(S)$, contradiction.
\end{itemize}
\end{itemize}
\end{proof}
\end{lem}

\begin{lem}
\label{intmin}
\begin{equation*}
\forall S \subset \mathbb{Z}, b \in \mathbb{R} \; . \;
 S \neq \emptyset \; \wedge \; \big [ \; \forall s \in S \; . \; s \geq b \;
\big ] \; \Longrightarrow \; \exists s^* \in S \; . \; \forall s \in S \; . \;
s \geq s^*
\end{equation*}
\begin{proof} Assume the antecedent, and take $S' = -S$. Since $S \neq
\emptyset$, $S' \neq \emptyset$. Also, since $b \leq s$ for all $s \in S$,
$b' = -b \geq -s$ for all $s \in S$, hence $b'$ is an upper bound for
$-S$. From ~\ref{intmax}, we know there is an $s^* \in S'$ with
$s^* \geq s'$ for all $s' \in S'$. This in turn means $-s^* \leq -s'$ for all
$s' \in S'$, and $s^* = -s^*$ is the minimum for $S$ (probably some tedious
details to sort out here).
\end{proof}
\end{lem}

\begin{thm}
\label{maxabsexists}
\begin{equation*}
\forall S \subset F, b \in \mathbb{R}^+ \; . \; S \neq \emptyset \; \wedge \;
\big [ \; \forall s \in S \; . \; |s| \leq b \; \big ] \;
\Longrightarrow \exists s^* \in S \; . \; \forall s \in S \; . \; |s| \leq |s^*|
\end{equation*}
\begin{proof} Assume the antecedent. If $S = \{0\}$, then $s^* = 0$ and we're
done. Otherwise, assume there is $s \neq 0$ in $S$. Take $S' = \{ \; s \in S \;
| s \neq 0 \; \}$.

From ~\ref{realpow2}, we know there is an integer $i$ with $r^i > b$.
This implies that $|s| < r^i$ for all $s \in S'$. From ~\ref{expupper}, we
know $ne(s) < i$ for all $s \in S'$. Let
\begin{equation*}
Exp = \{ \; e \; | \; \exists s \in S' \; . \; e = ne(s) \; \}
\end{equation*}
$i$ is therefore an upper bound for $Exp$. From ~\ref{intmax}, there is a
maximal element $e^*$ in $Exp$.

Next, take
\begin{equation*}
Frac = \{ \; f \; | \; \exists s \in S' \; . \; f = nf(s) \; \wedge \;
ne(s) = e^* \; \}
\end{equation*}
Since $f < r^p$ for all $f \in Frac$, from ~\ref{intmax}, there is a maximal
element $f^* \in Frac$. Take $s^* = f^* \cdot r^{e^* - p + 1}$.

Consider any $s \in S'$. We know $ne(s) \leq e^*$. If $ne(s) < e^*$, then
\begin{align*}
|s| &= nf(s) \cdot r^{ne(s) - p + 1}\\
& < r^p \cdot r^{ne(s) - p + 1}\\
&\leq r^{e^*}\\
& = r^{p - 1} \cdot r^{e^* - p + 1}\\
& \leq f^* \cdot r^{e^* - p + 1}\\
& = |s^*|
\end{align*}
If $ne(s) = e^*$, then, since $nf(s) \leq f^*$,
\begin{equation*}
|s| = nf(s) \cdot r^{e^* - p + 1} \leq f^* \cdot r^{e^* - p + 1} = |s^*|
\end{equation*}
Finally, since $f^* > 0$, $|s^*| > 0$, so the claim is true for $s = 0$, and
hence true for all $s \in S$.
\end{proof}
\end{thm}

\end{document}


\documentclass[11pt, a4paper]{article}

\usepackage{amsmath}
\usepackage{tikz}
\usepackage{placeins}

\begin{document}

\title{FISHER'S LINEAR DISCRIMINANT ANALYSIS}
\date{}
\maketitle

Fisher's Linear Discriminant Analysis (FDA) is a dimensionality reduction technique to ease classification.

\section{Two Class Case}

Consider the case of $N$ points in $d$ dimensions, with each point belonging to one of the two classes $C_1$ and $C_2$. The idea is to find the optimal direction to project the vector of these points to. Such a projection can be represented as,

\begin{align*}
	y = \boldsymbol{w}^T\boldsymbol{x}
\end{align*}

where $\boldsymbol{w}$ is a $d$ dimensional vector defining the direction of projection, $\boldsymbol{x}$ is the vector being projected and $y$ is a scalar representing the magnitude of projection.

The projection should serve two purposes as discussed in the following subsections:

\subsection{Maximizing Between-Class Scatter}

The class means should be projected as far apart as possible. Let $\boldsymbol{m_1}$ and $\boldsymbol{m_2}$ denote the mean of class $C_1$ and $C_2$ respectively.

\begin{align*}
	\boldsymbol{m_1} & = \frac{1}{N} \sum_{\boldsymbol{x_i} \in C_1} \boldsymbol{x_i} \\
	\boldsymbol{m_2} & = \frac{1}{N} \sum_{\boldsymbol{x_i} \in C_2} \boldsymbol{x_i}
\end{align*}

If $\boldsymbol{m_1}$ is projected to $m_1$ and $\boldsymbol{m_2}$ is projected to $m_2$, then the between-scatter is defined by,

\begin{align*}
	(m_1 - m_2)^2 & = (\boldsymbol{w}^T\boldsymbol{m_1} - \boldsymbol{w}^T\boldsymbol{m_2})^2                                    \\
	              & = (\boldsymbol{w}^T(\boldsymbol{m_1} - \boldsymbol{m_2}))^2                                                  \\
	              & = \boldsymbol{w}^T(\boldsymbol{m_1} - \boldsymbol{m_2})\boldsymbol{w}^T(\boldsymbol{m_1} - \boldsymbol{m_2}) \\
	              & = \boldsymbol{w}^T(\boldsymbol{m_1} - \boldsymbol{m_2})(\boldsymbol{m_1} - \boldsymbol{m_2})^T\boldsymbol{w} \\
	              & = \boldsymbol{w}^T\boldsymbol{S_B}\boldsymbol{w}
\end{align*}

where $\boldsymbol{S_B}$ represents the between-class scatter matrix.

\subsection{Minimizing Within-Class Scatter}

The projections of each class should be as condensed as possible. The within-class scatter of the transformed data belonging to class $C_k$ is denoted by,

\begin{align*}
	s_k^2 & = \sum_{i \in C_k} (y_i - m_k)^2                                                                                \\
	      & = \sum_{i \in C_k} (\boldsymbol{w}^T\boldsymbol{x_i} - \boldsymbol{w}^T\boldsymbol{m_k})^2                                  \\
	      & = \sum_{i \in C_k} (\boldsymbol{w}^T(\boldsymbol{x_i} -\boldsymbol{m_k}))^2                                                 \\
	      & = \sum_{i \in C_k} \boldsymbol{w}^T(\boldsymbol{x_i} -\boldsymbol{m_k})(\boldsymbol{x_i} -\boldsymbol{m_k})^T\boldsymbol{w} \\
	      & = \boldsymbol{w}^T \boldsymbol{S_k} \boldsymbol{w}                                                                          \\
\end{align*}

where,
\begin{align*}
	\boldsymbol{S_k} = \sum_{i \in C_k}(\boldsymbol{x_i} -\boldsymbol{m_k})(\boldsymbol{x_i} -\boldsymbol{m_k})^T
\end{align*}

In the case of two classes, total within class scatter is denoted by,

\begin{align*}
	s_1^2 + s_2^2 & = \boldsymbol{w}^T \boldsymbol{S_1} \boldsymbol{w} + \boldsymbol{w}^T \boldsymbol{S_1} \boldsymbol{w} \\
	              & = \boldsymbol{w}^T (\boldsymbol{S_1}+\boldsymbol{S_2}) \boldsymbol{w}                                 \\
	              & = \boldsymbol{w}^T \boldsymbol{S_W} \boldsymbol{w}
\end{align*}

where $\boldsymbol{S_W}$ represents the within-class scatter matrix.

\subsection{Combining Minimization and Maximization}

A reasonable way to simultaneously maximize the between-class scatter and minimize the within-class scatter is to maximize their fraction defined as follows,

\begin{align*}
	J(\boldsymbol{w}) = \frac{\boldsymbol{w}^T \boldsymbol{S_B} \boldsymbol{w}}{\boldsymbol{w}^T \boldsymbol{S_W} \boldsymbol{w}}
\end{align*}

Note that $J(\boldsymbol{w})$ is invariant under rescalings of the form $\boldsymbol{w} \Rightarrow \alpha \boldsymbol{w}$. This sets up the reformulation of this problem as per the following,

\begin{align*}
	\text{maximize} & \ \ \boldsymbol{w}^T \boldsymbol{S_B} \boldsymbol{w}     \\
	\text{s.t.}     & \ \ \boldsymbol{w}^T \boldsymbol{S_W} \boldsymbol{w} = 1
\end{align*}

Using the concept of Lagrangian,

\begin{align*}
	L(\boldsymbol{w}, \lambda) = \boldsymbol{w}^T \boldsymbol{S_B} \boldsymbol{w} - \lambda(\boldsymbol{w}^T \boldsymbol{S_W} \boldsymbol{w} - 1)
\end{align*}

Differentiating w.r.t. $\boldsymbol{w}$,

\begin{align*}
	\frac{\partial L(\boldsymbol{w}, \lambda)}{\partial \boldsymbol{w}} = 2\boldsymbol{S_B}\boldsymbol{w} - 2\lambda \boldsymbol{S_W}\boldsymbol{w}
\end{align*}

Equating the diffenrential to zero, the solution follows,

\begin{align*}
	\boldsymbol{S_B}\boldsymbol{w} = \lambda \boldsymbol{S_W}\boldsymbol{w}
\end{align*}

This is the generalized eigenvalue problem that can be solved easily. Note that since $\boldsymbol{S_B}$ is a product of two vectors and thus of rank one, the above equation will only yield one eigenvalue, eigenvector pair. The eigenvector is the sought projection direction.

\section{Example}

Consider the following set of six points in two dimesions distributed equally among the two classes,

\FloatBarrier\clearpage
\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]
		\tikzstyle {margin} = [dashed]

		\draw[step=1.0, gray, very thin] (-5.5, -2.5) grid (5.5, 4.5);

		\draw[-latex] (-6,0) -- (6,0) node[right]{x1};
		\draw[-latex] (0,-3) -- (0,5) node[left]{x2};

		\draw[point line] (-1.3, 1) -- (-0.7, 1);
		\draw[point line] (-1, 0.7) -- (-1, 1.3);
		\draw (-1, 1) node [below right] {(-1, 1)};

		\draw[point line] (-4.3, -1) -- (-3.7, -1);
		\draw[point line] (-4, -1.3) -- (-4, -0.7);
		\draw (-4, -1) node [below left] {(-4, -1)};

		\draw[point line] (-4.3, 3) -- (-3.7, 3);
		\draw[point line] (-4, 2.7) -- (-4, 3.3);
		\draw (-4, 3) node [above left] {(-4, 3)};

		\draw[point line] (1.7, 1) -- (2.3, 1);
		\draw (2, 1) node [above left] {(2, 1)};

		\draw[point line] (2.7, 0) -- (3.3, 0);
		\draw (3, 0) node [above left] {(3, 0)};

		\draw[point line] (0.7, 2) -- (1.3, 2);
		\draw (1, 2) node [above left] {(1, 2)};

	\end{tikzpicture}
\end{figure}

The means for the respective classes are,

\begin{align*}
	\boldsymbol{m_{+}} & = \begin{pmatrix}
	-3 \\
	1
	\end{pmatrix} \\
	\boldsymbol{m_{-}} & = \begin{pmatrix}
	2 \\
	1
	\end{pmatrix}
\end{align*}

Now, the between-class scatter matrix is calculated as follows,

\begin{align*}
	\boldsymbol{S_B} & = (\boldsymbol{m_{+}} - \boldsymbol{m_{-}})(\boldsymbol{m_{+}} - \boldsymbol{m_{-}})^T \\
	                 & = \begin{pmatrix}
	-5 \\
	0
	\end{pmatrix} \begin{pmatrix}
	-5               & 0
	\end{pmatrix} \\
	                 & = \begin{pmatrix}
	25               & 0                                                                                \\
	0                & 0
	\end{pmatrix}
\end{align*}

And then, the within-class scatter matrix is calculated,

\begin{align*}
	S_+              & = \sum_{i \in +}(\boldsymbol{x_i} -\boldsymbol{m_+})(\boldsymbol{x_i} -\boldsymbol{m_+})^T \\
	                 & = \begin{pmatrix}
	2 \\
	0
	\end{pmatrix} \begin{pmatrix}
	2                & 0
	\end{pmatrix} + \begin{pmatrix}
	-1 \\
	2
	\end{pmatrix} \begin{pmatrix}
	-1               & 2
	\end{pmatrix} + \begin{pmatrix}
	-1 \\
	-2
	\end{pmatrix} \begin{pmatrix}
	-1               & -2
	\end{pmatrix}  \\
	                 & = \begin{pmatrix}
	6                & 0                                                                                \\
	0                & 8
	\end{pmatrix} \\
	S_-              & = \sum_{i \in -}(\boldsymbol{x_i} -\boldsymbol{m_-})(\boldsymbol{x_i} -\boldsymbol{m_-})^T \\
	                 & = \begin{pmatrix}
	0 \\
	0
	\end{pmatrix} \begin{pmatrix}
	0                & 0
	\end{pmatrix} + \begin{pmatrix}
	-1 \\
	1
	\end{pmatrix} \begin{pmatrix}
	-1               & 1
	\end{pmatrix} + \begin{pmatrix}
	1 \\
	-1
	\end{pmatrix} \begin{pmatrix}
	1                & -1
	\end{pmatrix}  \\
	                 & = \begin{pmatrix}
	2                & -2                                                                                \\
	-2               & 2
	\end{pmatrix} \\
	\boldsymbol{S_W} & = \boldsymbol{S_+} + \boldsymbol{S_-}                                                      \\
	                 & = \begin{pmatrix}
	8                & -2                                                                                \\
	-2               & 10
	\end{pmatrix}
\end{align*}

The eigenvalue problem is formulated as,

\begin{align*}
	\boldsymbol{S_B}\boldsymbol{w} = \lambda \boldsymbol{S_W}\boldsymbol{w}
\end{align*}

Equating the determinant of $\boldsymbol{S_B} - \lambda \boldsymbol{S_W}$ to zero,

\begin{align*}
	\begin{vmatrix}
	25-8\lambda   & 2\lambda   \\
	2\lambda      & -10\lambda
	\end{vmatrix} & = 0        \\
	\lambda = \frac{125}{38}
\end{align*}

The corresponding eigenvector amounts to $\frac{1}{\sqrt{26}}\begin{pmatrix}
5 \\
1
\end{pmatrix}$ which is the direction of projection. The below plot helps to visualize how this projection direction divides the two classes.

\FloatBarrier\clearpage
\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]
		\tikzstyle {margin} = [dashed]

		\draw[step=1.0, gray, very thin] (-5.5, -2.5) grid (5.5, 4.5);

		\draw[-latex] (-6,0) -- (6,0) node[right]{x1};
		\draw[-latex] (0,-3) -- (0,5) node[left]{x2};

		\draw[point line] (-1.3, 1) -- (-0.7, 1);
		\draw[point line] (-1, 0.7) -- (-1, 1.3);
		\draw (-1, 1) node [below right] {(-1, 1)};

		\draw[point line] (-4.3, -1) -- (-3.7, -1);
		\draw[point line] (-4, -1.3) -- (-4, -0.7);
		\draw (-4, -1) node [below left] {(-4, -1)};

		\draw[point line] (-4.3, 3) -- (-3.7, 3);
		\draw[point line] (-4, 2.7) -- (-4, 3.3);
		\draw (-4, 3) node [above left] {(-4, 3)};

		\draw[point line] (1.7, 1) -- (2.3, 1);
		\draw (2, 1) node [above left] {(2, 1)};

		\draw[point line] (2.7, 0) -- (3.3, 0);
		\draw (3, 0) node [above left] {(3, 0)};

		\draw[point line] (0.7, 2) -- (1.3, 2);
		\draw (1, 2) node [above left] {(1, 2)};

		\draw[dashed] (5.5, 1.1) -- (-5.5, -1.1);
		\draw[thin] (-4, -1) -- (-4.04, -0.81);
		\draw[thin] (-1, 1) -- (-0.77, -0.15);
		\draw[thin] (-4, 3) -- (-3.27, -0.66);

		\draw[thin] (1, 2) -- (1.35, 0.27);
		\draw[thin] (2, 1) -- (2.12, 0.42);
		\draw[thin] (3, 0) -- (2.88, 0.58);

	\end{tikzpicture}
\end{figure}

\section{Multi-Class Case}

In two class case dimensionality was reduced to 1. In $K$ class case, it is reduced to $K-1$. Hence, instead of finding an optimal $d$-dimensional vector $\boldsymbol{w}$ in the two class case, a matrix of $K-1$ $d$-dimensional vectors $\boldsymbol{W}^T = \begin{pmatrix}
\boldsymbol{w_1}^T \\
\boldsymbol{w_1}^T \\
\vdots \\
\boldsymbol{w_{K-1}}^T \\
\end{pmatrix}$ must be found. The projection is defined as $\boldsymbol{y}=\boldsymbol{W}^T\boldsymbol{x}$ where $\boldsymbol{y}$ is a $K-1$ dimensional vector as expected.

The generalization for within-class matrix is as follows,

\begin{align*}
	\boldsymbol{S_W} = \sum_{k=1}^K \sum_{i \in C_k} (\boldsymbol{x_i} - \boldsymbol{m_k})(\boldsymbol{x_i} - \boldsymbol{m_k})^T
\end{align*}

For generalizing between-class matrix, consider the total scatter matrix $\boldsymbol{S_T}$ as follows,

\begin{align*}
	\boldsymbol{S_T} & = \sum_{i=1}^N (\boldsymbol{x_i} - \boldsymbol{m})(\boldsymbol{x_i} - \boldsymbol{m})^T                                                                                \\
	                 & = \sum_{k=1}^K \sum_{i \in C_k} (\boldsymbol{x_i} - \boldsymbol{m})(\boldsymbol{x_i} - \boldsymbol{m})^T                                                                                \\
	                 & = \sum_{k=1}^K \sum_{i \in C_k} (\boldsymbol{x_i} - \boldsymbol{m_k} + \boldsymbol{m_k} - \boldsymbol{m})(\boldsymbol{x_i} - \boldsymbol{m_k} + \boldsymbol{m_k} - \boldsymbol{m})^T                                  \\
	                 & = \sum_{k=1}^K \sum_{i \in C_k} (\boldsymbol{x_i} - \boldsymbol{m_k})(\boldsymbol{x_i} - \boldsymbol{m_k})^T + \sum_{k=1}^K \sum_{i \in C_k} (\boldsymbol{m_k} - \boldsymbol{m})(\boldsymbol{m_k} - \boldsymbol{m})^T \\
	                 & = \boldsymbol{S_W} + \sum_{k=1}^K N_k(\boldsymbol{m_k} - \boldsymbol{m})(\boldsymbol{m_k} - \boldsymbol{m})^T                                                                                \\
	                 & = \boldsymbol{S_W} + \boldsymbol{S_B}
\end{align*}

The maximization problem is then generalized as follows,

\begin{align*}
	J(\boldsymbol{W}) = \frac{det(\boldsymbol{W}^T \boldsymbol{S_B} \boldsymbol{W})}{det(\boldsymbol{W}^T \boldsymbol{S_W} \boldsymbol{W})}
\end{align*}

The solution to find the matrix $\boldsymbol{W}$ is to find the largest $K-1$ eigenvalues of the following equation and arrange the corresponding eigenvectors in a matrix.

\begin{align*}
	\boldsymbol{S_B}\boldsymbol{w} = \lambda \boldsymbol{S_W}\boldsymbol{w}
\end{align*}

Additionally, there are no more than $K-1$ non-zero eigenvectors to the above
equation due to the redundancy of the matrix $\boldsymbol{S_B}$ as was seen in the two class case.

\end{document}

Fisher's Linear Discriminant Analysis/Fisher's Linear Discriminant Analysis.tex


\subsection{Representing natural numbers}

We will use a byte to describe natural numbers. This gives us a range of \(2^8=256\). As we include \(0\) the largest number here is \(255\).

\subsection{Zero}

We describe zero using all \(0\)s.

\(0=00000000\)

\subsection{Natural numbers}

We show other natural numbers by iterating through the possible bits. To increase by one we take the last bit and use the NOT operator on it. If this bit is \(1\) we are done.

If this bit is now \(0\) we move to the next bit and flip that. We then repeat the check and repeat until we either run out of bits, or flip a bit to \(1\).


\chapter{Paths in Graphs}
\label{chapter:paths}
\section{Connectivity}
Imagine you are developing a game, where the map is generated automatically.
In this gate there are several areas connected by portals. So you need to check
that all the areas in your map are reachable from one another.

First we need to somehow understand what we mean by ``reachable'', we say that
an area $A$ is reachable from an area $B$ if there is a path from $A$ to
$B$. To formalize this notion using graphs we need to introduce a graph
corresponding to the map, consider a graph $G = (V, E)$ such that vertices of
the graph are areas in your map and $(A, B) \in E$ iff the areas $A$ and $B$
are connected by a portal. So a path from $A$ to $B$ is a sequence of areas
$A = C_1$, \dots, $C_\ell = B$ such that $C_i$ and $C_{i + 1}$ are connected by
a portal (i.e. $(C_i, C_{i + 1}) \in E$).
\begin{definition}
  Let $G = (V, E)$ be a graph. We say that a path from $u$ to $v$ is
  a sequence $w_1, \dots, w_\ell \in V$\footnote{%
    Usually such an object is called a walk, and it is called a path if
    all the vertices $w_1$, \dots, $w_\ell$ are different. However,
    for our applications it does not matter and we will use the word ``path''.
  } such that
  \begin{itemize}
    \item $w_1 = u$, $w_\ell = v$, and
    \item $(w_i, w_{i + 1}) \in E$ for $i \in [\ell - 1]$.
  \end{itemize}

  We say that $u, v \in V$ are connected iff there is a path from $u$ to $v$.
  So the graph is connected iff any $u, v \in V$ are connected.
\end{definition}

\begin{exercise}
  Let $G = ([2n], E)$ be a graph such that $(i, j) \in E$ if $|i - j| = 2$.
  Is $G$ connected?
\end{exercise}

So, using this notation, we need to check whether the graph corresponding to
the map is connected. There are numerous ways to do it, we consider a simple
algorithm just to see how it works.
\begin{algorithm}
  \begin{algorithmic}[1]
    \Function{Connected}{$n$, $E$}
      \State $S \gets \emptyset$
      \State $Q \gets \set{1}$

      \label{algorithm:connectivity-cycle}
      \While{$Q \neq \emptyset$}
        \State Choose an element $v$ from $Q$
        \State $Q \gets S \setminus \set{v}$

        \State $S \gets S \cup \set{v}$

        \State $Q \gets Q \cup
          \set[{(v, u) \in E \text{ and } u \notin S}]{u \in \range{n}}$
      \EndWhile
      \label{line:connectivity-last}
      \State \Return{$S = \range{n}$}
    \EndFunction
  \end{algorithmic}
  \caption{An algorithm checking whether the graph on $[n]$ with the set of
  edges $E$ is connected.}
  \label{algorithm:connectivity}
\end{algorithm}

\begin{theorem}
  Algorithm~\ref{algorithm:connectivity} checks whether the graph $([n], E)$
  is connected.
\end{theorem}
\begin{proof}
  First of all, note that the algorithm has a finite running time since
  size of $S$ increases by $1$ in the cycle starting on
  line~\ref{algorithm:connectivity-cycle}.
  It is also easy to see that if a vertex $v \in Q$ at some point it is
  in $S$ on line~\ref{line:connectivity-last}. In addition, if $v \in Q$
  at some point, then $\set[{(v, u) \in E}]{u \in \range{n}} \subseteq S$ on
  line~\ref{line:connectivity-last}.

  Therefore if $u \notin S$ and $(v, u) \in E$,
  then $v \notin S$. Using this observation we may prove that
  if $G = ([n], E)$ is connected, then Algorithm~\ref{algorithm:connectivity}
  returns true. Indeed, assume the opposite. Consider $u \in \range{n} \setminus S$,
  and $N_i \subseteq \range{n}$ such that \[
    N_0 = \set{u},
    N_{i + 1} = N_i \cup \set[{w \in N_i, (v, w) \in E}]{v \in \range{n}}.
  \]
  Note that by the previous observation if $v \in N_i$, then $v \notin S$.
  Since $G$ is connected, there is a path $u = v_1, \dots, v_k = 1$.
  Note that $u \in N_0$, $v_2 \in N_1$, \dots, $v_k \in N_{k - 1}$.
  Therefore $1 = v_k \notin S$ which is a contradiction.

  To finish the proof we need to show that if $S = \range{n}$, then the graph is
  connected. To prove the statement we prove by induction that there is a path
  from $1$ to any element of $S$ and $Q$ in every iteration of
  line~\ref{algorithm:connectivity-cycle}. Indeed, initially $S$ is empty and
  $Q$ contains only $1$. After an iteration of
  line~\ref{algorithm:connectivity-cycle} we choose an element $v$ from $Q$
  and by the induction hypothesis there is a path from $1$ to $v$.
  We add it to $S$ and the statement about $S$ holds, afterwards we
  add all the neighbours of $v$ to $Q$. So the statement about $Q$ is
  also true.
\end{proof}

Not all the graphs are connected, but it is always possible to split the
graph into connected parts, such parts are called connected components.
\begin{definition}
  Let $G = (V, E)$ be a graph. We say that $U \subseteq V$ is a connected
  component if for any $u \in U$ and $v \in V$,
  $v \in U$ iff there is a path from $u$ to $v$ in $G$.
\end{definition}

\begin{theorem}
  Let $G = (V, E)$ be a graph.
  If $U_1$ and $U_2$ are connected components of $G$, then they
  either equal to each other or disjoint.
  Moreover there are connected components
  $V_1$, \dots, $V_k$ in $G$ such that $V_1 \cup \dots \cup V_k = V$
  and $V_1$, \dots, $V_k$ are disjoint.
\end{theorem}

\begin{exercise}
  Let $G = ([2n], E)$ be a graph such that $(i, j) \in E$ if $|i - j| = 2$.
  Find all the connected components of $G$.
\end{exercise}

\begin{exercise}
  Find a modification of Algorithm~\ref{algorithm:connectivity}
  that can find all the connected components of $([n], E)$.
\end{exercise}

\section{Eulerian Paths}

Graph theory originated from a simple question asked by Leonard Euler:
``Is it possible to walk through the town of K\"{o}nigsberg, starting and
ending at the same place, so that we use each bridge exactly once?'' (the map of
K\"{o}nigsberg is depicted on Figure~\ref{figure:konigsberg}).
\begin{figure}
  \begin{center}
    \begin{tikzpicture}[thick]
      \draw plot [smooth] coordinates {(0,0) (1, 0) (2,0.3) (3,0) (4,-0.3) (5, 0) (6, 0)};
      \draw plot [smooth] coordinates {(0, -3) (3.5, -3.3) (6, -3)};

      \draw plot [smooth] coordinates {(0,-1) (1, -1.1) (2, -1.5) (1, -1.9) (0,-2)};

      \draw plot [smooth cycle] coordinates {(4, -1) (5, -1.1) (6, -1.5) (5, -1.9) (4, -2)};

      \draw [line width=2mm] (0.5, 0.3) -- (0.5, -1.3);
      \draw [line width=2mm] (0.5, -3.3) -- (0.5, -1.7);
      \draw [line width=2mm] (4.5, 0.5) -- (4.5, -1.3);
      \draw [line width=2mm] (4.5, -3.5) -- (4.5, -1.7);
      \draw [line width=2mm] (5.5, 0.3) -- (5.3, -1.3);
      \draw [line width=2mm] (5.5, -3.3) -- (5.3, -1.7);
      \draw [line width=2mm] (1.7, -1.5) -- (4.1, -1.5);
    \end{tikzpicture}
  \end{center}
  \caption{K\"{o}nigsberg's map}
  \label{figure:konigsberg}
\end{figure}
It is possible to see that the geometry of the islands is not important for
this problem, the only important property is the number of bridges between
islands.

In other words, all the necessary information can be described by the graph
(the islands are vertices and the bridges are edges) depicted on
Figure~\ref{figure:konigsberg-graph}.
\begin{figure}
  \begin{center}
    \begin{tikzpicture}[thick]
      \node[circle, draw, inner sep=0pt, minimum size=6pt] (v1) at (0,0) {};
      \node[circle, draw, inner sep=0pt, minimum size=6pt] (v2) at (1,1) {};
      \node[circle, draw, inner sep=0pt, minimum size=6pt] (v3) at (-1,1) {};
      \node[circle, draw, inner sep=0pt, minimum size=6pt] (v4) at (0,2) {};

      \draw (v4) to[out=-60, in=150] (v2);
      \draw (v4) to[out=-30, in=120] (v2);
      \draw (v1) to[out=60, in=-150] (v2);
      \draw (v1) to[out=30, in=-120] (v2);
      \draw (v2) -- (v3);
      \draw (v1) -- (v3);
      \draw (v4) -- (v3);
    \end{tikzpicture}
  \end{center}
  \caption{The graph of K\"{o}nigsberg's bridges}
  \label{figure:konigsberg-graph}
\end{figure}
Hence, to formalize the problem we need to give the following definition.
\begin{definition}
  A path $v_1$, \dots, $v_k$ in a graph $G = (V, E)$ is called Eulerian if for
  any edge $(u_1, u_2) \in E$ there is exactly one $i \in [k - 1]$ such that
  $u_1 = v_i$ and $u_2 = v_{i + 1}$.

  An Eulerian path is called an Eulerian cycle if $v_1 = v_k$.
\end{definition}
Using this definition the question is whether there exists an Eulerian cycle in
the graph of K\"{o}nigsberg's bridges.

\begin{exercise}
  Check whether the graph of K\"{o}nigsberg's bridges has an Eulerian cycle or
  not.
\end{exercise}

The following theorem gives a simple criterion that allows us to solve the
problem in the general case.
\begin{theorem}
\label{theorem:eulerian}
  A connected graph $G$ has an Eulerian cycle if and only if all vertices
  of $G$ have even degree.
  (Note that the statement holds even if $G$ has parallel edges).
\end{theorem}

\begin{proof}
  Assume that such a cycle exists
  If a vertex $v$ appears $k$ times in the cycle, then there are $2k$ edges
  involving $v$ in the cycle (because, each time $v$ is visited, there is an
  edge used to step on $v$ and one to leave from $v$); since the cycle contains
  all the edges of the graph, $v$ has degree $2k$. Therefore all vertices have
  even degree. This shows that if a connected graph contains an Eulerian cycle,
  then every vertex has even degree.

  To prove this statement in the other direction, we will prove by induction a
  stronger statement, we will prove that if $G$ is a graph in which every
  vertex has even degree, then every connected non-trivial connected
  component of $G$ (a connected component is trivial if it contains only an
  isolated vertex of degree zero) has an Eulerian cycle. We will proceed by
  induction on the number of edges.

  If there are zero edges, then every connected component has only one vertex
  and so it is nothing to prove. This is the base case of the induction.

  If we have a graph $G = (V,E)$ with a non-empty set of edges and in which
  every vertex has even degree, then let $V_1$, \dots, $V_m$ be the non-trivial
  connected components of $V$. If $m \ge 2$, then every connected component has
  strictly less vertices than $G$, and so we can apply the inductive hypothesis
  and find Eulerian cycles in each of $V_1$, \dots, $V_m$.

  It remains to consider the case in which the set $V'$ of vertices of non-zero
  degree of $G$ are all in the same connected component. Let $G' = G[V']$.
  Since every vertex of $G'$ has degree at least, there must be a cycle in
  $G'$. Let $C$ be a simple cycle (that is, a cycle with no vertices repeated)
  in $G'$, and let $G'' - C$. Since we have removed two edges from every
  vertex, we have that $G''$ is still a graph in which every vertex has even
  degree. Since $G''$ has fewer edges than $G'$ we can apply the induction
  hypothesis, and find an Eulerian cycle in each non-trivial connected
  component of $G''$. We can then patch together these Eulerian cycles with $C$
  as follows: we traverse $C$, starting from any vertex; the first time we
  reach one of the non-trivial connected components of $G''$, we stop
  traversing $C$, and we traverse the Eulerian cycle of the component, then
  continue on $C$, until we reach for the first time one of the non-trivial
  connected components of $G''$ that we haven’t traversed yet, and so on. This
  describes a Eulerian path into all of $G'$
\end{proof}

\begin{exercise}
  Finish the proof of Theorem~\ref{theorem:eulerian} by proving that if a graph
  $G$ has only vertices of an odd degree, then there is a simple cycle in $G$.
\end{exercise}

\begin{corollary}
  A graph $G$ has an Eulerian path starting and ending in two different
  vertices if and only if in $G$ there are exactly two vertices with odd
  degrees.
  (Note that the statement holds even if $G$ has parallel edges).
\end{corollary}
\begin{proof}
  Let $G = (V, E)$ and $u$ and $v$ be the vertices with odd degrees. Let us
  consider the graph $G + (u, v) = (V, E \cup (u, v))$ (if there are edges
  between $u$ and $v$ we increase their number by one). Note that all
  the degrees in $G + (u, v)$ are even.
  \nomenclature[G]{$G + e$}{denotes the graph $(V, E \cup \set{e})$}
  Therefore by Theorem~\ref{theorem:eulerian}, there is an Eulerian cycle in
  $G + (u, v)$. Without loss of generality the cycle is in the form $u$, $v$,
  $w_1$, \dots, $w_k$, $v$. Therefore, there is an Eulerian path $v$,
  $w_1$, \dots, $w_k$, $v$ in $G$.
\end{proof}

\section{Hamiltonian Paths}

Another example of a path that mathematicians are interested in is Hamiltonian
path.
\begin{definition}
  Let $G$ be a graph. We say that a path in $G$ is Hamilton
  if it visits every vertex in $G$ exactly once. We say that such a path is a
  Hamiltonian cycle if its starting and ending vertices are
  connected.\footnote{%
    Hamiltonian paths and cycles are named after William Rowan Hamilton who
    invented the icosian game, now also known as Hamilton's puzzle, which
    involves finding a Hamiltonian cycle in the edge graph of the dodecahedron.
  }
\end{definition}

The greatest difference with Eulerian cycles is that it is not known whether
there is a fast (polynomial-time) algorithm that allows to find the Hamiltonian
cycles in a graph.\footnote{%
  Proving or disproving that there is a polynomial-time algorithm allowing to
  check whether a graph $G$ has a Hamiltonian path is one of the Millennium
  Problems. Clay Mathematics Institute offers a prize of \$1 million to a person
  who solves the problem.
}

It is easy to design an algorithm that checks whether a path exists in
$O((n - 1)!)$ by just brute forcing all the possible candidates for such a path.
However, using the ideas of the inclusion-exclusion principle, we may design
a much faster algorithm.
\begin{theorem}
\label{theorem:hamiltonian-algorithm}
  There is an algorithm with the running time $O(2^n n^3)$ such that it finds
  the number of Hamiltonian cycles in a graph $G = ([n], E)$.
\end{theorem}

Before we prove the theorem, recall that if $U$ and
$A_1, \dots, A_n \subseteq U$ are some finite sets, then
\[
  \left| \bigcap_{i = 1}^n A_i \right| =
  \sum_{X \subseteq \range{n}} (-1)^{|X|}
    \left| \bigcap_{i \in X} \overline{A}_i \right|,
\]
where $\overline{A}_i = U \setminus A_i$ and
$\bigcap_{i \in \emptyset} \overline{A}_i = U$.

\begin{proof}[Proof of Theorem~\ref{theorem:hamiltonian-algorithm}]
  As we mentioned before we the inclusion-exclusion principle to find the number
  of Hamilton cycles. Let $U$ be the set of all the cycles of length $n$
  (length is the number of edges in the path) going
  via the vertex $1$ and $A_v \subseteq U$ ($v \in \range{n})$ be the set of cycles of
  length $n$ going via the vertices $1$ and $v$.

  It is clear that the answer is $\left| \bigcap_{i = 1}^n A_i \right|$.
  Therefore it is enough to find all the cardinalities of
  $\left| \bigcap_{i \in X} \overline{A}_i \right|$. Note that
  $\bigcap_{i \in X} \overline{A}_i$ is equal to the set of all the cycles of
  length $n$ going via the vertex $1$ in $G - X$. We denote the cardinality of
  this set by $C_X$.

  To find the value of $C_X$ we use the following notation.
  Let $E_X$ be the set of edges in $G - X$ and let $T_X(d, x)$ be
  the number of length $d$ paths from $1$ to $x \in \range{n} \setminus X$ in $G - X$.
  Clearly $T_X(0, x) = 1$ if $x = 1$ and $T_X(0, x) = 0$ otherwise. In addition,
  $T_X(d + 1, x) = \sum_{y ~:~ (y, x) \in E_X} T_X(d, y)$. Therefore, we may
  compute $T_X(n, x)$ for all $x \in \range{n} \setminus X$ in $n^3$ steps. As a
  result, we may find the value of $\sum_{X \subseteq \range{n}} (-1)^{|X|} C_X$ in
  $2^n n^3$ steps.
\end{proof}

However, one may prove that if all the vertices in a graph have large degree,
then the graph has a Hamiltonian cycle.
\begin{theorem}[Dirac]
  Let $G$ be a graph on $n \ge 3$ vertices. If every vertex $v$ in $G$ has
  degree at least $n / 2$, then there is a Hamiltonian cycle in $G$.
\end{theorem}
\begin{proof}
  For the sake of contradiction, let us assume that $G$ has no Hamiltonian
  cycle but all the vertices have degree at least degree $n / 2$, where $n$ is
  the number of vertices in $G$.

  Let us start adding edges to $G$ as long as we are not creating a
  Hamiltonian cycle. When we stop we get a graph $H = (V, E)$ such that
  all the vertices of $H$ have degree at least $n / 2$, $H$ does
  not have a Hamiltonian cycle, but adding any new edge would create a
  Hamiltonian cycle.

  Consider any two vertices $x$ and $y$ that are not connected by
  an edge. We know that in the graph $H + (x, y)$ there is
  a Hamiltonian cycle $x = v_1$, \dots, $v_n = y$. Note that
  $|\set[{(x, v) \in E \text{ or } (y, v) \in E}]{v \in V}| \ge n$
  since $\deg_H(x) \ge n / 2$ and $\deg_H(y) \ge n / 2$.
  Therefore by the pigeonhole principle, there is $2 \le i \le n - 1$
  such that $(x, v_i) \in E$ and $(v_{i - 1}, y) \in E$. As a result,
  $x$, $v_2$, \dots, $v_{i - 1}$, $y$, $v_{n - 1}$, \dots, $v_i$
  is a Hamiltonian cycle in $H$.
\end{proof}

There are plenty of different applications of Hamiltonian paths. Here we describe
the one that comes from bioinformatics.

Imagine that we want to read a DNA strand, i.e., determine the order in which
nucleotides occur on a strand of DNA. One of the methods, called
``Sequencing by Hybridization'', is based on Hamiltonian paths.

The method works as follows.
\begin{itemize}
  \item Attach all possible DNA probes of length $k$ to a flat surface, each
    probe at a distinct and known location. This set of probes is called
    the DNA microarray.
  \item Apply a solution containing fluorescently labeled copies of a DNA
    fragment to the array.
  \item The DNA fragment hybridizes with those probes that are
    complementary to substrings of length $k$ of the fragment.
  \item Using a spectroscopic detector, determine which probes hybridize to
    the DNA fragment to obtain the $k$-mer composition of the DNA fragment.
  \item Reconstruct the sequence of the DNA fragment from the $k$-mer
    composition.
\end{itemize}

In other words, we need to reconstruct a string $s$ from all $n - k + 1$
substrings of length $k$; e.g, we need to reconstruct the string TATGGTGC
from the strings ATG, GGT, GTG, TAT, TGC, TGG (in this example $k = 3$).
(Note that different strings may have the same sets of substrings.
Strings GTATCT and GTCTAT correspond to the strings
AT, CT, GT, TA, TC when $k = 2$.)

By a given set $p_1$, \dots, $p_\ell$ of strings ($k$-mers) of length $k$
we construct the following graph. There are $\ell$ vertices corresponding to
the strings $p_1$, \dots, $p_\ell$; there is an edge between $p_i$ and $p_j$
whenever the same string of length $k - 1$ is a suffix of $p_i$ and a
prefix of $p_j$ (for example, TG is a suffix of ATG and a prefix of TGG).
It is easy to see that we can find a string corresponding to
$p_1$, \dots, $p_\ell$ if we have a Hamiltonian path in the graph.

\begin{chapterendexercises}
  \exercise Is it true that if a graph has a closed Eulerian walk, then it has
    an even number of edges?
  \exercise Let $G$ be a graph such that every vertex has degree $4$. Show that
    it is possible to color edges of the graph into two colors, red and blue,
    such that every vertex has exactly two red edges and two blue edges.
    \begin{solution}
      Without loss of generality the graph is connected. Since all the vertices
      of $G$ have even degree, there is an Euler cycle $e_1$, \dots, $e_m$ in
      $G$. We color $e_{2k}$ in red and $e_{2k + 1}$ in blue, for all possible
      $k$. Note that for every vertex $v$ (except the first in the path) there
      are $j_1 \neq j_2$ such that $e_{j_1}$, $e_{j_1 + 1}$, $e_{j_2}$, $e_{j_2
      + 1}$ are going from $v$. Hence, $v$ has exactly two red edges and two
      blue edges. As about the first vertex, note that number of edges in the
      graph is equal to $m = 2n$, hence, $e_m$ is colored on red and $e_1$ is
      colored in blue. Additionally, there is $j_1$ sucht that $e_{j}$ and $e_{j
      + 1}$ are going from the first vertex. Hence, the first vertex has exactly
      two red edges and two blue edges.
    \end{solution}
  \exercise[recommended] Let $G$ be a graph such that there are only $2$
    vertices with odd degree. Prove that they belong to the same connected
    component.
  \exercise Let $G = (V, E)$ be a connected graph and $c : V \to \set{0, 1}$
    be a function.
    \begin{enumerate}
      \item Assume that $\sum_{v \in V} c(v)$ is odd.
        Show that for any $s : E \to \set{0, 1}$, there is a vertex
        $v \in V$ such that $\sum_{(u, v) \in E} s(u, v)$ and $c(v)$
        have different remainders modulo $2$.
      \item Assume that $\sum_{v \in V} c(v)$ is even.
        Show that there is a function $s : E \to \set{0, 1}$
        such that $\sum_{(u, v) \in E} s(u, v)$ is odd iff $c(v)$ is odd
        for all $v \in V$.
    \end{enumerate}
  \exercise Find a minimal $k(n)$ such that for any graph $G$ on $n$ vertices,
    if $G$ has at least $k(n)$ edges, then $G$ is connected.
    \begin{solution}
      Let $G$ be some graph. The graph $G$ has at most $\binom{n}{2}$ many edges
      by \Cref{exercise:maximal-number-of-edges}.

      Let us assume that $G$ is not connected i.e. we may split $G$ into two
      disjoint subgraphs $G_1$ and $G_2$ of $G$ such that vertices of $G_1$ are
      not connected with vertices of $G_2$. Let $n_1$ and $n_2$ denote the
      number of vertices in $G_1$ and $G_2$ respectively. Note that $G_i$ has at
      most $\binom{n_i}{2}$ edges for every $i \in \range{2}$. Hence, $G$ has at
      most $\binom{n_1}{2} + \binom{n_2}{2}$ edges.

      However, $f(n_1) = \binom{n_1}{2} + \binom{n - n_1}{2}$ reaches the minimum for
      $1 \le n_1 \le n - 1$ at $n_1 = 1$. As a result, $k(n) \le \binom{n - 1}{2} + 1$.

      Additionally, note that there is a not connected graph $G$ with $\binom{n
      - 1}{2}$ edges (it is a complete graph on $n - 1$ vertices and one
      additional vertex). Thus $k(n) = \binom{n-1}{2} + 1$.
    \end{solution}
  \exercise Show that the following graph has no Hamiltonian cycle.
    \begin{center}
    	\begin{tikzpicture}[scale=0.5, thick]
		    % Draw a 7,11 network
    		% First we draw the vertices
		    \foreach \pos/\name in {{(0,0)/1}, {(3,0)/2}, {(5,0)/3},
        	                    {(7,0)/4}, {(10,0)/5}, {(4,2)/6}, {(6,2)/7},
			  	    {(5,4)/8}, {(4,-2)/9}, {(6,-2)/10}, {(5,-4)/11}}
			    \node[circle, draw, inner sep=0pt, minimum size=6pt] (\name) at \pos {};
    		\foreach \source/ \dest in {1/2, 4/5, 2/6, 2/9, 3/6, 3/9, 3/7,
		  		    3/10, 4/7, 4/10, 8/6, 8/7, 11/9, 11/10, 1/8, 1/11, 5/8, 5/11}
			    \draw (\source) -- (\dest);
        \end{tikzpicture}
  	\end{center}
    \begin{solution}
      Note that we can color all the vertices in black and yellow so that
      vertices of the same color are not adjacent.
      \begin{center}
        \begin{tikzpicture}[scale=0.5, auto,swap]
        	% Draw a 7,11 network
        	% First we draw the vertices
      	  \foreach \pos/\name in {{(3,0)/2}, {(5,0)/3},	{(7,0)/4},
              {(5,4)/8}, {(5,-4)/11}}
            \node[circle, fill=black, draw, inner sep=0pt, minimum size=6pt]
              (\name) at \pos {};
          \foreach \pos/\name in {{(0,0)/1}, {(10,0)/5}, {(4,2)/6},
              {(6,2)/7}, {(4,-2)/9}, {(6,-2)/10}}
      		  \node[circle, fill=yellow, draw, inner sep=0pt, minimum size=6pt]
              (\name) at \pos {};
      	  \foreach \source/ \dest in {1/2, 4/5, 2/6, 2/9, 3/6, 3/9, 3/7,
              3/10, 4/7, 4/10, 8/6, 8/7, 11/9, 11/10, 1/8, 1/11, 5/8, 5/11}
      		  \draw (\source) -- (\dest);
        \end{tikzpicture}
      \end{center}
      Note that there are $11$ vertices in total, there are $6$ yellow vertices
      and $5$ black. Assume that there is a Hamiltonian path $v_1$, \dots,
      $v_11$. Without loss of generality $v_1$ is yellow. This implies that
      $v_1$, $v_3$, $v_5$, $v_7$, $v_9$, and $v_11$ are also yellow since
      vertices of the same color are not adjacent. However, it also mean that
      $v_1$ and $v_11$ are adjacent and of the same color, which is a contradiction.
    \end{solution}
\end{chapterendexercises}