%!TEX root = morusAC.tex

\section{Symmetric Linear Combinations and \MiniMORUS}
\label{sec/introminimorus}

To simplify the description of the attack, we assume all plaintext blocks are zero. This assumption will be removed in \Cref{subsec:variable}, where we will show that plaintext bits only contribute linearly to the trail. Recall that the inner state of the cipher consists of five $4w$-bit registers $S_0,\dots,S_4$, each containing four $w$-bit words.

\subsection{Symmetric Linear Combinations}

We begin with a few observations about the \StateUpdate{} function. Besides XOR and AND operations, the \StateUpdate{} function uses two types of bit rotations:
\begin{enumerate}
\item \emph{word-wise} rotations perform a circular shift on each word within a register;
\item \emph{register-wise} rotations perform a circular shift on a whole register.
\end{enumerate}
The second type of rotation always shifts registers by a multiple of the word size $w$. This amounts to a (circular) permutation of the words within the register: for example, if a register contains the words $(A, B, C, D)$, and a register-wise rotation by $w$ bits to the left is performed, then the register now contains the words $(B, C, D, A)$.

To build our linear trail, we start with a linear combinations of bits within a single register.
\begin{definition}
Recall that $w$ denotes the word size in bits, and $4w$ is the size of a register. A linear combination of the form:
\[
S^t_{i,j(0)} \oplus S^t_{i,j(1)} \oplus \dots \oplus S^t_{i,j(k)}
\]
is said to be \emph{symmetric} iff the set of bits $S^t_{i,j(0)}, \dots, S^t_{i,j(k)}$ is left invariant by a circular shift by $w$ bits; that is, iff:
\[
\{j(i) : i\leq k\} = \{j(i) + w \text{\rm{} mod } 4w : i\leq k\}.
\]
\end{definition}
\emph{Example.} The following linear combination is symmetric for \MORUS[640], i.e. $w = 32$:
\begin{equation}
S^t_{0,0} \oplus S^t_{0,32} \oplus S^t_{0,64} \oplus S^t_{0,96}.
\label{eq:symmetric}
\end{equation}

This definition naturally extends to a linear combination across multiple registers, and also across ciphertext blocks.
The value of such a linear combination is unaffected by register-wise rotations, since those rotations always shift registers by a multiple of the word size.
On the other hand, since word-wise rotations always shift all four words within a register by the same amount, word-wise rotations preserve the symmetry property. Moreover, the XOR of two symmetric linear combinations is also symmetric.%; and the same holds for the AND operation (if we extend the symmetric property to non-linear combinations in the natural way).

This naturally leads to the idea of building a linear trail using only symmetric linear combinations, which is what we are going to do. As a result, the effect of register-wise rotations can be ignored. Moreover, since all linear combinations we consider are going to be symmetric, they can be described by truncating the linear combination to the first word of a register. Indeed, an equivalent way of saying a linear combination is symmetric, is that it involves the same bits in each word within a register. For example, in the case of (\ref{eq:symmetric}) above, the four bits involved are the first bit of each of the four words.

\subsection{\MiniMORUS}

In fact, we can go further and consider a reduced version of \MORUS where each register contains a single word instead of four. The \StateUpdate{} function is unchanged, except for the fact that register-wise rotations are removed: see \Cref{fig:minimorus}. We call these reduced versions \MiniMORUS[640] and \MiniMORUS[1280], for \MORUS[640] and \MORUS[1280] respectively. Since registers in \MiniMORUS contain a single word, word-wise and register-wise rotations are the same operation; for simplicity we write $\lll$ for word-wise rotations.

Since the trail we are building is relatively complex, we will first describe it on \MiniMORUS. We will then extend it to the full \MORUS via the previous symmetry.

\begin{figure}[h]
  \substatesfalse
  % \substatesfalse to label state words and/or masks
  \centering
  \tikzsetnextfilename{minimorus_plain}
  \begin{tikzpicture}[xscale=1.0,yscale=1.5]%{{{
    \printstate
  \end{tikzpicture}%}}}
  \caption{\MiniMORUS state update function.}
  \label{fig:minimorus}
\end{figure}


\documentclass{article}
\usepackage{amsmath}
\title{Numerical simulation of PWL models of gene regulatory networks}
\author{V. Acary}
\begin{document}
\maketitle
\subsection*{Standard form}

\begin{equation}
  \label{eq:std}
  \left\{\begin{array}{l}
      M \dot x  = f(x,t) +  r(t) \\
      r(t) = g(x,t,\lambda) \text{ (input) }  \\
      y(t) = h(x,t,\lambda) \text{ (output) } \\
   \end{array}\right.
\end{equation}

with one of the following nonsmooth law
\begin{enumerate}
\item Sign function
  \begin{equation}
    \label{eq:sign}
    \lambda \in \mbox{Sign} (-y) \Longleftrightarrow  - y  \in N_{[-1,1]}(\lambda)
  \end{equation}
\item Step function
  \begin{equation}
    \label{eq:step}
    \lambda \in \mbox{Step} (-y) \Longleftrightarrow  - y  \in N_{[0,1]}(\lambda)
  \end{equation}
\end{enumerate}

We have
\begin{equation}
  \label{eq:signstep}
  \begin{array}{l}
  s^+(x,\theta) = \mbox{Step}(x-\theta) = \frac 1 2 (1+ \mbox{Sign}(x-\theta))\\ \\
  s^-(x,\theta) = 1 - \mbox{Step}(x-\theta) = \frac 1 2 (1- \mbox{Sign}(x-\theta))
\end{array}
\end{equation}
\subsection*{First 2D example}
\begin{equation*}
\left\{\begin{array}{l} \dot{x}_{a} = - \gamma_{a}\, x_{a}+ \kappa_{a} \, s^{+}(x_{b},\theta_{b}^1)\, s^{-}(x_{a},\theta_{a}^2) \\
\dot{x}_{b} = - \gamma_{b}\, x_{b} + \kappa_{b} \,  s^{+}(x_{a},\theta_{a}^1)\,s^{-}(x_{b},\theta_{b}^2) \end{array}\right. .
\end{equation*}

The parameters are: $\theta_a^1=\theta_b^1=4, \theta_a^2=\theta_b^2=8, k_a=k_b=40, \gamma_a=4.5$ and $\gamma_b=1.5$.

The vector fields $f$ and $h$ can be identified as

\begin{equation}
  \label{eq:f1}
  f(x,t) = \left[
    \begin{array}{l}
      - \gamma_a x_a \\
      - \gamma_b x_b \\
    \end{array}
  \right]=\left[
    \begin{array}{l}
      - 4.5 x_a \\
      - 1.5 x_b \\
    \end{array}
  \right]
\end{equation}

\begin{equation}
  \label{eq:h1}
  h(x,t,\lambda) = -  \left[
    \begin{array}{l}
       x_a - \theta_a^1 \\
       x_b - \theta_b^1 \\
       x_a - \theta_a^2 \\
       x_b - \theta_b^2 \\
    \end{array}
  \right]=\left[
    \begin{array}{l}
      4- x_a \\
      4- x_b \\
      8- x_a \\
      8- x_b \\
    \end{array}
  \right]
\end{equation}

\subsubsection*{With the sign function}
\begin{equation}
  \label{eq:g1sign}
  \begin{array}{lcl}
  g(x,t,\lambda) &=&\displaystyle\frac 1 4  \left[
    \begin{array}{l}
      \kappa_a      (1- \mbox{sign}( x_a - \theta_a^2))((1+ \mbox{sign}( x_b - \theta_b^1)))\\
      \kappa_b      (1+ \mbox{sign}( x_a - \theta_a^1))((1- \mbox{sign}( x_b - \theta_b^2)))
    \end{array}
  \right]\\ \\
  &=& 10   \left[
    \begin{array}{l}
          (1-\lambda_3)(1+\lambda_2)\\
          (1+\lambda_1)(1-\lambda_4)
    \end{array}
  \right]
\end{array}
\end{equation}

\subsubsection*{With the Step function}
\begin{equation}
  \label{eq:g1step}
  \begin{array}{lcl}
  g(x,t,\lambda) &=&  \left[
    \begin{array}{l}
      \kappa_a      (1- \mbox{step}( x_a - \theta_a^2))( \mbox{step}( x_b - \theta_b^1)))\\
      \kappa_b      (\mbox{step}( x_a - \theta_a^1))((1- \mbox{step}( x_b - \theta_b^2)))
    \end{array}
  \right]\\ \\
  &=& 40   \left[
    \begin{array}{l}
          \lambda_2 (1-\lambda_3)\\
          \lambda_1 (1-\lambda_4)
    \end{array}
  \right]
\end{array}
\end{equation}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\chapter{Heuristic Strategies}

\section{Overview}

[motivation]

Heuristic strategies are interesting to study because they tend to uncover some interesting links between an optimal strategy and the immediate step. Good heuristics have an intuitive rationale as why the heuristic is constructed that way.

In this sense, the objective of a heuristic may not only be to yield an optimal solution quickly. [See e.g. neuwirth].

It is fair to expect that a tailored-heuristic to one configuration of the rules may not perform well in another configuration. This is the defect of tailored heuristics. (Show some examples)

See \cite{pepperdine10} for a list of heuristic functions.

[tie-breaker policy]

All known heuristic strategies work on the partition of the remaining secrets by a candidate guess. Notations: Let $S$ be the set of remaining possibilities. Let $\tilde{S}$ be the set of remaining possibilities after the guess, which is a random variable depending on the guess and the secret. Let $n = \#\{S\}$ be the number of remaining possibilities, and let $\tilde{n} = \#\{\tilde{S}\}$ be the number of remaining possibilities after a guess. Let a guess partition the remaining possibilities into $r$ cells $V_1, \ldots, V_r$, where the number of secrets in cell $i$ is $n_i = \#\{V_i\}$. Let $x$ be the unknown secret.

\section{The Min-Max heuristic}

Knuth \cite{knuth76} published the first systematic paper on Mastermind, where he introduced a heuristic strategy aiming at minimizing the worst-case number of remaining possibilities.

The same heuristic function first appeared in \cite{aleph71} for the Bulls and cows game, but was not elaborated.

Rationale: fewer remaining possibilities is better. We want to play safe and reduce the worst-case number of remaining possibilities.

Note that this strategy does not need an assumption on the distribution of the secret. [see neu] (This, e.g. is suitable in mastermind with lie (or dynamic mastermind.))

Note: If two guesses yield the same worst-case partition, the second-to-worst partition size is compared, etc.

\section{The Min-Average heuristic}

Rationale: fewer remaining possibilities is better. We want to minimize the expected partition size (number of remaining possibilities).

The objective function to minimize is
\[
E[\tilde{n}] = \sum_{i=1}^r P(x \in V_i) \# \{ V_i \}
\]
Now we assume each remaining possibility is equally likely to be the secret. Then the above is transformed to
\[
E[\tilde{n}] = \sum_{i=1}^r \frac{n_i}{n} n_i = \frac{1}{n}  \sum_{i=1}^r n_i^2 .
\]
Minimizing this expectation is equivalent to minimizing the heuristic function
\[
h(P) = \sum_{i=1}^r n_i^2 .
\]

[Gini Index]

A correction for perfect match is useful. If the guess is among the remaining possibilities, then one of the responses is the perfect match, which accounts for a partition of size 1. However, we don't need to make any further guesses in this case. So this partition should be excluded when calculating the expected \emph{remaining} count. This can be corrected by applying the following correction term:
\[
-1 .
\]

\section{The Max-Entropy heuristic}

Rationale: we want a guess to provide as much information as possible as to determine what is the secret.

The entropy heuristic was first introduced by Neuwirth \cite{neuwirth81} for Mastermind and by Larmouth \cite{aleph71} for Bulls and cows. It is a theoretically advanced heuristic that scores a guess by the ``amount of information'' brought by its partitioning of the potential secret set. To be precise, this heuristic aims to maximize the \emph{entropy} of the partition, defined as
\[
H = -\sum_i \frac{n_i}{n} \log \frac{n_i}{n} .
\]
The base of the logarithm is not specified but it does not impact the choice.

Rearranging terms, it can be written as
\begin{align}
H
&= - \frac{1}{n} \left[ \sum_i n_i (\log n_i - \log n ) \right] \notag \\
&= - \frac{1}{n} \left( \sum_i n_i \log n_i \right) + \log n . \notag
\end{align}
When $n$ is fixed, maximizing $H$ is equivalent to minimizing the heuristic function
\[
h(P) = \sum_i n_i \log n_i .
\]
If we loosely interpret $\log n_i$ as an estimate of the number of further guesses needed for a partition of size $n_i$, then we can interpret the heuristic function (when divided by $n$) as an estimate of the expected number of further guesses needed.

[Note: floating point precision?]

Heeffer \cite{heeffer07} tested various heuristic algorithms on the Mastermind game with 5 pegs and 8 colors, and found the entropy heuristic to perform the best.

See \href{http://en.wikipedia.org/wiki/Entropy\_(information\_theory)\#Further\_properties}{Wikipedia}.

Each feedback from a guess can be thought of as a "alphabet"
For p4c10n, there are 14 alphabets, but some letters are more likely to follow certain letters than others. However, such likelyhood depends on the guess chosen.

For example, if a guess partitions the possibility set into discrete partition, then all letters in that alphabet

If alphabet is equally likely, then the entropy is maximized.

***************

Why entropy heuristic doesn't yield best (worst step) and (average step)?

The apparent underperformance of the entropy heuristic could be explained by noting the fact that there is a distinction between \emph{determining} the secret and \emph{revealing} the secret. Suppose we are left with 2 possibilities: 5678 and 7890. We can \emph{determine} the secret with one guess (e.g. 5678). However, to actually \emph{reveal} the secret, we need to make an extra guess (7890) if 5689 returns \fb{0}{2}. In total we need maximum 2 guesses and average 1.5 guesses.

However, from a information theory's perspective, the extra guess is totally redundant in that there is no uncertainty of the outcome: we know for sure that we will get \fb{4}{0} when we guess \cw{7890}. In fact, when we guess 5678, the entropy of the resulting partition \{(5678:4A0B),(7890:0A2B)\} is zero (ignoring the constant denominator), which means uncertainty removed.

The extra step to reveal the secret is necessary in the traditional human game because otherwise it's difficult to judge that the code breaker wins. On the other hand, the human game rules could be slightly modified to remove the need for the extra guess. Instead of required to \emph{reveal} the guess with a 4A0B feedback, the codebreaker is required to \emph{assert} the guess after a number of rounds. If the assertion is correct, he wins; if the assertion is wrong, he loses. The number of guesses one needs before making an assertion is equal to the number of steps one needs to determine the secret, and this number is consistent with an information-theory perspective.

To cope with subtle discrepancy of determining and revealing the secret, the entropy heuristic could be amended to distinguish the difference, though in this case the theory is not that sound. See [taiwan wang you] For example, Larmouth \cite{aleph71} used the following entropy heuristic \emph{with correction}:
\[
h'(P) = \sum_i n_i \log n_i - (2 \log 2) \delta(\fb{4}{0})  .
\]
The correction term applies when the partition contains \fb{4}{0}. However, how the coefficient $(2 \log 2)$ is derived is unclear.

Another issue with the entropy heuristic is that when computing the entropy, it only depends on the probablity of each partition (i.e. the size of each partition). This is because in entropy theory, it assumes that we know absolutely nothing about the underlying random variable (the secret), except which partition it resides in. Under this assumption, two partitions with the same size gives the same amount of information; for example, if partition A and partiton B both contain 3 possibilities, then if either case turns out to contain the secret, then we are equipped with the knowledge that we have 3 possibilities left, without any more knowledge.

However, in the scenario of Mastermind, the situation is different, because we have extra knowledge about the codewords apart from the size. Consider two partitions of the same size:

A = (1234, 1235, 1236), and

B = (1234, 1235, 2135)

Though both partitions contain the same number of elements, their information content is different; partition A has more uncertainty (in Mastermind sense) than partition B. This is because it requires at least 2 steps to determine the secret in A (this can be verified by an exhaustive search). However, partition B can be determined with one guess (for example, any one of the three secrets). Thus, with the extra knowledge not present in a vanilla entropy theory, partition A has more uncertainty.

Thus the assumptions of applying the entropy theory do not exactly hold. Consequenty, the strategy produced by entropy theory may not be as good as it apparently suggests. This may also mean that the logarithm base in computing the entropy may need to be adapted to the actual structure of the partition. However, if we continue this step recursively, then it is essentially an exhaustive strategy, in which case we don't need the heuristic any more.

\section{The Max-Parts Heuristic}

Rationale is problematic. Subject to choice of equal heuristic element. (We can perform a randomized test to permute the codeword list.)

The \maxpar{} heuristic was introduced by Kooi \cite{kooi05} who applied it to the Mastermind game and outperformed all other heuristic strategies. It proved to work well for a Mastermind game with 4 pegs and 6 colors, but didn't work well for one with 5 pegs and 8 colors.

The formula is
\[
h(P) = r ,
\]
where $r$ is the number of n(on-empty) partitions.

%\subsection{The Min-Steps Heuristic}

For Bulls and cows, the \maxpar{} strategy might have been tested in as early as 1969 by Ken Thompson \cite{ritchie01}. However, it is apparent that this strategy doesn't perform as well as the other heuristic strategies. See appendix [???] for an table.

\section{Other heuristics}

%http://mercury.webster.edu/aleshunas/Support\%20Materials/Analysis/Dowelll\%20-\%20Mastermind%20v2-0.doc

%Defeating Mastermind
%By Justin Dowell

%- WideDev
%- LongRect both hybrid strats

These are not good. Because we need a "rationale" for the heuristic.

A few more heuristics have been tested by various authors. They are listed below for completeness. However, the rationale for each of the heuristics is not clear, so their performance may be expected to vary widely with the configuration of the rules.

[Move this to appendix]

For a large class of heuristics, the heuristic function is computed as the expectation of some function, $f$ of the partition size, optionally minus a correction term, $\lambda$ if the guess is in the possibilities. That is,
\[
h(P) = \frac{1}{n} \left[\sum_i n_i f(n_i) - \tau(\lambda) \right].
\]

The following list of functions appear in \cite{pepperdine10}:
\begin{center}
\begin{tabular}{c l l}
\hline
Name & $f$ & $\lambda$ \\
\hline
Modified entropy & $\log (1+n_i)$ & 0 \\
Landy's function & $L(n_i)$ where $L(n)$ is the solution of $x^x = n$ & 0 \\
Exponential asymptote & $1-e^{-n_i}$ & $(1-e^{-1})$ \\
square root & $\sqrt{n_i}$ & 1 \\
Logarithmic integral & $\text{li}(1+n_i)$ & $2 \cdot \text{li}(3)$ \\
\hline
\end{tabular}
\end{center}
More examples can be found in his document.

In addition, some hybrid strategies are used.

\section{Comparison of heuristics}

When several candidate guesses yield the same heuristic value, a choice must be made as to pick which one as the guess. Standard way is to choose the ``first'' candidate as it appears in the list, or the lexicographically minima. However, some evidence (where??) shows that the performance of a heuristic does depend on which choice is made. This is not ideal.

\section{Building a strategy tree}

% -------------------------------- %

% -------------------------------- %


\documentclass[twoside]{article}

\usepackage{mystyle}
\usepackage{tikz}

\newcommand{\I}{\mathcal{I}}

\title{Introduction to Matroids}
\author{Travis Westura}
\date{\today}

\begin{document}
\maketitle
\thispagestyle{empty}

\section{Introduction}

In this course we've seen several examples of algorithms that are ``greedy.''
A greedy algorithm is an algorithm that makes a ``locally best'' decision.
For example, in Dijkstra's algorithm we use a priority queue to keep track of nodes on the friontier and pop them off using the priority of shortest path distance.
And in Kruskal's algorithm for finding minimum weight spanning trees, at each step we select the minimum weight edge that does not form a cycle.

We can generalize the idea of ``locally make a best decision'' by using matroids.
The word \emph{matroid} should make you think of the word \emph{matrix}, which you use in linear algebra and multivariable or vector calculus.
We'll begin by reviewing the concepts of linear independence, then we'll discuss analogous concepts in graph theory.
Then we will relate these concepts by generalizing them and giving the definition of a matroid.
And finally we will use matroids to give a proof of the correctness of Kruskal's algorithm.

\section{Independence in Linear Algebra}

As you take Linear Algebra and Multivariable Calculus you will gain lots of experience working with vectors and matrices.
In these notes we'll denote vectors using boldface letters at the end of the alphabet, such as $\bv{u}$ and $\bv{v}$, and matrices using capital letters at the beginning of the alphabet, such as $A$ and $B$.
We'll denote by $\bv{0}$ the vector of all $0$'s, and we'll also write vectors as columns and matrices as rectangles containing numbers:
\begin{equation*}
  \bv{u} = \mat{3\\-1\\3}, \quad \bv{v} = \mat{v_1\\v_2\\v_3}, \quad \bv{0} = \mat{0\\0\\0}, \quad A = \mat{1 & 2 & 5\\-2 & 3 & 0}, \quad B = \mat{b_{1,1} & b_{1, 2} & b_{1, 3}\\b_{2, 1} & b_{2, 2} & b_{2, 3}}.
\end{equation*}
We say that vectors are elements of a set called a Vector Space, and we also have the ability to add vectors and to multiply them by scalars.
In multivariable calculus the most common example of vector spaces are $\R^2$ and $\R^3$, where vectors consist of tuples of $2$ and $3$ numbers, respectively.
We can add vectors and multiply them by scalars as follows:
\begin{equation*}
  \mat{u_1\\u_2} + \mat{v_2\\v_2} = \mat{u_1 + v_1\\v_2 + v_2}, \quad a\mat{u_1\\u_2} = \mat{a u_1\\ a u_2}.
\end{equation*}
A linear combination of vectors is a sum of scalar multiples of vectors.
For example
\begin{equation*}
  2\mat{2\\-1} + 3\mat{-3\\4} = \mat{-5\\10}.
\end{equation*}
We could say that the $3$rd vector depends on the first two vectors, since it is a linear combination of them.

A set of vectors\footnote{
  We can also treat the columns of matrices as vectors and define a notion of linear independence on them, although we should be careful that a column may be repeated, whereas a vector in a set is not repeated.
}
$\{\bv{v}_1, \bv{v}_2, \ldots, \bv{v}_n\}$ is \emph{linearly independent} if the only scalars ${a_1, a_2, \ldots, a_n}$ that satisfy
\begin{equation*}
  a_1\bv{v}_1 + a_2\bv{v}_2 + \cdots + a_n\bv{v}_n = \bv{0}
\end{equation*}
are ${a_1 = a_2 = \cdots = a_n = 0}$.This means there is no way to write one of the vectors as a combination of the others, unless we make all the coefficients~$0$.
That is, none of the vectors in the set depend on the others.
For example, the standard basis vectors in $\R^3$ are linearly independent,
\begin{equation*}
  \left\{\mat{1\\0\\0}, \mat{0\\1\\0}, \mat{0\\0\\1}\right\},
\end{equation*}
as none of them can be written as a linear combination of the others.
The following set of vectors is linearly dependent:
\begin{equation*}
  \left\{\mat{1\\2}, \mat{3\\2}, \mat{5\\6}\right\}, \quad 2\mat{1\\2} + \mat{3\\2} = \mat{2 + 3\\4 + 2} = \mat{5\\6}.
\end{equation*}

There is a limit to the number of vectors that we can add to a set while maintaining independence.
For example, in~$\R^3$, and set of $4$~vectors is linearly dependent.
An independent set to which we can't add any more vectors is called a \emph{maximal} independent set, with the word maximal meaning we have included as many vectors as possible.\footnote{
  A maximal linearly independent set of vectors is called a \emph{basis}.
}

Further, if we have an independent set of vectors, we can remove vectors and still have an independent set.
For example, the subset of standard basis vectors,
\begin{equation*}
  \left\{\mat{1\\0\\0}, \mat{0\\1\\0}\right\},
\end{equation*}
is still independent in~$\R^3$.

\section{Independence in Graphs}

Now let's discuss independence in graphs.
Recall that a graph~${G = (V, E)}$ consists of a set~$V$ of vertices and a set~$E$ of edges.
We will consider only undirected graphs for now.

Recall that a \emph{tree} is a connected graph that does not have any cycles.
A \emph{forest} is a graph in which each connected component is a tree.
That is, a forest is essentially a collection of trees grouped together.
A \emph{spanning tree} of an undirected\footnote{ There is also a notion of spanning tree in an undirected graph, called an \emph{arborescence}.} graph~$G$ is a subgraph that is a tree and that includes all of the vertices of~$G$.
The ``span'' part of a spanning tree refers to the tree containing all of the vertices.
In the following diagram, from left to right we have: a tree that does not span the graph, a cycle, two spanning trees, and a forest consisting of two trees.
\begin{center}
  \begin{tikzpicture}[every node/.style={circle,thick,draw}]
    \node (A) at (0, 0) {A};
    \node (B) at (1.5, 0) {B};
    \node (C) at (0, -1.5) {C};
    \node (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] (B);
    \path[-] (A) edge[draw=black,very thick] (C);
    \path[-] (B) edge[draw=gray,thin] (D);
    \path[-] (C) edge[draw=black,very thick] (D);
    \path[-] (A) edge[draw=gray,thin] (D);
  \end{tikzpicture}
  \quad
  \begin{tikzpicture}[every node/.style={circle,thick,draw}]
    \node (A) at (0, 0) {A};
    \node (B) at (1.5, 0) {B};
    \node (C) at (0, -1.5) {C};
    \node (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] (B);
    \path[-] (A) edge[draw=black,very thick] (C);
    \path[-] (B) edge[draw=gray,thin] (D);
    \path[-] (C) edge[draw=black,very thick] (D);
    \path[-] (A) edge[draw=black,very thick] (D);
  \end{tikzpicture}
  \quad
  \begin{tikzpicture}[every node/.style={circle,thick,draw}]
    \node (A) at (0, 0) {A};
    \node (B) at (1.5, 0) {B};
    \node (C) at (0, -1.5) {C};
    \node (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] (B);
    \path[-] (A) edge[draw=black,very thick] (C);
    \path[-] (B) edge[draw=black,very thick] (D);
    \path[-] (C) edge[draw=black,very thick] (D);
    \path[-] (A) edge[draw=gray,thin] (D);
  \end{tikzpicture}
  \quad
  \begin{tikzpicture}[every node/.style={circle,thick,draw}]
    \node (A) at (0, 0) {A};
    \node (B) at (1.5, 0) {B};
    \node (C) at (0, -1.5) {C};
    \node (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] (B);
    \path[-] (A) edge[draw=black,very thick] (C);
    \path[-] (B) edge[draw=black,very thick] (D);
    \path[-] (C) edge[draw=gray,thin] (D);
    \path[-] (A) edge[draw=black,very thick] (D);
  \end{tikzpicture}
  \quad
  \begin{tikzpicture}[every node/.style={circle,thick,draw}]
    \node (A) at (0, 0) {A};
    \node (B) at (1.5, 0) {B};
    \node (C) at (0, -1.5) {C};
    \node (D) at (1.5, -1.5) {D};
    \node (E) at (3, 0) {E};
    \node (F) at (3, -1.5) {F};

    \path[-] (A) edge[draw=gray,thin] (B);
    \path[-] (A) edge[draw=black,very thick] (C);
    \path[-] (B) edge[draw=gray,thin] (D);
    \path[-] (C) edge[draw=gray,thin] (D);
    \path[-] (A) edge[draw=black,very thick] (D);
    \path[-] (B) edge[draw=black,very thick] (E);
    \path[-] (E) edge[draw=black,very thick] (F);
    \path[-] (D) edge[draw=gray,thin] (F);
  \end{tikzpicture}
\end{center}
The independent sets of a graph are the forests, that is, the subgraphs that do not include cycles.
The dependent sets are the subgraphs that include cycles.
To see the analogy to linear algebra, here note that if we have a forest that is not a spanning tree, then there is still an edge we can add to it to obtain a larger subggraph without creating a cycle.
A spanning tree gives us a notion of maximal independent set, meaning that we have included as many edges as we possibly can.

And given a forest, removing edges does not create a cycle.
Thus we have the same property that removing elements from an independent set preserves independence.

\section{Definition of Matroids}

Now that we have some intuition about the concept of ``independence,'' let's give a definition of matroids.\footnote{
  We could define matroids in many ways.
  Matroids are referred to as \emph{cryptomorphic}, which means they have many equivalent but ostensibly unrelated definitions.}
Many mathematical definitions involve placing a set inside of parentheses with other sets or functions that further describe that set.
For example, a graph is given by two sets ${G = (V, E)}$.
A vector space is given by ${(V, k, +, \cdot)}$, where $V$ is a set of vectors, $k$ is a field of scalars, $+$ is an addition of vectors, and $\cdot$ is a multiplication between a scalar and a vector.
A familiar example is ${(\R^3, \R, +, \cdot)}$.
If you have taken CS~$2800$ or other more advanced courses, you will have seen groups, where are often described by writing ${(G, +)}$, where $G$ is a set and $+$ is a binary operation on that set.
Matroids have a similar definition, where we have a set that we write in parentheses together with some other information describing that set.
In this definition the additional information is a collection of subsets.

\begin{defn}[Matroid]
  A \emph{matroid}~$M$ is a pair~$(E, \I)$ consisting of a finite set~$E$ and a collection~$\I$ of subsets of~$E$, called the \emph{independent sets}, satisfying the following properties:
  \begin{enumerate}
    \item The collection is nonempty.
      That is, ${\I \ne \varnothing}$.
    \item All subsets of an independent set are independent.
      That is, if ${A \in \I}$ and ${B \subseteq A}$, then ${B \in \I}$.
    \item If $A$ and $B$ are independent sets and $A$ is larger than $B$, then we can take an element that is in $A$ but not in $B$ and add it to $B$ to construct an independent set.
    That is, if ${A, B \in \I}$ with ${|A| > |B|}$, then there exists ${x \in A \setminus B}$ such that ${B \cup \{x\} \in \I}$.
    This property is knows as the \emph{independent set exchange property}---we can ``exchange'' an element from one set to another.
  \end{enumerate}
  A subset of~$E$ that is not independent is called \emph{dependent}.
\end{defn}

Let's see how this definition fits in with our first two examples.
Recalling that a vector space has a set of vectors~$V$, we form a matroid~$(V, \I)$, where $\I$ is the collection of independent sets of vectors of~$V$.
Recall what we noted previously: given a set of independent vectors, removing vectors still results in an independent set.
And if there are two independent sets of vectors ${A = \{\bv{a}_1, \ldots, \bv{a}_n\}}$ and ${B = \{\bv{b}_1, \ldots, \bv{b}_m\}}$ with ${m < n}$, then clearly $B$ is not a maximal independent set, so we can add more vectors to it while maintaining independence.\footnote{
  Being formal about explicitly satisfying the thrid property requires slightly more linear algebra than I want to use right now, so I'll leave this verification as an exercise for those interested.
}

Next, given an undirected graph ${G = (V, E)}$, we form a matroid~$(E, \I)$ with the ground set given by the graph's edges and the independent sets given by the graph's forests.
Let's verify each part of the definition.
\begin{enumerate}
  \item The empty collection of edges forms a forest vacuously.
  \item If a graph does not form a cycle, then removing edges cannot yield a cycle.
  \item Consider two forests $A$ and $B$ where $A$ has more edges than $B$.
    Then there is some edge ${e \in A}$ such that ${B \cup \{e\}}$ does not have a cycle.
    The number of connected components in a graph ${(V, F)}$ is given by ${|V| - |F|}$.
    Note that individual vertices form their own component.
    If $A$ and $B$ are two forests with ${|A| > |B|}$, then there are more connected components in ${(V, B)}$ than in ${(V, A)}$.
    There there is some edge~$e$ in~$A$ that connects two of the components in~$B$.
    Thus ${B \cup \{e\}}$ forms a forest with one more edge than~$B$.
\end{enumerate}

\section{Kruskal's Algorithm and Matroid Optimization}

Given a weighted undirected graph, we construct a spanning tree by beginning by starting with the empty set and repeatedly adding the minimum weight edge that does not form a cycle.
That is, at each step, add the minimum weight edge that still produces an independent set.
The algorithm terminates when we have a spanning tree, or in the language of matroids, when we have a maximal independent set.

In the following example we start with the empty set of edges.
The minimum weight edge is between~$A$ and~$C$, the we first choose that edge.
Next we select the edge between~$A$ and~$D$.
We then can't select the edge between~$C$ and~$D$, as that edge would form a cycle.
Thus we select the edge between~$B$ and~$D$.
At this point there are no other edges we can add without creating a cycle, so we have a spanning tree and terminate out algorithm here.
\begin{center}
  \begin{tikzpicture}
    \node[shape=circle,draw=black,thick] (A) at (0, 0) {A};
    \node[shape=circle,draw=black,thick] (B) at (1.5, 0) {B};
    \node[shape=circle,draw=black,thick] (C) at (0, -1.5) {C};
    \node[shape=circle,draw=black,thick] (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] node[above] {5} (B);
    \path[-] (A) edge[draw=gray,thin] node[left] {1} (C);
    \path[-] (B) edge[draw=gray,thin] node[right] {4} (D);
    \path[-] (C) edge[draw=gray,thin] node[below] {3} (D);
    \path[-] (A) edge[draw=gray,thin] node[above right] {2} (D);
  \end{tikzpicture}
  \quad
  \begin{tikzpicture}
    \node[shape=circle,draw=black,thick] (A) at (0, 0) {A};
    \node[shape=circle,draw=black,thick] (B) at (1.5, 0) {B};
    \node[shape=circle,draw=black,thick] (C) at (0, -1.5) {C};
    \node[shape=circle,draw=black,thick] (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] node[above] {5} (B);
    \path[-] (A) edge[draw=black,very thick] node[left] {1} (C);
    \path[-] (B) edge[draw=gray,thin] node[right] {4} (D);
    \path[-] (C) edge[draw=gray,thin] node[below] {3} (D);
    \path[-] (A) edge[draw=gray,thin] node[above right] {2} (D);
  \end{tikzpicture}
  \quad
  \begin{tikzpicture}
    \node[shape=circle,draw=black,thick] (A) at (0, 0) {A};
    \node[shape=circle,draw=black,thick] (B) at (1.5, 0) {B};
    \node[shape=circle,draw=black,thick] (C) at (0, -1.5) {C};
    \node[shape=circle,draw=black,thick] (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] node[above] {5} (B);
    \path[-] (A) edge[draw=black,very thick] node[left] {1} (C);
    \path[-] (B) edge[draw=gray,thin] node[right] {4} (D);
    \path[-] (C) edge[draw=gray,thin] node[below] {3} (D);
    \path[-] (A) edge[draw=black,very thick] node[above right] {2} (D);
  \end{tikzpicture}
  \quad
  \begin{tikzpicture}
    \node[shape=circle,draw=black,thick] (A) at (0, 0) {A};
    \node[shape=circle,draw=black,thick] (B) at (1.5, 0) {B};
    \node[shape=circle,draw=black,thick] (C) at (0, -1.5) {C};
    \node[shape=circle,draw=black,thick] (D) at (1.5, -1.5) {D};

    \path[-] (A) edge[draw=gray,thin] node[above] {5} (B);
    \path[-] (A) edge[draw=black,very thick] node[left] {1} (C);
    \path[-] (B) edge[draw=black,very thick] node[right] {4} (D);
    \path[-] (C) edge[draw=gray,thin] node[below] {3} (D);
    \path[-] (A) edge[draw=black,very thick] node[above right] {2} (D);
  \end{tikzpicture}
\end{center}

We are interested in proving the correctness of this abstract algorithm.
For now we don't consider the implementation details.\footnote{
  For the details of an efficient implementation, look up the Union-Find data structure.
}
Kruskal's spanning tree algorithm is actually a case of the more general greedy algorithm that is used for weighted matroids.
A \emph{weighted matroid} is a matroid in which every element of the ground set has a nonnegative weight.
That is, a weighted matroid~${(E, \I)}$ has a weight~${w_e \ge 0}$ for each edge~${e \in E}$.
Finding a minimum weight spanning tree then becomes the problem of finding a minmum weight maximal independent set.\footnote{
  To use more terminalogy from linear algebra, a maximal independent set is called a \emph{base}.
  All maximal independent sets contain the same number of elements, and this number is called the \emph{rank} of the matroid.
}

In this general case the greedy algorithm works the same way.
Begin with the empty set.
Repeated select the minimum weight element of~$E$ that maintains independence.
Terminate when the only remaining elements of~$E$ would form a dependent set if added.

\begin{thm}[]
  Let ${(E, \I)}$ be a matroid with weights ${w_e \ge 0}$ for each~${e \in E}$.
  Then the greedy algorithm produces a maximal independent set of minimum weight.
\end{thm}
\begin{proof}
  Suppose that the greedy algorithm selects elements ${[e_1, e_2, \ldots , e_r]}$, in that order.
  These elements form an independent set, and their weights are in ascending order: ${w_{e_1} \le w_{e_2} \le \cdots \le w_{e_r}}$.
  To see this set is of minimum weight, suppose there is another maximal independent set consisting of ${[x_1, \ldots, x_r]}$ of lower weight.
  Then for some $i$ we must have ${w_{x_i} \le w_{e_i}}$.
  Use the least such $i$, which we note must be greater than~$1$, since the greedy algorithm picks the min cost edge at the first step.
  Now consider ${B = \{e_1, \ldots, e_{i-1}\}}$ and ${A = \{x_1, \ldots, x_i\}}$.
  Since ${|A| > |B|}$ and both sets are independent, we use the exchange property.
  There is some $j$ such that ${x_j \in A \setminus B}$ and ${B \cup \{x_j\}}$ is independent.
  But since ${w_{x_j} \le w_{x_i} < w_{e_i}}}$, we have a contradiction, as the greedy algorithm would have picked $x_j$ before $e_i$.
\end{proof}

There is also a converse to the previous result: if an independence system ${(E, \I)}$ is not a matroid, then the greedy algorithm fails for some choice of integer weights.
You'll learn more about Greedy Algorithms if you take the undergraduate algorithms course CS~$4820$, and you'll learn more about matroids if you take the graduate algorithms course CS~$6820$ or the combinatorics course sequence~Math~$4410$ and~$4420$.

\end{document}


\subsection{Asymmetric encryption}

Here we use different keys to encrypt and decrypt the file.

Consider two users who wish to send a message securely.

One option would be to use symmetric encrpytion. They would have to meet and share this key securely, however, as transferring it over an insecure network would mean it could be copied.

With public key encryption each user has a public and a private key. The private key is kept secure locally, while the public key can be broadcasted.

In order to encrypt the file, the recipient's public key is used, while both the private and public key are needed to decrypt the file.

As a result anyone can encrypt a file to send to the user, but only the user can read what is sent.

Public-key encryption can be used to facilitate symmetric encryption. If only one party has a public key then the other user can send a symmetric key securely using the public key.

Using this, asymmetric encrpyiton is only used at the start.

This is how HTTPS operates, where the website has a public key, but the client does not.

Each user still needs to trust that the public key is accurate. This could be done by hosting the public key on a secure location.

RSA is an algorithm used for public-key encrpytion, including for HTTPS handshakes and PGP.


\begin{lem}
% \emph{
If $A$ and $B$ are the adjacency matrices of simple graphs that are isomorphic to one another, then the minimum of rQAP is equal to the minimum of QAP.
\end{lem}

\begin{proof}
Because any feasible solution to GM is also a feasible solution to rQAP, we must only show that the optimal objective function value to rQAP can be no better than the optimal objective function value of QAP.  Let $A=PBP\T$, so that $\langle A, PBP\T\rangle=2m$, where $m$ is the number of edges in $A$.  If rQAP could achieve a lower objective value, then it must be that there exists a $D \in \mc{D}$ such that $\langle A, DBD\T\rangle > \langle A, PBP\T\rangle = 2m$ (remember that we are minimizing the negative Euclidean inner product). For that to be the case, it must be that $(DBD\T)_{uv} \geq 1$ for some $(u,v)$.  That this is not so may be seen by the submultiplicativity of the norm induced by the $\ell_{\infty}$ norm:
$\norm{Dx}_\infty \leq \norm{D}_{\infty,\infty} \norm{x}_\infty$.  Applying this twice (once for each doubly stochastic matrix multiplication) yields our result.
% Consider $d_i=\langle D, \text{col}_i(BD\T) \rangle$, where $\text{col}_i(\cdot)$ indicates the $i^{th}$ column of the matrix.  $d_i \leq 1$ for all $i \in [n]$, therefore, our result holds.
\end{proof}

\paragraph{Linear Assignment Problems} % (fold)
% \label{ssub:linear_assignment_problems}

% subsubsection linear_assignment_problems (end)

The standard way of writing a Linear Assignment Problem (LAP) is
\begin{equation*}
% \text{(LAP)} \qquad
\begin{array}{cl}
			\text{minimize}   & \sum_{u,v \in [n]} a_{u \pi(v)} b_{uv} \\
			\text{subject to}  &P \in \mc{P}.
\end{array} %\label{eq:LAP}
\end{equation*}
The LAP objective function, like the QAP objective function, enjoys a number of equivalent formulations, including
% , similar to the QAP objective function (cf Eq. \eqref{eq:equiv})
% \begin{multline} \label{eq:equiv}
% (a_{uv}-b_{\pi(u)\pi(v)})^2 = \norm{A - PBP\T}_F^2 \\ = tr \{ (A - PBP\T)\T (A - PBP\T)\}.
% \end{multline}
%
%
% which can be written equivalently in a number of ways using the notion of permutation matrix introduced in the main text, including
\begin{equation}
\text{(LAP)} \qquad
\begin{array}{cl}
			\text{minimize}   & \langle P, AB\T \rangle \\
			\text{subject to}  &P \in \mc{P}.
\end{array}\label{eq:LAP}
\end{equation}
%
% \begin{subequations} \label{eq:LAP2}
% \begin{align}
% 	&\argmin_{\PmcP} \norm{PA - B}_F =\\
% 	&\argmin_{\PmcP} \, tr(PA-B)\T (PA-B)=\\
% 	% &\argmin_{\PmcP} tr (A\T P\T PA) - tr(2PAB\T) + tr(B\T B)=\\
% 	&\argmin_{\PmcP}  -tr (P AB\T) = \argmin_{\PmcP}  -\langle P\T, AB\T \rangle = \label{eq:2c} \\
% 	% &\argmin_{\PmcP}  -\sum_{u,v \in [n]} p_{ij} a_{ij} b_{ji}
% 	% =\\% &\argmin_{\PmcP}  - \text{vec}(P)\T \text{vec}(AB\T).=\\
% 	&\argmin_{\PmcP}  -\langle P, AB\T \rangle, \label{eq:dotLAP}
% \end{align}
% \end{subequations}
% where $\langle \cdot,\cdot \rangle$ %the equality on the second to last line defines is the usual Euclidean inner product, i.e., $\langle X,Y\rangle \defn tr(X\T Y)= \sum_{ij} x_{ij} y_{ij}$.
% While the objective function and the first two constraints of LAP are linear, t
The binary constraints of LAP---like those of QAP---make solving even this problem computationally tricky.  Nonetheless, in the last several decades, there has been much progress in accelerating algorithms for solving LAPs, starting with exponential time, all the way down to $\mc{O}(n^3)$ for general LAPs, and even faster for certain special cases (e.g., sparse matrices) \cite{Jonker1987, Burkard2009}.

% That Eq.~\eqref{eq:dir} is a LAP is evident by considering Eq.~\eqref{eq:dotLAP}.
To see that Eq.~\eqref{eq:dir} is identical to Eq.~\eqref{eq:LAP},
simply let $A=\nabla_P^{(i)}$ and $B=I$ (the $n\times n$ identity matrix).

% The last form indicates that LAP is a linear programming problem (hence the name).  Yet, the constraints, $\mc{P}$, make it a bit trickier.  The feasible region $\mc{P}$ can be written as a set of three constraints: two linear equality constraint sets and a binary constraint.  The LAP objection function with constraints can explicitly be written:
% \begin{align}
% 		&\text{minimize}_P  &&\sum_{u \in \mc{V}} -p_{ij} a_{ij} b_{ji} \nonumber \\
% 		&\text{subject to } && \sum_{u \in \mc{V}} p_{ij} = 1 \, \forall u \in \mc{V} \nonumber \\
% 		& && \sum_{v \in \mc{V}} p_{ij} = 1 \, \forall v \in \mc{V}, \nonumber \\
% 		& &&p_{ij} \in \{0,1\} \, \forall u,v. \label{eq:rLAP}
% \end{align}
% Perhaps because LAP comes up in a wide variety of contexts, a large number of algorithms have been developed to solve LAP \cite{Burkard2009}.  These algorithms have become increasing efficient.
% One of the most popular algorithms, the so-called ``Hungarian algorithm'' has time complexity $\mc{O}(n^3)$ \cite{Jonker1987}.  Under certain conditions (for example, when $AB\T$ is sparse), faster implementations are also available.  As will be seen below, LAP is a key subroutine to our inexact QAP solution.

To solve a LAP, consider a continuous relaxation of LAP, specifically, relaxing the permutation matrix constraint to a doubly stochastic matrix constraint:
% A matrix $P$ is doubly stochastic precisely when $P$ satisfies the following three conditions:
% \begin{enumerate}
% \item	$P\mb{1} = \mb{1}$,
% \item	$P\T \mb{1}=\mb{1}$, %\\
% \item 	$P \in  \Real_+^{n \times n}$,
% \end{enumerate}
% where the third constraint relaxes the binary constraints of the permutation matrices with a non-negativity constraint.
% Let $\mc{D}$ be the set of doubly stochastic matrices.
% With this, we now state a relaxed LAP problem:
\begin{equation}
\text{(rLAP)} \qquad
\begin{array}{cl}
			\text{minimize}   & \langle P, AB\T \rangle \\
			\text{subject to}  &P \in \mc{D}.
\end{array}\label{eq:LAP}
\end{equation}
% \begin{subequations} \label{eq:rLAP}
% \begin{align}
% 		\text{(rLAP) } \quad &\underset{P}{\text{minimize}}  &&-\langle P, AB\T \rangle \\
% 		&\text{subject to } && P \in \mc{D}.
% 		% && \sum_{u \in \mc{V}} p_{ij} = 1 \, \forall u \in \mc{V} \nonumber \\
% 		% 		& && \sum_{v \in \mc{V}} p_{ij} = 1 \, \forall v \in \mc{V}, \nonumber \\
% 		% 		& &&p_{ij} \geq 0 \, \forall u,v, \label{eq:ALAP}
% \end{align}
% \end{subequations}
As it turns out, the minima of LAP and rLAP are equal to one anther \cite{Burkard2009}.
% solving rLAP is equivalent to solving LAP.
% \begin{prop}
% 	LAP and rLAP are equivalent, meaning that they have the same optimal objective function value.
% \end{prop}
% \begin{proof}
% 	Although this proposition is typically proven by invoking total unimodularity, we present a  proof here that we find simpler.	Let $P^L$ be a solution to LAP and let $P^r = \sum_{i\in[k]} \alpha_i P_i$ be a solution to rLAP for some positive integer $k$, permutation matrices $\{P_i\}_{i \in [k]}$, and positive real numbers $\{\alpha_i\}_{i \in[k]}$ such that $\sum_{i \in [k]} \alpha_i=1$.  Note that it must be the case that
% \begin{align}
% 	\langle P^L, AB\T \rangle \geq \langle P^r, AB\T \rangle,
% \end{align}
% because $P^L$ is also a solution for rLAP. Expanding $P^r$, we have
% \begin{align}
% 	\langle P^r, AB\T \rangle = \langle \sum_i \alpha_i P_i, AB\T \rangle = \sum_i \alpha_i \langle  P_i, AB\T \rangle.
% \end{align}
% But we know that
% \begin{align}
% 	\langle P^L, AB\T \rangle \leq \langle P, AB\T \rangle
% \end{align}
% for any $P \in \mc{P}$ because $P^L$ solves LAP, so
% 	%
% 	%
% 	% \begin{multline}
% 	% \langle P^L,AB\T \rangle = \langle  \sum_{i\in[k]} \alpha_i P_i^L, AB\T \rangle=  \sum_{i\in[k]} \alpha_i \langle  P_i^L, AB\T \rangle	 \\
% 	% \leq \sum_{i\in[k]} \alpha_i \langle P^r, AB\T  \rangle = \langle P^r, AB\T \rangle \leq \langle P^L, AB\T \rangle,
% 	% \end{multline}
% 	% % then we have a contradiction,
% 	% because $P^r$ is feasible in rLAP.
% 	\end{proof}
This relaxation motivates our approach to approximating QAP.

\begin{algorithm}
	\caption{\FAQ~ for finding a local optimum of rQAP} \label{alg:1}
\begin{algorithmic}[1]
	\REQUIRE graphs $A$ and $B$ as well as stopping criteria
	\ENSURE $\wh{P}$, an estimated permutation matrix
	\STATE Choose an initialization, $P^{(0)}=\mb{1}\mb{1}\T/n$ \label{step:init} %\COMMENT{although points in $\mc{D}$ would also be feasible}
	\WHILE{stopping criteria not met}
	\STATE Compute the gradient of $f$ at the current point via Eq.~\eqref{eq:grad}
	\STATE Compute the direction $Q^{(i)}$ by solving Eq.~\eqref{eq:dir} via the Hungarian algorithm
	\STATE Compute the step size $\alpha^{(i)}$ by solving Eq.~\eqref{eq:step}
	\STATE Update $P^{(i)}$ according to Eq.~\eqref{eq:update}  %$P^{(i+1)} \leftarrow P^{(i)} + \alpha^{(i)} Q^{(i)}$
	\ENDWHILE
	\STATE Obtain $\wh{P}$ by solving Eq.~\eqref{eq:proj} via the Hungarian algorithm.
\end{algorithmic}
\end{algorithm}

\begin{table}[h!]
\caption{Comparison of \FAQ~ with optimal objective function value and previous state-of-the-art for directed graphs.  The best (lowest) value is in \textbf{bold}. Asterisks indicate achievement of the global minimum.  The number of vertices for each problem is the number in its name (second column).}
\begin{center}
\begin{tabular}{|r|r|r||l|l|l|l|l|}
	\hline
	          \# &  Problem &      Optimal & \FAQ~ & \Epath~& \Grad~ \\
	\hline
	           1 &  lipa20a &     3683 & \textbf{3791} &     3885 &     3909 \\
	           2 &  lipa20b &    27076 & \textbf{27076}$^*$ &    32081 &    \textbf{27076}$^*$ \\
	           3 &  lipa30a &    13178 & \textbf{13571} 	&    13577 &    13668 \\
	           4 &  lipa30b &   151426 & \textbf{151426}$^*$ & \textbf{151426}$^*$ &   \textbf{151426}$^*$ \\
	           5 &  lipa40a &    31538 & \textbf{32109} 	&    32247 &    32590 \\
	           6 &  lipa40b &   476581 & \textbf{476581}$^*$ &   \textbf{476581}$^*$ &   \textbf{476581}$^*$ \\
	           7 &  lipa50a &    62093 & \textbf{62962} &    63339 &    63730 \\
	           8 &  lipa50b &  1210244 & \textbf{1210244}$^*$ &  \textbf{1210244}$^*$ &  \textbf{1210244}$^*$ \\
	           9 &  lipa60a &   107218 & \textbf{108488} &   109168 &   109809 \\
	          10 &  lipa60b &  2520135 & \textbf{2520135}$^*$ &  \textbf{2520135}$^*$ &  \textbf{2520135}$^*$ \\
	          11 &  lipa70a &   169755 & \textbf{171820} &   172200 &   173172 \\
	          12 &  lipa70b &  4603200 & \textbf{4603200}$^*$ &  \textbf{4603200}$^*$ &  \textbf{4603200}$^*$ \\
	          13 &  lipa80a &   253195 & \textbf{256073} &   256601 &   258218 \\
	          14 &  lipa80b &  7763962 & \textbf{7763962}$^*$ &  \textbf{7763962}$^*$ &  \textbf{7763962}$^*$ \\
	          15 &  lipa90a &   360630 & \textbf{363937} &   365233 &   366743 \\
	          16 &  lipa90b & 12490441 & \textbf{12490441}$^*$ & \textbf{12490441}$^*$ & \textbf{12490441}$^*$ \\
	\hline
	\end{tabular}
\end{center}
\label{tab:directed}
\end{table}%

\begin{table}[h!]
\caption{Comparison of \FAQ~ with optimal objective function value and the best result  on the undirected benchmarks.  Note that \FAQ~ restarted 100 times finds the optimal objective function value in 3 of 16 benchmarks, and that \FAQ~ restarted 3 times finds a minimum better than the previous state-of-the-art on all 16 particularly difficult benchmarks.}
\begin{center}
\begin{tabular}{|r|r|r||l|l|l|l|l|}
\hline
\# & Problem  &   Optimal    & \FAQ$_{100}$ & \FAQ$_{3}$ & previous min \\
\hline
1&    chr12c &   11156 &    \textbf{12176} &   13072 & 13072 \\
2&    chr15a &    9896 &    \textbf{9896}$^*$ &   17272 &  19086 \\
3&    chr15c &    9504 &    \textbf{10960} &   14274 &  16206 \\
4&   chr20b &    2298 &     \textbf{2786} &    3068 &    3068 \\
5&    chr22b &    6194 &    \textbf{7218} &    7876 &   8482 \\
6&    esc16b & 	292 & 		\textbf{292}$^*$ & 294 &    296 \\
7& 	   rou12 &  235528 &  \textbf{235528}$^*$ &  238134 &    253684 \\
8& 	   rou15 &  354210 &  \textbf{356654} &  371458 &    371458 \\
9&      rou20 &  725522 &  \textbf{730614} &  743884 &    743884 \\
10&    tai10a &  135028 &  \textbf{135828} &  148970 &    152534 \\
11&    tai15a &  388214 &  \textbf{391522} &  397376 &    397376 \\
12&    tai17a &  491812 &  \textbf{496598} &  511574 &    529134 \\
13&    tai20a &  703482 &  \textbf{711840} &  721540 &    734276 \\
14&    tai30a & 1818146 & \textbf{1844636} & 1890738 &  1894640 \\
15&    tai35a & 2422002 & \textbf{2454292} & 2460940 &  2460940 \\
16&    tai40a & 3139370 & \textbf{3187738} & 3194826 &  3227612 \\
    \hline
\end{tabular}
\end{center}
\label{tab:restarts}
\end{table}%


% !TEX root = scombinatorics.tex
\documentclass[scombinatorics.tex]{subfiles}
\begin{document}
\chapter{Vapnik-Chervonenkis theory}
\label{sauer}

\def\medrel#1{\parbox[t]{5ex}{$\displaystyle\hfil #1$}}
\def\ceq#1#2#3{\parbox[t]{20ex}{$\displaystyle #1$}\medrel{#2}{$\displaystyle #3$}}

%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
\section{The Vapnik-Chervonenkis dimension}\label{VCdim}

If all subsets of $A\subseteq\U$ are definable, that is $\P A=\phi(A,b)_{b\in\V}$ we say that $A$ is \emph{shattered\/} by $\phi(x\,;z)$.
%Paraphrase the the definition,  $A$ is shattered by $\phi$ if fr every $C\subseteq A$ there is a $b_C\in\V$ such that $C=\phi(A,b_C)$.
The following is called the \emph{shatter function\/}

%\ceq{\hfill\emph{$\pi_\phi(n)$}}{=}{\max\Big\{|\phi(A,b)_{b\in\V}|\ \ :\ \ A\subseteq\U,\ |A|=n\Big\}.}

\ceq{\hfill\emph{$\pi_\phi(n)$}}{=}{\max\bigg\{|\phi(A,b)_{b\in\V}|\ \ :\ \ A\in {\U\choose n}\bigg\}}

So, $\pi_\phi(n)$ gives the maximal number of definable subsets that a set of cardinality $n$ can have.
Trivially, $\pi_\phi(n)\le2^n$ for all $n$.
Moreover, if $\pi_\phi(n)=2^n$ for some $n$, then $\pi_\phi(k)=2^k$ for every $k\le n$.

The \emph{Vapnik-Chervonenkis dimension\/} of $\phi(\U\,;b)_{b\in\V}$, or of $\phi(x\,;z)$, abbreviated by \emph{\vc-dimension}, is the maximal cardinality of a finite set $A\subseteq\U$ that is shattered by $\phi(x\,;z)$.
Equivalently, it is the maximal $k$ such that $\pi_\phi(k)=2^k$.
If such a maximum does not exist,
we say that the \vc-dimension is infinite or that $\phi(x\,;z)$ has \emph{\ip\/} (the independence property).
Otherwise, we say that $\phi(x\,;z)$ has \emph{\nip\/} (not the independence property).
We may also say: \textit{is\/} \ip, or \textit{is\/} \nip.

As $\U$ and $\V$ are usually clear from the context, we may say \vc-dimension of $\phi(x\,;z)$ for the \vc-dimension of $\phi(\U\,;b)_{b\in\V}$.

\begin{example}
  If $\phi(x\,;z)$ is either $\top$ or $\bot$, then it shatters only the empty set, therefore it has \vc-dimension $0$.\QED
\end{example}

\begin{example}
  If $\phi(x\,;z)$ has ladder dimension $n$ then it has \vc-dimension at most $n$. Hence stable formulas are \nip.\QED
\end{example}

\begin{example}
  If $\phi(\U\,;b)_{b\in\V}$ is a non trivial chain of sets, then its \vc-dimension is $1$.\QED
\end{example}

\begin{example}
  Let $\U=\RR$ and $\V=\RR^2$.
  Let $\phi(x\,;z_1,z_2)$ be the formula $z_1<x<z_2$.
  Then its \vc-dimension $2$.\QED
\end{example}

\begin{example}
  Let $\U=\V=\RR^2$.
  Let $\phi(x_1,x_2\,;z_1,z_2)$ be the formula $x_2<z_1\cdot x_1 + z_2$.
  Then its  \vc-dimension $3$ (by Radon's Theorem).\QED
\end{example}

\begin{example}\label{ex_vcdim_opt}
  If $\phi(\U\,;b)_{b\in\V}$ is the set of all subsets of $\U$ of cardinality $\le k$.
  Then its \vc-dimension is $k$ and

  \ceq{\hfill\pi_\phi(n)}{=}{\sum^k_{i=0} \binom{n}{i}.}

  Incidentally, we note that this is also the shatter function of the collection of all subsets of $\U$ of cardinality exactly $k$.
  In fact we always assume $\U$ is infinite (or at least very large, in this case, size $\ge2k$ suffices).\QED
\end{example}

The \vc-dimension of $\phi(x\,;z)^{\rm op}$ is called the \emph{dual \vc-dimension\/} or \emph{\vc-codimension\/} of $\phi(x\,;z)$.

\begin{proposition}\label{prop_bound_VCcodim}
  If $\phi(x\,;z)$ has \vc-dimension $k$, then its \vc-codimension is $\le2^{k+1}-1$.
\end{proposition}

\begin{proof}
  Suppose that the \vc-dimension of $\phi(x\,;z)^{\rm op}$ is $\ge 2^{k+1}$.
  We prove that the \vc-dimension of $\phi(x\,;z)$ is at least $k+1$.
  Let $B=\big\{b_I\,:\,I\subseteq [k+1)\big\}$ be a set of cardinality $2^{k+1}$ shattered by $\phi(x\,;z)^{\rm op}$.
  That is, for every $\J\subseteq \P[k+1)$ there is $a_\J$ such that

  \ceq{\hfill\phi(a_\J, b_I)}{\IFF}{I\in\J}\hfill for all $I\subseteq [k+1)$

  Let $a_i=a_{\{I\; :\ i\,\in\, I\}}$. Then from the equivalence above we obtain

  \ceq{\hfill\phi(a_i, b_I)}{\IFF}{i\in I}

  That is, $\phi(x\,;z)$ shatters $A=\big\{a_i\,:\, i\in [k+1)\big\}$.
\end{proof}

We prove that the bound in the proposition above is optimal.

\begin{proposition}
  For every $k$, there is a formula $\phi(x\,;z)$ that has \vc-dimension $k$ and \vc-codimension $2^{k+1}-1$.
\end{proposition}
\begin{proof}
  Let $k$ be given.
  We claim that there is a formula $\phi(x\,;z)$ with \vc-dimension $2^{k+1}-1$ and \vc-codimension $k$.
  As the dual of the dual is the primal, this claim is equivalent to the proposition.
  Let $\U$ an infinite set, let $\V=\P(\U)$.
  Fix an equivalence relation on $\U$ with $2^{k+1}-1$ many classes.
  We say that $b\in\V$ is compatible if it is union of equivalence classes.
  Define

  \ceq{\hfill\phi(x\,;z)}{=}{x\in z \ \wedge\ z\textrm{ is compatible}}%\A x_1,x_2\big[x_1\sim x_2\imp [x_1\in z\iff x_2\in z]\big]}

  (Recall that in the preliminaries to this notes, we have agreed that $\U$ and $\V$ are some fixed infinite sets. Without this constraint we could have taken as $\U$ a set of cardinality $2^{k+1}-1$; defined $\phi(x\,;z)=x\in z$; and forgot the equivalence relation altogether.)

  Clearly, $\phi(x\,;z)$ has \vc-dimension $2^{k+1}-1$.
  Note that the dimension of $\phi(x\,;z)^{\rm op}$ is at least $k$.
  Otherwise $\phi(x\,;z)$ would have dimension $\le2^k-1$ by Proposition~\ref{prop_bound_VCcodim}.

  So, it suffices to prove that the dimension of $\phi(x\,;z)^{\rm op}$ is exactly $k$.
  Assume for a contradiction that some $B=\big\{b_i\ :\ i\in[k+1)\big\}\subseteq\V$ is shattered by $\phi(x\,;z)^{\rm op}$.
  Then there is some set $A=\big\{a_I:I\subseteq[k+1)\big\}\subseteq\U$ such that

  \ceq{\hfill\phi(a_I, b_i)}{\IFF}{i\in I}

  The $b_i\in B$ are necessarily compatible sets, therefore any two distinct $a_I\in A$ are non equivalent.
  But this is impossible by cardinality reasons.
\end{proof}

\begin{exercise}
  Let $\phi(x\,;z)$ have \vc-dimension $k$.
  Assume that there is no $A\subseteq\U$ of cardinality $\le k+1$ such that $\phi(a\,;\V)_{a\in A}$ covers $\V$.
  Prove that $\phi(x\,;z)$ has \vc-codimension $\le2^{k+1}-2$.
  Prove that the bound is optimal.
  Hint: let $\U$ be an infinite set, $\V = \P(\U)$, and consider an equivalence relation $\sim$ s.t. $\U / {\sim} = C \cup \{c_0, \dots, c_k\}$ with $|C| = 2^{k+1}-2$ and take the same formula used in the previous Proposition, but with $b \in \V$ compatible if it is the union of equivalence classes that belongs to $C$ or if it coincides with $c_i$ for some $i \in \{0,\dots, k\}$.
  Prove that $\phi(x\,;z)$ has \vc-codimension $k$.\QED
\end{exercise}

% \begin{exercise}
%   Let $A\subseteq\U$ have finite cardinality $n$.
%   Let $\phi(x\,;z)$ have \vc-dimension $k$.
%   Define an equivalence relation on $\V$ as follows.

%   \ceq{\hfill b\sim_Ab'}
%   {\IFF}
%   {}

% \end{exercise}

%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%
\section{The Sauer-Shelah lemma}\label{sauer}

\def\ceq#1#2#3{\parbox[t]{15ex}{$\displaystyle #1$}\medrel{#2}{$\displaystyle #3$}}

According to Gil Kalai in~\cite{kalai}, Sauer-Shelah's Lemma can been described as an \textit{eigentheorem\/} because it is important in many different areas of mathematic (model theory, learning theory, probability theory, ergodic theory, Banach spaces, to name a few).
No wonder it has been discovered and rediscovered may times.

It has been proved independently by Shelah~\cite{shelah72}, Sauer~\cite{sauer}, and Vapnik-Cher\-vo\-nen\-kis~\cite{VC} around 1970 (Shelah gives credit to Micha Perles).
Saharon Shelah was working in model theory while Norbert Sauer, Vladimir Vapnik and Alexey Chervonenkis were in statistical learning theory.

\begin{void_thm}[Sauer-Shelah Lemma]\label{lem_sauer}
If $\phi(x\,;z)$ has \vc-dimension $k$ then for every $n\ge k$

\ceq{\hfill\pi_\phi(n)}{\le}{\bigsum^{k}_{i=0} \binom{n}{i}.}
\end{void_thm}

The set system presented in Example~\ref{ex_vcdim_opt} shows that the bound is optimal.

An alternative proof of the Sauer-Shelah Lemma derives it as corollary of a lemma by Alain Pajor~\cite{pajor}.

\begin{void_thm}[Pajor's Lemma]\label{lem_pajor}
  Let $A\subseteq\U$ be finite.

  \ceq{\hfill|\phi(A,b)_{b\in\V}|}{\le}{\Big|\{C\subseteq A\; :\; C \textrm{\ is\ shattered\ by\ }\phi(x\,;z)\}\Big|}.
\end{void_thm}

\begin{proof}
  If $A$ is empty then $|\phi(A,b)_{b\in\V}|=1$ and $\0$ is the only subset of $A$ that $\phi$ shatters, so the inequality holds trivially.
  Otherwise, pick an $a\in A$ and assume the lemma holds for $A'=A\sm\{a\}$.
  Define

  \ceq{\hfill\psi(x\,;y)}{=}{\phi(x\,;y)\ \wedge\neg \phi(a\,;y)\ \wedge\ \E y'\, \Big[ \phi(a\,;y')\ \wedge\ \phi(A'\,;y')=\phi(A'\,;y)\Big].}

  Notice that

  \ceq{\hfill\big|\phi(A,b)_{b\in\V}\big|}{=}{\bigg|\phi(A',b)_{b\in\V}\quad \cup\quad \Big\{\{a\}\cup\psi(A',b)\ :\ b\in\V\Big\}\bigg|.}

  as the two sets in the r.h.s.\@ are disjoint

  \ceq{\hfill\big|\phi(A,b)_{b\in\V}\big|}{=}{\big|\phi(A',b)_{b\in\V}\big|\ +\ \big|\psi(A',b)_{b\in\V}\big|.}

  By induction hypothesis,

  \ceq{\hfill\big|\phi(A',b)_{b\in\V}\big|}{\le}{\Big|\{C\subseteq A'\; :\; C \textrm{\ is\ shattered\ by\ }\phi(x\,;z)\}\Big|}\hfill(1)

  and

  \ceq{\hfill\big|\psi(A',b)_{b\in\V}\big|}{\le}{\Big|\{C\subseteq A'\; :\; C \textrm{\ is\ shattered\ by\ }\psi(x\,;z)\}\Big|}

  \ceq{}{=}{\Big|\{C\subseteq A'\; :\; C\cup\{a\} \textrm{\ is\ shattered\ by\ }\phi(x\,;z)\}\Big|.}\hfill(2)

  In fact, $C\subseteq A'$ is shattered by $\psi(x\,;y)$ if an only if $C\cup\{a\}$ it is shattered by $\phi(x\,;y)$.
  Clearly,

  \ceq{\hfill(1)+(2)}{=}{\Big|\{C\subseteq A\; :\; C \textrm{\ is\ shattered\ by\ }\phi(x\,;z)\}\Big|,}

  so the lemma follows.
\end{proof}

\begin{proof}[Proof of the Sauer-Shelah Lemma]
  Given $\phi(x\,;z)$ and $n\ge k$ as in the lemma

  \ceq{\hfill\pi_\phi(n)}{=}{\max_{|A|=n}\big|\phi(A,b)_{b\in\V}\big|}\smallskip

  \ceq{\hfill\pi_\phi(n)}{\le}{\max_{|A|=n}\big|\{C\subseteq A\; :\; C\textrm{\ shattered\ by\ }\phi(x\,;z)\}\big|}\hfill by  Pajor's Lemma\smallskip

  \ceq{}{\le}{\sum^k_{i=0}{n\choose i}}\hfill because $\phi(x\,;z)$ has \vc-dimension $k$
\end{proof}

We write $f(n)=O(g(n))$ if there is a constant $C$ such that $|f(n)|\le C g(n)$ holds for all (sufficiently large) $n$.

The \emph{\vc-density\/} of $\phi(x\,;z)$ is the infimum over all real number $r$ such that $\pi_\phi(n)= O(n^r)$.
It is infinite if no such $r$ exist.
The  \emph{dual \vc-density\/} is defined accordingly.

By the Sauer-Shelah lemma the \vc-density is at most as large as the \vc-dimension.
It could be smaller, however it is usually rather difficult to compute.

We conclude this section with a couple of inequalities that is useful to have at hand.

\ceq{\hfill\sum^k_{i=0}{n\choose i}}
{=}
{\sum^k_{i=0}\frac{n!}{i!\,(n-i)!}}

\ceq{}
{\le}
{\sum^k_{i=0}\frac{n^i}{i!}}

\ceq{}
{\le}
{\sum^k_{i=0}\frac{n^i\,k!}{i!(k-i)!}}

\ceq{}
{=}
{\sum^k_{i=0}n^i{k\choose i}}

\ceq{}
{=}
{(n+1)^k}
\hfill by the binomial theorem.\smallskip

There is a second bound, which is better when $k\ge 3$ and holds for $n>k$

\ceq{\hfill\sum^k_{i=0}{n\choose i}}
{\le}
{\Big(\frac{n}{k}\Big)^{\!k}\sum^k_{i=0}\Big(\frac{k}{n}\Big)^{\!i}{n\choose i}}
\hfill because $\dfrac{k}{n}<1$

\ceq{}
{\le}
{\Big(\frac{n}{k}\Big)^{\!k}\sum^n_{i=0}\Big(\frac{k}{n}\Big)^{\!i}{n\choose i}}

\ceq{}
{=}
{\Big(\frac{n}{k}\Big)^{\!k}\Big(1+\frac{k}{n}\Big)^n}
\hfill by the binomial theorem

\ceq{}
{\le}
{\Big(\frac{n\,e}{k}\Big)^{\!k}}
\hfill where $e$ is the base of the natural logarithm.

\end{document}


\pagebreak
\null
\newpage
\section{Discrete Laplacians}\label{sec:Chapter3: other discrete laplacians}
In this Chapter we describe different methods to approximate the Laplace-Beltrami operator on manifolds, with particular attention to what we will call the Heat Kernel Graph Laplacian.

Let $f$ be a sufficiently smooth function on a compact, closed infinitely differentiable manifold $\mathcal M$. The Laplacian eigenvalue problem on $\mathcal M$ is defined as
\begin{equation}\label{eq:continous eigenvalue problem}
\triangle_{\mathcal M}\psi = -\lambda \psi
\end{equation}
Being the Laplace-Beltrami operator self-adjoint and semi-positive definite, there exists a basis $\mathcal B=\{\psi_i\}_i$of the space $L^2(\mathcal M)$ such that $\triangle_\mathcal M \psi_i = -\lambda_i\psi_i,\ \lambda_0\leq\lambda_1\leq...,\lambda_i\leq\lambda_{i+1}...\leq+\infty$. See \cite{rosenberg_1997} for an introduction to the Laplace-Beltrami operator on manifolds.

There's been a lot of work in trying to calculate solutions to equation (\ref{eq:continous eigenvalue problem}), leading to different ways to approximate the Laplace-Beltrami operator through what we call a \textit{discrete Laplacian}. By discrete Laplacian we mean an operator $L$ that, once evaluated on a signal $f$ and on a vertex $x_i$ can be written as a matrix $\mathbf L$ in the following way:

\begin{equation}\label{eq:discrete laplacian}
\mathbf L f\left(\mathbf{x}_{i}\right)=\frac{1}{d_{i}} \sum_{j} w_{i j}\left(f\left(\mathbf{x}_{i}\right)-f\left(\mathbf{x}_{j}\right)\right)
\end{equation}

Note that with unit \textit{masses} $d_i=1$ we recover the same definition of Graph Laplacian. We need now to introduce some basic concepts of Differential Geometry, especially the definition of mean curvature of a manifold and its link with the Laplace-Beltrami operator.
\subsection{Notions of Differential Geometry}

 For this short introduction to basic concepts of Differential Geometry, set the manifold $\mathcal M$ to be a differentiable, two dimensional surface embedded in $\mathbb R^3$.
 \begin{figure}[h]
 	\centering
 	\includegraphics[width=0.7\textwidth]{figs/Chapter3/curvature.png}
 	\caption{\label{fig:curvature}Curvature normal of a manifold}
 \end{figure}
The \textit{curvature} of a curve on a plane is defined as the inverse of the radius $R$ of the tangent circle. For each point on the manifold $\mathcal M$, define its tangent plane $H$, orthogonal to the normal vector $\mathbf n$. For every unit vector $\mathbf e_\theta$ lying on the tangent plane $H$, where $\theta$ is an angle that measures the direction on the tangent plane of $\mathbf e_\theta$, the \textit{normal curvature} $\kappa(\theta)$ is defined as the curvature of the curve that is the intersection of the manifold $\mathcal M$ and the plane containing both $\mathbf n$ and $\mathbf e_\theta$. The \textit{mean curvature} $\overline \kappa $ is defined as the average on $\theta$ of the normal curvatures:

 \begin{equation}\label{eq:mean curvature}
 	\overline \kappa=\frac{1}{2 \pi} \int_{0}^{2 \pi} \kappa(\theta) d \theta
 \end{equation}

It can be proved that the Laplace-Beltrami operator applied on the identity function $\mathbf x \rightarrow \mathbf x, \ \forall \mathbf x\in \mathcal M$ is directly linked to the \textit{mean curvature normal} $\overline{\kappa}\mathbf n$ by the following formula:
\begin{equation}\label{eq:laplacian and curvature}
	\triangle_\mathcal M \mathbf x  = -2\overline{\kappa}\mathbf n
\end{equation}
This equation provides us a way to approximate the Laplace-Beltrami operator through the approximation of the mean curvature normal. This fact is exploited by methods presented in the next section.

\subsection{Discrete Laplacians from Differential Geometry}
\begin{figure}[h]
	\begin{center}
		\includegraphics[width=0.45\textwidth]{figs/Chapter3/MyDesbrun.png}
		\includegraphics[width=0.45\textwidth]{figs/Chapter3/Voronoi}
	\end{center}
	\caption{\label{fig:Desbrun}One term of curvature normal formula and one Voronoi cell constructed around the node $x_i$}
\end{figure}
Desbrun et al. \cite{Desbrun1999} construct a triangulation $\mathcal T_h$ with the vertices in the sampling $x_0, x_1, ..., x_{n-1}$ approximating the manifold $\mathcal M$, and then use the following discrete expression for the \textit{discrete normal curvature} $\overline{\kappa_h} \mathbf{n}$ of the manifold $\mathcal M$:
\begin{equation}\label{eq:curvature normal}
	-\overline{\kappa_h} =\frac{1}{4 A_i} \sum_{j \in N_{1}(i)}\left(\cot \alpha_{ij}+\cot \beta_{ij}\right)\left(x_{j}-x_{i}\right)
\end{equation}
where $A_i$ is the area of all the triangles of the mesh sharing the node $x_i$; $N_1(i)$ is the first ring of neighbors of the $i$th node; $\alpha_{i j},\ \beta_{i j}$ are the angles of the triangles of the mesh that lie on the opposite side to the edge $(i,j)$ with respect to the node $x_i$ (Figure \ref{fig:Desbrun}). Observe that for a flat surface the discrete curvature is equal to zero $\overline{\kappa_h}=0$. This is a geometric approach that relies on the intrinsic properties of the triangulation $\mathcal T_h$ and is based on the geometric meaning of the curvature normal $\overline{\kappa}$. Using equations (\ref{eq:curvature normal}) and (\ref{eq:laplacian and curvature}) it can be shown \cite{REUTER2009381} that this approach leads to a discrete Laplacian with masses
$$
d_i=\frac{A_i}{3}
$$
where $A_i$ is the area of all the triangles of the mesh with a vertex in $x_i$, and weights
$$
w_{i j}=\frac{\cot \left(\alpha_{i j}\right)+\cot \left(\beta_{i j}\right)}{2}
$$

\subsection{Linear Finite Element Method Laplacian}
The eigenvalue problem (\ref{eq:continous eigenvalue problem}) can be rewritten in the equivalent weak form
\begin{equation}\label{eq:weak}
	\langle \nabla f, \nabla v\rangle_{L^2(\mathbb S^2)} = \lambda \langle  f, v\rangle_{L^2(\mathbb S^2)} \quad \forall v \in L^2(\mathbb S^2)
\end{equation}
The Finite Element Method (FEM) is a numerical algorithm that allows to calculate a discrete approximation of the solution $f$ through a functional discretization of the weak eigenvalue problem (\ref{eq:weak}). We will discuss this method deeper in section \ref{sec:Chapter3: Using the Finite Element Method to approximate the Laplace-Beltrami operator on a manifold}. By projecting equation (\ref{eq:weak}) on a finite dimensional functional subspace of $L^2(\mathbb S^2)$ spanned by $n$ basis functions $\phi_i$, by writing $n$ times the equation (\ref{eq:weak eigenvalue problem}), setting each time the \textit{test} function $v$ equal to the $i$th basis function $\phi_i$ we obtain the generalized algebraic eigenvalue problem
$$
\begin{aligned}
&\text{Find }(f,\lambda)\text{ such that }\mathbf A\mathbf f = \lambda \mathbf B \mathbf f\\
&\begin{cases}
(\mathbf A)_{ij} &= \int_{\mathbb S^2}\nabla \phi_i(\mathbf{x})\cdot \nabla \phi_j(\mathbf{x})d\mathbf{x}\\
(\mathbf B)_{ij} &= \int_{\mathbb S^2} \phi_i(\mathbf{x}) \phi_j(\mathbf{x})d\mathbf{x}\\
(\mathbf f)_i &= f_i:\quad f(\mathbf x) = f_0\phi_0(\mathbf x)+ ... + f_{n-1}\phi_{n-1}(\mathbf x)
\end{cases}
\end{aligned}
$$
that can be solved through usual algebraic solvers.

\paragraph{FEM Laplacian as a Differential Geometry Laplacian.} Levy \cite{levy} showed that by using the \textit{lumped} mass matrix $\mathbf D_{ii} = \sum_j \mathbf B_{ij}$ the FEM Laplacian
$$
\mathbf D^{-1}\mathbf A
$$
is the same Laplacian of Desbrun et al. \cite{Desbrun1999} introduced in the previous section. Meyer et al. \cite{Meyer02discretedifferential-geometry} proposed another discrete Laplacian by modifying the masses of Desbrun et al. setting them to
$$
d_i=a_{V}(i),
$$
where \(a_{V}(i)\) is the area of the polygon obtained by joining the circumcenters of the triangles surrounding node $i$ (i.e. the Voronoi cell, figure \ref{fig:Desbrun}).

\subsection{Graph Laplacian for manifolds}\label{sec:Chapter1:theoretical foundations}
Belkin et al. \cite{NIPS2006_2989} prove convergence of eigenvectors of the \textit{Heat Kernel Graph Laplacian} $\mathbf L_n^t$ (HKGL) of a data point cloud to the eigenfunctions of the Laplace Beltrami operator $\Delta_\mathcal M$ on the manifold $\mathcal M$, when the data is sampled from a uniform distribution on $\mathcal M$.
For this result to hold, they suppose the manifold $\mathcal M$ to be compact, infinitely differentiable and without boundary. We just point out that being $\mathcal M$ compact, $\Delta_\mathcal M$ has a discrete spectrum. The graph they use to approximate the manifold is constructed as follows: given a sampling $ \mathcal P = \{x_i\in\mathcal M\}_{i=0}^{n-1}$ on a $k$-dimensional manifold $\mathcal M\subset \mathbb R^N$ they construct the full graph defined by the weights
$$
w_{ij}=\exp\left({-\frac{||x_i-x_j||^2}{4t}}\right)
$$
where $\norm\cdot$ is the Euclidean norm in the ambient space $\mathbb R^N$ and whose Laplacian matrix
$$
\mathbf L_n^t
$$ we call \textit{Heat Kernel Graph Laplacian}.

Observe that given a function $f: \mathcal P \rightarrow \mathbb R$ defined on the sampling $ \mathcal P$ and defined the vector $\mathbf f\in\mathbb R^n$ such that $\mathbf f_i = f(x_i)$, the Heat Kernel Graph Laplacian matrix acts on $\mathbf f$ in the following way:
\begin{equation}\label{eq:HKGL}
(\mathbf L_n^t \mathbf f)_i:=  \sum_{j=0}^{n-1} e^{-\frac{||x_i-x_j||^2}{4t}}\left(f(x_i)-f(x_j)\right)
\end{equation}
This graph construction is motivated by the fact that the HKGL is nothing else than the natural discretization of the continuous \textit{functional approximation to the Laplace-Beltrami operator} $L^t:  L^{2}(\mathcal{M}) \rightarrow L^{2}(\mathcal{M})$ whose eigenvectors and eigenvalues are proven to converge to the ones of $\Delta_\mathcal M$.
\vspace{0.5cm}
\begin{definition}{}(\cite[Belkin et al.]{Belkin:2005:TTF:2138147.2138189}Functional approximation to the Laplace-Beltrami operator)\\ \label{eq: my L^t} Let $\mu$ be the uniform probability measure on the manifold $\mathcal M$, where $\text{vol}(\mathcal M)$ is the volume of $\mathcal M$. We define the functional approximation to the Laplace-Beltrami operator to be the operator $L^t: L^{2}(\mathcal{M}) \rightarrow L^{2}(\mathcal{M})$ such that
	\label{def:Functional approximation to the Laplace-Beltrami operator}
	$$ L^tf(y) = \int_{\mathcal M}e^{-\frac{||y-x||^2}{4t}}\left(f(y)-f(x)\right)d\mu(x)$$
\end{definition}
We end this Chapter by stating the theorem of spectral convergence of the HKGL to the Laplace-Beltrami operator $\Delta_\mathcal M$, that makes it a really good candidate to construct rotation invariant graphs.
\vspace{0.5cm}
\begin{snugshade*}
	\begin{theorem}(Belkin et al., \cite{NIPS2006_2989})\label{theo:spectral convergence}
		Let \(\lambda_{n, i}^{t}\) be the $i$th eigenvalue of
		$$
		\frac{(4\pi t)^{-(k+2)/2}}{n}\mathbf L^t_n
		$$
		and \(\mathbf v_{n, i}^{t}\) be the corresponding eigenvector. Let \(\lambda_{i}\) and \(v_{i}\) be the corresponding eigenvalue and eigenfunction of \(\Delta\) respectively. Then there exists a sequence \(t_{n} \rightarrow 0,\) such that
		\begin{equation}
		\begin{array}{c}{\lim _{n \rightarrow \infty} \lambda_{n, i}^{t_{n}}=\lambda_{i}} \\
		{\lim _{n \rightarrow \infty}\left\|\mathbf v_{n, i}^{t_{n}}-v_{i}(\mathbf x)\right\|_{2}=0}\end{array}
		\end{equation}
		where the limits are in probability.
	\end{theorem}
\end{snugshade*}


\documentclass{scrartcl}

\input{../../shared.tex}
\theoremstyle{definition}
\newtheorem{definition}{Definition}[section]
\newtheorem{lemma}{Lemma}[section]
\usepackage{enumerate}

\begin{document}

\section*{Problem set: Gradient descent}%

\paragraph{Exercise (i)} (7P) Enter the missing code snippets in the jupyter notebook. Partial credit will be awarded.

\begin{definition}
  For a matrix $Q \in \R^{m \times d}$ its \textbf{operator norm} is defined as
  \begin{equation}
    \Vert Q \Vert_{op} := \sup_{x \in \R^d: \Vert x \Vert \le 1} \Vert Q x \Vert,
  \end{equation}
  but for convenience we will mostly write $\Vert Q \Vert$ (so no subscript) but mean the operator norm.
\end{definition}

\begin{lemma}%
  The operator norm of a matrix $Q$ fulfills
  \begin{equation}
    \Vert Qx \Vert \le \Vert Q \Vert_{op} \Vert x \Vert.
  \end{equation}
  In particular, every linear map (given by a matrix $Q$) is Lipschitz continuous with Lipschitz constant $\Vert Q \Vert$.
\end{lemma}

\paragraph{Exercise (ii)} (2P) Prove that the quadratic function
\begin{equation}
  f(x) = \frac{1}{2} x^T Q x +b^T x + c
\end{equation}
is \textbf{smooth} with parameter $\Vert Q \Vert$. Note that without loss of generality $Q$ can be assumed to symmetric, however, this should not change anything.

\paragraph{Exercise (iii)} (1P) Suppose that we have observations $(x_i, y_i)$ which are \textbf{centered}, meaning that $\sum_{i=1}^{n}x_i = 0 = \sum_{i=1}^{n}y_i$. Let $(b^*, w^*)$ be the global minimum of the least squares objective
\begin{equation}
  f(b, w) = \sum_{i=1}^{n} {(b + w^T x_i - y_i)}^2.
\end{equation}
Prove that $b^*=0$.

\end{document}