\documentclass{beamer}

\input{../../shared_slides.tex}
\DeclareMathOperator*{\perm}{perm}

\title{Matrix scaling}

\begin{document}
\maketitle
\frame{\tableofcontents}

\section{Introduction}%

\begin{frame}
  \frametitle{Introduction}
  \textbf{given:} a matrix $A \in \R^{m\times n}_+$, vectors $r \in \R_{++}^m$ and $c \in \R^n_{++}$\\
  \textbf{find:} diagonal matrices $X$ and $Y$ such that for $B = XAY$ it holds:
  \begin{equation}
    B \mathbbm{1}_n = r \quad \text{and} \quad B^T \mathbbm{1}_m = c
  \end{equation}
  where $\mathbbm{1}_n = (1, \dots, 1)$ exactly $n$-times.
  Equivalently
  \begin{equation}
    \Vert B_{i,:} \Vert_1 = r_i \quad \text{and} \Vert B_{:, j} \Vert = c_j.
  \end{equation}

  \begin{block}{}
    In this case $A$ is called $(r,c)$-scalable.
  \end{block}
  \onslide<2->{%
    If $\Vert r \Vert_1 \neq \Vert c \Vert_2$ this is not possible.
  }
\end{frame}

\begin{frame}
  \frametitle{Visualization of diagonal scaling}

  \begin{equation}
    \begin{aligned}
    B = \begin{bmatrix}
      x_1 & & & \\
      & x_2 & & \\
      & & \ddots & \\
      & & & x_m
    \end{bmatrix}
    A
    \begin{bmatrix}
      y_1 & & & \\
      & y_2 & & \\
      & & \ddots & \\
      & & & y_n
    \end{bmatrix}
    \\
    =
    \begin{bmatrix}
      a_{1,1}x_1y_1 & a_{1,2}x_1y_2 & \cdots & a_{1,n} x_1y_m \\
      \vdots   & \ddots & & \\
      a_{m,1}x_m y_1 & & \cdots & a_{m,n}x_m y_m
    \end{bmatrix}
    \end{aligned}
  \end{equation}
  \begin{block}{\textbf{Application:} Ill conditioned linear system $Az = b$.}
    Can multiply both sides by $X$  and substitute $z= Yv$ to get instead
    \begin{equation}
      XAz = X
    \end{equation}
  \end{block}
\end{frame}

\section{Matchings}%
\label{sec:}

\begin{frame}
  \frametitle{$(0-1)$ matrices | bipartite graphs}
  \begin{minipage}{0.5\textwidth}
    \begin{equation}
      \begin{bmatrix}
        0 & 1 & 1 \\
        1 & 0 & 0 \\
        0 & 1 & 1
      \end{bmatrix}
    \end{equation}
  \end{minipage}
  \begin{minipage}{0.25\textwidth}
    \begin{figure}[ht]
      \centering
      \includegraphics[width=\textwidth]{bipartite-graph.png}
      \caption{bipartite graph}
    \end{figure}
  \end{minipage}

  \onslide<2->{%
    \begin{definition}
      A \textbf{matching} is a set of edges without common vertices.
    \end{definition}
    % Say something about applications of matching like:
    % - marriage theorem
    % - hitchcock transport problem
    \begin{definition}
      A \textbf{perfect matching} is a matching which covers all vertices.
    \end{definition}
  }
\end{frame}

\begin{frame}
  \frametitle{Finding the number of perfect matchings}
  Finding one is easy (polynomial time).
  Finding all is in \# P (i.e.\ hard!).
  \begin{block}{Consider $m=n$, $A\in \R^{n \times n}$}
    Recall:
    \begin{equation}
      \begin{aligned}
        \text{(determinant)} \quad \det A = \sum_{\sigma} \sign(\sigma) \prod_{i=1}^n a_i \sigma(i) \\
        \text{(permanent)} \quad \perm A = \sum_{\sigma} \prod_{i=1}^n a_i \sigma(i)
      \end{aligned}
    \end{equation}
  \end{block}
  \begin{block}{Observation}
    For a $(0,1)-$matrix $A$, $\perm A$ is the number of perfect matchings.
    % This means that computing permanent must be hard.
  \end{block}
  One is easy to compute the other one hard. How can this be?
  % Diagionalize via gaussion elimination gives O(n^3) algorithm for computing determinant.
  % This doesn't work for permanent because it is not invariant under these operation (not multilinear).
\end{frame}

\section{Permanent}%
\label{sec:}

\begin{frame}
  \frametitle{Lower bounding the permanent}
  % This notion also appears in optimal transport
  \begin{definition}
    A matrix $A \in \R^{m\times n}_+$ is called \textbf{doubly stochastic}, if sum of every row and every column is $1$.
  \end{definition}

  \begin{block}{van der Waerden (1926) conjectured}
    For doubly stochastic matrices the following \emph{lower bound} holds
    \begin{equation}
      \perm A \ge \frac{n!}{n^n}.
    \end{equation}
  \end{block}
  \begin{equation}
   \text{Is tight for} A = \begin{bmatrix}
      1/n & \cdots & 1/n \\
      \vdots & \ddots & 1/n \\
      1/n & \dots & 1/n
    \end{bmatrix}
  \end{equation}
  Proved independently by Jegortschow and Falikman in '80 / '81.
\end{frame}

% do we use this at all?
\begin{frame}
  \frametitle{Upper bounding the permanent}
  \begin{block}{Bregman-Minc}
    For $(0,1)$-matrices
    \begin{equation}
      \perm A \le \prod_{i=1}^n {(r_i !)}^{1/r_i} \quad \text{where $r_i := \Vert A_{i,:} \Vert$}
    \end{equation}
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Matrix scaling to approx.\ permanent}
  \begin{block}{}
    If a $(0, 1)$-matrix $A$ can be scaled to be doubly stochastic, i.e.\ it is $\left(\mathbbm{1}, \mathbbm{1}\right)$-scalable, then
    we can apply lower bound
    \begin{equation}
      \perm B = \perm (XAY) = \left(\prod_i x_i\right) \left(\prod_j y_j\right) perm A
    \end{equation}
  \end{block}
\end{frame}

\begin{frame}
  \frametitle{Matrix scaling as an optimization problem}
  \begin{itemize}
    \item \textbf{given:} $A, r, c$
    \item \textbf{find:} $X, Y$ such that $B=XAY$ fulfills $B\mathbbm{1}_m=r$ and $B \mathbbm{1}_n=c$.
    \item $m+n$ unknowns
          \item $m+n$ constraints
  \end{itemize}
  Consider the (\emph{nonconvex}) function
  \begin{equation}
    g(x,y) = \langle x, A y \rangle - \langle r, \log x \rangle - \langle c, \log y \rangle
  \end{equation}
  with derivative (coordinatewise)
  \begin{equation}
    \label{eq:grad-original-formulation}
    \begin{aligned}
      \nabla_x g(x,y) = Ay - \frac{r}{x} \\
      \nabla_y g(x,y) = A^T x - \frac{c}{y}
    \end{aligned}
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Reparametrizing this system}
  Via reparametrization $x= e^\xi$ and $y=e^{\eta}$ we get
  \begin{equation}
    f(\xi, \eta) = \sum_{i,j} a_{i,j} e^{\xi_i + \eta_j} - \langle r, \xi \rangle - \langle c, \eta \rangle
  \end{equation}
  which is \emph{convex}. It's gradient is given by
  \begin{equation}
    \label{eq:grad-convex-reformulation}
    \frac{\partial f}{\partial \xi_i} = \sum_{j=1}^{n} a_{i,j} e^{\xi_i + \eta_j} - r_i
  \end{equation}
  Easy to see that the optimality condition of~\eqref{eq:grad-convex-reformulation} and~\eqref{eq:grad-original-formulation} agree.
  Implies that even the nonconvex function only has \emph{global} minimizers.
\end{frame}

\begin{frame}
  \frametitle{Matrix scaling as an optimization problem [contd]}
  It is easy to see that a solution $(x,y)$ of
  \begin{equation}
    \begin{aligned}
      Ay - \frac{r}{x} = 0 \\
      A^T x - \frac{c}{y} = 0
    \end{aligned}
  \end{equation}
  defines a solution to the \emph{matrix scaling} problem via $X=\diag x$ and $Y=\diag y$
  \begin{equation}
    \left(\begin{array}{c}
            a_{1 1} y_1 + a_{1 2} y_2 + \cdots \\
            a_{2 1} y_1 + a_{2 2} y_2 + \cdots \\
            a_{m 1} y_1 + a_{m 2} y_2 + \cdots \\
          \end{array}\right)
        \begin{array}{c}
          \cdot x_1 = r_1\\
          \cdot x_2 = r_2\\
          \cdot x_m = r_m
        \end{array}
      \end{equation}
\end{frame}

\begin{frame}
  \frametitle{}
  The question remains: how to minimize
  \begin{equation}
    g(x,y) = \langle x, A y \rangle - \langle r, \log x \rangle - \langle c, \log y \rangle
  \end{equation}
  \onslide<2->{%
    \begin{block}{alternating minimiziation}
      Given a problem
      \begin{equation}
        \begin{aligned}
          &\min_{x,y} \phi(x,y)\\
          x_{k+1} &= \argmin_x \phi(x,y_k) \\
          y_{k+1} &= \argmin_y \phi(x_{k+1}, y)
        \end{aligned}
      \end{equation}
    \end{block}
    makes sense as long as the subproblems are easy (e.g. convex).
  }
  \begin{equation}
    g(x,y) = \langle x, A y \rangle - \langle r, \log x \rangle - \langle c, \log y \rangle
  \end{equation}
  \begin{equation}
    \text{opt. cond.\ for $x$} \quad Ay - \frac{r}{x} = 0
    \text{opt. cond.\ for $y$} \quad Ay - \frac{c}{y} = 0
  \end{equation}
\end{frame}

% change this to algorithmx package
\begin{frame}
  \frametitle{}

  \begin{block}{Sinkhorn '60}
    Given $(x_0, y_0)$, for $k=1,\dots$
    \begin{equation}
      \begin{aligned}
        x_{k+1}&= \frac{r}{A y_k} \\
        y_{k+1}&= \frac{r}{A x_{k+1}} \\
      \end{aligned}
    \end{equation}
  \end{block}
  Linear convergence if $a_{i,j} > 0$.
  \textbf{Q:} What if $A$ is not $(r,c)$-scalable?
  % Then the optimization problem has no solution as FOC are never fulfilled
\end{frame}

\end{document}


\subsection{Reals or rationals for analysis}

Why can't we use rationals for analysis?

If discontinous at not rational number, it can still be continous for all rationals.

eg \(f(x)=-1\) unless \(x^2>2\), where \(f(x)=1\).

Continous for all rationals, because rationals dense in reals

But can't be differentiated


%----------------------------------------------------------------------------------------
%	PACKAGES AND OTHER DOCUMENT CONFIGURATIONS
%----------------------------------------------------------------------------------------

\documentclass[paper=a4, fontsize=11pt]{scrartcl} % A4 paper and 11pt font size

\usepackage[T1]{fontenc} % Use 8-bit encoding that has 256 glyphs
\usepackage{fourier} % Use the Adobe Utopia font for the document - comment this line to return to the LaTeX default
\usepackage[english]{babel} % English language/hyphenation
\usepackage{amsmath,amsfonts,amsthm} % Math packages

\usepackage{sectsty} % Allows customizing section commands
\allsectionsfont{\centering \normalfont\scshape} % Make all sections centered, the default font and small caps

\usepackage{fancyhdr} % Custom headers and footers
\pagestyle{fancyplain} % Makes all pages in the document conform to the custom headers and footers
\fancyhead{} % No page header - if you want one, create it in the same way as the footers below
\fancyfoot[L]{} % Empty left footer
\fancyfoot[C]{} % Empty center footer
\fancyfoot[R]{\thepage} % Page numbering for right footer
\renewcommand{\headrulewidth}{0pt} % Remove header underlines
\renewcommand{\footrulewidth}{0pt} % Remove footer underlines
\setlength{\headheight}{13.6pt} % Customize the height of the header

\numberwithin{equation}{section} % Number equations within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)
\numberwithin{figure}{section} % Number figures within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)
\numberwithin{table}{section} % Number tables within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)

\setlength\parindent{0pt} % Removes all indentation from paragraphs - comment this line for an assignment with lots of text

%----------------------------------------------------------------------------------------
%	TITLE SECTION
%----------------------------------------------------------------------------------------

\newcommand{\horrule}[1]{\rule{\linewidth}{#1}} % Create horizontal rule command with 1 argument of height

% \title{
% \normalfont \normalsize
% \textsc{university, school or department name} \\ [25pt] % Your university, school and/or department name(s)
% \horrule{0.5pt} \\[0.4cm] % Thin top horizontal rule
% \huge Assignment Title \\ % The assignment title
% \horrule{2pt} \\[0.5cm] % Thick bottom horizontal rule
% }

% \author{John Smith} % Your name

% \date{\normalsize\today} % Today's date or a custom date

\begin{document}

% \maketitle % Print the title q

\section{Co-clustering by Block Value Decomposition}

This is a co-clustering algorithm called Block Value Decomposition (BVD) based on Nonnegative Matrix Factorization (NMF) technique. The goal is to find a factorization for the data matrix $X \in \mathbb{R}_{+}^{N \times M}$, where $N$ is the number of objects, $M$ is the number of features of these objects and the factorization takes the form $$X \approx USV^T$$, where $U \in \mathbb{R}_{+}^{N \times L}$ is a matrix of rows factors representing features clusters, $S \in \mathbb{R}_{+}^{L \times K}$ is a block matrix representing how blocks are related, and $V \in \mathbb{R}_{+}^{M \times K}$ is a matrix of columns factors representing rows clusters.

This algorithm solves the following optimization problem:
$$\textit{min } ||X - USV^T||^2 \textit{ s.t. } U \geq 0, S \geq 0, V \geq 0$$

The optimization problem can be solved using Lagrange multipliers ($\lambda$), optimizing the following Lagrange function:
$${\cal L} = |X - USV^T||^2 - tr(\lambda_1U^T) - tr(\lambda_2S^T) - tr(\lambda_3V^T)$$

Then ${\cal L}$ must satisfy the K.K.T. conditions:
$$\frac{\partial {\cal L}}{\partial U} = 0$$
$$\frac{\partial {\cal L}}{\partial S} = 0$$
$$\frac{\partial {\cal L}}{\partial V} = 0$$
$$\lambda_1 \odot U = 0$$
$$\lambda_2 \odot S = 0$$
$$\lambda_3 \odot V = 0$$

Solving the derivatives and equal them to $0$, is possible to solve the optimization problem by applying gradient ascending on ${\cal L}$ with the following update rules:
$$U \gets U \odot \frac{XVS^T}{USV^TVS^T}$$
$$V \gets V \odot \frac{U^TXV}{U^TUSV^TV}$$
$$S \gets S \odot \frac{S^TU^TX}{S^TU^TUSV^T}$$

\section{Fast Nonnegative Matrix Tri factorization}

In this case, the goal is to optimize the following problem:
$$\textit{min } ||X - USV^T||^2 \textit{ s.t. } U \in \Psi^{n \times k}, S \in \mathbb{R}_{+}^{l \times k}, V \in \Psi^{m \times l}$$ where $U$ and $V$ turns into cluster indicator matrices, with vectors $\vec{u_i}$ and $\vec{v_j}$ that contains $1s$ in only one position, indicating the cluster that that this vector belongs, and $0s$ in the rest.

Similar to the other algorithm, it optimizes $S$ with a multiplicative update rule and the following subproblems:
$$S \gets (U^TU)^{-1}U^TXV(V^TV)^{-1}$$
$$v_{ij} \left\{
\begin{array}{ll}
        1 & j = \textit{argmin}_l ||\vec{x_i} - \vec{\tilde{u_l}}||^2 \\
        0 & \textit{otherwise}
    \end{array}
\right.$$
$$u_{ij} \left\{
\begin{array}{ll}
        1 & i = \textit{argmin}_k ||\vec{x_j} - \vec{\tilde{v_k}}||^2 \\
        0 & \textit{otherwise}
    \end{array}
\right.$$
where $\tilde{U} = US$ and $\tilde{V} = SV^T$

\section{Overlapping Orthogonal Nonnegative Matrix Tri Factorization}

This is a proposal algorithm that aims to solve the following problem:
$$\textit{min } ||X - UV'||^{2}_{F}$$
$$\textit{s.t. } U^TU = I$$
$$\left[ \begin{array}{c} V^{(1)} \\ V^{(2)} \\ \vdots \\ V^{(k)} \end{array} \right]^T\left[ \begin{array}{c} V^{(1)} \\ V^{(2)} \\ \vdots \\ V^{(k)} \end{array} \right] = I$$
$$\left[ \begin{array}{c} V^{(1)} \\ V^{(2)} \\ \vdots \\ V^{(k)} \end{array} \right] \geq 0$$
$$U, S \geq 0$$
where $V^{(c)} \in \mathbb{R}^{M \times L}$ and $V^{'}_{c \cdot} = S_{c \cdot} V^{(c)^T}$, $\forall c \in \{1, \dots, k\}$.

This way the objects (lines) in $X$ can belong to multiple clusters.

\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% PROBLEM 2 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 2}

A neutron beam with an intensity of $2 \times 10^{12}\text{ neutrons/(cm}^2{\cdot}\text{s)}$ is incident on an unknown shielding material and has a beam spot of 5 cm$^2$. The shielding material has a thickness of 10 cm.
\begin{enumerate}[a)]
\item On average, $3.0\times10^9$ neutrons/s make it through the shield uncollided. What is the macroscopic cross section of the shield material?
\item What is the mean free path of a neutron in the shielding material?
\item If a single beam pulse is 10 $\mu$s, how many collisions are expected to take place in the shielding material?
\end{enumerate}


% \documentclass[draft,11pt]{article}
\chapter{Different Perspectives on Gaussian
  Elimination}
\label{cha:ge}
% \usepackage{cleveref}
%
%\allowdisplaybreaks

%%% for this lecture
%\newcommand\gap{\text{gap}}
%\newcommand{\Hongjie}[1]{{\color{red} Hongjie: #1}}

%\begin{document}
\sloppy
%\lecture{7 --- Wednesday, April 1}
%{Spring 2020}{Rasmus Kyng}{More Gaussian
  %Elimination,
  %Introduction to Random Matrix Concentration}

\section{An Optimization View of Gaussian Elimination for Laplacians}
In this section, we will explore how to exactly minimize a Laplacian
quadratic form by minimizing over one variable at a time.
It turns out that this is in fact Gaussian Elimination in disguise --
or, more precisely, the variant of Gaussian elimination that we tend
to use on symmetric matrices, which is called Cholesky factorization.

Consider a Laplacian $\LL$ of a connected graph $G = (V,E,\ww)$, where
$\ww \in \R^E$ is a vector of positive edge weights.
Let $\WW \in \R^{E \times E}$ be the diagonal matrix with the edge
weights on the diagonal, i.e. $\WW = \diag(\ww)$ and
$\LL = \BB\WW\BB^\trp$.
Let $\dd \in \R^V$ be a demand vector s.t.\ $ \dd \perp \vecone$.

Let us define an energy
\[
  \energy(\xx) =
-\dd^\trp\xx +
  \frac{1}{2}\xx^\trp\LL\xx
\]
Note that this function is convex and is minimized at $\xx$ s.t.\ $\LL
\xx = \dd$.

We will now explore an approach to solving the minimization problem
\[
\min_{\xx \in \R^V } \energy(\xx)
  \]
%
Let $\xx =
\begin{pmatrix}
  y \\ \zz
\end{pmatrix}$ where $y \in \R$ and $\zz \in \R^{V \setminus
  \setof{1}}$.

We will now explore how to minimize over $y$, given any $\zz$.
Once we find an expression for $y$ in terms of $\zz$,
we will be able to reduce  it to a
new quadratic minimization problem in $\zz$,
\[
  \energy'(\zz) =
-\dd'^\trp\zz +
  \frac{1}{2}\zz^\trp\LL'\zz
\]
where $\dd'$ is a demand vector on the remaining vertices, with $\dd
\perp \vecone$ and $\LL'$ is a Laplacian of a graph on the remaining
vertices $V' = V \setminus \setof{1}$.
We can then repeat the procedure to eliminate another variable and so on.
Eventually, we can then find all the solution to our original
minimization problem.

To help us understand how to minimize over the first variable, we
introduce some notation for the first row and column of the Laplacian:
\begin{equation}
  \label{eq:lapuniformlayout}
  \LL =
\begin{pmatrix}
W & -\aa^\trp \\
-\aa& \diag(\aa) + \LL_{-1}
\end{pmatrix}.
\end{equation}
Note that $W$ is the weighted degree of vertex 1, and that
\begin{equation}
\begin{pmatrix}
W & -\aa^\trp \\
-\aa& \diag(\aa)
\end{pmatrix}
\end{equation}
is the Laplacian of the subgraph of $G$ containing only the edges incident on
vertex 1, while $\LL_{-1}$ is the Laplacian of the subgraph of $G$
containing all edges \emph{not} incident on vertex 1.

Let us also write $\dd =
\begin{pmatrix}
  b \\ \cc
\end{pmatrix}$ where $b \in \R$ and $\cc \in \R^{V \setminus
  \setof{1}}$.

Now,
\begin{align*}
  \energy(\xx)
  &=
  -\dd^\trp\xx +
  \frac{1}{2}\xx^\trp\LL\xx
  =
  -\begin{pmatrix}
  b \\ \cc
\end{pmatrix}^\trp \begin{pmatrix}
  y \\ \zz
\end{pmatrix}
+
  \frac{1}{2}\begin{pmatrix}
  y \\ \zz
\end{pmatrix}^\trp
\left(
\begin{array}{ccc}
W & -\aa^\trp \\
-\aa& \diag(\aa) + \LL_{-1}
\end{array} \right)
\begin{pmatrix}
  y \\ \zz
\end{pmatrix}
  \\
  &=
    - by  -\cc^\trp\zz
    +
    \frac{1}{2}
    \left(
    y^2 W - 2 y \aa^\trp \zz
    +
    \zz^\trp \diag(\aa) \zz
    +
    \zz^\trp \LL_{-1} \zz
    \right).
\end{align*}
Now, to minimize over $y$, we set $\frac{\partial  }{\partial  y}
\energy(\xx) = 0$ and get
\[
    - b
    +
    y W - \aa^\trp \zz
    = 0
   .
  \]
  Solving for $y$, we get that the minimizing $y$ is
\begin{equation}
\label{eq:laponevarmin}
y = \frac{1}{W}(b + \aa^\trp \zz).
\end{equation}

Observe that
\begin{align*}
  \energy(\xx)
  &=
    - by  -\cc^\trp\zz
    +
    \frac{1}{2}
    \left(
    y^2 W - 2 y \aa^\trp \zz
    +
    \zz^\trp \diag(\aa) \zz
    +
    \zz^\trp \LL_{-1} \zz
    \right)
  \\* % the * prohibits page break
  &=
  - by  -\cc^\trp\zz
    +
    \frac{1}{2}
    \left(
    \frac{1}{W}( yW - \aa^\trp \zz )^2
    \underbrace{
    -
    \frac{1}{W}\zz^\trp \aa\aa^\trp \zz
    +
    \zz^\trp \diag(\aa) \zz
    +
    \zz^\trp \LL_{-1} \zz
    }_{\text{Let } \SS = \diag(\aa) - \frac{1}{W}\aa\aa^\trp +\LL_{-1}  }
    \right)
  \\* % the * prohibits page break
  &=
  - by  -\cc^\trp\zz
    +
    \frac{1}{2}
    \left(
    \frac{1}{W}( yW - \aa^\trp \zz )^2
    +
    \zz^\trp\SS \zz
    \right),
\end{align*}
where we simplified the expression by defining $\SS = \diag(\aa) -
\frac{1}{W}\aa\aa^\trp +\LL_{-1}$.
Plugging in ${y = \frac{1}{W}(b + \aa^\trp \zz),}$
we get
\begin{align*}
  \min_{y}
  \energy
  \begin{pmatrix}
  y \\ \zz
\end{pmatrix}
  &=
  -\left(\cc +  b \frac{1}{W}\aa \right)^\trp\zz
    -
    \frac{b^2}{2W}
    +
    \frac{1}{2}
    \zz^\trp\SS \zz
    .
\end{align*}
Now, we define $\dd' = \cc +  b \frac{1}{W}\aa$
and $  \energy'(\zz) =
-\dd'^\trp\zz +
\frac{1}{2}\zz^\trp\SS\zz$.
And, we can see that
\begin{align*}
  \argmin_{\zz} \min_{y}
  \energy
  \begin{pmatrix}
  y \\ \zz
\end{pmatrix}
  =
\argmin_{\zz}
    \energy'(\zz),
\end{align*}
since dropping the constant term $    -
    \frac{b^2}{2W}$ does not change what the minimizing $\zz$ values
    are.
\begin{claim}
\label{clm:optimgaussclosed}
\noindent
  \begin{enumerate}
  \item $ \dd' \perp \vecone$
  \item $ \SS = \diag(\aa) - \frac{1}{W}\aa\aa^\trp +\LL_{-1} $ is a
    Laplacian of a graph on the vertex set $V\setminus\setof{1}$.
  \end{enumerate}
\end{claim}
We will prove Claim~\ref{clm:optimgaussclosed} in a moment.
From the Claim, we see that the problem of finding $\argmin_{\zz}
    \energy'(\zz)$, is exactly of the same form as finding $\argmin_{\xx}
    \energy(\xx)$, but with one fewer variables.

We can get a minimizing $\xx$ that solves $\argmin_{\xx}
    \energy(\xx)$ by repeating the variable elimination procedure until we get
down to a single variable and finding its value.
We then have to work back up to getting a solution for $\zz$, and then
substitute that into Equation~\eqref{eq:laponevarmin} to get the value
for $y$.
\begin{remark}
  In fact, this perspective on Gaussian elimination also makes sense for any
  positive definite matrix.
  In this setting, minimizing over one variable will leave us with
  another positive definite quadratic minimization problem.
\end{remark}

%
\begin{proof}[Proof of Claim~\ref{clm:optimgaussclosed}]
To establish the first part, we note that  $\vecone^\trp \dd' = \vecone^\trp \cc + b\frac{\vecone^\trp\aa}{W} =
\vecone^\trp \cc + b= \vecone^\trp \dd = 0$.
To establish the second part, we notice that $\LL_{-1}$ is a graph
Laplacian by definition.
Since the sum of two graph Laplacians is another graph Laplacian, it
now suffices to show that $\SS $ is a graph Laplacian.
\begin{claim}
\label{clm:lapconditions}
  A matrix $\MM$ is a graph Laplacian if and only it satisfies the following conditions:
  \begin{itemize}
  \item  $\MM^\trp = \MM$.
  \item The diagonal entries of $\MM$ are non-negative, and the
    off-diagonal entries of $\MM$ are non-positive.
  \item $\MM \vecone = \veczero$.
  \end{itemize}
\end{claim}
Let's see that Claim~\ref{clm:lapconditions} is true. Firstly, when the conditions hold we
can write $\MM = \DD - \AA$ where $\DD$ is diagonal and non-negative,
and $\AA$ is non-negative, symmetric, and zero on the diagonal, and from the last condition
$\DD(i,i) = \sum_{j \neq i} \AA(i,j)$.
Thus we can view $\AA$ as a graph adjacency matrix and $\DD$ as the
corresponding diagonal matrix of weighted degrees.
Secondly, it is easy to check that the conditions hold for any graph
Laplacian, so the conditions indeed hold if and only if.
Now we have to check that the claim applies to $\SS$. We leave
this as an exercise for the reader.

Finally, we want to argue that the graph corresponding to $\SS$ is
connected.
Consider any $i,j \in V\setminus\setof{1}$.
Since $G$, the graph of $\LL$, is connected, there exists a simple path in
$G$ connecting $i$ and $j$.
If this path does not use vertex $1$, it is a path in the graph of $\LL_{-1}$
and hence in the graph of $\SS$.
If the path does use vertex $1$, it must do so by reaching the vertex
on some edge $(v,1)$ and leaving on a different edge $(1,u)$.
Replace this pair of edges with edge $(u,v)$, which appears in the graph of
$\SS$ because $\SS(u,v) < 0$. Now we have a path in the graph of $\SS$.
\end{proof}

\begin{figure}[h]
  \centering
  \includegraphics[width=1
  \textwidth]{fig/lecture7_schur-clique.jpeg}
% \caption{Plotting $\exp(z)$ compared to $1+z+z^2$.}
\label{fig:schurclique}
\end{figure}

\section{An Additive View of Gaussian Elimination}

\paragraph{Cholesky decomposition basics.}  Again we consider a graph Laplacian $\LL
\in \R^{n \times n}$ of a conected graph $G = (V,E,\ww)$, where as usual
$\abs{V} = n$ and $\abs{E} = m$.

In this Section, we'll study how to decompose a graph Laplacian as $\LL = \matlow \matlow^\trp$, where $\matlow \in \R^{n \times n}$
is a lower triangular matrix, i.e.
$\matlow(i,j) = 0$ for $i < j$.
Such a factorization is called a Cholesky
decomposition. It is essentially the result of Gaussian elimination with a slight twist
to ensure the matrices maintained at intermediate steps of the
algorithm remain symmetric.

We use $\nnz(\AA)$ to denote the number of non-zero entries of matrix $\AA$.
\begin{lemma}
Given an \emph{invertible} square lower triangular matrix $\matlow$,
we can solve the linear equation $\matlow \yy = \bb$ in time
$O(\nnz(\matlow))$.
Similarly, given an upper triangular matrix $\matup$, we can solve
linear equations $\matup \zz = \cc$ in time $O(\nnz(\matup))$.
\end{lemma}
We omit the proof, which is a straight-forward exercise.
The algorithms for solving linear equations in upper and lower
triangular matrices are known as forward and back substitution respectively.
\begin{remark}
  Strictly speaking, the lemma requires us to have access an adjacency
  list representation of $\matlow$ so that we can quickly tell where
  the non-zero entries are.
\end{remark}

Using forward and back substitution, if we have a decomposition of an
invertible matrix $\MM$ into $\MM = \matlow \matlow^\trp$, we can now
solve linear equations in $\MM$ in time $O(\nnz(\matlow))$.

\begin{remark}
 We have learned about decompositions using a lower triangular matrix, and later we
 will see an algorithm for computing these.
 In fact, we can have more flexibility than that. From an algorithmic perspective, it is
 sufficient that there exists a permutation matrix $\PP$ s.t. $\PP
 \matlow \PP^{\trp}$ is lower triangular. If we know the ordering
 under which the matrix becomes lower triangular, we can perform
 substitution according to that order to solve linear equations in the
 matrix without having to explicitly apply a permutation to the matrix.
\end{remark}

\paragraph{Dealing with pseudoinverses.} But how can we solve a linear equation in $\LL = \matlow
\matlow^\trp$, where $\LL$ is not invertible? For graph
Laplacians we have a simple characterization the kernel, and because
of this, dealing with the lack of invertibility turns out
to be fairly easy.

We can use the following lemma which you will prove in an exercise
next week.
\begin{lemma}
  Consider a real symmetric matrix $\MM = \XX \YY \XX^\trp$, where
  $\XX$ is real and invertible and
  $\YY$ is real symmetric.
  Let $\proj_{\MM}$ denote the orthogonal projection to the image
  of $\MM$.
  Then $\MM^{\pinv} = \proj_{\MM} (\XX^\trp)^{-1}\YY^{\pinv} \XX^{-1}\proj_{\MM}$.
\end{lemma}

The factorizations $\LL = \matlow \matlow^\trp$ that we produce will
have the property that all diagonal entries of $\matlow$ are strictly
non-zero, except that $\matlow(n,n) = 0$.
From let us $\matlowhat$ as the matrix whose entries
agree with $\matlow$, except that $\matlowhat(n,n) = 1$.
Let $\calDD$ be the diagonal matrix with $\calDD(i,i) = 1$ for $i < n$
and $\calDD(n,n) = 0$.
Then $\matlow \matlow^{\trp} = \matlowhat\calDD
\matlowhat^{\trp}$, and $\matlowhat$ is invertible, and $\calDD^{\pinv} = \calDD$.
Finally, $\proj_{\LL} = \II - \frac{1}{n}\vecone\vecone^{\trp}$,
because this matrix is acts like identity on vectors orthogonal to
$\vecone$ and ensures $\proj_{\LL} \vecone = \veczero$,
and this matrix can be applied to a vector in $O(n)$ time.
Thus $\LL^{\pinv}  = \proj_{\LL}(\matlowhat^\trp)^{-1}\calDD
\matlowhat^{-1}\proj_{\LL}$, and this matrix can be applied in time $O(\nnz(\matlow))$.

\paragraph{An additive view of Gaussian Elimination.}
The following theorem describes Gaussian Elimination / Cholesky
decompostion of a graph Laplacian.
\begin{theorem}[Cholesky Decomposition on graph Laplacians]
 Let $\LL
 \in \R^{n \times n}$ be a graph Laplacian of a connected
 graph $G = (V,E,\ww)$, where
$\abs{V} = n$.
 Using Gaussian Elimination, we can compute in $O(n^3)$ time a
 factorization $\LL = \matlow\matlow^{\trp}$ where $\matlow$ is lower
 triangular, and has positive diagonal entries except $\matlow(n,n) = 0$.
\end{theorem}
\begin{proof}
  Let $\LL^{(0)} = \LL$.
We will use $\AA(:,i)$ to denote the the $i$th column of a matrix $\AA$.
Now, for $i = 1$ to $i = n-1$ we define
\[
  \ll_i = \frac{1}{\sqrt{\LL^{(i-1)}(i,i)}} \LL^{(i-1)}(:,i)
  \text{ and }
   \LL^{(i)}  = \LL^{(i-1)} - \ll_i \ll_i^{\trp}
 \]
 Finally, we let $\ll_n = \veczero_{n \times 1}$.
 We will show later that
 \begin{equation}
   \label{eq:matlowlastentry}
   \LL^{(n-1)} = \matzero_{n \times n}.
 \end{equation}
 It follows that $\LL = \sum_{i} \ll_i \ll_i^{\trp}$, provided this
 procedure is well-defined, i.e. $\LL^{(i-1)}(i,i) \neq 0$ for all $i
< n$. We will sketch a proof of this later, while also establishing
several other properties of the procedure.

Given a matrix $\AA \in \R^{n \times n}$ and $U \subseteq [n]$, we will use $\AA(U, U)$ to
denote the principal submatrix of $\AA$ obtained by restricting to the
rows and columns with index in $U$, i.e. all entries $\AA(i,j)$ where
$i,j \in U$.
\begin{claim}
  \label{clm:schurconnected}
  Fix some $i < n$.
  Let $U = \setof{i+1,\ldots,n}$.
  Then $\LL^{(i)}(i,j) = 0$ if $i \not\in U$ or $j \not\in U$.
  And $\LL^{(i)}(U, U)$ is a graph Laplacian of a connected graph on
  the vertex set $U$.
\end{claim}
From this claim, it follows that $\LL^{(i-1)}(i,i) \neq 0$ for $i<n-1$,
since a connected graph Laplacian on a graph with $\abs{U} > 1$
vertices cannot have a zero on the diagonal, and it follows that
$\LL^{(n-1)}(i,i) = 0$, because the only graph we allow on one vertex
is the empty graph. This shows Equation~\eqref{eq:matlowlastentry} holds.
\end{proof}

\begin{proof}[Sketch of proof of Claim~\ref{clm:schurconnected}]
  % We now sketch a proof of Claim~\ref{clm:schurconnected}.
We will focus on the first elimination, as the remaining are similar.
Adopting the same notation as in Equation~\eqref{eq:lapuniformlayout},
we write
 \[
\LL^{(0)} = \LL  =
\left(
\begin{array}{ccc}
W & -\aa^\trp \\
-\aa& \diag(\aa) + \LL_{-1}
\end{array} \right)
\]
and, noting that
\[
  \ll_1 \ll_1^{\trp} =
  \begin{pmatrix}
   W & -\aa^\trp \\
-\aa & \frac{1}{W} \aa \aa^{\trp}
\end{pmatrix}
\]
we see that
\[
  \LL^{(1)}  = \LL^{(0)} - \ll_1 \ll_1^{\trp} =
  \begin{pmatrix}
   0 & \veczero \\
\veczero & \diag(\aa) - \frac{1}{W} \aa \aa^{\trp}  + \LL_{-1}
\end{pmatrix}
.
\]
Thus the first row and column of $\LL^{(1)}$ are zero claimed.
It also follows by Claim~\ref{clm:optimgaussclosed} that
$\LL^{(1)}(\setof{2,\ldots,n}, \setof{2,\ldots,n})$ is the
Laplacian of a connected graph.
This proves Claim~\ref{clm:schurconnected} for the case $i = 1$.
An induction following the same pattern can be used to prove the claim
for all $i < n$.
\end{proof}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "agao21_script"
%%% TeX-engine: luatex
%%% End:

% \section{Gaussian Elimination Recap and Structure Claim}

% %Last time, we studied a convex function minimization problem, and saw
% %how solve it by coordinatewise minimizaiton, which I claimed is really
% %Gaussian elimination in disguise.
% %Let's recap part of what we saw.

% Consider a Laplacian $\LL$ of a connected graph $G = (V,E,\ww)$, where
% $\ww \in \R^E$ is a vector of positive edge weights.
% Let $\WW \in \R^{E \times E}$ be the diagonal matrix with the edge
% weights on the diagonal, i.e. $\WW = \diag(\ww)$ and
% $\LL = \BB\WW\BB^\trp$.
% Let $\dd \in \R^V$ be a demand vector s.t. $ \dd \perp \vecone$.

% We defined an energy
% \[
%   \energy(\xx) =
% -\dd^\trp\xx +
%   \frac{1}{2}\xx^\trp\LL\xx
% \]

% Note that this function is convex and is minimized at $\xx$ s.t. $\LL
% \xx = \dd$.

% To understand how to minimize over the first variable, we
% introduce some notation for the first row and column of the Laplacian:
% \begin{equation}
%   \label{eq:lapuniformlayout}
%   \LL =
% \left(
% \begin{array}{ccc}
% W & -\aa^\trp \\
% -\aa& \diag(\aa) + \LL_{-1}
% \end{array} \right)
% \end{equation}
% Note that $W$ is the weighted degree of vertex 1, and that
% \begin{equation}
% \begin{pmatrix}
% W & -\aa^\trp \\
% -\aa& \diag(\aa)
% \end{pmatrix}
% \end{equation}
% is the Laplacian of the subgraph of $G$ containing only the edges incident on
% vertex 1, while $\LL_{-1}$ is the Laplacian of the subgraph of $G$
% containing all edges \emph{not} incident on vertex 1.

% Let us also write $\dd =
% \begin{pmatrix}
%   b \\ \cc
% \end{pmatrix}$ where $y \in \R$ and $\cc \in \R^{V \setminus
%   \setof{1}}$.

% Now,
% \begin{align*}
%   \energy(\xx)
%   &=
%   -\dd^\trp\xx +
%   \frac{1}{2}\xx^\trp\LL\xx
%   =
%   -\begin{pmatrix}
%   b \\ \cc
% \end{pmatrix}^\trp \begin{pmatrix}
%   y \\ \zz
% \end{pmatrix}
% +
%   \frac{1}{2}\begin{pmatrix}
%   y \\ \zz
% \end{pmatrix}^\trp
% \left(
% \begin{array}{ccc}
% W & -\aa^\trp \\
% -\aa& \diag(\aa) + \LL_{-1}
% \end{array} \right)
% \begin{pmatrix}
%   y \\ \zz
% \end{pmatrix}
% \end{align*}
% Now, to minimize over $y$, we set $\frac{\partial  }{\partial  y}
% \energy(\xx) = 0$ and get
% \[
%     - b
%     +
%     y W - \aa^\trp \zz
%     = 0
%    .
%   \]
%   Solving for $y$, we get that the minimizing $y$ is
% \begin{equation}
% \label{eq:laponevarmin}
% y = \frac{1}{W}(b + \aa^\trp \zz).
% \end{equation}
% By substituting in this value of $y$, we found
% \begin{align*}
%   \min_{y}
%   \energy
%   \begin{pmatrix}
%   y \\ \zz
% \end{pmatrix}
%   &=
%   -\left(\cc +  b \frac{1}{W}\aa \right)^\trp\zz
%     -
%     \frac{b^2}{2W}
%     +
%     \frac{1}{2}
%     \zz^\trp\SS \zz
%     .
% \end{align*}
% where
%  $\SS = \diag(\aa) -
%  \frac{1}{W}\aa\aa^\trp +\LL_{-1}$.
%  We let $\dd' = \cc +  b \frac{1}{W}\aa$, and $\energy'(\zz) =   -\dd^\trp\zz
% +
%     \frac{1}{2}
%     \zz^\trp\SS \zz
%     $, and noted that
% \begin{align*}
%   \argmin_{\zz} \min_{y}
%   \energy
%   \begin{pmatrix}
%   y \\ \zz
% \end{pmatrix}
%   =
% \argmin_{\zz}
%     \energy'(\zz),
% \end{align*}
%  and we ended the lecture by stating with the following claim.
%  \begin{claim}
% \label{clm:optimgaussclosed}
% \noindent
%   \begin{enumerate}
%   \item $ \dd' \perp \vecone$ when $\dd \perp \vecone$.
%   \item $\SS = \diag(\aa) - \frac{1}{W}\aa\aa^\trp +\LL_{-1} $ is a
%     Laplacian of a connected graph on the vertex set $V\setminus\setof{1}$.
%   \end{enumerate}
% \end{claim}
% Now we're ready to prove it.


\chapter{Conclusion}

Based on the research that was carried out is it clear now that procedural content generation (PCG) is a vast and very complicated field in computer science. We have observed a number of different techniques that can be used to generate content, each with their advantages and disadvantages. Amongst all these techniques are there is no single best PCG algorithm. Every content generator generally focuses on a particular type of environment and so a different technique should be used depending on the desired result. For example, using a search-based or rhythm-based method to generate mountains is probably not the best approach. Because of this, picking what kind of algorithm to use and how to implement it is very crucial.

To generate random content, one must have access to some sort of random elements. The randomness of the content is moderated through the use of random integers and floating-point numbers. Getting access to pure random numbers, meaning numbers that are truly random, is not really possible in computing. Instead, we use random number generators that create pseudo-random numbers. These numbers are entirely deterministic (apart from the time-based seed) but, if good parameters are used, they will have a distribution that appears to be random \citep{yu_random}.

Pseudo-random number generators including the Coupled Congruential Generator, the Mersenne Twister and the Blum Blum Shub are suitable in the field of cryptography. However cryptographically secure number generators are not required in simulations and therefore Linear Congruential Generators (LCG) are generally good enough for the use of procedural content generation.

Finding and picking out a good LCG is an exceptionally difficult task to manoeuvre around. LCGs require a period that is as long as possible because those with a short period have historically caused a few issues. Large word-sizes for the modulo must be used along with other constraints in order to guarantee this long period. However a LCG with long period will not necessary provide good results. Statistical tests need to be performed on a LCG to insure that it produces numbers with adequate statistical randomness. Many different LCGs were examined in order to chose a satisfying number generator.

In addition to a good number generator, generating content requires the help of a solid and well-detailed mathematical model describing the patterns of the environment as closely as possible. These models vary tremendously for different types of environments and can be very difficult to conceptualise due to their complexity. For instance, to generate mountain-like environment Perlin noise can be used to generate a random height-map. Multiple random sine waves can also be used to generate a wavy height-map of a 2D game. Rhythm-based approaches can be used whenever timing is a crucial factor, such as 2D side-scroll platform games \citep{DBLP:journals/tciaig/SmithWMTMC11}.

The model used to generate a dungeon-like environment is merely a tangent of maze generation. Mazes on their own already have a very elaborate theory established. Properties such as perfection, sparseness, biased and river are rigorously defined. There are multiple different algorithms that can be used to generate a perfect maze. Some of these algorithms are very similar to one another, such as the Depth-first search and Hunt-and-Kill algorithms, the Aldous-Broder and Wilson's algorithm, or the Binary-tree and Sidewinder. On the other hand some algorithms, such as the Recursive Division algorithm, are entirely different to all other algorithms. All these different algorithms have all been conceived in such a way that they all generate perfect mazes, however they all produce different types of mazes that can look very distinctive to one another.

Because of this wide variety of maze generators, picking which one to use is not obvious. The depth-first search and Hunt-and-Kill algorithm both have a large river factor which makes generally makes them produce more satisfying results \citep{JBuck}. However, algorithms like the Binary-tree algorithm are highly biased \citep{ThinkLabyrinth} which can potentially make them useful to one who wants to generate a dungeon that targets a specific direction.

Generating a basic mesh for the maze is not too complicated if we simply use 3D boxes to model the walls and tiled-textures for the UV mapping. From a programming perspective this is an efficient way to procedurally model the dungeon. But from an aesthetic point of view the result is not very satisfying. The aesthetic can be improved upon by using an occupancy-based algorithm; the method chosen consists of adding 3D meshes that are modelled by humans.

Despite the efforts, there is too much repetition and the dungeon looks similar almost everywhere. This is one of the major problems with PCG which was discussed very early on. PCG simply cannot produce levels that are unique. Even though levels are generated with different seeds and may look different they will still have obvious similarities. Some things can be done to reduce the repetition effect but it can never be blotted out completely. For instance, occupancy allows level designers to make specific parts of level look unique but chunks can still be re-used in several levels or within a level itself.

Overall we noticed that creating a generic framework for content generation is not simple because the possibilities are only limited to the imagination of designers. The algorithm used in the developed framework is essentially completely blank at the start and it is up to the user to define precisely how it should behave. There is an apparent reason for this: as specific details and constraints are added to desired result the generator's algorithm become increasingly more complex. In fact a slight change in the specification of the desired result can have a major impact on the algorithm used. Therefore resorting to a game-specific algorithm  might not always be a bad idea and this is probably why most game-developers focus on this.


\lab{Algorithms}{Policy Function Iteration}{Policy Function Iteration}
\objective{This section teaches how to improve dynamic programming convergence using policy function iteration.}

Now that we have covered how to solve simple dynamic programming problems by value function iteration, we consider the convergence of the algorithm.  We demonstrate two other methods known as policy function iteration, or Howard's Improvement, and modified policy function iteration.

\section*{Policy Function Iteration}
For infinite horizon dynamic programming problems, it can be shown that value function iteration converges at the rate $\beta$, where $\beta$ is the discount factor.  In practice, $\beta$ is usually close to one which means this algorithm often converges slowly.

In order to examine the value function iteration algorithm, it is helpful to see which functions take the most runtime.
\begin{problem}
\label{prob:profile}
In IPython, enter
\begin{lstlisting}
%run -p -s cum Value_Function_Iteration.py
\end{lstlisting}
where \texttt{Value\_Function\_Iteration.py} is the name of your script that solves the infinite horizon problem from the Value Function Iteration lab.  This will list the function calls made by your code, sorted by the cumulative time it spends within each function (including time spent in sub-functions).
Run the same command, this time changing the number of grid points $N$ to be 1000.
Run the command once more, this time setting $N=1000$ and $\beta = .95$.
\end{problem}

In Problem \ref{prob:profile} you should have noticed that runtime was significantly longer to run for larger $N$ or $\beta$ closer to 1.  The profiler gives more detailed information than just the overall runtime, however.  The results of Problem \ref{prob:profile} should look something like the following.
\begin{verbatim}
%run -p -s cum Value_Function_Iteration.py
         622 function calls in 2.542 seconds

   Ordered by: cumulative time

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
        1    0.000    0.000    3.065    3.065 <string>:1(<module>)
        1    0.001    0.001    3.065    3.065 {execfile}
        1    0.955    0.955    3.064    3.064 Value_Function_Iteration.py:5(<module>)
       59    0.000    0.000    1.418    0.024 fromnumeric.py:683(argmax)
       59    1.417    0.024    1.417    0.024 {method 'argmax' of 'numpy.ndarray' objects}
       59    0.001    0.000    0.613    0.010 fromnumeric.py:1774(amax)
       59    0.612    0.010    0.612    0.010 {method 'max' of 'numpy.ndarray' objects}
\end{verbatim}
We notice that the most time was spent in the maximization step.  Remember, the value function iteration method maximizes $V$ (and determines the corresponding $\psi$) at every step.  Because we find both a new $V$ and a new $\psi$ at every step, we only apply the new policy function for one iteration.  This gives us a crude approximation to the value function that corresponds to the new policy function, resulting in slow convergence of the value function.  Instead, we might consider finding the exact value function associated with each new policy function.

This is the idea behind the policy function iteration algorithm.  In this way we iterate on the policy functions rather than the value functions.  The algorithm for the policy function iteration can be summarized as follows:
\begin{enumerate}
\item Set an initial policy rule $W' = \psi_0(W)$ and a tolerance $\delta$.

\item \label{item:step2} Compute the value function assuming this rule is used forever:
\begin{equation*}
V(W_0) = \sum_{t=0}^\infty \beta^t u(\psi_k(W)-W)
\end{equation*}

\item Determine a new policy $\psi_{k+1}$ so that
\begin{equation*}
\psi_{k+1}(W) = \text{argmax}_{W'} u(W-W') + \beta V(W-W')
\end{equation*}

\item If $\delta_k = ||\psi_{k+1} - \psi_k|| < \delta$, stop, otherwise go back to step \ref{item:step2} with subscript $k+1$.
\end{enumerate}
In order to compute the value function, $V_k$ corresponding to a given policy $\psi_k$, we must solve
\begin{equation}
\label{Val_Fun}
V_k(W) = u(W-W') + \beta V_k(W')
\end{equation}
for $V_k$.

Once we have discretized $W$, equation \eqref{Val_Fun} is a linear system which we can rewrite as
\begin{equation*}
V_k(W) = u(W-W') + \beta QV_k(W)
\end{equation*}
where if $W$ is a vector of length $N$, then $Q$ is the $N\times N$ matrix
\begin{equation*}
Q_{ij} = \left\{
     \begin{array}{ll}
       1 & \text{if} \quad  W_j = W' = W_i\\
       0 & \text{otherwise}
     \end{array}
   \right.
\end{equation*}
Thus we have $V_k = (I-\beta Q)^{-1}u(W-W')$.  Although $Q$ may be large, we can take advantage of the fact that it is sparse, containing only $N$ nonzero entries out of $N^2$ total entries.

\begin{problem}
\label{prob:cake_eating_policyfun}
Solve the infinite horizon cake eating problem from the Value Function Iteration lab again, this time using policy function iteration.  In order to take advantage of the sparse matrices $I$ and $Q$, use the following imports from the SciPy \li{sparse} library.
\begin{lstlisting}
from scipy import sparse
from scipy.sparse import linalg
\end{lstlisting}
and the following code to initialize $I$ (outside the loop)
\begin{lstlisting}
I = sparse.identity(N)
\end{lstlisting}
and $Q$ (inside the loop)
\begin{lstlisting}
rows = np.arange(0, N)
columns = psi_ind
data = np.ones(N)
Q = sparse.coo_matrix((data, (rows, columns)), shape=(N, N))
Q = Q.tocsr()
\end{lstlisting}
where $N$ is the size of the $W$ grid.  Rather than compute $(I-\beta Q)^{-1}$ directly, use Scipy's sparse solver
\begin{lstlisting}
V = linalg.spsolve(I-beta*Q, u(W-W[psi_ind]))
\end{lstlisting}
where \texttt{psi\_ind} gives the indices of $W'$ for a given $W$ according to the current policy.
Take $N = 1000$ and $\beta = .95$.
Plot the policy function and compare with your policy function from the Value Function Iteration Lab.
\end{problem}

\section*{Modified Policy Function Iteration}
While policy function iteration converges in fewer iterations, solving the linear system can be slow, especially for problems with a large state space.  There is an alternative to this called modified policy function iteration.

In modified policy function iteration, we don't compute the exact value function corresponding to a policy.  Instead, at step \ref{item:step2} of the policy iteration algorithm we iterate $m$ times on the value function equation (Bellman equation) to get an approximation of the new value function.  This is faster than solving for the exact value function for large state spaces.  There is no strict rule on the value of $m$, the number of value function iterations.  In practice values such as $m=10$ or $m=15$ often work well.

Note that our methods for solving dynamic programs boil down to some combination of two things: iterating on the value function and iterating on the policy function.  Modified policy function does a combination of the two, taking advantage of the advantages of both methods.  Because modified policy iteration takes only slightly more work to code than value function iteration, it is often preferred in practice.  Whether policy or modified policy iteration will perform better may depend on the problem.

\begin{problem}
Solve the same problem as in problem \ref{prob:cake_eating_policyfun}, this time using the modified policy function iteration method with $m=15$.  In this case let convergence be determined in the same way (computing $\delta_k$) in the same way as in the value function iteration problem.

Use the same code as in Problem \ref{prob:cake_eating_policyfun} to initialize the sparse matrix $Q$.
\end{problem}

\begin{problem}
Solve the cake eating problem with each of the three methods and report how many iterations each takes.  Use $N= 1000$ as the number of grid points for $W$ and $\beta = 0.95$.  It is important that you use the same initial guess in each case in order to make the results comparable.  The accuracy of the initial guess greatly effects the number of iterations to convergence.  Take your initial guess as $V = 0$ which corresponds to an initial guess of the policy function with indices $[0,1,2,\ldots, N-1]$ (meaning $\psi = 0$).
\end{problem}

In general we should see that value function iteration takes more iterations than modified policy function iteration which in turn takes more iterations than policy function iteration.  It is important to note that this does not directly say anything about runtime.  Each iteration of policy iteration may take longer than an iteration of value function iteration.


\section*{Appendix}
  Let source $A \in \mathcal{V}$, sink $B \in \mathcal{V}$, capacity configuration $C$ and a candidate flow $X$. For $X$ to be
  valid with respect to $C$ the following two conditions must hold:
  \begin{align}
  \label{flow1}
    \forall (v, w) \in \mathcal{E}&, x_{vw} \leq c_{vw} \\
  \label{flow2}
    \forall v \in \mathcal{V} \setminus \{A,B\}&, \sum\limits_{w \in N^{+}(v)}x_{wv} = \sum\limits_{w \in N^{-}(v)}x_{vw}
  \end{align}

  \subimport{common/sepproofs/}{saturationsepproof.tex}

  \subimport{common/sepproofs/}{maxflowcontinuitysepproof.tex}

  \subimport{common/sepproofs/}{fcfscorrectness.tex}

  \subimport{common/sepproofs/}{fcfscomplexity.tex}

  \subimport{common/sepproofs/}{abscorrectness.tex}

  \subimport{common/sepproofs/}{abscomplexity.tex}

  \subimport{common/sepproofs/}{absDinfnormminproof.tex}

  \subimport{common/sepproofs/}{propcorrectness.tex}

  \subimport{common/sepproofs/}{propcomplexity.tex}

  \subimport{common/sepproofs/}{maxflowmonotonicitysepproof.tex}

  We will now prove that \texttt{BinSearch} returns the desired $\delta^*$ when we provide it with an appropriate interval as
  input.

  \ \\
  \subimport{common/sepproofs/}{dinfbinsearchcorrectness.tex}

  \subimport{common/sepproofs/}{dinfbinsearchcomplexity.tex}

  \subimport{common/sepproofs/}{dinfmincorrectness.tex}

  \subimport{common/sepproofs/}{dinfmincomplexity.tex}


\documentclass[12pt]{article}

\usepackage{mathpb}

\begin{document}

\section*{Examples}

\noindent\rule{\textwidth}{1pt}

\begin{verbatim}
\begin{theorem}{A}
    $\N \subset \Z \subset \Q \subset \R \subset \C \subset \Ha$
\end{theorem}
\end{verbatim}

\begin{theorem}{A}
$\N \subset \Z \subset \Q \subset \R \subset \C \subset \Ha$
\end{theorem}

\noindent\rule{\textwidth}{1pt}

\begin{verbatim}
\begin{exercise}{2}
    Let $z = \alpha + i\beta \in \C$,
    $\Rea(z) = \alpha$, $\Ima(z) = \beta$
\end{exercise}
\end{verbatim}

\begin{exercise}{2}
    Let $z = \alpha + i\beta \in \C$,
     $\Rea(z) = \alpha$, $\Ima(z) = \beta$
\end{exercise}

\noindent\rule{\textwidth}{1pt}

\begin{verbatim}
\begin{lemma}{3}
    Denote $\card(\N)  = \aleph_0$. Then $\card \R = 2^{\aleph_0}$
\end{lemma}
\end{verbatim}

\begin{lemma}{3}
    Denote $\card(\N)  = \aleph_0$. Then $\card \R = 2^{\aleph_0}$
\end{lemma}

\noindent\rule{\textwidth}{1pt}

\begin{verbatim}
\begin{problem}{4}
    Show that $T \in L(V,V)$ is injective
    if and only if $\Ker(T) = \left\{ 0 \right\}$
\end{problem}
\end{verbatim}

\begin{problem}{4}
    Let $T \in L(V.)$. Show $\Range(T) = V$  $\Ker(T) = \left\{ 0 \right\}$
\end{problem}

\noindent\rule{\textwidth}{1pt}

\begin{verbatim}
\begin{question}{5}
    Consider $\Id \in \mathcal{L}(V)$. What is $\Inv (\Id)$ ?
\end{question}
\end{verbatim}

\begin{question}{5}
    Consider $\Id \in S(n)$. What is $\Inv (\Id)$ ?
\end{question}

\noindent\rule{\textwidth}{1pt}

\begin{verbatim}
\begin{lemma}{2B}
    \[ \limto{x}{0} \frac{\sin(x)}{x} = 1 \]
\end{lemma}
\end{verbatim}

\begin{lemma}{2B}
\[ \limto{x}{0} \frac{\sin(x)}{x} = 1 \]
\end{lemma}

\noindent\rule{\textwidth}{1pt}

\begin{verbatim}
    \begin{lemma}{2C}
        Let $x,y \in V$, $\lambda \in F$. Then
        \innerp{\lambda x}{y} = \lambda \innerp{x,y}
    \end{lemma}
\end{verbatim}

\begin{lemma}{2C}
    Let $x,y \in V$, $\lambda \in F$. Then
      $\innerp{\lambda x}{y} = \lambda \innerp{x}{y}$
\end{lemma}

\noindent\rule{\textwidth}{1pt}

\end{document}


\chapter{Target Propagation}\label{targetprop}

\VUname{Target propagation} (sometimes abbreviated to targetprop) is an alternative approach to computing the gradients of a deep feedforward neural network. The idea dates back to \cite{lecun_learning_1986}, but has recently gained more interest (\cite{bengio_how_2014}, \cite{bengio_towards_2015}, \cite{lee_difference_2015}). Unlike the back-propagation algorithm, the target propagation algorithm is only defined for optimization of feedforward neural networks used in classification problems. For a high-level overview, the back-propagation algorithm can be summarized as follows: The error of the network is computed at the end of the network (after the loss function) and then back-propagated through the network to each layer, where it is used to update the layer parameters. The target propagation algorithm can be viewed as operating conversely: The desired output of the network is back-propagated through the network, creating a \VUname{target} for each layer. Such targets are then used to compute the error of each individual layer, which in turn is used to update the layer parameters.

\section{General target propagation overview}\label{targetprop_general}

Using notation from section \ref{backprop_application}, the principle of target propagation can be expressed as a problem of finding a target \( \widehat{\VUvec{h}}^{(i)} \) for each layer of the neural network so that this target minimizes the loss function of the network, that is
\begin{equation}\label{targetprop_loss_bound}
	L \left( \widehat{\VUvec{y}}, \VUvec{y} \right) = L \left( f^{(n)} \circ \dots \circ f^{(i + 1)} \left( \VUvec{h}^{(i)} \right), \VUvec{y} \right) > L \left( f^{(n)} \circ \dots \circ f^{(i + 1)} \left( \widehat{\VUvec{h}}^{(i)} \right), \VUvec{y} \right)
\end{equation}
With such a target, a \VUname{layer-local loss function} may be defined as \( L_i = L \left( \widehat{\VUvec{h}}^{(i)}, \VUvec{h}^{(i)} \right) \). This loss value can in turn be used to compute the gradients for the parameters of the layer as
\[ \nabla_{\VUvec{a}^{(i)}} J = \nabla_{\VUvec{h}^{(i)}} L \left( \widehat{\VUvec{h}}^{(i)}, \VUvec{h}^{(i)} \right) \odot {\sigma^{(i)}}' \left( \VUvec{a}^{(i)} \right)  = \nabla_{\VUvec{h}^{(i)}} L_i \odot {\sigma^{(i)}}' \left( \VUvec{a}^{(i)} \right) \]
\[ \nabla_{\VUmat{W}^{(i)}} J = \nabla_{\VUvec{a}^{(i)}} J {\VUvec{h}^{(i - 1)}}^T \]
\[ \nabla_{\VUvec{b}^{(i)}} J = \nabla_{\VUvec{a}^{(i)}} J \]
To summarize, once a proper target for each layer is chosen, the derivatives can be computed using only the input of the layer and the derivatives of the layer-local loss function and the activation function of the layer.

The open question remaining is how to choose the targets for the layers. For the last layer of the network, the obvious choice of target is
\begin{equation}\label{targetprop_last_layer_target}
	\widehat{\VUvec{h}}^{(n)} = \widehat{\VUvec{y}} - \eta \nabla_{\widehat{\VUvec{y}}} L \left( \widehat{\VUvec{y}}, \VUvec{y} \right)
\end{equation}
where \( \eta \) is a step size, i. e. a hyperparameter. Choosing \( \eta = 0.5 \) effectively makes \( \widehat{\VUvec{h}}^{(n)} \) the same as it would be for back-propagation.

If all the necessary functions were invertible, a trivial choice of the target for lower levels of the network would be the inverse image of the desired output \( \VUvec{y} \). Then the layer-local loss function value would be \( L_i = L \left( \widehat{\VUvec{h}}^{(i)}, \widehat{\VUvec{h}}^{(i)} \right) = 0 \). This trivial choice is, however, practically unfeasible -- the functions in question may not be invertible (e. g. when a ReLU activation function is used). Moreover, such targets may actually not be in the domains of the layer functions \( f^{(i)} \). To circumvent these issues, two variations of target propagation have been proposed. (\cite{bengio_how_2014} and \cite{lee_difference_2015})

\section{Vanilla target propagation}\label{vanilla_targetprop}

Vanilla target propagation tries to replace the actual inverse image by an approximation. That is, for each layer \( f^{(i)} \) there is defined an approximate inverse function \( \widetilde{f}^{(i)} \), such that
\[ f^{(i)} \left( \widetilde{f}^{(i)} \left( \VUvec{h}^{(i)} \right) \right) \approx \VUvec{h}^{(i)} \qquad \text{and} \qquad \widetilde{f}^{(i)} \left( f^{(i)} \left( \VUvec{h}^{(i - 1)} \right) \right) \approx \VUvec{h}^{(i - 1)} \]
The target can then be chosen as the approximate inverse image represented by
\[ \widehat{\VUvec{h}}^{(i - 1)} = \widetilde{f}^{(i)} \left( \widehat{\VUvec{h}}^{(i)} \right) \]
The approximate inverse function \( \widetilde{f}^{(i)} \) is realized by a neural network layer (in a sense introducing a dual neural network), that is
\[ \widetilde{f}^{(i)} \left( \VUvec{h}^{(i)} \right) = \widetilde{\sigma}^{(i)} \left( \left( {\widetilde{\VUmat{W}}}^{(i)} \right)^T \VUvec{h}^{(i)} + \widetilde{\VUvec{b}}^{(i)} \right) \]
This approach is grounded in the fact that using a neural network layer as the approximate inverse function makes the pair \( \left( f^{(i)}, \widetilde{f}^{(i)} \right) \) effectively an \VUname{auto-encoder} (\cite{bourlard_auto-association_1988}). This connection can be used to apply results holding for auto-encoders to target propagation.

Introducing another neural network allows the original neural network to be effectively trained using target propagation, but only at the cost of having to train the dual network. However, one of the aforementioned benefits of having the connection to auto-encoders is the fact that the dual network can be trained as a decoder layer in an auto-encoder -- training \( \widetilde{f}^{(i)} \) only to be a good predictor of \( {f^{(i)}}^{-1} \). This is achieved by introducing a \VUname{dual layer-local loss function}
\[ \widetilde{L}_i = L \left( \widetilde{f}^{(i)} \left( f^{(i)} \left( \VUvec{h}^{(i - 1)} \right) \right), \VUvec{h}^{(i - 1)} \right) \]
This choice would, however, only minimize the error at \( \VUvec{h}^{(i - 1)} \). It is more desirable to minimize the error not only at this singular point but also in its neighbourhood. To achieve this, random noise is injected into the input of the dual layer-local loss function, effectively making it
\[ \widetilde{L}_i = L \left( \widetilde{f}^{(i)} \left( f^{(i)} \left( \VUvec{h}^{(i - 1)} + \varepsilon \right) \right), \VUvec{h}^{(i - 1)} + \varepsilon \right) \quad \text{where} \quad \varepsilon \sim \mathcal{N} \left( 0, \sigma^2 \right) \]

To prove that target propagation is an effective way to learn neural networks, \cite{lee_difference_2015} state and prove the following theorem:

\begin{theorem}\label{targetprop_works}
	Assume that \( \forall i \in \left\{ 1, \dots, n \right\} \) there is \( \widetilde{f}^{(i)} = {f^{(i)}}^{-1} \), \( \VUvec{h}^{(i)} = f^{(i)} \left( \VUvec{h}^{(i - 1)} \right) = \VUmat{W}^{(i)} \sigma^{(i)} \left( \VUvec{h}^{(i - 1)} \right) \) where \( \sigma^{(i)} \) is a differentiable monotonically increasing element-wise function. Let \( \delta \VUmat{W}^{(i)}_{\mathrm{BP}} \) and \( \delta \VUmat{W}^{(i)}_{\mathrm{TP}} \) be the update for the \( i \)-th layer from back-propagation and target propagation respectively. If \( \eta \) in equation \ref{targetprop_last_layer_target} is sufficiently small, then the angle \( \alpha \) between \( \delta \VUmat{W}^{(i)}_{\mathrm{BP}} \) and \( \delta \VUmat{W}^{(i)}_{\mathrm{TP}} \) is bounded by
	\[ 0 < \frac{1 + \Delta_1 \left( \eta \right)}{\frac{\lambda_{\mathrm{max}}}{\lambda_{\mathrm{min}}} + \Delta_2 \left( \eta \right)} \leq \cos \left( \alpha \right) \leq 1 \]
		where \( \lambda_{\mathrm{min}} \) and \( \lambda_{\mathrm{max}} \) are the smallest and largest singular values of \( \left( \VUmat{J}_{f^{(n)}} \dots \VUmat{J}_{f^{(i + 1)}} \right)^T \) where \( \VUmat{J}_{f^{(k)}} \) is the Jacobian matrix of \( f^{(k)} \) and \( \Delta_1 \left( \eta \right) \) and \( \Delta_2 \left( \eta \right) \) are close to \( 0 \) for sufficiently small \( \eta \).
\end{theorem}

\section{Difference target propagation}

An issue with vanilla target propagation is the fact that even when a layer has already reached a minimum of its loss function, some of the lower layers may still be optimizing and thus disturbing the inputs of the layer and with that increasing the loss again. A solution to this problem is to require that a level reaches its optimum only when all the lower levels have already reached theirs. In other words:
\[ \widehat{\VUvec{h}}^{(i)} = \VUvec{h}^{(i)} \implies \widehat{\VUvec{h}}^{(i - 1)} = \VUvec{h}^{(i - 1)} \]
A way to fulfil this restriction is to restrict the targets in the following way:
\[ \widehat{\VUvec{h}}^{(i - 1)} - \VUvec{h}^{(i - 1)} = \widetilde{f}^{(i)} \left( \widehat{\VUvec{h}}^{(i)} \right) - \widetilde{f}^{(i)} \left( \VUvec{h}^{(i)} \right) \]
This restriction gives the following formula for the layer target:
\begin{equation}\label{difference_targetprop}
	\widehat{\VUvec{h}}^{(i - 1)} = \widetilde{f}^{(i)} \left( \widehat{\VUvec{h}}^{(i)} \right) + \VUvec{h}^{(i - 1)} - \widetilde{f}^{(i)} \left( \VUvec{h}^{(i)} \right)
\end{equation}
This augmented definition of a target has the same computational benefits as vanilla target propagation -- only the target values have to be back-propagated. All other necessary values are layer-local or have already been computed in the forward pass of the network.

\begin{remark}
	If the approximate inverse function \( \widetilde{f}^{(i)} \) is substituted by the actual inverse function \( {f^{(i)}}^{-1} \), then
	\[ \widehat{\VUvec{h}}_{\mathrm{DTP}}^{(i - 1)} = {f^{(i)}}^{-1} \left( \widehat{\VUvec{h}}^{(i)} \right) + \VUvec{h}^{(i - 1)} - {f^{(i)}}^{-1} \left( \VUvec{h}^{(i)} \right) = \]
	\[ = {f^{(i)}}^{-1} \left( \widehat{\VUvec{h}}^{(i)} \right) + \VUvec{h}^{(i - 1)} - \VUvec{h}^{(i - 1)} = {f^{(i)}}^{-1} \left( \widehat{\VUvec{h}}^{(i)} \right) = \widehat{\VUvec{h}}_{\mathrm{VTP}}^{(i - 1)} \]
	and Vanilla target propagation and Difference target propagation become equivalent.
\end{remark}

\cite{lee_difference_2015} state and prove the following theorem to show that difference target propagation satisfies the bound in equation \ref{targetprop_loss_bound}:

\begin{theorem}
	Let the target for layer \( i - 1 \) be given by the difference target propagation algorithm (i. e. equation \ref{difference_targetprop}). Let \( f^{(i)} \) and \( \widetilde{f}^{(i)} \) be differentiable and the corresponding matrices \( \VUmat{J}_{f^{(i)}} \) and \( \VUmat{J}_{\widetilde{f}^{(i)}} \) satisfy
	\[ \rho \left( \left( \VUmat{I} - \VUmat{J}_{f^{(i)}} \VUmat{J}_{\widetilde{f}^{(i)}} \right)^T \left( \VUmat{I} - \VUmat{J}_{f^{(i)}} \VUmat{J}_{\widetilde{f}^{(i)}} \right) \right) < 1\]
	where \( \rho \) denotes the spectral radius. If \( \widehat{\VUvec{h}}^{(i)} - \VUvec{h}^{(i)} \) is sufficiently small, then
	\[ \left\lVert \widehat{\VUvec{h}}^{(i)} - f^{(i)} \left( \widehat{\VUvec{h}}^{(i - 1)} \right) \right\rVert^2_2 < \left\lVert \widetilde{\VUvec{h}}^{(i)} - \VUvec{h}^{(i)} \right\rVert^2_2 \]
\end{theorem}
The condition on the spectral radius is easily satisfied because \( \widetilde{f}^{(i)} \) learns the inverse of \( f^{(i)} \) and so the matrix in question is close to zero.