\documentclass{article}
\usepackage{graphicx}
\usepackage{amsmath}

\usepackage{pgfplots}
\usepackage{tikz}
\usepackage{nicefrac}
\pgfplotsset{every axis legend/.append style={
at={(0,0)},
anchor=north east}}
\usetikzlibrary{shapes,positioning,intersections,quotes}

\definecolor{darkgreen}{rgb}{0.0, 0.6, 0.0}
\definecolor{darkred}{rgb}{0.7, 0.0, 0.0}

\title{Interpolation}
\begin{document}
\def\horzbar{\text{magic}}

  \pagenumbering{gobble}
  \maketitle
  \newpage
  \pagenumbering{arabic}

\section*{Introduction}

We have to arrays of numbers $X$ and $Y$. Array $X$ contains independent data points. Array $Y$ contains dependent data points $y_i,i=1,…,m$.

We want to find a function $\hat{y}(x)$, which gets the exact same value with given points.\\

\section*{Linear Interpolation}

Linear interpolation is achieved by connecting two data points with a straight line.

For $x_i < x < x_{i+1}$:

$$\hat{y}(x) = y_i + \frac{(y_{i+1} - y_{i})(x - x_{i})}{(x_{i+1} - x_{i})}.$$

\section*{Derivation}

\begin{tikzpicture}

	\draw [dashed]  (-3, -1.9) -- (3, -1.9);
	\draw [dashed]  (3, -5) -- (3, -1.9);
	\draw [-stealth](-3,-5) -- (9,-5);
	\draw [-stealth](-3,-5) -- (-3,1);

    \draw [stealth-stealth, blue](0,-3) -- (8,0);
    \draw [stealth-stealth, red](8,0) -- (8,-3);
    \draw [stealth-stealth, green](0,-3) -- (8,-3);
    \draw [stealth-stealth, black](0,-3.1) -- (3,-3.1);

	\node[above right=0pt of {(8, 0)}, outer sep=2pt,fill=none] {$(x_2, y_2)$};
	\node[above right=0pt of {(8, -1.8)}, outer sep=2pt,fill=none, darkred] {$y_2 - y_1$};
	\node[above right=0pt of {(4.5, -3.6)}, outer sep=2pt,fill=none, darkgreen] {$x_2 - x_1$};
	\node[above right=0pt of {(1, -3.6)}, outer sep=2pt,fill=none] {$x - x_1$};
	\node[above right=0pt of {(-1,-3.7)}, outer sep=2pt,fill=none] {$(x_2, y_2)$};
	\node[above right=0pt of {(-1,-1.7)}, outer sep=2pt,fill=none] {$y$};
	\node[above right=0pt of {(3,-4.2)}, outer sep=2pt,fill=none] {$x$};
	\node[above right=0pt of {(3,-2.7)}, outer sep=2pt,fill=none] {$h$};

\end{tikzpicture}

$$\alpha = \frac{y_2 - y_1}{x_2 - x_1}$$

$$h = \alpha \cdot (x - x_1)$$

$$y = y_1 + h$$

$$y = y_1 + (x - x_1) \cdot \frac{y_2 - y_1}{x_2 - x_1}$$

\section*{Example}

We are given two points A(-2, 0) and B (2, 2).\\

\begin{tikzpicture}
    \begin{axis}[
        axis x line=middle,
        axis y line=middle,
        width=8cm,
        height=8cm,
        xmin=-5,   % start the diagram at this x-coordinate
        xmax= 5,   % end   the diagram at this x-coordinate
        ymin=-5,   % start the diagram at this y-coordinate
        ymax= 5,   % end   the diagram at this y-coordinate
        xlabel=$x$,
        ylabel=$y$,
        legend cell align=left,
        legend pos=south east,
        legend style={draw=none},
        tick align=outside,
        enlargelimits=false]
      % plot the function
      \addplot[domain=-5:5, blue, ultra thick,samples=500] {0.5*x + 1};
      \fill[red] (700,700) circle (3pt);
    	  \fill[red] (300, 500) circle (3pt);
    	  \draw [dashed]  (500, 700) -- (680, 700);
    	  \draw [dashed]  (700, 500) -- (700, 680);
    	  \node[above right=0pt of {(255,510)}, outer sep=2pt,fill=none] {A};
	  \node[above right=0pt of {(655,710)}, outer sep=2pt,fill=none] {B};
	\legend{$\nicefrac{1}{2} \cdot x$ + 1}
    \end{axis}
\end{tikzpicture}

Let's try to evaluate the value of the function at $x=1$

$$\hat{y}(x) = y_i + \frac{(y_{i+1} - y_{i})(x - x_{i})}{(x_{i+1} - x_{i})} = 0 + \frac{(2 - 0)(1 - (-2))}{(2 - (-2))} = 1.5$$

\newpage
\section*{Cubic Spline}

The interpolating function in cubic spline interpolation is a set of piecewise cubic functions.\\

For $x_i < x < x_{i+1}$:\\

We have two points $(x_i, y_i)$ and $(x_{i+1}, y_{i+1})$  joined with a cubic polynomial:

$$S_i(x) = a_i x^3 + b_i x^2 + c_i x + d_i$$

For $n$ points, there are $n-1$ cubic functions to find, and each cubic function requires four coefficients ($a_i, b_i, c_i, d_i$).

There are $4(n-1)$ unknowns to find.\\

\begin{tikzpicture}
    \begin{axis}[
        axis x line=middle,
        axis y line=middle,
        width=13cm, height=13cm,     % size of the image
        grid = none,
        grid style={dashed, gray!0},
        %xmode=log,log basis x=10,
        %ymode=log,log basis y=10,
        xmin=-2,     % start the diagram at this x-coordinate
        xmax= 4,    % end   the diagram at this x-coordinate
        ymin=-7,     % start the diagram at this y-coordinate
        ymax= 7,   % end   the diagram at this y-coordinate
        %/pgfplots/xtick={0,1,...,60}, % make steps of length 5
        %extra x ticks={23},
        %extra y ticks={0.507297},
        axis background/.style={fill=white},
        ylabel=y,
        xlabel=x,
        %xticklabels={,,},
        %yticklabels={,,},
        tick align=outside,
        tension=0.08]
      % plot the stirling-formulae
      \addplot[name path global=a, domain=-2:4, blue, thick,samples=500] {-x*x*x + 4*x*x-x-4};
       \fill[red] (211, 39.3) circle (3pt);
       \fill[red] (455, 108.8) circle (3pt);
       \node[above right=0pt of {(211, 39.3)}, outer sep=2pt,fill=none] {$y_1$};
    	  \node[above right=0pt of {(455, 108.8)}, outer sep=2pt,fill=none] {$y_2$};
    	   \node[above right=0pt of {(250, 98.8)}, outer sep=2pt,fill=none] {$S_1(x)$};
\draw [-stealth](280,98) -- (350,82
);
    \end{axis}
\end{tikzpicture}

\section*{Derivation}

We are trying to find a function $S_i(x) = a_i x^3 + b_i x^2 + c_i x + d_i$ going trough both points: $(x_i, y_i)$ and $x_{i+1}, y_{i+1}$.

\begin{align}
S_i(x_i) &= y_i,\quad i = 1,\ldots,n-1,
\end{align}
\begin{align}
S_i(x_{i+1}) &= y_{i+1},\quad i = 1,\ldots,n-1,
\end{align}

Smoothness condition:
\begin{align}
S'_i(x_{i+1}) &= S^{\prime}_{i+1}(x_{i+1}),\quad i = 1,\ldots,n-2,
\end{align}
\begin{align}
S''_i(x_{i+1}) &= S''_{i+1}(x_{i+1}),\quad i = 1,\ldots,n-2,
\end{align}

Boundry condition: The curve is a “straight line” at the end points:
\begin{align}
S''_1(x_1) &= 0
\end{align}
\begin{align}
S''_{n-1}(x_n) &= 0
\end{align}

Let $h_{i}=x_{i}-x_{i-1}$

Let $S_i^{''}(x_i) = S_i^{''}(x_{i+1}) = M_i$

$S_1^{''}(x_1)= M_0 = 0$ and $S_n^{''}(x_n) = M_n = 0$

Other $M_i$ are unknown.

By Lagrange interpolation, we can interpolate each $S''_{i}$ on  $[x_{i-1},x_{i}]$:

$$S''_{i}(x)=M_{i-1}{\frac {x_{i}-x}{h_{i}}}+M_{i}{\frac {x-x_{i-1}}{h_{i}}} \quad for \quad x\in [x_{i-1},x_{i}]$$

Integrating the above equation twice and using the condition that $C_{i}(x_{i-1})=y_{i-1}$ and $ C_{i}(x_{i})=y_{i}$ to determine the constants of integration, we have.

$$ S_{i}(x)=M_{i-1}{\frac {(x_{i}-x)^{3}}{6h_{i}}}+M_{i}{\frac {(x-x_{i-1})^{3}}{6h_{i}}}+\left(y_{i-1}-{\frac {M_{i-1}h_{i}^{2}}{6}}\right){\frac {x_{i}-x}{h_{i}}}+\left(y_{i}-{\frac {M_{i}h_{i}^{2}}{6}}\right){\frac {x-x_{i-1}}{h_{i}}}$$
$${\text{for}}\quad x\in [x_{i-1},x_{i}] $$\\

This expression gives us the cubic spline S(x) if $ M_{i},i=0,1,\cdots ,n$ can be determined.
$$S'_{i+1}(x)=-M_{i}{\frac {(x_{i+1}-x)^{2}}{2h_{i+1}}}+M_{i+1}{\frac {(x-x_{i})^{2}}{2h_{i+1}}}+{\frac {y_{i+1}-y_{i}}{h_{i+1}}}-{\frac {M_{i+1}-M_{i}}{6}}h_{i+1}$$

$$S'_{i+1}(x_{i})=-M_{i}{\frac {h_{i+1}}{2}}+{\frac {y_{i+1}-y_{i}}{h_{i+1}}}-{\frac {M_{i+1}-M_{i}}{6}}h_{i+1}$$

Similarly, when $x\in [x_{i-1},x_{i}]$, we can shift the index to obtain

\begin{align}
S'_{i}(x) &=-M_{i-1}{\frac {(x_{i}-x)^{2}}{2h_{i}}}+M_{i}{\frac {(x-x_{i-1})^{2}}{2h_{i}}}+{\frac {y_{i}-y_{i-1}}{h_{i}}}-{\frac {M_{i}-M_{i-1}}{6}}h_{i}
\end{align}

$$ S'_{i}(x_{i})=M_{i}{\frac {h_{i}}{2}}+{\frac {y_{i}-y_{i-1}}{h_{i}}}-{\frac {M_{i}-M_{i-1}}{6}}h_{i}$$

Since $ S'_{i+1}(x_{i})=S'_{i}(x_{i})$, we can derive:

$$\mu _{i}M_{i-1}+2M_{i}+\lambda _{i}M_{i+1}=d_{i}\quad {\text{for}}\quad i=1,2,\cdots ,n-1,$$

$$\mu _{i}={\frac {h_{i}}{h_{i}+h_{i+1}}},\quad \lambda _{i}=1-\mu _{i}={\frac {h_{i+1}}{h_{i}+h_{i+1}}},\quad {\text{and}}\quad d_{i}=6f[x_{i-1},x_{i},x_{i+1}]$$

and $f[x_{i-1},x_{i},x_{i+1}]$ is a divided difference.\\

According to different boundary conditions, we can solve the system of equations above to obtain the values of $M_{i}$'s.\\

$S'_{1}(x_{0})=f'_{0}$ and $S'_{n}(x_{n})=f'_{n}$. According to equation (7), we can obtain:

$$S'_{1}(x_{0})=-M_{0}{\frac {(x_{1}-x_{0})^{2}}{2h_{1}}}+M_{1}{\frac {(x_{0}-x_{0})^{2}}{2h_{1}}}+{\frac {y_{1}-y_{0}}{h_{1}}}-{\frac {M_{1}-M_{0}}{6}}h_{1}$$
$$\Rightarrow f'_{0}=-M_{0}{\frac {h_{1}}{2}}+f[x_{0},x_{1}]-{\frac {M_{1}-M_{0}}{6}}h_{1}$$
$$\Rightarrow 2M_{0}+M_{1}={\frac {6}{h_{1}}}(f[x_{0},x_{1}]-f'_{0})=6f[x_{0},x_{0},x_{1}]$$

Analogously:

$$ S'_{n}(x_{n})=-M_{n-1}{\frac {(x_{n}-x_{n})^{2}}{2h_{n}}}+M_{n}{\frac {(x_{n}-x_{n-1})^{2}}{2h_{n}}}+{\frac {y_{n}-y_{n-1}}{h_{n}}}-{\frac {M_{n}-M_{n-1}}{6}}h_{n}$$

$$M_{n-1}+2M_{n}={\frac {6}{h_{n}}}(f'_{n}-f[x_{n-1},x_{n}])=6f[x_{n-1},x_{n},x_{n+1}]$$

Let:\\
$\lambda _{0}=\mu _{n}=1,$\\
$d_{0}=6f[x_{0},x_{0},x_{1}]$ and\\
$d_{n}=6f[x_{n-1},x_{n},x_{n}]$

\begin{equation*}
  \begin{bmatrix}
    2 & \lambda_0 \\
    \mu_1 & 2 & \lambda_1 \\
    & \ddots & \ddots & \ddots \\
    && \ddots & \ddots & \ddots \\
	&&& \ddots & \ddots & \ddots \\
	&&&& \mu_{n-1} & 2 & \lambda_{n-1} \\
	&&&&& \mu_{n} & 2 \\
  \end{bmatrix}
  %
   \begin{bmatrix}
    M_0 \\
    M_1 \\
    \vdots \\
    \vdots \\
    \vdots \\
    M_{n-1} \\
    M_n \\
  \end{bmatrix}
  =
   %
  \begin{bmatrix}
    d_0 \\
    d_1 \\
    \vdots \\
    \vdots \\
    \vdots \\
    d_{n-1} \\
    d_n \\
  \end{bmatrix}
\end{equation*}

 \newpage
\section*{Lagrange Polynomial Interpolation}

Lagrange polynomial interpolation gives us a single polynomial that connects all of the data points.

That polynomial is denoted as $L(x)$. It is true that $L(x_i) = y_i$ for all points $(x_i, y_i)$ .

$$L(x) = \sum_{i = 1}^n y_i P_i(x).$$

Each polynomial appearing in the sum is called a Lagrange basis polynomials, $P_i(x)$.

$$P_i(x) = \prod_{j = 1, j\ne i}^n\frac{x - x_j}{x_i - x_j},$$

\section*{Example}

We are given three points A(-1, 1), B(2, 3) and C(3,5).\\

$$P_1(x) = \frac{(x - x_2)(x - x_3)}{(x_1-x_2)(x_1-x_3)} = \frac{(x - 2)(x - 3)}{(-1-2)(-1-3)} = \frac{1}{12}(x^2 - 5x + 6)$$

$$P_2(x) = \frac{(x - x_1)(x - x_3)}{(x_2-x_1)(x_2-x_3)} = \frac{(x + 1)(x - 3)}{(2 + 1)(2-3)} = -\frac{1}{3}(x^2 - 2x - 3)$$

$$P_3(x) = \frac{(x - x_1)(x - x_2)}{(x_3-x_1)(x_3-x_2)} = \frac{(x + 1)(x - 2)}{(3 + 1)(3-2)} =\frac{1}{4}(x^2 -x - 2)$$

$$ L(x) = 1 \cdot P_1(x) + 3 \cdot P_2(x) + 5 \cdot P_3(x) $$

$$ L(x) = 1 \cdot P_1(x) + 3 \cdot P_2(x) + 5 \cdot P_3(x) $$

$$ L(x) = \frac{1}{3} x^2 + \frac{1}{3} x + 1 $$

\begin{tikzpicture}
    \begin{axis}[
        axis x line=middle,
        axis y line=middle,
        width=10cm,
        height=10cm,
        xmin=-5,   % start the diagram at this x-coordinate
        xmax= 6,   % end   the diagram at this x-coordinate
        ymin= -1,   % start the diagram at this y-coordinate
        ymax= 8,   % end   the diagram at this y-coordinate
        xlabel=$x$,
        ylabel=$y$,
        legend cell align=left,
        legend pos=north east,
        legend style={draw=none},
        tick align=outside,
        enlargelimits=false,
        xtick distance=1,
        ytick distance=1]

      % plot the function
      \addplot[domain=-5:10, blue, ultra thick,samples=500] {1/3*(x^2 + x + 3)};

      \fill[red] (400, 20) circle (3pt);
    	  \fill[red] (700, 40) circle (3pt);
    	  \fill[red] (800, 60) circle (3pt);

    	  \node[above right=0pt of {(340, 13)}, outer sep=2pt,fill=none] {A};
	  \node[above right=0pt of {(630, 40)}, outer sep=2pt,fill=none] {B};
	 \node[above right=0pt of {(730, 60)}, outer sep=2pt,fill=none] {C};

	\legend{$\frac{1}{3} x^2 + \frac{1}{3} x + 1$}

    \end{axis}
\end{tikzpicture}

\end{document}


\documentclass[12pt]{article}
\usepackage[margin=0.7in]{geometry}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{hyperref}
\hypersetup{colorlinks=true, urlcolor=green}
\DeclareMathOperator*{\argmax}{argmax}
\begin{document}

\title{The Gory Details}
\author{Pavel Komarov}
\date{March 12, 2018}
\maketitle

Here lies a full explanation of how multivariate Projection Pursuit Regression (PPR) and univariate Classification work, to the degree I currently understand. It is as much for me as for you, because I much prefer my own notation, and every time I have to dig in to the bones of the code and faff with the loss function, I end up having to refer to \href{http://www.slac.stanford.edu/cgi-wrap/getdoc/slac-pub-3824.pdf}{the original paper}'s somewhat ambiguous, derivationless equations. Formerly I was attempting to deposit some of this knowledge in code comments, but they took up too much space while somehow remaining marvelously unreadable.

This is all in \LaTeX\ because \href{https://github.com/github/markup/issues/897}{native math in markdown is amazingly still not supported by github}, so a \texttt{.md} would really be no less uncomfortable than code comments. If you happen to want to know how I accomplished all this gorgeous formatting, the source \texttt{.tex} is in the \href{https://github.com/pavelkomarov/projection-pursuit/tree/master/doc}{/doc} subdirectory of the \href{https://github.com/pavelkomarov/projection-pursuit}{repo} and can be compiled with \texttt{pdflatex}.

I attempt to follow a structure based on the \href{https://www.youtube.com/watch?v=H6ygbGX8OV8&t=9m19s}{five components of any machine learning algorithm}:

\begin{enumerate}
	\setlength\itemsep{-2mm}
	\item A Task/Problem Reduction
	\item A Model
	\item A Loss Function
	\item An Optimization Scheme
	\item Data
\end{enumerate}

--or at least the middle three, since by the time the data for your task meets my algorithm you'll have abstracted away concerns about where it came from and what it actually means.

\section{The Model}

PPR is a statistical model of the form:

$$\vec{y_i} = \sum_{j=1}^{r} f_j(\vec{x_i} \cdot \vec{\alpha_j}) \otimes \vec{\beta_j}^T$$

where:
\begin{itemize}
	\setlength\itemsep{-2mm}
	\item $i$ iterates examples, the rows of input and output matrices
	\item $j$ iterates the number of terms in the PPR ``additive model"
	\item $r$ is the total number of projections and functions (terms) in the PPR
	\item $\vec{y_i}$ is a $d$-dimensional vector, the $i$\textsuperscript{th} row in an output matrix $\pmb{Y} \in \mathbb{R}^{n \times d}$
	\item $\vec{x_i}$ is a $p$-dimensional vector, the $i$\textsuperscript{th} row of an input matrix $\pmb{X} \in \mathbb{R}^{n \times p}$
	\item $\vec{\alpha_j}$ is the $j$\textsuperscript{th} projection vector in the mdoel, a $p$-dimensional vector inner-producted with $\vec{x_i}$
	\item $f_j$ is the $j$\textsuperscript{th} function in the model, mapping from $\mathbb{R}^1 \rightarrow \mathbb{R}^1$
	\item $\vec{\beta_j}^T$ is the transpose of $\vec{\beta_j}$, a $d$-dimensional vector outer-producted with the result of $f_j$ to yield a result in the output space
	\item $\cdot$ is an inner product
	\item $\otimes$ is an outer product
\end{itemize}

This can be ``vectorized" as

$$\pmb{\hat{Y}} = \sum_{j=1}^r f_j(\pmb{X} \cdot \vec{\alpha_j}) \otimes \vec{\beta_j}^T$$

where the ``hat" \string^ distinguishes the answers predicted, as opposed to known training answers.\newline

I also term this the ``evaluation function". It may seem complicated, but the idea is simple:

\begin{enumerate}
	\setlength\itemsep{-2mm}
	\item Linearly project the input down to one dimension where it is easier to work with, thereby sidestepping the curse of dimensionality.
	\item Find a sensible mapping from this reduced space to weighted ``residuals", unexplained variance in the outputs. This is where the nonlinearity happens.
	\item Unpack from the single-dimensional residual space to the output space with a kind of inverse projection.
\end{enumerate}

\subsection{A Word on Additive Models}

In practice a single projection-mapping-expansion is not descriptive enough to capture the richness of what may be a very complicated underlying relationship between $\pmb{X}$ and $\pmb{Y}$, so it is repeated $r$ times, each new ``stage" only accounting for the variance left unexplained by the stages that have come before. Notice that, as per Taylor's Theorem and the no-doubt familiar universal approximation theorems, for certain classes of functions $f$, as $r$ goes to infinity the evaluation function can approximate any continuous functional relationship between inputs and outputs.

\section{The Loss Function}

The (supervised) learning process consists of minimizing a standard quadratic cost, or ``loss", function:

$$loss = \sum_{i=1}^n w_i (y_i - \hat{y_i})^2$$

where:
\begin{itemize}
	\setlength\itemsep{-2mm}
	\item $i$ iterates all training examples
	\item $n$ is the total number of training examples
	\item $w_i$ is the weight of the $i$\textsuperscript{th} example
	\item $y_i$ is the known answer for example $i$
	\item $\hat{y_i}$ (``y-i-hat") is the answer predicted by the model for example $i$
\end{itemize}

In words: Get as close as you can for all examples, where the penalty for predicting an example incorrectly scales with the square of how wrong you are.

There are other choices of loss function, including an absolute value loss with no square penalty. This can work better in cases where a nonliner penalty might cause the algorithm to account too strongly for a few outlier examples.

(TODO) It is also possible to add regularization terms, which bias the optimization process to find smaller parameter values. This is good for numerical stability and has the added benefit of reducing the likelihood of overfit.

Unfortunately, modifying the loss function in either of these ways means altering the optimization process, and that's a mathematical bear I just haven't chosen to wrestle with.\newline

Plugging the evaluation function in to the cost function yields a relationship between model parameters and loss. Because there are multiple dimensions in our vector $\vec{y_i}$, we introduce a sum over them so the PPR is motivated to make good predictions for all entries of the output:

$$loss = \sum_{i=1}^n w_i \sum_{k=1}^d w_k [y_{ik} - \sum_{j=1}^r f_j(\vec{x_i} \cdot \vec{\alpha_j}) \otimes \beta_{jk}]^2$$

where this new fauna:
\begin{itemize}
	\setlength\itemsep{-2mm}
	\item $k$ iterates the columns of the output $\pmb{Y}$
	\item $d$ is the number of outputs, the width of the output matrix $\pmb{Y}$
	\item $w_k$ is a scalar weight, the relative importance of the $k$\textsuperscript{th} output dimension
	\item $y_{ik}$ is the scalar $k$\textsuperscript{th} entry in the vector $\vec{y_i}$, itself the $i$\textsuperscript{th} row of $\pmb{Y}$
	\item $\beta_{jk}$ is the scalar $k$\textsuperscript{th} entry of $\vec{\beta_j}$ from the evaluation function
\end{itemize}

This can be vectorized as

$$loss = (\vec{w_I}^T \cdot (\pmb{Y} - \pmb{\hat{Y}})^2) \cdot \vec{w_K}$$

where
\begin{itemize}
	\setlength\itemsep{-2mm}
	\item $\vec{w_I}$ is an $n$-vector containing all example weights, $w_i$
	\item $\vec{w_K}$ is a $d$-vector containing all output dimension weights, $w_k$
\end{itemize}
and the sums disappear inside the inner products.\newline

The parameters we need to optimize to make the PPR ``learn" are $\vec{\alpha_j}$, $f_j$, and $\vec{\beta_j}$. $w_k$ are hyperparameters chosen by the user, just as $r$ is chosen. If output dimensions are on differing scales and the user wishes for the loss to be relative to those scales, then choosing $w_k$ inversely proportional to the variance of outputs causes the loss to treat outputs more evenly. If some output dimensions are more important than others, then setting their weights higher causes predictions in those dimensions to be closer.

\section{The Optimization Scheme}

The macroscopic optimization scheme to solve for so many different parameters is non-obvious but straightforward:
\begin{enumerate}
	\setlength\itemsep{-2mm}
	\item Initialize all $\vec{\alpha_j}$, $f_j$ and $\vec{\beta_j}$ to something random. Let $j=1$.
	\item Find the ``residual" variance unexplained by all stages fit so far.
	\item Project the input in to single dimension: $\pmb{X} \cdot \vec{\alpha_j}$.
	\item Fit $f_j$ to a weighted residual target versus projections.
	\item Use this $f_j$ to find a better setting for $\vec{\beta_j}$.
	\item Use a Gauss-Newton scheme to solve for an update to $\vec{\alpha_j}$.
	\item Repeat steps 3-6 until $f_j$, $\vec{\beta_j}$, and $\vec{\alpha_j}$ converge.
	\item (optional) Use the newly converged parameters to retune all previous $f_t$, $\vec{\beta_t}$, $\vec{\alpha_t}$ where $t \leq j$. (backfitting)
	\item Increment $j$ and go back to step 2 until $j$ reaches $r$.
\end{enumerate}

The inner loop here is a form of \textit{alternating optimization}, wherein all parameters except one are held constant, the best setting for that parameter given those constants is found, and the process cycled through all parameters until convergence. The outer loop builds up the additive model.

But this leaves some details unexplained. How exactly is the residual found? How are parameters found given fixed solutions to the others? How do we test for convergence?

\subsection{Finding The Residual}

The residual trick cleverly separates the contribution of the $j$\textsuperscript{th} stage from the other terms in the additive model. Consider rephrasing the loss function as:

$$loss = \sum_{i=1}^n w_i \sum_{k=1}^d w_k [y_{ik} - \sum_{t \neq j} f_t(\vec{x_i} \cdot \vec{\alpha_t}) \otimes \beta_{tk} - f_j(\vec{x_i} \cdot \vec{\alpha_j}) \otimes \beta_{jk}]^2$$

Now if we let

$$r_{ijk} = y_{ik} - \sum_{t \neq j} f_t(\vec{x_i} \cdot \vec{\alpha_t}) \otimes \beta_{tk}$$

then

$$loss,\ L = \sum_{i=1}^n w_i \sum_{k=1}^d w_k [r_{ijk} - f_j(\vec{x_i} \cdot \vec{\alpha_j}) \otimes \beta_{jk}]^2$$

In practice we will wish to find all $r_{ijk}$ for a particular $j$. Call this $\pmb{R}_j \in \mathbb{R}^{n \times d}$, the same space as the output. It can be found with

$$\pmb{R}_j = \pmb{Y} - \sum_{t \neq j} f_t(\pmb{X} \cdot \vec{\alpha_t}) \otimes \vec{\beta_t}$$

where $f_t(\pmb{X} \cdot \vec{\alpha_t})$ is actually an $n$-vector.

\subsection{Optimizing $\vec{\beta_j}$ Given $\vec{\alpha_j}$ and $f_j$}

$$loss\ for\ the\ jth\ term,\ L_j = \sum_{i=1}^n w_i \sum_{k=1}^d w_k [r_{ijk} - f_j(\vec{x_i} \cdot \vec{\alpha_j}) \otimes \beta_{jk}]^2$$

To optimize with respect to a parameter, use good ol' calculus: Take a derivative, set equal to zero, and solve. Let's select $\beta_{jk'}$, the $(k=k')$\textsuperscript{th} entry of $\vec{\beta_j}$ as the parameter of interest.

$$\frac{\partial L_j}{\partial \beta_{jk'}} = \sum_{i=1}^n w_i w_{k'} [2(r_{ijk'} - f_j(\vec{x_i} \cdot \vec{\alpha_j}) \otimes \beta_{jk'})(-f_j(\vec{x_i} \cdot \vec{\alpha_j}))] = 0$$

Notice that the sum over $k$ disappears because no term where $k \neq k'$ will contain our variable $\beta_{jk'}$, so for the purposes of differentiation they are constant, and the derivative of constants is zero. Only the weight $w_{k'}$ remains.

Also, I've been using $\otimes$ for consistency, but since $\beta_{jk'}$ is scalar, an outer product is nothing special, just an ordinary multiplication. So we can do some algebra:

$$ -2 w_{k'} \sum_{i=1}^n w_i [r_{ijk'} f_j(\vec{x_i} \cdot \vec{\alpha_j})] + 2 w_{k'} \beta_{jk'} \sum_{i=1}^n w_i [f_j^2(\vec{x_i} \cdot \vec{\alpha_j})] = 0 $$

$$\rightarrow \beta_{jk'} = \frac{\sum_{i=1}^n w_i [r_{ijk'} f_j(\vec{x_i} \cdot \vec{\alpha_j})]}{\sum_{i=1}^n w_i [f_j^2(\vec{x_i} \cdot \vec{\alpha_j})]}$$

This can be vectorized to find all entries of $\vec{\beta_j}$ at once:

$$ \vec{\beta_j} = \frac{\pmb{R}_j^T \cdot (\vec{w_I} \odot f_j(\pmb{X} \cdot \alpha_j))}{f_j(\pmb{X} \cdot \alpha_j) \cdot (\vec{w_I} \odot f_j(\pmb{X} \cdot \alpha_j))}$$

where $\odot$ is a Hadamard product, $\vec{w_I}$ is an $n$-vector, $\pmb{R}_j$ is the residuals, and the dot products are taken along the length-$n$ dimensions of these objects.

\subsection{Optimizing $f_j$ Given $\vec{\alpha_j}$ and $\vec{\beta_j}$}

Now a similar argument, but this time consider the parameter of interest to be $f_{i'j}$, the $(i=i')$\textsuperscript{th} entry of the $n$-vector formed by taking the inner product of $\pmb{X}$ with $\alpha_j$ and applying $f_j$ to each entry.

$$\frac{\partial L_j}{\partial f_{i'j}} = w_{i'} \sum_{k=1}^d w_k [2(r_{i'jk} - f_{i'j} \otimes \beta_{jk})(-\beta_{jk})] = 0$$

The sum over $i$ disappears because only the single term where $i=i'$ isn't constant to the derivative.

$$\rightarrow w_{i'} \sum_{k=1}^d w_k [-2 r_{i'jk} \beta_{jk} + 2 f_{i'j} \beta_{jk}^2 ] = 0$$

$$\rightarrow 2 f_{i'j} \sum_{k=1}^d w_k \beta_{jk}^2 = 2 \sum_{k=1}^d w_k r_{i'jk} \beta_{jk}$$

$$\rightarrow f_{i'j} = \frac{\sum_{k=1}^d w_k r_{i'jk} \beta_{jk}}{\sum_{k=1}^d w_k \beta_{jk}^2}$$

As in the case of $\vec{\beta_j}$, this can be vectorized.

$$ f_j(\pmb{X} \cdot \vec{\alpha_j}) = \frac{\pmb{R}_j \cdot (\vec{w_K} \odot \vec{\beta_j})}{\vec{\beta_j} \cdot (\vec{w_K} \odot \vec{\beta_j})}$$

This provides targets for the funtion $f_j$. The task is to find the function that maps from this input to this output, for which there are numerous solvers (finding a polynomial by reducing to a linear inverse problem, for example). The example weights $w_i$ disappear in the algebra and so do not affect the targets, but they can be passed on to the function-fitter so it considers some examples more important than others.

\subsection{Optimizing $\vec{\alpha_j}$ Given $\vec{\beta_j}$ and $f_j$}

This is by far the toughest set of parameters to optimize, because they are nested inside the function. This time express the loss as:

$$L_j = \sum_{k=1}^d w_k [\vec{w_I} \odot \vec{g_{jk}}^2]$$

where

$$ \vec{g_{jk}} = \vec{r_{jk}} - f_j(\pmb{X} \cdot \vec{\alpha_j}) \otimes \beta_{jk}$$

where $\vec{r_{jk}}$ is the $n$-vector formed by stacking $r_{ijk} \forall i$ together, or equivalently the $k$\textsuperscript{th} column of $\pmb{R}_j$.

The weights can be factored in to the square to yield a form solveable with \href{https://en.wikipedia.org/wiki/Gauss%E2%80%93Newton_algorithm#Description}{Gauss-Newton}.

$$L_j = \sum_{k=1}^d \vec{g_{jkw}}(\vec{\alpha_j})^2,$$

$$\vec{g_{jkw}} = \sqrt{w_k} \sqrt{\vec{w_I}} \odot (\vec{r_{jk}} - f_j(\pmb{X} \cdot \vec{\alpha_j}) \otimes \beta_{jk})$$

Find the Jacobian:

$$J_{jk}[u,v] = \frac{\partial \vec{g_{jkw}}(\vec{\alpha_j})[u]}{\partial \vec{\alpha_j}[v]} = -\sqrt{w_k} \sqrt{w_u} \dot{f_j}(\vec{x_u} \cdot \vec{\alpha_j}) \beta_{jk} x_{uv}$$

where $w_u$ is $\vec{w_I}[u]$, the $u$\textsuperscript{th} example weight, as distinct from $w_k$, which, remember, are output dimension weights.

\[
\rightarrow J_{jk} = -\sqrt{w_k} \beta_{jk}
\begin{bmatrix}
	\sqrt{w_0} \dot{f_j}(\vec{x_0} \cdot \vec{\alpha_j}) x_{00} & \sqrt{w_0} \dot{f_j}(\vec{x_0} \cdot \vec{\alpha_j}) x_{01} & \dots & \sqrt{w_0} \dot{f_j}(\vec{x_0} \cdot \vec{\alpha_j}) x_{0p} \\
    \sqrt{w_1} \dot{f_j}(\vec{x_1} \cdot \vec{\alpha_j}) x_{10} & \sqrt{w_1} \dot{f_j}(\vec{x_1} \cdot \vec{\alpha_j}) x_{11} & \dots & \sqrt{w_1} \dot{f_j}(\vec{x_1} \cdot \vec{\alpha_j}) x_{1p}\\
    \vdots & & \ddots & \vdots\\
    \sqrt{w_n} \dot{f_j}(\vec{x_n} \cdot \vec{\alpha_j}) x_{n0} & \hdots & \hdots & \sqrt{w_n} \dot{f_j}(\vec{x_n} \cdot \vec{\alpha_j}) x_{np}
\end{bmatrix}
\]

$$= -\sqrt{w_k} \beta_{jk} (\sqrt{\vec{w_I}} \odot \dot{f_j}(\pmb{X} \cdot \vec{\alpha_j})) \odot \pmb{X}$$

where everything left of the last $\odot$ simplifies to an $n$-vector, so the Hadamard product is taken with each column of $\pmb{X}$ individually.\newline

As per Gauss-Newton, the update to the parameter $\vec{\alpha_j}$ to the function $\vec{g_{jkw}}$ is given by the solution $\vec{\delta}$ to:

$$\left[ \sum_{k=1}^d J_{jk}^T J_{jk} \right] \vec{\delta} = \sum_{k=1}^d J_{jk}^T \vec{g_{jkw}}$$

On the left side is a $p \times p$ matrix, and on the right a $p \times 1$ vector, so we have an easy-to-solve linear inverse problem.

$$ \vec{\alpha_j} = \vec{\alpha_j} + \vec{\delta} $$

In practice $\vec{\alpha_j}$ is renormalized after this update to avoid numerical drift and keep it from blowing up. Because $\vec{\alpha_j}$ is a projection vector only meant to flatten the data along some dimension, its magnitude does not really matter.

\subsection{Testing for Convergence}

Parameters are considered ``converged" when their values across iterations cause a small-enough change in the value of the loss function.

Because we only make updates to the parameters governing stage $j$, we can make use the convenient identity

$$\pmb{Y} - \pmb{\hat{Y}} = \pmb{R}_j - f_j(\pmb{X} \cdot \vec{\alpha_j}) \otimes \vec{\beta_j}^T$$

to calculate the loss quickly.

\section{Classification}

Let the miscalssification risk $R$ be

$$R = \sum_{i=1}^n \min_{k \in [1,q]} \sum_{c=1}^q l_{ck} p(c | \vec{x_i})$$

where
\begin{itemize}
	\setlength\itemsep{-2mm}
	\item $i$ iterates over examples
	\item $q$ is the total number of classes in the problem
	\item $\min_k$ implements the optimal decision rule for each example
	\item $l_{ck}$ is the user-specified loss for predicting $y=k$ when in truth $y=c$
	\item the inner sum is the total loss for predicting $y=k$
	\item $p(c | \vec{x_i})$ is the true probability $y=c$ given input $\vec{x_i}$
\end{itemize}

The unknown here is that conditional probability. If we define an indicator variable

$$h_{ci} = 1\ if\ y_i=c,\ 0\ otherwise$$

then Friedman says we can rewrite the conditional probability as

$$p(c | \vec{x_i}) = \frac{\pi_c S}{s_c} E[h_{ci} | \vec{x_i}]$$

where
\begin{itemize}
	\setlength\itemsep{-2mm}
	\item $\pi_c$ is the prior probability that $y_i=c$ ($h_{ci}=1$), calculable from the training set with $\frac{\sum_{i=1}^n h_{ci}}{n}$
	\item $s_c = \sum_{i=1}^n w_i h_{ci}$, the cumulative weight of examples with classification $c$
	\item $S = \sum_{c=1}^q s_c$, the cumulative weight of everything, a constant
	\item $E$ means the expected value
\end{itemize}

The coefficient can be expanded to

$$\frac{\sum_{c=1}^q \sum_{i=1}^n w_i h_{ci} * \frac{\sum_{i=1}^n h_{ci}}{n}}{\sum_{i=1}^n w_i h_{ci}} \propto \frac{\sum_{i=1}^n h_{ci}}{\sum_{i=1}^n w_i h_{ci}}$$

This doesn't make any sense, because it says probabilities are \textit{inversely} related to example weights, but we really want the probability of predicting a given class to be \textit{directly} related to the importances of examples in that class. Convinced Friedman accidentally inverted the equation, the code and the following equations reinvert his inversion.

Notice that if the weights $w_i$ in $s_c$ are uniform (so no example is considered any more important than any other), then $\pi_c = s_c/S$, and all those terms cancel (regardless of the inversion).

Additionally, $l_{ck}$ is often simplified as

$$l_{ck} = 1\ if\ c \neq k,\ 0\ if\ c=k$$

So rewrite the risk as

$$R = \sum_{i=1}^n \min_{k \in [1,q]} \frac{1}{S} \sum_{c=1}^q \frac{s_c l_{ck}}{\pi_c} E[h_{ci} | \vec{x_i}]$$

And with the simplifying assumptions that all examples are equally important and misclassification is equally bad between all class pairs it becomes:

$$R = \sum_{i=1}^n \min_{k \in [1,q]} \sum_{c \neq k} E[h_{ci} | \vec{x_i}]$$

Or equivalently:

$$R = \sum_{i=1}^n \max_{k \in [1,q]} E[h_{ci} | \vec{x_i}]$$

because the sum is minimized by excluding the largest expectation.\newline

Now, recognize $E[\vec{h_c} | \pmb{X}]$ is a vector of the values $E[h_{ci} | \vec{x_i}]$, and for training data the expectation that $h_{ci}$ has a given value given $\vec{x_i}$ is known to be either a one or a zero, so $E[\vec{h_c} | \pmb{X}] = \vec{h_c}$, $h_{ci} \forall i$ stacked together.

Further, recognize that stacking $\vec{h_c}$ for all classes $c$ together as columns yields $\pmb{H}$, a one-hot representation of the true classifications $\pmb{Y}$. That is:

\[
\pmb{Y} =
\begin{bmatrix}
1\\
0\\
2\\
.\\
.\\
3\\
\end{bmatrix}
%%%%%
\ \ \ \ \ \ \ \pmb{H} =
\begin{bmatrix}
0 & 1 & 0 & 0\\
1 & 0 & 0 & 0\\
0 & 0 & 1 & 0\\
& \hdots & \hdots\\
& \hdots & \hdots\\
0 & 0 & 0 & 1
\end{bmatrix}
\]

And now we can model $\pmb{H}$ with a multivariate projection pursuit regression model, where we take the predicted class of example $i$ to be:

$$\hat{y_i} = \argmax_c \hat{h_{ci}} $$

That is: the predicted class is the index of the column where the largest value in the $i$\textsuperscript{th} row of $\hat{\pmb{H}}$, the predicted $\pmb{H}$, is located. If $\pmb{Y}$ is filled with generalized categories rather than numbers, then categoricals can be assigned numbers for the construction of $\pmb{H}$, and argmaxes can be translated back at prediction-time.

And just like that we have reduced univariate classification to multivariate regression! There is one caveat: Training the model to make these predictions should ideally involve optimizing the the misclassification risk as the loss function, not the sum-of-squares loss function from section 2 as is done for actual regression. But the $\max_k$ in the risk equation makes it nonconvex, which means we can no longer employ the methods detailed in section 3 to find model parameters.

Fortunately, Friedman assures us (and experiment bears out) that using the quatratic loss function is acceptable, and if we wish to account for examples being of differing importances or specify a funky non-uniform pairwise loss scheme, all we have to do is use weights:

$$w_c = \frac{s_c}{S \pi_c} \sum_{k \in [1,q]} l_{ck}$$

Since weights are relative, the constant $S$ and the normalization by the constant $n$ in the calculation of $\pi_c$ can be dropped to yeild:

$$w_c = \frac{\sum_{i=1}^n w_i h_{ci}}{\sum_{i=1}^n h_{ci}} \sum_{k \in [1,q]} l_{ck}$$

Vectorizing to obtain this weight for all classes at once we get:

$$\vec{w_C} = \frac{\vec{w_I} \cdot \pmb{H}}{\sum_{i=1}^n \pmb{H}} \odot \sum_{k=1}^q \pmb{L}$$

where $\pmb{L}[c,k] = l_{ck}$, the sum over $k$ amounts to a sum along the horizontal (second) axis, the sum over $n$ of $\pmb{H}$ amounts to a sum along the vertical (first) axis, and the division is pointwise on the vectors involved.

\end{document}


\chapter{Breaking the continuum hypothesis}
We now use the technique of forcing to break the
Continuum Hypothesis by choosing a good poset $\Po$.
As I mentioned earlier, one can also build a model
where the Continuum Hypothesis is true;
this is called the \emph{constructible universe},
(this model is often called ``$V=L$'').
However, I think it's more fun when things break\dots

%\section{Forcing $V \neq L$ is really easy}
%As a small aside, to check we're on the right track we show the following result.
%
%\begin{theorem}[$V \ne L$]
%	Let $M$ be a countable transitive model of $\ZFC$.
%	Let $\Po \in M$ be \emph{any} splitting poset,
%	and let $G \subseteq \Po$ be $M$-generic.
%	Then $M[G] \vDash (V \neq L)$.
%\end{theorem}
%\begin{proof}
%	Since $L$ has a $\Sigma_1$ definition,
%	we have \[ L^{M[G]} = L^M \subseteq M \subsetneq M[G] \]
%	where the last part follows from $G \notin M[G]$.
%\end{proof}
%
%Thus $M[G] \vDash \ZFC + (V \ne L)$ for any splitting poset $\Po$,
%and we are one step closer to breaking $\CH$.

\section{Adding in reals}
Starting with a \emph{countable} transitive model $M$.

We want to choose $\Po \in M$ such that $(\aleph_2)^M$ many real numbers appear,
and then worry about cardinal collapse later.

Recall the earlier situation where we set $\Po$ to be the infinite complete binary tree; its nodes can be thought of as partial functions $n \to 2$ where $n < \omega$.
Then $G$ itself is a path down this tree; i.e.\ it can be encoded as a total function $G : \omega \to 2$,
and corresponds to a real number.

\begin{center}
	\begin{asy}
		size(8cm);
		pair P = Drawing("\varnothing", (0,4), dir(90), red);
		pair P0 = Drawing("0", (-5,2), 1.5*dir(90), red);
		pair P1 = Drawing("1", (5,2),  1.5*dir(90));
		pair P00 = Drawing("00", (-7,0), 1.4*dir(120));
		pair P01 = Drawing("01", (-3,0), 1.4*dir(60), red);
		pair P10 = Drawing("10", (3,0),  1.4*dir(120));
		pair P11 = Drawing("11", (7,0),  1.4*dir(60));

		pair P000 = Drawing("000", (-8,-3));
		pair P001 = Drawing("001", (-6,-3));
		pair P010 = Drawing("010", (-4,-3), red);
		pair P011 = Drawing("011", (-2,-3));

		pair P100 = Drawing("100", (2,-3));
		pair P101 = Drawing("101", (4,-3));
		pair P110 = Drawing("110", (6,-3));
		pair P111 = Drawing("111", (8,-3));

		draw(P01--P0--P00);
		draw(P11--P1--P10);
		draw(P0--P--P1);
		draw(P000--P00--P001);
		draw(P100--P10--P101);
		draw(P010--P01--P011);
		draw(P110--P11--P111);

		draw(P--P0--P01--P010--(P010+2*dir(-90)), red+1.4);
		MP("G", P010+2*dir(-90), dir(-90), red);
	\end{asy}
\end{center}

We want to do something similar,
but with $\omega_2$ many real numbers instead of just one.
In light of this, consider in $M$ the poset
\[
	\Po = \opname{Add} \left( \omega_2, \omega \right)
	\defeq \left( \left\{ p : \omega_2 \times \omega \to 2,
		\dom(p) \text{ is finite} \right\},
	\supseteq \right).
\]
These elements $p$ (conditions) are ``partial functions'':
we take some finite subset of $\omega_2 \times \omega$ and map it into $2=\{0,1\}$.
(Here $\dom(p)$ denotes the domain of $p$,
which is the finite subset of $\omega_2 \times \omega$ mentioned.)
Moreover, we say $p \le q$ if $\dom(p) \supseteq \dom(q)$
and the two functions agree over $\dom(q)$.

\begin{ques}
	What is the maximal element $1_\Po$ here?
\end{ques}

\begin{exercise}
	Show that a generic $G$ can be encoded as a function $\omega_2 \times \omega \to 2$.
\end{exercise}

%Let $G \subseteq \opname{Add}(\omega_2, \omega)$ be an $M$-generic.
%We claim that, like in the binary case, $G$ can be encoded as a function $\omega_2 \times \omega \to 2$.
%To see this, consider $\alpha \in \omega_2$ and $n \in \omega$; we have the dense set
%\[ D_{\alpha, n}
%	= \left\{ p \in \opname{Add}(\omega_2, \omega)
%	\mid (\alpha, n) \in \dom(p) \right\}
%\]
%(this is obviously dense, given any $p$ add in $(\alpha, n)$ if it's not in there already).
%So $G$ hits this dense set, meaning that for every $(\alpha, n)$ there's a function in $G$ which defines it.
%Using the fact that $G$ is upwards closed and a filter, we may as before we may interpret $G$ as a function $\omega_2 \times \omega \to 2$.

\begin{lemma}[$G$ encodes distinct real numbers]
	For $\alpha \in \omega_2$ define
	\[ G_\alpha = \left\{ n \mid G\left( \alpha,n \right) = 0 \right\} \in \PP(\NN). \]
	Then $G_\alpha \neq G_\beta$ for any $\alpha \neq \beta$.
\end{lemma}
\begin{proof}
	We claim that the set
	\[ D = \left\{ q \mid \exists n \in \omega :
		q\left( \alpha, n \right) \neq q\left( \beta, n \right)
		\text{ are both defined}
	\right\} \]
	is dense.
	\begin{ques}
		Check this.
		(Use the fact that the domains are all finite.)
	\end{ques}
%	This is pretty easy to see.
%	Consider $p \in \opname{Add}(\omega_2, \omega)$.
%	Then you can find an $n$ such that
%	neither $(\alpha, n)$ nor $(\beta, n)$ is defined,
%	just because $\dom(p)$ is finite.
%	Then you make $p'$ as $p$ plus $p'( (\alpha, n) ) = 1$
%	and $p'( (\beta, n) ) = 0$.
%	Hence the set is dense.

	Since $G$ is an $M$-generic it hits this dense set $D$.
	Hence $G_\alpha \neq G_\beta$.
\end{proof}

Since $G \in M[G]$ and $M[G] \vDash \ZFC$,
it follows that each $G_\alpha$ is in $M[G]$.
So there are at least $\aleph_2^M$ real numbers in $M[G]$.
We are done once we can show there is no cardinal collapse.

\section{The countable chain condition}
It remains to show that with $\Po = \opname{Add}(\omega, \omega_2)$, we have that
\[ \aleph_2^{M[G]} = \aleph_2^M. \]
In that case, since $M[G]$ will have $\aleph_2^M = \aleph_2^{M[G]}$ many reals, we will be done.

To do this, we'll rely on a combinatorial property of $\Po$:
\begin{definition}
	We say that $A \subseteq \mathcal P$ is a \vocab{strong antichain}
	if for any distinct $p$ and $q$ in $A$, we have $p \perp q$.
\end{definition}
\begin{example}[Example of an antichain]
	In the infinite binary tree,
	the set $A = \{00, 01, 10, 11\}$ is a strong antichain
	(in fact maximal by inclusion).
\end{example}
This is stronger than the notion of ``antichain'' than you might be used to!\footnote{%
	In the context of forcing, some authors use ``antichain'' to refer to ``strong antichain''.
	I think this is lame.}
We don't merely require that every two elements are incomparable,
but that they are in fact \emph{incompatible}.
\begin{ques}
	Draw a finite poset and an antichain of it which is not strong.
\end{ques}

\begin{definition}
	A poset $\Po$ has the \vocab{$\kappa$-chain condition}
	(where $\kappa$ is a cardinal) if all strong antichains
	in $\Po$ have size less than $\kappa$.
	The special case $\kappa = \aleph_1$ is called the \vocab{countable chain condition},
	because it implies that every strong antichain is countable.
\end{definition}

We are going to show that if the poset has the $\kappa$-chain condition
then it preserves all cardinals greater than $\kappa$.
% or was it > \kappa?
In particular, the countable chain condition will show that $\Po$ preserves all the cardinals.
Then, we'll show that $\opname{Add}(\omega, \omega_2)$ does indeed have this property.
This will complete the proof.

We isolate a useful lemma:
\begin{lemma}[Possible values argument]
	Suppose $M$ is a transitive model of $\ZFC$ and $\Po$ is a partial order
	such that $\Po$ has the $\kappa$-chain condition in $M$.
	Let $X,Y \in M$ and let $f: X \to Y$
	be some function in $M[G]$, but $f \notin M$.

	Then there exists a function $F \in M$, with $F: X \to \PP(Y)$ and such that
	for any $x \in X$,
	\[ f(x) \in F(x) \quad\text{and}\quad \left\lvert F(x) \right\rvert^M < \kappa. \]
\end{lemma}
What this is saying is that if $f$ is some new function that's generated,
$M$ is still able to pin down the values of $f$ to at most $\kappa$ many values.

\begin{proof}
	The idea behind the proof is easy: any possible value of $f$ gives us some condition in
	the poset $\Po$ which forces it.
	Since distinct values must have incompatible conditions,
	the $\kappa$-chain condition guarantees
	there are at most $\kappa$ such values.

	Here are the details.
	Let $\dot f$, $\check X$, $\check Y$ be names for $f$, $X$, $Y$.
	Start with a condition $p$ such that $p$ forces the sentence
	\[ \text{``$\dot f$ is a function from $\check X$ to $\check Y$''}. \]
	We'll work just below here.

	For each $x \in X$, we can consider (using the Axiom of Choice) a maximal strong antichain $A(x)$
	of incompatible conditions $q \le p$ which forces $f(x)$ to equal some value $y \in Y$.
	Then, we let $F(x)$ collect all the resulting $y$-values.
	These are all possible values, and there are less than $\kappa$ of them.
\end{proof}

\section{Preserving cardinals}
As we saw earlier, cardinal collapse can still occur.
For the Continuum Hypothesis we want to avoid this possibility,
so we can add in $\aleph_2^M$ many real numbers and have $\aleph_2^{M[G]} = \aleph_2^M$.
It turns out that to verify this, one can check a weaker result.

\begin{definition}
	For $M$ a transitive model of $\ZFC$ and $\Po \in M$ a poset,
	we say $\Po$ \vocab{preserves cardinals} if
	$\forall G \subseteq \Po$ an $M$-generic,
	the model $M$ and $M[G]$ agree on the sentence ``$\kappa$ is a cardinal'' for every $\kappa$.
	Similarly we say $\Po$ \vocab{preserves regular cardinals} if $M$ and $M[G]$
	agree on the sentence ``$\kappa$ is a regular cardinal'' for every $\kappa$.
\end{definition}
Intuition:
In a model $M$, it's possible that two ordinals which are in bijection in $V$ are no longer in bijection in $M$.
Similarly, it might be the case that some cardinal $\kappa \in M$ is regular,
but stops being regular in $V$ because some function $f : \ol\kappa \to \kappa$ is cofinal but happened to only exist in $V$.
In still other words, ``$\kappa$ is a regular cardinal '' turns out to be a $\Pi_1$ statement too.

Fortunately, each implies the other.
We quote the following without proof.
\begin{proposition}[Preserving cardinals $\iff$ preserving regular cardinals]
	Let $M$ be a transitive model of $\ZFC$.
	Let $\Po \in M$ be a poset.
	Then for any $\lambda$,
		$\Po$ preserves cardinalities less than or equal to $\lambda$
		if and only if $\Po$ preserves regular cardinals less than or equal to $\lambda$.
	Moreover the same holds if we replace ``less than or equal to''
	by ``greater than or equal to''.
\end{proposition}

Thus, to show that $\Po$ preserves cardinality and cofinalities
it suffices to show that $\Po$ preserves regularity.
The following theorem lets us do this:
\begin{theorem}[Chain conditions preserve regular cardinals]
	Let $M$ be a transitive model of ZFC, and let $\Po \in M$ be a poset.
	Suppose $M$ satisfies the sentence ``$\Po$ has the $\kappa$ chain condition and $\kappa$ is regular''.
	Then $\Po$ preserves regularity greater than or equal to $\kappa$.
\end{theorem}
\begin{proof}
	Use the Possible Values Argument.
	\Cref{prob:chain}.
\end{proof}

In particular, if $\Po$ has the countable chain condition then $\Po$ preserves \emph{all} the cardinals (and cofinalities).
Therefore, it remains to show that $\opname{Add}(\omega, \omega_2)$ satisfies the countable chain condition.

\section{Infinite combinatorics}
We now prove that $\opname{Add}(\omega, \omega_2)$ satisfies the countable chain condition.
This is purely combinatorial, and so we work briefly.

\begin{definition}
	Suppose $C$ is an uncountable collection of finite sets.
	$C$ is a \vocab{$\Delta$-system} if there exists a \vocab{root} $R$
	with the condition that for any distinct $X$ and $Y$
	in $C$, we have $X \cap Y = R$.
\end{definition}

\begin{lemma}
	[$\Delta$-System lemma] Suppose $C$ is an uncountable collection of finite sets.
	Then $\exists \ol C \subseteq C$ such that $\ol C$ is an uncountable $\Delta$-system.
\end{lemma}
\begin{proof}
	There exists an integer $n$ such that $C$ has uncountably many guys of length $n$.
	So we can throw away all the other sets, and just assume that all sets in $C$ have size $n$.

	We now proceed by induction on $n$.
	The base case $n=1$ is trivial, since we can just take $R = \varnothing$.
	For the inductive step we consider two cases.

	First, assume there exists an $a \in C$ contained in uncountably many $F \in C$.
	Throw away all the other guys.
	Then we can just delete $a$, and apply the inductive hypothesis.

	Now assume that for every $a$, only countably many members of $C$ have $a$ in them.
	We claim we can even get a $\ol C$ with $R = \varnothing$.
	First, pick $F_0 \in C$.
	It's straightforward to construct an $F_1$ such that $F_1 \cap F_0 = \varnothing$.
	And we can just construct $F_2, F_3, \dots$
\end{proof}

\begin{lemma}
	For all $\kappa$, $\opname{Add}(\omega, \kappa)$ satisfies the countable chain condition.
\end{lemma}
\begin{proof}
	Assume not. Let
	\[ \left\{ p_\alpha : \alpha < \omega_1 \right\} \]
	be a strong antichain.  Let
	\[ C = \left\{ \dom(p_\alpha) : \alpha < \omega_1 \right\}. \]
	Let $\ol C \subseteq C$ be such that $\ol C$ is uncountable, and $\ol C$ is a $\Delta$-system with root $R$.
	Then let
	\[ B = \left\{ p_\alpha : \dom(p_\alpha) \in R \right\}. \]
	Each $p_\alpha \in B$ is a function $p_\alpha : R \to \{0,1\}$,
	so there are two that are the same.
\end{proof}

Thus, we have proven that the Continuum Hypothesis cannot be proven in $\ZFC$.

\section\problemhead
\begin{problem}
	\label{prob:chain}
	Let $M$ be a transitive model of ZFC, and let $\Po \in M$ be a poset.
	Suppose $M$ satisfies the sentence ``$\Po$ has the $\kappa$ chain condition and $\kappa$ is regular''.
	Show that $\Po$ preserves regularity greater than or equal to $\kappa$.
	\begin{hint}
		Assume not, and take $\lambda > \kappa$ regular in $M$;
		if $f : \ol \lambda \to \lambda$,
		use the Possible Values Argument on $f$ to generate a function in $M$
		that breaks cofinality of $\lambda$.
	\end{hint}
	\begin{sol}
		It suffices to show that $\Po$ preserves regularity greater than or equal to $\kappa$.
		Consider $\lambda > \kappa$ which is regular in $M$,
		and suppose for contradiction that $\lambda$ is not regular in $M[G]$.
		That's the same as saying that there is a function $f \in M[G]$,
		$f : \ol \lambda \to \lambda$ cofinal, with $\ol \lambda < \lambda$.
		Then by the Possible Values Argument,
		there exists a function $F \in M$ from $\ol \lambda \to \PP(\lambda)$
		such that $f(\alpha) \in F(\alpha)$ and $\left\lvert F(\alpha) \right\rvert^M < \kappa$
		for every $\alpha$.

		Now we work in $M$ again.
		Note for each $\alpha \in \ol\lambda$,
		$F(\alpha)$ is bounded in $\lambda$ since $\lambda$ is regular in $M$ and
		greater than $\left\lvert F(\alpha) \right\rvert$.
		Now look at the function $\ol \lambda \to \lambda$ in $M$ by just
		\[ \alpha \mapsto \cup F(\alpha) < \lambda. \]
		This is cofinal in $M$, contradiction.
	\end{sol}
\end{problem}


%
%  Teori.tex
%  Part of VertEgg documentation
%

\chapter{Theory of Vertical Egg Distributions}

\section{The Equations}\label{sec:equations}

Neglecting horizontal processes, the evolution of the vertical
distribution
of a class of fish eggs is governed by a simple \emph{conservation}
principle\index{conservation principle}

\begin{equation}\label{eq:consprin}
\begin{split}
  \text{The change}\ & \text{in the number of eggs between two depth levels} \\
  & = \text{the number of eggs entering at the lower level} \\
  & - \text{the number of eggs leaving at the upper level} \\
  & + \text{the number of eggs spawned} \\
  & - \text{the number of eggs that hatched or died}
\end{split}
\end{equation}

To put this in a more mathematical formulation, let the variable
$z$ denote depth and $t$ time. The $z$-axis is chosen to point
\emph{up}, that is $z = 0$ at the surface and negative values are
used in the water column. This choice is opposite from
\cite{sund83,sund91} and \cite{west89}. The present choice is
motivated by the most
common axis-convention in 3D hydrodynamic ocean modelling.

To make the theory more convenient mathematically, a
\emphi{continuum} approach is used, where the eggs are
replaced by a continuous egg \emphi{concentration}.  Let $\phi =
\phi(z,t)$ denote this egg concentration at depth $z$ and time
$t$. Let $F = F(z,t)$ be the upwards egg \emphi{flux}, i.e.\ the net
number of eggs passing upwards per time unit at depth $z$ and time
$t$. Similarly, let $Q = Q(z,t)$ denote the \emphi{source term}, the
number of eggs spawned minus the numbers that hatched or died per
depth and time unit at depth $z$ and time $t$.  The source term can
also be used to take care of other processes, such as loss of eggs by
horizontal advection.

The conservation principle \ref{eq:consprin} can then be formulated
in \emphi{integral form}
\begin{equation}\label{eq:intlaw}
\int_{z_1}^{z_2} \!\phi(z,t_2)\,dz - \int_{z_1}^{z_2}\!\phi(z,t_1)\,dz
   = \int_{t_1}^{t_2} \!F(z_1,t)\,dt
   - \int_{t_1}^{t_2} \!F(z_2,t)\,dt
   + \int_{t_1}^{t_2}\!\int_{z_1}^{z_2} \!Q(z,t)\,dzdt
\end{equation}
Mathematically this can be reformulated in mixed form, differential in time
and integral in space
\begin{equation}\label{eq:diffintlaw}
  \pddt \left( \int_{z_1}^{z_2} \!\phi(z,t)\,dz \right) =
  F({z_1},t) - F({z_2},t) + \int_{z_1}^{z_2}\!Q(z,t)\,dz
  \quad \mbox{for} \ -H \le {z_1} < {z_2} \le 0 .
\end{equation}
or in pure \emph{differential} form\index{differential form}
\begin{equation}\label{eq:difflaw}
  \Pd{\phi}{t} =  - \Pd{F}{z} + Q
\end{equation}

The flux is decomposed in two parts, the \emphi{convective flux} and the
\emphi{diffusive flux}.  The convection is due to the terminal buoyant
velocity $w = w(z,t)$ of the egg which is computed by the density
difference from the surroundings  and the egg size as described in section
\ref{sec:velocity}. The formulation is simply,
\begin{equation}\label{eq:convflux}
  \Fconv = w \phi .
\end{equation}
The diffusion is caused by turbulent mixing and is modelled by
Fick's law\index{Fick's law} using the vertical eddy diffusion
\index{eddy diffusion} $K = K(z,t)$,
\begin{equation}\label{eq:fick}
  \Fdiff = - K \frac{\partial \phi}{\partial z} .
\end{equation}

Often the source term $Q$ can be separated in a
\emph{spawning}\index{spawning term} or \emph{production}
term\index{production term} independent of the egg concentration
\begin{equation}\label{eq:spawn}
  \Qspawn = P
\end{equation}
and a \emph{mortality}\index{mortality term} or \emph{loss}
term\index{loss term} depending on the concentration
\begin{equation}\label{eq:loss}
  \Qloss = - \alpha \phi
\end{equation}
Using this the differential conservation law (\ref{eq:difflaw})
becomes
\bel{eq:difflawfull}
\Pd{\phi}{t} =  - \pddz (w \phi) + \pddz (K \Pd{\phi}{z}) + P - \alpha \phi
\end{equation}
This parabolic partial differential equations is
called the \emph{convection-diffusion}\index{convection-diffusion equation}
or \emph{transport} equation\index{transport equation}.

The boundary conditions\index{boundary conditions} are simply no flux
across the surface and bottom,
\bel{eq:boundcond}
  F(0,t) = F(-H,t) = 0,
\end{equation}
and the initial condition is given by
\bel{eq:initcond}
  \phi(z,0) = \phi_0(z),  \quad -H \le z \le 0.
\end{equation}

\section{Solutions of the equations}

Under simplified circumstances the equations in section~\ref{sec:equations}
can be solved analytically. This section examines some of these solutions.

\subsection{Vertical integrated equation}\label{sec:vertint}

Let $\Phi$ denote the total concentration, $\Phi = \Vint \phi dz$.
Take the vertical integral of equation~(\ref{eq:difflawfull}) and
use the boundary conditions (\ref{eq:boundcond}) gives
\be
\Od{\Phi}{t} = P^{tot} - \Vint \!\alpha \phi \, dz
\end{equation}
where $P^{tot} = \Vint P dz$ is the total spawning contribution.
Without any source terms the total concentration $\Phi$ is constant.
To reach a steady state solution with spawning, the loss term $\alpha$
must be nonzero.

If $\alpha$ is a positive
constant, the solution to the equation above is
\be
  \Phi(t) = \e^{-\alpha t} \Phi(0)
         + (1 - \e^{-\alpha t})  \frac{P^{tot}}{\alpha}
\end{equation}
with steady state solution $\Phi = P^{tot}/\alpha$.
If $\alpha = 0$ the solution is simply
\be
  \Phi(t) = \Phi(0) + t P^{tot} .
\end{equation}

\subsection{Stationary solution}\index{stationary solution}

A stationary solution solves the conservation law
\pref{eq:difflaw} without the time derivative
\begin{equation}\label{eq:stateq}
  \Od{F}{z} = Q
\end{equation}
with boundary conditions $F(0) = F(-H) = 0$ given by \pref{eq:boundcond}.
The condition for the existence of this solution is that the total
source term vanishes,
\begin{equation}
  Q^{tot} = \Vint \!Q \, dz= 0
\end{equation}
the flux function is then given by integrating \pref{eq:stateq}
\begin{equation}
  F(z) = \int_{-H}^z \! Q(s) \, ds = - \int_z^0 \! Q(s) \, ds .
\end{equation}

Without source term the expression above reduces to $F = 0$, i.e.\ the
net flux is zero everywhere. This simply says that in a steady state
convection is balanced by diffusion
\bel{eq:sstate}
  w \phi - K \Pd{\phi}{z} = 0 .
\end{equation}
This is an ordinary differential equation for $\phi(z)$.
The boundary conditions \pref{eq:boundcond} does not contain more information.
A unique solution can nevertheless be singled out by the
integral condition
\begin{equation}\label{eq:intcond}
  \Vint \!\phi(z)\,dz = \Phi .
\end{equation}
In other words, among all the solutions of (\ref{eq:sstate})
choose the one with correct vertically integrated concentration.

To simplify the notation put $m = w / K$ and let $M(z) = - \int_z^0 m(s)
ds$. Then the stationary solution is
\begin{equation}\label{eq:sstatesol}
  \phi(z) = \frac{\Phi}{\int_{-H}^0 \e^{M(s)} ds} \e^{M(z)} .
\end{equation}

\subsubsection{Constant coefficients}

The case with constant coefficients was studied by Sundby
\shortcite{sund83}.  In this case $M(z) = m z$ and the solution is a
truncated exponential distribution,
\begin{equation}\label{eq:eggsact}
  \phi_m(z) = \Phi \frac{m}{1 - \e^{-mH}} \e^{mz} .
\end{equation}
In the toolbox, this solution is computed by the function \edbi{eggsact}.
This solution has the following symmetry between ascending and
descending velocity,
\begin{equation}\label{eq:symmetry}
  \phi_{-m}(z) = \phi_m(-H-z)
\end{equation}

With large depth and/or high ascending velocity  the
effect of the bottom may be neglected. In this case the distribution
is well approximated by an exponential distribution \index{exponential
distribution} with parameter $\lambda = 1 / m$, that is
\begin{equation}
  \phi_m(z) \approx \Phi m \e^{mz}, \quad \text{for $mH \gg 1$} .
\end{equation}

For positive values of $m$ the following series development from
\citep{sund83} is valid
\begin{equation}
  \phi_m(z) = \Phi m \e^{mz} \sum_{j = 0}^{\infty} \e^{-jmH}, \quad m > 0 .
\end{equation}
The zero-th term of this expansion is the exponential distribution above.
For negative values of $m$ the symmetry relation (\ref{eq:symmetry})
can be used,
\begin{equation}
  \phi_m(z) = - \Phi m \e^{mz} \sum_{j = 1}^{\infty} \e^{jmH}, \quad m < 0 .
\end{equation}

To compute the mean and variance of the distribution the constant
factor $\Phi$ can be dropped. The moment generating function is then
\begin{equation}\label{eq:momgen}
  \psi(t) = \int_{-H}^0 \frac{m}{1 - \e^{-mH}} \e^{(m+t)z} dz
          = \frac{m}{1 - \e^{-mH}} \frac{1 - \e^{-(m+t)H}}{m+t}.
\end{equation}
Differentiating $\psi$ gives the moments of the distribution.
The mean depth is given by
\begin{equation}\label{eq:eggsactmean}
  \mu = \psi'(0) = - \frac{1}{m} + \frac{H}{\e^{mH} - 1}
\end{equation}
The function $\mu = \mu(m)$ with $H = 100\m$ is plotted in
figure~\ref{fig:mdepth}.  The variance is
\begin{equation}\label{eq:eggsactvar}
       \psi''(0) - \mu^2
      = \frac{2 - \e^{-mH}(m^2 H^2 + 2 m H + 2)}{m^2 ( 1 - \e^{-mH})} - \mu^2
\end{equation}

\begin{figure}
\begin{center}
\includegraphics[height=6cm]{mdepth}
\end{center}
\caption{Mean depth $\mu$ as a function of $m$}\label{fig:mdepth}
\end{figure}

\subsubsection{Piecewise constant coefficients}

The solution (\ref{eq:eggsact}) can easily be extended to a piecewise
constant $m$.
Let $0 = z_0 > \cdots > z_m = -H$ be a partition of the water column
with $m(z) = m_i$ on the interval $I_i = (z_i, z_{i-1})$.
Then the stationary solution restricted to the interval $I_i$ is
given as
\begin{equation}\label{eq:pwconst}
  \phi(z) = C_i \e^{m_i z}, \quad \mbox{for} \ z_i < z < z_{i-1}
\end{equation}
The $C_i$-s are determined by continuity
of the solution
\begin{equation}
  C_i \e^{m_i z_i} = C_{i+1} \e^{m_{i+1} z_i}, \quad  i = 1, \ldots, m-1
\end{equation}
and the integral condition~(\ref{eq:intcond})
\begin{equation}
   \sum_{i = 1}^m C_i \int_{z_i}^{z_{i-1}} \e^{m_i z} dz
   = \sum_{i = 1}^m C_i \frac{\e^{m_i z_{i-1}} - \e^{m_i z_i}}{m_i}
   = \Phi .
\end{equation}
This method is used in the function \edbi{sstate} to approximate the
general steady state solution. More details of the implementation is
given in section~\ref{sec:statprob}.

\subsubsection{Linear coefficients}

In Sundby \shortcite{sund91} the solution is derived for $m$ linear.
Let $m$ be given as $m(z) = a (z-z_0)$. Then $M(z) =
\half a ((z - z_0)^2 - z_0^2)$. Taking the last term into the
constant $C$, the solution becomes
\begin{equation}
  \phi(z) = C \e^{\half a (z - z_0)^2} .
\end{equation}
Bathypelagic eggs are neutral buoyant at $z = z_0$, raising if deeper
and sinking if higher in the water column. In this case the coefficient
$a$ is negative and the concentration has a normal distribution about $z =
z_0$ with variance $\sigma^2 = {1 / |a|}$.

%In a sample of eggs the diameters and buoyancies differ from egg to egg.
%A more realistic model of the stationary situation will therefore be to
%use a distribution of the $m$-values. This task was studied by Sundby (1983).
%...

\subsubsection{Stationary solution with source terms}

With source terms, the steady state equation is
\begin{equation}\label{eq:srcsstate}
  0 = - \pddz (w \phi) + \pddz (K \Pd{\phi}{z}) + P - \alpha \phi .
\end{equation}
This is a general second order ordinary
differential equation. With constant coefficients the equation becomes
\begin{equation}\label{eq:srcsacteq}
  K \phi'' - w \phi' - \alpha \phi = - P
\end{equation}
and the no flux boundary conditions are
\begin{equation}
  K \phi' - w \phi, \quad z = 0, z = -H .
\end{equation}
The solution can be written
\bel{eq:srssact}
  \phi(z) = A \e^{az} - B \e^{-bz} + \frac{P}{\alpha}
\end{equation}
with
\begin{align}
  a & = \frac{1}{2K} ( \sqrt{w^2 + 4 \alpha K} + w ) \\
  b & = \frac{1}{2K} ( \sqrt{w^2 + 4 \alpha K} - w ) \\
  A & = \frac{w P}{\alpha b K} \frac{1-\e^{-b H}}{1-\e^{-(a+b)H}} \\
  B & = \frac{w P}{\alpha a K} \e^{-b H} \frac{1-\e^{-a H}}{1-\e^{-(a+b)H}} .
\end{align}

This function is computed in the toolbox by the function \edbi{srcsact}.
The signs are chosen such that positive velocity $w$ makes all terms
positive. For negative velocities a symmetry property
similar to equation~(\ref{eq:symmetry})
can be used, if
$\phi(z)$ is a solution to equation (\ref{eq:srcsacteq}) and boundary
conditions (\ref{eq:boundcond}) then $\phi(-H-z)$ is a solution the
same equation and boundary conditions with the opposite sign on $w$.
In other words,
\begin{equation}
  \phi_{-w}(z) = \phi_w(-H-z) .
\end{equation}

\section{The Terminal Egg Velocity}\label{sec:velocity}
\index{egg velocity}\index{terminal velocity}

An egg in sea water will reach its terminal velocity $w$, where the
buoyant forcing balances the frictional drag. This velocity is a
function of the difference $\Delta \rho = \rho - \rho_e$ between the
density of the water and the egg, the egg diameter $d$, the
acceleration $g$ due to gravity and the molecular
viscosity\index{molecular viscosity}. Here the \emph{dynamic}
molecular viscosity\index{dynamic viscosity} $\mu$ is used.

The situation is characterised by the non-dimensional \emphi{Reynolds
number},
\begin{equation}
  Re = \frac{\rho d w}{\mu}
\end{equation}
For low values, $Re < 0.5$, the terminal velocity is given by
\emphi{Stokes' formula}
\begin{equation}\label{eq:stokes}
  w = \frac{1}{18} \frac{g d^2 \Delta \rho}{\mu} .
\end{equation}
This formula was obtained by Stokes \shortcite{stok1851}. The
derivation of the formula is given in almost any textbook on
fluid dynamics, for instance \citep{yih77}.

Combining these equations, one obtains an expression for $D$, the maximum
diameter for which Stokes' velocity applies,
\begin{equation}
  D^3 = \frac{9 \mu^2}{\rho g \Delta \rho}
\end{equation}

In the intermediate region $0.5 < Re < 5$, Dallavalle (1948) gave an
empirical formula\index{Dallavalle's formula}
\begin{equation}\label{eq:dallavalle}
  w = K_I (d - \zeta D) \Delta \rho^{2/3} \mu^{-1/3}
\end{equation}
where $\zeta = 0.4$ for a sphere. The coefficient $K_I$ is determined
by the requirement that both formulas should give the same answer for
$Re = 0.5$ or equivalently $d = D$.
This gives
\begin{equation}
  K_I = \frac{5}{54} 9^{1/3} g^{2/3} \rho^{-1/3}
         = 0.0875 \text{kg}^{-1/3} \,
             \text{m}^{5/3} \text{s}^{-4/3} .
\end{equation}

The formulas above are implemented as the function \edbi{eggvel} in
the VertEgg toolbox. The buoyancy of a fish egg is often given as the
salinity $S_e$ where the egg is neutrally buoyant. The function
\edbi{eggvelst} computes the terminal velocity in this case.

To compute the egg velocity the density, $\rho$ of water is needed.
In the toolbox only density at surface pressure is presently
available.  This is computed by the function \edbi{dens0} by the
UNESCO formula \citep{UNES81}.  The function \edbi{sw\_dens} in the
SEAWATER toolbox\index{SEAWATER} \citep{morg94} has implemented the
full UNESCO equation of state.

The dynamic molecular
viscosity\index{viscosity!molecular}\index{viscosity!dynamic} $\mu$ of
sea water is tabulated in table \ref{tab:molvisc} taken from Sverdrup
\emph{et al.}
\shortcite{sver52}. The values decrease  with temperature and increase
slowly with salinity. The dependence on pressure is insignificant, and is
neglected here.

\begin{table}[h]
\begin{center}
\begin{tabular}{|c|c|c|c|c|c|c|c|}
\hline\hline
Salinity   & \multicolumn{7}{c|}{Temperature [\deg C]} \\
\cline{2-8}
[psu] & 0 & 5 & 10 & 15 & 20 & 25 & 30 \\
\hline
 0 & 1.79 &  1.52 & 1.31 & 1.14 & 1.01 & 0.89 & 0.80 \\
10 & 1.82 &  1.55 & 1.34 & 1.17 & 1.03 & 0.91 & 0.82 \\
20 & 1.85 &  1.58 & 1.36 & 1.19 & 1.05 & 0.93 & 0.84 \\
30 & 1.88 &  1.60 & 1.38 & 1.21 & 1.07 & 0.95 & 0.86 \\
35 & 1.89 &  1.61 & 1.39 & 1.22 & 1.09 & 0.96 & 0.87 \\
\hline
\end{tabular}
\end{center}
 \caption{Dynamic molecular viscosity of sea water
  with unit $10^{-3} \,\text{kgm}^{-1}\text{s}^{-1}$}
  \label{tab:molvisc}
\end{table}

Using linear least squares regression the table is approximated by the
following function where $T$ is the temperature in \deg C and $S$ is
the salinity in psu.
\begin{equation}\label{eq:molvisc}
  \mu = 10^{-3} \, (1.7915 - 0.0538 \, T + 0.007 \, T^2 - 0.0023 \, S )
            \:   \text{kgm}^{-1}\text{s}^{-1} ,
\end{equation}
This reproduces table \ref{tab:molvisc} with an absolute error less
than $2 \times 10^{-5}\, \text{kgm}^{-1}\text{s}^{-1}$ and a relative
error of 1.7 \%. This is good enough to compute egg velocities where
the uncertainty in the other variables are larger.  This formula is
implemented by the function
\edbi{molvisc} in the toolbox. Riley and Skirrow \shortcite{rile75}
give a more precise formula which requires more computational effort.


\documentclass[10pt]{article}
\usepackage[utf8]{inputenc}
\usepackage{xcolor}
\usepackage[T1]{fontenc}
\usepackage{pagecolor}
\usepackage{amssymb}
\usepackage{lmodern}
\usepackage{mathtools, nccmath}
\usepackage{courier}
\usepackage[dvipsnames]{xcolor}

\definecolor{myyellow}{RGB}{225,225,100}
\definecolor{myred}{RGB}{220,100,100}
\definecolor{mygreen}{RGB}{120,225,120}
\definecolor{myblue}{RGB}{100,200,255}
\definecolor{mypurple}{RGB}{200,50,255}
\color{white}

\title{Geometry with complex numbers \#1}
\author{@all.about.mathematics}

\begin{document}

\maketitle
\pagecolor{black}
\LARGE
\section{Introduction}
In this series of posts, we discuss representations of geometrical objects with complex numbers and use them to prove some famous geometry theorems! We'll focus on lines and points in this post!

\noindent{Enjoy :D}
\newpage
\Large
\section{Lines with complex numbers}
First, let's think about lines segments on the complex plane. Let $z_1, z_2 \in\mathbb{C}$. How can we represent the line segment from $z_1$ and $z_2$?

\noindent{We know that the vector from $z_1$ to $z_2$ is $z_2-z_1$, so we can represent the line segment by the equation}
$$l=z_1+t(z_2-z_1)=z_1+tv$$
Where $t$ is a parameter so that $0\leq t\leq 1$ and $v$ is a non-zero vector in the direction of the line. Therefore, the line segment is the collection of all possible points $l$. Besides, if we let $t$ be any real number, then we get the line passing through $z_1$ and $z_2$.

\noindent
For example, let $z_1=3-2i$ and $z_2=4+5i$. Then the equation
$$L:3-2i+t(1+7i)$$
With $0\leq t\leq 1$ is the line segment from $z_1$ to $z_2$, and if $t\in\mathbb{R}$ then it is the line passing through $z_1$ and $z_2$.

\newpage
\section{Finding points of division}
Let's say we have 2 points $z_1,z_2\in\mathbb{C}$. Suppose we want to find a $z_3$ on the line segment $z_1 z_2$ such that it divides the length of the line segment by the ratio $a:b$, which is equivalent to
$$|z_1-z_3|:|z_2-z_3|=a:b$$
Consider the equation
$$L:=z_1+t(z_2-z_1)\qquad 0\leq t \leq 1$$
To find $z_3$,
$$t=\frac{a}{a+b}\implies z_3=z_1+\frac{a}{a+b}(z_2-z_1)=\frac{bz_1+az_2}{a+b}$$

\noindent
For example, let $z_1=4+5i\:,\: z_2=-4-3i$ and suppose we want to find $z_3$ such that it divides $z_1z_2$ by $3:5$. Then
$$z_3=\frac{5(4+5i)+3(-4-3i)}{3+5}=\frac{8+16i}{8}=1+2i$$

\newpage
\section{Parallel lines}
Given 2 lines $L_1\:,\: L_2$ and their equations
$$L_1:z_1+tv_1 \qquad t\in\mathbb{R}$$
$$L_1:z_2+sv_2 \qquad s\in\mathbb{R}$$
It is easy to see that if $v_1$ and $v_2$ are parallel, i.e.
$$\exists k\in\mathbb{R}:v_1=kv_2 $$
Stay tuned for the next post! We are going to use what we've learned here to easily prove a very useful theorem!
\end{document}

Tig. and Geometry/all.about.mathematics' questions/Complex number and geometry 1.tex


\chapter{Separating Hyperplanes, Lagrange Multipliers, and Convex Duality}
% \documentclass[draft,11pt]{article}
%\documentclass[11pt]{article}
%\input{../defs}
%\input{../layout-defs}
%\usepackage{amsmath}
%\usepackage{amssymb}
%\usepackage{amsthm}
%\usepackage{graphicx}
%\usepackage{float}
%\usepackage[ruled,vlined]{algorithm2e}
%\SetKwBlock{Repeat}{repeat}{}
%%% for this lecture
%\newcommand{\gap}{\text{gap}}
% \newtheorem{theorem}{Definition}
%\interfootnotelinepenalty=10000
% \newtheorem{lemma}{Lemma}

%\usepackage{tikz}
%\usetikzlibrary{arrows}

%%%% ADD MACROS HERE
% feel free to add more macros here

%\begin{document}

\sloppy
%\lecture{12 --- Wednesday, May 13th}
%{Spring 2020}{Rasmus Kyng, Scribe: Tim Taubner}{
% Separating Hyperplanes, Lagrange Multipliers, and Convex Duality}

\section{Overview}
First part of this chapter introduces the concept of a separating hyperplane of two sets followed by a proof that for two closed, convex and disjoint sets a separating hyperplane always exists.
This is a variant of the more general \emph{separating hyperplane theorem}%
\footnote{Wikipedia is good on this: \href{https://en.wikipedia.org/wiki/Hyperplane_separation_theorem}{https://en.wikipedia.org/wiki/Hyperplane\_separation\_theorem}} %
due to Minkowski.
Then \emph{Lagrange multipliers} $\xx$, $\ss$ of a convex optimization
problem
\begin{align*}
    \min_y{}\ & \energy(\yy) \\ \nonumber
\text{s.t.}\  & \AA\yy = \bb \\ \nonumber
              & c(\yy) \leq 0
\end{align*}
are introduced and with that, the \emph{Lagrangian} \begin{equation*} \LL(\yy, \xx, \ss) = \energy(\yy) + \xx^{\trp}(\bb - \AA\yy) + \ss^{\trp}c(\yy) \end{equation*}
is defined.
Finally, we deal with the dual problem
\begin{align*}
     \max_{\xx, \ss, \ss \geq 0}{} L(\xx,  \ss),
\end{align*}
where $L(\xx, \ss) = \min_y L(\yy, \xx, \ss)$. We show \emph{weak
  duality}, i.e.\ $L(\yy, \xx, \ss) \leq \energy(\yy)$ and that assuming \emph{Slater's condition} the values of both the primal and dual is equal, which is referred to as \emph{strong duality}.

\section{Separating Hyperplane Theorem}
Suppose we have two convex subsets $A, B \subseteq \R^n$ that are disjoint ($A \cup B = \emptyset$).
We wish to show that there will always be a (hyper-)plane $H$ that separates these two sets, i.e.~ $A$ lies on one side, and $B$ on the other side of $H$.

So what exactly do we mean by Hyperplane? Let's define it.
\begin{definition}[Hyperplane]
A \emph{hyperplane} $H$ of dimension $n$ is the subset $H := \{\xx \in \R^n : \ip{\nn, \xx} = \mu\}$.
We say $H$ has \emph{normal} $\nn \in \R^n$ and \emph{threshold} $\mu$.
It is required that $\nn \neq \veczero$.
\end{definition}

Every hyperplane divides $\R^n$ into two halfspaces $\{\xx : \ip{\vv, \xx} \geq \mu\}$ and $\{\xx : \ip{\vv, \xx} \leq \mu\}.$
It separates two sets, if they lie in different halfspaces. We formally define separating hyperplane as follows.

\begin{definition}[Separating Hyperplane]
We say a hyperplane $H$ \emph{separates} two sets $A, B$ iff
\begin{align*}
  \forall \aa \in A:& \ip{\nn, \aa} \geq \mu \\
  \forall \bb \in B:& \ip{\nn, \bb} \leq \mu
\end{align*}
If we replace $\geq$ with $>$ and $\leq$ with $<$ we say $H$ \emph{strictly} separates $A$ and $B$.
\end{definition}

It is easy to see that there exists disjoint non-convex sets that can not be
separated by a hyperplane (e.g.\ a point cannot be separated from a
ring around it). But can two disjoint convex sets always be strictly separated by a hyperplane?
The answer is no: consider the two-dimensional case depicted in
\autoref{fig:lefthyper} with $A = \{ (x,y): x \leq 0\}$ and $B = \{
(x, y) : x> 0 \text{ and } y \geq \frac{1}{x}\}$. Clearly they are disjoint; however the only separating hyperplane is $H = \{ (x, y): x = 0\}$ but it intersects $A$.

\begin{figure}[ht]
  \centering
  \includegraphics[width=1\textwidth]{fig/lec12-non-strict-separator.png}
  \caption{The sets $A = \{ (x, y): x \leq 0\}$ and $B = \{ (x, y): x
    > 0 \text{ and } y \geq \frac{1}{x} \}$ only permit a non-strictly separating hyperplane.}
  \label{fig:lefthyper}
\end{figure}

One can prove that there exists a non-strictly separating hyperplane for any two disjoint convex sets.
We will prove that if we further require $A$,$B$ to be closed and
bounded, then a strictly separating hyperplane always exists. (Note in
the example above how our choice of $B$ is not bounded.)

\begin{theorem}[Separating Hyperplane Theorem; closed, bounded sets] \label{th:shtcb}
For two closed, bounded, and disjoint convex sets $A, B \in \R^n$ there exists a strictly separating hyperplane $H$.
One such hyperplane is given by normal $\nn = \dd - \cc$ and threshold $\mu = \frac{1}{2}\left(\norm{\dd}_2^2 - \norm{\cc}_2^2\right)$, where $\cc \in A$, $\dd \in B$ are the minimizers of the distance between $A$ and $B$
\begin{equation*} \text{dist}(A, B) = \min_{\aa\in A, \bb \in B}\norm{\aa-\bb}_2 > 0.\end{equation*}
% Note that this distance is strictly larger than zero.
\end{theorem}
\begin{proof}
  We omit the proof that $\text{dist}(A, B) = \min_{\aa\in A, \bb \in
    B}\norm{\aa-\bb}_2 > 0$, which follows from $A,B$ being disjoint,
  closed, and bounded.
Now, we want to show that $\ip{\nn,\bb} > \mu$ for all $\bb \in B$; then $\ip{\nn,\aa} < \mu$ for all $\aa\in A$ follows by symmetry.
Observe that
\vspace{-1em}
\begin{align*}
  \ip{\nn, \dd} - \mu&= \ip{\dd-\cc,\dd} - \frac{1}{2}\left(\norm{\dd}_2^2-\norm{\cc}_2^2\right)\\
                    &= \norm{\dd}_2^2 - \dd^{\trp}\cc - \frac{1}{2}\norm{\dd}_2^2 + \frac{1}{2}\norm{\cc}_2^2 \\
										&= \frac{1}{2}\norm{\dd-\cc}_2^2 > 0.
\end{align*}
So suppose there exists $\uu \in B$ such that $\ip{\nn, \uu} - \mu \leq 0$.
We now look at the line defined by the distance minimizer $\dd$ and the point on the ``wrong side'' $\uu$.
Define $\bb(\lambda) = \dd + \lambda(\uu - \dd)$, and take the derivative of the distance between $\bb(\lambda)$ and $\cc$.
Evaluated at $\lambda=0$ (which is when $\bb(\lambda) = \dd$), this yields
\begin{equation*}
  \left.\frac{d}{d\lambda} \norm{\bb(\lambda) - \cc}_2^2\right\rvert_{\lambda=0} =
	\left.2\ip{\dd-\lambda\dd+\lambda\uu-\cc,\uu-\dd}\right\rvert_{\lambda=0} =
	2\ip{\dd-\cc, \uu-\dd}.
\end{equation*}

However, this would imply that the gradient is strictly negative since
\begin{align*}
\ip{\nn, \uu} - \mu & = \ip{\dd-\cc,\uu} - \ip{\dd-\cc,\dd} + \ip{\dd-\cc,\dd} - \mu \\
                  & = \ip{\dd-\cc,\uu-\dd} + \norm{\dd}_2^2 - \ip{\cc,\dd} - \frac{1}{2}\norm{\dd}_2^2+\frac{1}{2}\norm{\cc}_2^2 \\
                  & = \ip{\dd-\cc,\uu-\dd} + \frac{1}{2}\norm{\dd-\cc}_2^2 \leq 0.
\end{align*}
This contradicts the minimality of $\dd$ and thus concludes this proof.
\end{proof}

A more general separating hyperplane theorem holds even when the sets
are not closed and bounded:

\begin{theorem}[Separating Hyperplane Theorem] \label{th:sht}
Given two disjoint convex sets $A, B \in \R^n$ there exists a hyperplane $H$
separating them.
\end{theorem}

\section{Lagrange Multipliers and Duality of Convex Problems}

In this Section, we'll learn about \emph{Langrange Multipliers} and
how they lead to convex duality.
But first, let's see an example to help illustrate where these ideas
come from.

Imagine you were to prove that for all $\xx \in \R^n$ we have $\norm{\xx}_p \leq n^{\frac{1}{2} - \frac{1}{p}}\norm{\xx}_2$ for some $1 \leq p \leq 2;$.
We can look at this as optimizing $\max_{\xx} \norm{\xx}_p$ subject to $\norm{\xx}_2$ being constant, e.g.\ simply $\norm{\xx}_2 = 1$. Then the statement above follows from a scaling argument.

\begin{figure}[ht]
  \centering
  \includegraphics[width=.48\textwidth]{fig/lec12-2p-norms.jpeg}
  \caption{Looking at fixed $\norm{\xx}_p = \alpha$ and $\norm{\xx}_2=1$. (Here, $p = 1.5$.)}
  \label{fig:2-p-norm}
\end{figure}

If we move from $\xx$ to $\xx + \ddelta$ with $\ddelta \perp \grad_{\xx}\norm{\xx}_2$ and $\ddelta \not\perp \grad_{\xx}\norm{\xx}_p$ means that for infinitesimally small $\ddelta$ the 2-norm stays constant but the $p$-norm changes.
That means for either $\xx - \ddelta$ or $\xx + \ddelta$ the $p$-norm increases
while the 2-norm stays constant. Hence at the maximum of $\norm{\xx}_p$ the gradients of both norms have to be parallel, i.e.~\begin{equation*}\grad_{\xx}\left(\norm{\xx}_p - \lambda\norm{\xx}_2\right) = 0.\end{equation*}
%Here, $\lambda$ is the
This insight is the core idea of Lagrange multipliers (in this case $\lambda$).

Note that here the problem is not convex, because $\{ \xx: \norm{\xx}_2^2 =
1 \}$ is not convex and because we are asking to \emph{maximize} a norm.
In the following we will study Lagrange multipliers for general convex problems.

\subsection{General Convex Problems}

A full formal treament of convex duality would require us to be more careful
about using $\inf$ and $\sup$ in place of $\min$ and $\max$, as well
as considering problems that have no feasible solutions.
Today, we'll ignore these concerns.

Let us consider a general convex optimization problem with convex objective, linear equality constraints and convex inequality constraints
\begin{align}
  \label{eq:primalprob}
     \min_{y \in S}{} & \energy(\yy) \\ \nonumber
\text{s.t.}\  & \AA\yy = \bb \\ \nonumber
              & \cc(\yy) \leq \veczero,
\end{align}
where $\energy(\yy): S \rightarrow \R$ is defined on a convex subset $S \subseteq \R^n$, $\AA \in \R^{m\times n}$
and $\cc(\yy)$ is a vector of constraints $\cc(\yy) = \left(c_i(\yy)\right)_{i \in [k]}$.
For every $i \in [k]$ the function $c_i: S \rightarrow \R$ is convex.

\begin{definition}[Primal feasibility]
We say that $\yy \in S$ is \emph{primal feasible} if all constraints are satisfied, i.e.~$\AA\yy = \bb$ and $\cc(\yy) \leq \veczero$.
\end{definition}

In the following we will denote by $\alpha^* = \energy(\yy^*)$ the optimal value of the primal program where $\yy^*$ is an minimizer.

\begin{definition}
Next we introduce the \emph{dual variables} $\xx \in \R^m$, $\ss \in \R^k$ and define the \emph{Lagrangian} as
\begin{equation*} L(\yy, \xx, \ss) = \energy(\yy) + \xx^{\trp}(\bb-\AA\yy) + \ss^{\trp}\cc(\yy). \end{equation*}
We also define a Lagrangian only in terms of the dual variables by minimizing over $\yy$ as
\begin{equation*} L(\xx, \ss) = \min_{\yy}L(\yy, \xx, \ss). \end{equation*}
\end{definition}

\begin{definition}[Dual feasibility]
We say $(\xx, \ss)$ is dual feasible if $\ss \geq 0$.
If additionally $\yy$ is primal feasible, we say $(\yy, \xx, \ss)$ is primal-dual feasible.
\end{definition}

\begin{definition}[Dual problem]
We define the \emph{dual problem} as
\begin{align}
  \label{eq:dualprob}
  \max_{\substack{\xx, \ss \\ \ss \geq 0}}
  \min_{\yy}L(\yy, \xx, \ss)
  =
  \max_{\substack{\xx, \ss \\ \ss \geq 0}} L(\xx,  \ss)
  \end{align}
and denote the optimal dual value by $\beta^*$.
\end{definition}

For each $\yy$, the Lagrangian $L(\yy, \xx,
\ss)$ is linear in $(\xx,\ss)$ and hence also concave in them.
Hence $L(\xx,  \ss)$ is a concave function, because it is the pointwise
minimum (over $\yy$), of a collection of concave functions in
$(\xx,\ss)$.

This also means that the dual problem is really a convex optimization
problem in disguise, because we can flip the sign of $-L(\xx,  \ss)$
to get a convex function and minimizing this is equivalent to
maximizing $L(\xx,  \ss)$.
\begin{align*}
  \max_{\substack{\xx, \ss \\ \ss \geq 0}} L(\xx,  \ss)
  =
  -
  \min_{\substack{\xx, \ss \\ \ss \geq 0}} -L(\xx,  \ss)
\end{align*}

% Of course, we can write the dual problem using the
% $\min_{\yy}L(\yy, \xx, \ss)$ notation as
% \[
%   \max_{\substack{\xx, \ss \\ \ss \geq 0}}
%   \min_{\yy}L(\yy, \xx, \ss)
%   =
%   \max_{\substack{\xx, \ss \\ \ss \geq 0}} L(\xx,  \ss)
%   \]

\subsection{Weak Duality}
\label{sec:weakdual}
First we see that the primal problem can be written in terms of the Lagrangian as
\begin{equation}
  \label{eq:primalminmax}
  \alpha^* = \min_{\yy}\max_{\xx; \ss\geq\veczero} L(\yy, \xx, \ss)
\end{equation}
This is because for a minimizing $\yy$ all constraints have to be satisfied and the Lagrangian simplifies to $L(\yy,\xx,\ss) = \energy(\yy)$.
If $\AA\xx - \bb = \veczero$ was violated, making $\xx$ large sends $L(\yy,\xx,\ss) \rightarrow \infty$.
And if $\cc(\yy) \leq \veczero$ is violated, we can make $L(\yy,\xx,\ss) \rightarrow \infty$ by choosing large $\ss$.

Note that we require $\ss \geq \veczero$, as we only want to penalize the violation of the inequality constraints in one direction, i.e.\ when $\cc(\yy) > 0$.

%\begin{theorem}[Weak Duality]
For any primal-dual feasible $\yy,\xx,\ss$ we have
$L(\yy, \xx, \ss) \leq \energy(\yy)$ and hence also $L(\xx, \ss) = \min_y L(\yy, \xx, \ss) \leq \energy(\yy)$.

In other words $\max_{\xx; \ss \geq 0} L(\xx, \ss) = \beta^* \leq \alpha^*$.
This is referred to as \emph{weak duality}.

Using the forms in Equations~\eqref{eq:dualprob} and
\eqref{eq:primalminmax}, we can also state this as
\begin{equation*}
  \alpha^* = \min_{\yy}\max_{\xx; \ss\geq\veczero} L(\yy, \xx, \ss)
  \geq
  \max_{\xx; \ss\geq\veczero} \min_{\yy} L(\yy, \xx, \ss)
  =
  \beta^*
  .
\end{equation*}

\subsection{Strong Duality}
So now that we have proved weak duality $\beta^* \leq \alpha^*$,
what is strong duality? $\beta^* = \alpha^*$?
The answer is yes, but strong duality only holds under some conditions.

One sufficient condition we look at today is a variant of \emph{Slater's condition}%

\begin{definition}[Slater's condition with full domain] \label{def:slater}
A (primal) problem as defined in \eqref{eq:primalprob} with $S = \R^n$ fulfills
Slater's condition if there exists a \emph{strictly feasible} point,
i.e.\ there exists $\yytil$ s.t.\ $\AA\yytil = \bb$ and $\cc(\yytil) <
\veczero$.
This means that the strictly feasible point $\yytil$ lies strictly inside the set $\{\yy : \cc(\yy) \leq \veczero\}$ defined by the inequality constraints.
\end{definition}

\begin{theorem}
  \label{thm:slaterstrongduality}
For a problem satisfying Slater's condition, strong duality holds, i.e.\ $\alpha^* = \beta^*$.
In other words, the optimal value of the primal problem $\alpha^*$ is equal to the optimal value of the dual.
\end{theorem}

To extend Slater's condition to the case when the domain $S$ is a
strict subset of $\R^n$, we need the notion of a ``relative interior''.

% \begin{definition}[Affine hull] \label{def:affhull}
% Given a set $S \subset \R^n$, the \emph{affine hull} of $S$, denoted $\operatorname{aff}(S)$, is the
% union of $S$ and all lines through any pair of points in $S$.
% We can also write this as
% \[
%   \operatorname{aff}(S) = \setof{\alpha \xx + (1-\alpha) \yy : \xx,
%     \yy \in S, \alpha \in \R}.
%   \]
% \end{definition}

\begin{definition}[Relative interior] \label{def:relint}
  Given a convex set $S \subset \R^n$, the \emph{relative interior} of $S$ is
  \[
    \operatorname{relint}(S) = \setof{x \in S : \text{ for all } \yy
      \in S \text{ there exists } \epsilon > 0 \text{ such that }
      \xx - \epsilon(\yy - \xx) \in S}
    .
  \]
\end{definition}
In other words, $\xx \in \operatorname{relint}(S)$ if starting at $\xx
\in S$ we can move
``away'' from any $\yy \in S$ by a little and still be in $\SS$.
As an example, suppose $S = \setof{(s,t) \in \R^2 \text{ such that } s
  \geq 0 \text{ and } t = 0}$.
Then $(0,0) \in S$ but $(0,0) \not\in \operatorname{relint}(S)$, while
$(1,0) \in \operatorname{relint}(S)$.

Now, we can state a more general version of Slater's condition.

\begin{definition}[Slater's condition] \label{def:slatergeneral}
A (primal) problem as defined in \eqref{eq:primalprob} fulfills
Slater's condition if there exists a \emph{strictly feasible} point
$\yy \in \operatorname{relint}(S)$.
We require $\AA\yytil = \bb$ and $\cc(\yytil) <
\veczero$.
This means that the strictly feasible point $\yytil$ lies strictly inside the set $\{\yy : \cc(\yy) \leq \veczero\}$ defined by the inequality constraints.
\end{definition}

Theorem~\ref{thm:slaterstrongduality} also holds given the general
Slater's condition of Definition~\ref{def:slatergeneral}.

\paragraph{How are we going to prove this?} Before we prove the theorem, let's make a few observations to get us
warmed up. If you get bored, skip ahead to the proof.

It is sufficient to prove that $\alpha^* \leq \beta^*$, as the statement then follows in conjunction with weak duality.
We define the set
\begin{equation*} G = \{(\energy(\yy), \AA\yy - \bb, \cc(\yy)) : \yy \in S \}, \end{equation*}
where $S \subseteq \R^n$ is the domain of $\energy$.

Immediately, we observe that we can write the optimal primal value as
\begin{equation*} \alpha^* = \min \{t:(t,\vv,\uu) \in G, \vv = \veczero, \uu \leq \veczero\}. \end{equation*}
Similarly, we can write the Lagrangian (after minimizing over $\yy$)
\begin{equation*} L(\xx, \ss) = \min_{(t,\vv,\uu) \in G} (1, \xx, \ss)^{\trp} (t, \vv, \uu). \end{equation*}
This is equivalent to the inequality, for $(t,\vv,\uu) \in G$,
\begin{equation*} (1, \xx, \ss)^{\trp}(t, \vv, \uu) \geq L(\xx, \ss). \end{equation*}
which defines a hyperplane with $\nn = (1,\xx,\ss)$ and $\mu = L(\xx,
\ss)$ such that $G$ is on one side.

To establish strong duality, we would like to show the existence of a
hyperplane such that for $(t,\vv,\uu) \in G$
\[
  \nn^{\trp}(t, \vv, \uu) \geq \alpha^* \text{ and } \nn = (1,\xxhat,\sshat)
  \text{ with } \sshat \geq \veczero.
\]
Then we would immediately get
\begin{equation*}
 \beta^* \geq L(\xxhat,\sshat)= \min_{(t,\vv,\uu) \in G} (1, \xx,
 \ss)^{\trp} (t, \vv, \uu) \geq \alpha^* .
\end{equation*}

Perhaps not surprisingly, we will use the Separating
Hyperplane Theorem.
What are the challenges we need to deal with?
\begin{itemize}
\item We need to replace $G$ with a convex set (which we will call
  $A$) and separate $A$ from some other convex set (which we will call $B$).
\item We need to make sure the hyperplane normal $\nn$ has $1$ in the first
  coordinate and $\ss \geq \veczero$, and the hyperplane threshold is $\alpha^*$.
\end{itemize}

% The proof we are about to see handles these things in one neat
% stroke: Essentially, we replace $G$ with the convex set that comes from replacing
% the convex functions $\energy$ and $\cc$ with their epigraphs.

% If strong duality indeed holds,
% and is obtained at some $L(\yyhat,\xxhat,\sshat)$
% then the reasoning we used to prove weak duality in
% Section~\ref{sec:weakdual} seems to suggest that a dual point
% $(\xx^*,\ss^*)$ maximizing $L(\xx,\ss)$ will occur at a feasible
% $\yy$, since otherwise $L(\yy,\xx,\ss)$ could be reduced further by
% changing $(\xx,\ss)$ to penalize $\yy$.
% Of course, it's not clear whether this could then be counter-acted by
% changing $\yy$.

% \begin{itemize}
% \item and correspond to a feasible $\yy$ in the sense that
%   $L(\xx^*,\ss^*) = (\yy,\xx^*,\ss^*)$
%   \item be dual feasible,
% \end{itemize}
% If the minimizing $\yy$ is not feasible,  but

%  Then by considering a feasible $\yyhat$ for the primal problem, we
%   get
%   \begin{align**}
%     \beta^*
%     & \geq L(\xxhat,\sshat) \\
%     &= (1, \xxhat,\sshat)^{\trp}(\energy(\yyhat), \AA\yyhat - \bb, \cc(\yyhat))
%       \geq \energy(\yyhat) \geq \alpha^*
%   \end{align**}
% This sandwiching also tells us that $\yyhat$ and $(\xxhat,\sshat)$
% must be optimal for the primal and dual problems respectively.

% Clearly, we like to ensure that

% We'd like to show that
% for some dual-feasible $(\xxhat,\sshat)$ maximizing $L(\xx,
% \ss)$, then the hyperplane inequality doesn't have any slack, i.e. it
% holds with equality at some point, and furthermore, equality is
% obtained at some $(t, \vv, \xx) = (\energy(\yyhat), \AA \yyhat - \bb,
% \cc(\yyhat))$ where $\yyhat$ is feasible.

% When we proved weak duality in Section~\ref{sec:weakdual}, our reasoning

\begin{proof}[Proof of Theorem~\ref{thm:slaterstrongduality}.]
  For simplicity, our proof will assume that $S = \R^n$, but only a little
  extra work is required to handle the general case.

  Let's move to on finding two convex disjoints sets $A, B$ to enable the use of the separating hyperplane \autoref{th:sht}.

First set we define $A$, roughly speaking, as a multi-dimensional epigraph of
$G$. More precisely
\begin{equation*} A = \{(t, \vv, \uu) : \exists \yy \in S, t \geq \energy(\yy), \vv = \AA\yy - \bb, \uu \geq \cc(\yy)\}. \end{equation*}
Note that $A$ is a convex set. The proof is similar to the proof that the epigraph of a convex function is a convex set.
The optimal value of the primal program can be now written as
\begin{equation*} \alpha^* = \min_{(t,\veczero,\veczero) \in A} t. \end{equation*}
And we define another set $B$ of the same dimensionality as $A$ by
\begin{equation*} B := \{(r \in \R, \veczero \in \R^m, \veczero \in \R^k) : r < \alpha^* \}. \end{equation*}
This set $B$ is convex, as it is a ray.
An example of two such sets $A,B$ is illustrated in \autoref{fig:ab}.

We show that $A \cap B = \emptyset$ by contradiction.
Suppose $A, B$ are not disjoint; then there exists $\yy$ such that
\begin{equation*} (\energy(\yy), \AA\yy-\bb, \cc(\yy)) = (r, \veczero, \uu) \end{equation*}
with $\uu \leq \veczero$.
But this means that $\yy$ is feasible and  $\energy(\yy) = r < \alpha^*$; contradicting the optimality of $\alpha^*$.

% Now let's move back to proving that strong duality follows from Slater's condition.
%As Slater's condition requires the existence of a $\yy$ such that $\AA\yy = \bb$
To make things simpler,
we assume that our linear constraint matrix $\AA \in \R^{m\times n}$,
has full row rank and $m < n$ (but very little extra work is required to
deal with the remaining cases, which we omit).

As we just proved, $A$ and $B$ are convex and disjoint sets and hence the separating hyperplane theorem (\autoref{th:sht}) we introduced earlier in this chapter implies the existence a separating hyperplane.
This means there exists a normal
$\nn = (\tilde\rho, \xxtil, \sstil)$ and threshold
% \footnote{To prevent confusion, the threshold here is denoted $\mu$ instead of $alpha$} %
$\mu$ and with $A$ on one side, i.e.\
%\begin{equation*} (t, \vv, \uu) \in A \implies \begin{pmatrix}t \vv^{\trp} \uu^{\trp}\end{pmatrix}\begin{pmatrix}\tilde\rho \\ \xxtil \\ \sstil\end{pmatrix} \geq \alpha \end{equation*}
\begin{equation} \label{eq:A}
(t, \vv, \uu) \in A \implies (t, \vv, \uu)^{\trp}(\tilde\rho, \xxtil, \sstil) \geq \mu
\end{equation}
and the set $B$ on the other side:
\begin{equation} \label{eq:B}
(t, \vv, \uu) \in B \implies (t, \vv, \uu)^{\trp}(\tilde\rho, \xxtil, \sstil) \leq \mu.
\end{equation}

Now, we claim that $\sstil \geq 0$. Suppose $\sstil(i) < 0$, then for $\uu(i) \rightarrow \infty$ the threshold would grow unbounded, i.e.\ $\mu \rightarrow -\infty$ contradicting that the threshold $\mu$ is finite by the separating hyperplane theorem.
Similarly we claim $\tilde\rho \geq 0$, as if this were not the case, having $t \rightarrow \infty$ implies that $\mu \rightarrow -\infty$ again contradicting the finiteness of $\mu$.

From Equation~\eqref{eq:B} it follows that $t\tilde\rho \leq \mu$ for all $t < \alpha^*$ which implies that $t\tilde\rho \leq \mu$ for $t = \alpha^*$ by taking the limit. Hence we have $\alpha^*\tilde\rho \leq \mu$.
From $(t, \vv, \uu) \in A$ we get from  Equation~\eqref{eq:A}
\begin{equation*}
(\tilde\rho, \xxtil, \sstil)^{\trp}(t, \vv, \uu) \geq \mu \geq \alpha^*\tilde\rho
\end{equation*}
and thus
\begin{equation}\label{eq:rhotilde}
(\tilde\rho, \xxtil, \sstil)^{\trp}(\energy(\yy), \AA\yy-\bb,\cc(\yy)) \geq \alpha^*\tilde\rho.
\end{equation}

Now we consider two cases; starting with the ``good'' case where $\tilde\rho > 0$.
Dividing  Equation~\eqref{eq:rhotilde} by $\tilde\rho$ gives
\begin{equation*}
  \energy(\yy) + \frac{\xxtil^{\trp}}{\tilde\rho}(\AA\yy - \bb) + \frac{\sstil^{\trp}}{\tilde\rho}\cc(\yy) \geq \alpha^*.
\end{equation*}
Noting that the left hand side above is $L(\yy, \frac{\xxtil}{\tilde\rho}, \frac{\sstil}{\tilde\rho})$
and that the equation holds for arbitrary $\yy$; therefore also for the minimum we get
\begin{equation*} \min_{\yy}{L\left(\yy, \frac{\xxtil}{\tilde\rho}, \frac{\sstil}{\tilde\rho}\right)}\geq \alpha^* \end{equation*}
and hence via definition of $\beta^*$ finally
\begin{equation*} \beta^* \geq L\left(\frac{\xxtil}{\tilde\rho},\frac{\sstil}{\tilde\rho}\right) \geq \alpha^*. \end{equation*}

Next consider the ``bad'' case $\tilde\rho = 0$.
As $\alpha^*\tilde\rho \leq \mu$, we have $0 \leq \mu$.
From  Equation~\eqref{eq:A} we get
\begin{equation*}
  \cc(\yy)^{\trp}\ss + \xx^{\trp}(\bb-\AA\yy) \geq \mu \geq 0.
\end{equation*}
As Slater's condition holds, there is an interior point $\yytil$, i.e.\ it satisfies $\bb - \AA\yytil = \veczero$ and $\cc(\yytil) < \veczero$. Together with the equation above this yields
\begin{equation*} \cc(\yytil)^{\trp}\sstil + \xxtil^{\trp}\veczero \geq 0 \end{equation*}
which implies
$\cc(\yytil)^{\trp}\sstil \geq 0 $
and as $\cc(\yytil) < \veczero$ this means $\sstil = \veczero$.

As the normal $(\tilde\rho,\sstil,\xxtil)$ of the hyperplane can not be all zeroes, this means the last ``component'' $\xxtil$ must contain a non-zero entry, i.e.\ $\xxtil \neq \veczero$.
Furthermore $\xxtil^{\trp}(\bb - \AA\yytil) = \veczero$, $\cc(\yytil) < \veczero$ and $\AA$ has full row rank, hence there exists $\ddelta$ such that
\begin{equation*}
\xxtil^{\trp}(\bb-\AA(\yytil+\ddelta)) < \veczero \text{ and } \cc(\yytil + \ddelta) < 0.
\end{equation*}
This, however, means that there is a point in $A$ on the wrong side of the hyperplane, as
\begin{equation*}
(\tilde\rho, \xxtil, \sstil)^{\trp}(\energy(\yytil+\ddelta), \bb-\AA(\yytil+\ddelta), \cc(\yytil+\ddelta)) < 0
\end{equation*}
but the threshold is $\mu \geq 0$.
\end{proof}

\begin{remark*}
  Note that our reasoning about why $\ss \geq \veczero$ in the proof
  above is very similar to our reasoning for why the primal program
  can be written as Problem~\eqref{eq:primalminmax}.
\end{remark*}

\paragraph{Example.}
As an example of $A$ and $B$ as they appear in the above proof,
consider
\[
\min_{\substack{y \in (0,\infty) \\ 1/y
    -1 \leq 0}}
y^2
\]
 This leads to $\alpha^* = 1,
    y^* =1$, and $A = \setof{ (t,u): y \in (0,\infty) \text{ and } t >y^2
       \text{ and } u \geq 1/y-1}$, and $B = \setof{ (t,0) : t < 1}$ and the separating hyperplane normal is $\nn = (1,2)$.
These two sets $A,B$ are illustrated in \autoref{fig:ab}.
\begin{figure}[H]
  \centering
  \includegraphics[width=.9\textwidth]{fig/lec12-the-sets-a-b.jpg}
  \caption{Example of the convex sets $A$ and $B$ we wish to separate by hyperplane.}
\label{fig:ab}
\end{figure}

\subsection{The Gradient Perspective}
Let's come back to what we said earlier about parallel gradient.
Suppose $\yy^*$ is an optimizer of the primal problem and $\xx^*, \ss^*$ for the dual.
We thus have
\[ L(\yy^*, \xx^*, \ss^*) = \alpha^* = \beta^*. \]
Because $L(\yy, \xx^*, \ss^*)$ is a convex function in $\yy$, it also
follows that if $\energy: S \to \R$ and $\cc$ are differentiable and the minimizer
$\yy^*$ is not on the boundary of $S$, then we must have that the
gradient w.r.t. $\yy$ is zero, i.e.
\begin{equation*} \left.\grad_{\yy}L(\yy, \xx^*, \ss^*)\right|_{\yy=\yy^*} = 0 \end{equation*}
and plugging in
\[ L(\yy, \xx^*, \ss^*) = \energy(\yy) + \xx^{\trp}(\bb - \AA\yy) + \ss^{\trp}\cc(\yy) \]
yields
\begin{equation*} \grad\energy(\yy) + \xx^{\trp}\grad_y(\bb-\AA\yy) + \ss^{\trp}\grad\cc(\yy) = \veczero. \end{equation*}
And this connects to our point of the parallel gradients from the beginning of this section.

\subsection{Complementary Slackness}

We will see more of this next time, but if we look at $\yy^*$ we see that
\begin{equation*}\energy(\yy^*) =  \alpha^* = \energy(\yy^*) + \xx^{\trp}(\bb - \AA\yy^*) + \ss^{\trp}\cc(\yy^*) = \energy(\yy^*) + \ss^{\trp}\cc(\yy^*) \end{equation*}
and hence when the $i$-th convex constraint is not active, i.e.\ $c_i(\yy^*) < 0$ the \emph{slack} must be zero, i.e.\ $\ss(i) = 0$.
Conversely if the slack is non-zero, that is $\ss(i) \neq 0$ implies that the constraint is active, i.e.\ $c_i(\yy^*) = 0$.

A good reference for this is Boyd's free online book ``Convex
optimization'' (linked to on the course website).
It provides a number of different interpretations of duality. One
particularly interesting one comes from economics:
economists see the slack variables $\ss$ as prices for violating the constraints.

%\end{document}
%%% Local Variables:
%%% mode: latex
%%% TeX-master: "agao21_script"
%%% End: