\chapter{Preliminaries}\label{chap:preliminaries}

\section{General Notation}

We'll refer to \(\reals\) for the reals, \(\utri\) represents
the unit triangle (or unit simplex) in \(\reals^2\):
\(\utri = \left\{(s, t) \mid 0 \leq s, t, s + t \leq 1\right\}\).
When dealing with sequences with multiple indices, e.g.
\(s_{m, n} = m + n\), we'll use bold symbols to represent
a multi-index: \(\bm{i} = (m, n)\). We'll use \(\left|\bm{i}\right|\) to
represent the sum of the components in a multi-index.
The binomial coefficient
\(\binom{n}{k}\) is equal to \(\frac{n!}{k! (n - k)!}\) and the trinomial
coefficient \(\binom{n}{i, j, k}\) is equal to \(\frac{n!}{i! j! k!}\)
(where \(i + j + k = n\)). The notation \(\delta_{ij}\) represents the
Kronecker delta, a value which is \(1\) when \(i = j\) and \(0\)
otherwise.

\section{Floating Point and Forward Error Analysis}

We assume all floating point operations obey
\begin{equation}
  a \star b = \fl{a \circ b} = (a \circ b)(1 + \delta_1) =
  (a \circ b) / (1 + \delta_2)
\end{equation}
where \(\star \in \left\{\oplus, \ominus, \otimes, \oslash\right\}\), \(\circ
\in \left\{+, -, \times, \div\right\}\) and \(\left|\delta_1\right|,
\left|\delta_2\right| \leq \mach\). The symbol \(\mach\) is the unit round-off
and \(\star\) is a floating point operation, e.g.
\(a \oplus b = \fl{a + b}\). (For IEEE-754 floating point double precision,
\(\mach = 2^{-53}\).) We denote the computed result of
\(\alpha \in \reals\) in floating point arithmetic by
\(\widehat{\alpha}\) or \(\fl{\alpha}\) and use \(\floats\) as the set of
all floating point numbers (see \cite{Higham2002} for more details).
Following \cite{Higham2002}, we will use the following classic properties in
error analysis.

\begin{enumerate}
  \item If \(\delta_i \leq \mach\), \(\rho_i = \pm 1\), then
      \(\prod_{i = 1}^n (1 + \delta_i)^{\rho_i} = 1 + \theta_n\),
  \item \(\left|\theta_n\right| \leq \gamma_n \coloneqq
      n \mach / (1 - n \mach)\),
  \item \((1 + \theta_k)(1 + \theta_j) = 1 + \theta_{k + j}\),
  \item \(\gamma_k + \gamma_j + \gamma_k \gamma_j \leq \gamma_{k + j}
    \Longleftrightarrow (1 + \gamma_k)(1 + \gamma_j) \leq 1 + \gamma_{k + j}\),
  \item \((1 + \mach)^j \leq 1 / (1 - j \mach) \Longleftrightarrow
  (1 + \mach)^j - 1 \leq \gamma_j\).
\end{enumerate}

\section{B\'{e}zier Curves}

A \emph{B\'{e}zier curve} is a mapping from the unit interval
that is determined by a set of control points
\(\left\{\bm{p}_j\right\}_{j = 0}^n \subset \reals^d\).
For a parameter \(s \in \left[0, 1\right]\), there is a corresponding
point on the curve:
\begin{equation}
b(s) = \sum_{j = 0}^n \binom{n}{j} (1 - s)^{n - j} s^j \bm{p}_j \in
  \reals^d.
\end{equation}
This is a combination of the control points weighted by
each Bernstein basis function
\(B_{j, n}(s) = \binom{n}{j} (1 - s)^{n - j} s^j\).
Due to the binomial expansion
\(1 = (s + (1 - s))^n = \sum_{j = 0}^n B_{j, n}(s)\),
a Bernstein basis function is in
\(\left[0, 1\right]\) when \(s\) is as well. Due to this fact, the
curve must be contained in the convex hull of it's control points.

\subsection{de Casteljau Algorithm}

Next, we recall\footnote{We have used slightly non-standard notation for the
terms produced by the de Casteljau algorithm: we start the superscript at
\(n\) and count down to \(0\) as is typically done when describing Horner's
algorithm. For example, we use \(b_j^{(n - 2)}\) instead of
\(b_j^{(2)}\).} the de Casteljau algorithm:

\begin{breakablealgorithm}
  \caption{\textit{de Casteljau algorithm for polynomial evaluation.}}
  \label{alg:de-casteljau}

  \begin{algorithmic}
    \Function{\(\mathtt{result} = \mathtt{DeCasteljau}\)}{$b, s$}
      \State \(n = \texttt{length}(b) - 1\)
      \State \(\widehat{r} = 1 \ominus s\)
      \\
      \For{\(j = 0, \ldots, n\)}
        \State \(\widehat{b}_j^{(n)} = b_j\)
      \EndFor
      \\
      \For{\(k = n - 1, \ldots, 0\)}
        \For{\(j = 0, \ldots, k\)}
          \State \(\widehat{b}_j^{(k)} = \left(
              \widehat{r} \otimes \widehat{b}_j^{(k + 1)}\right) \oplus
              \left(s \otimes \widehat{b}_{j + 1}^{(k + 1)}\right)\)
        \EndFor
      \EndFor
      \\
      \State \(\mathtt{result} = \widehat{b}_0^{(0)}\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

\begin{theorem}[\cite{Mainar1999}, Corollary 3.2]
If \(p(s) = \sum_{j = 0}^n b_j B_{j, n}(s)\) and \(\mathtt{DeCasteljau}(p, s)\)
is the value computed by the de Casteljau algorithm then\footnote{In the
original paper the factor on \(\widetilde{p}(s)\) is \(\gamma_{2n}\),
but the authors did not consider round-off when computing
\(1 \ominus s\).}
\begin{equation}
\left|p(s) - \mathtt{DeCasteljau}(p, s)\right| \leq \gamma_{3n}
\sum_{j = 0}^n \left|b_j\right| B_{j, n}(s).
\end{equation}
\end{theorem}

The relative condition number of the evaluation of \(p(s) = \sum_{j = 0}^n
b_j B_{j, n}(s)\) in Bernstein form used in this work is (see
\cite{Mainar1999, Farouki1987}):
\begin{equation}
\cond{p, s} = \frac{\widetilde{p}(s)}{\left|p(s)\right|},
\end{equation}
where
\(\widetilde{p}(s) \coloneqq \sum_{j = 0}^n \left|b_j\right| B_{j, n}(s)\).

To be able to express the algorithm in matrix form, we define
the vectors
\begin{equation}
b^{(k)} = \left[\begin{array}{c c c} b_0^{(k)} & \cdots &
b_k^{(k)}\end{array}\right]^T, \quad
\widehat{b}^{(k)} = \left[\begin{array}{c c c} \widehat{b}_0^{(k)} & \cdots &
    \widehat{b}_k^{(k)}\end{array}\right]^T
\end{equation}
and the reduction matrices:
\begin{equation}
U_k = U_k(s) = \left[\begin{array}{c c c c c c}
    1 - s  & s      & 0      & \cdots & \cdots & 0      \\
    0      & 1 - s  & s      & \ddots &        & \vdots \\
    \vdots & \ddots & \ddots & \ddots & \ddots & \vdots \\
    \vdots &        & \ddots & \ddots & \ddots & 0 \\
    0      & \cdots & \cdots & 0      & 1 - s  & s
\end{array}\right] \in \reals^{k \times (k + 1)}.
\end{equation}
With this, we can express (\cite{Mainar1999}) the de Casteljau algorithm as
\begin{equation}\label{eq:matrix-de-casteljau}
b^{(k)} = U_{k + 1} b^{(k + 1)}
\Longrightarrow b^{(0)} = U_1 \cdots U_n b^{(n)}.
\end{equation}

In general, for a sequence \(v_0, \ldots, v_n\) we'll refer to \(v\)
as the vector containing all of the values:
\(v = \left[\begin{array}{c c c} v_0 & \cdots &
    v_n\end{array}\right]^T.\)

\section{B\'{e}zier Triangles}

A \emph{B\'{e}zier triangle} (\cite[Chapter~17]{Farin2001}) is a
mapping from the unit triangle
\(\utri\) and is determined by a control net
\(\left\{\bm{p}_{i, j, k}\right\}_{i + j + k = n} \subset \reals^d\).
A B\'{e}zier triangle is a particular kind of B\'{e}zier surface, i.e. one
in which there are two cartesian or three barycentric input parameters.
Often the term B\'{e}zier surface is used to refer to a tensor product or
rectangular patch.
For \((s, t) \in \utri\) we can define barycentric weights
\(\lambda_1 = 1 - s - t, \lambda_2 = s, \lambda_3 = t\) so that
\begin{equation}
1 = \left(\lambda_1 + \lambda_2 + \lambda_3\right)^n =
  \sum_{\substack{i + j + k = n \\ i, j, k \geq 0}} \binom{n}{i, j, k}
  \lambda_1^i \lambda_2^j \lambda_3^k.
\end{equation}
Using this we can similarly define a (triangular) Bernstein basis
\begin{equation}
B_{i, j, k}(s, t) = \binom{n}{i, j, k} (1 - s - t)^i s^j t^k
  = \binom{n}{i, j, k} \lambda_1^i \lambda_2^j \lambda_3^k
\end{equation}
that is in \(\left[0, 1\right]\) when \((s, t)\) is in \(\utri\).
Using this, we define points on the B\'{e}zier triangle as a
convex combination of the control net:
\begin{equation}
b(s, t) = \sum_{i + j + k = n} \binom{n}{i, j, k}
  \lambda_1^i \lambda_2^j \lambda_3^k
  \bm{p}_{i, j, k} \in \reals^d.
\end{equation}

\begin{figure}
  \includegraphics{../images/preliminaries/main_figure01.pdf}
  \centering
  \captionsetup{width=.75\linewidth}
  \caption{Cubic B\'{e}zier triangle}
  \label{fig:cubic-bezier-example}
\end{figure}

\noindent Rather than defining a B\'{e}zier triangle by the control net, it can
also be uniquely determined by the image of a standard lattice of
points in \(\utri\): \(b\left(j/n, k/n\right) = \bm{n}_{i, j, k}\);
we'll refer to these as \emph{standard nodes}.
Figure~\ref{fig:cubic-bezier-example} shows these standard nodes for
a cubic triangle in \(\reals^2\). To see the correspondence,
when \(p = 1\) the standard nodes \emph{are} the control net
\begin{equation}
b(s, t) = \lambda_1 \bm{n}_{1, 0, 0} +
\lambda_2 \bm{n}_{0, 1, 0} + \lambda_3 \bm{n}_{0, 0, 1}
\end{equation}
and when \(p = 2\)
\begin{multline}
b(s, t) = \lambda_1\left(2 \lambda_1 - 1\right) \bm{n}_{2, 0, 0} +
\lambda_2\left(2 \lambda_2 - 1\right) \bm{n}_{0, 2, 0} +
\lambda_3\left(2 \lambda_3 - 1\right) \bm{n}_{0, 0, 2} + \\
4 \lambda_1 \lambda_2 \bm{n}_{1, 1, 0} +
4 \lambda_2 \lambda_3 \bm{n}_{0, 1, 1} +
4 \lambda_3 \lambda_1 \bm{n}_{1, 0, 1}.
\end{multline}
However, it's worth noting that the transformation between
the control net and the standard nodes has condition
number that grows exponentially with \(n\) (see \cite{Farouki1991}, which
is related but does not directly show this).
This may make working with
higher degree triangles prohibitively unstable.

A \emph{valid} B\'{e}zier triangle is one which is
diffeomorphic to \(\utri\), i.e. \(b(s, t)\) is bijective and has
an everywhere invertible Jacobian. We must also have the orientation
preserved, i.e. the Jacobian must have positive determinant. For example, in
Figure~\ref{fig:inverted-element}, the image of \(\utri\) under
the map \(b(s, t) = \left[\begin{array}{c c} (1 - s - t)^2 + s^2 & s^2 + t^2
\end{array}\right]^T\) is not valid because the Jacobian is zero along
the curve \(s^2 - st - t^2 - s + t = 0\) (the dashed line). Elements that
are not valid are called \emph{inverted} because they have regions with
``negative area''. For the example, the image \(b\left(\utri\right)\)
leaves the boundary determined by the edge curves: \(b(r, 0)\),
\(b(1 - r, r)\) and \(b(0, 1 - r)\) when \(r \in \left[0, 1\right]\).
This region outside the boundary is traced twice, once with
a positive Jacobian and once with a negative Jacobian.
\begin{figure}
  \includegraphics{../images/preliminaries/inverted_element.pdf}
  \centering
  \captionsetup{width=.75\linewidth}
  \caption{The B\'{e}zier triangle given by \(b(s, t) = \left[
    (1 - s - t)^2 + s^2 \; \; s^2 + t^2 \right]^T\) produces an
    inverted element. It traces the same region twice, once with
    a positive Jacobian (the middle column) and once with a negative
    Jacobian (the right column).}
  \label{fig:inverted-element}
\end{figure}

\section{Curved Elements}\label{sec:curved-elements}

We define a curved mesh element \(\mathcal{T}\) of degree \(p\)
to be a B\'{e}zier triangle in \(\reals^2\) of the same degree.
We refer to the component functions of \(b(s, t)\) (the map that
gives \(\mathcal{T} = b\left(\utri\right)\)) as \(x(s, t)\) and \(y(s, t)\).

This fits a typical definition (\cite[Chapter~12]{FEM-ClaesJohnson})
of a curved element, but gives a special meaning to the mapping from
the reference triangle. Interpreting elements as B\'{e}zier triangles
has been used for Lagrangian methods where
mesh adaptivity is needed (e.g. \cite{CardozeMOP04}). Typically curved
elements only have one curved side (\cite{McLeod1972}) since they are used
to resolve geometric features of a boundary. See also
\cite{Zlmal1973, Zlmal1974}.
B\'{e}zier curves and triangles have a number of mathematical properties
(e.g. the convex hull property) that lead to elegant geometric
descriptions and algorithms.

Note that a B\'{e}zier triangle can be
determined from many different sources of data (for example the control net
or the standard nodes). The choice of this data may be changed to suit the
underlying physical problem without changing the actual mapping. Conversely,
the data can be fixed (e.g. as the control net) to avoid costly basis
conversion; once fixed, the equations of motion and other PDE terms can
be recast relative to the new basis (for an example, see \cite{Persson2009},
where the domain varies with time but the problem is reduced to
solving a transformed conservation law in a fixed reference configuration).

\subsection{Shape Functions}\label{subsec:shape-functions}

When defining shape functions (i.e. a basis with geometric meaning) on a
curved element there are (at least) two choices. When the degree of the
shape functions is the same as the degree of the function being
represented on the B\'{e}zier triangle,
we say the element \(\mathcal{T}\) is \emph{isoparametric}.
For the multi-index
\(\bm{i} = (i, j , k)\), we define \(\bm{u}_{\bm{i}} =
\left(j/n, k/n\right)\) and the corresponding standard node
\(\bm{n}_{\bm{i}} = b\left(\bm{u}_{\bm{i}}\right)\).
Given these points, two choices for shape functions present
themselves:
\begin{itemize}
  \itemsep 0em
  \item \emph{Pre-Image Basis}:
    \(\phi_{\bm{j}}\left(\bm{n}_{\bm{i}}\right) =
      \widehat{\phi}_{\bm{j}}\left(\bm{u}_{\bm{i}}\right) =
      \widehat{\phi}_{\bm{j}}\left(b^{-1}\left(
      \bm{n}_{\bm{i}}\right)\right)\)
    where \(\widehat{\phi}_{\bm{j}}\) is a canonical basis function
    on \(\utri\), i.e.
    \(\widehat{\phi}_{\bm{j}}\) a degree \(p\) bivariate polynomial and
    \(\widehat{\phi}_{\bm{j}}\left(\bm{u}_{\bm{i}}\right) =
    \delta_{\bm{i} \bm{j}}\)
  \item \emph{Global Coordinates Basis}:
    \(\phi_{\bm{j}}\left(\bm{n}_{\bm{i}}\right) =
    \delta_{\bm{i} \bm{j}}\), i.e. a canonical basis function
    on the standard nodes \(\left\{\bm{n}_{\bm{i}}\right\}\).
\end{itemize}

\noindent For example, consider a quadratic B\'{e}zier triangle:
\begin{gather}
b(s, t) = \left[ \begin{array}{c c}
    4 (s t + s + t) & 4 (s t + t + 1)
  \end{array}\right]^T \\
\Longrightarrow
\left[ \begin{array}{c c c c c c}
    \bm{n}_{2, 0, 0} &
    \bm{n}_{1, 1, 0} &
    \bm{n}_{0, 2, 0} &
    \bm{n}_{1, 0, 1} &
    \bm{n}_{0, 1, 1} &
    \bm{n}_{0, 0, 2}
  \end{array}\right] = \left[ \begin{array}{c c c c c c}
    0 & 2 & 4 & 2 & 5 & 4 \\
    4 & 4 & 4 & 6 & 7 & 8
  \end{array}\right].
\end{gather}
In the \emph{Global Coordinates Basis}, we have
\begin{equation}
\phi^{G}_{0, 1, 1}(x, y) = \frac{(y - 4) (x - y + 4)}{6}.
\end{equation}
For the \emph{Pre-Image Basis}, we need the inverse
and the canonical basis
\begin{equation}
b^{-1}(x, y) = \left[ \begin{array}{c c}
    \frac{x - y + 4}{4} & \frac{y - 4}{x - y + 8}
  \end{array}\right] \quad \text{and} \quad
\widehat{\phi}_{0, 1, 1}(s, t) = 4 s t
\end{equation}
and together they give
\begin{equation}
\phi^{P}_{0, 1, 1}(x, y) = \frac{(y - 4) (x - y + 4)}{x - y + 8}.
\end{equation}
In general \(\phi_{\bm{j}}^P\) may not even be a rational bivariate
function; due to composition with \(b^{-1}\) we can only guarantee that
it is algebraic (i.e. it can be defined as the zero set of polynomials).

\subsection{Curved Polygons}\label{subsec:curved-polygons}

\begin{figure}
  \includegraphics{../images/preliminaries/main_figure26.pdf}
  \centering
  \captionsetup{width=.75\linewidth}
  \caption{Intersection of B\'{e}zier triangles form a curved polygon.}
  \label{fig:bezier-triangle-intersect}
\end{figure}

When intersecting two curved elements, the resulting surface(s) will
be defined by the boundary, alternating between edges of each
element.
For example, in Figure~\ref{fig:bezier-triangle-intersect}, a
``curved quadrilateral'' is formed when two B\'{e}zier triangles
\(\mathcal{T}_0\) and \(\mathcal{T}_1\) are intersected.

A \emph{curved polygon} is defined by a collection of B\'{e}zier curves
in \(\reals^2\) that determine the boundary. In order to be
a valid polygon, none of the boundary curves may cross, the
ends of consecutive edge curves must meet and the curves must be right-hand
oriented. For our example in
Figure~\ref{fig:bezier-triangle-intersect}, the triangles
have boundaries formed by three B\'{e}zier curves:
\(\partial \mathcal{T}_0 = b_{0, 0} \cup b_{0, 1} \cup b_{0, 2}\) and
\(\partial \mathcal{T}_1 = b_{1, 0} \cup b_{1, 1} \cup b_{1, 2}\).
The intersection \(\mathcal{P}\) is defined by four boundary
curves: \(\partial \mathcal{P} =
C_1 \cup C_2 \cup C_3 \cup C_4\). Each boundary
curve is itself a B\'{e}zier curve\footnote{A specialization of a
B\'{e}zier curve \(b\left(\left[a_1, a_2\right]\right)\)
is also a B\'{e}zier curve.}:
\(C_1 = b_{0, 0}\left(\left[0, 1/8\right]\right)\),
\(C_2 = b_{1, 2}\left(\left[7/8, 1\right]\right)\),
\(C_3 = b_{1, 0}\left(\left[0, 1/7\right]\right)\) and
\(C_4 = b_{0, 2}\left(\left[6/7, 1\right]\right)\).

Though an intersection can be described in terms of the B\'{e}zier triangles,
the structure of the control net will be lost. The region will not in general
be able to be described by a mapping from a simple space like
\(\utri\).

\section{Error-Free Transformation}

An error-free transformation is a computational method where both
the computed result and the round-off error are returned. It
is considered ``free'' of error if the round-off can be represented
exactly as an element or elements of \(\floats\).
The error-free transformations used in this work are
the \texttt{TwoSum} algorithm by Knuth (\cite{Knuth1997}) and
\texttt{TwoProd} algorithm by Dekker (\cite{Dekker1971}, Section 5),
respectively.

\begin{theorem}[\cite{Ogita2005}, Theorem 3.4]\label{thm:eft}
For \(a, b \in \floats\) and \(P, \pi, S, \sigma \in \floats\),
\texttt{TwoSum} and \texttt{TwoProd} satisfy
\begin{alignat}{4}
\left[S, \sigma\right] &= \mathtt{TwoSum}(a, b), & \, S &= \fl{a + b},
  S + \sigma &= a + b, \sigma &\leq \mach \left|S\right|,
  & \, \sigma &\leq \mach \left|a + b\right| \\
\left[P, \pi\right] &= \mathtt{TwoProd}(a, b),
  & \, P &= \fl{a \times b}, P + \pi &= a \times b,
  \pi &\leq \mach \left|P\right|,
  & \, \pi &\leq \mach \left|a \times b\right|.
\end{alignat}
The letters \(\sigma\) and \(\pi\) are used to indicate that the
errors came from sum and product, respectively. See
Appendix~\ref{chap:appendix-algo} for implementation details.
\end{theorem}


% Part: first-order-logic
% Chapter: model-theory
% Section: partial-iso

\documentclass[../../../include/open-logic-section]{subfiles}

\begin{document}

\olfileid{mod}{bas}{dlo}
\section{Dense Linear Orders}

\begin{defn}
  A \emph{dense linear ordering without endpoints} is !!a{structure}
  $\Struct{M}$ for the !!{language} containg a single 2-place
  !!{predicate}~$<$ satisfying the following sentences:
  \begin{enumerate}
  \item $\lforall[x][\lnot x < x]$;
  \item $\lforall[x][\lforall[y][\lforall[z][(x < y \lif (y < z \lif x
    <z ))]]]$;
  \item $\lforall[x][\lforall[y][(x< y \lor \eq[x][y] \lor y < x)]]$;
  \item $\lforall[x][\lexists[y][x < y]]$;
  \item $\lforall[x][\lexists[y][y < x]]$;
  \item $\lforall[x][\lforall[y][(x < y \lif \lexists[z][(x < z \land
        z < y))]]]$.
 \end{enumerate}
\end{defn}

\begin{thm}\ollabel{thm:cantorQ}
  Any two !!{enumerable} dense linear orderings without
  endpoints are isomorphic.
\end{thm}

\begin{proof}
  Let $\Struct{M_1}$ and $\Struct{M_2}$ be !!{enumerable} dense linear
  orderings without endpoints, with ${<_1} = \Assign{<}{M_1}$ and ${<_2} =
  \Assign{<}{M_2}$, and let $\PIso{I}$ be the set of all partial
  isomorphisms between them. $\PIso{I}$ is not empty since at least
  $\emptyset \in \PIso{I}$. We show that $\PIso{I}$ satisfies the
  Back-and-Forth property.  Then $\Struct{M_1} \iso[p] \Struct{M_2}$,
  and the theorem follows by \olref[pis]{thm:p-isom1}.

  To show $\PIso{I}$ satisifes the Forth property, let $p \in
  \PIso{I}$ and let $p(a_i) = b_i$ for $i = 1$, \dots,~$n$, and
  without loss of generality suppose $a_1 <_1 a_2 <_1 \cdots <_1
  a_n$. Given $a \in \Domain{M_1}$, find $b \in \Domain{M_2}$ as
  follows:
  \begin{enumerate}
  \item if $a <_2 a_1$ let $b \in \Domain{M_2}$ be such that $b <_2
    b_1$;
  \item if $a_n <_1 a$ let $b \in \Domain{M_2}$ be such that $b_n <_2 b$;
 \item if $a_i <_1 a <_1 a_{i+1}$ for some $i$, then let $b \in
   \Domain{M_2}$ be such that $b_i <_2 b <_2 b_{i+1}$.
  \end{enumerate}
  It is always possible to find a $b$ with the desired property since
  $\Struct{M_2}$ is a dense linear ordering without endpoints. Define
  $q = p \cup \{ \langle a, b \rangle \}$ so that $q \in \PIso{I}$ is
  the desired extension of $p$. This establishes the Forth
  property. The Back property is similar. So $\Struct{M_1} \iso[p]
  \Struct{M_2}$; by \olref[pis]{thm:p-isom1}, $\Struct{M_1} \iso
  \Struct{M_2}$.
\end{proof}

\begin{prob}
  Complete the proof of \olref[mod][bas][dlo]{thm:cantorQ} by
  verifying that $\PIso{I}$ satisfies the Back property.
\end{prob}

\begin{rem}
  Let $\Struct{S}$ be any !!{enumerable} dense linear ordering without
  endpoints. Then (by \olref{thm:cantorQ}) $\Struct{S} \iso
  \Struct{Q}$, where $\Struct{Q} = (\Rat, <)$ is the !!{enumerable}
  dense linear ordering having the set $\Rat$ of the rational numbers
  as its domain. Now consider again the !!{structure} $\Struct{R} =
  (\Real, <)$ from \olref[thm]{remark:R}. We saw that there is
  !!a{enumerable} !!{structure} $\Struct{S}$ such that $\Struct{R}
  \elemequiv \Struct{S}$. But $\Struct{S}$ is !!a{enumerable} dense
  linear ordering without endpoints, and so it is isomorphic (and
  hence elementarily equivalent) to the !!{structure}~$\Struct{Q}$. By
  transitivity of elementary equivalence, $\Struct{R} \elemequiv
  \Struct{Q}$. (We could have shown this directly by establishing
  $\Struct{R} \iso[p] \Struct{Q}$ by the same back-and-forth
  argument.)
\end{rem}
\end{document}


\problemname{Mixing Drinks}
Pia is a famous bartender at the hip Stockholm night club Supernova.
One of her most impressive feats is the mixing a series of drinks using each of the $N$ distinct drink ingredients in the bar exactly once.
She does this in the following way.

First, Pia chooses a number of drinks to make.
Each of the drink ingredients are then lined up in front of her in order $1, 2, \dots, N$.
For the first drink, she uses some positive number $K$ of ingredients starting from the left, i.e. $1, 2, ..., K$.
For the next drink, she uses some positive number $L$ of ingredients starting from the first unused ingredient, i.e. $K + 1, K + 2, \dots, K + L$.
She continues this process until the final drink, which uses some set of ingredients $N - M, N - M + 1, \dots, N$.

However, not every pair of ingredients work well in a drink.
For example, milk and water would not go very well together.
She may not include a bad pair of ingredients in any drink.

So far, she has managed to make a different set of drinks every night.
For how many nights can she mix a new set of drinks?
We call two sets of drinks different if they do not consist of the exact same drinks (though they are allowed to have drinks in common).

\section*{Input}
The first line of the input contains two integers $1 \le N \le 100\,000$ and $0 \le P \le 100\,000$, the number of ingredients and bad pairs of ingredients.

Each of the next $P$ lines contains two integers $1 \le a \not= b \le N$, two ingredients that do not work well together in a drink.
The same pair of ingredients may appear multiple times in this list.

\section*{Output}
Output a single integer, the number of nights Pia can construct a different set of drinks.
Since this number may be large, output the remainder when divided by $10^9 + 7$.


\chapter{The Cox Proportional Hazards Model \label{chapter:cox}}

We just encountered the Kaplan-Meier estimate of the survival function in Chapter~\ref{chapter:km}. Now we are going to talk about models that essentially treat the survival function as the \emph{outcome} in a supervised learning problem. These models are called \textbf{Cox proportional hazards models}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Survival and Hazard Functions}

Consider a situation where we have some process that generates events, and we're trying to model the time to first event. The probability density for when the event occurs is $f(t)$. The cumulative probability of the event's having occurred by time $t$, also called the \textbf{cumulative incidence}, is
$$ F(t) = \int_0^t f(t) dt.$$
The probability of an individual's \emph{not} having experienced the event by time $t$ is
$$ S(t) = 1 - F(t), $$
which we call the \textbf{survival}. The probability of experiencing the event in an infinitesimally small interval starting at $t$, given that one has not experienced it by time $t$, is:
$$ \lambda(t) = \frac{f(t)}{S(t)} $$
and is called the \textbf{hazard}. The \textbf{cumulative hazard} function is equal to
$$ \Lambda(t) = \int_0^t \lambda(t') dt' = -\log S(t) $$
so $S(t) = \exp(-\Lambda(t))$.

Here is a graphical representation of some of these quantities. Remember that the probability distribution $f(t)$ must integrate to one.

\begin{center}
\includegraphics[width=0.85\textwidth]{img/survival-function-example-fx.png}
\end{center}

\vspace{3mm}

\begin{question}{}
What's the interpretation of the hazard, $\lambda(t)$, on this image? What happens to the hazard if $S(t)$ is low vs. high for the same $f(x)$?
\end{question}

\begin{question}{}
Assume $f(t)$ is exponential: $f(t) = b \cdot \exp(-bt)$, where $b$ is constant. Then $F(t) = 1 - \exp(-bt)$ and $S(t) = \exp(-bt)$. What is the hazard, $\lambda(t)$? What is the cumulative hazard, $\Lambda(t)$?
\end{question}

\vspace{3mm}

\begin{question}{}
The concept of a ``cumulative hazard'' is pretty weird. How should this quantity be interpreted?
% 1. The total amount of risk accumulated up to time t.
% 2. The number of times per subject that we would expect to observe failures over a given period if the failure event were repeatable.
% 3. It is useful in terms of allowing us to test assumptions of Cox models. Sort of like the logit in that way. It pops out of the modeling, so we care about it, but it isn't what we usually think about.
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Estimating Survival and Cumulative Hazard}

The Kaplan-Meier estimate of survival (Chapter~\ref{chapter:km}) is the most common estimate of the survival function. One can estimate the cumulative hazard using a couple of different methods.
\begin{enumerate}
\item Take the negative log of the Kaplan-Meier estimate of survival:
\begin{align*} \hat{\Lambda}_{KM}(t) &= -\log \hat{S}_{\text{KM}}(t) \\
&= -\sum_{i:t_i < t} \log \left( 1 - \frac{d_i}{n_i} \right) \end{align*}
\item Use the \textbf{Nelson-Aalen estimator}
$$ \hat{\Lambda}_{NA}(t) = \sum_{i:t_i < t} \frac{d_i}{n_i} $$
\end{enumerate}

One thing that is confusing about the R \texttt{survival} package's output is that is uses the Kaplan-Meier estimate for survival by default, but it uses the Nelson-Aalen estimator for cumulative hazard by default. So if you exponentiate the negative cumulative hazard that comes out of \texttt{survfit}, it won't match the survival estimate. The two are close in most cases, though. Here are some pictures for the ovarian cancer survival dataset we discussed in Chapter~\ref{chapter:km}:

\begin{center}
\includegraphics[width=0.85\textwidth]{img/ovarian-overall-survival.png}\\[3mm]
\includegraphics[width=0.85\textwidth]{img/ovarian-overall-cumhaz.png}
\end{center}

% From survfit.formula docs: "The routine returns both an estimated probability in state and an estimated cumulative hazard estimate. The cumulative hazard estimate is the Nelson-Aalen (NA) estimate or the Fleming-Harrington (FH) estimate, the latter includes a correction for tied event times. The estimated probability in state can estimated either using the exponential of the cumulative hazard, or as a direct estimate using the Aalen-Johansen approach. For single state data the AJ estimate reduces to the Kaplan-Meier and the probability in state to the survival curve; for competing risks data the AJ reduces to the cumulative incidence (CI) estimator. For backward compatability the type argument can be used instead."

\newpage

\begin{question}{}
Here is a plot showing the function $-\log(1-x)$ vs. $x$. Look at the expressions for the two estimators for the cumulative hazard, above. Under what conditions will they be similar? Under what conditions will they be different?
\begin{center}
\includegraphics[width=0.6\textwidth]{img/survival-na-km-comparison.png}
\end{center}
\end{question}

\vspace{3mm}

\begin{question}{}
The curvature of the Nelson-Aalen estimator gives you an idea of how the hazard varies with time. A concave shape is an indication of \emph{deceleration} of the hazard; for example, if the event in question is death and time is patient age, this would represent higher infant/childhood mortality than adult mortality. A convex shape is an indication of \emph{acceleration} of the hazard; in the death/age example, this would represent a process that accelerates as one ages (so called ``wear-out mortality'').

Looking at the graph of the overall cumulative hazard, what do you notice about how the hazard for ovarian cancer changes since the initiation of treatment?
\end{question}

\newpage

\begin{question}{}
Here are the raw data from treatment group $1$ of the \texttt{ovarian} dataset. These are the same data we looked at when building the Kaplan-Meier curve in Chapter~\ref{chapter:km}, Question~\ref{question:kmo1}. Using these data, fill in the remaining cells of the table below. Here we are using the Nelson-Aalen estimator for the cumulative hazard.
{\footnotesize
\begin{center}
\begin{tabular}{rlrr}
  \toprule
 & rx & futime & fustat \\
  \midrule
  1 & 1 & 59 & 1 \\
  2 & 1 & 115 & 1 \\
  3 & 1 & 156 & 1 \\
  4 & 1 & 268 & 1 \\
  5 & 1 & 329 & 1 \\
  6 & 1 & 431 & 1 \\
  7 & 1 & 448 & 0 \\
  8 & 1 & 477 & 0 \\
  9 & 1 & 638 & 1 \\
  10 & 1 & 803 & 0 \\
  11 & 1 & 855 & 0 \\
  12 & 1 & 1040 & 0 \\
  13 & 1 & 1106 & 0 \\
  \bottomrule
\end{tabular}
\end{center}
}
\vspace{-5mm}
{\footnotesize
\begin{center}
\begin{tabular}{rrrrll}
  \toprule
$j$ & $t_j$ & $n_j$ & $d_j$ & $\hat{\Lambda}(t_j)$ & Calculation \\
  \midrule
  0 & 0 & 13 & 0 & $0.000$ & $\frac{0}{13}$ \\
  1 & 59 & 13 & 1 & $0.077$ & $\hat{\Lambda}(t_0) + \frac{1}{13}$ \\
  2 & 115 & 12 & 1 & $0.160$ & $\hat{\Lambda}(t_1) + \frac{1}{12}$ \\[2mm]
  3 & 156 & \\[2mm] % 11 & 1 & $0.251$ & $\hat{\Lambda}(t_2) + \frac{1}{11}$ \\
  4 & 268 & \\[2mm] % 10 & 1 & $0.351$ & $\hat{\Lambda}(t_3) + \frac{1}{10}$ \\
  5 & 329 & 9 & 1 & $0.462$ & $\hat{\Lambda}(t_4) + \frac{1}{9}$ \\
  6 & 431 & 8 & 1 & $0.587$ & $\hat{\Lambda}(t_5) + \frac{1}{8}$ \\
  7 & 448 & 7 & 0 & $0.587$ & $\hat{\Lambda}(t_6) + \frac{0}{7}$ \\
  8 & 477 & 6 & 0 & $0.587$ & $\hat{\Lambda}(t_7) + \frac{0}{6}$ \\
  9 & 638 & 5 & 1 & $0.787$ & $\hat{\Lambda}(t_8) + \frac{1}{5}$ \\[2mm]
  10 & 803 & 4 & 0 & \\[2mm] % $0.787$ & $\hat{\Lambda}(t_9) + \frac{0}{4}$ \\
  11 & 855 & 3 & 0 & \\[2mm] % $0.787$ & $\hat{\Lambda}(t_{10}) + \frac{0}{3}$ \\
  12 & 1040 & 2 & 0 & \\[2mm] % $0.787$ & $\hat{\Lambda}(t_{11}) + \frac{0}{2}$ \\
  13 & 1106 & 1 & 0 & \\[2mm] % $0.787$ & $\hat{\Lambda}(t_{12}) + \frac{0}{1}$ \\
  \bottomrule
\end{tabular}
\end{center}
}
\end{question}

\newpage

\begin{question}{}
Here are plots of the cumulative hazard (Nelson-Aalen estimator) for patients by sex and by ECOG performance score status:
\begin{center}
\includegraphics[width=0.45\textwidth]{img/ovarian-rx-cumhaz.png}
\hspace{4mm}
\includegraphics[width=0.45\textwidth]{img/ovarian-ecog-cumhaz.png}
\end{center}
How does treatment group appear to impact the cumulative hazard? What about ECOG score? Do the cumulative hazards appear proportional (i.e., related by a common multiplier) in each case?
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Deriving the Cox Model}

We have spent considerable time on the hazard and cumulative hazard because these play important roles in what is perhaps the most famous survival analysis tool: the Cox proportional hazards model (``Cox model'', for short), developed by D.R. Cox in 1972. The model has the following form:
$$ \lambda(t|x) = \lambda_0(t)\exp(\beta_1 x_1 + \beta_2 x_2 + \dots + \beta_p x_p) $$
and another way of writing it is:
$$ \log \left( \frac{\lambda(t|x)}{\lambda_0(t)} \right) = \beta_1 x_1 + \beta_2 x_2 + \dots + \beta_p x_p. $$
Note that for now we are assuming that the covariates do not depend on time. There is a variant of the Cox model called the \textbf{extended Cox model} that allows time-dependent covariates. For now, we will just consider the fixed covariate case.

\vspace{3mm}

\begin{question}{}
Compare the Cox model to a logistic regression model. What is the same? What is different?
\end{question}

Here $\lambda_0(t)$ is called the \textbf{baseline hazard}. As usual, we symbolize the linear sum of the $\beta$s as $\beta^Tx$. Importantly, the baseline hazard can take any shape as a function of time. The $\lambda_0(t)$ part of the equation is, therefore, referred to as the ``nonparametric'' part, while the $\beta^Tx$ part is called the ``parametric'' part. The overall model is referred to as \textbf{semiparametric}.

The ratio of the hazards for two different sets of covariates, $x$ and $z$, is
$$ \frac{\lambda(t|x)}{\lambda(t|z)} = \frac{\lambda_0(t)\exp(\beta^Tx)}{\lambda_0(t)\exp(\beta^Tz)} = \exp(\beta^T (x-z)). $$
Taking the log of this, we arrive at
$$ \log \left( \frac{\lambda(t|x)}{\lambda(t|z)} \right) = \beta^T (x-z). $$
A single coefficient, $\beta_j$, is therefore the \textbf{hazard ratio} when the corresponding predictor, $x_j$, increases by one. This ratio is assumed to be constant over time. The hazard ratio is also called the \textbf{relative risk}.

\vspace{5mm}

\begin{question}{}
Why doesn't the Cox model have an intercept, $\beta_0$?
\end{question}

\vspace{1mm}

\begin{question}{}
Compare the interpretation of the coefficients in a Cox model to their interpretation in a logistic regression model.
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Fitting the Cox Model \label{section:coxfit}}

Fitting a Cox model means taking a sample of possibly right-censored data and deriving estimates for the parameters, $\beta_1, \dots, \beta_p$. Cox models are fit using a variant of maximum likelihood estimation called \textbf{partial likelihood estimation}.

Let's consider all of the unique times, $t_i$, that events are observed. For now, we will assume that exactly one event is observed at each of these times (no ties). We will use $R(t_i)$ to refer to the set of subjects who are ``at risk'' (i.e., not censored) just prior to time $t_i$. At each failure time, $t_i$, the contribution to the partial likelihood is:
\begin{align*} \mathcal{L}_i(\beta) &= \frac{P(\text{person $i$ experiences event|still around at $t_i$)}}{\sum_{l \in R(t_i)} P(\text{person $l$ experiences event|still around at $t_i$)}} \\
&= \frac{\lambda(t_i|x_i)}{\sum_{l \in R(t_i)} \lambda(t_i|x_l)} = \frac{\exp(\beta^Tx^{(i)})}{\sum_{l \in R(t_i)} \exp( \beta^Tx^{(l)})} \end{align*}
The complete partial likelihood over all $K$ observed event times, $t_i = t_1, \dots, t_K$, is:
$$ \mathcal{L}(\beta) = \prod_{i=1}^K \frac{\exp(\beta^Tx^{(i)})}{\sum_{l \in R(t_i)} \exp( \beta^Tx^{(l)})}.  $$
This is the thing that the model fitting process is trying to maximize. It is optimized numerically, using the same types of optimization procedures used by logistic regression and other generalized linear models.

\vspace{3mm}

\begin{question}{}
Why is this quantity called a ``partial likelihood'', instead of just a ``likelihood''?
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Interpreting Cox Models}

Here is a simple Cox model for survival time vs. age, residual disease, ECOG score, and treatment group in the ovarian cancer dataset.

\begin{center}
\includegraphics[width=0.9\textwidth]{img/ovarian-coxph-model.png}
\end{center}

\vspace{3mm}

\begin{question}{}
Interpret the coefficients, exponentiated coefficients, standard errors of the coefficients, Z scores, and $p$-values in this model. Also try your hand at interpreting the second block of model output, which includes the exponentiated coefficients (again), the exponentiated negative coefficients, and the lower and upper bounds of a 95\% confidence interval for the exponentiated coefficients.
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Making Predictions with Cox Models}

The Cox model fitted using the \texttt{coxph} function can be used to make various predictions on the original dataset. This yields a lot of output. All of the available outputs for the \texttt{ovarian} dataset are shown below.

{\scriptsize
\begin{center}
\begin{tabular}{rrrrrrrrrrrrr}
  \toprule
 & age & lp\_age & resid.ds & lp\_resid.ds & ecog.ps & lp\_ecog.ps & rx & lp\_rx & lp & risk & expected & surv \\
  \midrule
  1 & 72.33 & 9.03 & 2 & 0.83 & 1 & 0.00 & 1 & 0.00 & 2.67 & 14.43 & 0.16 & 0.85 \\
  2 & 74.49 & 9.30 & 2 & 0.83 & 1 & 0.00 & 1 & 0.00 & 2.94 & 18.90 & 0.46 & 0.63 \\
  3 & 66.47 & 8.30 & 2 & 0.83 & 2 & 0.34 & 1 & 0.00 & 2.27 & 9.71 & 0.40 & 0.67 \\
  4 & 53.36 & 6.66 & 2 & 0.83 & 1 & 0.00 & 2 & -0.91 & -0.61 & 0.54 & 0.11 & 0.89 \\
  5 & 50.34 & 6.28 & 2 & 0.83 & 1 & 0.00 & 1 & 0.00 & -0.08 & 0.93 & 0.25 & 0.78 \\
  6 & 56.43 & 7.04 & 1 & 0.00 & 2 & 0.34 & 1 & 0.00 & 0.19 & 1.21 & 0.33 & 0.72 \\
  7 & 56.94 & 7.11 & 2 & 0.83 & 2 & 0.34 & 2 & -0.91 & 0.17 & 1.18 & 0.40 & 0.67 \\
  8 & 59.85 & 7.47 & 2 & 0.83 & 2 & 0.34 & 2 & -0.91 & 0.53 & 1.71 & 0.70 & 0.49 \\
  9 & 64.18 & 8.01 & 2 & 0.83 & 1 & 0.00 & 1 & 0.00 & 1.65 & 5.21 & 2.15 & 0.12 \\
  10 & 55.18 & 6.89 & 1 & 0.00 & 2 & 0.34 & 2 & -0.91 & -0.88 & 0.42 & 0.24 & 0.79 \\
  11 & 56.76 & 7.08 & 1 & 0.00 & 2 & 0.34 & 1 & 0.00 & 0.24 & 1.27 & 0.94 & 0.39 \\
  12 & 50.11 & 6.25 & 1 & 0.00 & 1 & 0.00 & 2 & -0.91 & -1.84 & 0.16 & 0.12 & 0.89 \\
  13 & 59.63 & 7.44 & 2 & 0.83 & 2 & 0.34 & 2 & -0.91 & 0.51 & 1.66 & 1.23 & 0.29 \\
  14 & 57.05 & 7.12 & 2 & 0.83 & 1 & 0.00 & 2 & -0.91 & -0.15 & 0.86 & 0.63 & 0.53 \\
  15 & 39.27 & 4.90 & 1 & 0.00 & 1 & 0.00 & 1 & 0.00 & -2.28 & 0.10 & 0.08 & 0.93 \\
  16 & 43.12 & 5.38 & 1 & 0.00 & 2 & 0.34 & 1 & 0.00 & -1.47 & 0.23 & 0.17 & 0.84 \\
  17 & 38.89 & 4.85 & 2 & 0.83 & 2 & 0.34 & 1 & 0.00 & -1.17 & 0.31 & 0.23 & 0.79 \\
  18 & 44.60 & 5.57 & 1 & 0.00 & 1 & 0.00 & 1 & 0.00 & -1.62 & 0.20 & 0.15 & 0.86 \\
  19 & 53.91 & 6.73 & 1 & 0.00 & 1 & 0.00 & 2 & -0.91 & -1.37 & 0.25 & 0.19 & 0.83 \\
  20 & 44.21 & 5.52 & 2 & 0.83 & 1 & 0.00 & 2 & -0.91 & -1.76 & 0.17 & 0.13 & 0.88 \\
  21 & 59.59 & 7.44 & 1 & 0.00 & 2 & 0.34 & 2 & -0.91 & -0.33 & 0.72 & 0.53 & 0.59 \\
  22 & 74.50 & 9.30 & 2 & 0.83 & 2 & 0.34 & 1 & 0.00 & 3.28 & 26.49 & 1.65 & 0.19 \\
  23 & 43.14 & 5.38 & 2 & 0.83 & 1 & 0.00 & 1 & 0.00 & -0.97 & 0.38 & 0.04 & 0.96 \\
  24 & 63.22 & 7.89 & 1 & 0.00 & 2 & 0.34 & 2 & -0.91 & 0.13 & 1.14 & 0.18 & 0.84 \\
  25 & 64.42 & 8.04 & 2 & 0.83 & 1 & 0.00 & 2 & -0.91 & 0.77 & 2.16 & 0.45 & 0.64 \\
  26 & 58.31 & 7.28 & 1 & 0.00 & 1 & 0.00 & 2 & -0.91 & -0.82 & 0.44 & 0.09 & 0.91 \\
   \bottomrule
\end{tabular}
\end{center}
}

The term \texttt{age} is the raw value for age for each patient, and the term \texttt{lp\_age} is the linear predictor, $\beta_{\text{age}} x_{\text{age}}^{(i)}$, for patient $i$. The same is true for the other predictors. The term \texttt{lp} is the entire linear predictor, $\beta^Tx$, for each $x^{(i)}$. Confusingly, it has been centered, so its value is shifted from the sum of columns 2, 4, 6, and 8 by a fixed amount (Exercise: What is this amount?).  The \texttt{risk} term is just the overall risk score, $\exp(\texttt{lp})$. The \texttt{expected} term is the expected number of events given the covariates and follow-up time. The survival probability, \texttt{surv}, for each subject is $\exp(-\texttt{expected})$. % answer: 7.184752

Cox models make no assumptions about the shape of the baseline hazard, so to make predictions, they simply use the empirical survival (or cumulative hazard) curve for the entire dataset and then adjust it up or down depending on the values of the covariates. The way R does this is super confusing - it estimates the baseline hazard at the means of the covariates after centering, so the baseline hazard is not very interpretable. You can get it out of the model using the \texttt{basehaz} function. In any case, here's what you get when you ask the model to predict the survival and cumulative hazard curves for all of the patients in the training set:

\begin{center}
\includegraphics[width=0.9\textwidth]{img/ovarian-survplot-bypatient.png}\\
\includegraphics[width=0.9\textwidth]{img/ovarian-cumhaz-bypatient.png}
\end{center}

\newpage

\begin{question}{}
Which patients have the lowest and highest risk scores? Where do they appear on the patient-level survival and cumulative hazard graphs?
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Testing the Proportional Hazards Assumption}

The Cox model makes three important assumptions:
\begin{enumerate}
\item \emph{Common baseline hazard.} At any time, $t$, all individuals experience the same baseline hazard, $\lambda_0(t)$.
\item \emph{Proportional hazards.} The hazard for one individual is proportional to the hazard of any other individual.
\item \emph{Time-invariance.} The constant of proportionality between the hazards of any two individuals does not depend on time.
\end{enumerate}
All of them are potentially problematic. In particular, it's hard to come up with situations in which the hazards for \emph{any} two individuals can reasonably be assumed to be proportional. Thus, it's important to check this assumption.

There are whole book chapters and papers devoted to model diagnostics for the Cox model. I will present a couple of common methods here and leave the others to the course website.

\subsection{Schoenfeld Residuals}

In Section~\ref{section:coxfit}, we saw how the Cox model was fit using maximization of the partial likelihood. The quantity
$$ \frac{\exp(\beta^Tx^{(i)})}{\sum_{l \in R(t_i)} \exp( \beta^Tx^{(l)})} $$
was important because it gave us the probability, according to the model, that the person observed to experience the event at time $t_i$ would experience it, given all the people in the risk set just prior to time $t_i$. The \textbf{Schoenfeld residual} capitalizes on this idea.

\begin{quote}
\textbf{Schoenfeld residual:} The covariate value $x_j^{(i)}$ for the person ($i$) who actually experienced the event at time $t_i$, minus the expected value of the covariate for the risk set at $t_i$. Or:
$$ \text{residual} = x_j^{(i)} - \sum_{l \in R(t_i)} x_j^{(l)} \frac{\exp(\beta^Tx^{(l)})}{\sum_{m \in R(t_i)} \exp( \beta^Tx^{(m)})} $$
\end{quote}

There is one Schoenfeld residual for each combination of observed event and covariate. Typically, they will be plotted against time to assess if there is a trend. The test for trend comes from a simple linear regression model of the residuals against time, conducted separately for each covariate.

\begin{center}
\includegraphics[width=0.9\textwidth]{img/ovarian-cox-diagnostics-schoenfeld.png}
\end{center}

\begin{question}{}
How would you conduct the test for trend for each covariate using a simple linear regression model?
\end{question}

\begin{question}{}
Try calculating the Schoenfeld residual for a single covariate and failure time (your choice). All of the information you need is in the table below. The \texttt{risk} column is the same as in the previous table. It is $\exp{(\beta^Tx)}$.
{\small
\begin{center}
\begin{tabular}{rrrrrrrr}
  \hline
 & futime & fustat & age & resid.ds & ecog.ps & rx & risk \\
  \hline
1 & 59 & 1 & 72.33 & 2 & 1 & 1 & 14.43 \\
  2 & 115 & 1 & 74.49 & 2 & 1 & 1 & 18.90 \\
  3 & 156 & 1 & 66.47 & 2 & 2 & 1 & 9.71 \\
  22 & 268 & 1 & 74.50 & 2 & 2 & 1 & 26.49 \\
  23 & 329 & 1 & 43.14 & 2 & 1 & 1 & 0.38 \\
  24 & 353 & 1 & 63.22 & 1 & 2 & 2 & 1.14 \\
  25 & 365 & 1 & 64.42 & 2 & 1 & 2 & 2.16 \\
  26 & 377 & 0 & 58.31 & 1 & 1 & 2 & 0.44 \\
  4 & 421 & 0 & 53.36 & 2 & 1 & 2 & 0.54 \\
  5 & 431 & 1 & 50.34 & 2 & 1 & 1 & 0.93 \\
  6 & 448 & 0 & 56.43 & 1 & 2 & 1 & 1.21 \\
  7 & 464 & 1 & 56.94 & 2 & 2 & 2 & 1.18 \\
  8 & 475 & 1 & 59.85 & 2 & 2 & 2 & 1.71 \\
  9 & 477 & 0 & 64.18 & 2 & 1 & 1 & 5.21 \\
  10 & 563 & 1 & 55.18 & 1 & 2 & 2 & 0.42 \\
  11 & 638 & 1 & 56.76 & 1 & 2 & 1 & 1.27 \\
  12 & 744 & 0 & 50.11 & 1 & 1 & 2 & 0.16 \\
  13 & 769 & 0 & 59.63 & 2 & 2 & 2 & 1.66 \\
  14 & 770 & 0 & 57.05 & 2 & 1 & 2 & 0.86 \\
  15 & 803 & 0 & 39.27 & 1 & 1 & 1 & 0.10 \\
  16 & 855 & 0 & 43.12 & 1 & 2 & 1 & 0.23 \\
  17 & 1040 & 0 & 38.89 & 2 & 2 & 1 & 0.31 \\
  18 & 1106 & 0 & 44.60 & 1 & 1 & 1 & 0.20 \\
  19 & 1129 & 0 & 53.91 & 1 & 1 & 2 & 0.25 \\
  20 & 1206 & 0 & 44.21 & 2 & 1 & 2 & 0.17 \\
  21 & 1227 & 0 & 59.59 & 1 & 2 & 2 & 0.72 \\
   \hline
\end{tabular}
\end{center}
}
\end{question}

\begin{question}{}
The R function \texttt{cox.zph} performs the test of trend for all predictors as well as a global test of trend using ANOVA (don't worry, we'll get to this later). Here is the output for this model:
\begin{center}
\includegraphics[width=0.4\textwidth]{img/ovarian-coxzph.png}
\end{center}
For which predictors is there a potentially worrying association between the Schoenfeld residuals and time?
\end{question}

\subsection{What to do if Violations are Found}

Interpretation of the Cox model is relatively insensitive to deviations from proportionality, especially for large sample sizes. However, if nonproportionality is a huge issue for one or more predictors, there are a few strategies to deal with it.

\begin{enumerate}
\item \emph{Stratify.} One can stratify the model by different levels of the problematic predictor(s), essentially building separate models for the other covariates at each different level of the problematic predictor(s). This only works for predictors that have discrete levels, however; otherwise, one would need to discretize. A potential downside is that stratification eliminates the model's ability to quantify the effect of the stratification variable(s).
\item \emph{Partition the time axis.} Sometimes proportionality holds for the first part of the time axis but falls apart at the end. In that case, one can analyze the data from the first part of the study separately. The disadvantage, of course, is that one must throw out information from later parts of the study.
\item \emph{Add a nonlinear effect term.} Continuous covariates with nonlinear effects on the outcome may lead to nonproportional hazards. Including transformations of these covariates may help to alleviate the nonproportional hazards.
\end{enumerate}


%Background (a brief technical discussion of the D* Lite search method you implemented); figures may be helpful here;

\section{Background}
	This section will contain a brief technical discussion and intuition for each of the three approaches.

	\subsection{Naive Re-planning A*}
	    The NRA* algorithm is an adaption of the classic A* Search to an environment that is partially observable. In an A* Search, a rational agent prioritizes nodes $s$ in the fringe based on a value $f(s) = g(s)+h(s)$, which corresponds to the cost to reach the current node, plus the heuristic cost to reach the goal. In NRA*, the only information that the agent can perceive are the four adjacent cells in the assumed grid-world environment. Therefore, the NRA* agent generates an optimal path, and follows it until one of two things occur: (a) it has reached the goal or (b) it encounters a previously-unobserved obstacle. In this second case, it updates the edge weights into the obstacle square, re-runs A* search from its current location, and repeats the process as before. N.B. that when this re-planning takes place, any previous route knowledge that might have been computed previously does not carry over to the next search, resulting in significant recomputation.

	\subsection{Lifelong Planning A*}
	    The LPA* search algorithm is based on an incremental heuristic search \cite{koenig2002d} \cite{koenig2002improved}. Unlike NRA*, LPA* is able to use information from previous routefinding computations to inform new searches. To do so, LPA* keeps track of $g(s), rhs(s)$ 2-tuples for each node $s$ in the graph. Intuitively, $g(s)$ is an estimate of the cost so far, and corresponds to the same value from A* search. $rhs(s)$ is a one-step look-ahead estimate that takes into consideration the predecessors $s'$ of $s$, potentially providing a better estimate than $g(s)$. More formally, the $rhs$ is defined as 0 if $s=s_{start}$ and $min_{s'\in pred(s)} g(s')+c(s',s)$ otherwise. The queuing strategy for fringe nodes is more complex than in (NR)A*, as it uses a tuple of keys that establish a dual-priority system. Since the $rhs(s)$ values are a ``potentially better informed'' version of the $g(s)$ values, a node is called locally inconsistent if $g(s) \neq rhs(s)$. When a node becomes locally inconsistent, either through initialization or replanning, it is added to this queue.

	    Initially, only the start state is locally inconsistent. By making the nodes locally consistent, and therefore making sure that the $rhs$ values are precisely the same as the $g$ values, an optimal path can be found based on the currently-known edge weights. Upon discovering environmental changes which affect the edge weights between nodes, it updates those affected nodes, and makes them locally inconsistent (or in other words, adds them to the priority queue) if necessary. It then repeats the pathfinding process until the graph is once again consistent. Note that edge weights can increase (locally overconsistent) or decrease (locally underconsistent) in general, but in our example scenario, edges can only increase. This corresponds to the agent encountering an impassable barrier. Once the graph is locally consistent, the shortest path to the goal can be found by starting at the goal state, transitioning back to the predecessor through the nodes which minimize the cost, continuing until the start state is reached.

		A key drawback for LPA* is that it does not allow for changing start positions. For our motivational scenario, where the agent must directly observe the updated edge weights, this can cause significant backtracking.

	\subsection{D* Lite}
		 D* Lite \cite{koenig2004lifelong} \cite{simmons1995probabilistic} aims to improve on this by only updating nodes on the path from the in-progress current position to the goal state. To do so, modest changes to the LPA* algorithm are required; however, the core replanning approach of LPA* is maintained.

		LPA* iteratively attempts to find shortest paths from the start to the goal, incorporating new edge costs as it observes them through the local consistency approach described above. In contrast, D* Lite iteratively attempts to find the shortest path from the $current$ node to the goal, also incorporating new edge costs as it observes them, but only towards the goal. There are some other differences as well. In particular, the search direction in now switched, and the $g(s)$ values now represent the goal distance, not the backwards cost. Similarly, the $rhs$ values are now forward looking, i.e. one step look-ahead with respect to successors and their respective goal distances. Similar to LPA*, D* Lite also prioritizes nodes based on a tuple that is defined in terms of  $g(s), rhs(s)$. However, the definition of this tuple differs from that of LPA* in that its priorities are effectively lower bounds on those priorities from LPA*. It also maintains a priority offset $k_m$, which allows it to bound the replanning propagation. This re-formulation, along with a less strict terminating condition allow D* Lite to have better performance than LPA*.

		The framework for D* Lite is as follows: it first attempts to compute the shortest path from the start to the goal node. It then takes steps along this path, and makes any changes to the edge costs and associated priority queue values when necessary. It then computes the shortest path using this new information, and repeats. The currently-optimal shortest path to the goal can be found by starting at the current start state, transitioning to the successor which minimizes the $rhs$ condition, i.e. one step look-ahead, and continuing until the goal state is reached. However, to extract the actual path taken by the agent including replanning, this information must be extracted retrospectively, by recording the steps taken.


\section{Logistic Regression II}

\subsection{Parameterization}

\begin{frame}
  \frametitle{Parameterization}

  \begin{itemize}
    \item Until now, $F(\vec x)$ was some arbitrary function in $\vec x$ \\[.1cm]
      \structure{Example:} $ F(\vec x) = \vec x^T \mat A \vec x + \vec \alpha^T\vec x + \alpha_0 $ with components defined by Gaussian distributions \\[.25cm] \pause
    \item We can express a nonlinear $F(\vec x)$ as a scalar product by lifting $\vec x$ into a higher dimensional space: \\
      Given \\
      \hspace{0.5cm} $\vec x = \left(x_1, x_2\right)^T \in \real^2,~ \mat A = \left( \begin{array}{cc} a_{11} & a_{12} \\ a_{21} & a_{22} \end{array} \right), ~ \vec \alpha = \left(\alpha_1, \alpha_2 \right)^T, ~\alpha_0 $\,, \\
      then \\
      \hspace{0.5cm} $F(\vec x) = a_{11} x_1^2 + (a_{12} + a_{21}) x_1 x_2 + a_{22} x_2^2 + \alpha_1 x_1 + \alpha_2 x_2 + \alpha_0$\,.\\[.25cm] \pause
    \item Rewrite $F(\vec x) = \vec \theta^T \vec x'$ with $\vec \theta, \vec x' \in \real^6$: \\ \pause
      \hspace{0.5cm} $\vec \theta = (a_{11}, a_{12} + a_{21}, a_{22}, \alpha_1, \alpha_2, \alpha_0)^T$ \\
      \hspace{0.5cm} $\vec x' = ( x_1^2, x_1 x_2, x_2^2, x_1, x_2, 1)^T$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Parameterization \cont}

  \begin{citeblock}{Definition}

    We write the parameterized logistic function in the following:
    \begin{eqnarray*}
      g(\vec \theta^T \vec x) &=& \frac{1}{1+e^{-\vec \theta^T \vec x}}
    \end{eqnarray*}
    where $\vec \theta, \vec x$ are the lifted parameters of the original decision function $F$ \\
    (if it was not already linear).
  \end{citeblock}
\end{frame}

\subsection{Learning in Logistic Regression}

\subsubsection{Log-Likelihood Function}

\begin{frame}
  \frametitle{Log-Likelihood Function}

  \begin{itemize}
    \item Let us assume the posteriors are given by
      \begin{eqnarray*}
        p(y=0|\vec x) &=&  1-g(\vec \theta^T\vec x) \\
        p(y=1|\vec x) &=&  g(\vec \theta^T\vec x)
      \end{eqnarray*}
      where $g(\vec \theta^T\vec x)$ is the sigmoid function parameterized in $\vec \theta$. \\[.3cm]
    \item The parameter vector $\vec{\theta}$ has to be estimated from a set $S$ of $m$ training samples:
      \begin{eqnarray*}
        S &=& \{ (\vec x_1, y_1), (\vec x_2, y_2), (\vec x_3, y_3), \dots, (\vec x_m, y_m) \}\quad .
      \end{eqnarray*}
      \pause
    \item Method of choice: Maximum Likelihood Estimation
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Log-Likelihood Function \cont}

  Before we work on the formulas of the ML-estimator, we rewrite the posteriors
  using Bernoulli probability:

  \begin{eqnarray*}
    p(y|\vec x) &=& \pause g(\vec \theta^T\vec x)^{y}(1-g(\vec \theta^T\vec x))^{1-y}\\[.3cm]
  \end{eqnarray*}

  which shows the great benefit of the chosen notation for class numbers.
\end{frame}

\begin{frame}
  \frametitle{Log-Likelihood Function \cont}

  Now we can compute the log-likelihood function \\
  (assuming that the training samples are mutually independent):

  \begin{eqnarray*}
    \mathcal{L} (\vec \theta) &=& \log \left( \prod_{i=1}^m p(y_i|\vec x_i) \right) \\ \pause
                              &=& \sum_{i=1}^m \log \left( g(\vec \theta^T\vec x_i)^{y_i}\,\big(1-g(\vec \theta^T\vec x_i)\big)^{1-y_i} \right) \\ \pause
                              &=& \sum_{i=1}^m \left(y_i \log g(\vec \theta^T\vec x_i) + (1-y_i)\log\big(1-g(\vec \theta^T\vec x_i)\big) \right)
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{Log-Likelihood Function \cont}

  Simplification of the log-likelihood function:

  \begin{eqnarray*}
    \mathcal{L} (\vec \theta)
      &=& \sum_{i=1}^m \left( y_i \log g(\vec \theta^T\vec x_i) + (1-y_i)\log\big(1-g(\vec \theta^T\vec x_i)\big) \right) \\ \pause
      &=& \sum_{i=1}^m \left( y_i \log \frac{e^{\vec \theta^T \vec x_i}}{1 + e^{\vec \theta^T \vec x_i}} + (1 - y_i) \log \frac{1}{1 + e^{\vec \theta^T \vec x_i}} \right) \\ \pause
      &=& \sum_{i=1}^m \left( y_i \vec \theta^T \vec x_i + \log \frac{1}{1 + e^{\vec \theta^T \vec x_i}} \right) \\ \pause
      &=& \sum_{i=1}^m \left( y_i \vec \theta^T \vec x_i + \log \big( 1 - g(\vec \theta^T \vec x_i) \big) \right)
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{Log-Likelihood Function \cont}

  \structure{Notes for the expert:}

  \begin{itemize}
    \item The negative of the log-likelihood function is the cross entropy of\\
      $y$ and $g(\vec \theta^T\vec x)$. \\[.5cm]
    \item The negative of the log-likelihood function is a convex function.
  \end{itemize}
\end{frame}

\input{nextTime.tex}

\subsubsection{Newton-Raphson Iteration}

\begin{frame}
  \frametitle{Maximization of the log-likelihood function}

  \begin{itemize}
    \item The log-likelihood function is concave.
    \item We use the \point\href{http://www.stat.washington.edu/quinn/classes/560/Newton.pdf}{\structure{Newton-Raphson}} algorithm to solve the unconstrained optimization problem:
      \spread

      For the $(k+1)$-st iteration step, we get:
%
      \begin{eqnarray*}
        \vec \theta^{(k+1)} &=& \vec \theta^{(k)} - \left(  \frac{\partial^2}{\partial\vec \theta \partial \vec \theta^T} \mathcal{L} \left(\vec \theta^{(k)} \right)  \right)^{-1}
       \frac{\partial}{\partial\vec \theta} \mathcal{L}\left(\vec \theta^{(k)}\right)
      \end{eqnarray*}
  \end{itemize}
  \spread

  \structure{Note:} If you write the Newton-Raphson iteration in matrix form, you will end up with a weighted least squares iteration scheme.
\end{frame}

\begin{frame}
  \frametitle{Newton-Raphson Iteration}

  \structure{Taylor's Theorem:}\\[.3cm]

  Approximation of a $k$-times differentiable function $f(x)$ \\
  around a given point $x_0$:

  {\small
  \begin{displaymath}
    f(x_0+h) = f(x_0) + f'(x_0) h + \frac{f''(x_0)}{2!} h^2 + \ldots +
           \frac{f^{(k)}(x_0)}{k!} h^k + r_k(x_0 + h) h^k,
           \quad \lim_{h \rightarrow 0} r_k(x_0 + h) = 0
  \end{displaymath}
  }
  \pspread

  \structure{Second order Taylor polynomial:}
  \begin{displaymath}
    f(x_0 + h) \approx f(x_0) + f'(x_0) h + \frac{1}{2} f''(x_0) h^2
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Newton-Raphson Iteration \cont}

  \structure{Extremum:}

  \begin{eqnarray*}
    f'(x_0 + h)         & = & f'(x_0) + f''(x_0) h ~\stackrel{!}{=}~ 0 \\[.3cm]
                \hat{h} & = & - \frac{f'(x_0)}{f''(x_0)} \\[.3cm]
    x_1 = x_0 + \hat{h} & = & x_0 - \frac{f'(x_0)}{f''(x_0)}
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{Newton-Raphson Iteration \cont}

  \begin{center}
    \resizebox{.7\linewidth}{!}{
      \alt<8->{
        \input{\texfigdir/newton-raphson8.pstex_t}
      }{\alt<7>{
        \input{\texfigdir/newton-raphson7.pstex_t}
      }{\alt<6>{
        \input{\texfigdir/newton-raphson6.pstex_t}
      }{\alt<5>{
        \input{\texfigdir/newton-raphson5.pstex_t}
      }{\alt<4>{
        \input{\texfigdir/newton-raphson4.pstex_t}
      }{\alt<3>{
        \input{\texfigdir/newton-raphson3.pstex_t}
      }{\alt<2>{
        \input{\texfigdir/newton-raphson2.pstex_t}
      }{
        \input{\texfigdir/newton-raphson1.pstex_t}
      }}}}}}}
    }
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Gradient of the Log-Likelihood Function}

  \structure{The gradient:}
 %
  {\small
  \begin{eqnarray*}
    \frac{\partial}{\partial \theta_j} \mathcal{L}(\vec \theta)
      &=& \frac{\partial}{\partial \theta_j} \left( \sum_{i=1}^m \left( y_i \vec \theta^T \vec x_i + \log \big( 1 - g(\vec \theta^T \vec x_i) \big) \right) \right)\\ \pause
      &=& \sum_{i=1}^m \left( y_i x_{i,j} - \frac{1}{1-g(\vec \theta^T\vec x_i)} \frac{\partial}{\partial \theta_j}g(\vec \theta^T \vec x_i) \right)
  \end{eqnarray*}
  }
  \pause
%
  Now we use the derivative of the sigmoid function and get
%
  {\small
  \begin{eqnarray*}
     \frac{\partial}{\partial \theta_j} \mathcal{L}(\vec \theta)
       &=& \sum_{i=1}^m \left( y_i x_{i,j} - \frac{1}{1-g(\vec \theta^T\vec x_i)} g(\vec \theta^T \vec x_i) \big(1-g(\vec \theta^T \vec x_i)\big) x_{i,j} \right) \\ \pause
       &=& \sum_{i=1}^m \left( y_i - g(\vec \theta^T\vec x_i) \right) x_{i,j}
  \end{eqnarray*}
  }
%
   where $x_{i,j}$ is the $j$-th component of the $i$-th training feature vector.
\end{frame}

\begin{frame}
  \frametitle{Gradient of the Log-Likelihood Function \cont}

  Finally, we have a quite simple gradient:

  {\small
  \begin{eqnarray*}
    \frac{\partial}{\partial \theta_j} \mathcal{L}(\vec \theta)
    &=& \sum _{i=1}^m  \left( y_i - g(\vec \theta^T\vec x_i) \right) x_{i,j}
  \end{eqnarray*}
  }

  where $x_{i,j}$ is the $j$--th component of the $i$--th training feature vector. \\[.3cm]

  Or in vector notation:
  {\small
  \begin{eqnarray*}
    \frac{\partial}{\partial\vec \theta} \mathcal{L}(\vec \theta)
    &=& \sum _{i=1}^m  \left( y_i-g(\vec \theta^T\vec x_i) \right)\vec x_{i}
  \end{eqnarray*}
  }
\end{frame}

\begin{frame}
  \frametitle{Hessian of the Log-Likelihood Function}

  \begin{itemize}
    \item The Newton-Raphson algorithm requires the Hessian matrix.
    \item Remember the derivative of the sigmoid function!
  \end{itemize}

  \begin{eqnarray*}
    \frac{\partial^2}{\partial\vec \theta \partial \vec \theta^T} \mathcal{L}(\vec \theta) &=& -
    \sum _{i=1}^m  g(\vec \theta^T\vec x_i) \left(1-g(\vec \theta^T\vec x_i)\right)\vec x_i \vec x_i^T
  \end{eqnarray*}
\end{frame}

\subsection{Perceptron and Logistic Regression}

\begin{frame}
   \frametitle{Perceptron and Logistic Regression}

   \begin{center}
     \resizebox{.7\linewidth}{!}{
       \input{\texfigdir/perceptron.pstex_t}
     }
   \end{center}
\end{frame}

\subsection{Lessons Learned}

\begin{frame}
  \frametitle{Lessons Learned}

  \begin{itemize}
    \item Posteriors can be rewritten in terms of a logistic function.\\[.5cm]
    \item Given the decision boundary $F(\vec x)=0$, we can write down the posterior $p(y|\vec x)$ right away.\\[.5cm]
    \item Decision boundary for normally distributed feature vectors for each class is a quadratic function.\\[.5cm]
    \item If Gaussians share the same covariances, the decision boundary is a linear function.
  \end{itemize}
\end{frame}

\input{nextTime.tex}

\subsection{Further Readings}

\begin{frame}
  \frametitle{Further Readings}

  \begin{itemize}
    \item T. Hastie, R. Tibshirani, and J. Friedman: \\
      \structure{The Elements of Statistical Learning --}\\
      \structure{ Data Mining, Inference, and Prediction},\\
      2nd edition, Springer, New York, 2009. \\[.3cm]
    \item David W. Hosmer, Stanley Lemeshow: \\
      \structure{Applied Logistic Regression}, 2nd Edition, \\
      John Wiley \& Sons, Hoboken, 2000.
  \end{itemize}
\end{frame}

\subsection{Comprehensive Questions}

\begin{frame}
  \frametitle{Comprehensive Questions \cont}

  \begin{itemize}
    \item How can a nonlinear function be written as a scalar product? \\[.7cm] \pause
    \item What is the objective function for the ML-estimation of the logistic regression parameters? \\[.7cm] \pause
    \item What is the difference between a gradient descent and Newton-Raphson numerical optimization scheme? \\[.7cm] \pause
    \item What is the parameter update rule for the logistic regression parameters using the Newton-Raphson scheme?
  \end{itemize}
\end{frame}


\chapter{One-dimensional spaces}

We can divide one-dimensional spaces into two categories:
\emph{lines} and \emph{curves}.
Lines are straight.
Curves may bend.
Thus every line is a curve.

\ResearchQuestion{%
Drawing a line is easy,
but how do we describe a line \emph{algebraically}?
}

\ResearchQuestion{%
A line is a straight one-dimensional space.
\emph{Straight} is defined by the ambient space that contains the line.
How do we define \emph{straight}?
}

\section{Defining a line}

If two points \(a\) and \(b\) are on a line,
then their midpoint \((a + b) / 2\) is also on the line.

(These aren't obvious to the uninitiated?)

If both \(a\) and \(b\) are on a line,
then the point \(k \cdot (b - a) + a\) is also on the line, for every \(k:\Real\).

If three points \(a,b,c\) are on a line,
then the displacements \(b-a\) and \(c-b\) are parallel.

\paragraph{Infinite extension of a line segment}
A \emph{line segment} is what is drawn using a straightedge.
A \emph{line} is obtained by infinitely extending a line segment in both directions.

\paragraph{Embedding of \(\Real\)}
\emph{A line is a straight embedding of \(\Real\).}
A line is something straight-shaped and isomorphic to \(\Real\).

But there is a problem with that definition.
That definition may include a pathological sheet, which should be a two-dimensional object.
This is a mapping from \(\Real^2\) to \(\Real\):
Let there be two real numbers \(a\) and \(b\).
Define the number \(c\) as \(\ldots a_1 b_1 a_0 b_0 . a_{-1} b_{-1} a_{-2} b_{-2} \ldots\).%
\footnote{\url{https://math.stackexchange.com/questions/75107/injective-map-from-mathbbr2-to-mathbbr}}%
\footnote{\url{https://math.stackexchange.com/questions/183361/examples-of-bijective-map-from-mathbbr3-rightarrow-mathbbr}}

We can define a line by a parametric equation: \( x(k) = k g + p \).

We can define a line by an algebraic equation: \( a \cdot x + b = 0 \).

\subsection{Defining a line as a curve with constant velocity}

The \emph{velocity} of the curve \( x : \Real \to \Real^n \) is the derivative of \(x\).
The velocity of \(x\) is the rate of change of \(x\).

\emph{A line is a curve whose velocity is constant.}

\subsection{Defining a line as a geodesic}

\section{Describing lines in a two-dimensional ambient space}

Every line can be described as the set
\( \{ (x,y) ~|~ (x,y) \in \Real^2, ~ a x + b y = c \} \).
(Why?)

\subsection{Describing a line that passes two points}

Describe a line that passes \((x_1,y_1)\) and \((x_2,y_2)\).

The description is
\begin{align*}
    a x_1 + b y_1 &= c
    \\
    a x_2 + b y_2 &= c
\end{align*}
Rearrange:
\begin{align*}
    x_1 a + y_1 b &= c
    \\
    x_2 a + y_2 b &= c
\end{align*}
Solve for \(a,b,c\).
\begin{align*}
    \Matrix{x_1 & y_1 \\ x_2 & y_2} \Matrix{a \\ b} = \Matrix{c \\ c}
\end{align*}
We can solve it using GNU Octave by typing \verb@[x1,y1;x2,y2] \ [c;c]@
but we have to substitute the variables with numbers first.

\subsection{Finding the angle formed by two lines}

\subsection{Translating lines and describing parallel lines}

Translating a line produces another line that is parallel to the original line.

Two lines \(ax+by=c\) and \(a'x+b'y=c'\) are parallel iff \(\abs{a/b} = \abs{a'/b'}\)?

\subsection{Describing orthogonal lines}

This is important for tangents, normals, and osculating circles.

\section{Describing higher-dimensional lines}

Every \(n\)-dimensional line can be described as
\( \{ k g + p ~|~ k \in \Real \} \)
if \(g, p : \Real^n\).

Describe a line that passes \(x_1\) and \(x_2\).
The description is
\begin{align*}
    x_1 &= k_1 g + p
    \\
    x_2 &= k_2 g + p
\end{align*}

\begin{align*}
    x_i - p &= k_i g
\end{align*}

How do we solve the equation \(a = kb\) if \(a,b\) are vectors and \(k\) is a scalar?


\section*{Probabilities}
$\mathbb{E}_x[X] = \begin{cases}
   \int x \cdot p(x) \partial x  &|\mathbb{E}_x[f(x)] =\\
   \sum_x x \cdot p(x) &|\int f(x) \cdot p(x) \partial x
  \end{cases}$\\
%$\mathbb{E}_x[f(x)] = \int f(x) \cdot p(x) \partial x $\\
$Var[X] = \mathbb{E}[(X-\mu_X)^2] = \mathbb{E}[X^2] - \mathbb{E}[X]^2$\\
$P(A|B) = \frac{P(B|A) \cdot P(A)}{P(B)}$; $p(Z|X,\theta) = \frac{p(X,Z|\theta)}{p(X|\theta)}$\\
$P(x,y) = P(x \cap y)= P(y|x) \cdot P(x) = P(x|y) \cdot P(y)$

%\subsection*{Linearity of expectation}
%$X, Y$ rand. var., $a, b \in \mathbb{R}$:\\
%$\mathbb{E}_{x,y}[aX + bY] = a\mathbb{E}_x[X] + b \mathbb{E}_y[Y]$


\begin{intro}
  \putindex{multigrid} Multigrid methods avoid the problems discussed
  in the section on two-level Schwarz methods by using not only two,
  but a whole hierarchy of mesh levels. On each level, an approximate
  solver, a so called smoother is employed, which improves the error
  somewhat, and then an approximation on a coarser level is used to
  improve further. This is done down to the coarsest level, where we
  assume that the solution process is cheap.
\end{intro}

\begin{Definition}{hierarchy-spaces}
  A \define{hierarchy of spaces} $\{V_\ell\}_0\le \ell \le L$ is a
  sequence of the form
  \begin{gather}
    V_0 \subset V_1 \subset \dots \subset V_L.
  \end{gather}
  We assume that $V_L = V$ is the high resolution space on which we
  want to solve~\eqref{eq:itintro:1}, but where the condition number
  of the matrix $\mat A$ is bad. On the other end of the spectrum, we
  assume that the solution of~\eqref{eq:itintro:1} on $V_0$ is easily
  possible.
\end{Definition}

\begin{Definition}{multigrid-method}
  A \define{multigrid method} consists of the following components:
  \begin{enumerate}
  \item A \define{smoother} $R_\ell$ acting on the level space
    $V_\ell$, usually an iterative method like Richardson, Jacobi,
    Gauss-Seidel or a Schwarz method.
  \item A \define{coarse grid solver} solving the problem on $V_0$
    exactly.
  \item Transfer operators between the levels $V_{\ell}$ and
    $V_{\ell+1}$. For standard finite element methods, this is
    typically the embedding operator. The transfer in opposite
    direction is achieved by the $L^2$-projection.
  \end{enumerate}

  On a given level $V_{\ell}$, the multigrid level consists of an
  alternating sequence of \putindex{smoothing step}s and
  \putindex{coarse grid correction}s, where the latter consist of a
  projection of the residual to the space $V_{\ell-1}$ and then
  recursive application of the same sequence. This is easiest
  described by the function
  \begin{subequations}
    \label{eq:mg:5}
  \begin{gather}
    \label{eq:mg:1}
    u_1 = MG_\ell(u^{(0)}, g),
  \end{gather}
  which takes an initial value $u^{(0)}$ and computes an approximation
  $u_1$ to the solution to $A_\ell u = g$ by the following scheme:
  first, for $\ell = 0$ let
  \begin{gather*}
    MG_0(u^{(0)}, g) = A_0^{-1} g_0.
  \end{gather*}
  On levels $\ell \neq 0$, perform the three steps
  \begin{itemize}
  \item Pre-smoothing: apply $m_{\text{pre}}$ steps of a Richardson
    iteration preconditioned with the smoother $R_\ell$:
    \begin{gather}
      \label{eq:mg:2}
      u^{(k+1)} = u^{(k)} - R_{\ell}^{-1} \left(A_\ell u^{(k)} - g_\ell\right),
      \qquad 0 \le k < m_{\text{pre}}.
    \end{gather}
    \item Coarse grid correction: let $v^{(0)} \in V_{\ell-1}$ and
      $g_{\ell-1} \in V_{\ell-1}^*$ such that
      \begin{gather}
        \label{eq:mg:3}
        g_{\ell-1} = \Pi_{\ell-1}^T \left(g_\ell - A_\ell u^{(m_{\text{pre}})}\right),
        \qquad
        v^{(0)} = 0.
      \end{gather}
      Then, compute
      \begin{gather}
        \label{eq:mg:4}
        v^{(k+1)} = MG_{\ell-1}(v^{(k)}, g_{\ell-1}),
      \qquad 0 \le k < m_{\text{coarse}}.
      \end{gather}
      Let $w^{(0)} \in V_{\ell}$ be given by $w^{(0)} =
      u^{(m_{\text{pre}})} + v^{(m_{\text{coarse}})}$.
  \item Post-smoothing: apply $m_{\text{post}}$ steps of a Richardson
    iteration preconditioned with the smoother $R_\ell$:
    \begin{gather}
      \label{eq:mg:2a}
      w^{(k+1)} = w^{(k)} - R_{\ell}^{-1} \left(A_\ell w^{(k)} - g_\ell\right),
      \qquad 0 \le k < m_{\text{post}}.
    \end{gather}
    Assign $MG(u^{(0)}, g_\ell) = w^{(m_{\text{post}})}$.
  \end{itemize}
  \end{subequations}

  This method has three parameters, the numbers of pre- and post
  smoothing steps $m_{\text{pre}}$ and $m_{\text{post}}$ as well as
  the number of coarse grid iterations $m_{\text{coarse}}$. Here, it
  is the last one which has a strong impact on the structure of the
  iteration. It defines what is called the \define{cycle type}, which
  is either \define{V-cycle} for $m_{\text{coarse}} = 1$ or
  \define{W-cycle} for $m_{\text{coarse}} = 2$. The structure of the
  cycles can be seen in Figure~\ref{fig:mg:1}.
\end{Definition}
\begin{figure}[tp]
  \centering
  \begin{minipage}[t]{.49\linewidth}
  \begin{tikzpicture}[thick,scale=.7]
    % Levels
    \draw[dotted](0,0) node[anchor=east]{$V_0$} -- (7,0);
    \draw[dotted](0,1) node[anchor=east]{$V_1$} -- (7,1);
    \draw[dotted](0,2) node[anchor=east]{$V_1$} -- (7,2);
    \draw[dotted](0,4) node[anchor=east]{$V_{L-2}$} -- (7,4);
    \draw[dotted](0,5) node[anchor=east]{$V_{L-1}$} -- (7,5);
    \draw[dotted](0,6) node[anchor=east]{$V_L$} -- (7,6);

    % Transfers
    \draw(0.5,6) -- (1.5,4);
    \draw[dashed](1.5,4) -- (2.5,2);
    \draw(2.5,2) -- (3.5,0) -- (4.5,2);
    \draw[dashed](4.5,2) -- (5.5,4);
    \draw(5.5,4) -- (6.5,6);

    % Smoothers and coarse grid correction
    \node at (0.5,6) [circle,draw=blue,fill=blue] {};
    \node at (1.0,5) [circle,draw=blue,fill=blue] {};
    \node at (1.5,4) [circle,draw=blue,fill=blue] {};
    \node at (2.5,2) [circle,draw=blue,fill=blue] {};
    \node at (3.0,1) [circle,draw=blue,fill=blue] {};
    \node at (4.0,1) [circle,draw=blue,fill=blue] {};
    \node at (4.5,2) [circle,draw=blue,fill=blue] {};
    \node at (5.5,4) [circle,draw=blue,fill=blue] {};
    \node at (6.0,5) [circle,draw=blue,fill=blue] {};
    \node at (6.5,6) [circle,draw=blue,fill=blue] {};
    \node at (3.5,0) [rectangle,draw=red,fill=red] {};
  \end{tikzpicture}
  \end{minipage}
  \begin{minipage}[t]{.49\linewidth}
  \begin{tikzpicture}[thick,yscale=.7,xscale=.6]
    % Levels
    \draw[dotted](0,0) node[anchor=east]{$V_0$} -- (8.0,0);
    \draw[dotted](0,1) node[anchor=east]{$V_1$} -- (8.0,1);
    \draw[dotted](0,2) node[anchor=east]{$V_1$} -- (8.0,2);
    \draw[dotted](0,4) node[anchor=east]{$V_{L-2}$} -- (8.0,4);
    \draw[dotted](0,5) node[anchor=east]{$V_{L-1}$} -- (8.0,5);
    \draw[dotted](0,6) node[anchor=east]{$V_L$} -- (8.0,6);

    % Transfers and smoothers
    \draw(0.2,6) node[circle,draw=blue,fill=blue] {}
    -- (0.4,5) node[circle,draw=blue,fill=blue] {}
    -- (0.6,4) node[circle,draw=blue,fill=blue] {};
    \draw[dashed](0.6,4) -- (1.0,2);
    \draw (1.0,2) node[circle,draw=blue,fill=blue] {}
    -- (1.2,1) node[circle,draw=blue,fill=blue] {}
    -- (1.4,0) node[rectangle,draw=red,fill=red] {}
    -- (1.6,1)
    -- (1.8,0) node[rectangle,draw=red,fill=red] {}
    -- (2.0,1) node[circle,draw=blue,fill=blue] {}
    -- (2.2,2)
    -- (2.4,1) node[circle,draw=blue,fill=blue] {}
    -- (2.6,0) node[rectangle,draw=red,fill=red] {}
    -- (2.8,1)
    -- (3.0,0) node[rectangle,draw=red,fill=red] {}
    -- (3.2,1) node[circle,draw=blue,fill=blue] {}
    -- (3.4,2) node[circle,draw=blue,fill=blue] {};
    \draw[dashed](3.4,2) -- (3.5,3) -- (3.7,0) -- (3.8,4);
    \draw(3.8,4) node[circle,draw=blue,fill=blue] {}
    -- (4.0,5)
    -- (4.2,4) node[circle,draw=blue,fill=blue] {};
    \draw[dashed] (4.2,4) -- (4.4,0) -- (4.5,3)
    -- (4.6,2)
    ;
    \draw (4.6,2) node[circle,draw=blue,fill=blue] {}
    -- (4.8,1) node[circle,draw=blue,fill=blue] {}
    -- (5.0,0) node[rectangle,draw=red,fill=red] {}
    -- (5.2,1)
    -- (5.4,0) node[rectangle,draw=red,fill=red] {}
    -- (5.6,1) node[circle,draw=blue,fill=blue] {}
    -- (5.8,2)
    -- (6.0,1) node[circle,draw=blue,fill=blue] {}
    -- (6.2,0) node[rectangle,draw=red,fill=red] {}
    -- (6.4,1)
    -- (6.6,0) node[rectangle,draw=red,fill=red] {}
    -- (6.8,1) node[circle,draw=blue,fill=blue] {}
    -- (7.0,2) node[circle,draw=blue,fill=blue] {};
    \draw[dashed] (7.0,2) -- (7.4,4);
    \draw (7.4,4) node[circle,draw=blue,fill=blue] {}
    -- (7.6,5) node[circle,draw=blue,fill=blue] {}
    -- (7.8,6) node[circle,draw=blue,fill=blue] {};
  \end{tikzpicture}
  \end{minipage}
  \caption{Smoothing and grid transfer of the V-cycle (left) and
    W-cycle (right). Black lines indicate grid transfer, blue dots are
  smoothing operations and red squares are coarse grid
  solvers. ``Time'' is left to right.}
  \label{fig:mg:1}
\end{figure}

\begin{remark}
  Figure~\ref{fig:mg:1} shows that the recursive structure of the
  W-cycle is much more complex than that of the V-cycle. The
  complexity analysis below will show that higher values of
  $m_{\text{coarse}}$ do not lead to efficient algorithms.
\end{remark}

\begin{Definition}{variable-v-cycle}
  If the numbers of pre- and post smoothing steps in the V-cycle are
  dependent on the level $\ell$, we speak of the \define{variable
    V-cycle}. A typical choice is $m_\ell = 2^{L-\ell} m_L$, thus
  doubling the number of smoothing steps whenever stepping down one
  level.
\end{Definition}

\begin{remark}
  The variable V-cycle with $m_\ell$ as mentioned in the previous
  definition has as many smoothing steps per iteration as the W-cycle.
\end{remark}

\begin{remark}
  If we use an additive or multiplicative Schwarz method (omitting the
  coarse grid) as our smoother $R_\ell$, it should be possible in
  principle to use the analytical tools of
  Chapter~\ref{cha:iteration:schwarz-methods}. The difficulty then
  consists in ensuring that the spectral radius of the iteration
  matrix does not grow towards one if we proceed upwards on our scale
  of spaces $V_\ell$. This remark is a todo for the author and an
  encouragement for the reader. Hints may be found
  in~\cite{GriebelOswald95,Xu92}.
\end{remark}

\begin{remark}
  It turns out that the techniques used for the analysis of the
  V-cycle and the W-cycle, respectively are quite
  different. Therefore, we separate them into two sections.
\end{remark}

\begin{Theorem}{mg-complexity}
  Let $n_\ell$ be the dimension of $V_\ell$. Assume that the effort
  needed to for the operations in equations~\eqref{eq:mg:5}b/c/e is
  linear in $n_\ell$ and assume that $ n_{\ell+1}/n_\ell \approx 2^d$,
  where $d$ is the space dimension of the grid. Assume that the effort
  for the coarse grid solver is negligible. Then, the effort for
  one step of the V-cycle is of order $n_L$. The effort for one step
  of the W-cycle is of order $n_L$ for $d \ge 2$, while it is of order
  $n_L \log(n_L)$ in one dimension.
\end{Theorem}

\begin{proof}
  Start the recursion on level $L$ with the function $MG_L(0,
  g)$. This function calls $MG_{L-1}(\ldots)$ $m_{\text{coarse}}$
  times. Thus, by recursion, $MG_{\ell}(\ldots)$ is executed
  $m_{\text{coarse}}^{L-\ell}$ times.

  By our assumptions, the amount of operations $\breve N_\ell$ in $MG_\ell(\ldots)$
  without the coarse grid correction is linear in $n_\ell$, say
  bounded by $Cn_\ell$. Then, the overall effort $N_L$ on level $L$ is
  \begin{gather}
    \label{eq:mg:6}
    N_L \le C \sum_{\ell=1}^L n_\ell m_{\text{coarse}}^{L-\ell} \le C
    \sum_{\ell=1}^L n_L 2^{d(l-L)}m_{\text{coarse}}^{L-\ell}
    = C n_L \sum_{\ell=1}^L \left(\frac{m_{\text{coarse}}}{2^d}\right)^l.
  \end{gather}
  It remains to notice that the sum converges and is bounded
  independent of $L$ if and only if $m_{\text{coarse}}/2^d < 1$. The
  statements of the theorem follow immediately, observing that $L
  \simeq \log n_L$.
\end{proof}

\begin{Lemma}{mg-error-step}
  Let $B_\ell^{-1}$ be the operator associated with the action of the
  multigrid preconditioner on level $\ell$ for
  $\ell=0,\ldots,L$. Then, the error after one step of the multigrid
  method has the form
  \begin{gather}
    \label{eq:mg:7}
    u^{(k+1)} - u = E_L \left(u^{(k)} - u \right),
  \end{gather}
  where for $\ell=0,\ldots,L$ we denote by $E_\ell$ the
  \putindex{error propagation operator}
  \begin{gather}
    \label{eq:mg:8}
    E_\ell = \left(I-R_\ell^{-1} A_\ell\right)^{m_{\text{post}}}
    \left(I-B_{\ell-1}^{-1}A_{\ell-1} P_{\ell-1}\right)^{m_{\text{coarse}}}
     \left(I-R_\ell^{-1} A_\ell\right)^{m_{\text{pre}}}
  \end{gather}
\end{Lemma}

\begin{proof}
  For the smoother, we use the standard technique for Richardson's
  method outlined in Lemma~\ref{lemma:richardson:1}. For the coarse
  grid correction, we use Lemma~\ref{lemma:schwarz:2}.
\end{proof}

\begin{remark}
  The structure of the error propagation operator~\eqref{eq:mg:8}
  already suggests the course of the multigrid analysis (as well as
  the design of smoothers). Namely, we will have to decompose $V_l$
  into $V_{l-1}$ and its $A$-orthogonal complement. Then, we use the
  induction argument that $I-B_{\ell-1}^{-1}A_{\ell-1}$ is is small on
  $V_{l-1}$, while bounded on its complement. Vice versa,
  $I-R_\ell^{-1} A_\ell$ must be bounded on all of $V_\ell$, while
  providing good reduction properties on the complement of $V_{l-1}$.
\end{remark}

\section{The V-cycle}

\begin{assumption}
  \label{assumption:mg:1}
  Let the smoother $R_\ell$ be symmetric, positive definite and let
  the following two conditions hold, the second for some positive
  constant $\alpha$ independent of $\ell$:
  \begin{subequations}
    \label{eq:mg:9}
    \begin{xalignat}{2}
      \label{eq:mg:10}
      a\bigl((I-R_\ell^{-1}A_\ell)v,v\bigr) & \ge 0
      & \forall v &\in V_\ell, \\
      \label{eq:mg:20}
      r(w,w) &\le \alpha a(w,w)
      & \forall v &\in V_\ell, \quad w = (I-P_{\ell-1}) v
    \end{xalignat}
  \end{subequations}
\end{assumption}

\begin{Theorem}{v-cycle-convergence}
  Let $a(.,.)$ be symmetric, positive definite and let
  Assumption~\ref{assumption:mg:1} hold. Then, the V-cycle operator
  with $m_{\text{pre}} =m_{\text{post}} = m$ admits the estimate
  \begin{gather}
    \label{eq:mg:11}
    0 \le a(\bigl((I-B_\ell^{-1}A_\ell)v,v\bigr) \le \delta a(v,v),
    \qquad \forall v \in V_\ell,
  \end{gather}
  where
  \begin{gather}
    \label{eq:mg:12}
    \delta = \frac{\alpha}{\alpha+2m}.
  \end{gather}
  In particular, the contraction number of the multigrid method is
  bounded by a number less than 1, independent of the level.
\end{Theorem}

\begin{proof}
  \footnote{This version of the proof is taken
    from~\cite{ArnoldFalkWinther97Hdiv}. It can also be found
    in~\cite{BraessHackbusch83,Bramble93}.}
  First, abbreviate $K_{\ell} = I-R_{\ell}^{-1} A_\ell$, the error
  propagation operator of a smoothing step. Now we will prove the
  theorem by induction over $\ell$. First, since $B_0 = A_0$, it holds
  on level zero. For higher levels, we derive from~\eqref{eq:mg:8} the
  relation
  \begin{gather}
    \label{eq:mg:13}
    E_\ell = I-B_\ell^{-1} A_\ell = K_\ell^m\Bigl(
    (I-P_{\ell-1}) + (I-B_{\ell-1}^{-1} A_{\ell-1}) P_{\ell-1}
    \Bigr) K_\ell^m.
  \end{gather}
  Non-negativity follows readily by the induction argument and
  the same properties of the smoother and the Ritz-projection. For the
  upper bound, let $w = K_\ell^m v$ to obtain by the induction
  hypothesis
  \begin{multline}
    \label{eq:mg:14}
    a(E_\ell v,v) \le a\bigl((I-P_{\ell-1}) w,w\bigr) + \delta
    a(P_{\ell-1}w,w)
    \\
    = (1-\delta) a\bigl((I-P_{\ell-1}) w,w\bigr)
   + \delta a(w,w).
  \end{multline}
  Now we use the smoothing hypothesis anf the
  Bunyakovsky-Cauchy-Schwarz inequality for the bilinear form $r(.,.)$
  associated with the smoothing operator $R_\ell$ to obtain
  \begin{align*}
    a\bigl((I-P_{\ell-1}) w,w\bigr)
    &= \scal({(I-P_{\ell-1}) w}, A_\ell w) \\
    &= \scal(R_\ell {(I-P_{\ell-1})} w, R_\ell^{-1} A_\ell w)
    \\
    &= r\bigl((I-P_{\ell-1}) w, R_\ell^{-1} A_\ell w\bigr) \\
    &\le \sqrt{r\bigl((I-P_{\ell-1}) w,(I-P_{\ell-1}) w\bigr)}
    \sqrt{r\bigl(R_\ell^{-1} A_\ell w, R_\ell^{-1} A_\ell w\bigr)}
    \\
    &\le \sqrt{\alpha a\bigl((I-P_{\ell-1}) w,(I-P_{\ell-1}) w\bigr)}
    \sqrt{a\bigl(R_\ell^{-1} A_\ell w, w\bigr)}
  \end{align*}
  Using the projection property of $I-P_{\ell-1}$, we obtain
  \begin{gather}
    \label{eq:mg:15}
    a\bigl((I-P_{\ell-1}) w,w\bigr) \le \alpha a\bigl(R_\ell^{-1} A_\ell
    w, w\bigr)
    = \alpha a\bigl((I-K_\ell) K_\ell^{2m} v,v\bigr).
  \end{gather}

  The smoothing assumption also guarantees that the spectrum of
  $K_\ell$ is contained in the interval $[0,1]$. Therefore,
  \begin{gather}
    \label{eq:mg:16}
    a\bigl((I-K_\ell) K_\ell^{2m} v,v\bigr)
    \le a\bigl((I-K_\ell) K_\ell^{i} v,v\bigr),
    \qquad i=0,\dots,2m,
  \end{gather}
  yielding by deflating the telescoping sum
  \begin{gather}
    \label{eq:mg:17}
    a\bigl((I-K_\ell) K_\ell^{2m} v,v\bigr)
    \le \frac1{2m} \sum_{i=0}^{2m-1}
    a\bigl((I-K_\ell) K_\ell^{i} v,v\bigr)
    = \frac1{2m} a\bigl((I-K_\ell^{2m}) v,v\bigr)
  \end{gather}
  Combining~\eqref{eq:mg:14}, \eqref{eq:mg:15}, and~\eqref{eq:mg:17},
  we obtain
  \begin{gather}
    \begin{split}
      a(E_\ell v,v) &\le (1-\delta)\frac\alpha{2m}
      a\bigl((I-K_\ell^{2m}) v,v\bigr)
      + \delta a(K_\ell^m v,K_\ell^m v)
      \\
      &= (1-\delta)\frac\alpha{2m} a(v,v)
      + \left(\delta - (1-\delta)\frac\alpha{2m}\right)
      a(K_\ell^m v,K_\ell^m v).
    \end{split}
  \end{gather}
  Finally, we enter $\delta=\alpha/(\alpha+2m)$ to obtain
  \begin{gather*}
    \delta - (1-\delta)\frac\alpha{2m} = 0,
  \end{gather*}
  and thus
  \begin{gather}
    \label{eq:mg:18}
    a(E_\ell v,v) \le \left(1-\frac\alpha{\alpha+2m}\right)\frac\alpha{2m}
    a(v,v) = \frac\alpha{\alpha+2m}a(v,v).
  \end{gather}
\end{proof}

\begin{Lemma}{mg-additive}
  Let $R_\ell$ be the scaled additive Schwarz method
  \begin{gather}
    \label{eq:mg:19}
    R_a^{-1} = \omega \sum_{j=1}^J P_j A^{-1},
  \end{gather}
  where the subspaces $V_j$ are defined by overlapping subdomains
  $\Omega_j$ as in~\eqref{eq:schwarz:9}. Not that we do not include
  the coarse space here. Then, for $\omega$ sufficiently small, this
  smoother fulfills Assumption~\ref{assumption:mg:1}.
\end{Lemma}

\begin{proof}
  The positive definiteness and symmetry of the smoother have been
  proven in an abstract way in Lemma~\ref{lemma:schwarz:3}. In order
  to prove estimate~\eqref{eq:mg:10}, we observe that by
  Lemma~\ref{lemma:schwarz:5} and Lemma~\ref{lemma:schwarz:6}
  \begin{gather*}
    r(v,v) = \omega \min_{v=\sum v_j} a(v_j, v_j) \gtrsim \omega a(v,v),
  \end{gather*}
  where the implicit constant depends on the number of overlaps in
  Definition~\ref{definition:schwarz:finite-covering}. Thus, we can
  choose $\omega$ independent of $\ell$ such that
  \begin{gather*}
    r(v,v) \ge a(v,v) \qquad \forall v\in V_\ell.
  \end{gather*}
  Accordingly, $R_\ell - A_\ell$ is positive definite, and since
  $R_\ell^{-1}$ is as well, so is $I-R_\ell^{-1} A_\ell$.
  It remains to prove~\eqref{eq:mg:20}, but this is exactly the second
  half of the proof of Lemma~\ref{lemma:schwarz:stable-decomposition},
  if we replace the Clément interpolant into the coarse space by the
  Ritz projection.
\end{proof}

%\section{The W-cycle and two-grid convergence}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: