\section{Functions on Euclidean Space}
\textbf{NOTE:} My notes differ from Spivak's text in that I use subscripts to denote
components instead of superscripts.

\subsection{Norm and Inner Product}

\subsubsection{Exercise 1}
\begin{align*}
        \sum_{i = 1}^n x_i^2  = \sum_{i = 1}^n \abs{x_i}^2 \leq \bigg(\sum_{i = 1}^n \abs{x_i}\bigg)^2
        \implies \sqrt{\sum_{i = 1}^n x_i^2} \leq \sum_{i = 1}^n \abs{x_i}^2
\end{align*}

\subsubsection{Exercise 2}
We need $\sum_{i = 1}^n x_i y_i = \abs{x} \abs{y}$. Linear dependence by itself is not enough, since if
$x$ and $y$ are opposite sign we see that the lefthand sum will be negative. Thus, we need linear dependence
as well as $x$ and $y$ having the same sign.

\subsubsection{Exercise 3}
The proof is identical to the $\abs{x + y}$ case, except now we have a $-2 \sum_{i = 1}^n x_i y_i$ term.
Thus, equality holds when $x$ and $y$ are linearly dependent and have opposite signs.

\subsubsection{Exercise 4}
$\abs{\abs{x} - \abs{y}}^2 = \abs{x}^2 + \abs{y}^2 - 2\abs{x} \abs{y}$. Since $\abs{x} \abs{y} \geq \langle x, y \rangle$, we have the desired inequality.

\subsubsection{Exercise 5}
\begin{align*}
        \abs{z - x} = \abs{z - y + y - x} \leq \abs{z - y} + \abs{y - x}
\end{align*}
Geometrically, this is just the fact that any sidelength of a triangle must be bounded by the sum of the
other two sidelengths.

\subsubsection{Exercise 6}
(a) We can proceed as Spivak hints by noting that for the $\int_{a}^{b} (f - \lambda g)^2 > 0$ case, the
proof is identical to that of Theorem 1-1 (2). For the $\int_{a}^{b} (f - \lambda g)^2 = 0$ case (which is when equality is obtained), we
can use the fact that $(f - \lambda g)^2 = 0$ almost everywhere.

However, I think it's a little smoother to handle both cases at once:
\begin{align*}
        \int_{a}^{b} (f - \lambda g)^2 = \int_{a}^{b} f^2 - 2\lambda \int_{a}^{b} fg + \lambda^2 \int_{a}^{b} g^2 &\geq 0 \\
        \int_{a}^{b} f^2 - \frac{2 \bigg(\int_{a}^{b} fg\bigg)^2}{\int_{a}^{b} g^2} + \frac{\bigg(\int_{a}^{b} fg\bigg)^2}{\int_{a}^{b} g^2} &\geq 0 \quad \text{for} \: \: \lambda = \frac{\int_{a}^{b} fg}{\int_{a}^{b} g^2} \\
        \sqrt{\int_{a}^{b} f^2 \int_{a}^{b} g^2} &\geq \abs{\int_{a}^{b} fg}
\end{align*}

(b) Equality does not necessarily imply that $f = \lambda g$, since we could consider $f$ and $g$ to be
0 everywhere except two points $a$ and $b$ such that $f(a) \neq \lambda g(a)$ and $f(b) \neq \lambda g(b)$.
However, if $f$ and $g$ are continuous, then $\int_{a}^{b} (f - \lambda g)^2 = 0$ implies that $f = \lambda g$.

(c) Define $f(m) = x_i$ and $g(m) = y_i$ for $m \in [i, i + 1)$. Then $\int_{1}^{n} f g = \sum_{1}^n x_i y_i$
(we can break up the integral at the points of discontinuity) and Theorem 1-1 (2) follows.

\subsubsection{Exercise 7}
(a) If $T$ is inner product preserving then we have $\langle Tx, Tx \rangle = \langle x, x \rangle$, so
$T$ is norm preserving. If $T$ is norm preserving, then
\begin{align*}
        \langle T(x - y), T(x - y) \rangle &= \abs{T(x)}^2 + \abs{T(y)}^2 - 2\langle T(x), T(y) \rangle \\
        \langle x - y, x - y \rangle &= \abs{x}^2 + \abs{y}^2 - 2 \langle x, y \rangle \\
        \implies \langle T(x), T(y) \rangle &= \langle x, y \rangle
\end{align*}
so $T$ is inner product preserving.

(b) Since $Tx = Ty \implies T(x - y) = 0 \implies \abs{x - y} = 0$, $T$ is injective. Furthermore,
$Tx = 0 \implies \abs{x} = 0$, so the nullspace of $T$ is trivial and $T$ is thus surjective. Now if we
consider $T^{-1} y = x$, then we have $\langle y, y \rangle = \langle TT^{-1} y, TT^{-1} y \rangle = \langle T^{-1} y, T^{-1} y \rangle$.

\subsubsection{Exercise 10}
Let $\norm{T}_F = \sqrt{\sum_{i = 1}^n \sum_{j = 1}^m T_{ij}^2}$ (Frobenius norm). Then we have
\begin{align*}
        \abs{T h}^2 = \sum_{i = 1}^n \bigg(\sum_{j = 1}^m T_{ij} h_j\bigg)^2 &\leq \sum_{i = 1}^n \sum_{j = 1}^m T_ij^2 \sum_{j = 1}^m h_j^2 = \norm{T}_F^2 \abs{h}^2
\end{align*}
so letting $M = \norm{T}_F$ gives the desired inequality.

\subsubsection{Exercise 12}
Linearity and injectivity of $T$ follow from linearity of inner product. To see surjectivity, we note that
any element $f \in (\mathbb{R}^n)^*$ is determined entirely by $f(e_1), ..., f(e_n)$ due to linearity. Thus,
$f(y) = \langle x, y \rangle$ for the unique $x$ satisfying $x_i = f(e_i)$.

\subsubsection{Exercise 13}
Expanding $\langle x + y, x + y \rangle$ gives the desired result.

\subsection{Subsets of Euclidean Space}

\subsubsection{Exercise 14}
Any point $a$ in the union is also in some set which contains an open set around $a$, and by the definition of
union this open set is also in the union. For finite intersection, if a point is in two open sets, then both
sets must contain some open rectangles around that point; the smaller of these rectangles is in the
intersection of both sets. For infinite intersection, we can consider the intersection of
$(-\frac{1}{n}, \frac{1}{n})$ for all $n \in \mathbb{N}$, as it consists only of the single point 0.

\subsubsection{Exercise 17}
The procedure hinted by Spivak seems to be to split the square into 4 quadrants and then select a point from
each quadrant while satisfying the constraint, then repeat this procedure indefinitely (split the 4 quadrants
into 16 quadrants, etc.). However, I'm not sure how to make this procedure more rigorous.

\subsubsection{Exercise 19}
We can use the facts that the rationals are dense in $\mathbb{R}$ and that closed sets contain all of their
limit points (although neither fact is proved so far in this book) to immediately get the desired result.

\subsubsection{Exercise 21}
(a) Since $A$ is closed, $A^c$ is open, which means $x$ has an open rectangle (and thus, an open ball) around
it that is contained within $A^c$. Therefore, there exists $d > 0$ such that $\abs{y - x} \geq d$ for all
$y \in A$.

(b) Suppose that for every $d > 0$, we could choose $y_i \in A$ and $x_i \in B$ such that $\abs{y_i - x_i} < d$.
This would imply that we could pick a sequence $\{y_n\} \in A$ and a sequence $\{x_n\} \in B$ such that
 $\lim_{n \to \infty} y_n = \lim_{n \to \infty} x_n$. Since $B$ is compact, this limit must be a point in
  $B$, which contradicts the disjointness of  $A$ and  $B$.

(c) Consider $A = \{(0, i) \: | \: i \in \mathbb{N}\}$ and $B = \{(0, i + \frac{1}{i}) \: | \: i \in \mathbb{N}\}$.
For any $d$, we can choose $i$  such that $\frac{1}{i} < d$.

\subsubsection{Exercise 22}
This is easier to argue with open/closed balls instead of open/closed rectangles (for me). Every point
$c \in C$ must have an open ball $B_{\delta} (c) \subset U$ for some $\delta > 0$. For each such ball,
consider instead $B_{r} (c)$ with $r < \delta$. The closure of $B_r(c)$ is a closed ball that is contained
within $U$. Now, since $C$ is compact, it can be covered by finitely many of these closed balls. Since
closed sets are closed under finite union, we can take $D$ to be the union of these balls to get a compact
set whose interior contains $C$.

\subsection{Functions and Continuity}

\subsubsection{Exercise 23}
Suppose $\lim_{x \to a} f(x) = b$. Then we can choose $\delta$ such that $\abs{x - a} < \delta$ implies
$\abs{f(x) - b} < \epsilon$ for any $\epsilon > 0$. Rewriting $\abs{f(x) - b}$ as
$\sqrt{\sum_{i = 1}^m (f_i (x) - b_i)^2}$ gives $\abs{f_i(x) - b_i} < \epsilon$ for the same $\delta$, so
we get $\lim_{x \to a} f_i(x) = b_i$. The other direction reverses the steps after choosing
$\abs{f_i(x) - b_i} < \frac{\epsilon}{\sqrt{m}}$.

\subsubsection{Exercise 24}
This is basically the same as the previous exercise.

\subsubsection{Exercise 25}
From Exercise 1-10 we have that there exists  $M$  such that $\abs{T(x)} \leq M \abs{x}$. Thus, for any
$\epsilon > 0$ and $a \in \mathbb{R}^n$, we can choose $\delta = \frac{\epsilon}{M}$. Using this $\delta$,
we get $\abs{x - a} < \delta$ implies $\abs{T(x) - T(a)} = \abs{T(x - a)} < \epsilon$, so $T$ is continuous.

\subsubsection{Exercise 28}
Choose any $x$ on the boundary of  $A$ and let  $f(y) = \frac{1}{\abs{y - x}}$. Since $f$ is the quotient
of two continuous functions ($g(y) = 1$ and  $h(y) = \abs{y - x}$) with non-zero denonimator, $f$ is continuous.
Furthermore, by construction we can choose  $y$ arbitrarily close to  $x$, so  $f$ is unbounded.

\subsubsection{Exercise 29}
Since $f$ is continuous,  $f(A)$ is compact. By Heine-Borel,  $f(A) \subset \mathbb{R}$ is closed and bounded,
so it contains its maximum and minimum.

\subsubsection{Exercise 30}
Let $x_0 = a$. We have that
\begin{align*}
        \sum_{i = 1}^n o(f, x_i) &= \lim_{\delta \to 0} \sum_{i = 1}^n [M(f, x_i, \delta) - m(f, x_i, \delta)] \\
                                 &\leq \lim_{\delta \to 0} \sum_{i = 1}^n [M(f, x_i, \delta) - M(f, x_{i - 1}, \delta)] \\
                                 &\leq \lim_{\delta \to 0} M(f, x_n, \delta) - m(f, a, \delta) \\
                                 &\leq f(b) - f(a)
\end{align*}
Where we used the fact that $f$ is increasing to get $m(f, x_i, \delta) \geq M(f, x_{i - 1}, \delta)$.


\subsection{Complex functions}\label{subsec:complex_functions}

\begin{definition}\label{def:sequence_spaces}
  We will define multiple Banach spaces of sequences over \( \BbbC \).

  \begin{thmenum}
    \thmitem{def:sequence_spaces/c00} The simplest nontrivial sequence space is that of all sequences with only finitely many nonzero elements. It is denoted by \( c_{00} \). It can be defined as
    \begin{equation*}
      c_{00} \coloneqq \bigcup_{i=1}^\infty \BbbC^k,
    \end{equation*}
    where \( \BbbC^k \) is the corresponding \hyperref[def:module_of_tuples]{tuple space}.

    This space can be generalized to modules over \hyperref[def:module]{semirings}.
  \end{thmenum}
\end{definition}

\begin{definition}\label{def:function_spaces}
  We will define multiple Banach spaces of functions over \( \BbbK \).

  \begin{thmenum}
    \thmitem{def:function_spaces/c0} Define the set of functions \term{vanishing at infinity}:
    \begin{equation*}
      C_0(\BbbC) \coloneqq \{ f: \BbbC \to \BbbC \colon f(x) \xrightarrow[\abs{x} \to \infty]{} 0 \}.
    \end{equation*}

    \thmitem{def:function_spaces/c} Fix \hyperref[def:topological_space]{topological space} \( X \). The set \( C(X) = C(X, \BbbK) \) of all \( \BbbK \)-valued continuous functions on \( X \) in a Banach space over \( \BbbK \).
  \end{thmenum}
\end{definition}

\begin{theorem}[Arzela-Ascoli]\label{thm:arzela_ascoli}\mcite[cor. 10.49]{Knapp2016BasicAlgebra}
  Let \( X \) be a \hyperref[def:compact_space]{compact} \hyperref[def:separation_axioms/T2]{Hausdorff} space.

  A family \( \mscrF \subseteq C(X, \BbbR) \) of continuous real-valued functions is totally \hyperref[def:totally_bounded_set]{bounded} if and only if it is pointwise \hyperref[def:bounded_function/pointwise]{bounded} and \hyperref[def:function_set_continuity/equicontinuous]{equicontinuous}.
\end{theorem}


\subsection{Lattices}\label{subsec:lattices}

\begin{definition}\label{def:semilattice}\mcite[3]{Gratzer1978}
  Lattices are \hyperref[def:partially_ordered_set]{partially ordered sets} in which \hyperref[def:partially_ordered_set_extremal_points/supremum_and_infimum]{suprema and infima} are taken as basic operations called \enquote{joins} and \enquote{meets}. See \fullref{rem:lattice_operation_etymology} for a discussion of the operation names. This shifts the focus from ordering to operations, i.e. from predicates to functions.

  Joins and meets may also be defined axiomatically as binary operations (see \fullref{thm:binary_lattice_operations}) rather than via some partial order, however this restricts us to taking suprema of finite sets and prevents us from taking the supremum of an arbitrary set. In other words, it is possible for the order to carry more information than joins and meets. See \fullref{thm:binary_lattice_operations/new_lattice} for a discussion. Unless explicitly noted otherwise, we assume that lattices have their partial order defined.

  \begin{thmenum}[series=def:semilattice]
    \thmitem{def:semilattice/join} A \term{join-semilattice} is a \hyperref[def:partially_ordered_set_extremal_points/top_and_bottom]{bounded from above} partially ordered set in which every finite supremum exists. The operation itself is denoted by \( \vee \) and referred to as \term{join} and rather than supremum. In contrast to suprema, joins are usually written in \hyperref[rem:first_order_formula_conventions/infix]{infix} notation, e.g. \( x \vee y \vee z \) rather than \( \sup\set{ x, y, z } \).

    \thmitem{def:semilattice/meet} Analogously, a \term{meet-semilattice} is a partially ordered set in which every finite infimum exists. The infimum is denoted by \( \wedge \) and called \term{meet}.

    \thmitem{def:semilattice/bounded} A \term{bounded semilattice} is a semilattice that is \hyperref[def:partially_ordered_set_extremal_points/top_and_bottom]{bounded} as a \hyperref[def:partially_ordered_set]{partially ordered set}, either from above for join-semilattices or from below for meet-semilattices.

    \thmitem{def:semilattice/complete}\mcite[24]{Gratzer1978} A semilattice is said to be \term{complete} if the corresponding operation is defined for arbitrary sets rather than only finite ones. Finite lattices are trivially complete and complete lattices are trivially bounded.

    \thmitem{def:semilattice/lattice} A \term{lattice} is a partially ordered set which is both a join-semilattice and a meet-semilattice. It is called \term{bounded} if both semilattices are bounded, i.e. if the partially ordered set itself is \hyperref[def:partially_ordered_set_extremal_points/top_and_bottom]{bounded}. It is called \term{complete} if both semilattices are complete.

    \thmitem{def:semilattice/distributive_lattice}\mcite[30]{Gratzer1978} A lattice is said to be \term{distributive} if the following two distributive conditions hold:
    \begin{align}
      x \vee (y_1 \wedge y_2) &= (x \vee y_1) \wedge (x \vee y_2) \label{eq:def:semilattice/distributive_lattice/finite/join_over_meet} \\
      x \wedge (y_1 \vee y_2) &= (x \wedge y_1) \vee (x \wedge y_2) \label{eq:def:semilattice/distributive_lattice/finite/meet_over_join}.
    \end{align}

    If the lattice is \hyperref[def:semilattice/complete]{complete}, the above conditions are not enough. A complete lattice \( \mscrX \) it is said to be \term{distributive} if any of the following more general distributive axioms hold for any \( x \in \mscrX \) and \hyperref[def:cartesian_product/indexed_family]{family} \( \seq{ y_k }_{k \in \mscrK} \subseteq \mscrX \):
    \begin{align}
      x \vee \parens*{ \bigwedge_{k \in \mscrK} y_k } &= \bigwedge_{k \in \mscrK} \parens{ x \vee y_k } \label{eq:def:semilattice/distributive_lattice/arbitrary/join_over_meet} \\
      x \wedge \parens*{ \bigvee_{k \in \mscrK} y_k } &= \bigvee_{k \in \mscrK} \parens{ x \wedge y_k } \label{eq:def:semilattice/distributive_lattice/arbitrary/meet_over_join}
    \end{align}
  \end{thmenum}

  Lattices have the following metamathematical properties:
  \begin{thmenum}[resume=def:semilattice]
    \thmitem{def:semilattice/theory} The language of the theory of lattices consists of the language of the \hyperref[def:partially_ordered_set/theory]{theory of partially ordered sets} with the addition of the binary infix functional symbols \( \vee \) and \( \wedge \). If we only want to restrict ourselves to semilattices, we can add only one of the two operations as functional symbols. If we wish to study \hyperref[def:semilattice/bounded]{bounded lattices}, as it is often done, we must also add the constants \( \top \) and \( \bot \).

    For meet-semilattices, we add the following axiom schema to the theory to ensure compatibility between infima and meets (we use \( \mathbin\& \) to denote \hyperref[def:propositional_language/connectives/conjunction]{logical conjunction} to avoid symbol collision with meets):
    \begin{equation}\label{eq:def:semilattice/theory/meet_compat}
      \parens[\Big]{ \xi \wedge \eta \doteq \alpha } \leftrightarrow \parens[\Big]{ \alpha \leq \xi \mathbin\& \alpha \leq \eta \mathbin\& \qforall \alpha ((\alpha \leq \xi \mathbin\& \alpha \leq \eta) \rightarrow \alpha \leq \alpha) }
    \end{equation}
    and, for bounded lattices, the following axiom to ensure that \( \bot \) is indeed the minimum:
    \begin{equation}\label{eq:def:semilattice/theory/bottom_compat}
      \qforall \xi (\bot \leq \xi).
    \end{equation}

    Analogous axioms need to be added for join-semilattices.

    We cannot proper express the theory of complete (semi)lattices as an extension of this theory since we must define join and meet as unary operations on subsets of the domain rather than binary operations on members of the domain. Complete semilattices can instead be defined within \hyperref[def:zfc]{\logic{ZFC}}.

    \thmitem{def:semilattice/submodel} Unlike for partially ordered sets (see \fullref{def:partially_ordered_set/submodel}), not any subset of a semilattice is a subsemilattice because \( \vee \) and \( \wedge \) are now regarded as functional symbols. The axiom \eqref{eq:def:semilattice/theory/meet_compat} is not a positive formula, but does not cause trouble itself as it merely specifies compatibility of \( \leq \) and \( \wedge \).

    For bounded semilattices, the relevant constants should be present in any bounded subsemilattice.

    \thmitem{def:semilattice/trivial} The \hyperref[thm:substructures_form_complete_lattice/bottom]{trivial join-semilattice} and the trivial meet-semilattice are the empty set. The trivial bounded join-semilattice is the singleton \( \set{ \top } \) and the trivial bounded meet-semilattice is \( \set{ \bot } \).

    The trivial bounded lattice is \( \set{ \top, \bot } \).

    Note that the elements \( \top \) and \( \bot \) formally differ between different semilattices, however all trivial bounded lattices are isomorphic and hence it makes sense to speak of \enquote{the} bounded lattice.

    \thmitem{def:semilattice/homomorphism} \hyperref[def:first_order_homomorphism]{Homomorphisms} between (semi)lattices are simply the monotone maps.

    Alternatively, without referring to the order, we can characterize homomorphisms as functions preserving joins, meets and constants. No axioms follow automatically as in \fullref{thm:group_homomorphism_single_condition}.

    \thmitem{def:semilattice/category} The \hyperref[def:category_of_small_first_order_models]{categories of models} for (semi)lattices are full subcategories of \hyperref[def:partially_ordered_set/category]{\( \cat{Pos} \)}. We only give a special name for the category \( \cat{Lat} \) of lattices.

    \thmitem{def:semilattice/lattice_duality} The \hyperref[def:partially_ordered_set/duality]{principle of duality for partially ordered sets} holds for lattices if we also swap the binary operations \( \vee \) and \( \wedge \).

    If the lattice is bounded, we must additionally swap the constants \( \top \) and \( \bot \).

    If the lattice is bounded from only one side, the principle of duality does not hold unless we restrict ourselves to formulas that do not contain the constants.
  \end{thmenum}
\end{definition}

\begin{remark}\label{rem:lattice_operation_etymology}
  The terms \hyperref[def:semilattice/join]{\enquote{join}} for \( \vee \) and \hyperref[def:semilattice/meet]{\enquote{meet}} for \( \wedge \) are notoriously difficult to remember. A helpful accident is the ability to write \enquote{meet} as \enquote{\( \wedge \wedge \)eet}.
\end{remark}

\begin{proposition}\label{thm:binary_lattice_operations}
  Let \( (\mscrP, \leq) \) be a partially ordered set.

  \begin{thmenum}
    \thmitem{thm:binary_lattice_operations/semilattices} If it is a \hyperref[def:semilattice/join]{join-semilattice} (resp. \hyperref[def:semilattice/meet]{meet-semilattice}), then \( \vee \) (resp. \( \wedge \)) is \hyperref[def:magma/associative]{associative}, \hyperref[def:magma/commutative]{commutative} and \hyperref[def:magma/idempotent]{idempotent} when considered as a binary operation.

    \thmitem{thm:binary_lattice_operations/identity} If \( \mscrP \) is a (semi)lattice, the constants act as \hyperref[def:magma_identity]{magma identities}. That is, for each \( x \in \mscrP \),
    \begin{align}
      x \vee \bot = x \label{eq:thm:binary_lattice_operations/identity/join} \\
      x \wedge \top = x \label{eq:thm:binary_lattice_operations/identity/meet}
    \end{align}

    \thmitem{thm:binary_lattice_operations/absorption} If \( \mscrP \) is a lattice, then the following absorption laws hold:
    \begin{align}
      x \vee (x \wedge y) &= x \label{eq:thm:binary_lattice_operations/absorption/join} \\
      x \wedge (x \vee y) &= x \label{eq:thm:binary_lattice_operations/absorption/meet}.
    \end{align}

    \thmitem{thm:binary_lattice_operations/compatibility} The following conditions for compatibility with \( \leq \) hold:
    \begin{align}
      x \leq y &\T{if and only if} x \vee y = y \label{eq:thm:binary_lattice_operations/compatibility/join} \\
      x \leq y &\T{if and only if} x \wedge y = x \label{eq:thm:binary_lattice_operations/compatibility/meet}.
    \end{align}

    \thmitem{thm:binary_lattice_operations/new_lattice} If \( S \) is an arbitrary \hyperref[def:set]{set} and if \( \vee \) is a binary operation that is associative, commutative and idempotent (the conclusion of \fullref{thm:binary_lattice_operations/semilattices}), then \( (S, \leq) \) is a join-semilattice with an ordering defined by \eqref{eq:thm:binary_lattice_operations/compatibility/join}. If there exists a distinguished element \( \top \) such that \eqref{eq:thm:binary_lattice_operations/identity/join} holds, then \( (S, \leq) \) is a meet-semilattice.

    A completely analogous statement holds for meet-semilattices.

    If \( (S, \leq) \) is both a join-semilattice and meet-semilattice and if \( \vee \) and \( \wedge \) satisfy the absorption conditions \eqref{eq:thm:binary_lattice_operations/absorption/join} and \eqref{eq:thm:binary_lattice_operations/absorption/meet}, then \( (S, \leq) \) is a lattice. Furthermore, proving idempotence for \( \vee \) or \( \wedge \) is unnecessary because both follow from the absorption conditions.

    It may turn out that \( (S, \leq) \) is a complete lattice under this definition. This can allow us, for example, to transparently extend the binary operations join and meet into infinitary operations.
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{thm:binary_lattice_operations/semilattices} Suprema and infima are obviously associative and commutative as binary operations because ordering is immaterial for pure sets and \( x \vee y \) is defined as \( \sup\set{ x, y } \).

  Idempotence is also obvious because \( x \vee x = \sup\set{ x } = x \).

  \SubProofOf{thm:binary_lattice_operations/identity} Obvious since \( \bot \leq x \leq \top \) for all \( x \in \mscrP \).

  \SubProofOf{thm:binary_lattice_operations/absorption} If we rewrite \eqref{eq:thm:binary_lattice_operations/absorption/join} using suprema and infima, we obtain
  \begin{equation*}
    \sup\set{ x, \inf\set{ y, x } } = x.
  \end{equation*}

  If \( x \leq y \), then \( \inf\set{ y, x } = x \) and \( \sup\set{ x, \inf\set{ y, x } } = \sup\set{ x, x } = x \).

  If \( x \geq y \), then \( \inf\set{ y, x } = y \) and \( \sup\set{ x, \inf\set{ y, x } } = \sup\set{ x, y } = x \).

  This proves \eqref{eq:thm:binary_lattice_operations/absorption/join}. Since \( \wedge \) is \( \vee \) in the \hyperref[def:preordered_set/duality]{dual partially ordered set}, \eqref{eq:thm:binary_lattice_operations/absorption/meet} follows automatically.

  \SubProofOf{thm:binary_lattice_operations/compatibility} We have
  \begin{equation*}
    x \vee y
    =
    \sup\set{ x, y }
    =
    \begin{cases}
      y, &x \leq y \\
      x, &x > y
    \end{cases}
  \end{equation*}
  and dually for \( \wedge \).

  \SubProofOf{thm:binary_lattice_operations/new_lattice} Since the binary join and/or meet are defined for all members of the set \( S \), it is indeed a join-semilattice because all finite joins and meets exist by definition.

  Idempotence of \( \vee \) follows from \eqref{eq:thm:binary_lattice_operations/absorption/meet}:
  \begin{equation*}
    x \vee x = x \vee (x \wedge (x \vee x)) = x
  \end{equation*}
  and dually for \( \wedge \).
\end{proof}

\begin{definition}\label{def:fixed_point}
  Given a \hyperref[def:function]{function} \( f: A \to A \) between arbitrary sets, we call \( x \in A \) a \term{fixed point} of \( f \) if \( x = f(x) \).
\end{definition}

\begin{theorem}[Knaster-Tarski theorem]\label{thm:knaster_tarski_theorem}
  The \hyperref[def:fixed_point]{fixed points} of a \hyperref[def:partially_ordered_set/homomorphism]{monotone} \hyperref[def:multi_valued_function/endofunction]{endofunction} in a \hyperref[def:semilattice/lattice]{complete lattice} form a complete sublattice. In particular, the function has at least one fixed point.
\end{theorem}
\begin{proof}
  Let \( (\mscrX, \leq) \) be a complete lattice and let \( \varphi: \mscrX \to \mscrX \) be a monotone function. Define
  \begin{equation*}
    L \coloneqq \{ x \in \mscrX \colon f(x) \leq x \}.
  \end{equation*}

  We know that \( L \) is nonempty because \( \top \in L \).

  Since the lattice is complete, we can take \( l \coloneqq \inf L \). Note that \( f(l) \) is a lower bound of \( L \) because for any \( y \in L \) we have
  \begin{equation*}
    f(l) \leq f(y) \leq y.
  \end{equation*}

  But \( l \) is the largest lower bound of \( L \), hence
  \begin{equation}\label{eq:thm:knaster_tarski/f_lower}
    f(l) \leq l.
  \end{equation}

  Therefore, \( f(f(l)) \leq f(l) \) and \( f(l) \in L \). Hence, \( l \) is a lower bound for \( \{ f(l) \} \) and
  \begin{equation}\label{eq:thm:knaster_tarski/f_upper}
    l \leq f(l).
  \end{equation}

  From \eqref{eq:thm:knaster_tarski/f_lower} and \eqref{eq:thm:knaster_tarski/f_upper} it follows that \( l = f(l) \), that is, \( l \) is a fixed point of \( f \).

  Denote by \( F \) the set of all fixed points of \( \mscrX \). We just showed that \( F \) is nonempty. Let \( G \subseteq F \). We will show that the infimum and supremum of \( G \) is in \( F \).

  Denote
  \begin{equation*}
    l_G \coloneqq \inf G.
  \end{equation*}

  For any \( g \in G \) we have \( l_G \leq g \). From monotonicity of \( f \),
  \begin{equation*}
    f(l_G) \leq f(g) = g,
  \end{equation*}
  therefore \( f(l_G) \leq l_G \) because \( l_G \) is the greatest lower bound of \( G \). But, from monotonicity of \( f \), we have \( l_G \leq f(l_G) \). Therefore, \( f(l_G) = l_G \) and \( l_G \in F \).

  We can analogously show that \( \sup G \in F \) and conclude that \( (F, \leq) \) is itself a complete lattice.
\end{proof}

\begin{remark}\label{def:semilattice/lattice_categorical_product}
  The existence of finite joins and meets is equivalent to the existence of finite products and coproducts in the respective partially ordered set category defined in \fullref{thm:order_category_isomorphism/partially_ordered}.
\end{remark}


\subsection{Complex smooth functions}

If a function is complex differentiable, it is smooth.


\section{Extraction of Linearized Models}
Models that represent the linearized dynamics of a system in the neighborhood of an equilibrium point (trim condition) can be used to gain insight into system stability characteristics. Consider the dynamics represented by Eqs. (\ref{eqn:allodes}). Expanding the left hand side in a Taylor series, we obtain
\begin{equation}
\vector{f} \quad + \quad \frac{\partial \vector{f}}{\partial \dot{\vector{y}}} \Delta \dot{\vector{y}} \quad + \quad \frac{\partial \vector{f}}{\partial \vector{y}} \Delta \vector{y} \quad + \quad \frac{\partial \vector{f}}{\partial \vector{u}} \Delta \vector{u} \quad + \quad \cdots \quad = \quad \grkvec{\epsilon} \quad
\end{equation}
At equilibrium (trim), $\vector{f}$ = $\grkvec{\epsilon}$ $\stackrel{\text{def}}{=}$ $\vector{0}$. Let the Jacobian matrices be
\[ \textbf{E} \quad = \quad \frac{\partial \grkvec{\epsilon}}{\partial \dot{\vector{y}}}\biggr\rvert_\textrm{trim} \qquad \textrm{;} \qquad \textbf{F} \quad = \quad \frac{\partial \grkvec{\epsilon}}{\partial \vector{y}}\biggr\rvert_\textrm{trim} \qquad \textrm{;} \qquad \textbf{G} \quad = \quad \frac{\partial \grkvec{\epsilon}}{\partial \vector{u}}\biggr\rvert_\textrm{trim} \]
 Neglecting the higher-order terms, we obtain the linearized system dynamics about equilibrium, given by
\begin{equation}
\textbf{E} \textrm{ }\Delta \dot{\vector{y}} \quad + \quad \textbf{F} \textrm{ }\Delta \vector{y} \quad + \quad \textbf{G} \textrm{ }\Delta \vector{u} \quad = \quad \vector{0}
\end{equation}
Rearranging the equations and isolating $\Delta \dot{\vector{y}}$, we obtain
\begin{align}
\label{eqn:lindyn}
\Delta \dot{\vector{y}} \quad = \quad &\textbf{A} \textrm{ }\Delta \vector{y} \quad + \textbf{B} \textrm{ }\Delta \vector{u} \\
\textrm{Where} \qquad \qquad \qquad & \qquad \qquad \qquad \qquad \qquad \qquad \notag \\
\textbf{A} \quad = \quad &-\textbf{E}^{-1} \textbf{F} \notag \\
\textbf{B} \quad = \quad &-\textbf{E}^{-1} \textbf{G} \notag
\end{align}
The linearization process is performed by the routine \textbf{linrzODEs}, which calls \textbf{ODEResiduals} repeatedly. Using Eq. (\ref{eqn:lindyn}) and an appropriate state-to-output conversion matrix \textbf{C}, transfer functions between \textit{pilot inputs} and system outputs for the relevant physical quantities can be constructed as
\begin{equation}
\textbf{H}(s) \quad = \quad \textbf{C} \textrm{ }\left(s \textrm{ }\textbf{I} \textrm{ }- \textrm{ }\textbf{A}_\textrm{NR}\right)^{-1} \spc\textbf{B}_\textrm{NR} \spc + \spc \textbf{D}
\end{equation}
The transfer function magnitude and phase are obtained from the system \textbf{A} and \textbf{B} matrices using a wrapper routine \textbf{freq\_response\_wrapper} for a NETLIB program.

\subsection{Multiblade Coordinate Transform}
Before the system frequency response can be constructed, it is necessary to obtain the system \textbf{A}  and \textbf{B} matrices in the fixed frame. However, the \textbf{A} and \textbf{B} matrices obtained from the application of finite-difference perturbations on \textbf{ODEResiduals} are azimuth dependent, since the rotor states are defined in the rotating preconed undeformed axes. Therefore, an additional coordinate transform is required to convert the resulting \textbf{A} and \textbf{B} matrices to the fixed frame.

Let the matrix \textbf{P} represent the time-varying coordinate transform that converts the system states from the fixed frame to the rotating frame, and let \textbf{y$_\textrm{NR}$} denote the system state vector in the fixed frame. By definition,
\[ \textbf{y}(t) \quad = \quad \textbf{P}(t) \spc \textbf{y}_\textrm{NR}(t) \]

Substituting the above expression in the linearized governing equations
\begin{align*}
\vector{P} \spc \Delta \dot{\vector{y}}_\textrm{NR} \spc + \spc \dot{\vector{P}} \spc \Delta \vector{y}_\textrm{NR} \quad = \quad & \quad \textbf{A} \spc \vector{P} \spc & \Delta \vector{y}_\textrm{NR} \quad + \quad & \textbf{B} \spc\Delta \vector{u} \\
\Rightarrow \vector{P} \spc \Delta \dot{\vector{y}}_\textrm{NR} \quad = \quad & \left( \textbf{A} \spc \vector{P} - \dot{\vector{P}} \right) &\Delta \vector{y}_\textrm{NR} \quad + \quad & \textbf{B} \spc\Delta \vector{u} \\
\Rightarrow \Delta \dot{\vector{y}}_\textrm{NR} \quad = \quad & \vector{P}^{-1} \left( \textbf{A} \spc \vector{P} - \dot{\vector{P}} \right) &\Delta \vector{y}_\textrm{NR} \quad + \quad & \vector{P}^{-1} \spc \textbf{B} \spc\Delta \vector{u}
\end{align*}
This equation is of the form
\[ \Delta \dot{\vector{y}}_\textrm{NR} \quad = \quad \vector{A}_\textrm{NR} \Delta \vector{y}_\textrm{NR} \quad + \quad \textbf{B}_\textrm{NR} \spc\Delta \vector{u} \]
Where
\begin{align}
\vector{A}_\textrm{NR} \quad = \quad & \vector{P}^{-1} \left( \vector{A} \spc \vector{P} - \dot{\vector{P}} \right) \\
\vector{B}_\textrm{NR} \quad = \quad & \vector{P}^{-1} \spc \vector{B}
\end{align}

The system \textbf{A} and \textbf{B} matrices are obtained at multiple azimuths, converted to the fixed frame and averaged to obtain an engineering representation of a Linear Time-Invariant (\textbf{LTI}) system. The fixed-frame conversion at each azimuth is performed by a modified version of the Heli-UM routine \textbf{linea2}.

\subsubsection*{The Multiblade Coordinate Transformation}
The system state vector can be partitioned into two sets of vectors: those states defined in the fixed frame, \textbf{y}$_\textrm{NR}$ and the rotor states defined in the rotating frame \textbf{y}$_\textrm{R}$. The rotation matrix that converts the \textit{entire} system state vector \textbf{y} to its fixed-frame counterpart is \textbf{P}, given by
\[ \vector{P} = \begin{bmatrix}
\vector{P}_\textrm{NR} & \vector{0} \\
\vector{0} & \vector{P}_\textrm{R} \end{bmatrix} \]
It is assumed that the rotor states constitute the last partition in the state vector. The partition \textbf{P}$_\textrm{NR}$ is an identity matrix whose size is equal to the number of states defined in the non-rotating frame. \textbf{P}$_\textrm{R}$ is the multi-blade transformation matrix. Before writing out an expansion for \textbf{P}$_\textrm{R}$, we recall the partition of state vector corresponding to the rotor modes is given by
\[ \vector{y}_\textrm{rotor} \quad = \quad \renewcommand\arraystretch{0.5}\begin{Bmatrix} \spc \dot{\grkvec{\eta}}_1\tr \quad \grkvec{\eta}_{1}\tr \qquad \dot{\grkvec{\eta}}_2\tr \quad \grkvec{\eta}_{2}\tr \quad \cdots \quad \dot{\grkvec{\eta}}_\textrm{Nm}\tr \quad \grkvec{\eta}_\textrm{Nm}\tr \spc  \end{Bmatrix}\tr \]
The vector of generalized displacements for the `` $j^\textrm{th}$ '' mode is given by
\[ \grkvec{\eta}_j \quad = \quad \renewcommand\arraystretch{0.5}\begin{Bmatrix} \spc  \eta_{j,1} \qquad \eta_{j,2} \quad \cdots \quad \eta_{j,\textrm{Nb}} \spc  \end{Bmatrix}\tr \]
$\eta_{j,i}$ represents the `` $j^\textrm{th}$ '' generalized displacement of blade `` $i$ ''. These generalized displacements are the coefficients of the normal modes corresponding to the rotating beam structure of the blade. This particular arrangement of the rotor states is useful, since it allows us to construct the partition of the fixed-frame transformation \textbf{P}$_\textrm{R}$ as repeating diagonal blocks, given by
\[ \textbf{P}_\textrm{R} \quad = \quad \begin{bmatrix}
\textbf{P}_\textrm{mode} & \textbf{0} & \cdots & \textbf{0} & \textbf{0} \\
\textbf{0} & \textbf{P}_\textrm{mode}  & \cdots & \textbf{0} & \textbf{0} \\
\vdots & \vdots & \ddots & \vdots & \vdots \\
\textbf{0}& \textbf{0} & \cdots  & \textbf{P}_\textrm{mode}  & \textbf{0} \\
\textbf{0} & \textbf{0} & \cdots & \textbf{0} & \textbf{P}_\textrm{mode}
\end{bmatrix} \]
The diagonal block \textbf{P}$_\textrm{mode}$ repeats N$_\textrm{m}$ times, where N$_\textrm{m}$ is the number of blade modes. All that remains is to obtain an expression for the matrix \textbf{P}$_\textrm{mode}$, which is a square matrix of size $2\textrm{N}_\textrm{b} \times 2\textrm{N}_\textrm{b}$.

Consider a mode $\eta$ of a blade contained in a rotor system. $\eta(t)$ is defined in the rotating frame, and the frame is unique to each blade. This mode can be expressed as a Fourier series expansion in azimuth as
\[\eta_i(t) \quad = \quad \eta_0(t) \spc + \spc \sum_{k=1}^{N} \left[ \spc \eta_{kc} \spc \cos (k \psi_i) \quad + \quad \eta_{ks} \spc \sin (k \psi_i) \right] \]
For an even bladed rotor, an additional term exists, corresponding to the differential mode, and the mode is
\[\eta_i(t) \quad = \quad \eta_0(t) \spc + \spc \sum_{k=1}^{N} \left[ \spc \eta_{kc} \spc \cos (k \psi_i) \quad + \quad \eta_{ks} \spc \sin (k \psi_i) \spc \right] + \spc \eta_d (-1)^{i-1} \]
$\eta_0$, $\eta_{kc}$, $\eta_{ks}$ (and $\eta_d$ for even-bladed rotors) are the fixed-frame equivalents of the blade mode $\eta(t)$ for all blades. These transformations from fixed to rotating frame (or vice versa) are exact, hence there are as many fixed-frame coefficients as there are blades. In matrix-vector form, we have
\begin{equation}
\grkvec\eta \quad = \quad \vector{T} \spc \grkvec\eta_\textrm{F}
\end{equation}
Differentiate once with respect to time to obtain
\[ \dot{\grkvec\eta} \quad = \quad \dot{\vector{T}} \spc \grkvec\eta_\textrm{F} \spc + \spc \vector{T} \spc \dot{\grkvec\eta}_\textrm{F} \]
In matrix-vector product form,
\begin{equation*}
\begin{Bmatrix} \dot{\grkvec\eta} \\ \grkvec\eta \end{Bmatrix} \quad = \quad \begin{bmatrix}
\vector{T} & \dot{\vector{T}} \\
\vector{0} & \vector{T}
\end{bmatrix} \begin{Bmatrix} \dot{\grkvec\eta}_\textrm{F} \\ \grkvec\eta_\textrm{F} \end{Bmatrix}
\end{equation*}
Where
\begin{equation}
\grkvec\eta_\textrm{F} \quad = \quad \begin{Bmatrix} \eta_0 \\ \eta_\textrm{1c} \\ \eta_\textrm{1s} \\ \vdots \\ \eta_d \end{Bmatrix}
\end{equation}
\begin{equation}
\grkvec\eta \quad = \quad \begin{Bmatrix} \eta_1 \\ \eta_2 \\ \vdots \\ \eta_{\textrm{N}_b}\end{Bmatrix}
\end{equation}
For an even bladed rotor, the transformation between rotating blade modes and their fixed-frame counterparts is
\begin{equation}
\vector{T}_\textrm{even} \quad = \quad \begin{bmatrix}
1 & \cos \psi_1 & \sin \psi_1 & \cdots & -1 \\
1 & \cos \psi_2 & \sin \psi_2 & \cdots &  1 \\
\vdots & \vdots & \vdots & \vdots & \vdots \\
1 & \cos \psi_{\textrm{N}_b} & \sin \psi_{\textrm{N}_b} & \cdots & 1
\end{bmatrix}
\end{equation}
For an odd-bladed rotor, the corresponding transformation between rotating blade modes and their fixed-frame counterparts is
\begin{equation}
\vector{T}_\textrm{odd} \quad = \quad \begin{bmatrix}
1 & \cos \psi_1 & \sin \psi_1 & \cdots \\
1 & \cos \psi_2 & \sin \psi_2 & \cdots \\
\vdots & \vdots & \vdots & \vdots \\
1 & \cos \psi_{\textrm{N}_b} & \sin \psi_{\textrm{N}_b} & \cdots
\end{bmatrix}
\end{equation}
\subsection{LQR and the Riccati Equation}
Feedback control inputs based on linearized models are used in the present analysis to track prescribed vehicle motions and obtain the control inputs required to fly a certain trajectory. To obtain the feedback gains \textbf{K} from the linearized dynamics, the Linear Quadratic Regulator (LQR)  (Ref. \cite{Kirk}) is used, which provides a methodology to stabilize and control a linear system by minimizing a designer-weighted quadratic cost functional in the state deviations from targets and the control inputs. For an LTI system with dynamics given by Eq. (\ref{eqn:lindyn}), the infinite-horizon continuous-time LQR controller yields state feedback gains \textbf{K} to minimize the quadratic cost functional
\begin{equation}
\label{eqn:quadcost}
J \quad = \quad \int_{0}^{\infty} \left(\textrm{ } \vector{x}^T \textrm{ } \textbf{Q} \textrm{ } \vector{x} \textrm{ } + \textrm{ } \Delta \vector{u}^T \textrm{ } \textbf{R} \textrm{ } \Delta \vector{u}\textrm{ } \right) dt
\end{equation}
Where
\[ \vector{x} \quad = \quad ( \textrm{ } \vector{y} \textrm{ } - \textrm{ } \vector{y}_\textrm{target} \textrm{ } ) \]
The state feedback controls are given by
\begin{equation*}
\Delta \vector{u} \quad = -\textbf{K} \textrm{ } \vector{x}
\end{equation*}
The feedback gains are obtained from
\begin{equation}
\label{eqn:LQRfeedback}
\textbf{K} \quad = \quad \textbf{R}^{-1} \textbf{B}^T \textbf{P}
\end{equation}
\textbf{P} is the unique positive definite steady-state solution of the continuous-time Riccati Equation
\begin{equation}
\label{eqn:Riccati}
\frac{d \textbf{P}}{dt} \textrm{ } + \textrm{ } \textbf{A}^T \textbf{P} \textrm{ } + \textrm{ } \textbf{P} \textbf{A} \textrm{ } - \textrm{ } \textbf{P} \textbf{B}\textbf{R}^{-1}\textbf{B}^T \textbf{P} \textrm{ } + \textrm{ } \textbf{Q} \quad = \quad \textbf{0}
\end{equation}
At steady-state, the time derivatives vanish, and the stabilizing solution satisfies the algebraic Riccati equation
\begin{equation}
\label{eqn:SSR1}
\textbf{A}^T \textbf{P} \textrm{ }+\textrm{ } \textbf{P} \textbf{A} \textrm{ }-\textrm{ } \textbf{P} \textbf{B} \textbf{R}^{-1} \textbf{B}^T \textbf{P} \textrm{ }+ \textrm{ }\textbf{Q} \quad = \quad \textbf{0}
\end{equation}
Taking transpose on both sides
\begin{equation*}
\textbf{A}^T \textbf{P}^T \textrm{ }+ \textrm{ }\textbf{P}^T \textbf{A} \textrm{ }- \textrm{ }\textbf{P}^T \textbf{B}{\textbf{R}^{-1}}^T\textbf{B}^T \textbf{P}^T \textrm{ }+ \textrm{ }\textbf{Q}^T \quad = \quad \textbf{0}
\end{equation*}
The vehicle and load rigid-body states are assigned non-zero weights, and the control inputs are penalized with a unit weight. All other states and off-diagonal weights (entries of \textbf{Q},\textbf{R}) are assigned to zero. The diagonal form of the weighting matrices for controls and states allows for further simplification of the solution procedure to obtain the feedback gains \textbf{K} from the Riccati equation. The steady-state equation simplifies to
\begin{equation}
\label{eqn:SSR2}
\textbf{A}^T \textbf{P}^T \textrm{ }+ \textrm{ }\textbf{P}^T \textbf{A} \textrm{ }- \textrm{ }\textbf{P}^T \textbf{B}{\textbf{R}^{-1}}\textbf{B}^T \textbf{P}^T \textrm{ }+ \textrm{ }\textbf{Q} \quad =\quad \textbf{0}
\end{equation}
Eq. (\ref{eqn:SSR2}) is very similar to Eq. (\ref{eqn:SSR1}), with \textbf{P} replaced by $\textbf{P}^T$. Thus, if \textbf{P} is a stabilizing solution, \textbf{P}$^T$ is also a stabilizing solution and if a unique stabilizing solution exists, the matrix \textbf{P} must be symmetric. For a matrix of size \emph{n}$\times$\emph{n}, the number of elements of \textbf{P} to find are ${\frac{n \left(n+1\right)}{2}}$. Thus, Eq. (\ref{eqn:Riccati}) can be integrated numerically from an initial condition towards the stabilizing solution by exploiting symmetry and updating the upper or lower-triangular elements of \textbf{P}.

\subsection{Practical Considerations}
\begin{itemize}
\item \textbf{Controllability} \\
A few candidate flight conditions (straight and turning flight at various speeds) were considered to determine whether the system is controllable, which is a necessary condition for a stabilizing solution of the Riccati equation to exist. In all cases, the Grammian was found to be full-rank.
\item \textbf{Solving the Riccati Equation} \\
A Runge-Kutta (fourth-order) scheme is used to advance the Riccati equation forward in time starting from \textbf{0}. For computational efficiency, the time step is increased as the infinity-norm of $\dot{\textbf{P}}$ decreases, and marching is terminated when it falls below a threshold value ($10^{-8}$)
\item \textbf{Computation efficiency} \\
Additional time savings are obtained by marching the Riccati equation forward from the previous steady-state solution instead of the original initial condition (\textbf{0}). In case the Riccati equation does not converge to a steady-state solution within a prefixed number of iterations (30000 in this case), the feedback gains from the previous update are used and the initial condition is reset to \textbf{0}.
\item \textbf{Gain Scheduling and Control Smoothness} \\
It is possible, but practically cumbersome, to generate feedback gain matrices for a combination of speeds, climb angles, turn rates and altitudes. Every additional parameter (e.g. fin pitch settings) increases the number of potential pre-computations exponentially. Instead, a dynamic update of the system \textbf{A} and \textbf{B} matrices is performed every 30 revolutions following a re-trim based on the current flight condition and altitude, and the feedback gains \textbf{K} are smoothly transitioned from the previous set to the current one over a few rotor revolutions to avoid abrupt changes in the control inputs.
\end{itemize}

\begin{Figure}
 \centering
 \includegraphics[width=1.3\textwidth, angle=90]{images/feedback_gains_callgraph.png}
 \vspace{-0.5cm}
 \captionof{figure}{Abbreviated call graph to perform linearized analysis}
 \label{fig:cg}
\end{Figure}


\documentclass[../../main.tex]{subfiles}
\begin{document}

\subsubsection{Map}
The $map$ operation takes in a function $fn?$, Collection $coll?$ and additional Arguments $args?$ (as necessary)
and returns a modified Collection $coll!$ with members $fn!_{n}$. The ordering of $coll?$ is maintained within $coll!$
\begin{schema}{Map[(\_~\pfun~\_), Collection, V]}
  fn? : (\_~\pfun~\_) \\
  args? : V \\
  coll?, coll! : Collection \\
  map~\_ : (\_~\pfun~\_) \cross Collection \cross V \surj Collection
  \where
  coll! = map(fn?, coll?, args?) @ \\
  \t3 \langle ~\forall n : i~..~j \in coll? ~|~ i \leq n \leq j ~\land ~j = ~\# ~coll? @ \\
  \t4 \exists_1 ~fn!_{n} : V ~|~ fn!_{n} = \\
  \t5 (fn?(coll?_{n}, args?) \iff args? \not = \emptyset) ~\lor \\
  \t5 (fn?(coll?_{n}) \iff args? = \emptyset) \rangle \implies fn!_{i} \cat fn!_{n} \cat fn!_{j} \\
\end{schema}
Above, $fn!_{n}$ is introduced to handle the case where $fn?$ only requires a single argument.
Additional arguments may be necessary but if they are not ($args? = \emptyset$) then only $coll?_{n}$ is passed to $fn?$.
\begin{argue}
  X = \langle 1, 2, 3 \rangle \\
  \t1 map~(succ, X) = \langle 2, 3, 4 \rangle & increment each member of $X$ \\
  \t1 map~(+, X, 2) = \langle 3, 4, 5 \rangle & add 2 to each member of $X$
\end{argue}

\end{document}


\subsection{Complete metric spaces}\label{subsec:metric_convergence}

\begin{definition}\label{def:complete_metric_space}
  A metric space is said to be \term{complete} if
  \begin{thmenum}
    \thmitem{def:complete_metric_space/sequences} Every fundamental sequence converges.
    \thmitem{def:complete_metric_space/uniform} It is complete as a uniform space in the sense of \fullref{def:complete_uniform_space}
  \end{thmenum}
\end{definition}
\begin{proof}
  The equivalence is due to \fullref{thm:def:metric_topology/sequential} and \fullref{thm:def:metric_topology/hausdorff}.
\end{proof}

\begin{proposition}\label{thm:fundamental_sequence_is_bounded}
  In a metric space, any \hyperref[def:fundamental_net]{fundamental sequence} \( \{ x_k \}_{i=1}^n \) is \hyperref[def:metric_space/bounded_sequence]{bounded}.
\end{proposition}
\begin{proof}
  Since the set
  \begin{equation*}
    I \coloneqq \{ x_k \colon k \leq k_0 \}
  \end{equation*}
  is finite, it has a finite \hyperref[def:metric_space/diameter]{diameter}.

  Fix \( \varepsilon > 0 \). Since the sequence is fundamental, there exists an index \( k_0 \) such that
  \begin{equation*}
    \rho(x_k, x_m) < \varepsilon \quad\forall k, m \geq k_0.
  \end{equation*}

  We are only interested in the case \( \rho(x_{k_0}, x_m) < \varepsilon \).

  Let \( k < k_0 \) and \( m \geq k_0 \). Then
  \begin{equation*}
    \rho(x_k, x_m) \leq \rho(x_k, x_{k_0}) + \rho(x_{k_0}, x_m) < \diam(I) + \varepsilon,
  \end{equation*}
  which is a finite number.

  Thus, the distance between any two elements of the sequence is finite and the sequence is bounded.
\end{proof}

\begin{proposition}\label{thm:fundamental_subsequence_convergence}
  In any \hyperref[def:complete_metric_space]{metric space}, a \hyperref[def:fundamental_net]{fundamental sequence} converges to a value if and only if it has a subsequence that converges to the same value.
\end{proposition}
\begin{proof}
  Let \( (X, \rho) \) be a metric space and let \( \{ x_k \}_{k=1}^\infty \) be a fundamental sequence.

  \SufficiencySubProof Obvious
  \NecessitySubProof Assume that the subsequence \( \{ x_{k_n} \}_{n=1}^\infty \) converges to \( x \). Fix \( \varepsilon > 0 \). There exist \( k_0 \) and \( n_0 \) such that
  \begin{balign*}
     & \rho(x_k, x_m) < \tfrac \varepsilon 2 \quad\forall k, m \geq k_0
     & \rho(x, x_{k_n}) < \tfrac \varepsilon 2 \quad\forall n \geq n_0.
  \end{balign*}

  Fix \( k \geq k_0 \) and let \( n \geq n_0 \) be such that \( k_n \geq k_0 \). Then
  \begin{equation*}
    \rho(x, x_k) \leq \rho(x, x_{k_n}) + \rho(x_{k_n}, x_k) < \varepsilon.
  \end{equation*}

  Since \( \varepsilon \) was arbitrary, we conclude that \( \lim_{k \to \infty} x_k = \lim_{n \to \infty} x_{k_n} = x \).
\end{proof}

\begin{lemma}\label{thm:metric_space_completion_uniqueness}
  Let \( X \) be a metric space. If both \( f: X \to Y \) and \( g: X \to Z \) are \hyperref[def:complete_metric_space]{completions} of \( X \), then \( Y \) and \( Z \) are isometric.
\end{lemma}
\begin{proof}
  Let \( y \in Y \) and let \( \{ x_k \}_{k \to \infty} \subseteq X \) be a sequence such that
  \begin{equation*}
    f(x_k) \xrightarrow[k \to \infty]{} y.
  \end{equation*}

  Such a sequence exists since \( f(X) \) is dense in \( Y \).

  Define \( z \coloneqq \lim_{k \to \infty} g(x_k) \). Since both \( f \) and \( g \) are isometries, \( z \) does not depend on the choice of sequence \( \{ x_k \}_{k \to \infty} \) such that \( f(x_k) \to y \). Furthermore, if \( z \in Z \) is given rather than \( y \in Y \), an analogous process allows us to determine \( y \) uniquely based on \( z \).

  Thus, we have a bijective isometry between \( Y \) and \( Z \).
\end{proof}

\begin{theorem}[Metric space completion]\label{thm:metric_space_completion}
  Every metric space has a unique (up to an isometry) \hyperref[def:complete_metric_space]{completion}.

  This is a special case of \fullref{thm:uniform_space_completion} that we prove fully.
\end{theorem}
\begin{proof}
  Let \( (X, \rho) \) be a metric space. Uniqueness of the completion follows from \fullref{thm:metric_space_completion_uniqueness}. We will only show existence.

  \begin{thmenum}
    \thmitem{thm:metric_space_completion/part_a} First, we build the pseudometric space \( (F, \rho) \). We deal with fundamental sequences and isometries in pseudometric spaces, where the definitions, however, does not change.

    Define \( F \) to be the set of all fundamental \hyperref[def:fundamental_net]{sequences} in X. Define the pseudometric
    \begin{balign*}
       & \rho: F \times F \to \BbbR_{\geq 0}                                                                               \\
       & \rho\left( \{ x_k \}_{k=1}^\infty, \{ y_k \}_{k=1}^\infty \right) \coloneqq \lim_{k \to \infty} \rho(x_k, y_k).
    \end{balign*}

    We first show that is well-defined as a \hyperref[def:function]{function}. Let \( \{ x_k \}_{k=1}^\infty \) and \( \{ y_k \}_{k=1}^\infty \) be two sequences. Fix \( \varepsilon > 0 \). Then there exists an \( k_0 \) such that
    \begin{equation*}
      \rho(x_k, x_k) < \tfrac \varepsilon 2 \text{ and } \rho(y_k, y_m) <  \quad\forall k, m \geq k_0.tfrac \varepsilon 2.
    \end{equation*}

    Fix \( k, m \geq k_0 \). Then
    \begin{equation*}
      \rho(x_k, y_k) \leq \rho(x_k, x_m) + \rho(x_m, y_m) + \rho(y_m, y_k) < \rho(x_m, y_m) + \varepsilon,
    \end{equation*}
    hence
    \begin{equation*}
      \abs{\rho(x_k, y_k) - \rho(x_m, y_m)} < \varepsilon.
    \end{equation*}

    Thus, the sequence \( \{ \rho(x_k, y_k) \}_{k=1}^\infty \) is fundamental and, by \fullref{def:set_of_real_numbers_complete_metric_space}, it is convergent.

    Now we check that \( \rho \) is indeed a pseudometric:
    \SubProofOf{def:metric_space/pseudometric_identity} For every sequence \( x \in F \),
    \begin{equation*}
      \rho(x, x) = \lim_{k \to \infty} \rho(x_k, x_k) = 0.
    \end{equation*}
    \SubProofOf{def:metric_space/M2} For all sequences \( x, y \in F \),
    \begin{equation*}
      \rho(x, y) = \lim_{k \to \infty} \rho(x_k, y_k) = \lim_{k \to \infty} \rho(y_k, x_k) = \rho(y, x).
    \end{equation*}

    \SubProofOf{def:metric_space/M3} For all sequences \( x, y, z \in F \),
    \begin{equation*}
      \rho(x, z) = \lim_{k \to \infty} \rho(x_k, z_i) \leq \lim_{k \to \infty} \rho(x_k, y_k) + \lim_{k \to \infty} \rho(y_k, z_i) = \rho(x, y) + \rho(y, z).
    \end{equation*}

    \thmitem{thm:metric_space_completion/part_b} We prove that every fundamental sequence in \( (F, \rho) \) is convergent.

    Let \( \{ c^{(k)} \}_{k=1}^\infty \) be a fundamental sequence (of sequences) in \( (F, \rho) \). Thus, for every \( k = 1, 2, \ldots \), there exists an index \( n_k \) such that
    \begin{equation*}
      \rho(c_m^{(k)}, c_{n_k}^{(k)}) < \tfrac 1 k \quad\forall m \geq n_k.
    \end{equation*}

    Define the sequence
    \begin{equation*}
      d_k \coloneqq c_{n_k}^{(k)}, k = 1, 2, \ldots
    \end{equation*}

    To see that it is fundamental, fix \( \varepsilon > 0 \). Now since the sequence \( \{ c^{(k)} \} \) in \( F \) is fundamental, there exists \( k_0 \) such that
    \begin{equation*}
      \rho(c^{(k)}, c^{(m)}) = \lim_{k \to \infty} \rho(c_k^{(k)}, c_k^{(m)}) < \frac \varepsilon 2 \quad\forall k, m \geq k_0.
    \end{equation*}

    Let \( m_0 \geq k_0 \) be an index such that
    \begin{equation*}
      \frac 2 {m_0} < \frac \varepsilon 2.
    \end{equation*}

    Fix \( k \geq m \geq m_0 \). Let \( l \geq \max \{ n_k, n_m \} \) be such that
    \begin{equation*}
      \rho(c_l^{(k)}, c_l^{(m)}) < \frac \varepsilon 2.
    \end{equation*}

    Then
    \begin{balign*}
      \rho(d_k, d_m)
       & =
      \rho(c_{n_k}^{(k)}, c_{n_m}^{(m)})
      \leq \\ &\leq
      \rho(c_{n_k}^{(k)}, c_l^{(k)}) + \rho(c_l^{(k)}, c_l^{(m)}) + \rho(c_l^{(m)}, c_{n_m}^{(m)})
      \leq \\ &\leq
      \frac 1 k + \frac \varepsilon 2 + \frac 1 m
      \leq
      \frac 2 m + \frac \varepsilon 2
      <
      \varepsilon.
    \end{balign*}

    Thus, we have
    \begin{equation*}
      \rho(d_k, d_m) < \varepsilon \quad\forall k \geq m \geq m_0,
    \end{equation*}
    which proves that the sequence \( \{ d_k \}_{k=1}^\infty \) is fundamental in \( (X, \rho) \).

    Now it remains to show that \( c^{(k)} \xrightarrow[k \to \infty]{} d \) in \( (F, \rho) \).

    Fix \( \varepsilon > 0 \) and let \( k_0 \) be such that
    \begin{equation*}
      \frac 1 {k_0} \leq \frac \varepsilon 2.
    \end{equation*}
    and
    \begin{equation*}
      \rho(d_k, d_m) < \frac \varepsilon 2 \quad\forall k, m \geq k_0.
    \end{equation*}

    Now fix \( i \geq k_0 \). We have, for all \( k \geq i \),
    \begin{balign*}
      \rho(c_{n_k}^{(k)}, d_k)
       & =
      \rho(c_{n_k}^{(k)}, c_{n_k}^{(k)})
      \leq \\ &\leq
      \rho(c_{n_k}^{(k)}, c_{n_k}^{(k)}) + \rho(c_{n_k}^{(k)}, c_{n_k}^{(k)})
      =    \\ &=
      \rho(c_{n_k}^{(k)}, c_{n_k}^{(k)}) + \rho(d_k, d_k)
      <    \\ &<
      \frac 1 k + \frac \varepsilon 2
      <
      \varepsilon.
    \end{balign*}

    Hence,
    \begin{balign*}
      \rho(c^{(k)}, d)
      =
      \lim_{k \to \infty} \rho(c_k^{(k)}, d_k)
      =
      \lim_{k \to \infty} \rho(c_k^{(k)}, c_{n_k}^{(k)})
      <
      \varepsilon.
    \end{balign*}

    Thus, given \( \varepsilon > 0 \), we found an index \( k_0 \) such that
    \begin{equation*}
      \rho(c^{(k)}, d) < \varepsilon \quad\forall g \geq k_0.
    \end{equation*}

    Thus, \( d = \lim_{k \to \infty} c^{(k)} \) and \( (F, \rho) \) is a complete pseudometric space.

    \thmitem{thm:metric_space_completion/part_c} We construct an isometry of \( (X, \rho) \) into \( (F, \rho) \).

    Define the function
    \begin{balign*}
       & \iota: X \to F                        \\
       & \iota(x) \coloneqq (x, x, x, \ldots),
    \end{balign*}
    which sends each element of \( X \) into the corresponding constant sequence in \( F \).

    It is an \hyperref[def:isometry]{isometry} since
    \begin{equation*}
      \rho(\iota(x),\iota(y)) = \lim_{k \to \infty} \rho(x, y) = \rho(x, y).
    \end{equation*}

    \thmitem{thm:metric_space_completion/part_d} We show that the image \( \iota(X) \) is dense in \( (F, \rho) \).

    Fix the fundamental sequence \( y \coloneqq \{ y_k \}_{k=1}^\infty \). Define the sequence \( x \) of sequences
    \begin{equation*}
      x^{(k)} \coloneqq \iota(y_k), i = 1, 2, \ldots
    \end{equation*}

    It is fundamental in \( (F, \rho) \) since \( e \) is an isometry and since \( y \) is fundamental in \( (X, \rho) \).

    Fix \( \varepsilon > 0 \). Let \( k_0 \) be such that
    \begin{equation*}
      \rho(y_k, y_m) < \varepsilon \quad\forall k, m \geq k_0.
    \end{equation*}

    For \( i, k \geq k_0 \), we have
    \begin{balign*}
      \rho(x_k^{(k)}, y_k)
      \leq
      \rho(x_k^{(k)}, y_k) + \rho(y_k, y_k)
      =
      0 + \rho(y_k, y_k)
      <
      \varepsilon,
    \end{balign*}
    hence
    \begin{equation*}
      \rho(x^{(k)}, y) = \lim_{k \to \infty} \rho(x_k^{(k)}, y_k) < \varepsilon.
    \end{equation*}

    We conclude that \( x^{(k)} \xrightarrow[k \to \infty]{} y \) in \( (F, \rho) \), which implies that \( e(X) \) is dense in \( (F, \rho) \).

    \thmitem{thm:metric_space_completion/part_e} We build a complete metric space \( (C, \nu) \) from \( (F, \rho) \).

    We use \fullref{thm:pseudometric_to_metric} to construct a complete metric space \( (C, \nu) \) from the complete pseudometric space \( (F, \rho) \).

    We adapt \( \iota \) to the equivalence classes on \( C \):
    \begin{balign*}
       & \hat\iota: X \to C                 \\
       & \hat\iota(x) \coloneqq [\iota(x)].
    \end{balign*}

    Thus, \( \hat\iota \) embeds \( X \) into the complete metric space \( C \).
  \end{thmenum}
\end{proof}

\begin{theorem}[Cantor's nested compact theorem]\label{thm:cantors_nested_compact_theorem}
  A descending sequence of nonempty compact sets \( F_1 \supseteq F_2 \supseteq \ldots \) in a complete metric space such that \( \diam(F_i) \to 0 \) intersects at exactly one point (compare with \fullref{thm:noncompact_kuratowskis_lemma}).
\end{theorem}
\begin{proof}
  Choose an element \( x_k \in F_k \) for any \( i = 1, 2, \ldots \). Then the sequence \( \{ x_k \}_{k=1}^\infty \) is fundamental. To see this, let \( \varepsilon > 0 \) and let \( k_0 \) be an index such that \( \diam(F_{k_0}) < \varepsilon \). Then if \( j \geq i \geq k_0 \), \( x_m \) is contained in \( F_k \) and \( \rho(x_k, x_m) < \varepsilon \). Thus, the sequence is indeed fundamental and, since the space is complete, it has a limit point \( x \).

  The point \( x \) is contained in every set \( F_k, i = 1, 2, \ldots \) since all of the sets \( F_k \) are closed (by \fullref{thm:complete_metric_space_compact_conditions}) and contain their limit \hyperref[thm:limit_point_iff_in_closure]{points}. Thus,
  \begin{equation*}
    x \in \bigcap_{k=1}^\infty F_k.
  \end{equation*}

  Furthermore,
  \begin{equation*}
    \diam\left( \bigcap_{k=1}^\infty F_k \right) = 0,
  \end{equation*}
  hence \( x \) is the only point in the intersection.
\end{proof}


\subsection{Integer rings}

The integers with addition and multiplication form a ring.


\section{Ordinal Numbers}

\subsection{Well Ordering}
\begin{definition}
    A binary relation $<$ is a \cindex{partial ordering} of $P$ if :
    \begin{enumerate}
        \item $\forall p \in P (p \nless p)$.
        \item $p < q \wedge q < r \rightarrow p < r$.
    \end{enumerate}
\end{definition}

\begin{definition}
    A partial order $(P, <)$ is \cindex{linear ordering} if $\forall p \forall q (p < q \wedge p = q \wedge q < p )$.
\end{definition}

\begin{definition}
    $\alpha$ is the \cindex{supremum} of $X$ if $\alpha$ is the \cindex{least upper bound} of $X$: $\alpha = \supremum{X}$.
\end{definition}

\begin{definition}
    $\alpha$ is the \cindex{infimum} of $X$ if $\alpha$ is the \cindex{greatest lower bound} of $X$: $\alpha = \infimum {X}$.
\end{definition}

\begin{definition}
    If $(P, <)$ and $(Q,<)$ are partially ordered sets and $f: P \rightarrow Q$, then $f$ is \cindex{order-preserving} if $x < y \rightarrow f(x) < f(y)$. If $P$ and $Q$ are linearly ordered, $f$ is called \cindex{increasing}.
\end{definition}

\begin{definition}
    $f: P \rightarrow Q$  is \cindex{isomorphism} of $P$ and $Q$ if $f$ and $f^{-1}$ are order-preserving. An isomorphism of $P$ onto itself is \cindex{automorphism}.
\end{definition}

\begin{definition}
    A linear ordering $<$ is \cindex{well-ordering} if every nonempty subset of $P$ has a least element.
\end{definition}

\begin{theorem}\label{wellorderedsetisomorphismisbigger}
    If $(W,<)$ is a well-ordered set and $f:W \rightarrow W$ is an increasing function, then $\forall x\in W \left( f(x) \geq x \right)$.
\end{theorem}
\begin{proof}
    If the set $X = \set{x \in W: f(x) < x}$ is nonempty, let $z$ be its least element and $w = f(z)$. Then $f(w) = ff(z) < f(z) = w $. So $f(w) < w \rightarrow w \in X \wedge w < z$.
\end{proof}

\begin{theorem}
    The only automorphism of a well-ordered set is the identity.
\end{theorem}
\begin{proof}
    $f(x) \geq x$ and $f^{-1} \geq x$.
\end{proof}

\begin{theorem}
    If two well-ordered set $W_1$ and $W_2$ are isomorphic, then the isomorphism is unique.
\end{theorem}
\begin{proof}
    construct a automorphism using two isomorphism.
\end{proof}

\begin{definition}
    Let $(W,<)$ be an well-ordered set. $\alpha \in W$, the \cindex{initial segment} $W_\alpha$ of $W$ is defined as
    \begin{equation}
        W_\alpha = \set{x \in W: x < \alpha}
    \end{equation}
\end{definition}

\begin{theorem}
    no well-ordered set is isomorphic to an initial segment of itself.
\end{theorem}
\begin{proof}
    If $\range{f} = \set{x: x < u}$ is an initial segment, then $f(u) < u$, contrary to \thmref{wellorderedsetisomorphismisbigger}.
\end{proof}

\begin{theorem}
    If $W$ and $V$ are well-ordered sets, then one of the following holds:
    \begin{enumerate}
        \item $W$ is isomorphic to $V$.
        \item $W$ is isomorphic to an initial segment of $V$.
        \item an initial segment of $W$ is isomorphic to $V$.
    \end{enumerate}
\end{theorem}
\begin{proof}
    Define a set $f = \set{(x,y)\in W \times V: W_x \text{ is isomorphic to } V_y}$. Check the $\domain{f}$ and $\range{f}$.
\end{proof}

% Ordinal Numbers
\subsection{Ordinal Numbers}

\begin{definition}
    A set $T$ is \cindex{transitive} if every element of $T$ is a subset of $T$:
    \begin{equation}
        a \in T \rightarrow a \subset T
    \end{equation}
    or $\cup T \subset T$.
\end{definition}

\begin{definition}
    A set is an \cindex{ordinal number} if it is transitive and well-ordered by $\in$. The class of all ordinals is $\allordinals$.
\end{definition}

\begin{definition}
    For two sets $\alpha$ and $\beta$, define a relation $<$ as $\alpha < \beta \leftrightarrow \alpha \in \beta$.
\end{definition}

\begin{theorem}
    $\emptyset \in \allordinals$
\end{theorem}
\begin{proof}
    by definition.
\end{proof}

\begin{theorem}
    $\alpha \in \allordinals \wedge \beta \in \alpha \rightarrow \beta \in \allordinals$
\end{theorem}
\begin{proof}
    $\forall x \in \beta$, $x \in \beta \wedge \beta \subset \alpha \rightarrow x \in \alpha \rightarrow x \subset \alpha \rightarrow x \subset \beta $.
\end{proof}

\begin{theorem}
    $\alpha \in \allordinals \wedge \beta \in \allordinals \wedge \alpha \neq \beta \wedge \alpha \subset \beta  \rightarrow \alpha \in \beta$
\end{theorem}
\begin{proof}
    Let $\gamma$ be the least element of $\beta - \alpha$. Since $\alpha$ is transitive, $\alpha$ is an initial segment of $\beta_\gamma$. So $\alpha = \set{\epsilon \in \beta: \epsilon < \gamma} = \gamma$, so $\alpha \in \beta$.
\end{proof}

\begin{theorem}
    $\forall \alpha, \beta \in \allordinals \rightarrow \alpha \subset \beta \vee \beta \subset \alpha$
\end{theorem}
\begin{proof}
    Let $\gamma = \alpha \cap \beta$. $\gamma$ is an ordinal. So $\gamma \subset \alpha \rightarrow \gamma \in \alpha$, and $\gamma \in \beta$, so $\gamma \in \alpha \cap \beta = \gamma$ and $\gamma \in \gamma$.
\end{proof}

\begin{theorem}
    The facts about ordinal numbers are:
    \begin{enumerate}
        \item $\alpha = \set{\beta: \beta < \alpha}$
        \item If $C$ is a nonempty class of ordinals, then $\cap C$ and $\cup C$ are ordinals.
        \item $\forall \alpha \in \allordinals \left(\alpha \cup \set{\alpha} \in \allordinals \right)$ and $\alpha \cup \set{\alpha} = \mathbf{inf} \set{\beta: \beta > \alpha}$.
    \end{enumerate}
\end{theorem}

\begin{definition}
    We define $\alpha + 1 = \alpha \cup \set{\alpha}$, the \cindex{successor} of $\alpha$.
\end{definition}

\begin{theorem}
    Every well-ordered set is isomorphic to a unique ordinal number.
\end{theorem}

\begin{definition}
    If $\alpha = \beta + 1$, $\alpha$ is a \cindex{successor ordinal}. If $\alpha$ is not a successor ordinal, then $\alpha = \supremum{\beta: \beta < \alpha} = \cup \alpha$, and is a \cindex{limit ordinal}. $0$ is defined as a limit ordinal.
\end{definition}

\begin{definition}[natural numbers]
    The least nonzero limit ordinal is denoted as $\omega$. The ordinals less than $\omega$ is called \cindex{finite ordinals}, or \cindex{natural numbers}.
\end{definition}

\begin{theorem}[\cindex{Transfinite Induction}]
    Let $C$ be a class of ordinals and assume that:
    \begin{enumerate}
        \item $0 \in C$
        \item $\alpha \in C \rightarrow \alpha + 1 \in C$
        \item If $\alpha$ is a nonzero limit ordinal and $\forall \beta \in \alpha (\beta \in C) \rightarrow \alpha \in C$.
    \end{enumerate}
    Then $C = \allordinals$.
\end{theorem}
\begin{proof}
    choose the least $\alpha \notin C$.
\end{proof}

\begin{definition}
    A \cindex{transfinite sequence} is a function that the domain is an ordinal:
    \begin{equation}
        \transfinitesequence{\alpha_\xi : \xi < \alpha}
    \end{equation}
\end{definition}

\begin{theorem}[\cindex{Transfinite Recursion}]
    Let $G$ be a function on the class of transfinite sequence, then there is a unique function $F$ on $\allordinals$ that $\forall \alpha \in \allordinals$:
    \begin{equation}
        F(\alpha) = G(F\restriction_\alpha)
    \end{equation}
\end{theorem}

\begin{definition}
    Let $\alpha>0$ be a limit ordinal and $\transfinitesequence{\gamma_\xi : \xi < \alpha}$ be a nondecreasing sequence of ordinals. The \cindex{limit} of the sequence is
    \begin{equation}
        \lim_{\xi \rightarrow \alpha} \gamma_\xi = \supremum{\gamma_\xi : \xi < \alpha}
    \end{equation}
    It is possible that $\displaystyle \lim_{\xi \rightarrow \alpha} \gamma_\xi \notin \transfinitesequence{\gamma_\xi : \xi < \alpha}$.
\end{definition}

\begin{definition}
    A sequence of ordinal $\transfinitesequence{\gamma_\alpha : \alpha \in \allordinals}$ is \cindex{normal} if it is increasing and \cindex{continuous}, that is for every limit ordinal $\alpha$, $\displaystyle \gamma_\alpha = \lim_{\beta \rightarrow \alpha} \gamma_\beta$.
\end{definition}

% Ordinal Arithmetic
\subsection{Ordinal Arithmetic}

\begin{theorem}
    For all ordinal $\alpha$ and $\beta$, we have:
    \begin{enumerate}
        \item $\alpha + 0 = \alpha$
        \item $\alpha + (\beta + 1) = (\alpha + \beta) + 1$
        \item $\displaystyle \alpha + \beta = \lim_{\xi \rightarrow \beta} (\alpha + \xi)$ for all limit ordinal $\beta > 0$.
        \item $\alpha \cdot 0 = 0$
        \item $\alpha \cdot (\beta + 1) = \alpha \cdot \beta + \alpha$
        \item $\displaystyle \alpha \cdot \beta = \lim_{\xi \rightarrow \beta} \alpha \cdot \beta$ for all limit ordinal $\beta > 0$
        \item $\alpha^0 = 1$
        \item $\alpha^{\beta + 1} = \alpha^\beta \cdot \alpha$
        \item $\displaystyle \alpha^\beta = \lim_{\xi \rightarrow \beta} \alpha^\xi$ for all limit ordinal $\beta > 0$.
    \end{enumerate}
    So $\alpha + \beta$, $\alpha \cdot \beta$, and $\alpha^\beta$ are normal function in second variable $\beta$. Note that neither $+$ nor $\cdot$ is commutative:
    \begin{equation}
        \begin{aligned}
            1 + \omega = \omega &\neq \omega + 1 \\
            2 \cdot \omega = \omega &\neq \omega \cdot 2 = \omega + \omega
        \end{aligned}
    \end{equation}
\end{theorem}

\begin{theorem}
    For all ordinal $\alpha$ and $\beta$, we have:
    \begin{enumerate}
        \item $\beta < \gamma \rightarrow \alpha + \beta < \alpha + \gamma$
        \item If $ \alpha < \beta$, there is a unique $\delta$ that $\alpha + \delta = \beta$.
        \item $\beta < \gamma \wedge \alpha > 0 \rightarrow \alpha \cdot \beta < \alpha \cdot \gamma$
        \item If $\alpha > 0$, there is a unique $\beta$ and $\rho < \alpha$ that $\gamma = \alpha \cdot \beta + \rho$.
        \item $\beta < \gamma \wedge \alpha > 1 \rightarrow \alpha^\beta < \alpha^\gamma$
    \end{enumerate}
\end{theorem}

\begin{theorem}[\cindex{Cantor's Normal Form Theorem}]
    Every ordinal $\alpha > 0$ has a unique representation:
    \begin{equation}
        \alpha = \omega^{\beta_1} \cdot k_1 + \dots + \omega^{\beta_n} \cdot k_n
    \end{equation}
    where $n \geq 1$, $\alpha \geq \beta_1 > \dots > \beta_n$, and $k_i$ are nonzero natural numbers.
\end{theorem}
\begin{proof}
    use induction. $\forall \alpha > 0$, let $\beta$ be the greatest ordinal number that $\omega^\beta \leq \alpha$. There is a unique $\delta$ and $\rho < \omega^\beta$ that $\alpha = \omega^\beta + \rho$.
\end{proof}

% well-founded relations
\subsection{Well-Founded Relations}

\begin{definition}
    A binary relation $E$ on a set $P$ is \cindex{well-founded} if every nonempty $X \subset P$ has a $E$-minimal element, that is $\forall a \in X$ there is no $x \in X$ that $x E a$.
\end{definition}

\begin{theorem}
    If $E$ is a well-founded relation on $P$, there is a unique function $\rho : P \rightarrow \allordinals$ that $\forall x \in P$:
    \begin{equation}
        \rho(x) = \supremum{\rho (y) + 1: y E x}
    \end{equation}

    The range of $\rho$ is an initial segment of ordinals and is an ordinal number, which is the \cindex{height} of $E$.
\end{theorem}
\begin{proof}
    Define a $P$ that
    \begin{equation}
        \begin{aligned}
            P_0 &= \emptyset \\
            P_{\alpha + 1} &= \set{x \in P: \forall y (y E x \rightarrow y \in P_\alpha )} \\
            P_\alpha &= \bigcup_{\xi < \alpha} P_\xi \text{ , if } \alpha \text{ is a limit ordinal}
        \end{aligned}
    \end{equation}
    Let $\theta$ be the least ordinal that $P_{\theta +1} = P_\theta$.
\end{proof}


\lab{Algorithms}{Krylov Subspaces}{Finding Eigenvalues Using Iterative Methods}
\label{lab:kry_arnoldi}

\objective{Discuss simple Krylov Subspace Methods for finding eigenvalues and show some interesting applications.}

In the realm of numerical linear algebra, Krylov Subspace Methods are considered some of the most successful  methods ever invented.
They are simple and robust and can be used to find approximate solutions to linear systems and eigenvalue problems involving extremely
large matrices.
One of the reasons for this success is that they do not require copies or modifications of the original
matrix.
Krylov subspace methods are used to estimate certain properties of a matrix based on how it acts on vectors through matrix multiplication.
This is especially useful for large sparse matrices (allowing us to deal with the matrix in compressed form),
and for matrices that have symmetries that reduce storage and enable faster matrix multiplication (allowing us to use
specialized algorithms like the Fast Fourier Transform rather than the full matrix).

The general approach of Krylov subspace methods is to consider how a given matrix $A$ acts on the span of
$\{ x, Ax, A^2 x, \ldots, A^{N-1} x \}$ (called an \emph{order-$N$ Krylov Subspace generated by $A$ and $x$}), where $N$ is significantly
smaller than the number of rows of $A$.
The formation of these subspaces in an efficient and numerically stable manner is usually based on either the \emph{Arnoldi
iteration} or the \emph{Lanczos iteration}. We will discuss both of these algorithms here.

\section*{The Arnoldi Iteration}

Before discussing the specific uses of the Krylov subspace in linear systems and eigenvalue problems, let us address a very
practical concern: how can we best compute a basis for the Krylov subspace? The obvious answer is simply to
calculate the vectors $x, Ax, A^2x, \ldots, A^{N-1} x$, which we can accomplish using only matrix-vector multiplication.
Straightforward though this may be, there is a major problem: $A^n x$ tends to converge to a dominant eigenvector of $A$
as $n$ gets large, and consequently these vectors become nearly parallel. Thus, the basis $\{x, Ax, A^2x, \ldots, A^{N-1} x\}$
is far from orthogonal, and matrix computations associated with this basis will likely be ill-conditioned and prone to
numerical instability. To redress this problem, we may think to apply the Gram-Schmidt orthogonalization process to the
basis, obtaining an orthonormal basis for the Krylov subspace that enjoys much better numerical properties. This turns out to
be a useful thought, and is the basis for the Arnoldi iteration.

You may recall from lab \ref{lab:QRdecomp} that the Modified Gram-Schmidt algorithm allows us to find an increasing number of orthogonal
vectors but does not require that we run the algorithm to is completion to find a full basis.
Our goal is to find an orthonormal set of vectors $q_1,\ldots,q_N$ having the same span as $x, Ax, A^2x, \ldots, A^{N-1} x$.
We start
 things off by setting
\[
q_1 = \frac{x}{\|x\|_2}.
\]
Now, assuming we have obtained $q_1,\ldots,q_n$, we obtain $q_{n+1}$ by projecting $A q_n$ onto the previous vectors,
subtracting out these projections, and then normalizing. To make this more precise, let $h_{i,n} = \langle q_i, A q_n\rangle$
for $i = 1,\ldots, n$. Subtract out these projections by calculating
\[
p_{n+1} = A^n x - \sum_{i=1}^n h_{i,n}q_i.
\]
Define $h_{n+1,n} = \|p_{n+1}\|_2$, and normalize $p_{n+1}$ by calculating
\[
q_{n+1} = \frac{p_{n+1}}{h_{n+1,n}},
\]
our next basis vector. This procedure is outlined (in slightly more Python-friendly notation) in Algorithm \ref{alg:arnoldi_iteration}.

Perhaps you noticed a slight discrepancy between the Arnoldi iteration as described above, and the usual Gram-Schmidt procedure.
Specifically, you might have expected to compute $q_{n+1}$ by projecting $A^n x$, rather than $A q_n$, onto the previous vectors.
Thankfully, it is straightforward to show that our algorithm produces a valid orthonormal basis for the Krylov subspace,
despite this difference. And because of this detail, we do not need to compute and store the original Krylov basis
$x, Ax, \ldots, A^{N-1}x$.
Additionally, each iteration only requires one matrix-vector calculation, and the individual entries in $A$ are never
referenced or modified. Thus, even if the matrix $A$ is very large in theory, as long as we have a reasonably efficient
subroutine to calculate $Ax$ for any vector $x$, the Arnoldi iteration is computationally tractable.

This algorithm produces an othonormal basis $q_1,\ldots,q_N$ for the order-$N$ Krylov subspace generated by $A$ and $x$, as well
as a collection of numbers $h_{i,j}$. If we define a matrix $H_N$ whose $i,j$'th entry is $h_{i,j}$ for $i \leq j+1$ and
is $0$ otherwise, we now have an upper Hessenberg matrix.
Recall that an upper Hessenberg matrix has the property that all entries below the first subdiagonal
are equal to zero. Any square matrix is unitarily similar to an upper Hessenberg matrix. Dealing with a Hessenberg matrix is
often more convenient than dealing with a general matrix, especially when it comes to finding eigenvalues or solving systems
of equations, since efficient algorithms designed for these types of matrices exist.
It turns out that there is a Hessenberg factorization of $A$, given by
\[
A  = QHQ^*,
\]
where $Q$ is a unitary matrix and $H$ is upper Hessenberg such that the first $N$ columns of $Q$ are $q_1,\ldots,q_N$, and
the upper left $N \times N$ submatrix of $H$ is equal to $H_N$. Hence, the Arnoldi iteration provides a connection between
the Krylov subspace and the Hessenberg factorization of a matrix. Each step in the Arnoldi iteration can be thought of as computing
another step in the Hessenberg reduction of $A$. Each $H_N$ is really just the $N \times N + 1$ upper-left block of $H$.
Solving eigenvalue problems or systems of equations for
a general square matrix can thus be reduced, via Arnoldi iteration, to solving these problems for a Hessenberg matrix,
a much easier task.

At this point, we can view the Arnoldi iteration as a means to compute an orthonormal basis for a Krylov subspace, or
alternatively, to compute a partial Hessenberg factorization of a matrix.
But in Lab \ref{lab:Canonical_Transformations}, we discussed how orthogonal transformations can be used to transform a matrix to Upper
Hessenberg form.
We were able to find the eigenvalues of such Upper Hessenberg matrices in Lab \ref{lab:EigSolve}.
So what have we really gained by this new approach?
These previous approaches were based on matrix-matrix multiplication and required us to manipulate individual entries of the matrix.
The Arnoldi iteration avoids this and relies only on our ability to calculate matrix-vector multiplication.
Further, our present approach will allow us to compute only a partial Hessenberg factorization. This is advantageous when, as
is often the case, the behavior and properties of a matrix can be well-approximated by only a small portion of its Hessenberg
form.

%The idea is to construct a set of vectors that allow us to roughly approximate the action of a very large matrix on the subspace.
%This involves forming only a portion of the actual Hessenberg reduction of the Hessenberg factorization of the matrix.
%You may recall from lab \ref{lab:QRdecomp} that the Modified Gram-Schmidt algorithm allows us to find an increasing number of orthogonal
%vectors but does not require that we run the algorithm to is completion to find a full basis.
%In the Arnoldi iteration we use the Modified Gram-Schmidt algorithm to compute progressively larger portions of the matrices $Q$ and $H$
%in the Hessenberg factorization of $A$.
%These intermediate matrices often carry a significant amount of information about $A$, and we can get away with storing small portions of
%them instead of having to store a dense version of the matrix $A$.
%The Arnoldi iteration is outlined in Algorithm \ref{alg:arnoldi_iteration}.
%We will now consider the ideas underlying the Arnoldi iteration in further detail.
%
%Let $A$ be an extremely large matrix that we do not wish to modify.
%We will assume that we have some sort of method for computing $A v$ for any vector $v$, but we will not make any other assumptions
%relating to how $A$ is stored or what properties it has.
%Write the Hessenberg factorization of $A$ as
%\[
%Q^* A Q = H,
%\]
%where $Q$ is a unitary matrix and $H$ is Upper Hessenberg.
%
%We would like to compute the columns of $Q$ one-by-one.
%Let $q_k$ be the $k$'th column of $Q$ and $h_{i,j}$ be the $i,j$'th element of $H$.
%From matrix multiplication we have that
%\[A q_n = h_{0, n} q_0 + \dots + h_{n, n} q_n + h_{n+1, n} q_{n+1}\]
%This can be rewritten as
%\[h_{n+1, n} q_{n+1} = A q_n - h_{0,n} q_0 - \dots - h_{n,n} q_n\]
%Since each column of $Q$ has norm $1$, this allows computation of each $q_k$ based on each of the previous columns.
%This recurrence relation does not provide any constraint on the first column of $Q$.
%In practice, if we are searching for eigenvalues we can use a normalized random vector as the first column of $Q$.
%These columns of $Q$ allow compution of partial reductions of $A$ to Hessenberg form without any modification of $A$.
%This can seen as follows:
%
%Let $Q_k$ be the first $k$ columns of $Q$.
%Let $Q_u$ be the remaining columns of $Q$.
%We then have that
%\[H = Q^T A Q =
%\begin{bmatrix}
%Q_k^* A Q_k & Q_k^* A Q_u \\
%Q_u^* A Q_k & Q_u^* A Q_u
%\end{bmatrix}\]
%Since $H$ is Upper Hessenberg, so is the matrix $Q_k^* A Q_k$ in the upper left corner.
%Let $H_k = Q_k^* A Q_k$.
%Note that we really only need to store the columns of $Q$ that we have already computed.
%We can also construct the $H_k$ as we go.
%
%\begin{info}
%The recurrence relation for computing $q_{n+1}$ is the same expression that would be used to perform Gram-Schmidt orthogonalization
%to project $A q_n$ orthogonal to the vectors $\{ q_0, \dots, q_n \}$.
%This allows for an alternate characterization of the Arnoldi iteration.
%We first choose a normalized random vector $q_0$, then perform the Modified Gram-Schmidt algorithm on the vectors
%$A q_0, A q_1, A q_2, \dots$.
%\end{info}

\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{arnoldi}{$b, Amul, k, tol=1E-8$}
	\State $m \gets b.size$						\Comment{Some initialization steps}
	\State $Q \gets \text{empty}\left(m, k+1\right)$
	\State $H \gets \text{zeros}\left( k+1, k\right)$
	\State $q_0 = b$							\Comment{Set $q_0$ equal to $b$.}
	\State $q_0 /= \|q_0\|_2$						\Comment{Normalize $q_0$.}
	\For{$j=0$, $j<k$}							\Comment{Perform the actual iteration.}
		\State $q_{j+1} = Amul \left(q_j\right)$		\Comment{Compute $A q_j$.}
		\For{$i=0$, $i<j+1$}					\Comment{Modified Gram-Schmidt.}
			\State $h_{i,j} = \langle q_i, q_{j+1}\rangle$		\Comment{Set values of $H$.}
			\State $q_{j+1} -= h_{i,j} q_i$
		\EndFor
		\State $h_{j+1,j} = \|q_{j+1}\|_2$			\Comment{Set subdiagonal element of $H$.}
            \If{$|h_{j+1,j}|<tol$}					\Comment{Stop if $\|q_{j+1}\|_2$ is too small.}
			\State \pseudoli{return} $H[:j,:j]$
		\EndIf
		\State $q_{j+1} /= h_{j+1,j}$				\Comment{Normalize $q_{j+1}$.}
	\EndFor
	\State \pseudoli{return} $H[:-1]$				\Comment{Return $H_{k}$.}
\EndProcedure
\end{algorithmic}
\caption{The Arnoldi Iteration}
\label{alg:arnoldi_iteration}
\end{algorithm}

\begin{warn}
To avoid errors involving the casting of complex numbers to real numbers make the datatypes of $H$ and $Q$ complex.
Make sure you account for the use of complex numbers when computing the norm.
You will also want to use the complex version of the \li{sqrt} function.
This function is in the built in \li{cmath} library.
\end{warn}

Notice that in Algorithm \ref{alg:arnoldi_iteration}, $k$ is the number of times to multiply by $A$.
This will result in a dimension $k+1$ Krylov Subspace.

\begin{problem}
Write a Python function that performs the Arnoldi iteration given a nonzero starting vector $b$, a function to multiply a
vector on the left by some matrix $A$, and a number $n$ of steps to perform.
Also have it accept a tolerance parameter that defaults to \li{1E-8}.
Have it return the computed $H_n$ and $Q_n$.
\end{problem}

\begin{info}
Depending on the matrix $A$, the Arnoldi iteration may end quickly.
This happens when there is a subspace $G$ that is fixed under left multiplication by $A$.
In this particular case, the matrix $H_k$ exactly matches the way $A$ acts on $G$.
This means that the eigenvalues of $H_k$ are eigenvalues of $A$.
The eigenvectors of $H$ will also be eigenvectors of $A$ after the change of basis defined by $Q_k$.
\end{info}

% Make a 3D plot showing the projection. Label the initial vector x and Ax.

\section*{Finding Eigenvalues Using Arnoldi iteration}

It is relatively easy to understand how Arnoldi iteration finds eigenvalues of a matrix.
In simple terms, the eigenvalues of the $H_k$ converge to the eigenvalues of $A$.
The eigenvalues of the $H_k$ are called Ritz values.
This can happen very quickly, but, in general, the rate of convergence depends on the matrix $A$.
This works because each $H_k$ is a kind of lower-dimensional approximation of $A$.
In adding the projection of $A q_n$ orthogonal to $q_0, \ldots, q_n$, we are really including the only portion
of the space spanned by the image of $q_0, \dots, q_n$ under $A$ that is not already included in their span.
Generally speaking, the Ritz values converge most quickly to the eigenvalues of largest magnitude.
Convergence is also faster for eigenvalues that are relatively far from the other eigenvalues of $A$.
Figures \ref{fig:arnoldi_random_eig_conv} and \ref{fig:arnoldi_random_val_conv} show the convergence of the
Ritz values to the largest eigenvalues of some different matrices.

\begin{figure}
\includegraphics[width=\textwidth]{rand_eigs_conv.pdf}
\caption{The convergence of the Ritz values to the largest eigenvalues of a matrix with random eigenvalues between $0$ and $1$.}
\label{fig:arnoldi_random_eig_conv}
\end{figure}

\begin{figure}
\includegraphics[width=\textwidth]{rand_vals_conv.pdf}
\caption{The convergence of the Ritz values to the largest eigenvalues of a matrix with random entries between $0$ and $1$.
Matrices of this form generally have a single isolated eigenvalue that is much larger than the rest.
It can be seen here that the Ritz values converge to this eigenvalue much more quickly.}
\label{fig:arnoldi_random_val_conv}
\end{figure}

\begin{problem}
Run the Arnoldi iteration on a random $100 \times 100$ array $A$.
Use a random vector as a starting point.
Find $H_{40}$ and compare the $10$ eigenvalues of largest magnitude of $H_{40}$ with the $10$ eigenvalues of largest magnitude of $A$.
Arnoldi iteration is not the most effective way to work with dense matrices without significant patterns, but this example does illustrate the desired convergence.
\end{problem}

\begin{problem}
\label{prob:fourier_eigs}
The Discrete Fourier Transform is a linear operator, so it has a matrix representation.
This matrix representation is well known and the symmetries that arise in it are used in computing the Fast Fourier Transform.
Fast Fourier Transforms are discussed in much greater detail in Volume 2.

Since the Fast Fourier Transform is a linear operator, it also has eigenvalues and eigenvectors.
The eigenvalues are known to be $\{ -1, 1, -i, i \}$.
Use your implementation of the Arnoldi iteration to estimate the eigenvalues of the Discrete Fourier Transform.
This can be done by passing the \li{fft} function from either \li{pyfftw.interfaces.scipy_fftpack} or \li{scipy.fftpack} as the argument \li{Amul} to your function for the Arnoldi iteration.
Let $b$ be a random real-valued array of size $2^{20}$.
Let $k = 10$.
You will need to either divide your computed eigenvalues or your computed value for each $A q_i$ by $2^{10}$ (the square root of the length of $b$).
This is a normalizing factor that comes from the way the Fast Fourier Transform is defined.

All the eigenvalues of the Discrete Fourier Transform should show up most of the times this code is run.
\end{problem}

Problem \ref{prob:fourier_eigs} shows how Arnoldi Iteration works nicely on matrices with useful patterns.
What is remarkable is that the matrix representation of the Fast Fourier Transform for length $2^{20}$ vectors would be a $2^{20} \times 2^{20}$ complex valued matrix.
At that size it would take $64$ TB of space just to store it.
Thanks to the Fast Fourier Transform algorithm we are able to compute the Fourier Transform without ever explicitly forming this massive matrix.
Using the Arnoldi iteration we can also estimate its eigenvalues.

% Once the eigenvalue lab has been rewritten, have them use their own solver to find the eigenvalues of the $H_k$.

\begin{problem}
Finding the roots of a polynomial can be represented as an eigenvalue problem.
Finding the roots of a monic polynomial (a polynomial with leading coefficient 1) $p = c_0 + c_1 x + \dots + c_{n-1} x^{n-1} + x^n$ is equivalent to finding the eigenvalues of the matrix
\[C = \begin{bmatrix}
0 & 0 & \dots & 0 & -c_0 \\
1 & 0 & \dots & 0 & -c_1 \\
0 & 1 & \dots & 0 & -c_2 \\
\vdots & \vdots & \ddots & \vdots & \vdots \\
0 & 0 & \dots & 1 & -c_{n-1} \end{bmatrix}\]
This matrix is called the companion matrix of the polynomial $p$.
As it happens, every matrix is similar to the companion matrix of its characteristic polynomial, but we won't use that fact here.

The following is a function that, given an array containing the coefficients $c_0, c_1, \dots, c_{n-1}$ for a monic polynomial $p$, performs matrix multiplication by the corresponding companion matrix.
\begin{lstlisting}
def companion_multiply(c, u):
    v = np.empty_like(u)
    v[0] = - c[0] * u[-1]
    v[1:] = u[:-1] - c[1:] * u[-1]
    return v
\end{lstlisting}

Use the Arnoldi iteration to estimate the five zeros of largest norm of a degree $1000$ monic polynomial with randomly chosen coefficients (the leading coefficient still needs to be 1).
Run $50$ steps of the Arnoldi iteration.
Compare your results with the roots of the polynomial computed using NumPy's \li{poly1d} class.
This computation can be done like this (where \li{c} is the array of random coefficients for the polynomial)
\begin{lstlisting}
p = np.poly1d([1] + list(c[::-1]))
roots = p.roots
# Now sort by absolute value from largest to smallest
roots = roots[np.absolute(roots).argsort()][::-1]
\end{lstlisting}
How close are the first few zeros of largest norm?
\end{problem}

\section*{Lanczos Iteration}

Depending on the symmetry of the problem we may be able to make the Arnoldi iteration more efficient.
Consider the case that $A$ is symmetric.
This means that the matrx $H$ is both Upper Hessenberg and symmetric, so it is tridiagonal.
Since $H$ is tridiagonal, we \textit{should} have that each $A q_n$ is orthogonal to $q_0, \dots, q_{n-2}$.
This means that storage of all the columns of $Q_k$ is no longer necessary.
We can run the entire algorithm while storing only the previous two columns of $Q$ that we have computed.
We can also represent $H$ as two vectors: a vector $\alpha$ storing the values along the main diagonal of $H$, and a vector $\beta$ storing the values in the first subdiagonal (the values in the first superdiagonal are the same).
This change in the way things are stored allows Algorithm \ref{alg:arnoldi_iteration} to be simplified to Algorithm \ref{alg:lanczos_iteration}.
Algorithm \ref{alg:lanczos_iteration} is known as the Lanczos iteration.

\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{lanczos}{$b, Amul, k, tol=1E-8$}
	\State $q_0 \gets 0$								\Comment{Some initialization}
	\State $q_1 \gets \frac{b}{\|b\|_2}$
	\State $\alpha \gets \text{empty}\left(k\right)$
	\State $\beta \gets \text{empty}\left(k\right)$
	\State $\beta_{-1} = 0$
	\For{$i=0$, $i<k$}									\Comment{Perform the iteration.}
		\State $z \gets Amul\left(q_1\right)$					\Comment{$z$ is a temporary vector to store $q_{i+1}$.}
		\State $\alpha_i = \langle q_1, z \rangle$				\Comment{$q_1$ is used to store the previous $q_i$.}
		\State $z -= \alpha_i q_1 + \beta_{i-1} q_0$				\Comment{$q_0$ is used to store $q_{i-1}$.}
		\State $\beta_{i} = \|z\|_2$						\Comment{Initialize $\beta_i$.}
		\If{$\beta_i<tol$}								\Comment{Stop if $\|q_{i+1}\|_2$ is too small.}
			\State \pseudoli{return} $\alpha [: i+1]$, $\beta [: i]$
		\EndIf
		\State $z /= \beta_i$
		\State $q_0, q_1 = q_1, z$						\Comment{Store new $q_{i+1}$ and $q_i$ on top of $q_1$ and $q_0$.}
	\EndFor
	\State \pseudoli{return} $\alpha$, $\beta [: -1]$
\EndProcedure
\end{algorithmic}
\caption{The Lanczos Iteration}
\label{alg:lanczos_iteration}
\end{algorithm}

\begin{problem}
\label{prob:lanczos}
Write a Python function that performs the Lanczos iteration.
Have it accept a starting vector $b$, a function $Amul$ that computes $A x$ for any vector $x$, a number $k$ of iterations to perform, and an optional argument $tol$ that defaults to \li{1E-8}.
\end{problem}

In its most basic form the Lanczos iteration is not stable.
In exact arithmetic the vectors $q_i$ are exactly orthogonal, but in the presence of roundoff error this may be absolutely false.
In imprecise arithmetic, it is possible for the $q_i$ to suffer from so much roundoff error that \textit{they may no longer even be linearly independent}.
There are a variety of modifications to the Lanczos iteration that address this instability.
The library used for Lanczos iteration in Scipy uses an algorithm called the Implicitly Restarted Lanczos Method.
We will not discuss these algorithms in detail here.

% If needed we could make a separate lab on the Lanczos iteration and the Implicitly Restarted Lanczos Method.
% There isn't time or space here for it though.

\begin{problem}
The following code performs matrix multiplication by a tridiagonal symmetric matrix.
It accepts vectors $a$ and $b$ and $u$.
$a$ stores the entries in the main diagonal of the matrix.
$b$ stores the entries in the first sub/superdiagonal.
The function returns the image of $u$ under the matrix represented by $a$ and $b$.

\begin{lstlisting}
def tri_mul(a, b, u):
    v = a * u
    v[:-1] += b * u[1:]
    v[1:] += b * u[:-1]
    return v
\end{lstlisting}

Use the Lanczos iteration function you wrote for Problem \ref{prob:lanczos} to estimate the $5$ largest eigenvalues of a symmetric tridiagonal matrix $A$ with random values in its nonzero diagonals (i.e. make $a$ and $b$ random).
For demonstration purposes, let $A$ be $1000 \times 1000$.
Perform $100$ iterations.
Compare the $5$ eigenvalues of largest absolute value with the $5$ Ritz values of largest norm.
How do they compare?

Try running your simulation a few times for different vectors $a$ and $b$.
You may notice that, occasionally, the largest eigenvalue is repeated in the Ritz values.
This happens because of the lack of orthogonality between the vectors used in the Lanczos iteration.
These erroneous eigenvaleus are called ``ghost eigenvalues."
They generally converge to actual eigenvalues of the matrix and can make the multiplicity of an eigenvalue look higher than it really is.
\end{problem}

\section*{Arnoldi Iteration in SciPy}

SciPy interfaces with a Fortran library called ARPACK that has good implementations of the Arnoldi and Lanczos algorithms.
The Arnoldi iteration is found in \li{scipy.sparse.linalg.eigs}.
The Implicitly Restarted Lanczos iteration is found in \li{scipy.sparse.linalg.eigsh}
These functions allow you to find either the largest or smallest eigenvalues of a sparse or dense matrix.
The function \li{scipy.sparse.linalg.svds} uses the Implicitly Restarted Lanczos iteration on $A^* A$ to find singular values.

\begin{problem}
In Lab \ref{lab:MarkovGraph} we discussed how to find the Laplacian Matrix of a graph.
The second-smallest eigenvalue of a graph is known as the ``Fiedler Value" or the ``algebraic connectivity."
The algebraic connectivity of a graph is positive if the graph is connected and zero if it is not.
In general, the multiplicity of the eigenvalue $0$ in the Laplacian matrix of a graph is the number of connected components of that graph.
The following code constructs the Laplacian matrix of a graph composed of a line of nodes.
The matrix is stored in \li{dia_matrix} format.
\begin{lstlisting}
import scipy.sparse as ss
m = 1000
d = np.ones(m)
d[1:-1] += np.ones(m-2)
l = ss.diags([-np.ones(m-1), d, -np.ones(m-1)], [-1, 0, 1])
\end{lstlisting}
Use the \li{eigsh} function to verify that this graph is connected.
You should look at documentation for the \li{scipy.sparse} library and find the options you need to use.
For proper convergence you will want to leave the number of eigenvalues computed at its default value.

The following code constructs the Laplacian matrix the same graph as before except that a single edge has been removed.
\begin{lstlisting}
m = 1000
cut = 500
d = np.ones(m)
d[1:-1] += np.ones(m-2)
d1 = -np.ones(m-1)
d1[cut] = 0
d[[cut, cut+1]] =1
l = ss.diags([d1, d, d1], [-1, 0, 1])
\end{lstlisting}
Verify that this graph is not connected.
\end{problem}


% !TEX root = main.tex

%------------------------------------------------
\chapter{Introductory Examples}

\begin{example}
Radioactive isotope decays according to the differential $$\frac{dm}{dt} = -km$$ where $k > 0$ is constant. $m$ is the mass present at time $t$.
Find the half life such that $m(\tau) = \frac{1}{2}m(0)$
\end{example}
\bigskip

\begin{example}  A student goes home for Christmas leaving a mouldy burger under the sofa. The mould grows according to $$\frac{dm}{dt} = km$$ where $ k > 0$ is constant. Find the time $\tau = 0$ at which $m(\tau) > M$, assuming $m(0)$ is known. Here $M$ is the mass of the sofa.
\end{example}
\bigskip

\begin{example}  The previous model is changed to $$\frac{dm}{dt} = km(M-m)$$ where $k > 0$, $M > 0$ are constants. Show how the solution behaves.
\end{example}
\bigskip

\begin{example}  Show that the differential equation $$\frac{dy}{dt} = 2\sqrt[]{y}$$ has more than one solution with :
$$y_1(t) = t^2 \mbox{ for t } {\geq} \mbox{ 0}$$
$$y_2(t) = \mbox{ 0 for all t }$$
A third solution :
$$
y_3 (t) = \begin{cases}
y_2(t) \mbox{ for t } \leq \mbox{ 0} \\
y_1(t) \mbox{ for t } > \mbox{ 0}
\end{cases} $$
\end{example}
\bigskip

\begin{example}
Show that the solutions of the equation : $$\frac{dy}{dt} = 1 + y^2$$ blows up in finite time.
\end{example}

%------------------------------------------------
\endinput


\subsubsection{Hyperboloids of Two Sheets}
\noindent
A hyperboloid of two sheets has 2 -'s and 1 + in its equation. It is made up of two disconnected, mirror image, surfaces.

\begin{figure}[H]
	\centering
	\includegraphics[width=0.33\textwidth]{./Images/differentialMultivariableCalculus/two_sheets.png}
	\caption{A hyperboloid of two sheets}
\end{figure}