\chapter{Normal hidden Markov models}

\section{Forward-backward variables}

Let $(X_t)_{t = 1}^T$ be a homogeneous Markov chain over the state
space $S = \{1, \ldots, s\}$ with transition matrix $P = [p_{ij}]$,
$i, j \in S$, and initial state distribution $p = [p_i]$, $i \in S$.
Then, for each $i_1, \ldots, i_T \in S$,
\[
\Pr(X_1 = i_1, X_2 = i_2, \ldots, X_T = i_T) = p_{i_1}p_{i_1i_2}
\ldots p_{i_{T - 1}i_T}.
\]
For each state $i \in S$, let $f_i(y, \theta_i)$ be a corresponding
probability density function. In each moment $t = 1, \ldots, T$, a
value $y_t$ of a random variable $Y_t$ is observed which comes from
the density $f_{i_t}$. The likelihood of the sample $y_1, \ldots, y_T$
is
\begin{eqnarray*}
  \lefteqn{\mathcal{L} = L(p, P, \theta_1, \ldots, \theta_s) =} \\ & =
  & \sum_{i_1, \ldots, i_T = 1}^s p_{i_1}f_{i_1}(y_1, \theta_{i_1})
  p_{i_1i_2}f_{i_2}(y_2, \theta_{i_2}) \ldots p_{i_{T - 1}i_T}
  f_{i_T}(y_T, \theta_{i_T}) = \\ & = & \sum_{i_1 = 1}^s
  p_{i_1}f_{i_1}(y_1, \theta_{i_1}) \sum_{i_2 = 1}^s
  p_{i_1i_2}f_{i_2}(y_2, \theta_{i_2}) \ldots \sum_{i_T = 1}^s p_{i_{T
      - 1}i_T}f_{i_T}(y_T, \theta_{i_T}).
\end{eqnarray*}
The last expression can be calculated using forward variables
\begin{eqnarray}
  \label{eq:alpha1}
  \alpha_1(j) & = & p_j f_j(y_1, \theta_j), \hspace{1em} j \in S, \\
  \label{eq:alphat}
  \alpha_t(j) & = & \sum_{i = 1}^s (\alpha_{t - 1}(i)p_{ij})
  f_j(y_t, \theta_j),
  \hspace{1em} j\in S, \hspace{1em} t = 2, \ldots, T
\end{eqnarray}
or backward variables
\begin{eqnarray}
  \label{eq:betaT}
  \beta_T(i) & = & 1, \hspace{1em} i \in S, \\
  \label{eq:betat}
  \beta_t(i) & = & \sum_{j = 1}^s p_{ij}f_j(y_{t + 1},
  \theta_j)\beta_{t + 1}(j),
  \hspace{1em} i \in S, \hspace{1em} t = T - 1, \ldots, 1
\end{eqnarray}
or both as
\begin{equation}
  \label{eq:likelihood}
  \mathcal{L} = \sum_{i = 1}^s \alpha_T(i) = \sum_{i = 1}^s p_i f_i(y_1,
  \theta_i) \beta_1(i) = \sum_{i = 1}^s
  \alpha_t(i)\beta_t(i), \hspace{1em} t = 1, \ldots, T.
\end{equation}

Moreover, if we define
\begin{eqnarray}
  \label{eq:gamma}
  \gamma_t(i) & = & \alpha_t(i)\beta_t(i) / \mathcal{L}, \hspace{1em} t
  = 1, \ldots, T, \hspace{1em} i \in S, \\
  \label{eq:xi}
  \xi_t(i, j) & = &
  \alpha_t(i)\beta_{t + 1}(j)p_{ij} f_j(y_{t + 1}, \theta_j) / \mathcal{L},
  \hspace{1em}
  t = 1, \ldots, T - 1, \hspace{1em} i, j \in S,
\end{eqnarray}
the following interpretations are possible:
\begin{eqnarray*}
  \alpha_t(i) & = & \Pr(Y_1 = y_1, \ldots, Y_t = y_t, X_t = i),
  \\ \beta_t(i) & = & \Pr(Y_{t + 1} = y_{t + 1}, \ldots, Y_T = y_T, X_t
  = i), \\
  \gamma_t(i) & = & \Pr(Y_1 = y_1, \ldots, Y_T = y_T, X_t = i), \\
  \xi_t(i, j) & = & \Pr(Y_1 = y_1, \ldots, Y_T = y_T, X_t = i,
  X_{t + 1} = j),
\end{eqnarray*}
where $\Pr$ denotes likelihood of the respective event.

\section{Baum-Welch algorithm}

If $f_i(y, \theta_i) = \phi((y - \mu_i) / \sigma_i)$, where $\phi$ is
standard normal probability density, the following formulas can be
used for $i, j \in S$ to increase the likelihood $\mathcal{L}$:
\begin{eqnarray}
  \label{eq:baumwelchp}
  \overline{p}_i & = & \gamma_1(i), \\
  \label{eq:baumwelchP}
  \overline{p}_{ij} & = & \frac{\sum_{t = 1}^{T - 1} \xi_t(i,
    j)}{\sum_{t = 1}^{T - 1} \gamma_t(i)}, \\
  \label{eq:baumwelchmu}
  \overline{\mu}_{i} & = & \frac{\sum_{t = 1}^T \gamma_t(i)y_t}{\sum_{t
      = 1}^T \gamma_t(i)}, \\
  \label{eq:baumwelchsigma}
  \overline{\sigma}_i^2 & = & \frac{\sum_{t = 1}^T \gamma_t(i)(y_t -
    \overline{\mu}_i)^2}{\sum_{t = 1}^T \gamma_t(i)}.
\end{eqnarray}

\section{Viterbi algorithm}

Having found $p$, $P$, $\theta_1$, \ldots, $\theta_s$, one may need to
find the best sequence of states, that is a sequence
\begin{equation} \label{eq:bestseq}
  i_1, \ldots, i_T
\end{equation}
which maximizes
\begin{equation} \label{eq:bestseqprob}
  p_{i_1}f_{i_1}(y_1, \theta_{i_1}) p_{i_1i_2}f_{i_2}(y_2, \theta_{i_2})
  \ldots p_{i_{T - 1}i_T}f_{i_T}(y_T, \theta_{i_T}).
\end{equation}
The Viterbi algorithm proceeds as follows. Let
\begin{displaymath}
  \delta_1(i) = p_i f_i(y_1, \theta_i), \hspace{1em} \psi_1(i) =
  0, \hspace{1em} i \in S.
\end{displaymath}
For $t = 2, \ldots, T$, let
\begin{displaymath}
  \delta_t(j) = \max_{i \in S} (\delta_{t - 1}(i) p_{ij}) f_j(y_t,
  \theta_j),
  \hspace{1em}
  \psi_t(j) = \argmax_{i \in S} (\delta_{t - 1}(i) p_{ij}),
  \hspace{1em} i \in S.
\end{displaymath}
Then the maximized probability~(\ref{eq:bestseqprob}) is equal to
$\max_{i \in S} \delta_T(i)$ and the best sequence~(\ref{eq:bestseq})
can be backtracked by
\begin{displaymath}
  i_T = \argmax_{i \in S} \delta_T(i),
  \hspace{1em}
  i_t = \psi_{t + 1}(i_{t + 1}), \hspace{1em} t = T - 1, \ldots, 1.
\end{displaymath}

\section{Scaling}

If the forward and backward variables are scaled, i.~e.
\begin{eqnarray}
  \label{eq:alpha1scaled}
  \hat{\alpha}_1(j) & = & c_1 \alpha_1(j), \hspace{1em} j \in S, \\
  \label{eq:alphatscaled}
  \hat{\alpha}_t(j) & = & c_t \sum_{i = 1}^s (\hat{\alpha}_{t -
    1}(i)p_{ij}) f_j(y_t, \theta_j),
  \hspace{1em} j \in S, \hspace{1em} t = 2, \ldots, T,
\end{eqnarray}
and
\begin{eqnarray}
  \label{eq:betaTscaled}
  \hat{\beta}_T(i) & = & d_T \beta_T(i), \hspace{1em} i \in S, \\
  \label{eq:betatscaled}
  \hat{\beta}_t(i) & = & d_t \sum_{j = 1}^s p_{ij}f_j(y_{t + 1},
  \theta_j)\hat{\beta}_{t + 1}(j),
  \hspace{1em} i \in S, \hspace{1em} t = T - 1, \ldots, 1
\end{eqnarray}
are calculated instead of~(\ref{eq:alpha1}--\ref{eq:betat}), where
\begin{eqnarray}
  \label{eq:c1}
  c_1^{-1} & = & \sum_{j = 1}^s \alpha_1(j), \\
  \label{eq:ct}
  c_t^{-1} & = & \sum_{j = 1}^s \sum_{i = 1}^s (\hat{\alpha}_{t -
    1}(i)p_{ij}) f_j(y_t, \theta_j), \hspace{1em} t = 2, \ldots, T, \\
  \label{eq:dT}
  d_T^{-1} & = & \sum_{i = 1}^s \beta_T(i) = s, \\
  \label{eq:dt}
  d_t^{-1} & = & \sum_{i = 1}^s \sum_{j = 1}^s p_{ij}f_j(y_{t + 1},
  \theta_j)\hat{\beta}_{t + 1}(j),
  \hspace{1em} t = T - 1, \ldots, 1,
\end{eqnarray}
then
\begin{eqnarray}
  \label{eq:alphabyscaled}
  \hat{\alpha}_t(j) & = & c_1 \ldots c_t \alpha_t(j) =
  \frac{\alpha_t(j)}{\sum_{j = 1}^s \alpha_t(j)}, \\
  \label{eq:betabyscaled}
  \hat{\beta}_t(i) & = & d_T \ldots d_t \beta_t(i) =
  \frac{\beta_t(i)}{\sum_{i = 1}^s \beta_t(i)},
\end{eqnarray}
for $i, j \in S$, $t = 1, \ldots, T$. The logarithm of likelihood may be
calculated using the first equality~(\ref{eq:likelihood})
and~(\ref{eq:alphabyscaled}) for $t = T$ as
\begin{equation}
  \label{eq:loglikelihood}
  \log \mathcal{L} = - \sum_{t = 1}^T \log c_t,
\end{equation}
since $\sum_{i = 1}^s \alpha_T(i) = (c_1 \ldots c_T)^{-1}$.
The values~(\ref{eq:gamma}) and~(\ref{eq:xi}) may be calculated as
\begin{eqnarray}
  \label{eq:gammabyscaled}
  \gamma_t(i) & = & \frac{\hat{\alpha}_t(i)\hat{\beta}_t(i)}{\sum_{i =
      1}^s \hat{\alpha}_t(i) \hat{\beta}_t(i)}, \hspace{1em} t = 1,
  \ldots, T, \\
  \label{eq:xibyscaled}
  \xi_t(i, j) & = & d_t\frac{\hat{\alpha}_t(i)\hat{\beta}_{t +
      1}(j)p_{ij} f_j(y_{t + 1}, \theta_j)}{\sum_{i = 1}^s
    \hat{\alpha}_t(i) \hat{\beta}_t(i)},
  \hspace{1em} t = 1, \ldots, T - 1
\end{eqnarray}
for $i, j \in S$.

The Baum-Welch
adjustments~(\ref{eq:baumwelchp}--\ref{eq:baumwelchsigma}) can be
calculated as above except for~(\ref{eq:baumwelchP}), which should be
calculated as
\begin{equation}
  \label{eq:baumwelchPscaled}
  \overline{p}_{ij} = \frac{\sum_{t = 1}^{T - 1} \frac{\hat{\alpha}_t(i)
      \hat{\beta}_{t + 1}(j) p_{ij} f_j(y_{t + 1}, \theta_j)}{\sum_{i =
        1}^s \hat{\alpha}_t(i) \hat{\beta}_t(i)}d_t }{\sum_{t = 1}^{T - 1}
    \gamma_t(i)}
\end{equation}
for $i, j \in S$.

The Viterbi algorithm needs not scaling, but what should be maximized
is logarithm of~(\ref{eq:bestseqprob}) rather
than~(\ref{eq:bestseqprob}) itself.

% TODO: Put citations in proper places.
References: \cite{rabiner-1989}, \cite{baum-petrie-soules-weiss-1970},
\cite{cappe-moulines-ryden-2005}.

\section{Forecast normal pseudo-residuals}

Forecast normal pseudo-residuals are defined as follows \cite [p.~97]
{zucchini-macdonald-2009}. If $X_t$ is a continuous random variable
with distribution function $F_{X_t}$, then $F_{X_t}(X_t)$ is uniformly
distributed on $(0, 1)$ and $u_t = \Pr(X_t \leq x_t) = F_{X_t}(x_t)$
is the uniform pseudo-residual. The random variable
$\Phi^{-1}(F_{X_t}(X_t))$ is distributed standard normal and
\[
z_t = \Phi^{-1}(u_t) = \Phi^{-1}(F_{X_t}(x_t))
\]
is the normal pseudo-residual. If we take
\[
F_{X_t}(x_t) = \Pr(X_t \leq x_t\ |\ \mathbf{X}^{(t - 1)} =
\mathbf{x}^{(t - 1)}),
\]
we get forecast normal pseudo-residuals, while taking
\[
F_{X_t}(x_t) = \Pr(X_t \leq x_t\ |\ \mathbf{X}^{(-t)} =
\mathbf{x}^{(-t)}),
\]
we get ordinary normal pseudo-residuals. Therefore, we calculate
density of forecast according to formula
\[
\Pr(X_t = x\ |\ \mathbf{X}^{(t - 1)} = \mathbf{x}^{(t - 1)})
= \frac{\alpha_{t - 1} \Gamma P(x) 1^T}{\alpha_{t - 1} 1^T}.
\]


\documentclass[11pt]{article}
\usepackage{enumitem}
\usepackage{amsmath}
\usepackage{booktabs}

\begin{document}
\title{MAT1830 --- Assignment 9}
\author{Dylan Pinn --- 24160547}
\maketitle

\section*{Question 1}

A biased coin flips heads with probability $\frac{4}{9}$ and tails with
probability $\frac{5}{9}$. The coin is flipped 90 times. What is the probability
that heads is flipped exactly 40 times?

\begin{align*}
  \Pr(H) &= \frac{4}{9} \\
  \Pr(T) &= \frac{5}{9} \\
  \Pr(X = k) &= \binom{n}{k} p^k {(1-p)}^{n-k} \text{ for } k \in \{ 0,
    \dots, n \} \\
  n &= 90 \\
  k &= 40 \\
  p &= \frac{4}{9} \\
  \Pr(X = 40) &= \binom{90}{40} {\frac{4}{9}}^{40} {(1-\frac{4}{9})}^{90-40} \\
  &= \frac{90!}{40!50!} \times \frac{4^{40}}{9^{40}} \times
  \frac{5^{50}}{9^{50}} \\
  &\approx 0.08
\end{align*}

\break{}

\section*{Question 2}

Cars pass through a road junction according to a Poisson distribution. An
average of 5 cars per minute pass through the junction.

\begin{enumerate}[label= (\alph*)]
  \item What is the probability that exactly one car passes through the
    junction in a certain minute?

  $\lambda = 5$ and using the Poisson distribution formula gives;

  \begin{align*}
    \Pr(X=k) &= \frac{{\lambda}^k {e}^{- \lambda}}{k!} \\
    \Pr(X=1) &= \frac{5^1 e^{-5}}{1!} \\
    &= \frac{5}{e^5} \\
    &\approx 0.03
  \end{align*}

  \item Write down the expected number of cars to pass through the junction in
    four minutes.

    20 cars in 4 minutes.

  \item What is the probability that exactly 10 cars pass through the junction
    in a certain four minute period?

    $\lambda = 20$ from part (b) and using the Poisson distribution gives;

    \begin{align*}
      \Pr(X=10) &= \frac{20^{10} e^{-20}}{10!} \\
      &= \frac{1600000000}{567 e^{20}} \\
      &\approx 0.0058
    \end{align*}
\end{enumerate}

\break{}
\section*{Question 3}

A random variable $Y$ can only take values in $ \{ -5, 0, 5 \}$. The expected
value of $Y$ is 0 and its variance is 24. Find the probability distribution of
$Y$.

\begin{center}
  \begin{tabular}{c c c c}
    \toprule
    $y$ & -5 & 0 & 5 \\
    $\Pr(Y=y)$ & $a$ & $b$ & $c$ \\
    \bottomrule
  \end{tabular}
\end{center}

We know that $a + b + c = 1$ and $-5 + 5c = 0$ which gives $a = c$.

\[ a{(-5)}^2 + b{(0)}^2 + c{(5)}^2 = 24 \]
\[ 25a + 25c = 24 \]
\[ 25(a + c) = 24 \]
\[ a + c = \frac{24}{25} \]
as $a = c$ \[ 2a = \frac{24}{25} \]
\[ a = \frac{24}{50} = \frac{12}{25} \]

We can substitute this into the original formula to give:

\[ a + b + c = 1 \]
\[ b = 1 - a - c \]
\[ = 1 - \frac{12}{24} - \frac{12}{24} \]
\[ = \frac{1}{25} \]

\begin{center}
  \begin{tabular}{c c c c}
    \toprule
    $y$ & -5 & 0 & 5 \\
    $\Pr(Y=y)$ & $\frac{12}{24}$ & $\frac{1}{24}$ & $\frac{12}{24}$ \\
    \bottomrule
  \end{tabular}
\end{center}

\break{}
\section*{Question 4}

Write down the first five values $(r_0, \dots, r_4 \text{ and } s_0, \dots,
s_4)$ of each of the following recursive sequences.

\begin{enumerate}[label = (\alph*)]
  \item $r_0 = 2$, $r_n = {(r_{n-1})}^2 - 3n + 1$ for all integers $n \geq 1$.

    \begin{align*}
      r_0 &= 2 \\
      r_1 &= {(r_0)}^2 - 3 \times 1 + 1 \\
      &= 2^2 - 3 + 1 \\
      &= 0 \\
      r_2 &= {(r_1)}^2 - 3 \times 2 + 1 \\
      &= 0^2 - 6 + 1 \\
      &= -7 \\
      r_3 &= {(r_2)}^2 - 3 \times 3 + 1 \\
      &= {(-7)}^2 - 9 + 1 \\
      &= 39 \\
      r_4 &= {(r_3)}^2 - 3 \times 4 + 1 \\
      &= 39^2 - 12 + 1 \\
      &= 1510
    \end{align*}

  \item $s_0 = 2$, $s_n = {(s_{n-1})}^2 + {(s_{n-2})}^2 + \cdots + {(s_0)}^2$
  for all integers $n \geq 1$.

    \begin{align*}
      s_0 &= 2 \\
      s_1 &= 2^2 \\
          &= 4 \\
      s_2 &= 4^2 + 2^2 \\
          &= 20 \\
      s_3 &= 20^2 + 4^2 + 2^2 \\
          &= 420 \\
      s_4 &= 420^2 + 20^2 + 4^2 + 2^2 \\
          &= 176820
    \end{align*}
\end{enumerate}

\end{document}


\chapter{Estimating the CDF and Statistical Functionals}

% 1
\begin{ex}
  Fix a value $x$.
  Let $\widehat{F}_n(x)=\frac{1}{n}\sum_{i=1}^nI(X_i\leq x)$ and note that then
  \[
    \E{\widehat{F}_n(x)}
    =\E{\frac{1}{n}\sum_{i=1}^nI(X_i\leq x)}
    =\frac{1}{n}\sum_{i=1}^n\E{I(X_i\leq x)}
    =\frac{1}{n}\sum_{i=1}^n\P{X_i\leq x}
    =F(x).
  \]
  Likewise,
  \begin{align*}
    \E{\widehat{F}^2_n(x)}
     & =\E{\left[\frac{1}{n}\sum_{i=1}^nI(X_i\leq x)\right]^2} \\
     & =\frac{1}{n^2}\E{\sum_{i=1}^nI^2(X_i\leq x)
      +\sum_{j=1}^n\sum_{\substack{k = 1                       \\ j \neq k}}^nI(X_j\leq x)I(X_k\leq x)}                                                          \\
     & =\frac{1}{n^2}\sum_{i=1}^n\E{I(X_i\leq x)}
    +\frac{1}{n^2}\sum_{j=1}^n\sum_{\substack{k = 1            \\ j \neq k}}^n\E{I(X_j\leq x)I(X_k\leq x)}                                                          \\
     & =\frac{F(x)}{n}+\frac{n+1}{n}F^2(x),
  \end{align*}
  and therefore
  \[
    \var{\widehat{F}_n(x)}
    =\E{\widehat{F}^2_n(x)}-\E{\widehat{F}_n(x)}^2
    =\frac{F(x)(1-F(x)}{n}.
  \]
  Thus,
  \begin{align*}
    \mse{\widehat{F}_n(x)}
     & =\left[\bias{\widehat{F}_n(x)}\right]^2+\var{\widehat{F}_n(x)}               \\
     & =\left[\E{\widehat{F}_n(x)}-\widehat{F}_n(x)\right]^2+\var{\widehat{F}_n(x)} \\
     & =\var{\widehat{F}_n(x)}                                                      \\
     & =\frac{F(x)(1-F(x))}{n}\to 0,
  \end{align*}
  and we can conclude that $\widehat{F}_n(x)\to F(x)$ in quadratic mean (and
  hence in probability as well).
\end{ex}

\begin{ex}
  The expected value of a $\text{Bernoulli}(p)$ variable is $p$, and therefore
  the plug-in estimator for $p$ is given by $\overline{X}_n$. Since $\var{X_i}
    =\sqrt{p(1-p)}$, the plug-in estimator for the standard error is given
  by
  \[
    \frac{\widehat{\sigma}}{\sqrt{n}}
    =\sqrt{\frac{\overline{X}_n(1-\overline{X}_n)}{n}}.
  \]

  Thus, a 90 percent confidence interval for $p$ is given by
  \[
    \widehat{p}\pm z_{0.05}\se(\widehat{p})
    =\overline{X}_n\pm 1.645\sqrt{\frac{\overline{X}_n(1-\overline{X}_n)}{n}}.
  \]

  The plug-in estimator for $p-q$ is $\overline{X}_n-\overline{Y}_n$. Since
  \[
    \var{\widehat{p}-\widehat{q}}
    =\var{\widehat{p}}+\var{\widehat{q}}
    =\frac{\overline{X}_n(1-\overline{X}_n)}{n}
    +\frac{\overline{Y}_m(1-\overline{Y}_m)}{m},
  \]
  it follows that
  \[
    \se(\widehat{p}-\widehat{q})
    =\sqrt{\frac{\overline{X}_n(1-\overline{X}_n)}{n}
      +\frac{\overline{Y}_m(1-\overline{Y}_m)}{m}},
  \]
  and therefore a 90 percent confidence interval for $p-q$ is given by
  \[
    \widehat{p}-\widehat{q}
    \pm z_{0.05}\se(\widehat{p}-\widehat{q})
    =\overline{X}_n-\overline{Y}_m\pm
    1.645\sqrt{\frac{\overline{X}_n(1-\overline{X}_n)}{n}
      +\frac{\overline{Y}_m(1-\overline{Y}_m)}{m}}.
  \]
\end{ex}

\begin{ex}~
  \inputminted{python}{../code/07-03.py}
  \inputminted{text}{../output/07-03.txt}
\end{ex}

\begin{ex}
  Fix an $x\in\R$. Let $Y_i=I(X_i\leq X)$. Then
  \[
    \widehat{F}_n(x)
    =\frac{1}{n}\sum_{i=1}^nI(X_i\leq x)
    =\frac{1}{n}\sum_{i=1}^nY_i,
  \]
  where $Y_i$ is a $\text{Bernoulli}(p)$ random variable with outcome $1$
  whenever $X_i \leq x$, i.e.\ with $p=F(x)$. Thus, $\widehat{F}_n(x)$
  is the average of $n$ such random variables and by the central limit theorem,
  \[
    \frac{\sqrt{n}(\widehat{F}_n(x)-\mu_Y)}{\sigma_Y}
    =\frac{\sqrt{n}(\widehat{F}_n(x)-F(x))}{F(x)(1-F(x))}
    \rightsquigarrow Z,
  \]
  or
  \[
    \sqrt{n}\left(\widehat{F}_n(x)-F(x)\right)
    \rightsquigarrow N\left(0, F(x)(1-F(x))\right).
  \]
\end{ex}

% 5
\begin{ex}
  We assume, without loss of generality, that $x\leq y$. Then, since
  \[
    \widehat{F}_n(x)\widehat{F}_n(y)
    =\left(\frac{1}{n}\sum_{i=1}^nI(X_i\leq x)\right)
    \left(\frac{1}{n}\sum_{i=1}^nI(X_i\leq y)\right)
    =\frac{1}{n^2}\sum_{i=1}^n\sum_{j=1}^nI(X_i\leq x)I(X_j\leq y),
  \]
  \begin{align*}
    \E{\widehat{F}_n(x)\widehat{F}_n(y)}
     & =\frac{1}{n^2}
    \sum_{i=1}^n\sum_{j=1}^n\E{I(X_i\leq x)I(X_j\leq y)}      \\
     & =\frac{1}{n^2}\sum_{i=1}^n\E{I(X_i\leq x)I(X_i\leq y)}
    +\frac{1}{n^2}\sum_{i=1}^n\sum_{\substack{j=1             \\ i\neq j}}^n\E{I(X_i\leq x)I(X_j\leq y)} \\
     & =\frac{1}{n}F(x)+\frac{n-1}{n}F(x)F(y),
  \end{align*}
  \begin{align*}
    \cov{\widehat{F}_n(x),\widehat{F}_n(y)}
     & =\E{\widehat{F}_n(x)\widehat{F}_n(y)}
    -\E{\widehat{F}_n(x)}\E{\widehat{F}_n(y)} \\
     & =\frac{1}{n}F(x)+\frac{n-1}{n}F(x)F(y)
    -F(x)F(y)                                 \\
     & =\frac{F(x)(1-F(y))}{n}.
  \end{align*}
\end{ex}

\begin{ex}
  Note that
  \begin{align*}
    \var{\widehat{\theta}}
     & =\var{\widehat{F}_n(b)-\widehat{F}_n(a)}                                                \\
     & =\var{\widehat{F}_n(b)}+\var{\widehat{F}_n(a)}-2\cov{\widehat{F}_n(b),\widehat{F}_n(a)} \\
     & =\frac{F(b)(1-F(b))}{n}
    +\frac{F(a)(1-F(a))}{n}
    +\frac{F(a)(1-F(b))}{n}                                                                    \\
     & =\frac{[F(b)-F(a)](1-[F(b)-F(a)])}{n}                                                   \\
     & =\frac{\theta(1-\theta)}{n}
  \end{align*}
  by Theorem 7.3 and Exercise 7.5.

  Hence,
  \[
    \sehat(\widehat{\theta})
    =\sqrt{\frac{\widehat{\theta}(1-\widehat{\theta})}{n}},
  \]
  and we can define a normal-based $1-\alpha$ interval by
  \[
    \widehat{\theta}\pm z_{\alpha/2}
    \sqrt{\frac{\widehat{\theta}(1-\widehat{\theta})}{n}}.
  \]
\end{ex}

\begin{ex}~
  \inputminted{python}{../code/07-07.py}
  \begin{figure}[H]
    \centering
    \includegraphics[scale=1.1]{../images/07-07}
    \caption{Graph of a 95\% confidence band for the empirical CDF.}
  \end{figure}
  \inputminted{text}{../output/07-07.txt}
\end{ex}

\begin{ex}~
  \inputminted{python}{../code/07-08.py}
  \inputminted{text}{../output/07-08.txt}
\end{ex}

\begin{ex}
  We have $\widehat{p}_1=0.9$ and $\widehat{p}_2=0.85$. Thus,
  $\widehat{\theta}=\widehat{p}_1-\widehat{p}_2=0.9-0.85=0.05$ with standard
  error
  \[
    \sqrt{\frac{\widehat{p}_1(1-\widehat{p}_1)}{n}
      +\frac{\widehat{p}_2(1-\widehat{p}_2)}{n}}
    =0.0466369,
  \]
  an 80 percent confidence interval
  \[
    \widehat{\theta}\pm z_{0.1}\se(\thetahat)
    =0.05\pm 1.28155\cdot 0.04664
    =0.05\pm 0.059768,
  \]
  and a 95 percent confidence interval
  \[
    \widehat{\theta}\pm z_{0.025}\se(\thetahat)
    =0.05\pm 1.96\cdot 0.04664
    =0.05\pm 0.091408.
  \]
\end{ex}

% 10
\begin{ex}~
  \inputminted{python}{../code/07-10.py}
  \inputminted{text}{../output/07-10.txt}
\end{ex}


\documentclass[]{article}
\usepackage{caption,subcaption,graphicx,float,url,amsmath,amssymb,tocloft,wasysym,amsthm,thmtools}
\usepackage[hidelinks]{hyperref}
\usepackage[toc,acronym,nonumberlist]{glossaries}
\setacronymstyle{long-short}
\usepackage{glossaries-extra}
\graphicspath{{figs/}}
\setlength{\cftsubsecindent}{0em}
\setlength{\cftsecnumwidth}{3em}
\setlength{\cftsubsecnumwidth}{3em}
\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}
\newtheorem{thm}{Theorem}
\newtheorem{cor}[thm]{Corollary}

%opening
\title{
	Computation in Complex Systems\\
	Peer Review Assignment\\
	Week 1
}

%\makeglossaries

\begin{document}

\maketitle

\tableofcontents

\section{Polynomials \& Exponentials}

\begin{quotation}
	Today, your computer can do $T$ steps in a week. According to Moore's law, next year, your computer will be able to do $2T$ steps in a week. How does doubling $T$ change the $n$ that can be computed in a week?
\end{quotation}

a) This year $T = n^2$. Next year my (new) computer can do $2T$ steps. Let $n^\prime$ denote the size of problem I can handle next year. Then we have:
\begin{align*}
	T =& n^2 \text{, and}\\
	2T =& (n^\prime)^2 \text{, from Moore's Law, whence}\\
	 (n^\prime)^2=& 2n^2 \text{, or, taking square roots of both sides}\\
	n^\prime=& \sqrt{2} n
\end{align*}
So $n$ changes by a factor of $\sqrt{2}$.

b) This year $T = 2^n$. Next year my (new) computer can do $2T$ steps. Once again let $n^\prime$ denote the size of problem I can handle next year. Then we have:

\begin{align*}
	T =&2^n \text{, and}\\
	2T =& 2^{n^\prime} \text{, from Moore's Law, whence}\\
	2^{n^\prime}=& 2\cdot 2^n\\
	=& 2^{n+1} \text{, so taking logarithms to base 2}\\
	n^\prime =& n+1
\end{align*}

\section{Divide \& Conquer}
Given:
\begin{align*}
	f(1)=&1 \numberthis \label{eq:hypothesis}\\
	f(n)=&2f(n-1)+1 \numberthis \label{eq:recurrence}
\end{align*}
I'll begin by computing the first few values:
\begin{table}[H]
	\begin{center}
		\begin{tabular}{|l|r|} \hline
			n&$f(n)$\\\hline
			1&1\\\hline
			2&3\\\hline
			3&7\\\hline
			4&15\\\hline
			5&32\\\hline
			6&63\\\hline
		\end{tabular}
	\end{center}
\end{table}
We already have enough values to formulate a hypothesis\footnote{this sounds better than ''guess''}--$f(n)=2^{n+1}-1$:
\begin{table}[H]
	\begin{center}
		\begin{tabular}[H]{|l|r|r|}\hline
			n&$f(n)$&$2^n-1$\\\hline
			1&1&2-1\\\hline
			2&3&4-1\\\hline
			3&7&8-1\\\hline
			4&15&16-1\\\hline
			5&32&32-1\\\hline
			6&63&64-1\\\hline
		\end{tabular}
	\end{center}
\end{table}

\begin{thm}
	If $f$ satisfies (\ref{eq:hypothesis}) and (\ref{eq:recurrence}), $f(n)=2^n-1$
\end{thm}

\begin{proof}[Mathematical Induction]

	From (\ref{eq:hypothesis}), $f(1)=1$ and (\ref{eq:recurrence}) gives $f(2)= 2f(1)+1 = 2\cdot1+1 =3$. But if $n=2$, $2^n-1=4-1=3$, so the hypothesis is correct for $n=1$.

	Assume the hypothesis is correct for a particular $n$. We want to show it is correct for $n+1$, i.e. that $f(n+1)=2^{n+1}-1$.
	\begin{align*}
		f(n)=&2^n-1	\text{ by hypothesis. We use (\ref{eq:recurrence})}\\
		f(n+1)=&2f(n) + 1 \\
		=& 2(2^n-1) + 1\\
		=& 2\cdot 2^n -2 +1\\
		=& 2^{n+1} -1
	\end{align*}
\end{proof}

Although this proof is valid, it doesn't shed much light on why $f$ should be close to an exact exponential. The following proof is motivated by the fact that \ref{eq:recurrence} is close to an exponential doubling assuming that $f(n)$ is large when $n$ is large. Can we find a quantity that is close to $f(n)$ and which satisfies an exact exponential recurrence relation. It turn out that we can.
\begin{proof}[Alternative]
	We define a new quantity:
	\begin{align*}
		g(n)\triangleq& f(n) +1 \\
		g(n+1) = & f(n+1) +1 \text{, so (\ref{eq:recurrence}) gives:}\\
		=& \big(2f(n)+1\big)+1\\
		=& \big[2\big(g(n) -1\big) +1\big] +1\\
		=& 2 g(n) \text{, so the exponential growth of $g(n)$ is obvious, i.e.} \\
		g(n) =& C\cdot 2^n\text{, for some constant $C>0$. Now (\ref{eq:hypothesis}) becomes:}\\
		g(1) =&2\text{, whence}\\
		C=& 1 \text{, so}\\
		g(n) =& 2^n \text{, and} \\
		f(n) =& 2^n - 1
	\end{align*}
\end{proof}
\begin{cor}
	\begin{align*}
		f(64) =& 2^{64}-1\\
		=& 36,893,488,147,419,103,231
	\end{align*}
\end{cor}

\begin{proof}
	The value was calculated by the following Python code:
	\begin{verbatim}
		N = 1
		for i in range(65):
	 	     N*=2
		print (N-1)
	\end{verbatim}
\end{proof}
% glossary
%\printglossaries

% bibliography go here

%\bibliographystyle{unsrt}
%\addcontentsline{toc}{section}{Bibliography}
%\bibliography{origins,wikipedia}

\end{document}


\section{The \minelement algorithm}
\Label{sec:minelement}

The \minelement  algorithm in the \cxx Standard Library \cite[\S 28.7.8]{cxx-17-draft}
searches the minimum in a general sequence.
The signature of our version of \minelement reads:

\begin{lstlisting}[style = acsl-block]

  size_type min_element(const value_type* a, size_type n);
\end{lstlisting}

The function \minelement finds the smallest element in the range \inl{a[0..n-1]}.
More precisely, it returns the unique valid index \inl{i} such that
\inl{a[i]} is minimal among the values \inl{a[0]}, \ldots,
\inl{a[n-1]}, and \inl{i} is the first position with that property.
The return value of \minelement is \inl{n} if and only if \inl{n == 0}.

We use the predicate \logicref{LowerBound} that
basically expresses that a given value is less or equal than all
elements of a given array (section).
%
Closely related to the predicate \LowerBound is the predicate \logicref{StrictLowerBound}.
%
We also use the predicate \logicref{MinElement} which states that the element
at a given index \inl{min} is a \emph{lower bound} of the sequence \inl{a[0..n-1]},
and, by construction, a member of that sequence.

\subsection{Formal specification of \minelement}

The following listing contains the specification of \specref{minelement}.
Note that we also use the predicate \logicref{StrictLowerBound} in order to
express that \minelement returns the \emph{first} minimum position in \inl{a[0..n-1]}.

\input{Listings/min_element.h.tex}

\clearpage

\subsection{Implementation of \minelement}

The implementation of \implref{minelement} uses the predicates \logicref{LowerBound}
and \logicref{StrictLowerBound} in its loop annotations.

\input{Listings/min_element.c.tex}

\clearpage


\subsection{Matrices over rings}\label{subsec:matrices_over_rings}

We will define and prove some fundamental notions about matrices. We will start with matrices over plain sets and end up with matrices over nontrivial noetherian commutative rings. This is about as general as we want to go without the underlying ring being a field. The main benefit is being able to work with the \hyperref[def:set_of_integers]{ring of integers} or more general semirings, like the \hyperref[def:tropical_semiring]{tropical semirings}.

\begin{definition}\label{def:array}\mimprovised
  Let \( S \) be any nonempty \hyperref[def:set]{set} and \( n_1, \ldots, n_k \) be \hyperref[def:integer_signum]{positive integers}. An \term{array} of shape \( n_1 \times \cdots \times n_k \) is a \hyperref[def:function]{function} with signature
  \begin{equation*}
    A: \set{ 1, 2, \ldots, n_1 } \times \ldots \times \set{ 1, 2, \ldots, n_k } \to S.
  \end{equation*}

  \enquote{Multi-dimensional array} is also used as a term, but we will avoid it because the terminology conflicts with \hyperref[thm:vector_space_dimension]{vector space dimensions}.

  We can regard \enquote{\( n_1 \times \cdots \times n_k \)} simultaneously as a convenient notation and as a \hyperref[def:cartesian_product]{Cartesian product} of finite \hyperref[def:ordinal]{ordinals} (modulo the fact that finite ordinals are zero-based).

  In particular:
  \begin{thmenum}
    \thmitem{def:array/matrix} A two-dimensional array of shape \( m \times n \) is usually called a \term{matrix}. Let \( A \) be an \( m \times n \)-matrix. We will denote \( A \) as
    \begin{equation*}
      A = \seq{ a_{i,j} }_{i,j=1}^{m,n}
    \end{equation*}
    or graphically as the table
    \begin{equation*}
      \begin{pmatrix}
        a_{1,1} & a_{1,2} & \cdots & a_{1,n} \\
        a_{2,1} & a_{2,2} & \cdots & a_{2,n} \\
        \vdots  & \vdots  & \ddots & \vdots  \\
        a_{m,1} & a_{m,2} & \cdots & a_{m,n}
      \end{pmatrix}.
    \end{equation*}

    \thmitem{def:array/square_matrix} A \term{square matrix} of order \( n \) is simply an \( n \times n \) matrix.

    \thmitem{def:array/column_vector} A \term{column vector} of dimension \( m \) is simply a \( m \times 1 \) matrix
    \begin{equation*}
      \begin{pmatrix}
        a_{1,1} \\
        \vdots  \\
        a_{m,1}
      \end{pmatrix}.
    \end{equation*}

    When \( S \) is a \hyperref[def:semiring]{semiring} \( R \), we often identify the set of all \( m \)-dimensional column vectors with the free semimodule \hyperref[def:standard_basis]{\( R^m \)}.

    \thmitem{def:array/row_vector} A \term{row vector} of dimension \( n \) is simply an \( 1 \times n \) matrix
    \begin{equation*}
      \begin{pmatrix}
        a_{1,1} & \cdots & a_{1,n}
      \end{pmatrix}.
    \end{equation*}
  \end{thmenum}
\end{definition}

\begin{remark}\label{rem:vector_etymology}
  In practice, the terms \enquote{vector}, \enquote{tuple} and \enquote{finite sequence} are used interchangeably. Formally, the concepts differ slightly:

  \begin{itemize}
    \item \enquote{Vector} refers to an element of a \hyperref[def:vector_space]{vector space} or, more generally, a \hyperref[def:semimodule]{semimodule}. \hyperref[def:array/column_vector]{Column vectors} and \hyperref[def:array/row_vector]{row vectors} are important special cases.

    \item Tuples are defined and discussed in \fullref{def:cartesian_product/tuple}. Tuples are technically \hyperref[def:cartesian_product/indexed_family]{indexed families} and the latter are defined without reference to functions, which we use to define both arrays and sequences.

    \item Sequences are defined and discussed in \fullref{def:sequence}. Formally, a finite sequence of length \( n \) is the same as an array of shape \( n \).
  \end{itemize}
\end{remark}

\begin{definition}\label{def:block_matrix}
  A \term{block matrix} is a \enquote{matrix of matrices}. That is, a matrix of the form
  \begin{equation*}
    \begin{pmatrix}
      A_{1,1} & A_{1,2} & \cdots & A_{1,n} \\
      A_{2,1} & A_{2,2} & \cdots & A_{2,n} \\
      \vdots  & \vdots  & \ddots & \vdots  \\
      A_{m,1} & A_{m,2} & \cdots & A_{m,n}
    \end{pmatrix},
  \end{equation*}
  where all \( A_{i,j} \) are matrices of compatible dimensions.

  We can write the block matrix
  \begin{equation*}
    \begin{pmatrix}
      A      & \cdots & B      \\
      \vdots & \ddots & \vdots \\
      C      & \cdots & D
    \end{pmatrix}
  \end{equation*}
  as
  \begin{equation*}
    \parens*
      {
        \begin{array}{ccc|c|ccc}
          a_{1,1}   & \cdots & a_{1,n_A}   & \cdots & b_{1,1}   & \cdots & b_{1,n_B} \\
          \vdots    & \ddots & \vdots      & \cdots & \vdots    & \ddots & \vdots \\
          a_{m_A,1} & \cdots & a_{m_A,n_A} & \cdots & b_{m_B,1} & \cdots & b_{m_B,n_B} \\
          \hline
          \vdots    & \vdots & \vdots      & \ddots & \vdots    & \vdots & \vdots \\
          \hline
          c_{1,1}   & \cdots & c_{1,n_C}   & \cdots & d_{1,1}   & \cdots & d_{1,n_D} \\
          \vdots    & \ddots & \vdots      & \cdots & \vdots    & \ddots & \vdots \\
          c_{m_C,1} & \cdots & c_{m_C,n_C} & \cdots & d_{m_D,1} & \cdots & d_{m_D,n_D} \\
        \end{array}
      }.
  \end{equation*}

  Given any matrix \( A = \seq{ a_{i,j} }_{i,j=1}^{n,m} \), we can represent it via its block matrix of rows
  \begin{equation*}
    \parens*
      {
        \begin{array}{c}
          a_{1,\anon*} \\
          \hline
          a_{2,\anon*} \\
          \hline
          \vdots \\
          \hline
          a_{n,\anon*}
        \end{array}
      },
  \end{equation*}
  and its block matrix of columns
  \begin{equation*}
    \parens*
      {
        \begin{array}{c|c|c|c}
          a_{\anon*,1} & a_{\anon*,2} & \cdots & a_{\anon*,m}
        \end{array}
      }
  \end{equation*}
\end{definition}

\begin{definition}\label{def:matrix_diagonal}
  The \term{main diagonal} of the matrix \( A = \seq{ a_{i,j} }_{i,j=1}^{m,n} \) is the sequence \( a_{1,1}, \ldots, a_{i,i}, \ldots, a_{k,k} \), where \( k \coloneqq \min\set{ m, n } \). The \term{antidiagonal} is instead \( a_{1,k}, \ldots, a_{i,k-i}, \ldots, a_{k,n-k} \). These can be visualized as follows:
  \begin{equation*}
    \begin{pmatrix}
      \fbox{\( a_{1,1} \)} & a_{1,2}                & \cdots & a_{1,k-1}                & \fbox{\( a_{k,k} \)} & \cdots \\
      a_{2,1}              & \fbox{\( a_{2,2} \)}   &        & \fbox{\( a_{2,k-1} \)}   & a_{2,k}              &        \\
      \vdots               &                        & \ddots &                          & \vdots               &        \\
      a_{k-1,1}            & \fbox{\( a_{k-1,2} \)} &        & \fbox{\( a_{k-1,k-1} \)} & a_{k-1,k}            & \cdots \\
      \fbox{\( a_{k,1} \)} & a_{k,2}                & \cdots & a_{k,k-1}                & \fbox{\( a_{k,k} \)} &        \\
      \vdots               &                        &        & \vdots                   &                      & \ddots
    \end{pmatrix}
  \end{equation*}

  Over a \hyperref[def:semiring]{semiring}, we say that a square matrix \term{diagonal} if all entries outside the main diagonal are zero. For brevity, we write
  \begin{equation*}
    \op{diag}(a_1, \ldots, a_n)
    \coloneqq
    \begin{pmatrix}
      a_1    & 0      & \cdots & 0      \\
      0      & a_2    & \cdots & 0      \\
      \vdots & \vdots & \ddots & \vdots \\
      0      & 0      & \cdots & a_n
    \end{pmatrix}
  \end{equation*}

  The notation \( \op{diag}(A) \) is also used to denote the sequence of diagonal entries of \( A \).
\end{definition}

\begin{proposition}\label{thm:matrix_algebra}
  Denote by \( R^{m \times n} \) the set of \( m \times n \) \hyperref[def:array/matrix]{matrices} over the \hyperref[def:semiring]{semiring} \( R \). We define three operations on matrices:
  \begin{thmenum}
    \thmitem{def:matrix_algebra/addition} We define \term{matrix addition} \( +: R^{m \times n} \times R^{m \times n} \to R^{m \times n} \) componentwise as
    \begin{equation*}
      \begin{pmatrix}
        a_{1,1} & \cdots & a_{1,n} \\
        \vdots  & \ddots & \vdots  \\
        a_{m,1} & \cdots & a_{m,n}
      \end{pmatrix}
      +
      \begin{pmatrix}
        b_{1,1} & \cdots & b_{1,n} \\
        \vdots  & \ddots & \vdots  \\
        b_{m,1} & \cdots & b_{m,n}
      \end{pmatrix}
      \coloneqq
      \begin{pmatrix}
        a_{1,1} + b_{1,1} & \cdots & a_{1,n} + b_{1,n} \\
        \vdots            & \ddots & \vdots            \\
        a_{m,1} + b_{m,1} & \cdots & a_{m,n} + b_{m,n}
      \end{pmatrix}.
    \end{equation*}

    With addition, \( R^{m \times n} \) becomes an \hyperref[def:monoid/commutative]{commutative monoid} with neutral element the \term{zero matrix}
    \begin{equation}\label{eq:def:matrix_algebra/matrix_multiplication/zero}
      \begin{pmatrix}
        0       & 0      & \cdots & 0      \\
        0       & 0      & \cdots & 0      \\
        \vdots  & \cdots & \ddots & \vdots \\
        0       &        & \cdots & 0
      \end{pmatrix}.
    \end{equation}

    \thmitem{def:matrix_algebra/scalar_multiplication} We define \term{scalar multiplication} \( \cdot: R \times R^{m \times n} \to R^{m \times n} \) as
    \begin{equation*}
       \lambda \cdot \begin{pmatrix}
        a_{1,1} & \cdots & a_{1,n} \\
        \vdots  & \ddots & \vdots  \\
        a_{m,1} & \cdots & a_{m,n}
      \end{pmatrix}
      \coloneqq
      \begin{pmatrix}
        \lambda a_{1,1} & \cdots & \lambda a_{1,n} \\
        \vdots          & \ddots & \vdots          \\
        \lambda a_{m,1} & \cdots & \lambda a_{m,n}
      \end{pmatrix}.
    \end{equation*}

    Under \hyperref[def:matrix_algebra/addition]{addition} and \hyperref[def:matrix_algebra/scalar_multiplication]{scalar multiplication}, \( R^{m \times n} \) becomes an \( R \)-\hyperref[def:semimodule]{semimodule}.

    \thmitem{def:matrix_algebra/matrix_multiplication} We define \term{matrix multiplication} in two steps. The definition is justified by \fullref{thm:matrix_and_linear_function_algebras}. First, if \( \seq{ a_{1,j} }_{j=1}^n \) is a \hyperref[def:array/row_vector]{row vector} and \( \seq{ b_{i,1} }_{i=1}^m \) is a \hyperref[def:array/column_vector]{column vector}, we define their \term{inner product} as
    \begin{equation*}
      a \cdot b \coloneqq \sum_{i=1}^n a_i b_i.
    \end{equation*}

    We can now define matrix multiplication \( \cdot: R^{m \times k} \times R^{k \times n} \to R^{m \times n} \) as
    \begin{equation*}
     \parens*
       {
         \begin{array}{c}
            a_{1,-} \\
            \hline
            a_{2,-} \\
            \hline
            \vdots \\
            \hline
            a_{m,-}
          \end{array}
        }
      \cdot
      \parens*
        {
          \begin{array}{c|c|c|c}
            b_{-,1} & b_{-,2} & \cdots & b_{-,n}
          \end{array}
        }
      \coloneqq
      \begin{pmatrix}
        a_{1,-} \cdot b_{-,1} & a_{1,-} \cdot b_{-,2} & \vdots & a_{1,-} \cdot b_{-,n} \\
        a_{2,-} \cdot b_{-,1} & a_{2,-} \cdot b_{-,2} & \vdots & a_{2,-} \cdot b_{-,n} \\
        \vdots                & \vdots                & \ddots & \vdots                \\
        a_{m,-} \cdot b_{-,1} & a_{m,-} \cdot b_{-,2} & \cdots & a_{m,-} \cdot b_{-,n}
      \end{pmatrix}.
    \end{equation*}

    If \( n \) and \( m \) are equal, \( R^{n \times n} \) becomes an \( R \)-\hyperref[def:algebra_over_semiring]{algebra} under \hyperref[def:matrix_algebra/matrix_multiplication]{matrix multiplication} with multiplicative identity the \term{identity matrix} of order \( n \)
    \begin{equation}\label{eq:def:matrix_algebra/matrix_multiplication/identity}
      \op{diag}(\underbrace{ 1, \cdots, 1 }_{n \T*{ones}})
      =
      \begin{pmatrix}
        1       & 0      & \cdots & 0      \\
        0       & 1      & \cdots & 0      \\
        \vdots  & \ddots & \ddots & \vdots \\
        0       &        & \cdots & 1
      \end{pmatrix}.
    \end{equation}
  \end{thmenum}
\end{proposition}
\begin{proof}
  The semimodule structure is inherited by the \hyperref[thm:semiring_is_semimodule]{semimodule structure} on \( R \). We will show that, if \( n = n \), matrix multiplication is associative and bilinear. Fix matrices
  \begin{equation*}
    \begin{aligned}
      A = \seq{ a_{i,j} }_{i,j=1}^{m,k} && B = \seq{ b_{i,j} }_{i,j=1}^{k,l} && C = \seq{ c_{i,j} }_{i,j=1}^{l,n}.
    \end{aligned}
  \end{equation*}

  \SubProofOf[def:magma/associative]{associativity} The \( (i, j) \)-th entry in \( D \coloneqq (AB)C \) is
  \begin{equation*}
    d_{i,j} = \sum_{s=1}^n \parens*{ \sum_{r=1}^n a_{i,r} \cdot b_{r,s} } \cdot c_{s,j}.
  \end{equation*}

  Due to distributivity,
  \begin{equation*}
    d_{i,j}
    =
    \sum_{s=1}^n \sum_{r=1}^n a_{i,r} \cdot b_{r,s} \cdot c_{s,j}
    =
    \sum_{r=1}^n a_{i,r} \cdot \parens*{ \sum_{s=1}^n b_{r,s} \cdot c_{s,j} },
  \end{equation*}
  which is the \( (i, j) \)-th entry in \( A(BC) \).

  Therefore, \( (AB)C = A(BC) \).

  \SubProofOf[def:semimodule/homomorphism/additive]{additivity} Again due to distributivity,
  \begin{equation*}
    \sum_{r=1}^n \parens*{ a_{i,r} + b_{i,r} } \cdot c_{r,j}
    =
    \sum_{r=1}^n a_{i,r} \cdot c_{r,j} + \sum_{r=1}^n b_{i,r} \cdot c_{r,j}.
  \end{equation*}

  Therefore, \( (A + B)C = AC + BC \). The proof that \( A(B + C) = AB + AC \) is analogous.

  \SubProofOf[def:semimodule/homomorphism/homogeneity]{homogeneity} Again due to distributivity,
  \begin{equation*}
    t \cdot \sum_{r=1}^n a_{i,r} \cdot b_{r,j}
    =
    \sum_{r=1}^n (t \cdot a_{i,r}) \cdot b_{r,j}
    =
    \sum_{r=1}^n a_{i,r} \cdot (t \cdot b_{r,j}).
  \end{equation*}

  Therefore, \( t(AB) = (tA)B = A(tB) \).
\end{proof}

\begin{example}\label{ex:matrix_multiplication_is_noncommutative}
  For \( n > 1 \), the \hyperref[thm:matrix_algebra]{matrix algebra} \( R^{n \times n} \) is a noncommutative ring. Consider the following example:
  \begin{align*}
    \begin{pmatrix}
      0 & 0 \\
      0 & 1
    \end{pmatrix}
    \begin{pmatrix}
      1 & 0 \\
      1 & 0
    \end{pmatrix}
    &=
    \begin{pmatrix}
      0 & 0 \\
      1 & 0
    \end{pmatrix},
    \\
    \begin{pmatrix}
      1 & 0 \\
      1 & 0
    \end{pmatrix}
    \begin{pmatrix}
      0 & 0 \\
      0 & 1
    \end{pmatrix}
    &=
    \begin{pmatrix}
      0 & 0 \\
      0 & 0
    \end{pmatrix}.
  \end{align*}
\end{example}

\begin{remark}\label{rem:matrices_as_functions}
  Let \( R \) be a \hyperref[def:ring/commutative]{commutative ring} and let \( e_1, \ldots, e_n \) be the \hyperref[def:standard_basis]{standard basis} of \( R \). The \hyperref[def:basis_decomposition]{coordinate projections} \( \pi_{e_1}, \ldots, \pi_{e_n} \) allow us to identify \( R^n \) with the module \( R^{n \times 1} \) of \hyperref[def:array/column_vector]{column vectors} by regarding the vector \( x \) from \( R^n \) as the column vector
  \begin{equation*}
    \begin{pmatrix}
      \pi_{e_1}(x) \\
      \vdots \\
      \pi_{e_n}(x)
    \end{pmatrix}.
  \end{equation*}

  Under this identification, the columns on the identity matrix \eqref{eq:def:matrix_algebra/matrix_multiplication/identity} are precisely the column vectors of the \hyperref[def:standard_basis]{standard basis}.

  Let \( A = \seq{ a_{ij} }_{i,j=1}^{m,n} \) be an \( m \times n \) matrix over \( R \). If we regard \( R^n \) as a set of column vectors, then \hyperref[thm:matrix_algebra/matrix_multiplication]{matrix multiplication} allows us to regard \( A \) as the function \( x \mapsto Ax \), which maps column vectors from \( R^n \) to column vectors in \( R^m \).

  This justifies using juxtaposition for application of linear maps, e.g. \( Lx \) rather than \( L(x) \).

  Conversely, let \( e_1, \ldots, e_n \) be the standard basis of \( R^n \) and \( f_1, \ldots, f_m \) --- of \( R^m \). The linear map \( L: R^n \to R^m \) corresponds to the following matrix:
  \begin{equation*}
    \begin{pmatrix}
      \pi_{e_1}(L f_1) & \cdots & \pi_{e_1}(L f_1) \\
      \vdots           & \ddots & \vdots       \\
      \pi_{e_n}(L f_m) & \cdots & \pi_{e_n}(L f_m)
    \end{pmatrix}.
  \end{equation*}
\end{remark}

\begin{proposition}\label{thm:matrix_and_linear_function_algebras}
  For a \hyperref[def:ring/commutative]{commutative ring} \( R \), the \hyperref[def:matrix_algebra]{matrix algebra} \( R^{m \times n} \) is \hyperref[def:algebra_over_semiring/homomorphism]{isomorphic} to the \hyperref[thm:functions_over_algebra]{linear function algebra} \( \hom(R^n, R^m) \)\footnote{Note that the maps are from \( R^n \) to \( R^m \) and not vice versa}.
\end{proposition}
\begin{proof}
  Follows from our discussion in \fullref{rem:matrices_as_functions} due to linearity.
\end{proof}

\begin{remark}\label{rem:double_index_maps}
  We want to be able to map single indices to double indices and vice versa, for example for the purpose of \fullref{thm:matrix_spaces_are_free_modules}. As an example, we want to be able to \enquote{linearize} an \( m \times n \) matrix such as the \( 2 \times 3 \) matrix
  \begin{equation}\label{eq:rem:double_index_maps/example/matrix}
    \begin{pmatrix}
      1 & 2 & 3 \\
      4 & 5 & 6
    \end{pmatrix}
  \end{equation}
  into the tuple
  \begin{equation}\label{eq:rem:double_index_maps/example/row_major}
    (1, 2, 3, 4, 5, 6)
  \end{equation}
  and vice versa. This is called \term{row-major order} of the elements of a matrix. The \term{column-major order} would instead be
  \begin{equation}\label{eq:rem:double_index_maps/example/column_major}
    (1, 4, 2, 5, 3, 6).
  \end{equation}

  Let \( m \) and \( n \) be \hyperref[def:integer_signum]{positive integers}. We will explicitly define functions for linearizing a matrix like \eqref{eq:rem:double_index_maps/example/matrix} into its row-major order \eqref{eq:rem:double_index_maps/example/row_major}. Consider the sets
  \begin{align*}
    S &\coloneqq \overbrace{ \set{ 1, \ldots, mn - 1, mn } }^{\T{single indices}}
    \\
    D &\coloneqq \underbrace{ \set{ 1, \ldots, m } \times \set{ 1, \ldots, n } }_{\T{double indices}}
  \end{align*}
  and the mutually inverse operations
  \begin{align}
    &\begin{aligned}\label{eq:rem:double_index_maps/sharp}
      &\sharp: S \to D \\
      &\sharp(k) \coloneqq \parens[\Big]{ \quot(k - 1, m) + 1, \rem(k - 1, m) + 1 } \\
    \end{aligned}
    \\[0.5\baselineskip]
    &\begin{aligned}\label{eq:rem:double_index_maps/flat}
      &\flat: D \to S \\
      &\flat(i, j) \coloneqq (i - 1) \cdot m + (j - 1) + 1.
    \end{aligned}
  \end{align}

  The operation \( \sharp \) encodes the matrix \eqref{eq:rem:double_index_maps/example/matrix} into its row-major order \eqref{eq:rem:double_index_maps/example/row_major} and \( \flat \) does the opposite. Both operations are trivial except for the shifting needed in to allow us to use \hyperref[def:euclidean_domain]{remainders and quotients}.

  We can easily verify that \( \sharp \) is a \hyperref[def:morphism_invertibility/left_invertible]{left inverse} of \( \flat \) (note that \( j < m \)):
  \begin{align*}
    \sharp(\flat(i, j))
    &=
    \sharp\parens[\Big]{ (i - 1) \cdot m + (j - 1) + 1 }
    = \\ &=
    \parens[\Big]{ \quot(\cdots, m) + 1, \rem(\cdots, m) + 1 }
    = \\ &=
    \parens[\Big]{ (i - 1) + 1, (j - 1) + 1 }
    = \\ &=
    (i, j).
  \end{align*}

  We can just as easily verify that \( \flat \) is a \hyperref[def:morphism_invertibility/right_invertible]{right inverse} of \( \sharp \):
  \begin{align*}
    \flat(\sharp(k))
    &=
    \flat\parens[\Big]{ \quot(k, m) + 1, \rem(k, m) + 1 }
    = \\ &=
    \quot(k, m) \cdot m + \rem(k, m)
    = \\ &=
    k.
  \end{align*}

  Hence, \( \sharp \) is fully invertible with inverse \( \flat \). By \fullref{thm:function_invertibility_categorical/fully_invertible}, it is bijective.
\end{remark}

\begin{proposition}\label{thm:matrix_spaces_are_free_modules}
  The \hyperref[thm:matrix_algebra]{matrix algebra} \( R^{m \times n} \) is isomorphic as a \hyperref[def:semimodule]{semimodule} to \( R^{mn} \).
\end{proposition}
\begin{proof}
  \Fullref{rem:double_index_maps} gives us a semimodule isomorphism between \( m \times n \) matrices and \( mn \)-dimensional column vectors when extended to linear maps via \fullref{thm:free_semimodule_universal_property}.
\end{proof}

\begin{definition}\label{def:matrix_determinant}\mcite[215]{Knapp2016BasicAlgebra}
  The \term{determinant} for the \hyperref[thm:matrix_algebra]{matrix algebra} \( R^{n \times n} \) over the \hyperref[def:semiring/commutative]{commutative semiring} \( R \) is the function
  \begin{equation}\label{eq:def:matrix_determinant}
    \begin{aligned}
      &\det: R^{n \times n} \to R \\
      &\det(\seq{ a_{i,j} }_{i,j=1}^n) \coloneqq \sum_{\sigma \in S_n} \sgn(\sigma) \prod_{i=1}^n a_{i,\sigma(i)},
    \end{aligned}
  \end{equation}
  where \( S_n \) is the \hyperref[def:symmetric_group]{symmetric group} and \( \sgn \) is the \hyperref[def:permutation_parity]{sign} of the permutation \( \sigma \).

  See the proof of \fullref{thm:determinant_on_columns} for a justification of the definition.
\end{definition}

\begin{definition}\label{def:symmetric_function}\mimprovised
  Given a function \( f: X^n \to Y \), where \( X \) and \( Y \) are \hyperref[def:set]{plain sets}, we say that \( f \) is \term{symmetric} if, for any \hyperref[def:symmetric_group/permutation]{permutation} \( \sigma \in S_n \), we have
  \begin{equation*}
    f(x_1, \ldots, x_n) = f(x_{\sigma(1)}, \ldots, x_{\sigma(n)}).
  \end{equation*}

  A permutation can be decomposed into \hyperref[def:symmetric_group/transposition]{transpositions} due to \fullref{thm:permutation_decomposition_existence}. Hence, the above condition reduces to the simpler condition of \( f \) being invariant with respect to swapping any two arguments. That is,
  \begin{equation*}
    f(\ldots, x_{i-1}, \fbox{\( x_i \)}, x_{i+1}, \cdots, x_{j-1}, \fbox{\( x_j \)}, x_{j+1}, \ldots)
    =
    f(\ldots, x_{i-1}, \fbox{\( x_j \)}, x_{i+1}, \cdots, x_{j-1}, \fbox{\( x_i \)}, x_{j+1}, \ldots).
  \end{equation*}

  In the case where \( n = 2 \), this reduces to the simple condition
  \begin{equation*}
    f(x, y) = f(y, x).
  \end{equation*}

  Symmetric functions should not be confused with symmetric binary relations defined in \fullref{def:binary_relation/symmetric}.
\end{definition}

\begin{definition}\label{def:antisymmetric_function}\mimprovised
  Given a function \( f: X^n \to Y \), where \( X \) is a \hyperref[def:set]{plain set} and \( Y \) is an \hyperref[rem:additive_magma]{additive group}, we say that \( f \) is \term{antisymmetric} if, for any \hyperref[def:symmetric_group/permutation]{permutation} \( \sigma \in S_n \), we have
  \begin{equation*}
    f(x_1, \ldots, x_n) = \sgn(\sigma) \cdot f(x_{\sigma(1)}, \ldots, x_{\sigma(n)}).
  \end{equation*}

  A permutation can be decomposed into \hyperref[def:symmetric_group/transposition]{transpositions} due to \fullref{thm:permutation_decomposition_existence}. Hence, the above condition reduces to the simpler condition of \( f \) changing sign when swapping any two arguments. That is,
  \begin{equation*}
    f(\ldots, x_{i-1}, \fbox{\( x_i \)}, x_{i+1}, \cdots, x_{j-1}, \fbox{\( x_j \)}, x_{j+1}, \ldots)
    =
    -f(\ldots, x_{i-1}, \fbox{\( x_j \)}, x_{i+1}, \cdots, x_{j-1}, \fbox{\( x_i \)}, x_{j+1}, \ldots).
  \end{equation*}

  In the case where \( n = 2 \), this reduces to the simple condition
  \begin{equation*}
    f(x, y) = -f(y, x).
  \end{equation*}

  Antisymmetric functions should not be confused with antisymmetric binary relations defined in \fullref{def:binary_relation/antisymmetric}.
\end{definition}

\begin{definition}\label{def:alternating_function}\mimprovised
  Given a commutative ring \( R \), and \( R \)-module \( M \) and a \hyperref[def:multilinear_function]{multilinear function} \( f: M \to R \), we say that \( f \) is \term{alternating} if, \( x_i = x_j \) implies that
  \begin{equation*}
    f(x_1, \ldots, x_i, \ldots, x_j, \ldots x_n) = 0.
  \end{equation*}
\end{definition}

\begin{proposition}\label{thm:alternating_multilinear_is_antisymmetric}
  If a \hyperref[def:multilinear_function]{multilinear map} is \hyperref[def:alternating_function]{alternating}, it is \hyperref[def:antisymmetric_function]{antisymmetric}. The converse holds if \( 2 \) is a unit.
\end{proposition}
\begin{proof}
  \SufficiencySubProof If \( f \) is an alternating multilinear map, then
  \begin{equation*}
    0
    =
    f(\cdots, x_i + x_j, \cdots, x_i + x_j, \cdots)
    =
    f(\cdots, x_i, \cdots, x_j, \cdots)
    +
    f(\cdots, x_j, \cdots, x_i, \cdots).
  \end{equation*}

  Therefore,
  \begin{equation*}
    f(\cdots, x_i, \cdots, x_j, \cdots)
    =
    -f(\cdots, x_j, \cdots, x_i, \cdots).
  \end{equation*}

  \NecessitySubProof If \( f \) is an antisymmetric multilinear map, then
  \begin{equation*}
    0
    =
    f(\cdots, x_i + x_i, x_i, \cdots)
    =
    2 f(\cdots, x_i, x_i, \cdots).
  \end{equation*}

  If \( 2 \) is a unit, this implies
  \begin{equation*}
    f(\cdots, x_i, x_i, \cdots) = 0.
  \end{equation*}
\end{proof}

\begin{proposition}\label{thm:determinant_on_columns}
  In the \hyperref[thm:matrix_algebra]{matrix algebra} \( R^{n \times n} \) over the commutative ring \( R \), the determinant function \( \det: R^{n \times n} \to R \) can be regarded as a function that maps \( n \) column vectors from \( R^n \) to \( R \). That is,
  \begin{equation}\label{eq:thm:determinant_on_columns}
    \det(v_1, \cdots, v_n) = \sum_{\sigma \in S_n} \sgn(\sigma) \prod_{i=1}^n \pi_{\sigma(i)} (v_i).
  \end{equation}

  The determinant is an \hyperref[def:alternating_function]{alternating} \hyperref[def:multilinear_function]{multilinear function} on columns. Furthermore, it is the unique alternating multilinear function \( f(v_1, \ldots, v_n) \) such that \( f(e_1, \ldots, e_n) = 1 \), where \( e_1, \ldots, e_n \) are vectors of the \hyperref[def:standard_basis]{standard basis} in \( R^n \).
\end{proposition}
\begin{proof}
  Let \( \pi_1, \ldots, \pi_n \) be the \hyperref[def:basis_decomposition]{projection functionals} corresponding to the \hyperref[def:standard_basis]{standard basis} \( e_1, \ldots, e_n \).

  \SubProof{Proof of multilinearity} Due to linearity of the coordinate projection functionals \( \pi_i \) and due to distributivity in \( R \), for every \( j \) we have
  \begin{align*}
    &\phantom{{}={}}
    \det(\cdots, v_{j-1}, ty + rz, v_{j+1}, \cdots)
    = \\ &=
    \sum_{\sigma \in S_n} \sgn(\sigma) \cdot \pi_{\sigma(j)} (ty + rz) \prod_{i \neq j} \pi_{\sigma(i)} (v_i)
    = \\ &=
    t \sum_{\sigma \in S_n} \sgn(\sigma) \cdot \pi_{\sigma(j)} (y) \prod_{i \neq j} \pi_{\sigma(i)} (v_i) + r \sum_{\sigma \in S_n} \sgn(\sigma) \cdot \pi_{\sigma(j)} (z) \prod_{i \neq j} \pi_{\sigma(i)} (v_i)
    = \\ &=
    t \cdot \det(\cdots, y, \cdots) + r \cdot \det(\cdots, z, \cdots).
  \end{align*}

  \SubProof{Proof of alternation} If \( v_i = v_j \), then for every even (resp. odd) permutation \( \sigma \), the permutation \( \cycle{i, j} \bincirc \sigma \) is odd (resp. even), and hence they cancel out in the sum \eqref{eq:thm:determinant_on_columns}. This holds for every permutation, hence it remains for the determinant to be zero.

  \SubProof{Proof of \( \det(I_n) = 1 \)} Note that
  \begin{equation*}
    \prod_{i=1}^n \pi_i (e_{\sigma(i)}) \neq 0
  \end{equation*}
  if and only if \( i = \sigma(i) \) for every \( i = 1, \ldots, n \). This only holds for the identity permutation, hence
  \begin{equation*}
    \det(e_1, \ldots, e_n) = \prod_{i=1}^n \pi_i(e_i) = \prod_{i=1}^n 1 = 1.
  \end{equation*}

  \SubProof{Proof of uniqueness} Suppose that \( f(v_1, \ldots, v_n) \) is an alternating multilinear function such that \( f(e_1, \ldots, e_n) = 1 \).

  For an arbitrary column vector \( v_j \) in \( R^n \), we have
  \begin{equation*}
    v_j = \sum_{i=1}^n \pi_i(v_j) \cdot e_i.
  \end{equation*}

  Then
  \begin{align*}
    f(v_1, \ldots, v_n)
    &=
    f\parens*{ \sum_{i_1=1}^n \pi_{i_1}(v_1) \cdot e_{i_1}, \ldots, \sum_{i_n=1}^n \pi_{i_n}(v_n) \cdot e_{i_n} }
    = \\ &=
    \sum_{i_1=1}^n \pi_{i_1}(v_1) \cdots \sum_{i_n=1}^n \pi_{i_n}(v_n) f(e_{i_1}, \ldots, e_{i_n})
    = \\ &=
    \sum_{\sigma \in S_n} \pi_{\sigma(i)}(v_i) f(e_{\sigma(1)}, \ldots, e_{\sigma(n)}).
  \end{align*}

  The last step is valid because \( f \) is \hyperref[def:alternating_function]{alternating} and thus \( f(e_{i_1}, \cdots, e_{i_n}) \) is zero when not all of \( i_1, \ldots, i_n \) are distinct, and they are necessarily distinct if the indices are given by a permutation from \( S_n \).

  Finally, since \( f \) is \hyperref[def:antisymmetric_function]{antisymmetric} due to \fullref{thm:alternating_multilinear_is_antisymmetric},
  \begin{equation*}
    f(e_{\sigma(1)}, \ldots, e_{\sigma(n)}) = \sgn(\sigma) \underbrace{f(e_1, \ldots, e_n)}_{1 \T*{by assumption}} = \sgn(\sigma).
  \end{equation*}

  Therefore,
  \begin{equation*}
    f(v_1, \ldots, v_n) = \det(v_1, \ldots, v_n).
  \end{equation*}
\end{proof}

\begin{definition}\label{def:transpose_matrix}
  The \term{transpose matrix} of
  \begin{equation*}
    A = \begin{pmatrix}
      a_{1,1} & a_{1,2} & \cdots & a_{1,n} \\
      a_{2,1} & a_{2,2} & \cdots & a_{2,n} \\
      \vdots  & \vdots  & \ddots & \vdots  \\
      a_{m,1} & a_{m,2} & \cdots & a_{m,n}
    \end{pmatrix}
  \end{equation*}
  is defined as
  \begin{equation*}
    A^T = \begin{pmatrix}
      a_{1,1} & a_{1,2} & \cdots & a_{n,1} \\
      a_{2,1} & a_{2,2} & \cdots & a_{n,2} \\
      \vdots  & \vdots  & \ddots & \vdots  \\
      a_{1,m} & a_{2,m} & \cdots & a_{n,m}
    \end{pmatrix}.
  \end{equation*}

  A matrix that is equal to its transpose is called \term{symmetric}.
\end{definition}

\begin{proposition}\label{thm:def:matrix_determinant}
  In the \hyperref[thm:matrix_algebra]{matrix algebra} \( R^{n \times n} \) over the commutative ring \( R \), the \hyperref[def:matrix_determinant]{determinant} as function on matrices has the following basic properties:
  \begin{thmenum}
    \thmitem{thm:def:matrix_determinant/transpose} \( \det(A^T) = \det(A) \).
    \thmitem{thm:def:matrix_determinant/homogeneous} \( \det(tA) = t^n \cdot \det(A) \).
    \thmitem{thm:def:matrix_determinant/homomorphism}\mcite[sec. 6.7]{Тыртышников2004Лекции} \( \det(AB) = \det(A) \cdot \det(B) \).

    That is, \( \det: R^{n \times n} \to R \) is a \hyperref[def:monoid/homomorphism]{monoid homomorphism} from the \hyperref[def:semiring]{multiplicative monoid} of the ring \( R^{n \times n} \) to the multiplicative monoid of \( R \).
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{thm:def:matrix_determinant/transpose} The inverse of any permutation in \( S_n \) is also a permutation in \( S_n \), hence
  \begin{equation*}
    \det(A^T)
    =
    \sum_{\sigma \in S_n} \sgn(\sigma) \prod_{i=1}^n a_{\sigma(i),i}
    =
    \sum_{\sigma \in S_n} \sgn(\sigma^{-1}) \prod_{i=1}^n a_{i,\sigma^{-1}(i)}
    =
    \det(A).
  \end{equation*}

  \SubProofOf{thm:def:matrix_determinant/homogeneous} Follows from \fullref{thm:determinant_on_columns}.

  \SubProofOf{thm:def:matrix_determinant/homomorphism} The \( j \)-th column of the product \( C = AB \) is
  \begin{equation*}
    c_{\anon*,j}
    =
    \sum_{i=1}^n b_{i,j} a_{\anon*,i}
    =
    \begin{pmatrix}
      \sum_{i=1}^n a_{1,i} b_{i,j} \\
      \vdots \\
      \sum_{i=1}^n a_{n,i} b_{i,j} \\
    \end{pmatrix}.
  \end{equation*}

  Since the determinant is a multilinear function on columns,
  \begin{balign*}
    \det(c_{\anon*,1}, \cdots, c_{\anon*,n})
    &=
    \det\parens*{ \sum_{i_1=1}^n b_{i_1,1} a_{\anon*,i_1}, \cdots, \sum_{i_n=1}^n b_{i_n,n} a_{\anon*,i_n} }
    = \\ &=
    \sum_{i_1=1}^n b_{i_1,1} \det\parens*{ a_{\anon*,i_1}, \cdots, \sum_{i_n=1}^n b_{i_n,n} a_{\anon*,i_n} }
    = \\ &=
    \sum_{i_1=1}^n b_{i_1,1} \cdots \sum_{i_n=1}^n b_{i_n,n} \det(a_{\anon*,i_1}, \cdots, a_{\anon*,i_n})
    = \\ &=
    \sum_{i_1=1}^n \cdots \sum_{i_n=1}^n b_{i_1,1} \cdots b_{i_n,n} \det(a_{\anon*,i_1}, \cdots, a_{\anon*,i_n}).
  \end{balign*}

  Since the determinant is \hyperref[def:alternating_function]{alternating} on columns, \( \det(a_{\anon*,i_1}, \cdots, a_{\anon*,i_n}) \) is zero when not all of \( i_1, \ldots, i_n \) are distinct. They are necessarily distinct if the indices are given by a permutation from \( S_n \). Therefore,
  \begin{balign*}
    \det(AB)
    &=
    \sum_{\sigma \in S_n} \prod_{i=1}^n a_{i,\sigma(i)} \cdot \sigma(a_{\anon*, \sigma(1)}, \ldots, a_{\anon*, \sigma(n)})
    = \\ &=
    \sum_{\sigma \in S_n} \prod_{i=1}^n a_{i,\sigma(i)} \cdot \sgn(\sigma) \cdot \sigma(a_{\anon*, 1}, \ldots, a_{\anon*, n})
    = \\ &=
    \det(B) \det(A).
  \end{balign*}
\end{proof}

\begin{definition}\label{def:submatrix}\mimprovised
  If for the matrices \( A = \seq{ a_{i,j} }_{i,j=1}^{m,n} \) and \( B = \seq{ b_{i,j} }_{i,j=1}^{k,k} \) over a commutative ring there exist \hyperref[def:partially_ordered_set/homomorphism]{monotone functions}
  \begin{align*}
    &h: \set{ 1, \ldots, k } \to \set{ 1, \ldots, m }, \\
    &w: \set{ 1, \ldots, l } \to \set{ 1, \ldots, n },
  \end{align*}
  such that, for every \( i = 1, \ldots, k \) and \( j = 1, \ldots, l \) we have
  \begin{equation*}
    b_{i,j} = a_{h(i),w(j)}.
  \end{equation*}
\end{definition}

\begin{definition}\label{def:matrix_minor}\mimprovised
  A \term{minor} of a matrix is a \hyperref[def:matrix_determinant]{determinant} of a square \hyperref[def:submatrix]{submatrix}.
\end{definition}

\medskip

\begin{theorem}[Laplace expansion]\label{thm:laplace_expansion}\mcite[prop. 2.36]{Knapp2016BasicAlgebra}
  For a square matrix \( A = \seq{ a_{i,j} }_{i,j=1}^{n,n} \) over a commutative ring and a row index \( i \), we have
  \begin{equation*}
    \det A = \sum_{j=1}^n (-1)^{i + j} a_{i,j} \det A_{i,j},
  \end{equation*}
  where \( A_{i,j} \) is the \hyperref[def:submatrix]{submatrix} of \( A \) obtained by removing the \( i \)-th row and the \( j \)-th column.

  By \fullref{thm:def:matrix_determinant/transpose}, we can also expand along a column rather than a row.
\end{theorem}
\begin{proof}
  Denote the ring by \( R \). We will show that, for the \( i \)-th row,
  \begin{equation*}
    \begin{aligned}
      &\Phi: R^{n \times n} \to R, \\
      &\Phi(A) \coloneqq \sum_{j=1}^n (-1)^{i + j} a_{i,j} \det A_{i,j}.
    \end{aligned}
  \end{equation*}
  is an \hyperref[def:alternating_function]{alternating} \hyperref[def:multilinear_function]{multilinear function} on columns.

  Multilinearity follows from the multilinearity of determinants. For proving alternation, suppose that the \( k \)-th and \( l \)-th columns are equal. Then
  \begin{equation*}
    \Phi(A) = (-1)^{i + k} a_{i,k} \det A_{i,k} + (-1)^{i + l} a_{i,l} \det A_{i,l}.
  \end{equation*}

  The matrix \( A_{i,l} \) can be obtained from \( A_{i,k} \) by swapping \( \abs{k - l} \) columns. Since determinants are antisymmetric, it follows that
  \begin{equation*}
    \det A_{i,l} = (-1)^{k - l} \det A_{i,k}.
  \end{equation*}

  Furthermore, \( a_{i,k} = a_{i,l} \). Therefore,
  \begin{equation*}
    \Phi(A) = (-1)^{i + k} a_{i,k} \det A_{i,k} + (-1)^{(i + l) + (k - l)} a_{i,k} \det A_{i,k} = 0.
  \end{equation*}
\end{proof}

\begin{definition}\label{def:adjugate_matrix}\mimprovised
  The \term{cofactor matrix} of the \( m \times n \) matrix \( A \) is
  \begin{equation*}
    \seq{ (-1)^{i + j} \det A_{i,j} }_{i,j=1}^{m,n},
  \end{equation*}
  where \( A_{i,j} \) is the \hyperref[def:submatrix]{submatrix} of \( A \) obtained by removing the \( i \)-th row and the \( j \)-th column.

  The \term{adjugate matrix}, also called the \term{classical adjoint matrix}, is the transpose of the cofactor matrix.
\end{definition}

\begin{proposition}\label{thm:inverse_via_adjunction}
  The \hyperref[def:adjugate_matrix]{adjugate matrix} of the square \( n \times n \) matrix \( A \) satisfies
  \begin{equation*}
    A \cdot A^{\op{adj}} = \det A \cdot I_n.
  \end{equation*}
\end{proposition}
\begin{proof}
  From \fullref{thm:laplace_expansion} it follows that the \( (i, i) \)-th entry of the matrix \( A \cdot A^{\op{adj}} \) is
  \begin{equation*}
    \sum_{k=1}^n (-1)^{i + k} a_{i,k} \det A_{i,k}
    =
    \det A.
  \end{equation*}

  For \( i \neq j \), the \( (i, j) \)-th entry is
  \begin{equation*}
    \sum_{k=1}^n (-1)^{i + j} a_{i,k} \det A_{j,k}
    =
    \det \widehat A_{j \mapsto i},
  \end{equation*}
  where \( \widehat A_{j \mapsto i} \) is the matrix obtained by replacing the \( j \)-th column in \( A \) with the \( i \)-th. The determinant is then zero because it is an alternating function on the columns.

  Thus, the proposition follows.
\end{proof}

\begin{definition}\label{def:inverse_matrix}
  We say that \( B \in R^{n \times m} \) is a \term{left inverse matrix} of \( A \in R^{m \times n} \) if \( BA \) is the identity matrix \( I_n \) and a \term{right inverse matrix} if \( AB \) is \( I_m \). These are precisely the left and right inverse linear maps in the correspondence described in \fullref{thm:matrix_and_linear_function_algebras}.

  Due to \fullref{thm:square_matrix_left_invertible_iff_right_invertible}, for square matrices, the two notions coincide, and we say that \( B \) is simply an \term{inverse} of \( A \). An inverse matrix, if it exists, is unique. We denote this inverse of \( A \) by \( A^{-1} \).

  We say that \( A \) is \term{invertible} if an inverse exists, and \term{singular} otherwise.
\end{definition}
\begin{defproof}
  The inverse is unique by \fullref{thm:monoid_inverse_unique}.
\end{defproof}

\begin{proposition}\label{thm:square_matrix_left_invertible_iff_right_invertible}
  Over a nontrivial \hyperref[def:noetherian_semiring]{noetherian} commutative ring \( R \), a square matrix is \hyperref[def:inverse_matrix]{left invertible} if and only if it is \hyperref[def:inverse_matrix]{right invertible}.
\end{proposition}
\begin{proof}
  \NecessitySubProof Suppose that \( A \) is a right invertible matrix. When regarding \( A \) as a linear map via the identification from \fullref{rem:matrices_as_functions}, this implies that \( A \), as a linear map from \( R^n \) to \( R^n \), is right invertible. Then it is surjective and, by \fullref{thm:surjective_endomorphism_in_free_module}, an isomorphism. Therefore, \( A \) is a fully invertible matrix.

  \SufficiencySubProof Now suppose that \( A \) is a left inverse of \( B \). Then \( B \) is a right inverse of \( A \), and, by the other direction of the proposition, a two-sided inverse of \( A \).
\end{proof}

\begin{proposition}\label{thm:matrix_invertibility}
  In the \hyperref[def:inverse_matrix]{matrix algebra} \( R^{n \times n} \) over a nontrivial \hyperref[def:noetherian_semiring]{noetherian} \hyperref[def:ring/commutative]{commutative ring} \( R \), the following are equivalent:
  \begin{thmenum}
    \thmitem{thm:matrix_invertibility/invertible} The matrix \( A \) is \hyperref[def:inverse_matrix]{invertible}.
    \thmitem{thm:matrix_invertibility/determinant} The \hyperref[def:matrix_determinant]{determinant} of \( A \) is \hyperref[def:divisibility/unit]{invertible}.
    \thmitem{thm:matrix_invertibility/columns} The \hyperref[def:block_matrix]{columns} of \( A \) are \hyperref[thm:linear_dependence]{linearly independent}.
  \end{thmenum}
\end{proposition}
\begin{proof}
  \ImplicationSubProof{thm:matrix_invertibility/invertible}{thm:matrix_invertibility/determinant} Suppose that \( A \) is invertible.

  By \fullref{thm:def:matrix_determinant/homomorphism},
  \begin{equation*}
    \det(A^{-1}) \det(A) = \det(A^{-1} A) = \det(I_n) = 1,
  \end{equation*}
  hence \( \det(A) \) has a multiplicative inverse.

  \ImplicationSubProof{thm:matrix_invertibility/determinant}{thm:matrix_invertibility/columns} As in \fullref{thm:determinant_on_columns}, regard \( \det(v_1, \ldots, v_n) \) are an alternating multilinear function on the columns of a matrix.

  Suppose that \( \det(v_1, \ldots, v_n) \) is a unit in \( R \) and, aiming at a contradiction, suppose that the column vectors \( v_1, \ldots, v_n \) are linearly dependent. Then there exists a nontrivial linear combination that sums to zero:
  \begin{equation*}
    \sum_{i=1}^n t_i v_i = 0.
  \end{equation*}

  Suppose that \( t_k \) is nonzero. Then
  \begin{align*}
    0
    &=
    \det\parens*{ v_1, \ldots, v_{k-1}, 0, v_{k+1}, \ldots, v_n }
    = \\ &=
    \det\parens*{ v_1, \ldots, v_{k-1}, \sum_{i=1}^n t_i v_i, v_{k+1}, \ldots, v_n }
    = \\ &=
    \sum_{i=1}^n t_i \det( v_1, \ldots, v_{k-1}, v_i, v_{k+1}, \ldots, v_n )
    = \\ &=
    t_k \det( v_1, \ldots, v_{k-1}, v_k, v_{k+1}, \ldots, v_n ).
  \end{align*}

  But we have assumed that \( \det( v_1, \ldots, v_{k-1}, v_k, v_{k+1}, \ldots, v_n ) \) is a unit and that \( t_k \) is nonzero. Hence, the determinant can only be a zero divisor if the ring is trivial, which we have assumed it is not. The obtained contradiction shows that \( v_1, \ldots, v_n \) are linearly independent.

  \ImplicationSubProof{thm:matrix_invertibility/columns}{thm:matrix_invertibility/invertible} Suppose that the columns of \( A \) are linearly independent. Consider the matrix equation
  \begin{equation*}
    Ax
    =
    \parens*
    {
      \begin{array}{c|c|c}
        a_{\anon*,1} & \cdots & a_{\anon*,n}
      \end{array}
    }
    \begin{pmatrix}
      x_1 \\ \vdots \\ x_n
    \end{pmatrix}
    =
    \sum_{k=1}^n x_k a_{\anon*,k}
    =
    \vect 0.
  \end{equation*}

  Since the columns are linearly independent, only \( x_1 = \cdots = x_n \) is a solution to this equation. Thus, when regarding \( A \) as the linear map \( x \mapsto Ax \), the \hyperref[def:module/kernel]{kernel} of \( A \) becomes trivial. By \fullref{thm:def:group/zero_kernel}, this map is injective. As discussed in \fullref{def:module/category}, the injective linear maps are exactly the left invertible linear maps. Hence, there exists a left inverse of \( A \). Since \( A \) is a square matrix, by \fullref{thm:square_matrix_left_invertible_iff_right_invertible}, this implies that \( A \) is invertible.
\end{proof}


% % % % % % % % % % % % % % % % % % % % % % % % %
\chapter{Homotopy theory}
% % % % % % % % % % % % % % % % % % % % % % % % %

\section{CW-complexes}

\begin{definition}
	A map $f \colon X \rightarrow Y$ is called a
	\textit{weak homotopy equivalence} \index{homotopy equivalence!weak}
	if it induces isomorphisms
	\[
	\pi_n(X, x_0) \rightarrow \pi_n(Y, f(x_0))
	\]
	for all $n \ge 0$ and all choices of basepoints $x_0$ in $X$.
\end{definition}

\begin{theorem}[Whitehead's Theorem]
	A weak homotopy equivalence between CW-complexes is a homotopy equivalence.
\end{theorem}

% {\cite[Proposition 4.15]{hatcher2002algebraic}}
\begin{proposition}[Geometric interpretation of $n$-connectedness]
	If $(X, A)$ is an $n$-connected CW-pair, then there exists
	a CW-pair $(Z, A) \sim_{\rel A} (X, A)$
	such that all cells of $Z \setminus A$ have dimension greater than $n$.
\end{proposition}

\section{Homology}

\begin{definition}[Acyclic]
	A space $X$ is called \textit{acyclic}\index{acyclic} if $\widetilde{H}_{i}(X) = 0$ for all $i$,
	i.e. if its reduced homology vanishes.
\end{definition}

\begin{example}
	Removing a point from a homology sphere yields an acyclic space.
	If the dimension was at least $3$ this does not change
	the fundamental group, so if we started with a nontrivial homology sphere
	(i.e.\ $\pi_1 \ne 1$) this will give an example of an acyclic, but
	non-contractible space.

	This example for the Poincar\'e homology sphere is described in
	\citep[Example 2.38]{hatcher2002algebraic}.
\end{example}


\problemname{Gnoll Hypothesis}

You are a huge fan of the RPG \emph{The Eldest Scrolls: Earthrim} and know all of the game's internal mechanics.
For instance, when spawning a new monster, each of the $n$ different types of monsters in the game has some fixed probability of appearing,
and you know exactly what this probability distribution over monster types is.

However, in the latest update the developers seem to have changed the way monsters are spawned.  After some testing and reverse engineering,
you realise that instead of spawning all $n$ types of monsters, each spawn location only has a spawn pool of $k$ monster types. These spawn pools are
chosen randomly at the start of the game, independently for each spawn location, with every monster type having the same chance of being chosen for the spawn pool.  And apparently
a developer was lazy with adjusting the spawn chances.  Instead of normalising the spawn chances of the $k$ chosen types, the developer decided that
if a type of monster is not chosen, its spawn chance is added to the next chosen monster type in the list of types (and if monster types at the end of the list are not
chosen, their spawn chances are added to the first chosen monster type in the list).  For example, Figure~\ref{spawning monsters sample 1} shows a small example with $n=5$ monsters, a possible random choice of $k=3$ of those monsters, and the resulting spawn probabilities for those $3$ monsters.

After the update, some monster types seem to appear less often than before, and
some more often (for instance now there seem to be Gnolls all over
the place).  You believe that the new spawning logic may be
responsible for this by having changed the effective spawn chances of
the monsters.  In order to test this hypothesis, you decide to compute
these effective spawn chances after the update.

\begin{figure}[h] \centering
  \includegraphics[width=.4\textwidth]{sample1}
  \caption{Sample Input 1 and one possible spawn pool with adjusted spawn chances.}
  \label{spawning monsters sample 1}
\end{figure}

\section*{Input}
The input consists of:
\begin{itemize}
  \item One line with two integers $n$ and $k$ ($1 \le k \le n \le 500$), the number of different types of monsters and the number of monsters that are randomly chosen
    for the spawn pool of each spawn location.
  \item One line with $n$ real numbers $s_1, s_2, \ldots, s_n$ ($s_i \ge 0$ for each $i$, $\sum^{n}_{j=1}s_j = 100$), where $s_i$ is the spawn chance in percent for the $i$th type in the list of monster types.  Every real number has at most six digits after the decimal point.
\end{itemize}

\section*{Output}
Output a single line containing $n$ real numbers, the effective spawn chance in percent of each type of monster. The $i$th number in your output should correspond to the $i$th type of monster.
Your answers should have an absolute or relative error of at most $10^{-6}$.


\section{Linear independence}

\begin{outcome}
  \begin{enumerate}
  \item Find the redundant vectors in a set of vectors.
  \item Determine whether a set of vectors is linearly independent.
  \item Find a linearly independent subset of a set of spanning vectors.
  \item Write a vector as a unique linear combination of a set of
    linearly independent vectors.
  \end{enumerate}
\end{outcome}

% ----------------------------------------------------------------------
\subsection{Redundant vectors and linear independence}

In Example~\ref{exa:redundant-span}, we encountered three vectors
$\vect{u}$, $\vect{v}$, and $\vect{w}$ such that
$\sspan\set{\vect{u},\vect{v},\vect{w}} =
\sspan\set{\vect{u},\vect{v}}$.  If this happens, then the vector
$\vect{w}$ does not contribute anything to the span of
$\set{\vect{u},\vect{v},\vect{w}}$, and we say that $\vect{w}$ is
\textbf{redundant}%
\index{redundant vector}%
\index{vector!redundant}. The following definition generalizes this
notion.

\begin{definition}{Redundant vectors, linear dependence, and linear independence}{redundant-vectors}
  Consider a sequence of $k$ vectors $\vect{u}_1,\ldots,\vect{u}_k$.
  We say that the vector $\vect{u}_j$ is \textbf{redundant}%
  \index{redundant vector}%
  \index{vector!redundant} if it can be written as a linear
  combination of earlier vectors in the sequence, i.e., if
  \begin{equation*}
    \vect{u}_j = a_1\,\vect{u}_1 + a_2\,\vect{u}_2 + \ldots + a_{j-1}\,\vect{u}_{j-1}
  \end{equation*}
  for some scalars $a_1,\ldots,a_{j-1}$. We say that the sequence of
  vectors $\vect{u}_1,\ldots,\vect{u}_k$ is \textbf{linearly
    dependent}%
  \index{linear dependence}%
  \index{vector!linearly dependent} if it contains one or more
  redundant vectors. Otherwise, we say that the vectors are
  \textbf{linearly independent}%
  \index{linear independence}%
  \index{vector!linearly independent}.
\end{definition}

\begin{example}{Redundant vectors}{redundant-vectors}
  Find the redundant vectors in the following sequence of vectors. Are
  the vectors linearly independent?
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{c} 0 \\ 0 \\ 0 \\ 0\end{mymatrix},
    \quad
    \vect{u}_2 = \begin{mymatrix}{c} 1 \\ 2 \\ 2 \\ 3\end{mymatrix},
    \quad
    \vect{u}_3 = \begin{mymatrix}{c} 1 \\ 1 \\ 1 \\ 1\end{mymatrix},
    \quad
    \vect{u}_4 = \begin{mymatrix}{c} 2 \\ 3 \\ 3 \\ 4\end{mymatrix},
    \quad
    \vect{u}_5 = \begin{mymatrix}{c} 0 \\ 1 \\ 2 \\ 3\end{mymatrix},
    \quad
    \vect{u}_6 = \begin{mymatrix}{c} 3 \\ 3 \\ 2 \\ 2\end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  \begin{itemize}
  \item The vector $\vect{u}_1$ is redundant, because it is a linear
    combination of earlier vectors. (Although there are no earlier
    vectors, recall from Example~\ref{exa:span-empty-set} that the empty
    sum of vectors is equal to the zero vector $\vect{0}$. Therefore,
    $\vect{u}_1$ is indeed an (empty) linear combination of earlier
    vectors.)
  \item The vector $\vect{u}_2$ is not redundant, because it cannot be
    written as a linear combination of $\vect{u}_1$. This is because
    the system of equations
    \begin{equation*}
      \begin{mymatrix}{l|l}
        0 & 1 \\
        0 & 2 \\
        0 & 2 \\
        0 & 3 \\
      \end{mymatrix}
    \end{equation*}
    has no solution.
  \item The vector $\vect{u}_3$ is not redundant, because it cannot be
    written as a linear combination of $\vect{u}_1$ and $\vect{u}_2$.
    This is because the system of equations
    \begin{equation*}
      \begin{mymatrix}{ll|l}
        0 & 1 & 1 \\
        0 & 2 & 1 \\
        0 & 2 & 1 \\
        0 & 3 & 1 \\
      \end{mymatrix}
    \end{equation*}
    has no solution.
  \item The vector $\vect{u}_4$ is redundant, because $\vect{u}_4 =
    \vect{u}_2 + \vect{u}_3$.
  \item The vector $\vect{u}_5$ is not redundant, because
    This is because the system of equations
    \begin{equation*}
      \begin{mymatrix}{llll|l}
        0 & 1 & 1 & 2 & 0 \\
        0 & 2 & 1 & 3 & 1 \\
        0 & 2 & 1 & 3 & 2 \\
        0 & 3 & 1 & 4 & 3 \\
      \end{mymatrix}
    \end{equation*}
    has no solution.
  \item The vector $\vect{u}_6$ is redundant, because $\vect{u}_6 =
    \vect{u}_2 + 2\vect{u}_3-\vect{u}_5$.
  \end{itemize}
  In summary, the vectors $\vect{u}_1$, $\vect{u}_4$, and $\vect{u}_6$
  are redundant, and the vectors $\vect{u}_2$, $\vect{u}_3$, and
  $\vect{u}_5$ are not. It follows that the vectors
  $\vect{u}_1,\ldots,\vect{u}_6$ are linearly dependent.
\end{solution}

% ----------------------------------------------------------------------
\subsection{The casting-out algorithm}

The last example shows that it can be a lot of work to find the
redundant vectors in a sequence of $k$ vectors. Doing so in the naive way
require us to solve up to $k$ systems of linear equations!
Fortunately, there is a much faster and easier method, the so-called
{\em casting-out algorithm}.

\begin{algorithm}{Casting-out algorithm}{casting-out}
  \index{casting-out algorithm}%
  \index{linear independence!casting-out algorithm}%
  \index{vector!casting-out algorithm}%
  \textbf{Input:} a list of $k$ vectors
  $\vect{u}_1,\ldots,\vect{u}_k\in\R^n$.  \smallskip

  \textbf{Output:} the set of indices $j$ such that $\vect{u}_j$ is
  redundant.
  \smallskip

  \textbf{Algorithm:} Write the vectors $\vect{u}_1,\ldots,\vect{u}_k$
  as the columns of an $n\times k$-matrix, and reduce to {\ef}. Every
  {\em non-pivot} column, if any, corresponds to a redundant vector.
\end{algorithm}

\begin{example}{Casting-out algorithm}{casting-out}
  Use the casting-out algorithm to find the redundant vectors among
  the vectors from Example~\ref{exa:redundant-vectors}.
\end{example}

\begin{solution}
  Following the casting-out algorithm, we write the vectors
  $\vect{u}_1,\ldots,\vect{u}_6$ as the columns of a matrix and reduce
  to {\ef}.
  \begin{equation*}
    \begin{mymatrix}{rrrrrr}
      0 & 1 & 1 & 2 & 0 & 3 \\
      0 & 2 & 1 & 3 & 1 & 3 \\
      0 & 2 & 1 & 3 & 2 & 2 \\
      0 & 3 & 1 & 4 & 3 & 2 \\
    \end{mymatrix}
    \roweq\ldots\roweq
    \begin{mymatrix}{rrrrrr}
      0 & \circled{1} & 1 & 2 & 0 & 3 \\
      0 & 0 & \circled{1} & 1 & -1 & 3 \\
      0 & 0 & 0 & 0 & \circled{1} & -1 \\
      0 & 0 & 0 & 0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  The pivot columns are columns $2$, $3$, and $5$. The non-pivot
  columns are columns $1$, $4$, and $6$. Therefore, the vectors
  $\vect{u}_1$, $\vect{u}_4$, and $\vect{u}_6$ are redundant. Note
  that this is the same answer we got in
  Example~\ref{exa:redundant-vectors}.
\end{solution}

The above version of the casting-out algorithm only tells us which of
the vectors (if any) are redundant, but it does not give us a specific
way to write the redundant vectors as linear combinations of previous
vectors. However, we can easily get this additional information if we
reduce the matrix all the way to {\rref}. We call this version of the
algorithm the {\em extended casting-out algorithm}.

\begin{algorithm}{Extended casting-out algorithm}{extended-casting-out}
  \index{extended casting-out algorithm}%
  \index{casting-out algorithm!extended}%
  \index{linear independence!casting-out algorithm!extended}%
  \index{vector!casting-out algorithm!extended}%
  \textbf{Input:} a list of $k$ vectors
  $\vect{u}_1,\ldots,\vect{u}_k\in\R^n$.
  \smallskip

  \textbf{Output:} the set of indices $j$ such that $\vect{u}_j$ is
  redundant, and a set of coefficients for writing each redundant
  vector as a linear combination of previous vectors.
  \smallskip

  \textbf{Algorithm:} Write the vectors $\vect{u}_1,\ldots,\vect{u}_k$
  as the columns of an $n\times k$-matrix, and reduce to
  {\rref}. Every {\em non-pivot} column, if any, corresponds to a
  redundant vector. If $\vect{u}_j$ is a redundant vector, then the
  entries in the $j\th$ column of the {\rref} are coefficients for
  writing $\vect{u}_j$ as a linear combination of previous
  non-redundant vectors.
\end{algorithm}

\begin{example}{Extended casting-out algorithm}{extended-casting-out}
  Use the casting-out algorithm to find the redundant vectors among
  the vectors from Example~\ref{exa:redundant-vectors}, and write each
  redundant vector as a linear combination of previous non-redundant
  vectors.
\end{example}

\begin{solution}
  Once again, we write the vectors $\vect{u}_1,\ldots,\vect{u}_6$ as
  the columns of a matrix. This time we use the extended casting-out
  algorithm, which means we reduce the matrix to {\rref} instead of
  {\ef}.
  \begin{equation*}
    \begin{mymatrix}{rrrrrr}
      0 & 1 & 1 & 2 & 0 & 3 \\
      0 & 2 & 1 & 3 & 1 & 3 \\
      0 & 2 & 1 & 3 & 2 & 2 \\
      0 & 3 & 1 & 4 & 3 & 2 \\
    \end{mymatrix}
    \roweq\ldots\roweq
    \begin{mymatrix}{rrrrrr}
      0 & \circled{1} & 0 & 1 & 0 & 1 \\
      0 & 0 & \circled{1} & 1 & 0 & 2 \\
      0 & 0 & 0 & 0 & \circled{1} & -1 \\
      0 & 0 & 0 & 0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  As before, the non-pivot columns are columns $1$, $4$, and $6$, and
  therefore, the vectors $\vect{u}_1$, $\vect{u}_4$, and $\vect{u}_6$
  are redundant. The non-redundant vectors are $\vect{u}_2$,
  $\vect{u}_3$, and $\vect{u}_5$. Moreover, the entries in the sixth
  column are $1$, $2$, and $-1$.  Note that this means that the sixth
  column can be written as $1$ times the second column plus $2$ times
  the third column plus $(-1)$ times the fourth column. The same
  coefficients can be used to write $\vect{u}_6$ as a linear
  combination of previous {\em non-redundant} columns, namely:
  \begin{equation*}
    \vect{u}_6 = 1\,\vect{u}_2 + 2\,\vect{u}_3 - 1\,\vect{u}_5.
  \end{equation*}
  Also, the entries in the fourth column are $1$ and $1$, which are
  the coefficients for writing $\vect{u}_4$ as a linear combination of
  previous non-redundant columns, namely:
  \begin{equation*}
    \vect{u}_4 = 1\,\vect{u}_2 + 1\,\vect{u}_3.
  \end{equation*}
  Finally, there are no non-zero entries in the first column. This
  means that $\vect{u}_1$ is the empty linear combination
  \begin{equation*}
    \vect{u}_1 = \vect{0}.
  \end{equation*}
\end{solution}

% ----------------------------------------------------------------------
\subsection{Alternative characterization of linear independence}

Our definition of redundant vectors depends on the order in which the
vectors are written. This is because each redundant vector must be a
linear combination of {\em earlier} vectors in the sequence. For example,
in the sequence of vectors
\begin{equation*}
  \vect{u}=\begin{mymatrix}{r} 1 \\ 1 \\ 1 \end{mymatrix},\quad
  \vect{v}=\begin{mymatrix}{r} 3 \\ 2 \\ 1 \end{mymatrix},\quad
  \vect{w}=\begin{mymatrix}{r} 11 \\ 8 \\ 5 \end{mymatrix},
\end{equation*}
the vector $\vect{w}$ is redundant, because it is a linear combination
of earlier vectors $\vect{w} = 2\,\vect{u}+3\,\vect{v}$. Neither
$\vect{u}$ nor $\vect{v}$ are redundant. On the other hand, in the sequence
of vectors
\begin{equation*}
  \vect{u}=\begin{mymatrix}{r} 1 \\ 1 \\ 1 \end{mymatrix},\quad
  \vect{w}=\begin{mymatrix}{r} 11 \\ 8 \\ 5 \end{mymatrix},\quad
  \vect{v}=\begin{mymatrix}{r} 3 \\ 2 \\ 1 \end{mymatrix},
\end{equation*}
$\vect{v}$ is redundant because
$\vect{v} = \frac{1}{3}\vect{w} - \frac{2}{3}\vect{u}$, but neither
$\vect{u}$ nor $\vect{w}$ are redundant. Note that none of the vectors
have changed; only the order in which they are written is
different. Yet $\vect{w}$ is the redundant vector in the first sequence,
and $\vect{v}$ is the redundant vector in the second sequence.

Because we defined linear independence in terms of the absence of
redundant vectors, you may suspect that the concept of linear
independence also depends on the order in which the vectors are
written. However, this is not the case. The following theorem gives an
alternative characterization of linear independence that is more
symmetric (it does not depend on the order of the vectors).

\begin{theorem}{Characterization of linear independence}{characterization-linear-independence}
  Let $\vect{u}_1,\ldots,\vect{u}_k$ be vectors. Then
  $\vect{u}_1,\ldots,\vect{u}_k$ are linearly independent%
  \index{linear independence!alternative characterization}%
  \index{vector!linearly independent!alternative characterization}
  if and only if the homogeneous equation
  \begin{equation*}
    a_1\,\vect{u}_1 + \ldots + a_k\,\vect{u}_k = \vect{0}
  \end{equation*}
  has only the trivial solution%
  \index{trivial solution}%
  \index{solution!trivial}%
  \index{system of linear equations!trivial solution}.
\end{theorem}

\begin{proof}
  Let $A$ be the $n\times k$-matrix whose columns are
  $\vect{u}_1,\ldots,\vect{u}_k$. We know from the theory of
  homogeneous systems that the system
  $a_1\,\vect{u}_1 + \ldots + a_k\,\vect{u}_k = \vect{0}$ has no
  non-trivial solution if and only if every column of the {\ef} of $A$
  is a pivot column. By the casting-out algorithm, this is the case if
  and only if none of the vectors $\vect{u}_1,\ldots,\vect{u}_k$ are
  redundant, i.e., if and only if the vectors are linearly independent.
\end{proof}

\begin{example}{Characterization of linear independence}{characterization-linear-independence}
  Use the method of
  Theorem~\ref{thm:characterization-linear-independence} to determine
  whether the following vectors are linearly independent in $\R^4$.
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 1 \\ 2 \\ 0 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{r} 0 \\ 1 \\ 1 \\ 1 \end{mymatrix},\quad
    \vect{u}_3 = \begin{mymatrix}{r} 1 \\ 2 \\ 3 \\ 2 \end{mymatrix},\quad
    \vect{u}_4 = \begin{mymatrix}{r} 2 \\ 3 \\ 3 \\ 1 \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We must check whether the equation
  \begin{equation*}
    a_1 \begin{mymatrix}{r} 1 \\ 1 \\ 2 \\ 0 \end{mymatrix}
    + a_2 \begin{mymatrix}{r} 0 \\ 1 \\ 1 \\ 1 \end{mymatrix}
    + a_3 \begin{mymatrix}{r} 1 \\ 2 \\ 3 \\ 2 \end{mymatrix}
    + a_4 \begin{mymatrix}{r} 2 \\ 3 \\ 3 \\ 1 \end{mymatrix}
    = \begin{mymatrix}{r} 0 \\ 0 \\ 0 \\ 0 \end{mymatrix}
  \end{equation*}
  has a non-trivial solution. If it does, the vectors are linearly
  dependent. On the other hand, if there is only the trivial solution,
  the vectors are linearly independent. We write the augmented matrix
  and solve:
  \begin{equation*}
    \begin{mymatrix}{rrrr|r}
      1 & 0 & 1 & 2 & 0 \\
      1 & 1 & 2 & 3 & 0 \\
      2 & 1 & 3 & 7 & 0 \\
      0 & 1 & 2 & 1 & 0 \\
    \end{mymatrix}
    \roweq
    \ldots
    \roweq
    \begin{mymatrix}{rrrr|r}
      \circled{1} & 0 & 1 & 2 & 0 \\
      0 & \circled{1} & 1 & 1 & 0 \\
      0 & 0 & \circled{1} & 0 & 0 \\
      0 & 0 & 0 & \circled{2} & 0 \\
    \end{mymatrix}.
  \end{equation*}
  Since every column is a pivot column, there are no free variables;
  the system of equations has a unique solution, which is
  $a_1=a_2=a_3=a_4=0$, i.e., the trivial solution. Therefore, the
  vectors $\vect{u}_1,\ldots,\vect{u}_4$ are linearly independent.
\end{solution}

\begin{example}{Characterization of linear independence}{characterization-linear-independence2}
  Use the method of
  Theorem~\ref{thm:characterization-linear-independence} to determine
  whether the following vectors are linearly independent in $\R^3$.
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 1 \\ 0 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{r} 1 \\ 3 \\ 1 \end{mymatrix},\quad
    \vect{u}_3 = \begin{mymatrix}{r} 0 \\ 4 \\ 2 \end{mymatrix},\quad
  \end{equation*}
\end{example}

\begin{solution}
  As in the previous example, we must check whether the equation
  $a_1\,\vect{u}_1 + a_2\,\vect{u}_2 + a_3\,\vect{u}_3 = \vect{0}$ has
  a non-trivial solution. Once again, we write the augmented matrix
  and solve:
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      1 & 1 & 0 & 0 \\
      1 & 3 & 4 & 0 \\
      0 & 1 & 2 & 0 \\
    \end{mymatrix}
    \roweq
    \ldots
    \roweq
    \begin{mymatrix}{rrr|r}
      \circled{1} & 1 & 0 & 0 \\
      0 & \circled{2} & 4 & 0 \\
      0 & 0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  Since column $3$ is not a pivot column, $a_3$ is a free
  variable. Therefore, the system has a non-trivial solution, and the
  vectors are linearly dependent.

  With a small amount of extra work, we can find an actual non-trivial
  solution of
  $a_1\,\vect{u}_1 + a_2\,\vect{u}_2 + a_3\,\vect{u}_3 =
  \vect{0}$. All we have to do is set $a_3=1$ and do a back
  substitution. We find that $(a_1,a_2,a_3)=(2,-2,1)$ is a
  solution. In other words,
  \begin{equation*}
    2\vect{u}_1 - 2\vect{u}_2 + \vect{u}_3 = \vect{0}.
  \end{equation*}
  We can also use this information to write $\vect{u}_3$ as a linear
  combination of previous vectors, namely,
  $\vect{u}_3 = -2\vect{u}_1 + 2\vect{u}_2$.
\end{solution}

The characterization of linear independence in
Theorem~\ref{thm:characterization-linear-independence} is mostly
useful for theoretical reasons. However, it can also help in solving
problems such as the following.

\begin{example}{Related sets of vectors}{related-linear-independence}
  Let $\vect{u},\vect{v},\vect{w}$ be linearly independent vectors in
  $\R^n$. Are the vectors $\vect{u}+\vect{v}$, $2\vect{u}+\vect{w}$,
  and $\vect{v}-5\vect{w}$ linearly independent?
\end{example}

\begin{solution}
  By Theorem~\ref{thm:characterization-linear-independence}, to check
  whether the vectors are linearly independent, we must check whether
  the equation
  \begin{equation}\label{eqn:related-linear-independence1}
    a(\vect{u}+\vect{v}) + b(2\vect{u}+\vect{w}) +
    c(\vect{v}-5\vect{w})=\vect{0}
  \end{equation}
  has non-trivial solutions.  If it does, the vectors are linearly
  dependent, if it does not, they are linearly independent. We can
  simplify the equation as follows:
  \begin{equation}\label{eqn:related-linear-independence2}
    (a+2b)\vect{u} + (a+c)\vect{v} + (b-5c)\vect{w}=\vect{0}.
  \end{equation}
  Since $\vect{u}$, $\vect{v}$, and $\vect{w}$ are linearly
  independent, we know, again by
  Theorem~\ref{thm:characterization-linear-independence}, that
  equation {\eqref{eqn:related-linear-independence2}} only has the
  trivial solution. Therefore,
  \begin{eqnarray*}
    a + 2b & = & 0, \\
    a + c & = & 0, \\
    b - 5c & = & 0.
  \end{eqnarray*}
  We can solve this system of three equations in three variables, and
  we find that it has the unique solution $a=b=c=0$. Therefore,
  $a=b=c=0$ is the only solution to equation
  {\eqref{eqn:related-linear-independence1}}, which means that the
  vectors $\vect{u}+\vect{v}$, $2\vect{u}+\vect{w}$, and
  $\vect{v}-5\vect{w}$ are linearly independent.
\end{solution}

% ----------------------------------------------------------------------
\subsection{Properties of linear independence}

The following are some properties of linearly independent sets.

\begin{proposition}{Properties of linear independence}{properties-linear-independence}
  \index{linear independence!properties}%
  \index{vector!linearly independent!properties}%
  \index{properties of linear independence}%
  \begin{enumerate}
  \item \textbf{Linear independence and reordering.} If a sequence
    $\vect{u}_1,\ldots,\vect{u}_k$ of $k$ vectors is linearly
    independent, then so is any reordering of the sequence (i.e.,
    whether or not the vectors are linearly independent does not
    depend on the order in which the vectors are written down).
  \item \textbf{Linear independence of a subset.} If
    $\vect{u}_1,\ldots,\vect{u}_k$ are linearly independent, then so
    are $\vect{u}_1,\ldots,\vect{u}_j$ for any $j<k$.
  \item \textbf{Linear independence and dimension.}
    \label{properties-linear-independence-c}
    Let $\vect{u}_1,\ldots,\vect{u}_k$ be a sequence of $k$ vectors in
    $\R^n$. If $k>n$, then the vectors are linearly dependent (i.e.,
    not linearly independent).
  \end{enumerate}
\end{proposition}

\begin{proof}
  \begin{enumerate}
  \item This follows from
    Theorem~\ref{thm:characterization-linear-independence}, because
    whether or not the equation
    $a_1\vect{u}_1 + \ldots + a_k\vect{u}_k = \vect{0}$ has a
    non-trivial solution does not depend on the order in which the
    vectors are written.
  \item If one of the vectors in the sequence
    $\vect{u}_1,\ldots,\vect{u}_j$ were redundant, then it would be
    redundant in the longer sequence $\vect{u}_1,\ldots,\vect{u}_k$ as
    well.
  \item Let $A$ be the $n\times k$-matrix that has the vectors
    $\vect{u}_1,\ldots,\vect{u}_k$ as its columns and suppose that
    $k>n$. Then the rank of $A$ is at most $n$, so the {\ef} of $A$
    has some non-pivot columns. Therefore, the system
    $a_1\vect{u}_1 + \ldots + a_k\vect{u}_k = \vect{0}$ has
    non-trivial solutions, and the vectors are linearly dependent by
    Theorem~\ref{thm:characterization-linear-independence}.
  \end{enumerate}
\end{proof}

\begin{example}{Linear dependence}{linear-dependence}
  Are the following vectors linearly independent?
  \begin{equation*}
    \begin{mymatrix}{r}
        1 \\
        4
      \end{mymatrix},\quad
      \begin{mymatrix}{r}
        2 \\
        3
      \end{mymatrix},\quad
      \begin{mymatrix}{r}
        3 \\
        2
      \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  Since these are $3$ vectors in $\R^2$, they are linearly dependent
  by Proposition~\ref{prop:properties-linear-independence}. No calculation
  is necessary.
\end{solution}

% ----------------------------------------------------------------------
\subsection{Linear independence and linear combinations}

In general, there is more than one way of writing a given vector as a
linear combination of some spanning vectors. For example, consider
\begin{equation*}
  \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 1 \\ 0 \end{mymatrix},\quad
  \vect{u}_2 = \begin{mymatrix}{r} 0 \\ 1 \\ 1 \end{mymatrix},\quad
  \vect{u}_3 = \begin{mymatrix}{r} 1 \\ 2 \\ 1 \end{mymatrix},\quad
  \vect{v} = \begin{mymatrix}{r} 1 \\ 3 \\ 2 \end{mymatrix}.
\end{equation*}
We can write $\vect{v}$ in many different ways as a linear
combination of $\vect{u}_1,\ldots,\vect{u}_3$, for example
\begin{equation*}
  \begin{array}{r@{~}c@{~}c}
    \vect{v} &=& -\vect{u}_1 + 2\vect{u}_3, \\
    \vect{v} &=& \vect{u}_2 + \vect{u}_3, \\
    \vect{v} &=& \vect{u}_1 + 2\vect{u}_2, \\
    \vect{v} &=& 2\vect{u}_1 + 3\vect{u}_2 - \vect{u}_3. \\
  \end{array}
\end{equation*}
However, when the vectors $\vect{u}_1,\ldots,\vect{u}_k$ are linearly
independent, this does not happen. In this case, the linear
combination is always unique, as the following theorem shows.

\begin{theorem}{Unique linear combination}{unique-linear-combination}
  Assume $\vect{u}_1,\ldots,\vect{u}_k$ are linearly independent. Then
  every vector $\vect{v}\in\sspan\set{\vect{u}_1,\ldots,\vect{u}_k}$
  can be written as a linear combination of
  $\vect{u}_1,\ldots,\vect{u}_k$ in a unique way.
\end{theorem}

\begin{proof}
  We already know that every vector
  $\vect{v}\in\sspan\set{\vect{u}_1,\ldots,\vect{u}_k}$ can be written
  as a linear combination of $\vect{u}_1,\ldots,\vect{u}_k$, because
  that is the definition of span. So what must be proved is the
  uniqueness. Suppose, therefore, that there are two ways of writing
  $\vect{v}$ as such a linear combination, i.e., that
  \begin{equation*}
    \begin{array}{l}
    \vect{v} = a_1\,\vect{u}_1 + a_2\,\vect{u}_2 + \ldots + a_k\,\vect{u}_k\quad\mbox{and} \\
    \vect{v} = b_1\,\vect{u}_1 + b_2\,\vect{u}_2 + \ldots + b_k\,\vect{u}_k. \\
    \end{array}
  \end{equation*}
  Subtracting one equation from the other, we get
  \begin{equation*}
    \vect{0} = (a_1-b_1)\vect{u}_1 + (a_2-b_2)\vect{u}_2 + \ldots + (a_k-b_k)\vect{u}_k.
  \end{equation*}
  Since $\vect{u}_1,\ldots,\vect{u}_k$ are linearly independent, we
  know by Theorem~\ref{thm:characterization-linear-independence} that
  the last equation only has the trivial solution, i.e., $a_1-b_1=0$,
  $a_2-b_2=0$, \ldots, $a_k-b_k=0$. It follows that $a_1=b_1$,
  $a_2=b_2$, \ldots, $a_k=b_k$. We have shown that any two ways of
  writing $\vect{v}$ as a linear combination of
  $\vect{u}_1,\ldots,\vect{u}_k$ are equal. Therefore, there is only
  one way of doing so.
\end{proof}

% ----------------------------------------------------------------------
\subsection{Removing redundant vectors}

Consider the span of some vectors $\vect{u}_1,\ldots,\vect{u}_k$. As
we just saw in the previous subsection, the span is especially nice
when the vectors $\vect{u}_1,\ldots,\vect{u}_k$ are linearly
independent, because in that case, every element $\vect{v}$ of the
span can be {\em uniquely} written in the form
$\vect{v} = a_1\,\vect{u}_1 + \ldots + a_k\,\vect{u}_k$.

But what if we have a span of some vectors
$\vect{u}_1,\ldots,\vect{u}_k$ that are not linearly independent? It
turns out that we can always find some linearly independent vectors
that span the same set.  In fact, this can be done by simply removing
the redundant vectors from $\vect{u}_1,\ldots,\vect{u}_k$. This is the
subject of the following theorem.

\begin{theorem}{Removing redundant vectors}{linearly-independent-subset}
  \index{redundant vector!removing}%
  \index{vector!redundant!removing}%
  Let $\vect{u}_1,\ldots,\vect{u}_k$ be a sequence of vectors, and
  suppose that $\vect{u}_{j_1},\ldots,\vect{u}_{j_\ell}$ is the
  subsequence of vectors that is obtained by removing all of the
  redundant vectors. Then $\vect{u}_{j_1},\ldots,\vect{u}_{j_\ell}$
  are linearly independent and
  \begin{equation*}
    \sspan\set{\vect{u}_{j_1},\ldots,\vect{u}_{j_\ell}}
    =
    \sspan\set{\vect{u}_1,\ldots,\vect{u}_k}.
  \end{equation*}
\end{theorem}

\begin{proof}
  Remove the redundant vectors one by one, from right to left. Each
  time a redundant vector is removed, the span does not change; the
  proof of this is similar to Example~\ref{exa:redundant-span}.
  Moreover, the resulting sequence of vectors
  $\sspan\set{\vect{u}_{j_1},\ldots,\vect{u}_{j_\ell}}$ is linearly
  independent, because if any of these vectors were a linear
  combination of earlier ones, then it would have been redundant in
  the original sequence of vectors, and would have therefore been removed.
\end{proof}

\begin{example}{Finding a linearly independent set of spanning vectors}{linearly-independent-subset}
  Find a subset of $\set{\vect{u}_1,\ldots,\vect{u}_4}$ that is
  linearly independent and has the same span as
  $\set{\vect{u}_1,\ldots,\vect{u}_4}$.
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 0 \\ -2 \\ 3 \end{mymatrix},
    \quad
    \vect{u}_2 = \begin{mymatrix}{r} -2 \\ 0 \\ 4 \\ -6 \end{mymatrix},
    \quad
    \vect{u}_3 = \begin{mymatrix}{r} 1 \\ 2 \\ 2 \\ 1 \end{mymatrix},
    \quad
    \vect{u}_4 = \begin{mymatrix}{r} 3 \\ 4 \\ 2 \\ 5 \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We use the casting-out algorithm to find the redundant vectors:
  \begin{equation*}
    \begin{mymatrix}{rrrr}
      1  & -2 & 1 & 3 \\
      0  &  0 & 2 & 4 \\
      -2 &  4 & 2 & 2 \\
      3  & -6 & 1 & 5 \\
    \end{mymatrix}
    \roweq\ldots\roweq
    \begin{mymatrix}{rrrr}
      \circled{1}  & -2 & 1 & 3 \\
      0  &  0 & \circled{2} & 4 \\
      0  &  0 & 0 & 0 \\
      0  &  0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  Therefore, the redundant vectors are $\vect{u}_2$ and
  $\vect{u}_4$. We remove them (``cast them out'') and are left with
  $\vect{u}_1$ and $\vect{u}_3$. Therefore, by
  Theorem~\ref{thm:linearly-independent-subset},
  $\set{\vect{u}_1,\vect{u}_3}$ is linearly independent and
  $\sspan\set{\vect{u}_1,\vect{u}_3} =
  \sspan\set{\vect{u}_1,\ldots,\vect{u}_4}$.
\end{solution}