\chapter{Dynamic Programming}

\section{Introduction}
The core philosophy of dp:
\begin{enumerate}
\item The definition of \textbf{states}
\item The definition of the \textbf{transition functions} among states
\end{enumerate}

The so called concept dp as memoization of recursion does not grasp the core philosophy of dp.

The formula in the following section are unimportant. Instead, what is important is the definition of dp array and transition function derivation.
\subsection{Common practice}
\runinhead{Dummy.} Use dummies to avoid using if-else conditional branch.
\begin{enumerate}
\item Use $n+1$ dp arrays to reserve space for dummies.
\item Iteration range is $[1, n+1)$.
\item $n+k$ for k dummies
\end{enumerate}
\runinhead{State definition.} Two general sets of state definitions - the state
\begin{enumerate}
\item ends \textit{at} index $i$
\item ends \textit{before} index $i$
\item ends \textit{at} or \textit{before} index $i$
\end{enumerate}

\runinhead{Space optimization.} To avoid MLE, we need to carry out space optimization. Let $o$ be other subscripts, $f$ be the transition function.

Firstly,
$$
F_{i, o} = f\big(F_{i-1, o'}\big)
$$

should be reduced to
$$
F_{o} = f\big(F_{o'}\big)
$$

Secondly,
$$
F_{i, o} = f\big(F_{i-1, o'}, F_{i-2. o'}\big)
$$

should be reduced to
$$
F_{i, o} = f\big(F_{(i-1)\%2, o'}, F_{(i-2)\%2. o'}\big)
$$

More generally, we can be $(i-b)\%a$ to reduce the space down to $a$.

Notice:
\begin{enumerate}
\item Must iterate $o$ \textbf{backward} to un-updated value.
\end{enumerate}

\section{Sequence}\label{dpSequence}

\subsection{Single-state dp}
\runinhead{Longest common subsequence.} Let $F_{i, j}$ be the LCS at string $a[:i]$ and $b[:j]$. We have two situations: $a[i]==b[j]$ or not.
\begin{eqnarray*}
F_{i. j} = \left\{ \begin{array}{rl}
  F_{i-1, j-1}+1 &\mbox{// if $a[i]==b[j]$} \\
  \max\Big(F_{i-1, j},&\mbox{// otherwise} \\
  F_{i,j-1}\Big)
       \end{array} \right.
\end{eqnarray*}

\runinhead{Longest common substring.} Let $F_{i, j}$ be the LCS at string $a[:i]$ and
$b[:j]$. We have two situations: $a[i]==b[j]$ or not.
\begin{eqnarray*}
F_{i. j} = \left\{ \begin{array}{rl}
  F_{i-1, j-1}+1 &\mbox{// if $a[i]==b[j]$} \\
  0 &\mbox{// otherwise}
       \end{array} \right.
\end{eqnarray*}

Because it is not necessary that $F_{i,j}\geq F_{i',j'}, \forall i,j\cdot i>i', j>j'$, the $gmax=max\big(\{{F_{i,j}\}\big)$.
\runinhead{Longest increasing subsequence}. Find the longest increasing subsequence of an array $A$.

let $F_i$ be the LIS length ends at $A_i$.
\begin{eqnarray*}
F_i = \max(F[j]+1 \cdot\forall j < i) \text{ // if $A_i>A_j$}
\end{eqnarray*}

Then the global $maxa$ is:
$$
maxa = \max(F_i\cdot \forall i)
$$

Time complexity: $O(n^2)$

In code, notice the \pyinline{else}.
\begin{python}
F[i] = max(
    F[j] + 1 if A[i] > A[j] else 1
    for j in xrange(i)
)
\end{python}

Alternative solution using binary search in $O(n \log n)$ - Section \ref{extremeValueProblem}.

\runinhead{Maximum subarray sum.} Find the maximum subarray sum of $A$.

Let $F_i$  be the maximum subarray sum ending at $A_{i}$
$$
F_i = \max(F_{i-1}+A_{i}, 0)
$$

Then the global $maxa$ is:
$$
maxa = \max(F_i\cdot \forall i)
$$

\runinhead{Maximum sum of non-adjacent cells.} Get the maximum sum of non-adjacent
cells of an array $A$.

Let $F_i$ be the maximum sum of non-adjacent cells for $A[:i]$. You have tow options:
choose $A_{i-1}$ or not.
\begin{align*}
F_{i} = \max\big(F_{i-1}, F_{i-2}+A_{i-1}\big)
\end{align*}
\runinhead{Edit distance} Find the minimum number of steps required to convert words $A$ to $B$ using inserting, deleting, replacing.

Let $F_{i, j}$ be the minimum number
of steps required to convert $A[:i]$ to $B[:j]$.
\begin{eqnarray*}
F_{i, j} = \left\{ \begin{array}{rl}
  F_{i-1, j-1} &\mbox{// if $a[i]==b[j]$} \\
  \min\Big(F_{i, j-1}+1, &\mbox{// otherwise, insert}\\
  F_{i-1, j}+1, &\mbox{// delete}\\
  F_{i-1, j-1}+1\Big) &\mbox{// replace}\\
       \end{array} \right.
\end{eqnarray*}

\runinhead{Maximal square.} Find the largest rectangle in the matrix:
\begin{lstlisting}
1 0 1 0 0
1 0 1 1 1
1 1 1 1 1
1 0 0 1 0
\end{lstlisting}
Let $F_{i, j}$ represents the max square's length ended at $mat_{i, j}$ (lower right corner).
\begin{eqnarray*}
F_{i, j} = \left\{ \begin{array}{rl}
  \min\big(F_{i-1, j-1}, F_{i-1, j}, F_{i, j-1}\big)+1 &\mbox{// if $mat_{i, j}==1$} \\
  0 &\mbox{// otherwise}
       \end{array} \right.
\end{eqnarray*}

\subsection{Dual-state dp}
\runinhead{Maximal product subarray.} Find the subarray within an array $A$ which has the largest product.
\begin{itemize}
\item Let $small_i$ be the smallest product end with $A_i$.
\item Let $large_i$ be the largest product end with $A_i$.
\item The states can be negative.
\end{itemize}
\begin{eqnarray*}
&& small_i = \min\big( A_i,\ small_{i-1}\cdot A_i,\ large_{i-1}\cdot A_i \big)
\nonumber \\
&& large_i = \max\big( A_i,\ small_{i-1}\cdot A_i,\ large_{i-1}\cdot A_i \big)
\end{eqnarray*}

It can be optimized to use space $O(1)$.

\runinhead{Trapping Rain Water}
Given n non-negative integers representing an elevation map where the width of each
bar is 1, compute how much water it is able to trap after raining.
\begin{figure}[]
    \centerline{\includegraphics[height = 1in]{rainwatertrap}}
    \caption{Trapping Rain Water}
  \label{fig:rainwatertrap}
\end{figure}

Let $maxL_i$ be the $\max(A[:i])$; let $maxR_i$ be the $\max(A[i:n])$. The dp of obtaining max is trivial.

The the total volume $vol$:
$$
vol = \sum_i(\max\big(0,\min(maxL_i, maxR_{i+1})-A[i]\big))
$$
\runinhead{Zigzag subsequence.} Find the max length zigzag subsequence which goes up and down alternately within the array $A$.

Let $U_i$ be the max length of zigzag subsequence end at $A_i \wedge$ going up.

Let $D_i$ be the max length of zigzag subsequence end at $A_i \wedge$ going down.
\begin{align*}
U_i &= max(D_j+1 \cdot \forall j < i) \text{ // if $A_i > A_j$} \\
D_i &= max(U_j+1 \cdot \forall j < i) \text{ // if $A_i < A_j$}
\end{align*}

Notice in python implementation, don't use list comprehension since two states are interleaved and interdependent.

\section{String}
\runinhead{Word break.} Given a string $s$ and a dictionary of words $dict$, determine if $s$ can be segmented into a space-separated sequence of $dict$ words.

Let $F_i$ be whether \pyinline{s[:i]} can be segmented.
\begin{eqnarray*}
F_{i} = \left\{ \begin{array}{rl}
  F_{i-len(w)} &\mbox{// if $\exists w\in dict$, \pyinline{s[i-len(w):i]==w}}
\\
  false &\mbox{// otherwise}
       \end{array} \right.
\end{eqnarray*}
Return all such possible sentences. In original case, we use a bool array to record whether a dp could be segmented. Now we should use a vector for every dp to record how to construct that dp from another dp.

Let $F_i$ be all possible segmented words ends at \pyinline{s[i-1]}. $F_i$ is a list. $\exists F_i$ means $F_i$ is not empty.
\begin{eqnarray*}
F_{i} = \left\{ \begin{array}{rl}
  F_{i}+[w] &\mbox{// $\forall w\in dict\cdot$} \\
  & \mbox{ if \pyinline{s[i-len(w):i]==w} $\wedge\ \exists
F_{i-len(w)$ } \\
  F_{i}} &\mbox{// otherwise}
       \end{array} \right.
\end{eqnarray*}

Reconstruct the sentence from $F_i$. It is like building path for the tree. Using backtracking:
\begin{python}
def build(self, dp, i, cur, ret):
    if cur_index == 0:
        ret.append(" ".join(list(cur)))
        return

    # backtracking
    for word in dp[i]:
        cur.appendleft(word)
        self.build(dp, i-len(word), cur, ret)
        cur.popleft()

\end{python}

\runinhead{Is palindrome.} Given a string $s$, use an array to determine whether $s[i:j]$.

Let $P_{i,j}$  indicates whether $s[i:j]$ is palindrome. We have one condition - whether the head and the end letter are equal:
\begin{eqnarray*}
P_{i. j} = P_{i-1, j+1}\ \wedge\ s[i] = s[j-1]
\end{eqnarray*}

The code for palindrome dp is error-prone due to indexing. Notice that $i \in [0, n), j \in [i, n+1)$.
\begin{python}
n = len(s)
pa = [[False for _ in xrange(n+1)] for _ in xrange(n)]
for i in xrange(n):
    pa[i][i] = True
    pa[i][i+1] = True

for i in xrange(n-2, -1, -1):
    for j in xrange(i+2, n+1):
        pa[i][j] = pa[i+1][j-1] and s[i] == s[j-1]
\end{python}
\runinhead{Minimum palindrome cut.} Given a string s, partition s such that every substring of the partition is a palindrome. Return the minimum cuts needed for a palindrome partitioning of s.

Let $C_i$ be the min cut for $s[:i]$. We have 1 more cut from previous state to make $S[:i]$ palindrome.
\begin{eqnarray*}
C_{i} = \left\{ \begin{array}{rl}
  \min\big(C[k]+1 \cdot \forall k<i \big) &\mbox{// if $s[k:i]$ is palindrome}
\\
  0 &\mbox{// otherwise}
       \end{array} \right.
\end{eqnarray*}
\begin{python}
def minCut(self, s):
  n = len(s)

  P = [[False for _ in xrange(n+1)] for _ in xrange(n+1)]
  for i in xrange(n+1):  # len 0
    P[i][i] = True
  for i in xrange(n):  # len 1
    P[i][i+1] = True

  for i in xrange(n, -1, -1):  # len 2 and above
    for j in xrange(i+2, n+1):
      P[i][j] = P[i+1][j-1] and s[i] == s[j-1]

  C = [i for i in xrange(n+1)]  # max is all cut
  for i in xrange(n+1):
    if P[0][i]:
      C[i] = 0
    else:
      C[i] = min(
          C[j] + 1
          for j in xrange(i)
          if P[j][i]
      )

  return C[n]
\end{python}
\runinhead{ab string.} Change the char in a str only consists of `a' and `b' to non-decreasing order. Find the min number of char changes.

Two-state dp: `a' $\rightarrow$ `b' and  `b' $\rightarrow$ `a'. 1 cut into 2 segment.

\runinhead{abc string.} Follow up for ab string. Three-state dp: $chr \neq a, chr \neq b, chr \neq c$. 2 cuts into 3 segments.

\section{Combinatorics}
\subsection{Tree}
\runinhead{Number of different BSTs.} It can be solved using Catalan number (Section \ref{section:catalanNumber}), but here goes the dp solution.
\begin{lstlisting}
   1         3     3      2      1
    \       /     /      / \      \
     3     2     1      1   3      2
    /     /       \                 \
   2     1         2                 3
\end{lstlisting}

Let $F_i$ be the \#BSTs constructed from $i$ elements. The pattern is:
\begin{align*}
F_3 = F_0*F_2 + F_1*F_1 + F_2*F_0
\end{align*}

Thus, in general,
\begin{align*}
F_i = \sum(F_{j}*F_{i-1-j} \cdot \forall j< i)
\end{align*}

\section{Backpack}
\subsection{Classical}
Given $n$ items with weight $w_i$ and value $v_i$, an integer $C$ denotes the size of a backpack. What is the max value you can fill this backpack?

Let $F_{i, c}$ be the max value we can carry for index $0..i$ with capacity $c$. We have 2 choices: take the $i$-th item or not.
\begin{eqnarray*}
F_{i, c}= \max\big(&&F_{i-1, c}, \\
&&F_{i-1, c-w_i}+v_i\big)
\end{eqnarray*}
Advanced backpack problem\footnote{\href{http://github.com/tianyicui/pack}{Nine Lectures in Backpack Problem}.}.

\subsection{Sum}
\runinhead{k sum.} Given $n$ distinct positive integers, integer $k$ ($k \leq n$) and a number target. Find $k$ numbers where sum is target. Calculate the number of solutions. Since we only need the number of solutions, thus it can be solved using dp. If we need to enumerate all possible answers, need to do dfs instead.

$$
sum{j \choose i} = v
$$

Let $F_{i, j, v}$ means the \#ways of selecting $i$ elements from the first $j$ elements so that their sum equals to $v$. $j$ is the scanning pointer.

You have two options: either select $A_{j-1}$ or not.
$$
F_{i, j, v} = F_{i-1, j-1, v-A_{j-1}} + F_{i, j-1, v}
$$
Time complexity: O(n^2 k)
\section{Local and Global Extremes}
\subsection{Long and short stocks}
The following formula derives from the question: Best Time to Buy and Sell Stock IV. Say you have an array for which the $i$-th element is the price of a given stock on day $i$. Design an algorithm to find the maximum profit. You may complete at most $k$ transactions.

Let $local_{i, j}$ be the max profit with $j$ transactions with last transactions \textbf{ended at} day $i$. Let $global_{i, j}$ be the max profit with transactions \textbf{ended at} or \textbf{before} day $i$ with $j$ transactions.

To derive transition function for $local$, for any given day $i$, you have two options: 1) transact in one day; 2) hold the stock one more day than previous and then transact. The latter option is equivalent to revert yesterday's transaction and instead transact today.

To derive transition function for $global$, for any given day $i$, you have two options: 1) transact today; 2) don't transact today.
\begin{eqnarray*}
&& local_{i,j} = \max\Big(global_{i-1.j-1}+\Delta, local_{i-1,j}+\Delta\Big) \nonumber \\
&& global_{i,j} = \max\Big(local_{i, j}, global_{i-1,j}\Big)
\end{eqnarray*}
, where $\Delta$ is the price change (i.e. profit) at day $i$.\\
Notice:
\begin{enumerate}
\item Consider opportunity costs and reverting transaction.
\item The global min is not $glocal[-1]$ but $\max\big(\{global[i]\}\big)$.
\item You must sell the stock before you buy again (i.e. you can not have higher than 1 in stock position).
\end{enumerate}

\runinhead{Space optimization.}
\begin{eqnarray*}
&& local_{j} = \max\Big(global_{j-1} + \Delta, local_{j}+\Delta\Big)
\nonumber \\
&& global_{j} = \max\Big(local_{j}, global_{j}\Big)
\end{eqnarray*}

Notice,
\begin{enumerate}
\item Must iterate $j$ \textbf{backward}; otherwise we will use the updated value.
\end{enumerate}

\runinhead{Alternative definitions.}
Other possible definitions: let $global_{i, j}$ be the max profit
with transactions ended at or before day $i$ with \textbf{up to} $j$ transactions. Then,
\begin{eqnarray*}
&& local_{i,j} = \max\Big(global_{i-1.j-1} + \max(0, \Delta), local_{i-1,j}+\Delta\Big)
\nonumber \\
&& global_{i,j} = \max\Big(local_{i, j}, global_{i-1,j}\Big)
\end{eqnarray*}
and $global[-1]$ is the global max.

The complexity of the alternative definitions is the same as the original definitions. The bottom line is that different definitions of states result in different transition functions.

\section{Game theory - multi players}
Assumption: the opponent take the optimal strategy for herself.

\subsection{Coin game}
\runinhead{Same side} There are $n$ coins with different value in a line. Two players take turns to take 1 or 2 coins from left side. The player who take the coins with the most value wins.

let $F_i^p$ represents maximum values he can get for index $i..last$, for the person p. There are 2 choices: take the $i$-th coin or take the $i$-th and $(i+1)$-th coin.
\begin{eqnarray*}
F_i^p = \max\big(&A_i&+S[i+1:]-F_{i+1}^{p'},  \\
&A_i&+A_{i+1}+S[i+2:]-F_{i+2}^{p'}\big)
\end{eqnarray*}
The above equation can be further optimized by merging the sum $S$.

\runinhead{Dual sides}There are n coins in a line. Two players take turns to take a coin from one of the ends of the line until there are no more coins left. The player with the larger amount of money wins.

let $F_{i, j}^p$ represents maximum values he can get for index $i..j$, for
the person p. There are 2 choices: take the $i$-th coin or take the $j$-th coin.
\begin{eqnarray*}
F_{i,j}^p = \max\big(&A_i&+S[i+1:j]-F_{i+1,j}^{p'},  \\
&A_j&+S[i:j-1]-F_{i,j-1}^{p'}\big)
\end{eqnarray*}


\section{The Riemann-Stieltjes Integral}

\subsection{Exercise 1}
We first note that $\inf{f(x)} = 0$ on any interval $ [p_{i-1}, p_i] \subset [a, b]$. Similarly,
$\sup{f(x)} = 0$ if $x_0 \notin [p_{i-1}, p_i]$ and 1 otherwise. Thus, we proceed by constructing a
partition $P$ of $[a, b]$ such that $x_0 \in [p_{i-1}, p_i]$ and $\alpha(p_i) - \alpha(p_{i-1}) < \epsilon$.
This is possible since $\alpha$ is continuous at $x_0$, so there exists $\delta$ such that choosing
$p_i - p_{i-1} < \delta$ gives us the previous inequality. We then have that
$U(P, f, \alpha) - L(P, f, \alpha) < \epsilon$, so $f \in \mathscr{R}(\alpha)$. Furthermore, since
$L(P, f, \alpha) = 0$ for all $P$, we have that $\int_{a}^{b} f d\alpha = 0$.

\subsection{Exercise 2}
Since we're given $f(x) \geq 0$ and $\int_{a}^{b} f(x) dx = 0$, we have that $m = \inf{f(x)} = 0$. Letting
$m_i$ and $M_i$ denote the infimum and supremum of $f$ on the interval $[x_{i-1}, x_i] \subset [a, b]$,
we further have that $m_i \leq m \implies m_i = 0 \: \forall i$. Additionally, since $f$ is continuous
on $[a, b]$ and therefore uniformly continuous (from compactness), we can choose a $\delta$ such that
\begin{align*}
        \abs{x_i - x_{i-1}} < \delta &\implies \abs{f(x_i) - f(x_{i-1})} < \epsilon \\
                                     &\implies \abs{M_i - m_i} < \epsilon \implies M_i < \epsilon
\end{align*}
Thus, all of the $M_i$ can be made arbitrarily small, implying that $\sup{f(x)} < \epsilon$ for every
positive $\epsilon$. This gives us that $f(x) = 0$.

\subsection{Exercise 3}
(a) Suppose $f(0+) = f(0)$. We consider the partition $P = {x_0, x_1, x_2, x_3}$ where
$x_0 = -1, x_1 = 0, x_3 = 1$. Then $U(P, f, \beta_1) = M_2$ and $L(P, f, \beta_1) = m_2$, and
we have that $M_2 \to f(0)$ and $m_2 \to f(0)$ as $x_2 \to 0$, so $f \in \mathscr{R}(\beta_1)$.
For the other direction, if $f \in \mathscr{R}(\beta_1)$, then there exists $P$ such that
$U(P, f, \beta_1) - L(P, f, \beta_1) < \epsilon$. We can then consider the refinement $P^*$ of $P$
that contains an interval of the form $[0, x_i]$. Suppose now that $\abs{f(0+) - f(0)} = \epsilon > 0$.
Then
\begin{align*}
        M_i - m_i \geq \abs{f(0+) - f(0)} > \epsilon
\end{align*}
which is a contradiction, so $f(0+) = f(0)$.

(b) Only difference from (a) is that we need $f(0-) = f(0)$ instead of $f(0+) = f(0)$. The only changes that
need to be made to the proof of (a) involve replacing $[0, x_i]$ with $[x_{i-1}, 0]$.

(c) If $f$ is continuous at 0, we can consider the partition $P$ that contains $x_{i-1} < x_i = 0 < x_{i+1}$.
Then $U(P, f, \beta_2) = \frac{M_{i} + M_{i+1}}{2}$ and $L(P, f, \beta_2) = \frac{m_i + m_{i+1}}{2}$.
Since $f$ is continuous at 0, $M_i, M_{i+1}, m_i, m_{i+1} \to f(0)$ as $x_{i-1} \to 0$ and $x_{i+1} \to 0$,
so $f \in \mathscr{R}(\beta_2)$.
For the other direction, we proceed similarly to the proof of (a) and see that
\begin{align*}
        \frac{1}{2} (M_i + M_{i + 1}) - \frac{1}{2} (m_i + m_{i + 1}) &= \frac{1}{2} ((M_i - m_i) + (M_{i+1} - m_{i+1})) \\
                                                                      &\geq \frac{1}{2} (\abs{f(0-) - f(0)} + \abs{f(0+) - f(0)}) \\
                                                                      &> \epsilon
\end{align*}
for some $\epsilon > 0$ unless $f(0-) = f(0+) = f(0)$, so  $f$ must be continuous at 0 if  $f \in \mathscr{R}(\beta_2)$.

(d) If $f$ is continuous at 0 then we have $f(0+) = f(0-) = f(0)$, so we are done by parts (a)-(c).

\subsection{Exercise 4}
From the density of the rationals and irrationals in the reals, we have that $M_i - m_i = 1 \: \forall i$, so
$f \notin  \mathscr{R}$.

\subsection{Exercise 5}
If we consider $f(x) = 1$ for all rational $x$ and $f(x) = -1$ for all irrational $x$, we have that
$f^2 \in \mathscr{R}$ but $f \notin \mathscr{R}$. However, if $f^3 \in \mathscr{R}$, then $f \in \mathscr{R}$.
This is because $m \leq f^3 \leq M$ (since $f$ is bounded) and $x^{\frac{1}{3}}$ is continuous on any
$[m, M]$, so we can apply Theorem 6.11.

\subsection{Exercise 6}
$P$ is compact, so the open cover consisting of neighborhoods around each point of $P$ has a finite subcover.
Thus, $P$ can be covered by finitely many segments. Additionally, the total length of these segments can be
made arbitrarily small, since $P$ contains no segments itself. We can then proceed exactly as in the proof of
Theorem 6.10 to get the desired result.

\subsection{Exercise 7}
(a) If $f \in \mathscr{R}$, then by Theorem 6.12 (c) we have
\begin{align*}
        \abs{\int_{c}^{1} f dx - \int_{0}^{1} f dx} &= \abs{\int_{0}^{c} f dx} \\
                                                    &\leq cM < \epsilon \quad \text{for} \:\:  c < \frac{\epsilon}{M}
\end{align*}
Where $M = \sup{f}$ on $[0, 1]$.

(b) Consider $f$ such that
\begin{align*}
        f(x) =
        \begin{cases}
                0 & x = 0 \\
                \frac{1}{x} & 0 < x < \frac{1}{2} \\
                0 & x = \frac{1}{2} \\
                -\frac{1}{x} & \frac{1}{2} < x \leq 1
        \end{cases}
\end{align*}
Then $\int_{0}^{1} f dx = 0$, but $\int_{0}^{1} \abs{f} dx$ diverges.

\subsection{Exercise 8}
We see that the partition $P = {1, 2, ..., b + 1}$ of the interval $[1, b + 1]$ corresponds to the upper sum
$U(P, f) = \sum_{n = 1}^b f(n)$, and $\int_{1}^{b + 1} f(x) dx \leq U(P, f)$.
Thus, if $\sum_{n = 1}^\infty f(n)$ converges, then we have that
$\int_{1}^{b + 1} f(x) dx$ is bounded and monotonically increasing (since $f(x) \geq 0$),
so $\int_{1}^{\infty} f(x) dx $ converges. For the other direction, we can take the same partition $P$
and consider $L(P, f) = \sum_{n = 1}^b f(n + 1)$. Since $L(P, f) \leq \int_{1}^{b + 1} f(x) dx$, if
$\int_{1}^{\infty} f(x) dx$ converges then so does $\sum_{n = 2}^\infty f(n)$, and we are done (as
$f(1) < \infty$).

\subsection{Exercise 9}
Theorem: Let $f$ and $g$ be differentiable functions on $[0, \infty)$ such that
$\lim_{x \to \infty} f(x)g(x) = 0$ and $f', g' \in \mathscr{R}$ for every $[0, b]$. Then we have that
\begin{align*}
        \int_{0}^{\infty} f(x) g'(x) dx = -f(0)g(0) - \int_{0}^{\infty} f'(x) g(x)
\end{align*}
if both improper integrals converge.

Proof: Take limits on both sides of the integration by parts formula.

Letting $f(x) = \frac{1}{1 + x}$ and $g'(x) = cos(x)$ in the above formula shows that
\begin{align*}
        \int_{0}^{\infty} \frac{cos(x)}{1 + x} dx = \int_{0}^{\infty} \frac{sin(x)}{(1 + x)^2} dx
\end{align*}

We have that $\abs{cos(n)} + \abs{cos(n + 1)} \geq c$ for some
constant $c$ since $cos(n)$ and $cos(n + 1)$ cannot both be 0. As such,
$\sum_{n = 1}^\infty \frac{cos(n)}{1 + n}$ diverges since the harmonic series diverges and therefore
the integral on the left also diverges by the integral test.

\subsection{Exercise 10}
(a) We can use the convexity of $e^x$ to show this. Let $\lambda = \frac{1}{p}$ and let $u^p = e^a, v^q = e^b$
for some $a, b$ (this is possible because $u, v \geq 0$). Then we have
\begin{align*}
        e^{\lambda a + (1- \lambda) b} &\leq \lambda u^p + (1 - \lambda) v^q \\
        e^{\lambda a} e^{(1 - \lambda) b} &\leq \frac{u^p}{p} + \frac{v^q}{q} \\
        uv &\leq \frac{u^p}{p} + \frac{v^q}{q}
\end{align*}

(b) From part (a) we have that
\begin{align*}
        f(x) g(x) &\leq \frac{f^p(x)}{p} + \frac{g^q(x)}{q} \\
        \int_{a}^{b} fg d\alpha &\leq 1
\end{align*}

(c) We can use part (a) with  $u = \frac{\abs{f(x)}}{\bigg(\int_{a}^{b} f^p d\alpha\bigg)^{\frac{1}{p}}}$
and $v = \frac{\abs{g(x)}}{\bigg(\int_{a}^{b} g^q d\alpha\bigg)^{\frac{1}{q}}}$ to get the desired result.

(d) Assuming the limits exist, the inequality follows for impromper integrals by taking limits on both sides
and then using limit rules.


\documentclass[oneside]{book}
\usepackage[utf8]{inputenc}
\usepackage{authblk}
\usepackage{setspace}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{geometry}
\usepackage{amsthm}
\usepackage{runic}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage[breaklinks=true,a4paper=true,pagebackref=true]{hyperref}
\graphicspath{ {figures/} }
\geometry{
 a4paper,
 total={170mm,257mm},
 left=20mm,
 top=20mm,
}
\hypersetup{
    colorlinks=true,
    linktoc=true,
    linkcolor=blue,
}
\title{Libraria Algebrae}
\author{Liam Gardner}
\date{\today}
%\doublespacing
\newcommand\tab[1][1cm]{\hspace*{#1}}
\newcommand\nextline{\newline\tab}
\newcommand\nextquestion{\newline\newline}
\newcommand\soln{$\text{sol}^\text{n}\text{ }$}
\newcommand\fs{\mbox{\large $\mathrlap{f}s\,$}\,}
\newcommand\thm[2]{\section*{Theorem: #1}\label{sec:#2}\addcontentsline{toc}{section}{Theorem: #1}}
\newcommand\propn[2]{\section*{Proposition: #1}\label{sec:#2}\addcontentsline{toc}{section}{Proposition: #1}}
\newcommand\defn{\textbf{Definition}: }

\renewcommand\mod[1]{\text{ }\left(\text{mod }#1\right)}

\begin{document}
\DeclarePairedDelimiter\abs{\lvert}{\rvert}

\maketitle
\tableofcontents
\chapter{Linear Diophantine Equations in $\mathbb{Z}^2$}
\tab
Note that for the general equation $ax+by$, we assume that $ab\neq0$, since if one is 0, then the equation is trivial. We wish to answer three fundamental problems
\begin{enumerate}
\item Does there exist an integer solution?
\item If the answer is yes, find an integer solution.
\item Can we find \textit{all} solutions?
\end{enumerate}
\tab
It is common for existential theorems (those that say solutions exist) to not give a means of how to find said solutions.
\subsection{Example}
Solve $506x + 391y = 23$. Notice that $\gcd(506,391)=23$. Thus, by b\'ezout's\ lemma, a solution exists. We can use the EEA (Extended Euclidean Algorithm) to find a solution to the equation.
\newline
\begin{center}
\begin{tabular}{|c|c|c|c|}
\hline
$x$ & $y$ & $r$ & $q$ \\
\hline
\hline
1 & 0 & 506 & 0 \\
0 & 1 & 391 & 0 \\
1 & -1 & 115 & 1.0 \\
-3 & 4 & 46 & 3.0 \\
7 & -9 & 23 & 2.0 \\
-17 & 22 & 0 & 2.0 \\
\hline
\end{tabular}
\end{center}
\tab
Thus, we know that $(7,-9)$ is a solution. Now, we can subtract the equation $506x+391y=23$ with $506x_0 + 391y_0=23$, which gives $506(x-x_0) + 391(y-y_0) = 0$. Thus, we get $506(x-x_0) = -391(y-y_0)$. We can divide by the GCD of 506 and 391 to get the equation $22(x-x_0) = -17(y-y_0)$. Since the GCD is a common divisor to 506 and 391, we get that both $\frac{506}{23}$ and $\frac{391}{23}$ are integers and are coprime to each other. Now, since we know that $-17\lvert -17(y-y_0)$, and that $-17(y-y_0)=22(x-x_0)$, we get that $-17\lvert 22(x-x_0)$. Thus, by CAD, we get $17\lvert (x-x_0)$. Therefore, we find that $x-x_0 = 17n$ for some $n\in\mathbb{Z}$. Thus, a solution for $x$ can be found with $x_0 + 17n$, $\forall n\in\mathbb{Z}$.
\nextline
Following the same process, we get that $y=y_0+22n$. Therefore, all solutions to the Linear Diophantine Equation $506x + 391y = 23$ are given by the points $(7 + 17n, -9 - 22n)$.
\newline
\newline
Solve $506x + 391y = 24$.
\nextline
There are no \soln: Since $23=\gcd(506,391)$, we get that $23\lvert 506x+391y$ however, $23\nmid 24$, therefore, we've run into a contradiction.
\newline
\newline
Solve $506x + 391y = 46 = 2\cdot23$. We know that $506\cdot7 + 391\cdot(-9) = 23$, thus if we multiply both sides by two, we get that $506(7\cdot2) + 391(-9\cdot2) = 2\cdot23 = 46$. $506(14) + 391(-18) = 46$ gives the \soln (14,-18).

\thm{LDET Part 1}{ldeta}

Suppose $a,b,c\in\mathbb{Z}$ and $ab\neq0$ then $ax+by=c$ has a \soln in integers if and only if $\gcd(a,b)\lvert c$.
\subsection{Proof of forwards direction}
\tab
Assume $ax+by=c$ has an integer \soln $(x_0, y_0)$. Let $d=\gcd(a,b)$. Since $d\lvert a$ and $d\lvert b$, and since $c=ax_0+by_0$, then by Divsibility of Integer Combinations, $d\lvert c$.
\subsection{Proof of backwards direction}
\tab
Let $d=\gcd(a,b)$, and assume that $d\lvert c$, thus $c=kd$ \fs integer $k$. Then By B\'ezout's\ Lemma, we can find $x_0$ and $y_0$ $\in\mathbb{Z}$ such that $ax_0+by_0=d$, but then $k(ax_0+by_0)=kd$. Thus $a(kx_0) + b(ky_0) = c$
\subsection{Remark}
\tab
if $d\lvert c$ then the proof tells us how to find a \soln.
\begin{enumerate}
\item solve $ax+by=d$ using EEA to get $(x,y)=(x_0,y_0)$.
\item take $x=kx_0$ and $y=ky_0$, where $k=\frac{c}{d}$.
\end{enumerate}

\thm{LDET Part 2}{ldetb}

Suppose that $(x_0, y_0)$ is a particular solution to the LDE $ax+by=c$.
\nextline
Then the set of all \soln $\in\mathbb{Z}$ is given by the following set:
$$S = \left\{
(x,y) \Huge\mid x=x_0 + \frac{b\cdot n}{\gcd(a,b)},\,\,\, y=y_0 - \frac{a\cdot n}{\gcd(a,b)}
\right\}$$
\subsection{Proof}
\tab
Let $D$ be the set of all integer solutions to $ax+by=c$. I.E.
$$D = \left\{(x,y) \mid x,y,\in\mathbb{Z}, ax+by=c \right\}$$
\tab
This can be proven by showing $S \subseteq D$ and $D \subseteq S$
\newline $S\subseteq D:$
\nextline
Let $(x,y) \in S$, thus $x=x_0 + \frac{bn}{d}$ and $y=y_0-\frac{an}{d}$.
$$ax+by = a\left(x_0+\frac{bn}{d}\right) + b\left(y_0 - \frac{an}{d}\right)$$
$$= ax_0 + by_0 = c$$
\tab
since by definition, we know that $x_0$ and $y_0$ are a particular solution to the equation. Therefore, $S\subseteq D$
\newline $D\subseteq S:$
\nextline
Let $(x,y)\in D$, thus $x,y\in\mathbb{Z}$ and $ax+by=c$. Since, $(x_0,y_0)\in D$, we know that $ax_0 + by_0 = c$. We can subtract $ax_0+by_0=c$ from $ax + by = c$ to get $a(x-x_0) + b(y-y_0) = 0$. Dividing by the GCD of $a$ and $b$, we get that $\frac{a(x-x_0)}{d} = \frac{-b(y-y_0)}{d}$. Thus, since $\frac{b}{d}\lvert \frac{a}{d}(x-x_0)$ then by CAD, we get that $\frac{b}{d}\lvert (x-x_0)$, since $\frac{b}{d}$ and $\frac{a}{d}$ are coprime. Then we get $x-x_0=n\frac{b}{d}\Rightarrow x=x_0+\frac{bn}{d}$ \fs $n\in\mathbb{Z}$. Similarly, we get $y=y_0-\frac{an}{d}$. Therfore, $(x,y)\in S$.
\section{More Examples}
$12x+18y=13$.
\nextline
$\gcd(12,18) = 6$. Since $6\nmid13$ the equation has no \soln by \hyperref[sec:ldeta]{LDET1}
\newline
\newline
$14x-49y=28$
\nextline
$\gcd(14,-49) = 7$. Since $7\lvert 28$ the equation has solutions by \hyperref[sec:ldetb]{LDET2}
\nextline
Consider $14x-49y=7\Rightarrow 2x-7y=1$ which has \soln $(4,1)$. If we multiply $14(4) - 49(1) = 7$ by 4, we get that $14(x_0\cdot4) - 49(y_0\cdot4) = 7$, and from this we can get all \soln using LDET2.
\newline
\newline
\tab
Find all \soln to $15x+35=5$.
\newline
\begin{center}
\begin{tabular}{|c|c|c|c|}
\hline
$x$ & $y$ & $r$ & $q$ \\
\hline
\hline
1 & 0 & 15 & 0 \\
0 & 1 & 35 & 0 \\
1 & 0 & 15 & 0.0 \\
-2 & 1 & 5 & 2.0 \\
7 & -3 & 0 & 3.0 \\
\hline
\end{tabular}
\end{center}
\tab
Thus, we know that $\gcd(15,35)=5$, and $5\lvert 5$, there is a \soln. By EEA, we find $x=-2, y=1$ is a \soln. Then, we can find the general solution using
\hyperref[sec:ldetb]{LDET2}
 to be $x=-2+\frac{35n}{5}\Rightarrow 7n-2$, $y=1+\frac{15n}{5}\Rightarrow 1+3n$
\subsection{Geometric Understanding}
\tab
Graphing the line $15x+35y=5$ or $3x+7y=1$, we can rearrange for $y$ to get $y=\frac{-3}{7}x+\frac{1}{7}$. Thus, picking any lattice point, we can construct a triangle of length 7 and height 3 from that point to find the next lattice point.

\begin{figure}[h]
\centering
\includegraphics[width=0.5\textwidth]{l24f0}
\caption{Triangle formed from moving between two lattice points}
\end{figure}

\subsection{Nonnegative \soln}
\tab
The solutions to $15x+35y=5$ is given by $(-2+7n, 1-3n)$, these will be nonnegative if $x\geq0$ and $y\geq0$.
$$-2+7n\geq 0 \iff n\geq \frac{2}{7}$$
$$1-3n \geq 0 \iff n \leq \frac{1}{3}$$
Thus, since there are no integer solutions in the range $\frac{2}{7}\leq n \leq \frac{1}{3}$, as $n\in\mathbb{Z}$ there are no nonnegative solutions.
\subsection{Find all integer \soln to $15x+35y^2 = 5$}
\tab
Let $Y=y^2$, then, since we have the \soln to $15x+35Y=5$, given by $(-2+7n, 1-3n)$, all we have to do is see when $1-3n$ is a perfect square.
One way to solve this is to say let $z=y^2$ and solve $1-3n=z$. We can also solve this algebraically
\newline
\begin{center}
\begin{tabular}{c c c}
$\iff$ & $1-3n = y^2$ & \\
$\iff$ & $-3n=y^2-1$ & \\
$\iff$ & $3\lvert y^2-1$ & by euclid's lemma \\
$\iff$ & $3\lvert(y-1)(y+1)$ & \\
\end{tabular}
\end{center}
\tab
By Euclid's Lemma, we know either $3\lvert(y-1)$ or $3\lvert(y+1)$, though not both. Suppose $3\lvert(y-1) \iff y-1=3k\iff y=3k+1\, \fs k\in\mathbb{Z}$
Then, if $y=3k+1$, we get that $y^2=(1+3k)^2=1-3(-2k-3k^2)$. Let $n=(-2k-3k^2)$, then we get $y^2=1-3n$. Thus, if we take $x=-2+7n=-2+7(-2-3k^2)$ and $y=1-3k$, we get a perfect square \soln to the diophantine equation.
\nextline
If $3\lvert y+1$, then we have $y=-1+3l\, \fs l \in \mathbb{Z}$. Then $y^2=(-1+3l)^2=1-3(2l+3l^2)$. Thus, $y=1-3l$ and $x=-2+7(2l+3l^2)$. Therfore, we can generate infinitely many perfect square solutions.
\chapter{Congruence and Modular Arithmetic}
\section{Clockwork Arithmetic Analogy}
\tab
Imagine a clock, we know that a clock has 12 spokes. If we look at it and see the hour hand at 2, and we know that 12 has already passed, then we also know that the clock really means it's 14. Thus, we can say $2\approx14$
\section{Definition}
\tab
$\forall a,b\in\mathbb{Z}$, we say that ``$a$ is congruent to $b$ mod(ulo) 12'' if $m\lvert(a-b)$.
\nextline
Notation: $a\equiv b\mod{m}$
\section{Examples}
\tab
$m=1$ then $a\equiv b\mod{1} \iff 1\lvert (a-b)$ which is true $\forall a,b\in\mathbb{Z}$
\nextline
$m=2$ then $a\equiv b\mod{2} \iff 2\lvert (a-b) \iff a-b$ is even $\iff a$ and $b$ are both even or both odd.
\nextline
$2\equiv-116\mod{2}$, however $3\not\equiv 10024\mod{2}$
\nextline
$14\equiv 2\mod{12} \iff 12\lvert(14-2)$
\nextline
$6\equiv 26\mod{10} \iff 10\lvert(6-26)$
\nextline
$6\not\equiv -26\mod{10} \implies 10\nmid(6+26)$
\propn{Congruence is an Equivalence Relation}{CER}
\begin{center}
\begin{tabular}{l|l}
$\forall m\in\mathbb{N}, \forall a,b,c,\in\mathbb{Z}$ & Congruence is \\
\hline
\hline
$a\equiv a\mod{m}$ & symmetric \\
if $a\equiv b\mod{m}$ and $b\equiv c\mod{m}$ then $a\equiv c\mod{m}$ & transitive \\
$a\equiv b \mod{m}\implies b\equiv a\mod{m}$ & Reflexive
\end{tabular}
\end{center}
\tab
Remark: Any relation $a\sim b$ that satisfies all above properties is called an equivalence relation.
\nextline
In calculus, we say two functions $f(x)\sim g(x)$ are an equivalence relation if $f^\prime(x) = g^\prime(x)$
\section{Recap}
\tab
Fix $m\in\mathbb{N}$, $\forall a,b\in\mathbb{Z}$
\nextline
\begin{tabular}{l c l}
$a\equiv b\mod{m}$ & $\iff$ & $m\lvert (a-b)$ \\
& $\iff$ & $a-b=mk\,\, \fs k\in\mathbb{Z}$ \\
& $\iff$ & $a=b+mk$
\end{tabular}
\newline
\propn{Arithmetic Rules of Congruence}{ARC}
Suppose $a\equiv a^\prime \mod{m}$ and $b\equiv b^\prime \mod{m}$ then,
\begin{enumerate}
\item $a+b\equiv a^\prime + b^\prime \mod{m}$
\item $a-b\equiv a^\prime - b^\prime \mod{m}$
\item $ab\equiv a^\prime b^\prime \mod{m}$
\end{enumerate}
\subsection{Examples}
$$2\equiv 9\mod{7}\land3\equiv 17\mod{7}\implies 2+3\equiv9+17\mod{7}\implies 5=26$$
\newline
$$56\cdot30 \mod{40}$$
$$56=16+40\equiv 16\mod{40}$$
$$30=-10\equiv \mod{40}$$
$$56\cdot30 \equiv 16\cdot(-10)\mod{40}$$
$$\equiv-160\mod{40}$$
$$\equiv-40\cdot4\mod{40}\equiv 0\mod{40}$$
\subsection{Proof of addition}
\tab
Since $a\equiv a^\prime \mod{m}$ and $b\equiv b^\prime \mod{m}$ then $m\lvert(a-a^\prime)$ and $m\lvert(b-b^\prime)$
\nextline
We have $(a+b)-(a^\prime-b^\prime)=a-a^\prime + b-b^\prime$, thus by DIC we get $m\lvert((a+b)-(a^\prime+b^\prime))$. Therefore $a+b\equiv a^\prime + b^\prime \mod{m}$
\subsection{Remark on Division}
\tab Care is needed with division
$ab\equiv ac\mod{m} \not\implies b\equiv c\mod{m}$ even if $a\not\equiv 0 \mod{m}$
\nextline
$$10\equiv 4\mod{6}$$
$$2\cdot5 \equiv 2\cdot2 \mod{6}$$
$$5\not\equiv 2\mod{6}$$
\propn{Congruent Division}{CD}
\tab
If $ab\equiv ac\mod{m}$ and $a$ is coprime to $m$, then $b\equiv c\mod{m}$
\propn{Congruent Powers}{CP}
\tab
$a\equiv b\mod{m}\implies a^n\equiv b^n\mod{m}$
\subsection{Proof}
\tab
$$ab\equiv bc\mod{m}\iff m\lvert (ab-ac)\iff m\lvert a(b-c)$$
\nextline
Then by CAD we get $m\lvert(b-c)$ since $m$ and $a$ are coprime\nextline
$\implies b\equiv c \mod{m}$
\nextline
By applying the above proposition repeatedly; if $a_1\equiv a^\prime_1 \mod{m}, a_2\equiv a^\prime_2 \mod{m}, \cdots a_n\equiv a^\prime_n \mod{m}$, then we get the following result
\begin{enumerate}
\item $a_1+\cdots+a_n\equiv a^\prime_1 + \cdots a^\prime_n \mod{m}$
\item $a_1-\cdots-a_n\equiv a^\prime_1 - \cdots a^\prime_n \mod{m}$
\item $a_1\cdots a_n\equiv a^\prime_1 \cdots a^\prime_n \mod{m}$
\item (special case) $\forall q \in \mathbb{N}, a^q \equiv \left(a^\prime\right)^q \mod{m}$
\end{enumerate}
\subsection{More Examples}
Simplify $4^10 \mod{18}$\nextline
$4^{10} = \left(4^2\right)^5 = 16^5=(18-2)^5$ \nextline
$(18-2)^5\equiv-2^5\mod{18}$ \nextline
$\equiv-32\mod{18}$ \nextline
$-32+2\cdot 18\mod{18}$ \nextline
$4\mod{18}$
\newline
\newline
Is $3^9 + 62^{2020} - 20$ divisible by 7?
\nextline
Let $n=3^9+62^{2020}-20$. We know that $7\lvert n \iff 7\lvert(n-0) \iff n\equiv0\mod{7}$
\nextline
We can compute $3^9=\left(3^3\right)^3=27^3=(28-1)^3$ \nextline
$\equiv (-1)^3\mod{7}$\nextline
$\equiv -1\mod{7}$\nextline
We also know that $62^{2020} = (63-1)^{2020} \equiv (-1)^{2020}\mod{7} \equiv 1$ \nextline
$20=21-1\equiv -1\mod{7}$ \nextline
Using \hyperref[sec:ARC]{The arithmetic rules} we get that $n\equiv -1+1-(-1)\mod{7}\equiv1\mod{7}$ and thus $n$ is not divisible by 7.
\thm{Congruence and Remainders}{CR}
\subsection{Example}
\tab
What day of the week is it going to be a year from now?
\nextline
Since days cycle every 7, let's determine $365\mod{7}$. We know that $350=50\cdot7$ and thus \nextline $365 = 350 + 15$ \nextline
$=7\cdot50+14+1$ \nextline
$=7\cdot50+7\cdot2+1$\nextline
$=7(50+2)+1$\nextline
$\equiv 1\mod{7}$.\nextline
$365\equiv1\mod{7}$\nextline
Therefore, the day of the week one year from now is the same as the day of the week tomorrow.
\subsection{Observation}
\tab
if $n\in\mathbb{Z}$ \nextline
Any block of consecutive numbers will cycle through the numbers 0-6 inclusive:
$$\{\cdots,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,\cdots\}$$
$$\equiv\{\cdots,0,1,2,3,4,5,6,0,1,2,3,4,5,6,0,1,2,3,4,5,6,\cdots\}\mod{7}$$
\section{Warning}
\tab
$\forall a,b,b^\prime \in \mathbb{N}$, if $b\equiv b^\prime \mod{m}$ then in general $a^b\not\equiv a^{b^\prime}\mod{m}$
\subsection{Example}
\tab
$4\equiv1\mod{3}$\nextline
$2^4=16\equiv1\mod{3}$ however $2^1\equiv 2\mod{3}$ \nextline
Thus $4\equiv1\mod{3}$ however $2^4\not\equiv2^1\mod{3}$
\propn{Finite Integers}{FI}
$\forall a,b,\in\mathbb{Z} a\equiv b \mod{m}\iff a$ and $b$ have the same remainder after division by $m$
\subsection{Proof}
\tab
Applying the division algorithm, we get $a=qm+b$ and $b=q^\prime m+r^\prime$ where $0\leq r,r^\prime < m$.\nextline
Notice if $a\equiv b\mod{m}$, we get that $m\lvert(a-b)$ and thus $m\lvert(qm+r - q^\prime m - r^\prime)$\nextline
$\implies m\lvert(m(q+q^\prime) + (r-r^\prime))$ Then by DIC it follows that \nextline
$\iff m\lvert (r-r^\prime)$ \nextline
Now since $0\leq r < m$ and $0 \leq r^\prime < m$, we get that $-m\leq r-r^\prime < m$. Now, since $m\lvert(r-r^\prime)$, we get that by BBD, that $m\leq \abs{r-r^\prime}$ and so for both inequalities to hold, $r=r^\prime$.
\propn{Congruent if and only if same remainder}{CISR}
\tab
$a\equiv b\mod{m} \iff a$ and $b$ have the same remainder after division by $m$.
\propn{Congruent to Remainder}{CTR}
$\forall a,b\in\mathbb{Z}\,, 0\leq b \leq m-1 : a\equiv b\mod{m}\iff$ the remainder of $a$ after division by $m$ is $b$.
\nextline
Consequently, every $a\in\mathbb{Z}$ is congruent to a unique integer in $[0,m-1]\subseteq\mathbb{Z}$  mod $m$.
\subsection{Examples}
\begin{tabular}{r l l}
25 & $\equiv 32$ & $\mod{7}$ \\
& $\equiv 18$ & $\mod{7}$ \\
& $\equiv 11$ & $\mod{7}$ \\
& $\equiv \mathbf{4}$ & $\mod{7}$ \\
& $\equiv -3$ & $\mod{7}$ \\
& $\equiv -10$ & $\mod{7}$ \\
& $\cdots$ & $\mod{7}$
\end{tabular}
\nextline
4 is distinguished, as it is the remainder of 25 after division by 7.
\newline
\nextline
\textit{Find the remainder of $5^{10}$ after division by 7}
\nextline
We want to compute $5^{10}\mod{7}$. Since $5^{10} = \left(5^2\right)^5 = 25^5$. We know $25\equiv 4\mod{7}$ and by
\hyperref[sec:CP]{Congruent Powers}
we get $\equiv 4^5\mod{7}$
\nextline
$\equiv 4^3\cdot 4^2\mod{7}$ \nextline
$\equiv (63+1) \cdot (14+2) \mod{7}$ \nextline
$\equiv 1\cdot 2 \mod{7}$ \nextline
$\equiv 2\mod{7}$ \nextline
Therefore, the remainder of $5^{10}$ after division by 7 is 2.
\newline
\nextline
\textit{Find the remainder of} $77^{100}\cdot 999 - 6^{83} \mod{4}$
\nextline
We know that $77=80-3\equiv -3\mod{4}\equiv 1\mod{4}$ by \hyperref[sec:CP]{Congruent Powers}.
$999=1000-1\equiv -1\mod{4}\equiv 3\mod{4}$. Now, notice that $6^{83}=6^2\cdot6^{81}$ and since $6^2\equiv 0\mod{4}$ we get that $0\cdot6^{81}\equiv 0\mod{4}$ by \hyperref[sec:CP]{Congruent Powers}
\nextline
$77^{100}\cdot999 - 6^{83} \equiv 1^{100}\cdot3 - 0\mod{4} \equiv 3\mod{4}$
\hyperref[sec:ARC]{Congruence and Multiplication}
\subsection{Sum of Factorial Example}
\tab
\textit{What is the last decimal of the following expression?}
$$\sum_{n=1}^{100} n!$$
\nextline
Notice that the last digit of a number is the remainder mod 10. As a smaller example, notice that $7!\equiv 0\mod{10}$ because $7!$ contains a factor $2\cdot5=10$ and thus is a multiple of 10. Therefore, we know that if $k\geq 5$ we get that $k!\equiv 0\mod{10}$
\nextline
Going back to our original problem, we can notice that $\forall n\geq 5$ the sum of $n!\mod{10}$ will be zero, and thus we only have to compute $1!+2!+3!+4!=1+2+6+24=33\equiv 3\mod{10}$
\section{Divisibility Rules}
\subsection{Divisibility by 3}
\tab
$\forall a \in \mathbb{Z} 3\lvert a \iff 3\lvert$ the digit sum of $a$.
\nextline
$3\lvert 2046 \iff 3\lvert(2+0+4+6)$. $2+4+6=12$ and since $3\lvert12$ we know $3\lvert 2046$. \nextline
$3\nmid271 \iff 3\nmid(2+7+1)$ $2+7+1=10$ and since we know $3\nmid 10$ we know $3\nmid 271$.
\subsection{Proof of divisibility by 3}
\tab
If the digits of $a$ are $d_k, d_{k-1}, d_{k-2}, \cdots, d_1+d_0$ then $a=10^kd_k + 10^{k-1}d_{k-1}+10^{k-2}d_{k-2}, \cdots, 10d_1, d_0$. This is called the decimal expansion of $a$. Since $10\equiv 1\mod{3}$ we get that \newline$a\equiv d_k + d_{k-1} + d_{k-2} + \cdots + d_1 + d_0\mod{3}$
\subsection{Divisibility by 11}
\tab
$11\lvert a \iff 11\lvert$ the alternatign sum of the digits of $a$\nextline
$11\lvert108097\iff 11\lvert(1+8+9) - (0+0+7)$ and since $1+8+9-7=11$ and $11\lvert 11$ we get that $11\lvert 108097$\nextline
$11\nmid133 \iff 11\nmid(1+3-3)$, thus $11\nmid1\implies 11\nmid133$
\subsection{Proof of divisibility by 11}
\tab take
$a=10^kd_k + 10^{k-1}d_{k-1}+10^{k-2}d_{k-2}, \cdots, 10d_1+ d_0$ to be the decimal representation of $a$. Since $10\equiv-1\mod{11}\equiv 10\mod{11}$  thus
$a\equiv (-1)^kd_k + (-1)^{k-1}d_k-1 + \cdots + (-1)^1d_1 + 1d_0$. Now, notice that this is the sum of the digits of $a$ indexed by even values of $k$ subtracted by the odd-indexed digits of $a$ mod 11.
\section{Linear Congruence Relations}
\subsection{Problem: does $x^3+x^2-x+1=0$ have an integer solution?}
\tab
Suppse $x=a\in\mathbb{Z}$ is a \soln. Thus, $a^3+a^2-a+1=0$. Thus, since both sides are integers, we know that $\forall m \in \mathbb{N}$, $a^3+a^2-a+1\equiv 0 \mod{m}$.
\nextline
Consider the equation in modulo 3. Notice now that $a$ can be either 0, 1, or 2 (mod 3). If $a\equiv0\mod{3}$ we get that $1\equiv0\mod{3}$ which is false. If $a\equiv1\mod{3}$ we get that $1+1=2\equiv0\mod{3}$ which is still false. If $a\equiv2\mod{3}$ then $a\equiv-1\mod{3}$ and thus we get $-1+1-(-1)+1=2\equiv0\mod{3}$ which is still false. Therefore, since there are no integer solutions moldulo 3, there are no integer solutions to the equation $a^3+a^2-a+1=0$.
\chapter{Linear Congruence}
\tab $y^2=4x+2$ can be solved over the integers by solving the relation mod m. If there are no solutions for a particular $m$, then there are no solutions over the integers. $y^2\equiv4x+2\mod{m}$. There's a finite process to check $y^2 \equiv 4x+2\mod{m}$ compared to the infinite process to check $y^2=4x+2$.
\nextline
\defn A Linear Congruence Equation is an equation of the form $ax\equiv c\mod{m}$ where $a,c\in\mathbb{Z}$ and $m\in\mathbb{N}$ are fixed and $a\not\equiv 0\mod{m}$. We wish to find \soln for $x$ over the integers.
\section{Methods of Solving}
\subsection{Brute Force}
$5x\equiv 2\mod{3}$
\begin{center}
\begin{tabular}{|c|c|c|c|}
$x\mod{3}$ & 0 & 1 & 2 \\
\hline
$5x$ & 0 & 5 & 10 \\
\hline
$5x\mod{3}$ & 0 & 2 & 1 \\
\end{tabular}
\end{center}
\tab
We see that the only solution to this is when $x\equiv 1\mod{3}$.
\subsection{LDE}
$5x\equiv 2\mod{3} \iff 5x=2+3k$ $\fs k\in\mathbb{Z}$. \nextline
$\iff 5x-3k=2$ \nextline
$\iff 5x+3y=2$ for $y=-k$ \nextline
Let $d=\gcd(5,3)$, then notice that $d\lvert 2$ thus by \hyperref[sec:ldeta]{LDET1} there are infinite solutions.
Solve for $x,y$.
\nextline
By inspection we see that a particular \soln is $(x_0,y_0) = (1,-1)$. Using \hyperref[sec:ldetb]{LDET2} we can get the general solutions given by
$$\begin{cases}
x=1 + 3n \\
y=-1 - 5n
\end{cases}$$
\section{Examples}
$2x\equiv3\mod{4}$
\nextline
By Method 1:
\begin{center}
\begin{tabular}{|c|c|c|c|c|}
$x\mod{4}$ & 0 & 1 & 2 & 3 \\
\hline
$2x$ & 0 & 2 & 4 & 6 \\
\hline
$2x\mod{4}$ & 0 & 2 & 0 & 2 \\
\end{tabular}
\end{center}
\tab
Since there is no value of $3$ in the table, there is no \soln to $2x\equiv 3\mod{4}$
\nextline
By Method 2:
\nextline
$2x\equiv3\mod{4} \iff 2x+4y=3$. Since $\gcd(2,4)\nmid3$, by \hyperref[sec:ldeta]{LDET1} there is no \soln to the equation.
\thm{Linear Congruence Theorem}{LC}
\tab
Consider the Linear Congruence Equation $ax\equiv c\mod{m}$ where $a,c\in\mathbb{Z}$ and $m\in\mathbb{N}$ are fixed and $a\not\equiv 0\mod{m}$. Let $d=\gcd(a,m)$. Then, solutions exist if and only if $d\lvert c$ by \hyperref[sec:ldeta]{LDET1}. If $d\lvert c$ and if $x_0$ is a \soln then the general solution set is given by the following (using \hyperref[sec:ldetb]{LDET2})
$$\left\{
x\in\mathbb{Z}\mid x=x_0+\frac{m}{d}n\, \fs n\in\mathbb{Z}
\right\}$$
$$\left\{
x\in\mathbb{Z}\mid x\equiv x_0\mod{\frac{m}{d}}
\right\}$$
\tab
Notice for the second set, there are $d$ \soln mod $m$
\subsection{Proof}
$ax\equiv c\mod{m}\iff ax+my=c$ so \soln exist $\iff d\lvert c$ by \hyperref[sec:ldeta]{LDET1}.\nextline
If $x_0$ is a \soln for x, then by \hyperref[sec:ldetb]{LDET2} the general \soln set is
$$\left\{
x\in\mathbb{Z} \mid x=x_0+\frac{m}{d}n\,\fs n\in\mathbb{Z}
\right\}$$
\tab
Since $x=x_0+\frac{m}{d}n \iff x\equiv x_0\mod{\frac{m}{d}}$
\nextline
Finally, if $x=x_0+\frac{m}{d}n$ then by applying the Division Algorithm, we get $n=qd+r$, $0\leq r < d$. Thus $x=x_0+\frac{m}{d}n \iff x=x_0+(qd+r)\frac{m}{d} = x_0 + mq + r\frac{m}{d}$\nextline
$\iff x\equiv x_0+r\frac{m}{d}\mod{m}$, $0\leq r \leq d-1$
\newline
\null\hfill$\mathcal{QED}$
\subsection{More Examples}
Solve $12x\equiv 9\mod{15}$\nextline
\soln: Step1: (gcd check)
$d=\gcd(12,15)=3$ and $3\lvert9$ thus solutions exist\newline
Step2 (Particular \soln)\nextline
$12x+15y=9$. By EEA we get $(x_0, y_0) = (-3,3)$.\nextline
$x\equiv x_0\mod{\frac{m}{d}}$.\nextline
$\equiv -3\mod{5}$\nextline
$\equiv 2\mod{5}$\nextline
$x=2+5k$\nextline
Since the only unique solutions are in the integer range $[0,14]$, we know that the only solutions to $x=2+5k$ in that interval are $x\equiv2,7,12\mod{15}$.
\section{Nonlinear Congruence Equations}
\tab
There is no general/efficient method of finding solutions.\nextline
\subsection{Examples}
$x^2\equiv 1\mod{2}$.
\begin{center}
\begin{tabular}{|c|c|c|}
$x$ & 0 & 1 \\
\hline
$x^2$ & 0 & 1
\end{tabular}
\end{center}
\tab
Thus $x\equiv 1\mod{2}$ is the only \soln
\nextline
$x^2\equiv 1\mod{4}$
\begin{center}
\begin{tabular}{|c|c|c|c|c|}
$x$ & 0 & 1 & 2 & 3\\
\hline
$x^2$ & 0 & 1 & 4 & 9 \\
\hline
$x^2\mod{4}$ & 0 & 1 & 0 & 1
\end{tabular}
\end{center}
\tab
Therefore, the solutions are $x\equiv 1,3\mod{4}$.
\nextline
Solving $x^2\equiv 1\mod{8}$ gives 4 solutions $\left(x\equiv1,3,5,7\mod{8}\right)$. Solving $x^2\equiv 1\mod{2^k}\, \fs k\geq3$ will have only 4 solutions.
\end{document}


\subsection{Total variation}\label{subsec:total_variation}

\begin{definition}\label{def:riemann_stieltjes_integral}
  The most common generalization of the \hyperref[def:riemann_integral]{Riemann integral} is the Riemann-Stieltjes integral. It is not as well-behaved, hence we will give the most general definition and not attempt to prove equivalences.

  Let \( \mscrX \) be a real \hyperref[def:separation_axioms/T2]{Hausdorff} \hyperref[def:topological_vector_space]{topological vector space}. Fix two \hyperref[def:function]{functions} \( f, \alpha: [a, b] \to X \).

  The \term{Riemann sum} of \( f \) with respect to \( \alpha \) corresponding to the \hyperref[def:riemann_partition/tagged]{tagged partition} \eqref{eq:def:riemann_partition/tagged} is defined as
  \begin{equation*}
    S(f, \alpha, \Delta, \Xi) \coloneqq \sum_{k=1}^n f(\xi_k) (\alpha(x_k) - \alpha(x_{k-1})).
  \end{equation*}

  The limit of the net
  \begin{equation}\label{eq:def:riemann_stieltjes_integral/net}
    \{ S(f, \alpha, \Delta, \Xi) \}_{(\Delta, \Xi) \in \op{tpart}([a, b])},
  \end{equation}
  if it exists, is called the \term{Riemann-Stieltjes integral} of \( f \) with respect to \( \alpha \) and is denoted by
  \begin{equation*}
    \int_a^b f(x) d \alpha(x).
  \end{equation*}
\end{definition}


\section{Linear hypothesis using linear regression}

\subsection{Data}
\begin{figure}[!ht]
  \includegraphics[width=\textwidth,height=0.4\textheight,keepaspectratio]{scatter_plot.png}
  \caption{Scatter plot of data}
  \label{fig:scatter_plot}
\end{figure}

\subsection{Implementation}
A linear function given by equation \ref{eq:simple_linear_model} was selected as the hypothesis.
Linear regression with learning learning rate($\alpha$) = 0.01 and gradient descent approach was used for optimizing the parameters.
The training stopped after reaching the maximum allowed number of iterations which was 10000.
For this specific dataset we can use a threshold of 0.5 to classify the data. The predicting data
can be assigned a class 1 (fail) if $h_{\theta}$(x) $<$ 0.5 and class 2 (pass) otherwise.

\begin{equation}
\label{eq:simple_linear_model}
Y = \theta_0 + \theta_1 * X
\end{equation}

\subsection{Observation}
The final values of $\theta$s for a run are as follows:

% Theta 0 final value
\begin{equation}
\theta_0 = -0.15184881886181856
\end{equation}

% Theta 1 final value
\begin{equation}
\theta_1 = 0.23405702659773173
\end{equation}

\begin{figure}[!ht]
  \includegraphics[width=\textwidth,height=0.4\textheight,keepaspectratio]{regression_line_0_01.png}
  \caption{Linear regression line}
  \label{fig:regression_line}
\end{figure}

The regression line corresponding to the above $\theta$s is shown in figure \ref{fig:regression_line}

\subsection{Source Code}
\lstinputlisting[language=python]{task_1.py}


% !TeX root = ./apxthy.tex

\section{Least Squares Methods}

\subsection{Motivation}
%
We first describe least squares methods in abstract terms. Let
$[a,b]$ be an interval and
$b_1, \dots, b_N \in C([a,b])$ be $N$ linearly independent basis functions
for an approximation space
\[
    \AA_N   := {\rm span}\b\{ b_1, \dots, b_N \b\}.
\]
Given $w \in C(a,b) \cap L^1(a,b)$ (note the open interval!) we can
define a weighted $L^2$-inner product
\[
    \< f, g \>_{L^2_w} := \int_a^b w(x) f(x) g(x)^* \,dx,
\]
with associated norm $\|f\|_{L^2_w} := \<f,f\>_{L^2_w}^{1/2}$. The best
approximation of a function $f \in C([a,b])$ with respect to this weighted norm
is then given by
\begin{equation} \label{eq:lsq:ctslsq}
    p_N \in \min_{p \in \AA_N} \b\| f - p \b\|_{L^2_w}^2.
\end{equation}
We call this a continuous least squares problem.

Computationally, we typically need to discretise \eqref{eq:lsq:ctslsq}.
To that end, we choose points $x_1, \dots, x_M \in [a, b]$ and weights
$w_1, \dots w_M$ and define the discrete inner product
\begin{equation} \label{eq:lsq:disip}
    \< f, g \>_{\ell^2_w} := \sum_{m = 1}^M w_m f(x_m) g(x_m)^*
\end{equation}
with associated norm $\|f\|_{\ell^2_w} := \<f,f \>_{\ell^2_w}^{1/2}$. This
gives the discrete least squares problem
\begin{equation}
    \label{eq:lsq:dislsq}
    p_N \in \min_{p \in \AA_N} \b\| f - p \b\|_{\ell_w^2}.
\end{equation}
This is the typical kind of least squares problem encountered in
real applications.

We distinguish two scenarios:
\begin{enumerate}
\item {\bf User Chooses Data: } In this scenario the ``user'' is given a
function $f$ to be approximated. She may then choose the points $x_m$, weights
$w_m$ and evaluations $f(x_m)$ in order to fine-tune and optimise the fit $p_N$.
For example it is then feasible to start from \eqref{eq:lsq:ctslsq} and design a
discrete LSQ system \eqref{eq:lsq:dislsq} that approximates
\eqref{eq:lsq:ctslsq} in a suitable sense. An arbtirary amount of data $(x_m,
f(x_m))$ may then be generated to ensure a good fit.

\item {\bf Data is provided: } Some data has been collected outside the control
of the person (``user'') designing the fit. Given the data points $(x_m, f(x_m))$
(possibly subject to noise, i.e. $y_m = f(x_m) + \eta_m$ is then provided)
one then needs to choose an appropriate approximation space $\AA_N$,
approximation degree $N$ and weights $w_m$ to ensure a good fit in a sense
dictated by the application.
\end{enumerate}

We will study both scenarios but note that the second one is the more
typical in applications.

\subsection{Solution methods}
%
\label{sec:lsq:soln}
%
We convert \eqref{eq:lsq:dislsq} into a linear algebra problem.
By writing
\[
    Y_m := f(x_m),  \sqrt{W} := {\rm diag}(\sqrt{w_1}, \dots, \sqrt{w_m}) \in \R^{M \times M}
\]
and
\[
    p(x_m) = \sum_{n = 1}^N c_n b_n(x_m) = A c,
    \quad \text{where }  A_{mn} = b_n(x_m),
\]
then $A \in \R^{M \times N}$ and we obtain
\[
    \sum_{m = 1}^M w_m | p(w_m) - f(x_m)|^2
    = \big\| \sqrt{W} A c - \sqrt{W} Y \big\|^2,
\]
where $\|\cdot\|$ denotes the standard Euclidean norm in $\R^M$.
We write $\tilde{A} := \sqrt{W} A, \tilde{Y} := \sqrt{W} Y$ and
write the least squares functional equivalently as
\[
    \Phi(c) := \big\| \sqrt{W} A c - \sqrt{W} Y \big\|^2
    = c^T \tilde{A}^T \tilde{A} c - 2 c^T \tilde{A}^T \tilde{Y} + \|\tilde{Y}\|^2.
\]
A minimiser must satisfy $\nabla\Phi(c) = 0$, which gives the linear system
\begin{equation} \label{eq:lsq:normaleqns}
    \tilde{A}^T\tilde{A} c = \tilde{A}^T \tilde{Y}.
\end{equation}
These are called the normal equations, which can be solved using
the LU or Cholesky factorisation.

It turns out that they are often (though not always) ill-conditioned.
An alternative approach is therefore to perform the (numerically stable)
{\em thin QR factorisation}
\[
    \tilde{A} = Q R,
\]
where $R \in \R^{N \times N}$ is upper triangular and  $Q \in \R^{M \times N}$
has ortho-normal columns, i.e., $Q^T Q = I \in \R^{N \times N}$.
With the QR factorisation in hand the normal equations can be rewritten as
\begin{align*}
    \tilde{A}^T\tilde{A} c &= \tilde{A}^T \tilde{Y} \\
    \Leftrightarrow \qquad
    R^T Q^T Q R c &= R^T Q^T \tilde{Y} \\
    \Leftrightarrow \qquad
    R c &= Q^T \tilde{Y},
\end{align*}
provided that $R$ is invertible (which is equivalent to $A^T A$ being invertible
and to $A$ having full rank). Thus, the solution of the least squares problem
becomes
\begin{equation}
    \label{eq:lsq:qr}
    R c = Q^T \sqrt{W} Y, \qquad \text{where} \qquad
    \sqrt{W} A = QR.
\end{equation}

It is worthwhile comparing the computational cost of the two approaches.
\begin{enumerate}
\item The assembly of thenormal equations requires the multiplication
$A^T A$ which requires $O(M N^2)$ operations, followed by  the
Cholesky factorisation of $A^T A$ which requires $O(N^3)$ operiations.
Thus the cost of solving \eqref{eq:lsq:normaleqns} is $O(M N^2)$.
\item The cost of the QR factorisation in \eqref{eq:lsq:qr} is
$O(M N^2)$ as well, while the inversion of $R$ is only $O(N^2)$ and the
multiplication with $Q^T$ is $O(NM)$.

Thus both algorithms scale like $O(M N^2)$.
\end{enumerate}

\subsection{Orthogonal Polynomials}
%
\label{sec:lsq:orthpolys}
%
We have so far encountered orthogonal polynomials in the context of the
Chebyshev basis, which arise naturally due to their connection to trigonometric
polynomials. More generally, we can consider orthogonal polynomials with respect
to {\em any} inner product $\<\cdot, \cdot\>$. For simplicity we will continue
to work on the domain $[-1, 1]$. In the context of least squares problems, we
can think of \eqref{eq:lsq:ctslsq} or \eqref{eq:lsq:dislsq} and the inner
continuous or discrete products associated with these least squares problems.

The main result we want to discuss here is that the three-point recursion
\eqref{eq:poly:chebrecursion} for the Chebyshev basis is not special, but that
all families of orthogonal polynomials satisfy such a recursion. That is,
given an inner product $\< \cdot, \cdot\>$ we will construct sequences of
coefficients, $A_k, B_k, C_k$ such that the sequence of polynomials given by
%
\begin{equation} \label{eq:lsq:general_3ptrec}
    \phi_{k+1}(x) := (x - B_k) \phi_k(x) - C_k \phi_{k-1}(x)
\end{equation}
%
are orthogonal. By construction, we immediately see that the leading term in
$\phi_{k}$ is $x^k$; hence they also span the space of all polynomials.

Taking the inner product of \eqref{eq:lsq:general_3ptrec} with $\phi_k$ and then
$\phi_{k-1}$ we obtain
%
\begin{align*}
    0 &= \< x \phi_k, \phi_k \> -  B_k \< \phi_k, \phi_k \>, \\
    0 &= \< \phi_k, x \phi_{k-1} \> - C_k \< \phi_{k-1}, \phi_{k-1} \>,
\end{align*}
%
which gives expressions for $B_k, C_k$,
%
\begin{equation} \label{eq:lsq:coeffs_3pt}
    \begin{split}
        B_k &:= \frac{\< x \phi_k, \phi_k \>}{\< \phi_k, \phi_k \>}, \\
        C_k &:= \frac{\< \phi_k, x \phi_{k-1} \>}{\< \phi_{k-1}, \phi_{k-1} \>}.
    \end{split}
\end{equation}
%
It is worth noting that this construction is simply the Gram-Schmidt procedure,
but truncated at a three-term recursion rather than the full recursion to
$\phi_0$. In particular, by construction, we have that $\phi_{k+1} \perp \phi_k,
\phi_{k-1}$ and it only remains to show that they it is also orthogonal to
$\phi_0, \dots, \phi_{k-2}$. Concretely we obtain the following result.

\begin{proposition}
    Suppose that $\< \cdot, \cdot\>$ is an inner product on  the space of
    polynomials such that the operator $p \mapsto x \cdot p$ is self-adjoint,
    (i.e., $\<x p, q\> = \< p, xq\>$ for all polynomials $p, q$). Suppose,
    moreover, that $\phi_0 = 1, \phi_1 = x - \<1, x\>$, and that $\phi_k, k \geq 2,$
    is given by the three-point recursion \eqref{eq:lsq:general_3ptrec} with
    coefficients \eqref{eq:lsq:coeffs_3pt}. Then $\{ \phi_k : k \in \N \}$ is
    a basis of the space of polynomials which is orthogonal with respect to
    $\< \cdot, \cdot \>$.
\end{proposition}
\begin{proof}
    By construction we have that $\phi_1 \perp \phi_0$ and that $\phi_{k+1}
    \perp \phi_k, \phi_{k-1}$ for $k \geq 2$. Thus, it only remains to prove
    that, for $k \geq 2$, $\phi_{k+1} \perp \phi_j$ for $j = 0, \dots, k-2$.

    By induction we may assume that $\< \phi_j, \phi_i \> = 0$ for $i \neq j$
    and $i \leq k$. Then, we have
    \begin{align*}
        \< \phi_{k+1}, \phi_j\>
        &=
        \< x \phi_k, \phi_j \> - B_k \< \phi_k, \phi_j\> - C_k \< \phi_{k-1}, \phi_j\> \\
        &=  \< \phi_k, x \phi_j \>,
    \end{align*}
    where we also used self-adjointness of multiplication by $x$. Since
    the degree of $x \phi_j$ is at most $k-1$ and, again by induction, $\phi_k$
    is orthogonal to $\phi_0, \dots, \phi_{k-1}$ it follows that
    $\< \phi_k, x \phi_j \> = 0$. This completes the proof.
\end{proof}

\begin{exercise}
    Derive a recursion for an {\em orthonormal} basis of the form
    \begin{align*}
        A_0 \phi_0 &= 1, \\
        A_1 \phi_1 &= x - B_1, \\
        A_k \phi_k &= (x - B_k) \phi_{k-1} - C_k \phi_{k-2}.
    \end{align*}
    Make sure to prove that all $A_k$ are non-zero.
\end{exercise}

\begin{exercise}
    Consider the inner product
    \[
        \< p, q\> =  \int_{-1}^1 pq + p'q' \, dx;
    \]
    prove that the multiplication operator $p \mapsto xp$ is not self-adjoint.

    If we were to construct a sequence of orthogonal polynomials by  the
    Gram-Schmidt procedure, would we again obtain a three-term recursion?

    {\it Hint: The involved calculations are somewhat boring. You may wish to use
    a computer algebra system to explore this question.}
\end{exercise}

\begin{remark}
    A discrete inner product of the form \eqref{eq:lsq:disip} is not strictly
    an inner product on the space of all polynomials, but depending on the
    summation points it may be an inner product on a subspace $\mathcal{P}_N$.
    In this case the recursion formula can simply be terminated at degree
    $k = N$ to obtain an orthogonal (or orthonormal) basis of $\mathcal{P}_N$.
\end{remark}

\subsection{Accuracy and Stability I: Least Squares and Nodal Interpolation}
%
Consider fitting trigonometric polynomials $\AA_{2N} = \TT_N'$ with equispaced
grid points $x_n = \pi n / N$ and uniform weights $w_n = 1$. Then the
least squares fit
\[
    \min \sum_{n = 0}^{2N-1} |f(x_n) - t(x_n)|^2
\]
is equivalent to trigonometric interpolation, for which we have sharp and error
estimates that predict a close to optimal rate of approximation.

We could leave it at this, but it is still interesting to observe what happens
to the least squares system. The matrix $A$ is now given by
\[
    A_{nk} = e^{ikx_n}
\]
and the entries in the normal equation by
\[
    [A^* A]_{kk'} = \sum_{n} e^{-ikx_n} e^{ik' x_n} = 2 N \delta_{kk'}
\]
according to Exercise~\ref{exr:trig:trapezoidal rule}(i).
This is due to the fact that the discrete inner product
\eqref{eq:lsq:disip}  (up to a constant factor) identical to
the $L^2(\TT)$-inner product on the space $\TT_N'$, that is,
\[
    \< f, g \>_{\ell^2} = 2N \mint_{-\pi}^\pi f g^* \,dx \qquad
    \forall f, g \in \TT_N'.
\]
No QR factorisation is needed and the lsq fit is given by
\[
    c = (2 N)^{-1} A^* Y,
\]
where the operation $(2N)^{-1} A^T Y$ can be performed at $O(N \log N)$
computational cost using the FFT.

Analogous observations are of course true for connecting least squares
methods and algebraic polynomials.

\subsection{Accuracy and Stability II: Random data}
%
\label{sec:lsq:rand}
%
The situation gets more interesting when we are not allowed to optimise the
points at which to fit the approximant. There is an infinite variety of
different situations that can occur when the provided data is application
driven, which goes far beyond the scope of this module. Here, we will assume
that the points $x_m$ are distributed according to some probability law. That
is, they are random. This is in fact a rather common situation in applications
as well. Note also that we are now in the Case-2 situtation where we are given a
fixed amount of data $(x_m, f(x_m))_{m = 1}^M$ and should choose $N$ ensure the
best possible fit given the data we have. In particular this means that we
should not choose $N$ too large!

Specifically, we shall assume throughout this section that
\begin{equation}
    \label{eq:lsw:wm_law}
    x_m \sim w \,dx, \qquad \text{are iid}, \quad \text{for } m = 1, \dots, M.
\end{equation}
(identically and independently distributed) and without loss of generality that
$\int_a^b w \,dx = 1$ as this can always be achieved by rescaling. We also
assume that $w \in C(a,b) \cap L^1(a,b)$ as before. In this case, we can
construct a sequence of orthogonal polynomials as in \S~\ref{sec:lsq:orthpolys}
with respect to the $L^2_w$-inner product and we will target best approximation
with respect to the same inner product.

We will discuss two fundamental results due to Cohen, Davenport and Leviatan
\cite{Cohen2013-yj}, but we won't prove them.  The first result concerns the
{\em stability} of the normal equations. Specifically, we will show that if we
use an $L^2_w$-orthogonal basis then $A^T A$ will be close to identity (and in
particular invertible) for a sufficiently large number of sample points $x_m$.

\begin{theorem}[Stability] \label{th:lsq:randstab}
    Let $\phi_1, \dots, \phi_N$ be $L^2_w$-orthonormal, $A_{mk} := \phi_k(x_m)$,
    then
    \[
        \mathbb{P}\B[ \| A^* A - I \|_{\rm op} > \delta\B]
        \leq 2 N \exp\B( - \smfrac{C_\delta M}{K(N)}\B),
    \]
    where $C_\delta = (1+\delta) \log (1+\delta) - \delta$ and
    \[
        K(N) = \sup_{x \in [a,b]} \sum_{k = 1}^N |\phi_k(x)|^2.
    \]
\end{theorem}

Let us specifically focus on the Chebyshev measure where $w(x) =
(1-x^2)^{-1/2}$, the Chebyshev basis $T_k(x)$ on the interval $[a,b]= [-1,1]$.
Since $|T_k(x)| \leq 1$ it readily follows that $K(N) \leq N$. Moreover, the
recursion formulat for $T_k$ implies that $T_k(1) = 1$ for all $k$, hence this
bound is sharp, i.e., $K(N) = N$ in this case.

To make $N \exp( - \smfrac{C_\delta M}{N})$ small, we therefore need
to choose $N \leq \alpha M / \log M$. With this choice,
\[
    N \exp( - \smfrac{C_\delta M}{N})
    =
    N \exp\b( - \alpha C_\delta \log M\b)
    \leq
    M^{1 -\alpha C_\delta} / \log M
\]
and by choosing $\alpha$ sufficiently large we can ensure that this value tends
to zero as $M \to \infty$. (the case of sufficiently large amounts of data).
Conversely, if $\alpha$ is too small, then $M^{1 -\alpha C_\delta} / \log M \to
\infty$ as $M \to \infty$ which shows that the choice $N \leq \alpha M/\log M$
is sharp. This is a very mild restriction!

The next result  we discuss concerns the approximation $p_{NM} = \sum_n c_n
\phi_n$ we obtain by solving the least squares problem $\| f - p_{NM}
\|_{\ell^2(\{x_m\})}^2 \to \min$.

\begin{theorem} \label{th:lsq:randerr}
    There exists a constant $c$ such that, if
    \[
        K(N) \leq \frac{c}{1+r} \frac{M}{\log M},
    \]
    then,
    \[
        \mathbb{E}\b[ \|f - p_{NM}\|_{L^2_w}^2 \b]
        \leq
        (1+\epsilon(M)) \|f - \Pi_N f\|_{L^2_w}^2
        + 8 \|f\|_{L^\infty(a,b)}^2 M^{-r},
    \]
    where $\epsilon(M) \to 0$ as $M \to \infty$, and $\Pi_N$ denotes the
    best-approximation operator with respect to the $L^2_w$-norm.
\end{theorem}

As a first comment, we observe that our restriction $N \leq \alpha M / \log M$
for sufficiently small $\alpha$ re-appears. (In fact, this prerequisite is
required to be able to apply Theorem~\ref{th:lsq:randstab}.)

We can now ask what consequence the choice $N = \alpha M / \log M$ has
on the error. In this case, $\alpha = c / (1+r)$, or equivalently,
$r = c/\alpha - 1$ hence (sacrifycing just a log-factor)
\[
    M^{-r}  \leq N^{-r} = N^{1 - c/\alpha} =: N^{-\alpha'},
\]
where $\alpha' > 0$ provided that $\alpha$ is chosen sufficiently small. In this
case, we can conclude that
\[
    \mathbb{E}\b[ \|f - p_{NM}\|_{L^2_w}^2 \b]
    \lesssim
    \|f - \Pi_N f\|_{L^2_w}^2
    +
    N^{- \alpha'}
\]
for some $\alpha' > 0$. Thus, for differentiable functions $f$, such a choice is
quasi-optimal.

However, for analytic functions the rate in the error estimate is
reduced. Let us assume that $f \in A(E_\rho)$, then
$\| f - \Pi_N f \|_{L^2_w} \lesssim \rho^{-N}$ hence we must balance the
two constributions
\[
    \rho^{-N} + M^{-r}.
\]
We have already seen that $N = \alpha M/\log M$ leads to $\rho^{-N} \ll M^{-r}$,
hence we instead attempt to choose $N = a (M/\log M)^{\alpha}$ for some $0 < \alpha < 1$,
which gives
\[
    r = c' (M / \log M)^{1-\alpha}.
\]
Thus, we wish to balance
\begin{align*}
    \exp\B[ - (\log \rho) N\B]  &+ \exp\B[ - r \log M \B]  \\
    = \exp\B[ - c'' M^\alpha (\log M)^{-\alpha} \B]  &+ \exp\B[ - c M^{1-\alpha} (\log M)^{-\alpha} \B]
\end{align*}
We can now see that the two terms are balanced when $\alpha = 1/2$, that is,
the quasi-optimal choice of $N$ appears to be
\[
    N = a \b(M / \log M\b)^{1/2}.
\]
This is also somewhat consistent with the observation that in the $C^j$ case we
need to decrease $\alpha$ for increasing $j$.

That said, we should remember that we have balanced an error estimate and not
the actual error. At this point, it is highly advisable to test these
predictions numerically, which is done in \nblsq, where we see --- for some
limited examples --- that the stability condition $N \lesssim M/\log M$ appears
to be crucial but the stronger requirement $N \lesssim (M/\log M)^{1/2}$ seems
to not be required.

In summary, the foregoing analysis is intended to demonstrate how different
competing contributions to approximation by fitting from data can be balanced at
least in principle but also the limitations of analysis.

\subsection{Exercises}
%
\label{sec:lsq:exercises}
%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\chapter{Regression}
\label{chap:regression}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Linear Regression (OLS)}
\label{regression:linear}

Linear regression fits the best hyperplane, or line in 1D,
to a collection of $m$ points $\vb{x}_{i}, y_{i}$,
typically via the method of least squares.
If $\vb{x}$ has $n$ features we can represent the
linear relationship between $\vb{x}$ and $y$ as:

\begin{equation}\label{eq:linear:one_point}
y_{i} = \beta_{0} + \sum_{j=1}^{n}\, \beta_{j} x_{ij} + \epsilon_{i}\,,
\end{equation}

\noindent where $\beta_{j}$ are the parameters of the regression
and $\epsilon$ represent random errors.
Transitioning to matrix notation\footnote{Note
that \textit{linear} regression refers to the linearity in the model parameters
$\vb*{\beta}$, not $\mathbf{X}$.
The components of $\mathbf{X}_{i}$ can be, and often are,
non-linear functions of other input features.}, this is simply:

\begin{equation}\label{eq:linear:matrix}
\vb{y} = \mathbf{X} \vb*{\beta} + \vb*{\epsilon}\,,
\end{equation}

\noindent where we have set $X_{i0} =1$.
Here $\mathbf{X}$ is $m \times n$,
$\vb*{\beta}$ is $n \times 1$,
and $\vb{y}$, $\vb*{\epsilon}$ are $m \times 1$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Derivation}
\label{regression:linear:derivation}

The ordinary least squares (OLS) estimate\footnote{When
the errors are assumed to be normal,
as in \cref{item:regression:linear:exogeneity,item:regression:linear:spherical,item:regression:linear:normality},
OLS is equivalent to the maximum likelihood estimation (MLE) approach of \cref{opt:MLE}.
See one derivation \href{https://machinelearningmastery.com/linear-regression-with-maximum-likelihood-estimation/}{here}.} of
the parameters $\hat{\vb*{\beta}}$
can be found by minimizing the squares of the errors,
\ie the objective function $S\left(\vb*{\beta}\right) = \norm{\vb*{\epsilon}}^{2}$:

\begin{subequations} \label{eq:linear:ols}
\begin{align}
\hat{\vb*{\beta}} &= \argmin_{\vb*{\beta}} S\left(\vb*{\beta}\right)\,, \label{eq:linear:argmin} \\
S\left(\vb*{\beta}\right)
&= \norm{\vb{y} - \mathbf{X} \vb*{\beta}}^{2} = \left(\vb{y} - \mathbf{X} \vb*{\beta}\right)\transpose \left(\vb{y} - \mathbf{X} \vb*{\beta}\right) \label{eq:linear:S_matrix} \\
&= \sum_{i=1}^{m} \, \abs{y_{i} - \sum_{j=0}^{n} \, \beta_{j} x_{ij}}^{2}\,. \label{eq:linear:S_components}
\end{align}
\end{subequations}

We then find the minimum of $S\left(\vb*{\beta}\right)$ with respect to $\vb*{\beta}$
by taking the gradient and setting it equal to zero:

\begin{subequations} \label{eq:linear:ols_derivation}
\begin{align}
S\left(\vb*{\beta}\right) &=
 \vb{y}\transpose \vb{y}
-\vb*{\beta}\transpose \mathbf{X}\transpose \vb{y}
-\vb{y}\transpose \mathbf{X} \vb*{\beta}
+\vb*{\beta}\transpose \mathbf{X}\transpose \mathbf{X} \vb*{\beta} \label{eq:linear:S_expand} \\
&= \vb{y}\transpose \vb{y}
-2\, \vb*{\beta}\transpose \mathbf{X}\transpose \vb{y}
+\vb*{\beta}\transpose \mathbf{X}\transpose \mathbf{X} \vb*{\beta}\,, \label{eq:linear:S_expand_simplified} \\
\partial_{\vb*{\beta}} \, S\left(\vb*{\beta}\right) &=
0
-2\, \partial_{\vb*{\beta}} \, \vb*{\beta}\transpose \mathbf{X}\transpose \vb{y}
+\partial_{\vb*{\beta}} \, \vb*{\beta}\transpose \mathbf{X}\transpose \mathbf{X} \vb*{\beta} \label{eq:linear:grad_S_expand} \\
&= -2\, \mathbf{X}\transpose \vb{y} + 2\, \mathbf{X}\transpose \mathbf{X} \vb*{\beta} = 0. \, \implies \label{eq:linear:grad_S} \\
\mathbf{X}\transpose \vb{y} &= \mathbf{X}\transpose \mathbf{X} \vb*{\beta}\,, \label{eq:linear:penultimate}
\end{align}
\end{subequations}

\noindent where we have used\footnote{See \href{https://economictheoryblog.com/2015/02/19/ols_estimator/}{here}
and \href{https://economictheoryblog.com/2018/10/17/derivation-of-the-least-squares-estimator-for-beta-in-matrix-notation-proof-nr-1/}{here}
for the component-wise proof, but it is essentially the same as the 1D $\partial_{x} b x = b$, $\partial_{x} b x^{2} = 2 b x$.
Also note that \cref{eq:linear:S_expand_simplified} results from $\vb{y}\transpose \mathbf{X} \vb*{\beta}$ being scalar
and thus equal to its own transpose, $\vb*{\beta}\transpose \mathbf{X}\transpose \vb{y}$.}:

\begin{subequations} \label{eq:grad_relations}
\begin{align}
\partial_{\vb*{\beta}} \, \vb*{\beta}\transpose \mathbf{X}\transpose \vb{y} &= \mathbf{X}\transpose \vb{y}\,, \label{eq:grad_relations:1} \\
\partial_{\vb*{\beta}} \, \vb*{\beta}\transpose \mathbf{X}\transpose \mathbf{X} \vb*{\beta} &= 2 \, \mathbf{X} \transpose \mathbf{X} \vb*{\beta}\,. \label{eq:grad_relations:2}
\end{align}
\end{subequations}

Consequently, the optimal $\hat{\vb*{\beta}}$ of \cref{eq:linear:ols}
has a closed form solution\footnote{The fitted prediction is $\hat{\vb{y}} = \mathbf{X} \hat{\vb*{\beta}}$, with residuals $\hat{\vb*{\epsilon}} = \vb{y} - \hat{\vb{y}}$.}:

\begin{equation}\label{eq:linear:betahat_OLS}
\hat{\vb*{\beta}}^{\text{OLS}} = \left(\mathbf{X}\transpose\mathbf{X}\right)^{-1}\mathbf{X}\transpose \vb{y}\,,
\end{equation}

\noindent which is the best linear unbiased estimator (BLUE),
as can be shown with the Gauss-Markov Theorem provided
the following assumptions hold\footnote{For additional details see
\href{https://economictheoryblog.com/2015/04/01/ols_assumptions/}{here},
\href{http://people.duke.edu/~rnau/testing.htm}{here}, and
\href{https://economictheoryblog.com/2015/02/26/markov_theorem/}{here}.}:

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Assumptions}
\label{regression:linear:assumptions}

\begin{enumerate}[noitemsep]
  \item The underlying relationship between $\vb{x}$ and $y$ is linear, and there are no major outliers.\label{item:regression:linear:linear}
  \item The columns of $\mathbf{X}$, \ie features, are linearly independent, \ie $\mathrm{rank}\left(\mathbf{X}\right) = n$ (no multicollinearity). This allows $\mathbf{X}\transpose\mathbf{X}$ to be inverted.\label{item:regression:linear:multicollinearity}
  \item The errors $\epsilon$ have conditional mean 0, $\expvalE{\epsilon \mid \mathbf{X}} = 0$ (exogeneity). The errors thus:\label{item:regression:linear:exogeneity}
  \begin{enumerate}[noitemsep]
    \item Have a mean of zero, $\expvalE{\epsilon} = 0$.
    \item Are not correlated with the input features, $\expvalE{\mathbf{X}\transpose\epsilon} = 0$.
  \end{enumerate}
  \item The errors are spherical, $\mathrm{var}\left(\epsilon \mid \mathbf{X}\right) = \sigma^{2} \identity$. Thus:\label{item:regression:linear:spherical}
  \begin{enumerate}[noitemsep]
    \item Each observation $\vb{x}_{i}$ has the same constant variance $\sigma^{2}$ (homoscedasticity).
    \item The errors are uncorrelated between observations, $\expvalE{\epsilon_{i}\epsilon_{j \neq i} \mid \mathbf{X}} = 0$ (no autocorrelation).
  \end{enumerate}
  \item The errors are normally distributed (multivariate normality)\footnote{This is not required for OLS to be the BLUE, but hypothesis testing works if true.}.\label{item:regression:linear:normality}
\end{enumerate}

If these assumptions are violated the following issues arise,
namely the model may be biased and or have a large or invalid estimated variance:

\begin{itemize}[noitemsep]
  \item[\cref{item:regression:linear:linear}.] If you are fitting nonlinear data the predictions will have large errors,
particularly when extrapolated beyond the range of the fitted data.
This will show up as systematic errors in the residuals plot,
or may be obvious when comparing observed vs predicted values.
Possible fixes include applying a nonlinear transformation to some of the features to linearize the data, \eg take the log,
adding more combinations of features, \eg higher polynomial terms,
or finding new independent features which may explain the nonlinearity.

  \item[\cref{item:regression:linear:multicollinearity}.] If some of the features are not linearly independent (multicollinearity),
they can be biasing the model and should be removed in turn until linear independence is restored.
Multicollinearity can be spotted in the input feature correlation matrix,
with the variance inflation factor of \cref{regression:linear:VIF},
or if the residuals correlate to any of the features.

  \item[\cref{item:regression:linear:exogeneity}.] If something is wrong with the errors
such that they have a non-zero mean or correlate to the input features
the OLS $\hat{\vb*{\beta}}$ is biased and inconsistent.
This can happen if there are omitted variables or measurement errors.
Furthermore, if $\expvalE{\epsilon \mid \mathbf{X}} = c$ only the $\beta_{0}$ intercept is affected.

  \item[\cref{item:regression:linear:spherical}.] If something is wrong with the errors
such that they have a changing variance or correlate across observations\footnote{Thus the residuals correlate with row number, \ie autocorrelation.}
the reported confidence intervals on the model parameters may be over or underestimated.
The OLS $\hat{\vb*{\beta}}$ remains unbiased and has consistent coefficients, but will be biased for standard errors.

  \item[\cref{item:regression:linear:normality}.] If the errors are not normally distributed the confidence intervals are again suspect.
This can be diagnosed by comparing the errors to the normal distribution with a normal probability plot, or normal quantile plot,
or through a statistical method like the Anderson-Darling and Kolmogorov-Smirnov tests.
Note that violating normality in the errors is not as much of an issue compared to the other assumptions
since the fit will still give usable coefficients provided the assumed form of the model is correct.
Problems of this kind can arise from nonlinear data or influential outliers.
If the errors really are non-normal, a generalized linear model (GLM) could be employed to model them correctly.
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Goodness of Fit}
\label{regression:linear:goodness_of_fit}

\subsubsection{Coefficient of Determination ($R^{2}$)}
\label{regression:linear:goodness_of_fit:R2}

The coefficient of determination\footnote{For additional details see
\href{https://economictheoryblog.com/2014/11/05/the-coefficient-of-determination-latex-r2/}{here},
\href{https://economictheoryblog.com/2014/11/05/proof/}{here}, and
\href{http://people.duke.edu/~rnau/rsquared.htm}{here}.}, $R^{2}$,
measures how much of $\vb{y}$'s variance is explained by the fitted model $\hat{\vb{y}}$,
versus by random errors $\vb*{\epsilon}$ or other unknown sources,
as $\vb{y} = \hat{\vb{y}} + \vb*{\epsilon}$.
$R^{2} \approx 1$ is ideal and, depending on the particular definition,
it can take values outside the typical $0 \leq R^{2} \leq 1$ range.

\begin{equation}\label{eq:regression:linear:goodness_of_fit:R2}
R^{2} = \frac{
\sum_{i=1}^{m} \left(\yhat_{i} - \expval{\yhat}\right)^{2}
}{
\sum_{i=1}^{m} \left(y_{i} - \expval{y}\right)^{2}
} = 1 - \frac{
\sum_{i=1}^{m} \hat{\epsilon}_{i}^{\,2}
}{
\sum_{i=1}^{m} \left(y_{i} - \expval{y}\right)^{2}
} = \rho_{y,\yhat}^{2}
\end{equation}

The adjusted $R^{2}$, $R^{2}_{\text{adj}}$, accounts for the degrees of freedom in the
fitted sample, $m-1$, and in the model, $m-n-1$, and is thus a better metric.

\begin{equation}\label{eq:regression:linear:goodness_of_fit:R2_adj}
R^{2}_{\text{adj}} = 1 - \frac{
\left(\sum_{i=1}^{m} \hat{\epsilon}_{i}^{\,2}\right)/\left(m-n-1\right)
}{
\left(\sum_{i=1}^{m} \left(y_{i} - \expval{y}\right)^{2}\right)/\left(m-1\right)
}
= 1 - \left(1-R^{2}\right)\frac{m-1}{m-n-1}
\end{equation}

\subsubsection{Standard Error of the Regression and $\chi_{\nu}^{2}$}
\label{regression:linear:goodness_of_fit:reduced_chi2}

The standard error of the regression, $s$,
and reduced \chiSqstat, $\chi_{\nu}^{2}$,
are additional ways of quantifying goodness of fit.
$s$ is a measure of the typical residual, in the units of $y$.
$\chi_{\nu}^{2}$ is a convenient way of reporting this spread in a dimensionless manner,
particularly in more sophisticated regressions where each $y_{i}$ has an \apriori estimated uncertainty, $\sigma_{i}$.

\begin{subequations}\label{eq:regression:linear:goodness_of_fit:s_red_chi2}
\begin{align}
s^{2} &= \frac{\norm{\hat{\vb*{\epsilon}}}^{2}}{m-n} = \frac{1}{m-n} \sum_{i=1}^{m} \left(\yhat_{i} - y_{i}\right)^{2} \label{eq:regression:linear:goodness_of_fit:s} \\
\chi_{\nu}^{2} &= \frac{1}{m-n} \sum_{i=1}^{m} \frac{\left(\yhat_{i} - y_{i}\right)^{2}}{\sigma_{i}^{2}} \label{eq:regression:linear:goodness_of_fit:red_chi2}
\end{align}
\end{subequations}

$\chi_{\nu}^{2}$ can be interpreted as follows:

\begin{table}[H]
  \centering
  \begin{tabular}{c | p{8cm}}
$\chi_{\nu}^{2} \gg 1$ & The true model may be different than the fitted model. \\
$\chi_{\nu}^{2} > 1$ & Fit doesn't fully explain data (underfitting), or the \apriori $\sigma_{i}$ are underestimated. \\
$\chi_{\nu}^{2} \approx 1$ & Good fit! \\
$\chi_{\nu}^{2} < 1$ & Fit overexplains data (overfitting), or the \apriori $\sigma_{i}$ are overestimated. \\
$\chi_{\nu}^{2} \ll 1$ & Extreme overfitting or overestimated $\sigma_{i}$.
  \end{tabular}
%  \caption{}
  \label{table:red_chi2_interp}
\end{table}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Coefficient Significance}
\label{regression:linear:coeff_significance}
% TODO what is the significance of any particular \beta_{i} coefficient, via \ttest, \chiSqtest, \Ftest?? Should be able to get distribution of any \beta_{i} analytically for OLS, but can use bootstrap sampling for more complex models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Variance Inflation Factor (VIF)}
\label{regression:linear:VIF}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ridge Regression (L2, or Tikhonov)}
\label{regression:linear:ridge}

Adding ridge (L2, or Tikhonov) regularization to OLS is much more straight forward than LASSO,
so we shall discuss it first\footnote{See \href{https://stats.stackexchange.com/a/164546}{here} for an interesting geometric explanation.}.
We wish to minimize the square errors $\norm{\vb*{\epsilon}}^{2}$,
now subject to the condition that $\norm{\vb*{\beta}}^{2} < t$ for some $t \geq 0$.
For any $t \geq 0$ there is a $\lambda \geq 0$ such that minimizing the following
objective function $S\left(\vb*{\beta}\right)$ without conditions is
equivalent\footnote{This is sometimes termed soft-thresholding, but is really an application of Lagrange multipliers, as described in \cref{opt:lagrange_mult}.}:

\begin{equation} \label{eq:linear:ridge}
S\left(\vb*{\beta}\right) = \norm{\vb{y} - \mathbf{X} \vb*{\beta}}^{2} + \lambda \norm{\vb*{\beta}}^{2}\,.
\end{equation}

After taking the gradient, we have the same terms as \cref{eq:linear:grad_S},
plus a new term\footnote{Note the similarity to \cref{eq:grad_relations:2}.} \cref{eq:linear:ridge_derivation:new}.
Rearranging \cref{eq:linear:ridge_derivation:penultimate} we can move the new term in with the old,
and quickly arrive at a modified version of the OLS solution \cref{eq:linear:ridge_derivation:betahat}.

\begin{subequations} \label{eq:linear:ridge_derivation}
\begin{align}
\partial_{\vb*{\beta}} \, S\left(\vb*{\beta}\right)
&= -2\, \mathbf{X}\transpose \vb{y} + 2\, \mathbf{X}\transpose \mathbf{X} \vb*{\beta} \label{eq:linear:ridge_derivation:grad_S_repeat} \\
&+ 2\,\lambda \vb*{\beta} = 0 \,\, \implies \label{eq:linear:ridge_derivation:new} \\
\mathbf{X}\transpose \vb{y} &= \left(\mathbf{X}\transpose \mathbf{X} + \lambda\,\identity\right) \vb*{\beta} \label{eq:linear:ridge_derivation:penultimate} \\
\hat{\vb*{\beta}}^{\text{ridge}} &= \left(\mathbf{X}\transpose\mathbf{X} + \lambda\,\identity\right)^{-1}\mathbf{X}\transpose \vb{y} \label{eq:linear:ridge_derivation:betahat}
\end{align}
\end{subequations}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{LASSO Regression (L1)}
\label{regression:linear:lasso}

As promised, adding LASSO (L1) regularization to OLS is challenging
as the condition, $\norm{\vb*{\beta}} < t$ for some $t \geq 0$,
is not differentiable, see the corners of \cref{fig:ml:l1l2:l1}, and must be dealt with carefully.
In the special case that $\mathbf{X}$ is orthonormal, $\mathbf{X}\transpose \mathbf{X} = \identity$,
a closed form solution \cref{eq:linear:lasso:betahat:1} can be derived on a case-by-case basis
at the $\beta_{i}$ coordinate-level\footnote{See \href{https://stats.stackexchange.com/questions/17781/derivation-of-closed-form-lasso-solution}{here},
\href{https://en.wikipedia.org/wiki/Lasso_(statistics)\#Orthonormal_covariates}{here},
and \href{https://xavierbourretsicotte.github.io/lasso_derivation.html}{here}.}.
In the general case, more sophisticated methods can find numerical solutions.

\begin{subequations} \label{eq:linear:lasso:betahat_all}
\begin{align}
\hat{\vb*{\beta}}^{\text{LASSO}}_{j}
&= \hat{\vb*{\beta}}^{\text{OLS}}_{j} \max\left(0, \frac{\lambda}{\abs{\hat{\vb*{\beta}}^{\text{OLS}}_{j}}}\right) \label{eq:linear:lasso:betahat:1} \\
&= \sign{\hat{\vb*{\beta}}^{\text{OLS}}_{j}} \left(\abs{\hat{\vb*{\beta}}^{\text{OLS}}_{j}} - \lambda\right)^{+} \label{eq:linear:lasso:betahat:2} \\
\hat{\vb*{\beta}}^{\text{OLS}}
&= \left(\mathbf{X}\transpose\mathbf{X}\right)^{-1}\mathbf{X}\transpose \vb{y}
= \mathbf{X}\transpose \vb{y} \label{eq:lasso:modified_betahat_OLS}
\end{align}
\end{subequations}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{LASSO vs Ridge Regression}
\label{regression:linear:lasso_vs_ridge}

All the usual concepts from \cref{ml_general:reg} apply, but specifically for linear regression
both LASSO and ridge allow us to relax the multicollinearity condition.
Lasso tends to use one feature per group of correlated $\vb{x}_{j}$'s, setting the rest to $\approx 0$,
while ridge will keep all of the associated $\beta_{j}$'s at a similar magnitude.
As one may expect from LASSO shrinking $\beta_{j}$'s to zero,
it does a better job when only some of the $n$ input features infuence \yhat,
while ridge does better if most of the features have similar importances.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{OLS Example}
\label{regression:linear:example}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Bayesian Linear Regression}
\label{regression:bayesian_linear}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{LASSO and Ridge as Priors}
\label{regression:bayesian_linear:lasso_vs_ridge}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Weighted Least Squares}
\label{regression:WLS}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Generalized Least Squares (GLS)}
\label{regression:GLS}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Generalized Linear Models (GLM)}
\label{regression:GLM}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Binomial Regression}
\label{regression:GLM:binomial}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Poisson Regression}
\label{regression:GLM:poisson}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Gaussian Process Regression (Kriging)}
\label{regression:kriging}
% TODO

% TODO see \cite{Brochu2010} section 2.6 and the additional references it cites

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Difference in Differences (DID)}
\label{regression:DID}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Fixed-Effects Model}
\label{regression:fixed_effects}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Mixed-Effects Model}
\label{regression:mixed_effects}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Simpson's Paradox}
\label{regression:simpsons_paradox}
% TODO


\section{Direct Sums and Free Modules}
\begin{definition}
    If $M_1,\ldots,M_n$ are $R$-modules, then their direct sum $M_1\oplus\cdots\oplus M_n$ is the set $M_1\times \cdots\times M_n$ with entry-wise addition and scalar multiplications.
\end{definition}
\begin{example}
    1. $R^n$ is simply $R\oplus\cdots\oplus R$ of $n$ copies of $R$.\\
    2. If $M_1,M_2\le M$, then the $R$-module homomorphism $M_1\oplus M_2\to M$ by $(m_1,m_2)\mapsto m_1+m_2$ is an isomorphism iff $M_1\cap M_2=\varnothing$ and $M_1+M_2=M$.
\end{example}
\begin{lemma}
    If $M=\bigoplus_{i=1}^nM_n$, and $N_1\le M_i$.
    Take $N=\bigoplus_{i=1}^nN_i$, then
    $$M/N\cong\bigoplus_{i=1}^nM_i/N_i$$
\end{lemma}
\begin{proof}
    Apply the first isomorphism theorem to the surjective $R$-module map $\phi:M\to\bigoplus_{i=1}^nM_i/N_i$ by $(m_1,\ldots,m_n)\mapsto(m_1+N_1,\ldots,m_n+N_n)$.
\end{proof}
\begin{example}
    Taking $R=\mathbb Z$ then $\mathbb Z^2=\mathbb Z\oplus \mathbb Z$, then we have $(\mathbb Z\oplus\mathbb Z)/(m\mathbb Z\oplus n\mathbb Z)\cong(\mathbb Z/m\mathbb Z)\oplus(\mathbb Z/n\mathbb Z)$.
\end{example}
\begin{definition}
    Let $m_1,\ldots,m_n\in M$.
    The set $\{m_1,\ldots,m_n\}$ is independent if $r_1m_1+\cdots +r_nm_n=0\implies \forall i,r_i=0$.
\end{definition}
\begin{definition}
    A subset $S$ of an $R$-module $M$ generates $M$ freely if $S$ generates $M$ and any function $\psi:S\to N$ for another $R$-module $N$ extends to an $R$-module homomorphism $M\to N$.
\end{definition}
Note that if such an extension exists then it is necessarily unique.
\begin{definition}
    A freely-generated $R$-module is called a free $R$-module.
    The corresponding $S$ is called the free basis.
\end{definition}
\begin{proposition}
    For an $R$-module $M$ and a subset $S=\{m_1,\ldots,m_n\}\subset M$, the followings are equivalent:\\
    1. $S$ generates $M$ freely.\\
    2. $S$ generates $M$ and $S$ is independent.\\
    3. Every $m\in M$ can be written uniquely in the form $m=r_1m_1+\cdots +r_nm_n$ for $r_1,\ldots,r_n\in R$.\\
    4. The $R$-module homomorphism $R^n\to M$ by $(r_1,\ldots,r_n)\mapsto r_1m_1+\ldots r_nm_n$ is an isomorphism.
\end{proposition}
\begin{proof}
    $1\implies 2$: We already knows that $S$ generates $M$, so it suffices to show that $S$ is independent.
    Suppose for sake of contradiction that $r_1m_1+\ldots+r_nm_n=0$ for some $r_i\in R$ and some $r_j$ is nonzero.
    Consider the function $\psi:S\to R$ by $m_j\mapsto 1$ and $m_i\mapsto 0$ for any $i\neq j$.
    Suppose this extends to an $R$-module map $\theta:M\to R$, then $0=\theta(0)=\theta(r_1m_1+\cdots +r_nm_n)=r_j$, contradiction.\\
    Remaining implications $2\implies 3\implies 1$ and $3\iff 4$ are just as easy if not easier.
\end{proof}
Sadly not all $R$-modules are free.
Even if it is, the free basis does not behave like what we expect from a vector space.
\begin{example}[non-example]
    1. Suppose we have a nontrivial finite abelian group $A$, then $A$ is not free as a $\mathbb Z$-module since it is not isomorphic to $\mathbb Z^n$ which is infinite.\\
    2. The set $\{2,3\}\subset\mathbb Z$ generates $\mathbb Z$ as a $\mathbb Z$-module, but it is not independent and no subset of it gives a free basis.
\end{example}
\begin{proposition}[Theorem on Invariant of Dimension]
    Let $R$ be a nonzero ring.
    If $R^m\cong R^n$ as $R$-modules, then $m=n$.
\end{proposition}
We introduce the following general construction: Let $R$ be a ring and $I\unlhd R$ and $M$ is an $R$-module.
We write $IM=\{im:i\in I,m\in M\}\le M$.
Then the quotient $M/(IM)$ is an $R/I$ module by $(r+I)(m+IM)=rm+IM$.
Also by Zorn's Lemma, for any proper ideal $I$ in a ring $R$, there is a maximal ideal containing $I$ (this is obvious when $R$ is Noetherian).
\footnote{I think we can prove the proposition without using AC (or equivalence)}
\begin{proof}
    Return to our proof, suppose $R^m\cong R^n$.
    Choose $I\unlhd R$ maximal, then we have
    $$(R/I)^m\cong R^m/(IR^m)\cong R^n/(IR^n)\cong (R/I)^n$$
    But $R/I$ is a field, so $m=n$.
\end{proof}


\documentclass{article}
\input{preamble}

\newcommand{\version}{Draft: 02 April 2020}

\lhead{\scriptsize}
\chead{\scriptsize}
\rhead{\scriptsize \thepage}

\lfoot{\scriptsize}
\cfoot{\scriptsize}
\rfoot{\scriptsize \version}

%==============================================================================
\begin{document}
%==============================================================================

We define the Oneka-type regional flow model by the following discharge potential:
%
\begin{equation}\label{1}
    \Phi(x,y) = A x^2 + B y^2 + C xy + Dx + Ey + F
\end{equation}
%
Equation~\eqref{1} is a quadratic form in real variables. We define $z$ and $\bar{z}$ as
%
\begin{equation}\label{2}
    z = x + i y \qquad \qquad \bar{z} = x - i y
\end{equation}
%
Inverting yields
%
\begin{equation}\label{3}
    x = \frac{z + \bar{z}}{2} \qquad \qquad y = \frac{z - \bar{z}}{2i}
\end{equation}
%
Substituting \eqref{3} into \eqref{1} yields
%
\begin{equation}\label{4}
    \Phi(z,\bar{z})
    = A \left(\frac{z + \bar{z}}{2}\right)^2 + B \left(\frac{z - \bar{z}}{2i}\right)^2 +
        C \left(\frac{z + \bar{z}}{2}\right)\left(\frac{z - \bar{z}}{2i}\right) +
        D \left(\frac{z + \bar{z}}{2}\right) + E \left(\frac{z - \bar{z}}{2i}\right) + F
\end{equation}
%
Rearranging \eqref{4} yields
%
\begin{equation}\label{5}
    \Phi(z, \bar{z}) = a z^2 + b \bar{z}^2 + c z \bar{z} + d z + e \bar{z} + f
\end{equation}
%
where
%
\begin{align}
    a &= \frac{A - B - iC}{4} \\
    b &= \frac{A - B + iC}{4} \\
    c &= \frac{A + B}{2} \\
    d &= \frac{D - iE}{2} \\
    e &= \frac{D + iE}{2} \\
    f &= F
\end{align}
%
Alternatively,
%
\begin{equation}
    \frac{1}{4}
    \begin{bmatrix}
    1& -1& -1i& 0& 0& 0 \\
	1& -1& 1i& 0& 0& 0 \\
	2& 2& 0& 0& 0& 0 \\
	0& 0& 0& 2& -2i& 0 \\
	0& 0& 0& 2& 2i& 0 \\
	0& 0& 0& 0& 0& 4
    \end{bmatrix}
    \begin{bmatrix}A \\ B \\ C \\ D \\ E \\ F \end{bmatrix}
    =
    \begin{bmatrix}a \\ b \\ c \\ d \\ e \\ f \end{bmatrix}
\end{equation}
%
Inverting yields
%
\begin{equation}
    \begin{bmatrix}
	1& 1& 1& 0& 0& 0 \\
	-1& -1& 1& 0& 0& 0 \\
	2i& -2i& 0& 0& 0& 0 \\
	0& 0& 0& 1& 1& 0 \\
	0& 0& 0& 1i& -1i& 0 \\
	0& 0& 0& 0& 0& 1
    \end{bmatrix}
    \begin{bmatrix}a \\ b \\ c \\ d \\ e \\ f \end{bmatrix}
    =
    \begin{bmatrix}A \\ B \\ C \\ D \\ E \\ F \end{bmatrix}
\end{equation}

Note that $b = \bar{a}$ and $e = \bar{d}$, so \eqref{5} simplifies to
%
\begin{align}\label{7}
    \Phi(z, \bar{z})
    &= a z^2 + \bar{a} \bar{z}^2 + c z \bar{z} + d z + \bar{d} \bar{z} + f \\
    &= \real{a z^2 + d z + f} + c z \bar{z}
\end{align}
%
where $a$ and $d$ are complex, but $c$ and $f$ are real.

%==============================================================================
\end{document}
%==============================================================================

\newpage
%------------------------------------------------
\subsection{Regional recharge}
%------------------------------------------------
The regional recharge is given by
%
\begin{equation}\label{2.3}
    N = -2 \left( A + B \right)
\end{equation}
%
\begin{equation}\label{2.4}
    \ev{N} = -2 \left( \ev{A} + \ev{B} \right)
\end{equation}
%
\begin{equation}\label{2.5}
    \var{N} = 4 \left( \var{A} + \var{B} + 2\cov{A,B} \right)
\end{equation}