\documentclass[paper=a4, fontsize=11pt]{scrartcl} % A4 paper and 11pt font size
\usepackage{physics}
\usepackage[T1]{fontenc} % Use 8-bit encoding that has 256 glyphs
\usepackage{fourier} % Use the Adobe Utopia font for the document - comment this line to return to the LaTeX default
\usepackage[english]{babel} % English language/hyphenation
\usepackage{amsmath,amsfonts,amsthm} % Math packages
\usepackage{braket}
\usepackage{lipsum} % Used for inserting dummy 'Lorem ipsum' text into the template
\usepackage{tikz}
\usepackage{amsmath}
\usepackage{sectsty} % Allows customizing section commands
\allsectionsfont{\centering \normalfont\scshape} % Make all sections centered, the default font and small caps
\usepackage[mathscr]{euscript}
\usepackage{bm}
\newcommand{\uvec}[1]{\boldsymbol{\hat{\textbf{#1}}}}
\usepackage[thinlines]{easytable}
\usepackage{fancyhdr} % Custom headers and footers
\pagestyle{fancyplain} % Makes all pages in the document conform to the custom headers and footers
\fancyhead{} % No page header - if you want one, create it in the same way as the footers below

\usepackage{multicol}
\fancyfoot[L]{} % Empty left footer
\fancyfoot[C]{} % Empty center footer
\fancyfoot[R]{\thepage} % Page numbering for right footer
\renewcommand{\headrulewidth}{0pt} % Remove header underlines
\renewcommand{\footrulewidth}{0pt} % Remove footer underlines
\setlength{\headheight}{13.6pt} % Customize the height of the header
\usepackage{float}
\numberwithin{equation}{section} % Number equations within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)
\numberwithin{figure}{section} % Number figures within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)
\numberwithin{table}{section} % Number tables within sections (i.e. 1.1, 1.2, 2.1, 2.2 instead of 1, 2, 3, 4)

\setlength\parindent{0pt} % Removes all indentation from paragraphs - comment this line for an assignment with lots of text
\usepackage{pgfplots}

\pgfplotsset{
  compat=newest,
  xlabel near ticks,
  ylabel near ticks
}
%----------------------------------------------------------------------------------------
%	TITLE SECTION
%----------------------------------------------------------------------------------------

\newcommand{\horrule}[1]{\rule{\linewidth}{#1}} % Create horizontal rule command with 1 argument of height

\title{
\normalfont \normalsize
\textsc{California State University San Marcos \\ Dr. De Leone, Physics 323} \\ [25pt] % Your university, school and/or department name(s)
\horrule{0.5pt} \\[0.4cm] % Thin top horizontal rule
\huge H.W. 5 \\ % The assignment title
\horrule{2pt} \\[0.5cm] % Thick bottom horizontal rule
}

\author{Josh Lucas} % Your name

\date{\normalsize\today} % Today's date or a custom date

\begin{document}

\maketitle % Print the title

%----------------------------------------------------------------------------------------
%	PROBLEM 1
%----------------------------------------------------------------------------------------

\section*{Problem 1.13}
\textbf{Consider a quantum system with an observable A that has three possible measurement
results: a1, a2, and a3.}\\
\textbf{a) Write down the three kets $\ket{a_1}, \ket{a_2}, \ket{a_3},$ corresponding to these possible results
using matrix notation.}\\
\\
We can write the state vectors in matrix notation by choosing  orthogonal directions for each observable, giving them a magnitude of one and writing them in column form.
$$
\ket{a_1} =
\begin{pmatrix}
1\\ 0\\0
\end{pmatrix},
\quad
\ket{a_2} =
\begin{pmatrix}
0\\ 1\\0
\end{pmatrix},
\quad
\ket{a_3} =
\begin{pmatrix}
0\\0\\1
\end{pmatrix}
$$
\textbf{b) The system is prepared in the state}\\
$$\ket{\psi} = 1\ket{a_1} -2\ket{a_2} +5\ket{a_3}$$
\textbf{Write this state in matrix notation and calculate the probabilities of all possible measurement
results of the observable A. Plot a histogram of the predicted measurement results.}\\
\\
The magnitude of each observable is multiplied by its unit vector.
\begin{equation*}
\ket{\psi} = \begin{pmatrix}
1\\0\\0
\end{pmatrix} -
2 \begin{pmatrix}
0\\1\\0
\end{pmatrix} +
5 \begin{pmatrix}
0\\0\\1
\end{pmatrix}\ =\
\begin{pmatrix}
1\\-2\\5
\end{pmatrix}
\end{equation*}
We need to normalize the wave function by dividing the ket by its magnitude,
\begin{align*}
\ket{\psi} & = C
\begin{pmatrix}
1\\-2\\5
\end{pmatrix} \\
\braket{\psi|\psi} & = C^2 (\ 1\quad -2\quad 5\ )
\begin{pmatrix}
1\\-2\\5
\end{pmatrix}\\
& = C^2(1+4+25) \\
C & = \frac{1}{\sqrt{30}}\\
\ket{\psi} & = \frac{1}{\sqrt{30}}
\begin{pmatrix}
1\\-2\\5
\end{pmatrix}
\end{align*}
We can find the probabilities of each result by multiplying possible bra measurement direction with the normalized ket,
\begin{multicols}{3}
\noindent
\begin{align*}
\mathscr{P}_{a_1} & = \bigg| \braket{a_1|\psi} \bigg|^2 \\
& = \Bigg| (\ 1\quad 0\quad 0\ )
\frac{1}{\sqrt{30}}\begin{pmatrix}
1\\-2\\5
\end{pmatrix}
 \Bigg|^2 \\
& = \Big |
\frac{1}{\sqrt{30}}
  \Big |^2 \\
  \mathscr{P}_{a_1} & = \frac{1}{30}
\end{align*}
\begin{align*}
\mathscr{P}_{a_2} & = \bigg| \braket{a_2|\psi} \bigg|^2 \\
& = \Bigg| (\ 0\quad 1\quad 0\ )
\frac{1}{\sqrt{30}}\begin{pmatrix}
1\\-2\\5
\end{pmatrix}
 \Bigg|^2 \\
& = \Big |
\frac{-2}{\sqrt{30}}
  \Big |^2 \\
  \mathscr{P}_{a_2} & = \frac{4}{30}
\end{align*}
\begin{align*}
\mathscr{P}_{a_3} & = \bigg| \braket{a_3|\psi} \bigg|^2 \\
& = \Bigg| (\ 0\quad 0\quad 1\ )
\frac{1}{\sqrt{30}}\begin{pmatrix}
1\\-2\\5
\end{pmatrix}
 \Bigg|^2 \\
& = \Big |
\frac{5}{\sqrt{30}}
  \Big |^2 \\
  \mathscr{P}_{a_3  } & = \frac{25}{30} = \frac{5}{6}
\end{align*}
\end{multicols}
\begin{center}
\begin{tikzpicture}
    \begin{axis}[
      ybar,
      bar width=15pt,
      xlabel={$\bra{\psi}$},
      ylabel={Probability},
      ymin=0,
      ymax =100,
      ytick=\empty,
      xtick=data,
      axis x line=bottom,
      axis y line=left,
      enlarge x limits=0.2,
      symbolic x coords={ $ a_1$, $ a_2$, $a_3$ },
      xticklabel style={anchor=base,yshift=-\baselineskip},
      nodes near coords={\pgfmathprintnumber\pgfplotspointmeta\%}
    ]
      \addplot[fill=white] coordinates {
        ($ a_1$,3.33)
        ($ a_2$,13.3)
        ($a_3$, 83.3)
      };
    \end{axis}
  \end{tikzpicture}
\end{center}
\textbf{c) In a different experiment, the system is prepared in the state}\\
$$
\ket{\psi} = 2\ket{a_1} +3i\ket{a_2}
$$
\textbf{Write this state in matrix notation and calculate the probabilities of all possible measurement
results of the observable A. Plot a histogram of the predicted measurement results.}
\begin{align*}
\ket{\psi} & =
\begin{pmatrix}
2\\3i\\0
\end{pmatrix} \quad \text{Normalize the function} \\
\braket{\psi|\psi} & = C^* (\ 2\quad -3i\quad 0\ )\
C \begin{pmatrix}
2\\ 3i\\ 0
\end{pmatrix}\\
& = C^2 (4 + 9)\\
C & = \frac{1}{\sqrt{13}} \\
\ket{\psi} & = \frac{1}{\sqrt{13}}
\begin{pmatrix}
2\\3i \\ 0
\end{pmatrix}
\end{align*}

\begin{multicols}{3}
\noindent
\begin{align*}
\mathscr{P}_{a_1} & = \bigg| \braket{a_1|\psi} \bigg|^2 \\
& = \Bigg| (\ 1\quad 0\quad 0\ )
\frac{1}{\sqrt{13}}\begin{pmatrix}
2\\3i\\0
\end{pmatrix}
 \Bigg|^2 \\
& = \Big |
\frac{2}{\sqrt{13}}
  \Big |^2 \\
  \mathscr{P}_{a_1} & = \frac{4}{13}
\end{align*}
\begin{align*}
\mathscr{P}_{a_2} & = \bigg| \braket{a_2|\psi} \bigg|^2 \\
& = \Bigg| (\ 0\quad 1\quad 0\ )
\frac{1}{\sqrt{13}}\begin{pmatrix}
2\\3i\\0
\end{pmatrix}
 \Bigg|^2 \\
& = \Big |
\frac{3i}{\sqrt{13}}
  \Big |^2 \\
  \mathscr{P}_{a_2} & = \frac{9}{13}
\end{align*}
\begin{align*}
\mathscr{P}_{a_3} & = \bigg| \braket{a_3|\psi} \bigg|^2 \\
& = \Bigg| (\ 0\quad 0\quad 1\ )
\frac{1}{\sqrt{13}}\begin{pmatrix}
2\\3i\\0
\end{pmatrix}
 \Bigg|^2 \\
& = 0 \\
  \mathscr{P}_{a_3} & = 0
\end{align*}
\end{multicols}

\begin{center}
\begin{tikzpicture}
    \begin{axis}[
      ybar,
      bar width=15pt,
      xlabel={$\bra{\psi}$},
      ylabel={Probability},
      ymin=0,
      ymax =100,
      ytick=\empty,
      xtick=data,
      axis x line=bottom,
      axis y line=left,
      enlarge x limits=0.2,
      symbolic x coords={ $ a_1$, $ a_2$, $a_3$ },
      xticklabel style={anchor=base,yshift=-\baselineskip},
      nodes near coords={\pgfmathprintnumber\pgfplotspointmeta\%}
    ]
      \addplot[fill=white] coordinates {
        ($ a_1$,30.7)
        ($ a_2$,69.2)
        ($a_3$, 0)
      };
    \end{axis}
  \end{tikzpicture}
\end{center}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 1.15}
\textbf{Consider a quantum system described by a basis} $\ket{a_1}, \ket{a_2},$ and $\ket{a_3}$. \textbf{The system is initially in a state}\\
\begin{equation*}
\ket{\psi_i} = \tfrac{i}{\sqrt{3}} \ket{a_1} + \sqrt{\tfrac{2}{3}} \ket{a_2}.
\end{equation*}
\textbf{Find the probability that the system is measured to be in the final state}
\begin{equation*}
\ket{\psi_f} = \tfrac{1+i}{\sqrt{3}} \ket{a_1} + \frac{1}{\sqrt{6}} \ket{a_2} +\tfrac{1}{\sqrt{6}} \ket{a_3}.
\end{equation*}
\begin{align*}
\mathscr{P}_{\psi_f} & = \Big |\braket{\psi_f|\psi_i} \bigg|^2 \\
& = \Bigg|\bigg(\tfrac{1-i}{\sqrt{3}}\quad \tfrac{1}{\sqrt{6}}\quad \tfrac{1}{\sqrt{6}} \bigg)
\begin{pmatrix}
\tfrac{i}{\sqrt{3}} \\
\sqrt{\tfrac{2}{3}} \\
0
\end{pmatrix}
 \Bigg|^2 \\
 & =  \Bigg |  \frac{1-i}{3} + \sqrt{\frac{2}{9}} + 0  \Bigg |^2 \\
 & =  \Bigg |  \frac{1-i}{3} + \frac{1}{3}  \Bigg |^2 \\
 & = \Bigg (  \frac{2}{3} + \frac{1}{3}  \Bigg )^2 \\
 \mathscr{P}_{\psi_f} & = \frac{5}{9}
\end{align*}
%----------------------------------------------------------------------------------------
\section*{Problem 1.16}
\textbf{The spin components of a beam of atoms prepared in the state} $\ket{\psi_{in}}$ \textbf{are measured and the following
experimental probabilities are obtained:}
\begin{multicols}{3}
\noindent
\begin{align*}
\mathscr{P}_+ & = \tfrac{1}{2} \\
\mathscr{P}_- & = \tfrac{1}{2}
\end{align*}
\begin{align*}
\mathscr{P}_{+x} & = \tfrac{3}{4} \\
\mathscr{P}_{-x} & = \tfrac{1}{4}
\end{align*}
\begin{align*}
\mathscr{P}_{+y} & = 0.067 \\
\mathscr{P}_{-y} & = 0.933
\end{align*}
\end{multicols}
\textbf{From the experimental data, determine the input state.}
\begin{align*}
\frac{1}{2} & = \big | \braket{+|\psi} \big |^2  &  \frac{1}{2} & = \big | \braket{-|\psi} \big |^2                                    \\
& = \bigg | \bra{+} \bigg \{ a\ket{+} + b\ket{-} \Bigg \} \bigg|^2 &    & = \bigg | \bra{-} \bigg \{ a\ket{+} + b\ket{-} \Bigg \} \bigg|^2                   \\
& = \bigg | a \braket{+|+} \bigg|^2         &        & = \bigg | b \braket{-|-} \bigg|^2                                      \\
& = \bigg | a \bigg |^2       &         & = \bigg | b \bigg |^2                                                    \\
a & = \frac{1}{\sqrt{2}}  &   b & = \frac{1}{\sqrt{2}}
\end{align*}
\horrule{2pt}
\begin{multicols}{2}
\noindent
\begin{align*}
\tfrac{3}{4} & = \big | \braket{+_x| \psi}  \big |^2 \\
\tfrac{3}{4} & = \bigg |  \tfrac{1}{\sqrt{2}} \bigg ( \bra{+} + \bra{-} \bigg ) \tfrac{1}{\sqrt{2}} \bigg ( \ket{+}  + e^{i\phi}\ket{-} \bigg )  \bigg |^2 \\
& = \bigg | \tfrac{1}{2} \big ( 1 + e^{i\phi}  \big )  \bigg|^2 \\
& = \tfrac{1}{4} \big ( 1 + e^{i\phi}  \big ) \big ( 1 + e^{-i\phi}   \big ) \\
& = \tfrac{1}{4} \big (  1^2 + 2\cos(\phi) + e^0 \big ) \\
& = \tfrac{1}{4} \big ( 2 + 2\cos(\phi) \big ) \\
& + \tfrac{1}{2} \big (  1 + \cos(\phi) \big ) \\
\tfrac{3}{2} - 1 & = \cos(\phi) \\
\cos^{-1}(\tfrac{1}{2}) &= \pm \tfrac{\pi}{3}
\end{align*}
\begin{align*}
\tfrac{1}{4} & = \big | \braket{-_x| \psi}  \big |^2 \\
\tfrac{1}{4} & = \bigg |  \tfrac{1}{\sqrt{2}} \bigg ( \bra{+} - \bra{-} \bigg ) \tfrac{1}{\sqrt{2}} \bigg ( \ket{+}  + e^{i\phi}\ket{-} \bigg )  \bigg |^2 \\
& = \bigg | \tfrac{1}{2} \big ( 1 - e^{i\phi}  \big )  \bigg|^2 \\
& = \tfrac{1}{4} \big ( 1 - e^{i\phi}  \big ) \big ( 1 - e^{-i\phi}   \big ) \\
& = \tfrac{1}{4} \big (  1^2 - 2\cos(\phi) + e^0 \big ) \\
& = \tfrac{1}{4} \big ( 2 - 2\cos(\phi) \big ) \\
& = \tfrac{1}{2} \big (  1 - \cos(\phi) \big ) \\
\cos(\phi) & = 1- \tfrac{1}{2}  \\
\cos^{-1}(\tfrac{1}{2}) & =  \pm \tfrac{\pi}{3}
\end{align*}
\end{multicols}
\horrule{2pt}
\begin{multicols}{2}
\noindent
\begin{align*}
0.067 & = \big | \braket{+_y| \psi}  \big |^2 \\
 & = \bigg |  \tfrac{1}{\sqrt{2}} \bigg ( \bra{+} -i \bra{-} \bigg ) \tfrac{1}{\sqrt{2}} \bigg ( \ket{+}  + e^{i\phi}\ket{-} \bigg )  \bigg |^2 \\
& = \bigg | \tfrac{1}{2} \big ( 1 - ie^{i\phi}  \big )  \bigg|^2 \\
& = \tfrac{1}{4} \big ( 1 - ie^{i\phi}  \big ) \big ( 1 + ie^{-i\phi}   \big ) \\
& = \tfrac{1}{4} \big (  1^2 + 2\sin(\phi) + e^0 \big ) \\
& = \tfrac{1}{4} \big ( 1+ 2\sin(\phi) \big ) \\
& = \tfrac{1}{2} \big (  1 + \sin(\phi) \big ) \\
\sin(\phi) & = 2(0.067) - 1 \\
\sin^{-1}(-0.866) & = - \tfrac{\pi}{3 } \ or\ -\tfrac{2\pi}{3}
\end{align*}
\begin{align*}
0.933 & = \big | \braket{+_y| \psi}  \big |^2 \\
 & = \big |  \tfrac{1}{\sqrt{2}} \big ( \bra{+} +i \bra{-} \bigg ) \tfrac{1}{\sqrt{2}} \big ( \ket{+}  + e^{i\phi}\ket{-} \big )  \bigg |^2 \\
& = \bigg | \tfrac{1}{2} \big ( 1 + ie^{i\phi}  \big )  \bigg|^2 \\
& = \tfrac{1}{4} \big ( 1 + ie^{i\phi}  \big ) \big ( 1 - ie^{-i\phi}   \big ) \\
& = \tfrac{1}{4} \big (  1^2 + 2\sin(\phi) + e^0 \big ) \\
& = \tfrac{1}{4} \big ( 1+ 2\sin(\phi) \big ) \\
& = \tfrac{1}{2} \big (  1 + \sin(\phi) \big ) \\
\sin(\phi) & = 2(0.933) - 1 \\
\sin^{-1}(0.866) & = \pm \tfrac{\pi}{3 } \
\end{align*}
\end{multicols}
\begin{equation*}
\ket{\psi_{in}} = \tfrac{1}{\sqrt{2}} \big ( \ket{+} + e^{-i\tfrac{\pi}{3}} \ket{-} \big )
\end{equation*}
\end{document}


\documentclass[12pt]{article}
\usepackage[fleqn]{amsmath}
\usepackage{textcomp}
\usepackage{gensymb}
\usepackage{amsfonts}
\usepackage{enumitem}
%\usepackage{tikz}  % Include for figures.
%\usepackage{subfiles}  % Include for subfiles.

\newcommand{\HOMEWORKNUM}{6}
\newcommand{\NAME}{D. Choi}
\newcommand{\DATE}{2020-05-29}

\title{\vspace{-4\baselineskip}MATH 225 - Homework \#\HOMEWORKNUM}
\author{\NAME}
\date{\DATE}

%\pagenumbering{gobble}  % Include for single-page document.

\begin{document}
\maketitle

\section*{1.}
\textit{Solve.}
\begin{align*}
	1x + 0y + 0z &= 0 \\
	0x + \frac{1}{2}y + \frac{1}{2}z &= 0 \\
	0x + \frac{1}{2}y + \frac{1}{2}z &= 1
\end{align*}
This system can be rewritten in matrix form $A\vec{v} = \vec{b}$ as
\begin{equation*}
	\begin{pmatrix}
		1 & 0 & 0 \\
		0 & \frac{1}{2} & \frac{1}{2} \\
		0 & \frac{1}{2} & \frac{1}{2}
	\end{pmatrix}
	\begin{pmatrix}
		x \\
		y \\
		z
	\end{pmatrix}
	=
	\begin{pmatrix}
		0 \\
		0 \\
		1
	\end{pmatrix}
	.
\end{equation*}
Such a matrix $A$ is an orthographic projection onto the plane $z = y$. \\
Since the vector $\vec{b}$ does not lie on the plane $z = y$,
\boxed{\text{no solutions exist}}
for $\vec{v}$.

\section*{2.}
\textit{Solve.}
\begin{align*}
	1x + 0y + 0z &= 10 \\
	0x + \frac{1}{2}y + \frac{1}{2}z &= 5 \\
	0x + \frac{1}{2}y + \frac{1}{2}z &= 5
\end{align*}
This system can be rewritten (again) in matrix form $A\vec{v} = \vec{c}$ as
\begin{equation*}
	\begin{pmatrix}
		1 & 0 & 0 \\
		0 & \frac{1}{2} & \frac{1}{2} \\
		0 & \frac{1}{2} & \frac{1}{2}
	\end{pmatrix}
	\begin{pmatrix}
		x \\
		y \\
		z
	\end{pmatrix}
	=
	\begin{pmatrix}
		10 \\
		5 \\
		5
	\end{pmatrix}
	.
\end{equation*}
Such a matrix $A$ is (still) an orthographic projection onto the plane
$z = y$. \\
Thus, solutions to the system are in the form
\begin{equation*}
	\vec{v} =
	\boxed{
		\vec{c} + s \begin{pmatrix} 0 \\ 1 \\ -1 \end{pmatrix}
	}
	,
\end{equation*}
where $s \in \mathbb{R}$.

\section*{3.}
\textit{Solve.}
\begin{equation*}
	\begin{pmatrix}
		\cos(30 \degree) & -\sin(30 \degree) & 0 \\
		\sin(30 \degree) & \cos(30 \degree) & 0 \\
		0 & 0 & 1
	\end{pmatrix}
	\begin{pmatrix}
		x \\
		y \\
		z
	\end{pmatrix}
	=
	\begin{pmatrix}
		\cos(40 \degree) \\
		\sin(40 \degree) \\
		10
	\end{pmatrix}
\end{equation*}
The matrix
\begin{equation*}
	\begin{pmatrix}
		\cos(30 \degree) & -\sin(30 \degree) & 0 \\
		\sin(30 \degree) & \cos(30 \degree) & 0 \\
		0 & 0 & 1
	\end{pmatrix}
\end{equation*}
is an instance of the rotation matrix about the z-axis
\begin{equation*}
	\text{Rot}_z(\theta) =
	\begin{pmatrix}
		\cos \theta & -\sin \theta & 0 \\
		\sin \theta & \cos \theta & 0 \\
		0 & 0 & 1
	\end{pmatrix}
\end{equation*}
where $\theta = 30 \degree$. \\
From this, the equation can be rewritten as
\begin{equation*}
	\text{Rot}_z(30 \degree)
	\begin{pmatrix} x \\ y \\ z \end{pmatrix}
	=
	\text{Rot}_z(40 \degree)
	\begin{pmatrix} 0 \\ 0 \\ 10 \end{pmatrix}
	.
\end{equation*}
Thus,
\begin{align*}
	\begin{pmatrix} x \\ y \\ z \end{pmatrix}
	&=
	(\text{Rot}_z(30 \degree))^{-1}
	\text{Rot}_z(40 \degree)
	\begin{pmatrix} 0 \\ 0 \\ 10 \end{pmatrix} \\
	&=
	\text{Rot}_z(-30 \degree)
	\text{Rot}_z(40 \degree)
	\begin{pmatrix} 0 \\ 0 \\ 10 \end{pmatrix} \\
	&=
	\text{Rot}_z(10 \degree)
	\begin{pmatrix} 0 \\ 0 \\ 10 \end{pmatrix} \\
	&=
	\boxed{
		\begin{pmatrix}
			\cos(10 \degree) \\
			\sin(10 \degree) \\
			10
		\end{pmatrix}
	}
	.
\end{align*}

\end{document}


\subsubsection{Time Series Normalization}

\begin{frame}{Time Series Normalization}{$\eta$ Normalization}
    Given is a time series $Q = (q_1, \dots, q_n)$ of length $n$
    \begin{block}{$\eta$ Normalization}
        $\eta(q_i) = q_i - \mu$
    \end{block}
    \begin{block}{Mean of $Q$}
        $\mu = \frac{1}{n} \sum \limits_{i=1}^{n} q_i$
    \end{block}
\end{frame}

\begin{frame}<handout:0>{Time Series Normalization $\eta$}{Example}
    \begin{center}
        \resizebox {\textwidth} {!} {
            \begin{tabular}{cc}
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            xmin=0,
                            xmax=47,
                            xlabel=time,
                            ylabel=acceleration,
                            width=\axisdefaultwidth,
                            height=0.7*\axisdefaultheight,
                            reverse legend,
                            legend pos=south east]
                            \addplot[red, thick, mark=none] table {../data/fig/norm1/q.dat};
                            \addlegendentry{Q}
                            \addplot[blue, thick, mark=none] table {../data/fig/norm1/c.dat};
                            \addlegendentry{C}
                        \end{axis}
                    \end{tikzpicture}
                } & \quad
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tabular}[b]{ll}
                        \begin{turn}{90}
                            \begin{tikzpicture}
                                \begin{axis}[
                                    xmin=0,
                                    xmax=47,
                                    ymin=-40,
                                    ymax=40,
                                    hide x axis,
                                    hide y axis,
                                    width=\axisdefaultwidth,
                                    height=0.7*\axisdefaultheight]
                                    \addplot[red, ultra thick, mark=none] table {../data/fig/norm1/q.dat};
                                \end{axis}
                            \end{tikzpicture}
                        \end{turn} \hspace*{3em} &
                        \begin{tikzpicture}
                            \begin{axis}[
                                enlargelimits=false,
                                ymin=0,
                                ymax=47,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=\axisdefaultwidth,
                                colorbar,
                                colormap/viridis high res]
                                \addplot[matrix plot*,
                                    mesh/cols=48,
                                    point meta=explicit] table[meta=C] {../data/fig/norm1/matrix.dat};
                            \end{axis}
                        \end{tikzpicture}\\
                        &
                        \\[1em]
                        &
                        \begin{tikzpicture}
                            \begin{axis}[
                                xmin=0,
                                xmax=47,
                                ymin=-40,
                                ymax=40,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=0.7*\axisdefaultheight]
                                \addplot[blue, ultra thick, mark=none] table {../data/fig/norm1/c.dat};
                            \end{axis}
                        \end{tikzpicture}
                    \end{tabular}
                }
            \end{tabular}
        }
    \end{center}
\end{frame}

\begin{frame}{Time Series Normalization $\eta$}{Example}
    \begin{center}
        \resizebox {\textwidth} {!} {
            \begin{tabular}{cc}
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            xmin=0,
                            xmax=47,
                            xlabel=time,
                            ylabel=acceleration,
                            width=\axisdefaultwidth,
                            height=0.7*\axisdefaultheight,
                            reverse legend,
                            legend pos=south east]
                            \addplot[gray, quiver={u=\thisrow{u}, v=\thisrow{v}}] table {../data/fig/norm1/path.dat};
                            \addplot[red, thick, mark=none] table {../data/fig/norm1/q.dat};
                            \addlegendentry{Q}
                            \addplot[blue, thick, mark=none] table {../data/fig/norm1/c.dat};
                            \addlegendentry{C}
                        \end{axis}
                    \end{tikzpicture}
                } & \quad
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tabular}[b]{ll}
                        \begin{turn}{90}
                            \begin{tikzpicture}
                                \begin{axis}[
                                    xmin=0,
                                    xmax=47,
                                    ymin=-40,
                                    ymax=40,
                                    hide x axis,
                                    hide y axis,
                                    width=\axisdefaultwidth,
                                    height=0.7*\axisdefaultheight]
                                    \addplot[red, ultra thick, mark=none] table {../data/fig/norm1/q.dat};
                                \end{axis}
                            \end{tikzpicture}
                        \end{turn} \hspace*{3em} &
                        \begin{tikzpicture}
                            \begin{axis}[
                                enlargelimits=false,
                                ymin=0,
                                ymax=47,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=\axisdefaultwidth,
                                colorbar,
                                colormap/viridis high res]
                                \addplot[matrix plot*,
                                    mesh/cols=48,
                                    point meta=explicit] table[meta=C] {../data/fig/norm1/matrix.dat};
                                \addplot[white, ultra thick, mark=*, mark size=1] table {../data/fig/norm1/matrix_path.dat};
                            \end{axis}
                        \end{tikzpicture}\\
                        &
                        \\[1em]
                        &
                        \begin{tikzpicture}
                            \begin{axis}[
                                xmin=0,
                                xmax=47,
                                ymin=-40,
                                ymax=40,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=0.7*\axisdefaultheight]
                                \addplot[blue, ultra thick, mark=none] table {../data/fig/norm1/c.dat};
                            \end{axis}
                        \end{tikzpicture}
                    \end{tabular}
                }
            \end{tabular}
        }
    \end{center}
\end{frame}

\begin{frame}{Time Series Normalization}{$z$ Normalization}
    Given is a time series $Q = (q_1, \dots, q_n)$ of length $n$
    \begin{block}{$z$ Normalization}
        $z(q_i) = \frac{q_i - \mu}{\sigma}$
    \end{block}
    \begin{block}{Standard deviation of $Q$}
        $\sigma = \frac{1}{n} \sum \limits_{i=1}^{n} (q_i - \mu)^2$
    \end{block}
\end{frame}

\begin{frame}<handout:0>{Time Series Normalization $z$}{Example}
    \begin{center}
        \resizebox {\textwidth} {!} {
            \begin{tabular}{cc}
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            xmin=0,
                            xmax=47,
                            xlabel=time,
                            ylabel=acceleration,
                            width=\axisdefaultwidth,
                            height=0.7*\axisdefaultheight,
                            reverse legend,
                            legend pos=south east]
                            \addplot[red, thick, mark=none] table {../data/fig/norm2/q.dat};
                            \addlegendentry{Q}
                            \addplot[blue, thick, mark=none] table {../data/fig/norm2/c.dat};
                            \addlegendentry{C}
                        \end{axis}
                    \end{tikzpicture}
                } & \quad
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tabular}[b]{ll}
                        \begin{turn}{90}
                            \begin{tikzpicture}
                                \begin{axis}[
                                    xmin=0,
                                    xmax=47,
                                    ymin=-2,
                                    ymax=2,
                                    hide x axis,
                                    hide y axis,
                                    width=\axisdefaultwidth,
                                    height=0.7*\axisdefaultheight]
                                    \addplot[red, ultra thick, mark=none] table {../data/fig/norm2/q.dat};
                                \end{axis}
                            \end{tikzpicture}
                        \end{turn} \hspace*{3em} &
                        \begin{tikzpicture}
                            \begin{axis}[
                                enlargelimits=false,
                                ymin=0,
                                ymax=47,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=\axisdefaultwidth,
                                colorbar,
                                colormap/viridis high res]
                                \addplot[matrix plot*,
                                    mesh/cols=48,
                                    point meta=explicit] table[meta=C] {../data/fig/norm2/matrix.dat};
                            \end{axis}
                        \end{tikzpicture}\\
                        &
                        \\[1em]
                        &
                        \begin{tikzpicture}
                            \begin{axis}[
                                xmin=0,
                                xmax=47,
                                ymin=-2,
                                ymax=2,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=0.7*\axisdefaultheight]
                                \addplot[blue, ultra thick, mark=none] table {../data/fig/norm2/c.dat};
                            \end{axis}
                        \end{tikzpicture}
                    \end{tabular}
                }
            \end{tabular}
        }
    \end{center}
\end{frame}

\begin{frame}{Time Series Normalization $z$}{Example}
    \begin{center}
        \resizebox {\textwidth} {!} {
            \begin{tabular}{cc}
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            xmin=0,
                            xmax=47,
                            xlabel=time,
                            ylabel=acceleration,
                            width=\axisdefaultwidth,
                            height=0.7*\axisdefaultheight,
                            reverse legend,
                            legend pos=south east]
                            \addplot[gray, quiver={u=\thisrow{u}, v=\thisrow{v}}] table {../data/fig/norm2/path.dat};
                            \addplot[red, thick, mark=none] table {../data/fig/norm2/q.dat};
                            \addlegendentry{Q}
                            \addplot[blue, thick, mark=none] table {../data/fig/norm2/c.dat};
                            \addlegendentry{C}
                        \end{axis}
                    \end{tikzpicture}
                } & \quad
                \resizebox* {!} {0.3\textwidth} {
                    \begin{tabular}[b]{ll}
                        \begin{turn}{90}
                            \begin{tikzpicture}
                                \begin{axis}[
                                    xmin=0,
                                    xmax=47,
                                    ymin=-2,
                                    ymax=2,
                                    hide x axis,
                                    hide y axis,
                                    width=\axisdefaultwidth,
                                    height=0.7*\axisdefaultheight]
                                    \addplot[red, ultra thick, mark=none] table {../data/fig/norm2/q.dat};
                                \end{axis}
                            \end{tikzpicture}
                        \end{turn} \hspace*{3em} &
                        \begin{tikzpicture}
                            \begin{axis}[
                                enlargelimits=false,
                                ymin=0,
                                ymax=47,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=\axisdefaultwidth,
                                colorbar,
                                colormap/viridis high res]
                                \addplot[matrix plot*,
                                    mesh/cols=48,
                                    point meta=explicit] table[meta=C] {../data/fig/norm2/matrix.dat};
                                \addplot[white, ultra thick, mark=*, mark size=1] table {../data/fig/norm2/matrix_path.dat};
                            \end{axis}
                        \end{tikzpicture}\\
                        &
                        \\[1em]
                        &
                        \begin{tikzpicture}
                            \begin{axis}[
                                xmin=0,
                                xmax=47,
                                ymin=-2,
                                ymax=2,
                                hide x axis,
                                hide y axis,
                                width=\axisdefaultwidth,
                                height=0.7*\axisdefaultheight]
                                \addplot[blue, ultra thick, mark=none] table {../data/fig/norm2/c.dat};
                            \end{axis}
                        \end{tikzpicture}
                    \end{tabular}
                }
            \end{tabular}
        }
    \end{center}
\end{frame}


\documentclass[12pt,letterpaper]{article}

\usepackage[margin=1.9cm]{geometry}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{pgfplots}
\usepackage{siunitx}

\pgfplotsset{compat=1.16}

\begin{document}
\section*{Part 1 Series RC}
\subsection*{1.}
\begin{tikzpicture}
    \begin{axis}[
        title = {Voltage vs Time},
        xlabel = Time (\si{\micro\second}),
        ylabel = Voltage (\si{\volt}),
        xmin = 0,
        xmax = 85,
        ymin = 0,
        ymax = 11,
        axis lines=left,
        grid = both,
        height = 6cm,
        width = 17cm,
        legend pos=south west
    ]

    \addplot table [x=t, y=Output, col sep=comma, mark=none] {Part1.csv};
    \addlegendentry{Output Voltage}
    \addplot table [x=t, y=Input, col sep=comma, mark=none] {Part1.csv};
    \addlegendentry{Input Voltage}
    \addplot [mark=none, black, domain=0:45.65] {3.59};
    \draw (45.65,0) -- (45.65, 3.59);
    \addplot [mark=*] coordinates {(45.65, 3.59)};
\end{axis}
\end{tikzpicture}
\subsection*{2.}
The time constant $\tau$ appears at $V = \frac{V_0}{e} = \frac{10}{e} = \SI{3.59}{\volt}$

At $\SI{3.59}{\volt}$, $\tau = \SI{45.65}{\micro\second}$
\subsection*{3a.}
$$\frac{|47-45.65|}{47} \times 100 \% = 2.88\%$$
\subsection*{3b.}
$$\frac{|47-45.21|}{47} \times 100 \% = 3.83\%$$
\subsection*{3c.}
$$\frac{|47-43.1|}{47} \times 100 \% = 8.51\%$$
\subsection*{4.}
Using KVL:
$$V_c = V_o - V_r$$
where:
\begin{gather*}
    V_r = V_o e^{\frac{t}{\tau}}, \tau = \SI{43}{\micro\second}\\
V_c = V_o - V_o e^{\frac{t}{\tau}} = V_o (1-e^{\frac{t}{\tau}})
\end{gather*}

Comparison:

Recorded $V_c = \SI{1.84}{\volt}$
Calculated $V_c = 10(1-e^{-\frac{10}{43}}) = \SI{2.07}{\volt}$
\section*{Part 2 Series RL}
\subsection*{5.}
\begin{tikzpicture}
    \begin{axis}[
        title = {Current vs Time},
        xlabel = Time (\si{\micro\second}),
        ylabel = Current (\si{\ampere}),
        xmin = 0,
        xmax = 80,
        ymin = 0,
        ymax = 0.08,
        axis lines=left,
        grid = both,
        height = 7cm,
        width = 17cm,
        legend pos=south east
    ]
    \addplot table [x=t, y=Theoretical, col sep=comma, mark=none] {Part2.csv};
    \addlegendentry{Theoretical Current}
    \addplot table [x=t, y=Calculated, col sep=comma, mark=none] {Part2.csv};
    \addlegendentry{Actual Current}
    \end{axis}
\end{tikzpicture}

\subsection*{6.}
\begin{gather*}
\tau_{RL} = \SI{14.6}{\micro\second} = \frac{L}{R}\\
L = \SI{14.6}{\micro\second} * \SI{150}{\ohm} = \SI{2.19}{\milli\henry}
\end{gather*}

Comparison between calculated and labelled:

$$\frac{|2.5-2.19|}{2.5} \times 100 \% = 12.4\%$$
\section*{Part 3 Series RLC}
\subsection*{7.}
Apply Ohm's law to voltages
\begin{center}
    \begin{tabular}{|c|c|}
        \hline
        {Peak Current (\si{\ampere})} & {Time (\si{\second})} \\ \hline
        0.015&7.2\\  \hline
        -0.0086&22.8\\\hline
        0.0054&38.8\\\hline
        -0.003&54.8\\\hline
        0.00168&70.8\\\hline
        -0.00104&86.8\\\hline
    \end{tabular}
\end{center}
Example calculation:

$$\text{Voltage peak at } t = \SI{7.2}{\micro\second} \text{ is } \SI{1.5}{\volt} \text{ which is } \frac{\SI{1.5}{\volt}}{\SI{100}{\ohm}} = \SI{0.015}{\ampere}$$
\subsection*{8.}
\begin{tikzpicture}
    \begin{axis}[
        title = {$\ln|i|$ vs Time},
        xlabel = Time (\si{\micro\second}),
        ylabel = $\ln|i|$,
        xmin = 0,
        xmax = 80,
        ymin = -8,
        ymax = -2,
        grid = both,
        height = 5.5cm,
        width = 17cm
    ]
    \addplot table [green, x=t, y=i, col sep=comma, mark=none] {Part3.csv};
    \addlegendentry{Calculated}
    \addplot [mark=none, black, domain=0:80, opacity=0.5] {-0.033747*x - 3.96};
    \addlegendentry{Linear Regression}

\end{axis}
\end{tikzpicture}

The slope, through linear regression, was calculated to be $\alpha = 33747 = \frac{R}{2L}$
$$L=\frac{R}{2\alpha} = \frac{\SI{150}{\ohm}}{2\times33747} = \SI{2.22}{\milli\henry}$$

\subsection*{9.}
\begin{gather*}
\ln\left(\frac{V_o}{\omega_D L}\right) = - 3.96\\
\frac{V_o}{\omega_D L} = e^{-3.96}\\
L=\frac{V_o}{\omega_D L} e^{-3.96} = \frac{\SI{10}{\volt}}{2\pi} (\SI{31.65}{\kilo\hertz}) e^{-3.96} = \SI{2.64}{\milli\henry}
\end{gather*}

\subsection*{10.}
\begin{gather*}
f = \frac{\omega}{2\pi}
\end{gather*}
Where the damped frequency is given by $\omega_D = \sqrt{\omega_o^2-\alpha ^2}$

Since $\omega_o^2 \gg \alpha ^2$, the damped frequency yields a result such that $\omega_D \simeq \sqrt{\omega_o^2} = \omega_o$

Example:
\begin{gather*}
\omega_o = 2\pi f_o = 2\pi \SI{31.8}{\kilo\hertz} = \SI{199805.29}{\radian\per\second}\\
\omega_o^2 = 3.9922\times 10^{10} \gg \alpha^2 = 1.14\times 10^9\\
\omega_D = \sqrt{\omega_o ^2 - \alpha^2} = \SI{196934.75}{\radian\per\second} \simeq \omega_o
\end{gather*}

\subsection*{11.}
\begin{gather*}
31.8\times10^3 = \frac{1}{2\pi \sqrt{L \cdot 10\times10^{-9}}}\\
L = \SI{0.00250}{\henry} = \SI{2.50}{\milli\henry}
\end{gather*}

\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Convergence of Measurable Functions %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Convergence of Measurable Functions}

\begin{exercise}{5.1}{}

    Let $(f_n)_{n=1}^{\infty}$ be a sequence of $\CalF$-measurable functions $f_n: \Omega \to \mathbb{R}$. Then the set $A$ of those $\omega \in \Omega$ such that $\lim_{n \to \infty} f_n(\omega)$ converges to some (finite) number belongs to $\CalF$.

\end{exercise}

\begin{exercise}{5.2}{Almost Finite, Converging Sequence is Bounded}

    Assume that $\mu(\Omega) < \infty$. Let $(f_n)_{n=1}^{\infty}$ be \emph{$\mu$-a.e. finite}, converging in measure to $\mu$ to some $f: \Omega \to \mathbb{R}$. Then the sequence of $f_n$ is \emph{bounded in measure $\mu$, uniformly in $n$}, i.e.:

        \begin{align*}
            \lim_{K \to \infty} \sup_{n \geq 1} \mu(|f_n| \geq K) = 0.
        \end{align*}

    \Hint $f_n$ \emph{$\mu$-a.e. finite} \emph{and} $\mu(\Omega) < \infty$ $\Rightarrow$ $f_n$ bounded in measure (not necessarily uniformly), so

        \begin{align*}
            \lim_{K \to \infty} \sup_{n \geq 1} \mu(|f_n| \geq K) = \\ \lim_{K \to \infty} \limsup_{n \to \infty} \mu(|f_n| \geq K).
        \end{align*}

    Then use observation of splitting measures of inequalities.

\end{exercise}

\begin{exercise}{5.3}{Product of Bounded \& Zero Convergent is Zero Convergent}

    Let $(f_n)_{n=1}^{\infty}$ and $(g_n)_{n=1}^{\infty}$ be sequences of $\mu$-a.e. finite measurable functions such that the $f_n$ are bounded in measure $\mu$, uniformly in $n$ and $g_n \to 0$ in measure $\mu$, as $n \to \infty$. Then $f_ng_n \to 0$ in measure $\mu$, as $n \to \infty$.

\end{exercise}

\begin{exercise}{Ws 3, 1}{}

    Let $\mu-\lim f_n = f$, then there exists a subsequence $(f_{n_k})_{k=1}^{\infty}$ such that $(n_k)_{k=1}^{\infty}$ is increasing and $f_{n_k} \to f$ ($\mu$-a.e.).

    \Hint Borel-Cantelli with $A_k = \{ |f_{n_k} - f| \geq 1/k \}$ s.t. $\mu(A_k) \leq 1/k^2$.

\end{exercise}

\begin{theorem}{5.4}{Measure Convergence Has Almost Everywhere Converging Subsequence}

    Let $(f_n)_{n=1}^{\infty}$ be a sequence of functions converging in measure $\mu$ to some $\mu$-a.e. finite function $f$. Then there exists a (strictly) increasing sequence $(n_k)_{k=1}^{\infty}$ of positive integers such that $\lim_{k \to \infty} f_{n_k} = f$ $\mu$-almost everywhere.

\end{theorem}

\begin{exercise}{5.5}{}

    Convergence in measure $\mu$ does not imple convergence $\mu$-almost everywhere.

    \Hint $(\mathbb{R}, \mathcal{B}(\mathbb{R}), \lambda)$ with $f_n = \Indicator{[k/2^m, (k+1)/2^m]}$ where $k = 0,1,\hdots,2^m - 1$ and $m=0,1,\hdots$ such that $n = 2^m + k$.

\end{exercise}

\begin{exercise}{Ws 3, 2}{Convergence Implication}

    Let $\mu(\Omega) < \infty$. Then $\lim_{n \to \infty} f_n = f$ ($\mu$-a.e.) $\Rightarrow$ $\mu-\lim_{n \to \infty} f_n = f$.

\end{exercise}

\begin{exercise}{Ws 3, 3}{Relaxed Domnitated Convergence}

    Lebegue's Theorem on Dominated convergence holds under the following, relaxed conditions:

        \begin{enumerate}[(i)]
            \setlength{\parskip}{0em}
            \item $\lim_{n \to \infty} f_n = f$ \emph{$\mu$-a.e.},  $|f_n| \leq g|$ $\mu$-a.e. and $g \in L_1(\Omega, \CalF, \mu)$, i.e. $\int_{\Omega} |g| \, d\mu < \infty$; and
            \item $\mu-\lim_{n \to \infty} f_n = f$,  $|f_n| \leq g|$ $\mu$-a.e. and $g \in L_1(\Omega, \CalF, \mu)$, i.e. $\int_{\Omega} |g| \, d\mu < \infty$.
        \end{enumerate}

\end{exercise}


%!TEX root =  ../main.tex

\subsection{Exponential Equations}

\objective{Solve exponential/logarithmic equations through a variety of techniques}

Whatever function we are dealing with, if $f(x)=f(3)$ then $x$ must equal 3.  This is true
if $f(x)$ is a logarithmic, exponential or other kind of function.  So, if $\log_{\pi}(x+3) =
\log_{\pi}{7-x}$, then $x+3 = 7 - x$ and $2x = 4$, so $x$ must equal 2.  Similarly,
if $3^{2x+1} = 3^7$ then $2x+1 =7$ and $x=3$.  These are the most basic kinds of
exponential equations.

You are probably pretty good at spotting when numbers are related via multiplication
and division because of so many years of practicing such arithmetic.  $56x+49=0$ should
leap out to you as $7(8x+7)=0$ or even $x=-\frac{7}{8}$ because you have known
your seven times table for so many years now.  Exponents are not memorized to the
same extent --- and nor should they be --- but perhaps you might consider expanding your
familiarity with them just a little more.  The return on investment is pretty low after cubes,
but you will be pleasantly surprised at the added reach knowing the numbers on the following
table will give you:

$$
\begin{matrix}
2 & 4 & 8 & 16 & 32 \\
3 & 9 & 27 & 81 & 243 \\
4 & 16 & 64 & 256 & 1024 \\
5 & 25 & 125 & 625 &  \\
6 & 36 & 216 &  &  \\
7 & 49 & 343 &  &  \\
8 & 64 & 512 &  &  \\
9 & 81 & 729 &  &
\end{matrix}
$$

\begin{example}{Exponential Base Manipulation}
\exProblem
Solve $9^{x+1}=\left(\frac{1}{27}\right)^{2x}$.

\exSolution
With a familiarity with exponents, we see that both numbers are power of 3.  We can
re-write each base to show this, and use the same equality principle as before.
\begin{align*}
	9^{x+1} &= \left(\frac{1}{27}\right)^{2x}\\
	(3^2)^{x+1} &= (3^{-3})^{2x}\\
	3^{2x+2} &= 3^{-6x}\\
	2x + 2 &= -6x\\
	8x &= -2\\
	x &= -\frac{1}{4}
\end{align*}
\end{example}

\begin{example}{Logarithmic Base Manipulation}
\exProblem
Solve $\log_7{x} = \log_{49}{2x}$.

\exSolution
There are lot of transformation we could try at this point which would be valid.
Let us consider how we can make both sides have a logarithmic base of 49.
$\log_7{7} = \log_{49}{49}$, which shows that is you square the base, you must square
what you are taking the log of.  This will allow us to rewrite the equation and solve.
\begin{align*}
	\log_7{x} &= \log_{49}{2x}\\
	\log_{49}{x^2} &= \log_{49}{2x}\\
	x^2 &= 2x \\
	x^2 - 2x &= 0\\
	x(x-2) &= 0\\
	x &= \{0, 2\}
\end{align*}
If we check our two solutions, however, we find a contradiction.  $\log_b{0}$ does
not exist: there is no exponent you can raise a number to and get 0.  We can check that
2 works as a solution in our TI-8*.  (Newer TI-8*'s have a LOGBASE function under MATH,
but everyone can check by typing $\frac{\log{2}}{\log{7}}$ etc.)
\end{example}

\begin{example}{Combining Logs}
\exProblem
Solve for $x$: $\log_4{(x+10)}+\log_4{(x+34)}=4$

\exSolution
We can (must) combine the two logs, in order to make the problem simpler.
\begin{align*}
	\log_4{(x+10)(x+34)} &= 4\\
	\log_4{x^2+44x+340)} &= 4\\
	4^4 &= x^2+44x+340\\
	0 &= x^2+44x+340-256\\
	0 &= (x+2)(x+42)
\end{align*}
Of the two solutions, only -2 works in the original problem; -42 does not
\end{example}

\subsection{Substitution}
Finally, there are some problems which do not yield to combination, manipulation, or
changing log-form.  These problems require substitution.  For example, nothing in
$e^{2x}+5e^x=1$ seems to fit what we have so far described.  We cannot combine
any terms of the left, so we must ``explain away'' for a moment the troublesome
exponents.  We pick a variable to represent what we cannot deal with: $e^x$.

The problem now becomes $u^2+5u=1$.  This is not magic: we must un-substitute at the
end, or else we are solving a different problem.  But the magical aspect is that we can now
see that this is a quadratic problem.  $u^2+5u-1=0$ does not yield to factoring, so
we must resort to the Quadratic Formula.

$$
u=\frac{-5\pm\sqrt{29}}{2}
$$

We check with the TI-8*, and the ```plus'' solution is positive, while the ``minus'' one is not.
Since $u$ is really just a cipher for $e^x$, we need only the positive solution.  This means
$e^x=\frac{-5+\sqrt{29}}{2}$ or $x=\ln{-5+\sqrt{29}}-\ln{2}$, which is around -1.65.


\input{../../style/preamble}
\input{../../latex-math/basic-math}
\input{../../latex-math/basic-ml}

\newcommand{\titlefigure}{figure_man/ridge_hat.png}
\newcommand{\learninggoals}{
  \item Know the regularized linear model
  \item Know Ridge regression ($L2$ penalty)
  \item Know Lasso regression ($L1$ penalty)
}

\title{Introduction to Machine Learning}
\date{}

\begin{document}

\lecturechapter{Lasso and Ridge Regression}
\lecture{Introduction to Machine Learning}

\begin{vbframe}{Regularization in the linear model}

  \begin{itemize}
  \item Linear models can also overfit if we operate in a high-dimensional space with not that many observations.
  \item OLS usually require a full-rank design matrix.
  \item When features are highly correlated, the least-squares estimate becomes highly sensitive to random errors in the observed response, producing a large variance in the fit.
  \item We now add a complexity penalty to the loss:
  $$
  \riskrt = \sumin \left(\yi - \thetab^\top \xi \right)^2 + \lambda \cdot J(\thetab).
  $$
  \item Intuitive to measure model complexity as deviation from the 0-origin, as the 0-model is empty and contains no effects. Models close to this either have few active features or only weak effects.
  \item So we measure $J(\thetab)$ through a vector norm.
    This shrinks coefficients closer 0, hence the term \textbf{shrinkage methods}.
  \end{itemize}

\end{vbframe}

% \section{Ridge Regression}

\begin{vbframe}{Ridge Regression}

  \textbf{Ridge regression} uses a simple $L2$ penalty:
  \begin{eqnarray*}
  \thetah_{\text{Ridge}} &=& \argmin_{\thetab} \sumin \left(\yi - \thetab^T \xi \right)^2 + \lambda \|\thetab\|_2^2 \\
  &=& \argmin_{\thetab} \left(\yv - \Xmat \thetab\right)^\top \left(\yv - \Xmat \thetab\right) + \lambda \thetab^\top \thetab.
  \end{eqnarray*}

Optimization is possible (as in the normal LM) in analytical form:
$$\thetah_{\text{Ridge}} = ({\Xmat}^T \Xmat  + \lambda \id)^{-1} \Xmat^T\yv$$

Name comes from the fact that we add positive entries along the diagonal "ridge" $\Xmat^T \Xmat$.

\framebreak

We understand the geometry of these 2 mixed components in our regularized risk objective much better, if we formulate the optimization as a constrained problem (see this as Lagrange multipliers in reverse).

\vspace{-0.5cm}

\begin{eqnarray*}
\min_{\thetab} && \sumin \left(\yi - \fxit\right)^2 \\
  \text{s.t. } && \|\thetab\|_2^2  \leq t \\
\end{eqnarray*}

\vspace{-1.0cm}

\begin{figure}
\includegraphics[width=0.3\textwidth]{figure_man/ridge_hat.png}
\end{figure}

\begin{footnotesize}
NB: Relationship between $\lambda$ and $t$ will be explained later.
\end{footnotesize}

\framebreak

\begin{columns}
\begin{column}{0.5\textwidth}
\begin{figure}
\includegraphics[width=\textwidth]{figure_man/ridge_hat.png}
\end{figure}
\end{column}

\begin{column}{0.5\textwidth}
\begin{footnotesize}
\begin{itemize}
  \item We still optimize the $\risket$, but cannot leave a ball around the origin.
  \item $\risket$ grows monotonically if we move away from $\thetah$.
  \item Inside constraints perspective: From origin, jump from contour line to contour line (better) until you become infeasible, stop before.
\item Outside constraints perspective: From $\thetah$, jump from contour line to contour line (worse) until you become feasible, stop then.
  \item So our new optimum will lie on the boundary of that ball.
\end{itemize}
\end{footnotesize}
\end{column}
\end{columns}

\end{vbframe}

\begin{vbframe}{Example: Polynomial Ridge Regression}

True (unknown) function is \(f(x) = 5 + 2x +10x^2 - 2x^3 + \epsilon\) (in red).

\lz

Let us consider a \(d\)th-order polynomial
\[ f(x) = \theta_0 + \theta_1 x + \cdots + \theta_d x^d = \sum_{j = 0}^{d} \theta_j x^j\text{.} \]
Using model complexity $d = 10$ overfits:

\begin{center}
\includegraphics[width = 10cm ]{figure/poly_ridge_1.png} \\
\end{center}

\framebreak

With an $L2$ penalty we can now select $d$ "too large" but regularize our model by shrinking its coefficients. Otherwise we have to optimize over the discrete $d$.

\vfill

\begin{center}
\includegraphics[width = 11cm ]{figure/poly_ridge_2.png} \\
\end{center}

\begin{center}
\tiny
\begin{tabular}{ c| c c c c c c c c c c c c}
 $\lambda$ & $\beta_0$ & $\beta_1$ & $\beta_2$ & $\beta_3$ & $\beta_4$ & $\beta_5$ & $\beta_6$ & $\beta_7$ & $\beta_8$ & $\beta_9$ & $\beta_{10}$ \\
 \hline
 0.00 & 12.00 & -16.00 & 4.80 & 23.00 & -5.40 & -9.30 & 4.20 & 0.53 & -0.63 & 0.13 & -0.01 \\
 10.00 & 5.20 &1.30 & 3.70 & 0.69 & 1.90 & -2.00 & 0.47 & 0.20 & -0.14 & 0.03 & -0.00 \\
 100.00 & 1.70 & 0.46 & 1.80 & 0.25 & 1.80 & -0.94 & 0.34 & -0.01 & -0.06 & 0.02 & -0.00
\end{tabular}
\end{center}

\end{vbframe}

% \section{Lasso Regression}

\begin{vbframe}{Lasso Regression}

Another shrinkage method is the so-called \textbf{Lasso regression}, which uses an $L1$ penalty on $\thetab$:

\begin{eqnarray*}
\thetah_{\text{Lasso}} &=&  \argmin_{\thetab} \sumin \left(\yi - \thetab^T \xi\right)^2 + \lambda \|\thetab\|_1 \\
  &=& \argmin_{\thetab} \left(\yv - \Xmat \thetab\right)^\top \left(\yv - \Xmat \thetab\right) + \lambda \|\thetab\|_1.
\end{eqnarray*}

Note that optimization now becomes much harder. $\riskrt$ is still convex, but we have moved from an optimization problem with an analytical solution towards a non-differentiable problem.

\lz

Name: least absolute shrinkage and selection operator.

\framebreak

We can also rewrite this as a constrained optimization problem. The penalty results in the constrained region to look like a diamond shape.

\begin{eqnarray*}
\min_{\thetab} && \sumin \left(\yi - \fxit\right)^2\\
\text{subject to: } && \|\thetab\|_1 \leq t \\
\end{eqnarray*}

\vspace*{-1cm}

  \begin{figure}
\includegraphics[width=0.3\textwidth]{figure_man/lasso_hat.png}\\
\end{figure}

\end{vbframe}

\endlecture
\end{document}


%
% Chapter 3.3
%

\section*{3.3 Graphing Derivatives}

\subsection*{Increasing/Decreasing Test}

\begin{itemize}
    \item If \(f'(x)>0\) on an interval, then \(f\) is increasing on that interval.
    \item If \(f'(x)<0\) on an interval, then \(f\) is decreasing on that interval.
\end{itemize}

\subsection*{First Derivative Test}

\begin{itemize}
    \item If \(f'\) changes from positive to negative at \(c\), then \(f\) has a \textbf{local maximum} at \(c\).
    \item If \(f'\) changes from negative to positive at \(c\), then \(f\) has a \textbf{local minimum} at \(c\).
    \item If \(f'\) is positive to the left and right of \(c\), or negative to the left and right of \(c\), then \(f\) has no local maximum or minimum at \(c\).
\end{itemize}

\subsection*{Concavity Test}

\begin{itemize}
    \item If \(f''(x) > 0\) for all \(x\) in \(I\), then the graph of \(f\) is concave upward on
\(I\).
    \item If \(f''(x) < 0\) for all \(x\) in \(I\), then the graph of \(f\) is concave downward on \(I\).
\end{itemize}
If the graph of \(f\) lies above all of its tangents on an interval \(I\), then it is called concave upward on \(I\). If the graph of \(f\) lies below all of its tangents on \(I\), it is called concave downward on \(I\).
\\\\
A point \(P\) on a curve \(y=f(x)\) is called an \textbf{inflection point} if \(f\) is continuous there and the curve changes from concave upward to concave downward or vice versa.

\subsection*{Second Derivative Test}

Suppose \(f''(x)\) is continuous near \(c\).
\begin{itemize}
    \item If \(f'(c) = 0\) and \(f''(c) > 0\), then \(f\) has a local minimum at \(c\).
    \item If \(f'(c) = 0\) and \(f''(c) < 0\), then \(f\) has a local maximum at \(c\).
\end{itemize}


\subsection{Edgeworth box}

So now we have a framework for how agents can interact. We want to model trade, what games can do this?

To do this we introduce the Edgeworth box. This is a rectangle where the length of the \(x\) and \(y\) axes represent the total amount of those goods, and points on the box represent allocations of the goods. There is an initial endowment of goods.

\(x=\{0.2,0.8\}\)

\(y=\{0.8,0.2\}\)

[Put box here]

Their respective utility functions are:

\(u_a=f_a(x_a,y_a)\)

\(u_b=f_b(x_b,y_b)\)

In this example we use:

\(u_{a,b}=x^{0.5}y^{0.5}\)

We can add indifference curves to this box, which intercept at the endowment.

[Put box here]

The agents would be better off if they could trade so that they both had half of a unit of each good.

\(x=\{0.5,0.5\}\)

\(y=\{0.5,0.5\}\)

However they could also both be better off with

\(x=\{0.6,0.6\}\)

\(y=\{0.4,0.4\}\)

[Box here]

There are many such trades which could be made, which agents would rank differently.

There are also points where no further trade would be agreed by both parties. For example with the above outcome, any further trade would make at least one party worse off.

Such points are called Pareto efficient. That is, if a point is not Pareto efficient at least one party can be made better off without harming others.

[Box here]


\section{Artificial Neural Networks}
To understand neural networks, first it is necessary to understand its building blocks, artificial neurons. Commonly called just neurons, nodes, or units (the preferred term used in this document will be unit), these were historically inspired by biological neurons. The idea behind them is that a single unit is a very simple element that receives some inputs and produces an output, the real power comes from connecting many of these together, from thousands, to millions, or even billions of connections\footnote{
  As of writing this document GPT-3 is the biggest neural network model of all time, having 175 billion parameters \cite{gpt3_2020}.
}.
\begin{figure}[hbt]
    \centering
    \caption{Single unit representation}
    \includegraphics[width=0.4\textwidth]{chapters/NeuralNets/figures/neuron.pdf}
    \fonte{From the author (2021)}
    \label{fig:neuron}
\end{figure}

\autoref{fig:neuron} shows a representation of an artificial unit, it receives a set of inputs $[x_0, x_1, ..., x_n]$, denoted as a vector $\bm{x}$, and each input has a corresponding scalar value $w_i$ called its weight. Units will also often have a bias term $b$ that is independent of the inputs and that is useful for shifting the output.

By changing the set of weights (in vector form $\bm{w}$) and the bias term, it is possible to achieve different behaviours from the unit based on its inputs. \autoref{eq:weighted_input} shows how these parameters are used to calculate what is called the \textit{weighted input} ($z$) of the unit \cite[Chapter 2]{NN&DL2015}.
\begin{equation} \label{eq:weighted_input}
    z = \sum_{i}^{n}{w_i x_i} + b =
    %
    \begin{bmatrix}w_0, & w_1, & ... & w_n\end{bmatrix}
    \begin{bmatrix}x_0, & x_1, & ... & x_n\end{bmatrix}^T + b =
    %
    \bm{w} \cdot \bm{x}^T + b
\end{equation}

The weighted input is usually passed through a non-linear function $f$, called the \textit{activation function}, to produce the output $a$, called the unit activation, as seen in \autoref{eq:activation}.
\begin{equation} \label{eq:activation}
    a = f\left(z\right) = f\left(\bm{w} \cdot \bm{x}^T + b\right)
\end{equation}

A neural network is built by combining multiple units together and the learning process consists of adjusting all the weights and biases in order to produce the expected results given the dataset. Any combination of connections can be considered a neural network, however for the overwhelming majority of cases the networks are divided into layers, and for most of these cases the connections form an acyclic graph. This means that there are no cycles in the network, the input flows from one layer to the next, and there is usually no connection between layers that are not consecutive \textbf{--} exceptions to this are \gls{LSTM} networks \cite{lstm1997} and Residual Networks \cite{resnet2015}, but they are not the focus of this document.

The layers of the network are commonly divided into input layer, hidden layers, and output layer. The input layer represents the input data, usually in diagram representations the inputs are drawn like units, this however is just an stylistic choice since the values do not pass through any calculation in this layer.

The output layer contains the units that will be interpreted as the result produced from the input. For example, in a case where the network is trying to predict the price of apartments given area, number of rooms, and others properties (classical machine learning example), the output would be a single unit whose activation is the predicted price. Problems where the output can assume a range of values are usually called \textit{regression} problems.

On the other hand, problems where the output is better interpreted as a discrete value are called \textit{classification} problems. Predicting the digits of \gls{MNIST} falls into the category of classification problems, where the output represents which of the 10 possible digits the input image represents.

In the case of \gls{MNIST} the output layer can be made of 10 units, where each of the activations gives the probability of the input belonging to the corresponding digit. A natural question to raise from this description is: why would there be a need for one unit to represent each class, when the number 10 can be more efficiently represented using only 4 bits (4 units)? Or even, why not use just one unit that outputs the predicted number?

One reason for this is that these simpler representations would lose the property of interpretability of the output as a probability distribution. But the most important reason can be validated empirically, it is easier for a network to classify the inputs into isolated units then it is to try and correlate them with specific bits for each class \cite[Chapter 1]{NN&DL2015} or to reduce the output into a single unit.

For most classification problems using a neural network the output layer will contain one unit for each possible class and the desired output will be a vector where all elements are 0's, except for the element corresponding to the true label that will be a 1, indicating the 100\% probability for that class. This way of encoding the outputs is called \textit{one-hot encoding}.

The rest of the layers in a neural network, called the hidden layers, are all the layers between the input and output. A neural network does not need to have any hidden layers, but they are fundamental for building more complex relations and robust models.

\textcite{universalApproximator1989} showed that, given sufficient hidden units, any neural network with a single hidden layer can be used to approximate any function to any amount of precision, in other words, neural networks with at least a hidden layer are \textit{universal approximators}. But in practice it is observed that more hidden layers usually perform better, they are able to divide the problems into steps and gradually reach the result. For example, an image classifier might use the first layers to distinguish lower level features like edges, while deeper layers recognize shapes, textures, and all the way to complex patterns like faces \cite{deepLearningBook2016}.

More recent years have seen a resurgence of \gls{DL}, this is generally understood as learning with networks having at least two hidden layers, but modern models can have much more than that\footnote{
    Google's Inception v3 image classifier model has 42 layers in total \cite{inceptionV3_2015}
}.

\autoref{fig:network} shows an example of the layered structure of a neural network, for simplicity the diagram shows only fully connected layers (all the units of a layer are connected to all the units of the previous layer, see \autoref{subsub:fully_connected}) but it could also have different types of layer without losing the general idea of input, hidden, and output.
\begin{figure}[hbt]
    \centering
    \caption{Diagram of a fully connected neural network}
    \includegraphics[width=0.8\textwidth]{chapters/NeuralNets/figures/network.pdf}
    \fonte{From the author (2021)}
    \label{fig:network}
\end{figure}

\subsection{Types of layers}
The networks constructed for this document will all use a combination of fully connected and convolutional layers. This section will explain how these work, how they differ, and their advantages in relation to each other.

Before proceeding however, it is important to define the notation that will be used. Since from now on the focus will changes from single units to an entire network, it is necessary to establish a notation that allows for indexing individual units, in any layer, and also their parameters.

The bias, weighted input, and activation for unit $i$ in layer $l$ will be written as $b_{i}^{(l)}$, $z_{i}^{(l)}$, and $a_{i}^{(l)}$ respectively. The weight connecting unit $j$ in layer $l-1$, to unit $i$ in layer $l$ will be written as $w_{ij}^{(l)}$.

The set of biases, weighted inputs, and activations in layer $l$ can also be written as the vectors $\bm{b}^{(l)}$, $\bm{z}^{(l)}$ and $\bm{a}^{(l)}$. The weights connecting the units in layer $l-1$ to units in layer $l$ can be written as the weight matrix $\bm{W}^{(l)}$.
\begin{equation*}
    \renewcommand{\arraystretch}{1.4}
    \setlength\arraycolsep{4pt}
    \bm{z}^{(l)} = \begin{bmatrix}
        z_{0}^{(l)} \\
        z_{1}^{(l)} \\
        \vdots \\
        z_{n}^{(l)}
    \end{bmatrix}
    %
    \quad
    \bm{a}^{(l)} = \begin{bmatrix}
        a_{0}^{(l)} \\
        a_{1}^{(l)} \\
        \vdots \\
        a_{n}^{(l)}
    \end{bmatrix}
    %
    \quad
    \bm{b}^{(l)} = \begin{bmatrix}
        b_{0}^{(l)} \\
        b_{1}^{(l)} \\
        \vdots \\
        b_{n}^{(l)}
    \end{bmatrix}
    %
    \quad
    \bm{W}^{(l)} = \begin{bmatrix}
        w_{00}^{(l)} & w_{01}^{(l)} & \dots  & w_{0m}^{(l)} \\
        w_{10}^{(l)} & w_{11}^{(l)} & \dots  & w_{1m}^{(l)} \\
        \vdots       & \vdots       & \ddots & \vdots \\
        w_{n0}^{(l)} & w_{n1}^{(l)} & \dots  & w_{nm}^{(l)}
    \end{bmatrix}
\end{equation*}

Recall that the activation of a unit is simply its activation function, denoted here as $f$, applied to the weighted input. \autoref{eq:activation_again} rewrites this relation as shown in \autoref{eq:activation} using the revised notation.
\begin{equation} \label{eq:activation_again}
    \bm{a}^{(l)} = f\left( \bm{z}^{(l)} \right)
\end{equation}

Observe that in \autoref{eq:activation_again} the activation function is applied to the vector $\bm{z}^{(l)}$, this is a common shorthand notation to represent the element-wise application of the function to the vector. This notation will be used throughout the entirety of this document, all functions applied to vectors will be applied element-wise unless otherwise noted.

\subsubsection{Fully connected layer} \label{subsub:fully_connected}
Fully connected, also called Dense layers, are a type of layer where all the inputs are connected to all outputs of the previous layer. \autoref{eq:dense_weighted_input} shows how the weighted inputs of this layer are calculated from the activations of the previous layer.
\begin{equation} \label{eq:dense_weighted_input}
    z_{i}^{(l)} = \sum_{j}{w_{ij}^{(l)} a_{j}^{(l-1)}} + b_i^{(l)}
\end{equation}

If the previous layer is the input layer, the values for $\bm{a}^{(l-1)}$ will be the input of the network. To abstract the type of input it is useful to just replace the notation with a general layer input $\bm{x}$. The activation can be written more simply by using vector form as shown in \autoref{eq:dense_activation}, the layer superscripts were also omitted for more clarity.
\begin{equation} \label{eq:dense_activation}
    \bm{a} = f\left( \bm{W}\bm{x} + \bm{b} \right)
\end{equation}

Dense layers are extremely common, being used in many types of neural networks. They are very useful for mapping their inputs, that can have any number of dimensions, to a vector with any different number of dimensions, this is used in almost all classifiers to reduce the detected features into a one-hot encoded vector of the possible classes. For example, in the 2012 ImageNet challenge the three last layers of the network AlexNet were fully connected, they mapped the $43,264$ features into a $1000$ dimensional vector corresponding to all classes of images in the challenge \cite{alexnet2012}.

This type of layer can also be used to apply transformations to features that will be used later in the network, this is used in \acp{GAN} to map the latent space to a vector that is transformed into the generated image (see\autoref{cha:gans}). They can even be used for full feature extraction, but this is usually not the best choice since they are quite expensive given the high number of connections and, as will be seen, they lack some useful properties that are present in convolutional layers.

\subsubsection{Convolutional layer}
One drawback of fully connected layers is that they do not leverage the structure of the data when calculating the activations. Consider for example the case of images, when dealing with random values all pixels are uncorrelated with one another, but in real world situations the pixels of an image group together to make edges, shapes and complex figures. The same could be said for audio, video, language and many other situations \cite{guide_conv2018}.

Consider the example of a simple image of a digit in the \gls{MNIST} dataset, by translating the image by a couple of pixels or by slightly warping the strokes the digit represented does not change. But since the fully connected layer has no sense of neighbouring pixels (or temporal coherence for audio and video, etc.), then it has no choice other than learning weights for all possible slight transformations to the inputs. This not only makes learning more difficult, but also introduces many unnecessary parameters to the network.

Convolutional layers are a way of dealing with this problem. Initially inspired by the visual cortex of vertebrates \cite{neurocognitron1982}, the idea behind them is to detect the presence of features, no matter where they are present or if they are slight perturbed (e.g. an edge should be seen as an edge, no matter where it is located in the image or if it is slightly rotated).

Neural networks that employ convolutional layers are usually called \acp{CNN}, their history is very long and they were already used in the 1990's for learning the \gls{MNIST} dataset when it was introduced \cite{mnist1998}. However they were not very popular for larger problems and only grew in popularity after the great breakthrough in the ImageNet challenge achieved by the \gls{CNN} AlexNet in 2012 \cite{alexnet2012}. Since then they have become very common and are used in a variety of situations, even outside of image recognition.

This section will only explain how they work for the 2-dimensional case, but the idea can be easily generalized to the 1-dimensional or multidimensional cases. The inputs of a convolutional layer share the properties that they are represented as multidimensional arrays, have one or more axis where the ordering matters (for images these are the width and height), and can have an axis representing different views of the data (e.g. the RGB channels for a colored image) \cite{guide_conv2018}.

The name convolution is not a coincidence, the layer operation is related to the convolutions seen in signal processing for 1D discrete signals, the image convolutions like gaussian and Sobel filters, or higher dimensions mathematical convolutions. The operation consists of sliding a window of weights, called the kernel, over the entire input and calculating the sum of the weighted inputs covered by this window to produce the resulting values.
\begin{figure}[hbt]
    \centering
    \caption{Convolutional layer kernel properties}
    \includegraphics[width=0.55\textwidth]{chapters/NeuralNets/figures/cnn.pdf}
    \fonte{From the author (2021)}
    \label{fig:kernel_properties}
\end{figure}

It is simpler to understand this by looking at an image representation, \autoref{fig:kernel_properties} shows an example of a kernel being applied at the corner of an image with a single channel, whose values are $255$, and that is padded with zeros.

As seen in the figure, the kernel will start at the top left of the image (including some optional zero padding) and calculate a value for that position, then it will step a number of pixels, called the stride, and calculate the next value. When there is no more room to step horizontally, the kernel will start again at the left of the image and step a stride size downward, this is repeated until the whole image is traversed.

The convolution value between the kernel window and the pixels is simply the sum of the element wise products between the pixel values and the corresponding kernel weight. An image can be used again to help visualize this process, \autoref{fig:convolution} shows how a convolution is calculated for a $3\times3$ kernel, on a $3\times3$ image, with $1$ pixel zero padding, and stride of $1$ for both horizontal and vertical directions.
\begin{figure}[hbt]
    \centering
    \caption{Simple convolution process}
    \includegraphics[width=0.7\textwidth]{chapters/NeuralNets/figures/convolution.pdf}
    \fonte{From the author (2021)}
    \label{fig:convolution}
\end{figure}

Also note that the kernel does not change when calculating the outputs of the convolution, in other words, the weights of the kernel are shared between the units. These weight are learned instead of being predefined, this allows for the network to decide which features the kernels should detect in order to solve the problem; these could be for example edges, textures, shapes, specific colors or others.

One important detail to mention is that the examples shown until now only consist of single channel images. For colored images with 3 channels, and more general cases with $n$ channels, the kernel is not just a 2D matrix but is more accurately represented as a volume (i.e. one 2D kernel for each channel) . This means that in \autoref{fig:convolution} the kernel is a $3\times3\times1$ volume, if the input were a colored image it would be a $3\times3\times3$ volume. For the general case of an $n$ channel input, the volume would be $h\times w\times n$ for a kernel with height $h$ and width $w$.

The convolution for multiple channels is basically the same as for a single channel, each $h\times w$ slice of the volume is convoluted with one of the channels and the results are added together to get the final convolution. These resulting values can be considered as the weighted inputs $\bm{z}$ of the convolutional layer, so the activation function should be applied in order to get the final layer output.

In summary, for the case of images, a convolutional layer takes as input a 3D volume (i.e. two spatial dimensions to convolve with, and 1 channel dimension to give different views of the image) and operates on it using a kernel volume, the spatial sizes of the kernel are free to choose, but the number of channels must match. The convolution operation with a single kernel will reduce the input volume to a 2D feature map, where each point in the map indicates how much the feature is present at that region of the input covered by the kernel window. In general it is desirable to detect many different features from the input image, this means that convolutional layers will have multiple kernels and all the resulting convolutions will be stacked to produce an output volume.

The output will have as many channels as the number of kernels used in the convolutional layer, but the width and height will depend on the kernel size, stride, and padding. Consider sliding the kernel in the horizontal direction (the same logic applies to any direction and for all input dimensions), the number of values calculated by the convolution will be the number of possible positions that the kernel can be placed in this direction.

Suppose that in the horizontal direction the sizes of input, kernel, zero padding, and stride are $i$, $k$, $p$, and $s$ respectively. Then the output size $o$ in this direction is given by \autoref{eq:conv_output_size} \cite{guide_conv2018}, where $\lfloor{.}\rfloor$ is the floor function.
\begin{equation} \label{eq:conv_output_size}
    o = \left\lfloor{
        \frac{i + 2p - k}{s}
    }\right\rfloor + 1
\end{equation}

Convolutions can be used to enlarge the input image, but are most commonly used to reduce the dimensions while raising the number of features detected. For example, the GoogLeNet Inception architecture reduces the $224\times224\times3$ input image to a feature map of size $7\times7\times1024$ \cite{inceptionV1_2014}. This reduction is usually not done using only a single convolutional layer, it is most common for the input volume to pass through multiple convolutions that gradually reduce its width and height while increasing its depth. Looking at the Inception model again, the whole network is $22$ layers deep with most of those being convolutions for feature extraction \cite{inceptionV1_2014}.

According to \textcite[chap. 9]{deepLearningBook2016}, convolutional layers leverage the ideas of sparse interactions, shared parameters, and equivariant representations to improve a machine learning system. They describe sparse interactions in the sense that kernels are smaller than the input, reducing the number of parameters, the memory used, the processing cost, and improving statistical efficiency. And the idea of equivariant representations is related with the invariance to translation, since at least in principle, the use of a sliding window allows for detecting the same feature no matter where it might appear on the image.

\subsubsection{Transposed Convolutions}
Any convolution operation can be converted to a matrix multiplication where the input is flattened to a 1-dimensional vector and the kernel is converted to a sparse matrix $\bm{C}$. The forward pass through the network, which is when the convolution is applied, is equivalent to multiplying the flattened input by $\bm{C}$, and the backward pass is equivalent with multiplying by $\bm{C}^T$ \cite{guide_conv2018}. The multiplication with $\bm{C}^T$ converts the output volume to the input volume and is commonly called the transposed convolution, or sometimes \textit{deconvolution}.

Any kernel represents both a convolution or transposed convolution, it all depends on how the values are interpreted as a matrix. It is important to note that the transposed convolution is not a way to reverse the convolution step, it is generally not possible to calculate the input of a convolution based on the kernel and output values. But the transpose operation can be considered as a reverse in the sense of transforming the output volume into the input volume.

Given the property of producing the opposite volume in relation to convolutions, transposed convolution layers are commonly used to upscale data to higher dimensions. For example, they are used in \acp{GAN} to upscale the latent space vector into the corresponding image (see\autoref{cha:gans}) \cite{nipsGAN2017}. However, upscaling with this method has been shown to produce image artifacts \cite{deconvolutionArtifacts2016} and some models, like styleGAN \cite{styleGAN2018}, already drop the use of transposed convolution layers for different upscaling approaches.

\subsubsection{Pooling layer}
Pooling layers are very similar to convolutional layers in the sense that they both slide an window of some width and height through the image, using some stride value, optional padding, and producing a number for each possible position of the window. But pooling layers do not have any learnable parameters and just execute a predefined function over all the inputs inside the sliding window to produce the output.

The two most common types of pooling are max and average pooling. Max pooling, as the name suggests, simply returns the maximum value present in the sliding window, while average pooling returns the average of those values. Pooling layers are very commonly used together with convolutional layers, \textcite[p. 355-336]{deepLearningBook2016} describe that a usual convolutional layer in a \gls{CNN} consists of a convolution operation, followed by the nonlinear activation, and lastly by a pooling layer; they say that this operation helps to make the representation approximately invariant to input translations.

\subsubsection{Embedding Layer}
This layer is an alternative way of representing discrete valued inputs. Recall that one way to represent a discrete value (e.g. the class of a given input) is to use one-hot encoding, this allows for mapping $n$ possible values into a $n$ dimensional vector of all 0's and a single 1 representing the value.

Embedding layers offer a way to map $n$ values into a vector with $m$ dimensions, where $m$ can be any number. The mapping from value to vector in an embedding layer is not something fixed, but is also learned during training. This type of layer is very useful in language models, where encoding thousands of possible words into one-hot vectors is infeasible, embeddings allow for much smaller representations that can be learned by the model to best fit a given problem.

When conditioning \acp{GAN} with class information for the \glsunset{CGAN} \gls{CGAN} variant (see \autoref{sub:cgan}), it is necessary to combine the label of the data together with the input. Embedding layers offer a more robust solution to this when compared to one-hot vectors, because of that they were used for conditioning in the experiments seen in\autoref{cha:experiments}.

\subsection{Activation Functions}
Historically one of the first implementations of artificial neural networks used the step function as activation for the units \cite[Chapter 1]{NN&DL2015}, this means that for positive inputs the step function produces a $1$ and for all other values it produces a $0$. It also could use the sign function \cite{thePerceptron2017}, producing a $-1$ for non positive inputs.

This type of approach is called a Perceptron, one notable example of its implementation was the MARK I Perceptron, a hardware solution where all weights were regulated by potentiometers and automatically adjusted by motors to train the network \cite{perceptron1960}. However it was later shown that this type of binary activation was very limited and could only solve linear separable problems \cite{thePerceptron2017}.

One may question the need for activation functions or why do they need to be nonlinear, what is the reason for introducing nonlinearity to the network? To understand this, first note that not using an activation function is the same as setting the output $y = x$, this is also a linear relationship between the values, so it is only necessary to explain why a linear activation is a problem.

The nonlinearity is introduced to make the network able to learn more complex relationships in the data, since not all real world situations have a linear dependence between input and output. By transforming the input through multiple nonlinear activations it is possible to create very elaborate mappings from input to output. This does not hold true for linear transformations, since applying a sequence of them to some input is equivalent to applying just a single one.

To see this consider an input vector $\bm{x}$ with two transformations matrices $\bm{A}_1$, $\bm{A}_2$ and vectors $\bm{b}_1$, $\bm{b}_2$. The output $\bm{y}_2$ is obtained by applying two linear transformations to the input vector as follows.
\begin{equation} \label{eq:linearity_1}
    \bm{y}_1 = \bm{A}_1 \bm{x} + \bm{b}_1
    \qquad \text{and} \qquad
    \bm{y}_2 = \bm{A}_2 \bm{y}_1 + \bm{b}_2 \\
\end{equation}

By rearranging the terms it can be confirmed that the two linear transformations are equivalent to a single one, this can be seen by the following derivation.
\begin{align*}
    \bm{y}_2& = \bm{A}_2 (\bm{A}_1 \bm{x} + \bm{b}_1) + \bm{b}_2 \\
    & = \bm{A}_2 \bm{A}_1 \bm{x} +  \bm{A}_2 \bm{b}_1 + \bm{b}_2 \\
    & = \bm{A} \bm{x} + \bm{b}
\end{align*}

This logic holds true for any number of linear transformations. Now it should be hopefully clear to see that there is inherently no difference between a multilayered neural network with linear activations and a simple two layered input-output network. It is necessary to introduce nonlinearities in the hidden layers in order to build more complex models \textbf{--} linear transformations can however be used in the output layer to map the values to a more desirable range, regression problems are an example.

Although nonlinear, the binary property of the step function limits the capabilities of a neural network, to work around this limitations it is necessary to introduce a different type of activation. To avoid the problem of jumping values it is best to have a continuous function instead of a discrete one, it is also important that this function be differentiable in its domain and that the derivatives are not zero everywhere (see \glsunset{ReLU} \gls{ReLU} activation in \autoref{subsub:relu} for more remarks about this restrictions). The derivative requirements are necessary to make possible the use of the learning algorithm Gradient Descent with Backpropagation (see sections \ref{sec:loss_&_gradient_descent} and \ref{sec:backpropagation}).

This section will briefly introduce some important activation functions that are widely used in multiple machine learning problems and that were used in the experiments found in\autoref{cha:experiments}.

\subsubsection{Sigmoid} \label{subsub:sigmoid}
Usually denoted by \gls{sigmoid}, the sigmoid function maps all real numbers to the interval $(0, 1)$. For a given input $x$, the value for $\sigma(x)$ is given by \autoref{eq:sigmoid}.
\begin{equation} \label{eq:sigmoid}
    \gls{sigmoid}(x) = \frac{1}{1 + e ^ {-x}}
\end{equation}

The sigmoid function can be considered the continuous version of the step function, as the absolute value of $x$ grows, $\sigma(x)$ gets exponentially closer to $\mathsf{step}(x)$, but there is a continuous transition close to $0$. This can be seen more clearly in\autoref{fig:activations}.
\begin{figure}[hbt]
    \centering
    \caption{Curves of sigmoid, tanh and ReLU activation functions}
    \includegraphics[width=0.6\textwidth]{chapters/NeuralNets/figures/Activations.pdf}
    \fonte{From the author(2021)}
    \label{fig:activations}
\end{figure}

This function can be useful to convert a single output to a probability value or to normalize a set of outputs. However, the fact that the outputs are always positive raises some problems, \textcite{efficientBackprop2012} showed that in these situations the backpropagation algorithm must update all the network parameters in the same direction, this means that the parameters are not free to wander the parameter space in the best direction.

Most of the time the network will benefit more by adding to some parameters while subtracting from others, the restriction to always update in the same direction makes learning more difficult and can greatly reduce the speed of convergence. \textcite{efficientBackprop2012} also argue that any deviation in the average of the outputs will bias the update direction, so it is better to have activations that are zero centered.

Knowing this problem, it only makes sense to consider sigmoid activations in the output layer, since for most cases it is better to use a zero centered alternative like the \gls{tanh} in the hidden layers.

\subsubsection{Hyperbolic Tangent}
The hyperbolic tangent function, as seen in \autoref{eq:tanh}, is a zero centered, scaled, and shifted version of the sigmoid function.
\begin{equation} \label{eq:tanh}
    \tanh(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}} = 2\sigma(2x) - 1
\end{equation}

This function is almost always better than sigmoid since it has the same shape and properties without having the disadvantage of not being zero centered. It can be used in any layer, including the output. The times were a sigmoid would be preferred are when it is desired that the output be bounded to $(0,1)$, as is the case for a probability value.

\subsubsection{Rectified Linear Unit} \label{subsub:relu}
\glsreset{ReLU}
Both the sigmoid and hyperbolic tangent functions have two characteristics that can raise some problems when training a machine learning model. The first one is that they can only produce very close approximations of the number zero, but not the exact value \textbf{--} the only exception would be when the input of the \gls{tanh} function is also exactly zero, but this is very unlikely to happen and would only work for very specific inputs.

The lack of a true zero can be undesirable when the goal of the network is to build a sparse representation of the data, that is, a representation that only depends on a small number of inputs that strongly correlate to the output. This is in contrast with a model that depends on many inputs, but most of them have very little impact on the output.

The sparsity argument not only has some biological support, but has been shown to also positively influence the quality of a model \cite{relu2011}. Promoting sparsity is found in many other areas of science (e.g. statistical modeling, image compression) and it is very useful for producing simpler representations of the data \cite{dataDrivenScience2019}.

The other problem present in the sigmoid and \gls{tanh} functions is that they saturate for high absolute values of the input. Most of the variation in these functions occur close to zero, but for larger values there is barely any difference between outputs. For example, the difference between the \gls{tanh} activation between inputs $1$ and $2$ is around $0.2$, while the difference from $2$ to $100$ is just $0.036$. One other way to say this is that the derivatives of these functions are very close to zero for inputs far from the origin (see \autoref{fig:derivatives}), this can give rise to the vanishing gradients problem and make learning extremely slow (see \autoref{sub:vanishing_gradients}).
%
\begin{figure}[hbt]
    \centering
    \caption{Derivative of sigmoid, tanh and ReLU activation functions}
    \includegraphics[width=0.6\textwidth]{chapters/NeuralNets/figures/Derivatives.pdf}
    \fonte{From the author(2021)}
    \label{fig:derivatives}
\end{figure}

The \gls{ReLU} activation function is an alternative that addresses both of the mentioned problems with sigmoid and \gls{tanh}. It is constructed from combining different linear regions (called a piecewise linear function), this allows for the activation to inherit many desirable properties of linear transformations while still retaining nonlinearity and being able to build complex relations \cite{deepLearningBook2016}.

The \gls{ReLU} function is simply composed of a constant zero for all negative inputs and the identity function for everything else, this means that for an input $x$ the activation is calculated as shown in \autoref{eq:relu}.
\begin{equation} \label{eq:relu}
    \ReLU(x) = \max(0, x)
\end{equation}

This definition allows for \gls{ReLU} to produce exact zeros for any negative inputs, promoting sparsity in the model representations. The constant positive derivative (see \autoref{fig:derivatives}) also removes the problem of vanishing gradients, since the units never saturate. Another big advantage of \gls{ReLU} is that it is extremely easy to compute, a simple \texttt{if} statement is enough to get the result; compared with the need to calculate exponentials in the sigmoid and \gls{tanh} functions, \gls{ReLU} performance is much faster.

Since around 2010, with papers like \cite{relu2011} exploring the \gls{ReLU} activation, there were many popular methods that showed impressive results using this function (e.g. \cite{alexnet2012} and \cite{inceptionV3_2015}). At the current time \gls{ReLU} is the standard recommended activation function to be used in most problems \cite{deepLearningBook2016}.

There are however some downsides to this activation. First there is the sharp change in the function behavior at the value $0$, making its derivative undefined at that point, this however does not seem to be a problem in practice \cite{relu2011}. \gls{ReLU} also loses the desirable zero centered property that made \gls{tanh} a good substitute for the sigmoid, the argument from \textcite{efficientBackprop2012} holds for all activations, this means that all the network updates for units that use \gls{ReLU} must be made in the same direction, making learning more difficult.

Lastly, for a unit to be able to learn it is necessary that it outputs a value in the range where the activation function has a non-zero derivative for at least one example in the training data. But it is possible that for all the training data in a given problem, there exists some units using \gls{ReLU} activations that will always output zero, this makes learning impossible in these units and effectively freezes them on their state forever.

There are many other alternatives proposed to address these problems, some relevant examples are Maxout, LeakyReLU, ELU, GELU and PReLU, these have found some success in big machine learning problems\footnote{
    GELU is used in natural language processing models like GPT-3 \cite{gpt3_2020}
} and the benchmark by \cite{CaffeNetBench2017} showed good results for some of these alternatives \textbf{--} although \gls{ReLU} also fared well in those tests.

Between these alternatives, LeakyReLU is specially relevant for \acp{GAN}, being an important piece of the \glsunset{DCGAN}\gls{DCGAN} variant that is the base of many \gls{GAN} architectures. It consists of a simple change from ReLU that just guarantees that the units will always have at least some positive derivative. This activation is calculated as shown in \autoref{eq:leaky_relu}, where $\alpha$ is a hyperparameter that must be a small value (usually $0.3$).
\begin{equation} \label{eq:leaky_relu}
    \LeakyReLU(x) = \max(\alpha x, x)
\end{equation}

\subsubsection{Softmax}
In classification problems it is very common to have the output layer encode the input class has a $n$ dimensional, one-hot encoded vector, where $n$ is the number of classes. Since in one-hot encoding each element corresponds to the probability of the input belonging to the corresponding class (all zeros and a single $1$ for the correct class), it is desirable for the output of a classification model to also be a probability distribution. Besides the fact that it aligns with the encoding representation, it also gives a meaningful representation for the output of the network, by observing the output probabilities it is possible to have an idea of how confident the network is in the results\footnote{
    Since the values in a probability distribution should always add to $100\%$ this may not always give good representations, this is the case for adversarial examples that can fool the network to misclassify images with a high degree of confidence \cite{adversarialExamples2013}.
}.

The softmax activation function offers a way to map all the units weighted inputs to a probability distribution. One difference when compared to the other activation functions is that the softmax does not take a single number as input, instead it uses all values in the layer for calculating the unit activation. This makes sense since the probabilities are dependent on the proportions of each unit weighted input. \autoref{eq:softmax} shows how the activation for the unit $i$ is calculated based on the layer weighted inputs $(z_0 \dots z_n)$, the layer superscripts were omitted for clarity.
\begin{equation} \label{eq:softmax}
    a_i = \frac{e^{z_i}}{ \sum\limits_{k=0}^{n}{e^{z_k}} }
\end{equation}