\newcommand{\R}{\mathbb{R}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\F}{\mathbb{F}}
\newcommand{\rcl}[2]{{{[#1]}_{#2}}}
\newcommand{\modulo}[3]{#1\equiv #3(\bmod\, #2)}
\newcommand\inv[1]{{{#1}^{-1}}}
\newcommand\ftwox{{ \F_2[x]}}
\newcommand\ftwoxpx{{ \F_2[x]/p(x)}}
\newcommand\ftwoeight{{ \F_{2^8}}}
\newcommand\cto{\longrightarrow}
\newcommand\EA{\mathbf{E_A}}
\newcommand\EB{\mathbf{E_B}}
\newcommand\EK{\mathbf{E_K}}
\newcommand\EM{\mathbf{E_M}}

\newcommand\Zpstar{{ \Z_p^*}}
\newcommand\Znstar{{ \Z_n^*}}
\newcommand\xor{{\oplus}}
\newcommand{\set}[1]{\{#1\}}

\def\EXN{3}
\def\DATE{\Week\EXN}

%%%\ifHandout

\ifText \NoHead \summary{Block Ciphers continued --- AES (Rijndael) --- AACS --- Finite
  Fields --- Revision: Matrix Arithmetic}

\subsubsection{AES (Rijndael)}

\myglos{AES}{The advanced encryption standard that is the successor of DES.}
\myglos{Rijndael}{The original name of AES.}  The \emph{advanced encryption
  standard} (AES) is the successor of the outdated DES. It was developed by the
two Belgian cryptographers Joan Daemen and Vincent Rijmen. It was originally
named \emph{Rijndael}, but renamed to AES when it was adopted as official US
standard in November 2001. Rijndael is a block cipher, however, it does not rely
on the basic design of the Feistel cipher. In particular is has distinct
encryption and decryption algorithms. But similar to DES, it is round based and
relies on a combination of substitutions, permutations and key addition. AES is
motivated by arithmetic operations in the field $\F_{2^8}$, and its
implementation in hardware and software is compact and fast.

Rijndael is parameterisable in that it can work with:
\begin{itemize}
\item block sizes of 128, 192, and 256 bits,
\item key sizes of 128, 192, and 256 bits, and
\item 10, 12, or 14 rounds of encryption.
\end{itemize}

Rijndael performs encryption, decryption and computes the key schedule using
arithmetic in $\F_{2^8}$ with respect to the irreducible polynomial
$p(x)=x^8+x^4+x^3+x+1$. That is, $\F_{2^8}=\F_{2}[x]/(x^8+x^4+x^3+x+1)$.

\myitem{Operation of AES}

We will discuss the basic operations of Rijndael for the case of $128$ bit block
and key size and $10$ rounds of encryption. Rijndael arranges both message and
key in $4\times4$ matrices of $8$-bit elements, i.e., each element is exactly
one byte and each column and each row contain $32$-bit words.

If $m=m_0\|m_1\|\ldots\|m_{15}$ is the message, then Rijndael initialises the so
called \emph{state matrix} $A$ as follows:
\[
\begin{bmatrix}
  m_0 & m_4 & m_8 & m_{12}\\
  m_1 & m_5 & m_9 & m_{13}\\
  m_2 & m_6 & m_{10} & m_{14}\\
  m_3 & m_7 & m_{11} & m_{15}\\
\end{bmatrix}\longrightarrow
A=
\begin{bmatrix}
  a_{0,0} & a_{0,1} & a_{0,2} & a_{0,3}\\
  a_{1,0} & a_{1,1} & a_{1,2} & a_{1,3}\\
  a_{2,0} & a_{2,1} & a_{2,2} & a_{2,3}\\
  a_{3,0} & a_{3,1} & a_{3,2} & a_{3,3}\\
\end{bmatrix}
\]

Each round applies the following manipulations to the state matrix:
\begin{enumerate}
\item a substitution operation on every single byte \textit{SubBytes},
\item a byte permutation \textit{ShiftRows},
\item a column manipulation \textit{MixColumns},
\item and an xor of the state with the round key \textit{AddRoundKey}.
\end{enumerate}
An exception is the last round, where the \textit{MixColumns} operation is
skipped.

For each of these operations, except for \textit{AddRoundKey}, we need
corresponding inverse operations for the decryption. Here is an overview of the
encryption and decryption algorithms:

\begin{minipage}{.5\textwidth}
  $E_K(M)$
  \begin{tabbing}
    $A := M$\\
    $A :=$ AddRoundKey($A$, $K_0$)\\
    \textbf{for} \=$i$ = 1 \textbf{to} 9 \textbf{do}\\
    \>$A :=$ SubBytes($A$)\\
    \>$A :=$ ShiftRows($A$)\\
    \>$A :=$ MixColumns($A$)\\
    \>$A :=$ AddRoundKey($A$, $K_i$)\\
    \textbf{end}\\
    $A :=$ SubBytes($A$)\\
    $A :=$ ShiftRows($A$)\\
    $A :=$ AddRoundKey($A$, $K_{10}$)\\
    $C := A$
\end{tabbing}
\end{minipage}
\begin{minipage}{.5\textwidth}
$D_K(C)$
  \begin{tabbing}
    $A := C$\\
    $A :=$ AddRoundKey($A$, $K_{10}$)\\
    $A :=$ InverseShiftRows($A$)\\
    $A :=$ InverseSubBytes($A$)\\
    \textbf{for} \=$i$ = 9 \textbf{downto} 1 \textbf{do}\\
    \>$A :=$ AddRoundKey($A$, $K_i$)\\
    \>$A :=$ InverseMixColumns($A$)\\
    \>$A :=$ InverseShiftRows($A$)\\
    \>$A :=$ InverseSubBytes($A$)\\
    \textbf{end}\\
    $A :=$ AddRoundKey($A$, $K_0$)\\
    $M := A$
  \end{tabbing}
\end{minipage}

Here are the single operations in detail.

\textit{1. SubBytes:} \myglos{SubBytes}{An S-Box like substitution operation
  used during AES encryption.}  This operation is similar to the S-Box
substitution of DES. Each byte $a_{i,j}$ of the state is substituted by the
output of a single S-Box.  This S-Box corresponds to an algebraic operation in
Rijndael's finite field $\ftwoeight$. Each byte $a_{i,j}=[z_7,\ldots,z_0]$ is
considered as a polynomial in $\ftwoeight$. Its substitution is then computed in
two steps:
\begin{enumerate}
\item First we compute the multiplicative inverse of $a_{i,j}$ in $\ftwoeight$
  to get $\inv{a_{i,j}}=[x_7,\ldots, x_0]$. (The zero element is mapped to
  $[0,\ldots,0]$.)
\item We then compute a new bit vector $b_{i,j}=[y_7,\ldots,y_0]$ with the
  following transformation in $\F_2$ (observe that the vector addition is the
  same as an xor $\xor$):
  \[\begin{bmatrix}y_0\\y_1\\y_2\\y_3\\y_4\\y_5\\y_6\\y_7\end{bmatrix} =
  \begin{bmatrix} 1&0&0&0&1&1&1&1 \\ 1&1&0&0&0&1&1&1 \\ 1&1&1&0&0&0&1&1 \\
    1&1&1&1&0&0&0&1 \\ 1&1&1&1&1&0&0&0 \\ 0&1&1&1&1&1&0&0 \\ 0&0&1&1&1&1&1&0 \\
    0&0&0&1&1&1&1&1\end{bmatrix}\cdot
  \begin{bmatrix}x_0\\x_1\\x_2\\x_3\\x_4\\x_5\\x_6\\x_7\end{bmatrix} +
  \begin{bmatrix}0\\1\\1\\0\\0\\0\\1\\1\end{bmatrix} \]
\end{enumerate}
The substitution can be schematically displayed as:

% [[ 1,0,0,0,1,1,1,1],[1,1,0,0,0,1,1,1],[1,1,1,0,0,0,1,1],[
%   1,1,1,1,0,0,0,1],[1,1,1,1,1,0,0,0],[0,1,1,1,1,1,0,0],[0,0,1,1,1,1,1,0],[
%   0,0,0,1,1,1,1,1]]

\begin{center}
  \includegraphics[height=5cm]{AES-SubBytes}
  \rput[b]{90}(0,2){\tiny Source: Wikipedia}%, \url{http://en.wikipedia.org/wiki/Feistel_cipher}}
\end{center}

For the decryption algorithm we can define an inverse operation
\textit{InverseSubBytes} by first reversing the transformation:
\myglos{InverseSubBytes}{An S-Box like substitution algorithm used during AES
  decryption. The inverse of SubBytes.}

\[\begin{bmatrix}x_0\\x_1\\x_2\\x_3\\x_4\\x_5\\x_6\\x_7\end{bmatrix} =
\begin{bmatrix} 0&0&1&0&0&1&0&1\\ 1&0&0&1&0&0&1&0\\ 0&1&0&0&1&
0&0&1\\ 1&0&1&0&0&1&0&0\\ 0&1&0&1&0&0&1&0\\ 0&0&1&0&1&0&0&1\\
1&0&0&1&0&1&0&0\\ 0&1&0&0&1&0&1&0
\end{bmatrix}
\left(\begin{bmatrix}y_0\\y_1\\y_2\\y_3\\y_4\\y_5\\y_6\\y_7\end{bmatrix} +
  \begin{bmatrix}0\\1\\1\\0\\0\\0\\1\\1\end{bmatrix}\right) \] and then
computing the multiplicative inverse of $[x_7,\ldots, x_0]$ in $\ftwoeight$.

\textit{2. ShiftRows:} This operation performs a cyclic shift on the state
matrix by shifting each row separately.  This ensures that the columns of the
state matrix interact over several rounds of encryption. The lower three rows
are shifted by one, two, and three positions, respectively: \myglos{ShiftRows}{A
  byte permutation used during AES encryption.}  \myglos{InverseShiftRows}{A
  byte permutation used during AES decryption. Inverse of Shift Rows.}
\begin{center}
  \includegraphics[width=.65\textwidth]{AES-ShiftRows}
  \rput[b]{90}(0,2){\tiny Source: Wikipedia}%, \url{http://en.wikipedia.org/wiki/Feistel_cipher}}
\end{center}
  For the decryption the \textit{InverseShiftRows}
does the reverse shift.%\pagebreak

\textit{3. MixColumns:} \myglos{MixColumns}{A column manipulation operations
  used during AES encryption.}  \myglos{InverseMixColumns}{A column manipulation
  operations used during AES decryption. Inverse of MixColumns.}  This operation
ensure interaction of the rows of the state matrix by mixing each column
separately. For this it performs the following matrix multiplication for each
column $i=1,2,3,4$ over $\ftwoeight$, where the entries of the matrix are
hexadecimal representations of polynomials of degree $7$ (e.g. $0x03$ corresponds
to $x+1$.)
\[\begin{bmatrix}b_{0,i}\\b_{1,i}\\b_{2,i}\\b_{3,i}\end{bmatrix}=
\begin{bmatrix} 0x02&0x03&0x01&0x01 \\ 0x01&0x02&0x03&0x01 \\
  0x01&0x01&0x02&0x03 \\ 0x03&0x01&0x01&0x02 \end{bmatrix}\cdot
\begin{bmatrix}a_{0,i}\\a_{1,i}\\a_{2,i}\\a_{3,i}\end{bmatrix}\] This operation
actually corresponds to a polynomial multiplication in $\ftwoeight[x]/(x^4+1)$,
the ring of polynomials whose coefficient are in $\ftwoeight$, i.e., they are
polynomials themselves. Observe that $x^4+1$ is not irreducible in $\ftwoeight$
and therefore $\ftwoeight[x]/(x^4+1)$ is indeed a ring not a field.

\[a(x)\cdot c(x)=(a_3x^3+a_2x^2+a_1x+a_0)\cdot (0x03x^3+0x01x^2+0x01x+0x02)
({\bmod}\; x^4+1)\] We can picture
\textit{MixColumns} as
\begin{center}
  \includegraphics[width=.6\textwidth]{AES-MixColumns}
  \rput[b]{90}(0,2){\tiny Source: Wikipedia}%, \url{http://en.wikipedia.org/wiki/Feistel_cipher}}
\end{center}
The inverse operation \textit{InverseMixColumns} is then simply the
multiplication with $\inv{c(x)}$, which has an inverse in the ring
$\ftwoeight[x]/(x^4+1)$.

\textit{4. AddRoundKey:} \myglos{AddRoundKey}{The round key addition operation
  used during AES encryption and decryption.}  As mentioned earlier, the key
size of AES is $128$ bits. All the round keys $K_0,\ldots,K_{10}$ derived from
the key $K$ are also $128$ bits and can therefore be expressed as a $4\times4$
matrix:
\[
K_i=
\begin{bmatrix}
  k_{0,0} & k_{0,1} & k_{0,2} & k_{0,3}\\
  k_{1,0} & k_{1,1} & k_{1,2} & k_{1,3}\\
  k_{2,0} & k_{2,1} & k_{2,2} & k_{2,3}\\
  k_{3,0} & k_{3,1} & k_{3,2} & k_{3,3}\\
\end{bmatrix}
\]
Adding the round key is simply the xor-ing of the state matrix and the round key
matrix byte by byte: $A\xor K_i$. The inverse of this operation is obviously the
same and we do not need a special operation for the decryption.

\begin{center}
  \includegraphics[width=.5\textwidth]{AES-AddRoundKey}
  \rput[b]{90}(0,2){\tiny Source: Wikipedia}%, \url{http://en.wikipedia.org/wiki/Feistel_cipher}}
\end{center}

\myitem{Key Schedule of AES}

The round keys $K_0,\ldots,K_{10}$ are derived from the key $K$. Since $K$ is
$128$ bits we can divide it into 4 words of $32$ bits each:
$K=W_0\|W_1\|W_2\|W_3$, which also corresponds to round key $K_0$. All
subsequent round keys $K_i=W_{4i}\|W_{4i+1}\|W_{4i+2}\|W_{4i+3}$ are then
computed using an $8$ bit left rotation, an S-Box substitution with the
\textit{SubBytes} function, and a scrambling sequence, which is started with a
round constant $RC_i$ that is computed in $\ftwoeight$ by \[{RC_i}=x^i
({\bmod}\; x^8+x^4+x^3+x+1).\] The algorithm then looks like this:
\begin{center}
  \begin{minipage}{1.0\linewidth}
    \begin{tabbing}
      KeySchedule($K$)\\[.2cm]
      $W_0\|W_1\|W_2\|W_3 := K$\\
      \textbf{for} \=$i$ := 1 \textbf{to} 10 \textbf{do}\\
      \>$T := W_{4i-1}\lll 8$\\
      \>$T :=$ SubBytes($T$)\\
      \>$T := T \xor RC_i$\\
      \>$W_{4i} := W_{4i-4} \xor T$\\
      \>$W_{4i+1} := W_{4i-3} \xor W_{4i}$\\
      \>$W_{4i+2} := W_{4i-2} \xor W_{4i+1}$\\
      \>$W_{4i+3} := W_{4i-1} \xor W_{4i+2}$\\
      \textbf{end}
    \end{tabbing}
  \end{minipage}
\end{center}

Note that \textit{SubBytes} is applied to the four $8$ bit bytes of the $32$ bit
word $T$ individually.

\newpage
\subsubsection{AACS --- Advanced Access Content System}

The Advanced Access Content System (AACS) is a Digital Rights Management system
for HD-DVD and Blu-Ray Discs developed by a consortium that includes Disney,
Intel, Microsoft, Matsushita (Panasonic), Warner Brothers, IBM, Toshiba, and
Sony. The general idea of Digital Rights Management (DRM) is to restrict access
control to electronic media and playback devices to retain full control by the
copyright owner. We first give a brief overview of AACS and then dicuss some of
the issues arising from AACS and DRM in general.

\myglos{DRM}{Short for Digital Rights Management}
\myglos{Digital Rights Management}{Access control technologies that limit usage of digital media or devices.}
\myglos{AACS}{Short for Advanced Access Content System}
\myglos{Advanced Access Content System}{A Digital Rights Management system for HD-DVDs and Blue-Ray Discs}

\myitem{Overview} AACS uses encryption, hash functions, and watermarking schemes
based on AES. Exact specifications for parts of the technology are published by
the AACS licensing authority at \url{http://www.aacsla.com}. Below is a
schematic overview of the mechanism of AACS.

\begin{minipage}{.5\linewidth}
  The content of a disc is encrypted with AES using a collection of \emph{title
    keys}. The encrypted content, together with the encrypted title keys, a
  \emph{media key block (MKB)}, and the \emph{unique disc ID} are all stored on
  the disc.

  An AACS encrypted disc can only be accessed with a fully licensed player. Each
  licensed player gets a unique set of \emph{device keys} as well as a unique
  \emph{Host Private Key}. The latter makes it possible that particular playback
  devices or programmes can be individually revoked.

  The device keys are used to compute a key from the MKB. The host private key
  is needed to retrieve the disc ID. Both values are then processed by an
  AES-based one-way function, AES-G.  (One-way functions are functions easy to
  compute, but infeasible to invert. We will discuss one-way functions in more
  detail later in the lecture.)
\end{minipage}
\begin{minipage}{.5\linewidth}
  \begin{center}
    \includegraphics[height=9cm]{AACS-Simple}
    \rput[b]{90}(0,2){\tiny Source:
      arstechnica.com}%, \url{http://en.wikipedia.org/wiki/Feistel_cipher}}
  \end{center}
\end{minipage}
The result of AES-G is then used to decrypt the title keys, which in turn are
used to finally decrypt the content.

\myitem{Security Issues}

Here are some of the security requirements for AACS implementations to ensure
DRM requirements:
\begin{itemize}\itemsep0pt
\item The content should ``not be present on any User-Accessible Bus in analog
  or unencrypted, compressed form'', because users could possibly record or
  redirect that content.
\item Implementations must use ``encryption, execution of a portion of the
  implementation in ring zero or supervisor mode (i.e., in kernel mode), and/or
  embodiment in a secure physical implementation,'' to keep encryption keys
  secret at all times.
\item They must also use ``techniques of obfuscation clearly designed to
  effectively disguise and hamper attempts to discover the approaches used''.
\end{itemize}
Thus, video content must travel through the system encrypted and must only
interact with authorized components over authorized pathways.

For example in Windows Vista AACS is implemented via the Protected Video Path
(PVP), which stops DRM-restricted content from playing while unsigned software
is running in order to prevent the unsigned software from accessing the
content. Additionally, PVP can encrypt information during transmission to the
monitor or the graphics card, which makes it more difficult to make unauthorized
recordings. In other word, you cannot watch a video, while running your own
programmes.
\newpage

\myitem{Legal and Ethical Issues}

It is obvious that to fully comply with the DRM requirement specification, AACS
has to be deeply embedded into an operating system. This essentially rules out
any open source implementation of the AACS as the algorithms to decrypt keys
would be easily accessible and therefore security would be compromised. But even
if only proprietary implementations are licensed and allowed, any Operating
System can (at least theoretically) be emulated by a virtual machine. But
running an AACS compliant software player on the VR would give access to the
unsecured data streams. This has indeed already been done:

In December 2006 the first software was announced and subsequently published
that enables to backup AACS encrypted content. In order to keep the software
legal, it cannot actually be used on its own, but an appropriate key must be
supplied manually. Since extracting a key without license would be illegal, 128
bit keys were quickly made available throughout the Internet. The AACS licensing
agency tried to put a stop to this by suing websites publishing keys as well as
by revoking keys (in particular for Windows based software like WinDVD).
However, the more they sued, the more keys were published, etc.

The issue remains unsolved to date, in particular since much is a legal grey
area.  Some of the issues of DRM highlighted by this controversy are:
\begin{itemize}
\item The AACS licensing agency argued that publishing the keys is illegal since
  they would fall under their software patent.  However, is it possible to
  patent single 128-bit numbers? And if so, in which form? Only the hexadecimal
  representation, or all other representations as well, i.e. decimal, binary,
  octal, sum of two or several values, etc.?  AACS can be used with millions
  of keys. Are they all patented? Can I be sued if I use one, publish one, etc.
\item Under the laws of many countries making one backup of media content is
  legal. AACS however effectively prevents this.
\item AACS protected media can only be played on specially licensed devises, not
  necessarily on the playback devise of my choice.  In other words, although I
  own the content of the disc (not the Copyright!) I can not play it the way I
  want to.

  As a comparison: If I buy a bottle of Coke, I am not allowed to reproduce its
  content, as it is a patented formula. However, no-one can force me to only
  drink it out of a particular licensed glass.
\item The DRM requirements of AACS can only be realised when fully embedded into
  the operating system. In order not to compromise security (and thus violate
  the licensing terms) information on the embedding cannot be made public or
  even given to competitors.  However, this amounts essentially to a similar
  controversy that has led to anti-trust law suits, by companies like Netscape,
  RealPlayer etc, against Microsoft.
\end{itemize}
These are just some issues. There are plenty more\ldots

\endText

\ifText
\ExHead{\EXN}\HandIn{18 October, 14pm}

\begin{enumerate}%%\setcounter{enumi}{0}\itemsep0pt
\ResumeExercises
\EX{EXmcw}{8+8}
\EX{EXmcp}{6+6+8}
\StoreExercises
\end{enumerate}
\fi

\ifSolution
\beginSolution
\SolHead{\EXN}{21 October}
\begin{enumerate}%\setcounter{enumi}{0}\itemsep0pt
  \ResumeSolutions
  \SOL{EXmcw}
  \SOL{EXmcp}
  \StoreSolutions
\end{enumerate}
\endSolution
\fi

\ifMaths\beginMaths{Finite Fields}

Finite fields are an important algebraic concepts used in more advanced
cryptographic techniques. This section will introduce the very basics, as far as
we need them in this lecture. For a more in-depth introduction, see for instance:
Lidl \& Niederreiter, \emph{Introduction to finite fields and their
  applications}, Cambridge University Press, 1994.

We first define some basic algebraic notions characterising sets together with
binary operations.

\begin{definition}{\textbf{(Group)}}
  Let $G$ be a set and $\circ$ be a binary operation defined on $G$, i.e.,
  $\circ:G\times G\to G$. We say $(G,\circ)$ is a \textbf{group} if $\circ$
  \begin{romanenum}
  \item is \textbf{closed}: that is for each $a,b\in G$ we have $a\circ b\in G$.
  \item is \textbf{associative}: that is for each $a,b,c\in G$ we have $(a\circ
    b)\circ c=a\circ (b\circ c)$.
  \item has an \textbf{identity element}: that is there exists $e\in G$ s.t. for
    each $a\in G$ we have $a\circ e=e\circ a=a$.
  \item every element has an \textbf{inverse}: that is for every $a\in G$ there
    is $\inv{a}\in G$ with $a\circ\inv{a}=\inv{a}\circ a=e$.
  \end{romanenum}
  We call $(G,\circ)$ a \textbf{commutative group} if in addition $\circ$
  \begin{romanenum}\addtocounter{enumi}{4}
  \item is \textbf{commutative}: that is for each $a,b\in G$ we have $a\circ b=b
    \circ a$.
  \end{romanenum}
\end{definition}

\example An easy example are the integers with addition $(\Z,+)$. They form a
commutative group, since they are clearly closed and associative, the identity
element is $0$ and each element $a\in\Z$ has an inverse, namely $-a\in\Z$.

On the contrary the integers with times $(Z,\cdot)$ are not a group! Although
they are closed under $\cdot$, times is associative, and $1\in\Z$ is an identity
element, not every element has an inverse. For example $2\in\Z$ would have
$\frac12$ as inverse, which is not an integer!

However, the relationship between $+$ and $\cdot$ gives rise to the following
definition:

\begin{definition}{\textbf{(Ring)}}
  Let $R$ be a set with two binary operations $+$ and $\cdot$, then
  $(R,+,\cdot)$ is a ring if
  \begin{itemize}
  \item $(R,+)$ is a commutative group,
  \item $(R,\cdot)$ is closed, associative and has an identity.
  \item $+$ and $\cdot$ are
    \begin{romanenum}
    \item \textbf{left distributive}: that is for each $a,b,c\in R$ we have
      $a\cdot(b+c)=(a\cdot b)+(a\cdot c)$,
    \item \textbf{right distributive}: that is for each $a,b,c\in R$ we have $(b+c)\cdot a=(b\cdot a)+(c\cdot a)$.
    \end{romanenum}
  \end{itemize}
\end{definition}

\example $(\Z,+,\cdot)$ is a ring, since we can easily verify the distributivity
laws.

If the multiplication has inverses as well, we can extend our ring definition to
the following:

\begin{definition}{\textbf{(Field)}}
  Let $F$ be a set with two binary operations $+$ and $\cdot$. Let $F^*$ be the
  set that contains all elements of $F$ except the identity for $+$, i.e. we let
  $F^*=F\setminus\set{0}$, where $0$ is the identity for $+$. Then $(F,+,\cdot)$
  is a field if
  \begin{itemize}
  \item $(F,+)$ is a commutative group,
  \item $(F^*,\cdot)$ is a commutative group,
  \item $+$ and $\cdot$ are left and right distributive.
  \end{itemize}
\end{definition}

\example The rational numbers with addition and multiplication form a field:
$(\Q,+,\cdot)$. Both operations are obviously closed, associative and
commutative. For addition $0$ is the identity and for every $a\in\Q$ the
additive inverse is $-a\in\Q^*$.  Then $\Q^*=\Q\setminus\set{0}$, $1$ is the
identity for multiplication and every $a\in\Q^*$ has a multiplicative inverse,
namely $\frac1a\in\Q^*$.

\pagebreak

\textbf{\large Some Finite Examples}

In the following we want to restrict ourselves to finite sets. We have already
seen some examples of finite sets in Mathematics 1 + 2. For example, the set of
all permutations of $n$ elements forms non-commutative(!) group, the symmetric
group $S_n$. Other finite sets are the sets of residue classes modulo some $n$,
$\Z_n$, for which we now want to check what structure they form.

For finite sets one can use a very easy technique to verify the properties of a
particular operation, by simply writing down the entire operation in the form of
a \emph{multiplication table}. For example we can use the following
multiplication tables

\begin{minipage}[t]{.2\textwidth}
  {$(\Z_2,+)$:}
  $\begin{array}{c|cc}
           +  &\rcl{0}{2}&\rcl{1}{2}\\\hline
    \rcl{0}{2}&\rcl{0}{2}&\rcl{1}{2}\\
    \rcl{1}{2}&\rcl{1}{2}&\rcl{0}{2}\\
  \end{array}$
\end{minipage}
\begin{minipage}[t]{.2\textwidth}
  {$(\Z_2,\cdot)$:}
  $\begin{array}{c|cc}
       \cdot  &\rcl{0}{2}&\rcl{1}{2}\\\hline
    \rcl{0}{2}&\rcl{0}{2}&\rcl{0}{2}\\
    \rcl{1}{2}&\rcl{0}{2}&\rcl{1}{2}\\
  \end{array}$
\end{minipage}
\begin{minipage}[t]{.3\textwidth}
  {$(\Z_3,+)$:}
  $\begin{array}{c|ccc}
           +  &\rcl{0}{3}&\rcl{1}{3}&\rcl{2}{3}\\\hline
    \rcl{0}{3}&\rcl{0}{3}&\rcl{1}{3}&\rcl{2}{3}\\
    \rcl{1}{3}&\rcl{1}{3}&\rcl{2}{3}&\rcl{0}{3}\\
    \rcl{2}{3}&\rcl{2}{3}&\rcl{0}{3}&\rcl{1}{3}\\
  \end{array}$
\end{minipage}
\begin{minipage}[t]{.3\textwidth}
  {$(\Z_3,\cdot)$:}
  $\begin{array}{c|ccc}
       \cdot  &\rcl{0}{3}&\rcl{1}{3}&\rcl{2}{3}\\\hline
    \rcl{0}{3}&\rcl{0}{3}&\rcl{0}{3}&\rcl{0}{3}\\
    \rcl{1}{3}&\rcl{0}{3}&\rcl{1}{3}&\rcl{2}{3}\\
    \rcl{2}{3}&\rcl{0}{3}&\rcl{2}{3}&\rcl{1}{3}\\
  \end{array}$
\end{minipage}
to determine that $(\Z_2,+)$ and $(\Z_3,+)$ are commutative groups. Obviously
$(\Z_2,\cdot)$ and $(\Z_3,\cdot)$ are not groups, since neither $\rcl{0}{2}$ nor
$\rcl{0}{3}$ have a multiplicative inverse. However, if we get rid of the $0$
element in both tables, it is easy to see that we get commutative groups for
$\Z_2^*$ and $\Z_3^*$:
\begin{center}
\begin{minipage}[t]{.3\textwidth}
  {$(\Z_2^*,\cdot)$:} $\begin{array}{c|c} \cdot &\rcl{1}{2}\\\hline
    \rcl{1}{2}&\rcl{1}{2}\\
  \end{array}$
\end{minipage}
\begin{minipage}[t]{.3\textwidth}
  {$(\Z_3^*,\cdot)$:} $\begin{array}{c|cc} \cdot &\rcl{1}{3}&\rcl{2}{3}\\\hline
    \rcl{1}{3}&\rcl{1}{3}&\rcl{2}{3}\\
    \rcl{2}{3}&\rcl{2}{3}&\rcl{1}{3}\\
  \end{array}$
\end{minipage}
\end{center}
Since one can also easily check that both distributivity laws hold, we can thus
conclude that both $(\Z_2,+,\cdot)$ and $(\Z_3,+,\cdot)$ are fields.

The natural next question is: Do all residue class sets form a field together
with addition and multiplication? Let's have a look at $\Z_4$:

\begin{center}
  {$(\Z_4,+)$:} $\begin{array}{c|cccc} +
    &\rcl{0}{4}&\rcl{1}{4}&\rcl{2}{4}&\rcl{3}{4}\\\hline
    \rcl{0}{4}&\rcl{0}{4}&\rcl{1}{4}&\rcl{2}{4}&\rcl{3}{4}\\
    \rcl{1}{4}&\rcl{1}{4}&\rcl{2}{4}&\rcl{3}{4}&\rcl{0}{4}\\
    \rcl{2}{4}&\rcl{2}{4}&\rcl{3}{4}&\rcl{0}{4}&\rcl{1}{4}\\
    \rcl{3}{4}&\rcl{3}{4}&\rcl{0}{4}&\rcl{1}{4}&\rcl{2}{4}\\
  \end{array}$\qquad
  {$(\Z_4^*,\cdot)$:} $\begin{array}{c|ccc} \cdot
    &\rcl{1}{4}&\rcl{2}{4}&\rcl{3}{4}\\\hline
    \rcl{1}{4}&\rcl{1}{4}&\rcl{2}{4}&\rcl{3}{4}\\
    \rcl{2}{4}&\rcl{2}{4}&\rcl{0}{4}&\rcl{2}{4}\\
    \rcl{3}{4}&\rcl{3}{4}&\rcl{2}{4}&\rcl{1}{4}\\
  \end{array}$
\end{center}
We can see, while $(Z_4,+)$ is a commutative group, $(\Z_4^*,\cdot)$ is not a
group: It is not even closed, as $\rcl{2}{4}\cdot\rcl{2}{4}=\rcl{0}{4}$ and
$\rcl{0}{4}$ is not an element of $\Z_4^*$, and there is also no inverse element
for $\rcl{2}{4}$. Thus $(\Z_4,+,\cdot)$ is a ring ($\rcl{1}{4}$ is the neutral
element for $\cdot$), but not a field.

Indeed one can show the following two theorems:
\begin{theorem}{}
  $(\Z_n,+,\cdot)$ is a ring for every $n\geq 2$.
\end{theorem}
\begin{theorem}{}
  $(\Z_p,+,\cdot)$ is a field if and only if $p$ is a prime number.
\end{theorem}

\begin{definition}{\textbf{(Finite Field)}}
  Let $(F,+,\cdot)$ be a field. If $F$ is a finite set with $p$ elements, we
  call $(F,+,\cdot)$ a \textbf{finite field} or order $p$ and denote it by
  $\F_p$.
\end{definition}
\example We can now write $\F_2=(\Z_2,+,\cdot)$, $\F_3=(\Z_3,+,\cdot)$, etc. In
general we have for every prime number $p$: $\F_p=(\Z_p,+,\cdot)$.

So far we know that for every prime number $p$ there exists a finite field of
that order. In addition one can easily show that this is (up to isomorphism) the
only finite field of that order. That is, every finite field of prime order has
the structure of $\Z_p$.

Our next question is, are these the only finite fields, or are there any others,
i.e. of an order that is not a prime number. In order to answer this question,
we have to make a little detour via the theory of polynomials.

\pagebreak

\textbf{\large Polynomials}

I assume that everyone is familiar with polynomials. The following is just to
recall some important concepts. While we can define polynomials essentially over
any ring, we will restrict ourselves, for now, to polynomials over the integers
$\Z$.

\begin{definition}{\textbf{(Polynomial)}}
  We call an expression of the form $a_nx^n+\ldots+a_2x^2+a_1x+a_0$ a \textbf{polynomial}
  in the variable $x$ over $\Z$, if all $a_i\in\Z, i=0,\ldots,n$ and all
  exponents $0,\ldots,n$ are non-negative integers.

  We denote the set of all polynomials in one variable over $\Z$ as $\Z[x]$.

  We call a summand $a_ix^i$ of a polynomial a \textbf{monomial} of
  \textbf{degree} $i$ with \textbf{coefficient} $a_i$.

  We say a polynomial $p\in\Z[x]$ is of \textbf{degree} $n$ if its greatest
  non-zero monomial is of degree $n$. We generally write $deg(p)=n$.

\end{definition}

\example $p(x)=x^4+3x^3+2x^2-10$ is a polynomial of degree $4$.

\begin{definition}{\textbf{(Polynomial Arithmetic)}}
  Let $p(x),q(x)\in\Z[x]$ be $p(x)=a_nx^n+\ldots+a_2x^2+a_1x+a_0$ and
  $q(x)=b_nx^n+\ldots+b_2x^2+b_1x+b_0$. We define addition $+$ and multiplication
  $\cdot$ as component-wise operations as:

  \begin{romanenum}
  \item $p(x)+q(x)=(a_n+b_n)x^n+\ldots+(a_2+b_2)x^2+(a_1+b_1)x+(a_0+b_0)$
  \item $\begin{array}[t]{lllllll}
      p(x)*q(x)&=&&(a_n*b_m)x^{(n+m)}&+(a_n*b_{m-1})x^{(n+(m-1))}&+\ldots&+(a_n*b_{0})x^n\\
      & &&&\qquad\qquad\qquad\qquad\vdots\\
      && + &(a_0*b_m)x^m&+(a_0*b_{m-1})x^{m-1}&+\ldots&+(a_0*b_{0})
    \end{array}$
  \end{romanenum}

\end{definition}

\example Let $p(x)=x^4+3x^3+2x^2-10$ and $q(x)=2x^3-9x^2+2x-3$. Then we have
\begin{eqnarray*}
p(x)+q(x)&=&x^4+5x^3-7x^2+2x-13\\
p(x)\cdot q(x)&=&2x^7-3x^6-21x^5-15x^4-25x^3+84x^2-20x+30.
\end{eqnarray*}

Recall that $(\Z,+,\cdot)$ forms a ring. Similarly we can show that
$(\Z[x],+,\cdot)$ forms a ring with the addition and multiplication over
polynomials. We now observe some more parallels between $\Z$ and $\Z[x]$:

%% \begin{tabular}{p{.5\textwidth}|p{.5\textwidth}}
%%   \centering{$\Z$} & \qquad\qquad\qquad\qquad$\Z[x]$\\
%%   \multicolumn{1}{c}\textbf{Division:}\\
%% \end{tabular}

\begin{tabular}{p{.3\textwidth}|p{.7\textwidth}}
  \centering{$\Z$} & \qquad\qquad\qquad\qquad$\Z[x]$\\\hline
\end{tabular}

\vspace{.4cm}\textbf{Division with Remainder}\vspace{.1cm}

\begin{tabular}{p{.3\textwidth}|p{.7\textwidth}}
  Divide $323$ by $7$ & Divide $x^3+4x^2+6x-1$ by $x^2+2x+1$\\
  $\begin{array}{lcccc}
      \phantom{-}323 & / & 7 & = & 46\\
      -28 & & & & \\\cline{1-1}
      \phantom{-3}43 & & & & \\
      \phantom{}-42 & & & & \\\cline{1-1}
      \phantom{-34}1 & & & & \\
    \end{array}$
  &\arraycolsep1pt
  $\begin{array}{llllllcccc}
    \phantom{-}x^3 & +4x^2 & +6x & -1 & / & x^2+2x+1 & = & x + 2 \\
    -x^3 & - 2x^2 & -\phantom{6}x \\\cline{1-4}
    & \phantom{-}2x^2 & +5x & -1\\
    & - 2x^2 & -4x & -2\\\cline{2-4}
    & & \phantom{-4}x & -3
  \end{array}$\\[.4cm]
  In general for every $a,b\in\Z$ with $a\geq b$ we find $s,r\in\Z$
  with $|s|<|a|$ and $|r|<|b|$ such that $a=s\cdot b + r$.
  &
  In general for every $p(x),q(x)\in\Z[x]$ with $deg(p)\geq deg(q)$ we find $s(x),r(x)\in\Z[x]$
  with $deg(s)<deg(p)$ and $deg(r)<deg(q)$ such that $k\cdot p(x)=s(x)\cdot q(x) + r(x)$, with
  $k\in\Z$.\newline Observe that $k$ guarantees integer division
  for the coefficients (e.g. $x^2+1$ is not divisible by $2x+1$ in $\Z$, but $4x^2+4$ is).
  Such a $k$ always exists!
\end{tabular}

\vspace{.4cm}\textbf{Modular Arithmetic}\vspace{.1cm}

\begin{tabular}{p{.3\textwidth}|p{.7\textwidth}}
  %%\centering{$\Z$} & \qquad\qquad\qquad\qquad$\Z[x]$\\
  Recall: $\modulo{323}{7}{1}$. &
  Similarly we can write $\modulo{x^3+4x^2+6x-1}{x^2+2x+1}{x-3}$\\[.2cm]
  The modulo operation divides $\Z$ into a finite number of residue classes, e.g., for mod $n$:
  $\Z_n=\set{0,1,\ldots,n-1}$ &
  Given a polynomial $p(x)\in\Z[x]$, the modulo $p(x)$ operation induces residue classes on $\Z[x]$.
  We denote the set of all residue classes modulo $p(x)$ by $\Z[x]/p(x)$. It contains one
  residue class for each polynomial in $\Z[x]$ of degree less then $p(x)$.   These are, however,
  infinitely many!.
\end{tabular}

%% \textbf{Residue Classes}

%% \begin{tabular}{p{.3\textwidth}|p{.7\textwidth}}
%%   \centering{$\Z$} & \qquad\qquad\qquad\qquad$\Z[x]$\\
%% \end{tabular}

\vspace{.4cm}\textbf{Irreducibility}\vspace{.1cm}

\begin{tabular}{p{.3\textwidth}|p{.7\textwidth}}
  $p\in\Z$ is prime if it is only divisible by $1$ and $p$.  &
  $p(x)\in\Z[x]$ is called \emph{irreducible} if it is only divisible by $p(x)$ and
  the trivial polynomial $a_0x^0=a_0\in\Z$. E.g., $x^2+1$ is irreducible in $\Z[x]$.
\end{tabular}

\newpage
\textbf{\large Polynomials over Finite Fields}

As mentioned earlier we can construct polynomials over arbitrary rings and
therefore also fields. We will now look at the residue class construction for
polynomials over finite fields. While the following could be done with any
finite field of the form $\F_p$, where $p$ is a prime number, we will restrict
ourselves to the finite field $\F_2$, in which we are most interested in.

Recall that $\F_2=(\Z_2,+,\cdot)$, i.e. contains only the elements $0$ and $1$.
We will from now on omit the residue class notation and write $0$ and $1$
instead of $\rcl{0}{2}$ and $\rcl{1}{2}$, respectively!

We now define the polynomial ring over $\F_2$ as $\ftwox$. We first have a look
at the general polynomial arithmetic in $\ftwox$, which works modulo $2$, that
is we only have $0$ and $1$ as coefficients and the addition and multiplication
of coefficient is performed modulo $2$. We observe this with an example:

\example Let $x^2+x+1$ and $x^3+x^2+x$ be polynomials over $\ftwox$ then we have:\vspace{-.1cm}
\begin{eqnarray*}
  (x^2+x+1)+(x^3+x^2+x) = & x^3+2x^2+2x+1 & = x^3+1\\
  (x^2+x+1)\cdot(x^3+x^2+x) = & x^5+2*x^4+3*x^3+2*x^2+x & = x^5+x^3+x
\end{eqnarray*}
This also means that we do not need negative coefficients, as
$x+1{=}-x+1{=}x-1{=}-x-1$ in $\ftwox$.\vspace{.2cm}

\textbf{Modular Arithmetic}\quad
Let $p(x)=x^4+x+1\in\ftwox$. Then we have \vspace{-.1cm}
\[\modulo{(x^2+x+1)\cdot(x^3+x^2+x)= x^5+x^3+x}{x^4+x+1}{x^3+x^2}\]\vspace{-.0cm}
To verify this we look at the following polynomial division:\vspace{-.0cm}
\[\begin{array}{rrrrrrrcccc}
  x^5 & & +x^3 & & +x& & / & x^4+x+1 = x \\
  x^5 & &      &+x^2& +x \\\cline{1-6}
  & & x^3     &+x^2& \\
\end{array}\]\vspace{-.25cm}

In a next step we can now define the residue classes for \textbf{$\ftwoxpx$}.
For simplification we take $p(x)$ to be polynomial of degree $2$ first. Thus let
$p(x)=x^2+1\in\ftwoxpx$. We then get four residue classes modulo $x^2+1$,
represented by $[0]$, $[1]$, $[x]$, $[x+1]$, i.e., all polynomials in $\ftwox$
with degree less than $2$. We can then construct the following tables for $+$
and $\cdot$:

\begin{minipage}{.5\textwidth}
\begin{center}
  {$(\ftwox/(x^2+1),+)$:}\newline $\begin{array}{c|cccc} +
             &\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\\hline
    \rcl{0}{}&\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\
    \rcl{1}{}&\rcl{1}{}&\rcl{0}{}&\rcl{x+1}{}&\rcl{x}{}\\
    \rcl{x}{}&\rcl{x}{}&\rcl{x+1}{}&\rcl{0}{}&\rcl{1}{}\\
    \rcl{x+1}{}&\rcl{x+1}{}&\rcl{x}{}&\rcl{1}{}&\rcl{0}{}\\
  \end{array}$
\end{center}
\end{minipage}
\begin{minipage}{.5\textwidth}
\begin{center}
  {$(\ftwox/(x^2+1),\cdot)$:}\newline $\begin{array}{c|cccc} \cdot
    &\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\\hline
    \rcl{0}{}&\rcl{0}{}&\rcl{0}{}&\rcl{0}{}&\rcl{0}{}\\
    \rcl{1}{}&\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\
    \rcl{x}{}&\rcl{0}{}&\rcl{x}{}&\rcl{1}{}&\rcl{x+1}{}\\
    \rcl{x+1}{}&\rcl{0}{}&\rcl{x+1}{}&\rcl{x+1}{}&\rcl{0}{}\\
  \end{array}$
\end{center}
\end{minipage}

We can see that, even when we delete the $\rcl{0}{}$ lines in the multiplication
table, the group axioms will not hold for `$\cdot$', as $\rcl{x+1}{}$ does not
have an inverse element. This can be explained by the fact that $x^2+1$ is not
an irreducible polynomial in $\ftwox$, since it can be factorised into
$x^2+1=(x+1)(x+1)$. We recall that for those $\Z_n$ where $n$ was a prime number
we could construct a finite field. If we replace $x^2+1$ by an irreducible
polynomial we should therefore also get a finite field. Let's try instead the
polynomial $p(x)=x^2+x+1$, which is indeed irreducible over $\ftwox$.

\begin{minipage}{.5\textwidth}
\begin{center}
  {$(\ftwox/(x^2+x+1),+)$:}\newline $\begin{array}{c|cccc} +
             &\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\\hline
    \rcl{0}{}&\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\
    \rcl{1}{}&\rcl{1}{}&\rcl{0}{}&\rcl{x+1}{}&\rcl{x}{}\\
    \rcl{x}{}&\rcl{x}{}&\rcl{x+1}{}&\rcl{0}{}&\rcl{1}{}\\
    \rcl{x+1}{}&\rcl{x+1}{}&\rcl{x}{}&\rcl{1}{}&\rcl{0}{}\\
  \end{array}$
\end{center}
\end{minipage}
\begin{minipage}{.5\textwidth}
\begin{center}
  {$(\ftwox/(x^2+x+1),\cdot)$:}\newline $\begin{array}{c|cccc} \cdot
    &\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\\hline
    \rcl{0}{}&\rcl{0}{}&\rcl{0}{}&\rcl{0}{}&\rcl{0}{}\\
    \rcl{1}{}&\rcl{0}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}\\
    \rcl{x}{}&\rcl{0}{}&\rcl{x}{}&\rcl{x+1}{}&\rcl{1}{}\\
    \rcl{x+1}{}&\rcl{0}{}&\rcl{x+1}{}&\rcl{1}{}&\rcl{x}{}\\
  \end{array}$
\end{center}
\end{minipage}

The two tables demonstrate that both $(\ftwox/(x^2+x+1),+)$ and
{$((\ftwox/(x^2+x+1))^*,\cdot)$} form commutative groups. [Recall that
$(\ftwox/(x^2+x+1))^*=\ftwox/(x^2+x+1)\setminus\set{\rcl{0}{}}$.]  We can also
show that the two distributivity laws hold and that therefore
{$((\ftwox/(x^2+x+1))^*,+,\cdot)$} is a finite field of order $4$.

This construction demonstrates that for every irreducible polynomial
$p(x)\in\ftwox$ of degree $n$, $\ftwoxpx$ yields a finite field of order $2^n$,
independent of the concrete choice of $p(x)$. The more general
result is:
\begin{theorem}{}
  For every prime $p$ and every positive integer $n$ there exists one finite
  field of order $\F_{p^n}$.
\end{theorem}

\newpage
\textbf{\large Finite Fields as Binary Operations}

What does all this have to do with computer science? Recall that we restricted
ourselves to polynomials in $p(x)\in\ftwox$. This means each monomial in $p(x)$
has as coefficient either $1$ or $0$, i.e. the coefficients are binary. We can
therefore straightforwardly translate polynomials of degree $d$ into bit strings
of length $d+1$ by just taking the coefficients of each monomial.

\example Consider the translation of a polynomial of degree $7$ into $8$ bits:
\[\begin{array}{cccccccr}
 x^7 & + x^6 & & + x^4 & + x^3 & & & +1\\
1 & 1 & 0 & 1 & 1& 0 & 0& 1
\end{array}\]

We now can use the operations on the finite field $\F_{2^n}$ to define
operations on bit strings of length $n+1$. For brevity we will take our examples
from $\F_8=\ftwoxpx$ with $p(x)=x^3+x+1$ as irreducible polynomial.

We first observe that addition on $\F_{2^n}$ is the same as the xor operation
$\xor$ on bits.

\example $\begin{array}{cccccccr}
(x^2+x+1) &+ & (x^2+1) & = & x\\
111 & \xor & 101 & = & 010
\end{array}$

While addition does not give us a new operation, we can use multiplication on
$\F_8$ to define a new bitwise operation $\otimes$.  First here is the
multiplication table for $\F_8^*$:
\begin{center}\small\arraycolsep2pt
  $\begin{array}{c|ccccccc}
    {(\F_8^*,\cdot)} & \rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}&\rcl{x^2}{}&\rcl{x^2+1}{}&\rcl{x^2+x}{}&\rcl{x^2+x+1}{}\\\hline
    \rcl{1}{}&\rcl{1}{}&\rcl{x}{}&\rcl{x+1}{}&\rcl{x^2}{}&\rcl{x^2+1}{}&\rcl{x^2+x}{}&\rcl{x^2+x+1}{}\\
    \rcl{x}{}&\rcl{x}{}&\rcl{x^2}{}&\rcl{x^2+x}{}&\rcl{x+1}{}&\rcl{1}{}&\rcl{x^2+x+1}{}&\rcl{x^2+1}{}\\
    \rcl{x+1}{}&\rcl{x+1}{}&\rcl{x^2+x}{}&\rcl{x^2+1}{}&\rcl{x^2+x+1}{}&\rcl{x^2}{}&\rcl{1}{}&\rcl{x}{}\\
    \rcl{x^2}{}&\rcl{x^2}{}&\rcl{x+1}{}&\rcl{x^2+x+1}{}&\rcl{x^2+x}{}&\rcl{x}{}&\rcl{x^2+1}{}&\rcl{1}{}\\
    \rcl{x^2+1}{}&\rcl{x^2+1}{}&\rcl{1}{}&\rcl{x^2}{}&\rcl{x}{}&\rcl{x^2+x+1}{}&\rcl{x+1}{}&\rcl{x^2+x}{}\\
    \rcl{x^2+x}{}&\rcl{x^2+x}{}&\rcl{x^2+x+1}{}&\rcl{1}{}&\rcl{x^2+1}{}&\rcl{x+1}{}&\rcl{x}{}&\rcl{x^2}{}\\
    \rcl{x^2+x+1}{}&\rcl{x^2+x+1}{}&\rcl{x^2+1}{}&\rcl{x}{}&\rcl{1}{}&\rcl{x^2+x}{}&\rcl{x^2}{}&\rcl{x+1}{}\\
  \end{array}$
\end{center}
The operation $b_1\otimes b_2$ on 3-bit strings $b_1,b_2$ is then defined taking
the two polynomials corresponding to $b_1$ and $b_2$,
respectively, multiplying them according to the above multiplication table and
transforming the result again into a 3-bit string.

\example $\begin{array}{cccccccr}
(x^2+x+1) &\cdot & (x^2+1) & \equiv & x^2+x & ({\bmod}\, x^3+x+1)\\
111 & \otimes & 101 & = & 110
\end{array}$

Observe that the choice of irreducible polynomial really matters in the
definition of $\otimes$. For instance, if our choice of irreducible polynomial
were $x^3+x^2+1$ then the example would look like this:

\example $\begin{array}{cccccccr}
(x^2+x+1) &\cdot & (x^2+1) & \equiv & 1 &({\bmod}\, x^3+x^2+1)\\
111 & \otimes & 101 & = & 001
\end{array}$

Finally we consider another two, more complex examples, where again the choice
of irreducible polynomial matters:

\example \[\begin{array}{cccccccr}
(x^6+1) &\cdot & (x^4+1) & \equiv & x^5+x^4+x^3+x^2+1&({\bmod}\, x^8+x^4+x^3+x+1)\\
01000001 & \otimes & 00010001 & = & 00111101
\end{array}\]
\[\begin{array}{cccccccr}
(x^6+1) &\cdot & (x^4+1) & \equiv & x^5+x^2+1 &({\bmod}\, x^8+x^4+x^3+x^2+1)\\
01000001 & \otimes & 00010001 & = & 00100101
\end{array}\]

Observe that the bit strings are of length $8$, which is a multiple of $4$. We
can therefore express the above multiplications in terms of hexadecimal numbers,
indicated by prefix $0x$:

\centerline{$0x41 \otimes 0x11 = 0x3\mathrm{D}$\qquad and\qquad $0x41 \otimes 0x11 = 0x25$}

\endMaths\fi

\ifMaths
\beginMaths{Matrix Arithmetic}

I assume that everyone is familiar with matrices and basic operations on them.
This handout should serve as a reminder.

Recall that a matrix is a rectangular array of elements. Abstractly a $m\times
n$ matrix can be displayed as:
\[A=\begin{bmatrix}
  a_{1,1}  & \cdots  & a_{1,n}\\
  \vdots & \cdot & \vdots \\
  a_{m,1} & \cdots & a_{m,n}
\end{bmatrix}\]
We will refer to a particular element in the $i$'s row and $j$'s column as $A[i,j]=a_{i,j}$.

\example $A = \begin{bmatrix} 1 & 2 & 3 \\ 1 & 2 & 7 \\ 4&9&2 \\
  6&0&5\end{bmatrix}$ is a $4\times3$ matrix. The element $a_{2,3}$ is $7$.

The matrix $R = \begin{bmatrix} 1 & 2 & 3 & 4 & 5 & 6 & 7 & 8 & 9 \end{bmatrix}$
is a $1\times 9$ matrix, or 9-element row vector.\vspace{.5cm}

\textbf{\large Matrix Addition}

Given $m\times n$ matrices $A$ and $B$, their sum $A + B$ is the $m\times n$
matrix computed by adding corresponding elements (i.e. $(A + B)[i, j] = A[i, j]
+ B[i, j]$).

\example $\begin{bmatrix} 1 & 3 \\ 1 & 0 \\ 1 & 2 \end{bmatrix} + \begin{bmatrix} 0 & 0 \\
  7 & 5 \\ 2 & 1 \end{bmatrix} = \begin{bmatrix} 1+0 & 3+0 \\ 1+7 & 0+5 \\ 1+2 &
  2+1 \end{bmatrix} = \begin{bmatrix} 1 & 3 \\ 8 & 5 \\ 3 & 3 \end{bmatrix}$\vspace{.5cm}

\textbf{\large Matrix Multiplication}

Multiplication of two matrices is well-defined only if the number of columns of
the left matrix is the same as the number of rows of the right matrix. If $A$ is
an $m\times n$ matrix and $B$ is an $n\times p$ matrix, then their matrix
product $AB$ is the $m\times p$ matrix ($m$ rows, $p$ columns) given by:
\[(AB)[i,j] = A[i,1] B[1,j] + A[i,2] B[2,j] + ... + A[i,n] B[n,j]\quad \mbox{for each pair $i$ and $j$.}\]

\example $\begin{bmatrix} 1 & 0 & 2 \\ -1 & 3 & 1 \\ \end{bmatrix} \cdot \begin{bmatrix}
  3 & 1 \\ 2 & 1 \\ 1 & 0 \end{bmatrix} = \begin{bmatrix} (1 \cdot 3 + 0 \cdot
  2 + 2 \cdot 1) & (1 \cdot 1 + 0 \cdot 1 + 2 \cdot 0) \\ (-1 \cdot 3 + 3
  \cdot 2 + 1 \cdot 1) & (-1 \cdot 1 + 3 \cdot 1 + 1 \cdot 0) \\
\end{bmatrix} = \begin{bmatrix} 5 & 1 \\ 4 & 2 \\ \end{bmatrix}$\vspace*{.5cm}

\textbf{\large Matrix Multiplication over $\F_2$}

So far we have implicitly assumed that our matrices are defined over the
integers. However, we can likewise define matrices with elements from $\F_2$.
Addition and multiplication are then similarly defined as above with the
exception that the operations on the single components are performed modulo $2$.

\example Addition: $\begin{bmatrix} 1 & 0\\ 1 & 1\\\end{bmatrix}+
\begin{bmatrix} 1 & 1\\ 0 & 1\\\end{bmatrix} =
\begin{bmatrix} 0 & 1\\ 1 & 0\\\end{bmatrix}
$

\example Multiplication: $\begin{bmatrix} 1 & 0 & 1 & 1\\ 1 & 1 & 0 & 1\\ 0 & 1 & 0 & 1\\\end{bmatrix}\cdot
\begin{bmatrix} 1 \\ 0\\ 1\\ 1\\\end{bmatrix} =
\begin{bmatrix} 1+1+1 \\ 1+1\\ 1 \\\end{bmatrix}=
\begin{bmatrix} 1 \\ 0\\ 1 \\\end{bmatrix}
$

Similarly, we can define matrices and matrix arithmetic over other fields, in
particular over all finite fields of the form $\F_{2^n}$.

\endMaths\fi

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "handouts"
%%% End:


% Part: first-order-logic
% Chapter: model-theory
% Section: theory-of-m

\documentclass[../../../include/open-logic-section]{subfiles}

\begin{document}

\olfileid{mod}{bas}{thm}

\section{The Theory of a \printtoken{S}{structure}}

Every !!{structure}~$\Struct{M}$ makes some !!{sentence}s true, and
some false. The set of all the !!{sentence}s it makes true is called
its \emph{theory}. That set is in fact a theory, since anything it
entails must be true in all its models, including~$\Struct{M}$.

\begin{defn}
  Given a !!{structure}~$\Struct M$, the \emph{theory} of
  $\Struct{M}$ is the set $\Theory{M}$ of !!{sentence}s
  that are true in $\Struct{M}$, i.e., $\Theory{M} =
  \Setabs{!A}{\Sat{M}{!A}}$.
\end{defn}

We also use the term ``theory'' informally to refer to sets
of !!{sentence}s having an intended interpretation, whether deductively
closed or not.

\begin{prop}
For any $\Struct{M}$, $\Theory{M}$ is complete.
\end{prop}

\begin{proof}
For any !!{sentence}~$!A$ either $\Sat{M}{!A}$ or $\Sat{M}{\lnot !A}$,
so either $!A \in \Theory{M}$ or $\lnot !A \in \Theory{M}$.
\end{proof}

\begin{prop}\ollabel{prop:equiv}
  If $\Struct{N} \models !A$ for every $!A \in \Theory{M}$, then
  $\Struct{M} \elemequiv \Struct{N}$.
\end{prop}

\begin{proof}
Since $\Sat{N}{!A}$ for all $!A \in \Theory{M}$, $\Theory{M} \subseteq
\Theory{N}$. If $\Sat{N}{!A}$, then $\Sat/{N}{\lnot !A}$, so $\lnot !A
\notin \Theory{M}$. Since $\Theory{M}$ is complete, $!A \in
\Theory{M}$. So, $\Theory{N} \subseteq \Theory{M}$, and we have
$\Struct{M} \elemequiv \Struct{N}$.
\end{proof}

\begin{rem}\ollabel{remark:R}
  Consider $\Struct{R} = \langle\Real, <\rangle$, the !!{structure}
  whose domain is the set $\Real$ of the real numbers, in the !!{language}
  comprising only a 2-place !!{predicate} interpreted as the $<$
  relation over the reals. Clearly $\Struct{R}$ is !!{nonenumerable};
  however, since $\Theory{R}$ is obviously consistent, by the
  L\"owenheim-Skolem theorem it has !!a{enumerable} model, say
  $\Struct{S}$, and by \olref{prop:equiv}, $\Struct{R}
  \equiv \Struct{S}$. Moreover, since $\Struct{R}$ and $\Struct{S}$
  are not isomorphic, this shows that the converse of
  \olref[iso]{thm:isom} fails in general.
\end{rem}

\end{document}


\chapter{Common Random Variables}\label{S:CommonRVs}

%\remove{
For a continuous RV $X$ with a closed-form expression for the inverse DF $F^{[-1]}$, we can employ \hyperref[A:InvS]{Algorithm~\ref*{A:InvS}} to draw samples from $X$.  \hyperref[T:ContinRVsInvS]{Table~\ref*{T:ContinRVsInvS}}  summarises some random variables that are amenable to \hyperref[A:InvS]{Algorithm~\ref*{A:InvS}}.

\begin{table}[htpb]
\begin{center}
\caption{Some continuous RVs that can be simulated from using \hyperref[A:InvS]{Algorithm~\ref*{A:InvS}}. \label{T:ContinRVsInvS}}
\begin{tabular}{| c | c | c | c |}
\hline
Random Variable $X$ & $F(x)$ & $X=F^{[-1]}(U), \quad U \sim \uniform(0,1)$ & Simplified form \\ \hline
$\uniform(a,b)$ &
\eqref{E:Uniformabcdf} & $a+(b-a)U$ & -- \\
$\exponential(\lambda)$ & \eqref{E:Exponentialpdfcdf} & $\frac{-1}{\lambda} \log(1-U)$ &  $\frac{-1}{\lambda} \log(U)$ \\
$\laplace(\lambda)$ & \eqref{E:LaplaceInvcdf} & $- \frac{1}{\lambda} \ \sign\left( U-\frac{1}{2}\right) \log \left(1 - 2 \left| U-\frac{1}{2} \right| \right)$ & -- \\
$\cauchy$ & \eqref{E:StandardCauchycdf} & $\tan \left(\pi \left(U- \frac{1}{2} \right) \right)$ & $\tan \left(\pi U \right)$ \\
\hline
\end{tabular}
\end{center}
\end{table}
%}%end remove

Next, we familiarise ourselves with the Gaussian or $\normal$ RV.
\begin{model}[$\normal(\mu,\sigma^2)$]
$X$ has a $\normal(\mu,\sigma^2)$ or $\gaussian(\mu,\sigma^2)$ distribution with the location parameter $\mu \in \Rz$ and the scale or variance parameter $\sigma^2 > 0$, if:
\begin{equation}\label{E:Normalpdf}
f(x; \mu, \sigma^2) = \frac{1}{\sigma \sqrt{2 \pi}}
 \exp{\left( - \frac{1}{2 \sigma^2} (x-\mu)^2 \right)}, \qquad x \in \Rz
\end{equation}
$\normal(0,1)$ distributed RV, which plays a fundamental role in asymptotic statistics, is conventionally denoted by $Z$.  $Z$ is said to have the {\bf Standard Normal} distribution with PDF $f(z; 0,1)$ and DF $F(z;0,1)$ conventionally denoted by $\phi(z)$ and $\Phi(z)$, respectively.

There is no closed form expression for $\Phi(z)$ or $F(x;\mu,\sigma)$.  The latter is simply defined as:
\[
F(x;\mu,\sigma^2) = \int_{-\infty}^x f(y;\mu,\sigma)\,dy
\]
We can express $F(x;\mu,\sigma^2)$ in terms of the error function ($\erf$) as follows:
\begin{equation}\label{E:DFNormalviaErf}
F(x;\mu,\sigma^2) = \frac{1}{2} \ \erf \left(  \frac{x-\mu}{\sqrt{2 \sigma^2}} \right)+ \frac{1}{2}
\end{equation}
\end{model}
We
%\remove{
implement the PDF \eqref{E:Normalpdf} and DF \eqref{E:DFNormalviaErf} for a $\normal(\mu,\sigma^2)$ RV $X$ as \Matlab functions {\tt NormalPdf} and {\tt NormalCdf}, respectively, in \hyperref[Mf: NormalCdfPdf]{Labwork \ref*{Mf: NormalCdfPdf}},  and then
%}%end remove
produce plots for various $\normal(\mu,\sigma^2)$ RVs, shown in \hyperref[F:plotPdfCdfNormals]{Figure \ref*{F:plotPdfCdfNormals}}.  Observe the concentration of probability mass, in terms of the PDF and DF plots, about the location parameter $\mu$ as the variance parameter $\sigma^2$ decreases.
\begin{figure}[htpb]
\caption{Density and distribution function of several $\normal(\mu,\sigma^2)$ RVs.\label{F:plotPdfCdfNormals}}
\centering   \makebox{\includegraphics[width=6.5in]{figures/plotPdfCdfNormals}}
\end{figure}

% cse snipped
%\remove{
\begin{labwork}[Compute the  the $\P(X \in (a,b))$ for the $\normal(0,1)$ RV $X$]\label{LW:NormalIntervalProb}
Write a function to evaluate the $\P(X \in (a,b))$ for the $\normal(0,1)$ RV $X$ for user-specified values of $a$ and $b$. [Hint: one option is by making two calls to {\tt NormalCdf} and doing one arithmetic operation.]
\end{labwork}

Simulations \ref*{SIM:Uniformab} and \ref*{SIM:Exponential}, \ref*{SIM:Laplace} and \ref*{SIM:StdCauchy}
produce samples from a continuous RV $X$ with a closed-form expression for the inverse DF $F^{[-1]}$ via \hyperref[A:InvS]{Algorithm~\ref*{A:InvS}} (\hyperref[T:ContinRVsInvS]{Table~\ref*{T:ContinRVsInvS}}).  But only a few RVs have an explicit $F^{[-1]}$.  For example, $\normal(0,1)$ RV does not have an explicit $F^{[-1]}$.
\hyperref[A:InvSbyNumSol]{Algorithm~\ref*{A:InvSbyNumSol}} is a more general but inexact method that relies on an approximate numerical solution of $x$, for a given $u$, that satisfies the equation $F(x)=u$.

\begin{algorithm}
\caption{Inversion Sampler by Numerical Solution of $F(X)=U$ via Newton-Raphson Method}
\label{A:InvSbyNumSol}
\begin{algorithmic}[1]
\STATE {{\it input:} $F(x)$, the DF of the target RV $X$}
\STATE {{\it input:} $f(x)$, the density of $X$}
%\STATE {{\it input:} The fundamental sampler}
\STATE {{\it input:} A reasonable {\tt Stopping Rule}, \\e.g.~a specified tolerance $\epsilon >0$ and a maximum number of iterations {\tt MAX}}
\STATE {{\it input:} a careful mechanism to specify $x_0$}
%\STATE {\it initialise:} set the seed, if any, for the fundamental sampler
\STATE {\it output:} a sample from $X$ distributed according to $F$
\STATE {{\it draw:} $u \sim \uniform(0,1)$}
\STATE{{\it initialise:} $i \gets 0, \qquad x_i \gets x_0, \qquad x_{i+1} \gets x_0 - \frac{F(x_0)-u}{f(x_0)}$}
\WHILE{{\tt Stopping Rule} is not satisfied,\\e.g.~$|F(x_{i})-F(x_{i-1})| > \epsilon$ AND $i < {\tt MAX}$}
\STATE $x_i \gets x_{i+1}$
\STATE $x_{i+1} \gets \left( x_{i} - \frac{F(x_i)-u}{f(x_i)} \right)$
\STATE $i \gets i+1$
\ENDWHILE
\STATE{{\it return:} $x \gets x_i$}
\end{algorithmic}
\end{algorithm}

\begin{simulation}[$\normal(\mu,\sigma^2)$]\label{SIM:NormalByNewRap}
We may employ \hyperref[A:InvSbyNumSol]{Algorithm~\ref*{A:InvSbyNumSol}} to sample from the $Normal(\mu,\sigma^2)$ RV $X$ using the following function.
 \VrbMf[label=Sample1NormalByNewRap.m]{scripts/Sample1NormalByNewRap.m}

We draw five samples from the $\normal(0,1)$ RV $Z$ and store them in $z$ as follows.  The vector $z$ can be obtained by a Newton-Raphson-based numerical transformation of the vector $u$ of $5$ IID samples from the $\uniform(0,1)$ RV.  We simply need to apply the function {\tt Sample1NormalByNewRap} to each element of an array of $\uniform(0,1)$ samples.  \Matlab's {\tt arrayfun} command can be used to apply {\tt @(u)(Sample1NormalByNewRap(u,0,1))} (i.e., {\tt Sample1NormalByNewRap} as a function of $u$) to every element of our array of $\uniform(0,1)$ samples, say {\tt Us}.  Note that $F(z)$ is the same as the drawn $u$ from $U$ at least up to four significant digits.
\begin{VrbM}
>> rand('twister',563987);
>> Us=rand(1,5); % store 5 samples from Uniform(0,1) RV in array Us
>> disp(Us); % display Us
    0.8872    0.2569    0.5275    0.8650    0.8517
>> z=Sample1NormalByNewRap(Us(1),0,1); %transform Us(1) to a Normal(0,1) sample z
>> disp(z); % display z
    1.2119
>> z = arrayfun(@(u)(Sample1NormalByNewRap(u,0,1)),Us); %transform array Us via arrayfun
>> % dislay array z obtained from applying Sample1NormalByNewRap to each element of Us
>> disp(z);
    1.2119   -0.6530    0.0691    1.1031    1.0439
>> % check that numerical inversion of F worked, i.e., is F(z)=u ?
>> disp(NormalCdf(z,0,1));
    0.8872    0.2569    0.5275    0.8650    0.8517
\end{VrbM}
Next we draw five samples from the $\normal(-100.23,0.01)$ RV $X$, store it in an array $x$ and observe that the numerical method is reasonably accurate by the equality of $u$ and $F(x)$.
\begin{VrbM}
>> rand('twister',563987);
>> disp(Us); % display Us
    0.8872    0.2569    0.5275    0.8650    0.8517
>> % transform array Us via arrayfun
>> x = arrayfun(@(u)(Sample1NormalByNewRap(u,-100.23,0.01)),Us);
>> disp(x);
 -100.1088 -100.2953 -100.2231 -100.1197 -100.1256
>> disp(NormalCdf(x,-100.23,0.01));
    0.8872    0.2569    0.5275    0.8650    0.8517
\end{VrbM}
One has to be extremely careful with this approximate simulation algorithm implemented in floating-point arithmetic.  More robust samplers for the $\normal(\mu,\sigma^2)$ RV exist.
However, \hyperref[A:InvSbyNumSol]{Algorithm~\ref*{A:InvSbyNumSol}} is often the only choice when simulating from an arbitrary RV with an unknown closed-form expression for its $F^{[-1]}$.
\end{simulation}
%%% begin of informal CLT excursion
Next, we use our simulation capability to gain an informal and intuitive understanding of one of the most elementary theorems in probability and statistics, namely, the Central Limit Theorem (CLT).  We will see a formal treatment of CLT later.

Informally, the CLT can be stated as follows:\\
``The sample mean of a large number of IID samples, none of which is dominant, tends to the $\normal$ distribution as the number of samples increases.''

\begin{labwork}[Investigating the Central Limit Theorem with IID $\exponential(\lambda=0.1)$ RVs]\label{LW:CLTOfExponentials}
Let us investigate the histograms from $10000$ simulations of the sample mean of $n=10,100,1000$ IID $\exponential(\lambda=0.1)$ RVs as follows:
\begin{VrbM}
>> rand('twister',1973); % initialise the fundamental sampler
>> % a demonstration of Central Limit Theorem (CLT) -- Details of CLT are in the sequel
>> % the sample mean should be a Normal(1/lambda,lambda/n) RV
>> lambda=0.1; Reps=10000; n=10; hist(sum(-1/lambda * log(rand(n,Reps)))/n)
>> lambda=0.1; Reps=10000; n=100; hist(sum(-1/lambda * log(rand(n,Reps)))/n,20)
>> lambda=0.1; Reps=10000; n=1000; hist(sum(-1/lambda * log(rand(n,Reps)))/n,20)
\end{VrbM}
Do you see a pattern in the histograms?

See the histograms generated from the following code that produces sample means from the $\cauchy$ RV:
\begin{VrbM}
>> Reps=10000; n=1000; hist(sum(tan(pi * rand(n,Reps)))/n,20)
>> Reps=10000; n=1000; hist(sum(tan(pi * rand(n,Reps)))/n,20)
>> Reps=10000; n=1000; hist(sum(tan(pi * rand(n,Reps)))/n,20)
\end{VrbM}
\end{labwork}
\begin{classwork}[Why doesn't the sample mean of the $\cauchy$ RV ever settle down?]
Explain in words why the mean of $n$ IID samples from the $\cauchy$ RV ``is {\bf not} obeying'' the Central Limit Theorem.  Also relate it to \hyperref[F:plot5RunningMeansStandardcauchyUnif010]{Figure~\ref*{F:plot5RunningMeansStandardcauchyUnif010}} of \hyperref[LW:RunningMeanCauchy]{Labwork~\ref*{LW:RunningMeanCauchy}}.
\end{classwork}
%}% end remove

\begin{model}[$\gammA(\lambda,k)$ RV]
Given a shape parameter $\alpha>0$ and a rate parameter $\beta>0$, the RV $X$ is said to be $\gammA(\alpha,\beta)$ distributed if its PDF is:
\[
f(x;\alpha,\beta) = \frac{\beta^{\alpha}}{\Gamma(\alpha)} x^{\alpha-1} \exp(-\beta x), \qquad x > 0 \ ,
\]
where, the gamma function which interpolates the factorial function is:
\[
\Gamma(\alpha) := \int_0^{\infty} \exp(-y) y^{\alpha-1} dy \ .
\]
When $k \in \Nz$, then $\Gamma(k)=(k-1)!$.  The DF of $X$ is:
\[
F(x;\lambda,k) = \BB{1}_{\Rz_{>0}}(x) \frac{\beta^{\alpha}}{\Gamma(\alpha)} \int_0^{x} y^{\alpha-1} \exp(-\beta y) dy =
\begin{cases}
0 & \text{if } x \leq 0 \\
 \frac{\gamma(\alpha,\beta x)}{\Gamma(\alpha)} & \text{if } x > 0
\end{cases}
\]
where $\gamma(\alpha,\beta x)$ is called the lower incomplete Gamma function.
\end{model}
The expectation and variance of a $\gammA(\alpha,\beta)$ RV are $\alpha/\beta$ and $\alpha/\beta^2$, respectively.
%\remove{
The Gamma function and the incomplete Gamma function are available as \Matlab functions {\tt gamma} and {\tt gammainc}, respectively.  Thus, {\tt gamma(k)} returns $\Gamma(k)$ and {\tt gammainc(lambda*x,k)} returns $F({\tt x};{\tt lambda},{\tt k})$.  Using these functions, it is straightforward to evaluate the PDF and CDF of $X \sim \gammA(\lambda,k)$.  We use the following script to get a sense for the impact upon the PDF and CDF of the shape parameter $k$ as it ranges in $\{1,2,3,4,5\}$ for a given scale parameter $\lambda=0.1$.
\VrbMf[label=PlotPdfCdfGamma.m]{scripts/PlotPdfCdfGamma.m}
%}%end remove

\begin{figure}[htpb]
\caption{PDF and CDF of $X \sim \gammA(\beta=0.1,\alpha)$ with $\alpha \in \{1,2,3,4,5\}$.\label{F:PlotPdfCdfGamma}}
\centering   \makebox{\includegraphics[width=6.50in]{figures/PlotPdfCdfGamma}}
\end{figure}

Note that if $X \sim \gammA(1,\beta)$ then $X \sim \exponential(\beta)$, since:
\[
f(x;1,\beta)
= \frac{1}{(1-1)!}  \beta \exp(-\beta x) = \beta \exp(-\beta x) \ .
\]
More generally, if $X \sim \gammA(\alpha,\beta)$ and $\alpha \in \Nz$, then $X \sim \sum_{i=1}^{\alpha} Y_i$, where $Y_i \overset{IID}{\sim} \exponential(\beta)$ RVS, i.e.~ the sum of $\alpha$ IID $\exponential(\beta)$ RVs forms the model for the $\gammA(\alpha,\beta)$ RV.  If you model the inter-arrival time of buses at a bus-stop by IID $\exponential(\beta)$ RV, then you can think of the arrival time of the $k^{\text th}$ bus as a $\gammA(\alpha,\beta)$ RV.

%\section{Discrete Random Variables}
%}%end remove

%\remove{
\section*{Summary of Random Variables}\label{S:SummaryRVs}
\begin{table}[ht]
\centering
\begin{tabular}{|c|c|c|c|}%|c}
\hline
Model &PDF&Mean&Variance\\ \hline%&{\tt MGF}\\
%$\pointmass(\theta)$&$\BB{1}_{\{\theta\}}(x)$&$\theta$&$0$\\%&$e^{\theta t}$\\
$\bernoulli(\theta)$ &$\theta^x(1-\theta)^{1-x} \BB{1}_{\{0,1\}}(x)$&$\theta$&$\theta(1-\theta)$\\%&$\theta e^t+(1-\theta)$\\
$\binomial(n,\theta)$&$(^n_{\theta})\theta^x(1-\theta)^{n-x} \BB{1}_{\{0,1,\ldots,n\}}(x)$&$n \theta$&$n \theta(1-\theta)$\\%&$(\theta e^t+(1-\theta))^n$\\
$\geometric(\theta)$ &$\theta(1-\theta)^{x} \BB{1}_{\Zz_+}(x)$&$ \frac{1}{\theta}-1$&$\frac{1-\theta}{\theta^2}$\\%&$\frac{\theta e^t}{1-(1-\theta)e^t}(t<-log(1-\theta))$\\
$\poisson(\lambda)$&$\frac{\lambda^xe^{-\lambda}}{x!} \BB{1}_{\Zz_+}(x)$&$\lambda$&$\lambda$\\%&$e^{\lambda(e^t-1)}$\\
$\uniform(\theta_1,\theta_2)$&$\BB{1}_{[\theta_1,\theta_2]}(x)/(\theta_2-\theta_1)$&$\frac{\theta_1+\theta_2}{2}$&$\frac{(\theta_2-\theta_1)^2}{12}$\\%&$\frac{e^{\theta_2 t}-e^{\theta_1 t}}{(\theta_2-\theta_1)t}$\\
$\exponential(\lambda)$&$\lambda e^{-\lambda x}$&$\lambda^{-1}$&$\lambda^{-2}$\\%&$\frac{1}{1-\frac{1}{\lambda} t}(t<\lambda)$\\
$\normal(\mu,\sigma^2)$&$\frac{1}{\sigma\sqrt{2\pi}}e^{(x-\mu)^2/(2\sigma^2)}$&$\mu$&$\sigma^2$\\%&$exp\{ut+\frac{\sigma^2t^2}{2}\}$\\
$\gammA(\alpha,\beta)$&$\frac{\beta^{\alpha}}{\Gamma(\alpha)}{x^{\alpha-1}e^{-\beta x}}$&$\alpha/\beta$&$\alpha/\beta^2$\\%&$(\frac{1}{1-\beta t})^{\alpha}(t<1/\beta)$\\
%$\betA(\alpha,\beta)$ & $\frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha)\Gamma(\beta)}x^{\alpha-1}(1-x)^{\beta-1}$ & $\frac{\alpha}{\alpha+\beta}$ & $\frac{\alpha\beta}{(\alpha+\beta)^2 (\alpha+\beta+1)} $ \\%& $ 1+\Sigma^\infty_{k+1}(\Pi^{k-1}_{r=0}\frac{\alpha+r}{\alpha+\beta+r})\frac{t^k}{k\!}$\\
%$t_v$ & $\frac{\Gamma((v+1)/2)}{\Gamma(v/2)}  \frac{1}{(1+x^2/v)^{(v+1)/2}$ &0 (if $v>1$ ) & $\frac{v}{v-2}$  (if $v>2$) & does not exist \\
% $\chi^2_p$&$\frac{1}{\Gamma(p/2)2^{p/2}}x^{(p-2)-1}e^{-x/2}$&$p$&$2p$\\ \hline%&$(\frac{1}{1-2t})^{p/2}(t<1/2)$\\
\hline
\end{tabular}
\caption{Random Variables with PDF, Mean and Variance}
%\label{tab:}
\end{table}

We need a new notion for the variance of two RVs.
\begin{definition}[Covariance]
Suppose $X_1$ and $X_2$ are random variables, such that $\E(X_1^2) < \infty$ and $\E(X_2)^2 < \infty$.  Then, $\E(|X_1 X_2|) < \infty$ and $\E(|(X_1-\E(X_1))(X_2-\E(X_2))|) < \infty$.  We therefore define the covariance $\cv(X_1,X_2)$ of $X_1$ and $X_2$ as:
\[
\cv(X_1,X_2) := \E \left((X_1-\E(X_1))(X_2-\E(X_2))\right) = \E(X_1 X_2) - \E(X_1) \E(X_2)
\]
\end{definition}

Let us consider the natural two-dimensional analogue of the $\bernoulli(\theta)$ RV in the real plane $\Rz^2 := (-\infty,\infty)^2 := (-\infty,\infty) \times (-\infty,\infty)$.  A natural possibility is to use the {\bf ortho-normal basis vectors} in $\Rz^2$:
$$ \boxed{
e_1 := (1,0), \qquad e_2 := (0,1)
} \ .$$
Recall that vector addition and subtraction are done component-wise, i.e.~$(x_1,x_2) \pm (y_1,y_2) = (x_1\pm y_1,x_2 \pm y_2)$.

\begin{classwork}[Geometry of Vector Addition]
Recall elementary vector addition in the plane.  What is $(1,0)+(1,0)$, $(1,0)+(0,1)$, $(0,1)+(0,1)$?  What is the relationship between $(1,0)$, $(0,1)$ and $(1,1)$ geometrically? How does the diagonal of the parallelogram relate the its two sides in the geometry of addition in the plane?  What is  $(1,0)+(0,1)+(1,0)$?
\end{classwork}

\begin{figure}[htpb]
\caption{Quincunx on the Cartesian plane.  Simulations of $\binomial(n=10,\theta=0.5)$ RV as the x-coordinate of the ordered pair resulting from the culmination of sample trajectories formed by the accumulating sum of $n=10$ IID $\bernoulli(\theta=0.5)$ random vectors over $\{(1,0),(0,1)\}$ with probabilities $\{\theta,1-\theta\}$, respectively.  The blue lines and black asterisks perpendicular to and above the diagonal line, i.e.~the line connecting $(0,10)$ and $(10,0)$, are the density histogram of the samples and the PDF of our $\binomial(n=10,\theta=0.5)$ RV, respectively.\label{F:BinomQuincunxn10r10r1000}}
\centering
\mbox{\subfigure[Ten samples]{\hspace{-2cm} \includegraphics[width=3.250in]{figures/BinomQuincunxn10r10}} \hspace{-2cm}
	   \subfigure[Thousand samples]{\includegraphics[width=3.250in]{figures/BinomQuincunxn10r1000}} }
\end{figure}

\begin{model}[$\bernoulli(\theta)$ \rv]
Given a parameter $\theta \in [0,1]$, we say that $X := (X_1,X_2)$ is a $\bernoulli(\theta)$ random vector (\rv) if it has only two possible outcomes in the set $\{e_1,e_2\} \subset \Rz^2$, i.e.~$x:=(x_1,x_2) \in \{(1,0),(0,1)\}$.  The PMF of the \rv~$X:= (X_1,X_2)$ with realisation $x:=(x_1,x_2)$ is:
\[
f(x;\theta) := \P(X=x) = \theta \, \BB{1}_{\{e_1\}}(x) + (1-\theta) \, \BB{1}_{\{e_2\}}(x) =
\begin{cases}
\theta & \text{if } \quad x=e_1:=(1,0) \\
1- \theta & \text{if } \quad x=e_2:=(0,1) \\
0 & \text{otherwise}
\end{cases}
\]
\end{model}

\begin{classwork}[Expectation and Variance of $\bernoulli(\theta)$ \rv]
What is the Expectation of $\bernoulli(\theta)$ \rv?
\[
\E_{\theta}(X) = \E_{\theta}((X_1,X_2)) = \sum_{(x_1,x_2) \in \{e_1,e_2\}} (x_1,x_2) f((x_1,x_2);\theta) = (1,0) \theta + (0,1) (1-\theta) = (\theta,1-\theta) \ .
\]

How about the variance ? [Hint: Use the definitions of $\E(X)$ and $\V(X)$ for the \rv~$X$.  $\E(X^2)$ is not a single number and you may need new words such as covariance to deal with terms like $\E(X_1 X_2)$.]
\end{classwork}

%\remove{
We can write the $\binomial(n,\theta)$ RV $Y$ as a $\binomial(n,\theta)$ \rv~$X:=(Y,n-Y)$.  In fact, this is the underlying model and the {\bf bi} in the $\binomial(n,\theta)$ does refer to two in Latin.  In the coin-tossing context this can be thought of keeping track of the number of Heads and Tails out of an IID sequence of $n$ tosses of a coin with probability $\theta$ of observing Heads.  In the Quincunx context, this amounts to keeping track of the number of right and left turns made by the ball as it drops through $n$ levels of pegs where the probability of a right turn at each peg is independently and identically $\theta$.  In other words, the $\binomial(n,\theta)$ \rv~$(Y,n-Y)$ is the sum of $n$ IID $\bernoulli(\theta)$ {\rv}s $X_1:=(X_{1,1},X_{1,2}), X_2:=(X_{2,1},X_{2,2}), \ldots, X_n:=(X_{n,1},X_{n,2})$:
\[
(Y,n-Y) = X_1+X_2+\cdots + X_n = (X_{1,1},X_{1,2}) + (X_{2,1},X_{2,2}) + \cdots + (X_{n,1},X_{n,2})
\]

Go the Biomathematics Research Centre on the 6th floor of Erskine to play with the Quincunx built by Ryan Lawrence in 2007 (See the project by Ashman and Lawrence at  \href{http://www.math.canterbury.ac.nz/~r.sainudiin/courses/STAT218/projects/Stat218StudentProjects2007.pdf}{\url{http://www.math.canterbury.ac.nz/~r.sainudiin/courses/STAT218/projects/Stat218StudentProjects2007.pdf}} % \ref{S:AshmanLawrenceQuincunx}
for details).  It is important to gain a physical intimacy with the Quincunx to appreciate the following model of it.  We can make a statistical model of Galton's observations earlier regarding the dynamics of lead shots through the Quincunx as the sum of $n$ IID $\bernoulli(0.5)$ \rv{s}, where $n$ is number of pegs that each ball bounces on before making a left or right turn with equal probability.

\begin{exercise}[Number of paths and the binomial coefficient]
How does the number of paths that lead to a bucket $(x_1,x_2)$ with $x_1+x_2=n$ relate to the binomial coefficient $\binom{n}{x_1} ?$
%\vspace{2cm}
\end{exercise}

\begin{labwork}[Quincunx Sampler Demo -- Sum of $n$ IID $\bernoulli(1/2)$ \rv{s}]\label{LW:QuincunxSampler}
Let us understand the Quincunx construction of the $\binomial(n,1/2)$ \rv $X$ as the sum of $n$ independent and identical $\bernoulli(1/2)$ \rv{s} by calling the interactive visual cognitive tool as follows:
\begin{VrbM}
>> guiMultinomial
\end{VrbM}
The M-file {\tt guiMultinomial.m} will bring a graphical user interface (GUI) as shown in \hyperref[F:guiMultinomialQuincunx]{Figure \ref*{F:guiMultinomialQuincunx}}.  Using the drop-down menu at ``How many levels?'' change the number of levels to $2$ ($n=2$).  Now click the ``Do one'' button as many times as you like and comprehend the simulation process -- the path taken by the ball as it falls through two levels.  Next, from the drop-down menu at ``How many  Replication?'' change it from $10$ to $100$.  You can press ``Do all'' to watch all 100 balls drop into their possible values at level $2$.  Change the number of levels or $n$ in $\binomial(n,1/2)$ \rv to $3$ or $5$ or $10$ and do more simulations until you are comfortable with the construction that the sum of $n$ IID $\bernoulli(1/2)$ \rv{s} is the $\binomial(n,1/2)$ \rv.

When we drop $1000$ balls into the simulated Quincunx the density histogram is much closer to the PDF of $\binomial(n=10,\theta=0.5)$ RV than when we only drop $10$ balls.  See \hyperref[F:BinomQuincunxn10r10r1000]{Figure \ref*{F:BinomQuincunxn10r10r1000}} for a description of the simulations.  Try to replicate such a simulation on your own.
\end{labwork}

\begin{figure}[htpb]
\caption{Visual Cognitive Tool GUI: Quincunx.\label{F:guiMultinomialQuincunx}}
\centering   \makebox{\includegraphics[width=6.50in]{figures/guiMultinomialQuincunx}}
\end{figure}
%}%end remove

We are now ready to extend the $\binomial(n,\theta)$ RV or \rv~to its multivariate version called the $\multinomial(n,\theta_1,\theta_2,\ldots,\theta_k)$ \rv.  We develop this \rv~as the sum of $n$ IID $\demoivre(\theta_1,\theta_2,\ldots,\theta_k)$ \rv~that is defined next.

\begin{model}[$\demoivre(\theta_1,\theta_2,\ldots,\theta_k)$ \rv]\label{M:deMoivreRVec}
The PMF of the $\demoivre(\theta_1,\theta_2,\ldots,\theta_k)$ \rv~$X := (X_1,X_2,\ldots,X_k)$ taking value $x := (x_1,x_2,\ldots,x_k) \in \{e_1,e_2,\ldots,e_k\}$, where the $e_i$'s are ortho-normal basis vectors in $\Rz^k$ is:
\[
f(x;\theta_1,\theta_2,\ldots,\theta_k) := \P(X=x) = \sum_{i=1}^k \theta_i \BB{1}_{\{e_i\}}(x) =
\begin{cases}
\theta_1 & \text{if} \quad x=e_1:=(1,0,\ldots,0) \in \Rz^k \\
\theta_1 & \text{if} \quad x=e_1:=(0,1,\ldots,0) \in  \Rz^k \\
\vdots \\
\theta_k & \text{if} \quad x=e_k:=(0,0,\ldots,1) \in  \Rz^k \\
0 & \text{otherwise}
\end{cases}
\]
Of course, $\sum_{i=1}^k \theta_i = 1$.
\end{model}

When we add $n$ IID $\demoivre(\theta_1,\theta_2,\ldots,\theta_k)$ \rv~together, we get the $\multinomial(n,\theta_1,\theta_2,\ldots,\theta_k)$ \rv~as defined below.

\begin{model}[$\multinomial(n,\theta_1,\theta_2,\ldots,\theta_k)$ \rv]\label{M:Multinomial}
We say that a \rv~$Y:=(Y_1,Y_2,\ldots,Y_k)$ obtained from the sum of $n$ IID $\demoivre(\theta_1,\theta_2,\ldots,\theta_k)$ \rv{s} with realisations
$$y:=(y_1,y_2,\ldots,y_k) \in \Yz:= \{(y_1,y_2,\ldots,y_k) \in \Zz_+^k : \sum_{i=1}^k y_i = n\}$$ has the PMF given by:
\[
f(y;n,\theta) := f(y;n,\theta_1,\theta_2,\ldots,\theta_k) := \P(Y=y;n,\theta_1,\theta_2,\ldots,\theta_k) = \binom{n}{y_1,y_2,\ldots,y_k} \prod_{i=1}^k \theta_i^{y_i} \ ,
\]
where, the multinomial coefficient:
\[
 \binom{n}{y_1,y_2,\ldots,y_k} := \frac{n!}{y_1! y_2! \cdots y_k!} \ .
\]
Note that the marginal PMF of $Y_j$ is $\binomial(n,\theta_j)$ for any $j=1,2,\ldots,k$.
\end{model}

\begin{figure}[htpb]
\caption{Septcunx on the Cartesian co-ordinates.  Simulations of $\multinomial(n=2,\theta_1=1/3,\theta_2=1/3,\theta_3=1/3)$ \rv~as the sum of $n$ IID $\demoivre(\theta_1=1/3,\theta_2=1/3,\theta_3=1/3)$ \rv{s} over $\{(1,0,0),(0,1,0),(0,0,1)\}$ with probabilities $\{\theta_1,\theta_2,\theta_3\}$, respectively.  The blue lines perpendicular to the sample space of the $\multinomial(3,\theta_1,\theta_2,\theta_3)$ \rv, i.e.~the plane in $\Rz^3$ connecting $(n,0,0)$, $(0,n,0)$ and $(0,0,n)$, are the density histogram of the samples.\label{F:MultinomSeptcunxn2n10r1000}}
\centering
\mbox{\subfigure[Thousand Samples with $n=2$]{\hspace{-2cm} \includegraphics[width=4.250in]{figures/MultinomSeptcunxn2r1000}} \hspace{-2cm}
	   \subfigure[Thousand Samples with $n=10$]{\includegraphics[width=4.250in]{figures/MultinomSeptcunxn10r1000}} }
\end{figure}

We can visualise the $\multinomial(n,\theta_1,\theta_2,\theta_3)$ process as a sum of $n$ IID $\demoivre(\theta_1,\theta_2,\theta_3)$ \rv{s} via a three dimensional extension of the Quincunx called the ``Septcunx'' and relate the number of paths that lead to a given trivariate sum $(y_1,y_2,y_3)$ with $\sum_{i-1}^3 y_i = n$ as the multinomial coefficient $\frac{n!}{y_1! y_2! y_3!}$.  In the Septcunx, balls choose from one of three paths along $e_1$, $e_2$ and $e_3$ with probabilities $\theta_1$, $\theta_2$ and $\theta_3$, respectively, in an IID manner at each of the $n$ levels, before they collect at buckets placed at the integral points in the $3$-simplex, $\Yz = \{(y_1,y_2,y_3) \in \Zz_+^3 : \sum_{i=1}^3 y_i=n \}$.  Once again, we can visualise that the sum of $n$ IID $\demoivre(\theta_1,\theta_2,\theta_3)$ \rv{s} constitute the $\multinomial(n,\theta_1,\theta_2,\theta_3)$ \rv~as depicted in \hyperref[F:MultinomSeptcunxn2n10r1000]{Figure \ref*{F:MultinomSeptcunxn2n10r1000}}.

%\remove{
\begin{labwork}[Septcunx Sampler Demo -- Sum of n IID $\demoivre(1/3,1/3,13/)$ \rv{s}]\label{LW:SeptcunxSampler}
Let us understand the Septcunx construction of the $\multinomial(n,1/3,1/3,1/3)$ \rv $X$ as the sum of $n$ independent and identical $\demoivre(1/3,1/3,13/)$ \rv{s} by calling the interactive visual cognitive tool as follows:
\begin{VrbM}
>> guiMultinomial
\end{VrbM}
The M-file {\tt guiMultinomial.m} will bring a GUI as shown in \hyperref[F:guiMultinomialQuincunx]{Figure \ref*{F:guiMultinomialQuincunx}}.  Using the drop-down menu at ``How many dimensions?'' change to ``3-d (the septcunx)'' and you will see a septcunx as shown in \hyperref[F:guiMultinomialSeptcunx]{Figure \ref*{F:guiMultinomialSeptcunx}}.  Next, using the drop-down menu at ``How many levels?'' change the number of levels to $2$ ($n=2$).  Now click the ``Do one'' button as many times as you like and comprehend the simulation process -- the path taken by the ball as it falls through two levels in three dimensional space.  Feel free to change the up-down and left-right sliders for the view angles.  Next, from the drop-down menu at ``How many  Replication?'' change it from $10$ to $100$.  You can press ``Do all'' to watch all 100 balls drop into their possible values at level $2$.  Change the number of levels or $n$ in $\multinomial(n,1/3,1/3,1/3)$ \rv to $5$ or $10$ and do more simulations until you are comfortable with the construction that the sum of $n$ IID $\demoivre(1/3,1/3,1/3)$ \rv{s} is the $\multinomial(n,1/3,1/3,1/3)$ \rv.
\end{labwork}

\begin{figure}[htpb]
\caption{Visual Cognitive Tool GUI: Septcunx.\label{F:guiMultinomialSeptcunx}}
\centering   \makebox{\includegraphics[width=6.50in]{figures/guiMultinomialSeptcunx}}
\end{figure}

\begin{labwork}[PDF of $\multinomial(n,\theta)$ \rv]\label{LW:MultinomialPdf}
We can implement the following \Matlab function {\tt MultinomialPdf} to compute the PDF of the $\multinomial(n,\theta)$ \rv~where $\theta:=(\theta_1,\theta_2,\ldots,\theta_k)$ is a point in the $k$-simplex $\bigtriangleup_k$ as follows:
\VrbMf[label=MultinomialPdf.m]{scripts/MultinomialPdf.m}
We can call this function to evaluate the PDF at a specific sample $x=(x_1,x_2,\ldots,x_k)$ as follows:
\begin{VrbM}
>> MultinomialPdf([2 0 0],2,[1/3 1/3 1/3])
ans =    0.1111
>> MultinomialPdf([0 2 0],2,[1/3 1/3 1/3])
ans =    0.1111
>> MultinomialPdf([0 0 2],2,[1/3 1/3 1/3])
ans =    0.1111
>> MultinomialPdf([1 1 0],2,[1/3 1/3 1/3])
ans =    0.2222
>> MultinomialPdf([1 0 1],2,[1/3 1/3 1/3])
ans =    0.2222
>> MultinomialPdf([0 1 1],2,[1/3 1/3 1/3])
ans =    0.2222
\end{VrbM}
\end{labwork}

\begin{simulation}[A simple multinomial simulation]\label{SIM:Multinomial}
Using the identity matrix $I$ in $\Rz^3$ that can be created in \Matlab using the {\tt eye(3)} command, and the $\demoivre(1/3,1/3,1/3)$ RV sampler, simulate vector-valued samples from $\demoivre(1/3,1/3,1/3)$ \rv.  Finally add up $n=10$ samples from $\demoivre(1/3,1/3,1/3)$ \rv~to produce samples from $\multinomial(10,1/3,1/3,1/3)$ \rv.
\end{simulation}

%%% begin Dominic's Material
%%% WORK: We need to introduce sampling from a large discrete distribution, say the Alias method before this material % theoretically explain randsample function in Matlab
\remove{
\section{Importance Resampler}
The rejection method cannot be used when the constant $a$ or $\tilde{a}$ that guarantees the envelape condition cannot be found. The importance resampler, also known as the method of {sampling/importance resampling}, does not require the constant, but it produces a random variable that is only approximately distributed according to $f$. As for the rejection method, we need a density/mass function $g$ that we can generate from and that has support at least as large as the support of $f$.

\begin{algorithm}
\caption{Importance Resampler}
\label{A:ImpReSampler}
\begin{algorithmic}[1]
\STATE {
{\it input:}
\begin{itemize}
\item[(1)] shape of a target density $\tilde{f}(x) = \left({\int \tilde{f}(x)dx}\right) f(x)$,
\item[(2)] a proposal density $g(x)$ satisfying only (a) and (b) above.
\item[(3)] a large enough integer $m$.
\end{itemize}
}
\STATE {\it output:} a sample $x{\prime}$ from RV $X^{\prime}$ with density $f^{\prime}$ that is close to  $f$
\STATE Generate $y_1,\ldots, y_m \sim g$
\STATE Compute
$$
w_i=\frac{f(y_i)/g(y_i)}{\sum^m_{j=1}f(y_j)/g(y_j)},i=1,\ldots,m \enspace .
$$
\STATE Resample $x^{\prime}$ from $\{y_1,\ldots, y_m\}$ with weights $\{w_1,\ldots, w_m\}$
\end{algorithmic}
\end{algorithm}

\begin{prop}The Importance Resampler of Algorithm~\ref{A:ImpReSampler} produces samples from a variable $X^{\prime}$ that is approximately distributed according to $f$, in the sense that:
\begin{equation}
\lim_{m\rightarrow \infty}\P( X^{\prime} \leq t)=\int^t_{-\infty}f(x)dx
\end{equation}
for any real number $t$.
\begin{proof}
\begin{displaymath}
\begin{split}
&\P(X^{\prime}\leq t)=\sum^m_{i=1}w_iI_{(-\infty,t]}(y_i)=\frac{\frac{1}{m}\sum^m_{i=1}\frac{f(y_i)}{g(y_i)}I_{(-\infty,t]}(y_i)}{\frac{1}{m}\sum^m_{i=1}\frac{f(y_i)}{g(y_i)}}\\
%\end{displaymath}
%\begin{displaymath}
&\underrightarrow{^{m\rightarrow\infty}}\frac{E [\frac{f(y)}{g(y)}I_{(-\infty,t]}(y)]}{E[\frac{f(y)}{g(y)}]}=\frac{\int^t_{-\infty}f(y)dy}{{\int^{\infty}_{-\infty}f(y)dy}}=\int^t_{-\infty}f(y)dy\\
\end{split}
\end{displaymath}
\end{proof}
\end{prop}

Let us visualise the Importance Resampler in action from \hyperref[Mf:ImpResamplerCauchyViaNormal]{Labwork~\ref*{Mf:ImpResamplerCauchyViaNormal}}.

\begin{labwork}[$\cauchy$ RV via Importance Resampler]
Use the sampling/importance resampling method to generate $1000$ approximate $\cauchy$ samples  by using the $\normal(0,1)$ samples:

$$f(x)=\frac{1}{\pi(1+x^2)} \textrm{ and  } g(x)=\frac{1}{\sqrt{2\pi}}\exp\left(-\frac{x^2}{2}\right).$$

\begin{VrbM}
n = 1000;
m = 10000;
y = randn(1,m); % randn is the N(0,1) generator in Matlab
y2 = y .* y;
w = exp(0.5 * y2) ./ (1 + y2);
w = w / sum(w);
x = randsample(y,n,true,w); % resample n values from y weighted by w
\end{VrbM}
\end{labwork}

Note that to get $n$ sample points from $f$ using sampling/importance resampling, we must start with a sample from $g$ of size $m$ larger than $n$.

As for the rejection method, the sampling/importance resampling method can still be used if only the un-normalised form of $f$ or $g$ (or both) is known, simply by using the un-normalised densities/mass functions to compute the weights.
%%%%%%end Dominic's material
}

\section{Other Continuous Random Variables}\label{S:OtherContRVs}
Here, we see other common continuous RVs that can be simulated from transforming RVs we have already encountered.
%}%end remove
%\remove{
\begin{simulation}[$\gammA(\lambda,k)$ for integer $k$]\label{SIM:Gamma}
Using this relationship we can simulate from $X \sim \gammA(\lambda,k)$, for an integer-valued $k$, by simply summing $k$ IID samples from $\exponential(\lambda)$ RV as follows:
\begin{VrbM}
>> lambda=0.1; %declare some lambda parameter
>> k=5; % declare some k parameter (has to be integer)
>> rand('twister',7267); % initialise the fundamental sampler
>> % sum k IID Exponential(lambda) samples for one desired sample from Gamma(lambda,k)
>> x= sum(-1/lambda*log(rand(k,1)))
x =   28.1401
>> % sum the 10 columns of k X 10 IID Exponential(lambda) samples for 10 desired samples from Gamma(lambda,k)
>> x= sum(-1/lambda*log(rand(k,10)))
x =
   83.8150   61.2674   80.3683  103.5748   48.4454   20.2269   93.8310   56.1909   77.0656   29.0851
\end{VrbM}
\end{simulation}

\begin{model}[$\lognormal(\lambda,\zeta)$]
$X$ has a $\lognormal(\lambda,\zeta) $ distribution if $\log(X)$ has a $\normal(\lambda,\zeta^2)$ distribution.  The location parameter $\lambda = \E(\log(X)) > 0$ and the scale parameter $\zeta > 0$.  The PDF is:
\begin{equation}\label{E:LogNormalpdf}
f(x; \lambda, \zeta) = \frac{1}{\sqrt{2 \pi} \zeta x }
 \exp{\left( - \frac{1}{2 \zeta^2} (\log(x)-\lambda)^2 \right)}, \qquad x > 0
\end{equation}
No closed form expression for $F(x;\lambda,\zeta)$ exists and it is simply defined as:
\[
F(x;\lambda,\zeta) = \int_{0}^x f (y;\lambda,\zeta)\,dy
\]
We can express $F(x;\lambda,\zeta) $ in terms of $\Phi$ (and, in turn, via the associated  error function $\erf$) as follows:
\begin{equation}\label{E:DFLogNormalviaErf}
F(x;\lambda,\zeta) = \Phi \left( \frac{\log(x) - \lambda}{\zeta} \right) = \frac{1}{2} \ \erf \left(  \frac{\log(x)-\lambda}{\sqrt{2}\zeta} \right)+ \frac{1}{2}
\end{equation}
\end{model}
%We implement the pdf \eqref{E:LogNormalpdf} and DF \eqref{E:DFLogNormalviaErf} for a $\normal(\mu,\sigma^2)$ RV $X$ as \Matlab functions {\tt NormalPdf} and {\tt NormalCdf}, respectively, in \hyperref[Mf: NormalCdfPdf]{Labwork \ref*{Mf: NormalCdfPdf}}  and then render their plots for various $\normal(\mu,\sigma^2)$ RVs in \hyperref[F:plotPdfCdfNormals]{Figure \ref*{F:plotPdfCdfNormals}}.

\begin{labwork}[Simulations with the $\lognormal(\lambda_C,\zeta_C)$ RV]\label{LW:lognormal}
Transform a sequence of samples obtained from the fundamental sampler to those from the $\lognormal(\lambda_C,\zeta_C)$ RV $C$ by using only  \hyperref[A:InvSbyNumSol]{Algorithm~\ref*{A:InvSbyNumSol}} or \Matlab's {\tt randn} as an intermediate step.   [Hint: %Example 4.13 in Ang \& Tang p.166-167.  There is a typo in this Example.  Note:
If $Y$ is a $\normal(\lambda,\zeta^2)$ RV, then $Z=e^Y$ is said to be a $\lognormal(\lambda,\zeta)$ RV.  ]
\begin{enumerate}
\item Seed the fundamental sampler by your Student ID,
\item generate $1000$ samples from an RV $C\sim \lognormal(\lambda=10.36, \zeta=0.26)$ by exponentiating the samples from the $\normal(10.36,0.26^2)$ RV and
\item and report:
\begin{enumerate}
\item how many of the samples are larger than $35000$,
\item the sample mean, and
\item the sample standard deviation.
\end{enumerate}
\end{enumerate}

\end{labwork}

Beta RV

Chi-Square

F distribution

t-distribution

Weibul

Heavy-tail family

\section{Other Random Vectors}\label{S:OtherRVecs}

Multivariate Normal

Uniform Distribution on Sphere

Dirichlet Distribution
%}% end remove

\begin{ExerciseList}
\Exercise
{**}The covariance of two random variables $X$ and $Y$ is defined as
\[
\cv(X,Y) := \E \left((X-\E(X))(Y-\E(Y))\right) = \E(X Y) - \E(X) \E(Y) \enspace .
\]
\begin{itemize}
\item[(a)] Show, starting from the definition, that $\cv(X,Y) = \E(XY)-\E(X) \E(Y)$.
\item[(b)] When $\cv(X,Y)=0$, $X$ and $Y$ are said to be ``uncorrelated''.
Show that if $X$ and $Y$ are independent, then they are also uncorrelated.
\end{itemize}

\Exercise
{**} Let $X_1,X_2,\ldots,X_n$ be random variables.
Their joint CDF is defined as
\[
F(x_1,x_2,\ldots,x_n) := \P (X_1 \leq x_1, X_2 \leq x_2, \ldots, X_n \leq x_n) \enspace.
\]
By repeated application of the definition of conditional probability, show that the joint CDF admits the following ``telescopic'' representation:
\begin{eqnarray*}
F(x_1,x_2,\ldots,x_n)
&=&
F(x_n \mid x_1,\ldots,x_{n-1}) F(x_{n-1} \mid x_1,\ldots,x_{n-2})\cdots F(x_2 \mid x_1) F(x_1)\\
&=&
F(x_1) \prod_{i=2}^n F(x_i \mid x_1,\ldots,x_{i-1}) \enspace ,
\end{eqnarray*}
where, $F(x_i \mid x_1,\ldots,x_{i-1})$ denotes the conditional probability, $\P(X_i \leq x_i \mid X_1 \leq x_1,\ldots,X_{i-1} \leq x_{i-1})$.
\end{ExerciseList}

%infinite coin-tosses and the fundamental model
%\remove{
\section{Problems}
\begin{exercise}
If $u\sim U[0,1]$, show that the distribution of $1-u$ is also $U[0,1]$.
\end{exercise}

\begin{exercise}
Write a Matlab function to generate n random variables from the distribution with the following mass function:
$$\begin{array}{|c|c|c|c|c|c|}\hline
x	&1.7	&3.4	&5.9	&7.2&	9.6\\ \hline
f(x)	&0.15&	0.4	&0.05	&0.1	&0.3\\ \hline
\end{array}$$

Use your Matlab function to generate 1000 sample values from the distribution, and compare the relative frequencies obtained with the mass function probabilities.
\end{exercise}

\begin{exercise}
The Laplacian distribution is also called the double exponential distribution because it can be regarded as the extension of the exponential distribution for both positive and negative values. An easy way to generate a Laplacian$(0, 1)$ random variable is to generate an exponential(1) random variable and then change its sign to negative with probability 0.5. Write a Matlab function to generate $n$ Laplacian$(0, 1)$ random variables using the {\tt expornd} function from Exercise 2.6.5. Call your function {\tt laprnd}. It should take $n$ as input and produce a row vector containing the $n$ Laplacian$(0, 1)$ random variables as output.
\end{exercise}

\begin{exercise}
\begin{asparaenum}[(a)]
\item	Referring to Example 2.2.3, write a \Matlab function to generate $n$ $N(0, 1)$ random variables using the rejection method with the Laplacian$(0, 1)$ distribution. Include a counter for the number of iterations in your function.

\item	Use your Matlab function to generate 1000 $N(0, 1)$ random variables. Plot the density histogram for your generated values and superimpose the $N(0, 1)$ density onto it. Compare the average number of iterations to get a single $N(0, 1)$ random variable with the constant a.

\item	Now suppose that we know only the un-normalised $N(0, 1)$ and Laplacian$(0, 1)$ densities, i.e.:
$$\tilde{f}(x)=\exp\left( -\frac{x^2}{2}\right)\textrm{ and }\tilde{g}(x)=\exp(-|x|)$$

What is the constant $\tilde{a}$ for the rejection method in this case? Implement the rejection method in Matlab, including a counter for the number of iterations, and use it to generate 1000 $N(0, 1)$ random variables. Compare the average number of iterations to get a single $N(0, 1)$ random variable with $a$ and $\tilde{a}$.
\end{asparaenum}
\end{exercise}

\begin{exercise}
Consider (Ross, p.64.) the use of the rejection method to generate from the density:
$$f(x)=20x(1-x)^3.$$
for $0 \leq x\leq   1$, using the $U (0, 1)$ distribution as proposal distribution.
\begin{asparaenum}[(a)]
\item Show that the constant for using the rejection method is $a = 2.1094$.

\item Write a \Matlab function to generate n random variables from $f$ using the rejection method. Include a counter for the number of iterations in your function.

\item Use your \Matlab function to generate 1000 random variables from $f$. Plot the density histogram for your generated values and superimpose the density curve onto it. Compare the average number of iterations to get a single random variable with the constant $a$.

\end{asparaenum}
\end{exercise}

\begin{exercise}
Consider (Ross, .p65.) the use of the rejection method to generate from the density:
$$f(x)=\frac{2}{\sqrt{\pi}}x^{1/2}e^{-x}$$
for $x\geq 0$, and using the exponential distribution with mean $m$ as proposal distribution.

\begin{asparaenum}[(a)]
\item Show that the constant for using the rejection method is:
$$a=\sqrt{\frac{2}{\pi e}}\frac{m^{3/2}}{(m-1)^{1/2}}$$

\item Show that the best exponential distribution to use is the one with a mean of 3/2.

\item	Write a \Matlab function to generate $n$ random variables from $f$ using the rejection method. Include a counter for the number of iterations in your function.

\item	Use your \Matlab function to generate 1000 random variables from $f$. Plot the density histogram for your generated values and superimpose the density curve onto it. Compare the average number of iterations to get a single random variable with the constant $a$.
\end{asparaenum}
\end{exercise}

\begin{exercise}
\begin{asparaenum}[(a)]
\item Referring to Example 2.3.3, implement the Matlab function to generate 1000 approximate Cauchy$(0,1)$ random variables using sampling/importance resampling, starting with $m = 10,000$ $N(0,1)$ sample values. Plot the density histogram for your generated values and superimpose the Cauchy$(0,1)$ density onto it.

\item Explore what happens if you start with
\begin{inparaenum}[(i)]
\item $m = 1000 N(0,1)$ sample values,
\item $m = 100000 N(0,1)$ sample values.
\end{inparaenum}
\end{asparaenum}
\end{exercise}

\begin{exercise}
Write a \Matlab function to generate 1000 approximate Laplacian$(0,1)$ random variables using sampling/importance resampling with the $N(0,1)$ distribution. Plot the density histogram for your generated values and superimpose the Laplacian$(0,1)$ density onto it.
\end{exercise}

\begin{exercise}
Implement the RWMH sampler in Example 2.4.8. Perform 10,000 iterations and plot the outputs sequentially. Comment on the appearance of the plot with regard to convergence to the target density. Plot the density histogram for the last 5000 iterations and superimpose the target density onto it. Investigate what happens when $g(\cdot|x)=U(x-c,x+c)$ is used as the proposal density with different values of $c$ that are smaller or larger than 1. (Note: In \Matlab, the modified Bessel function of the first kind is available as {\tt besseli}.)
\end{exercise}
%}


\section{Definition}
\begin{definition}
	Let $f(t)$ be a function that is defined for $t \geq 0$.
	\begin{equation*}
		\Laplace{f} = \int_{0}^{\infty}{e^{-st}f(t) \mathrm{d}t}.
	\end{equation*}
	The domain of $F$ is all values of $s$ where the integral is defined and finite.
\end{definition}

\input{./laplaceTransforms/definition/stillLinear.tex}


\documentclass[a4j]{jarticle}
\usepackage{fancybox,ascmac,amsmath,amssymb,graphicx}
\begin{document}

\section{Design of Butterworth Lowpass Filter}

\begin{itembox}[l]{{\large {\bf Butterworth Approximation}}}
1. Define specification of the filter, \( H_0 \): DC Gain, \( H_c \): Gain at cutoff frequency, \( H_s \) Gain at stopband edge frequency, \( \omega_c \): Cutoff frequency (rad/s), \( \omega_s \): Stopband frequency (rad/s).
Also choose the optimization strategy: stopband edge frequency gain optimized or passband edge gain optimized.\\
2. Calculate normalized cutoff frequency \( \Omega_s \)
\begin{eqnarray*}
\Omega_{c} &=& 1 \\
\Omega_{s} &=& \frac{\omega_s}{\omega_c}
\end{eqnarray*}
3. Calculate \( \beta = \beta_{max} \) on stopband edge frequency gain optimized or \( \beta = \beta_{min} \) on passband edge gain optimized.
\begin{eqnarray*}
\beta_{max} = \sqrt{\left(\frac{H_0}{H_c}\right)^2-1}\\
\beta_{min} = \sqrt{\frac{\left(\frac{H_0}{H_s}\right)^2-1}{\Omega_s^2}}
\end{eqnarray*}
4. Calculate the order of the filter \( N \) by rounding up \( n_{fmin} \) to the next integer:
\begin{eqnarray*}
n_{fmin} &=& \frac{\log\left(\frac{{\frac{H_0^2}{H_s^2}-1}}{{\frac{H_0^2}{H_c^2}-1}}\right)}{2\log\Omega_s}
\end{eqnarray*}
5. Calculate \( s_{k+} \), the roots of the transfer function denominator polynomial equation:
\begin{eqnarray*}
s_{k+} &=& \sqrt[N]{\frac{1}{\beta}}\mathrm{e}^{j\left(\frac{2k+1}{2N}\pi+\frac{\pi}{2}\right)}\quad k=0,1, \cdots N-1
\end{eqnarray*}
\end{itembox}

\clearpage

{\large {\bf Example 1}} Determine the transfer function of Butterworth lowpass filter of stopband edge frequency gain optimized,
 \( H_0 \) = 1, \( H_c \) = 0.891 ( = -1dB) , \( H_s \) = 0.00398 ( = -48dB) ,
 Cutoff frequency = 125664 (=20kHz), Stopband edge frequency = 1108353 (=176.4kHz) \\

{\large {\bf 1. Perform Butterworth Approximation}}

\begin{eqnarray*}
\Omega_{c} &=& 1 \\
\Omega_{s} &=& \frac{\omega_s}{\omega_c} = 8.82 \\
n_{fmin} &=& 2.85 \\
N &=& Ceiling(2.85) = 3 \\
\beta_{max} &=& 0.509 \\
\textmc{angle of } s_k &=& \frac{2k+1}{2N}\pi + \frac{\pi}{2} \\
\textmc{magnitude of } s_k &=& \beta^{\frac{-1}{N}} \\
s_k &=& magnitude \left\{ \cos( angle ) + i\sin( angle) \right\} \\
s_0 &=& 1.25 \left\{ \cos(2.09) + i\sin(2.09) \right\} \\
s_1 &=& 1.25 \left\{ \cos(\pi) + i\sin(\pi) \right\} = -1.25\\
s_2 &=& 1.25 \left\{ \cos(4.19) + i\sin(4.19) \right\} \\
\textmc{Transfer function} H(s) &=& \frac{1}{(s-1.25 \left\{ \cos(2.09) + i\sin(2.09) \right\})(s+1.25)(s-1.25 \left\{ \cos(4.19) + i\sin(4.19) \right\})}
\end{eqnarray*}

{\large {\bf 2. Partial fraction decomposition}}
\begin{eqnarray*}
H(s) &=& \frac{1}{(s-1.25 \left\{ \cos(2.09) + i\sin(2.09) \right\})(s+1.25)(s-1.25 \left\{ \cos(4.19) + i\sin(4.19) \right\})}\\
     &=& \frac{c_1}{s-1.25 \left\{ \cos(2.09) + i\sin(2.09) \right\}} + \frac{c_2}{s+1.25} + \frac{c_3}{s-1.25 \left\{ \cos(4.19) + i\sin(4.19) \right\}}\\
c_1  &=& \frac{1}{(s+1.25)(s-1.25 \left\{ \cos(4.19) + i\sin(4.19) \right\})}, s = 1.25 \left\{ \cos(2.09) + i\sin(2.09) \right\} \\
     &=& -0.6263-0.3616i \\
c_2  &=& \frac{1}{(s-1.25 \left\{ \cos(2.09) + i\sin(2.09) \right\})(s-1.25 \left\{ \cos(4.19) + i\sin(4.19) \right\})}, s = -1.25 \\
     &=& 1.253 \\
c_3  &=& \frac{1}{(s-1.25 \left\{ \cos(2.09) + i\sin(2.09) \right\})(s+1.25)}, s = 1.25 \left\{ \cos(4.19) + i\sin(4.19) \right\} \\
     &=& -0.6263+0.3616i \\
\therefore H(s) &=& \frac{-0.6263-0.3616i}{s+0.6263-1.085i} + \frac{1.253}{s+1.253} + \frac{-0.6263+0.3616i}{s+0.6263+1.085i}
\end{eqnarray*}

\clearpage

{\large {\bf 3. Multiply conjugate complex 1st order rational polynomial to get 2nd order rational polynomial with real value coefficients}}

\begin{eqnarray*}
H(s) &=& \frac{-0.6263-0.3616i}{s+0.6263-1.085i} + \frac{1.253}{s+1.253} + \frac{-0.6263+0.3616i}{s+0.6263+1.085i} \\
     &=& \frac{1.253}{s+1.253} + \frac{-0.6263-0.3616i}{(s+0.6263-1.085i)(s+0.6263-1.085i)} \\
     &=& \frac{1.253}{s+1.253} + \frac{0.523}{s^2+1.253s+1.569}
\end{eqnarray*}

{\large {\bf 4. Create lossy integrator from 1st order rational polynomial}}

\begin{eqnarray*}
H_{s1} &=& \frac{A}{s+a} \\
\omega_c &=& 20000 * 2 * \pi = 125663 \\
R_{n0} &=& 1 \\
C_{n0} &=& \frac{a}{\omega_c * R_0} \\
R_{0}  &=& R_{n0} * 10000 = 10 (k\Omega) \quad  \textmc{(frequency scaling)}\\
C_{0}  &=& \frac{C_0}{10000} = 996 \textmc{(pF)} \quad  \textmc{(frequency scaling)} \\
\end{eqnarray*}

Bug: resulted circuit does not reflect A of the rational polynomial! \\

\begin{figure}
    \centering
    \includegraphics[width=90mm,height=50mm,natwidth=900,natheight=500]{circuit.png} \\
Figure 1
\end{figure}

{\large {\bf 5. Create Sallen-Key lowpass filter from 2nd order rational polynomial}}

\begin{eqnarray*}
H_{s2}   &=& \frac{A}{s^2+as+b} \\
\omega_0 &=& \sqrt{b} \\
\omega_c &=& 20000 * 2 * \pi = 125663 \\
Q        &=& \frac{\omega_0}{a} \\
C_{n2}   &=& 1 \\
C_{n1}   &=& \sqrt{3}*Q*C_{n2} \\
R_{n1}   &=& \frac{1}{\omega_0 * Q * C_{n2}} \\
R_{n2}   &=& \frac{1}{\sqrt(3)*\omega_0 *C_{n2}} \\
C_{1}    &=& \frac{C_{n1}}{\omega_c * 10000} = 1.38 \textmc{(nF)} \quad \textmc{(frequency scaling)} \\
C_{2}    &=& \frac{C_{n2}}{\omega_c * 10000} = 796 \textmc{(nF)} \quad \textmc{(frequency scaling)}  \\
R_{1}    &=& R_{n1} * 10000 = 7.98 (k\Omega) \quad \textmc{(frequency scaling)}  \\
R_{2}    &=& R_{n2} * 10000 = 4.61 (k\Omega) \quad \textmc{(frequency scaling)}
\end{eqnarray*}

Bug: resulted circuit does not reflect A of the rational polynomial!

\end{document}