\documentclass[a4paper,10pt]{article}
\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{eucal}
\usepackage{amscd}
\usepackage{url}

\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\I}{\mathbb{I}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\Pee}{\mathbb{P}}
\newcommand{\EuO}{\mathcal{O}}
\newcommand{\Qbar}{\overline{\mathbb{Q}}}
\newcommand{\code}{\lstinline}

\newcommand{\ljk}[2]{\left(\frac{#1}{#2}\right)}
\newcommand{\trans}[2]{(\,#1\;\;#2\,)}
\newcommand{\modulo}[1]{\;\left(\mbox{mod}\;#1\right)}
\newcommand{\fr}{\mathfrak}
\newcommand{\qed}{\square}

\DeclareMathOperator{\Log}{Log}

\def\notdivides{\mathrel{\kern-3pt\not\!\kern4.5pt\bigm|}}
\def\nmid{\notdivides}
\def\nsubseteq{\mathrel{\kern-3pt\not\!\kern2.5pt\subseteq}}

\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}

\newenvironment{proof}[1][Proof]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{definition}[1][Definition]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{example}[1][Example]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{remark}[1][Remark]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}

\parindent=0pt
\parskip 4pt plus 2pt minus 2pt

\title{Complex Variables}
\author{William Hart}

\begin{document}
\maketitle

\section{Limits}

\begin{definition}
We say that the \textbf{limit} of a complex function $f(z)$ as $z$ approaches $z_0$ is $w$, written
$$\lim_{z\to z_0}f(z) = w$$
if for each $\epsilon > 0$ there is a $\delta > 0$ such that $|f(z) - w| < \epsilon$ whenever $0 < |z - z_0| < \delta$.
\end{definition}

\begin{theorem}
Let $f(z) = u(x, y) + iv(x, y)$ and $z_0 = x_0 + iy_0$ for real valued functions $u$ and $v$ and real variables $x_0$ and $y_0$. Then
$$\lim_{z \to z_0}f(z) = u_0 + iv_0$$
iff
$$\lim_{(x, y) \to (x_0, y_0)}u(x, y) = u_0 \;\;\mbox{and}\;\; \lim_{(x, y) \to (x_0, y_0)}v(x, y) = v_0.$$
\end{theorem}

Pick $\delta_1$ and $\delta_2$ such that $|u - u_0|$ and $|v - v_0|$ are both less than $\epsilon/2$ whenever $||(x, y) - (x_0, y_0)||$ is less than $\delta_1$ or $\delta_2$ respectively.

Both hold if we use $\delta =$ min$(\delta_1, \delta_2)$ instead.

Using the triangle inequality, we can bound $|(u + iv) - (u_0 + iv_0)|$ by $\epsilon$ and the result follows from the definition of the limit.

The converse also follows by application of the triangle inequality.

\begin{theorem}
Suppose that
$$\lim_{z\to z_0}f_1(z) = w_1 \;\;\mbox{and}\;\; \lim_{z\to z_0}f_2(z) = w_2$$
then
\begin{itemize}
\item $\lim_{z \to z_0}f_1(z) + f_2(z) = w_1 + w_2$
\item $\lim_{z \to z_0}f_1(z)f_2(z) = w_1w_2$
\item If $w_2 \neq 0$ then $\lim_{z \to z_0}\frac{f_1(z)}{f_2(z)} = \frac{w_1}{w_2}.$
\end{itemize}
\end{theorem}

These follow from the corresponding results for real functions.

\begin{definition}
By
$$\lim_{z \to z_0}f(z) = \infty$$
we mean that for each $\epsilon > 0$ there is a $\delta > 0$ such that $|f(z)| > \frac{1}{\epsilon}$ whenever $0 < |z - z_0| < \delta$ and by
$$\lim_{z\to \infty}f(z) = w$$
we mean that for each $\epsilon > 0$ there exists a $\delta > 0$ such that $|f(z) - w| < \epsilon$ whenever $|z| > \frac{1}{\delta}$.
\end{definition}

\section{Continuity}

\begin{definition}
A function $f(z)$ of a complex variable is \textbf{continuous} at $z_0$ if
$$\lim_{z \to z_0}f(z) = f(z_0).$$
We say that $f$ is continuous in a region $R$ if it is continuous at every point in $R$.
\end{definition}

\begin{theorem}
We have that
\begin{itemize}
\item the sum and product of continuous functions is continuous.
\item excluding division by zero, the quotient of continuous functions is continuous.
\item The composition $f(g(z))$ of continuous functions $f(w)$, $g(z)$ is continuous.
\item the function $f(z) = u(x, y) + iv(x, y)$ for $z = x + iy$ is continuous iff the components $u, v$ are continuous functions of two real variables.
\end{itemize}
\end{theorem}

All of these follow easily from the definition and relevant properties of limits.

\begin{corollary}
Any polynomial function $f(z) = a_0 + a_1z + \cdots + a_nz^n$ for $a_i \in \C$ is continuous everywhere in the complex plane.
\end{corollary}

\begin{theorem}
If $f(z)$ and $g(z)$ are continuous at $z_0$ then
\begin{itemize}
\item $f(z) + g(z)$ is continuous at $z_0$
\item $f(z)g(z)$ is continuous at $z_0$
\item $f(z)/g(z)$ is continuous at $z_0$ if $g(z_0) \neq 0$
\end{itemize}
\end{theorem}

These follow from the corresponding results on limits.

\begin{theorem}
If $f(x)$ is continuous at $x_0$, $z_0 = f(x_0)$ and $g(z)$ is continuous at $z_0$ then $(g\circ f)(x)$ is continuous at $x_0$.
\end{theorem}

We have an $(\epsilon, \delta)$ statement for $g(z)$ at $z_0$. By continuity of $f(x)$ at $x_0$ we know there is a $\gamma > 0$ such that $|f(x) - f(x_0)| < \delta$ whenever $|x - x_0| < \gamma$. Thus we have an $(\epsilon, \gamma)$ statement for $(g\circ f)(x)$.

\begin{corollary}
If $f(z)$ is continuous at $z_0$ and $f(z_0) \neq 0$, then there is a neighbourhood of $z_0$ for which $f(z) \neq 0$.
\end{corollary}

If $|f(z_0)| = \epsilon$, there is a $\delta$ neighbourhood of $z_0$ for which $|f(z) - f(z_0)| < \epsilon$, i.e. $f(z)$ can't reach as far as $0$ in this neighbourhood.

\begin{theorem}
A function $f(z) = u(x, y) + iv(x, y)$ for real-valued functions $u$ and $v$ is continuous at $z_0$ iff $u$ and $v$ are.
\end{theorem}

This follows from the corresponding theorem for limits.

\begin{corollary}
If $f(z)$ is continuous on a closed, bounded region $R$, then $f$ is bounded on $R$ and $|f(z)|$ achieves a maximum somewhere on $R$.
\end{corollary}

Write $f(z) = u(x, y) + iv(x, y)$ for real-valued $u$ and $v$. By the theorem $u$ and $v$ are continuous on $R$. Thus $u$ and $v$ and hence $f$ are bounded on $R$.

The function $\sqrt{u(x, y)^2 + v(x, y)^2}$ achieves a maximum on $R$, and thus so does $|f(z)|$.

\begin{definition}
We say that a function $f(z)$ is \textbf{uniformly continuous} on a region $R$ if for any $\epsilon > 0$ there is a single value of $\delta > 0$ such that
$$|f(z) - f(z_0)| < \epsilon \;\;\mbox{whenever}\;\; |z - z_0| < \delta$$
for all $z_0 \in R$.
\end{definition}

\begin{theorem}
A function $f(z)$ which is continuous in a closed, bounded region $R$ is uniformly continuous there.
\end{theorem}

Write $f(z) = u(x, y) + iv(x, y)$ for real-valued $u$ and $v$. The result follows from the corresponding theorem for real-valued functions.

\section{Derivatives}

\begin{definition}
Let $f(z)$ be a function defined in a neighbourhood of a point $z_0$. The \textbf{derivative} of $f$ at $z_0$ is defined to be
$$f'(z_0) = \lim_{z\to z_0} \frac{f(z) - f(z_0)}{z - z_0}$$
if the limit exists. The function is said to be \textbf{differentiable} at $z_0$ if it does.
\end{definition}

\begin{theorem}
If the derivative of $f(z)$ exists at $z_0$ then $f(z)$ is continuous there.
\end{theorem}

Multiply the expression for the derivative at $z_0$ above by $\lim_{z\to z_0}(z - z_0) = 0$ and we have that $\lim_{z\to z_0}(f(z) - f(z_0)) = 0$. This is precisely the statement of the continuity of $f(z)$ at $z_0$.

\begin{theorem}
If $c \in \C$ and $f(z)$ is differentiable then
\begin{itemize}
\item $\frac{d}{dz}c = 0$
\item $\frac{d}{dz}z = 1$
\item $\frac{d}{dz}cf(z) = cf'(z)$
\end{itemize}
\end{theorem}

These follow directly from the definition.

\begin{theorem}
We have
$$\frac{d}{dz}z^n = nz^{n-1}$$
for all $n > 0$, and also for $n < 0$ when $z \neq 0$.
\end{theorem}

To prove this, we use the definition of the derivative and write $\delta = z - z_0$. We expand $(z_0 + \delta)^n$ using the binomial theorem.

\begin{theorem}
If $f(z)$ and $g(z)$ are both differentiable at $z$ then
\begin{itemize}
\item $\frac{d}{dz}\left(f(z) + g(z)\right) = f'(z) + g'(z)$
\item $\frac{d}{dz}\left(f(z)g(z)\right) = f(z)g'(z) + f'(z)g(z)$
\item $\frac{d}{dz}\left(f(z)/g(z)\right) = \frac{f'(z)g(z) - f(z)g'(z)}{g(z)^2}$ when $g(z) \neq 0$
\end{itemize}
\end{theorem}

The first part follows directly from the definitions.

For the second part we use
\begin{multline*}
f(z + \delta)g(z + \delta) - f(z)g(z) = f(z)\left[g(z + \delta) - g(z)\right] \\
         + \left[f(z + \delta) - f(z)\right]g(z + \delta).
\end{multline*}

For the third part, we prove the result first for $f(z) = 1$, which follows from the definition, and then use the second part.

\begin{theorem}
If $f(z)$ has a derivative at $z_0$ and $g(z)$ has a derivative at $f(z_0)$ then $h(z) = g(f(z))$ has a derivative at $z_0$, and
$$h'(z_0) = g'\left[f(z_0)\right]f'(z_0).$$
\end{theorem}

Writing $w = f(z)$ and $w_0 = f(z_0)$ we must show that
$$\lim_{z\to z_0}\frac{g(w) - g(w_0)}{z - z_0} = g'(w_0)\lim_{z\to z_0}\frac{w - w_0}{z - z_0}.$$

This would be true if
$$g(w) - g(w_0) = (g'(w_0) + T(w))(w - w_0),$$
for $T(w)$ defined in a neighbourhood of $w_0$, so long as $\lim_{z\to z_0}T(w) = 0$.

The function defined by
$$T(w) = \frac{g(w) - g(w_0)}{w - w_0} - g'(w_0),$$
when $w \neq w_0$ and $T(w) = 0$ at $w = w_0$, has precisely the required properties.

It is easy to see that $T(w)$ is continuous at $w = w_0$ and $w = f(z)$ is continuous at $z = z_0$. Thus $T(w) = T(f(z))$ is continuous at $z_0$. Thus
$$\lim_{z\to z_0}T(f(z)) = T(f(z_0)) = T(w_0) = 0,$$
as required.

\section{The Cauchy-Riemann identities}

We can find some necessary conditions for $f(z) = u(x, y) + iv(x, y)$ to be differentiable at $z_0 = x_0 + iy_0$.

\begin{theorem}
If $f(z) = u(x, y) + iv(x, y)$ is differentiable at $z_0 = x_0 + iy_0$ then
\begin{equation}
\begin{split}
f'(z_0) & = u_x(x_0, y_0) + iv_x(x_0, y_0) \\
        & = -i\left[u_y(x_0, y_0) + iv_y(x_0, y_0)\right]
\end{split}
\end{equation}
\end{theorem}

This follows by expanding the expression for $f'(z)$ in terms of $u$ and $v$ and noting that the limits must still hold if we approach $z_0$ along the line $z = x + iy_0$ or along the line $z = x_0 + iy$, respectively.

\begin{corollary} (Cauchy-Riemann)
If $f(z) = u(x, y) + iv(x, y)$ is differentiable at $z_0 = x_0 + iy_0$ then
$$u_x(x_0, y_0) = v_y(x_0, y_0)$$
and
$$u_y(x_0, y_0) = -v_x(x_0, y_0).$$
\end{corollary}

We obtain this by equating the two expressions in the theorem and comparing real and imaginary parts.

The Cauchy-Riemann equations are only a necessary condition. But with some additional continuity conditions, we can find sufficient conditions for differentiability.

\begin{theorem}
If $f(z) = u(x, y) = iv(x, y)$ is defined in a neighbourhood of $z_0 = x_0 + iy_0$ then $f'(z_0)$ exists if
\begin{itemize}
\item the partial derivatives of $u$ and $v$ exist everywhere in the neighbourhood
\item the partial derivatives are continuous at $(x_0, y_0)$
\item the Cauchy-Riemann equations hold at $(x_0, y_0)$
\end{itemize}
\end{theorem}

Because the partial derivatives are continuous at $(x_0, y_0)$ we can write
$$\Delta u = u_x(x_0, y_0)\Delta x + u_y(x_0, y_0)\Delta y + \epsilon_1 |\Delta z|$$
and
$$\Delta v = v_x(x_0, y_0)\Delta x + v_y(x_0, y_0)\Delta y + \epsilon_2 |\Delta z|$$
for some functions $\epsilon_1$, $\epsilon_2$ that tend to $0$ as $(\Delta x, \Delta y)$ tends to $(0, 0)$.

Making use of the Cauchy-Riemann relations, we obtain an expression for $f'(z_0)$ which clearly exists.

One can restate the Cauchy-Riemann equations in polar coordinates when $z_0 \neq 0$.

To convert to polar form, we write
$$x = r\cos\theta \;\;\mbox{and}\;\; y = r\sin\theta.$$

\begin{theorem} (Cauchy-Riemann)
If $f(z) = u(r, \theta) + iv(r, \theta)$ is differentiable at $z_0 \neq 0$ then
$$u_r = v_\theta/r \;\;\mbox{and}\;\; u_\theta = -rv_r.$$
The derivative is given by
$$f'(z_0) = e^{-i\theta_0}\left(u_r(r_0, \theta_0) + iv_r(r_0, \theta_0)\right).$$
\end{theorem}

We have
$$u_r = u_x\cos\theta + u_y\sin\theta$$
$$u_\theta = -u_x r\sin\theta + u_y r\cos\theta$$
$$v_r = v_x\cos\theta + v_y\sin\theta$$
$$v_\theta = -v_x r\sin\theta + v_y r\cos\theta.$$

Applying the Cauchy-Riemann relations yields the stated result.

The necessary conditions for differentiability become the following.

\begin{theorem}
If $f(z) = u(r, \theta) + iv(r, \theta)$ is defined in a neighbourhood of $z_0 = r_0\exp(i\theta_0) \neq 0$ then $f'(z_0)$ exists if
\begin{itemize}
\item the partial derivatives of $u$ and $v$ with respect to $r$ and $\theta$ exist
\item the partial derivatives are continuous at $(r_0, \theta_0)$
\item the partial derivatives satisfy the polar form of the Cauchy-Riemann relations
\end{itemize}
\end{theorem}

This is a straightforward translation of the result for rectangular coordinates.

\begin{definition}
A region of the complex plane is \textbf{connected} if any two points in the region can be joined by a series of straight lines.
\end{definition}

\begin{definition}
A \textbf{domain} is an open region of the complex plane that is connected.
\end{definition}

\begin{theorem}
If $f'(z) = 0$ at all points in a domain $D$ then $f(z)$ is constant in $D$.
\end{theorem}

We write $f(z) = u(x, y) + iv(x, y)$ and note that if $f'(z) = 0$ then $u_x + iv_x = 0$. By the Cauchy-Riemann equations we then have
$$u_x = u_y = v_x = v_y = 0.$$

But this means that any directional derivative of $u$ at any point in $D$ is zero, and similarly for $v$. In particular, this means that the value of $f$ along any line in $D$ must be constant.

As $D$ is connected, the result follows.

\section{Analytic functions}

\begin{definition}
A function $f(z)$ is \textbf{analytic} (or holomorphic) in an open set if it is differentiable at each point in that set. The function $f(z)$ is analytic at a point $z_0$ if it is analytic in a neighbourhood of $z_0$.
\end{definition}

\begin{definition}
A function $f(z)$ is \textbf{entire} if it is analytic at every point in the complex plane.
\end{definition}

\begin{definition}
If a function $f(z)$ is analytic at every point in a neighbourhood of $z_0$ except $z_0$ itself, then $z_0$ is called a \textbf{singular} point of $f$.
\end{definition}

\begin{theorem}
If $f(z)$ and $g(z)$ are analytic in a domain $D$ then
\begin{itemize}
\item $f(z) + g(z)$ is analytic in $D$
\item $f(z)g(z)$ is analytic in $D$
\item $f(z)/g(z)$ is analytic in $D$ if $g(z) \neq 0$ for all $z \in D$
\end{itemize}
\end{theorem}

\begin{theorem}
If $f(z)$ is analytic in a domain $D$ and $g(z)$ is analytic in a domain containing the image of $D$ under $z \mapsto f(z)$, then $g(f(z))$ is analytic in $D$.
\end{theorem}

These follow from the corresponding facts about differentiability.

\section{The exponential function}

For real-valued $x$, the exponential function $e^x$ satisfies two important identities.

\begin{itemize}
\item $e^x$ is entire and $\frac{d}{dx}e^x = e^x$
\item for all $x, y \in \R$ we have $e^{x + y} = e^xe^y$
\end{itemize}

When defining the exponential function $e^z$ for complex $z = x + iy$ it is natural to define it to be a function that has the above properties and that reduces to the ordinary real-valued function when $y = 0$.

In fact, there is a unique complex-valued function with these properties.

\begin{theorem}
The function
$$f(z) = e^x(\cos y + i\sin y)$$
is entire and $f'(z) = f(z)$.
\end{theorem}

Writing $f(z) = u(x, y) + iv(x, y)$ we have $u_x = v_y$ and $u_y = -v_x$, which are everywhere continuous. Thus $f(z)$ is differentiable everywhere.

We have that $f'(z) = u_x + iv_x = f(z)$.

Note that $f(z)$ as defined in the theorem reduces to $e^x$ when $y = 0$.

\begin{theorem}
The only complex-valued function $f(z)$ which satisfies
\begin{itemize}
\item $f(x) = e^x$ for $x \in \R$
\item $f$ is entire and $f'(z) = f(z)$ for all $z \in \C$
\end{itemize}
is the function $f(z) = e^x(\cos y + i\sin y)$.
\end{theorem}

Let us write $e^z$ as a shorthand for $e^x(\cos y + i\sin y)$. Let $f(z)$ be a function with the required properties. Write $g(z) = f(z)e^{-z}$.

As $f(z)$ and $e^{-z}$ are entire, so is $g(z)$. By the product rule and chain rule, we have
$g'(z) = f'(z)e^{-z} - f(z)e^{-z} = 0$.

Thus $g'(z) = c$ for some constant $c \in \C$, i.e. $f(z) = c/e^{-z} = ce^z$ as can be verified by expanding out $e^{-z}$.

Clearly $c = 1$ in order for the first condition to hold.

It makes sense to define the complex exponential as follows.

\begin{definition}
We define
$$e^z = e^x(\cos y + i\sin y),$$
for all $z = x + iy \in \C$.
\end{definition}

\begin{theorem}
We have
$$e^{z_1 + z_2} = e^{z_1}e^{z_2},$$
for all $z_1, z_2 \in \C$.
\end{theorem}

Write $z_1 = x_1 + iy_1$ and $z_2 = x_2 + iy_2$. We check that
$$(\cos y_1 + i\sin y_1)(\cos y_2 + i\sin y_2) = \cos (y_1 + y_2) + i\sin (y_1 + y_2)$$
using standard trig identities. The rest of the result is straightforward.

\begin{corollary}
For $n \in \Z$ we have
$$e^{nz} = (e^z)^n.$$
\end{corollary}

Follows from the theorem by induction for positive $n$. As we already saw above, $e^{-z} = 1/e^z$, so the result also follows for negative $n$. For $n = 0$, use $e^0 = e^ze^{-z} = 1$.

\section{Trigonometric functions}

From $e^{iy} = \cos y + i\sin y$ and $e^{-iy} = \cos y - i\sin y$ we obtain
$$\sin y = \frac{e^{iy} - e^{-iy}}{2i} \;\;\mbox{and}\;\; \cos y = \frac{e^{iy} + e^{-iy}}{2}.$$

This leads us to the following definition.

\begin{definition}
For $z \in C$ we define
$$\sin z = \frac{e^{iz} - e^{-iz}}{2i} \;\;\mbox{and}\;\; \cos y = \frac{e^{iz} + e^{-iz}}{2}.$$
\end{definition}

These are entire function by the rules of sums and quotients of entire functions.

\begin{theorem}
We have
$$\frac{d}{dz}\sin z = \cos z \;\;\mbox{and}\;\; \frac{d}{dz}\cos z = -\sin z.$$
\end{theorem}

This follows directly from the definition and the derivative of the exponential function.

We also have
$$\sin(-z) = -\sin z \;\;\mbox{and}\;\; \cos(-z) = \cos z.$$

\begin{theorem}
We have the following identities
$$\sin(z_1 + z_2) = \sin z_1\cos z_2 + \cos z_1\sin z_2$$
$$\cos(z_1 + z_2) = \cos z_1\cos z_2 - \sin z_1\sin z_2$$
$$\sin^2 z + \cos^2 z = 1$$
$$\sin(z + \pi/2) = \cos z$$
$$\sin(z - \pi/2) = -\cos z$$
$$\sin(z + \pi) = -\sin z$$
$$\cos(z + \pi) = -\cos z$$
$$\sin(z + 2\pi) = \sin z$$
$$\cos(z + 2\pi) = \cos z$$
\end{theorem}

These follow by straightforward algebraic manipulation and standard trig identities for real variables.

Recall that for $y \in \R$ we have
$$\sinh y = \frac{e^y - e^{-y}}{2} \;\;\mbox{and}\;\; \cosh y = \frac{e^y + e^{-y}}{2}.$$

We can use these to obtain the real and complex parts of $\sin$ and $\cos$.

\begin{theorem}
We have
$$\sin z = \sin x\cosh y + i\cos x \sinh y$$
$$\cos z = \cos x\cosh y - i\sin x \sinh y$$
\end{theorem}

We note that $\sin(iy) = i\sinh y$ and $\cos(iy) = \cosh y$.

\begin{theorem}
We have
$$|\sin z|^2 = \sin^2 x + \sinh^2 y$$
$$|\cos z|^2 = \cos^2 x + \sinh^2 y$$
\end{theorem}

We use the previous theorem and the fact that $\sin^2 y + \cos^2 y = 1$ and $\cosh^2 y - \sinh^2 y = 1$.

\begin{theorem}
The zeroes of $\sin z$ are $z = n\pi$ for $n \in \Z$, and the zeroes of $\cos z$ are offset from those by $\pi/2$.
\end{theorem}

We use the previous theorem and note that we must have $\sin x = 0$ and $\sinh y = 0$.

\begin{definition}
We define
$$\tan z = \frac{\sin z}{\cos z}, \;\; \cot z = \frac{\cos z}{\sin z},$$
$$\sec z = \frac{1}{\cos z}, \;\; \csc z = \frac{1}{\sin z}.$$
\end{definition}

\begin{theorem}
We have
$$\frac{d}{dz}\tan z = \sec^2 z, \;\; \frac{d}{dz}\cot z = -\csc^2 z,$$
$$\frac{d}{dz}\sec z = \sec z\tan z, \;\; \frac{d}{dz}\csc z = -\csc z\cot z.$$
\end{theorem}

We use the quotient rule in the definitions and standard trig identities.

\section{The logarithm}

Recall that $\ln x$ is the inverse of the exponential function $e^x$ for $x \in \R$. As the latter takes on every positive real value, the domain of $\ln x$ is $\R^{+}$.

\begin{theorem}
If $z = re^{i\theta}$ is a complex number in modulus/argument format, with $-\pi < \theta \leq \pi$, the multi-valued function
$$\log z = \ln r + i(\theta + 2n\pi), \;\; n \in \Z$$
satisfies
$$e^{\log z} = z.$$
\end{theorem}

As $e^{2\pi i} = 1$, the result follows by substitution.

\begin{definition}
We define the complex logarithm by
$$\log z = \ln r + i(\theta + 2n\pi), \;\; n \in \Z$$
for all $z = re^{i\theta}$ with $-\pi < \theta \leq \pi$. The function
$$\Log z = \ln r + i\theta$$
is called the \textbf{principal branch} of $\log z$.
\end{definition}

Recall that $\ln x$ is continuous and differentiable for $x > 0$ with
$$\frac{d}{dx}\ln x = \frac{1}{x}.$$

\begin{theorem}
The function $\Log z$ is analytic (and therefore continuous) on the domain $r > 0$, $-\pi < \theta < \pi$.
\end{theorem}

Writing $\Log z$ in polar form we get $u(r, \theta) = \ln r$ and $v(r, \theta) = \theta$. The first order partial derivatives are continuous on the stated domain.

We also have
$$u_r = v_\theta/r \;\;\mbox{and}\;\; u_\theta = -rv_r,$$
so that the polar form of the Cauchy-Riemann equations is satisfied. Thus $\Log z$ is analytic on the stated domain.

\begin{theorem}
We have
$$\frac{d}{dz}\Log z = \frac{1}{z}$$
on the domain $r > 0$, $-\pi < \theta < \pi$.
\end{theorem}

We have
$$\frac{d}{dz}\Log z = e^{-i\theta}(u_r + iv_r) = \frac{1}{re^{i\theta}} = \frac{1}{z}.$$

\begin{definition}
A \textbf{branch} of a multi-valued function $f(z)$ is a single-valued function $F(z)$ analytic on some domain where it takes one of the values of $f(z)$.
\end{definition}

\begin{definition}
A \textbf{branch cut} is a line or curve that is used to define a branch $F(z)$ of a multi-valued function $f(z)$. Points on a branch cut are singular values for the single valued function $F(z)$. Any point that is on every branch cut of $f$ is called a \textbf{branch point}.
\end{definition}

For example, the ray $\theta = \pi$ is a branch cut for $\log z$, and the origin is a branch point.

The following theorem holds for the multivalued function $\log$, by which we mean that if two of the values in the identity are specified, there is a value of the third function making the identity hold.

\begin{theorem}
We have
$$\log(z_1z_2) = \log z_1 + \log z_2$$
for all $z_1, z_2 \in \C$.
\end{theorem}

This follows immediately from the definition of $\log z$.

Note that this result doesn't hold everywhere for the principal branch.

\section{Complex exponents}

\begin{definition}
For $z \neq 0$ and $c$ any complex number, we define
$$z^c = e^{c\log z}.$$
\end{definition}

Note that this is may be multi-valued function. For example, if $c = 1/n$ for an integer $n$, then there are $n$ distinct values of the function.

\begin{definition}
The \textbf{principal value} of $z^c$ is given when $\log z$ is replaced with the principal branch $\Log z$ in the definition.
\end{definition}

\begin{theorem}
The derivative of a branch of $z^c$ is given by
$$\frac{d}{dz}z^c = cz^{c - 1},$$
which is single-valued on the same domain as $z^c$.
\end{theorem}

Follows by the chain rule.

\section{Inverse trigonometric functions}

\begin{theorem}
The inverse sine function is given by the multi-valued function
$$\sin^{-1} z = -i\log\left[iz + (1 - z^2)^{1/2}\right].$$
\end{theorem}

We want $w$ where
$$z = \frac{e^{iw} - e^{-iw}}{2i}.$$

Expressing as a quadratic and solving for $e^{iw}$ we get
$$e^{iw} = iz + (1 - z^2)^{1/2}.$$

Taking logarithms of both sides yields the stated result.

The same technique can be used to show the following.

\begin{theorem}
$$\cos^{-1} z = -i\log\left[z + i(1 - z^2)^{1/2}\right]$$
\end{theorem}

\begin{theorem}
$$\tan^{-1} z = \frac{i}{2}\log\frac{i+z}{i-z}.$$
\end{theorem}

We easily differentiate these expressions to obtain the following.

\begin{theorem}
$$\frac{d}{dz}\sin^{-1} z = \frac{1}{(1 - z^2)^{1/2}},$$
$$\frac{d}{dz}\cos^{-1} z = -\frac{1}{(1 - z^2)^{1/2}},$$
$$\frac{d}{dz}\tan^{-1} z = \frac{1}{1 + z^2}.$$
\end{theorem}

The first two depend on the choice of square root made, whereas the last is single-valued regardless of what branch of $\log$ is taken.

\section{Complex-valued functions of a real variable}

\begin{definition}
If $f(t) = u(t) + iv(t)$ is a complex-valued function of a real parameter $t$, the derivative of $f$ with respect to $t$ is given by
$$f'(t) = u'(t) + iv'(t).$$
\end{definition}

\begin{definition}
If $f(t) = u(t) + iv(t)$ is a complex-valued function of a real parameter $t$, the definite integral of $f$ with respect to $t$ in the interval $a \leq t \leq b$ is given by
$$\int_a^b f(t)dt = \int_a^b u(t) + i\int_a^b v(t)dt.$$
\end{definition}

The definite integral exists if both $u$ and $v$ are piecewise continuous on the interval $a \leq t \leq b$.

Similar definitions exist for improper integrals.

The Fundamental Theorem of Calculus holds for functions of this type.

\begin{theorem}
If $f(t) = u(t) + iv(t)$ and $F(t) = U(t) + iV(t)$ are continuous on $a \leq t \leq b$ with $F'(t) = f(t)$ then
$$\int_a^b f(t)dt = F(b) - F(a).$$
\end{theorem}

Follows from the definition of the integral.

\begin{theorem}
For $a \leq b$ we have
$$\left|\int_a^b f(t)dt\right| \leq \int_a^b |f(t)|dt.$$
\end{theorem}

We obtain the absolute value of the integral on the left side by multiplying it by the inverse of $e^{i\theta}$ where $\theta$ is its argument. The scaled integral then has a real value.

This means we only need to consider the real part of this scaled integral. But
$$\mbox{Re}(e^{-i\theta}f(t)) \leq \left|e^{-i\theta}f(t)\right| = \left|f(t)\right|.$$

This gives the stated inequality by standard inequalities for integrals of real-valued functions.

\section{Contours}

Integrals of complex-valued functions of a complex variable $z$ are defined for curves in the complex plane. The curve is first parameterised by a real parameter $t$, so that we can use the integrals of the previous section to define such path integrals.

\begin{definition}
An \textbf{arc} $C$ in the complex plane is a set of points $z = x(t) + iy(t)$ for $a \leq t \leq b$ for continuous real-valued functions $x(t)$ and $y(t)$.
\end{definition}

\begin{definition}
An arc $C$ is said to be \textbf{simple} if it doesn't cross itself. An arc $C$ is said to be a \textbf{simple closed curve} if it doesn't cross itself except for $z(a) = z(b)$.
\end{definition}

Examples include straight line segments and circles, or arcs of circles.

\begin{definition}
An arc $C$ with points $z(t) = x(t) + iy(t)$ for $a \leq t \leq b$ is said to be a \textbf{differentiable arc} if $x'(t)$ and $y'(t)$ exist and are continuous.
\end{definition}

We can compute the length of a differentiable arc.

\begin{theorem}
If $C$ is a differentiable arc with points $z(t) = x(t) + iy(t)$ for $a \leq t \leq b$ then the length of the arc is given by
$$L = \int_a^b \left|z'(t)\right| dt.$$
\end{theorem}

Thinking of the points in the complex plane as points in the $(x, y)$ plane, this is just the standard formula for arc length from calculus.

The parameterisation of an arc by a parameter $t$ is not unique. If we have another parameter $\tau$ such that
$$t = \phi(\tau), \;\; \alpha \leq \tau \leq \beta,$$
we can convert between the two parameterisations.

In order to ensure that $t$ increases whenever $\tau$ does, we require that $\phi'(\tau) > 0$ for all $\alpha \leq \tau \leq \beta$.

\begin{theorem}
If $C$ is a differentiable arc with points $z(t) = x(t) + iy(t)$ for $a \leq t \leq b$ and $t = \phi(\tau)$ for $\alpha \leq \tau \leq \beta$ with $\phi'(\tau) > 0$ for all $\alpha \leq \tau \leq \beta$ then the length of $C$ is given by
$$L = \int_{\alpha}^\beta \left|z'(\phi(\tau))\right|\phi'(\tau)d\tau.$$
\end{theorem}

Follows by splitting $z(t)$ into real and imaginary parts and then making use of the corresponding result for real-valued functions.

Note that if $z(t) = Z(\tau) = z(\phi(\tau))$ for $\alpha \leq \tau \leq \beta$ then
$$Z'(\tau) = z'(\phi(\tau))\phi'(\tau)$$
by the chain rule.

\begin{definition}
A differentiable arc $C$ given by $z(t) = x(t) + iy(t)$ for $a \leq t \leq b$ is said to be \textbf{smooth} if $z'(t) \neq 0$ on the open interval $a < t < b$.
\end{definition}

\begin{definition}
The unit tangent vector to a smooth arc $C$ given by $z(t) = x(t) + iy(t)$ for $a \leq t \leq b$ is given by
$$\underbar{T} = \frac{z'(t)}{|z'(t)|}.$$
\end{definition}

Note that the unit tangent vector doesn't depend on the parameter $t$, since changing the parameter $t$ by $t = \phi(\tau)$ for $\phi'(\tau) > 0$ only changes $z'$ by a positive real factor, which is subsequently scaled out.

The unit tangent vector is in fact the unit tangent vector to the curve $C$ when thought of as a curve in the $(x, y)$ plane.

\begin{definition}
A \textbf{contour} is a piecewise smooth arc.
\end{definition}

The integral of a function $f(z)$ along a contour $C$ is denoted
$$\int_C f(z)dz$$

\begin{definition}
If the contour $C$ is given by points $z = z(t)$ for $a \leq t \leq b$ and $f(z)$ is a piecewise continuous function on $C$, then the \textbf{contour integral} of $f$ along $C$ is given by
$$\int_C f(z)dz = \int_a^b f(z(t))z'(t)dt$$
\end{definition}

Note that as $C$ is a contour, $z'(t)$ is piecewise continuous. As $f(z)$ is piecewise continuous, so is $f(z(t))$ and thus the integral exists.

\begin{theorem}
The value of the contour integral is independent is independent of the representation of $C$.
\end{theorem}

The proof is essentially the same as the proof of independence of arc length.

\begin{theorem}
For any constant $z_0$ and piecewise continuous functions $f(z)$ and $g(z)$ we have
$$\int_C z_0f(z)dz = z_0\int_C f(z)dz$$
and
$$\int_C [f(z) + g(z)]dz = \int_C f(z)dz + \int_C g(z) dz$$
\end{theorem}

These follow immediately from the definitions of the contour integral and the corresponding definitions of complex-valued functions of a real parameter.

\begin{theorem}
If we write $-C$ for the contour $C$ in reverse, then for a piecewise continuous function $f(z)$ we have
$$\int_{-C}f(z)dz = -\int_C f(z)dz$$
\end{theorem}

The contour $-C$ has points $z = z(-t)$ for $-b \leq t \leq -a$. Thus
$$\int_{-C}f(z)dz = \int_{-b}^{-a} f(z(-t))(-z'(t))dt$$

The result follows by making a change of variables $s = -t$.

\begin{theorem}
If $C_1$ is a contour whose endpoint is the starting point of a contour $C_2$, then for any piecewise continuous function $f(z)$ we have
$$\int_C f(z)dz = \int_{C_1}f(z)dz + \int_{C_2}f(z)dz,$$
where $C$ is the contour that first follows $C_1$ then $C_2$.
\end{theorem}

We note that shifting the representation of a contour with points $z(t)$ for $a \leq t \leq b$ so that it has points $Z(s) = z(s - d)$ for $a + d \leq s \leq b + d$ doesn't change the value of the integral, since $s'(t) = 1$.

Thus the points of $C$ can be represented as a function of a single parameter $t$, with the points of $C_2$ shifted so that the starting point of $C_2$ is the end point of $C_1$.

The result follows by the corresponding theorem for the sum of two integrals of a real-valued parameter.

\begin{theorem}
If $f(z)$ is a piecewise continuous function on a contour $C$ of length $L$ such that $|f(z)| \leq M$ for some nonnegative constant $M$ for all points on $C$ then
$$\left|\int_C f(z)dz\right| \leq LM$$
\end{theorem}

If the points of $C$ are given by $z(t)$ for $a \leq t \leq b$ then
$$\left|\int_C f(z)dz\right| \leq \int_a^b |f(z(t))z'(t)|dt.$$

The result now follows by replacing $|f(z)|$ by $M$ and pulling out the constant $M$, and noticing that what remains is the length of the contour.

\section{Antiderivatives}

Sometimes contour integrals are independent of the path that is taken between the endpoints. We examine when this is the case.

\begin{definition}
An \textbf{antiderivative} of a continuous function $f$ in a domain $D$ is a function $F$ such that $F'(z) = f(z)$ for all $z \in D$.
\end{definition}

Note that an antiderivative is analytic on $D$.

\begin{theorem}
The antiderivate $F(z)$ of a continuous function $f(z)$ is unique up to addition of a constant, if it exists.
\end{theorem}

If $G(z)$ is also an antiderivative then $F(z) - G(z)$ has derivative zero. This implies that $F(z) - G(z) = c$ for some constant $c$.

\begin{theorem}
If $f(z)$ is continuous on a domain $D$ then the following are equivalent
\begin{itemize}
\item $f$ has an antiderivative $F$ on $D$
\item integrals of $f$ along contours lying entirely in $D$ with the same start and end points, have the same value
\item integrals of $f$ around closed contours lying entirely in $D$ have value zero
\end{itemize}
\end{theorem}

To show the first implies the second, consider first the case of smooth arcs $C$ between fixed endpoints $z_1$ and $z_2$.

If $C$ is parameterised by $z = z(t)$ for $a \leq t \leq b$ then
$$\frac{d}{dz}F(z(t)) = F'(z(t))z'(t) = f(z(t))z'(t).$$

By the Fundamental Theorem of Calculus extends to complex functions of a real variable, we have
$$\int_C f(z)dz = \int_a^b f(z(t))z'(t)dt = F(z(b)) - F(z(a)).$$

This clearly only depends on the values of $F$ at the endpoints.

The result holds for an arbitrary contour by additivity.

The third part obviously follows from the second, by considering any two distinct points on the closed contour and thinking of the closed contour in two parts, between those points.

Clearly the converse of this implication also holds, by reversing the argument. Thus to show that the third part of the theorem implies the first, it is sufficient to show that the second part implies the first.

To accomplish this, we define
$$F(z) = \int_{z_0}^z f(s)ds,$$
where the integral notation means to take any contour from $s = z_0$ to $s = z$ lying wholly within $D$.

We are done if we show that $F$ is an antiderivative of $f$, i.e. that $F'(z) = f(z)$ on $D$.

To do so, let $\Delta z$ be any point distinct from $z$ in a neighbourhood of $z$ contained in $D$. By additivity
$$F(z + \Delta z) - F(z) = \int_z^{z + \Delta z} f(s)ds.$$

By path independence, we can take the path of integration to be a straight line.

It is easy to show that
$$f(z) = \frac{1}{\Delta z}\int_z^{z + \Delta z}f(z) ds,$$
for any fixed value of $z$ (so that $f(z)$ is a constant in the integral).

Thus
$$\frac{F(z + \Delta z) - F(z)}{\Delta z} - f(z) = \frac{1}{\Delta z}\int_z^{z + \Delta z}f(s) - f(z)ds.$$

The derivative we are after is the limit of the first expression on the left side as $\Delta z \to 0$.

But $f$ is continuous at $z$. Thus the right hand side can be made as close to zero as one wishes.

Thus $F'(z) = f(z)$ and we are done.

\section{The Cauchy-Goursat theorem}

Suppose that $C$ is a simple, closed contour given by points $z(t)$ for $a \leq t \leq b$ and that the contour is counterclockwise. Also suppose that $f(z)$ is analytic in the interior of, and at each point of $C$.

We recall that
$$\int_C f(z)dz = \int_a^b f(z(t))z'(t)dt.$$

Writing
$$f(z) = u(x(t), y(t)) + iv(x(t), y(t)),$$
for $z = x + iy$ and making use of the chain rule, we have
$$\int_C f(z)dz = \int_a^b (ax' - vy')dt + i\int_a^b (vx' + uy')dt.$$

In terms of line integrals of real-valued functions of real variables, this yields
$$\int_C f(z)dz = \int_C udx - vdy + i\int vdx + udy.$$

Now recall the following theorem from multivariable calculus.

\begin{theorem} (Green)
If two real-valued functions $P(x, y)$ and $Q(x, y)$ and their first-order partial derivatives are continuous throughout the closed region $R$ with boundary $C$ then
$$\int_C Pdx + Qdy = \int \int_R (Q_x - P_y)dA.$$
\end{theorem}

Since the function $f$ above is analytic on $R$, it is continuous there. Thus $u$ and $v$ are also continuous in $R$.

Suppose in addition that $f'$ is also continuous in $R$. Then the first order partial derivatives of $u$ and $v$ will be too. Then Green's theorem gives us
$$\int_C f(z) dz = \int \int_R (-v_x - u_y)dA + i\int \int_R (u_x - v_y)dA.$$

But since $f$ is analytic on $R$ we have by the Cauchy-Riemann equations that
$$u_x = v_y \;\;\mbox{and}\;\; u_y = -vx.$$

Thus the previous integral is zero, i.e. we have
$$\int_C f(z) dz = 0.$$

Along the way, we added the assumption that $f$ had continuous derivative. But Goursat was able to remove this assumption. In fact, the following theorem holds.

\begin{theorem} (Cauchy-Goursat)
If a function $f$ is analytic at all points interior to and on a simple, closed countour $C$, then
$$\int_C f(z)dz = 0.$$
\end{theorem}

We give only a sketch of the proof of this result, since the full proof is quite long.

The idea is that we will divide the region $R$ enclosed by $C$ into a collection of squares, and partial squares along the boundary.

We claim that it is possible to divide the region in such a way, using squares not necessarily of the same size, such that for each of the squares, indexed by $j = 1, 2, \ldots, n$ there is a point $z_j$ for which
$$\left|\frac{f(z) - f(z_j)}{z - z_j} - f'(z_j)\right| < \epsilon$$
for all $z \neq z_j$ in that square, where $\epsilon$ is fixed and as small as one desires.

This follows from the definition of the derivative as a limit. If the number of squares were not finite, then it must be possible to subdivide a given square into four equal sized squares, and one of those into four more an so on, so that the inequality never holds. We end up with a sequence of every smaller squares converging on a point, for which the inequality doesn't hold. But this then contradicts the definition of the derivative at that point. Thus there are a finite number of squares.

Next we define the function
$$\delta_j = \begin{cases}\frac{f(z) - f(z_j)}{z - z_j} - f'(z_j) & \mbox{when}\;\; z \neq z_j,\\ 0 & \mbox{when}\;\; z = z_j\end{cases}.$$

On the $j$-th square, we have $|\delta_j(z)| < \epsilon$. The function $\delta_j(z)$ is also continuous on that region.

Let $C_j$ be the boundary of the $j$-th square or partial square. Critically, we have
$$f(z) = f(z_j) - z_jf'(z_j) + f'(z_j)z + (z - z_j)\delta_j(z)$$
on such a region.

Because each of these terms has an antiderivative except the last, their integrals along $C_j$ are zero, leading to
$$\int_{C_j} f(z)dz = \int_{C_j} (z - z_j)\delta_j(z)dz.$$

The integral of $f(z)$ on $C$ is the sum of the integrals along the boundaries of all the squares and partial squares, since each boundary of a square on the interior of $R$ is part of the boundary of two different squares in the opposite sense and so their integrals cancel.

Letting $s_j$ be the side length of the $j$-th square, we have
$$|z - z_j| \leq \sqrt{s}s_j.$$

The length of the boundary of the $j$-th square is either $4s_j$ if it is a square, or if not, it is bounded by $4s_j + L_j$ where $L_j$ is the length of the portion of the region $R$ included in the boundary of the partial square.

Using these inequalities, it is possible to obtain a bound of the following form
$$\left|\int_C f(z)dz\right| < D\epsilon,$$
where $D$ is some constant in terms of say the area of a square bounding $R$ and the length of the boundary of $R$.

Since $\epsilon$ may be taken as small as one desires, the result follows.

\end{document}


% !TEX root = main.tex
%=====================================================================
\chapter{Random processes}\label{chap:randomprocesses}

\newcommand{\Znn}{\mathbb{Z}_{\geq 0}}

%-------------------------------------------------
%\section{Random processes}\label{sec:rprocs}
Let $\Omega$ be the sample space of some random experiment and let $T\subseteq\R$ be a set of \emph{times}. The canonical examples are $T=\{0,1,2,\ldots\}$ which defines a discrete-time process, and $T=[0,\infty)$ which defines a continuous-time process.

\begin{definition}
A \emph{random process} on $\Omega$ is a collection of random variables $\{X_t:t\in T\}$, where each $X_t:\Omega\to\R$ is a random variable on $\Omega$.
\ben
\it If $T$ is countable, $\{X_t\}$ is called a \emph{discrete-time} random process.
\it If $T$ is uncountable, $\{X_t\}$ is called a \emph{continuous-time} random process.
\een
\end{definition}

We can think of a random process $\{X_t\}$ as a mapping:
\[
\begin{array}{rccl}
\{X_t\}: 	& T\times\Omega		& \to		& \R \\
			& (t,\omega)		& \mapsto	& X_t(\omega).
\end{array}
\]

\begin{definition}
For a fixed outcome $\omega\in\Omega$, the associated realisation $\{X_t(\omega):t\in T\}$ of the random process $\{X_t:t\in T\}$ is called a \emph{trajectory} or \emph{sample path} of the process.
\end{definition}

If $T$ is a finite set then $\{X_t\}$ is random vector which is defined by its joint CDF. If $T$ is an infinite set (either countable or uncountable) it is not easy to define a CDF to describe $\{X_t\}$. To do it, we have to deal with the joint distributions of $(X_{t_1},X_{t_2},\ldots,X_{t_n})$ for all $n\in\N$ and all choices of $t_1,t_2,\ldots,t_n\in T$. These are called the \emph{finite-dimensional distributions} of the random process.

%-----------------------------
\subsection{The canonical probability space}

Let $\Omega = [0,1]^{\N}$ be the set of infinite sequences of real numbers from the unit interval $[0,1]$, let $\mathbf{\omega}\in\Omega$ and write this as
\[
\mathbf{\omega} = (\omega_0,\omega_1,\omega_2,\ldots).
\]
The $n$th term of the sequence is extracted by the random variable
\[
\begin{array}{rccl}
	\gamma_n:	& \omega			& \to		& [0,1] \\
				& \mathbf{\omega}	& \mapsto	& \omega_n.
\end{array}
\]
We state the following theorem without proof.
\begin{theorem}
There exists a probability measure $\prob$ on $\Omega)$ such that the random variables $\gamma_0,\gamma_1,\ldots$ are independent and uniformly distributed on $[0,1]$.
\end{theorem}

We think of the single outcome $\mathbf{\omega} = (\omega_0,\omega_1,\omega_2,\ldots)$ as the source of all randomness in our experiment, and all other quantities are deterministic functions of this outcome (random variables).

%-----------------------------
\subsection{The Bernoulli process}\label{sec:bernoulliprocs}
\begin{definition}
A random process $\{X_n\}$ consisting of independent and identically distributed $\text{Bernoulli}(p)$ variables is called the $\text{Bernoulli}(p)$ process.
\end{definition}

In terms of the canonical probability space,
\[
X_n = \begin{cases}
	1	& \gamma_n\leq p, \\
	0	& \text{otherwise.}
\end{cases}
\]
Trajectories of the Bernoulli process are binary sequences (of infinite length).

\begin{example}
Let $\{X_n\}$ be a $\text{Bernoulli}(p)$ process. If $p>0$, show that a `success' is eventually observed with probability $1$.
\begin{solution}
\bit
\it $E 		= \{\omega\in\Omega:X_n(\omega)=1 \text{ for some } n\}$
\it $E_n 	= \{\omega\in\Omega:X_m(\omega)=1 \text{ for some } m\leq n\}$
\it $E_1\subset E_2\subset E_3\subset\ldots$ is an expanding sequence, with $E=\cup_{n=1}^{\infty} E_i$.
\it By continuity,
\[
\prob(E) = \lim_{n\to\infty}\prob(E_n) = \lim_{n\to\infty}\big(1 - (1-p)^n\big) = 1.
\]
\eit
\end{solution}
\end{example}

% ex: hitting times
\begin{example}
Suppose have observed the first $n$ terms of a $\text{Bernoulli}(p)$ process. Let $\tau$ be the time until we next observe a `success'. Show that $\tau\sim\text{Geometric}(p)$.
\begin{solution}
\[
\tau = \min\{m>0:X_{n+m}=1\}.
\]
Because the $X_i$ are independent,
\begin{align*}
\prob(\tau > m | X_1,X_2,\ldots,X_n)
	& = \prob(X_{n+1}=0,X_{n+2}=0,\ldots,X_{n+m}=0 | X_1,X_2,\ldots,X_n) \\
	& = \prob(X_{n+1}=0)\prob(X_{n+2}=0)\cdots\prob(X_{n+m}=0) \\
	& = (1-p)^{m}
\end{align*}
so $\tau\sim\text{Geometric}(p)$ with $\prob(\tau=m)=(1-p)^{m-1}p$.
\end{solution}
\end{example}

% ex: memoryless
\begin{exercise}
Show that the geometric distribution has the so-called memoryless property: if $\tau\sim\text{Geometric}(p)$, then
\[
\prob(\tau > m+n|\tau > n) = \prob(\tau > m).
\]
\begin{answer}
\begin{align*}
\prob(\tau > m+n|\tau > n)
	& = \prob(\tau>m+n,\tau>n)/\prob(\tau>n) \\
	& = \prob(\tau>m+n)/\prob(\tau>n) \\
	& = (1-p)^{m+n}/(1-p)^n \\
	& = (1-p)^m \\
	& = \prob(\tau > m).
\end{align*}
\end{answer}
\end{exercise}

%-------------------------------------------------
%\subsection{The Poisson process}\label{sec:poissonprocess}

%-------------------------------------------------
\section{Random walks}\label{sec:rwalks}

%intro
The \emph{simple random walk} is the simplest model of a \emph{diffusion process}. We consider a particle which inhabits the set of integer points $\Z$ and at each (discrete) time step the particle moves either one step to the right with probability $p$ or one step to the left with probability $q=1-p$.
\bit
\it For fixed $\omega$ the random variable $X_n$ describes the trajectory of the particle over time.
\it For fixed $n$ the random variable $X_n$ describes the (spatial) distribution of the particle at time $n$.
\eit

\begin{definition}
A discrete random process $\{X_n\}$ is called a  \emph{simple random walk} with parameter $p\in(0,1)$ if
\ben
\it $X_{n+1}-X_n$ is independent of $X_0,X_1,\ldots,X_n$,
\it $\prob(X_{n+1} = X_n + 1) = p$ and $\prob(X_{n+1} = X_n - 1) = q$ where $q=1-p$.
\een
If $p=1/2$ the random walk is called \emph{symmetric}.
\end{definition}

% link with canonical probability space
In terms of the random variables $\gamma_n$ defined on the canonical probability space let us define a new sequence of random variables,
\[
\xi_n = \begin{cases}
	\phantom{-}1	& \gamma_n\leq p, \\
	-1	& \text{otherwise.}
\end{cases}
\]
\begin{lemma}
The random process $\{X_n\}$ where $X_n = X_0 + \sum_{k=1}^n \xi_k$ is a simple random walk.
\begin{proof}
\ben
\it The $\xi_n$ are independent (they are transforms of the $\gamma_n$, which are independent), so the increment $X_{n+1}-X_n$ is independent of $\xi_1,\xi_2,\ldots,\xi_n$, and because $X_0,X_1,\ldots,X_n$ are just linear combinations of $\xi_1,\xi_2,\ldots,\xi_n$, it follows that $X_{n+1}-X_n$ is independent of $X_0,X_1,\ldots,X_n$.
\it Because $\gamma_{n+1}\sim\text{Uniform}[0,1]$,
\begin{align*}
\prob(X_{n+1} = X_n + 1)
	& = \prob(\xi_{n+1} = 1) \prob(\gamma_{n+1}\leq p) = p.
\prob(X_{n+1} = X_n - 1)
	& = \prob(\xi_{n+1} = -1) \prob(\gamma_{n+1} > p) = 1-p.
\end{align*}
\een
\end{proof}
\end{lemma}

%-----------------------------
\subsection{Properties}

\begin{definition}
A \emph{stationary} random process is one whose behaviour does not change when shifted in time and space.
\end{definition}

\begin{lemma}
The simple random walk satisfies
\[
\begin{array}{lcll}
\prob(X_n=x\,|\,X_0=a)
	& = & \prob(X_n=x+b\,|\,X_0=a+b) 	&\qquad\text{(spatial homogeneity), and} \\
\prob(X_n=x\,|\,X_0=a)
	& = & \prob(X_{m+n}=x\,|\,X_m=a) 	&\qquad\text{(temporal homgeneity)}
\end{array}
\]
and is therefore a stationary process.
\begin{proof}
\[
\begin{array}{lll}
\prob(X_n=x|X_0=a)
	& = \prob\left(a+\sum_{k=1}^n \xi_k = x\right) \\
	& = \prob\left((a+b)\sum_{k=1}^n \xi_k = x+b\right) = \prob(X_n=x+b|X_0=a+b) \\
\prob(X_n=x|X_0=a)
	& = \prob\left(a+\sum_{k=1}^n \xi_k = x\right) \\
	& = \prob\left(a+\sum_{k=m+1}^{m+n} \xi_k = x\right) = \prob(X_{m+n}=x|X_m=a).
\end{array}
\]
\end{proof}
\end{lemma}

%%-----------------------------
%\begin{theorem}
%Let $\{X_n\}$ be a simple random walk with parameter $p$, and suppose that $X_0=0$. Then $X_n$ is a discrete random variable, with
%\[
%\prob(X_n=\ell) = \binom{n}{\frac{n+\ell}{2}} p^{(n+\ell)/2}q^{(n-\ell)/2}
%\quad\text{for } \ell\in\{-n,-n+2,\ldots,-2,0,2,\ldots n-2, n\},
%\]
%where $q=1-p$, and zero otherwise.
%\begin{proof}
%
%\end{proof}
%\end{theorem}

%--------------------------------------------------------------------------
\subsection{Sample paths}
%--------------------------------------------------------------------------
The motion of a particle can be represented by the sequence $\{(n,X_n)\,:\,n=0,1,2,\ldots\}$., which is called the \emph{trajectory} or \emph{path} of the particle. By convention, sample paths are plotted with time on the horizontal axis, and displacement on the vertical axis.

Any event can be expressed in terms of an appropriate set of paths: the probability of the event is the probability that one of the associated paths is realized. The set of sample paths therefore serves as a \emph{sample space} for analysing the simple random walk.

Let $C_n$ be the set of all possible paths of length $n$,
\[
C_n = \big\{(x_0,x_1,\ldots,x_n)\in\Z^{n+1}: x_{k+1}-x_k = \pm 1\text{ for } k=0,1,2,\ldots,n-1\big\}.
\]

The probability that the first $n$ steps of the random walk follows any particular path $\mathbf{x}=(x_0,x_1,\ldots,x_n)$ is $p^cq^d$ where
\bit
\it $c$ is the number of positive steps (right/up), and
\it $d$ is the number of negative steps (left/down).
\eit

%-------------------------
\subsubsection{The distribution of the particle at time $n$}
We can compute the PMF of $X_n$ by examining the ensemble of all possible paths over the first $n$ steps. Let $C_n(a,b)$ be the set of paths from $(0,a)$ to $(n,b)$,
\[
C_n(a,b) = \big\{\mathbf{x}\in C_n: x_0=a, x_n=b\}.
\]

% lemma: PMF of $X_n$
\begin{lemma}
The number of possible paths from $(0,a)$ to $(n,b)$ is
\[
|C_n(a,b)| = \binom{n}{\frac{1}{2}(n+b-a)}
\]
provided that $(n+b-a)/2$ belongs to the set $\{0,1,2\ldots,n\}$.
\end{lemma}
If $X_0=0$, the condition says that the particle can occupy only odd-numbered sites after an odd number of steps, and only even-numbered sites after an even number of steps.)

\begin{proof}
Choose a path from $(0,a)$ to $(n,b)$. Let $u$ be the number of positive steps, and $d$ the number of negative steps.
\bit
\it $u+d=n$ is the total number of steps;
\it $u-d=b-a$ is the overall displacement to the right.
\eit
Solving for $c$ and $d$, we get $c=\frac{1}{2}(n+b-a)$ and $d=\frac{1}{2}(n-b+a)$. The number of paths from $(0,a)$ to $(n,b)$ is the number of ways of choosing exactly $u$ positive steps from the $n$ available steps, so
\[
|C_n(a,b)| = \binom{n}{u} = \binom{n}{\frac{1}{2}(n+b-a)}
\]
\end{proof}

% corollary: PMF of X_n
\begin{corollary}
%The probability that $X_n=b$ given that $X_0=a$ is
\[
\prob(X_n=b|X_0=a) = \binom{n}{\frac{1}{2}(n+b-a)}p^{\frac{1}{2}(n+b-a)}q^{\frac{1}{2}(n-b+a)}
\]
\end{corollary}
\begin{proof}
Each path in $C_n(a,b)$ occurs with probability $p^{\frac{1}{2}(n+b-a)}q^{\frac{1}{2}(n-b+a)}$, so
\[
\prob(X_n=b|X_0=a) = \binom{n}{\frac{1}{2}(n+b-a)}p^{\frac{1}{2}(n+b-a)}q^{\frac{1}{2}(n-b+a)}
\]
\end{proof}

%--------------------------------------------------------------------------
\subsection{The Reflection Principle}
%--------------------------------------------------------------------------
Counting sample paths is made easier by using the \emph{reflection principle}.

\begin{figure}
\centering\resizebox{0.5\linewidth}{!}{\includegraphics{reflection_principle}}
\caption{The Reflection Principle\label{reflection_principle}}
\end{figure}

%Let $a,b>0$ and define $C^{0}_n(a,b)$ to be the set of paths from $(0,a)$ to $(n,b)$ which contain a point of the form $(k,0)$, i.e.\ those trajectories which return to the (spatial) origin at some time step $k=0,1,2,\ldots,n$. We can use the reflection principle to show that the number of such paths is equal to the total number of paths from $(0,-a)$ to $(n,b)$.
%
%-------------------------
% theorem: reflection principle
\begin{theorem}
Let $a,b>0$, and let $C^{0}_n(a,b)$ be the set of paths from $(0,a)$ to $(n,b)$ which visit the (spatial) origin:% at some time step $k\in\{1,2,\ldots,n-1\}$,
\[
C^{0}_n(a,b) = \big\{\mathbf{x}\in C_n(a,b): x_k=a \text{ for some } k=1,2,\ldots,n-1\}.
\]
The number of such paths is equal to the total number of paths from $(0,-a)$ to $(n,b)$,
\[
|C^0_n(a,b)| = |C_n(-a,b)|.
\]
\end{theorem}
\begin{proof}
Each path from $(0,a)$ to $(n,b)$ intersects the horizontal axis at some earliest point $(k,0)$. If we reflect the segment of the path with $0\leq t\leq k$ in the horizontal axis, we obtain a path from $(0,a)$ to $(n,b)$ which intersects the horizontal axis. This operation puts the elements of $C^0_n(a,b)$ and $C_n(-a,b)$ in one-to-one correspondence, which proves the theorem.
\end{proof}

%For simplicity, we now focus our attention on simple random walks that start at the origin, i.e.\ those for which $X_0=0$.
%
%For $\ell > 0$, let $C^{*}_n(\ell)$ be the set of paths from $(0,0)$ to $(n,\ell)$ which do \emph{not} revisit the initial location $X_0=0$.
%
%

%%--------------------------------------------------------------------------
%\subsubsection{The Ballot Theorem}
%
%For $b>a\geq 0$, let $C^{*}_n(a,b)$ to be the set of paths from $(0,a)$ to $(n,b)$ which do \emph{not} revisit the initial location $X_0=a$.
%\[
%C^{*}_n(a,b) = \big\{\mathbf{x}\in C_n(a,b): x_k=0 \text{ for some } k=1,2,\ldots,n-1\}.
%\]

% ballot theorem
\begin{theorem}[The ballot theorem]
Let $b>a\geq 0$, and let $C^{*}_n(a,b)$ to be the set of paths from $(0,a)$ to $(n,b)$ which do \emph{not} revisit the initial location $X_0=a$,
\[
C^{*}_n(a,b) = \big\{\mathbf{x}\in C_n(a,b): x_k=a \text{ for some } k=1,2,\ldots,n-1\}.
\]
The number of such paths is given by
\[
|C^{*}_n(a,b)| = \frac{b-a}{n} |C_n(a,b)|,
\]
where $C_n(a,b)$ is the set of all paths from $(0,a)$ to $(n,b)$.
\end{theorem}

%-------------------------
\begin{proof}
Without loss of generality, let $a=0$. The first step of all such paths must be to $(1,1)$, so by the reflection principle the number of such paths is
\begin{align*}
|C^{*}_n(0,b)|
	& = |C_{n-1}(1,b)| - |C^0_{n-1}(1,b)| \\
	& = |C_{n-1}(1,b)| - |C_{n-1}(-1,b)|
\end{align*}
The result then follows by the fact that the total number of paths $(0,0)$ to $(n,b)$ is equal to
\[
|C_n(0,b)| = \binom{n}{\frac{1}{2}(n+b)}
\]
and
\[
\begin{array}{lll}
|C_{n-1}(1,b)| 	& = \binom{n-1}{\frac{1}{2}(n+b)} 	& = \frac{n+b}{2n} |C_n(0,b)| \\
|C_{n-1}(-1,b)|	& = \binom{n-1}{\frac{1}{2}n+b-1}	& = \frac{n-b}{2n} |C_n(0,b)| \\
\end{array}
\]
\end{proof}

%-------------------------
% example
\begin{example}
In an election, candidate A scores $\alpha$ votes and candidate B scores $\beta$ votes, where $\alpha > \beta$. What is the probability that candidate $A$ was always ahead of candidate $B$ during the election?
\begin{solution}
Assume that each possible combination of $\alpha$ votes for $A$ and $\beta$ votes for $B$ is equally likely.

\bigskip
After $n=\alpha+\beta$ steps, the required probability is the proportion of paths from $(0,0)$ to $(\alpha+\beta,\alpha-\beta)$ which do not re-visit the horizontal axis. By the ballot theorem,
\[
\prob(\text{$A$ was always ahead of $B$}) = \frac{\alpha-\beta}{\alpha+\beta}.
\]
\end{solution}
\end{example}

%% thm: maximum
%\begin{theorem}
%Let $\{X_n\}$ be a symmetric simple random walk with $X_0=0$, and let $M_n=\max\{X_0,X_2,\ldots,X_n\}$. Then
%\[
%\prob(M_n\geq \ell)	= \prob(X_n=\ell) + \prob(X_n=\ell+1)
%\]
%Note that only one of $\prob(X_n=\ell)$ and $\prob(X_n=\ell+1)$ is non-zero, the first if $n$ and $\ell$ are both odd or both even, and the second if one is odd and the other is even.
%\end{theorem}
%
%\begin{proof}
%Let $A_n(\ell)$ be the set of paths whose maximum is at least equal to $\ell$:
%\begin{align*}
%A_n(\ell)
%	& = \{\mathbf{x}\in C_n: \max x_k \geq \ell, k=1,2,\ldots,n\} \\
%	& = \big\{\mathbf{x}\in C_n: x_k \geq \ell \text{ for at least one } k\in\{0,1,\ldots,n\}\big\}
%\end{align*}
%There are $2^n$ possible sample paths, all equally likely, so
%\[
%\prob(M_n \geq \ell) = 2^{-n}|A_n(\ell)|.
%\]
%Let
%\bit
%\it $A_n^{(1)}(\ell) = \big\{\mathbf{x}\in A_n(\ell): x_n > \ell\big\}$,
%\it $A_n^{(2)}(\ell) = \big\{\mathbf{x}\in A_n(\ell): x_n = \ell\big\}$,
%\it $A_n^{(3)}(\ell) = \big\{\mathbf{x}\in A_n(\ell): x_n < \ell\big\}$.
%\eit
%Then
%\[
%|A_n(\ell)| = |A_n^{(1)}(\ell)| + |A_n^{(2)}(\ell)| + |A_n^{(3)}(\ell)|
%\]
%
%By the reflection principle, the trajectories in $A_n^{(1)}(\ell)$ are in one-to-one correspondence with those in $A_n^{(3)}(\ell)$, so
%\[
%|A_n(\ell)| = 2|A_n^{(1)}(\ell)| + |A_n^{(2)}(\ell)|
%\]
%
%Now,
%\bit
%\it $A_n^{(1)}(\ell) = \cup_{k=\ell+1}^{\infty}\{\mathbf{x}: x_n=k\}$ and
%\it $A_n^{(2)}(\ell) = \{\mathbf{x}: x_n=\ell\}.
%\eit
%
%There are $2^n$ sample paths, each equally likely, so
%\begin{align*}
%\prob(M_n\geq \ell)
%	& = 2\sum_{k=\ell+1}^{\infty}\prob(X_n=k) + \prob(X_n=\ell) \\
%	& = \sum_{k=\ell+1}^{\infty}\prob(X_n=k) + \sum_{k=\ell}^{\infty}\prob(X_n=\ell) \\
%\end{align*}
%
%Hence,
%\begin{align*}
%\prob(M_n = \ell)
%	& = \prob(M_n\geq \ell) - \prob(M_n\geq \ell+1) \\
%	& = \prob(M_n\geq \ell) - \prob(M_n\geq \ell+1) \\
%	& = \left(\sum_{k=\ell+1}^{\infty}\prob(X_n=k) + \sum_{k=\ell}^{\infty}\prob(X_n=k)\right)
%			- \left(\sum_{k=\ell+2}^{\infty}\prob(X_n=k) + \sum_{k=\ell+1}^{\infty}\prob(X_n=k)\right) \\
%	& = \sum_{k=\ell}^{\infty}\prob(X_n=k) - \left(\sum_{k=\ell+2}^{\infty}\prob(X_n=k) \\
%	& = \prob(X_n=\ell)\right) + \prob(X_n=\ell+1)\right) \\
%\end{align*}
%
%\end{proof}

%-----------------------------
\subsection{Hitting times}
Let $X_n=\sum_{k=1}^{n}\xi_k$ be a simple random walk with $X_0=0$. The \emph{first hitting time} at level $\ell$ is the time at which the trajectory first reaches level $\ell\in\N$,
\[
T_{\ell} = \min\{n: X_n=\ell\}
\]
If $T_{\ell}=n$, we must have that (1) $X_n = \ell$ and (2) $X_k < \ell$ for all $k=0,1,2,\ldots,n-1$.

\begin{example}[Gambler's ruin]
A gambler starts with $\pounds x$ and plays a game in which a fair coin is tossed repeatedly. Each time, if the coin shows heads then he wins $\pounds 1$, but if the coin shows tails he loses $\pounds 1$. The gambler stops when either he goes bankrupt or otherwise reaches some pre-determined amount $\pounds a$. Find the probability that the gambler goes bankrupt.
%Show that the probability that the gambler goes bankrupt is equal to $(a-x)/x$.
\end{example}

\begin{solution}
Let $X_n$ denote the gambler's \emph{winnings} after $n$ steps. The sequence $X_0,X_1,X_2,\ldots$ can be modelled by a symmetric simple random walk starting at $X_0=0$, with
\[
X_n = \sum_{k=1}^n \xi_k \quad\text{where $\prob(\xi_k=1)=1/2$ and $\prob(\xi_k=-1)=1/2$.}
\]
Let $T$ be the (random) time at which the game stops:
\bit
\it $T=\min\{T_0,T_a\}$ where $T_0$ and $T_a$ are the hitting times of levels $0$ and $a$ respectively.
\eit

Consider the random variable $X_T = \sum_{k=1}^{T}\xi_k$, the gambler's winnings at the random time $T$.
\bit
\it Clearly $X_T = -x$ or $X_T = a-x$.
\it Let $p_0=\prob(X_T = -x)$ and $p_1=\prob(X_T = a-x)$.
\eit

Let $\xi$ be a random variable with $\prob(\xi_k=1)=1/2$ and $\prob(\xi_k=-1)=1/2$, and let $\xi_1,\xi_2,\ldots$ be independent increments from the distribution of $\xi$. By Corollary~\ref{cor:GF-NofX} (Wald's identity),
\[
\expe(X_T) = \expe(T)\expe(\xi) = \expe(T)\times 0 = 0,
\]
We also know that $\expe(X_T) = (-x)p_0 + (a-x)p_a$, so $xp_0 = (a-x)p_a$. Thus, using the fact that $p_0+p_a=1$, we obtain
\[
p_0 = \frac{a-x}{a} \quad\text{and}\quad p_1 = \frac{x}{a}.
\]
Note that we have assumed that $\expe(T)<\infty$, which requires that $\prob(T<\infty)=1$.

\end{solution}

%-----------------------------

\begin{theorem}
Let $\{X_n\}$ be a simple random walk with $X_0=0$, and let $T$ be the first hitting time at level $\ell=1$. The PMF of $T$ is given by the following recursive formula,
\[
p_n = q\sum_{k=1}^{n-2}p_{k}p_{n-k-1}, \qquad p_0=0, p_1=p, q=1-p..
\]
where $p_n = \prob(T=n)$.
\end{theorem}
\begin{proof}
\bit
\it We cannot move from $0$ to $1$ in an even number of steps, so $p_{2n}=0$ for $n\in\Znn$.
\it For $n=1$, the first step must be upwards, so $p_1=p$.
\it For $n>1$, the first step must be downwards (which occurs with probability $q$), and then we need to climb from $-1$ to $0$, and then from $0$ to $1$.
\eit
\begin{align*}
\prob(T=n)
	& = q\sum_{k=1}^{n-2}\prob(\text{$k$ steps to first hit $0$ from $-1$, and $n-k-1$ steps to first hit 1 from $0$}) \\
	& = q\sum_{k=1}^{n-2}\prob(\text{$k$ steps to first hit $0$ from $-1$})\prob(\text{$n-k-1$ steps to first hit $1$ from $0$}) \\
	& = q\sum_{k=1}^{n-2}\prob(\text{$k$ steps to first hit $1$ from $0$})\prob(\text{$n-k-1$ steps to first hit $1$ from $0$}) \\
	& = q\sum_{k=1}^{n-2}p_{k}p_{n-k-1}
\end{align*}
\end{proof}

\begin{theorem}
The PGF of $T$ is given by $\displaystyle G(t) = \frac{1 - \sqrt{1-4pqt^2}}{2qt}$.
\end{theorem}

\begin{proof}
Let $G(t) = \sum_{k=0}^{\infty} p_k t^k$ be the PGF of $T$. The square of the PGF can be written as
\[
G(t)^2 = \sum_{k=0}^{\infty} \left(\sum_{i=0}^{k} p_ip_{k-i}\right) t^k
\]
Because $p_0=0$ and $p_{k+1}=q\sum_{i=1}^{k-1}p_{i}p_{k-i}$, the inner sum can be written as
\[
\sum_{i=0}^{k} p_ip_{k-i}
	= \sum_{i=1}^{k-1} p_ip_{k-i}
	=  q^{-1}p_{k+1} \quad\text{for $k\geq 2$.}
\]
(and zero for $k=0$ and $k=1$). Hence,
\[
G(t)^2 = q^{-1}\sum_{k=2}^{\infty} p_{k+1} t^{k+1} = G(t) - pt
\]
so
\[
qtG(t)^2 = \sum_{k=2}^{\infty} p_{k+1} t^{k+1} = \sum_{k=0}^{\infty} p_k t^k - pt = G(t) - pt.
\]
Thus we obtain a quadratic equation for $G(t)$,
\[
qtG(t)^2 - G(t) + pt = 0
\]
and solving for $G(t)$ we obtain
\[
G(t) = \frac{1\pm\sqrt{1-4pqt^2}}{2qt}
\]
For any PGF we must have that $G(t)\leq 1$, so we conclude that
\[
G(t) = \frac{1 - \sqrt{1-4pqt^2}}{2qt}
\]
\end{proof}

\begin{remark}
The probabilities $p_n$ can be computed by taking successive derivatives of $G(t)$ and evaluating these at $t=1$. An explicit expression is given by
\[
p_{2n-1} = p^n q^{n-1}\frac{2}{n}\binom{2n-3}{n-2}.
\]
This result can also be obtained by the reflection principle.
\end{remark}

% probability that we don't hit level 1 at all
\begin{remark}
Since $G(1)=\sum_{k=0}^{\infty}p_k$ we might be inclined to think that $G(1)=1$. In fact,
\[
G(1) = \frac{1 - \sqrt{1-4pq}}{2q} = \frac{1-|p-q|}{2q}
	= \begin{cases}
		1 	& \text{ for $p\geq 1/2$,} \\
		p/q & \text{ for $p < 1/2$.}
	\end{cases}
\]
Thus if $p<q$ we see that $G(1)<1$, which show that the random walk might never reach level $1$ when the probability of a positive step is smaller than the probability of a negative step. In this case, $G(1)=\sum_{k=0}^{\infty}p_k = \prob(T<\infty)$, so $\prob(T=\infty)>0$. It is a remarkable fact that if $p=1/2$, the random walk will \emph{always} hit level $1$ sooner or later, but this need not happen if $p<1/2$. This behaviour is known as \emph{criticality} -  many systems exhibit qualitatively different behaviour when the value of a parameter $p$ lies either side of some critical value $p_c$.
\end{remark}

% expected first hitting time
\begin{remark}
We can compute the \emph{expected} time before the random walk hits $1$ for the first time. If $p<1/2$ then $\prob(T=\infty)>0$ so $\expe(T)=\infty$. For the case $p\geq 1/2$, note that
\[
G'(t) = \frac{2p}{\sqrt{1-4pqt^2}} - \frac{1-\sqrt{1-4pqt^2}}{2qt^2}.
\]
\[
\begin{array}{lcl}
p=1/2:	& \qquad	& \expe(T) = \lim_{t\nearrow 1}G'(t)	= \lim_{t\nearrow 1}\left(\frac{1}{\sqrt{1-t^2}} - \frac{1-\sqrt{1-t^2}}{t^2}\right) = +\infty. \\
p>1/2:	&			& \expe(T) = \lim_{t\nearrow 1}G'(t) = \frac{1}{p-q}.
\end{array}
\]
\end{remark}

%==========================================================================
\begin{exercise}
Consider a simple random walk $X_0,X_1,X_2,\ldots$ with $X_0=0$. Let $q$ be the probability that the random walk eventually returns to the starting position $X_0$. If $q=1$, position $X_0$ is called \emph{recurrent}; if $q<1$, position $X_0$ is called \emph{transient}. Show that $X_0$ is transient if and only if $\sum_{n=1}^{\infty} \prob(X_n = 0) < \infty$. [\textit{Hint}: find expressions for the expected number of times that $X_0$ is re-visited.]
\begin{answer}
Let
\[
I_n = \begin{cases}
1 & \text{if } X_n = 0 \\
0 & \text{otherwise.}
\end{cases}
\]
and let $N = \sum_{n=1}^{\infty} I_n$ be the number of times that state $X_0=0$ is revisited.

The expected value of $N$ is given by
\[
\expe(N)
	= \expe\left(\sum_{n=1}^{\infty} I_n\right)
	= \sum_{n=1}^{\infty} \expe(I_n)
	= \sum_{n=1}^{\infty} \prob(X_n = 0)
\]

The expected value of $N$ is also given by
\begin{align*}
\expe(N)
	& = \sum_{k=1}^{\infty} k\prob(N=k) \\
	& = \sum_{k=1}^{\infty} \big[ k\prob(N\geq k) - k\prob(N\geq k+1)\big] \\
	& = \sum_{k=1}^{\infty} k\prob(N\geq k) - \sum_{k=2}^{\infty} (k-1)\prob(N\geq k) \\
	& = \sum_{k=1}^{\infty} \prob(N\geq k) \\
	& = \sum_{k=1}^{\infty} q^k
\end{align*}
where the last equality follows by the fact that every return occurs independently with probability $q$. Combining these results, we get
\[
\sum_{n=1}^{\infty}\prob(X_n=0) = \sum_{k=1}^{\infty} q^k
\]
which diverges if $q=1$, and converges if $q<1$. Thus the random walk is recurrent precisely when $\sum_{n=1}^{\infty} \prob(X_n = 0)$ is infinite.
\end{answer}
\end{exercise}

%% reflection principle (first passage time distribution)
%%==========================================================================
%\begin{exercise}
%Consider a simple random walk $X_0,X_1,X_2,\ldots$ with $X_0=0$. Let $T_{\ell}$ be the first hitting time at level $\ell$, defined by $T_{\ell} = \min\{n\geq 0\,:\, X_n = \ell\}$. Use the reflection principle to show that the distribution of $T_{\ell}$ satisfies
%\[
%\prob(T_{\ell}\leq n) = \prob(X_n = \ell) + 2\prob(X_n > \ell).
%\]
%\begin[answer}
%Consider the sequence $X_n^{*}$ defined by
%\[
%X_n^{*} = \begin{cases}
%	X_n			& \text{if } n\leq\tau(m) \\
%	2m - X_n		& \text{if } n\geq\tau(m)
%\end{cases}
%\]
%By the reflection principle, $X_n^{*}$ is a simple random walk starting from $X_0^{*}=0$.
%%
%Consider the event $\tau(m)\leq n$. If this event occurs, $X_n$ and $X_n^{*}$ are on opposite sides of $m$ (unless they are both at $m$), and correspond under reflection. Since both processes are simple random walks starting at zero,
%\[
%\prob(X_n^{*} = m+k) = \prob(X_n = m+k) \qquad\text{for all } k\geq 0
%\]
%
%If $k\geq 0$, the event $X_n=m+k$ is impossible unless $\tau(m)\leq n$, so
%\begin{align*}
%\prob(X_n = m+k)
%	& = \prob\big(X_n 	= m+k \text{ and } \tau(m)\leq n\big) \\
%	& = \prob\big(X_n^{*}= m+k \text{ and } \tau(m)\leq n\big) \\
%	& = \prob\big(X_n 	= m-k \text{ and } \tau(m)\leq n\big) \\
%\end{align*}
%and therefore
%\begin{align*}
%\prob\big(\tau(m)\leq n\big)
%	& = \sum_{k=-\infty}^{\infty} \prob\big(X_n = m+k\text{ and }\tau(m)\leq n\big) \\
%	& = \prob(X_n = m) + 2\prob(X_n > m)
%\end{align*}
%as required.
%\end{answer}
%\end{exercise}

%-------------------------------------------------
\section{Branching processes}\label{sec:branching}
In Victorian England, several aristocratic families realised that their family names could become extinct. In 1873, Sir Francis Galton posed the following question in the \textit{Educational Times}:
\begin{quote}
How many male children (on average) must each generation of a family have for the family name to continue in perpetuity?
\end{quote}
The first complete answer was put forward by Reverend Henry Watson. Galton and Watson published a paper entitled \textit{On the probability of extinction of families} in 1874. Their model is as follows.

\ben
\it A population starts with a single individual, $Z_0=1$
\it At time $n=1$, this individual gives birth to $Z_1$ offspring, where $Z_1\in\{0,1,2,\ldots\}$, then dies.
\it If $Z_1=0$, the population is extinct and $Z_n=0$ for all $n\geq 2$.
\it If $Z_1>0$, each of the $Z_1$ individuals in the first generation gives birth to a random number of offspring at time $n=2$: the first has $Z_{1,1}$ offspring, the second has $Z_{1,2}$ offspring, ..., and the has last $Z_{1,Z_1}$ offspring.
\it Assume that every individual in every generation has the same offspring distribution, and the number of offpring born to any individual is independent of the number born to any other individual.
\it The total number of individuals in the second generation is
\[
Z_2 = \sum_{k=1}^{Z_1} Z_{1,k}.
\]
\it The third, fourth, fifth etc. generations are produced in the same way.
\it If it happens that $Z_m=0$ for some $m$ then $Z_n=0$ for all $m\geq n$, and the population is \emph{extinct}.
\een

\begin{definition}
A random process $Z_0,Z_1,Z_2\ldots$ with the properties described above is called a \emph{simple branching process} (or Galton-Watson process).
\end{definition}

The offspring distribution determines the evolution of a branching process. Galton's question is to find conditions on the offspring distribution under which
\[
\prob\big(Z_n\geq 1 \text{ for all } n=0,1,2,\ldots\big) = 1.
\]

Let $p_0,p_1,p_2,\ldots$ denote the offspring distribution, and let $G(t)=\sum_{k=1}^{\infty}p_k t^k$ be its PGF.

\begin{theorem}
The PGF of $Z_n$ is the $n$-fold composition of $G(t)$ with itself,
\[
G_{Z_n}(t) = \underbrace{G(G(\ldots G(t)\ldots))}_{\text{$n$ times}} \quad\qquad (n\geq 1).
\]
\begin{proof}
For $n=1$, $Z_1$ has distribution $p_0,p_1,p_2,\ldots$ so $G_{Z_1}(t) = G(t)$.
Suppose the statement holds for some $n\in\N$. Then
\[
Z_{n+1} = \sum_{i=1}^{Z_n} Z_{n,i}.
\]
is a random sum of $Z_n$ independent variables with PMF $p_0,p_1,\ldots$, and where the number of summands $Z_n$ is independent of the summands $Z_{n,1},Z_{n,2},\ldots,Z_{n,Z_n}$.

\bigskip
Hence, by Theorem~\ref{thm:GF-NofX},
\[
G_{Z_{n+1}}(t) = G_{Z_n}\big[G(t)\big]
\]
and by the inductive hypothesis,
\[
G_{Z_{n+1}}(t) = \underbrace{G(G(\ldots G(t)\ldots))}_{\text{$n+1$ times}}
\]
as required.
\end{proof}
\end{theorem}

%-------------------------
% theorem
\begin{theorem}
Let $Z_0,Z_1,Z_2,\ldots$ be a simple branching process, and let $\mu$ and $\sigma^2$ be the mean and variance of its offspring distribution. Then $\expe(Z_n)=\mu^n$ and
\[
\var(Z_n)
	= \sigma^2\mu^n(1+\mu+\mu^2+\cdots+\mu^n)
	= \begin{cases}
		\sigma^2(n+1)							& \text{if } \mu=1,\\
		\sigma^2\mu^n\left(\frac{1-\mu^{n+1}}{1-\mu}\right)	& \text{if } \mu\neq 1.
	\end{cases}
\]
\end{theorem}
%-------------------------
\begin{proof}
For brevity, let $G_n(t)$ denote the PGF $G_{Z_n}(t)$ of $Z_n$
\ben
\it % mean
To find the mean, differentiate $G_n(t) = G\big(G_{n-1}(t)\big)$,
\[
G'_n(t) = G'\big(G_{n-1}(t)\big)G'_{n-1}(t).
\]
Evaluating this at $t=1$, and using the fact that $G_{n-1}(1)=1$,
\[
\expe(Z_n) = G'_n(1) = G'(1)G'_{n-1}(1) = \mu\expe(Z_{n-1})
\]
The result then follows by induction.
\it % variance
To find the variance, differentiate $G_n(s)=G\big(G_{n-1}(s)\big)$ twice to obtain
\[
G''_n(1) = G''(1)G'_{n-1}(1)^2 + G'(1)G''_{n-1}(1)
\]
and substitute in the expression $\var(Z_n) = G''_n(1) + G'_n(1) - G'_n(1)^2$.
\een
\end{proof}

%-----------------------------
\subsection{Extinction probability}

The event that the population becomes extinct can be written as
\[
E = \big\{\omega\in\Omega:Z_n(\omega)=0 \text{ for some } n\in\N\}.
\]
This can be written as the union of an expanding sequence of events $E_1\subseteq E_2\subseteq\ldots$,
\[
E = \medcup_{n=1}^{\infty} E_n \quad\text{where}\quad E_n = \{\omega\in\Omega:Z_n(\omega)=0\}.
\]
By the continuity of probability measures, the \emph{extinction probability} is
\[
\prob(E) = \lim_{n\to\infty}\prob(E_n).
\]
Using the fact that $\prob(E_n) = \prob(Z_n=0) = G_{Z_n}(0)$,
\[
\prob(E) = \lim_{n\to\infty}G_{Z_n}(0) = \lim_{n\to\infty} \underbrace{G(G(\ldots G(0)\ldots))}_{\text{$n$ times}}
\]

Remarkably, the extinction probability $\prob(E)$ can be computed even when $G_{Z_n}$ is not known explicitly.
\begin{theorem}
The extinction probability is the smallest non-negative solution of the so-called \emph{extinction equation},
\[
x = G(x)
\]
where $G$ is the PGF of the offspring distribution.
\end{theorem}

\begin{proof}
Let $e=\prob(E)$ be the extinction probability. First we show that $e$ is a solution of $x=G(x)$. Let
\[
x_n = \underbrace{G(G(\ldots G(0)\ldots))}_{\text{$n$ times}}
\]
Then (1) $e=\lim_{n\to\infty}x_n$ and (2) $G(x_n) = x_{n+1}$, so
\[
e 	= \lim_{n\to\infty}x_n
	= \lim_{n\to\infty}x_{n+1}
	= \lim_{n\to\infty}G(x_n)
	= G(\left(\lim_{n\to\infty}x_n\right)
	= G(e),
\]
where we have used the fact that $G$ is a continuous function.

To show that $e=\prob(E)$ is the smallest solution, let $e'$ be another solution of $x=G(x)$ in $[0,1]$. Since $e'\geq 0$ and $G(t)$ is a non-decreasing function,
\[
G(0) \leq G(e') = e'.
\]
Applying $G$ to both sides, since $G(t)$ is increasing,
\[
G(G(0)) \leq G(G(e')) = G(e') = e'.
\]
Repeating this procedure, we get
\[
\prob(E_n) = \underbrace{G(G(\ldots G(0)\ldots))}_{\text{$n$ times}} \leq e'.
\]
Hence,
\[
e =\prob(E) = \lim_{n\to\infty}\prob(E_n) \leq \lim_{n\to\infty} e' = e',
\]
so $e$ is not larger than any other solution $e'$ of $x=G(x)$.
\end{proof}

%-------------------------------------------------
\section{Martingales}\label{sec:martingales}
%-------------------------
% definition
\begin{definition}
A random process $X_0,X_1,\ldots$ is called a \emph{martingale} with respect to the sequence of random variables $\xi_1,\xi_2,\ldots$ if
\ben
\it $\expe(|X_n|) < \infty$ and
\it $\expe(X_{n+1}|\xi_1,\ldots,\xi_n) = X_n$.
\een
\bit
\it A \emph{sub-martingale} has $\expe(X_{n+1}|\xi_1,\ldots,\xi_n) \geq X_n$ (the process tends to increase over time).
\it A \emph{super-martingale} has $\expe(X_{n+1}|\xi_1,\ldots,\xi_n) \leq X_n$ (the process tends to decrease over time).
\eit
\end{definition}

%-------------------------
% example
\begin{example}[Random walk]
Let $X_n=X_0+\sum_{k=1}^n \xi_n$ be a symmetric simple random walk (so $\prob(\xi_k=1)=\prob(\xi_k=-1) = 1/2$). Show that $\{X_n\}$ is a martingale with respect to the increments $\xi_1,\xi_2,\ldots$.
\begin{solution}
For the first condition, $\expe(|\xi_i|)=1$ so
\[
\expe(|X_n|) = \expe\left(\left|\sum_{i=1}^{n} \xi_i\right|\right) \leq \sum_{i=1}^{n} \expe(|\xi_i|) < \infty.
\]
For the second condition,
\begin{align*}
\expe(X_{n+1}|\xi_1,\xi_2,\ldots,\xi_n)
	& = \expe(\xi_1+\xi_2\ldots+\xi_{n+1}|\xi_1,\ldots,\xi_n) \\
	& = \expe(\xi_1|\xi_1,\ldots,\xi_n) + \expe(\xi_2|\xi_1,\ldots,\xi_n) + \ldots + \expe(\xi_{n+1}|\xi_1,\ldots,\xi_n) \\
	& = \expe(\xi_1|\xi_1) + \expe(\xi_2|\xi_2) + \ldots + \expe(\xi_n|\xi_n) + \expe(\xi_{n+1}) \qquad \text{by independence} \\
	& = \xi_1 + \xi_2 + \ldots + \xi_n + 0 \\
	& = X_n.
\end{align*}
\end{solution}
\end{example}

%-------------------------
% example
\begin{example}[Sub-martingale]
Let $\xi_1,\xi_2,\ldots$ be independent random variables with zero means, finite variances and partial sums $X_n=\sum_{i=1}^n \xi_i$. Show that $X_n^2$ is a sub-martingale with respect to $\xi_1,X_2,\ldots$.
\begin{solution}
Since $X^2_{n+1} = (X_n + \xi_{n+1})^2$,
\begin{align*}
\expe(X^2_{n+1}|\xi_1,\xi_2,\ldots,\xi_n)
	& = \expe(X_n^2 + 2X_n\xi_{n+1} + \xi_{n+1}^2 | \xi_1,\ldots,\xi_{n}) \\
	& = X^2_n + 2\expe(\xi_{n+1})\expe(X_n | \xi_1,\ldots,\xi_n) + \expe(\xi_{n+1}^2) \qquad \text{by independence,} \\
	& = X^2_n + \expe(\xi_{n+1}^2) \geq T_n \qquad\text{because $\expe(\xi_{n+1})=0$.}
\end{align*}
Since $\xi_{n+1}^2>0$, $\expe(X^2_{n+1}|\xi_1,\xi_2,\ldots,\xi_n)\geq X^2_n$, so $X^2_n$ is a sub-martingale.
\end{solution}
\end{example}

%-------------------------
% thm: MCT
\begin{theorem}[Martingale Convergence Theorem]
Let $X_0,X_1,\ldots$ be a martingale such that $\expe(|X_n|)$ is bounded for all $n=0,1,2,\ldots$. Then there exists a finite random variable $X$ such that $X_n\to X$ with probability one as $n\to\infty$.
\end{theorem}

The MCT can be used to prove the following remarkable theorem.
%-------------------------
% example (application of MCT}
\begin{theorem}
A symmetric simple random walk on $\Z$ will visit every point with probability 1.
\end{theorem}
\begin{proof}
Let $X_0=0$ and $b\in\Z$, and suppose (without loss of generality) that $b<0$. Let $T$ be the first time $n$ for which $X_n=b$, and consider the corresponding \emph{stopped process} $\tilde{X}_0,\tilde{X}_1,\tilde{X}_2,\ldots$ defined by $\tilde{X}_n = X_{\min\{n,T\}}$.
\bit
\it The stopped process remains in position $b$ from time $T$ onwards.
\eit

It is easy to show that $\tilde{X}_n$ is a martingale, and because $\tilde{X}_n - b$ is non-negative, $\expe(|\tilde{X}_n - b|) = \expe(\tilde{X}_n - b) < \infty$.

\bigskip
By the Martingale Convergence Theorem, the limit $\tilde{X} = \lim_{n\to\infty}\tilde{X}_n$ exists and is finite with probability one.
\bit
\it In particular, $|\tilde{X}_{n+1} - \tilde{X}_n|$ converges to zero, and must therefore  be less than 1 for large $n$.
\it However $|\tilde{X}_{n+1} - \tilde{X}_n| = 1$ whenever $n < T$.
\it Thus we have $T<\infty$, and hence $X_n = b$ for some $n$.
\eit
\end{proof}

% ex: branchingn process
\begin{exercise}
Let $Z_0,Z_1,Z_2,\ldots$ be a simple branching process with $Z_0=1$. Show that the sequence $W_1,W_2,\ldots$ with $W_n=Z_n/\expe(Z_n)$ is a martingale with respect to $Z_1,Z_2,\ldots$.
\begin{answer}
Conditioned on $Z_n=z_n$, the number $Z_{n+1}$ is the sum of $z_n$ independent familiy sizes,
\[
\expe(Z_{n+1}: Z_n=z_n) = \mu z_n
\]
where $\mu$ is the expected family size. By the Markov property,
\[
\expe(Z_{n+1}: Z_1,Z_2,\ldots,Z_n) = \mu Z_n
\]
Since $\expe(Z_n)=\mu^n$, we conclude that
\[
\expe(W_{n+1}: Z_1,Z_2,\ldots,Z_n) = W_n
\]
as required.
\end{answer}
\end{exercise}

%==========================================================================
\begin{exercise}
An urn contains one red ball and one green ball. At each time step, we choose one ball uniformly at random from the urn, and replace it along with another ball of the same colour. Let $R_n$ and $G_n$ respectively denote the number of red balls and green balls after $n$ steps, and let $M_n$ denote the fraction of green balls in the urn.
\ben
\it Show that $M_n$ is a martingale.
\it Show that $M_n$ converges to a finite limit with probability 1 as $n\to\infty$
\een
\begin{answer}
\ben
\it % (a)
$M_n$ is a martingale because
\begin{align*}
\expe(M_{n+1} | R_0,G_0,\ldots,R_n,G_n)
	& = \left(\frac{R_n}{R_n+G_n}\right)\left(\frac{G_n}{R_n+G_n+1}\right)
	  + \left(\frac{G_n}{R_n+G_n}\right)\left(\frac{G_n+1}{R_n+G_n+1}\right) \\
	& = \frac{G_n}{R_n+G_n} \\
	& = M_n
\end{align*}
\it % (b)
Since $M_n\geq 0$ is bounded for all $n\in\N$, it follows by the martingale that there exists a finite random variable $M$ such that $M_n\to M$ with probability one as $n\to\infty$. In fact, it can be shown that
\[
\prob(G_n = m+1) = \binom{m}{n}\frac{m!(n-m)!}{(n+1)!} = \frac{1}{n+1}
\]
and hence
\[
\prob(M_n\leq x) = \frac{\lfloor x(n+2)-1\rfloor}{n+1} \to x \quad\text{as }n\to\infty
\]
Thus the distribution of $M_n$ approaches a uniform distribution on $[0,1]$ as $n\to\infty$.
\een
\end{answer}
\end{exercise}

\endinput


% !TEX root = main.tex

%-------------------------------------------------
\section{The Cram\'{e}r-Rao lower bound}\label{sec:crlb}

We now show that the variance of any unbiased estimator cannot be smaller than a certain fixed value which depends only on the Fisher information of the associated distribution. First we need the following lemma.
%\bit
%\it Let $X$ be a random variable having PDF $f(x,\theta)$, where $\theta$ is an unknown scalar parameter.
%\it Let $X_1,X_2,\ldots,X_n$ be a random sample from the distribution of $X$.
%\it Let $I(\theta)$ be the Fisher information of a single observation,
%\[
%I(\theta) = \var\big[u(\theta;X)\big]
%\quad\text{where}\quad
%u(\theta;X) = \frac{\partial}{\partial\theta}\log f(X;\theta).
%\]
%\eit
%
\begin{lemma}
%Let $X$ be a random variable, let $f(x,\theta)$ denote its PDF, l
Let $T(X)$ be an unbiased estimator of $\theta$ and let $U(X)$ be the score function of $X$. Then
\[
\cov(T,U) = 1
\]
\end{lemma}

\begin{proof}
We prove the lemma for continuous random variables.
\par
By Lemma~\ref{lem:expe_score_function}, $\expe(U)=0$ so
\begin{align*}
\cov(T,U)
	& = \expe(TU) - \expe(T)\expe(U) = \expe(TU) \\
%	= \expe(TU) \\
	& = \expe\left[T(X)\left(\frac{\partial}{\partial\theta}\log f(X;\theta)\right)\right] \\
	& = \expe\left[T(X)\left(\frac{1}{f(X;\theta)}\frac{\partial}{\partial\theta}f(X;\theta)\right)\right] \\
	& = \int T(x)\left(\frac{1}{f(x;\theta)}\frac{\partial}{\partial\theta}f(x;\theta)\right)f(x;\theta)\,dx \\
	& = \int T(x)\left(\frac{\partial}{\partial\theta}f(x;\theta)\right)\,dx\\
	& = \frac{\partial}{\partial\theta}\int T(x)f(x;\theta)\,dx \quad\text{(by the regularity conditions)} \\
	& = \frac{\partial}{\partial\theta}\expe(T)
	= \frac{\partial}{\partial\theta}\theta
	= 1\qquad\text{(because $T$ is unbiased).}
\end{align*}
\end{proof}

% thm: CRLB
\begin{theorem}[CRLB]\label{thm:crlb}
Let $X$ be a random variable and let $f(x;\theta)$ denote its PDF where $\theta$ is an unknown scalar. If $T(X)$ is an unbiased estimator of $\theta$ then
\[
\var(T) \geq \frac{1}{I(\theta)}.
\]
\end{theorem}

% proof
\begin{proof}
%Because $\expe(U)=0$ and $\expe(T)=0$,
By the Cauchy-Schwarz inequality applied to $T-\expe(T)$ and $U-\expe(U)$,
\begin{align*}
1 = \cov(T,U)^2
	& = \expe\big([T-\expe(T)][U-\expe(U)]\big)^2 \\
	& \leq \expe\big([T-\expe(T)]^2\big)\expe\big([U-\expe(U)]^2\big) \\
	& = \var(T)\var(U)
\end{align*}
Hence
\[
\var(T) \geq \frac{1}{\var(U)}.
\]
Thus, because $I(\theta)=\var(U)$, i.e. the variance of the score function, we conclude that
\[
\var(T) \geq \frac{1}{I(\theta)}.
\]
\end{proof}

% thm: CRLB
\begin{corollary}\label{cor:crlb}
Let $X_1,X_2,\ldots,X_n$ be a random sample and let $f(x,\theta)$ denote their common PDF, where $\theta$ is an unknown scalar. If $T_n$ is an unbiased estimator of $\theta$ then
\[
\var(T_n) \geq \frac{1}{nI(\theta)}
\]
where $I(\theta)$ is the Fisher information of a single observation.
\end{corollary}

% remark
\begin{remark}
\bit
\it The CRLB provides a \emph{lower limit} on the variance of an unbiased estimator.
%\it The CRLB provides an \emph{upper limit} on the amount of information we can extract from a random sample.
\it Similar results hold for biased estimators, vectors of parameters, non-independent samples, and so on.
\it The CRLB has deep connections with the \emph{Heisenberg Uncertainty Principle}.
\eit
\end{remark}

%----------------------------------------------------------------------
\section{Efficiency}
%----------------------------------------------------------------------

% definition: efficiency
\begin{definition}
Let $T$ be an unbiased estimator of the parameter $\theta$.
\ben
\it
The \emph{efficiency} of $T$ as an estimator of $\theta$ is defined by
$e(T) = \displaystyle\frac{1/I(\theta)}{\var(T)}$.
\it
If $e(T)=1$ then $T$ is called an \emph{efficient} estimator of $\theta$.
\een
\end{definition}
An efficient estimator is thus an unbiased estimator whose variance achieves the Cram\'{e}r-Rao lower bound (for all $\theta$) and which therefore has the smallest variance among all possible unbiased estimators.

%----------------------------------------
% example: mean of a Bernoulli sample
\begin{example}
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $\text{Bernoulli}(\theta)$ distribution, where $0<\theta<1$ is unknown. Show that the sample mean $\bar{X}$ is an efficient estimator of $\theta$.
\end{example}

\begin{solution}
Let $X\sim\text{Bernoulli}(\theta)$. Then $\var(X)=\theta(1-\theta)$, so
\[
\var(\bar{X})=\frac{\theta(1-\theta)}{n}.
\]
\par
\par
The PMF of $X$ is
$
f(x;\theta)=\theta^x(1-\theta)^{1-x}\quad\text{for $x\in\{0,1\}$ (and zero otherwise).}
$
%for $x\in\{0,1\}$, and zero otherwise.
\par
The log-likelihood function of a single observation is
\[
%\log f(x;\theta) = \displaystyle\frac{x-\theta}{\theta(1-\theta)}
\log f(x;\theta) = x\log\theta + (1-x)\log(1-\theta).
\]
The score function of a single observation is
\[
u(\theta;x)
	= \frac{\partial}{\partial\theta}\log f(x,\theta)
%	= \frac{\partial}{\partial\theta}\left(\frac{1}{\theta(1-\theta)}\right)
	= \frac{x}{\theta} - \frac{1-x}{1-\theta}
	= \frac{x-\theta}{\theta(1-\theta)}.
\]
Since $\expe(X)=\theta$ and $\var(X)=\theta(1-\theta)$, the Fisher information of a single observation is
\[
I(\theta) = \expe\left[\left(\frac{\partial\log f(x,\theta)}{\partial\theta}\right)^2\right]
	= \frac{\expe\big[(X-\theta)^2\big]}{\theta^2(1-\theta)^2}
	= \frac{\var(X)}{\theta^2(1-\theta)^2}
	= \frac{1}{\theta(1-\theta)}.
\]
Hence for a random sample of size $n$, because $\bar{X}$ is an unbiased estimator of $\theta$, the Cram\'{e}r-Rao lower bound yields
\[
\var(\bar{X}) \geq \frac{\theta(1-\theta)}{n}.
\]
But we know that $\var(\bar{X})=\theta(1-\theta)/n$, so the variance of $\bar{X}$ attains the CRLB. Thus it follows that $\bar{X}$ is an efficient estimator of $\theta$.
\end{solution}

% example: mean of normal sample
\begin{example}
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $N(\theta,\sigma^2)$ distribution, whose mean $\theta$ is unknown but whose variance $\sigma^2$ is known. Show that $\bar{X}$ is an efficient estimator of $\theta$.
\end{example}
\begin{solution}
Let $X\sim N(\theta,\sigma^2)$. By Example~\ref{example:fisher_information_normal}, the Fisher information of $X$ is
\[
I(\theta) = \displaystyle\frac{1}{\sigma^2}.
\]
$\bar{X}$ is an unbiased estimator of $\theta$, so its variance is bounded below by the CRLB:
\[
\var(\bar{X}) \geq \frac{1}{nI(\theta)}
\qquad\text{so}\qquad
\var(\bar{X}) \geq \frac{\sigma^2}{n}.
\]
Since $\var(\bar{X}) = \displaystyle\frac{\sigma^2}{n}$, we see that $\var(\bar{X})$ attains the CRLB, so $\bar{X}$ is an efficient estimator for $\theta$.
\end{solution}

\begin{example}
Let $X$ be a continuous random variable with the following PDF:
\[
f(x) = \begin{cases}
	\displaystyle\frac{3\theta^3}{(x+\theta)^4}	& \text{ for $x>0$,} \\
	0											& \text{ otherwise.}
\end{cases}
\]
\ben
\it Show that $\expe(X)=\theta/2$ and $\var(X) = 3\theta^2/4$.
\it Show that $T = 2\bar{X}$ is an unbiased esimator of $\theta$, and find its variance.
\it Show that the efficiency of $T$ as an estimator of $\theta$ is $5/9$.
\een
\begin{solution}
\ben
\it % <<< mean and variance of sample points
Integration by parts:
\begin{align*}
\expe(X)
	& = 3\theta^3 \int_0^{\infty} \frac{x}{(x+\theta)^4}\,dx \\
	& = 3\theta^3\left(\left[\frac{-x}{(x+\theta)^3}\right]_0^\infty - \int_0^\infty \frac{-1}{3(x+\theta)^2}\,dx\right)
	= 3\theta^3\left[\frac{-1}{6(x+\theta)^2}\right]_0^\infty = \frac{\theta}{2}. \\
\expe(X^2)
	& = 3\theta^3 \int_0^{\infty} \frac{x^2}{(x+\theta)^4}\,dx \\
	& = 3\theta^3\left(\left[\frac{-x^2}{(x+\theta)^3}\right]_0^\infty - \int_0^\infty 2x\frac{-1}{3(x+\theta)^3}\,dx\right) \\
	& = 2\theta^3\int_0^\infty \frac{x}{(x+\theta)^3}\,dx \\
	& = 2\theta^3\left(\left[\frac{-x}{2(x+\theta)^2}\right]_0^\infty - \int_0^\infty \frac{-1}{2(x+\theta)^2}\,dx\right)
	= \theta^3\left[\frac{-1}{x+\theta}\right]_0^\infty = \theta^2.
	\end{align*}
Hence $\var(X) = \expe(X^2)-\expe(X)^2 = \theta^2 - \theta^2/4 = 3\theta^2/4$.
\it % <<< bias & variance of T
The mean and variance of $T$ are
\begin{align*}
\expe(T)
	& = 2\expe(\bar{X}) = 2\expe(X) = \theta \quad\text{so $T$ is unbiased}, \\
\var(T)
	& = 4\var(\bar{X}) = 4\var(X)/n = 3\theta^2/n.
\end{align*}
\it % <<< efficiency
\bit
\it $f(x) = 3\theta^3/(x+\theta)^4$.
\it $\ell(\theta) = \log 3 + 3\log\theta - 4\log(x+\theta)$.
\it $\ell'(\theta) = \displaystyle\frac{3}{\theta} - \frac{4}{x+\theta}$.
\it $\ell''(\theta) = -\displaystyle\frac{3}{\theta^2} + \frac{4}{(x+\theta)^2}$.
\it $I(\theta) = -\expe\big[\ell''(\theta;X)\big] = \displaystyle\frac{3}{\theta^2} - 4\expe\left(\frac{1}{(X+\theta)^2}\right)$.
\eit
Now,
\begin{align*}
\expe\left(\frac{1}{(X+\theta)^2}\right)
	& = \int_0^\infty \left(\frac{1}{(x+\theta)^2}\right)\left(\frac{3\theta^3}{(x+\theta)^4}\right)\,dx \\
	& = 3\theta^3 \int_0^\infty \frac{1}{(x+\theta)^6}\,dx \\
	& = 3\theta^3\left[\frac{-1}{5(x+\theta)^5}\right]_0^\infty
	= \frac{3}{5\theta^2}.
\end{align*}
Hence,
\[
I(\theta) = \frac{3}{\theta^2} - \frac{12}{5\theta^2} = \frac{3}{5\theta^2}
\quad\text{and}\quad
I_n(\theta) = nI(\theta) = \frac{3n}{5\theta^2}
\]
Since $\var(T)=3\theta^2/n$, we see that
\[
\text{Efficiency}(T) = \frac{1/I_n(\theta)}{\var{T}} = \frac{5\theta^2/3n}{3\theta^2/n} = \frac{5}{9}.
\]
\een
\end{solution}
\end{example}

%%-----------------------------
%% example: variance of normal sample
%\begin{example}
%Let $X_1,X_2,\ldots,X_n$ be a random sample from the $N(\mu,\theta)$ distribution, whose mean $\mu$ is known but whose variance $\theta$ is unknown. Using the fact that, for the normal distribution, $\expe\big((X-\mu)^4\big)=3\theta^2$, show that the statistic
%\[
%T(X_1,X_2,\ldots,X_n) = \frac{1}{n}\sum_{i=1}^{n} (X_i-\mu)^2
%\]
%is an efficient estimator of the variance $\theta$.
%\end{example}
%
%\begin{solution}
%Let $X\sim N(\mu,\theta)$. First we note that $T$ is unbiased, because
%\[
%\expe(T) = \frac{1}{n}\sum_{i=1}^{n} \expe\big[(X_i-\mu)^2\big] = \var(X) = \theta.
%\]
%
%Furthermore, because the $X_i$ are independent, the variance of $T$ is
%\begin{align*}
%\var(T)
%	= \frac{1}{n}\var\big[(X-\mu)^2\big]
%	& = \frac{1}{n}\Big(\expe\big[(X-\mu)^4\big] - \expe\big[(X-\mu)^2\big]^2\Big) \\
%	& = \frac{1}{n}\big(3\theta^2 - \theta^2\big) \\
%	& = \frac{2\theta^2}{n}
%\end{align*}
%
%Let $f(x;\theta)$ denote the PDF of the $N(\mu,\theta)$ distribution:
%\begin{align*}
%f(x;\theta)
%	& = \ \frac{1}{\sqrt{2\pi\theta}}\exp\left(-\frac{(x-\mu)^2}{2\theta}\right), \\[2ex]
%\log f(x;\theta)
%	& = \ \log\left(\frac{1}{\sqrt{2\pi}}\right) + \log\left(\frac{1}{\sqrt{\theta}}\right)  - \frac{(x-\mu)^2}{2\theta}, \\[2ex]
%\frac{\partial}{\partial\theta}\log f(x;\theta)
%	& = \ -\frac{1}{2\theta} + \frac{(x-\mu)^2}{2\theta^2}, \\[2ex]
%\frac{\partial^2}{\partial\theta^2}\log f(x;\theta)
%	& = \ \frac{1}{2\theta^2} - \frac{(x-\mu)^2}{\theta^3}.
%\end{align*}
%
%The Fisher information of a single observation is thus given by
%\begin{align*}
%I(\theta)
%	= -\expe\left(\frac{\partial^2}{\partial\theta^2}\log f(X;\theta)\right)
%	& = -\expe\left(\frac{1}{2\theta^2} - \frac{(X-\mu)^2}{\theta^3}\right) \\
%	& = -\frac{1}{2\theta^2} + \frac{\expe(X-\mu)^2}{\theta^3} \\
%	& = -\frac{1}{2\theta^2} + \frac{\theta}{\theta^3}
%	= \frac{1}{2\theta^2}.
%\end{align*}
%
%By the Cram\'{e}r-Rao theorem,
%\[
%\var(T) \geq \frac{1}{nI(\theta)} = \frac{2\theta^2}{n}
%\]
%
%\bit
%\it The variance of $T$ attains the CRLB for all values of $\theta>0$.
%\it $T$ is therefore an efficient estimator of the variance of the Normal distribution.
%\eit
%\end{solution}

% !TEX root = main.tex
%----------------------------------------------------------------------
\begin{exercise}
\begin{questions}
%----------------------------------------
\question
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $\text{Poisson}(\theta)$ distribution, where $\theta>0$ is unknown. Show that the sample mean $\bar{X}$ is an efficient estimator of $\theta$.

\begin{answer}
The PMF of the $\text{Poisson}(\theta)$ distribution is
\[
f(x;\theta) = \frac{\theta^x\exp(-\theta)}{x!} \quad \text{for $x=0,1,2,3,\ldots$ (zero otherwise)}
\]
Let $X\sim\text{Poisson}(\theta)$. The score function of $X$ is
\[
u(\theta;X)
	= \frac{\partial}{\partial\theta}\log f(X,\theta)
	= \frac{\partial}{\partial\theta}(X\log\theta - \theta - \log x!)
	= \frac{X-\theta}{\theta}.
\]
Since $\expe(X)=\theta$ and $\var(X)=\theta$, the Fisher information of a single observation is therefore
\[
I(\theta)
	= \expe\left[\left(\frac{\partial}{\partial\theta}\log f(x,\theta)\right)^2\right]
	= \frac{\expe\big[(X-\theta)^2\big]}{\theta^2} = \frac{\var(X)}{\theta^2} = \frac{1}{\theta}.
\]
Hence for a random sample of size $n$, the CRLB is equal to
\[
\frac{1}{nI(\theta)} = \frac{\theta}{n}.
\]
This is equal to the variance of the sample mean $\bar{X}$, so $\bar{X}$ is an efficient estimator of $\theta$.

\end{answer}

%----------------------------------------
\question
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $N(\mu,\theta)$ distribution, whose mean $\mu$ is known but whose variance $\theta$ is unknown. Using the fact that $\expe\big((X-\mu)^4\big)=3\theta^2$ for the normal distribution, show that the statistic
\[
T = \frac{1}{n}\sum_{i=1}^{n} (X_i-\mu)^2
\]
is an efficient estimator of the variance $\theta$.

\begin{answer}
Let $X\sim N(\mu,\theta)$. First we note that $T$ is unbiased, because
\[
\expe(T) = \frac{1}{n}\sum_{i=1}^{n} \expe\big[(X_i-\mu)^2\big] = \var(X) = \theta.
\]

Furthermore, because the $X_i$ are independent, the variance of $T$ is
\begin{align*}
\var(T)
	= \frac{1}{n}\var\big[(X-\mu)^2\big]
	& = \frac{1}{n}\Big(\expe\big[(X-\mu)^4\big] - \expe\big[(X-\mu)^2\big]^2\Big) \\
	& = \frac{1}{n}\big(3\theta^2 - \theta^2\big) \\
	& = \frac{2\theta^2}{n}
\end{align*}

Let $f(x;\theta)$ denote the PDF of the $N(\mu,\theta)$ distribution:
\begin{align*}
f(x;\theta)
	& = \ \frac{1}{\sqrt{2\pi\theta}}\exp\left(-\frac{(x-\mu)^2}{2\theta}\right), \\[2ex]
\log f(x;\theta)
	& = \ \log\left(\frac{1}{\sqrt{2\pi}}\right) + \log\left(\frac{1}{\sqrt{\theta}}\right)  - \frac{(x-\mu)^2}{2\theta}, \\[2ex]
\frac{\partial}{\partial\theta}\log f(x;\theta)
	& = \ -\frac{1}{2\theta} + \frac{(x-\mu)^2}{2\theta^2}, \\[2ex]
\frac{\partial^2}{\partial\theta^2}\log f(x;\theta)
	& = \ \frac{1}{2\theta^2} - \frac{(x-\mu)^2}{\theta^3}.
\end{align*}

The Fisher information of a single observation is thus given by
\begin{align*}
I(\theta)
	= -\expe\left(\frac{\partial^2}{\partial\theta^2}\log f(X;\theta)\right)
	& = -\expe\left(\frac{1}{2\theta^2} - \frac{(X-\mu)^2}{\theta^3}\right) \\
	& = -\frac{1}{2\theta^2} + \frac{\expe(X-\mu)^2}{\theta^3} \\
	& = -\frac{1}{2\theta^2} + \frac{\theta}{\theta^3}
	= \frac{1}{2\theta^2}.
\end{align*}

By the Cramer-Rao theorem,
\[
\var(T) \geq \frac{1}{nI(\theta)} = \frac{2\theta^2}{n}
\]

\bit
\it The variance of $T$ attains the CRLB for all values of $\theta>0$.
\it $T$ is therefore an efficient estimator of the variance of the Normal distribution.
\eit
\end{answer}

\question % rayleigh
Let $X\sim\text{Rayleigh}(\theta)$ whose CDF is given by
\[
F(x) = 1 - e^{-x^2/2\theta} \quad\text{for $x\geq 0$ and zero otherwise.}
\]
\ben
\it Show that $X^2\sim\text{Exponential}(2\theta)$ where $2\theta$ is a scale paremter.
\it Show that the MLE of $\theta$ is $\displaystyle T=\frac{1}{2n}\sum_{i=1}^n X_i^2$.
\it Show that the $T$ is an unbiased estimator of $\theta$.
\it Show that the $T$ is an efficient estimator of $\theta$.
\een
\begin{answer}
\ben
\it % <<< transformation
Let $Y=g(X)$ where $g(x)=x^2$. This is one-to-one and increasing over $\supp(f_X)=[0,\infty)$. The inverse transformation is $g^{-1}(y) = \sqrt{y}$ and $\supp(f_Y)=[0\infty)$. Hence
\[
F_Y(y) = F_X\big[g^{-1}(y)\big] = 1 - e^{-y/2\theta} \quad\text{for $y\geq 0$, and zero otherwise.}
\]
This is the CDF of the $\text{Exponential}(2\theta)$ distribution, where $2\theta$ is a scale parameter.
\it % <<< MLE
\bit
\it $f(x,\theta) = \displaystyle\frac{x}{\theta}e^{-x^2/2\theta}$.
\it $L(\theta) = \prod_{i=1}^n \displaystyle\frac{x_i}{\theta}e^{-x_i^2/2\theta}$.
\it $\ell(\theta) = -n\log\theta + \sum_{i=1}^n\log x_i - \displaystyle\frac{1}{2\theta}\sum_{i=1}^n x_i^2$.
\it $\ell'(\theta) = -\displaystyle\frac{n}{\theta} + \frac{1}{2\theta^2}\sum_{i=1}^n x_i^2$.
\it $\ell''(\theta) = \displaystyle\frac{n}{\theta^2} - \frac{1}{\theta^3}\sum_{i=1}^n x_i^2$.
\eit
Setting $\ell'(\theta)=0$, the MLE of $\theta$ is $\displaystyle T=\frac{1}{2n}\sum_{i=1}^n X_i^2$.
\it % <<< bias
Because $X^2\sim\text{Exponential}(2\theta)$ we have $\expe(X^2)=2\theta$, so
\[
\expe(T) = \frac{1}{2n}\sum_{i=1}^n \expe(X_i^2) = \theta.
\]
so $T$ is unbiased.

\it % <<< efficiency
Again using the fact that $\expe(X^2) = 2\theta$,
\begin{align*}
I_n(\theta) = -\expe\big[\ell''(\theta;\mathbf{X})\big]
	& = -\frac{n}{\theta^2} + \frac{1}{\theta^3}\sum_{i=1}^n \expe(X_i^2)
	= -\frac{n}{\theta^2} + \frac{n}{\theta^3}2\theta
	= \frac{n}{\theta^2}.
\end{align*}
Because $X^2\sim\text{Exponential}(2\theta)$ we have $\expe(X^2)=2\theta$ and $\var(X^2) = 4\theta^2$. By indepedence,
\[
\var(T) = \frac{1}{4n^2}\sum_{i=1}^n\var(X_i) = \frac{n}{4n^2}4\theta^2 = \frac{\theta^2}{n}
\]
Thus $\var(T)$ achieves the CRLB, so $T$ is an efficient estimator for $\theta$.
\een
\end{answer}

%----------------------------------------
\question
Let $X\sim\text{Exponential}(\lambda)$ where $\lambda>0$ is an unknown rate parameter and let $X_1,X_2,\ldots,X_n$ be a random sample from the distribution of $X$. The PDF of $X$ is
\[
f(x) = \begin{cases}
	\lambda e^{-\lambda x}	& \text{ for $x>0$,} \\
	0						& \text{ otherwise.}
\end{cases}
\]
Let $S_n=\sum_{i=1}^n X_i$. This is the sum of $n$ independent $\text{Exponential}(\lambda)$ random variables, and has the so-called \emph{Erlang} distribution with parameters $n\in\N$ and $\lambda>0$, whose PDF is given by
\[
f_S(s) = \frac{\lambda^n s^{n-1}e^{-\lambda s}}{\Gamma(k)} \quad\text{for $s>0$ and zero otherwise,}
\]
where $\Gamma(k)$ is the so-called \emph{gamma function},
\[
\Gamma(k) = \int_{0}^{\infty} t^{k-1}e^{-t}\,dt.
\]
$\Gamma(k)$ is an extension of the factorial function and has the property $\Gamma(k+1) = k\Gamma(k)$.

\ben
\it Show that
$\expe(S_n^{-1}) = \displaystyle\frac{\lambda}{n-1}$ and
$\var(S_n^{-1}) = \displaystyle\frac{\lambda^2}{(n-1)^2(n-2)}$.
%$\displaystyle\expe\left(\frac{1}{S_n}\right) = \frac{\lambda}{n-1}$ and
%%$\displaystyle\expe\left(\frac{1}{S_n^2}\right) = \frac{\lambda^2}{(n-1)(n-2)}$.
%$\displaystyle\var\left(\frac{1}{S_n}\right) = \frac{\lambda^2}{(n-1)^2(n-2)}$.
\it Show that the MLE of $\lambda$ is given by $T_n = \displaystyle\frac{n}{\sum_{i=1}^n X_i}$.
\it Show that $T_n$ is an asymptotically unbiased estimator for $\lambda$ as $n\to\infty$.
\it Show that $T_n$ is an asymptotically efficient estimator of $\lambda$, in the sense that its variance converges to the CRLB as $n\to\infty$.
\een

\begin{answer}
\ben
\it % <<< inverse moments
\begin{align*}
\expe(S_n^{-1})
	& = \frac{\lambda^n}{\Gamma(n)}\int_0^\infty s^{n-2}e^{-\lambda s}\,ds
	= \frac{\lambda}{\Gamma(n)}\int_0^\infty u^{n-2}e^{-u}\,du
	= \frac{\lambda\Gamma(n-1)}{\Gamma(n)} = \frac{\lambda}{n-1}. \\
\expe(S_n^{-2})
	& = \frac{\lambda^n}{\Gamma(n)}\int_0^\infty s^{n-3}e^{-\lambda s}\,ds
	= \frac{\lambda^2}{\Gamma(n)}\int_0^\infty u^{n-3}e^{-u}\,du
	= \frac{\lambda^2\Gamma(n-2)}{\Gamma(n)} = \frac{\lambda^2}{(n-1)(n-2)}. \\
\var(S_n^{-1})
	& = \expe(S_n^{-2}) -\expe(S_n^{-1})^2 =  \frac{\lambda^2}{(n-1)^2(n-2)}.
\end{align*}
\it % <<< mle
\bit
\it $L(\lambda) = \prod_{i=1}^n \lambda e^{-\lambda x_i}$.
\it $\ell(\lambda) = n\log\lambda + (\theta-1)\sum_{i=1}^n x_i$.
\it $\ell'(\lambda) = n/\theta + \sum_{i=1}^n x_i$.
\it $\ell''(\lambda) = -n/\theta^2 < 0$.
\eit
Setting $\ell'(\lambda)=0$ we obtain $T=n/\sum_{i=1}^n X_i$ as the MLE of $\lambda$.
\it % <<< bias
$T_n$ is an asymptotically unbiased estimator for $\lambda$ because
\[
\expe(T_n)
	= n\expe\left(\frac{1}{S_n}\right)
	= \left(\frac{n}{n-1}\right)\lambda.
	= \left(\frac{1}{1-1/n}\right)\lambda
	\to \lambda \text{ as $n\to\infty$.}
\]
\it % <<< efficiency
The Fisher information of the sample is
\[
I_n(\lambda) = -\expe\big[\ell''(\lambda)\big] = \frac{n}{\lambda^2}
\]
so the CRLB is $\lambda^2/n$. Now,
\[
\var(T_n)
	= n^2\var\left(\frac{1}{S_n}\right)
	= \frac{\lambda^2}{n}\left(\frac{n^3}{(n-1)^2(n-2)}\right)
	= \frac{\lambda^2}{n}\left(\frac{1}{(1-1/n)^2(1-2/n)}\right)
	\to \frac{\lambda^2}{n} \text{ as $n\to\infty$.}
\]
Thus $\var(T_n)$ converges to the CRLB as $n\to\infty$.
\een
\end{answer}

%----------------------------------------
\end{questions}
\end{exercise}
%----------------------------------------------------------------------

%======================================================================
\endinput
%======================================================================