% !TEX root = index.tex
\section{How Curved is a Potato?}
\epigraph{The heart of mathematics consists of concrete examples and concrete problems. Big general theories are usually afterthoughts based on small but profound insights; the insights themselves come from concrete special cases.}{Paul Halmos}

Surfaces are much more varied than curves. The standard examples of surfaces are a plane, a cylinder, a saddle, a sphere, a torus etc. It is hard to come up with a single number that completely describes how curved these objects are. So the better question is what's the right amount of information we need to quantify curvature of these objects?

\subsection{Defining Curvature for Graphs}
For us a surface $S$ is just a connected smooth 2-dimensional subset of $\R^3$. We want our surfaces to be nice, with no sharp edges and no self-intersections.\footnote{A surface is a connected smooth 2-dimensional submanifold of $ \R^n$ without a boundary.}

It is unclear that we can use an approximating sphere or something similar to define curvature. But we know that for curves the curvature measures the \emph{quadratic behavior} of the curve. As with curves, we'll start by analyzing graphs of functions $f(x,y):\R^2 \rightarrow \R$
\begin{align*}
  S = \{ (x,y,z) : z = f(x,y)\}
\end{align*}
We'll further assume that $p=(0,0)$ is a critical point of $f(x,y)$ i.e.
\begin{align*}
   f_x(p) = 0 = f_y(p)
\end{align*}
Our experience with curves suggests that we should define the curvature of $ S$ at $ p$ to be the triple
$$\kappa \stackrel{?}{=} (f_{xx}(p), f_{xy}(p), f_{yy}(p))$$
but there is a catch.

\subsection{The Catch}
We can rotate $ \R^3$ about the $ z-\mbox{axis}$ and the new surface $ S'$ will still satisfy the properties mentioned above. The new surface $ S'$ will be described by a completely different function $ g(x,y)$ and the second derivatives of $ g(x,y)$ will be different from those of $ f(x,y)$. However, the curvatures of $ S$ and $S'$ should be the same. This problem did not arise for curves because there is no way to {rotate $ \R^2$ about the $ y$-axis}.

\begin{figure}[H]
  \centering
  \begin{subfigure}[t]{0.495\textwidth}
    \centering
    \includegraphics[width=6cm]{parabola1}
    \caption{$ f(x,y) = x^2 $ has second derivatives $ (2,0,0)$}
  \end{subfigure}
  \begin{subfigure}[t]{0.495\textwidth}
    \centering
    \includegraphics[width=6cm]{parabola2}
    \caption{$ f(x,y) = y^2 $ has second derivatives $ (0,0,2)$}
  \end{subfigure}
  \caption{The two surfaces are related to each other by a rotation about the $ z$-axis but they have very different second derivatives.}
\end{figure}

\begin{ques}
  \label{ques:paraboloid}
  Rotate the parabolic cylinder $z=x^2$ about the  $z$-axis by an angle $\theta$ in the counter-clockwise direction. Describe the new surface as the graph of some function $f_{\theta}(x,y)$. Find the second derivatives of $f_{\theta}(x,y)$.
\end{ques}

\subsection{Rotations and Second Derivatives}
We need to understand how the second derivatives change when we rotate the $ xy$-plane about the $ z$-axis. There are two tricks to do this efficiently.\footnote{In order to avoid clutter we'll stop writing the $ (p)$. All the derivatives are being taken at the critical point $p = (0,0)$.}

\begin{description}
  \item{\bf Trick \#1} Instead of analyzing the second derivatives we analyze the degree 2 Taylor polynomial
  \begin{align*}
    2f(x,y) \approx 2f(p) + f_{xx}\cdot {x^2} + 2f_{xy} \cdot {xy} + f_{yy} \cdot {y^2}
  \end{align*}

  \item{\bf Trick \#2} We rewrite the degree 2 Taylor polynomial in matrix form
  \begin{align*}
    2f(x,y)
    &\approx 2f(p) +
    \begin{bmatrix} x & y \end{bmatrix}
    \begin{bmatrix}
      f_{xx} & f_{xy} \\
      f_{xy} & f_{yy}
    \end{bmatrix}
    \begin{bmatrix} x \\ y \end{bmatrix}
  \end{align*}
\end{description}
The rotation or the reflection of the $ xy$-plane about the $z$-axis is a linear transformation and hence can be described by some orthogonal matrix $ A$
\begin{align*}
  \begin{bmatrix} x \\ y \end{bmatrix}
    &= A
  \begin{bmatrix} x' \\ y' \end{bmatrix}
\end{align*}
Plugging this in the right hand side of the Taylor polynomial we get
\begin{align}
  \nonumber
  2f(x,y)
  &\approx 2f(p) + f_{xx}{x^2} + 2f_{xy} {xy} + f_{yy}{y^2} \\
  \nonumber
  &= 2f(p) +
  \begin{bmatrix} x & y \end{bmatrix}
  \begin{bmatrix}
    f_{xx} & f_{xy} \\
    f_{xy} & f_{yy}
  \end{bmatrix}
  \begin{bmatrix} x \\ y \end{bmatrix} \\
    \label{eq:Hessian}
    &= 2f(p) +
    \begin{bmatrix} x' & y' \end{bmatrix} A^T
    \begin{bmatrix}
      f_{xx} & f_{xy} \\
      f_{xy} & f_{yy}
    \end{bmatrix}
    A \begin{bmatrix} x' \\ y' \end{bmatrix}
\end{align}
Notice that the right hand side is still quadratic, and not surprisingly this is \textbf{the Taylor approximation of $ f$ in terms of $ (x',y')$}.
\begin{ques}
  Let $A$ denote reflection of the $xy$-plane about the line $x=y$.
  \begin{enumerate}
    \item Express $A$ as a $2 \times 2$ matrix.
    \item Expand the right hand side of \eqref{eq:Hessian} for this $A$ and verify that this is indeed the degree 2 Taylor approximation in the new variables.
  \end{enumerate}
\end{ques}
\begin{ques}
  Suppose that $A = \begin{bmatrix} m & 0 \\ 0 & n \end{bmatrix}$ where $m,n$ are non-zero real numbers. This linear transformation corresponds to scaling the $x,y$ axes by $m,n$ respectively. Expand the right hand side of \eqref{eq:Hessian} for this $A$ and verify that this is indeed the degree 2 Taylor approximation in the new variables.
\end{ques}
\begin{definition}
  The matrix $\begin{bmatrix} f_{xx} & f_{xy} \\ f_{xy} & f_{yy} \end{bmatrix}$ is called the \textbf{Hessian} of $ f$, denoted $ \hess(f)(x,y)$.
\end{definition}
\begin{thm}
  \label{thm:change_of_coords_Hessian}
  At a critical point $p$, if the coordinates change according to the linear transformation $$\begin{bmatrix} x \\ y \end{bmatrix} = A \begin{bmatrix} x' \\ y' \end{bmatrix}$$ then the Hessian $ \hess(f)$ changes as
  \begin{align}
    \label{eq:2-tensor}
    \begin{bmatrix}
      f_{x'x'} & f_{x'y'} \\
      f_{x'y'} & f_{y'y'}
    \end{bmatrix}
      &=
       A^T
      \begin{bmatrix}
        f_{xx} & f_{xy} \\
        f_{xy} & f_{yy}
      \end{bmatrix}
      A
  \end{align}
  We say that $\hess(f)$ is a 2-tensor at a critical point.
\end{thm}
\begin{remark}
  The assumption that we're at a critical point i.e. $ f_x = 0 = f_y$ is extremely crucial here and cannot be dropped. If, for example, we further had $ f_x = 0 = f_y$ and $ f_{xx} = 0 = f_{xy} = f_{yy}$ then we will get a similar change of coordinate rule for the third derivatives, which will be a 3-tensor.
\end{remark}

\subsection{The Curvatures}
The Curvature(s) should be \emph{invariants} of the matrix $ \hess(f)$ which remain unchanged when we apply the transformation \eqref{thm:change_of_coords_Hessian} where $ A$ is either a rotation or a reflection. When $A$ is one of these we further have $A^T = A^{-1}$ so that
\begin{align*}
  \begin{bmatrix}
    f_{x'x'} & f_{x'y'} \\
    f_{x'y'} & f_{y'y'}
  \end{bmatrix}
    &=
     A^{-1}
    \begin{bmatrix}
      f_{xx} & f_{xy} \\
      f_{xy} & f_{yy}
    \end{bmatrix}
    A
\end{align*}
We say that $\hess(f)(x,y)$ and $\hess(f)(x',y')$ are {\bf similar} or {\bf conjugate} to each other i.e. they represent the same linear transformation. If two matrices represent the same linear transformation then they have the same \emph{determinant} and \emph{trace}.
\begin{ques}
  Let $A$ be an $n\times n$ matrix and let $P$ be an invertible $n \times n$ matrix. Prove that
  \begin{align*}
    \det A = \det (P A P^{-1}) \mbox{ \: and \: }\tr A = \tr(P A P^{-1})
  \end{align*}
\end{ques}
We now have well defined notions of curvature which remain unchanged when we rotate or reflect the $xy$-plane.
\begin{definition}
  If $ S$ is the graph of a smooth function $ f(x,y)$ satisfying $ f_x(p) = 0 = f_y(p)$ then
  \begin{enumerate}
    \item The \textbf{Mean Curvature} of $ S$ at $ p$ is defined to be
    \begin{align*}
      H = \tr(\hess(f))/2 = (f_{xx} + f_{yy})/2
    \end{align*}
    \item The \textbf{Gaussian Curvature} of $ S$ at $ p$ is defined to be \textbf{determinant}
    \begin{align*}
      K = \det(\hess(f)) = f_{xx}f_{yy} - f_{xy}^2
    \end{align*}
  \end{enumerate}
\end{definition}

\begin{ques}
  Go back to your computation in Question \ref{ques:paraboloid} and verify that the Gaussian and Mean curvatures for $z=f_\theta(x,y)$ do not depend on $\theta$.
\end{ques}

\begin{ques}
  For each of the following surfaces, find the second order Taylor polynomial, the Hessian, and the Mean and Gaussian curvatures at $(0,0)$.
  \begin{description}
    \item[The Perfect \textbf{Potato Chip}: ]  $ z = x^2 - y^2 $
    \item[Cylindrical Potato: ] $z = -\sqrt{r^2 - x^2} $
    \item[Spherical Potato: ] $z = -\sqrt{r^2 - x^2 - y^2}$
    \item[Parabolic Cylinder: ] $z =  x^2$
  \end{description}
  (You can use the estimate $-\sqrt{r^2 - \alpha} \approx -r + \frac{\alpha}{2r} $. This is called the \textbf{binomial approximation}.)
\end{ques}

\begin{remark}
  Although we're only analyzing critical points of graphs of functions, we can always rotate $ \R^3$ (which should not change curvatures) so that the surface looks like a graph near the point of interest, and the point becomes a critical point. As such, the above method defines curvature in all generality.
\end{remark}

\subsection{Appendix: Taylor approximation}
Taylor polynomials are a way to approximate functions by polynomials.
\begin{definition}
	The degree $ n$ \textbf{Taylor approximation} of a function $ f: \R \rightarrow \R$ at a point $ x=a$ is defined to be
	\begin{align*}
		f(a) + \dfrac{f'(a)}{1!} (x-a) + \dfrac{f''(a)}{2!} (x-a)^2 + \cdots + \dfrac{f^{(n)}(a)}{n!} (x-a)^n
	\end{align*}
	where $f^{(n)}(a)$ denotes the $ n^{th}$ derivative of $ f$ at $ a$.
\end{definition}
We're only interested in the degree 2 approximation at $ x=0$ i.e. $$ f(x)
	\approx f(0) + f'(0) x + f''(0) \frac{x^2}{2}$$
This generalizes to functions in multiple variables easily.
\begin{definition}
	The degree 2 or \textbf{second order Taylor approximation} of a function $ f: \R^2 \rightarrow \R$ at the point $ p=(0,0)$ is defined to be
	\begin{align*}
		f(x,y) \approx f(p) + f_x(p)\cdot x + f_y(p)\cdot y + f_{xx}(p)\cdot \frac{x^2}{2} + f_{xy}(p) \cdot xy + f_{yy}(p)\cdot \frac{y^2}{2}
	\end{align*}
	where $f_{*}(p)$ denotes the partial derivatives.\footnote{$f_{xy}$ has coefficient 1 instead of $1/2$ as it is secretly a sum of two terms $f_{xy}$ and $f_{yx}$ which happen to be equal for all twice differential functions.}
\end{definition}
\begin{ques}
	If $ f(x,y)$ is polynomial in 2 variables $x,y$ find it's the second order Taylor approximation at $(0,0)$.
\end{ques}
\begin{ques}
  Let $T_2(f)$ denote the degree 2 Taylor approximation of $f$ at $(0,0)$. Let $P$ denote the vector space of smooth functions $\R^2 \rightarrow \R$ and let $P_2$ denote the vector space of polynomials of degree $\le 2$ with real coefficients.
  \begin{enumerate}
    \item Show that $T_2$ defines a linear transformation $P \rightarrow P_2$.
    \item Further show that $T_2(T_2(f)) = T_2(f)$.
  \end{enumerate}
\end{ques}
Linear transformations $L$ which satisfy $L \circ L = L$ are called \textbf{projections}, and thus taking the degree 2 Taylor approximation is like projecting onto the space of degree 2 polynomials.

\subsection{Appendix: Orthogonal Transformations}
\begin{ques}$ $
  \begin{enumerate}
    \item Prove that every rotation of $\R^2$ about the origin is given by a linear transformation of the form
    \begin{align*}
      \begin{bmatrix}
        \cos \theta & -\sin \theta \\
        \sin \theta & \cos \theta
      \end{bmatrix}
    \end{align*}
    \item Prove that every reflection of $\R^2$ about a line passing through the origin is given by a linear transformation of the form
    \begin{align*}
      \begin{bmatrix}
        -\cos \theta & \sin \theta \\
        \sin \theta & \cos \theta
      \end{bmatrix}
    \end{align*}
    \item Prove that both reflections and rotations satisfy
    \begin{align*}
      A^{-1} = A^T
    \end{align*}
    \item Show that if $A$ is a $2 \times 2$ matrix satisfying $A^{-1} = A^T$ then $A$ is either a rotation or a reflection.
  \end{enumerate}
\end{ques}
In higher dimensions it is difficult to give explicit descriptions of rotations and reflections, instead we define \textbf{orthogonal matrices} to be the ones that satisfy $A^{-1} = A^T$. It is easy to show that orthogonal matrices preserve distances and angles and hence are the correct generalizations of rotations and reflections.


\section{Point Estimation}

\begin{definition}[Point Estimator]
    A \emph{point estimator} is any function $W(X_1, \dots, X_n)$ of a sample; that is, any statistic is a point estimator. An \emph{estimate} is a realised value $w(x_1, \dots, x_n)$.
\end{definition}

Note that there is the implicit restriction that the estimator is not a function of the parameter you are trying to estimate.

\subsection{Methods of Finding Estimators}

\subsubsection{Method of Moments}
The method of moments is performed using a random sample $X_1, \dots, X_n$ from a population with unknown parameters $\theta_1, \dots, \theta_k$ by computing the first $k$ empirical moments $m_k = \frac{1}{n}\sum_{i=1}^nX_i^k$ and matching them with the corresponding moments of the population $\mu_k' = \E{}[X^t]$.
\begin{align*}
    m_1 &= \mu_1'(\theta_1, \dots, \theta_k)\\
        &\vdotswithin{=} \\
    m_k &= \mu_k' (\theta_1, \dots, \theta_k)
\end{align*}
From this you get $k$ simultaneous equations that you can use to solve for the parameters of the population.

\subsubsection{Maximum Likelihood Estimators}
\begin{definition}[Maximum Likelihood Estimator]
    For each sample point $\vec{x}$, let $\hat{\theta}(\vec{x})$ be a parameter value at which the likelihood $L(\theta \vert{} \vec{x})$ attains its maximum as a function of $\theta$, with $\vec{x}$ held fixed. A \emph{maximum likelihood estimator} (MLE) of the parameter $\theta$ based on a sample $\vec{X}$ is $\hat{\theta}(\vec{X})$.
\end{definition}

Suppose we want to find the MLE for some function of the parameter $\tau(\theta)$.

\begin{definition}[Induced Likelihood]
    Given some function of the parameter $\tau(\theta)$, we define the \emph{induced likelihood function} $L^*$ by
    \[
        L^*(\eta \vert{} \vec{x}) = \sup_{\theta : \tau(\theta) = \eta} L(\theta \vert{} \vec{x}).
    \]
    The value $\hat{\eta}$ that maximises $L^*(\eta \vert{} \vec{x})$ will be called the MLE of $\eta = \tau(\theta)$. It can be seen that the maxima of $L^*$ and $L$ coincide.
\end{definition}

\begin{theorem}[Invariance Property of MLEs]
    IF $\hat{\theta}$ is the MLE of $\theta$, then for any function $\tau(\theta)$, the MLE of $\tau(\theta)$ is $\tau(\hat{\theta})$.
\end{theorem}

\begin{remark}
    \mbox{}
    \begin{enumerate}[1)]
        \item The MLE can be an unstable function of the data.
        \item When verifying maxima for multi-dimensional problems try to avoid going down the hessian route, which can be tedious.
    \end{enumerate}
\end{remark}

\subsubsection{Bayes Estimators}
If we denote the prior distribution by $\pi(\theta)$ and the sampling distribution by $f(\vec{x} \vert{} \theta)$, then the posterior distribution of $\theta$ given the sample $\vec{x}$ is
    \[
        \pi(\theta \vert{} \vec{x}) = \frac{f(\vec{x}\vert{}\theta)\pi(\theta)}{m(\vec{x})},
    \]
    where $m(\vec{x})$ is the marginal distribution of the sample $\vec{X}$
    \[
        m(\vec{x}) = \int f(\vec{x}\vert{}\theta) \pi(\theta) \d{}\theta.
    \]

\begin{definition}[Conjugate Priors]
    Let $\mathcal{F}$ denote the class of pdfs/pmfs $f(\vec{x} \vert{} \theta)$ (indexed by $\theta$). A class $\Pi$ of prior distribution is a \emph{conjugate family} for $\mathcal{F}$ if, $\forall f \in \mathcal{F}$, $\forall \, \text{priors} \in \mathcal{F}$ and $\forall \vec{x} \in \X{}$, the posterior distribution is in $\Pi$.
\end{definition}

Note that this relation is not said to be symmetric.

\begin{remark}[Some Examples]
    \mbox{}
    \begin{itemize}
        \item $\n{}$ is self-conjugate as a family.
        \item Beta distribution is conjugate to binomial.
    \end{itemize}
\end{remark}

\subsection{Methods of Evaluating Estimators}

\begin{definition}[Mean Squared Error]
    The \emph{mean squared error} (MSE) of an estimator $W$ of a parameter $\theta$ is defined by $\E{}[(W - \theta)^2]$.
\end{definition}

\begin{lemma}[Bias-Variance Decomposition]
    \[
        \E{}[(W - \theta)^2] = \Var{}[W] + (\E{}[W] - \theta)^2 = \Var{}[W] + \text{Bias}[W]^2
    \]
\end{lemma}

\begin{definition}[Bias]
    The \emph{bias} of a point estimator $W$ of a parameter $\theta$ is given by
    \[
        \text{Bias}[W] = \E{}[W] - \theta.
    \]
    An estimator whose bias is 0 is called an \emph{unbiased estimator} and has $\E{}[W] = \theta$ $\forall \theta$.
\end{definition}

\begin{remark}
    \mbox{}
    \begin{itemize}
        \item Clearly, if an estimator is unbiased, then its MSE is equal to its variance.
        \item The MSE makes sense for location parameters but not so much for scale parameters, since it is symmetric and scale parameters have a natural floor at 0.
        \item The MSE may be a function of the thing you're trying to estimate. So which estimator you choose as being the `best' may depend on the range you expect the parameter to lie within.
    \end{itemize}
\end{remark}

\subsubsection{Best Unbiased Estimators}

\begin{definition}[Best Unbiased Estimator]
    An estimator $W^*$ is \emph{best unbiased estimator} of of $\tau(\theta)$ if it satisfies $\E{}[W^*] = \tau(\theta) \,\, \forall \theta$, and for any other estimator with $\E{}[W] = \tau(\theta) \,\, \forall \theta$ we have $\Var{}[W*] \leq \Var{}[W] \,\, \forall \theta$. We also call $W^*$ the \emph{uniform minimum variance unbiased estimator} (UMVUE) of $\tau(\theta)$.
\end{definition}

\begin{remark}
    Suppose that we are trying to estimate $\theta$ and consider the class of estimators
    \[
        \mathcal{C}_\tau = \{W: \E{}[W] = \tau(\theta)\}.
    \]
    All estimators in this class have the same bias, so we can compare their MSEs by comparing their variances alone. (So the best estimator in this class is just the minimum variance one.) This means that the considerations of this chapter can be applied to classes like $\mathcal{C}_\tau$, even if $\tau(\theta) \neq \theta$.
\end{remark}

The best unbiased estimator, if it exists, could be hard to find. The following lower bound at least gives a stopping criterion to our search.

\begin{theorem}[Cram\'{e}r-Rao Inequality]
    Let $X_1, \dots, X_n$ be a sample with pdf $f(\vec{x}\vert{}\theta)$ and let $W(\vec{X}) = W(X_1, \dots, X_n)$ be any estimator with finite variance satisfying
    \[
        \frac{\d{}}{\d{} \theta} \E{}[W(\vec{X})] = \int_{\X{}} \frac{\pd{}}{\pd{} \theta} \left(W(\vec{x}) f(\vec{x}\vert{}\theta) \right)\d{}\vec{x}.
    \]
    Then
    \[
        \Var{}[W(\vec{X})] \geq \frac{\left(\frac{\d{}}{\d{} \theta} \E{}[W(\vec{X})] \right)^2}{\E{}\left[ \left(\frac{\pd{}}{\pd{} \theta} \log f(\vec{X} \vert{} \theta) \right)^2 \right]}.
    \]
\end{theorem}

The proof of this considers the correlation between the gradient of the log likelihood and the statistic. Note that the sample in the above theorem is not necessarily iid.

\begin{remark}
    This hold for discrete distributions too, replacing integrals with sums.
\end{remark}

\begin{corollary}
    If the random sample $\X_1, \dots, X_n$ is iid then the result becomes
    \[
        \Var{}[W(\vec{X})] \geq \frac{\left(\frac{\d{}}{\d{} \theta} \E{}[W(\vec{X})] \right)^2}{n\E{}\left[ \left(\frac{\pd{}}{\pd{} \theta} \log f(X \vert{} \theta) \right)^2 \right]}.
    \]
\end{corollary}

\begin{definition}[Fisher Information]
    The quantity $\E{}\left[ \left(\frac{\pd{}}{\pd{} \theta} \log f(\vec{X} \vert{} \theta) \right)^2 \right]$ is called the \emph{Fisher Information} of the sample $\vec{X}$.
\end{definition}

\begin{lemma}
    If $f(x \vert{} \theta)$ satisfies
    \[
        \frac{\d{}}{\d{} \theta} \E{}\left[ \frac{\pd}{\pd \theta} \log f(X \vert{} \theta) \right] = \int \frac{\pd}{\pd \theta} \left[ \left( \frac{\pd}{\pd \theta} \log f(x \vert{} \theta) \right) f(x \vert{} \theta) \right] \d{}x
    \]
    (as is true for an exponential family), then the Fisher information can be written
    \[
        \E{}\left[ \left(\frac{\pd{}}{\pd{} \theta} \log f(X \vert{} \theta) \right)^2 \right] = - \E{}\left[ \frac{\pd^2}{\pd \theta^2} \log f(X \vert{} \theta) \right].
    \]
\end{lemma}

\begin{remark}
    Even if the Cram\'{e}r-Rao bound is applicable, it may not be sharp -- there may not be an estimator that attains this bound.
\end{remark}

\begin{lemma}[Attainment]
    Let $X_1, \dots, X_n$ be iid $X \sim f(x \vert{} \theta)$, where $f(x \vert{} \theta)$ satisfies the conditions of the Cram\'{e}r-Rao Theorem. Let $L(\theta \vert{} \vec{x}) = \prod_{i=1}^n f(x_i \vert{} \theta)$ denote the likelihood function. If $W(\vec{X})$ is any unbiased estimator of $\tau(\theta)$ then $W(\vec{X})$ attains the Cram\'{e}r-Rao lower bound if and only if $\exists a(\theta)$ such that
    \[
        a(\theta) [W(\vec{X}) - \tau(\theta)] = \frac{\pd}{\pd \theta} \log L(\theta \vert{} \vec{x}).
    \]
\end{lemma}

\subsubsection{Sufficiency and Unbiasedness}

\begin{theorem}[Rao-Blackwell]
    Let $W$ be any unbiased estimator of $\tau(\theta)$ and let $T$ be a sufficient statistic for $\theta$. Define $\phi(T) = \E{}[W \vert{} T]$. Then $\E{}[\phi(T)] = \tau(\theta)$ and $\Var{}[\phi(T)] \leq \Var{}[W] \,\, \forall \theta$. That is, $\phi(T)$ is a \emph{uniformly better unbiased estimator} of $\tau(\theta)$.
\end{theorem}

\begin{remark}
    \mbox{}
    \begin{itemize}
        \item Conditioning on any unbiased estimator on a sufficient statistic will result in a uniform improvement, so we need consider only functions of a sufficient statistic when looking for best unbiased estimators.
        \item The proof doesn't require that the statistic we condition on is sufficient, but if it isn't then the resulting quantity will probably depend on the parameter we are trying to estimate.
    \end{itemize}
\end{remark}

\begin{theorem}
    If $W$ is a best unbiased estimator of $\tau(\theta)$ then $W$ is unique.
\end{theorem}

The following theorem is mostly useful to show that a given estimator \emph{isn't} best unbiased.

\begin{theorem}
    If $\E{}[W] = \tau(\theta)$, then $W$ is the best unbiased estimator of $\tau(\theta)$ if and only if $W$ is uncorrelated with all unbiased estimators of 0.
\end{theorem}

The idea comes from considering $\phi_a = W + aU$ where $\E{}[U] = 0$, then considering the variance.

\begin{remark}[Unbiased estimator of 0]
    Note that an unbiased estimator of 0 is simply noise (one should estimate 0 with 0). If an estimator can be improved by adding noise, then it is probably defective.
\end{remark}

We are now in a position such that, if we can characterise all of the unbiased estimators of 0 then we can check if a given estimator is best unbiased. In general this is not easy and requires conditions on the distribution. However, if a distribution is complete then it admits no unbiased estimators of 0 other than 0 itself (by definition), so we will be done.\\

Note that due to the Rao-Blackwell theorem, only the distribution of the sufficient statistic needs to be complete (not the underlying population distribution).

\begin{theorem}
    Let $T$ be a complete sufficient statistic for a parameter $\theta$ and let $\phi(T)$ be any estimator based only on $T$. Then $\phi(T)$ is the unique best unbiased estimator of its expected value.
\end{theorem}

\begin{theorem}[Lehmann-Scheff\'{e}]
    Unbiased estimators based on complete sufficient statistics are unique.
\end{theorem}


\section{Orthogonal Matching Pursuit}
\label{JM:sec:GOMP}

In the following section, preconditioned generalized orthogonal matching pursuit (GOMP) \cite{JMTongEtAl2020} will be implemented.
Matching pursuit algorithms are a class of greedy algorithms designed to solve the sparse signal recovery problem

\begin{equation}
\begin{aligned}
    \min_x \quad & \left\lVert x \right\rVert_0 \\
    \textrm{s.t.} \quad & y = \Psi x
\end{aligned}
\end{equation}

where $x \in \mathbb{R}^n$ represents an unknown, $K$ sparse basis for a signal $y \in \mathbb{R}^m$ in $\Psi \in \mathbb{R}^{m \times n}$, also called the sampling matrix.
This is done by iteratively reducing the estimation error of the recovered signal by selecting $S$ most similar, new components from the sampling matrix.\\

The main algorithm can be described as follows

\begin{algorithm}[H]
    \SetAlgoLined
    \KwResult{$x$}
    \KwData{$y$, $\Psi$, $K$, $S = 1$}
    Projection onto span\;
    $P = \Psi^T \left(\Psi\Psi^T\right)$;
    $\tilde{y} = P y$;
    $\tilde{\Psi} = P \Psi$\;
    Initialize residual and support\;
    $r = y$;
    $\Lambda = \emptyset$\;
    \While{Not converged}{
        $\Omega = \delta_S\left( \left| \tilde{\Psi}^T~r \right| \right)$\;
        $\Lambda = \Lambda \cup \Omega$\;
        $x = \min \left\lVert \tilde{y} - \tilde{\Psi}u \right\rVert_2~, \quad supp(u) = \Lambda$\;
        $r = \tilde{y} - \tilde{\Psi} x$\;
    };
\end{algorithm}

Here $\delta_S\left( x \right) : \mathbb{R}^n \mapsto \lbrace 0, 1 \rbrace^n$ denotes (with abuse of notation) the mapping of the $S$ largest elements onto a corresponding
indicator vector. A similar notation to the pseudocode above can be achieved in \textit{Julia}. For brevity, only necessary elements of the source code will be shown, which can be found in detail in REF. The function is defined as follows

\lstinputlisting[language=Julia,firstline=4, lastline=6]{../scripts/gomp.jl}

First, we will focus on the preconditioning via the matrix $P \in \mathbb{R}^{m \times m}$ mapping onto the
column span of $\Psi$

\lstinputlisting[language=Julia,firstline=22, lastline=26]{../scripts/gomp.jl}

Next, the iterative computation of the support, corresponding coefficients and residuals is performed

\lstinputlisting[language=Julia,firstline=37, lastline=50]{../scripts/gomp.jl}

As can be seen from above, the implementation is quite similar to its mathematical counterpart, enhancing the interpretability and readability.
The results of the algorithm are shown in Fig. \ref{JM:fig:GOMP}, where a 100 dimensional sparse vector $x$ has been recovered.

\begin{figure}
    \includegraphics[width = 0.9\textwidth]{../figures/merged.pdf}
    \caption{Performance of GOMP for a single sparse 100 dimensional vector (left) and with different sparsities (right). The signal $y$ is of dimension 300 and has been generated by a random dense sampling matrix.
    GOMP has been used with $K=50, S=1$ and run for at most 100 iterations with a absolute tolerance $\epsilon = 0.1$.}
    \label{JM:fig:GOMP}
\end{figure}

\newpage


\chapter{Sort}

\section{Introduction}
List of general algorithms:
\begin{enumerate}
\item Selection sort: invariant
\begin{enumerate}
\item Elements to the left of $i$ (including $i$) are fixed and in ascending order (fixed and sorted).
\item No element to the right of $i$ is smaller than any entry to the left of $i$ ($A[i]  \leq\min(A[i+1:n])$.
\end{enumerate}
\item Insertion sort: invariant
\begin{enumerate}
\item Elements to the left of $i$ (including $i$) are in ascending order (sorted).
\item Elements to the right of $i$ have not yet been seen.
\end{enumerate}
\item Shell sort: h-sort using insertion sort.
\item Quick sort: invariant
\begin{enumerate}
\item $|A_p|..\leq..|..unseen..|..\geq..|$ maintain the 3 subarrays.
\end{enumerate}
\item Heap sort: compared to quick sort it is guaranteed $O(n \lg n)$, compared to merge sort it is $O(1)$ extra space.
\end{enumerate}

\section{Algorithms}
\subsection{Quick Sort}
\subsubsection{Normal pivoting}\label{section:pivot}
The key part of quick sort is pivoting:
\newpage
\begin{python}
def pivot(self, A, lo, hi):
    """
    pivoting algorithm:
    | p | closed set | open set |
    | closed set | p | open set |
    """
    p = lo
    closed = p
    for i in xrange(lo, hi):
        if A[i] < A[p]:
            closed += 1
            A[i], A[closed] = A[closed], A[i]

    A[closed], A[p] = A[p], A[closed]
    return closed
\end{python}

Notice that this implementation goes $O(N^2)$ for arrays with all duplicates.

\textbf{Problem with duplicate keys}: it is important to stop scan at duplicate
keys (counter-intuitive); otherwise quick sort will goes $O(N^2)$ for the
array with all duplicate items, because the algorithm will put all items
equal to the $A[p]$ on \textbf{a single side}.

Example: quadratic time to sort random arrays of 0s and 1s.

\subsubsection{Stop-at-equal pivoting}
Alternative pivoting implementation with optimization for duplicated keys:
\begin{python}
def pivot_optimized(self, A, lo, hi):
    """
    Fix the pivot as the 1st element
    Scan from left to right and right to left simultaneously
    Avoid the case that the algo goes O(N^2) with duplicated keys
    """
    p = lo
    i = lo
    j = hi
    while True:
        while True:
            i += 1
            if i >= hi or A[i] >= A[lo]:
                break
        while True:
            j -= 1
            if j < lo or A[j] <= A[lo]:
                break

        if i >= j:
            break

        A[i], A[j] = A[j], A[i]

    A[lo], A[j] = A[j], A[lo]
    return j

\end{python}
\subsubsection{3-way pivoting}
This problem is also known as \textit{Dutch national flag problem}.

3-way pivoting: pivot the array into 3 subarrays:

$$|..\leq..|..=..|..unseen..|..\geq..|$$
\begin{python}
def pivot_3way(self, A, lo, hi):
    lt = lo-1  # pointing to end of array LT
    gt = hi  # pointing to the end of array GT (reversed)

    v = A[lo]
    i = lo  # scanning pointer
    while i < gt:  # not n or hi
        if A[i] < v:
            lt += 1
            A[lt], A[i] = A[i], A[lt]
            i += 1
        elif A[i] == v:
            i += 1
        else:
            gt -= 1
            A[gt], A[i] = A[i], A[gt]

    return lt, gt
\end{python}
\subsection{Merge Sort}
\begin{figure}[!htp]
\centering
\subfloat{\includegraphics[scale=.80]{msort}}
\caption{Merge Sort}
\label{fig:msort}
\end{figure}
\runinhead{Normal merge} Normal merge sort with extra space
\begin{python}
def merge_sort(self, A):
  if len(A) <= 1:
    return

  mid = len(A)/2
  L, R = A[:mid], A[mid:]
  self.merge_sort(L)
  self.merge_sort(R)

  i, j, k = 0, 0, 0
  while i < len(L) and j < len(R):
    if L[i] < R[j]:
      A[k] = L[i]
      i += 1
    else:
      A[k] = R[j]
      j += 1
    k += 1

  if i < len(L):
    A[k:] = L[i:]
  if j < len(R):
    A[k:] = R[j:]
\end{python}

\runinhead{Merge backward.} Merge two arrays in place of one of the arrays.
\begin{python}
def merge(self, A, m, B, n):
  """
  Arrays in asc order.
  Assume A has enough space.
  CONSTANT SPACE: starting backward.
  """
  i = m-1
  j = n-1
  closed = m+n

  while i >= 0 and j >= 0:
    closed -= 1
    if A[i] > B[j]:
      A[closed] = A[i]
      i -= 1
    else:
      A[closed] = B[j]
      j -= 1

  # either-or
  # dangling
  if j >= 0: A[:closed] = B[:j+1]
  # if i >= 0: A[:closed] = A[:i+1]
\end{python}
\runinhead{In-place merge}
In-place merge sort of array without recursive. The basic idea is to avoid the recursive call while using iterative solution.

The algorithm first merge chunk of length of 2, 4, 8 ... until $2^k$ where $2^k$ is large than the length of the array.
\begin{python}
def merge_sort(self, A):
  n = len(A)
  l = 1
  while l <= n:
    for i in range(0, n, l*2):
      lo, hi = i, min(n, i+2*l)
      mid = i + l
      p, q = lo, mid
      while p < mid and q < hi:
        if A[p] < A[q]:
          p += 1
        else:
          tmp = A[q]
          A[p+1:q+1] = A[p:q]
          A[p] = tmp
          p, mid, q = p+1, mid+1, q+1

    l *= 2

  return A
\end{python}
The time complexity may be degenerated to $O(n^2)$.
\subsection{Do something while merging}
During the merging, the left half and the right half are both sorted; therefore, we can carry out operations like:
\begin{enumerate}
\item inversion count
\item range sum count
\end{enumerate}

\runinhead{Count of Range Sum.} Make an array $A$ of sums, where \pyinline{A[i] = sum(nums[:i])}; and then feed to merge sort. Since both the left half and the right half are sorted, we can diff $A$ in $O(n)$ time to find range sum.

\begin{python}
def msort(A, lo, hi):
  if hi - lo <= 1: return 0

  mid = (lo + hi)/2
  cnt = msort(A, lo, mid) + msort(A, mid, hi)

  temp = []
  i = j = r = mid
  for l in xrange(lo, mid):
    # range count
    while i < hi and A[i] - A[l] <  LOWER: i += 1
    while j < hi and A[j] - A[l] <= UPPER: j += 1
    cnt += j - i

    # normal merge
    while r < hi and A[r] < A[l]:
      temp.append(A[r])
      r += 1

    temp.append(A[l])

  while r < hi:  # dangling right
    temp.append(A[r])
    r += 1

  A[lo:hi] = temp
  return cnt
\end{python}

Here, the implementation of merge sort use: 1 for-loop for the left half and 2 while-loop for the right half.

\section{Properties}
\subsection{Stability}
Definition: a stable sort preserves the \textbf{relative order of items with equal keys} (scenario: sorted by time then sorted by location).

Algorithms:
\begin{enumerate}
\item Stable
\begin{enumerate}
\item Merge sort
\item Insertion sort
\end{enumerate}
\item Unstable
\begin{enumerate}
\item Selection sort
\item Shell sort
\item Quick sort
\item Heap sort
\end{enumerate}
\end{enumerate}
\textbf{Long-distance swap} operation is the key to find the unstable case during sorting.
\begin{figure}[hbtp]
\centering
\subfloat{\includegraphics[scale=.50]{stable_sort}}
\caption{Stale sort vs. unstable sort}
\label{fig:trie}
\end{figure}

\subsection{Sorting Applications}
\begin{enumerate}
\item Sort
\item Partial quick sort (selection), k-th largest elements
\item Binary search
\item Find duplicates
\item Graham scan
\item Data compression
\end{enumerate}

\subsection{Considerations}
\begin{enumerate}
\item Stable?
\item Distinct keys?
\item Need guaranteed performance?
\item Linked list or arrays?
\item Caching system? (reference to neighboring cells in the array?
\item Usually randomly ordered array?
(or partially sorted?)\item Parallel?
\item Deterministic?
\item Multiple key types?
\end{enumerate}

$O(N\lg N)$ is the lower bound of comparison-based sorting; but for other
contexts, we may not need $O(N \lg N)$:
\begin{enumerate}
\item Partially-ordered arrays: insertion sort to achieve $O(N)$. \textbf{Number of inversions}: 1 inversion $=$ 1 pair of keys that are out
of order.
\item Duplicate keys
\item Digital properties of keys: radix sort to achieve $O(N)$.
\end{enumerate}

\subsection{Sorting Summary}
See Figure \ref{fig:sortSummary}.
\begin{figure*}[hbtp]
\centering
\subfloat{\includegraphics[scale=1.2]{sort_summary}}
\caption{Sort summary}
\label{fig:sortSummary}
\end{figure*}
\section{Partial Quicksort}
\subsection{Find $k$ smallest}
\runinhead{Heap-based solution.} $O(n \log k)$

Version 1, construct heap with $n$ numbers, and take $k$: $O(n+k\log n)$, where $O(n)$ is for constructing heap.

Version 2. construct heap with $k$ numbers, and iterate $n$: $O(k+n\log k)$.

The 2nd version is much master than 1st based on emperical analysis; additionally, it has smaller memory impact.

In python there are:
\begin{python}
heapq.nlargest(n, iterable[, key])
heapq.nsmallest(n, iterable[, key])
\end{python}

\newpage
\runinhead{Partial Quicksort}  Then the $A[:k]$ is sorted $k$ smallest. The algorithm recursively sort the $A[lo:hi]$

The average time complexity is
\begin{eqnarray*}
F(n) = \left\{ \begin{array}{rl}
  F(\frac{n}{2})+O(n) &\mbox{// if $\frac{n}{2} \geq k$} \\
  2F(\frac{n}{2})+O(n) &\mbox{// otherwise}
       \end{array} \right.
\end{eqnarray*}
Therefore, the complexity is $O(n+k \log k)$.
\begin{python}
def partial_qsort(self, A, lo, hi, k):
    if lo >= hi: return

    p = self.pivot(A, lo, hi)
    self.partial_qsort(A, lo, p, k)
    if k <= p+1: return
    self.partial_qsort(A, p+1, hi, k)
\end{python}
The partial quick sort will find the $k$ smallest number in sorted order. If the top $k$ elements are not required to be sorted, then use find $k$-th algorithm

\subsection{Find $k$-th: Quick Select}
Use partial quick sort to find $k$-th smallest element in the unsorted array. The algorithm recursively sort the $A[lo:hi]$

The average time complexity is
\begin{align*}
F(n) &= F(n/2) + O(n) \\
&= O(n)
\end{align*}

Find $k$-th with 2-way partitioning
\begin{python}
def find_kth(self, A, lo, hi, k):
    if lo >= hi: return

    p = self.pivot(A, lo, hi)
    if k == p: return p
    if k < p:  return self.find_kth(A, lo, p, k)
    else:      return self.find_kth(A, p+1, hi, k)
\end{python}
Find $k$-th with 3-way partitioning. Pay attention to the indexing. $lt$, $gt$ means the last index of less-than portion and larger-than partion.
\begin{python}
def find_kth(self, A, lo, hi, k):
    if lo >= hi: return

    lt, gt = self.pivot(A, lo, hi)
    if lt < k < gt: return k
    if k <= lt: return self.find_kth(A, lo, lt+1, k)
    else:       return self.find_kth(A, gt, hi, k)
\end{python}

Pivoting see section - \ref{section:pivot}.

\runinhead{Find $k$-th in union of two sorted array.} Given sorted two arrays $A, B$, find the $k$-th element (0 based index).

Core clues:
\begin{enumerate}
\item To reduce the complexity of $O(\log (m+n))$, need to half the arrays.
\item Decide which half of the array to disregard.
\item Decide whether to disregard the median (i.e. boundary point).
\end{enumerate}
\begin{python}
def find_kth(self, A, B, k):
  if not A:  return B[k]
  if not B:  return A[k]
  if k == 0: return min(A[0], B[0])

  m, n = len(A), len(B)
  if A[m/2] >= B[n/2]:
    if k > m/2 + n/2:
      return self.find_kth(A, B[n/2+1:], k-n/2-1)  # exclude median
    else:
      return self.find_kth(A[:m/2], B, k)  # exclude median
  else:
    return self.find_kth(B, A, k)  # swap
\end{python}
\subsection{Applications}
\runinhead{Wiggle Sort.} Given an unsorted array $A$, reorder it such that $A_0 < A_1 > A_2 < A_3$. Do it in $O(n)$ time and $O(1)$ space.

Core clues:
\begin{enumerate}
\item Quick selection for finding median (Average $O(n)$)
\item Three-way partitioning to split the data
\item Re-mapping the index to do in-place partitioning
\end{enumerate}

\runinhead{Pre-processing} Sorting can be an important pre-processing step as to:
\begin{enumerate}
\item Satisfying the output order (e.g. if multiple results are possible, output the one that's smallest in terms of the natural order).
\end{enumerate}

\begin{python}
class Solution(object):
  def wiggleSort(self, A):
    n = len(A)
    median_idx = self.find_kth(A, 0, n, n/2)
    v = A[median_idx]

    idx = lambda i: (2*i+1)%(n|1)
    lt = -1
    hi = n
    i = 0
    while i < hi:
      if A[idx(i)] > v:
        lt += 1
        A[idx(lt)], A[idx(i)] = A[idx(i)], A[idx(lt)]
        i += 1
      elif A[idx(i)] == v:
        i += 1
      else:
        hi -= 1
        A[idx(hi)], A[idx(i)] = A[idx(i)], A[idx(hi)]

  def pivot(self, A, lo, hi, pidx=None):
    lt = lo-1
    gt = hi
    if not pidx: pidx = lo

    v = A[pidx]
    i = lo
    while i < gt:
      if A[i] < v:
        lt += 1
        A[lt], A[i] = A[i], A[lt]
        i += 1
      elif A[i] == v:
        i += 1
      else:
        gt -= 1
        A[gt], A[i] = A[i], A[gt]

    return lt, gt

  def find_kth(self, A, lo, hi, k):
    if lo >= hi: return

    lt, gt = self.pivot(A, lo, hi)

    if lt < k < gt:
      return k
    if k <= lt:
      return self.find_kth(A, lo, lt+1, k)
    else:
      return self.find_kth(A, gt, hi, k)
\end{python}

\section{Inversion}
If $a_i > a_j$ but $i<j$, then this is considered as 1 Inversion. That is, for an element, the count of other elements that are \textit{larger} than the element but appear \textit{before} it. This is the default definition.

There is also an alternative definition: for an element, the count of other elements that are \textit{samller} than the element but appear \textit{after} it.

\subsection{MergeSort \& Inversion Pair}
MergeSort to calculate the reverse-ordered paris. The only difference from a normal
merge sort is that - when pushing the 2nd half of the array to the place, you calculate
the inversion generated by the element $A_2[i_2]$ compared to $A_1[i_1:]$.

Therefore the Merge-and-count key is \pyinline{ret += len(A1) - i1}

\begin{figure}[hbtp]
\centering
\subfloat{\includegraphics[scale=1.20]{mergeAndSort.png}}
\caption{Merge and Count}
\label{fig:mergeAndSort}
\end{figure}

\begin{python}
def merge(A1, A2, A):
  i1 = i2 =0
  ret = 0
  for i in xrange(len(A)):
    if i1 == len(A1):
      A[i] = A2[i2]
      i2 += 1
    elif i2 == len(A2):
      A[i] = A1[i1]
      i1 += 1
    else:
      # use array diagram to illustrate
      if A1[i1] > A2[i2]:  # push the A2 to A
        A[i] = A2[i2]
        i2 += 1
        # number of reverse-ordered pairs
        ret += len(A1) - i1
      else:
        A[i] = A1[i1]
        i1 += 1

  return ret

def merge_sort(a):
  n = len(a)
  if n == 1:
    return 0

  a1 = a[:n/2]
  a2 = a[n/2:]

  ret1 = merge_sort(a1)
  ret2 = merge_sort(a2)
  # merge not merge_sort
  ret = ret1+ret2+merge(a1, a2, a)
  return ret
\end{python}

\subsection{Binary Index Tree \& Inversion Count}
Given $A$, calculate each element's inversion number.

Construct a BIT (\ref{BIT}) with length $max(A)+1$. Let BIT maintains the index of values. Scan the element from left to right (or right to left depends on the definition of inversion number), and set the index equal val to 1. Use the prefix sum to get the inversion number.

\pyinline{get(end) - get(a)} get the count of number that appears \textit{before} $a$ (i.e. already in the BIT) and also \textit{larger} than $a$.

Possible to extend to handle duplicate number.
\\
Core clues:
\begin{enumerate}
\item BIT maintains \textbf{index of values} to count the number of at each value.
\item \pyinline{get(end) - get(a)} to get the inversion count of $a$.
\end{enumerate}
\begin{python}
def inversion(self, A):
    bit = BIT(max(A)+1)
    ret = []
    for a in A:
        bit.set(a, 1)  # += 1 if possible duplicate
        inversion = bit.get(max(A)+1) - bit.get(a)
        ret.append(inversion)

    return ret
\end{python}

\subsection{Segment Tree \& Inversion Count}\label{segmentTreeInversionCount}
Compared to BIT, Segment Tree can process queries of both $idx \rightarrow sum$ and $sum \rightarrow idx$; while BIT can only process $idx \rightarrow sum$.

Core clues:
\begin{enumerate}
\item Segment Tree maintains \textbf{index of values} to count the number of at each value.
\item \pyinline{get(root, end) - get(root, a)} to get the inversion count of $a$.
\end{enumerate}
\begin{python}
class SegmentTree(object):
  def __init__(self):
    self.root = None

  def build(self, root, lo, hi):
    if lo >= hi: return
    if not root: root = Node(lo, hi)

    root.left = self.build(root.left, lo, (lo+hi)/2)
    if root.left:
      root.right = self.build(root.right, (lo+hi)/2, hi)

    return root

  def set(self, root, i, val):
    if root.lo == i and root.hi-1 == root.lo:
      root.cnt_this += val
    elif i < (root.lo+root.hi)/2:
      root.cnt_left += val
      self.set(root.left, i, val)
    else:
      self.set(root.right, i, val)

  def get(self, root, i):
    if root.lo == i and root.hi-1 == root.lo:
      return root.cnt_left
    elif i < (root.lo+root.hi)/2:
      return self.get(root.left, i)
    else:
      return (
          root.cnt_left + root.cnt_this +
          self.get(root.right, i)
      )

class Solution(object):
  def _build_tree(self, A):
    st = SegmentTree()
    mini, maxa = min(A), max(A)
    st.root = st.build(st.root, mini, maxa+2)
    # maxa+1 is the end dummy
    return st

  def countOfLargerElementsBeforeElement(self, A):
    st = self._build_tree(A)
    ret = []
    end = max(A)+1
    for a in A:
      ret.append(
          st.get(st.root, end) - st.get(st.root, a)
      )
      st.set(st.root, a, 1)

    return ret
\end{python}

\subsection{Reconstruct Array from Inversion Count}\label{inversionReconstruct}
Given a \textit{sorted} numbers with their associated inversion count (\# larger numbers before this element). $A[i].val$ is the value of the number, $A[i].inv$ is the inversion number. Reconstruct the original array $R$ that consists of each $A[i].val$.

Brute force can be done in $O(n^2)$. Put the $A[i].val$ into $R$ at slot s.t. the \# \textit{empty} slots before it equals to $A[i].inv$.

\rih{BST}. Possible to use BST to maintain the empty slot indexes in the original array. Each node's rank indicates the count of empty indexes in its left subtree. But need to maintain the deletion.

\rih{Segment Tree}. Use a segment tree to maintain the size of empty slots. Each node has a $start$ and a $end$ s.t slot indexes $\in [start, end)$. Go down to find the target slot, go up to decrement the size of empty slots.

Caveat: need to sort the array in the preprocessing step.

Reconstruction of array cannot use BIT since there is no map of $prefixSum \rightarrow i$.
\newpage
\begin{python}
class Node(object):
  def __init__(self, lo, hi, cnt):
    self.lo = lo
    self.hi = hi
    self.cnt = cnt  # size of empty slots

    self.left = None
    self.right = None

  def __repr__(self):
    return repr("[%d,%d)" % (self.lo, self.hi))

class SegmentTree(object):
  """empty space"""
  def __init__(self):
    self.root = None

  def build(self, lo, hi):
    """a node can have right ONLY IF has left"""
    if lo >= hi: return
    if lo == hi-1: return Node(lo, hi, 1)

    root = Node(lo, hi, hi-lo)
    root.left = self.build(lo, (hi+lo)/2)
    root.right = self.build((lo+hi)/2, hi)
    return root

  def find_delete(self, root, sz):
    """
    :return: index
    """
    root.cnt -= 1
    if not root.left:
      return root.lo
    elif root.left.cnt >= sz:
      return self.find_delete(root.left, sz)
    else:
      return self.find_delete(root.right,
                  sz - root.left.cnt)

class Solution(object):
  def reconstruct(self, A):
    st = SegmentTree()
    n = len(A)
    st.root = st.build(0, n)
    A = sorted(A, key=lambda x: x[0])
    ret = [0]*n
    for a in A:
      idx = st.find_delete(st.root, a[1]+1)
      ret[idx] = a[0]

    return ret

if __name__ == "__main__":
  # (val, inv)
  A = [(5, 0), (2, 1), (3, 1), (4, 1,), (1, 4)]
  assert Solution().reconstruct(A) == [5, 2, 3, 4, 1]
\end{python}

\runinhead{Duplicate.} What if the array contains duplicate elements? Use a \pyinline{Counter} to count the duplicate items already in the result.


\documentclass{article}
\usepackage{amsmath}

\title{ My first document}
\date{2020/11/09}
\author{Steven Huang}

\begin{document}
\pagenumbering{gobble}
\maketitle
\newpage
\pagenumbering{arabic}

\section{section1}
Hello World!

\section{section2}

\begin{equation}\label{1}
\begin{aligned}
f(x)=x^2 + sigma(x) + log(10) +\\ x^3 + 3x^2 + 5x + 1.8\\
\end{aligned}
\end{equation}

\begin{equation}
f(x)=x^2 + sigma(x) + log(10) + x^3 + 3x^2 + 5x + 1.8\\
\end{equation}

\begin{align}
f(x)=x^2 + 5x + 1.8\\
g(x)=\lambda*2+3
\end{align}

\begin{equation}
l(x)=(x^2 + 2*x)/(3+5*x)
\end{equation}

\begin{align}
  1 + x^2 & + 4*x= 3\\
  1 &= 3 - 2\\
  4*x+8*x^3 &+5&=10
\end{align}

\section{section3-functions}
\begin{align}
f(x)  &= x^2 \\
g(x) &= \frac{1}{x}\\
F(x) &= \int^0_x \frac{1}{3}x^3\\
L(x) = \int^a_b \frac{1+x}{1+x^2}\sqrt{x}e^2\\
f(x) = \frac{1}{d \sqrt{2*p}}\\
f(x) = \left( \frac{1}{d \sqrt{2*p}} \right)\\
f(x) = \left[ \frac{1}{d \sqrt{2*p}} \right]\\
f(x) = \left\{ \frac{1}{d \sqrt{2*p}} \right\}\\
F(x) = \int_{a}^{b}\left( \sum_{n=1}^{+\infty}u_{n}(x) \right)\,\mathrm{d}x
    =\sum_{n=1}^{+\infty}\int_{a}^{b}u_{n}(x)\,\mathrm{d}x \\
Loss(y_true, y_pred) = \frac{1}{x}\\
Loss(y_{true}, y_{pred}) = 1 - \frac{2\sum_{pixel}y_{true}y_{pred} + smooth }{\sum_{pixel}y_{true}^2 + \sum_{pixel}y_{pred}^2 + smooth}\\
\\
Loss(y_{true}, y_{pred})  = \log(1+\exp(-1 * \sum_{Scoremap}y_{true}y_{pred} ))\\
\end{align}

\section{section4-matrixs}
\begin{equation}
A_{m,n} =
\begin{pmatrix}
a_{1,1} & a_{1,2} & \cdots & a_{1,n} \\
a_{2,1} & a_{2,2} & \cdots & a_{2,n} \\
\vdots  & \vdots  & \ddots & \vdots  \\
a_{m,1} & a_{m,2} & \cdots & a_{m,n}
\end{pmatrix}
\end{equation}

\begin{equation}
A =
\begin{pmatrix}
1 & 2 & 3 \\
4 & 5 & 6 \\
7 & 8 & 9
\end{pmatrix}
\end{equation}

\begin{equation*}
B =
\begin{bmatrix}
a & b & c \\
d & e & f \\
g & h & i
\end{bmatrix}
\end{equation*}

\begin{equation}
   \begin{matrix}
   a_{11} & a_{12} & a_{13}  \\
   a_{21} & a_{22} & a_{23}  \\
   a_{31} & a_{32} & a_{33}  \\
   \end{matrix}
\end{equation}

\begin{equation}
  \left[
   \begin{matrix}
   a_{11} & a_{12} & a_{13}  \\
   a_{21} & a_{22} & a_{23}  \\
   a_{31} & a_{32} & a_{33}  \\
   \end{matrix}
  \right]
\end{equation}

\begin{equation}
  \left\{
   \begin{matrix}
   a_{11} & a_{12} & a_{13}  \\
   a_{21} & a_{22} & a_{23}  \\
   a_{31} & a_{32} & a_{33}  \\
   \end{matrix}
  \right\}
\end{equation}

\end{document}


\documentclass[8 pt]{article}

\usepackage[utf8x]{inputenc}
\usepackage{dsfont}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{tensor}
\usepackage{mathtools}
\usepackage[T1]{fontenc}
%\usepackage[spanish]{babel}
\usepackage[cm]{fullpage}
\usepackage{graphicx}
\usepackage{float}
\usepackage{bm}
\usepackage{setspace}
\usepackage{enumitem}
\usepackage{mdwlist}
\usepackage{parskip}
\usepackage{listings}
\usepackage{color}
%\usepackage{epstopdf}
\usepackage{tikz,datatool}
\usepackage{hyperref}
\usepackage{mathabx}
\usepackage{multicol}
\usepackage{eurosym}

\newcommand{\HRule}{\rule{\linewidth}{0.5mm}}

\AtBeginDocument{
  \let\myThePage\thepage
  \renewcommand{\thepage}{\oldstylenums{\myThePage}}
}

\newcommand{\gra}{$^\text{o}$}
\newcommand{\dif}{\text{d}}
\newcommand{\avg}[1]{\left\langle #1 \right\rangle}
\newcommand{\ket}[1]{\left| #1 \right\rangle}
\newcommand{\bra}[1]{\left\langle #1 \right|}
\newcommand{\bket}[2]{\left\langle #1 \middle| #2 \right\rangle}
\newcommand{\der}[2]{\frac{\text{d} #1}{\text{d} #2}}
\newcommand{\prt}[2]{\frac{\partial #1}{\partial #2}}
\newcommand{\dert}[3]{\frac{\text{d}^#3 #1}{\text{d} #2^#3}}
\newcommand{\prtt}[3]{\frac{\partial^#3 #1}{\partial #2^#3}}
\newcommand{\dl}{\mathcal{L}}
\newcommand{\dha}{\mathcal{H}}
\newcommand{\vol}{\text{vol}}
\renewcommand{\vec}[1]{\pmb{#1}}

\DeclarePairedDelimiter\ceil{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}

\begin{document}

\begin{minipage}{\textwidth}
    \centering
    \Large \textbf{\textsc{Homework 1: An option's fair price}}\\[0.5cm]

    \small \textsc{Francisco García Flórez, Joris van Lammeren, Wouter Varenkamp}\\[0.5cm]

    \begin{minipage}{0.6\textwidth}
      \textbf{Abstract.} In this first homework we will be computing using a Monte Carlo method the exercise value of a share, which we will then use to determine a fair price for a call option of that share.
    \end{minipage}
\end{minipage}

\section{Computing the share's value via a Wiener process}

Firstly, we need to compute the value of the share itself after a certain amount of time. For this purpose we will solve the following stochastic differential equation, known as \emph{Wiener process} \cite{Wilmott}:

\begin{equation}
  \frac{\dif S}{S} = \sigma \dif X + \mu \dif t ~~~ ,
\end{equation}

where $\sigma$ is the volatility, $\mu$ is the drift and $S(t)$ the value of the option as a function of time. Instead of solving it analytically, we can discretize it as in \cite{sde} to find

\begin{equation}
  S_{i+1} = S_i \left( 1 + \sigma \Delta W_i + \mu \Delta t_i \right) ~~~ ,
\end{equation}

where $\Delta W_i = \sqrt{\Delta t_i} N(0, 1)$, simulating the random walk. In our case the time step is constant, so every $\Delta t_i$ will have the same value. The value after $nt$ time steps is the value at expiration date, which we will use to determine the price of the option.

\section{Determining fair prices for call options}

Now that we can compute the exercise value of the share, we can use it to determine how much we can pay for a call option for that share. Since this is not a value that can be derived via an objective procedure, we will consider two approaches to compute the price: the \emph{risk-free approach} and the \emph{risky approach}.

\subsection{Risk-free approach}

For the \textbf{risk-free approach}, we consider the probability distribution of the value of the share at expiration date, which will be a normal $N(\mu, \sigma)$. Now, since buying the call option gives us the \emph{right} to buy the share at the exercise price, we can choose not to use it if that means losing money. To represent this fact, instead of using the previous normal for our computations, we take every instance with share price lower than the exercise price and assign a profit of zero to it, instead of a negative value.

In the following plots we can clearly see that transformation:

\begin{figure}[H]
  \begin{center}
    \includegraphics[width=0.4\textwidth]{graphs/ex-normal-1.eps}
    \includegraphics[width=0.4\textwidth]{graphs/ex-normal-2.eps}
    \caption{(a) Example of normal distribution $N(1,1)$. (b) Distribution after applying the mentioned transformation, $\tilde N(1,1)$.}
    \label{fig:ex_normal}
  \end{center}
\end{figure}

Using this new probability distribution we can compute the risk-free price as the \textbf{average value} of it. This price is the minimum profit that on average we expect to make, so the fair price using this approach would be any amount smaller than the sum of the exercise price plus this value, as much as the writer of the option is willing to accept.

We can write this as

\begin{equation}
  \avg{oP}_{rf} \equiv \avg{\tilde N(\mu, \sigma)} ~~~ ,
\end{equation}

where $aE$ is the exercise price and $\avg{oP}_{rf}$ is the option price computed risk-free, and $\tilde N(\mu, \sigma)$ is the adjusted normal distribution.

\subsection{Risky approach}

On the other hand, we can take a \textbf{risky approach} and pay more than the risk-free price with a smaller probability of having profits. For this to be quantitative we have to assign a risk value to a price, in a way that zero corresponds to making the average profit and one corresponds to the impossible case of having profit paying an infinite amount of money.

So, assuming the expiry values of the share is a random variable with normal distribution, we can introduce the risk as a value $P \in [0, 1)$ and compute the probability of making average profit using the cumulative distribution function (cdf) as

\begin{equation*}
  \text{cdf}(\avg{oP}_{rf}, \mu, \sigma) ~~~ ,
\end{equation*}

and then computing the final probability using $P$ as

\begin{equation*}
  \text{cdf}(\avg{oP}_{rf}, \mu, \sigma)(1 - P) + P ~~~ .
\end{equation*}

Now, what we are doing here is mapping the interval $[\text{cdf}(\avg{oP}_{rf}, \mu, \sigma), 1)$ to $[0, 1)$, so that by computing the inverse cdf of this probability we can map the expected profits interval, $[\avg{oP}_{rf}, \infty)$, to $P \in [0, 1)$. We can write this calculation as

\begin{equation}
  \avg{oP}_r \equiv \text{cdf}^{-1} \left( \text{cdf}(\avg{oP}_{rf}, \mu, \sigma)(1 - P) + P \right) ~~~ .
\end{equation}

\section{Results}

In this section we will use the previous theory with the following values: $\mu = 0.02$, $\sigma = 0.2$, $S(0) = 14$ and $aE = 15$, over a period of $101$ days. In the following histogram we can see the results of performing $N=10^5$ random walks to compute the price of the option at exercise date. From here we can compute the values of $\mu$ and $\sigma$ mentioned in the previous section.

\begin{figure}[H]
  \begin{center}
    \includegraphics[width=0.6\textwidth]{graphs/rw-hist.eps}
    \caption{Histogram showing the distribution of share values after $N=10^5$ random walks.}
    \label{fig:rw_hist}
  \end{center}
\end{figure}

From this plot we can see that a drift ratio of $0.02$ is not enough to increase the value of the share significantly, however the volatility increases the width of the normal, and therefore for some value there will be a profit. Now we can proceed to compute the risk-free and risky value for the option. The two graphs below show the profit-risk relation in absolute profit and in percentage given these values.

\begin{figure}[H]
  \begin{center}
    \includegraphics[width=0.4\textwidth]{graphs/P-oP.eps}
    \includegraphics[width=0.4\textwidth]{graphs/P-Profit.eps}
    \caption{(a) Expected profit at the exercise date of the option. (b) Expected profit over exercise value of the option.}
    \label{fig:oP_plots}
  \end{center}
\end{figure}

As we can see in Figure \ref{fig:oP_plots}a, the risk-free approach gives a value for the option of $\sim 0.25$\euro, however it can be increased by taking more risk as we explained before. In this case we can consider a risk $P = 0.5$ acceptable for a profit\% of around $\sim 5\%$, although still very small. If this were a real option, we would advise against buying it in the first place, and find others with greater expected profits.

\begin{thebibliography}{28}
\raggedright
\addcontentsline{toc}{section}{Bibliography}

\bibitem{Wilmott} P. Wilmott et al, \emph{The Mathematics of Financial Derivatives}, 1995.

\bibitem{sde} Timothy Sauer, \emph{Handbook of Computational Statistics}, Springer, pp 529-550, July 2011.

\end{thebibliography}

\end{document}