\section{Ice Thickness Evolution}
The evolution of the ice thickness, $H$, stems from the continuity equation and can be expressed as
\begin{equation}
  \label{kin.eq.ice_thickness}
  \frac{\pd H}{\pd t} = -\vec\nabla\cdot(\overline{\vec{u}} H) + B,
\end{equation}
where $\overline{\vec{u}}$ is the vertically averaged ice velocity, $B$ is the surface mass balance and $\vec\nabla$ is the horizontal gradient operator \citep{Payne1997}.

For large--scale ice sheet models, the \emph{shallow ice approximation} is generally used. This approximation states that bedrock and ice surface slopes are assumed sufficiently small so that the normal stress components can be neglected \citep{Hutter1983}. The horizontal shear stresses ($\tau_{xz}$ and $\tau_{yz}$) can thus be approximated by
\begin{equation}
  \label{kin.eq.horiz_shear}
  \begin{split}
    \tau_{xz}(z)&=-\rho g(s-z)\frac{\pd s}{\pd x},\\
    \tau_{yz}(z)&=-\rho g(s-z)\frac{\pd s}{\pd y},
  \end{split}
\end{equation}
where $\rho$ is the density of ice, $g$ the acceleration due to gravity and $s=H+h$ the ice surface.

Strain rates $\dot{\epsilon}_{ij}$ of polycrystalline ice are related to the stress tensor by the non--linear flow law:
\begin{equation}
  \label{kin.eq.flowlaw}
  \dot{\epsilon}_{iz}=\frac12\left(\frac{\pd u_i}{\pd z}+\frac{\pd u_z}{\pd i}\right)=A(T^\ast)\tau_\ast^{(n-1)}\tau_{iz}\qquad i=x,y,
\end{equation}
where $\tau_\ast$ is the effective shear stress defined by the second invariant of the stress tensor, $n$ the flow law exponent and $A$ the temperature--dependent flow law coefficient. $T^\ast$ is the absolute temperature corrected for the dependence of the melting point on pressure \cite[$T^\ast=T+8.7\cdot10^{-4}(H+h-z)$, $T$ in Kelvin,][]{Huybrechts1986}. The parameters $A$ and $n$ have to be found by experiment. $n$ is usually taken to be 3. $A$ depends on factors such as temperature, crystal size and orientation, and ice impurities. Experiments suggest that $A$ follows the Arrhenius relationship:
\begin{equation}
  \label{kin.eq.arrhenius}
  A(T^\ast)=fae^{-Q/RT^\ast},
\end{equation}where $a$ is a temperature--independent material constant, $Q$ is the activation energy for creep and $R$ is the universal gas constant \citep{Paterson1994}. $f$ is a tuning parameter used to `speed--up' ice flow and accounts for ice impurities and the development of anisotropic ice fabrics \citep{Payne1999,Tarasov1999,Tarasov2000,Peltier2000}.

Integrating \eqref{kin.eq.arrhenius} with respect to $z$ gives the horizontal velocity profile:
\begin{equation}
  \label{kin.eq.horiz_velo}
  \vec u(z)-\vec u(h) = -2(\rho g)^n|\vec\nabla s|^{n-1}\vec\nabla s\int_h^zA(s-z)^ndz,
\end{equation}
where $\vec u(h)$ is the basal velocity (sliding velocity). Integrating \eqref{kin.eq.horiz_velo} again with respect to $z$ gives an expression for the vertically averaged ice velocity:
\begin{equation}
  \label{kin.eq.avg_velo}
  \overline{\vec u}H=-2(\rho g)^n|\vec\nabla s|^{n-1}\vec\nabla s\int_h^s\int_h^zA(s-z)^ndzdz'.
\end{equation}

The vertical ice velocity stems from the conservation of mass for an incompressible material:
\begin{equation}
  \label{kin.eq.incompress}
  \frac{\pd u_x}{\pd x} + \frac{\pd u_y}{\pd y} + \frac{\pd u_z}{\pd z} = 0.
\end{equation}
Integrating \eqref{kin.eq.incompress} with respect to $z$ gives the vertical velocity distribution of each ice column:
\begin{equation}
  \label{kin.eq.vert_velo}
  w(z)=-\int_h^z\vec\nabla\cdot\vec u(z)dz+w(h),
\end{equation}
with lower, kinematic boundary condition
\begin{equation}
  w(h)=\frac{\pd h}{\pd t}+\vec u(h)\cdot\vec\nabla h+S,
\end{equation}
where $S$ is the melt rate at the ice base given by Equation \eqref{temp.eq.meltrate}. The upper kinematic boundary is given by the surface mass balance and must satisfy:
\begin{equation}
  \label{kin.eq.upper_bc}
  w(s)=\frac{\pd s}{\pd t}+\vec u(s)\cdot\vec\nabla s+B.
\end{equation}

\input{\dir/grid.tex}

\subsection{Ice Sheet Equations in $\sigma$--Coordinates}
The horizontal velocity, Equation \eqref{kin.eq.horiz_velo}, becomes in the $\sigma$--coordinate system
\begin{equation}
  \label{kin.eq.vert_velo_sigma}
  \vec u(\sigma) = -2(\rho g)^nH^{n+1}|\vec\nabla s|^{n-1}\vec\nabla s\int_1^\sigma A\sigma^nd\sigma+\vec u(1)
\end{equation}
and the vertically averaged velocity
\begin{equation}
  \label{kin.eq.avg_velo_scaled}
  \overline{\vec u} H=H\int_0^1\vec ud\sigma+\vec u(1)H
\end{equation}
The vertical velocity, Equation \eqref{kin.eq.vert_velo}, becomes
\begin{equation}
  \label{kin.eq.vert_velo_scaled}
  w(\sigma)=-\int_1^\sigma\left(\frac{\pd\vec u}{\pd\sigma}\cdot(\vec\nabla s-\sigma\vec\nabla H)+H\vec\nabla\cdot\vec u\right)d\sigma+w(1)
\end{equation}
and lower boundary condition
\begin{equation}
  w(1)=\frac{\pd h}{\pd t}+\vec u(1)\cdot\vec\nabla h+S.
\end{equation}

\subsection{Calculating the Horizontal Velocity and the Diffusivity}
Horizontal velocity and diffusivity calculations are split up into two parts:
\begin{subequations}
  \begin{align}
    \vec u(\sigma)&=c\vec\nabla s+\vec u(1)\\
    D &=H\int_0^1cd\sigma\\
    \vec q&=D\vec\nabla s+H\vec u(1)\\
    \intertext{with}
    c(\sigma)&=-2(\rho g)^nH^{n+1}|\vec\nabla s|^{n-1}\int_1^\sigma A\sigma^nd\sigma
  \end{align}
\end{subequations}

Quantities $\vec u$ and $D$ are found on the velocity grid. Integrating from the ice base ($k=N-1$), the discretised quantities become
\begin{subequations}
  \begin{equation}
    \tilde{c}_{r,s,N}=0
  \end{equation}
  \begin{multline}
    \tilde{c}_{r,s,k}=-2(\rho g)^nH_{r,s}^{n+1}\left(({\tilde{s}^x_{r,s}})^2+({\tilde{s}^y_{r,s}})^2\right)^{\frac{n-1}{2}}\\
    \sum_{\kappa=N-1}^k\frac{A_{r,s,\kappa}+A_{r,s,\kappa+1}}2 \left(\frac{\sigma_{\kappa+1}+\sigma_\kappa}2\right)^n(\sigma_{\kappa+1}-\sigma_\kappa)
  \end{multline}
  \begin{equation}
    \tilde{D}_{r,s}=H_{r,s}\sum_{k=0}^{N-1}\frac{\tilde{c}_{r,s,k}+\tilde{c}_{r,s,k+1}}2(\sigma_{k+1}-\sigma_k)
  \end{equation}
\end{subequations}
Expressions for $\vec{u}_{i,j,k}$ and $\vec{q}_{i,j}$ are straight forward.

\subsection{Solving the Ice Thickness Evolution Equation}
Equation \eqref{kin.eq.ice_thickness} can be rewritten as a diffusion equation, with non--linear diffusion coefficient $D$:
\begin{equation}
  \label{kin.eq.ice_evo}
  \frac{\pd H}{\pd t}=-\vec\nabla\cdot D\vec\nabla s+B=-\vec\nabla\cdot\vec q+B
\end{equation}
This non--linear partial differential equation can be linearised by using the diffusion coefficient from the previous time step. The diffusion coefficient is calculated on the $(r,s)$--grid, i.e. staggered in both $x$ and $y$ direction. Figure \ref{kin.fig.staggered_grid} illustrates the staggered grid. Using finite differences, the fluxes in $x$ direction, $q^x$ become
\begin{subequations}
\begin{align}
  q^x_{i+\frac12,j}&=-\frac12(\tilde{D}_{r,s}+\tilde{D}_{r,s-1})\frac{s_{i+1,j}-s_{i,j}}{\Delta x}\\
  q^x_{i-\frac12,j}&=-\frac12(\tilde{D}_{r-1,s}+\tilde{D}_{r-1,s-1})\frac{s_{i,j}-s_{i-1,j}}{\Delta x}\\
  \intertext{and the fluxes in $y$ direction}
  q^y_{i,j+\frac12}&=-\frac12(\tilde{D}_{r,s}+\tilde{D}_{r-1,s})\frac{s_{i,j+1}-s_{i,j}}{\Delta y}\\
  q^y_{i,j-\frac12}&=-\frac12(\tilde{D}_{r,s-1}+\tilde{D}_{r-1,s-1})\frac{s_{i,j}-s_{i,j-1}}{\Delta y}.
\end{align}
\end{subequations}

\begin{figure}[htbp]
  \centering
  \includegraphics{\dir/figs/staggered_grid.eps}
  \caption{Illustration of the staggered grid used to calculate ice thicknesses, diffusivities and mass fluxes.}
  \label{kin.fig.staggered_grid}
\end{figure}

\subsubsection{ADI Scheme}
The alternating--direction implicit method (ADI) uses the concept of operator splitting where Equation \eqref{kin.eq.ice_evo} is first solved in the $x$--direction and then in the $y$--direction, \citep{Press1992}. The time step $\Delta t$ is devided into two time steps $\Delta t/2$. The descretised version of Equation \eqref{kin.eq.ice_evo} becomes \citep{Huybrechts1986}:
\begin{subequations}
\begin{align}
  \label{kin.eq.adi_1}
  2\frac{H_{i,j}^{t+\frac12}-H_{i,j}^{t}}{\Delta t} &= -\frac{q_{i+\frac12,j}^{x,t+\frac12}-q_{i-\frac12,j}^{x,t+\frac12}}{\Delta x} - \frac{q_{i,j+\frac12}^{y,t}-q_{i,j-\frac12}^{y,t}}{\Delta y} + B_{i,j} \\
  \label{kin.eq.adi_2}
  2\frac{H_{i,j}^{t+1}-H_{i,j}^{t+\frac12}}{\Delta t} &= -\frac{q_{i+\frac12,j}^{x,t+\frac12}-q_{i-\frac12,j}^{x,t+\frac12}}{\Delta x} - \frac{q_{i,j+\frac12}^{y,t+1}-q_{i,j-\frac12}^{y,t+1}}{\Delta y} + B_{i,j}
\end{align}
\end{subequations}
Gathering all $t+\frac12$ terms on the left side, Equation \eqref{kin.eq.adi_1} can be expressed as a tri--diagonal set of equations for each row $j$:
\begin{equation}
  -\alpha_{i,j}H_{i-1,j}^{t+\frac12} + (1-\beta_{i,j})H_{i,j}^{t+\frac12} - \gamma_{i,j}H_{i+1,j}^{t+\frac12} = \delta_{i,j}
\end{equation}
with
\begin{subequations}
  \begin{align}
  \alpha_{i,j} &=\frac{\tilde{D}_{r-1,s}+\tilde{D}_{r-1,s-1}}{4\Delta x^2}\Delta t\\
  \beta_{i,j}  &=-\frac{\tilde{D}_{r,s}+2\tilde{D}_{r-1,s}+\tilde{D}_{r-1,s-1}}{4\Delta x^2}\Delta t = -(\alpha_{i,j}+\gamma_{i,j})\\
  \gamma_{i,j} &=\frac{\tilde{D}_{r,s}+\tilde{D}_{r,s-1}}{4\Delta x^2}\Delta t
  \end{align}
and the RHS,
\begin{equation}
  \delta_{i,j} = H_{i,j}^t-\frac{\Delta t}{2\Delta y}\left(q_{i,j+\frac12}^{y,t}-q_{i,j-\frac12}^{y,t}\right) + \frac{\Delta t}2B_{i,j} + \alpha_{i,j}h_{i-1,j} -\beta_{i,j}h_{i,j} + \gamma_{i,j}h_{i+1,j}.
\end{equation}
\end{subequations}

A similar tri--diagonal system is found for each column, $i$ of Equation \eqref{kin.eq.adi_2}.

\subsubsection{Linearised Semi--Implicit Scheme}
Using the Crank--Nicolson scheme, the semi--implicit temporal discretisation of \eqref{kin.eq.ice_evo} is then:
\begin{multline}
\label{kin.eq.ice_evo_disc1}
  \frac{H^{t+1}_{i,j}-H^t_{i,j}}{\Delta t}=\frac{q^{x,t+1}_{i+\frac12,j}-q^{x,t+1}_{i-\frac12,j}}{2\Delta x}+\frac{q^{y,t+1}_{i,j+\frac12}-q^{y,t+1}_{i,j-\frac12}}{2\Delta y} \\
  +\frac{q^{x,t}_{i+\frac12,j}-q^{x,t}_{i-\frac12,j}}{2\Delta x}+\frac{q^{y,t}_{i,j+\frac12}-q^{y,t}_{i,j-\frac12}}{2\Delta y}+ B_{i,j}
\end{multline}
The superscripts $^t$ and $^{t+1}$ indicate at what time the ice thickness $H$ is evaluated. Collecting all $H^{t+1}$ terms of \eqref{kin.eq.ice_evo_disc1} on the LHS and moving all other terms to the RHS we can rewrite \eqref{kin.eq.ice_evo_disc1} as
\begin{equation}
  \label{kin.eq.evo_matrix}
  -\alpha_{i,j}H^{t+1}_{i-1,j} - \beta_{i,j}H^{t+1}_{i+1,j} - \gamma_{i,j}H^{t+1}_{i,j-1} - \delta_{i,j}H^{t+1}_{i,j+1}+ (1-\epsilon_{i,j})H^{t+1}_{i,j} = \zeta_{i,j}
\end{equation}
with the RHS,
\begin{multline}
  \zeta_{i,j} = \alpha_{i,j}H^{t}_{i-1,j} + \beta_{i,j}H^{t}_{i+1,j} + \gamma_{i,j}H^{t}_{i,j-1} + \delta_{i,j}H^{t}_{i,j+1} + (1+\epsilon_{i,j})H^{t}_{i,j} \\
  + 2(\alpha_{i,j}h_{i-1,j} + \beta_{i,j}h_{i+1,j} + \gamma_{i,j}h_{i,j-1} + \delta_{i,j}h_{i,j+1}+ \epsilon_{i,j}h_{i,j}) + B_{i,j}\Delta t
\end{multline}
with the elements of the sparse matrix
\begin{subequations}
  \begin{align}
    \alpha_{i,j} &=\frac{\tilde{D}_{r-1,s}+\tilde{D}_{r-1,s-1}}{4\Delta x^2}\Delta t\\
    \beta_{i,j} &=\frac{\tilde{D}_{r,s}+\tilde{D}_{r,s-1}}{4\Delta x^2}\Delta t\\
    \gamma_{i,j} &=\frac{\tilde{D}_{r,s-1}+\tilde{D}_{r-1,s-1}}{4\Delta y^2}\Delta t\\
    \delta_{i,j} &=\frac{\tilde{D}_{r,s}+\tilde{D}_{r-1,s}}{4\Delta y^2}\Delta t\\
    \epsilon_{i,j} &=-(\alpha_{i,j}+\beta_{i,j}+\gamma_{i,j}+\delta_{i,j})
  \end{align}
\end{subequations}

This matrix equation is solved using an iterative matrix solver for non-symmetric sparse matrices. The solver used here is the bi--conjugate gradient method with incomplete LU decomposition preconditioning provided by the SLAP package.

\subsubsection{Non--Linear Scheme}
The non--linearity of Equation \eqref{kin.eq.ice_evo} arises from the dependance of $D$ on $s$. A non--linear scheme for \eqref{kin.eq.ice_evo} can be formulated using Picard iteration, which consists of two iterations: an outer, non--linear and an inner, linear equation. The scheme is started off with the diffusivity from the previous time step, i.e.
\begin{subequations}
  \begin{equation}
    D^{(0),t+1}=D^{t}
  \end{equation}
and Equation \eqref{kin.eq.evo_matrix} becomes
\begin{multline}
  \label{kin.eq.evo_matrix_nonlin}
  -\alpha^{(\xi),t+1}_{i,j}H^{t+1}_{i-1,j} - \beta^{(\xi),t+1}_{i,j}H^{(\xi+1),t+1}_{i+1,j} - \gamma^{(\xi),t+1}_{i,j}H^{(\xi+1),t+1}_{i,j-1} \\
  - \delta^{(\xi),t+1}_{i,j}H^{(\xi+1),t+1}_{i,j+1}+ (1-\epsilon^{(\xi),t+1}_{i,j})H^{(\xi+1),t+1}_{i,j} = \zeta^{(0),t}_{i,j}
\end{multline}
\end{subequations}
Equation \eqref{kin.eq.evo_matrix_nonlin} is iterated over $\xi$ until the maximum ice thickness residual is smaller than some threshold:
\begin{equation}
  \max\left(\left|H^{(\xi+1),t+1}-H^{(\xi),t+1}\right|\right)<H_{\text{res}}
\end{equation}

\begin{figure}[htbp]
  \centering
  \includegraphics[width=0.5\textwidth]{\dir/figs/thick_evo.eps}
  \caption{Flow diagram showing how the linearised solver (on the left) and the non--linear solver work. The inner, linear iteration is contained within the box labeled ``calculate new ice distribution''.}
  \label{kin.fig.solvers}
\end{figure}

\input{\dir/vert_velo.tex}


\section{Majorana Fermions}
Define $\nu=S_0 S\pi$ to be a topological band invariant.
$\nu=+1$ is the non-topological phase.

The band looks like $E_k = -2t\cos k$,
which mas a minimum of $\mu = -2t$ and a maximum of $\mu= 2t$.

$\nu = -1$ is the topological phase.

Majorana fermions obey
\begin{align}
    \left\{ \gamma_i, \gamma_j \right\} &= 2\delta_{ij}
\end{align}
which implies
\begin{align}
    \gamma_{i}^2 = 1.
\end{align}
Here the operators
$\gamma_i$ act on sites $i=1,\ldots,2N$.

Let $\tilde{\gamma_i}:=\sum_{j} W_{ij}\gamma_j$
where $w\in O(2N)$ is an orthogonal matrix.
Then you still get
\begin{align}
    \left\{ \tilde{\gamma}_i, \tilde{\gamma}_j \right\} = 2\delta_{ij}
\end{align}

Thus the largest possible symmetry of $2N$ Majoranas is
$O(2N)$.

One may define creation and annihilation operators
$c_i$ and $c_i^\dagger$ for ``complex fermions''
which obey
\begin{align}
    \left\{ c_i^\dagger, c_j \right\} &= \delta_{ij}\\
    \left\{ c_i, c_j \right\} &= 0.
\end{align}
The two can be related by
\begin{align}
    c_i &= \frac{1}{2}\left(
        \gamma_{2i - 1} + 1\gamma_{2i}
    \right)
\end{align}
where $i=1,\ldots,N$.

Thus $2N$ Majoranas is equivalent to $N$ complex fermions.

$N$ complex fermions has $2^{N}$ states,
since the number operator $c_i^\dagger c_i = 0$ or $1$.

So we can say that 1 complex fermion has a
``quantum dimension'' of 2,
but 1 Majorana fermion has a quantum dimension of $\sqrt{2}$.

Let us introduce the fermion parity operator
\begin{align}
    P_i = {\left( -1 \right)}^{c_i^\dagger c_i}
    = -i \gamma_{2i - 1}\gamma_{2i}=
    \begin{cases}
        +1\\
        -1
    \end{cases}
\end{align}
From this we can also define the total fermion parity
\begin{align}
    P &= \prod_{i=1}^{N} P_i.
\end{align}
Under $O(2N)$ rotation this transforms as
\begin{align}
    P \to P\det W
\end{align}
so $SO(2N)$ fixes $P$.

Consider just 1 pair with
\begin{align}
    \gamma_{1} &= c + c^\dagger\\
    \gamma_2 &= \frac{1}{2}\left( c - c^{\dagger} \right)
\end{align}
and such as system only has two states
$\ket{0}$ for empty and $\ket{1}$ for filled.
Then the Majorana operators act like
\begin{align}
    \gamma_1\ket{0} &= \ket{1}\\
    \gamma_2\ket{0} &= i\ket{1}
\end{align}
so in fact
\begin{align}
    \gamma_1 &=
    \begin{pmatrix}
        0 & 1\\
        1 & 0
    \end{pmatrix}
    = \sigma^{x}\\
    \gamma_1 &=
    \begin{pmatrix}
        0 & -i\\
        i & 0
    \end{pmatrix}
    = \sigma^{y}
\end{align}
and their product satsifies
\begin{align}
    -i \gamma_1 \gamma_2 = -i^2 \sigma^z = \sigma^z.
\end{align}
The most general free ferion Hamiltonian assuming there being
$2N$ Majoranas is
\begin{align}
    H &= \frac{i}{4} \sum_{ij} A_{ij} \gamma_i \gamma_j
\end{align}
where $A_{ij} = - A_{ji} = A_{ij}^*$
so $A$ is a $2N\times 2N$ real skew symmetric matrix.

We can always bring $A$ into canonical form
\begin{align}
    D &= WAW^\dagger =
    \begin{pmatrix}
        0 & \varepsilon_1 & & & & & \\
        -\varepsilon_1 & 0 & & & & & \\
        & & 0 & \varepsilon_2 & & & \\
        & & -\varepsilon_2 & 0 & & &\\
        & & & & \ddots & &\\
        & & & & & 0 & \varepsilon_N\\
        & & & & & -\varepsilon_N & 0\\
    \end{pmatrix}
\end{align}
so we can set $\tilde{\gamma} = W\gamma$
and
\begin{align}
    \tilde{c}_j &= \frac{1}{2}\left( \tilde{\gamma}_{2j} - 1
    + \tilde{\gamma}_{2i}\right)
\end{align}
The Hamiltonian in this form is
\begin{align}
    H &=
    \frac{i}{2}\sum_{j=1}^{N}
    \varepsilon_j \tilde{\gamma}_{2j - 1} \tilde{\gamma}_{2j}\\
    &=
    \sum_{j=1}^{N} \varepsilon_j\left( \tilde{c}_j^\dagger \tilde{c}_j -
    \frac{1}{2}\right)
\end{align}
We can then do the so-called ``particle-hole transformation''
\begin{align}
    C: \quad
    \tilde{c}_j^\dagger &\to \tilde{c}_j\\
    \tilde{\gamma}_j &\to {(-1)}^{j+1}\tilde{\gamma}_j
\end{align}
which also can transform the Hamiltonian under conjugation with
\begin{align}
    C:\quad
    H\to CHC = -H.
\end{align}
This means tthat eigenvalues must come in pairs $(E, -E)$.

If all $\varepsilon_i \ne 0$
then the lowest energy state is uniqued and gapped.

Consider the fermion parity of the ground state.
Consider a single pair
\begin{align}
    H &= \frac{i}{2} \sum \gamma_1 \gamma_2
\end{align}
which for the ground state if $\varepsilon > 0$ with
$i\gamma_1 \gamma_2 = -1$
implies
$P = -i \gamma_1 \gamma_2 = \pm 1$.
The parity $P=\sgn(\varepsilon)$.

For the whole system,
\begin{align}
    P &=
    \prod_{i=1}^{N} \sgn\left( \varepsilon_i \right)
    =
    \sgn\left[ \Pf(D) \right]
    = \sgn\left( \Pf\left( A \right) \det W \right)
\end{align}
where $\Pf$ denotes the Pfaffian,
which is defined for an $N\times N$ matrix with $N$ even matrix by
\begin{align}
    \Pf(M) &=
    \frac{1}{2^{N}}
    \frac{1}{N!}
    \sum_{\text{permutations }\sigma}
    {(-1)}^{\sgn(\sigma)}
    \prod_{i=1}^{N}
    A_{\sigma(2 i - 1), \sigma(2i)}.
\end{align}

For example,
\begin{align}
    \Pf
    \begin{pmatrix}
        0 & \varepsilon\\
        -\varepsilon & 0
    \end{pmatrix}
    = \varepsilon
\end{align}
and
\begin{align}
    \Pf
    \begin{pmatrix}
        0 & \varepsilon & &\\
        -\varepsilon & 0 & &\\
        & & 0 & \varepsilon'\\
        & & -\varepsilon' & 0
    \end{pmatrix}
    = \varepsilon\cdot \varepsilon'
\end{align}
A useful property is that
\begin{align}
    \det(X) = {\left[ \Pf(X) \right]}^2
\end{align}

If we fixed the definition of fermion parity,
only consider $W\in SO(2N)$,
then
\begin{align}
    P(H) &= \sgn \Pf A
\end{align}

\section{Boundary Majorana Zero Modes}
Consider the Hamiltonian
\begin{align}
    H &=
    \sum_{i}\left[
        -t \left( c_i^\dagger c_{i + 1} + c_{i+1}^\dagger c_i\right)
        - \mu\left( c_i^\dagger c_i^\dagger - \frac{1}{2} \right)
        + \Delta c_i c_{i+1}
        + \Delta^* c_{i+1}c_{i}
    \right]
\end{align}
where $\Delta = |\Delta| e^{i\phi}$,
in which case we can just redefine
\begin{align}
    c_i \to \tilde{c}_i = e^{i\phi/2}c_i
\end{align}
to absorb the $\phi$.
Then you would write
\begin{align}
    \tilde{c}_i &= \frac{1}{2}
    \left( \gamma_{2i - 1} + i\gamma_{2i} \right).
\end{align}
Then
\begin{align}
    H &= \frac{i}{2}\sum_{i}\left[
        \left( |\Delta| + t \right)\gamma_{2i}\gamma_{2i+1}
        + \left( |\Delta| - t \right) \gamma_{2i - 1} \gamma_{2i + 2}
        - \mu \gamma_{2i - 1} \gamma_{2i}
    \right]
\end{align}
Let us consider two limiting cases with open boundary conditions.

Case 1: $|\Delta| = t = 0$ and $\mu > 0$.
Then
\begin{align}
    H &= -\frac{i}{2}\mu\sum_{i}\gamma_{2i - 1}\gamma_{2i}
\end{align}
so each site has
\begin{align}
    -i\gamma_{2i - 1}\gamma_{2i} = 1 = P_i
\end{align}
This is an ``atomic superconductor''
with tightly bound Cooper pairs
and strong pairing phases.
The cooper wave function is exponentially decaying.

Case 2: $|\Delta| = t > 0$ and $\mu = 0$.
$\gamma_1$ and $\gamma_{2N}$ are boundary Majorana zero modes with
\begin{align}
    \left[ H, \gamma_1 \right] =
    \left[ H, \gamma_{2N} = 0 \right]
\end{align}
which means there is a 2-fold degeneracy of the 0 energy state
corresponding to
\begin{align}
    i\gamma_1 \gamma_{2N} = \pm 1
\end{align}

Away from the idealized points,
still free fermion,
we can look for solutions to
\begin{align}
    \left[ H, \tilde{\gamma}_{L} \right]
    = \left[ H, \tilde{\gamma}_{R} \right] = 0
\end{align}
in the limit $N\to\infty$ where
\begin{align}
    \tilde{\gamma}_L &= \alpha_1 \gamma_1 + \alpha_2 \gamma_2 + \cdots\\
    \tilde{\gamma}_R &= \beta_{2N}\gamma_{2N} + \beta_{2N - 1}\gamma_{2N - 1} +
    \cdots
\end{align}
where
\begin{align}
    \alpha_x &\sim e^{-x/\xi}\\
    \beta_{2N - x} &\sim e^{-x/\xi}.
\end{align}
For $\nu=-1$,
can find solution $\alpha$ and $\beta$
decay experimentally away from the boundary.

The finite size system boundayr Majorana zero modes will have some wave function
overlap.
This effective Hamiltonian ground state is in a subspace
with effective Hamiltonian
\begin{align}
    H_{\textrm{eff}} &=
    \frac{i}{2} t_{\textrm{eff}} \tilde{\gamma}_L \tilde{\gamma}_R
\end{align}
where the effective hopping coefficient is
$t_{\textrm{eff}}\sim e^{-N/\xi}$.

At the phase transition,
$\xi\to\infty$,
the MZMs become delocalized.
This is the critical point that is a field theory of 1D gapless Majorana modes.

Note that when
\begin{align}
    \left[ H, \tilde{\gamma}_L \right]
    = \left[ H, \tilde{\gamma}_R \right] = 0
\end{align}
it is called a ``string zero mode.''.

If we turn on interactions,
\begin{align}
   \delta H \propto
   \sum_{ijkl} \gamma_i \gamma_j \gamma_k \gamma_l
\end{align}
The boundary MZMs still exist,
but they have a doubly degenerate ground state
in the $N\to\infty$ limit,
but generically no longer commute with $H$.

The ground state degeneracy is ``topological''
in the following sense.
\begin{enumerate}
    \item Robust to perturbations.
    \item No \emph{local} bosonic operator that can distinguish between the
        ground states.
        Specifically,
        \begin{align}
            \bra{j}\mathcal{O}\ket{i} \propto \delta_{ij}
            + O\left( e^{-L/\xi} \right)
        \end{align}
        for any local bosonic operator $\mathcal{O}$.
        It is not protected against local fermionic operators though.
        \begin{align}
            \bra{j}c_1\ket{i}, \bra{j} c_N\ket{i}
            \not\propto \delta_{ij}
        \end{align}
        As long as we can protect the system from stray fermions,
        we have a
        ``topologically protected qubit''.
        Otherwise,
        we have so-called ``quasi-particle'' poisoning.
\end{enumerate}
Boundary Majorana Zero Modes can be understood from field theory.

Suppose we have a spinless $p$-wave superconductor with Hamiltonian
\begin{align}
    H &= \frac{1}{2}\sum_{k\in\mathrm{BZ}}
    \Psi_k^\dagger
    \begin{pmatrix}
        \varepsilon_k & \Delta_k^*\\
        \Delta_k & -\varepsilon_k
    \end{pmatrix}
    \Psi_k
\end{align}
where
\begin{align}
    \Psi_k =
    \begin{pmatrix}
        \Psi_k\\
        \Psi_k^\dagger
    \end{pmatrix}
\end{align}
and $t=|\Delta|$, $\mu\approx -2t$ and
\begin{align}
    \varepsilon_k &= -2t\cos k - \mu\\
    \Delta_k &= -2i\Delta\sin k.
\end{align}

The the limit $k\to 0$ and we get
\begin{align}
    \varepsilon_k &\to -2t - \mu = m\\
    \Delta_k &\to -2i\Delta k
\end{align}
where $m$ is defined as the mass term in the field theory.
Assume $\Delta$ is real so that $\tilde{\Delta}:= 2\Delta$.
The Hamiltonian thus can be written as
\begin{align}
    H_k &=
    \begin{pmatrix}
        m & i\tilde{\Delta}k\\
        -i\tilde{\Delta}k & -m
    \end{pmatrix}
    = \tilde{\Delta}k\sigma^y + m\sigma^z.
\end{align}
In real space continuous variables,
\begin{align}
    H &=
    \frac{1}{2}\int dx\,
    \Psi^\dagger(x)
    \left( -i\tilde{\Delta}\sigma^y \partial_x + m\sigma^z \right)
    \Psi(x)
\end{align}
where
\begin{align}
    \Psi(x) &=
    \begin{pmatrix}
        \Psi(x)\\
        \Psi^\dagger(x).
    \end{pmatrix}
\end{align}
here, we have
\begin{align}
    \Psi^*(x) &= \left( \Psi^\dagger(x) \right)^T
\end{align}
and the Majorana condition is
\begin{align}
    \Psi(x) &= \sigma^x \Psi^*(x).
\end{align}
The ``trivial phase'' is for $\mu + 2t < 0$ which means $m>0$.
The topological phase occurs when
$\mu + 2t > 0$ which implies $m < 0$.
And the critical point is when $\mu = -2t$ and we get $m=0$.
This is a massless $(1+1)D$ Majorana field.

\subsection{Digression: Jackiw-Rabbi system}
Consider a $(1 + 1)D$ massive Dirac fermion.
Then if we define
\begin{align}
    \Psi =
    \begin{pmatrix}
        \Psi_1\\
        \Psi_2
    \end{pmatrix}
\end{align}
consider a $(1+1)D$ massive Dirac fermion
\begin{align}
    H_{\textrm{Dirac}} &=
    \frac{1}{2}\int dx\,\left[
    -i \nu \Psi^\dagger \sigma^y \partial_x \Psi
    + m\Psi^\dagger \sigma^z \Psi
    \right]
\end{align}
which arises as the continuum limit of the SSH model
\begin{align}
    H_{\mathrm{SSH}} &=
    -\sum_{i}\left[
    \left( t + {(-1)}^{i} \delta t \right) c_i^\dagger c_{i+1} + \textrm{h.c.}
    \right]
\end{align}
This is used to model polyaccetaline if we enable two spin $\frac{1}{2}$.
Lable each unit cell by
\begin{align}
    \Psi_r &=
    \begin{pmatrix}
        c_{2r - 1}\\
        c_{2r}
    \end{pmatrix}
\end{align}
Also $\delta t \approx m$ of Dirac point.

Consider a domain wall in $m(x)$.
Look for eigenstates of $H_{\textrm{Dirac}}$.
\begin{align}
    q^\dagger &= \int dx\,\left[
    \phi_1(x) \psi_1^\dagger(x)
    + \phi_2(x) \psi_2^\dagger(x)
    \right]
\end{align}
defines eigenstate of energy $\varepsilon$ since
\begin{align}
    \left[ H_{\mathrm{Dirac}}, q^\dagger \right] &=
    \varepsilon q^\dagger.
\end{align}
where $\phi =
\begin{pmatrix}
    \phi_1\\
    \phi_2
\end{pmatrix}$.

Consider a simple limit with 1 fermion pre unit cell.
And we have 2 chains.
1 fermion = 2 domain wall.
1 domain wall = $\pm\frac{1}{2}$ charges.
Example of charge fractionalization in charge topological insulator.

Generalization: $\frac{1}{m}$ fillings.
CDW with on sites per unit cell $\to$
elementary domain wall
$\implies$
$\pm \frac{1}{m}$ charges.

\begin{enumerate}
    \item Modulation arises from spontaneous symmetry breaking
        both dimerization patterns have some energy.
        Domain walls are ``deconfined''.
    \item Modulation from ``explicit'' symmetry breaking terms domain wall.
        Then two dimerization patterns have different energy
        $\to$ confined linear energy cost.
\end{enumerate}

\subsection{MZMs and J-R soliton}
\begin{align}
    \Phi(x) &=
    \begin{pmatrix}
        \Psi\\
        \Psi^\dagger
    \end{pmatrix}\\
    \Psi &= \sigma_x \Psi^*\\
    \Phi_0 &=
    \begin{pmatrix}
        \phi_0\\
        \lambda \phi_0
    \end{pmatrix}\\
    q^\dagger &=
    \int dx\, \left[
    \phi_0\Psi^\dagger(x) + \lambda \phi_0\Psi(x)
    \right]
\end{align}
$\phi_0$ is real $\implies$ $q= \lambda q^\dagger$ and
\begin{align}
    \gamma &=
    \begin{cases}
        q & \lambda = 1\\
        iq & \lambda = -1
    \end{cases}
\end{align}
which implies
\begin{align}
    \gamma^\dagger &= \gamma\\
    \gamma^2 &= 1
\end{align}
which are Majorana zero modes bound to domain walls.


\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\begmath 4.2 Linear Least-Squares and Covariance Matrix

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

The principal expected application of this set of subroutines is the
solution of the linear least-squares problem, $A{\bf x} \simeq {\bf b}$,
where $A$ is an $m \times n$ matrix with $m > n$, and ${\bf b}$ is an $m$%
-vector. The solution vector, ${\bf x}$, is an $n$-vector. Computation of
the covariance matrix of the solution vector, ${\bf x}$, is also supported.

This software is not limited to the usual case of $m > n$ and $A$ being of
full rank. A pseudoinverse solution is provided for all of the cases, $m > n$%
, $m = n$, and $m < n$, including the case of $A$ being rank-deficient.
Furthermore, the right-side of the problem may be an $m \times k$ matrix, $B$,
in which case the solution will be an $n \times k$ matrix, $X$.

\subsection{Usage}

\subsubsection{Solving a Least-Squares Problem}

\paragraph{Program Prototype, Single Precision}

\begin{description}
\item[INTEGER]  \ {\bf LDA, M, N, LDB, NB, KRANK, IP}($\geq $N)

\item[REAL]  \ {\bf A}(LDA, $\geq $N){\bf , TAU, RNORM}($\geq $NB){\bf ,\\
WORK}($\geq $N){\bf , B}(LDB) or {\bf B}(LDB, $\geq $ NB)
\end{description}

Assign values to A(,), LDA, M, N, B(,), LDB, NB, and TAU.

\begin{center}
\fbox{\begin{tabular}{@{\bf }c}
CALL SHFTI(A, LDA, M, N, B, LDB, NB,\\
TAU, KRANK, RNORM, WORK, IP)\\
\end{tabular}}
\end{center}

Computed quantities are returned in A(,), B(,), KRANK, RNORM(), and IP().

\paragraph{Argument Definitions}

\begin{description}
\item[A(,)]  \ [inout] On entry contains the M$\times $N matrix, $A$, of the
least-squares problem. Permit M $>$ N, M = N, or M $<$ N. On return contains
an upper triangular matrix that can be used by subroutine, SCOV2, to compute
the covariance matrix.

\item[LDA]  \ [in] First dimensioning parameter for the array, A(,). Require
LDA $\geq $ M.

\item[M]  \ [in] Number of rows of data in A(,) and B(,) on entry. Require
M $\geq 1.$

\item[N]  \ [in] Number of columns of data in A(,) on entry. Require N $\geq
0.$

\item[B(,)]  \ [inout] May be a singly or doubly subscripted array. On entry
contains the right-side M-vector, ${\bf b}$, or (M$\times $NB)-matrix, $B$,
for the least-squares problem. On return contains the solution N-vector, $%
{\bf x}$, or (N$\times $NB)-matrix, $X$.

\item[LDB]  \ [in] First dimensioning parameter for the array B(,). Require
LDB $\geq \max (\text{M},\text{N})$ when NB $\geq 1$, and LDB $\geq 1$
when NB $=0.$

\item[NB]  \ [in] Number of right-side vectors for the least-squares
problem. Require NB $\geq 0$. If NB = 1, the array, B(,), may be either
singly or doubly subscripted. If NB $>1$, the array B(,) must be doubly
subscripted. If NB = 0, this subroutine will not access B(,).

\item[TAU]  \ [in] Absolute tolerance parameter provided by the user.
Ideally should indicate the noise level of the data in the given matrix, $A$.
The value, zero, is acceptable. Will be used in estimating the rank of $A$.
Larger values of TAU will lead to a smaller estimated rank for $A$.

\item[KRANK]  \ [out] Rank of $A$ estimated by the subroutine. Will be in the
range, $0\leq \text{KRANK}\leq \min (\text{M},\text{N}).$

\item[RNORM()]  \ [out] On return, RNORM(i) will contain the square root of
sum of squares of residuals for the $i^{th}$ right-side vector.

\item[WORK()]  \ [scratch] This array, of length at least N, is used
internally by the subroutine as working space.

\item[IP()]  \ [out] On return contains a record of column interchanges.
\end{description}

\subsubsection{Computing the Covariance Matrix}

Subroutine SCOV2 is designed to be used following SHFTI; but only in cases
in which KRANK determined by SHFTI is N.

\paragraph{Program Prototype, Single Precision}

\begin{description}
\item[INTEGER]  \ {\bf LDA, N, IP}($\geq $N){\bf , IERR}

\item[REAL]  \ {\bf A}(LDA, $\geq $N){\bf , VAR}
\end{description}

On entry the values in A(,), LDA, N, and IP() should be the same as on
return from a previous call to SHFTI. Also, assign a value to VAR.
$$
\fbox{{\bf CALL SCOV2(A, LDA, N, IP, VAR, IERR)}}
$$
Computed quantities are returned in A(,) and IERR.

\paragraph{Argument Definitions}

\begin{description}
\item[A(,)]  \ [inout] On entry contains an N$\times $N upper-triangular
matrix produced by the subroutine, SHFTI. On return contains the
upper-triangular part of the symmetric covariance matrix of the original
least-squares problem. Elements in A(,) below the diagonal will not be
referenced.

\item[LDA, N]  \ [in] Must be the same as in a previous call to SHFTI.

\item[IP()]  \ [in] Contains a record of column interchanges performed by a
previous call to SHFTI.

\item[VAR]  \ [in] Estimate of the variance of the data errors in the
original right-side vector, ${\bf b}$. To compute the covariance matrix for
the $i^{th}$ right-side vector of the original problem, the user's code can
compute VAR in terms of output quantities from SHFTI as
{\tt \begin{tabbing}
\hspace{.2in}\=DOF = M - N\\
\>STDDEV = RNORM($i$)/sqrt(DOF)\\
\>VAR = STDDEV**2
\end{tabbing}}
\item[IERR]  \ [out] Error flag. Zero indicates no error was detected. A
positive value indicates that A(IERR, IERR) was zero on entry. In this
latter case no result will be produced.
\end{description}

\subsubsection{Modifications for Double Precision}

Change the REAL type statements to DOUBLE PRECISION, and change the
subroutine names from SHFTI and SCOV2 to DHFTI and DCOV2, respectively.

\subsection{Examples and Remarks}

\subsubsection{A least-squares example}

Data for a sample linear least-squares problem were generated by computing
values of%
\begin{equation*}
y=0.5+0.25\ \sin (2\pi x)+0.125\ \exp (-x)
\end{equation*}
at eleven points, $x=0.0$, 0.1, ..., 1.0, and rounding the resulting $y$%
-values to 4~decimal places, thus introducing errors bounded in magnitude by
0.00005. The program, DRSHFTI, uses SHFTI to compute a least-squares fit to
this data using the model%
\begin{equation*}
c_1+c_2\sin (2\pi x)+c_3\exp (-x)
\end{equation*}
and uses SCOV2 to compute the covariance matrix for the computed
coefficients. Results are shown in the output file, ODSHFTI.

\subsubsection{Large problems}

If M $>>$ N and storage limitations make it awkward or impossible to
allocate M$\times $N locations for the array, A(,), one can use sequential
accumulation of the rows of data to produce a smaller matrix to which SHFTI
and SCOV2 can then be applied. See Chapter~4.4 for sequential accumulation.

\subsubsection{Underdetermined problems}

If M $<$ N, or, more generally, whenever Rank($A$) $<$ N, there will be
infinitely many vectors, ${\bf x}$, that achieve the same minimal residual
norm for the least-squares problem. For such cases SHFTI computes the
pseudoinverse solution, $i.e$. the solution vector, ${\bf x}$, of least
Euclidean norm.

\subsubsection{Computing the pseudoinverse matrix}

If one sets B(,) to be the M$\times$M identity matrix, the resulting
N$\times $M solution matrix, $X$, will be the pseudoinverse matrix of $A$.

\subsubsection{Reliability issues regarding KRANK}

KRANK must be understood as an estimate of the rank of $A$ that depends not
only on $A$, but on the user's setting of TAU and the particular algorithm
used in SHFTI. A small change in the value of TAU could result in a
different value being assigned to KRANK, which in turn could result in a
large change in the solution vector.

Although this subroutine will produce a solution whatever the value of
KRANK, the occurrence of KRANK $< \min (\text{M}$, N$)$ should be regarded
as exceptional. The user should investigate such instances as it could be
due to a programming error or a very ill-conditioned model. Singular Value
Analysis (see Chapter~4.3) can be useful in analyzing an ill-conditioned model.

\subsection{Functional Description}

This software is an adaptation to Fortran~77 of the subroutine, HFTI,
given and described in \cite{Lawson:1974:SLS}.  The name, HFTI, denotes
Householder Forward Transformation with column Interchanges.

To avoid nonessential complications, we shall describe the algorithm only
for the case of M $\geq $ N and NB = 1. A sequence of up to N Householder
orthogonal transformations is applied from the left, with column
interchanges, the total effect of which may be summarized by the equation%
\begin{equation*}
Q\left[ A:{\bf b}\right] \left[
\begin{array}{cc}
P & 0 \\
0 & 1
\end{array}
\right] =\left[
\begin{array}{cc}
R & {\bf g} \\ 0 & {\bf h}
\end{array}
\right]
\end{equation*}
where Q is the M$\times $M product of the Householder matrices, $P$ is an N$%
\times $N permutation matrix accounting for the column interchanges, $R$ is an
N$\times $N upper-triangular matrix with diagonal elements in order of
decreasing magnitudes, ${\bf g}$ is an N-vector, and ${\bf h}$ is an
(M $-$ N)-vector.

If all diagonal elements of $R$ exceed TAU in magnitude, the solution, ${\bf x}
$, is computed by solving $R{\bf x}={\bf g}$. The subroutine sets KRANK $=$
N and RNORM(1)\ $=\Vert {\bf h}\Vert .$

Alternatively, if the diagonal elements of $R$ beyond position $i$ are less
than TAU, the subroutine sets KRANK $=i$. Let $[R_1:{\bf g}_1]$ denote the
first KRANK rows of $[R:{\bf g}]$, and let ${\bf g}_2$ denote the last
N $-$ KRANK components of ${\bf g}$. The subroutine applies up to KRANK
Householder transformations to $R_1$ from the right, effecting the
transformation
\begin{equation*}
R_1K=[W:0]
\end{equation*}
where $K$ is the N$\times $N product of Householder transformations and $W$ is a
KRANK$\times $KRANK non-singular upper-triangular matrix. A KRANK-vector, $%
{\bf y}_1$, is computed by solving $W{\bf y}_1={\bf g}_1$. An N-vector, $%
{\bf y}$, is formed by appending zeros to the end of ${\bf y}_1$, and the
minimal length solution vector, ${\bf x}$, is computed as ${\bf x}=\text{PK}%
{\bf y}$. RNORM(1) is computed as the Euclidean norm of the (M $-$ KRANK)-vector
formed by concatenating ${\bf g}_2$ and ${\bf h}.$

Subroutine, SCOV2, is intended for use only when Rank($A$) = N.
Algorithm, COV, from \cite{Lawson:1974:SLS} is used.  The covariance
matrix is traditionally defined as $ C=\sigma ^2(A^tA)^{-1}$, where
$\sigma ^2$ is the variance of the data error.  Using the triangular
matrix, $R$, and the (orthogonal) permutation matrix, $P$, defined above,
one can also write $C=\sigma ^2(PR^tRP^t)^{-1}=\sigma ^2PR^{-1}R^{-t}P^t$,
where $R^{-t}$ denotes the transpose of $R^{-1}$.  Thus, SCOV2 computes%
\begin{align*}
E&=R^{-1} \\ F&=EE^t\\ C&=\sigma ^2PFP^t
\end{align*}

\bibliography{math77}
\bibliographystyle{math77}

\subsection{Error Procedures and Restrictions}

In SHFTI (or DHFTI) an error message will be issued and an immediate return
will be made, setting KRANK = 0, if any of the following conditions are
noted:%
\begin{equation*}
\text{M}<1,\text{ N}<0,\text{ LDA}<\text{M, or LDB}<\max (\text{M, N})
\end{equation*}
Most commonly, on return KRANK will have the value $\min (\text{M}$, N$)$.
A smaller value of KRANK may be valid, but unless the user has reason to
expect this possibility it is likely to be due to a usage error.

In SCOV2 (or DCOV2) the N diagonal elements of A(,) must be nonzero on
entry. If so, IERR is set to zero. If not, IERR is set to the index of the
first zero element, an error message will be issued, and a return will be
made with the computation being incomplete.

\subsection{Supporting Information}

The source language is ANSI Fortran~77.

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
DCOV2 & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
DCOV2, DDOT, DSWAP, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
DHFTI & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
AMACH, DAXPY, DDOT, DHFTI, DHTCC, DHTGEN, DNRM2, ERFIN, ERMOR, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
SCOV2 & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
ERFIN, ERMSG, IERM1, IERV1, SCOV2, SDOT, SSWAP\rule[-5pt]{0pt}{8pt}}\\
SHFTI & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
AMACH, ERFIN, ERMOR, ERMSG, IERM1, IERV1, SAXPY, SDOT, SHFTI, SHTCC, SHTGEN, SNRM2}\\\end{tabular}

Adapted to Fortran~77 from \cite{Lawson:1974:SLS} by C.  L.  Lawson and S.
Y.  Chiu, JPL, May~1986, June~1987.

\begcode
\bigskip
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRSHFTI}\vspace{10pt}
\lstinputlisting{\codeloc{shfti}}

\vspace{30pt}\centerline{\bf \large ODSHFTI}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{shfti}}
\end{document}


\section{Quaternion integration\label{quatProofs}}

Let us first consider a geometric view on quaternions, taken from~\cite{Shoemake:85}. Treat
the four components of a quaternion as Cartesian coordinates of a four-dimensional vector
space. The set of unit quaternions is then the surface of a unit hypersphere (also called a
\emph{glome}~\cite{MathWorld:4D}) in this vector space. Each point on this hypersphere
corresponds to a particular rotation. It also turns out that each pair of
opposite points on this sphere represent exactly the same rotation; hence all possible
rotations are contained in one hemisphere, no matter where the sphere is cut in half.

\subsection{Conservation of magnitude\label{quatIntegrationMagnitude}}
Define the quaternion dot product, in analogy to the 4D vector dot product, to be
\begin{equation}
\q{p}\cdot\q{q} =
    (p_w + p_x\qi + p_y\qj + p_z\qk)\cdot (q_w + q_x\qi + q_y\qj + q_z\qk) =
    p_w q_w + p_x q_x + p_y q_y + p_z q_z
\end{equation}
The dot product is commutative, contrary to the quaternion juxtaposition product.

The instantaneous rate of change is given~\cite{BaraffWitkin:97,Eberly:04,Saunders:PhD} to be
\begin{eqnarray*}
\dot{\q{q}} & = & \frac{1}{2}\tilde{\ve{\omega}}\q{q} =
    \frac{1}{2}(\omega_1\qi + \omega_2\qj + \omega_3\qk)
    (q_w + q_x\qi + q_y\qj + q_z\qk) \\*
& = & \frac{1}{2} ( - \omega_1 q_x - \omega_2 q_y - \omega_3 q_z ) +
    \frac{\qi}{2} ( \omega_1 q_w + \omega_2 q_z - \omega_3 q_y ) + \\*
&&  \frac{\qj}{2} (-\omega_1 q_z + \omega_2 q_w + \omega_3 q_x ) +
    \frac{\qk}{2} ( \omega_1 q_y - \omega_2 q_x + \omega_3 q_w )
\end{eqnarray*}

We now treat \q{q} and $\dot{\q{q}}$ as 4D vectors and calculate
their dot product:
\begin{eqnarray*}
\q{q}\cdot\dot{\q{q}} & = & \frac{1}{2} (
    - q_w \omega_1 q_x - q_w \omega_2 q_y - q_w \omega_3 q_z
    + q_x \omega_1 q_w + q_x \omega_2 q_z - q_x \omega_3 q_y \\*
&&  - q_y \omega_1 q_z + q_y \omega_2 q_w + q_y \omega_3 q_x
    + q_z \omega_1 q_y - q_z \omega_2 q_x + q_z \omega_3 q_w ) \\*
& = & 0.
\end{eqnarray*}
The rate of change is orthogonal to \q{q}, and therefore it is always
a tangent to the sphere, touching it at the point corresponding to \q{q}. The set of all possible
values of $\dot{\q{q}}$ is thus a hyperplane (a three-dimensional subspace) tangential to the
sphere at the point \q{q} in 4D space.

We can determine the magnitude $\norm{\dot{\q{q}}}$ from the sum of squares of the
components given above and find it to be
$\norm{\dot{\q{q}}} = \frac{1}{2}\norm{\ve{\omega}}\,\norm{\q{q}}$. Since we always
require $\q{q}$ to be a unit quaternion, we can reduce this to
\begin{equation}
\label{quatRateOfChangeMagnitude}
\norm{\dot{\q{q}}} = \frac{1}{2}\norm{\ve{\omega}}.
\end{equation}

Now let us determine what happens if we calculate $\q{q} + h\dot{\q{q}}$ for some finite $h$.
Note that this operation is required by all common numerical solvers of differential equations.
Consider the magnitude of the result:
\begin{eqnarray*}
\norm{\q{q} + h\dot{\q{q}}}^2 & = & (\q{q} + h\dot{\q{q}})\cdot(\q{q} + h\dot{\q{q}}) \\
&=& \q{q}\cdot\q{q} + 2h\q{q}\cdot\dot{\q{q}} + h^2\dot{\q{q}}\cdot\dot{\q{q}} \\
&=& 1 + 0 + \frac{h^2}{4}\norm{\ve{\omega}}^2 \\
&>& 1 \quad\quad\mbox{whenever}\quad \norm{\ve{\omega}} > 0.
\end{eqnarray*}

Hence, if the body in question is rotating, it is not possible for a standard numerical ODE solver
to preserve a quaternion's property of unit magnitude.

\subsection{Normalization is not enough\label{quatNormalization}}

One should think that given the derivative of a quaternion \q{q} (equation~\ref{quatRateOfChange},
page~\pageref{quatRateOfChange}), one can find $\q{q}(t + h)$ for some time step $h$ within
the accuracy of the ODE solver employed ($O(h^5)$ error for fourth-order Runge-Kutta).
Unfortunately this is not the case. This shall be demonstrated using Euler's method; it should,
however, be pointed out that more sophisticated methods like RK4 are also affected. Consider the
value of \q{q} at the next time step, $\q{q}(t + h) = \q{q}(t) + h \dot{\q{q}}(t)$. For any
non-zero $h$ and $\dot{\q{q}}$ this point will always lie outside the unit quaternion sphere due
to the orthogonality of \q{q} and $\dot{\q{q}}$. This is usually compensated by renormalizing
the quaternion after the ODE solving step. Geometrically, this renormalization can be understood
as drawing a straight line through the origin and the point $\q{q}(t) + h \dot{\q{q}}(t)$,
intersecting this line with the unit sphere and replacing $\q{q}(t + h)$ by this point of
intersection (see figure~\ref{quatNormalizationFigure}).

\begin{figure}
\psfrag{frag:q}{\q{q}}
\psfrag{frag:qdot}{$\dot{\q{q}}$}
\psfrag{frag:qplusqdot}{$\q{q} + h\dot{\q{q}}$}
\centerline{\includegraphics[width=6cm]{figures/quaternion1}}
\caption{Normalizing a quaternion after performing an ODE solving step.
    \label{quatNormalizationFigure}}
\end{figure}

Following the tangent to the sphere is a reasonable approximation to following its curve if the
magnitude of $h \dot{\q{q}}(t)$ is small compared to the curvature of the sphere.
For large time steps or large magnitudes of \ve{\omega}, however, this gets increasingly
erroneous. Consider the limiting case, a body rotating infinitely fast
($\norm{\ve{\omega}} \rightarrow \infty$): after renormalisation, $\q{q}$ will have moved merely
a quarter of the way around the unit sphere, which equates to concatenating the rotation of
quaternion \q{q} with some rotation by $180^\circ$. This is a strictly finite amount of rotation
per time step, while it would actually have been correct to perform an infinite number of
revolutions around the quaternion sphere.

If a polynomial approximation method like RK4 had been used instead of Euler's method, a parametric
polynomial space curve would have been fitted to the surface of the sphere instead of the straight
line. Note however that the Taylor series of the $\sin$ and $\cos$ functions are non-terminating,
and that it is therefore not possible for a finite polynomial curve to lie exactly in the surface
of a sphere. These ODE solvers will therefore suffer the same problems, albeit less pronounced.

\subsection{Corrected quaternion integration\label{quatIntegrationDerivation}}

Assume that the body we are simulating is rotating at a constant angular velocity.
(This assumption is later weakened by the use of a more sophisticated ODE solver,
but for now we will stick with Euler's method.) Furthermore assume without loss of
generality that the body is rotating clockwise about its $x$ axis, which corresponds
to the world's $x$ axis, and that at time $t=0$ the body's frame and the world frame
coincide. Then the orientation of the body (the quaternion describing the linear
transformation from the body's frame of reference to the world frame) is given as a
function of time by
\begin{equation}
\label{quatDerivationExact}
\q{q}(t) = \cos\left(\frac{\norm{\ve{\omega}}t}{2}\right) +
    \sin\left(\frac{\norm{\ve{\omega}}t}{2}\right)\qi
\end{equation}
(cf.\ equation~\ref{quatRotation}, figure~\ref{quatIntFig1}) and its angular velocity is
\begin{equation}
\ve{\omega} = (\omega_1, \omega_2, \omega_3)^T = (\norm{\ve{\omega}}, 0, 0)^T
\end{equation}
for some arbitrary $\norm{\ve{\omega}}$, measured in radians per unit time.

\begin{figure}
\psfrag{frag:omegat}{$\frac{\norm{\ve{\omega}} t}{2}$}
\psfrag{frag:qdotoft}{$\dot{\q{q}}(t)$}
\psfrag{frag:real}{Re}
\psfrag{frag:iimag}{$\mathsf{i}$-Im}
\centerline{\includegraphics[width=6cm]{figures/quaternion2}}
\caption{Assumed situation for the derivation in
section~\ref{quatIntegrationDerivation}.\label{quatIntFig1}}
\end{figure}

Now assume w.l.o.g.\ that we take a time step from $t = 0$ to $t = h$.
Then we require that the result returned by Euler's method for $\q{q}(h)$
after renormalization be equal to its exact value in equation~\ref{quatDerivationExact}:
\begin{equation}
\label{quatDerivationSetup}
\cos\left(\frac{\norm{\ve{\omega}} h}{2}\right) +
    \sin\left(\frac{\norm{\ve{\omega}} h}{2}\right)\qi =
    \frac{\q{q}(0) + h \dot{\q{q}}(0)}
        {\norm{\q{q}(0) + h \dot{\q{q}}(0)}}
\end{equation}

We know from examining the 4D geometry that the value assigned to $\dot{\q{q}}$
in equation~\ref{quatRateOfChange} has the correct direction and merely needs to be
corrected in magnitude. In other words, we are searching for a scalar function
$f(h, \norm{\ve{\omega}})$ which will allow $\dot{\q{q}}$ to satisfy
equation~\ref{quatDerivationSetup}:
\begin{equation}
\dot{\q{q}}_h(t) = f\tilde{\ve{\omega}}(t)\q{q}(t)
\end{equation}

Observe that under the above assumptions $\q{q}(0) = 1$, and thus
$\dot{\q{q}}_h(0) = f \norm{\ve{\omega}} \qi$. Substituting this
into equation~\ref{quatDerivationSetup} and considering only the real part:
\begin{eqnarray*}
&& \cos\left(\frac{\norm{\ve{\omega}} h}{2}\right) =
    \left[1 + \left( f \norm{\ve{\omega}} h \right)^2 \right]^{-\frac{1}{2}} \\
&\Leftrightarrow&
    \left( f \norm{\ve{\omega}} h \right)^2 =
    \frac{1}{\cos^2\left(\frac{\norm{\ve{\omega}} h}{2}\right)} - 1 \\
&\Leftrightarrow&
    f(h, \norm{\ve{\omega}}) =
    \frac{1}{\norm{\ve{\omega}} h} \sqrt{\frac{
        1 - \cos^2\left(\frac{\norm{\ve{\omega}} h}{2}\right)}{
        \cos^2\left(\frac{\norm{\ve{\omega}} h}{2}\right)}} =
    \frac{1}{\norm{\ve{\omega}} h}
        \tan\left(\frac{\norm{\ve{\omega}} h}{2}\right)
\end{eqnarray*}

To check, we substitute this result into the $\qi$-imaginary part of
equation~\ref{quatDerivationSetup}:
\begin{eqnarray*}
\sin\left(\frac{\norm{\ve{\omega}} h}{2}\right) & = &
    \tan\left(\frac{\norm{\ve{\omega}} h}{2}\right)
    \left[ 1 + \tan^2\left(\frac{\norm{\ve{\omega}} h}{2}\right)
    \right]^{-\frac{1}{2}} \\
&=& \tan\left(\frac{\norm{\ve{\omega}} h}{2}\right)
    \left[ \frac{\cos^2\left(\frac{\norm{\ve{\omega}} h}{2}\right) +
    \sin^2\left(\frac{\norm{\ve{\omega}} h}{2}\right) }{
    \cos^2\left(\frac{\norm{\ve{\omega}} h}{2}\right) }
    \right]^{-\frac{1}{2}} \\
&=& \tan\left(\frac{\norm{\ve{\omega}} h}{2}\right)
    \cos\left(\frac{\norm{\ve{\omega}} h}{2}\right) \\
&=& \sin\left(\frac{\norm{\ve{\omega}} h}{2}\right)
\end{eqnarray*}

Thus we establish the validity of this expression for $f$. Observe that by using
L'Hospital's rule, we can find value of $f$ for an infinitesimally small time step:
$$
\lim_{h \to 0} f = \lim_{h \to 0} \frac{ \frac{\norm{\ve{\omega}}}{2}
    \cos^{-2}\left(\frac{\norm{\ve{\omega}} h}{2}\right) }{ \norm{\ve{\omega}} } =
    \frac{1}{2}
$$
i.e.\ we obtain the original equation~\ref{quatRateOfChange} for the
instantaneous rate of change.

Now let $\Delta \q{q} = h \dot{\q{q}} =
    \frac{h}{2} \tilde{\ve{\omega}} \q{q}$.
From equation~\ref{quatRateOfChangeMagnitude} we find that
$\norm{\Delta\q{q}} = \frac{\norm{\ve{\omega}} h}{2}$.
Hence we can simplify the expression for the quaternion correcting factor by expressing it
in terms of $\Delta \q{q}$ as follows:
$$
h f \tilde{\ve{\omega}}\q{q} = \frac{h}{\norm{\ve{\omega}} h}
    \tan\left(\frac{\norm{\ve{\omega}} h}{2}\right) \tilde{\ve{\omega}} \q{q} =
    \tan\left(\norm{\Delta\q{q}}\right) \frac{\Delta\q{q}}{\norm{\Delta\q{q}}}
$$

This expression now has a clear geometric interpretation with respect to the 4D geometry
(see figure~\ref{quatIntFig2}):
$\norm{\Delta\q{q}}$ is measured in radians, and it corresponds to the \emph{correct}
angle between the old and the new vector $\q{q}$. Since $\q{q}$ and
$\dot{\q{q}}$ are orthogonal, we have a right-angled triangle between the origin,
the old and the new points $\q{q}$, and hence we can use the $\tan$ function to
evaluate the required length of the side in direction $\Delta\q{q}$.

\begin{figure}
\psfrag{frag:real}{Re}
\psfrag{frag:iimag}{$\mathsf{i}$-Im}
\psfrag{frag:q}{\q{q}}
\psfrag{frag:deltaq}{$\Delta\q{q}$}
\psfrag{frag:quergs1}{$\q{q}+\tan(\norm{\Delta\q{q}})\frac{\Delta\q{q}}{\norm{\Delta\q{q}}}$}
\psfrag{frag:quergs2}{$\mathrm{Quergs}(\q{q},\,\Delta\q{q})$}
\centerline{\includegraphics[width=7.7cm]{figures/quaternion3}}
\caption{Illustration of the operation of Quergs.\label{quatIntFig2}}
\end{figure}

Finally we can combine this correction and the subsequent quaternion normalisation into
a single function, which I call Quergs (for \emph{Qu}at\emph{er}nion inte\emph{g}ration
\emph{s}tep)\footnote{This naming follows the spirit of Shoemake~\cite{Shoemake:85}, whose
``Slerp'' function is an `acronym' of \emph{S}pherical \emph{l}inear int\emph{erp}olation.}:
\begin{eqnarray*}
\q{q}(t+h) = \mathrm{Quergs}(\q{q}(t), \Delta\q{q}) &=&
    \frac{\q{q}(t) + \tan\left(\norm{\Delta\q{q}}\right)
        \frac{\Delta\q{q}}{\norm{\Delta\q{q}}}}{
    \norm{\q{q}(t) + \tan\left(\norm{\Delta\q{q}}\right)
        \frac{\Delta\q{q}}{\norm{\Delta\q{q}}}}} \\
&=& \frac{\q{q}(t) + \tan\left(\norm{\Delta\q{q}}\right)
        \frac{\Delta\q{q}}{\norm{\Delta\q{q}}}}{
    \sqrt{1 + \tan^2\left(\norm{\Delta\q{q}}\right)}} \\
&=& \left[\q{q}(t) + \tan\left(\norm{\Delta\q{q}}\right)
        \frac{\Delta\q{q}}{\norm{\Delta\q{q}}}\right]
    \cos\left(\norm{\Delta\q{q}}\right)
\end{eqnarray*}
The last expression is simplest (and again allows geometric interpretation), but probably
the first of the three expressions is more useful for numerical evaluation, since it involves
only one trigonometric function and minimizes numerical errors.

When implementing this formula, care must be taken around the discontinuities of the $\tan$
function, where numerical instability may occur. These discontinuities are reached whenever a
body performs an odd multiple of half revolutions during a single time step.


\documentclass[11pt, a4paper]{article}

\usepackage{tikz}
\usetikzlibrary{shapes,arrows}
\usepackage{amsmath}
\usepackage{placeins}
\usepackage{amssymb}

\begin{document}

\title{LINEAR REGRESSION}
\date{}
\maketitle

Linear regression models assume a linear relationship between the inputs $X_1, X_2,\ ...,\ X_p$ and the output $Y$. These models are simple and often provide an insight into the effect of input variables on the output variable.
Linear models can be expanded to transformations of the input variables, thus making them widely applicable.

\section{Single Variable Regression}

Given training data of the form $(x_1, y_1), (x_2, y_2),\ ...,\ (x_N, y_N)$; the idea is to come up with the best estimate $\hat{y}$ such that,

\begin{align*}
	\hat{y} = \beta_0 + \beta_1x
\end{align*}

This scheme makes an estimation error at each point,
\begin{align*}
	\epsilon_i & = y_i - \hat{y}_i            \\
	           & = y_i - (\beta_0 + \beta_1x)
\end{align*}

Generally, sum of squares of all the $N$ errors is minimized to define the best fit. Thus, the objective is to choose $\beta_0$ and $\beta_1$ such that this Residual Square Sum (RSS) is as small as possible. The errors are visualized below,

\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}

		\draw[-latex] (0,0) -- (6,0) node[right]{x};
		\draw[-latex] (0,0) -- (0,5) node[left]{y};

		\draw (0, 1) -- (5, 4);

		\draw (5, 4) node [above right] {$y = \beta_0 + \beta_1x$};
		\draw (0, 1) node [left] {$\beta_0$};

		\foreach \Point in {(1,3), (2, 3), (4, 4), (3, 2), (1.5, 1)}{
			\node at \Point {\textbullet};
		};

		\draw (1, 3) node [above] {$x_1, y_1$};
		\draw (1.5, 1) node [below] {$x_2, y_2$};
		\draw (2, 3) node [above right] {$x_3, y_3$};
		\draw (3, 2) node [below] {$x_4, y_4$};
		\draw (4, 4) node [above] {$x_5, y_5$};

		\draw [thick] (1, 3) -- (1, 1.6);
		\draw [thick] (1.5, 1) -- (1.5, 1.9);
		\draw [thick] (2, 3) -- (2, 2.2);
		\draw [thick] (3, 2) -- (3, 2.8);
		\draw [thick] (4, 4) -- (4, 3.4);
	\end{tikzpicture}
\end{figure}

Note that as line moves away from the data points the $RSS$ gets bigger since each error becomes bigger. In fact there is no upper bound on $RSS$, only a lower bound. Hence to get the minimum $RSS$, equating partial differentials w.r.t. $\beta_0$ and $\beta_1$ to zero is sufficient without worrying if the extrema is maxima or minima, it is always guaranteed to be a minima.

\begin{align*}
	RSS(\beta_0, \beta_1)                 & = \sum\limits_{i = 1}^N (y_i - \beta_0 - \beta_1x_i)^2    \\
	\frac{\partial RSS}{\partial \beta_0} & = -2\sum\limits_{i = 1}^N (y_i - \beta_0 - \beta_1x_i)    \\
	\frac{\partial RSS}{\partial \beta_1} & = -2\sum\limits_{i = 1}^N (y_i - \beta_0 - \beta_1x_i)x_i \\
\end{align*}

Equating both to zero,

\begin{align*}
	\beta_0N + \beta_1\sum x_i = \sum y_i             \\
	\beta_0\sum x_i + \beta_1\sum x_i^2 = \sum x_iy_i
\end{align*}

This can be written in matrix form as follows,

\begin{align*}
	\begin{pmatrix} N & \sum x_i \\ \sum x_i & \sum x_i^2 \end{pmatrix} \begin{pmatrix} \beta_0 \\ \beta_1 \end{pmatrix} = \begin{pmatrix}
	\sum y_i \\ \sum x_iy_i \end{pmatrix}
\end{align*}

Multiplying by inverse of the $2\times2$ matrix on both sides,

\begin{align*}
	\begin{pmatrix} \beta_0 &   &   \\ \beta_1 \end{pmatrix} &= \begin{pmatrix} N & \sum x_i \\ \sum x_i & \sum x_i^2 \end{pmatrix}^{-1} \begin{pmatrix}
	\sum y_i \\ \sum x_iy_i \end{pmatrix} \\
	& = \frac{1}{N\sum x_i^2 - (\sum x_i)^2} \begin{pmatrix} \sum x_i^2 & -\sum x_i \\ -\sum x_i & N \end{pmatrix} \begin{pmatrix}
	\sum y_i \\ \sum x_iy_i \end{pmatrix} \\
	&= \frac{1}{N\sum x_i^2 - (\sum x_i)^2} \begin{pmatrix}
	\sum x_i^2 \sum y_i -\sum x_iy_i  \sum x_i  \\ N \sum x_i y_i - \sum x_i \sum y_i \end{pmatrix} \\
\end{align*}

Dividing both numerator and denominator by $N^2$, $\beta_0$ and $\beta_1$ can be represented in terms of averages as follows,

\begin{align*}
	\beta_0 & = \frac{Avg(x^2)Avg(y) - Avg(xy)Avg(x)}{Avg(x^2) - [Avg(x)]^2}                    \\
	\beta_1 & = \frac{Avg(xy) - Avg(x)Avg(y)}{Avg(x^2) - [Avg(x)]^2} = \frac{Cov(x, y)}{Var(x)} \\
\end{align*}

After the calculation of $\beta_0$ and $\beta_1$, the goodness of fit is measured by correlation,

\begin{align*}
	r = \frac{Cov(x, y)}{\sqrt{Var(x)Var(y)}}
\end{align*}

Correlation always resides in the interval $[-1, 1]$. It is 1 or -1 if all the data points $(x_i, y_i)$ lie on a line.

\subsection{Input Transformations}

\begin{itemize}
	\item Relationship between $x$ and $y$ is not linear if $y = x^n$. Taking $log$ on both sides, $log(y) = nlog(x)$. Clearly relationship between $log(x)$ and $log(y)$ is linear.
	\item Relationship between $x$ and $y$ is not linear if $y = a^x$. Taking $log$ on both sides, $log(y) = xlog(a)$. Clearly, relationship between $log(y)$ and $x$ is linear.
	\item Due to scenarios like this, the inputs are sometimes transformed appropriately before being fed into a linear regression model.
\end{itemize}

\section{Multiple Variable Regression}

In the general case, the equation becomes

\begin{align*}
	\hat{y} = \beta_0 + \sum\limits_{j = 1}^p \beta_j x_j
\end{align*}

Let $\mathbf{X}$ be $N\times(p+1)$ matrix where the first column is all ones and the $j^{th}$ column represents the $N$ values of the input variable $x_j$. Similarly, let $\mathbf{Y}$ be a vector of the $N$ values of the output variable $y$. For all the $N$ given observations, it can be written,

\begin{align*}
	\mathbf{y} = \mathbf{X}\boldsymbol{\beta}
\end{align*}

The error vector becomes,

\begin{align*}
	\mathbf{\epsilon} = \mathbf{y} - \mathbf{X}\boldsymbol{\beta}
\end{align*}

The residual sum of squares can be represented as,

\begin{align*}
	RSS(\boldsymbol{\beta}) & = \mathbf{\epsilon}^T\mathbf{\epsilon}                                                                                \\
	                        & = (\mathbf{y} - \mathbf{X}\boldsymbol{\beta})^T(\mathbf{y} - \mathbf{X}\boldsymbol{\beta})                                                                                \\
	                        & = (\mathbf{y}^T - \boldsymbol{\beta}^T\mathbf{X}^T)(\mathbf{y} - \mathbf{X}\boldsymbol{\beta})                                                                                \\
	                        & = \mathbf{y}^T\mathbf{y} - \boldsymbol{\beta}^T\mathbf{X}^T\mathbf{y} - \mathbf{y}^T\mathbf{X}\boldsymbol{\beta} + \boldsymbol{\beta}^T\mathbf{X}^T\mathbf{X}\boldsymbol{\beta}
\end{align*}

Taking the derivative w.r.t. $\boldsymbol{\beta}$

\begin{align*}
	\frac{\partial RSS(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}} & =
	\frac{\partial}{\partial \boldsymbol{\beta}}(\mathbf{y}^T\mathbf{y}) - \frac{\partial}{\partial \boldsymbol{\beta}}(\boldsymbol{\beta}^T\mathbf{X}^T\mathbf{y}) - \frac{\partial}{\partial \boldsymbol{\beta}}(\mathbf{y}^T\mathbf{X}\boldsymbol{\beta}) + \frac{\partial}{\partial \boldsymbol{\beta}}(\boldsymbol{\beta}^T\mathbf{X}^T\mathbf{X}\boldsymbol{\beta}) \\
	                                                                     & = 0 - \mathbf{X}^T\mathbf{y} - \mathbf{X}^T\mathbf{y} + 2\mathbf{X}^T\mathbf{X}\boldsymbol{\beta} \\
	                                                                     & = -2\mathbf{X}^T\mathbf{y} + 2\mathbf{X}^T\mathbf{X}\boldsymbol{\beta}
\end{align*}

Equating the derivative to zero for minimization,

\begin{align*}
	\mathbf{X}^T\mathbf{X}\boldsymbol{\hat{\beta}} & = \mathbf{X}^T\mathbf{y}                              \\
	\boldsymbol{\hat{\beta}}                       & = (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{y}
\end{align*}

These estimators are also known as OLS(Ordinary Least Square) estimators.

\section{Bias and Variance}
\subsection{Definitions}
Note that the expectation of a vector $\mathbf{V}$ is defined as,

\begin{align*}
	\mathbb{E}[\mathbf{V}] = \begin{pmatrix} \mathbb{E}(v_1) \\ \mathbb{E}(v_2) \\ . \\ . \\ .  \\ \mathbb{E}(v_n) \end{pmatrix}
\end{align*}

The variance of a vector is a little trickier,
\begin{align*}
	Var[\mathbf{V}] = \begin{pmatrix} Var(v_1) & Cov(v_1, v_2) & . & . & . & Cov(v_1, v_n) \\ Cov(v_2, v_1) & . & . & . & . & . \\ . & . & . & . & . & . \\ . & . & . & . & . & . \\ . & . & . & . & . & . \\ Cov(v_n, v_1) & . & . & . & . & Var(v_n, v_n) \\ \end{pmatrix}
\end{align*}

\subsection{Bias}
Let the true population process of $\mathbf{y}$ be

\begin{align*}
	\mathbf{y} = \mathbf{X}\boldsymbol{\beta} + \mathbf{u}
\end{align*}

where $\boldsymbol{\beta}$ is the true population variable and $\mathbf{u}$ is the population error. Least squares estimate as derived above is,

\begin{align*}
	\boldsymbol{\hat{\beta}} & = (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{y}                                   \\
	                         & = (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T (\mathbf{X}\boldsymbol{\beta} + \mathbf{u}) \\
	                         & = \boldsymbol{\beta} + (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{u}
\end{align*}

Taking expectation on both sides,

\begin{align*}
	\mathbb{E}[\hat{\boldsymbol{\beta}}] & = \mathbb{E}[\boldsymbol{\beta}] + \mathbb{E}[(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{u}] \\
	                                     & = \boldsymbol{\beta} + \mathbb{E}[(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{u}]
\end{align*}

Assuming $\mathbf{u}$ and $\mathbf{X}$ are independent, the equation can be simplified to,

\begin{align*}
	\mathbb{E}[\hat{\boldsymbol{\beta}}] & = \boldsymbol{\beta} + (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbb{E}[\mathbf{u}] \\
\end{align*}

Assuming $\mathbb{E}[\mathbf{u}] = 0$, the equation gets further simplified,

\begin{align*}
	\mathbb{E}[\hat{\boldsymbol{\beta}}] & = \boldsymbol{\beta}
\end{align*}

Under the conditional independence zero mean assumption [1], the least squares estimator is unbiased and on average the estimate is equivalent to the true population parameter.

\subsection{Variance}

Once again start with the least squares estimate equation,

\begin{align*}
	\boldsymbol{\hat{\beta}} & = (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{y} \\
\end{align*}

Taking variance on both sides,

\begin{align*}
	Var[\boldsymbol{\hat{\beta}}] & = Var[(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{y}]
\end{align*}

Since the variance of the product of a non-stochastic matrix $\mathbf{A}$ and a vector $\mathbf{V}$ is $\mathbf{A}Var(\mathbf{V})\mathbf{A}^T$,

\begin{align*}
	Var[\boldsymbol{\hat{\beta}}] & = (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\ Var[\mathbf{y}]\ ((\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T)^T \\
	                              & = (\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\ Var[\mathbf{y}]\  \mathbf{X}(\mathbf{X}^T\mathbf{X})^{-1}
\end{align*}

Assume that $Var[\mathbf{v}] = \sigma^2\mathbf{I}$. This is same as saying that,

\begin{itemize}
	\item All $y_i's$ are  homoscedastic i.e. $Var(y_i) = \sigma^2 \ \forall\ i\ [2]$.
	\item Distinct $y_i's$ are uncorrelated i.e. $Cov(y_i, y_j) = 0\  \forall\ i \neq j\ [3]$.
\end{itemize}

\begin{align*}
	Var[\boldsymbol{\hat{\beta}}] & = \sigma^2 (\mathbf{X}^T\mathbf{X})^{-1}
\end{align*}

\section{Gauss-Markov Theorem}

This theorem states that under above assumptions [1], [2] and [3], the OLS estimators are BLUE (Best Linear Unbiased Estimators). This means that there are no other unbiased linear estimators which have a lower variance of $\boldsymbol{\beta}$ than OLS.

\section{Ridge Regression}

A penalty proportional to sum of squares of all $\beta$ coefficients is added to the residual sum of squares in ridge regression to make sure that coefficients are reasonably bounded and don't grow arbitarily large.

\begin{align*}
	\mathbf{Error}(\boldsymbol{\beta}) & = \boldsymbol{\epsilon}^T\boldsymbol{\epsilon} + \lambda\boldsymbol{\beta}^T\boldsymbol{\beta}
\end{align*}

The solution to the above ridge regression problem is given below,

\begin{align*}
	\boldsymbol{\hat{\beta}} & = (\mathbf{X}^T\mathbf{X} + \lambda\mathbf{I})^{-1}\mathbf{X}^T\mathbf{y}
\end{align*}

As $\lambda$ tends to zero, the ridge solution tends to the OLS solution and as $\lambda$ tends to infinity, the ridge solution tends to zero because of the infinite penalty for non-zero coefficients.

If $\mathbf{X}^T\mathbf{X}$ is non-invertible, OLS has no unique solution, but this problem does not occur with ridge regression as the matrix $\mathbf{X}^T\mathbf{X} + \lambda\mathbf{I}$ is always guaranteed to be invertible, therfore always ensuring unique solution.

The fundamental idea with ridge regression is to incur some bias in order to reduce variance which may often be desirable in practical situations.

Now, a short excursion on Lagrange multipliers before the  introduction to Lasso regression.

\section{Lagrange Multipliers}

Suppose a function $f$ is to be minimized subject to the constraint that the value of another function $g$ on the same inputs is $t$. For illustration, consider two inputs $x$ and $y$ with $f(x, y) = x^2 + y^2$, $g(x, y) = 4x + 3y$ and $t = 25$. Hence the problem is posed as follows:

\begin{align*}
	minimize\ \ &x^2 + y^2 &              \\
	subject\ to\ \ &4x + 3y = 25 &              \\
\end{align*}

$f(x, y) = x^2 + y^2$ has a circular countour centered at origin. All points on this circle have the same value of $f$. On the ohter hand, the curve $4x + 3y = 25$ is a line offset from the origin.

One can imagine an expanding circle at origin which finally hits the line. The point where the circle hits the line is the solution as it is the minimum value of $f$ satisfying the constraint $g=25$.

At this intersection of the circle and the line, their gradients must be aligned to the same direction. More precisely, for some scalar $\lambda$,

\begin{align*}
	\begin{pmatrix}\frac{\partial f}{\partial x} \\
	                                             \\  \frac{\partial f}{\partial y} \end{pmatrix} = \lambda  \begin{pmatrix}\frac{\partial g}{\partial x} \\ \\ \frac{\partial g}{\partial y} \end{pmatrix}
\end{align*}

Note that if one were to minimize $f - \lambda g$, taking the partials and equating them to zero would have have led to the same equation above. Hence it makes no difference to pose the same problem differently as follows,

\begin{align*}
	minimize\ \ &f - \lambda g &                            \\
	            & for\ some\ scalar\ \lambda \\
\end{align*}

The function $f - \lambda g$ is called the Langrangian function and $\lambda$ the Langragial multiplier.

\section{Lasso Regression}

Instead of ridge regression's penalty which is proportional to the sum of squares of the $\boldsymbol\beta$s, Lasso penalizes the sum of moduli of the $\boldsymbol\beta$s.

\begin{align*}
	\mathbf{Error}(\boldsymbol{\beta}) & = \boldsymbol{\epsilon}^T\boldsymbol{\epsilon} + \lambda \sum |\boldsymbol\beta_i|
\end{align*}

From the discussion of Lagrange mutlipliers, the same problem can be posed as,

\begin{align*}
	minimize\ \ &\boldsymbol{\epsilon}^T\boldsymbol{\epsilon} &                                               \\
	            & subject\ to\ \sum |\boldsymbol\beta_i| \leq t
\end{align*}

The graphs of $\sum |\boldsymbol\beta_i| \leq t$ are right angle rotated squares centered at origin in two dimensions.

As before, imagine the expanding contours of $\boldsymbol{\epsilon}^T\boldsymbol{\epsilon}$ till they hit the constraint graph to deliver the solution. Since the constraint graph has corners, it is highly likely that the expanding contour of $\boldsymbol{\epsilon}^T\boldsymbol{\epsilon}$ will hit one of the corners. At that corner, one or more of the $\boldsymbol\beta$s will be zero.

Hence, Lasso not only shrinks the $\boldsymbol\beta$s but also makes some of them zero introducing sparsity in the solution and only retains only the most important $\boldsymbol\beta$s. This feature makes it considerably varied from the ridge regression.

\end{document}