\hypertarget{multiple-regression}{%
\chapter{Multiple Regression}\label{multiple-regression}}
\section{Introduction}
Multiple regression is like linear regression, but with more than one
independent value, meaning that we try to predict a value based on two
or more variables. Multiple regression is a statistical technique that
can be used to analyze the relationship between a single dependent
variable and several independent variables. The objective of multiple
regression analysis is to use the independent variables whose values are
known to predict the value of the single dependent value. Each predictor
value is weighed, the weights denoting their relative contribution to
the overall prediction.

\[
\hat{y} = \beta_0 + \beta_1 x_1 + \beta_2 x_2 + ... + \beta_n x_n
\]

Here \(\hat{y}\) is the dependent variable, and \(x_1, x_2, ..., x_n\)
are the \(n\) independent variables. In calculating the weights,
\(\beta_0\), \(x_1, x_2, ..., x_n\), regression analysis ensures maximal
prediction of the dependent variable from the set of independent
variables. This is usually done by least squares estimation.
\section{Implementation}
    \begin{tcolorbox}[breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder]
\prompt{In}{incolor}{1}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
\PY{k+kn}{import} \PY{n+nn}{pandas} \PY{k}{as} \PY{n+nn}{pd}
\PY{k+kn}{from} \PY{n+nn}{sklearn} \PY{k+kn}{import} \PY{n}{linear\PYZus{}model}
\PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{model\PYZus{}selection} \PY{k+kn}{import} \PY{n}{train\PYZus{}test\PYZus{}split}
\PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
\PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt}
\end{Verbatim}
\end{tcolorbox}

    \begin{tcolorbox}[breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder]
\prompt{In}{incolor}{2}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
\PY{n}{dataset} \PY{o}{=} \PY{n}{pd}\PY{o}{.}\PY{n}{read\PYZus{}csv}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{./data/house\PYZus{}data.csv}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
\PY{n}{dataset}\PY{o}{.}\PY{n}{head}\PY{p}{(}\PY{p}{)}
\end{Verbatim}
\end{tcolorbox}

            \begin{tcolorbox}[breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0]
\prompt{Out}{outcolor}{2}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
   sqft  bedrooms    price
0  1180         3  3540000
1  2570         3  7710000
2   770         2  1540000
3  1960         4  7840000
4  1680         3  5040000
\end{Verbatim}
\end{tcolorbox}

    \begin{tcolorbox}[breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder]
\prompt{In}{incolor}{3}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
\PY{n}{X} \PY{o}{=} \PY{n}{dataset}\PY{o}{.}\PY{n}{iloc}\PY{p}{[}\PY{p}{:}\PY{p}{,}\PY{l+m+mi}{0}\PY{p}{:}\PY{o}{\PYZhy{}}\PY{l+m+mi}{1}\PY{p}{]}
\PY{n}{y} \PY{o}{=} \PY{n}{dataset}\PY{o}{.}\PY{n}{iloc}\PY{p}{[}\PY{p}{:}\PY{p}{,}\PY{o}{\PYZhy{}}\PY{l+m+mi}{1}\PY{p}{]}
\PY{n}{dataset}\PY{o}{.}\PY{n}{corr}\PY{p}{(}\PY{p}{)}
\end{Verbatim}
\end{tcolorbox}

            \begin{tcolorbox}[breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0]
\prompt{Out}{outcolor}{3}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
              sqft  bedrooms     price
sqft      1.000000  0.410243  0.945462
bedrooms  0.410243  1.000000  0.654177
price     0.945462  0.654177  1.000000
\end{Verbatim}
\end{tcolorbox}

    \begin{tcolorbox}[breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder]
\prompt{In}{incolor}{4}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
\PY{n}{regr} \PY{o}{=} \PY{n}{linear\PYZus{}model}\PY{o}{.}\PY{n}{LinearRegression}\PY{p}{(}\PY{p}{)}
\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{y\PYZus{}train}\PY{p}{,} \PY{n}{y\PYZus{}test} \PY{o}{=} \PY{n}{train\PYZus{}test\PYZus{}split}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n}{y}\PY{p}{,} \PY{n}{test\PYZus{}size}\PY{o}{=}\PY{o}{.}\PY{l+m+mi}{25}\PY{p}{,} \PY{n}{random\PYZus{}state}\PY{o}{=}\PY{l+m+mi}{42}\PY{p}{)}
\PY{n}{regr}\PY{o}{.}\PY{n}{fit}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{y\PYZus{}train}\PY{p}{)}
\PY{n}{regr}\PY{o}{.}\PY{n}{score}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{y\PYZus{}test}\PY{p}{)}
\end{Verbatim}
\end{tcolorbox}

            \begin{tcolorbox}[breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0]
\prompt{Out}{outcolor}{4}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
0.976264543946519
\end{Verbatim}
\end{tcolorbox}

    \begin{tcolorbox}[breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder]
\prompt{In}{incolor}{5}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
\PY{n}{y\PYZus{}pred}\PY{o}{=}\PY{n}{regr}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X}\PY{p}{)}
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Predicted:}\PY{l+s+se}{\PYZbs{}t}\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{y\PYZus{}pred}\PY{p}{[}\PY{p}{:}\PY{l+m+mi}{10}\PY{p}{]}\PY{p}{)}
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Actual:}\PY{l+s+se}{\PYZbs{}t}\PY{l+s+se}{\PYZbs{}t}\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{y}\PY{o}{.}\PY{n}{values}\PY{p}{[}\PY{p}{:}\PY{l+m+mi}{10}\PY{p}{]}\PY{p}{)}
\end{Verbatim}
\end{tcolorbox}

    \begin{Verbatim}[commandchars=\\\{\}]
Predicted:       [ 3488660.82843106  8375217.66544624   329918.68771236
7948141.12000992
  5246415.08635019 20111800.5848103   5369457.88440453  3066799.80653047
  5597965.93793402  5984671.87467623]
Actual:          [ 3540000  7710000  1540000  7840000  5040000 21680000  5145000
3180000
  5340000  5670000]
    \end{Verbatim}

    \begin{tcolorbox}[breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder]
\prompt{In}{incolor}{6}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
\PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{n}{dpi}\PY{o}{=}\PY{l+m+mi}{90}\PY{p}{)}
\PY{n}{plt}\PY{o}{.}\PY{n}{scatter}\PY{p}{(}\PY{n+nb}{range}\PY{p}{(}\PY{n}{y\PYZus{}pred}\PY{o}{.}\PY{n}{size}\PY{p}{)}\PY{p}{,} \PY{n}{y\PYZus{}pred}\PY{p}{,} \PY{n}{label}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Predicted}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
\PY{n}{plt}\PY{o}{.}\PY{n}{scatter}\PY{p}{(}\PY{n+nb}{range}\PY{p}{(}\PY{n}{y\PYZus{}pred}\PY{o}{.}\PY{n}{size}\PY{p}{)}\PY{p}{,} \PY{n}{y}\PY{p}{,} \PY{n}{label}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Actual}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
\PY{n}{plt}\PY{o}{.}\PY{n}{legend}\PY{p}{(}\PY{p}{)}
\end{Verbatim}
\end{tcolorbox}

            \begin{tcolorbox}[breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0]
\prompt{Out}{outcolor}{6}{\boxspacing}
\begin{Verbatim}[commandchars=\\\{\}]
<matplotlib.legend.Legend at 0x1dc847c2af0>
\end{Verbatim}
\end{tcolorbox}

    \begin{center}
    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{./figures/MR1.png}
    \end{center}
    { \hspace*{\fill} \\}

MCA/Machine Learning/Project file/Tex source/Multiple Regression.tex


% !TEX root = ../main.tex
\newpage
\section{\theory The Theta Neuron Model} \label{TheThetaNeuronModel}
%\subsection{Canonical neuron models}
A number of neuron model families have been identified, and often there exists a continuous change of variables from models of the same family into a \textit{canonical} model that can represent the whole family \cite{Hoppensteadt2001CanonicalNM}. As the transformation is not required to be invertible, we can study the universal neurocomputational properties of the family in a low dimensional model.
It was Hodgkin \cite{Hodgkin1948} who classified neurons into two types based on their excitability, upon experimenting with the electrical stimulation of cells. Class 1 models begin to spike at an arbitrarily slow rate, and the spiking frequency increases when the applied current is increased. Class 2 models spike as soon as their internal threshold is exceeded and the spiking frequency stays relatively constant within a certain frequency band \cite{Hoppensteadt2001CanonicalNM}.

\subsection{Theta Neuron model description} \label{sec:TheThetaNeuronModelDescription}
In \cite{Ermentrout1986}, a Class 1 canonical phase model was proposed:
\begin{align}
\dot{\theta} = (1-\cos \theta)+(1+\cos \theta) \cdot I \qquad \theta \in \T \label{eq:thetaneuron}
\end{align}
with $I$ a bifurcation parameter on the supplied current. We can visualise the dynamics on the unit circle, like in Figure \ref{fig:thetaneuronbifurcationtikz}. The neuron produces a spike when $\theta$ surpasses $\pi$, upon which $\theta \leftarrow -\pi$.

\begin{figure}[H]
\minipage{0.33\linewidth}
\centering
\begin{tikzpicture}
    \draw (0,0) circle [radius=1];
    \draw (0,-1.2) node[below]{Excitable regime: $I < 0$};
    \draw (-1,0) node[left]{$\pi$};
    \draw[fill=black, black] (1,0) circle [radius=0.025];
    \draw (1,0) node[right]{0};
    \draw[fill=red, red] (-1,0) circle [radius=0.05];
    \draw (-1,0) node[right]{spike};

    \draw[black, ->] (0.866, 0.5)to[out=-60,in=90](1,0);
    \draw[fill=white, draw=black] (0.866,0.5) circle [radius=0.1];
    \draw (0.866,0.5) node[left]{\small{threshold}};

    \draw[fill=black, draw=black] (0.866,-0.5) circle [radius=0.1];
    \draw (0.866,-0.5) node[left]{\small{rest}};

    \draw[black, ->] (0.5,0.866)to[out=150,in=0](0,1);
    \draw[black, ->] (-0.5,-0.866)to[out=-30,in=180](0,-1);
\end{tikzpicture}
\endminipage
\minipage{0.33\linewidth}
\centering
\begin{tikzpicture}
    \draw (0,0) circle [radius=1];
    \draw (0,-1.2) node[below]{Bifurcation: $I = 0$};
    \draw (1,0) node[right]{0};
    \draw[fill=red, red] (-1,0) circle [radius=0.05];
    \draw (-1,0) node[right]{spike};
    \draw (-1,0) node[left]{$\pi$};

    \draw[fill=gray, draw=black] (1,0) circle [radius=0.1];

    \draw[black, ->] (0.5,0.866)to[out=150,in=0](0,1);
    \draw[black, ->] (-0.5,-0.866)to[out=-30,in=180](0,-1);
\end{tikzpicture}
\endminipage
\minipage{0.33\linewidth}
\centering
\begin{tikzpicture}
    \draw (0,0) circle [radius=1];
    \draw (0,-1.2) node[below]{Periodic regime: $I > 0$};
    \draw (-1,0) node[left]{$\pi$};
    \draw (1,0) node[right]{0};
    \draw[fill=black, black] (1,0) circle [radius=0.025];
    \draw[fill=red, red] (-1,0) circle [radius=0.05];
    \draw (-1,0) node[right]{spike};

    \draw[black, dotted] (0,0)to(1,0);
    \draw(0,0) node[above]{$\theta$};
    \draw[black, dotted] (0,0)to(0.866,0.5);

    \draw[black, ->] (0.5,0.866)to[out=150,in=0](0,1);
    \draw[black, ->] (-0.5,-0.866)to[out=-30,in=180](0,-1);
\end{tikzpicture}
\endminipage
\caption{SNIC bifurcation of the theta neuron model. A spike occurs when $\theta = \pi$. For $I < 0$, the neuron is in a rest state but \textsl{excitable} and we observe one stable and one unstable equilibrium point. For $I > 0$, $\dot{\theta} > 0$ so that $\theta$ moves continuously around the circle and we can observe \textsl{periodic} sustained spiking. The saddle-node bifurcation occurs at $I = 0$, so that $\theta$ will spike when it is larger than 0.}
\label{fig:thetaneuronbifurcationtikz}
\end{figure}

We can recognise the features of the class 1 model in Figure \ref{fig:ThetaNeuronResponseToCurrent}. This makes \eqref{eq:thetaneuron} the normal form of the \textit{saddle-node-on-invariant-circle} ($\SNIC$) bifurcation \cite{Luke2013}, as it collapses $\R$ to $\T$.

\begin{figure}[H]
\centering
\includegraphics[width = \textwidth]{../Figures/ThetaNeuronResponseToCurrent.pdf}
\caption{Properties of the theta neuron model, with solutions of \eqref{eq:thetaneuron} in blue, spikes marked in dotted lines, and the current $I$ in red. Left: the spike frequency of $\theta$ increases as $I$ is increased over time, which is the distinguishing feature of class 1 canonical models. Middle: spikes occur within a finite time period when $I > 0$ and within infite time when $I = 0$. Right: when $I$ is large, the neuron \textsl{bursts}.}
\label{fig:ThetaNeuronResponseToCurrent}
\end{figure}

Equilibria only exist for the \textsl{excitable} regime $I < 0$:
\begin{align*}
\dot{\theta} &= 1-\cos \theta+I+I \cdot \cos \theta = (I+1)+(I-1) \cdot \cos \theta \\
\theta^{\ast}_{1, 2} &= \pm \arccos \left(\frac{I+1}{1-I}\right)+2 \pi n
\end{align*}
We can find the stability of the equilibria through:
\begin{align*}
\frac{\mathop{d}}{\mathop{d \theta}}((1-\cos \theta)+(1+\cos \theta) \cdot I) &= \sin \theta-\sin \theta \cdot I = (1-I) \cdot \sin \theta
\end{align*}
In the equilibria this yields:
\begin{align*}
\frac{\mathop{d}}{\mathop{d \theta}}\left( \theta^{\ast}_{1, 2} \right) &= \pm(1-I) \cdot \sqrt{1 - \left( \frac{I+1}{1-I} \right)^2 } = \pm(1-I) \cdot \frac{2 \sqrt{-I}}{1-I} = \pm2 \sqrt{-I}
\end{align*}
We find that $\theta^{\ast}_{1}$ is an unstable equilibrium point, and that $\theta^{\ast}_{2}$ is stable. This means that as $\theta$ gets perturbed above $\theta^{\ast}_{1}$, a spike occurs and $\theta$ converges to $\theta^{\ast}_{2}$. This is demonstrated in Figure \ref{fig:ThetaModelEquilibriumPoints}.
\begin{figure}[H]
\centering
\includegraphics[width = \textwidth]{../Figures/ThetaModelEquilibriumPoints.pdf}
\caption{Equilibria $\theta^{\ast}$ for different values of $I < 0$. Left: $I = -1$ yields $\theta^{\ast}_{1,2} = \pm \frac{\pi}{2}$, one of the simulations is started exactly on the unstable equilibrium and stays there. Middle: $I = -0.5$, we see how a spike occurs when $\theta > \theta^{\ast}_{1}$ upon which $\theta \rightarrow \theta^{\ast}_{2}$. Right: bifurcation diagram of the \SNIC bifurcation, with the stable equilibria in blue, and the unstable in red. At $I = 0$ the equilibria merge into one.}
\label{fig:ThetaModelEquilibriumPoints}
\end{figure}

\subsection{Solutions for static currents} \label{sec:TheThetaNeuronModelSolutionPeriodics}
Gaining insight into \eqref{eq:thetaneuron} is hard, due to the difficulty of finding an analytical solution. However, it has been noted that there exists a simple transformation which yields (see \ref{app:TransformationToQIF}):
\begin{align}
V &\equiv \tan \left( \frac{\theta}{2} \right) \label{eq:QIFtransformation} \\
\dot{V} &= V^2 + I \label{eq:QIFmodel}
\end{align}
This model is called the \textsl{Quadratic Integrate and Fire model} (\QIF). \eqref{eq:QIFmodel} models the membrane potential of a neuron, which spikes at $V=\infty$ and resets to $V \leftarrow -\infty$. The transformation \eqref{eq:QIFtransformation} is continuous between spikes, so insights from a solution for $V$ can be transformed directly to $\theta$. The equilibria of the \QIF model are simply $\pm \sqrt{-I}$ (as $I < 0$) so that we can express $\theta^{\ast}_{1, 2} = 2 \arctan \left( \pm \sqrt{-I} \right)$ \cite{Gutkin2014}. \\

The solution for the excitable regime $I < 0$ is :
\begin{align}
V(t) = \frac{2 \sqrt{-I}}{1 - e^{2 t \sqrt{-I}}}-\sqrt{-I} \label{eq:ThetaNeuronModelSolutionPeriodicExcitable}
\end{align}
The solution at the bifurcation $I = 0$ is :
\begin{align}
V(t) = \frac{-1}{t} \label{eq:ThetaNeuronModelSolutionPeriodicBifurcation}
\end{align}
The solution for the periodic regime $I > 0$ is :
\begin{align}
V(t) = -\sqrt{I} \cdot \cot (t \sqrt{I}) \label{eq:ThetaNeuronModelSolutionPeriodic}
\end{align}
These equations assume that at $t=0$ a spike has occured. The steps required to find the solutions (\ref{eq:ThetaNeuronModelSolutionPeriodicExcitable}) to (\ref{eq:ThetaNeuronModelSolutionPeriodic}) are described in \ref{app:ThetaModelSolutions}. Solutions for $\theta$ are found by simply taking the inverse of the transformation in \eqref{eq:QIFtransformation}.

If the \QIF model is so much simpler, then why bother using the Theta model? Simulating the \QIF model requires an artificial reset threshold, because we cannot expect a computer to represent infinity easily. Finite thresholds make the analytical solutions more difficult and convoluted. By using the Theta model the dynamics remain smooth and bounded on $\T$. %Since we take the cosine of the phase angle in \eqref{eq:thetaneuron}, we do not even

\subsection{Numerical solutions} \label{sec:TheThetaNeuronModelODE45}
When $I$ is not static, we need to revert to numerical solutions. A fixed-step 4-stage Runge-Kutta method (Dormand-Prince 45) was implemented to numerically solve all differential equations. A fixed-step algorithm makes it possible to finely tune the large memory demand of the systems presented in this work.

\subsection{Frequency response} \label{sec:TheThetaNeuronModelFrequencyResponse}
\begin{wrapfigure}{r}{0.39\textwidth}
\centering
\vspace{-\baselineskip}
\includegraphics[width = \linewidth]{../Figures/ThetaNeuronfI.pdf}
\caption{Frequency response of the Theta model. For $I \leq 0$ the spike period is infinite, which is why we see the solutions to (\ref{eq:thetaneuron}) approach $\theta = 0$ for $I = 0$ in Figure \ref{fig:ThetaNeuronResponseToCurrent}. }
\label{fig:ThetaNeuronfI}
\end{wrapfigure}

As we already saw in Figure \ref{fig:ThetaNeuronResponseToCurrent}, an increasing current increases the spiking frequency. We can compute this relationship by measuring how long it takes for $V$ to reach a spike: we solve \eqref{eq:ThetaNeuronModelSolutionPeriodic} for $t$ at $V(t) = +\infty$ in \ref{app:ThetaModelFrequencyResponse}. This yields the oscillation period $T = \frac{\pi}{\sqrt{I}}$, which we can see in Figure \ref{fig:ThetaNeuronfI}.

We know that when $\theta > \theta^{\ast}_{1}$ a spike occurs in the excitable regime, or in any case in the periodic regime. But the time that it takes to reach the spike can be arbitrarily long, depending on how far we are over $\theta^{\ast}_{1}$. So, spikes will occur, but after a delay that is dependant on the stimulus. Explicitly, if we perturb $\theta(0) = \theta^{\ast}_{1} + \varepsilon$ we obtain from \cite{Gutkin2014}:
\begin{align*}
T_{\text {spike}} = \frac{-\tanh ^{-1}\left(1+\frac{\epsilon}{\sqrt{I}}\right)}{\sqrt{I}}
\end{align*}
The delay to the spike blows up as $\varepsilon \rightarrow$ 0 so that spikes may occur after a very large delay. \\

In most of our future work, $I$ will not be a static current. We ask ourselves: how sensitively does $T$ depend on $I$ when $I$ is perturbed? We can measure this as a \textsl{relative} perturbation using $\mathop{dI}/I$ and $\mathop{dT/T}$ \cite{IntroductionModelingDynamics} :
\begin{align*}
\left| \frac{dT}{dI} \frac{I}{T} \right| &= \left| \frac{dT / T}{dI / I}\right|
= \left|- \frac{\pi}{2} \left(\frac{1}{\sqrt{I}}\right)^3 \frac{I}{T} \right|
= \left| \frac{\pi}{2} \left(\frac{T}{\pi}\right)^3 \frac{I}{T} \right|
= \frac{1}{2} \left|\left(\frac{T}{\pi}\right)^2 \left(\frac{\pi}{T}\right)^2 \right| = \frac{1}{2}
\end{align*}
Hence, a 1\% change in $I$ will result in a 0.5 \% change in the period.

\subsection{Phase response} \label{sec:TheThetaNeuronModelPhaseResponse}
Perturbations on the period can also be understood from the perspective of the phase. Changes to the phase $\theta$ can delay or advance the event of a spike, and in general this depends on exactly when the stimulus occurs. The phase response curve (\PRC) gives us exactly that relation \cite{Perez2020, Gutkin2014}.

Let us define $\phi \in [0, T[$, which represents the time since the last event of a spike. When we add a small bifurcation $\varepsilon$ to $\theta$ at time $\phi$, a spike will occur at $T_{\phi}$, and we have that $\theta(\phi_{\rm new}) = \theta(\phi) + \varepsilon$. The time to the new spike is now $T_{\phi} = T + (\phi - \phi_{\text{new}})$. The \PRC can then be defined as:
\begin{align}
\PRC(\phi) = T - T_{\phi} \label{eq:PRC1}
\end{align}
The \PRC is thus the expected delay of the period in function of the timing of that delay. This process has been visualised in Figure \ref{fig:ThetaNeuronPRC}, after \cite{Perez2020}. For infinitesimally small perturbations to the phase, we can find the \PRC as the \textsl{adjoint} of the solution \cite{Gutkin2014}:
\begin{align}
\PRC(\phi) = \frac{1}{d V(\phi) / d \phi} = \frac{1}{2 \sqrt{I}} \left(1-\cos \left( \frac{2 \pi}{T} \phi \right) \right) \label{eq:PRC2}
\end{align}
We can use $\phi \in [0, T[$ and $\theta \in \T$ to see that \eqref{eq:PRC2} can be expressed as:
\begin{align}
\PRC(\theta) \sim 1 + \cos \theta  \label{eq:PRC3}
\end{align}
which is the magnitude with which $I$ excites the model, see \eqref{eq:thetaneuron} \cite{Ermentrout1996}. Analysis of the \PRC thus also allows us to study how the bifurcation of $\theta$ with magnitude $I$ occurs. The \PRC is always positive, which indicates that a positive bifurcation will accelerate the time of the spike, and vice versa. This has also been reported as a distinguishing feature of Class 1 models \cite{Ermentrout1996}. \\

As bifurcations are not always small, we need to dig a little deeper. An exact formulation for $T_{\phi}$ can be obtained by integrating after the bifurcation, see \ref{app:ThetaModelPhaseResponse}. The \PRC then becomes:
\begin{align}
\PRC(\phi, \varepsilon) = \frac{1}{\sqrt{I}}\left(\frac{\pi}{2} + \arctan \left(\frac{\varepsilon}{\sqrt{I}} - \cot \left(\phi\sqrt{I} )\right)\right)\right) - \phi \label{eq:PRCepsilon}
\end{align}
When plotting \eqref{eq:PRCepsilon} for different values of $\varepsilon$ in Figure \ref{fig:ThetaNeuronPRC}, we can see that larger bifurcations yield larger delays on the phase. For infinitesimally small bifurcations the response of \eqref{eq:PRC2} is symmetric about the middle of the period. However, here we see that with increasing $\varepsilon$ the \PRC loses symmetry and skews to the left. For large bifurcations, we can expect the greatest perturbation of the phase briefly after the action potential. This tendency continues to skew to the left for increasing $I$, and the \PRC converges to a value of $T$ at $\phi = 0$. A delay with magnitude $T$ is really the largest delay we can achieve with a single bifurcation. The effect of a bifurcation diminishes over the period: as $\theta$ approaches $\pi$, the ability to advance a spike in time disappears.\\

\begin{figure}[H]
\centering
\includegraphics[width = \textwidth]{../Figures/ThetaNeuronPRC.pdf}
\caption{Response of the Theta model to bifurcations on the phase. Left: a bifurcation $\varepsilon < 0$ at time $\phi$ perturbs $\theta(t)$ (in blue) which results in a delayed spike (trajectory in red). For $\varepsilon > 0$ spikes are advanced in time. Right: the \PRC, given by \eqref{eq:PRCepsilon}, plotted in orange for $\varepsilon$ ranging from 0.1 to 1 - a weak bifurcation is more translucent. A solution for $\theta$ (in blue) shows when the model is the most susceptible to bifurcations over the course of one period.}
\label{fig:ThetaNeuronPRC}
\end{figure}


\documentclass[11pt,twocolumn]{amsart} % twocolumn
\usepackage{geometry}                % See geometry.pdf to learn the layout options. There are lots.
\geometry{a4paper}                   % ... or a4paper or a5paper or ...
\usepackage{layout}
%\geometry{landscape}                % Activate rotated page geometry
%\usepackage[parfill]{parskip}    % Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{epstopdf}
\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}
\DeclareMathOperator*{\argmin}{arg\,min}

% shortcuts
\newcommand{\ve}[1]{\boldsymbol{#1}}
\newcommand{\ma}[1]{\boldsymbol{#1}}
\newenvironment{m}{\begin{bmatrix}}{\end{bmatrix}}

\title{Some stuff for \textbf{home}}
\author{Gaspard Buma}
%\date{}                                           % Activate to display a given date or no date

\begin{document}

\twocolumn[
\maketitle
]
\section{Notations}
In the following discussion the following typographic notations has been used:
\begin{itemize}
  \item \emph{Lower case bold face} means vector (ex. $\ve\mu, \ve{x}$). All vectors are row vectors like $\begin{m} x_1 & x_2 \end{m}$ not $\begin{m} x_1 \\ x_2 \end{m}$.
  \item \emph{Upper case bold face} means matrix (ex. $\ma{A}, \ma{S}, \ma{\Sigma}$).
  \item The transpose of a matrix is noted $\ma{A}^T$, not $\ma{A}'$.
  \item $\ve{x}', \ma{A}'$ denotes a vector/matrix that has been transformed.
\end{itemize}
\section{Variance of a value}

The \textbf{variance} (expressed with the symbol $\sigma^2$ or $\sigma_S^2$) measures the dispersion of a set of values $\ma{S}$ from the mean value $\mu$ or expected value $E(\ma{S})$. The mean value is~:

\[
  \mu = E(\ma{S}) = \frac{1}{l} \sum_1^{l} s_l
\]

$l$ being the number of elements in the set. The variance can be computed as~:
\[
  \sigma^2 = E[(\ma{S} - \mu)(\ma{S} - \mu)] = \frac{1}{l} \sum_1^{l} (s_l - \mu)^2
\]

For example, given the set of values $\ma{S} = {1,4,3,6}$, the mean value $\mu$ is~:
\[
  \mu = \frac{1+4+3+6}{4} = 3.5,
\]

and the variance $\sigma^2$~:
\begin{align*}
  \sigma^2 = & (1-3.5)^2 + (4-3.5)^2 +\\
             & \frac{(3-3.5)^2 + (6-3.5)^2}{4} = \frac{13}{4} = 3.25
\end{align*}

\section{variance-covariance matrix}

The \textbf{variance-covariance matrix} is the generalisation to greater dimensions of the variance seen before. Imagine we have a set $\ma{S}$ of vectors of dimension $n$ and we want to compute the variance of these vectors. We must first compute the expected value $E(\ma{S})$ which is the mean vector $\mu$~:
\begin{align*}
  E(\ma{S}) = & \ve{\mu} = \begin{m} \mu_1 \hdots \mu_n \end{m} \\
       = & \frac{1}{l} \begin{m} \begin{m} s_{11} \hdots s_{1n} \end{m} \\ + \\ \vdots \\ + \\ \begin{m} s_{l1} \hdots s_{ln} \end{m} \end{m}
\end{align*}

We can build a matrix $\Theta$ (theta) of size ($l,n$) from the vector $\mu$ (mu)~:
\[
  \ma\Theta = \begin{m} \ve{\mu} \\ \vdots \\ \ve{\mu} \end{m}
\]

From the definition of the variance, we have
\[
  \Sigma = E \begin{m} (\ma{S} - \ma\Theta)^T (\textbf{S} - \ma\Theta)\end{m}
\]

thus each entry in the variance-covariance matrix $\ma\Sigma$ (sigma) is~:
\[
  \Sigma_{ij} = \frac{\sum_{k=1}^{l} (s_{ki} - \mu_i)(s_{kj} - \mu_j)}{l}.
\]

For example, given the set $\ma{S}$ formed with three experiments measuring two values~:
\begin{displaymath}
  \ma{S} = \begin{m} 1 & 1 \\ 0 & 4 \\ 2 & 1 \end{m},
\end{displaymath}

we can compute the expected value~:
\[
  \ve{\mu} = \frac{1}{3} \begin{m} \left( \begin{array}{c} 1 \\ + \\ 0 \\ + \\ 2 \end{array} \right) \left( \begin{array}{c} 1 \\ + \\ 4 \\ + \\ 1 \end{array} \right) \end{m} = \begin{m} 1 & 2 \end{m}.
\]

To find the variance-covariance matrix, let us first compute the \emph{mean deviation form} of $\ma{S}$ where we remove the mean value from each experiment in the set~:
\[
  \ma{T} = \ma{S} - \ma\Theta = \begin{m} (1-1) & (1-2) \\ (0-1) & (4-2) \\ (2-1) & (1-2)\end{m} = \begin{m} 0 & -1 \\ -1 & 2 \\ 1 & -1\end{m}.
\]

We can now find $\ma\Sigma$~:
\begin{align*}
  \ma\Sigma = & E \begin{m} \ma{T}^T\ma{T}\end{m} = \frac{1}{n-1} \begin{m} \ma{T}^T\ma{T}\end{m} \\
         = & E \begin{m} \begin{m} 0 & -1 & 1 \\ -1 & 2 & -1\end{m} \begin{m} 0 & -1 \\ -1 & 2 \\ 1 & -1\end{m} \end{m} \\
         = & E \begin{m} 0*0 + -1*-1 + 1*1 & \hdots \\ -1*0 + 2*-1 + -1*1 & \hdots \end{m} \\
         = & E \begin{m} 2 & -3 \\ -3 & 6 \end{m} = \frac{1}{2} \begin{m} 2 & -3 \\ -3 & 6 \end{m}
         = \begin{m} 1 & -\frac{3}{2} \\ -\frac{3}{2} & 3\end{m}
\end{align*}

From the result above, $1$ is the variance along $x$, $3$ is the variance along $y$ and $-\frac{3}{2}$ is the covariance between the two dimensions. There are proofs on why we should use $\frac{1}{n-1}$ for the correct normalization factor instead of the straight-forward $\frac{1}{n}$ but this is beyond the scope of my competences...

\section{Mahalanobis distance}

Let us compute the \textbf{Mahalanobis distance} from a vector $\ve{t}$ to the set $\ma{S}$ with the variance-covariance matrix $\ma\Sigma$ and the mean value $\ve{\mu}$~:
\[
  D_M(\ve{t}) = \sqrt{(\ve{t} - \ve{\mu})\Sigma^{-1}(\ve{t} - \ve{\mu})^T}
\]

If we remove the covariances from the matrix $\ma\Sigma$, we have a diagonal matrix whose entries are the variances (standard deviations) for each dimension ($1$ and $3$ in the example above). The Mahalanobis distance becomes the \textbf{normalized Euclidean distance}~:
\[
  D_{nE}(\ve{t}) = \sqrt{\sum_{i=1}^{n}\frac{(t_i - \mu_i)^2}{\sigma_i^2}}.
\]

From the example we used above we compute the different distances for two vectors $\ve{t_1}$ and $\ve{t_2}$~:

\includegraphics{points.pdf}
\begin{align*}
  \ve{t_1}  = & \begin{m} 2 & 2 \end{m} \\
  \ve{t_2}  = & \begin{m} 1 & 3 \end{m} \\
  \ve{\mu} = & \begin{m} 1 & 2 \end{m} \\
  \Sigma    = & \begin{m} 1 & -\frac{3}{2} \\ -\frac{3}{2} & 3\end{m}
\end{align*}

The \textbf{Euclidean distance} $d$ is the same for both points, even though it is visible that $\ve{t_1}$ is more ``out of the way" then $\ve{t_2}$ ~:
\begin{align*}
  d_1 = & \sqrt{\sum_{i=1}^{n}(t_i - \mu_i)^2} \\
     = & \sqrt{(2-1)^2 + (2-2)^2} = 1 \\
  d_2 = & \sqrt{(1-1)^2 + (3-2)^2} = 1
\end{align*}

The \textbf{normalized Euclidean distance} $d_{nE}$ should show that $\ve{t_2}$ matches the set $\ma{S}$ better then $\ve{t_1}$~:
\begin{align*}
  d_{1nE} = & \sqrt{\frac{(2-1)^2}{1} + \frac{(2-2)^2}{3}} = 1 \\
  d_{2nE} = & \sqrt{\frac{(1-1)^2}{1} + \frac{(3-2)^2}{3}} = \frac{1}{\sqrt{3}} \approx 0.6.
\end{align*}

This is better. We can see that $\ve{t_1}$ is penalized for being out of the way in the wrong direction. The ``penalty" for being badly adjusted along the $x$ axis is $1$ when it is only $\frac{1}{3}$ along the $y$ axis.

Let's now see what the Mahalanobis distance tells us. First we need to compute $\Sigma^{-1}$~:
\[
  \Sigma^{-1} = \frac{1}{(1*3 - -\frac{3}{2}*-\frac{3}{2})} \begin{m} 3 & \frac{3}{2} \\ \frac{3}{2} & 1\end{m} = \begin{m} 4 & 2 \\ 2 & \frac{4}{3} \end{m}
\]

We can now find the distances~:
\begin{align*}
  d_{1M} = & \sqrt{(\ve{t_1} - \ve{\mu})\Sigma^{-1}(\ve{t_1} - \ve{\mu})^T} \\
         = & \sqrt{\begin{m} 1 & 0 \end{m} \begin{m} 4 & 2 \\ 2 & \frac{4}{3}\end{m} \begin{m} 1 \\ 0 \end{m}} \\
         = & \sqrt{\begin{m} 4 & 2 \end{m} \begin{m} 1 \\ 0 \end{m}} = 2\\
  d_{2M} = & \sqrt{\begin{m} 0 & 1 \end{m} \begin{m} 4 & 2 \\ 2 & \frac{4}{3}\end{m} \begin{m} 0 \\ 1 \end{m}} \\
         = & \sqrt{\begin{m} 2 & \frac{4}{3}\end{m} \begin{m} 0 \\ 1 \end{m}} = \frac{2}{\sqrt{3}} \approx 1.2\\
\end{align*}

These distances classify the two vectors in the exactly the same way as the normalized Euclidean distances. Why is that so ? The reason is that our vectors are always away from the mean value in a single direction at a time, thus the ``covariance'' part of the error is never used.

What happens with the two points $\ve{t_3}$ and $\ve{t_4}$ ? From the first look at the picture we can see that their distance to the mean value is the same~:
\begin{align*}
  d_3 = & \frac{1}{\sqrt{2}} \approx 0.7 \\
  d_4 = & \frac{1}{\sqrt{2}} \approx 0.7
\end{align*}

Their normalized Euclidean values are also the same as they differ from the mean value exactly in the same way along $x$ and $y$~:
\begin{align*}
  d_{3nE} = & \frac{1}{\sqrt{3}} \approx 0.6\\
  d_{4nE} = & \frac{1}{\sqrt{3}} \approx 0.6
\end{align*}

Now comes the Mahalanobis distances~:
\begin{align*}
  d_{3M} = & \frac{1}{\sqrt{3}} \approx 0.6\\
  d_{4M} = & \frac{\sqrt{14}}{\sqrt{6}} \approx 1.5
\end{align*}

This type of distance calculation is the only one that reflect the difference between the two vectors $\ve{t_3}$ and $\ve{t_4}$ and classifies $\ve{t_4}$ as beeing further ``away'' then $\ve{t_3}$, like our eyes tell us from a quick look at the picture~!

\section{Eigenvalues}

Before we study \emph{Principal Component Analysis}, we need some understanding of eigenvalues. Imagine we have a transformation matrix $\ma{P}$~:
\[
  \ma{P} = \begin{m} 1 & 1 \\ 0 & 2 \end{m}.
\]

\includegraphics{eigen.pdf}

What are the vectors in this transformation that keep their direction (are not rotated) ? Mathematically speaking we want to find $\ve{e}$ such that~:
\[
  \ve{e'} = \ve{e} * \ma{P} = \lambda \ve{e}
\]

We can see from the picture that $\ve{y}$ satisfy this as $\ve{y'}$ is just $2 * \ve{y}$. Such vectors are called \emph{eigenvectors} and their corresponding lambda values \emph{eigenvalues}. To find these vectors, we can rewrite the equation above~:

\[
  \ve{e'} = \ve{e} * \ma{P} = \lambda \ve{e}
\]

as

\[
  \ve{e} * \ma{P} - \lambda \ve{e} = 0
\]

or

\[
  \ve{e} (\ma{P} - \lambda \ma{I}) = 0
\]

We know that an equation of the form~:
\[
  \ve{x} \ma{A} = 0
\]
has only the trivial solution $\ve{x} = \ve{0}$ if the determinant of $\ma{A}$ is not null. We can therefore find the non-trivial solutions to the equation above by computing the determinant when the latter \emph{is} null. This equation is called the \emph{characteristic equation} of $\ma{P}$~:
\[
  det(\ma{P} - \lambda \ma{I}) = 0
\]

We can now calculate the values of $\lambda$~:
\begin{align*}
  det(\begin{m} 1 & 1 \\ 0 & 2\end{m} - \lambda \begin{m} 1 & 0 \\ 0 & 1 \end{m}) = & 0 \\
  det(\begin{m} 1-\lambda & 1 \\ 0 & 2-\lambda\end{m}) = & 0 \\
  (1-\lambda)(2-\lambda) - 1 * 0 = & 0 \\
  (1-\lambda)(2-\lambda) = & 0
\end{align*}

which has the solutions
\[
\lambda_1 = 1, \lambda_2 = 2
\]

The corresponding \emph{eigenvectors} can be found by substituting $\lambda_1$ and $\lambda_2$ in $\ve{e} (\ma{P} - \lambda \ma{I}) = \ve{0}$~:
\begin{align*}
  \begin{m} x & y \end{m} \begin{m} 1-\lambda & 1 \\ 0 & 2-\lambda \end{m} = & 0 \\
  \begin{m} x_1 & y_1 \end{m} \begin{m} 1-1 & 1 \\ 0 & 2-1 \end{m} = & 0 \\
  \begin{m} x_1 & y_1 \end{m} \begin{m} 0 & 1 \\ 0 & 1 \end{m} = & 0 \\
  x_1 + y_1 = & 0 \\
  \ve{e_1} = & \begin{m} t & -t \end{m}
\end{align*}

and for $\lambda_2$~:
\begin{align*}
  \begin{m} x_2 & y_2 \end{m} \begin{m} 1-2 & 1 \\ 0 & 2-2 \end{m} = & 0 \\
  \begin{m} x_2 & y_2 \end{m} \begin{m} -1 & 1 \\ 0 & 0 \end{m} = & 0 \\
  -x_2  = & 0 \\
   x_2  = & 0 \\
  \ve{e_2} = \begin{m} 0 & t \end{m} = \ve{y}
\end{align*}

If we look at our transformation picture again, we see that the relation in the direction $\ve{e_1}$ is kept and the relation in the direction of $\ve{e_2}$ is just doubled. This reflects the eigenvalues $1$ and $2$ with the directions of the eigenvectors $\ve{e_1}$ and $\ve{e_2}$.

Before moving on to the following section, we need to establish some properties of \emph{symmetric} matrices. The first one is that such a matrix $\ma{A}$ can be expressed by the multiplication of a diagonal matrix $\ma{D}$ (all entries not on the diagonal are $\ve{0}$) and an invertible matrix $\ma{E}$~:
\begin{equation*}
  \ma{A} = \ma{E}^T\ma{D}\ma{E}. \label{symmetric}
\end{equation*}

Because $\ma{A}$ is symmetric, the transpose $\ma{A}^T$ of $\ma{A}$ (mirror across the diagonal) is the same as $\ma{A}$, we can therefore write~:
\begin{align*}
  \ma{A}^T = & (\ma{E}^T\ma{D}\ma{E})^T \\
           = &  \ma{E}^T(\ma{E}^T\ma{D})^T \\
           = &  \ma{E}^T\ma{D}^T\ma{E}^{TT}
\end{align*}
now we simply replace $\ma{D}^T$ by $\ma{D}$ since a diagonal matrix is equivalent to its transpose and we replace $\ma{E}^{TT}$ by $\ma{E}$ since transposing twice is equivalent to doing nothing. We get
\[
  \ma{A}^T = \ma{E}^T\ma{D}\ma{E} = \ma{A}.
\]

This proves that any matrix that can be expressed as a product of an invertible matrix $\ma{E}$ and a diagonal matrix $\ma{D}$ is indeed symmetric. We should now prove the reverse~: that any symmetric matrix can be expressed as stated above. This proof is called the \emph{spectral theorem}. While trying to understand it, I fell across strange names like \emph{Hermitian matrices}, \emph{Schur decomposition} and \emph{Hilbert spaces}. It also involved matrix calculations with complex numbers. That was too much me. Sorry. For those interested, I found a proof which you can consult on the web\footnote{(http://users.utu.fi/jkari/compression/kltaddition.pdf)}.

Anyway, let's pass on to the interesting part~: the matrix $\ma{E}$ used in the little talk above is in fact a matrix of the orthonormal eigenvectors of $\ma{A}$ (those vectors that do not rotate when transformed by $\ma{A}$). The proof is interesting but quite long, you can safely pass on to the section on \emph{Principal Component Analysis} if you do not have fun with letters in capital bold.

\emph{Please note that the following proofs are a rewrite of the first appendix of the tutorial on PCA by Jonathon Shlens\footnote{www.snl.salk.edu/~shlens/pub/notes/pca.pdf}}.

First, lets build this matrix $\ma{E}$ from the eigenvectors of $\ma{A}$~:
\[
  \ma{E} = \begin{m} \ve{e_1} \\ \vdots \\ \ve{e_n} \end{m}.
\]

Now let $\ma{D}$ be a diagonal matrix with the corresponding eigenvalues of $\ma{A}$~:
\[
  \ma{D} = \begin{m} \lambda_1 & 0 & .. \\ 0 & \lambda_2 & .. \\ \vdots & \vdots & \ddots \end{m}.
\]

We first try to prove that $\ma{E}\ma{A} = \ma{D}\ma{E}$~:
\begin{align*}
  \ma{E}\ma{A} = \begin{m} \ve{e_1} \\ \ve{e_2} \\ \vdots \end{m} \ma{A} = &
                 \begin{m} \ve{e_1}\ma{A} \\ \ve{e_2}\ma{A} \\ \vdots \end{m} \\
  \ma{D}\ma{E} = \begin{m} \lambda_1 & 0 & .. \\ 0 & \lambda_2 & .. \\ \vdots & \vdots & \ddots \end{m} \ma{E} = &
                 \begin{m} \lambda_1\ve{e_1} \\ \lambda_2\ve{e_2} \\ \vdots \end{m}.
\end{align*}

For $\ma{E}\ma{A}$ to be equal to $\ma{D}\ma{E}$, we must show that $\ve{e_i}\ma{A} = \lambda_i\ve{e_i}$ for all $i$. Remember that an eigenvector is \emph{a vector that is scaled onto itself by the transformation}, that is $\ve{e'} = \lambda\ve{e}$. This is exactly what the result of our equation states, thus the entries of $\ma{E}$ are the eigenvectors of $\ma{A}$ if $\ma{E}\ma{A} = \ma{D}\ma{E}$.

To finish the proof, we just need to show that $\ma{E}\ma{A} = \ma{D}\ma{E}$ implies $\ma{A} = \ma{E}^T\ma{D}\ma{E}$. By multiplying each side by $\ma{E}^{-1}$, we obtain $\ma{A} = \ma{E}^{-1}\ma{D}\ma{E}$. If we could show that $\ma{E}^{-1} = \ma{E}^T$, we would have proven (\ref{symmetric}). We know that $\ma{E}^{-1} = \ma{E}^T$ if $\ma{E}$ is an orthogonal matrix since~:
\begin{align*}
  \ma{E}\ma{E}^T = & \begin{m} \ve{e_1} \\ \ve{e_2} \\ \vdots \end{m} \begin{m} \ve{e_1}^T & \ve{e_2}^T & \hdots \end{m} \\
                 = & \begin{m} \ve{e_1}\ve{e_1}^T & \ve{e_1}\ve{e_2}^T & \hdots \\ \ve{e_2}\ve{e_1}^T & \ve{e_2}\ve{e_2}^T & \hdots \\ \vdots & \vdots & \ddots \end{m} \\
                 = & \begin{m} 1 & 0 & \hdots \\ 0 & 1 & \hdots \\ \vdots & \vdots & \ddots \end{m} \\
                 = & \ma{I}
\end{align*}

because each vectors in an orthogonal matrix is $\perp$ to other vectors, thus the dot product is zero except for the diagonal entries where it is equal to the squared norm of the vector. If we choose our eigenvectors so that their norm is 1, we obtain the identity matrix $\ma{I}$. We just need to show that $\ma{E}$ is an orthogonal matrix and pulling the thread out of this maze, we prove that a symmetric matrix can be expressed by (\ref{symmetric}). We want to know if $\vec(e_i) \dot \vec{e_j} = 0$ for all $i \not= j$. Let $\lambda_i$ and $\lambda_j$ be two distinct eigenvalues.
\begin{align*}
  \lambda_i\ve{e_i} \cdot \ve{e_j} = & \lambda_i\ve{e_i}\ve{e_j}^T \\
                                  = & \ve{e_i}\ma{A}\ve{e_j}^T \\
                                  = & \ve{e_i}(\ve{e_j}\ma{A}^T)^T \\
                                  = & \ve{e_i}(\ve{e_j}\ma{A})^T \\
                                  = & \ve{e_i}(\lambda_j\ve{e_j}^T) \\
  \lambda_i\ve{e_i} \cdot \ve{e_j} = & \ve{e_i} \cdot \lambda_j\ve{e_j}
\end{align*}

Thus $(\lambda_i - \lambda_j)\ve{e_i} \cdot \ve{e_j} = 0$. Since we have conjectured that $\lambda_i \not= \lambda_j$, it must be that $\ve{e_i}$ and $\ve{e_j}$ are orthonormal (their dot product is $0$). Therefore $\ma{E}^{-1} = \ma{E}^T$ and we can write~:
\[
  \ma{A} = \ma{E}^T\ma{D}\ma{E}.
\]

All this work, just to reexpress an arbitrary symmetric matrix $\ma{A}$\dots But this result is the ground on which the next section is built.

\section{Principal Component Analysis}

Principal Component Analysis is a way to extract the ``most proeminent features'' of a data set. It assumes that the data has a good signal to noise ratio which means that \emph{great variance} in the data is a synonym for \emph{relevant feature} and not \emph{big noise}.

In all the calculations below, we need to use the \emph{variations} of the measures. We will thus remove the mean values from our data set and use this new set called the \emph{mean deviation form} of $\ma{S}$~:
\[
  \ma{T} = \ma{S} - \begin{m}\ve{\mu} \\ \vdots \\ \ve{\mu}\end{m}.
\]

In order to extract the ``most proeminent features'' of our data, a good idea would be to look at the directions in which the data varies most. Is it along the $\ve{x}$ axis ? The $\ve{y}$ axis ? Somewhere in between ?

Recall from the previous sections that the variance-covariance matrix $\ma\Sigma$ displays variances along the diagonal and covariances off-diagonal. We can use this matrix to extract directions in which the data varies most~:
\[
  \ma\Sigma = \frac{1}{n-1}\ma{T}^T\ma{T}.
\]

By looking at the diagonals of $\Sigma$, we pick the greatest absolute value and there we are, we found our direction ($3 \Rightarrow \ve{y}$). Is that it ? No, this solution misses the fact that the main direction might not follow one of the axis (dimension) of our recorded data (the ellipsoid is not vertical nor horizontal). What we need is to apply a transformation (rotation + stretch) to our data $\ma{T}$ into an new basis $\ma{P}$ prior to looking at the variance-covariance matrix, so that the principal components align with the new basis' axis.
\begin{align*}
  \ma{T'} = & \ma{T}\ma{P} \\
  \ma\Sigma'     = & \frac{1}{n-1}\ma{T'}^T\ma{T'}
\end{align*}

Our goal is thus to find a new basis $\ma{P}$ that gives us great absolute values along the diagonals of $\ma\Sigma'$ and small absolute values off-diagonal (small redundancy). If possible, it would be great to see large variances along the top of the diagonal which would mean we could just cut off the rightmost part of the matrix $\ma{T'}$ and still have most of the information on the varying signal. To summarize~:
\begin{itemize}
  \item Find a new basis $\ma{P}$ that reduces the redundancy of the signals (low covariance).
  \item The $\ma\Sigma'$ matrix for $\ma{T'}$ should ideally be a diagonal matrix with very high values at the top and very small at the bottom, thus showing signals in the first dimensions and pure noise in the last ones.
  \item We could then choose to remove the columns of $\ma{T'}$ that have greater indices than the number of dimensions we want to keep thus reducing the dimensionality of our data.
\end{itemize}

A simple way to find the solution would be~:
\begin{enumerate}
  \item Find the direction $\ve{e_1}$ that shows the greatest variance in the data $\ma{T}$. Normalize this vector. This is our first basis vector.
  \item Find another normalized vector showing the greatest variance that is orthogonal to those already found ($\forall \ve{b}\in \{\text{found vectors}\}, \ve{e_n} \perp \ve{b}$).
  \item Repeat until we have $n$ basis vectors ($n$ is the size of vector space).
\end{enumerate}

The resulting ordered set of vectors are the \emph{principal components} of our data set \textbf{T}.

\section{computing PCA}

Mathematically stated, what we are looking for is a diagonalized version of $\ma\Sigma'$~:
\begin{align*}
  \ma\Sigma'     = & \frac{1}{n-1}\ma{T'}^T\ma{T'} \\
                 = & \frac{1}{n-1}(\ma{T}\ma{P})^T(\ma{T}\ma{P}) \\
                 = & \frac{1}{n-1}\ma{P}^T\ma{T}^T\ma{T}\ma{P}
\end{align*}

Does this look familiar to you ? Recall all our lengthy discussion on symmetrical matrices ? Well, $\ma{T}^T\ma{T}$ is such a matrix. Lets replace it by $\ma{A}$. We get~:
\[
  \ma\Sigma'     = \frac{1}{n-1} \ma{P}^T\ma{A}\ma{P}
\]

Now what happens if we set $\ma{P} = \ma{E}^T$ and replace $\ma{A}$ by $\ma{E}^T\ma{D}\ma{E}$ from equation (\ref{symmetric}) ?
\begin{align*}
  \ma\Sigma' = & \frac{1}{n-1}\ma{E}\ma{A}\ma{E}^T \\
             = & \frac{1}{n-1}\ma{E}(\ma{E}^T\ma{D}\ma{E})\ma{E}^T \\
             = & \frac{1}{n-1}\ma{I}\ma{D}\ma{I} \\
             = & \frac{1}{n-1}\ma{D}
\end{align*}

We have just found the transformation that diagonalizes the variance-covariance matrix. It's the transpose of the matrix of eigenvectors of $\ma{T}^T\ma{T}$ !
\[
  \ma{P} = \ma{E}^T = \begin{m} \ve{e_1}^T & \ve{e_2}^T & \hdots \end{m}
\]
where $\ve{e_i}$ is $i$th eigenvector of $\ma{T}^T\ma{T}$.

\section{applying PCA to $\ma{S}$}

Using the same sample data as above, we imagine that the set of points $\textbf{S}$ represents the measures of a moving element. The object actually moves only along the ellipsoid axis $\ve{e} = \begin{m}-1 & 2 \end{m}$, the deviations from this axis is just noise. The goal of PCA is to re-express our set of data in a new basis that reflects this behaviour (data along $\ve{e}$, noise along the other direction). The naive basis (in which we recorded the data) is~:
\[
  \ma{B} = \begin{m} \ve{x} \\ \ve{y} \end{m} = \begin{m} 1 & 0 \\ 0 & 1 \end{m} = \ma{I}
\]

We will use PCA to find a new basis $\ma{P}$ that is a linear combination of the original basis $\ma{B}$ and in which we will only look at the data along the greatest axis of the ellipsoid (where the data exhibits the greatest variance). In this new basis, the set of data $\ma{S}$ becomes~:
\[
  \ma{S'} = \ma{S}\ma{P}
\]

The new basis should optimise signal to noise ratio if we consider the first dimension as representing the signal and the second the noise~:
\[
  SNR = \frac{\sigma_{signal}^2}{\sigma_{noise}^2}
\]

Before delving any further, we must first transform our data into its \emph{mean deviation form} $\ma{T}$.
\[
  \ma{T} = \ma{S} - \begin{m}\ve{\mu} \\ \vdots \\ \ve{\mu}\end{m} = \begin{m} 0 & -1 \\ -1 & 2 \\ 1 & -1\end{m}.
\]

We can now easily compute the actual SNR~:
\[
  SNR = \frac{\sum_{k=1}^l T_{k1}^2}{\sum_{k=1}^l T_{k2}^2} = \frac{0+1+1}{1+4+1} = \frac{1}{3}
\]

How do we find the new basis that gives us the greatest SNR ? Expressed in another way, we could say~: how do we find the basis that shows greatest variance in the first dimension and mostly noise in the second ? From the preceding section, we know how to build $\ma{P}$. Let's do it with our data. First we need to find the symmetric matrix $\ma{A}$~:
\begin{align*}
  \ma{A} = & \ma{T}^T\ma{T} = \begin{m} 0 & -1 & 1 \\ -1 & 2 & -1 \end{m} \begin{m} 0 & -1 \\ -1 & 2 \\ 1 & -1\end{m} \\
         = & \begin{m} 2 & -3 \\ -3 & 6 \end{m}
\end{align*}

Now we must find the eigenvalues of this matrix~:
\begin{align*}
  det(\ma{A} - \lambda\ma{I}) = & 0 \\
  det(\begin{m} 2-\lambda & -3 \\ -3 & 6-\lambda \end{m}) = & 0 \\
  (2-\lambda)(6-\lambda) - 9 = & 0 \\
  \lambda^2 - 8\lambda + 3 = & 0 \\
  \lambda_1 \approx 7.6, \lambda_2 \approx 0.4.
\end{align*}

The corresponding eigenvectors are~:

\begin{align*}
  \begin{m}x & y\end{m}\begin{m} 2-\lambda & -3 \\ -3 & 6-\lambda \end{m} = & 0 \\
  \left\{ \begin{array}{rl} x = & \frac{3}{2-\lambda} t \\ y = & t \end{array} \right.
\end{align*}

We want normed vectors so we add the rule $x^2 + y^2 = 1$ and we get~:
\begin{align*}
  \frac{9t^2}{(2 - \lambda)^2} + t^2 = & 1 \\
  t = & \frac{2-\lambda}{\sqrt{9 + (2 - \lambda)^2}} \\
  t_1 \approx & -0.9 \\
  t_2 \approx & 0.5 \\
  \ma{E} \approx & \begin{m}0.5 & -0.9 \\ 0.9 & 0.5 \end{m}.
\end{align*}

Now that we have our transformation matrix $\ma{P}$
\[
  \ma{P} = \ma{E}^T = \begin{m}0.5 & 0.9 \\ -0.9 & 0.5 \end{m}.
\]
we can transform our data $\ma{T}$ into $\ma{T}'$~:
\begin{align*}
  \ma{T}' = & \ma{T}\ma{P} \approx \begin{m} 0 & -1 \\ -1 & 2 \\ 1 & -1\end{m}\begin{m}0.5 & 0.9 \\ -0.9 & 0.5 \end{m} \\
    \approx & \begin{m} 0.9 & -0.5 \\ -2.2 & 0.1 \\ 1.4 & 0.4 \\ \end{m}
\end{align*}

With this new transformation, we should have great variance along the first dimension and a very small one in the second (that was our goal). By looking at the values, this seems to be the case. Let's see with a picture.

\includegraphics{points2.pdf}

Just to make sure our work was correct, let's compute the variance-covariance matrix $\ma\Sigma'$~:
\[
  \ma\Sigma' = \frac{1}{n-1}\ma{T'}^T\ma{T'} \approx \begin{m} 7.6 & 0 \\ 0 & 0.4 \end{m}.
\]

It works ! And by the way, this is $\ma{D}$ which is the diagonal matrix made out of the eigenvalues of $\ma{A} = \ma{T}^T\ma{T}$ ($7.6$ and $0.4$).

To finish our goal of dimensionality reduction, we can build another version of the matrix $\ma{P}$ that automatically strips off the ``noisy'' dimensions, only keeping the principal components of our data~:
\[
  \ma{Q} = \ma{P}\begin{m} 1 \\ 0 \end{m} = \begin{m}0.5 \\ -0.9\end{m}.
\]

And before closing this section, we need to compute our new signal to noise ratio~:
\[
  SNR = \frac{\sum_{k=1}^l {T'}_{k1}^2}{\sum_{k=1}^l {T'}_{k2}^2} \approx \frac{7.6}{0.4} = 19.3.
\]

By just looking at the $\ve{y}$ axis, we would have obtained $3$. Applying the transformation $\ma{P}$ made the signal much easier to analyse.

That's it (I'm glad I made it this far) !

\subsection{summary on PCA}

PCA can be used as a filter to reduce the dimensionality of the data and ease the computation of the \textbf{Mahalanobis distance} as this distance becomes the \textbf{normalized Euclidean distance} with the new transformation since the covariances have been removed ($\ma\Sigma'$ is a diagonal matrix). The calculation of the transformation matrix $\ma{P}$ involves the following computations~:
\begin{itemize}
  \item Find the mean vector $\ve\mu$.
  \item Substract this vector to each vector of the training set $\ma{S}$ to obtain the \emph{mean deviation form} $\ma{T}$.
  \item Find the eigenvectors of the symmetric matrix $\ma{T}^T\ma{T}$, sorted by decreasing eigenvalues.
  \item Build the transformation matrix $\ma{P} = \begin{m}\ve{e_1}^T & \ve{e_2}^T & \hdots & \ve{e_k}^T \end{m}$ with $k \leq n$ to keep the $k$ most relevant dimensions of the transformed data.
  \item Apply this transformation $\ma{P}$ to any test vector $\ve{t}$ before computing the normalized Euclidean distance to the mean value $\ve\mu$.
  \item The smaller the distance, the bigger the probability that $\ve{t}$ belongs to the set $\ma{S}$.
\end{itemize}

Among all the steps above, two are problematic for large matrices: finding the eigenvalues and eigenvectors. These steps imply solving the \emph{characteristic polynomial} of $\ma{A}$. If our data contains 12 measures x 100 samples per row, the dimension of our polynomial is 1200 ! To find the eigenvector we have to solve n times a system of n-1 equations. This can become daunting. That was for the bad news. The good news is that these calculations must only be done on the training set, not live. The even better news is that there exists a library written in Fortran called \emph{LAPACK} that can do the job as fast as the most clever people could make it. Thanks.
\subsection{notes on PCA}

For a deaper understanding of PCA and it's limits, one may have a look at~:
\begin{itemize}
  \item \emph{kernel PCA} to apply a non-linear filter prior to applying PCA for problems where linearity is an issue.
  \item \emph{ICA} for sets of data whose distribution probabilities are not exponential (Gaussian, Exponential, etc).
  \item \emph{Central Limit Theorem} on why PCA should work most of the time in real world as probabilities usually \textbf{are} gaussian.
\end{itemize}

\section{Movement recognition}

From the knowledge we have gathered above, we can now try to find a way to compare the movements. Imagine we have 3 movements (classes) labeled $a$, $b$ and $c$. Each movement has been recorded several times. We create the training sets $\ma{S_a}$, $\ma{S_b}$ and $\ma{S_c}$ with the recordings of the different movements. Each move is made of 2 measures recorded for 10 samples. We write it as a row vector of dimension 20 ($\ve{s_{ai}}$).

\section{Filtering (PCA)}

We start by computing the mean vector for each movement. We will note the number of recordings for the current movement $r$~:
\[
  \ve\mu_a = \frac{1}{r}\sum_{k=1}^r \begin{m}\ve{s_{a1k}} & \ve{s_{a2k}} & \hdots & \ve{s_{a20k}} \end{m}
\]
We do this for each class $a$, $b$ and $c$. We then build a matrix out of these prototypes~:
\[
  \ma{S} = \begin{m}\ve\mu_a \\ \ve\mu_b \\ \ve\mu_c \end{m}
\]

We now find the mean deviation form of $\ma{S}$ after finding the mean value of all the prototypes ($\ve\mu$)~:
\[
  \ma{T} = \begin{m}\ve\mu_a - \ve\mu \\ \ve\mu_b - \ve\mu \\ \ve\mu_c - \ve\mu \end{m}
\]

Now that we have found the \emph{least varying parts} of each movement (the ``platonic idea'') by computing the average, we will find a transformation that shows the \emph{most varying parts} \textbf{between} these movements. We apply PCA on the matrix $\ma{T}$ and find the transformation matrix $\ma{P}$.
\[
  \ma{P} = \ma{E}^T \ma{I_{reduc}}
\]
Where $\ma{I_{reduc}}$ is a truncated identity matrix to reduce the dimensionality of the data to the principal components.

\section{Recognition}

To label new vectors, we could use the Mahalanobis distance or Euclidean distance in the transformed vector space produced by PCA. This is the first solution discussed below. The other approach is to use \emph{Support Vector Machines}. This is the solution which we will try to use if our hardware is fast enough since it produces better results.

\subsection{Distances}

We now have two options. Either we use the information contained in the eigenvalues (matrix $\ma{D}$) to compute the normalized Euclidean distances between our test vector and the prototypes in the new coordinate system defined by the transformation $\ma{P}$ or we map all our training data into the new coordinate system and compute the Mahalanobis distance to each set. This last solution loses some information from the \emph{differences between the different classes $a$, $b$ and $c$} but takes into account what is noise and what is a signal in each movement. Since we have already included the information on what makes the differences between the classes when we chose to remove the dimensions with least variance, the second solution seems to keep most of the information contained in the training data.

Transforming each training set into the new coordinate system is done by
\begin{align*}
  \ma{S_{a}'} = & \ma{S_a}\ma{P} \\
  \ma{S_{b}'} = & \ma{S_b}\ma{P} \\
  \ma{S_{c}'} = & \ma{S_c}\ma{P} \\
\end{align*}

We now compute the variance-covariance for each of these matrices~:
\begin{align*}
  \ma\Sigma_a' = & \frac{1}{r-1}\ma{S_{a}'}^T\ma{S_{a}'} \\
  \ma\Sigma_b' = & \frac{1}{r-1}\ma{S_{b}'}^T\ma{S_{b}'} \\
  \ma\Sigma_c' = & \frac{1}{r-1}\ma{S_{c}'}^T\ma{S_{c}'} \\
\end{align*}

Normalization of the values in these matrices ?
\begin{align*}
  \ma\Sigma_{aij}' = & \frac{\ma\Sigma_{aij}'}{\sum_{k=1}^3 \ma\Sigma_{kij}'}
\end{align*}

To label an arbitrary movement $\ve{t}$, we find the minimal Mahalanobis distance to the sets in the new coordinate system.
\begin{align*}
  \ve{t_k'} = & \ve{t'} - \ve{\mu_k'} \\
            = & \ve{t}\ma{P} - \ve{\mu_k'} \\
  \argmin_{k} D_M(\ve{t}) = & \argmin_{k} \sqrt{\ve{t_k'}\Sigma_k'^{-1}\ve{t_k'}^T}
\end{align*}

The value returned by argmin ($k$) is the label for the test movement.

\subsection{Support Vector Machines}

\emph{Support Vector Machines} is a new mathematical tool for pattern recognition. Basically, it involves mapping the data into a new vector space (once more) with higher dimensions this time. The goal is to find a vector space in which the data is linearly separable (you can draw a line/plane/... between the different classes). For example, imagine you have two sets of data $a$ and $b$: all values for set $a$ are inside a circle located on the origine and all values for $b$ are outside of this circle. If we add a third dimension that is the distance to the origine, we will obtain a cone pointing downward. All points from the set $a$ are located below a certain height and all point from set $b$ are above this height. We can thus split the set using a plane that cuts the cone at this height.

To apply SVM, we first filter all our data set with PCA:
\begin{align*}
  \ma{S_{a}'} = & \ma{S_a}\ma{P} \\
  \ma{S_{b}'} = & \ma{S_b}\ma{P} \\
  \ma{S_{c}'} = & \ma{S_c}\ma{P} \\
\end{align*}

We then feed the svm library\footnote{http://www.csie.ntu.edu.tw/~cjlin/libsvm} (we could never have written this code) with two arrays~: a list of labels and a list of vectors. We thus feed it with set $S_{a}'$, telling it these vectors have the label ``a'', $S_{b}'$ with label ``b'' and $S_{c}'$ with label ``c''. We then call the learn function.

To test an arbitrary movement $\ve{t}$, we must first move it into the vector space defined by PCA and then send it into SVM. The SVM black box will return the label for the test vector.
\begin{align*}
  \ve{t_k'} = & \ve{t'} - \ve{\mu_k'} \\
            = & \ve{t}\ma{P} - \ve{\mu_k'} \\
  \text{label} = & \text{svmlib.predict}(\ve{t_k'})
\end{align*}

\onecolumn

\section{Ruby code}
\subsection{variance}
\begin{verbatim}
set   = [1,4,3,6]
mu    = set.inject(0) {|s,v| s + v } / set.size.to_f
sigma = set.inject(0) {|s,v| s + ((v - mu)**2) } / set.size.to_f
\end{verbatim}

\subsection{Mahalanobis distance}
\begin{verbatim}
class Array
  # transpose
  def t
    cols = []
    self[0].each_index do |i|
      cols[i] = map {|v| v[i]}
    end
    cols
  end

  def *(m)
    res = []
    each_index do |row|
      res[row] = []
      m[0].each_index do |col|
        res[row][col] = 0
        self[0].each_index do |i|
          res[row][col] += self[row][i] * m[i][col]
        end
      end
    end
    res
  end

  def det
    (self[0][0] * self[1][1]) - (self[1][0] * self[0][1])
  end

  def /(scalar)
    map {|row| row.map {|v| v / scalar}}
  end

  # vector substraction
  def -(v)
    res = [[]]
    self[0].each_index {|i| res[0][i] = self[0][i] - v[0][i]}
    res
  end

  def to_s
    if self.size > 1 || self[0].size > 1
      if self.size == 1
        row_format = '[' + (" %3.1f" * self[0].size) + " ]\n"
      else
        row_format = '|' + (" %3.1f" * self[0].size) + " |\n"
      end
      self.inject("") {|s,row| s + sprintf(row_format, *row)}
    else
      self[0][0].to_s
    end
  end

  def map_index(&block)
    res = []
    self.each_index do |i|
      res[i] = yield(i,self[i])
    end
    res
  end
end

set = [[1,1],
       [0,4],
       [2,1]]
# mean vector mu
mu = [[]]
set[0].each_index do |i|
  mu[0][i] = set.inject(0) {|s,v| s + v[i] } / set.size.to_f
end

# set - mu
s_mu = set.map do |v|
  r = []
  v.each_index do |i|
    r[i] = v[i] - mu[0][i]
  end
  r
end

# variance-covariance matrix
sigma = (s_mu.t * s_mu) / (set.size - 1.0).to_f

# inverse of sigma
if sigma.det == 0
  puts "Matrix not invertible"
  return
end

# invers of sigma (only for a 2x2 matrix)
sigma_inv = [[  sigma[1][1], -sigma[0][1] ],
             [ -sigma[1][0],  sigma[0][0] ]] / sigma.det

puts "Sigma\n\n"
puts sigma.to_s
puts "\nSigma's inverse\n\n"
puts sigma_inv.to_s

puts "\n\nAll results express value^2 (before square root)"

[
[[[2  , 2  ]],
[ [1  , 3  ]]],
[[[1.5, 1.5]],
[ [1.5, 2.5]]]
].each do |t1,t2|
  puts "\n-----------------------------"
  puts "t1   = #{t1}"
  puts "t2   = #{t2}"
  puts "t1-u = #{t1 - mu}"
  puts "t2-u = #{t2 - mu}\n\n"

  d1 = (t1 - mu).flatten.inject(0) {|s,v| s + v**2}
  d2 = (t2 - mu).flatten.inject(0) {|s,v| s + v**2}

  puts "d1   = #{d1}"
  puts "d2   = #{d2}\n\n"

  d1nE = (t1 - mu).flatten.map_index{|i,v| v**2 / sigma[i][i] }.inject(0) {|s,v| s + v }
  d2nE = (t2 - mu).flatten.map_index{|i,v| v**2 / sigma[i][i] }.inject(0) {|s,v| s + v }

  puts "d1nE = #{d1nE}"
  puts "d2nE = #{d2nE}\n\n"

  d1M = (t1 - mu) * sigma_inv * (t1 - mu).t
  d2M = (t2 - mu) * sigma_inv * (t2 - mu).t

  puts "d1M  = #{d1M}"
  puts "d2M  = #{d2M}\n\n"
end
\end{verbatim}
\end{document}


% -----------------------------------------------------------------------------------
% Section   :
% -----------------------------------------------------------------------------------

\section*{Data Structure}

First we describe the setting and data structure. We have a stream of items $\set{i_1, i_2, ...}$ where each $i$ comes from some finite universe $\pmb{I} = \set{1,..,i,..n}$, and we are interested in the approximate frequence of each $i$. The Count-Min Sketch algorithms $Init$ialize a fixed array of counters, of width $w$ and depth $d$, where the counters are initialized to all zeros. Each row $j$ of counters is associated with a different pairwise independent hash function $\mathcal{H}_j : i \send \set{1,..,w}$ mapping items $i$ uniformly onto the column index. Now the hash functions do not need to be particularly strong a la cryptographic hash functions. In our setting where items are simply integers, we have:
	\[
		\mathcal{H}_j = a_j \times i + b_j \quad mod \quad p \quad mod \quad w
	\]

where p is some prime number far larger than $n$, say $p = 2^n - 1$, and $a_j$ and $b_j$ are chosen uniformly from $(1,p)$. But note each $\mathcal{H}_j$ must be different, otherwise there is no benefit from the repetition.\newline

\section*{Functions over the data structure}

Once the counters or sketch and hash functions are initalized, we $Update$ the sketch as items $i$ come in from the stream. Meanwhile we may $Estimate$ the frequency of any $i$ using our counters and hash table. Consider the pseudocode below. \newline

\begin{algorithm}
\caption{Init(w,d,p)}\label{euclid}
\begin{algorithmic}[1]
	\State $C[1,1]...C[d,w] \gets 0$
	\For {$j \gets 1 \textit{ to d}$}
		\State \textit{pick $a_j,b_j$ uniformly from $[1..p]$}
	\EndFor
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
\caption{Update(i)}\label{euclid}
\begin{algorithmic}[1]
	\For {$j \gets 1 \textit{ to d}$}
		\State $h_j(i) = (a_j \times i + b_j) \quad mod \quad p \quad mod \quad w$
		\State $C[j,h_j(i)]  \leftarrow C[j,h_j(i)] + 1$
	\EndFor
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
\caption{Estimate(i)}\label{euclid}
\begin{algorithmic}[1]
	\State $e \gets \infty$
	\For {$j \gets 1 \textit{ to d}$}
		\State $h_j(i) = (a_j \times i + b_j) \quad mod \quad p \quad mod \quad w$
		\State $e \leftarrow min(e,C[j,h_j(i)]$
	\EndFor
	\State $return \quad e$
\end{algorithmic}
\end{algorithm}


\documentclass[12pt]{article}
\usepackage[fleqn]{amsmath}
\usepackage{tikz}
\usepackage{subfiles}

\title{\vspace{-4\baselineskip}MATH 225 - Homework \#2}
\author{D. Choi}
\date{2020-05-22}

%\pagenumbering{gobble}

\begin{document}
\maketitle

\section*{1.}
\textit{Find the matrix that reflects space across the
$y = \tan(25^\circ)x$
line.
} \\[\baselineskip]
Let $A$ be such a matrix. \\
The elements of $A$ can be found by computing where the standard basis vectors
will land after being reflected across the
$y = \tan(25^\circ)x$
line.
\begin{equation*}
	A
	\begin{pmatrix}
		1 \\
		0
	\end{pmatrix}
	=
	A
	\begin{pmatrix}
		\cos0 \\
		\sin0
	\end{pmatrix}
	=
	\begin{pmatrix}
		\cos(2 \cdot 25^\circ) \\
		\sin(2 \cdot 25^\circ)
	\end{pmatrix}
	=
	\begin{pmatrix}
		\cos(50^\circ) \\
		\sin(50^\circ)
	\end{pmatrix}
\end{equation*}
\begin{equation*}
	A
	\begin{pmatrix}
		0 \\
		1
	\end{pmatrix}
	=
	A
	\begin{pmatrix}
		\cos(90^\circ) \\
		\sin(90^\circ)
	\end{pmatrix}
	=
	\begin{pmatrix}
		\cos(90^\circ - 2 \cdot 65^\circ) \\
		\sin(90^\circ - 2 \cdot 65^\circ)
	\end{pmatrix}
	=
	\begin{pmatrix}
		\cos(-40^\circ) \\
		\sin(-40^\circ)
	\end{pmatrix}
\end{equation*}
\begin{equation*}
	A =
	\boxed{
		\begin{pmatrix}
			\cos(50^\circ) & \cos(-40^\circ) \\
			\sin(50^\circ) & \sin(-40^\circ)
		\end{pmatrix}
	}
\end{equation*}

\section*{2.}
\begin{equation*}
	M =
	\begin{pmatrix}
		\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}} \\
		-\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
	\end{pmatrix}
\end{equation*}
\textit{Find $M^2$, $M^3$, $M^4$, $M^{17}$.} \\[\baselineskip]
Let $R(\theta)$ be the rotation matrix where
\begin{equation*}
	R(\theta) =
	\begin{pmatrix}
		\cos \theta & -\sin \theta \\
		\sin \theta & \cos \theta
	\end{pmatrix}
	.
\end{equation*}
$M$ can be considered as a specific instance of $R(\theta)$ where
$\theta = -45^\circ$.
\begin{equation*}
	M =
	\begin{pmatrix}
		\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}} \\
		-\frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
	\end{pmatrix}
	=
	\begin{pmatrix}
		\cos(-45^\circ) & -\sin(-45^\circ) \\
		\sin(-45^\circ) & \cos(-45^\circ)
	\end{pmatrix}
	=
	R(-45^\circ)
\end{equation*}
Because $R(\theta)$ has the property of
\begin{equation*}
	(R(\theta))^n = R(\theta n),
\end{equation*}
it can be said for $M$ that
\begin{equation*}
	M^n = R(-45^\circ \cdot n).
\end{equation*}
Thus,
\begin{gather*}
	\boxed{M^2 = R(-45^\circ \cdot 2) = R(-90^\circ)}, \\
	\boxed{M^3 = R(-45^\circ \cdot 3) = R(-135^\circ)}, \\
	\boxed{M^4 = R(-45^\circ \cdot 4) = R(-180^\circ)},\text{ and} \\
	\boxed{M^{17} = R(-45^\circ \cdot 17) = R(-45^\circ) = M} .
\end{gather*}

\section*{3.}
\textit{Which is linear?}
\begin{gather*}
	f(x,y) = \langle 3x^2 + y, 2x + y \rangle \\
	g(x,y) = \langle 3x + y, x + 3y \rangle
\end{gather*}
\textit{Find the corresponding matrix for the linear transformation.}
\bigskip
\begin{equation*}
	f(2
	\begin{pmatrix}
		1 \\
		0
	\end{pmatrix}
	)
	=
	f
	\begin{pmatrix}
		2 \\
		0
	\end{pmatrix}
	=
	\begin{pmatrix}
		3(2)^2 + (0) \\
		2(2) + 0
	\end{pmatrix}
	=
	\begin{pmatrix}
		12 \\
		4
	\end{pmatrix}
\end{equation*}
\begin{equation*}
	2f
	\begin{pmatrix}
		1 \\
		0
	\end{pmatrix}
	=
	2
	\begin{pmatrix}
		3(1)^2 + 0 \\
		2(1) + 0
	\end{pmatrix}
	=
	2
	\begin{pmatrix}
		3 \\
		2
	\end{pmatrix}
	=
	\begin{pmatrix}
		6 \\
		4
	\end{pmatrix}
\end{equation*}
As
\begin{equation*}
	f(2
	\begin{pmatrix}
		1 \\
		0
	\end{pmatrix}
	)
	\neq
	2f
	\begin{pmatrix}
		1 \\
		0
	\end{pmatrix},
\end{equation*}
$f$ is not homogeneous of degree 1, a property which linearity requires. \\
Thus, $f$ is not linear.
\begin{equation*}
	g
	\begin{pmatrix}
		x \\
		y
	\end{pmatrix}
	=
	\begin{pmatrix}
		3x + y \\
		x + 3y
	\end{pmatrix}
	=
	\boxed{
		\begin{pmatrix}
			3 & 1 \\
			1 & 3
		\end{pmatrix}
		\begin{pmatrix}
			x \\
			y
		\end{pmatrix}
	}
\end{equation*}

\section*{4.}
\textit{Use geometry (graph paper) to find}
\begin{equation*}
	\begin{pmatrix}
		4 & 1 \\
		1 & 4
	\end{pmatrix}
	\begin{pmatrix}
		1 \\
		1
	\end{pmatrix}
	.
\end{equation*}
\bigskip
\subfile{fig}
\begin{equation*}
	\begin{pmatrix}
		4 & 1 \\
		1 & 4
	\end{pmatrix}
	\begin{pmatrix}
		1 \\
		1
	\end{pmatrix}
	=
	\boxed{
		\begin{pmatrix}
			5 \\
			5
		\end{pmatrix}
	}
\end{equation*}

\end{document}


\documentclass[aspectratio=169]{beamer}

% because we need to claim weird things
\newtheorem{claim}{Claim}
\newtheorem{defn}{Definition}
%\newtheorem{lemma}{Lemma}
\newtheorem{thm}{Theorem}
\newtheorem{vita}{Vit\ae}
\newtheorem{qotd}{Quote of the Day}
\renewcommand{\qedsymbol}{$\blacksquare$}

\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{listings}
\usepackage{color}
\usepackage{graphics}
\usepackage{ulem}
\bibliographystyle{unsrt}

% background image
\usebackgroundtemplate%
{%
    \includegraphics[width=\paperwidth,height=\paperheight]{../artifacts/stemulus.pdf}%
}
\setbeamertemplate{caption}[numbered]
\lstset{%
	breaklines=true,
	captionpos=b,
	frame=single,
	keepspaces=true,
	showstringspaces=false
}

% page numbers
\addtobeamertemplate{navigation symbols}{}{%
    \usebeamerfont{footline}%
    \usebeamercolor[fg]{footline}%
    \hspace{1em}%
    \insertframenumber/\inserttotalframenumber
}

% presentation header
\usetheme{Warsaw}
\title{Big $\mathcal{O}$ Notation}
\author{Dylan Lane McDonald}
\institute{CNM STEMulus Center\\Web Development with PHP}
\date{\today}

\begin{document}
\lstset{language=Java}
\begin{frame}
\titlepage
\end{frame}

\begin{frame}
\frametitle{Outline}
\tableofcontents
\end{frame}

\section{Mathematics}
\subsection{Limits}
\begin{frame}
\frametitle{Limits}
Intuitively, the \textbf{limit} of a function $f(x)$ as $x$ approaches a value $c$ is where the function will arrive at $c$. Consider the function:
\begin{equation}
f(x) = \frac {x - 1}{x - 1}
\label{eqn:ratio}
\end{equation}
At $f(1)$, we get $f(1) = \frac 00$, which does not exist. But, as we approach for values $x \pm \varepsilon$ for $\varepsilon \in \mathbb{R}^+$ and $\varepsilon$ is very small, $f(x) = 1$. Informally, since both sides ``agree'' on the fact $f(x)$ is really tending toward $1$, we define the limit of $f(x)$ as $x$ approaches 1 to be 1. Symbolically, this is written:
\[
\lim_{x \rightarrow 1} f(x) = \lim_{x \rightarrow 1} \left(\frac {x - 1}{x - 1}\right) = 1
\]
\end{frame}

\begin{frame}
\frametitle{Limits to Infinity}
In computer science, the most useful limit to consider is the limit of a function as it approaches infinity. That is, as the function's input grow larger, where does the function approach? Most algebraic and transcendental functions will also tend toward $\infty$. Consider a function that doesn't tend toward $\infty$:
\[
\lim_{x \rightarrow \infty} \left(\frac 1x\right) = 0
\]
Intuitively, this is so because as the denominator gets larger and larger, it ``dominates'' the one in the numerator and brings the ratio closer and closer to zero.
\end{frame}

\section{Algorithmic Complexity}
\begin{frame}
\frametitle{Algorithmic Complexity: English}
Intuitively, computer scientists are interested in how long it takes a computer to solve a problem. But how long something takes in real time is too complicated to address because run times will vary from Windows to Linux to iPads, etc. Instead, the theoretical bounds on the problem with respect to the input size is studied. This is an introduction to a broad field called \textbf{algorithmic analysis}.

\mbox{}\\
\pause
The main measure of algorithmic complexity is known as $\mathcal{O}$ (``Big O'') notation. The intuitive question $\mathcal{O}$ sets out to answer is, ``What is the longest I can expect this program to take?'' That is, as the problem grows in size, how much longer will this program take to process the additional input?
\end{frame}

\subsection{Graphical}
\begin{frame}
\begin{figure}
\includegraphics[scale=0.21]{../artifacts/big-o-knuth.png}
\caption{Adventures in $\mathcal{O}()$}
\label{fig:xkcd}
\end{figure}
\end{frame}

\begin{frame}
\frametitle{Algorithmic Complexity: Graphical}
\begin{figure}
\includegraphics[scale=0.725]{../artifacts/big-o-plot.pdf}
\caption{Two $\mathcal{O}()$ Functions}
\label{fig:big-o}
\end{figure}
\end{frame}

\subsection{Algebra}
\begin{frame}
\frametitle{Algorithmic Complexity: Algebraic}
Suppose the two functions in Figure \ref{fig:big-o} are two different programs that solve the same problem. The $\mathcal{O}(n)$ function is more efficient for small inputs and can be used when data sets a very small. The $\mathcal{O}(\log n)$\footnote{In computer science, logarithms are base 2.} is more scaleable and suitable for larger inputs.

\mbox{}\\
\pause
Note the $\mathcal{O}(n)$ function is not exactly $f(x) = x$, but instead $f(x) = \frac 14 x$. This sets up one of the fundamental problems $\mathcal{O}$ answers: Given $c \in \mathbb{R}^+$ and $n_0 \in \mathbb{R}^+$, when does is function $c \cdot f(n) \le g(n)$ for all $n \ge n_0$? If such $c$ and $n_0$ exist, it follows that $f(n) = \mathcal{O}(g(n))$.

\mbox{}\\
\pause
In the example in Figure \ref{fig:big-o}, $\log(n)$ is $\mathcal{O}(n)$ for $c = \frac 14$ and $n_0 = 16$.
\end{frame}

\subsection{Calculus}
\begin{frame}
\frametitle{Algorithmic Complexity: Calculus}
Let $f(n)$ and $g(n)$ be two functions that return positive real numbers (i.e., $f : \mathbb{D}_f \rightarrow \mathbb{R}^+$ and $g : \mathbb{D}_g \rightarrow \mathbb{R}^+$). Define $f(n)$ as $\mathcal{O}(g(n))$ if and only if:
\begin{equation}
\lim_{n \rightarrow \infty} \left| \frac{f(n)}{g(n)}\right| < \infty
\label{eqn:limit}
\end{equation}
That is, as $n$ approaches $\infty$ in Equation \ref{eqn:limit}, the quotient of the algorithm and the function will approach some value $\alpha$. If $\alpha$ is a finite number, this implies that $f(n)$ and $g(n)$ are ``sufficiently similiar'' for the algorithm $f(n)$ as $\mathcal{O}(g(n))$. On the other hand, if $\alpha = \pm \infty$ (and the limit in Equation \ref{eqn:limit} therefore does not exist), then $f(n)$ and $g(n)$ are not similar and it is not the case that $f(n)$ is $\mathcal{O}(g(n))$.
\end{frame}

\begin{frame}
\frametitle{Common $\mathcal{O}()$ Values}
\begin{table}
\begin{tabular}{|l|l|l|}
\hline
\textbf{Value} & \textbf{Name} & \textbf{Example}\\
\hline
$\mathcal{O}(1)$ & Constant & Accessing an array member\\
\hline
$\mathcal{O}(\log n)$ & Logarithmic & Searching an index or tree\\
\hline
$\mathcal{O}(n)$ & Linear & Na\"{i}vely searching an array\\
\hline
$\mathcal{O}(n \log n)$ & Loglinear & Sorting an array\\
\hline
$\mathcal{O}(n^2)$ & Quadratic &  Matrix multiplication\\
\hline
$\mathcal{O}(n^3)$ & Cubic & Finding a determinant of a matrix\\
\hline
$\mathcal{O}(a^n)$ & Exponential & Traveling salesman: dynamic programming\\
\hline
$\mathcal{O}(n!)$ & Factorial & Traveling salesman: directly\\
\hline
\end{tabular}
\caption{Common $\mathcal{O}$ Values \& Examples}
\label{tbl:values}
\end{table}
\end{frame}

\subsection{Impact}
\begin{frame}
\frametitle{Impact}
Each line of code has an associated $\mathcal{O}()$ value. As each line of code executes, the performance impact is felt. For instance, if we have three lines of code: two of which are $\mathcal{O}(n)$ and one is $\mathcal{O}(n \log n)$, the program is $\mathcal{O}(n \log n)$ since the $\mathcal{O}(n \log n)$ is larger and ``dominates'' the $\mathcal{O}(n)$ lines of code.

\mbox{}\\
\pause
A loop has the general effect of multiplying the complexity into $n$. Using the three lines of code in the previous example, there are two lines of code that are $\mathcal{O}(n^2)$ and one that is $\mathcal{O}(n^2 \log n)$. Again, the $\mathcal{O}(n^2 \log n)$ ``dominates'' and the program is $\mathcal{O}(n^2 \log n)$.
\end{frame}

\begin{frame}
\frametitle{Example}
\begin{theorem}
A function that has an exact run time of $f(n) = 2^{n + 10}$ is $\mathcal{O}(2^n)$.
\end{theorem}
\begin{proof}
Rewrite $f(n)$ as $f(n) = 2^{10} \cdot 2^n = 1024 \cdot 2^n$. Defining the limit as seen in Equation \ref{eqn:limit}, we get:
\[
\lim_{n \rightarrow \infty} \left| \frac{1024 \cdot 2^n}{2^n} \right| = \lim_{n \rightarrow \infty} \left| 1024 \right| = 1024
\]
Since, the limit exists, it follows that $f(n)$ is $\mathcal{O}(2^n)$.
\end{proof}
It should be noted that, in this case, $c = 1024$ and $n_0 = 10$,\\
similar to what was seen in the Algebraic section.
\end{frame}

\begin{frame}
\frametitle{Final Thought}
The selection of algorithms and data structures often leads to very large consequences in how programs, whether on or off the web, perform. These are measured in $\mathcal{O}()$ form. Knowing common complexities for every day tasks is valuable and facilitates the informed choices of which algorithms to use as well being a powerful tool in performance optimization of programs.

\mbox{}\\
Also, carefully consider using slower algorithms within loops, as they tend to be exacerbated by the fact loops generally introduce a factor of $n$ to any operation being performed. This has been one of the largest sources of slow down in my professional experience.
\end{frame}
\end{document}


% !TeX root = ../main.tex
\documentclass[../main.tex]{subfiles}
\begin{document}
\section{Model Foundation}
\subsection{Trigonometry}
\begin{definition}\label{M:Trigonometry}
\textit{Generalized trigonometric functions} \(f_\lambda\colon \R\to\R\) and \(f_\lambda^\ast:\R\to\R\) are defined as
\begin{align*}
f_\lambda\left(\theta\right)&\coloneqq
\begin{cases}
g\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
h\left(\lambda\theta\right)&\text{otherwise,}\\
\end{cases}\\
f_\lambda^\ast\left(\theta\right)&\coloneqq
\begin{cases}
g\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
h\left(-\lambda\theta\right)&\text{otherwise,}\\
\end{cases}
\end{align*}
where \(g\) (resp. \(h\)) are the associated trigonometric (resp. hyperbolic) function.
\end{definition}
\begin{example}[Generalized sine functions]\label{M:Trigonometry:Sine}
\begin{align*}
\sin_\lambda{\theta}
&\coloneqq
\begin{cases}
\sin\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
\sinh\left(\lambda\theta\right)&\text{otherwise.}\\
\end{cases}\\
\sin_\lambda^\ast{\theta}
&\coloneqq
\begin{cases}
\sin\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
\sinh\left(-\lambda\theta\right)&\text{otherwise.}\\
\end{cases}\\
&=
\begin{cases}
\sin\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
-\sinh\left(\lambda\theta\right)&\text{otherwise.}\\
\end{cases}\\
&=\begin{cases}
\sin\left(\abs{\lambda}\theta\right)&\text{if \(\lambda\geq0\),}\\
\sinh\left(\abs{\lambda}\theta\right)&\text{otherwise.}\\
\end{cases}
\end{align*}
(see\cref{TrigonometrySinePlotted})
\end{example}
\begin{example}[Generalized cosine functions]\label{M:Trigonometry:Cosine}
\begin{align*}
\cos_\lambda{\theta}
&\coloneqq
\begin{cases}
\cos\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
\cosh\left(\lambda\theta\right)&\text{otherwise.}\\
\end{cases}\\
\cos_\lambda^\ast{\theta}
&\coloneqq
\begin{cases}
\cos\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
\cosh\left(-\lambda\theta\right)&\text{otherwise.}\\
\end{cases}\\
&=\begin{cases}
\cos\left(\lambda\theta\right)&\text{if \(\lambda\geq0\),}\\
\cosh\left(\lambda\theta\right)&\text{otherwise.}\\
\end{cases}\\
&=\cos_\lambda{\theta}
\end{align*}
(see\cref{TrigonometryCosinePlotted})
\end{example}
\begin{theorem}[Pythagorean's identity equivalence]\label{M:Trigonometry:Pythagorean}
\begin{align*}
\cos_\lambda^2{\theta}+\sign{\lambda}\sin_\lambda^2{\theta}&=1\\
\sec_\lambda^2{\theta}-\sign{\lambda}\tan_\lambda^2{\theta}&=1\\
\end{align*}
\end{theorem}
\begin{proof}[\proofof{M:Trigonometry:Pythagorean}]
Proof by exhaustion.
\end{proof}
\begin{proposition}[Generalized trigonometric functions of sum of arguments]\label{M:Trigonometry:Sum}
\begin{align*}
\sin_\lambda\left(\theta+\phi\right)
&=\sin_\lambda{\theta}\cos_\lambda{\phi}+\cos_\lambda{\theta}\sin_\lambda{\phi}\\
\sin_\lambda^\ast\left(\theta+\phi\right)
&=\sin_\lambda^\ast{\theta}\cos_\lambda{\phi}+\cos_\lambda{\theta}\sin_\lambda^\ast{\phi}\\
\cos_\lambda\left(\theta+\phi\right)
&=\cos_\lambda{\theta}\cos_\lambda{\phi}-\sign{\lambda}\sin_\lambda{\theta}\sin_\lambda{\phi}\\
&=\cos_\lambda{\theta}\cos_\lambda{\phi}-\sin_\lambda^\ast{\theta}\sin_\lambda{\phi}\\
&=\cos_\lambda{\theta}\cos_\lambda{\phi}-\sin_\lambda{\theta}\sin_\lambda^\ast{\phi}
\end{align*}
\end{proposition}
\begin{proof}[\proofof{M:Trigonometry:Sum}]
Proof by exhaustion.
\end{proof}
\begin{proposition}[Derivative of generalized trigonometric functions]\label{M:Trigonometry:Derivative}
\begin{align*}
{\sin_\lambda}^\prime{\theta}&=\lambda\cos_\lambda{\theta}\\
{\sin_\lambda^\ast}^\prime{\theta}&=\abs{\lambda}\cos_\lambda{\theta}\\
{\cos_\lambda}^\prime{\theta}&=-\lambda\sin_\lambda^\ast{\theta}\\
{\tan_\lambda}^\prime{\theta}&=\lambda\sec_\lambda^2{\theta}\\
{\tan_\lambda^\ast}^\prime{\theta}&=\abs{\lambda}\sec_\lambda^2{\theta}
\end{align*}
\end{proposition}
\begin{proof}[\proofof{M:Trigonometry:Derivative}]
Proof by exhaustion.
\end{proof}
\subsection{Matrices}
\begin{definition}\label{M:Rotation}
\textit{Generalized rotation matrix} is defined as
\begin{align*}
R_\lambda\left(\theta\right)&\coloneqq
\begin{bmatrix}
\cos_\lambda{\theta}&-\sin_\lambda^\ast{\theta}\\
\sin_\lambda{\theta}&\cos_\lambda{\theta}\\
\end{bmatrix}\text{,}
\end{align*}
where \(\theta\in\R\).
\end{definition}
\begin{corollary}[Generalized rotation matrix at zero]\label{M:Rotation:Identity}
\[
R_\lambda\left(0\right)
=
I_2
\]
\end{corollary}
\begin{proof}[\proofof{M:Rotation:Identity}]
Obvious
\end{proof}
\begin{corollary}[Generalized rotation matrix of sum of arguments]\label{M:Rotation:Sum}
\[
R_\lambda\left(\theta\right)R_\lambda\left(\phi\right)=R_\lambda\left(\theta+\phi\right)
\]
\end{corollary}
\begin{proof}[\proofof{M:Rotation:Sum}]
\begin{align*}
R_\lambda\left(\theta\right)R_\lambda\left(\phi\right)
&=\begin{bmatrix}
\cos_\lambda{\theta}&-\sin_\lambda^\ast{\theta}\\
\sin_\lambda{\theta}&\cos_\lambda{\theta}\\
\end{bmatrix}
\begin{bmatrix}
\cos_\lambda{\phi}&-\sin_\lambda^\ast{\phi}\\
\sin_\lambda{\phi}&\cos_\lambda{\phi}\\
\end{bmatrix}
&\text{(\cref{M:Rotation})}\\
&=\begin{bmatrix}
\cos_\lambda{\theta}\cos_\lambda{\phi}+\left(-\sin_\lambda^\ast{\theta}\right)\sin_\lambda{\phi}&
\cos_\lambda{\theta}\left(-\sin_\lambda^\ast{\phi}\right)+\left(-\sin_\lambda^\ast{\theta}\right)\cos_\lambda{\phi}\\
\sin_\lambda{\theta}\cos_\lambda{\phi}+\cos_\lambda{\theta}\sin_\lambda{\phi}&
\sin_\lambda{\theta}\left(-\sin_\lambda^\ast{\phi}\right)+\cos_\lambda{\theta}\cos_\lambda{\phi}\\
\end{bmatrix}
&\text{(\cref{Matrix:Product})}\\
&=\begin{bmatrix}
\cos_\lambda{\theta}\cos_\lambda{\phi}-\sin_\lambda^\ast{\theta}\sin_\lambda{\phi}&
-\left(\sin_\lambda^\ast{\theta}\cos_\lambda{\phi}+\cos_\lambda{\theta}\sin_\lambda^\ast{\phi}\right)\\
\sin_\lambda{\theta}\cos_\lambda{\phi}+\cos_\lambda{\theta}\sin_\lambda{\phi}&
\cos_\lambda{\theta}\cos_\lambda{\phi}-\sin_\lambda{\theta}\sin_\lambda^\ast{\phi}\\
\end{bmatrix}
&\text{(simplify)}\\
&=\begin{bmatrix}
\cos_\lambda{\theta+\phi}&-\sin_\lambda^\ast{\theta+\phi}\\
\sin_\lambda{\theta+\phi}&\cos_\lambda{\theta+\phi}\\
\end{bmatrix}
&\text{(\cref{M:Trigonometry:Sum})}\\
&=R_\lambda\left(\theta+\phi\right)
&\text{(\cref{M:Rotation})}\\
R_\lambda\left(\theta\right)
R_\lambda\left(\phi\right)
&=R_\lambda\left(\theta+\phi\right)
&\qedhere
\end{align*}
\end{proof}
\begin{corollary}[Inverse of generalized rotation matrix]\label{M:Rotation:Inverse}
\[
R_\lambda\left(\theta\right)^{-1}=R_\lambda\left(-\theta\right)
\]
\end{corollary}
\begin{proof}[\proofof{M:Rotation:Inverse}]
\begin{align*}
R_\lambda\left(\theta\right)
R_\lambda\left(-\theta\right)
&=R_\lambda\left(0\right)&\text{(\cref{M:Rotation:Sum})}\\
&=I_2&\text{(\cref{M:Rotation:Identity})}\\
R_\lambda\left(-\theta\right)
R_\lambda\left(\theta\right)
&=R_\lambda\left(0\right)&\text{(\cref{M:Rotation:Sum})}\\
&=I_2&\text{(\cref{M:Rotation:Identity})}
\end{align*}
\[
R_\lambda\left(\theta\right)R_\lambda\left(-\theta\right)
=R_\lambda\left(-\theta\right)R_\lambda\left(\theta\right)
=I_2
\]
\[
{R_\lambda\left(\theta\right)}^{-1}
=
R_\lambda\left(-\theta\right)
\qedhere
\]
\end{proof}
\begin{definition}\label{M:Position}
\textit{Position matrix} is defined recursively as
\begin{align*}
P_{\lambda,n}\left(\left\{\tensor{\theta}{^1},\dots,\tensor{\theta}{^n}\right\}\right)
&\coloneqq
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}\text{,}\\
P_{\lambda,0}
\lambda&\coloneqq I_1\text{,}
\end{align*}
where \(\theta=\left\{\tensor{\theta}{^i}\right\}\in\R^n\) for \(i\in\range{1}{n}\).
\end{definition}
\begin{definition}\label{M:Position:Set}
Let \(P\left(n,\lambda\right)\) be set of position matrices.
\end{definition}
\begin{corollary}[Position matrix at zero]\label{M:Position:Set:Identity}
\[
P_{\lambda,n}
\left(0_{n}\right)
=
I_{n+1}
\]
\end{corollary}
\begin{proof}[\proofof{M:Position:Set:Identity}]
Prove by mathematical induction on \(n\),
Let
\begin{equation}\label{M:Position:Set:Identity:Proof:Induction}
P_{\lambda,n-1}\left(0_{n-1}\right)=I_n
\end{equation}
\begin{align*}
P_{\lambda, n}\left(0_{n}\right)
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(0_{n-1}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2, n+1}
\begin{bmatrix}
R_{\lambda}\left(0\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2, n+1}
&\text{(\cref{M:Position})}\\
&=
\begin{bmatrix}
I_{n}&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2, n+1}
\begin{bmatrix}
R_{\lambda}\left(0\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2, n+1}
&\text{(\cref{M:Position:Set:Identity:Proof:Induction})}\\
&=
\begin{bmatrix}
I_{n}&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2, n+1}
\begin{bmatrix}
I_{2}&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2, n+1}
&\text{(\cref{M:Rotation:Identity})}\\
&=
I_{n+1}
T_{2, n+1}
I_{n+1}
T_{2, n+1}
&\text{(\cref{Matrix:Identity:Block})}\\
&=
T_{2, n+1}
T_{2, n+1}
&\text{(\cref{Matrix:Identity})}\\
&=
I_{n+1}
&\text{(\cref{Matrix:Permutation:Square})}
\end{align*}
\[
P_{\lambda,n}
\left(0_{n}\right)
=
I_{n+1}
\qedhere
\]
\end{proof}
\begin{definition}\label{M:Orientation}
\textit{Orientation matrix} is defined as
\begin{align*}
Q^{\pm}_n\left(\phi_{n-1},\phi_{n-2},\dots,\phi_1\right)
&\coloneqq
\begin{bmatrix}
1&0_{1\times n}\\
0_{n\times 1}&X^{\pm}_{+1,n-1}\left(\phi_{n-1},\phi_{n-2},\dots,\phi_1\right)\\
\end{bmatrix}\text{,}\\
Q^{\pm}_0
&\coloneqq\pm I_1\text{,}
\end{align*}
where \(\phi_m\in\R^m\text{for } m\in\range{1}{n-1}\).
\end{definition}
\begin{definition}\label{M:Orientation:Set}
Let \(Q\left(n\right)\) be set of orientation matrices.
\end{definition}
\begin{corollary}[Orientation matrix at zero]\label{M:Orientation:Set:Identity}
\[
Q^{+}_{n}
\left(0_{n-1}, 0_{n-2},\dots\right)
=
I_{n+1}
\]
\end{corollary}
\begin{proof}[\proofof{M:Orientation:Set:Identity}]
Prove by mathematical induction on \(n\),
Let
\begin{equation}\label{M:Orientation:Set:Identity:Proof:Induction}
Q^{+}_{n-1}\left(0_{n-2}, 0_{n-3},\dots\right)=I_n
\end{equation}
\begin{align*}
Q^{+}_n\left(0_{n-1}, 0_{n-2},\dots\right)
&=
\begin{bmatrix}
1&0_{1\times n}\\
0_{n\times 1}&X^{\pm}_{+1,n-1}\left(0_{n-1},0_{n-2},\dots\right)\\
\end{bmatrix}
&\text{(\cref{M:Orientation})}\\
&=
\begin{bmatrix}
1&0_{1\times n}\\
0_{n\times 1}&P_{+1,n-1}\left(0_{n-1}\right)Q^{+}_{n-1}\left(0_{n-2}, 0_{n-3},\dots\right)\\
\end{bmatrix}
&\text{(\cref{M:Point})}\\
&=
\begin{bmatrix}
1&0_{1\times n}\\
0_{n\times 1}&P_{+1,n-1}\left(0_{n-1}\right) I_n\\
\end{bmatrix}
&\text{(\cref{M:Orientation:Set:Identity:Proof:Induction})}\\
&=
\begin{bmatrix}
1&0_{1\times n}\\
0_{n\times 1}&P_{+1,n-1}\left(0_{n-1}\right)\\
\end{bmatrix}
&\text{(\cref{Matrix:Identity})}\\
&=
\begin{bmatrix}
1&0_{1\times n}\\
0_{n\times 1}&I_{n}\\
\end{bmatrix}
&\text{(\cref{M:Position:Set:Identity})}\\
&=
I_{n+1}
&\text{(\cref{Matrix:Identity:Block})}
\end{align*}
\[
Q^{+}_n
\left(0_{n-1}, 0_{n-2},\dots\right)
=
I_{n+1}
\qedhere
\]
\end{proof}
\begin{definition}\label{M:Point}
\textit{Point matrix} is defined as
\begin{align*}
X^{\pm}_{\lambda,n}\left(\theta,\phi_{n-1},\phi_{n-2},\dots,\phi_1\right)&\coloneqq
P_{\lambda,n}\left(\theta\right)
Q^{\pm}_n\left(\phi_{n-1},\phi_{n-2},\dots,\phi_1\right)\text{,}
\end{align*}
where \(\theta\in\R^n\) and \(\phi_m\in\R^m\text{for } m\in\range{1}{n-1}\).
\end{definition}
\begin{definition}\label{M:Point:Set}
Let \(X\left(n,\lambda\right)\) be set of point matrices.
\end{definition}
\begin{corollary}[Position matrix as subset of point matrix]\label{M:Point:Position}
\[
P\left(\lambda,n\right)\subset X\left(\lambda,n\right)
\]
\end{corollary}
\begin{proof}[\proofof{M:Point:Position}]
\begin{align*}
\forall{P\in P\left(\lambda,n\right)}
\forall{Q\in Q\left(n\right)},
&P Q\in X\left(\lambda,n\right)
&\text{(\cref{M:Point})}\\
\implies
&P I\in X\left(\lambda,n\right)
&\text{(\cref{M:Orientation:Set:Identity})}\\
\implies
&P\in X\left(\lambda,n\right)
&\text{(\cref{Matrix:Identity})}
\end{align*}
\[
P\left(\lambda,n\right)\subset X\left(\lambda,n\right)\qedhere
\]
\end{proof}
\begin{corollary}[Point matrix at zero]\label{M:Point:Set:Identity}
\[
X^{+}_{\lambda,n}
\left(0_{n}, 0_{n-1},\dots\right)
=
I_{n+1}
\]
\end{corollary}
\begin{proof}[\proofof{M:Point:Set:Identity}]
It can be implied from\cref{M:Position:Set:Identity,M:Orientation:Set:Identity}.
\end{proof}
\subsection{Group structure}
\begin{proposition}
Group of position matrix with multiplication is a subgroup of an orthogonal group for \(\lambda>0\).
\[
\left(P\left(n,\lambda\right),\cdot\right)\cong O\left(n+1\right)
\]
\end{proposition}
\begin{proof}
\begin{align*}
P&=P_{\lambda,n}\left(\left\{\tensor{\theta}{^1},\dots,\tensor{\theta}{^n}\right\}\right)\\
&\coloneqq
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}
&&\text{(\cref{M:Position})}\\
P^T
&=
T_{2,n+1}^T
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}^T
T_{2,n+1}^T
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}^T
&&\text{(Transpose of product)}\\
&=
T_{2,n+1}^T
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)^T&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}^T
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
&&\text{(Transpose of block)}\\
&=
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)^T&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
&&\text{(Transpose of permutation matrix)}\\
PP^T
&=
\left(\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}\right)\\
&\left(T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)^T&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}\right)
&&\text{(equation above)}\\
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}\\
&
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)^T&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
&&\text{()}\\
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)R_\lambda\left(\tensor{\theta}{^n}\right)^T&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}\\
&
T_{2,n+1}
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}&&\text{(\cref{Matrix:Product:Block})}\\
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}\\
&
T_{2,n+1}
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
&&\text{()}\\
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}&&\text{()}\\
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}&&\text{(\cref{Matrix:Product:Block})}\\
&=I_{n+1}&&\text{(mathematical induction)}
\end{align*}
\end{proof}
\begin{proposition}
Group of position matrix with multiplication is a subgroup of an \(\left(1,n\right)\)-orthochronus indefinite orthogonal group for \(\lambda<0\).
\[
\left(P\left(n,\lambda\right),\cdot\right)\cong O_+\left(n,1\right)
\]
\end{proposition}
\begin{proof}
\begin{align*}
P
&=P_{\lambda,n}\left(\left\{\tensor{\theta}{^1},\dots,\tensor{\theta}{^n}\right\}\right)\\
&\coloneqq
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}&&\text{(\cref{M:Position})}\\
P^T
&=
T_{2,n+1}^T
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}^T
T_{2,n+1}^T
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}^T&&\text{(Transpose of product)}\\
&=
T_{2,n+1}^T
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)^T&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}^T
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
&&\text{(Transpose of block)}\\
&=
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)^T&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)^T&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
&&\text{(Transpose of permutation matrix)}\\
g
&=\diag{-1,1,\dots,1}\\
gPgP^T
&=I_{n+1}&&\text{(mathematical induction)}
\end{align*}
\begin{align*}
\tensor{P}{^1_1}&> 0
\end{align*}
\end{proof}
\begin{proposition}
Group of position matrix with multiplication is isomorphic to translation group for \(\lambda\to0\).
\[
\left(P\left(n,\lambda\right),\cdot\right)\cong E\left(n\right)
\]
\end{proposition}
\begin{proof}
\begin{align*}
R_\lambda\left(\theta\right)
&\coloneqq\begin{bmatrix}
\cos_\lambda\left(\theta\right)&-\sin_\lambda^\ast\left(\theta\right)\\
\sin_\lambda\left(\theta\right)&\cos_\lambda\left(\theta\right)\\
\end{bmatrix}&&\text{(\cref{M:Rotation})}\\
&\to\begin{bmatrix}
1&0\\
\lambda\theta&1\\
\end{bmatrix}&&\text{(Limits of the functions)}\\
P_{\lambda,n}\left(\left\{\tensor{\theta}{^1},\dots,\tensor{\theta}{^n}\right\}\right)
&\coloneqq
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
R_\lambda\left(\tensor{\theta}{^n}\right)&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}&&\text{(\cref{M:Position})}\\
&\to
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
T_{2,n+1}
\begin{bmatrix}
\begin{matrix*}1&0\\
\lambda\tensor{\theta}{^n}&0\\
\end{matrix*}&0_{{2}\times{n-1}}\\
0_{{n-1}\times{2}}&{I}_{n-1}\\
\end{bmatrix}
T_{2,n+1}&&\text{(equation above)}\\
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times 1}\\
0_{1\times n}&1\\
\end{bmatrix}
\begin{bmatrix}
{I}_{n}&0_{n\times1}\\
\begin{matrix*}\lambda\tensor{\theta}{^n}&0_{1\times n-1}\end{matrix*}&1\\
\end{bmatrix}&&\text{(property of permutation matrix)}\\
&=
\begin{bmatrix}
P_{\lambda,n-1}\left(\left\{\tensor{\theta}{^1_{n}},\dots,\tensor{\theta}{^{n-1}}\right\}\right)&0_{n\times1}\\
\begin{matrix*}\lambda\tensor{\theta}{^n}&0_{1\times n-1}\end{matrix*}&1\\
\end{bmatrix}&&\text{(\cref{Matrix:Product:Block})}\\
P_{\lambda,n}\left(\tensor{\theta}{}\right)
&\to
\begin{bmatrix}
1&0_{n\times1}\\
\lambda\tensor{\theta}{}&I_n\\
\end{bmatrix}&&\text{(mathematical induction)}\\
P_{\lambda,n}\left(\tensor{\theta}{}\right) P_{\lambda,n}\left(\tensor{\phi}{}\right)
&\to
\begin{bmatrix}
1&0_{n\times1}\\
\lambda\tensor{\theta}{}&I_n\\
\end{bmatrix}\begin{bmatrix}
1&0_{n\times1}\\
\lambda\tensor{\phi}{}&I_n\\
\end{bmatrix}&&\text{(equation above)}\\
&=
\begin{bmatrix}
1&0_{n\times1}\\
\lambda\tensor{\theta}{}+\lambda\tensor{\phi}{}&I_n\\
\end{bmatrix}&&\text{(\cref{Matrix:Product:Block})}
\end{align*}
\end{proof}
\begin{proposition}
Group of orienatation with multiplication is isomorphic to orthogonal group.
\[
\left(Q\left(n\right),\cdot\right)\cong O\left(n\right)
\]
\end{proposition}
\begin{proposition}
Group of point with multiplication is isomorphic to orthogonal group for \(\lambda>0\).
\[
\left(X\left(n\right),\cdot\right)\cong O\left(n\right)
\]
\end{proposition}
\begin{proposition}
Group of point with multiplication is isomorphic to \(\left(1,n\right)\)-orthochronus indefinite orthogonal group for \(\lambda<0\).
\[
\left(X\left(n\right),\cdot\right)\cong O_+\left(n,1\right)
\]
\end{proposition}
\begin{proposition}
Group of point with multiplication is isomorphic to Euclidean group for \(\lambda\to0\).
\[
\left(X\left(n\right),\cdot\right)\cong E\left(n\right)
\]
\end{proposition}
\end{document}