%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Copyright (c) 2003-2018 by The University of Queensland
% http://www.uq.edu.au
%
% Primary Business: Queensland, Australia
% Licensed under the Apache License, version 2.0
% http://www.apache.org/licenses/LICENSE-2.0
%
% Development until 2012 by Earth Systems Science Computational Center (ESSCC)
% Development 2012-2013 by School of Earth Sciences
% Development from 2014 by Centre for Geoscience Computing (GeoComp)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Slip on a Fault}\label{Slip CHAP}
\begin{figure}[ht]
\centerline{\includegraphics{Slip1}}
\caption{Domain $\Omega=[0,1]^2$ with a vertical fault of length $0.5$.}
\label{fig:slip.1}
\end{figure}
%
In this example we illustrate how to calculate the stress distribution around
a fault\index{fault} in the Earth's crust caused by a slip\index{slip} through
an earthquake.

To simplify the presentation we assume a simple domain $\Omega=[0,1]^2$ with
a vertical fault in its center as illustrated in \fig{fig:slip.1}.
We assume that the slip distribution $s_{i}$ on the fault is known.
We want to calculate the distribution of the displacements $u_{i}$\index{displacement}
and stress $\sigma_{ij}$\index{stress} in the domain.
Further, we assume an isotropic, linear elastic material model of the form
\begin{eqnarray} \label{Slip  stress}
\sigma_{ij} & = & \lambda u_{k,k} \delta_{ij} + \mu ( u_{i,j} + u_{j,i})
\end{eqnarray}
where $\lambda$ and $\mu$ are the Lam\'e coefficients\index{Lam\'e coefficients}
and $\delta_{ij}$ denotes the Kronecker symbol\index{Kronecker symbol}.
On the boundary the normal stress is given by
\begin{eqnarray} \label{Slip natural fault}
\sigma_{ij}n_{j}=0
\end{eqnarray}
and normal displacements are set to zero:
\begin{eqnarray} \label{Slip constraint}
u_{i}n_{i} =0
\end{eqnarray}
The stress needs to fulfill the momentum equation\index{momentum equation}
\begin{eqnarray}\label{Slip general problem}
- \sigma_{ij,j}=0
\end{eqnarray}
This problem is very similar to the elastic deformation problem presented in \Sec{ELASTIC CHAP}.
However, we need to address an additional challenge: the displacement
$u_{i}$ is in fact discontinuous across the fault, but we are in the
lucky situation that we know the jump of the displacements across the fault.
This is in fact the given slip $s_{i}$.
So we can split the total distribution $u_{i}$ into a component
$v_{i}$ which is continuous across the fault and the known slip $s_{i}$
\begin{eqnarray}\label{Slip Split}
u_{i} = v_{i} + \frac{1}{2} s^{\pm}_{i}
\end{eqnarray}
where $s^{\pm}=s$ when right of the fault and $s^{\pm}=-s$ when left of the fault.
We assume that $s^{\pm}=0$ when sufficiently away from the fault.

We insert this into the stress definition in \eqn{Slip stress}
\begin{eqnarray} \label{Slip stress split}
\sigma_{ij} & = &
\sigma^c_{ij} +
\frac{1}{2} \sigma^s_{ij}
\end{eqnarray}
 with
\begin{eqnarray} \label{Slip stress split 1}
\sigma^c_{ij} = \lambda v_{k,k} \delta_{ij} + \mu ( v_{i,j} + v_{j,i})
\end{eqnarray}
and
\begin{eqnarray} \label{Slip stress split 2}
\sigma^s_{ij} = \lambda s^{\pm}_{k,k} \delta_{ij} + \mu ( s^{\pm}_{i,j} + s^{\pm}_{j,i}).
\end{eqnarray}
In fact, $\sigma^s_{ij}$ defines a stress jump across the fault.
An easy way to construct this function is to use a function $\chi$ which is
$1$ on the right and $-1$ on the left side from the fault.
One can then set
\begin{eqnarray} \label{Slip  stress split 23 }
\sigma^s_{ij} = \chi \cdot  ( \lambda s_{k,k} \delta_{ij} + \mu ( s_{i,j} + s_{j,i}) )
\end{eqnarray}
assuming that $s$ is extended by zero away from the fault.
After inserting \eqn{Slip stress split} into (\ref{Slip general problem}) we
get the differential equation\index{momentum equation}
\begin{eqnarray}\label{Slip general problem 2 }
- \sigma^c_{ij,j}=\frac{1}{2} \sigma^s_{ij,j}
\end{eqnarray}
Together with the definition (\ref{Slip stress split 1}) we have a
differential equation for the continuous function $v_i$.
Notice that the boundary condition (\ref{Slip constraint}) and (\ref{Slip natural fault})
transfer to $v_i$ and $\sigma^c_{ij}$ as $s$ is zero away from the fault.
In \Sec{ELASTIC CHAP} we have discussed how this problem is solved using
the \LinearPDE class. We refer to this section for further details.

To define the fault we use the \class{FaultSystem} class introduced in \Sec{Fault System}.
The following statements define a fault system \var{fs} and add the fault \var{1} to the system:
\begin{python}
  fs=FaultSystem(dim=2)
  fs.addFault(fs.addFault(V0=[0.5,0.25], strikes=90*DEG, ls=0.5, tag=1)
\end{python}
The fault added starts at point $(0.5,0.25)$ has length $0.5$ and points north.
The main purpose of the \class{FaultSystem} class is to define a
parameterization of the fault using a local coordinate system.
One can inquire the class to get the range used to parameterize a fault.
\begin{python}
  p0,p1 = fs.getW0Range(tag=1)
\end{python}
Typically \var{p0} is equal to zero while \var{p1} is equal to the length of the fault.
The parameterization is given as a mapping from a set of local coordinates
onto a parameter range (in our case the range \var{p0} to \var{p1}).
For instance, to map the entire domain \var{mydomain} onto the fault one can
use
\begin{python}
  x = mydomain.getX()
  p,m = fs.getParametrization(x, tag=1)
\end{python}
Of course there is the problem that not all locations are on the fault.
For those locations which are on the fault \var{m} is set to 1, otherwise 0 is used.
So on return the values of \var{p} define the value of the fault parameterization
(typically the distance from the starting point of the fault along the fault)
where \var{m} is positive.
On all other locations the value of \var{p} is undefined.
Now \var{p} can be used to define a slip distribution on the fault via
\begin{python}
  s = m*(p-p0)*(p1-p)/((p1-p0)/2)**2*slip_max*[0.,1.]
\end{python}
Notice the factor \var{m} which ensures that \var{s} is zero away from the fault.
It is important that the slip is zero at the ends of the faults.

We can now put all components together to get the script:
\begin{python}
  from esys.escript import *
  from esys.escript.linearPDEs import LinearPDE
  from esys.escript.models import FaultSystem
  from esys.finley import Rectangle
  from esys.weipa import saveVTK
  from esys.escript.unitsSI import DEG

  #... set some parameters ...
  lam=1.
  mu=1
  slip_max=1.
  mydomain = Rectangle(l0=1.,l1=1.,n0=16, n1=16)  # n1 needs to be a multiple of 4!
  # .. create the fault system
  fs=FaultSystem(dim=2)
  fs.addFault(V0=[0.5,0.25], strikes=90*DEG, ls=0.5, tag=1)
  # ... create a slip distribution on the fault
  p, m=fs.getParametrization(mydomain.getX(), tag=1)
  p0,p1= fs.getW0Range(tag=1)
  s=m*(p-p0)*(p1-p)/((p1-p0)/2)**2*slip_max*[0.,1.]
  # ... calculate stress according to slip:
  D=symmetric(grad(s))
  chi, d=fs.getSideAndDistance(D.getFunctionSpace().getX(), tag=1)
  sigma_s=(mu*D+lam*trace(D)*kronecker(mydomain))*chi
  #... open symmetric PDE ...
  mypde=LinearPDE(mydomain)
  mypde.setSymmetryOn()
  #... set coefficients ...
  C=Tensor4(0., Function(mydomain))
  for i in range(mydomain.getDim()):
    for j in range(mydomain.getDim()):
       C[i,i,j,j]+=lam
       C[j,i,j,i]+=mu
       C[j,i,i,j]+=mu
  # ... fix displacement in normal direction
  x=mydomain.getX()
  msk=whereZero(x[0])*[1.,0.] + whereZero(x[0]-1.)*[1.,0.] \
     +whereZero(x[1])*[0.,1.] + whereZero(x[1]-1.)*[0.,1.]
  mypde.setValue(A=C, X=-0.5*sigma_s, q=msk)
  #... solve pde ...
  mypde.getSolverOptions().setVerbosityOn()
  v=mypde.getSolution()
  # .. write the displacement to file:
  D=symmetric(grad(v))
  sigma=(mu*D+lam*trace(D)*kronecker(mydomain))+0.5*sigma_s
  saveVTK("slip.vtu", disp=v+0.5*chi*s, stress=sigma)
\end{python}
The script creates the file \file{slip.vtu} which contains the total
displacements and stress.
These values are stored as cell-centered data.
%
\begin{figure} [ht]
\centerline{\includegraphics[width=\figwidth]{Slip2}}
\caption{Total Displacement after the slip event}
\label{fig:slip.2}
\end{figure}
%
See \fig{fig:slip.2} for a visualization of the result.


\chapter{Multiscale Methods for polarized maps on the Sphere}
\label{ch_mms_pola}

% chapter multiscale transform for pola data

\section{Module-phase non linear multiscale transform}
%---------------------------------------------------------------------------------------------------------------------------------------
\label{sec:modphase}

\subsection{Introduction}
Given a polarized map in the standard Q-U representation, consider a different point of view and define the modulus $M$ and phase $P$ maps as follows~:
%
%From a different point of view, the combined $Q$ and $U$ maps can be considered as a vector field. Let define such combined map $\mathcal{V}$ as follows~:
%\begin{equation}
%{\mathcal{V}} = \left[ Q \, U\right]
%\end{equation}
%Each pixel of $\mathcal{V}$ is then vector valued. A classical approach amounts to decomposing each vector into its modulus and phase part. $\mathcal{V}$ can then be decomposed into a modulus map $M$ and a phase map $P$~:
\begin{eqnarray}
\forall k,\,\,\, M_k & = & \sqrt{Q_k^2 +  U_k^2} \\
\forall k,\,\,\, P_k & = & \exp(i \theta_k) \mbox{ where } tan(\theta_k) = U_k/Q_k
\end{eqnarray}
Because the smoothness of the $Q$ and $U$ maps should result in some smoothness of the modulus map $M$ and the phase map $P$,
one may consider devising a multiscale modulus/phase decomposition of the spin 2 field ${\mathcal{V}} = \left[ Q \, U\right]$.\\

The specificity of the modulus/phase decomposition of $\mathcal{V}$ is twofold~: i) the modulus field is non-negative and ii) the phase
field takes its values on the unit circle $S^1$. Recently, \cite{rahman05} introduced a multiscale analysis technique for manifold valued
data that will be described in the following paragraph. We then define the modulus/phase (MP) multiscale transform as follows~:
\vspace{.1cm}
\begin{center}
\begin{minipage}[b]{0.85\linewidth}
\footnotesize{
\begin{enumerate}
\item Apply a classical multiscale transform (\textit{i.e.} wavelets) to the modulus map $M$.
\item Apply the multiscale analysis technique for manifold valued data described in \cite{rahman05} to the phase map $P$.
\end{enumerate}}
\end{minipage}
\end{center}
\vspace{.1cm}

\subsection{Decimated MP-multiscale transform}

\begin{figure*}[htb]
\centerline{
 \vbox{
 \includegraphics[width=\textwidth]{fig_modphase_back.pdf}
 }
 }
\caption{Examples of MP-multiscale coefficients backprojection.}
\label{fig_modphase_back}
\end{figure*}
Let us provide some essential notation~: we assume that the entries of the phase map $P$ lie in a manifold $\mathcal{M}$
(\textit{e.g.} $\mathcal{M}\equiv S^1$). According to \cite{rahman05}, take $p_0,p_1 \in \mathcal{M}$ and define $Log_{p_0}(p_1)$
as the log-map of $p_1$ onto the tangent space $\mathcal{T}_{p_0}$ of $\mathcal{M}$ at $p_0$. The back-projection is obtained
using the inverse of the log-map $Exp_{p_0}$. \footnote{In differential geometry, the Exp map and Log map are generalizations of
the usual exponential and logarithm function. Here the manifold $\mathcal{M}$ is a Riemannian manifold. In that case, the Exp map
at point $p_0$, $Exp_{p_0}(s)$ is the map which takes a vector $s$ of the tangent space of $\mathcal{M}$ at $p_0$ and provides the
point $p_1$ by travelling along the geodesic starting at $p_0$ in the direction s.}\\
For instance, if we choose $\mathcal{M} \equiv S^1$ then $p_0 = \exp(i \theta_0)$ and $p_1 = \exp(i \theta_1)$. The $Exp_{p_0}$ and
$Log_{p_0}$ maps are then defined as follows~:
\begin{eqnarray}
\forall p_1 \in S^, \,\,\,  Log_{p_0} (p_1) & = & \theta_1 - \theta_0 \\
\forall s \in \mathbb{R} \,\,\, Exp_{p_0} (s) & = & exp(i(\theta_0 + s))
\end{eqnarray}
The multiscale transform for manifold valued data introduced in \cite{rahman05} is equivalent to a two-step interpolation-refinement
scheme similar to the lifting scheme described in~\cite{wave:sweldens98}. The wavelet coefficients and low pass approximation pixels
are then computed as follows at each scale $j$ and pixel $k$~:
\begin{eqnarray}\label{eq:mani}
w_{j+1,k}^P & = & Log_{c_{j,2k+1}^P}\left(\mathcal{P}(c_{j,2k}^P)\right)   \\
c_{j+1,k}^P & = & Exp_{c_{j,2k}^P} ( -\mathcal{U}(w_{j+1,k}^P))
\label{eq:mani2}
\end{eqnarray}
The wavelet coefficient $w_{j+1,k}^P$ at pixel $k$ and scale $j$ is the projection of its prediction/interpolation $\mathcal{P}(c_{j,2k}^P)$
onto the tangent space $\mathcal{T}_{c_{j,2k+1}^P}$ of $\mathcal{M}$ at $c_{j,2k+1}^P$. The low pass approximation $c_{j+1,k}^P$ at scale $j+1$
is computed by updating $c_{j,2k}^P$ from the wavelet coefficient $w_{j+1,k}^P$.\\
The main advantage of this scheme is its ability to capture local regularities while guaranteeing the low pass approximation to belong to
the manifold $\mathcal{M}$. Indeed, the wavelet coefficient $w_{j+1,k}^P$ at pixel $k$ and scale $j+1$ is computed as the $Exp$ map at
$c_{j,2k+1}^P$ of an approximation $\mathcal{P}(c_{j,2k}^P)$ of $c_{j,2k}^P$.\\
Note also that even if the definitions of the $Exp_{p_0}$ and $Log_{p_0}$ maps involve the absolute phase $\theta(k)$ (\textit{i.e.} $tan(\theta(k)) = U_k/Q_k$),
at least they only require the computation of differences of phases values thus avoiding the explicit manipulation of an absolute phase.\\
However the non-linearity of the proposed transform is a major drawback when considering denoising and restoration applications.\\

\paragraph{Illustration~:\\}
In the case of polarized data, the entries of the phase map $P$ lie in $\mathcal{M} \equiv S^1$. In the following experiments, $\mathcal{P}$ and $\mathcal{U}$ are chosen such that~:
\begin{eqnarray}
w_{j+1}^P & = & Log_{c_{j,2k+1}^P}(c_{j,2k}^P)  \\
c_{j+1,k}^P & = & Exp_{c_{j,2k}^P} \left( - \frac{w_{j+1}^P}{2}\right)
\end{eqnarray}
This multiscale transform is invertible and its inverse is computed as follows~:
\begin{eqnarray}
c_{j,2k}^P & = & Exp_{c_{j+1,k}^P} \left(\frac{w_{j+1}^P}{2}\right)\\
c_{j,2k+1}^P & = & Exp_{c_{j,2k}^P}\left( w_{j+1}^P \right)
\end{eqnarray}
The picture in Figure~\ref{fig_modphase_back} features some examples of backprojections of MP-multiscale coefficients.

\subsection{Undecimated MP-multiscale transform}
For image restoration purposes, the use of undecimated multiscale transforms has been shown to provide better results than decimated transforms~\cite{starck:book98,starck:book06}.
The aforementioned modulus/phase multiscale analysis can be extended to an undecimated scheme consisting in~: i) applying an undecimated wavelet transform to the modulus map,
ii) analyzing the phase map P using an extension to the undecimated case of the multiscale transform described in \cite{rahman05}. In that case, Equations~\eqref{eq:mani}
and \eqref{eq:mani2} are replaced with the following equations~:
\begin{eqnarray}\label{eq:maniu}
c_{j+1,k}^P & = & Exp_{c_{j,k}^P} ( \mathcal{F}(c_{j,.}^P))\\
w_{j+1}^P & = & Log_{c_{j+1,k}^P}\left(c_{j,k}^P\right)
\end{eqnarray}
where $\mathcal{F}(c_{j,.}^P) = \sum_l h_{l} Log_{c_{j,k}}\left(c_{j,k-2^jl}\right)$ with $\sum_l h_l = 1$ and $h_l > 0$. The low pass
approximation $c_{j+1,k}^P$ is then computed from a linear combination (linear filter) of a neighborhood $\{c_{j,k-2^jl}\}_l$ of $c_{j,k}$
weighted by the positive scalars $\{h_l\}_l$. Note that from scale $j$ to scale $j+1$, the spatial size of the neighborhood increases
by a factor $2$ which would be equivalent to downsize by a factor $2$ the band pass filter of the classical wavelet decomposition scheme.\\

\subsection{Example}
In the case of polarized data, the entries of the phase map $P$ lie in $\mathcal{M} \equiv S^1$. In the following experiments, $\mathcal{F}$ is chosen such that~:
\begin{eqnarray}
c_{j+1,k}^P & = & Exp_{c_{j,k}^P}\left(\sum_l h_{l}Log_{c_{j,k}}\left(c_{j,k-2^jl}^P\right)\right)  \\
w_{j+1,k}^P & = & Exp_{c_{j,k}^P} \left(c_{j+1,k}^P\right)
\end{eqnarray}
where~:
\begin{equation}
h_l = \left\{
\begin{array}{ccc}
0 & \mbox{ if } & l < -2 \mbox{ or } l > 2 \\
1/16 &\mbox{ if }& l=-2 \mbox{ or } l=2 \\
1/4 &\mbox{ if }& l=-1 \mbox{ or } l=1\\
3/8 &\mbox{ if }& l= 0
\end{array}
\right.
\end{equation}
This multiscale transform is invertible and its inverse is computed as follows~:
\begin{equation}
 c_{j,k}^P  =  Exp_{c_{j+1,k}^P } \left(- w_{j+1,k}^P\right)
\end{equation}

\begin{figure*}[htb]
 \vbox{
 \centerline{
 \hbox{
 \includegraphics[width=7cm]{fig_mol_synchrotron.pdf}
 \includegraphics[width=7cm]{fig_mol_synchrotron_noise.pdf}
 }
 }
 \centerline{
 \hbox{
 \includegraphics[width=7cm]{fig_mol_synchrotron_noise_no_scale1-3.pdf}
 \includegraphics[width=7cm]{fig_mol_synchrotron_noise_no_scale1-5.pdf}
 }
 }
 }
\caption{ Polarized field smoothing - \textit{top left~:} simulated synchroton emission.  \textit{top right~:} same field corrupted by additive noise.
 \textit{bottom left~:} MP-multiscale reconstruction after setting to zero all coefficients from the three first scales.
\textit{bottom right~:} MP-multiscale reconstruction after setting to zero all coefficients from  the five first scales.}
\label{fig_modphase_smoothfield}
\end{figure*}
Fig.~\ref{fig_modphase_smoothfield} top  shows a simulated polarized field of the synchrotron emission and its noisy version.
We have applied the MP-multiscale transform and we remove the first three scales (i.e. we put all coefficients to zero) before
reconstructing. The resulting image is shown on the bottom left of Fig.~\ref{fig_modphase_smoothfield}. The bottom right of
Fig.~\ref{fig_modphase_smoothfield} corresponds to the same experiment, but by removing the five first scales. We can see that
the field is smoother and smoother, but respecting the large scale structure of the field. This transform will be very well suited
to CMB studies where the phase is analyzed independently of the modulus, such as in~\cite{coles05,naselsky05}.

%---------------------------------------------------------------------------------------------------------------------------------------
%---------------------------------------------------------------------------------------------------------------------------------------

\section{Polarized Wavelet Transform using Spherical Harmonics}
\label{sec:pol_iwt}

\subsection{Isotropic Undecimated Wavelet Transform on the Sphere (UWTS) }
\label{sec:UWTS}

\begin{figure*}[htb]
\centerline{
 \hbox{
 \includegraphics[width=7cm]{fig_q_iwt_back.pdf}
 \includegraphics[width=7cm]{fig_u_iwt_back.pdf}
 }
 }
\caption{Q-isotropic wavelet transform backprojection (left) and U-isotropic wavelet backprojection (right).}
\label{fig_qu_iwt_back}
\end{figure*}

%---------------------------------------------------------------------------------------------------------------------------------------
The undecimated isotropic transform on the sphere described in~\cite{starck:sta05_2} is similar in many respects to the usual
\emph{ \`a trous} isotropic wavelet transform. It is obtained using a zonal scaling function $\phi_{l_c}(\vartheta, \varphi)$
which depends only on colatitude $\vartheta$ and is invariant with respect to a change in longitude $\varphi$. It follows that
the spherical harmonic coefficients $\hat \phi_{l_c} (l,m)$ of $\phi_{l_c}$ vanish when $m \ne 0$ which makes it simple to compute
those spherical harmonic coefficients $\hat c_{0}(l,m)$ of $c_0 = \phi_{l_c} * f$ where $*$ stands for convolution :
\begin{eqnarray}
 \hat c_{0}(l,m) = \widehat{\phi_{l_c} * f} (l,m) = \sqrt{\frac{4\pi}{2l+1} } \hat \phi_{l_c} (l,0) \hat f(l,m)
\end{eqnarray}
A possible scaling function~\cite{starck:book98}, defined in the spherical harmonics representation, is $\phi_{l_c}(l, m) = {2 \over 3} B_{3} ( { {2 l} \over {l_{c} } } )$
where $B_{3}$ is the cubic B-spline compactly supported over $[-2, 2]$. Denoting $\phi_{2^{-j} l_{c} }$ a rescaled version of
$\phi_{l_{c}}$ with cut-off frequency $2^{-j} l_{c}$, a multi-resolution decomposition of $f$ on a dyadic scale is obtained recursively :
\begin{eqnarray}
c_0   & = &  \phi_{ l_{c} }  * f    \nonumber    \\
c_j    &=&   \phi_{2^{-j}  l_{c}  }  * f  =   c_{j-1} * h_{j-1} \nonumber    \\
\end{eqnarray}
where the zonal low pass filters $h_{j}$ are defined by
\begin{eqnarray}
 \hat{H}_{j}(l,m)  & =  &  \sqrt{\frac{4\pi}{2l+1} }  \hat h_{j}(l,m)  \nonumber \\
 &  =  & \left\{
  \begin{array}{ll}
  \frac {   \hat \phi_{\frac{l_{c}}{2^{j+1}} }(l,m)   }   {  \hat  \phi_{  \frac{l_{c}}{2^{j}} }(l,m)   } & \mbox{if }  l  < \frac{ l_{c}} {2^{j+1}} \quad \textrm{and}\quad m = 0\\
0 & \mbox{otherwise } \
  \end{array}
  \right.
\end{eqnarray}
The cut-off frequency is reduced by a factor of $2$ at each step so that in applications where this is useful such as compression,
the number of samples could be reduced adequately. Using a pixelization scheme such as Healpix \cite{pixel:healpix}, this can easily
be done by dividing by 2 the Healpix {\it nside} parameter when computing the inverse spherical harmonics transform.
% Of course, this is only an approximate \emph{Sampling Theorem} but it proved sufficient for numerical purposes.  However, in the present isotropic undecimated transform, no downsampling is performed  and the maps have the same number of pixels on each scale. Hence the orthogonality requirement is relaxed, which provides us with a higher degree of freedom in the choice and design  of the wavelet function $\psi_{l_c}$ to be used with the scaling function $\phi_{l_c}$.
As in the \emph{\`a trous} algorithm, the wavelet coefficients can be defined as the difference between two consecutive resolutions,
$w_{j+1}(\vartheta, \varphi) = c_{j}(\vartheta, \varphi) - c_{j+1}(\vartheta, \varphi)$. This defines a zonal wavelet function $\psi_{l_c}$ as
\begin{eqnarray}\label{wavelet}
\hat \psi_{\frac{l_c}{2^{j}}}(l,m) = \hat \phi_{\frac{l_c}{2^{j-1}}} (l,m)  - \hat \phi_{\frac{l_c}{2^{j}}}(l,m)
\end{eqnarray}

\begin{figure*}[htb]
\centerline{
\hbox{
 \includegraphics[width=7cm]{fig_dust_input.pdf}
}}
\caption{Simulated observations on the sphere of the polarized galactic dust emission.}
\label{fig_simu_pol_dust}
\end{figure*}

With this particular choice of wavelet function, the decomposition is readily inverted by summing the coefficient maps on all wavelet scales
 \begin{eqnarray}\label{IWT}
   %c_{0}(\vartheta, \varphi) = c_{J}(\vartheta, \varphi) + \sum_{j=1}^{J} w_j(\vartheta, \varphi)
   f(\vartheta, \varphi) = c_{J}(\vartheta, \varphi) + \sum_{j=1}^{J} w_j(\vartheta, \varphi)
\end{eqnarray}
where we have made the simplifying assumption that $f$ is equal to $c_0$. Obviously, other wavelet functions $\psi$ could be used just as well, such as the needlet function~\cite{marinucci08}.

% Also,  because of the redundancy of the described decomposition, the inverse transform  is not unique and in fact this can profitably be used to impose additional constraints on the synthesis functions (\emph{e.g.} smoothness, positivity) used in the reconstruction \cite{starck:sta06}.

\begin{figure*}[htb]
\centerline{
\vbox{
 \hbox{
 \includegraphics[width=7cm]{fig_ebiwt_scale1.pdf}
 \includegraphics[width=7cm]{fig_ebiwt_scale2.pdf}
 }
 \hbox{
 \includegraphics[width=7cm]{fig_ebiwt_scale3.pdf}
 \includegraphics[width=7cm]{fig_ebiwt_scale4.pdf}
 }
  \hbox{
 \includegraphics[width=7cm]{fig_ebiwt_scale5.pdf}
 \includegraphics[width=7cm]{fig_ebiwt_scale6.pdf}
 }
  }
 }
\caption{QU-Undecimated Wavelet Transform of the simulated polarized map of galactic dust emission shown in figure~(\ref{fig_simu_pol_dust}).}
\label{fig_quwt_trans_dust}
\end{figure*}

\subsection{Extension to Polarized Data}
By applying the above scalar isotropic wavelet transform to each component $T$, $Q$, $U$ of a polarized map on the sphere, we have~:
\begin{eqnarray}
\label{tqu_iwt}
T(\vartheta, \varphi) & = c_{J}^T (\vartheta, \varphi)+ \sum_{j=1}^{J} w_j^T  (\vartheta, \varphi)\\ \nonumber
Q(\vartheta, \varphi) & = c_{J}^Q (\vartheta, \varphi)+ \sum_{j=1}^{J} w_j^Q (\vartheta, \varphi)\\ \nonumber
U(\vartheta, \varphi) & = c_{J}^U (\vartheta, \varphi)+ \sum_{j=1}^{J} w_j^U (\vartheta, \varphi)
\end{eqnarray}
where $c_{J}^X$ stands for the low resolution approximation to component $X$ and $w_j^X$ is the map of wavelet coefficients of that component on scale $j$. This leads to the following decomposition~:
\begin{eqnarray}
(Q \pm iU)[k] =    (c^Q_{J} \pm c^U_{J,p})[k]   +   \sum_{j=1}^J   ( w_{j}^Q \pm w_{j}^U )[k]
\label{eq_qu_rec_uwt}
\end{eqnarray}
Fig.\ref{fig_qu_iwt_back} shows the backprojection of a Q-wavelet coefficient (left) and a $U$-wavelet coefficient (right).
Fig.~\ref{fig_quwt_trans_dust} shows the undecimated isotropic polarized wavelet transform of the dust image shown on
Fig.~\ref{fig_simu_pol_dust} using six scales, \textit{i.e.} five wavelet scales and the coarse approximation.
%---------------------------------------------------------------------------------------------------------------------------------------
%---------------------------------------------------------------------------------------------------------------------------------------
\section{Polarized Curvelet Transform}
\label{sec:pol_cur}
% \begin{figure*}
% \centerline{
% \hbox{
% \psfig{figure=fig_back_cur_sphere.ps,bbllx=1cm,bblly=7cm,bburx=20cm,bbury=20cm,height=8.5cm,width=12cm,clip=}
% }}
% \caption{Backprojection of  various curvelet coefficients at different scales and orientations on the sphere.  Each map is obtained by setting all but one of the curvelet coefficients to zero, and applying an inverse curvelet transform. Depending on the scale and the position of the non zero curvelet coefficient, the reconstructed image presents a feature with a given width, length and orientation.}
 %\label{Figure:back_cur}
% \end{figure*}
 \begin{figure*}[htb]
\centerline{
\vbox{
 \hbox{
 \includegraphics[width=7cm]{fig_mol_backproj_qucur_qj3.pdf}
 \includegraphics[width=4cm]{fig_backproj_qucur_qj3.pdf}
 }
 \hbox{
 \includegraphics[width=7cm]{fig_mol_backproj_qucur_uj3.pdf}
 \includegraphics[width=4cm]{fig_backproj_qucur_uj3.pdf}
 }
  }
 }
\caption{Top, Q-curvelet backprojection (left)  and zoom (right). Bottom, U-curvelet backprojection (left)  and zoom. }
\label{fig_qucur_back}
\end{figure*}
The 2D ridgelet transform \cite{cur:candes99_1} was developed in an attempt to overcome some limitations inherent in former multiscale methods
\emph{e.g.} the 2D wavelet, when handling smooth images with edges \textit{i.e.} singularities along smooth curves. Ridgelets are translation
invariant \emph{ridge} functions with a wavelet profile in the normal direction. Although ridgelets provide sparse representations of smooth
images with straight edges, they fail to efficiently handle edges along curved lines. This is the framework for curvelets which were given a
first mathematical description in \cite{Curvelets-StMalo}. Basically, the curvelet dictionary is a multiscale pyramid of localized directional
functions with anisotropic support obeying a specific parabolic scaling such that at scale $2^{-j}$, its length is $2^{-j/2}$ and its width is $2^{-j}$.
This is motivated by the parabolic scaling property of smooth curves. Other properties of the curvelet transform as well as decisive optimality results
in approximation theory are reported in \cite{Curvelets-StMalo,CandesDonohoCurvelets}. Notably, curvelets provide optimally sparse representations
of manifolds which are smooth away from edge singularities along smooth curves. Several digital curvelet transforms \cite{cur:donoho99,starck:sta01_3,cur:demanet06}
have been proposed which attempt to preserve the essential properties of the continuous curvelet transform and many papers \cite{starck:sta04,felix2008,starck:sta04}
report on their successful application in image processing experiments. The so-called first generation discrete curvelet described in \cite{cur:donoho99,starck:sta01_3}
consists in applying the ridgelet transform to sub-images of a wavelet decomposition of the original image. By construction, the sub-images are
well localized in space and frequency and the subsequent ridgelet transform provides the necessary directional sensitivity. This latter implementation
in combination with the good geometric properties of the Healpix pixelization scheme, inspired the digital curvelet transform on the sphere~\cite{starck:sta05_2}.
The digital curvelet transform on the sphere is clearly invertible in the sense that each step of the overall transform is itself invertible.
The curvelet transform on the sphere has a redundancy factor of $16J + 1$ when $J$ scales are used, which may be a problem for handling huge data sets
such as from the future Planck-Surveyor experiment. This can be reduced by substituting the pyramidal wavelet transform to the undecimated wavelet
transform in the above algorithm. More details on the wavelet, ridgelet, curvelet algorithms on the sphere can be found in \cite{starck:sta05_2}.
As for the isotropic wavelet on the sphere, a straightforward extension to polarized data will consist in applying successively the curvelet transform
on the sphere to the three components $T$, $Q$ and $U$. Figure~\ref{fig_qucur_back} shows the backprojection of a Q-curvelet coefficient and
U-curvelet coefficient. Clearly, the shapes of these polarized curvelet functions are very different from the polarized wavelet functions.%---------------------------------------------------------------------------------------------------------------------------------------
%---------------------------------------------------------------------------------------------------------------------------------------
\section{Polarized E/B Wavelet and E/B Curvelet}
\label{sec:pol_eb}

\subsection{Introduction}
We have seen that the generalization of the Fourier representation for polarized data on the sphere is the spin-2 spherical harmonics basis denoted $_{\pm 2}Y_{\ell m}$:
\begin{equation}
Q \pm i U  = \sum_{\ell, m}  { _{\pm 2}a_{\ell m}}   {_{\pm 2}Y_{\ell m} }
\end{equation}

At this point, it is convenient~\cite{zalda} to introduce the two quantities denoted $E$ and $B$ which are defined on the sphere by
\begin{eqnarray}\label{EB}
E = &  \sum_{\ell, m}   a_{\ell m} ^E Y_{\ell m} =  \sum_{\ell, m}  - \frac{ 1}{2}   ({_{ 2}a_{\ell m}}  +  {_{- 2}a_{\ell m}} )    Y_{\ell m} \\ \nonumber
B = & \sum_{\ell, m}   a_{\ell m} ^B Y_{\ell m} =  \sum_{\ell, m}  i \frac{ 1}{2}    ({_{ 2}a_{\ell m}}  -  {_{- 2}a_{\ell m}} )   Y_{\ell m}
\end{eqnarray}
%\begin{equation}\label{EB}
%E =  \sum_{\ell, m}   a_{\ell m} ^E Y_{\ell m} =  \sum_{\ell, m}  - \frac{   {_{ 2}a_{\ell m}}  +  {_{- 2}a_{\ell m}}  }  {2}  Y_{\ell m}  \quad \quad
%B =  \sum_{\ell, m}   a_{\ell m} ^B Y_{\ell m} =  \sum_{\ell, m}  i \frac{   {_{ 2}a_{\ell m}}  -  {_{- 2}a_{\ell m}}  }  {2}   Y_{\ell m}
%\end{equation}
where $Y_{\ell m}$ stands for the usual spin 0 spherical harmonics basis functions. The quantities $E$ and $B$ are derived by applying
the spin lowering operator twice to $Q + i U$  and the spin raising operator twice to $Q - i U$ so that $E$ and $B$ are real scalar
fields on the sphere, invariant through rotations of the local reference frame. The normalization of $a_{\ell m} ^E$ and $a_{\ell m} ^B$
chosen in the latter definition is purely conventional but it appears to be rather popular~\cite{1997PhRvD..55.1830Z,2003PhRvD..67b3501B}.
Still, we could multiply $a_{\ell m} ^E$ and $a_{\ell m} ^B$ by some $A_{\ell}$ and we would have just as good a representation of the initial
polarization maps. Through a change of parity $E$ will remain invariant whereas the sign of pseudo-scalar $B$ will change. The $E$ and $B$
modes defined here are not so different from the \emph{gradient} (\emph{i.e. curl} free) and \emph{curl} (\emph{i.e. divergence} free) components
encountered in the analysis of vector fields. Finally, the spatial anisotropies of the Gaussian CMB temperature and polarization fields are
completely characterized in this new linear representation by the power spectra and cross spectra of $T$, $E$ and $B$. Thanks to the different
parities of $T$ and $E$ on one side and $B$ on the other, the sufficient statistics reduce to only four spectra namely $C_\ell^{EE}, C_\ell^{TE},
C_\ell^{TT}, C_\ell^{BB}$. For a given cosmological model, it is possible to give a theoretical prediction of these spectra. Aiming at inverting
the model and inferring the cosmological parameters, an important goal of CMB temperature and polarization data analysis is then to estimate the
latter power spectra, based on sampled, noisy sometimes incomplete $T$, $Q$ and $U$ spherical maps.

\subsection{E/B Isotropic Wavelet}
Following the above idea of representing CMB polarization maps by means of $E$ and $B$ modes, we propose a formal extension of the previous
undecimated isotropic wavelet transform that will allow us to handle linear polarization data maps $T$, $Q$ and $U$ on the sphere. Practically,
the maps we consider are pixelized using for instance the Healpix pixelization scheme. In fact, we are not concerned at this point with the
recovery of E and B modes from pixelized or incomplete data maps which itself is not a trivial task. The extension of the wavelet transform
on the sphere we describe here makes use of the $E$ and $B$ representation of polarized maps described above in a formal way. Given polarization
data maps $T$, $Q$ and $U$, the proposed wavelet transform algorithm consists of the following steps :
\vspace{.1cm}
\begin{center}
\begin{minipage}[b]{0.85\linewidth}
\footnotesize{
\begin{enumerate}
\item Apply the spin $\pm 2$ spherical harmonics transform to $Q+iU$ and $Q-iU$. Practically, the Healpix software package provides an implementation
of this transform for maps that use this pixelization scheme. Otherwise, a fast implementation was recently proposed by \cite{wiauxspin2}.
\item Combine the decomposition coefficients ${ _{2}a_{\ell m}}$ and ${ _{-2}a_{\ell m}}$ from the first step into $a_{\ell m}^E$ and $a_{\ell m}^B$
and build \emph{formal} $E$ and $B$ maps associated with $Q$ and $U$ by applying the usual inverse spherical harmonics transform, as in equation~\ref{EB}.
For numerical and algorithmic purposes, it may be efficient to stay with the spherical harmonics representation of $E$ and $B$.
\item Apply the undecimated isotropic transform on the sphere described above to map $T$ and to the $E$, $B$ representation of the polarization maps.
\end{enumerate}}
\end{minipage}
\end{center}
\vspace{.1cm}
The wavelet coefficient maps  $w_j^T$, $w_j^E$, $w_j^B$ and the low resolution approximation maps $c_J^T$, $c_J^E$, $c_J^B$ are obtained by applying
the isotropic undecimated wavelet transform described in section~\ref{sec:UWTS} to the $T$, $E$, $B$ representation of the polarized data. Figure~\ref{fig:UWTSpol}
shows the result of applying the proposed transform to the polarized CMB data map \emph{ka} \footnote{available at http://lambda.gsfc.nasa.gov/product/map/current/ }
from the WMAP experiment. The top two images show the initial $Q$ and $U$ maps while the subsequent maps are the low pass and wavelet coefficients'maps
in a four scale decomposition. The scaling function we used is a cubic box spline as proposed in section~\ref{sec:UWTS}. The wavelet coefficients were
obtained as the difference between two successive low pass approximations of the multiresolution decomposition of the $E$ and $B$ maps. The proper choice
for the scaling and wavelet functions will depend on the application and the existence of constraints to be enforced.
\begin{figure*}
\vbox{
\centerline{
\hbox{
\psfig{figure=ka_q_nb_hi.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
\psfig{figure=ka_u_nb_hi.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
}
}
\centerline{
\hbox{
\psfig{figure=ka__hi3_1_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
\psfig{figure=ka__hi3_2_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
}
}
\centerline{
\hbox{
\psfig{figure=ka__hi2_1_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
\psfig{figure=ka__hi2_2_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
}
}
\centerline{
\hbox{
\psfig{figure=ka__hi1_1_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
\psfig{figure=ka__hi1_2_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
}
}
\centerline{
\hbox{
\psfig{figure=ka__hi0_1_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
\psfig{figure=ka__hi0_2_nb.pdf,bbllx=5cm,bblly=2cm,bburx= 19cm,bbury=28cm,height=7cm,width=4cm,angle = 90,clip=}
}
}
}
\caption{\textbf{top~:} $Q$ and $U$ CMB polarization data maps from channel \emph{ka} of the WMAP experiment. \textbf{left~:} low pass and wavelet coefficients in three scales of the formal E mode. \textbf{right~:} low pass and wavelet coefficients in three scales of the formal B mode.}
\label{fig:UWTSpol}
\end{figure*}

%---------------------------------------------------------------------------------------------------------------------------------------
\subsection*{Reconstruction}
%---------------------------------------------------------------------------------------------------------------------------------------
\begin{figure*}[htb]
\centerline{
\vbox{
 \hbox{
 \includegraphics[width=7cm]{fig_e_iwt_back_scale2.pdf}
 \includegraphics[width=7cm]{fig_b_iwt_back_scale2.pdf}
 }
  \hbox{
 \includegraphics[width=7cm]{fig_e_iwt_back_scale3.pdf}
 \includegraphics[width=7cm]{fig_b_iwt_back_scale3.pdf}
 }
 \hbox{
 \includegraphics[width=7cm]{fig_e_iwt_back_scale4.pdf}
 \includegraphics[width=7cm]{fig_b_iwt_back_scale4.pdf}
 }
 }
 }
\caption{E-isotropic wavelet transform backprojection (left) and B-isotropic wavelet backprojection (right).}
\label{fig_eb_iwt_back}
\end{figure*}
Obviously, the transform described above is invertible and the inverse transform is readily obtained by applying the inverse
of each of the three steps in reverse order. If, as in the example decomposition above, we take the wavelet function to be
the difference between two successive low pass approximations, the third step is inverted by simply summing the last low pass
approximation with the maps of wavelet coefficients from all scales as in equation~\ref{IWT} :
\begin{eqnarray}
T  & = & c_{J}^T + \sum_{j=1}^{J} w_j^T \quad \quad \nonumber \\
E & = & c_{J}^E + \sum_{j=1}^{J} w_j^E \quad \quad \nonumber \\
B & = &  c_{J}^B + \sum_{j=1}^{J} w_j^B
\end{eqnarray}
where $c_{J}^X$ stands for the low resolution approximation to component $X$ and $w_j^X$ is the map of wavelet coefficients of that component on scale $j$. Finally, noting that :
\begin{eqnarray}
Q  & =  & -\frac{1}{2} \sum_{\ell, m}   a_{\ell m} ^E   ( {_{ 2} Y}_{\ell m} +  {_{ -2} Y}_{\ell m} ) +  i a_{\ell m} ^B ( {_{ 2} Y}_{\ell m} -  {_{ -2} Y}_{\ell m} )  \nonumber \\
     & =  & \sum_{\ell, m}   a_{\ell m} ^E   Z_{\ell m}^+ +  i a_{\ell m} ^B Z_{\ell m}^-   \\ \nonumber
U  & =   & -\frac{1}{2} \sum_{\ell, m}   a_{\ell m} ^B   ( {_{ 2} Y}_{\ell m} +  {_{ -2} Y}_{\ell m} ) -  i a_{\ell m} ^E ( {_{ 2} Y}_{\ell m} -  {_{ -2} Y}_{\ell m} )  \nonumber \\
      & =  & \sum_{\ell, m}   a_{\ell m} ^B Z_{\ell m}^+ -  i a_{\ell m} ^E Z_{\ell m}^-
\end{eqnarray}
the initial representation of the polarized data in terms of $T$, $Q$ and $U$ maps is reconstructed from its wavelet coefficients using the following equations :
% \begin{eqnarray}
%T= c_{J}^T + \sum_{j=1}^{J} w_j^T \\ \nonumber
%Q = \Big \{ \sum_{\ell, m}   <  c_{J}^E , Y_{\ell m}>  {_{ +} Z}_{\ell m} +  i <  c_{J}^B , Y_{\ell m}>  {_{ -} Z}_{\ell m} \Big \}        +   \sum_{j=1}^{J}  \Big \{ \sum_{\ell, m}  <  w_j^E , Y_{\ell m}>  {_{ +} Z}_{\ell m} +  i <  w_j^B , Y_{\ell m}>  {_{ -} Z}_{\ell m} \Big \}    \\ \nonumber
%U = \Big \{ \sum_{\ell, m}   <  c_{J}^B , Y_{\ell m}>  {_{ +} Z}_{\ell m}  -  i <  c_{J}^E , Y_{\ell m}>  {_{ -} Z}_{\ell m} \Big \}        +   \sum_{j=1}^{J}  \Big \{ \sum_{\ell, m}  <  w_j^B , Y_{\ell m}>  {_{ +} Z}_{\ell m} -  i <  w_j^E , Y_{\ell m}>  {_{ -} Z}_{\ell m} \Big \}
%\end{eqnarray}
%\begin{eqnarray}
%T =& c_{J}^T + \sum_{j=1}^{J} w_j^T \\ \nonumber
%Q =& c_{J}^E  \sum_{\ell, m} Y_{\ell m}^{\dagger} Z_{\ell m}^+ + i c_{J}^B  \sum_{\ell, m} Y_{\ell m} ^{\dagger} Z_{\ell m}^-  +   \sum_{j=1}^{J}  \Big \{  w_j^E  \sum_{\ell, m} Y_{\ell m}^{\dagger} Z_{\ell m}^+ +{i} w_j^B  \sum_{\ell, m} Y_{\ell m}^{\dagger}  Z_{\ell m}^- \Big \}    \\ \nonumber
%U =& c_{J}^B  \sum_{\ell, m} Y_{\ell m}^{\dagger} Z_{\ell m}^+ -  i c_{J}^E  \sum_{\ell, m} Y_{\ell m} ^{\dagger} Z_{\ell m}^-  +   \sum_{j=1}^{J}  \Big \{  w_j^B  \sum_{\ell, m} Y_{\ell m}^{\dagger} Z_{\ell m}^+ - {i} w_j^E  \sum_{\ell, m} Y_{\ell m}^{\dagger}  Z_{\ell m}^- \Big \}
%\end{eqnarray}
\begin{eqnarray}\label{eq:recons}
T =& c_{J}^T + \sum_{j=1}^{J} w_j^T \\ \nonumber
Q =& c_{J}^{E,+} + i c_{J}^{B,-} + \sum_{j=1}^{J} \Big \{ w_j^{E,+} + i w_j^{B,-} \Big \} \\ \nonumber
U =& c_{J}^{B,+} - i c_{J}^{E,-} + \sum_{j=1}^{J} \Big \{ w_j^{B,+} - i w_j^{E,-} \Big \}
\end{eqnarray}
where
\begin{eqnarray}\label{eq:change}
c_{J}^{X,+} = c_{J}^X \sum_{\ell, m} Y_{\ell m}^{\dagger} Z_{\ell m}^+ \quad \textrm{and} \quad c_{J}^{X,-} = c_{J}^X \sum_{\ell, m} Y_{\ell m}^{\dagger} Z_{\ell m}^-
\end{eqnarray}
with $W^\dagger$ denoting the transpose conjugate of $W$ so that $\tilde{W} W^\dagger$ is the scalar dot product of $\tilde{W}$ and $W$
while $W^\dagger \tilde{W}$ is an operator (or matrix) acting on its left hand side as a projection along $W$ and \emph{reconstruction}
along $\tilde{W}$. In practice, the Healpix software package provides us with an implementation of the forward and inverse spin $0$ and
spin $2$ spherical harmonics transforms which we need to implement the proposed inverse transform given by equations~\ref{eq:recons}
and~\ref{eq:change}. Clearly, as mentioned earlier in section~\ref{sec:UWTS}, we could have chosen some other wavelet function than merely
the difference between two consecutive scaling functions, and the transformation would still be nearly as simple to invert. Fig.\ref{fig_eb_iwt_back}
shows, on the left, backprojections of E-wavelet coefficients, and, on the right, backprojections of B-wavelet coefficients on the right hand side at different scales.

%---------------------------------------------------------------------------------------------------------------------------------------
\subsection*{E-B  Curvelet}
%---------------------------------------------------------------------------------------------------------------------------------------
\begin{figure*}[htb]
\centerline{
\vbox{
 \hbox{
 \includegraphics[width=12cm]{fig_ecur_back.pdf}
 }
  }
 }
\caption{E-curvelet coefficient backprojection.}
\label{fig_ecur_back}
\end{figure*}

\begin{figure*}[htb]
\centerline{
\vbox{
 \hbox{
 \includegraphics[width=12cm]{fig_bcur_back.pdf}
 }
 }
 }
\caption{B-curvelet coefficient backprojection.}
\label{fig_bcur_back}
\end{figure*}
Similarly to the EB-wavelet constructions, we can easily construct an EB-curvelet transform by first computing the E and B components using
the spin $\pm 2$ spherical harmonics transform, and then applying a curvelet transform on the sphere separately on each of these two components.
Fig.~\ref{fig_ecur_back} shows the backprojection of an E-curvelet coefficient and Fig.~\ref{fig_bcur_back} shows the backprojectionof a B-curvelet coefficient.

%---------------------------------------------------------------------------------------------------------------------------------------


\chapter{Machine Learning techniques} \label{chap3}
\section{Introduction}
In this chapter, we will focus on the more traditional methods used in natural language processing such as Na\"{i}ve-Bayes, decision trees, linear SVM and others. These will serve as a baseline for comparing the performances of the more two advanced models that will be analysed later on: LSTM and Attention Mechanism. The first thing to do when working with text is the do words and texts embedding, indeed, in order to use machine learning algorithms on texts, a mathematical representation of these texts is required.
\section{Text to vectors}
As explained before, text needs to be represented in a way that gives more meaningful information than a simple sequence of bits, which have additional drawbacks such that for a given word, the sequence of bits representing it depends on the coding.
 The first and simplest coding that comes to mind is a one-hot encoding: a matrix $M$ of size number of texts $\times$ number of words where $M_{ij} = 1$ if the word $j$ is present in the text  $i$ and $0$ in the other case. But this is still not enough as each word is given the same weight, no matter how often it appears in the text. \\

 In order to overcome this problem, term-frequency might be used, that is, rather than setting $M_{ij}$ to 0 or 1 we set it to the amount of time it appears in the text. \\

 It is possible to use even better text embedding. It is called term-frequency, inverse document frequency. The main idea is that a word that appears often in all the documents is not helpful in order to classify the documents. For example, if the task is to classify books of biology and physics, words atoms, cells or light are more useful than today or tomorrow. \\

In order to compute tf-idf, it is separated in two parts, the first one being term frequency and the second one inverse document frequency. We have that \begin{equation}
 tf_{ij} = \#(W_j | W_j \in D_i)\\
\end{equation}
That it $tf_{ij}$ is the number of times the word j appears in the document i.
Secondly, we have that \begin{equation*}
 idf_{j} = \log(\frac{\#D}{\#(D_i | W_j \in D_i)}) \\
\end{equation*}
this is the log of the total number of documents, over the number of documents that contains the word j.
Finally, the value tfidf value is computed by \begin{equation}
 tf-idf_{ij} = tf_{ij} * idf_{j} \\
\end{equation}
This the text embedding methods that will be used in this section.
\section{Methodology}
\paragraph{} All the methods presented will be tested in two different ways:
\begin{itemize}
 \item On the liar-liar dataset
 \item On the fake corpus dataset, excluding the news from \textit{beforeitsnews.com} and \textit{nytimes.com}
\end{itemize}
To be more precise, in the first case, the models will be trained on a training set, tuned using validation set and finally tested using test set. In the second case, the same methodology will be used, the dataset has been split be choosing $60\%$ of the text from each domain for training, and $20\%$ for validation and testing. This way of splitting has been chosen because of the uneven representation of each domain in the dataset in order to ensure representation of all the domains in the tree subsets.
\subsection{Evaluation Metrics}
In order to evaluate each model, multiple evaluation metrics have been used. There are recall, precision and f1-score. It is needed to use multiple metrics because they don't all account for the same values. For instance, it is possible to have a model with a recall of 1 that behave extremely bad because it simply classifies all the inputs in the same single class.
Remember That Precision Is Defined As \begin{equation}
 Precision = \frac{TP}{TP + FP}
\end{equation}
Which means that we can have two different precision, depending on which classes is considered as being positive. This is the proportion of correctly classified positive elements over the number of elements classified as positive. It is equals to 1 when there is no false positive, but it does not mean that all the positive elements are correctly classified as it might be some false negative. The recall helps to solve this problem.
It Is Defined As \begin{equation}
 recall = \frac{TP}{TP + FN}
\end{equation}
The f1-score combines the recall and the precision. It is defined by
\begin{equation}
 f1-score = \frac{2 * precision * recall}{precision + recall}\\
\end{equation}
It is also possible to look at the weighted average of all these values. For instance, it is possible to compute the global recall by averaging the recall for both classes by the respective class ratio. \\

Finally, raw output can be used by looking at the confusion matrix.\\

The first parameter to tune is the max number of features used by tf-idf. This is the maximum number of words that will be kept to create the text encoding. The words that are kept are the most frequent words.
\section{Models}
Four models have been used in order to classify texts represented as a TF-IDF matrix. These are Multinomial Na\"{i}ve-Bayes, Linear SVM, Ridge Classifier and Decision Tree. I will start by a very brief recap of each model and how they work.
\subsection{Na\"{i}ve-Bayes\cite{zhang_optimality_nodate}}
The basic idea of Na\"{i}ve-Bayes model is that all features are independent of each other. This is a particularly strong hypothesis in the case of text classification because it supposes that words are not related to each other. But it knows to work well given this hypothesis.
Given an element of class y and vector of features $\mathbf{X} = (x_1,...,x_n)$. The probability of the class given that vector is defined as
\begin{equation}
 P(y | \mathbf{X}) = \frac{P(y)*P(\mathbf{X} | y)}{P(\mathbf{X})}
\end{equation}
Thanks to the assumption of conditional independence, we have that
\begin{equation}
 P(x_i |y,x_1, ...,x_{i-1},x_{i+1},...,x_n) = P(x_i | y)
\end{equation}
Using Bayes rules we have that
\begin{equation}
 P(y|x_1,...,x_n) = \frac{P(y)\prod_{i=1}^n P(x_i | y)}{P(x_1,...,x_n)}
\end{equation}
Because $P(x_1,...,x_n)$ is constant, we have the classification rule
\begin{equation}
 \hat{y} = \underset{y}{argmax} P(y)\prod_{i=1}^n P(x_i | y)
\end{equation}
\subsection{Linear SVM}
Linear SVM is a method for large linear classification. Given pairs of features-label $(\mathbf{x_i}, y_i), y_i \in \{-1, 1\}$, it solves the following unconstrained optimization problem.
\begin{equation}
 \underset{w}{min} \frac{1}{2} \mathbf{w^Tw} + \mathbf{C} \sum_{i=1}^l \xi(\mathbf{w;x_i},y_u)
\end{equation}
Where $\xi$ is a loss function, in this case L2 loss function has been used, and $\mathbf{C} > 0$ a penalty parameter.
Class of new examples are assigned by looking at the value of $\mathbf{w^Tw}$. The class 1 is assigned if $\mathbf{w^Tw} \geq 0$ and the class $-1$ if $\mathbf{w^Tw} < 0$.
\subsection{Decision Tree\cite{Rokach:2014:DMD:2755359}}
Decision tree works by recursively selecting features and splitting the dataset on those features. These features can either be nominal or continuous. \\

In order to find the best split, it uses gini impurity.
\begin{equation}
 G = \sum_{i=1}^C p(i) * (1-p(i))\\
\end{equation}
Where $p(i)$ is the probability of class i in the current branch. The best split is chosen as the one that decreases the most the impurity. For instance, beginning from the root, the gini impurity is computed on the complete dataset, then the impurity of each branch is computed over all features, weighting it by the number of elements in each branch. The chosen feature is the one that has the highest impurity.
\subsection{Ridge Classifier}
Ridge classifier works the same way as ridge regression. It states the problem as a minimization of the sum of square errors with penalization. It can be expressed as in \textbf{Equation \ref{eq:ridge}}.
\begin{equation}
 \underset{w}{min} ||Xw-y||^2_2 + \alpha ||w||^2_2 \label{eq:ridge}
\end{equation}
The predicted class if positive if Xw is positive and negative otherwise.
\section{Models on liar-liar dataset}
\subsection{Linear SVC}
In the case of linear SVC there is one parameter to tune up, which is the penalty parameters for the error term. \textbf{Figure \ref{fig:chap3:linearSVC}} shows the three main metrics with respect to the penalty parameter. This show that a parameter of around $0.1$ is the best one.
\begin{figure*}[]
 \centering
 \makebox[\textwidth][c]{\includegraphics[width=1\textwidth]{images/chapitre3/svc_liar.pdf}}
 \caption{Tuning linearSVC parameters }
 \label{fig:chap3:linearSVC}
\end{figure*}
\subsection{Decision Tree}
With decision trees, it is possible to reduce overfitting by pruning the tree. It is possible to do pre-pruning or post pruning. Pre-pruning means that a stopping criterion is used to stop tree growth earlier and post pruning cut the tree once it has been fully grown. It this case pre-pruning is done by limiting the maximum depth of the tree.
\begin{figure*}[]
 \centering
 \makebox[\textwidth][c]{\includegraphics[width=1\textwidth]{images/chapitre3/liar-dt.pdf}}
 \caption{Tuning Decision Tree Parameters }
 \label{fig:chap3:dt}
\end{figure*}
\textbf{Figure \ref{fig:chap3:dt}} shows metrics values for different depths. It seems that tree of depth 1000 are the best ones.
\subsection{Ridge Classifier}
With the ridge classifier model, it is also possible to tweak the penalty value of the optimization problem. At \textbf{Figure \ref{fig:chap3:ridge1}} we can see that the optimal parameter is around 10 or 20, depending of the metrics that we want to maximize. Later on, the value of 10 will be chosen as a compromise between precision and recall. It is the value that maximizes the f1-score.
\begin{figure*}
 \centering
 \includegraphics[width=1\textwidth]{images/chapitre3/liar-ridge}
 \caption{Average metrics for ridge classifiers with respect to the penalty parameter.}
 \label{fig:chap3:ridge1}
\end{figure*}
\subsection{Max Feature Number}
Starting with the maximum number of features, the precision of each model can be analysed when limiting the maximum number of words. The results for each model can be seen at \textbf{Figure \ref{fig:chap3:max_feature3}, \ref{fig:chap3:max_feature1}} and \textbf{\ref{fig:chap3:max_feature2}}. Shows that depending on the metrics we want to optimize it is better to choose different parameters. For instance, in order to maximize F1-score, it is better to use a maximum number of features of 1000.\\

The results are slightly different if the goal is to optimize the precision because if the best value stays the same for Linear SVM and Ridge Classifier, the Na\"{i}ve-Bayes work better when using the maximum number of features and it goes the same way for recall.
Based on \textbf{Figure \ref{fig:chap3:max_feature3}} we can say that when it comes to precision and recall, Na\"{i}ve-Bayes is the one that performs the best.\\

Row results for max features selection are available at \textbf{Appendix \ref{Appendix1}}.
\begin{figure*}[]
 \centering
 \makebox[\textwidth][c]{\includegraphics[width=1.2\textwidth]{images/chapitre3/liar-liar_f1_ML}}
 \caption{Weighted average of f1-score, precision and recall of each class. }
 \label{fig:chap3:max_feature3}
\end{figure*}
It goes differently when we focus on a single class. For example, the precision for fake detection is at its maximum for Linear SVM and Ridge Classifier when only 10 features are used. But at the same time, it is at its minium for reliable class. It shows that when trying to optimize the overall model and not only for a single class, it is better to look at the weighted average than at the value for a single class. But it is still important to look at the metrics for a single class because it indicates how it behaves for this class. For instance, in the case of automatic fake news detection, it is important to minimize the number of reliable news misclassified in order to avoid what could be called censorship.
\begin{figure*}[]
 \centering
 \makebox[\textwidth][c]{\includegraphics[width=1.2\textwidth]{images/chapitre3/liar-liar_precision_ML}}
 \caption{Precision of the model for each class, the x axes is log scale of the number of features}
 \label{fig:chap3:max_feature1}
\end{figure*}
\begin{figure*}[]
 \centering
 \makebox[\textwidth][c]{\includegraphics[width=1.2\textwidth]{images/chapitre3/liar-liar_recall_ML}}
 \caption{Precision of the model for each class, the x axes is log scale of the number of features}
 \label{fig:chap3:max_feature2}
\end{figure*}
\section{Models on fake corpus dataset}
\subsection{SMOTE: Synthetic Minority Over-sampling Technique\cite{Chawla2011}}
As it has been shown in \textbf{Chapter \ref{chap2}}, the fake news corpus is unbalanced. Synthetic minority oversampling is a technique that allows generating fake samples from the minor class. It works by randomly choosing one or many nearest neighbours in the minority class. For instance, if the algorithm is set to use 5 nearest neighbours, for each sample it will choose one of its nearest neighbours, and generate a new sample on the segment joining the sample and its neighbour.
\begin{algorithm}
  \KwData{k = Number of nearest neighbours}
  \KwData{T = number of minority class samples}
  \For{$i \leftarrow 1...T$}{Compute k-nearest neighbours of sample i\;
   Populate(knn, i)\;}
  \caption{SMOTE}
  \label{algo:SMOTE}
\end{algorithm}
\begin{algorithm}
 \KwData{knn = the k-nearest neighbour of sample i}
 \KwData{s = ith sample}
 nn = random\_choice(knn)\;
 newSample = s + rand(0, 1) * (nn - s)\;
 \caption{Populate}
 \label{algo:populate}
\end{algorithm}
\textbf{Algorithm \ref{algo:SMOTE}} and \textbf{\ref{algo:populate}} shows how it works. The first one computes the k-nearest neighbours and the second one computes a new element by randomly choosing one of these neighbours.
\subsection{Model selection without using SMOTE}
\subsubsection{Hyperparameters tuning}
As for the models trained on the \textbf{liar-liar corpus}, hyper-parameters can be optimized the same way. The average metric for each model with respect to their parameters are shown at \textbf{Figure \ref{fig:chap3:ridge2}, \ref{fig:chap3:dt2}} and \textbf{\ref{fig:chap3:lsvm2}}. \\

\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ridge}
 \caption{Metrics value with respect to the penalty parameter for ridge classifier}
 \label{fig:chap3:ridge2}
\end{figure*}
The optimal parameter for the ridge classifier is clearly 1. As well as for the decision tree trained on \textbf{liar-liar} dataset, the optimal maximum depth is of 1000.
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/fake-dt}
 \caption{Optimal depth of decision tree.}
 \label{fig:chap3:dt2}
\end{figure*}
And finally, the optimal value for the penalty parameter of the svm is also 1.\\

\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/svc_fake}
 \caption{Optimal penalty parameters for linear svm}
 \label{fig:chap3:lsvm2}
\end{figure*}
By looking at \textbf{Figure \ref{fig:chap3:max_feature1}, \ref{fig:chap3:max_feature2}} and \textbf{\ref{fig:chap3:max_feature3}} we can find optimal parameters for the number of features used in TF-IDF. It shows that linear svm and ridge classifiers are the ones that perform the best, having an average precision of sligtly more than $94\%$ for the linear svm and $94\%$ for the ridge classifier. They achieve these performances from $50,000$ features and does not decrease. On the other hand, Na\"{i}ve-Bayes reaches a pike at $100,000$ features and greatly decrease afterward. \\

\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ML_fake_average}
 \caption{Average recall, precision and f1-score wti respect to the maximum number of features.}
 \label{fig:chap3:max_feat1}
\end{figure*}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ML_fake_precision}
 \caption{Precision for fake and reliable class for each model with respect to the maximum number of features}
 \label{fig:chap3:max_feat2}
\end{figure*}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ML_fake_recall}
 \caption{Recall for fake and reliable class for each model with respect to the maximum number of features}
 \label{fig:chap3:max_feat3}
\end{figure*}

\textbf{Figure \ref{fig:chap3:max_feature3}} shows why it is important to look at all the metrics, because Na\"{i}ve-Bayes reaches a recall of 1 for the reliable class and close to 0 for the fake class, which means that almost all texts are classified as reliable. This can be verified by looking at \textbf{Figure \ref{fig:chap3:confMat1}}, only a small proportion of true fake is actually classified as it.
\begin{figure*}
 \centering
 \includegraphics[width=0.5\textwidth]{images/chapitre3/confMat_fake_364070}
 \caption{Confusion matrix for each model using $364,070$ features}
 \label{fig:chap3:confMat1}
\end{figure*}
\subsection{Model selection with SMOTE}
The first thing that can be noticed at \textbf{Figure \ref{fig:chap3:ridge3}, \ref{fig:chap3:dt3}, \ref{fig:chap3:lsvm3}, \ref{fig:chap3:smote_max_feat1}, \ref{fig:chap3:smote_max_feat2}} and \textbf{\ref{fig:chap3:smote_max_feat3}} is that the two models that worked the best without applying SMOTE method, linear SVM and ridge classifiers are still the ones that perform the best. By comparing \textbf{Figure \ref{fig:chap3:lsvm2}} and \textbf{Figure \ref{fig:chap3:lsvm3}} we can see that it works better when applying a smaller regularization parameter. It goes from $0.66\%$ of accuracy to  $0.86\%$ of accuracy thus acting as a regularization. The same does not apply to ridge classifiers. \\

\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ridge+smote}
 \caption{Metrics value with respect to the penalty parameter for ridge classifiers when using SMOTE.}
 \label{fig:chap3:ridge3}
\end{figure*}
It also has a huge impact on how Na\"{i}ve-Bayes behaves as it removes overfitting when using a larger number of features in TF-IDF, leading to a few percent of accuracy increase. \\
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/fake-dt-SMOTE}
 \caption{Optimal depth of decision tree when using SMOTE.}
 \label{fig:chap3:dt3}
\end{figure*}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/svc_fake_smote}
 \caption{Optimal penalty parameters for linear svm when using SMOTE}
 \label{fig:chap3:lsvm3}
\end{figure*}

\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ML_SMOTE_fake_average}
 \caption{Average recall, precision and f1-score wti respect to the maximum number of features.}
 \label{fig:chap3:smote_max_feat1}
\end{figure*}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ML_SMOTE_fake_precision}
 \caption{Precision for fake and reliable class for each model with respect to the maximum number of features}
 \label{fig:chap3:smote_max_feat2}
\end{figure*}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/ML_SMOTE_fake_recall}
 \caption{Recall for fake and reliable class for each model with respect to the maximum number of features}
 \label{fig:chap3:smote_max_feat3}
\end{figure*}
It conclusion for SMOTE method we can say that it does help models that do not have a regularization parameter or when the regularization parameter is low. Thus it does help prevent overfitting.
\section{Results on testing set}
\subsection{Methodology}
Now that all the models have been tuned, they need to be tested independently on testing set. Each dataset contains a testing set. \\
For the \textbf{liar-liar} dataset the following parameters will be used:
\begin{itemize}
 \item Linear SVM with regularization parameters of $0.1$ of a max TF-IDF features of 500,
 \item Ridge Classifier with $\alpha = 10$ and also max TF-IDF features of 500,
 \item Decision Tree with maximum depth of 1000 and the maximum number of features for TF-IDF,
 \item Na\"{i}ve-Bayes will also use the maximum number of features for TF-IDF.
\end{itemize}
For the \textbf{Fake News Corpus}, the following setting will be used:
\begin{itemize}
 \item Linear SVM with regularization parameters of $1$,
 \item Ridge Classifier with $\alpha = 1$,
 \item Decision Tree with maximum depth of 100.
\end{itemize}
They will all be trained using $100,000$ features for TF-IDF.
For the \textbf{Fake News Corpus} with SMOTE, the same parameters will be used, but the maximum number of features for TF-IDF will be used.\\
All the models will be trained on train and validation set and tested on test set.
\subsection{Results}
\subsubsection{Liar-Liar Corpus}
By looking at the row results, based on average accuracy Na\"{i}ve-Bayes, Linear SVM and ridge classifiers perform very close, but when looking at the recall per class it shows that Na\"{i}ve-Bayes is bad at detecting fake news and classifies most of the text as reliable, when Linear SVM and Ridge classifiers are more balanced.
\begin{table}
\begin{subtable}{\textwidth}
\begin{tabular}{lrrrrr}
\toprule
{} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
\midrule
f1-score  &     0.514399 &     0.679764 &  0.614049 &     0.597082 &      0.607588 \\
precision &     0.570485 &     0.638376 &  0.614049 &     0.604430 &      0.608744 \\
recall    &     0.468354 &     0.726891 &  0.614049 &     0.597623 &      0.614049 \\
support   &  1106.000000 &  1428.000000 &  0.614049 &  2534.000000 &   2534.000000 \\
\bottomrule
\end{tabular}
\caption{Raw results for Linear SVM}
\end{subtable}
\begin{subtable}{\textwidth}
\begin{tabular}{lrrrrr}
\toprule
{} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
\midrule
f1-score  &     0.412107 &     0.698507 &  0.601421 &     0.555307 &      0.573504 \\
precision &     0.578431 &     0.608741 &  0.601421 &     0.593586 &      0.595512 \\
recall    &     0.320072 &     0.819328 &  0.601421 &     0.569700 &      0.601421 \\
support   &  1106.000000 &  1428.000000 &  0.601421 &  2534.000000 &   2534.000000 \\
\bottomrule
\end{tabular}
\caption{Raw results for Na\"{i}ve-Bayes}
\end{subtable}
\begin{subtable}{\textwidth}
\begin{tabular}{lrrrrr}
\toprule
{} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
\midrule
f1-score  &     0.496366 &     0.691279 &  0.617206 &     0.593822 &      0.606207 \\
precision &     0.582927 &     0.633606 &  0.617206 &     0.608266 &      0.611486 \\
recall    &     0.432188 &     0.760504 &  0.617206 &     0.596346 &      0.617206 \\
support   &  1106.000000 &  1428.000000 &  0.617206 &  2534.000000 &   2534.000000 \\
\bottomrule
\end{tabular}
\caption{Raw results for Ridge Classifer.}
\end{subtable}
\begin{subtable}{\textwidth}
\begin{tabular}{lrrrrr}
\toprule
{} &         fake &     reliable &  accuracy &    macro avg &  weighted avg \\
\midrule
f1-score  &     0.479354 &     0.591549 &  0.542226 &     0.535451 &      0.542580 \\
precision &     0.475936 &     0.594901 &  0.542226 &     0.535418 &      0.542977 \\
recall    &     0.482821 &     0.588235 &  0.542226 &     0.535528 &      0.542226 \\
support   &  1106.000000 &  1428.000000 &  0.542226 &  2534.000000 &   2534.000000 \\
\bottomrule
\end{tabular}
\caption{Raw results for Decision Tree}
\end{subtable}
\caption{Raw results on \textbf{Liar-Liar Corpus}.}
\end{table}
Finally, it is possible to look at the ROC curve at \textbf{Figure \ref{fig:chap3:roc1}}. One more time, it shows that Na\"{i}ve-Bayes, linear svm and ridge classifier have similar performance, but in this case it shows that NB has a little advantage, with a slightly larger AUC. There is only one point for the decision tree as it does not output probabilities for each class. \\

\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/roc1}
 \caption{ROC curve for each model}
 \label{fig:chap3:roc1}
\end{figure*}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/test_liar_confMat}
 \caption{Confusion Matrix for each models}
 \label{fig:chap3:confMat1}
\end{figure*}
When it comes to the \textbf{Fake news corpus}, linear models are still the ones that perform the best, with linear svm reaching an accuracy of $94.7\%$ and ridge classifiers $93.98\%$. Surprisingly, decision tree outperform Na\"{i}ve-Bayes in this case, with an accuracy of $89.4\%$ when Na\"{i}ve-Bayes only gets $85.3\%$.\\

\begin{table}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.894700 &      0.965364 &  0.947874 &      0.930032 &      0.947620 \\
 precision &      0.907861 &      0.960783 &  0.947874 &      0.934322 &      0.947494 \\
 recall    &      0.881916 &      0.969989 &  0.947874 &      0.925952 &      0.947874 \\
 support   &  17496.000000 &  52181.000000 &  0.947874 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results for Linear SVM on \textbf{Fake News Corpus}}
\end{subtable}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.674458 &      0.905634 &  0.853682 &      0.790046 &      0.847585 \\
 precision &      0.764127 &      0.875841 &  0.853682 &      0.819984 &      0.847790 \\
 recall    &      0.603624 &      0.937525 &  0.853682 &      0.770574 &      0.853682 \\
 support   &  17496.000000 &  52181.000000 &  0.853682 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results for Na\"{i}ve-Bayes on \textbf{Fake News Corpus}}
\end{subtable}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.874220 &      0.960438 &  0.939808 &      0.917329 &      0.938788 \\
 precision &      0.919674 &      0.945736 &  0.939808 &      0.932705 &      0.939192 \\
 recall    &      0.833048 &      0.975604 &  0.939808 &      0.904326 &      0.939808 \\
 support   &  17496.000000 &  52181.000000 &  0.939808 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results for Ridge Classifier on \textbf{Fake News Corpus}}
\end{subtable}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.791687 &      0.929799 &  0.894987 &      0.860743 &      0.895119 \\
 precision &      0.788700 &      0.930987 &  0.894987 &      0.859844 &      0.895258 \\
 recall    &      0.794696 &      0.928614 &  0.894987 &      0.861655 &      0.894987 \\
 support   &  17496.000000 &  52181.000000 &  0.894987 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results for Decision Tree on \textbf{Fake News Corpus}}
\end{subtable}
\caption{Results on \textbf{Fake News Corpus} without using SMOTE.}
\end{table}
In this case, the ROC curve (\textbf{Figure \ref{fig:chap3:roc2}}) shows almost the same ranking of models, except for Decision Tree that is the last one, and Na\"{i}ve-Bayes being juste above it.
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/roc2}
 \caption{ROC curve for each model}
 \label{fig:chap3:roc2}
\end{figure*}
Confusion matrix (\textbf{Figure \ref{fig:chap3:confMat2}}) shows that Na\"{i}ve-Bayes has a tendency of classifying fake news as being reliable. And the other hand, ridge classifier is the one that makes the least misclassification for reliable news, which is a good point. \\

\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/test_fake_confMat}
 \caption{Confusion Matrix for Each models}
 \label{fig:chap3:confMat2}
\end{figure*}
Finally, there is the results of the models trained with SMOTE data augmentation. Using it shows little to no improvements. The only benefit is to balance a little bit the recall for Na\"{i}ve-Bayes on fake and reliable news.
\begin{table}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.891373 &      0.962340 &   0.94407 &      0.926857 &      0.944520 \\
 precision &      0.869960 &      0.970623 &   0.94407 &      0.920291 &      0.945346 \\
 recall    &      0.913866 &      0.954198 &   0.94407 &      0.934032 &      0.944070 \\
 support   &  17496.000000 &  52181.000000 &   0.94407 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results of linear svm on \textbf{Fake News Corpus} when training using SMOTE}
\end{subtable}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.714816 &      0.873538 &  0.824777 &      0.794177 &      0.833683 \\
 precision &      0.604424 &      0.950521 &  0.824777 &      0.777472 &      0.863615 \\
 recall    &      0.874543 &      0.808091 &  0.824777 &      0.841317 &      0.824777 \\
 support   &  17496.000000 &  52181.000000 &  0.824777 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results of Na\"{i}ve-Bayes on \textbf{Fake News Corpus} when training using SMOTE}
\end{subtable}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.877755 &      0.956129 &  0.935431 &      0.916942 &      0.936449 \\
 precision &      0.836588 &      0.973317 &  0.935431 &      0.904953 &      0.938984 \\
 recall    &      0.923182 &      0.939537 &  0.935431 &      0.931360 &      0.935431 \\
 support   &  17496.000000 &  52181.000000 &  0.935431 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results of Ridge Classifier on \textbf{Fake News Corpus} when training using SMOTE}
\end{subtable}
\begin{subtable}{\textwidth}
 \begin{tabular}{lrrrrr}
 \toprule
 {} &          fake &      reliable &  accuracy &     macro avg &  weighted avg \\
 \midrule
 f1-score  &      0.787226 &      0.921178 &  0.884969 &      0.854202 &      0.887542 \\
 precision &      0.734992 &      0.946085 &  0.884969 &      0.840539 &      0.893079 \\
 recall    &      0.847451 &      0.897549 &  0.884969 &      0.872500 &      0.884969 \\
 support   &  17496.000000 &  52181.000000 &  0.884969 &  69677.000000 &  69677.000000 \\
 \bottomrule
 \end{tabular}
 \caption{Raw results of Decision tree on \textbf{Fake News Corpus} when training using SMOTE}
\end{subtable}
\caption{Results on \textbf{Fake News Corpus} when training with SMOTE.}
\end{table}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/roc3}
 \caption{ROC curve for each model}
 \label{fig:chap3:roc3}
\end{figure*}
\begin{figure*}
 \centering
 \includegraphics[width=0.8\textwidth]{images/chapitre3/test_SMOTE_fake_confMat}
 \caption{Confusion Matrix for each models}
 \label{fig:chap3:confMat3}
\end{figure*}
\section{Conclusion}
In this chapter we have analysed how traditional machine learning algorithms words on two different datasets, the second being imbalanced a data augmentation technique, called SMOTE, has been used in order to see if it improves the results. \\

We can conclude that in all the cases linear models are the ones that work the best, with a top accuracy of $61.7\%$ on the \textbf{liar-liar corpus} using Ridge Classifier, and a top accuracy of $94.7\%$ on the \textbf{Fake News Corpus} using linear svm. At the end, the result obtains on the second data set are really good, when those obtain on the first when are mitigated. \\

As explained earlier, it might be important to choose to model that makes the smaller misclassification rate on reliable news in order to avoid possible censorship and confusion matrix shows that in both case Ridge Classifiers is the ones that make the fewer errors in that case. \\

In addition, we have shown that Synthectic Minority Over Sampling Techniques acts as a regularizers, as it does improve performance when the penalization term in small on linear models. \\

In the next section, the focus will be put on trying to improve results on the \textbf{Liar-Liar corpus} as there is room for improvement and that the second dataset already as very good results. But models will still be trying on it for comparison.


\appendix
\section{Proof}

\begin{proof}[Proof of Proposition~\ref{thm:1}]
From the cross-validation criterion, for linear models we have the well-known result that
    \begin{equation*}
    \frac{1}{T} \sum_{i=1}^T \tilde{e_t}^2 = \frac{1}{T} \sum_{i=1}^T \frac{\tilde{e_t}^2}{(1-h_t)^2}
    \end{equation*}
where $h_t = x_t'(X'X)^{-1}x_t$ is the leverage associated with observation $t$. Applying Taylor expansion, we can expand the above equation as
    \begin{eqnarray*}
    \frac{1}{T} \sum_{i=1}^T \tilde{e_t}^2 & = & \frac{1}{T} \sum_{i=1}^T \frac{\tilde{e_t}^2}{(1-h_t)^2} \\
                                           & \approx & \frac{1}{T} \sum_{i=1}^T \hat{e_t}^2 + \frac{2}{T} \sum_{i=1}^T \hat{e_t}^2 h_t \\
                                           & = & \hat{\sigma}^2 + \frac{2}{T} \sum_{i=1}^T \hat{e_t}^2 x_t' (X'X)^{-1} x_t
    \end{eqnarray*}
Under regularity conditions listed in Assumption~\ref{asump:1}, we have $\hat{\sigma}^2 \stackrel{p}{\rightarrow} \sigma^{2}$, and for the penalty term,  $\frac{1}{T} \sum_{i=1}^T \hat{e_t}^2 x_t' (X'X)^{-1} x_t \stackrel{p}{\rightarrow} E(\boldmath{e'Pe})$, putting these two parts together, we can see that CV is asymptotically equivalent to Mallows' Cp under our assumptions except for conditionally homoscedastic errors.
\end{proof}
<<<<<<< HEAD

\begin{proof}[Proof of Corollary~\ref{corollary:1}]
Since CV is asymptotically equivalent to Mallows' Cp, following Hansen's \cite{hansen2009averaging} proof, write the sample CV criterion for the weighted model as a function of the break model weight $w$,
    \begin{equation*}
      \mathrm{CV}(w) = (w\hat{e} + (1-w)\tilde{e})'(w\hat{e} + (1-w)\tilde{e}) + 2(T - 2k)^{-1}(k + w\bar{p})\hat{e}'\hat{e}
    \end{equation*}
where $\bar{p}$ proposed by Hansen is used to approximate the infeasible expected value of the population penalty term. The CV weight is the value in $[0, 1]$ that minimizes $\mathrm{CV}(w)$, so
    \begin{equation*}
      \hat{w} = \frac{(T - 2k)(\sum_{t=1}^{T}\tilde{e}_{t}^{2} - \sum_{t=1}^{T}\hat{e}_{t}^{2}) - \bar{p}\sum_{t=1}^{T}\hat{e}_{t}^{2}}{(T - 2k)(\sum_{t=1}^{T}\tilde{e}_{t}^{2} - \sum_{t=1}^{T}\hat{e}_{t}^{2})}
    \end{equation*}
if $(T - 2k)(\sum_{t=1}^{T}\tilde{e}_{t}^{2} - \sum_{t=1}^{T}\hat{e}_{t}^{2})(\sum_{t=1}^{T}\hat{e}_{t}^{2})^{-1} \geq \bar{p}$ while $\hat{w} = 0$ otherwise.
\end{proof}
=======
\begin{proof}[Proof of Lemma~\ref{thm:2}]
We relax the conditional homoscedasticity assumption in Hansen's Mallow's model averaging method when there is a possible structural break in the underlying DGP. The proof is adapted from Hansen \cite{hansen2009averaging} and Andrews \cite{andrews93} without assuming conditional homoscedasticity. The CV penalty term can be expanded as:
\begin{eqnarray*}
e'P(k)e & = & e'Pe + e'P^{*}(k)e \\
        & = & e'Pe + e'X^{*}(k)(X^{*}(k)'X^{*}(k))^{-1}X^{*}(k)'e
\end{eqnarray*}
where the notations for $P$,$P^{*}(k)$ and $X^{*}(k)$ are the same as in Hansen \cite{hansen2009averaging}.

For the second term, $e'X^{*}(k)(X^{*}(k)'X^{*}(k))^{-1}X^{*}(k)'e$, we can see
\begin{eqnarray*}
X^{*}(k)'X^{*}(k) & = & (X(k)-X(X'X)^{-1}X(k)'X(k))'(X(k)-X(X'X)^{-1}X(k)'X(k)) \\
                  & = & X(k)'X(k) - X(k)'X(X'X)^{-1}X(k)'X(k) \\
				  &   & - X(k)'X(k)(X'X)^{-1}X'X(k) + X(k)'X(k)(X'X)^{-1}X(k)'X(k) \\
				  & \stackrel{P}{\rightarrow} & \pi Q - \pi QQ^{-1} \pi Q - \pi QQ^{-1} \pi Q + \pi QQ^{-1} \pi Q \\
				  & = & \pi (1-\pi)Q
\end{eqnarray*}

By continuous mapping theorem we have $(X^{*}(k)'X^{*}(k))^{-1}\stackrel{P}{\rightarrow}(\pi (1-\pi))^{-1}Q^{-1}$.
>>>>>>> 2966618e04912eba46807909c13546d2481a77e9

\begin{proof}[Proof of Proposition~\ref{thm:2}]
The proof of this proposition is adapted from Hansen \cite{hansen2009averaging}. By projection arguments, $P(m) = P + P^{*}(m)$, where $P = X(X'X)^{-1}X'$, $P^{*}(m) = X^{*}(m)(X^{*}(m)'X^{*}(m))^{-1}X^{*}(m)'$, $X^{*}(m) = X(m) - PX(m) = X(m) - X(X'X)^{-1}X'X(m) = X(m) - X(X'X)^{-1}X(m)'X(m)$, and $X(m)$ is the matrix of stacked regressors $x_{t}(t < m)$, the cross-validation penalty term can be expanded as:
    \begin{eqnarray*}
    e'P(m)e & = & e'Pe + e'P^{*}(m)e \\
            & = & e'Pe + e'X^{*}(m)(X^{*}(m)'X^{*}(m))^{-1}X^{*}(m)'e
    \end{eqnarray*}

\noindent We start by showing the asymptotic distribution of the second term on the right-hand-side of the above equation, $e'P^{*}(m)e = e'X^{*}(m)(X^{*}(m)'X^{*}(m))^{-1}X^{*}(m)'e$. For the meat part, $X^{*}(m)'X^{*}(m)$, we have
    \begin{eqnarray*}
    X^{*}(m)'X^{*}(m) & = & (X(m)-X(X'X)^{-1}X(m)'X(m))'(X(m)-X(X'X)^{-1}X(m)'X(m)) \\
                      & = & X(m)'X(m) - X(m)'X(X'X)^{-1}X(m)'X(m) \\
    				  &   & \quad - X(m)'X(m)(X'X)^{-1}X'X(m) \\
                      &   & \quad + X(m)'X(m)(X'X)^{-1}X(m)'X(m) \\
    				  & = & X(m)'X(m) - X(m)'X(X'X)^{-1}X(m)'X(m)
    \end{eqnarray*}
\noindent From our assumptions and $\frac{m}{T} \rightarrow \pi$, by laws of large numbers, we have
    \begin{equation*}
        \frac{1}{T} X(m)'X(m) \stackrel{P}{\rightarrow} \pi Q
    \end{equation*}
and
    \begin{equation*}
        \frac{1}{T} X(m)'X(X'X)^{-1}X(m)'X(m) \stackrel{P}{\rightarrow} \pi QQ^{-1} \pi Q
    \end{equation*}
so
    \begin{equation*}
        \frac{1}{T} X^{*}(m)'X^{*}(m) \stackrel{P}{\rightarrow} \pi (1-\pi)Q
    \end{equation*}

\noindent By continuous mapping theorem we have
    \begin{equation*}
        (\frac{1}{T} X^{*}(m)'X^{*}(m))^{-1}\stackrel{P}{\rightarrow}(\pi (1-\pi))^{-1}Q^{-1}
    \end{equation*}
\noindent For the bread part, $X^{*}(m)'e = X(m) - X(X'X)^{-1}X(m)'X(m))'e$, we can show
    \begin{eqnarray*}
         X(m) - X(X'X)^{-1}X(m)'X(m))'e & = & X(m)'e - X(m)'X(m)(X'X)^{-1}X'e \\
                                        & = & \jia x_t e_t - \jia x_t x_t' \left( \jian x_t x_t' \right) ^{-1} \left( \jian x_t e_t \right)
    \end{eqnarray*}

\noindent Next, applying laws of large numbers and the mixing functional central limit theorem, we have
    \begin{equation*}
        \frac{1}{\sqrt{T}} \jia x_t e_t \Rightarrow W(\pi)
    \end{equation*}

    \begin{equation*}
        \frac{1}{T} \jia x_t x_t' \stackrel{P}{\rightarrow} \pi Q
    \end{equation*}

    \begin{equation*}
        \left( \frac{1}{T} \jian x_t x_t' \right) ^{-1} \stackrel{P}{\rightarrow} Q
    \end{equation*}

    \begin{equation*}
        \frac{1}{\sqrt{T}} \jian x_t e_t \Rightarrow W(1)
    \end{equation*}
where $W(1)$ is the Brownian motion vector with covariance matrix $\Sigma \equiv \lim\limits_{n\to\infty}$VAR$(\rn\jian X_i e_i)$, and $W(\pi)$ is the Brownian vector at time $\pi$.

<<<<<<< HEAD
\noindent Putting together results obtained above, we have
    \begin{equation*}
        \frac{1}{\sqrt{T}} X^{*}(m)'e \Rightarrow W(\pi) - \pi W(1)
    \end{equation*}
=======
\begin{eqnarray*}
e'P^{*}(k)e & = & e'X^{*}(k)(X^{*}(k)'X^{*}(k))^{-1}X^{*}(k)'e \\
            & \stackrel{P}{\rightarrow} & \frac{1}{\pi(1-\pi)} (W(\pi) - \pi W(1))'Q^{-1}(W(\pi) - \pi W(1)) =  \frac{\mathbf{B}(\pi)'\mathbf{B}(\pi)}{\pi(1-\pi)}\equiv J_0(\xi_{\delta})
\end{eqnarray*}
where $\mathbf{B}(\pi)$ is a Brownian bridge. Combined with Hansen's \cite{hansen2009averaging} theorem 1 without assuming conditional homoscedasticity or Andrews' \cite{andrews93} theorem 4, our results follow.
>>>>>>> 2966618e04912eba46807909c13546d2481a77e9

\noindent Then we have
    \begin{equation*}
        \frac{1}{T} e'P^{*}(m)e \Rightarrow \frac{1}{\pi(1-\pi)} (W(\pi) - \pi W(1))'Q^{-1}(W(\pi) - \pi W(1)) =  \frac{\mathbf{B}(\pi)'\mathbf{B}(\pi)}{\pi(1-\pi)}
    \end{equation*}
where $\mathbf{B}(\pi)$ is a Brownian bridge. Combined with Hansen's \cite{hansen2009averaging} theorem 1 without assuming conditional homoscedasticity or Andrews' \cite{andrews93} theorem 4, we have $\frac{1}{T} e'P^{*}(m)e \Rightarrow J_0(\xi_{\delta})$.

\noindent For the first component in the penalty term, $e'Pe$, we have
    \begin{equation*}
      e'Pe = (\frac{1}{\sqrt{T}} \jian x_t e_t)'(\frac{1}{T} \jian x_t x_t')^{-1}(\frac{1}{\sqrt{T}} \jian x_t e_t)
    \end{equation*}
\noindent Again, applying relevant laws of large numbers and central limit theorem,
    \begin{equation*}
      \frac{1}{\sqrt{T}} \jian x_t e_t \Rightarrow W(1)
    \end{equation*}

<<<<<<< HEAD
    \begin{equation*}
      \frac{1}{T} \jian x_t x_t' \stackrel{p}{\rightarrow} Q
    \end{equation*}
so
    \begin{equation*}
      e'Pe \stackrel{p}{\rightarrow} \Xi' Q^{-1} \Xi
    \end{equation*}
where $\Xi \sim N(0, \Sigma)$ .
=======
Under the assumption of conditional homoscedasticity, the above distribution is a $\chi^2$ distribution times the variance of the population error. If we relax the assumption of conditional homoscedasticity, $e'Pe$ does not converge to a $\chi^2$ distribution, instead, it converges to a weighted sum of $\chi^2$ distribution of one degree of freedom.
>>>>>>> 2966618e04912eba46807909c13546d2481a77e9

\noindent $\Sigma$ is symmetric and positive definite, $Q^{-1}$ is of the same rank of $\Sigma$, applying results of the distribution of quadratic forms (see section 5.4 of Ravishanker and Dipak \cite{linear_model_textbook}), we have
    \begin{equation*}
      e'Pe \stackrel{d}{\rightarrow} \sum_{j=1}^{k} \lambda_j \chi^2(1)
    \end{equation*}
where $\lambda_j$s are the eigenvalues of the matrix $Q^{-1}\Sigma$, $\chi^2(1)$ is a random variable having the $\chi^2$ distribution with degree of freedom one.

<<<<<<< HEAD
\noindent Collecting all results shown above, we have
    \begin{equation*}
      e'P(\hat{m})e \stackrel{d}{\rightarrow} \sum_{j=1}^{k} \lambda_j \chi^2(1) + J_0(\xi_{\delta})
    \end{equation*}
\end{proof}

\begin{proof}[Proof of Corollary~\ref{corollary:2}]
From proposition~\ref{thm:2}, take expectation of the CV penalty term,
    \begin{equation*}
      E(e'P(\hat{m})e) = E(\sum_{j=1}^{k} \lambda_j \chi^2(1)) + E(J_0(\xi_{\delta}))
    \end{equation*}
we have $E(\sum_{j=1}^{k} \lambda_j \chi^2(1)) = \sum_{j=1}^{k} \lambda_j$, applying Hansen's technique, approximate the value of $E(J_0(\xi_{\delta}))$ by averaging two extreme cases, so $E(J_0(\xi_{\delta})) \approx \frac{1}{2}(\mathrm{tr}(\hat{Q}^{-1}\hat{\Sigma}) + 2\bar{p} - k) \equiv \bar{p}^{*}$. Then by the same procedure in the proof of corollary~\ref{corollary:2},
    \begin{equation*}
      \mathrm{CV}(w) = (w\hat{e} + (1-w)\tilde{e})'(w\hat{e} + (1-w)\tilde{e}) + 2(\mathrm{tr}(\hat{Q}^{-1}\hat{\Sigma}) + w \bar{p}^{*})
    \end{equation*}
\noindent The CV weight is the value in $[0, 1]$ that minimizes $\mathrm{CV}(w)$, so
    \begin{equation*}
      \hat{w} = 1 - \frac{\mathrm{tr}\left(\hat{Q}^{-1}\hat{\Sigma}\right) + 2\bar{p} - k}{2\left(\sum_{t=1}^{T}\tilde{e}_t^2 - \sum_{t=1}^{T}\hat{e}_t^2\right)}
    \end{equation*}
if $(\sum_{t=1}^{T}\tilde{e}_{t}^{2} - \sum_{t=1}^{T}\hat{e}_{t}^{2}) \geq \bar{p}^{*}$ while $\hat{w} = 0$ otherwise.
=======
\begin{eqnarray*}
e'Pe & \stackrel{d}{\rightarrow} & \sum_{j=1}^{k} \lambda_j \chi^2(1)
\end{eqnarray*}
where $\lambda_j$s are the eigenvalues of the matrix $L'Q^{-1}L$, $\Sigma = LL'$. Although the asymptotic distribution is not pivotal, since we are only interested in the first moment of this distribution, we can easily obtain its value by $\mathrm{E}(e'Pe) = \sum_{j=1}^{k} \lambda_j = \tr{(L'Q^{-1}L)} = \tr{(Q^{-1} \Sigma)}$, where $Q$ and $\Sigma$ can be estimated by their sample analogues. This can be done without relying on potentially time-consuming Monte Carlo simulation.
\end{proof}
\begin{proof}[Proof of Theorem~\ref{lem:2}]
The proof of this theorem follows Hansen\cite{hansen2009averaging}, just replace all relevant terms according to Theorem~\ref{thm:2}.
>>>>>>> 2966618e04912eba46807909c13546d2481a77e9
\end{proof}


\chapter*{Symbols}
\label{symbols}
\addToTOC{symbols}

\section*{Notation}

\begin{deflist}
\notation{y}{Scalars are denoted in lowercase italic.}
\notation{\z}{Vectors are denoted in lowercase boldface.}
\notation{\A}{Matrices are denoted in uppercase boldface.}
\notation{\ev{\cdot}}{Time-average of a signal.}
\notation{\abs{\cdot}}{Number of elements in a set; Magnitude.}
\notation{\had}{Elementwise multiplication. (``Hadamard product'').}
\notation{\sigmoid(\cdot)}{Sigmoid `squashing' function, $\reals \rightarrow (0, 1)$. $ \ \sigmoid(x) = \frac{1}{1 + \exp(-x)}$.}
\notation{\tanh(\cdot)}{Hyperbolic tangent, $\reals \rightarrow (-1, 1)$. $ \ \tanh(x) = 2\, \sigmoid(x) - 1$.}
\end{deflist}

\section*{Signals}
\label{sec:signals}

\begin{deflist}
\notation{\z_t}{Digitized LFP sample at discrete time step $t$. $\z_t \in \reals^C$, with $C$ the number of channels (i.e. the number of electrodes simultaneously recorded from). Input to an SWR detection algorithm.}
\notation{o_t}{Output signal of an SWR detection algorithm, $\in \reals$.}
\notation{n_t}{`Envelope'. Transformation of $o_t$, so that it is constrained to $\reals^+$. Should be high when the corresponding input sample $\z_t$ is part of an SWR segment, and low when it is not. $\ n_t = \abs{o_t}$ for online linear filters; $n_t = \sigmoid(o_t)$ for the RNN's of \cref{ch:RNN}.}
\notation{y_t}{Binary target signal, used when training data-driven SWR detection algorithms. We define $y_t = 1$ when the corresponding input sample $\z_t$ is part of an SWR segment, and $y_t = 0$ when it is not.}
\end{deflist}

\section*{Measures \& parameters}

\begin{deflist}
\notation{T}{Detection threshold applied to the envelope $n_t\,$. $T \in [\min{n_t},\ \max{n_t}]$. Each threshold $T$ yields a different $P$-value, $R$-value, $F_1$-value, etc.}
\notation{P}{Precision. Also known as positive predictive value. The fraction of correct detections, out of all detections.}
\notation{R}{Recall. Also known as sensitivity, hit rate, or true positive rate. The fraction of detected reference SWR segments, out of all reference SWR segments.}
\notation{F_\beta}{F-score: weighted harmonic mean of recall and precision. $F_\beta = \frac{(1+\beta^2) P R}{\beta^2 P + R}$. Measures detection performance ``for a user who attaches $\beta$ times as much importance to recall as to precision.'' \cite{Rijsbergen1979}}
\notation{F_1}{F-score where recall and precision are weighted equally.}
\notation{f_s}{Sampling frequency of a signal.}
\end{deflist}