\documentclass{report}
\usepackage{amsmath}
\usepackage{amssymb}
\begin{document}
\section{Marginal distribution and Conditional distribution}
\subsection{Abstract}
In this section, we study the marginal probability and conditional probability of multivariate Gaussian distribution
\subsection{Prior Knowledge}
In the previous chapter, we derived the probability density function of multivariate Gaussian distribution:
$$
x \backsim N(\mu, \Sigma) = \frac{1}{(2\pi)^{\frac{p}{2}}|\Sigma|^\frac{1}{2}}\exp(-\frac{1}{2}(x-\mu)^T \Sigma^{-1}(x-\mu))
$$
Now let's split the random variable $x$ into two parts:
$$
x \in \mathcal{R}^p \quad x_a \in \mathcal{R}^m \quad x_b \in \mathcal{R}^n \quad m+n=p \quad
$$
$$
x =
\left (
\begin{matrix}
x_a \\
x_b \\
\end{matrix}
\right )
\quad
\mu =
\left (
\begin{matrix}
\mu_a \\
\mu_b \\
\end{matrix}
\right ) \quad
\Sigma=
\left (
\begin{matrix}
\Sigma_{aa} & \Sigma_{ab}\\
\Sigma_{ba} & \Sigma_{bb}\\
\end{matrix}
\right )
$$
\subsection{Theorem}
$$
X \sim N(\mu, \Sigma) \quad Y=AX+B  \Longrightarrow Y \sim N(A\mu+B, A \Sigma A^T)
$$
\subsection{Derive Marginal Distribution}
$$
x_a = \left (\begin{matrix}I & 0\end{matrix}\right )\left (\begin{matrix}x_a \\x_b\\\end{matrix}\right ) + 0
$$
$$
E[x_a] = \left (\begin{matrix}I & 0\end{matrix}\right )\left (\begin{matrix}\mu_a \\\mu_b \\\end{matrix}\right ) = \mu_a
$$
$$
\begin{aligned}Var[x_a] &= \left (\begin{matrix}I & 0\end{matrix}\right )\left (\begin{matrix}\Sigma_{aa} & \Sigma_{ab} \\\Sigma_{ba} & \Sigma_{bb} \\\end{matrix}\right )\left (\begin{matrix}I \\0\end{matrix}\right ) \\&= \left (\begin{matrix}\Sigma_{aa} & \Sigma_{ab} \end{matrix}\right )\left (\begin{matrix}I \\0\end{matrix}\right )\\&=\Sigma_{aa}\end{aligned}
$$
$$
\therefore x_a \sim N(\mu_a, \Sigma_{aa})
$$
\subsection{Derive conditional Distribution}
Let's set:
$$
\begin{cases}
x_{b.a}=x_b - \Sigma_{ba} \Sigma_{aa}^{-1} x_a \\
\mu_{b.a} = \mu_b - \Sigma_{b.a} \Sigma_{aa}^{-1} \mu_a \\
\Sigma_{bb.a}=\Sigma_{bb} - \Sigma_{ba} \Sigma_{aa}^{-1} \Sigma_{ab}
\end{cases}
$$
$$
\begin{aligned}
x_{b.a} &= x_b - \Sigma_{ba} \Sigma_{bb}^{-1} x_a \\
&=
\left (
\begin{matrix}
- \Sigma_{ba} \Sigma_{bb}^{-1} & I\\
\end{matrix}
\right )
\left (
\begin{matrix}
x_a \\
x_b \\
\end{matrix}
\right ) + 0
\end{aligned}
$$
$$
\begin{aligned}
E[x_{b.a}]
&=
\left (
\begin{matrix}
-\Sigma_{ba} \Sigma_{aa}^{-1} & I
\end{matrix}
\right )
\left (
\begin{matrix}
\mu_a \\
\mu_b \\
\end{matrix}
\right )\\
&=\mu_b - \Sigma_{ba} \Sigma_{aa}^{-1} \mu_a\\
&=\mu_{b.a}
\end{aligned}
$$
$$
\begin{aligned}
Var[x_{b.a}]
&= \left ( \begin{matrix}
-\Sigma_{ba} \Sigma_{aa}^{-1} & I
\end{matrix} \right )
\left ( \begin{matrix}
\Sigma_{aa} & \Sigma_{ab} \\
\Sigma_{ba} & \Sigma_{bb} \\
\end{matrix} \right )
\left ( \begin{matrix}
-\Sigma_{aa}^{-1} \Sigma_{ba}^T \\
I
\end{matrix} \right ) \\
&=\left ( \begin{matrix}
0 & \Sigma_{bb} - \Sigma_{ba} \Sigma_{aa}^{-1} \Sigma_{ab} \\
\end{matrix} \right )
\left ( \begin{matrix}
-\Sigma_{aa}^{-1} \Sigma_{ba}^T \\
I
\end{matrix} \right ) \\
&=\Sigma_{bb} - \Sigma_{ba} \Sigma_{aa}^{-1} \Sigma_{ab}\\
&=\Sigma_{bb.a}
\end{aligned}
$$
$$
\therefore x_{b.a} \sim N(\mu_{b.a}, \Sigma_{bb.a})
$$
$$
\begin{cases}
\mu_{b.a} = \mu_b - \Sigma_{ba} \Sigma_{aa}^{-1} \mu_a \\
\Sigma_{bb.a}=\Sigma_{bb} - \Sigma_{ba} \Sigma_{aa}^{-1} \Sigma_{ab}
\end{cases}
$$
We slightly modify the formula:

$$
\begin{aligned}
x_{b.a}
&=x_b - \Sigma_{ba} \Sigma_{aa}^{-1} x_a\\
x_b | x_a&=x_{b.a} + \Sigma_{ba} \Sigma_{aa}^{-1} x_a\\
&=I x_{b.a} + C\\
\end{aligned}
$$
Here, all parts of the covariance matrix  $\Sigma_{ba} \ \Sigma_{aa}$  can be calculated, so it can be regarded as a constant.\\\\
And what we're asking for here is $x_b | x_a$, so $x_a$ is also known, so the second term of the above formula can be regarded as a constant.\\\\
therefore:
$$
E[x_b|x_a]=I E[x_{b.a}] + C = \mu_{b.a} + \Sigma_{ba} \Sigma_{aa}^{-1} x_a
$$
$$
Var[x_b|x_a]=IVar[x_{b.a}]I^T=\Sigma_{bb.a}
$$
then we get the conditional probability of multivariate gaussian distribution:
$$
x_b|x_a \sim N(\mu_{b.a} + \Sigma_{ba} \Sigma_{aa}^{-1} x_a, \Sigma_{bb.a})
$$
\end{document}


\chapter{Blind Source Separation on the Sphere}
\label{ch_mrs_ica}

\section{Introduction}
\index{blind source separation}
\index{ICA}

Blind Source Separation (BSS) is a problem that occurs in multi-dimensional data processing. The overall goal is to recover
unobserved signals, images or sources $S$ from mixtures of these sources $X$ observed typically at the output of an
array of sensors. The simplest mixture model takes the form:
\begin{equation}\label{model0}
X = A S
\end{equation}
where $X$ and $S$ are random vectors of respective sizes $m \times 1$, $n \times 1$ and $A$ is an $m \times n$ matrix. The
entries of $S$ are assumed to be independent random variables. Multiplying $S$ by $A$ linearly mixes the $n$ sources into
$m$ observed processes. \\

Independent Component Analysis methods were developed to solve the BSS problem, i.e. given a batch of $T$ observed
samples of $X$, estimate the mixing matrix $A$ and reconstruct the corresponding $T$ samples of the source vector $S$, relying
mostly on the statistical independence of the source processes. Note that with the above model, the independent sources can
only be recovered up to a multiplication by a non-mixing matrix i.e. up to a permutation and a scaling of the
entries of $S$. Although independence is a strong assumption, it is in many cases physically plausible. The point is that
it goes beyond the simple second order decorrelation obtained for instance using Principal Component Analysis (PCA) : decorrelation
is not enough to recover the source processes since any rotation of a white random vector remains a white random vector.\\

Algorithms for blind component separation and mixing matrix estimation depend on the model used for the probability distribution
of the sources. In a first set of techniques, source separation is achieved in a noise-less setting, based on the
non-Gaussianity of all but possibly one of the components. Most mainstream Independent Component Analysis (ICA)  techniques belong to this category : JADE~\citep{ica:jade},
FastICA, Infomax~\citep{ica:icabook}. In a second set of blind techniques, the components are modeled as Gaussian processes, either
stationary or non stationary and, in a given representation, separation requires that the sources have diverse, i.e. non
proportional, variance profiles. The Spectral Matching ICA method (SMICA) ~\citep{ica:Del2003}, considers in this sense the case of
mixed stationary Gaussian components and goes further than the above model Eq.~\eqref{model0} by taking into account additive
instrumental noise $N$:
\begin{equation}\label{model1}
X = A S + N
\end{equation}
Moving to a Fourier representation, the idea is that colored components can be separated based on the diversity of their power spectra.\\

The next  section give a short overview of two significant ICA methods mentioned above and implemented in the \mrs package:
JADE  and FastICA ~\citep{ica:icabook}.
This is followed by a description of ways to
combine wavelets and ICA techniques. Some useful properties of wavelet transforms can indeed come enhance the performance of ICA  methods in several situations.
Finally, we present the method GMCA, which performs a blind source separation using the the sparsity concept.

\section{JADE}
\index{ICA!jade}

The Joint Approximate Diagonalization of Eigenmatrices method (JADE) assumes the observed data $X$ follows the noiseless mixture
model~\eqref{model0} where the independent sources $S$ are non-Gaussian i.i.d.\footnote{The letters i.i.d. stand for
independently and identically distributed meaning that each entries of $X$ at a given time $t$ are independent of $X$ at any other
time $t'$ and that the distribution of $X$ does not depend on time.} random processes. The mixing matrix is assumed to be square
and invertible so that (de)mixing is actually just a change of basis.

As mentioned above, second order statistics do not retain enough information for source separation in this context: finding a change of
basis in which the data covariance matrix is diagonal will not in general enable to identify the independent sources properly. Nevertheless,
decorrelation is half the job~\citep{ica:tutorial} and one may seek the basis in which the data is represented by maximally independent
processes among those bases in which the data is decorrelated. This leads to so-called orthogonal algorithms: after a proper whitening of
the data by multiplication with the inverse of a square root of the covariance matrix of the data $W$, one is then seeking a rotation $R$
(which leaves things white) so that $\hat{ S}$ defined by
\begin{equation}
\hat{ S} = W^{-1} \, Y =  W^{-1}\, R \, X_{\textrm{white}}  = W^{-1}\, R \, W \, X
\end{equation}
and $\hat{B} = \widehat{A^{-1}} =  W^{-1}\, R \, W$ are estimations of the sources and of the inverse of the mixing matrix.\\

JADE is such an orthogonal ICA method and, like most mainstream ICA techniques, it exploits higher order statistics so as to achieve some
sort of non linear decorrelation. Precisely, in the case of JADE, statistical independence is assessed using fourth order cross cumulants :
%\begin{eqnarray} \nonumber
%F_{ijkl} & = & \textrm{cum}( y_i, y_j, y_k, y_l ) \nonumber \\
%  & = & \mathcal{E} (y_i y_j y_k y_l) - \mathcal{E} (y_i y_j)\mathcal{E} (y_k y_l) \nonumber \\
%  & & -\mathcal{E} (y_iy_l)\mathcal{E} ( y_j y_k)-\mathcal{E} (y_iy_k)\mathcal{E} (y_j y_k)
%\end{eqnarray}
\begin{eqnarray} \nonumber
F_{ijkl} & = & \textrm{cum}( y_i, y_j, y_k, y_l ) \nonumber \\
 & = & \mathcal{E} (y_i y_j y_k y_l) - \mathcal{E} (y_i y_j)\mathcal{E} (y_k y_l) -\mathcal{E} (y_iy_l)\mathcal{E} ( y_j y_k)-\mathcal{E} (y_iy_k)\mathcal{E} (y_j y_k)
\end{eqnarray}
where $\mathcal{E}$ stands for statistical expectation and the $y_i$'s are the entries of vector $Y$ modeled as random variables,
and the correct change of basis (i.e. rotation) is found by somehow diagonalizing the fourth order cumulant tensor.
Indeed, if the $y_i$'s were independent, all the cumulants with at least two different indices would be zero. As a consequence of
the independence assumption of the source processes $S$ and of the whiteness of $Y$ for all rotations $R$, the fourth order
tensor $F$ is well structured: JADE was precisely devised to take advantage of the algebraic properties of $F$. JADE's objective
function is given by
\begin{eqnarray}  \nonumber
%\mathcal{J}_{\textrm{jade}}( R )   &=& \sum _{ijkl \ne ijkk}  \textrm{cum}(  y_i, y_j, y_k, y_l )^2  \nonumber    \\
  \mathcal{J}_{\textrm{jade}}( R ) & =&  \sum _{ij}   \sum_{k \ne l} \textrm{cum}(  y_i, y_j, y_k, y_l )^2
\end{eqnarray}
which can be interpreted as a joint diagonalization criterion. Fast and robust algorithms are available for the minimization
of $\mathcal{J}_{\textrm{jade}}( R )$ with respect to $R$ based on Jacobi's method for matrix diagonalization~\citep{ica:pham2001}.
More details on JADE can be found in~\citep{ica:jade,ica:tutorial,ica:icabook}.

\subsubsection{JADE for spherical maps}

Applying JADE on multichannel data mapped to the sphere does not require any particular modification of the algorithm. Indeed, JADE estimates
the fourth order cumulant tensor from the available data samples assuming an i.i.d. random field. Hence, given a pixelization scheme on
the sphere such as provided by the Healpix package, JADE can be directly applied to the multichannel spherical data pixels.

\section{FastICA}
\index{ICA!fastica}

FastICA is by now a standard technique in ICA. Like JADE, it is meant for the analysis of mixtures of independent non-Gaussian sources in
a noise-less setting. A complete description of this method can be found in \citep{ica:icabook} and references therein\footnote{Many papers on this
algorithm are available at http://www.cs.helsinki.fi/u/ahyvarin/papers/fastica.shtml}. We give here a brief and simplified account
of the algorithm. FastICA, again like JADE, is a so-called orthogonal ICA method: the independent components are sought by maximizing a
measure of non-Gaussianity under the constraint that they are decorrelated. Intuitively, one should understand that mixtures of independent
non-Gaussian random variables tend to look more Gaussian. An enlightening view on the relation between mutual information, which is
a natural measure of independence, decorrelation and non-Gaussianity can be found in~\citep{ica:3easy,ica:geomindep}. Non-Gaussianity is assessed
in FastICA using a contrast function $G$ based on a non-linear approximation to negentropy~\citep{ica:icabook}. In practice, depending
on the application, different approximations or non-linear (non-quadratic) functions should be experimented with. In a simple deflation scheme,
for sphered data, the directions are found sequentially : a direction $r$ of maximal non-Gaussianity is sought by maximizing
\begin{equation}
J_G(r) = \Big( \mathcal{E} \{ G(r^T x_{\textrm{white}}  ) \} - \mathcal{E} \{ G(\nu ) \} \Big)^2
\end{equation}
where $\nu$ stands for centered unit variance Gaussian variable, under the constraint that $r$ has unit norm and that $r$ is orthogonal
to the directions found previously.\\ \\
The contrast function $G$ can for instance be chosen among the following~\citep{ica:icabook}:
\begin{eqnarray}
G_0 (u)  & = &  \frac{1}{a} \textrm{log}\,\textrm{cosh} (a u ) \nonumber    \\
G_1 (u)  & = &  -\frac{1}{a} \textrm{exp}(- a u^2 / 2 )   \nonumber	   \\
G_2 (u)  & = &  \frac{1}{4} u^4 \nonumber    \\
\end{eqnarray}
where $a$ is a constant to be determined depending on the application. It can be shown that the maxima of $J_G$ occur at certain maxima
of $\mathcal{E} \{ G(r^T x_{\textrm{white}} ) \} $. These are obtained for $r$ solution to :
\begin{equation}
\mathcal{E} \{ x_{\textrm{white}} g(r^T x_{\textrm{white}}  ) \} - \lambda r = 0
\end{equation}
where $\lambda$ is a constant easily expressed in terms of the optimal direction $r_0$, and $g$ is the derivative of $G$. Solving this
equation using Newton's method, and a few approximations, a fixed-point algorithm is derived which consists in repeating the
following two steps until convergence :
\begin{eqnarray}
r  & \leftarrow & \mathcal{E} \{ x_{\textrm{white}} g(r^T x_{\textrm{white}}  ) \} - \mathcal{E} \{ g'(r^T x_{\textrm{white}}  ) \} r     \nonumber  \\
r  & \leftarrow  &  \frac{r}{\| r \|}   \nonumber	   \\
\end{eqnarray}
A simple implementation of this algorithm is included in the present package. It is largely based on the $\textbf{Matlab}^{TM}$ code
available at www.cis.hut.fi/projects/ica/fastica/.

\section{Wavelet and BSS}
\index{wavelet!ICA}
\index{ICA!wavelet}

\label{sec:wjade}
\index{ICA!wjade}
\index{jade!wavelet}

Wavelets come into play as a sparsifying transform. Applying a wavelet transform on both sides of~\eqref{model0} does not affect the
mixing matrix and the model structure is preserved. Also, moving the data to a wavelet representation does not affect its information
content. However, the statistical distribution of the data coefficients in the new representation is different: wavelets are known to
lead to sparse i.i.d. representations of structured data. Further, the local (coefficient wise) signal to noise ratio
depends on the choice of a representation. A wavelet transform tends to grab the informative coherence between pixels while averaging
the noise contributions, thus enhancing structures in the data. Although the standard ICA model~\eqref{model0} is for a noiseless setting,
the derived methods can be applied to real data. Performance will depend on the detectability of significant coefficients i.e. on
the sparsity of the statistical distribution of the coefficients. Moving to a wavelet representation will often lead to more robustness to noise.

Once the data has been transformed to a proper representation (e.g. wavelets but also ridgelets and curvelets in the case of strongly
anisotropic 2D or 3D data), WJADE (resp. WFastICA) consists in applying the standard JADE (resp. FastICA) method to the new multichannel coefficients. Once the mixing matrix
is estimated, the initial source maps are obtained using the adequate inverse transform after some non linear denoising or thresholding of
the coefficients if necessary.

\section{Sparse Blind Source Separation: the GMCA method}
% \section{Generalized Morphological Component Analysis on the sphere}
\label{sec:gmca}

\subsection{Morpho-Spectral Diversity}
%\label{sec:morph-spec-diversity}
Extending the redundant representation framework to the multichannel case requires defining what a multichannel overcomplete representation is.
Let us assume in this section that $\A = [\varphi_{\nu,1}, \cdots, \varphi_{\nu, N_c}] \in \RR^{N_c \times N_s}$ is a \textit{known spectral} dictionary,
and $\W = [ \varphi_{1}, \cdots, \varphi_{T}] \in \RR^{N \times T}$ is a \textit{spatial} or \textit{temporal} dictionary\footnote{The adjectives
\textit{spectral} and \textit{spatial} that characterize the dictionaries are not formal. Owing to the symmetry of the multichannel sparse decomposition problems,
$\A$ and $\W$ have no formal difference. In practice and more particularly in multi/hyperspectral imaging, $\A$ will refer to the dictionary of physical spectra
and $\W$ to the dictionary of image/signal waveforms. In the BSS problem, $\A$ is unknown.}. We assume that each source $s_i$ can be represented as
a (sparse) linear combination of atoms in $\W$; $s_i=\W\alpha_i$. Let $\balpha$ the $N_s \times T$ matrix whose rows are $\alpha_i^\Tr$.

From Eq.~\eqref{model0}, the multichannel noiseless data $\bY$ can be written as
\be
\label{eq:tensor1}
\bY = \A\balpha\W^\Tr = \sum_{i=1}^{N_s}\sum_{j=1}^{T} \parenth{\varphi_{\nu,i}\varphi_{j}^\Tr}\alpha_i[j] ~.
\ee
Consequently, each column in of $\bY$ reads
\be
\label{eq:tensor2}
\bY[.,l] = \parenth{\A \otimes \W[l,.]} \mathrm{vect}(\balpha) ~, \quad \forall ~ l=1,\cdots,N ~,
\ee
and finally
\be
\label{eq:tensor3}
\mathrm{vect}(\bY) = \parenth{\A \otimes \W} \mathrm{vect}(\balpha) ~,
\ee
where $\otimes$ is the tensor (Kronecker) product and the operator $\mathrm{vect}$ stacks the columns of its argument in a long 1D vector.
This latter equation brings a clear and simple insight: the sparsity of the sources in $\W$ translates into sparsity of the multichannel data
$\bY$ in the multichannel tensor product dictionary ${\bf \Psi}=\A \otimes \W$.

The multichannel dictionary $\bf \Psi$ can also be seen as concatenation of multichannel atoms ${\bf \Psi}^{(ij)}=\varphi_{\nu,i}\varphi_{j}^\Tr$
which are rank-one matrices obtained from each atomic spectrum $\varphi_{\nu,i}$ and each spatial elementary atom $\varphi_{j}$ (see Eq.~\eqref{eq:tensor1}).

Some of the popular recovery results in sparse component analysis algorithm rely on the mutual coherence of
the dictionary~\citep{cur:elad02,miki:Gribonval-Nielsen,mca:Donoho-Elad}. In the multichannel case a quantity
of that kind can be defined. In fact, by standard properties of the tensor product, one can easily show that
the Gram matrix of a tensor product is the tensor product of the Gram matrices. Thus the mutual coherence of the multichannel dictionary $\bf \Psi$ is:
\index{coherence}
\begin{equation}
\label{eq:mmc}
0 \le \mu_{\bf \Psi}  =  \max\left\{\mu_{\A},\mu_{\W}\right\} < 1 ~ .
\end{equation}

This expression of mutual coherence is instructive as it tells us that multichannel atoms can be distinguished
based on their spatial or spectral morphology. In other words, discriminating two multichannel atoms $\Psi_{ij}$ and $\Psi_{i'j'}$ may put on different faces:
\begin{itemize}
\item{Spatial or temporal (respectively spectral) diversity:} in this case $i=i'$ and $j \neq j'$ (respectively $i \neq i'$ and $j = j'$).
These atoms have the same spectrum (respectively, spatial shape) but one can discriminate between them based on their
spatial (respectively, spectral) diversity. From \eqref{eq:mmc}, their coherence is lower than $\mu_{{\W}}$ (respectively $\mu_{{\A}}$).
Disentangling these multichannel atoms can equivalently be done in \ref{ch_sca_datarest}.

\item{Both diversities:} $i \neq i'$ and $j \neq j'$, this seems to be a more favorable scenario to differentiate
the atoms as they do not share neither the same spectrum nor the same spatial (or temporal) ``shape". Note that
from \eqref{eq:mmc}, the coherence between these atoms in this case is lower than $\mu_{{\A}}\mu_{{\W}} \le \max\left\{\mu_{\A},\mu_{\W}\right\}$.
\end{itemize}

\subsection{Multichannel Sparse Decomposition}

We embark from Eq.~\eqref{eq:tensor1}, where the multichannel dictionary ${\bf \Psi}$ is supposed to be overcomplete, i.e. $NN_c < TN_s$.
The goal is to recover the sparsest solution $\balpha$ from $\bY$ which requires solving:
\begin{equation}
\label{eq:multi_l0}
\min_{\balpha \in \RR^{N_s \times T}} \sum_{i=1}^{N_s}\norm{\alpha_i}_{0} \st \bY  = \A \balpha \W^\Tr.
\end{equation}
As justified in \ref{sect_mca}, this combinatorial problem can be replaced by its convex relaxation substituting the $\ell_1$ norm for the $\ell_0$ pseudo-norm, hence giving:
\begin{equation}
\label{eq:multi_l1}
\min_{\balpha \in \RR^{N_s \times T}} \sum_{i=1}^{N_s}\norm{\alpha_i}_{1} \st \bY  = \A \balpha \W^\Tr.
\end{equation}

As \eqref{eq:tensor3} is a vectorized monochannel form of \eqref{eq:tensor1}, what we are trying so do is actually to find
the sparsest solution of a monochannel underdetermined system of linear equations where the solution is sparse in an
overcomplete tensor product dictionary. Recovery properties of monochannel sparse decomposition by $\ell_1$ minimization
were overviewed in chapter~\ref{ch_sca_datarest}. Therefore, if one is able to translate those identifiability criteria
in the language of tensor product dictionaries, then we are done.

In particular, the coherence-based sparse recovery criterion given in \citep{DonohoHuo} is trivial to adapt owing to \eqref{eq:mmc}.
Indeed, if $\bY$ is $k$-sparse in the multichannel dictionary ${\bf \Psi}$ with $k < C(\mu_{{\bf \Psi}}^{-1}+1)$ for some $C > 0$ (typically $C=1/2$),
and the dictionary is sufficiently incoherent (both spectrally and spatially), then the solution of Eq.~\eqref{eq:multi_l1} is unique,
is a point of equivalence of Eq.~\eqref{eq:multi_l0} and Eq.~\eqref{eq:multi_l1}, and the recovery is stable to bounded noise on $\bY$.

Above, we addressed the multichannel sparse decomposition problem without assuming any constraint on the sparsity pattern
of the different channels. It is worth however pointing out that sparse recovery conditions from multichannel measurements
can be refined if some structured sparsity is hypothesized. For instance, for structured multichannel representation
(e.g. sources with disjoint supports) \citep{GN05} provided coherence-based sufficient recovery conditions by solving Eq.~\eqref{eq:multi_l1}.
One should note that despite apparent similarities, the multichannel sparse decomposition problem discussed here is conceptually
different from the one targeting \textit{simultaneous} sparse recovery of multiple measurements vectors (MMV) considered by several authors, see e.g.
\citep{CREK05,MalioutovMMV05,TroppMMV06,ChenHuo06,ArgyriouMMVLearning08,BachMMVLearning08,GribonvalMMV08,EldarMMV08,LouniciMMVLearning09,WainwrightMMV09}.
The latter are not aware of any mixing process via $\A$, and their goal is to recover $\balpha$ from MMV $\bY=\balpha\W^\Tr$ in which
the vectors $\alpha_i$, i.e. rows of $\balpha$, have a common sparsity pattern. However the MMV model can also be written
$\mathrm{vect}(\bY^\Tr) = \parenth{\W \otimes \I} \mathrm{vect}(\balpha^\Tr)$ as in Eq.~\eqref{eq:tensor3}. The most widely used approach
to solve the simultaneous sparse recovery problem with joint sparsity is to minimize a mixed $\ell_p-\ell_q$ norm of the form
$\sum_{j=1}^T\parenth{\norm{\balpha[.,j]}_p^q}^{1/q}$ for $p \geq 1, 0 \leq q \leq +\infty$.

\subsection{Generalized Morphological Component Analysis}
\label{subsec:gmca}
We now turn to the BSS problem and highlight the role of sparsity and morphological diversity as a source of contrast to solve it.
Towards this goal, we assume that the sources are sparse in the spatial dictionary $\W$ that is the concatenation of $K$ orthonormal bases
$\parenth{{\W}_{k}}_{k=1,\cdots,K}$: $\W = \left[{\W}_{1},\cdots,{\W}_{K} \right]$. The restriction to orthonormal bases is only formal
and the algorithms to be presented later still work in practice even with redundant sub-dictionaries $\W_k$.

\newpage
The Generalized Morphological Component Analysis framework assumes a priori that each source is modeled as the linear combination
of $K$ morphological components where each component is sparse in a specific basis:
\begin{eqnarray}
\label{eq:sourcecomponents}
\forall i \in \{1,\cdots,N_s\}; \qquad s_i & = & \sum_{k=1}^K x_{i,k} = \sum_{k=1}^K \W_k\alpha_{i,k} \\
& = & \W \alpha_i \qquad \mbox{ where } \alpha_i =  \left[\alpha_{i,1}^\Tr,\cdots,\alpha_{i,K}^\Tr \right]^\Tr ~ \nonumber
\end{eqnarray}
GMCA seeks an unmixing scheme, through the estimation of $\A$, which leads to the sparsest sources $\bf S$ in the dictionary $\W$.
This is expressed by the following optimization problem written in the augmented Lagrangian form:
\begin{multline}
\label{eq:optimgmca}
\min_{{\A},\alpha_{1,1},\cdots,\alpha_{N_s,K}} \frac{1}{2}\norm{\bY - {\A}\balpha\W^\Tr}^2_{\mathrm{F}} + \lambda \sum_{i=1}^{N_s} \sum_{k=1}^K \norm{\alpha_{i,k}}_p^p \\
\st \norm{a_i}_2 = 1 ~ \forall i \in \{1,\cdots,N_s\} ~
\end{multline}
where typically $p=0$ or its relaxed convex version with $p=1$, and $\norm{{\bf X}}_{\mathrm{F}}=\parenth{\trace({\bf X}^\Tr{\bf X})}^{1/2}$ is the Frobenius norm.
The unit $\ell_2$-norm constraint on the columns of $\A$ avoids the classical scale indeterminacy of the product $\bf AS$ in \eqref{model1}.
The reader may have noticed that the MCA problem in chapter~\ref{ch_sca_datarest} is a special case of the GMCA problem Eq.~\eqref{eq:optimgmca}
when there is only one source $N_s=1$ and one channel $N_c=1$ (no mixing). Thus GMCA is indeed a multichannel generalization of MCA.

The program \eqref{eq:optimgmca} is a notoriously difficult non-convex optimization problem even for convex penalties when $p \geq 1$.
More conveniently, following \eqref{model0}, the product $\bf AS$ can be split into $N_s \cdot K$ multichannel morphological components:
${\bf AS} = \sum_{i,k} a_i x_{i,k}^\Tr = \sum_{i,k} (a_i \alpha_{i,k}^\Tr) \W_k^\Tr$. Based on this decomposition, and inspired by
the block-coordinate relaxation as for MCA, GMCA yields an alternating minimization algorithm to estimate iteratively one term at a time \citep{starck:bobin07}.
We will show shortly that the estimation of each morphological component $x_{i,k} = \W_k\alpha_{i,k}$ assuming $\A$ and $x_{\{i',k'\} \neq \{i,k\} }$
are fixed is obtained by simple hard or soft thresholding for $p=0$ and $p=1$.

Define the $(i,k)^{\textrm{th}}$ multichannel marginal residual by:
\begin{equation}
\label{eq:gmca_resi}
{\bf R}_{i,k} = {\bY} - \sum_{i' \neq i}   \sum_{k'\neq k}    a_{i'} x_{i',k'}^\Tr ~
\end{equation}
as the part of the data $\bY$ unexplained by the multichannel morphological component $a_i x_{i,k}^\Tr$. Estimating $x_{i,k} = \W_{k} \alpha_{i,k}$,
assuming $\A$ and the other components $x_{(i',k') \neq (i,k)}$ are fixed, leads to the component-wise optimization problem:
\begin{equation}
\label{eq:componentwisephi}
\min_{x_{i,k} \in \RR^{N}} \frac{1}{2}\norm{{\bf R}_{i,k} - (a_i \alpha_{i,k}^\Tr)\W^\Tr}_{\mathrm{F}}^2 +  \lambda \norm{\alpha_{i,k}}_p^p ~
\end{equation}

Since here ${\W}_k$ is an orthogonal matrix, with calculations using proximity operators, it can be shown that the unique solution
of Eq.~\eqref{eq:componentwisephi} is obtained by a hard ($p=0$) or soft ($p=1$) thresholding. Hence, the closed-form estimate of
the morphological component $x_{i,k}$ is:
\begin{equation}
\label{eq:st_update}
\tilde{x}_{i,k} = \Delta_{\W_k,\lambda^\prime} \parenth{\frac{1}{\norm{a_i}_2^2} {\bf R}_{i,k}^\Tr a_i} ~
\end{equation}
where $\lambda^\prime=\lambda/\norm{a_i}_2^2$ for soft thresholding and $\lambda^\prime=\sqrt{2\lambda}/{\norm{a_i}_2}$ for hard thresholding.
The operator $\Delta_{{\bf D}, \lambda}(x)$ consists of (i) computing the coefficients of $x$ in the dictionary ${\bf D}$, (ii)
thresholding (soft or hard) the obtained coefficients with the threshold $\lambda$, and (iii) reconstructing from thresholded coefficients:
\begin{equation}
\Delta_{{\bf D},\lambda}(x) = {\bf D} \Thres_{\lambda} \left( {\bf D}^\Tr x \right)
\end{equation}
$\Thres_{\lambda}$ is either a hard or a soft thresholding. When $\W_k$ is redundant, \eqref{eq:st_update} is only the first iteration
of a forward-backward splitting recursion, and which should be used when $\W_k$ is overcomplete.
However in practice \eqref{eq:st_update} can still be used to save computation time.
\index{iterative!hard thresholding}
\index{iterative!soft thresholding}

Now, considering $\{a_{i'}\}_{i' \neq i}$ and all morphological components as fixed, and recalling that $N_c \geq N_s$, updating the column $a_i$ is then just a least-squares estimate
\begin{equation}
\label{eq:a_update}
\tilde{a}_i = \frac{1}{\norm{s_i}^2_2} \left({\bf Y} - \sum_{i' \neq i} a_{i'} s_{i'}^\Tr\right) s_i ~
\end{equation}
where $s_i = \sum_{k=1}^K x_{i,k}$. This estimate is then projected onto the unit sphere to meet the unit $\ell_2$-norm constraint in Eq.~\eqref{eq:optimgmca}.
The GMCA algorithm is summarized in Algorithm~\ref{algo_gmca}.

{\linespread{1}
\begin{algorithm}[htb]
\caption{GMCA algorithm.}
\label{algo_gmca}
\noindent{\bf Task:} Sparse Blind Source Separation.\\
\noindent{\bf Parameters:} The data $\bY$, the dictionary $\W=[\W_1 \cdots \W_K]$, number of iterations $\niter$, number of sources $N_s$ and channels $N_c$, stopping threshold $\lambda_{\min}$, threshold update schedule.\\
\noindent{\bf Initialization:} $x_{i,k}^{(0)} = 0$ for all $(i,k)$, $\A^{(0)}$ random and threshold $\lambda_0$.\\
\noindent{\bf Main iteration:} \\
\For{$t=1$ {\bf to} $\niter$}{
    \For{$i=1,\cdots,N_s$ }{
    \For{$k=1,\cdots,K$ }{
       Compute the marginal residuals: $${\bf R}_{i,k}^{(t)} = {\bY}- \sum_{(i',k') \neq (i,k)} {a}_{i'}^{{(t-1)}}{x}_{i',k'}^{{(t-1)}^\Tr}.$$
       Estimate the current component ${x}_{i,k}^{(t)}$ via thresholding with threshold $\lambda_t$:\\
       ${x}_{i,k}^{(t)} = \Delta_{\W_k, \lambda_t}\left({\bf R}_{i,k}^{{(t)}^\Tr}{a}_i^{{{(t-1)}}}\right)$.
    }
Update $i$th source $s_i^{(t)} = \sum_{k=1}^K x_{ik}^{(t)}$. \\
Update $a_i$ assuming $a_{i' \neq i}^{(t)}$ and the morphological components $ {x}_{i,k}^{(t)} $ are fixed~:\\
 $ {a}_i^{{(t)}} = \frac{1}{\|{s}_i^{(t)}\|_2^2} \left({\bY} - \sum_{i' \neq i}^{N_s} {a}_{i'}^{(t-1)} {s}_{i'}^{{(t)}^\Tr} \right){s}_i^{{(t)}}$ and normalize to a unit $\ell_2$ norm.
}
Update the threshold $\lambda_t$ according to the given schedule.\\
\lIf{$\lambda_t \leq \lambda_{\min}$} stop.
}
\noindent{\bf Output:} Estimated sources $\big(s^{(\niter)}_i\big)_{i=1,\cdots,N_s}$ and mixing matrix ${\A}^{(\niter)}$.
\end{algorithm}}

For $p=1$ and fixed threshold $\lambda$, Algorithm~\ref{algo_gmca} can be shown to converge to a stationary point, see \citep{Tseng01,bobin-gmca-cmb}.
This point is not guaranteed to be even a local minimum of the energy, and this is even less clear for $p=0$. Thus, in the same vein as MCA, GMCA relies
on a salient-to-fine strategy using a varying threshold to mitigate the problem of sensitivity to initialization. More precisely, GMCA first computes
coarse versions of the morphological components for any fixed source $s_i$. These raw sources are estimated from their most significant coefficients in $\W$.
Then, the corresponding column $a_i$ is estimated from the most significant features of $s_i$. Each source and its corresponding column of $\A$ are then
alternately and progressively refined as the threshold decreases towards $\lambda_{\min}$. This particular iterative thresholding scheme provides robustness
to noise and initialization by working first on the most significant features in the data and then progressively incorporating smaller details to finely
tune the model parameters. GMCA can be used with either linear or exponential decrease of the threshold as for MCA.

If $\A$ were known and fixed, the GMCA would be equivalent to performing an MCA sparse decomposition of $\bY$ in the tensor product
multichannel dictionary ${\A} \otimes \W$. But as GMCA also updates the mixing matrix at each iteration, it is able to learn the
spectral part of the multichannel dictionary directly from the data.

\subsubsection{The Thresholding Strategy}

\index{iterative!hard thresholding}
\index{iterative!soft thresholding}

\paragraph*{Hard or soft thresholding?} In practice, it was observed that hard thresholding leads to better results \citep{starck:bobin06,starck:bobin07}.
Furthermore, if $\A$ is known and no noise contaminates the data, GMCA with hard thresholding will enjoy the sparse recovery guarantees given in \citep{starck:bobin_2,BobinJMIV},
with the proviso that the morphological components are contrasted and sparse in a sufficiently incoherent multichannel dictionary ${\A} \otimes \W$.
%Furthermore in \citet{starck:bobin_2}, it was shown empirically that the use of hard-thresholding is likely to provide the $\ell_0$ sparse solution for
%the single channel sparse decomposition problem. By analogy, the use of a hard-thresholding operator is assumed to solve the multichannel $\ell_0$ quasi-norm
%problem instead of \eqref{eq:optim_l1}. Recent results give also theoretical support to iterative hard thresholding methods \citep{blumensath08,blumensath09,maleki09,donoho09}.

\paragraph*{Handling additive Gaussian noise.}
The GMCA algorithm is well suited to deal with data contaminated with additive Gaussian noise (see the next section for a Bayesian interpretation).
For instance, assume that the noise $\bf E$ in \eqref{model1} is additive white Gaussian in each channel, i.e. its covariance matrix
${\boldsymbol \Sigma}_{\bf E}$ is diagonal, and let $\sigma_{\bf E}$ be its standard deviation supposed equal for all channels for simplicity.
Then, Algorithm~\ref{algo_gmca} can be applied as described above with $\lambda_{\min}=\tau\sigma_{\bf E}$, where $\tau$ is chosen as in denoising methods,
typically taking its value in the range $[3,4]$. This attribute of GMCA makes it a suitable choice for use in noisy BSS.
GMCA not only manages to separate the sources, but also succeeds in removing additive noise as a by-product.

\subsection{The Bayesian Perspective}
GMCA can be interpreted from a Bayesian standpoint. For instance, let us assume that the entries of the mixtures $\parenth{y_i}_{i=1,\cdots,N_c}$,
the mixing matrix $\A$, the sources $\parenth{s_i}_{i=1,\cdots,N_s}$ and the noise matrix $\bf E$ are random processes. We assume that the noise
$\bf E$ is zero-mean Gaussian where the noise vector $\veps_i$ in each channel is white, but the noise between channels is possibly correlated with
known covariance matrix ${\boldsymbol \Sigma}_{\bf E}$. This means that the log-likelihood function takes the form:
\begin{equation}
LL(\bY\big|{\bf S},{\A},{\boldsymbol \Sigma}_{\bf E}) = \frac{1}{2} \norm{{\bY} - {\bf{AS}}}_{{\boldsymbol \Sigma}_{\bf E}}^2 ~,\ \text{where} \norm{\bf X}_{{\boldsymbol \Sigma}_{\bf E}}^2 = \trace\big({\bf X}^\Tr{\boldsymbol \Sigma}_{\bf E}^{-1}{\bf X}\big)
\end{equation}

We further assume that the uniform prior is imposed on entries of $\A$. Other priors on $\A$ could be imposed; e.g. known fixed column for example.
As far as the sources are concerned, they are known from \eqref{eq:sourcecomponents} to be sparse in the dictionary $\W$. Thus their coefficients
$\balpha=[\alpha_1,\cdots,\alpha_{N_s}]^\Tr$ will be assumed as drawn independently from a leptokurtic PDF with heavy tails
such as the generalized Gaussian distribution form:
\begin{multline}
\label{eq:indepas}
\qquad \pdf_{\balpha}(\alpha_{1,1},\ldots,\alpha_{N_s,K}) \propto \prod_{i=1}^{N_s}\prod_{k=1}^{K}\exp\parenth{-\lambda_{i,k}\norm{\alpha_{i}}_{p_{i,k}}^{p_{i,k}}} ~ \\
0 \leq p_{i,k} < 2 ~ \forall (i,k) \in \{1,\cdots,N_s\}\times\{1,\cdots,K\} ~
\end{multline}
Putting together the log-likelihood function and the priors on $\A$ and $\balpha$, the MAP estimator leads to the following optimization problem:
\begin{equation}
\label{eq:optim_bayes}
\min_{{\A},\alpha_{1,1},\cdots,\alpha_{N_s,K}} \frac{1}{2}\norm{\bY - {\A}\balpha\W^\Tr}^2_{{\boldsymbol \Sigma}_{\bf E}} + \sum_{i=1}^{N_s} \sum_{k=1}^K \lambda_{i,k}\norm{\alpha_{i,k}}_{p_{i,k}}^{p_{i,k}} ~
\end{equation}
This problem has strong similarity with that of Eq.~\eqref{eq:optimgmca}. More precisely, if the noise is homoscedastic and decorrelated between channels
(i.e. ${\boldsymbol \Sigma}_{\bf E} = \sigma_{\bf E}^2 {{\bf I}}$), if the shape parameters $p_{i,k}$ of the generalized Gaussian distribution prior
are all equal to $p$ and the scale parameters are all taken as $\lambda_{i,k}=\lambda/\sigma_{\bf E}^2$, and if the columns of $\A$ are assumed uniform
on the unit sphere, then Eq.~\eqref{eq:optim_bayes} is exactly Eq.~\eqref{eq:optimgmca}. Note that in the development above, the independence assumption in \eqref{eq:indepas}
does not necessarily entail independence of the sources. Rather it means that there are no a priori assumptions that indicate any dependency between the sources.

\subsection{The Fast GMCA Algorithm}
\label{gmca_algo}
\label{fast_gmca}
The goal here is to speed up the GMCA algorithm. As a warm-up, assume that the dictionary ${\W}$ is no longer redundant and reduces to a single orthobasis (i.e. $K=1$).
Let us denote $\Ya=\bY\W$ the matrix where each of its rows stores the coefficients of each channel $y_i$. The optimization problem Eq.~\eqref{eq:optimgmca} then becomes
(we omit the $\ell_2$ constraint on $\A$ to lighten the notation):
\begin{equation}
\label{eq:optim2}
\min_{{\A},{\bf \balpha}} \frac{1}{2}\norm{\Ya - {\A} \balpha}_\mathrm{F}^2 + \lambda \sum_{i=1}^{N_s} \norm{\alpha_{i}}_p^p ~
\end{equation}
where $p=0$ or $p=1$. The GMCA algorithm no longer needs to apply the analysis and synthesis operators at each iteration as only the channels $\bY$
have to be transformed once in $\W$. Clearly, this case is computationally much cheaper.

However, this is rigorously valid only for an orthobasis dictionary, and no orthonormal basis is able to sparsely represent large variety of signals
and yet we would like to use very sparse signal representations which motivated the use of redundancy in the first place. Arguments supporting the
substitution of \eqref{eq:optim2} for \eqref{eq:optimgmca} for a redundant dictionary ${\W}$ were given in \citep{starck:bobin07,bobin08_aiep}. The idea is to first
compute the sparsest representation of each channel $y_i$ in the redundant dictionary $\W$ using an appropriate (non-linear) decomposition algorithm (e.g. BP, MCA).
Now, $\Ya$ denotes the matrix where each row contains the sparse decomposition of the corresponding channel. Because the channels are linear mixtures
of the sources via the mixing matrix $\A$, the key argument developed by \citep{starck:bobin07} is that the sparse decomposition algorithm must preserve linear mixtures.
Descriptively, the sparsest decomposition provided by the algorithm when applied to each channel must be equal to the linear combination of the sparsest
decompositions of the sources. This statement is valid if the sources and the channels are identifiable, meaning that they verify sufficient conditions
so that their unique sparsest representation can be recovered by the decomposition algorithm. For instance, if MCA is used, then it is sufficient as in
\citep{starck:bobin_2,BobinJMIV} that the channels and the sources be sparse enough in an incoherent dictionary $\W$, and their morphological components
be sufficiently contrasted. See \citep{starck:bobin07,bobin08_aiep} for details.

Hence, under these circumstances, a fast GMCA algorithm can be designed to solve \eqref{eq:optim2} by working in the transform domain after decomposing each
observed channel $y_i$ in ${\W}$ using a sparse decomposition algorithm such as MCA. There is an additional important simplification when substituting problem
\eqref{eq:optim2} for \eqref{eq:optimgmca}. Indeed, since $N_c \geq N_s$ (i.e. overdetermined BSS), it turns out that \eqref{eq:optim2} is a multichannel overdetermined
least-squares fit with $\ell_0/\ell_1$-sparsity penalization. We again use an alternating minimization scheme to solve for $\A$ and $\balpha$:
\index{iterative!hard thresholding}
\index{iterative!soft thresholding}
\begin{itemize}
\item Update the coefficients: when $\A$ is fixed, since the quadratic term is strictly convex ($\A$ has full column-rank), the marginal optimization problem
can be solved by a general form of the forward-backward splitting iteration \citep{ChenRockafellar97}:
\begin{equation}
\balpha^{(t+1)} = \Thres_{\mu\lambda} \parenth{\balpha^{(t)} + \mu{\boldsymbol \Xi} \A^\Tr(\Ya - {\A}\balpha^{(t)})} ~
\end{equation}
where ${\boldsymbol \Xi}$ is a relaxation matrix such that the spectral radius of $({\bf I} - \mu{\boldsymbol \Xi}\A^\Tr\A)$ is bounded above by 1,
and the step-size $0 < \mu \leq 1/\opnorm{{\boldsymbol \Xi}\A\A^\Tr}$. Taking ${\boldsymbol \Xi} = (\A^\Tr\A)^{-1}$ ($\A^\Tr\A$ is non-singular
and a kind of Newton's method ensues) yields the closed-form
\begin{equation}
\tilde{\balpha}  =  \Thres_{\lambda}\parenth{\A^{+}\Ya}
\end{equation}
where $\Thres_{\lambda}$ is a thresholding operator (hard for $p=0$ and soft for $p=1$).
\item If $\balpha$ is fixed, and since $\balpha$ is full row-rank, the mixing matrix $\A$ is given by the least-squares estimate:
\begin{equation}
{\bf \tilde{A}} = \Ya\balpha^\Tr\parenth{\balpha\balpha^\Tr}^{-1} = \Ya\balpha^+ ~
\end{equation}
and the columns of ${\bf \tilde{A}}$ are then normalized.
\end{itemize}
Note that the latter two-step estimation scheme has a flavor of the alternating sparse coding/dictionary learning algorithm presented by \citep{ksvd:elad,fadili:peyrespie07} in a different framework.

This two-stage iterative process leads to the accelerated version of GMCA summarized in Algorithm~\ref{algo_fast_gmca}.
{\linespread{1}
\begin{algorithm}[htb]
\caption{Fast GMCA algorithm.}
\label{algo_fast_gmca}
\noindent{\bf Task:} Sparse Blind Source Separation.\\
\noindent{\bf Parameters:} The data $\bY$, the dictionary $\W=[\W_1 \cdots \W_K]$, number of iterations $\niter$, number of sources $N_s$ and channels $N_c$,
stopping threshold $\lambda_{\min}$, threshold update schedule.\\
\noindent{\bf Initialization:}
\begin{itemize}
\item $\balpha^{(0)} = 0$,  $\A^{(0)}$ a random matrix.
\item Apply the MCA Algorithm~\ref{algo_mca} with $\W$ to each data channel $y_i$ to get $\Ya$.
\item Set threshold $\lambda_0 = \max_{i,l}\abs{\Ya[i,l]}$.
\end{itemize}
\noindent{\bf Main iteration:} \\
\For{$t=1$ {\bf to} $\niter$}{
\begin{itemize}
\item  Update the coefficients $\balpha$:
      ${\balpha}^{(t+1)} =  \Thres_{\lambda_t}\big({\A}^{(t)^+} \Ya\big)$.
\item  Update the mixing matrix $\A$:
      ${\A}^{(t+1)} =   \Ya\balpha^{(t+1)^+}$, normalize columns to a unit $\ell_2$ norm.
\item Update  the threshold $ \lambda_t$ according to the given schedule.
\end{itemize}
\lIf{$\lambda_t \leq \lambda_{\min}$} stop.
}
Reconstruct the sources: $\tilde{s}_i   =    \sum_{k=1}^K {\W}_{k}  \alpha^{(\niter)}_{i,k}, i=1,\cdots,N_s$.\\
\noindent{\bf Output:} Estimated sources $\big(\tilde{s}_i\big)_{i=1,\cdots,N_s}$ and mixing matrix ${\A}^{(\niter)}$.
\end{algorithm}}

In the same vein as in Section~\ref{subsec:gmca}, the coarse-to-fine process is also at the heart of this fast version of GMCA
with the threshold that decreases with increasing iteration count. This again brings robustness to noise and initialization.

%\subsection{The GMCA model}
%\label{sec:model}
%
%The observation with detector $i$ is then a noisy linear mixture of $n$ independent sources $\{s_j\}_{j=1,\cdots,n}$ :
%$x_i = \sum_{j=1}^n a_{ij} s_j + n_i$. The coefficient $a_{ij}$ reflects the emission law of source $s_j$ in the
%frequency band of the $i$-th sensor; $n_i$ models instrumental noise. When $m$ sensors provide observations at
%different frequencies, this linear mixture model can be rewritten in a more convenient matrix formulation :
%\begin{equation}
%\label{eq:lm_model}
%{\bf X} = {\bf AS} + {\bf N}
%\end{equation}
%where ${\bf X}$ is the $m \times t$ data matrix the rows of which are the observed data maps in each channel, ${\bf A}$
%is the $m \times n$ mixing matrix, ${\bf S}$ is the $n \times t$ source matrix the rows of which are the sources $s_j$,
%and ${\bf N}$ is the $m \times t$ noise matrix.
%
%We further assume that all the protagonists of the model in Equation~\ref{eq:lm_model} are random components (variables
%or vectors). More particularly, the entries of the noise matrix ${\bf N}$ are assumed to be \textit{independently}
%distributed according to a zero mean Gaussian distribution with variance $\sigma_i^2$ depending on the detector.
%From physical considerations, ${\bf N}$ models instrumental noise the level of which varies independently from one
%detector to another. ${\bf N}$ is thus a random Gaussian variable with zero mean and covariance matrix
%${\bf \Gamma_N} = \mbox{diag}(\sigma_1^2,\cdots,\sigma_m^2)$. In practice, as the detectors are assumed to be accurately calibrated,
%${\bf \Gamma_N}$ is known with high precision. The log-likelihood function is then the following one :
%\begin{equation}
%\label{eq:ll}
%\log P({\bf X} \big| {\bf A},{\bf S},{\bf \Gamma_N}) = -\frac{1}{2} \|{\bf X} - {\bf AS}\|_{2,{\bf \Gamma_N}}^2 + C
%\end{equation}
%where $C$ is a constant. The notation $\| . \|_{2,{\bf \Gamma_N}}^2$ stands for the Frobenius norm of ${\bf Y}$ in the noise
%covariance metric : $\| Y \|_{2,{\bf \Gamma_N}}^2 = \mbox{Trace}\left( {\bf Y}^T {\bf \Gamma_N}^{-1} {\bf Y}\right)$.
%From a Bayesian point of view, adding physical priors should help the separation task. We first assume no particular knowledge
%about the emission laws of the components modeled by ${\bf A}$. For simplicity, we consider that each entry of the mixing
%matrix ${\bf A}$ is \textit{i.i.d.}\footnote{Independently and identically distributed.} from a uniform zero mean distribution.
%Note that it would be possible to add some physical constraint on the emission laws reflected in ${\bf A}$.
%
%In the general case, source separation is merely a question of diversity and contrast between the sources (see \citep{Cardo1}).
%For instance, on the one hand JADE relies on non-gaussianity to distinguish between the sources. On the other, SMICA takes advantage
%of the diversity of the mixed components' power spectra to achieve the separation task. ``Non-gaussianity" and ``power spectra diversity"
%are contrasts between the sources. A combination of both characteristics, ``Non-gaussianity" and ``power spectra diversity",
%was also proposed to separate CMB from kinetic SZ signal which are otherwise undistinguishable \citep{forni}. Recent work has
%already emphasized on sparsity as a source of diversity to improve component separation (see \citep{Zibu} and \citep{MMCA}).
%In that setting, each source $\{s_j\}_{j=1,\cdots,n}$ is assumed to be sparse in a representation (potentially overcomplete) $\mathcal{D}$.
%Formally, $\mathcal{D}$ is a fixed dictionary of signal waveforms written as a $T \times t$ matrix. We define the set of projection
%coefficients $\alpha_j$ such that : $\forall j \in \{1,\cdots,n\}, \quad s_j = \alpha_j \mathcal{D}$. Any source $s_j$ is said
%to be sparse in $\mathcal{D}$ if most of the entries of $\alpha_j$ are nearly zero and only a few have ``significant" amplitudes.
%When $\mathcal{D}$ is overcomplete ($T > t$), $\mathcal{D}$ is called a dictionary. Overcomplete representations attractiveness
%in image processing theory leans on their potential to generate very sparse representations of data based on their morphological
%content (see e.g. \citep{DH} and references therein).
%
%In the field of basic source separation we showed in \citep{MMCA} that morphological diversity and sparsity are key properties
%leading to better separation. We noticed that the gist of sparsity-based source separation methods leans on the rationale :
%``\textit{independent sources are distinctly sparse in a dictionary $\mathcal{D}$}". In that study, we considered the simple
%case of morphologically different sources : components were assumed to be sparsely represented in different sub-dictionaries.
%We illustrated that such sparsity prior provides a very effective way to distinguish between sources. In the present paper,
%we focus on a more general setting : the sources can have similar morphologies (\textit{i.e.} all the sources are sparsely
%represented over the whole $\mathcal{D}$). When the overcomplete dictionary $\mathcal{D}$ is made of the union of $D$ orthonormal
%bases (\textit{i.e.} $\mathcal{D} = \left[\Phi_1,\cdots,\Phi_D\right]$) then each source is modeled as the linear combination
%of $D$ so-called morphological components (see \citep{SED} for details on Morphological Component Analysis) - each morphological
%component being sparse in a different orthonormal basis $\{\Phi_1,\cdots,\Phi_D\}$:
%\begin{eqnarray}
%\forall j\in \{1,\cdots,n\}, \quad s_j & = & \sum_{k=1}^D \varphi_{jk} = \sum_{k=1}^D \alpha_{jk} \Phi_k
%\end{eqnarray}
%From a statistical viewpoint, we assume that the entries of $\alpha_{jk} = \varphi_{jk}\Phi_k^T$ are \textit{i.i.d} from
%a Laplacian probability distribution with scale parameter $1/\mu$:
%\begin{equation}
%\label{eq:source_prior}
%P(\varphi_{jk}) \propto \exp\left(- \mu \|\varphi_{jk}\Phi_k^T\|_1\right)
%\end{equation}
%where the $\ell_1$-norm $\|.\|_1$ stands for $\|x\|_1 = \sum_{p=1}^t |x[p]|$ in which $x[p]$ is the $p$-th entry of $x$. In practice,
%the Laplacian prior is well adapted to model leptokurtic sparse signals. We classically assume that the morphological components are
%statistically mutually independent : $P({\bf S}) = \prod_{j,k} P(\varphi_{jk})$. Estimating the sources ${\bf S}$ is then equivalent
%to estimating the set of morphological components $\{\varphi_{jk}\}_{j=1,\cdots,n;k=1,\cdots,D}$. In this Bayesian context, we propose
%to estimate those morphological components $\{\varphi_{jk}\}$ and the mixing matrix ${\bf A}$ from a \textit{maximum a posteriori} (MAP)
%leading to the following optimization problem:
%\begin{equation}
%\left\{\{\hat{\varphi}_{jk}\},{\bf \hat{A}}\right\} = {\arg\max}_{\{\varphi_{jk}\},{\bf A}} P({\bf X}|{\bf A},\{\varphi_{jk}\},{\bf \Gamma_N}) \prod_{j,k}P(\varphi_{jk}) P({\bf A})
%\end{equation}
%where we further assumed that the morphological components $\{\varphi_{jk}\}$ are independent of ${\bf A}$. Owing to Equations~\ref{eq:ll}
%and \ref{eq:source_prior}, the mixing matrix ${\bf A}$ and the morphological components $\{\varphi_{jk}\}$ are obtained by minimizing
%the following negative log \textit{a posteriori}:
%\begin{equation}
%\label{eq:optim}
%\left\{\{\hat{\varphi}_{jk}\},{\bf \hat{A}}\right\} = {\arg\min}_{\{\varphi_{jk}\},{\bf A}}\|{\bf X} - {\bf AS}\|_{2,{\bf \Gamma_N}}^2 + 2 \mu \sum_{j=1}^n \sum_{k=1}^D \|\varphi_{jk}\Phi_k^T\|_1
%\end{equation}
%where $\forall j \in \{1,\cdots,n\},\quad s_j = \sum_{k=1}^D \varphi_{jk}$. Equation~\ref{eq:optim} leads to the GMCA estimates
%of the sources and the mixing matrix in a general sparse component separation context. Interestingly, in the case of CMB data,
%the sources we look for (CMB, galactic dust and SZ) are quite sparse in the same unique orthonormal wavelet basis. The dictionary
%$\mathcal{D}$ then reduces to a single orthonormal basis $\Phi$. In that case, since $\Phi$ is unitary, Equation~\ref{eq:optim}
%can be rewritten as follows :
%\begin{eqnarray}
%\label{eq:foptim}
%\left\{{\bf \hat{\alpha}},{\bf \hat{A}}\right\} &=& {\arg\min}_{{\boldsymbol \alpha},{\bf A}} \|{\bf X}\Phi^T - {\bf A\alpha}\|_{2,{\bf \Gamma_N}}^2 + 2 \mu \|{\boldsymbol \alpha}\|_1 \nonumber \\
%						&=& {\arg\min}_{{\boldsymbol \alpha},{\bf A}} f_\mu({\boldsymbol \alpha},{\bf A}) = {\arg\min}_{{\boldsymbol \alpha},{\bf A}} f_0({\bf A}, {\boldsymbol \alpha}) + 2\mu f_1({\boldsymbol \alpha})
%\end{eqnarray}
%where ${\boldsymbol \alpha} = {\bf S}\Phi^T$. Note that the estimation is done in the sparse representation $\Phi$ requiring a single
%transform of the data ${\bf X}\Phi^T$. To remain computationally efficient, GMCA relies on practical transforms which generally involve
%fast implicit operators (typical complexity of $\mathcal{O}\left(t\right)$ or $\mathcal{O}\left(t \log t \right)$). In \citep{Zibu},
%the authors also used a unique orthonormal wavelet basis. While a gradient descent is used in \citep{Zibu}, we use a fast and efficient
%iterative thresholding optimization scheme which we describe in the next section.
%
%\subsection{Solving the optimization problem}
%\label{sec:algo}
%The \textit{maximum a posteriori} estimates of the coefficients ${\boldsymbol \alpha}$ and the mixing matrix in Equation~\ref{eq:foptim}
%lead to a non-convex minimization problem. Note that in Equation~\ref{eq:foptim} the functional to be minimized suffers from several
%invariances : any permutation or rescaling of the sources and the mixing matrix leaves the product $\bf A{\boldsymbol \alpha}$ unaltered.
%The scale invariance is computationally alleviated by forcing the columns of ${\bf A}$ to have unit $\ell_2$ norm~: $\forall i\in{1,\cdots,n},\quad a^{i^T}a^i = 1$
%where $a^i$ is the $i$-th column of ${\bf A}$.
%
%As solutions of problem~(\ref{eq:foptim}) have no explicit formulation, we propose solving it by means of a block-coordinate relaxation
%iterative algorithm such that each iteration $(h)$ is decomposed into two steps : (i) estimation of the sources ${\bf S}$ assuming the
%mixing matrix is fixed to its current estimate ${\bf \hat{A}}^{(h-1)}$ and (ii) estimation of the mixing matrix assuming the sources are
%fixed to their current estimates ${\bf \hat{S}}^{(h)}$. It is not difficult to see that the objective MAP functional in (\ref{eq:foptim})
%is continuous on its effective domain and has compact level sets. Moreover, this objective function is convex in the source coefficient
%vectors $(\alpha_1,\ldots,\alpha_n)$, and $f_0$ has an open domain, is continuous and G\^ateaux differentiable. Thus by \cite[Theorem 4.1]{Tseng2001},
%the iterates generated by our alternating algorithm are defined and bounded, and each accumulation point is a stationary point of the MAP
%functional. In other words, our iterative algorithm will converge. Hence, at iteration $(h)$, the sources are estimated from a \textit{maximum a posteriori}
%assuming ${\bf A} = {\bf \hat{A}}^{(h-1)}$. By classical ideas in convex analysis, a necessary condition for ${\boldsymbol \alpha}$ to be
%a minimizer is that the zero is an element of the subdifferential of the objective at ${\boldsymbol \alpha}$. We calculate\footnote{For clarity,
%we drop the upper script $(h-1)$ and write $\hat{\bf A} = \hat{\bf A}^{(h-1)}$.}:
%\begin{equation}
%\label{eq:subdiff}
%\partial_{\boldsymbol \alpha} f_\mu({\boldsymbol \alpha},{\bf A})= -2{\bf {\bf A}}^T{\bf \Gamma_N}^{-1}({\bf X}\Phi^T - {\bf A}{\boldsymbol \alpha}) + 2\mu \partial_{\boldsymbol \alpha} \|{\boldsymbol \alpha}\|_1
%\end{equation}
%where $\partial_{\boldsymbol \alpha} \|{\boldsymbol \alpha}\|_1$ is defined as (owing to the separability of the prior):
%\[
%\partial_{\boldsymbol \alpha} \|{\boldsymbol \alpha}\|_1 = \left\{U \in \mathbb{R}^{n \times t} \Bigg|
%\begin{array}{ccc}
%U{j,k} & = \mbox{ sign}(\alpha_{j,k}), & ~ \alpha_{j,k} \neq 0 \\
%U{j,k} & \in [-1,1], & ~ \alpha_{j,k} = 0
%\end{array} \right\}.
%\]
%Hence, Equation \ref{eq:subdiff} can be rewritten equivalently as two conditions leading to the following (proximal) fixed point equation:
%\begin{equation}
%\label{eq:it_estim1}
%\begin{array}{cc}
%\hat{\alpha}_{j,k} = 0, & \text{if} ~ \left|{\left({\bf A}^T{\bf \Gamma_N}^{-1}{\bf X}\Phi^T\right)}_{j,k} \right| \leq \mu \\
%{\bf {\bf A}}^T{\bf \Gamma_N}^{-1}({\bf X}\Phi^T - {\bf A}\hat{\boldsymbol \alpha}) = \mu \mbox{ sign}\left(\hat{\boldsymbol \alpha}\right), & \text{otherwise}.
%\end{array}
%\end{equation}
%Unfortunately, Equation~\ref{eq:it_estim1} has no closed-form solution in general. It must be iterated and is thus computationally demanding.
%Fortunately, it can be simplified when ${\bf A}$ has nearly orthogonal columns in the noise covariance matrix (\textit{i.e.}
%${\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf \hat{A}} \simeq \mbox{diag}\left({\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf \hat{A}}\right)$).
%Let ${\bf C} = {\left({\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf \hat{A}}\right)}^{-1}{\bf A}^T{\bf \Gamma_N}^{-1}{\bf X}\Phi^T$, Equation~\ref{eq:it_estim1}
%boils down to the following set of equations $\forall j\in\{1,\cdots,n\}$:
%\begin{equation}
%\label{eq:it_estim}
%\begin{array}{ccc}
%\hat{\alpha}_{j,k} & = 0, \quad \text{if} ~ \left|{{\bf C}}_{j,k} \right| \leq \mu^{(h)} \sigma_j^2 \\
%\hat{\alpha}_j & = {\left[{\bf C}\right]}_j - \mu \sigma_j^2  \mbox{ sign}\left(\hat{\alpha}_j\right), \quad \text{otherwise}.
%\end{array}
%\end{equation}
%where $[{\bf Y}]_j$ is the $j$-th row of ${\bf Y}$. In practice, even if the approximation we make is not strictly valid, such a simplification
%leads to good computational results. These equations are known as soft-thresholding with threshold $\mu^{(h)} \sigma_j^2$. We define $\mathrm{ST}_{\delta}(.)$,
%the soft-thresholding operator with threshold $\delta$. At iteration $(h)$, the sources are thus estimated such that:
%\begin{equation}
%\hat{\alpha}_j^{(h)} = \mathrm{ST}_{\mu^{(h)} \sigma_j^2}\left(\left[{\bf C}\right]_j\right)
%\end{equation}
%The $j$th source is reconstructed as $\hat{s}_j^{(h)} = \hat{\alpha}_j^{(h)}\Phi$. The mixing matrix ${\bf A}$ is then estimated by a maximum
%likelihood estimate amounting to a simple least-squares update assuming ${\bf S}$ is fixed. The GMCA algorithm is then described in \ref{algo_gmca}:
%
%%\begin{flushleft}
%%\vspace{0.15in}
%%\centering
%%\begin{tabular}{|c|} \hline
%%\begin{minipage}[h]{0.95\linewidth}
%%\vspace{0.025in} \footnotesize{\textsf{1. Set the number of iterations $I_{\max}$ and thresholds $\delta_j^{(0)} = \mu^{(0)}\sigma_j^2$\\}
%%\textsf{2. While each $\mu^{(h)}$ is higher than a given lower bound $\mu_{min}$ (e.g. can depend on the noise variance), \\}
%%\hspace{0.1in} \textsf{-- Proceed with the following iteration to estimate source coefficients ${\boldsymbol \alpha}$ at iteration $h$ assuming ${\bf A}$ is fixed:}
%%\hspace{0.2in} \textsf{$\hat{\alpha}_j^{(h)} = \mathrm{ST}_{\mu^{(h)} \sigma_j^2}\left(\left[{\left({\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf \hat{A}}\right)}^{-1}{\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf X}\Phi^T\right]_j\right)$:\\}
%%\hspace{0.1in} \textsf{-- Update $\bf A$ assuming ${\boldsymbol \alpha}$ is fixed :}
%%\hspace{0.2in} \textsf{${\bf \hat{A}}^{(h)} = {\bf X}\Phi^T{\bf \hat{\boldsymbol \alpha}}^T\left({\bf \hat{\boldsymbol \alpha}}{\bf \hat{\boldsymbol \alpha}}^T\right)^{-1}$\\}
%%\textsf{-- Decrease the threshold $\mu^{(h)}$ following a given strategy}}
%%\vspace{0.05in}
%%\end{minipage}
%%\\\hline
%%\end{tabular}
%%\vspace{0.15in}
%%\end{flushleft}
%
%{\linespread{1}
%\begin{algorithm}[h]
%\caption{The Generalized Morphological Component Analysis algorithm.}
%\label{algo_gmca}
%\noindent{\bf Task:} Compute the GMCA of a discrete $X$.\\
%\noindent{\bf Parameters:} Data samples $X$, number of estimated sources $J$, dictionnary $\Phi$, noise covariance matrix ${\bf \Gamma_N}$.\\
%\noindent{\bf Initialization:}
%\begin{itemize}
%\item Set the number of iterations $I_{\max}$
%\item Set the initial thresholds $\delta_j^{(0)} = \mu^{(0)}\sigma_j^2$
%\item Set the final thresholds $\mu_{\min}$, it can depend on the noise standard deviation
%\end{itemize}
%\While{ $\mu^{(h)}_{j} > \mu_{\min}$ }{
%\begin{enumerate}[1.]
%\item estimate source coefficients ${\boldsymbol \alpha}$ at iteration $h$ assuming ${\bf A}$ is fixed :
%
%\lFor{$j = 1, \cdots, J$}{ $\hat{\alpha}_j^{(h)} = \mathrm{ST}_{\mu^{(h)} \sigma_j^2}\left(\left[{\left({\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf \hat{A}}\right)}^{-1}{\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf X}\Phi^T\right]_j\right)$ }
%\item Update $\bf A$ assuming ${\boldsymbol \alpha}$ is fixed :
%
%${\bf \hat{A}}^{(h)} = {\bf X}\Phi^T{\bf \hat{\boldsymbol \alpha}}^T\left({\bf \hat{\boldsymbol \alpha}}{\bf \hat{\boldsymbol \alpha}}^T\right)^{-1}$
%\item Decrease the threshold $\mu^{(h)}$ following a given strategy
%\end{enumerate}
%}
%\noindent{\bf Output:} $(\hat{\alpha}_j^{(m)})$ $j=1,\ldots,J$ with $m = I_{\max}$: coefficients of the separated components.
%\end{algorithm}
%}
%
%Note that the overall optimization scheme is based on an iterative and alternate thresholding algorithm involving a
%\textit{coarse to fine} estimation process. Indeed, \textit{coarse} versions of the sources (\textit{i.e.} containing
%the most ``significant" features of the sources) are first computed with high values of $\mu^{(h)}$.
%%\begin{equation}
%%\forall j\in\{1,\cdots,n\},\quad \hat{s}_j = \mathrm{ST}_{\mu^{(h)} \sigma_j^2}\left({\left({\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf \hat{A}}\right)}^{-1}{\bf \hat{A}}^T{\bf \Gamma_N}^{-1}{\bf X}\Phi^T\right)\Phi
%%\end{equation}
%In the early stages of the algorithm, the mixing matrix is then estimated from the most ``significant" features of the sources
%which are less perturbed by noise. The estimation of ${\bf A}$ and ${\bf S}$ is then refined at each iteration as $\mu^{(h)}$
%(and thus the thresholds $\{\mu^{(h)}\sigma_j^2\}_{j=1,\cdots,n}$) decreases towards a final value $\mu_{min}$. We already used
%this minimization scheme in \citep{MMCA} where this optimization process provided robustness and helped convergence even in a
%noisy context. Experiments in Section~\ref{sec:results} illustrate that it achieves good results with GMCA as well.


\hypertarget{classnumpp_1_1differentiation_1_1symbolic_1_1divide}{}\section{numpp\+:\+:differentiation\+:\+:symbolic\+:\+:divide$<$ Left, Right $>$ Class Template Reference}
\label{classnumpp_1_1differentiation_1_1symbolic_1_1divide}\index{numpp\+::differentiation\+::symbolic\+::divide$<$ Left, Right $>$@{numpp\+::differentiation\+::symbolic\+::divide$<$ Left, Right $>$}}
\subsection*{Public Types}
\begin{DoxyCompactItemize}
\item
\mbox{\Hypertarget{classnumpp_1_1differentiation_1_1symbolic_1_1divide_ac1d7c214342c5429411607c70c1eba2a}\label{classnumpp_1_1differentiation_1_1symbolic_1_1divide_ac1d7c214342c5429411607c70c1eba2a}}
{\footnotesize template$<$std\+::size\+\_\+t Active$>$ }\\using {\bfseries derivative} = simplify\+\_\+division$<$ simplify\+\_\+subtraction$<$ simplify\+\_\+multiplication$<$ typename Left\+::template derivative$<$ Active $>$, Right $>$, simplify\+\_\+multiplication$<$ Left, typename Right\+::template derivative$<$ Active $>$ $>$$>$, \hyperlink{classnumpp_1_1differentiation_1_1symbolic_1_1power}{power}$<$ Right, 2 $>$ $>$
\end{DoxyCompactItemize}
\subsection*{Static Public Member Functions}
\begin{DoxyCompactItemize}
\item
\mbox{\Hypertarget{classnumpp_1_1differentiation_1_1symbolic_1_1divide_abcc37b0e5f45acc06b30bef6f1431a5e}\label{classnumpp_1_1differentiation_1_1symbolic_1_1divide_abcc37b0e5f45acc06b30bef6f1431a5e}}
static C\+O\+N\+S\+T\+E\+X\+PR auto {\bfseries calculate} (auto \&\&values)
\end{DoxyCompactItemize}

The documentation for this class was generated from the following file\+:\begin{DoxyCompactItemize}
\item
differentiation/symbolic/arithmetic.\+hpp\end{DoxyCompactItemize}


\section{XN: A normalization method based on ReLU1}
This section introduces a new kind of parameters normalization methods based on Prof. Xu's idea for ReLU1 activation function.

\subsection{XN in progress}
Suppose we have a dataset $\mathcal{X}\subset\mathbb{R}^n$, which has $N$ samples, each sample $x\in \mathcal{X}$ with a label $y_x=e_i,i=1,2,...,c$, where $c$ is the number of classes.

Consider a one-hidden-layer neural network
\begin{equation}
	\label{1hidden}
	f(x)=W_2\text{ReLU1}(W_1x+b_1)+b_2,
\end{equation}
where $W_1\in\mathbb{R}^{n_1\times n},\ W_2\in\mathbb{R}^{c\times n_1}$, and a loss function \[L(X)=\frac{1}{N}\sum_{x\in X}\|f(x)-y_x\|^2_2,\] where $x\in\mathbb{R}^n$ is a sample and $y_x=e_i$ is the label.
%\subsubsection{Version 1}
%\begin{enumerate}
%	\item For an full-batch input matrix $X\in\mathbb{R}^{n\times N}$, where each column of $X$ is a sample, find the rows $y$ of $W_1X+b_1$ s.t. the $i$th element of $y$, denoted as $y_i$, satisfying
%	\begin{equation}
%		y_i\notin [0,1], \quad\forall i.
%	\end{equation}
%	Denoted the corresponding rows of $W_1$ and $b_1$ as $\bar W$ and $\bar b$.
%	\item Solve the optimization problem
%	\begin{equation}
%		\begin{aligned}
%			\min_{W,b}&\quad\|WX+b - \text{ReLU1}(\bar WX+\bar b)\|_F\\
%			\text{subject to}&\quad (WX+b)_{ij}\leq 0\text{ or } \geq 1,\quad \forall i,j.
%		\end{aligned}
%	\end{equation}
%
%	\item Replace $\bar W$ and $\bar b$ by $W$ and $b$.
%\end{enumerate}
\subsubsection{XN Version 2}
XN is independent with the SGD training process.

After an update of the neural network, XN can be applied to each hidden layer successively, and won't change the output of it. In this example, we have only one hidden layer. The XN algorithm can be written as follows.
\begin{enumerate}
	\item Denote $\Theta=(W_1\ b_1)\in\mathbb{R}^{n_1\times(n+1)}$, $\hat X=(X;\mathbf{1})\in\mathbb{R}^{(n+1)\times m}$ and $Y=\text{ReLU1}(W_1X+b_1)$.
	Split the columns of $\hat X$ to $X_I$, $X_J$, $X_K$, s.t. for every row $\theta_i$ of $\Theta$, $i=1,...,n_1$,
	\begin{equation}
	\begin{aligned}
	\theta_i X_I\leq 0,\\
	\theta_i X_J\geq 1,\\
	\theta_i X_K\in[0,1].
	\end{aligned}
	\end{equation}
	Denote IJ as the union of I and J. For every $\theta_i$, the optimization problem can be rewritten as
	\begin{equation}\label{standardv2opt}
	\begin{aligned}
	\min_\theta&\quad\|\theta  X_{IJ}-Y_{IJ}\|_2\\
	\text{subject to}&\quad\theta X_{I}\leq 0,\\
	&\quad\theta X_{J}\geq 1,\\
	&\quad\theta X_{K}=Y_{K}.
	\end{aligned}
	\end{equation}

	\paragraph{Note}
	\begin{enumerate}
	\item
	If $X_K$ is row full-rank, the problem has only one solution, which is the original $w_1,b_1$.
	\begin{proof}
		Because $X_K$ is row full-rank, we have, by definition,
		\begin{equation}
		xX_K=0\Rightarrow x=0.
		\end{equation}
		If there are $\theta$, $\theta'$ s.t.
		\begin{equation}
		\theta X_K=Y_K,\quad \theta' X_K=Y_K,
		\end{equation}
		then
		\begin{equation}
		(\theta-\theta') X_K=0\Rightarrow \theta-\theta'=0.
		\end{equation}
	\end{proof}
	\item
	In practice, the cases when the equality constraint
	\begin{equation}
		\theta X_K=Y_K
	\end{equation}
    is under-determined	appear after several epochs of SGD training.
	\item
	\ref{standardv2opt} is a standard constraint optimization problem, with inequality constraints. It has no explicit solution, and can be solved by algorithms such as interior-point methods, the simplex algorithm, etc. In practice, we may turn to standard libraries to solve it.

	\end{enumerate}
%	\begin{equation}\label{v2opt}
%		\begin{aligned}
%			\min_{W,b}&\quad\|(WX+b - \text{ReLU1}(W_1X+b_1)\|_F\\
%			\text{subject to}&\quad (WX+b)_{ij}<0\text{ or } >1,\quad \forall (i,j)\in Z,\\
%			&\quad (WX+b)_{ij}=(W_1X+b_1)_{ij},\quad \forall (i,j)\notin Z.
%		\end{aligned}
%	\end{equation}
%
%	\eqref{v2opt} is equivalent to solve
%	\begin{equation}\label{v2eq}
%		WX-b=Y,\quad Y_{ij}=(W_1X+b_1)_{ij},\quad \forall (i,j)\notin Z,
%	\end{equation}
%	which is in some sense easier to solve when $Z$ is larger. If $N<n+1$, \eqref{v2eq} is undetermined in most cases, however this is hard to satisfy.
	\item Replace $W_1$ and $b_1$ by $W$ and $b$.
\end{enumerate}

%Consider \eqref{v2opt}. Denote $\Theta=(W\ b)\in\mathbb{R}^{n_1\times(n+1)}$, $\hat X=(X;\mathbf{1})\in\mathbb{R}^{(n+1)\times m}$ and $Y=\text{ReLU1}(W_1X+b_1)$,
%\begin{equation}\label{simplifiedv2opt}
%	\begin{aligned}
%		\min_\Theta&\quad\|\Theta \hat X-Y\|_F\\
%		\text{subject to}&\quad(\Theta\hat X)_{ij}>1\ or\ <0,\quad \forall(i,j)\in Z,\\
%		&\quad(\Theta\hat X)_{ij}=Y_{ij},\quad \forall (i,j)\notin Z.
%	\end{aligned}
%\end{equation}

\subsubsection{Numerical experiments}
Our network has two hidden layers, and XN will be applied to the first, second and both hidden layers, separately, i.e. after each update, which is SGD,
\begin{enumerate}
	\item apply XN to the first hidden-layer, or
	\item apply XN to the first and the second hidden-layer, or
	\item apply XN to the second hidden-layer.
\end{enumerate}
\begin{enumerate}
%	\item \textbf{Version 1.} In \eqref{v1}, no row $y$ of $W_1X+b_1$ satisfies $y_i\notin[0,1],\forall i$, no matter which layer to apply XN. In fact, there are always almost a half of $y_i\in[0,1]$.
	\item \textbf{XN Version 2.}

	Whether XN is used or not depends on the equality constraints and the error.

	Figure \ref{XNv2_Points} shows the loss function on point-separation problem, with XN on different layers. XN is applied only at a few time steps, as follows,
	\begin{enumerate}
		\item L, XN.

		[0, 1, 2, 3, 4, 5, 6, 7, 8, 12, 14, 15, 16, 17, 34, 35, 36, 101, 102, 975, 978, 982, 987, 988, 989, 1126, 1127, 1128, 1134, 1136, 1141, 1144, 1158, 1163, 1168, 1170, 1171, 1172, 1173, 11411, 11910, 11920, 11922, 11923, 17831, 17836, 17864, 17901, 17902, 17914, 17916].
		\item XN, XN. Only second layer uses XN.

		[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 17, 18, 115, 127, 16179, 16229, 16240, 16243, 16244, 16248, 16250].

		\item XN, L. No layer uses XN.
	\end{enumerate}

	\begin{figure}[H]
		\center
		\includegraphics*[width=0.8\textwidth]{./figures/XNv2_Points.png}
		\caption{Loss function on point-separation problem. [2, 20, 64, 9] shows the number of neurons at each layer (one input, two hidden, and one output layer). ['L', 'XN'] means hidden-layer 1 is Linear, and hidden-layer 2 applies XN.}
		\label{XNv2_Points}
	\end{figure}
%	In \eqref{standardv2opt}, if $\theta X_{K}=Y_{K}$ is overdetermined (for MNIST, it's may be overdetermined, because $X_K\in\mathbb{R}^{129\times m'}$, $m'\approx 30000$ (the size of the full-batch is 60000)), and
	\begin{figure}[H]
		\center
		\includegraphics*[width=0.8\textwidth]{./figures/XNv2_mnist100.png}
		\caption{Loss function on 100 samples of MNIST. \emph{blank} means without XN.}
		\label{XNv2_Points}
	\end{figure}

	\begin{figure}[H]
		\center
		\includegraphics*[width=0.8\textwidth]{./figures/XNv2_mnist500.png}
		\caption{Loss function on 500 samples of MNIST. \emph{blank} means without XN. XN is applied successfully after every epoch.}
		\label{XNv2_Points}
	\end{figure}

	\begin{figure}[H]
		\center
		\includegraphics*[width=0.8\textwidth]{./figures/XNv2_mnist500_XN0.png}
		\caption{Loss function on 500 samples of MNIST. XN is applied once before SGD.}
		\label{XNv2_Points}
	\end{figure}

\end{enumerate}

\subsubsection{Problems}
\begin{enumerate}
	\item
	The optimization problem is very large and the main matrix, $X_{IJ}X_{IJ}^T$, is not SPD, which means the solution is not unique. These make the solving process very time-consuming. This may be worked around by not requiring an optimal solution, but only a slightly better one.
	\item
	XN ia not directly applicable for the full MNIST dataset, because the equality constraint is over-determined at each epoch.
	\item
	The generalization ability of XN on MNIST is hard to test, because the subset dataset is too small, and, as a result, both SGD with XN and blank SGD can not make good prediction with it.
\end{enumerate}

\subsection{An example of XN (mini-batch version)}
Suppose we have a dataset $\mathcal{X}\subset\mathbb{R}^n$, which has $N$ samples, each sample $x\in \mathcal{X}$ with a label $y_x=e_i,i=1,2,...,c$, where $c$ is the number of classes.

Consider a one-hidden-layer neural network
\begin{equation}
  \label{1hidden}
f(x)=W_2\text{ReLU1}(W_1x+b_1)+b_2,
\end{equation}
where $W_1\in\mathbb{R}^{n_1\times n},\ W_2\in\mathbb{R}^{c\times n_1}$, and a loss function \[L(X)=\frac{1}{N}\sum_{x\in X}\|f(x)-y_x\|^2_2,\] where $x\in\mathbb{R}^n$ is a sample and $y_x=e_i$ is the label.

To train $f$ with XN, we have the following algorithm.
\begin{enumerate}
	\item Choose an $m$-sample mini-batch $X\subset\mathcal{X}$ randomly. $X$ is represented as a matrix $X\in\mathbb{R}^{n\times m}$ , where each column of $X$ is a sample.
	\item Compute the value $f_1$ of the hidden layer on $X$,
	\begin{equation}
		f_1(X)=
		\begin{pmatrix}
			W_1 & b_1
		\end{pmatrix}
		\begin{pmatrix}
			X \\ \mathbf{1}
		\end{pmatrix}
		\in\mathbb{R}^{n_1\times m}
	\end{equation}
	and the value $f_2$ of the output layer,
	\begin{equation}
		f_2(x)=
		\begin{pmatrix}
		W_2 & b_2
		\end{pmatrix}
		\begin{pmatrix}
		\text{ReLU1}(f_1(X)) \\ \mathbf{1}
		\end{pmatrix}
		\in\mathbb{R}^{c\times m}.
	\end{equation}
	\item (XN) Change the parameters of the hidden layer $W_1$, $b_1$ to $W$, $b$ such that
	\begin{equation}
		\begin{pmatrix}
			W & b
		\end{pmatrix}
		\begin{pmatrix}
			X \\ \mathbf{1}
		\end{pmatrix}
		=
		\text{ReLU1}\left(
		\begin{pmatrix}
			W_1 & b_1
		\end{pmatrix}
		\begin{pmatrix}
			X \\ \mathbf{1}
		\end{pmatrix}\right)
		=\text{ReLU1}(f_1(X)).
	\end{equation}
	Denote $\Theta=(W\ b)\in\mathbb{R}^{n_1\times(n+1)}$, $\hat X=(X;\mathbf{1})\in\mathbb{R}^{(n+1)\times m}$ and $Y=\text{ReLU1}(f_1(X))\in\mathbb{R}^{n_1\times m}$, the above equation can be rewritten as
	\begin{equation}\label{XNeq}
		\Theta \hat X=Y.
	\end{equation}
	The least square problem
	\begin{equation}\label{XNlst}
		\min_\Theta\|\Theta \hat X-Y\|_F
	\end{equation}
	gives a solution of \eqref{XNeq} if the minimal value is $0$. To assure $0$ is reachable, note that \eqref{XNeq} can be expanded as
	\begin{equation}
		\bar X\bar\Theta=\bar Y,\quad
		\bar X\in\mathbb{R}^{n_1m\times(nn_1+n_1)},\ \bar\Theta\in\mathbb{R}^{nn_1+n_1},\ \bar Y\in\mathbb{R}^{n_1m\times(nn_1+n_1)},
	\end{equation}
	so we can make $\bar X$ an underdetermined matrix by requiring $n_1m < nn_1+n_1$, or
	\begin{equation}
		m < n+1.
	\end{equation}
	That is to say, in a layer, the number of input samples should not be larger than the dimension of a sample. \eqref{XNlst} can then be solved by LAPACK package directly.
	If \eqref{XNlst} is not solvable, for example because of a $\bar X$ with a zero-row, then XN is not applied, i.e. $W_1,\ b_1$ is not changed.

	\textbf{Notes}.
	\begin{enumerate}
		\item Because $\hat X$ is not necessarily square, a plausible way to solve \eqref{XNeq} is by pseudo-inverse,
		\begin{align*}
		\Theta \hat X=Y&\Rightarrow\Theta \hat X{\hat X}^T=Y{\hat X}^T\\
		&\Leftrightarrow\Theta =Y{\hat X}^T(\hat X{\hat X}^T)^{-1}.
		\end{align*}
		However, because $\hat X\in\mathbb{R}^{(n+1)\times m}$ and we require $m<n+1$, $\hat X{\hat X}^T$ is not invertible. Even if $\hat X{\hat X}^T$ is invertible, we can not get $\Theta \hat X=Y$ if $\hat X$ is not invertible. If it is, we can solve the equation directly with $\hat X^{-1}$.
		\item Another way is to assume $\Theta=Z\bar X^T$, and we have
		\begin{align*}\\
		\Theta \hat X=Y&\Leftrightarrow Z\bar X^T \hat X=Y\\
		&\Leftrightarrow Z =Y({\hat X}^T\hat X)^{-1}.\\
		&\Rightarrow
		\Theta=Y({\hat X}^T\hat X)^{-1}\bar{X}^T.
		\end{align*}
		This is a feasible method. However, this method won't bring an obviously difference as the least square method, so we won't show the results of it.
	\end{enumerate}

	\item Update the parameters with gradient descent methods on the mini-batch.
	\item Go to 1 until a convergence.
\end{enumerate}

For a fully connected neural network with several hidden layers, we just change parameters at each layer successively. The above example shows the XN algorithm on mini-batches.
\subsection{XN and ReLU1}
ReLU1, denoted as $\tau$, is defined as
\begin{equation}
\tau(x) =
	\begin{cases}
		0 \quad & 0 < x, \\
		x \quad &0 \le x < 1, \\
		1 \quad &1 \le  x.
	\end{cases}
\end{equation}

This function has two inflection points $x=0$ and $x=1$. When $x\leq0$ or $x\geq1$, the gradient of $\tau$ is
\begin{equation}
	\frac{\D\tau}{\D x}=0.
\end{equation}

In the above example, for an input $\mathbf{x}=\Theta\hat X$, $\tau$ is applied to each element, thus does not change the dimension of $\mathbf{x}$.

The idea of XN is to adjust $\Theta$ so that if $\mathbf{x}_{ij}\notin [0,1]$, $\mathbf{x}_{ij}$ is moved to the nearest inflection point, as shown in \eqref{XNeq}. XN does not affect the output of each layer, thus does not change the model.

Recall that a neural network is defined as $f^J$, where
\begin{equation}
f^j = \theta^j \circ g^j \circ f^{j-1},\quad j=1,2,...,J.
\end{equation}

%With XN, the function is rewritten as
%
%\begin{equation}
%	f^j = \theta^j \circ \text{ReLU1} \circ\text{XN}\circ f^{j-1},\quad j=1,2,...,J.
%\end{equation}
%Note that
%\begin{equation}\label{key}
%	\text{XN}\circ f^{j-1}=\text{XN}\circ \theta^j \circ g^{j-1},
%\end{equation}
%the operator XN in fact changes $\theta^{j-1}$ to $\theta^*$, where
%\begin{equation}
%	\theta^*\circ g^{j-1}=\text{ReLU1}\circ f^{j-1}.
%\end{equation}
%
%Denote $\text{ReLU1}\circ f^{j-1}$ on a mini-batch $B$ as $G\in\mathbb{R}^{n\times m}$, where $m$ is the size of mini-batch and $n$ is the dimension of the input, and $g^{j-1}$ on $B$ as $\bar G$, we have
%\begin{equation}
%\theta\circ g^{j-1}=
%\begin{pmatrix}
%	\bar G^T & 1_m
%\end{pmatrix}
%\begin{pmatrix}
%	W^T\\b
%\end{pmatrix}
%:=\bar X\Theta=G.
%\end{equation}
%
%The above equation can then be viewed as an optimization problem
%\begin{align}\label{op_xn}
%	\min_{\Theta\in\mathbb{R}^{n_x}}\|\bar X\Theta-G\|_F,
%\end{align}
%which may have infinite solutions, in which case we may add a constraint to it, such as $\min \|\theta-\bar\theta\|$ or $\min\|\theta\|$. However, any solution of \eqref{op_xn} is adoptable.
%
%Another quick way to solve \eqref{op_xn} is as follows. Assuming $\Theta_k=\bar X^T y,k=1,2,...,n$, we have
%\begin{align*}
%	\bar X\bar X^Ty&=G_k,\\
%	y&=(\bar X\bar X^T)^{-1}G_k,\\
%	\Theta_k&=\bar{X}^T(\bar X\bar X^T)^{-1}G_k.
%\end{align*}
%
%To guarantee $(\bar X\bar X^T)^{-1}$ exists, a necessary condition is $m<n$.
%
%One can also use a package to compute the linear square problem directly. For example, in Python, numpy.linalg.lstsq() provides such an API. This function in facts uses LAPACK library, which further uses divide and conquer methods to solve the problem.
%
%The algorithms are as follows.

\begin{algorithm}[htb]
	\caption{Training Algorithm with XN}
	\label{alg0_XN}
	\begin{algorithmic}[1]
		\Require $\Theta_0$, learning rate $\eta_t$
		\For{$t = 0,1 \cdots$}
		\State Select a mini-bath $B_t$.
		\State Replace $\Theta_t$ with $\hat{\Theta}_t$,
		\begin{equation}
			\Theta_t\leftarrow \hat{\Theta}_t = \text{XN}(f^J, \Theta_t, B_t).
		\end{equation}
		\State Compute the gradient (BP),
		\begin{equation}
			g_t = \nabla_{\Theta} L_{B_t}(\Theta) |_{\Theta = \hat{\Theta}_t}.
		\end{equation}
		\State Update parameters $\Theta$ with
		\begin{equation}
			\Theta_{t+1} = \Theta_{t} - \eta_t g_t.
		\end{equation}
		\EndFor
	\end{algorithmic}
\end{algorithm}

\begin{algorithm}[htb]
\caption{XN: solve $\hat \Theta_t$ from $\Theta_t$ and $f^J$ on $B_t$}
\label{XN_solve}
\begin{algorithmic}[1]
\Require The neural network $f^J$,  current parameters $\Theta_t$, mini-batch $B_t$
\For{$j = 0:J-1$}
\For{$k = 1:n_{j+1}$}
\State Compute $\hat \theta^j_{t,k}$ such that
\begin{equation}\label{eq1:XN}
\hat{\theta}_{t,k}^j \bar{x}^j_i = g^{j+1}({\theta}_{t,k}^j \bar{x}^j_i),
\end{equation}
with $x^j_i = g^j(f^{j-1}(x_i))$ for $i \in B_t$. For example, $\theta=\bar{X}^T(\bar X\bar X^T)^{-1}g$.

\State Replace $\hat\theta$ by $\theta$,
\begin{equation}
	\hat\theta\leftarrow\theta.
\end{equation}
%\State If \eqref{eq1:XN} have infinite solutions, we can add those kind of constrains:
%\begin{equation}\label{eq1:constrain}
%\min \|\hat{\theta}_{t,k}^j  - \theta_{t,k}^j\|^2  \quad \text{or} \quad \min  \|\hat{\theta}_{t,k}^j  - \theta_{t,k}^j\|_{1}
%\end{equation}
\EndFor
\EndFor
\end{algorithmic}
\end{algorithm}

\section{Implementation}

XN is implemented with PyTorch as an independent layer object.

\section{Numerical Results}

We use a fully connected neural network, with two hidden layers, each has 100 neurons. Thus, the mini-batch size should be smaller than 101.

We will only do XN on the second hidden layer.

We want XN to be properly solved, so there is a parameter \emph{xn\_error} to control the error, i.e.
\begin{equation}
	\|\Theta \hat X-Y\|_F<\text{xn\_error}.
\end{equation}

A max-norm is more convenient, because it's not sensitive to batch-size. We still use F-norm here.

Also, we have two methods to solve XN, i.e. the least square and the pseudo-inverse one. We will denote them as \emph{lst} and \emph{inv} respectively.

We test XN on different datasets.

\subsection{Points-separation problem}
\subsubsection{Introduction}
The samples of points-separation problem with $9$ classes are shown as follows.

\begin{figure}[H]
	\center
	\includegraphics*[width=0.5\textwidth]{./figures/XN_Points_show.png}
	\caption{The training data of 9-class points-separation problem.}
\end{figure}

This problem can be easily solved by plain SGD, i.e. without BN or XN, the prediction of which is as follows.

\begin{figure}[H]
	\center
	\includegraphics*[width=0.5\textwidth]{./figures/XN_Points_SGD_pred.png}
	\caption{The prediction of the neural network trained by plain SGD.}
\end{figure}

\subsubsection{Loss on mini-batches}

The setup is as follows.

\begin{center}
	\begin{tabular}{|l|c|}
		\hline
		Batch size & 32\\
		\hline
		xn\_error & $10^{-4}$\\
		\hline
		Learning rate & $0.01$\\
		\hline
	\end{tabular}
\end{center}

Due to the size of xn\_error, not all training step will apply XN. Figure \ref{fig:XN_points_1} shows the value of loss function and whether using XN at each step.

\begin{figure}[H]
	\center
	\includegraphics*[width=10cm]{./figures/XN_Points.png}
	\caption{Above: loss function on mini-batch at each step; below: if using XN at each step, 1 if use XN, otherwise 0.}
	\label{fig:XN_points_1}
\end{figure}

\textbf{Observations on Figure \ref{fig:XN_points_1}}
\begin{enumerate}
	\item Within the first 100 steps, XN decreases the loss function.
	\item After 1000 steps, although the loss function on the current mini-batch won't change, it will increase sharply on the next mini-batch. XN will break the prediction on other mini-batches.
\end{enumerate}

XN breaks the prediction on other mini-batches even without SGD, as Figure \ref{fig:XN_points_onlyXN} shows.
\begin{figure}[H]
	\center
	\includegraphics*[width=10cm]{./figures/XN_points_onlyXN.png}
	\caption{At each step, when using XN, SGD is not used.}
	\label{fig:XN_points_onlyXN}
\end{figure}

We may add a parameter xn\_prob to control the probability of using XN. By changing the probability of XN, we can see how the loss function will change. Figure \ref{fig:XN_points_02} shows loss function on mini-batches with xn\_prob=0.2.

\begin{figure}[H]
	\center
	\includegraphics*[width=10cm]{./figures/XN_Points_02.png}
	\caption{Loss function on mini-batch, with xn\_prob=0.2.}
	\label{fig:XN_points_02}
\end{figure}

\subsubsection{Train the full-batch and adjust on mini-batch}
Another way to use XN is doing gradient descent on the full-batch, and doing XN on the mini-batch.

With the same xn\_error=$10^{-4}$ and other setups, figure \ref{fig:XN_points_full} shows value of loss function on the whole training dataset.
\begin{figure}[H]
	\center
	\includegraphics*[width=10cm]{./figures/XN5.png}
	\caption{Loss on the full-batch.}
	\label{fig:XN_points_full}
\end{figure}
\textbf{Observations on Figure \ref{fig:XN_points_full}}
\begin{enumerate}
	\item Within the first 100 steps, XN decreases the full loss function.
	\item After 1000 steps, it will increase sharply after XN. XN will break the prediction on other samples.
\end{enumerate}

We can decrease the XN error to $10^{-5}$, so  that only the first 100 steps will use XN. The loss is shown in figure \ref{fig:XN_points_full_100}. The first several XN speed up the training.
\begin{figure}[H]
	\center
	\includegraphics*[width=10cm]{./figures/XN1.png}
	\includegraphics*[width=10cm]{./figures/XN_Points_full_1e5.png}
	\caption{Loss on the full-batch. Only step 8, 21 and 50 use XN.}
	\label{fig:XN_points_full_100}
\end{figure}

\subsection{MNIST}
We now use the dataset MNIST. xn\_error is $10^{-6}$.

\subsubsection{Loss function with xn\_prob 0.2}
Figure \ref{fig:XN_MNIST_1} shows the loss function on mini-batches with XN-probability $0.2$.
\begin{figure}[H]
	\center
	\includegraphics*[width=8cm]{./figures/XN14.png}
	\caption{Loss function on mini-batches with xn\_prob $0.2$.}
	\label{fig:XN_MNIST_1}
\end{figure}

\subsubsection{Train the full-batch and adjust on mini-batch}
Using GD on the full-batch and XN on the mini-batch, Figure \ref{fig:XN_MNIST_full} shows the loss on the full training dataset.
\begin{figure}[H]
	\center
	\includegraphics*[width=8cm]{./figures/XN15.png}
	\caption{Loss function on the full-batch with xn\_prob $0.2$.}
	\label{fig:XN_MNIST_full}
\end{figure}
Decrease xn\_prob to $0.001$, we have Figure \ref{fig:XN_MNIST_full_0001} showing the loss function on the full-batch.
\begin{figure}[H]
	\center
	\includegraphics*[width=\textwidth]{./figures/XN17.png}
	\caption{Loss function on the full-batch with xn\_prob $0.001$.}
	\label{fig:XN_MNIST_full_0001}
\end{figure}

\textbf{Observations on Figure \ref{fig:XN_MNIST_full_0001}}. Every XN will increase the loss on other samples.

\subsection{Conclusion}
From the above results, although XN keeps the loss on the current mini-batch, it will increase on other samples.

Changing batch size (e.g. 2, 8, 32, 64) or learning rate (e.g. 0.1, 0.01) won't give different results, so we omit them.

To keep the loss on the full-batch is hard. First, changing $\Theta$ for a mini-batch will change the whole neural network, which will change the loss on other samples. Second, it's hard to do XN on the full-batch, because the number of the input samples, $m$, should not be larger than the dimension of a sample, $n$. $m$ is $50,000$ for MNIST full-batch, which is too large.

\subsection{Batch Normalization}

The model VGG11 is used to compare the performance of XN, XN as loss, BN and a blank setup. ImageNet, which has $1000$ classes, is used as dataset.

VGG11 can be defined as a function $f=f_2\circ f_1$, where $f_1$ is of convolutional layers and $f_2$ is of fully connected layers,
\begin{equation}
f_2=W_3\text{ReLU1}(W_2\text{ReLU1}(W_1x+b_1)+b_2)+b_3.
\end{equation}

Dropout is used after each ReLU1 in $f_2$.

\begin{center}
	\includegraphics*[width=\textwidth]{./figures/BN_ImageNet.png}
\end{center}

The figure above compares $90$ epochs of training, with or without BN. \emph{training top1} means the accuracy of prediction on a mini-batch, and \emph{training top5} means the top5 accuracy, i.e. if the largest 5 predictions of a sample include the right one, then the prediction is viewed as correct.

\subsection{XN, XN as loss and BN, ImageNet}
\begin{center}
	\includegraphics*[width=\textwidth]{./figures/XN_ImageNet.png}
\end{center}

There are two hidden layers in $f_2$, but we only do XN on the first one. If do both, the solution of the least square problem has large error. The figure above compares the first $4000$ steps of training.

The batch size is $256$ on training data, and the optimizer is SGD with momentum $0.9$.

During the training, the error of the least square problem is kept under $10^{-4}$, except for larger numbers like $0.1$ for several times.


\section{The {\variant{\nodeitems}} problem}
\label{sec:nodes}

In this section, we provide the formal problem definition
of the {\variant{\nodeitems}} problem variant and describe a greedy
polynomial-time algorithm for solving it.
\begin{problem}[{\variant{\nodeitems}}]
Given $G=(V,E)$, transition matrix $\transition$, initial distribution of items to nodes
$\initial$ and integer $k$, find
$S\subseteq V$ such that $|S|=k$ such that
$\objective_\shortnodeitems\left(S\right)$ is minimized.
\label{problem:nodes-variant}
\end{problem}
A brute-force way to solve Problem~\ref{problem:nodes-variant}
would be to evaluate the objective function over all node-sets of size $k$.
Obviously such an algorithm is infeasible -- and we thus
study a natural greedy algorithm for the problem, namely \nodegreedy.

\spara{The {\nodegreedy} algorithm:}
This is a greedy algorithm that performs $k$ iterations; at each iteration,
it adds one more
node in the solution.
If $S^t$ is the solution at iteration $t$,
then solution $S^{t+1}$ is constructed by finding the
node $u\in V\setminus S^t$ such that:
\begin{equation}\label{eq:nodegreedy}
v^\ast=\argmin_{v\in V\setminus S^t}\objective_\shortnodeitems\left(S^{t}\cup\{v\}\right).
\end{equation}
Although in the majority of our experiments that compare the brute-force solutions with those
of {\nodegreedy} the two solutions were identical, we identified some contrived instances
for which this was not the case. Thus, {\nodegreedy} is not an optimal algorithm for
Problem~\ref{problem:nodes-variant}.

\spara{Running time}
{\nodegreedy} evaluates Equation~\eqref{eq:nodegreedy} at each iteration.
A naive implementation of this would require computing Equation~\eqref{eq:nodevariance}
$O(|V|)$ times per iteration, each time using $O(|V|^2)$ numerical operations.
As a first improvement, we avoid the full double summation over $V$
via a summation over edges $E$,
\begin{align}
	\objective_{_\shortparentstransitions}(S) = & \sum_{u\in V}\initial^{\prime}(u)\sum_{v\in V\setminus S}\transition^{\prime}(u,v)\left(1-\transition^{\prime}(u,v)\right) \nonumber \\
	= &  \sum_{(u,v)\in E, v\in V\setminus S}
	\initial^{\prime}(u)\transition^{\prime}(u,v)\left(1-\transition^{\prime}(u,v)\right),
\end{align}
that involves $O(k|V||E|)$ numerical operations.

% Clearly, the above running time would make {\nodegreedy}
% infeasible to run even for small-size datasets.
We can further speed-up the algorithm if
we re-use at each step the computations
done in the previous one.
To see how, let $S_t$ (resp.\ $S_{t+1}$)
be the solution we construct after
$t$ (resp.\ $(t+1)$) iterations and
let $v^\ast$ be the node such that
$S_{t+1}=S_t\cup v^\ast$.
Then, for any $u\in V$ we have $
\rho(u,S_t) = \sum_{v\in S_t}\transition(u,v)
$, and therefore
\begin{eqnarray}\label{eq:rho}
% \rho(u,S_{t+1}) & = &\sum_{v\in S_{t+1}}\transition(u,v)\\
% & = & \sum_{v\in S_{t}}\transition(u,v) + \transition(u,v^\ast)\nonumber\\
% & = & \rho(u,S_{t})+ \transition(u,v^\ast)\nonumber
\rho(u,S_{t+1}) & = \rho(u,S_{t})+ \transition(u,v^\ast).
\end{eqnarray}
Moreover, for any $S\subseteq V$ let
\begin{align}
B(u,S) & = \sum_{(u,v)\in E\ s.t.\ v\in V\setminus S}\transition^{\prime}(u,v)\left(1-\transition^{\prime}(u,v)\right) \\
& = \sum_{v\in V\setminus S}\frac{\transition(u,v)}{1-\rho(u,S)}\left(1-\frac{\transition(u,v)}{1-\rho(u,S)}\right)\nonumber .
\end{align}
We can then express $B(u,S_{t+1})$ in terms of
$B(u, S_{t})$:
\begin{eqnarray}\label{eq:B}
\lefteqn{
B(u,S_{t+1})  =} \nonumber \\
& & B(u,S_t)-2\transition(u,v^\ast)\left(1-\rho(u,S_t)-\transition(u,v^\ast)\right).
\end{eqnarray}
Finally, using Equations~\eqref{eq:rho} and~\eqref{eq:B}
and algebraic manipulations,
we can express $\objective_{\shortnodeitems}(S_{t+1})$ as follows:
\begin{eqnarray}\label{eq:noderewrite}
\lefteqn{
\objective_{\shortnodeitems}(S_{t+1}) = }\\
&&\sum_{u\in V}\initial(u)\left(\frac{B(u,S_t)}{1-\rho(u,S_{t+1})}-2\transition(u,v^\ast)\right)\nonumber
\end{eqnarray}
Thus, if we store $B(u,S_t)$ and $\rho(u,S_t)$ at iteration $t$,
then evaluating
Equation~\eqref{eq:noderewrite} at iteration $t+1$
takes only $O(|V|)$ numerical operations.

For all iterations but the first one,
the above sequence of rewrites enables us to achieve
a speedup from $O(|V||E|)$ to $O(|V|^2)$ numerical operations
per iteration.
For the first iteration, initializing
the auxiliary quantities $B(u, \emptyset)$, $u\in V$, still takes
$O(|E|)$.
With this book-keeping, the running time of {\nodegreedy} is  reduced
from $O(k|V||E|)$ to $O(|E| + k|V|^2) = O(k|V|^2|)$.
Note also that \nodegreedy\ is amenable to parallelization,
as, given the auxiliary quantities from the previous step,
we can compute the objective function independently for each
candidate node.

% \emph{Optimality proof:} Our proof for the optimality of {\nodegreedy}, will
% first demonstrate that if node $v_1$ is the node picked by {\nodegreedy}
% in its first iteration, then $v_1$ is included in the optimal solution.
% Then,
% we will show that the solution to our original problem
% consists of $v_1$ plus the optimal solution to a modified problem that does
% not contain $v_1$.

% For the first part of the proof, let's assume that $S^\ast$ is the optimal solution
% of size $k$. Also, let $S$ be the solution of size $k$ constructed by {\nodegreedy} and
% let $v_1$ be the node picked by {\nodegreedy} in its first iteration. Before setting
% its solution in iteration $1$ {\nodegreedy} sets its solution to $S_0=\{\}$ and thus
% $B(u,S_0)$ is equal to some constant $C$, i.e., $B(u,S_0)=C$ for every $u\in V$.
% Therefore, from Equations~\eqref{eq:rho},~\eqref{eq:B}
% and~\eqref{eq:noderewrite} we know that
% \[
% v_1=\argmin_{v\in V}\sum_{u\in V}\initial(u)\left(\frac{C}{1-\transition(u,v)}-2\transition(u,v)\right).
% \]
% Now let's assume that $v_1\notin S^\ast$ and let $T^\ast\subset S^\ast$ such that
% $\left |T^\ast\right|=k-1$ and $S^\ast\setminus T^\ast = v'$.

% From the first step of of the {\nodegreedy} algorithm we know that
% \begin{eqnarray*}
% \sum_{u\in V}\initial(u)\left(\frac{C}{1-\transition(u,v_1)}-2\transition(u,v_1)\right)& \leq & \\\sum_{u\in V}\initial(u)\left(\frac{C}{1-\transition(u,v')}-2\transition(u,v')\right), & &
% \end{eqnarray*}
% which means that
% \begin{eqnarray*}
% \sum_{u\in V}\initial(u)\left(\frac{C}{1-\transition(u,v_1)}-2\transition(u,v_1)\right)& \leq & \\\sum_{u\in V}\initial(u)\left(\frac{C}{1-\transition(u,v')}-2\transition(u,v')\right), & &
% \end{eqnarray*}