\documentclass{article}
\usepackage{amsmath}

\title{A Simple Solver for Matrix Equations}
\author{Andrew Winkler, Ph.~D.}

\begin{document}

\maketitle

\section{Abstract}

We derive a solver for linear systems \begin{math}{}Ax=b\end{math}, which works for real, complex, rational,
and certain other scalars. It is related to the Cholesky decomposition, but sidesteps creating a factorization
to directly construct the solution. We include a C implementation of the most important case, that where
real numbers are represented as double precision numbers and the matrix \begin{math}A\end{math} is symmetric and positive semi-definite.

\section{Preliminaries}

We consider the linear system \begin{math}{}Ax=b\end{math}. If your only interest is in the real numbers,
you can skip the rest of this section with the notation that \begin{math}{}A^*\end{math} means the transpose of the matrix \begin{math}{}A\end{math}. In particular a number, which can be seen as a 1x1 matrix, is its own transpose.

The algorithm we describe works perfectly well for finite dimensional Hilbert spaces over various fields, to be described. However, since the method implicitly constructs a basis for the vector space, for ease of exposition and with no real loss of generality, we assume already chosen an orthonormal basis, with respect to which A is a matrix, and x and b are column vectors.

We can view rational numbers as lying in the larger field of real numbers; likewise the real numbers in the complex numbers, and the complex numbers in the quaternions. A quaternion satisfies \begin{math}{}(a + bi +cj + dk)^* = (a -bi -cj -dk)\end{math}, from which it is immediate that \begin{math}{}q^*q = a^2 + b^2 + c^2 + d^2 >= 0\end{math}. Moreover if \begin{math}{}q^*q =0\end{math} then \begin{math}{}q =0\end{math}. If
\begin{math}{}q* = q\end{math} then \begin{math}q\end{math} is real, and conversely.

The algorithm assumes only that the field is some subfield of the quaternions, which is closed under this conjugation operation, which is the same thing as saying that if \begin{math}{}\alpha\end{math} is in the field, so is \begin{math}{}\alpha^*\alpha\end{math}, or equivalently \begin{math}{}\alpha + \alpha^*\end{math}. This is of course true of the rationals, the reals, any subfield of the reals, the complexes, and the quaternions.

Recall that the adjoint of a matrix \begin{math}A\end{math}, denoted \begin{math}A^*\end{math}, is the transpose of the matrix obtained from \begin{math}A\end{math} by replacing each number \begin{math}{}a_{ij}\end{math} by its complex/quaternion conjugate \begin{math}{}a_{ij}^*\end{math}.

\section{Linear Equations}

We seek all \begin{math}{}x\end{math} for which \begin{math}{}Ax=b\end{math}. We start by noticing that if
\begin{math}{}Ax=b\end{math}, then also \begin{math}{}A^*Ax=A^*b\end{math}. The converse need not be true,
but \begin{math}{}\{x|Ax=b\} \subset \{x|A^*Ax = A^*b\}\end{math}, and in fact it's the simplest sort of subset:
it's either empty, or it's everything. To see this, note that if \begin{math}{}A^*Ax_1 = A^*Ax_2 \end{math},
then \begin{math}{}A^*A(x_1-x_2)=0 \end{math}, so \begin{math}{}0=(x_1-x_2)^*A^*A(x_1-x_2)=(A(x_1-x_2))^*(A(x_1-x_2)) \end{math}. But then \begin{math}{}A(x_1-x_2)=0 \end{math}, or \begin{math}{}Ax_1 = Ax_2 \end{math}.

In summary if \begin{math}{}A^*Ax_1 = A^*Ax_2 = A^*b \end{math}, then either \begin{math}{}Ax_1 = Ax_2 = b \end{math}, or else \begin{math}{}Ax_1 = Ax_2 \ne b \end{math}, in which case there is no solution.

So having found all solutions to the auxiliary equation, either they are precisely the solutions to the original equation, or the original equation has no solutions, as determined by picking any one of the candidate solutions and computing \begin{math}{}Ax\end{math}.

Notice that \begin{math}{}(A^*A)^* = A^*A^{**} = A^*A\end{math}, and that \begin{math}{}x^*(A^*A)x=(Ax)^*(Ax) >= 0\end{math}. Thus the general problem reduces to the special case considered in the next section.

\section{Linear Positive Equations}

We consider then the problem \begin{math}{}Ax=b\end{math} in the special case where
\begin{math}{}A^*=A\end{math} and \begin{math}{}x^*Ax >= 0\end{math} for all \begin{math}{}x\end{math}. Such a matrix is called positive semi-definite.

We write \begin{math}{}A\end{math} as \begin{math}{}\begin{bmatrix}\alpha & b^* \\ a & A^{'}\end{bmatrix}\end{math}. It is possible
that \begin{math}{}A\end{math} is 1x1 in which case all expressions involving
\begin{math}{}A\end{math},
\begin{math}{}a\end{math}, or
\begin{math}{}b\end{math} are empty sums, and therefor 0. If
\begin{math}{}A\end{math} is
\begin{math}{}n\end{math} by
\begin{math}{}n\end{math}, then
\begin{math}{}A^{'}\end{math} is
\begin{math}{}n-1\end{math} by
\begin{math}{}n-1\end{math}, and
\begin{math}{}a\end{math} and
\begin{math}{}b\end{math} are
\begin{math}{}n-1\end{math} column vectors, while
\begin{math}{}\alpha\end{math} is a scalar.

Since \begin{math}{}A=A^*\end{math},
 \begin{math}{}
\begin{bmatrix}\alpha & b^* \\ a & A^{'}\end{bmatrix} =
\begin{bmatrix}\alpha & b^* \\ a & A^{'}\end{bmatrix}^* =
\begin{bmatrix}\alpha^* & a^* \\ b & A^{'*}\end{bmatrix} \end{math}, so
\begin{math}{}\alpha^*=\alpha\end{math}, \begin{math}{}a=b\end{math}, and
\begin{math}{}A^{'*} = A^{'}\end{math}.

Moreover,
\begin{math}{}
0 <= \begin{bmatrix}1\\0\end{bmatrix}^*\begin{bmatrix}\alpha & a^* \\ a & A^{'}\end{bmatrix}\begin{bmatrix}1\\0\end{bmatrix}
= \begin{bmatrix}1&0\end{bmatrix}\begin{bmatrix}\alpha \\ a\end{bmatrix}
= \alpha
\end{math}

\subsection{Case 1}
\begin{math}{}\alpha=0\end{math}

Then in fact it's also true that \begin{math}{}a=0\end{math}. For
\begin{math}0 <= \begin{bmatrix}\chi\\a\end{bmatrix}^*\begin{bmatrix}0 & a^* \\ a & A^{'}\end{bmatrix}\begin{bmatrix}\chi\\a\end{bmatrix} = \begin{bmatrix}\chi^* & a^*\end{bmatrix} \begin{bmatrix}a^*a\\a\chi+A^{'}a\end{bmatrix}
= \chi^*a^*a + a^*a\chi + a^*A^{'}a
\end{math}.

Notice that it is here that we make use of the assumption that we started with
a positive semi-definite matrix.

If we restrict attention to rational numbers, then we get the equation of a line. The only lines that have no negative values are horizontal, so the slope \begin{math}{}2a^*a\end{math} must be zero, which means \begin{math}{}a\end{math} must be zero.

Our equation simplifies to
\begin{math}{}
\begin{bmatrix}0 & 0 \\ 0 & A^{'}\end{bmatrix}\begin{bmatrix}\chi\\x^{'}\end{bmatrix} =\begin{bmatrix}\beta\\b^{'}\end{bmatrix}
\end{math}, which reduces to the pair of equations \begin{math}0=\beta{}\end{math}, and
\begin{math}{}A^{'}x^{'}=b^{'} \end{math}. The first equation tells us that there can be no solution unless
\begin{math}{}\beta=0\end{math}. If, however, it is, then any solution \begin{math}{}x^{'}\end{math} of the
second equation, which we note, satisfies the same properties we required of
\begin{math}{}A\end{math}, can be combined with any number \begin{math}{}z_n\end{math} (where we use the dimension n to tag the variable, to prevent recursive name collisions)
into a solution, \begin{math}{}A\begin{bmatrix}z_n\\x^{'}\end{bmatrix} = b\end{math}

In particular, we can always take \begin{math}{}z_n\end{math} to be 0.

\subsection{Case 2}
\begin{math}{}\alpha>0\end{math}

Then
\begin{math}{}
\begin{bmatrix}\alpha & a^* \\ a & A^{'}\end{bmatrix}\begin{bmatrix}\chi\\x^{'}\end{bmatrix}=\begin{bmatrix}\beta\\b^{'}\end{bmatrix}
\end{math}
if and only if
\begin{math}{}\alpha\chi + a^*x^{'} = \beta\end{math} and
\begin{math}{}a\chi + A^{'}x^{'} = b^{'}\end{math}
if and only if
\begin{math}{}\alpha\chi= \beta - a^*x^{'} \end{math} and
\begin{math}{}a\chi + A^{'}x^{'} = b^{'}\end{math}
if and only if
\begin{math}{}\chi= \alpha^{-1}(\beta - a^*x^{'})\end{math} and
\begin{math}{}a\alpha^{-1}(\beta - a^*x^{'}) + A^{'}x^{'} = b^{'}\end{math}
if and only if
\begin{math}{}\chi= \alpha^{-1}(\beta - a^*x^{'})\end{math} and
\begin{math}{}a\alpha^{-1}(- a^*x^{'}) + A^{'}x^{'} = b^{'}-a\alpha^{-1}\beta\end{math}
if and only if
\begin{math}{}\chi= \alpha^{-1}(\beta - a^*x^{'})\end{math} and
\begin{math}{}(A^{'} - a\alpha^{-1}a^*)x^{'} = b^{'}-a\alpha^{-1}\beta\end{math}

So if we can solve the second equation, the first equation shows us how to convert it into a solution
of the original equation. Notice that if
\begin{math}{}b =0\end{math}, then so is the derived right hand side; a homogeneous problem generates
a homogeneous subproblem.

But this second equation is again of the same form, because
\begin{gather}{}(A^{'} - a\alpha^{-1}a^*)^* =
A^{'*} - a^{**}\alpha^{-1}a^* =
A^{'} - a\alpha^{-1}a^*
\end{gather}
and moreover
\begin{gather}{}
0 <= \begin{bmatrix}-\alpha^{-1}a^*x\\x\end{bmatrix}^*\begin{bmatrix}\alpha & a^* \\ a & A^{'}\end{bmatrix}\begin{bmatrix}-\alpha^{-1}a^*x\\x\end{bmatrix}
=
x^*(A^{'} - a\alpha^{-1}a^*)x
\end{gather}. So the derived submatrix is nonnegative, and positive if the original matrix is.

As in either case we have reduced to either the base case of dimension one or to a similar problem in one dimension smaller, the solution, whether it exists or
not, and whether or not multiple solutions exist, is complete.

\section{After-thought}
This section can be safely skipped by anyone whose interest is confined to
the real numbers, the complex numbers, the rational numbers, or even the quaternions.

The method works provided the scalars are taken from any subfield of the
quaternions which is closed under the * operation. More precisely, we really
need \begin{math}\alpha^* = \alpha\end{math} if and only if \begin{math}\alpha\end{math} is real, that \begin{math}\alpha^*\alpha >= 0\end{math}, and that
\begin{math}\alpha^*\alpha = 0\end{math} only if \begin{math}\alpha=0\end{math}.

But the method works, at least in some cases, even if the field is not *-closed.We can take the *-closure of any subfield of the quaternions, by intersecting all *-closed subfields containing it. Since the quaternions are *-closed, that *-closure is a *-closed subfield of the quaternions. We can then find all solutions composed of vectors having entries in the closure; it remains to determine
which of those solutions, if any, lie in the original field. In the case where
\begin{math}A\end{math} is positive, the solution is unique, and a solution whose entries lie in the original field does exist, so must coincide with the solution obtained by this method, which then must necessarily also have entries lying in the original field.

\section{Implementation Notes}
At each level of recursion, we only need to retain
\begin{math}{}a\end{math},
\begin{math}{}\alpha\end{math}
and
\begin{math}{}\beta\end{math}, allowing us to overwrite
\begin{math}{}A^{'}\end{math}
with \begin{math}{}(A^{'} - a\alpha^{-1}a^*)\end{math}
and
\begin{math}{}b^{'}\end{math} with
\begin{math}{}b^{'}-a\alpha^{-1}\beta\end{math}
.

We conclude with a C-language implementation of a solver for the positive semi-definite case, where
\begin{math}{}x^*Ax >=0 \end{math} for all \begin{math}{}x\end{math}.

\begin{verbatim}
$ cat positive_solver.h
extern
double * positive_solver(double ** A, double * b, int n);

$ cat positive_solver.c
/*
 * Andrew Winkler

This code solves the equation Ax=b, where A is a positive
semi-definite matrix, so that x^t A x >= 0 for all x,
provided a solution exists. If A is positive definite,
it will be an isomorphism, but in general conditions
must be imposed on b, for it to lie in the range of A.

It has the virtue of dramatic simplicity - there's no need
to explicitly construct the Cholesky decomposition, no need
to do the explicit back-substitutions.  Yet it's essentially
equivalent to that more labored approach, so its
performance/stability/memory, etc. should be at least as good.

There are two kinds of checks, both of which are disabled.
This means that a solution will be generated; it is up to
the caller to verify that the equation Ax=b is satisfied to
the desired precision. If it does not, then no solution exists.

The first kind of check is that Avec is 0 whenever A[0][0]
is 0, which will always be true if A is positive semi-definite,
but which could become false because of any small perturbation
caused by roundoff error upstream in the computation of A.
The second kind of check is that b[0] is 0 whenever A[0][0] is 0,
which is necessary for b to lie in the range of A. The checks are
disabled because the code does not deal with real numbers,
but rather floating point numbers; it's up to the user to decide
what is "close enough" to zero.

*/

#include <stdlib.h>
#include <stdio.h>
#include "positive_solver.h"

void _consistency_checker(double * v, int n) {
    for (int i=0; i<n; i++) {
        if ( v[i] != 0.0 ) exit(-1);
    }
}

void _positive_solver(
    double ** A,
    double * b,
    double * x,
    int n
    )
{
  if (n < 1) exit(-1);

  if (n == 1) {
    if (A[0][0] == 0.0) {
        /* _consistency_checker(b, 1); */
        x[0] = 0.0;
        return;
    }
    x[0] = b[0] / A[0][0];
    return;
  }

  double * bvec = b + 1;
  double * Avec = A[0] + 1;
  double ** Asub = A + 1;
  double * xvec = x + 1;

  int m = n -1;

  if (A[0][0] == 0.0) {
      /*
      _consistency_checker(b, 1);
      _consistency_checker(Avec, m);
      */
  } else {
      for(int j=0; j < m; j++){
        bvec[j] -= Avec[j] * b[0] / A[0][0];
        for(int i=0; i < m - j; i++)
          Asub[i][j] -= Avec[i] * Avec[i+j] / A[0][0];
      }
  }

  _positive_solver(Asub, bvec, xvec, m);

  if (A[0][0] == 0.0) {
      x[0] = 0.0;
      return;
  }

  double p = 0; for(int k=0; k<m; k++) p += Avec[k] * xvec[k];

  x[0] = (b[0] - p) / A[0][0];

  return;
}

#include <malloc.h>
double * positive_solver(
    double ** A,
    double * b,
    int n
    )
{
  double * x = (double *) malloc(n * sizeof(double));
  _positive_solver(A, b, x, n);
  return x;
}

\end{verbatim}
\end{document}


\documentclass{amsart}

\usepackage{psfrag}
\usepackage{graphicx}
\usepackage{url}
\usepackage{amsmath,amsfonts}
%\usepackage{fullpage}

\newcommand\br{{\mathbf r}}
\newcommand\w{{\mathbf w}}
\newcommand\bL{{\mathbf L}}
\newcommand\boldf{{\mathbf f}}
\newcommand\bz{{\mathbf z}}
\newcommand\bnabla{{\boldsymbol \nabla}}
\newcommand\e{{\mathbf e}}
\newcommand\q{{\mathbf q}}

\begin{document}

\section{Differential equations on the unit sphere}

The theoretical underpinnings of this package are described in \cite{montgomerysmith:10}.

The objective of the software is to provide a systematic spherical harmonic method to numerically solve partial differential equations on the two
dimensional sphere, $S = \{\br = (x,y,z):|\br| = x^2+y^2+z^2 = 1\}$, or in spherical coordinates, $\br = (x,y,z) = (\sin\theta
\cos\phi,\sin\theta\sin\phi,\cos\theta)$, where $0\le\theta\le\pi$, $0\le\phi\le2\pi$.  The partial differential equations considered here are of the
form
\begin{equation}
\label{pde}
\frac\partial{\partial t}\psi = F(\br,\bnabla) \psi,
\end{equation}
where $\psi$ is a function (for example, the probability distribution function) defined on the sphere. Here $\bnabla$ is the gradient operator
restricted to the sphere that is defined as
\begin{equation}
\begin{split}
\bnabla &= (\nabla_x,\nabla_y,\nabla_z) \\
&= \left(\cos\theta\cos\phi\frac{\partial}{\partial\theta} - \csc\theta\sin\phi \frac{\partial}{\partial\phi},\cos\theta\sin\phi\frac{\partial}{\partial\theta} + \csc\theta\cos\phi \frac{\partial}{\partial\phi},-\sin\theta\frac{\partial}{\partial\theta} \right) \\
&= \left((1-x^2)\frac\partial{\partial x}-xy\frac\partial{\partial y}-xz\frac\partial{\partial z},
         -xy\frac\partial{\partial x}+(1-y^2)\frac\partial{\partial y}-yz\frac\partial{\partial z},\right.\\
&\qquad\qquad\left.
         -xz\frac\partial{\partial x}-yz\frac\partial{\partial y}+(1-z^2)\frac\partial{\partial
         z}\right) .
\end{split}
\end{equation}
In the above, $F$ is a polynomial in six variables, with the proviso that it matters in which order the terms of each monomial part are written. We
believe that this includes many of the published, if not all, of the partial differential equations that describe the evolution of
the orientation distribution function for short fiber suspensions.

The spherical harmonic approach converts equation~\eqref{pde} to a
system of ordinary differential equations written as
\begin{equation}
\label{spherical-pde}
\frac\partial{\partial t} \hat\psi_l^m = \sum_{l'=0}^\infty \sum_{m'=-l'}^{l'} c_{l,l'}^{m,m'} \hat\psi_{l'}^{m'} ,
\end{equation}
where $\hat\psi_l^m$ ($l \ge 0$, $|m| \le l$) are the spherical harmonic coefficients of $\psi$, defined in
equation~\eqref{spherical-harmonic-coefficient} below, and $c_{l,l'}^{m,m'}$ are the coefficients defined through equation~\eqref{decompose F}.

The software provides a systematic algorithm for calculating the coefficients $c_{l,l'}^{m,m'}$ appearing in equation~\eqref{spherical-pde} (and also in equation~\eqref{decompose F} below). This algorithm makes it possible to readily solve all differential
equations on the unit sphere of the form shown in equation~\eqref{pde}.

\section{Description of the model equations}

To illustrate the software, we consider variations of Jeffery's equation \cite{jeffery:23} which describes the motion of fibers in a
moving fluid with vorticity $\w$ and rate of deformation tensor $\Gamma$. Jeffery's equation is
often written in terms of the fiber orientation distribution function $\psi$ and the fiber aspect ratio parameter $-1 \le \lambda\le1$ as
\begin{equation}
\label{jeffery's}
\frac\partial{\partial t}\psi = J \psi := -
\tfrac12\bnabla\cdot(\w \times \br \psi + \lambda(\Gamma\cdot\br -
\Gamma:\br\br\br) \psi) ,
\end{equation}
where we note that the right hand side is in the form of equation~\eqref{pde}. A variation of equation~\eqref{jeffery's} is Jeffery's equation with
rotary diffusion as expressed by Bird \cite{bird:87b} as
\begin{equation}
\label{jeffery's-folgar-tucker}
\frac\partial{\partial t}\psi = J \psi + \bnabla\cdot\bnabla(D_r
\psi) ,
\end{equation}
where $D_r$ captures the effect of fiber interaction and depends upon the flow kinetics. Folgar and Tucker \cite{folgar:84} selected $D_r = C_I \dot{\gamma}$ where $\dot{\gamma} =
\left(\frac12\Gamma:\Gamma\right)^{1/2}$ and $C_I$ is a constant that depends upon the volume fraction and aspect ratio of the
fibers.

Another example is anisotropic diffusion such as that proposed by Koch \cite{koch:95} in the
differential equation
\begin{equation}
\label{koch} \frac\partial{\partial t}\psi = J \psi +
\bnabla\cdot(I-\br\br)\cdot D_r\cdot\bnabla\psi = J \psi +
(\bnabla-2\br)\cdot D_r\cdot\bnabla\psi
\end{equation}
where the anisotropic diffusion matrix $D_r$ is defined in terms of the model parameters $C_1$ and $C_2$ (see Koch \cite{koch:95} for more detail) as
\begin{equation}
\label{kochDr}
D_r = C_1 {\dot\gamma}^{-1} (\Gamma:\mathbb A:\Gamma)I + C_2
{\dot\gamma}^{-1} \Gamma:\mathcal A:\Gamma .
\end{equation}
In the above, the $(I-\br\br)$ term serves to project vectors onto the surface of the sphere. We note that this term is not explicitly included in
Koch's formula in \cite{koch:95}, however it's existence is implied in the paragraph following the original introduction.

The anisotropic diffusion matrix $D_r$ defined in equation~\eqref{kochDr} is written in terms of the 2nd, 4th and 6th order moment tensors of $\psi$
which are, respectively, defined as
\begin{equation}
\label{momenttensors}
 A := \int_S \psi \br\br \, d\br ,\qquad
\mathbb A := \int_S \psi \br\br\br\br \, d\br ,\qquad \text{and}
\qquad \mathcal A := \int_S \psi \br\br\br\br\br\br \, d\br .
\end{equation}
In these integrals, we adopt the common definition for the integral of a function $F(\br)$ over the surface of the unit sphere as
\begin{equation}
\int_S F(\br) \, dr := \int_{\theta=0}^\pi \int_{\phi=0}^{2\pi}
F(\br) \sin\theta\,d\phi\,d\theta ,
\end{equation}
and we note here that the moment tensors $A$, $\mathbb A$, and $\mathcal{A}$ can be expressed in terms of spherical harmonics as shown below in
equation~\eqref{moment tensor}.

Care must be exercised in not confusing the differential geometry on the surface of the sphere $S$ with the differential geometry
in the three dimensional space $\mathbb R^3$ it is embedded in.  Thus, for example, integration by parts
is the slightly unexpected form
\begin{equation}
\label{by-parts}
\int_S \boldf \cdot \bnabla g\, d\br  = - \int_S (\bnabla\cdot (I-\br\br)\cdot\boldf) g\, d\br = \int_S ((2\br - \bnabla)\cdot\boldf)g \, d\br ,
\end{equation}
which reduces to the usual integration by parts when $\boldf$ is
tangential to the surface of the sphere. We also recall the so
called angular momentum operator
\begin{equation}
\label{ang-momentum-ops}
\bL = -i\br\times\bnabla = (L_x,L_y,L_z) =
-i\left(y\frac\partial{\partial z} - z\frac\partial{\partial y} ,
z\frac\partial{\partial x} - x\frac\partial{\partial z} ,
x\frac\partial{\partial y} - y\frac\partial{\partial x} \right) ,
\end{equation}
where, as usual,  $i$ denotes the complex number satisfying $i^2=-1$.  Integration by parts for the angular momentum operator is more straightforward, that is,
\begin{equation}
\label{by-parts-l}
\int_S \boldf \cdot \bL g\, d\br  = -\int_S (\bL\cdot\boldf) g\, d\br .
\end{equation}
Note that $\bL\times\bL = \bnabla\times\bnabla = i\bL$, and in particular $\nabla_x$, $\nabla_y$ and $\nabla_z$
do not commute with each other. We also use the formulae $\br\cdot\bnabla f=\br\cdot\bL f=\bL\cdot(\br f) = 0$ and $\bnabla\cdot(\br f) = 2f$.

\section{Spherical harmonics solutions}

Any square integrable function $\psi$ defined on the unit sphere may be written as a Fourier series like representation in terms of the spherical
harmonics $Y_l^m(\theta,\phi)$ for $l \ge 0$ and $|m| \le l$ as
\begin{equation}
\label{spherical-series}
\psi = \sum_{l=0}^\infty \sum_{m=-l}^l
\hat\psi_l^m Y_l^m ,
\end{equation}
where $Y_l^m$ are the spherical harmonic functions as defined in \cite{weisstein:06a}.
The coefficients $\hat\psi_l^m$ in equation~\eqref{spherical-series}
are evaluated from
\begin{equation}
\label{spherical-harmonic-coefficient}
\hat\psi_l^m = \int_S \psi
\bar Y_l^m d\br .
\end{equation}

To obtain the system of ordinary differential equations~\eqref{spherical-pde}, we integrate equation~\eqref{pde} against $\bar Y_l^m$ over the unit
sphere to obtain its adjoint or weak form as
\begin{equation}
\label{weakform}
\frac\partial{\partial t} \int_S \psi \bar Y_l^m
\,d\br = \int_S \psi F^*(\br,\bnabla) \bar Y_l^m \,d\br ,
\end{equation}
where the right hand side follows from integration by parts and the term $F^*$ denotes any polynomial $F$ in which each monomial term is written in
reverse order with the substitution of $2\br - \bnabla$ for $\bnabla$. Therefore, applying equation~\eqref{weakform}, Jeffery's
equation~\eqref{jeffery's} is defined by
\begin{equation}
J^* = \tfrac12(\w\cdot(\br\times\bnabla) + \lambda\br\cdot\Gamma\cdot\bnabla),
\end{equation}
Similarly, the two extensions of Jeffery's equation that include diffusion appearing in equations~\eqref{jeffery's-folgar-tucker} and~\eqref{koch}
(expressed as $\frac\partial{\partial t}\psi = F \psi$), respectively become
\begin{equation}
\label{FStarFT}
F^* = J^* + D_r \bnabla\cdot\bnabla ,
\end{equation}
and
\begin{equation}
F^* = J^* + (\bnabla-2\br)\cdot D_r\cdot\bnabla .
\end{equation}
In all cases, we can decompose
\begin{equation}
\label{decompose F}
F^*(\br,\bnabla) \bar Y_l^m = \sum_{l'=0}^\infty \sum_{m'=-l'}^{l'} c_{l,l'}^{m,m'} \bar Y_{l'}^{m'} ,
\end{equation}
and therefore obtain the spherical harmonic representation shown in equation~\eqref{spherical-pde} above.

\section{The ``spherical'' program algorithm}

An automated algorithm is implemented to compute the coefficients $c_{l,l'}^{m,m'}$ in equation~\eqref{decompose F} for
any differential equation satisfying the criteria given above (and, therefore, the $\mathcal C_{l,m}$ in equation~\eqref{moment tensor}). The
algorithm is to recursively apply the replacement rules described in equation~\eqref{rules} until no further substitutions can be made. Here ``op''
denotes any of $z$, $L_z$, $L_+$, or $L_-$ operations, and $\mathcal Y$ denotes any linear combination of the $\bar Y_l^m$'s.
\begin{equation}
\label{rules}
\begin{split}
\text{op}(\mathcal Y \pm c \bar Y_l^m) &\to \text{op}(\mathcal Y) \pm c \, \text{op}(\bar Y_l^m), \\
x \mathcal Y &\to  z i L_y(\mathcal Y) - i L_y(z\mathcal Y), \\
y \mathcal Y &\to  i L_x(z\mathcal Y) - z i L_x(\mathcal Y), \\
z\bar Y_l^m &\to \sqrt{\frac{(l+m)(l-m)}{(2l-1)(2l+1)}} \bar Y_{l-1}^m + \sqrt{\frac{(l+m+1)(l-m+1)}{(2l+1)(2l+3)}} \bar Y_{l+1}^m ,\\
\nabla_x \mathcal Y &\to z i L_y(\mathcal Y) - y i L_z(\mathcal Y),  \\
\nabla_y \mathcal Y &\to x i L_z(\mathcal Y) - z i L_x(\mathcal Y),  \\
\nabla_z \mathcal Y &\to y i L_x(\mathcal Y) - x i L_y(\mathcal Y),  \\
L_x (\mathcal Y) &\to \tfrac 12 \left(L_+ (\mathcal Y) + L_- (\mathcal Y)\right), \\
L_y (\mathcal Y) &\to -\tfrac i2 \left(L_+ (\mathcal Y) - L_- (\mathcal Y)\right), \\
L_z \bar Y_l^m &\to -m\bar Y_l^m ,\\
L_+ \bar Y_l^m &\to - \sqrt{(l+m)(l-m+1)}\bar Y_l^{m-1}, \\
L_- \bar Y_l^m &\to - \sqrt{(l-m)(l+m+1)}\bar Y_l^{m+1}
\end{split}
\end{equation}
It is important to note that $x \bar Y_l^m$, $y \bar Y_l^m$, $z \bar Y_l^m$, $\nabla_x\bar Y_l^m$, $\nabla_y \bar Y_l^m$, and $\nabla_z \bar Y_l^m$
involve $\bar Y_{l'}^{m'}$ for $l'$ and $m'$ that differ from $l$ and $m$, respectively, by at most one.  The script described in the appendix uses
the recursive algorithm in equation~\eqref{rules} to create threaded functions in the programming language
C which in turn are used in an iterative procedure for computing the solution to the differential equation in equation~\eqref{pde}.

Moment tensors may be computed using this algorithm as well. For example, to compute the 6th order moment tensor $\mathcal A$ in
equation~\eqref{momenttensors} we simply expand
\begin{equation}
\label{expand Y_0^0} \br\br\br\br\br\br \bar Y_0^0 = \sum_{l=0}^6
\sum_{m=-l}^l \mathcal C_{l,m} \bar Y_l^m,
\end{equation}
where $\mathcal C_{l,m}$ is the tensor of rank six composed of coefficients calculated by applying $x$, $y$ and $z$ to $\bar Y_0^0$ six times using the algorithm. It follows that since $\bar Y_0^0 = 1/\sqrt{4\pi}$, the 6th order orientation tensor
becomes
\begin{equation}
\label{moment tensor} \mathcal A = \sqrt{4\pi} \int_S \psi
\br\br\br\br\br\br \bar Y_0^0 \, d\br = \sqrt{4\pi} \sum_{l=0}^6
\sum_{m=-l}^l \mathcal C_{l,m} \hat\psi_l^m ,
\end{equation}
where the orthogonality of the spherical harmonics are used to simplify the final result.

\section{The ``spherical'' software package}

The program developed to implement our spherical harmonics
simulation procedure described above which computes the coefficients
in equation~\eqref{spherical-pde}, and then convert these terms into
a C program, may be found at
\url{https://github.com/stephenmontgomerysmith/spherical}. The
program is written in \emph{perl}, and makes use of either the
commercial computer algebra system \emph{Mathematica} by Wolfram
Research, or the open-source computer algebra system \emph{Maxima}.
The scripts are designed to work in a Unix like environment.

The program that evaluates spherical harmonics coefficients is a
\emph{perl} script which applies the rules in~\eqref{rules}. It is
invoked with the command sequence ``{\tt perl expand-method.pl}''.
The user may type in an expression involving {\tt x}, {\tt y}, {\tt
z}, {\tt dx}, {\tt dy}, {\tt dz}, {\tt lx}, {\tt ly}, or {\tt lz}
representing ``multiply by $x$,'' ``multiply by $y$,'' ``multiply by
$z$,'' $\nabla_x$, $\nabla_y$, $\nabla_z$, $L_x$, $L_y$, or $L_z$
respectively, or any number, including {\tt I} for $i$, and any
appropriate combination of parentheses, addition and subtraction,
and ``{\tt *}'' which represents composition of the operators.  It
outputs a sequence of lines, which when added together, represents
the effect of the expression upon $\bar Y_l^m$.  Each line has three
entries separated by semicolons, $l'$;$m'$;$c$, and represents
$c\bar Y_{l+l'}^{m+m'}$.  The third entry is written in the
programming language C, using complex numbers as described in
\cite{c:99}.  So, for example, the input
\begin{verbatim}
%perl expand-method.pl
> -2*I*(dy-y)
\end{verbatim}
yields the output
\begin{verbatim}
-1;-1;((l*sqrt(-1+l+m)*sqrt(l+m))/(sqrt(-1+2*l)*sqrt(1+2*l)))+(0)*I
-1;1;((l*sqrt(-1+l-m)*sqrt(l-m))/(sqrt(-1+2*l)*sqrt(1+2*l)))+(0)*I
1;-1;(((1+l)*sqrt(1+l-m)*sqrt(2+l-m))/(sqrt(1+2*l)*sqrt(3+2*l)))+(0)*I
1;1;(((1+l)*sqrt(1+l+m)*sqrt(2+l+m))/(sqrt(1+2*l)*sqrt(3+2*l)))+(0)*I
\end{verbatim}
This result reflects the equation
\begin{equation*}
\begin{split}
-2i(\nabla_y - y)\bar Y_l^m = &
\tfrac{l\sqrt{(l+m-1)(l+m)}}{\sqrt{(2l-1)(2l+1)}} \bar Y_{l-1}^{m-1}
+ \tfrac{l\sqrt{(l-m-1)(l-m)}}{\sqrt{(2l-1)(2l+1)}} \bar Y_{l-1}^{m+1} \\
& + \tfrac{(l+1)\sqrt{(l-m+1)(l-m+2)}}{\sqrt{(2l+1)(2l+3)}} \bar Y_{l+1}^{m-1}
+ \tfrac{(l+1)\sqrt{(l+m+1)(l+m+2)}}{\sqrt{(2l+1)(2l+3)}} \bar Y_{l+1}^{m+1} .
\end{split}
\end{equation*}

The program ``{\tt perl expand-method.pl}'' is called by another
script invoked as ``{\tt perl expand-iterate.pl <filename>}'' which
converts a configuration file into a C program to numerically solve
the spherical harmonics differential equation. For example,
Jeffery's equation (written here in weak form) with Koch diffusion
can be written as below
\begin{verbatim}
void compute_psidot(complex* psidot, complex* psi,
                    double lambda, double w[3], double g[3][3]) {
  double a4[3][3][3][3], a6[3][3][3][3][3][3], Dr[3][3];

  tensor4(psi,a4); tensor6(psi,a6);
  ... compute Dr from g, a4 and a6 ...

  @spherical_iterate {
    psidot[@index] =
      /* Jeffery's equation */
        w[0]*@method(psi,0.5*y*dz-0.5*z*dy)
      + w[1]*@method(psi,0.5*z*dx-0.5*x*dz)
      + w[2]*@method(psi,0.5*x*dy-0.5*y*dx)
      + lambda*g[0][0]*@method(psi,0.5*x*dx)
      + lambda*g[0][1]*@method(psi,0.5*x*dy+0.5*y*dx)
      + lambda*g[0][2]*@method(psi,0.5*x*dz+0.5*z*dx)
      + lambda*g[1][1]*@method(psi,0.5*y*dy)
      + lambda*g[1][2]*@method(psi,0.5*y*dz+0.5*z*dy)
      + lambda*g[2][2]*@method(psi,0.5*z*dz)
      /* Koch diffusion */
      + Dr[0][0]*@method(psi,(dx-2*x)*dx)
      + Dr[0][1]*@method(psi,(dx-2*x)*dy+(dy-2*y)*dx)
      + Dr[0][2]*@method(psi,(dx-2*x)*dz+(dz-2*z)*dx)
      + Dr[1][1]*@method(psi,(dy-2*y)*dy)
      + Dr[1][2]*@method(psi,(dy-2*y)*dz+(dz-2*z)*dy)
      + Dr[2][2]*@method(psi,(dz-2*z)*dz);
  }
}
\end{verbatim}

To calculate the coefficients for the moment tensors, as in
equation~\eqref{expand Y_0^0}, if the environmental variable {\tt
TENSOR} is set to a non-zero value, then {\tt expand-method.pl}
applies the input expression to $\bar Y_0^0$, and outputs the
answers in floating point format.  So, for example the input
\begin{verbatim}
%env TENSOR=1 perl expand-method.pl
> x*y*z*z
\end{verbatim}
yields
\begin{verbatim}
2;-2;(0)+(-0.026082026547865053022)*I
2;2;(0)+(0.026082026547865053022)*I
4;-2;(0)+(-0.030116930096841707924)*I
4;2;(0)+(0.030116930096841707924)*I
\end{verbatim}
This results reflects the 1233 component of 4th order moment tensor
\begin{equation*}
\mathbb A_{1233} =
i\sqrt{4\pi}\left(\tfrac{\sqrt{30}}{210}(\hat\psi_2^2-\hat\psi_2^{-2})
+\tfrac{\sqrt{10}}{105}(\hat\psi_4^2-\hat\psi_4^{-2})\right).
\end{equation*}
This is used by ``{\tt perl make-tensor.pl $n$}'' and ``{\tt perl
make-reverse-tensor.pl $n$}'', which create C programs to convert,
respectively, spherical harmonic coefficients to moment tensors of
rank $n$, and vice-versa.

\bibliographystyle{plain}
\bibliography{info_08}

\end{document}


We wish to predict useful dependencies for a proof goal, or conjecture.
For this we define rankings, which are generated by predictors:

\begin{definition}\defgls{rankings}
  A ranking $r \in \rankings$ is a partial function which yields the likelyhood a
  dependency $\phi \in \depset$ is useful in the current context.

  $$
    r : \depset \rightharpoonup \mathbb{R}
  $$
\end{definition}

\begin{definition}\defgls{predictors}
  A predictor $P \in \predictors$ is a function which given an unproven conjecture $c \in \types$
  computes the likelyhood a dependency $\phi \in \depset$ is useful in proof of conjecture $c$.
  In other words, for a conjecture $c \in \types$ it yields a ranking $r \in \rankings$.
  $$
    P : \types \rightarrow \rankings
  $$
\end{definition}

A weak total ordered subset of definitions useful for conjecture $c$ is formed by sorting on the resulting value of type $\mathbb{R}$.
In the next few chapters we define a few of these predictors.

\subsubsection{\knn}
\input{document-approach-predictors-knn}

\subsubsection{\knnadaptive}
\input{document-approach-predictors-knnadaptive}

\subsubsection{\nb}
\input{document-approach-predictors-nb}

\subsubsection{Adarank}
\input{document-approach-predictors-adarank}

\subsubsection{Ensembles}
\input{document-approach-predictors-ensemble}


\section{Methodology for Library Evaluation}

The three dimensions of library evaluation are (i)efficiency, (ii)scalability, and (iii)reliability.    The overall evaluation of the library is the geometric mean of numerical measurements of each dimension.  Standard numerical measurements  for evaluating efficiency has been found in literature.  A similar approach may apply to the evaluation of scalability.  A numerical measure of reliability, unfortunately, requires further study.  The first section of this part will review methodology of hardware efficiency evaluation, which I believe applies to software efficiency evaluation as well.  In the rest three sections of this part, approaches to evaluate each dimension will be examined respectively.

\subsection{Measuring Hardware Efficiency}
In practice, hardware efficiency is measured by its performance regarding to a benchmark suite, which contains a set of test programs.  Performance is defined as the reciprocal of mean execution time of all tests\cite{HePa06}.  This project will use geometric mean instead of weighted arithmetic mean for three reasons.

Firstly and most importantly, as proved by Fleming and Wallace \cite{Fleming}, geometric mean is the only correct average of normalised measurements.  As users often compare the performance of one system to another, performance is usually normalised to a reference system.

\begin{comment}
 Precisely, let the benchmark suite has $n$ tests indexed from $1$ to $n$, performance of system A is calculated as:

 $performance_A$
= $\frac{time_{ref}}{time_{A}}$
= $\frac{\sqrt[n]{\prod^{n}_{i=1} time_{ref_i}}}{\sqrt[n]{\prod^{n}_{i=1} time_{A_i}}}$
= $\sqrt[n]{\prod^{n}_{i=1}{\frac{time_{ref_i}}{time_{A_i}}}}$

\hspace{0.8 cm}where $time_{S_i}$ is the execution time of program $i$ on system $S$,

\hspace{1.0 cm} and $time_{S}$ is the time measurement of system $S$.
\end{comment}

Secondly, the order of calculating means and normalisation is flexible in practice.  Therefore, the suite developer will only implement a generic tool for measuring unnormalised execution time. The work of comparing performance of two interested platforms will be left to users.  Besides, users are free to extend the benchmark suite.  When new tests are added to the benchmark suite, performance measure for old examples becomes a valid partial measure.

Finally, statistic tools could be employed to verify the soundness of chosen suite.  For example, the geometric standard deviations is an indicator to assess the variability of the chosen suite.

\subsection{Measuring Library Efficiency}
The Methodology of measuring hardware efficiency also applies to measuring library efficiency, where both the library implementation and the running platform may differ.

To test the efficiency of different distributed programming libraries, a set of benchmarking examples need to be identified.  To avoid unintentional optimisation in the TAkka implementation, benchmarks are scrupulously selected from existing applications written in other OTP-like libraries and modifications are made at the minimal level.

Theoretically, any program that involves a time measurement could be a candidate efficiency test.  Such examples could be easily found in both the correctness test suite and the scalability test suite.  However, we would like to exclude examples whose results significantly determine the overall measure.  To this end, statistic methods such as principal component analysis will be employed.

\subsection{Measuring Scalability}

To assess the scalability of different OTP-like libraries, suitable examples from the BenchErl suite\cite{RELEASE} have been reimplemented using Akka and TAkka.  A summary of selected examples will be given at \S\ref{scalability}.

Similar to efficiency evaluation, geometric mean of scalability measures of all examples could be the indicator of the library scalability.  Unfortunately, numerical evaluation for scalability is not found in the literature.  This project will use linear regression techniques to analysis the scalability of an OTP-like library in two aspects: (i) the degrees to which the performance speedup deviate from the ideal linear relationship regarding to the number of active CPU cores.  (ii) the slope of the linear equation, assuming that the relationship between performance speedup and the number of active CPU cores is linear.

Finally, scalability of OTP-like libraries will be tested on a variety of platforms.  The three candidate platforms are Beowulf cluster, Google App Engine, and Amazon EC2.

\subsection{Measuring Reliability}

Identify types of failures and their impact is a crucial part of reliability evaluation.  The next two months will be devoted to the study of possible failures that could be handled at the language level.  Methodology for numerically evaluating reliability requires more studies and careful verifications.


\section{Asymptotic Analysis for Closed Systems}
\label{sec:Asymptotic-Analysis-Closed-Systems}

\begin{theorem}[Asymptotic Bounds for Closed Systems]
	\label{thm:Asymptotic-Bounds-Closed-Systems}
	For any closed system with load $N$, we have that

	\begin{equation}
	\label{eqn:Asymptotic-Bounds-Closed-Systems}
	\begin{split}
	X & \leq \min \Big\{ \frac{N}{D + \expected{Z}} , \frac{1}{D_{max}} \Big\} \\
	\expected{R} & \geq \max\{D , N \cdot D_{max} - \expected{Z} \}
	\end{split}
	\end{equation}

	where the first term in each clause is an asymptote for small $N$, and the second one for large $N$.

	\begin{proof}
		See \cite{harchol2013performance} for a formal demonstration.
	\end{proof}
\end{theorem}

It is possible to get even tighter bounds by using the \textit{Balanced Bounds technique}, described in \cite{lazowska1984quantitative}.

The lower and the upper bound intercept in
\begin{equation}
\label{eqn:Asymptotic-Bounds-Closed-Systems-Interception}
N^{*} = \frac{D + \expected{Z}}{D_{max}}
\end{equation}
That is the point beyond which there must be some queueing in the system, because $\expected{R} > D$.

If $N > N^{*}$, performance can be improved ($X$ increased and $\expected{R}$ decreased) only decreasing $D_{max}$.

If $N >> N^{*}$, decreasing some $D_{i} < D_{max}$ have no effect.
If $N << N^{*}$, decreasing some $D_{i} < D_{max}$ have little effect.

In a batch system ($Z=0$), $N^{*}$ decreases, meaning that the domination of $D_{max}$ occurs with smaller load.

The bottleneck server is the server with demand $D_{max}$. It is the key limiting factor to performance improvement. Thus, the first step in improving performance is to identify the bottleneck server.

The asymptotic bounds hold for open systems only if $\lambda . X \rightarrow \frac{1}{D_{max}}$.


%\section{A result on expectation}
%\begin{lemma}   \label{MC}
%Let $\lambda\ge 0$ be a probability density function such that
%\begin{equation}
%  \label{density}
%\int_{\mathbb R^d}\lambda(\omega)dx =1.
%\end{equation}
%\begin{equation}
%  \label{E}
%\mathbb{E}g:=\int_{\mathbb{R}^d}
%  g(\omega)\lambda(\omega)d\omega
%\end{equation}
%and
%\begin{equation}
%  \label{En}
%\mathbb{\bar E}h:=\int_{\mathbb{R}^d\times \mathbb{R}^d\times\ldots\times \mathbb{R}^d}
%h(\omega_1,\omega_2,\ldots,\omega_n) \lambda(\omega_1) \lambda(\omega_n)\ldots \lambda(\omega_n)
%d\omega_1d\omega_2\ldots d\omega_n.
%\end{equation}
%
%Then for any $g\in L^\infty(\mathbb{R}^d)$, we have
%  \begin{equation}
%   \mathbb{\bar E}(\mathbb{E}g-\frac1n\sum_{i=1}^n
%    g(\omega_i))^2=\frac{1}{n}(\mathbb{E}(g-\mathbb{E}(g))^2)
%\le \frac{1}{n}\mathbb{E}(g^2)\le \frac{1}{n}\|g\|^2_{L^\infty},
%  \end{equation}
%  where
%\end{lemma}
%\begin{proof}
% \begin{equation}
%    \label{eqn}
%    \begin{aligned}
%      &\quad \mathbb{\bar E}\left(\mathbb{E} g-\frac1n\sum_{i=1}^ng(\omega_i)\right)^2 \\
%  &=\frac{1}{n^2} \mathbb{\bar E}\left(n\mathbb{E} g-\sum_{i=1}^ng(\omega_i)\right)^2 \\
%  &=\frac{1}{n^2} \mathbb{\bar E}\left(\sum_{i=1}^n(\mathbb{E} g-g(\omega_i))\right)^2 \\
%  &=\frac{1}{n^2} \mathbb{\bar E}\sum_{i,j=1}^n(\mathbb{E} g-g(\omega_i))(\mathbb{E} g-g(\omega_j))\\
%  &=\frac{1}{n^2} \mathbb{\bar E}\sum_{i=1}^n(\mathbb{E} g-g(\omega_i))^2\\
%&+\frac{1}{n^2} \mathbb{\bar E}\sum_{i\neq  j}^n((\mathbb{E}g)^2-\mathbb{E}(g)(g(\omega_i)+
%g(\omega_j))+g(\omega_i)g(\omega_j)))
%    \end{aligned}
%  \end{equation}
%We note that
%$$
%\mathbb{\bar E} g(\omega_i)=\mathbb{\bar E} g(\omega_j) =\mathbb E(g)
%$$
%and, for $i\neq j$,
%$$
%\mathbb{\bar E} ( g(\omega_i)g(\omega_j)) =\mathbb{\bar E} (
%g(\omega_i))\mathbb{\bar E}(g(\omega_j))
%=[\mathbb E(g)]^2.
%$$
%Thus
% \begin{equation}
%\mathbb{\bar E}\sum_{i\neq j}^n((\mathbb{E}g)^2-\mathbb
%  E(g)(g(\omega_i)+ g(\omega_j))+(g(\omega_i)g(\omega_j)))=0.
%  \end{equation}
%Furthermore
%$$
%\mathbb{\bar E}(\mathbb{E} g-g(\omega_i))^2
%=\mathbb{\bar E}(\mathbb{E} g-g(\omega))^2
%=[\mathbb{E} g]^2-2[\mathbb E g(\omega)]^2 +\mathbb E(g^2)
%=\mathbb E(g^2) -[\mathbb E(g)]^2
%\le\mathbb E(g^2)
%$$
%Consequently
% \begin{equation}
%    \label{eqn}
%    \begin{aligned}
%      &\quad \mathbb{\bar E}\left(\mathbb{E} g-
%      \frac1n\sum_{i=1}^ng(\omega_i)\right)^2 \\
%      &=\frac{1}{n^2} \mathbb{\bar E}\left(n\mathbb{E} g-
%      \sum_{i=1}^ng(\omega_i)\right)^2 \\
%      &=\frac{1}{n^2} \mathbb{\bar E}\left(\sum_{i=1}^n(\mathbb{E}
%      g-g(\omega_i))\right)^2 \\
%      &=\frac{1}{n^2} \sum_{i=1}^n(\mathbb{\bar E}(\mathbb{E}      g-g(\omega_i))^2) \\
%      &=\frac{1}{n}(\mathbb{E}(g-\mathbb{E}(g))^2) \\
%      &=\frac{1}{n}(\mathbb{E}(g^2)-(\mathbb{E}(g))^2) \\
%      &\le\frac1n \mathbb E(g^2)\\
%      &\le\frac1n\|g\|^2_{L^\infty}
%    \end{aligned}
%  \end{equation}
%\end{proof}
%
%The above result be extended to the following general case easily.
%\begin{lemma} \label{MC1} Given $1\le i\le n$, let $\Omega $ be a
%  probability space with a probability measure $\mu$ with density
%  function $\lambda\ge 0$ ($1\le i\le n$), namely
%$$
%\mu(G)=\int_G \lambda(\omega)d\mu, \quad
%\mu(\Omega)=1, \quad G\subset \Omega.
%$$
%Define
%\begin{equation}
%\label{Ei}
%\mathbb{E}g:=\int_{\Omega}  g(\omega)\lambda(\omega)d\mu
%\end{equation}
%and
%\begin{equation}
%  \label{En}
%\mathbb{\bar E}h:=\int_{\Omega\times\Omega\times\ldots\times\Omega}
%h(\omega_1,\omega_2,\ldots,\omega_n) \lambda(\omega_1) \lambda(\omega_n)\cdots \lambda(\omega_n)
%d\mu_1d\mu_2\ldots d\mu_n.
%\end{equation}
%Then for any $g\in L^\infty(\Omega)$, we have
%  \begin{equation}
%    \mathbb{\bar E}(\mathbb{E}g-\frac1n\sum_{i=1}^n
%    g(\omega_i))^2=\frac{1}{n}(\mathbb{E}(g-\mathbb{E}(g))^2)
%\le \frac{1}{n}\|g\|^2_{L^\infty}.
%  \end{equation}
%\end{lemma}
%\section{Fourier transformation and Barron spaces}
%Given a bounded domain $B\subset\mathbb R^d$, we consider the function
%$$
%f: B\mapsto \mathbb R
%$$
%Let
%$$
%f^e: \mathbb R^d\mapsto \mathbb R
%$$
%be any extension of $f$ so that
%$$
%f^e|_B=f(x), \quad x\in B.
%$$
%Most time, we will drop the superscript $``e"$ to still use $f$ to
%denote an extension of $f$.
%
%Consider the Fourier transform:
%\begin{equation}
%  \label{Fourier}
%  \hat f(\omega)=\frac{1}{(2\pi)^d}\int_{\mathbb{R}^d}e^{-i\omega\cdot x}f(x)dx
%  \quad \forall \omega \in \mathbb R^d,
%\end{equation}
%Using the Fourier inversion formula, we can have a Fourier
%representation of $f(x)$ as follows
%\begin{equation}
%  \label{eqn1}
%  f(x)=\int_{\mathbb{R}^d}e^{i\omega\cdot x}\hat{f}(\omega)d\omega
%  \quad \forall x \in B,
%\end{equation}
%From \eqref{eqn1}, we have
%\begin{equation}
%  \label{f0}
% f(x_B)=\int_{\mathbb{R}^d}e^{i\omega\cdot x_B}\hat{f}(\omega)d\omega.
%\end{equation}
%Let us write
%\begin{equation}
%  \label{theta-omega}
%  \hat{f}(\omega)=e^{i\theta(\omega)}|\hat{f}(\omega)|.
%\end{equation}
%Since $f(x)$ is real-valued, it follows from \eqref{eqn1} and
%\eqref{f0} that, for $x, x_B\in B$
%  \begin{equation}
%    \label{eqn4.3}
%    \begin{aligned}
%f(x)-f(x_B)
%&={\rm Re}\int_{\mathbb{R}^d}
%(e^{i\omega\cdot x}-e^{i\omega\cdot x_B})
%\hat{f}(\omega)d\omega \\
%&={\rm Re}\int_{\mathbb{R}^d}
%(e^{i\omega\cdot x}-e^{i\omega\cdot x_B})
% e^{i\theta
%    (\omega)}|\hat{f}(\omega)|d\omega \\
%    &=\int_{\mathbb{R}^d}(\cos(\omega\cdot
%    x+\theta(\omega))-\cos(\omega\cdot x_B+\theta(\omega)))|\hat{f}(\omega)|d\omega \\
%&=\int_{\mathbb{R}^d}(\cos(\omega\cdot(x-x_B)+\theta_B(\omega))-\cos(\theta_B(\omega)))|\hat{f}(\omega)|d\omega \\
%&=\int_{\mathbb{R}^d}g(x,\omega)|\hat{f}(\omega)|d\omega \\
%&=\|f\|_{B^m}\int_{\mathbb{R}^d}|\omega|_B^{-m}g(x,\omega)\lambda^{m}(\omega)d\omega
%%\\&=\int_{\mathbb{R}^d}\|f\|_{B^m}
%%(\cos(\omega\cdot x+\theta(\omega))-\cos(\omega\cdot x_B+\theta(\omega)))\lambda(\omega)d\omega \\
%%  &=\int_{\mathbb{R}^d} g(x,\omega)\lambda(\omega)d\omega,
%\end{aligned}
%  \end{equation}
%where
%$$
%\theta_B(\omega)=\omega\cdot x_B+\theta(\omega)
%$$
%and $g: B\times \mathbb{R}^d\rightarrow \mathbb{R}$
%is given by
%\begin{equation}\label{gz}
%  g(x,\omega):=
%\cos(\omega\cdot (x-x_B)+\theta_B(\omega))  -\cos(\theta_B(\omega)),
%\end{equation}
%
%\begin{equation}
%\label{lambda}
%\lambda^m(\omega)=\frac{|\omega|_B^m|\hat{f}(\omega)|}{\|f\|_B^m}
%\end{equation}
%is a  probability distribution density function.
%
%We define
%\begin{equation}
%\|f\|_{B^m}:=\int_{\mathbb R^d}|\omega|_B^m|\hat{f}(\omega)|d\omega,
%\end{equation}
%\begin{equation}
%\|f\|_{B^m}:=\inf_{f^e|_B=f}\int_{\mathbb R^d}|\omega|_B^m|\hat{f^e}(\omega)|d\omega,
%\end{equation}
%where
%$$
%|\omega|_B:=\sup\limits_{x\in B}|\omega\cdot(x-x_B)|
%$$
%We note that, if, for example,  $B=B_r=\{x:|x|\le r\}$, we then have $|\omega|_{B_r}
%=r\cdot |\omega|$ if $x_B=0$.  In general
%$$
%|\omega|_B\le {\rm diam}(B)|\omega|.
%$$
%
%We then obtain the following result.
%\begin{lemma}
%\label{lem2.2}
%\begin{equation}
%\frac{f(x)-f(x_B)}{\|f\|_{B^m}}
%=\int_{\mathbb R^d} \frac{g(x,\omega)}{|\omega|^m}\lambda(\omega)d\omega =\mathbb E_\omega(\frac{g(x, \omega)}{|\omega|_B^m})
%\end{equation}
%and
%$g$ is given by \eqref{gz} and satisfies
%\begin{equation}\label{g-bound1}
%|g(x,\omega)|\le  \min(2,|\omega|_B)\quad \forall x\in B,  \omega\in \mathbb
%R^d.
%\end{equation}
%\end{lemma}
%
%\begin{theorem}
%\label{thm:cos} For a probability measure $\mu$ on $B$ and every
%  function $f$ with $\|\hat f\|_{L^1}< \infty$, there exist
% $\omega_1,\ldots,\omega_n\in \mathbb R^d$ such that
%\begin{equation}
% \int_{B}\left|f(x)-f(x_B)-\frac{\|\hat f\|_{L^1}}{n}
%    \sum_{i=1}^ng(x,\omega_i)\right|^2\/d\mu(x) \le \frac{4\|\hat f\|^2_{L^1}}{n}.
%\end{equation}
%\end{theorem}
%\begin{theorem}
%  \label{thm2.1} For a probability measure $\mu$ on $B$ and every
%  function $f$ with $\|f\|_{B}< \infty$, there exist
%  $\omega_1,\ldots,\omega_n\in \mathbb R^d$ such that
%\begin{equation}
%\int_{B}\left|f(x)-f(x_B)-\frac{\|f\|_{B}}{n} \sum_{i=1}^n\frac{g(x,\omega_i)}{|\omega_i|}\right|^2\/d\mu(x) \le \frac{\|f\|^2_B}{n}.
%\end{equation}
%\end{theorem}
%\begin{proof}
%Denote
%$$
%\tilde f(x)=\frac{f(x)-f(x_B)}{\|f\|_{B^m}}
%$$
%By Lemma \ref{MC} and Lemma \ref{lem2.2}, we have for $m=0,1$,
%\begin{equation}
%    \label{eqn3.10}
%    \begin{aligned}
%      &\quad \mathbb{\bar E}\left(\tilde f(x)-\frac1n\sum_{i=1}^n\frac{g(x,\omega_i)}{|\omega|_B^m}\right)^2 \\
%      &=\mathbb{\bar E}(\mathbb{E}_\omega(\frac{g(x,\omega)}{|\omega|_B^m})-
%      \frac1n\sum_{i=1}^n \frac{g(x,\omega_i)}{|\omega|_B^m})^2 \\
%      &\le\frac{1}{n}\max_{x\in B, \omega\in \mathbb R^d}\frac{|g(x,\omega)|^2}{|\omega|_B^{2m}}\\
%      &\le \frac{4^{1-m}}{n}
%    \end{aligned}
%\end{equation}
%
%By the Fubini's theorem,
%  \begin{equation}
% \label{eqn3.11}
%  \begin{aligned}
% &\quad  \mathbb{\bar E}\int_B(\tilde f(x)-\frac1n\sum_{i=1}^n\frac{g(x,\omega_i)}{|\omega_i|_B^m})^2 \mu(dx)\\
% &=\int_B \mathbb{\bar E} (\tilde f(x)-\frac1n\sum_{i=1}^n\frac{g(x,\omega_i)}{|\omega_i|_B^m})^2 \mu(dx)\\
%    &\le \frac{4^{1-m}}{n}.
%  \end{aligned}
%  \end{equation}
%Therefore, there exists a sequence of $\{\omega_i\}_{i=1}^n$ where
%$\omega_i\in \mathbb{R}^d$, such that
%$$
%\int_B  (\tilde
%f(x)-\frac1n\sum_{i=1}^n\frac{g(x,\omega_i)}{|\omega_i|_B^m})^2
%\mu(dx)
%\le \frac{4^{1-m}}{n}.
%$$
%This complete the proof.
%\end{proof}
%Recalling \eqref{gz}, we have
%\begin{eqnarray*}
%f_n
%&\equiv& f(0)+\frac{\| f\|_B}{n} \sum_{i=1}^n\frac{g(\cdot,\omega_i)}{|\omega_i|_B}\\
%&=& f(0)+\frac1n \sum_{i=1}^n \frac{\|f\|_B}{|\omega_i|_B}(\cos(\omega_i\cdot x+\theta(\omega_i))  -\cos(\theta(\omega_i)))\\
%&=&
%\frac1n \sum_{i=1}^n \frac{\|f\|_B}{|\omega_i|_B}(\cos(\omega_i\cdot
%x+\theta(\omega_i))  +\left(
%f(0)-\frac1n \sum_{i=1}^n \frac{\|f\|_B}{|\omega_i|_B}\cos(\theta(\omega_i)) \right)\\
%&=&
%\sum_{i=1}^n a_i\cos(\omega_i\cdot x+b_i)  +c
%\end{eqnarray*}
%where
%$$
%a_i=\frac1n \frac{\|f\|_B}{|\omega_i|_B}, b_i=\theta(\omega_i), c=f(0)-\frac1n \sum_{i=1}^n \frac{\|f\|_B}{|\omega_i|_B}\cos(\theta(\omega_i)).
%$$
%
%\begin{theorem}
%  \begin{equation}
%\min_{a_i,b_i,c\in \mathbb R^1, \omega_i\in \mathbb R^d}
%\int_{B}|f(x)-\sum_{i=1}^n a_i\cos(\omega_i\cdot x+b_i)
%-c|^2\mu(dx)\le \frac{4\|\hat f\|^2_{L^1(\mathbb R^d)}}{n}.
%  \end{equation}
% \end{theorem}

\section{An improved analysis}
\subsection{Heaviside Function}
Define $g_i: [-1,1]\mapsto \mathbb R$ as
follows:
\begin{equation}
  \label{psi}
g_i(t)=\frac{1}{|\omega_i|_B}[\cos(|\omega_i|_Bt+\theta_B(\omega_i))  -\cos(\theta_B(\omega_i))],
\end{equation}
In view of \eqref{gz}, we have
\begin{equation}
  \label{gpsi}
g_i(s_i)=\frac{g(x,\omega_i)}{|\omega_i|_B}, \quad s_i=\omega_i^B\cdot(x-x_B),\quad \omega^B=\frac{\omega}{|\omega|_B}
\end{equation}
Now, we take an integer
\begin{equation}
  \label{k}
k\ge \sqrt{n}
\end{equation}
and consider a partition of $[-1,1]$ with the following grid points
$$
t_j=jh_k, j=-k:k
$$
with
$$
 h_k=\frac{1}{k}\le \frac{1}{\sqrt{n}}.
$$
We first take a piecewise constant interpolation for $g_i$ on $[0,1]$ to get
$$
g_{i,k}(t)=(\Pi_kg_i)(t)=\sum_{j=0}^{k-1}g_i(t_j) M_j(t),
$$
where
$$
M_j(t)=M_0(\frac{t-t_j}{h_k})
$$
and
\begin{equation}
  \label{cardinal}
M_0(x)=
\left\{
  \begin{array}{ll}
0 & x\le0 \\
1 & 0< x\le1    \\
0 & x > 1
  \end{array}
\right.
\end{equation}
We note that
$$
M_0(x)=H(x)-H(x-1)
$$
where $H$ is the Heaviside function
Thus
$$
M_0(\frac{t-t_{j}}{h_k})
=H(\frac{t-t_{j}}{h_k})-H(\frac{t-t_{j}}{h_k}-1)=H(\frac{t-t_{j}}{h_k})-H(\frac{t-t_{j+1}}{h_k})
\equiv H_{j}(t)-H_{j+1}(t).
$$
Thus, since $g_i(t_0)=0, H_k=0$, we have
\begin{equation}  \label{gi0}
g_{i,k}(t)=\sum_{j=0}^{k-1}g_i(t_j) M_j(t)
=\sum_{j=1}^{k-1}(g_i(t_j) - g_i(t_{j-1})) H_{j}(t), \quad t\in [0,1]
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%
Now we consider
\begin{equation}
h_i(t) = g_i(-t), \quad t\in [0,1].
\end{equation}
Similar to \eqref{gi0}, we have
$$
(\Pi_kh_i)(t)=\sum_{j=1}^{k-1}(h_i(t_j) - h_i(t_{j-1}))H_j(t)=
=\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(t)
$$
Namely
$$
(\Pi_k g_i)(-t)=\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(t), \quad t\in [0,1]
$$
or
\begin{equation}\label{gi1}
(\Pi_k g_i)(t)=\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(-t), \quad t\in [-1,0]
\end{equation}
By combining \eqref{gi0} and \eqref{gi1}, we get a piecewise constant
interpolation of $g_i$ on $[-1,1]$ as follows:
\begin{eqnarray}
g_{i,k}(t)&=&
\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(-t)+\sum_{j=1}^{k-1}(g_i(t_j) - g_i(t_{j-1})) H_{j}(t)\nonumber \\
&=&\sum_{j=1}^{k-1}[a_{ij}^-H_j(-t)+a_{ij}^+H_{j}(t)] \label{gih}
\quad t\in [-1,1]
\end{eqnarray}
where
$$
a_{ij}^{\pm}=g_i(\pm t_j) - g_i(\pm t_{j-1})
$$
It is easy to see that
\begin{equation}
|g_i(t)-g_{i,k}(t)|\le h_k, \quad t\in [-1,1].
\end{equation}

\begin{equation}
\|\frac1n \sum_{i=1}^n \frac{g(\cdot,\omega_i)}{|\omega_i|_B}-f^*\|_{L^2(\mu,B)}\le h_k
\end{equation}
where
\begin{equation}
  \label{fstar}
f^*(x)=
\frac1n\sum_{i=1}^ng_{i,k}(\omega_i^B\cdot (x-x_B)).
\end{equation}
By the approximation in last section, we have
\begin{equation}
  \|\tilde f-f^*\|_{L^2(\mu,B)}\le \frac{2}{\sqrt{n}}
\end{equation}
Let us rewrite
$$
f^*(x)
=\sum_{i=1}^n\sum_{j=1}^{k-1}[\gamma_{ij}^- f_{ij}^-+ \gamma_{ij}^+f_{ij}^+]
$$
where
$$
\gamma_{ij}^{\pm}=\frac{|a^\pm_{ij}|}{nd_i},
f^\pm_{i,j}=d_i\/{\rm sign}(a^\pm_{ij})H_j(\pm \omega_i^B\cdot (x-x_B))
$$
and
$$
d_i=\sum_{j=1}^{k-1}(|a^-_{ij}|+|a^+_{ij}|)\le 2
$$
By definition
\begin{equation}
  \label{gammaij}
\sum_{i=1}^n\sum_{j=1}^{k-1}[\gamma_{i,j}^-+\gamma_{i,j}^+]=1.
\end{equation}
With re-numeration as
$$
p_\ell=\gamma_{ij}^{\pm}, f_\ell = f_{ij}^{\pm}, 1\le \ell \le N=2n(k-1)
$$
We have
$$
f^*(x)=\sum_{\ell=1}^N p_\ell f_\ell
$$
Consider
$$
\mathcal N=\{1,2,\ldots, N\}
$$
and
$$
\bar f: \mathcal N\mapsto \mathbb R^1
$$
such that
$$
\bar  f(\ell)=f_{\ell}, \ell\in \mathcal N
$$
With the probability measure
$$
\mu(\mathcal M)=\sum_{m\in \mathcal M}p_m \quad \mathcal M\subset\mathcal N.
$$
By definition.
$$
\mathbb E(\bar  f) = f^*(x).
$$
By the basic result on expectation in Lemma \ref{MC1}, we have
$$
\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\left(f^*(x)-
{1\over n}\sum_{i=1}^n f_{\ell_i}\right)^2
=\mathbb E_{\mathcal N^n} \left(\mathbb E(\bar f)-{1\over
    n}\sum_{i=1}^n  \bar  f(\ell_i)\right)^2\\
\le\frac1n \|\bar  f\|_{\infty}^2 \le\frac4n.
$$
By taking the $L^2(\mu,B)$ on the above inequality, we get
$$
\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\|f^*-{1\over n}\sum_{i=1}^n f_{\ell_i}\|_{L^2(\mu,B)}^2
\le\frac4n.
$$
Thus, there exisit $\ell_1^*, \ldots, \ell_n^*\in \mathcal N$ such that
$$
\|f^*-{1\over n}\sum_{i=1}^n f_{\ell_i^*}\|_{L^2(\mu,B)}^2
\le\frac4n.
$$
where
$$
f_n(x)={1\over n}\sum_{i=1}^n f_{\ell_i^*}(x).
$$
Then we have
\begin{equation}
\|\tilde f-f_n\|^2_{L^2(\mu,B)}\le \frac{9}{n}.
\end{equation}
Consequently
\begin{equation}
\left\|f(x)-f(x_B)-\|f\|_Bf_n\right\|^2_{L^2(\mu,B)}\le \frac{9\|f\|^2_B}{n}.
\end{equation}

\subsection{Piecewise linear function}
The proof here is almost the same as the proof for Heaviside function in the last part.

Now we take a piecewise linear interpolation for $g_i$ on $[0,1]$, since $g_i(t_0)=0$ , we get
$$
g_{i,k}(t)=(\Pi_kg_i)(t)=\sum_{j=1}^{k}[g_i(t_{j})-g_i(t_{j-1})]\sigma_{j-1}(t),   \quad t\in [0,1]
$$
where
$$
\sigma_j(t)=M_0(\frac{t-t_j}{h_k})
$$
and
\begin{equation}
\label{cardinal}
M_0(x)=
\left\{
\begin{array}{ll}
0 & x\le0 \\
x & 0< x\le1    \\
1 & x > 1
\end{array}
\right.
\end{equation}
%We note that
%$$
%M_0(x)=ReLU(x)-ReLU(x-1)
%$$

%%%%%%%%%%%%%%%%%%%%%%%%%%%
Consider
\begin{equation}
h_i(t) = g_i(-t), \quad t\in [0,1].
\end{equation}
Similarly, we have
$$
(\Pi_kh_i)(t)=\sum_{j=1}^{k}(h_i(t_j) - h_i(t_{j-1}))\sigma_{j-1}(t)=
=\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(t)
$$
Namely
$$
(\Pi_k g_i)(-t)=\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(t), \quad t\in [0,1]
$$
or
\begin{equation}\label{gi1}
(\Pi_k g_i)(t)=\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(-t), \quad t\in [-1,0]
\end{equation}
Combine together we get a piecewise linear
interpolation of $g_i$ on $[-1,1]$ as follows:
\begin{eqnarray}
g_{i,k}(t)&=&
\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(-t)+\sum_{j=1}^{k}(g_i(t_j) - g_i(t_{j-1}))\sigma_{j-1}(t)\nonumber \\
&=&\sum_{j=1}^{k}[a_{ij}^-\sigma_{j-1}(-t)+a_{ij}^+\sigma_{j-1}(t)] \label{gih}
\quad t\in [-1,1]
\end{eqnarray}
where
$$
a_{ij}^{\pm}=g_i(\pm t_j) - g_i(\pm t_{j-1})
$$
%It is easy to see that
%\begin{equation}
%|g_i(t)-g_{i,k}(t)|\le h_k, \quad t\in [-1,1].
%\end{equation}
%\begin{equation}
%\|\frac1n \sum_{i=1}^n \frac{g(\cdot,\omega_i)}{|\omega_i|_B}-f^*\|_{L^2(\mu,B)}\le h_k
%\end{equation}
%where
%\begin{equation}
%\label{fstar}
%f^*(x)=
%\frac1n\sum_{i=1}^ng_{i,k}(\omega_i^B\cdot (x-x_B)).
%\end{equation}
%
%By Theorem \ref{jones}, we have
%\begin{equation}
%%\|\tilde f-f^*\|_{1,B}\le \frac{C_1(d,B)}{\sqrt{n}}
%  \|\tilde f-f^*\|_{L^2(\mu,B)}\le \frac{2}{\sqrt{n}}
%\end{equation}
%here $C_1(d,B)=\sqrt{\mu(B)}[\sqrt{(d+{\rm diam}(B))}+{\rm diam}(B)(1+d)]$.

%Let us rewrite
%$$
%f^*(x)
%=\sum_{i=1}^n\sum_{j=1}^{k}[\gamma_{ij}^- f_{ij}^-+ \gamma_{ij}^+f_{ij}^+]
%$$
%where
%$$
%\gamma_{ij}^{\pm}=\frac{|a^\pm_{ij}|}{nd_i},
%f^\pm_{i,j}=d_i\/{\rm sign}(a^\pm_{ij})\sigma_{j-1}(\pm \omega_i^B\cdot (x-x_B))
%$$
%and
%$$
%d_i=\sum_{j=1}^{k}(|a^-_{ij}|+|a^+_{ij}|)\le 2
%$$
%By definition
%\begin{equation}
%\label{gammaij}
%\sum_{i=1}^n\sum_{j=1}^{k}[\gamma_{i,j}^-+\gamma_{i,j}^+]=1.
%\end{equation}
%With re-numeration as
%$$
%p_\ell=\gamma_{ij}^{\pm}, f_\ell = f_{ij}^{\pm}, 1\le \ell \le N=2nk
%$$
%We have
%$$
%f^*(x)=\sum_{\ell=1}^N p_\ell f_\ell
%$$
%Consider
%$$
%\mathcal N=\{1,2,\ldots, N\}
%$$
%and
%$$
%\bar f: \mathcal N\mapsto \mathbb R^1
%$$
%such that
%$$
%\bar f(\ell)=f_{\ell}, \ell\in \mathcal N
%$$
%With the probability measure
%$$
%\lambda(\mathcal M)=\sum_{m\in \mathcal M}p_m \quad \mathcal M\subset\mathcal N.
%$$
%By definition.
%$$
%\mathbb E(\bar f) = f^*(x).
%$$
%
%By the basic result on expectation:
%\begin{equation}
%\begin{aligned}
%&\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\left((f^*(x)-
%{1\over n}\sum_{i=1}^n f_{\ell_i})^2\right)\\
%=&\mathbb E_{\mathcal N^n} \left(\mathbb E(\bar f)-{1\over
%	n}\sum_{i=1}^n  \bar f(\ell_i)\right)^2
%\le\frac1n \|\bar f\|_{\infty}^2 \le\frac{4}{n}.
%\end{aligned}
%\end{equation}
%
%By taking the $L^2(\mu,B)$ on the above inequality, we get
%$$
%\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\|f^*-{1\over n}\sum_{i=1}^n f_{\ell_i}\|_{L^2(\mu,B)}^2
%\le\frac4n.
%$$
%Thus, there exisit $\ell_1^*, \ldots, \ell_n^*\in \mathcal N$ such that
%$$
%\|f^*-f_n(x)\|_{L^2(\mu,B)}^2
%\le\frac4n.
%$$
%where
%$$
%f_n(x)={1\over n}\sum_{i=1}^n f_{\ell_i^*}(x).
%$$
%Then we have
%\begin{equation}
%\|\tilde f-f_n\|^2_{L^2(\mu,B)}\le \frac{9}{n}.
%\end{equation}

%Consequently
%\begin{equation}
%\left\|f(x)-f(x_B)-\|f\|_Bf_n\right\|^2_{L^2(\mu,B)}\le \frac{9\|f\|^2_B}{n}.
%\end{equation}

Follow the procedure in last section, and notice that $\sigma(x)=ReLU(x)-ReLU(x-1)$, we obtain the following theorem.

\begin{theorem}
	For a probability measure $\mu$ on B and every function with $\|f\|_B<\infty$, there exists $\omega_1,\dots,\omega_n\in\mathbb{R}^d$ such that
$$
\left\|f(x)-f_n(x)\right\|^2_{L^2(\mu,B)}\le \frac{C\|f\|^2_B}{n}.
$$
where $f_n(x)=\sum_{i=1}^{n}a_iReLU(\omega_i x+b_i)+c$.
\end{theorem}


% !TEX root = main.tex

\section{Mathematics}

\begin{itemize}
\item
An inline equation with TeX delimiters (dollars): $\sum_{n=1}^{\infty} a_n = 1$.
\item
An inline equation with LaTeX delimiters: \(\sum_{n=1}^{\infty} a_n = 1\).
\end{itemize}

\bigskip
An \texttt{equation} environment:
\begin{equation}
\sum_{n=1}^{\infty} a_n = 1.
\end{equation}

An \texttt{equation*} environment:
\begin{equation*}
\sum_{n=1}^{\infty}\frac{1}{n^2} = \frac{\pi^2}{6}.
\end{equation*}

A \texttt{display} environment with LaTeX delimiters:
\[
x = \frac{-b\pm\sqrt{b^2-4ac}}{2a}.
\]

A \texttt{display} environment with TeX delimiters (double-dollars):
$$
x = \frac{-b\pm\sqrt{b^2-4ac}}{2a}.
$$

A \texttt{display} environment containing a \texttt{text} command, which itself contains (inline) mathmode:
\[
E=mc^2 \text{ where $c$ is the speed of light}.
\]

\subsection{Labelled equations}

An equation with a label:
\begin{equation}\label{eq:euler}
e^{i\pi}+1=0.
\end{equation}

Equation -\eqref{eq:euler}- is due to Euler.

An \texttt{align} environment:
\begin{align}\label{eq:polar}
x & = r\sin\theta \\
y & = r\cos\theta
\end{align}

%\bigskip
%A labelled \texttt{equation} environment:
%\begin{equation}\label{eq:einstein}
%E = mc^2
%\end{equation}
%Equation -\eqref{eq:einstein}- is due to AE.

\bigskip
An {\tt equation} containing three {\tt pmatrix} environments:
\begin{equation}
\begin{pmatrix}0 & 1 \\ 1 & 0\end{pmatrix}
\qquad
\begin{pmatrix}0 & -i \\ i & 0\end{pmatrix}
\qquad
\begin{pmatrix}1 & 0 \\ 0 & -1\end{pmatrix}
\end{equation}


% !TeX root = ../main.tex
\documentclass[../main.tex]{subfiles}
\begin{document}
\section{Geometric transformation}
\begin{lemma}\label{M:transform:distance}
For any point \(p,q\in N\)
and element \(X\in X\left(n,\lambda\right)\),
\[
\norm{\left(Xp\right)\left(Xq\right)}=\norm{pq}\text{.}
\]
\end{lemma}
\begin{proof}[\proofof{M:transform:distance}]
If \(\lambda>0\),
\begin{align*}
X^{-1}&=X^T\\
&=X^TI\\
I&=X^TIX\\
g&=X^TgX\text{.}
\end{align*}
If \(\lambda\to 0\),
\begin{align*}
X&\to I\\
X^T
&\to I^T\\
&=I\\
g&=g\\
g&=IgI\\
g&\to X^TgX\text{.}
\end{align*}
If \(\lambda<0\),
\begin{align*}
X^{-1}&=gX^Tg\\
I&=gX^TgX\\
g^{-1}&=X^TgX\\
g&=X^TgX\text{.}
\end{align*}
\begin{align*}
\norm{pq}
&=\sqrt{g\left(q-p,q-p\right)}\\
&=\sqrt{\left(q-p\right)^Tg\left(q-p\right)}\\
\norm{\left(Xp\right)\left(Xq\right)}
&=\sqrt{g\left(Xq-Xp,Xq-Xp\right)}\\
&=\sqrt{g\left(X\left(q-p\right),X\left(q-p\right)\right)}\\
&=\sqrt{\left(X\left(q-p\right)\right)^Tg\left(X\left(q-p\right)\right)}\\
&=\sqrt{\left(q-p\right)^TX^TgX\left(q-p\right)}\\
&=\sqrt{\left(q-p\right)^Tg\left(q-p\right)}\\
&=\norm{pq}\qedhere
\end{align*}
\end{proof}
\end{document}


\chapter{Mathematical derivations of MOFA+} \label{appendix:mofa}

\section{Deriving the variational inference algorithm}

The theoretical foundations for the variational inference scheme are described in \Cref{section:variational_inference}. Just to brief, we need to define a variational distribution of a factorised form and subsequently look for the member of this family that most closely resembles the true posterior using the KL divergence as a \textit{distance} metric. Following the mean-field principle, in MOFA+ we factorised the variational distribution as follows:

\begin{equation} \begin{aligned}
	q(\bfX) &= q \left(\{\widehat{\bfZ^\bfg}, \bfS^\bfg, \bf{\alpha^g}, \bf{\theta^g}\}, \{\widehat{\bfW^m}, \bfS^m, \bf\alpha^m, \bf\theta^m\}, \{ \bf\tau^{gm}\} \right)\\
	&= \prod_{g=1}^{G} \prod_{n=1}^{N_g} \prod_{k=1}^{K} q(\hat{z}_{nk}^g, s_{nk}^g) \prod_{g=1}^{G} \prod_{k=1}^{K} q(\alpha_k^g) \prod_{g=1}^{G} \prod_{k=1}^{K} q(\theta_k^g) \\
	&\times \prod_{m=1}^{M} \prod_{d=1}^{D_m} \prod_{k=1}^{K} q(\hat{w}_{kd}^m,s_{kd}^m) \prod_{m=1}^{M} \prod_{k=1}^{K} q(\alpha_k^m) \prod_{m=1}^{M} \prod_{k=1}^{K} q(\theta_k^m)  \\
	&\times \prod_{g=1}^{G} \prod_{m=1}^{M} \prod_{d=1}^{D_m} q(\tau_d^{gm})
\end{aligned} \end{equation}

However, inspired by \cite{Titsias2011}, we did not adopt a fully factorised distribution as $\hat{w}_k^m$ and $s_k^m$ can hardly be assumed to be independent.

To derive the variational updates we can proceed in two ways, as described in \Cref{section:variational_inference}. One option is to use exploit the  mean-field assumption and use calculus of variations to find the optimal distribution $q(\bfX)$ that maximises the lower bound $\Lagr(\bfX)$\cite{Bishop2006,Murphy}. The alternative and possibly easier approach is to define a parametric form for the distribution $q(\bfX)$ with some parameters $\bTheta$ to be of the same form as the corresponding prior distribution $p(\bfX)$. Then, one can find the gradients with respect to the parameters to obtain the coordinate ascent optimisation scheme. In our derivations we followed the first approach, but because we used conjugate priors the second one should converge to the same result.\\
Below we give the explicit update equations for every hidden variable in the MOFA+ model which are applied at each iteration of the variational inference algorithm.

\section{Variational update equations}

\paragraph*{Factors}

For every group $g$, sample $n$ and factor $k$: \\

Prior distribution $p(\hat{z}_{nk}^g,s_{nk}^g)$:
\begin{align}
	p(\hat{z}_{nk}^g,s_{nk}^g) &= \mathcal{N} (\hat{z}_{nk}^g \,|\, 0, ~ 1/\alpha_k^g)\, \text{Ber}(s_{nk}^g \,|\,\theta_k^g)
\end{align}

Variational distribution $q(\hat{z}_{nk}^g,s_{nk}^g)$:\\

Update for $q(s_{nk}^g)$:
\begin{equation}
	q(s^g_{nk}) = \mathrm{Ber}(s^g_{nk}|\gamma^g_{nk})
\end{equation}
with
\begin{equation} \begin{aligned}
	&\gamma^g_{nk} = \frac{1}{1+\exp(-\lambda_{nk}^g)}\\
	& \lambda_{nk}^g = \la \ln\frac{\theta}{1-\theta} \ra + 0.5\ln\frac{\la\alpha_k^g\ra}{\la\tau_d^{gm}\ra} - 0.5\ln\left( \sum_{m=1}^M\sum_{d=1}^{D_m} \la (w_{kd}^m)^2 \ra + \frac{\la\alpha_k^g\ra}{\la\tau_d^{gm}\ra} \right) \\
	&+ \frac{\la\tau_d^{gm}\ra}{2} \frac{ \left( \sum_{m=1}^M\sum_{d=1}^{D_m} y_{nd}^{gm} \la w^m_{kd} \ra - \sum_{j \neq k} \la s_{nj}^g\hat{z}_{nj}^g\ra \sum_{m=1}^M\sum_{d=1}^{D_m} \la w^m_{kd} \ra \la w^m_{jd} \ra \right)^2} {\sum_{m=1}^M\sum_{d=1}^{D_m} \la (w_{kd}^m)^2 \ra + \frac{\la\alpha_k^g\ra}{\la\tau_d^{gm}\ra} }
\end{aligned} \end{equation}

Update for $q(\hat{z}_{nk}^g)$:
\begin{equation} \begin{aligned}
      q(\hat{z}_{nk}^g|s_{nk}^g=0) &= \mathcal{N} \left(\hat{z}_{nk}^g \middle| 0, 1/\alpha_k^g \right) \\
      q(\hat{z}_{nk}^g|s_{nk}^g=1) &= \mathcal{N} \left( \hat{z}_{nk}^g \middle| \mu_{z_{nk}^g}, \sigma_{z_{nk}^g}^2\right)
  \end{aligned} \end{equation}
with
\begin{equation} \begin{aligned}
  	\mu_{z_{nk}^g} &= \frac{ \sum_{m=1}^M\sum_{d=1}^{D_m} y_{nd}^{m,g} \la w^m_{kd} \ra - \sum_{j \neq k} \la s_{nj}^g\hat{z}_{nj}^g \ra \sum_{m=1}^M\sum_{d=1}^{D_m} \la w^m_{kd} \ra \la w^m_{jd} \ra } { \sum_{m=1}^M\sum_{d=1}^{D_m} \la (w_{kd}^m)^2 \ra + \frac{\la\alpha_k^g\ra}{\la\tau_d^{gm}\ra} }\\
  	\sigma_{z_{nk}^g}^2 &= \frac{ \la\tau_d^{gm}\ra^{-1} } { \sum_{m=1}^M\sum_{d=1}^{D_m} \la (w_{kd}^m)^2 \ra + \frac{\la\alpha_k^g\ra}{\la\tau_d^{gm}\ra} }
\end{aligned} \end{equation}

\paragraph*{ARD prior on the factors}

For every group $g$ and factor $k$: \\

Prior distribution:
\begin{align}
	p(\alpha_k^g) &= \Gdist{\alpha_k^g}{a_0^\alpha, b_0^\alpha}
\end{align}

Variational distribution $q(\alpha_k^g)$:
\begin{equation}
    q(\alpha^g_{k}) = \Gdist{\alpha_k^g}{\hat{a}_{gk}^{\alpha}, \hat{b}_{gk}^{\alpha}}
\end{equation}
where:
\begin{equation} \begin{aligned}
	\hat{a}_{gk}^\alpha &= a_0^\alpha + \frac{N_g}{2}\\
	\hat{b}_{gk}^\alpha &= b_0^\alpha +\frac{ \sum_{n=1}^{N_g} \la (\hat{z}_{nk}^g)^2 \ra }{2}
\end{aligned} \end{equation}

\paragraph*{Sparsity parameter of the Factors}

For every group $g$ and factor $k$: \\

Prior distribution:
\begin{align}
	p(\theta_k^g) &= \Bdist{\theta_k^g}{a_0^\theta,b_0^\theta}
\end{align}

Variational distribution:
\begin{equation}
	q(\theta_k^g) = \Bdist{\theta_k^g}{\hat{a}_{gk}^{\theta}, \hat{b}_{gk}^{\theta}}
\end{equation}
where
\begin{equation}
     \begin{aligned}
  	\hat{a}_{gk}^\theta &= \sum_{n=1}^{N_g} \la s^g_{nk}\ra + a_0^\theta\\
  	\hat{b}_{gk}^\theta &= b_0^\theta - \sum_{n=1}^{N_g} \la s^g_{nk}\ra + N_g
     \end{aligned}
\end{equation}

\paragraph*{Feature weights}
For every view $m$, feature $d$ and factor $k$: \\

Prior distribution $p(\hat{w}_{kd}^m,s_{kd}^m)$:
\begin{align}
	p(\hat{w}_{kd}^m,s_{kd}^m) &= \mathcal{N} (\hat{w}_{kd}^m \,|\, 0, ~ 1/\alpha_k^m)\, \text{Ber}(s_{kd}^m \,|\,\theta_k^m)
\end{align}

Variational distribution $q(\hat{w}_{kd}^m,s_{kd}^m)$:\\

Update for $q(s_{kd}^m)$:
\begin{equation}
	q(s^m_{kd}) = \mathrm{Ber}(s^m_{kd}|\gamma^m_{kd})
\end{equation}
with
\begin{equation} \begin{aligned}
	&\gamma^m_{kd} = \frac{1}{1+\exp(-\lambda_{kd}^m)}\\
	& \lambda_{kd}^m = \la \ln\frac{\theta}{1-\theta} \ra + 0.5\ln\frac{\la\alpha_k^m\ra}{\la\tau_d^{gm}\ra} - 0.5\ln\left( \sum_{g=1}^G\sum_{n=1}^{N_g} \la (z_{nk}^g)^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^{gm}\ra} \right) \\
	&+ \frac{\la\tau_d^{gm}\ra}{2} \frac{ \left( \sum_{g=1}^G\sum_{n=1}^{N_g} y_{nd}^{gm} \la z^g_{nk} \ra - \sum_{j \neq k} \la s_{jd}^m\hat{w}_{jd}^m\ra \sum_{g=1}^G\sum_{n=1}^{N_g} \la z^g_{nk} \ra \la z^g_{nj} \ra \right)^2} {\sum_{g=1}^G\sum_{n=1}^{N_g} \la (z_{nk}^g)^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^{gm}\ra} }
\end{aligned} \end{equation}

Update for $q(\hat{w}_{kd}^m)$:
\begin{equation} \begin{aligned}
      q(\hat{w}_{kd}^m|s_{kd}^m=0) &= \mathcal{N} \left(\hat{w}_{kd}^m \middle| 0, 1/\alpha_k^m \right) \\
      q(\hat{w}_{kd}^m|s_{kd}^m=1) &= \mathcal{N} \left( \hat{w}_{kd}^m \middle| \mu_{w_{kd}^m}, \sigma_{w_{kd}^m}^2\right)
  \end{aligned} \end{equation}
with
\begin{equation} \begin{aligned}
  	\mu_{w_{kd}^m} &= \frac{ \sum_{g=1}^G\sum_{n=1}^{N_g} y_{nd}^{gm} \la z^g_{nk} \ra - \sum_{j \neq k} \la s_{jd}^m\hat{w}_{jd}^m \ra \sum_{g=1}^G\sum_{n=1}^{N_g} \la z^g_{nk} \ra \la z^g_{nj} \ra } { \sum_{g=1}^G\sum_{n=1}^{N_g} \la (z_{nk}^g)^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^{gm}\ra} }\\
  	\sigma_{w_{kd}^m}^2 &= \frac{ \la\tau_d^{gm}\ra^{-1} } { \sum_{g=1}^G\sum_{n=1}^{N_g} \la (z_{nk}^g)^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^{gm}\ra} }
\end{aligned} \end{equation}

  	% Taken together this means that we can update $q(\hat{w}_{kd}^m,s_{kd}^m)$ using:
  	% \begin{equation*}
  	% q(\hat{w}_{kd}^m|s_{kd}^m) q(s_{kd}^m) = \Ndist{ \hat{w}_{kd}^m } { s_{kd}^m \mu_{w_{kd}^m}, s_{kd}^m\sigma_{w_{kd}^m}^2 + (1-s_{kd}^m)/\alpha_k^m}    (\new{\gamma_{kd}^m})^{s_{kd}^m} (1-\new{\gamma_{kd}^m})^{1-s_{kd}}
  	% \end{equation*}

\paragraph*{ARD prior on the weights}

For every view $m$ and factor $k$: \\

Prior distribution $p(\alpha_k^m)$:
\[
	p(\alpha_k^m) = \Gdist{\alpha_k^m}{a_0^\alpha, b_0^\alpha}
\]
Variational distribution $q(\alpha_k^m)$:
\begin{equation}
    q(\alpha^m_{k}) = \Gdist{\alpha_k^m}{\hat{a}_{mk}^{\alpha}, \hat{b}_{mk}^{\alpha}}
\end{equation}
where:
\begin{equation} \begin{aligned}
	\hat{a}_{mk}^\alpha &= a_0^\alpha + \frac{D_m}{2}\\
	\hat{b}_{mk}^\alpha &= b_0^\alpha +\frac{ \sum_{d=1}^{D_m} \la (\hat{w}_{kd}^m)^2 \ra }{2}
\end{aligned} \end{equation}

\paragraph*{Sparsity parameter of the weights}

For every view $m$ and factor $k$: \\

Prior distribution:
\[
	p(\theta_k^m) = \Bdist{\theta_k^m}{a_0^\theta,b_0^\theta}
\]

Variational distribution:
\begin{equation}
	q(\theta_k^m) = \Bdist{\theta_k^m}{\hat{a}_{mk}^{\theta}, \hat{b}_{mk}^{\theta}}
\end{equation}
where
\begin{equation}
     \begin{aligned}
  	\hat{a}_{mk}^\theta &= \sum_{d=1}^{D_m} \la s^m_{kd}\ra + a_0^\theta\\
  	\hat{b}_{mk}^\theta &= b_0^\theta - \sum_{d=1}^{D_m} \la s^m_{kd}\ra + D_m
     \end{aligned}
\end{equation}

\paragraph*{Noise (Gaussian)}

For every view $m$, group $g$ and feature $d$:\\

Prior distribution $p(\tau_d^{gm})$:
\[
	p(\tau_d^{gm}) = \Gdist{\tau_{dg}^m}{a_0^\tau,b_0^\tau},
\]

Variational distribution $q(\tau_d^{gm})$:
\begin{equation}
	q(\tau_d^{gm}) = \Gdist{\tau_d^{gm}}{\hat{a}_{d}^{gm} , \hat{b}_{d}^{gm}}
\end{equation}
where:
\begin{equation} \begin{aligned}
	\hat{a}_{d}^{gm} &= a_0^{\tau} + \frac{N_g}{2}\\
	\hat{b}_{d}^{gm} &= b_0^{\tau} + \frac{1}{2} \sum_{n=1}^{N_g}  \la\left(y_{nd}^{gm} - \sum_k^{K} w_{kd}^m z_{nk}^{g}\right)^2 \ra
\end{aligned} \end{equation}

\section{Evidence Lower Bound}

Although computing the ELBO is not necessary in order to estimate the posterior distribution of the parameters, it is used to monitor the convergence of the algorithm. As shown in \Cref{eq_elbo2}, the ELBO can be decomposed into a sum of two terms: (1) the expected log likelihood under the current estimate of the posterior distribution of the parameters and (2) the KL divergence between the prior and the variational distributions of the parameters:\\

\begin{equation} \begin{aligned}
    \Lagr = \E_{q(X)} \ln p(Y|X) - \KL\left(q(X) \middle|\middle| p(X) \right)
\end{aligned} \end{equation}

\paragraph*{Log likelihood term}

Assuming a Gaussian likelihood:
\begin{equation} \begin{aligned}
	\E_{q(X)} \ln p(Y|X) = & -\sum_{m=1}^M \frac{ND_m}{2} \ln(2\pi) + \sum_{g=1}^G \frac{N_g}{2} \sum_{m=1}^M \sum_{d=1}^{D_m} \la \ln(\tau_{d}^{gm}) \ra \\
	&-\sum_{g=1}^G \sum_{m=1}^M \sum_{d=1}^{D_m} \frac{\la \tau_{d}^{gm} \ra}{2} \sum_{n=1}^{N_g} \big( y_{nd}^{m,g} - \sum_{k=1}^{K}\la s_{kd}^m \hat{w}_{kd}^m \ra \la z_{nk}^g \ra \big)^2
\end{aligned} \end{equation}

\paragraph*{KL divergence terms}

Note that $\KL\left(q(X) \middle|\middle| p(X) \right) = \E_q(q(X)) - \E_q(p(X))$. \\
Below, we will write the analytical form for these two expectations.

\paragraph*{Weights}

\begin{equation} \begin{aligned}
    \E_q[\ln p(\hat{W},S)] =& -\sum_{m=1}^{M}\frac{KD_m}{2}\ln(2\pi) + \sum_{m=1}^{M}\frac{D_m}{2}\sum_{k=1}^{K} \ln(\alpha_k^m) - \sum_{m=1}^{M} \frac{\alpha_k^m}{2} \sum_{d=1}^{D_m} \sum_{k=1}^{K} \la (\hat{w}_{kd}^m)^2 \ra \\
    & + \la \ln(\theta) \ra \sum_{m=1}^{M} \sum_{d=1}^{D_m} \sum_{k=1}^{K} \la s_{kd}^m \ra + \la \ln(1-\theta) \ra \sum_{m=1}^{M} \sum_{d=1}^{D_m}\sum_{k=1}^{K} (1- \la s_{kd}^m \ra)
\end{aligned} \end{equation}

\begin{equation} \begin{aligned}
	\E_q[\ln q(\hat{W}, S)] =&-\sum_{m=1}^{M}\frac{KD_m}{2}\ln(2\pi) + \frac{1}{2}\sum_{m=1}^{M}\sum_{d=1}^{D_m}\sum_{k=1}^{K}\ln(\la s_{kd}^m \ra \sigma_{w_{kd}^m}^2 + (1-\la s_{kd}^m \ra)/\alpha_k^m) \\
	&+ \sum_{m=1}^{M} \sum_{d=1}^{D_m} \sum_{k=1}^{K} (1-\la s_{kd}^m \ra) \ln(1 - \la s_{kd}^m \ra) - \la s_{kd}^m \ra \ln \la s_{kd}^m \ra
\end{aligned} \end{equation}

\paragraph*{Factors}

\begin{equation} \begin{aligned}
    \E_q[\ln p(\hat{Z},S)] =& -\sum_{g=1}^{G}\frac{N_g K}{2}\ln(2\pi) + \sum_{g=1}^{G}\frac{N_g}{2}\sum_{k=1}^{K} \ln(\alpha_k^g) - \sum_{g=1}^{G} \frac{\alpha_k^g}{2} \sum_{n=1}^{N_g} \sum_{k=1}^{K} \la (\hat{z}_{nk}^g)^2 \ra \\
    & + \la \ln(\theta) \ra \sum_{g=1}^{G} \sum_{n=1}^{N_g} \sum_{k=1}^{K} \la s_{nk}^g \ra + \la \ln(1-\theta) \ra \sum_{g=1}^{G} \sum_{n=1}^{N_g}\sum_{k=1}^{K} (1- \la s_{nk}^g \ra)
\end{aligned} \end{equation}

\begin{equation} \begin{aligned}
	\E_q[\ln q(\hat{Z}, S)] =&-\sum_{g=1}^{G}\frac{N_g K}{2}\ln(2\pi) + \frac{1}{2}\sum_{g=1}^{G}\sum_{n=1}^{N_g}\sum_{k=1}^{K}\ln(\la s_{nk}^g \ra \sigma_{z_{nk}^g}^2 + (1-\la s_{nk}^g \ra)/\alpha_k^g) \\
	&+ \sum_{g=1}^{G} \sum_{n=1}^{N_g} \sum_{k=1}^{K} (1-\la s_{nk}^g \ra) \ln(1 - \la s_{nk}^g \ra) - \la s_{nk}^g \ra \ln \la s_{nk}^g \ra
\end{aligned} \end{equation}

\paragraph*{ARD prior on the weights}

\begin{equation} \begin{aligned}
	\E_q [\ln p(\balpha)] &= \sum_{m=1}^{M}\sum_{k=1}^{K}\Big(a_0^\alpha\ln b_0^\alpha +   (a_0^\alpha - 1) \la \ln \alpha_k \ra - b_0^\alpha \la \alpha_k \ra - \ln \Gamma(a_0^\alpha) \Big) \\
	\E_q [\ln q(\balpha)] &= \sum_{m=1}^{M}\sum_{k=1}^{K} \Big( \hat{a}_{k}^\alpha \ln \hat{b}_{k}^\alpha + (\hat{a}_{k}^\alpha - 1) \la \ln \alpha_k \ra - \hat{b}_{k}^\alpha \la \alpha_k \ra - \ln \Gamma(\hat{a}_{k}^\alpha) \Big)
\end{aligned} \end{equation}

\paragraph*{Sparsity parameter of the weights}

\begin{equation} \begin{aligned}
	  &\E_q\left[ \ln p(\btheta) \right] = \sum_{m=1}^M \sum_{k=1}^K\sum_{d=1}^{D_m}\left( (a_0 - 1) \times \la \ln(\pi^m_{d, k}) \ra + (b_0 -1) \la \ln(1 - \pi^m_{d, k}) \ra - \ln (\mathrm{B} (a_0, b_0))\right) \\
	  &\E_q\left[ \ln q(\btheta) \right] = \sum_{m=1}^M \sum_{k=1}^K\sum_{d=1}^{D_m}\left( (a^m_{k,d} - 1) \times \la \ln(\pi^m_{d, k}) \ra + (b^m_{k,d} -1) \la \ln(1 - \pi^m_{d, k}) \ra - \ln (\mathrm{B} (a^m_{k,d}, b^m_{k,d})) \right) \\
\end{aligned} \end{equation}

\paragraph*{ARD prior on the Factors}

\begin{equation} \begin{aligned}
	\E_q [\ln p(\balpha)] &= \sum_{g=1}^{G}\sum_{k=1}^{K}\Big(a_0^\alpha\ln b_0^\alpha +   (a_0^\alpha - 1) \la \ln \alpha_k \ra - b_0^\alpha \la \alpha_k \ra - \ln \Gamma(a_0^\alpha) \Big) \\
	\E_q [\ln q(\balpha)] &= \sum_{g=1}^{G}\sum_{k=1}^{K} \Big( \hat{a}_{k}^\alpha \ln \hat{b}_{k}^\alpha + (\hat{a}_{k}^\alpha - 1) \la \ln \alpha_k \ra - \hat{b}_{k}^\alpha \la \alpha_k \ra - \ln \Gamma(\hat{a}_{k}^\alpha) \Big)
\end{aligned} \end{equation}

\paragraph*{Sparsity parameter of the Factors}
\begin{equation} \begin{aligned}
	  &\E_q\left[ \ln p(\btheta) \right] = \sum_{g=1}^G \sum_{k=1}^K\sum_{n=1}^{N_g}\left( (a_0 - 1) \times \la \ln(\pi^g_{n, k}) \ra + (b_0 -1) \la \ln(1 - \pi^g_{n, k}) \ra - \ln (\mathrm{B} (a_0, b_0))\right) \\
	  &\E_q\left[ \ln q(\btheta) \right] = \sum_{g=1}^G \sum_{k=1}^K\sum_{n=1}^{N_g}\left( (a^g_{k,n} - 1) \times \la \ln(\pi^g_{n, k}) \ra + (b^g_{k,n} -1) \la \ln(1 - \pi^g_{n, k}) \ra - \ln (\mathrm{B} (a^g_{k,n}, b^g_{k,n})) \right) \\
\end{aligned} \end{equation}

\paragraph*{Noise}

\begin{equation} \begin{aligned}
	\E_q [\ln p(\btau)] &= \sum_{m=1}^{M} D_m a_0^\tau \ln b_0^\tau + \sum_{g=1}^{G}\sum_{m=1}^{M}\sum_{d=1}^{Dm} (a_0^\tau - 1) \la \ln \tau_d^{gm} \ra - \sum_{g=1}^{G}\sum_{m=1}^{M}\sum_{d=1}^{Dm} b_0^\tau \la \tau_d^{gm} \ra - \sum_{m=1}^{M} D_m \ln \Gamma(a_0^\tau)\\
	\E_q [\ln q(\btau)] &= \sum_{g=1}^{G} \sum_{m=1}^{M} \sum_{d=1}^{D_m} \left( \hat{a}_{dgm}^\tau \ln \hat{b}_{dgm}^\tau + (\hat{a}_{dgm}^\tau - 1) \la \ln \tau_d^{gm} \ra - \hat{b}_{dgm}^\tau \la \tau_d^{gm} \ra - \ln \Gamma(\hat{a}_{dgm}^\tau) \right)
\end{aligned} \end{equation}