\chapter{Concepts in Spin Conductance Models}

\section{Converting from $\Pi$-model to T-model}

Depending on the situation, it might be easier to work with the $\Pi$-model or the T-model. However, the derivation of the conductance matrix is straightforward using the $\Pi$-model, as described earlier. Hence, we need to derive the equivalent conductances in the T-model from the $\Pi$-model conductances. Before we begin, note that in the spin conductance model, the conductances are tensors instead of scalars. Hence, it is important to preserve the order of multiplication.

Let us begin by looking at the $\Pi$-model as shown in Fig.~\ref{fig:modelDiagrams}. We can first write the following equations:
\begin{IEEEeqnarray}{rCl}
I_{out}&=&I_{p}+I_{2} \label{eq:piIout}\\
I_{in}&=&I_{p}-I_{1} \label{eq:piIin}\\
I_{1}&=&G_{sf\pi}V_{1} \label{eq:piI1}\\
I_{2}&=&G_{sf\pi}V_{2} \label{eq:piI2} \\
I_{p}&=&G_{se\pi}(V_{2}-V_{1}) \label{eq:piIp}
\end{IEEEeqnarray}Then, we add Eqs.~(\ref{eq:piIout}) and (\ref{eq:piIin}) to get \begin{IEEEeqnarray}{rCl}
I_{out}+I_{in}&=&2I_{p}+I_{2}-I_{1} \label{eq:piCurrSum}
\end{IEEEeqnarray}Substituting Eqs.~(\ref{eq:piI1}), (\ref{eq:piI2}) and (\ref{eq:piIp}) into Eq.~(\ref{eq:piCurrSum}), we obtain \begin{IEEEeqnarray}{rCl}
I_{out}+I_{in}&=&2G_{se\pi}(V_{2}-V_{1})+G_{sf\pi}V_{2}-G_{sf\pi}V_{1} \nonumber \\
&=& 2G_{se\pi}(V_{2}-V_{1})+G_{sf\pi}(V_{2}-V_{1}) \label{eq:piCurrSum2} \\
&=& (2G_{se\pi}+G_{sf\pi})(V_{2}-V_{1}) \nonumber
\end{IEEEeqnarray}Next, we subtract Eq.~(\ref{eq:piIin}) from Eq.~(\ref{eq:piIout}) to get \begin{IEEEeqnarray}{rCl}
I_{out}-I_{in}&=&I_{2}+I_{1} \label{eq:piCurrDiff}
\end{IEEEeqnarray}Substituting Eqs.~(\ref{eq:piI1}) and (\ref{eq:piI2}) into Eq.~\ref{eq:piCurrDiff}, we get \begin{IEEEeqnarray}{rCl}
I_{out}-I_{in}&=&G_{sf\pi}(V_{2}+V_{1}) \label{eq:piCurrDiff2}
\end{IEEEeqnarray}

Now, let us look at the T-model in Fig.~\ref{fig:modelDiagrams}. The objective is to write $I_{out}+I_{in}$ and $I_{out}-I_{in}$, and compare them to the equations for the $\Pi$-model so that we can obtain equations for $G_{seT}$ and $G_{sfT}$ in terms of $G_{se\pi}$ and $G_{sf\pi}$. First, we can write the following equations for the T-model:\begin{IEEEeqnarray}{rCl}
I_{out}&=&G_{seT}(V_{2}-V_{m}) \label{eq:tIout}\\
I_{in}&=&G_{seT}(V_{m}-V_{1}) \label{eq:tIin}\\
I_{m}&=&G_{sfT}V_{m} \label{eq:tIm}\\
I_{out}&=&I_{in}+I_{m} \label{eq:tKCL1}
\end{IEEEeqnarray}Adding Eqs.~(\ref{eq:tIout}) and (\ref{eq:tIin}), we get \begin{IEEEeqnarray}{rCl}
I_{out}+I_{in}&=&G_{seT}(V_{2}-V_{1}) \label{eq:tCurrSum}
\end{IEEEeqnarray}\begin{figure}
\centering
\includegraphics[scale=1.75]{ResearchNotes_SpinConductanceModels/figs/2Ports}
\caption{These circuit diagrams show how conductances are arranged in the $\Pi$-model and the T-model.}
\label{fig:modelDiagrams}
\end{figure}Subtracting Eq.~(\ref{eq:tIin}) from Eq.~(\ref{eq:tIout}), we get \begin{IEEEeqnarray}{rCl}
I_{out}-I_{in}&=&G_{seT}(V_{2}+V_{1}-2V_{m}) \label{eq:tCurrDiff}
\end{IEEEeqnarray}Then, we rearrange Eq.~(\ref{eq:tKCL1}) and substituting into Eq.~(\ref{eq:tIm}):\begin{IEEEeqnarray}{rCl}
I_{out}-I_{in}&=&G_{sfT}V_{m} \label{eq:tCurrDiff2}
\end{IEEEeqnarray}Pre-multiplying both sides of Eq.~(\ref{eq:tCurrDiff2}) with $G^{-1}_{sfT}$:\begin{IEEEeqnarray}{rCl}
V_{m}&=&G^{-1}_{sfT}(I_{out}-I_{in}) \label{eq:tVm}
\end{IEEEeqnarray}Substituting Eq.~(\ref{eq:tVm}) into Eq.~(\ref{eq:tCurrDiff}):\begin{IEEEeqnarray}{rCl}
I_{out}-I_{in}&=&G_{seT}(V_{2}+V_{1})-2G_{seT}G^{-1}_{sfT}(I_{out}-I_{in}) \label{eq:tCurrDiff3}
\end{IEEEeqnarray}Rearranging Eq.~(\ref{eq:tCurrDiff3}):\begin{IEEEeqnarray}{rCl}
G_{seT}(V_{2}+V_{1})&=&(I+2G_{seT}G^{-1}_{sfT})(I_{out}-I_{in}) \label{eq:tCurrDiff4}
\end{IEEEeqnarray}where $I$ is the identity matrix.

Comparing Eqs.~(\ref{eq:piCurrSum2}) and (\ref{eq:tCurrSum}), we get \begin{IEEEeqnarray}{rCl}
G_{seT}&=&2G_{se\pi}+G_{sf\pi} \label{eq:gset}
\end{IEEEeqnarray}Next, we compare Eqs.~(\ref{eq:piCurrDiff2}) and (\ref{eq:tCurrDiff4}) and obtain \begin{IEEEeqnarray}{rCl}
(I+2G_{seT}G^{-1}_{sfT})G_{sf\pi}&=&G_{seT} \label{eq:gset2}
\end{IEEEeqnarray}Substituting Eq.~(\ref{eq:gset}) into the right-hand side of (\ref{eq:gset2}): \begin{IEEEeqnarray}{rCl}
(I+2G_{seT}G^{-1}_{sfT})G_{sf\pi}&=&2G_{se\pi}+G_{sf\pi} \\
G_{sf\pi}+2G_{seT}G^{-1}_{sfT}G_{sf\pi}&=&2G_{se\pi}+G_{sf\pi} \\
2G_{seT}G^{-1}_{sfT}G_{sf\pi}&=&2G_{se\pi} \label{eq:gsePiA}
\end{IEEEeqnarray}Post-multiplying both sides of Eq.~(\ref{eq:gsePiA}) by $\frac{G^{-1}_{sf\pi}}{2}$, we obtain \begin{IEEEeqnarray}{rCl}
G_{seT}G^{-1}_{sfT}&=&G_{se\pi}G^{-1}_{sf\pi} \label{eq:gProd}
\end{IEEEeqnarray}Post-multiplying both sides of Eq.~(\ref{eq:gProd}) by $G_{sfT}$:\begin{IEEEeqnarray}{rCl}
G_{seT}&=&G_{se\pi}G^{-1}_{sf\pi}G_{sfT} \label{eq:gset3}
\end{IEEEeqnarray}Substitute Eq.~(\ref{eq:gset}) into Eq.~(\ref{eq:gset3}), we get:\begin{IEEEeqnarray}{rCl}
G_{se\pi}G^{-1}_{sf\pi}G_{sfT}&=&2G_{se\pi}+G_{sf\pi} \label{eq:gsft}
\end{IEEEeqnarray}Pre-multiplying both sides of Eq.~(\ref{eq:gsft}) by $G^{-1}_{se\pi}$: \begin{IEEEeqnarray}{rCl}
G^{-1}_{sf\pi}G_{sfT}&=&2I+G^{-1}_{se\pi}G_{sf\pi} \label{eq:gsft2}
\end{IEEEeqnarray}Finally, pre-multiply both sides of Eq.~(\ref{eq:gsft2}) by $G_{sf\pi}$: \begin{IEEEeqnarray}{rCl}
G_{sfT}&=&G_{sf\pi}G^{-1}_{se\pi}G_{sf\pi}+2G_{sf\pi} \label{eq:gsft3}
\end{IEEEeqnarray}

Thus, we can convert from the $\Pi$-model to the T-model using \begin{IEEEeqnarray}{rCl}
G_{seT}&=&2G_{se\pi}+G_{sf\pi} \\
G_{sfT}&=&G_{sf\pi}G^{-1}_{se\pi}G_{sf\pi}+2G_{sf\pi}
\end{IEEEeqnarray}


\documentclass{aastex6}

\usepackage{graphicx}
\usepackage[suffix=]{epstopdf}
\usepackage{natbib}
\usepackage{amsmath}
\usepackage{url}
\usepackage{xspace}
\usepackage{color}

\makeatletter
\def\env@matrix{\hskip -\arraycolsep % taken from amsmath.sty lines 895ff
  \let\@ifnextchar\new@ifnextchar
  \array{*{\c@MaxMatrixCols}c}}
\makeatother

\usepackage{geometry}
\geometry{
	tmargin=4.5cm,
	bmargin=0.5cm,
	lmargin=2.0cm,
	rmargin=0.5cm
}
\linespread{1} % c

\newcommand{\ie}{{\textit{i.e.,}~}}
\newcommand{\eg}{{\textit{e.g.,}~}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\equref}[1]{{\xspace}Eq.~(\ref{#1})}
\newcommand{\figref}[1]{{\xspace}Fig.~\ref{#1}}
\newcommand{\figrefs}[2]{{\xspace}Figs.~\ref{#1}~and ~\ref{#2}}
\newcommand{\equrefbegin}[1]{{\xspace}Equation~(\ref{#1})}
\newcommand{\figrefbegin}[1]{{\xspace}Figure~\ref{#1}}
\newcommand{\secref}[1]{{\xspace}Sec.~\ref{#1}}
\newcommand{\appref}[1]{{\xspace}Appendix.~\ref{#1}}
\renewcommand{\d}{{\mathrm{d}}}
\newcommand{\equ}[1]{\begin{equation}#1\end{equation}}
\newcommand{\eqn}[1]{\begin{eqnarray}#1\end{eqnarray}}
\renewcommand{\vec}[1]{\boldsymbol{#1}}
\newcommand{\negsp}[1]{\hspace*{-#1mm}}
\newcommand{\ntypes}{{\mathrm{N}_\mathrm{types}}}
\newcommand{\nobj}{{\mathrm{N}_\mathrm{obj}}}

\newcommand{\todo}[1]{\textcolor{blue}{[TODO: #1]}}

\begin{document}

\title{Optimized simulation-based estimators for power spectra and covariance matrices}
\author{Boris Leistedt \textit{(send comments and add your name here!)}}
\date{\today}
\maketitle

\vspace*{-3mm}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\textit{Monte-Carlo (MC) estimators} are widely used in cosmology to compute the mean and covariance of observables such as 3D or 2D tomographic auto and cross power spectra of the matter density field.
However, the variance of the simplest MC estimators badly scale as $N^{-1}$ with $N$ the number of simulations, and are proportional to the power spectrum, assuming the draws are uncorrelated.
We show how state-of-the-art \textit{Monte-Carlo variance reduction techniques} such as \textit{antithetic} and \textit{stratified estimators}, as well as \textit{control variates}, can be applied in the cosmological context to drastically reduce the variance of all simulation-based estimates of interest.
We also show how those relate to the recent proposals of using  \textit{paired} and \textit{constrained} simulations.
Even though most of our discussion will be about the 3D power spectrum and its covariance, we will make explicit how our results apply to other observables and their covariances, such as tomographic power spectra, bispectra, etc.
Intermediate conclusions and take-away messages are highlighted in \textbf{bold} and summarized at the end (so go and read those if you only have a limited amount of time!).

Note that we will *not* discuss how to incorporate the uncertainties of the estimated quantities (model power spectra, covariance matrices, etc) into cosmological analyses. We only aim to reduce those uncertaintes. There are lots of techniques to deal with subsequent steps and include those uncertainties in cosmological likelihoods, or even compress observables of interest \citep[\eg][]{SellentinHeavens2015, FriedrichEifler2017, Heavensetal2017}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Monte-Carlo covariance estimator with uncorrelated samples}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Introduction}

In what follows, we will use the notation $\mathcal{N}(\mu, \Sigma)$ for a uni- or multi-variate Gaussian of mean $\mu$ and (co)variance $\Sigma$.

Let us first consider a very simple example: we have a random variable $X \sim \mathcal{N}(\mu, \Sigma)$, and we want to estimate the mean $\mu$ with an estimator $\hat{\mu}$ based on $N$ realizations $X_i$ with $i=1, \cdots, N$.
Our choice by default is obviously the sample mean $\hat{\mu} =  N^{-1} \sum_i X_i$.
Unfortunately, the variance of this estimator is $\mathrm{Var}[\hat{\mu}] = N^{-1}  \sigma^2 $, which scales badly.

There is a standard trick in the field of statistics: take $X_i = Y_i + Z_i$ with  $Y_i$ and $Z_i$ two realizations of $\mathcal{N}(\mu, \Sigma)$ with a non-zero cross-correlation $\mathrm{Corr}[Y_i, Z_i] = \beta$.
In this case, we can show that  $\mathrm{Var}[\hat{\mu}] = N^{-1}  \sigma^2 (1+\beta) $.

From this trivial example we conclude that we can reduce the variance of our sample mean estimator by applying it to averaged anti-correlated draws instead of un-correlated draws.
Although this is fairly trivial for Gaussian random variables, this property is generally very powerful, and extends to other distributions and estimators.
In this document we will apply this type of trick to the problem of estimating 2-pt and 4-pt correlations in cosmology.
In particular, we will apply this property at various levels, when $X$ will sometimes be the density field, its power spectrum, or its covariance.
We will investigate the effect of introducing correlations and how they reduce the variance of sample estimators.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Generalities}

Let's consider $\delta_k = \delta(k)$ a complex-valued Gaussian random field with zero mean and diagonal covariance kernel $P_{k}$, also called the power spectrum. In other words,
\equ{
	\mathrm{Real}[\delta_k] \sim \mathcal{N}(0, P_k/2) \quad \quad \mathrm{Imag}[\delta_k] \sim \mathcal{N}(0, P_k/2) \label{eq:p_delta}
}

Note that the modulus $|\delta_k|^2 = \delta_k \delta_k^*$ follows a zero-mean Gaussian distribution of variance $P_k$.

The $k$ index/variable will implicitly refer to the $k$th Fourier mode, since we want to identify $\delta$ to the (initial) cosmological matter density field. Despite having cosmology in mind, we remain fairly general in most of this discussion.

By construction, we set $\mathrm{Cov}[\delta_k, \delta_{k'}] = 0$ for $k\neq k'$. We also introduce a convenient reduced complex-valued random variable $x_k = \frac{\delta_k}{\sqrt{P_k/2}}$, the real and imaginary parts of which follow standard Gaussian distributions $\mathcal{N}(0, 1)$.

Let us write down Monte Carlo estimators of the power spectrum and its covariance, i.e., estimators based on random draws of $\delta$ (the random draws are assumed to be in our control, not from an existing or predefined set). The simplest estimator will involve $N_k$ independent draws of the $k$th mode, $\delta_{k,p}$ with $p=1, \cdots, N_k$ from \equref{eq:p_delta}.

The unbiased estimator for the power spectrum (assuming we know the mean is zero) is
\eqn{
	\hat{P}_{k} = \frac{1}{N_k} \sum_{p=1}^{N_k} |\delta_{k,p}|^2 =\frac{P_k}{N_k } \underbrace{\sum_{p=1}^{N_k} |x_{k,p}|^2 }_{\hat{\sigma}_k}
}
Since the $x_k$'s are standard normal random variables, $\hat{\sigma}_k$ follows a chi-squared distribution with $N_k$ degrees of freedom, $\hat{\sigma}_k \sim \chi^2(N_k)$.
Thus, $\hat{P}_{k} \sim \frac{P_k}{N_k }  \chi^2(N_k)$, with the mean value being $P_k$, demonstrating that the previous estimator is unbiased.

We are interested in the covariance of this estimator:
\eqn{
	C_{kk'} = \mathrm{Cov}[\hat{P}_{k}, \hat{P}_{k'}]
}
In our simple setup, we know that the $k$ modes are uncorrelated, so we have
\eqn{
	C_{kk'} = \frac{2{P}^2_{k}}{N_k} \delta^D(k-k')
}
where $ \delta^D$ is the Dirac delta. Yet, we want to estimate this covariance via Monte Carlo.
A concrete example is the estimation of the covariance of the power spectrum in a density field evolved from Gaussian initial conditions, which is a fairly standard setup in cosmology.
Hence, we introduce another set of draws: we repeat our experiment (drawing $N_k$ modes) $N$ times. The estimator from the $i$th draw is denoted by $\hat{P}_{k, i}$.
At this stage, we assume they are uncorrelated, \ie $\mathrm{Cov}[\hat{P}_{k, i}\hat{P}_{k, j}] = 0$ if $i\neq j$.

In the case of galaxy survey analyses in cosmology, we would generate $N$ random Gaussian initial conditions and evolve them in an $N$-body fashion. Each 3D Fourier mode is estimated from $N_k$ modes. In what follows we will assume that $N_k=1$ for simplicity.

We turn our attention to the sample covariance estimator
\eqn{
	\hat{C}_{kk'} = \frac{1}{\nu}  \sum_{i=1}^{N} (\hat{P}_{k, i} - \bar{P}_{k})(\hat{P}_{k', i} - \bar{P}_{k'})
}
We can either assume that the mean is know and set $\bar{P}_{k}=P_k$, in which case $\nu=N$, or directly estimate it with
\eqn{
	\bar{P}_{k} = \frac{1}{N}  \sum_{i=1}^{N} \hat{P}_{k, i}
}
in which case we must set $\nu=N-1$. In what follows, we will consider the latter.

We also know that if each $\hat{P}_{k, i}$ were Gaussian, $\hat{C}_{kk'} (N-1)$ would follow a Wishard distribution. It is unbiased, so the mean is the true covariance $C_{kk'}$, and its variance is
\eqn{
	\mathrm{Cov}[\hat{C}_{ij} \hat{C}_{kl}] = \frac{C_{ij}C_{kl} + C_{ik}C_{jl}}{N-1}
}
This assumption is pretty accurate as soon as $N_k>50$, because the chi-squared distribution is then very well approximated by a Gaussian (but still assuming that $\delta$ is Gaussian!).
We will make that assumption for the derivations below.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Three problems}

First, we see that the scaling of the standard MC covariance estimator is $(N-1)^{-1}$.
This is a fairly standard result in statistics, and there are numerous strategies to decrease this scaling.
We will go through some of those below.

Second, we see that our covariance of interest is exceedingly simple in the case of a Gaussian random field. We don't really need the MC estimator.
In the case of the cosmological matter power spectrum or its covariance, there are a number of theoretical models that are satisfactory over some range of scales and redshifts.
We absolutely want to make use of that and focus on estimating the residuals (unmodelled) part with Monte-Carlo, since this significantly reduces the number of simulations required.
In other words, let's not use large number of simulations just to estimate the components that we already know how to model analytically.
Again, there are standard techniques for including incomplete theoretical models in MC estimators, and we will explore those.

Third, there are various other considerations that we may need to consider in our estimators. A good example is the effect of box size and resolution on N-body simulations and power spectrum or covariance estimators.
We will briefly discuss those issues below. Standard techniques exist to deal with them, and they don't affect our main results.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Monte-Carlo covariance estimator correlated samples}

Let us relax one of the previous assumptions and introduce correlated draws, \ie
\eqn{
	\mathrm{Cov}[\hat{P}_{k, i}\hat{P}_{k', j}] \ = \ \mathbb{E}[\hat{P}_{k, i}\hat{P}_{k', j}] - \mathbb{E}[\hat{P}_{k, i}]\mathbb{E}[\hat{P}_{k', j}]  \ =  \ C_{kk'} \left( \delta^K_{ij}(1-\beta_{ij}) + \beta_{ij} \right)
}
where $\delta^K$ is Kronecker's delta. This slightly complicated expression is to recover $C_{kk'}$ at $i=j$ and $\beta_{ij}C_{kk'}$ at $i\neq j$. In other words, the correlation coefficient is simply $\beta_{ij}$. The only condition we must impose is that the full correlation matrix (constructed with the $\beta_{ij}$s) is symmetric positive definite. In particular, we note that if we take $\beta_{ij}=\beta$, we must have $\beta > -1/(N-1)$. This condition will be obvious when we study the estimators below.

We will now compute the covariance of the sample estimator, on the diagonal only (we only consider $\mathrm{Cov}[\hat{C}_{kk'}\hat{C}_{lm}]$ for $l=k, k'=m$):
\eqn{
	\mathrm{Var}[\hat{C}_{kk'}] \ =\    \frac{1}{(N-1)^2}\sum_{i=1}^{N}\sum_{j=1}^{N} \mathrm{Cov}[\hat{C}_{kk',i} , \hat{C}_{kk',j} ]
}
After a rather long derivation presented in \appref{sec:samplecovariancecorrelateddraws}, we obtain
\eqn{
	&&  \ \mathrm{Var}[\hat{C}_{kk'}] \ =    \bigl(C_{kk}C_{k'k'} + C_{kk'}^2 \bigr) \times s_N
}
with the scaling term
\eqn{
	s_N = \frac{ \sum_{ij} B_{ij}^2  - \frac{(\sum_{ij} B_{ij})^2}{N^2}}{(N-1)^2}
}
	 with $B_{ij} =  \delta^K_{ij}(1-\beta_{ij}) + \beta_{ij} $ and $\beta_{ij} \in [-1, 1]$ the correlation coefficient between the $i$th and the $j$th samples $\hat{P}_{k, i}$ and $\hat{P}_{k, j}$.

Using the same ingredients of \appref{sec:samplecovariancecorrelateddraws}, we can compute the expectation of the estimator,

\eqn{
	\mathbb{E}(\hat{C}_{kk'}) = C_{kk'} b_N
}
with the bias
\eqn{
	b_N = \frac{\sum_i B_{ii} - \frac{B}{N}}{N-1}
}

If $b_N\neq 1$, the estimator is biased, so we must apply a correction factor to $\hat{C}_{kk'}$.
The variance of the de-biased estimator is
\eqn{
	 \mathrm{Var}\left[ \frac{\hat{C}_{kk'}}{b_N}\right]  = \bigl(C_{kk}C_{k'k'} + C_{kk'}^2 \bigr) \times s_N^\prime  \label{ref:samplecovariancevariance}
}
with
\eqn{
	s_N^\prime  = \frac{s_N}{b^2_N} =  \frac{ \sum_{ij} B_{ij}^2  - \frac{B^2}{N^2}}{ \left(\sum_i B_{ii} - \frac{B}{N}\right)^2 }
	}

Finally, we check that the sample mean estimator is unbiased, $\mathbb{E}(\bar{P}_k) = P_k$, and that its covariance is
\eqn{
	\mathrm{Cov}[\bar{P}_k, \bar{P}_{k'}] = C_{kk'} \underbrace{ \frac{B}{N^2} }_{v_N}
}

\begin{table}\centering
\begin{tabular}{|c|c|c|c|c|c|c|c|}
	\hline	Name	& $B_{ij}$	&	$B_{i}$	&	$B$	&	$v_N$	&	$s_N $ 	&	$b_N$	&	$s_N^\prime$	\\\hline
	Uncorrelated	&$ \delta^K_{ij}$		&	1	&	$1/N$	&	$N$	&	$1/(N-1)$	&	$1$	&	$1/(N-1)$	\\
	Correlated	&$\delta^K_{ij}(1-\beta) + \beta$		&	$1 + (N-1)\beta$	&	$N + (N^2-N)\beta$	&	$\frac{1 + (N-1)\beta}{N}$	&	$(1-\beta)^2/(N-1)$	&	$1-\beta$	&	$1/(N-1)$	\\
	Fully correlated	&$ 1$		&	$N$	&	$N^2$	&	$1$	&	$0$	&$0$		&	$0$	\\
	Fully anticorrelated	&$2 \delta^K_{ij} - 1$		&	$2-N$	&	$N(2-N)$	&	$(2-N)/N$	&	$4/(N-1)$	&	$2$	&	$1/(N-1)$	\\
	\hline
\end{tabular}
\caption{Summary of our results for the mean and variance of the sample mean and sample variance estimators. We define $B_{ij} =  \delta^K_{ij}(1-\beta_{ij}) + \beta_{ij}$, $B_i=\sum_{j=1}^NB_{ij}$, $B=\sum_{i=1}^NB_{i}$. As defined in the text, $v_N$ and $s_N^\prime$ characterize the scaling of the variance of the \textit{unbiased} sample mean and sample covariance estimators (the constant terms in those are $C_{kk'}$ and $C_{kk}C_{k'k'} + C_{kk'}^2$, respectively).
}
\label{table:correlateddrawbiasesandvariances}
\end{table}

Our findings are summarized in Table~\ref{table:correlateddrawbiasesandvariances}. \textbf{We conclude that that 1) we can easily reduce the variance of the sample mean by adopting correlated samples, 2) this unfortunately doesn't work for the sample covariance. Unlike for the sample mean, adjusting the correlations cannot cancel or even reduce the variance of the estimator.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Stratified sampling}

Let us now take a step back from the $P_k$ and $C_{kk'}$ notation, and introduce an important concept.
We will assume that we have a set of correlated samples $X_{ij}$.
We group them in packets of size $M$, and for each packet, we compute a statistic with a MC estimator $f$.
We then compute a final statistic with another MC estimator $g$.
\eqn{
	\hat{X}_i &=& f\bigl(\{X_{ij} | j=1, \cdots, N\}\bigr) \quad i=1, \cdots M \\
	\hat{X} &=& g\bigl(\{\hat{X}_i | i=1, \cdots M\}\bigr)
}
Having multiple layers of MC estimators is known as \textbf{stratified sampling}. It is a powerful variance reduction technique.
Let us study a few cases of interest, where we can simply use the results of the previous section on both $f$ and $g$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Stratified estimator for the mean}

First, let us consider both estimators to be the sample mean,
\equ{
	\hat{X}_i = \frac{1}{N}\sum_{j=1}^N X_{ij}  \quad\quad \hat{X} =  \frac{1}{M}\sum_{i=1}^M \hat{X}_i
	}
All of the samples have a mean $\mu$ and a variance $V$. It is pretty obvious that $\mathbb{E}[\hat{X}] = \mu$.
The samples within each of the packets $i=1, \cdots M$ are correlated with a correlation coefficient $\beta$.
There are no correlations between packets. Thus,
\eqn{
	\mathrm{Cov}[X_{ij}, X_{ml}] &=& \delta^K_{im} V \bigl(\delta^K_{jl}(1-\beta) + \beta\bigr) \
}
Using the expressions derived above, we find
\eqn{
	% \mathrm{Var}[\hat{X}_i] &=& V \times \frac{1 + (N-1)\beta}{N}\\
	\mathbb{E}[\hat{X}] &=& \mu\quad\quad
	\mathrm{Var}[\hat{X}] =  V \times \frac{1 + (N-1)\beta}{N M}
}

We see that the variance is cancelled if
\eqn{
	\beta^\mathrm{VC} \rightarrow^{>} - \ \frac{1}{N-1}
}
with VC as in Variance Cancellation. The arrow means that we cannot actually take $- \ \frac{1}{N-1}$ because the correlation matrix wouldn't be semi-positive definite, hence valid. However, we can approach this value as much as we want.

The specific case of $N=2$ is known as the \textbf{antithetic estimator for the sample mean}. In this case, the optimal choice for cancelling the variance is $\beta^\mathrm{RC}=-1$, \ie perfectly anti-correlated variables.

Conversely, imagining the correlation $\beta$ and the number of simulations $MN$ are fixed, one can infer $N$ to obtain the minimum-variance stratified estimator.

%Note that in the case of Gaussian random variables, the Gaussian covariance matrix is singular when $\beta \pm 1$, and perfectly anti-correlated samples are obtained by flipping the sign of the variable. There is only one problem: in this case, we cancel out the variable because there is indeed no variability whatsoever in the sampling. So one may  want to set $\beta$ close but not exactly $\pm1$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Two-layer stratified estimators for the covariance}

There are two estimators for the covariance: taking $f$ as the sample covariance and $g$ as the sample mean estimator, or conversely. We will go through both. Again, we only consider correlations within (not between) packets:
\eqn{
	\mathrm{Cov}[X_{ij}, X_{ml}] &=& \delta^K_{im} V \bigl(\delta^K_{jl}(1-\beta) + \beta\bigr) \
}
The first stratified estimator is
\eqn{
	\hat{X}_i &=& \frac{1}{N-1}\sum_{j=1}^N \frac{(X_{ij} - \bar{X}_{i})^2}{1-\beta}	\quad\quad \bar{X}_{i}	=  \frac{1}{N}\sum_{j=1}^N X_{ij}	\quad\quad \hat{X} =  \frac{1}{M}\sum_{i=1}^M \hat{X}_i
}
and its mean and the variance are
\eqn{
	\mathbb{E}[\hat{X}] 	&=&	V 	\quad \quad\mathrm{Var}[\hat{X}] = \frac{2V^2}{M(N-1)}
}
The extra $ (1-\beta)$ in the definition of $\hat{X}_i$ was to make the estimator unbiased.
The second estimator reads
\eqn{
	\hat{X}_i &=& \frac{1}{N}\sum_{j=1}^N X_{ij}	\quad\quad
	\hat{X} =   \frac{N}{(1 + (N-1)\beta)} \frac{1}{M-1}\sum_{i=1}^M (\hat{X}_i  - \bar{X})^2 \quad\quad
	\bar{X} =  \frac{1}{M}\sum_{i=1}^M \hat{X}_i
}

and its mean and the variance are
\eqn{
	\mathbb{E}[\hat{X}] &=& V \quad\quad
	 \mathrm{Var}[\hat{X}] = \frac{2V^2}{(M-1)N}
}
Again, the extra $\beta$ term in the definitions is to make the variance estimator unbiased.

Because of the terms at the denominator of the scaling of those two covariance estimators ($(N-1)M $ and $(M-1)N$) and the absence of a $\beta$ term, it is actually. better to \textbf{not} bin them into groups (straightforward to prove), and use the standard (unstratified) sample variance estimator.

\textbf{In conclusion, we can use correlations to reduce the variance of the sample mean, but not that of the variance. }

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Connection to cosmology}

Reconnecting to the previous section, should we identify $X$ to $\delta_{k, i}$, or to $\hat{P}_{k,i}$? Again, we assume that we will do $i=1, \cdots, N$ draws, potentially grouped into packets and correlated.

The setup is fairly clear: we want to generate Gaussian initial conditions for $\delta$, then evolve them via a N-body code, and estimate the power spectrum and the covariance matrix of the evolved field.

If we identify $X$ to $\delta_{k, i}$, then we need to add another layer of estimation on top of our formalism to get the covariance matrix.
If we identify $X$ to $\hat{P}_{k,i}$, we immediately face the question of controlling the correlation properties of $\hat{P}_{k,i}$ given that we generate $\delta_{k, i}$.

We should immediately note that the latter case might be easily achievable: if we generate anti-correlated density fields, any residual anti-correlation in the power spectrum will decrease the variance of the stratified sample estimators described above. So the real question is quantitative: how can we generate anti-correlated power spectra when drawing density fields ?

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Can we generate anti-correlated power spectra from (anti-)correlated density? No.}

We consider correlated Gaussian draws of the density field, such that
\eqn{
	\mathrm{Cov}\bigl[\mathrm{Real}[\delta_{k, p, i}], \mathrm{Real}[\delta_{k', q, j}]\bigr] &=& \delta^D(k-k') \ \delta^K_{pq}\  \frac{P_k}{2}\ \underbrace{( \delta^K_{ij}(1+\beta_{ij}) + \beta_{ij})}_{B_{ij}}	\\
	\mathrm{Cov}\bigl[\mathrm{Imag}[\delta_{k, p, i}], \mathrm{Imag}[\delta_{k', q, j}] \bigr] &=& \delta^D(k-k') \ \delta^K_{pq}\  \frac{P_k}{2}\ \underbrace{( \delta^K_{ij}(1+\gamma_{ij}) + \gamma_{ij})}_{G_{ij}}\\
	\mathrm{Cov}\bigl[\mathrm{Real}[\delta_{k, p, i}], \mathrm{Imag}[\delta_{k', q, j}] \bigr] &=& \delta^D(k-k') \ \delta^K_{pq}\  \frac{P_k}{2}\ \underbrace{( \delta^K_{ij}(1+h_{ij}) + h_{ij})}_{H_{ij}}
}
\ie without mixing the $k$ or $p-q$ modes.
Since the mean value is zero, we have $\mathrm{Cov}[...] = \mathbb{E}[...]$ for all expression involving pairs of densities.
Like before, we will use the notation $B_{ij}$ (and $G_{ij}$ and $H_{ij}$) for the elements of the correlation matrices, which must be symmetric semi-positive definite.

\eqn{
	\mathrm{Cov}[\hat{P}_{k, i}, \hat{P}_{k', j}] &=& \frac{1}{N_k^2} \sum_{p=1}^{N_k}\sum_{q=1}^{N_k} \mathrm{Cov}\Bigl[ \delta_{k, p, i} \delta_{k, p, i}^*, \delta_{k', q, j}\delta_{k', q, j}^* \Bigr] \\
	&=&  \frac{1}{N_k^2} \sum_{p=1}^{N_k}\sum_{q=1}^{N_k}   \mathbb{E}[\delta_{k, p, i} \delta_{k', q, j}]\mathbb{E}[ \delta_{k, p, i}^* \delta_{k', q, j}^*] + \mathbb{E}[\delta_{k, p, i} \delta_{k', q, j}^*]\mathbb{E}[ \delta_{k, p, i}^* \delta_{k', q, j}]
}
where we have used Wick's theorem, \equref{eq:wickstheorem}. Unfortunately, to simply this expression we must split the real and imaginary part s of the complex numbers. Luckily, many of the cross-terms vanish since all the real and imaginary parts are independent.
\eqn{
	\mathrm{Cov}[\hat{P}_{k, i}, \hat{P}_{k', j}] &=&  \frac{1}{N_k^2} \sum_{p=1}^{N_k}\sum_{q=1}^{N_k}\\
	&&
	\mathbb{E}[\mathrm{Real}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] +i\mathrm{Imag}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] +i\mathrm{Imag}[\delta_{k',q,j}] \mathrm{Real}[\delta_{k, p, i}]  - \mathrm{Imag}[\delta_{k, p, i}] \mathrm{Imag}[\delta_{k',q,j}] ] \nonumber\\
	&& \times\mathbb{E}[ \mathrm{Real}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] -i\mathrm{Imag}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] -i\mathrm{Imag}[\delta_{k',q,j}] \mathrm{Real}[\delta_{k, p, i}] -\mathrm{Imag}[\delta_{k, p, i}] \mathrm{Imag}[\delta_{k',q,j}] ]\nonumber \\
	&&+ \  \mathbb{E}[\mathrm{Real}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] +i\mathrm{Imag}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] -i\mathrm{Real}[\delta_{k, p, i}] \mathrm{Imag}[\delta_{k',q,j}] +\mathrm{Imag}[\delta_{k, p, i}] \mathrm{Imag}[\delta_{k',q,j}] ] \nonumber\\
	&& \times \mathbb{E}[ \mathrm{Real}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] -i\mathrm{Imag}[\delta_{k, p, i}] \mathrm{Real}[\delta_{k',q,j}] +i\mathrm{Imag}[\delta_{k',q,j}]  \mathrm{Real}[\delta_{k, p, i}]+\mathrm{Imag}[\delta_{k, p, i}] \mathrm{Imag}[\delta_{k',q,j}] ] \nonumber \\
	%&=& \delta^D(k-k')   \frac{P^2_k}{4} \frac{1}{N_k} \left[ \left(B_{ij} + 2iH_{ij} - G_{ij}\right)  \left(B_{ij} - 2iH_{ij} - G_{ij}\right) + \left(B_{ij} + G_{ij}\right)  \left(B_{ij} + G_{ij}\right) \right]	\\
	%&=& \delta^D(k-k')   \frac{P^2_k}{4} \frac{1}{N_k} \left[  (B_{ij} - G_{ij})^2 + 4H^2_{ij} + \left(B_{ij} + G_{ij}\right)^2 \right] \\
	&=& \delta^D(k-k')   P^2_k \frac{1}{N_k} \left[  \frac{B^2_{ij} + G^2_{ij}}{2}  + H^2_{ij} \right]
}

%\eqn{
%	&&\mathbb{E}[(a_1+ib_1)(a_3+ib_3)]\mathbb{E}[ (a_1-ib_1)(a_3-ib_3)] + \mathbb{E}[(a_1+ib_1)(a_3-ib_3)]\mathbb{E}[ (a_1-ib_1)(a_3+ib_3)] 	\\
%	&&\quad = \ \mathbb{E}[a_1a_3+ib_1a_3+ib_3a_1 - b_1b_3]\ \mathbb{E}[ a_1a_3-ib_1a_3-ib_3a_1-b_1b_3] \\
%	&&\quad \quad + \  \mathbb{E}[a_1a_3+ib_1a_3-ia_1b_3+b_1b_3]\ \mathbb{E}[ a_1a_3-ib_1a_3+ib_3a_2+b_1b_3]
%}

The correlation coefficient is $\left[  \frac{B^2_{ij} + G^2_{ij}}{2}  + H^2_{ij}\right]$ and is strictly positive regardless of the correlations we introduce between $\delta_{k, p, i}$ and $\delta_{k', q, j}$.
\textbf{We conclude that unfortunately we cannot generate anti-correlated power spectra from correlated density field draws. We could have seen this from our previous findings too.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Drawing the density field from anti-correlated power spectra.}

Let us explore another strategy: rather than drawing the density field from the theory power spectrum, let's generate the density field from anti-correlated draws of the theory power spectrum.
In other words, we now construct power spectra such that $Q_{k, i}$ for $i=1, \cdots, N$ from the mean $P_{k}$ and with the covariance structure $\mathrm{Cov}[{Q}_{k, i}, {Q}_{k', j}] \ = \ C_{kk'} \left( \delta^K_{ij}(1-\beta_{ij}) + \beta_{ij} \right)$
We use $Q$ instead of $P$ or even $\hat{P}$ to emphasize the fundamental difference of the scheme proposed here.

We now draw the corresponding density field $\delta_{k, p, i}$ from $Q_{k, i}$. Importantly, we want it to be a Gaussian random field of mean $P_{k}$, in order to have Gaussian initial conditions for an N-body simulation.
Since we have drawn $Q_{k, i}$ already, we cannot construct $\delta_{k, p, i}$ from a naive Gaussian draw.
Luckily, as described in \cite{AnguloPontzen2016} (see also \citealt{Schmittfull2015, PradaEtAl2016, DingEtAl2017}), this can be achieved by setting
$\delta_{k, p, i} = \sqrt{Q_{k, i}} \exp(i\theta_{k, p, i})$
where the complex phase $\theta_{k, p, i}$ is drawn uniformly between $0$ and $2\pi$.
Again, we perform $N_k$ independent random draws for the $k$-th mode in the $i$-th simulation.

The resulting density fields $\delta_{k, p, i} $ are indeed Gaussian random fields in $P_k$, but constructed with anti-correlated power spectra $Q_{k, i}$.
Concretely, the probability distribution of the modulus is no longer
\eqn{
	p\bigl(|\delta_{k, p, i}|, \theta_{k, p, i}\bigr) = \frac{|\delta_{k, p, i}|}{2\pi P_{k, i}} \exp\left( -\frac{|\delta_{k, p, i}|^2}{P_{k, i}}\right)
}
it is now
\eqn{
	p\bigl(|\delta_{k, p, i}|, \theta_{k, p, i}\bigr) = \frac{1}{2\pi} \delta^D\left(|\delta_{k, p, i}| - \sqrt{Q_{k, i}}\right)
}
\todo{How well does this procedure work for Gaussian random fields?}

We can check that the measured power spectra are indeed anticorrelated:
\eqn{
	\mathrm{Cov}[\hat{P}_{k, i}, \hat{P}_{k', j}] \ = \ \frac{1}{N_k^2} \sum_{p=1}^{N_k}\sum_{q=1}^{N_k} \mathrm{Cov}\Bigl[ |\delta_{k, p, i}|^2, |\delta_{k', q, j}|^2 \Bigr]
	\ =\  \frac{1}{N_k^2} \sum_{p=1}^{N_k}\sum_{q=1}^{N_k}   \mathrm{Cov}[{Q}_{k, i}, {Q}_{k', j}]
	\ = \ C_{kk'} \left( \delta^K_{ij}(1-\beta_{ij}) + \beta_{ij} \right)
}
which is the desired property.

\textbf{We conclude that by using this trick we can draw density fields that have anti-correlated power spectra, and therefore use the previous stratified estimators.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Non-Gaussian random variables}

So far we have assumed that most variables are Gaussian distributed. In practice, there are two sources of non-Gaussianity: in $P_k$ since it is calculated from $\delta$, and in $\delta$ itself.

We have assumed that the estimated power spectra $\hat{P}_{k, i}$ are Gaussian distributed. This is not true even if $\delta_k$ is a Gaussian random field: we showed that $\hat{P}_{k, i}$ follows a chi-squared distribution. As a consequence, our application of Wick's theorem at \equref{eq:wickstheorem} does not hold. However, we expect it to be relatively accurate given the small deviations between the variances of the chi-squared and Gaussian distributions. For most observables it will be the case: we usually average out enough modes in each bins so that the statistical uncertainties are close to Gaussian.

The real non-Gaussianity is in the \textit{evolved} density field itself, and the impossibility to model the covariance matrix $C_{kk'}$ accurately (which is why we needed simulations in the first place).  Schematically, we can write
\eqn{
	\delta^\mathrm{ini}_k \ &\longrightarrow& \ \delta^\mathrm{fin}_k	\nonumber\\
	P^\mathrm{ini}_k \ &\longrightarrow& \ P^\mathrm{fin}_k	\nonumber\\
	C_{kk'}^\mathrm{ini}  \ &\longrightarrow& \ C_{kk'}^\mathrm{fin} \nonumber\\
	\mathrm{Corr}[\hat{P}^\mathrm{ini}_{k, i}, \hat{P}^\mathrm{ini}_{k', j}]  \ &\longrightarrow& \ \mathrm{Corr}[\hat{P}^\mathrm{fin}_{k, i}, \hat{P}^\mathrm{fin}_{k', j}]   \nonumber
}
where the arrows denote cosmological evolution, for example evolving the initial density field in an N-body fashion.
We are truly interested in $P^\mathrm{fin}_k$ and $C_{kk'}^\mathrm{fin}$.
Even if we generate anti-correlated initial power spectra and the corresponding density fields, there is no guarantee that those anti-correlations persist in the evolved field, especially in $\mathrm{Corr}[\hat{P}^\mathrm{fin}_{k, i}, \hat{P}^\mathrm{fin}_{k', j}]$.
Yet, those are essential to achieve variance reduction.
Luckily, we can intuitively guess that most of the anti-correlation should be reduced by not totally cancelled, especially on large scales, which is where we care about variance cancellation the most due to the fewer number of modes.
\todo{Can we write down first order analytic predictions for $\mathrm{Corr}[\hat{P}^\mathrm{fin}_{k, i}, \hat{P}^\mathrm{fin}_{k', j}] / \mathrm{Corr}[\hat{P}^\mathrm{ini}_{k, i}, \hat{P}^\mathrm{ini}_{k', j}]$ ?}
\todo{Run simulations and make experiments to quantify this!}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Including theoretical predictions in the estimators}

We are interested in $P^\mathrm{fin}_k$ and $C_{kk'}^\mathrm{fin}$ but only have partial models, that are accurate only in certain limited regimes, such as large scales, high redshift, etc.
Can we make use of those in the previous MC estimators?

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Setup and naive approach}

In a first attempt, let's suppose we have models for $P^\mathrm{fin}_k$ and $C_{kk'}^\mathrm{fin}$, denoted by $\tilde{P}^\mathrm{fin}_k$ and $\tilde{C}_{kk'}^\mathrm{fin}$, that are imperfect.
We now seek to estimate the deviations from those models to the target, true power spectrum and its covariance matrix:
\eqn{
	{D}_k &=&P^\mathrm{fin}_k - \tilde{P}^\mathrm{fin}_k		 \quad \quad\quad
	\Delta_{kk'} = C_{kk'}^\mathrm{fin}  - \tilde{C}_{kk'}^\mathrm{fin}
}
Let us write down the classic sample estimators first:
\eqn{
	\bar{D}_k &=&	\frac{1}{N} \sum_{i} \hat{D}_{k, i}	 \quad \quad\quad
	\hat{\Delta}_{kk'} = \frac{1}{N-1} \sum_{i} \left( \hat{D}_{k, i} - \bar{D}_{k} \right) \left( \hat{D}_{k', i} - \bar{D}_{k'} \right)
}
with $\hat{D}_{k, i} = \hat{P}^\mathrm{fin}_{k, i} - \tilde{P}^\mathrm{fin}_k$.

Let's remember that \textit{in the Gaussian case}, the constant term in all of our sample mean estimators was $C_{kk'}$, and the constant in the sample covariance estimators was ${C}_{kk}{C}_{k'k'} + {C}_{kk'}^2$.

So we immediately conclude that if the power spectrum estimates $P^\mathrm{fin}_{k, i}$ are Gaussian distributed (potentially correlated), we have
\eqn{
	\mathrm{Cov}[\bar{D}_k, \bar{D}_{k'}] = \Delta_{kk'} v_N \quad \quad\quad
	 \mathrm{Var}\left[ \frac{\hat{\Delta}_{kk'}}{b_N}\right]  = \bigl(\Delta_{kk}\Delta_{k'k'} + \Delta_{kk'}^2 \bigr) s_N^\prime
}
analogous to \equref{ref:samplecovariancevariance}. Again, the $b_N$ term is to debias the estimator in the case of correlated power spectra.

The scalings $v_N$ and $s_N^\prime$ have not changed - only the constant terms did. The variances of the estimators now depend on the deviations from the model.
If those are small, then we have indeed improved!
Needless to say we can use the previous stratified estimators in this case too.

\textbf{In conclusion, including theoretical predictions when MC estimating the power spectra and covariances reduces the variances of the estimators, but not their scaling with the number of simulations.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Control variates}

What if we wanted to further decrease the scaling of the estimators by including theoretical predictions?
This can be achieved with \textbf{control variates}.
The idea is again to exploit (anti-)correlation, but this time not between our samples (previously the power spectra), but with \textit{auxiliary variables}.
Specifically, we want the auxiliary variables to have a known target statistic.

In the context of cosmological N-body simulations, an obvious choice is to run an auxiliary set of simulations where we can actually accurately model the power spectra and covariance matrices.
For example, we could run LPT simulations.

Like before, we will look at differences between the estimated power spectra of our full complicated simulation $\hat{P}^\mathrm{fin}_{k, i} $ and those of our simplified simulation, denoted $\hat{S}^\mathrm{fin}_{k, i}$.
The significant difference between those is that  we do have exact (or very accurate models) of the power spectrum and covariance matrix for the latter, which we write $\tilde{P}^\mathrm{fin}_k$ and $\tilde{C}_{kk'}^\mathrm{fin}$.

We will not discuss how the auxiliary simulations are constructed now - let us first write the sample estimators
\eqn{
	\hat{D}_{k, i} &=&  \hat{P}_{k, i} + \alpha (\hat{S}_{k, i} - \tilde{P}^\mathrm{fin}_k)  \quad \quad\quad
	\bar{D}_k =	\frac{1}{N} \sum_{i=1}^N	\hat{D}_{k, i}	\quad \quad\quad
	\hat{\Delta}_{kk'} = \frac{1}{N-1} \sum_{i=1}^N \left( \hat{D}_{k, i} - \bar{D}_{k} \right) \left( \hat{D}_{k', i} - \bar{D}_{k'} \right)
}
We have
\eqn{
	\mathbb{E}[\hat{D}_{k, i}] &=& {P}^\mathrm{fin}_k 	\\
	\mathrm{Covar}[\hat{D}_{k, i}, \hat{D}_{k', j}] &=& \underbrace{\mathrm{Covar}[\hat{P}_{k, i}, \hat{P}_{k', j}]}_{C_{kk'} B_{ij}}  + \alpha^2 \underbrace{\mathrm{Covar}[\hat{S}_{k, i}, \hat{S}_{k', j}] }_{\tilde{C}_{kk'}^\mathrm{fin} G_{ij}} + \alpha \underbrace{\mathrm{Covar}[\hat{P}_{k, i}, \hat{S}_{k', j}]}_{ R_{kk'} S_{ij}} + \alpha \underbrace{\mathrm{Covar}[\hat{S}_{k, i}, \hat{P}_{k', j}]}_{ R_{kk'} S_{ij}}
}
where we have defined the relevant covariances as before. In particular, the auxiliary simulations are correlated with $G_{ij}$ (subject to the same constraints as $B_{ij}$ as before), and $ R_{kk'} S_{ij}$ characterizes the covariance between the main and the auxiliary simulations.
Those might need to be estimated from the simulations themselves, but may be partially estimated analytically.
We will discuss that below.

The mean and variance of the sample mean (power spectrum) estimator are
\eqn{
	\mathbb{E}[\bar{D}_{k}] &=& {P}^\mathrm{fin}_k 	\\
	\mathrm{Covar}[\bar{D}_{k}, \bar{D}_{k'}] &=& \frac{1}{N^2}\sum_{i=1}^N\sum_{j=1}^N \left( C_{kk'} B_{ij} + \alpha^2  \tilde{C}_{kk'}^\mathrm{fin} G_{ij} + 2 \alpha  R_{kk'} S_{ij}  \right)
}
By differentiating, we find that the value minimizing the variance is
\eqn{
	{\alpha^\mathrm{opt}_{kk'}} = \ - \ \frac{ \sum_{i=1}^N\sum_{j=1}^N  R_{kk'} S_{ij} }{ \sum_{i=1}^N\sum_{j=1}^N \tilde{C}_{kk'}^\mathrm{fin} G_{ij} }
}
Of course those covariances/correlations might not be known and may need to be estimated directly from the data, especially because $R_{kk'} S_{ij}$ is a covariance and depends on the unknown target	 covariance ${C}_{kk'}^\mathrm{fin}$.
\todo{Discuss how to estimate unbiased $ R_{kk'} S_{ij}$ from data.}

Let's derive the mean and variance of the sample covariance, using a short-cut notation $F_{kk',ij} = \mathrm{Covar}[\hat{D}_{k, i}, \hat{D}_{k', j}]$,
\eqn{
	\mathbb{E}[\hat{\Delta}_{kk'}] &=&  \frac{ \sum_{i}F_{kk',ii} - \frac{\sum_{ij} F_{kk',ij}}{N} } {N-1}	 \\
	 \mathrm{Var}\left[ \hat{\Delta}_{kk'}\right]  &=&	\frac{\sum_{ij} \bigl(F_{kk,ij}F_{k'k',ij} + F_{kk',ij}^2 \bigr) - \frac{1}{N^2}\bigl( \sum_{ij}F_{kk,ij}F_{k'k',ij}\bigr) - \frac{1}{N^2}\bigl(  F^2_{kk',ij} \bigr)^2 }{(N-1)^2}
	}
	\todo{Double check!!}
	Note that this estimator, in this current form, is biased since  $F_{kk',ij}$ contains the target unknown covariance ${C}_{kk'}^\mathrm{fin}$ and also the known auxiliary covariance $\tilde{C}_{kk'}^\mathrm{fin}$.

It would be good to understand this estimator, even in a simpler setting. For this purpose, we assume that the known covariance is close to the true one $\tilde{C}_{kk'} \approx {C}_{kk'}$, and that the cross-correlations with the auxiliary simulations are such that $R_{kk',ij} \approx {C}_{kk'} \rho_{ij}$.
Then, we can write $F_{kk',ij} ={C}_{kk'}  \mathcal{B}_{ij}$ with $\mathcal{B}_{ij} = B_{ij} + 2 \alpha \rho_{ij} + \alpha^2 G_{ij}$.
With those assumptions, we find
\eqn{
	 \mathrm{Var}\left[ \frac{\hat{\Delta}_{kk'}}{\mathbb{E}[\hat{\Delta}_{kk'}] /{C}_{kk'} }\right]  &=&  \frac{ \sum_{ij} \mathcal{B}_{ij}^2  - \frac{\left(\sum_{ij}\mathcal{B}_{ij}\right)^2}{N^2}}{ \left(\sum_i \mathcal{B}_{ii} - \frac{\sum_{ij}\mathcal{B}_{ij}}{N}\right)^2 }
	}
where I haven't written out the bias explicitely for conciseness.
We see that the scaling takes unsurprisingly the same form as in our original investigations. However, there is a major difference: the coefficients $\mathcal{B}_{ij}$ depend on $\alpha$, which we can adjust to minimize the variance just like we did for the sample mean a few line above.
By differentiating, we find that the value(s) minimizing the variance, ${\alpha^\mathrm{opt}_{kk'}}$ is (are) the zero(s) of a third order polynomial
\eqn{
	 \alpha^3  \times 2 \left(  \sum_{ij} G_{ij}^2 - \frac{(\sum_{ij} G_{ij})^2}{N}\right) +
	  \alpha^2 \times 6 \left(  \sum_{ij} G_{ij} \rho_{ij} - \frac{\sum_{ij} G_{ij} \sum_{ij} \rho_{ij}}{N} \right) + \\
	  \alpha^1 \times 2 \left(  \sum_{ij} G_{ij} B_{ij} - \frac{\sum_{ij} G_{ij} \sum_{ij} B_{ij}}{N}  + 2 \sum_{ij} \rho^2_{ij} - 2\frac{(\sum_{ij}  \rho_{ij})^2}{N}  \right) +
	    2 \left(  \sum_{ij} B_{ij} \rho_{ij} - \frac{\sum_{ij} B_{ij} \sum_{ij} \rho_{ij}}{N}  \right)
}
Note that this is again for each pair of mode $kk'$.
This can be simplified further by adopting $G_{ij}=B_{ij}$ or $G_{ij} = 0$, but it is probably best to study this system numerically.

Let's understand another version of this estimator, more connected to what we are probably going to run in practice. Let's assume the only correlation is between pairs of original-auxiliary simulations, not between original-original or auxiliary-auxiliary.  So we take $B_{ij} = G_{ij} = \delta_{ij}$ and $R_{kk'} S_{ij} = R_{kk'}  \delta_{ij}$.
In this case $F_{kk',ij} = \delta_{ij}  \left( {C}_{kk'} + \alpha^2 \tilde{C}_{kk'}^\mathrm{fin}  + 2\alpha R_{kk'} \right) = F_{kk'}$.
\eqn{
	\mathbb{E}[\hat{\Delta}_{kk'}] &=&  F_{kk'} 	 \\
	 \mathrm{Var}\left[ \hat{\Delta}_{kk'}\right]  &=&  \frac{F_{kk}F_{k'k'} + F_{kk'}^2 }{N-1}	\\
	 	&=& \frac{1}{N-1} \Bigl[ \left( {C}_{kk}{C}_{k'k'} + {C}_{kk'}^2 \right)  + \alpha \left( 2R_{kk} {C}_{k'k'} + 2 R_{k'k'} C_{kk} + 4 {C}_{kk'}{R}_{kk'}\right) \\
		&&  +  \alpha^2 \left( \tilde{C}_{kk}^\mathrm{fin} {C}_{k'k'} +  \tilde{C}_{k'k'}^\mathrm{fin}{C}_{kk} + {R}_{kk'}^2 + 4{R}_{kk}R_{k'k'} + 2  \tilde{C}_{kk'}^\mathrm{fin}{C}_{kk'} \right)  \\
		&&  + \alpha^3 \left( 2R_{kk}\tilde{C}_{k'k'}^\mathrm{fin}  +R_{k'k'} \tilde{C}_{kk}^\mathrm{fin}  + 4R_{k'k'} \tilde{C}_{kk'}^\mathrm{fin}  \right)  + \alpha^4 \left( \tilde{C}_{kk}^\mathrm{fin}   \tilde{C}_{k'k'}^\mathrm{fin} + (\tilde{C}_{kk'}^\mathrm{fin})^2    \right)  \Bigr]
	}

The estimator is easily debiased by considering $\hat{\Delta}_{kk'} -   \alpha^2 \tilde{C}_{kk'}^\mathrm{fin}  - 2\alpha R_{kk'} $ which does not change its variance. Again, we cannot reduce the scaling of the estimator with $N$, but we can reduce the constant term. Compared to the theoretical prediction setup, we have  extra degrees of freedom: the choice of $\alpha$ for each $kk'$ pair. By solving the system $\partial  \mathrm{Var}[ \hat{\Delta}_{kk'}] /  \partial \alpha = 0$ numerically we could find the optimal $\alpha$ that reduces the variance of the
Again, we can only reduce the constant term of the variance of the sample covariance. However, there is a big difference: the reduction does not rely on our ability to model the unknown covariance, but rather to correlate simulations. It doesn't matter what the covariance is for the auxiliary simulations as long as we know it. The more we correlate the simulations the small will the variance of our the mean and covariance estimators be. This is not true for the previous section on theoretical predictions.

Of course we may not know $R_{kk'}$. Often we will have to estimate it from the simulations themselves, but of course not re-use the same set for the mean and covariance estimators of interest. This may require splitting the full set of simulations.
\todo{How to estimate that in practice?}.

\textbf{The main conclusion is that the variance of the sample covariance estimator can indeed be reduced by using control variates, especially an auxiliary set of simulations. However, this must be studied numerically.}

\todo{What is the actual reduction of the variance? What if $G_{ij}=B_{ij}$ or $G_{ij} = 0$? Can we actually study this system analytically, or should we just solve it numerically with real simulations? This would be reasonable given that all the correlation coefficients are pretty much unknown in reality.}

\todo{Compute product and ratio control variates for comparison.}
\todo{Discuss how to construct the auxiliary simulations with the right correlations}.
\todo{Discuss how to extend this to multiple simulations and control variates.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Summary}

We have introduced three concepts that can be used to decrease the variance of simulation-based estimates in general: stratified sampling, theory proxy, and control variates.
Let us briefly summarise them, for a random variable $X$.
\textbf{Stratified sampling} consists of performing correlated draws/simulations of $X$, and averaging them out in packets before (or somewhere in) the estimator of interest.
The \textbf{theory proxy} simply consists of using an approximate analytical solution for the observable of interest, and only using the simulations to estimate the difference with the truth.
The use of \textbf{control variates} consists of running an auxiliary set of simulations correlated with out main draws of $X$.
The three techniques exploit extra sources of information and can reduce the variance of any MC estimate.

\cite{PontzenEtAl2016} introduced the use of paired simulations, identifying $X$ to the initial density field, drawing two fully anti-correlated Gaussian initial conditions (thus simply flipping the sign), then evolving them before looking at the auto- and cross-power spectra.
They showed that the two evolved simulations are still significantly correlated, and that averaging their power spectra leads to a cancellation of some of the noise terms, which can be explained by perturbation theory to some extent.
In light of our findings, this is a case of stratified sampling, in fact antithetic estimators, where we exploit the anti-correlation that propagated from initial to evolved density field.
\cite{PontzenEtAl2016} just didn't run multiple pairs of simulations, but they did demonstrate the variance reduction with analytical arguments.
The true stratified sampling for the mean power spectrum would indeed rely on multiple groups of anti-correlated simulations. To obtain the maximum variance reduction, one should indeed run pairs of simulations with flipped signs.

\cite{AnguloPontzen2016} explored an extension: fixing the amplitude of the power spectrum to the theory, and only drawing the phases. Again, this leads to the cancellation of a few noise terms,  and it isn't surprising that this would further reduce the variance of the mean power spectrum.
What is more interesting is that this trick is exactly what we need to use to design a stratified estimator for the covariance matrix.
Indeed, we have seen that we cannot identify $X$ to the density field because anti-correlations in the density will not translate into variance reduction of the covariance matrix.
What we need to do is generate density fields that have anti-correlated power spectra. As discussed above, by using the trick of \cite{AnguloPontzen2016} we can design a good stratified estimator for the covariance matrix.

We also conclude that we cannot generate anti-correlated N-body simulations in a way that optimizes both the 2-pt and 4-pt estimators (the power spectrum and its covariance).

Our findings should be generally applicable to 2-pt and 4-pt functions (thus to tomographic density and shear power spectra, covariance matrices, etc). We should even be able to design simulation strategies optimized to reduce the variance of higher-order estimates such as bi- and tri-spectra.

In other words, we have shown that:
\begin{itemize}
	\item If we can run simulations that have anti-correlated observables (\eg 3D power spectra), we can significantly reduce the variance of the sample mean and covariance estimators (estimating $P_k$ and its covariance). This is a generalization of the techniques described in \cite{PontzenEtAl2016} and \cite{AnguloPontzen2016}, which were partially exploiting the so-called antithetic estimator of the mean.
	\item This variance reduction can be improved even further by considering stratified estimators, \ie running groups of correlated simulations. In the specific case of antithetic "paired" simulations as explored in \cite{PontzenEtAl2016} and \cite{AnguloPontzen2016}, we have derived a new estimator for the covariance matrix.
	\item Many of our derivations assumed Gaussian random variables, but our conclusions, and in particular the variance reduction, hold for non-Gaussian random variables. Those must be studied numerically, but it is very likely (and it has been found in other studies) that the anti-correlations introduced in the initial fields remain fairly conserved as it is evolved in an N-body simulation.
	Thus, we should be able to significantly reduce the variance of the mean and covariance estimators applied to the evolved density field (in other words, we'll need fewer simulations).
	\item We have only discussed the 3D power spectrum and its covariance, but our conclusions will also apply to any 2-pt estimators of the initial or evolved density field, and their covariances (tomographic power spectra, etc). They should also extend to n-pt correlations, such as the bispectrum, although the estimators will have to be modified.
	\item Very concretely, we only need to tweak the correlations in the initial conditions of N-body simulations and adopt the previous stratified estimators. In particular, we must construct density fields (initial conditions for N-body simulations) which have anti-correlated observed power spectra. Inspired by \cite{AnguloPontzen2016}, we have shown a simple way to achieve that.
\end{itemize}

All of those improvements require no changes in the way the simulations are run. We only need to generate the initial conditions differently, and adopt more complex estimators to compute the final power spectra and covariances.

Furthermore, we have shown that including extra information such as (imperfect) theoretical modeling can also significantly reduce the number of simulations needed. In a naive approach, one can simply focus on estimating differences between the simulation and the imperfect model. In a much better approach, one can run an auxiliary set of simulations, where the analytic form of the target power spectrum or covariance is known, and exploit anti-correlations with the primary set of simulations to significantly reduce the variance of the estimators.
For this latter technique, one could in fact use an existing set of uncorrelated N-body simulations with a new set of simpler correlated simulations (\eg  LPT) to reduce the variance.

\todo{Write down the final expressions combining stratified estimators with theoretical predictions and control variates!}

\todo{Check Weinberg and Coles papers}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Extensions}

\todo{Write down the estimators for the mean bispectrum and its covariance! All of the previous methods should work too.}

\todo{Sections about partial averaging, conditioning, Rao-Blackwellization, or importance sampling? Are those useful in this context? I don't think so but I'll check.}

\todo{Discuss what changes and does not change if we consider other observables, their mean and covariance.}

\todo{Precision matrices!}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliography{bib}

\newpage
\appendix
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Derivation of the variance of the sample covariance estimator with correlated draws}\label{sec:samplecovariancecorrelateddraws}

We first write
\eqn{
	\hat{C}_{kk',i} \ = \ (\hat{P}_{k, i} - \bar{P}_{k})(\hat{P}_{k', i} - \bar{P}_{k'})
	\ = \  \hat{P}_{k, i}\hat{P}_{k', i} - \bar{P}_{k}\hat{P}_{k', i} - \hat{P}_{k, i}\bar{P}_{k'}  + \bar{P}_{k}\bar{P}_{k'}
}
Our derivation requires unraveling the various components and going though some long expressions,
\eqn{
	 && \mathrm{Cov}[\hat{C}_{kk',i} , \hat{C}_{kk',j} ] \\
	 &=&
	  \mathrm{Cov}[\hat{P}_{k, i}\hat{P}_{k', i}, \hat{P}_{k, j}\hat{P}_{k', j} ] - \mathrm{Cov}[ \bar{P}_{k}\hat{P}_{k', i}, \hat{P}_{k, j}\hat{P}_{k', j} ] - \mathrm{Cov}[ \hat{P}_{k, i}\bar{P}_{k'} ,\hat{P}_{k, j}\hat{P}_{k', j} ] + \mathrm{Cov}[\bar{P}_{k}\bar{P}_{k'}, \hat{P}_{k, j}\hat{P}_{k', j} ]	\\
	 &-& \mathrm{Cov}[\hat{P}_{k, i}\hat{P}_{k', i} ,\bar{P}_{k}\hat{P}_{k', j} ] + \mathrm{Cov}[ \bar{P}_{k}\hat{P}_{k', i}, \bar{P}_{k}\hat{P}_{k', j} ] + \mathrm{Cov}[\hat{P}_{k, i}\bar{P}_{k'}, \bar{P}_{k}\hat{P}_{k', j} ] - \mathrm{Cov}[\bar{P}_{k}\bar{P}_{k'}, \bar{P}_{k}\hat{P}_{k', j} ]\\
	 &-& \mathrm{Cov}[\hat{P}_{k, i}\hat{P}_{k', i}, \hat{P}_{k, j}\bar{P}_{k'}] + \mathrm{Cov}[ \bar{P}_{k}\hat{P}_{k', i} ,\hat{P}_{k, j}\bar{P}_{k'}] + \mathrm{Cov}[\hat{P}_{k, i}\bar{P}_{k'} ,\hat{P}_{k, j}\bar{P}_{k'}] -\mathrm{Cov}[\bar{P}_{k}\bar{P}_{k'}, \hat{P}_{k, j}\bar{P}_{k'}]\\
	&+&  \mathrm{Cov}[\hat{P}_{k, i}\hat{P}_{k', i}, \bar{P}_{k}\bar{P}_{k'}] - \mathrm{Cov}[ \bar{P}_{k}\hat{P}_{k', i} , \bar{P}_{k}\bar{P}_{k'} ] - \mathrm{Cov}[\hat{P}_{k, i}\bar{P}_{k'} , \bar{P}_{k}\bar{P}_{k'}] + \mathrm{Cov}[\bar{P}_{k}\bar{P}_{k'} , \bar{P}_{k}\bar{P}_{k'}]
}

Thanks to Isserlis' \& Wick's theorem, we can use the following identity for any quadruplet of Gaussian random variables ABCD:
\eqn{
	\mathrm{Cov}[AB,CD] &=& \mathbb{E}[ABCD] - \mathbb{E}[AB]\mathbb{E}[CD] \\
	&=& \mathbb{E}[AB]\mathbb{E}[CD] + \mathbb{E}[AC]\mathbb{E}[BD] + \mathbb{E}[AD]\mathbb{E}[BC] - \mathbb{E}[AB]\mathbb{E}[CD] \\
	&=& \mathbb{E}[AC]\mathbb{E}[BD] + \mathbb{E}[AD]\mathbb{E}[BC] \label{eq:wickstheorem}
}

We can expand the covariances into expectations,
\eqn{
	 && \mathrm{Cov}[\hat{C}_{kk',i} , \hat{C}_{kk',j} ] =  \\
	 &&
	  \EE[\hat{P}_{k, i}\hat{P}_{k, j}] \EE[\hat{P}_{k', i}\hat{P}_{k', j} ]
	+	  \EE[\hat{P}_{k, i}\hat{P}_{k', j} ] \EE[\hat{P}_{k', i} \hat{P}_{k, j}]
	  - \EE[ \bar{P}_{k}\hat{P}_{k, j}] \EE[\hat{P}_{k', i} \hat{P}_{k', j} ]
	  -  \EE[ \bar{P}_{k} \hat{P}_{k', j} ] \EE[\hat{P}_{k', i} \hat{P}_{k, j}] \\
	&&	  - \EE[ \hat{P}_{k, i}\hat{P}_{k, j}]  \EE[ \bar{P}_{k'} \hat{P}_{k', j} ]
	  - \EE[ \hat{P}_{k, i}\hat{P}_{k', j}]  \EE[  \bar{P}_{k'} \hat{P}_{k, j}]
	 + \EE[\bar{P}_{k}\hat{P}_{k, j}]  \EE[ \bar{P}_{k'} \hat{P}_{k', j} ]
	  + \EE[\bar{P}_{k} \hat{P}_{k', j} ]  \EE[\bar{P}_{k'} \hat{P}_{k, j} ]	\\
	&& - \EE[\hat{P}_{k, i} \bar{P}_{k}]  \EE[ \hat{P}_{k', i} \hat{P}_{k', j} ]
	 - \EE[\hat{P}_{k, i} \hat{P}_{k', j} ]  \EE[\hat{P}_{k', i} \bar{P}_{k} ]
	+ \EE[ \bar{P}_{k} \bar{P}_{k}] \EE[ \hat{P}_{k', i} \hat{P}_{k', j} ]
	 + \EE[ \bar{P}_{k} \hat{P}_{k', j}]  \EE[  \hat{P}_{k', i} \bar{P}_{k}] \\
	&& + \EE[\hat{P}_{k, i} \bar{P}_{k}]  \EE[ \bar{P}_{k'} \hat{P}_{k', j} ]
	 + \EE[\hat{P}_{k, i} \hat{P}_{k', j}]  \EE[  \bar{P}_{k'} \bar{P}_{k}]
	 - \EE[\bar{P}_{k} \bar{P}_{k}]  \EE[ \bar{P}_{k'} \hat{P}_{k', j} ]
	 - \EE[\bar{P}_{k} \hat{P}_{k', j} ]  \EE[\bar{P}_{k'} \bar{P}_{k} ] \\
	&& - \EE[\hat{P}_{k, i} \hat{P}_{k, j}]  \EE[ \hat{P}_{k', i} \bar{P}_{k'}]
	 - \EE[\hat{P}_{k, i} \bar{P}_{k'} ]  \EE[\hat{P}_{k', i} \hat{P}_{k, j}]
	 + \EE[ \bar{P}_{k} \hat{P}_{k, j}]  \EE[ \hat{P}_{k', i} \bar{P}_{k'}]
	 + \EE[ \bar{P}_{k} \bar{P}_{k'}] \EE[ \hat{P}_{k', i} \hat{P}_{k, j}] \\
	&& + \EE[\hat{P}_{k, i}  \hat{P}_{k, j} ]  \EE[\bar{P}_{k'}\bar{P}_{k'}]
	 + \EE[\hat{P}_{k, i} \bar{P}_{k'}]  \EE[ \bar{P}_{k'} \hat{P}_{k, j}]
	 -\EE[\bar{P}_{k} \hat{P}_{k, j}]  \EE[ \bar{P}_{k'} \bar{P}_{k'}]
	 -\EE[\bar{P}_{k} \bar{P}_{k'}] \EE[ \bar{P}_{k'} \hat{P}_{k, j}]\\
	&& +  \EE[\hat{P}_{k, i} \bar{P}_{k} ]  \EE[\hat{P}_{k', i} \bar{P}_{k'}]
	+  \EE[\hat{P}_{k, i} \bar{P}_{k'} ]  \EE[\hat{P}_{k', i} \bar{P}_{k}]
	- \EE[ \bar{P}_{k} \bar{P}_{k} ] \EE[\hat{P}_{k', i}  \bar{P}_{k'} ]
	- \EE[ \bar{P}_{k} \bar{P}_{k'} ]  \EE[\hat{P}_{k', i}  \bar{P}_{k} ] \\
	&&- \EE[\hat{P}_{k, i} \bar{P}_{k} ]  \EE[\bar{P}_{k'}  \bar{P}_{k'}]
	- \EE[\hat{P}_{k, i} \bar{P}_{k'} ]  \EE[\bar{P}_{k'} \bar{P}_{k}]
	+ \EE[\bar{P}_{k}  \bar{P}_{k} ]  \EE[\bar{P}_{k'} \bar{P}_{k'}]
	+ \EE[\bar{P}_{k} \bar{P}_{k'} ]  \EE[\bar{P}_{k'}  \bar{P}_{k}]
	  }

Thanks to commutativity of the terms within the expectations and also of the indices ($i, j, k, k'$), the only three terms we need to compute are:\
\eqn{
	 \EE[\hat{P}_{k, i}\hat{P}_{k', j}] \ &=& \  \mathrm{Cov}[\hat{P}_{k, i}\hat{P}_{k', j}] + \mathbb{E}[\hat{P}_{k, i}]\mathbb{E}[\hat{P}_{k', j}]  \ =  \ C_{kk'} \underbrace{\left( \delta^K_{ij}(1-\beta_{ij}) + \beta_{ij} \right)}_{B_{ij}} + P_k P_{k'} \\
	 \EE[\bar{P}_{k}\hat{P}_{k', j}] \ &=&\   \frac{1}{N}  \sum_{l=1}^{N}\EE[\hat{P}_{k, l}\hat{P}_{k', j}]  \ = \ P_k P_{k'} + \frac{C_{kk'} }{N}  \underbrace{ \sum_{l=1}^{N} \left( \delta^K_{lj}(1-\beta_{lj}) + \beta_{lj} \right)}_{B_j} \\
	 \EE[\bar{P}_{k}\bar{P}_{k'}] \ &=& \ \frac{1}{N^2}  \sum_{l=1}^{N}  \sum_{m=1}^{N} \EE[\hat{P}_{k, l}\hat{P}_{k', m}]  \  = \ P_k P_{k'} + \frac{C_{kk'}}{N^2} \underbrace{ \sum_{l=1}^{N}  \sum_{m=1}^{N}   \left( \delta^K_{lm}(1-\beta_{lm}) + \beta_{lm} \right)}_{B}
}
where we have assumed that we estimate the mean too, using the sample estimator. Obviously, we have $B=\sum_{l=1}^{N}  B_{l}$ and $B_l=\sum_{m=1}^{N}  B_{lm}$.
Without off-diagonal correlations, we have $B_{ij} = \delta^K_{ij}$,  $B_l = 1$, and $B=N$. In this case, we recover the standard results
\eqn{
	 \EE[\hat{P}_{k, i}\hat{P}_{k', j}] \ = \  C_{kk'} \delta^K_{ij} + P_k P_{k'} \quad\quad\quad \EE[\bar{P}_{k}\hat{P}_{k', j}] \ =  \  \EE[\bar{P}_{k}\bar{P}_{k'}]  \ =\ P_k P_{k'} + \frac{C_{kk'} }{N}
}

Thanks to the symmetry in $kk'$, there are only six unique terms in $\mathrm{Cov}[\hat{C}_{kk',i} , \hat{C}_{kk',j} ] $:
\eqn{
	A_0 = &&  \sum_{ij} \EE[\hat{P}_{k, i}\hat{P}_{k, j}] \EE[\hat{P}_{k', i}\hat{P}_{k', j} ]  \ = \ C_{kk}C_{k'k'}  \sum_{ij} B_{ij}^2 + (C_{kk}P^2_{k'} + P_k^2C_{k'k'}) B  + N^2 P^2_{k}P^2_{k'} \\
	A_0^\prime = &&  \sum_{ij}  \EE[\hat{P}_{k, i}\hat{P}_{k', j}] \EE[\hat{P}_{k', i}\hat{P}_{k, j} ]  \ = \  C^2_{kk'}  \sum_{ij} B^2_{ij} +  2 C_{kk'} B P_k P_{k'} +  N^2P_k^2 P_{k'}^2 \\
	A_1 = &&  \sum_{ij}  \EE[ \bar{P}_{k}\hat{P}_{k, j}] \EE[\hat{P}_{k', i} \hat{P}_{k', j} ] \ = \   N^2 P^2_{k}P^2_{k'} + B \Bigl( P^2_{k}C_{k'k'} + P^2_{k'} C_{kk} \Bigr) + \frac{C_{kk} C_{k'k'}  }{N} \sum_{j}  B^2_j	\\
	A_1^\prime = &&  \sum_{ij} \EE[ \bar{P}_{k} \hat{P}_{k', j} ] \EE[\hat{P}_{k', i} \hat{P}_{k, j}] \ = \ N^2 P^2_k P^2_{k'} + 2 BP_k P_{k'}C_{kk'}  +  \frac{C^2_{kk'} }{N}  \sum_{j} B^2_j	\\
	&& \sum_{ij} 	 \EE[\bar{P}_{k}\hat{P}_{k, j}]  \EE[ \bar{P}_{k'} \hat{P}_{k', j} ]	= A_1	\\
	 && \sum_{ij} 	\EE[\bar{P}_{k} \hat{P}_{k', j} ]  \EE[\bar{P}_{k'} \hat{P}_{k, j} ]	= A_1^\prime \\
	A_2 = && \sum_{ij}   \EE[ \bar{P}_{k} \bar{P}_{k}] \EE[ \hat{P}_{k', i} \hat{P}_{k', j} ] =		N^2P^2_kP^2_{k'} + B (P^2_k C_{k'k'}  + P^2_{k'} C_{kk} ) +	 \frac{C_{kk}C_{k'k'}}{N^2}  B^2\\
	A_2^\prime = && \sum_{ij}   \EE[ \bar{P}_{k} \bar{P}_{k'}] \EE[ \hat{P}_{k, i} \hat{P}_{k', j} ] =		N^2 P^2_k P^2_{k'} + 2BC_{kk'}P_k P_{k'} +\frac{C^2_{kk'}}{N^2} B^2 \\
	 && \sum_{ij}  \EE[\bar{P}_{k}  \bar{P}_{k} ]  \EE[\bar{P}_{k'} \bar{P}_{k'}]	=   \sum_{ij}  \EE[\bar{P}_{k} \bar{P}_{k}]  \EE[ \bar{P}_{k'} \hat{P}_{k', j} ] = A_2\\
	&& \sum_{ij}  \EE[\bar{P}_{k} \bar{P}_{k'} ]  \EE[\bar{P}_{k'}  \bar{P}_{k}]	=  \sum_{ij} \EE[\bar{P}_{k} \hat{P}_{k', j} ]  \EE[\bar{P}_{k'} \bar{P}_{k} ] = A_2^\prime
}
We can then reduce the giant summation to
\eqn{
	&&  \ \mathrm{Var}[\hat{C}_{kk'}] \ = \frac{
	 A_0   +	  A_0^\prime  - A_2 - A_2^\prime}{(N-1)^2}  =    \bigl(C_{kk}C_{k'k'} + C_{kk'}^2 \bigr) \times \frac{ \sum_{ij} B_{ij}^2  - \frac{(\sum_{ij} B_{ij})^2}{N^2}}{(N-1)^2}
}

%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{document}
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\section{Conclusion}
\label{sec:conclusion}

We have proven mathematically that the total-field anomaly data caused by a set of magnetic sources
with uniform magnetization direction can be exactly reproduced by a continuous and planar layer of
dipoles having an all-positive magnetic-moment distribution.
This theoretical property holds true for the case in which a layer has the same
magnetization direction as that of the true sources, regardless of whether they have
a purely induced magnetization or not.
By using this generalized positivity constraint, we presented a new iterative method for
estimating the total magnetization direction of 3D magnetic sources based on the equivalent-layer technique.
At each iteration, we impose a positivity constraint on the estimated magnetic-moment distribution of the layer
and solve a non-linear inverse problem for estimating the magnetization direction of the equivalent sources.
Prior knowledge about the shape and depth of magnetic sources is not required, neither the use of an evenly spaced
data set. This methodology can be applied for determining the magnetization direction of multiple sources,
considering all of them with the same magnetization direction.
Results obtained with synthetic data produced by multiple sources show that the estimated magnetization direction
obtained by our iterative method successfully retrieves the true one.
Tests with synthetic data illustrate how the presence of a relatively shallow-seated source affects the
result obtained by our method for the cases in which it has a magnetization direction equal to and different from
the other sources. In both cases, the equivalent layer yielded large data misfits above the shallow source;
however, we cannot distinguish whether the shallow source has a magnetization direction equal to or different from the other sources.
Moreover, our method produces the most poorly estimated magnetization direction when shallow-seated source is magnetized in a
direction that differs from the other sources.
An application to field data over the Goi{\' a}s alkaline province, center of Brazil, has confirmed that our method can
be a reliable tool for interpreting complex geological scenarios. The result over the Montes Claros complex suggests the presence
of a strong remanent magnetization component and corroborates a previous study conducted independently at the same area.
The estimated magnetic-moment distribution over the layer has led to a very acceptable reduction to the pole, but also
produced large data-misfits at some isolated regions. We presume that these locally large data-misfits are due to shallow sources,
however we cannot infer if they have the same magnetization direction of the other bodies.


\documentclass{article}
\usepackage[a4paper, left=25.4mm, top=25.4mm, right=25.4mm, bottom=25.4mm]{geometry}
\usepackage[shortlabels]{enumitem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{authoraftertitle}
\usepackage{blindtext}
\usepackage{bussproofs}
\usepackage{color}
\usepackage{graphicx}
\usepackage{newpxtext}
%\usepackage{newpxmath}
\usepackage{ragged2e}
\usepackage{textcomp}
\usepackage{textgreek}
\renewcommand{\thesubsection}{\arabic{subsection}}
\newcommand\tab[1][1cm]{\hspace*{#1}}
\newcommand\pfun{\mathrel{\ooalign{\hfil$\mapstochar\mkern5mu$\hfil\cr$\to$\cr}}}
\newcommand\lowermidtilde{\raisebox{-0.8ex}{\textasciitilde}}
\newcommand\midtilde{\raisebox{-0.6ex}{\textasciitilde}}
\newcommand{\partfarrow}{\rightharpoonup}
\newcommand{\partf}{\rightharpoonup\!\!\!\!\!\!\!\!\!\raisebox{2.5pt}{$\rightharpoonup$}}

\author{Victor Zhao\\xz398@cam.ac.uk}

\begin{document}
\centering
\section*{Discrete Mathematics\\CST Part IA Paper 2}
\MyAuthor

\justifying
\subsection{Proof}
\begin{enumerate}
    \item Some mathematical jargon:
        \begin{itemize}[topsep=0pt]
            \item \textbf{Statement}: A sentence that is either true or false — but not both.
            \item \textbf{Predicate}: A statement whose truth depends on the value of one or more variables.
            \item \textbf{Theorem}: A very important true statement.
            \item \textbf{Proposition}: A less important but nonetheless interesting true statement.
            \item \textbf{Lemma}: A true statement used in proving other true statements.
            \item \textbf{Corollary}: A true statement that is a simple deduction from a theorem or proposition.
            \item \textbf{Conjecture}: A statement believed to be true, but for which we have no proof.
            \item \textbf{Proof}: Logical explanation of why a statement is true; a method for establishing truth.
            \item \textbf{Logic}: The study of methods and principles used to distinguish good (correct) from bad (incorrect) reasoning.
            \item \textbf{Axiom}: A basic assumption about a mathematical situation. Axioms can be considered facts that do not need to be proved (just to get us going in a subject) or they can be used in definitions.
            \item \textbf{Definition}: An explanation of the mathematical meaning of a word (or phrase). The word (or phrase) is generally defined in terms of properties.
            \item A statement is \textit{simple} (or \textit{atomic}) when it cannot be broken into other statements, and it is \textit{composite} when it is built by using several (simple or composite statements) connected by logical expressions
        \end{itemize}
    \item Contraposition:\\
        The contrapositive of $P\implies Q$ is $\neg Q\implies\neg P$.
    \item Modus Ponens:
        If $P$ and $P\implies Q$ holds then so does $Q$.
        \begin{prooftree}
            \AxiomC{$P$}
            \AxiomC{$P\implies Q$}
            \BinaryInfC{$Q$}
        \end{prooftree}
    \item Some notations:
        \begin{itemize}[topsep=0pt]
            \item Implication: $\implies$
            \item Bi-implication: $\Longleftrightarrow$
            \item Universal quantification: $\forall x.P(x)$
            \item Existential quantification: $\exists x.P(x)$
            \item Unique existence: $\exists!x.P(x)$
                $$\exists!x.P(x)\Longleftrightarrow\exists x.P(x)\wedge\Big(\forall y.\forall z.\big(P(y)\wedge P(z)\big)\implies y=z\Big)$$
            \item Conjunction: $\wedge$
            \item Disjunction: $\vee$
            \item Negation: $\neg$
        \end{itemize}
    \item Equality axioms:
        \begin{itemize}[topsep=0pt]
            \item Every individual is equal to itself.
                $$\forall x.x=x$$
            \item (Leibniz equality) For any pair of equal individuals, if a property holds for one of them, then ait also holds for the other one.
                $$\forall x.\;\forall y.\;x=y\implies (P(x)\implies P(y))$$
        \end{itemize}
\end{enumerate}
\subsection{Numbers}
\begin{enumerate}
    \item Definitions of real numbers. A real number is:
        \begin{itemize}[topsep=0pt]
            \item \textbf{rational} if it is of the form $\frac{m}{n}$ for a pair of integers $m$ and $n$; otherwise it is \textbf{irrational};
            \item \textbf{positive} if it is greater than 0, and \textbf{negative} if it is smaller than 0;
            \item \textbf{nonnegative}~if~it~is~greater~than~or~equal~to~0,~and~\textbf{nonpositive}~if~it~is~smaller~than~or~equal~to~0;
            \item \textbf{natural} if it is a nonnegative integer.
        \end{itemize}
    \item Additive structure $(\mathbb{N},0,+)$ of natural numbers with zero and addition is a commutative monoid\\(a \textit{monoid} is a semigroup with an identity element; a \textit{semigroup} preserves closure and associativity):
        \begin{itemize}[topsep=0pt]
            \item Monoid laws:
                $$\begin{aligned}
                	0+n&=n+0=n&&\text{(identity)} \\
                	(l+m)+n&=l+(m+n)&&\text{(associativity)}
                \end{aligned}$$
            \item Commutativity law:
                $$m+n=n+m$$
        \end{itemize}
    \item Multiplicative structure $(\mathbb{N},1,\cdot)$ of natural numbers with one and multiplication is a commutative monoid:
        \begin{itemize}[topsep=0pt]
            \item Monoid laws:
                $$1\cdot n=n\cdot 1=n$$
                $$(l\cdot m)\cdot n=l\cdot(m\cdot n)$$
            \item Commutativity law:
                $$m\cdot n=n\cdot m$$
        \end{itemize}
    \item The overall structure $(\mathbb{N},0,+,1,\cdot)$ is a commutative semiring:
        \begin{itemize}[topsep=0pt]
            \item $(\mathbb{N},0,+)$ is a commutative monoid;
            \item $(\mathbb{N},1,\cdot)$ is a monoid;
            \item Multiplication is distributive over addition:
                $$l\cdot(m+n)=l\cdot m+l\cdot n$$
            \item Multiplication by 0 annihilates $\mathbb{N}$:
                $$0\cdot n=n\cdot 0=0$$
        \end{itemize}
    \item Cancellation:
        \begin{itemize}[topsep=0pt]
            \item Additive cancellation: for all natural numbers $k$, $m$, $n$,
                $$k+m=k+n\implies m=n$$
            \item Multiplicative cancellation: for all natural numbers $k$, $m$, $n$,
                $$\text{if }k\neq0\text{ then }k\cdot m=k\cdot n\implies m=n$$
        \end{itemize}
    \item Inverses:
        \begin{itemize}[topsep=0pt]
            \item A number $x$ is said to admit an \textbf{additive inverse} whenever there exists a number $y$ such that $x+y=0$;
            \item A number $x$ is said to admit an \textbf{multiplicative inverse} whenever there exists a number $y$ such that $x\cdot y=1$.
        \end{itemize}
    \item The integers $\mathbb{Z}$ form a commutative ring, and the rationals $\mathbb{Q}$ form a field:
        \begin{itemize}[topsep=0pt]
            \item A \textit{group} is a monoid in which every element has an inverse;
            \item A \textit{ring} is a semiring $(0,+)$, $(1,\cdot)$ where $(0,+)$ is a commutative group. It is commutative if $(1,\cdot)$ is also commutative;
            \item A \textit{field} is a ring where every non-zero element has a multiplicative inverse.
        \end{itemize}
    \item Divisibility and congruence:
        \begin{itemize}[topsep=0pt]
            \item Let $d$ and $n$ be integers. We say that $d$ \textit{devides} $n$, and write $d|n$, whenever there exists an integer $k$ such that $n=k\cdot d$;
            \item Fix a positive integer $m$. For integers $a$ and $b$, we say that $a$ \textit{is congruent to} $b$ \textit{modulo} $m$, and write $a\equiv b\;(\text{mod } m)$, whenever $m|(a-b)$.
        \end{itemize}
    \item For all prime numbers $p$ and integers $0\leq m\leq p$, either $\binom{p}{m}\equiv 0\;(\text{mod } p)$ or $\binom{p}{m}\equiv 1\;(\text{mod } p)$.\\
        For $0<m<p$, $p|\binom{p}{m}$ and $(p-m)|\binom{p-1}{m}$.
    \item The Freshman's Dream: For all natural numbers $m$, $n$ and primes $p$,
        $$(m+n)^p\equiv m^p+n^p\;(\text{mod } p)$$
    \item The Dropout Lemma: For all natural numbers $m$ and primes $p$,
        $$(m+1)^p\equiv m^p+1\;(\text{mod } p)$$
    \item The Many Dropout Lemma: For all natural numbers $m$ and $i$, and primes $p$,
        $$(m+i)^p\equiv m^p+i\;(\text{mod } p)$$
    \item Fermat's Little Theorem: For all natural numbers $i$ and primes $p$,
        \begin{itemize}[topsep=0pt]
            \item $i^p\equiv i\;(\text{mod } p)$, and
            \item $i^{p-1}\equiv 1\;(\text{mod } p)$ whenever $i$ is not a multiple of $p$.
        \end{itemize}
    \item The Division Theorem: For every natural number $m$ and positive natural number $n$, there exists a unique pair of integers $q$ and $r$ such that $q\geq0$, $0\leq r\leq n$, and $m=q\cdot n+r$.
    \item Modular arithmetic: For all natural numbers $m>1$, the modular-arithmetic structure
        $$(\mathbb{Z}_m,0,+_m,1,\cdot_m)$$
        is a commutative ring.\\
        For prime $p$, $\mathbb{Z}_p$ is a field.
    \item Greatest Common Divisor: For all positive integers $m$ and $n$,
        $$\text{gcd}(m,n)=\begin{cases}
            n &, \text{if }n|m\\
            \text{gcd}\big(n,\text{rem}(m,n)\big) &, \text{otherwise}
        \end{cases}$$
    \item Some fundamental properties of gcds:
        \begin{itemize}[topsep=0pt]
            \item Commutativity: $\text{gcd}(m,n)=\gcd(n,m)$,
            \item Associativity: $\text{gcd}\big(l,\gcd(m,n)\big)=\text{gcd}\big(\gcd(l,m),n\big)$,
            \item Distributivity: $\text{gcd}(l\cdot m,l\cdot n)=l\cdot\text{gcd}(m,n)$.
        \end{itemize}
    \item Theorem: For positive integers $k$, $m$, and $n$, if $k|(m\cdot n)$ and $\text{gcd}(k,m)=1$ then $k|n$.\\
        Corollary (Euclid's Theorem): For positive integers $m$, $n$, and prime $p$, if $p|(m\cdot n)$ then $p|m$ or $p|n$.
    \item For all positive integers $m$ and $n$,
        \begin{itemize}[topsep=0pt]
            \item $n\cdot\text{lc}_2(m,n)\equiv\text{gcd}(m,n)\;(\text{mod }m)$, and
            \item whenever $\text{gcd}(m,n)=1$,\\
                $[\text{lc}_2(m,n)]_m$ is the multiplicative inverse of $[n]_m$ in $\mathbb{Z}_m$.
        \end{itemize}
\newpage
    \item Principle of Induction:\\
        Let $P(m)$ be a statement for $m$ ranging over the natural numbers greater than or equal to a fixed natural number $l$. If
        \begin{itemize}[topsep=0pt]
            \item $P(l)$ holds, and
            \item $\forall n\geq l\text{ in }\mathbb{N}.\big(P(n)\implies P(n+1)\big)$ also holds,
        \end{itemize}
        then
        \begin{itemize}[topsep=0pt]
            \item $\forall m\geq l\text{ in }\mathbb{N}.P(m)$ holds.
        \end{itemize}
    \item Principle of Strong Induction:\\
        Let $P(m)$ be a statement for $m$ ranging over the natural numbers greater than or equal to a fixed natural number $l$. If
        \begin{itemize}[topsep=0pt]
            \item $P(l)$ holds, and
            \item $\forall n\geq l\text{ in }\mathbb{N}.\Big(\big(\forall k\in[l..n].P(k)\big)\implies P(n+1)\Big)$ also holds,
        \end{itemize}
        then
        \begin{itemize}[topsep=0pt]
            \item $\forall m\geq l\text{ in }\mathbb{N}.P(m)$ holds.
        \end{itemize}
    \item Well-Founded Induction:\\
        \textbf{Definition:} a \textit{well-founded relation} is a binary relation $\prec$ on a set $A$ such that there are no infinite descending chains $\cdots\prec a_i\prec\cdots\prec a_1\prec a_0$. When $a\prec b$ we say $a$ is a \textit{predecessor} of $b$.\\
        \textbf{Principle of Well-Founded Induction:} Let $\prec$ be a well-founded relation on a set $A$. if
        \begin{itemize}[topsep=0pt]
            \item $\forall a\in A.\Big(\big(\forall b\prec a.P(b)\big)\implies P(a)\Big)$ holds,
        \end{itemize}
        then
        \begin{itemize}[topsep=0pt]
            \item $\forall a\in A.P(a)$ holds.
        \end{itemize}
    \item Fundamental Theorem of Arithmetic: For every positive integer $n$ there is a unique finite ordered sequence of primes $(p_1\leq\cdots\leq p_l)$ with $l\in\mathbb{N}$ such that
        $$n=\displaystyle\prod_{i=1}^{l}p_i.$$
\end{enumerate}
\newpage
\subsection{Sets}
\begin{enumerate}
    \item Axioms:
        \begin{itemize}[topsep=0pt]
            \item Extensionality axiom: Two sets are equal if they have the same elements.
                $$\forall\text{ sets }A, B\;.\;A=B\Longleftrightarrow(\forall x.x\in A\Longleftrightarrow x\in B)$$
            \item Powerset axiom: For any set, there is a set consisting of all its subsets.
            \item Pairing axiom: For every $a$ and $b$, there is a set with $a$ and $b$ as its only elements.
            \item Union axiom: Every collection of sets has a union.
            \item Infinity axiom: There is an infinite set, containing $\emptyset$ and closed under successor. \\($\text{Succ}(x)=_\text{def}x\cup\{x\}$)
            \item Axiom of choice: Every surjection has a section (right inverse).
            \item Replacement axiom: The direct image of every definable functional property on a set is a set.
        \end{itemize}
    \item Cardinality:
        \begin{itemize}[topsep=0pt]
            \item $\forall\text{ finite set }U.\#\mathcal{P}(U)=2^{\#U}$
            \item $\forall\text{ sets }A,B.\#(A\times B)=\#A\times\#B$
            \item $\forall\text{ sets }A,B.\#(A\uplus B)=\#A+\#B$
        \end{itemize}
    \item Subsets:
        $$A\subseteq B\Longleftrightarrow(\forall x.x\in A\implies x\in B)$$
        $$A\subset B\Longleftrightarrow(A\subseteq B\wedge A\neq B)$$
        Reflexivity: $\forall\text{ set }A\;.\;A\subseteq A$\\
        Transitivity: $\forall\text{ set }A,B,C\;.\;(A\subseteq B\wedge B\subseteq C)\implies A\subseteq C$\\
        Antisymmetry: $\forall\text{ set }A,B\;.\;(A\subseteq B\wedge B\subseteq A)\implies A=B$
    \item Separation principle: For any set $A$ and any definable property $P$, there is a set containing precisely those elements of $A$ for which the property $P$ holds.
        $$\{x\in A\;|\;P(x)\}$$
    \item The powerset Boolean algebra: $\big(\mathcal{P}(U), \emptyset, U, \cup, \cap, (\cdot)^c\big)$
        \begin{itemize}[topsep=0pt]
            \item For all $A, B\in\mathcal{P}(U)$,
                $$A\cup B=\{x\in U\;|\;x\in A\vee x\in B\}\in\mathcal{P}(U)$$
                $$A\cap B=\{x\in U\;|\;x\in A\wedge x\in B\}\in\mathcal{P}(U)$$
                $$A^c=\{x\in U\;|\;\neg(x\in A)\}\in\mathcal{P}(U)$$
            \item The union operateion $\cup$ and the intersection operation $\cap$ are associative, commutative, and idempotent:
                $$(A\cup B)\cup C=A\cup(B\cup C),\;A\cup B=B\cup A,\;A\cup A=A$$
                $$(A\cap B)\cap C=A\cap(B\cap C),\;A\cap B=B\cap A,\;A\cap A=A$$
            \item The \textit{empty set} $\emptyset$ is a neutral element for $\cup$ and the \textit{universal set} $U$ is a neutral element for $\cap$:
                $$\emptyset\cup A=U\cap A=A$$
            \item The empty set $\emptyset$ is an annihilator for $\cap$ and the universal set $U$ is an annihilator for $\cup$:
                $$\emptyset\cap A=\emptyset$$
                $$U\cup A=U$$
            \item With respect to each other, the union operation $\cup$ and the intersection operation $\cap$ are distributive and absorptive:
                $$A\cap(B\cup C)=(A\cap B)\cup(A\cap C)$$
                $$A\cup(B\cap C)=(A\cup B)\cap(A\cup C)$$
                $$A\cup(A\cap B)=A\cap(A\cup B)=A$$
            \item The complement operation $(\cdot)^c$ satisfies complementation laws:
                $$A\cup A^c=U,\;A\cap A^c=\emptyset$$
        \end{itemize}
    \item Ordered pair: $\langle a, b\rangle=_{\text{def}}\big\{\{a\},\{a, b\}\big\}$\\
    Fundamental property or ordered pairing:
        $$\forall a,b,x,y\;.\;\langle a,b\rangle=\langle x,y\rangle\Longleftrightarrow(a=x\wedge b=y)$$
    \item Big Unions: Let $U$ be a set. For a collection of sets $\mathcal{F}\in\mathcal{P}(\mathcal{P}(U))\;(\text{i.e. }\mathcal{F}\subseteq\mathcal{P}(U))$,
        $$\bigcup\mathcal{F}=_{\text{def}}\{x\in U\;|\;\exists A\in\mathcal{F}.x\in A\}\in\mathcal{P}(U)$$
    Idea:
        $$\bigcup\{A_1, A_2, \cdots\}=(A_1\cup A_2\cup\cdots)\subseteq U$$
    \item Big Intersections: Let $U$ be a set. For a collection of sets $\mathcal{F}\in\mathcal{P}(\mathcal{P}(U))\;(\text{i.e. }\mathcal{F}\subseteq\mathcal{P}(U))$,
        $$\bigcap\mathcal{F}=_{\text{def}}\{x\in U\;|\;\forall A\in\mathcal{F}.x\in A\}\in\mathcal{P}(U)$$
    Idea:
        $$\bigcap\{A_1, A_2, \cdots\}=(A_1\cap A_2\cap\cdots)\subseteq U$$
    \item Tagging: $\{l\}\times A$
    \item Disjoint Unions: $A\uplus B=_\text{def}\big(\{1\}\times A\big)\cup\big(\{2\}\times B\big)$
        $$\forall x.x\in(A\uplus B)\Longleftrightarrow\big(\exists a\in A.x=(1,a)\big)\vee\big(\exists b\in B.x=(2,b)\big)$$
\end{enumerate}
\newpage
\subsection{Relations}
\begin{enumerate}
    \item Some notations and definitions:
        \begin{itemize}[topsep=0pt]
            \item Relation: $\pfun$\\
                For all finite sets $A$ and $B$, $\#\text{Rel}(A, B)=2^{\#A\cdot\#B}$
            \item Partial function: $\rightharpoonup$\\
                Set of partial functions: $\partf$\\
                Every partial function $f:A\rightharpoonup B$ satisfies that: for each element $a$ of $A$ there is at most one element $b$ of $B$ such that $a\;f\;b$.
                $$\forall f\in\text{Rel}(A, B).\;f\in(A\partf\;B)\Longleftrightarrow\forall a\in A.\forall b_1,b_2\in B.\;a\;f\;b_1\wedge a\;f\;b_2\implies b_1=b_2$$
                For all finite sets $A$ and $B$, $\#(A\partf\;B) = (\#B+1)^{\#A}$
            \item Mapping: $\mapsto$
            \item Function: $\rightarrow$\\
                Set of functions: $\Rightarrow$\\
                A partial function is total if its domain of definition coincides with its source.
                $$\forall f\in(A\partf\;B).\;f\in(A\Rightarrow B)\Longleftrightarrow\forall a\in A.\;\exists b\in B.\;a\;f\;b$$
                $$\forall f\in\text{Rel}(A, B).\;f\in(A\Rightarrow B)\Longleftrightarrow\forall a\in A.\;\exists!b\in B.\;a\;f\;b$$
                For all finite sets $A$ and $B$, $\#(A\Rightarrow B) = \#B^{\#A}$
            \item Injection: $\rightarrowtail$\\
                A function $f:A\rightarrow B$ is injective whenever
                $$\forall a_1,\;a_2\in A.\;f(a_1)=f(a_2)\implies a_1=a_2$$
            \item Surjection: $\twoheadrightarrow$\\
                A function $f:A\rightarrow B$ is surjective whenever
                $$\forall b\in B.\;\exists a\in A.\;f(a)=b$$
                For all finite sets $A$ and $B$, $\#\text{Sur}(A,B)=$
            \item Bijection: A function $f:A\rightarrow B$ is bijective whenever there exists a (necessarily unique) function $g:B\rightarrow A$ (referred to as the inverse of $f$) such that
                $$g\circ f=\text{id}_A\quad\text{and}\quad f\circ g=\text{id}_B$$
                For all finite sets $A$ and $B$,
                $$\#\text{Bij}(A,B)=\begin{cases}
                    0 &, \text{if }\#A\neq\#B\\
                    n! &, \text{if }\#A=\#B=n
                \end{cases}$$
        \end{itemize}
    \item Composition:\\
        Composition of two relations $R:A\pfun B$ and $S:B\pfun C$:
            $$S\circ R:A\pfun C$$
        Relational composition is associative and has the identity relation as neutral element:
            $$\forall R:A\pfun B,\;S:B\pfun C,\;T:C\pfun D\;.\;(T\circ S)\circ R=T\circ(S\circ R)$$
            $$\forall R:A\pfun B\;.\;R\circ\text{id}_A=\text{id}_B\circ R=R$$
        $R^{\circ n}$: $R$ composed with itself $n$ times.\\
        $R^{\circ*}=\bigcup_{n\in\mathbb{N}}R^{\circ n}$
    \newpage
    \item Preorders:\\
        A preorder $(P,\sqsubseteq)$ consists of a set $P$ and a relation $\sqsubseteq$ on $P$ satisfying the following two axioms:
        \begin{itemize}[topsep=0pt]
            \item Reflexivity: $\forall x\in P.x\sqsubseteq x$
            \item Transitivity: $\forall x,y,z\in P.(x\sqsubseteq y\wedge y\sqsubseteq z)\implies x\sqsubseteq z$
        \end{itemize}
        $R^{\circ*}$ is the reflexive-transitive closure of $R$\\
        $R^{\circ*}$ is the least preorder containing $R$\\
        $R^{\circ*}$ is the preorder freely generated by $R$
    \item Isomorphism: $\cong$\\
        Two sets $A$ and $B$ are isomorphic (and have the same cardinality) whenever there is a bijection between them,
    \item Equivalence relations:\\
        A relation $E$ on a set $A$ is an equivalence relation whenever it is:
        \begin{itemize}[topsep=0pt]
            \item Reflexive: $\forall x\in A.\;x\;E\;x$
            \item Symmetric: $\forall x, y\in A.\; x\;E\;y\implies y\;E\;x$
            \item Transitive: $\forall x, y, z\in A.\;(x\;E\;y\wedge y\;E\;z)\implies x\;E\;z$
        \end{itemize}
    \item Set partitions:\\
        A partition $P$ of a set $A$ is a set of non-empty subsets of $A$ (that is, $P\subseteq\mathcal{P}(A)$ and $\emptyset\notin P$), whose elements are typically referred to as blocks, such that
        \begin{itemize}[topsep=0pt]
            \item The union of all blocks yields $A$: $\bigcup P=A$, and
            \item All blocks are pairwise disjoint: $\forall B_1, B_2\in P.\;B_1\neq B_2\implies B_1\cap B_2=\emptyset$
        \end{itemize}
        For every set $A$: $\text{EqRel}(A)\cong\text{Part}(A)$
    \item Enumerability:\\
        A set $A$ is enumerable whenever there exists a surjection ($\mathbb{N}\twoheadrightarrow A$), or a injection ($A\rightarrowtail\mathbb{N}$), referred to as an enumeration.\\
        A countable set is one that is either empty or enumerable.
    \item Relational images and functional images:\\
        Let $R:A\pfun B$ be a relation.
        \begin{itemize}[topsep=0pt]
            \item The direct image of $X\subseteq A$ under $R$ is the set $\overrightarrow{R}(X)\subseteq B$:
                $$\overrightarrow{R}(X)=\{b\in B|\exists x\in X\;.\;x\;R\;b\}$$
                This construction yields a function $\overrightarrow{R}:\mathcal{P}(A)\to\mathcal{P}(B)$.
            \item The inverse image of $Y\subseteq B$ under $R$ is the set $\overleftarrow{R}(X)\subseteq A$:
                $$\overleftarrow{R}(Y)=\{a\in A|\forall b\in B\;.\;a\;R\;b\implies b\in Y\}$$
                This construction yields a function $\overleftarrow{R}(Y):\mathcal{P}(B)\to\mathcal{P}(A)$.
        \end{itemize}
        Let $f:A\to B$ be a function.
        \begin{itemize}[topsep=0pt]
            \item For all $X\subseteq A$, $\overrightarrow{f}(X)=\{b\in B|\exists a\in X\;.\;f(a)=b\}$;
            \item For all $Y\subseteq B$, $\overleftarrow{f}(Y)=\{a\in A|f(a)\in Y\}$.
        \end{itemize}
\end{enumerate}
\end{document}


\subsection{Implementation of temporal constraint with Huang style transfer}
In this implementation we have used the idea that style is a distribution of features described in Section \ref{sec:Style is a distribution of features}. We have utilized the Wasserstein metric in order to change the style loss of our previous implementation for style transfer for videos described in \ref{sec:Ruder style transfer}. This implementation is one of our own experiments with style transfer for video, as we combine the Ruder implementation with temporal constraint and change up the style loss to the one proposed in the paper by Huang et al. \cite{Huang:1}.
\newline\newline
Since calculating the Wasserstein distance is computationally heavy, we have simplified this calculation. We characterize the distribution of features with simply the mean value and co-variance of the distributions, then using these to calculate the Wasserstein distance between them. For these calculations we have used TensorFlow, so we can more easily run the computations on a GPU for faster calculations.\newline\newline
The loss function for this implementation uses the same layers in the VGG-19 CNN as our implementation from Section \ref{sec:Ruder style transfer}, but the weights are different as the style loss is a lot smaller. The weight for the style loss here is 400 times larger than for our Ruder implementation. The results for this implementation can be found in Section \ref{sec:Results with wasserstein metric}.
\begin{figure}[!ht]
\begin{center}
\includegraphics[scale=0.12]{report/Method/images/shrek_comparison.png}
\caption{Leftmost we see a frame from the first Shrek movie. The middle image is styled with the Gram Matrices from Gatys paper. The rightmost image is styled with Wasserstein as loss function. Both were styled with La Muse (Picasso) as style reference.}
\label{fig:architecture}
\end{center}
\end{figure}