% !TEX root = ../main.tex
% chktex-file 21
% chktex-file 46
\section{Spectral Graph Theory}%
\label{sec:sgt}

We start with an introduction to spectral graph theory, which will allow us to characterize and compare the structure of graphs.
Graphs are most commonly described in their vertex base, i.e.\  the connection strength $w_{i j}$ of pairs $(v_i, v_j)$ of vertices.
\begin{align}
	G :=&\, (\mathcal{V}, \mathcal{E}, W)\quad\text{with edges } \mathcal{E} \subseteq \mathcal{V} \times \mathcal{V}\text{, weights } W \in \mathbb{R}^{N \times N}\text{ and vertex count } N := |\mathcal{V}|
\end{align}
In this paper we will only consider undirected graphs with non-negative real weights and without self-loops ($\forall i: w_{i i} = 0$).

The core idea of spectral graph theory is to perform a change of basis of $W$ and describe graphs in terms of their so called \textit{spectral base} instead of their \textit{vertex base}.
To see what this means, we interpret the adjacency/weight matrix $W$ as a linear operator that operates on so called signals $x \in \mathbb{R}^N$.
A signal $x$ can be interpreted as a function $x: \mathcal{V} \to \mathbb{R}$ that assigns a signal strength to each vertex.
By applying $Wx$ the given signal strengths $x_i$ are shifted according to the connection strenghts $w_{i j}$ to neighboring vertices $v_j$.
This interpretation of graphs is very similar to that of Markov chains where signals represent probability distributions.

\subsection{Relating Graph Signals to Real-valued Functions}%
\label{sec:sgt:real}

Let us now compare discrete graph signals $x: \mathcal{V} \to \mathbb{R}$ to continuous real-valued functions $f: \mathbb{R} \to \mathbb{R}$.
Both signatures only differ in their domain.
The domain $\mathbb{R}$ of $f$ has an inherent structure, the real number line, which provides a strict ordering of its elements and a notion of distance between them.
The domain $\mathcal{V}$ of $x$ however has no such inherent structure, i.e.\  $v_1 < v_2$ for two vertices $v_1, v_2$ does not have a clear meaning.
The structure of $\mathcal{V}$ fully depends on the graph $G$ that is acting on it.
Intuitively graph signals can thus be understood as a discretized generalization of real-valued functions, where the underlying structure of the input domain is not fixed but can be freely chosen.
\begin{figure}[ht]
	\centering
	\includegraphics[width=0.8\linewidth]{gfx/sgt/real-graph.pdf}
	\caption{%
		Illustration of how the discretized real number line can be interpreted as an infinite linear graph, compared to some arbitrary finite non-linear graph.
		The bars show the signal strengths $f(t)$ and $x_i$ at the vertices $b_t$ and $b_i$ respectively.
	}\label{fig:sgt:realGraph}
\end{figure}

\Cref{fig:sgt:realGraph} shows that all real-valued functions $f$ can be seen as signals $x$ of the graph described by the real number line\footnote{%
	Technically this is not correct, since $\mathbb{R}$ is continuous whereas all vertex sets $\mathcal{V}$ have to be discrete.
	To build an intuition for graph signals, this detail can however be ignored.
}.
Both, real-valued functions and graph signals, can be described as vectors in their time and vertex base respectively:
\begin{equation*}
	\begin{split}
		f = \int_{\mathbb{R}} f(t) b_t dt
		\Rightarrow \langle b_t, f \rangle = f(t)
	\end{split}
	\quad\text{and}\quad
	\begin{split}
		x = \sum_{i = 1}^{N} x_i b_i
		\Rightarrow \langle b_i, x \rangle = x_i
	\end{split}
\end{equation*}
In this equation ${\{ b_i \}}_{v_i \in \mathcal{V}}$ denotes the standard basis of the adjacency matrix $W$, i.e.\ $b_i \in \mathbb{R}^N$ has a signal strength of $0$ for all $v_{j \neq i}$ and a signal strength of $1$ for $v_i$.
Similarly ${\{ b_t \}}_{t \in \mathbb{R}}$ denotes the infinite dimensional standard basis of the space of real functions, where $\langle b_t, \cdot \rangle := \delta_t(\cdot)$, with $\delta$ denoting the Dirac delta function.

\vspace{-1em}%
\subsection{Extending the Fourier Transform to Graphs}%
\label{sec:sgt:fourier}

As mentioned at the beginning of this section, the core idea of spectral graph theory is to express graph signal vectors $x$ in the spectral basis ${\{ u_i \}}_{i = 1}^{N}$ instead of the standard vertex basis ${\{ b_i \}}_{v_i \in \mathcal{V}}$.
This idea is analogous to the classical Fourier transform on real-valued functions.
In the first step we are going to give an intuition for the classical Fourier transform.
Afterwards we will extend this intuition to the graph domain.

The Fourier basis ${\{ u_\xi \}}_{\xi \in \mathbb{R}}$ describes a function $f$ not in terms of the values $f(t) = \langle b_t, f \rangle$ it takes at position $t$ but instead in terms of the amplitudes $\hat{f}(\xi) = \langle u_\xi, f \rangle$ of the complex exponentials $u_\xi(t) = e^{2\pi i \xi t}$.
Using this perspective, the Fourier transform can be viewed as a change of basis operator from the orthonormal standard basis ${\{ b_t \}}_{t \in \mathbb{R}}$ to the also orthonormal Fourier basis ${\{ u_\xi \}}_{\xi \in \mathbb{R}}$:
\begin{align*}
	f = \int_{\mathbb{R}} \underbrace{\langle b_t, f \rangle}_{f(t)} b_t dt\quad=\quad\int_{\mathbb{R}} \underbrace{\langle u_\xi, f \rangle}_{\hat{f}(\xi)} u_\xi d\xi
\end{align*}
This property of the Fourier transform by itself is not special; in fact there are infinitely many orthonormal bases on the space of real-valued functions.
The distinguishing property of ${\{ u_\xi \}}_{\xi \in \mathbb{R}}$, making it useful in many domains, is that it is an \textit{eigenbasis} of the \textit{Laplacian} $\Updelta$, i.e.\  $\Updelta u_\xi = \lambda_\xi u_\xi$ for the eigenvalue $\lambda_\xi \in \mathbb{R}$.
The Laplacian is a multi\-dimensional generalization of the second derivative.
For real-valued functions this boils down to $\Updelta u_\xi = \frac{\partial^2}{\partial t^2} u_\xi = -{(2 \pi \xi)}^2 e^{2 \pi i \xi t}$ with the eigenvalue $\lambda_\xi = {-(2\pi\xi)}^2$ depending solely on the frequency $\xi$.
The original motivation to express functions in terms of the Laplacian's eigenbasis was to solve the physical heat equation\footnote{%
	More generally the Fourier basis turns out to be meaningful for all \textit{linear time-invariant} (LTI) systems, of which the heat equations are only one instance.
}.
For this reason it is a useful intuition to think about signals as temperature distributions that will converge to an equilibrium state over time.
This intuition also works in the graph setting where heat only flows between neighboring vertices in proportion to their connection strengths.

Now that we have looked at the Fourier transform of real-valued functions, we will extend this notion to graphs.
Just like the classical Fourier transform, the graph Fourier transform performs a change of basis of a signal $x$ from the vertex basis ${\{ b_i \}}_{v_i \in \mathcal{V}}$ to the Fourier basis ${\{ u_k \}}_{k = 1}^{N}$.
This Fourier basis again is characterized by it being an eigenbasis of the Laplacian, more specifically the so called \textit{combinatorial graph Laplacian} $L$ in this case:
\begin{align}
	L := D - W\quad\text{with the degree matrix } D := {
		\renewcommand*{\arraystretch}{0.5}
		\begin{pmatrix}
			d_1 & & \\
			& \ddots & \\
			& & d_N
		\end{pmatrix}
	}, d_i := \sum_{j = 1}^{N} w_{i j}
\end{align}
Using this definition, the application $L x$ is a discrete generalized analogue of the second derivative $\frac{\partial^2}{\partial t^2} f$.
Putting both Laplacian variants, $\frac{\partial^2}{\partial t^2}$ and $L$, side-by-side gives an intuition for why this is the case:
\begin{equation*}
	\begin{split}
		\left(-\frac{\partial^2}{\partial t^2} f\right)\mkern-4mu(t) = \lim_{h \to 0} \frac{1}{h^2} (\underbrace{f(t) - f(t-h)}_{\Delta_{t, t-h}} + \underbrace{f(t) -  f(t+h)}_{\Delta_{t, t+h}})
	\end{split}
	\begin{split}\ \left|\ %
		{(L x)}_i = \sum_{j = 1}^{N} w_{i j} \underbrace{(x_i - x_j)}_{\Delta_{i, j}}
	\right.\end{split}
\end{equation*}
The second derivative of a function $f$ essentially averages the differences in signal strength in the neighborhood of a point $t$.
For real-valued functions this neighborhood only consists of the two infinitesimally close points to the left and to the right of $t$, i.e.\ $t - h$ and $t + h$.
The graph Laplacian represents the same operation, where each point/vertex might however have more than two neighbors that need to be averaged.

Based on the graph Laplacian $L$ we just defined, the graph Fourier basis ${\{ u_k \}}_{k = 1}^{N}$ is the set of eigenvectors of $L$.
In the rest of this paper we will assume that these eigenvectors $u_k$ are sorted in ascending order w.r.t.\  their eigenvalues $\lambda_k$, i.e.\  $\lambda_1 \leq \lambda_2 \leq \cdots \leq \lambda_N$.
\Cref{fig:sgt:graphFourier} shows how this definition generalizes the Fourier basis from the real number line to an arbitrary graph.
It also shows that the eigenvalues $\lambda_k$ of the graph Fourier basis encode some notion of frequency, just like they do for the classical Fourier basis.
Analogous to the classical Fourier transform, the eigenvalues of a graph's Laplacian are therefore also called its \textit{spectrum}.
\begin{figure}[ht]
	\centering
	\makebox[\textwidth][c]{\includegraphics[width=\linewidth]{gfx/sgt/graph-fourier.pdf}}
	\caption{%
		Comparison between the basis functions/vectors of the classical Fourier transform and the graph Fourier transform.
		For the eigenfunctions on the upper half only the real cosine components of the complex exponentials are shown.
		\source[based on]{Shuman2013}
	}\label{fig:sgt:graphFourier}
\end{figure}

\subsection{Spectral Properties of Graphs}%
\label{sec:sgt:spectrum}

Next we will give an intuition for the relation between the spectrum of a graph and its structural properties\footnote{%
	Only a general overview will be given. For a more detailed discussion we refer to \citet{Shuman2013}.
}.
For any graph the smallest eigenvalue always is $\lambda_1 = 0$ with the associated eigenvector $u_1 = \frac{1}{\sqrt{N}} {(1, \dots, 1)}^\top$ being a uniform signal over all vertices.
Using the heat analogy, this simply means that any system in which everything has the same temperature is in an equilibrium state and no heat is flowing.
More generally for graphs with $c$ separate connected components the first $c$ eigenvalues are all $0$ since each component can have its own equilibrium temperature without causing heat flow.
If the spectrum of a graph is known, this fact can be used to quickly determine whether a graph is connected or not.

Another meaningful eigenvalue is $\lambda_2$ and its eigenvector $u_2$, also called the \textit{Fiedler value} and \textit{Fiedler vector} respectively.
As we have just seen, a Fiedler value of $\lambda_2 = 0$ means that a graph is not connected.
More generally the Fielder value can be interpreted as a measure of overall graph connectivity\footnote{%
	Formally this measure is called \textit{algebraic connectivity}, as opposed to regular \textit{graph connectivity}, which is defined as $\min_{v_i \in \mathcal{V}} \deg(v_i)$.
}.
If there are two clusters of vertices $\mathcal{V}_+$ and $\mathcal{V}_-$ in a graph that are connected via relatively few edges (compared to the overall number of edges), the Fielder value is small;
for well connected graphs on the other hand the Fiedler value is large.
Using the heat analogy, the Fielder value essentially measures the ``width'' of a bottleneck between two parts of a graph through which heat will only flow very slowly or even not at all in case of $\lambda_2 = 0$.
The partitioning of vertices into $\mathcal{V}_+$ and $\mathcal{V}_-$ can be retrieved via the Fiedler vector $u_2$;
vertices with a positive signal strength in $u_2$ are in $\mathcal{V}_+$, the other vertices are in $\mathcal{V}_-$.
The so called \textit{spectral clustering} algorithm is based on this relationship.
\Cref{fig:sgt:graphFourier} shows how the Fiedler vector $u_2$ partitions a graph via the signs of the vertex signal strengths.

To summarize, expressing a graph $G$ in terms of the spectrum of its Laplacian $L$ gives access to graph characteristics like its overall connectivity.
Similarly to how the low-frequency components of the Fourier transform of a function represent the overall shape of that function, the eigenvectors associated with the small eigenvalues of $L$ represent overall characteristics of $G$.
The details of a function or graph on the other hand are encoded in the high-frequency components of its Fourier transform, i.e.\  the eigenvectors associated with large eigenvalues.
This perspective already hints at how spectral graph analysis might be useful to approximate graphs.


\documentclass[]{article}

\usepackage[utf8]{inputenc}
\usepackage{hyperref}
\usepackage{makecell}
\usepackage{url}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{array}
\usepackage{times,color}
\usepackage[rflt]{floatflt}
\usepackage{epsfig,times,graphicx}
\usepackage{amsmath,amssymb,amsopn,algorithm,algorithmic,theorem,float,bbm,bm,enumerate,color,multirow}
\usepackage{rotating}
\usepackage{array}
%\usepackage{slashbox}
\usepackage{makecell}
\usepackage{multirow}
\usepackage{hhline}
\usepackage{xspace}
\usepackage{mathtools}
\usepackage{bm}

\input{notation}

\title{Projection onto Elastic Net Ball}

\begin{document}

How can we project to an arbitrary norm ball?

\section{Major Lessons}
I think the general way of doing this is to go for proximal operators. Proximal operator of a closed proper convex function $f(\cdot)$ is defined as:
\beq
\prox_f (\v) = \argmin_\x \frac{1}{2} \norm{\v - \x}{2}^2 + f(\x)
\eeq
Note that the objective of the above optimization is strongly convex, so there is a unique minimizer for it.

\subsection{Generalizing Projection}
Proximal operator is a generalization of projection in the following sense. Remember the indicator of a set $\cS$, i.e.,
\beq
\indic_\cS(\x) =
\begin{cases}
	0& \x \in \cS \\
	\infty& \x \notin \cS \\
\end{cases}
\eeq
Then the proximal operator reduces to Euclidian projection:
\beq
\prox_f (\v) = \argmin_{\x\in \cS} \frac{1}{2} \norm{\v - \x}{2}^2 = \Pi_\cS(\v)
\eeq
\subsection{Generalizing Orthogonal Decomposition}
Let $\cS$ and $\cS^\perp$ be two orthogonal subspaces. Then for any vector $\v$ we have:
\beq
\v = \Pi_{\cS}(\v) + \Pi_{\cS^\perp}(\v)
\eeq
We have the same decomposition for closed proper convex $f$ and its convex conjugate $f^*(\y) = \argmin_\x \x^T\y - f(\x)$:
\beq
\v = \prox_f (\v) + \prox_{f^*} (\v)
\eeq
The above result is known as {\bf Moreau Decomposition}.

{\bf Proof:}
For the $\u = \prox_f (\v)$, from the optimality condition we should have:
\beq
\v - \u \in \partial f(\u) \iff \u \in \partial f^*(\v - \u ) \iff (\v - \u) = \prox_{f^*}(\u)
\eeq

Above, we used a useful property which relates subgradient of $f$ and $f^*$:
\beq
\u \in \partial f(\v) \iff \v \in \partial f^*(\u) \iff f^*(\u) + f(\v) = \u^T \v
\eeq

\subsection{Moreau Decomposotion for a Norm}
Here we first need to following for an arbitrary norm $f(\x) = \norm{\x}{}$:
\beq
f^*(\y) = \indic_{\norm{\cdot}{*} \leq 1}(\y)
\eeq
where $\norm{\cdot}{*}$ is the daul norm. In other words, the convex conjugate of a norm is the indicator function of its dual norm.

From the above equality and the Moreau decomposition we get:
\be
\v
&= \prox_{\norm{\cdot}{}} + \prox_{\indic_{\norm{\cdot}{*} \leq 1}(\cdot)}
\\
&= \prox_{\norm{\cdot}{}} + \Pi_{\indic_{\norm{\cdot}{*} \leq 1}} (\v)
\ee
Above equality gives us a way to switch between proximal operator $\prox$ and projection $\Pi$.

\end{document}


\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

\usepackage{graphicx}

\title{MATH 505B Homework 4}
\author{Saket Choudhary\\skchoudh@usc.edu}

\begin{document}
\maketitle

\section*{Problem 6.14.1}

\begin{align*}
\langle\ \mathbf{x}, \mathbf{Py} \rangle &= \sum_{k\in \theta} x_k (\mathbf{Py})_k\pi_k \\
&= \sum_{k\in \theta} x_k (\sum_jp_{kj}y_j)\pi_k\\
&= \sum_{k\in \theta} x_k (\sum_jp_{kj}\pi_ky_j)\\
&= \sum_{k,j} x_k p_{kj}\pi_ky_j\\
&= \sum_{k,j} x_k (p_{jk}\pi_jy_j) \text{ using reversibility criterion  } \pi_jp_{jk} = \pi_kp_{kj}\\
&= \sum_{j} p_{jk}x_k \pi_jy_j\\
&= \sum_{j \in \theta} (\sum_k p_{jk}x_k) \pi_jy_j\\
&= \langle \mathbf{Px,y} \rangle
\end{align*}

\section*{Problem 6.14.2}
For reversibility: $\pi_ip_{ij} = \pi_jp_{ji}$:

$p'_{ij}= b_{ij}g_{ij} = \frac{\pi_jg_{ji}g_{ij}}{\pi_ig_{ij}+\pi_jg_{ji}}$ and $p'_{ji} = \frac{\pi_ig_{ij}g_{ij}}{\pi_ig_{ij}+\pi_jg_{ji}}$

Hence, $\pi_ip'_{ij} = \pi_jp'_{ji}$ and hence $b_{ij}$ satisfies reversibility criterion besides $ 0 \leq b_{ij} \leq 1$.

\section*{Problem 7.2.1}
\subsection*{7.2.1(a)}

\begin{align*}
\{E(|X+Y|^p)\}^{1/p} &= \{E|X^p|\}^{1/p} + \{E|Y^p|^{1/p}\}\\
E[|X|] &= E[|X_n+X-X_n|]\\
\{E[|X|^p]\}^{1/p} &= \{E[|X_n+X-X_n|^p]\}^{1/p}\\
&\leq \{E[|X_n|^p]\}^{1/p}+\{E[|X-X_n|^p]\}^{1/p}\\
\implies \lim_{n \longrightarrow \infty} \inf E[|X|^p] &\leq E[|X_n|^p]^{1/p}\numberthis \label{eqn1} \\
\end{align*}

Similarly,

\begin{align*}
\{E[|X_n|^p]\}^{1/p} &= \{E[|X_n-X+X|^p]\}^{1/p}\\
&\leq \{E[|X|^p]\}^{1/p} + \{E[|X_n-X|]\}^{1/p}\\
\implies  \lim_{n \longrightarrow \infty} \sup E[|X_n|^p] &\leq E[|X|^p] \numberthis \label{eqn2}
\end{align*}

Combining \ref{eqn1}, \ref{eqn2} : $E[|X_n|^p] \longrightarrow
E[|X|^p]$ $p \geq 1$

\subsection*{7.2.1(b)}
Using $p=1$ in part (a)
\subsection*{7.2.1(c)}
Using part (a) $E[X_n^2] \longrightarrow E[X^2]$
$(X_n \xrightarrow{2} X) \Rightarrow (X_n \xrightarrow{1} X)\ \implies E[X_n] \longrightarrow E[X]$ and hence $Var(X_n) \longrightarrow Var(X)$

\section*{Problem 7.2.3}

Consider $k \geq 0$, $ n\geq 1$,  $X_n=k/n \leq X < (k+1)/n$.
$X-1/n \leq X_n \leq X$

Define similarly $Y_n$. $Y_n,X_n$ are independent by definition

$E[X_n] \longrightarrow E[X]$ and $E[Y_n] \longrightarrow E[Y]$

Thus, using independence and convergence relations $E[X_nY_n] = E[X_n]E[Y_n] \longrightarrow E[X]E[Y] $

Now,
\begin{eqnarray*}
(X-1/n)(Y-1/n)\leq X_nY_n \leq XY\\
\implies E[(X-1/n)(Y-1/n)]\leq E[X_nY_n] \leq E[XY]\\
E[(X-1/n)(Y-1/n)] = E[XY-\frac{X+Y}{n}+1/n^2]
\implies E[X_nY_n] \longrightarrow E[XY]
\end{eqnarray*}

Thus combining, the above two results $E[X_nY_n] \longrightarrow E[XY]$ and $E[X_nY_n] \longrightarrow E[X][Y]$
we get $E[XY]=E[X]E[Y]$

\section*{Problem 7.2.10}

$\sum_r X_r \sim Poisson(\sum_r \lambda_r)$

Define $t=\sum_{r=1}^n \lambda_r$:
\begin{align*}
P(\sum_r X_r \leq x) &= \sum_{i=0}^x \frac{e^{-t}t^i}{i!}\\
\lim_{n \longrightarrow \infty}P(\sum_r X_r \leq x)  &= \begin{cases} 0 & t\longrightarrow \infty\\
Poisson(t) & t \text{is finite}
\end{cases}
\end{align*}

\section*{Problem 7.4.1}

\begin{align*}
E[X_1] &= 0*(1-\frac{1}{n\log{n}} + 0*\frac{1}{2n\log n}\\
&=0\\
E[X_1^2] &= \frac{2n^2}{2n\log{n}}+0*(1-\frac{1}{n\log{n}}\\
&= \frac{n}{\log{n}}\\
E[(\frac{1}{n}S_n-0)^2] &= \frac{1}{n^2}Var(S_n)\\
&= \frac{1}{n^2}\frac{n}{\log{n}}\\
&= \frac{1}{n \log{n}}
&\longrightarrow 0
\end{align*}

$\sum_i P(|X_i| \geq i)
\longrightarrow \infty$ Hence, using Borel-Cantelli Lemma(7.3.10b)
we have $P(|X_j| \geq j) =1$ for some $j$
$|X_j| = |S_j-S_{j-1}| \geq j$ and hence $S_j/j$ diverges.

\section*{Problem 7.5.1}
Define $I_{i}(j)$ as the indicator variable denoting if the $X_j$ lies in the $i^{th}$ interval,
\begin{align*}
\log{R_m} &= \sum_{i=1}^n Z_m(i) \log p_i\\
&= \sum_{i=1}^n \sum_{j=1}^m I_i(j) p_i\\
\end{align*}
Define $\sum_{i=1}^m I_i(j)=Y_j$, then $\log{R_m} = \sum_{j=1}^m Y_j$

$E[Y_j]=\sum_{i=1}^np_i\log{p_i}=-h$
Thus, by strong law of convergence $\frac{1}{m}\sum_{j=1}^m Y_j = \ \longrightarrow -h=E[Y_j]$

\section*{Problem 7.5.3}
Transient $P(X_n=i|X_0=i)< 1$

Using strong law $S_n/n \longrightarrow E[X_1]$ If $E[X_1]\neq 0$ then $P[S_n=0|S_1=0] < 1$ as $S_n=0$ happens only finitely often

\section*{Problem 7.7.1}
\begin{align*}
E[X_iX_j] &= E[E[X_iX_j|X_0,X_1,\dots, X_{j-1}]]\\
&= E[E[X_i(S_j-S_{j-1})|X_0,X_1,\dots,X_{j-1}]]\\
&= E[X_i(E[S_j-S_{j-1}|X_0,X_1,\dots,X_{j-1}])]\\
&= E[X_i(E[S_j|X_0,X_1,\dots,X_{j-1}]-S_{j-1})]\\
&= E[X_i(S_{j-1}-S_{j-1})]\\
&=0
\end{align*}

\section*{Problem 7.7.3}
\begin{align*}
E[X_{n+1}|X_0,X_1,\dots,X_n] &= aX_n+X_{n-1} \\
E[S_{n+1}|X_0,X_1,\dots, X_n] &= E[\alpha X_{n+1}+X_{n}|X_0,X_1,\dots,X_n]\\
&= \alpha E[X_{n+1}|X_0,X_1,\dots, X_n] + X_n\\
&= (\alpha a+1)X_n + \alpha bX_{n-1}\\
&= S_n = \alpha X_n+X_{n-1}\\
\implies \alpha = \frac{1}{1-a}, b=\frac{1}{\alpha}
\end{align*}

\section*{Problem 7.7.4}
$X_n$: Net profit per unit stake on $n^{th}$ play.

$S_{i} = S_{i-1}+f_{i}(X_1,X_2,\dots,X_i)$
such that $S_1=X_1Y$

Thus, $S_{n} = \sum_{i=1}^n X_if_{i-1}(X_1,X_2,\dots, X_{i-1})$

\begin{align*}
S_{n+1} &=  S_n + f_{n+1}(X_1,X_2,\dots, X_n)X_{n+1}\\
E[S_{n+1}-S_n|X_1,X_2,\dots,X_n] &= E[X_{n+1}f_{n+1}(X_1,X_2,\dots, X_n)|X_1,X_2,\dots,X_n]\\
&= f_{n+1}(X_1,X_2,\dots, X_n)E[X_{n+1}|X_1,X_2,\dots,X_n]\\
&= 0\\
\implies E[S_{n+1}|X_1,X_2,\dots,X_n] &= S_n
\end{align*}

\section*{Problem 7.8.1}
$E[X_i]=0$
By Doob-Kolmogorov inquality:

\begin{align*}
P(\max_{i\leq j \leq n} |S_j| > \epsilon) &\leq \frac{1}{\epsilon^2} \sum_{j=1}^{n}E[S_n^2]\\
E[S_n^2] &= Var(S_n)+E[S_n]^2\\
&= Var(S_n)\\
&= \sum Var(X_i)\\
\implies P(\max_{i\leq j \leq n} |S_j| > \epsilon) \leq \frac{1}{\epsilon^2} \sum_{j=1}^{n}Var(X_j)
\end{align*}

\section*{Problem 7.8.3}
By theorem 7.8.1 $S_n$ converges to $S$ almost surely
Now, using the above proved fact that $S_n \longrightarrow S \implies Var(S_n) \longrightarrow Var(S) \implies Var(S) \longrightarrow 0$

\end{document}


\section*{Appendix}
\begingroup
\def\thetheorem{\ref{lem:decrease}}
\begin{lemma}
If $f_i$ is continuously differentiable and log-concave, then the functions $l_i,d_i,v_i$ are continuous, strictly decreasing, and
\[l_i(p) < d_i(p) < v_i(p) \text{ for all }p.\]
\end{lemma}
\addtocounter{theorem}{-1}
\endgroup
\begin{proof}
Continuity of $F_i,f_i,f'_i$ implies that of $l_i,d_i,v_i$. It's known~\cite{concave} that log-concavity of $f_i$ implies log-concavity of both $F_i$ and $1-F_i$. As a result, $l_i$, $d_i$, and $v_i$ are derivatives of strictly concave functions; therefore, they are strictly decreasing. In particular, each of

\[v'_i(p) = \frac{f'_i(p)}{F_i(p)} - \frac{f_i(p)^2}{F_i(p)^2},\quad
l'_i(p) = \frac{-f'_i(p)}{1-F_i(p)} - \frac{f_i(p)^2}{(1-F_i(p))^2},\]

are negative for all $p$, so we conclude that

\begin{align*}
d_i(p) - v_i(p)
= \frac{f'_i(p)}{f_i(p)} - \frac{f_i(p)}{F_i(p)}
&= \frac{F_i(p)}{f_i(p)} v'_i(p)
< 0,
\\l_i(p) - d_i(p)
= -\frac{f'_i(p)}{f_i(p)} -\frac{f_i(p)}{1-F_i(p)}
&= \frac{1-F_i(p)}{f_i(p)} l'_i(p)
< 0.
\end{align*}

\end{proof}

\begingroup
\def\thetheorem{\ref{thm:uniq-max}}
\begin{theorem}
Suppose that for all $j$, $f_j$ is continuously differentiable and log-concave. Then the unique maximizer of $\Pr(P_i=p\mid E^L_i,E^W_i)$ is given by the unique zero of
\[Q_i(p) = \sum_{j \succ i} l_j(p) + \sum_{j \sim i} d_j(p) + \sum_{j \prec i} v_j(p).\]
\end{theorem}
\addtocounter{theorem}{-1}
\endgroup

\begin{proof}
First, we rank the players by their buckets according to $\floor{P_j/\epsilon}$, and take the limiting probabilities as $\epsilon\rightarrow 0$:
\begin{align*}
    \Pr(\floor{\frac{P_j}\epsilon} > \floor{\frac{p}\epsilon})
    &= \Pr(p_j \ge \epsilon\floor{\frac{p}\epsilon} + \epsilon)
    \\&= 1 - F_j(\epsilon\floor{\frac{p}\epsilon} + \epsilon)
    \rightarrow 1 - F_j(p),
    \\\Pr(\floor{\frac{P_j}\epsilon} < \floor{\frac{p}\epsilon})
    &= \Pr(p_j < \epsilon\floor{\frac{p}\epsilon})
    \\&= F_j(\epsilon\floor{\frac{p}\epsilon})
    \rightarrow F_j(p),
    \\\frac 1\epsilon \Pr(\floor{\frac{P_j}\epsilon} = \floor{\frac{p}\epsilon})
    &= \frac 1\epsilon \Pr(\epsilon\floor{\frac{p}\epsilon} \le P_j < \epsilon\floor{\frac{p}\epsilon} + \epsilon)
    \\&= \frac 1\epsilon\left( F_j(\epsilon\floor{\frac{p}\epsilon} + \epsilon) - F_j(\epsilon\floor{\frac{p}\epsilon}) \right)
    \rightarrow f_j(p).
\end{align*}

Let $L_{jp}^\epsilon$, $W_{jp}^\epsilon$, and $D_{jp}^\epsilon$ be shorthand for the events $\floor{\frac{P_j}\epsilon} > \floor{\frac{p}\epsilon}$, $\floor{\frac{P_j}\epsilon} < \floor{\frac{p}\epsilon}$, and $\floor{\frac{P_j}\epsilon} = \floor{\frac{p}\epsilon}$. respectively. These correspond to a player who performs at $p$ losing, winning, and drawing against $j$, respectively, when outcomes are determined by $\epsilon$-buckets. Then,
\begin{align*}
\Pr(E^W_i,E^L_i\mid P_i=p)
&= \lim_{\epsilon\rightarrow 0}
\prod_{j \succ i} \Pr(L_{jp}^\epsilon)
\prod_{j \prec i} \Pr(W_{jp}^\epsilon)
\prod_{j \sim i, j\ne i} \frac{\Pr(D_{jp}^\epsilon)}\epsilon
\\&= \prod_{j \succ i} (1 - F_j(p)) \prod_{j \prec i} F_j(p) \prod_{j \sim i, j\ne i} f_j(p),
\\\Pr(P_i=p \mid E^L_i,E^W_i)
&\propto f_i(p) \Pr(E^L_i,E^W_i\mid P_i=p)
\\&= \prod_{j \succ i} (1 - F_j(p)) \prod_{j \prec i} F_j(p) \prod_{j \sim i} f_j(p),
\\\ddp\ln \Pr(P_i=p \mid E^L_i,& E^W_i) = \sum_{j \succ i} l_j(p) + \sum_{j \prec i} v_j(p) + \sum_{j \sim i} d_j(p) = Q_i(p).
\end{align*}

Since \Cref{lem:decrease} tells us that $Q_i$ is strictly decreasing, it only remains to show that it has a zero. If the zero exists, it must be unique and it will be the unique maximum of $\Pr(P_i=p \mid E^L_i,E^W_i)$.

To start, we want to prove the existence of $p^*$ such that $Q_i(p^*) < 0$. Note that it's not possible to have $f'_j(p) \ge 0$ for all $p$, as in that case the density would integrate to either zero or infinity. Thus, for each $j$ such that $j\sim i$, we can choose $p_j$ such that $f'_j(p_j) < 0$, and so $d_j(p_j) < 0$. Let $\alpha = -\sum_{j\sim i} d_j(p_j) > 0$.

Let $n = |\{j:\,j \prec i\}|$. For each $j$ such that $j \prec i$, since $\lim_{p\rightarrow\infty}v_j(p) = 0/1 = 0$, we can choose $p_j$ such that $v_j(p_j) < \alpha/n$. Let $p^* = \max_{j\preceq i} p_j$. Then,
\[
\sum_{j \succ i} l_j(p^*) \le 0, \quad \sum_{j \sim i} d_j(p^*) \le -\alpha, \quad \sum_{j \prec i} v_j(p^*) < \alpha.
\]

Therefore,
\begin{align*}
Q_i(p^*)
&= \sum_{j \succ i} l_j(p^*) + \sum_{j \sim i} d_j(p^*) + \sum_{j \prec i} v_j(p^*)
\\&< 0 - \alpha + \alpha = 0.
\end{align*}

By a symmetric argument, there also exists some $q^*$ for which $Q_i(q^*) > 0$. By the intermediate value theorem with $Q_i$ continuous, there exists $p\in (q^*,p^*)$ such that $Q_i(p) = 0$, as desired.
\end{proof}


% This LaTeX was auto-generated from an M-file by MATLAB.
% To make changes, update the M-file and republish this document.

%%% \documentclass{article}
%%% \usepackage{graphicx}
%%% \usepackage{color}

%%% \sloppy
%%% \definecolor{lightgray}{gray}{0.5}
\setlength{\parindent}{0pt}

%%% \begin{document}

\subsubsection*{Contents}

\begin{itemize}
\setlength{\itemsep}{-1ex}
   \item Calibrations Curve Computing
   \item Generate sample data
   \item Call algorithm
   \item Display results
   \item Interpolate values
   \item Plot results
\end{itemize}
\begin{lstlisting}[style=mcode]
close all
\end{lstlisting}

\subsubsection*{Calibrations Curve Computing}

\begin{par}
Example for algorithm CCC.
\end{par} \vspace{1em}
\begin{par}
Calibration Curves Computing is a software for the evaluation of instrument calibration curves
\end{par} \vspace{1em}

\subsubsection*{Generate sample data}

\begin{par}
An dependence of amplitude error (Volts, ppm) on signal frequency (Hz) of an ADC was measured and uncertainties of measurement was estimated. The uncertainty of frequency can be considered as negligible.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
%3.7+12.*x+3.*x.^2
f = [10 1e2 1e3 1e4 1e5];
err = [19.700 32.700 69.700 90.700 148.700];
err_unc = [4 10 13 20 33];
\end{lstlisting}
\begin{par}
Set independent and dependent variables for \lstinline{CCC} algorithm. Lets operate in semi logarithm space for easy plotting.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI = [];
DI.x.v = log10(f);
DI.x.u = [];
DI.y.v = err;
DI.y.u = err_unc;
\end{lstlisting}
\begin{par}
Suppose the ADC has quadratic dependence of the error on the signal frequency.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI.exponents.v = [0 1 2];
\end{lstlisting}

\subsubsection*{Call algorithm}

\begin{par}
Use QWTB to apply algorithm \lstinline{CCC} to data \lstinline{DI}.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DO = qwtb('CCC', DI);
\end{lstlisting}

        \begin{lstlisting}[style=output]
QWTB: no uncertainty calculation
QWTB: CCC wrapper: model was set by CCC wrapper to a value `Model 2a`.
\end{lstlisting} \color{black}

\subsubsection*{Display results}

\begin{par}
Results is
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
disp(['offset          : ' num2str(DO.coefs.v(1)) ' +- ' num2str(DO.coefs.u(1))])
disp(['linear coeff.   : ' num2str(DO.coefs.v(2)) ' +- ' num2str(DO.coefs.u(2))])
disp(['quadratic coeff.: ' num2str(DO.coefs.v(3)) ' +- ' num2str(DO.coefs.u(3))])
\end{lstlisting}

        \begin{lstlisting}[style=output]
offset          : 12.6828 +- 16.9884
linear coeff.   : 1.9434 +- 19.1198
quadratic coeff.: 4.9055 +- 3.9754
\end{lstlisting} \color{black}

\subsubsection*{Interpolate values}

\begin{par}
Interpolate fitted polynom at values \lstinline{t}.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
t = [0:0.1:6];
ty = DO.func.v(t, DO.coefs.v);
\end{lstlisting}
\begin{par}
Calculate uncertainties of interpolated values (\lstinline{S} is sensitivity matrix, \lstinline{CC} is covariance matrix of coefficients, \lstinline{CT} is covariance matrix of interpolated values, \lstinline{uty} is uncertainty of interpolated values).
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
for i = 1:length(t);
        S = t(i).^DI.exponents.v;
        CC = diag(DO.coefs.u,0)*DO.coefs.c*diag(DO.coefs.u,0);
        CT(i)=S*CC*S';
end
uty=CT.^0.5;
\end{lstlisting}

\subsubsection*{Plot results}

\begin{lstlisting}[style=mcode]
hold on
errorbar(DI.x.v, DI.y.v, DI.y.u, 'xb')
errorbar(DI.x.v, DO.yhat.v, DO.yhat.u, 'og')
plot(t, ty, '-r');
plot(t, ty + uty, '-r');
plot(t, ty - uty, '-r');
xlabel('log(f)')
ylabel('error of amplitude')
legend('original data','fitted values','interpolated values', 'uncer. of int. val.','location','southeast')
hold off
\end{lstlisting}

\begin{center}
\includegraphics[width=0.7\textwidth]{algs_examples_published/CCC_alg_example_01.pdf}
\end{center}

%%% \end{document}


\section*{Web Appendix D}

The publication bias and the \textit{p}-hacking models have been defined as a selection model and a mixture model, respectively. To appreciate the difference between the models, and relate the selection biases to them, consider the following example. We want to perform a meta-analysis based on 10 studies. In the ideal situation with no publication bias and no \textit{p}-hacking, 10 researchers perform their studies fairly, observe an effect size from its distribution $\phi(x_{i}\mid\theta_i,\sigma^2_{i})$ and publish the result. To perform our meta-analysis, we do not need much more than to estimate $\theta_0$ by using $\phi(x_{i}\mid\theta_i,\sigma^2_{i})$ and $\phi(\theta_{i}\mid\theta_0,\tau^2)$.

In a publication bias scenario, some of these 10 studies may not be published because their \textit{p}-values $u_i$ are too large. Let us say that the first 3 have a \textit{p}-value smaller than $0.025$ (group A), the following 4 between $0.025$ and $0.05$ (group B), and the remaining 3 larger than $0.05$ (group C). With an editor publishing all studies ($\rho_{1} = 1$) with $u_i < 0.025$, half ($\rho_{2} = 0.5$) of those with $0.025 \leq u_i < 0.05$ and one-third ($\rho_{3} = 0.33$) when $u_i \geq 0.05$, we would expect to see all 3 studies in group A, 2 of Group B and only one of group $C$. It is clear that the density is not anymore the original $\phi(x_{i}\mid\theta_i,\sigma^2_{i})$, but a transformed one $f(x_{i}\mid\theta_i,\sigma^2_{i})$ for which some values of $\theta_i$ are underrepresented by the effect of a selection mechanism. In this case, by the selection probability function $w(u_i) = 1_{[0,0.025)}(u_i) + 0.5 \cdot 1_{[0.25,0.05)}(u_i) + 0.33 \cdot 1_{[0.05, 1]}(u_i)$. In practice we do not know $\rho_{1}$, $\rho_{2}$ and $\rho_{3}$, and we need to estimate them in our model. In the ideal situation, they are all equal to 1, that means no selection bias, so $f(x_{i}\mid\theta_{i},\sigma^2_{i}) = \phi(x_{i}\mid\theta_i,\sigma^2_{i})$.

In the \textit{p}-hacking scenario, instead, we observe all our 10 studies, but not with their original effect size, because they have been modified in order to reach a specific significance value $\alpha$. In mathematical terms, $x_i$ does not come from the original Gaussian $\phi(x_{i}\mid\theta_i,\sigma^2_{i})$ but from a truncated Gaussian $\phi_\alpha^{\star}(x_{i}\mid\theta_i,\sigma^2_{i})$, where the truncation excludes the possibility to get an $u_i$ larger than $\alpha$. If every researchers decided to $p$-hack all studies at the same level, let us say $\alpha = 0.05$, we could make inference on $\theta_i$ using $\phi_{0.05}^{\star}(x_{i}\mid\theta,\sigma^2_{i})$ and $\phi(\theta_{i}\mid\theta_0,\tau^2)$. But researchers may choose to $p$-hack at a different $\alpha$, for example at $\alpha_1 = 0.025$ with probability $\pi_1=  0.1$, at $\alpha_2 = 0.05$ with probability $\pi_2 = 0.7$ and no-hack ($\alpha_3 = 1$) with probability $\pi_3 = 0.2$. So the density from which our 10 studies will be in this case a mixture of the three truncated Gaussian, with mixing distribution $\omega(\alpha) = 0.2 \cdot 1(\alpha = 0.025) + 0.7 \cdot 1(\alpha = 0.05) + 0.2 \cdot 1(\alpha = 1)$. Also in this case we cannot know the probabilities $\pi_1$, $\pi_2$ and $\pi_3$ in advance and we need to estimate them from the data. Here the ideal situation, no $p$-hacking, is $\pi_1 = \pi_2 = 0$ and $\pi_3 = 1$.

The main difference between the models is then related to $\theta_i$. The publication bias mechanism affects $\phi(x_{i}\mid\theta_i,\sigma^2_{i})$ once the study is done, i.e., we have already observed an instance of $\theta_i$. If the study is not selected, we generate a new $\theta_i$, i.e., we make a new study until we reach out 10. In contrast, in the $p$-hacking mechanism all studies are retained, just modified, so we keep all the generated $\theta_i$. This difference affects the distribution of $\theta_i$, $\phi(\theta_{i}\mid\theta_0,\tau^2)$. If $\theta_i = \theta$ in all studies (fixed effect meta-analysis), it does not matter.


% Created by Matthew Riehl  matthew.e.riehl@gmail.com
% Copyright (c) <2021>, <matthew.e.riehl@gmail.com>
% All rights reserved.

% This source code is licensed under the MIT license found in the
% README file in the root directory of this source tree.
%

\documentclass[]{article}
\usepackage{fullpage}
\usepackage{tikz}
\usepackage{amsmath}
\usepackage{pgfplots}
\pgfplotsset{compat=newest}
\title{Projectile Motion}
\author{Dr. Matthew Riehl}
\date {17 August 2021}

\begin{document}

	\section*{Projectile Motion}
	Motion of object with $x$-component and $y$-component which are independent of each other.  Can  be described with parametric equations ($x=f(t)\ \text{and }y=f(t)$).  Initial velocity is: $$\vec{v}_0=v_{0x}\hat\i+v_{0y}\hat\j$$ and $x$- and $y$-components are $$v_{x0}=v_0\cos{\theta _0}\text{ and }v_{y0}=v_0\sin{\theta _0}$$ where $\theta _0$ is the angle between $v_0$ and the positive $x$ direction.

	\begin{center}
		\begin{tikzpicture}[scale=1]
			\draw[very thin,color=gray!50] (0,0) grid (4,4);
			\draw[->, blue] (0,0) -- (4,0) node[below right] {$x$};
			\draw[->, blue] (0,0) -- (0,4) node[left] {$y$};
			\draw[-stealth,red, line width =2pt] (0,0) -- (3.7,2.8) node[right]{$v_0$};
			\draw[-stealth, line width = 1.5pt, green!60!black] (-.1,0) -- (-.1,2.8) node[left]{$v_{0y}$};
			\draw[-stealth, line width = 1.5pt, yellow!60!black] (0,-.1) -- (3.7,-.1) node[below]{$v_{0x}$};
			\draw[<->, red, thick] (1,0) arc (0:37.12:1cm) node[below right]{\ \footnotesize $\theta=37\,^\circ$};
		\end{tikzpicture}
	\end{center}

	Motion is split into $x$-component and $y$-component.  Typically, $x$ is horizontal (no acceleration) and $y$ is vertical (acceleration due to gravity, $a=-9.81\,\text{m/s}^2$).  If $v_0=12\,\text{m/s}$ and $\theta=37^\circ$,  the motion is:

		\begin{tikzpicture}[scale=1]
			\draw[very thin,color=gray!50,step=1cm] (0,0) grid (15.5,3.5);
			\draw[->, blue] (0,0) -- (15.5,0) node[below right] {$x$};
			\draw[->, blue] (0,0) -- (0,3.5) node[left] {$y$};
			\node[below left] at (0,0) {\footnotesize $0$};
			\foreach \x in {1,2,...,15}
			\draw[shift={(\x,0)}] (0pt,2pt) -- (0pt,-2pt) node[below] {\footnotesize $\x$};
			\foreach \x in {1,2,3}
			\draw[shift={(0,\x)}] (2pt, 0pt) -- (-2pt,0pt) node[left] {\footnotesize $\x$};
			\draw[red, thick, domain=0:1.48, samples=100] plot ({12*cos(37.12)*\x}, {-0.5*9.81*\x^2+12*sin(37.12)*\x});

			\foreach \x in {0,.1,.2,...,1.4,1.48}
			{
				\node at ({12*cos(37.12)*\x}, {-0.5*9.81*\x^2+12*sin(37.12)*\x})  {\footnotesize \textcolor{red}{\textbullet}};

				\draw[-stealth,green!80!black,line width=1.5pt] ({12*cos(37.12)*\x}, {-0.5*9.81*\x^2+12*sin(37.12)*\x}) -- ({12*cos(37.12)*\x+1.2*cos(37.12)}, {-0.5*9.81*\x^2+12*sin(37.12)*\x});

				\draw[-stealth,yellow!80!black,line width=1.5pt] ({12*cos(37.12)*\x}, {-0.5*9.81*\x^2+12*sin(37.12)*\x}) -- ({12*cos(37.12)*\x}, {(-0.5*9.81*\x^2+12*sin(37.12)*\x)+0.1*(-9.81*\x+12*sin(37.12))});
			}
		\end{tikzpicture}
		where the arrows represent the \colorbox{pink!50}{\emph{velocity}} in the $x$- and $y$-directions.  The parametric equations for \colorbox{pink!50}{\emph{position} vs. time} are: $$\vec{x}=(12\cos\theta_0) t$$ and $$\vec{y}=(12\sin\theta_0) t-\frac{1}{2}\cdot9.81 t^2$$ where $\theta_0=37^\circ$
	\vspace{3cm}
	\newpage
	This is the same picture, but with different code. This requires the pgfplots package while the first requires only tikz.\\
	\begin{tikzpicture}
		\begin{axis}[width=15cm, height=5cm,xlabel={$x$}, ylabel={$y$},grid, thick,ymin=-1,
			ymax=3,
			xmin=-1,
			xmax=15.5]

			\addplot+[no marks,red,smooth,variable=t,domain=0:1.48]plot ({12*cos(37.12)*t}, {-0.5*9.81*t^2+12*sin(37.12)*t});

			\foreach \yValue in {0,.1,.2,...,1.4,1.48} {
				\edef\temp{\noexpand\draw [-stealth,green!80!black,line width=1.5pt] (axis cs:9.568*\yValue,-0.5*9.81*\yValue^2+12*0.6035*\yValue) -- (axis cs:1.2*.9586+9.568*\yValue,-0.5*9.81*\yValue^2+12*0.6035*\yValue);}
				\temp

				\edef\temp{\noexpand\draw[-stealth,yellow!60!black,line width=1.5pt] (12*.7974*\yValue, -0.5*9.81*\yValue^2+12*.6035*\yValue) -- (12*.7974*\yValue, -0.5*9.81*\yValue^2+12*.6035*\yValue+0.1*(-9.81*\yValue+12*.6035);}
				\temp
			}
		\end{axis}

	\end{tikzpicture}
\end{document}


\subsubsection{Proof of Safety}

The proof of safety relies on this observation from
Lemma~\ref{lem:threshold_subsets}: two subsets with at least
a threshold number of validators in each subset will share at least one
honest validator.

\begin{lem}
\label{lem:threshold_subsets}
Let $f$ be the number of malicious validators.
Given any two subsets with at least $2f+1$ validators in a system
containing $3f+1$ validators, those two subsets share at least $f+1$ validators.
Thus, these two subsets share at least one honest validator.
\end{lem}

\begin{proof}
Let $A$ and $B$ be two subsets with at least $2f+1$ validators
chosen from $N = 3f+1$ validators.
From De Morgan’s laws, we know

\begin{equation*}
    A \cap B = (A^{c} \cup B^{c})^{c}.
\end{equation*}

\noindent
Here, $A^{c}$ denotes the complement of $A$ (that is, the elements of the
system not in A).
In what follows, $\abs{A}$ denotes the number of elements in $A$.
Thus, we have

\begin{align*}
    \abs{A\cap B} &= \abs{(A^{c} \cup B^{c})^{c}} \\
        &= N - \abs{A^{c} \cup B^{c}} \\
        &\ge N - \parens{\abs{A^{c}} + \abs{B^{c}}} \\
        &\ge N - 2f \\
        &= f + 1.
\end{align*}

\noindent
Their intersection shares at least $f+1$ validators and there are $f$
malicious validators, so we see that $A$ and $B$ share at least one honest
validator.
\end{proof}

This lemma ensures that honest participants will always agree when
PreCommitting a proposal or submitting a NextHeight message in a given
round, as the next two lemmas show.

\begin{lem}
\label{lem:2_honest_precommit}
In any round, any 2 honest participants who PreCommit a proposal will PreCommit
the same proposal.
\end{lem}

\begin{proof}
Suppose two honest participants PreCommit proposals $P_{1}$ and $P_{2}$.
To PreCommit a proposal, they both must have knowledge of at least $2f+1$
PreVote messages from participants.
There corresponds subsets $S_{1}$ and $S_{2}$, where $S_{1}$ contains
the participants who submitted prevotes for $P_{1}$ and $S_{2}$ contains
the participants who submitted PreVotes for $P_{2}$.
By Lemma~\ref{lem:threshold_subsets},
$S_{1}$ and $S_{2}$ share at least one honest participant.
An honest participant will only PreVote for one value in a round, so the
proposals $P_{1}$ and $P_{2}$ must agree.
\end{proof}

\begin{lem}
\label{lem:2_honest_nextheight}
In any round, any 2 honest participants who submit a NextHeight message
will submit a NextHeight message for the same proposal.
\end{lem}

\begin{proof}
Mutatis mutandis, the proof is the same as that of
Lemma~\ref{lem:2_honest_precommit}.
\end{proof}

We are now able to show the multiple distinct NextHeight messages from honest
participants are not able to occur before the DeadBlockRound.
This is a stronger result than Lemma~\ref{lem:2_honest_nextheight}
and follows from the fact that a
threshold number of LockedValues must occur before submitting a NextHeight
message.

\begin{lem}
\label{lem:2_distinct_nextheight}
In any round before the DeadBlockRound, any 2 honest participants
who submit a NextHeight message will submit a NextHeight message
for the same proposal.
\end{lem}

\begin{proof}
Suppose we have not yet reached the DeadBlockRound.
Let $P_{1}$ and $P_{2}$ be NextHeight messages from two honest participants
with corresponding signing subsets $S_{1}$ and $S_{2}$.
When $P_{1}$ is signed, all members of $S_{1}$ are supposed to set
LockedValue to the proposal corresponding to $P_{1}$ because they
PreCommitted $P_{1}$.
Similarly, all members of $S_{2}$ are supposed to set LockedValue
to the proposal corresponding to $P_{2}$ because they PreCommitted $P_{2}$.
This implies $S_{1}$ and $S_{2}$ are two subsets of participants of size
at least $2f+1$ with LockedValue set.
Once an honest participant sets his LockedValue, it is never unset before the
DeadBlockRound.
By Lemma~\ref{lem:threshold_subsets},
$S_{1}$ and $S_{2}$ have at least one honest participant in common
with his LockedValue set to one proposal.
Thus, we see that the $P_{1}$ and $P_{2}$ NextHeight messages must be
for the same proposal when they are submitted by honest participants.
\end{proof}

Lemma~\ref{lem:honest_signed_nextround}
is useful in bounding the number of honest validators who may not
proceed to the next round.

\begin{lem}
\label{lem:honest_signed_nextround}
In order for $2f+1$ validators to enter a round, at least $f+1$
honest validators must have signed a NextRound message and at most $f$
honest validators failed to sign a NextRound message.
\end{lem}

\begin{proof}
We recall an honest validator may not enter a round without a valid
RoundCertificate.
A NextRound message contains two objects.
The first object is a RoundCertificate for the current round.
The second is a RoundShare object for the next round.
A RoundCertificate contains both round number and block height as internal
fields, and in order to form a valid Round Certificate, at least $2f+1$
validators must have signed a RoundShare.
Because there are at most $f$ dishonest validators who signed for the
RoundCertificate, at least $f+1$ honest validators must have also signed the
RoundCertificate.
Thus, at most $f$ honest validators did not sign the RoundCertificate.
\end{proof}

Once validators reach the DeadBlockRound, they will not acknowledge any
NextHeight messages for any preceding round.
This ensures that dishonest validators are not able to fork the chain by
producing a block based on those NextHeight messages.

\begin{lem}
\label{lem:sign_rc_dbr}
If at least $2f+1$ participants sign the RoundCertificate to the DeadBlockRound,
then it is not possible for $2f+1$ participants to form a valid NextHeight
message for any round preceding the DeadBlockRound.
\end{lem}

\begin{proof}
Given Lemma~\ref{lem:honest_signed_nextround}
and the rule that any honest validator who has signed a
NextRound message for the DeadBlockRound will never sign or acknowledge any
NextHeight message from a previous round, at least $f+1$ honest validators must
have signed a NextRound message for the DeadBlockRound if there exists a
RoundCertificate for the DeadBlockRound.
If at least $f+1$ honest validators are in the DeadBlockRound,
at most $f$ honest validators may remain in a round preceding
the DeadBlockRound.
If at most $f$ honest validators remain in a preceding round, then the malicious
validators are unable to use the signatures of $f$ honest validators to form a
set of $2f+1$ valid NextHeight messages.
Therefore, the malicious validators may not form a block from those NextHeight
messages.
\end{proof}

With this assurance, we allow for participants to safely proceed to the
DeadBlockRound even if they previously were locked onto a NextHeight message.

\begin{lem}
\label{lem:safe_unlock_nh_dbr}
It is safe for any participant who is locked on a NextHeight message to unlock
and proceed to the DeadBlockRound upon receiving a RoundCertificate for the
DeadBlockRound.
\end{lem}

\begin{proof}
This follows from Lemma~\ref{lem:sign_rc_dbr}.
\end{proof}

The previous work allows us to show that we will converge to the EmptyBlock
when a RoundCertificate for the DeadBlockRound exists.
This ensures new blocks will be created even if no transactions are performed.

\begin{lem}
\label{lem:emptyblock_rc_dbr}
The DeadBlockRound must converge to the EmptyBlock if a RoundCertificate for
the DeadBlockRound exists.
\end{lem}

\begin{proof}
Upon entering the DeadBlockRound, every valid process will immediately PreVote
the EmptyBlock and ignore all other contradicting votes.
These contradicting votes include any PreVote for a Proposal that is not the
EmptyBlock, any NextRound message, any PreCommit that is not a PreCommit for
the EmptyBlock, and any PreVoteNil or PreCommitNil message as well.
As a result of Lemma~\ref{lem:sign_rc_dbr},
at least $f+1$ honest validators enter the DeadBlockRound.
Therefore, the honest validators who enter the DeadBlockRound will only
progress once at least $f$ other validators also PreVote in the DeadBlockRound.
Because we assume all messages are eventually received, the at most $f$ honest
validators who did not sign the NextRound Certificate for the DeadBlockRound
will eventually receive the RoundCertificate for the DeadBlockRound and PreVote
for the EmptyBlock in the DeadBlockRound.
It follows that the round eventually converges to the EmptyBlock and no other
possible block.
\end{proof}

The previous work also allows us to show we will not converge to more
than one valid block at a given block height.

\begin{lem}
It is not possible for our system to converge to more than one valid block for
any given block height.
\end{lem}

\begin{proof}
If the round enters the DeadBlockRound, then Lemma~\ref{lem:emptyblock_rc_dbr}
shows that we will converge to the EmtpyBlock.
Lemma~\ref{lem:2_distinct_nextheight}
proves that it is not possible to have more than one proposal for which
an associated NextHeight message has been validly formed before the
DeadBlockRound.
Lemma~\ref{lem:safe_unlock_nh_dbr}
allows for a safe transition into the DeadBlockRound.
\end{proof}

The next few lemmas assure the behavior of honest validators as it relates to
voting.

\begin{lem}
\label{lem:honest_must_prevote}
An honest validator who enters a round must eventually PreVote or PreVoteNil in
that round.
\end{lem}

\begin{proof}
All honest validators will either PreVote or PreVoteNil at the termination of
the ProposalTimeout.
Therefore, they must eventually prevote.
\end{proof}

\begin{lem}
\label{lem:2fp1_precommit}
As long as $2f+1$ honest validators enter a round, there will be at least
$2f+1$ PreCommits or PreCommitNils in that round.
\end{lem}

\begin{proof}
By Lemma~\ref{lem:honest_must_prevote},
all honest validators will eventually PreVote or PreVoteNil in a round.
In the event that a PreVote is received for a competing Proposal from the
perspective of a validator who has already PreVoted, that validator will count
this PreVote as a PreVoteNil.
From there, the honest validators will be able to either PreCommit or
PreCommitNil, thus leading to $2f+1$ PreCommits or PreCommitNils.
\end{proof}

\begin{lem}
\label{lem:2fp1_nextround}
As long as $2f+1$ honest validators enter a round, there will be at least
$2f+1$ NextRound or NextHeight messages in that round.
\end{lem}

\begin{proof}
Mutatis mutandis, the proof is the same as that of
Lemma~\ref{lem:2fp1_precommit}.
\end{proof}

We now show that a round must terminate or a higher block is formed.

\begin{lem}
\label{lem:valid_rouncert_exists}
If at any time a valid RoundCertificate exists for a round, that round must
eventually terminate or a higher block must be formed.
\end{lem}

\begin{proof}
We first focus on the case when no validator has signed a DeadBlockRound
RoundCertificate.
In this case, we may have that $f+1$ honest validators have signed a NextHeight
message and the round cannot terminate but a new block will be formed because
the validators will eventually observe the previous NextHeight messages and
will follow them.
Otherwise, at least $f+1$ honest validators will have entered the current round
and it must eventually terminate; we will fall back to the previous case if any
of these validators observe a NextHeight message.

In the case that a DeadBlockRound RoundCertificate exists, we have already
proven termination by Lemma~\ref{lem:emptyblock_rc_dbr}.
\end{proof}

Taken together, we are now able to show that our blockchain will make forward
progress provided there are a limited number of faults.

\begin{lem}
Our blockchain will always make forward progress so long as there are no more
than $f$ faults in the system.
\end{lem}

\begin{proof}
If there are at most $f$ faults, then all rounds must terminate or a new block
will be formed by Lemma~\ref{lem:valid_rouncert_exists}.
\end{proof}


\documentclass[a4paper]{article}
\usepackage[a4paper,margin=25mm]{geometry}
\usepackage{graphicx,subcaption}
\usepackage{amsmath,amsfonts}
\usepackage{amssymb}
\usepackage{accents}

\newcommand{\rvec}[1]{\accentset{\leftarrow}{#1}}
\renewcommand{\v}[1]{\mathbf{#1}}
\renewcommand{\c}[1]{{\cal #1}}

\title{Gaussian Restricted Boltzmann Classifier}
\author{G.A. Jarrad}

\begin{document}
\maketitle
\numberwithin{equation}{section}
\numberwithin{figure}{section}
\numberwithin{table}{section}
\section{Definition}\label{sec:intro}
Consider a restricted Boltzmann machine (RBM) with a real-valued input layer,
a binary-valued hidden layer, and a binary-valued output layer, as shown in
Figure~\ref{fig:rbm}.
A suitable energy function is given by
\begin{eqnarray}
    E(\v{x},\v{h},\v{y}; \Theta) & = & \frac{1}{2}\|\v{x}-\v{a}\|^2
    - \v{b}^T\v{h} - \v{h}^T W\v{x} - \v{c}^T\v{y} - \v{h}^T U\v{y}\,,
\label{eq:energy}
\end{eqnarray}
with input feature vector $\v{x}=(x_1,x_2,\ldots,x_F)\in\mathbb{X}\subseteq\mathbb{R}^F$,
hidden binary vector $\v{h}=(h_1,h_2,\ldots,h_H)\in\mathbb{H}=\{0,1\}^H$, and output
binary vector $\v{y}=(y_1,y_2,\ldots,y_C)\in\mathbb{Y}=\{0,1\}^C$.
The model parameters are $\Theta=(\v{a}, \v{b}, \v{c}, W, U)$.
The joint probability of $\v{x}$, $\v{y}$ and $\v{h}$ is then
\begin{eqnarray}
    p(\v{x},\v{h},\v{y}\;|\;\Theta) & = & \frac{e^{-E(\v{x},\v{h},\v{y}; \Theta)}}
{\int_{\mathbb{X}}\sum_{\v{h}'\in\mathbb{H}}\sum_{\v{y}'\in\mathbb{Y}}
e^{-E(\v{x}',\v{h}',\v{y}'; \Theta)}\,d|\v{x}'|
}
\,,
\end{eqnarray}
which is intractible to compute in general.

In order to turn the RBM into a restricted Boltzmann classifier (RBC), let us now suppose that the binary vector $\v{y}$
is really a one-in-$C$ vector of $C-1$ zeros and a single one, restricted to the set
$\mathbb{Y}'=\{\v{y}\in\mathbb{Y}\;|\;\sum_{k=1}^{C}y_k=1\}$. Then there is a one-to-one correspondence between each
vector $\v{y}\in\mathbb{Y}'$ and some scalar $y\in\{1,2,\ldots,C\}$, such that, for example,
the term $U\v{y}$ selects the $y$-th column of $U$, denoted by $\v{u}_y$.
Hence we obtain a final mapping to a multinomial output, suitable for a classifier.
The joint probability then becomes
\begin{eqnarray}
     p(\v{x},\v{h},y\;|\;\Theta) &=&
\frac{
    e^{-\frac{1}{2}\|\v{x}-\v{a}\|^2+\v{b}^T\v{h}+\v{h}^T W\v{x}+c_y+\v{h}^T \v{u}_y}
}
{\int_{\mathbb{X}}\sum_{y'=1}^{C}\sum_{\v{h}'\in\mathbb{H}}
   e^{-\frac{1}{2}\|\v{x}'-\v{a}\|^2+\v{b}^T\v{h}'+\v{h}'^T W\v{x}'+c_{y'}+\v{h}^T \v{u}_{y'}}
   \,d|\v{x}'|
}
\nonumber\\&=&
\frac{
    e^{c_y-\frac{1}{2}\|\v{x}-\v{a}\|^2}
\prod_{i=1}^{H}e^{h_i(b_i+ \v{w}_i^T\v{x}+u_{iy})}
}
{\int_{\mathbb{X}}\sum_{y'=1}^{C}
   e^{c_{y'}-\frac{1}{2}\|\v{x}'-\v{a}\|^2}
  \prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}+u_{iy}}\right]
  \,d|\v{x}'|
}
\,,
\end{eqnarray}
where $\v{w}_i^T$ is the $i$-th row of $W$.
The discriminative form of the RBC can then be specified as
\begin{eqnarray}
    p(y\;|\;\v{x},\Theta) & = &
\frac{\sum_{\v{h}'\in\mathbb{H}}p(\v{x},\v{h}',y\;|\;\Theta)}
{\sum_{y'=1}^{C}\sum_{\v{h}'\in\mathbb{H}}p(\v{x},\v{h}',y'\;|\;\Theta)}
\nonumber\\&=&
\frac{
    e^{c_y-\frac{1}{2}\|\v{x}-\v{a}\|^2}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}+u_{iy}}\right]
}
{\sum_{y'=1}^{C}
   e^{c_{y'}-\frac{1}{2}\|\v{x}-\v{a}\|^2}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}+u_{iy'}}\right]
}
\nonumber\\&=&
\frac{
   e^{c_y}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}+u_{iy}}\right]
}
{
\sum_{y'=1}^{C}
e^{c_{y'}}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}+u_{iy'}}\right]
}
\,,
\label{eq:p_y_x}
\end{eqnarray}
which is a nonlinear form of logistic classifier.

Now, the bipartite restriction depicted in Figure~\ref{fig:rbm} ensures that $\v{x}$ and $y$
are conditionally independent given $\v{h}$.
Observe, for instance, that
\begin{eqnarray}
   p(\v{x}\;|\;\v{h},y,\Theta) & = &
\frac{ p(\v{x},\v{h},y\;|\;\Theta)}
{ \int_{\mathbb{X}}p(\v{x}',\v{h},y\;|\;\Theta)\,d|\v{x}'|}
\nonumber\\&=&
\frac{
    e^{-\frac{1}{2}\|\v{x}-\v{a}\|^2+\v{b}^T\v{h}+\v{h}^T W\v{x}+c_y+\v{h}^T \v{u}_y}
}
{
\int_{\mathbb{X}}
    e^{-\frac{1}{2}\|\v{x}'-\v{a}\|^2+\v{b}^T\v{h}+\v{h}^T W\v{x}'+c_y+\v{h}^T \v{u}_y}
\,d|\v{x}'|
}
\nonumber\\&=&
\frac{
    e^{-\frac{1}{2}\|\v{x}-\v{a}\|^2+\v{h}^T W\v{x}}
}
{
\int_{\mathbb{X}}
    e^{-\frac{1}{2}\|\v{x}'-\v{a}\|^2+\v{h}^T W\v{x}'}
\,d|\v{x}'|
}
\nonumber\\&=&
\frac{
    e^{-\frac{1}{2}\|\v{x}-\v{a}-W^T\v{h}\|^2+\v{h}^T W\v{a}+\frac{1}{2}\v{h}^T WW^T \v{h}}
}
{
    \int_{\mathbb{X}}
    e^{-\frac{1}{2}\|\v{x}-\v{a}-W^T\v{h}\|^2+\v{h}^T W\v{a}+\frac{1}{2}\v{h}^T WW^T \v{h}}
    \,d|\v{x}'|
}
\nonumber\\&=&
\frac{
    e^{-\frac{1}{2}\|\v{x}-\v{a}-W^T\v{h}\|^2}
}
{
    \int_{\mathbb{X}}e^{-\frac{1}{2}\|\v{x}-\v{a}-W^T\v{h}\|^2}\,d|\v{x}'|
}
\nonumber\\&=& N(\v{x}\;|\;\v{a}+W^T\v{h},I)\,.
\end{eqnarray}
Hence, $\v{x}$ is conditionally normally distributed with mean $\v{a}+W^T\v{h}$ and unit spherical
variance $I$ (the identity matrix).

Similarly, observe that
\begin{eqnarray}
    p(y\;|\;\v{x},\v{h},\Theta) & = &
\frac{p(\v{x},\v{h},y\;|\;\Theta)}
{\sum_{y'=1}^{C}p(\v{x},\v{h},y'\;|\;\Theta)}
\nonumber\\&=&
\frac{
    e^{-\frac{1}{2}\|\v{x}-\v{a}\|^2+\v{b}^T\v{h}+\v{h}^T W\v{x}+c_y+\v{h}^T\v{u}_y}
}
{
   \sum_{y'=1}^{C}
    e^{-\frac{1}{2}\|\v{x}-\v{a}\|^2+\v{b}^T\v{h}+\v{h}^T W\v{x}+c_{y'}+\v{h}^T\v{u}_{y'}}
}
\nonumber\\&=&
\frac{e^{c_y+\v{h}^T\v{u}_y}}
  {\sum_{y'=1}^{C}e^{c_{y'}+\v{h}^T \v{u}_{y'}}}\,.
\end{eqnarray}
This result is just the {\em soft-max} function, or standard logistic classifier.

Conversely, $\v{h}$ depends upon both $\v{x}$ and $\v{y}$ via
\begin{eqnarray}
    p(\v{h}\;|\;\v{x},y,\Theta) & = &
    \frac{p(\v{x},\v{h},y\;|\;\Theta)}
{\sum_{\v{h}'\in\mathbb{H}}p(\v{x},\v{h}',y\;|\;\Theta)}
\nonumber\\&=&
\frac{
    e^{c_y-\frac{1}{2}\|\v{x}-\v{a}\|^2}
\prod_{i=1}^{H}e^{h_i(b_i+ \v{w}_i^T\v{x}+u_{iy})}
}
{
   e^{c_{y}-\frac{1}{2}\|\v{x}-\v{a}\|^2}
  \prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}+u_{iy}}\right]
}
\nonumber\\&=&
\frac{
\prod_{i=1}^{H}e^{h_i(b_i+ \v{w}_i^T\v{x}+u_{iy})}
}
{
  \prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}+u_{iy}}\right]
}
\nonumber\\&=&
  \prod_{i=1}^{H}p(h_i\;|\;\v{x},y,\Theta)
\,,
\end{eqnarray}
where
\begin{eqnarray}
  p(h_i=1\;|\;\v{x},y,\Theta) & = &
  \frac{e^{b_i+\v{w}_i^T\v{x}+u_{iy}}}{1+e^{b_i+\v{w}_i^T\v{x}+u_{iy}}}
~=~\sigma(b_i+\v{w}_i^T\v{x}+u_{iy})
  \,.
\label{eq:ph1}
\end{eqnarray}
This is just the logistic sigmoid function.

\section{Supervised Discriminative Optimisation}
Consider the problem of estimating the RBC parameters $\Theta$ from a data-set of fully labelled feature vectors,
$\c{X}=(\v{x}_1,\v{x}_2,\ldots,\v{x}_N)$, with corresponding labels $\c{Y}=(y_1,y_2,\ldots,y_N)$.
Assuming that the data items are independent, the discriminative likelihood is given by
\begin{eqnarray}
  p(\c{Y}\;|\;\c{X},\Theta) & = & \prod_{d=1}^N p(y_d\;|\;\v{x}_d,\Theta)
\,,
\end{eqnarray}
 and hence, from equation~\eqref{eq:p_y_x}, the average discriminative log-likelihood is given by
\begin{eqnarray}
  \c{L}_{\c{Y}|\c{X}}(\Theta) & = & \frac{1}{N}\ln p(\c{Y}\;|\;\c{X},\Theta)
~=~
\frac{1}{N}\sum_{d=1}^N \ln p(y_d\;|\;\v{x}_d,\Theta)
\nonumber\\& = &
\frac{1}{N}\sum_{d=1}^N\left\{
  \sum_{y'=1}^C \delta_{y',y_d}\ln\left(
   e^{c_{y'}}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}_d+u_{iy'}}\right]\right)\right.
\nonumber\\&&
{}-\left.\ln\sum_{y'=1}^{C}
e^{c_{y'}}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}_d+u_{iy'}}\right]
\right\}
\nonumber\\& = &
\frac{1}{N}\sum_{d=1}^N\left\{
  \sum_{y'=1}^C \delta_{y',y_d}\left(
   c_{y'}+\sum_{i=1}^{H}\ln\left[1+e^{b_i+ \v{w}_i^T\v{x}_d+u_{iy'}}\right]\right)\right.
\nonumber\\&&
{}-\left.\ln\sum_{y'=1}^{C}
e^{c_{y'}}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}_d+u_{iy'}}\right]
\right\}\,.
\end{eqnarray}
Hence, the gradient with respect to $c_y$ is
\begin{eqnarray}
\frac{\partial\c{L}_{\c{Y}|\c{X}}}{\partial c_y}
& = &
\frac{1}{N}\sum_{d=1}^N\left\{
\delta_{y,y_d}-
\frac{e^{c_{y}}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}_d+u_{iy}}\right]}
{\sum_{y'=1}^{C}
e^{c_{y'}}\prod_{i=1}^{H}\left[1+e^{b_i+ \v{w}_i^T\v{x}_d+u_{iy'}}\right]}
\right\}
\nonumber\\&=&
\frac{1}{N}\sum_{d=1}^N\left\{
\delta_{y,y_d}-p(y\:|\;\v{x}_d,\Theta)
\right\}
\nonumber\\&=&
\frac{N_y}{N}-\frac{1}{N}\sum_{d=1}^N p(y\:|\;\v{x}_d,\Theta)
\,,
\end{eqnarray}
where $N_y$ is the number of data labelled with class $y$.

In order to develop the remaining derivatives, we first observe that
\begin{eqnarray}
\frac{\partial}{\partial\theta_i}\left(
c_{y}+\sum_{i'=1}^{H}\ln\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y}}\right]
\right)
&=&
\frac{\partial}{\partial\theta_i}\ln\left[1+e^{b_{i}+ \v{w}_{i}^T\v{x}_d+u_{iy}}\right]
\nonumber\\&=&
\frac{e^{b_{i}+ \v{w}_{i}^T\v{x}_d+u_{iy}}}
{1+e^{b_{i}+ \v{w}_{i}^T\v{x}_d+u_{iy}}}
\frac{\partial}{\partial\theta_i}(b_{i}+ \v{w}_{i}^T\v{x}_d+u_{iy})
\nonumber\\&=&
p(h_i=1\;|\;\v{x}_d,y,\Theta)
\frac{\partial}{\partial\theta_i}(b_{i}+ \v{w}_{i}^T\v{x}_d+u_{iy})\,,
\end{eqnarray}
from equation~\eqref{eq:ph1},
and then use the fact that
$\nabla f(\theta)=f(\theta)\nabla\ln f(\theta)$ to deduce that
\begin{eqnarray}
&\frac{\partial}{\partial\theta_i}e^{c_{y}}\prod_{i'=1}^{H}\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y}}\right]&
\nonumber\\&=&\hspace*{-25mm}
e^{c_{y}}\prod_{i'=1}^{H}\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y}}\right]
\frac{\partial}{\partial\theta_i}\left(
c_{y}+\sum_{i'=1}^{H}\ln\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y}}\right]
\right)
\nonumber\\&=&\hspace*{-25mm}
e^{c_{y}}\prod_{i'=1}^{H}\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y}}\right]
p(h_i=1\;|\;\v{x}_d,y,\Theta)
\frac{\partial}{\partial\theta_i}(b_{i}+ \v{w}_{i}^T\v{x}_d+u_{iy})\,.
\end{eqnarray}
Hence, the gradient of the log-likelihood with respect to $u_{iy}$ is
\begin{eqnarray}
\frac{\partial\c{L}_{\c{Y}|\c{X}}}{\partial u_{iy}}
& = &
\frac{1}{N}\sum_{d=1}^N\left\{\rule{0pt}{5mm}
\delta_{y,y_d}\,p(h_i=1\;|\;\v{x}_d,y,\Theta)\right.
\nonumber\\&&
{}-\left.\frac{e^{c_{y}}\prod_{i'=1}^{H}\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y}}\right]}
{\sum_{y'=1}^{C}e^{c_{y'}}\prod_{i'=1}^{H}\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y'}}\right]}
p(h_i=1\;|\;\v{x}_d,y,\Theta)
\right\}
\nonumber\\&=&
\frac{1}{N}\sum_{d=1}^N\left\{
\delta_{y,y_d}-p(y\;|\;\v{x}_d,\Theta)\right\}\,p(h_i=1\;|\;\v{x}_d,y,\Theta)
\,.
\end{eqnarray}
Similarly, the gradient of the log-likelihood with respect to $b_{i}$ is
\begin{eqnarray}
\frac{\partial\c{L}_{\c{Y}|\c{X}}}{\partial b_{i}}
& = &
\frac{1}{N}\sum_{d=1}^N\left\{\sum_{y'=1}^{C}
\delta_{y',y_d}\,p(h_i=1\;|\;\v{x}_d,y',\Theta)\right.
\nonumber\\&&
{}-\left.\frac{\sum_{y'=1}^{C}e^{c_{y'}}\prod_{i'=1}^{H}\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y'  }}\right]
\,p(h_i=1\;|\;\v{x}_d,y',\Theta)}
{\sum_{y'=1}^{C}e^{c_{y'}}\prod_{i'=1}^{H}\left[1+e^{b_{i'}+ \v{w}_{i'}^T\v{x}_d+u_{i'y'}}\right]}
\right\}
\nonumber\\&=&
\frac{1}{N}\sum_{d=1}^N\sum_{y'=1}^{C}\left\{
\delta_{y',y_d}-p(y'\;|\;\v{x}_d,\Theta)\right\}\,p(h_i=1\;|\;\v{x}_d,y',\Theta)
\nonumber\\&=&
\sum_{y=1}^{C}\frac{\partial\c{L}_{\c{Y}|\c{X}}}{\partial u_{iy}}
\,,
\end{eqnarray}
and the gradient with respect to $\v{w}_i$ is
\begin{eqnarray}
\frac{\partial\c{L}_{\c{Y}|\c{X}}}{\partial \v{w}_{i}}
& = &
\frac{1}{N}\sum_{d=1}^N\v{x}_d\sum_{y'=1}^{C}\left\{
\delta_{y',y_d}-p(y'\;|\;\v{x}_d,\Theta)\right\}\,p(h_i=1\;|\;\v{x}_d,y',\Theta)
\,.
\end{eqnarray}
Consequently, the discriminative log-likelihood $\c{L}_{\c{Y}|\c{X}}$ can be maximised using standard or accelerated
gradient ascent. Note, however, that the parameter $\v{a}$ from equation~\eqref{eq:energy}
does not appear in the RBC~\eqref{eq:p_y_x}, and therefore cannot be optimised discriminatively.

\end{document}


\documentclass{standalone}

\begin{document}

	\chapter{Differential Equations}

	\section{First order differential equations}
	\subsection{Exact differential equations}
	When solving \textit{exact differential equations} we recognize that one side of the equation is of the form $u\frac{dv}{dx} + v\frac{du}{dx}$ and hence  $\int u\frac{dv}{dx} + v\frac{du}{dx}\,dx$ can be quoted as $uv+k$.

	\subsection{Integrating factor}
	Consider a first order differential equation that can be written in the form $\frac{dy}{dx} + f(x)y + g(x)$. The LHS of the differential equation ia not yet exact, but suppose that it becomes exact when it is multiplied by a function $I(x)$.\\

	Thus, we have the exact differential equation: \[I(x)\left(\frac{dy}{dx} + f(x)y =  g(x)\right)\]

	Which is simplified to: \[I(x)\frac{dy}{dx} + I(x)f(x)y = I(x)g(x)\]

	Comparing the LHS with $v\frac{du}{dx} + u\frac{dv}{dx}$ we have:
	\begin{align*}  v&=I &\quad u&=y\\    \frac{du}{dx} &= I(x)f(x) &\quad \frac{dv}{dx} &= \frac{dy}{dx} \end{align*}

	Which results in:
	\begin{align*}
		\int\frac{1}{I(x)}\,dI(x)      & = \int f(x)\,dx         \\
		\implies \ln\left| I(x)\right| & = \int f(x)\, dx        \\
		\therefore \quad	I(x)          & = e^{\int f(x)\,dx} \qed
	\end{align*}

	\newpage
	\section{Second order differential equations}
	A second order differential equation is one of the form: \[a\frac{d^2y}{dx^2} + b\frac{dy}{dx} + cy= f(x)\] with a general solution: \[y=\text{C.F.} + \text{P.I.}\]

	\subsection{The complementary function}
	To obtain this part of the solution, the general solution of the quadratic equation $ax^2 + bx + c=0$ by comparing it to $a\frac{d^2y}{dx^2} + b\frac{dy}{dx} + cy = 0$ when $f(x)$, or the \textbf{RHS} $= 0$, and thus the two so called roots can be found.

	\begin{center}
		\renewcommand{\arraystretch}{1.6}
		\begin{tabular}{|c|c|}
			\hline
			Roots                                        & General Solution                 \\ \hline
			Two real distinct roots $\alpha$ and $\beta$ & $y=Ae^{\alpha x} + Be^{\beta x}$ \\ \hline
			Two real equal roots $\alpha$ and $\alpha$   & $y=(A+Bx)e^{\alpha x}$           \\ \hline
			Two complex roots $p\pm qi$                  & $y=e^{px}(A\cos qx + B\sin qx)$  \\\hline
		\end{tabular}
	\end{center}

	\subsection{Type 2: $\mathbf{f(x) \neq 0}$}
	When $f(x) \neq 0$ the general solution of such a differential equation is of the form $y=\text{C.F. + P.I.}$, where C.F. is the complimentary function and P.I. is the particular integral.

	The C.F. is obtained by finding the general solution of the differential equation $a\frac{d^2y}{dx^2} + b\frac{dy}{dx} + cy = 0$, similarly to the previous case.\\

	The P.I. is any solution of the given differential equation. It depends on the function $f(x)$ and is usually a general form of it.
	\begin{center}
		\renewcommand{\arraystretch}{1.6}
		\begin{tabular}{|c|c|}
			\hline
			$\mathbf{f(x)}$		& \textbf{Trial Solution}\\\hline
			$5$					& $y=k$\\\hline
			$2x+7$				& $y=px+q$\\\hline
			$3x^2+x-6$			& $y=px^2+qx+r$\\\hline
			$3e{7x}$			& $y=ke^{7x}$\\\hline
			$3xe^{-2x}$     	& $y=(px+q)e^{-2x}$\\\hline
			$2\sin3x + 4\cos3x$ & $y=p\sin3x + q\cos3x$\\\hline
			$\cos8x$ 			& $y=p\sin8x + q\cos8x$\\\hline
			$\sin x - 3\cos2x$  & $y=p\sin x + q\cos x +r\sin2x + t\sin2x$\\\hline
			$2x + 6e^{4x}$		& $y=px+q+ke^{4x}$\\\hline
		\end{tabular}
	\end{center}
	\newpage

	The trial solution is to be chosen according to the above table and differentiated twice. These values are then substitued in the given differential equation, so that the unknown constant/s of the trial solution can be found.\\

	The general solution $y=\text{C.F.} + \text{P.I.}$ is then written.\\

	The unknown constants of the C.F. can be found if additional information is given  (i.e. $y=1$ and $\frac{dy}{dx}=0$ when $x=0$).\\

	In some cases (failure cases) the trial solution listed in the above table leads to an \textit{inconsistent equation}. This usually happens when the trial solution is included in the complementary function. The correct trial solution is obtained by multiplying the trial solution in the above table by $x$ or $x^2$. Such trial solutions are usually given by the question.

\end{document}


%!TEX root = fastZKP.tex

\subsection{GKR Protocol}\label{subsec::GKR}

In~\cite{GKR}, Goldwasser et al. proposed an efficient interactive proof protocol for layered arithmetic circuits, which we use as a building block for our new zero-knowledge argument and is referred as the \emph{GKR} protocol. We present the detailed protocol here.

\subsubsection{Sumcheck Protocol.}
\label{subsec::sumcheck}
The sumcheck problem is a fundamental problem that has various applications. The problem is to sum a polynomial $f: \mathbb{F}^\ell \rightarrow \mathbb{F}$ on the binary hypercube $$\sum\nolimits_{b_1,b_2,\ldots,b_\ell\in\{0,1\}}f(b_1,b_2,...,b_\ell).$$
Directly computing the sum requires exponential time in $\ell$, as there are $2^\ell$ combinations of $b_1,\ldots,b_\ell$. Lund et al.~\cite{sumcheck} proposed a \emph{sumcheck} protocol that allows a verifier $\mathcal{V}$ to delegate the computation to a computationally unbounded prover $\mathcal{P}$, who can convince $\mathcal{V}$ that $H$ is the correct sum. We provide a description of the sumcheck protocol in Protocol~\ref{prot::sumcheck}.
\begin{figure}[t!]
\small{
\centering{\centering
\framebox{\parbox{.99\linewidth}{
\begin{protocol}[\textbf{Sumcheck}]
	\label{prot::sumcheck}
	The protocol proceeds in $\ell$ rounds.
	\begin{itemize}
		\item In the first round, $\P$ sends a univariate polynomial $$f_1(x_1)\overset{def}{=}\sum\limits_{b_2,\ldots,b_\ell\in\{0,1\}}f(x_1,b_2,\ldots,b_\ell)\, ,$$ $\V$ checks $H=f_1(0)+f_1(1)$. Then $\V$ sends a random challenge $r_1\in\mathbb{F}$ to $\P$.
		\item In the $i$-th round, where $2\le i \le l-1$, $\P$ sends a univariate polynomial
		$$f_{i}(x_{i})\overset{def}{=}\sum\limits_{b_{i+1},\ldots,b_\ell\in\{0,1\}}f(r_1,\ldots, r_{i-1}, x_{i}, b_{i+1},\ldots, b_{\ell})\, ,$$
		$\V$ checks $f_{i-1}(r_{i-1})=f_{i}(0)+f_{i}(1)$, and sends a random challenge $r_{i}\in\mathbb{F}$ to $\P$.
		\item In the $\ell$-th round, $\P$ sends a univariate polynomial $$f_{\ell}(x_{\ell})\overset{def}{=}f(r_1, r_2, \ldots, r_{l-1}, x_{\ell})\, ,$$ $\V$ checks $f_{\ell-1}(r_{\ell-1})=f_{\ell}(0)+f_{\ell}(1)$. The verifier generates a random challenge $r_{\ell}\in\mathbb{F}$. Given oracle access to an evaluation $f(r_1, r_2, \ldots, r_\ell)$ of $f$, $\V$ will accept if and only if $f_{\ell}(r_\ell) = f(r_1, r_2, \ldots, r_\ell)$. The instantiation of the oracle access depends on the application of the sumcheck protocol.
	\end{itemize}
\end{protocol}}}}}
\vspace{-0.2in}
\end{figure}
The proof size of the sumcheck protocol is $O(d\ell)$, where $d$ is the variable-degree of $f$, as in each round, $\P$ sends a univariate polynomial of one variable in $f$%\babis{you mean the maximum degree accross all variableS?}\yupeng{formally defined in 2.1}\dawn{minor comment: this sentence can be interpreted wrongly; in each round, P only sends one univariate polynomial, not a univariate polynomial for each variable}
, which can be uniquely defined by $d+1$ points. The verifier time of the protocol is $O(d\ell)$. The prover time depends on the degree and the sparsity of $f$, and we will give the complexity later in our scheme. The sumcheck protocol is complete and sound with $\epsilon = \frac{d\ell}{|\mathbb{F}|}$.

\begin{definition}[\textbf{Multi-linear Extension}]
	Let $V:\{0, 1\}^\ell \rightarrow \mathbb{F}$ be a function. The \textit{multilinear extension} of $V$ is the unique polynomial $\tilde{V}: \mathbb{F}^l \rightarrow \mathbb{F}$ such that $\tilde{V}(x_1, x_2, ..., x_{l}) = V(x_1, x_2, ..., x_{l})$ for all $x_1, x_2, \ldots, x_{l}\in\{0,1\}^l$.

	$\tilde{V}$ can be expressed as:
	$$\tilde{V}(x_1, x_2, ..., x_{l})=\sum\nolimits_{b\in\{0,1\}^\ell}\prod\nolimits_{i=1}^{l}[((1-x_i)(1-b_i)+x_ib_i) \times V(b)]$$
	where $b_i$ is $i$-th bit of b.

\end{definition}

\paragraph{Multilinear extensions of arrays.} Inspired by the close form equation of the multilinear extension given above, we can view an array $\textbf{A} = (a_0, a_1, \ldots, a_{n-1})$ as a function $A: \binary^{\log n}\rightarrow \F$ such that $\forall i\in[0,n-1], A(i) = a_i$. Therefore, in this paper, we abuse the use of multilinear extension on an array as the multilinear extension $\tilde{A}$ of $A$. Using the sumcheck protocol as a building block, Goldwasser et al.~\cite{GKR} showed an interactive proof protocol for layered arithmetic circuits.

\smallskip\noindent\textbf{High Level Ideas.} Let $C$ be a layered arithmetic circuit with depth $d$ over a finite field $\mathbb{F}$. Each gate in the $i$-th layer takes inputs from two gates in the $(i+1)$-th layer; layer $0$ is the output layer and layer $d$ is the input layer. The protocol proceeds layer by layer. Upon receiving the claimed output from $\P$, in the first round, $\V$ and $\P$ run the sumcheck protocol to reduce the claim about the output to a claim about the values in the layer above. In the $i$-th round, both parties reduce a claim about layer $i-1$ to a claim about layer $i$ through the sumcheck protocol. Finally, the protocol terminates with a claim about the input layer $d$, which can be checked directly by $\V$, or is given as an oracle access. If the check passes, $\V$ accepts the claimed output.

\paragraph{Notation.} Before describing the GKR protocol, we introduce some additional notations. We denote the number of gates in the $i$-th layer as $S_i$ and let $s_i = \ceil{\log S_i}$. (For simplicity, we assume $S_i$ is a power of 2, and we can pad the layer with dummy gates otherwise.) We then define a function $V_i:\binary^{s_i}\rightarrow\mathbb{F}$ that takes a binary string $b\in\binary^{s_i}$ and returns the output of gate $b$ in layer $i$, where $b$ is called the gate label. With this definition, $V_0$ corresponds to the output of the circuit, and $V_d$ corresponds to the input layer. Finally, we define two additional functions $add_i, mult_i: \binary^{s_{i-1}+2s_i}\rightarrow\binary$, referred as \emph{wiring predicates} in the literature. $add_i$ ($mult_i$) takes one gate label $z\in\binary^{s_{i-1}}$ in layer $i-1$ and two gate labels $x,y\in\binary^{s_i}$ in layer $i$, and outputs 1 if and only if gate $z$ is an addition (multiplication) gate that takes the output of gate $x,y$ as input. With these definitions, $V_i$ can be written as follows:
\begin{equation}
    \begin{aligned}
	V_i(z)=\sum_{x, y \in\binary^{s_{i+1}}}(add_{i+1}(z,x,y)(V_{i+1}(x)+V_{i+1}(y))\\
	+mult_{i+1}(z,x,y)(V_{i+1}(x)V_{i+1}(y)))
	\end{aligned}
\end{equation}
for any $z\in\binary^{s_i}$.

\ignore{
\yupeng{try to remove $\beta$.}

\begin{definition}[Multilinear extension of identity function]
Here we present a multilinear extension of identity function
\begin{eqnarray}\beta_{l}(a, b)=
	\begin{cases}
	1, &a=b\cr 0, &a\neq b
	\end{cases}
\end{eqnarray}
Where $a, b$ are binary strings with length $l$. The multilinear extension is the following:

$$\tilde{\beta_{l}}(a,b)\overset{def}{=}\prod_{j=1}^{l}((1-a_{j})(1-b_{j})+a_{j}b_{j})$$

We have $\tilde{\beta_{l}}(a,b)=\beta_{l}(a, b)$ when $a, b$ are binary.
\end{definition}

\begin{definition}[Multilinear extension of $add$, $mult$]
We will define the multilinear extension of two component of $V_i$.
$$\tilde{add}_{i}(g, u, v)=\sum_{(g', u', v') \in G_{i, add}}\tilde{\beta}_{s_i}(g, g')\tilde{\beta}_{s_{i+1}}(u, u')\tilde{\beta}_{s_{i+1}}(v, v')$$

$$\tilde{mult}_{i}(g, u, v)=\sum_{(g', u', v') \in G_{i, mult}}\tilde{\beta}_{s_i}(g, g')\tilde{\beta}_{s_{i+1}}(u, u')\tilde{\beta}_{s_{i+1}}(v, v')$$
\end{definition}

\begin{definition}[Multilinear extension of $V_i(g)$]
	\label{def::multilinear}
	%$$\tilde{V}_{i}(z)=\sum_{g\in\{0,1\}^{s_i} u, v\in \{0,1\}^{s_{i+1}}}f_{i,z}(g,u,v)$$

	%$$f_{i,z}(g,u,v)\overset{def}{=}\tilde{\beta_{i}}(z, g)[\tilde{mult}(g, u, v)(\tilde{V}_{i+1}(u)\tilde{V}_{i+1}(v))+\tilde{add}(g,u,v)(\tilde{V}_{i+1}(u)+\tilde{V}_{i+1}(v))]$$
	%where $$\tilde{\beta_{i}}(z,g)\overset{def}{=}\prod_{j=1}^{s_{i}}((1-g_{j})(1-z_{j})+g_{j}z_{j})$$

	%$$\tilde{mult}(g,u,v)\overset{def}{=}\sum_{(g', u', v')\in G_{i,mult}}\tilde{\beta_{i}}(g,g')\tilde{\beta_{i+1}}(u,u')\tilde{\beta_{i+1}}(v,v')$$

	%$$\tilde{add}(g,u,v)\overset{def}{=}\sum_{(g', u', v')\in G_{i,add}}\tilde{\beta_{i}}(g,g')\tilde{\beta_{i+1}}(u,u')\tilde{\beta_{i+1}}(v,v')$$

	%when $a, b$ are binary strings, $\tilde{\beta_{i}}(a, b)$ will output $1$ when $a=b$, and $0$ otherwise. $\tilde{mult},\tilde{add}$ is the multilinear extension of the wiring predicates. A wiring predicates is a function of three gates $g_1, g_2, g_3$, and returns $1$ if $g_1$ takes $g_2, g_3$ as it's input.

	We define the multilinear extension of $V_i(g)$ in the following way:

	$$\tilde{V}_i(g)=\sum_{u, v \in \{0,1\}^{s_{i+1}}}(\tilde{add}_{i+1}(g, u, v)(\tilde{V}_{i+1}(u)+\tilde{V}_{i+1}(v))+\tilde{mult}_{i+1}(g,u,v)(\tilde{V}_{i+1}(u)\tilde{V}_{i+1}(v)))\textnormal{,}$$
	where $\tilde{mult}$ and $\tilde{add}$ are multilinear extension of $mult$ and $add$.
\end{definition}

}

In the equation above, $V_i$ is expressed as a summation, so $\V$ can use the sumcheck protocol to check that it is computed correctly. As the sumcheck protocol operates on polynomials defined on $\mathbb{F}$, we rewrite the equation with their multilinear extensions:

\begin{align}\label{eq:GKR}
\tV_i(g)=&\sum\nolimits_{x, y \in\binary^{s_{i+1}}}f_i(x,y)\nonumber\\
=&\sum\nolimits_{x, y \in\binary^{s_{i+1}}}(\tadd_{i+1}(g,x,y)(\tV_{i+1}(x)+\tV_{i+1}(y))\nonumber\\
&+\tmult_{i+1}(g,x,y)(\tV_{i+1}(x)\tV_{i+1}(y)))\,,
\end{align}
where $g\in\mathbb{F}^{s_i}$ is a random vector.

\paragraph{Protocol.} With Equation~\ref{eq:GKR}, the GKR protocol proceeds as following. The prover $\P$ first sends the claimed output of the circuit to $\V$. From the claimed output, $\V$ defines polynomial $\tV_0$ and computes $\tV_0(g)$ for a random $g\in\mathbb{F}^{s_0}$. $\V$ and $\P$ then invoke a sumcheck protocol on Equation~\ref{eq:GKR} with $i=0$. As described in Section~\ref{subsec::sumcheck}, at the end of the sumcheck, $\V$ needs an oracle access to $f_i(u,v)$, where $u,v$ are randomly selected in $\mathbb{F}^{s_{i+1}}$. To compute $f_i(u,v)$, $\V$ computes $\tadd_{i+1}(u,v)$ and $\tmult_{i+1}(u,v)$ locally (they only depend on the wiring pattern of the circuit, but not on the values), asks $\P$ to send $\tV_1(u)$ and $\tV_1(v)$ and computes $f_i(u,v)$ to complete the sumcheck protocol. In this way, $\V$ and $\P$ reduces a claim about the output to two claims about values in layer 1. $\V$ and $\P$ could invoke two sumcheck protocols on $\tV_1(u)$ and $\tV_1(v)$ recursively to layers above, but the number of claims and the sumcheck protocols would increase exponentially in $d$.

\smallskip\noindent\textbf{Combining two claims: condensing to one claim.} In~\cite{GKR}, Goldwasser et al. presented a protocol to reduce two claims $\tV_i(u)$ and $\tV_i(v)$ to one as following. $\V$ defines a line $\gamma: \mathbb{F} \rightarrow \mathbb{F}^{s_i}$ such that $\gamma(0)=u, \gamma(1)=v$. $\V$ sends $\gamma(x)$ to $\P$. Then $\P$ sends $\V$ a degree $s_i$ univariate polynomial $h(x)=\tilde{V_i}(\gamma(x))$. $\V$ checks that $h(0)=\tV_i(u), h(1)=\tV_i(v)$. Then $\V$ randomly chooses $r\in\mathbb{F}$ and computes a new claim $h(r) = \tV_i(\gamma(r)) = \tV_i(w)$ on $w=\gamma(r) \in \mathbb{F}^{s_i}$. $\V$ sends $r, w$ to $\P$. In this way, the two claims are reduced to one claim $\tV_i(w)$. Combining this protocol with the sumcheck protocol on Equation~\ref{eq:GKR}, $\V$ and $\P$ can reduce a claim on layer $i$ to one claim on layer $i+1$, and eventually to a claim on the input, which completes the GKR protocol.

\smallskip\noindent\textbf{Combining two claims: random linear combination.}  In~\cite{zksumcheck}, Chiesa et al. proposed an alternative approach using random linear combinations. Upon receiving the two claims $\tV_i(u)$ and $\tV_i(v)$, $\V$ selects $\alpha_i, \beta_i\in\mathbb{F}$ randomly and computes $\alpha_i\tV_i(u)+\beta_i\tV_i(v)$. Based on Equation~\ref{eq:GKR}, this random linear combination can be written as
{\footnotesize
\begin{align}\label{eq:randomGKR}
&\alpha_i\tV_i(u)+\beta_i\tV_i(v)\nonumber\\
=&\alpha_i\sum_{x, y \in\binary^{s_{i+1}}}(\tadd_{i+1}(u,x,y)(\tV_{i+1}(x)+\tV_{i+1}(y))+\tmult_{i+1}(u,x,y)(\tV_{i+1}(x)\tV_{i+1}(y)))\nonumber\\+&\beta_i\sum_{x, y \in\binary^{s_{i+1}}}(\tadd_{i+1}(v,x,y)(\tV_{i+1}(x)+\tV_{i+1}(y))+\tmult_{i+1}(v,x,y)(\tV_{i+1}(x)\tV_{i+1}(y)))\nonumber\\
=&\sum_{x, y \in\binary^{s_{i+1}}}((\alpha_i\tadd_{i+1}(u,x,y)+\beta_i\tadd_{i+1}(v,x,y))(\tV_{i+1}(x)+\tV_{i+1}(y))\nonumber\\
&+(\alpha_i\tmult_{i+1}(u,x,y)+\beta_i\tmult_{i+1}(v,x,y))(\tV_{i+1}(x)\tV_{i+1}(y)))
\end{align}
}
$\V$ and $\P$ then execute the sumcheck protocol on Equation~\ref{eq:randomGKR} instead of Equation~\ref{eq:GKR}. At the end of the sumcheck protocol, $\V$ still receives two claims about $\tV_{i+1}$, computes their random linear combination and proceeds to an layer above recursively until the input layer.

In our new ZKP scheme, we will mainly use the second approach. The full GKR protocol using random linear combinations is given in Protocol~\ref{protocol::GKR} of Appendix~\ref{app:gkr}.

\begin{theorem}\cite{VSA13}\cite{JT_Thesis}\cite{CMT}\cite{GKR}. Let $C$ : $\mathbb{F}^n \rightarrow \mathbb{F}^k$ be a depth-$d$ layered arithmetic circuit. Protocol \ref{protocol::GKR} is an interactive proof for the function computed by $C$ with soundness $O(d\log {|C|}/|\mathbb{F}|)$. It uses $O(d \log |C|)$ rounds of interaction and running time of the prover $\mathcal{P}$ is $O(|C|\log |C|)$. Let the optimal computation time for all $\tilde{add_i}$ and $\tilde{mult_i}$ be $T$, the running time of $\V$ is $O(n+k+d\log |C|+T)$. For log-space uniform circuits it is $T=\poly{\log |C|}$.
\end{theorem}
\babis{is it possible to argue briefly why the prover time is $C\log C$? This is important for the paper motivation.}\yupeng{Explained in section 3.}


%%% Latex preamble
\documentclass[11pt]{article}
\oddsidemargin 0pt
\textwidth 6.5in
\topmargin -0.65in
\textheight 9.25in

\usepackage{epsfig}
\usepackage{natbib}
\usepackage{amsmath, mathrsfs,amsfonts,amssymb}

%Get the citation punctuation correct
\bibpunct{(}{)}{;}{a}{}{,}

%Set the space between paragraphs.
%Uncomment this for single spacing, comment for double spacing
\setlength{\parskip}{1.0ex plus0.5ex minus0.2ex}

%Allow colour in your output (pdf only?)
% \usepackage[usenames,dvipsnames,svgnames,table]{xcolor}
% \definecolor{darkgreen}{rgb}{0,.5,0}  %For example
%%%End of preamble

\newcommand{\Plwr}{\ensuremath{P_{\mathrm{lwr}}}}
\newcommand{\Pupr}{\ensuremath{P_{\mathrm{upr}}}}
\renewcommand{\hat}[1]{\ensuremath{\vec{#1}}}
\newcommand{\vareps}{\varepsilon}

\begin{document}
\begin{center}
\begin{Huge} Determining the duty cycle for Pandora \end{Huge}
\end{center}

Pandora will be launched into a low Earth orbit, aligned so the spacecraft orbits in a plane roughly perpendicular the direction from the Earth to the sun. The duty cycle for a celestial object (e.g. a star) is the fraction of time that object is observable by Pandora and not obscured by the Earth, or the sun. The duty cycle depends on the star staying within both the Earth avoidance cone, and the sun avoidance cone.

\section{Calculating duty cycle due to Earth avoidance}
Let \hat{t} be the unit vector from the centre of the Earth to the telescope.
Let \hat{s} be the unit vector from the centre of the coordinate system (the barycentre of the solar system) to the target star. Assume the star is at infinity, so we don't have to adjust \hat{s} for the exact location of the Earth.

From the perspective of the telescope, a star is observable if it is near the zenith angle (away from the Earth, or ``up''), and unobservable if it is near the nadir-angle. The telescope (or zenith) vector will of course change as a function of orbital phase.  Let us define a maximum angle away from nadir for which a star is still observable, $\vareps$. The the condition of observability is therefore that

\begin{equation}
\hat{t} \cdot \hat{s} < \cos{\vareps}
\end{equation}

If the right ascension and declination are labeled $\lambda, \phi$, respectively,
\begin{equation}
\hat{s}
=
\begin{bmatrix}
s_x\\
s_y\\
s_z\\
\end{bmatrix}
=
\begin{bmatrix}
\cos{\lambda} \cos{\phi}\\
\sin{\lambda} \cos{\phi}\\
\sin{\phi} \\
\end{bmatrix}
\end{equation}

All that remains is to determine \hat{t}. The telescope vector will vary as a function of both orbital phase of the telescope around the Earth, and the orbital phase of the Earth around the Sun.

Let \hat{e} be the unit vector from the sun to the Earth. Assume, for the sake of simplicity, that the Earth's orbit is circular. Then, in ecliptic plane coordinates,

\begin{equation}
\hat{e} = ( \cos{\alpha}, \sin{\alpha}, 0)
\end{equation}

where $\alpha$ is the Earth orbital phase angle, which varies between 0 and $2\pi$ over the course of a year.

Call the plane of the telescope's orbit around the Earth {\bf uv}. We seek basis vectors within the {\bf uv}-plane that we can project the telescope vector onto.
Let \hat{u} and \hat{v} be two vectors in the plane {\bf uv} such that the vectors \hat{u}, \hat{v} and \hat{t} are all perpendicular and satisfy the right hand rule. We further arbitrarily\footnote{It is tempting to think of this is the condition that sets the orbital plane, but it just sets the direction of ``up'' in the plane.} define \hat{v} = (0,0,1)$^\intercal$. Then

\begin{eqnarray}
\hat{e} &=& \hat{u} \times \hat{v} \\
\Rightarrow \hat{u} &=& \hat{v} \times \hat{u} \\
\Rightarrow \hat{u} &=& (-\sin{\alpha}, \cos{\alpha}, 0)^\intercal
\end{eqnarray}

If we assume a circular orbit for the telescope around the Earth, we can write

\begin{equation}
\hat{t} = \cos{\rho} \hat{u} + \sin{\rho} \hat{v}
\end{equation}

%Want vertical vectors here
where $\rho$ is the telescope's orbital phase. We can then convert \hat{t} into ecliptic plane coordinates as

\begin{equation}
\hat{t} =
    \begin{bmatrix}
    \hat{t} \cdot \hat{x} \\
    \hat{t} \cdot \hat{y} \\
    \hat{t} \cdot  \hat{z}
    \end{bmatrix}
    =
    \begin{bmatrix}
    -\sin{\alpha} \cos{\rho} \\
    \cos{\alpha}, \cos{\rho} \\
    \sin{\rho})
    \end{bmatrix}
\end{equation}

\subsection{Finding the extrema of observability}
The points in the orbit where the star comes in and out of observability are given by $\hat{t} \cdot \hat{s} = \cos{\vareps}$
Working through the arithmetic, we end up with

\begin{equation}
\cos{\rho} (\cos{\alpha} s_y - \sin{\alpha} s_x) + \sin{\rho} s_z - \cos{\vareps} = 0
\end{equation}

Set
\begin{equation}
\begin{matrix}
T_1 \\
T_2 \\
T_3 \\
\end{matrix}
=
\begin{matrix}
\cos{\alpha} s_y - \sin{\alpha} s_x \\
s_z \\
\cos{\vareps} \\
\end{matrix}
\end{equation}

Then
\begin{equation}
T_1\cos{\rho} = T_3 - T_2 \sin{\rho}  \label{trig}
\end{equation}

Squaring both sides, and replacing $\cos^2{\rho}$ with $1 - \sin^2{\rho}$ gives, after a little arithmetic

\begin{equation}
(T_1^2 + T_2^2) \sin^2{\rho} - 2T_1 T_2 \sin{\rho} + (T_3^2 - T_1^2) = 0
\end{equation}

We can solve this quadratic equation to obtain two values for $\sin{\rho}$, the max and min orbital phase that a star can be observed at. Note that if a star is either continuously observable, or never observable, this equation will have no solutions, and we'll need to treat that separately.

Now, the sine of an angle is only strictly defined within a quarter of a circle, and the arcsin function admits two acceptable answers, each 180 degrees away from each other. To figure out which is the correct answer, we return to Eqn~\ref{trig} and re-write it as

$$T_2 \sin{\rho}  = T_3 - T_1\cos{\rho}  $$

Again we square it, and replace $\sin^2{\rho}$ with $1- \cos^2{\rho}$, and end up with

$$
(T_1^2 + T_2^2) \cos^2{\rho} - 2T_1 T_3 \sin{\rho} + (T_3^2 - T_2^2) = 0
$$

If we solve for both $\sin{\rho}$ and $\cos{\rho}$, we can figure out the correct quadrant for $\rho$.

\end{document}