\chapter{Newton and quasi-Newton methods}
\label{chapter:Newton}

\section{Basics of nonlinear iterations}
\begin{intro}
  The efficient solution of nonlinear problems is an important
  ingredient to implicit timestepping schemes as well as shooting
  methods. Without attempting completeness, we present some important
  facts about iterative methods for this problem. We introduce the two
  generic schemes, Newton and gradient methods, discuss their
  respective pros and cons and combine their features in order to
  obtain better methods.
\end{intro}

\input{definitions/nonlinear}
\input{definitions/iteration-order}
\input{definitions/newton}
\input{theorems/newton-kantorovich}

\begin{remark}
  Instead of proving the Newton-Kantorovich theorem, we discuss its
  main assumptions and features. First, we note that it does not
  require that the initial value be close to a root, or even assumes
  the existence of a root. The theorem is actually an existence proof.

  The Lipschitz condition on $\nabla f$ can be seen as the deviation
  of $f$ from being linear. Indeed, if $f$ were linear, then $L=0$ and
  provided $M\neq 0$ the method converges in a single step for any
  initial value.

  The larger the constant $M$, the smaller wone of the eigenvalues of % wone?
  the Jacobian. Therefore, the function becomes flat in that
  direction and the root finding problem becomes unstable.

  If we have convergence due to $\beta_0 \le 1/2$ (the proof shows
  contraction) there holds $\beta_1 := LM \left\| f\left(x^{(1)}\right)
  \right\| < 1/2$ and we have quadratic convergence from the second step on.
\end{remark}

\input{definitions/gradient-method}
\input{theorems/gradient-method}

% \begin{proof}
%   First, we observe that in any point $x$ with $\nabla F(x) \neq 0$,
%   there exists $\epsilon > 0$ such that
%   \begin{gather*}
%     F(x) - F(x-\epsilon\nabla F(x))
%     = \epsilon \abs{\nabla F(x)}^2 > 0.
%   \end{gather*}
%   Thus, $F(x-\epsilon\nabla F(x)) < F(x)$. We conclude, that for such
%   $x$, the line search obtains a positive value of $\alpha$. Thus, the
%   sequence of the gradient iteration is monotonically decreasing and
%   stays within the set $K$.
% \end{proof}

\section{Globalization}

\begin{intro}
  The convergence of the Newton method is only local, and it is the
  faster, the closer to the solution we start. Thus, finding good
  initial guesses is an important task. A reasonable initial guess in
  a one-step method seems to be $y_0$, but on closer inspection, this
  is true only if the time step is small. Therefore, the convergence
  requirements of Newton's method would insert a new time step
  restriction, which we want to avoid in the context of implicit
  methods. Therefore, and for other cases like the shooting
  methods of chapter~\ref{chapter:rwa}, we present
  methods which extend the domain of convergence.

  As a rule, Newton'a method should never be implemented without some
  globalization strategy!
\end{intro}

\input{definitions/newton-line-search}
\input{definitions/newton-step-size}

\begin{remark}
  The step size control algorithm can be implemented with very low
  overhead. In fact, in each Newton step we only have to compute the
  norm of the residual, which is typically needed for the stopping
  criterion anyway. Additional work is only needed if the residual
  grows. But this is the case, when the original method was likely to
  fail.

  The convergence proof does not guarantee that the values of $j$
  remain bounded. Practically, this is irrelevant, since typically the
  step size control only triggers within the first few steps, then the
  quadratic convergence of the Newton method starts.
\end{remark}

\input{definitions/descent-methods}

\begin{remark}
  Obviously, the gradient method is a descent method, where the direction $s$
  is chosen parallel to $\nabla F(x^{(k)})$ and $\mu$ is chosen in an
  optimal way. It is also called the method of \define{steepest
    descent}.
\end{remark}

\begin{Lemma}{newton-descent}
  The Newton method applied to the function $f(x)$ is a descent method
  applied to the functional $F(x) = \abs{f(x)}^2$. The same holds for
  the Newton method with line search or step size control.
\end{Lemma}

\begin{proof}
  By the product rule, there holds
  \begin{gather*}
    \nabla F(x) = 2 f^T(x) \nabla f(x).
  \end{gather*}
  The search direction of the Newton method is
  \begin{gather*}
    s = -\frac{d^{(k)}}{\abs{d^{(k)}}}
    = \frac{\bigl(\nabla f(x^{(k)})\bigr)^{-1}f(x^{(k)})}{\abs{\dots}}
  \end{gather*}
  Thus, omitting the arguments $x^{(k)}$, we obtain
  \begin{gather*}
    \frac{\nabla F\, s}{\norm{\nabla F}}
    = \frac{f^T \nabla f(x)\bigl(\nabla f\bigr)^{-1} f}%
    {\norm{\bigl(\nabla f\bigr)^{-1} f}\,\norm{f^T \nabla f(x)}}
    \ge \frac{\abs{f}^2}{\norm{\bigl(\nabla f\bigr)^{-1}}\;
      \norm{f}^2 \norm{\nabla f(x)} }
    = \frac1{\operatorname{cond}_2 (\nabla f(x))},
  \end{gather*}
  where we used the operator norm $\norm{.}$ of matrices with respect
  tot the Euclidean norm of $\R^d$. $\operatorname{cond}_2(A)$ is the
  spectral condition of $A$, namely
  \begin{gather*}
    \operatorname{cond}_2(A) = \norm{A} \, \norm{A^{-1}}.
  \end{gather*}
  With ~\ref{eq:descent-methods:1} we conclude that $s\in \mathcal S_\gamma(\nabla F)$ for any
  $\gamma$ with
  \begin{gather*}
    \gamma \le \frac1{\operatorname{cond}_2 (\nabla f(x))}.
  \end{gather*}
  The different variants of the Newton method are only distinguished by
  a different choice of the scaling parameter $\mu$.
\end{proof}

\begin{Lemma}{downhill}
  Let $F: \R^d \to \R$ be continuously differentiable. For a given
  point $x$, assume $\nabla F = \nabla F(x) \neq 0$.  Then, there is a
  constant $\lambda > 0$ such that for any
  $s\in \mathcal S_\gamma(\nabla F(x))$ and any
  $0 \le \mu \le \lambda$ there holds
  \begin{gather}
    \label{eq:downhill:1}
    F(x-\mu s) \le F(x) - \frac{\gamma\mu}2 \abs{\nabla F(x)}.
  \end{gather}
  In particular, a positive scaling factor $\mu$ for the descent method can
  always be found.
\end{Lemma}

% Stoer/Bulirsch I, 4. Auflage, p. 238
\begin{proof}
  First, define
  \begin{gather*}
    U_1(x) = \bigl\{ y\in \R^d \big\vert \abs{\nabla F(y)-\nabla F(x)}
    \le \tfrac{\gamma}{2} \abs{\nabla F(x)}\bigr\}.
  \end{gather*}
  Since $\nabla F$ is continuous and $\nabla F(x) \neq 0$, this set is
  a nonempty neighborhood of $x$.
% Similarly, the set
%  \begin{gather*}
%    U_2(x) =
%    \bigl\{ y\in \R^d \big\vert \mathcal S_\gamma(\nabla F(y)) \subseteq
%    \mathcal S_{\nicefrac{\gamma}{2}}(\nabla F(x))\bigr\}
%  \end{gather*}
%  is a nonempty neighborhood of $x$.
Choose now $\lambda$ such that
  \begin{gather*}
    B_{\lambda}(x) \subseteq U_1(x),% \cap U_2(x).
  \end{gather*}
  Hence, for any $\mu\in(0,\lambda)$ and
  $s\in \mathcal S_\gamma(\nabla F(x))$, there holds by the mean value
  theorem with $0<\theta<1$
  \begin{multline*}
    F(x)-F(x-\mu s) = \mu \nabla F(x-\theta\mu s)s \\
    = \mu \Bigl(\bigl(\nabla F(x-\theta\mu s) - \nabla F(x)\bigr)
    +\nabla F(x)\Bigr).
  \end{multline*}
  Using the definitions of $U_1(x)$ and $U_2(x)$, we obtain
  \begin{align*}
    F(x)-F(x-\mu s)
    &\ge -\tfrac{\gamma\mu}{2}\abs{\nabla F(x)}
      + \mu DF(x)s\\
    &\ge -\tfrac{\gamma\mu}{2}\abs{\nabla F(x)}
      + \mu\gamma\abs{\nabla F(x)}\\
    &= \tfrac{\gamma\mu}{2}\abs{\nabla F(x)}.
  \end{align*}
\end{proof}

\section{Practical considerations}

\begin{intro}
  Quadratic convergence is an asymptotic statement, which for any
  practical purpose can be replaced by ``fast'' convergence. Most of
  the effort spent in a single Newton step consists of setting up the
  Jacobian $J$ and solving the linear system in the second line
  of~\eqref{eq:newton-def:1}. Therefore, we will consider techniques
  here, which avoid some of this work. We will have to consider two
  cases
  \begin{enumerate}
  \item Small systems with $d\lesssim 1000$. For such systems, a
    direct method like $LU$- or $QR$-decomposition is advisable in
    order to solve the linear system. To this end, we compute the
    whole Jacobian and compute its decomposition, an effort of order
    $d^3$ operations. Comparing to $d^2$ operations for applying the
    inverse and order $d$ for all other tasks, this must be avoided as
    much as possible.
  \item Large systems, where the Jacobian is typically sparse (most of
    its entries are zero). For such a system, the effort of order
    $d^2$ for a full matrix vector multiplication is already not
    affordable. Therefore, the linear problem is solved by an
    iterative method and we will not have to compute the Jacobian at
    all.
  \end{enumerate}
\end{intro}

\begin{remark}
  In order to save numerical effort constructing and inverting
  Jacobians, the following strategies have been successful.
  \begin{itemize}
  \item Fix a threshold $0<\eta<1$ which will be used as a bound for
    error reduction. In each Newton step, first compute the update
    vector $\widehat d$ using the Jacobian $\widehat J$ of the
    previous step. This yields the modified method
    \begin{gather}
      \label{eq:newton:1}
      \begin{alignedat}2
        &&J_k &= J_{k-1} \\
        &&\widehat x &= x^{(k)} - J_{k}^{-1}f(x^{(k)})\\
        \text{If }\abs{f(\widehat x)} &\le \eta
          \abs{f(x^{(k)})} \quad& x^{(k+1)} &= \widehat x\\
          \text{Else } J_k &= \bigl(\nabla f(x^{(k)}\bigr)^{-1}
          \quad & x^{(k+1)} &=x^{(k)}- J_{k}^{-1}f(x^{(k)}).
      \end{alignedat}
    \end{gather}
    Thus, an old Jacobian and its inverse are used until convergence
    rates deteriorate. This method is again a quasi-Newton method
    which will not converge quadratically. However, we can obtain linear
    convergence at any rate $\eta$.
  \item If Newton's method is used within a time stepping scheme, the
    Jacobian of the last Newton step in the previous time step is
    often a good approximation for the Jacobian of the first Newton
    step in the new time step. This holds in particular for small time
    steps and constant extrapolation. Therefore, the previous method
    should also be extended over the bounds of time steps.
  \item An improvement of the method above can be achieved by so
    called rank-1 updates. Given $x^{(k)}$ and $x^{(k-1)}$, compute
    \begin{gather}
      \label{eq:newton:2}
      \begin{split}
        p &= x^{(k)} - x^{(k-1)}\\
        q &= f(x^{(k)}) - f(x^{(k-1)})\\
        J_k &= J_{k-1} + \frac1{\abs{p}^2}
        \left(q - J_{k-1} p\right)p^T
      \end{split}
    \end{gather}
    The fact that the rank of $J_k - J_{k-1}$ is at most one can be
    used to obtain a decomposition of $J_k$ in a cheap way from one
    for $J_{k-1}$.
  \end{itemize}
\end{remark}

\begin{remark}
  For problems leading to large, sparse Jacobians, typically space
  discretizations of partial differential equations, computing
  inverses of $LU$-decompositions is infeasible. These matrices
  typically only feature a few nonzero elements per row, while the
  inverse and the $LU$-decomposition is fully populated, thus
  increasing the amount of memory from $d$ to $d^2$.

  Linear systems like this are often solved by iterative methods,
  leading for instance to so called Newton-Krylov methods. Iterative
  methods approximate the solution of a linear system
  \begin{gather*}
    J d = f
  \end{gather*}
  only using multiplications of a vector with the matrix $J$. On
  the other hand, for any vector $v\in \R^d$, the term $Jv$ denotes
  the directional derivative of $f$ in direction $J$. Thus, it can be
  approximated easily by
  \begin{gather*}
    J v \approx \frac{f\left(x^{(k)}+\epsilon v\right) -
      f\left(x^{(k)}\right)}{\epsilon}.
  \end{gather*}
  The term $f\left(x^{(k)}\right)$ must be calculated anyway as it is
  the current Newton residual. Thus, each step of the iterative linear
  solver requires one evaluation of the nonlinear function, and no
  derivatives are computed.

  The efficiency of such a method depends on the number of linear
  iteration steps which is determined by two factors: the gain in
  accuracy and the contraction speed. It turns out that typically
  gaining two digits in accuracy is sufficient to ensure fast
  convergence of the Newton iteration. The contraction number is a
  more difficult issue and typically requires preconditioning, which
  is problem-dependent and as such must be discussed when needed.
\end{remark}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "notes"
%%% End:


%This is my super simple Real Analysis Homework template

\documentclass{article}
\usepackage[margin=1in]{geometry}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage[]{amsthm} %lets us use \begin{proof}
\usepackage[]{amssymb} %gives us the character \varnothing
\usepackage{amsmath}

\title{Debye length}
\author{John ``Jack'' Brooks}
\date\today
%This information doesn't actually show up on your document unless you use the maketitle command below

\begin{document}

\maketitle %This command prints the title based on information entered above

\section{Debye length derivation}
The Debye length is the scale length associated with plasma screening out external electric fields, $\mathbf{E}$.  To show this, we start with Gauss's law and convert $\mathbf{E}$ to potential, $\phi$ with $\mathbf{E} = - \nabla \phi$.

\begin{equation}
\begin{split}
	%\label{eq:asdf}
	\nabla^2 \phi &= \frac{1}{\epsilon_0}\left( \rho_{plasma} \right) \\
	&= \frac{1}{\epsilon_0}\left[ q n_0 \, \textrm{exp} \left( \frac{-q \phi (r)}{k_b T} \right)\right]
\end{split}
\end{equation}
where and the plasma charge is assuming a Boltzmann distribution.

If we assume that $k_bT \gg q\phi$, then we can Taylor expand the exponential term to get
\begin{equation}
\begin{split}
%\label{eq:asdf}
\nabla^2 \phi = \frac{1}{\epsilon_0}\left[ q n_0 \,  \left(1- \frac{q \phi (r)}{k_b T} \right)\right]
\end{split}
\end{equation}
Assuming that we have equal ion and electron charge density, $q_e n_{oe}$ cancels with $q_i n_{oi}$ leaving
\begin{equation}
\begin{split}
%\label{eq:asdf}
\nabla^2 \phi = \frac{1}{\epsilon_0}\left[ -   \frac{q^2 n_0 \phi (r)}{k_b T}  \right].
\end{split}
\end{equation}
The solution to this is
\begin{equation}
\begin{split}
\phi &= \phi_0 \, \textrm{exp} \left( -r / \left( \frac{q^2n_0}{\epsilon_0 k_b T} \right)^{1/2} \right) \\
 &= \phi_0 \, \textrm{exp} \left( -r /\lambda_D^{1/2} \right)
\end{split}
\end{equation}
where the scale length,
\begin{equation}
\begin{split}
 \lambda_D =  \left( \frac{q^2n_0}{\epsilon_0 k_b T} \right)^{1/2},
\end{split}
\end{equation}
is called the Debye length.

\section{Debye length screening}

If we now include a test charge, $Q_{tc}$, to our Gauss's law equation,
\begin{equation}
\begin{split}
%\label{eq:asdf}
\nabla^2 \phi &= \frac{1}{\epsilon_0}\left( \rho_{plasma} - \rho_{tc}\right) \\
&= \frac{1}{\lambda_D^2}\phi - \frac{Q_{tc}}{\epsilon_0}\delta (r-r_0)
\end{split}
\end{equation}
we can now investigate how the plasma screens out this charge.

TODO

\end{document}


\documentclass{article}

\begin{document}

Author: Carl Cortright

\section{Problem 10.1}

Training Data:\\

\begin{tabular}{c|cccc}

  Class & CornFlakes & Frosties & Sugar Puffs & Branflakes \\
  \hline
  Greater than 60 & 1 & 0 & 0 & 0\\
  Greater than 60 & 1 & 0 & 0 & 1\\
  Greater than 60 & 1 & 1 & 1 & 1\\
  Greater than 60 & 0 & 0 & 0 & 1\\
  Less than 60 & 0 & 1 & 1 & 0\\
  Less than 60 & 1 & 1 & 1 & 0\\

\end{tabular}

\subsection{(a)}

$P(Less than 60 | <0,1,1,0>) = \frac{P(Less than 60) * P(<0,1,1,0>|Less than 60)}{P(<0,1,1,0>)} = \frac{\frac{1}{3} * \frac{1}{2}}{\frac{1}{6}} = 1$

\subsection{(b)}

$P(age) = \frac{ P(age) * P(<a,b,c,d> | age) }{ P(<a,b,c,d>) }$

\section{Problem 10.5}

\subsection{}

$P(C=1) = \sum_{i=0}^N P(X_i | C=1)$ \\

$P(X_i = 1 | C = 1) = \frac{P(C=1) * P(C=1 | X_i=1) }{P(X_i = 1}$ \\

$P(X_i=1 | C = 0) =  \frac{P(C=0) * P(C=0 | X_i=1) }{P(X_i = 1}$ \\

\subsection{}

To form the classifier, we must take the probability of an input vector $\vec{x}$ and class $C$ and divide it by the probability of the input vector:\\

$P(C|\vec{x}) = \frac{P(\vec{x},C)}{P(\vec{x})}$

\subsection{}

If the word viagra doesn't appear in the training set, then $P(Viagra | C) = 0$ meaning via Bayes rule $P(C | Viagra) = 0$ . Via intuition, we know this must not be true. To fix this, we need to use a technique called smoothing, which basically means adding some arbitrary value $z$ to all of our probabilities to insure that they are always non-zero.

\section{Dirichlet Distributions}

\subsection{(a)}

One situation might be where we want to model classes of classes. An example would be if we were to classify spam or not spam, and then classify the sender of that particular email.

\subsection{(b)}

From Lowest to highest entopy:\\
$Dir(2,2,2)$\\
$Dir(1,1,1)$\\
$Dir(0.1,0.1,0.1)$\\

\end{document}


\documentclass[12pt]{cdblatex}
\usepackage{examples}

\begin{document}

\section*{The Gauss relation for the curvature of a hypersurface}

\begin{cadabra}
   {a,b,c,d,e,f,g,i,j,k,l,m,n,o,p,q,r,s,t,u#}::Indices.

   \nabla_{#}::Derivative.

   K_{a b}::Symmetric.
   g^{a}_{b}::KroneckerDelta.

   # Define the projection operator

   hab:=h^{a}_{b} -> g^{a}_{b} - n^{a} n_{b}.

   # 3-covariant derivative obtained by projection on 4-covariant derivative

   vpq:=v_{p q} -> h^{a}_{p} h^{b}_{q} \nabla_{b}{v_{a}}.

   # Compute 3-curvature by commutation of covariant derivatives

   vpqr:= h^{a}_{p} h^{b}_{q} h^{c}_{r} ( \nabla_{c}{v_{a b}} - \nabla_{b}{v_{a c}} ).

   substitute (vpq,hab)
   substitute (vpqr,vpq)

   distribute   (vpqr)
   product_rule (vpqr)
   distribute   (vpqr)
   eliminate_kronecker(vpqr)

   # Standard substitutions

   substitute (vpqr,$h^{a}_{b} n^{b} -> 0$)
   substitute (vpqr,$h^{a}_{b} n_{a} -> 0$)
   substitute (vpqr,$\nabla_{a}{g^{b}_{c}} -> 0$)
   substitute (vpqr,$n^{a} \nabla_{b}{v_{a}} -> -v_{a} \nabla_{b}{n^{a}}$)
   substitute (vpqr,$v_{a} \nabla_{b}{n^{a}} -> v_{p} h^{p}_{a}\nabla_{b}{n^{a}}$)
   substitute (vpqr,$h^{p}_{a} h^{q}_{b} \nabla_{p}{n_{q}} -> K_{a b}$)
   substitute (vpqr,$h^{p}_{a} h^{q}_{b} \nabla_{p}{n^{b}} -> K_{a}^{q}$)

   # Tidy up and display the results

   {h^{a}_{b},\nabla_{a}{v_{b}}}::SortOrder.

   sort_product   (vpqr)
   rename_dummies (vpqr)
   canonicalise   (vpqr)
   factor_out     (vpqr,$h^{a?}_{b?}$)
   factor_out     (vpqr,$v_{a?}$)       # cdb(gauss,vpqr)
\end{cadabra}

\subsection*{The Gauss relation for the curvature of a hypersurface}

\begin{align*}
   D_{r}(D_{q}v_p) - D_{q}(D_{r}v_p) = \cdb{gauss}
\end{align*}

\vspace{15pt}

\begin{latex}
   \begin{align*}
      D_{r}(D_{q}v_p) - D_{q}(D_{r}v_p) = \cdb{gauss}
   \end{align*}
\end{latex}

\end{document}


\subsection{Analysis of the wave energy cascade}

\subsubsection{Formulation of the spectral energy budget for
non-quadratic energy}

In order to analyze the flux of energy, we derive the spectral energy
budget, i.e.\ the governing equations for spectral KE and APE
functions.
%
This is not as straightforward as in incompressible turbulence since
the expression of the kinetic energy $E_K = \JJ\cdot\uu/2$ is not
quadratic.
%
In order to define the spectral KE function, we use the relation
\begin{equation}
\meanx{ E_K } = \sum_\kk \scalarprod{\uu}{\JJ}/2,
\end{equation}
where
\begin{equation}
\scalarprod{\baa}{\bb} \equiv \Re\{ \hat \baa(\kk)^* \cdot \hat \bb(\kk) \},
\end{equation}
where $\Re$ denotes the real part, $\kk$ is the wave number and the
hat denotes the Fourier transform.
%
The spectral KE function can therefore be defined as
\begin{equation}
E_K(\kk) \equiv \scalarprod{\uu}{\JJ}/2,
\end{equation}
such as $\meanx{ E_K } = \sum_\kk E_K(\kk)$.
%
Similarly, the PE can be written as the sum over all wave numbers of
the spectral PE function $E_P(\kk) = c^2 |\hat h|^2/2 = c^2
\scalarprod{h}{h}/2$, where, by definition,
\begin{equation}
\scalarprod{a}{b} \equiv \Re\{ \hat a(\kk)^* \hat b(\kk) \}.
\end{equation}

The equation for the spectral KE and PE functions can be written as
\begin{eqnarray}
\p_t E_K(\kk) &=& T_K(\kk) + C_K(\kk),\\
\p_t E_P(\kk) &=& T_P(\kk) - C_P(\kk),
\end{eqnarray}
where
\begin{equation}
C_K(\kk)
= -\scalarprod{\uu}{\bnabla E_P}/2 - c^2 \scalarprod{\JJ}{\bnabla h}/2,
\end{equation}
\begin{equation}
C_P(\kk) = c^2 \scalarprod{h}{hd} /2,
\end{equation}
\begin{equation}
T_P(\kk) = -c^2 \scalarprod{h}{\uu \cdot \bnabla h + hd/2}
\end{equation}
and
\begin{equation}
T_K(\kk)
= -\scalarprod{\uu}{\uu \cdot \bnabla \JJ}/2
-\scalarprod{\JJ}{\uu \cdot \bnabla \uu}/2
-\scalarprod{\uu}{d\JJ}/2.
\end{equation}
The mean conversion from potential energy to kinetic energy can be
computed from the two conversion spectral functions as $C = \sum_\kk
C_K(\kk) = \sum_\kk C_P(\kk)$.

\subsubsection{Exact Kolmogorov law for irrotational flows}

As in incompressible homogeneous isotropic turbulence, an exact
Kolmogorov law for irrotational flows can be derived.
%
It is convenient to start from the following form of the governing
equations
\begin{eqnarray}
\p_t \uu &=& -\bnabla \cdot (|\uu|^2/2) - c^2 \bnabla h
- \zeta_a \eez \wedge \uu, \\
\p_t \JJ &=& -\p_j (u_j \JJ) - \bnabla E_p,\\
\p_t h   &=& -\bnabla \cdot \JJ.
\end{eqnarray}
Assuming that the flow is irrotational and that there is no global
rotation ($\zeta_a=0$), it can be shown that
\begin{equation}
\p_t ( J_i' u_i ) = - \p_i( J_i'|\uu|^2/2) - \p_i'(u_j J_j' u_i')
-\p_i (J_i' c^2 h)  - \p_i' (u_i E_p')
\end{equation}
and
\begin{equation}
\p_t ( h' h ) = - \p_i( h'J_i ) - \p_i'( hJ_i' ),
\end{equation}
which gives
\begin{eqnarray}
\p_t ( \JJ'\cdot\uu + \JJ\cdot\uu' + 2c^2 h'h )
&=& - \p_i( J_i'|\uu|^2/2) - \p_i'( J_i|\uu'|^2/2) %\nonumber \\&&
- \p_i'(u_j J_j' u_i') - \p_i(u_j' J_j u_i) \nonumber\\
&& -c^2\p_i (J_i'h) - c^2\p_i' (J_ih') %\nonumber\\&&
- \p_i' (u_i E_p') - \p_i (u_i' E_p) \nonumber\\
&& -2c^2\p_i' (J_i'h) - 2c^2\p_i (J_ih').
\end{eqnarray}
We then assume homogeneity and take the ensemble average denoted by
$\meane{}$.  The separation vector is noted $\rr = \xx' - \xx$.  For
every function $g(\xx, \xx')$, we have $\p_i' \meane{g} =
-\p_i\meane{g} = \bnabla_{\rr} \meane{g}|_i$. We obtain
\begin{eqnarray}
\p_t \meane{ \JJ'\cdot\uu + \JJ\cdot\uu' + 2c^2 h'h }
&=& + \bnabla_{\rr}\cdot\meane{ \JJ'|\uu|^2/2 - \JJ|\uu'|^2/2 } \nonumber\\
&& - \bnabla_{\rr}\cdot\meane{ u_j J_j' \uu' - u_j' J_j \uu } \nonumber\\
&& - \bnabla_{\rr}\cdot\meane{ \uu E_p'- \uu' E_p } \nonumber\\
&& -c^2\bnabla_{\rr}\cdot\meane{ \JJ'h - \JJ h' }
\end{eqnarray}
%
We then introduce the structure functions and the operator $\delta$
returning the increment of a variable between two points separated by $\rr$,
for example $\delta h (\xx, \rr) = h(\xx+\rr)- h(\xx)$.
%
Using again homogeneity, we get
\begin{equation}
\meane{ (\delta h)^2 \delta\uu } =
-\meane{ h'^2 \uu } + \meane{ h^2 \uu' }
+2\meane{ hh' \uu } - 2\meane{ hh' \uu' }
\end{equation}
and
\begin{equation}
\meane{ |\delta \uu|^2 \delta\JJ } =
\meane{  |\uu|^2 \JJ' } - \meane{ |\uu'|^2\JJ }
+\meane{ u_ju_j' \JJ' } - \meane{  u_ju_j' \JJ },
\end{equation}
which gives
\begin{equation}
2\p_t \meane{ \JJ'\cdot\uu + \JJ\cdot\uu' + 2c^2 h'h }
= \bnabla_{\rr} \cdot ( \meane{ |\delta \uu|^2 \delta\JJ }
+ c^2\meane{ (\delta h)^2 \delta\uu } ).
\end{equation}
Using isotropy and integrating yield an exact Kolmogorov law for
one-layer shallow water irrotational turbulence
\begin{equation}
\meane{ |\delta \uu|^2 \delta J_L }
+ c^2\meane{ (\delta h)^2 \delta u_L } = -4 \varepsilon r, \label{eq_Kolmo}
\end{equation}
where $\eps$ is the energy dissipation rate and $J_L \equiv
\JJ\cdot\rr / |\rr|$ and $u_L \equiv \uu\cdot\rr / |\rr|$ are
longitudinal increments.

\subsection{Simple model for structure functions determined by a
random set of shocks}
\label{subsection_shock_model}

\cite{BouchaudMezardParisi1995} and \cite{Kuznetsov2004} showed that
the presence of discontinuities can strongly influence the statistics
and in particular lead to very strong intermittency.
%
We now present an extremely simple model based on the assumption that
shocks totally dominate the flow.  The structure functions are
calculated by averaging increments over the total surface of the
domain $\mA$.  Considering that the increments are produced only by
discontinuity lines, we get
\begin{eqnarray}
\meane{|\delta h|^p}
&=& \frac{1}{\mA} \int_{\mA}  d^2\xx |\delta h(\xx)|^p \nonumber\\
&\simeq& \frac{r}{\mA} \int_\shocks ds |\sin\theta| |\Delta h (s)|^p,
\end{eqnarray}
where $\theta$ is the angle between the shock line and the separation
vector $\rr$ %
and $\Delta h (s)$ is the step in $h$ at a shock.
%
Assuming isotropy, the integral can be split and we get
\begin{equation}
\meane{|\delta h|^p}
\simeq
r \frac{L_s}{\mA} \meant{|\sin\theta|} \means{|\Delta h (s)|^p}
=
r \frac{L_s}{\mA} \frac{2}{\pi} \means{|\Delta h (s)|^p},
\end{equation}
where $\meant{}$ denotes the mean over $\theta$, %
$L_s$ is the total length of the shocks in the domain %
and $\means{}$ denotes the mean over all shocks.

For velocity increments, we can also use a characteristic of the
velocity singularities.  The step in velocity is only in the component
perpendicular to the shocks, which implies that the longitudinal and
transverse increments are related by $\delta u_L = \delta u
\sin\theta$, $\delta u_T = \delta u \cos\theta$.
%
Applying the same averaging method, we obtain
\begin{equation}
\meane{|\delta u_L|^p}
\simeq
r \frac{L_s}{\mA}
\meant{|\sin\theta|^{1+p}}
\means{|\Delta u (s)|^p}
\end{equation}
and
\begin{equation}
\meane{|\delta u_T|^p}
\simeq
r \frac{L_s}{\mA}
\meant{|\sin\theta||\cos\theta|^{p}}
\means{|\Delta u (s)|^p},
\end{equation}
where $\Delta u (s)$ is the velocity step at a shock.
%
These assumptions completely determine the ratio between the
longitudinal and transverse structure functions, which can be
analytically computed as
\begin{eqnarray}
R_p(r) \equiv \frac{\meane{|\delta u_L|^p}}{\meane{|\delta u_T|^p}} =
\frac{\meant{|\sin\theta|^{1+p}}}{\meant{|\sin\theta||\cos\theta|^p}},
\end{eqnarray}
giving the numerical values $R_2 = 2$, $R_3 = 6\pi/8$ and $R_4 = 8/3$.

Interestingly, for an isotropic and purely divergent flow ($\zeta =
0$), the longitudinal and transverse second order structure functions
are exactly related by $\meane{(\delta u_L)^2} = \p_r( r\meane{(\delta
u_T)^2} )$ \cite[]{Lindborg2007jas}.
%
If we further assume than both second order structure functions follow
the same scaling law $\meane{ (\delta u_L)^2} \propto \meane{ (\delta
u_T)^2} \propto r^\alpha$, then we have $\meane{ (\delta u_L)^2} =
(1+\alpha) \meane{ (\delta u_T)^2} $, which gives $ R_2 = 2
\Leftrightarrow \alpha = 1$.

The shock model also provides predictions of the flatness factors of
the velocity increments
\begin{eqnarray}
F_L &=&  \frac{\meane{|\delta u_L|^4}}{\meane{|\delta u_L|^2}^2}
= r^{-1} \frac{\means{|\Delta u|^4}}{\means{|\Delta u|^2}^2}
\frac{\meant{|\sin\theta|^5}}{\meant{|\sin\theta|^3}^2},\\
F_T &=&  \frac{\meane{|\delta u_T|^4}}{\meane{|\delta u_T|^2}^2}
= r^{-1} \frac{\means{|\Delta u|^4}}{\means{|\Delta u|^2}^2}
\frac{\meant{|\sin\theta||\cos\theta|^4}}{\meant{|\sin\theta||\cos\theta|^2}^2},
\end{eqnarray}
and on the ratio of the two flatness factors, which is simply equal to
$F_T/F_L = 1.5$ according to the model.


\section{Hermitian Forms and Real Skew-Symmetric Forms}
\begin{definition}
    A sesquilinear form $\phi:V\times V\to\mathbb C$  is called Hermitian if $\phi(u,v)=\overline{\phi(v,u)}$.
\end{definition}
\begin{remark}
    In particular, $\phi(u,u)=\overline{\phi(u,u)}$, so $\phi(u,u)$ is real.
    Moreover, for any $\lambda\in\mathbb C$ we have $\phi(\lambda u,\lambda u)=|\lambda|^2\phi(u,u)$.
    Therefore it makes sense to talk about positive/definite (semi)definite Hermitian forms.
\end{remark}
\begin{lemma}
    A sesquilinear form $\phi:V\times V\to\mathbb C$ is Hermitian iff for any basis $B$ of $V$, $[\phi]_B=\overline{[\phi]}_B^\top$.
\end{lemma}
\begin{proof}
    If $\phi$ is Hermitian, then write $A=[\phi]_B=(a_{ij})_{i,j}=(\phi(e_i,e_j))_{i,j}$ where we have $a_{ji}=\phi(e_j,e_i)=\overline{\phi(e_i,e_j)}=\bar{a}_{ij}$.
    Conversely if $[\phi]_B=A$ with $A=(a_{ij})_{ij}=\bar{A}^\top$ and $u=\sum_i\lambda_ie_i,v=\sum_i\mu_ie_i$, then
    \begin{align*}
        \phi(u,v)&=\phi\left( \sum_{i=1}^n\lambda_ie_i,\sum_{j=1}^n\mu_je_j \right)=\sum_{i=1}^n\sum_{j=1}^n\lambda_i\bar\mu_ja_{ij}\\
        &=\overline{\sum_{i=1}^n\sum_{j=1}^n\bar\lambda_i\mu_ja_{ji}}\\
        &=\overline{\phi\left( \sum_{j=1}^n\mu_je_j,\sum_{i=1}^n\lambda_ie_i \right)}\\
        &=\overline{\phi(v,u)}
    \end{align*}
    So $\phi$ is Hermitian.
\end{proof}
The polarisation identity becomes
$$\phi(u,v)=\frac{1}{4}(Q(u+v)-Q(u-v)+iQ(u+iv)-iQ(u-iv))$$
for a Hermitian $\phi$ and $Q(w)=\phi(w,w)$.
\begin{theorem}[Hermitian Formulation of Sylvester's Law]
    Let $V$ be an $n$-dimensional vector space over $\mathbb C$ and $\phi:V\times V\to\mathbb C$ a Hermitian form on $V$, then $V$ has a basis $\{v_1,\ldots,v_n\}$ such that
    $$[\phi]_B=\begin{pmatrix}
        I_p&&\\
        &-I_q&\\
        &&0
    \end{pmatrix}$$
    where $p,q$ depends only on $\phi$.
\end{theorem}
The proof is nearly identical to the real symmetric case.
\begin{proof}
    If $\phi=0$ then we are done.
    Otherwise, by the polarisation identity, there exists $e_1\neq 0$ such that $\phi(e_1,e_1)\neq 0$.
    Set $v_1=e_1/\sqrt{|\phi(e_1,e_1)|}$, then we get $\phi(v_1,v_1)=\pm 1$.
    Consider $W=\{w\in V:\phi(v,w)=0\}$, then easily $V=\langle\{v_1\}\rangle\oplus W$.
    Then we can do induction on the dimension to show that $\phi$ is diagonal in some basis, which implies the existence of $p,q$.
    The uniqueness follows from the observation that $p$ (resp. $q$) is the minimal dimension of a subspace on which $\phi$ is positive (resp. negative) definite.
\end{proof}
\begin{definition}
    Let $V$ be a vector space over $\mathbb R$.
    A bilinear form on a real vector space is skew-symmetric if $\phi(u,v)=-\phi(v,u)$ for all $u,v\in V$.
\end{definition}
\begin{remark}
    1. For any $u\in V$, $\phi(u,u)=-\phi(u,u)$ therefore $\phi(u,u)=0$.\\
    2. The definition is equivalent to say that for any basis $B$ of $V$ we have $[\phi]_B=-[\phi]_B^\top$.\\
    3. For any $A\in M_n(\mathbb R)$, we can decompose it
    $$A=\frac{A+A^\top}{2}+\frac{A-A^\top}{2}$$
    into symmetric and skew-symmetric parts.
\end{remark}
\begin{theorem}[Sylvester Form]
    Let $\phi$ be a skew-symmetric bilinear form over a real vector space $V$ with $\dim V=n<\infty$, then there is a basis $B=\{v_1,w_1,\ldots,v_m,w_m,v_{2m+1},\ldots,v_n\}$ of $V$ such that
    $$[\phi]_B=\begin{pmatrix}
        A&&&\\
        &\ddots&&\\
        &&A&\\
        &&&0
    \end{pmatrix},A=\begin{pmatrix}
        0&1\\
        -1&0
    \end{pmatrix}$$
    where there are $m$ copies of $A$.
\end{theorem}
\begin{proof}
    Induction on $n$.
    If $\phi=0$, then we are done.
    Otherwise $\phi\neq 0$, so there is some $v_1,w_1$ such that $\phi(v_1,w_1)\neq 0$.
    After rescaling we might as well assume that $\phi(v_1,w_1)=1$, so correspondingly $\phi(w_1,v_1)=-1$.
    We know that $v_1,w_1$ has to be linearly independent as $\phi$ is skew-symmetric.
    Let $U=\langle\{v_1,v_2\}\rangle$ and $W=\{v\in V:\phi(v_1,v)=\phi(w_1,v)=0\}$, then $V=U\oplus W$.
    We are then done since we can use induction hypothesis on $W$ and $[\phi|_U]=A$.
\end{proof}
\begin{corollary}
    Skew-symmetric bilinear forms have even rank.
\end{corollary}
\begin{proof}
    Immediate.
\end{proof}
\begin{definition}
    Let $V$ be a vector space over $\mathbb R$ (resp. $\mathbb C$).
    An inner product on $V$ is a positive definite symmetric (resp. Hermitian) form $\phi$ on $V$.
    The pair $(V,\phi)$ is then called a real (resp. complex) inner product space.
\end{definition}
Sometimes we write $\langle u,v\rangle=\phi(u,v)$ if it is understood.
\begin{example}
    In $\mathbb R^n$, the usual real scalar product is an inner product.\\
    In $\mathbb C^n$, the usual complex scalar product is an inner product.\\
    In $C([0,1],\mathbb C)$ (over $\mathbb C$), the form
    $$\langle f,g\rangle=\int_0^1f(t)\overline{g(t)}w(t)\,\mathrm dt$$
    is an inner product for any $w\in C([0,1],\mathbb R_+)$.
\end{example}
\begin{definition}
    Let $\langle\cdot,\cdot\rangle$ be an inner product, its induced norm is $\|v\|=\sqrt{\langle v,v\rangle}$.
\end{definition}
\begin{remark}
    $\|v\|\ge 0$ and the equality holds iff $v=0$.
\end{remark}


\input{../header.tex}
\title{\vspace{-2cm}INF3490/INF4490 Exercise Solutions - Neural Networks}
\author{Eivind Samuelsen\input{../author_footnote.tex}}
\date{}

% Removing paragraph indents is sometimes useful:
\setlength\parindent{0pt}
% ==============================================================================

% ================================= DOCUMENT ===================================
\begin{document}
    \renewcommand\marginsymbol[1][0pt]{%
  \tabto*{0cm}\makebox[-1cm][c]{$\mathbb{P}$}\tabto*{\TabPrevPos}}

\maketitle
\input{../intro.tex}

\section{Single Layer Perceptron}
\subsection{}
In the perceptron below, what will the output be when the input is (0, 0)?
What about inputs (0, 1), (1, 1) and (1, 0)?
What if we change the bias weight to -0.5?

\begin{figure}[H]
\begin{center}
\includegraphics[width=0.3\textwidth]{fig1.png}
\caption{Single Layer Perceptron. b = 1 }
\label{fig:slp}
\end{center}
\end{figure}

\noindent\textit{Answer:}

\noindent
\begin{center}
\begin{tabular}{|c|c|c|c|c|c|}
\hline
\multicolumn{3}{|c|}{Bias $=-1.5$} & \multicolumn{3}{c|}{Bias $=-0.5$}\tabularnewline
\hline
Input & Weighted sum & Output & Input & Weighted sum & Output\tabularnewline
\hline
\hline
$\left(0,0\right)$ & $-1.5$ & $0$ & $\left(0,0\right)$ & $-0.5$ & $0$\tabularnewline
\hline
$\left(0,1\right)$ & $-0.5$ & $0$ & $\left(0,1\right)$ & $0.5$ & $1$\tabularnewline
\hline
$\left(1,0\right)$ & $-0.5$ & $0$ & $\left(1,0\right)$ & $0.5$ & $1$\tabularnewline
\hline
$\left(1,1\right)$ & $0.5$ & $1$ & $\left(1,1\right)$ & $1.5$ & $1$\tabularnewline
\hline
\end{tabular}
\par\end{center}

\subsection{}
Starting with random weights, how do you proceed in order to train the perceptron above to perform any given binary operation?\\

\noindent\textit{Answer:}

\noindent
See the source code for the next task.

\subsection{Implement a perceptron \marginsymbol}
Implement the perceptron, and train it to perform the logical functions NOT (use only one of the inputs), NAND, and NOR.
What happens when you try to train it do the XOR function?\\

\textit{Source code:}
\inputminted{Python}{bool_perceptron.py}
The source code is available online on GitHub:\\
\href{https://github.com/olehermanse/INF3490-AI_Machine_Learning/tree/master/material/week5}{\textbf{github.com/olehermanse/INF3490-AI\_Machine\_Learning/tree/master/material/week5}}

\section{Multi Layer Perceptron (MLP)}
The figure below shows a multilayer perceptron that constructs the XOR function.
How would you rewrite it to construct the binary equivalence function
(i.e. the output is above threshold when both inputs are either 0 or 1)?
Can you construct it so that it will detect equivalence for any combination of integer inputs?

\begin{figure}[H]
\begin{center}
\includegraphics[width=0.6\textwidth]{fig2.png}
\caption{An illustrated example of a multi layer perceptron}
\label{fig:mlp}
\end{center}
\end{figure}

\noindent\textit{Answer:}

\noindent
If we just want it to work in the binary case, we can simply switch
signs on all the output weights to create the opposite function:

\begin{figure}[H]
\begin{center}
\includegraphics[width=0.4\textwidth]{opposite.png}
\end{center}
\end{figure}

If we want it to work for all integers, we have to change the weights
in the first layer. First, note that the output layer encodes the
function $h_{2}-h_{1}-0.5>0$ which is equivalent to $h_{2}\land\lnot h_{1}$
since $h_{1}$ and $h_{2}$ are either 0 or 1. If we equate true with
1 and false with 0 we can write $\mbox{\ensuremath{h_{1}}:\,\ \ensuremath{i_{1}}+\ensuremath{i_{2}}-1.5>0}$
and $\mbox{\ensuremath{h_{2}}:\,\ \ensuremath{i_{1}}+\ensuremath{i_{2}}-0.5>0}$,
so the output is

\[
o:\, i_{1}+i_{2}>0.5\ \land\ i_{1}+i_{2}\leq1.5\quad\Leftrightarrow\quad o:\,0.5<i_{1}+i_{2}\leq1.5
\]
 Limiting $i_{1}$ and $i_{2}$ to integers gives us $o:\, i_{1}+i_{2}=1$.
So the output is 1 if and only if the sum of the inputs are 1. We
can get the equivalence function by changing this to $o:\, i_{1}-i_{2}=0$.
Going back a step or two we see that we can achieve this by making
the output
\[
o:\, i_{1}-i_{2}>-0.5\ \land\ i_{1}-i_{2}\leq0.5
\]
 or something similar. This has the same logical form that we already
have, so we don't need to change the weights of the output layer,
but only those of the hidden layer.

Tracing back, we see that we would need $h_{1}:\, i_{1}-i_{2}-0.5>0$
and $h_{2}:\, i_{1}-i_{2}+0.5>0$ as the hidden layer outputs. This
implies the following weights:
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.4\textwidth]{all_integers.png}
\end{center}
\end{figure}

\input{../contact.tex}
\end{document}
% ==============================================================================


\section{Stationarity}
Broadly speaking, a time series is said to be {\sl stationary} if
there is no systematic trend, no systematic change in variance,
and if strictly
periodic variations or seasonality do not exist. Most processes in
nature appear to be non-stationary. Yet much of the theory in
time-series literature is only applicable to stationary processes.

One way of describing a stochastic process is to specify the joint
distribution of the observations $Y(t_1), \dots, Y(t_n)$ for any
set of times $t_1, \dots, t_n$ and any value of $n$. A time series
is said to be strictly stationary if the joint distribution of
$Y(t_1), \dots, Y(t_n)$ is the same as that of $Y(t_1 + h), \dots
Y(t_n + h)$ for all $t_1, \dots, t_n$ and $h$. To see how this is
a useful assumption, notice that the above condition implies that
the expected value and covariance structure of any two components,
$Y_a(t)$ and $Y_b(t)$, of a time series are
constant in time
\begin{equation}
\label{sos}
\E \{Y_a(t)\} = \mu_a \, , \, \var \{Y_a(t)\} = \sigma_a^2 \mbox{ and }
\mbox{corr}\{ Y_a(t), Y_b(t+h) \} = \gamma_{ab}(h).
\end{equation}
The function
$\gamma_{ab}(h)$ is called the cross-correlation function if $a\neq b$
and the auto-correlation function if $a=b$.

In practice it is often useful to define stationarity in a less
restricted way than that described above. In many cases,
the statistical structure of the processes can be
completely described with the second-order properties of equation
(\ref{sos}). We can estimate the
quantities in (\ref{sos}) using
standard statistical procedures,
for example we may estimate the cross-correlation at {\it lag}
$h$, $\gamma_{a,b}(h)$ with the sample correlation of
$Y_a(1),\dots,Y_a(T-h)$ and $Y_b(h+1),\dots,Y_b(T)$.

\subsection{An example: Fetal Monitoring}

Measurements of fetal heart rate (FHR) and fetal movement (FM) are
generated by maternal-fetal monitoring.
Approximately 5 measurements per second are taken during 50 minutes on
120 subjects that are monitored at 20,24,28,32,36,
and 38-39 weeks of
gestation. Both FHR and FM are recorded giving us a multiple time
series $Y(t), t=1,\dots,50\times 60\times 5$, where $Y(t)$ is a vector
with 2 entries.

The association between accelerations of FHR and
FM has been documented since the 1930s. For
example, it has been observed
that in the third trimester most large fetal heart
accelerations are
associated with fetal activity.
In Section 4.2 we will describe how relatively straight-forward time
series techniques provide a visual descriptions of how these
associations vary with weeks gestation. These description have
motivated a methodology that will provide us is with a more rigorous
assessment of this relationship.

If we consider the FM and FHR  measurements, seen in Figure 5, as
outcomes from a two component time series, we may consider the
cross-correlation function of these two components as a description of
the association between these two processes. Notice that the
measurements taken for each
fetus at each gestation week has a cross-correlation function associated
with them. In Figure 6, as a descriptive plot, we show the average,
over individuals,
of these functions for each gestation
week. Notice that a peak at around the $-6$ second lag starts to
appear in the plot for the 24 week gestation. As the fetus gets older,
this peak grows and becomes more defined. This result
can be
considered a first step in the characterization of the relationship
between FM and FHR.

\centerline{\epsfig{figure=Plots/plot-10-02.ps,width=\textwidth}}
\centerline{\epsfig{figure=Plots/plot-10-03.ps,width=\textwidth}}


\section{Notation}

In addition to the notation in the Shelley spec, we use the following notations:

\begin{description}
\item[Functions with finite support] For a monoid $M$, $A \to_0 M$
  are the functions $f : A \to M$ with finite support, i.e. such that
  there exist only finitely many $a \in A$ such that $f a \neq 0$. We
  denote the support of $f$ by $\supp f$, which is defined as
  $\supp f := \{ a \in A : f a \neq 0 \}$.
\end{description}


\begin{figure*}[!ht]
\includegraphics[width=\textwidth]{figs/waterfall}
\caption{A waterfall plot of the \texttt{cartoon-spring.wav} from 0-200 ms between frequencies of 20-8000 Hz, where the height indicates the amplitude of each frequency. This plot was created with the REW tool~\cite{REWTool}.}
\label{fig:waterfall}
\end{figure*}

\section{Aural Distance}
\label{sec:distance}

As a distance metric, we used as a starting point the literature on acoustic fingerprinting~\cite{fingerprinting}.
Acoustic fingerprinting is the concept of creating a condensed, distinct summary of an audio file that can be used later to identify that audio file or to look it up in a database.
Acoustic fingerprints turn an audio file into a represent of how the file will sound to the human ear regardless of how it is represented in a digital format~\cite{fingerprinting}.
There are numerous ways to develop acoustic fingerprints and companies like Shazam and Sound-Hound have developed complex algorithms to create accurate fingerprints even from low quality files recorded on a cellphone mic.
For this work, we used the work of Shazam~\cite{wang2003industrial} as an inspiration for our distance metric calculation.

As an intuition, the psycho-acoustic identity of a sound file (how humans distinguish between one sound and the next) can be captured by taking every ``moment'' of audio, and listing the predominate frequencies for that slice of time.
This intuition can be represented with a waterfall plot, as shown in Figure~\ref{fig:waterfall}, which plots how the frequencies change over time.
A waterfall plot uses the Fast Fourier Transform (FFT) to calculate many discrete Fourier transforms over small times slices.
In this way, a waterfall plot is a representation of an audio file as a list of spectrograms plotted over time.
In the Shazam method, the peaks are selected from each time slice of audio and used to create a ``constellation'' of peaks over time.
This constellation is then used to build a hash that acts as a fingerprint to uniquely identify the audio sample.
We use a similar strategy by first performing a real Fast Fourier Transform on the audio file and then picking out the frequency peaks in each time frame.
However, the key difference in DSP-PBE is that we do not use the constellation as a hash for lookup in a database (as Shazam and SoundHound do), but instead, we need a distance metric between two constellations to provide a measure of how close we are to synthesizing the correct DSP filter.
Distance metrics are common in music synthesis tasks, for example, in the generation of jazz improvisations, where the improvisation should stay close by some measure to the original melody~\cite{donze2014machine}.

Fast Fourier Transforms (FFT) are the key to a good acoustic fingerprint.
The FFT, however, cannot be taken as a blackbox in our application.
The two factors we need to consider are 1) the window-size for how many samples will be used to calculate the FFT, and 2) the bin size which roughly speaking, defines the resolution of the FFT.

Each return element is a frequency \textit{bin}, and depending on the scale of your return array the size of these bins varies.
In order for each bin to correspond to 1 Hz the size of the return vector must be equal to the sampling frequency (44,100 Hz).
If each bin is not 1 Hz, the effects of spectral leakage will be seen.
This occurs when the bins do not correspond to the exact frequency peaks of the sound.
The amplitude from the peaks that fall in between bins will \textit{leak} over into the closest bin and create a distorted spectrogram.
For this reason we had to adjust the size of the FFT return arrays to be 44,100, as 44,100 Hz is a common format for audio.
Although this slows down the process of FFT, it provides the most accurate representation of the sound and for our purposes frequency accuracy is paramount.

With our constellations created from the waterfall plot, we constructed a \texttt{dist} function that measures the aural distance and is faithful to the psycho-acoustics of the human ear.
Our implementation takes the Euclidean distance of the peaks in a time slice on the frequency-amplitude axis.
In order to define this distance more formally, we introduce the notation $c@t$ to indicate selecting time slice $t$ from constellation $c$.
We also use the function $peak:: Int \to Constellation \to Peak$ to select a peak from a constellation, where the peaks are in sorted order based on frequency.
Then, for an audio clip $x$ and an audio clip $y$, and a function $toC :: Audio \to Constellation$ to transform the audio clip into a constellation with $ts$ time slices and $p$ peaks in each time slice:
%
\begin{align*}
\sum_{t=0}^{ts}\ \sum_{i=0}^{p} euclid\Big(\ &peak(i,toC(x)@t), \\ &peak(i,toC(y)@t)\ \Big)
\end{align*}
%
Note that this definition requires the audio clips to be temporally aligned, which is not always a fair assumption in the real world. We leave the exploration of a temporal offset between two example audio samples to future work.

As a sanity check that this distance metric matches the psycho-acoustic definition of distance, we used the test cases listed in Table~\ref{table:dist}.

\begin{table*}[!h]
\centering
\caption{Test cases to evaluate distance metric. The exact values are only important in relationship to the others.}
\begin{tabular}{|l | c | c|}
 \hline
 Test Name \& Expected Result & Value 1 & Value 2 \\
 \hline
 \hline
 Identity & \multirow{2}{*}{(PianoC, PianoC) = 0} & \multirow{2}{*}{NA}\\
   \quad Value 1 = 0 &  & \\
 \hline
 Commutativity & \multirow{2}{*}{(PianoC, PianoCSharp) = 5.635} & \multirow{2}{*}{(PianoCSharp, PianoC) = 5.635} \\
   \quad Value 1 = Value 2  & & \\
 \hline
 Commutativity & \multirow{2}{*}{(PianoC, HornCSharp) = 20.500} & \multirow{2}{*}{(HornCSharp, PianoC) = 20.500} \\
   \quad  Value 1 = Value 2 & & \\
 \hline
 Filter less than pitch & \multirow{2}{*}{(PianoC, PianoFilterC) = 3.749} & \multirow{2}{*}{(PianoC, PianoCSharp) = 5.635} \\
   \quad Value 1 $<$ Value 2 & & \\
 \hline
 Filter less than pitch+instrument & \multirow{2}{*}{(PianoC, PianoFilterC) = 3.749} & \multirow{2}{*}{(PianoC, HornCSharp) = 20.500} \\
   \quad Value 1 $<$ Value 2 & & \\
 \hline
 Pitch less than pitch+instrument & \multirow{2}{*}{(PianoC, PianoCSharp) = 5.635} & \multirow{2}{*}{(PianoC, HornCSharp) = 20.500} \\
   \quad Value 1 $<$ Value 2 & & \\
 \hline
\end{tabular}
\label{table:dist}
\end{table*}

The goal in the synthesis procedure is to find a DSP filter program, $F$, such that \texttt{dist$(O, F(I))$ = 0}.
However, in practice the DSP-PBE Synthesizer can only get us so close to this metric and we instead just minimize this distance.
To do this, the user specifies a default threshold distance for the aural distance.
The threshold distance defines how close is acceptably close, and can be changed by the user depending on their needs or requirements.


\documentclass[a4paper,11pt]{article}

\usepackage{physics}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm, mathtools}
%\usepackage{hyperref}
\usepackage{color}
\usepackage{jheppub}
\usepackage[T1]{fontenc} % if needed

% My Documents
\newcommand{\be}{\begin{equation}}
\newcommand{\ee}{\end{equation}}
\newcommand{\bes}{\begin{equation*}}
\newcommand{\ees}{\end{equation*}}
\newcommand{\bea}{\begin{flalign*}}
\newcommand{\eea}{\end{flalign*}}

%\linespread{1.0}
%\setlength{\parindent}{0em}
%\setlength{\parskip}{0.8em}

\title{\textbf{Binaries and Tides}}
\author{Aditya Vijaykumar}
\affiliation{International Centre for Theoretical Sciences, Bengaluru, India.}
\emailAdd{aditya.vijaykumar@icts.res.in}
\abstract{}
\begin{document}
\maketitle

\section{Tanja's Lectures at Gravitational Wave School 2017}

Non-black hole objects differ from black holes in that  :-
\begin{itemize}
	\item They can deform due to their rotational motion
	\item They can have effects due to no presence of horizon
	\item They can get tidally deformed
\end{itemize}
Tanja's lectures focus on tidal effects, which are the most promising candidates (as of 2017) to detect parameters of neutron stars.

We start off with tides in Newtonian physics.

\subsection{Newtonian Physics}
Notation - $ \vb{a} $ is 3-vector and $ \va{a} $ is 4-vector.

The force between two bodies of masses $ m  $ and $ M $ is,
\begin{equation}\label{key}
\vb{F} = - \dfrac{GmM}{r^2} \vu{n} \qq{,} U = -\dfrac{GM}{r} \qq{,} \vb{a} = \grad{U}
\end{equation}
where $ U $ and $ \vb{a} $ are the gravitational potential due to mass $ M $ and the acceleration respectively.

To make things more specific, consider a body of mass $ m_A $ with position vector $ \vb{z}_A $. The potential at $\vb{x} $ due to $ m_A$ is,
\begin{equation}\label{key}
U_A(\vb{x}) = \dfrac{Gm_A}{\abs{\vb{x} - \vb{z}_A}}
\end{equation}
Alternatively, if we consider extended bodies with density $ \rho(x) $, the potential can be written as,
\begin{align}\label{key}
U_A(\vb{x}) &= G \int \dd^3{\vb{x}'}\dfrac{\rho(\vb{x}' )}{\abs{\vb{x} - \vb{x}'}}\\
\implies \laplacian{U_A} &= G \int \dd^3{\vb{x}'} \rho(\vb{x}' ) \laplacian{\dfrac{1}{\abs{\vb{x} - \vb{x}'}}} \\
&=  G \int \dd^3{\vb{x}'} \rho(\vb{x}' ) \qty(-4 \pi \delta(\vb{x} - \vb{x}'))  \\
\laplacian{U_A} &= -4 \pi G \rho(\vb{x} )
\end{align}
\end{document}


\section{CIFAR10 example} % Jonathan / b-j8501 / Jul 11 & % b-j8505 / Jul 12
In this section, we will give more details about training a neural network with PyTorch. We will start with the simple CIFAR10 example introduced before, and explain how to modify each component of it.

The first of all, the first six lines here just consist the \emph{import} statements. Basically all this does is bringing code from this imported libraries from PyTorch in your program allows to use the classes, the functions that for implement each of these libraries.
\begin{python}
import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
\end{python}

\subsection{The \emph{`Net'} class}
Then each neural network we should to implement should exactly contain the class in the following. Which you does is that you define a new class called Net, which is a subclass of nn.Module. The nn.Module is a class implemented in the package torch.nn. Bascially you don't have to know exactly what the Module class does, but the point here is Network is going to be a subclass of it. Let go to it step by step.

Any class defined in Python has to have the \emph{\_\_init\_\_} function. It will run when the class be created. Late on, in the code when you implement like \emph{`conv\_net = Net()'}, it will create an instance of the Net class. And when the instance is created, the \emph{\_\_init\_\_} function is run. And we can see that the \emph{\_\_init\_\_} function create each of the pieces of the neural network, which are the operators we need for our neural network. This linear (\emph{nn.Linear}), convolution (\emph{nn.Conv2d}) and max pool (\emph{nn.MaxPool2d}) is implement in somewhere else. Roughly speaking, they simply contain how much parameters and a forward function. For example the \emph{`nn.Linear'} class:
\begin{itemize}
\item contains parameters which consists from a matrix and a vector $(W,b)$,
\item and the `forward' function is
$${\rm forward}(x) = Wx+b.$$
\end{itemize}
Later on, we will talk about what \emph{nn.Conv2d} and \emph{nn.MaxPool2d} do.

And then to make is useful, we need to define this forward function. Basically, it tells you, given this neural network and some inputs, what the output is. In this particular case, what the network does is to apply the first convolution, and a ReLU function. The ReLU function is
$${\rm ReLU}(x) = \max\{0,x\}.$$
And then to apply a max pooling. Then applies the next convolution, a ReLU function and pooling layer.
Then, what the \emph{`x.view'} does is take $x$, which is still an image (2 dimensional) and flat it out into a vector. Then applies the first linear layer, a ReLu function, and the second linear layer, a ReLU function, and finally the third linear layer. And it returns $x$. What is contained in the class is all the parameters for each of these layers and the forward function which tells you the order and how to apply them. So that is what network consist.
\begin{python}
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x
\end{python}

\subsection{The `main' function}
What we want to do is to load a bunch of data, and train this network (all the parameters) on these data. So the \emph{main} function is going to do that. The two lines \emph{trainset} and \emph{trainloader} load the train set, and the two lines \emph{testset} and \emph{testloader} load the test set. The \emph{trainloader} is a class which specifies some way of giving you the data. In our particular case, we construct the \emph{trainloader}, we pass the \emph{trainset}, the images, to it and also a bunch of the parameters, like the batch size, shuffle, number of workers. When you calculate the gradient, you don't use the whole dataset at once, and we only use some of them. The number of data we use to calculate the gradient is the batch size. Here we pass the \emph{`batch\_size'} to the \emph{DataLoader} what happen is the later on when we loop over everything in the \emph{trainloader} (see the for loop in the main function), it will return the images four at a time because we set `batch\_size=4'. The shuffle is true means that you return the data without replacement. It the same thing to the test set.

The \emph{criterion} is a class containing the loss function. Given the output of the neural network and the true label applies the loss function. The optimize class, we will talk about it later on. When you construct the \emph{`optim'} class,  the parameters you have to pass are \emph{`net.parameters'} that tell the optimizer which parameters need to be optimized, and other hyperparameters. It has a function called \emph{`step'}. Whenever the \emph{`step'} function is called it going to modify the parameters in some way based on whatever the gradients are. The optimizer assume you already calculated the gradients and then it does some sort of step based on the gradients. That explains why when you actually run the training we have to call \emph{`optimizer.zero\_grad'} and \emph{` loss.backward'} independently of \emph{`optimizer.step'}. \emph{`optimizer.step'} doesn't handle taking the gradient, it assumes that all the gradients already been taking properly.

\begin{python}
def main():
	transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
	trainset = torchvision.datasets.CIFAR10(root='./data', train=True,download=True, transform=transform)
	trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, shuffle=True, num_workers=2)
	testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
	testloader = torch.utils.data.DataLoader(testset, batch_size=4, shufflex=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

conv_net = Net()

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')

correct = 0
total = 0
with torch.no_grad():
    for data in testloader:
        images, labels = data
        outputs = net(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

print('Accuracy of the network on the 10000 test images: %d %%' % (
    100 * correct / total))

\end{python}

\subsection{\emph{`DataLoader'}}
Next, we want to talk about how can we change the Dataloader and how can we make it provide the data samples in a different way. In particular, we already talk about the difference between sampling with and without replacement. Let us recall that a little bit.

The loss function:
$$L(\theta) = \sum_{i=1}^n l(x_i,\theta)$$
we want to calculate the gradient of the loss function.
$$\nabla L(\theta) = \sum_{i=1}^n \nabla_\theta l(x_i,\theta)$$
and $n$ generally could be very large, $n \sim 10^4 - 10^7$. In this situation, to compute the gradients over all the data point is computationally not feasible. So just approximate the gradients by sampling some of the data points.
Initially, consider the computation complexity. Stochastic gradient descent
\begin{itemize}
\item approximate the gradient by considering a sample of data points
$$\{x_{i_1}, \ldots, x_{i_k}\}$$
where $\{i_1, \ldots, i_k\}$ is randomly chosen in each iteration, then
$$\nabla L(\theta) \approx \sum_{l=1}^k \nabla_\theta l(x_{i_l},\theta)$$
\item $k$ is called the mini-batch size (controls the accuracy of "noise" in the sample).
This is a sampled gradient, containing some noise. If I have larger mini-batch size, we will get a more accuracy gradients, so the mini-batch size $k$ is an important hyper-parameter.
\item $i_1, \ldots, i_k$ sampled with/without replacement.
\begin{itemize}
\item with replacement: there can be repetitions
\item without replacement: there can't be repetitions
\end{itemize}
\end{itemize}
What we implement is sampling without replacement cross the whole epoch. When we randomly choose these data points, we can't choose the same data points again until we around the whole data set.

Now, let's see how we can change it. First, go to the PyTorch documents page\footnote{\url{https://pytorch.org/docs/stable/data.html}}. Here is an explanation what exactly is this class does, and the important thing for us is what the possible inputs. So when I construct a DataLoader, I need to pass some variables. Whenever we construct a DataLoader, we have to pass it a dataset because there is no default value. In our program, we passed the train set, CIFAR10. You also can pass something else you want to it. And all the other variables here have some default values. There are options here to keep the defaults or to change it to what you want We can see that some of the defaults actually already changed in our program. Instead of the default value 1, we set the batch size as 4, and shuffle as true instead of false. Now, we know how to change the sampling strategies. We probably want to change some of these other inputs. Look here, we can see the description are. In particular, you will see sampler, the default value be none, which defined the strategy. Most likely, if we construct the samlper and pass it to the DataLoader, we can change the with and without replacement strategy.
\begin{figure}[H]
\centering
\includegraphics[scale=0.5]{./figures/497Proj_DataLoader}
%\caption{DataLoader}
\end{figure}

Now, we need to look at the sampler class. If we want to change the sampler, we need to define a class that is a subclass of the sampler, and contain the sampler strategy.
\begin{figure}[H]
\centering
\includegraphics[scale=0.4]{./figures/497Proj_Sampler}
%\caption{}
\end{figure}
In our case, if you want change that with or without replacement, somebody has already written it.
It turns out the RandomSampler.
\begin{figure}[H]
\centering
\includegraphics[scale=0.4]{./figures/497Proj_RandomSampler}
%\caption{}
\end{figure}
Let's use our program as an example to show how to use a sampler class. First, we need to create an instance of the RandomSampler class and pass it to the DataLoader as the sampler variable.
\begin{python}
	transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
	trainset = torchvision.datasets.CIFAR10(root='./data', train=True,download=True, transform=transform)
	with_replacement_sampler = torch.utils.data.RandomSampler(trainset, replacement=True)
	trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, shuffle=True, sampler=with_replacement_sampler, num_workers=2)
\end{python}
Now if we run this code, the data will be sampled with replacement. This is just to illustrate how you go about changing something about the model and the training process. You just look up the classes, look up in the documentation, see what they do. And often the most of thing that you may want to do, someone already program them in a simple way to do them. You can just turn on like we just doing it. Just notice that you don't have to implement it yourself in this case. You just need to look out these classes do and figure out which tools for use. So we recommend just for fun just try to the CIFAR10 example with and without replacement to see difference.

\subsubsection{Codes for the CIFAR10 example}
\begin{python}
# cifar10_example.py

import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

def main():
	transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
	trainset = torchvision.datasets.CIFAR10(root='./data', train=True,download=True, transform=transform)
	trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, shuffle=True, num_workers=2)
	testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
	testloader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

conv_net = Net()

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')

correct = 0
total = 0
with torch.no_grad():
    for data in testloader:
        images, labels = data
        outputs = net(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

print('Accuracy of the network on the 10000 test images: %d %%' % (
    100 * correct / total))

\end{python}

\subsection{The training part of CIFAR10 example}
In this section, we will analysis the training part code in the CIFAR10 example.
\begin{python}
for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')
\end{python}
In particular, we will see what the \emph{loss.backward} and \emph{optimizer.step} do
\subsubsection{\emph{loss.backward}}
Let's start with the automatically differentiation of PyTorch. This function \emph{loss.backward} hides some pretty complicated code that automatically figure out how to calculate the gradient w.r.t. the output.
Let's start with some simple example.
\begin{python}
import torch
from torch.autograd import Variable
x = Variable(torch.randn(3,3), requires_grad = True)
\end{python}
\emph{Variable} is a fundamental data type in PyTorch, which contains a tensor and some other things, like a flag in our example called \emph{requires\_grad}. So if I set \emph{requires\_grad} is true, it just indicating to PyTorch that in the future whatever \emph{x} is involved in (computational graph), it going to calculate the gradient of x. Also, one of the other things the Variable contains is the gradient value. Which right now should be empty. You can check it by running \emph{x.grad}.
Now let's see what happen when we use this Variable x in some calculations.
\begin{python}
y = torch.sum(x)
\end{python}
Here, \emph{torch.sum} means all the values of x. Here, we create a new variable y which consists of the sum of the entries of x. And PyTorch allows us to automatically calculate the gradient of the output w.r.t. any inputs. This is an extremely simple example.
\begin{equation}
\begin{split}
x &= \left(\begin{array}{ccc}
	* &\cdots &* \\
	\vdots &\ddots &\vdots \\
	* &\cdots &*
	\end{array}\right) \\
y &=  \left(\begin{array}{c}
	1 \\ \vdots \\ 1
	\end{array}\right)^\top
	x\
	\left(\begin{array}{c}
	1 \\ \vdots \\ 1
	\end{array}\right)
	=\sum_{i=1}^3 \sum_{j=1}^3 x_{ij} \\
\frac{{\rm d} y}{{\rm d} x} &= \left(\begin{array}{ccc}
	1 &1 &1 \\
	1 &1 &1 \\
	1 &1 &1
	\end{array}\right)
\end{split}
\end{equation}
How to let PyTorch compute this. Simply call
\begin{python}
torch.autograd.grad(y,x)
\end{python}
This function automatically calculate the derivative of the second augment w.r.t the first. And other way, which is more commonly used, is to call
\begin{python}
y.backward()
\end{python}
This function will calculate the derivative of y w.r.t everything that depends on the requirements of gradients (see the flag \emph{requires\_grad}). In our case, it will calculate the gradient of y w.r.t. x and store the gradient value in \emph{x,grad}.
These are two common ways that let PyTorch to calculate the gradients. We just give a simple example with one input and one output, but this whole thing works even for much more complicated examples with multiple input variables. And PyTorch can automatically calculate the gradients of these variables.

And there is one important thing to note is now if I call the \emph{y.backwark} again, what happen to the \emph{x.grad}? It will keep the same? or it will be changed? The answer is, unfortunately, it adds the gradient to whatever is already stored in the \emph{x.grad}. So, this explains why we have to zero the gradients of all of the parameters (see \emph{optimizer.zero\_grad}) before we call
\emph{loss.backward} in our codes.

\subsubsection{\emph{optimizer.step}}
The \emph{loss.backward} step calculates the gradients of all of our parameters and stores in \emph{.grad} fields. Now, let's look at our \emph{optimizer}. This \emph{optimizer} may be a class you want to write yourself if you want to try some new algorithm. It will help to know how it works.
Notice that the function we are calling here is \emph{optimizer.step}. For example, we set optimizer as SGD. After we calculate the gradients of all of the parameters, then we call this step function, it will add the values of the multiplication of learning rate and negative gradients to current values of the parameters. The \emph{optimizer} class assume that gradients already been calculated and stored in \emph{.grad} fields, and the class just decide what to do with the gradients.

Now, let use Logistic regression as an example.
\begin{python}
# lr_example.py

import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

class LogisticRegression(nn.Module):
    def __init__(self):
        super(LogisticRegression, self).__init__()
        self.linear = nn.Linear(10, 5)

    def forward(self, x):
        return self.linear(x)
\end{python}
We just write a logistic regression class with 10 input features and 5 output features. Then we will use this example to see what a the network class exactly contains.
Run the following codes in python terminal
\begin{python}
import torch
from lr_example import LogisticRegression

net = LogisticRegression()
x = torch.randn(10)
y = net(x)
\end{python}
Here we create an instance of LogisticRegression class, and set x as input and y as output. The variables this class contains is the parameters which is a 10-by-5 matrix and a vector as the bias. And take a vector x as input, and apply the forward step (see \emph{net(x)}, `forward' can be dropped), it will run the forward function to calculate $W x+b$.

There is one thing have to know is that when you call \emph{backward} on something has to be a scalar. It can't be for example a tensor. For example, if I try to call \emph{y.backward}, it will be given an error, because you can only call this for scalar outputs. It won't to calculate a Jacobian matrix of a vector output w.r.t. a vector input. However, if you pass it the derivatives of each of this components, then it can continuous do the \emph{.backward} function. So, for example, if we we want to calculate the derivative of the sum of $y$, we can do it as follows:
\begin{python}
y.backward(torch.ones(5))
\end{python}
If you have a vector that you want to call \emph{.backward} on, you have to pass it a vector with the same size, it gives the gradients of each of the components of that vector w.r.t. the output. Actually, it calculate the Jacobian times the vector. In general, it can't calculate the Jacobian of a non-scalar output, but if you know what you want to multiply the Jacobian with, then you can pass it to \emph{.backward} as well.

Then we can see the parameter of a network
\begin{python}
params = list(net.parameters())
\end{python}
Now \emph{params} contains matrix $W$ and vector $b$. Also, we can see the gradient values of them.
\begin{python}
params[0].grad
params[1].grad
\end{python}

Here we give an explanation about it. For $x\in\mathbb{R}^n$ and $y\in\mathbb{R}^d$, $y$ is some function of $x$.
\begin{equation}
\begin{split}
y &= f(x) \\
\frac{{\rm d} y}{{\rm d} x} &= \left(\begin{array}{ccc}
	\frac{{\rm d} y_1}{{\rm d} x_1} &\cdots &\frac{{\rm d} y_d}{{\rm d} x_1} \\
	\vdots &\ddots &\vdots \\
	\frac{{\rm d} y_1}{{\rm d} x_n} &\cdots &\frac{{\rm d} y_d}{{\rm d} x_n}
	\end{array}\right)
\end{split}
\end{equation}
In this case, \emph{y.backward()} will return an error, but we can call \emph{y.backward(v)} for some vector $v\in\mathbb{d}$. Then it will calculate
\begin{equation}
\frac{{\rm d} y}{{\rm d} x}\cdot v = \left(\begin{array}{ccc}
	\frac{{\rm d} y_1}{{\rm d} x_1} &\cdots &\frac{{\rm d} y_d}{{\rm d} x_1} \\
	\vdots &\ddots &\vdots \\
	\frac{{\rm d} y_1}{{\rm d} x_n} &\cdots &\frac{{\rm d} y_d}{{\rm d} x_n}
	\end{array}\right)
	\left(\begin{array}{c}
	v_1 \\ \vdots \\v_d
	\end{array}\right)
= \nabla(y\cdot v)
\end{equation}

This is basically because if you want to compute the whole Jacobian matrix, it requires a lot of time and space and it always not necessary.

Maybe this will make more sense if I set another variable $z$ as the sum of $y$,
\begin{python}
net.zero.grad()
y = net(x)
z =torch.sum(y)
\end{python}

In our example, we can see $z$ is a function of $y$ and $z = g(y) = y \cdot v$.
If I take \emph{z.backward}, this is the same computation with the previous did. Because the derivatives of $z$ w.r.t. $y$ is $\frac{{\rm d} z}{{\rm d}y} = (1,1,1,1,1)^\top$. This is what the \emph{.backward} does. It calculate the gradients multiplied by the Jacobian of previous layer to get the previous gradients, and so on.
You can call
\begin{python}
z.backward()
params[0].grad
params[1].grad
\end{python}
and compare the output with the previous one.

In practice, we would construct some loss function, and call \emph{backward} on it. Particular, in our cifar10 example, it consists of a bunch of parameters, particular, each of these layers has its parameters, and when you call forward, it performs some complicated calculations. It will build some graph that contains all the dependencies of these calculations. And then, after you apply the network
and apply a loss function to it, you call \emph{loss.backward}, this step gives you all the gradients of the parameters, and then this is a special optimizer class, which takes those stored gradients and use them to perform like a forward step.
The most important thing to remember are don't forget the \emph{optimizer.zero\_grad} because the \emph{backward} step adds the new calculated gradients to that already stored in the \emph{grad} field. Of course, if you have two loss functions, called \emph{loss1} and \emph{loss2}, you can call
\begin{python}
loss1.backward()
loss2.backward()
\end{python}
and gradients now is the sum of those two gradients.


\selectlanguage{english}
\begin{abstract}
    \noindent For the first project of the Computer Simulations (ELTE Physics MSc) course I propose a concept of an N-body simulation, which aims to reproduce the satellite formation inside asteroid belts around a larger stellar object (eg. a gas giant or a star). Due to the N-body simulations' high computational difficulty, my spare objective is to observe at least some form of clustering process inside these type of systems. To achieve these goals, I will use Newton's law of universal gravitation, solving the particles' equation of motion by a simple 4th order Runge-Kutta function. Other numerical methods - described in Sec. III. will be optionally tested.
\end{abstract}

\begin{multicols}{2}
\section{Introduction}
The problem of gravitational- or electromagnetic attraction of more, than 2 bodies are impossible to solve analitically, except for some finite special cases. To study the motion of the particles in such systems we're ought to rely on numerical simulations and approximations. The computational difficulty of the problem grows non-linearly as we try to simulate more and more particles, and thus these simulations are required to be reinforced by some clever numerical tricks to overcome the barrier of immense computational times.

\section{Motivation}
As I've already worked with numerical simulations of many-body systems, using two- and three-body approximations, I've choosen the topic of N-body simulations unhesitantly to work on as the first project of the Computer Simulations course. Also being fascinated by astronomical questions and problems, the choosing of the topic was an overally trivial decision for me. My goals are to improve my knowledge about simple N-body simulations and to learn to use some generally useful numerical tricks while working on the assignment.

\section{Description of the proposed assignment}
Inside a dense asteroid or planetary debris belt it is inevitable for larger objects to form. This belt could be either orbiting around a central star, or a planet. In both cases, moons or dwarf planets will be created by the constant collisions of the individual smaller bodies. My proposed assignment is to simulate the N-body problem of such system of maximally $10^5$ bodies and monitor its density changes and observe the formation of clusters or even greater satellites with DBSCAN or mean-shift methods. Furthermore to achieve somewhat more realistic results, I propose to implement a simple collision system between the colliding bodies. In a star - planet - asteroids system it is also possible to observe the formation of asteroid clusters around Lagrange points, which could be an extra task for the project. \newline
To evade all possible infinities or huge numerical errors, I will work with spherical rigid bodies, instead of point-like particles. To speed up the processing, I propose to experimentally use a neighbour list method with some reasonable cut-off distance, similar to used in molecular dynamics simulation. Reason for this, that at great distances the gravitational pull between smaller ojects can be bravely neglected, and thus the efficiency of the simulation could be greatly enhanched. However I have no idea how does this approximation affects or deteriorates the accuracy of the simulation. I think at least it deserves a chance.

\section{Theoretical background}
Due to my very limited free time and the short deadline for the final submission, I decided to work using the Newtonian gravitational theory only. In this case it becomes pretty straightforward to write down the necessary equations of motion. The force, acting on the $i$th body could be described by the following sum:

\begin{equation}
m_{i} \boldsymbol{\ddot{r}}_{i}
=
- G \sum_{i \neq j} \frac{m_{i} m_{j}}{\left| r_{i} - r_{j} \right|^{2}} \frac{\boldsymbol{r}_{i} - \boldsymbol{r}_{j}}{\left| r_{i} - r_{j} \right|}
\end{equation}
Using this, one can simply describe the acceleration acting on the $i$th body:

\begin{equation}
\boldsymbol{\ddot{r}}_{i}
=
- G \sum_{i \neq j} \frac{m_{j}}{\left| r_{i} - r_{j} \right|^{2}} \frac{\boldsymbol{r}_{i} - \boldsymbol{r}_{j}}{\left| r_{i} - r_{j} \right|}
\end{equation}
Which differential equation essentially should be solved numerically for the $x$, $y$ and $z$ components of the $\boldsymbol{r}$ vector to acquire the coordinates and velocities of the individual particles.

\section{Problems and spare objective}
N-body simulations are $\mathcal{O} \left( N ^{2} \right)$ hard problems, which in the absence of powerful computational systems, makes these simulations pretty useless for the observation of longterm processes, when too much particle is present. If my initial idea - to observe formation of compact satellites and moons - fails, I aim to try to at least monitor the formation of denser clusters and groups inside my asteroid system. \newline
It could be also provide unsatisfactory results if the initial system is not stable, and all particles simply ejected from the solar system after a while. I really do hope this won't happen...

\section{Technical deteails}
I'm the most confident working in Python 3.X enviroments, and prefering mostly Jupyter Notebooks. The used tools and libraries will be only Python-related. I also use FFmpeg to generate animations of the simulated systems. If it is possible, I will upload some animations of my project on my YouTube channel.

\end{multicols}


\section{Laurent Series, Isolated Singularities and the Residue Theorem}
\begin{theorem}
    Consider an open annulus $A=\{z\in\mathbb C:r<|z-a|<R\}$ and let $f$ be holomorphic on $A$.
    Then $f$ has a Laurent series expansion
    $$f(z)=\sum_{n\in\mathbb Z}c_n(z-a)^n=\left( \sum_{n=0}^\infty c_n(z-a)^n \right)+\left( \sum_{n=1}^\infty c_{-n}(z-a)^{-n} \right)$$
    for every $z\in A$.
    In addition, for any $r<\rho<R$, we have
    $$c_n=\frac{1}{2\pi i}\oint_{\partial D_\rho(a)}\frac{f(z)}{(z-a)^{n+1}}\,\mathrm dz$$
    Also for any $r<\rho_1<\rho_2<R$ the series converges uniformly on $\{z\in\mathbb C:\rho_1<|z-a|<\rho_2\}$
\end{theorem}
\begin{proof}
    Let $w\in A$, and choose $r<\rho_2<|w-a|<\rho_1<R$ and let $\gamma_1$ be the anticlockwise curve as the boundary of the a polar slice of the annulus $\{z\in\mathbb C:\rho_2<|z-a|<\rho_1\}$ that contains $w$, and $\gamma_2$ be the boundary of the rest such that $\gamma_{1,2}$ agrees on the slices.
    Now
    $$\oint_{\gamma_2}\frac{f(z)}{z-w}\,\mathrm dz=0,\frac{1}{2\pi i}\oint_{\gamma_1}\frac{f(z)}{z-w}\,\mathrm dz=I(\gamma,w)f(w)$$
    But since $\gamma_1$ is homotopic to a circle around $w$, $I(\gamma,w)=1$.
    Hence
    \begin{align*}
        f(w)&=\frac{1}{2\pi i}\left( \oint_{\gamma_1}\frac{f(z)}{z-w}+\oint_{\gamma_2}\frac{f(z)}{z-w} \right)\\
        &=\frac{1}{2\pi i}\left( \oint_{\partial D_{\rho_1}(a)}\frac{f(z)}{z-w}\,\mathrm dz-\oint_{\partial D_{\rho_2}(a)}\frac{f(z)}{z-w}\,\mathrm dz \right)\\
        &=f_1(w)+f_2(w)\\
        f_1(w)&=\frac{1}{2\pi i}\oint_{\partial D_{\rho_1}(a)}\frac{f(z)}{z-w}\,\mathrm dz\\
        f_2(w)&=-\frac{1}{2\pi i}\oint_{\partial D_{\rho_2}(a)}\frac{f(z)}{z-w}\,\mathrm dz
    \end{align*}
    Expanding $f_1$ as Taylor series about $a$ would give the nonnegative terms of the Laurent series.
    We shall produce the negative terms from $f_2$ by a trick we used before.
    Observe
    $$-\frac{1}{z-w}=\frac{1}{w-a}\frac{1}{1-(z-a)/(w-a)}=\sum_{m=1}^\infty\frac{(z-a)^{m-1}}{(w-a)^m}$$
    which converges uniformly as a geometric series.
    So we can change the order of integration and get
    $$f_2(w)=\sum_{m=1}^\infty\left(\frac{1}{2\pi i}\oint_{\partial D_{\rho_2}(a)}f(z)(z-a)^{m-1}\,\mathrm dz\right)(w-a)^{-m}$$
    writing $m=-n$ gives the existence of the Laurent series.
    The rest is trivial.
\end{proof}
\begin{definition}
    A complex valued function $f$ has an isolated singularity at a point $a\in\mathbb C$ if $f$ is defined and holomorphic in a punctured disk $D_r(a)\setminus\{a\}$ for some $r>0$ but not in $D_r(a)$ (i.e. either not defined at $a$ or not holomorphic there).
\end{definition}
\begin{example}
    1. $f(z)=1/z$ has an isolated singularity at $z=0$.\\
    2. $f(z)=(e^z-1)/z$ also has one at $0$.\\
    3. $f(z)=e^{1/z}$ has one at $0$.
\end{example}
\begin{definition}
    An isolated singularity $a$ of $f$ is removable if there is a holomorphic $g$ defined on $D_r(a)$ for some $r>0$ and $f=g$ on $D_r(a)\setminus\{a\}$.
\end{definition}
\begin{proposition}
    $f$ has a removable singularity at $a$ iff $\lim_{z\to a}(z-a)f(z)=0$.
\end{proposition}
\begin{proof}
    The ``only if'' part is immediate.
    For the other direction, Suppose $f$ is holomorphic on $D_r(a)\setminus\{a\}$ and $\lim_{z\to a}(z-a)f(z)=0$, then let
    $$h(z)=\begin{cases}
        (z-a)^2f(z)\text{, if $z\neq 0$}\\
        0\text{, if $z=0$}
    \end{cases}$$
    So $h$ is holomorphic in some disk $D_r(a)\setminus\{a\}$.
    Also
    $$\frac{h(z)-h(a)}{z-a}=(z-a)f(z)\to 0$$
    as $z\to a$, so $h$ is actually holomorphic on $D_r(a)$ with $h^\prime(a)=0$.
    But $h$ has a zero of order at least $2$, hence there is a holomorphic $g$ on $D_r(a)$ such that $h(z)=(z-a)^2g(z)$, but then $g$ equals $f$ on $D_r(a)\setminus\{a\}$.
\end{proof}
So the isolated singularities of bounded functions are removable.
\begin{definition}
    Let $a$ be an isolated singularity of $f:D_r(a)\to\mathbb C$, then $a$ is a pole if the limit $\lim_{z\to a}|f(z)|=\infty$.\\
    If $a$ is neither a pole nor a removable singularity, we say $a$ is an essential singularity.
\end{definition}
\begin{proposition}\label{pole_equiv}
    Let $U$ be a domain and $f:U\setminus\{a\}\to\mathbb C$ is holomorphic for $a\neq U$.
    Then the followings are equivalent:\\
    1. $a$ is a pole of $f$.\\
    2. There is $\epsilon>0$ such that there is a holomophic $h:D_\epsilon(a)\to\mathbb C$ with $h(z)=0\iff z=a$ and $\forall z\in D_\epsilon(a)\setminus\{a\},f(z)=1/h(z)$.\\
    3. There is a holomorphic $g:U\to\mathbb C$ such that $g(a)\neq 0$ and
    $$f(z)=(z-a)^{-k}g(z)$$
    for some integer $k\ge 1$.
    Also such $g,k$ are uniquely determined by $f$.
\end{proposition}
\begin{proof}
    $1\implies 2$: Choose $\epsilon$ such that $|f(z)|\ge 1$ whenever $0<|z-a|<\epsilon$, so $1/f$ is holomorphic on $D_\epsilon(a)\setminus\{a\}$.
    But $a$ becomes a removable singularity of $1/f$ by the preceding proposition, so the extension of $1/f$ to the entire disk $D_\epsilon(a)$ would be the desired $h$.\\
    $2\implies 3$: Suppose $h$ has a zero of order $k$, then $h(z)=(z-a)^kq(z)$ where $q$ is holomorphic and $q(a)\neq 0$, so $q$ is nonzero in a disk $D\subset D_\epsilon(a)$ around $a$, hence
    $$g(z)=\begin{cases}
        1/q(z)\text{, if $z\in D$}\\
        (z-a)^kf(z)\text{, if $z\in U\setminus\{a\}$}
    \end{cases}$$
    which is well-defined by the definition of $h$ and is holomorphic.
    Uniqueness follows.\\
    $3\implies 1$ is trivial.
\end{proof}
\begin{corollary}
    If $z\in\mathbb C$ is an essential singularity of $f$, then the limit of $|f(z)|$ as $z\to a$ does not exist, either as a real number or infinity.
\end{corollary}
\begin{example}
    If we compute $e^{1/z}$ as $z\to 0$, we will find that it does not exist, so $0$ is an essential singularity of it.
\end{example}
\begin{definition}
    If $f$ has a pole at $a$, then the integer $k$ as stated in Proposition \ref{pole_equiv} is called the order of the pole $a$.
    If $k=1$, then we call $a$ a simple pole.\\
    Let $U$ be a domain and $S\subset U$ is a set of isolated points in $U$.
    If $f$ is holomorphic on $U\setminus S$ and each $s\in S$ is either a removable singularity or a pole of $f$, we say $f$ is meromorphic on $U$.
\end{definition}
Meromorphic functions can be taken (maybe) as holomorphic functions from $U$ to $\mathbb C_\infty$.
\begin{remark}
    By the preceding corollary, at an isolated essential singularity $a$ of $f$, then $f$ oscillates around $a$.
    More precisely we have the Cosorati-Weierstrass Theorem (proven in example sheet), where we find that the image of any punctured neighbourhood around $a$ has dense image in $\mathbb C$.
    A much much harder theorem will show that this image is the entire complex plane $\mathbb C$ except possibly one point.
\end{remark}
\begin{proposition}
    Let $f$ be holomorphic in $D_R(a)\setminus\{a\}$, then we have the series
    $$f(z)=\sum_{n\in\mathbb Z}c_n(z-a)^n,\forall z\in D_R(a)\setminus\{a\}$$
    Also, $\forall n<0,c_n=0$ implies that $a$ is removable.\\
    If now $c_{-k}\neq 0$ for some $k>0$ and $c_n=0$ for $n<-k$, then there is a holomorphic $g:D_R(a)\to\mathbb C$ such that
    $$f(z)=\sum_{n=-k}^\infty c_n(z-a)^n=\frac{g(z)}{(z-a)^k}$$
    So $f$ has a pole of order $k$ at $a$.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
Let rhe series expansion be as above, then by uniform convergence, the integral
$$\int_{\partial D_\rho(a)}f(z)\,\mathrm dz=2\pi ic_{-1}$$
So we define
\begin{definition}
    $c_{-1}$ is defined as the residue $\operatorname{Res}_f(a)$ of $f$ at $a$.
    And the series
    $$\sum_{n=-\infty}^{-1}c_n(z-a)^n$$
    is called the principal part of $f$.
\end{definition}
\begin{theorem}[Residue Theorem]
    Let $U$ be a domain and $S=\{a_1,\ldots,a_k\}\subset U$.
    Suppose $f$ is holomorphic on $U\setminus S$, then for any closed piecewise $C^1$ curve $\gamma:[0,1]\to U\setminus S$ homologous to $0$ in $U$, we have
    $$\int_\gamma f(z)\,\mathrm dz=2\pi i\sum_{j=1}^kI(\gamma;a_j)\operatorname{Res}_f(a_j)$$
\end{theorem}
\begin{proof}
    Let $g$ be the principal part of $f$ at $a_i$.
    Then $g$ is holomorphic on $\mathbb C\setminus\{a_i\}$, so $f-(g_1+\cdots+g_k)$ is holomorphic in $U$ except at removable singularities, so by Cauchy's Theorem,
    $$\int_\gamma f(z)\,\mathrm dz=\sum_{j=1}^k\int_\gamma g_j(z)\,\mathrm dz=2\pi i\sum_{j=1}I(\gamma,a_j)\operatorname{Res}_f(a)$$
    as desired.
\end{proof}
There are some useful facts about residues.
\begin{proposition}
    If $f$ has a simple pole at $a$, then $\operatorname{Res}_f(a)=\lim_{z\to a}(z-a)f(a)$.
    More generally, if $f$ has a pole of order $k$, then if we write $f(z)=(z-a)^{-k}g(z)$ for a holomorphic $g$, then $\operatorname{Res}_f(a)=g^{(k-1)}(a)/(k-1)!$.
    If $f=g/h$ with $g,h$ holomorphic and at $a$, $g(a)\neq 0$ and $h$ has a simple zero at $a$, then $f(a)=g(a)/h^\prime(a)$.
\end{proposition}
\begin{proof}
    Easy.
\end{proof}
\begin{proposition}[Jordan's Lemma]
    If $f$ is holomorphic in $\{z\in\mathbb C:|z|>r\}$ for some $r>0$ and if $zf(z)$ is bounded for large $|z|$, then
    $$\int_{\gamma_R}f(z)e^{i\alpha z}\,\mathrm dz\to 0,\gamma_R:[0,\pi]\ni t\mapsto Re^{it}$$
    as $R\to\infty$.
\end{proposition}
\begin{proof}
    Example sheet.
    Use the fact that $\sin t/t\ge 2/\pi$ for $t\in (0,\pi/2]$.
\end{proof}
\begin{proposition}
    Let $f$ be holomorphic on $D_R(a)\setminus\{a\}$ with a simple pole at $z=a$ and $\gamma_\epsilon:[\alpha,\beta]\ni t\mapsto a+\epsilon e^{it}$, then
    $$\lim_{\epsilon\to 0^+}\int_{\gamma_\epsilon}f(z)\,\mathrm dz=(\beta-\alpha)i\operatorname{Res}_f(a)$$
\end{proposition}
\begin{proof}
    Just write $f(z)=\operatorname{Res}_f(a)/(z-a)+g(z)$.
\end{proof}
\begin{example}
    Let $R>\epsilon>0$.
    Consider $f(z)=e^{iz}/z$ along the contour $\gamma=\gamma_1-\gamma_\epsilon+\gamma_2+\gamma_R$ where $\gamma_1$ is the segment $[-R,-\epsilon]$, $\gamma_2$ is $[\epsilon,R]$, and $\gamma_R(t)=Re^{it},\gamma_\epsilon(t)=\epsilon e^{it},t\in [0,\pi]$.
    Now by Jordan's Lemma and the preceding proposition respectively, we have
    $$\lim_{R\to\infty}\int_{\gamma_R}f(z)\,\mathrm dz=0,\lim_{\epsilon\to 0}\int_{\gamma_\epsilon}f(z)\,\mathrm dz=\pi i\operatorname{Res}_f(a)=\pi i$$
    Now for $\epsilon,R$ as before we have
    $$0=\int_\gamma f(z)\,\mathrm dz=\int_{-R}^{-\epsilon}\frac{e^{ix}}{x}\,\mathrm dx+\int_{-\gamma_\epsilon}f(z)\,\mathrm dz+\int_\epsilon^R\frac{e^{ix}}{x}\,\mathrm dx+\int_{\gamma_R}f(z)\,\mathrm dz$$
    Now let $R\to\infty,\epsilon\to 0$, we get
    $$2i\int_0^\infty\frac{\sin x}{x}\,\mathrm dx=\pi i\implies\int_0^\infty\frac{\sin x}{x}\,\mathrm dx=\frac{\pi}{2}$$
\end{example}


\problemname{Elementary Math}

\illustration{.35}{elementarymath.jpg}{Example exam by Ellen}%
Ellen is teaching elementary math to her students and the time for the final exam has
come. The exam consists of $n$ questions. In each question the students have to
add ($+$), subtract ($-$) or multiply ($*$) a pair of numbers.

Ellen has already chosen the $n$ pairs of numbers. All that remains is to decide
for each pair which of the three possible operations the students should perform.
To avoid students getting bored, Ellen wants to make sure that the $n$ correct answers
to her exam are all different.

Please help Ellen finish constructing the exam by automating this task.

\section*{Input}

The input consists of:
\begin{itemize}
   \item one line with one integer $n$ ($1\leq n \leq 2\,500$), the number of pairs of numbers;
   \item $n$ lines each with two integers $a$ and $b$ ($-10^6 \leq a, b \leq 10^6$), a pair of numbers used.
\end{itemize}

\section*{Output}

For each pair of numbers $(a,b)$ in the same order as in the input,
output a line containing a valid equation.
Each equation should consist of five parts: $a$, one of the three operators, $b$,
an equals sign ($=$), and the result of the expression.
All the $n$ expression results must be different.

If there are multiple valid answers you may output any of them.
If there is no valid answer, output a single line with the string ``\texttt{impossible}'' instead.


\section{Eigenvalues and eigenvectors of a matrix}

\begin{outcome}
  \begin{enumerate}
  \item Describe eigenvalues geometrically and algebraically.
  \item Find eigenvalues and eigenvectors for a square matrix.
  \end{enumerate}
\end{outcome}