%!TEX root =  ../main.tex

\mychapters{Triangles}{triangles}{\chapdir/pics/Northern_lights_in_Greenland}

Amazingly, the trigonometric functions defined by right-triangles are
meaningful on non-right triangles.  The Pythagorean theorem is actually
just a special case of the Law of Cosines.  Triangles, it seems, are the
building blocks of the universe.

Why are triangles so productive?  What is so special about triangles?
Is there another shape with such descriptive power?

\newpage
\chapterminitoc

%									11 - 1
\newpage
\section{Area Formulae}
\noindent\makebox[\textwidth]{\includegraphics[width=\paperwidth]{ch11/1101p.pdf}}
\subsection{Sine Formulae}
Every child with any years under his or her belt in school knows that the area of a triangle is
$\frac{1}{2}bh$.  But what if we don't know $h$, the height?  It turns out, other information
is just as good.

Consider a triangle with side $b$ on the bottom.  For convenience's sake, we will put angle
A on the left, angle C on the right, and have angle B above.  If we drop an altitude from angle
C onto side $b$, we have $h$, and have bisected ABC into two right triangles.  We see,
therefore, that $\sin A = \frac{h}{c}$ and that $\sin C = \frac{h}{a}$.  Solving both equations
for $h$, we see that $h = c \cdot \sin A = a \cdot \sin C$.  If we don't know $h$ and we want to
find the area, we can substitute in either of these expressions.  This procedure would also have
worked for the other rotations of triangle ABC.

\begin{equation}
\text{the area of triangle ABC} = \frac{1}{2} a \cdot b \cdot \sin C = \frac{1}{2} a \cdot \sin B \cdot c = \frac{1}{2} \sin A \cdot b \cdot \sin C
\end{equation}

\subsection{Heron's Formula}
While the proof is very lengthy, the ancient Greek mathematician Hero proved that the area of a
triangle is related to itself \textbf{semi-perimeter}, that is, half the total length around the outside.
If we have some triangle with sides $a$, $b$, and $c$, then $s = \frac{a + b + c}{2}$.

\begin{equation}
\text{the area of a triangle ABC} = \sqrt{s(s-a)(s-b)(s-c)}
\end{equation}

\newpage
\subsection{Exercises}
To be done in Kuta

%									11 - 2
%\newpage
\invisiblesection{Law of Cosines}
\subsection{Problems}
\noindent\makebox[\textwidth]{\includegraphics[width=\paperwidth]{ch11/1102p.pdf}}
\newpage
\subsection{SSS}
\subsection{SAS}
\newpage
\subsection{Exercises}
to be done in kuta

%									11 - 3
\newpage
\invisiblesection{Law of Sines}
\subsection{Problems}
\noindent\makebox[\textwidth]{\includegraphics[width=\paperwidth]{ch11/1103p.pdf}}
\newpage
\subsection{Imagining the Height}
\subsection{SAA}
\subsection{ASA}
\newpage
\subsection{Exercises}
to be done in kuta

%									11 - 4
\newpage
\invisiblesection{The Ambiguous Case}
\subsection{Problems}
\noindent\makebox[\textwidth]{\includegraphics[width=\paperwidth]{ch11/1104p.pdf}}
\newpage
\subsection{SSA}
\subsection{Related Rates}
\subsection{Exercises}
to be done in kuta

%									11 - 5
\newpage
\invisiblesection{2D Vectors}
\subsection{Problems}
\noindent\makebox[\textwidth]{\includegraphics[width=\paperwidth]{ch11/1105p.pdf}}
\newpage
\subsection{Definition and Magnitude}
Vectors have a length, called their magnitude, written with absolute value bars.
\index{Absolute Value!of vectors}
\subsection{Vector Components}
\subsection{Heading and Bearing}
\newpage
\subsection{Exercises}
to be done in kuta

%									11 - 6
\newpage
\section{Review}
\subsection{Chapter Review}
\subsection{Chapter Test}


\section{M/M/1 Queues}
\label{sec:M-M-1-queues}

A $M/M/1$ is a queue where
(i) the arrival process is Poissonian with rate $\lambda$,
(ii) the service process is Exponential with rate $\mu$,
(iii) there is one servers,
(iv) the buffer has infinite capacity,
(v) the scheduling policy is FCFS.

%\begin{figure}[tp]
%\label{fig:M-M-1-queue}
%	\centering
%	\includegraphics{fig/M-M-1-Queue}
%	\caption{An M/M/1 queue and its corresponding CTMC.}
%\end{figure}

\begin{theorem}[State Probability]
\label{thm:M-M-1-probability-state}
	For any $M/M/1$, the state probability is

	\begin{equation}
	\label{eqn:M-M-1-probability-state}
	\pi_{i} = \varrho^{i}(1-\varrho)
	\end{equation}

	\begin{proof}
		For a formal demonstration, see \cite{harchol2013performance} on page 258-259.
	\end{proof}
\end{theorem}

\begin{theorem}[Mean System Jobs]
\label{thm:M-M-1-System-Jobs}
	For any $M/M/1$, the expected number of jobs is

	\begin{equation}
	\label{eqn:M-M-1-System-Jobs}
	\expected{N} = \frac{\varrho}{(1-\varrho)}
	\end{equation}

	\begin{proof}
		\begin{equation*}
		\expected{N} = \sum_{i=0}^{\infty} i \pi_{i} = \frac{\varrho}{(1-\varrho)}
		\end{equation*}
	\end{proof}
\end{theorem}

The remaining metrics ($\expected{N_{Q}},\expected{T},\expected{T_{Q}}$) could be determined by applying the Little's Law, the basic definitions
$\expected{N}=\expected{N_{Q}}+\expected{N_{S}}$,
$\expected{N_{S}}=\varrho$,
$\expected{T}=\expected{T_{Q}}+\expected{T_{S}}$, and
$\expected{T_{S}}=\frac{1}{\mu}$.

In particular we obtain:

\begin{equation}
\label{eqn:M-M-1-Queue-Jobs}
\expected{N_{Q}} = \frac{\varrho^{2}}{(1-\varrho)}
\end{equation}

\begin{equation}
\label{eqn:M-M-1-Delay}
\expected{T_{Q}} = \frac{1}{\mu} \cdot \frac{\varrho}{(1-\varrho)}
\end{equation}

\begin{equation}
\label{eqn:M-M-1-Response-Time}
\expected{T} = \frac{1}{\mu - \lambda}
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Written By Michael Brodskiy
% Class: Analytic Geometry & Calculus III (Math-292)
% Professor: V. Cherkassky
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass[12pt]{article}
\usepackage{alphalph}
\usepackage[utf8]{inputenc}
\usepackage[russian,english]{babel}
\usepackage{titling}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{enumitem}
\usepackage{amssymb}
\usepackage[super]{nth}
\usepackage{everysel}
\usepackage{ragged2e}
\usepackage{geometry}
\usepackage{fancyhdr}
\geometry{top=1.0in,bottom=1.0in,left=1.0in,right=1.0in}
\newcommand{\subtitle}[1]{%
  \posttitle{%
    \par\end{center}
    \begin{center}\large#1\end{center}
    \vskip0.5em}%

}
\usepackage{hyperref}
\hypersetup{
colorlinks=true,
linkcolor=blue,
filecolor=magenta,
urlcolor=blue,
citecolor=blue,
}

\urlstyle{same}

\title{Lecture XIX Notes}
\date{\today}
\author{Michael Brodskiy\\ \small Professor: V. Cherkassky}

% Mathematical Operations:

% Sum: $$\sum_{n=a}^{b} f(x) $$
% Integral: $$\int_{lower}^{upper} f(x) dx$$
% Limit: $$\lim_{x\to\infty} f(x)$$

\begin{document}

\maketitle

\section{Vector Fields $-$ 16.1}

If the region $D$ lies in $\mathbb{R}^2$, then the function for the field is as follows:

$$\bold{\bar{F}}(x,y)=P(x,y)\bold{\hat{i}}+Q(x,y)\bold{\hat{j}}$$

If $D$ lies in $\mathbb{R}^3$:

$$\bold{\bar{F}}(x,y)=P(x,y)\bold{\hat{i}}+Q(x,y)\bold{\hat{j}}+R(x,y)\bold{\hat{k}}$$

The formula for Newton's law of gravitation is as follows:

$$|\bold{\bar{F}}|=\frac{GmM}{r^2}$$

This may be transformed into a vector field by using the following steps:

$$|\bold{\bar{F}}|=\frac{GmM}{r^2}$$
$$|\bold{\bar{F}}|=\frac{GmM}{r^3}\bold{\hat{r}}$$
$$\bold{\hat{r}}=\langle x, y, z\rangle,\,r=\sqrt{x^2+y^2+z^2}$$
$$|\bold{\bar{F}}|=\frac{GmMx\bold{\hat{i}}}{(x^2+y^2+z^2)^{\frac{3}{2}}}+\frac{GmMy\bold{\hat{j}}}{(x^2+y^2+z^2)^{\frac{3}{2}}}+\frac{GmMz\bold{\hat{k}}}{(x^2+y^2+z^2)^{\frac{3}{2}}}$$

Calling back to the gradient, which is written as $\nabla f(x,y)=\frac{\partial f}{\partial x}\bold{\hat{i}} + \frac{\partial f}{\partial y}\bold{\hat{j}}$, where $f(x,y)$ is a scalar function. This may be used in conjunction with vector fields.\\

Any vector field which has $\bold{\bar{F}}=\nabla f$ is called a conservative vector field. In such a case, $f$ is called a potential function for $\bold{\bar{F}}$

\textit{In Physics:}
Newton's formula for gravitation is a conservative vector field.
In addition to this, the potential function, $f$, for conservative vector fields was used to find potential electrical and magnetic field, in addition to the aforementioned gravitational field.

\section{Line Integrals $-$ 16.2}

A line integral may be found using the formula:

$$\int_C f(x,y)\,ds$$

As used in an earlier chapter, $ds=\sqrt{(\frac{dx}{dt})^2+(\frac{dy}{dt})^2}$

Therefore, the line integral may be found by using:

$$\int_C f(x,y)\,ds=\int_a^b f(x(t),y(t)) \sqrt{\left(\frac{dx}{dt}\right)^2+\left(\frac{dy}{dt}\right)^2}\,dt$$

Much like any other integral, when applied to a piecewise function, the integral may be broken up:

$$\int_C f(x,y)\,ds=\int_{C_1} f(x,y)\,ds+\int_{C_2} f(x,y)\,ds+\int_{C_3} f(x,y)\,ds+\dots \int_{C_n} f(x,y)\,ds+$$

\end{document}


\documentclass[]{article}
\usepackage{amsmath}
\usepackage[a4paper]{geometry}
\usepackage{graphicx}
\usepackage{microtype}
\usepackage{siunitx}
\usepackage{booktabs}
\usepackage[colorlinks=false, pdfborder={0 0 0}]{hyperref}
\usepackage{cleveref}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{float}

\begin{document}

\title{Regularized Optimization}
\author{Dun Wang}
\maketitle

\section{L2 Regularization}

In PLM, we need to do the L2 form regularized optimization as following:

\begin{align*}
  I_{mn}^{*}=\sum_{m' \in M_{m}} a_{mnm'}I_{m'n'}
\end{align*}

\begin{align*}
  \chi^2_{mn}    = \sum_{n'\in{N}_n} \frac{[I_{mn'} - I^{\ast}_{mnn'}]^2}{\sigma^2_{mn'}}
		+ \sum_{m'\in{M}_m} \frac{a_{mnm'}^2}{\Sigma^2}
\end{align*}
\\
where $\sum_{m' \in {M}_m} \frac{a_{mnm'}^2}{\Sigma^2}$ is the L2 regularization term and $\frac{1}{\Sigma^2}$ is the regularization strength

\section{Linear Algerbra}
Take derivative to the $\chi^2$ equation with respect to $a_{mnk}$,  we can get the normal equation with L2 regualrization:

\begin{align*}
  0   = \sum_{n'\in{N}_n} \frac{1}{\sigma^2_{mn'}}[I_{mn'} -  \sum_{m'\in{M}_m}a_{mnm'}I_{m'n'}]I_{kn'}
                 + \sum_{m'\in{M}_m} \frac{a_{mnk}}{\Sigma^2}
\end{align*}
\\
where $k \in{M}_m$
\\
\\
Construct the matrix:
\\
\begin{align*}
A=
 \begin{pmatrix}
  I_{1,1} & I_{1,2} & \cdots & I_{1,m'} \\
  I_{2,1} & I_{2,2} & \cdots & I_{2,m'} \\
  \vdots  & \vdots  & \ddots & \vdots  \\
  I_{n,1} & I_{n,2} & \cdots & I_{n,m'}
 \end{pmatrix}
\end{align*}

\begin{align*}
C =
 \begin{pmatrix}
  \sigma^2_{1,1} & 0 & \cdots & 0 \\
  0 & \sigma^2_{2,2} & \cdots & 0 \\
  \vdots  & \vdots  & \ddots & \vdots  \\
  0 & 0& \cdots & \sigma^2_{n,n}
 \end{pmatrix}
\end{align*}

\begin{align*}
Y =
 \begin{pmatrix}
  I_{1,1} & I_{1,2} & \cdots & I_{1,m}\\
  I_{2,1} & I_{2,2} & \cdots & I_{2,m} \\
  \vdots  & \vdots  & \ddots & \vdots  \\
  I_{n,1}& 0& \cdots & I_{n,m}
 \end{pmatrix}
\end{align*}

\begin{align*}
X=
 \begin{pmatrix}
  a_{1,1} & a_{1,2} & \cdots & a_{1,n} \\
  a_{2,1} & a_{2,2} & \cdots & a_{2,n} \\
  \vdots  & \vdots  & \ddots & \vdots  \\
  a_{m',1} & a_{m',2} & \cdots & a_{m',n}
 \end{pmatrix}
\end{align*}
\\
The equation is equivalent to:
\begin{align*}
(A^TC^{-1}A+\frac{1}{\Sigma^2}I)X &= A^TC^{-1}Y
\end{align*}
\\
Solve the linear system above, we can get the regularized optimization parameter set:
\begin{align*}
X &=(A^TC^{-1}A+\frac{1}{\Sigma^2}I)^{-1} A^TC^{-1}Y
\end{align*}

\end{document}


\subsection{Sensitivity Test}
Based on the analysis above, we modify the Multi-groups Logistic Model.
\begin{equation}
  \begin{cases}
    \frac{1}{N_x(t)}\frac{dN_x(t)}{dt} = r_x(t)(1-\frac{N_x(t)}{{{N'}_x}_{max}} - RGB_x\frac{N_y(t)}{{{N'}_y}_{max}}) \\ \\
    \frac{1}{N_y(t)}\frac{dN_y(t)}{dt} = r_y(t)(1-\frac{N_y(t)}{{{N'}_y}_{max}} - RGB_y\frac{N_x(t)}{{{N'}_x}_{max}})
  \end{cases}
\end{equation}
\par
We take $F_A$ and $F_B$ as examples to analyze the sensitivity of our model when environment fluctuates rapidly. Because they are all drought-tolerant fungi, when rapid fluctuations in the environment have an adverse effect, they will appear as heavy precipitation in a short period of time. We artificially set rapid fluctuations in the environment on the fifth day and draw curve figure of $N_A'(t)$ and $N_B'(t)$ as follows.
\begin{figure}[H]
  \centering
  \includegraphics[width=\textwidth]{figures/A&B_sen.png}
  \caption{Curves of $N_A'(t)$ and $N_B'(t)$.}
\end{figure}
\par
From the above figure, it is found that the fungal population maintained an exponential growth in the first $5$ days. After rapid environmental fluctuations occurred on the $5th$ day, the populations of $F_A$ and $F_B$ both declined significantly. Around the $11th$ day, the population began to increase. After $20$ days, the populations of $F_A$ and $F_B$ both stabilized, at $713$ and $439$, respectively. Compared with the stable fungal population of $F_A$ and $F_B$ without the influence of rapid environmental fluctuations, the values have dropped by $10.4\%$ and $15.9\%$, respectively.
\par
Therefore, when rapid environmental fluctuations occur, the \textbf{advanced Multi-groups Logistic Model} could well reflect the influence of harsh environments on the growth and reproduction of fungi, which shows that the model has \textbf{good sensitivity}.


\section*{Introduction}
Our algorithm is composed by two main stages:
\begin{itemize}
\item Find a feasible solution.
\item Optimize the feasible solution found in the previous stage.
\end{itemize}

In order to improve the diversification of the algorithm and to explore as much as possible the solution space, we run several parallel threads and finally we choose the best of the provided solutions.

\section*{Feasibile solution search}
In our approach, we combine a greedy strategy with a taboo list which stores the ``bad'' couples timeslot, exam.

Starting from a random order of exams, we assign each one to the first timeslot where it does not conflict with any other exam. At the end of this loop, it is highly probable that some exams are not inserted in any timeslot.

Hence, we start a loop that continues until all exams are correctly assigned. In this loop we exploit the following algorithm:
\begin{enumerate}
\item Check if a mutation is needed and mutate consequently.
\item For each unassigned exam:
\begin{enumerate}
\item Search the timeslot with the smallest number of conflicts and where the exam is not taboo.
\item Assign the exam to it.
\item Remove all the exams in conflict with the added exam and mark the couple timeslot, exam as taboo.
\end{enumerate}
\item If the number of unassigned exams is lower than the minimum number found so far, update it.
\item If the number of conflicts is lower than the minimum number found so far, update it.
\end{enumerate}
The size of the taboo list is a critical parameter. A large value would store a large number of forbidden couples timeslot, exam making the timeslot choice a difficult operation, while a small value would force a behavior similar to a steepest descent, which conduces to an unfeasible solution. After several attempts, we notice that a good empiric size is 100.

A constant mutation frequency would enforce a strongly modification of an almost feasible solution into a worse one or it may mutate too low frequently. Hence, we adopted for a dynamic approach that changes the control value according to the quality of the current solution.
\paragraph*{Mutation}
For each taken exam:
\begin{enumerate}
\item Search the timeslots where the exam does not conflict and where the move is not taboo.
\item Move the exam in a random timeslot among the selected ones and mark the couple timeslot, exam as taboo.
\end{enumerate}


\chapter{Introduction}

I remind some definitions from volume~1~\cite{volume-1}.

I denote a set definition like $\setcond{x\in A}{P(x)}$ instead of customary
$\{ x\in A \mid P(x) \}$ (in order to reduce formulas size).

I denote partial order as~$\sqsubseteq$. I denote lattice operations as
$\bigsqcap$, $\bigsqcup$, $\sqcap$,~$\sqcup$.

The following generalizes monovalued morphisms in category~$\mathbf{Rel}$.

Let $\Hom$-sets be complete lattices.
\begin{defn}
\index{morphism!metamonovalued}A morphism $f$ of a partially ordered
category is \emph{metamonovalued} when $\left(\bigsqcap G\right)\circ f=\bigsqcap_{g\in G}(g\circ f)$
whenever $G$ is a set of morphisms with a suitable source and destination.
\end{defn}

\begin{defn}
\index{morphism!metainjective}A morphism $f$ of a partially ordered
category is \emph{metainjective} when $f\circ\left(\bigsqcap G\right)=\bigsqcap_{g\in G}(f\circ g)$
whenever $G$ is a set of morphisms with a suitable source and destination.\end{defn}
\begin{obvious}
Metamonovaluedness and metainjectivity are dual to each other.\end{obvious}
\begin{defn}
\index{morphism!metacomplete}A morphism $f$ of a partially ordered
category is \emph{metacomplete} when $f\circ\left(\bigsqcup G\right)=\bigsqcup_{g\in G}(f\circ g)$
whenever $G$ is a set of morphisms with a suitable source and destination.
\end{defn}

\begin{defn}
\index{morphism!co-metacomplete}A morphism $f$ of a partially ordered
category is \emph{co-metacomplete} when $\left(\bigsqcup G\right)\circ f=\bigsqcup_{g\in G}(g\circ f)$
whenever $G$ is a set of morphisms with a suitable source and destination.
\end{defn}
Let now $\Hom$-sets be meet-semilattices.
\begin{defn}
\index{morphism!weakly metamonovalued}A morphism $f$ of a partially
ordered category is \emph{weakly metamonovalued} when $(g\sqcap h)\circ f=(g\circ f)\sqcap(h\circ f)$
whenever $g$ and $h$ are morphisms with a suitable source and destination.
\end{defn}

\begin{defn}
\index{morphism!weakly metainjective}A morphism $f$ of a partially
ordered category is \emph{weakly metainjective} when $f\circ(g\sqcap h)=(f\circ g)\sqcap(f\circ h)$
whenever $g$ and $h$ are morphisms with a suitable source and destination.
\end{defn}
Let now $\Hom$-sets be join-semilattices.
\begin{defn}
\index{morphism!weakly metacomplete}A morphism $f$ of a partially
ordered category is \emph{weakly metacomplete} when $f\circ(g\sqcup h)=(f\circ g)\sqcup(f\circ h)$
whenever $g$ and $h$ are morphisms with a suitable source and destination.
\end{defn}

\begin{defn}
\index{morphism!weakly co-metacomplete}A morphism $f$ of a partially
ordered category is \emph{weakly co-metacomplete} when $(g\sqcup h)\circ f=(g\circ f)\sqcup(h\circ f)$
whenever $g$ and $h$ are morphisms with a suitable source and destination.\end{defn}
\begin{obvious}
~
\begin{enumerate}
\item Metamonovalued morphisms are weakly metamonovalued.
\item Metainjective morphisms are weakly metainjective.
\item Metacomplete morphisms are weakly metacomplete.
\item Co-metacomplete morphisms are weakly co-metacomplete.
\end{enumerate}
\end{obvious}

\begin{defn}
\index{morphism!monovalued}For a partially ordered dagger category
I will call \emph{monovalued} morphism such a morphism $f$ that $f\circ f^{\dagger}\sqsubseteq1_{\Dst f}$.
\end{defn}

\begin{defn}
\index{morphism!entirely defined}For a partially ordered dagger category
I will call \emph{entirely defined} morphism such a morphism $f$
that $f^{\dagger}\circ f\sqsupseteq1_{\Src f}$.
\end{defn}

\begin{defn}
\index{morphism!injective}For a partially ordered dagger category
I will call \emph{injective} morphism such a morphism $f$ that $f^{\dagger}\circ f\sqsubseteq1_{\Src f}$.
\end{defn}

\begin{defn}
\index{morphism!surjective}For a partially ordered dagger category
I will call \emph{surjective} morphism such a morphism f that $f\circ f^{\dagger}\sqsupseteq1_{\Dst f}$.\end{defn}
\begin{rem}
It is easy to show that this is a generalization of monovalued, entirely
defined, injective, and surjective functions as morphisms of the category
$\mathbf{Rel}$.\end{rem}
\begin{obvious}
``Injective morphism'' is a dual of ``monovalued morphism'' and
``surjective morphism'' is a dual of ``entirely defined morphism''.\end{obvious}


\lab{Discrete Hidden Markov Models}{Discrete Hidden Markov Models}
\label{lab:hmm}
\objective{Understand how to use discrete Hidden Markov Models.}

Given a discrete state-space Hidden Markov Model (HMM) with parameters $\lambda$ and an observation sequence $O$, we would like to answer three questions:
\begin{enumerate}
 \item What is $\mathbb{P}(O |\, \lambda)$? In other words, what is the likelihood that our model generated the observation sequence?
 \item What is the most likely state sequence to have generated $O$, given $\lambda$?
 \item How can we choose the parameters $\lambda$ that maximize $\mathbb{P}(O | \, \lambda)$?
\end{enumerate}
The answers to these questions are centered around the \emph{forward-backward} algorithm for HMMs.
For the second question, the approach taken in this lab will be to find the state sequence maximizing the expected number of correct states.
The third question is an example of \emph{unsupervised learning}, since we are attempting to learn (or fit) model parameters using data (the observation sequence $O$) that is devoid
of human-provided labels (the corresponding state sequence); the algorithm does not rely on human supervision or input.

We assume throughout this lab that the HMM has a discrete state space of cardinality $N$ and a discrete observation space of cardinality $M$.
In this context $\lambda = \left( A, B, \mathbf{\pi} \right)$, where $A$ is a $N \times N$  column-stochastic matrix (the state transition model), $B$ is a $M \times N$ column-stochastic matrix (the
state observation model), and $\mathbf{\pi}$ is a stochastic vector of length $N$ (the initial state distribution).
Further, $O$ is a vector of length $T$ with values in the set $\{1,2,\ldots,M\}$.

\begin{warn}
The mathematical exposition in the lab assumes the standard 1-based indexing of vectors and matrices.
Be sure to carefully translate the various formulae into 0-based indexing when implementing these methods for Python coding.
This means that, in Python, your array containing the observation sequence $O$ will actually have values
in the set $\{0,1,\ldots,M-1\}$ so that they may be used to index the matrix $B$ correctly.
\end{warn}

Throughout this lab, we will be using the following toy HMM to verify your code.
\begin{lstlisting}
>>> # toy HMM example to be used to check answers
>>> A = np.array([[.7, .4],[.3, .6]])
>>> B = np.array([[.1,.7],[.4, .2],[.5, .1]])
>>> pi = np.array([.6, .4])
>>> obs = np.array([0, 1, 0, 2])
\end{lstlisting}

\begin{problem}
\begin{comment}
The following was listed as a problem, but gave the student nothing to do, so we removed the given code.

To start off your implementation of the HMM, define a class using the following code.
You will be adding class methods throughout the remainder of the lab.
\begin{lstlisting}
class hmm(object):
    """
    Finite state space hidden markov model.
    """
    def __init__(self):
        """
        Initialize model parameters.
        """
        self.A = None
        self.B = None
        self.pi = None
\end{lstlisting}
\end{comment}
To start off your implementation of the HMM, define a class object which you should call ``hmm".
Then add the initialization method, in which you should set the \emph{self} aspects A, B, and pi to be None objects.
You will be adding methods throughout the remainder of the lab.
\end{problem}

\subsection*{The Forward Pass}
Our first task is to efficiently compute $\log \mathbb{P}(O | \lambda)$.
We can do this using the \emph{forward pass} of the forward-backward algorithm.
We must take care to compute all values in a numerically stable way; we do this by properly scaling values as necessary.

We compute a scaled forward probability matrix $\widehat{\alpha}$ of dimension $T \times N$ as follows:
Let $\widehat{\alpha}_{i,:}, B_{i,:}$ denote the $i$-th rows of $\widehat{\alpha}$ and $B$, respectively, let $\odot$ denote the Hadamard (or entry-wise) product of arrays,
and let $\langle \cdot, \cdot \rangle$ denote the standard dot product.
(Note that here, using 0-based indexing and the toy HMM example, $B_{O_3,:}$ would refer to $[.5,.1]$.)
Then
\begin{itemize}
 \item $c_1 = \langle \pi, B_{O_1,:}\rangle^{-1}$
 \item $\widehat{\alpha}_{1,:} = c_1(\pi\odot B_{O_1,:})$
 \item For $t = 2, \ldots, T$:
 \begin{itemize}[]
     \item $c_t = \langle A\widehat{\alpha}_{t-1,:}, B_{O_t,:}\rangle^{-1}$
	 \item $\widehat{\alpha}_{t,:} = c_t((A\widehat{\alpha}_{t-1,:})\odot B_{O_t,:})$
 \end{itemize}
\end{itemize}
The matrix $\widehat{\alpha}$ will be of use when fitting parameters, but we can compute the desired log probability using the scaling factors $c_t$ as follows:
\[
\log \mathbb{P}(O | \lambda) = -\sum_{t=1}^T \log c_t.
\]

\begin{problem}
Implement the forward pass by adding the following method to your class:
\begin{lstlisting}
def _forward(self, obs):
    """
    Compute the scaled forward probability matrix and scaling factors.

    Parameters
    ----------
    obs : ndarray of shape (T,)
        The observation sequence

    Returns
    -------
    alpha : ndarray of shape (T,N)
        The scaled forward probability matrix
    c : ndarray of shape (T,)
        The scaling factors c = [c_1,c_2,...,c_T]
    """
    pass
\end{lstlisting}
To verify that your code works, you should get the following output using the toy HMM:
\begin{lstlisting}
>>> h = hmm()
>>> h.A = A
>>> h.B = B
>>> h.pi = pi
>>> alpha, c = h._forward(obs)
>>> print -(np.log(c)).sum() # the log prob of observation
-4.6429135909
\end{lstlisting}
\end{problem}

\subsection*{The Backward Pass}
The backward pass of the forward-backward algorithm produces values that can be used to calculate the most likely state sequence corresponding to an observation sequence.

We compute a scaled backward probability matrix $\widehat{\beta}$ of dimension $T \times N$ as follows:
\begin{itemize}
 \item $\widehat{\beta}_{T,i} = c_{T}$ for $i = 1,\ldots, N$
 \item $\widehat{\beta}_{t,:} = c_{t}A^T(B_{O_{t+1},:}\odot \widehat{\beta}_{t+1,:})$ for $t = T-1, \ldots, 1$
\end{itemize}
(Above, $A^T$ is the \emph{transpose} of $A$, not the $T$-th power of $A$.)

It turns out that
\begin{equation*}
\mathbb{P}(\mathbf{x}_{t} = i | O, \lambda) = \frac{\widehat{\alpha}_{t,i}\widehat{\beta}_{t,i}}{\sum_{j=1}^{N} \widehat{\alpha}_{t,j}\widehat{\beta}_{t,j}}
\end{equation*}
and so we can easily compute the most likely state at time $t$ by
\begin{equation*}
\mathbf{x}_{t}^{*} = \argmax_{i} \widehat{\alpha}_{t,i} \widehat{\beta}_{t,i}.
\end{equation*}
This is the solution to the second question posed at the beginning of the lab.

\begin{problem}
Implement the backward pass by adding the following method to your class:
\begin{lstlisting}
def _backward(self, obs, c):
    """
    Compute the scaled backward probability matrix.

     Parameters
    ----------
    obs : ndarray of shape (T,)
        The observation sequence
    c : ndarray of shape (T,)
        The scaling factors from the forward pass

    Returns
    -------
    beta : ndarray of shape (T,N)
        The scaled backward probability matrix
    """
    pass
\end{lstlisting}
Using the same toy example as before, your code should produce the following output:
\begin{lstlisting}
>>> beta = h._backward(obs, c)
>>> print beta
[[ 3.1361635   2.89939354]
 [ 2.86699344  4.39229044]
 [ 3.898812    2.66760821]
 [ 3.56816483  3.56816483]]
\end{lstlisting}
\end{problem}

\subsection*{Computing the $\delta$ and $\gamma$ Probabilities}
Having implemented both parts of the forward-backward algorithm, we are closing in on the solution to question three, namely that of fitting parameters $\lambda$ that maximize $\mathbb{P}(O | \, \lambda)$.
At this stage, we combine the information accumulated in the forward-backward algorithm to produce a three-dimensional array $\widehat{\delta}$
of shape $(T-1)\times N \times N$ whose entries are related to $\mathbb{P}(\mathbf{x}_t = i, \mathbf{x}_{t+1} = j|\, O, \lambda)$, as well as
a $T \times N$ matrix $\widehat{\gamma}$ whose entries are related to $\mathbb{P}(\mathbf{x}_t=i | \, O, \lambda)$.
The relevant formulae are
\[
\widehat{\delta}_{t,i,j} = \frac{\widehat{\alpha}_{t,i}A_{j,i}B_{O_{t+1},j}\widehat{\beta}_{t+1,j}}{\sum_{k,l}\widehat{\alpha}_{t,k}A_{l,k}B_{O_{t+1},l}\widehat{\beta}_{t+1,l}}
\]
for $t = 1, \ldots, T-1$ and $i,j = 1, \ldots, N$,
\[
\widehat{\gamma}_{t,i} = \sum_{j=1}^N \widehat{\delta}_{t,i,j}
\]
for $t = 1,\ldots,T-1$ and $i=1,\ldots,N$, and finally
\[
\widehat{\gamma}_{T,:} = \frac{\widehat{\alpha}_{T,:}\odot \widehat{\beta}_{T,:}}{\langle\widehat{\alpha}_{T,:}, \widehat{\beta}_{T,:}\rangle}.
\]

\begin{problem}
Add the following method to your class to compute the $\delta$ and $\gamma$ probabilities.
\begin{lstlisting}
def _delta(self, obs, alpha, beta):
    """
    Compute the delta probabilities.

    Parameters
    ----------
    obs : ndarray of shape (T,)
        The observation sequence
    alpha : ndarray of shape (T,N)
        The scaled forward probability matrix from the forward pass
    beta : ndarray of shape (T,N)
        The scaled backward probability matrix from the backward pass

    Returns
    -------
    delta : ndarray of shape (T-1,N,N)
        The delta probability array
    gamma : ndarray of shape (T,N)
        The gamma probability array
    """
    pass
\end{lstlisting}
While writing a triply-nested loop may be the simplest way to convert the formula into code,
it is possible to use array broadcasting to eliminate two of the loops, which will speed up your code.

Check your code by making sure it produces the following output, using the same toy example as before.
\begin{lstlisting}
>>> delta, gamma = h._delta(obs, alpha, beta)
>>> print delta
[[[ 0.14166321  0.0465066 ]
  [ 0.37776855  0.43406164]]

 [[ 0.17015868  0.34927307]
  [ 0.05871895  0.4218493 ]]

 [[ 0.21080834  0.01806929]
  [ 0.59317106  0.17795132]]]
>>> print gamma
[[ 0.18816981  0.81183019]
 [ 0.51943175  0.48056825]
 [ 0.22887763  0.77112237]
 [ 0.8039794   0.1960206 ]]
\end{lstlisting}
\end{problem}

\subsection*{Choosing Better Parameters}
After running the forward-backward algorithm and computing the $\delta$ probabilities, we are now in a position to choose new parameters $\lambda' = (A', B', \pi')$
that increase the probability of observing our data, i.e.
\[
\mathbb{P}(O|\,\lambda') \geq \mathbb{P}(O|\,\lambda).
\]
The update formulas are given by
\begin{align*}
A'_{i,j} &= \frac{\sum_{t=1}^{T-1}\widehat{\delta}_{t,j,i}}{\sum_{t=1}^{T-1}\widehat{\gamma}_{t,j}}\\
B'_{i,j} &= \frac{\sum_{t=1}^{T}\widehat{\gamma}_{t,j}1_{\{O_t=i\}}}{\sum_{t=1}^{T}\widehat{\gamma}_{t,j}}\\
\pi' &= \widehat{\gamma}_{1,:}
\end{align*}
where $1_{\{O_t=i\}}$ is one if $O_t=i$ and zero otherwise.
\begin{problem}
Implement the parameter update step by adding the following method to your class:
\begin{lstlisting}
def _estimate(self, obs, delta, gamma):
    """
    Estimate better parameter values.

    Parameters
    ----------
    obs : ndarray of shape (T,)
        The observation sequence
    delta : ndarray of shape (T-1,N,N)
        The delta probability array
    gamma : ndarray of shape (T,N)
        The gamma probability array
    """
    # update self.A, self.B, self.pi in place
    pass
\end{lstlisting}
Verify that your code produces the following output on the toy HMM from before:
\begin{lstlisting}
h._estimate(obs, delta)
>>> print h.A
[[ 0.55807991  0.49898142]
 [ 0.44192009  0.50101858]]
>>> print h.B
[[ 0.23961928  0.70056364]
 [ 0.29844534  0.21268397]
 [ 0.46193538  0.08675238]]
>>> print h.pi
[ 0.18816981  0.81183019]
\end{lstlisting}
\end{problem}

\subsection*{Fitting the Model}
We are now ready to put everything together into a learning algorithm.
Given a sequence of observations, a maximum number of iterations $K$, and a convergence tolerance threshold $\epsilon$, we fit a HMM model using the following procedure:
\begin{itemize}
\item Randomly initialize parameters $\lambda = (A, B, \pi)$
\item Compute $\log \mathbb{P}(O |\, \lambda)$
\item For $i=1, 2, \ldots, K$:
\begin{itemize}
\item Run forward pass
\item Run backward pass
\item Compute $\delta$ probabilities
\item Update model parameters
\item Compute $\log \mathbb{P}(O |\, \lambda)$ according to new parameters
\item If change in log probabilities is less than $\epsilon$, break
\item Else, continue
\end{itemize}
\end{itemize}

The most convenient way to randomly initialize stochastic matrices is to draw from the Dirichlet distribution,
which produces vectors with nonnegative entries that sum to 1.
The following Python code initializes $A$, $B$, and $\pi$ using this technique:
\begin{lstlisting}
>>> # assume N and M are defined
>>> A = np.random.dirichlet(np.ones(N), size=N).T
>>> B = np.random.dirichlet(np.ones(M), size=N).T
>>> pi = np.random.dirichlet(np.ones(N))
\end{lstlisting}

\begin{figure}
\centering
\includegraphics[width=\textwidth]{logProbs.pdf}
\caption{The log probabilities for a HMM trained on the Declaration of Independence
data with 200 iterations. It takes over 100 iterations for the algorithm to work itself out
of a poor local maximum.}
\label{fig:logprobs}
\end{figure}

The learning algorithm is essentially an optimization over the parameter space (i.e. the space of tuples of
stochastic arrays having the proper dimensions) with respect to the objective function $\mathbb{P}(O |\, \lambda)$.
The algorithm is guaranteed to increase the objective function at each iteration, so it is sure to converge.
However, the objective function is riddled with local maxima, and so the outcome depends heavily on the randomly
selected starting values for $A$, $B$, and $\pi$. Figure \ref{fig:logprobs} illustrates the issues involved.
The log probability stays approximately constant for the first 100
iterations. This indicates that the algorithm is not exploring the parameter space enough, and the parameters
found at the 100-th iteration are virtually the same as those found at the first or second iteration. After the first
100 iterations, however, the algorithm is finally able to explore more of the parameter space and hence make
better progress toward increasing the objective function. The moral of the story is that you may need to train
the HMM a few times, using different starting values, and then keep the model that has the highest log likelihood.

\begin{problem}
Implement the learning algorithm by adding the following method to your class:
\begin{lstlisting}
def fit(self, obs, A, B, pi, max_iter=100, tol=1e-3):
    """
    Fit the model parameters to a given observation sequence.

    Parameters
    ----------
    obs : ndarray of shape (T,)
        Observation sequence on which to train the model.
    A : stochastic ndarray of shape (N,N)
        Initialization of state transition matrix
    B : stochastic ndarray of shape (M,N)
        Initialization of state observation matrix
    pi : stochastic ndarray of shape (N,)
        Initialization of initial state distribution
    max_iter : integer
        The maximum number of iterations to take
    tol : float
        The convergence threshold for change in log-probability
    """
    # initialize self.A, self.B, self.pi
    # run the iteration
    pass
\end{lstlisting}
\end{problem}

We now turn to the data found in the file {\tt declaration.txt}.
This file contains the text of the Declaration of Independence.
We will use the sequence of characters (after stripping out punctuation and converting everything to lower-case) as our observation sequence.
In order to convert the raw text into a useable data structure, we need to read in the file, process the string as necessary, and then map the characters to integer values.
We provide sample code below to accomplish this task:
\begin{lstlisting}
>>> import numpy as np
>>> import string

>>> with open("declaration.txt", 'r') as f: # read in the text
>>>     dec = f.read(-1).lower() # convert to lower-case

>>> # next, remove punctuation and newline characters
>>> dec = dec.translate(string.maketrans("",""), string.punctuation+"\n\r")

>>> # create a list of the unique characters in the text
>>> char_map = list(set(dec))

>>> # map each character to its index in char_map
>>> obs = []
>>> for char in dec:
>>>     obs.append(char_map.index(char))
>>> obs = np.array(obs)
\end{lstlisting}

\begin{problem}
You are now ready to train a HMM using the Declaration of Independence data.
Use $N=2$ states and $M=27$ observation values (26 lower case characters and 1 whitespace character),
and run for 200 iterations with the default value for \li{tol}.
Generally speaking, if you converge to a log probability greater than $-21550$, then you have reached
an acceptable set of parameters for this dataset.

Once the learning algorithm converges, analyze the state observation matrix $B$.
Note which rows correspond to the largest and smallest probability values in each column of $B$,
and check the corresponding characters.
The code below displays typical results for a well-converged HMM:
\begin{lstlisting}
>>> for i in xrange(len(h.B)):
>>>     print  "{0}, {1:0.4f}, {2:0.4f}".format(char_map[i], h.B[i,0], h.B[i,1])
 , 0.0051, 0.3324
a, 0.0000, 0.1247
c, 0.0460, 0.0000
b, 0.0237, 0.0000
e, 0.0000, 0.2245
d, 0.0630, 0.0000
g, 0.0325, 0.0000
f, 0.0450, 0.0000
i, 0.0000, 0.1174
h, 0.0806, 0.0070
k, 0.0031, 0.0005
j, 0.0040, 0.0000
m, 0.0360, 0.0000
l, 0.0569, 0.0001
o, 0.0009, 0.1331
n, 0.1207, 0.0000
q, 0.0015, 0.0000
p, 0.0345, 0.0000
s, 0.1195, 0.0000
r, 0.1062, 0.0000
u, 0.0000, 0.0546
t, 0.1600, 0.0000
w, 0.0242, 0.0000
v, 0.0185, 0.0000
y, 0.0147, 0.0058
x, 0.0022, 0.0000
z, 0.0010, 0.0000
\end{lstlisting}
What do you notice about the second column of $B$? It seems that the HMM has detected a vowel state and a consonant state, without any prior input from an English speaker.
Interestingly, the whitespace character is grouped together with the vowels. A HMM can also detect the vowel/consonant distinction in other languages. It appears that
this distinction is a statistically significant aspect of much of human language.
\end{problem}


\documentclass[../main/thesis_msc.tex]{subfiles}

\begin{document}

\chapter{Appendix}

\section{Radio Interferometry Measurement Equation- RIME formalism}

The calibration techniques used for LOFAR are based on RIME formalism. The explanation closely follows the one given in \citep{rime}, since the Direction Dependant Effects (DDE) are well explained in it. This section describes the calculation of RIME from the voltages obtained at the antennae.\\
\noindent Consider a quasi-monochromatic source giving out a signal. This can be represented by a complex vector \textbf{\textit{e}} (the ``original signal"), and be written in the form of a matrix using an orthogonal coordinate system \textit{xyz}, with `\textit{z}' along the direction of propagation.
\begin{center}
\(
\textbf{\textit{e}} =
  \begin{pmatrix}
    e_x \\
    e_y
  \end{pmatrix}
\)
\end{center}
\noindent The signal encounters multiple effects on its path towards the antennae. These effects are assumed to affect the signal \textit{\textbf{linearly}}. The signal changes to \textbf{\textit{e$^\backprime$}} due to these effects and can be written in the form given in the equation \ref{eq}.
\begin{equation}
\textbf{\textit{e$^{\backprime}$}} = \underbrace{\textbf{\textit{J}}_n \textbf{\textit{J}}_{n-1} ..... \textbf{\textit{J}}_1}_{\textbf{Jones chains}} \textbf{\textit{e}} = \textbf{\textit{J}} \textbf{\textit{e}}
\label{eq}
\end{equation}
\noindent Here, \textit{\textbf{J}} is a 2$\times$2 complex matrix known as Jones matrix. Since there are multiple effects along the path of the signal, the Jones chain is written taking all these effects into account and the final cumulative Jones matrix can be written as \textit{\textbf{J}}.
\noindent If a and b are two linear dipole feeds, the signal one reaching and gets converted to complex voltages $v_a$ and $v_b$ representing the two polarizations.
\begin{equation}
\textrm{
\(
\textbf{\textit{v}} =
  \begin{pmatrix}
    v_a \\
    v_b
  \end{pmatrix}
\ = \textbf{\textit{J}} \textbf{\textit{e}} \)
}
\label{J}
\end{equation}
\noindent An interferometer consists of several antennae elements, so let us consider two such spatially separated elements - \textit{p} and \textit{q}, giving independent voltage vectors $\emph{v}_p$ and $\emph{v}_q$.  Their voltages are correlated to give the visibility matrix V$_{pq}$. Hence, the visibility matrix can be written as:
\begin{center}
\(
\textrm{V}_{pq} = 2
  \begin{pmatrix}
    \langle v_{pa}v^*_{qa} \rangle & \langle v_{pa}v^*_{qb} \rangle \\
    \langle v_{pb}v^*_{qa} \rangle & \langle v_{pb}v^*_{qb} \rangle
  \end{pmatrix}
\)
\end{center}
\noindent Here, $v^*$ represents the complex conjugate of $v$. V$_{pq}$ can be written as matrix product of $\emph{v}_p$ and complex conjugate of $\emph{v}_q$. \textit{H} represents the conjugate transpose operation.
\begin{equation}
\textrm{V}_{pq} = 2\langle v_{p}v^H_{q} \rangle
\label{H}
\end{equation}

\noindent Combining equations \ref{J} and \ref{H}, we get:
\begin{equation}
\textrm{V}_{pq} = 2\langle \textbf{\textit{J}}_{p}(\textbf{\textit{ee}}^H) \textbf{\textit{J}}_{q} \rangle
\end{equation}
\noindent We assume that \textbf{\textit{J}}$_{p}$ and \textbf{\textit{J}}$_{q}$  are constant over the averaging interval.
\begin{equation}
\textrm{V}_{pq} = 2\langle v_{p}v^H_{q} \rangle = 2\textbf{\textit{J}}_p \begin{pmatrix}
    \langle e_{x}e^*_{x} \rangle & \langle x_{x}e^*_{y} \rangle \\
    \langle e_{y}e^*_{x} \rangle & \langle e_{y}e^*_{y} \rangle
  \end{pmatrix} \textbf{\textit{J}}^H_q
\end{equation}
\noindent A relation can be obtained between the source signal and Stokes parameters (I, Q, U, V) as shown in \citep{RIME1}, and a new matrix called the brightness matrix B is defined.
\begin{equation}
2\begin{pmatrix}
    \langle e_{x}e^*_{x} \rangle & \langle x_{x}e^*_{y} \rangle \\
    \langle e_{y}e^*_{x} \rangle & \langle e_{y}e^*_{y} \rangle
  \end{pmatrix}   = \underbrace{ \begin{pmatrix}
    I + Q & U + iV \\
    U + iV & I - Q
  \end{pmatrix} }_{\textrm{Brightness matrix B}}
\end{equation}
\noindent The signal undergoes sequential layers of corruption, and the resulting Jones matrix \textbf{\textit{J}}$_{p}$  can be written in the form of Jones chain: \textbf{\textit{J}}$_{p}$  = (\textbf{\textit{J}}$_{pn}$ . \textbf{\textit{J}}$_{p(n - 1)}$ .....\textbf{\textit{J}}$_{p1}$ ).
\noindent The final Jones matrix can be thus written as a cumulative of the following terms:
\begin{itemize}
\item The \textbf{phase} term: Phase difference exists between the two antennae due to the path-length difference between the paths from source to \textit{p} and \textit{q}. This in turn corrupts the signal. Phase center is the direction in which the antennae are steered towards to minimize the phase difference\footnote{The net measured visibility is lowered in amplitude due to the presence of this complex term in the Jones matrix, which is variable in terms of both frequency and time. This effect is called ``smearing".}.  \\
Considering the conventional coordinate system, with \textit{z} axis pointed towards the phase center, antenna \textit{p}'s location can be defined as \textit{\textbf{u}}$_p$ = ($u_p, v_p, w_p$). Thus, the phase term in Jones matrix formalism can be written as: \\
\begin{center}
$\textbf{\textit{K}}_p = e^{i \kappa_p} = e^{-2i(u_pl + v_pm + w_p(n-1))}$
\end{center}
Here, \textit{\textbf{u}} is defined in terms of wavelength. The visibility can hence be written (taking inly the phase term into account)
\begin{center}
$\textrm{V}_{pq} = \textbf{\textit{K}}_p \textrm{B} \textbf{\textit{K}}_q^H$
\end{center}
The phase term is a scalar matrix, and hence can be moved around in the Jones chain.
\item The \textbf{source-independent antenna} gain term: The interferometer itself also has some corrupting effects, and these can be considered in the Jones chain in the form $\textbf{\textit{G}}_p$.Thsi describes the Direction Independent Effects (DIEs) or the uv- Jones term.
\item The \textbf{source-dependent} gain term: This is the remainder of the Jones chain of the form \textbf{\textit{E}}$_{sp}$. This represents the Direction Dependent Effects (DDEs) or the sky-Jones term.
\end{itemize}
\noindent Hence, in total, the Jones chain can be written as:
\begin{equation}
\textbf{\textit{J}}_{sp} =  \textbf{\textit{G}}_{p} \textbf{\textit{E}}_{sp} \textbf{\textit{E}}_{sp}
\end{equation}
\noindent The final visibility matrix can be written\footnote{This equation is in the ``\textbf{onion form}". In this way, the various effects/corruptions are sequentially applied to the signal} as:
\begin{equation}
\textrm{V}_{pq}=\textbf{\textit{G}}_{p} \left( \sum\limits_{s} \textbf{\textit{E}}_{sq} \textbf{\textit{K}}_{sq} \textrm{B}_s \textbf{\textit{K}}^H_{sq} \textbf{\textit{E}}^H_{sq} \right) \textbf{\textit{G}}^H_{q}
\end{equation}
In LOFAR, this equation is taken over all the sufficiently bright sources in the horizon. However, sky is a continuous brightness distribution B($\sigma$) with $\sigma$ being the unit direction vector. Hence, the total visibility for the interferometer should be in the form of an integration, not as a summation of discrete sources. When we write the visibility matrix in terms of the plane projection as described earlier (as opposed to the unit sphere integral, which is not very traceable) it is of the form in the equation \ref{wterm}, with $ n = \sqrt{1 - l^2 - m^2}$.
\begin{equation}
\textrm{V}_{pq}= \textbf{\textit{G}}_{p} \left( \int\limits_{l} \int\limits_{m} \frac{1}{n} \bar{\textbf{\textit{E}}}_{p}  (\textbf{\textit{l,m}}) \text{ B } \bar{\textbf{\textit{E}}}^H_{q} (\textbf{\textit{l,m}}) e^{-2 \pi i(u_{pq}l + v_{pq}m + w_{pq}(n-1))} dl dm \right) \textbf{\textit{G}}^H_{q}
\label{wterm}
\end{equation}
\noindent Decomposing $w_{pq} = w_p - w_q$ and writing the non-coplanetary term as per antenna terms, we can substitute $W_p= \frac{1}{\sqrt{n}} e ^{-2 \pi i w_p (n-1)}$ and write $\textbf{\textit{E}}_p =  \bar{\textbf{\textit{E}}}_p W_p $. This would give us the visibility matrix in the form of a 2D Fourier Transform  of the apparent sky brightness for the baseline $pq$ (with the term B$_{pq}$ in the place of  $\textbf{\textit{E}}_{p}$B$\textbf{\textit{E}}_{q}$).
\begin{equation}
\textrm{V}_{pq} = \textbf{\textit{G}}_{p}  \left( \int\limits_{l} \int\limits_{m} \textrm{B}_{pq} e^{-2 \pi i(u_{pq}l + v_{pq}m)} \right) \textbf{\textit{G}}^H_{q}
\end{equation}
\noindent This is the general form of the \textbf{Van Cittert Zernike theorem}. This is the formalism used by the LOFAR calibration software. This equation effectively takes care of the direction dependent effects. Using this fourier relation, and applying the Inverse Fast Fourier Transform (IFFT) algorithm, we can synthesis an image.

\section{NGC 628- correlation plot}
The Figure \ref{ngc_acd} shows the correlation plot for the thermal region for A, C and D regions, for NGC\,628. The regions are marked in the Chapter 3 of the thesis in Figure 3.12.

    \begin{figure}[h]
	\centering
	\includegraphics[scale = 0.65]{13Regions_thermal_region_ACandD.png}
	\caption{Correlation of the flux density value with the spectral index map values, for each pixel. The various regions have different 			slops of correlation, which is interesting. This plot shows the regions A, C and D.}
	\label{ngc_acd}
	\end{figure}

\section{Making of thermal maps}
To understand how the thermal map is made, one can refer to \citet{2017A&A...600A...6M}. In this section I will explain why a simpple H-$\alpha$ map might not be sufficient if one needs to do an in-depth analysis of thermal emission in the galaxy, and for an accurate determination of the magnetic fields in the galaxy. Classically, the separation of the thermal and non-thermal components has been done by assuming the spectral index of synchrotron emission to be $\alpha_{\text{Syn}}$ = $-$1.0. This method overestimates the amount of thermal emission in the spiral arms, as the $\alpha_{\text{Syn}}$ in such regions is flatter than the assumed value. The $\alpha_{\text{Syn}}$ is also steeper in the inter-arm regions and the outer disk of the galaxy, where a higher amount of synchrotron emission takes place. This leads to an overestimation of the thermal emission in such regions. Another approach is to use 24~$\mu$m infrared (IR) map to estimate the thermal emission. This method also hasa its own set of caviates viz, IR emission may also come from non-thermal sources such as super massive black holes, and energetic cosmic rays (CRs)may also produce IR, thus overestimating the thermal fractionsw in such regions. In this thesis, the thermal map has been made using an extinction corrected H-$\alpha$ map. An H-$\alpha$ map in itself suffers from dust extinction, which would underestimate the amount of thermal emission. In the method where extinction correction is done, the optical depth for the whole galaxy is obtained using high resolution FIR data at 70 and 160~$\mu$m. Using this, one can determine the amount of extinction, thus producing a very good estimate of the thermal tempelate. This method was developed by \citet{2007A&A...475..133T}, and is used in the thermal map that I use in my thesis for NGC\,628. The thermal map for IC\,342 was based on the separation of the thermal and non- thermal components by assuming a constant spectral index, as the extintion corrected thermal map was not available. My thesis does not require an accurate determination of magnetic fields and cosmir ray propagation. Thus, a superficial understanding of the thermal regions in the galaxy was sufficient.
\end{document}


\documentclass[main.tex]{subfiles}
\begin{document}

\section{Fisher Matrix}

\marginpar{Tuesday\\ 2020-11-24, \\ compiled \\ \today}

Suppose we have a data vector \(\vec{x}\), such that each element is distributed according to a Gaussian with mean \(\mu \) and standard deviation \(\sigma \).
We could construct an estimator for the mean like \(\hat{\theta} = f(\vec{x}) = x_3\).
This is unbiased: \(\expval{x_3} = \mu = \expval{x_i}\) for any \(i\).

However, this is not the best estimator we could choose, as its variance is \(\sigma^2\); we could instead use the arithmetic mean, which is also unbiased but has a variance of \(\sigma^2 / N\).

An \emph{unbiased} estimator \(\theta \) for \(\hat{\theta}\) is one such that \(\expval{\hat{\theta}} = \theta \).

Given a likelihood \(\mathscr{L}(\vec{x} | \vec{\theta})\), the question is: what is the \textbf{best unbiased estimator} for \(\theta \)?
In order to answer this, we need to define the \textbf{Fisher information matrix}:
%
\begin{align}
F_{\alpha \beta } = \expval{- \pdv[2]{\log \mathscr{L}}{\theta_\alpha}{\theta _\beta }}
\,.
\end{align}

The average is an ensemble one, over all the possible realizations of the data.
This is precisely the Hessian of the log-posterior, computed at the maximum of the posterior, which estimates the precision matrix of the parameters.
We have nice theorems for these.

\begin{theorem}
    For any unbounded estimator \(\hat{\theta}_\alpha \) of \(\theta _\alpha \), its error is bounded by
    %
    \begin{align}
    \sqrt{\expval{\hat{\theta}_\alpha^2} - \expval{\hat{\theta}_\alpha }^2} \geq \sqrt{F_{\alpha \alpha }^{-1}}
    \,.
    \end{align}
\end{theorem}

This is known as the \textbf{Cramer-Rao} bound. So, if we can saturate the inequality, then we know we have the best possible estimator.

\begin{theorem}
    If there exists a Best Unbiased Estimator, then this is also the Maximum Likelihood estimator.
\end{theorem}

\begin{theorem}
    The ML estimate is asymptotically the BUE.
\end{theorem}

Consider a likelihood \(P(\vec{d} | \theta)\); then the variance is given by
%
\begin{align}
\expval{(\hat{\theta} - \expval{\hat{\theta}})^2} = \int \dd{\vec{d}} P(\vec{d} | \theta ) (\theta - \expval{\hat{\theta}})^2
\,.
\end{align}

Let us define the product of two functions as
%
\begin{align}
\qty(u (\vec{d}), v (\vec{d})) = \int \dd{\vec{x}} u (\vec{x}) v (\vec{x}) P(\vec{x} | \theta )
\,.
\end{align}

Then, the variance of the estimator \(\hat{\theta} \) can be expressed by defining \(v(\vec{d}) = \hat{\theta} - \expval{\hat{\theta}}\), and then \(\var{\hat{\theta}} = \qty(v(\vec{d}), v(\vec{d}))\).

Now we will use the Schwarz inequality, which says that in an inner product space:
%
\begin{align}
(v, v) \geq \frac{(u, v)^2}{(u, u)}
\,.
\end{align}

We will apply this, with the aforementioned \(v\), and with the so-called \emph{score function}:
%
\begin{align}
u(\vec{d}) = \pdv{}{\theta } \log P(\vec{d} | \theta )
\,.
\end{align}
%

Then,
%
\begin{align}
(u, w) &= \int \dd{\vec{x}} u(\vec{x}) w(\vec{x}) P(\vec{x} | \theta )  \\
&= \int \dd{\vec{x}} \pdv{}{\theta } \log P(\vec{x} | \theta ) w(\vec{x}) P(\vec{x} | \theta )  \\
&= \int \dd{\vec{x}} \frac{1}{P(\vec{x} | \theta )} \pdv{P}{\theta } w(\vec{x} ) P(\vec{x} | \theta )  \\
&= \pdv{}{\theta } \expval{w}
\,.
\end{align}

Then,
%
\begin{align}
(u, v) &= \int (\hat{\theta} - \expval{\hat{\theta}}) \pdv{\log P}{\theta } P(\vec{x} | \theta ) \dd{\vec{x}} \\
&= \int \hat{\theta} \pdv{\log P}{\theta } P(\vec{x} | \theta ) \dd{\vec{x}} -
\expval{\hat{\theta}} \underbrace{\int \pdv{\log P}{\theta } P(\vec{x} | \theta ) \dd{\vec{x}}}_{= \pdv{}{\theta } \int P \dd{\vec{x}} = 0}  \\
&= \int \hat{\theta} \pdv{\log P}{\theta } P(\vec{x} | \theta ) \dd{\vec{x}}  \\
&= \pdv{}{\theta } \expval{\hat{\theta}} = 1
\,,
\end{align}
%
therefore, we can apply Schwarz's inequality so that
%
\begin{align}
\var{\hat{\theta}} = (v, v) \geq \frac{(u, v)^2}{(u, u)} = \frac{1}{(u, u)}
= \qty(\int \dd{x} \qty(\pdv{}{\theta } \log P)^2 P(\vec{x} | \theta ))^{-1} = \expval{\qty(\pdv{\log P}{\theta })^2}^{-1}
\,,
\end{align}
%
so what we are left to prove is that
%
\begin{align}
\expval{\qty(\pdv{\log P}{\theta })^2} = \expval{\pdv[2]{\log P}{\theta }}
\,.
\end{align}

\end{document}


\chapter{Nonlinear Ordinary Differential Equations and Iteration}

The implicit explicit method avoids the direct solution of nonlinear problems. This can be advantageous for some problems, but can also lead to severe time step restrictions in others. Furthermore, the resulting numerical schemes can sometimes have undesirable qualitative properties. For this reason, we need to describe methods that allow us to solve the nonlinear equations generated in fully-implicit numerical schemes.

We consider an ordinary differential equation
\begin{equation}
\frac{\mathrm{d}y}{\mathrm{d}t}=f(t,y)
\end{equation}
for $t\in[t_{0},t^{*}]$, and for which $f(t,y)$ is not necessarily a linear function of $y$. We want to use an implicit numerical method to obtain an approximate solution of this problem -- for example backward Euler's method. If we want to demonstrate the convergence of the numerical scheme, we need to demonstrate convergence of functional iteration which we use to find the solution for the nonlinear equation term in using backward Euler's method.

The results that follow are primarily taken from Iserles~\cite{Ise09}, although this material is also often found in calculus texts such as Lax, Burstein and Lax~\cite{LaxBurLax76}, and Hughes et al.~\cite{HugEtAl08}. We will let $t_{i}$ denote the time at time step $i$, $y_{i}$ denote the approximate solution at time step $i$ and $h$ denote the time step. We will assume $f$ is Lipschitz continuous, a condition that is weaker than differentiable but stronger than continuous, which we will give a precise definition of.  There are two classical iteration methods:
\begin{itemize}
\item fixed-point iteration
\item Newton's (Newton-Raphson) method.
\end{itemize}
We will prove convergence of these two methods (a proof of the convergence of the modified Newton-Raphson method is in Iserles~\cite[p. 130]{Ise09}). We will analyze the specific problem $y'(t)=y^{2}$ with initial data
$y(0)=1$ and $t\in[0,0.99]$.

%%%%%%%
%Section
%%%%%%%
\section{Exact Solution to an Example Nonlinear Ordinary Differential Equation}

We consider
\begin{equation}
\frac{\mathrm{d}y}{\mathrm{d}t}=y^{2}
\end{equation}
with initial data $y(t=0)=1$ and $t\in[0,0.99]$.  Whenever the solution $y(t)$ exists, it will be positive all the time, because the initial value is positive and $\frac{\mathrm{d}y}{\mathrm{d}t}$ is positive.

To integrate this equation explicitly, we use separation of variables to find that
\begin{align}
&{}\int_{y(0)}^{y(t)}\frac{1}{\tilde{y}^{2}}\mathrm{d}\tilde{y}=\int_0^t\mathrm{d}\tau
\end{align}
which implies
\begin{equation}
-\frac{1}{y(t)}=t+c
\end{equation}
where $c$ is the constant of integration. Using our initial data we get $c=-1$, so
\begin{equation}
y(t)=\frac{1}{1-t}
\end{equation}
is our exact solution for this problem. We will use this exact solution to compare the numerical solutions obtained by the different iterative methods. Notice that this exact solution becomes infinite as $t\rightarrow1$.

%%%%%%%
%Section
%%%%%%%
\section{Definitions Required to Prove Convergence}

\begin{definition}{\bf The Lipschitz Condition}
A function $f(x):x\in D\subset\mathbb{R}$ is Lipschitz if $\left\Vert f(x_1)-f(x_2)\right\Vert \leq\lambda\left\Vert x_1-x_2\right\Vert $ for all $x_1$ and $x_2$ in the domain $D$.
\end{definition}

There are two specific definitions of the Lipschitz condition.

\begin{definition}{\bf Locally Lipschitz Condition}
The function $f(x)$ is called locally Lipschitz if, for each $z\in\mathbb{R}$, there exists an $L>0$ such that $f$ is Lipschitz on the open ball of center $z$ and radius $L$.
\end{definition}

\begin{definition}{\bf Globally Lipschitz Condition}
If $f(x)$ is Lipschitz on all of the space $\mathbb{R}$ (i.e. The open ball is $\mathbb{R}$ in above definition), then $f$ is globally Lipschitz.
\end{definition}

Note the fundamental difference between the local and global versions of the Lipschitz-condition. Whereas in the local version the Lipschitz ``constant''($\lambda)$ and the open ball depend on each point $x\in\mathbb{R}$ , in the global version the ``constant''($\lambda)$ is fixed and the open ball is $\mathbb{R}$. In particular, a globally Lipschitz function is locally Lipschitz continuous, but the converse is not true.

%%%%%%%
% Section
%%%%%%%
\section{Existence and Uniqueness of Solutions to Ordinary Differential Equations}

Peano's theorem states that if $f(x)$ is continuous, then a solution to the ordinary differential equation $x'(t)=f(x)$ with initial condition $x(t_0)=x_0$ exists  at least in some neighbourhood of time $t_0$ -- this solution need not be unique. Picard's theorem states that if $f(x)$ is locally Lipschitz, then the solution for the ordinary differential equation $x'(t)=f(x)$ with initial condition $x(t_0)=x_0$ is unique when it exists. A comprehensive statement of these theorems is in Iserles~\cite[p. 445]{Ise09}, and there are proofs of these theorems in many books on ordinary differential equations (for example Birkhoff and Rota~\cite[Chap. 6, pg.\ 192]{BirRot89}).

%%%%%%%
%Section
%%%%%%%
\section{Backward Euler}

We recall that the backward Euler method is given by
\begin{equation}
y^{n+1}=y^{n}+hf(y^{n+1}).
\end{equation}
Note that if $f$ is nonlinear, we need to solve a nonlinear equation in each step advancing the solution (numerical). It is usually hard to solve a nonlinear equation exactly using analytical methods, so we also use numerical methods.  For our example equation, we get
\begin{equation}
y^{n+1}=y^{n}+h\left(y^{n+1}\right)^2
\end{equation}
This example has the advantage that we can find its solutions algebraically, so we can then examine the behavior of numerical schemes.

%%%%%%%
%Section
%%%%%%%
\section{Convergence of Functional Iteration}

We often use functional iteration to solve nonlinear equations. We recall that there are two popular methods: fixed-point iteration and Newton's method.

%subsection
\subsection{Convergence of the Fixed-Point Method}

We want to find a root of $x=f(x).$ We try to use the fixed-point method and to construct a sequence $x_{n+1}=f(x_{n})$ where $n=0,1,2\ldots$.

\begin{theorem}
Let $f(x)$ have a fixed-point $\tilde{x}=f(\tilde{x})$, be Lipschitz continuous for $x\in(a, b)\subset\mathbb{R}$ with Lipschitz constant $k<1$ and $f(x)$ be continuous on $[a,b]$. Then the fixed point method $x_{n+1}=f(x_{n})$ converges to the unique fixed-point of $\tilde{x}=x_{\infty}=f(x_{\infty})$ for $x\in[a,b]$.
\end{theorem}

\begin{proof}
Since $f(x)$ is Lipschitz continuous, we find that,
\begin{equation}
\left|x_{n+1}-x_{\infty}\right|=\left|f(x_{n})-f(x_{\infty})\right|\leq k\left|x_{n}-x_{\infty}\right|
\end{equation}
for $n=1,2\ldots$. Hence by induction we conclude that
\begin{equation}
\left|x_{n+1}-x_{\infty}\right|\leq k^{n}\left|x_{1}-x_{\infty}\right|.
\end{equation}
Since $k<1$, $\lim_{n\rightarrow\infty}k^n\lvert x_1-x_{\infty}\rvert=0$, so we obtain a solution $x_{\infty}=f(x_{\infty})$, where $x_{\infty}$ is the fixed point. We can show that the limit is unique by supposing that there are two different limits and reaching a contradiction.\end{proof}

For a proof of the existence of the fixed-point under the assumptions used in this theorem, see a book on numerical analysis, such as Bradie~\cite{Bra06} or Iserles~\cite{Ise09}.

Regarding our problem, we apply fixed-point iteration, we want to find the root of an equation of the form:
\begin{equation}\label{eq:Fp}
w=hw^{2}+\beta=f(w).
\end{equation}
When the timestep $h$ is small enough then $f'(w)=2hw\leq200h<1$. So fixed-point iteration is convergent provided the time-step is small enough. We note that eq.\ \eqref{eq:Fp} has two roots, and so the domain of the initial iterate plays an important role in determining which root is choosen.

%subsection
\subsection{Convergence of Newton's Method}

We now consider Newton's method. We want to find a root, $x^*$ of $f(x)$ such that $f(x^*)=0.$ Newton's method is a fixed-point method where the iterates are constructed by
\begin{equation}\label{eq:Newton}
x_{n+1}=x_{n}-\frac{f(x_{n})}{f'(x_{n})}
\end{equation}
where $n=0,1,2\ldots$. If the function $f(x)$ is sufficiently well behaved, then Newton's method has a quadratic rate of convergence.

\begin{theorem}
Suppose $f(x)$ is twice continuously differentiable and that its second derivative is bounded. Suppose also that there exists $x^*$ for which $f(x^{*})=0$. Suppose $f'(x)\neq0$ in the interval $\left[x^{*}-\left|x^{*}-x_{0}\right|,x^{*}+\left|x^{*}-x_{0}\right|\right]$, $f''(x)$ is finite in the same interval and $\left\lvert x_{0} - x^{*}\right\rvert$ is small. Then, Newton's method is of quadratic convergence.
\end{theorem}

\begin{proof}
\begin{equation}
f(x^{*})=f(x_{n})+f'(x_{n})(x^{*}-x_{n})+\frac{1}{2!}f''(z_{n})(x^{*}-x_{n})^{2}
\end{equation}
by Taylor expansion with Lagrange form remainder. In the above $z_{n}\in[x_{n},x^{*}]$. Since $f(x^*)=0$, we have
\begin{equation}
0=f(x_{n})+f'(x_{n})(x^{*}-x_{n})+\frac{1}{2!}f''(z_{n})(x^{*}-x_{n})^{2},
\end{equation}
so
\begin{equation}
\frac{f(x_{n})}{f'(x_{n})}+(x^{*}-x_{n})=-\frac{1}{2!}\frac{f''(z_{n})}{f'(z_{n})}(x^{*}-x_{n})^{2}.
\end{equation}
Plug in the formula for $x_{n+1}$, from eq.\ \eqref{eq:Newton}  we have
\begin{equation}
x^{*}-x_{n+1}=-\frac{1}{2!}\frac{f''(z_{n})}{f'(z_{n})}(x^{*}-x_{n})^{2}.
\end{equation}
Let
\begin{equation}
e_{n}=\left|x^{*}-x_{n}\right|.
\end{equation}
We have
\begin{equation}
e_{n+1}=\left|\frac{1}{2!}\frac{f''(z_{n})}{f'(z_{n})}\right|e_{n}^{2}
\end{equation}
and by our assumption, we know there is a constant $c$ such that
\begin{equation}
\left|\frac{1}{2!}\frac{f''(z_{n})}{f'(z_{n})}\right|<c.
\end{equation}
Hence we have $e_{n+1}<me_{n}^{2}$ for some finite constant $m$. So Newton's method is convergent provided $e_0=\lvert x_0-x^*\rvert$ is sufficiently small.
\end{proof}

Regarding our problem, we consider
\begin{equation}
f(y)=y-hy^{2}-\beta.
\end{equation}
Hence $f'(y)=1-2hy\neq0$ and $f"(y)$ is finite, so our problem satisfies all assumptions if we choose our initial data and initial iterates suitably. Hence the Newton iterations will converge and give an approximation to the nonlinear term in backward Euler's method.

%%%%%%%
%Section
%%%%%%%
\section{Convergence of the Theta Method}

The backward Euler, forward Euler and Crank-Nicolson methods are special case of the theta method, so we will first prove the convergence of the theta method to encompass these three methods. The theta method is the following algorithm,
\begin{equation}
y^{n+1}=y^{n}+h[\theta f(t^{n},y^{n})+(1-\theta)f(t^{n+1},y^{n+1})]
\end{equation}
where
$n=0,1,\ldots$ and $\theta\in[0,1]$. Notice that for $\theta=1/2$ we obtain the Crank-Nicolson method or trapezoidal rule.

First, substituting the exact solution $y(t)$ and using the Taylor expansion we have
\begin{align}
&{}y(t^{n+1})-y(t^{n})-h[\theta f(t^{n},y(t^{n}))+(1-\theta)f(t^{n+1},y(t^{n+1}))]
\\&{}=y(t^{n+1})-y(t^{n})-h[\theta y'(t^{n})+(1-\theta)y'(t^{n+1})] \notag
\\&{} =[y(t^{n})+hy'(t^{n})+\frac{1}{2}h^{2}y''(t^{n})+\frac{1}{6}h^{3}y^{\prime\prime\prime}(t^{n})] \notag
\\&{}\hspace{2em}-y(t^{n})-h\{\theta y'(t^{n})+(1-\theta)[y'(t^{n})+hy''(t^{n})+\frac{1}{2}h^{2}y^{\prime\prime\prime}(t^{n})]\}+\mathcal{O}(h^{4}) \notag
\\&{}=\left(\theta-\frac{1}{2}\right)h^{2}y''(t^{n})+\left(\frac{1}{2}\theta-\frac{1}{3}\right)h^{3}y^{\prime\prime\prime}(t^{n})+\mathcal{O}(h^{4}). \notag
\end{align}
Subtracting the last expression from
\begin{equation}
y^{n+1}-y^{n}-h[\theta f(t^{n},y^{n})+(1-\theta)f(t^{n+1},y^{n+1})]=0,
\end{equation}
we have that when $h$ is small enough
\begin{align}
&{}e^{n+1,h}
\\&{}=e^{n,h}+\theta h[f(t^{n},y(t^{n})+e^{n,h})-f(t^{n},y(t^{n}))] \notag
\\&{}\hspace{1em}+(1-\theta)h[f(t^{n+1},y(t^{n+1})+e^{n+1,h})-f(t^{n+1},y(t^{n+1}))] \notag
\\&{} \begin{cases}
-\frac{1}{12}h^{3}y^{\prime\prime\prime}(t^{n})+\mathcal{O}(h^{4}), & \theta=\frac{1}{2}\\
+(\theta-\frac{1}{2})h^{2}y''(t^{n})+\mathcal{O}(h^{3}), & \theta\neq\frac{1}{2}
\end{cases} \notag
\end{align}
where $e^{i}=y^{i}-y(t^{i})$. Using the triangle inequality and by the Lipschitz continuity of $f$, there exist constants $c$ and $\lambda$ such that
\begin{align}
&{}\left\Vert e^{n+1,h}\right\Vert
\\&{} \leq\left\Vert e^{n,h}\right\Vert +\theta h\lambda\left\Vert e^{n,h}\right\Vert +(1-\theta)h\lambda\left\Vert e^{n+1,h}\right\Vert +\begin{cases}
ch^{3} & \theta=\frac{1}{2}\\
ch^{2} & \theta\neq\frac{1}{2}
\end{cases}. \notag
\end{align}
When $\theta=\frac{1}{2}$, the theta method reduces to the trapezoidal rule. It is possible to show that the Crank-Nicolson method has second order convergence, see for example, Iserles~\cite{Ise09}. Now let's consider $\theta\neq\frac{1}{2}$,
\begin{align}
&{}\left\Vert e^{n+1,h}\right\Vert \leq\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\left\Vert e^{n,h}\right\Vert +\frac{c}{1-(1-\theta)h\lambda}h^{2}.
\end{align}

We claim that
\begin{equation}
\left\Vert e^{n,h}\right\Vert \leq\frac{c}{\lambda}\left[\left(\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\right)^{n}-1\right]h
\end{equation}
We prove this statement by induction.  When $n=0$, $\left\Vert e^{n,h}\right\Vert =0$, since the initial conditions is exactly calculated. Now suppose this statement is true for $n=k$, where $k\geq0$ and is a integer. We want to show this statement is true for $n=k+1$. Consider
\begin{equation}
\left\Vert e^{k+1,h}\right\Vert \leq\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\left\Vert e^{k,h}\right\Vert +\frac{c}{1-(1-\theta)h\lambda}h^{2},
\end{equation}
then plug in
\begin{equation}
\left\Vert e^{kn,h}\right\Vert \leq\frac{c}{\lambda}\left[\left(\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\right)^{k}-1\right]h.
\end{equation}
We have
\begin{align}
&{} \left\Vert e^{k+1,h}\right\Vert
\\&{} \leq\frac{c}{\lambda}\left[\left(\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\right)^{k+1}-\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\right]h+\frac{c}{1-(1-\theta)h\lambda}h^{2} \notag
\\&{}=\frac{c}{\lambda}\left[\left(\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\right)^{k+1}-1\right]h. \notag
\end{align}
So our claim is true for all $n$ . Note that
\begin{align}
\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda} &{}=1+\frac{h\lambda}{1-(1-\theta)h\lambda}
\\&{}\leq \exp\left(\frac{h\lambda}{1-(1-\theta)h\lambda}\right) \notag
\end{align}
by a Taylor expansion of the exponential function. Thus, we have
\begin{align}
\left\Vert e^{n,h}\right\Vert&{} \leq\frac{c}{\lambda}\left[\left(\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\right)^{n}-1\right]h
\\&{}\leq\frac{c}{\lambda}\left(\frac{1+\theta h\lambda}{1-(1-\theta)h\lambda}\right)^{n}h \notag
\\&{}\leq\frac{ch}{\lambda}\exp\left(\frac{nh\lambda}{1-(1-\theta)h\lambda}\right). \notag
\end{align}
By our condition, $nh\leq t^{*}.$
Therefore
\begin{equation}
\left\Vert e^{n,h}\right\Vert \leq\frac{ch}{\lambda}\exp\left(\frac{t^{*}\lambda}{1-(1-\theta)h\lambda}\right).
\end{equation}
So we have $\lim_{h\rightarrow0}\left\Vert e^{n,h}\right\Vert =0$ and $0\leq nh\leq t^{*}$. Hence the theta method is convergent
of order 1 when $\theta\neq\frac{1}{2}.$

Note that the backward Euler method is a special case of the theta method when $\theta=0$, so backward Euler's method is convergent of order 1. We arrive at our theorem.

\begin{theorem}
Backward Euler's method is convergent of order 1.
\end{theorem}

\begin{remark}
If $f$ is globally Lipschitz, then we can apply the above argument with respect to any time interval. If $f$ is only locally Lipschitz, then we need to analyze the situation more carefully. First, by Picard's theorem,  there is a unique solution of this ordinary differential equation for a short amount of time.  Indeed, we just need to know that the Lipschitz constant is finite without necessarily needing to know the exact value.
\end{remark}

\begin{remark} If one did not know of Picard's theorem, one could deduce the existence and uniqueness of solutions to ODEs by using time discretization.
\end{remark}

Now we consider $y'=y^{2}$ and $t\in[0,0.99]$. The exact solution of this problem is $y(t)=\frac{1}{1-t}$. So $1\leq y\leq100$. In our problem, $f=y^{2}$ is clearly analytic and it is locally Lipschitz. It is easy to show $f$ is not globally Lipschitz. If a function $f(x)$ is globally Lipschitz condition then there is a finite constant $\lambda$ such that
\begin{equation}
\frac{\left\Vert f(x)-f(y)\right\Vert }{\left\Vert x-y\right\Vert }\leq\lambda
\end{equation}
for all $x,y\in \mathbb{R}$. In our problem, let $x=0$ and $\left\Vert y\right\Vert \rightarrow\infty,$ it is easy to check
\begin{equation}
\frac{\left\Vert f(x)-f(y)\right\Vert }{\left\Vert x-y\right\Vert }\rightarrow\infty.
\end{equation}
We now discuss how one can find local Lipschitz constants $\lambda$. When $f$  is differentiable, we often just differentiate $f$ and find the maximum value of its derivative in the domain of interest. In our example, $f$ is simple and we only need to know that the Lipschitz constant is finite. So we use a more rough method to show that the Lipshitz constant is finite,
\begin{align}
&{}\left\Vert f(y^{1})-f(y^{2})\right\Vert
=\left\Vert y^{1}+y^{2}\right\Vert \left\Vert y^{1}-y^{2}\right\Vert \leq\left(\left\Vert y^{1}\right\Vert +\left\Vert y^{2}\right\Vert\right)\left\Vert y^{1}-y^{2}\right\Vert.
\end{align}
So it suffices to find the maximal value of $\left\Vert y\right\Vert$ in this problem. In our problem, $y(t)$ is continuous. Furthermore, $y(t)$ will be positive all the time, because the initial value is positive and $y'$ is positive. A continuous function has finite maximal value in a closed and bounded set. Note that the exact solution of our problem is $y(t)=\frac{1}{1-t}$ , so $1\leq y\leq100$. So we know that the Lipschitz constant in our problem is finite.

Finally, we get the convergence of functional iteration and backward Euler's method of our problem. Thus our numerical scheme for $y'=y^{2}$ with initial data $y(0)=1$ and $t\in[0,0.99]$ is convergent.

\begin{corollary}
By the theorems for existence and uniqueness of the solution for ordinary differential equations and Theorem 4.1 ,Theorem 4.2 and Theorem 4.3, we arrive at our final goal that the numerical solution generated by backward Euler's method with functional iteration exists and is unique when the time-step, $h0$ approaches zero.
\end{corollary}

\begin{remark} This requires careful choice of initial iterates when doing functional iteration.
\end{remark}

\begin{remark} Typically, the exact solution of an ODE is not known, although it is possible to deduce local Lipschitz continuity. Should the solution become infinite, a numerical method will either not converge or display very large values if the approximate solution closely approximates the exact solution. Some care is required in interpreting such numerical simulations in these cases.
\end{remark}
%%%%%%%
%Section
%%%%%%%
\section{Example Programs which use Iteration to Solve a Nonlinear Ordinary Differential Equation}

The following two Matlab programs demonstrate backward Euler's method for the example equation. The first one uses fixed-point iteration to solve for the nonlinear term and the second one uses Newton's method to solve for the nonlinear term.

\lstinputlisting[style=matlab_style,language=Matlab,label=lst:BEFPiterMatlab,caption={A Matlab program to demonstrate fixed-point iteration.}]{./NonlinearOdeAndIteration/Programs/BackwardEulerFixedPoint.m}

\lstinputlisting[style=matlab_style,language=Matlab,label=lst:BENewIterMatlab,caption={A Matlab program to demonstrate Newton iteration.}]{./NonlinearOdeAndIteration/Programs/BackwardEulerNewtonIteration.m}
%%%%%%%
%  Section
%%%%%%%
\section{Exercises}
\begin{enumerate}
\item[1)] Run the fixed-point iteration program in Matlab and check that the outcome is reasonable. Now investigate how changing the number of time steps taken to go from a time of 0 to a time of 0.99, and the tolerance for fixed point iterations affects the maximum error. In particular try a range of 1,000-1,000,000 (in powers of 10) for the number of time steps and a tolerance ranging from $10^{-1}-10^{-7}$ (in powers of $10^{-1}$). You should observe that there is an ``ideal"  combination of subdivisions and tolerance to minimize the error. What are these combinations? Do this whole process again using Newton iteration instead. How have the answers changed?
\item[2)] Write a Matlab program to solve $y'=y^2$ with $y(0)=1$ using the Crank-Nicolson method and fixed point iteration. Explain why there are two fixed-points to which the fixed-point iteration can converge. Which of these fixed-points gives the correct approximation to the solution of the differential equation? Comment on how the choice of initial iterate for the fixed-point iteration determines the fixed-point to which the method converges.
\item[3)]
\begin{enumerate}
\item[a)] Show that the differential equation $y'=\sqrt{\lvert y\rvert}$, with $y(0)=0$ is not Lipschitz continuous.
\item[b)] Find at least two analytical solutions to this differential equation.
\item[c)] Compute a numerical solution to this differential equations using the forward Euler method.
\item[d)] Compute a numerical solution to this differential equations using the backward Euler method. Be sure to try different initial guesses for the fixed-point iteration, not just the value at the previous time step; you should be able to calculate the influence of the choice of initial iterate on the selection of solution by the numerical method. Comment on this.
\item[e)] Compute a numerical solution to this differential equations using the implicit midpoint rule. Be sure to try different initial guesses for the fixed point iteration, not just the value at the previous time step; you should be able to calculate the influence of the choice of initial iterate on the selection of ``solution'' by the numerical method. Comment on this.
\item[f)] Repeat (d) and (e) with Newton iteration.
\item[g)] Comment on the applicability of numerical methods for solving differential equations without unique solutions.
\end{enumerate}
\item[4)] Modify the program for the 1-D Allen-Cahn equation so that it uses the Crank-Nicolson and fixed-point iteration for the nonlinear term. You will need to calculate the nonlinear term in real space, so that your resulting scheme is
\begin{equation}
\frac{\hat{u}^{n+1,k+1}-\hat{u}^n}{\delta t}=\frac{\hat{u}_{xx}^{n+1,k+1}+\hat{u}^n_{xx}}2 + \frac{1}{2}\widehat{\left[u^{n+1,k}- \left(u^{n+1,k} \right)^3\right]}+\frac{1}{2}\widehat{\left[u^{n}- \left(u^{n}\right)^3\right]},
\end{equation}
where $n$ denotes the time step and $k$ denotes the iterate. Stop the iterations once the maximum difference between successive iterates is sufficiently small.
\item[5)] Modify the program for the 2-D Allen-Cahn equation so that it uses the Crank-Nicolson method and fixed-point iteration for the nonlinear term. You will need to calculate the nonlinear term in real space.
\end{enumerate}


\documentclass{article}

\usepackage{amsmath}
\usepackage{numprint}

\author{Daniel Fernandes Martins (danielfmt)}
\title{Question \#7 Solution}

\begin{document}

\maketitle

\textbf{Disclaimer.} This is the reasoning I used to solve the problem; it
may be wrong though. This is intended just as food for thought.

\section{Regularization With Polynomials}

This question asks about a hypothesis set based on the Legendre polynomial
$L_{10}(x)$:

\begin{equation*}
L_{10}(x) = \frac{1}{256}(46189x^{10} - 109395x^8 + 90090x^6 - 30030x^4 + 3465x^2 - 63)
\end{equation*}

The regularization parameter $C$ can be either 0 or 1; when $C=0$, all terms
in the polynomial with degree $d \geq Q_o$ are cancelled.

\subsection{The Answer}

The correct answer to this question can be found just by proving the first
option wrong.

The first option, $\mathcal{H}(10,0,3) \cup \mathcal{H}(10,0,4) = \mathcal{H}_4$,
is incorrect because the regularization on both $\mathcal{H}(10,0,3)$ and
$\mathcal{H}(10,0,4)$ ommits the same terms on the polynomial, resulting in

\begin{equation*}
L_{10}'(x) = \frac{1}{256}(3465x^2 - 63),
\end{equation*}

so the union $\mathcal{H}(10,0,3) \cup \mathcal{H}(10,0,4)$ cannot possibly be
$\mathcal{H}_4$. However, it's plausible that the intersection
$\mathcal{H}(10,0,3) \cap \mathcal{H}(10,0,4)$ might be $\mathcal{H}_2$.

\end{document}


\documentclass{amsart}
\usepackage{amsmath}
\usepackage{amsthm}
%\usepackage{a4wide}
%\usepackage{enumerate}

\newcommand{\R}{\mathbb{R}}
\newcommand{\Z}{\mathbb{Z}}
\DeclareMathOperator{\Spa}{Spa}
\DeclareMathOperator{\Spv}{Spv}
\DeclareMathOperator{\pre}{pre}

\theoremstyle{plain}
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{proposition}[theorem]{Proposition}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
\newtheorem*{remarkn}{Remark}

\begin{document}

\title{Projective limits.}
\maketitle

\section{The set-up}

Let $X$ be a type and let $R$ be a preorder on $X$ (that is, a binary relation which is reflexive and transitive). Corresponding to $(X,R)$ is a ``small thin category'' (``small'' means the objects form a set, ``thin'' apparently means that there's at most one morphism between any two objects). We have to make a choice here: let's be covariant (it doesn't matter). So if $x\ R\ y$ then there's a morphism from $x$ to $y$.

If now $i$ is a (covariant) functor from $(X,R)$ to some category $\mathcal{C}$ then for all $x\in X$, $A_x:=i(x)$ is an object of $\mathcal{C}$, and for all $x\ R\ y$ we have a morphism $f_{xy}:=i(x\ R\ y)$ from $A_x$ to $A_y$. If $\mathcal{C}$ has all limits, then one can take the (projective) limit of $i$; in the category of commutative rings or types this would be the subring of the product ring $\prod_{x\in X}A_x$ consisting of elements $(a_x)_{x\in X}$ such that $f_{xy}(a_x)=a_y$.

{\it TODO.} If $\mathcal{C}$ is the category of topological rings, is the projective limit still the same thing, equipped with subspace topology of the product topology? Surely this is true but I never checked. I will remark that a sheaf of topological rings is not just a presheaf of topological rings such that the underlying presheaf of rings is a sheaf; I believe one needs the ring isomorphism between $\mathcal{F}(U)$ and the relevant subspace of $\prod_i\mathcal{F}(U_i)$ to be a homeomorphism, not just continuous.

Now say $S$ is a sub-preorder of $R$, that is, $S$ is another preorder on $X$ and $x\ S\ y\implies x\ R\ y$. This can certainly happen -- for example every equivalence relation is a preorder, so just imagine splitting up $R$'s equivalence classes into smaller ones. As another example one can imagine the usual total order on $\{1,2,3\}$ and then a subrelation where $1\leq 3$ and $2\leq 3$ but $1$ is no longer at most 2.

There's a natural functor $(X,S)\to(X,R)$. So the functor $i:(X,R)\to\mathcal{C}$ which sends $x$ to $A_x$ and sends the morphism $x\ R\ y$ to $f_{xy}:A_x\to A_y$, induces a functor $j:(X,S)\to\mathcal{C}$ which still sends $x$ to $A_x$ and $x\ S\ y$ to $f_{xy}:A_x\to A_y$. If $\mathcal{C}$ has all small limits one could ask what the relationship between the limits $A_R$ of $i$ and $A_S$ of $j$ is. First a thought experiment: in the category of types one can see that there is a natural map from $A_R$ to $A_S$; the $j$-limit is a (possibly) bigger subset of the same product. But of course general nonsense gives you a morphism from $A_R$ to $A_S$ in general; as part of the package of the universal object $A_R$ we know that for all $x\in X$ there's a map from $A_R$ to $A_x$, and these maps commute with all the $f_{xy}$ for $x\ R\ y$ and hence for all the $f_{xy}$ with $x\ S\ y$; now by the universal property of $A_S$ we get a map from $A_R$ to $A_S$.

In this note I'll explain a criterion for this map to be an isomorphism in $\mathcal{C}$.

\section{The criterion.}

Say the sub-preorder $S$ has the following property: For all $x,y\in X$, if $x\ R\ y$ then there exists $z\in X$ such that $x\ S\ z$, $y\ S\ z$ and $z\ R\ y$. What does this buy us? Well, $y\ S\ z$ implies $y\ R\ z$, and then $z\ R\ y$ implies that $y$ and $z$ are $R$-isomorphic and hence $f_{yz}$ is an isomorphism in $\mathcal{C}$. In particular even though it might not be true that $x\ S\ y$, it is true that $x\ S\ z$ and that $y$ and $z$ are $R$-isomorphic. In particular the morphism $f_{xy}:A_x\to A_y$ coming from $R$ is isomorphic in $\mathcal{C}$ to the $S$-morphism $f_{xz}:A_x\to A_z$.

I claim that if $S$ has this property, then the $i$-limit $A_R$ and the $j$-limit $A_S$ are isomorphic. The fact that $S\subseteq R$ gives us a map from the $i$-limit to the $j$-limit as we saw above; what we need is a map from the $j$-limit to the $i$-limit. We want to get this map by using the universal property of the $i$-limit; to do this we need to check that the $j$-limit $A_S$ equipped with its maps $\pi_x:A_S\to A_x$ satisfies $f_{xy}\circ\pi_x=\pi_y$ for all $x,y$ with $x\ R\ y$. What we know (as part of the package of the universal object $A_S$) is that this is true if $x\ S\ y$. But now say $x\ R\ y$. By our hypothesis on~$S$ we can choose $z$ with $x\ S\ z$, $y\ S\ z$ and $z\ R\ y$; hence $f_{yz}$ is an isomorphism and to prove $f_{xy}\circ\pi_x=\pi_y$ it to prove this after applying $f_{yz}$.

It thus suffices to prove $f_{yz}f_{xy}\pi_x=f_{yz}\pi_y$; but the left hand size is $f_{xz}\pi_x$ because $i$ is a functor, and hence both sides are $f_z$ because $x\ S\ z$ and $y\ S\ z$. Hence both sides are equal and by the universal property of $A_R$ we get a map $A_S\to A_R$. The composite maps in both directions commute with all triangles formed by the maps $A_R\to A_x$ and $A_S\to A_x$ so must be the identity map by uniqueness.

\end{document}


\section{Overview of the core language}
\subsection{Expressions}
There are two kind of expressions for an object in a type:
\begin{description}
\item[Application:] This is always on the form \verb|i e1 ... en|,
  where \verb|i| is an identifier (a constant or a variable).
\item[Abstraction:] This is always on the form \verb|\x -> e| where
  \verb|x| is a variable.
\end{description}
We have the following expressions for types:
\begin{description}
\item[Set:] The type of sets is \verb|Set|.
\item[Functions:] The dependent function type is denoted by  \verb|(x : E) -> E'|.
\item[Elements in a set:] If the object expression \verb|E| has the
  type \verb|Set|, then we use the notation \verb|E| for the type of
  elements in \verb|E|. Hence, we can use applications and
  abstractions to form type expressions.
\end{description}

\subsection{Declarations}
A program is a list of declarations, each of which is on one of the
forms:
\begin{description}
\item[Typing declaration:] This has the form \verb|c : E| and it
  introduces the constant \verb|c| of the type \verb|E|.
\item[Explicit definition:] This has the form \verb|c = e| and defines
  the constant \verb|c| to be equal to the expression \verb|e|.
\item[Implicit definition:] A declaration of an implicitly defined
  constant \verb|f| has the shape
\begin{verbatim}
fun f (x1:t1) ... (xn:tn) : te =
                 c1 y1 ... ym -> e1 |
                    ...
                 ck z1 ... zm -> ek
\end{verbatim}
Here \verb|c1| \dots \verb|ck| are constructors for the data type \verb|te|.
The intuition is that the constant \verb|f| is defined so that
\begin{verbatim}
f x1 ... xn (c1 y1 ... ym) = e1
                          ...
f x1 ... xn (ck z1 ... zl) = ek
\end{verbatim}

\item[Data declaration:] A data type (i.e. an object in \verb|Set|) is
  defined by giving the typing of its constructors:
\begin{verbatim}
 data D (x1:t1) ... (xn:tn) :tm =
                c1 : e1|
                  ...
                cn : en
\end{verbatim}
\end{description}


\problemname{Antimatter Rain}

You've heard of acid rain but have you heard of antimatter rain?
Antimatter rain is so potent that when it comes into contact
with another object, it immediately disintegrates
both itself and the object.
Kayla's job as a SpaceFleet
Researcher is gathering weather data on exotic planets.
This time, their assignment is to monitor the antimatter
rainfall.

Sensors are set up in the planet's atmosphere and are about
to be rained on with antimatter rain. Oh no! Kayla monitors a
single 2D section. Each sensor is either a single horizontal strip
or a single point. When one or more antimatter droplet fall
on a single sensor, all of those droplets and the sensor
disintegrate simultaneously. That is, they disappear. All other
droplets will drop past where the sensor used to be.

Kayla sees all the antimatter rain drops the moment before
they all start to fall. All droplets fall at exactly the same
rate.

For each droplet, Kayla wants to know if and where it
will disintegrate. Help them out with this demanding task!

\begin{center}
 \includegraphics[width=0.4\textwidth]{antimatter.pdf}
\end{center}
Illustration of the first sample. The vertical lines connect
the drops to the sensor they hit. The drop with no associated vertical
line will not hit any sensor.

\section*{Input}

The first line of input contains two integers $D$ ($1 \leq D \leq 100\,000$),
which is the number of antimatter droplets, and $S$ ($1 \leq S \leq 100\,000$),
which is the number of sensors.

The next $D$ lines describe the droplets, in order. Each of these lines contains two
integers $x$ ($1 \leq x \leq 10^9$), which is the $x$-coordinate of the droplet
and $y$ ($1 \leq y \leq 10^9$), which is the $y$-coordinate of the droplet.

The next $S$ lines describe the sensors. Each line contains three integers
$x_1$, $x_2$ ($1 \leq x_1 \leq x_2 \leq 10^9$), which is the leftmost and the rightmost
$x$-coordinate of the sensor, and $y$ ($1 \leq y \leq 10^9$), which is the
$y$-coordinate of the sensor.

It is guaranteed that no two drops will start in the same location, no drop will
start on any sensor, and no two sensors touch (not even at a single point).

\section*{Output}
For each droplet, in order, display a single number indicating the $y$-coordinate that it will disintegrate.
If the droplet does not disintegrate, display $0$ instead.
These values should appear on separate lines.


\paragraph{Zonotope} $\hat{x_j} = a_0^j + \sum_{i=1}^k a_i^j \epsilon_i$, $\epsilon_i \in [-1, 1]$
\paragraph{Symmetry} center is $a_0$, $2 \cdot a_0 - X$ is $X$ flipped
\paragraph{Affine} Zonotope is linear
\paragraph{ReLU} Enumerate $\epsilon_i \in [-1,1]$ to find $l_x$, $u_x$.
If $u_x \le 0$, $\hat{y} = 0$. If $l_x \ge 0$, $\hat{y} = \hat{x}$.
Otherwise, $\hat{y} = \lambda \hat{x} - \epsilon_\text{new} \frac{\lambda l_x}{2} - \frac{\lambda l_x}{2}$, $\lambda := \frac{u_x}{u_x-l_x}$


%---------------------------Maximum Aspect Frobenius-----------------------------
\section{Maximum Aspect Frobenius}

For quadrilaterals, there is not a unique definition of the aspect Frobenius.
Instead, we use the aspect Frobenius
defined for triangles (see section~\S\ref{s:tri-aspect-Frobenius}).
Consider the four triangles formed by pairs of neighboring quadrilateral edges.
Given three counterclockwise, consecutively ordered quadrilateral vertices $i$, $j$, and $k$
denote the triangular aspect frobenius $F_{ijk}$.
To obtain a single value for the metric, we take the maximum of the four unique triangular aspects
\[
  q = \max\left(F_{301}, F_{012}, F_{123}, F_{230}\right).
\]

\quadmetrictable{maximum aspect frobenius}%
{$1$}%                                      Dimension
{$[1,1.3]$}%                                Acceptable range
{$[1,DBL\_MAX]$}%                           Normal range
{$[1,DBL\_MAX]$}%                           Full range
{$1$}%                                      Unit square
{\cite{pebay:04}}%                          Citation
{v\_quad\_max\_aspect\_frobenius}%          Verdict function name


%!TEX root = ../notes.tex
\section{April 22, 2022}
\subsection{Short Vectors \emph{continued}}
\recall last class we stated Hermite's Theorem:
\begin{theorem*}[Hermite's Theorem]
    Let $L$ be a lattice of dimension $n$. Then $0\neq v\in L$ with
    \[||\bvec{v|}|\leq \sqrt{n}\cdot \det(L)^\frac{1}{n}\]
\end{theorem*}
\begin{proof}
    Apply Minkowski's Theorem to the cube
    \[S = \left\{ (x_1, \dots, x_n) \Bigm\vert -\det(L)^{1/n}\leq x_i \leq \det(L)^{1/n} \right\}\]
    Minkowski's states that $S$ contains a nonzero lattice vector. By inspection, this lattice vector $\bvec{v}\in S$ implies that
    \[||\bvec{v}|| \leq \underbrace{\sqrt{(\det(L)^{1/n})^2 + \cdots + (\det(L)^{1/n})^2}}_{n\text{ times}} = \sqrt{n}(\det(L)^{1/n})\]
\end{proof}

\begin{theorem*}[Variant of Hermite's Theorem]
    There is a basis $\bvec{v}_1, \dots, \bvec{v}_n$ with
    \[||\bvec{v}_1||\cdot||\bvec{v}_2||\cdots ||\bvec{v}_n|| \leq n^{n/2}\det(L)\]
\end{theorem*}
\begin{definition}[Hadamard Ratio]
    The \ul{Hadamard Ratio} is
    \[\mathcal{H} = \left( \frac{|\det(L)|}{||\bvec{v}_1||\cdot||\bvec{v}_2||\cdots ||\bvec{v}_n||} \right)^{1/n}\]
\end{definition}

The variant of Hermite's Theorem says that there is a basis for which $\mathcal{H}\geq \frac{1}{\sqrt{n}}$. For any basis, $0 < \mathcal{H}\leq 1$. $\mathcal{H} = 1$ if and only if our basis is orthogonal.

This ratio makes precise how orthogonal our basis is.

We can write this in code (\emph{now enhanced with NumPy!}):
\lstinputlisting[]{code/linalg_np.py}

Instead of using a hypercube, we can use a hypersphere. Let
\[S = B_R(\bvec{0}),\]
a ball with radius $R$ centered at $0$. Minkowski's theorem says $S$ contains a nonzero lattice vector if $\textsf{Vol}(B_R(\bvec{0})) \geq 2^n\cdot \det(L)$. Let $\textsf{Vol}(B_R(\bvec{0})) = C_n\cdot R^n$ where $C_n$ is the volume of a unit ball in $n$ dimensions.

This is also to say that
\[R\geq \frac{2}{C_n^{1/n}}\det(L)^{1/n}\]
Fact from analysis/calculus (Stirling's Formula):
\[\lim_{n\to\infty}C_n^{1/n}\cdot\sqrt{n} = \sqrt{2\pi e}.\]
The key point is
\[R\geq \frac{2}{C_n^{1/n}}\det(L)^{1/n} \approx \sqrt{\frac{2}{\pi e}}\sqrt{n}(\det(L))^{1/n}\]

\begin{ques*}
    What is the real truth? Given a random lattice, how long is the shortest vector? What is the actual length (probably)?
\end{ques*}
How many lattice points \emph{``should''} a ball of radius $R$ contain? Probably the volume divided by the volume of the fundamental domain:
\[\frac{\mathsf{Vol}(S)}{\mathsf{Vol(\mathcal{F})}} = \frac{\mathsf{Vol}(S)}{\det(L)}.\]
So when \emph{should} $B_R(\textbf{0})$ contain nonzero $\bvec{v}\in L$? This is probably when $\mathsf{Vol}(B_R(\bvec{0})) \geq \det(L)$.

It's around when
\[R \geq \sqrt{\frac{1}{2\pi e}}\sqrt{n}(\det(L))^{1/n}\]

\begin{definition}[Gaussian Expected Shortest Length]
    $\sigma(L) = \sqrt{\frac{n}{2\pi e}}\cdot (\det(L))^{1/n}$ is called the \ul{Gaussian expected shortest length}.
\end{definition}
We \emph{expect} $||\bvec{v}_\mathsf{shortest}||\approx\sigma(L)$. We \emph{know} $||\bvec{v}_\mathsf{shortest}||\lesssim 2\cdot \sigma(L)$

\subsection{Babai's Algorithm}
\emph{Goal:} we want to solve, approximately, the closest vector problem. We have $L\subseteq\RR^n$ with basis $\bvec{v}_1, \dots, \bvec{v}_n$. We have some $\bvec{w}\in\RR^n$. We want to find a close lattice vector to $\bvec{w}$. That is, we want \[a_1, a_2, \dots, a_n\in\ZZ\] where $||\bvec{w} - \sum a_i\bvec{v}_i||$ is small.

\emph{Method:}
\begin{enumerate}
    \item Write $\bvec{w} = \alpha_1\bvec{v}_1 + \cdots \alpha_n\bvec{v}_n$.
    \item Round $\alpha_i$ to nearest integer $a_i$.
\end{enumerate}

\begin{ques*}
    When does this work well?
\end{ques*}

We implement this in code:
\lstinputlisting[]{code/babai.py}


\section{Discussion}\label{sec:discussion}
In this section, we will elaborate on the interpretation of some results, reflect on some concepts,
and connect to the current and future research.

Sec.~\ref{sec:data-encoding} explained the three different encoding methods leveraged in this
research and cited some literature on the effect of encoding on network accuracy. Mathematically
speaking, encoding is seen as feature transformation $\mathcal{T}$, where a character $X$ is
transformed to $\mathcal{T}(X)$ in the new encoding space. Since the lossless encoding is
invertible, it is clear for any two functions (networks) and any two encodings (transformations)
that
$\eta_1\left(\mathcal{T}_1(X)\right) = \left(\eta_1\cdot\mathcal{T}_1\cdot
  \mathcal{T}_2^{-1}\right)\left(\mathcal{T}_2(X)\right) =
\eta_2\left(\mathcal{T}_2(X)\right)$. This means that if the network $\eta_1$ is the most accurate
network for the encoding $\mathcal{T}_1$, using another encoding $\mathcal{T}_2$ for the same
problem requires designing another network
$\eta_2 = \eta_1\cdot\mathcal{T}_1\cdot \mathcal{T}_2^{-1}$. However, this network may be of
complicated architecture to be able to ``decode'' a terse or complex pattern $\mathcal{T}_2(X)$. The
behavior of the three encodings BinE, OneE, and TwoE in this paper can be seen in the light of this
discussion. The most terse representation is the BinE ($n=8$) and the most sparse representation is
the OneE ($n=181$); and in between comes our TwoE ($n=41$) as a smart design and compromise between
the low dimensionality of BinE and the self-decoded nature of the OneE
(Sec.~\ref{sec:data-encoding}). This may be a qualitative interpretation to why the accuracy of the
best models was always possessed by the TwoE, yet with one exception at the BinE
(Sec.~\ref{sec:arabic-results}). However, from Figures~\ref{fig:ArabicModelsResults} and
\ref{english_results}, the rug plots reveal that the populations of accuracy at different
encodings do interleave and each encoding can perform better than others at some experiments. We
emphasize that this effect is an artifact to the non exhaustive network configuration parameters and
experiments conducted in this research. Had we covered the configuration parameter space then all
encoding methods would produce the same accuracy, yet at different network architectures, as each
encoding requires the right network architecture to learn from (or to ``decode'').

Sec.~\ref{sec:param-netw-conf} detailed the network configuration parameters for both Arabic
datasets (\{4L, 7L\} $\times$ \{82U, 50U\} $\times$ \{0W, 1W\} $\times$ \{LSTM, BiLSTM\} = 16
networks) and for English dataset (\{3L, 4L, 5L, 6L, 7L, 8L\} $\times$ \{30U, 40U, 50U, 6U\}
$\times$ \{LSTM,\ BiLSTM,\ GRU,\ BiGRU\} = 96 networks). Each experiment runs almost in one hour (30
epochs $\times$ 2 min/epoch) on the mentioned hardware (Sec.~\ref{sec:model}). The total run time of
all network configurations on all data representations for both Arabic and English datasets was
$16\times 12+96\times 2 = 384$ hours, i.e., more than two weeks! We are currently working on more
exhaustive set of experiments to cover a good span of the network configuration parameter space to
both confirm the above discussion on encoding and to boost the per-class accuracy on both datasets.

\bigskip

The per-class accuracy for both datasets needs investigation; in particular, the interesting trend
between the per-class accuracy and the class size of the Arabic dataset needs more investigation. We
speculate that this is a mere correlation that does not imply causation; and the reason for this
trend may be attributed to the difficulty of, or the similarity between, the meters having small
class size. This difficulty, or similarity, may be what is responsible for the low accuracy
(Figure~\ref{fig:footn-both-models}) on a hand, and the lack of interest of poets to compose at
these meters, which resulted in their scarcity (Figure~\ref{fig:footn-footn-class}), on the other
hand.

Diacritic effect is explained in Sec.~\ref{sec:results}; experiments with diacritics scored higher
than those without diacritics only when small class size were trimmed from the datasets (1T). When
including the whole dataset (0T) the effect of diacritics was not consistent. This interesting
phenomenon needs more investigation, since the phonetic pattern of any meter is uniquely identified
by diacritics (Sec.~\ref{sec:arab-poetry-text}). This may be connected to the observation above of
the per-class accuracy.

For more investigation of both phenomena, we are working on a randomized-test-like experiments in
which all classes will be forced to have equal size $n$. We will study how the per-class accuracy or
overall accuracy, along with their two individual components (precision and recall), behave and how
the diacritic effect changes in terms of both $n$ and the number of involved classes $k$, where
$2 \leq k \leq K$, and $K(=16)$ is the total number of meters.