\documentclass[jou]{apa6}

\usepackage[american]{babel}

\usepackage{csquotes}
\usepackage[style=apa,sortcites=true,sorting=nyt,backend=biber]{biblatex}
\DeclareLanguageMapping{american}{american-apa}
\addbibresource{bibliography.bib}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Discrete Structures
%% The start of RBS stuff
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Working internal and external links in PDF
\usepackage{hyperref}
% Extra math symbols in LaTeX
\usepackage{amsmath}
\usepackage{gensymb}
\usepackage{amssymb}
% Enumerations with (a), (b), etc.
\usepackage{enumerate}
\usepackage[framemethod=TikZ]{mdframed}
\usepackage{xcolor}

\let\OLDitemize\itemize
\renewcommand\itemize{\OLDitemize\addtolength{\itemsep}{-6pt}}

\usepackage{etoolbox}
\makeatletter
\preto{\@verbatim}{\topsep=3pt \partopsep=3pt }
\makeatother

% These sizes redefine APA for A4 paper size
\oddsidemargin 0.0in
\evensidemargin 0.0in
\textwidth 6.27in
\headheight 1.0in
\topmargin -24pt
\headheight 12pt
\headsep 12pt
\textheight 9.19in

\title{Sample Quiz 8}
\author{Discrete Structures, Spring 2020}
\affiliation{RBS}

\leftheader{Discrete Sample Quiz 8}

\abstract{%
}

%\keywords{}

\setlength\parindent{0pt}

\begin{document}

%\thispagestyle{empty}

\twocolumn
\section{Final Exam, 2020-04-23}

\vspace{4pt}
{\bf Question 1.}\\
By $U$ we denote the set of all positive integers
between $1$ and $120$. This is the {\em universe} in which
we define several subsets:
$$\left\{ \begin{array}{l}
A = \{ x \in U\;\mid\;2\mid{}x\},\\
B = \{ x \in U\;\mid\;3\mid{}x\},\\
C = \{ x \in U\;\mid\;5\mid{}x\},\\
X = \{ x \in U\;\mid\;2\mid{}x\;\vee\;3\mid{}x\},\\
Y = \{ x \in U\;\mid\;(3\mid{}x\;\wedge\;5\mid{}x)\;\vee\neg(2\mid{}x)\}.
\end{array} \right.$$

{\bf (A)} Express $X$ using the sets $A,B,C$ (using set union $V \cup W$,
set intersection $V \cap W$, set complement $\overline{V}$ operations).\\
{\bf (B)} Express $Y$ using the sets $A,B,C$ in a similar way.\\
{\bf (C)} Find $|X|$ - the size of the set $X$.\\
{\bf (D)} Find $|Y|$ - the size of the set $Y$.

\vspace{10pt}
{\bf Question 2.}\\
Let $A$ and $B$ be sets with sizes $|A| = 8$ and $|B| = 5$
and $|A \cap B| = 3$.

Calculate the largest and the smallest possible values
for each of the following set sizes:

{\bf (A)} $|A \cup B|$.\\
{\bf (B)} $|A \times (B \times B)|$.\\
{\bf (C)} $\left| \mathcal{P}(\mathcal{P}(A \cap B)) \right|$ - the
powerset of a powerset of $A \cap B$.\\
{\bf (D)} $|A \oplus B|$ - the symmetric difference of the sets $A$ and $B$.

\vspace{10pt}
{\bf Question 3.}\\
Consider the following recurrent sequence:

$$\left\{ \begin{array}{l}
a_0 = 3\\
a_1 = 4\\
a_{n+2} = 5a_{n+1} - 6a_n,\;\text{if}\;n \geq 0 \\
\end{array} \right.$$

Assume that $b_n$ is another sequence satisfying the
recurrence rule
$$b_{n+2} = 5b_{n+1} - 6b_n,\;\text{if}\;n \geq 0$$
(The first two members $b_0,b_1$ are not known.)

{\bf (A)} Write the first $6$ members of this sequence ($a_0,\ldots,a_5$).\\
{\bf (B)} Write the characteristic equation for this sequence.\\
{\bf (C)} Write the general expression for an arbirary sequence $b_n$
satisfying the recurrent expression as a sum of two geometric progressions
(you can leave unknown coefficients in your answer; just explain which ones they are).\\
{\bf (D)} Write the formula to compute $a_n$ (that would satisfy
the initial conditions $a_0 = 3$ and $a_1 = 4$).

\vspace{10pt}
{\bf Question 4.}\\
Consider this code snippet in Python:

\begin{verbatim}
n = 1000
sum = 0
for i in range(1, n*n+1):
    for j in range(1,i+1):
        sum += i % j
\end{verbatim}

And a similar one in R:

\begin{verbatim}
n <- 1000
sum <- 0
for (i in 1:(n*n)) {
    for (j in 1:i) {
        sum <- sum + i %% j
    }
}
\end{verbatim}

{\bf (A)} Explain in human language what this algorithm does.\\
{\bf (B)} Denote by $f(n)$ the
number of times the variable `sum` is incremented.
Write the Big-O-Notation for $f(n)$. Find
a function $g(n)$ such that
$f(n)$ is in $O(g(n))$. (If there are multiple functions,
pick the one with the slowest growth.)\\
{\bf (C)} Express the function $f(n)$ precisely -
how many times `sum` is incremented in terms of variable $n$.

\vspace{10pt}
{\bf Question 5.}

Let $A$ be the set of all positive divisors of the number $120$
(including $1$ and $120$ itself).

{\bf (A)} What is the multiplication of all
numbers in the set $A$?\\
{\bf (B)} Express this number as the product of prime powers.

\vspace{10pt}
{\bf Question 6.}

Define the following binary relationship on the set of
integer numbers $\mathbb{Z}$:
We say that $aRb$ (numbers $a,b \in \mathbb{Z}$ are in the relation $R$) iff
$$\left\{ \begin{array}{l}
a - b \equiv 0\,(\text{mod}\;11)\\
a - b \equiv 0\,(\text{mod}\;12)\\
a - b \equiv 0\,(\text{mod}\;13)
\end{array} \right.$$

\begin{tabular}{|l|l|l|} \hline
{\bf Item} & {\bf Statement} & {\bf True or False?} \\ \hline
{\bf (A)} & $R$ is reflexive &  \\ \hline
{\bf (B)} & $R$ is symmetric &  \\ \hline
{\bf (C)} & $R$ is antisymmetric &  \\ \hline
{\bf (D)} & $R$ is transitive &  \\ \hline
{\bf (E)} & $aRb$ iff $a=b$ &  \\ \hline
\end{tabular}

For all items where you answered `FALSE`, specify a counterexample
(values for some numbers that would make the condition true, but the
conclusion false).
If the statement was true, write "none".

{\bf (A)} counterexample: $\ldots$\\
{\bf (B)} counterexample: $\ldots$\\
{\bf (C)} counterexample: $\ldots$\\
{\bf (D)} counterexample: $\ldots$\\
{\bf (E)} counterexample: $\ldots$

\vspace{10pt}
{\bf Question 7.}\\
Four people $A,B,C,D$ each has his own hat.
After the meeting they leave their
building in a hurry, everyone grabs some hat at random
so that all $4!$ permutations of the hats have equal probabilities.

Let the random variable $X$ denote the number of hats that
were picked up correctly. (For example, if
the hat assignment is this: $(A \rightarrow A,
B \rightarrow B, C \rightarrow D, D \rightarrow C)$, then
$X = 2$, because two people got their own hats.)

{\bf (A)}  Find $E(X)$ - the expected value of $X$.\\
{\bf (B)}  Find $V(X)$ - the variance of $X$.

\vspace{10pt}
{\bf Question 8.}\\
There was a crooked man who had a crooked 1 euro coin.
On lucky days it would flip the *heads* with probability $p=\frac{2}{3}$,
and the *tails* with probability $p=\frac{1}{3}$, but on unlucky days
it was the opposite ($p(\mathtt{heads})=\frac{1}{3}$, but
$p(\mathtt{tails})=\frac{2}{3}$).
There were equal probabilities of $\frac{1}{2}$ for lucky and unlucky days.

One morning he flipped the coin $5$ times and altogether got three {\em heads}
and two {\em tails}.

Let us introduce the following events:

\begin{itemize}
\item $E$ (evidence): Five coin tosses result in three {\em heads} and two {\em tails}.
\item $H$ (hypothesis): The current day is lucky.
\end{itemize}

{\bf (A)} Find $P(E|H)$ - the conditional probability of $E$ given that the
day is lucky.\\
{\bf (B)} Find $P(E|H)\cdot P(H)$ - the probability that the day
is lucky and $E$ happens.\\
{\bf (C)} Find $P(E|\overline{H})$ - the conditional probability of $E$
given that the day is not lucky.\\
{\bf (D)} Find $P(E|\overline{H})\cdot P(\overline{H})$ - the probability
that the day is unlucky and $E$ happens.\\
{\bf (E)} Find $P(E)$ - as the sum of two probabilities ($E$ happened
on a lucky day and also $E$ happened on unlucky day).\\
{\bf (F)} Find the conditional probability $P(H|E)$ -
the likelyhood that the croocked man has a lucky day, given
that the event $E$ has happened.

\vspace{10pt}
{\bf Question 9.}

\begin{figure}[!htb]
\center{\includegraphics[width=2in]{final/final-syntax-tree.png}}
\caption{\label{fig:final-syntax-tree} A tree for an expression.}
\end{figure}

The syntax tree describes an algebraic expression (please note
the difference between the unary minus that
flips the value of the variable $y$ and
the binary minus that subtracts the
two subexpressions: $-y$ and $6 \times 3$).

{\bf (A)} Write the preorder DFS traversal of
this tree.\\
{\bf (B)} Write the inorder DFS traversal of
this tree.\\
{\bf (C)} Write the postorder DFS traversal of
this tree.

{\em Note.} In all $3$ answers denote the unary
minus with the tilde sign $\sim$,
but the regular/binary minus with $-$.

\vspace{10pt}
{\bf Question 10.}\\
\begin{figure}[!htb]
\center{\includegraphics[width=2in]{final/prim-weighted-graph.png}}
\caption{\label{fig:prim-weighted-graph} A graph with 9 vertices.}
\end{figure}

Run the Prim's algorithm on the weighted graph in Figure~\ref{fig:prim-weighted-graph},
start growing the tree from the vertex $I$.

\begin{tabular}{|l|l|} \hline
{\bf Step} & {\bf Newly Added Edge} \\ \hline
{\bf Step 1} & \\ \hline
{\bf Step 2} & \\ \hline
{\bf Step 3} & \\ \hline
{\bf Step 4} & \\ \hline
{\bf Step 5} & \\ \hline
{\bf Step 6} & \\ \hline
{\bf Step 7} & \\ \hline
{\bf Step 8} & \\ \hline
\end{tabular}

What is the total weight of the obtained Minimum Spanning Tree?

\mbox{}
\newpage
\subsection{Answers}

Every problem is worth $15$ points. The total for this final is 150 points.

\vspace{4pt}
{\bf Question 1.}\\

{\bf (A)} $X = A \cup B$ (Boolean OR means set union)\\
{\bf (B)} $Y = (B \cap C) \cup \overline{A}$ (Boolean and means set intersection; negation means set complement)\\
{\bf (C)} $|X| = |A| + |B| - |A \cap B| = 60+40-20 = 80$ (principle of inclusion-exclusion).\\
{\bf (D)} $|Y|$ is all odd numbers and also four even numbers divisible by $15$
($30, 60, 90, 120$). The total is $60 + 4 = 64$.

{\scriptsize
{\em Grading.}
\begin{itemize}
\item Each correct answer is $3$ points (total $12$).
\item Explaining both {\bf (C}) and {\bf (D)} is another $3$ points (total $3$).
\item Using wrong set notation ($\wedge$ instead of $\cap$ etc.) divides the number of points in half.
\item $68$ instead of $64$ is $2$ points (instead of $3$).
\end{itemize}
}

\vspace{10pt}
{\bf Question 2.}\\
In all the answers the largest and the smallest value are equal, because
we know exactly how the two sets intersect; how many elements belong to just
one of the sets $A$, $B$, and how many elements belong to the both sets.

{\bf (A)} $|A \cup B| = |A| + |B| - |A \cap B| = 8+5-3 = 10$ (the principle of inclusion-exclusion).\\
{\bf (B)} $|A \times (B \times B)| = 8 \cdot 5 \cdot 5 = 200$\\
(Cartesian product has size that is the product of all participant sets:
one can combine three elements from the sets $A$, $B$ and $B$ in this many ways).\\
{\bf (C)} $2^{2^3} = 2^8 = 256$ (the number of elements in the powerset
of any set $X$ can be obtained by raising $2$ to the power $|X|$).\\
{\bf (D)} $|A \oplus B| = (8-3) + (5-3) = 7$ (we remove the common elements from both $A$ and $B$).

{\scriptsize
{\em Grading.}
\begin{itemize}
\item Each correct answer is $3$ points (total $12$).
\item Expressions or verbal explanations of the answers is $3$ points (total $3$).
\end{itemize}
}

\vspace{10pt}
{\bf Question 3.}\\
{\bf (A)} $a_0 = 3$,\\
$a_1 = 4$,\\
$a_2 = 5\cdot 4 - 6 \cdot 3 = 2$,\\
$a_3 = 5\cdot 2 - 6 \cdot 4 = -14$,\\
$a_4 = 5\cdot(-14) - 6 \cdot 2 = -82$,\\
$a_5 = 5\cdot(-82) - 6 \cdot (-14) = -326$,\\
$a_6 = 5\cdot(-326) - 6 \cdot (-82) = -1138$.

{\bf (B)} The characteristic equation is obtained, if we try to find $a_n$
in the form of a geometric progression $r^n$:
$r^{n+2} = 5r^{n+1} - 6r^n,$ or
$r^2 -5r + 6 = 0$.
It has two roots: $r_1 = 2$, $r_2 = 3$.

{\bf (C)} The general form of the expression for any iterative
sequence $b_n$ satisfying the relationship $b_{n+2} = 5b_{n+1} - 6b_n$ is as follows:
$$b_n = A \cdot 2^n + B \cdot 3^n,$$
where $A,B$ are two constants that depend on the two initial values of the sequence $b_n$.

{\bf (D)} We need to solve a system of two equations, to ensure that the formula
$a_n = A \cdot 2^n + B \cdot 3^n$ has correct values for $n=0$ and $n=1$.
We get the following system:
$$\left\{ \begin{array}{l}
A + B = 3,\\
2A + 3B = 4.\\
\end{array} \right.$$
Substitute $B = 3-A$ into the second equation. We get that
$2A + 9 - 3A = 4$ and $A = 5$. We also get that $B = -2$.
Therefore the exact formula to calculate the sequence $a_n$ is this:
$$a_n  = 5 \cdot 2^n - 2 \cdot 3^n,\;\text{where}\;n \geq 0.$$
This actually works, if we plug in the values calculated in {\bf (A)} for $n = 0,\ldots,6$.

{\scriptsize
{\em Grading.}
\begin{itemize}
\item Answer in {\bf (A)} is $4$ points.
\item Answer in {\bf (B)} is $4$ points.
\item Answer in {\bf (C)} is $3$ points.
\item Answer in {\bf (D)} is $4$ points.
\end{itemize}
}

\vspace{10pt}
{\bf Question 4.}\\
{\bf (A)} The algorithm takes all numbers $i$ from $1$ to $n^2$ and
divides them by all the smaller numbers $j < i$, and adds up all the obtained remainders.

{\bf (C)} The outer loop is repeated $n^2$ times. The inner loop is repeated
$1 + 2 + 3 + \ldots + n^2$ times. This is an arithmetic progression.
The sum of an arithmetic progression is the arithmetic mean of the first and the last
member multiplied by the number of members:
$$f(n) = \frac{1 + n^2}{2} \cdot n^2 = \frac{n^4 + n^2}{2}.$$

{\bf (B)} $f(n)$ is in $O(n^4)$. Therefore we can take $g(n) = n^4$. We can
pick another $g(n)$ that is multiplied by some nonzero constant
(such as $\frac{n^4}{2}$ or $17n^4$ or anything else - that also counts
as a valid answer).
Certainly, $f(n)$ is also in $O(n^k)$ for any $k > 4$, but the function $g(n) = n^4$
is the slowest growing.

{\scriptsize
{\em Grading.}
\begin{itemize}
\item Answer for {\bf (A)} is $5$ points.
\item Answer for {\bf (B)} (any 4th degree polynomial of $n$) is $5$ points.
An attempt to estimate some arithmetic progression with a different upper limit
(say, $1 + 2 + \ldots + n$) gets $2$ points.
\item Answer for {\bf (C)} is $5$ points. If the answer is
provided just for $n=1000$ (not for any variable), then it is $4$ points.
\end{itemize}
}

\vspace{10pt}
{\bf Question 5.}\\
{\bf (A)} If expressed as a product of two positive integers $120 = ab$,
one of the divisors $a$ or $b$ would be smaller than $\sqrt{120} \approx 11$, and the other
one would be bigger. We can easily list all the ways to express $120$
as a product of two integers:
$$1 \cdot 120 = 2 \cdot 60 = 3 \cdot 40 = 4 \cdot 30 = $$
$$= 5 \cdot 24 = 6 \cdot 20 = 8 \cdot 15 = 10 \cdot 12,$$
and there are no other factorizations, since all the divisors less than $11$ are
already listed.
Multiplying them all together would give
$$(120)^8 = 42998169600000000$$

{\bf (B)} As a product of prime factors:
$$(120)^8 = (2^3 \cdot 3 \cdot 5)^8 = 2^{24} \cdot 3^8 \cdot 5^8.$$

{\scriptsize
{\em Grading.}
\begin{itemize}
\item Correct item {\bf (A)} is $7$ points (also floating point answers were fine -
not all had easy access to the big integer arithmetic).
\item Correct item {\bf (B)} is $8$ points.
\end{itemize}
}

\vspace{10pt}
{\bf Question 6.}\\

\begin{tabular}{|l|l|l|} \hline
{\bf Item} & {\bf Statement} & {\bf True or False?} \\ \hline
{\bf (A)} & $R$ is reflexive & TRUE \\ \hline
{\bf (B)} & $R$ is symmetric & TRUE \\ \hline
{\bf (C)} & $R$ is antisymmetric & FALSE \\ \hline
{\bf (D)} & $R$ is transitive & TRUE \\ \hline
{\bf (E)} & $aRb$ iff $a=b$ & FALSE \\ \hline
\end{tabular}

{\bf (A)} Counterexample: None\\
{\bf (B)} Counterexample: None\\
{\bf (C)} Consider counterexample $a=0$, $b = 11 \cdot 12 \cdot 13 = 1716$.
While it is true that $aRb$ and $bRa$, nevertheless $a \neq b$.\\
{\bf (D)} Counterexample: None\\
{\bf (E)} Counterexample is same as in {\bf (C)}: $a=0$, $b = 1716$.

{\scriptsize
{\em Grading.}
\begin{itemize}
\item One correct answer is $2$ points (total $10$).
\item Counterexamples for {\bf (C)} and {\bf (E)} is $5$ points (they are, in fact, the same).
\end{itemize}
}

\vspace{10pt}
{\bf Question 7.} Answer: {\tt 17}\\

\begin{itemize}
\item For $1$ of $24$ permutations $X = 4$ (all hats stay in place),
\item For $0$ permutations $X = 3$ (it is not possible for exactly three hats to stay in place, because
then the 4th hat also returns to its owner),
\item For $6$ of $24$ permutations $X = 2$ (there are ${4 \choose 2} = 6$ ways how to pick $2$ hats
that stay in place; and the remaining two hats can switch places only in one way),
\item For $8$ of $24$ permutations $X = 1$ (there are ${4 \choose 1} = 4$ ways how to pick $1$ hat
that stays in place; and the remaining three hats can rotate in two ways).
\item For the remaining $24 - (1 + 6 + 8) = 9$ permutations $X = 0$ (no hats stay in place).
\end{itemize}

{\scriptsize
\begin{table}[h!]
\begin{center}
\caption{Random Variable $X$ for the Hat Problem}
\begin{tabular}{|l|r|r|r|} \hline
\label{table:t1}
{\bf Permutation} & $X$ & $X-E(X)$ & $(X-E(X))^2$ \\ \hline
$\textcolor{red}{\mathtt{ABCD}}$ & $4$ & $3$ & $9$ \\ \hline
$\textcolor{red}{\mathtt{AB}}\mathtt{DC}$ & $2$ & $1$ & $1$ \\ \hline
$\textcolor{red}{\mathtt{A}}\mathtt{CB}\textcolor{red}{\mathtt{D}}$ & $2$ & $1$ & $1$ \\ \hline
$\textcolor{red}{\mathtt{A}}\mathtt{CDB}$ & $1$ & $0$ & $0$ \\ \hline
$\textcolor{red}{\mathtt{A}}\mathtt{DBC}$ & $1$ & $0$ & $0$ \\ \hline
$\textcolor{red}{\mathtt{A}}\mathtt{D}\textcolor{red}{\mathtt{C}}\mathtt{D}$ & $2$ & $1$ & $1$ \\ \hline

{\tt BACD} & $2$ & $1$ & $1$ \\ \hline
{\tt BADC} & $0$ & $-1$ & $1$ \\ \hline
{\tt BCAD} & $1$ & $0$ & $0$ \\ \hline
{\tt BCDA} & $0$ & $-1$ & $1$ \\ \hline
{\tt BDAC} & $0$ & $-1$ & $1$ \\ \hline
{\tt BDCA} & $1$ & $0$ & $0$ \\ \hline

{\tt CABD} & $1$ & $0$ & $0$ \\ \hline
{\tt CADB} & $0$ & $-1$ & $1$ \\ \hline
{\tt CBAD} & $2$ & $1$ & $1$ \\ \hline
{\tt CBDA} & $1$ & $0$ & $0$ \\ \hline
{\tt CDAB} & $0$ & $-1$ & $1$ \\ \hline
{\tt CDBA} & $0$ & $-1$ & $1$ \\ \hline

{\tt DABC} & $0$ & $-1$ & $1$ \\ \hline
{\tt DACB} & $1$ & $0$ & $0$ \\ \hline
{\tt DBAC} & $1$ & $0$ & $0$ \\ \hline
{\tt DBCA} & $2$ & $1$ & $1$ \\ \hline
{\tt DCAB} & $0$ & $-1$ & $1$ \\ \hline
{\tt DCBA} & $0$ & $-1$ & $1$ \\ \hline\hline
{\bf Mean} & $1$ & $0$ & $1$ \\ \hline
\end{tabular}
\end{center}
\end{table}
}

{\bf (A)} $E(X) = \frac{1}{24} \cdot 4 + \frac{6}{24} \cdot 2 + \frac{8}{24} \cdot 1 = 1$.
This means that the expected number of hats that stay in place is exactly $1$.\\
{\bf (B)} For all $24$ permutations, subtract the value $E(X) = 1$ from
every hat experiment outcome. Define $x_1,\ldots,x_{24}$ - all $24$ values of the
random variable $X$ (exactly one value is $4$, exactly six values are $2$,
exactly $8$ values are $1$, exactly $9$ values are $0$):
$$V(X) = \frac{\sum_{i=1}^{24} (x_i - 1)^2}{24} = \frac{24}{24} = 1.$$

Therefore, $V(X) = 1$ (variance also equals $1$, but the unit of measurement
is not hats but ``hats squared'').

{\em Note.} $E(X)=1$ is the arithmetic mean over the column $X$, but $V(X)=1$
is the arithmetic mean over the column $(X-E(X))^2$ (see Table~\ref{table:t1}).

{\scriptsize
{\em Grading (theoretically max=20, but most results are not that high).}
\begin{itemize}
\item Correctly found $E(X)$ is $5$ point.
\item Correctly found $V(X)$ is $5$ point.
\item Justified computation for $E(X)$ is $5$ points.
\item Justified computation for $V(X)$ is $5$ points.
\end{itemize}
}

\vspace{10pt}
{\bf Question 8.} \\

{\bf (A)} $P(E|H)$ is the outcome of the Binomial distribution:
There are $n=5$ coin-toss experiments; the probability of success for any single experiment
is $p = \frac{2}{3}$ (since we know that the day is lucky and hypothesis $H$ holds).
Therefore,
$$P(E|H) = {5 \choose 3} p^3 (1-p)^2 = 10 \left( \frac{2}{3} \right)^3 \left( \frac{1}{3} \right)^2 =
\frac{80}{243}$$

{\bf (B)} $P(E|H)\cdot P(H) = \frac{80}{243}\cdot\frac{1}{2}= \frac{40}{243}$,
since $P(H) = \frac{1}{2}$ (the {\em a priori} probability of a lucky day is exactly $1/2$).

{\bf (C)} $P(E|\overline{H})$ is the outcome of the Binomial distribution:
Again, there are $n=5$ coin-toss experiments, but now the probability of a single
experiment is just $p = \frac{1}{3}$. Therefore,
$$P(E|\overline{H}) = {5 \choose 3} p^3 (1-p)^2 = 10 \left( \frac{1}{3} \right)^3 \left( \frac{2}{3} \right)^2 =
\frac{40}{243}$$

{\bf (D)} $P(E|\overline{H}) \cdot P(\overline{H}) = \frac{40}{243}\cdot\frac{1}{2} = \frac{20}{243}$.

{\bf (E)} We can compute $P(E)$ as the sum of two mutually incompatible events:
event $E$ can happen either on a lucky day or on an unlucky day:
$$P(E) = P(E|H)\cdot P(H) + P(E|\overline{H}) \cdot P(\overline{H}) =
\frac{40}{243} + \frac{20}{243} = \frac{60}{243}.$$

{\bf (F)} Use Bayes formula:
\begin{align}
P(H|E) = & \frac{P(E|H) \cdot P(H)}{P(E|H) \cdot P(H) + P(E|\overline{H}) \cdot P(\overline{H})} = \nonumber \\
 = & \frac{P(E|H) \cdot P(H)}{P(E)} =  \frac{ \frac{40}{243}}{ \frac{60}{243}} = \frac{2}{3}. \nonumber
\end{align}

Bayes formula is intuitive: It shows the proportion of the
subcase $P(E|H) \cdot P(H)$ (i.e. event $E$ hapens on a lucky day) out of the
whole probability $P(E) = P(E|H)\cdot P(H) + P(E|\overline{H}) \cdot P(\overline{H})$
(i.e. event $E$ happens either on a lucky or unlucky day).

{\scriptsize
{\em Grading (theoretically max=20, but most results are not that high).}
\begin{itemize}
\item Any item from {\bf (A)} to {\bf (E)} is $3$ points.
\item Bayes formula or a similar expression finding the reverse conditional probability
in {\bf (F)} in $5$ points.
\end{itemize}
}

\vspace{10pt}
{\bf Question 9.}\\
{\bf (A)} $\mathtt{+:-\sim{}y\times6\;3\;z\;2}$,\\
{\bf (B)} $\mathtt{y\sim{}-\,6\,\times3:z\,+\,2}$,\\
{\bf (C)} $\mathtt{y\sim{}6\;3\,\times-\,z\,:2\,+}$.

{\bf Note.} In inorder traversal {\bf (B)} we first visit
the first subtree (e.g., $\mathtt{y}$), and only then the
parent node (e.g., unary minus $\sim$).
See (Rosen2019, p.811).

{\scriptsize
{\em Grading.}
\begin{itemize}
\item Each correctly written expression is
\item Item {\bf (B)} with switched order of the unary minus and its child node $y$ is
$3$ points instead of $5$.
\item Minor typos in single characters get $4$ or $5$ points.
\item Any major differences from the correct result do not get points.
\end{itemize}
}

%\vspace{10pt}
\newpage
{\bf Question 10.}\\

We start from vertex $I$. At every step we
grow the tree by a single edge (so that it stays connected
and the newly added edge has the smallest possible weight).

\begin{tabular}{|l|l|} \hline
{\bf Step} & {\bf Newly Added Edge} \\ \hline
{\bf Step 1} & $IG$, $w=1$ \\ \hline
{\bf Step 2} & $IE$, $w=2$ \\ \hline
{\bf Step 3} & $ED$, $w=1$ \\ \hline
{\bf Step 4} & $DC$, $w=2$ \\ \hline
{\bf Step 5} & $CH$, $w=2$ \\ \hline
{\bf Step 6} & $GA$, $w=3$ \\ \hline
{\bf Step 7} & $AB$, $w=2$ \\ \hline
{\bf Step 8} & $IF$, $w=5$ \\ \hline
\end{tabular}

The total weight of all added edges (same as the total weight of the MST) is $18$.

\begin{figure}[!htb]
\center{\includegraphics[width=1.5in]{final/prim-weighted-mst.png}}
\caption{\label{fig:prim-weighted-mst} MST edges shown in blue.}
\end{figure}

{\scriptsize
{\em Grading.}
\begin{itemize}
\item Incorrectly adding up weights could subtract $1$ or $2$ points from the total.
\item Adding $9$ edge weights (or any other number instead of $8$ weights) and getting incorrect sum is $11$ points (instead of $15$).
\item Not showing the edges in answers (or displaying them in an order that differs from Prim's algorithm),
but still getting something close to MST is about $8$ points.
\end{itemize}
}

\end{document}


% !TEX root = ../zeth-protocol-specification.tex

\section{Data structures and representation}\label{preliminaries:structured-data}

\newcommand{\STR}{\mathsf{STR}}

\subsection{Structured data}\label{preliminaries:data-types:formulation}

When describing the operations to be performed and the data to be manipulated as part of the protocol, we commonly employ tuples of related data where each element of the tuple has some associated semantic meaning and which must often satisfy some conditions. In this section, we develop a framework to reason about such \emph{structured} data, where a single datum may consist of one or more logical parts (called \emph{fields}). The framework is built on top of simple mathematical concepts such as sets, and mappings between them, ensuring that we can always reason about structured data in a rigorous way. We also define notation to aid the specification of structured data, and to refer to specific components of a datum. This will be used extensively in the specification of the protocol.

As a simple motivating example, consider a protocol that processes data relating to individual people. This fictional system may send and receive data such as \emph{name}, \emph{age} and \emph{address} for a single person, grouping this data into a logical unit. Further, each piece of data must satisfy specific conditions (\emph{name} must be a series of characters from some alphabet, \emph{age} must be a positive integer, etc.) We shall make use of this example several times during the formulation below.

In what follows, let $\STR = \smallset{a, b, \ldots, y, z}^{*}$ (the Kleene star of the \emph{Roman alphabet}). In our formulation, field names $f_i$ will be elements in this set.

\begin{remark}
  Note that a similar formulation could be made using an arbitrary set, such as the same alphabet augmented with specific symbols, or the alphabet of a different language. Our choice of $\STR{}$ here is for simplicity.
\end{remark}

We begin by defining a data type as a set of values called ``fields'', each with a ``name'' from $\STR$. Abstract sets are used to constrain the values of each field.

\begin{definition}[Structured Data Type]\label{preliminaries:def:datatype}
  Let $f_0, \ldots, f_{n-1}$ be $n$ distinct elements of $\STR$ and let $V_0, \ldots, V_{n-1}$ be sets, for some $n \in \NN$. We define \emph{the structured data type $\datatypestyle{T}$ with fields $\smallset{(f_i, V_i)}_{i \in [n]}$} to be a set of values:
  \[
    \datatypestyle{T} = V_0 \times \cdots \times V_{n-1}
  \]
  with associated post-fix ``dot'' operators $.f_i : \datatypestyle{T} \to V_i$ for $i = 0, \ldots, n-1$, acting on values $\mathbf{x} \in \datatypestyle{T}$ to extract the individual elements:
  \[
    \mathbf{x}.f_i = v_i \text{, where } \mathbf{x} = (v_0, \ldots, v_{n-1}) \in \datatypestyle{T}
  \]

  Here, we say that the $i$-th field has \emph{field name} $f_i$, with \emph{value set} $V_i$. Each ``dot'' operator $.f_i$ \emph{extracts} the $i$-th component, or the \emph{value with field name $f_i$}.
\end{definition}

\begin{example}\label{preliminaries:eg:datatype-person}
  Consider our example protocol that processes information about people. A potentially useful structured data type $\datatypestyle{Person}$ may be defined with fields:
  \[
    \smallset{ (\varstyle{name}, \STR), (\varstyle{age}, \NN), (\varstyle{height}, \RR^+) }
  \]
  Values $\mathbf{p}$ in $\datatypestyle{Person}$ are simply tuples in $\STR{} \times \NN \times \RR^+$, with semantic meaning (name, age, height) assigned to each component of $\mathbf{p}$.

  Examples of valid elements in $\datatypestyle{Person}$ include $\mathbf{a} = (\varstyle{alice}, 28, 1.65)$ and $\mathbf{b} = (\varstyle{bob}, 31, 1.74)$, where the following equalities hold:
  \begin{align*}
    \mathbf{a}.\varstyle{name} &= \varstyle{alice}, \\
    \mathbf{b}.\varstyle{age} &= 31, \\
    \mathbf{b}.\varstyle{height} &= 1.74;
  \end{align*}
\end{example}

For clarity, structured data types may be specified using tables of names, descriptions and value sets, rather than sets of the form $\smallset{(f_i, V_i)}_{i \in [n]}$. Similarly, it is frequently convenient to include the \emph{field names} alongside values when specifying structured data values.

\begin{example}\label{preliminaries:eg:datatype-person-table}
  $\datatypestyle{Person}$ from \cref{preliminaries:eg:datatype-person} might be described in table-form as follows:
  \begin{table}[H]
    \centering
    \begin{tabular}{cp{15em}c}
      Field                & Description        & Data type \\ \toprule
      $\varstyle{name}$   & Name of the person & $\STR{}$ \\ \midrule
      $\varstyle{age}$    & Age in years       & $\NN$ \\ \midrule
      $\varstyle{height}$ & Height in meters   & $\RR^+$ \\ \midrule
    \end{tabular}
  \end{table}
\end{example}

\begin{example}\label{preliminaries:eg:datatype-person-value-with-field-names}
  The values $\mathbf{a}$ and $\mathbf{b}$ in \cref{preliminaries:eg:datatype-person} might be written as follows:
  \begin{align*}
    \mathbf{a} &=
    \{\varstyle{name}:\: \varstyle{alice}, \; \varstyle{age}:\: 28, \; \varstyle{height}:\: 1.65\} \\
    \mathbf{b} &=
    \{\varstyle{name}:\: \varstyle{bob}, \; \varstyle{age}:\: 31, \; \varstyle{height}:\: 1.74\}
  \end{align*}
\end{example}

\begin{remark}[``dot'' operators in assignment]\label{preliminaries:rem:dot-assignment}
  The ``dot'' operators may be used in algorithm descriptions to indicate \emph{assignment to a specific component}. For example $\mathbf{a}.\varstyle{age} \gets 29$ means that the value of the $\varstyle{age}$ field of $\mathbf{a}$ is replaced by the value $29$.

  Formally, for a structured data type $\datatypestyle{T}$ with fields $\smallset{(f_i, V_i)}_{i \in [n]}$ where $\mathbf{x} = (v_0, \ldots, v_{n-1}) \in \datatypestyle{T}$ and $v_i^\prime \in V_i$:
  \[
    \mathbf{x}.f_i \gets v_i^\prime
  \]
  is equivalent to:
  \[
    \mathbf{x} \gets (v_0, \ldots, v_{i-1}, v_i^\prime, v_{i+1}, \ldots, v_{n-1})
  \]
\end{remark}

We define one further operator and related assignment notation, convenient in cases where $V_i = X^m$ for sets $X$ and $m \in \NN$.
\begin{definition}[Square bracket operator]\label{def:squareb-operator}
  For $m \in \NN$ and set $X$, define the operator $[\ ] : X^m \times [m] \to X$ as:
  \[
    \mathbf{x}[i] = x_i \text{ where } \mathbf{x} = (x_0, \ldots, x_m)
  \]
  For the set $X^*$, the operator takes the form $[\ ] : X^* \times \NN \to X$, defined as:
  \[
    \mathbf{x}[i] =
    \begin{cases}
        x_i & \text{ if } \len{\mathbf{x}} > i \text{ where } \mathbf{x} = (x_0, \ldots) \\
        \bot & \text{otherwise}
    \end{cases}
  \]
\end{definition}

\begin{remark}[Square bracket operators in assignment]
  Similarly to \cref{preliminaries:rem:dot-assignment}, we develop assignment notation for the square bracket operator $[\ ]$.
  Let $\mathbf{x} = (x_0, \ldots, x_{m-1})$ be a member of $X^m$, and $x_i^\prime$ be some element in $X$. The statement:
  \[
    \mathbf{x}[i] \gets x_i^\prime
  \]
  is equivalent to:
  \[
    \mathbf{x} \gets (x_0, \ldots, x_{i-1}, x_i^\prime, x_{x+1}, \ldots, x_{m-1})
  \]
  Informally, this can be interpreted as replacing the $i$-th component of $\mathbf{x}$ with $x_i^\prime$.
\end{remark}

\begin{remark}[Deep structures and chained ``dot'' operators]
Consider the case of structured data $\datatypestyle{T}$ with fields $\{(f_i, V_i)\}_{i \in [n]}$ for $n \in \NN$. Let $\datatypestyle{T^\prime}$ be another structured data type with fields $\{(f^\prime_i, V^\prime_i)\}_{i \in [n^\prime]}$ for $n^\prime \in \NN$, and assume that $V_j = \datatypestyle{T^\prime}$ for some $j \in [n]$. Informally, the values of the $j$-th field of elements of $\datatypestyle{T}$ are themselves structured data of type $\datatypestyle{T^\prime}$.

In this case, ``dot'' operators may be \emph{chained}, so that $\mathbf{x}.f_j.f^\prime_k$ refers to the $k$-th field $v^\prime_k$ of the $j$-th field $v_j$ of $\mathbf{x} \in \datatypestyle{T}$.
\end{remark}

\begin{example}\label{preliminaries:eg:datatype-person-deep}
  Define a structured data type $\datatypestyle{Address}$ with fields $(\varstyle{country}, \STR{}), (\varstyle{zipcode}, \STR{})$. We redefine the structured data type $\datatypestyle{Person}$ from \cref{preliminaries:eg:datatype-person}, with an extra field $\varstyle{address}$ of type $\datatypestyle{Address}$. That is, $\datatypestyle{Person}$ is the structured data type with fields:
  \begin{table}[H]
    \centering
    \begin{tabular}{cp{15em}c}
      Field                 & Description           & Data type \\ \toprule
      $\varstyle{name}$    & Name of the person    & $\STR{}$ \\ \midrule
      $\varstyle{age}$     & Age in years          & $\NN$ \\ \midrule
      $\varstyle{height}$  & Height in meters      & $\RR^+$ \\ \midrule
      $\varstyle{address}$ & Address of the person & $\datatypestyle{Address}$ \\ \midrule
    \end{tabular}
  \end{table}
  An example element $\mathbf{a}$ in $\datatypestyle{Person}$ is:
  \begin{align*}
    \mathbf{a} =
    \{ &\\
       & \varstyle{name}:\: \varstyle{alice}, \\
       & \varstyle{age}:\: 28, \\
       & \varstyle{height}:\: 1.65, \\
       & \varstyle{address}:\: (\varstyle{country}:\: \varstyle{UK}, \varstyle{zipcode}:\: \varstyle{SW1A})\\
    \}
  \end{align*}
  In this case, the following equalities using the dot and square bracket operators all hold:
  \begin{align*}
    & \mathbf{a}.\varstyle{name} = \varstyle{alice} \\
    & \mathbf{a}.\varstyle{height} = 1.65 \\
    & \mathbf{a}.\varstyle{address}.\varstyle{country} = \varstyle{UK} \\
    & \mathbf{a}.\varstyle{address}.\varstyle{zipcode} = \varstyle{SW1A} \\
    & \mathbf{a}.\varstyle{address}.\varstyle{country}[1] = \varstyle{K}
  \end{align*}
\end{example}

\subsection{Representations}\label{preliminaries:data-types:representation}

The binary alphabet $\bin$, denoted $\BB$, is used to represent the presence or absence of an electrical signal in a computer. In fact, every piece of information in a computer is represented as a string of bits.
We assume the existence of an efficient binary representation for some set of primitive datatypes (such as the natural numbers $\NN$, or alphanumeric characters). Structured data types built up from primitive types (as described above) can then recursively be assigned similarly efficient representations.
This is used to define the following functions to \emph{encode} data to its bit-string representation, and to \emph{decode} such bit-strings back to elements of the original type.

\begin{definition}
  For a set $X$ of values which are to be represented as bit strings, we define functions:
  \begin{align*}
    \encode{}{X} &: X \to \BB^* \\
    \decode{}{X} &: \BB^* \to X \cup \bot
  \end{align*}
  satisfying
  \[
    \decode{\encode{x}{X}}{X} = x\ \forall x \in X
  \]
  to be the functions which encode (resp.~decode) elements of $X$ into (resp.~from) the bit-string representations chosen above.
  We note that $\decode{}{X}$ may return $\bot$ in the case that the input bit-string is malformed.
\end{definition}

Without ambiguity, we overload the functions $\encode{}{}$ and $\decode{}{}$ to mean $\encode{}{X}$ and $\decode{}{X}$ where the set $X$ is clear from context.

In the following sections, we assume that elements of $\NN$ are encoded as big-endian binary numbers in the natural way. We denote by $\NN_{b}$ the set of natural numbers that can be uniquely encoded in this way using $b$ bits (possibly with padding). In other words,
\[
  \NN_{b} = \smallset{x \in \NN\ \suchthat\ \encode{x}{\NN} \in \BB^{b}}
\]


\documentclass{article}
\usepackage[utf8]{inputenc}

\title{Common activation functions used in neural net}
\author{}
\date{}

\usepackage{natbib}
\usepackage{graphicx}
\usepackage{amsmath,amssymb}
\usepackage[left=2.5cm,right=2.5cm,top=1cm,bottom=1.25cm]{geometry}
\usepackage{hyperref}
\usepackage{multicol}
\usepackage[export]{adjustbox}
\usepackage{sidecap}
\hypersetup{colorlinks=true,urlcolor=blue}
\renewcommand{\thesubsection}{\arabic{subsection}.}

\begin{document}

\maketitle

\section*{Activation function}
The activation function of a node in an artificial neural network converts an input signal to an output signal. This is loosely based on the biological neuron where the activation function decides whether or not to pass the signal accumulated in a cell body to the next cell.

\section*{Why we need activation functions}

\begin{itemize}
    \item Activation functions determine the firing potential of a neuron and can act as a decision function(i.e., whether the neuron should fire or not).
    \item  A feedforward neural network with linear activation and any number of hidden layers is equivalent to just a linear neural neural network with no hidden layer. For example lets consider the neural network in figure with two hidden layers and no activation:\\
    \includegraphics[scale=0.75, center]{neural_net_three_layers.png}
    \begin{align*}
        y &= h_2 W_3 + b_3 \\
          &= (h_1 W_2 + b_2) W_3 + b_3 \\
          &= h_1 W_2 W_3 + b_2 W_3 + b_3 \\
          &= (x W_1 + b_1) W_2 W_3 + b_2 W_3 + b_3 \\
          &= x W_1 W_2 W_3 + b_1 W_2 W_3 + b_2 W_3 + b_3 \\
          &= x W' + b'
    \end{align*}
    So we could replace this neural net with a single layer neural net.This can be extended to $n$ layers. This indicates adding layers doesn't increase the approximation power of a linear neural net at all. We need non-linear activation functions to approximate non-linear functions and most real world problems are highly complex and non-linear. In fact when the activation function is non-linear, then a two-layer neural network with sufficiently large number of hidden units can be proven to be a universal function approximator.
    \item Activation functions are also needed for squashing the unbounded linearly weighted sum from neurons.
\end{itemize}

\section*{Some desirable properties of activation functions}
\begin{itemize}
    \item \textit{Nonlinear:} To approximate complex non-linear functions the activation function also needs to be non-linear.
    \item \textit{Continuously differentiable:} This is desirable for gradient based optimization methods.
    \item \textit{Monotonic:} When the activation function is monotonic, the error surface associated with a single-layer model is guaranteed to be convex.
    \item \textit{Approximates Identity function($f(x)=x$) near origin:} When activation functions have this property, the neural network will learn efficiently when its weights are initialized with small random values. This is because gradient near origin will be close to $1$. When the activation function does not approximate identity near the origin, special care must be used when initializing the weights. Activation functions where $f(0)=0$, $f'(0)=1$ and $f'(x)$ is continuous at $0$ have this property.
    \item \textit{Range:} When the range of activation function is finite, gradient-based training methods tend to be more stable because only limited range of values affect gradient significantly. When the range is infinite, training is generally more efficient(faster) because most of the values contribute significantly to gradients. In the latter case, smaller learning rates are typically necessary.

\end{itemize}

\section*{Common activation functions}

\subsection{Sigmoid}
\begin{itemize}
    \item Equation: $f(x) = \sigma(x) = \frac{1}{1+e^{-x}}$
    \item Derivative: $f'(x) = \sigma(x) (1 - \sigma(x))$
    \item Graph:\\ \includegraphics[ center]{sigmoid.png}
    \item Pros
    \begin{enumerate}
        \item Its non-linear, continuously differentiable, monotonic.
        \item Output is in the range of $[0, 1]$ so it can be used as a binary classifier or estimate probability.
        \item The bounded output means the activations won't blow up.
    \end{enumerate}
    \item Cons
    \begin{enumerate}
        \item Towards either end of the sigmoid function the change in activation is very small as input is changed and gradient is very close to zero. This gives rise to the \textbf{vanishing gradients problem}. This means that neurons in those regions will get very little gradient update and become saturated. The learning will become very slow.
        \item Sigmoid outputs are not zero centered and it does not approximate identity function near origin.
        \item Exponential function is computationally expensive.
    \end{enumerate}
\end{itemize}

\subsection{Tanh}
\begin{itemize}
    \item Equation: $f(x) = \tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}}$
    \item Derivative: $f'(x) = 1 - f(x)^2$
    \item Graph:\\ \includegraphics[ center]{tanh.png}
    \item Pros
    \begin{enumerate}
        \item Its non-linear, continuously differentiable, monotonic.
        \item Output is bounded in the range of $[-1, +1]$ so the activation won't blow up.
        \item It approximates identity function near origin. Optimization is easier than sigmoid because gradient is steeper.
    \end{enumerate}
    \item Cons
    \begin{enumerate}
        \item Suffers vanishing gradients problem just like sigmoid.
        \item Also computationally expensive.
    \end{enumerate}
\end{itemize}

\subsection{ReLU(Rectified Linear Unit)}
\begin{itemize}
    \item Equation:
    \begin{equation*}
        f(x)=\begin{cases}
        x, & \text{$x>=0$}.\\
        0, & \text{$x<0$}.
        \end{cases}
    \end{equation*}
    \item Derivative:
    \begin{equation*}
        f'(x)=\begin{cases}
        1, & \text{$x>=0$}.\\
        0, & \text{$x<0$}.
        \end{cases}
    \end{equation*}
    \item Graph:\\ \includegraphics[ center]{relu.png}
    \item Pros
    \begin{enumerate}
        \item Its non-linear, monotonic.
        \item The gradient does not saturate like sigmoid or tanh so it rectifies the vanishing gradient problem.
        \item The second derivative of the rectifying operation is $0$ almost everywhere, and the derivative of the rectifying operation is $1$ everywhere that the unit is active. This means that the gradient direction is far more useful for learning than it would be with activation functions that introduce second-order effects(sigmoid and tanh).
        \item It is computationally inexpensive.
    \end{enumerate}
    \item Cons
    \begin{enumerate}
        \item When the activation is zero for a neuron it doesn't get any further update via gradient descent. So such neuron will stop responding to variations in error or input signal.This is called \textbf{dying relu problem}. When initializing the parameters of the network, it can be a good practice to set all elements of bias to a small, positive value, such as $0.1$. This makes it very likely that the rectified linear units will be initially active for most inputs in the training set and allow the derivatives to pass through.
        \item It cannot be used in the output layer of a network. We usually use sigmoid(binary) or softmax(multiclass) for classification task and linear function for regression task.
        \item The range of relu is $[0, \infty)$. This means it can blow up the activation.
        \item ReLU is not differentiable at $x=0$. However it sill works for gradient based learning in practice because neural training training algorithms do not usually arrive at a local minimum of the cost function, but instead merely reduce its value significantly and gradient based optimization on a digital computer is subject to numerical error anyway.
    \end{enumerate}
\end{itemize}

\subsection{Leaky ReLU and Parametric ReLU}
\begin{itemize}
    \item Equation:
    \begin{equation*}
        f(x)=\begin{cases}
        x, & \text{$x>=0$}.\\
        \alpha x, & \text{$x<0$}.
        \end{cases}
    \end{equation*}
    $\alpha=0.01$ for Leaky ReLU
    \item Derivative:
    \begin{equation*}
        f'(x)=\begin{cases}
        1, & \text{$x>=0$}.\\
        \alpha, & \text{$x<0$}.
        \end{cases}
    \end{equation*}
    \item Graph:\\ \includegraphics[center]{leaky_relu.png}
    \item Pros
    \begin{enumerate}
        \item It has non-zero slope everywhere and so doesn't have the dying relu problem.
    \end{enumerate}
    \item Cons
    \begin{enumerate}
        \item The result is not always consistent and in practice doesn't always perform better than plain relu.
    \end{enumerate}
\end{itemize}

\subsection{ELU(Exponential Linear Unit)}
\begin{itemize}
    \item Equation:
    \begin{equation*}
        f(x)=\begin{cases}
        x, & \text{$x>0$}.\\
        \alpha (e^x-1), & \text{$x<=0$}.
        \end{cases}
    \end{equation*}
    \item Derivative:
    \begin{equation*}
        f'(x)=\begin{cases}
        1, & \text{$x>=0$}.\\
        \alpha e^x, & \text{$x<0$}.
        \end{cases}
    \end{equation*}
    \item Graph:\\ \includegraphics[center]{elu.png}
    \item Pros
    \begin{enumerate}
        \item It doesn't have dying relu problems.
        \item It becomes smooth slowly for negative inputs unlike relu which sharply becomes smooth.
    \end{enumerate}
    \item Cons
    \begin{enumerate}
        \item Output becomes saturated for large negative values and gradient becomes zero for neurons in that region.
    \end{enumerate}
\end{itemize}

\end{document}

source/notes/Common activation functions used in neural net/main.tex


%---------------------------Taper---------------------------
\section{Taper\label{s:hex-taper}}

Taper measures the maximum ratio of a cross-derivative to its shortest associated principal axis.
Given a pair of principal axes $f$ and $g$, the taper is
\[
  T_{fg} = \frac{\normvec{ X_{fg}}}{\min\left\{\normvec{ X_f},\normvec{X_g}\right\}}
\]
The metric is then the maximum taper of any cross-derivative
\[
  q = \max\left\{ T_{12}, T_{13}, T_{23} \right\}
\]

Note that if $\normvec{X_1}$ or $\normvec{X_2}$ or $\normvec{X_3} < DBL\_MIN$, we set $q = DBL\_MAX$.

\hexmetrictable{taper}%
{$1$}%                                        Dimension
{$[0,0.5]$}%                                  Acceptable range
{$[0,DBL\_MAX]$}%                             Normal range
{$[0,DBL\_MAX]$}%                             Full range
{$0$}%                                        Cube
{Adapted from \cite{tf:89}}%                  Citation
{v\_hex\_taper}%                              Verdict function name