% 	\begin{exercise}
% 		\begin{align*}
% 			\sqrt{1 - x^2}
% 		\end{align*}
% 	\end{exercise}

\section*{Solutions to Selected Problems}

\begin{exercise*}
  Find
  $\lim \limits_{x \rightarrow 0} f(x)$ where $$f(x) = \begin{cases}
            x & \mbox{ if $x$ is rational}   \\
            0 & \mbox{ if $x$ is irrational}
          \end{cases}$$
\end{exercise*}
\begin{proof}
  We'll prove that $\lim \limits_{x \rightarrow 0} f(x) = 0$. For this we need to show that for every $\epsilon$, there exists a $\delta$ such that for every $x$, if $0 < |x| < \delta$ then $|f(x)| < \epsilon$.

  Let $\epsilon > 0$, let $\delta = \epsilon$ and let $0 < |x| < \delta$ then either $x$ is a rational number or it is an irrational number.

  If $x$ is a rational number then
  \begin{align*}
    |f(x)| = |x| < \delta = \epsilon
  \end{align*}
  If $x$ is an irrational number then
  \begin{align*}
    |f(x)| = 0 < \epsilon
  \end{align*}
  which shows that $|f(x)|< \epsilon$ for all $0 < |x| < \delta$, which is what we wanted to prove.
\end{proof}

\begin{exercise*}
  Let $
    f(x) = \begin{cases}
      1 & \mbox{if $x$ is rational,}   \\
      0 & \mbox{if $x$ is irrational.}
    \end{cases}
  $
  \begin{enumerate}
    \item Prove that for every real number $L$, $\lim \limits_{x \rightarrow 0} f(x) \neq L$.
    \item Prove that $\lim \limits_{x \rightarrow 0} f(x) \neq \infty$. (Similarly for $-\infty$.)
  \end{enumerate}
  Hence the limit of $f$ at $0$ does not exist.
\end{exercise*}
\begin{proof}
  \begin{enumerate}
    \item   For the first part, we need to show that for any real number $L$, there exists an $\epsilon > 0$ such that for every $\delta > 0$ there exists an $x$ such that $0 < |x| < \delta$ and $|f(x) - L | \ge \epsilon$. We'll show that this is true for $\epsilon = 1/2$.

    Let $\epsilon = 1/2$, let $\delta > 0$, let $x_1$ be an irrational number satisfying $0 < |x_1| < \delta$ and let $x_2$ be a rational number satisfying $0 < |x_2 | < \delta$.

    We have already shown that for any $L$ at least one of $|L|$ or $|1 - L|$ is $\ge 1/2$. Hence either \begin{align*}
      |f(x_1) - L| \ge 1/2 \quad \mbox{ or } \quad |f(x_2) - L | \ge 1/2
    \end{align*}
    Thus the inequality $|f(x) - L | \ge \epsilon$ is true for either $\epsilon = x_1$ or $x_2$, which completes the proof.

    \item For the second part, we need to show that there exists an $\epsilon > 0$ such that for every $\delta > 0$ there exists an $x$ such that $0 < |x| < \delta$ and $|f(x)| \le \epsilon$. We'll show that this is true for $\epsilon = 1$.

    Let $\epsilon = 1$, let $\delta > 0$, let $x$ be an irrational number satisfying $0 < |x| < \delta$.

    Then $|f(x)| = 0 \le 1 = \epsilon$ which is what we wanted to prove.

    Similar proof works for $-\infty$.\\
  \end{enumerate}
  Since $\lim \limits_{x \rightarrow 0} f(x)$ does not equal any real number, $\infty$, or $-\infty$, the limit of $f$ at 0 does not exist.
\end{proof}


\documentclass[main.tex]{subfiles}
\begin{document}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%   heap and priprity queue
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
In this chapter, we introduce heap data structures which is essentially an array object but it can be viewed as a nearly complete binary tree. The concept of the data structures in this chapter is between liner and non-linear, that is using linear data structures to mimic the non-linear data structures and its behavior for higher efficiency under certain context.
%%%%%%%%%%%%%%%%%heap%%%%%%%%%%%%%%%%%
\section{Heap}
\label{sec_heap}
Heap is a tree based data structures that satisfies \textbf{heap property} but implemented as an array data structure. There are two kinds of heaps: \textbf{max-heaps} and \textbf{min-heaps}. In both kinds, the values in the nodes satisfy a \textbf{heap property}. For max-heap, the property states as for each node in the heap at $i$, $A[p[i]] <= A[i]$.  Normally, heap is based on binary tree, which makes it a binary heap. Fig.~\ref{fig:max-heap-1} show a binary max-heap and how it looks like in a binary tree data structure. In the following content, we default our heap is a binary heap. Thus, the largest element in a max-heap is stored at the root. For a heap of $n$ elements the height is $\log n)$. % as for every node i other than root. $A[PARENT(i)]>= A[i]$.  The unique usage of Heap, including miniHeap and maxiHeap, Monotic Heap.  The (binary) heap data structure is an array that we can view as a nearly complete binary tree. The tree is completely filled on all levels except possibly the lowest, which is filled from left up to a point.
% \subsection{Introduction}
\begin{figure}[h!]
    \centering
    \includegraphics[width = 0.98\columnwidth]{fig/binary_tree.png}
    \caption{Max-heap be visualized with binary tree structure on the left, and implemnted with Array on the right.}
    \label{fig:max-heap-1}
\end{figure}

As we can see we can implement heap as an array due to the fact that the tree is complete. A complete binary tree is one in which each level must be fully filled before starting to fill the next level. Array-based heap is more space efficient compared with tree based due to the non-existence of the child pointers for each node. To make the math easy, we iterate node in the tree starting from root in the order of level by level and from left to right with beginning index as 1 (shown in Fig.~\ref{fig:max-heap-1}).  According to such assigning rule, the node in the tree is mapped and saved in the array by the assigned index (shown in Fig.~\ref{fig:max-heap-1}). In heap, we can traverse the imaginary binary tree in two directions:  \textbf{root-to-leaf} and  \textbf{leaf-to-root}. Given a parent node with p as index, the left child of can be found in position $2p$ in the array. Similarly, the right child of the parent is at position $2p + 1$ in the list. To find the parent of any node in the tree,  we can simply use $\lfloor p/2\rfloor$.  In Python3, use integer division $n//2$. \textit{Note: we can start index with 0 as used in \textbf{heapq} library introduced later in this section. Given a node $x$, the left and right child will be $2*x+1$, $2*x+2$, and the parent node will have index $(x-1)//2$.}

The common application of heap data structure include:
\begin{itemize}
    \item Implementing a priority-queue data structure which will be detailed in the next section so that insertion and deletion can be implemented in $O(\log n)$; Priority Queue is an important component in algorithms like Kruskal's for minimum spanning tree (MST) problem and Dijkstra's for single-source shortest paths (SSSP) problem.
    \item Implementing heapsort algorithm,
\end{itemize}

Normally, there is usually no notion of 'search' in heap, but only insertion and deletion, which can be done by traversing a $O(\log n)$ leaf-to-root or root-to-leaf path.
%%%%%%%%%%%%%%%%%%Basic Implementation%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Basic Implementation}
The basic methods of a heap class should include: \textbf{pop}, \textbf{push}, and \textbf{heapify}. \textbf{push} an item into the heap and \textbf{pop} the root item at the heap out, and still maintain the heap property.  And \textbf{heapify} denotes the operation needed for an given array, to convert it to a heap directly and efficiently.

Let's implement a heap class using list. Because the first element of the heap is actually empty, we define our class as follows:
\begin{lstlisting}[language=Python]
class Heap:
    def __init__(self):
        self.heap = [None]
        self.size = 0
    def __str__(self):
        out = ''
        for i in range(1, self.size + 1):
            out += str(self.heap[i]) + ' '
        return out
\end{lstlisting}
Assuming we already have got a heap shown in Fig.~\ref{fig:max-heap-1}, push or pop an item from the current heap requires us to do post-processing in order to maintain the heap property. Let's discuss the two cases. Change it to use max heap as example.

\paragraph{Push with Floating} When we push an item into, to maintain the complete binary tree property, the new item goes to the end of the heap(array) first. Assuming the new item a[i] is the smallest item up till now, there will be violation of the heap property through the \textbf{a[i]->root path}. To correct the potential violation, we traverse the path a[i]->root, and compare each node and its parent  to decide if a swap operation is needed. For a min-heap, if the child node is smaller than the parent, that is a violation, and we swap these two nodes to let a[i] \textbf{float up} to make sure the subtree of a[i].parent obey the min-heap property.  For example, in the min-heap.  The time complexity is the same as the height of the complete tree, which is $O(\log n)$.
\begin{lstlisting}[language=Python]
    def _float(self, index): # enforce min-heap, leaf-to-root
      while index // 2: # while parent exist
          p_index = index // 2
          print('p', p_index, index)
          if self.heap[index] < self.heap[p_index]: # a violation
              # swap
              self.heap[index], self.heap[p_index] = self.heap[p_index], self.heap[index]
          else:
            break
          index = p_index # move up the node
    def insert(self, val):
        self.heap.append(val)
        self.size += 1
        self._float(index = self.size)
\end{lstlisting}

\paragraph{Pop with Sinking} When we pop out the item at root node,  or delete any item a[i], an empty spot appears at that position. To maintain the complete binary tree, we first  simply use the last item to fill in this spot. However, in a min-heap, the last item will mostly not be the smallest item among the subtree rooted at a[i]. The smallest item will appear anywhere in the subtree. We simply do a search starts from node a[i] and compare its value with left and right child. The left and right subtree obey the min-heap property already, therefore the smallest item is among a[i], left, right. If the node is larger than its smaller child node, we swap the parent with the smaller child, and move our pointer to the smaller child node and repeat the above process until the current node is the smallest among these three nodes.   This process is called like sinking down a[i] along the \textbf{path a[i]->leaf}. Same as the insert in the case of complexity, $O(\log n)$.
\begin{lstlisting}[language=Python]
    def _sink(self, index): # enforce min-heap, root-to-leaf
        while 2 * index <= self.size:
            li = 2 * index
            ri = li + 1
            mi = index
            if self.heap[li] < self.heap[mi]:
              mi = li
            if ri <= self.size and self.heap[ri] < self.heap[mi]:
              mi = ri
            if mi != index:
                # swap index with mi
                self.heap[index], self.heap[mi] = self.heap[mi], self.heap[index]
            else:
              break
            index = mi
    def pop(self):
        val = self.heap[1]
        self.heap[1] = self. heap.pop()
        self.size -= 1
        self._sink(index = 1)
        return val
\end{lstlisting}
Now, let us run an example:
\begin{lstlisting}[language=Python]
h = Heap()
lst = [21, 1, 45, 78, 3, 5]
for v in lst:
    h.insert(v)
print('heapify with insertion: ', h)
h.pop()
print('after pop(): ', h)
\end{lstlisting}
The output is listed as:
\begin{lstlisting}
heapify with insertion: 1 3 5 78 21 45
after pop(): 3 21 5 78 45
\end{lstlisting}

\paragraph{Heapify with Bottom-up Sinking}  Heapify is a procedure that convert a list to a heap data structure.  We have learned the insert procedure. To heapify a list, we can do it through a series of insert iterating through the items in the list and we get an upper-bound complexity of $O(n\log n)$. However, a more efficient way to do it is is to treat the given list as a tree and to heapify directly on the list. %There are two possibly two ways to do this: (1) through sinking and (2) through floating.
To satisfy the heap property, we need to first start from the smallest subtree. For leaf nodes, they have no children which satisfies the heap property naturally. Therefore we can jumpy to the last parent node, which will be at position a[n//2]. We apply the sinking process as used in \textbf{pop} so that this subtree rooted at current node obeys the heap property. And we   iterate through all the parents nodes that is a[1...n//2] in reversed order, we can guarentee that final complete binary tree still obeys the heap property. This follows a divide-and-conquer (DP) fashion. Instead of heaipfy A[1...n], we first, heaipfy A[n], A[n-1...n], A[n-2...n], ..., A[1...n].  The process is shown in Fig.~\ref{fig:heapify}. With this process, it can give us a tighter upper bound and close to $O(n)$.
\begin{figure}[h!]
    \centering
    \includegraphics[width = 0.98\columnwidth]{fig/heapify.png}
    \caption{Heapify for a given list.}
    \label{fig:heapify}
\end{figure}
\begin{lstlisting}[language=Python]
  def heapify_sink(self, lst):
      self.heap = [None] + lst
      self.size = len(lst)
      for i in range(self.size//2, 0, -1):
        self._sink(i)
\end{lstlisting}

Now, run the following code:
\begin{lstlisting}[language=Python]
h = Heap()
h.heapify(lst)
print('heapify with heapify:', h)
\end{lstlisting}
Out put is:
\begin{lstlisting}
heapify with heapify: 1 5 21 78 3 45
\end{lstlisting}
\begin{bclogo}[couleur = blue!30, arrondi=0.1,logo=\bccrayon,ombre=true]{Which way is more efficient building a heap from a list?} Using insertion or heapify? What is the efficiency of each method? The experimental result can be seen in the code.
\end{bclogo}
When we are solving a problem, unless specifically required for implementation, we can always use an existent Python module/package. Here, we introduce one Python module: heapq that implements heap data structure for us.
% \begin{enumerate}
% \item MAX-HEAPIFY, runs in $O(lgn)$, is the key to maintaining the max-heap property
% \item BUILD-MAX-HEAP, runs in linear time, produces a maxheap from an unordered input array
% \item MAX-HEAP-INSERT, HEAP-EXTRACT-MAX, HEAP-INCREASE-KEY, and HEAP-MAXIMUM, runs in $O(lgn)$ time, allow the heap data structure to implement a priority queue
% \end{enumerate}
%%%%%%%%%%%%%%%%%%Python Built-in Module: heapq%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Python Built-in Library: heapq}
\textbf{heapq}: heapq is a built-in library in Python that implements relevant functions to carry out various operations on heap data structure. These functions are listed and described in Table~\ref{tab:functions_in_heapq}. \textit{To note that heapq is not a data type like queue.Queue() or collections.deque(), it is a library (or class) that can do operations like it is on a heap.} %, which can be used to maintain a priority queue. Operations include heappush, heappop, and nsmallest. heapq in python to maintain a priority queue with $O(logn)$.
\begin{table}[h]
\begin{small}
\centering
\noindent\captionof{table}{ Methods of \textbf{heapq}}
 \noindent \begin{tabular}{|p{0.25\columnwidth}|p{0.75\columnwidth}| }
  \hline
Method & Description   \\ \hline
heappush(h, x)  &  Push the value item onto the heap, maintaining the heap invariant.  \\\hline
heappop(h)  &Pop and return the \textit{smallest} item from the heap, maintaining the heap invariant. If the heap is empty, IndexError is raised.\\ \hline
heappushpop(h, x)  &Push item on the heap, then pop and return the smallest item from the heap. The combined action runs more efficiently than heappush() followed by a separate call to heappop().\\ \hline
heapify(x) & Transform list x into a heap, in-place, in linear time.\\ \hline
heapreplace(h, x) & Pop and return the smallest item from the heap, and also push the new item. The heap size doesn’t change. If the heap is empty, IndexError is raised. This is more efficient than heappop() followed by heappush(), and can be more appropriate when using a fixed-size heap.\\ \hline
nlargest(k, iterable, key = fun) & This function is used to return the k largest elements from the iterable specified and satisfying the key if mentioned. \\ \hline
nsmallest(k, iterable, key = fun) & This function is used to return the k smallest elements from the iterable specified and satisfying the key if mentioned. \\ \hline
\end{tabular}
  \label{tab:functions_in_heapq}
  \end{small}
\end{table}
heapq has some other functions like merge(), nlargest(), nsmallest() that we can use. Check out \url{https://docs.python.org/3.0/library/heapq.html} for more detials.

\paragraph{Min-Heap} Now, let us try to heapify the same examplary list as used in the last section, [21, 1, 45, 78, 3, 5], we use need to call the function heapify(). The time complexity of heapify is $O(n)$
\begin{lstlisting}[language = Python]
'''implementing with heapq'''
from heapq import heappush, heappop, heapify
h = [21, 1, 45, 78, 3, 5]
heapify(h) # inplace
print('heapify with heapq: ', h)
\end{lstlisting}
The print out is:
\begin{lstlisting}
heapify with heapq:  [1, 3, 5, 78, 21, 45]
\end{lstlisting}

 Here we demonstrate how to use function nlargest() and nsmallest() if getting the first n largest or smallest is what we need, we do not need to heapify() the list as we needed in the heap and pop out the smallest. The step of heapify is built in these two functions.
\begin{lstlisting}[language=Python]
''' use heapq to get nlargest and nsmallest'''
li1 = [21, 1, 45, 78, 3, 5]
# using nlargest to print 3 largest numbers
print("The 3 largest numbers in list are : ", end="")
print(heapq.nlargest(3, li1))

# using nsmallest to print 3 smallest numbers
print("The 3 smallest numbers in list are : ", end="")
print(heapq.nsmallest(3, li1))
\end{lstlisting}
The print out is:
\begin{lstlisting}
The 3 largest numbers in list are : [78, 45, 21]
The 3 smallest numbers in list are : [1, 3, 5]
\end{lstlisting}

\paragraph{Max-Heap} As we can see the default heap implemented in the heapq library is forcing the heap property of the min-heap. What if we want a max-heap instead? In heapq library, it does offer us function, but it is intentionally hided from users. It can be accessed like: heapq.\_[function]\_max(). Now, let us implement a max-heap instead.
\begin{lstlisting}[language = Python]
# implement a max-heap
h = [21, 1, 45, 78, 3, 5]
heapq._heapify_max(h) # inplace
print('heapify max-heap with heapq: ', h)
\end{lstlisting}
The print out is:
\begin{lstlisting}
heapify max-heap with heapq:  [78, 21, 45, 1, 3, 5]
\end{lstlisting}

Also, in practise, a simple hack for the max-heap is to save data as negative. Also, in the priority queue.
% What is we want a max-heap which returns the largest number instead of the smallest each time?
% \subsubsection{Max-heap and Min-heap}
% We can write our own MinHeap and MaxHeap class wrapper as follows so that it can be easier to use:
% \begin{lstlisting}[language = Python]
% class MaxHeapObj(object):
%     def __init__(self,val): self.val = val
%     def __lt__(self,other): return self.val > other.val
%     def __eq__(self,other): return self.val == other.val
%     def __str__(self): return str(self.val)

% class MinHeap(object):
%     def __init__(self): self.h = []
%     def heappush(self,x): heapq.heappush(self.h,x)
%     def heappop(self): return heapq.heappop(self.h)
%     def __getitem__(self,i): return self.h[i]
%     def __len__(self): return len(self.h)

% class MaxHeap(MinHeap):
%     def heappush(self,x): heapq.heappush(self.h,MaxHeapObj(x))
%     def heappop(self): return heapq.heappop(self.h).val
%     def __getitem__(self,i): return self.h[i].val
% \end{lstlisting}
\paragraph{More Private Functions}
\begin{table}[h]
\begin{small}
\centering
\noindent\captionof{table}{ Private Methods of \textbf{heapq}}
 \noindent \begin{tabular}{|p{0.25\columnwidth}|p{0.75\columnwidth}| }
  \hline
Method & Description   \\ \hline
heappush(h, x)  &  Push the value item onto the heap, maintaining the heap invariant.  \\\hline
heappop(h)  &Pop and return the \textit{smallest} item from the heap, maintaining the heap invariant. If the heap is empty, IndexError is raised.\\ \hline
heappushpop(h, x)  &Push item on the heap, then pop and return the smallest item from the heap. The combined action runs more efficiently than heappush() followed by a separate call to heappop().\\ \hline
heapify(x) & Transform list x into a heap, in-place, in linear time.\\ \hline
heapreplace(h, x) & Pop and return the smallest item from the heap, and also push the new item. The heap size doesn’t change. If the heap is empty, IndexError is raised. This is more efficient than heappop() followed by heappush(), and can be more appropriate when using a fixed-size heap.\\ \hline
nlargest(k, iterable, key = fun) & This function is used to return the k largest elements from the iterable specified and satisfying the key if mentioned. \\ \hline
nsmallest(k, iterable, key = fun) & This function is used to return the k smallest elements from the iterable specified and satisfying the key if mentioned. \\ \hline
\end{tabular}
  \label{tab:functions_in_heapq}
  \end{small}
\end{table}

\paragraph{With Tuple/List or Customized Object as Elements}
Any object that supports comparison (\texttt{\_cmp\_()}) can be used in heap with \texttt{heapq}. When we want our item includes information as (priority, task), we can either put it in tuple or list. In the heap, we can change the value of any item just as in the list. However, the problem occurs after the change that the list will violate the heap priority. What we can do is use function such as \texttt{\_siftdown(heap, 0, len(heap)-1)} (used to implement heappush, and called with decreased priority ) and \texttt{\_siftup(heap, 0)} (used to implement heappop, and called with increased priority).
\begin{lstlisting}[language=Python]
import heapq

heap = [[3, 'a'], [10, 'b'], [5,'c']]
heapq.heapify(heap)
print(heap)

heap[0] = [6, 'a']
print(heap)
heapq._siftup(heap, 0) #simlar to remove heap[0], put this item at the end
print(heap)
\end{lstlisting}

%%%%%%%%%%%%%%%%%%%%priority Queue%%%%%%%%%%%%%%%%%%%
\section{Priority Queue}
\label{sec_priority_queue}
A priority queue is an abstract data type(ADT) and an extension of queue with properties: (1) additionally each item has a priority associated with it. (2) In a priority queue, an item with high priority is served (dequeued) before an item with low priority. (3) If two items have the same priority, they are served according to their order in the queue.

Heap is generally preferred for priority queue implementation because of its better performance compared with arrays or linked list. Also, in Python queue module, we have \texttt{PriorityQueue()} class that provided us the implementation. Beside, we can can implement priority queue with  \texttt{heapq} library too. These contents will be covered in the next two subsection.

Applications of Priority Queue:
\begin{enumerate}
    \item CPU Scheduling
    \item Graph algorithms like Dijkstra’s shortest path algorithm, Prim’s Minimum Spanning Tree, etc
    \item All queue applications where priority is involved.
\end{enumerate}

\subsubsection{Implement with \texttt{heapq}  Library} The core function is the ones used to implement the heap: \texttt{heapify()}, \texttt{push()}, and \texttt{pop()}. The official document:\url{https://docs.python.org/2/library/heapq.html} gave the exact implementation. However, we are still going to summarize and organize this information in our book. In order to implement priority queue, our binary heap needs to have the following features:
\begin{enumerate}
    \item Sort stability: when we get two tasks with equal priorities, we return them in the order as of they were originally added. A potential solution is to modify the original 2-element list (priority, task) into a 3-element list as (priority, count, task). The entry \texttt{count} serves as a tie-breaker so that two tasks with the same priority are returned in the order they were added. And also, since no two entry counts are the same the tuple comparison will never attemp to directly compare two tasks.
    \item Find a task in the heap, and either remove it or update its priority. Situations like the priority of a task changes or if a pending task needs to be removed. We understand how inconvenient it can be to find the non-root item and update its value. Normally, finding the item is a linear search which takes $O(n)$ and update its value using either \texttt{\_siftdown()} or \texttt{\_siftup()} can be $O(\log n)$. The solution is: (1) do not remove the task other than the \texttt{pop} operation, but mark it as REMOVED instead; (2) to define a dictionary that use \texttt{task} as key and the 3-element list as value. We name it \texttt{entry\_finder}. When the entry is a list, in the heap that encompass these items will only get pointers. Therefore, we can execute the find/mark as removed operation using task as key and do it in the \texttt{entry\_finder} instead.
\end{enumerate}
Python code:
\begin{lstlisting}[language=Python]
from heapq import heappush, heappop, heapify
from typing import List
import itertools
class PriorityQueue:
  def __init__(self, items:List[List]=[]):
      self.pq = []                         # list of entries arranged in a heap
      self.entry_finder = {}               # mapping of tasks to entries
      self.REMOVED = '<removed-task>'      # placeholder for a removed task
      self.counter = itertools.count()     # unique sequence count
      # add count to items
      for p, t in items:
        item = [p, next(self.counter), t]
        self.entry_finder[t] = item
        self.pq.append(item)
      heapify(self.pq)

  def add_task(self, task, priority=0):
      'Add a new task or update the priority of an existing task'
      if task in self.entry_finder:
          self.remove_task(task)
      count = next(self.counter)
      entry = [priority, count, task]
      self.entry_finder[task] = entry
      heappush(self.pq, entry)

  def remove_task(self, task):
      'Mark an existing task as REMOVED.  Raise KeyError if not found.'
      entry = self.entry_finder.pop(task)
      entry[-1] = self.REMOVED

  def pop_task(self):
      'Remove and return the lowest priority task. Raise KeyError if empty.'
      while self.pq:
          priority, count, task = heappop(self.pq)
          if task is not self.REMOVED:
              del self.entry_finder[task]
              return task
      raise KeyError('pop from an empty priority queue')
\end{lstlisting}
Let's run an example with our customized \texttt{PriorityQueue} class:
\begin{lstlisting}[language=Python]
pq = PriorityQueue(items=[[6, 'task 6'], [5, 'task5'], [19, 'task19']])
print(pq.pq)
pq.add_task('task 10', 10)
print(pq.pq)
pq.remove_task('task5')
print(pq.pq)
pq.pop_task()
\end{lstlisting}
With output as:
\begin{lstlisting}[numbers=none]
[[5, 1, 'task5'], [6, 0, 'task 6'], [19, 2, 'task19']]
[[5, 1, 'task5'], [6, 0, 'task 6'], [19, 2, 'task19'], [10, 3, 'task 10']]
[[5, 1, '<removed-task>'], [6, 0, 'task 6'], [19, 2, 'task19'], [10, 3, 'task 10']]
'task 6'
\end{lstlisting}

\subsubsection{Implement with \texttt{PriorityQueue} class} Class \texttt{PriorityQueue()} is the same as \texttt{Queue()}, \texttt{LifoQueue()}, they have same member functions as shown in Table~\ref{tab:methods_of_queue}. Therefore, we skip the semantic introduction. \texttt{PriorityQueue()} normally thinks that the smaller the value is the higher the priority is. We use a similar example as above to demonstrate its function.
\begin{lstlisting}[language=Python]
import queue
pq = queue.PriorityQueue()
items=[[6, 'task 6'], [5, 'task5'], [19, 'task19']]
for item in items:
  pq.put(item)

print(pq.queue)
next_job = pq.get()
print('processing job:', next_job)
print(pq.queue)
\end{lstlisting}
The output is:
\begin{lstlisting}
[[5, 'task5'], [6, 'task 6'], [19, 'task19']]
processing job: [5, 'task5']
[[6, 'task 6'], [19, 'task19']]
\end{lstlisting}
If we want to give the number with larger value as higher priority, a simple hack is to pass by negative value. Another more professional way is to pass by a customized object and rewrite the comparison operator: < and == in the class with \_\_lt\_\_() and \_\_eq\_\_(). In the following code, we show how to use higher value as higher priority.
\begin{lstlisting}[language = Python]
class Job(object):
    def __init__(self, priority, description):
        self.priority = priority
        self.description = description
        print('New job:', description)
        return
    # def __cmp__(self, other):
    #     return cmp(self.priority, other.priority)
    '''customize the comparison operators '''
    def __lt__(self, other): # <
        try:
            return self.priority > other.priority
        except AttributeError:
            return NotImplemented
    def __eq__(self, other): # ==
        try:
            return self.priority == other.priority
        except AttributeError:
            return NotImplemented

q = Queue.PriorityQueue()

q.put( Job(3, 'Mid-level job') )
q.put( Job(10, 'Low-level job') )
q.put( Job(1, 'Important job') )

while not q.empty():
    next_job = q.get()
    print('Processing job:', next_job.priority)
\end{lstlisting}
The print out is:
\begin{lstlisting}
Processing job: 10
Processing job: 3
Processing job: 1
\end{lstlisting}
If we want the priority queue to be able to update the priority of a task, we can apply similar wrapper in the \texttt{heapq} section.
\begin{bclogo}[couleur = blue!30, arrondi=0.1,logo=\bccrayon,ombre=true]{In single thread programming, is \textbf{heapq} or \textbf{PriorityQueue} more efficient?} In fact, the PriorityQueue implementation uses heapq under the hood to do all prioritisation work, with the base Queue class providing the locking to make it thread-safe. While heapq module offers no locking, and operates on standard list objects. This makes the heapq module faster; there is no locking overhead. In addition, you are free to use the various heapq functions in different, noval ways, while the PriorityQueue only offers the straight-up queueing functionality.
\end{bclogo}
Let us take these knowledge into practice with a LeetCode Problem:
347. Top K Frequent Elements (medium). Given a non-empty array of integers, return the k most frequent elements.
\begin{lstlisting}[numbers=none]
Example 1:

Input: nums = [1,1,1,2,2,3], k = 2
Output: [1,2]

Example 2:

Input: nums = [1], k = 1
Output: [1]
\end{lstlisting}

Analysis: to solve this problem, we need to first using a hashmap to get information as: item and its freqency. Then, we need to obtain the top frequent elements. The second step can be down with sorting, or using heap we learned.

\textbf{Solution 1: Use Counter().} Counter() has a function most\_common(k) that will return the top k most frequent items. However, its complexity will be $O(n \log n)$.
\begin{lstlisting}[language=Python]
from collections import Counter
def topKFrequent(self, nums, k):
    return [x for x, _ in Counter(nums).most_common(k)]
\end{lstlisting}

\textbf{Solution 2: Use dict and heapq.nlargest()}. The complexity should be better than $O(n \log n)$.
\begin{lstlisting}[language=Python]
from collections import Counter
import heapq
def topKFrequent(self, nums, k):
    count = collections.Counter(nums)
    return heapq.nlargest(k, count.keys(), key=count.get)
\end{lstlisting}

We can also use PriorityQueue().
\begin{lstlisting}[language=Python]
from queue import PriorityQueue
class Solution:
def topKFrequent(self, nums, k):
    h = PriorityQueue()

    # build a hashmap (element, frequency)
    temp = {}
    for n in nums:
        if n not in temp:
            temp[n] = 1
        else:
            temp[n] += 1
    # put them as (-frequency, element) in the queue or heap
    for key, item in temp.items():
        h.put((-item, key))

    # get the top k frequent ones
    ans = [None]*k
    for i in range(k):
        _, ans[i] = h.get()
    return ans
\end{lstlisting}

\section{Bonus}
\label{heap_sec_bonus}
\paragraph{Fibonacci heap} With fibonacc heap, insert() and getHighestPriority() can be implemented in O(1) amortized time and deleteHighestPriority() can be implemented in O(Logn) amortized time.
%%%%%%%%%%%%%%%%%%%%%%%LeetCode problems%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Exercises}

\textbf{selection with key word: kth. These problems can be solved by sorting, using heap, or use quickselect}
\begin{enumerate}
\item 703. Kth Largest Element in a Stream (easy)
    \item 215. Kth Largest Element in an Array (medium)
    \item 	347. Top K Frequent Elements (medium)
    \item 373. Find K Pairs with Smallest Sums (Medium
	\item 378. Kth Smallest Element in a Sorted Matrix (medium)
\end{enumerate}
\textbf{priority queue or quicksort, quickselect}
\begin{enumerate}
    \item 23. Merge k Sorted Lists (hard)
    \item 253. Meeting Rooms II (medium)
    \item 621. Task Scheduler (medium)
\end{enumerate}
\end{document}


\section{Introduction}
Let us first briefly describe finite difference methods and finite element methods used to solve the
following boundary value problem
\begin{equation}
\label{laplace}
-\Delta u = f,  \mbox{ in } \Omega,\quad
u=0  \mbox{ on } \partial\Omega,\quad
\Omega=(0,1)^2.
\end{equation}
For the $x$ direction and the $y$ direction, we consider the partition:
\begin{equation}\label{partitionyx}
 0=x_0<x_1<\cdots<x_{n+1}=1, \quad x_j=\frac{j}{n+1},\quad (j=0,\cdots,n+1);
 \end{equation}
 \begin{equation}\label{partitiony}
 0=y_0<y_1<\cdots<y_{n+1}=1, \quad y_j=\frac{j}{n+1},\quad (j=0,\cdots,n+1).
\end{equation}
Such a uniform partition in the $x$ and $y$ directions leads us to a special example in two dimensions,
a uniform square mesh $\R_h^2 = \big\{(mh,nh); n, m \in \Z\big\}$ (Figure \ref{fig:2dpartition}).
Let $\Omega_h = \Omega\cap\R_h^2$, the set of interior mesh points and $\partial\Omega_h = \partial\Omega\cap\R_h^2$, the set of boundary mesh points.

\begin{figure}
\begin{center}
\setlength{\unitlength}{0.5mm}
\begin{picture}(45,45)(50,0)
\linethickness{0.25mm}
\multiput(0,0)(10,0){6}{\line(0,1){50}}
\multiput(0,0)(0,10){6}{\line(1,0){50}}
\put(0,0){\line(1,1){50}}
\put(10,0){\line(1,1){40}}
\put(20,0){\line(1,1){30}}
\put(30,0){\line(1,1){20}}
\put(40,0){\line(1,1){10}}
\put(0,10){\line(1,1){40}}
\put(0,20){\line(1,1){30}}
\put(0,30){\line(1,1){20}}
\put(0,40){\line(1,1){10}}
\put(47,34){$\displaystyle \left. \begin{array}{l}~ \\ ~\end{array}
\right\} h={1\over n+1}$}
\put(54,14){$\displaystyle N = n^2$}
\multiput(100,0)(10,0){6}{\line(0,1){50}}
\multiput(100,0)(0,10){6}{\line(1,0){50}}
\put(147,34){$\displaystyle \left. \begin{array}{l}~ \\ ~\end{array}
\right\} h={1\over n+1}$}
\put(154,14){$\displaystyle N = n^2$}
\end{picture}
\setlength{\unitlength}{0.5mm}
\end{center}
\label{fig:2dpartition}
\caption{Two-dimensional uniform grid for finite element and finite difference}
\end{figure}

\subsection{Five-point finite difference methods}
We can use the center difference to approximate the derivatives as follows:
$$
\frac{\partial^2 u}{\partial x^2}(x_i,y_j)=
\frac{u(x_{i+1},y_j)-2u(x_i,y_j)+u(x_{i-1},y_j)}{h^2}+{\mathcal O}(h^2),
$$
$$
\frac{\partial^2 u}{\partial y^2}(x_i,x_j)=
\frac{u(x_{i},y_{j+1})-2u(x_i,y_j)+u(x_{i},y_{j-1})}{h^2}+{\mathcal
 O}(h^2).
$$
Thus,
\begin{equation}\label{2dfd-truncation}
(-\Delta_hu)(x_i,y_j)=(-\Delta u)(x_i,y_j)+{\mathcal O}(h^2)
\end{equation}
where $-\Delta_h$ is a discretized operator for $-\Delta$ given by
\begin{equation}
  \label{Delta-h}
(-\Delta_hu)(x_i,y_j)= \frac{1}{h^2}(4u(x_i,y_j)-(u(x_{i+1},y_j)+u(x_{i-1},y_j)
+u(x_i,y_{j+1})+u(x_i,y_{j-1}))).
\end{equation}
The truncation error \eqref{2dfd-truncation} can be proved by applying Taylor's expansion directly.

The finite difference scheme is then formed by
\begin{equation}
  \label{2d-fd0}
4u_{ij}-(u_{i+1,j}+u_{i-1,j}+u_{i,j+1}+u_{i,j-1})=f_{i,j},~~u_{i,j}=0~~\hbox{if}~~i ~~\hbox{or}~~ j\in \{0, n+1\}
\end{equation}
where
\begin{equation}
  \label{fij-fd}
f_{i,j} =h^2 f(x_i, y_j).
\end{equation}

\begin{proposition}
The mapping $A$ in \eqref{uniform-laplace} has following properties
\begin{enumerate}
\item $A$ is symmetric, namely
$$
(Au, v)_F=(u,Av)_F, ~~~\hbox{where}~~~ (u,v)_F=\sum_{i,j=1}^nu_{i,j}v_{i,j}.
$$
\item  $(Av, v)_F>0, ~~~\hbox{if}~~~v\neq 0.$
\item  $\rho(A)\le 8$.
\end{enumerate}
\end{proposition}

\subsection{Finite element methods}
We consider two finite elements: continuous linear element and bilinear element. These two finite element methods find $u_h\in V_h$ such that
$$
(\nabla u_h, \nabla v_h)=(f, v_h),\ \forall v_h\in V_h.
$$
The basis functions $\phi_{i,j}\in V_h$ such that
\begin{equation*}
\phi_{i,j}(x_k,y_l)=\delta_{(i,j), (k,l)}=
\left\{
\begin{array}{rl}
1& (i,j)= (k,l)\\
0& (i,j)\neq (k,l)
\end{array}
\right.
\end{equation*}
The above formulation can be written as
$$
Au=f,
$$
with $A_{(j-1)n+i, (l-1)n+k}=(\nabla \phi_{kl}, \nabla \phi_{ij})$ and $f_{(j-1)n+i, (l-1)n+k}=(f, \phi_{ij})$.
\begin{enumerate}
\item Continuous linear finite element discretization of
\eqref{laplace} on the left triangulation in Fig \ref{fig:2dpartition}. The discrete space for linear finite element is
$$
V_h=\{v_h: v_h|_K\in P_1(K) \text{ and } v_h \text{ is continuous on each interior edge}\}.
$$
The basis functions according to the three vertice on the element below are $1-x$, $y$ and $x-y$.
%Here we need to notice that, $n_\ell = 2^{k_\ell} + 1$ for general PDEs grid
%with the above boundary condition. For general images, we can take them as
%discrete functions on grid with size $n_\ell = 2^{k_\ell}m$ with small $m = 1,3,\cdots$.
%Then generally speaking, the coarse grid size is $n_{\ell+1} = \frac{n_\ell}{2}=2^{k\ell - 1}m$.
It is easy to verify that the formulation for the linear element method is exactly the same as the five-point finite difference scheme, namely
\begin{equation}
  \label{2d-fe0}
4u_{ij}-(u_{i+1,j}+u_{i-1,j}+u_{i,j+1}+u_{i,j-1})=f_{i,j},~~u_{i,j}=0~~\hbox{if}~~i ~~\hbox{or}~~ j\in \{0, n+1\}.
\end{equation}
The linear finite element method solves
\begin{equation}
\label{laplace}
Au=f
\end{equation}
with
\begin{equation}
  \label{2d-fd}
A=tridiag (-I, B, -I), \quad B=tridiag (-1, 4, -1).
\end{equation}
The matrix $A$ is a block tridiagonal matrix.

\begin{figure}[!ht]
\begin{center}
\begin{tikzpicture}[xscale=2,yscale=2]
\tikzstyle{every node}=[font=\Large,scale=0.9]
\draw[-] (0,0) -- (1,0);
\draw[-] (0,0) -- (1,1);
\draw[-] (1,0) -- (1,1);
\node[below] at (0,0) {$(0,0)$};
\node[below] at (1,0) {$(1,0)$};
\node[above] at (1,1) {$(1,1)$};
\draw[-] (3,0) -- (4,0);
\draw[-] (3,0) -- (3,1);
\draw[-] (4,0) -- (4,1);
\draw[-] (3,1) -- (4,1);
\node[below] at (3,0) {$(0,0)$};
\node[below] at (4,0) {$(1,0)$};
\node[above] at (3,1) {$(1,1)$};
\node[above] at (4,1) {$(0,1)$};
\end{tikzpicture}
\end{center}
\end{figure}
\item Continuous bilinear finite element discretization of
\eqref{laplace} on the right mesh in
Fig. \ref{fig:2dpartition}. The discrete space for linear finite element is
$$
V_h=\{v_h: v_h|_K\in \{1,\ x,\ y,\ xy \} \text{ and } v_h \text{ is continuous on each interior edge}\}.
$$
The basis functions according to the four vertice on the element above
are $\frac{1}{4}(1-x)(1-y)$, $\frac{1}{4}(1+x)(1-y)$, $\frac{1}{4}(1-x)(1+y)$ and $\frac{1}{4}(1+x)(1+y)$.
And we have
\begin{equation}
  \label{2d-fe1}
8u_{ij}-(u_{i+1,j}+u_{i-1,j}+u_{i,j+1}+u_{i,j-1}+u_{i+1,j+1}+u_{i-1,j-1}+u_{i-1,j+1}+u_{i+1,j-1})=f_{i,j},
\end{equation}
and
$$
u_{i,j}=0~~\hbox{if}~~i ~~\hbox{or}~~ j\in \{0, n+1\}.
$$

The bilinear finite element method solves
\begin{equation}
\label{laplace-h}
Au=f
\end{equation}
with
%Here we need to notice that, $n_\ell = 2^{k_\ell} + 1$ for general PDEs grid
%with the above boundary condition. For general images, we can take them as
%discrete functions on grid with size $n_\ell = 2^{k_\ell}m$ with small $m = 1,3,\cdots$.
%Then generally speaking, the coarse grid size is $n_{\ell+1} = \frac{n_\ell}{2}=2^{k\ell - 1}m$.
\begin{equation}\label{uniformbilinear-laplace}
\tilde A=
\begin{pmatrix}
\tilde A_1&\tilde A_2&\\
\tilde A_2&\tilde A_1&\tilde A_2&\\
     &\ddots&\ddots&\ddots&\\
     &     &\tilde A_2&\tilde A_1&\tilde A_2\\
     &     &     &\tilde A_2&\tilde A_1
\end{pmatrix},
\end{equation}
\begin{equation*}
\tilde A_1=
\begin{pmatrix}
8&-1&\\
-1&8&-1&\\
     &\ddots&\ddots&\ddots&\\
     &     &-1&8&-1\\
     &     &     &-1&8
\end{pmatrix},
\tilde A_2=
\begin{pmatrix}
-1&-1&\\
-1&-1&-1&\\
     &\ddots&\ddots&\ddots&\\
     &     &-1&-1&-1\\
     &     &     &-1&-1
\end{pmatrix},
\end{equation*}
\end{enumerate}


In CMSC250, there are only about two ways in which you might prove a number $\sqrt[n]{q}$ irrational for some $n \in \mathbb{Z}^{>0}$ and some $q \in \mathbb{Z}^{>1}$:

\begin{itemize}
    \item A Euclidean argument. That is, you prove some lemma of the form ``if $q \mid a^n$ then $q \mid a$''.
    \item Use the unique prime factorization theorem and compare the exponents on prime factors modulo $n$.
\end{itemize}

There are two main problems with both approaches:
\begin{itemize}
    \item The Euclidean argument scales poorly to large values of $n$ and large values of $q$, especially when proving a statement like this by contrapositive, where the naive thing to do is to prove separately each remainder case out of the quotient remainder theorem.
    \item Proofs by UPFT are hard to write correctly, are a mess of indices, and are difficult to grade and give feedback. Especially if you have an equation of the form
    \[2p_1^{2e_1}\cdots p_n^{2e_n} = q_1^{2f_1}\cdots q_k^{2f_k},\]
    it is tricky to talk about the exponent of the $2$ in the unique prime factorization of the number on the left hand side.
\end{itemize}

Here in this handout I outline better tools (Bezout's Lemma, and the $p \mid ab$ lemma) which completely resolve these two issues. Proofs using these two methods both scale well to large numbers and are not too hard to write correctly.

\section{Preliminary Definitions and Results}

\begin{definition}
 Given integers $a$ and $b$, we say that $a$ divides $b$ (and we write $a \mid b$) if there exists an integer $k$ such that $a \cdot k = b$.
\end{definition}

\begin{definition}
 Given integers $a$ and $b$ and positive integer $m$ we say that $a$ is congruent to $b$ modulo $m$ (and we write $a \equiv b \pmod{m}$) if $m \mid (a - b)$.
\end{definition}

Here in this document $\mathbb{Z}$ denotes the integers and $\mathbb{N}$ denotes the integers greater than $0$. \textbf{Note that this is different from standard CMSC250 convention!} I'll also abuse the following fact a lot, which is often taken as a given (axiomatic) property of the natural numbers:

\begin{tcolorbox}
 Let $S \subseteq \mathbb{N}$. Then $S$ has a least element. That is, there is an $s \in S$ such that for all $t \in S$, $s \leq t$.
\end{tcolorbox}

This property is called the \textbf{well ordering property} of $\mathbb{N}$. As an aside, note that this property that $T$ is equivalent to the principle of mathematical induction, as proven in the box below.
\begin{tcolorbox}
 To fill in later. It's not too important.
\end{tcolorbox}

\begin{definition}
 Let $a$ and $b$ be \textbf{non-zero} integers. Then the greatest common denominator of $a$ and $b$ (we write $\gcd(a, b)$) is the largest number $g$ which divides $a$ and $b$.
\end{definition}

\begin{example} Here are some simple examples illustrating the gcd.
    \begin{tcolorbox}
      The greatest common denominator of $12$ and $8$ is $4$. \\ For any integer $n$, the greatest common denominator of $n$ and $n + 1$ is $1$. How do we show this? Suppose $a \geq 2$, $a \mid n$ (that is, $ak = n$ for some $k \in \mathbb{Z}$). Then $ak + 1 = n + 1$, and by the quotient remainder theorem it follows that $a$ does not divide $n + 1$. We've deduced that no divisor of $a$ greater than $2$ can also divide $n + 1$. Hence the \textit{greatest} common divisor of $n$ and $n + 1$ is $1$.
    \end{tcolorbox}
\end{example}

\begin{definition}
 A prime number $p$ is a natural number greater than $1$ with the property that the only natural numbers which divide $p$ are $1$ and $p$.
\end{definition}

The following lemma is the key to most irrationality applications. It says that if $p$ is prime and $p \nmid a$, then $p$ and $a$ have no commmon factors other than $1$. Intuitively and formally, this is clear because $p$ only has $2$ factors, $1$ and $p$.
\begin{lemma}
    Suppose $p$ is a prime and $a$ is an integer with $p \nmid a$. Then $\gcd(p, a) = 1$.
\end{lemma}
\begin{proof}
The only divisors of $p$ are $1$ and $p$. The condition $p \nmid a$ shows that $p$ is not a divisor of $a$. Hence $1$ must be the \textit{greatest} common divisor of $p$ and $a$, as desired.
\end{proof}

We also use the following lemma a lot, which is just a way to bound numbers by divisors.
\begin{lemma}
    Suppose $a$ and $b$ are integers, $b$ non-zero, with $a \mid b$. Then $|a| \leq |b|$. In particular, if $a$ and $b$ are positive, then $a \leq b$.
\end{lemma}
\begin{proof}
    $a \mid b$ implies that there is an integer $k$ such that $a\cdot k = b$. Since $b \neq 0$ it follows that $k \neq 0$. So $|k| \geq 1$. It follows that
    \[|a| = |a| \cdot 1 \leq |a||k| = |ak| = |b|,\] as desired.
\end{proof}

\section{Bezout's Lemma, and the $p \mid ab$ Lemma}

\begin{definition}
 Suppose $m$ and $n$ are integers. An \textbf{integer linear combination} of $m$ and $n$ is any integer of the form
 \[mx + ny\] where $x, y \in \mathbb{Z}$.
\end{definition}

\begin{example}
    $1$ is an integer linear combination of $13$ and $9$, since
    \[1 = 91 - 90 = 13(7) - 9(10).\]
\end{example}

Given non-zero integers $m$ and $n$, let \[S = \{mx + ny \mid x, y \in \mathbb{Z}, mx + ny > 0\}.\]
In English, $S$ is all the positive integer linear combinations. Since $S \subseteq \mathbb{N}$, it has a least element $\ell$. Bezout's Lemma states that $\ell = \gcd(m, n)$, a surprising result to take in at first.

\begin{lemma}
    Let $m$ and $n$ be non-zero integers, and let $g = \gcd(m, n)$. Then $g$ is the smallest positive integer linear combination of $m$ and $n$, that is, there exist some integers $x_0$, $y_0$ where
    \[g = mx_0 + ny_0,\] and moreover $g$ is the smallest number where such $x_0$ and $y_0$ exist.
\end{lemma}

\begin{proof}
The proof strategy is as follows: define $\ell$ as earlier, that is, as the least positive integer linear combination of $m$ and $n$ (that is, the smallest natural number where $mx_0 + ny_0 = \ell$ for some integer $x_0$ and $y_0$). We prove two intermediate results:
\begin{enumerate}
    \item $\ell \mid n$ and $\ell \mid n$.
    \item $g \mid \ell$
\end{enumerate}

These two things imply $\ell = g$, for the first step tells us $\ell$ is a common divisor of $m$ and $n$, and the second step tells us that the \textit{greatest} common divisor divides $\ell$, hence is less than or equal to $\ell$. But $g$ is the greatest common divisor, so $g = \ell$.

Let's prove these two steps. Suppose (by contradiction), $\ell \nmid m$. Then by the quotient remainder theorem there exist integers $q$ and $r$ where $0 < r < \ell$ such that $m = q\ell + r$. Then we have
\[r = m - \ell q = m - (mx_0 + ny_0)\ell = m(1 - x_0\ell) + n(b_0\ell).\]
Both expressions in the parentheses in the right hand side are integers by closure under addition and multiplication. So we have expressed $r > 0$, strictly less than $\ell$, as an integer linear combination of $m$ and $n$. But $\ell$ was defined as the \textbf{least} integer linear combination! This is a contradiction, so $\ell \mid m$. Similarly, $\ell \mid n$.

Let $g = \gcd(m, n)$. There exist $k_1, k_2 \in \mathbb{Z}$ such that $gk_1 = m$ and $gk_2 = n$. Then using the $x_0$ and $y_0$ as above we get
\[\ell = mx_0 + ny_0 = g(k_1x_0 + k_2y_0),\]
which shows that $g \mid \ell$, as desired.
\end{proof}

What immediately follows is a very nice lemma about a prime dividing a product.

\begin{lemma}
    Let $p$ be a prime and $a$ and $b$ be integers. If $p \mid ab$ then $p \mid a$ or $p \mid b$.
\end{lemma}
\begin{proof}
    To prove this statement we need to show that the statement
    \[(p \mid a) \lor (p \mid b).\]

    is always true. To prove this or statement, there are two cases. Either $(p \mid a)$, which means the or statement is true. So there is nothing to do here. So suppose $p \nmid a$. We need to show that $(p \mid b)$ to show that the or statement is true.

    If $p \nmid a$ then $\gcd(a, p) = 1$. So there exist integers $x, y$ such that $ax + py = 1$. Multiplying this equation by $b$, we get $abx + pby = b$. But since $p \mid ab$ there exists an integer $k$ such that $pk = ab$. So substituting this into the above equation we get
    \[pkx + pby = p(kx + by) = b,\] and since $kx + by$ is an integer it follows that $p \mid b$, as desired.
\end{proof}

\section{Immediate applications to irrationality proofs}

Using the $p \mid ab$ lemma we can immediately deduce the Euclidean lemma for any prime and any exponent, as highlighted below.

\begin{lemma}
Suppose $p$ is a prime and $a$ is an integer with $p \mid a^2$. Then $p \mid a$.
\end{lemma}
I'll give two different proofs of this fact, one using Bezout's Lemma and one using the $p \mid ab$ lemma.
\begin{proof}
    By contrapositive. Suppose $p \nmid a$. Then $\gcd(p, a) = 1$, so there exist integers $x$ and $y$ such that $px + ay = 1$. Squaring this equation we get
    \[p^2x^2 + 2pxay + a^2y^2 = p(px^2 + 2pxy) + a^2(y^2) = 1.\]
    This shows that $1$ is an integer linear combination of $p$ and $a^2$. Since it is the smallest possible integer linear combination we can conclude that $\gcd(a^2, p) = 1$. That is,
    $p \nmid a^2$, as desired.
\end{proof}
Isn't that neat? No nightmare of indices. No dealing with $p-1$ cases (especially as $p$ gets very large). Just square the equation $ax + py = 1$ to get what you want. The second proof is even easier.

\begin{proof}
    Suppose $p \mid a^2$. Then $p \mid a \cdot a$. So $p \mid a$ or $p \mid a$, which means that $p \mid a$, as desired.
\end{proof}

In fact, this proof is so nice that it leads the way to an easy proof by induction.

\begin{lemma} \label{gen_euclid}
Suppose that $p$ is a prime and $a$ is an integer with $p \mid a^n$. Then $p \mid a$.
\end{lemma}

\begin{proof}
We'll prove this by induction on $n$. For the case $n = 1$, clearly $p \mid a$ implies $p \mid a$. Now assume that for $n \geq 1$ that $p \mid a^n$ implies $p \mid a$. Suppose $p \mid a^{n + 1}$. Then $p \mid a$ or $p \mid a^n$. Clearly if $p \mid a$ we would be done. Even if we suppose not, then $p \mid a^n$ and our inductive step gives us $p \mid a$ anyway. So the inductive step is complete and we are done.
\end{proof}

The next lemma shows that mod a prime, any non-zero integer has an inverse modulo $p$.

\begin{lemma} \label{inverse}
Suppose $p$ is any prime, and $k$ is not a multiple of $p$. Then there exists an integer $\ell$ such that
\[k \ell \equiv 1 \pmod{p}.\]
\end{lemma}

\begin{proof}
Since $k$ is not a multiple of $p$, it follows that $\gcd(k, p) = 1$. So there exist integers $x$ and $y$ such that $kx - py = 1$, or $kx - 1 = py$. This implies that $p \mid (kx - 1)$, or $kx \equiv 1 \pmod{p}$, as desired.
\end{proof}

\begin{example}
We'll find all the solutions $n$ to the equation $5n \equiv 3 \pmod{7}$.
\begin{tcolorbox}
 By Lemma \ref{inverse} there exists some $k$ such that $5k \equiv 1 \pmod{7}$. After some searching we see that $k = 3$ is an integer such that $5k \equiv 1 \pmod{7}$. Hence
 \[5kn \equiv 3k \pmod{7} \implies n \equiv 3(3) \equiv 2 \pmod{7}.\]

 So all such $n$ must be of the form $7k + 2$ for some integer $k$. Conversely, if $n = 7k + 2$, then
 \[5n = 35k + 10 = 7(5k + 1) + 3,\] or $5n \equiv 3 \pmod{7}$. This shows that all solutions $n$ to the equation are the integers $7k + 2$ where $k$ is \textbf{any} integer.
\end{tcolorbox}
\end{example}

\section{Exercises}
 \begin{enumerate}
   \item \input{Ch6/1_ex/problem_1}
   \item \input{Ch6/1_ex/problem_2}
   \item \input{Ch6/1_ex/problem_3}
   \item \input{Ch6/1_ex/problem_4}
   \item \input{Ch6/1_ex/problem_5}
   \item \input{Ch6/1_ex/problem_6}
   \item \input{Ch6/1_ex/problem_7}
   % add more problem files here
 \end{enumerate}


\section{Abstract}
A model for the solar system using ordinary differential equations was developed with the Velocity Verlet method. The earth sun system, earth sun jupiter system and the whole solar system was examined.

\section{Introduction}
In this project I will develop a model simulating the solar system using a set of ordinary differential equations based on Newton's laws of motion:

$\frac{d^2x}{dt^2} = \frac{F_{G,x}}{M_{earth}}$\\

$\frac{d^2y}{dt^2} = \frac{F_{G,y}}{M_{earth}}$\\

$\frac{d^2y}{dt^2} = \frac{F_{G,z}}{M_{earth}}$\\

$F_{G,x}$ = $\frac{-GM_{\odot}M_{earth}}{r^2}cos\theta sin\psi$= $\frac{-GM_{\odot}M_{earth}}{r^3}x$\\

and\\

$F_{G,y}$ = $\frac{-GM_{\odot}M_{earth}}{r^2}sin\theta sin\psi$= $\frac{-GM_{\odot}M_{earth}}{r^3}y$\\

similarly\\

$F_{G,z}$ = $\frac{-GM_{\odot}M_{earth}}{r^2}cos\psi$= $\frac{-GM_{\odot}M_{earth}}{r^3}z$\\

Numerical methods will be used to solve these equations. An initial test for various methods include the Forward Euler, Central Euler and the velocity Verlet methods. For the complete system the velocity Verlet method will be used.\\

In the process of ending up with a solar system I will start by considering basic interaction between the Earth and Sun, before expanding the model to a 3-body problem involving Jupiter, and finally including all planets of the solar system.

\section{Methods}
%*Formalism/methods: Discussion of the methods used and their basis/suitability. Total number of possible points 20*
\subsection{Forward Euler}
The Euler method tries to calculate an unknown curve starting in a given point satisfying a certain differential equation.\\
\begin{equation}
\frac{dy}{dt}=f(t,y)
\label{Eq:diffeq}
\end{equation}
, where $y(t_0)=y_0$ and $y'(t_0) = v_0$\\

Forward Euler tries to approximate a function curve by a polynomial. The derivative on the left hand side is approximated with a forward step using a Taylor expansion:\\
$y(t+\Delta t) = y(t) + y'(t)\Delta t + O(\Delta t^2)$\\
Neglecting 2nd order terms and higher and solving for y'(t):\\
$y'(t) = \frac{y(t+\Delta t)-y(t)}{\Delta t}$\\
Inserting into equation~\ref{Eq:diffeq} and solving for $y(t+\Delta t)$:\\
$y(t+\Delta t) = y(t)+\Delta t f(t,y(t))$\\

Let $t_n = t_0+n\Delta t$\\

The total solution can be found by iterating from n=0 to n:\\
\begin{equation}
y_{n+1} = y_n+f(t_n, y_n)\Delta t
\label{Eq:Forward_Euler}
\end{equation}\\

\subsection{Central Euler}
Central Euler method approximates the derivative $\frac{dy}{dt}$ with a central approximation:
$\frac{dy}{dt} \approx \frac{y(t+\Delta t) -y(t-\Delta t)}{2\Delta t}$\\

Inserting into equation~\ref{Eq:diffeq} and solving for $y(t+\Delta t)$:\\

$y(t+\Delta t) = y(t-\Delta t)+2\Delta t f(t,y(t))$\\

Let $t_n = t_0+n\Delta t$.\\

The total solution can be found by iterating from n=0 to n:\\
\begin{equation}
y_{n+1} = y_{n-1}+2f(t_n, y_n)\Delta t
\label{Eq:Central_Euler}
\end{equation}\\

Computing $y_1$ at the start of Central Euler iteration is problematic, because it depends on position $y_{-1}$, which is unknown. Therefore in the first step we use the Forward Euler method. The error of the first time step is then of order $O(\Delta t^2)$. This is not considered a problem because the error of the initial value is small compared to the total error accumulated at time $t=t_n$

\subsection{Velocity Verlet}
The Velocity Verlet method uses a central method to approximate the 2nd order derivative. Velocity Verlet method builds on the basic Verlet method:\\

\begin{equation}
\frac{d^2y}{dt^2}=f(t,y)
\label{Eq:diffeq_verlet}
\end{equation}
, where $y(t_0)=y_0$ and $y'(t_0) = v_0$\\

$\frac{d^2y}{dt^2} \approx \frac{y(t+\Delta t) -2y(t)+y(t-\Delta t)}{\Delta t^2}$\\

inserting into equation~\ref{Eq:diffeq_verlet} and solving for $y(t+\Delta t)$:\\

$y(t+\Delta t) = 2y(t)-y(t-\Delta t)+\Delta t^2 f(t,y(t))$\\

Let $t_n = t_0+n\Delta t$.\\

The total solution can be found by iterating from n=0 to n:\\
\begin{equation}
y_{n+1} = 2y_n - y_{n-1} + f(t_n, y_n)\Delta t^2
\label{Eq:Verlet}
\end{equation}\\

Computing $y_2$ at the start of Verlet iteration at n=1, time $t=t_1=\Delta t$, one already needs the position vector $y_1$ at time $t=t_1$. At first sight this could give problems, because the initial conditions are known only at the initial times $t_0 = 0$. However, from these the acceleration $a_0 = f(y_0, t_o)$ is known, and a suitable approximation for the first time step position can be obtained using the taylor polynomial of second degree:\\

$y_1 = y_0 +v_0\Delta t + \frac{a_0\Delta t^2}{2}\approx y(\Delta t)+O(\Delta t^3)$\\

The error of the first time step calculation then is of order $O(\Delta t^3)$. This is not considered a problem because on a simulation over a large number of time steps, the error of the first time step is negligibly small compared to the total error at time $t=t_n$.\\

The velocities are often needed to calculate physical properties of a system, like kinetic energy. The velocities are not given explicitly given by the basic Verlet method. The Velocity Verlet method incorporates velocities solving the first time step problem in the basic Verlet method by:
$y(t+\Delta t) = y(t) +v(t)\Delta t + \frac{a(t)\Delta t^2}{2}$\\

We can calculate the acceleration at the next step by using the differential equation~\ref{Eq:diffeq_verlet}\\

$a(t+\Delta t) = f((t+\Delta t), y(t+\Delta t))$\\

This acceleration is then used to calculate the velocity in the next step\\

$v(t+\Delta t) = v(t) +\frac{a(t)+a(t+\Delta t)}{2}\Delta t$\\

It can be shown that the error for the Velocity Verlet is of the same order as that of the basic Verlet. \\

Therefore the final solution set for the Velocity Verlet becomes:\\
\begin{gather}
		a_n = f(t_n, y_n)\\
        y_{n+1} = y_n + \Delta t (v_n + 0.5 a_n \Delta t) \\
        v_{n+1} = v_k + 0.5 \Delta t (a_n + f(y_{n+1}, t_{n+1}))
\end{gather}

\newpage
\section{Implementation}
For all programs, see:\\
$\href{https://github.com/larsjbro/FYS4150/tree/master/Project_3/source}{https://github.com/larsjbro/FYS4150/tree/master/Project_3/source}$
\subsection{Testing stability of algorithms}
In this section all main results come from this program found at github:\\
$\href{https://github.com/larsjbro/FYS4150/blob/master/Project_3/source/test_algorithms/test_algorithms.py}{https://github.com/larsjbro/FYS4150/blob/master/Project_3/source/test_algorithms/test_algorithms.py}$

\subsubsection{Initial velocity for circular orbit}
The centripetal force must equal the gravitational force in order to get a circular orbit:\\

$\frac{v^2}{r} = GM_{\odot} = 4\pi^2 [AU^3/Yr^2]$\\

Solving for v and using that the initial distance from the earth to the sun is 1 AU, we have that a circular orbit is obtained for:\\

$v = 2\pi$\\

\subsubsection{Stability for different time steps dt}
Earth orbit sun for different dt, stability test:\\

I can see that the Forward Euler is suspect for large step values of dt, and not even reliable compared to the other methods for the shortest time step. Large step lengths gave an unstable solution as shown in Figures 1, 2 and 4.

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k4beta200ForwardEuler.png}

\caption{Earth orbit around the sun using the Forward Euler method with $dt = \frac{1}{2^4}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_4}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k5beta200ForwardEuler.png}

\caption{Earth orbit around the sun using the Forward Euler method with $dt = \frac{1}{2^5}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_5}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k8beta200ForwardEuler.png}

\caption{Earth orbit around the sun using the Forward Euler method with $dt = \frac{1}{2^8}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k4beta200CentralEuler.png}

\caption{Earth orbit around the sun using the Central Euler method with $dt = \frac{1}{2^4}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k5beta200CentralEuler.png}

\caption{Earth orbit around the sun using the Central Euler method with $dt = \frac{1}{2^5}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k8beta200CentralEuler.png}

\caption{Earth orbit around the sun using the Central Euler method with $dt = \frac{1}{2^8}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k4beta200VelocityVerlet.png}

\caption{Earth orbit around the sun using the Velocity Verlet method with $dt = \frac{1}{2^4}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k5beta200VelocityVerlet.png}

\caption{Earth orbit around the sun using the Velocity Verlet method with $dt = \frac{1}{2^5}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/stability_test_k8beta200VelocityVerlet.png}

\caption{Earth orbit around the sun using the Velocity Verlet method with $dt = \frac{1}{2^8}$}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\subsubsection{Conservation of kinetic and potential energies and angular momentum}
Angular momentum for a body in motion under a central force F is defined as follows:\\
$\frac{dL}{dt}=r \times F$\\
, where L is the angular momentum, r is the radius and F is the central force. For a circular orbit, we must assume that the central force and the radius are parallel, which gives a change of angular momentum with respect to time being 0. Therefore the angular momentum must be conserved for circular motion. \\
\\
$E_p = -GM/r$\\
$E_k = 1/2mv^2$\\,
\\
$E_{tot} = constant$\\

Since the radius with respect to time is not changing as well, we have that both the kinetic energy and the potential energy must remain constant.

Now checking for a circular orbits, that the angular momentum, kinetic and potential energies are conserved:\\

For every position we have the velocities. These are decomposed in each direction, x, y and z. The total velocity squared is the sum of the squared velocities in each direction. This is directly linked to kinetic energy. Large maximum differences of the velocities squared for each method, indicates unstable solutions.\\

Low standard deviation for L(angular momentum), $E_k$(kinetic energy ~$v^2$) and $E_p$(potential energy ~$1/r$) would mean that the system is stable. \\

For both CentralEuler and VelocityVerlet, the solutions are decent, with the Velocity Verlet being the most stable. The ForwardEuler is the most unstable, also shown by figures 1, 2 and 3, showing that this method does not approximate an orbit, even when the two other methods do. \\

True values are shown in Table ~\ref{tab:Stability_methods_tabular}\\

\FloatBarrier
\begin{table}
\begin{tabular}{llrrr}
\toprule
Metric & Unit & Forward Euler & Central Euler & Velocity Verlet \\
\midrule
max diff($v^2$) & $[AU^2/Yr^2]$  & 0.08430  & 0.02381 & 0.00029 \\
std($v^2$) &$[AU^2/Yr^2]$ & 4.86822 & 0.02656 & 0.00840 \\
std(L/$M_{earth}$) &$[AU^2/Yr]$ & 1.52866 &  0.00134 & 1.12180e-14 \\
std(1/r) & $[1/AU]$ & 0.11873 & 0.00034 & 0.00011 \\
\bottomrule
\end{tabular}
\caption{Stability of methods}
\label{tab:Stability_methods_tabular}
\end{table}
\FloatBarrier

\subsubsection{Timing differences between methods and FLOPs}
Floating point operations are as follows:\\
Central Euler has 2 additions and 2 multiplications = 4n flops\\
Forward Euler has 2 additions and 1 multiplication = 3n flops\\
Velocity Verlet has 3 additions and 3 multiplications = 6n flops\\

Table ~\ref{tab:Timing_differences_FLOPS} shows the runtime differences for each method for varying time steps. The Forward Euler was the fastest method, but clearly this was compensated with bad precision. The Central Euler method had much better precision, and was just slightly slower. The VelocityVerlet was the most precise, and also the slowest. The runtimes can be directly linked to the number of flops as calculated above.

\FloatBarrier
\begin{table}

\begin{tabular}{lrrrr}
\toprule
{} &  k &  CentralEuler &  ForwardEuler &  VelocityVerlet \\
\midrule
0 &  4 &      0.022213 &      0.019871 &        0.048964 \\
1 &  5 &      0.065837 &      0.048205 &        0.128394 \\
2 &  6 &      0.109570 &      0.101608 &        0.188569 \\
3 &  7 &      0.267016 &      0.214553 &        0.437121 \\
4 &  8 &      0.404209 &      0.396895 &        0.856961 \\
\bottomrule

\end{tabular}

\caption{Runtime in seconds for different sampling time dt. $dt = 1/(2^k)$}
\label{tab:Timing_differences_FLOPS}
\end{table}
\FloatBarrier

%*Code/Implementations/test: Readability of code, implementation, testing and discussion of benchmarks. Total number of possible points 20*

\section{Results}
%*Analysis: of results and the effectiveness of their selection and presentation. Are the results well understood and discussed? Total number of possible points: 20*

\subsection{Escape velocities}
\subsubsection{Analytic solution}
Analytically the escape velocity should occur when the kinetic energy is great enough for the object to escape the gravitational force. We have that:\\

$v_{crit} = E_k>E_p$\\

$\frac{M_{earth}v^2}{2} = \frac{GM_{\odot}M_{earth}}{r}$\\

$\implies v_{crit}=\sqrt{\frac{2M_{\odot}G}{r}}$\\

using $GM_{\odot} = 4\pi^2 [AU^3/Yr^2]$\\

$v_{crit} = 2\sqrt{2}\pi [AU/yr]$\\

This means that analytically an initial velocity larger than $2\sqrt{2}\pi [AU/yr]$ would result in the Earth escaping from the sun.\\

\subsubsection{Numerical solution}
If the initial velocity is smaller than $2\pi$ I get the figure ~\ref{fig:Earth_sun_jupiter_v0_pi} showing that small velocities also do not result a circular orbit:

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/escape_velocity_test_k9beta200v010VelocityVerlet.png}

\caption{Initial velocity = $\pi$ [AU/Yr]}
\label{fig:Earth_sun_jupiter_v0_pi}
\end{figure}
\FloatBarrier

Figure ~\ref{fig:Earth_sun_jupiter_v0_2.6_pi} shows that an initial velocity of 2.6$\pi$ gives an elliptical orbit, since we are not at escape velocities.

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/escape_velocity_test_k9beta200v026VelocityVerlet.png}

\caption{Initial velocity = $2.6\pi$ [AU/Yr]}
\label{fig:Earth_sun_jupiter_v0_2.6_pi}
\end{figure}
\FloatBarrier

Figure ~\ref{fig:Earth_sun_jupiter_v0_3.0_pi} shows that an initial velocity of 3.0$\pi$ escapes the sun.

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/escape_velocity_test_k9beta200v030VelocityVerlet.png}

\caption{Initial velocity = $3.0\pi$ [AU/Yr]}
\label{fig:Earth_sun_jupiter_v0_3.0_pi}
\end{figure}
\FloatBarrier

\subsubsection{Varying beta values}

Increasing the beta value means reducing the gravitational force. Figure ~\ref{fig:Earth_sun_jupiter_beta_230_v0_2.0_pi}, ~\ref{fig:Earth_sun_jupiter_beta_270_v0_2.0_pi} and ~\ref{fig:Earth_sun_jupiter_beta_300_v0_2.0_pi} shows that a higher beta value results in a larger radius for the orbit. When beta approaches 3 the orbit of the Earth will spiral out and escape the sun.

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/escape_velocity_test_k9beta230v020VelocityVerlet.png}

\caption{Initial velocity = $2.0\pi$ and beta = 2.3 [AU/Yr]}
\label{fig:Earth_sun_jupiter_beta_230_v0_2.0_pi}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/escape_velocity_test_k9beta270v020VelocityVerlet.png}

\caption{Initial velocity = $2.0\pi$ and beta = 2.7 [AU/Yr]}
\label{fig:Earth_sun_jupiter_beta_270_v0_2.0_pi}
\end{figure}
\FloatBarrier

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_algorithms/escape_velocity_test_k9beta300v020VelocityVerlet.png}

\caption{Initial velocity = $2.0\pi$ and beta = 3.0 [AU/Yr]}
\label{fig:Earth_sun_jupiter_beta_300_v0_2.0_pi}
\end{figure}
\FloatBarrier

\subsection{3-body-problem}

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{test_three_body_problem/stability_2d_test_k70beta200VelocityVerlet.png}

\caption{The difference in discretization shows how vital the differences can be}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

Figure ~\ref{fig:Earth_jupiter} shows the Earth, Sun and Jupiter in one plot.

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{final_model/stability_2d_test_k8beta200VelocityVerlet.png}

\caption{Earth, Sun and Jupiter in one plot}
\label{fig:Earth_jupiter}
\end{figure}
\FloatBarrier

\subsection{Final Model}
Below is a final plot of the solar system:\\

\FloatBarrier
\begin{figure}[!ht]
\centering
\FloatBarrier
\includegraphics[width=0.70\textwidth]{final_model/stability_3d_test_k8beta200VelocityVerlet.png}

\caption{Final model of the solar system}
\label{fig:Earth_orbit_sun_Forward_Euler_k_8}
\end{figure}
\FloatBarrier

\subsection{The perihelion precession of Mercury}
I was not able to finish the general relativity problem trying to figure out if the observed perihelion precession of Mercury quite in time, but I have given a link to the program I have made here:
$\href{https://github.com/larsjbro/FYS4150/blob/master/Project_3/source/test_perihelion/test_perihelion.py}{https://github.com/larsjbro/FYS4150/blob/master/Project_3/source/test_perihelion/test_perihelion.py}$

\section{Conclusion}
To start with the test for numerical method of choice showed that the Velocity Verlet method was the best of the three. I could see that the method of choice plays a huge role in the search for reliable results. Also I found that the runtimes depended very much on number of floating point operations. Nevertheless the precision needs to be at a minimum regardless of speed. The sun is huge, and it took a lot of effort to neutralize the gravitational force\\

%*Conclusions, discussions and critical comments: on what was learned about the method used and on the results obtained. Possible directions and future improvements? Total number of possible points: 10*

\section{References}
\subsection{Internet sources}

-$\href{https://github.com/CompPhysics/ComputationalPhysics/blob/gh-pages/doc/Lectures/lectures2015.pdf}{https://github.com/CompPhysics/ComputationalPhysics/blob/gh-pages/doc/Lectures/lectures2015.pdf}$, October 2017\\

$\href{https://ssd.jpl.nasa.gov/horizons.cgi}{https://ssd.jpl.nasa.gov/horizons.cgi}$, NASA, October 2017, used to generate ephemerides\\

%\FloatBarrier
%\begin{figure}[!ht]
%\centering
%\FloatBarrier
%\includegraphics[width=0.45\textwidth]{eigenvector_rho29n128omega500.png}
%
%\caption{Normalized energy for the three lowest eigenvalues for repulsive Coulomb interaction with n=128 and $\omega_r$=5}
%\label{fig:Eigenvalue_states_n_320_omega_500}
%\end{figure}
%\FloatBarrier


\chapter{Permutations and Sorting}

In the previous chapter, we talked about permutations. If you have a list
of three letters, like $[a, b, c, d]$, you can rearrange them in $4!$
ways:\index{permutations}

\begin{tabular}{c c c c c c}
  a,b,c,d & a,b,d,c & a, d, b, c & a, d, c, b & a, c, b, d & a, c, d, b \\
  b,a,c,d & b,a,d,c & b, d, a, c & b, d, c, a & b, c, a, d & b, c, d, a \\
  c,b,a,d & c,b,d,a & c, d, b, a & c, d, a, b & c, a, b, d & c, a, d, b \\
  d,b,c,a & d,b,a,c & d, a, b, c & d, a, c, b & d, c, b, a & d, c, a, b
\end{tabular}
% KA: https://www.khanacademy.org/math/precalculus/x9e81a4f98389efdf:prob-comb/x9e81a4f98389efdf:combinations/v/combination-formula

You can make Python generate all the permutations for you:

\begin{Verbatim}
from itertools import permutations
all_permutations = permutations(('a', 'b', 'c', 'd'))
for p in all_permutations:
    print(p)
\end{Verbatim}
% KA: https://www.khanacademy.org/math/precalculus/x9e81a4f98389efdf:prob-comb/x9e81a4f98389efdf:combinations/v/introduction-to-combinations

\section{Notation}

How do we define or write down a single permutation? You could say
something like ``Swap the first and second items and swap the third
and fourth items.'' However, that gets pretty difficult to read. So we
usually write a permutation as two lines: the first line is before
permutation and the second line is after.  Like this:

$$\begin{pmatrix}
  1 & 2 & 3 & 4 \\
  2 & 1 & 4 & 3
\end{pmatrix}$$

And can we assign permutations to variables. For example, if we wanted the
variable $A$ to represent ``swapping the first and second item'', we
would write this:

$$A = \begin{pmatrix}
  1 & 2 & 3 & 4 \\
  2 & 1 & 3 & 4
\end{pmatrix}$$

And if we wanted $B$ to represent ``swapping the third and fourth item'', we would write:

$$B = \begin{pmatrix}
  1 & 2 & 3 & 4 \\
  1 & 2 & 4 & 3
\end{pmatrix}$$

Now, we can \textit{compose} permutations together. For example, we might say:\index{permutations!composing}

$$B \circ A = \begin{pmatrix}
  1 & 2 & 3 & 4 \\
  2 & 1 & 4 & 3
\end{pmatrix}$$

That is, if we have the list $[a, b, c, d]$ and we apply permutation $A$ and then permutation $B$, we get $[b, a, d, c]$.

\textbf{Important:} Note that permutations are applied from right to
left. $B \circ A$ means ``Applying $A$ and then $B$.''  Why does this matter?
Permutations are not necessarily commutative. That is, if you have two
permutations $S$ and $T$, $S \circ T$ is not always the same as $T
\circ S$.

Also, note that ``don't change anything'' is a permutation. We call it
\textit{the identity permutation}. If you have four items, the identity
permutation would be written:\index{permutations!identity permutation}

$$I = \begin{pmatrix}
  1 & 2 & 3 & 4 \\
  1 & 2 & 3 & 4
\end{pmatrix}$$

(We use capital ``I'' for the identity.)

\subsection{Challenge} Find an example of two permutations $S$ and $T$ such that $S \circ T$ does not equal $T \circ S$.

\section{Sorting in Python}

One of the common forms of permutation in software is sorting.
Sorting is putting data in a particular order. For example, in Python,
if you had a list of numbers, you can sort it in ascending order like
this:\index{sorting}

\begin{Verbatim}
my_grades = [92, 87, 76, 99, 91, 93]
grades_worst_to_best = sorted(my_grades)
\end{Verbatim}

You want to sort backwards?

\begin{Verbatim}
my_grades = [92, 87, 76, 99, 91, 93]
grades_best_to_worst = sorted(my_grades, reverse=True)
\end{Verbatim}

Note that \pyfunction{sorted} makes a new list with the correct
order. If you want to sort the array in place, you can use the
\pyfunction{sort} method:
% ADD: Need to define array

\begin{Verbatim}
my_grades = [92, 87, 76, 99, 91, 93]
my_grades.sort(reverse=True)
\end{Verbatim}

\section{Inverses}

Think for a second about this permutation:

$$S = \begin{pmatrix}
  1 & 2 & 3 & 4 \\
  3 & 4 & 2 & 1
\end{pmatrix}$$

You could say this permutation shuffles a list a bit.  What is its
inverse? That is, what is the permutation that unshuffles the items
back to where they were originally?\index{permutations!inverses}

$$S^{-1} = \begin{pmatrix}
  1 & 2 & 3 & 4 \\
  4 & 3 & 1 & 2
\end{pmatrix}$$

That is, the original moved an item in the first spot to the third
spot. The inverse must move what ever was in the third spot back to
the first spot.

(Notation note: Because in multiplication, $b \times b^{-1} = 1$, we
use ``to the negative one'' to indicate inverses in lots of places.)

Mechanically, how do you find the inverse? Flip the rows, and then sort the columns using the top number:
% ADD: Inveserse should probably be defined
$$\begin{pmatrix}
  1 & 2 & 3 & 4 \\
  3 & 4 & 2 & 1
\end{pmatrix}
\text{ flip }\rightarrow
\begin{pmatrix}
  3 & 4 & 2 & 1 \\
  1 & 2 & 3 & 4
\end{pmatrix}
\text{ sort }\rightarrow
\begin{pmatrix}
  1 & 2 & 3 & 4 \\
  4 & 3 & 1 & 2
\end{pmatrix}
$$

Let's say you have two permutations $A$ and $B$. Permuting by $B$ and then $A$ would look like this:

$$C = A \circ B$$

If you know $A^{-1}$ and $B^{-1}$, what is $C^{-1}$?  You would undo-$A$ and then undo-$B$, so

$$C^{-1} = B^{-1} \circ A^{-1}$$

\section{Cycles}

Here is a permutation:

$$\begin{pmatrix}
  1 & 2 & 3 & 4 & 5 \\
  2 & 4 & 5 & 1 & 3
\end{pmatrix}$$

When this is applied, whatever is at 1 gets moved to 2, 2 gets moved
to 4, and 4 gets moved to 1.  That is a \textit{cycle}: $1 \rightarrow
2 \rightarrow 4$ and then it goes back to 1. It involves three locations, so we say
it is a \textit{3-cycle}.\index{permutations!cycles}

There is another cycle in this permutation: $3 \rightarrow 5$ and then it goes back to 3.

Because these cycles share no members, we say the cycles are \textit{disjoint}.

Every permutation can be broken down into a collection of disjoint cycles.

$$T = \begin{pmatrix}
  1 & 2 & 3 & 4 & 5 \\
  2 & 4 & 5 & 1 & 3
\end{pmatrix} = (1 \rightarrow 2 \rightarrow 4)(3 \rightarrow 5)$$

The first handy thing about this notation is that it makes it easy for
us to describe the inverse: we just run the cycles backwards:

$$T^{-1} = (4 \rightarrow 2 \rightarrow 1)(5 \rightarrow 3)$$

Starting with the list $[a, b, c, d, e]$, lets repeatedly apply the permutation $T$

\begin{tabular}{r | l}
  Initial & {\color{red} a, b,} {\color{blue} c,} {\color{red} d,} {\color{blue} e} \\
  $T$ applied & {\color{red} d, a,}  {\color{blue} e,} {\color{red} b,} {\color{blue} c}\\
  $T \circ T$ applied & {\color{red} b, d,} {\color{blue} c,} {\color{red} a,}  {\color{blue} e} \\
  $T \circ T \circ T$ applied & {\color{red} a, b,}  {\color{blue} e,} {\color{red} d,} {\color{blue} c} \\
  $T \circ T \circ T \circ T$ applied & {\color{red} d, a,} {\color{blue} c,} {\color{red}b,}  {\color{blue} e} \\
  $T \circ T \circ T \circ T \circ T$ applied & {\color{red} b, d,}  {\color{blue} e,} {\color{red} a,} {\color{blue} c} \\
  $T \circ T \circ T \circ T \circ T \circ T$ applied & {\color{red} a, b,} {\color{blue} c,} {\color{red} d,}  {\color{blue} e}\\
\end{tabular}

This permutation, results in six combinations, and then it loops back
on itself. The number of combinations is the least common multiple of
all the cycles.  In this case, there is a 3-cycle and a 2-cycle. The
least common multiple of 2 and 3 is 6.


\chapter{A Point-Classification Neural Network}
%\usepackage{amsfonts}
%\usepackage{amsmath}
%\usepackage{graphicx}
%\usepackage{listings}

\section{Introduction}
In this article we implement a simplest neural network with python 3. This implementation is mainly for learning and understanding, without a regularization term and any use of existing deep learning framework.

\subsection{What's A Point-classification Neural Network}
Suppose we draw a curve on a canvas, which separates the canvas, or every point on it, into several classes. Now, suppose we have only finite points and their classes, we want to reconstruct the curve. The reconstruction is done by a neural network, trained with the given points. We use it to predict the class of every point on the canvas, and the boundary of the different classes points is the curve we want.
\begin{center}
\includegraphics[width=0.3\textwidth]{figures/pcNN_train3.png}
\end{center}

\subsection{What Does the Neural Network Look Like}
Viewed as a function $f$, the neural network can be written as
\[f(x)=L_3\circ L_2\circ L_1(x)\]
where $x$ denotes the input vector,  $L_1$ and $L_2$ denote the two hidden layers, and $L_3$ denotes the output layer.
\begin{center}
\includegraphics[width=0.5\textwidth]{figures/pcNN_Fig2}
\end{center}

\subsection{Mathematical Expression of Each Part}
The neural network is \[f(x)=y,\quad x,y\in\mathbb{R}^2,\]
and the loss function is \[L(y)=\frac{1}{2}\|y-p\|^2=\frac{1}{2}\sum_{i=1}^2{(y_i-p_i)^2}\]
where $p$ is the true classification vector.\\
$f(x)=L_3\circ L_2\circ L_1(x)$, where
\[
\begin{split}
L_1(x)=A(xW_1+b_1)=\frac{1}{1+e^{-(xW_1+b_1)}}\in\mathbb{R}^{10},\\
x\in\mathbb{R}^{2},W_1\in\mathbb{R}^{10\times2},b_1\in\mathbb{R}^{10},
\end{split}
\]
\[
\begin{split}
L_2(x)=A(xW_2+b_2)=\frac{1}{1+e^{-(xW_2+b_2)}}\in\mathbb{R}^{10},\\
x\in\mathbb{R}^{10},W_2\in\mathbb{R}^{10\times10},b_2\in\mathbb{R}^{10},
\end{split}
\]
\[
\begin{split}
L_3(x)=A(xW_3+b_3)=\frac{1}{1+e^{-(xW_3+b_3)}}\in\mathbb{R}^{2},\\
x\in\mathbb{R}^{10},W_3\in\mathbb{R}^{2\times10},b_3\in\mathbb{R}^{2}.
\end{split}
\]
Each column of the matrix $W_i$ represents a neuron in the layer $i$. So, layer $1$ has $10$ neurons, layer $2$ has $10$ neurons, and layer $3$ has $2$ neurons.\\

\subsection{What Does A Digital Picture Look Like}
The input picture will first be transformed into a grey-scale picture. Then, the values of pixels correspond to a matrix, and we have a matrix as our picture.

\begin{center}
\includegraphics[width=0.5\textwidth]{figures/pcNN_mesh.png}
\end{center}

\section{The Implementation}
\subsection{Produce Training Data}

\subsection{Produce A Training Curve}

To give such a curve, or picture, for example, one can draw it by \emph{paint} on windows, or any other tools on different platforms.\\
If you use \emph{paint}, save it as \emph{train.png} and use the following to load it.

\begin{lstlisting}
from PIL import Image
import numpy as np
pil_im = Image.open('train.png').convert('L')
data = np.asarray(pil_im)
\end{lstlisting}

This will load the picture in \emph{pil\_im}, and transform it into an array \emph{data}. Now, each element of \emph{data} corresponds to the grey scale of a pixel.\\

\section{Generate A Training Dataset}

To get a training dataset, we generate a couple of coordinates randomly. We then generate the classification from the picture, and scale the dataset by dividing its length.

\subsection{Backward Propagation}
\subsubsection{Backward Propagation (BP) in A Layer}
Viewed as a unity, a layer will carry out the follows.
\begin{itemize}
\item Recive an input row vector $x$
\item Compute $y=xW+b$
\item Send $A(y)=\frac{1}{1+e^{-y}}$ to the next layer
\end{itemize}
In BP, We want to update $W$ and $b$ by the gradient descent method.\\
We already have a loss function $L$, and we know the update process is
\[W = W + \delta D_WL,\]
\[b = b+\delta D_bL.\]
Let's figure out how to compute $D_WL$ and $D_bL$.

\subsubsection{Backward Propagation (BP) in A Layer, Update W}
Let $W_i$ be the $i$th column of W, we have
\[\nabla_{W_i}L=D_{W_i}L=\partial_{y_i}L\nabla_{W_i}y_i\]
as a cloumn vector. We then have
\[
\begin{split}
D_WL&=
\begin{pmatrix} \nabla_{W_1}L & \dots & \nabla_{W_m}L
\end{pmatrix}\\
&=
\begin{pmatrix}
\partial_{y_1}L & \dots &\partial_{y_m}L
\end{pmatrix}
\times
\begin{pmatrix}
\nabla_{W_1}y_1 & \dots & \nabla_{W_m}y_m
\end{pmatrix}
\end{split}
\]
where the multiplication is element-wise.
We know that
\[
\nabla_{W_i}y_i=\nabla_{W_i}{(xW_i+b_i)}=x^T.
\]
Thus we have
\[
D_WL=x^T
\begin{pmatrix}
\partial_{y_1}L & \dots &\partial_{y_m}L
\end{pmatrix}.
\]

\subsubsection{Backward Propagation (BP) in A Layer, Update b}
Very similar to $W$, we update $b_i$ by
\[\partial_{b_i}L=\partial_{y_i}L\nabla_{b_i}y_i=\partial_{y_i}L.\]
So we have
\[D_bL=
\begin{pmatrix}
\partial_{y_1}L & \dots &\partial_{y_m}L
\end{pmatrix}.
\]
Now we show how to compute
\[
D_yL=\nabla_yL=
\begin{pmatrix}
\partial_{y_1}L & \dots &\partial_{y_m}L
\end{pmatrix}.
\]
Basically, one should follow the steps below.
\begin{itemize}
\item When you are in a hidden layer, just call the next layer to make it send $\nabla_yL$ to you.
\item If the prior layer asked you for $\nabla_{\hat y}L$, follow the steps below.
\end{itemize}
By the chain rule we have
\[
\nabla_{\hat y}L=D_{y}L\cdot D_xy\cdot D_{\hat y}x.
\]
We have already get $D_yL$. For the rest two, we have
\[
D_xy_i=\nabla_xy_i=\nabla_x(xW_i+b_i)=W_i^T,
\]
so we have
\[
D_xy=
\begin{pmatrix}
W_1^T \\ \dots\\ W_m^T
\end{pmatrix}
=W^T
\]

On the other hand,
\[
D_{\hat y}x_i=D_{\hat y}A({\hat y}_i)=(1-{\hat y}_i){\hat y}_i1_i,
\]
so we have
\[
D_{\hat y}x=D_{\hat y}A({\hat y}_i)=\text{diag}{((1-{\hat y}_i){\hat y}_i)}.
\]
As a result, we have
\[
\begin{split}
D_{\hat y}L&=D_yL\cdot
\begin{pmatrix}
W_1^T(1-{\hat y}_1){\hat y}_1 \\ \dots\\ W_m^T(1-{\hat y}_m){\hat y}_m
\end{pmatrix}\\
&=(D_yL\times
\begin{pmatrix}
(1-{\hat y}_1){\hat y}_1 & \dots & (1-{\hat y}_m){\hat y}_m
\end{pmatrix})
\cdot W^T\\
&=(D_yL\times(1-{\hat y}){\hat y})\cdot W^T
\end{split}.
\]

When you are in the last layer, there's no next layer to call. What we need is
\[
\nabla_yL=
\begin{pmatrix}
\partial_{y_1}L & \dots &\partial_{y_m}L
\end{pmatrix}.
\]
Our loss function is
\[
L(y)=\frac{1}{2}\sum_i{(y_i-p_i)}^2,
\]
so we have
\[
\nabla_yL=
\begin{pmatrix}
y_1-p_1 & \dots & y_n-p_n
\end{pmatrix}
\]
where $n$ is the number of classes.

We propagate only one vector $x$ in the above discussions, which is not efficient. Now we want to propagate a \emph{batch} of $x$, denote as a matrix $X$, with each row an input vector.
\begin{itemize}
    \item When update $W$ and $b$, we just sum up the gradients to get our new gradients,
\[W = W + \delta\sum_i D_WL(X_i)=W+\delta\Delta W,\]
\[b = b+\delta\sum_i D_bL(X_i)=b+\delta\Delta b.\]
where $X_i$ denotes the $i$th row of $X$, and $\delta$ is the step size.
\end{itemize}

This can be rewritten as
\[
\Delta W=X^T\nabla_yL,
\]
\[
\Delta b=1^T\nabla_yL.
\]
We now finished the backforward propagation
\[
W=W+\delta X^T\nabla_yL
\]
\[
b=b+\delta 1^T\nabla_yL
\]
\[
D_{\hat y}=(D_yL\times(1-{\hat y}){\hat y})\cdot W^T
\]
A note for batches. We can choose a batch, i.e. a subset of dataset randomly, and update them at one time. After this, we do this again, until the result is acceptable.

\subsection{The Step Size $\delta$}

Fixed step size and others like optimal step size show different performance in decreasing the loss function.
\begin{center}
\includegraphics[width=\textwidth]{figures/pcNN_a2.png}
\end{center}


\chapter{REMPS derivation}
\label{sec:remps_deriv}
\thispagestyle{empty}

\noindent This appendix provides the derivation of the REMPS solution.
We report here the formulation of the REMPS problem. For the sake of brevity we use $\mathcal{X} = \mathcal{S} \times \mathcal{A} \times \mathcal{S}$ and $(s,a,s') = x \in \mathcal{X}$, moreover we indicate with $d(\cdot)$ the optimized distribution and with $d^{P,\pi}(\cdot)$ the sampling distribution.
\begin{align}
	\underset{d}{\text{maximize}} & \; \int_\mathcal{X}d(x)R(x) \\
	\text{subject to} &  \; \int_\mathcal{X}d(x) \log \frac{d(x)}{d^{P,\pi}(x)} \mathrm{d}x \leq \epsilon \\
	& \int_\mathcal{X} d(x) \mathrm{d}x = 1 \ \, .
\end{align}

We solve the problem with Lagrangian multipliers. We denote with $\eta$ the langrangian multiplier associated with the KL constraint and with $\lambda$ the multiplier associated with the constraint of being a valid distribution.
\begin{align}
	\mathcal{L}(d(\cdot), \eta, \lambda) = &\int_\mathcal{X} d(x)R(x) \mathrm{d}x +\\ &+ \eta \left( \epsilon - \int_\mathcal{X} d'(x) \log \frac{d(x)}{d^{P,\pi}(x)} \mathrm{d}x \right) +\\ &+ \lambda \left(1 - \int_\mathcal{X} d(x) \mathrm{d}x \right) \, .
	\label{eq:lagrangian}
\end{align}
Observe that $\frac{\partial}{\partial f(x_0)} \int f(x) g(x) dx = g(x_0)$. So we take the derivative with respect to $d(x)$ to get:
\begin{equation}
	R(x) - \eta \log \frac{d(x)}{d^{P,\pi}(x)} + \eta - \lambda = 0 \, ,
\end{equation}
from which we get, solving for $d(x)$:
\begin{equation}
	d(x) = d^{P,\pi}(x) \exp \left(\frac{R(x)}{\eta} \right) \exp \left( 1 - \frac{\lambda}{\eta} \right) \, .
	\label{eq:remps-pol}
\end{equation}
By enforcing the constraint that $d$ should be a valid distribution we obtain:
\begin{equation}
	d(x) = \frac{d^{P,\pi}(x) \exp \left(\frac{R(x)}{\eta} \right)}{\int_\mathcal{X} d^{P,\pi}(x) \exp \left( \frac{R(x)}{\eta} \right) \mathrm{d}x} \, .
\end{equation}

Substituting into the Lagrangian function (\ref{eq:lagrangian}), we obtain the dual function:
\begin{align}
	g(\eta, \lambda) &= -\eta + \eta\epsilon + \lambda \\
	&= \eta \log \left( \int_\mathcal{X} d^{P,\pi}(x) \exp \left(\epsilon + \frac{R(x)}{\eta} \right)\mathrm{d}x \right) \, .
\end{align}

From (\ref{eq:remps-pol}) we extract the policy and model inducing the distribution $d$. We return to the original formulation for the sake of clarity.
\begin{align}
	\pi'(a | s) &= \frac{\int_\mathcal{S}d(s,a,s') \mathrm{d}s'}{\int_\mathcal{A}\int_\mathcal{S}d(s,a,s') \mathrm{d}s' \mathrm{d}a} \\
	&= \frac{\pi(a | s) \int_\mathcal{S} P(s' | s,a) \exp \left( \epsilon + \frac{R(x)}{\eta} \right) \mathrm{d}s'}{\int_\mathcal{A} \pi(a | s) \int_\mathcal{S} P(s'|s,a) \exp \left( \epsilon + \frac{R(x)}{\eta} \right) \mathrm{d}s' \mathrm{d}a}, \\
	P'(s' | a, s) &= \frac{d(s,a,s')}{\int_\mathcal{S}d(s,a,s') \mathrm{d}s'} \\
	&= \frac{P(s' | s,a) \exp \left( \epsilon + \frac{R(x)}{\eta} \right)}{\int_\mathcal{S} P(s' | s,a) \exp \left( \epsilon + \frac{R(x)}{\eta} \right) \mathrm{d}s'} \; .
\end{align}