% 19 April 1993 mg
\chapter{Catalog of Program packages and entries}
\section*{Arithmetic Routines}
\begin{DLtt}{12345678901}
\item[A105 MPA] Multiple-Precision Floating-Point Arithmetic
\end{DLtt}
\section*{Elementary Functions}
\begin{DLtt}{12345678901}
\item[B100 BINOM] Binomial Coefficient
\item[B101 ATG] Arc Tangent Function
\item[B102 ASINH] Hyperbolic Arcsine
\item[B300 RSRTNT] Integral of type $R(x,\sqrt{a+bx+cx^2})$
\end{DLtt}
\section*{Equations and Special Functions}
\begin{DLtt}{12345678901}
\item[C200 ZEROX] Zero of a Function of One Real Variable
\item[C201 SNLEQ] Numerical Solution of Systems of Nonlinear Equations
\item[C202 RMULLZ] Zeros of a Real Polynomial
\item[C205 RZERO] Zero of a Function of One Real Variable
\item[C207 RTEQ3] Roots of a Cubic Equation
\item[C208 RTEQ4] Roots of a Quartic Equation
\item[C209 CPOLYZ] Zeros of a Complex Polynomial
\item[C210 NZERFZ] Number of Zeros of a Complex Function
\item[C300 ERF] Error Function and Complementary Error Function
\item[C301 FREQ] Normal Frequency Function
\item[C302 GAMMA] Gamma Function for Positive Argument
\item[C303 GAMMF] Gamma Function for Real Argument
\item[C304 DILOG] Dilogarithm Function
\item[C306 CGAMMA] Complex Gamma Function
\item[C307 CDIGAM] Complex Digamma or Psi Function
\item[C309 CCLBES] Coulomb Wave, Bessel, and Spherical Bessel Functions for Complex Argument(s) and Order
\item[C310 ALGAMA] Logarithm of the Gamma Function
\item[C312 BESJ0] Bessel Functions J and Y of Orders Zero and One
\item[C313 BESI0] Modified Bessel Functions I and K of Orders Zero and One
\item[C315 RRIZET] Riemann Zeta Function
\item[C316 RPSIPG] Psi (Digamma) and Polygamma Functions
\item[C318 ELFUN] Jacobian Elliptic Functions sn, cn, dn
\item[C320 CELFUN] Jacobian Elliptic Functions sn, cn, dn for Complex Argument
\item[C321 CGPLG] Nielsen's Generalized Polylogarithm
\item[C322 FRSIN] Fresnel Integrals
\item[C323 FERDR] Fermi-Dirac Function
\item[C324 ATANI] Arctangent integral
\item[C327 BSIR4] Modified Bessel Functions I and K of Order 1/4, 1/2 and 3/4
\item[C328 CWHITM] Whittaker Function M of Complex Argument and Complex Indices
\item[C330 ASLGF] Legendre and Associated Legendre Functions
\item[C331 FCONC] Conical Functions of the First Kind
\item[C333 CLOGAM] Logarithm of the Complex Gamma Function
\item[C334 GAPNC] Incomplete Gamma Functions
\item[C335 CWERF] Complex Error Function
\item[C336 SININT] Sine and Cosine Integrals
\item[C337 EXPINT] Exponential Integral
\item[C338 CEXPIN] Complex Exponential Integral
\item[C339 DAWSON] Dawson's Integral
\item[C340 BSIR3] Modified Bessel Functions I and K of Order 1/3 and 2/3
\item[C342 STRH0] Struve Functions of Orders Zero and One
\item[C343 BSJA] Bessel Functions J and I with Positive Argument and Non-Integer Order
\item[C344 CBSJA] Bessel Functions J with Complex Argument and Non-Integer Order
\item[C345 BZEJY] Zeros of Bessel Functions J and Y
\item[C346 RELI1] Elliptic Integrals of First, Second, and Third Kind
\item[C347 RELI1C] Complete Elliptic Integrals of First, Second, and Third Kind
\item[C348 CELINT] Elliptic Integral for Complex Argument
\item[C349 RTHETA] Jacobian Theta Functions
\end{DLtt}
\section*{Integration, Minimization, Non-linear Fitting}
\begin{DLtt}{12345678901}
\item[D101 SIMPS] Integration by Simpson's Rule
\item[D103 GAUSS] Adaptive Gaussian Quadrature
\item[D104 CAUCHY] Cauchy Principal Value Integration
\item[D105 TRIINT] Integration over a Triangle
\item[D107 RGQUAD] N-Point Gaussian Quadrature
\item[D108 TRAPER] Trapezoidal Rule Integration with an Estimated Error
\item[D110 RGMLT] Gaussian Quadrature for Multiple Integrals
\item[D111 GPINDP] General Purpose Integration in Double-Precision
\item[D113 CGAUSS] Adaptive Complex Integration Along a Line Segment
\item[D114 RIWIAD] Adaptive Multidimensional Monte-Carlo Integration
\item[D115 CHEBQU] Double-Precision Clenshaw-Curtis Integration
\item[D151 DIVON4] Multidimensional Integration or Random Number Generation
\item[D200 RKSTP] First-order Differential Equations (Runge-Kutta)
\item[D201 DEQBS] First-order Differential Equations (Gragg--Bulirsch--Stoer)
\item[D202 DEQMR] First-order Differential Equations (Runge--Kutta--Merson)
\item[D203 RKNYS] Second-order Differential Equations (Runge--Kutta--Nystr{\accent "7F o}m)
\item[D300 EPDE1] Elliptic Partial Differential Equation
\item[D302 ELPAHY] Fast Partial Differential Equation Solver
\item[D401 DERIV] Numerical Differentiation
\item[D501 LEAMAX] Constrained Non-Linear Least Squares and Maximum
                   Likelihood Estimation
\item[D506 MINUIT] Function Minimization and Error Analysis
\item[D509 MINVAR] Minimum of a Function of One Variable
\item[D510 FUMILI] Fitting Chisquare and Likelihood Functions
\item[D601 RFRDH1] Solution of a Linear Fredholm Integral Equation of Second Kind
\item[D700 RFT] Real Fast Fourier Transform
\item[D701 FFTRC] Real and Complex Fast Fourier Transform
\item[D702 CFT] Complex Fast Fourier Transform
\item[D703 RFFT] Real Fast Fourier Transform
\item[D704 CFFT] Complex Fast Fourier Transform
\end{DLtt}
\section*{Interpolation, Approximations, Linear Fitting}
\begin{DLtt}{12345678901}
\item[E100 POLINT] Polynomial Interpolation
\item[E102 MAXIZE] Maximum and Minimum Elements of Arrays
\item[E103 AMAXMU] Largest Absolute Number in Scattered Vector
\item[E104 FINT] Multidimensional Linear Interpolation
\item[E105 DIVDIF] Function Interpolation
\item[E106 LOCATF] Binary Search for Element in Ordered Array
\item[E207 TRISUM] Summation of Trigonometric Series
\item[E208 LSQ] Least Squares Polynomial Fit
\item[E210 NORBAS] Polynomial Splines / Normalized B-Splines
\item[E211 RCSPLN] Cubic Splines and their Integrals
\item[E221 CHEB] Solution of Overdetermined Linear System in the Chebychev Norm
\item[E230 TL] Constrained and Unconstrained Linear Least Squares Fitting
\item[E250 LFIT] Least-Squares Fit to Straight Line
\item[E255 PARLSQ] Least-Squares Fit to Parabola
\item[E401 ECTRAD] Telescoping of Power Series, Double-Precision
\item[E406 DCHECF] Chebyshev Series Coefficients of a Function
\item[E407 CHSUM] Summation of Chebyshev Series
\item[E410 CPSC] Complex Power Series Coefficients
\end{DLtt}
\section*{Matrices, Vectors and Linear Equations}
\begin{DLtt}{12345678901}
\item[F001 LAPACK] Linear Algebra Package
\item[F002 RVADD] Elementary Vector Processing
\item[F003 RMADD] Elementary Matrix Processing
\item[F004 RMMLT] Matrix Multiplication
\item[F010 RINV] Linear Equations, Matrix Inversion
\item[F011 RFACT] Repeated Solution of Linear Equations, Matrix Inversion, Determinant
\item[F012 RSINV] Symmetric Positive-Definite Linear Systems
\item[F105 POLROT] Rotate a Three-Dimensional Polar Coordinate System
\item[F112 TR] Manipulation of Triangular and Symmetric Matrices
\item[F116 DOTI] Scalar Product of Two Space-Time Vectors
\item[F117 CROSS] Vector Product of Two 3-Vectors
\item[F118 ROT] Rotating a 3-Vector
\item[F121 VECMAN] Vector Algebra
\item[F122 SCATTER] Search Operations on Sparse Vectors
\item[F123 BVSL] Bit Vector Manipulation Package
\item[F150 MXDIPR] Direct or Tensor Matrix Product
%\item[F202 LRCH] Eigenvalues of a Real Symmetric Band Matrix
%\item[F220 EISPAC] Matrix Eigenvalue/Eigenvector Package
%\item[F221 EISCG1] Eigenvalues/Eigenvectors of Complex General Matrix
%\item[F222 EISCH1] Eigenvalues/Eigenvectors of Complex Hermitian Matrix
%\item[F223 EISRG1] Eigenvalues/Eigenvectors of Real General Matrix
%\item[F224 EISRS1] Eigenvalues/Eigenvectors of Real Symmetric Matrix
%\item[F225 EISST1] Eigenvalues/Eigenvectors of Real Tridiagonal Matrix
\item[F230 DEFLS] Deflate Matrix with Known Eigenvalue/Eigenvector
\item[F406 RBEQN] Banded Linear Equations
\item[F500 LIHOIN] Linear Homogenous Inequalities
%\item[F600 SVD] Singular Value Matrix Decomposition
\end{DLtt}
\section*{Statistical Analysis and Probability}
\begin{DLtt}{12345678901}
\item[G100 PROB] Upper Tail Probability of Chi-Squared Distribution
\item[G101 CHISIN] Inverse of Chi-Square Distribution
\item[G102 PROBKL] Kolmogorov Distribution
\item[G103 TKOLMO] Kolmogorov Test
\item[G104 STUDIS] Student's T-Distribution and Its Inverse
\item[G105 GAUSIN] Inverse of Gaussian Distribution
\item[G106 GAMDIS] Gamma Distribution
\item[G110 LANDAU] Landau Distribution
\item[G111 DISVAV] Vavilov Distribution and Its Inverse
\item[G900 RANF] Random Number Generator
\item[G901 RAN2VS] Random Points on a Circle or Sphere
\end{DLtt}
\section*{Operation Research Techniques and Management Science}
\begin{DLtt}{12345678901}
\item[H100 SIMPLE] Linear Optimization Using the Simplex Algorithm
\item[H300 ASSIGN] Assignment Problem
\end{DLtt}
\section*{Input/Output}
\begin{DLtt}{12345678901}
\item[I101 EPIO] EP Standard Format Input/Output Package
\item[I202 KUIP] KUIP - Kit for a User Interface Package
\item[I302 FFREAD] Format-Free Input Processing
\item[I303 RDWORD] Read a Format-Free Number
\end{DLtt}
\section*{Output and Graphical Data Presentation}
\begin{DLtt}{12345678901}
\item[J200 VIZPRI] Print Large Characters
\item[J401 BANNER] Print Banner Page in Large Characters
\item[J403 XBANNER] Print Banner Text
\item[J509 CONPRT] Print Function Contours in Two Variables
\item[J511 MAP] Table and Plot of Real Function
\item[J530 BINSIZ] Reasonable Intervals for Histogram Binning
\end{DLtt}
\section*{Executive Routines}
\begin{DLtt}{12345678901}
\item[L210 COMIS] COMIS - Compilation and Interpretation System
\item[L400 PATCHY] Source Code Maintenance
\end{DLtt}
\section*{Data Handling}
\begin{DLtt}{12345678901}
\item[M101 SORTZV] Sort One-Dimensional Array
\item[M103 FLPSOR] Sort One-Dimensional Array into Itself
\item[M104 SORCHA] Sort One-Dimensional Character Array into Itself
\item[M107 SORTR] Sort Rows of a Matrix
\item[M108 SORTMQ] Sort Rows of a Matrix
\item[M109 SORTRQ] Sort Rows of a Matrix
\item[M214 CVTVAX] Conversion To and From VAX Number Formats on IBM
\item[M215 PSCALE] Find Power-of-Ten Scale for Printing
\item[M216 GETWI] Read and Convert a CDC NOS/BE or Scope 2 W/I File
\item[M218 CVTCDC] Convert Between CDC and IBM Floating-Point Number Formats
\item[M220 IE3CONV] Conversion To and From IEEE Number Format
\item[M224 SETFMT] Edit Format for Printing a Floating-Point Vector
\item[M231 CVTIB] Convert Floating-Point Numbers Between Host Machine and IBM Formats
\item[M232 CVTND] Convert Floating-Point Numbers Between Host Machine and NORD Formats
\item[M233 TRTCH] Translate Between Different Character Sets
\item[M250 FLOARG] Assure Floating or Integer Representation
\item[M251 UFLINT] Assure Integer or Floating Representation of Numbers
\item[M400 CHTOI] Portable Conversion Between Type CHARACTER and Type INTEGER
\item[M409 UBUNCH] Concentrate and Disperse Character Strings
\item[M410 A1MANI] Manipulating BCD Strings in A1 Representation
\item[M416 UBLOW1] Concentrate and Disperse Bit Strings
\item[M421 BITBYT] Package for Handling Bits and Bytes
\item[M422 PACBYT] Handling Packed Vectors of Bytes
\item[M423 INCBYT] Increment a Byte of a Packed Vector
\item[M426 BLOW] Unpack Full Words into Bytes
\item[M427 PKCHAR] Pack/Unpack Continuous Byte-strings
\item[M428 LOCBYT] Search for Byte-Content
\item[M429 NUMBIT] Number of One-Bits in a Word
\item[M431 IFROMC] Convert Between Character String and Packed ASCII Form
\item[M432 CHPACK] Utility Routines for Character String Parsing and Construction
\item[M433 INDEXX] Utility Package for Character Manipulation
\item[M434 VXINV] Fast VAX Byte Inversion
\item[M436 BUNCH] Pack Bytes into Full Words
\item[M437 GETBIT] Set or Retrieve a Bit in a String
\item[M438 BTMOVE] Move Bit String
\item[M439 GETBYT] Set or Retrieve a Bit String
\item[M440 FIO999] Fortran Read/Write Simulation and Internal Input/Output Buffer
\item[M441 BITPAK] Handling Bits and Bytes, Bit Zero the Least Significant
\item[M442 NAMEFD] Fortran Emulation of VM/CMS NAMEFIND Command
\item[M501 IUSAME] Locating a String of Same Words
\item[M502 UOPTC] Decoding Options Characters
\item[M503 UBITS] Locate the One-Bits of a Word or an Array
\item[M506 IUMODE] Mode of Argument
\item[M507 LENOCC] Occupied Length of a Character String
\item[M508 BITPOS] Find One-Bits in a String
\end{DLtt}
\section*{Debugging, Error Handlng}
\begin{DLtt}{12345678901}
\item[N001 KERSET] Error Processing for Sections A-H of KERNLIB
\item[N002 MTLSET] Error Processing for MATHLIB
\item[N100 LOCF] Address of a Variable
\item[N103 IUWEED] Detect Indefinite and Infinite in an Array
\item[N105 TRACEQ] Print Trace-Back
\item[N203 TCDUMP] Memory Dump
\end{DLtt}
\section*{Service or Housekeeping Programming Aids}
\begin{DLtt}{12345678901}
\item[Q100 ZEBRA] Dynamic Data Structure and Memory Manager
\item[Q120 HIGZ] High Level Interface to Graphics and Zebra
\item[Q121 PAW] PAW - Physics Analysis Workstation Package
\item[Q122 SIGMA] SIGMA - System for Interactive Graphical Mathematical Applications
\item[Q123 FATMEN] Distributed File and Tape Management System
\item[Q124 CSPACK] Client Server Routines and Utilities
\item[Q180 HEPDB] Distributed Database Management System
\item[Q210 ZBOOK] Dynamic Memory Management
\item[Q901 INDENT] Indent Fortran Source
\item[Q902 FLOP] FLOP - Fortran Language Oriented Parser
\item[Q904 CONVERT] Fortran 77 to Fortran 90 source form conversion tool
\end{DLtt}
\section*{Logical and Symbolic}
\begin{DLtt}{12345678901}
\item[R205 REDUCE] REDUCE 3.3 - A System for Symbolic Algebra
\end{DLtt}
\section*{Magnet and Beam Design, Electronics}
\begin{DLtt}{12345678901}
\item[T604 POISCR] Solution of Poisson's or Laplace's Equation in Two-Dimensional Regions
\end{DLtt}
\section*{Quantum Mechanics, Particle Physics}
\begin{DLtt}{12345678901}
\item[U100 CLEBS] Clebsch-Gordan Coefficients in Algebraic Form
\item[U101 LOREN4] Lorentz Transformation
\item[U102 LORENF] Lorentz Transformations
\item[U110 CLEBSG] Clebsch-Gordan Coefficients; Wigner 3-j, 6-j, 9-j Symbols; Racah Coefficients; Jahn U-Function
\item[U501 DJMNB] Beta-Term in Wigner's D-Function
\end{DLtt}
\section*{Random Numbers and General Purpose Utilities}
\begin{DLtt}{12345678901}
\item[V100 RANNOR] Random Numbers in Normal Distribution
\item[V101 NORRAN] Fast Random Numbers in Normal Distribution
\item[V102 NORMCO] Pair of Random Numbers in Normal Distribution
\item[V103 IRND01] Random Bits
\item[V104 RNDM] Uniform Random Numbers
\item[V105 NRAN] Arrays of Uniform Random Numbers
\item[V106 RN32] Machine-Independent Uniform Random Numbers
\item[V107 RNDM2] IBM Uniform Random Number Generator
\item[V108 RG32] Portable Gaussian Random Number Generator
\item[V109 RANGAM] Random Numbers in Gamma or Chisquare Distribution
\item[V110 POISSN] Poisson Random Numbers
\item[V111 BINOMI] Binomial Random Numbers
\item[V112 MUNOMI] Multinomial Random Numbers
\item[V113 RANMAR] Uniform Random Number Generator
\item[V114 RANECU] Uniform Random Number Generator
\item[V130 RAN3D] Random Three-Dimensional Vectors
\item[V150 HISRAN] Random Numbers According to Any Histogram
\item[V151 FUNRAN] Random Numbers According to Any Function
\item[V202 PERMU] Permutations and Combinations
\item[V300 UZERO] Preset Parts of an Array
\item[V301 UCOPY] Copy an Array
\item[V302 UCOCOP] Copy a Scattered Vector
\item[V304 IUCOMP] Search a Vector for a Given Element
\item[V306 PROXIM] Adjusting an Angle to Another Angle
\item[V401 GRAPH] Find Compatible Node-Nets in an Incompatibility Graph
\item[V700 RVNSPC] Volume of Intersection of a Circular Cylinder with a Sphere
\end{DLtt}
\section*{High Energy Physics Simulation, Kinematics, Phase Space}
\begin{DLtt}{12345678901}
\item[W150 TRSPRT] Transport, Second-Order Beam Optics
\item[W151 TURTLE] Beam Transport Simulation, Including Decay
\item[W505 FOWL] General Monte-Carlo Phase-Space
\item[W515 GENBOD] N-Body Monte-Carlo Event Generator
\end{DLtt}
\section*{Statistical Data Analysis and Presentation}
\begin{DLtt}{12345678901}
\item[Y201 IUCHAN] Find Histogram-Channel
\item[Y250 HBOOK] Statistical Analysis and Histogramming
\item[Y251 HPLOT] HBOOK Graphics Interface for Histogram Plotting
\end{DLtt}
\section*{Miscellaneous System-Dependent Facilities}
\begin{DLtt}{12345678901}
\item[Z001 KERNGT] Print KERNLIB Version Numbers
\item[Z007 DATIME] Job Time and Date
\item[Z008 TIMAL] Job Time in IBM Accounting Units
\item[Z009 CALDAT] Calendar Date Conversion
\item[Z020 UMON] Usage Monitor for VAX/VMS
\item[Z029 NOARG] Number of Arguments Supplied in a Call Statement
\item[Z034 WHICH] Computer Mainframe Identification
\item[Z035 ABEND] Abnormal Termination of Fortran Programs
\item[Z036 ABUSER] Intercept a Fortran Abend on IBM
\item[Z037 VAXAST] Routines to Handle Control-C Interrupts on Vax
\item[Z041 QNEXTE] Restart of Next Event
\item[Z042 JUMPXN] Calling a subroutine by its address
\item[Z044 INTRAC] Identify Job as Interactive
\item[Z100 JOBNAM] Get User Job Name
\item[Z203 XINOUT] Short List Reading and Writing
\item[Z204 FNZERO] Cray File Name with Blank or Zero Fill
\item[Z262 GOPARM] Provide the User with the G Step PARM-String (IBM)
\item[Z264 IARGC] Returns UNICOS command line Arguments
\item[Z265 CINTF] Immediate Interface Routines to the C Library
\item[Z267 FTOVAX] Convert file-name to and from UNIX syntax
\item[Z300 IOPACK] IBM-Dependent Input/Output Package
\item[Z301 VAXTIO] VAX Fortran Interface for Reading and Writing 'Foreign' Tapes
\item[Z303 KAPACK] Random Access I/O Using Keywords
\item[Z304 IOSPACK] General Purpose IBM VM/CMS Non-Graphics Full Screen Interface Package
\item[Z305 VMPACK] IBM VM/CMS System Interface
\item[Z306 MAXDSK] Find CMS R/W minidisk with most free space
\item[Z307 JOB\dollar VM] Return details about CMS virtual machine
\item[Z308 IOSPAK2] 3270 Full Screen I/O Routines
\item[Z309 VMIO] CMS Macro I/O Package
\item[Z310 CFIO] Handle Fixed-length records on Unix streams
\item[Z311 CIO] Handle Unix Disk Files
\item[Z312 VAXTAP] VAX Tape Handling Utilities
\end{DLtt}


% !Mode:: "TeX:UTF-8"
% !TEX program  = xelatex
\section{Introduction}\label{S:introduction}
\subsection{Preliminary information}
We assume that the reader has already learned the parabola in the physics class and the rectangular coordinate system in the math class. If you do not have the required knowledge, it does not matter, you can follow the information provided in this article, learning while reading.

\subsection{Projectile Motion}
Perhaps we have been told by our parents since childhood that, 45 degrees is the ideal angle to throw a football without air resistance, to understand the reasons why 45 degrees is the best angle without air resistance, we should start with projectile motion\cite{wiki:projectile_motion}.

\begin{defnbox}{Projectile motion}
    Projectile motion is a form of motion experienced by an object or particle (a projectile) that is thrown near the Earth's surface and moves along a curved path under the action of gravity only\footnote{In particular, the effects of air resistance are assumed to be negligible.}.
\end{defnbox}

In the following article, we call the curved path a trajectory\cite{wiki:trajectory}. Through experiments or simulations, we can get the trajectories of football at different angles, as shown in Figure~\ref{F:trajectory}. From the picture we can see that, the footballs are thrown from the origin, the speed remains unchanged, while the angle has changed. Intuitively, 45 degrees is indeed the ideal angle to throw a football for maximum range, and complementary angles have the same range.

The trajectory was shown by Galileo to be a parabola, which is relevant to quadratic equation. No rush, we will learn the basics of quadratic equation, in the next section.

\begin{figure}
    \centering
    \includegraphics[width=.7\textwidth]{figures/quadratic_equation-1.png}
    \caption{Parabolic trajectories at different angles}\label{F:trajectory}
\end{figure}

\section{Quadratic Equation}\label{S:quadratic}
\subsection{Definition}
\begin{defnbox}{Quadratic equation\cite{wiki:quadratic_equation}}
    In algebra, a quadratic equation (from the Latin \emph{quadratus} for ``square'') is any equation having the form
    \begin{equation}\label{E:1}
        c_2 x^2 + c_1 x + c_0 = 0,
    \end{equation}
    where $x$ represents an unknown, and $c_n$ ($n=0,1,2$) represent known numbers, with $a\neq 0$. If $a=0$, then the equation is linear, not quadratic, as there is no $c_2 x^2$ term. Then we can divide both sides of the equation by $c_2$, and replace $c_1/c_2$ with $b$, $c_0/c_2$ with $c$.
    \begin{equation}\label{E:quadratic-equation}
        x^2 + bx + c = 0.
    \end{equation}
    That is, every quadratic equation can be transformed into the form of Equation~\eqref{E:quadratic-equation}.
\end{defnbox}

\subsection{Quadratic Formula and Its Derivation}
From Equation~\eqref{E:quadratic-equation}, we can complete the square to derive a general formula to solve quadratic equations. Then we have noticed that, coefficient of $x^2$ is 1, coefficient of $x$ is $b$, if we want to complete the square form, we have
\begin{equation}\label{E:quadratic-formula-1}
    \begin{aligned}
        x^2 + bx + c &= 0 \\
        x^2 + 2\frac{b}{2}x &= -c \\
        \left(x+\frac{b}{2}\right)^2 &= -c + \frac{b^2}{4} \\
        \left(x+\frac{b}{2}\right)^2 &= \frac{b^2-4c}{4}.
    \end{aligned}
\end{equation}

The left side of the Equation~\eqref{E:quadratic-formula-1} is a squared form, which means that it may have multiple solutions. That is, if $b^2-4c\geq 0$,
\begin{equation}\label{E:quadratic-formula-2}
    \begin{aligned}
        x + \frac{b}{2} &= \pm \frac{\sqrt{b^2-4c}}{2} \\
        x &= -\frac{b}{2} \pm \frac{\sqrt{b^2-4c}}{2} \\
        x &= \frac{-b\pm\sqrt{b^2-4c}}{2}.
    \end{aligned}
\end{equation}

If we substitute the Equation~\eqref{E:quadratic-formula-2} back to the Equation~\eqref{E:quadratic-equation}, we will find that the equation still holds. That is the quadratic formula is
\begin{equation}\label{E:quadratic-formula}
    x = \begin{cases}
        \frac{-b\pm\sqrt{b^2-4c}}{2} & \text{If $b^2-4c\geq 0$;} \\
        \frac{-b\pm i\sqrt{4c-b^2}}{2} & \text{If $b^2-4c<0$.}
    \end{cases}
\end{equation}

You can get an intuitive feel for the quadratic equation from Figure~\ref{F:quadratic-formula}. Moreover, we found other interesting phenomena --- vertex, which is the extreme point of the parabola, whether minimum or maximum. The $x$-coordinate of the vertex will be located at $x=-\tfrac{b}{2}$, and the $y$-coordinate of the vertex may be found by substituting this $x$-value into the function, which gives that $y=\tfrac{1-(b^2-4c)}{4}$.

\begin{figure}
    \centering
    \includegraphics[width=.35\textwidth]{figures/quadratic_equation-2.png}
    \caption{Visualization of quadratic equations (with $a=1$)}\label{F:quadratic-formula}
\end{figure}


\subsection{Performance estimation}
\label{sec:performance}

In this section, we describe the first phase of Elo-MMR. For notational convenience, we assume all probability expressions to be conditioned on the \textbf{prior context} $P_{i,< t}$, and omit the subscript $t$.

Our prior belief on each player's skill $S_i$ implies a prior distribution on $P_i$. Let's denote its probability density function (pdf) by
\begin{equation}
\label{eq:perf-prior}
f_i(p) := \Pr(P_i = p) = \int \pi_i(s) \Pr(P_i = p \mid S_i=s) \,\mathrm{d}s,
\end{equation}
where $\pi_i(s)$ was defined in \Cref{eq:pi-s}. Let
\[F_i(p) := \Pr(P_i\le p) = \int_{-\infty}^p f_i(x) \,\dx,\]
be the corresponding cumulative distribution function (cdf). For the purpose of analysis, we'll also define the following ``loss'', ``draw'', and ``victory'' functions:
\begin{align*}
l_i(p) &:= \ddp\ln(1-F_i(p)) = \frac{-f_i(p)}{1 - F_i(p)},
\\d_i(p) &:= \ddp\ln f_i(p) = \frac{f'_i(p)}{f_i(p)},
\\v_i(p) &:= \ddp\ln F_i(p) = \frac{f_i(p)}{F_i(p)}.
\end{align*}

Evidently, $l_i(p) < 0 < v_i(p)$. Now we define what it means for the deviation $P_i - S_i$ to be log-concave.
\begin{definition}
\label{def:log-concave}
An absolutely continuous random variable on a convex domain is \textbf{log-concave} if its probability density function $f$ is positive on its domain and satisfies
\[f(\theta x + (1-\theta) y) > f(x)^\theta f(y)^{1-\theta},\;\forall\theta\in(0,1),x\neq y.\]
\end{definition}

Log-concave distributions appear widely, and include the Gaussian and logistic distributions used in Glicko, TrueSkill, and many others. We'll see inductively that our prior $\pi_i$ is log-concave at every round. Since log-concave densities are closed under convolution~\cite{concave}, the independent sum $P_i=S_i+(P_i-S_i)$ is also log-concave. Log-concavity is made very convenient by the following lemma, proved in the extended version of this paper:
\begin{lemma}
\label{lem:decrease}
If $f_i$ is continuously differentiable and log-concave, then the functions $l_i,d_i,v_i$ are continuous, strictly decreasing, and
\[l_i(p) < d_i(p) < v_i(p) \text{ for all }p.\]
\end{lemma}

For the remainder of this section, we fix the analysis with respect to some player $i$. As argued in \Cref{sec:bayes_model}, $P_i$ concentrates very narrowly in the posterior. Hence, we can estimate $P_i$ by its MAP, choosing $p$ so as to maximize:
\[\Pr(P_i=p\mid E^L_i,E^W_i) \propto f_i(p) \Pr(E^L_i,E^W_i\mid P_i=p).\]

Define $j\succ i$, $j\prec i$, $j\sim i$ as shorthand for $j\in E^L_i$, $j\in E^W_i$, $j\in \mathcal P\setminus (E^L_i\cup E^W_i)$ (that is, $P_j>P_i$, $P_j<P_i$, $P_j=P_i$), respectively. The following theorem yields our MAP estimate:
\begin{theorem}
\label{thm:uniq-max}
Suppose that for all $j$, $f_j$ is continuously differentiable and log-concave. Then the unique maximizer of $\Pr(P_i=p\mid E^L_i,E^W_i)$ is given by the unique zero of
\[Q_i(p) := \sum_{j \succ i} l_j(p) + \sum_{j \sim i} d_j(p) + \sum_{j \prec i} v_j(p).\]
\end{theorem}
The proof appears in the extended version of this paper. Intuitively, we're saying that the performance is the balance point between appropriately weighted wins, draws, and losses. Let's look at two specializations of our general model, to serve as running examples in this paper.

\paragraph{Gaussian performance model}
If both $S_j$ and $P_j-S_j$ are assumed to be Gaussian with known means and variances, then their independent sum $P_j$ will also be a known Gaussian. It is analytic and log-concave, so \Cref{thm:uniq-max} applies.

We substitute the well-known Gaussian pdf and cdf for $f_j$ and $F_j$, respectively. A simple binary search, or faster numerical techniques such as the Illinois algorithm or Newton's method, can be employed to solve for the unique zero of $Q_i$.

\begin{figure}
    \centering
    \includegraphics[width=1.05\columnwidth]{images/l2-lr-plot.eps}
    \caption{$L_2$ versus $L_R$ for typical values (left). Gaussian versus logistic probability density functions (right).}
    \label{fig:l2-lr-plot}
\end{figure}

\paragraph{Logistic performance model}
Now we assume the performance deviation $P_j-S_j$ has a logistic distribution with mean 0 and variance $\beta^2$. In general, the rating system administrator is free to set $\beta$ differently for each contest. Since shorter contests tend to be more variable, one reasonable choice might be to make $1/\beta^2$ proportional to the contest duration.

Given the mean and variance of the skill prior, the independent sum $P_j = S_j + (P_j-S_j)$ would have the same mean, and a variance that's increased by $\beta^2$. Unfortunately, we'll see that the logistic performance model implies a form of skill prior from which it's tough to extract a mean and variance. Even if we could, the sum does not yield a simple distribution.

For experienced players, we expect $S_j$ to contribute much less variance than $P_j-S_j$; thus, in our heuristic approximation, we take $P_j$ to have the same form of distribution as the latter. That is, we take $P_j$ to be logistic, centered at the prior rating $\mu^\pi_j = \argmax \pi_j$, with variance $\delta_j^2 = \sigma_j^2 + \beta^2$, where $\sigma_j$ will be given by \Cref{eq:variance}. This distribution is analytic and log-concave, so the same methods based on \Cref{thm:uniq-max} apply.
Define the scale parameter $\bar\delta_j := \frac{\sqrt{3}}{\pi} \delta_j$. A logistic distribution with variance $\delta_j^2$ has cdf and pdf:
\begin{align*}
F_j(x) &= \frac { 1 } { 1 + e^{-(x-\mu^\pi_j)/\bar\delta_j} }
= \frac 12 \left(1 + \tanh\frac{x-\mu^\pi_j}{2\bar\delta_j} \right),
\\f_j(x) &= \frac { e^{(x-\mu^\pi_j)/\bar\delta_j} } { \bar\delta_j\left( 1 + e^{(x-\mu^\pi_j)/\bar\delta_j} \right)^2}
= \frac { 1 } { 4\bar\delta_j} \sech^2\frac{x-\mu^\pi_j}{2\bar\delta_j}.
\end{align*}

The logistic distribution satisfies two very convenient relations:
\begin{align*}
F'_j(x) = f_j(x) &= F_j(x) (1 - F_j(x)) / \bar\delta_j,
\\f'_j(x) &= f_j(x) (1 - 2F_j(x)) / \bar\delta_j,
\end{align*}
from which it follows that
\[d_j(p)
= \frac{1 - 2F_j(p)}{\bar\delta}
= \frac{-F_j(p)}{\bar\delta} + \frac{1 - F_j(p)}{\bar\delta}
= l_j(p) + v_j(p).\]

In other words, a tie counts as the sum of a win and a loss. This can be compared to the approach (used in Elo, Glicko, BAR, Topcoder, and Codeforces) of treating each tie as half a win plus half a loss.\footnote{Elo-MMR, too, can be modified to split ties into half win plus half loss. It's easy to check that \Cref{lem:decrease} still holds if $d_j(p)$ is replaced by
$w_l l_j(p) + w_v v_j(p)$,
provided that $w_l,w_v\in [0,1]$ and $|w_l-w_v|<1$.
In particular, we can set $w_l=w_v=0.5$. The results in \Cref{sec:properties} won't be altered by this change.}

Finally, putting everything together:
\[Q_i(p) = \sum_{j \succeq i} l_j(p) + \sum_{j \preceq i} v_j(p)
= \sum_{j \succeq i} \frac{-F_j(p)}{\bar\delta_j} + \sum_{j \preceq i} \frac{1 - F_j(p)}{\bar\delta_j}.\]
Our estimate for $P_i$ is the zero of this expression. The terms on the right correspond to probabilities of winning and losing against each player $j$, weighted by $1/\bar\delta_j$. Accordingly, we can interpret $\sum_{j\in \cP} (1-F_j(p))/\bar\delta_j$ as a weighted expected rank of a player whose performance is $p$. Similar to the performance computations in Codeforces and Topcoder, $P_i$ can thus be viewed as the performance level at which one's expected rank would equal $i$'s actual rank.


\section{Introduction}
In this paper we are going to study Lie algebras over unital commutative rings $R$, with $2 \nmid \mathrm{char}(R)$ which are projective of finite type as $R$-modules.
\subsection{Lie algebras and their coalgebras}
A Lie algebra $\lieg$ is an $R$-module with $R$-linear map:
$$\mu : \lieg \otimes_R \lieg \longrightarrow \lieg$$
that is antisymmetric ($\mu = -\tau \mu$) and Jacobi-identity holds:
$$\mu (1 \otimes \mu) (1 + \zeta_3 + \zeta_3^2)(x_1 \otimes x_2 \otimes x_3) = 0, x_i \in \lieg$$
where $\zeta_3$ is the cyclic permutation of the tensors, $\tau = [x \otimes y \longmapsto y \otimes x]$ the flip map and $1$ the identity map. In \cite{SongSu}, they proposed
$$\ker (1 - \tau) \subset \ker \mu$$
as an alternative for antisymmetry.
\begin{defi}
An $R$-module $\lieg$ with $R$-linear map $\Delta : \lieg \longrightarrow \lieg \otimes \lieg$ is called an Lie $R$-coalgebra if:
\bi
\item $\mathrm{im} \Delta \subset \mathrm{im} (1 - \tau)$
\item $(1 + \zeta_3 + \zeta_3^2) (1 \otimes \Delta) \Delta = 0$.
\ei
\end{defi}
We recall that an coassociative coalgebra fulfills $(1 \otimes \Delta)\Delta = (\Delta \otimes 1)\Delta)$. For clarity we repeat:
a bialgebra is an $R$-module $B$, with multiplication $\mu$, comultiplication $\Delta$ (and optional unit and counit), such that $\mu, \eta$ are homomorphisms of coalgebras and $\Delta, \eps$ are homomorphisms of algebras over $R$.
\begin{defi}
An Lie bialgebra $(\lieg,\mu,\Delta)$ is a:
\bi
\item Lie algebra $(\lieg,\mu)$ and
\item Lie coalgebra $(\lieg,\Delta)$,
\ei
such that the following diagram commutes:
$$\xymatrix{
\lieg \otimes \lieg \ar[rr]^\mu\ar[d]_{D \otimes D} && \lieg \ar[rr]^\Delta && \lieg \otimes \lieg\\
\lieg^{\otimes 4} \ar[d]_{id \otimes \tau\otimes id} &&&&\\
\lieg^{\otimes 4} \ar[d]_{id \otimes id \otimes \tau} &&&&\\
\lieg^{\otimes4} \ar[rr]_{id \otimes \Delta \otimes id \otimes \Delta} &&\lieg^{\otimes 6} \ar[rr]_{\mu \otimes id \otimes \mu \otimes id} && \lieg^{\otimes4}\ar[uuu]_{\pi_1 - \pi_2}\\
}$$
where $D : \lieg \longrightarrow \lieg \otimes \lieg, x \longmapsto x \otimes x$, $\pi_1 : \lieg^{\otimes 4} \longrightarrow \lieg\otimes\lieg, x \otimes y \otimes z \otimes u \longmapsto x \otimes y$, $\pi_2 : \lieg^{\otimes 4} \longrightarrow \lieg \otimes \lieg$, $x \otimes y \otimes z \otimes u \longmapsto z \otimes u$.
\end{defi}


\section{Minimum Cost Assignment with Expected Distances}


\section{Open questions}

Under previous insights, we ask respectively:
\begin{enumerate}
    \item let $\mathcal{R}_{h(t)}=\left(g(h(t)), h(t)\right)$ a $h$-characterization
        of $\mathcal{R}$, and consider the series expansion of funtion $g$:
        \begin{displaymath}
            \left.\left[g(y)=g_0 + g_1 y + g_2 y^2 + \ldots\right|y=h(t)\right]
        \end{displaymath}
        what's the interpretation of coefficients in the sequence
        $\lbrace g_i\rbrace_{i\in\mathbb{N}}$?

        We've seen that if $\mathcal{R}=\left(d(t), td(t)\right)$, then
        $[g(y)=A(y)|y=h(t)]$ where $A(t)$ is $\mathcal{R}$'s A-sequence:
        does exist a deep relation among those sequences for \emph{arbitrary} arrays?
        For instance, what about Fibonacci array $\mathcal{F}$:
        \begin{displaymath}
            \begin{split}
                \big[ g_{\mathcal{F}}(y) &= 1 + y + y^{2} - y^{3} -2y^{4}
                -3 y^{5}  +3 y^{7} \\
                &+7 y^{8} +4 y^{9} + \mathcal{O}\left(y^{10}\right) | y=h_{\mathcal{F}}(t) \big]
            \end{split}
        \end{displaymath}
        or about Delannoy array $\mathcal{D}$:
        \begin{displaymath}
            \begin{split}
                \big[ g_{\mathcal{D}}(y) &= 1 + y - y^{2} + 3 y^{3} -11y^{4} + 45 y^{5} -197y^{6}\\
                &+ 903 y^{7} -4279y^{8} + \mathcal{O}\left(y^{9}\right)| y=h_{\mathcal{D}}(t) \big]
            \end{split}
        \end{displaymath}
    \item let $\mathcal{P}$ and $\mathcal{C}$ be Pascal and Catalan arrays, respectively:
        what does $\mathcal{C}^{\stackrel{\frac{t}{1-t}}{\rightarrow}}$ count?
        More generally, chosen two Riordan arrays $\mathcal{A}$ and $\mathcal{B}$,
        $\mathcal{A}^{\stackrel{h_{\mathcal{B}}(t)}{\rightarrow}}$ also has a combinatorial meaning?
        So does $\mathcal{B}^{\stackrel{h_{\mathcal{A}}(t)}{\rightarrow}}$? What about considering
        combinations involving  $\mathcal{A}^{-1}$ and $\mathcal{B}^{-1}$ too?

    \item let $\mathcal{R}(d(t),h(t))$ be a Riordan array in
        natural notation. If function $\hat{h}$, the compositional inverse
        of function $h$, has the following structure:
        \begin{displaymath}
            \hat{h}(y) = \frac{y}{\Pi(y)}
        \end{displaymath}
        where $\Pi$ is a polynomial such that $\Pi(0)\not=0$, the question is:
        is it always the case that there exists a sequence of polynomials
        $\lbrace \Omega_{i} \rbrace_{i\in\mathbb{N}}$, where function $\Omega_{i}$
        is defined as:
        \begin{displaymath}
            \Omega_{i}(y)=\left(\frac{[t^{1+i}]d(t)h(t)} {\Pi(y)}\right)
        \end{displaymath}
        such that $\mathcal{R}$'s $A$-sequence can be be factored respect to
        function $\hat{h}$ as:
        \begin{displaymath}
                \left.\left[
                    A_{\mathcal{R}}(y) = \sum_{i \geq0}{
                        \Omega_{i}(y) \hat{h}(y)^{i}}
                        \right| y = h(t) \right]
        \end{displaymath}
        in other words, the sequence of coefficients of the function defining
        the second column of array $\mathcal{R}$, namely the convolution of functions
        $d$ and $h$, shifted by \emph{one} position, does occur in
        the given factorization?

\end{enumerate}

\subsection{An hint\ldots}

In this section we offer an hint to tackle question number $1$
asked in previous enumeration. Let $\mathcal{R}_{h(t)}(\gamma(h(t)), h(t))$
be a Riordan array written in $h$-characterization form, for some function $\gamma$.
Now, with abuse of notation, do a \emph{standard} matrix-vector product, namely
consider $\mathcal{R}_{h(t)}$ as a matrix: multiply it by a vector $\vect{\omega}$
whose coefficients are defined by a sequence $\lbrace \omega_i \rbrace_{i\in\mathbb{N}}$
and set equal to a vector $\vect{a}$ whose coefficients are defined by $\mathcal{R}$'s
$A$-sequence. Formally:

\begin{displaymath}
    \mathcal{R}_{h(t)}\left[\begin{array}{c} \omega_0 \\ \omega_1 \\ \omega_2 \\ \vdots \end{array}\right] =
        \left[\begin{array}{c} a_0 \\ a_1 \\ a_2 \\ \vdots \end{array}\right]
\end{displaymath}
Interpreting columns of $\mathcal{R}_{h(t)}$ and vector $\vect{a}$ via
corresponding generating functions, rewrite as follow:
\begin{displaymath}
    \begin{split}
            \omega_0\,\gamma(h(t))\,h(t)^{0} +
            \omega_1\,\gamma(h(t))\,h(t)^{1} +
            \omega_2\,\gamma(h(t))\,h(t)^{2} +
            \ldots &= A(t) \\
            \gamma(h(t))\left(\omega_0\,h(t)^{0} +
            \omega_1\,h(t)^{1} +
            \omega_2\,h(t)^{2} +
            \ldots \right) &= A(t) \\
            \gamma(h(t))\,\Omega(h(t)) &= A(t) \\
    \end{split}
\end{displaymath}
where functions $A$ and $\Omega$ are \ac{fps} over sequences
$\lbrace a_i \rbrace_{i\in\mathbb{N}}$ and
$\lbrace \omega_i \rbrace_{i\in\mathbb{N}}$, respectively. Note that abstracting
over function $h$ yield:
\begin{displaymath}
    \left.\left[
        \gamma(y)\,\Omega(y) = A(\hat{h}(y)) \right| y = h(t) \right]
\end{displaymath}
but this relationship is quite difficult since introduces function $\hat{h}$,
the compositional inverse of function $h$. Therefore, function
$\gamma$, parameterized by function $h$, is related to $\mathcal{R}$'s
$A$-sequence, parameterized over variable $t$, by the existence of
a function $\Omega$, parameterized over function $h$, such that:
\begin{displaymath}
    \gamma(h(t))\,\Omega(h(t)) = A(t)
\end{displaymath}
For the sake of clarity, let $\mathcal{M}$ be the Motzkin array, so:
\begin{displaymath}
        \left.\left[
            \Omega_{\mathcal{M}}(y) = \frac{y^{4} + 3 \, y^{3} + 5 \, y^{2} + 3 \, y + 1}{{\left(y^{2} + y + 1\right)}^{3}}
                \right| y = h(t) \right]
\end{displaymath}
and the first terms of fps expansion:
\begin{displaymath}
        \left.\left[
            \Omega_{\mathcal{M}}(y) = 1 -y^{2} -y^{3} + 4 y^{4} -2\,y^{5}
                -6\,y^{6} + 11 y^{7}+\mathcal{O}\left(y^{8}\right)
                \right| y = h(t) \right]
\end{displaymath}
In turn another question arises: \emph{there
exists a smart way to compute function $\Omega$}? If there exists such a method, then
$\mathcal{R}$'s $A$-sequence could be computed easily, since function $\gamma$
is read directly from the factorization, for \emph{any} Riordan array\ldots


\subsection{Gaussian Mixture}
\label{sec:ctm-gm}

The second clustering algorithm we tried was Gaussian Mixture \cite{dempster1977maximum}. Similar to the K-Means algorithm, it also groups unlabeled data points but with different criteria. As its name suggests, it assumes the points are randomly distributed following a Gaussian distribution. It iteratively tries to assign groups to maximize the likelihood of each data point belonging to their assigned groups. Likewise, we hypothesized that the closer a point (student solution) is to the centroid, in this case the central probability contour, the higher the probability of the student receiving a high score. In practice, we designated each student's score to be equal to the probability of them belonging to their assigned cluster, returned from the Gaussian Mixture algorithm.

\begin{figure}
\includegraphics[width=\textwidth]{conversion-to-mark/marking_paster_nbio_ece459-a1-w2017_gm}
\caption[Gaussian Mixture Clustering]{This is a probability contour graph of our data points grouped by the Gaussian Mixture algorithm. Points closer to the center of the contours, i.e. lighter areas, have a higher probability of belonging to their corresponding group.}
\label{fig:ctm-gm}
\end{figure}


%\documentclass{article}
\documentclass[18pt]{extarticle}
% Comment the following line to NOT allow the usage of umlauts
\usepackage[utf8]{inputenc}
% Uncomment the following line to allow the usage of graphics (.png, .jpg)
%\usepackage{graphicx}

\newcommand*{\pd}[3][]{\ensuremath{\frac{\partial^{#1} #2}{\partial #3}}}
% Start the document
\begin{document}

% Create a new 1st level heading
\section{Finding a function's derivative in chart coordinates}
Front matter
$
\\ f(x) = f(\psi^{-1}(\psi(x)))
%
\\ \frac{\partial}{\partial x} f(x) =
\frac{\partial }{\partial x}
f(\psi^{-1}(\psi(x)))
%
\\ =\pd{f}{x}(\psi^{-1}(\psi(x))) *
\pd{}{x}(\psi^{-1}(\psi(x)))
%
\\ =\pd{f}{x}(\psi^{-1}(\psi(x))) *
\pd{\psi^{-1}}{x}(\psi(x)) *
\pd{}{x}\psi(x)
\\ $
\newline
Now also consider the following:
\newline
$
\\ \pd{}{x}f(\psi^{-1}(x)) =
\pd{f}{x}(\psi^{-1}(x)) *
\pd{\psi^{-1}}{x}(x)
$
\newline
\newline
This is the derivative of the function in the chart coordinates on the left hand side and then applying the chain rule on the right.
\newline
But from the above equalities we see we can relate
the derivative of the function in chart coordinates, to its derivative in manifold coordinates by (a) evaluating it at $\psi(x)$ and then (b) multiplying by $\pd{\psi}{x}$.
\newline
\newline
So we finally have:
\newline
$
\\ \pd{}{x}f(x) = \pd{\psi}{x} *
\pd{}{x}f(\psi^{-1}(x)) @ \psi(x)
$
\end{document}


\newcommand\chapternumber{2}
\input{../tex_header/header.tex}
\usepackage{enumerate}
\usepackage{float}

\begin{document}

\section{Find constants / partition functions.}
\subsection{}
\begin{align*}
    f(x|a,b) = 1 \text{ on } x \in [a,b]
\end{align*}

First, let's take integral of $f(x)$:
\begin{align*}
    \int f(x) \ dx
    &= \int_a^b 1 \ dx\\
    &= b-a
\end{align*}

So, if we want $\int p(x) \ dx = 1$, and $p(x) = \frac{1}{\mathcal{Z}} f(x)$, we have:
\begin{align}
    \int \frac{1}{\mathcal{Z}} f(x) \ dx&= 1 \nonumber \\
    \frac{1}{\mathcal{Z}} \int  f(x) \ dx&= 1 \nonumber \\
    \int f(x) \ dx&= \mathcal{Z}  \\
    b-a &= \mathcal{Z} \nonumber
\end{align}

So, $\mathcal{Z} = b-a$.

Equation (1) can also apply to other functions in question 1, and it indicates that the normalizing constant is just the integral of the function.

\subsection{}
\begin{align*}
    f(x|\beta) = \exp(-\frac{x}{\beta}) \text{ on } x \in (0,\infty) \text{ and } \beta \in (0, \infty)
\end{align*}

First, let's take integral of $f(x)$:
\begin{align*}
    \int f(x) \ dx
    &= \int_0^{\infty} \exp(-\frac{x}{\beta})\ dx \\
    &= (-\beta \cdot \exp(-\frac{x}{\beta})) \biggr\rvert _0^{\infty} \\
    &= -\beta \cdot (\exp(-\infty)-\exp(0)) \\
    &= \beta
\end{align*}

So, substitude to equation (1), we have $\mathcal{Z} = \beta$.

\subsection{}
\begin{align*}
    f(x|\mu,\sigma)=\exp[-\frac{(x-\mu)^2}{2\sigma^2}]
\end{align*}
where $x \in (-\infty, \infty)$, $\mu \in (-\infty, \infty)$, $\sigma \in (0, \infty)$.

To find the integral of $f$, we need to square it:
\begin{align*}
    \int f(x|\mu, \sigma) \ dx &= \sqrt{\int_{-\infty}^{\infty} f(x|\mu, \sigma) \ dx \cdot \int_{-\infty}^{\infty} f(x|\mu, \sigma) \ dx} \\
    &= \sqrt{\int_{-\infty}^{\infty} f(x|\mu, \sigma) \ dx \cdot \int_{-\infty}^{\infty} f(y|\mu, \sigma) \ dy} \\
    &= \sqrt{\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} f(x|\mu, \sigma) \cdot f(y|\mu, \sigma) \ dx\ dy} \\
    &= \sqrt{\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} \exp[-\frac{(x-\mu)^2}{2\sigma^2}] \cdot \exp[-\frac{(y-\mu)^2}{2\sigma^2}] \ dx\ dy} \\
    &= \sqrt{\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} \exp[-\frac{(x-\mu)^2}{2\sigma^2}-\frac{(y-\mu)^2}{2\sigma^2}] \ dx\ dy} \\
    &= \sqrt{\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} \exp[-\frac{(x-\mu)^2 + (y-\mu)^2}{2\sigma^2}] \ dx\ dy} \\
\end{align*}

Let's set the new origin at $(\mu, \mu)$, and turn the equation into polar coordinate:
\begin{align*}
    \int f(x|\mu, \sigma) \ dx
    &= \sqrt{\int_{-0}^{2 \pi} \int_{0}^{\infty} \exp[-\frac{r^2}{2\sigma^2}] \cdot r \ dx\ dy} \\
    &= \sqrt{\int_{-0}^{2 \pi} \sigma^2 \ dy} \\
    &= \sqrt{2 \sigma^2 \pi } \\
\end{align*}

We know $\sigma > 0$, so $\mathcal{Z} = \sigma \sqrt{2 \pi}$.
% First, we start from the PDF of the Normal distribution:
% \begin{align*}
%     p(x) &= \frac{1}{\sigma \sqrt{2 \pi}} \cdot \exp[-\frac{(x-\mu)^2}{2\sigma^2}] \\
%     &= \frac{1}{\sigma \sqrt{2 \pi}} \cdot f(x|\mu, \sigma)
% \end{align*}

% So,
% \begin{align*}
%     f(x|\mu,\sigma) = \sigma \sqrt{2 \pi} \cdot p(x)
% \end{align*}

% We know that Normal distribution is a probability distribution, so $\int p(x) \ dx=1$.
% Thus $\int f(x|\mu, \sigma) \ dx  = \sigma \sqrt{2 \pi} \cdot 1$, and $\mathcal{Z} = \sigma \sqrt{2 \pi}$.
\subsection{}
\begin{align*}
    f(x|a,\gamma)=\frac{1}{x^\gamma}
\end{align*}
where $x \in (a, \infty)$, $a \in (0, \infty)$, $\gamma \in (0, \infty)$.

There are three cases: (1) $\gamma>1$, (2) $\gamma=1$, (3) $0<\gamma<1$.

In case (1) $\gamma>1$:
\begin{align*}
    \int_a^{\infty} \frac{1}{x^{\gamma}} \ dx
    &= \frac{x^{1-\gamma}}{1-\gamma} \biggr\rvert _a^{\infty} \\
    &= 0 - \frac{a^{1-\gamma}}{1-\gamma} \\
    &= \frac{1}{a^{\gamma-1} (\gamma-1)}
\end{align*}

So, in case (1), $\mathcal{Z} = \frac{1}{a^{\gamma-1} (\gamma-1)}$.

In case (2) $\gamma=1$:
\begin{align*}
    \int_a^{\infty} \frac{1}{x} \ dx
    &= \log(x) \biggr\rvert _a^{\infty} \\
    &= \infty
\end{align*}

So, in case (2), $\mathcal{Z}$ doesn't exist.

In case (3) $0<\gamma<1$:
\begin{align*}
    \int_a^{\infty} \frac{1}{x^{\gamma}} \ dx
    &= \frac{x^{1-\gamma}}{1-\gamma} \biggr\rvert _a^{\infty} \\
    &= \infty
\end{align*}

So, in case (3), $\mathcal{Z}$ doesn't exist.

Thus, in summary, if $f(x|a,\gamma)$ is defined on $\gamma \in (0, \infty)$, $\mathcal{Z}$ doesn't exist.

\section{1st and 2nd moment}
In the previous question, only 1.1 and 1.2 can be properly-normalized.

\subsection{}
For 1.1, $\mathcal{Z}=b-a$, and $p(x)=\frac{1}{b-a}$.
\begin{align*}
    m_1 &= E_{x \sim p(x)}[x] \\
    &= \int_a^b p(x) x \ dx\\
    &= \int_a^b \frac{x}{b-a} \ dx\\
    &= \frac{a+b}{2}
\end{align*}

\begin{align*}
    m_2 &= E_{x \sim p(x)}[(x-m_1)^2] \\
    &= E_{x \sim p(x)}[(x-\frac{a+b}{2})^2] \\
    &= \int_a^b p(x) (x-\frac{a+b}{2})^2 \ dx\\
    &= \int_a^b \frac{(x-\frac{a+b}{2})^2}{b-a}  \ dx\\
    &= \frac{1}{b-a} \int_a^b (x^2 - (a+b)x + \frac{1}{4}(a+b)^2) )  \ dx\\
    &= \frac{1}{b-a} (\frac{1}{3}(b^3-a^3) - \frac{1}{2}(a+b)(b^2-a^2) + \frac{1}{4}(b-a)(a+b)^2) \\
    &= \frac{1}{12} (b-a)^2
\end{align*}

\subsection{}
For 1.2, $\mathcal{Z}=\beta$, and $p(x)=\frac{1}{\beta} \exp(-\frac{x}{\beta})$.
\begin{align*}
    m_1 &= E_{x \sim p(x)}[x] \\
    &= \int_0^{\infty} p(x) x \ dx\\
    &= \int_0^{\infty} \frac{1}{\beta} \exp(-\frac{x}{\beta}) x \ dx\\
    &= \frac{1}{\beta} \int_0^{\infty} \exp(-\frac{x}{\beta}) x \ dx\\
    &= \frac{1}{\beta} [\beta (-\exp(-\frac{x}{\beta})(\beta+x))] \biggr\rvert _0^{\infty} \\
\end{align*}
Here, we have $\beta>0$, so, $\exp(-\frac{x}{\beta}) x \rightarrow 0$ when $x \rightarrow 0$.
\begin{align*}
    m_1 &= \frac{1}{\beta} [0 + \beta (\beta+0))] \\
    &= \beta
\end{align*}

\begin{align*}
    m_2 &= E_{x \sim p(x)}[(x-m_1)^2] \\
    &= E_{x \sim p(x)}[(x-\beta)^2] \\
    &= \int_0^{\infty} p(x) (x-\beta)^2 \ dx \\
    &= \int_0^{\infty} \frac{1}{\beta} \exp(-\frac{x}{\beta}) (x-\beta)^2 \ dx \\
    &= \frac{1}{\beta} \int_0^{\infty} \exp(-\frac{x}{\beta}) (x^2-2\beta x + \beta^2) \ dx \\
    &= \frac{1}{\beta} [ \int_0^{\infty} \exp(-\frac{x}{\beta}) x^2 \ dx -2 \int_0^{\infty} \exp(-\frac{x}{\beta}) \beta x \ dx + \int_0^{\infty} \exp(-\frac{x}{\beta}) \beta^2 \ dx ] \\
    &= \frac{1}{\beta} [ (\beta(-\exp(-\frac{x}{\beta})(2\beta^2+x^2+2\beta x)))|_0^{\infty} -2 (\beta^2(-\exp(-\frac{x}{\beta})(\beta+x)))|_0^{\infty} \\
    &\  + (\beta^3(-\exp(-\frac{x}{\beta})))|_0^{\infty} ]
\end{align*}
For the same argument in case of $\beta>0$, $\exp(-\frac{x}{\beta}) x \rightarrow 0$ and $\exp(-\frac{x}{\beta}) x^2 \rightarrow 0$ when $x \rightarrow 0$:
\begin{align*}
    m_2 &= \frac{1}{\beta} [ 2 \beta^3 -2 \beta^3 + \beta^3 ] \\
    &= \beta^2
\end{align*}

\subsection{}
For 1.3, $\mathcal{Z} = \sigma \sqrt{2 \pi}$, and $p(x)=\frac{1}{\sigma \sqrt{2 \pi}} \exp(-\frac{(x-\mu)^2}{2 \sigma^2})$.
\begin{align*}
    m_1 &= E_{x \sim p(x)}[x] \\
    &= \int_{-\infty}^{\infty} p(x) x \ dx \\
    &= \int_{-\infty}^{\infty} \frac{1}{\sigma \sqrt{2 \pi}} \exp(-\frac{(x-\mu)^2}{2 \sigma^2}) x \ dx \\
    &= \int_{-\infty}^{\infty} \frac{1}{\sigma \sqrt{2 \pi}} \exp(-\frac{(x-\mu)^2}{2 \sigma^2}) (x-\mu) + \mu \cdot \frac{1}{\sigma \sqrt{2 \pi}} \exp(-\frac{(x-\mu)^2}{2 \sigma^2}) \ dx \\
\end{align*}

Let $y = x - \mu$, we can see the first part is 0 (from question 1.3).
And we write the second part back into the form of $p(x)$, which is the normalized probability density function.
\begin{align*}
    m_1
    &= \int_{-\infty}^{\infty} \frac{1}{\sigma \sqrt{2 \pi}} \exp(-\frac{y^2}{2 \sigma^2}) y + \mu \cdot p(x) \ dx \\
    &= \int_{-\infty}^{\infty} \mu \cdot p(x) \ dx \\
    &= \mu
\end{align*}

\begin{align*}
    m_2 &= E_{x \sim p(x)}[(x-m_1)^2] \\
    &= E_{x \sim p(x)}[(x-\mu)^2] \\
    &= \int_{-\infty}^{\infty} p(x) (x-\mu)^2 \ dx \\
    &= \int_{-\infty}^{\infty} \frac{1}{\sigma \sqrt{2 \pi}} \exp[-\frac{(x-\mu)^2}{2 \sigma^2}] (x-\mu)^2 \ dx \\
\end{align*}

Let $y=\frac{x-\mu}{\sqrt{2}\sigma}$, so, $dx = \sqrt{2} \sigma\ dy$, we have:
\begin{align*}
    m_2 &=
    \int_{-\infty}^{\infty} \frac{1}{\sigma \sqrt{2 \pi}} \exp[-y^2] 2\sigma^2 y^2  \sqrt{2} \sigma \ dy \\
    &= \sigma^2 \int_{-\infty}^{\infty} \frac{2}{\sqrt{\pi}} \exp[-y^2]  y^2  \ dy \\
\end{align*}

We know from question 1, $\int_{-\infty}^{\infty} \exp[-\frac{(x-\mu)^2}{2\sigma^2}] \ dx = \sigma \sqrt{2 \pi}$.
If we differentiate both sides w.r.t. $y$, we have :
\begin{align*}
    \int_{-\infty}^{\infty} 2 y^2 \exp[-y^2] \sigma \sqrt{2} \ dy &= \sigma \sqrt{2 \pi} \\
    \int_{-\infty}^{\infty} \frac{2}{\sqrt{\pi}} \exp[-y^2]  y^2  \ dy &= 1
\end{align*}

Substitude back in, we have:
\begin{align*}
    m_2 &= \sigma^2
\end{align*}

\subsection{}
For 1.4, first, we have:
\begin{align*}
    m_1 &= E_{x \sim p(x)}[x] \\
    &= \int_{a}^{\infty} p(x) x \ dx \\
    &= \frac{1}{\mathcal{Z}} \int_{a}^{\infty} \frac{1}{x^\gamma} x \ dx \\
    &= \frac{1}{\mathcal{Z}} \int_{a}^{\infty} x^{1-\gamma} \ dx \\
    &= \frac{1}{\mathcal{Z}} \frac{1}{2-\gamma} [x^{2-\gamma}] \biggr\rvert_a^{\infty} \\
\end{align*}

We can see, (1) in the case of $\gamma<=2$, $m_1 \rightarrow \infty$;

(2) in the case of $\gamma>2$, $x^{2-\gamma} |_a^{\infty}=-a^{2-\gamma}$, so:
\begin{align*}
    m_1
    &= a^{\gamma-1} (\gamma-1) \frac{1}{2-\gamma} (-a^{2-\gamma}) \\
    &= \frac{\gamma-1}{\gamma-2} \cdot a \\
\end{align*}

Now in order to calculate $m_2$, we need $m_1$ to be finite, (and for case $\gamma<=2$ we don't have finite $m_2$ ):
\begin{align*}
    m_2 &= E_{x \sim p(x)}[(x-m_1)^2] \\
    &= \frac{1}{\mathcal{Z}} \int_{a}^{\infty} x^{-\gamma} (x-m_1)^2 \ dx \\
    &= \frac{1}{\mathcal{Z}} \int_{a}^{\infty} x^{-\gamma} (x^2 - 2 m_1 x + m_1 ^2) \ dx \\
    &= \frac{1}{\mathcal{Z}} \int_{a}^{\infty} (x^{2-\gamma} - 2 m_1 x^{1-\gamma} + m_1 ^2 x^{-\gamma}) \ dx \\
\end{align*}

For the same argument when calculating $m_1$, we can see, (1) in the case of $\gamma<=3$, $m_2 \rightarrow \infty$;

(2) in the case of $\gamma>3$, we have:
\begin{align*}
    m_2 &= \frac{1}{\mathcal{Z}} \int_{a}^{\infty} (x^{2-\gamma} - 2 m_1 x^{1-\gamma} + m_1 ^2 x^{-\gamma}) \ dx \\
    &= a^{\gamma-1} (\gamma-1) \biggr[ \int_{a}^{\infty} (x^{2-\gamma} - 2 m_1 x^{1-\gamma} + m_1 ^2 x^{-\gamma}) \ dx \biggr] \\
    &= a^{\gamma-1} (\gamma-1) \biggr[ \int_{a}^{\infty} x^{2-\gamma} \ dx - 2 m_1 \int_{a}^{\infty} x^{1-\gamma} \ dx + m_1 ^2 \int_{a}^{\infty} x^{-\gamma} \ dx \biggr] \\
    &= a^{\gamma-1} (\gamma-1) \biggr[ \frac{a^{3-\gamma}}{\gamma-3} - 2 m_1 \frac{a^{2-\gamma}}{\gamma-2} + m_1 ^2 \frac{a^{1-\gamma}}{\gamma-1} \biggr] \\
    &= a^{\gamma-1} (\gamma-1) \biggr[ \frac{a^{3-\gamma}}{\gamma-3} - 2 \frac{\gamma-1}{\gamma-2} \cdot a \frac{a^{2-\gamma}}{\gamma-2} + (\frac{\gamma-1}{\gamma-2} \cdot a)^2 \frac{a^{1-\gamma}}{\gamma-1} \biggr] \\
    &= a^2 (\gamma-1) \biggr[ \frac{1}{\gamma-3} - 2 \frac{\gamma-1}{\gamma-2} \frac{1}{\gamma-2} + (\frac{\gamma-1}{\gamma-2})^2 \frac{1}{\gamma-1} \biggr] \\
    &= a^2 (\gamma-1) \biggr[ \frac{1}{\gamma-3} - 2 \frac{\gamma-1}{(\gamma-2)^2} + \frac{\gamma-1}{(\gamma-2)^2} \biggr] \\
    &= a^2 (\gamma-1) \biggr[ \frac{1}{\gamma-3} - \frac{\gamma-1}{(\gamma-2)^2} \biggr] \\
    &= a^2 \biggr[ \frac{\gamma-1}{\gamma-3} - \frac{(\gamma-1)^2}{(\gamma-2)^2} \biggr] \\
\end{align*}

\subsection*{Comments}
When analyzing data, we can compute the 1st and 2nd moment of the samples, and guess the distribution according to them.

For example, if we found $m_2 = m_1^2$, it might suggest that the data is from an exponential distribution.

\section{Simulation}
Python code please refer to:

\href{https://github.com/liusida/ds2/blob/main/assignment2/code/}{https://github.com/liusida/ds2/blob/main/assignment2/code/}

\input{table}

\subsection*{Convergence}
The estimated $m_1$ and $m_2$ converge to analytical solutions of $m_1$ and $m_2$.
The variance of $m_1$ and $m_2$ decrease with $K$.
Yes, the simulation confirms the analytical solutions are good.

\section{Derivation from (11) to (12)}
Because the constraints are:
\begin{align*}
    E_{x \sim p(x)}[f_i(x)] = F_i
\end{align*}
for $1 \leq i \leq I$, let's write the function $J$ in this way:
\begin{align*}
    J = \sum_x p(x) \log \frac{1}{p(x)} - \sum_{1 \leq i \leq I} \lambda_i (E_{x \sim p(x)}[f_i(x)]-F_i)
\end{align*}
And we start to solve for $\partial_{p(x)} J = 0$, we have:
\begin{align*}
    \partial_{p(x)} \biggr[ \sum_x p(x) \log \frac{1}{p(x)} - \sum_{1 \leq i \leq I} \lambda_i \biggr(E_{x \sim p(x)}[f_i(x)]-F_i\biggr) \biggr] &= 0 \\
    \partial_{p(x)} \biggr[ \sum_x p(x) \log \frac{1}{p(x)} - \sum_{1 \leq i \leq I} \lambda_i \biggr(\sum_{x} p(x) f_i(x)-F_i\biggr) \biggr] &= 0 \\
    \sum_x \partial_{p(x)} p(x) \log \frac{1}{p(x)} - \sum_{1 \leq i \leq I} \partial_{p(x)} \lambda_i \biggr(\sum_{x} p(x) f_i(x)-F_i\biggr) &= 0 \\
    \sum_x \partial_{p(x)} p(x) \log \frac{1}{p(x)} - \sum_{1 \leq i \leq I} \partial_{p(x)} \lambda_i \sum_{x} p(x) f_i(x) - \sum_{1 \leq i \leq I} \partial_{p(x)} \lambda_i F_i &= 0 \\
    \sum_x \partial_{p(x)} p(x) \log \frac{1}{p(x)} - \sum_x \partial_{p(x)} p(x) \sum_{1 \leq i \leq I} \lambda_i f_i(x) - 0 &= 0 \\
    \sum_x \partial_{p(x)} p(x) \biggr[ \log \frac{1}{p(x)} - \sum_{1 \leq i \leq I} \lambda_i f_i(x) \biggr] &= 0 \\
\end{align*}
Because in general $\partial_{p(x)} p(x) \neq 0$, so:
\begin{align*}
    \log \frac{1}{p(x)} - \sum_{1 \leq i \leq I} \lambda_i f_i(x) &= 0 \\
    \log p(x) &= - \sum_{1 \leq i \leq I} \lambda_i f_i(x)  \\
    p(x) &= \exp \biggr( - \sum_{1 \leq i \leq I} \lambda_i f_i(x) \biggr) \\
\end{align*}
Though ``$\propto$'' is used in the lecture note, but since $\lambda_i$ can be any constants, I think it is OK to use ``$=$''.

\section{Prove there is a maximum entropy distribution on R}
Proof:

The Normal distribution is a maximum entropy distribution, and the Normal distribution has a support of $R$,
thus there is a maximum entropy distribution on $R$.

\end{document}


\section{Introduction: Covering spaces}
A continuous map between spaces $p: Y \rightarrow X$ is a \emph{cover} or a \emph{covering map} if every point $x \in X$ has an open neighborhood $U$ such that \begin{enumerate}
  \item $p^{-1}(U)$ is a disjoint union of open sets $\sqcup_{i \in \cali} U_i$,
  \item $p$ restricted to each $U_i$ is an isomorphism.
\end{enumerate}
% \footnote{We'll later on show that the size of fiber does not depend on the point $x$.}
\begin{figure}[H]
  \centering
  \input{images/pancake.tex}
  \caption{Pancake picture of covering spaces}
\end{figure}
$U$ is said to be an \emph{evenly covered neighborhood}.
If $\abs{p^{-1}(x)} = k$ for ever $x \in X$, we say that $p$ is a $k$-cover.

\begin{ex}
  The most basic example of a covering space is
  \begin{equation*}
    I \rightarrow \{*\}
  \end{equation*}
  where $I$ is a discrete set.
  This is not very interesting, so from now on \textbf{we will assume that all our spaces connected.}
\end{ex}

\begin{ex}
  The only connected cover of $[0,1]$ is the space $[0,1]$ inself.
\end{ex}

\begin{ex}
  The first non-trivial example of a cover comes from a circle.
  \begin{align*}
    S^1 &\longrightarrow S^1 \\
    \theta \mod 2 \pi &\longmapsto n \theta \mod 2\pi
  \end{align*}
  The real line is also a cover of $S^1$.
  \begin{align*}
    \bbr^1 &\longrightarrow S^1 \\
    \theta &\longmapsto \theta \mod 2\pi
  \end{align*}
\end{ex}

\begin{ex}
  Turns out, the cylinder is a 2-covering of a mobius strip.
  It is hard to describe this using coordinates or equations, we'll instead do this combinatorially.
\end{ex}

\subsection{Spaces}
We will use cell complexes as a combinatorial model for spaces.
A \emph{2-cell complex} $X$ is a triple $(V, E, F)$ consisting of:
  \begin{enumerate}
    \item A non-empty set of vertices $V$.
    \item A set of directed edges $E$. For each edge $e \in E$, denote by $d_0 e$ and $d_1 e$ the starting and ending vertex of $e$ respectively. Denote by $e^{-1}$ the same edge but going in the opposite direction so that $d_0 e^{-1} = d_1 e$ and $d_1 e^{-1} = d_0 e$.
    \item A set of faces $F$.
  \end{enumerate}
  We further assume that our cell complexes are
  \begin{enumerate}
    \item \emph{locally finite} i.e. there are finitely many edges between any two vertices, and
    \item \emph{connected} i.e. there is a path of undirected edges connecting any two vertices.
  \end{enumerate}

A \emph{simplical map} between two cell complexes $p: X_0 \rightarrow X_1$ is a compatible triple of maps.
\begin{align*}
  V_0 \rightarrow V_1 && E_0 \rightarrow E_1 && F_0 \rightarrow F_1
\end{align*}
A simplical map is a covering space map if the induced map on the underlying topological spaces is one.

\begin{remark}
  Not every continuous map can be represented by a simplicial map, but every covering map can be. Further, there are more than one ways to describe a space using a graph.
\end{remark}

\subsubsection{Example: Covers of $S^1$}
For every integer $n$, there exists a unique $n$-cover of a circle.
  \begin{figure}[H]
  \centering
    \begin{tikzpicture}[thick]
      \begin{scope}[shift={(3,0)}, scale=0.5]
        \input{images/circle.tex}
      \end{scope}

      \draw [->] (-0.5,0) to node[midway, above] {$p$} (2.5,0);

      \begin{scope}[shift={(-3,0)}]
        \input{images/circle.tex}
      \end{scope}
    \end{tikzpicture}
    \caption{2-covering of a circle}
  \end{figure}
  Further, these covering maps form a poset: $nS^1$ is a cover of $mS^1$ if and only if $m$ divides $n$.
  $S^1$ has an infinite covering given by the real line.

  \begin{figure}[H]
  \centering
    \begin{tikzpicture}[thick]
      \begin{scope}[shift={(3,0)}, scale=0.5]
        \filldraw (0,0) circle (4pt);
        \draw [->] (0,0) to [bend left=45] (1,1) to [bend left=45] (2,0);
        \draw (2,0) to [bend left=45] (1,-1)  node [below] {$S^1$} to [bend left=45] (0,0);
      \end{scope}

      \draw [->] (-0.5,0) to node[midway, above] {$p$} (2.5,0);

      \begin{scope}[shift={(-5,0)}, scale=0.5]
        \filldraw (0,0) circle (4pt)
                  (2,0) circle (4pt)
                  (4,0) circle (4pt)
                  (6,0) circle (4pt);
        \draw [dashed] (-1,0) to (0,0);
        \draw [->] (0,0) to (1,0);
        \draw (1,0) to (2,0);
        \draw [->] (2,0) to (3,0);
        \draw (3,0) to node[midway, below] {$\bbr^1$} (4,0);
        \draw [->] (4,0) to (5,0);
        \draw (5,0) to (6,0);
        \draw [dashed] (6,0) to (7,0);
      \end{scope}
    \end{tikzpicture}
    \caption{Covering of $S^1$ by $\bbr^1$.}
  \end{figure}

  We will show that these covering spaces correspond to the subgroups of $\bbz$ and the inclusions of subgroups correspond to covering maps.
  \begin{align*}
    n \bbz &\longleftrightarrow nS^1 \\
    \set{0} &\longleftrightarrow \bbr^1 \\
    n \bbz \subseteq d \bbz &\longleftrightarrow nS^1 \rightarrow dS^1
  \end{align*}
  This is because the fundamental group of the circle is $\bbz$.

  \subsection{Example: $S^1 \vee S^1$}
  The space $S^1 \vee S^1$ (two circles glued at point) with 1 vertex, 2 edges, and 0 faces has very interesting covering spaces, see Figure \ref{fig:CoveringsOfS1S1}. The meanings of the various notations in Figure \ref{fig:CoveringsOfS1S1} will become clear later.
  \begin{figure}[H]
  \centering
    \begin{tikzpicture}[thick, scale=0.75]
      \input{images/s1s1.tex}
    \end{tikzpicture}
    \caption{$S^1 \vee S^1$}
  \end{figure}

  \begin{figure}[p]
  \centering
    \includegraphics[width=\textwidth]{coveringsOfS1S1.jpg}
    \caption{Coverings of $S^1 \vee S^1$. Image from Algebraic Topology, Allen Hatcher, Chapter 1.}
    \label{fig:CoveringsOfS1S1}
  \end{figure}

  \subsubsection{Example: Gluing diagrams}
  We can form surfaces using 2-complexes, however these are harder to describe on paper.
  Instead, we use a trick called \emph{gluing diagrams}, Figure \ref{fig:GluingDiagrams}.
  These are ways to draw non-planar things on a plane, so there is some ambiguity and gluing going on in a gluing diagram that you should be careful about.

  \begin{qbox}
    In each of the gluing diagrams in Figure \ref{fig:GluingDiagrams}, count the number of vertices, edges, and faces.
  \end{qbox}

  \begin{qbox}
    Show that there are 2-cover maps
    \begin{align*}
      \mbox{Cylinder} &\longrightarrow \mbox{Mobius Strip} \\
      \mbox{Torus} &\longrightarrow \mbox{Klein Bottle} \\
      \mbox{Sphere } S^2 &\longrightarrow \mbox{Real projective space}
    \end{align*}
  \end{qbox}

  \begin{qbox}
    Guess the poset of covers of
    \begin{enumerate}
      \item Cylinder,
      \item Mobius Strip,
      \item Torus.
    \end{enumerate}
    Can you interpret these posets as subgroups of some groups?
  \end{qbox}

  \begin{figure}[p]
    \centering
    \includegraphics[width=0.8\textwidth]{cylinderGluingDiagram.png}
    \includegraphics[width=0.8\textwidth]{MobiusStripGluingDiagram.png}

    \includegraphics[width=0.8\textwidth]{torusGluingDiagram.png}
    \includegraphics[width=0.8\textwidth]{KleinBottleGluingDiagram.png}

    \includegraphics[width=0.8\textwidth]{RP2GluingDiagram.png}
    \caption{Gluing diagrams for cylinder, Mobius strip, torus, Klein bottle, real projective space respectively. Images from BMC Notes on Surfaces by Maia Averett.}
    \label{fig:GluingDiagrams}
  \end{figure}


%% SECTION HEADER /////////////////////////////////////////////////////////////////////////////////////
\section{Example: Equations}
\label{sec12}

%% SECTION CONTENT ////////////////////////////////////////////////////////////////////////////////////

This section shows a few equation examples. Labels can be used to reference equations.
\begin{itemize}
	\item Example (Equation \ref{eq:1_1})
	\item Example (Equation \ref{eq:1_2})
	\item Example (Equation \ref{eq:1_3})
	\item Example (Equation \ref{eq:1_4})
	\item Example (Equation \ref{eq:1_5})
\end{itemize}

\begin{equation}
\label{eq:1_1}
\overline{M}_{i}=\iint\limits_A \rho\;u_{i}\left(u_{k}\;n_{k} \right)\;dA
\end{equation}

\begin{equation}
\label{eq:1_2}
V_{REF} = \left( \frac{2 \times 144 \times P_{DYN}}{\rho} \right)^{\frac{1}{2}}
\end{equation}

\begin{equation}
\label{eq:1_3}
V_{MOM} = \left( \frac{A_{TIP} (\frac{V_{REF}}{2})^{2} + A_{MIDDLE} V_{REF}^{2} + A_{HUB} V_{HUB}^{2}}{A_{JET} V_{AVG}} \right)
\end{equation}

\begin{equation}
\label{eq:1_4}
\begin{aligned}
& Q_{J}= \int_0^\delta Vdy+\left(y-\delta \right)b\;V_{s}\\
& Q_{J}= b\;V_{s} \int_0^\delta \left(\frac{y}{\delta} \right)^\frac{1}{n}dy+\left(y-\delta \right)b\;V_{s}\\
& Q_{J}=\frac{b\;V_{s}}{\left(\frac{n+1}{n} \right)}\delta+\left(y-\delta \right)b\;V_{s}\\
& Q_{J}=b\;V_{s}\left(y-\frac{\delta}{n+1} \right)
\end{aligned}
\end{equation}

\begin{equation}
\label{eq:1_5}
B_{K_{T_{Jx}}}=\sqrt{\left(\theta_{Q_{J}}B_{Q_{J}} \right )^{2}+\left(\theta_{D}B_{D} \right )^{2}+\left(\theta_{n}B_{n} \right )^{2}+\left(\theta_{ \alpha }B_{ \alpha } \right )^{2}+\left(\theta_{\rho}\left(B_{\rho}+\theta_{\rho tw}B_{tw} \right ) \right )}
\end{equation}


\section{Leave One Covariate Out}

\begin{frame}[c]
\Huge{\centerline{Leave One Covariate Out}}
\end{frame}

%----------------------------------------------------------------------------------------
\begin{frame}\frametitle{Leave One Covariate Out}
	\begin{itemize}
			\item Leave-one-covariate-out (LOCO) provides the feature importance values at a per-observation level (i.e. what is the feature importance of $X_j$ for observation $\mathbf{x}^{(i)}$).
			\bigskip
			\item LOCO is calculated by subtracting a model's prediction, $g(\mathbf{x}^{(i)})$, for a row-observation with all its features, from that model's prediction for the same row \textit{without} the input feature $X_j$ of interest.
		\begin{equation}
                           \begin{aligned}\label{eq:rf}
                            g(\mathbf{x}_{(-j)}^{(i)}) - g(\mathbf{x}^{(i)}),
                            \end{aligned}
		\end{equation}
where $\mathbf{x}_{(-j)}^{(i)} \in \mathcal{P}_{(-j)}$, given that $\mathcal{P}_{(-j)}$ is the complement set to $\{X_j\} \in \mathcal{P}$ (i.e $\{X_j\} \cup \mathcal{P}_{(-j)} = \mathcal{P}$).
	\end{itemize}
\end{frame}
%----------------------------------------------------------------------------------------


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
    \DeclareGraphicsExtensions{.png, .jpeg}
\usepackage{caption}
\usepackage[top=1in, bottom=1in, left=1in, right=1in]{geometry}

\title{STAT 775: Machine Learning \\ HW 03}
\author{Terence Henriod}
\date{\today}

\begin{document}

\clearpage            % All
\maketitle            % this,
\thispagestyle{empty} % removes the page number from the title page

\begin{abstract}
Feature reduction techniques including Exploring All Subsets or Best Subset Selection, Ridge Regression, and Principle Component Analysis (PCA).
\end{abstract}

\newpage
\section{Best Subset Selection}
\subsection{Problem Description}
Reproduce Figure 3.5 from the \emph{Elements of Statistical Learning} book. In this exercise, we need to use the prostate cancer data set found on the book's website and explore the performance of linear regression using subsets of the given features. The error for each subset is plotted, and a line indicating the best subsets for each subset size $k$ is indicated.

\subsection{Results}
My figure looks pretty similar to the one in the book, however, I think that the book excluded some points from their figure, which mine does include. I think this because the regression weights (or $\vec{\beta}$) match those found in the book, so my results should match theirs very closely.

It should be noted that in the subsets, the intercept ``feature" was used \emph{along with} each subset; the term \emph{subsets} refers to subsets of the actual features.

  \begin{figure}[h!]
    \centering
    \includegraphics[width=.8\linewidth]{HW03_Exercise01}
    \caption{An error plot for all subsets of features for the prostate cancer data set.}
    \label{fig:HW03_Exercise01}
  \end{figure}

\subsection{Code}
The following R code was used to reproduce the figure from the book.
\begin{verbatim}
##################
# STAT775
# HW 03
# Exercise 01
#
# Reproduce Figure 3-5 from the
# Elements of Statistical Learning
# (This is a scatterplot of the
# errors produced by models built
# from all possible subsets of
# predictors, except for an
# intercept, for simplicity)
#
##################

#
# Initial Setup
#
setwd("C:/Users/Terence/Documents/GitHub/STAT775/HW03")
# setwd("~/STAT775/HW03/")

RESULTS.FILE.NAME = "HW03_Exercise01.png"
DATA.FILE.NAME <- "prostate_cancer/prostate_cancer_data"
# these include the intercept that gets added to the matrices
FEATURE.INDICES <- 1:9
TARGET.INDEX <- 10
TRAIN.SET.FLAG <- 11

data <- read.table(DATA.FILE.NAME)
# according to the dataset info, this needs to be done
data[, 1:8] <- scale(data[, 1:8], T, T)
intercept <- rep(1, nrow(data))
data <- data.frame(Intercept = intercept, data)

# store data with rows as observations
train.data <- data.matrix(subset(data, data$train == T))[, FEATURE.INDICES]
train.targets <- data.matrix(subset(data, data$train == T))[, TARGET.INDEX]
test.data <- data.matrix(subset(data, data$train == F))[, FEATURE.INDICES]
test.targets <- data.matrix(subset(data, data$train == F))[, TARGET.INDEX]

#
# Regression
#
model.betas <-
  solve(t(train.data) %*% train.data) %*% t(train.data) %*% train.targets

#
# Test errors
#
feature.subsets <- set_power(as.set(2:9))  # skip excluding intercept

k <- rep(0, length(feature.subsets))
subset.as.string <- rep("", length(feature.subsets))
error <- rep(0, length(feature.subsets))
best.for.k <- rep(F, length(feature.subsets))

subset.selection.results <- data.frame(
  K = k,
  Subset = subset.as.string,
  Error = error,
  Best.For.K = best.for.k,
  stringsAsFactors = F
)

index = 1
residual.sum.of.squares <- 0
for (subset in feature.subsets) {
  subset.betas <- matrix(rep(0, 9))
  subset.betas[1, ] <- model.betas[1, ]
  for (i in subset) {
    subset.betas[i, ] <- model.betas[i, ]
  }

  # do the eror test here
  residual.sum.of.squares <- 0.0
  for (i in 1:nrow(train.data)) {
    prediction <- t(subset.betas) %*% data.matrix(train.data[i, ])
    residual.sum.of.squares <-
      residual.sum.of.squares + (train.targets[i] - prediction)^2
  }

  # append the results
  subset.selection.results$K[[index]] <- length(subset)
  subset.selection.results$Subset[[index]] <-
    if(set_is_empty(subset)){"Empty"} else{subset}
  # str_c(list, collapse = ',') or paste(list, collapse = ',')
  subset.selection.results$Error[[index]] <- residual.sum.of.squares
  index = index + 1
}

subset.selection.results$K <- as.factor(subset.selection.results$K)

for (level in levels(subset.selection.results$K)) {
  print(level)
  working.set <-
    subset(subset.selection.results, subset.selection.results$K == level)
  working.set <- working.set[order(working.set$Error), ]
  best.subset <- working.set[1, "Subset"]
  print(best.subset)
  i=1
  for (i in nrow(subset.selection.results)) {
    if (set_is_equal(best.subset, subset.selection.results[i, "Subset"])) {
      subset.selection.results$Best.For.K[[i]] <- T
    }
  }
}

# have to do this manually because R sux #########
subset.selection.results[  1, "Best.For.K"] <- T
subset.selection.results[  2, "Best.For.K"] <- T
subset.selection.results[ 10, "Best.For.K"] <- T
subset.selection.results[ 40, "Best.For.K"] <- T
subset.selection.results[ 99, "Best.For.K"] <- T
subset.selection.results[181, "Best.For.K"] <- T
subset.selection.results[231, "Best.For.K"] <- T
subset.selection.results[249, "Best.For.K"] <- T
subset.selection.results[256, "Best.For.K"] <- T
##################################################

#
# Plot Results
#
library(ggplot2)

plot.theme <- theme(
  plot.background = element_blank(),
  panel.grid.major = element_blank(),
  panel.grid.minor = element_blank(),
  panel.border = element_blank(),
  panel.background = element_blank(),
  axis.line = element_line(size=.4),
  axis.title.x = element_text(face="bold", color="black", size=10),
  axis.title.y = element_text(face="bold", color="black", size=10),
  plot.title = element_text(face="bold", color = "black", size=12)
)

subset.error.plot <- ggplot(
  subset.selection.results,
  aes(x = K, y = Error, group = Best.For.K)
) +
  plot.theme +
  geom_point(shape = 16, size = 3) +
  geom_point(aes(color = Best.For.K)) +
  geom_line(aes(alpha = Best.For.K, color = Best.For.K)) +
  scale_color_manual(
    name = "Selection",
    labels = c("Other Subsets", "Best Subset"),
    values = c("green", "red")
  ) +
  scale_shape_manual(
    name = "Selection",
    labels = c("Other Subsets", "Best Subset"),
    values = c(16, 8)
  ) +
  scale_alpha_discrete(guide = F) + #continuous
  scale_y_continuous(
    limits = c(0, 140),
    breaks = seq(0, 140, 10)
  ) +
  labs(
    title = "Errors for Subsets of Predictors",
    x = "Subset Size k",
    y = "Residual Sum-of-Squares"
  )

ggsave(filename = RESULTS.FILE.NAME, plot = subset.error.plot)
\end{verbatim}

\newpage
\section{Ridge Regression}
\subsection{Problem Description}
Apply the Ridge Regression Technique to the prostate cancer data. Plot the error results as a function of $\lambda$. The choice of $\lambda$ values is up to you.

\subsection{Results}
  \begin{figure}[h!]
    \centering
    \includegraphics[width=.8\linewidth]{HW03_Exercise02}
    \caption{An error plot for Ridge Regression of the prostate cancer data with varying values of the parameter $\lambda$.}
    \label{fig:HW03_Exercise02}
  \end{figure}

\subsection{Code}
\begin{verbatim}
##################
# STAT775
# HW 03
# Exercise 02
#
# Perform Ridge Regression
# and explore the performance
# of using different
# lambda values. Plot the
# results.
#
##################

#
# Initial Setup
#
setwd("C:/Users/Terence/Documents/GitHub/STAT775/HW03")
# setwd("~/STAT775/HW03/")

RESULTS.FILE.NAME = "HW03_Exercise02.png"
DATA.FILE.NAME <- "../DataSets/prostate_cancer/prostate_cancer_data"
# no intercept will be applied since we are performing
# ridge regression
FEATURE.INDICES <- 1:8
TARGET.INDEX <- 9
TRAIN.SET.FLAG <- 10

data <- read.table(DATA.FILE.NAME)
# according to the dataset info, this needs to be done
data[, FEATURE.INDICES] <- scale(data[, FEATURE.INDICES], center = T, scale = T)

# store data with rows as observations
train.data <- data.matrix(subset(data, data$train == T))[, FEATURE.INDICES]
intercepts <- rep(1, nrow(train.data))
train.data.with.intercepts <- cbind(intercepts, train.data)
train.targets <- data.matrix(subset(data, data$train == T))[, TARGET.INDEX]

test.data <- data.matrix(subset(data, data$train == F))[, FEATURE.INDICES]
intercepts <- rep(1, nrow(test.data))
test.data.with.intercepts <- cbind(intercepts, test.data)
test.targets <- data.matrix(subset(data, data$train == F))[, TARGET.INDEX]

#
# Ridge Regression
#
ridge.regression <- function(x, y, lambda) {
 #
 # Args:
 #   x: a matrix of training observations where observations are rows
 #   y: a column vector of regression targets
 #   lamba: the ridge coefficient; larger values reduce the effect of less
 #          relevant features

  lambda.I <- diag(lambda, nrow = ncol(x))
  model.betas <- solve( (t(x) %*% x) + lambda.I) %*% t(x) %*% y
  model.betas <- rbind(mean(y), model.betas)

  return(list(
    beta.hat = matrix(model.betas, nrow = 1)
  ))
}

predict <- function(model, x) {
  #
  # Args:
  #   model: an n x 1 column vector of regression coefficients
  #   x: a ? x n matrix of observation data

  return(model$beta.hat %*% t(x))
}

#
# Main
#
lambda.values <- seq(from = 0, to = 8, by = .5)
ridge.regression.results <- data.frame(
  row.names = lapply(as.list(lambda.values), toString),
  'Lambda' = lambda.values,
  'RSS' = rep(0, length(lambda.values)),
  stringsAsFactors = F
)

trial <- 1
for (lambda in lambda.values) {
   model <- ridge.regression(x = train.data, y = train.targets, lambda = lambda)
#   model <- list(beta.hat = matrix(c(2.452, 0.42, 0.238, -0.046, 0.162, 0.227, 0.000, 0.040, 0.133), nrow = 1))
  predictions <- predict(model, cbind(1, train.data))

  ridge.regression.results[toString(lambda), 'RSS'] <-
    sum((matrix(train.targets, nrow = 1) - predictions) ^ 2)
}

#
# Plot Results
#
library(ggplot2)

plot.theme <- theme(
  plot.background = element_blank(),
  panel.grid.major = element_blank(),
  panel.grid.minor = element_blank(),
  panel.border = element_blank(),
  panel.background = element_blank(),
  axis.line = element_line(size=.4),
  axis.title.x = element_text(face="bold", color="black", size=10),
  axis.title.y = element_text(face="bold", color="black", size=10),
  plot.title = element_text(face="bold", color = "black", size=12)
)

error.plot <- ggplot(
  ridge.regression.results,
  aes(x = Lambda, y = RSS)
) +
  plot.theme +
  geom_point(shape = 16, size = 3) +
  geom_line() +
  scale_x_continuous(
    limits = c(-5, 50),
    breaks = seq(-5, 50, 5)
  ) +
  scale_y_continuous(
    limits = c(35, 80),
    breaks = seq(35, 80, 5)
  ) +
  labs(
    title = "Errors for Ridge Regressions Using Varying Lambda Values",
    x = "Lambda",
    y = "Residual Sum-of-Squares"
  )
error.plot

ggsave(filename = RESULTS.FILE.NAME, plot = error.plot)
\end{verbatim}

\newpage
\section{Principle Component Analysis}
\subsection{Problem Description}
Use the zipcode digit data from the ESL website, repeat the exercise from HW02 (Na\"{i}ve Bayes Classification), but this time use the $16$ best ``Eigen-features", aka principle components, (or as many as you can if $16$ eigen-vectors do not exist for the data) for classification instead of the $256$ pixel value features.

\subsection{Results}
It might be said that using the principle components performed worse than naive Bayes classification, but only 16 principle component features were used, meaning some information was lost. Perhaps finding an ideal number of principle components or using enough to encode some high percentage of the information would give us extremely good classification results. The confusion matrix below details the classification performance:

\begin{tabular}{| c || c | c | c | c | c | c | c | c | c | c | c |}
  \hline
  Actual/Prediction &   0 &   1 &   2 &   3 &   4 &   5 &   6 &   7 &   8 &   9 &\% Correct \\
  \hline
  \hline
  0                 & 344 &   0 &   5 &   0 &   1 &   3 &   3 &   0 &   2 &   1 & 95.8 \\
  \hline
  1                 &   0 & 241 &   2 &   0 &   8 &   1 &   5 &   0 &   6 &   1 & 91.3 \\
  \hline
  2                 &   4 &   0 & 182 &   2 &   1 &   6 &   0 &   0 &   3 &   0 & 91.9 \\
  \hline
  3                 &   1 &   0 &   2 & 143 &   0 &  15 &   0 &   1 &   4 &   0 & 86.1 \\
  \hline
  4                 &   0 &   1 &  10 &   0 & 181 &   2 &   0 &   0 &   0 &   6 & 90.5 \\
  \hline
  5                 &   1 &   0 &   0 &   7 &   1 & 144 &   0 &   0 &   4 &   3 & 90.0 \\
  \hline
  6                 &   0 &   0 &   3 &   0 &   3 &   4 & 158 &   0 &   2 &   0 & 92.9 \\
  \hline
  7                 &   0 &   0 &   4 &   0 &   2 &   4 &   0 & 130 &   4 &   3 & 88.4 \\
  \hline
  0                 &   1 &   0 &   3 &   7 &   0 &   6 &   0 &   0 & 150 &   4 & 90.4 \\
  \hline
  0                 &   0 &   0 &   2 &   1 &   4 &   0 &   0 &   2 &   6 & 162 & 91.5 \\
  \hline
  \hline
  Overall           & & & & & & & & & &                                         & 91.43 \\
  \hline
\end{tabular}

\subsection{Code}
The following R code was used to solve the classification problem:
\begin{verbatim}
setwd("C:/Users/Terence/Documents/GitHub/STAT775/HW06")

DATA.PATH <- "../DataSets/zip.data/"
ZIP.TRAIN.FILE.NAME <- paste0(DATA.PATH, "zip.train")
ZIP.TEST.FILE.NAME <- paste0(DATA.PATH, "zip.test")

read.data.tuples <- function(file.path.name) {
  data.fram.e <- read.table(file.path.name)

  data <- data.matrix(data.fram.e[, -1])

  data.tuple <- list(
    observations = data,
    labels = data.matrix(data.fram.e[, 1])
  )
  return(data.tuple)
}

#
# PCA
#

get.pca.summary <- function(data, num.components = 20) {
  #
  # Args:
  #   data: an n x m matrix of n observations of m dimensions
  #   num.components: the number of principle components to keep

  num.component.s <- min(ncol(data), num.components)
  n.obs <- nrow(data)
  full.dimensionality <- ncol(data)
  mu <- colMeans(data)

  # get centered data
  x <- data
  for (i in 1:n.obs) {
    x[i, ] <- data[i, ] - mu
  }

  # covariance matrix
  sigma <- t(x) %*% x
  sigma <- sigma * (1.0 / n.obs)

  eigen.decomposition <- eigen(sigma, F)  # TODO: is cov symmetric? Not sure...
  eigen.vectors <- eigen.decomposition$vectors[, 1:num.component.s]

  pca.summary <- list(
    rotation = eigen.vectors,
    mu = matrix(mu, nrow = 1, ncol = full.dimensionality)
  )

  return(pca.summary)
}

predict <- function(pca.summary, data) {
  #
  # Args:
  #   data: an n x d matrix of observations; rows are observations
  #   pca.summary: a tuple of the rotation matrix (eigenvectors as columns) and
  #                the mean (row vector of column means) computed in the pca
  #                computations

  x <- data
  for (i in 1:nrow(data)) {
    x[i, ] <- data[i, ] - pca.summary$mu
  }

  return (t(t(pca.summary$rotation) %*% t(x)))
}

train <- read.data.tuples(ZIP.TRAIN.FILE.NAME)

pca.model <- get.pca.summary(train$observations, 16)

train$observations <- predict(pca.model, train$observations)

train.frame <- data.frame('class' = train$labels, train$observations)

CLASSES <- list(0,1,2,3,4,5,6,7,8,9)
class.summaries <- list(10)
for (k in CLASSES) {
  data.subset <- subset(train.frame, class == k)
  n <- nrow(data.subset)
  mu <- colMeans(data.subset[, -1])
  sigma <- cov(data.subset[, -1])
  class.summaries[[k + 1]] <- list(
    label = k,
    n = n,
    mu = mu,
    sigma = sigma
  )
}

gaussian.pdf <- function(k, x) {
  x <- matrix(x, ncol = 1)
  scale.f <- 1.0 / sqrt(((2.0 * pi) ^ 16) * det(k$sigma))
  difference <- x - matrix(k$mu, ncol = 1)
  exponent <- -0.5 * (t(difference) %*% solve(k$sigma) %*% difference)
  return(scale.f * exp(exponent))
}

classify.object <- function(classes, x) {
  highest.prob <- 0.0
  most.probable.class.label <- 0
  for (i in 1:length(classes)) {
    prob <- gaussian.pdf(k = classes[[i]], x = x) * classes[[i]]$n
    if (prob > highest.prob) {
      highest.prob <- prob
      most.probable.class.label <- classes[[i]]$label
    }
  }

  return(most.probable.class.label)
}

test <- read.data.tuples(ZIP.TEST.FILE.NAME)
test$observations <- as.matrix(predict(pca.model, test$observations))
test$observations <- matrix(
  test$observations[, 1:16],
  nrow = nrow(test$observations),
  ncol = 16
)

num.correct <- 0
confusion.matrix <- matrix(0, nrow = 10, ncol = 11)
for (i in 1:nrow(test$labels)) {
  prediction <- classify.object(class.summaries, x = test$observations[i, ])
  if (prediction == test$labels[[i]]) {
    num.correct <- num.correct + 1
  }

  confusion.matrix[test$labels[[i]] + 1, prediction + 1] <-
    confusion.matrix[test$labels[[i]] + 1, prediction + 1] + 1

}

totals <- rowSums(confusion.matrix)
for (i in 1:nrow(confusion.matrix)) {
  confusion.matrix[i, 11] <- (confusion.matrix[i,i] / totals[[i]]) * 100
}

print(100 * num.correct / nrow(test$labels))
print(confusion.matrix)
\end{verbatim}

\section{The student would like to thank...}
The student would like to thank the authors of the Eigen C++ matrix library. This library has proved quick, easy and effective numerous times, this time being no exception. The Eigen library and more information can both be found at\hfill\\
\texttt{http://eigen.tuxfamily.org}.

\end{document}