\documentclass[12pt]{article}
\usepackage[usenames]{color} %used for font color
\usepackage{amsmath, amssymb, amsthm}
\usepackage{wasysym}
\usepackage[utf8]{inputenc} %useful to type directly diacritic characters
\usepackage{graphicx}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{float}
\usepackage{mathtools}
\usepackage [english]{babel}
\usepackage [autostyle, english = american]{csquotes}
\MakeOuterQuote{"}
\graphicspath{ {./} }
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\prob}{\mathbb{P}}
\newcommand{\degrees}{^{\circ}}
\DeclarePairedDelimiter\ceil{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}

\author{Tianshuang (Ethan) Qiu}
\begin{document}
\title{Math 104, HW11}
\maketitle
\newpage

\section{Q1}
Assume that $\lim_{n \to \infty} \sup \{|f_n(x)-f(x)|\} = 0$, then let $\epsilon > 0$. By our assumption there exists $N \in \R$ such that $\sup \{ |f_n(x) - f(x)| \}<\epsilon$ for all $x \in S, n > N$. Since the supremum is less than $\epsilon$, every member in that set must also be less than epsilon, which is the definition for uniform convergence. Therefore $f_n \to f$ uniformly.
\newline
Now assume that $f_n \to f$ uniformly. Assume that $\lim_{n \to \infty} \sup \{|f_n(x)-f(x)|\} = k > 0$, then let $\epsilon = k/2$. Since $f_n \to f$ uniformly for any $\epsilon > 0$ we can find $N \in \R$ such that $|f_m(x)-f(x)|<\epsilon$ for all $x \in S, m > N$.
Now consider this $m$. It implies that the supremum of the set $\{|f_m(x)-f(x)|\}$ is less than or equal to $k/2$, which is less than the supremum of the set as $n$ approaches infinity.
Since we know that $\lim \sup y_n \leq \sup y_n$, we have reached a contradiction, our assumption is incorrect and $\lim_{n \to \infty} \sup \{|f_n(x)-f(x)|\} = 0$. Thus we have proven the converse.
\newpage

\section{Q2}
We claim that $f_n \to f: [0,1] \to \R$, $f(x)=x^2$ uniformly.
\newline
Let $\epsilon > 0$, choose $N = \max \{1, \frac{4}{\epsilon}\}$, consider $n > N$. Take $x \in [0,1]$, $|f_n(x)-f(x)| = |(x-\frac{1}{n})^2-x^2|$
$$= |x^2+\frac{1}{n^2}-\frac{2x}{n}-x^2| = |\frac{1}{n^2}-\frac{2x}{n}|$$
Since we know that $n\geq 1$ and $x \in [0,1]$, we know that $\frac{1}{n^2} < \frac{2x}{n}$, so their difference is (nonstrictly) less than $\frac{2x}{n}$
\newline
Now since $n > N$, $\frac{2x}{n} \leq \frac{2}{n} < \frac{\epsilon}{2} < \epsilon$.
Therefore $|f_n(x)-f(x)|<\epsilon$, and we have proven that $f_n \to f$ uniformly.
\newpage

\section{Q3}
\subsection{Pointwise Convergent}
Let $\epsilon > 0$ and $x \in [0,1]$, choose $N = \frac{\epsilon}{2(1-x)}$. For all $n > N$, consider $|f_n(x)-f(x)|= |nx^n(1-x)-0| \leq |Nx^n(1-x)| \leq |\frac{\epsilon}{2}x^n| \leq |\epsilon/2| < \epsilon$
\newline
Thus given any point in $[0,1]$ and $\epsilon$, we can find a $N$ such that $|f_n(x)-f(x)|<\epsilon$

\subsection{Not Unif. Convergent}
Choose $\epsilon = \frac{1}{10e}$, let $N \in \R$. Consider $m > N$. Since $f_m(x)$ is a polynomial we know that it is differentiable. Furthermore, since $f_m(0)=0$, $f_m(1)=0$, the maximum of $f_m(x)$ in $[0,1]$ must appear in $(0,1)$, then its derivative must be 0. $f_m(x)=mx^m(1-x)=mx^m-mx^{m+1}$.
$$f_m'(x)= m^2x^{m-1}-m(m+1)x^{m} = 0$$
$$m^2x^{m-1}=m^2x^m+mx^m$$
$$m^2 = m^2x+mx$$
$$x = \frac{m}{m+1}$$
We find that $f_m'(x)=0$ when $x = \frac{m}{m+1}$, now we attempt to find $f_m(x)$ at this $x$. $f_m(x)=(\frac{m}{m+1})^m \frac{m}{m}$
\newline
Since we are intrested in the behavior of this value as $N \to \infty$, we know that the limit of the maximum of this function is $\lim_{m \to \infty}(\frac{m}{m+1})^m = \frac{1}{e}$. Recall that our $\epsilon = \frac{1}{10e}$ which is less than the maximum of the function $\lim_{m \to \infty} f_m$, therefore there is no $N$ that can satisfy the requirements for uniform convergence.
Thus it is not uniformly convergent.
$\blacksquare$
\newpage

\section{Ross 25.7}
Since $|\cos(k)| \leq 1$, $|\frac{1}{n^2}\cos(nx)| \leq \frac{1}{n^2}$, and since $\sum \frac{1}{n^2}$ converges by the $p$ series test, $\sum \frac{1}{n^2}\cos(nx)$ converges uniformly as well by the M-test.
\newline
Furthermore, since $\frac{1}{n^2}\cos(nx)$ is continuous for all $n \in \N$, and it converges uniformly, it must converge to a continuous function.
\newline
$\blacksquare$
\newpage

\section{Q5}
\subsection{Pointwise Convergent}
Let $x \in (0,1), \epsilon > 0$, pick $N = \log_x((1-x)\epsilon) = \frac{\ln((1-x)\epsilon)}{\ln(x)}$, consider an arbitrary $n > N$. $f_n(x) = \sum_{k=0}^n g_k =  \sum_{k=0}^n x_k$
$$|f_n(x)-f_x| = |(\sum_{k=0}^n x_k) - \frac{1}{1-x}|$$
We apply the formula for a geometric sum to the former, since $x<1$, $(1-x^n)$ is positive
$$= |\frac{1-x^n}{1-x} - \frac{1}{1-x}| = |\frac{-x^n}{1-x}| = \frac{x^n}{1-x}$$
Now recall our $N$, we know that the difference is strictly less than $\frac{x^N}{1-x} = \frac{\epsilon (1-x)}{1-x} = \epsilon$
\newline
Thus we have proven that $\sum_k=0^\infty g_k \to f$ pointwise.

\subsection{Not Uniformly Convergent}
We will use the fact that finite sums of finite numbers cannot be infinite to create a contradiction.
\newline
First for our function $f$, as $x \to 1$, $(1-x) \to 0$, and thus $\frac{1}{1-x} \to infty$. Assume that $f_n \to f$ uniformly. Then choose $\epsilon = 1$, by our assumption there exists $N \in \R$ such that for all $m > n$, $|f_m(x)-f(x)|<\epsilon$ for all $x \in (0,1)$.
\newline
Since $f_m$ is the sum of $m+1$ terms starting from $x^0$, and $0 < x < 1$, we know that $\lim_{x \to 1} f_m \leq (m+1) < \infty$. Now we just need to find this point.
\newline
We are looking for a value such that the difference is equal to our epislon: $f(x)=m+1+1 = m+2$. Now we try to find this point in $(0,1)$:
$$\frac{1}{1-x}=m+2$$
$$1 = (1-x)(m+2) = m +2 -xm -2x$$
$$x(m+2) = m+1$$
$$x = \frac{m+1}{m+2}$$
Since $m>0$, $x \in (0,1)$, and at this point, $|f_m(x)-f(x)|=\epsilon$, and we have a contradiction. Therefore our assumption is incorrect and $\sum_k=0^\infty g_k \not \to f$ uniformly.
\newpage

\section{Ross 25.3}
\subsection{a}
We claim that the series of functions converge go $f: \R \to \R$, $f(x) = \frac{1}{2}$. Consider $\lim_{n \to \infty} \sup \{ |f_n(x)-f(x)|\} = \lim_{n \to \infty} \sup \{ |\frac{n + \cos x}{2n + \sin^2x}-\frac{1}{2}|\}$
\newline
Since both cosine and sine are non-strictly between 0 and 1, the greatest $\frac{n + \cos x}{2n + \sin^2x}$ can be is $\frac{n+1}{2n}$, and the smallest it can be is $\frac{n}{2n+1}$, therefore we have the inequality:
$$|\frac{n + \cos x}{2n + \sin^2x}-\frac{1}{2}| \leq \max \{\frac{n+1}{2n}, \frac{n}{2n+1}\} - \frac{1}{2}$$
Now consider $\lim_{n \to \infty}\frac{n+1}{2n} = \frac{1}{2}$ since the difference $|\frac{n+1}{2n}-\frac{1}{2}| = |\frac{1}{2n}|$ approaches 0 as $n \to \infty$. Similarly $\lim_{n \to \infty}\frac{n}{2n+1} = \frac{1}{2}$
\newline
Therefore we know that $\max \{\frac{n+1}{2n}, \frac{n}{2n+1}\} - \frac{1}{2} = 0$, thus
$$\lim_{n \to \infty} \sup \{ |f_n(x)-f(x)|\} = 0$$
From our theorem in Q1 we know that $f_n \to f$ uniformly.

\subsection{b}
By the theorem that if $f_n$ are continuous and $f_n \to f$ uniformly, then $\lim_{n \to \infty} \int_a^b f_n(x) dx= \int_a^b f(x)dx$, we know that we can subsitute the integral for
$$\int_2^7 \frac{1}{2}dx = \frac{7}{2}-\frac{2}{2} = 2.5$$
\newpage

\section{Ross 23.1}
\subsection{a}
We compute $\lim \sup (a_n)^\frac{1}{n} = \lim \sup (n^2)^\frac{1}{n} = \lim \sup (n^\frac{1}{n})^2 = 1$
Therefore $R = 1/1 = 1$
\newline
Now we check -1: $\sum (-1)^nn^2$ does not converge by the alternating series test.
\newline
For $x=1$, $\lim_{n \to \infty} n^2 \not 0$, therefore it diverges.
\newline
Finally, the intereval of convergence is found to be $(-1,1)$

\subsection{c}
We compute $\lim \sup (a_n)^\frac{1}{n} = \lim \sup (\frac{2^n}{n^2})^\frac{1}{n} = \lim \sup \frac{2^{n\frac{1}{n}}}{1} = 2$
Therefore $R = 1/2 = \frac{1}{2}$
\newline
Now we check $-\frac{1}{2}$: $\sum (-1)^n\frac{1}{n^2}$ converges by the alternating series test.
\newline
For $x= \frac{1}{2}$, $\sum \frac{1}{n^2}$ converges by p-series test.
\newline
Finally, the intereval of convergence is found to be $[-\frac{1}{2},\frac{1}{2}]$

\subsection{e}
We compute $\lim \sup (a_n)^\frac{1}{n} = \lim \sup (\frac{2^n}{n!})^\frac{1}{n} = \lim \sup \frac{2}{n!^{\frac{1}{n}}} = 0$
Therefore $R = \infty$
\newline
The interval of convergence is defined to be $(-\infty, \infty)$

\subsection{g}
We compute $\lim \sup (a_n)^\frac{1}{n} = \lim \sup (\frac{3^n}{n4^n})^\frac{1}{n} = \lim \sup (\frac{3}{4})^{n\frac{1}{n}}\frac{1}{1} = \frac{3}{4}$
Therefore $R = 1/(3/4) = \frac{4}{3}$
\newline
Now we check $-\frac{4}{3}$: $\sum (-1)^n\frac{1}{n}$ converges by the alternating series test.
\newline
For $x= \frac{4}{3}$, $\sum \frac{1}{n}$ does not converge by p-series test.
\newline
Finally, the intereval of convergence is found to be $[-\frac{4}{3},\frac{4}{3})$

\end{document}


\documentclass[11pt, a4paper]{article}

\usepackage{tikz}
\usetikzlibrary{shapes,arrows}
\usepackage{placeins}
\usepackage{amsmath}
\usepackage{mathrsfs}
\usepackage{bm}
\usepackage{amssymb}

\begin{document}

\title{SUPPORT VECTOR MACHINES}
\date{}
\maketitle

Support Vector Machines (SVMs) are one of the most powerful supervised learning algorithms which work on the idea of finding the `safest' separator hyperplane (one with the largest `margin') to divide the labelled hyperspace.

\section{Optimal Linear Separator}

Given training data of the form $(x_1, y_1),\ (x_2, y_2),\ ...\ , (x_n, y_n)$ where $y_i \in \{+1, -1\}\ \forall\ x_i \in \mathbf{X}$, what is the optimal linear separator that can separate the positives from the negatives?

\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]
		\tikzstyle {margin} = [dashed]

		\draw[-latex] (0,0) -- (6,0) node[right]{};
		\draw[-latex] (0,0) -- (0,5) node[left]{};

		\draw[point line] (1.7, 1) -- (2.3, 1);
		\draw[point line] (.7, 2) -- (1.3, 2);

		\draw[point line] (2.7, 3) -- (3.3, 3);
		\draw[point line] (3, 2.7) -- (3, 3.3);

		\draw[point line] (2.7, 4) -- (3.3, 4);
		\draw[point line] (3, 3.7) -- (3, 4.3);

		\draw[point line] (3.7, 3) -- (4.3, 3);
		\draw[point line] (4, 2.7) -- (4, 3.3);

		\draw[margin] (4, 0.5) -- (.5, 4);
		\draw (4, 0.5) node [below right] {$H_1$};

		\draw[margin] (2.5, 4.5) -- (2.5, 0.5);
		\draw (2.5, 0.5) node [below] {$H_2$};
	\end{tikzpicture}
\end{figure}

It is clear that $H_1$ is a better linear separator as opposed to $H_2$. It is expected that $H_1$ will make fewer errors for predicting unseen points as comapred to $H_2$.

\section{Derivation}

Let $\vec{\mathbf{w}}$ be a vector perpendicular to a linear separator. The following figure visualizes the constraint that the separator is subjected to.

\FloatBarrier\clearpage
\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {margin} = [dashed]
		\tikzstyle {vector} = [very thick,->, >=stealth]

		\draw[-latex] (0,0) -- (5,0) node[right]{};
		\draw[-latex] (0,0) -- (0,5) node[left]{};

		\draw[margin] (4, 0.5) -- (.5, 4);

		\draw[vector] (0, 0) -- (2.25, 2.25);
		\draw (2.25, 2.25) node [above right] {$\vec{\mathbf{w}}$};

		\draw[vector] (0, 0) -- (1, 3.5);
		\draw (1, 3.5) node [above right] {$\vec{\mathbf{x}}$};

		\draw[vector] (2.25, 2.25) -- (1, 3.5);
		\draw (2, 2.75) node [below left] {$\vec{\mathbf{x}} - \vec{\mathbf{w}}$};

	\end{tikzpicture}
\end{figure}

\begin{align*}
	(\vec{\mathbf{x}} - \vec{\mathbf{w}}).\vec{\mathbf{w}} = 0   \\
	\vec{\mathbf{x}}.\vec{\mathbf{w}} - |\vec{\mathbf{w}}|^2 = 0 \\
	\vec{\mathbf{x}}.\vec{\mathbf{w}} - b = 0                    \\
\end{align*}

Assume two hyperplanes parallel to the separator hyperplane, one on each side of the separator, such that they cut through the closest training point(s) on the corresponding side. The equations for these `support' hyperplanes are,

\begin{align*}
	\vec{\mathbf{x_{+}}}.\vec{\mathbf{w}} = b + \delta \\
	\vec{\mathbf{x_{-}}}.\vec{\mathbf{w}} = b - \delta \\
\end{align*}

Note that the subscipts $+$ and $-$ denote the positive and negative side respectively. The equations are still satisfied if we scale $\vec{\mathbf{w}}$ and ($b$ and $\delta$) by the same amount. Hence, $\delta = 1$ is set to avoid this difficutly.

\begin{align*}
	\vec{\mathbf{x_{+}}}.\vec{\mathbf{w}} = b + 1 \\
	\vec{\mathbf{x_{-}}}.\vec{\mathbf{w}} = b - 1 \\
\end{align*}

The length of margin between the $+$ support hyperplane and the $-$ support hyperplane is calculated as follows,

\begin{align*}
	margin & = (\vec{\mathbf{x_{+}}} - \vec{\mathbf{x_{-}}}) . \frac{\vec{\mathbf{w}}}{|\vec{\mathbf{w}}|} \\
	       & = \frac{2}{|\vec{\mathbf{w}}|}                                                                \\
\end{align*}

All positive and negative points can be assumed to follow,

\begin{align*}
	\vec{\mathbf{x_{+}}}.\vec{\mathbf{w}} \geq b + 1 \\
	\vec{\mathbf{x_{-}}}.\vec{\mathbf{w}} \leq b - 1 \\
\end{align*}

These equations can be combined together to form,

\begin{align*}
	y \times (\vec{\mathbf{x}}.\vec{\mathbf{w}} - b) - 1 \geq 0 \\
\end{align*}

or,

\begin{align*}
	y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 \geq 0\ \forall\ i \\
\end{align*}

The problem can be formulated as follows,

\begin{align*}
	\text{maximize}\ \frac{2}{\vec{\mathbf{|w|}}}\ \text{subject to}\
	y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 \geq 0\ \forall\ i \\
\end{align*}

or,

\begin{align*}
	\text{minimize}\ \frac{1}{2} \times {\vec{\mathbf{|w|}}^2}\ \text{subject to}\
	y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 \geq 0\ \forall\ i    \\
\end{align*}

This form is called the primal form and it is a quadratic optimization problem but not readily kernelized. The primal form is converted the dual form by using a Langrangian.

\begin{align*}
	\mathscr{L}(\vec{\mathbf{w}},\ b,\ \bm{\alpha}) = \frac{1}{2} \times \vec{\mathbf{|w|}}^2 - \sum\limits_{i = 1}^N \alpha_i \times [ y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 ]
\end{align*}

Taking derivative w.r.t $\vec{\mathbf{w}}$ and $b$, equating to zero and solving, these two equations arise.

\begin{align}
	\vec{\mathbf{w}}^*                        & = \sum\limits_{i = 1}^N \alpha_i \times y_i \times \vec{\mathbf{x_i}} \\
	\sum\limits_{i = 1}^N \alpha_i \times y_i & = 0
\end{align}

Now, for a point $i$ on a support hyperplane,

\begin{align*}
	y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}}^* - b^*) = 1              \\
	y_i \times y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}}^* - b^*) = y_i \\
	\vec{\mathbf{x_i}}.\vec{\mathbf{w}}^* - b^* = y_i                         \\
	b^* = y_i - \vec{\mathbf{x_i}}.\vec{\mathbf{w}}^*
\end{align*}

Inserting (1) and (2) back into langragian, the dual form takes the form of,

\begin{align*}
	\text{maximize}\ \mathscr{L}(\vec{\mathbf{w}},\ b,\ \bm{\alpha}) & = \sum\limits_{i = 1}^N \alpha_i                                                                                \\
	                                                                 & \ \ \ - \frac{1}{2} \times \sum\limits_{i, j} \alpha_i \times \alpha_j \times y_i \times y_j \times \vec{\mathbf{x_i}} . \vec{\mathbf{x_j}} \\
	\text{subject to}\ \sum\limits_{i = 1}^N \alpha_i \times y_i     & = 0\ \ \ \text{and}                                                                                \\
	\alpha_i                                                         & >= 0\ \forall\ i
\end{align*}

The dual formation is also a quadratic optimization problem but is readily kernelized through the substitution $\vec{\mathbf{x_i}} . \vec{\mathbf{x_j}} \rightarrow k(\mathbf{x_i}, \mathbf{x_j})$.

The primal problem is convex and the dual formation is concave and due to duality theroy the duality gap is zero meaning the solutions for both (minima for primal and maxima for dual) the problems are the same.

An interesting equation which holds at the solution is,

\begin{align*}
	\alpha_i \times [y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1] = 0\ \forall\ i
\end{align*}

At the points not on the support hyperplanes, $y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 > 0$ or $y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 < 0$ and hence $\alpha_i$ must be zero for each such point. On the other hand, for the points on the support hyperplanes, $y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 = 0$ so $\alpha_i \geq 0$ for each such point.

The training vectors on the support hyperplanes for which $\alpha_i > 0$ are called support vectors.

\section{Non Linearly Separable Case}

The equations for the non separable case are,

\begin{align*}
	\vec{\mathbf{x_{i}}}.\vec{\mathbf{w}} & \geq b + 1 - \epsilon_i\ \forall\ y_i = +1 \\
	\vec{\mathbf{x_{i}}}.\vec{\mathbf{w}} & \leq b - 1 + \epsilon_i\ \forall\ y_i = -1 \\
	\epsilon_i                            & \geq 0\ \forall\ i
\end{align*}

The variables $\epsilon_i$s allow for violation of the constraint. Hence, we must penalize for that. One penalty function is $C \times \sum\limits_i \epsilon_i$ where C is the tradeoff between margin and penalty. To be on the wrong side of the hyperplane $\epsilon_i$ will be greater than one, so $\sum\limits_i \epsilon_i$ is an upper bound on number of violations.

The primal problem thus becomes,

\begin{align*}
	\text{minimize}\ \frac{1}{2} \times {\vec{\mathbf{|w|}}^2} + C \times \sum\limits_i \epsilon_i &                    \\ \text{subject to}\ y_i \times (\vec{\mathbf{x_i}}.\vec{\mathbf{w}} - b) - 1 + \epsilon_i &\geq 0\ \forall\ i\ and \\
	\epsilon_i                                                                                & \geq 0\ \forall\ i
\end{align*}

The dual formulation is also surprisingly similar,

\begin{align*}
	\text{maximize}\ \mathscr{L}(\vec{\mathbf{w}},\ b,\ \bm{\alpha}) & = \sum\limits_{i = 1}^N \alpha_i                                                                                \\
	                                                                 & \ \ \ - \frac{1}{2} \times \sum\limits_{i, j} \alpha_i \times \alpha_j \times y_i \times y_j \times \vec{\mathbf{x_i}} . \vec{\mathbf{x_j}} \\
	\text{subject to}\ \sum\limits_{i = 1}^N \alpha_i \times y_i     & = 0\ \ \ \text{and}                                                                                \\
	0 <= \alpha_i                                                    & <= C\ \forall\ i
\end{align*}

The multipliers $\alpha_i$s are now constrained by C. Note that the dual formation is still readily kernelized.

\section*{The Kernel Trick}

Kernel functions are functions which let you calculate dot product in higher dimensions given vectors in lower dimensions. Sincel SVMs use only dot products, kernel function effectively solve the problem in higher dimensions without expliclity transforming each point into the higher dimension. This leads SVMs to be very time efficient algorithms even when solving the problem in higher dimensions than the input space.

Say there is a $\mathbb{R}^2 \rightarrow \mathbb{R}^3$ transformation function $\phi$ such that $\phi(i, j) = (i^2, \sqrt{2} \times i \times j, j^2)$, the kernel function $K(\vec{\mathbf{x}}, \vec{\mathbf{z}}) = (\vec{\mathbf{x}}.\vec{\mathbf{z}})^2$ works because,

\begin{align*}
	\phi(\vec{\mathbf{x}}).\phi(\vec{\mathbf{z}}) & = (x_i^2, \sqrt{2} \times x_i \times x_j, x_j^2).(z_i^2, \sqrt{2} \times z_i \times z_j, z_j^2) \\
	                                              & = x_i^2 \times z_i^2 + 2 \times x_i \times x_j \times z_i \times z_j + x_j^2 \times z_j^2       \\
	                                              & = (x_i \times z_i + x_j \times z_j)^2                                                           \\
	                                              & = (\vec{\mathbf{x}}.\vec{\mathbf{z}})^2                                                         \\
	                                              & = K(\vec{\mathbf{x}}, \vec{\mathbf{z}})                                                         \\
\end{align*}

\section{Comments}

\begin{itemize}
	\item SVMs are helpful in tasks like text classification, image classification, digit recognition etc.
	\item SVMs can work with limited amount of training data and high dimensional data effectively.
	\item The maximal margin approach reduces overfitting allowing SVMs to generalize better.
	\item SVMs are typically quite robust against noise.
\end{itemize}

\end{document}

Support Vector Machines/Support Vector Machines.tex


\section{The Rate of Change of a Function}\label{sec:Slope}

Suppose that $y$ is a function of $x$, say $y = f(x)$.
It is often useful to know how sensitive the value of $y$
is to small changes in $x$.

\begin{example}{Small Changes in $x$}{SmallChanges}
Consider  $\ds y=f(x)=\sqrt{625-x^2}$ (the upper semicircle of radius
25 centered at the origin), and let's compute the changes of $y$
resulting from small changes of $x$ around $x=7$.
\end{example}

\begin{solution}
When $x=7$, we find that $\ds y=\sqrt{625-49}=24$.
Suppose we want to know how much $y$ changes when $x$ increases a little,
say to 7.1 or 7.01.

In the case of a straight line $y=mx+b$, the slope $m=\Delta y/\Delta
x$ measures the change in $y$ per unit change in $x$. This can be
interpreted as a measure of ``sensitivity''; for example, if
$y=100x+5$, a small change in $x$ corresponds to a change one hundred
times as large in $y$, so $y$ is quite sensitive to changes in $x$.

Let us look at the same ratio $\Delta y/\Delta x$ for our function
$\ds y=f(x)=\sqrt{625-x^2}$ when $x$ changes from 7 to $7.1$.  Here $\Delta
x=7.1-7=0.1$ is the change in $x$, and
\begin{align*}
\Delta y =f(x+\Delta x)-f(x)&=f(7.1)-f(7)\cr
         &=\sqrt{625-7.1^2}-\sqrt{625-7^2}\cr
	 & \approx 23.9706-24=-0.0294.\cr
\end{align*}
Thus, $\Delta y/\Delta x\approx -0.0294/0.1=-0.294$. This means that $y$
changes by less than one third the change in $x$, so apparently $y$ is
not very sensitive to changes in $x$ at $x=7$. We say ``apparently''
here because we don't really know what happens between 7 and
$7.1$. Perhaps $y$ changes dramatically as $x$ runs through the values
from 7 to $7.1$, but at $7.1$ $y$ just happens to be close to its
value at $7$. This is not in fact the case for this particular
function, but we don't yet know why.
\end{solution}

The quantity $\Delta y/\Delta x\approx -0.294$ may be interpreted as
the slope of the line through $(7,24)$ and
$(7.1,23.9706)$, called a \dfont{chord} of the circle.
In general, if we draw the chord from the point $(7,24)$ to a nearby
point on the semicircle $(7+\Delta x,\,f(7+\Delta x))$, the slope of this
chord is the so-called \dfont{difference quotient}
$$
\frac{f(7+\Delta x)-f(7)}{\Delta x}=
\frac{\sqrt{625-(7+\Delta x)^2}-24}{\Delta x}.
$$
For example, if $x$ changes only from 7 to 7.01, then the
difference quotient (slope of the chord) is approximately equal to
$(23.997081-24)/0.01=-0.2919$.  This is slightly different than for the
chord from $(7,24)$ to $(7.1,23.9706)$.

As $\Delta x$ is made smaller (closer to 0), $7+\Delta x$ gets closer to 7 and the chord joining
$(7,f(7))$ to $(7+\Delta x,f(7+\Delta x))$ shifts slightly, as shown
 in Figure~\ref{fig:chords}. The chord
gets closer and closer to the \dfont{tangent line}
to the circle at the point $(7,24)$.  (The
tangent line is the line that just grazes the circle at that point,
i.e., it doesn't meet the circle at any second point.)  Thus, as
$\Delta x$ gets smaller and smaller, the slope $\Delta y/\Delta x$ of
the chord gets closer and closer to the slope of the tangent line.
This is actually quite difficult to see when $\Delta x$ is small,
because of the scale of the graph. The values of $\Delta x$ used for
the figure are $1$, $5$, $10$ and $15$, not really very small values.
The tangent line is the one that is uppermost at the right hand
endpoint.

\smallskip
\begin{figure}[!ht]
\centerline{\beginpicture
\normalgraphs
%\ninepoint
\setcoordinatesystem units <2truemm,2truemm>
\setplotarea x from 0 to 26, y from 0 to 26
\circulararc 90 degrees from 25 0 center at 0 0
\axis left ticks numbered from 5 to 25 by 5 /
\axis bottom ticks numbered from 5 to 25 by 5 /
\plot 0 26.04 25 18.75 /
\plot 0 26.2 25 18.3379 /
\plot 0 26.896 25 16.554 /
\plot 0 27.97 25 13.8 /
\plot 3 27.5384 25 8.02 /
\endpicture}
\caption{Chords approximating the tangent line. \label{fig:chords}}
\end{figure}

So far we have found the slopes of two chords that should be close to
the slope of the tangent line, but what is the slope of the tangent
line exactly? Since the tangent line touches the circle at just one
point, we will never be able to calculate its slope directly, using
two ``known'' points on the line. What we need is a way to capture
what happens to the slopes of the chords as they get ``closer and
closer'' to the tangent line.

Instead of looking at more particular values of $\Delta x$, let's see
what happens if we do some algebra with the difference quotient using
just $\Delta x$. The slope of a chord from $(7,24)$ to a nearby point
$\left(7+\Delta x,f(7+\Delta x)\right)$ is given by
\begin{align*}
\frac{f(7+\Delta x)-f(7)}{\Delta x}&=\frac{\sqrt{625-(7+\Delta x)^2} - 24}{\Delta x}\\
&= \left( \ds\frac{\sqrt{625-(7+\Delta x)^2} - 24}{\Delta x} \right)
\left(\frac{\sqrt{625-(7+\Delta x)^2}+24}{\sqrt{625-(7+\Delta x)^2}+24} \right)\\
&=\frac{625-(7+\Delta x)^2-24^2}{\Delta x(\sqrt{625-(7+\Delta x)^2}+24)}\\
&=\frac{49-49-14\Delta x-\Delta x^2}{\Delta x(\sqrt{625-(7+\Delta x)^2}+24)}\\
&=\frac{\Delta x(-14-\Delta x)}{\Delta x(\sqrt{625-(7+\Delta x)^2}+24)}\\
&=\frac{-14-\Delta x}{\sqrt{625-(7+\Delta x)^2}+24}
\end{align*}
Now, can we tell by looking at this last formula what happens when
$\Delta x$ gets very close to zero? The numerator clearly gets very
close to $-14$ while the denominator gets very close to
$\ds \sqrt{625-7^2}+24=48$. The fraction is therefore very close to
$-14/48 = -7/24 \cong -0.29167$. In fact, the slope of the tangent line is exactly $-7/24$.

What about the slope of the tangent line at $x=12$? Well, 12 can't be
all that different from 7; we just have to redo the calculation with
12 instead of 7. This won't be hard, but it will be a bit
tedious. What if we try to do all the algebra without using a specific
value for $x$? Let's copy from above, replacing 7 by  $x$.
\begin{align*}
\frac{f(x+\Delta x)-f(x)}{\Delta x}&=\frac{\sqrt{625-(x+\Delta x)^2} - \sqrt{625-x^2}}{\Delta x}\\
&=\frac{\sqrt{625-(x+\Delta x)^2} - \sqrt{625-x^2}}{\Delta x}
\frac{\sqrt{625-(x+\Delta x)^2}+\sqrt{625-x^2}}{\sqrt{625-(x+\Delta x)^2}+\sqrt{625-x^2}}\\
&=\frac{625-(x+\Delta x)^2-625+x^2}{\Delta x(\sqrt{625-(x+\Delta x)^2}+\sqrt{625-x^2})}\\
&=\frac{625-x^2-2x\Delta x-\Delta x^2-625+x^2}{\Delta x(\sqrt{625-(x+\Delta x)^2}+\sqrt{625-x^2})}\\
&=\frac{\Delta x(-2x-\Delta x)}{\Delta x(\sqrt{625-(x+\Delta x)^2}+\sqrt{625-x^2})}\\
&=\frac{-2x-\Delta x}{\sqrt{625-(x+\Delta x)^2}+\sqrt{625-x^2}}
\end{align*}

Now what happens when $\Delta x$ is very close to zero? Again it seems
apparent that the quotient will be very close to
$$\frac{-2x}{\sqrt{625-x^2}+\sqrt{625-x^2}}
=\frac{-2x}{2\sqrt{625-x^2}}=\frac{-x}{\sqrt{625-x^2}}.
$$
Replacing $x$ by 7 gives $-7/24$, as before, and now we can easily do
the computation for 12  or any other value of
$x$ between $-25$ and 25.

So now we have a single expression, $\ds {-x/ \sqrt{625-x^2}}$,
that tells us the slope of the tangent line for any value of
$x$. This slope, in turn, tells us how sensitive the value of $y$ is
to small changes in the value of $x$.

The expression $\ds {-x/ \sqrt{625-x^2}}$ defines a new function
 called the \dfont{derivative} of the
original function (since it is derived from the original function).
 If the original is referred to as $f$ or $y$ then
the derivative is often written $f'$ or $y'$ (pronounced ``f
prime'' or ``y prime''). So in this case we might write $\ds f'(x)=-x/
\sqrt{625-x^2}$ or $y'=\ds {-x/ \sqrt{625-x^2}}$. At a particular point, say $x=7$, we write
$f'(7)=-7/24$ and we say that ``$f$ prime of 7 is $-7/24$'' or ``the derivative of
$f$ at 7 is $-7/24$.''

To summarize, we compute the derivative of $f(x)$ by forming the
difference quotient
$$
\frac{f(x+\Delta x)-f(x)}{\Delta x},
$$
which is the slope of a line, then we figure out what happens when
$\Delta x$ gets very close to 0.

At this point, we should note that the idea of letting $\Delta x$ get closer and closer to 0
is precisely the idea of a limit that we discussed in the last chapter. The limit here is a limit
as $\Delta x$ approaches 0. Using limit notation, we can write
$f'(x)=\lim_{\Delta x\to 0}\frac{f(x+\Delta x)-f(x)}{\Delta x}$.

In the particular case of a circle, there's a simple way to find the
derivative.  Since the tangent to a circle at a point is perpendicular to
the radius drawn to the point of contact, its slope is the negative
reciprocal of the slope of the radius.  The radius joining $(0,0)$ to
$(7,24)$ has slope 24/7.  Hence, the tangent line has slope
$-7/24$. In general, a radius to the point $\ds (x,\sqrt{625-x^2})$ has
slope $\ds \sqrt{625-x^2}/x$, so the slope of the tangent line is
$\ds {-x/ \sqrt{625-x^2}}$, as before. It is {\bf NOT} always true that a
tangent line is perpendicular to a line from the origin---don't use
this shortcut in any other circumstance.

As above, and as you might expect, for different values of $x$ we
generally get different values of the derivative $f'(x)$. Could it be
that the derivative always has the same value? This would mean that
the slope of $f$, or the slope of its tangent line, is the same
everywhere. One curve that always has the same slope is a line; it
seems odd to talk about the tangent line to a line, but if it makes
sense at all the tangent line must be the line itself. It is not hard
to see that the derivative of $f(x)=mx+b$ is $f'(x)=m$.


% binomial.tex %
% The Binomial Theorem proven in HOL %

\setlength{\unitlength}{1mm}
% \vskip10mm
% \begin{center}\LARGE\bf
% The Binomial Theorem proven in HOL.
% \end{center}
% \vskip10mm

\def\obj#1{\mbox{\tt#1}}

% ---------------------------------------------------------------------
% Parameters customising the document; set by whoever \input's it
% ---------------------------------------------------------------------
% \self{} is one word denoting this document, such as "article" or "chapter"
% \path{} is the path denoting the case-study directory
%
% Typical examples:
% \def\self{article}
% \def\path{\verb%Training/studies/binomial%}
% ---------------------------------------------------------------------

\section{Pascal's Triangle and the Binomial Theorem}

Pascal's Triangle\footnote{
According to Knuth \cite{knuth73}, the triangle gets its name from
its appearance in Blaise Pascal's {\em Trait\'e du triangle arithm\'etique}
in 1653, although the binomial coefficients were known long before then.
The earliest surviving description is from the tenth century,
in Hal\={a}yudha's commentary on the Hindu classic, the Chandah-S\^{u}tra.
The Binomial Theorem itself was first reported in 1676 by Isaac Newton.
}
is the infinite triangle of numbers which starts like this:
\[\begin{array}{ccccccc}
1  \\
1 & 1 \\
1 & 2 & 1 \\
1 & 3 & 3 & 1 \\
1 & 4 & 6 & 4 & 1 \\
1 & 5 & 10 & 10 & 5 & 1 \\
\cdot & \cdot & \cdot & \cdot & \cdot & \cdot & \cdot \\
\end{array}\]
The construction is as follows. The numbers at the edges
are $1$s. Each number in the interior is the sum of the number immediately
above it and the number to its left in the previous row.

The numbers in Pascal's Triangle are called Binomial Coefficients.
The number in the $n$th row and $k$th column (where $0 \leq k \leq n$, and the
topmost row and the leftmost column are numbered $0$) is written
$n \choose k$ and pronounced ``$n$ choose $k$''.  The coefficients are
pronounced this way because $n \choose k$ turns out to be the number of
ways to choose $k$ things out of $n$ things, but that is another story
(see for instance Knuth's book \cite{knuth73}).

A simple form of the Binomial Theorem \cite{maclane67} \cite{mostow63}
is the following equation, where $a$ and $b$ are integers and $n$ is any
natural number,
\[
(a + b)^n = \sum_{k=0}^n {n \choose k} a^k b^{n-k}
\]
The rest of this \self{} describes how the Binomial Theorem can be proven
in \HOL{}.  In fact, a more general theorem is proven, where $a$ and $b$
are elements of an arbitrary commutative ring, but the form displayed here
can be derived from the general result.

The motivation for the proof of the Binomial Theorem in \HOL{} is
tutorial.
% ---to be a medium sized worked example whose subject matter is
% more widely known than any specific piece of hardware or software.
The proof illustrates specific ways in \HOL{} to represent mathematical
notations and manage theorems proven about general structures such as
monoids, groups and rings. The following files accompany this \self{} in
directory \path{}:
\begin{description}
\item[{\tt mk\_BINOMIAL.ml}]
    The \ML{} program containing the definitions, lemmas and theorems\\
    needed to prove the Binomial Theorem in \HOL{}.
\item
    [{\tt BINOMIAL.th}]
    The theory file which is generated by \verb@mk_BINOMIAL.ml@.
\item
    [{\tt BINOMIAL.ml}]
    An \ML{} program which loads the theory \verb@BINOMIAL@ and declares
    a few derived rules and one tactic used in the proof.
    The Binomial Theorem is bound to the \ML{} identifier \verb@BINOMIAL@.
\end{description}
% The aim of this \self{} is to introduce the small amount of algebra and
% mathematical notation needed to state and prove the Binomial Theorem, show
% how this is rendered in \HOL{}, and outline the structure of the proof
% contained in \verb@mk_BINOMIAL.ml@.
The \self{} is meant to be intelligible
without examination of any of the accompanying files.
To avoid clutter not every detail is spelt out.
Often definitions and theorems are indicated
in the form displayed by \HOL{}, rather than as \ML{} source text. Specific
details of the \ML{} definitions or tactics can be found in the file
\verb@mk_BINOMIAL.ml@.

The way to work with this case study is first to study this \self{} to see
the structure of the proof, and then to work interactively with \HOL{}.
Start \HOL{} and say:
\begin{session}
\begin{verbatim}
#loadt `BINOMIAL`;;
...
File BINOMIAL loaded
() : void

#BINOMIAL;;
|- !plus times.
    RING(plus,times) ==>
    (!a b n.
      POWER times n(plus a b) =
      SIGMA(plus,0,SUC n)(BINTERM plus times a b n))
\end{verbatim}
\end{session}
(This is the first display of an interaction with \HOL{}.  The difference
between lines input to and lines output from \HOL{} is that only the former
are preceded by the \HOL{} prompt ``\verb@#@''.  A line of ellipsis
``\verb@...@'' stands for output from \HOL{} which has been omitted from the
display.)

The first input line in the display above loads in all the definitions,
theorems, derived rules and the tactic defined for the proof.  The second
input line asks \HOL{} to print theorem \verb@BINOMIAL@.  The constants
used in \verb@BINOMIAL@ are explained later in the \self{}.  The \HOL{}
session is now set up either to make use of \verb@BINOMIAL@ by specialising
it to a specific ring, or to study \verb@mk_BINOMIAL.ml@ by using the subgoal
package to work through proofs contained in it.

The remainder of the \self{} follows the plan:
\begin{description}
\item[Section~\ref{BinomialCoefficients}]
    shows how to define the number $n \choose k$ in \HOL{} as the term
    $\obj{CHOOSE}\,n\,k$, where $\obj{CHOOSE}$ is a function defined by
    primitive recursion.
\item[Section~\ref{MonoidsGroupsRings}]
    shows how to define elementary algebraic concepts like associativity,
    commutativity, monoid, group and ring in \HOL{}, and how to apply
    theorems proved in the general case to particular situations.
\item[Section~\ref{PowersReductionsRangesSums}]
    shows how to define the iterated operations of raising a term to a
    power and reducing (summing) a list of terms. These operations are
    part of the everyday mathematical language used to state the Binomial
    Theorem, but they are not built-in to \HOL{} and so need to be defined.
    This section and the previous two together describe enough definitions
    in \HOL{} to state the Binomial Theorem.
\item[Section~\ref{BinomialTheorem}]
    shows how the Binomial Theorem is proven in \HOL{}.
    The proof is by induction, and depends on three main lemmas.
\item[Section~\ref{BinomialTheoremForIntegers}]
    outlines how to apply the general theorem to a particular case:
    the ring of integers.
\item[Appendix~\ref{PrincipalLemmas}]
    states the principal lemmas used to prove the theorem.
    Some of these are {\em ad hoc} but others could be reused elsewhere.
\end{description}

% ----------------------------------------------------------------------------

\section{The Binomial Coefficients}
\label{BinomialCoefficients}

The definition of the binomial coefficients as the numbers in Pascal's Triangle
is formalised in three equations:
\begin{eqnarray*}
n \choose 0 &=& 1 \\
0 \choose {k+1} &=& 0 \\
{n+1} \choose {k+1} &=& {{n} \choose {k+1}} + {{n} \choose {k}}
\end{eqnarray*}
(These actually define $n \choose k$ to be $0$ if $k>n$, but this is
consistent with Pascal's Triangle: think of the spaces to the right of
the triangle as filled with $0$'s.) The desire is to define a constant
\verb@CHOOSE@ in \HOL{} such that for all numbers $n$, $k$ in the type
{\tt num}, $\verb@CHOOSE@\,n\,k$ denotes the number $n \choose k$.
Unfortunately these three equations cannot be used directly as the definition
of \verb@CHOOSE@ in \HOL{} because they are not in the form of a base case
(when $n=0$) together with an inductive case (when $n>0$).

To work towards the definition, consider a term that specifies a base case
and an inductive case intended to be equivalent to the three equations above:
\begin{session}
\begin{verbatim}
let base_and_inductive: term =
    "(CHOOSE 0 k = ((0 = k) => 1 | 0)) /\
     (CHOOSE (SUC n) k =
         ((0 = k) => 1 | (CHOOSE n (k-1)) + (CHOOSE n k)))";;
\end{verbatim}
\end{session}
The theory \verb@prim_rec@ contains a primitive recursion theorem which
says that any base case and inductive case identifies a unique function,
\verb@fn@:
\begin{session}
\begin{verbatim}
#num_Axiom;;
|- !e f. ?! fn.
    (fn 0 = e) /\
    (!n. fn (SUC n) = f (fn n) n)
\end{verbatim}
\end{session}
Given the theorem \verb@prim_rec@, the builtin function
\verb@prove_rec_fn_exists@ can prove that there is a function that satisfies
the property specified by the term \verb@base_and_inductive@:
\begin{session}
\begin{verbatim}
#let RAW_CHOOSE_DEF = prove_rec_fn_exists num_Axiom base_and_inductive;;
RAW_CHOOSE_DEF =
|- ?CHOOSE.
    (!k. CHOOSE 0 k = ((0 = k) => 1 | 0)) /\
    (!n k.
      CHOOSE(SUC n)k = ((0 = k) => 1 | (CHOOSE n(k - 1)) + (CHOOSE n k)))
\end{verbatim}
\end{session}
These equations would not do as the only definition of \verb@CHOOSE@ due
to the presence of the conditional operator, which makes rewriting
difficult.  But having obtained the theorem \verb@RAW_CHOOSE_DEF@ it is
easy to prove there is a function which satisfies the original three
equations:
\begin{session}
\begin{verbatim}
CHOOSE_DEF_LEMMA =
|- ?CHOOSE.
    (!n. CHOOSE n 0 = 1) /\
    (!k. CHOOSE 0(SUC k) = 0) /\
    (!n k. CHOOSE(SUC n)(SUC k) = (CHOOSE n(SUC k)) + (CHOOSE n k))
\end{verbatim}
\end{session}
Now the constant \verb@CHOOSE@ can be specified as originally desired:
\begin{session}
\begin{verbatim}
#let CHOOSE_DEF =
#    new_specification `CHOOSE_DEF` [`constant`,`CHOOSE`] CHOOSE_DEF_LEMMA;;
CHOOSE_DEF =
|- (!n. CHOOSE n 0 = 1) /\
   (!k. CHOOSE 0(SUC k) = 0) /\
   (!n k. CHOOSE(SUC n)(SUC k) = (CHOOSE n(SUC k)) + (CHOOSE n k))
\end{verbatim}
\end{session}
Two elementary properties of the binomial coefficients can now be proven
by induction:
\begin{session}
\begin{verbatim}
CHOOSE_LESS = |- !n k. n < k ==> (CHOOSE n k = 0)
CHOOSE_SAME = |- !n. CHOOSE n n = 1
\end{verbatim}
\end{session}

% ----------------------------------------------------------------------------

\section{Monoids, Groups and Commutative Rings}
\label{MonoidsGroupsRings}
%A simple way to define algebraic structures in \HOL{}.

\subsection{Associativity and Commutativity}

An {\em algebraic structure} is a set equipped with some operators that
obey some laws.  An elementary example of such a structure is the pair
$(A,+)$, where $A$ is a set and $+$ is a binary operator on $A$ that obeys
the laws of associativity,
\[
a + (b + c) = (a + b) + c
\]
and commutativity,
\[
a + b = b + a
\]

How might these laws be represented in \HOL{}?  The simplest scheme is
to deal with structures of the form $(\sigma, +)$, where $\sigma$ is a
type of the \HOL{} logic, and $+$ is a \HOL{} term of type
$\sigma\to\sigma\to\sigma$.  Notice that the set in the structure is
represented as a \HOL{} type: see the theory \verb@group@ presented in
Elsa Gunter's case study of Modular Arithmetic for a more sophisticated
approach where the set in a structure can be a subset of a \HOL{} type.

The two laws can be defined as two constants in \HOL{}:
\begin{session}
\begin{verbatim}
#let ASSOCIATIVE =
#    new_definition (`ASSOCIATIVE`,
#      "!plus: *->*->*. ASSOCIATIVE plus =
#          (!a b c. plus a (plus b c) = plus (plus a b) c)");;
...
#let COMMUTATIVE =
#    new_definition (`COMMUTATIVE`,
#      "!plus: *->*->*. COMMUTATIVE plus =
#          (!a b. plus a b = plus b a)");;
\end{verbatim}
\end{session}
By instantiating the type variable \verb@*@ and specialising the variable
\verb@plus@ these constants are applicable to any particular structure
of the form $(\sigma, +: \sigma\to\sigma\to\sigma)$.

A simple example to illustrate this methodology is a permutation theorem
about an arbitrary associative commutative operator (which is called
\verb@PERM@ in file \verb@mk_BINOMIAL.ml@).  It is worthwhile going
through the proof in detail because it illustrates how a difficulty in
making use of assumptions like \verb@ASSOCIATIVE plus@ and
\verb@COMMUTATIVE plus@ can be solved.
\begin{session}
\begin{verbatim}
#g "!plus: *->*->*.
#       ASSOCIATIVE plus /\ COMMUTATIVE plus ==>
#           !a b c. plus (plus a b) c = plus b (plus a c)";;
\end{verbatim}
\end{session}
The first step is to strip off the quantifiers, and break up the implication:
\begin{session}
\begin{verbatim}
#e (REPEAT STRIP_TAC);;
OK..
"plus(plus a b)c = plus b(plus a c)"
    [ "ASSOCIATIVE plus" ]
    [ "COMMUTATIVE plus" ]
\end{verbatim}
\end{session}
The next step is to rewrite \verb@plus a b@ to \verb@plus b a@ on the
left-hand side, from the assumption \verb@COMMUTATIVE plus@. If the theorem
was about a specific operator, such as \verb@$+: num->num->num@ from the
theory of numbers in \HOL{}, the rewriting could be done with
\verb@REWRITE_TAC@ and the specific commutativity theorem, such as
\verb@ADD_SYM@:
\[
\mbox{\tt |- !m n. m + n = n + m}.
\]
But in the general case all that is available is the definition of
\verb@COMMUTATIVE plus@ which cannot be used directly with
\verb@REWRITE_TAC@ to swap the terms \verb@a@ and \verb@b@.
One approach is first to prove the
following equation from the definition \verb@COMMUTATIVE plus@,
\[
\mbox{\tt COMMUTATIVE plus |- !a b. plus a b = plus b a},
\]
and then to use the equation with \verb@REWRITE_TAC@ in the same way as one
might use \verb@ADD_SYM@.  This is a valid tactic because the
assumption \verb@COMMUTATIVE plus@ is already present in the assumption
list.  A little forward proof achieves the first step, the derivation of the
equation:
\begin{session}
\begin{verbatim}
#top_print print_all_thm;;
- : (thm -> void)

#let forwards = fst(EQ_IMP_RULE (SPEC "plus: *->*->*" COMMUTATIVE));;
forwards = |- COMMUTATIVE plus ==> (!a b. plus a b = plus b a)

#let eqn = UNDISCH forwards;;
eqn = COMMUTATIVE plus |- !a b. plus a b = plus b a
\end{verbatim}
\end{session}
This forward proof is a special case of a derived
rule of the logic,
\[
\Gamma\turn \uquant{x} t_1 = t_2
\over
\Gamma, t_1[t'/x] \turn t_2[t'/x]
\]
which can be coded as the \ML{} function \verb@SPEC_EQ@:
\begin{session}
\begin{verbatim}
#let SPEC_EQ v th = UNDISCH(fst(EQ_IMP_RULE (SPEC v th)));;
SPEC_EQ = - : (term -> thm -> thm)
\end{verbatim}
\end{session}
Now the equation \verb@eqn@ can be used to prove swap terms
\verb@a@ and \verb@b@:
\begin{session}
\begin{verbatim}
#e(SUBST1_TAC (SPECL ["a:*";"b:*"] eqn));;
OK..
"plus(plus b a)c = plus b(plus a c)"
    [ "ASSOCIATIVE plus" ]
    [ "COMMUTATIVE plus" ]
\end{verbatim}
\end{session}
Now that the terms appear in both sides in the same order but with different
grouping, associativity can be used to make the grouping on both sides
the same.  Again the definition of \verb@ASSOCIATIVE@ cannot be used
directly, but the derived rule \verb@SPEC_EQ@ obtains from the definition
an equation analogous to \verb@eqn@ which can be used:
\begin{session}
\begin{verbatim}
#e(REWRITE_TAC [SPEC_EQ "plus: *->*->*" ASSOCIATIVE]);;
OK..
goal proved
...
|- !plus.
    ASSOCIATIVE plus /\ COMMUTATIVE plus ==>
    (!a b c. plus(plus a b)c = plus b(plus a c))
\end{verbatim}
\end{session}

The point of this example was to illustrate how to prove theorems about
an arbitrary operator whose behaviour is specified by general predicates
like \verb@ASSOCIATIVE@ and \verb@COMMUTATIVE@. The main difficulty is
in deriving equations for use with rewriting or substitution.  The solution
used in the example and extensively in the proof of the Binomial Theorem
is to derive equations using the derived rule \verb@SPEC_EQ@. Another derived
rule, \verb@SPEC_IMP@, is used extensively and is a variant of \verb@SPEC_EQ@:
\[
\Gamma\turn \uquant{x} t_1 \Rightarrow t_2
\over
\Gamma, t_1[t'/x] \turn t_2[t'/x]
\]
\begin{session}
\begin{verbatim}
#let SPEC_IMP v th = UNDISCH(SPEC v th);;
SPEC_IMP = - : (term -> thm -> thm)
\end{verbatim}
\end{session}
Two following two rules are versions of \verb@SPEC_EQ@ and \verb@SPEC_IMP@
generalised to take lists of variables rather than just one:
\begin{session}
\begin{verbatim}
SPECL_EQ = - : (term list -> thm -> thm)
SPECL_IMP = - : (term list -> thm -> thm)
\end{verbatim}
\end{session}
The final new derived rule used in the proof of the Binomial Theorem
is \verb@STRIP_SPEC_IMP@, a variant of \verb@SPEC_IMP@, which
splits up a conjunction in the antecedent into separate assumptions:
\[
\Gamma\turn \uquant{x} (t_1 \wedge \cdots \wedge t_n) \Rightarrow t_{n+1}
\over
\Gamma, t_1[t'/x], \ldots, t_n[t'/x] \turn t_{n+1}[t'/x]
\]
\begin{session}
\begin{verbatim}
#let STRIP_SPEC_IMP v th =
#    UNDISCH_ALL(SPEC v (CONV_RULE (ONCE_DEPTH_CONV ANTE_CONJ_CONV) th));;
STRIP_SPEC_IMP = - : (term -> thm -> thm)
\end{verbatim}
\end{session}

There are other methods for proving theorems which are conditional on
predicates such as \verb@ASSOCIATIVE@ and \verb@COMMUTATIVE@.  One is to
rewrite with the definition before assuming the predicate (see, for instance,
the proof of \verb@RIGHT_CANCELLATION@ in \verb@mk_BINOMIAL.ml@). This
method is fine for small proofs, but in larger proofs it presents the problem
of how to specify a particular assumption on the assumption list.  Another
method is to put the predicates on the assumption list, and use resolution
to extract the body of the definition (see the proof of
\verb@UNIQUE_LEFT_INV@).  When the assumption list is not short this is
rather a blunt instrument, in the sense that resolution can find many more
theorems than the one desired, and is also rather computationally costly
when compared to carefully constructing a theorem with \verb@SPEC_EQ@ and
its variants.

\subsection{Monoids}

A {\em monoid} is a structure $(M,+)$ where $M$ is a set, $+$ is an
associative binary operator on $M$, such that there is an {\em identity
element}, $u \in M$, which is a left and right identity of $+$, that is,
it satisfies $u+a = a+u = a$ for all $a \in M$. When the operator is written
as $+$, the structure $(M,+)$ is called an {\em additive monoid} and the
identity element is written as $0$; otherwise if the operator is written
as $\times$, the structure $(M,\times)$ is called a {\em multiplicative
monoid} and the identity element is written as $1$.
\begin{session}
\begin{verbatim}
#let RAW_MONOID =
#    new_definition (`RAW_MONOID`,
#      "!plus: *->*->*. MONOID plus =
#          ASSOCIATIVE plus /\
#          (?u. (!a. plus a u = a) /\ (!a. plus u a = a))");;
\end{verbatim}
\end{session}
This definition is inconvenient to manipulate as it stands, because of
the presence of the existential quantifier.  However it is possible via
Hilbert's $\epsilon$-operator to specify a function \verb@Id@ such that
\verb@Id plus@ stands for an identity element of \verb@plus@, if such
exists.  In fact it is easy to prove that the identity element in a monoid
is unique.  The first step is to prove that there exists a function \verb@Id@
with the property just described:
\begin{session}
\begin{verbatim}
ID_LEMMA =
|- ?Id.
    !plus.
     (?u. (!a. plus u a = a) /\ (!a. plus a u = a)) ==>
     (!a. plus(Id plus)a = a) /\ (!a. plus a(Id plus) = a)
\end{verbatim}
\end{session}
The proof of \verb@ID_LEMMA@ uses Hilbert's $\epsilon$-operator to
construct a witness for the outer existential quantifier,
\begin{verbatim}
    \plus.@u:*.(!a:*. (plus u a = a)) /\ (!a. (plus a u = a))
\end{verbatim}
Given \verb@ID_LEMMA@, the constant \verb@Id@ is specified as follows:
\begin{session}
\begin{verbatim}
#let ID = new_specification `Id` [`constant`, `Id`] ID_LEMMA;;
ID =
|- !plus.
    (?u. (!a. plus u a = a) /\ (!a. plus a u = a)) ==>
    (!a. plus(Id plus)a = a) /\ (!a. plus a(Id plus) = a)
\end{verbatim}
\end{session}
The condition that \verb@Id plus@ is a left and right identity is expressed
by the predicates \verb@LEFT_ID@ and \verb@RIGHT_ID@, respectively:
\begin{session}
\begin{verbatim}
LEFT_ID = |- !plus. LEFT_ID plus = (!a. plus(Id plus)a = a)
RIGHT_ID = |- !plus. RIGHT_ID plus = (!a. plus a(Id plus) = a)
\end{verbatim}
\end{session}
Given these abbreviations it is easy to prove in \HOL{} that
if a left and right identity element exists, then it is unique:
\begin{session}
\begin{verbatim}
UNIQUE_LEFT_ID =
|- !plus.
    LEFT_ID plus /\ RIGHT_ID plus ==>
    (!l. (!a. plus l a = a) ==> (Id plus = l))

UNIQUE_RIGHT_ID =
|- !plus.
    LEFT_ID plus /\ RIGHT_ID plus ==>
    (!r. (!a. plus a r = a) ==> (Id plus = r))
\end{verbatim}
\end{session}
The use of these last two theorems is to help identify the element
\verb@Id plus@ for some specific \verb@plus@, such as \verb@$+@ from the theory
of \verb@arithmetic@.  The abbreviations \verb@LEFT_ID@ and \verb@RIGHT_ID@
admit a new characterisation of the predicate \verb@MONOID@, which is easier
to manipulate in \HOL{} than \verb@RAW_MONOID@:
\begin{session}
\begin{verbatim}
MONOID =
|- !plus. MONOID plus = LEFT_ID plus /\ RIGHT_ID plus /\ ASSOCIATIVE plus
\end{verbatim}
\end{session}

\subsection{Groups}

A {\em group} is a monoid in which every element is invertible.
\begin{session}
\begin{verbatim}
RAW_GROUP =
|- !plus.
    Group plus =
    MONOID plus /\
    (!a. ?b. (plus b a = Id plus) /\ (plus a b = Id plus))
\end{verbatim}
\end{session}
An alternative characterisation of \verb@Group@\footnote{The mixed
  case identifier \texttt{Group} is used rather than \texttt{GROUP},
  because the latter is already defined in Elsa Gunter's theory
  \texttt{group}, a parent of her theory \texttt{integer}, which needs
  to coexist with the present theory so that integers can be used as
  an example ring (Section~\ref{BinomialTheoremForIntegers}).} which
makes the existential quantification implicit can be had by specifying
a function \verb@Inv@ so that, if \verb@Group plus@ holds, then the
element \verb@Inv plus a@ is an inverse of \verb@a@.  The function
\verb@Inv@ is specified in a way analogous to the specification of
\verb@Id@.  First a lemma is proven which says there does exist a
function \verb@Inv@ with the property desired:
\begin{session}
\begin{verbatim}
INV_LEMMA =
|- ?Inv.
    !plus.
     (!a. ?b. (plus b a = Id plus) /\ (plus a b = Id plus)) ==>
     (!a.
       (plus(Inv plus a)a = Id plus) /\ (plus a(Inv plus a) = Id plus))
\end{verbatim}
\end{session}
Given this lemma, which is proven by an easy tactic almost the same
as the one for \verb@ID_LEMMA@, the constant \verb@Inv@ can be specified:
\begin{session}
\begin{verbatim}
#let INV =
#    new_specification `Inv` [`constant`, `Inv`] INV_LEMMA;;
\end{verbatim}
\end{session}
The predicates which say that \verb@Inv@ produces left and right inverses
are defined as new constants:
\begin{session}
\begin{verbatim}
LEFT_INV =
|- !plus. LEFT_INV plus = (!a. plus(Inv plus a)a = Id plus)
RIGHT_INV =
|- !plus. RIGHT_INV plus = (!a. plus a(Inv plus a) = Id plus)
\end{verbatim}
\end{session}
Finally, the alternative characterisation of \verb@Group@ can be proven:
\begin{session}
\begin{verbatim}
GROUP = |- Group plus = MONOID plus /\ LEFT_INV plus /\ RIGHT_INV plus
\end{verbatim}
\end{session}
The theory of groups has been developed only as far as needed to prove
the Binomial Theorem; Appendix~\ref{PrincipalLemmas} shows the one lemma,
\verb@RIGHT_CANCELLATION@, specifically about groups.

\subsection{Rings}

A {\em ring} is a structure $(R,+,\times)$ such that structure $(R,+)$
is a group, structure $(R,\times)$ is a monoid, {\em addition}, $+$, is
commutative, and {\em multiplication}, $\times$, distributes (on both sides)
over addition.  A {\em commutative ring} is a ring in which multiplication
is commutative.  The word ring is used in the remainder of this \self{}
and in the \HOL{} code to mean a commutative ring:
\begin{session}
\begin{verbatim}
RING =
|- !plus times.
    RING(plus,times) =
    Group plus /\
    COMMUTATIVE plus /\
    MONOID times /\
    COMMUTATIVE times /\
    LEFT_DISTRIB(plus,times) /\
    RIGHT_DISTRIB(plus,times)
\end{verbatim}
\end{session}
\verb@LEFT_DISTRIB@ and \verb@RIGHT_DISTRIB@ are new constants
defined by the theorems:
\begin{session}
\begin{verbatim}
LEFT_DISTRIB =
|- !plus times.
    LEFT_DISTRIB(plus,times) =
    (!a b c. times a(plus b c) = plus(times a b)(times a c))

RIGHT_DISTRIB =
|- !plus times.
    RIGHT_DISTRIB(plus,times) =
    (!a b c. times(plus a b)c = plus(times a c)(times b c))
\end{verbatim}
\end{session}
Notice that the definition of \verb@RING@ abbreviates a conjunction of
predicates, some of which are atomic, in the sense that they are some basic
property of \verb@plus@ or \verb@times@ or both, and some are compound,
in the sense that they abbreviate a further conjunction of properties.
The definition is a tree of predicates with atomic ones like
\verb@LEFT_DISTRIB@ and \verb@ASSOCIATIVE@, at the leaves, and compound
ones like \verb@Group@ and \verb@MONOID@, at the branches.  Many theorems
conditional on \verb@RING(plus,times)@ need to make use of both atomic
and compound predicates contained in the tree.  To make access to all these
predicates easy, the following lemma is proven, which makes explicit all
the predicates in the tree:
\begin{session}
\begin{verbatim}
RING_LEMMA =
|- RING(plus,times) ==>
   RING(plus,times) /\
   Group plus /\
   MONOID plus /\
   LEFT_ID plus /\
   RIGHT_ID plus /\
   ASSOCIATIVE plus /\
   LEFT_INV plus /\
   RIGHT_INV plus /\
   COMMUTATIVE plus /\
   MONOID times /\
   LEFT_ID times /\
   RIGHT_ID times /\
   ASSOCIATIVE times /\
   COMMUTATIVE times /\
   LEFT_DISTRIB(plus,times) /\
   RIGHT_DISTRIB(plus,times)
\end{verbatim}
\end{session}
Goals of the form,
\[
\mbox{\tt !plus times. RING(plus,times) ==> ...}
\]
are consistently tackled with the following tactic, which uses \verb@RING_LEMMA@
to add all the atomic and compound predicates in the tree to the assumption
list:
\begin{session}
\begin{verbatim}
#let RING_TAC =
#    GEN_TAC THEN GEN_TAC THEN
#    DISCH_THEN (\th. STRIP_ASSUME_TAC (MP RING_LEMMA th));;
\end{verbatim}
\end{session}
The one lemma about rings needed here, \verb@RING_0@, is stated in
Appendix~\ref{PrincipalLemmas}.

% ----------------------------------------------------------------------------

\section{Powers, Reductions, Ranges and Sums}
\label{PowersReductionsRangesSums}
%Translation of mathematical definitions such as sums of series into HOL

The aim of this section is to explain how the mathematical notation
$\sum_{k=0}^n {n \choose k} a^k b^{n-k}$ can be rendered in \HOL{}.
If this is written out more explicitly than necessary for human
consumption it looks like this:
\[
\sum_{k=0}^n {n \choose k} \cdot (a^k \times b^{n-k})
\]
where $\times$ is multiplication in the ring, and where exponentiation
and $\cdot$ stand for the scalar powers of multiplication and addition,
respectively.  Since binomial coefficients were treated in
Section~\ref{BinomialCoefficients}, what remain to be defined in this section
are scalar powers and the $\Sigma$-notation.  The latter is tackled in
three steps: reduction of a list of monoid elements; generation of lists
comprising subranges of natural numbers, $[n,n+1,\ldots,n+k-1]$; and finally
the reduction of a list of ring elements indexed by a subrange of the natural
numbers.

\subsection{Scalar powers in a monoid}

If $(M,+)$ and $(M,\times)$ are additive and multiplicative
monoids respectively, the scalar powers of addition and multiplication are
defined informally by the following equations, where $n>0$:
\begin{eqnarray*}
n \cdot a &=& \overbrace{a + \cdots + a}^{n} \\
a^n &=& \overbrace{a \times \cdots \times a}^{n}
\end{eqnarray*}
When $n=0$, $n \cdot a = 0$ and $a^n = 1$.

Scalar power is defined in \HOL{} via the constant \verb@POWER@ which
is defined by primitive recursion:
\begin{session}
\begin{verbatim}
POWER =
|- (!plus a. POWER plus 0 a = Id plus) /\
   (!plus n a. POWER plus(SUC n)a = plus a(POWER plus n a))
\end{verbatim}
\end{session}
Three lemmas about \verb@POWER@ are stated in Appendix~\ref{PrincipalLemmas}.

\subsection{Reduction in a monoid}

This section makes use of the standard \HOL{} theory of lists, in particular
the constants \verb@NIL@, \verb@CONS@, \verb@APPEND@ and \verb@MAP@,
definition by primitive recursion and proof by induction. The theory of
lists is described in \DESCRIPTION.

If $(M,+)$ is a monoid, then the reduction of a list $[a_1,\ldots,a_n]$,
where each $a_i$ is an element of $M$, is the sum,
\[
a_1 + \cdots + a_n,
\]
or the monoid's identity element if $n=0$.

Reduction is represented in \HOL{} by the constant \verb@REDUCE@,
with type,
\begin{verbatim}
REDUCE : (*->*->*) -> (*)list -> *
\end{verbatim}
and which is defined by primitive recursion on lists on its second argument:
\begin{session}
\begin{verbatim}
#let REDUCE =
#  new_list_rec_definition (`REDUCE`,
#    "(!plus. (REDUCE plus NIL) = (Id plus):*) /\
#     (!plus (a:*) as. REDUCE plus (CONS a as) = plus a (REDUCE plus as))");;
REDUCE =
|- (!plus. REDUCE plus[] = Id plus) /\
   (!plus a as. REDUCE plus(CONS a as) = plus a(REDUCE plus as))
\end{verbatim}
\end{session}
Three lemmas about \verb@REDUCE@ are stated in Appendix~\ref{PrincipalLemmas}.

\subsection{Subranges of the natural numbers}

The constant \verb@RANGE@, which is a curried function of two numbers,
is defined by primitive recursion so that,
\[
\verb@RANGE@\,m\,n = [m, m+1, \ldots, m+n-1]
\]
This definition means that the subrange is the first $n$ numbers starting
at $m$.  In particular, when $n=0$ the list is empty.
Appendix~\ref{PrincipalLemmas} states two simple properties of \verb@RANGE@

\verb@RANGE@ generates a subrange given its least element and length;
an alternative method is to generate the subrange given its two endpoints,
via a constant \verb@INTERVAL@ which satisfies the equation:
\[
\verb@INTERVAL@\,m\,n =
    \left\{\begin{array}{ll}
    [m, m+1, \ldots, n] & \mbox{if $m \leq n$} \\
    {}[] & \mbox{otherwise}
    \end{array}\right.
\]

For comparison, the two methods are definable in \HOL{} as follows:
\begin{session}
\begin{verbatim}
#let RANGE =
#    new_recursive_definition false num_Axiom `RANGE`
#      "(RANGE m 0 = NIL) /\
#       (RANGE m (SUC n) = CONS m (RANGE (SUC m) n))";;
#
#let INTERVAL =
#    new_recursive_definition false num_Axiom `INTERVAL`
#      "(INTERVAL m 0 =
#          (m > 0) => NIL | [0]) /\
#      (INTERVAL m (SUC n) =
#          (m > (SUC n)) => NIL | APPEND (INTERVAL m n) ([SUC n]))";;
\end{verbatim}
\end{session}

\subsection{$\Sigma$-notation}

The standard $\Sigma$-notation is defined informally by the equation,
\[
\sum_{i=m}^n a_i =
    \left\{\begin{array}{ll}
    a_m + a_{m+1} + \cdots + a_n & \mbox{if $m \leq n$} \\
    0 & \mbox{otherwise}
    \end{array}\right.
\]
Therefore the standard $\Sigma$-notation is naturally definable via
a subrange generated by \verb@INTERVAL@.

An alternative notation, naturally definable via \verb@RANGE@, can be
defined informally as follows:
\[
\sum_{i=m}^{(n)} a_i =
    \left\{\begin{array}{ll}
    a_m + a_{m+1} + \cdots + a_{m+n-1} & \mbox{if $n > 0$} \\
    0 & \mbox{otherwise}
    \end{array}\right.
\]
The brackets around the $n$ above the $\Sigma$ distinguish this
notation from the standard one.

This second notation is used in the \HOL{} proof of the Binomial Theorem.
The difference between the two is just that the standard notation is
definable via \verb@INTERVAL@, and the second via \verb@RANGE@. The second
notation was chosen entirely because the constant \verb@RANGE@ is easier
to manipulate in \HOL{} than \verb@INTERVAL@, which is because the latter
has a more complex definition. The disadvantage of this choice is that
the proof of the theorem uses a non-standard notation, but that's all the
difference is: notation.  If desired, the Binomial Theorem stated in the
standard notation could be trivially deduced from its non-standard statement.

The non-standard $\Sigma$-notation is defined in \HOL{} as the following
abbreviation:
\begin{session}
\begin{verbatim}
#let SIGMA =
#    new_definition (`SIGMA`,
#      "SIGMA (plus,m,n) f = REDUCE plus (MAP f (RANGE m n)): *");;
\end{verbatim}
\end{session}
Note that an indexed term $a_i$ is represented in \HOL{} as a function
\verb@f@ of type \verb@num -> *@.

Appendix~\ref{PrincipalLemmas} states several properties of \verb@SIGMA@;
their proofs make use of properties of \verb@MAP@ from the theory
\verb@list@, and also the lemmas about \verb@REDUCE@ and \verb@RANGE@.

The functions \verb@REDUCE@ and \verb@RANGE@ are needed here only to define
\verb@SIGMA@, and in fact could be omitted if \verb@SIGMA@ were to be defined
directly  using primitive recursion.  It is worth going to the trouble
of defining \verb@REDUCE@ and \verb@RANGE@, however, because they are likely
to be useful in other situations.

% ----------------------------------------------------------------------------

\section{The Binomial Theorem for a Commutative Ring}
\label{BinomialTheorem}
%Division of a medium sized proof into manageable lemmas.

This section outlines the proof.  For full details see the tactics in
\verb@mk_BINOMIAL.ml@.

The statement and proof of the Binomial Theorem use a new \HOL{} constant,
\verb@BINTERM@, to abbreviate indexed terms of the form $\lambda k.\,{n
\choose k} \cdot a^{n-k} \times b^k$.  \verb@BINTERM@ is defined as a
curried function as follows:
\begin{session}
\begin{verbatim}
#let BINTERM_DEF =
#    new_definition (`BINTERM_DEF`,
#      "BINTERM (plus: *->*->*) (times: *->*->*) a b n k =
#          POWER plus (CHOOSE n k)
#              (times (POWER times (n-k) a) (POWER times k b))");;
\end{verbatim}
\end{session}
This abbreviation lets the Binomial Theorem be stated and proven in \HOL{} as
follows:
\begin{session}
\begin{verbatim}
BINOMIAL =
|- !plus times.
    RING(plus,times) ==>
    (!a b n.
      POWER times n(plus a b) =
      SIGMA(plus,0,SUC n)(BINTERM plus times a b n))
\end{verbatim}
\end{session}
In outline, the proof proceeds as follows.  The following equation is
proven by induction on $n$:
\begin{eqnarray*}
(a+b)^n &=& \sum_{k=0}^{(n+1)} {n \choose k} a^{n-k} b^k
\end{eqnarray*}
When $n=0$ both sides reduce to the multiplicative identity element, $1$,
of the ring. For the inductive step, the equation is assumed for $n$, and
the following goal must be proven:
\begin{eqnarray*}
(a+b)^{n+1} &=& \sum_{k=0}^{(n+2)} {{n+1} \choose k} a^{n+1-k} b^k
\end{eqnarray*}
Three lemmas are used in the inductive step:
\[\begin{array}{rcll}
\displaystyle
\sum_{k=1}^{(n)} {{n+1} \choose k} a^{n+1-k} b^k &=&
\displaystyle
    a \times \sum_{k=1}^{(n)} {n \choose k} a^{n+1-k} b^k +
    b \times \sum_{k=0}^{(n)} {n \choose k} a^{n+1-k} b^k
    & \mbox{({\tt LEMMA\_1})} \\
\displaystyle
a \times \sum_{k=0}^{(n+1)} {n \choose k} a^{n-k} b^k &=&
\displaystyle
    a^{n+1} +
    a \times \sum_{k=1}^{(n)} {{n+1} \choose k} a^{n+1-k} b^k
    & \mbox{({\tt LEMMA\_2})} \\
\displaystyle
b \times \sum_{k=0}^{(n+1)} {n \choose k} a^{n-k} b^k &=&
\displaystyle
    b \times \sum_{k=0}^{(n)} {{n+1} \choose k} a^{n+1-k} b^k
    + b^{n+1}
    & \mbox{({\tt LEMMA\_3})}
\end{array}\]
\verb@LEMMA_1@ is the key to the inductive step. The three stages of the
inductive step are indicated (a), (b) and (c) in \ML{} comments.  Step
(a) expands the right-hand side of the goal with \verb@LEMMA_1@:
\begin{eqnarray*}
\mbox{rhs} &=&
    a^{n+1} +
    a \times \sum_{k=1}^{(n)} {{n+1} \choose k} a^{n+1-k} b^k +
    b \times \sum_{k=0}^{(n)} {{n+1} \choose k} a^{n+1-k} b^k +
    b^{n+1}
\end{eqnarray*}
Step (b) expands the left-hand side with the induction hypothesis:
\begin{eqnarray*}
\mbox{lhs} &=&
    (a+b)\sum_{k=0}^{(n+1)} {n \choose k} a^{n-k} b^k
\end{eqnarray*}
Finally, in step (c) the expansions of the two sides are shown to be the
same, using distributivity of $\times$ over $+$, associativity of $+$,
and \verb@LEMMA_2@ and \verb@LEMMA_3@:
\begin{eqnarray*}
\mbox{lhs} &=&
    a \times \sum_{k=0}^{(n+1)} {n \choose k} a^{n-k} b^k
    +
    b \times \sum_{k=0}^{(n+1)} {n \choose k} a^{n-k} b^k \\
  &=&
    a^{n+1} +
    a \times \sum_{k=1}^{(n)} {{n+1} \choose k} a^{n+1-k} b^k
    +
    b \times \sum_{k=0}^{(n)} {{n+1} \choose k} a^{n+1-k} b^k
    + b^{n+1} \\
  &=&
      \mbox{rhs}
\end{eqnarray*}

% ----------------------------------------------------------------------------

\section{The Binomial Theorem for Integers}
\label{BinomialTheoremForIntegers}

The previous sections have dealt with the material in theory \verb@BINOMIAL@,
concerning how to prove the Binomial Theorem for an arbitrary ring.  A
natural next step is to produce some specific commutative ring, and prove
a Binomial Theorem for it, as an instance of the general case.

To see how to do so for Elsa Gunter's theory of integers, take a look
at the file {\tt mk\_BINOMIAL\_integer.ml}, contained in directory \path{}.
When loaded into \HOL{}, this \ML{} file proves that the integers are a
ring---by quoting laws contained in the theory \verb@integer@---and then
derives a Binomial Theorem for the integers---by Modus Ponens from the
general theorem \verb@BINOMIAL@.


\subsection{Data modeling}
\label{sec:model-creation}

The current implementation of \dBoost/ contains three data modeling strategies. Two of them are relatively simple machine-learning based applications that handle mostly continuous numerical data well.
They assume that data is distributed normally.
The last one makes much weaker assumptions about its input, and is best suited for studying discrete heterogeneous distributions produced by gathering human-input data. The following three subsections discuss the particulars of these three models; section~\ref{sec:partitioning} presents a way of extending models to deal with data that lends itself well to partitioning over a particular attribute.

\begin{figure}
  \centering
  \newcommand{\cramped}[3]{\subfloat[#2]{\includegraphics[width=.3\linewidth]{#1}\label{fig:#3}}}
  \cramped{../graphics/gaussians-preview.pdf}{Gaussian}{gaussian}\hspace*{.01\linewidth}
  \cramped{../graphics/mixtures-preview.png}{Mixture}{mixture}\hspace*{.01\linewidth}
  \cramped{../graphics/histograms-preview.pdf}{Histogram}{histogram}
  \caption{Simple visualization of the outlier detection strategy employed by each model. Possible outlier values are shown in red.}
  \label{fig:models}
\end{figure}

\subsubsection{Simple Gaussian Modeling}
\label{sec:gaus_model}
The univariate Gaussian model (Figure~\ref{fig:gaussian}) treats each value $x_i$ of the expanded tuples as random sample drawn from a normal distribution $\mathcal N(\mu_i, \sigma_i)$.

The model's parameters (a pair $(\mu, \sigma)$ for each numerical column) are computed as each column's mean and standard deviation. In the common case where the dataset has not significantly changed between the analysis and the modeling passes, the information obtained during the statistical analysis pass is sufficient to derive these parameters.

Despite its simplicity, this model presents the attractive property of requiring extremely little memory -- on the order of the size of one expanded tuple.

\subsubsection{Mixture Modeling}
\label{sec:mixture_model}
The multivariate Gaussian Mixture model (Figure~\ref{fig:mixture}) takes advantage of the correlation hints supplied by the statistical analysis pass to model sub-tuples of the expanded tuples as samples from multivariate Gaussian mixtures (GMMs), creating one model per group of correlated columns.

For example, if the statistical analysis phase outlines a pair of fields $(f_1, f_2)$ as good candidate for joint modeling, then the Mixture modeling strategy will learn a particular GMM to model this correlation. Pairs of values $(X_1, X_2)$ are here assumed to have been produced by random sampling off a distribution of the form
\begin{align*}
\sum_{j=1}^{N} \pi_j \mathcal N(\mu_j, \Sigma_j)
\end{align*}

where $N$ is the number of individual components that the GMM comprises ($N$ is a user-defined value in our implementation, but abundant literature exists on the subject of choosing $N$~\cite{Schwartz1978}~\cite{Akaike1974}), and $\pi_j, \mu_j$ and $\Sigma_j$ are parameters of the GMM learned as part of the modeling pass~\cite{Dempster1977}.

Unlike simple Gaussian models, the expectation maximization algorithm used in inferring the optimal model parameters for Gaussian mixtures does require retaining some data in memory. Still, most of the fields obtained after expanding each tuple are discarded after the relevant ones are extracted for learning purposes; in most cases we expect the set of values retained to be much smaller than the set of all attributes, thus limiting the memory usage.

In addition, when dealing with large amounts of data, it is possible -- and indeed, preferable -- to train the Mixture model on a randomly sampled subset of the data before running the full analysis. This approach is particularly relevant when using the Mixture model, but can be applied to all models to shorten the learning phase when dealing with very large datasets.

\subsubsection{Histogram-based Modeling}
\label{sec:histograms}
Simple Gaussian modeling and Gaussian Mixture modeling both offer good results for continuous numerical data such as the ones produced by sensors, but suffer from two limitations:

\begin{itemize}
\item They make strong assumptions about the distribution of the data under study
\item They fail to capture patterns in discrete numerical data, non-numerical data, or heterogeneous data
\end{itemize}

Our last model (Figure~\ref{fig:histogram}) does not make any assumption about the data under study. Instead, it counts the occurrences of each unique value in each column of the expanded tuple and for each set of potentially correlated sub-tuples (as suggested by the analysis module). These counts, accumulated over the entire dataset, provide a \emph{de facto} distribution of the data in each field and set of correlated fields.

%Described as above, the histogram modeling strategy is rather memory inefficient: it requires keeping track of each value of each expanded tuple that the model comes across.
To limit memory usage, and to speed up the modeling phase, we discard histograms as soon as they reach a certain size -- say, 16 bins. Discarding histograms when their number of bins reaches a fixed threshold is just one of a number of heuristics that could be implemented here; the idea is that a profusion of different values, all repeating infrequently, is unlikely to provide valuable insight as far as outlier detection is concerned (as an extreme example, the histogram of an attribute with no repeated values would only have one value per bin, and would not yield any insight about the data). With this discarding heuristic applied, histograms are quick to generate and extremely memory efficient.

Histograms also have the valuable property of treating sets of fields (obtained via correlation analysis) and single fields in the exact same way, thus permitting to model single columns or groups of attributes indifferently. Finally, because they make no assumption about the data they manipulate (aside from the requirement that it be of small cardinality), histograms are able to accurately describe a broad class of discrete distributions.

\subsubsection{Meta-modeling through attribute-based partitioning}
\label{sec:partitioning}

The models presented above treat attributes and sets of correlated attributes as a whole. In some cases, however, it is possible to identify sub-populations of tuples by scrutinizing certain expanded attributes of the data; these sub-populations can then be studied separately, yielding more insight and better outlier classification performance.

As an example, consider the case of an airline adjusting status levels for its frequent fliers, using the number of flights for each passenger as well as their status level. A non-partitioned analysis may not return any interesting information, but a partitioned analysis could single out passengers in lower status levels traveling significantly more than average, or passengers with higher status traveling rarely. This would work even if statuses were stored as textual data, with no indication of their relative rankings.

The general approach, given a dataset and a pre-existing model, therefore consists in extracting sets of attributes based on correlation hints provided by earlier stages of the pipeline, and dividing each group of attributes between a single key (in the example, the status level) and one or more sub-population attributes (in the example, the number of flights). One instance of the selected model is then built for each value of the key. For example, if the statistical analysis phase highlights a correlation between columns $A$ (status levels) and $B$ (number of flights), and column $A$ contains values $a_1, \dots, a_n$ (\texttt{bronze}, \texttt{silver}, \texttt{gold}, \dots), then we distribute the pairs $(A, B)$ into $n$ partitions based on the value of $A$; values of $B$ in each of these partitions are then modeled independently (in the example, this yields a different model of flights count for each status level).

This type of approach is useful when the distribution for an attribute or set of attributes is multi-modal. A high-level non-partitioned analysis will reveal values that fall in none of the classes; a partitioned approach, on the other hand, may more easily reveal discrepancies by suppressing interference between each class.

In addition to providing better classification accuracy, partitioning may lead to better runtime performance by diminishing the size of the dataset covered by each model. These benefits are especially important when model construction performance does not scale linearly, and when data volumes are too large to be analyzed on a single machine.

Finally, attribute-based partitioning allows for previously impossible analysis. Assuming for example that a dataset with two columns has 4 classes identified by the value in the first column, each with 10 distinct expected values in the second column, a generic histogram-based analysis would discard the histogram for the pair of values as having too many buckets (40). A partitioned analysis, on the other hand, would allow the construction of four histograms, each with 10 regular bins and potentially a few outliers.

In our prototype implementation, we focused on partitioning applied to the discrete histogram case; the technique, however, generalizes to all the models presented above.


\documentclass{article}
\pagestyle{empty}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{graphicx}
\usepackage{multicol}
\setlength{\oddsidemargin}{0in} \setlength{\evensidemargin}{0in}
\setlength{\topmargin}{0in} \setlength{\textheight}{8.5in}
\setlength{\textwidth}{6.5in}

\makeatletter
\renewcommand*\env@matrix[1][*\c@MaxMatrixCols c]{%
	\hskip -\arraycolsep
	\let\@ifnextchar\new@ifnextchar
	\array{#1}}
\makeatother

\begin{document}
\begin{flushleft}
	\bfseries{MATH 260, Linear Systems and Matrices, Fall `14}\\
	\bfseries{Activity 12:  Diagonalization}\\
%	\bfseries{Honor Code:} \hspace{3.5in}\bfseries{Names:}\\
\end{flushleft}
\begin{flushleft}

\section*{Warm-up:  Eigenvectors and Spaces of Repeated Roots}
%This is the matrix from Sect. 5.4, pg 339, #43

$\textbf{G}
=\begin{bmatrix}
1 & 0 & 0\\
-4 & 3 & 0 \\
-4 & 2 & 1
\end{bmatrix}
$
\hspace{0.5in}
The eigenvalues of \textbf{G} are $\lambda_{1,2}=1$ and $\lambda_3 = 3$.

\vspace{0.25in}
1) If I had asked you to find the eigenvalues of \textbf{G}, you would NOT need to calculate $\det(\textbf{G}-\lambda \textbf{I})=0$. Explain why.

\vspace{1in}

2) Find the three eigenvectors associated with the given eigenvalues.

\vspace{3in}

3) What is the dimension of each corresponding eigenspace (recall, these are the geometric multiplicities of each eigenvalue)? What is the sum of the eigenspace dimensions for \textbf{G}?

\vspace{1in}

\section*{Diagonalized Equations}
Define two matrices: $\textbf{P} = \begin{bmatrix}
| & | & & |\\
\vec{v}_1 & \vec{v}_2 & \ldots &\vec{v}_n \\
| & | & & |
\end{bmatrix} $
\hspace{0.5in}
$\textbf{D} = \begin{bmatrix}
\lambda_1 & 0 & \ldots & 0 \\
0 & \lambda_2 & \ldots & 0 \\
\vdots & \vdots & \ddots & \vdots \\
0 & 0 & \ldots & \lambda_n
\end{bmatrix} $

\vspace{0.1in}

We are going to check some matrix equations using \textbf{P}, $\textbf{P}^{-1}$ and \textbf{D}.
For each of the following be sure to show some intermediate steps, since we already know what the final result should be!

\vspace{0.1in}

For \# 4-8 use the matrix: $\textbf{A} = \begin{bmatrix} 1 & 1 \\ 4 & 1 \end{bmatrix}$ \hspace{0.2in} with eigenvalues $\lambda_1 = 3$ and $\lambda_2 = -1$ with corresponding eigenvectors $\vec{v}_1 = \begin{bmatrix} 1 \\ 2 \end{bmatrix}$ and $\vec{v}_2 = \begin{bmatrix} 1 \\ -2 \end{bmatrix}$ respectively.

\vspace{0.2in}

4) Find $\textbf{AP}$ and $\textbf{PD}$. How do they compare?

\vspace{2.5in}

5) Find $\textbf{PDP}^{-1}$. What does it equal that we have already defined? \\
\textit{Hint: You already calculated $\textbf{PD}$ you just need to find $\textbf{P}^{-1}$, and do one multiplication}

\vspace{2.5in}

6) Find $\textbf{P}^{-1} \textbf{AP}$. What does it equal that we have already defined?\\
\textit{Hint: You only need to do ONE new multiplication here!}

\vspace{2in}

7) Write all three of the equations you have calculated above here, using the variables \textbf{A}, \textbf{P}, \textbf{D}, and $\textbf{P}^{-1}$.

\vspace{1in}

8)a) Define the matrix $\textbf{F}=\begin{bmatrix} \lambda_2 & 0 \\ 0 & \lambda_1 \end{bmatrix} = \begin{bmatrix} -1 & 0 \\ 0 & 3 \end{bmatrix}$, and use the same \textbf{P} as above. Find $\textbf{PFP}^{-1}$.

\vspace{2.5in}

b) Does it matter to the equations you wrote in Problem 7 what order the eigenvalues and eigenvectors are in \textbf{P} and \textbf{D}? What relationship is required between \textbf{P} and \textbf{D}?

\vspace{1in}

\section*{Diagonalizing with Repeated Eigenvalues}
9) In the warm-up you found the eigenvectors for a matrix with a repeated eigenvalue, but which had 3 linearly independent eigenvectors. Pick your favorite equation from \# 6. Is the equation valid for \textbf{G}?

\vspace{2.5in}

%The matrix below is from Sect 5.4, pg 339, #29
10) The matrix $\begin{bmatrix} 3 & 1 \\ -1 & 5 \end{bmatrix}$ has a repeated eigenvalue of 4, with one eigenvector of $\vec{v}_1=\begin{bmatrix} 1 \\ 1 \end{bmatrix}$. \\
If we let $\textbf{P}=\begin{bmatrix} 1 & 1 \\ 1 & 1 \end{bmatrix}$ and $\textbf{D} = \begin{bmatrix} 4 & 0 \\ 0 & 4 \end{bmatrix}$. Does the equation $\textbf{A} = \textbf{PDP}^{-1}$ still hold? \\

\textit{Hint: Think about \textbf{P}. What is $det(\textbf{P})$? What does that tell you about $\textbf{P}^{-1}$? }

\vspace{2in}

11) If the results from \# 9 and \# 10 are generalizable (they are), what must be true about a matrix for us to diagonalize it?

\vspace{1in}

\end{flushleft}
\end{document}

Fall 2014 - Capaldi A/Activities/Activity12_Diagonalization.tex


\documentclass[12pt]{amsart}

\usepackage{graphicx}
\usepackage{amsthm}

\begin {document}

\title{Find Your Path}
\author {Shannon Lane}
%\address{...}

\begin{abstract}
We will demonstrate how to find a polynomial that passes through a given set of data using polynomial interpolation. % %source
In this paper we will look at a set of data given by the finite sequence $a_1, a_2,\hdots,a_n$. In order to find the polynomial we will use properties of the Vandermonde Matrix.
%We will demonstrate how to use polynomial interpolation to create a polynomial function which hits every point in the sequence.

\end{abstract}

\maketitle

\section{Introduction}
Consider the sequence $a_1, a_2,\hdots, a_n$. The goal is to find a polynomial of degree $n-1$, such that $p(k)=a_k$, for all $k=1, 2,\hdots,n.$ In other words, the graph of $p(x)$ must go through every point $(x,y)=(k,a_k)$ for all $k=1, 2,\hdots,n$.

\indent Since the general formula for a polynomial of degree $n-1$ can be written as
\begin{equation}
p(x)=\alpha_{n}x^{n-1}+\alpha_{n-1}x^{n-2}+\hdots+\alpha_2x+\alpha_1,
\end{equation}
our task is to determine the coefficients $\alpha_1, \alpha_2,\hdots, \alpha_k$ for all $k=1, 2,\hdots,n$.

\indent For example, in the case $n=3$, we obtain

\begin{equation*}
\begin{array}{l}
p(a_1)=\alpha_3(1)^2+\alpha_2(1)+\alpha_1 ,\\
p(a_2)=\alpha_3(2)^2+\alpha_2(2)+\alpha_1 ,\\
p(a_3)=\alpha_3(3)^2+\alpha_2(3)+\alpha_1.
\end{array}
\end{equation*}
The system of equations above would be used to solve for $\alpha_3$, $\alpha_2$, and $\alpha_1$. Another way to solve for the $\alpha$'s is through matrices. To demonstrate this method, lets consider a simple example.

%When given a sequence of

%The purpose of this paper is the take a sequence of numbers and find a polynomial that hits every point. We are give a sequence $a1, a2,...,ak$. the function p(x) must represent each p(n) values from n=1, 2,...,k. To better understand this it is important to identify what each variable corresponds to one another.

%n corresponds to the x values in the polynomial p(x) and the an correspond to the y values found by p(x).

\subsection{Example}
Find the line that passes through the points (1,1) and (2,4).
First the slope is $\frac{4-1}{2-1}=3$. The corresponding $y$-intercept is $y=3x+\alpha_1$, by using $(x,y)=(1,1)$, we obtain $1=3(1)+\alpha_1$, and $\alpha_1=-2$. Thus, the equation is $y=3x-2$.

Now lets look at this example using matrices.
The system of equations is derived by using the points (1,1) and (2,4) in the general equation $x\alpha_2+\alpha_1=y$.
The system of equations are
\begin{equation*}
\begin{array}{l}
1=(1)\alpha_2+\alpha_1,\\
4=(2)\alpha_2+\alpha_1.
\end{array}
\end{equation*}

Let us write the latter system of equations in the matrix form. We get
\begin{equation*}
\left[ {\begin{array}{cc}
1 & 1 \\
1 & 2 \\
\end{array} } \right]
\left[ {\begin{array}{c}
\alpha_1 \\
\alpha_2 \\
\end{array} } \right]=
\left[ {\begin{array}{c}
1 \\
4 \\
\end{array} } \right].
\end{equation*}
To find $\alpha_2$ and $\alpha_1$ the inverse of the matrix $\left[ {\begin{array}{cc}
1 & 1 \\
1 & 2 \\
\end{array} } \right]$ needs to be applied to both sides.
Thus, we obtain the coefficients as follows
\begin{equation*}
\left[ {\begin{array}{c}
\alpha_1 \\
\alpha_2 \\
\end{array} } \right]=
\left[ {\begin{array}{cc}
1 & 1 \\
1 & 2 \\
\end{array} } \right]^{-1}
\left[ {\begin{array}{c}
1 \\
4 \\
\end{array} } \right]
=\left[ {\begin{array}{c}
-2 \\
3 \\
\end{array} } \right].
\end{equation*}

This result is the same result from deriving the system through known equations.
Indeed, to find a straight line between two points ($x_1$,$y_1$) and ($x_2$,$y_2$). First the slope would be found by $\frac{y_2-y_1}{x_2-x_1}$. Then the corresponding $y$-intercept value would be calculated. These values would be inserted into the general formula $x\alpha_2+\alpha_1=y$.

In this paper we will expand this example for $k=1, 2,\hdots, n$ in the sequence.
\section{Examples}
In this section we will look at more general examples, which will allow us to draw a conclusion on how to find a general method to find a polynomial that goes through the sequence $a_1, a_2,\hdots, a_k$ for $k=1, 2,\hdots,n$.
\subsection{Example}
Given the sequence 2, 5, 12 find the polynomial $p(x)$. Let
\begin{equation*}
a_1=2, a_2=5, \text{ and }a_3=12.
\end{equation*}
Thus, below is the system in matrix form \begin{equation*}
\left[ {\begin{array}{ccc}
1&(2)^1 & (2)^2 \\
1&(5)^1 & (5)^2 \\
1&(12)^1 & (12)^2
\end{array} } \right]*
\left[ {\begin{array}{c}
\alpha_1 \\
\alpha_2 \\
\alpha_3
\end{array} } \right]=
\left[ {\begin{array}{c}
2\\
5\\
12
\end{array} } \right].
\end{equation*}
After solving for the $\alpha$'s, we get $p(x)=2x^2-3x+3$.

Figure \ref{smile} below shows the points given by the sequence and $p(x)$.

\subsection{Example}
Consider the sequence $a_1, a_2, a_3, a_4, a_5$.
The following values for $p(k)$ are
\begin{equation*}
p(1)=a_1, p(2)=a_2, p(3)=a_3, p(4)=a_4, p(5)=a_5.
\end{equation*}
We get the following system of equations
\begin{equation*}
\begin{array}{l}
\alpha_5(1)^4+\alpha_4(1)^3+\alpha_3(1)^2+\alpha_2(1)+\alpha_1=a_1 ,\\
\alpha_5(2)^4+\alpha_4(2)^3+\alpha_3(2)^2+\alpha_2(2)+\alpha_1=a_2 ,\\
\alpha_5(3)^4+\alpha_4(3)^3+\alpha_3(3)^2+\alpha_2(3)+\alpha_1=a_3 , \\
\alpha_5(4)^4+\alpha_4(4)^3+\alpha_3(4)^2+\alpha_2(4)+\alpha_1=a_4 ,\\
\alpha_5(5)^4+\alpha_4(5)^3+\alpha_3(5)^2+\alpha_2(5)+\alpha_1=a_5.
\end{array}
\end{equation*}
Notice the equations are to the $n-1$ degree, since a polynomial has $n$ coefficients and we are finding a polynomial of degree $n-1$ to go through all the points $(k,a_k)$ for $k=1, 2,\hdots,n$.
The system of equations in matrix form
\begin{equation*}
\left[ {\begin{array}{ccccc}
1 & (1)^1 & (1)^2 & (1)^3 & (1)^4 \\
1 & (2)^1 & (2)^2 & (2)^3 & (2)^4 \\
1 & (3)^1 & (3)^2 & (3)^3 & (3)^4 \\
1 & (4)^1 & (4)^2 & (4)^3 & (4)^4 \\
1 & (5)^1 & (5)^2 & (5)^3 & (5)^4
\end{array} } \right]
\left[ {\begin{array}{c}
\alpha_1 \\
\alpha_2 \\
\alpha_3 \\
\alpha_4 \\
\alpha_5 \\
\end{array} } \right]=
\left[ {\begin{array}{c}
a_1 \\
a_2 \\
a_3\\
a_4\\
a_5\\
\end{array} } \right].
\end{equation*}
Note, the left column in the first matrix is all ones because $\alpha_1$ is a single coefficient and not multiplied by any value of $x$. Now like the previous example the inverse matrix is applied to both sides. To get the following system
\begin{equation*}
\left[ {\begin{array}{c}
\alpha_1 \\
\alpha_2 \\
\alpha_3 \\
\alpha_4 \\
\alpha_5
\end{array} } \right]=
\left[ {\begin{array}{ccccc}
1& (1)^1 & (1)^2 & (1)^3 & (1)^4 \\
1& (2)^1 & (2)^2 & (2)^3 & (2)^4 \\
1& (3)^1 & (3)^2 & (3)^3 & (3)^4 \\
1& (4)^1 & (4)^2 & (4)^3 & (4)^4 \\
1& (5)^1 & (5)^2 & (5)^3 & (5)^4
\end{array} } \right]^{-1}
\left[ {\begin{array}{c}
a_1 \\
a_2 \\
a_3\\
a_4\\
a_5
\end{array} } \right].
\end{equation*}
The system above will solve for $\alpha_5$, $\alpha_4$, $\alpha_3$, $\alpha_2 \text{ and } \alpha_1$. These coeffients are put into Formula (1) to get a polynomial that goes through all $a_k$ where $k=1, 2, 3, 4, 5$.

\section{General Process and Connection with Vandermonde Matrix}
Given the sequence $a_1, a_2,\hdots,a_n$, find the polynomial that goes through $p(k)=a_k$ for $k=1, 2,\hdots,n$.
The system of equations are
\begin{align*}
%\begin{array}{l}
\alpha_{n}(x_1)^{n-1}+\alpha_{n-1}(x_1)^{n-2}+\hdots+\alpha_2(x_1)+\alpha_1&=a_1\\
\alpha_{n}(x_2)^{n-1}+\alpha_{n-1}(x_2)^{n-2}+\hdots+\alpha_2(x_2)+\alpha_1&=a_2\\
\alpha_{n}(x_3)^{n-1}+\alpha_{n-1}(x_3)^{n-2}+\hdots+\alpha_2(x_3)+\alpha_1&=a_3\\
&\,\,\,\vdots\\
\alpha_{n}(x_{n-1})^{n-1}+\alpha_{n-1}(x_{n-1})^{n-2}+\hdots+\alpha_2(x_{n-1})+\alpha_1&=a_{n-1}\\
\alpha_{n}(x_n)^{n-1}+\alpha_{n-1}(x_n)^{n-2}+\hdots+\alpha_2(x_n)+\alpha_1&=a_n.
%\end{array}
\end{align*}

For simpler notation let,
\begin{equation*}
\begin{array}{c}
V
\end{array}=
\left[ {\begin{array}{ccccc}
1& (x_1)^1 &\hdots& (x_1)^{n-2} & (x_1)^{n-1} \\
1& (x_2)^1 &\hdots& (x_2)^{n-2} & (x_2)^{n-1} \\
1& (x_3)^1 &\hdots& (x_3)^{n-2} & (x_3)^{n-1} \\
\vdots\\
1&(x_{n-1})^1&\hdots& (x_{n-1})^{n-2} & (x_{n-1})^{n-1}\\
1&(x_{n})^1&\hdots& (x_{n})^{n-2} & (x_{n})^{n-1}\\

\end{array} } \right],
\end{equation*}
\begin{equation*}
\begin{array}{c}
a
\end{array}=
\left[ {\begin{array}{c}
\alpha_1 \\
\alpha_2 \\
\alpha_3 \\
\vdots\\
\alpha_{n-1} \\
\alpha_n \\

\end{array} } \right],
\begin{array}{c}
Y
\end{array}=
\left[ {\begin{array}{c}
a_1 \\
a_2 \\
a_3\\
\vdots\\
a_{n-1}\\
a_n
\end{array} } \right].
\end{equation*}
Therefore, the corresponding matrix for the above system of equations is $Va=Y$.
Notice in matrix V in each column the $x$ value is raised to the power $n-1, n-2,\hdots,1, 0$. This matrix is more commonly known as a Vandermonde Matrix, after Alexander-Theophile Vandermonde who was a French musician and chemist in the late 1700's \cite{Klinger}.

The same inverse application in the previous examples is applied to the Vandermonde matrix. Thus, resulting in the final matrix calculation is $a=V^{-1}Y$.
After solving for the $\alpha$'s, we will be able to put the coeffencients in the equation
\begin{equation*}
p(x)=\alpha_{n}x^{n-1}+\hdots+\alpha_2x+\alpha_1
\end{equation*}
which will go through all the points in the sequence given. This process of finding the exact equation that passes through any set of data is called polynomial interpolation \cite{Walsh}.
The following theorem has been proven by the work above.

\begin{theorem}
Given a sequence $a_1, a_2,\hdots,a_n$ for $k=1, 2,\hdots, n$. The formula to find the coefficients of a polynomial of degree $n-1$ satisfying $p(k)=a_k$ for all $k=1, 2,\hdots, n$ is given by $a=V^{-1}Y$.
\end{theorem}
The next section will show an explicit formula for $V^{-1}$.
\section{Vandermonde Matrix Inverse}
The Vandermonde Matrix has an explicit inverse, proven by Richard Turner \cite{Turner}. The inverse can be found by the following matrices below. Let
\begin{equation*}
U=
\left[ {\begin{array}{ccccc}
1& -x_1 &x_1x_2&-x_1x_2x_3&\hdots \\
0&1&-(x_1+x_2)&(x_1x_2+x_2x_3+x_3x_1)&\hdots \\
0&0&1&-(x_1+x_2+x_3)&\hdots \\
0&0&0&1&\hdots\\
\vdots&\vdots&\vdots&\vdots&\ddots\\
\end{array} } \right],
\end{equation*}
\begin{equation*}
L=
\left[ {\begin{array}{cccc}
1& 0 &0&\hdots \\
\frac{1}{x_1-x_2} &\frac{1}{x_2-x_1}&0&\hdots \\
\frac{1}{(x_1-x_2)(x_1-x_3)} & \frac{1}{(x_2-x_1)(x_2-x_3)}&\frac{1}{(x_3-x_1)(x_3-x_1)}&\hdots \\
\vdots&\vdots&\vdots&\ddots\\
\end{array} } \right].
\end{equation*}
Note, if the $x$-values of the points of interpolation are known, then $U$ and $L$ can be directly computed. Along with, $U$ is a upper triangular matrix and $L$ is a lower triangular matrix. The product of $UL$ will generate $V^{-1}$ for a polynomial of degree $n-1$.
\subsection{Example} Consider the sequence 2, 5, 12 from Example 2.1 and find the polynomial $p(x)$. The composition of $U$ and $L$ are
\begin{equation*}
U=
\left[ {\begin{array}{ccc}
1&-1&2 \\
0&1&-3\\
0&0&1
\end{array} } \right],
L=
\left[ {\begin{array}{ccc}
1&0&0 \\
-1&1&0 \\
\frac{1}{2}&-1&\frac{1}{2}
\end{array} } \right].
\end{equation*}

Thus, $V^{-1}$ is
\begin{equation*}
UL=
\left[ {\begin{array}{ccc}
3&-3&1\\
-\frac{5}{2}&4&-\frac{3}{2}\\
\frac{1}{2}&-1&\frac{1}{2}
\end{array} } \right].
\end{equation*}
To compute $\alpha_3, \alpha_2, \text { and } \alpha_1$, the product $UL$ is multiplied with matrix $Y$, where Y
$=\left[ {\begin{array}{c}
2\\
5\\
12
\end{array} } \right].$ The final calculation is
\begin{equation*}(UL)*A = \left[ {\begin{array}{c}
3\\
-3\\
2
\end{array} } \right]
\end{equation*}
which is the same values for $\alpha$'s obtained in Example 2.1. This verifies that $UL=V^{-1}$.

Due to the length of this paper, we will not prove the explicit inverse since it has previously been proven.
\section{Conclusions and Outlook}

We have shown through polynomial interpolation we can find a polynomial of degree $n-1$ that passes through any give sequence $a_1, a_2,\hdots,a_n$ for $k=1, 2,\hdots, n$ at integer points. We can solve for the coefficients in the polynomial by the matrix equation $a=V^{-1}Y$. Our approach arises several questions. Can a sequence produce a piecewise function? Can exponential equations be represented better with a different type of interpolation?

\begin{thebibliography}{9}
\bibitem{Klinger} Klinger, Allen. "The Vandermonde Matrix." American Mathematical Monthly. 74.5 (1967): 571-574. Print.
\bibitem{Turner} Turner, L R. Inverse of the Vandermonde Matrix with Applications. Washington, D.C: National Aeronautics and Space Administration, 1966. Print.
\bibitem{Walsh} Walsh, JL. "Note on Polynomial Interpolation to Analytic Functions." Proc Natl Acad Sci U S A.. 19.11 (1933): 959-63. Print.
\end{thebibliography}
\end {document}