\documentclass[12pt]{article}
\usepackage{amsfonts}
\usepackage{bm}

% symbol used for sqrt(-1)
\newcommand{\Ci}{{\rm i}}

\newcommand{\C}{\mathbb{C}}
\newcommand{\R}{\mathbb{R}}

\newcommand{\code}[1]{{\tt{#1}}}

\newcommand{\boost}[1][]{{\ensuremath{ {\bf H}{#1}_{\bm{\hat m}}(\beta) }}}
\newcommand{\rotat}[1][]{{\ensuremath{ {\bf U}{#1}_{\bm{\hat n}}(\phi) }}}

\newcommand{\pauli}[1]{\ensuremath{ {\bm\sigma}_{\rm #1} }}

\begin{document}

\section{Introduction}

Quaternions were invented by William Rowan Hamilton when he carved the
fundamental formula of quaternion algebra,
\begin{equation}
i^2=j^2=k^2=ijk=-1
\end{equation}
into the stone of the Brougham bridge.
An arbitrary quaternion, $\bm{U}$, is given by the linear combination,
\begin{equation}
\bm{U}=u_0+u_1i+u_2j+u_3k, \hspace{5mm} u_i\in\R
\end{equation}
and, as with matrices, quaternion multiplication is non-commutative.

A quaternion with complex coefficients is known as a biquaternion.
The algebra of biquaternions is isomorphic to the $2\times2$ complex
matrix group, and may be
represented by the linear combination,
\begin{equation}
\bm{B}=b_0\pauli{0}+b_1\pauli{1}+b_2\pauli{2}+b_3\pauli{3},\hspace{5mm}b_i\in\C
\end{equation}
where $\pauli{0}$ equals the identity matrix, $\bm{I}$, and $\pauli{1-3}$
are the Pauli spin matrices:
\begin{eqnarray}
\pauli{1} = \left( \begin{array}{cc}
1 & 0 \\
0 & -1
\end{array}\right)
&
\pauli{2} = \left( \begin{array}{cc}
0 & 1 \\
1 & 0
\end{array}\right)
&
\pauli{3} = \left( \begin{array}{cc}
0 & -\Ci \\
\Ci & 0
\end{array}\right).
\label{eqn:pauli}
\end{eqnarray}
From the above definitions, it can be seen that the Pauli spin
matrices have the following properties:
\begin{equation}\label{eqn:properties}
\pauli{i}^2 = \bm{I}, \hspace{5mm}
\pauli{i}\pauli{j} = -\pauli{j}\pauli{i} = \Ci\pauli{k}
\end{equation}
where $\{i,j,k\}$ is chosen from cyclic permutations of $\{1,2,3\}$.

A very useful notation represents the biquaternion as a scalar plus
a three-vector:
\begin{equation}\label{eqn:biquaternion}
\bm{B}=[b+\bm{b}] = b\pauli{0} + \bm{b\cdot\sigma}, \hspace{5mm} b_i\in\C
\end{equation}
were $\bm{b}=(b_1,b_2,b_3)$ and $\bm\sigma$ is a three-vector whose
components are the Pauli spin matrices as defined above.  Using this
notation, the left-handed quaternion sub-group (see
Section~\ref{sec:qmult}) may be represented as
\begin{equation}\label{eqn:quaternion}
\bm{Q}=[q+i\bm{q}]= q\pauli{0} + \Ci\bm{q\cdot\sigma}, \hspace{5mm} q_i\in\R
\end{equation}

\section{Some Algebraic Derivations}

\subsection{Biquaternion Multiplication}
As with matrices, quaternion multiplication is non-commutative.  It is
useful to derive the result of the multiplication of two quaternions
in some detail. Start with two biquaternions, $\bm{A}=[a_0+\bm{a}]$
and $\bm{B}=[b_0+\bm{b}]$, as defined in Eq.~\ref{eqn:biquaternion},
and expand to yield
\begin{eqnarray}
\begin{array}{ccl}
\bm{A}\bm{B} & = &
(a_0\pauli{0} + a_1\pauli{1} + a_2\pauli{2} + a_3\pauli{3})
(b_0\pauli{0} + b_1\pauli{1} + b_2\pauli{2} + b_3\pauli{3}) \\
& = & a_0b_0\pauli{0}\pauli{0} + a_0b_1\pauli{0}\pauli{1}
	+ a_0b_2\pauli{0}\pauli{2} + a_0b_3\pauli{0}\pauli{3} \\
& + & a_1b_0\pauli{1}\pauli{0} + a_1b_1\pauli{1}\pauli{1}
	+ a_1b_2\pauli{1}\pauli{2} + a_1b_3\pauli{1}\pauli{3} \\
& + & a_2b_0\pauli{2}\pauli{0} + a_2b_1\pauli{2}\pauli{1}
	+ a_2b_2\pauli{2}\pauli{2} + a_2b_3\pauli{2}\pauli{3} \\
& + & a_3b_0\pauli{3}\pauli{0} + a_3b_1\pauli{3}\pauli{1}
	+ a_3b_2\pauli{3}\pauli{2} + a_3b_3\pauli{3}\pauli{3}
\end{array}
\end{eqnarray}
By applying the properties shown in Eq.~\ref{eqn:properties}, this may
be reduced to
\begin{eqnarray}\label{eqn:biquaternion_multiplication}
\begin{array}{ccl}
\bm{A}\bm{B}
& = & (a_0b_0 + a_1b_1 + a_2b_2 + a_3b_3) \pauli{0} \\
& + & (a_0b_1 + a_1b_0 + \Ci a_2b_3 - \Ci a_3b_2) \pauli{1} \\
& + & (a_0b_2 - \Ci a_1b_3 + a_2b_0 + \Ci a_3b_1) \pauli{2} \\
& + & (a_0b_3 + \Ci a_1b_2 - \Ci a_2b_1 + a_3b_0) \pauli{3}
\end{array}
\end{eqnarray}

\subsection{Left-Handed Quaternion Multiplication}
\label{sec:qmult}
Equation~\ref{eqn:biquaternion_multiplication} may be used to derive
the multiplication rule for left-handed quaternions.  Assuming that
$\bm{C}=[c_0+\Ci\bm{c}]$ and $\bm{D}=[d_0+\Ci\bm{d}]$, as defined in
Eq.~\ref{eqn:quaternion}, the quaternion product,
$\bm{X}=[x_0+\Ci\bm{x}]=\bm{CD}$, is given by
\begin{equation}\label{eqn:quaternion_multiplication}
\begin{array}{rl}
x_0 & = c_0d_0 - c_1d_1 - c_2d_2 - c_3d_3 \\
x_1 & = c_0d_1 + c_1d_0 - c_2d_3 + c_3d_2 \\
x_2 & = c_0d_2 + c_1d_3 + c_2d_0 - c_3d_1 \\
x_3 & = c_0d_3 - c_1d_2 + c_2d_1 + c_3d_0
\end{array}
\end{equation}

Notice that this result differs from the product of two quaternions as
originally defined by Hamilton.  This is because the left-handed
quaternion basis is given by the Pauli spin matrices multplied by the
imaginary number, $\Ci=\sqrt{-1}$, which satisfy:
\begin{equation}
\Ci\pauli{i} \Ci\pauli{j} = -\Ci\pauli{k}
\end{equation}

\section{Isomorphism with Jones matrices}

An arbitrary $2\times2$ complex, or Jones, matrix may be represented
by its polar decomposition,
\begin{equation}
{\bf J} = J \; \boost \rotat,
\label{eqn:polar_decomposition}
\end{equation}
where $J=(\det{\bf J})^{1/2}$,
\begin{eqnarray}
\boost &=& \exp(\bm{\sigma\cdot}\bm{\hat m}\beta)
        = [\cosh\beta,\sinh\beta\;\bm{\hat m}],
\label{eqn:boost} \\
\rotat &=& \exp(i\bm{\sigma\cdot}\bm{\hat n}\phi)
        = [\cos\phi,i\sin\phi\;\bm{\hat n}],
\label{eqn:rotation}
\end{eqnarray}
and $\bm{\hat m}$ and $\bm{\hat n}$ are real-valued unit 3-vectors.
The matrices, \rotat\ and \boost, are unimodular (ie. have unit
determinant) and are known as the axis-angle representation of unitary
and Hermitian transformations, respectively. The same matrix, ${\bf J}$,
may also be represented by
\begin{equation}
{\bf J}=J\;\rotat\boost[^\prime],
\end{equation}
where $\boost[^\prime]=\rotat[^\dagger]\boost\rotat$.  The polar
decomposition of Eq.~\ref{eqn:polar_decomposition} may be calculated
noting that
\begin{equation}
{\bf JJ}^\dagger = (J\boost\rotat)(\rotat[^\dagger]\boost[^\dagger]J^*)
	= |J|^2 \boost[^2]
\end{equation}
So that:
\begin{equation}\label{eqn:sqrt}
\boost = {\sqrt{{\bf JJ}^\dagger} \over |J|}
\end{equation}
and
\begin{equation}
\rotat = \boost[^{-1}] {{\bf J} \over J}
\end{equation}
Note that equation~\ref{eqn:sqrt} requires the definition of a matrix
square root, the solution of which is given in Hamaker 2000, Appendix B.5.

\section{Fast Quaternion Fourier Transform}

An efficient implementation of the two-dimensional quaternion Fourier
Transform has been implemented using the 2-D complex FFT \cite{pdc01}.
The result is summarized and simplified for the one-dimensional case
using left-handed quaternions.  The one-dimensional quaternion Fourier
transform is given by:
\begin{equation}
\bm{H}(\nu)=\intop_{-\infty}^\infty
	\exp(-\Ci\;\bm{\sigma\cdot\hat\mu}\;2\pi\nu t)\;\bm{H}(t)\;dt
\end{equation}
where $\bm{\hat\mu}$ is a real-valued unit three-vector and
$\bm{H}(t) = [h_0(t) + \Ci\bm{h}(t)]$ is a left-handed quaternion-valued
function of the real variable, $t$.

\noindent
By defining three new basis vectors:
\begin{equation}
\bm{\hat\mu}_k = \sum_{i=1}^3 \mu_{k,i}\Ci\pauli{i} \hspace{1cm} 1\le k \le 3
\end{equation}
such that $\bm{\hat\mu}_1=\Ci\bm{\hat\mu}$,
$\bm{\hat\mu}_k^2=-\pauli{0}$, and
$\bm{\hat\mu}_i\bm{\hat\mu}_j=-\bm{\hat\mu}_k$, $\bm{H}(t)$ may be
expressed as:
\begin{equation}
\bm{H}(t) = h_0(t) + \sum_{i=1}^3 h_i^\prime(t)\;\bm{\hat\mu}_i
\end{equation}
where $\bm{h}^\prime(t)=\bm{R}^{-1}\bm{h}(t)$ and
\begin{equation}
\bm{R} = \left( \begin{array}{ccc}
\mu_{1,1} & \mu_{2,1} & \mu_{3,1} \\
\mu_{1,2} & \mu_{2,2} & \mu_{3,2} \\
\mu_{1,3} & \mu_{2,3} & \mu_{3,3} \\
\end{array}\right)
\end{equation}
By the definitions of $\bm{\hat\mu}_k$, $\bm{H}(t)$ may be re-expressed as:
\begin{equation}
\bm{H}(t) = c_1(t) + c_2(t)\bm{\hat\mu}_2
\end{equation}
where
\begin{eqnarray}
c_1(t) & = & h_0(t)+h_1^\prime(t)\bm{\hat\mu}_1 \\
c_2(t) & = & h_2^\prime(t)-h_3^\prime(t)\bm{\hat\mu}_1
\end{eqnarray}
Finally, the quaternion Fourier transform may be re-expressed as:
\begin{equation}\label{eqn:fqft}
\bm{H}(\nu)=\intop_{-\infty}^\infty
		\exp(-2\pi\nu t\;\bm{\hat\mu}_1)\;c_1(t)\;dt
	+ \intop_{-\infty}^\infty
		\exp(-2\pi\nu t\;\bm{\hat\mu}_1)\;c_2(t)\;dt\;\bm{\hat\mu}_2
\end{equation}
As $\bm{\hat\mu}_1^2=-1$, both integrals in the right-hand side of
Equation~\ref{eqn:fqft} are simply complex Fourier transforms.

\newpage
\section{The \code{Quaternion} template class}

The \code{Quaternion} template class was designed to implement both
left-handed quaternions and biquaternions using the Pauli spin
matrices, with purely imaginary or purely real vector components, as
the basis.  When using the Pauli spin matrices with real components,
it is possible to represent a Hermitian matrix using only four real
numbers.  However, as the product of two Hermitian matrices is not
Hermitian (unless the matrices commute), it is not possible to
multiply two real quaternions in the Pauli basis.

In summary, there are four ways to create a null-constructed instance
of a Quaternion object:
\begin{verbatim}
// Left-handed quaternion (unitary basis)
Quaternion<float> q;

// Representation of Hermitian matrix (non-multiplicative)
Quaternion<float, Hermitian> h;

// Biquaternion in unitary basis
Quaternion<complex<float>, Unitary> bq;

// Biquaternion in Hermitian basis
Quaternion<complex<float>, Hermitian> bh;
\end{verbatim}
Note that the floating-point precision is specified in the first
template argument and the basis, \code{Unitary} or \code{Hermitian},
is specified in the second template argument. As the left-handed
quaternion is most closely related to Hamilton's quaternion, the
Unitary basis is used by default.

\begin{thebibliography}{}

\bibitem{pdc01}
Pei, S., Ding, J., Chang, J. 2001,  IEEE Trans. Signal Process. Vol. 49,
No. 11, 2783

\end{thebibliography}

\end{document}

   Efficient implementation of quaternion Fourier transform, convolution,
   and correlation by 2-D complex FFT.
   Pei, Soo-Chang; Ding, Jian-Jiun; Chang, Ja-Han
   IEEE Trans. Signal Process. 49 (2001), no. 11, 2783--2797, Math. Sci. Net.


\documentclass[]{article}
\usepackage{amsmath}

%opening
\title{MTH 343 Numerical Analysis Lecture 7:}
\author{Sheikh Abdul Raheem Ali}

\begin{document}

\maketitle

\section{Method of False Position}

Bisection/Secant Method that constructs approximating lines similar to those of secant method but always brackets the root in the manner of the bisection method.

\[ P_{n+1} = b_n - \frac{f(b_n)(b_n - a_n)}{f(b_n) - f(a_n)} \]

Example: Solve $ x^2 - 2 = 0 $ for $ x $. $ f(x) = x^2 - 2, a_1 = 1, b_1 = 2. $

\begin{tabular}{c c c c c}
	$ n $ & $ a_n $ & $ b_n $ & $ P_{n+1} $ & $ f(P_{n+1}) $ \\
	1&1&2&1.3333&-0.22222\\
	2&1.3333&2&1.4&-0.04\\
	3&1.4&2&1.4117&-0.00692\\
\end{tabular}

Although the method of False Position may appear superior to the Secant Method, it generally converges more slowly.

\section{Newton's Method}

One of the most widely used methods.

\section*{Technique}

\begin{enumerate}
	\item Start from a single initial point (estimate) $ x_0 $ that is close to the root, then move along the tangent line to its intersection with the x-axis \& take that point as the first approximation.
	\item The procedure is continued until either the successive x-values are sufficiently close or the values of the function is sufficiently near zero.
\end{enumerate}

\begin{align*}
	\frac{f(x_0) - 0}{x_0 - x_1} &= f'(x_0) \\
	\frac{f(x_0)}{f'(x_0)} &= x_0 - x_1 \\
	x_1 &= x_0 - \frac{f(x_0)}{f'(x_0)} \\
	x_{n+1} &= x_n - \frac{f(x_n)}{f'(x_n)}
\end{align*}

This method is of order two, which means that on average it converges to a solution at roughly two additional decimal points of precision per iteration.

It fails when it encounters a maximum or minimum (as $ f'(x_n) $ would be 0 and the tangent would be parallel to the x-axis), which means it's not as suitable for functions that oscillate.

\section*{Remarks}

\begin{enumerate}
	\item Widely used, it converges rapidly (quadratic convergence). The number of decimal places accuracy nearly doubles at each iteration.
	\item Method may converge to a root different from the expected one.
	\item Method may diverge if the starting value is not close enough to the root.
	\item If we reach the max or min of a curve we fly off to infinity.
\end{enumerate}

	\section*{Theorem}

	Newton's method is quadratically convergent.

	\section*{Proof}

	\[ f(x) = f(a) + f'(x)(x-x_0) + \frac{f''(x)(x-x_0)^2}{2!} + \frac{f^{(3)}(x)(x-x_0)^3}{3!} + ...  \]

	$ x = P $
	$ a = P_n $

	\begin{align*}
		f(P) = f(P_n) + f'(P_n)(P-P_n) + \frac{f''(\xi)}{2!}(P-P_n)^2 + ... \\
		0 = \frac{f(P_n)}{f'(P_n)} + P - P_n + \frac{f''(\xi)}{2f'(P_n)}(P-P_n)^2 + ...\\
		P - P_n + \frac{f(P_n)}{f'(P_n)} = - \frac{f''(\xi)}{2f'(P_n)}(P-P_n)^2 \\
		|P-P_{n+1}| = \frac{f'(\xi)}{2|f'(P_n)|}(P - P_n)^2 \\
		|P - P_{n+1}| \le \frac{M}{2f'(P_n)}|(P-P_n)^2
	\end{align*}

\end{document}


To begin the excursion into the calculation of uncertainties, we first shorten the notation in accordance with footnote \footref{fnote1}:
\begin{align*}
	P(E_{i}|C_{\mu}) &= P_{\mu i}
	&
	\phi(C_{\mu}) &= \phi_{\mu}
	&
	n(E_{j}) = n_{j}.
\end{align*}

As outlined in \cite{agostini} (section 4), the covariance matrix $V=V(\phi,\phi\textprime)$ from statistical contributions has two components:
$V^{\mathrm{Data}}$ from the counted measured effects distribution, and $V^{\mathrm{MC}}$ due to the limited MC statistics in $P_{\mu j}$.
This can be seen from considering the uncertainties from $n_{j}$ and $M_{\mu j}$ in eq. \ref{eq:fullUnf}.
Since $\phi=M\times n=M(P(E|C))\times n$, we can identify respectively the aforementioned error contributions as
\begin{align*}
% \begin{split}
  V^{Total} &= V^{Data}+V^{MC} \\
  &= \frac{\partial \phi}{\partial n} \, Cov({n,n\textprime}) \, \frac{\partial \phi\textprime}{\partial n}
  \\
  &
  +\frac{\partial \phi}{\partial P} \, Cov({P,P\textprime}) \, \frac{\partial \phi\textprime}{\partial P}.
% \end{split}
\end{align*}

% V_Data
\subsection{$V^{Data}$}
D\textquotesingle Agostini argues that since the data sample $n_{j}$ is a realization of a multinomial distribution,
then
\begin{equation} \label{eq:Vn_agos}
V^{Data} = M \, Cov(n,n\textprime) \, M
\end{equation}
where the $Cov(n,n\textprime)$ is the
covariance matrix of the measurements with respect to the estimated true number of events $\sum_{\mu}{\phi_{\mu}}=N_{true}$:

\begin{equation}
 \begin{split}
Cov(n_{k},n_{j}) =
  \begin{cases}
    n_{j}(1-\frac{n_{j}}{N_{true}})       & \quad \text{if } k = j \\
    -\frac{n_{j}n_{k}}{N_{true}}  & \quad \text{if } k \ne j \\
  \end{cases}.
 \end{split}
\end{equation}

However, Adye (\cite{adye} section 5) demonstrates that this error estimation is only valid for the first iteration, as subsequent $\phi^{i}$ are \textbf{not independent} of $n_{j}$.
Indeed, we should re-write eq. \ref{eq:Vn_agos} appropriately as
\begin{equation} \label{eq:Vn_matrix}
	V^{Data} =
	\frac{\partial \phi^{i+1}}{\partial n} \times
	Cov(n,n\textprime) \times
	\frac{\partial \phi^{i+1}\textprime}{\partial n} ,
\end{equation}
with
\begin{equation*}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial n_{j}}
	= M_{\mu j} + \frac{\phi^{i+1}_{\mu}}{\phi^{i}_{\mu}} \, \frac{\partial \phi^{i}_{\mu}}{\partial n_{j}}
	- \sum_{\sigma, k}{ \epsilon_{\sigma} \, \frac{n_{k}}{\phi^{i}_{\sigma}} \, M_{\mu k} \, M_{\sigma k} \, \frac{\partial \phi^{i}_{\sigma}}{\partial n_{j}}}
\end{equation*}
where again the superscripts $i$ and $i+1$ refer to the iteration number.
The full derivation of $\frac{\partial \phi^{i+1}}{\partial n}$ (eq. \ref{eq:dphidn}) is found in section \ref{ExpVn} below.
In some cases it is safe to use the Poisson form of $Cov(n,n\textprime)$:

\begin{equation}
 \begin{split}
Cov(n_{k},n_{j}) = n_{k} \, \delta_{kj} \,.
 \end{split}
\end{equation}

% V_MC
\subsection{$V^{MC}$}
The contribution from $V^{MC}$, while well outlined in \cite{agostini} and below, is quite a monster. If one simply implements the equation verbatim
into code, the expected time for calculating all elements $\sim \textnormal{(number of bins)}^{7}$. Thus, here we present the form of $V^{MC}$,
while in section \ref{ExpVm} we show the explicit expansion and further contraction of indices towards a more reasonable, practical calculation.

D'Agostini identifies $V^{MC}$ via $\frac{\partial}{\partial M}$ giving
\begin{equation} \label{eq:Vmc_dagos}
	V^{MC} =
	n\times Cov(M,M\textprime) \times n\textprime.
\end{equation}
Further expansion reveals
\begin{equation} \label{eq:CovMM}
	Cov(M_{\mu k},M_{\lambda j}) =
	\sum_{\{\sigma r\},\{\sigma s\}}
	\frac{\partial M_{\mu k}}{\partial P_{\sigma r}}
  \,
	\frac{\partial M_{\lambda j}}{\partial P_{\sigma s}}
  \,
	Cov \big( P_{\sigma r}, P_{\sigma s} \big),
\end{equation}

\begin{equation}
	\frac{\partial M_{\mu k}}{\partial P_{\sigma j}} =
	M_{\mu k}
  \,
	\Bigg[
	\frac{\delta_{\mu \sigma} \, \delta_{jk}}{P_{\sigma j}}
	-\frac{\delta_{\mu \sigma}}{\epsilon_{\sigma}}
	-\frac{\delta_{jk} \, M_{\sigma k} \, \epsilon_{\sigma}}{P_{\sigma k}}
	\Bigg],
\end{equation}

\begin{equation} \label{eq:CovPP}
Cov \big( P_{\sigma r}, P_{\sigma s} \big) =
  \begin{cases}
  \frac{1}{\tilde{n}_{\sigma}} \, P_{\sigma r} \, (1-P_{\sigma r})       & \quad \text{if } r = s \\
    -\frac{1}{\tilde{n}_{\sigma}} \, P_{\sigma r} \, P_{\sigma s}        & \quad \text{if } r \ne s \\
  \end{cases}.
\end{equation}
In the final expression, $\tilde{n}_{\mu}$ represents the number of simulated events which fell into the true cause bin $\mu$.
If our simulation is weighted, we identify $\tilde{n}$ with the effective number of events
$\tilde{n}_{\mu} = \frac{( \sum_{j} w_{\mu j} )^{2}}{\sum_{j} w_{\mu j}^{2}}$ for all $j$ events in bin $\mu$.

Once again, Adye (\cite{adye2}) shows this is a first order estimate, only valid for the first iteration.
Re-writing \ref{eq:Vmc_dagos} with $\frac{\partial}{\partial P}$,
\begin{equation} \label{eq:Vmc_matrix}
	V^{MC} =
	\frac{\partial \phi^{i+1}}{\partial P} \times
	Cov(P,P\textprime) \times
	\frac{\partial \phi^{i+1}\textprime}{\partial P},
\end{equation}
we identify $\frac{\partial \phi^{i+1}}{\partial P}$ as
\begin{equation*}
\begin{split}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial P_{\lambda k}}
	&= \frac{\delta_{\lambda \mu}}{\epsilon_{\mu}} \, \bigg(\frac { n_{k} \, \phi^{i}_{\mu} }{ f_{k}} - \phi^{i+1}_{\mu} \bigg)
	- \frac{n_{k} \phi^{i}_{\lambda} }{f_{k}} \, M_{\mu k} \\
	&
	+ \frac{\phi^{i+1}_{\mu}}{\phi^{i}_{\mu}} \, \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}
	- \sum_{\rho,j}{ n_{j} \, \frac{\epsilon_{\rho}}{\phi^{i}_{\rho}} \, M_{\rho j} \, M_{\mu j} \, \frac{\partial \phi^{i}_{\rho}}{\partial P_{\lambda k}} }
\end{split}
\end{equation*}
whose derivation (eq. \ref{eq:dphidP}) is found in section \ref{ExpVm} below.
Of course, D'Agostini's form of $Cov(P,P\textprime)$ remains valid for use with the new construction of the partials.
One may also use a Poisson covariance if justified appropriately:

\begin{equation} \label{eq:CovPPPoisson}
Cov \big( P_{\rho r}, P_{\lambda s} \big) =
   \sigma_{\rho r} \sigma_{\lambda s} \delta_{\rho \lambda} \delta_{rs},
\end{equation}
with $\sigma_{\rho r}$ being the error estimates on $P_{\rho r}$ estimated when filling $P$ with Monte Carlo.

% Updated Algorithm - Including Errors
\subsection{Updated Unfolding Algorithm}

The afore-outlined unfolding algorithm must be modified to include the propagation of systematic errors.
At each iteration we have $\phi^{i+1}$, so both $\frac{\partial \phi^{i+1}}{\partial n}$ and $\frac{\partial \phi^{i+1}}{\partial P}$
can be calculated. The results are propagated and saved until the full covariance matrix is required for error estimates on the final $\phi$.

\begin{algorithm}
  \caption{Unfolding Algorithm - Including Errors}
  \begin{algorithmic}
  	\State $\phi^{0}\gets$ Prior
	\State testStatistic$\gets$ Pass
  	\While{ ( testStatistic = Pass ) }
		\State $M\gets M(P(E|C),\phi^{i})$
		\State $\phi^{i+1}\gets M \times n$
		\State $\frac{\partial \phi^{i+1}}{\partial n}\gets$ eq. \ref{eq:dphidn}
		\State $\frac{\partial \phi^{i+1}}{\partial P}\gets$ eq. \ref{eq:dphidP}
		\State testStatistic$\gets$ TS$(\phi^{i},\phi^{i+1})$
	\EndWhile
	\State $V^{Total}\gets V^{Data}(\frac{\partial \phi^{i+1}}{\partial n})+V^{MC}(\frac{\partial \phi^{i+1}}{\partial P})$
	\State $\sigma^{2}_{\phi} \approx diag(V^{Total})$
  \end{algorithmic}
\end{algorithm}

% Expansions of components of full V
\subsection{Expansion of Components of $V$}\label{adye_full_derivation}

%Useful Formulae
\subsubsection{Some useful formulae}
Recalling the unfolding formulae from before,
\begin{align*}
	\phi^{i+1}_{\mu} &= \sum_{k}{M_{\mu k} \, n_{k}}
	&
	M_{\mu j} &= \frac{ P_{\mu j} \, \phi^{i}_{\mu} }{ \epsilon_{\mu} \, f_{j}},
\end{align*}
where the efficiency, $\epsilon$, and normalization, $f$, of $M$ are
\begin{align*} %\label{eq:useful_eq}
	\epsilon_{\mu} &= \sum_{j} {P_{\mu j}}
	&
	f_{j} &= \sum_{\mu}{P_{\mu j} \, \phi^{i}_{\mu}}.
\end{align*}
Of note is the presence of $\phi^{i}$, ie, the unfolded cause distribution from the previous iteration, or the
prior in the case $i=0$.

We will be taking derivatives of these objects with respect to $n_{k}$ and $P_{\lambda k}$, to wit,
\begin{align} \label{eq:useful_derivs}
	\frac{\partial P_{\mu j}}{\partial n_{k}} &= 0
	&
	\frac{\partial \epsilon_{\mu}}{\partial n_{k}} &= 0
	&
	\frac{\partial f_{j}}{\partial n_{k}} &= \sum_{\mu}{P_{\mu j} \, \frac{\partial \phi^{i}_{\mu}}{\partial n_{k}}} \\
  	\frac{\partial P_{\mu j}}{\partial P_{\lambda k}} &= \delta_{\mu \lambda} \, \delta_{jk}
	&
	\frac{\partial \epsilon_{\mu}}{\partial P_{\lambda k}} &= \delta_{\lambda \mu}
	&
	\frac{\partial f_{j}}{\partial P_{\lambda k}} &= \delta_{jk} \, \phi^{i}_{\lambda} + \sum_{\mu} {P_{\mu j} \, \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}}.
\end{align}

The explicit forms of $\frac{\partial \phi^{i}_{\mu}}{\partial n_{k}}$ and $\frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}$ will be shown below, but only for $i=0$ do
\begin{equation} \label{eq:first_deriv}
	\frac{\partial \phi^{i}_{\mu}}{\partial n_{k}} = 0 \quad , \quad \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}} = 0,
\end{equation}
as no unfolding has been performed. This will clearly not be the case for subsequent iterations when $\phi^{i}$ becomes dependent on $n_{k}$ and $P_{\lambda k}$.

%Expansion of V_Data
\subsubsection{Expansion of $V^{Data}$} \label{ExpVn}
Making the appropriate substitutions, the index representation of eq. \ref{eq:Vn_matrix} is
\begin{align*}
	V(\phi^{i+1}_{\mu},\phi^{i+1}_{\nu})^{Data} =
	\sum_{jk}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial n_{j}} \,
	Cov(n_{j},n_{k}) \,
	\frac{\partial \phi^{i+1}_{\nu}}{\partial n_{k}},
\end{align*}
with
\begin{align*}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial n_{j}} &= \frac{\partial}{\partial n_{j}} \sum_{k}{M_{\mu k} \, n_{k}} \\
	&= \sum_{k} \bigl ( {M_{\mu k} \, \frac{\partial n_{k}}{\partial n_{j}} + n_{k}  \, \frac{\partial M_{\mu k}}{\partial n_{j}}} \bigr) \\
	&= \sum_{k} \bigl ( {M_{\mu k} \, \delta_{jk} + n_{k}  \, \frac{\partial M_{\mu k}}{\partial n_{j}}} \bigr) \\
	&= M_{\mu j} + \sum_{k}{n_{k}  \, \underbrace{\frac{\partial M_{\mu k}}{\partial n_{j}}}}
\end{align*}
%
\begin{align*}
	\frac{\partial M_{\mu k}}{\partial n_{j}} &= \frac{\partial}{\partial n_{j}} \frac{P_{\mu k} \phi^{i}_{\mu} }{ \epsilon_{\mu} f_{k} } \\
	%
	&= \underbrace{ \frac{P_{\mu k}}{\epsilon_{\mu} f_{k}} }_{\frac{M_{\mu k}}{\phi^{i}_{\mu}}} \frac{\partial \phi^{i}_{\mu}}{\partial n_{j}}
	- \underbrace{ \frac{P_{\mu k} \phi^{i}_{\mu}}{\epsilon_{\mu} f_{k}} }_{M_{\mu k}}
	\frac{1}{f_{k}}\sum_{\sigma}{P_{\sigma k} \frac{\partial \phi^{i}_{\sigma}}{\partial n_{j}} } \\
	%
	&= \frac{M_{\mu k}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial n_{j}} - M_{\mu k}
	\sum_{\sigma}{ \epsilon_{\sigma} \underbrace{ \frac{ P_{\sigma k} }{  \epsilon_{\sigma} f_{k} } }_{\frac{M_{\sigma k}}{\phi^{i}_{\sigma}}}
	\frac{\partial \phi^{i}_{\sigma}}{\partial n_{j}}} \\
	%
	&= \frac{M_{\mu k}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial n_{j}} - \sum_{\sigma}{ \frac{\epsilon_{\sigma}}{\phi^{i}_{\rho}} M_{\mu k} M_{\sigma k} \frac{\partial \phi^{i}_{\sigma}}{\partial n_{j}}}
\end{align*}
%
\begin{align*}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial n_{j}}
	&= M_{\mu j} + \frac{1}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial n_{j}} \underbrace{\sum_{k} {M_{\mu k} n_{k}}}_{\phi^{i+1}_{\mu}}
	- \sum_{\sigma, k}{ \epsilon_{\sigma} \frac{n_{k}}{\phi^{i}_{\rho}} M_{\mu k} M_{\sigma k} \frac{\partial \phi^{i}_{\sigma}}{\partial n_{j}}} \\
\end{align*}
%
\begin{equation} \label{eq:dphidn}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial n_{j}}
	= M_{\mu j} + \frac{\phi^{i+1}_{\mu}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial n_{j}}
	- \sum_{\sigma, k}{ \epsilon_{\sigma} \frac{n_{k}}{\phi^{i}_{\sigma}} M_{\mu k} M_{\sigma k} \frac{\partial \phi^{i}_{\sigma}}{\partial n_{j}}}
\end{equation}

Recalling eq. \ref{eq:first_deriv}, $\frac{\partial \phi^{0}_{\mu}}{\partial n_{j}} = 0$ for the first iteration, eliminating the last two terms of eq. \ref{eq:dphidn} and recovering $\frac{\partial \phi^{1}_{\mu}}{\partial n_{j}} = M_{\mu j}$ as per \cite{agostini}. In practice, one need only calculate $\frac{\partial \phi^{i+1}_{\mu}}{\partial n_{j}}$ for each iteration, saving the result until the full calculation of $V^{Data}$ is required.

%Expansion of V_MC
\subsubsection{Expansion of $V^{MC}$} \label{ExpVm}

Similar to $V(\phi^{i+1}_{\mu},\phi^{i+1}_{\nu})^{Data}$, we identify the contributions to $V$ from the Monte Carlo:

\begin{align*}
	V(\phi^{i+1}_{\mu},\phi^{i+1}_{\nu})^{MC} =
	\sum_{\lambda j} \sum_{\rho k}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial P_{\lambda j}}
	Cov(P_{\lambda j},P_{\rho k})
	\frac{\partial \phi^{i+1}_{\nu}}{\partial P_{\rho k}}.
\end{align*}

Proceeding forward,

\begin{align*}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial P_{\lambda k}}
	&= \frac{\partial}{\partial P_{\lambda k}} \sum_{j}{M_{\mu j} n_{j}}
	= \sum_{j} { n_{j} \underbrace{\frac{\partial M_{\mu j}}{\partial P_{\lambda k}} } }
\end{align*}
\begin{align*}
	\frac{\partial M_{\mu j}}{\partial P_{\lambda k}}
	&= \frac{\partial}{\partial P_{\lambda k}} \frac{ P_{\mu j} \phi^{i}_{\mu} }{ \epsilon_{\mu} f_{j}} \\
	&= \frac{\phi^{i}_{\mu} }{ \epsilon_{\mu} f_{j}} \frac{\partial P_{\mu j}}{\partial P_{\lambda k}}
	+ \underbrace{ \frac{P_{\mu j}}{\epsilon_{\mu} f_{j}} }_{\frac{M_{\mu j}}{\phi^{i}_{\mu}} } \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}
	- \frac{1}{\epsilon_{\mu} f_{j}} \underbrace{\frac{P_{\mu j} \phi^{i}_{\mu}}{\epsilon_{\mu} f_{j}}}_{M_{\mu j}}
	\bigg( f_{j} \frac{\partial \epsilon_{\mu}}{\partial P_{\lambda k}} + \epsilon_{\mu} \frac{\partial f_{j}}{\partial P_{\lambda k}}\bigg)
	\\
	&= \frac {\phi^{i}_{\mu} }{\epsilon_{\mu} f_{j}} \delta _{\lambda \mu} \delta_{jk}
	+ \frac{M_{\mu j}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}
	- \frac{1}{\epsilon_{\mu} f_{j}} M_{\mu j} \bigg( f_{j} \delta_{\lambda \mu}
	+ \epsilon_{\mu}\delta_{jk} \phi^{i}_{\lambda} + \epsilon_{\mu} \sum_{\rho} {P_{\rho j} \frac{\partial \phi^{i}_{\rho}}{\partial P_{\lambda k}}}\bigg)
	\\
	&= \frac {\phi^{i}_{\mu} }{\epsilon_{\mu} f_{j}} \delta _{\lambda \mu} \delta_{jk}
	+ \frac{M_{\mu j}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}
	- \frac{M_{\mu j}}{\epsilon_{\mu}} \delta_{\lambda \mu}
	- \frac{M_{\mu j} \phi^{i}_{\lambda}}{f_{j}} \delta_{jk}
	- \sum_{\rho}{\epsilon_{\rho} M_{\mu j} \underbrace{\frac{P_{\rho j}}{\epsilon_{\rho} f_{j}}}_{\frac{M_{\rho j}}{\phi^{i}_{\rho}}} \frac{\partial \phi^{i}_{\rho}}{\partial P_{\lambda k}}}
	\\
	&= \frac {\phi^{i}_{\mu} }{\epsilon_{\mu} f_{j}} \delta _{\lambda \mu} \delta_{jk}
	+ \frac{M_{\mu j}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}
	- \frac{M_{\mu j}}{\epsilon_{\mu}} \delta_{\lambda \mu}
	- \frac{M_{\mu j} \phi^{i}_{\lambda}}{f_{j}} \delta_{jk}
	- \sum_{\rho} M_{\rho j}M_{\mu j} \frac{\epsilon_{\rho}}{\phi^{i}_{\rho}} \frac{\partial \phi^{i}_{\rho}}{\partial P_{\lambda k}},
\end{align*}

and going back to $\frac{\partial \phi^{i+1}_{\mu}}{\partial P_{\lambda k}}$ to include the sum over $j$,

\begin{align*}
\begin{split}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial P_{\lambda k}} =
\end{split}
\end{align*}
%
\begin{align*}
\begin{split}
	\sum_{j} n_{j} \bigg[ \frac {\phi^{i}_{\mu} }{\epsilon_{\mu} f_{j}} \delta _{\lambda \mu} \delta_{jk}
	+ \frac{M_{\mu j}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}
	- \frac{M_{\mu j}}{\epsilon_{\mu}} \delta_{\lambda \mu}
	- \frac{M_{\mu j} \phi^{i}_{\lambda}}{f_{j}} \delta_{jk}
	- \sum_{\rho} M_{\rho j}M_{\mu j} \frac{\epsilon_{\rho}}{\phi^{i}_{\rho}} \frac{\partial \phi^{i}_{\rho}}{\partial P_{\lambda k}} \bigg]
	\\
	= \frac { n_{k} \phi^{i}_{\mu} }{\epsilon_{\mu} f_{k}} \delta _{\lambda \mu}
	+ \frac{1}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}} \underbrace{\sum_{j}{ M_{\mu j} n_{j} }}_{\phi^{i+1}_{\mu}}
	- \frac{\delta_{\lambda \mu}}{\epsilon_{\mu}} \underbrace{\sum_{j}{ M_{\mu j} n_{j}}}_{\phi^{i+1}_{\mu}}
	- \frac{n_{k} M_{\mu k} \phi^{i}_{\lambda}}{f_{k}}
	\\
	- \sum_{j}{ \sum_{\rho} n_{j} \frac{\epsilon_{\rho}}{\phi^{i}_{\rho}} M_{\rho j}M_{\mu j} \frac{\partial \phi^{i}_{\rho}}{\partial P_{\lambda k}} },
\end{split}
\end{align*}

with final form

\begin{equation} \label{eq:dphidP}
\begin{split}
	\frac{\partial \phi^{i+1}_{\mu}}{\partial P_{\lambda k}}
	&= \frac{\delta_{\lambda \mu}}{\epsilon_{\mu}} \bigg(\frac { n_{k} \phi^{i}_{\mu} }{ f_{k}} - \phi^{i+1}_{\mu} \bigg)
	- \frac{n_{k} \phi^{i}_{\lambda} }{f_{k}} M_{\mu k} \\
	&
	+ \frac{\phi^{i+1}_{\mu}}{\phi^{i}_{\mu}} \frac{\partial \phi^{i}_{\mu}}{\partial P_{\lambda k}}
	- \sum_{\rho,j}{ n_{j} \frac{\epsilon_{\rho}}{\phi^{i}_{\rho}} M_{\rho j}M_{\mu j} \frac{\partial \phi^{i}_{\rho}}{\partial P_{\lambda k}} }.
\end{split}
\end{equation}

Again for the first iteration $\frac{\partial \phi^{0}_{\mu}}{\partial P_{\lambda k}} = 0$, eliminating the last two terms of eq. \ref{eq:dphidP}, and recovering
D'Agostini's version. Again, upon implementation one need only calculate $\frac{\partial \phi^{i+1}_{\mu}}{\partial P_{\lambda k}}$ at each iteration,
saving it until $V^{MC}$ is needed for error estimation.

%To obtain a simplified representation of eq. \ref{eq:CovMM}, we first re-write \ref{eq:CovPP}:
%
%\begin{equation}
%Cov \Bigg[ P(E_{r}|C_{u}), P(E_{s}|C_{u}) \Bigg] =
%    \frac{\delta_{rs}}{n_{u}}P(E_{r}|C_{u})-\frac{1}{n_{u}}P(E_{r}|C_{u})P(E_{s}|C_{u}).
%\end{equation}
%
%Next, we consider the combination of the two partial derivative factors in \ref{eq:CovMM}. In the following derivation, $P(E_{r}|C_{u})=P_{ru}$.
%
%\begin{equation}
% \begin{split}
%    \frac{\partial M_{ki}}{\partial P_{ru}} \times \frac{\partial M_{lj}}{\partial P_{su}} \\
%    = M_{ki}M_{lj}
%    \Bigg[
%    	\frac{\delta_{ku}\delta_{ri}}{P_{ru}}
%	-\frac{\delta_{ku}}{\epsilon_{u}}
%        -\frac{\delta_{ri}M_{ui}\epsilon_{u}}{P_{iu}}
%    \Bigg]
%    \Bigg[
%    	\frac{\delta_{lu}\delta_{sj}}{P_{su}}
%	-\frac{\delta_{lu}}{\epsilon_{u}}
%        -\frac{\delta_{sj}M_{uj}\epsilon_{u}}{P_{ju}}
%    \Bigg]  \\
%    = M_{ki}M_{lj}
%    \Bigg[
%    \frac{\delta_{ku}\delta_{ri}\delta_{lu}\delta_{sj}}{P_{ru}P_{su}}
%    -\frac{\delta_{ku}\delta_{ri}\delta_{lu}}{P_{ru}\epsilon_{u}}
%    -\frac{\delta_{ku}\delta_{ri}\delta_{sj}M_{uj}}{P_{ru}P_{ju}}
%    -\frac{\delta_{ku}\delta_{lu}\delta_{sj}}{P_{su}\epsilon_{u}}
%    +\frac{\delta_{ku}\delta_{lu}}{\epsilon_{u}\epsilon_{u}} \\
%    +\frac{\delta_{ku}\delta_{sj}M_{uj}\epsilon_{u}}{\epsilon_{u}P_{ju}}
%    -\frac{\delta_{lu}\delta_{ri}\delta_{sj}M_{ui}}{P_{su}P_{iu}}
%    +\frac{\delta_{lu}\delta_{ri}M_{ui}\epsilon_{u}}{\epsilon_{u}P_{iu}}
%    +\frac{\delta_{ri}\delta_{sj}M_{ui}M_{uj}\epsilon_{u}\epsilon_{u}}{P_{iu}P_{ju}}
%    \Bigg]  \\
% \end{split}
%\end{equation}
%
%Finally, we arrive at the simplified expression
%\begin{equation} \label{eq:CovMM_full}
% \begin{split}
%    Cov(M_{ki},M_{lj})
%    = M_{ki}M_{lj}
%    \Bigg[
%    	\frac{M_{kj}}{n_{k}}
%    	+\frac{M_{li}}{n_{l}}
%	-\delta_{ij}
%	\Bigg(
%	\frac{M_{kj}\epsilon_{k}}{n_{k}P_{jk}}
%	+\frac{M_{kj}\epsilon_{k}}{n_{k}P_{jk}}
%	\Bigg)  \\
%        +\delta_{kl}
%	\Bigg(
%	\frac{\delta_{ij}}{n_{k}P_{ik}}
%	-\frac{1}{n_{k}\epsilon_{k}}
%	\Bigg)
%	+\sum_{u}^{n_{C}}
%	\frac{M_{ui}M_{uj}\epsilon_{u}\epsilon_{u}}{n_{u}}
%	\Big(
%	\frac{\delta_{ij}}{P_{ju}}-1
%	\Big)
%        \Bigg].
% \end{split}
%\end{equation}
%The final summation term can be computed prior to solving for all elements of $Cov(M_{ki},M_{lj})$, allowing for a speedier than expected set of loops.


% Created 2021-10-12 Tue 12:29
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble, euscript}
\DeclareMathOperator{\atantwo}{atan2}
\newcommand*{\ctrb}{\EuScript{C}}
\newcommand*{\obsv}{\EuScript{O}}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{\today}
\title{Controllability and observability}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={Controllability and observability},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.4.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Repetition: State space models}
\label{sec:orgfa57760}

\section{Mobile robot unicycle model}
\label{sec:org684bf07}

\begin{frame}[label={sec:org8194325}]{Example: Mobile robot}
\begin{columns}
\begin{column}{0.4\columnwidth}
\begin{center}
 \includegraphics[width=.3\linewidth]{../../figures/x80pro.jpg}
\end{center}
\begin{center}
 \includegraphics[width=1.0\linewidth]{../../figures/unicycle-model}
\end{center}
\end{column}

\begin{column}{0.6\columnwidth}
\pause

\[ \xi = \begin{bmatrix} \theta\\x\\y \end{bmatrix},   \quad u = \begin{bmatrix} \omega\\v \end{bmatrix}\]

\[\frac{d}{dt} \xi = \begin{bmatrix} \dot{\theta}\\\dot{x}\\\dot{y} \end{bmatrix} = \begin{bmatrix} \omega\\ v\cos\theta\\v\sin\theta\end{bmatrix} \]

Called uni-cycle model.
\pause

\alert{Activity} Can we reach any point in state space \(\begin{bmatrix} x &  y & \theta \end{bmatrix}^T\) by a suitably designed input signal sequence \(u(t)\)?
\end{column}
\end{columns}
\end{frame}
\begin{frame}[label={sec:orgbb3a664}]{Example: Mobile robot}
\small
\begin{columns}
\begin{column}{0.4\columnwidth}
\begin{center}
 \includegraphics[width=1.0\linewidth]{../../figures/unicycle-model-op}
\end{center}

\[\frac{d}{dt} \xi = \begin{bmatrix} \dot{\theta}\\\dot{x}\\\dot{y} \end{bmatrix} = \begin{bmatrix} \omega\\ v\cos\theta\\v\sin\theta\end{bmatrix} \]
\end{column}
\begin{column}{0.6\columnwidth}
Linearized model using deviation variables
\[ \xi(t) = \xi_0 + z(t), \quad \frac{d}{dt} \xi = \frac{d}{dt} z\]
\[ \frac{d}{dt} z = \begin{bmatrix} \omega\\ v\cos\theta_0 \\ v\sin\theta_0 \end{bmatrix}
= \underbrace{0}_{A} z + \underbrace{\begin{bmatrix} 1 & 0 \\ 0 & 0\\ 0 & 1\end{bmatrix}}_{B} \begin{bmatrix}\omega\\v\end{bmatrix} \]

\pause

\begin{align*} \mathcal{C} &= \begin{bmatrix} B & AB & A^2B \end{bmatrix}\\
&= \begin{bmatrix} 1 & 0 & 0 & 0 & 0 & 0 \\ 0 & 0 & 0 & 0 & 0 & 0\\ 0 & 1 & 0 & 0 & 0 & 0\end{bmatrix}
\end{align*}

\pause

\alert{Activity} Is the linearized model controllable? (Hint: What is \(\text{rank}\, \mathcal{C}\)?)
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:org554dd27}]{Example: Mobile robot}
\begin{columns}
\begin{column}{0.4\columnwidth}
Single bearing measurement.

\begin{center}
 \includegraphics[width=1.0\linewidth]{../../figures/unicycle-model-bearing}
\end{center}
\end{column}
\begin{column}{0.6\columnwidth}
\[ \xi = \begin{bmatrix} \theta\\x\\y \end{bmatrix}\]

\pause

\alert{Activity} Is the system observable with one bearing-only measurement?
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:org23594f7}]{Example: Mobile robot}
\begin{columns}
\begin{column}{0.4\columnwidth}
Bearing and distance measurement.

\begin{center}
 \includegraphics[width=1.0\linewidth]{../../figures/unicycle-model-bearing-distance}
\end{center}
\end{column}
\begin{column}{0.6\columnwidth}
\[ \xi = \begin{bmatrix} \theta\\x\\y \end{bmatrix}\]

\pause

\alert{Activity} Is the system observable with one bearing and one distance measurement?
\end{column}
\end{columns}
\end{frame}
\end{document}


\subsection{Ensembling Techniques}

Ensemble learning techniques combine multiple trained baseline models to make the final set of predictions.  In traditional machine learning, ensemble methods are mainly divided in three categories: bagging, boosting, and stacking \cite{ensembleML2012}.  With bagging, the final prediction is obtained as a weighted majority over predictions from each baseline model.  In boosting, each model specializes on certain subsets of examples.  Finally, stacking ensembles use the prediction of each baseline model to train a new model which outputs a combined prediction.

\subsection{Bagging Ensemble Of CNNs}
\label{ss:ensembling}

In our solution we opted for using bagging as the ensemble method, and grid search to explore the search space of model weights and other hyperparameter values. We considered ensembling all the models by removing their respective final fully connected layers and fine-tuning all networks with a shared fully connected layer (or otherwise a ``stacking'' ensemble, as described in Section~\ref{ss:ensembling}). However, limitations on GPU memory made this approach difficult. So we decided on a simpler bagging approach.

One technique we consider in our grid search takes into account the class-wise accuracy or class-wise confidence calculated on the training set for each model. The class-wise accuracy is given by $\frac{\textbf{true positives}}{\textbf{ \# class examples}}$. the class-wise confidence is given by $\frac{\textbf{true positives}}{\textbf{ true positives + false positives}}$. For each model, on each class we calculated it's accuracy and confidence for its top-1 and top-5 guesses. We the weigh the output of each network based on its accuracy or confidence in guessing examples on the validation set.


\chapter{Universal Approximation}\label{ch:approx}

Three categories of approximation theory.
\begin{enumerate}
\item
In Barron's paper, there is a section on the lower bound of
approximation using linear subspaces. If the basis is fixed, then the
rate $n^{-1/d}$ is not improvable. The DNN uses a basis adapt to the
function.

The adaptive FEM we have studied before is indeed using linear
subspaces. For a given and fixed basis, select the best n term to
approximate a function. The non-linear approximation theory (by
DeVore) is to relax the smoothness of function to achieve the optimal
rate $n^{-1/d}$ but won't improve the rate.

Now the problem is a truly nonlinear approximation problem, even the
basis can be changed according to $f$. The dimension independent rate
$n^{-1/2}$ seems also optimal. What we can improve is the
characterization of the smoothness.
\end{enumerate}

\input{6DL/nonpoly}

\section{Semi-constructive Method}
A interesting problem is that how big this special function set $\mathcal{F}$ is? In 1989, there are many papers using some different methods to prove a surprising results that with some simple assumption for active function $g_k$, for K=1, then $\mathcal{F}$ is uniformly dense on compact in $C^0(R^{m})$. To state and proof those result simpler, we have the next definition:
\begin{definition}\label{def:space}
For $\mathcal{F}$ with $K=1$, we note it by the next function set:
\begin{equation}
\mathcal{F} = \Sigma(g) := \{ \sum_{j=1}^{n_1} \beta_j g (\omega_{j}\cdot x + \theta_{j}) ~ | ~  n_1 \in \mathbb{N}^{+}, \beta_i, \theta_j, \in \mathbb{R}, \bm \omega_j \in \mathbb{R}^m \}.
\end{equation}
\end{definition}

\begin{definition}\label{def:algebra-space}
We define the smallest function algebra which contains $\Sigma(g)$ as a subset:
\begin{equation}
\Pi \Sigma(g) := \{ \sum_{j}^{n_1} \beta_j \cdot \prod_{k}^{l_j}G({\omega}_{j}^k \cdot {x} + \theta_j^k) ~|~ {\omega}_{j}^{k} \in \mathbb{R}^m, \beta_j \in \mathbb{R}, l_j \in \mathbb{N}, n_1 \in \mathbb{N}^{+} \},
\end{equation}
\end{definition}

\begin{definition}\label{def:udoc}
A subset $S \subset C^0(\mathbb{R}^m)$ is said to be {\bf uniformly dense on compact in $C^0(\mathbb{R}^m)$ }if for any compact subset $K \subset \mathbb{R}^m$, then $S$ is dense in $C^0(K)$ when restrict $S$ to $K$.
\end{definition}

\begin{definition}\label{def:squashing}
A function $g: \mathbb{R} \to [0,1]$ is {\bf squashing} if it is non-decreasing, $\lim_{x\to \infty}g(x) = 1$ and $\lim_{x\to -\infty} g(x) = 0$.
\end{definition}

\begin{definition}\label{def:cosine-squasher}
We define the next cosine squasher function as:
\begin{equation}
\phi(x) =
\begin{cases}
0 & -\infty < x \le -\frac{\pi}{2} \\
\frac{cos(x+\frac{3\pi}{2}) + 1}{2} \quad &-\frac{\pi}{2} < x < \frac{\pi}{2} \\
1 &\frac{\pi}{2} \le x < \infty.
\end{cases}
\end{equation}
\end{definition}

To prove the next theorem by Hornik et. 1989 we need the Stone-Weierstrass theorem:
\begin{theorem}[Stone-Weierstrass]
Let $\mathcal{A}$ be an algebra of real continuous functions on a compact set $\Omega$, which means that $\mathcal{A} \subset C^0(\Omega)$ and $af(x) + bg(x) \in \mathcal{A}$, $fg \in \mathcal{A}$ for any $a, b \in \mathbb{R}$ and $f, g \in \mathcal{A}$. If $\mathcal{A}$ separates points on $\Omega$ and if $\mathcal{A}$ vanishes at no point of $\Omega$, i.e:
\begin{itemize}
\item for every $x \neq y \in \Omega$, then exists $f \in \mathcal{A}$ s.t $f(x) \neq f(y)$,
\item for every $x \in \Omega$, then exists $f \in \mathcal{A}$ s.t $f(x) \neq 0$.
\end{itemize}
Then the uniform closure of $\mathcal{A}$ w.r.t $L^{\infty}$ norm is equal to $C^{0}(\Omega)$.
\end{theorem}

 And we introduce a important lemma which might be the most important results to construct the dense result for $\Sigma(g)$ in Hornik et, 1989.
 \begin{lemma}\label{lemm:squashing}
 Let $f$ be a continuous squashing function and $g$ an arbitrary squashing function. Then for any $\epsilon > 0$, there is an function $\hat{f} \in \Sigma(g)$ such that
 $$ \|f - \hat{f} \|_{C^0(\mathbb{R})} < \epsilon .$$
 \end{lemma}
 \begin{proof}
 This is an interesting exercise. Remark: here $m = 1$ for $\Sigma(g)$.
 \end{proof}

Here is the result:
\begin{theorem}[Hornik 1989]
For any squashing function $g$, $\Sigma(g)$ is uniformly dense on compact in $C^0(\mathbb{R}^m)$.
%, if we just take the output layer active function $g_{2}(x) = x$, then there exist $n_1$ and $W_2$ such that:
%\begin{equation}
%\|\bm{f}_2(W_2; \bm{x}) - f(\bm{x})\|_{C^{0}(I^m)} \le \epsilon
%\end{equation}
%where $\bm{f}_2(W_2; \bm{x})$ is defined as
%\begin{equation}
%\bm{f}_2(W_2; \bm{x}) = \sum_{l=1}^{n_1} \omega_{2,l1}\sigma(\sum_{i=1}^m \omega_{1,li}x_i + \theta_{1,l}).
%= \bm{W}_{2}^{T} \sigma(\bm{W}_{1}\bm{x} + \bm{\theta}_1).
%\end{equation}
\end{theorem}

\begin{proof}
Without loss of generality, we consider a compact set $K$. And choose any $f \in C^{0}(\mathbb{R}^m)$ and any $\epsilon > 0$. We use the next steps to prove the theory.
\begin{itemize}
\item We can use the Stone-Weierstrass theorem which shows that $\Sigma(cos) = \Pi \Sigma(cos)$ is dense in $C^0(K)$. Which means we have $f_1 \in \Sigma(cos)$ such that
$$
\|f - f_1\|_{C^{0}(K)} \le \frac{\epsilon}{2},
$$
with
$$f_1 = \sum_{i=1}^P \hat \beta_i cos(\hat\omega_i \cdot x + \hat\theta_i).$$
Now we take $M \in \mathbb{N}^{+}$ such that
$$
(\hat\omega_i \cdot x + \hat\theta_i) |_K \subset [-2\pi M, 2\pi M] \quad \forall 1\le i \le P.
$$
\item We construct a new function $\hat{\phi}$ by M and $\phi$ defined in \ref{def:cosine-squasher}:
\begin{align}
\hat{\phi}(x) = \sum_{k = -M}^{M} 2(\phi(-x + \frac{\phi}{2} - 2m\phi) + \phi(x - \frac{3\phi}{2} + 2m\phi) - 1) \\=
\begin{cases}
0 & -\infty < x \le -2\pi M \\
cos(x) - 1 \quad &-2\pi M < x < 2\pi (M+1) \\
0 &  2\pi(M+1)\le x < \infty.
\end{cases}
\end{align}
So we have
\begin{equation}
f_1  = \sum_{i=1}^P \hat\beta_i \hat{\phi}(\hat\omega_i \cdot x + \hat\theta_i) + (\sum_{i=1}^P \beta_i )\hat{\phi}(0\cdot x + \frac{\pi}{2}) = \sum_{i=1}^Q\beta_i \phi(\omega_i \cdot x +  \theta_i),
\end{equation}
when restricted into $K$.

\item Then we use the Lemma \ref{lemm:squashing}, we can approximate $\phi$ by using:
$$
\tilde \phi(x) = \sum_{i=1}^{L} \alpha_ig(a_ix + \gamma_i) \quad a_i, \gamma_i \in \mathbb{R},
$$
such that
$$
\|\phi - \tilde \phi \|_{C^{0}(\mathbb{R})} \le \frac{\epsilon}{ 2 \sum_{i=1}^{Q} |\beta_i|}.
$$
So we let
$$
f_2 = \sum_{i=1}^Q \beta_i \tilde \phi(\omega_i \cdot x + \theta_i) \in \Sigma(g),
$$
when restricted to $K$, we have
$$
\|f_1 - f_2\|_{C^0(K)} \le \sum_{i=1}^{Q} |\beta_i| \| \phi - \tilde \phi \|_{C^0(K)} \le \frac{\epsilon}{2}.
$$
\end{itemize}

This finish this proof.
\end{proof}

However, this approximation property cannot tell us anything about how to set the net structure to get better approximation.

\section{Functional Analysis Methods}
\paragraph{Hahn-Banach Theorem: analytic form}
Let $X$ be a normed space over field $K$ of real or complex numbers. Let $f:V\rightarrow K$ be a bounded linear functional defined on a subspace $V\subset X$. Then $f$ can be extended to a bounded linear functional $F:X \rightarrow K$ having the same norm: $norm(F):=\mathop{sup} \limits_{x\in X,norm(x)\le 1}|F(x)|=\mathop{sup} \limits_{x\in V,norm(x)\le 1}|f(x)|=:norm(f)$

\paragraph{Hahn-Banach Theorem: geometric form}

Let $X$ be a normed space over the reals, and let $A,B$ be nonempty, disjoint, convex subsets of $X$:

(i) If $A$ is open, then there exists a bounded linear functional $\phi: X\rightarrow R$ and a number $c\in R$ such that: $\phi(a)<c\le\phi(b)$ for all $a\in A, b\in B$

(ii) If $A$ is compact and $B$ is closed, then there exists a bounded linear functional $\phi: X\rightarrow R$ and numbers $c_1, \ c_2$ such that: $\phi(a)\le c_1<c_2\le \phi(b)$ for all $a\in A, b\in B$

\paragraph{Riesz-Markov Theorem (Representation theorem for the continuous dual of $C_0(X)$)}

Let $X$ be locally compact Hausdorff space. For any continuous linear functional $\phi$ on $C_0(X)$, there is a unique regular countably additive complex Borel measure $\mu$ on $X$ such that $\phi(f)=\int_Xf(x)d\mu(x)$ for all $f\in C_0(X)$.

\paragraph{Some definitions}

Let $I_n$ denote the n-dimensional unit cube, $[0,1]^n$. The space of continuous functions on $I_n$ is denoted by $C(I_n)$ and we use $norm(f)$ to denote the supremum (or uniform) norm of an $f\in C(I_n)$. In general we use $norm(\cdot)$ to denote the maximum of a function on its domain. The space of finite, signed regular Borel measures on $I_n$ is denoted by $M(I_n)$.

Def: We say that $\sigma$ is discriminatory if for a measure $\mu\in M(I_n)$
$$\int_{I_n}\sigma(y^Tx+\theta)d\mu(x)=0$$
for all $y\in R^n$ and $\theta\in R$ implies that $\mu=0$.

Def: We say that $\sigma$ is sigmoidal if
$\sigma(t)\rightarrow \Big\{
\begin{aligned}
1  \ \ \ as \ \ \ t\rightarrow + \infty \\
0  \ \ \ as \ \ \ t\rightarrow - \infty
\end{aligned}$

\paragraph{Theorem 1}

Let $\sigma$ be any continuous discriminatory function. Then finite sums of the form:
$$G(x)=\mathop{\Sigma}\limits_{j=1}^N\alpha_j\sigma(y_j^Tx+\theta_j)$$
are dense in $C(I_n)$. In other words, given any $f\in C(I_n)$ and $\epsilon>0$, there is a sum, $G(x)$, of the above form, for which $|G(x)-f(x)|<\epsilon$ for all $x\in I_n$.
~\\~\\proof: Let $S\subset C(I_n)$ be the set of functions of the form $G(x)$ as in the statement of the theorem 1. Clearly $S$ is a linear subspace of $C(I_n)$. We claim that the closure of $S$ is all of $C(I_n)$.

Assume that the closure of $S$, say $R(=\bar{S})$, is a closed proper subspace of $C(I_n)$ [Notice: the closure of a linear space is also a linear space: $r_1\in R,r_2\in R, r_1=\mathop{lim}\limits_{n\rightarrow \infty}S_n^1, r_2=\mathop{lim}\limits_{n\rightarrow \infty}S_n^2 (S_n^1\in S, S_n^2\in S)$. So, $ar_1+br_2=\mathop{lim}\limits_{n\rightarrow \infty}aS_n^1+bS_n^2$. Because $S$ is a linear space, so $aS_n^1+bS_n^2\in S$. Then $ar_1+br_2\in R$. So $R$ is a linear space.]

Claim: there is a bounded linear functional on $C(I_n)$, call it $L$, with the property that $L\neq 0$ but $L(R)=L(S)=0$.

proof of the claim: Because $R\subsetneqq C(I_n)$, so there exists an $x_0\in C(I_n) \setminus R$. We denote $\tilde{R}=span(R,x_0)$. For $\forall \tilde{r}\in \tilde{R}, \exists! \ r\in R, t\in \mathbb{R}$ s.t. $\tilde{r}=r+tx_0$\ (if $\tilde{r}=r+tx_0=r'+t'x_0$, then $r-r'=(t-t')x_0$. So $(t-t')x_0\in R$. Then we get $t=t', r=r'$)
~\\ We define a funcitonal on $\tilde{R}$, say $\tilde{L}$.
$$\tilde{L}: \tilde{R} \rightarrow \mathbb{R}$$
$$\tilde{r}=r+tx_0\mapsto t$$
That is $\tilde{L}(\tilde{r})=\tilde{L}(r+tx_0)=t$.
~\\It's easy to show $\tilde{L}$ is linear. Now we prove that $\tilde{L}$ is bounded.
~\\$norm(\tilde{L})=\mathop{sup}\limits_{\tilde{r}\neq0}\frac{|L(\tilde{r})|}{norm(\tilde{r})}=\mathop{sup}\limits_{r+tx_0\neq0}\frac{|t|}{norm(r+tx_0)}=\mathop{sup}\limits_{r/t+x_0\neq0}\frac{|t|}{|t|\cdot norm(r/t+x_0)}=\mathop{sup}\limits_{r/t+x_0\neq0}\frac{1}{norm(r/t+x_0)}=\frac{1}{\mathop{inf}\limits_{r/t\in R}norm(r/t+x_0)}=\frac{1}{\mathop{inf}\limits_{v\in R}norm(x_0-v)}$
~\\Because $R,x_0$ are nonempty, disjoint, and convex subsets of $C(I_n)$, and $R$ is closed, $x_0$ is compact.
~\\By Hahn-Banach theorem(geometric form), there exists a bounded linear functional $\phi: C(I_n)\rightarrow \mathbb{R}$ and $c_1, c_2\in \mathbb{R}$ s.t. $\phi(x_0)\le c_1<c_2\le \phi(r)\ \ \forall r\in R$
~\\So $\phi(r)-\phi(x_0)=\phi(r-x_0)\ge c_2-c_1>0$, then we have $c_2-c_1\le\phi(r-x_0)\le norm(\phi)\cdot norm(r-x_0)\Rightarrow norm(r-x_0)\ge\frac{c_2-c_1}{norm(\phi)}>0\ \ \forall r\in R$.
~\\Then, $\mathop{inf}\limits_{v\in R}norm(x_0-v)\ge\frac{c_2-c_1}{norm(\phi)}>0$, so $norm(\tilde{L})<\infty$
~\\Now, we've proved that $\tilde{L}$ is a bounded linear functional on $\tilde{R}$, and $\tilde{L}\neq 0$, and $\tilde{L}(r)=\tilde{L}(r+0\cdot x_0)=0\ \ \forall r\in R$. So $\tilde{L}(R)=\tilde{L}(S)=0$.
~\\if $\tilde{R}=C(I_n)$, then we define $L:=\tilde{L}$, then we're done.
~\\if $\tilde{R}\subsetneqq C(I_n)$. By Hahn-Banach Theorem(analytic form), $\tilde{L}$ can be extended to a bounded linear functional $L: C(I_n)\rightarrow \mathbb{R}$ and $L|_{\tilde{R}}=\tilde{L}$. So $L\neq0$, and $L(R)=\tilde{L}(R)=0$. Then we have $L(R)=L(S)=0$ and $L\neq0$.

So we have proved the claim that there is a bounded linear functional on $C(I_n)$, call it $L$, with the property that $L\neq 0$ but $L(R)=L(S)=0$.

~\\Because $I_n$ is locally compact (actually, it is compact) and Hausdorff space. So $C_0(I_n)=C(I_n)$, and $L$ is a continuous linear functional on $C(I_n)$. By Riesz-Markov Theorem, there is a unique regular countably additive signed Borel measure $\mu$ on $I_n$ such that $L(h)=\int_{I_n}h(x)d\mu(x)$ for all $h\in C(I_n)$
~\\In particular, $\sigma(y^Tx+\theta)\in R$ (by definition), so $\int_{I_n}\sigma(y^Tx+\theta)d\mu(x)=L(\sigma(y^Tx+\theta))=0$ for all $y$ and $\theta$

~\\However, we assumed $\sigma$ wa discriminatory so that $\mu=0$. But $L\neq0$. So $\exists f\in C(I_n)$ such that $L(f)=\int_{I_n}fd\mu\neq0$, so $\mu\neq0$. We get the contradiction.
~\\Hence, $R=\bar{S}=C(I_n)$, that is $S$ is dense in $C(I_n)$.
~\\ Q.E.D.

~\\~\\Now, we specialize this result to show that any continuous sigmoidal $\sigma$ of the form discussed before, namely
$\sigma(t)\rightarrow \Big\{
\begin{aligned}
1  \ \ \ as \ \ \ t\rightarrow + \infty \\
0  \ \ \ as \ \ \ t\rightarrow - \infty
\end{aligned}$
is discriminatory.
\paragraph{Lebesgue Bounded Convergence Theorem}

Let ${f_n}$ be a sequence of measurable functions on a set of finite measure $E$. Suppose ${f_n}$ is uniformly pointwise bounded on $E$, that is, there is a number $M\ge0$ for which $|f_n|\le M$ on $E$ for all $n$.

If $f_n\rightarrow f$ pointwise on $E$, then $\mathop{lim}\limits_{n\rightarrow\infty}\int_Ef_n=\int_Ef$.
\paragraph{Lemma1}

Any bounded, measurable sigmoidal function, $\sigma$, is discriminatory. In particular, any continuous sigmoidal function is discriminatory.
~\\~\\proof:To prove this, we assume $\int_{I_n}\sigma(y^Tx+\theta)d\mu(x)=0$ for $\forall y,\theta$, we want to prove $\mu=0$. To demonstrate this, note that for any $x,y,\theta,\phi$, we have
$$\sigma(\lambda(y^Tx+\theta)+\phi) \Big\{
\begin{aligned}
\rightarrow1 \ \ \ for \ \ \ y^Tx+\theta>0 \ \ \ as \ \ \ \lambda\rightarrow + \infty \\
\rightarrow0 \ \ \ for \ \ \ y^Tx+\theta<0  \ \ \ as \ \ \ \lambda\rightarrow + \infty \\
=\sigma(\phi) \ \ \ for \ \ \ y^Tx+\theta=0 \ \ \ for \ \ \ all \ \ \lambda
\end{aligned}$$
Thus, the functions $\sigma_\lambda(x)=\sigma(\lambda(y^Tx+\theta)+\phi)$ converge pointwise to the function
$$\gamma(x) =\Big\{
\begin{aligned}
1 \ \ \ for \ \ \ y^Tx+\theta>0 \\
0 \ \ \ for \ \ \ y^Tx+\theta<0 \\
\sigma(\phi) \ \ \ for \ \ \ y^Tx+\theta=0
\end{aligned}$$
as $\lambda\rightarrow + \infty$
~\\Furthermore, $\sigma_\lambda(x)=\sigma(\lambda(y^Tx+\theta)+\phi)$ is uniformly bounded. (Because we've assumed that $\sigma$ is bounded).
~\\Let $\Pi_{y,\theta}$ be the hyperplane defined by $\{x|y^Tx+\theta=0, x\in I_n\}$ and let $H_{y,\theta}$ be the open half-space defined by $\{x|y^Tx+\theta>0, x\in I_n\}$. By the Lebesgue Bounded Convergence Theorem, we have:
$$\mathop{lim}\limits_{\lambda\rightarrow+ \infty}\int_{I_n}\sigma_{\lambda}(x)d\mu(x)=\int_{I_n}\gamma(x)d\mu(x)$$
Because $\int_{I_n}\sigma_{\lambda}(x)d\mu(x)=0$ for $\forall \lambda$ by assumption, so $\int_{I_n}\gamma(x)d\mu(x)=\mathop{lim}\limits_{\lambda\rightarrow+ \infty}\int_{I_n}\sigma_{\lambda}(x)d\mu(x)=0$
~\\Then $0=\int_{I_n}\gamma(x)d\mu(x)=\int_{\Pi_{y,\theta}}\gamma(x)d\mu(x)+\int_{H_{y,\theta}}\gamma(x)d\mu(x)+\int_{I_n\setminus (\Pi_{y,\theta}\cup H_{y,\theta})}\gamma(x)d\mu(x)$
~\\Because $$\gamma(x) =\Big\{
\begin{aligned}
1 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ x\in H_{y,\theta} \\
0 \ \ \ x\in I_n\setminus H_{y,\theta}\cup \Pi_{y,\theta} \\
\sigma(\phi) \ \ \ \ \ \ \ \ \ \ \ \ \ \ x\in \Pi_{y,\theta}
\end{aligned}$$
So, $0=\int_{\Pi_{y,\theta}}\gamma(x)d\mu(x)+\int_{H_{y,\theta}}\gamma(x)d\mu(x)=\sigma(\phi)\cdot\mu(\Pi_{y,\theta})+\mu(H_{y,\theta})$
~\\We now show that the measure of all half-planes being 0 implies that the measure $\mu$ itself must be 0.
~\\Fix y, for a bounded measurable function, h, define the linear functional, F, according to :
$$F(h)=\int_{I_n}h(y^Tx)d\mu(x)$$
$$|F(h)|\le\int_{I_n}|h(y^Tx)|d|\mu|(x)\le norm(h)_{L^{\infty}(\mathbb{R})}\cdot|\mu|(I_n)$$
Because $\mu$ is finite signed measure, so $|\mu|(I_n)<\infty$.
Then $norm(F)=\mathop{sup}\limits_{h\neq0}\frac{|F(h)|}{norm(h}_{L^{\infty}(\mathbb{R})})\le|\mu|(I_n)<\infty$, so $F$ is a bounded functional on $L^{\infty}(\mathbb{R})$.
~\\Let $h$ be the indicator function of the interval $[\theta, \infty)$ (That is, $h(u)=1$ if $u\ge\theta$ and $h(u)=0$ if $u<\theta$). So that
$$F(h)=\int_{I_n}h(y^Tx)d\mu(x)=\int_{\Pi_{y,-\theta}}h(y^Tx)d\mu(x)+\int_{H_{y,-\theta}}h(y^Tx)d\mu(x)+\int_{I_n\setminus (\Pi_{y,-\theta}\cup H_{y,-\theta})}h(y^Tx)d\mu(x)$$
(Recall that $\Pi_{y,\theta}=\{x|y^Tx+\theta=0, x\in I_n\}$ and $H_{y,\theta}=\{x|y^Tx+\theta>0, x\in I_n\}$)
~\\So $F(h)=\mu(\Pi_{y,-\theta})+\mu(H_{y,-\theta})$
~\\Because we've shown that $\sigma(\phi)\mu(\Pi_{y,\theta})+\mu(H_{y,\theta})=0$ for all $\phi, \theta, y$ and $\sigma$ is sigmoidal function. So we take $\phi\rightarrow +\infty$, then $\sigma(\phi)\rightarrow1$. So $\mu(\Pi_{y,\theta})+\mu(H_{y,\theta})=0$ for all $y, \theta$. So
$$F(h)=\mu(\Pi_{y,-\theta})+\mu(H_{y,-\theta})=0$$
If $h$ is the indicator function of the open interval $(\theta, \infty)$.
$$F(h)=\int_{I_n}h(y^Tx)d\mu(x)=\int_{\Pi_{y,-\theta}}h(y^Tx)d\mu(x)+\int_{H_{y,-\theta}}h(y^Tx)d\mu(x)+\int_{I_n\setminus (\Pi_{y,-\theta}\cup H_{y,-\theta})}h(y^Tx)d\mu(x)=\mu(H_{y,-\theta})$$
Similarly, because we have $\sigma(\phi)\mu(\Pi_{y,\theta})+\mu(H_{y,\theta})=0$ for all $\phi, \theta, y$ and $\sigma$ is sigmoidal function. We take $\phi\rightarrow-\infty$, so $\sigma(\phi)\rightarrow0$. Then $\mu(H_{y,\theta})=0$ for all $y,\theta$. So we have
$$F(h)=\mu(H_{y,-\theta})=0$$
By linearity, $F(h)=0$ for the indicator function of any open intervals. ($F(\chi_{(a,b)})=F(\chi_{(a,\infty)}-\chi_{[b,\infty)})=F(\chi_{(a,\infty)})-F(\chi_{[b,\infty)})=0-0=0$)
~\\So it is true for any step functions($\mathop{\Sigma}\limits_{n=1}^Na_n\chi_{c_n}, \ \{c_n\}$ are disjoint open intervals).
~\\Because $sin(z)$ and $cos(z)$ can be uniformly approximated by step functions on a closed bounded interval. So we have $\int_{I_n}sin(y^Tx)d\mu(x)=0$ and $\int_{I_n}cos(y^Tx)d\mu(x)=0$ for any $y$. Then we have:
$$\int_{I_n}cos(y^Tx)+i\cdot sin(y^Tx)d\mu(x)=\int_{I_n}exp(iy^Tx)d\mu(x)=0$$
for all $y$. Thus, the Fourier transform of $\mu$ is 0 and so $\mu$ must be zero as well (trigonometric polynomials are dense in $C(I_n)$(stone-weierstrass), we can approximate a simple function by step functions and approximate a step function by continuous functions. So $\mu(E)=0$ for any measurable set $E$). Hence, $\sigma$ is discriminatory.
~\\Q.E.D.

~\\~\\Now, we apply the previous results to the case of most interest in neural network theory. A straightforward combination of Theorem 1 and Lemma 1 shows that networks with one internal layer and an arbitrary continuous sigmoidal function can approximate continuous functions with arbitrary precision providing that no constraints are placed on the number of nodes or the size of the weights. This is Theorem 2 below.

\paragraph{Theorem 2}

Let $\sigma$ be any continuous sigmoidal function. Then finite sums of the form
$$G(x)=\mathop{\Sigma}\limits_{j=1}^N\alpha_j\sigma(y_j^Tx+\theta_j)$$
are dense in $C(I_n)$. In other words, given any $f\in C(I_n)$ and $\epsilon>0$, there is a sum, $G(x)$, of the above form, for which
$$|G(x)-f(x)|<\epsilon$$
for all $x\in I_n$

~\\proof: Combine Theorem 1 and Lemma 1.
~\\Q.E.D.


\section{Results}

\begin {itemize}
% After the previous selection of genes we worked with 191 genes
% decided to choose 5 clusters
\item Spline regression:
 The 2 first clusters included the genes with an L-shape

\begin{center}
\includegraphics[width=0.3\columnwidth]{./images/grafic_two_patterns.pdf}
\end{center}

\item Conditional Mutual Information

% No previous selection of the genes was needed
%\begin{block}{Three criteria}
We filtered for L-shapes using a combination of three criteria:
\begin{itemize}
\item the ratio $r<0.25$
\item unconditioned MI $\mathit{cMI}(0)>0.1$
\item the median expression on the left side of the optimal threshold $t^{\ast}$ is higher
than the median expression on the right side.
\end{itemize}
%\end{block}
% The parameters are chosen according to a random permutation test (see Liu(2012)).
% According to the above criteria, a total of 641 genes are selected to be L-shape genes.

\item Comparison between the methods:
\begin{center}
\begin{tabular}{|c|c|c|}
\hline
Initial selection & 191 & 641 \\
\hline
\hline
Cluster & Splines & cMI \\
\hline
1 & 140 & 102 \\
2 & 22 & 16 \\
\hline
Total & 162 & 118 \\
\hline
\end{tabular}
\end{center}

\item In summary...
  \begin{itemize}
  \item We have found similar results between both methods.
  \item Biological interpretation is in progress but preliminary (unpublished) results are consistent with the hypothesis.
  \item Sample size is a limiting factor: $cMI$ works better with hundreds of samples but one may have a very small number (real cases: 30, 12)
\end{itemize}

%\begin{center}
%\includegraphics[width=0.6\columnwidth]{./images/grafic_two_genes.pdf}
%\end{center}

\end{itemize}


\documentclass[12pt]{article}

\input{../preamble}

\makeatletter
%http://groups.google.com/group/comp.text.tex/msg/7e812e5d6e67fcc5
\def\convertto#1#2{\strip@pt\dimexpr #2*65536/\number\dimexpr 1#1}
\makeatother

\begin{document}

\section{Misc Notes}
See \url{https://www.overleaf.com/learn/latex/Lengths_in_LaTeX} for more

1in = 72.27pt = 72bp

\pagebreak

\section{Conversion Table}
The following table gives decimal approximations of the scaling factors between various units used by LaTeX.

\gap

\begin{center}\begin{tabular}
  {>{\def\colunit{pt}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{mm}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{cm}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{ex}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{em}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{bp}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{dd}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{pc}}l<{\convertto{\rowunit}{1\colunit}}
   >{\def\colunit{in}}l<{\convertto{\rowunit}{1\colunit}}
   >{\bfseries}l}
\multicolumn{1}{l}{\bfseries 1pt} & \multicolumn{1}{l}{\bfseries 1mm} & \multicolumn{1}{l}{\bfseries 1cm} & \multicolumn{1}{l}{\bfseries 1ex} & \multicolumn{1}{l}{\bfseries 1em} & \multicolumn{1}{l}{\bfseries 1bp} & \multicolumn{1}{l}{\bfseries 1dd} & \multicolumn{1}{l}{\bfseries 1pc} & \multicolumn{1}{l}{\bfseries 1in} & \\
\gdef\rowunit{pt} & & & & & & & & & \rowunit\\
\gdef\rowunit{mm} & & & & & & & & & \rowunit\\
\gdef\rowunit{cm} & & & & & & & & & \rowunit\\
\gdef\rowunit{ex} & & & & & & & & & \rowunit\\
\gdef\rowunit{em} & & & & & & & & & \rowunit\\
\gdef\rowunit{bp} & & & & & & & & & \rowunit\\
\gdef\rowunit{dd} & & & & & & & & & \rowunit\\
\gdef\rowunit{pc} & & & & & & & & & \rowunit\\
\gdef\rowunit{in} & & & & & & & & & \rowunit\\
\end{tabular}\end{center}

\vspace{0.2in}
Table taken from: https://tex.stackexchange.com/questions/8260/

\end{document}


% Copyright 2021 Google LLC
%
% Use of this source code is governed by an MIT-style
% license that can be found in the LICENSE file or at
% https://opensource.org/licenses/MIT.

%!BIB program = biber
%!TeX program = lualatex
%!TeX spellcheck = en-US

\documentclass[hctr2.tex]{subfiles}
\begin{document}
\section{Security of HCTR2}\label{security}
Following the approach described in \cite{concrete},
we prove that if the underlying block cipher is secure,
then HCTR2 has good security properties.
The security bound proven appears in \autoref{securitybound}.
\subsection{Definitions}\label{definitions}
We use \(x \sample S\) to mean ``\(x\) is sampled from \(S\)
uniformly at random'', and we write
\(A^{\mathcal{O}, \mathcal{O}'}\Rightarrow 1\) to refer
to the event ``adversary \(A\),
given access to oracles \(\mathcal{O}\) and \(\mathcal{O}'\),
returns 1''.

Let $\Perm(n)$ denote the set of all permutations on \(\bin^n\).
Per \cite{concrete}, for a block cipher
\(E: \mathcal{K} \times \bin^n \rightarrow \bin^n\)
the distinguishing advantage of an adversary \(A\) is:
%
\begin{align*}
    \advantage{\pm \mathrm{prp}}{E}[(A)] \defeq
    {}&\left\lvert\probsub{k \sample \mathcal{K}}{A^{E_k,E_k^{-1}}\Rightarrow 1}\right.
    \\
    {}&\left. - \probsub{\pi \sample \Perm(n)}{A^{\pi,\pi^{-1}}\Rightarrow 1}\right\rvert
    \\
    \intertext{Define}
    \advantage{\pm \mathrm{prp}}{E}[(q, t)] \defeq
    {}&\max_{A \in \mathcal{A}(q, t)} \advantage{\pm \mathrm{prp}}{E}[(A)]
\end{align*}
where $\mathcal{A}(q, t)$
is the set of all adversaries that make at most $q$ queries and take at most $t$ time.

Let $\Perm^\mathcal{T}(\mathcal{M})$
denote the set of all
tweakable length-preserving permutations
$\bm{\pi} : \mathcal{T} \times \mathcal{M} \rightarrow \mathcal{M}$
such that for all $T, M \in \mathcal{T} \times \mathcal{M}$,
$\abs{\bm{\pi}(T, M)} = \abs{M}$, and
for all $T \in \mathcal{T}$, $\bm{\pi}_{T}$ is a permutation on \(\mathcal{M}\).
In an abuse of notation
we use $\bm{\pi}^{-1}$ to refer to the function
such that $\bm{\pi}^{-1}(T, \bm{\pi}(T, M)) = M$ ie $(\bm{\pi}^{-1})_T = (\bm{\pi}_T)^{-1}$.

Per~\cite{cmc}, for a tweakable super-pseudorandom permutation
$\bm{E} : \mathcal{K} \times \mathcal{T} \times \mathcal{M} \rightarrow \mathcal{M}$
the distinguishing advantage of an adversary $A$ is:
%
\begin{align*}
    \advantage{\pm \widetilde{\mathrm{prp}}}{\bm{E}}[(A)] \defeq
    {}&\left\lvert\probsub{k \sample \mathcal{K}}{A^{\bm{E}_k,\bm{E}_k^{-1}}\Rightarrow 1}\right.
    \\
    {}&\left. - \probsub{\bm{\pi} \sample \Perm^\mathcal{T}(\mathcal{M})}
        {A^{\bm{\pi},\bm{\pi}^{-1}}\Rightarrow 1}\right\rvert
    \\
    \intertext{Define}
    \advantage{\pm \widetilde{\mathrm{prp}}}{\bm{E}}[(q, \sigma, t)]
    \defeq {}&
    \max_{A \in \mathcal{A}(q, \sigma, t)} \advantage{\pm \widetilde{\mathrm{prp}}}{\bm{E}}[(A)]
\end{align*}
where \(\mathcal{A}(q, \sigma, t)\)
is the set of all adversaries
that make at most \(q\) queries
and take at most \(t\) time,
such that the total number of blocks sent in all queries is
at most \(\sigma\) ie
\begin{displaymath}
    \sum_s \ceil{\abs{T^s}/n} + \ceil{\abs{P^s}/n} \leq \sigma
\end{displaymath}
where \(\abs{T^s}, \abs{P^s}\) are the length of the tweak and the message presented in query \(s\).

We use \(\HCTR[\pi]\) to refer to HCTR2 in which invocation
of the block cipher is replaced with invocation of the
permutation \(\pi \in \Perm(n)\).
\(\XCTR_\pi\) refers to a similar substitution.
\(\HCTR[E]\) refers to HCTR2 using the block cipher \(E\),
ie \(\HCTR[E_k]\) for \(k \sample \mathcal{K}\), while
\(\HCTR[\Perm(n)]\) refers to \(\HCTR[\pi]\)
for \(\pi \sample \Perm(n)\).

\subsection{Hash function}\label{hproperties}
Define \(\poly(M)\) to refer to the formal polynomial
\(\poly(M_0 \Concat \cdots \Concat M_{l-1})
\defeq  M_0\hpoly^{l-1} \xor \cdots \xor M_{l-1}\).
While for example \(\hpoly + 2\) and \(2\hpoly + 1\) can be equal
in value if \(\hpoly = 1\),
they are not equal as formal polynomials;
two formal polynomials are only equal
if every coefficient is equal. Thus \(\poly(M) = \poly(M')\)
only if \(M = 0^{ln} \Concat M'\) for some \(l\) or vice versa.

Define \(H(T, M)\) as the formal polynomial in \(\hpoly\) given
by that tweak and message.
\(H_{\hgen}(T, M)\) is then evaluation of this polynomial at
\(\hpoly = x^{-n}\hgen\); POLYVAL evaluates at this point for
performance reasons.
\begin{align*}
    & H(T, M) \\
    \defeq &
    \begin{cases}
        \poly(\fromint(2\abs{T} + 2) \Concat \pad(T) \Concat M \Concat 0^n) &
        \text{if } n \text{ divides } \abs{M} \\
        \poly(\fromint(2\abs{T} + 3) \Concat \pad(T) \Concat \pad(M \Concat 1) \Concat 0^n) &
        \text{otherwise}
    \end{cases}
\end{align*}

We depend on the following properties of this map
onto formal polynomials:
\begin{itemize}
    \item The map is injective
    \item The polynomial is never \(0\) or \(x^n\hpoly\)
    \item The constant term is always zero
    \item The polynomial is of degree at most
    \begin{displaymath}
        d(T, M) \defeq 1 + \ceil{\abs{T}/n} + \ceil{\abs{M}/n}
    \end{displaymath}
\end{itemize}

For the first property, see \autoref{injective}.
For the second, observe that \(H(T, M)\)
can be of degree 1 only if \(\abs{T} = \abs{M} = 0\),
in which case the polynomial is \(x\hpoly\).
Since \(x^{n-1} \neq 1\) we have that \(x^n \neq x\).

For any nonzero polynomial \(p(\hpoly)\)
in \(\GF(2^n)\), there are at most \(\deg(p)\) values \(\hpoly\)
such that \(p(\hpoly) = 0\), and therefore
\(\probsub{\hpoly\sample{\bin^n}}{p(\hpoly) = 0} \leq \deg(p)/2^n\).
Since multiplication by a nonzero field element
is a bijection of the field onto itself, it follows that
\(\probsub{\hgen\sample{\bin^n}}{p(x^{-n}\hgen) = 0} \leq \deg(p)/2^n\).
From this we infer three properties of \(H_{\hgen}(T, M)\):
\begin{description}
    \item[Property 1]
    For any \(T, M\) and any \(g \in \bin^n\),
    \begin{displaymath}
        \probsub{\hgen\sample{\bin^n}}{H_{\hgen}(T, M) = g} \leq d(T, M)/2^n
    \end{displaymath}
    Proof: since \(H(T, M)\) is nonzero and has
    a zero constant term,
    the polynomial \(H(T, M) \xor g\)
    is nonzero and has the same degree, at most \(d(T, M)\).
    \item[Property 2]
    For any \((T_1, M_1) \neq (T_2, M_2)\) and any \(g \in \bin^n\)
    \begin{align*}
        & \probsub{\hgen\sample{\bin^n}}{H_{\hgen}(T_1, M_1) \xor H_{\hgen}(T_2, M_2) = g} \\
        \leq  & \max(d(T_1, M_1), d(T_2, M_2))/2^n
    \end{align*}
    Proof: \(H\) is injective onto polynomials
    and the constant term is zero, therefore
    \(H(T_1, M_1) \xor H(T_2, M_2) \xor g\)
    is not the zero polynomial and
    has degree at most \(\max(d(T_1, M_1), d(T_2, M_2))\).
    This is the almost-XOR-universal property.
    \item[Property 3]
    For any \(T, M\) and any \(g \in \bin^n\)
    \begin{displaymath}
        \probsub{\hgen\sample{\bin^n}}{H_{\hgen}(T, M) \xor \hgen = g} \leq d(T, M)/2^n
    \end{displaymath}
    Proof: \(H(T, M)\) has a zero constant term and
    cannot be equal to the polynomial \(x^n\hpoly\).
    \(H(T, M) \xor g \xor \hgen = H(T, M) \xor g \xor x^n\hpoly\)
    thus cannot be the zero polynomial and has
    degree at most \(d(T, M)\).
\end{description}

\subsection{H-coefficient technique}\label{hco}
The H-coefficient technique was introduced by Patarin in 1991~\cite{ppdes,hco}.
We highly recommend the exposition
of~\cite{hco2} Section 3,
``The H-coefficient Technique in a Nutshell'';
we here present a simpler exposition that
does not cover the technique in its full
generality but only our use of it.
Our use of the symbol \(\mathcal{T}\) and the term
``compatible'' differ from~\cite{hco2}.

We wish to bound the adversary's ability to distinguish between
two ``worlds'', world X (the ``real world'') and world Y (the ``ideal world'').
Each world is a probability distribution over
deterministic oracles the adversary interacts with.

We consider only deterministic adversaries.
A randomized adversary can be considered as a random draw
from a population of deterministic adversaries, so
a bound on the advantage achievable by a deterministic
adversary bounds the whole population and therefore
the advantage of the randomized adversary. In what follows
we consider the adversary \(A\) fixed; only the world, and the
particular oracles drawn from that world, vary.

When the adversary interacts with the oracle,
a transcript \(\tau\) of queries and responses is created.
\(\Tc\) is the set of ``compatible transcripts'':
if \(\tau \in \Tc\) then for the fixed adversary,
there is some oracle
that results in its creation. For example,
since the adversary is deterministic, the first query
will always be the same; a transcript that
does not start with this query is not a compatible transcript.
For a given \(\tau \in \Tc\),
a deterministic adversary must always
return the same answer; call this answer \(A(\tau)\).

Let random variables \(X\) and \(Y\)
represent the distribution of transcripts
in world X and world Y respectively, so that
each transcript \(\tau\) has a probability \(\prob{X = \tau}\)
of arising in world X, and similarly \(\prob{Y = \tau}\) in world Y.
The adversary's distinguishing advantage is then
\(\left|\prob{A(Y) = 1} - \prob{A(X) = 1}\right|\).
Without loss of generality,
we assume that \(\prob{A(Y) = 1} \geq \prob{A(X) = 1}\).
We further assume that \(A(\tau)\) is optimal:
\(A(\tau) = 1\)
when \(\prob{Y = \tau} > \prob{X = \tau}\) and 0 otherwise.

In \autoref{mainlemma}, we partition \(\Tc\) into \(\Tg\) and \(\Tb\),
and prove that:
\begin{itemize}
    \item \(\prob{Y = \tau} \leq \prob{X = \tau}\) for all \(\tau \in \Tg\)
    \item \(\prob{Y \in \Tb} \leq \epsilon\)
\end{itemize}
It follows that \(A(\tau) = 0\) for all \(\tau \in \Tg\),
and therefore that \(\prob{A(Y) = 1} \leq \epsilon\),
from which we bound the distinguishing advantage:
\(\prob{A(Y) = 1} - \prob{A(X) = 1} \leq \epsilon\).

With this technique, only the first proof
need consider the probability
distribution of world X at all,
and this proof need only consider good transcripts.
The bulk of the work, proving
\(\prob{Y \in \Tb} \leq \epsilon\),
involves only world Y, which is far simpler
to reason about.

\subfile{mainlemma.tex}
\subsection{Security bound}\label{securitybound}
By a standard substitution argument~\cite{cbcsec,concrete} we have that
\begin{displaymath}
    \advantage{\HCTR[\Perm(n)]}{\HCTR[E]}[(q, \sigma, t)]
    \leq \advantage{\pm \mathrm{prp}}{E}[(\sigma + 2, t + \sigma t')]
\end{displaymath}
where \(t'\) is a small constant
representing the per-block cost of simulating HCTR2, and
\(\sigma + 2\) bounds the number of block cipher calls made by the simulator.

Halevi and Rogaway's PRP-RND lemma
\cite[Appendix C, Lemma 6]{cmc} tells us that
\begin{displaymath}
    \advantage{\pm \widetilde{\mathrm{prp}}}{\pm\widetilde{\mathrm{rnd}}}[(q, \sigma, t)]
    \leq \left.\binom{q}{2}\middle/2^n\right.
    \leq q^2/2^{n+1}
\end{displaymath}
Putting these together with our main lemma, we conclude
\begin{align*}
    &\advantage{\pm \widetilde{\mathrm{prp}}}{\HCTR[E]}[(q, \sigma, t)] \\
    \leq & \quad \advantage{\pm \mathrm{prp}}{E}[(\sigma + 2, t + \sigma t')] \\
    &+ \left.\left(3\sigma^2 + 2q\sigma + q^2 + 7\sigma + 2\right)\middle/2^{n+1}\right.
\end{align*}
\end{document}


\documentclass[letterpaper,12pt]{article}
\usepackage{amssymb,latexsym,amsmath}
\usepackage{wasysym}
\usepackage{graphicx}
\usepackage{hyperref}

\begin{document}

\begin{flushleft}
    MulitLayer Perceptrons\\
    AI/ML Guild Lab \#2b\\
    \today \\
\end{flushleft}

\section{Multilayer Perceptrons}

It's really a misnomer to call Multilayer Perceptrons \emph{multi-layer}. It's
not a perceptron with multiple layers but instead it is a network of multiple
``neuron-like'' processing units but not every layer in the network is a
perceptron.

Recall that the perceptron equation:

\[
    Z = \sum_{i=0}^{N} W_i x_i + b
\]

What would happen if you applied multiple layers of this equation?

\section{History}

Dudes named Rumelhart, Hinton, and Williams introduced/discovered this algorithm
in the mid-‘80s. Rumelhart was a mathematical pychologist working at UC San
Diego. He led a research group on neural networks, picking up where Rosenblatt
left off after the Minksy book pretty much destroyed public funding for this
type of research.

Rumelhart wanted to train a neural network with multiple
layers and sigmoidal units instead of threshold units (as in the perceptron)
or linear units but they did not how to do that. Rumelhart came up with an idea
called ``Back Propagation'' but it was Hinton that used it in conjunction with
the training model Rumelhart developed for sigmoidal model training.

This is what is typically referred to as a multilayered perceptron today.

\includegraphics[scale=0.5]{mlp-network}

\section{The Algorithm}

Formally, a multilayered perceptron is one where these conditions are present:

\begin{itemize}
    \item a linear function that aggregates the input values
    \item a sigmoid function (the activation function)
    \item an output function (i.e.\ a threshold function for classification)
    \item a loss function that computes the overall error of the network
    \item a learning procedure to adjust the weights of the network, i.e., the
        ``back propagation'' algorithm
\end{itemize}

\section{The Math}

\subsection{Linear Function}
The linear function is very similar to the single layer perceptron. It is a
weight sum of the inputs plus a bias. The difference is that the weights are
matrix and no longer a vector.
\[
    z_m = \sum_{i=0}^{N} W_{m,n} X_m + b
\]

\subsection{Sigmoid Function}
Every element of the $Z_m$ becomes an input to the sigmoid function.

\[
    a_m = \sigma(z_m) = \frac{1}{1 + e^{-z_m}}
\]

The output of the sigmoid function is a vector of size $m$ where every element
is a unit in the hidden layer.

$a$ stands for activation! Which is typically how the output of the hidden layer
is described. Using the sigmoid funciton is somewhat arbrary, you can use other
non-linear functions here like $\tanh$ or $ReLU$.

\subsection{Output Function}

Similar to single layer but adjusted for sigmoid.

\[
    f(a) =
    \begin{cases}
        1, & a \ge 0.5\\
        0, & \text{else}
    \end{cases}
\]

\subsection{Cost function}

We need a measure of how well the network is performing. We did this with the
single layer too. This is used in the training step. This often goes by
different names in the literature such \emph{objective function}, \emph{loss
function}, or \emph{error function}.

Typically this cost function is tuned for your problem at hand. The one that
Rumelhart came up with was

\[
    E = \frac{1}{2} \sum_{k} {(a_k - y_k)}^2
\]

This is commonly known as the sum of squared errors function. It's pretty basic
but often gets the job done.

\subsection{Training: Forward \& Back Propagation}

Multilayer Perceptron training is composed of two steps. Forward propagation
where information flows forward in the network to compute predictions and the
error. And then the back propagation stage where the error derivatives is
calculated and used to update the weight matrix.

The forward propagation stage is just chaining the linear, sigmoid and output
functions (just like the single layer!).

The back propagation part is more complicated but it's basically asking ``how
does the error change if we tweak the weights''? This question has additional
questions that need answers first. How does the error change when we change
the activation $a_m$ a bit?  How does the activation change if we fiddle with
linear function a bit?

It takes a fair amount of calculus to derive all of this. I'll leave it as an
excercise to the reader. The outcome of that derivation can be solved for using
the gradient descent algorithm.

\end{document}


\chapter{Geometry kitchen sink}
\label{chp:geometry}

\section{Combining synthetic and analytic geometry? Do we still need this distinction?}

Synthetic geometry uses axioms; it does not use formulas.%
\footnote{\url{https://en.wikipedia.org/wiki/Synthetic_geometry}}

Analytic geometry uses formulas from mathematical analysis.

Analytic geometry is a marriage between analysis and geometry.

When asked "What is a line in a 2-dimensional Euclidean space?",
a student of
synthetic geometry
may answer "It is the infinite extension (in both direction)
of the shortest geometric object connecting two points,"
while a student of
analytic geometry
may answer
``It is $\{ (x,y) ~|~ ax+by=c \text{ where } a,b,c\in\Real \}$.''
% https://en.wikipedia.org/wiki/Synthetic_geometry
% https://en.wikipedia.org/wiki/Analytic_geometry
% http://en.wikipedia.org/wiki/Foundations_of_geometry
% http://en.wikipedia.org/wiki/Timeline_of_geometry
% https://en.wikipedia.org/wiki/Algebraic_geometry
Algebraic geometry studies the solution of equations.

Topology studies topological spaces.
Analytic geometry uses coordinates to define spaces,
while topology uses neighborhood to define spaces.

\section{Understanding the advantages of synthetic geometry?}

Why do we begin with synthetic geometry?
Why don't we just go straight to analytic geometry?

\section{Rambling, too abstract}

What makes geometry interesting is not the spaces,
but the mapping between spaces?

(Why are we telling the reader this?)

There are many kinds of spaces we will talk about:

* algebraic structures (especially fields),
* coordinate space (a space of tuples),
* real coordinate space (a space of tuples of real numbers),
* vector space (a coordinate space with vector operations),
* inner product space (a space with an inner product),
* covector space (a space whose member is a covector, defined below).
* tensor space.

There are many ways to make a mapping between two spaces,
as we will see:

* physical field (mapping between two spaces, usually from a coordinate space),
* coordinate system (surjective mapping between two spaces),
* natural vectorization of a ring (mapping from a ring to one of its natural vector spaces),
* basis (linear coordinate system for a vector space),
* covector (linear mapping from a vector space to its ring space),
* cobasis (basis for a covector space).

Then, we will eat our own tail by stating that all those mappings are members of their respective function spaces
(a function space is a space whose members are functions):

* coordinate system transformation (a mapping between two mappings),

\section{Measuring distance using metrics}

With a metric, we can define
a (generalized) circle with radius $r$ and center $c$
to be the set of points
$\{x ~|~ d~c~x = r \}$.

An astronomer may need to tell another astronomer where the latter can find a celestial body.
He can give its coordinates in a coordinate system they have agreed on.
The astronomer can use a celestial coordinate system.
% https://en.wikipedia.org/wiki/Celestial_coordinate_system

\subsection{Measuring distance with Euclidean metrics}

\index{definitions!Euclidean space}%
The two-dimensional Euclidean space
may be imagined as an unbounded sheet of flat paper.

The \emph{Euclidean metric} for \(\Real^n\) is \( d(x,y) = \sqrt{\sum_{k=1}^n (y_k-x_k)^2} \).
It is the length of the shortest straight line segment from \(x\) to \(y\) in Euclidean geometry.
The two-dimensional Euclidean metric is the Pythagorean theorem.

Do not confuse Euclidean metric \(d\) and derivative operator \(d\).

\section{Not yet read}

\enquote{Two Approaches to Modelling the Universe: Synthetic Differential Geometry and Frame-Valued Sets},
John L. Bell\footnote{\url{http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.114.1930&rep=rep1&type=pdf}}

\enquote{The power of analytic geometry derives very largely from the fact
that it permits the methods of the calculus, and, more generally, of
mathematical analysis, to be introduced into geometry.} (p.~1)

Foundations of geometry\footnote{\url{https://en.wikipedia.org/wiki/Foundations_of_geometry}}

Hilbert's axioms\footnote{\url{https://en.wikipedia.org/wiki/Hilbert\%27s_axioms}}

Birkhoff's axioms\footnote{\url{https://en.wikipedia.org/wiki/Birkhoff\%27s_axioms}}

\section{Mapping \(\Real^n\) to itself? Why?}

We have shown that we can use a real $n$-tuple as a coordinate to the real $n$-space.
In other words, we can map the real $n$-space to itself.
The identity coordinate system for the real $n$-space is
$I : \mathbb{R}^n \to \mathbb{R}^n$ where $I(x) = x$;
this $I$ is also the standard basis for the real $n$-space.

We also call a vector space a linear space.

There is another way to think of a coordinate in $\mathbb{R}^n$:
as a function $N \to \mathbb{R}$ where $N = \{1,2,3,\ldots,n\}$.
We can think of an $n$-dimensional $F$-vector as a function $N \to F$ where $N = \{1,2,3,\ldots,n\}$.
We can forget about dimensions and think of a $F$-vector as a function $\mathbb{N} \to F$.

We can also use vector spaces to talk conveniently about geometric objects.
With vector spaces, we can define a line by vector collinearity,
and define a plane by its normal vector.

Let there be two bases $J$ and $K$.
Let $T$ be coordinate transformation from $J$ to $K$
(that is $J~x = K~(T~x)$)
and $U$ be basis transformation from $J$ to $K$
(that is $K = U~J$).
\[
J~x = (U~J)~(T~x)
\]
\[
J = U \circ J \circ T
\]

Interesting things happen when we change the basis.
$T : E \to F$.

\section{Projective spaces (Why are we talking about this?)}

To grasp projective space, see real projective space.

Descartes's Cartesian coordinate system.

Klein's erlangen program.

Using algebra, we can describe a circle whose radius is $r$
and whose center is the origin in a 2-dimensional Euclidean space with Cartesian coordinate system
as $\{C_2~(x,y) ~|~ x^2+y^2 = r^2\}$.
We can find the intersection of two geometric objects
by solving the system of their equations.

% Synthetic differential geometry
% http://home.sandiego.edu/~shulman/papers/sdg-pizza-seminar.pdf
% Synthetic Differential Geometry: An application to Einstein’s Equivalence Principle
% http://www.math.ru.nl/~landsman/scriptieTim.pdf
% see the references in
% https://ncatlab.org/nlab/show/synthetic+differential+geometry
% https://mathoverflow.net/questions/186851/synthetic-vs-classical-differential-geometry

\section{Avoiding the tensor notation in Ricci calculus and the Einstein summation convention?}

% https://en.wikipedia.org/wiki/Ricci_calculus
We try not use the tensor notation in Ricci calculus,
of which the Einstein notation is a part.

Every text should be more readable than writable.

\section{Using equations to constrain a system, reducing its degree of freedom? Why are we talking about this?}

% https://en.m.wikipedia.org/wiki/Constraint_(classical_mechanics)

A \emph{constraint} is?
A constraint is modeled by an equation.
A system is modeled by a set of equations.
The system satisfies all equations in that set.

Example:
The position of an object moving in straight line is modeled by \(x(t) = k t\).

Constraint force is the cause of the constraint.

\subsection{Understanding a system's degree of freedom}

Bead sliding in a straight wire at an angle to gravitational field line.

Motivate degree of freedom with analytic geometry.
To describe a circle, we need two parameters c and r.

The degree of freedom of a system is the minimum number of parameters required to describe it.
Each parameter has type \(\Real\).

% https://en.wikipedia.org/wiki/Classical_central-force_problem
% https://en.wikipedia.org/wiki/Central_force

\section{Bilinear function? Why are we talking about this?}

A bilinear function \(f\) is left-linear and right-linear:
\(
f(x+z,y) = f(x,y) + f(z,y)
\)
and
\(
f(x,y+z) = f(x,y) + f(x,z)
\).

We can generalize that to multilinearity, but we won't.

* A linear function is multilinear.
* Otherwise a function is multilinear iff every partial application of it is also multilinear.

\section{Measuring the distance between two points on a sphere}

The distance from \(P\) to \(Q\) is \(\angle PCQ \cdot r\)


\section{Densely Connected Convolutional Networks - DenseNet}

This section will elaborate on the implemented DenseNet. Article \citep{DENSE} has been used as the basis for the implementation of the DenseNet.

The DenseNet in this project consist of 12 layers where 5 layers are dense blocks, and 7 layers are convolutional layers. At the end of the DenseNet a flatten layer and a softmax layer are applied. Adam optimizer is used.

The creation of the 7 convolutional layers and 5 dense block's can be seen in listing \ref{lst:denseconvo}.

\begin{lstlisting}[language=Python, label=lst:denseconvo, caption=for loop whicifarh creates the dense block's mixed with the convolutional layers]
layer1 = slim.conv2d(input_layer,64,[3,3],
	normalizer_fn=slim.batch_norm,scope='conv_'+str(0))
for i in range(5):
	layer1 = denseBlock(layer1,i,units_between_stride)
	layer1 = slim.conv2d(layer1,64,[3,3],stride=[2,2],
		normalizer_fn=slim.batch_norm,scope='conv_s_'+str(i))

top = slim.conv2d(layer1,10,[3,3],
	normalizer_fn=slim.batch_norm,activation_fn=None,scope='conv_top')
\end{lstlisting}

The convolutional layers has a filter size of 3 by 3, and a stride with 2 by 2.

The dense block's consist of 6 convolutional layers which are connected in a direct connection to all subsequent layers and have a filter size of 3 by 3. The code for creating the dense block's can be seen in listing \ref{lst:densenetblock}. The last convolutional layer is the top layer which is a fully connected layer.

\begin{lstlisting}[language=Python, label=lst:densenetblock, caption=DenseNet Block function]
def denseBlock(input_layer,i,j):
	with tf.variable_scope("dense_unit"+str(i)):
	nodes = []
	a = slim.conv2d(input_layer,64,[3,3],normalizer_fn=slim.batch_norm)
	nodes.append(a)
	for z in range(j):
		b = slim.conv2d(tf.concat(nodes,3),64,[3,3],
				normalizer_fn=slim.batch_norm)
		nodes.append(b)
	return b
\end{lstlisting}

Listing \ref{lst:softmax} shows the last layer which is the softmax layer. This layer is appended to \emph{top} which contains all the previous layers.

\begin{lstlisting}[language=Python, label=lst:softmax, caption=Softmax layer appended to the end of the DenseNet]
output = slim.layers.softmax(slim.layers.flatten(top))
\end{lstlisting}

When all the layers are prepared the DenseNet is ready to be used with TensorFlow, and the training can commence.


\documentclass[a4paper]{article}
\usepackage{amsmath}
\numberwithin{equation}{section}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{paracol}
\usepackage[margin=15mm]{geometry}
\usepackage{url}

\newcommand{\x}{\mathbf{x}}
\newcommand{\z}{\mathbf{z}}
\newcommand{\vv}{\mathbf{v}}
\newcommand{\cov}{\boldsymbol{\Sigma}}
\newcommand{\bcov}{\bar{\cov}}
\newcommand{\C}{\mathbf{C}}
\newcommand{\I}[1]{\mathbf{I}_{#1}}
\newcommand{\Zero}[1]{\boldsymbol{0}_{#1}}
\newcommand{\Real}{\mathbb{R}}
\newcommand{\K}{\mathbf{K}}
\newcommand{\R}{\mathbf{R}}
\newcommand{\transpose}{{\intercal}}
\newcommand{\inverse}{{-1}}
\title{Notes on Bry et al. \cite{bry2015ijrr} Equations}
\date{\vspace{-5ex}}
\author{Marco Camurri}
\begin{document}
\maketitle
\begin{paracol}{2}
\appendix
 \section{Bry et al. 2015 \cite{bry2015ijrr}}
\begin{align}
  \x_t &= [ \x_t^m \; \x_t^p ] \label{eq:1}\\
  \z_t &= h(\x_t^m, \vv_t) \label{eq:2}\\
  \bar{\cov}_t &= \begin{bmatrix}
                  \bcov^{m^2} & \bcov^{mp} \\
                  \bcov^{pm} & \bcov^{p^2}
                 \end{bmatrix} \label{eq:3}\\
 \C &= \begin{bmatrix} \I{k} & \Zero{n-k} \end{bmatrix} \in \Real^{k\times n}
\label{eq:4}\\
 \K^m &= \bcov^m_t(\C^m)^\transpose(\C^m\bcov_t(\C^m)^\transpose+\R)^\inverse
\label{eq:5}\\
 \mu_t^m &= \bar{\mu}^m_t + \K^m(\z_t - \C^m \bar{\mu}_t^m) \label{eq:6}\\
 \cov^m_t &= (\I{n} - \K^m \C^m)\bcov^m_t \label{eq:7}\\
  \cov^m_t &= \bcov^m_t - \bcov^m_t(\C^m)^\transpose(\C^m
\bcov_t(\C^m)^\transpose + \R_t)^\inverse \bcov^m_t \label{eq:8}\\
\R_t &= (({\bcov^{m}_t})^\inverse \cov^m_t ({\bcov^m_t})^\inverse)^\inverse -
\bcov^m_t \nonumber\\
&= ((\cov^m)^{\inverse} - ({\bcov^m_t})^\inverse)^\inverse \label{eq:9}\\
\z_t &= (\K^{m})^\inverse(\mu^m_t - \bar{\mu}^m_t) + \bar{\mu}^m_t \label{eq:10}
\end{align}
\switchcolumn
\appendix
\setcounter{section}{1}
\section{Code implementation \cite{pronto}}
\begin{align}
  \x_t &= [ \x_t^m \in \Real^{k} \; \x_t^p\in\Real^{n-k} ]  \label{eqb:1}\\
 \z_t &\in \Real^k \label{eqb:2}\\
   \bar{\cov}_t &= \begin{bmatrix}
                  \bcov^{m^2} & \bcov^{mp} \\
                  \bcov^{pm} & \bcov^{p^2}
                 \end{bmatrix} \in \Real^{n \times n}\label{eqb:3}\\
 \C &= \begin{bmatrix} \I{k} & \Zero{n-k} \end{bmatrix} \in \Real^{k\times n}
\label{eqb:4}\\
 \K^\transpose &= (\R_t + \C \bcov_t \C^\transpose)^\inverse \C \bcov_t\quad
\text{with}\;
\K \in \Real^{n\times k} \label{eqb:5}\\
 \mu_t &= \bar{\mu}_t + \K (\z_t - \C\bar{\mu}_t) \label{eqb:6}\\
 \cov_t &= \bcov_t - \K \C \bcov_t \label{eqb:7}
\end{align}

\end{paracol}
The Equations \ref{eq:1}--\ref{eq:10} are taken directly from
\cite{bry2015ijrr} without modifications. The dimensionality of the matrices
were not specified and the subscript $m$ is used inconsistently (e.g., is
$\C^m$ the same as $\C$ or is it only its left part?).
The introductory part about partitioning the state makes the reader believe
that all matrices with the $m$ superscript are square $k \times k$. Instead,
they have standard size, therefore Equations \ref{eq:5}--\ref{eq:7} correspond
to standard Kalman updates equations where the Jacobian of $h$ is simply the
selector matrix $\C$.

Since the GPF keeps track of mean and covariance of a set of particles over
time, independently of the Kalman filter, on the left side it is assumed that
the prior mean $\bar{\mu}^m$, the posterior mean $\mu^m$ and their respective
covariance matrices are known, whereas the mesurement $\z_t$ and its covariance
matrix $\R_t$ are unknown, so they are recovered in Equations
\ref{eq:9}--\ref{eq:10} and then plugged back in to make the standard Kalman
update.

In the code related to non-GPF updates, this is not necessary because $\z_t$
and $\R_t$ are provided by the user. Therefore, a standard Kalman update is
carried out by Equations \ref{eqb:5}--\ref{eqb:7}, which are implemented in
\cite{pronto}.
For numerical reasons, in Equation \ref{eqb:5} the transpose of the Kalman
gain is computed. Howerver, since $\R_t$, $\bcov_t$, and
$\C\bcov_t\C^\transpose$ are symmetric, it is easy to prove that (\ref{eqb:5})
is equivalent to (\ref{eq:5}).
\bibliographystyle{plain}
\bibliography{bry_explanation}

\end{document}