\chapter{Riemann integrals}
\begin{quote}
	\footnotesize
	``Trying to Riemann integrate discontinuous
	functions is kind of outdated.'' \\
	--- Dennis Gaitsgory, \cite{ref:55b}
\end{quote}

We will go ahead and define the Riemann integral,
but we won't do very much with it.
The reason is that the Lebesgue integral is basically better,
so we will define it, check the fundamental theorem of calculus
(or rather, leave it as a problem at the end of the chapter),
and then always use Lebesgue integrals forever after.

\section{Uniform continuity}
\prototype{$f(x) = x^2$ is not uniformly continuous on $\RR$,
but functions on compact sets are always uniformly continuous.}
\begin{definition}
	Let $f \colon M \to N$ be a continuous
	map between two metric spaces.
	We say that $f$ is \vocab{uniformly continuous} if
	for all $\eps > 0$ there exists a $\delta > 0$ such that
	\[ d_M(p,q) < \delta \implies d_N(f(p), f(q)) < \eps. \]
\end{definition}
This difference is that given an $\eps > 0$ we must specify a $\delta > 0$
which works for \emph{every} choice $p$ and $q$ of inputs;
whereas usually $\delta$ is allowed to depend on $p$ and $q$.
(Also, this definition can't be ported
to a general topological space.)

\begin{example}
	[Uniform continuity failure]
	\listhack
	\begin{enumerate}[(a)]
		\ii The function $f \colon \RR \to \RR$ by $x \mapsto x^2$
		is not uniformly continuous.
		Suppose we take $\eps = 0.1$ for example.
		There is no $\delta$ such that if $|x-y| < \delta$
		then $|x^2-y^2| < 0.1$,
		since as $x$ and $y$ get large,
		the function $f$ becomes increasingly sensitive
		to small changes.

		\ii The function $(0,1) \to \RR$ by $x \mapsto x\inv$
		is not uniformly continuous.

		\ii The function $\RR_{>0} \to \RR$ by $x \mapsto \sqrt x$
		does turn out to be uniformly continuous
		(despite having unbounded derivatives!).
		Indeed, you can check that the assertion
		\[ \left\lvert x-y  \right\rvert < \eps^2
			\implies \left\lvert \sqrt x - \sqrt y \right\rvert
			< \eps \]
		holds for any $x, y, \eps > 0$.
	\end{enumerate}
\end{example}

The good news is that in the compact case all is well.
\begin{theorem}
	[Uniform continuity free for compact spaces]
	Let $M$ be a compact metric space.
	Then any continuous map $f \colon M \to N$ is
	also uniformly continuous.
\end{theorem}
\begin{proof}
	Assume for contradiction there is some bad $\eps > 0$.
	Then taking $\delta = 1/n$,
	we find that for each integer $n$
	there exists points $p_n$ and $q_n$
	which are within $1/n$ of each other,
	but are mapped more than $\eps$ away from each other by $f$.
	In symbols, $d_M(p_n, q_n) < 1/n$ but $d_N(f(p_n), f(q_n)) > 1/n$.

	By compactness of $M$,
	we can find a convergent subsequence
	$p_{i_1}$, $p_{i_2}$, \dots\ converging to some $x \in M$..
	Since the $q_{i_n}$ is within $1/i_n$ of $p_{i_n}$,
	it ought to converge as well, to the same point $x \in M$.
	Then the sequences $f(p_{i_n})$ and $f(q_{i_n})$
	should both converge to $f(x) \in N$,
	but this is impossible as they are always $\eps$
	away from  each other.
\end{proof}
This means for example that $x^2$ viewed
as a continuous function $[0,1] \to \RR$ is automatically
uniformly continuous.
Man, isn't compactness great?

\section{Dense sets and extension}
\prototype{Functions from $\QQ \to N$ extend to $\RR \to N$
if they're uniformly continuous and $N$ is complete.
See also counterexamples below.}

\begin{definition}
	Let $S$ be a subset (or subspace) of a topological space $X$.
	Then we say that $S$ is \vocab{dense}
	if every open subset of $X$ contains a point of $S$.
\end{definition}

\begin{example}
	[Dense sets]
	\listhack
	\begin{enumerate}[(a)]
		\ii $\QQ$ is dense in $\RR$.
		\ii In general, any metric space $M$ is dense
		in its completion $\ol M$.
	\end{enumerate}
\end{example}

Dense sets lend themselves to having functions completed.
The idea is that if I have a continuous function
$f \colon \QQ \to N$, for some metric space $N$,
then there should be \emph{at most} one way to extend it to a function
$\wt f \colon \RR \to N$.
For we can approximate each rational number by real numbers:
if I know $f(1)$, $f(1.4)$, $f(1.41)$, \dots\
$\wt f(\sqrt2)$ had better be the limit of this sequence.
So it is certainly unique.

However, there are two ways this could go wrong:
\begin{example}
	[Non-existence of extension]
	\listhack
	\begin{enumerate}[(a)]
		\ii It could be that $N$ is not complete,
		so the limit may not even exist in $N$.
		For example if $N = \QQ$,
		then certainly there is no way to
		extend even the identify function $f \colon \QQ \to N$
		to a function $\wt f \colon \QQ \to N$.

		\ii Even if $N$ was complete, we might run into issues
		where $f$ explodes.
		For example, let $N = \RR$ and define
		\[ f(x) = \frac{1}{x-\sqrt2} \qquad f \colon \QQ \to \RR. \]
		There is also no way to extend this
		due to the explosion of $f$ near $\sqrt2 \notin \QQ$,
		which would cause $\wt f(\sqrt2)$ to be undefined.
	\end{enumerate}
\end{example}
However, the way to fix this is to require $f$ to be uniformly continuous,
and in that case we do get a unique extension.

\begin{theorem}
	[Extending uniformly continuous functions]
	Let $M$ be a metric space, $N$ a \emph{complete} metric space,
	and $S$ a dense subspace of $M$.
	Suppose $\psi \colon S \to N$ is a \emph{uniformly} continuous function.
	Then there exists a unique continuous function $\wt \psi \colon M \to N$
	such that the diagram
	\begin{center}
	\begin{tikzcd}
		M \ar[r, "\wt \psi"] & N \\
		S \ar[u, hook] \ar[ru, "\psi"'] &
	\end{tikzcd}
	\end{center}
	commutes.
\end{theorem}
\begin{proof}
	[Outline of proof]
	As mentioned in the discussion,
	each $x \in M$ can be approximated by a
	sequence $x_1$, $x_2$, \dots\ in $S$ with $x_i \to x$.
	The two main hypotheses, completeness and uniform continuity,
	are now used:
	\begin{exercise}
		Prove that $\psi(x_1)$, $\psi(x_2)$, \dots\ converges in $N$
		by using uniform continuity to show that it is Cauchy,
		and then appealing to completeness of $N$.
	\end{exercise}
	Hence we define $\wt \psi(x)$ to be the limit of that sequence;
	this doesn't depend on the choice of sequence,
	and one can use sequential continuity to show $\wt \psi$ is continuous.
\end{proof}

\section{Defining the Riemann integral}
Extensions will allow us to define the Riemann integral.
I need to introduce a bit of notation so bear with me.

\begin{definition}
	Let $[a,b]$ be a closed interval.
	\begin{itemize}
		\ii We let $C^0([a,b])$ denote the set of
		continuous functions on $[a,b] \to \RR$.
		\ii We let $R([a,b])$ denote the set of
		\textbf{rectangle functions} on $[a,b] \to \RR$.
		These functions which are constant on the intervals
		$[t_0,t_1)$, $[t_1, t_2)$, $[t_2, t_3)$, \dots, $[t_{n-2}, t_{n-1})$, %chktex 9
		and also $[t_{n-1}, t_n]$,
		for some $a = t_0 < t_1 < t_2 < \dots < t_n = b$.
		\ii We let $M([a,b]) = C^0([a,b]) \cup R([a,b])$.
	\end{itemize}
\end{definition}
Warning: only $C^0([a,b])$ is common notation,
and the other two are made up.

See picture below for a typical a rectangle function.
(It is irritating that we have to officially assign a single
value to each $t_i$,
even though there are naturally two values we want to use,
and so we use the convention of letting
the left endpoint be closed).
\begin{center}
\begin{asy}
	import graph;
	graph.xaxis();
	graph.yaxis();
	label("$a=t_0$", (-3,0), dir(-90), red);
	label("$b=t_4$", ( 3,0), dir(-90), red);
	pen b = blue + 1.2;
	draw( (-3,3)--(-1.2,3), b );
	draw( (-1.2,1)--(0.8,1), b );
	draw( (0.8,4)--(2,4), b );
	draw( (2,2.5)--(3,2.5), b );

	draw( (-3,3)--(-3,0), red );
	draw( (-1.2,3)--(-1.2,0), dotted+heavycyan );
	draw( (0.8,4)--(0.8,0), dotted+heavycyan );
	draw( (2,4)--(2,0), dotted+heavycyan );
	draw( (3,2.5)--(3,0), red );
	label("$t_1$", (-1.2,0), dir(-90), heavycyan);
	label("$t_2$", (0.8,0), dir(-90), heavycyan);
	label("$t_3$", (2,0), dir(-90), heavycyan);

	dotfactor *= 2;
	dot( (-3,3), blue ); // x = a
	opendot( (-1.2, 3), blue );
	opendot( (0.8, 1), blue );
	opendot( (2, 4), blue );
	dot( (-1.2,1), blue );
	dot( (0.8,4), blue );
	dot( (2,2.5), blue );
	dot( (3, 2.5), blue ); // x = b
\end{asy}
\end{center}

\begin{definition}
	We can impose a metric on $M([a,b])$
	by defining
	\[ d(f,g) = \sup_{x \in [a,b]} \left\lvert f(x) - g(x) \right\rvert. \]
\end{definition}

Now, there is a natural notion of integral
for rectangle functions: just sum up the obvious rectangles!
Officially, this is the expression
\[ f(a)(t_1-a) + f(t_1)(t_2-t_1) +
	+ f(t_2) \left( t_3 - t_2 \right)
	+ \dots + f(t_n) \left( b - t_n \right).  \]
We denote this function by
\[ \Sigma : R([a,b]) \to \RR. \]

\begin{theorem}
	[The Riemann integral]
	\label{thm:gaitsgory_riemann}
	There exists a unique continuous map
	\[ \textstyle{\int_a^b} \colon M([a,b]) \to \RR \]
	such that the diagram
	\begin{center}
	\begin{tikzcd}
		M([a,b]) \ar[r, "\int_a^b"] & \RR \\
		R([a,b]) \ar[u, hook] \ar[ru, "\Sigma"'] &
	\end{tikzcd}
	\end{center}
	commutes.
\end{theorem}
\begin{proof}
	We want to apply the extension theorem,
	so we just have to check a few things:
	\begin{itemize}
		\ii We claim $R([a,b])$ is a dense subset of $M([a,b])$.
		In other words, for any continuous $f \colon [a,b] \to \RR$
		and $\eps > 0$,
		we want there to exist a rectangle function
		that approximates $f$ within $\eps$.

		This follows by uniform continuity.
		We know there exists a $\delta > 0$ such
		that whenever $|x-y| < \delta$ we have $|f(x)-f(y)| < \eps$.
		So as long as we select a rectangle function
		whose rectangles have width less than $\delta$,
		and such that the upper-left corner of each rectangle
		lies on the graph of $f$, then we are all set.

		\begin{center}
		\begin{asy}
			import graph;
			real f(real x) { return 3 + (x-2)*(x-2)*(x+5) / 35; }
			draw( (5, f(5))--(5, 0), red+dotted );
			for (int i=0; i<10; ++i) {
				filldraw(box( (i-5, 0), (i-4, f(i-5)) ),
					opacity(0.1)+lightgreen, deepgreen);
				dot( (i-5, f(i-5)), deepgreen);
			}
			draw(graph(f,-5.2,5.2,operator ..), blue, Arrows(TeXHead));
			graph.xaxis();
			graph.yaxis();
			label("$a$", (-5,0), dir(-90), blue);
			label("$b$", (5,0), dir(-90), blue);
		\end{asy}
		\end{center}

		\ii The ``add-the-rectangles'' map $\Sigma \colon R([a,b]) \to \RR$
		is \emph{uniformly} continuous.
		Actually this is pretty obvious:
		if two rectangle functions $f$ and $g$
		have $d(f,g) < \eps$,
		then $d(\Sigma f, \Sigma g) < \eps(b-a)$.

		\ii $\RR$ is complete.\qedhere
	\end{itemize}
\end{proof}

\section{Meshes}
The above definition might seem fantastical, overcomplicated,
hilarious, or terrible, depending on your taste.
But if you unravel it, it's really the picture you are used to.
What we have done is taking every continuous function $f \colon [a,b] \to \RR$
and showed that it can be approximated by a rectangle function
(which we phrased as a dense inclusion).
Then we added the area of the rectangles.
Nonetheless, we will give a definition that's
more like what you're used to seeing in other places.
\begin{definition}
	A \emph{tagged partition} $P$ of $[a,b]$
	consists of a partition of $[a,b]$ into $n$ intervals,
	with a point $\xi_i$ in the $n$th interval, denoted
	\[ a = t_0 < t_1 < t_2 < \dots < t_n = b
		\qquad\text{and}\qquad \xi_i \in [t_{i-1}, t_i]
		\quad \forall \; 1 \le i \le n.  \]
	The \emph{mesh} of $P$ is the width
	of the longest interval, i.e.\ $\max_i(t_i - t_{i-1})$.
\end{definition}

Of course the point of this definition
is that we add the rectangles,
but the $\xi_i$ are the sample points.

\begin{center}
\begin{asy}
	import graph;
	real f(real x) { return 3 + (x-2)*(x-2)*(x+5) / 35; }
	real[] t = {-4.4, -3.3, -0.2, 1.3, 3.5, 4.5};
	filldraw(box((-5,0), (-4.1,f(t[0]))), opacity(0.1)+lightcyan, heavycyan );
	filldraw(box((-4.1,0), (-2,f(t[1]))), opacity(0.1)+lightcyan, heavycyan );
	filldraw(box((-2,0), (0.6,f(t[2]))), opacity(0.1)+lightcyan, heavycyan );
	filldraw(box((0.6,0), (1.9,f(t[3]))), opacity(0.1)+lightcyan, heavycyan );
	filldraw(box((1.9,0), (3.7,f(t[4]))), opacity(0.1)+lightcyan, heavycyan );
	filldraw(box((3.7,0), (5,f(t[5]))), opacity(0.1)+lightcyan, heavycyan );
	for (int i=0; i<=5; ++i) { dot( (t[i], f(t[i])), blue ); }
	draw(graph(f,-5.2,5.2,operator ..), blue, Arrows(TeXHead));
	label("$\xi$", (t[1], f(t[1])), dir(90), blue);
	graph.xaxis();
	graph.yaxis();
	label("$a$", (-5,0), dir(-90), blue);
	label("$b$", (5,0), dir(-90), blue);
\end{asy}
\end{center}

\begin{theorem}
	[Riemann integral]
	Let $f \colon [a,b] \to \RR$ be continuous.
	Then
	\[ \int_a^b f(x) \; dx
		= \lim_{\substack{\text{$P$ tagged partition}
				\\ \opname{mesh} P \to 0}}
			\left( \sum_{i=1}^n f(\xi_i) (t_i - t_{i-1}) \right). \]
	Here the limit means that we can take any sequence
	of partitions whose mesh approaches zero.
\end{theorem}
\begin{proof}
	The right-hand side corresponds to the areas
	of some rectangle functions $g_1$, $g_2$, \dots
	with increasingly narrow rectangles.
	As in the proof \Cref{thm:gaitsgory_riemann},
	as the meshes of those rectangles approaches zero,
	by uniform continuity, we have $d(f, g_n) \to 0$ as well.
	Thus by continuity in the diagram of \Cref{thm:gaitsgory_riemann},
	we get $\lim_n \Sigma(g_n) = \int(f)$ as needed.
\end{proof}

Combined with the mean value theorem,
this can be used to give a short proof of
the fundamental theorem of calculus
for functions $f$ with a continuous derivative.
The idea is that for any choice of partition
$a \le t_0 < t_1 < t_2 < \dots < t_n \le b$,
using the Mean Value Theorem it should be possible
to pick $\xi_i$ in each interval to match
with the slope of the secant:
at which point the areas sum to the total change in $f$.
We illustrate this situation with three points,
and invite the reader to fill
in the details as \Cref{thm:FTC}.

\begin{center}
\begin{asy}
	size(9cm);
	import graph;
	graph.xaxis("$x$");
	graph.yaxis("$y$");
	real f(real x) { return x*x/2+0.4; }
	pair P(real x) { return (x, f(x)); }
	draw(graph(f,-2,2,operator ..), blue, Arrows(TeXHead));
	label("$f$", (2, f(2)), dir(-45), blue);
	for (real i=-1.3; i<=2; ++i) {
		if (i < 1) { draw(P(i)--P(i+1), deepgreen, EndArrow(TeXHead), Margins); }
		dot(P(i), deepgreen);
	}
	real m1 = f(-0.3) - f(-1.3);
	real m2 = f(0.7) - f(-0.3);
	real m3 = f(1.7) - f(0.7);
	real r = 0.4;
	void draw_tangent(real m) {
		dot(P(m), red);
		draw((m-r, f(m)-r*m)--(m+r, f(m)+r*m), red);
	}
	draw_tangent(m1);
	draw_tangent(m2);
	draw_tangent(m3);
	draw(P(-1.3)--(-1.3,0), deepgreen+dashed);
	draw(P(-0.3)--(-0.3,0), deepgreen+dashed);
	draw(P( 0.7)--( 0.7,0), deepgreen+dashed);
	draw(P( 1.7)--( 1.7,0), deepgreen+dashed);
	label("$t_0$", (-1.3,0), dir(-90), deepgreen);
	label("$t_1$", (-0.3,0), dir(-90), deepgreen);
	label("$t_2$", ( 0.7,0), dir(-90), deepgreen);
	label("$t_3$", ( 1.7,0), dir(-90), deepgreen);
	draw(P(m1)--(m1,0), red+dashed);
	draw(P(m2)--(m2,0), red+dashed);
	draw(P(m3)--(m3,0), red+dashed);
	label("$\xi_1$", (m1,0), dir(-90), red+fontsize(8pt));
	label("$\xi_2$", (m2,0), dir(-90), red+fontsize(8pt));
	label("$\xi_3$", (m3,0), dir(-90), red+fontsize(8pt));
	draw(P(-1.3)--P(1.7), blue+dashed, EndArrow, Margins);
	label("Net change", 0.3*P(-1.3)+0.7*P(1.7), dir(90), blue);
\end{asy}
\end{center}

One quick note is that although I've only defined
the Riemann integral for continuous functions,
there ought to be other functions for which it exists
(including ``piecewise continuous functions'' for example,
or functions ``continuous almost everywhere'').
The relevant definition is:
\begin{definition}
	If $f \colon [a,b] \to \RR$ is a function
	which is not necessarily continuous,
	but for which the limit
	\[ \lim_{\substack{\text{$P$ tagged partition}
			\\ \opname{mesh} P \to 0}}
		\left( \sum_{i=1}^n f(\xi_i) (t_i - t_{i-1}) \right). \]
	exists anyways,
	then we say $f$ is \vocab{Riemann integrable} on $[a,b]$
	and define its value to be that limit $\int_a^b f(x) \; dx$.
\end{definition}
We won't really use this definition much,
because we will see that every Riemann integrable function
is Lebesgue integrable, and the Lebesgue integral is better.

\begin{example}
	[Your AP calculus returns]
	We had better mention that \Cref{thm:FTC}
	implies that we can compute Riemann integrals in practice,
	although most of you may already know this
	from high-school calculus
	For example, on the interval $(1,4)$,
	the derivative of the function $F(x) = \frac13 x^3$ is $F'(x) = x^2$.
	As $f(x) = x^2$ is a continuous function $f \colon [1,4] \to \RR$, we get
	\[ \int_1^4 x^2 \; dx =
		F(4) - F(1) = \frac{64}{3} - \frac13 = 21. \]
	Note that we could also have picked $F(x) = \frac13x^3 + 2019$;
	the function $F$ is unique up to shifting,
	and this constant cancels out when we subtract.
	This is why it's common in high school to (really) abuse notation
	and write $\int x^2 \; dx = \frac13x^3+C$.
\end{example}

\section{\problemhead}
\begin{problem}
	Let $f \colon (a,b) \to \RR$ be differentiable
	and assume $f'$ is bounded.
	Show that $f$ is uniformly continuous.
	\begin{hint}
		Contradiction and mean value theorem (again!).
	\end{hint}
\end{problem}

\begin{sproblem}
	[Fundamental theorem of calculus]
	\label{thm:FTC}
	Let $f \colon [a,b] \to \RR$ be continuous,
	differentiable on $(a,b)$,
	and assume the derivative $f'$ extends to a
	continuous function $f' \colon [a,b] \to \RR$.
	Prove that
	\[ \int_a^b f'(x) \; dx = f(b) - f(a). \]
	\begin{hint}
		For every positive integer $n$,
		take a partition where every rectangle has width $w = \frac{b-a}{n}$.
		Use the mean value theorem to construct a tagged partition
		such that the first rectangle has area $f(a+w)-f(a)$,
		the second rectangle has area $f(a+2w) - f(a+w)$, and so on;
		thus the total area is $f(b) - f(a)$.
	\end{hint}
\end{sproblem}

\begin{sproblem}
	[Improper integrals]
	\label{prob:improper}
	For each real number $r > 0$,
	evaluate the limit\footnote{If you are not
		familiar with the notation $\eps \to 0^+$,
		you can replace $\eps$ with $1/M$ for $M > 0$,
		and let $M \to \infty$ instead.}
	\[ \lim_{\eps \to 0^+} \int_\eps^1 \frac{1}{x^r} \; dx \]
	or show it does not exist.

	This can intuitively be thought of as
	the ``improper'' integral $\int_0^1 x^{-r} \; dx$;
	it doesn't make sense in our original definition since
	we did not (and cannot) define the integral
	over the non-compact $(0,1]$ %chktex 9
	but we can still consider the integral over $[\eps,1]$
	for any $\eps > 0$.
\end{sproblem}

\begin{problem}
	Show that
	\[ \lim_{n \to \infty}
		\left( \frac{1}{n+1} + \frac{1}{n+2} + \dots + \frac{1}{2n} \right)
		= \log 2.  \]
	\begin{hint}
		Write this as $\frac1n \sum_{k=1}^n \frac{1}{1+\frac kn}$.
		Then you can interpret it as a rectangle sum
		of a certain Riemann integral.
	\end{hint}
\end{problem}


\section{Temporal Difference Learning}

\subsection{Constant-$\alpha$ \cindex{TD(0)} Prediction}

Suppose at time $t+1$, state becomes $S_{t+1}$ with reward $R_{t+1}$. The  \cindex{TD error} is defined as:
\begin{equation}
	\delta_t = R_{t+1} + \gamma V(S_{t+1}) - V(S_t)
\end{equation}

The simplest TD is:
\begin{equation}
	V(S_t) \gets V(S_t) + \alpha \delta_t
\end{equation}

Because TD use Markov property while MC is not, it is usually more efficient.

It use $R_{t+1} + \gamma V(S_{t+1})$ to estimate $G_t$ in formula $V(S_t) \gets V(S_t) + \alpha \Big ( G_t - V(S_t) \Big )$ which is an average of all $G_t$.

TD is \cindex{sample update} which explores just one condition while MC is \cindex{expected update} which explores all choices.

The advantage of TD:
\begin{description}
	\item [over DP] TD does not require a model of environment.
	\item [over MC] TD is an online, fully incremental fashion, more efficient.
\end{description}

TD is sensitive to initial value (guess).

Algorithm (\ref{algo:tdpe}) contains detail.

\begin{algorithm}
	\caption{TD(0) policy evaluation, estimate $v_\pi$}\label{algo:tdpe}

	\begin{algorithmic}[1]
		\State $ \alpha \in (0,1]$
		\State $V(s) \gets$ random \Comment{bootstrap}

		\Statex

		\Loop
			\State choose $S$
			\Repeat
				\State $A \gets$ action given by $\pi$ for $S$ \Comment following the given policy $\pi$
				\State take action $A$, get $R$ and $S'$
				\State $V(S) \gets V(S) + \alpha \Big (R + \gamma V(S') - V(S) \Big )$
				\State $S \gets S'$
			\Until{$S$ is terminal}
		\EndLoop
	\end{algorithmic}
\end{algorithm}

\subsection{Sarsa: On-policy TD Algorithm}

\subsubsection{Sarsa Prediction}

\cindex{Sarsa} is a process of $S_t,A_t,R_{t+1},S_{t+1},A_{t+1}$, and its formula is:
\begin{equation}
	Q(S_t,A_t) \gets Q(S_t,A_t) + \alpha \Big ( R_{t+1} + \gamma Q(S_{t+1},A_{t+1}) - Q(S_t,A_t) \Big )
\end{equation}

If $S_{t+1}$ is terminal, then $Q(S_{t+1},A_{t+1})$ is $0$.

\subsubsection{Sarsa Control}

The algorithm is almost the same as Sarsa prediction. The difference is to update the policy on the fly using $\varepsilon$-soft or $\varepsilon$-greedy policy.

Algorithm (\ref{algo:sarsa}) contains detail.

\begin{algorithm}
	\caption{on-policy Sarsa TD control, estimate $q_*$}\label{algo:sarsa}

	\begin{algorithmic}[1]
		\State $ \alpha \in (0,1]$
		\State $Q(s,a) \gets$ random

		\Statex

		\Loop
			\State choose $S$
			\State $A \gets$ action from a policy derived from $Q$ (e.g., $\varepsilon$-greedy) \Comment {update policy}
			\Repeat
				\State take action $A$, get $R$ and $S'$
				\State \Comment {update policy after each iteration}
				\State $A' \gets$ action from a policy derived from $Q$ (e.g., $\varepsilon$-greedy)
				\State $Q(S,A) \gets Q(S,A) + \alpha \Big (R + \gamma Q(S',A') - Q(S,A) \Big )$
				\State $S \gets S'$
				\State $A \gets A'$
			\Until{$S$ is terminal}
		\EndLoop
	\end{algorithmic}
\end{algorithm}

\subsection{Expected Sarsa Learning Algorithm}

\cindex{expected Sarsa} use expectation, rather than $\varepsilon$-greedy, to learn next $q$. It reduces variance by removing the random selection of $A_{t+1}$ during $\varepsilon$-soft policy :

\begin{equation}
	Q(S_t,A_t) \gets Q(S_t,A_t) + \alpha \left ( R_{t+1} + \gamma \sum_a \pi(a|S_{t+1}) Q(S_{t+1},a) - Q(S_t,A_t) \right )
\end{equation}

\subsection{\cindex{Q-learning}: Off-policy TD Control}

\cindex{Q-learning} is defined as:
\begin{equation}
	Q(S_t,A_t) \gets Q(S_t,A_t) + \alpha \Big ( R_{t+1} + \gamma \max_a Q(S_{t+1},a) - Q(S_t,A_t) \Big )
\end{equation}

Algorithm (\ref{algo:qlearning}) contains detail.

\begin{algorithm}
	\caption{off-policy Q-learning TD control, estimate $\pi_*$}\label{algo:qlearning}

	\begin{algorithmic}[1]
		\State $ \alpha \in (0,1]$
		\State $Q(s,a) \gets$ random

		\Statex

		\Loop
			\State choose $S$
			\Repeat
				\State $A \gets$ action given by $Q$ for $S$ (e.g., $\varepsilon$-greedy)
				\State take action $A$, get $R$ and $S'$
				\State $Q(S,A) \gets Q(S,A) + \alpha \Big ( R + \gamma \underset{a}{\max}\ Q(S',a) - Q(S,A) \Big )$
				\State $S \gets S'$
			\Until{$S$ is terminal}
		\EndLoop
	\end{algorithmic}
\end{algorithm}

\subsection{Double Learning}

Maximization has \cindex{maximization bias}. For example, if $\mathbb{E}[q(s,a)] = 0$, then $\max q(s,a) > 0$. So the selection of $q$ is always a positive bias. One way of viewing the problem is that maximization use the same sample to determine action and estimate its value. \cindex{double learning} uses two $q$ for determination and estimation.

Algorithm (\ref{algo:doublelearning}) contains detail of double q-learning TD control.

\begin{algorithm}
	\caption{double Q-learning TD control, estimate $q_*$}\label{algo:doublelearning}

	\begin{algorithmic}[1]
		\State $ \alpha \in (0,1]$
		\State $Q_1(s,a) \gets$ random
		\State $Q_2(s,a) \gets$ random

		\Statex

		\Loop
			\State choose $S$
			\Repeat
				\State $A \gets$ action given by $\varepsilon$-greedy policy of $Q_1+Q_2$ for $S$
				\State take action $A$, get $R$ and $S'$
				\State $p \gets \text{random}(0,1)$
				\If{$p > 0.5$}
					\State $Q_1(S,A) \gets Q_1(S,A) + \alpha \Big(R + \gamma Q_2(S',  \underset{a}{\max}\  Q_1(S',a)) - Q_1(S,A)\Big)$
				\Else
					\State $Q_2(S,A) \gets Q_2(S,A) + \alpha \Big(R + \gamma Q_1(S',\underset{a}{\max}\  Q_2(S',a)) - Q_2(S,A)\Big)$
				\EndIf
				\State $S \gets S'$
			\Until{$S$ is terminal}
		\EndLoop
	\end{algorithmic}
\end{algorithm}

There are double learning version for Sarsa and Expected-Sarsa as well.

\subsection{Comments}

Compared with MC, TD is sensitive to initial value. It explores Monte Carlo property and is more effective.

\begin{itemize}
    \item TD: average $V$ using $R + \gamma V$.
    \item Sarsa: from $V$ to $Q$.
    \item Expect Sarsa: remove randomness of Sarsa by average.
    \item Q: replace average by $\max$.
    \item Double-Q: reduce $\max$ bias by using two queues.
\end{itemize}


\section{Conclusions}

After applying three methods, $k$-NN, SVM, random forest, to Seattle climate dataset, SVM and random forest have similar results in terms of overall error rate and are better than $k$-NN. SVM has a kind of balance between type \uppercase\expandafter{\romannumeral1} and type \uppercase\expandafter{\romannumeral2} error, while random forest has larger type \uppercase\expandafter{\romannumeral2} error and lower type \uppercase\expandafter{\romannumeral1} error. So random forest may make more mistakes in predicting days with precipitation. In reality people may care more and change their action, like bring an umbrella, if they know it will rain tomorrow. So in that case SVM would be better than random forest. Also the results of applying the model to New York data suggest we can’t use the models directly on another city without modification. Overall, the classification methods are reliable if we have enough history data and just want to make prediction of precipitation.


\subsection{Estimating HMMs with the forward-backward algorithm}

We calculate state x at time t given all obs.


% Robert Lech - June 2015

\documentclass{article}
\usepackage{times}
\usepackage{color}
\usepackage{amsmath}

\begin{document}

% Article top matter
\title{How to Solve the Fibonacci Recursion using Linear Algebra} %\LaTeX is a macro for printing the Latex logo
\author{Robert Lech\\
		Carleton University,\\
		Ottawa, Canada,\\
		K1S 5B6\\
		\texttt{robert.lech123@gmail.com}}  %\texttt formats the text to a typewriter style font
\date{\today}  %\today is replaced with the current date
\maketitle

\begin{abstract}
We're curious to understand why the Fibonacci recursion $R(n)=R(n-1)+R(n-2), R(0)=R(1)=1$ has the closed form solution $R(n)=C_{1}\lambda_{1}^{n}+C_{2}\lambda_{2}^{n}$ where $C_{1}=\frac{1}{\sqrt{5}}$ and $C_{2}=-\frac{1}{\sqrt{5}}$.
In this paper, we'll see how to solve this recursion using Linear Algebra techniques.

\textcolor{red}{FINISH THIS}
\end{abstract}

\section{Introduction}
\textcolor{red}{FINISH THIS}

\section{Stuff}
We're interested in solving:

\begin{equation}
\label{eqn:fib_rec}
R(n)=R(n-1)+R(n-2), R(0)=0, R(1)=1
\footnote{The reader can note that solving this recursion with the initial
conditions $R(0)=0, R(1)=1$ or doing so with the intial conditions % chktex 36
$R(1)=1, R(2)=1$ gives us the same closed form solution as they output the % chktex 36
same sequence. We use the former set of initial conditions because it allows
for less computations.}
\end{equation}

We first want to rewrite this problem as a Linear Algebra problem. To do so,
we'll introduce a concept called the ``Companion Matrix''. Its purpose is to
transform a vector of sequence elements
$\begin{bmatrix} y_{n} \\ y_{n-1} \end{bmatrix}$
to $\begin{bmatrix} y_{n+1} \\ y_{n} \end{bmatrix}$.
Therefore, we'll say that:

\begin{align*}
\vec{y_{n}}
=
\begin{bmatrix}
y_{n+1} \\
y_{n}
\end{bmatrix}
&=
\begin{bmatrix}
1 & 1 \\
1 & 0
\end{bmatrix}
\begin{bmatrix}
y_{n} \\
y_{n-1}
\end{bmatrix} \\
&=
\vec{C}
\begin{bmatrix}
y_{n} \\
y_{n-1}
\end{bmatrix} \\
&=
\vec{C}\vec{C}
\begin{bmatrix}
y_{n-1} \\
y_{n-2}
\end{bmatrix} \\
&=
\vec{C}^{2}
\begin{bmatrix}
y_{n-1} \\
y_{n-2}
\end{bmatrix} \\
\vdots\\
&=
\vec{C}^{n}
\begin{bmatrix}
y_{1} \\
y_{0}
\end{bmatrix}
\end{align*}

Therefore, to solve $\vec{y_{n}}$, we'd need to solve $\vec{C}^{n}$. There is
no immediate answer to this as computing the general $n^{th}$ power of a
general matrix can get quite ugly. However, we can recall that calculating the
$n^{th}$ power of a diagonalizable matrix is a lot easier. Let $\lambda_{i}$
and $\vec{v_i}$ be $i^{th}$ eigenvalue and eigenvector of $\vec{C}$,
respectively. Then we can let $\vec{P}=(\vec{v_i})^{T}$ be the matrix create
by the eigenvectors of $\vec{C}$ and we can let $\vec{\Lambda}$ be the matrix
who's diagonal entries are the corresponding $\lambda$ values.
We can also use Gauss-Jordan Elimination to compute $\vec{P}^{-1}$. We can
recall that:

\begin{equation}
\vec{C}^{n}
=
\left(\vec{P}\vec{\Lambda}\vec{P}^{-1}\right)^{n}
=
\vec{P}\vec{\Lambda}^{n}\vec{P}^{-1}
\end{equation}

This is a lot easier to compute. We can obtain the eigenvalues
$\lambda_{1}=\frac{1-\sqrt{5}}{2}$ and $\lambda_{2}=\frac{1+\sqrt{5}}{2}$.
The respective eigenvectors are
$\vec{v_1}= \begin{bmatrix}\lambda_{1} \\ 1 \end{bmatrix}$ and
$\vec{v_2} = \begin{bmatrix}\lambda_{2} \\ 1 \end{bmatrix}$.
Therefore, we have that:

\begin{align*}
\begin{bmatrix}
1 & 1 \\
1 & 0
\end{bmatrix}^{n}
&=
\left(
\begin{bmatrix}
\lambda_{1} & \lambda_{2} \\
1 & 1
\end{bmatrix}
\begin{bmatrix}
\lambda_{1} & 0 \\
0 & \lambda_{2}
\end{bmatrix}
\begin{bmatrix}
-(\lambda_{2}-\lambda_{1})^{-1} & 1+(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1} \\
(\lambda_{2}-\lambda_{1})^{-1} & -(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1}
\end{bmatrix}
\right)^{n} \\
&=
\begin{bmatrix}
\lambda_{1} & \lambda_{2} \\
1 & 1
\end{bmatrix}
\begin{bmatrix}
\lambda_{1} & 0 \\
0 & \lambda_{2}
\end{bmatrix}^{n}
\begin{bmatrix}
-(\lambda_{2}-\lambda_{1})^{-1} & 1+(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1} \\
(\lambda_{2}-\lambda_{1})^{-1} & -(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1}
\end{bmatrix}\\
&=
\begin{bmatrix}
\lambda_{1} & \lambda_{2} \\
1 & 1
\end{bmatrix}
\begin{bmatrix}
\lambda_{1}^{n} & 0 \\
0 & \lambda_{2}^{n}
\end{bmatrix}
\begin{bmatrix}
-(\lambda_{2}-\lambda_{1})^{-1} & 1+(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1} \\
(\lambda_{2}-\lambda_{1})^{-1} & -(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1}
\end{bmatrix}\\
&=
\begin{bmatrix}
\lambda_{1}^{n+1} & \lambda_{2}^{n+1} \\
\lambda_{1}^{n} & \lambda_{2}^{n}
\end{bmatrix}
\begin{bmatrix}
-(\lambda_{2}-\lambda_{1})^{-1} & 1+(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1} \\
(\lambda_{2}-\lambda_{1})^{-1} & -(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1}
\end{bmatrix}\\
&=
\begin{bmatrix}
-\lambda_{1}^{n+1}(\lambda_{2}-\lambda_{1})^{-1} + \lambda_{2}^{n+1}(\lambda_{2}-\lambda_{1})^{-1}
& \lambda_{1}^{n+1}(1+(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1}) - \lambda_{2}^{n+1}(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1} \\
-\lambda_{1}^{n}(\lambda_{2}-\lambda_{1})^{-1} + \lambda_{2}^{n}(\lambda_{2}-\lambda_{1})^{-1}
& \lambda_{1}^{n}(1+(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1}) - \lambda_{2}^{n}(\lambda_{1})(\lambda_{2}-\lambda_{1})^{-1} \\
\end{bmatrix}\\
&=
\begin{bmatrix}
\frac{\lambda_{1}^{n+1} - \lambda_{2}^{n+1}}{\lambda_{1}-\lambda_{2}}
& \lambda_{1}^{n+1}(\frac{\lambda_{2}}{\lambda_{2}-\lambda_{1}}) - \lambda_{2}^{n+1}(\frac{\lambda_{1}}{\lambda_{2}-\lambda_{1}}) \\
 & \\
\frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}
& \lambda_{1}^{n}(\frac{\lambda_{2}}{\lambda_{2}-\lambda_{1}}) - \lambda_{2}^{n}(\frac{\lambda_{1}}{\lambda_{2}-\lambda_{1}}) \\
\end{bmatrix}\\
&=
\begin{bmatrix}
\frac{\lambda_{1}^{n+1} - \lambda_{2}^{n+1}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{1}^{n+1}\lambda_{2}-\lambda_{2}^{n+1}\lambda_{1}}{\lambda_{2}-\lambda_{1}}\\
 & \\
\frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{1}^{n}\lambda_{2}-\lambda_{2}^{n}\lambda_{1}}{\lambda_{2}-\lambda_{1}}\\
\end{bmatrix}\\
&=
\begin{bmatrix}
\frac{\lambda_{1}^{n+1} - \lambda_{2}^{n+1}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{2}^{n+1}\lambda_{1}-\lambda_{1}^{n+1}\lambda_{2}}{\lambda_{1}-\lambda_{2}}\\
 & \\
\frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{2}^{n}\lambda_{1}-\lambda_{1}^{n}\lambda_{2}}{\lambda_{1}-\lambda_{2}}\\
\end{bmatrix}
\end{align*}

We note that $\lambda_{1}\lambda_{2}=-1$, and therefore the last step simplifies to:

\begin{align*}
\vec{C}^{n}
&=
\begin{bmatrix}
\frac{\lambda_{1}^{n+1} - \lambda_{2}^{n+1}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}\\
 & \\
\frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{1}^{n-1} - \lambda_{2}^{n-1}}{\lambda_{1}-\lambda_{2}} \\
\end{bmatrix}
\end{align*}

We can now simplify our previous steps to get that:

\begin{align*}
\vec{y_{n}}
&=
\begin{bmatrix}
y_{n+1} \\
y_{n}
\end{bmatrix}\\
&=
\vec{C}^{n}
\begin{bmatrix}
y_{1} \\
y_{0}
\end{bmatrix}\\
&=
\begin{bmatrix}
\frac{\lambda_{1}^{n+1} - \lambda_{2}^{n+1}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}\\
 & \\
\frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}
& \frac{\lambda_{1}^{n-1} - \lambda_{2}^{n-1}}{\lambda_{1}-\lambda_{2}} \\
\end{bmatrix}
\begin{bmatrix}
1 \\
0
\end{bmatrix}\\
&=
\begin{bmatrix}
\frac{\lambda_{1}^{n+1} - \lambda_{2}^{n+1}}{\lambda_{1}-\lambda_{2}} \\
\\
\frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}
\end{bmatrix}
\end{align*}

This implies that
$y_{n}=\frac{\lambda_{1}^{n} - \lambda_{2}^{n}}{\lambda_{1}-\lambda_{2}}=\left(\frac{1}{\sqrt{5}}\right)\lambda_{1}^{n}+\left(-\frac{1}{\sqrt{5}}\right)\lambda_{2}^{n}$.
A similar argument can be applied to recursions of the form
$R(n)=\sum_{i=1}^{k}c_{i}R(n-i)$ where $c_{i}$ is a constant.

\section{Limitations}
This method has limitations. It cannot be applied to recursions of the form
$R(n)=\sum_{i=1}^{k}c_{i}R(n-i)$ where $c_{i}$ is not a constant. A
simple example would be $R(n)=nR(n-1), R(1)=1$. It is quite visible from
telescoping that $R(n)=(n!)$. However, one would not be able to put this into
any simpler form using this method since computing
$\vec{C}^n=\prod_{i=1}^{n}\begin{bmatrix}i\end{bmatrix}=n! $.

\begin{thebibliography}{9}
%The \bibitem is to start a new reference.  Ensure that the cite_key is
%unique.  You don't need to put each element on a new line, but I did
%simply for readability.

\textcolor{red}{WIKIPEDIA}

\end{thebibliography} %Must end the environment

\end{document}  %End of document.

Solving Linear Relations using Linear Algebra/paper.tex


\subsection{Hybrid orbital representation}
\label{sec:spo_hybrid}
The hybrid representation of the single particle orbitals combines a localized atomic basis set around atomic cores and B-splines in the interstitial regions to reduce memory use while retaining high evaluation speed and either retaining or increasing overall accuracy. Full details are provided in Ref.~\cite{Luo2018hyb}, and \textbf{users of this feature are kindly requested to cite this paper}.
In practice, we have seen that using a meshfactor=0.5 is often possible and achieves huge memory savings.
Figure~\ref{fig:hybridrep} illustrates how the regions are assigned. Orbitals within region A are computed as
\[
  \phi^A_n({\bf r})=R_{n,l,m}(r)Y_{l,m}(\hat{r})
\]
Orbitals in region C are computed as the regular B-spline basis described in subsection~\ref{sec:spo_spline} above. The region B interpolates between A and C as
\begin{align}
  \phi^B_n({\bf r}) &= S(r) \phi^A_n({\bf r}) + (1-S(r))\phi^C_n({\bf r}) .\\
                S(r) &= \frac{1}{2}-\frac{1}{2}\tanh\left[\alpha\left(\frac{r-r_{\rm A/B}}{r_{\rm B/C}-r_{\rm A/B}}-\frac{1}{2}\right)\right] .
\end{align}

\begin{figure}
\centering
\includegraphics[trim={0 152 0 0},clip,width=0.45\columnwidth]{./figures/hybrid_new.jpg}
\qquad
\includegraphics[trim={0 2 0 150},clip,width=0.45\columnwidth]{./figures/hybrid_new.jpg}
\caption{Regular and hybrid orbital representation. Regular B-spline representation (left panel) contains only one region and a sufficiently fine mesh to resolve orbitals near the nucleus. The hybrid orbital representation (right panel) contains near nucleus regions (A) where spherical harmonics and radial functions are used, buffers or interpolation regions (B), and an interstitial region (C) where a coarse B-spline mesh is used.}
\label{fig:hybridrep}
\end{figure}

To enable hybrid orbital representation, the input XML needs to see the tag \ixml{hybridrep="yes"} shown in Listing 8.6.
\begin{lstlisting}[style=QMCPXML,caption=Hybrid orbital representation input example.\label{listing:hybridrep}]
<determinantset type="bspline" source="i" href="pwscf.h5"
                tilematrix="1 1 3 1 2 -1 -2 1 0" twistnum="-1" gpu="yes" meshfactor="0.8"
                twist="0  0  0" precision="single" hybridrep="yes">
  ...
</determinantset>
\end{lstlisting}
Second, the information describing the atomic regions is required in the particle set, shown in Listing 8.7.
\begin{lstlisting}[style=QMCPXML,caption=particleset elements for ions with information needed by hybrid orbital representation.\label{listing:hybridrep_particleset}]
  <group name="Ni">
    <parameter name="charge">          18 </parameter>
    <parameter name="valence">         18 </parameter>
    <parameter name="atomicnumber" >   28 </parameter>
    <parameter name="cutoff_radius" > 1.6 </parameter>
    <parameter name="inner_cutoff" >  1.3 </parameter>
    <parameter name="lmax" >            5 </parameter>
    <parameter name="spline_radius" > 1.8 </parameter>
    <parameter name="spline_npoints">  91 </parameter>
  </group>
\end{lstlisting}

The parameters specific to hybrid representation are listed as
\begin{table}[h]
\centering
\begin{tabularx}{\textwidth}{l l l l l l }
\hline
\multicolumn{6}{l}{\texttt{attrib} element} \\
\hline
\multicolumn{2}{l}{Attribute:} & \multicolumn{4}{l}{}\\
   &   \bfseries Name            & \bfseries Datatype & \bfseries Values & \bfseries Default   & \bfseries Description \\
   &   \texttt{cutoff\_radius}             &  Real            &  $>=0.0$    &  \textit{None}    & Cutoff radius for B/C boundary  \\
   &   \texttt{lmax}         &  Integer            &  $>=0$ &  \textit{None} & Largest angular channel \\
   &   \texttt{inner\_cutoff}             &  Real            &  $>=0.0$    &  Dep.    & Cutoff radius for A/B boundary  \\
   &   \texttt{spline\_radius}         &  Real            &  $>0.0$ &  Dep. & Radial function radius used in spline \\
   &   \texttt{spline\_npoints}         &  Integer            &  $>0$ &  Dep. & Number of spline knots \\
  \hline
\end{tabularx}
\end{table}

\begin{itemize}
  \item \texttt{cutoff\_radius} is required for every species. If a species is intended to not be covered by atomic regions, setting the value 0.0 will put default values for all the reset parameters. A good value is usually a bit larger than the core radius listed in the pseudopotential file. After a parametric scan, pick the one from the flat energy region with the smallest variance.
  \item \texttt{lmax} is required if $\texttt{cutoff\_radius} > 0.0$. This value usually needs to be at least the highest angular momentum plus 2.
  \item \texttt{inner\_cutoff} is optional and set as $\texttt{cutoff\_radius}-0.3$ by default, which is fine in most cases.
  \item \texttt{spline\_radius} and \texttt{spline\_npoints} are optional. By default, they are calculated based on \texttt{cutoff\_radius} and a grid displacement $0.02$\,bohr.
        If users prefer inputing them, it is required that $\texttt{cutoff\_radius}<=\texttt{spline\_radius}-2\times\texttt{spline\_radius}/(\texttt{spline\_npoints}-1)$.
\end{itemize}

In addition, the hybrid orbital representation allows extra optimization to speed up the nonlocal pseudopotential evaluation using the batched algorithm listed in Section~\ref{sec:nlpp}.


% This is part of the TFTB Reference Manual.
% Copyright (C) 1996 CNRS (France) and Rice University (US).
% See the file refguide.tex for copying conditions.

\markright{tfrbj}
\section*{\hspace*{-1.6cm} tfrbj}

\vspace*{-.4cm}
\hspace*{-1.6cm}\rule[0in]{16.5cm}{.02cm}
\vspace*{.2cm}

{\bf \large \sf Purpose}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
Born-Jordan time-frequency distribution.
\end{minipage}
\vspace*{.3cm}

{\bf \large \sf Synopsis}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
[tfr,t,f] = tfrbj(x)
[tfr,t,f] = tfrbj(x,t)
[tfr,t,f] = tfrbj(x,t,N)
[tfr,t,f] = tfrbj(x,t,N,g)
[tfr,t,f] = tfrbj(x,t,N,g,h)
[tfr,t,f] = tfrbj(x,t,N,g,h,trace)
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Description}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
        {\ty tfrbj} computes the Born-Jordan distribution of a
        discrete-time signal {\ty x}, or the cross Born-Jordan
        representation between two signals. This distribution has the
        following expression :

\[BJ_x(t,\nu)=\int_{-\infty}^{+\infty} \frac{1}{|\tau|}\
\int_{t-|\tau|/2}^{t+|\tau|/2} x(s+\tau/2)\ x^*(s-\tau/2)\ ds\ e^{-j2\pi
\nu \tau} d\tau.\]

\hspace*{-.5cm}\begin{tabular*}{14cm}{p{1.5cm} p{8cm} c}
Name & Description & Default value\\
\hline
        {\ty x}     & signal if auto-BJ, or {\ty [x1,x2]} if cross-BJ. {\ty
			Nx=length(x)}\\
        {\ty t}     & time instant(s)          & {\ty (1:Nx)}\\
        {\ty N}     & number of frequency bins & {\ty Nx}\\
        {\ty g}     & time smoothing window with odd length, {\ty g(0)}
			being forced to {\ty 1}
                                         & {\ty window(odd(N/10))}\\
        {\ty h}     & frequency smoothing window with odd length, {\ty
			h(0)} being forced to {\ty 1}
                                         & {\ty window(odd(N/4))}\\
        {\ty trace} & if nonzero, the progression of the algorithm is shown
                                         & {\ty 0}\\
     \hline {\ty tfr}   & time-frequency representation\\
        {\ty f}     & vector of normalized frequencies\\

\hline
\end{tabular*}
\vspace*{.2cm}

When called without output arguments, {\ty tfrbj} runs {\ty tfrqview}.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Example}
\begin{verbatim}
         sig=fmlin(128,0.05,0.3)+fmlin(128,0.15,0.4);
         g=window(9,'Kaiser'); h=window(27,'Kaiser');
         t=1:128; tfrbj(sig,t,128,g,h,1);
\end{verbatim}
\vspace*{.5cm}

{\bf \large \sf See Also}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
all the {\ty tfr*} functions.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Reference}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
[1] L. Cohen ``Generalized Phase-Space Distribution Functions'',
J. Math. Phys., Vol. 7, No. 5, pp. 781-786, 1966.
\end{minipage}


% LaTeX template for MLSP papers. To be used with:
%   * mlspconf.sty - ICASSP/ICIP LaTeX style file adapted for MLSP, and
%   * IEEEbib.bst - IEEE bibliography style file.
% --------------------------------------------------------------------------
\documentclass{article}
\usepackage{amsmath,graphicx,mlspconf}

% Copyright notices.
% ------------------
% Select one of the four copyright notices below. Only required for the camera-ready paper submission.
%
% * For papers in which all authors are employed by the US government:

% Header
\toappear{EECE5644 Final Project Presentations, Dec.\ 9, 2020, Northeastern University}

% Example definitions.
% --------------------
%\def\x{{\mathbf x}}
%\def\L{{\cal L}}

% Title.
% ------
\title{ANALYSIS OF KALMAN FILTER PERFORMANCE ON LINEAR AND NON-LINEAR VEHICULAR MOTION}
%
% Double-blind peer review.
% -------------------------
% Anonymize your paper for the double-blind peer-review process using the
% following author and affiliation.
\name{%
    Nathaniel Gordon
    \qquad Eric Grimaldi
    \qquad Siddhant Gupta
}
\address{
    Northeastern University
}

\begin{document}
%\ninept

\maketitle

\begin{abstract}

With the growing prevalence of autonomous vehicles, the need for methods to predict and model the motion of said vehicles is ever increasing. One such method for trajectory estimation is the Kalman filter. In this project, we derive and implement two versions of a Kalman filter that are able to harness RADAR and LIDAR data of a simulated autonomous vehicle to predict the vehicle’s motion with an uncertainty of 4cm. This project demonstrates the effectiveness of the linear and lxtended variations of the Kalman filter, particularly when used in conjunction.

\end{abstract}

\section{Introduction}
\label{sec:intro}

Before proceeding to the applied Kalman filters, the algorithm will be defined in a general sense. The Kalman filter is built around two fundamental phases: the prediction phase, in which a state estimation is produced, and an update phase, during which uncertainty in the state space is propagated for use in the next iteration of the algorithm. The prediction phase can be defined as follows:

$$x' = Fx+u$$

Here, the state space and its projected prediction are denoted $x$ and $x'$ respectively, and $u$ is the control term. $F$ is a representation of the dynamics of the object's motion.

The update phase, in which the future state uncertainty $P'$ is determined, can be represented as a function of the prior uncertainty and measurement noise $Q$:

$$P'=FPF^T+Q$$

Once again, the dynamics matrix $F$ is present. Defining this quantity will be a crucial part of the implementation step.

\section{METHODS}
\label{sec:methods}

\subsection{Linear Kalman Filter Implementation}
\label{ssec:lkfimp}

First, the object dynamics for the autonomous vehicle must be discussed. Two values are pertinent to the prediction of the vehicle's motion: position and velocity. For the linear Kalman filter, linear motion will be assumed:

$$
F = \begin{cases}
p_x'=p_x+v_x\Delta t\\
p_y'=p_y+v_y\Delta t\\
v_x'=v_x\\
v_y'=v_y
\end{cases}
$$

Because there is no aspect of control in this application, we can then represent the prediction equation as:

$$
\begin{pmatrix}
p_x'\\
p_y'\\
v_x'\\
v_y'
\end{pmatrix}=
\begin{pmatrix}
1 && 0 && \Delta t && 0\\
0 && 1 && 0 && \Delta t\\
0 && 0 && 1 && 0\\
0 && 0 && 0 && 1
\end{pmatrix}
\begin{pmatrix}
p_x\\
p_y\\
v_x\\
v_y
\end{pmatrix}
$$

Now, the data may be applied to the algorithm. The data used for this project were borrowed from \cite{gaskey19}. One source of the measurements present in the data is LIDAR. These readings contain a measurement of the vehicle's position in Cartesian space. Let these readings be denoted $z$, the measurement vector:

$$
z = \begin{pmatrix}
p_x\\
p_y
\end{pmatrix}
$$

Because we are unable to directly reason about the vehicle's velocity from the LIDAR input, we will define a mask function $H$ to apply the LIDAR data to only the positional aspects of the state vector:

$$
H = \begin{pmatrix}
1 && 0 && 0 && 0\\
0 && 1 && 0 && 0
\end{pmatrix}
$$

Additionally, the instrument error for the LIDAR readings is 2.25 centimeters.

With these values defined, the update process for the LIDAR readings becomes the following:

$$P'=HPH^T+Q$$

The error, $y$, is the difference between the true reading and the masked projection:

$$y=z-Hx'$$

\begin{figure}[ht]
\begin{minipage}[b]{1.0\linewidth}
  \centering
  \centerline{\includegraphics[width=8.5cm]{1.png}}
\end{minipage}
\caption{Positional estimates generated from linear Kalman filter and LIDAR data.}
\end{figure}

Finally, the Kalman gain, $K$, which describes the relative weight placed on the recent measurements (as opposed to older ones), can be derived as thus:

$$K=P'H^TP^I$$

\subsection{Extended Kalman Filter Implementation}
\label{ssec:ekfimp}

The other source of readings in the data is a RADAR system. These readings are given in polar form, and are a description of the vehicle's range $\rho$, perpendicular velocity $\dot{\rho}$, and bearing $\phi$. To convert to Cartesian form, the following transformation is applied:

\begin{figure}[hb]
\begin{minipage}[b]{1.0\linewidth}
  \centering
  \centerline{\includegraphics[width=8.5cm]{2.png}}
\end{minipage}
\caption{Error (Euclidean distance) in linear Kalman filter estimation.}
\end{figure}

$$
h(x')=\begin{pmatrix}
\sqrt{p_x'^2+[_y'^2}\\
arctan(p_y'/p_x')\\
\frac{p_x'v_x'+p_y'v_y'}{\sqrt{p_x'^2+p_y'^2}}
\end{pmatrix}
$$

This transformation raises a pertinent issue: it introduces non-linearity into the system. To handle this linearity, the extended Kalman filter methodology is employed.

To adapt the algorithm to account for this non-linearity, the Jacobian of the transformation matrix $h_j$ must be calculated, as derived by \cite{gaskey19}:

$$
h_j=
$$$$\begin{bmatrix}
\frac{p_x}{\sqrt{p_x^2+p_y^2}} && \frac{p_y}{\sqrt{p_x^2+p_y^2}} && 0 && 0\\
-\frac{p_y}{p_x^2+p_y^2} && \frac{p_x}{p_x^2+p_y^2} && 0 && 0\\
\frac{p_y(v_xp_y-v_yp_x)}{(p_x^2+p_y^2)^{3/2}}&& \frac{p_y(v_yp_x-v_xp_y)}{(p_x^2+p_y^2)^{3/2}} && \frac{p_x}{\sqrt{p_x^2+p_y^2}} && \frac{p_y}{\sqrt{p_x^2+p_y^2}}
\end{bmatrix}$$

The rest of the process of constructing the extended Kalman filter is identical to the methods in the previous section, where the linear variant was defined.

\section{RESULTS}
\label{sec:results}

\subsection{Linear Kalman Filter Results}
\label{ssec:lkfresults}

To test the performance of the Kalman filter algorithms, a simplified trial using only LIDAR data was run to isolate the linear Kalman filter algorithm. This meant that while the procedure was unable to produce an estimate of the vehicle's velocity, it could predict its position. The trial contained 250 LIDAR readings of the vehicle traveling in a figure-eight pattern.

\begin{figure}[hb]
\begin{minipage}[b]{1.0\linewidth}
  \centering
  \centerline{\includegraphics[width=8.5cm]{3.png}}
\end{minipage}
\caption{Compiled LIDAR and RADAR readings from the dataset. The orientation of the triangle markers corresponds to the vehicle's measured bearing.}
\end{figure}

The results of this trial are visible in Figure 1, where the predicted positions are plotted alongside the corresponding true values. Additionally, in Figure 2, the error, found with Euclidean distance from the true value. For this trial, the empirically determined root mean squared error was found to be .21 meters.

While the linear Kalman filter generates accurate predictions of position, it is unable to capture the full state of the vehicle in motion. Thus, subsequent trials were run to determine if using pure RADAR data, or a mix of the two data streams, would improve the prediction quality.

\subsection{Extended Kalman Filter Results}
\label{ssec:ekfresults}

After the linear Kalman filter had been tested independently, a second trial was run, using only the RADAR data from the autonomous vehicle simulation. In this trial, the procedure was able to estimate the complete state vector, capturing both the position and velocity of the vehicle in the prediction. The results of this trial are visible in Figure 4.

Finally, a trial was run where both Kalman filters were run in tandem, analyzing both LIDAR and RADAR data. In estimating the vehicle's motion, the linear Kalman filter serves as the more accurate arbiter of positional data, as it draws from the less noisy LIDAR instrument. However, this model also has the benefit of ascertaining velocity information from the extended Kalman filter. While drawing on the RADAR data is necessary for projecting the vehicle's future state, the graph in Figure 5 demonstrates how the positional data gathered from the extended Kalman filter is much less accurate due to the RADAR's lesser precision. In conjunction, the two filters provide a robust estimation of vehicle movement, drawing from both linear and non-linear data streams.

\begin{figure}[hb]
\begin{minipage}[b]{1.0\linewidth}
  \centering
  \centerline{\includegraphics[width=8.5cm]{4.png}}
\end{minipage}
\caption{Predicted vehicle states collected from using only the extended Kalman filter on the RADAR data.}
\end{figure}

\begin{figure}[ht]
\begin{minipage}[b]{1.0\linewidth}
  \centering
  \centerline{\includegraphics[width=8.5cm]{5.png}}
\end{minipage}
\caption{Overlaid results of drawing predictions from both linear and extended Kalman filters. The state predictions draw on both LIDAR and RADAR data.}
\end{figure}

\section{CONCLUSION}
\label{sec:conclusion}

The outcomes of the three trials indicate that the Kalman filter is a highly flexible tool, and can be used in tandem with other filters to process data from multiple sources. Furthermore, the uncertainty measurement the filter maintains enables it to rely on measurements of different inputs for parts of the state vector, minimizing overall error in the output.

An future extension to this project could entail observing the behavior of the system with changes to the degree of noise in each measuring instrument. Alternatively, the use of an unscented Kalman filter for processing the non-linear data could be investigated.

This project served as an excellent opportunity for all three group members to gain experience implementing and testing the filtering techniques demonstrated in this report. The source code for this report can be found at \cite{gordon20}.

% To start a new column (but not a new page) and help balance the last-page
% column length use \vfill\pagebreak.
% -------------------------------------------------------------------------
%\vfill
%\pagebreak

% References should be produced using the bibtex program from suitable
% BiBTeX files (here: strings, refs, manuals). The IEEEbib.bst bibliography
% style file from IEEE produces unsorted bibliography list.
% -------------------------------------------------------------------------
\bibliographystyle{IEEEbib}
\bibliography{refs}

\end{document}


\chapter{Proof of the Linked-Diagram Theorem}\label{app:linked-diagram-theorem}

\begin{ntt}\label{ntt:operator-combinations}
Let
``$Y^m$ choose $Z^k$'', denoted ${}^mC_k(Y:Z)$,
refer to a sum over the $m$ choose $k$ permutations of $Y^{m-k}Z^k$,\,\footnote{For example,
$
  {}^4C_2(Y:Z)
=
  Y^2Z^2
+
  YZYZ
+
  YZ^2Y
+
  ZY^2Z
+
  ZYZY
+
  Z^2Y^2
$.
}
where $Y$ and $Z$ are operators that may or may not commute.\,\footnote{
  If they do commute, then ${}^mC_k(Y:Z)={n\choose k}Y^{m-k}Z^k$.
}
This defines a generalization of the binomial theorem.
\begin{align}
\label{eq:generalized-binomial-theorem}
  (
    Y
  +
    Z
  )^m
=
  \sum_{k=0}^m
  {}^mC_k(Y:Z)
\end{align}
Furthermore, let
$
  {}^mC(Y:Z_1,\ld,Z_k)
$
be a sum over permutations of
$
  Y^{m-k}
  Z_1\cd Z_k
$ that preserve the ordering of the $Z_i$'s.\,\footnote{
  For example,
$
  {}^4C(Y:Z_1,Z_2)
=
  Y^2Z_1Z_2
+
  YZ_1YZ_2
+
  YZ_1Z_2Y
+
  Z_1Y^2Z_2
+
  Z_1YZ_2Y
+
  Z_1Z_2Y^2
$.
}
When all of the $Z_i$'s equal $Z$, we can write
$
  {}^mC(Y:Z_1,\ld,Z_k)
=
  {}^mC_k(Y:Z)
$.
\end{ntt}

\begin{prop}
\label{prop:wavefunction-infinite-recursion}
\thmstatement{
$\ds{
  \Y(\la)
=
  \sum_{m=0}^\infty
  \pr{
    R_0
    (\la V_{\text{c}} - E(\la))
  }^m
  \F
}$
}
\thmproof{
  This follows by infinite recursion of equation~\ref{eq:lambda-dependent-recursive-series} with the assumption
  $\ds{
  \lim_{m\rightarrow\infty}
  \pr{
    R_0
    (\la V_{\text{c}} - E(\la))
  }^m
  \Y(\la)
  =
    0
  }$.
}
\end{prop}

\begin{dfn}\label{dfn:integer-compositions}
\thmtitle{Integer compositions}
The \textit{compositions} of an integer $m$ are the ways of writing $m$ as a sum of positive integers.
The full set of integer compositions of $m$ is given by
$
  \mc{C}(m)
=
  \mc{C}_1(m)
  \cup
  \mc{C}_2(m)
  \cup
  \cd
  \cup
  \mc{C}_m(m)
$
where
$
  \mc{C}_k(m)
=
  \{
    (r_1,\ld,r_k)\in\mb{N}_0^k
  \,|\,
    r_1+\cd+r_k
  =
    m
  \}
$
are the integer compositions of $m$ into $k$ parts.
\end{dfn}

\begin{lem}
\label{lem:energy-substitution-proof}
\thmtitle{The Energy Substitution Lemma}
\thmstatement{
  $\Y\ord{m}$ equals the sum of a ``principal term''
  $(R_0V_{\text{c}})^m\F$
  plus all possible substitutions of adjacent factors $(R_0V_{\text{c}})^{r_i}$ in the principal term by $R_0E_{\text{c}}\ord{r_i}$.
  Each term in the sum is weighted by a sign factor $(-)^k$, where $k$ is the number of substitutions.
}\vspace{5pt}
\thmproof{
Using equation~\ref{eq:generalized-binomial-theorem} and a double sum identity\footnote{
  Reverse double-sum reduction:
  $\ds{
    \sum_{m=0}^\infty
    \sum_{k=0}^m
    t_{m-k,k}
  =
    \sum_{k'=0}^\infty
    \sum_{k=0}^\infty
    t_{k',k}
  }$.
  See
  \url{http://functions.wolfram.com/GeneralIdentities/12/}.
} in the infinite recursion formula for $\Y(\la)$ gives the following.
{\footnotesize
\begin{align*}
  \Y(\la)
=
  \sum_{m=0}^\infty
  (
    R_0
    (
      \la V_{\text{c}}
    -
      E(\la)
    )
  )^m
  \F
=
  \sum_{m=0}^\infty
  \sum_{k=0}^m
  \la^{m-k}
  (-)^k\,\,
  {}^mC_k
  (
    R_0V_{\text{c}}:
    R_0E(\la)
  )
  \F
=
  \sum_{k'=0}^\infty
  \sum_{k=0}^\infty
  \la^{k'}
  (-)^{k}\,\,
  {}^{k'+k}\hspace{-1pt}C_k
  (
    R_0V_{\text{c}}:
    R_0E(\la)
  )
  \F
\end{align*}}%
The $k'=0$ term has no operators separating $\F$ from the resolvent and vanishes.
Taylor expansion of the energies gives
{\footnotesize
\begin{align*}
  \Y(\la)
=&\
  \sum_{k=0}^\infty
  \sum_{k'=1}^\infty
  {\sum_{p_1=1}^\infty}
  \cd
  {\sum_{p_k=1}^\infty}
  \la^{k' + p_1 + \cd + p_k}
  (-)^{k}\,\,
  {}^{k'+k}\hspace{-1pt}C
  (
    R_0V_{\text{c}}:
    R_0E_{\text{c}}\ord{p_1},\ld,
    R_0E_{\text{c}}\ord{p_k}
  )
  \F
\\
=&\
  \sum_{m=1}^\infty
  \sum_{k=0}^{m-1}
  \sum_{(r_1,\ld,r_{k+1})}^{\mc{C}_{k+1}(m)}
  \la^m
  (-)^k\,\,
  {}^{k+r_1}\hspace{-1pt}C
  (
    R_0V_{\text{c}}:
    R_0E_{\text{c}}\ord{r_2},\ld,
    R_0E_{\text{c}}\ord{r_{k+1}}
  )
  \F
\end{align*}}%
where we have grouped powers of $\la$ using a multi-sum reduction.
Writing the inner sums as a sum over $\mc{C}(m)$ we find
\begin{align}
  \Y\ord{m}
=
  \left.
  \fr{1}{m!}
  \pd{^m\Y(\la)}{\la^m}
  \right|_{\la=0}
=
  \sum_{(r_1,\ld,r_{k+1})}^{\mc{C}(m)}
  (-)^k\,\,
  {}^{k+r_1}\hspace{-1pt}C
  (
    R_0V_{\text{c}}:
    R_0E_{\text{c}}\ord{r_2},\ld,
    R_0E_{\text{c}}\ord{r_{k+1}}
  )
  \F
\end{align}
which, given \cref{ntt:operator-combinations} and definition~\ref{dfn:integer-compositions}, is an algebraic statement of the proposition, completing the proof.
}
\end{lem}

\begin{thm}
\thmtitle{The Bracketing Theorem}
\thmstatement{
  $\Y\ord{m}$ equals the principal term plus all possible insertions of nested brackets into the principal term.
  Each term in the sum is weighted by $(-)^k$ where $k$ is the total number of brackets.
}\vspace{5pt}
\thmproof{
  The proposition holds for $m=1$ because
  $\Y\ord{1}=R_0V_{\text{c}}\F$ and there are no possible bracketings.
  Assume it holds for $m-1$.
  Then by the energy substitution lemma it also holds for $m$ because $E_{\text{c}}\ord{r_i}$
  equals
  $\ip{\F|V_{\text{c}}|\Y\ord{r_i}}$
  which, by our inductive assumption, equals
  $\ip{V_{\text{c}}(R_0V_{\text{c}})^{r_i}}$
  plus all nested bracketings weighted by appropriate sign factors.
}
\end{thm}


\documentclass[a4paper,12pt]{article}

\usepackage{ucs}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage[english]{babel}
\usepackage[T1]{fontenc}
\usepackage[pdftex]{graphicx}
\usepackage[margin=2.5cm]{geometry}
\usepackage{listings}
\usepackage{color}

\usepackage[pdftex]{hyperref}

% Default fixed font does not support bold face
\DeclareFixedFont{\ttb}{T1}{txtt}{bx}{n}{12} % for bold
\DeclareFixedFont{\ttm}{T1}{txtt}{m}{n}{12}  % for normal

\definecolor{deepblue}{rgb}{0,0,0.5}
\definecolor{deepred}{rgb}{0.6,0,0}
\definecolor{deepgreen}{rgb}{0,0.5,0}

% Python style for highlighting
\newcommand\pythonstyle{\lstset{
language=Python,
basicstyle=\ttm,
otherkeywords={self},             % Add keywords here
keywordstyle=\ttb\color{deepblue},
emph={MyClass,__init__},          % Custom highlighting
emphstyle=\ttb\color{deepred},    % Custom highlighting style
stringstyle=\color{deepgreen},
frame=tb,                         % Any extra options here
showstringspaces=false            %
}}

% Python environment
\lstnewenvironment{python}[1][]
{
\pythonstyle
\lstset{#1}
}
{}

% Python for external files
\newcommand\pythonexternal[2][]{{
\pythonstyle
\lstinputlisting[#1]{#2}}}

% Python for inline
\newcommand\pythoninline[1]{{\pythonstyle\lstinline!#1!}}

\title{Point estimation}
\author{}
\date{}

\begin{document}
 \maketitle

 \section{Bayesian inference: Poisson}

 We come back to the example seen in class: $W^\pm W^\pm jj$ from ATLAS (PRL 113, 141803 (2014)).
 \begin{itemize}
  \item $D = 12$ observed events
  \item $B = 3.0 \pm 0.6$ background events (Poisson distributed, estimated from a control sample)
 \end{itemize}

 The unknowns are the expected background count $b$ and the expected signal count $s$. The expected event count is $d = b+s$.

 \begin{enumerate}
  \item Write the likelihood $\mathcal{L}(D | s,b)$.
  \item What is the MLE $\hat{s}_\text{MLE}$? Draw the profile likelihood $-2\ln\mathcal{L}_\text{prof}(D|s)$ (e.g. using a \texttt{TGraph}, or with any other tool).
  \item Draw the (marginalised) posterior for $s$, for each of the following priors for $s$ (and a flat prior for $b$). You will perform the marginalisation using the MCMC method, also
  drawing the Markov chain in one of the cases.
  \begin{enumerate}
   \item using a flat prior in $[0,+\infty[$
   \item using Jeffrey's prior for a Poisson process without background
   \item using Jeffrey's prior for a Poisson process with background?
  \end{enumerate}
 \end{enumerate}

 \section{Straight line tracking}

 \begin{center}
  \includegraphics[width=0.5\textwidth]{../figures/BLUE/fig2_a}
 \end{center}

 We consider a tracking detector without magnetic field (particle trajectories are straight lines $y = a + bx$). The detector is made of 2 tracking stations, each with 3 layers,
 situated at $x_{11} = 10$, $x_{12} = 11$, $x_{13} = 12$; $x_{21} = 20$, $x_{22} = 21$, $x_{23} = 22$ (distances in mm). We want to measure $a$, the impact parameter of the particle with respect to the origin, while we are not interested in $b$ (it is a ``nuisance parameter''). Each layer has a resolution of 1\,mm: in other words, each measurement $y_i$ is a random variable
 with a Gaussian distribution of mean $a + bx_i$ and variance 1.

 In the following we assume the true values of $a$ and $b$ to be $a_0=3$\,mm and $b_0=0.1$. The measured values are $y_{11} = 4.0$, $y_{12} = 3.8$, $y_{13} = 3.6$, $y_{21} = 5.2$, $y_{22} = 4.9$, $y_{23} = 4.8$\,mm.

 \begin{enumerate}
  \item Give the expression for the ML estimates $\hat{a}_\text{MLE}$ and $\hat{b}_\text{MLE}$. What is the corresponding covariance matrix? Compare the numerical value of $V(\hat{a})$
  when only the 3 layers of the first station are used, or those of the second station, or all 3 stations.
  \item Draw $-2\ln\mathcal{L}(a,\hat{b})$ as a function of $a$ for the 3 first layers, the 3 last layers, and the 6 layers. Draw them together with the profile likelihood
  $-2\ln\mathcal{L}_\text{prof}(a) = -2\ln\mathcal{L}(a,\hat{\hat{b}}(a))$. You can use \texttt{TGraph} for instance.
  \item An independent measurement adds the information that $\langle b \rangle = 0.1$, with a precision $\sigma_b = 0.05$. How to account for this information in the likelihood?
  Repeat the previous question (likelihood drawing) with the new likelihood.
  \item \textbf{Bayesian estimation.}
  \begin{enumerate}
    \item Assume a flat prior for $a$ and $b$. What is the posterior for $(a,b)$? What is the marginalised posterior for $a$? Give its mean and variance.
    \item What happens if we set the prior for $a$ to be 0 for $a<0$?
    \item Let's go back to a flat prior on $\mathbb{R}$ for $a$. How to account for $\langle b \rangle = 0.1$, $\sigma_b = 0.05$, in the prior for $b$?
    \item \textit{Numerical application}: perform the marginalisation on $b$ using the MCMC method, with and without the additional constraint. Draw the corresponding Markov chain
    (e.g. using a \texttt{TGraph}).
    Use it to draw the posterior for $(a,b)$ and the marginalised posterior for $a$. You could fill histograms (\texttt{TH1F}, \texttt{TH2F}) with the Markov chain.
  \end{enumerate}
  \item \textbf{Data combination.} Use the BLUE method to combine the estimates $\hat{a}_\text{MLE,1}$ and $\hat{a}_\text{MLE,2}$, with two different methods:
  \begin{enumerate}
   \item Combining simultaneously $a$ and $b$ (correlated measurements of several physical quantities).
   \item Combining only $a$, considering the uncertainty on $a$ coming from $b$ as correlated between the two stations.
  \end{enumerate}
 \end{enumerate}

 \section*{Technical tools}
 See \url{https://root.cern.ch/doc/master/}
 \begin{itemize}
  \item \texttt{TGraph}
  \begin{python}
import ROOT
from array import array
x = [1,2]
y = [3,4]
g = ROOT.TGraph(len(x), array('d',x), array('d',y))
c1 = ROOT.TCanvas()
g.Draw("AL")
c1.Draw()
  \end{python}
  \item \texttt{TH1F}
  \begin{python}
import ROOT
h = ROOT.TH1F("h","1D histo",10,0,10)
for i in range(0,100):
   h.Fill(ROOT.gRandom.Uniform(10))

c1 = ROOT.TCanvas()
h.Draw()
c1.Draw()
  \end{python}
  \item \texttt{TH2F}
  \begin{python}
import ROOT
h2 = ROOT.TH2F("h2","2D histo",10,0,10,10,0,10)
for i in range(0,100):
   h2.Fill(ROOT.gRandom.Uniform(10),ROOT.gRandom.Uniform(10))

c1 = ROOT.TCanvas()
h2.Draw("COLZ")
c1.Draw()
  \end{python}
 \end{itemize}

\end{document}