\documentclass[11pt]{scrartcl} % Font size
\input{../structure.tex} % Include the file specifying the document structure and custom commands

%----------------------------------------------------------------------------------------
%	TITLE SECTION
%----------------------------------------------------------------------------------------

\title{
	\normalfont\normalsize
	\textsc{Harvard Privacy Tools Project}\\ % Your university, school and/or department name(s)
	\vspace{25pt} % Whitespace
	\rule{\linewidth}{0.5pt}\\ % Thin top horizontal rule
	\vspace{20pt} % Whitespace
	{\huge Variance Sensitivity Proofs}\\ % The assignment title
	\vspace{12pt} % Whitespace
	\rule{\linewidth}{2pt}\\ % Thick bottom horizontal rule
	\vspace{12pt} % Whitespace
}

% \author{\LARGE} % Your name

\date{\normalsize\today} % Today's date (\today) or a custom date

\begin{document}

\maketitle

\begin{definition}
Let sample variance be defined as
$$ s^2(x) = \frac{1}{n-1} \sum_{i=1}^n (x_i - \mean)^2,$$
where $\mean$ refers to the sample mean of $x$.
\end{definition}

\section{Neighboring Definition: Change One}
\subsection{$\ell_1$-sensitivity}
\begin{lemma}
\label{lemma:meansum}
For arbitrary $a$,
$$ \sum_{i=1}^n (x_i - a)^2 = \sum_{i=1}^n (x_i - \bar{x})^2 + n(a-\bar{x})^2.$$
\end{lemma}

\begin{proof}
\begin{align*}
\sum_{i=1}^n (x_i - a)^2 &= \sum_{i=1}^n \left( (x_i - \bar{x}) - (a-\bar{x}) \right)^2\\
	&= \sum_{i=1}^n \left( (x_i - \bar{x})^2 -2(x_i - \bar{x})(a-\bar{x}) + (a-\bar{x})^2\right)\\
	&= \sum_{i=1}^n (x_i - \bar{x})^2 - 2\sum_{i=1}^n \left(x_ia-x_i\bar{x} -\bar{x}a + \bar{x}^2\right) + \sum_{i=1}^n \left( a^2 -2a\bar{x} + \bar{x}^2\right)\\
	&=  \sum_{i=1}^n (x_i - \bar{x})^2 -2a\sum_{i=1}^n x_i + 2\bar{x}\sum_{i=1}^n x_i + 2\bar{x}an - 2\bar{x}^2n + a^2n-2a\bar{x}n+\bar{x}^2n\\
	&=  \sum_{i=1}^n (x_i - \bar{x})^2 + a^2n-2a\bar{x}n+\bar{x}^2n\\
	&=  \sum_{i=1}^n (x_i - \bar{x})^2 + n(a-\bar{x})^2
\end{align*}

\end{proof}

\begin{theorem}
\label{thm:l1change}
Let
$$ f(\x) = \sum_{i=1}^n (x_i - \bar{x})^2.$$
Then for $\x$ bounded between $m$ and $M,$ $f(\cdot)$ has $\ell_1$-sensitivity in the change-one model bounded above by
$$\frac{n-1}{n} (M-m)^2.$$
\end{theorem}

\begin{proof}
Consider databases $\xprime$ and $\xprimeprime$ which differ in a single point. For notational ease, call $\x$ the part of $\xprime$ and $\xprimeprime$
that is the same, and say that $\x$ contains $n$ points. WLOG say that the last data point in the database is the one that differs. I.e.,
$\xprime = \x \cup \{x_{n+1}\},$ and $\xprimeprime = \x \cup \{x'_{n+1}\}$. This proof assumes that a ``neighboring database" is one that differs in a single
data-point, so we will ultimately be comparing $f(\xprime)$ and $f(\xprimeprime)$. However, it is useful to first write $f(\xprime)$ in terms of $f(\x)$.
Note that

\begin{align}
\label{eqn:meanprime}
\bar{x}' &= \frac{1}{n+1} \sum_{i=1}^{n+1} x_i \nonumber \\
	&= \frac{n\bar{x} + x_{n+1}}{n+1}.
\end{align}

Then,
\begin{align}
\label{eq:varAddOnePoint}
f(\xprime) &= \sum_{i=1}^n (x_i - \bar{x}')^2 + (x_{n+1} - \bar{x}')^2 \notag\\
	&= \sum_{i=1}^n (x_i - \bar{x})^2 + n(\bar{x}'-\bar{x})^2 + (x_{n+1} - \bar{x}')^2 &&\text{(By Lemma \ref{lemma:meansum})} \notag\\
	&= f(\x) + n\left( \frac{n\bar{x} + x_{n+1}}{n+1}-\bar{x}\right)^2 + \left(x_{n+1} - \frac{n\bar{x} + x_{n+1}}{n+1} \right)^2 &&\text{(By Equation \ref{eqn:meanprime})} \notag\\
	&= f(\x) + n\left(\frac{x_{n+1}-\bar{x}}{n+1}\right)^2 + \left( \frac{nx_{n+1}-n\bar{x}}{n+1}\right)^2 \notag\\
	&= f(\x) + (x_{n+1}-\bar{x})^2 \frac{n+n^2}{(n+1)^2} \notag\\
	&= f(\x) + (x_{n+1}-\bar{x})^2 \frac{n}{n+1} \notag\\
\end{align}

Now, to bound the sensitivity of $f$, note that

\begin{align}
	\label{eq:l1_np1_bound}
\left\vert f(\xprime) - f(\xprimeprime) \right\vert &= \left\vert (x_{n+1}-\bar{x})^2 \frac{n}{n+1} - (x_{n+1}'-\bar{x})^2 \frac{n}{n+1} \right\vert \\
	&\le (M-m)^2 \frac{n}{n+1}. \nonumber
\end{align}
The bound in the final line follows from the case where $x_{n+1} = M$ (resp. $m$) and $\bar{x} = x'_{n+1} = m$ (resp. $M$). \newline

So we have a bound on the sensitivity of $f(\cdot)$ for a data set of size $n+1$. Traditionally we consider sensitivities on a data set of size $n$.
Redefining $n+1$ as $n$ in the above equation gives
$$ (M-m)^2 \frac{n-1}{n}. $$
\end{proof}

\begin{corollary}
Variance has $\ell_1$-sensitivity in the change-one model bounded above by
$$ (M-m)^2 \frac{n-1}{n^2}. $$
\end{corollary}

\begin{proof}
This follows from Theorem \ref{thm:l1change} with a renormalization by $n$.
\end{proof}

\begin{corollary}
Sample variance has $\ell_1$-sensitivity in the change-one model bounded above by
$$ \frac{(M-m)^2 }{n}. $$
\end{corollary}

\begin{proof}
This follows from Theorem \ref{thm:l1change} with a renormalization by $n-1$.
\end{proof}

\subsection{$\ell_2$-sensitivity}
%\begin{theorem}
%\label{thm:l2change}
%	Let $X$ be a data set with $n$ elements, $x_1, \hdots, x_n$ and let
%	\[ f(X) = \sum_{i=1}^n (x_i - \bar{x})^2 \]
%	be the sample variance.
%	For $X$ bounded between $m$ and $M,$ $f(\cdot)$ has an $\ell_2$-sensitivity in the change-one model of
%	\[ \frac{n-1}{n} (M-m)^2 \]
%\end{theorem}

%\begin{proof}
%	Pick up from statement~\ref{eq:l1_np1_bound}, switching from an $\ell_1$ to an $\ell_2$ norm and interpret the data sets in question to be of size $n$ rather than $n+1$.
%%	\begin{align*}
%%		(f(x') - f(x''))^2
%%			&= \left( (x_{n}-\bar{x})^2 \frac{n-1}{n} - (x_{n}'-\bar{x})^2 \frac{n-1}{n} \right)^2 \\
%%			&= \left( \frac{n-1}{n} \right)^2 \left( (x_n - \bar{x})^2 - (x'_n - \bar{x})^2 \right)^2 \\
%%			&\leq \left( \frac{n-1}{n} \right)^2 \left( (M-m)^2 \right)^2 \\
%%			&= \left( \frac{n-1}{n} \right)^2 (M - m)^4 \\
%%			&= \left( \frac{n-1}{n} (M - m)^2 \right)^2.
%%	\end{align*}
%\end{proof}

\begin{corollary}
Variance has $\ell_2$-sensitivity in the change-one model bounded above by
$$ \frac{n-1}{n^2} (M-m)^2.$$
\end{corollary}

\begin{corollary}
Sample variance has $\ell_2$-sensitivity in the change-one model bounded above by
$$ \frac{1}{n} (M-m)^2.$$
\end{corollary}

%\begin{proof}
%This follows from Theorem \ref{thm:l2change} with a renormalization by $n-1$.
%\end{proof}

\section{Neighboring Definition: Add/Drop One}
\subsection{$\ell_1$-sensitivity}
 \begin{theorem}
 \label{thm:l1addsub}
 	Let $X$ be a data set with $n$ elements, $x_1, \hdots, x_n$ and
 	\[ f(X) = \sum_{i=1}^n (x_i - \bar{X})^2. \]
 	For $X$ bounded between $m$ and $M,$ $f(\cdot)$ has a global $\ell_1$-sensitivity in the add/drop one model of
 	\[ \frac{n}{n+1} (M-m)^2 \]
 \end{theorem}

 \begin{proof}
 	We must consider both adding and removing an element from $X$. \newline

 	Adding an element: \newline
 	Let $X' = X \cup x'_{n+1}$. Recall from Eq.~\ref{eq:varAddOnePoint} that for

	$$ f(x) = \sum_{i=1}^n (x_i - \bar{x})^2, $$
	$$ f(x') = f(x) + (x_{n+1} - \bar{x})^2 \frac{n}{n+1}.$$

	So,

	\begin{align}
	\label{eq:L1add}
	\left\vert f(x') - f(x) \right\vert &= \left\vert (x_{n+1} - \bar{x})^2 \frac{n}{n+1} \right\vert \nonumber\\
		&= \left\vert (x_{n+1} - \bar{x})^2 \frac{n}{n+1} \right\vert \nonumber \\
		&\le \left( M-m \right)^2 \frac{n}{n+1}
	\end{align}

	Removing an element: \\
	Let $X' = X \setminus \{x_n\}$. Then, rewriting Eq.~\ref{eq:varAddOnePoint} with $n$ set to $n+1$ since ``$x$'' in this case is the greater set,
	$$ f(x) = f(x') + (x_n - \bar{x}')^2 \frac{n-1}{n}.$$

	Then,
	\begin{align}
	\label{eq:L1sub}
	\left\vert f(x) - f(x') \right\vert &= \left\vert (x_n - \bar{x}')^2 \frac{n-1}{n} \right\vert \nonumber\\
		&\le (M-m)^2 \frac{n-1}{n},\\
	\end{align}

	Note that for any $n \ge 1$,
	\begin{equation}
	\label{ineq}
	 \frac{n}{n + 1} > \frac{n-1}{n}.
	\end{equation}

	So, the worst-case bound always occurs in the ``add-one'' case, and the $\ell_1$-sensitivity of $f(\cdot)$ is in general bounded by

\begin{equation}
\label{eq:fBoundaddsub}
\left( M-m \right)^2 \frac{n}{n + 1}.
\end{equation}

 \end{proof}

 \begin{corollary}
 \label{cor:l1addsub}
	Variance has $\ell_1$-sensitivity in the add/drop one model bounded above by
	$$\left( M-m \right)^2 \frac{1}{n+1}. $$
 \end{corollary}

 \begin{proof}
This follows from Theorem \ref{thm:l1addsub} with a renormalization by $n$.
\end{proof}

 \begin{corollary}
 \label{cor:l1addsub}
	Sample variance has $\ell_1$-sensitivity in the add/drop one model bounded above by
	$$\left( M-m \right)^2 \frac{n}{n^2 - 1}. $$
 \end{corollary}

 \begin{proof}
This follows from Theorem \ref{thm:l1addsub} with a renormalization by $n-1$.
\end{proof}

\subsection{$\ell_2$-sensitivity}

\begin{theorem}
\label{thm:l2addsub}
Variance has $\ell_2$-sensitivity in the add/drop one model bounded above by
	$$ \left( M-m \right)^2 \frac{1}{n+1} .$$
\end{theorem}

%\begin{proof}
%Because the bounds in Equations \ref{eq:L1add} and \ref{eq:L1sub} are positive, they hold for their square. Then, by the inequality for $n \ge 1$ in Equation \ref{ineq} it follows that the $\ell_2$ sensitivity of the variance is bounded by
%$$
%\left( \left( M-m \right)^2 \frac{1}{n+1} \right)^2,
%$$
%where the change in constant from Equations \ref{eq:L1add} to $1/(n+1)$ comes from the $1/n$ in the definition of variance.
%\end{proof}

\begin{corollary}
Sample variance has $\ell_2$-sensitivity in the add/drop one model bounded above by
	$$ \left( M-m \right)^2 \frac{n}{n^2 - 1}.$$
\end{corollary}

%\begin{proof}
%The logic here is the same as for \ref{thm:l2addsub} with a renormalization by $n-1$ rather than by $n$.
%\end{proof}

\bibliographystyle{alpha}
\bibliography{mean}

\end{document}


\documentclass[12pt]{article}
\usepackage{indentfirst}
\usepackage{siunitx}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{float}
\usepackage{amssymb}

\setlength{\parindent}{20pt}
\setlength{\oddsidemargin}{0.5cm}
\setlength{\evensidemargin}{0.5cm}
\setlength{\marginparsep}{0.75cm}
\setlength{\marginparwidth}{2.5cm}
\setlength{\marginparpush}{1.0cm}
\setlength{\textwidth}{150mm}
\setlength{\parindent}{1cm}
\renewcommand{\baselinestretch}{1.5}

\begin{document}
	\section{Row Space}
	Definition: Row $A = Span\{\vec{a}_1^T, \vec{a}_2^T,\dots,\vec{a}_m^T\}$ \par
	Property: Let $A$ be an m $\times$ n matrix. Row A is a subspace of $\mathbb{R}^n$ \par
	Property: Row $A$ = Col $A^T$ \par
	Theorem: A and B are row-equivalent matrices $\rightarrow$ Row A = Row B \par
	Corollary: dim(Col A) = dim(Row A) for any matrix A \par
	\section{Rank}
	Definition: The rank of a matrix A is the dimension of Col A.
\end{document}


\def\x{{\bf x}}
\def\q{{\bf q}}
\def\p{{\bf p}}
\def\v{{\bf v}}

\lab{Algorithms}{Fast Fourier Transform}{Fast Fourier Transform}
\objective{This lesson explores the fourier transform applied to audio signals.}

\section*{Audio Signals}

In this lab we will working with signals which represent sound. First we should understand how sound is produced and transmitted. As an example, consider a typical electronic speaker.

\begin{center}\includegraphics[width=45mm]{speaker}\end{center}

The essential component is a bowl-shaped membrane, called the \emph{cone} (or \emph{diaphragm}) of the speaker. Behind the cone is attached a lightweight electromagnet. When current flows through the electromagnet in a certain direction, a magnetic field is created which repels the electromagnet (and the attached cone) from a  heavier permanent magnet mounted behind it. The cone is thus thrust forward, compressing the air in front of the speaker, resulting a slight increase in air pressure which is propagated through the room in a wave. When the direction of the current is reversed, the electromagnet is attracted to the permanent magnet. The cone is thus pulled back, expanding the air in front of the speaker, resulting in a slight decrease in air pressure which, again, is propagated through the room in a wave. These pressure waves reach your ears and are perceived as sound. The basic mechanism of a speaker is entirely reversible; for instance, a pair of ordinary (non-amplifying) headphones can also, without any modification, be used as a stereo microphone (Try it!).

Play the audio clip \texttt{pulseramp.wav}. You will hear a series of pulses, or clicks. A click is simply a sudden (small) increase in pressure followed almost immediately by a return to normal pressure. By a speaker, a click is produced by the cone suddenly thrusting forward a small amount and then gradually, silently, returning to its original position. Notice that as the pulses occur closer and closer together, eventually your ears no longer perceive them as individual pulses. Instead they are perceived as a tone which increases in pitch as the pulses increase in frequency.

In python we can read in a \texttt{.wav} by importing wavfile from scipy.io and using wavfile.read() as follows
\begin{lstlisting}
from scipy.io import wavfile
rate, mywave = wavfile.read('pulseramp.wav')
\end{lstlisting}
Here \texttt{data} is the signal and \texttt{rate} is the sampling rate of the audio measured in Hz.  We can check the sampling rate and the size of the signal:
\begin{lstlisting}
rate
Out[2]: 44100

mywave.shape
Out[3]: (300000,)
\end{lstlisting}
We find that \texttt{mywave} is a vector of length $300000$. It has only one column because this is a mono (single-channel) \texttt{.wav} file. If we had used a stereo \texttt{.wav} file, then mywave would have been a matrix with two columns, one for the left channel and one for the right.

The entries of \texttt{mywave} are integers between $-32767$ and $+32767$ which represent the relative air pressure at a particular instant, sampled at regular intervals (these bounds are the range of 16-bit integers).   In our case, the \texttt{.wav} file is sampled at 44.1KHz (the standard sample-rate for high-quality audio), so there are 44100 samples per second. $+32767$ represents the maximum possible increase in air pressure which the speaker can produce (at its current volume level), while $-32767$ represents the maximum possible decrease in air pressure. In the case of our particular \texttt{.wav} file, you can check that most of the entries of \texttt{mywave} are $0$, indicating neutral pressure. Only occasionally you will find an entry which is $+32767$, which is where a pulse occurs.
We can see this most easily if we plot the waveform of our sound (see Figure \ref{pulseramp}):
\begin{lstlisting}
from matplotlib import pyplot as plt
plt.plot(mywave)
plt.show()
\end{lstlisting}
\begin{figure}[ht]\caption{A sequence of pulses of increasing frequency}\label{pulseramp}\centering\includegraphics[width=\textwidth]{pulseramp}\end{figure}

We should keep in mind that the deviations in air pressure that we hear as sound are extremely small relative to atmospheric pressure. For instance, in normal conversation, the deviation in air pressure reaching your ears might be around $.01$ pascals, about one ten-millionth of atmospheric pressure.

\section*{Using the Fourier Transform to Investigate the Spectrum of a Signal}

In many ways the most basic kind of sound is one whose waveform is a sinusoid (i.e., a scaled, shifted version of the sine function). The tone of a sinusoid is, to the ear, more mellow than any other sound of the same pitch. We have seen that, mathematically, sinusoids are important, because Fourier series provide a way of expressing any periodic, piecewise continuous function as a sum of sine and cosine waves of various frequencies and amplitudes, or, more conveniently, using complex exponential functions.

Here's one way of producing a a 1760Hz sine wave (which in musical terms is A6, the A between two and three octaves above middle C):

\begin{lstlisting}
import scipy as sp
from scipy.io import wavfile

samplerate=44100 # 44100 samples per second
freq=1760 # We're going to produce a 1760 Hz sine wave ...
length=2 # ... which will last for 2 seconds.
stepsize=freq*2*sp.pi/samplerate
sig=sp.sin(sp.arange(0,stepsize*length*samplerate,stepsize))
scaled = sp.int16(sig/sp.absolute(sig).max() * 32767)
wavfile.write('sinusound.wav',samplerate, scaled)
\end{lstlisting}

Notice before writing the file we had to scale it to be a 16-bit integer between -32767 and +32767 because wavfile.write expects the data to be in this form.  Now play this sound.

\begin{problem}
Produce a 60Hz sine wave 1 second in length. Then plot the signal. Play the sound. Is it audible?
\end{problem}

For a discrete signal, we are given finitely many data points (assumed to be evenly spaced on the $x$-axis); we can think of such a signal as a function $f$ whose domain consists of the integers $\{0,1,\dots,N-1\}$, where $N$ is the number of data points. For convenience, we will often extend $f$ to be a periodic function with period $N$, defined on all integers, by defining $f(a+bN)=f(a)$ for all integers $a$ and $b$. How might we find something analagous to a Fourier series for such a function? Well, it turns out that the functions, $e^{\frac{2\pi ikx}N}$, which are periodic with period $N$, form an orthonormal set with respect to the inner product
$$\langle f,g\rangle = \frac1N\sum_{k=0}^{N-1}\overline{f(k)}g(k)$$
In particular, these functions are linearly independent and hence form a basis for the $N$-dimensional (complex) vector space of complex-valued functions with domain $\{0,\dots,N-1\}$. The vector of coordinates of a function $f$ with respect to this basis constitutes what is called the \emph{Discrete Fourier Transform} (or \emph{DFT}) of the function $f$. In other words, any function $f$ whose domain consists of the integers $\{0,1,\dots,N-1\}$ may be expressed
$$f(x)=\sum_{k=0}^{N-1}c_ke^{\frac{2\pi ikx}N}$$
and we call the vector of coefficients $(c_0,c_1,\dots,c_{N-1})$ the DFT of $f$. Explicitly, these coefficients may be computed by taking the inner product of each basis function with $f$:
$$c_k=\langle e^{\frac{2\pi ikx}N}, f(x)\rangle=\frac1N\sum_{x=0}^{N-1}e^{\frac{-2\pi ikx}N}f(x)$$

In scipy, there is a module \texttt{scipy.fft} that contains functions for computing the DFT of a signal. (\texttt{fft} stands for Fast Fourier Transform.) For example, if we create a sine wave signal \texttt{sig} as above and then type,
\begin{lstlisting}
fsig = sp.fft(sig)
\end{lstlisting}
then \texttt{fsig} is an array of complex numbers representing the DFT of this signal. We will plot the absolute values of the entries of \texttt{fsig} (see Figure \ref{sinespec}).

Notice that there are two spikes, one on the left and one on the right. Why are there two spikes when there is only one tone? The answer is that for a real-valued signal, the right half of the DFT is always a mirror-image duplicate of the left half -- or, more precisely, each entry in the right half of the DFT is the complex conjugate of the corresponding entry in the left half. Thus the right half of the DFT gives us no new information, and we may as well ignore it.

In saying that the right half of the DFT is a complex-conjugate mirror-image of the left half, to be perfectly correct we actually need to ignore the very first entry of the DFT.

\begin{problem}
Implement a function to compute the DFT of a vector using the naive algorithm, directly from the definition of the DFT. Test it on several random vectors and make sure the results match those of scipy's \texttt{fft}.  Note that the scaling scipy uses does not divide by $N$, so you will need to account for that in comparing your results.
\end{problem}

The naive way of computing the DFT has $O(N^2)$ runtime.  There is a very important algorithm (and many variations of it) called the Fast Fourier Transform (FFT) which computes the DFT in $O(N \log N)$ runtime.  Essentially the FFT algorithm accomplishes this speedup by breaking the recursively breaking up the original array into smaller pieces, finding the Fourier Transform of those smaller pieces, and carefully recombining to get the Fourier Transform of the original array.  While scipy uses the FFT algorithm, it's implementation at the time of this writing is somewhat slow.

There is a library written in $C$ for computing the FFT called FFTW (Fastest Fourier Transform in the West) that is substantially quicker.  There is a python package available for download called ``pyfftw'' that wraps fftw in python. The pyfftw package has an interface called scipy\_fftpack that acts exactly like scipy's fft but uses the FFTW. It is important to note that it is only faster than scipy's version on larger arrays. You can also set the number of threads. The default is 1. The author got a significant speed up by setting the threads=8 .

\begin{figure}[ht]\caption{DFT of a sine wave}\label{sinespec}\centering\includegraphics[width=\textwidth]{sinespec}\end{figure}

Now let's look at a more complicated signal. Play the file \texttt{tada.wav}.  If you read \texttt{tada.wav} into python you will find it has two columns.  That is because it is a stereo recording meaning the two columns correspond to two audio signals (left and right).  When using pyFFTW, by default it will take the Fourier Transform along the last axis.  Because of this we will set axis=0.

Now we plot the DFT of \texttt{tada.wav} (see Figure \ref{tadaspec}):
\begin{lstlisting}
import scipy as sp
from scipy.io import wavfile
from pyfftw.interfaces import scipy_fftpack as fftw
rate, sig = wavfile.read('tada.wav')
sig = sp.float32(sig)
fsig = fftw.fft(sig,axis=0)
plt.figure()
plt.plot(sp.absolute(fsig))
plt.show()
\end{lstlisting}

\begin{figure}[ht]\caption{Absolute value of the DFT of \texttt{Tada.wav}}\label{tadaspec}\centering\includegraphics[width=\textwidth]{tadaspec}\end{figure}
Again, the right half of the plot is a mirror-image duplicate of the left half. We might as well exclude it by only plotting the left half. The absolute value of the left half of  the DFT is called the \emph{spectrum} of the signal (see Figure \ref{tadaspec2}):
\begin{lstlisting}
f = sp.absolute(fsig)
plt.plot(f[0:f.shape[0]/2,:])
plt.show()
\end{lstlisting}
\begin{figure}[ht]\caption{Spectrum of \texttt{Tada.wav} (Absolute value of left half of DFT)}\label{tadaspec2}\centering\includegraphics[width=\textwidth]{tadaspec2}\end{figure}
Notice that the number of spikes in the plot is much greater than the number of musical notes in the sound. This is because when a one note is played on a musical instrument, more than one frequency is produced: the lowest frequency is called the \emph{fundamental}; the others, which usually occur as integer multiples of the fundamental frequency, are called \emph{overtones}. In instruments with a mellow tone (such as a flute or vibraphone) the overtones are relatively soft, while in instruments with a brighter tone (such as a trumpet) the overtones are quite prominent. In the DFT of \texttt{Tada.wav}, you will notice spikes at about 2030, 4060, 6090, 8120, 10160, 12190, 14220, 16250, 18280, and 20320 units. These are approximately integer multiples of 2030 and represent overtones of a single musical note whose fundamental frequency is about 2030 units.

Now what are these units and how can we express them in Hz? Well, a frequency of 2030 units in the DFT means that the tone undergoes 2030 cycles through the extent of the entire recording. We use check the shape of \texttt{sig} to find the total number of samples in the recording:
\begin{verbatim}
sig.shape
Out[27]: (42752, 2)
\end{verbatim}
So the entire recording is 42752 samples long (in each channel -- this is a stereo recording.) The sample rate is 22050 samples per second. (You could find this out, for instance, by right clicking on the \texttt{.wav}, selecting \emph{Properties} and looking under the \emph{Summary} tab).  Thus, 2030 cycles through the entire recording is equivalent to
$$\frac{2030\text{ cycles}}{42752\text{ samples}} \times \frac{22050\text{ samples}}{\text{sec}} \approx 1047 \text{ Hz}.$$
So this tone is at 1047 Hz, which in musical terms is a (concert) C6, the C two octaves above middle C. There are also other, lower musical notes in the recording which each have their own sequence of overtones.

\begin{problem}
Listen to \texttt{pianoclip.wav}. Identify the audible frequency of greatest amplitude (Ignore the 0 frequency as explained in the following paragraph.).  What musical note does the fundamental frequency correspond to? (For this, you may wish to use the table on http://en.wikipedia.org/wiki/Piano\_key\_frequencies)
\end{problem}

You will notice that in this case the largest spike actually occurs in the very first entry of the DFT. This corresponds to a frequency of 0Hz. A cosine wave of 0Hz is nothing other than a constant function. The presence of this 0Hz frequency, if taken literally, would mean that there was a (slight) sustained increase above the normal atmospheric pressure throughout the recording. However, a speaker or musical instrument, because of its bounded ranges of motion, is incapable of producing a sustained increase (or decrease) in pressure; and a microphone, for the same reason, is incapable of detecting such a sustained change in pressure. An ordinary speaker cannot produce frequencies below about 15 to 20Hz, and even if it could, we would not be able to hear them. The 0Hz frequency which appears here is simply an artifact of an imperfect recording device. It indicates that some direct current ``leaked" through the device.

\section*{Down-sampling a signal}

Listen to \texttt{saw.wav}. You will hear a high-pitched buzzing sound \footnote{The technical term for it is a sawtooth wave; for the curious, see http://en.wikipedia.org/wiki/Sawtooth\_wave}. The sound is recorded at a sample rate of 44.1KHz. Now let's say we wanted to convert it to a 22.05KHz sample rate. How might we do that? Since we're cutting the sample rate in half, we might think that we could accomplish this simply by discarding every other sample, i.e., keep the even-index samples and discard the odd-index samples. Let's try that and see how it works:
\begin{lstlisting}
rate, in_sig = wavfile.read('saw.wav')
out_sig = in_sig[sp.arange(0,rate,2)]
wavfile.write('down_saw.wav',rate/2,out_sig)
\end{lstlisting}
What does it sound like? Well, it sounds much like the original except that there are some extra tones which weren't there to begin with; these extra tones, in this case, are not in a nice harmonic relationship with the original tone and clash rather badly. One of the extra tones is at a rather low frequency of about 99Hz compared to 1568Hz of the fundamental frequency in the original tone. Where are these mysterious ``extra" tones coming from?

To explain this, we need to understand some important facts about sampling. First of all, a basic fact is that the highest possible frequency that a sampled signal can represent is half the sampling rate. Such a frequency is represented by alternating between +32767 and -32767 at each sample. This frequency is known as the \emph{Nyquist frequency}. If, for instance, the sampling rate is 44.1KHz, then the Nyquist frequency will be 22.05KHz. This is slightly beyond the upper limit of human hearing; thus, for audio, generally no improvement in quality results from using a higher sampling rate than 44.1KHz, since at 44.1KHz the entire audible spectrum can already be represented.

Now let's consider what would happen if we tried to represent a frequency beyond the Nyquist frequency. We create a sinusoid of frequency $f$ Hz by setting
$$s_k = \cos\left(\frac{2\pi kf}r\right)$$
where $s_k$ is the value of the $k$th sample and $r$ is the sample rate in Hz. Suppose we take the frequency to be $f=\frac{r}2+a$, where $a$ is some positive number, so that $f$ is beyond the Nyquist frequency. Then we have
\begin{align*}
s_k &= \cos\left(\frac{2\pi kf}r\right) \\
&= \cos\left(\frac{2\pi k(\frac{r}2+a)}r\right)\\
&= \cos\left(\pi k+\frac{2\pi ka}r\right)\\
&= \cos\left(-\left(\pi k+\frac{2\pi ka}r\right)\right)\\
&= \cos\left(-\left(\pi k+\frac{2\pi ka}r\right)+2\pi k \right)\\
&= \cos\left(\pi k-\frac{2\pi ka}r\right)\\
&= \cos\left(\frac{2\pi k(\frac{r}2-a)}r\right).
\end{align*}
But this is a sinusoid of frequency $\frac{r}2-a$, which is a frequency below the Nyquist frequency. Thus a sinusoid above the Nyquist frequency, when sampled, is indistinguishable from a corresponding sinusoid below the Nyquist frequency. An electronic speaker, given the sampled signal, will actually produce the sinusoid which lies below the Nyquist frequency, not the one above. Thus,
\emph{A frequency above the Nyquist frequency will, when sampled, have its frequency reflected across the Nyquist frequency.}
This phenomenon is known as \emph{aliasing}.

What does this have to do with our problem? Our tone is at 1568Hz, which is well below the Nyquist frequency of 22050Hz. However, there are prominent overtones extending all the way up to 21951Hz (see Figure \ref{sawspec}). Since we are downsampling to 22050Hz, the new Nyquist frequency will be 11025Hz. The overtones in our signal which lie between 11025Hz and 22050Hz will then be reflected across the new Nyquist frequency in our downsampled sound. For example, the overtone at 21951Hz will be reflected to $11025-(21951-11025)=99$ Hz, the lowest ``extra" tone that we observed. See Figure \ref{sawspecdown}.

\begin{figure}[ht]\caption{Spectrum of \texttt{saw.wav}}\label{sawspec}\centering\includegraphics[width=\textwidth]{sawspec}\end{figure}

\begin{figure}[ht]\caption{Spectrum of \texttt{saw.wav} naively downsampled to 22050Hz}\label{sawspecdown}\centering\includegraphics[width=\textwidth]{sawspecdown}\end{figure}

So, how do we downsample our signal in a way that avoids this unwanted aliasing effect? What we do is take the DFT of our signal, cut out the upper half of the spectrum (which can no longer be represented, after we downsample), then take the inverse DFT of the result. It's as simple as that.

Well -- \emph{almost} that simple. Actually, when we say we're cutting out a set of frequencies from the spectrum, we mean we're removing those frequencies from the left half of the DFT and also the corresponding frequencies from the right half of the DFT. So, we're actually removing a big chunk out of the \emph{middle} of the DFT and then pasting the two ends back together. We can do this as follows where in\_sig is the signal from \texttt{saw.wav}:
\begin{lstlisting}
old_rate = 44100
new_rate = 22050
in_sig = sp.float32(in_sig)
fin = fftw.fft(in_sig)
nsiz = sp.floor(in_sig.size*new_rate/old_rate)
nsizh = sp.floor(nsiz/2)
fout = sp.zeros(nsiz)
fout = fout + 0j
fout[0:nsizh] = fin[0:nsizh]
fout[nsiz-nsizh+1:] = sp.conj(sp.flipud(fout[1:nsizh]))
out = fftw.ifft(fout)
\end{lstlisting}
There are just a couple minor problems we need to deal with before we're finished. First, because of roundoff error, you will find that some of the entries of the \texttt{out} have (slightly) nonzero imaginary components.  So we should first make sure the signal is purely real,
\begin{lstlisting}
out = sp.real(out) % Take the real component of the signal
\end{lstlisting}
Second, as always, we need to rescale and cast as a 16-bit integer:
\begin{lstlisting}
out = sp.int16(out/sp.absolute(out).max() * 32767)
\end{lstlisting}

\begin{figure}[ht]\caption{Spectrum of \texttt{saw.wav} correctly downsampled to 22050Hz}\label{sawspecdown2}\centering\includegraphics[width=\textwidth]{sawdownspec}\end{figure}

\begin{problem}
Convert the same sound, \texttt{saw.wav}, to a 11025Hz sampling rate. Do you notice a difference in the quality of the sound? If so, can you explain why?
\end{problem}

\begin{problem}
Can the technique above be used to change the sampling rate by something other than an integer factor, e.g., from 44100Hz to 36000Hz? Try it.
\end{problem}

\begin{comment}
\begin{problem}
Suppose we would like to upsample \texttt{Tada.wav} from a sampling rate of 22050Hz to 44100Hz. \begin{enumerate}
\item[(a)]
The naive way to do this would be to use linear interpolation on the original signal, i.e., the original signal would form the odd-index samples of the new signal, while the even-index samples of the new signal would be formed by taking the average of each two consecutive samples in the original signal. Write code to carry out the upsampling in this way; plot the spectrum of the result. What do you observe?
\item[(b)]Now write code to upsample the signal using the DFT, using a technique similar to what we did above for downsampling. Again, plot the spectrum of the result. Can you hear any difference between this and the result of (a)?
\end{enumerate}
\end{problem}
\end{comment}


\clearpage
\section*{Depth, Shape, Precision; VSIPL Naming}\addcontentsline{toc}{section}{Depth, Shape, Precision; VSIPL Naming}
In order to understand \cvl{} one needs to understand something about the convention used when naming functions, types, structures, scalars, etc. This also will help one understand some of the reasons behind the \ttbf{block} and \ttbf{view} structures in \cvl. I try to maintain the same conventions in this document and extend them to cover \pyjv{} type strings.
%
\subsection*{Depth}\addcontentsline{toc}{subsection}{Depth}
A scalar element has a \ttbf{depth}.  For \ttbf{VSIPL} this is pretty simple.  It is either complex or real. I suppose in the future it is possible other scalar depths could be defined. For instance a scalar defining a pixel in an image might have red, green, blue components.

Note that \ttbf{depth} is an attribute of a \ttbf{block}, and that \ttbf{block}s only store elements of a single type; so a \ttbf{block} will only have one depth associated with it.
%
\subsection*{Precision}\addcontentsline{toc}{subsection}{Precision}
Precision indicates how accurate the numbers are. In C this would be indicated by \ilCode{float}, \ilCode{double}, \ilCode{int}, etc. \jv{} only supports standard ANSI C89 precisions but the naming conventions for the \ttbf{VSIP} specification allow for just about any precision to be be declared if an implementation wants to support it.

Note that \ttbf{precision} is an attribute of a \ttbf{block}
%
\subsection*{Shape}\addcontentsline{toc}{subsection}{Shape}
A \ttbf{view} defines the \ttbf{shape} of a \ttbf{VSIPL} object. A \ttbf{block} is basically an abstract notion of memory storage. It has a \ttbf{depth} and a \ttbf{precision} and provides to a view a linear array of scalar elements.  How the elements are defined on the underlying memory of the compute device is implementation dependent. The \ttbf{view} then places a shape on the block allowing one to access the data as a vector, matrix, or tensor.

So the \ttbf{shape} is an attribute of the \ttbf{view}. A \ttbf{view} is basically an index set on a \ttbf{block}.
%
\subsection*{Function Naming for \cvl}\addcontentsline{toc}{subsection}{Function Naming for \cvl}
\subsubsection*{Depth Affix}\addcontentsline{toc}{subsubsection}{Depth Affix}
Generally the prefix \ttbf{c} is used to indicate complex and the prefix \ttbf{r} is used to indicate real. For real the precision is frequently understood with no \ttbf{r} except in some cases where both real and complex are needed. For instance \ilCode{vsip\_vadd\_f} is for real vectors and \ilCode{vsip\_cvadd\_f} is for complex vectors. The function \ttbf{add} has been defined to allow for adding a real vector to a complex vector resulting in \ilCode{vsip\_rcvadd\_f}.

We also have an \ttbf{mi} depth type which goes in the precision place-holder. This is a matrix index type the scalar of which is defined as a structure in the \cvl{} specification (similar to the way complex is defined).
%
\subsubsection*{Precision Affix}\addcontentsline{toc}{subsubsection}{Precision Affix}
There are many precisions available for use in \ttbf{VSIPL}. The ones used in \jv{} are contained in table \ref{tab:jvsipPrecisions}.

We note that the matrix index has elements that are the same precision as the vector index. The matrix index comes in the precision place when naming but it is much like the complex type and actually indicates a \ttbf{depth}.

The type \ttbf{ue32} is used in the definition of \ttbf{VSIPL} random numbers. There are no blocks defined for it, only a scalar.  For \jv{} this is defined to be an \ilCode{unsigned int} which normally is 32 bits long; but I don't think this is required by the C89 specification. So in general the declaration of this type (in \ttbf{vsip.h}) will be implementation dependent.
\input{JvsipPrecisions}
%
\subsubsection*{Shape Affix}\addcontentsline{toc}{subsubsection}{Shape Affix}
Shapes in the \cvl{} specification are basically indicated by an \ttbf{s} for a scalar, a \ttbf{v} for a vector, a \ttbf{m} for a matrix and a \ttbf{t} for a tensor.

\subsubsection*{Comments on Naming in C VSIPL specification}\addcontentsline{toc}{subsubsection}{Comments on Naming in C VSIPL specification}
In the \cvl{} specification we have characters in italic font for \ttbf{\emph{d}} (depth), \\* \ttbf{\emph{s}} (shape), \ttbf{\emph{p}} (any precision), \ttbf{\emph{f}} (any float), \ttbf{\emph{i}} (any integer), etc.

These special characters indicate an overloaded specification telling the implementor what general types may be defined for a function in an implementation.   I don't use these character types in this document because I am talking about an actual implementation; not a specification.  The characters I use indicate what is actually implemented.


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[super,square]{natbib}
\usepackage{tabularx}
\usepackage{parskip}
\usepackage[margin=1.4in]{geometry}
\usepackage{csquotes}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amsthm}
\usepackage{hyperref}

\hypersetup{
    colorlinks,
    citecolor=black,
    filecolor=black,
    linkcolor=black,
    urlcolor=black
}

\newcommand{\comment}[1]{}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{corollary}{Corollary}[theorem]
\newtheorem{proposition}{Proposition}[theorem]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{identity}[theorem]{Identity}

\title{\vspace{-3cm} Combinatorics}
\author{}
\date{}

\comment{
Outline:
-
}

\begin{document}
\maketitle
\vspace{-1.5cm}
\tableofcontents
\newpage

\section{Counting and Enumeration}
\subsection{Strings and sets}
An $X$-string of length $n$ is written $x_1x_2x_3 \cdots x_n$ for $x_i \in X$, where $X$ is an alphabet its elements are  characters.
\[
s: [n] \mapsto X, \ \ \  [n] = \{ 1, 2,3 \cdots, n \}
\]

\subsection{Exponential}
An enumeration such that: \begin{enumerate}
    \item Order of items matters.
    \item Counts include duplication or removals of items.
\end{enumerate}

If $X$ is a finite set, the number of all $X$-strings of length $n$ will be $|X|^n$.

\subsection{Permutations}
If $X$ is an $n$ element set, the number of $X$-strings of length $m$ is,
\[
    P(n,m) = \frac{n!}{(n-m)!}
\]

\begin{enumerate}
    \item Order of items matters.
    \item Counts do not include duplication or removals of items.
    \item Collection of counts could be stored in \emph{arrays}.
\end{enumerate}

\subsection{Combinations (Binomial Coefficient)}
The number of combinations of $n$ objects taken $k$ at a time. Given $n, k \in \mathbb Z, 0 \leq k \leq n$ then the binomial coefficient is,
\[
    C(n, k) = \binom{n}{k} = \frac{P(n,k)}{k!} = \frac{n!}{k!(n-k)!} = \binom{n}{n-k}
\]

\begin{enumerate}
    \item Order of items doesn't matter.
    \item Counts do not include duplication or removals of items.
    \item Collection of counts could be stored in \emph{sets}.
\end{enumerate}

\begin{identity}
\[
    \binom{n+1}{k} = \binom{n}{k} + \binom{n}{k+1}
\]
\end{identity}

\subsection{n-th Partial Sums}
\begin{identity}
\[
    1 +2 +3 + \cdots + n =  \frac{n(n+1)}{2}
\]
\end{identity}
\begin{identity}
\[
    1 +3 + 5 + \cdots + 2n -1 = n^2
\]
\end{identity}

\subsection{Pascal's Triangle}
A triangular array made up of the binomial coefficients in which the entry in the $n$th row and $k$th column is ${\tbinom {n}{k}}$.

\begin{identity}
\[
    \binom{n}{k} = \binom{n-1}{k-1} + \binom{n-1}{k}
\]
\end{identity}

\begin{identity}
\[
    \sum_{i=0}^n \binom{n}{i} = 2^n
\]
\end{identity}

\subsection{Binomial Theorem}
Let $x,y \in \mathbb R, x+y \neq 0$. Then $\forall n \in \mathbb Z^{+}$,
\[
    (x + y)^n = \sum_{i=0}^n \binom{n}{i} x^{n-i} y^i
\]

\subsection{Multinomial Coefficients}
The number of combinations in which $k$ elements are chosen from subsets of $X$ with $|X| = n$ will be,
\[
    \binom{n}{k_1, k_2, \cdots, k_n} = \frac{n!}{k_1! k_2! \cdots k_n!}
\]

\subsection{Stars and Bars}
Stars and bars is a graphical aid for deriving certain combinatorial theorems. It can be used to solve many simple counting problems, such as how many ways there are to put n indistinguishable balls into k distinguishable bins.

\begin{theorem}
For any pair of positive integers n and k, the number of k-tuples of positive integers whose sum is n is equal to the number of $(k - 1)$-element subsets of a set with $n - 1$ elements, i.e. $\binom{n - 1}{k-1}$.
\end{theorem}

\begin{theorem}
For any pair of positive integers n and k, the number of k-tuples of non-negative integers whose sum is n is equal to the number of multisets of cardinality $k - 1$ taken from a set of size $n + 1$, i.e. $\binom{n+1}{k-1}$.

\end{theorem}

\subsection{Multinomial Theorem}
Let $x_1,\cdots x_n \in \mathbb R^+$. Then $\forall n \in \mathbb N^+$,
\[
    (x_1 + \cdots x_n)^n = \sum_{k_1 + ... + k_r = n} = \binom{n}{k_1, \cdots k_r} x_1^{k_1} x_2 ^{k_2} \cdots x_r^{k_r}
\]

\subsection{Lattice Paths}
A sequence of ordered pairs $(m_1, n_1), (m_2, n_2), \cdots, (m_t, n_t)$ such that either:
\begin{enumerate}
    \item $m_{i+1} = m_{i}+1$ and $n_{i+1} = n_{i}$
    \item $m_{i+1} = m_i$ and $n_{i+1} = n_i +1$.
\end{enumerate}
The construction of lattice paths forms a bijection with $X$-strings where $X = \{ H, V\}$ with $H,V$ encoding horizontal or vertical moves. The number of lattice paths from $(m_1, n_1)$ to $(m_2,n_2)$ is,
\[
   \binom{m_2 - m_1 + m_2 - m_1}{m_2-m_1}.
\]

\subsection{Catalan Number}
The number of lattice paths from $(0,0)$ to $(n,n)$ that do not go above the diagonal line $y = x$, also known as Dyck paths, are equal to the $n$th Catalan number,
\[
    C(n) = \frac{1}{n+1} \binom{2n}{n}
\]

\subsection{The Rule of Sum}
The rule of sum or addition principle is a basic counting principle. Stated simply, it is the idea that if we have A ways of doing something and B ways of doing another thing and we can not do both at the same time, then there are A + B ways to choose one of the actions.

\subsection{Inclusion-Exclusion Principle}
The inclusion–exclusion principle is a counting technique that generalizes the method of obtaining the number of elements in the union of two finite sets without double counting elements,
\[
    |A\cup B|=|A|+|B|-|A\cap B|.
\]
The principle is particularly useful in the case of three sets, i.e. for the sets $A$, $B$ and $C$, the count of the union is given by,
\[
|A\cup B\cup C|=|A|+|B|+|C|-|A\cap B|-|A\cap C|-|B\cap C|+|A\cap B\cap C|.
\]

\subsection{The Euler Totient Function}
For a positive integer $n\geq2$, Euler's Totient function is defined by
\[
\phi(n)=|\{m\in \mathbb N: m\le n,  \gcd(m,n)=1\}|.
\]

\section{Number Theory}

\subsection{Well-Ordered Property}
$\forall A \subset \mathbb N^+, A \neq \emptyset$, $A$ must have a minimum element. This statement is equivalent to mathematical induction.

\subsection{Principle of Mathematical Induction}
Let $S_n$ be an open statement involving a positive integer $n$. If $S_1$ is true, and if for each positive integer $k$, assuming that the statement $S_k$ is true implies that the statement $S_k + 1$ is true, then $S_n$ is true for every positive integer $n$.

\subsection{Pigeonhole Principle}
Given $m$ items and $n$ containers, there exists a containers with at least $\lceil \frac{m}{n} \rceil$ items.

Expressed in terms of functions and their domains and ranges: if $f: X \mapsto Y$ and $|X| > |Y|$ then there exists a $y \in Y$ and $x, x' \in X, x \neq x'$ so that $f(x) = f(x') = y$

\subsection{Erdos–Szekeres theorem}
If $m,n \in \mathbb Z^+$, then any sequence of $mn+1$ distinct real numbers either has an increasing sub-sequence of $m+1$ items or a decreasing sub-sequence of $n+1$ terms.

\subsection{Fibonacci Sequence}
\[
\phi_n = \phi_{n-1} +\phi_{n-2}\text{; } \phi_1 = 1 \text{, } \phi_2 = 2
\]

\subsection{Euclidean Algorithm}
The Euclidean algorithm is an efficient method for computing the greatest common divisor (GCD) of two integers.

Given $m = q_i n_i + r_i$ where $m_i,n_i \in \mathbb Z^+$ with $0 \leq r_i < n_i$ we see that $r_i$ represents the remainder and $q_i$ represents the quotient. We use the fact that $\gcd(m, n_i) = \gcd(n_i, r_i)$ to iteratively calculate the $\gcd$ on the decomposition of the $n_i$ term and the remainder $r_i$.

\section{Graphs Theory}
Let $G = (V, E)$ be a graph with $V$ verticies and $E$ edges.

\subsection{Subgraph}
$H = (W, F)$ is a subgraph of $G$ if $W \subseteq V$ and $F \subseteq E$. That is, a subgraph does not contain any additional vertices or edges but may contain fewer.

\subsection{Induced Subgraph}
$H = (W, F)$ is an induced subgraph if $W \subseteq V$ and $F = \{ xy \in E : xy \in W \}$. That is,  an induced subgraph is formed from a subset of the vertices of the graph with all of the edges connecting pairs of vertices in the subset being kept from the original graph.

\subsection{Spanning}
$H = (W, F)$ is spanning when $W = V$.

\subsection{Isomorphic}
$G$ is isomorphic to $H$, denoted by $G \cong H$, if there exists an edge-preserving bijective mapping, $f: V \mapsto W$.

\subsection{Homeomorphic}
The subdivision of an edge $e$ with endpoints $\{u,v\}$ yields a graph containing one new vertex $w$ with an edge set replacing $e$ by two new edges, $\{u,w\}$ and $\{w,v\}$.

Two graphs $G$ and $H$ are homeomorphic if there is a graph isomorphism from some subdivision of $G$ to some subdivision of $H$.

\subsection{Contains}
$G$ contains $H$ if there exists a subgraph of $G$ that is isomorphic to $H$.

\subsection{Complete Graphs}
A complete graph is a graph in which each pair of graph vertices is connected by an edge, i.e. $x, y \in E$ for all distinct $x,y \in V$.

The complete graph with n graph vertices is denoted as $K_n$. $K_n$ has $\binom{n}{2}$ edges.

\subsection{Independent Graphs}
An inedpendent graph, denoted $I_n$, is a graph with $xy \not\in E$ for all distinct pairs of vertices $xy \in V$

\subsection{Paths and Cycles}
$P_n = \{ x_1, x_2, \cdots x_n \}$ is a path of length $n-1$.

$C_n = \{ x_1, x_2, \cdots x_n, x_1 \}$ is a cycle with length $n$.

\subsection{Connected}
A graph $G$ is connected if every vertex can be reached by any other vertex in the graph, i.e a path exists connecting $v_1$ to $v_2$ for all $v_1,v_2 \in V$.

\subsection{Component}
A maximal connected subgraph of a disconnected graph.

\subsection{Acyclic / Forrests}
A graph that does not contain any cycles.

The number of components is a forrest is $|V| - |E|$

\subsection{Tree}
A connected acyclic graph.

\subsection{Spanning Tree}
If $G$ is a connected tree and $H$ is a spanning subgraph, then we can call $H$ a spanning tree.

\subsection{Degree Sum Formula}
The degree sum formula states that, given a graph $G=(V,E)$,
\begin{identity}
\[
\sum _{v\in V}\deg(v)=2|E|.
\]
\end{identity}

The formula implies that in any undirected graph, the number of vertices with odd degree is even, which is known as the Handshaking lemma. The name comes from a popular problem: in any group of people the number of people who have shaken hands with an odd number of other people from the group must be even.

\subsection{Eulerian Cycles and Graphs}
A Eulerian cycle is a cycle that contains every edge in a given graph exactly once. A Eulerian graph is a graph that contains a Eulerian cycle.

\begin{theorem}
A graph admits a Eularian cycle if and only if all vertices have even degree
\end{theorem}

\subsection{Hamiltonian Cycles and Graphs}
A Hamiltonian cycle is a cycle that traverses each vertex exacltly once. A Hamiltonian graph is a graph that contains a Hamiltonian cycle.

\begin{theorem}
If a graph has $n$ vertices and $\forall v \in V, \deg(v) \geq  \lceil \frac{n}{2} \rceil$, then it must be Hamiltonian.
\end{theorem}

\begin{theorem}
(Ore's Theorem) If for all non-adjacent vertices $v,w \in V$, $deg(v) + deg(w) \geq n$, then $G$ must be Hamiltonian.
\end{theorem}

\subsection{Bipartite Graphs}
A graph is bipartite if there exists a partition of $V$ so that the resulting induced subgraphs are independent.

A graph is complete bipartite, denoted $K_{m,n}$, if it is bipartite with $V = V_1 \cup V_2, |V_1| = m, |V_2|= n$ and has edges $xy$ for all $x \in V_1, y \in V_2$.

\subsection{Graph Coloring}
A graph can be colored by assigning a color or label to each of its vertices. A proper coloring exists when no adjacent vertices have the same color.

\subsection{Chromatic Number}
The chromatic number of a graph, denoted $\chi (G)$, is the smallest number of $k$ colors so that $G$ is k-colorable.

\begin{proposition}
    Recall $C_n$ is a cycle of $n$ vertices. Then, $\chi (C_n) = \begin{cases}
2 ,\ \ \text{if } n \text{ is even} \\
3,\ \  \text{if } n \text{ is odd }
\end{cases}$
\end{proposition}

\subsection{Clique Number}
The clique number of a graph, denoted $\omega(G)$, is the maximum number $n$ so that $G$ has a subgraph isomorphc to the complete graph $K_m$.

\begin{proposition}
    For any graph $G$, $ \chi(G) \geq \omega(G)$.
\end{proposition}

If $ \chi(G) = \omega(G)$, then $G$ is called perfect or reduced.

\begin{proposition}
    For any $t \geq 3$, there exists a graph $G$ so that $\omega(G) = 2$ and $\chi(G) = t$.
\end{proposition}

This means that graphs can have an arbitrarily large chromatic number. If the original graph is triangle-free, a Mycielskian graph may be constructed to increase the graph's chromatic number.

\subsection{Planar Graph}
A graph is planar if it has a drawing or embedding so that its edges only intersect one another at one of the graph's vertices.

\subsection{(Euler's formula)}
\begin{theorem}
Given a finite, connected, planar graph where $v$ is the number of vertices, $e$ is the number of edges and $f$ is the number of faces,
\[
    v-e+f=2.
\]
\end{theorem}

\begin{theorem}
A planar graph on $n$ verticies has at most $3n-6$ edges, for $n \geq 3$.
\end{theorem}

\subsection{Kuratowski's Theorem}
\begin{theorem}
A graph is planar if and only if it does not contain a subgraph homeomorphic to $K_5$ or $K_{3,3}$.
\end{theorem}

\subsection{Four Color Theorem}
\begin{theorem}
Every planar graph has chromatic number of at most 4.
\end{theorem}

\subsection{Petersen Graph}
A Petersen graph has the following properties: $|V| = 10, |E| = 15$, there are no cycles of length 3 or 4, and $\deg(v) = 3, \forall v \in V$. A Petersen graph is never Hamiltonian.

\subsection{Dense Graph}
A dense graph is a graph in which the number of edges is close to the maximal number of edges. For undirected simple graphs, the graph density is:
\[
D={\frac {|E|}{\binom {|V|}{2}}}={\frac {2|E|}{|V|(|V|-1)}}
\]
\subsection{Sparse Graph}
Conversely, a sparse graph is a graph in which the number of edges is close to the minimal number of edges. A sparse graph can be a disconnected graph.

\section{Generating Functions}
Given a sequence $\sigma=\{a_n:n\ge0\}$ of real numbers, we define the ordinary generating function $F(x)$ of the sequence $\sigma$ as,
\[
F(x)=\sum_{n=0}^\infty a_n x^n.
\]

This differs from a formal power series in that we are not concerned with the series convergence or divergence and are generally not interested in substituting a value of $x$ and obtaining a specific value for $F(x)$. Instead we may use $F(x)$ to encode and generate the sequence $\sigma$.

\begin{identity} The geometric series:
\[
\frac {1}{1-x}=\sum _{n=0}^{\infty }x^{n}=1+x+x^{2}+x^{3}+\cdots
\]
\end{identity}

\subsection{Newton's Binomial Theorem}
\begin{theorem}
For all real $p$ with $p\neq 0$,
\[
(1+x)^p=\sum_{n=0}^\infty\binom{p}{n}x^n.
\]
\end{theorem}

\subsection{Partitions of an Integer}
A partition $P$ of an integer is a collection of (not necessarily distinct) positive integers such that,
\[
\sum_{i\in P} i = n.
\]
We can find the number of partitions of a given integer using a generator function
\[
P(x) = \left(\sum_{m=0}^\infty x^{m}\right)\left(\sum_{m=0}^\infty x^{2m}\right)         \left(\sum_{m=0}^\infty x^{3m}\right)\cdots         \left(\sum_{m=0}^\infty x^{km}\right)\cdots = \prod_{m=1}^\infty\frac{1}{1-x^m}.
\]
\begin{theorem}
For each $n\geq 1$, the number of partitions of $n$ into distinct parts is equal to the number of partitions of $n$ into odd parts.
\end{theorem}

\subsection{Exponential Generating Function}
The exponential generating function for the sequence $\{a_n\colon n\geq 0\}$ is $\displaystyle \sum_n \frac{a_n x^n}{n!}$.

\begin{identity}The exponential function formula:
\[
e^{x} =\sum _{n=0}^{\infty }{\frac {x^{n}}{n!}}=1+x+{\frac {x^{2}}{2!}}+{\frac {x^{3}}{3!}}+\cdots.
\]
\end{identity}

\section{Recurrence Equations}
A recurrence tells us how to compute the $n$th term of a sequence given some number of previous values. A recursion is linear when it has the following form,
\[
c_0a_{n+k}+ c_1a_{n+k-1} + \dots+c_ka_{n} = g(n),
\]
where $k \geq 1$ is an integer, $c_0,c_1,\dots,c_k$ are constants with $c_0, c_k \neq 0$, and $g: \mathbb Z \rightarrow \mathbb R$ is a function.

A linear equation is homogeneous if the function $g(n)$ is the zero function is non-homogeneous otherwise. The process of solving recurrence equations is similar to solving ordinary differential equations as both functions express future states in relation to differences to their previous states.

\subsection{Advancement Operator}
The advancement operator is a linear map that shifts or advances a function's input forward by one, i.e. $A: \mathbb R \mapsto \mathbb R$, where $(Af)(n) = f(n+1)$.

It is commutative and has the properties that $A^k =  \underbrace{A \circ \cdots \circ A }_\text{k times}$ and $(A^k f)(n) = f(n+k)$.

\subsection{Finding Closed-Form Expressions}
\begin{theorem}
The set of all solutions to the homogeneous linear equation of degree $k$ is a $k$-dimensional subspace of $V$.
\end{theorem}

This means that we may solve a linear recurrence stated in terms of advancement operations by finding a basis for the vector space of solutions.

\begin{lemma}
    Let $r \neq 0$ , and let $f$ be a solution to the operator equation $(A - r)f=0$. If $c=f(0)$, then $f(n)=c r^n$ for every $n \in \mathbb Z$.
\end{lemma}

This means that the roots of the advancement operator equation will give us solutions to its corresponding recurrence of the form $f(n) = c r^n$.

\begin{lemma}
Consider a non-homogeneous operator equation and let $W$ be the subspace of $V$ consisting of all solutions to its corresponding homogeneous equation.

If $f_0$ is a solution to the homogeneous equation, then every solution $f$ to has the form $f = f_0 + f_1$ where $f_1 \in W$ is a particular solution.
\end{lemma}

If the  polynomial in the advancement operator has distinct roots we may use the following theorem,

\begin{theorem}
Consider the following advancement operator equation $p(A) f = (A-r_1)(A-r_2)\cdots(A-r_k)f=0$. with $r_1 , r_2 , \cdots , r_k$ distinct non-zero constants. Then every solution has the form $f(n) = c_1 r_1^n + c_2 r_2^n + c_3 r_3^n + \cdots + c_k r_k^n$.
\end{theorem}

If there are repeated roots we use the following theorem,
\begin{theorem}
Let $k \geq 1$ and consider the equation $( A - r )^k f = 0$. Then the general solution has the form $f(n) = c_1 r_n + c_2 n r_n + c_3 n^2r^n + c_4 n^3 r^n + \cdots + c_k n^{k - 1} r ^n$.
\end{theorem}

\subsection{Using Generating Functions}
Alternatively, we may use generating function to solve recursive equations. This approach does not require determining an appropriate form for the particular solution but instead the method of generating functions often requires that the resulting generating function be expanded using partial fractions.

Generating functions can be used in solving certain nonlinear recurrence equation, in particular, we can use it to determine the number of rooted, unlabeled, binary, ordered trees.  A tree is rooted if we have designated a special vertex called its root. An unlabeled tree is one in which we do not make distinctions based upon names given to the vertices. A binary tree is one in which each vertex has 0 or 2 children, and an ordered tree is one in which the children of a vertex have some ordering (first, second, third, etc.).

\begin{theorem}
The generating function for the number $c_n$  of rooted, unlabeled, binary, ordered trees with $n$ leaves is
\[
C(x) = \frac{1-\sqrt{1-4x}}{2} = \sum_{n=1}^\infty \frac{1}{n}\binom{2n-2}{n-1}x^n.
\]
\end{theorem}
Notice that $c_n$ is a Catalan number, which also counts the number of lattice paths that do not cross the diagonal line $y=x$.

\begin{thebibliography}{}
\bibitem{}
Keller, M.T. and Trotter, W.T., Applied Combinatorics, Open Textbook Library, ISBN9781534878655. http://www.rellek.net/book/app-comb.html

\end{thebibliography}
\end{document}


\section{Comparison}\label{sec:comparison}
In this section we discuss comparison operators that can be used to compare objects in JavaScript.

\subsection{Equality operators}
There are two types of Equality operators called Equality, and strict equality. They are \texttt{==} and \texttt{===} respectively.

The equality operator converts the operands if they are not of the same type. Here is an example.

\begin{lstlisting}
> 1 == '1'
true
\end{lstlisting}

The \texttt{number} operator is casted to string for the equality check. When we use this equality operator the operands do need to be of the same type.

The strict equality on the other hand will not evaluate anything to \texttt{true} unless they are of the same type.

\begin{lstlisting}
> 1 === '1'
false
\end{lstlisting}

Note that when you compare mutable objects, such as Arrays and Objects your a comparing the pointer to the object. They will not evaluate to \texttt{true} even if they are the same. Only if their pointers are the same. When you have immutable objects, such as numbers and booleans, their value are compared.

When in doubt which one to use, always use strict equality. In matter of fact, always use strict equality unless you know what you are doing.

The negation equality check works the same, there we have \texttt{!=} and \texttt{!==}. The follow the same rules as we described above.

\subsection{Relational operators}
JavaScript supports the same relational operators as in C or Java. They are explained with examples in the following listing.

\begin{lstlisting}
> 2 > 1
true
> 3 <= 3
true
> 3 <= 3 && 2 != 2
false
\end{lstlisting}


  \section{Algorithms and their performance}
\label{sec:algorithm}

Waveform analysis is to obtain $t_i$ and $q_i$ estimators $\hat{t}_i$ and $\hat{q}_i$ from waveform $w(t)$, where the output indices $i$ are from 1 to $\hat{N}$ and $\hat{N}$ is an estimator of $N_\mathrm{PE}$ in eq.~\eqref{eq:lc-sample}. Figure~\ref{fig:io} illustrates the input $w(t)$ and the outputs $\bm{\hat{t}}, \hat{\bm{q}}$, where boldface $\hat{\bm{t}}$ denotes the vector $\hat{t}_i$.
\begin{figure}[H]
  \centering
  \begin{subfigure}{.45\textwidth}
    \resizebox{\textwidth}{!}{\input{figures/wave.pgf}}
    \caption{\label{fig:input} Input $w(t)$ reproduced from figure~\ref{fig:pile}.}
  \end{subfigure}
  \begin{subfigure}{.45\textwidth}
    \resizebox{\textwidth}{!}{\input{figures/charge.pgf}}
    \caption{\label{fig:output} Output $\hat{\bm{t}}, \hat{\bm{q}}$ obtained from \subref{fig:input}.}
  \end{subfigure}
  \caption{\label{fig:io} Waveform analysis takes PMT waveform as input in~\subref{fig:input} and outputs times $\hat{\bm{t}}$ and charges $\hat{\bm{q}}$ of the PEs in~\subref{fig:output}. }
\end{figure}

The fluctuation of $q_i$ makes $\hat{t}_i$ ambiguous and $\hat{N}$ fail to estimate $N_\mathrm{PE}$. For example, 1, 2 and even 3~PEs can generate exactly the same charge as $1.6$ units.  A single PE charged $1$ may be mis-interpreted as 2~PEs at consecutive $\hat{t}_i$ and $\hat{t}_{i+1}$ with $\hat{q}_i=\hat{q}_{i+1}=0.5$.

\subsection{Evaluation criteria}
\label{sec:criteria}
Subject to such ambiguity of $t_i/q_i$, we introduce a set of evaluation criteria to assess the algorithms' performance.

\subsubsection{Kullback-Leibler divergence}
\label{sec:pseudo}

We construct a light curve estimator $\hat{\phi}(t)$ from $\bm{\hat{t}}$, $\bm{\hat{q}}$ and $\hat{N}$,
\begin{equation}
  \label{eq:lc}
  \hat{\phi}(t) = \sum_{i=1}^{\hat{N}} \hat{q}_i\delta(t-\hat{t}_i),
\end{equation}
which resembles eq.~\eqref{eq:lc-sample}.
Consider the non-normalized Kullback-Leibler~(KL) divergence~\cite{mihoko_robust_2002} between $\hat{\phi}(t)$ and $\mu \phi(t-t_{0})$,
\begin{equation}
  \begin{aligned}
    D_\mathrm{KL}\left[\hat{\phi}(t) \parallel \mu\phi(t-t_0)\right] & =\int \left[\hat{\phi}(t) \log\frac{\hat{\phi}(t)}{\mu\phi(t-t_0)} + \mu\phi(t-t_0) - \hat{\phi}(t) \right]\mathrm{d}t \\
    & = - \int \hat{\phi}(t) \log\phi(t-t_0)\mathrm{d}t - \log\mu\int\hat{\phi}(t)\mathrm{d}t + \mu + \int \left[\hat{\phi}(t) \log\hat{\phi}(t) - \hat{\phi}(t) \right]\mathrm{d}t \\
    & = - \sum_{i=1}^{\hat{N}}\int \hat{q}_i\delta(t-\hat{t_i}) \log\phi(t-t_0)\mathrm{d}t - \log\mu\int\hat{q}_i\delta(t-\hat{t_i})\mathrm{d}t + \mu +  C \\
    & = -\log \prod_{i=1}^{\hat{N}} \left[\phi(\hat{t}_i-t_0)\right]^{\hat{q}_i} - \log\mu\sum_{i=1}^{\hat{N}} \hat{q}_i + \mu + C
  \label{eq:kl}
  \end{aligned}
\end{equation}
where $C$ is a constant regarding $t_0$ and $\mu$.  Define the time KL estimator as
\begin{equation}
  \begin{aligned}
  \label{eq:pseudo}
  \hat{t}_\mathrm{KL} &= \arg\underset{t_0}{\min}~D_\mathrm{KL}\left[\hat{\phi}(t) \parallel \mu\phi(t-t_0)\right] \\
  &= \arg\underset{t_0}{\max} \prod_{i=1}^{\hat{N}} \left[\phi(\hat{t}_i-t_0)\right]^{\hat{q}_i},
  \end{aligned}
\end{equation}
which reduces to an MLE like eq.~\eqref{eq:2} if $\hat{q}_i\equiv 1$.  $\hat{t}_\mathrm{KL}$ estimates $t_0$ when $t_i, q_i, N_\mathrm{PE}$ are all uncertain.
Denoting $\Delta t_0$ as $\hat{t}_\mathrm{KL} - t_0$, the standard deviation $\sigma_\mathrm{KL}$ of $\Delta t_0$ on a batch of waveforms is the resolution of an algorithm.

The intensity KL estimator is,
\begin{equation}
  \label{eq:pseudo-mu}
  \hat{\mu}_\mathrm{KL} = \arg\underset{\mu}{\min}~D_\mathrm{KL}\left[\hat{\phi}(t) \parallel \mu\phi(t-t_0)\right] = \sum_{i=1}^{\hat{N}} \hat{q}_i.
\end{equation}

\subsubsection{Residual sum of squares}
\label{sec:rss}

Following eqs.~\eqref{eq:1} and~\eqref{eq:lc}, we construct an estimator of a waveform,
\begin{equation}
  \label{eq:w-hat}
  \hat{w}(t) = \sum_{i=1}^{\hat{N}}\hat{q}_i V_\mathrm{PE}(t-\hat{t}_i) = \hat{\phi}(t) \otimes V_\mathrm{PE}(t).
\end{equation}

For a noise-free evaluation of $\hat{w}(t)$, residual sum of squares~(RSS) is a $\ell_2$-distance of it to $\tilde{w}(t)$,
\begin{equation}
  \label{eq:rss}
  \mathrm{RSS} \coloneqq\int\left[\hat{w}(t) - \tilde{w}(t)\right]^2\mathrm{d}t.
\end{equation}

Figure~\ref{fig:l2} demonstrates that if two functions do not overlap, the $\mathrm{RSS}$ of them remains constant regardless of their relative positions.  The sampled light curves $\hat{\phi}(t)$ and $\tilde{\phi}(t)$ consist with delta functions and hardly overlap, therefore the $\mathrm{RSS}$ of them is useless.  Futhermore, RSS cannot compare a discrete function with a continuous one.  We shall only consider the $\mathrm{RSS}$ of waveforms.

\begin{figure}[H]
  \centering
  \resizebox{0.6\textwidth}{!}{\input{figures/tab.pgf}}
  \caption{\label{fig:l2} The $\mathrm{RSS}$ of red and blue curves is a function of the two shaded regions. It is a constant when the curves shift horizontally provided they do not overlap.  In contrast, the Wasserstein distance $D_\mathrm{w}$ of the two curves is associated with the distance between them.  It complements $\mathrm{RSS}$ and offers a time-sensitive metric suitable for the sparse PE space.}
\end{figure}

\subsubsection{Wasserstein distance}
\label{sec:W-dist}

\input{fom}

In the following, we  evaluate the performance of waveform analysis algorithms ranging from heuristics, deconvolution, neuron network to regression, by the criteria discussed in this section.

\subsection{Heuristic methods}
By directly extracting waveform features, heuristic methods are straightforward and widely used.

\subsubsection{Waveform shifting}
\label{sec:shifting}
By \textit{waveform shifting}, we extract more time information than TDC in a similar manner: select all the $t_i$ where the waveform $w(t_i)$ exceeds a threshold to suppress noise, shift them by $\Delta t$ according to the shape of single PE response $V_\mathrm{PE}(t)$ in eq.~\eqref{eq:dayaspe} to get $\hat{t}_i$, then take $\hat{q}_i \propto w(\hat{t}_i)$ so that the total charge $\sum \hat{q}_i$ aligns with the integration of the waveform $w(t)$.  Though the $\hat{q}_i$'s obtained are smaller than PE charges as in figure~\ref{fig:shifting}, waveform shifting does minimal analysis and serves as a baseline method.

\begin{figure}[H]
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/threshold.pgf}}
    \caption{\label{fig:shifting} A waveform shifting example gives \\ $\Delta t_0=\SI{-0.69}{ns}$, $\mathrm{RSS}=\SI{369.6}{mV^2}$, $D_\mathrm{w}=\SI{3.11}{ns}$.}
  \end{subfigure}
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/findpeak.pgf}}
    \caption{\label{fig:peak} A peak finding example gives \\ $\Delta t_0=\SI{-1.93}{ns}$, $\mathrm{RSS}=\SI{266.9}{mV^2}$, $D_\mathrm{w}=\SI{2.40}{ns}$.}
  \end{subfigure}
  \caption{\label{fig:method}Demonstrations of heuristic methods on a waveform sampled from $\mu=4$, $\tau_\mathrm{l}=\SI{20}{ns}$, $\sigma_\mathrm{l}=\SI{5}{ns}$ light curve conditions.  Peak finding in~\subref{fig:peak} handles charges more realistically than waveform shifting in~\subref{fig:shifting}, giving better numbers by the $\mathrm{RSS}$ and $D_\mathrm{w}$ criteria in section \ref{sec:criteria}.}
\end{figure}

\subsubsection{Peak finding}
\label{sec:findpeak}

\textit{Peak finding} is more effective than waveform shifting by exploiting PE pulses.  We smooth the waveforms by a low-pass Savitzky-Golay filter~\cite{savitzky_smoothing_1964} and find all the peaks at $t_i$.  The following resembles waveform shifting: apply a shift to get $\hat{t}_i$, take $\hat{q}_i \propto w(\hat{t}_i)$ and normalize to the waveform integration. As shown in figure~\ref{fig:peak}, peak finding outputs charges close to 1 and works well for lower PE counts.  But when PEs pile up closely, peaks overlap intensively, making this method unreliable.

\subsection{Deconvolution}
\label{sec:deconv}
Deconvolution methods contribute more than heuristic ones by using the full shape of $V_\mathrm{PE}(t)$, thus can accommodate overshoots and pile-ups.  Smoothing is necessary since deconvolution does not model the white noise.

\subsubsection{Fourier deconvolution}
\textit{Fourier deconvolution} is an option considered by JUNO prototype~\cite{zhang_comparison_2019}. The deconvolution relation is evident in the Fourier transform $\mathcal{F}$ to eq.~\eqref{eq:1},
\begin{equation}
  \label{eq:fourier}
  %\begin{aligned}
  \mathcal{F}[w]  = \mathcal{F}[\tilde{\phi}]\mathcal{F}[V_\mathrm{PE}] + \mathcal{F}[\epsilon]
  \implies \mathcal{F}[\tilde{\phi}]  = \frac{\mathcal{F}[w]}{\mathcal{F}[V_\mathrm{PE}]} - \frac{\mathcal{F}[\epsilon]}{\mathcal{F}[V_\mathrm{PE}]}.
  %\end{aligned}
\end{equation}
By low-pass filtering the waveform $w(t)$ to get $\tilde{w}(t)$, we suppress the noise term $\epsilon$, take the inverse Fourier transform $\hat{\phi}_1(t) = \mathcal{F}^{-1}\left[\frac{\mathcal{F}[\tilde{w}]}{\mathcal{F}[V_\mathrm{PE}]}\right](t),$ and compute $\hat{\phi}(t)$ as the over-threshold $q_\mathrm{th}$ part of $\hat{\phi}_1(t)$,
\begin{equation}
  \label{eq:fdconv2}
    \hat{\phi}(t) = \hat{\alpha}\underbrace{\hat{\phi}_1(t) I\left(\hat{\phi}_1(t) - q_\mathrm{th}\right)}_{\text{over-threshold part of} \hat{\phi}_1(t)}
\end{equation}
where $I(x)$ is the indicator function, and $\hat{\alpha}$ is a scaling factor to minimize $\mathrm{RSS}$,
\begin{equation*}
  \begin{aligned}
  \label{eq:id}
  I(x) = \left\{
    \begin{array}{ll}
      1 & \mbox{, if $x\ge0$}, \\
      0 & \mbox{, otherwise}
    \end{array}
    \right.
    \quad~~~
    \hat{\alpha} = \arg \underset{\alpha}{\min}\mathrm{RSS}\left[\alpha\hat{w}(t),w(t)\right]. \\
  \end{aligned}
\end{equation*}
$\tilde{w}(t)$ unknown in data analysis, we replace it in eq.~(\ref{eq:rss}) with $w(t)$.

Figure~\ref{fig:fd} illustrates that Fourier deconvolution outperforms heuristic methods.  Meanwhile, noise and precision loss in the waveform lead to smaller and even negative $\hat{q}_i$. Those can be mitigated by thresholding and scaling in eq.~\eqref{eq:fdconv2}, but calls for a more elegant solution.

\begin{figure}[H]
  \begin{subfigure}{0.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/fftrans.pgf}}
    \caption{\label{fig:fd} A Fourier deconvolution example: \\ $\Delta t_0=\SI{-1.16}{ns}$, $\mathrm{RSS}=\SI{124.7}{mV^2}$, $D_\mathrm{w}=\SI{2.03}{ns}$.}
  \end{subfigure}
  \begin{subfigure}{0.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/lucyddm.pgf}}
    \caption{\label{fig:lucy} A Richardson-Lucy direct demodulation example:\\ $\Delta t_0=\SI{-0.75}{ns}$, $\mathrm{RSS}=\SI{70.3}{mV^2}$, $D_\mathrm{w}=\SI{1.10}{ns}$.}
  \end{subfigure}
  \caption{\label{fig:deconv}Demonstrations of deconvolution methods on a waveform sampled from the same setup as figure~\ref{fig:method}. Richardson-Lucy direct demodulation in~\subref{fig:lucy} imposes positive charges in iterations and obtains better results than Fourier deconvolution in~\subref{fig:fd}.}
\end{figure}

\subsubsection{Richardson-Lucy direct demodulation}
\label{sec:lucyddm}

\textit{Richardson-Lucy direct demodulation}~(LucyDDM)~\cite{lucy_iterative_1974} with a non-linear iteration to calculate deconvolution has a wide application in astronomy~\cite{li_richardson-lucy_2019} and image processing. We view $V_{\mathrm{PE}*}(t-s)$ as a conditional probability distribution $p(t|s)$ where $t$ denotes PMT amplified electron time and $s$ denotes the given PE time. By the Bayesian rule,
\begin{equation}
  \label{eq:lucy}
  \tilde{\phi}_*(s) V_{\mathrm{PE}*}(t-s) = \tilde{\phi}_*(s)p(t|s) = p(t,s) = \tilde{w}_*(t)p(s|t),
\end{equation}
where $p(t, s)$ is the joint distribution of amplified electron $t$ and PE time $s$, and $\tilde{w}$ is the smoothed $w$.  Cancel out the normalization factors,
\begin{equation}
  \label{eq:ptt}
  p(s|t) = \frac{\tilde{\phi}_*(s) V_{\mathrm{PE}*}(t-s)}{\tilde{w}_*(t)} = \frac{\tilde{\phi}(s) V_{\mathrm{PE}}(t-s)}{\int\tilde{\phi}(s') V_{\mathrm{PE}}(t-s')\mathrm{d}s'}.
\end{equation}
Then a recurrence relation $\phi_*$ is,
\begin{equation}
  \label{eq:iter}
  \begin{aligned}
    \tilde{\phi}_*(s) & = \int p(s|t) \tilde{w}_*(t)\mathrm{d}t = \int \frac{\tilde{\phi}(s) V_{\mathrm{PE}}(t-s)}{\int\tilde{\phi}(s') V_{\mathrm{PE}}(t-s')\mathrm{d}s'} \tilde{w}_*(t) \mathrm{d}t \\
    \implies \hat{\phi}^{n+1}(s) & = \int \frac{\hat{\phi}^n(s) V_{\mathrm{PE}*}(t-s)}{\int\hat{\phi}^n(s') V_{\mathrm{PE}}(t-s')\mathrm{d}s'} \tilde{w}(t) \mathrm{d}t,
  \end{aligned}
\end{equation}
where only $V_{\mathrm{PE}*}$ in the numerator is normalized and superscript $n$ denotes the iteration step.
Like Fourier deconvolution in eq.~\eqref{eq:fdconv2}, we threshold and scale the converged $\hat{\phi}^\infty$ to get $\hat{\phi}$.  As shown in figure~\ref{fig:lucy} the positive constraint of $\hat{\phi}$ makes LucyDDM more resilient to noise.

The remaining noise in the smoothed $\tilde{w}$ crucially influence deconvolution.  A probablistic method will correctly model the noise term $\epsilon$, as we shall see in section \ref{sec:regression}.  Before that let's try something different.

\subsection{Convolutional neural network}
\label{sec:cnn}
% Network structure
Convolutional neural networks~(CNN) made breakthroughs in various fields like computer vision~\cite{he_deep_2016} and natural language processing~\cite{vaswani_attention_2017}. As an efficient composition of weighted additions and non-linear functions, neural networks outperform many traditional algorithms.

We choose a shallow network structure of 5 layers to recognize patterns as shown in figure~\ref{fig:struct} motivated by the pulse shape and universality of $V_\mathrm{PE}(t)$ for all the PEs.  The convolutional widths are selected considering the localized nature of $V_\mathrm{PE}(t)$.

The workflow of data processing consists of training and predicting. We find a mapping from waveform $w(t)$ to PE $\tilde{\phi}(t)$ with backpropagation method and supervised learning. We formulate the training loss as optimizing $D_\mathrm{w}[\hat{\phi}, \tilde{\phi}] $, the Wasserstein distance between the truth $\tilde{\phi}$ and predicted $\hat{\phi}$. As discussed in section~\ref{sec:W-dist}, $D_w$ can handle the PE sparsity in training iterations. Figure~\ref{fig:loss} shows the convergence of Wasserstein distance during training.

The output of CNN is scaled by $\hat{\alpha}$ following eq.~\eqref{eq:fdconv2} to get $\hat{\phi}$.

\begin{figure}[H]
  \begin{subfigure}{.4\textwidth}
    \centering
    \begin{adjustbox}{width=0.5\textwidth}
      \input{model}
    \end{adjustbox}
    \caption{\label{fig:struct} Structure of the neural network.}
  \end{subfigure}
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/epoch.pgf}}
    \caption{\label{fig:loss} Evolution of loss.}
  \end{subfigure}
  \caption{\label{fig:CNN} Training process of a CNN. A shallow network structure of 5 layers in~\subref{fig:struct} is trained to converge in Wasserstein distance as shown in~\subref{fig:loss}.}
\end{figure}

\vspace{-0.5cm}
In figure~\ref{fig:cnn-npe}, we use box plot to describe $D_\mathrm{w}$ distributions. From figure~\ref{fig:cnn-npe}, we find $D_w$ is the smallest for one PE.  $D_w$ stops increasing with $N_\mathrm{PE}$ at about 6 PEs, where the PE times are the most challenging to extract.  When $N_\mathrm{PE}$ is more than 6, pile-ups tend to produce a continuous waveform and the average PE time accuracy stays flat. Thus waveform analysis is the most important in recovering time accuracy for PEs less than 10.

Such small $D_\mathrm{w}$ in figure~\ref{fig:cnn-npe} provides a precise matching of waveforms horizontally to guarantee effective $\hat{\alpha}$ scaling, explaining why $\mathrm{RSS}$ is also small in figure~\ref{fig:cnn}.

\begin{figure}[H]
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/takarachargestats.pgf}}
    \caption{\label{fig:cnn-npe} $D_\mathrm{w}$ histogram and its distributions conditioned \\ on $N_{\mathrm{PE}}$. ``arbi. unit'' means arbitrary unit.}
  \end{subfigure}
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/takara.pgf}}
    \caption{\label{fig:cnn}An example giving \\ $\Delta t_0=\SI{-3.05}{ns}$, $\mathrm{RSS}=\SI{10.0}{mV^2}$, $D_\mathrm{w}=\SI{0.64}{ns}$.}
  \end{subfigure}
  \caption{\label{fig:cnn-performance}Demonstration of CNN on $\num[retain-unity-mantissa=false]{1e4}$ waveforms in~\subref{fig:cnn-npe} and one waveform in~\subref{fig:cnn} sampled from the same setup as figure~\ref{fig:method}.  In figure~\subref{fig:cnn-npe}, the middle line is the median of distribution. The upper and lower hinge are the first quartile $Q_1$ at \SI{25}{\percent} and the third quartile $Q_3$ at \SI{75}{\percent}. The upper and lower whiskers are $Q_1 \pm 1.5(Q_3-Q_1)$. }
\end{figure}

\subsection{Regression analysis}
\label{sec:regression}
With the generative waveform model in eq.~\eqref{eq:1}, regression is ideal for analysis. Although computational complexity hinders the applications of regression by the vast volumes of raw data, the advancement of sparse models and big data infrastructures strengthens the advantage of regression.

We replace $\hat{N}$ with a fixed sample size $N_\mathrm{s}$ and $\hat{t}_i$ with a fixed grid of times $t'_i$ in eq.~\eqref{eq:w-hat},
\begin{equation}
  \label{eq:gd}
  w'(t) = \sum_{i=1}^{N_\mathrm{s}}q'_iV_\mathrm{PE}(t-t'_i).
\end{equation}
When $\{t'_i\}$ is dense enough, $\{\hat{q}_i\}$ determines the inferred PE distribution $\hat{\phi}(t)$,
\begin{equation}
  \label{eq:gd-phi}
  \hat{\phi}(t) = \sum_{i=1}^{N_\mathrm{s}}\hat{q}_i\delta(t-t'_i).
\end{equation}
From the output $\hat{\phi}_\mathrm{dec}(t)$ of a deconvolution method in section~\ref{sec:lucyddm}, we confidently leave out all the $t'_i$ that $\hat{\phi}_\mathrm{dec}(t_i')=0$ in eq.~\eqref{eq:gd-phi} to reduce the number of parameters and the complexity.

We attempted to replace the dense $\bm{t'}$ grid in eq.~\eqref{eq:gd} with a length-varying vector of sparse PEs. However, the truth $N_\mathrm{PE}$ is unknown and formulating an explicit trans-dimensional model is expansive.  We shall leave this possibility to the future.

\subsubsection{Direct charge fitting}
\label{sec:dcf}

Fitting the charges $q'_i$ in eq.~\eqref{eq:gd} directly by minimizing RSS of $w'(t)$ and $w(t)$, we get
\begin{equation}
  \label{eq:gd-q}
  \bm{\hat{q}} = \arg \underset{q'_i \ge 0}{\min}~\mathrm{RSS}\left[w'(t),w(t)\right].
\end{equation}
Slawski and Hein~\cite{slawski_non-negative_2013} prove that in deconvolution problems, the non-negative least squares formulation in eq.~\eqref{eq:gd-q} is self-regularized and gives sparse solutions of $q'_i$.  We optimize eq.~\eqref{eq:gd-q} by Broyden-Fletcher-Goldfarb-Shanno algorithm with a bound constraint~\cite{byrd_limited_1995}.  In figure~\ref{fig:fitting-npe} charge fitting is consistent in $D_\mathrm{w}$ for different $N_\mathrm{PE}$'s, showing its resilience to pile-up.

\begin{figure}[H]
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/xiaopeipchargestats.pgf}}
    \caption{\label{fig:fitting-npe} $D_\mathrm{w}$ histogram and its distributions conditioned \\ on $N_{\mathrm{PE}}$, boxplot explained in figure~\ref{fig:cnn-performance}.}
  \end{subfigure}
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/xiaopeip.pgf}}
    \caption{\label{fig:fitting}An example giving \\ $\Delta{t_0}=\SI{-3.23}{ns}$, $\mathrm{RSS}=\SI{7.64}{mV^2}$,$D_\mathrm{w}=\SI{0.68}{ns}$.}
  \end{subfigure}
  \caption{\label{fig:dcf}Demonstration of direct charge fitting with $\num[retain-unity-mantissa=false]{1e4}$ waveforms in~\subref{fig:fitting-npe} and one waveform in~\subref{fig:fitting} sampled from the same setup as figure~\ref{fig:method}.  Direct charge fitting shows a better performance than LucyDDM in figure~\ref{fig:lucy} and a comparable $D_\mathrm{w}$ to CNN in figure~\ref{fig:cnn}.}
\end{figure}

In figure~\ref{fig:fitting}, sparsity of $q'_i$ is evident as speculated.  However, the majority of the results $\hat{q}_i$ is less than 1.  That motivates us to incorporate prior knowledge of $q'_i$ towards a sparser and performant model than directly fitting charges.

\subsubsection{Markov chain Monte Carlo}
\label{subsec:mcmc}
Chaining the $q'_i$ distribution~(section~\ref{subsec:spe}), the charge fitting eq.~\eqref{eq:gd-q} and the light curve eq.~\eqref{eq:time-pro}, we arrive at a hierarchical Bayesian model,
\begin{equation}
  \begin{aligned}
    t_{0} &\sim \mathcal{U}(0, \overline{t_0}) \\
    \mu_i &= \mu \int_{t'_i-\frac{\Delta t'}{2}}^{t'_i+\frac{\Delta t'}{2}} \phi(t' - t_0)\mathrm{d}t' \approx \mu\phi(t'_i - t_0)\Delta{t'} \\
    z_i &\sim \mathcal{B}(\mu_i) \\
    q'_{i,0}&=0\\
    q'_{i,1}& \sim \mathcal{N}(1, \sigma_\mathrm{q})\\
    q'_i &= q'_{i,z_i}\\
    w'(t) & = \sum_{i=1}^{N_\mathrm{s}}q'_iV_\mathrm{PE}(t-t'_i)\\
    w(t) &\sim \mathcal{N}(w'(t), \sigma_\epsilon)
  \end{aligned}
  \label{eq:mixnormal}
\end{equation}
where $\mathcal{U}$ and $\mathcal{B}$ stands for uniform and Bernoulli distributions, $\overline{t_0}$ is an upper bound of $t_0$, and $q'_i$ is a mixture of 0 (no PE) and normally distributed $q'_{i,1}$ (1 PE). When expection $\mu_i$ of a PE hitting $(t'_{i} - \frac{\Delta t'}{2}, t'_{i} + \frac{\Delta t'}{2})$ is small enough, that 0-1 approximation is valid.  The inferred waveform $w'(t)$ differs from observable $w(t)$ by a white noise $\epsilon(t) \sim \mathcal{N}(0, \sigma_\epsilon)$ motivated by eq.~\eqref{eq:1}.  When an indicator $z_i=0$, it turns off $q'_i$, reducing the number of parameters by one.  That is how eq.~\eqref{eq:mixnormal} achieves sparsity.

We generate posterior samples of $t_0$ and $\bm{q'}$ by Hamiltonian Monte Carlo~(HMC)~\cite{neal_mcmc_2012}, a variant of Markov chain Monte Carlo that is suitable to high-dimensional problems. Construct $\hat{t}$ and $\hat{q}_i$ as the mean estimators of posterior samples $t_0$ and $q'_i$ at $z_i=1$.  Unlike the $\hat{t}_\mathrm{KL}$ discussed in section~\ref{sec:pseudo}, $\hat{t}_0$ is a direct Bayesian estimator from eq.~\eqref{eq:mixnormal}.  We construct $\hat{\phi}(t)$ by eq.~\eqref{eq:gd-phi} and $\hat{w}(t)$ by
\begin{equation}
  \label{eq:mcmc-w}
  \hat{w}(t) = \sum_{i=1}^{N_\mathrm{s}}\hat{q}_iV_\mathrm{PE}(t-t'_i).
\end{equation}
RSS and $D_\mathrm{w}$ are then calculated according to eqs.~(\ref{eq:rss}) and (\ref{eq:numerical}).

\begin{figure}[H]
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/mcmcchargestats.pgf}}
    \caption{\label{fig:mcmc-npe} $D_\mathrm{w}$ histogram and its distributions conditioned \\ on $N_{\mathrm{PE}}$, boxplot explained in figure~\ref{fig:cnn-performance}.}
  \end{subfigure}
  \begin{subfigure}{.5\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/mcmc.pgf}}
    \caption{\label{fig:mcmc}An example with \\ $\Delta{t_0}=\SI{-2.48}{ns}$, $\mathrm{RSS}=\SI{16.25}{mV^2}$, $D_\mathrm{w}=\SI{0.76}{ns}$.}
  \end{subfigure}
  \caption{\label{fig:mcmc-performance}Demonstration of MCMC with $\num[retain-unity-mantissa=false]{1e4}$ waveforms in~\subref{fig:mcmc-npe} and one waveform in~\subref{fig:mcmc} sampled from the same setup as figure~\ref{fig:method}.  Although using a more dedicated model, MCMC performs worse than direct charge fitting in figure~\ref{fig:dcf}. We suspect the Markov chain is not long enough.}
\end{figure}
Although we imposed a $\mathcal{N}(1, \sigma_\mathrm{q})$ prior, The charges $\hat{q}_i$ in figure~\ref{fig:fitting} are still less than 1.  The $D_\mathrm{w}$ marginal distribution in figure~\ref{fig:mcmc-npe} is less smooth than that of direct charge fitting in figure~\ref{fig:fitting-npe}.  Similarly, RSS in figure~\ref{fig:mcmc} is slightly worse than that in figure~\ref{fig:fitting}.  We suspect the markov chain has not finally converged due to the trans-dimensional property of eq.~\eqref{eq:mixnormal}.  Enlonging the burning and chain is not a solution because MCMC is already much slower than direct fitting in section~\ref{sec:dcf}.  We need an algorithm that pertains the model of eq.~\eqref{eq:mixnormal} but much faster than MCMC.

\subsubsection{Fast Bayesian matching pursuit}
\label{subsec:fbmp}
If we discretize waveform variable $t$ in eq.~\eqref{eq:mixnormal} into $t_j$, $w(t)$ becomes a vector $w_j \coloneqq w(t_j)$ and $V_\mathrm{PE}(t - t'_i)$ a matrix $V_{\mathrm{PE},ji} \coloneqq V_\mathrm{PE}(t_j - t'_i)$.  Such a formulation unveils a joint multivariate normal distribution of $\bm{w}$ and $\bm{q}'$ subject to $\bm{z}$,
\begin{equation}
  \label{eq:mgauss}
  \begin{aligned}
    \left.
      \begin{bmatrix}
        \bm{w} \\
        \bm{q}'
      \end{bmatrix}
    \right\vert\bm{z}
    &\sim \mathcal{N}\left(
      \begin{bmatrix}
        \bm{V}_\mathrm{PE}\bm{z} \\
        \bm{z}
      \end{bmatrix},
      \begin{bmatrix}
        \bm{\Sigma} & \bm{V}_\mathrm{PE}\bm{Z} \\
        \bm{Z}\bm{V}_\mathrm{PE}^\intercal & \bm{Z}
      \end{bmatrix}
    \right) \\
    \bm{Z} &= \sigma_\mathrm{q}^2 \begin{bmatrix}
      z_1 & & \\
      & \ddots \\
      && z_{N_\mathrm{s}}
    \end{bmatrix}\\
    \bm{\Sigma} &= \bm{V}_\mathrm{PE}\bm{Z}\bm{V}_\mathrm{PE}^\intercal+\sigma_\epsilon^2\bm{I}\\
    z_i &\sim \mathcal{P}(\mu_i) \\
    \mu_i &= \mu \int_{t'_i-\frac{\Delta t'}{2}}^{t'_i+\frac{\Delta t'}{2}} \phi(t' - t_0)\mathrm{d}t' \\
  \end{aligned}
\end{equation}
where $\mathcal{P}$ stands for Poisson distribution.  Unlike eq.~\eqref{eq:mixnormal}, $z_i$ can take non-negative integers larger than 1, encoding the number of PE's in the $i$-th time bin.  Marginalizing out $\bm{z}$,
\begin{equation}
  \begin{aligned}
    \label{eq:bayesianinter}
    p(\bm{w}|t_0, \mu) & = \sum_{\bm{z}}p(\bm{w}, \bm{z}|t_0,\mu) = \sum_{\bm{z}}p(\bm{w}|\bm{z})p(\bm{z}|t_0,\mu),\\
    \left(\hat{t}_0, \hat{\mu}\right) &= \arg\underset{t_0,\mu}{\max} ~p(\bm{w} | t_0, \mu),
  \end{aligned}
\end{equation}
gives the probability of $\bm{w}$ subject to $\mu$ and $t_0$ with the corresponding estimators. $p(\bm{w}|z)$ is in place of $p(\bm{w}|z, t_0, \mu)$ because $\bm{w}$ does not depend on $t_0$ or $\mu$ if $\bm{z}$ is given.  The summation over $\bm{z}$, however, takes exploding number of combinations. Only a few $\bm{z}$'s contribute significantly in eq.~\eqref{eq:bayesianinter}.  Schniter~et al.~\cite{schniter_fast_2008} propose fast Bayesian matching pursuit~(FBMP) to construct a dominating set $\Omega$ of $\bm{z}$ with non-negligible $p(\bm{w}|\bm{z})$.  In terms of variational inference~(e.g.~\cite{mackay_information_2003}), the dominating set $\Omega$ defines an approximation $p_\Omega(\bm{z})$ to $p(\bm{z}|\bm{w})$, dropping the context $t_0$ and $\mu$ for simplicity,

\begin{equation}
  \label{eq:VI}
  \begin{aligned}
    p_\Omega(\bm{z}) &= \begin{cases}
      \frac{p(\bm{w},\bm{z})}{\sum_{z' \in \Omega} p(\bm{w},\bm{z}')} & \text{ if } \bm{z} \in \Omega \\
      0 & \text{ if } \bm{z} \notin \Omega
    \end{cases}\\
    \log p(\bm{w}) - D_\mathrm{KL}\left[p_\Omega(\bm{z}) \parallel p(\bm{z}|\bm{w})\right] &= \sum_z p_\Omega(\bm{z}) \log \frac{p(\bm{w}) p(\bm{z}|\bm{w})}{p_\Omega(\bm{z})} \\
    &= \left[\sum_z p_\Omega(\bm{z})\right]\left[\log \sum_{z' \in \Omega} p(\bm{w}, \bm{z'}) \right] \\
    &= \log \sum_{z \in \Omega} p(\bm{w}, \bm{z}).
  \end{aligned}
\end{equation}
Since the KL divergence is non-negative, $\log \sum_{\bm{z} \in \Omega} p(\bm{w}, \bm{z})$ is the evidence lower bound~(ELBO) to $\log p(\bm{w})$.  Schniter et al.~\cite{schniter_fast_2008} construct $\Omega$ by repeated greedy search~(RGS) of $\bm{z}$'s with largest $p(\bm{w}, \bm{z})$.  But we find an RGS on $p(\bm{w}|\bm{z})$ instead gives better ELBO and less biased $\hat{\mu}$~(section~\ref{subsec:chargereconstruction}).  An iteration scheme~\cite{schniter_fast_2008} efficiently computes $p(\bm{w}|\bm{z})$ in eq.~(\ref{eq:mgauss}) and overcomes the difficulty of MCMC~(section~\ref{subsec:mcmc}) to make FBMP practical.

We take the maximum a posteriori estimation $\hat{\bm{z}}$ and the expectation estimation of $\hat{\bm{q}}$,
\begin{equation}
  \label{eq:fbmpcharge}
  \begin{aligned}
    \hat{\bm{z}} &= \arg \underset{\bm{z} \in \Omega}{\max} p(\bm{w}|\bm{z}) \\
    \hat{\bm{q}}_{\hat{\bm{z}}} &= E(\bm{q}'|\bm{w},\hat{\bm{z}}) = \hat{\bm{z}} + \bm{Z}\bm{V}_\mathrm{PE}^\intercal\bm{\Sigma}^{-1}(\bm{w}-\bm{V}_\mathrm{PE}\hat{\bm{z}})
  \end{aligned}
\end{equation}
and calculate RSS and $D_\mathrm{w}$ by eqs.~\eqref{eq:rss}, \eqref{eq:numerical}, \eqref{eq:gd-phi} and \eqref{eq:mcmc-w}.

\begin{figure}[H]
  \begin{subfigure}[b]{.45\textwidth}
    \centering
    \resizebox{1.05\textwidth}{!}{\input{figures/fbmpchargestats.pgf}}
    \caption{\label{fig:fbmp-npe} $D_\mathrm{w}$ histogram and distributions conditioned on $N_{\mathrm{PE}}$, boxplot explained in figure~\ref{fig:cnn-performance}.}
  \end{subfigure}
  \hspace{0.5em}
  \begin{subfigure}[b]{.55\textwidth}
    \centering
    \resizebox{\textwidth}{!}{\input{figures/demoe2c0.pgf}}
    \caption{\label{fig:fbmp} An example giving \\ $\Delta{t_0}=\SI{-3.51}{ns}$, $\mathrm{RSS}=\SI{15.8}{mV^2}$, $D_\mathrm{w}=\SI{0.59}{ns}$.}
  \end{subfigure}
  \caption{\label{fig:fbmp-performance}Demonstration of FBMP with $\num[retain-unity-mantissa=false]{1e4}$ waveforms in~\subref{fig:fbmp-npe} and one waveform in~\subref{fig:fbmp} sampled from the same setup as figure~\ref{fig:method}.  FBMP reconstructs the waveform and charges flawlessly.}
\end{figure}
In terms of $D_\mathrm{w}$, figure~\ref{fig:fbmp-npe} show that FBMP is on par with CNN in figure~\ref{fig:cnn-npe}.  Figure~\ref{fig:fbmp} demonstrates a perfect reconstruction example where the true and reconstructed charges and waveforms overlap.  A bias of $\Delta{t_0}=\SI{-3.51}{ns}$ aligns with $\hat{t}_\mathrm{ALL}$ in eq.~\eqref{eq:2}, which will be covered in section~\ref{subsec:timeresolution}.  The superior performance of FBMP attributes to sparsity and positiveness of $q'_i$, correct modeling of $V_\mathrm{PE}$, $q'$ distribution and white noise.

FBMP provides a Bayesian interface to event reconstruction by eq.~\eqref{eq:bayesianinter}.  We shall investigate it in the upcoming publications.


%!TEX root = /home/renaud/Documents/EPL/tfe/latex/tfe.tex
\addcontentsline{toc}{chapter}{Conclusion}
\chapter*{Conclusion}
This work starts with an introduction of the concept of communities in a network. In particular, chapter~\ref{chap:clustering} is entirely dedicated to the \textit{stability measure}, which allows to quantify the quality of a particular clustering:
\[
	r(t) = \max_{\P} \left\{\trace \left[ \b \b H_{\P}^{\t}(\b \Pi\b P(t) - \bs \pi^{\t}\bs \pi)\b H_{\P} \right]\right\},
\]
where $t$ is the Markov time, which acts as an intrinsic resolution parameter. This chapter is intended to be a standalone theoretical chapter providing a complete overview of the stability measure.

In chapter~\ref{chap:transportmodel}, the \textit{reactive transport} equation
\[
	\frac{\partial C}{\partial t} = q-\nabla \cdot \left(\b uC - \b K \nabla C\right).
\]
is presented and the main properties of the solution to that equation are shown. This allows to propose a compartment model structure in chapter~\ref{chap:compartment} that exhibits similar properties:
\[
	\Omega_i \frac{dC_i}{dt} = \Omega_i q_i - \sum_{\subalign{j&=1 \\j &\neq i}}^N \left[ U_{i,j}\frac{C_i + C_j}{2} - V_{i,j} (C_j-C_i)\right] - \phi_{i,e} \quad \mbox{for } i=1,\dots,N.
\]
The equivalent matrix form
\[
	\bs \Omega \b{\dot{c}} = \bs \Omega \b q + \b A \b c
\]
is also developed for convenience, and the previously mentioned properties are translated into the matrix language.

Anticipating the fact that we do not consider any problem involving reactive phenomena in this work, the attention is then focused on the \textit{advection-diffusion} equation in chapter~\ref{chap:numerical}. It is the particular form of the reactive transport equation obtained when the reactive term $q$ is equal to zero. In order to numerically solve such problems, the Eulerian methods are compared to the Lagrangian methods to arrive at the conclusion that a Lagrangian approach is better suited to the needs of this work. An effort is made to derive the Lagrangian equations describing the position of an individual particle which are consistent with the advection-diffusion transport model. This leads to the beautiful and complex theory of stochastic differential equations (SDEs). The backward-Itô interpretation of stochastic differential equations is shown to be particularly useful for dealing with problems involving a discontinuous diffusivity tensor. Indeed, at the contrary to the Itô or Stratonovich interpretations, the backward-Itô interpretation leads to Lagrangian equations that does not include \textit{gradient drift terms}, namely terms involving spatial derivatives of the diffusivity tensor:
\[
	\mathbf{(bI)} \quad
	\begin{cases}
		\rm d \b{x}(t) =  \b u(\b x(t),t) \rm dt + \b B(\b x(t),t) \rm d \b W(t),\\
		\b B \b B^\t = 2 \b K,\\
		\b x(t_0) = \b x_0.
	\end{cases}{}
\]
The most simple numerical scheme consistent with the backward-Itô interpretation of a SDE, the backward Euler scheme, is then presented. The \textit{box-counting} method which allows to estimate the concentration from a set of particles trajectories completes the tools needed to implement an efficient code to numerically solve a two-dimensional advection-diffusion problem. The code is briefly presented in section~\ref{sec:thecode}.

Finally, the whole method is summarized in chapter~\ref{chap:method}, allowing to tackle a simple test problem in chapters~\ref{chap:overturnercirculation} and \ref{chap:bioverturner}. The test problem is chosen such that a two compartments decomposition of the domain is intuitively obvious. Satisfactorily, the method leads to the expected box decomposition. For that simple problem, a continuous-time compartment model satisfying the properties from chapter~\ref{chap:compartment} is build relatively easily using an ad-hoc method. A more general approach leading to a discrete-time compartment model is also proposed. Both models are assessed by comparing their results to the ones obtained by numerical simulations.

For the simple test problem considered, the method leads to a satisfying two-compartments model. It would be presumptuous to draw conclusions on the method based only on the results obtained for such a simple test problem. Instead, this encouraging result strengthens our idea that further works are worthwhile. Ideas of problems on which the method may be tested include the idealized two-dimensional model of the meridian circulation in the Atlantic \cite{deleersnijder2006overturner}, the Great Barrier Reef, Australia \cite{thomas2014numerical} or the Pacific Ocean \cite{shah2017tracing}.

To conclude, let us mention the several prospects of amelioration that comes to our mind at the time of finishing this work. As already raised, the method should be applied on more complex problems, and at the very least on problems having a physical meaning. Furthermore, it could be interesting to compare the results obtained when using different clustering algorithms. For example, the \textit{infomap} algorithm proposed by Martin Rosvall also seems well adapted to this study.\footnote{See the excellent website \url{http://www.mapequation.org} for a presentation of the algorithm and links to the related publications.} Finally, we could extend the \Cpp code towards a complete toolbox for the simulation of two-dimensional SDEs. Such a toolbox should allow to choose the boundary conditions and to deal with the reactive term, and it could integrate higher order numerical schemes like Milstein's scheme, but this is of course outside the scope of the present work. A parallelization of the code with MPI also figures amongst the perspective of further work that have been considered.\footnote{In fact, an MPI version of the code has actually been implemented. But we have not managed to test that implementation with many nodes because of incompatibility problems with the versions of MPI on the clusters, which are still not solved at time time of finishing this work.}

% This work illustrates how the tools from applied mathematics could potentially be useful in the context of solving real-world physical problems. It