\input{../../style/preamble}
\input{../../latex-math/basic-math}
\input{../../latex-math/basic-ml}
\input{../../latex-math/ml-gp}

\newcommand{\titlefigure}{figure_man/discrete/marginalization-more.png} %not best picture
\newcommand{\learninggoals}{
  \item \textcolor{blue}{XXX}
  \item \textcolor{blue}{XXX}
}

\title{Introduction to Machine Learning}
\date{}

\begin{document}

\lecturechapter{Gaussian Processes}
\lecture{Introduction to Machine Learning}

\begin{vbframe}{Weight-Space View}

\begin{itemize}
  \item Until now we considered a hypothesis space $\Hspace$ of parameterized functions $\fxt$ (in particular, the space of linear functions).
  \item Using Bayesian inference, we derived distributions for $\thetab$ after having observed data $\D$.
  \item Prior believes about the parameter are expressed via a prior distribution $q(\thetab)$, which is updated according to Bayes' rule

  $$
  \underbrace{p(\thetab | \Xmat, \yv)}_{\text{posterior}} = \frac{\overbrace{p(\yv | \Xmat, \thetab)}^{\text{likelihood}}\overbrace{q(\thetab)}^{\text{prior}}}{\underbrace{p(\yv|\Xmat)}_{\text{marginal}}}.
  $$
\end{itemize}

\end{vbframe}

\begin{vbframe}{Function-space View}

Let us change our point of view:

\begin{itemize}
  \item Instead of \enquote{searching} for a parameter  $\thetab$ in the parameter space, we directly search in a space of \enquote{allowed} functions $\Hspace$.
  \item We still use Bayesian inference, but instead specifying a prior distribution over a parameter, we specify a prior distribution \textbf{over functions} and update it according to the data points we have observed.
\end{itemize}

\framebreak

Intuitively, imagine we could draw a huge number of functions from some prior distribution over functions $^{(*)}$.

\begin{figure}
  \includegraphics[width=0.8\textwidth]{figure_man/gp-sample/gp-sample-1-1.pdf}
\end{figure}

\vspace*{-0.5cm}

\begin{footnotesize}
  $^{(*)}$ We will see in a minute how distributions over functions can be specified.
\end{footnotesize}

\framebreak

\foreach \x in{1,2,3} {
    After observing some data points, we are only allowed to sample those functions, that are consistent with the data. \\
  \begin{figure}
    \includegraphics[width=0.8\textwidth]{figure_man/gp-sample/gp-sample-2-\x.pdf}
  \end{figure}
}

\framebreak

As we observe more and more data points, the variety of functions consistent with the data shrinks.
  \begin{figure}
    \includegraphics[width=0.8\textwidth]{figure_man/gp-sample/gp-sample-2-4.pdf}
  \end{figure}

\framebreak

Inutitively, there is something like \enquote{mean} and a \enquote{variance} of a distribution over functions.

  \begin{figure}
    \includegraphics[width=0.8\textwidth]{figure_man/gp-sample/gp-sample-2-4.pdf}
  \end{figure}

\end{vbframe}

\begin{frame}{Weight-space vs. Function-space View}

\begin{table}
  \begin{tabular}{cc}
  \textbf{Weight-Space View} & \textbf{Function-Space View} \vspace{4mm}\\
  Parameterize functions & \vspace{1mm}\\
  \footnotesize Example: $\fxt = \thetab^\top \xv$ & \vspace{3mm}\\
  Define distributions on $\thetab$ & Define distributions on $f$ \vspace{4mm}\\
  Inference in parameter space $\Theta$ & Inference in function space $\Hspace$
  \end{tabular}
\end{table}

\lz

Next, we will see how we can define distributions over functions mathematically.

\end{frame}

\section{Distributions on Functions}

\begin{vbframe}{Discrete Functions}

For simplicity, let us consider functions with finite domains first.

\lz

Let $\mathcal{X} = \left\{\xv^{(1)}, \dots , \xv^{(n)}\right\}$ be a finite set of elements and $\Hspace$ the set of all functions from $\mathcal{X} \to \R$.

\lz

Since the domain of any $h(.) \in \Hspace$ has only $n$ elements, we can represent the function $h(.)$ compactly as a $n$-dimensional vector $$\bm{h} = \left[h\left(\xv^{(1)}\right), \dots, h\left(\xv^{(n)}\right)\right].$$
\end{vbframe}

\begin{frame}{Discrete Functions}

\textbf{Example 1:} Let us consider $h: \Xspace \to \Yspace$ where the input space consists of \textbf{two} points $\Xspace = \{0, 1\}$.

\lz

Examples for functions that live in this space:

\begin{figure}[h]
\foreach \x in{1,2,3} {
  \includegraphics<\x>[width=0.7\linewidth]{figure_man/discrete/example_2_\x.pdf} \par
}
\end{figure}

\end{frame}

\begin{frame}{Discrete Functions}

\textbf{Example 2:} Let us consider $h: \Xspace \to \Yspace$ where the input space consists of \textbf{five} points $\Xspace = \{0, 0.25, 0.5, 0.75, 1\}$.

\lz

Examples for functions that live in this space:

\begin{figure}[h]
\foreach \x in{1,2,3} {
  \includegraphics<\x>[width=0.7\linewidth]{figure_man/discrete/example_5_\x.pdf}\par
}
\end{figure}

\end{frame}

\begin{frame}{Discrete Functions}

\textbf{Example 3:} Let us consider $h: \Xspace \to \Yspace$ where the input space consists of \textbf{ten} points.

\lz

Examples for functions that live in this space:

\begin{figure}[h]
\foreach \x in{1,2,3} {
  \includegraphics<\x>[width=0.7\linewidth]{figure_man/discrete/example_10_\x.pdf}\par
}
\end{figure}

\end{frame}

\begin{vbframe}{Distributions on Discrete Functions}

\vspace*{0.5cm}

One natural way to specify a probability function on discrete function $h \in \Hspace$ is to use the vector representation
$$
  \bm{h} = \left[h\left(\xi[1]\right), h\left(\xi[2]\right), \dots, h\left(\xi[n]\right)\right]
$$

of the function.

\lz

Let us see $\bm{h}$ as a $n$-dimensional random variable. We will further assume the following normal distribution:

$$
  \bm{h} \sim \mathcal{N}\left(\bm{m}, \bm{K}\right).
$$

\textbf{Note: } For now, we set $\bm{m} = \bm{0}$ and take the covariance matrix $\bm{K}$ as given. We will see later how they are chosen / estimated.

\end{vbframe}

\begin{frame}{Discrete Functions}

\textbf{Example 1 (continued):} Let $h: \Xspace \to \Yspace$ be a function that is defined on \textbf{two} points $\Xspace$. We sample functions by sampling from a two-dimensional normal variable

$$
\bm{h} = [h(1), h(2)] \sim \mathcal{N}(\bm{m}, \bm{K})
$$

\begin{figure}[H]
\foreach \x in{1,2,3} {
  \includegraphics<\x>[width=0.4\linewidth]{figure_man/discrete/example_norm_2_\x-a.pdf} ~  \includegraphics<\x>[width=0.4\linewidth]{figure_man/discrete/example_norm_2_\x-b.pdf}
} \par
\begin{footnotesize}
In this example, $m = (0, 0)$ and $K = \begin{pmatrix} 1 & 0.5 \\ 0.5 & 1 \end{pmatrix}$.
\end{footnotesize}
\end{figure}

\end{frame}

\begin{frame}{Discrete Functions}

\textbf{Example 2 (continued):} Let us consider $h: \Xspace \to \Yspace$ where the input space consists of \textbf{five} points. We sample functions by sampling from a five-dimensional normal variable

$$
\bm{h} = [h(1), h(2), h(3), h(4), h(5)] \sim \mathcal{N}(\bm{m}, \bm{K})
$$

\begin{figure}[h]
\foreach \x in{1,2,3} {
  \includegraphics<\x>[width=0.4\linewidth]{figure_man/discrete/example_norm_5_\x-a.pdf} ~  \includegraphics<\x>[width=0.4\linewidth]{figure_man/discrete/example_norm_5_\x-b.pdf}
}
\end{figure}

\end{frame}

\begin{frame}{Discrete Functions}

\textbf{Example 3 (continued):} Let us consider $h: \Xspace \to \Yspace$ where the input space consists of \textbf{ten} points. We sample functions by sampling from ten-dimensional normal variable

$$
\bm{h} = [h(1), h(2), \dots, h(10)] \sim \mathcal{N}(\bm{m}, \bm{K})
$$

\begin{figure}[h]
\foreach \x in{1,2,3} {
  \includegraphics<\x>[width=0.4\linewidth]{figure_man/discrete/example_norm_10_\x-a.pdf} ~  \includegraphics<\x>[width=0.4\linewidth]{figure_man/discrete/example_norm_10_\x-b.pdf}
}
\end{figure}

\end{frame}

\begin{vbframe}{Role of the Covariance Function}

Note that the covariance controls the \enquote{shape} of the drawn function. Consider two extreme cases where function values are

\begin{enumerate}
  \item[a)] strongly correlated: $\bm{K} = \begin{footnotesize}\begin{pmatrix} 1 & 0.99 & \dots & 0.99 \\
  0.99 & 1 & \dots & 0.99 \\
  0.99 & 0.99 & \ddots & 0.99 \\
  0.99 & \dots & 0.99 & 1 \end{pmatrix}\end{footnotesize}$
  \item[b)] uncorrelated: $\bm{K} = \id$
\end{enumerate}

\begin{figure}
  \includegraphics[width=0.35\linewidth]{figure_man/discrete/example_extreme_50-1.pdf} ~~  \includegraphics[width=0.35\linewidth]{figure_man/discrete/example_extreme_50-2.pdf}
\end{figure}

\framebreak

\begin{itemize}
  \item \enquote{Meaningful} functions (on a numeric space $\Xspace$) may be characterized by a spatial property: \vspace*{0.2cm}
  \begin{itemize}
    \item[] If two points $\xi, \xi[j]$ are close in $\Xspace$-space, their function values $f(\xi), f(\xi[j])$ should be close in $\Yspace$-space.
  \end{itemize} \vspace*{0.2cm}
  In other words: If they are close in $\Xspace$-space, their functions values should be \textbf{correlated}! \vspace*{0.4cm}
  \item We can enforce that by choosing a covariance function with
  $$
    \bm{K}_{ij} \text{ high, if } \xi[i], \xi[j] \text{ close.}
  $$

  \framebreak

  \item We can compute the entries of the covariance matrix by a function that is based on the distance between $\xi, \xi[j]$, for example:

  \vspace*{0.2cm}
  \begin{enumerate}
    \item[c)] Spatial correlation: \begin{footnotesize}$K_{ij} = k(\xi[i], \xi[j]) = \exp\left(-\frac{1}{2}\left|\xi - \xi[j]\right|^2\right)$\end{footnotesize}
  \end{enumerate}

\begin{figure}
  \includegraphics[width=0.45\linewidth]{figure_man/discrete/example_extreme_50-4.pdf} ~~  \includegraphics[width=0.45\linewidth]{figure_man/discrete/example_extreme_50-3.pdf}
\end{figure}

\end{itemize}

\begin{footnotesize}
\textbf{Note}: $k(\cdot,\cdot)$ is known as the \textbf{covariance function} or \textbf{kernel}. It will be studied in more detail later on.
\end{footnotesize}

\end{vbframe}

% \begin{vbframe}
% \begin{figure}
% 	\centering
% 	\includegraphics{figure_man/discrete/sample2.png} \\
% 	\begin{footnotesize} If we sample again, we get another function.
% 	\end{footnotesize}
% \end{figure}

% However, we are usually interested in functions with infinite domain size.

% \lz

% This idea is extended to infinite domain size via \textbf{Gaussian processes}.

% \end{vbframe}

\section{Gaussian Processes}

\begin{vbframe}{From Discrete to Continuous Functions}

\begin{itemize}
  \item We defined distributions on functions with discrete domain by defining a Gaussian on the vector of the respective function values
  $$
    \mathbf{h} = [h(\xi[1]), h(\xi[2]), \dots, h(\xi[n])] \sim \mathcal{N}(\bm{m}, \bm{K})
  $$

  \item We can do this for $n \to \infty$ (as \enquote{granular} as we want)
  \begin{figure}
    \includegraphics[width = 0.9\textwidth]{figure_man/discrete/example_limit.pdf}
  \end{figure}
\end{itemize}

\end{vbframe}

\begin{frame}{From Discrete to Continuous Functions}

\begin{itemize}
  \item No matter how large $n$ is, we are still considering a function over a discrete domain.
  \item How can we extend our definition to functions with \textbf{continuous domain} $\Xspace \subset \R$?
\end{itemize}

\end{frame}

\begin{frame}{Gaussian Processes: Intuition}

\begin{itemize}
  \only<1>{
    \item Intuitively, a function $f$ drawn from \textbf{Gaussian process} can be understood as an \enquote{infinite} long Gaussian random vector.
    \item It is unclear how to handle an \enquote{infinite} long Gaussian random vector!
  \lz
  \begin{figure}
    \includegraphics[width=0.3\textwidth]{figure_man/question.png}
  \end{figure}
  }
  \only<2-4>{
    \item Thus, it is required that for \textbf{any finite set} of inputs $\{\xi[1], \dots, \xi[n]\} \subset \Xspace$, the vector $\mathbf{f}$ has a Gaussian distribution
    $$
      \bm{f} = \left[f\left(\xi[1]\right), \dots, f\left(\xi[n]\right)\right] \sim \mathcal{N}\left(\bm{m}, \bm{K}\right),
    $$
    with $\bm{m}$ and $\bm{K}$ being calculated by a mean function $m(.)$ / covariance function $k(.,.)$.
    \item This property is called \textbf{Marginalization Property}.
    \begin{figure}
      \only<2>{\includegraphics[width=0.4\textwidth]{figure_man/discrete/example_marginalization_5.pdf}\includegraphics[width=0.5\textwidth]{figure_man/discrete/marginalization-5.png}}
      \only<3>{\includegraphics[width=0.4\textwidth]{figure_man/discrete/example_marginalization_10.pdf}\includegraphics[width=0.5\textwidth]{figure_man/discrete/marginalization-more.png}}
      \only<4>{\includegraphics[width=0.4\textwidth]{figure_man/discrete/example_marginalization_50.pdf}\includegraphics[width=0.5\textwidth]{figure_man/discrete/marginalization-more.png}}
   \end{figure}
    }
\end{itemize}

\end{frame}

\begin{vbframe}{Gaussian Processes}

This intuitive explanation is formally defined as follows:

\lz

A function $\fx$ is generated by a GP $\gp$ if for \textbf{any finite} set of inputs $\left\{\xv^{(1)}, \dots, \xv^{(n)}\right\}$, the associated vector of function values $\bm{f} = \left(f(\xv^{(1)}), \dots, f(\xv^{(n)})\right)$ has a Gaussian distribution

$$
\bm{f} = \left[f\left(\xi[1]\right),\dots, f\left(\xi[n]\right)\right] \sim \mathcal{N}\left(\bm{m}, \bm{K}\right),
$$

with

\begin{eqnarray*}
\textbf{m} &:=& \left(m\left(\xi\right)\right)_{i}, \quad
\textbf{K} := \left(k\left(\xi, \xv^{(j)}\right)\right)_{i,j},
\end{eqnarray*}

where $m(\xv)$ is called mean function and $k(\xv, \xv^\prime)$ is called covariance function.

\framebreak

\vspace*{0.5cm}

A GP is thus \textbf{completely specified} by its mean and covariance function

\vspace*{-0.2cm}
\begin{eqnarray*}
m(\xv) &=& \E[f(\xv)] \\
k(\xv, \xv^\prime) &=& \E\biggl[\left( f(\xv) - \E[f(\xv)] \right) \left( f(\xv^\prime) - \E[f(\xv^\prime)] \right)\biggr]
\end{eqnarray*}

\vfill

\textbf{Note}: For now, we assume $m(\xv) \equiv 0$. This is not necessarily a drastic limitation - thus it is common to consider GPs with a zero mean function.

% \framebreak

% \vspace*{0.5cm}

% Intuitively, one can think of a function $f$ drawn from a Gaussian process prior as a Gaussian distribution with an \enquote{infinitely} long mean vector and an \enquote{infinite by infinite} covariance matrix.

% \lz

% Each dimension of the Gaussian corresponds to an element $\xv$ from the domain $\mathcal{X}$. The corresponding component of the random vector represents the value of $f(\xv)$.

% \lz

% The \textbf{marginalization property} makes it possible to handle this \enquote{infinite} representation: evaluations of the process on any finite number of points follow a multivariate normal distribution.

\end{vbframe}

\begin{vbframe}{Sampling from a Gaussian process Prior}

We can draw functions from a Gaussian process prior. Let us consider $\fx \sim \mathcal{GP}\left(0, k(\xv, \xv^\prime)\right)$ with the squared exponential covariance function $^{(*)}$

$$
k(\xv, \xv^\prime) = \exp\left(-\frac{1}{2\ls^2}\|\xv - \xv^\prime\|^2\right), ~~ \ls = 1.
$$
\vspace{-4cm}
This specifies the Gaussian process completely.

\vspace{8cm}
\footnotesize
$^{(*)}$ We will talk later about different choices of covariance functions.

\normalsize

\framebreak

To visualize a sample function, we

\begin{itemize}
  \item choose a high number $n$ (equidistant) points $\left\{\xv^{(1)}, \dots, \xv^{(n)}\right\}$
  \item compute the corresponding covariance matrix $\Kmat = \left(k\left(\xi, \xv^{(j)}\right)\right)_{i,j}$ by plugging in all pairs $\xv^{(i)}, \xv^{(j)}$
  \item sample from a Gaussian $\bm{f} \sim \mathcal{N}(\bm{0}, \bm{K})$.
\end{itemize}

We draw $10$ times from the Gaussian, to get $10$ different samples.

% Using $100$ equidistant points, we repeat the process of generating the Gaussian $10$ times ($10$ different functions) and draw each function by connecting the sampled values.

% \lz

\begin{figure}
  \includegraphics[width=0.9\textwidth]{figure_man/different-samples.png}
\end{figure}

\vspace{-0.2cm}
Since we specified the mean function to be zero $m(\xv) \equiv 0$, the drawn functions have zero mean.

\end{vbframe}

\section{Gaussian Processes as Indexed Family}

\begin{vbframe}{Gaussian processes as an Indexed Family}

% \begin{block}{Definition}
% A \textbf{Gaussian process} is a (infinite) collection of random variables, any \textbf{finite} number of which have a \textbf{joint Gaussian distribution}.
% \end{block}

% \lz

A Gaussian process is a special case of a \textbf{stochastic process} which is defined as a collection of random variables indexed by some index set (also called an \textbf{indexed family}). What does it mean?

\lz

An \textbf{indexed family} is a mathematical function (or \enquote{rule}) to map indices $t \in T$ to objects in $\mathcal{S}$.

\begin{block}{Definition}
A \textbf{family of elements in $\mathcal{S}$ indexed by $T$} (indexed family) is a surjective function
\vspace*{-0.3cm}
\begin{eqnarray*}
s: T &\to& \mathcal{S} \\
   t &\mapsto& s_t = s(t)
\end{eqnarray*}
\end{block}

\end{vbframe}

\begin{vbframe}{Indexed Family}

Some simple examples for indexed families are:

\vspace*{0.3cm}

\begin{minipage}{0.43\linewidth}
  \begin{itemize}
  \item finite sequences (lists): $T = \{1, 2, \dots, n\}$ and $\left(s_t\right)_{t \in T} \in \R$ \vspace{1cm}
  \item infinite sequences: $T = \N$ and $\left(s_t\right)_{t \in T} \in \R$
  \end{itemize}
\end{minipage}
\begin{minipage}{0.55\linewidth}
\includegraphics{figure_man/indexed_family/indexed_family_1.png} \\
\includegraphics{figure_man/indexed_family/indexed_family_2.png}
\end{minipage}

\framebreak

But the indexed set $\mathcal{S}$ can be something more complicated, for example functions or \textbf{random variables} (RV):

\begin{minipage}{0.43\linewidth}
  \vspace*{0.5cm}
  \begin{itemize}
    \item $T = \{1, \dots, m\}$, $Y_t$'s are RVs: Indexed family is a random vector. \vspace*{0.2cm}
    \item $T = \{1, \dots, m\}$, $Y_t$'s are RVs: Indexed family is a stochastic process in discrete time \vspace*{0.2cm}
    \item $T = \Z^2$, $Y_t$'s are RVs: Indexed family is a 2D-random walk.
  \end{itemize}
\end{minipage}\hfill
\begin{minipage}{0.5\linewidth}
\includegraphics{figure_man/indexed_family/indexed_family_4.png} \\
\includegraphics{figure_man/indexed_family/indexed_family_3.png}
\end{minipage}

\end{vbframe}

\begin{frame}{Indexed Family}

\begin{itemize}
  \item A Gaussian process is also an indexed family, where the random variables $f(\xv)$ are indexed by the input values $\xv \in \Xspace$.
  \item Their special feature: Any indexed (finite) random vector has a multivariate Gaussian distribution (which comes with all the nice properties of Gaussianity!).
\end{itemize}

\begin{figure}
  \includegraphics<1>[width=0.7\textwidth]{figure_man/indexed_family/indexed_family_5.png} \par
  \only<1>{\begin{footnotesize} Visualization for a one-dimensional $\Xspace$. \end{footnotesize}}
  \includegraphics<2>[width=0.6\textwidth]{figure_man/indexed_family/indexed_family_6.png}\par
  \only<2>{\begin{footnotesize} Visualization for a two-dimensional $\Xspace$. \end{footnotesize}}
\end{figure}

\end{frame}

\endlecture
\end{document}


\section{Introduction}
Correlation clustering~\cite{Bansal-2002} also known as the multicut problem~\cite{chopra_1993_mp}
is a basic primitive in computer vision~\cite{andres_2011_iccv,kroeger_2012_eccv,yarkony_2012_eccv,alush_2013_simbad} and data mining~\cite{Chierichetti-2014,Arasu-2009,Sadikov-2010} as a first step
towards understanding an image or data in general.
The correlation clustering problem is the following:
Given an undirected edge weighted graph,
where positive weights (\emph{attractive}) encourage the adjacent nodes to be in the same cluster
and negative weights (\emph{repulsive}) encourage the nodes to stay in different cluster,
find the clustering of nodes such that the sum over weights of edges between clusters is minimized.
Note that this problem formulation does not included a predefined number of clusters.

Despite the clear mathematical formulation and nice properties of correlation clustering
to perform a one-shot agglomeration clustering,
the main draw back of correlation clustering is scalability~\cite{nunez_iglesias_2013}
which arise from the NP-hardness of the problem~\cite{Bansal-2002}.

In signed social networks, where positive and negative edges encodes friend and foe relationships, respectively,
correlation clustering is a natural way to detect communities~\cite{Chierichetti-2014,Chen-2012}.
Correlation clustering can also been used to cluster query refinements in web search~\cite{Sadikov-2010}.
Because social and web-related networks are often very huge, heuristic methods, \eg the PIVOT-algorithm~\cite{Ailon-2008},
are very popular~\cite{Chierichetti-2014}.

In computer vision applications, unsupervised image segmentation algorithms usually start with an over-segmentation
into superpixels (superregions), which are then clustered into ``perceptually meaningful''
regions by correlation clustering on sparse graphs.
Such an approach has been shown to yield
state-of-the-art results on the Berkeley Segmentation Database
\cite{andres_2011_iccv,yarkony_2012_eccv,alush_2013_simbad}.

Recently, Beier~\etal~\cite{beier_2014_cvpr} presented a promising
approximative method called Cut, Clue \& Cut.
While this scales well for planar graphs, it does not for non planar graphs and has a bad any-time performance.
A further inherent drawback of this method is that the subproblems are not limited in their size.
%
Consequently, partition problems on large scale data, \eg
huge volume images in computational neuroscience~\cite{kroeger_2012_eccv}
or social networks~\cite{Leskovec-2010},
are not tractable, because the runtime explodes or the
approximations are far away from being useful.

In the present work we will suggest some alternative approaches to deal with large scale correlation clustering problems.
In the end we ill present a method that can be combined with any other correlation clustering method and
allows to reduce the overall problem to a sequence of correlation clustering problems which are by magnitudes smaller.

\textbf{Contribution:}
In the present work we present some novel approaches that are designed for large scale correlation clustering problems.
First, we define a novel energy based agglomerative clustering algorithm that monotonically increase the energy.
With this at hand we show how to improve the anytime performance of Cut, Clue \& Cut.
Second, we introduce cluster-fusion moves, which extend the original fusion moves~\cite{Lempitsky-2010}
used in supervised segmentation to the unsupervised case and give a polyhedral interpretation of this algorithm.
We propose two versatile proposal generators, and evaluate the proposed methods on existing and new benchmark problems.
That experiments show that we can improve the computation time by one to two magnitudes without worsening the segmentation
quality significantly.

% Given an edge weighted graph, positive weights (\emph{attractive})
% encourage the adjacent nodes to be in the same
% connected component, while negative weights (\emph{repulsive}) encourage
% the nodes to stay in different connect components.
% Therefore the sign of the weights encodes if two
% nodes should be merged or not and the magnitude of the weights encodes
% the certainty of this desire.
% The objective of correlation clustering / multicuts
% is finding the cut with a minimal sum of cut edge weights.
% The number of connected components / clusters is discovered
% from the weights ( rewrite?!?).
% Correlation clustering / the multicut is NP-hard \cite{???}.
% %
% Despite the NP-hardness, correlation clustering has been
% successfully used for
% \begin{inparaenum}[(i)]
%     \item partitioning a superpixel region adjacency graph~\cite{andres_2011_iccv,kroeger_2012_eccv}
%     \item with optional long range repulsive edges~\cite{andres_2013_emmcvpr}.
%     \item Alush and Globerger showed how to average multiple segmentations with the multicut objective~\cite{alush_2012_pami}.
%     \item Multicuts can also be used for interactive segmentation~\cite{bagon_2011_arxiv},
%     \item for co-segmentation~\cite{glassner_2011_cvpr}
%     \item and to cluster sparse and graphs~\cite{???}.
% \end{inparaenum}

% Despite the nice properties of correlation clustering
% to perform a \say{one-shot agglomeration of supervoxels} ~\cite{nunez_iglesias_2013},
% the main draw back of correlation clustering is scalability~\cite{nunez_iglesias_2013}.
% Existing solvers for correlation clustering do not scale for non planar graphs.
% Even state of the art approximate move making algorithm fail
% to give good approximations non planar graphs.
% When increasing the problem sizes, either the runtime explodes or the
% approximations are far away from being useful.

% \textbf{Contribution:}
% \begin{inparaenum}[(i)]
% \item Within this work we propose a fast and scalable move making algorithm for correlation clustering,
% \item which generalized  fusion moves \cite{???} to correlation clustering problems.
% \item We reduce the inference to a series of moves where
% each moves optimizes over a subspace spanned by the current best solution
% and a proposal solution.
% \item We propose two versatile proposal generators,
% \item and show how to optimize these moves.
% \item We give a polyhedral interpretation of this algorithm,
% \item and evaluate the proposed method
% on existing benchmark problems.
% \item We show state of the art any time performance on those instances.
% \end{inparaenum}[(i)]

\textbf{Outline:} {\bf If we are short of space, the following can be ommited:}
In sec.~\ref{sec:problem_formulation} we will give a
detailed problem definition where we introduce
the correlation clustering objective.
In sec.~\ref{sec:related_work} we will
discuss existing methods for correlation
clustering and briefly explain the concept of fusion moves.
In sec: ~\ref{sec:cc_fm} we describe our proposed
method and show the properties of the algorithm.
In sec.~\ref{sec:exp} we show an evaluation
of the method on existing parameter  and discuss the effects of parameters.
Future work will be discussed in sec. \ref{sec:future} and
we will conclude in ~\ref{sec:conclusion}.
%-------------------------------------------------------------------------

\section{Notation and Problem Formulation}\label{sec:problem_formulation}
Let $G=(V,E, w)$ be a weighted graph of nodes $V$ and edges $E$.
%
The function $w : E \rightarrow \mathbb{R}$ assigns a weight to each edge.
A positive weight expresses the desire that two adjacent nodes should
be merged, whereas a negative weight indicates
that these nodes should be separated into two different regions.
%
A \emph{subgraph} $G_A = \{A, E_A, w\}$ consists
of nodes $A \subseteq V$ and edges $E_A := E\cap (A\times A)$.
%
A segmentation of the graph $G$ can by either given by an
node labeling $l \in \mathbb{N}^{|V|}$
or an edge labeling $y \in\{0,1\}^{|E|}$.
An edge labeling is only consistent if it contains no dangling edges~\cite{kappes_2013_arxiv}.
We denote the set of all consistent edge labelings by $P(G)\subset\{0,1\}^{|E|}$.
The convex hull of this set is known as the \emph{multicut polytope} $MC(G) = \textrm{conv}(P(G))$.

%$\rho_y : P(G) \to P(G_y)$
%$\rho_y^{-1} : P(G_y) \to P(G)$

% \begin{itemize}
%   \item Multicut in computer vision, applications, sparse segmentation -> IMPORTANT PROBLEM
%   \item Problem: Existing Methods does not scale, even CGC. - Large scale 3d problems
%   \item Contribution: Fast scalable method using novel fusion moves for correlation clustering
% \end{itemize}

% Def: Graph, weighted graph, cut, multicut

Given a weighted graph $G=(V,E,w)$ we consider the problem of segmenting $G$ such that the costs
of the edges between distinct segments is minimized. This can be formulated in the node domain
by assigning each node $v$ a label $l_v \in \mathbb{N}$
\begin{align}
  l^* &= \argmin_{L \in \mathbb{N}^{|V|}} \sum_{ (i,j) \in E } w_{ij} \cdot [l_{i} \neq l_{j}], \label{eq:nodeproblem}
\end{align}  % y_{ij}^* &=& [l_{u} \neq l_{v}]
or in the edge domain, by label each edge $e$ as cut $y_e=1$ or uncut $y_e=0$
\begin{align}
  y^* &= \argmin_{y \in P(G)} \sum_{ (i,j) \in E } w_{ij} \cdot y_{ij} \label{eq:edgeproblem}.%\\
\end{align}
As shown in~\cite{kappes_2013_arxiv} both problems are equivalent, but formulation \ref{eq:nodeproblem}
suffers from ambiguities in the formulation~\cite{kappes_2011_emmcvpr}.

% The exist an surjective mapping from a node-label $l$ to edge-labeling $y$ and
% a bijective mapping from a partitioning of $V$ to vertices of the multicut polytope $MC$.
% Consequently,
% (i) problems \ref{eq:nodeproblem} and \ref{eq:edgeproblem} are equivalent
% and (ii) the node-labeling is not unique for a given partitioning and introduce some
% ambiguities.

% \subsubsection{Multicut Objective}

% The multicut / correlation clustering objective
% can be formulated in different ways.

% \paragraph{Edge Indicator Variables:}
% \begin{center}
%     \begin{eqnarray}
%         y^* &=& \argmin_{y} \sum_{ e_{ij} \in E } w_{ij} \cdot y_{ij} \\
%         s.t.:& & y \in \textit{Multicut Polytope} \nonumber
%     \end{eqnarray}
% \end{center}

% \paragraph{Fully Connected Graph:}
% \begin{center}
%     \begin{eqnarray}
%         y^*   & = & \argmin_{y} \sum_{ i<j \in V } w_{ij} \cdot y_{ij} \\
%         s.t.: &  & y_{ij} + y_{jk} < y_{i,k} \quad \forall i, j, k   \nonumber
%     \end{eqnarray}
% \end{center}

% \paragraph{Node Coloring:}
% \begin{center}
%     \begin{eqnarray}
%         l^* &=& \argmin_{L} \sum_{ e_{ij} \in E } w_{ij} \cdot [l_{u} \neq l_{v}] \\
%         y_{ij}^* &=& [l_{u} \neq l_{v}]
%     \end{eqnarray}
% \end{center}

\input{inputs/fig_notation.tex}

\section{Related Work}\label{sec:related_work}%
%
Due to the ambiguity of formulation~\ref{eq:nodeproblem},
a major branch of research has focused on solving
relaxations of eq.~\ref{eq:edgeproblem}.
To keep the objective and system of inequalities small, they
work on sparse graphs and
use cutting plane methods
in combination with (integer) linear programming and efficient
separation procedures~\cite{kappes_2011_emmcvpr,andres_2011_iccv,kappes_2013_arxiv}.

With no time restrictions and integer constraints these methods can solve the problem to global optimality.
For huge problems both, separation and solving the ILP in each round, becomes very time consuming.

For planar problems, Yarkony \etal~\cite{yarkony_2012_eccv}
suggested a column generating strategy for the outer LP-relaxation
that includes all cycle-inequalities of problem \ref{eq:edgeproblem}.
The column generation base on solving planar max cut instances.
While this method is fast,
it only solve a relaxation of the problem and so requires additional rounding strategies,
lacks of a practical stopping condition for the column generation,
and most important  is restricted to planar problems.

An other branch of research uses move making algorithm
to optimize correlation clustering~\cite{bansal_2004_ml,beier_2014_cvpr,Kernighan-1970}.
Starting with an initial segmentation, auxiliary problems are solved that
strictly improve the segmentation.

Bansal and Bagon propose a modified $\alpha-$expansion~\cite{bansal_2004_ml}
algorithm suitable for correlation clustering, that allows all variables to
change to cluster $\alpha$ in a single move. While the authors claim that this
scales to large scale data, in~\cite{beier_2014_cvpr} it has been shown that
this is not the case in general. This is not surprising because the auxiliary
problems can have as many variables as the original problem.

A more efficient move making method has been presented by Beier \etal~\cite{beier_2014_cvpr}.
Their Cut, Glue and Cut method iteratively re-optimizes the cuts between adjacent clusters of the current solution.
While this method scales better then  modified $\alpha-$expansion, it still faces three problems:
Firstly, the subproblems can be as large as the original problem,
secondly, the subproblems are max-cut problems on non-planar graphs that are NP-hard
and even the used approximation has high polynomial complexity.
This limits the method, to be applicable to huge problems. The same holds for the
method of Kernighan and Lin~\cite{Kernighan-1970}.

   % \begin{itemize}
   % \item Multicut~\cite{kappes_2011_emmcvpr}
   % \item Expand and Explorer~\cite{bagon_2011_arxiv}
   % \item Fast Planar CC~\cite{yarkony_2012_eccv}
   % \item Break and Conquer \cite{alush_2013_simbad}.
   % \item Cut Glue And Cut~\cite{beier_2014_cvpr}
   % \end{itemize}

For energy minimization problems fusion moves have become increasingly popular~\cite{Lempitsky-2010,kappes_2014_ws}.
For many large scale computer vision applications fusion moves lead to good approximations
with state of the art any time performance~\cite{kappes_2014_ws}.

The fusion move algorithm iteratively fuse the current best solution with a proposal solutions
by optimizing over the subspace spanned by the two labeling.
Due to the ambiguity of a node-labeling, fusion moves can not be applied directly for correlation clustering.
We will show how to overcome this point later.

\input{inputs/fig_hc_alg.tex}

Outside computer vision greedy methods has been suggested for correlation clustering problems, see \cite{Elsner-2009} for an overview.
A common greedy approach~\cite{Soon-2001,Ng-2002,Elsner-2008} is to randomly permute the nodes and than assign
iteratively each node to an existing cluster or create a new cluster if costs cannot be decreased by assigning to a cluster.
%Common assigning strategies are:
%\emph{BEST}; assign to cluster which is linked by the most positive edge~\cite{Ng-2002},
%\emph{FIRST}; Assign to the cluster which is first linked by a positive edge~\cite{Soon-2001}, and
%\emph{VOTE}; Assign to cluster that minimizes objective function~\cite{Elsner-2008}.
%
The PIVOT Algorithm~\cite{Ailon-2008} iterate over all nodes in random order.
If the node is not assigned it construct a cluster containing the node and all its
unassigned positively linked neighbors.
%
%Typically these algorithms started for many random permutations,
%and pick the clustering with best objective value.

A widely use post-processing method is  Best One Element Move (BOEM)~\cite{Gionis-2007}.
Start with an initial clustering one node is removed from a cluster and reassigned to an existing or new cluster such that the costs are minimized.
BEOM stops if no move can decrease the costs.


%! Author = tstreule

\section{Membranes and Transport}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Diffusion}
%
\formula[\unitfrac{mol}{l}=\unitfrac{mol}{10^3\,cm^3}]{Concentration}{c(x,t)}
\formula[\unitfrac{mol}{cm^2\,s}]{Flux}{\phi(x,t)}

\formbox{\textbf{Fick's law}}{\phi(x,t) = -D_n \pderiv{c_n(x,t)}{x}}
$= \phi_0$ in \textbf{steady state}
\formtex{~}{$D_n$: diffusion coeff $[\unitfrac{cm^2}{s}]$}
\formbox{Continuity eq.}{-\pderiv{\phi_n(x,t)}{x} = \pderiv{c_n(x,t)}{t}}
\formtex{\hfill \vspace{-1mm}e.g.}{\textbf{Delta initial condition} $c(x,t)\vert_{t=0}=n_0\delta(x)$}
\formtex{~}{%
    $c(x,t) {=} \frac{n_0}{\sqrt{2\pi\sigma}} \eu^{-x^2\!/\sigma^2}$,\hfill
    $\sigma {=} \sqrt{2D_nt}$,\hfill
    $t_{\unitfrac{1}{2}} {=} \frac{1}{D_n} x_{\unitfrac{1}{2}}^2$
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Diffusion across membrane}
%
\formtex{\includegraphics[width=.3\columnwidth]{Transport_Diffusion}}{%
%    \hspace{1cm}%
    \parbox{.7\columnwidth}{%
        $\phi_n(t) = P_n\,(c_n^i(t) - c_n^o(t))$\\
        with permeability: $P_n = \frac{D_nk_n}{d}$\\[1em]
        state space (inside membrane)\\
        after $\tau\ped{SS} = \frac{d^2}{\pi^2D_n}$
    }}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Osmosis}
%
\vspace{-2mm}
\formbox{\vspace{2mm}Osmotic (back-)pressure}{\pi(x,t) = RT\; \sum\limits_n c_n(x,t)}
\vspace{2mm} {\small\enskip $c_\Sigma(x,t)$: \parbox{.2\columnwidth}{osmolarity/\\total conc.}}
\formula{Zero pressure if}{\pi_i - \pi_o = 0}
\formula{Volume of the cell}{v_c(\infty) = v_c' + \frac{N_\Sigma^i}{c_\Sigma^o}}
\quad $v_c'$: non-water volume
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Carrier mediated Transport}
%
\formtex{Hints for their}{$\bullet$ Saturation of solute trsp. (\lightning~Fick's law)}
\formtex{existance}{$\bullet$ Competitive inhib. \quad $\bullet$ Structure specificy}

\textbf{4-state carrier model}

\begin{minipage}{.5\columnwidth}
    \includegraphics[width=.9\columnwidth]{Transport_Carriers}
\end{minipage}%
\begin{minipage}{.5\columnwidth}
    \begin{enumerate}
        \item Conservation\par
        \quad $N_E^i + N_E^o + N_{ES}^i + N_{ES}^o = N_{ET}$
        \item Fast binding (always steady state)\par
        \quad $K = \frac{c_S^i N_E^i}{N_{ES}^i} = \frac{c_S^o N_E^o}{N_{ES}^o}$
        \item Translocation (char. by flux)\par
        \quad $\phi_{E(S)} = \alpha N_{E(S)}^i - \beta N_{E(S)}^o$
        \item Zero net flux\par
        \quad $\phi_E + \underbrace{\phi_{ES} \vspace{-1mm}}_{\phi_S} = 0$
    \end{enumerate}
\end{minipage}
\scalebox{.9}{
    \formula{Net flux \underline{out} of cell}{\phi_S {=} \frac{\alpha\beta}{\alpha+\beta}N_{ET} \paren*{\frac{c_s^i}{c_s^i + K} {-} \frac{c_s^o}{c_s^o + K}}
    \enskip (\phi_S)\ped{max} {=} \frac{\alpha\beta}{\alpha+\beta}N_{ET}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ion Transport}
%
\formbox{\textbf{Continuity eq.}}{\pderiv{J_n(x,t)}{x} = -z_n F \pderiv{c_n(x,t)}{t}}
\enskip $z_n$: valency
\formbox{\textbf{Poisson eq.}}{\pderiv[2]{\psi(x,t)}{x} = -\frac{1}{\epsilon} \sum_n z_nFc_n(x,t)}
\formtex{\centering i.e.}{electric field potential = local charge density}
\formbox{\vspace{3mm}\textbf{Nernst-Planck eq.}\\$u_n$: molar mobility}{\scriptsize
    \underbrace{J_n(x,t) \vspace{-1mm}}_\mathrm{current}
    =- \underbrace{z_nFD_n\pderiv{c_n(x,t)}{x} \vspace{-1mm}}_\textrm{diffusion of charge}
    - \underbrace{u_nz_n^2F^2c_n(x,t)\pderiv{\psi(x,t)}{x} \vspace{-1mm}}_\textrm{mobility of charge in E-field}
}

Debye Length ``depletion width'': $\lambda_D \simeq \unit[1]{nm}$ within $\tau_r \simeq \unit[1]{ns}$
\formbox{\textbf{Nernst Equi. Pot.}}{V_n = \frac{\unit[59]{mV}}{z_n} \log_{10}Q}
\enskip with
\enskip \highlight{$\!\!\textstyle Q = \frac{c_s^o}{c_s^i}\!\!$},
\enskip $\frac{D_n}{u_n} = RT$
\formula{~}{V\ped{K} \simeq \unit[-75]{mV}, \quad V\ped{Na} \simeq \unit[55]{mV}}
\formula{Donnan equil.}{V_m {=} V_n \Rightarrow I_n=0}
\textcolor{gray}{\enskip $\left(\frac{c_n^o}{c_n^i}\right)^{\pm\frac{1}{z_n}} {=} \eu^\frac{FV_m}{RT}$, $+$: anodic}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Resting (membrane) potential}
%
\begin{minipage}{.35\columnwidth}
    \centering
    Resting $\leftrightarrow$ $I_m = 0$\par
    \includegraphics[width=\columnwidth]{Transport_Resting}
\end{minipage}
\hfill
\begin{minipage}{.25\columnwidth}
    \includegraphics[width=\columnwidth]{Transport_Current}
\end{minipage}
\hfill
\begin{minipage}{.35\columnwidth}
    \includegraphics[width=\columnwidth]{Transport_Multiple_Ion}
\end{minipage}

\formtex{\textbf{3-ion model}}{permeable to \ce{K}, \ce{Na} and some others}
\formbox{Resting potential}{V_m\ap{rest} {=} \frac{V\ped{K}\;G\ped{K}}{G\ped{tot}} {+} \frac{V\ped{Na}\;G\ped{Na}}{G\ped{tot}} {+} \frac{V_o\;G_o}{G\ped{tot}}}
{\scriptsize $G\ped{tot} {\equiv} G_m {=} \mathrm{slope}$}
\formtex{$J_m = J\ped{K} \!+\! J\ped{Na} \!+\! J_o = 0$}{where $J_n\!\neq\!0$ (only momentarly rest)}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Active Transport}
%
Rest is only momentarly $\to$ maintain conc. grad with \textit{act. transport}
\begin{itemize}
    \item Ions moving against its conc. gradient $\to$ ATP needed
    \item Active pumps for $J_m\!=\!0$ (rest) and $J_{n,\mathrm{in}} \!+\! J_{n,\mathrm{out}} = 0$ (quasi equilib.)
\end{itemize}


\subsection{Simulation}

The implementation scheme is shown in Figure \ref{fig:block}.

\begin{figure}[H]
    \centering
    \includegraphics[width = .9\textwidth]{fig/block.png}
    \caption{Block scheme of the ideal channel}
    \label{fig:block}
\end{figure}

The various components of the communication chain are detailed hereafter. The channel encoder is not yet implemented, and the symbols mapping/demapping were provided.

\subsubsection{Halfroot Nyquist filter}

After the mapping, the complex symbols are convolved with a root raised-cosine filter. It allows to limit the communication bandwidth, as illustrated in Figure \ref{fig:filter} where the PSD of the symbol stream are shown before and after filtering, with a rolloff factor $\beta$ of $0.3$. The symbol frequency was fixed to 2 \si{\mega \hertz}, which translates to a cutoff frequency of 1 \si{\mega \hertz}.

\begin{figure}[H]
\centering
    \includegraphics{fig/filter.tex}
     \caption{PSD of the symbol stream at the transmitter before and after filtering}
    \label{fig:filter}
\end{figure}

At the receiver side, the stream is convolved again with the matched filter, forming a Nyquist filter and maximizing the SNR. The Nyquist filter allows to cancel the ISI in the sense that it reduces to a Dirac pulse when sampled at the symbol frequency, as shown in Figure \ref{fig:isi}. Hence, no symbol of the symbol stream overlaps with the current received symbol.

\begin{figure}[H]
\centering
    \includegraphics{fig/isi.tex}
     \caption{Illustration of the cancellation of ISI by the Nyquist halfroot filter}
    \label{fig:isi}
\end{figure}

\subsubsection{Noise addition}

The channel is corrupted by Additive White Gaussian Noise (AWGN), which is simulated using its baseband equivalent representation. The channel was simulated for various SNR (defined with respect to the bit energy) and various modulation schemes, and the results are plotted in Figure \ref{fig:ber}. The simulations confirm the theoretical results.

\begin{figure}[H]
\centering
    \includegraphics{fig/ber2.tex}
    \caption{Bit error rate as a function of the SNR for various modulations. The full lines correspond to the theoretical curves and the dots to the simulation.}
    \label{fig:ber}
\end{figure}

As expected, more energy is required to have the same BER when the constellation size is increased. It is worth noting that the worst BER value that could be achieved is 0.5, corresponding to a random choice of the binary values. Moreover, the BER curves for 2-PAM and 4-QAM are superposed because a 4-QAM modulation can be seen as two 2-PAM modulations in quadrature.

\newpage
\subsection{Questions}

\subsubsection{Questions regarding the simulation}

\paragraph{It is proposed to use the baseband equivalent model of the AWGN channel. Would it be possible to live with a bandpass implementation of the system ?} \mbox{}

The baseband model allows to implement the simulations regardless of the carrier frequency. If the system were simulated in bandpass, the required sampling frequency would be extremely high and unrealistic.

\paragraph{How do you choose the sample rate in Matlab ?} \mbox{}

The sampling rate should be at least twice as high as the symbol frequency. It will be increased further when simulating the sample time shift in the third step.

\paragraph{How do you make sure you simulated the desired $E_b/N_0$ ratio ?} \mbox{}

First the energy of a bit is evaluated, then the noise power is computed in order to obtain the desired $E_b/N_0$ ratio. The energy of the bit $E_b$ is given by

\begin{equation*}
    E_b = \frac{T_{samp}}{2 N_{bits}} \sum_{n} {\left| s \left[ n \right] \right|}^2
\end{equation*}

Then $N_0$ can be chosen according to the wanted SNR and the noise is added to the symbols using the baseband representation :

\begin{equation*}
    r \left[ k \right] = \sqrt{N_0 f_{samp}} \left\{ \mathcal{N}(0,1) + j \mathcal{N}(0,1) \right\}
\end{equation*}

The square root is present because of the baseband representation of the noise. Indeed, the computed power is equally distributed between the real and imaginary parts of the noise.

\paragraph{How do you choose the number of transmitted data packets and their length ?} \mbox{}

A rule of thumb is to send $10^{n+2}$ bits of data in order to reliably observe of BER of $10^{-n}$ as it corresponds to 100 detected errors. Moreover, the total number of bits should be a multiple of the number of bits per symbol used in the modulation scheme. The Figure \ref{fig:ber} was realised by sending around $10^8$ bits of data.

\subsubsection{Questions regarding the communication system}

\paragraph{Determine the supported (uncoded) bit rate as a function of the physical bandwidth.} \mbox{}

The bit rate $R$ is given by $R = \frac{log_2 (M)}{T}$ where $M$ is the number of symbols and $T$ is the symbol duration. The Nyquist filtering yields a relationship between the symbol duration and the bandwidth, $\Delta f = 2/T$ (this can be seen on Figure \ref{fig:filter} where T = 0.5 \si{\micro \second} and $\Delta f = 2$ \si{\mega \hertz}). This gives:

\begin{equation*}
    R = \frac{log_2 (M) \ \Delta f}{2}
\end{equation*}

\paragraph{Explain the trade-off communication capacity/reliability achieved by varying the constellation size.} \mbox{}

As detailed in the previous question, increasing the constellation size (the number of bits transmitted per symbol $M$) allows to increase the bit rate. However, the minimum euclidean distance between the symbols in the constellation size decreases, which causes the error probability to increase. As a result, to achieve a similar BER, more energy is required when the constellation size increases but the bit rate also increases.

\paragraph{Why do we choose the halfroot Nyquist filter to shape the complex symbols ?} \mbox{}

The halfroot Nyquist filter allows to limit the communication bandwidth (it is more spectrally efficient than the plain rectangular window) and allows to cancel ISI.

\paragraph{How do we implement the optimal demodulator? Give the optimization criterion.} \mbox{}

The optimal demodulator is implemented 	as a bank of K filters matched to the basis functions of the modulation scheme:

\begin{equation*}
	h_i(t) := s_i(-t) \hspace{1cm}  i=1,...,K
\end{equation*}

The outputs to such filters are the autocorrelations between the input signal and the basis functions. The filters are mathematically equivalent to a bank of K correlators. The optimisation criterion is the maximisation of the SNR at the output. It is realized by the matched filters, and the SNR will be equal to $\frac{2\mathcal{E}}{N_0}$.

\paragraph{How do we implement the optimal detector? Give the optimization criterion.} \mbox{}

There exist two possible optimization criteria for the detector. First, the maximum a posteriori (MAP) criterion, which minimises the error probability by exploiting the knowledge of the a priori probabilities $p(\underline{s}_m)$ and the conditional probabilities $p(\underline{r}|\underline{s}_m)$, where $\underline{r}$ is the observation. It selects the symbol which is the most likely to have been sent, knowing that we observed $\underline{r}$.

	\begin{equation*}
		\underline{\tilde{s}}^{\text{MAP}}_m = \underset{\underline{s}_m}{\text{max}} \ p(\underline{s}_m |\underline{r}) \stackrel{\text{Bayes}}{=}
		\underset{\underline{s}_m}{\text{max}} \ p(\underline{r} |\underline{s}_m) p(\underline{s}_m)
	\end{equation*}

The maximum likelihood criterion is equivalent to the MAP criterion when all the M symbols are equally probable, so that $p(\underline{s}_m) = 1/M$ is constant and can be dropped when maximizing the probabilities.

	\begin{equation*}
		\underline{\tilde{s}}^{\text{ML}}_m = \underset{\underline{s}_m}{\text{max}} \ p(\underline{r} |\underline{s}_m)
	\end{equation*}

The ML criterion determines what symbol is the most likely to have produced the received signal by selecting the symbol which has the lowest euclidean distance to the received symbol.


\section{The Measuring Numbers}

\begin{frame}
  \centering
  \resizebox{0.9\columnwidth}{!}{%
  \begin{tikzpicture}
    \draw[color=gmitred,dotted,-latex] (0,0) arc (180:0:1);
    \draw[latex-latex, color=gmitgrey] (-1.5,0) -- (3.5,0);
    \foreach \x in {-1,0,1,2} \draw[shift={(\x,0)}, color=gmitblue!80] (0pt,0pt) -- (0pt,-5pt) node[below] {\tiny $\x$};
    \draw[shift={(3.14,0)}, color=gmitblue!80] (0pt,0pt) -- (0pt,-5pt) node[below] {\tiny $\pi$};
    \draw[shift={(1.41,0)}, color=gmitblue!80] (0pt,0pt) -- (0pt,-5pt) node[below] {\tiny $\sqrt{2}$};
    \draw[very thick, color=gmitred] (0,0) -- (1,0);
    \end{tikzpicture}
  }
\end{frame}

\begin{frame}[standout]
  \begin{quote}
    What is the distance between 0 and 1? \\[8mm]
    What number comes directly after 0? \\[8mm]
    What number comes directly before 1? \\[8mm]
  \end{quote}
\end{frame}

\begin{frame}
  \begin{columns}
    \begin{column}{0.3\textwidth}
      \color{gmitblue} \fontsize{60}{70}
      \[0.\bar{9}\]
    \end{column}
    {\color{gmitgrey!30}\vrule{}} \hspace{0.01\textwidth}
    \begin{column}{0.5\textwidth}
      \begin{align*}
        &            &  1& &\times && 0.999\ldots & &= & &0&.999\ldots\\[4mm]
        & \Rightarrow& 10& &\times && 0.999\ldots & &= & &9&.999\ldots\\[4mm]
        & \Rightarrow&  9& &\times && 0.999\ldots & &= & &9&.000\ldots\\[4mm]
        & \Rightarrow&  1& &\times && 0.999\ldots & &= & &1&.000\ldots\\[4mm]
        & \Rightarrow&   & &       && 0.999\ldots & &= & &1& \\[4mm]
      \end{align*}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \begin{column}{0.3\textwidth}
      \color{gmitblue} \fontsize{60}{10}
      \[\mathbb{R}\]
    \end{column}
    {\color{gmitgrey!30}\vrule{}} \hspace{0.1\textwidth}
    \begin{column}{0.5\textwidth}
      \begin{align*}
        &\mathbb{N} &\leftrightarrow& &&\mathbb{R} \\[4mm]
        &1 &\leftrightarrow& &43&.17235759\ldots \\[2mm]
        &2 &\leftrightarrow& &3&.19535351\ldots \\[2mm]
        &3 &\leftrightarrow& &209&.00000000\ldots \\[2mm]
        &4 &\leftrightarrow& &0&.12345678\ldots \\[2mm]
        &\vdots &\leftrightarrow& &&\vdots \\[4mm]
        &{\color{gmitred}?} &{\color{gmitred}\leftrightarrow}& &{\color{gmitred}0}&{\color{gmitred}.2015\ldots}
      \end{align*}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}[standout]
  \begin{quote}
    Computer programs are countable. \\[8mm]
    Not all numbers are computable.
  \end{quote}
\end{frame}


\subsubsection{Influence of Time Series Distance Measure}

\begin{frame}{Influence of Time Series Distance Measure}{Normalization}
    \begin{center}
        \resizebox {\textwidth} {!} {
            \begin{tabular}{cc}
                \resizebox {!} {\height} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            legend pos=south west,
                            xmin=0.4,
                            xmax=0.9,
                            ymin=0.2,
                            ymax=0.7,
                            width=\axisdefaultwidth,
                            height=\axisdefaultwidth,
                            xlabel=$Precision_{\mu}$,
                            ylabel=$Recall_{\mu}$]
                            \addplot[blue, only marks, mark size=1] table {../data/fig/distance_measure_result/dtw.dat};
                            \addlegendentry{DTW}
                            \addplot[red, only marks, mark size=1] table {../data/fig/distance_measure_result/ndtw.dat};
                            \addlegendentry{$\eta$DTW}
                            \addplot[green, only marks, mark size=1] table {../data/fig/distance_measure_result/n1dtw.dat};
                            \addlegendentry{$\eta '$DTW}
                            \addplot[gray, domain=0.4:0.9] {(0.3 * x) / (2 * x - 0.3)};
                            \addplot[gray, domain=0.4:0.9] {(0.4 * x) / (2 * x - 0.4)};
                            \addplot[gray, domain=0.4:0.9] {(0.5 * x) / (2 * x - 0.5)};
                            \addplot[gray, domain=0.4:0.9] {(0.6 * x) / (2 * x - 0.6)};
                            \addplot[gray, domain=0.4:0.9] {(0.7 * x) / (2 * x - 0.7)};
                        \end{axis}
                    \end{tikzpicture}
                } &
                \resizebox {!} {\height} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            xmin=0,
                            xmax=1,
                            ymin=0,
                            ymax=1,
                            width=\axisdefaultwidth,
                            height=\axisdefaultwidth,
                            xlabel=$Precision_{\mu}$,
                            ylabel=$Recall_{\mu}$,
                            samples=100]
                            \addplot[blue, only marks, mark size=1] table {../data/fig/distance_measure_result/dtw.dat};
                            \addplot[red, only marks, mark size=1] table {../data/fig/distance_measure_result/ndtw.dat};
                            \addplot[green, only marks, mark size=1] table {../data/fig/distance_measure_result/n1dtw.dat};
                            \addplot[gray, domain=0.051:1] {(0.1 * x) / (2 * x - 0.1)};
                            \addplot[gray, domain=0.11:1] {(0.2 * x) / (2 * x - 0.2)};
                            \addplot[gray, domain=0.16:1] {(0.3 * x) / (2 * x - 0.3)};
                            \addplot[gray, domain=0.21:1] {(0.4 * x) / (2 * x - 0.4)};
                            \addplot[gray, domain=0.26:1] {(0.5 * x) / (2 * x - 0.5)};
                            \addplot[gray, domain=0.31:1] {(0.6 * x) / (2 * x - 0.6)};
                            \addplot[gray, domain=0.36:1] {(0.7 * x) / (2 * x - 0.7)};
                            \addplot[gray, domain=0.41:1] {(0.8 * x) / (2 * x - 0.8)};
                            \addplot[gray, domain=0.46:1] {(0.9 * x) / (2 * x - 0.9)};
                        \end{axis}
                    \end{tikzpicture}
                }
            \end{tabular}
        }
    \end{center}
\end{frame}

\input{experiment/evaluation/influence_of_time_series_distance_measure/influence_of_sakoe-chiba_band_size.tex}


\documentclass[../main.tex]{subfiles}
\begin{document}
Finding a solution to a problem in
Computer  Science  and  Artificial  Intelligence  is  often  thought
as a process of search through the space of  possible  solutions (state space), either carried on some data strutcures, or calculated in the search space of a problem domain.

\paragraph{Searching Strategies} In this part, we will first learn the basic searching algorithms carried out on explicit data strucutres in Chapter~\ref{chapter_basic_searching}. This will include:
\begin{enumerate}
    \item General Searching Strategies
    \item Linear Search
    \item Tree Search
    \item Graph Search
\end{enumerate}
In this chapter, we only explain different strategies on explicitly defineded data structures to keep it simple and clean. The main purpose is to learn the fundamenal concepts and properties to lay the ground for the mode advanced algorithms.

\paragraph{Uninformed search vs informed search} Searching can be categorized as \textbf{uninformed search} (also called \textbf{blind search}) and \textbf{informed search}(also called \textbf{heuristic search}) strategies. The uninformed search means that the strategies have no additional information about states beyond that provided in the problem definition. All they can do is to generate successors and distinguish a goal state from a non-goal state. And we categorize their strategies by the \textit{order} in which nodes are expanded. On the other hand, strategies that know whether one non-goal state is ``more promising" than another are the informed search. In this book, we only cover common uninformed search strategies.

\paragraph{Advanced uninformed searching} learning the basics and knowing the properties, we can move on to more advanced uninformed searching techniques in Chapter~\ref{} that has better efficiency. The content will included:
\begin{enumerate}
    \item Advanced Linear Searching such as Binary Search and Two-pointer technique;
    \item Recursive Backtracking:
    \item Bidirectional BFS.
\end{enumerate}

\textbf{ Complete Search} and \textbf{partial search} are the two main branches in the Searching paradigm.

Complete search is one that guarantees that if a path/solution to the goal/requirement exists, the algorithm will reach the goal given enough time, this is denoted as \textit{completeness}. Complete search is  thought of as a \textit{universal solution} to problem solving.  On the other hand, Partial Search a.k.a Local Search will not always find the correct or optimal solution if one exists because it usually sacrifice completeness for greater efficiency.

In this part of this book,  we will learn the Complete Search instead of the partial search due to its practicity solving the LeetCode Problems. The name ``Complete Search'' does not necessarily mean they are not efficient and a brute force solution all the time. For instance, the \textbf{backtracking} and \textbf{Bi-directional Search} they are more efficient that a brute force exhaustive search solutions.

Complete Search algorithms can be categorized into:
\begin{itemize}
    \item Explicit VS Virtual Search Space: Explicit complete search is carried on  data structures, linear structures or non-linear data structures like graph/ trees. In Explicit Search, the search space size is the size of the targeting data structure size. We will need to find a sub-structure of the given data structure. Virtual space based search is to find a set of value assignments to certain variables that satisfy specific mathematical equations/inequations, or sometimes to maximize or minimiaze a certain function of these variables. This type of problems is known as \textbf{constraint satisfaction problem}. Such as backtracking an optimized search algorithms for virtual space.
    \item Linear VS Non-linear Complete Search: Linear search checks every record in a linear fashion, such as sliding window algorithm, binary search, sweep linear. On the other hand, Non-linear Search is applied on non-linear data structures and follows graph fashion.
    \item Iterative VS Recursive Search: For example, most linear search is iterative. Breath-first-search for graph and level-by-level search for trees are iterative too. Recursive Search are algorithms implemented with recursion, such as Depth-first-search for graph, or DFS based tree traversal, or backtracking.
\end{itemize}

\paragraph{How to Learn Complete Search?} Up till now, we have already learned the explicit complete search carried out on different data structures in Part~\ref{part_data_structure}. In this part, we will complete the complete search topic with more advanced and efficient searching algorithms applied either on real data structures or result space. Also, this part will give us more examples of how to apply the algorithm design methodogy as divide and conquer, use the basic data structures we learned before to design \textbf{complete search} algorithms with efficiency and elegance.

\paragraph{Organization of Complete Search} This part follows the same organization as of  Part~\ref{part_data_structure},  it is composed of linear search or non-linear search. For each type of algorithm, we will explain how to use it on cases like: explicit or virtual search space.

\begin{itemize}
    \item Linear Search (Chapter~\ref{chapter_linear_searching}) which describes the common algorithms that carries on Linear data structures: Array, Linked List and String.
    \item Non-linear Search (Chapter~\ref{chapter_non_linear_searching}) encompasses the most common and basic search techniques for graph and tree data structures. The two most basic search techniques: Breadth-first-search and Depth-first-search serves as the basis to the following more advanced graph algorithms in the next chapter.
    \item Advanced Non-linear Search ( Chapter~\ref{chapter_advanced_non_linear_search}) includes more advanced concepts of graph and more advanced graph search algorithms that solve common problems defined in graph. The problems specifically we include are: Connected Components, topological sort, cycle detection, minimum spanning trees and shortest path related problems.
\end{itemize}

In this part, for graph algorithms, we will only cover medium level, and leave out the more advanced ones in Part~\ref{part_advanced_topics}.

\paragraph{Searching Methodology}
Before we head to more specific searching algorithms, we shall define the searching algorithm design methodology more clearer:

\end{document}


\synctex=1

\documentclass[a4paper,12pt]{scrartcl}

%%%%%%%%%%%%%%%%%
% Customization %
%%%%%%%%%%%%%%%%%

% Load all packages
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath}
\usepackage{geometry}
\usepackage{amsfonts}
\usepackage{amsthm}
\usepackage{mathtools}
\usepackage{biblatex}
\usepackage{setspace}
\usepackage{hyperref}
\usepackage[capitalize]{cleveref}
\usepackage{listings}

\newtheorem{lemma}{Lemma}
\addbibresource{lit.bib}

\renewcommand{\ttdefault}{pcr}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Definition for this document only %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand{\heps}{\ensuremath{\hat \varepsilon}}
\newcommand{\cvx}[1]{\ensuremath{\mathrm{conv}(#1)}}
\newcommand{\hPcond}[2]{\ensuremath{\hat{\mathrm{P}}\{#1|#2\}}}
\newcommand{\Pcond}[2]{\ensuremath{\mathrm{P}}\{#1|#2\}}
\newcommand{\hProb}{\ensuremath{\hat{\mathrm{P}}}}
\newcommand{\Prob}{\ensuremath{\mathrm{P}}}
\newcommand{\ul}[1]{\ensuremath{\underline{#1}}}
\newcommand{\wt}[1]{\ensuremath{\widetilde{#1}}}
\newcommand{\mat}[1]{\ensuremath{\mathbf{#1}}}
\newcommand{\vt}[1]{\ensuremath{\mathbf{#1}}}
\newcommand{\rv}[1]{\ensuremath{\mathsf{\uppercase{#1}}}}
\newcommand{\hrv}[1]{\ensuremath{\hat{\rv{#1}}}}
\newcommand{\transp}{\ensuremath{^{\mathrm T}}}
\newcommand{\AAA}{\ensuremath{\mathcal A}}
\newcommand{\HHH}{\ensuremath{\mathcal H}}
\newcommand{\SSS}{\ensuremath{\mathcal S}}
\newcommand{\OOO}{\ensuremath{\mathcal O}}
\newcommand{\bernoulli}[1]{\ensuremath{\mathcal{B}(#1)}}
\newcommand{\DSBS}[1]{\ensuremath{\mathrm{DSBS}(#1)}}
\newcommand{\RR}{\ensuremath{\mathbb R}}
\newcommand{\defas}{\vcentcolon=}

\def\barcirc{\mathrel{\barcirci}}
\def\barcirci{{%
    \setbox0\hbox{\ensuremath{\relbar\!\!\relbar}}%
    \rlap{\hbox to \wd0{\hss\ensuremath{\circ}\hss}}\box0
}}
\newcommand{\mkv}{\ensuremath{\barcirc}}

\newcommand{\binEnt}[1]{\ensuremath{\mathrm{H}(#1)}}
\newcommand{\binEntInv}[1]{\ensuremath{\mathrm{H}^{-1}(#1)}}
\newcommand{\mutInf}[2]{\ensuremath{\mathrm{I}(#1;#2)}}
\newcommand{\Nto}[1]{\ensuremath{\{1,2,\dots,#1\}}}

\newtheorem{theorem}{Theorem}

\lstloadlanguages{Octave}
\lstset{%
  language=Octave,
  numbers=left,
  stepnumber=2,
  numberstyle=\tiny,
  basicstyle=\footnotesize\ttfamily,
  keywordstyle=\bfseries,
  morekeywords={inf,sup,convhulln,precedes,infsupdec},
  breaklines=true,
}

% This makes sure that firstnumber = firstline
\makeatletter
\patchcmd{\lst@GLI@}% <command>
  {\def\lst@firstline{#1\relax}}% <search>
  {\def\lst@firstline{#1\relax}\def\lst@firstnumber{#1\relax}}% <replace>
  {\typeout{listings firstnumber=firstline}}% <success>
  {\typeout{listings firstnumber not set}}% <failure>
\makeatother

%%%%%%%%%%
% Layout %
%%%%%%%%%%
\geometry{a4paper, inner=2.5cm, outer=2.5cm, top=3.25cm, bottom=3.25cm, headsep=1cm}
\setstretch{1.2}
\setlength\parskip{0pt
	plus .1\baselineskip
}
\setlength\topsep{.5\baselineskip
	plus .1\baselineskip
}

\sloppy
\allowdisplaybreaks
\raggedbottom

%%%%%%%%%%%%%%%%%%%%
% Document Details %
%%%%%%%%%%%%%%%%%%%%
\title{Documentation}
\author{Georg Pichler}
\date{\today}

\begin{document}

% What would you like?
%\longversion %default
%\shortversion %or just put "shortversion" somewhere in the filename

% Title Page
\maketitle

\section{Overview}
\label{sec:overview}
Assume $p \in (0,\frac 12)$ and define the set $\SSS' \subseteq \RR^3$ as
\begin{align}
  \SSS' \defas \smash{\bigcup_{0 \le \alpha, \beta \le \frac 12}} \big\{
  (\mu, R_1, R_2) :
  R_1 &\ge 1 - \binEnt{\alpha},\, \nonumber\\*
  R_2 &\ge 1 - \binEnt{\beta},\, \nonumber\\*
  \mu &\le 1 - \binEnt{\alpha * p * \beta}
        \big\},
\end{align}
where $\binEnt{t} \defas -t\log_2(t) - (1-t) \log_2(1-t)$ is the binary entropy function and $a * b \defas a(1-b)+(1-a)b$ is the binary convolution.
Let $\SSS \defas \cvx{\SSS'}$ denote the convex hull of $\SSS'$.

Note that we can also define $\SSS'' \subseteq \RR^3$ as
\begin{align}
  \SSS'' \defas \smash{\bigcup_{0 \le \alpha, \beta \le \frac 12}} \big\{
  (\mu, R_1, R_2) :
  R_1 &= 1 - \binEnt{\alpha},\, \nonumber\\*
  R_2 &= 1 - \binEnt{\beta},\, \nonumber\\*
  \mu &= 1 - \binEnt{\alpha * p * \beta}
        \big\}
\end{align}
and obtain $\SSS' = \SSS'' + \OOO$, where $\OOO \defas \RR_- \times \RR_+ \times \RR_+ $. Letting $\wt\SSS \defas \cvx{\SSS''}$, we have $\SSS = \wt\SSS + \OOO$.

The random variables $(\rv x, \rv y) \sim \DSBS{p}$ are a doubly symmetric binary source with parameter $p$, i.e., $\rv x \sim \bernoulli{\frac12}$ is a Bernoulli random variable with parameter $\frac12$ and $\rv y \defas \rv x \oplus \rv z$, where $\oplus$ denotes the binary ``xor'' operation. The random variable $\rv z \sim \bernoulli{p}$ is independent of $\rv x$.

We want to show the following two statements:
\begin{theorem}
  \label{thm:1}
  For $p=0$, there exist random variables $(\rv u, \rv v)$ with\footnote{For random variables $\rv x$, $\rv y$ and $\rv z$ we use the notation $\rv x \mkv \rv y \mkv \rv z$ to denote that $\rv x$, $\rv y$ and $\rv z$ form a Markov chain in this order, i.e, $\rv x$ is independent of $\rv z$ given $\rv y$.} $\rv u \mkv \rv x \mkv \rv z \mkv \rv v$ such that $(\mu, R_1, R_2) \notin \SSS'$, where
  \begin{align}
    R_1 &= \mutInf{\rv u}{\rv x} \label{eq:1:1} \\
    R_2 &= \mutInf{\rv v}{\rv z} \label{eq:1:2} \\
    \mu &= \mutInf{\rv u}{\rv v} \label{eq:1:3} .
  \end{align}
\end{theorem}

\begin{theorem}
  \label{thm:2}
  For $p=0.1$, there exist random variables $(\rv u, \rv v)$ with $\rv u \mkv \rv x \mkv \rv z$ and $\rv x \mkv \rv z \mkv \rv v$, such that $(\mu, R_1, R_2) \notin \SSS$, where
  \begin{align}
    R_1 &= \mutInf{\rv u}{\rv x} \\
    R_2 &= \mutInf{\rv v}{\rv z} \\
    \mu &= \mutInf{\rv u}{\rv x} + \mutInf{\rv v}{\rv z} - \mutInf{\rv u \rv v}{\rv x \rv z} .
  \end{align}
\end{theorem}

\section{Proof of Theorem~\ref{thm:1}}
\label{sec:proof-1}

For $a \in [0,1]$ let $(\rv u, \rv v)$ satisfy $\rv u \mkv \rv x \mkv \rv z \mkv \rv v$ as well as
\begin{align}
  \textrm{p}_{\rv u | \rv x}(u | x) =
  \begin{cases}
    a & u=x=0 \\
    0 & u=0, x=1 \\
    1-a & u=1, x=0 \\
    1 & u=x=1
  \end{cases} ,
      &&
         \textrm{p}_{\rv v | \rv z}(v | z) =
         \begin{cases}
           0 & v=z=0 \\
           a & v=0, z=1 \\
           1 & v=1, z=0 \\
           1-a & v=z=1
         \end{cases} .
\end{align}
We obtain $R \defas R_1 = R_2 = \binEnt{\frac{a}{2}} - \frac{1}{2}\binEnt{a}$ and $\mu = \mutInf{\rv u}{\rv v} = 2R - a$ from \cref{eq:1:1,eq:1:2,eq:1:3}. Note that $\rv x = \rv z$ as $p = 0$.
We fix $a = 0.8$ and use interval arithmetic\footnote{We will use underlined letters to denote intervals and use the following notation to compare intervals: $\ul a < \ul b$ iff $a < b$ for all $a \in \ul a$ and $b \in \ul b$. Additionally, we define $b < \ul a$ iff $\{b\} < \ul a$.}~\cite{Moore2009Introduction} to compute compute intervals $\ul R$ and $\ul\alpha$ such that $R \in \ul R$ and $\ul R \subseteq 1-\binEnt{\ul\alpha}$.
We then compute $\ul\mu_{\text{b}} \defas 1 - \binEnt{\ul\alpha * \ul\alpha}$ and finish the poof by showing that $\ul\mu \defas 2\ul R - a > \ul\mu_{\text{b}}$.

The computation is done in \texttt{counterexample1.m}:
\lstinputlisting[%
firstline=17,
lastline=33,
]{../counterexample1.m}

Evaluation confirms that there is a gap with $\ul\mu - \ul\mu_{\text{b}} > 10^{-3}$:

\texttt{Gap is at least 7.948583e-03}

\section{Proof of Theorem~\ref{thm:2}}
\label{sec:proof-2}

\subsection{The Counterexample}
\label{sec:counterexample}

By random search we numerically found an approximate distribution $\hPcond{\hrv u=u, \hrv v=v}{\rv x=x, \rv z=z}$ with $u,v,x,z \in \{0,1\}$, that will serve as a candidate.
We interpret the conditional distribution $\hProb$ as a vector in $[0,1]^{16}$. Additionally $\hProb$ has the property that $\hPcond{\hrv u=u}{\rv x=x} \approx \hPcond{\hrv v=u}{\rv z=x}$, up to numerical error.
Notice that the required Markov chains $\rv u \mkv \rv x \mkv \rv z$ and $\rv x \mkv \rv z \mkv \rv v$, as well as the requirements $\sum_{u,v} \Pcond{\rv u=u, \rv v=v}{\rv x=x, \rv z=z} = 1$ for all $x,z \in \{0,1\}$ are linear constraints on $\Pcond{\rv u=u, \rv v=v}{\rv x=x, \rv z=z} \in [0,1]^{16}$.
By calculating the kernel of the corresponding matrix, we can find a symbolic approximation $\mathrm{P}$ of $\hat{\mathrm{P}}$ which is a probability distribution satisfying the required Markov chains, as well as $\Pcond{\rv u=u}{\rv x=x} = \Pcond{\rv v=u}{\rv z=x}$ for $u,x \in \{0,1\}$.
Hence, we found the candidate $(\mu^\circ, R^\circ, R^\circ)$ with $R^\circ=\mutInf{\rv u}{\rv x}=\mutInf{\rv v}{\rv z}$ and $\mu^\circ = 2R^\circ - \mutInf{\rv u \rv v}{\rv x \rv z}$. Using interval arithmetic\footnote{We will use underlined letters to denote intervals and use the following notation to compare intervals: $\ul a < \ul b$ iff $a < b$ for all $a \in \ul a$ and $b \in \ul b$. Additionally, we define $b < \ul a$ iff $\{b\} < \ul a$.}~\cite{Moore2009Introduction}, we compute intervals $\ul\mu^\circ \ni \mu^\circ$ and $\ul R^\circ \ni R^\circ$.

\subsection{The Lower Bound}
\label{sec:lower-bound}

Let $\hat\mu(R_1,R_2) \defas \max\{\mu : (\mu, R_1, R_2) \in \SSS \}$ and by slightly abusing notation $\hat\mu(R) \defas \hat\mu(R,R)$. We will upper bound $\hat\mu(R^\circ)$ in order to ultimately prove $\hat\mu(R^\circ) < \ul\mu^\circ$.

Given a rate pair $(R_1, R_2) \in [0,1]^2$, we can obtain intervals $\ul\alpha, \ul\beta$ such that $R_1 \in 1 - \binEnt{\ul\alpha}$ and $R_2 \in 1 - \binEnt{\ul\beta}$ using an interval implementation of the inverse binary entropy function. This yields an interval $\ul\mu(\ul\alpha, \ul\beta) \supseteq 1 - \binEnt{\ul\alpha * p * \ul\beta}$.
Given strictly increasing rates $R(i) \in [0,1]$ with $i \in \Nto{K}$ and $0 = R(1) < R(2) < \dots < R(K) = 1$,
we perform this process on the $K \times K$ grid $(R(i), R(j))_{i,j=1\dots K}$ where $K \approx 800$ and obtain $\ul\mu(i,j)_{i,j=1\dots K}$, an interval representation of $\SSS''$. However, we are interested in $\wt\SSS = \cvx{\SSS''}$.
We shift $\ul\mu$ by one and obtain $\ul{\wt\mu}(i,j) \defas \ul\mu\big(\min(i+1,K),\min(j+1,K)\big)$.

\begin{lemma}
  \label{lem:bound}
  Let $\vt a \in \OOO$, $b \in \RR$. If $\vt a \cdot (\ul{\wt\mu}(i,j), R(i), R(j)) \ge b$ for all $i,j = 1\dots K$, then $\vt a \cdot \vt x \ge b$ for all $\vt x \in \wt\SSS$.
\end{lemma}
\begin{proof}
  % Observe that $\mu(i,j)$ is increasing in the sense that $\mu(i,j) \le \mu(i+1,j)$ and $\mu(i,j) \le \mu(i,j+1)$.
  If there is a point in $\vt x \in \wt\SSS$ with $\vt a \cdot \vt x < b$, then there must also be a point $\vt x' = (\mu, R_1, R_2)  \in \SSS''$ with $\vt a \cdot \vt x' < b$. There is a square $[R(i), R(i+1)] \times [R(j), R(j+1)] \ni (R_1, R_2)$. Clearly $\mu \not\ge \ul{\wt\mu}(i,j)$ and hence $\vt a \cdot (\ul{\wt\mu}(i,j), R(i), R(j)) \not\ge \vt a \cdot \vt x' < b$.
\end{proof}

We obtain $\vt a = (a_1, a_2, a_3) \in \OOO$, $b \in \RR$ and verify the condition of \cref{lem:bound}. Clearly $\vt a \cdot \vt x \ge b$ for all $\vt x \in \SSS$. It remains to verify that
\begin{align}
  a_1 \ul\mu^\circ + (a_2 + a_3) \ul R^\circ < b  .
\end{align}

\subsection{Verification}
\label{sec:verification}

The verification of the counterexample is performed by \texttt{counterexample2.m}. After some preliminary checks and loading the required packages, the approximate distribution $\hPcond{\hrv u=u, \hrv v=v}{\rv x=x, \rv z=z}$ is imported from \texttt{numeric\_distribution.dat} and we set some parameters that will be used for calculating the sequence of rates $0 = R(1) < R(2) < \dots < R(K) = 1$. We also initialize the inverse binary entropy function class \texttt{ibinent}.

\lstinputlisting[%
firstline=14,
lastline=22,
]{../counterexample2.m}

The meaning of $K$, $Q$, $\mathrm{bdp}$ is a follows. The first $\frac K4$ rates will be spaced uniformly in $[0, \mathrm{bdp}]$. The remaining $\frac{3K}{4}$ rates are calculated as $R=1-x^Q$, where $x$ is uniformly spaced in $[0, (1-\mathrm{bdp})^{\frac 1Q})$.

Next we construct the symbolic distribution $\mathrm{P} = \texttt{Psc\_UVXZ}$. We do so by first constructing a symbolic distribution $\texttt{Psv\_XZ} = \Prob(\rv x=x, \rv z=z)$ and the symbolic reference distribution $\texttt{Psv\_UVgXZ\_ref} = \mathrm{P}_{\mathrm{ref}}(\hrv u=u, \hrv v=v|\rv x=x, \rv z=z)$ that satisfies the long Markov chain $\hrv u \mkv \rv x \mkv \rv z \mkv \hrv v$ and has two identical BSCs $\rv x \to \hrv u$, $\rv z \to \hrv v$.
Then we apply the projection matrix returned by \texttt{get\_projection\_matrix()} to the conditional distribution $\hPcond{\hrv u=u, \hrv v=v}{\rv x=x, \rv z=z} = \texttt{Psv\_UVgXZ}$.

\lstinputlisting[%
firstline=79,
lastline=84,
]{../counterexample2.m}

We use these symbolic distributions to calculate the intervals $\texttt{Ri} = \ul R^\circ$ and $\texttt{mui} = \ul \mu^\circ$.

\lstinputlisting[%
firstline=120,
lastline=124,
]{../counterexample2.m}

The last section is dedicated to the inner bound. First the rates $\texttt{Rv} = R(i)_{i=1\dots K}$ are calculated and we obtain the grid $\texttt{MUmi} = \ul\mu(i,j)_{i,j=1\dots K}$, which is subsequently shifted to obtain $\ul{\wt{\mu}}$.

\lstinputlisting[%
firstline=137,
lastline=151,
]{../counterexample2.m}

Finally, we use \texttt{upper\_concave\_envelope3\_tri} to obtain the triangularization $\mat T$. Using the function \texttt{tsearchn} we find the triangle that contains $(\ul R^\circ, \ul R^\circ)$. We construct a supporting hyperplane using \texttt{hyperplane3}, verify it with \texttt{verify\_hyperplane} and finally obtain the upper bound to our inner bound $\texttt{MUi} = \ul \mu$ by interpolating on the resulting hyperplane.

\lstinputlisting[%
firstline=168,
]{../counterexample2.m}

Indeed, evaluation confirms that there is a gap with $\ul \mu^\circ - \ul \mu > 10^{-4}$:

\texttt{Distance is at least 1.181411e-04}

\section{Source Files}
\label{sec:source-files}

\begin{itemize}
\item \texttt{utils/binent.m}

  The function $\texttt{binent(x)}$ computes the binary entropy function $\texttt{binent(x)} = \binEnt{x}$. If the argument is a vector, $\binEnt{x}$ it is computed element-wise.

  \lstinputlisting{../utils/binent.m}
\item \texttt{utils/hyperplane3.m}

  Given three interval vectors $\ul{\vt x}, \ul{\vt y}, \ul{\vt z} \subseteq \RR^3$, we obtain the three vectors $\ul{\vt w}_i = (\ul{x}_i, \ul{y}_i, \ul{z}_i)$, with $i \in \{1,2,3\}$.
  Let $\vt w_i$ denote the mean of the interval vector $\ul{\vt w}_i$ for $i \in \{1,2,3\}$. The function the computes a vector $\vt a \perp \{\vt w_1 - \vt w_2, \vt w_1 - \vt w_3\}$ with $a_3 \le 0$. It returns $(\vt a, b) = \texttt{hyperplane3(x,y,z)}$ such that $b$ is maximal with $\vt a \cdot \ul{\vt w}_i \ge b$ for $i \in \{1,2,3\}$.

  \lstinputlisting{../utils/hyperplane3.m}
\item \texttt{utils/ent.m}

  This function computes the entropy of a p.m.f.\ $\vt x$, $\texttt{ent(x)} = \binEnt{\vt x}$, where
  \begin{align}
    \binEnt{\vt x} = - \sum_{i=1}^{n} x_i \log_2 x_i .
  \end{align}
  If the argument has more than one dimension, $\binEnt{\vt x}$ is computed along the first dimension.

  \lstinputlisting{../utils/ent.m}
\item \texttt{utils/get\_projection\_matrix.m}

  Let $\AAA \in \RR^{16}$ be the affine space, such that $\AAA \cap [0,1]^{16}$ is the set of conditional probability distributions $\Pcond{\rv u=u, \rv v=v}{\rv x=x, \rv z=z}$ with $x,z,u,v \in \{0,1\}$, such that $\rv u \mkv \rv x \mkv \rv z$, $\rv x \mkv \rv z \mkv \rv v$ and the marginals $\Pcond{\rv u=u}{\rv x=x}$ and $\Pcond{\rv v=v}{\rv z=z}$ are constant.
  We may write $\AAA = \vt x + \AAA'$ where $\vt x \in \AAA$ and $\AAA'$ is a linear space.
  The function $\mat N = \texttt{get\_projection\_matrix()}$ returns a $16 \times 16$ symbolic matrix $\mat N$, that performs an orthogonal projection onto $\AAA'$.

  \lstinputlisting{../utils/get_projection_matrix.m}
\item \texttt{utils/ibinent.m}

  The class \texttt{ibinent} provides an interval version of the inverse binary entropy function $\binEntInv{\cdot}$. To construct it, we take advantage of the fact that $\binEnt{\cdot}$ is a concave function.

  \lstinputlisting{../utils/ibinent.m}

\item \texttt{utils/star.m}

  This functions computes the binary convolution $\texttt{star(a,b)} = a (1-b) + (1-a) b$. If the arguments are vectors, the function is computed element-wise.

  \lstinputlisting{../utils/star.m}

\item \texttt{utils/upper\_concave\_envelope3\_tri.m}

  Given three vectors $\vt x$, $\vt y$, and $\vt z$, each of length $n$, we interpret them as a points $\vt w_i = (x_i, y_i, z_i)$, where $i \in \Nto{n}$.
  These points span the polytope $\AAA \defas \cvx{\vt w_i : i \in \Nto{n}}$. Let $\OOO \defas \{(x,y,z) : x,y \ge 0, z \le 0 \}$. We are interested in the compact faces of $\AAA' \defas \AAA + \OOO$. The function $\mat T = \texttt{upper\_concave\_envelope3\_tri(x,y,z)}$ returns an $M \times 3$ matrix $\mat T = (\vt k^{(1)}, \vt k^{(2)}, \dots ,\vt k^{(M)})\transp$. The $M$ triangles $\cvx{\vt w_{k^{(m)}_i} : i \in \{1,2,3\}}$, $m \in \Nto{M}$, provide a triangularization of all compact faces of $\AAA'$.

  \lstinputlisting{../utils/upper_concave_envelope3_tri.m}

\item \texttt{utils/verify\_hyperplane.m}

  This function verifies that the pair $(\vt a, b)$ represents a supporting hyperplane. Given three (interval) vectors $\ul{\vt x}$, $\ul{\vt y}$, and $\ul{\vt z}$, each of length $n$, we interpret them as a (interval) points $\ul{\vt w}_i = (\ul x_i, \ul y_i, \ul z_i)$, where $i \in \Nto{n}$. The function $b=\texttt{verify\_hyperplane(q,x,y,z)}$ returns the Boolean $b = 1$ if $\ul{\vt w}_i \cdot \vt a \ge b$ for all $i \in \Nto{n}$, where $(\vt a, b) = \texttt{q}$. Otherwise the function returns $b = 0$.

  \lstinputlisting{../utils/verify_hyperplane.m}
\item \texttt{utils/z\_interp2.m}

  Given the hyperplane $\HHH \defas \{\vt v \in \RR^3: \vt a \cdot \vt v = b\}$ and two values (possibly intervals) $\ul x$ and $\ul y$, the function $\ul z =  \texttt{z\_interp2(q,x,y)}$ returns $\ul z$ such that $(x,y,z) \in \HHH \cap ( \ul x \times \ul y \times \RR )$ implies $z \in \ul z$. The first argument is $\texttt{q} = (\vt a, b)$.

  \lstinputlisting{../utils/z_interp2.m}
\end{itemize}

% \begin{appendix}
%   \section{Appendix}
% \end{appendix}

% Bibliography
%\cleardoublepage
\printbibliography[heading=bibintoc]

\end{document}


\documentclass[11pt]{scrartcl} % Font size
\input{../structure.tex} % Include the file specifying the document structure and custom commands

%----------------------------------------------------------------------------------------
%	TITLE SECTION
%----------------------------------------------------------------------------------------

\title{
	\normalfont\normalsize
	\textsc{Harvard Privacy Tools Project}\\ % Your university, school and/or department name(s)
	\vspace{25pt} % Whitespace
	\rule{\linewidth}{0.5pt}\\ % Thin top horizontal rule
	\vspace{20pt} % Whitespace
	{\huge Count Sensitivity Proofs}\\ % The assignment title
	\vspace{12pt} % Whitespace
	\rule{\linewidth}{2pt}\\ % Thick bottom horizontal rule
	\vspace{12pt} % Whitespace
}

% \author{\LARGE} % Your name

\date{\normalsize\today} % Today's date (\today) or a custom date

\begin{document}

\maketitle

\begin{definition}
Let $\mathcal{X}$ be the universe of possible rows (individuals) and let $I: \mathcal{X} \rightarrow \{0,1\}$ be a predicate on rows. Let $x \in \mathcal{X}^n$ be a dataset. Then a count over $x$ is defined as
$$ q(x) = \sum_{i=1}^n I(x_i).$$
\end{definition}

\begin{definition}
Let $q_1, \ldots, q_k$ be a series of counts with predicates $I_1, \ldots, I_k$. These counts are disjoint for every row in the database if only one of them evaluates to 1. In other words, they are disjoint if $\forall x_i \in \mathcal{X},$
$$ \sum_{j=1}^k I_j(x_i) = 1.$$
\end{definition}

\section{Neighboring Definition: Change One}

% l1 sensitivity
\subsection{$\ell_1$-sensitivity}

\begin{theorem}
\label{thm:change1L1}
A single count query has $\ell_1$-sensitivity 1 in the change-one model. A series of $k$ disjoint counts has $\ell_1$-sensitivity 2 in the change-one model.
\end{theorem}

\begin{proof}
Let $q$ be a count query with predicate $I$, and let $x'$ be equal to $x$ with point $x_i$ changed to $x_i'$. Then,

\begin{align*}
\left\vert q(x') - q(x) \right\vert &= \left\vert \sum_{j=1}^n I(x_j') - \sum_{j=1}^n I(x_j) \right\vert \\
	&= \left\vert \sum_{\{ i \in [n] \vert i \ne j\}} I(x_j) + I(x_i') - \sum_{\{ i \in [n] \vert i \ne j\}} I(x_j) - I(x_i) \right\vert \\
	&= \left\vert I(x_i') - I(x_i) \right\vert \\
	& \le 1.
\end{align*}

Consider a series of $k$ disjoint count queries $\mathbf{q} = \{q_1, \ldots, q_k\}$ on the same databases $x$ and $x'$. Note that since the counts are disjoint, $x_i$ and $x_i'$ can at most each increment a single one of the $k$ counts by 1. Call the count that $x_i$ impacts $q_i$, and the count that $x_i'$ impacts $q_j$. Then,

\begin{align*}
\left\vert \mathbf{q}(x) - \mathbf{q}(x') \right\vert &= \left\vert \left(q_1(x) - q_1(x')\right) + \ldots + \left(q_k(x) - q_k(x')\right) \right\vert \\
	&= \left\vert \right(q_i(x) - q_i(x')\left) + \right(q_i(x) - q_j(x')\left) \right\vert \\
	&\le 2
\end{align*}
\end{proof}

% l2 sensitivity
\subsection{$\ell_2$-sensitivity}

\begin{theorem}
A single count query has $\ell_2$-sensitivity 1 in the change-one model. A series of $k$ disjoint counts has $\ell_2$-sensitivity $\sqrt{2}$ in the change-one model.
\end{theorem}

\begin{proof}
From the proof of Theorem \ref{thm:change1L1}, the difference between counts on two neighboring databases is at most 1. Squaring this gives the same value. For a series of $k$ disjoint counts,
\begin{align*}
\left\vert \mathbf{q}(x) - \mathbf{q}(x') \right\vert_2 &= \left( \left(q_1(x) - q_1(x')\right)^2 + \ldots + \left(q_k(x) - q_k(x')\right)^2 \right)^{1/2} \\
 & \le \left( 1^2 + 1^2 \right)^{1/2} \\
 &= \sqrt{2}.
 \end{align*}
\end{proof}

\section{Neighboring Definition: Add/Drop One}
% l1 sensitivity
\subsection{$\ell_1$-sensitivity}
\begin{theorem}
\label{thm:addDropL1}
A single count query has $\ell_1$-sensitivity 1 in the add/drop-one model. A series of $k$ disjoint counts also has $\ell_1$-sensitivity 1 in the add/drop-one model.
\end{theorem}

\begin{proof}
Let $q$ be a count query with predicate $I$, and let $x'$ be equal to database $x$ with point $x_i$ removed. Then
\begin{align*}
\left \vert q(x) - q(x') \right\vert &= \left\vert \sum_{j=1}^n I(x_j) - \sum_{\{ i \in [n] \vert i \ne j\}} I(x_j) \right\vert\\
	&= \left\vert \sum_{\{ i \in [n] \vert i \ne j\}} I(x_j) + I(x_i) - \sum_{\{ i \in [n] \vert i \ne j\}} I(x_j) \right\vert\\
	&\le 1.
\end{align*}

A nearly identical argument holds for adding a point. \\

Consider a series of $k$ disjoint count queries $\mathbf{q} = \{q_1, \ldots, q_k\}$ and consider database $x'$ equal to database $x$ with point $x_i$ removed. Note that only a single one of the $k$ queries will be affected by the change from $x$ to $x'$, so
\begin{align*}
\left\vert \mathbf{q}(x) - \mathbf{q}(x') \right\vert &= \left\vert \left(q_1(x) - q_1(x')\right) + \ldots + \left(q_k(x) - q_k(x')\right) \right\vert \\
	&\le 1.
\end{align*}
The same argument holds for $x'$ equal to $x$ with a single point added.
\end{proof}
% l2 sensitivity
\subsection{$\ell_2$-sensitivity}
\begin{theorem}
A single count query has $\ell_2$-sensitivity 1 in the add/drop-one model. A series of $k$ disjoint counts also has $\ell_2$-sensitivity 1 in the add/drop-one model.
\end{theorem}

% \bibliographystyle{alpha}
% \bibliography{mean}

\end{document}


\documentclass[a4paper,11pt]{article}
%Declare document type and font size

\usepackage{latexsym,amssymb,enumerate,amsmath,epsfig,amsthm}
\usepackage[margin=1in]{geometry}
\usepackage{setspace,color}
\usepackage{graphicx}
\usepackage{subfigure}

%Specify the packages needed for compiling this document

\theoremstyle{definition}
\newtheorem{thm}{Theorem}
\newtheorem{df}{Definition}
\newcommand{\ds}{\displaystyle}
\setlength{\parindent}{0pt}

%Define new commands and environments

\title{Lecture notes on Multivariable Calculus}
\author{}
\date{}

\begin{document}

\maketitle
\section*{Change of Variables for Double Integrals}
The following material is taken from \cite{stewart12}.
\newline
\newline
Let \(z = f(x,y)\) and \({D}\) be a bounded region in $\(\mathbb{R}\)^2$. The double integral

$$\iint_D f(x,y) \,dA$$

is usually not easy to compute directly if \({D}\) is neither Type I nor Type II region. If \({D}\) is a simple polar region, we can change the integral to polar coordinates such that the double integral is easier to compute. More generally, suppose we have a new coordinate system (u, v) such that

\[
\left \{
  \begin{tabular}{ccc}
  $x =x(u,v)$ \\
  $y =y(u,v)$
  \end{tabular}
\]

The function $T (u, v) = (x(u, v), y(u, v))$ from $\(\mathbb{R}\)^2$ to $\(\mathbb{R}\)^2$ is called a \textbf{transformation}. We assume that T is \textbf{bijective} i.e. both one-to-one and onto. Hence $T^{-1}$ exists. Now the question is: How can we change a double integral $$\iint_D f(x,y) dA$$ in terms of the new coordinate system $(u,v)$?

\begin{thm}
Let $f(x,y)$ be a two-variable function defined on a bounded region \({D}\) in the $xy$-space. Let $S = T^{-1}(D)$ be the region in the $uv$-space. Then we have

    $$\iint_D f(x,y) \,dA = \iint_S f(x(u,v),y(u,v)) \left\Bigg|\frac{\partial(x,y)}{\partial(u,v)} \right\Bigg| \,du \,dv$$

where $\frac{\partial(x,y)}{\partial(u,v)}$ is called the \textbf{Jacobian}, which is defined as follows:

$$
\frac{\partial(x,y)}{\partial(u,v)} =
\left \Bigg|
  \begin{tabular}{ccc}
  $ \frac{\partial(x)}{\partial(u)}$ \smallskip $\frac{\partial(x)}{\partial(v)} $ \\
  $ \frac{\partial(y)}{\partial(u)}$ \smallskip $\frac{\partial(y)}{\partial(v)} $
  \end{tabular}
\right \Bigg|
= \frac{\partial x}{\partial u}\frac{\partial y}{\partial v} - \frac{\partial x}{\partial v}\frac{\partial y}{\partial u}
$$

\newpage
To understand why we need to multiply the Jacobian to the function in the double integral when we change the coordinate system, we first consider the small increments $\Delta u$ and $\Delta v$ in the $uv$-space. The rectangle $[u, u + \Delta u] \times [v, v + \Delta v]$ is transformed by $T$ into the region in the $xy$-space as shown in Figure 1.

\begin{figure}[h]
\centering
    \includegraphics[width=0.7\textwidth]{change_figure}}
\caption{The transformation $T$ : $\(\mathbb{R}\)^2$ \rightarrow $\(\mathbb{R}\)^2$}
\end{figure}

By definition, we have
\begin{gather*}
P = T(u,v) = (x(x,v),y(u,v)) \\
Q = T(u+\Delta u,v) = (x(u+\Delta u,v),y(u+\Delta u,v) \\
R = T(u,v+\Delta v) = (x(u,v+\Delta v),y(u,v+\Delta v)
\end{gather*}

\end{thm}

\begin{thebibliography}{1}
\bibitem{stewart12}{\sc J. Stewart}, {\em Calculus, Early Transcendentals}, Cengage Learning, 2012.
\end{thebibliography}

\end{document}


\subsection{Wirtinger derivatives}

Previously we had partial differentiation on the real line. We could use the partial differention operator

We want to find a similar operator for the complex plane.


\section{Title2Rec}
\label{sec:t2r}
Title2Rec recommends tracks taking as input the playlist title, following the procedure illustrated in Figure~\ref{fig:t2r_rec}. The title is translated into a vector $p_{t2r}$, named title embedding, computed by applying the strategy described in Section~\ref{sec:title_embs} to the playlists defined in the MPD dataset.

Given a new seed playlist, we compute its title embedding in the same way. Then, we select a subset $P$ including the top-300 most similar playlists to the given one by comparing its embeddings with $p_{t2r}$ using the cosine similarity. Finally, the required number of tracks are selected among the ones available in $P$. The tracks have been ordered to ensure that the most popular ones in $P$ are placed at the top of the list.

% source https://docs.google.com/presentation/d/1KV4eFuYvFxS1Z25ZwOqu5TJ2zuBbeRAZAkR2Hbf8h_A/edit#slide=id.g3b0c032961_0_82
\begin{figure}
    \centering
    \includegraphics[width=\textwidth, width=0.4\textwidth]{figures/t2r_rec.pdf}
    \caption{The Title2Rec algorithm compares the fastText representation of the title of a seed playlist to the known ones using the cosine similarity.}
    \label{fig:t2r_rec}
\end{figure}