\documentclass[report.tex]{subfiles}
\begin{document}

\chapter{Related Work} % (fold)
\label{cha:related_work}

% ==============================================================================
% ==============================================================================

\section{Non-Local Boxes} % (fold)
\label{sec:related_nlbs}
Non-local boxes (NLBs) are theoretical objects where multiple parties share some
correlation. The study of non-local boxes was introduced by physicists Sandu
Popescu and Daniel Rohrlich as part of their research on quantum non-locality.

Consider a device with two ends, held by Alice and Bob. Each provides an
input (\(x\) and \(y)\), and observe the output produced by the box (\(a\) and
\(b)\). The box is defined by the probability distribution \(P(a,b | x,y)\)
where \(a, b, x, y \in {0, 1}\) \cite{nlb_lamontagne}.

\begin{figure}[H]
  \centering
  \includegraphics[width=0.66\textwidth]{img/nlb}
  \caption{A non local box \cite[Section~I]{2006quant.ph..9166D}}
\end{figure}

This device will immediately produce an output when given an input, preventing
any communication as a result of a delayed input. These devices are
non-signalling, ensuring that Alice cannot get any information about Bob's input
from her output, forbidding faster than light communication.

Popescu and Rohrlich introduced the concept of a non-local box a correlation
that could achieve the algebraic maximum violation of 4 in the CHSH inequality
while also non-signalling \cite{ref1}, the CHSH inequality introduced by Clauser
et al being a constraint in quantum theory \cite{PhysRevLett.23.880}. This work
led to the further study of other non-local boxes as well, with expanded scope
in terms of the inputs and outputs of these boxes.
% subsubsection pr_box (end)

\subsection{Non-signalling} % (fold)
\label{sub:non_signalling}
As mentioned in Section \ref{sec:related_nlbs}, an important part of the
non-local box is that it is non-signalling. Non-signalling means that in the
described experiment, Alice cannot be made aware of Bob's choice of input or
vice-versa. This requirement imposes restrictions on the probability
distribution, since the probability of an output should depend solely on its
respective input.

The non-signalling restriction can be represented by the given formulae
\cite[Section~II.A]{PhysRevA.71.022101}:
\begin{equation} \label{eq:non-signal_1}
\sum_{b} P(a, b | x, y) = \sum_{b} P(a, b | x, y') = P(a | x)
\quad \forall a, x, y, y'
\end{equation}

\begin{equation} \label{eq:non-signal_2}
\sum_{a} P(a, b | x, y) = \sum_{a} P(a, b | x', y) = P(b | y)
\quad \forall b, x, x', y
\end{equation}

These restrictions allow for a definition of a reduced probability on the
outputs as well, \(P(a | x)\), which is useful information to have when
working with non-local boxes. For example, the modelling of NLBs in PRISM relies
heavily on having a definition of reduced probabilities, meaning that the lack
of such information would impede on the creation of such models.
% subsection non_signalling (end)

\subsection{Example NLBs} % (fold)
\label{sub:example_nlbs}
Provided are some examples of standard non-local boxes. It is possible to have
non-local boxes with more than two inputs and outputs, and it is also possible
to have non-local boxes that handle a larger set of values than binary digits
as well. Note that for the tables below, the rows indicate input and the columns
indicate output.

\subsubsection{Classical Model} % (fold)
\label{ssub:classical_model}
The classical model is a simple example of a non-local box. In our theoretical
device, a coin is toss inside the device, and whatever result it gives is sent
as the output to Alice or Bob. This behaviour produces the following probability
distribution.

\begin{table}[H]
  \centering
  \begin{tabular}{l | c c c c}
	  & 00 & 01 & 10 & 11 \\
	  \hline
	  00 & \(\frac{1}{2}\) & 0 & 0 & \(\frac{1}{2}\) \\
	  01 & \(\frac{1}{2}\) & 0 & 0 & \(\frac{1}{2}\) \\
	  10 & \(\frac{1}{2}\) & 0 & 0 & \(\frac{1}{2}\) \\
	  11 & \(\frac{1}{2}\) & 0 & 0 & \(\frac{1}{2}\) \\
  \end{tabular}
  \caption{Probability Distribution}
  \label{tab:classical}
\end{table}

As you can see, this upholds the non-signalling requirement, since the
distribution shows that a change of one input cannot influence the output at the
other end of the device.
% subsubsection classical_model (end)

\subsubsection{PR-Box} % (fold)
\label{ssub:pr_box}
The PR-box, originally proposed by and named after Popescu and Rohrlich, can be
considered the `Hello World' of non-local boxes. It is the most basic structure,
with two inputs and two outputs, and is the basis for this area of research.

The PR-Box has the following probability distribution:
\[
	P(a, b | x, y) =
	\begin{cases}
		\frac{1}{2} & \quad a \text{ XOR } B = x \text{ AND } y \\
		0 & \quad \text{otherwise} \\
	\end{cases}
\]

This set-up can again be described using a matrix denoting the probabilities as
in \ref{ssub:classical_model}, although the above notation is commonly used in
papers related to quantum non-locality.

\begin{table}[H]
  \centering
\begin{tabular}{l | c c c c}
  & 00 & 01 & 10 & 11 \\
  \hline
  00 & \(\frac{1}{2}\) & 0 & 0 & \(\frac{1}{2}\) \\
  01 & \(\frac{1}{2}\) & 0 & 0 & \(\frac{1}{2}\) \\
  10 & \(\frac{1}{2}\) & 0 & 0 & \(\frac{1}{2}\) \\
  11 & 0 & \(\frac{1}{2}\) & \(\frac{1}{2}\) & 0 \\
\end{tabular}
  \caption{Probability Distribution}
  \label{tab:pr}
\end{table}
% subsection example_nlbs (end)
% section related_nlbs (end)

% ==============================================================================

\section{PRISM} % (fold)
\label{sec:prism}
PRISM \cite{KNP11} is a probabilistic model checking tool. It allows for
the modelling of systems which exhibit some random nature to them. Examples
include quantum cryptography or wireless networking protocols. Models as such
discrete-time Markov chains can be created using the PRISM language, and
experiments can be ran using these models. Section \ref{sub:nlbs_in_prism}
provides an example of a non-local box in PRISM, breaking down the modules and
providing commentary on how it works.

PRISM uses a state based language, and includes tools for performing various
forms of analysis on a given model. A typical PRISM file will be made up of
multiple modules and variables, which interact with each other in various ways.
A module can read the values of other variables to update its own variables,
thus changing the state of the model. In addition, these modules can perform
synchronized actions with each other as well, meaning that multiple variables
can update in different modules simultaneously.

\subsection{NLBs in PRISM} % (fold)
\label{sub:nlbs_in_prism}
PRISM can be used for modelling non-local boxes, although it's weaknesses in
doing so are the inspiration for this project. The PR-box described in
Section \ref{ssub:pr_box} will be used as the example model, which will be
broken down for easier understanding.

The box is represented using a discrete-time Markov chain, meaning that the
model will transition from one state to the next based only on the current
state, with no memory of the steps preceding.

\subsubsection{Input} % (fold)
\label{ssub:input}
\lstinputlisting[
	style = snippet,
	firstnumber = 1,
	linerange = {3-9},
	caption = {pr.qgrady input module.},
	label = {lst:inputmodule}
]{files/pr.prism}

Each input of the box is given its own module in the model. These modules
contain the variable that represents its input, initialised at -1 to indicate
that it's value is yet to be determined. Line 3 includes a guard indicating that
its actions can only be taken if \texttt{x} is equal to -1. In this case, it has
a 50/50 chance of being 0 or 1, effectively a coin toss.

Lines 5 and 6 provide a way to synchronize depending on the value of \texttt{x},
this is how the output module of the model will be alerted of the input's value,
and can then determine the appropriate output. As you can see, this does not
change the value of \texttt{x}, which will remain the same no matter what,
avoiding the potential for unwanted side effects.
% subsubsection input (end)

\subsubsection{Output Declaration} % (fold)
\label{ssub:output_declaration}
\lstinputlisting[
	style = snippet,
	firstnumber = 1,
	linerange = {19-27, 51},
	caption = {pr.qgrady output module variables and synchronize flags.},
	label = {list:outputmodule}
]{files/pr.prism}

The output module includes all outputs produced by the box, with the addition of
a \texttt{ready} variable that acts as a block as part of the module. This has
the purpose of ensuring that when the module synchronizes with an input, the
respective output must be determined next before continuing.

The output variables have similar synchronized actions that can allow parties
interested in using the value of the output in determining the next state to
transition to.
% subsubsection output_declaration (end)

\subsubsection{Output Reduced Probabilities} % (fold)
\label{ssub:output_reduced_probabilities}
\lstinputlisting[
	style = snippet,
	firstnumber = 1,
    linerange = {29-32},
	caption = {Reduced probability of outputs.},
	label = {lst:reducedprob}
]{files/pr.prism}

Moving onto the reduced probabilities. This is one reason why non-signalling
must be cared about. Its ability to provide a definition of the reduced
probabilities means that we can use them as part of the model. In this instance,
each variable has a reduced probability of 50\% to be either 0 or 1.

Note that the module must be ready in order to perform this action, synchronized
with the input actions acting as a flag alerting that this assignment can take
place.
% subsubsection output_reduced_probabilities (end)

\subsubsection{Output Normalised Probabilities} % (fold)
\label{ssub:output_normalised_probabilities}
\lstinputlisting[
	style = snippet,
	firstnumber = 1,
	linerange = {34-41},
	caption = {Reduced probability of outputs.},
	label = {lst:normalisedprobs}
]{files/pr.prism}

Finally, the normalised probabilities are considered. These actions take
advantage of knowing the existing values of the other input and output to
normalise the probabilities of the remaining output. For example, line 8 shows
that \texttt{y} equal to 1 and \texttt{b} equal to 1 shows that \texttt{a} can
only be 0 since the action is synchronizing with \texttt{x0}. The probabilities
in \ref{tab:pr} must have the now impossible outcomes ignored so that the
normalised probabilities can be determined for use in the model.
% subsubsection output_normalised_probabilities (end)

% subsection nlbs_in_prism (end)
% section prism (end)
% chapter related_work (end)

\newpage
\end{document}


\section{Causal Graphs}

\subsection{An Overview}
\label{sec:graph-overview}
The previous section, \ref{sec:graph-importance}, motivated why the knowledge encoded in one's DAG is important.
This section gives a high level overview of what DAGs are and provides references for more thorough readings on the subject.
Causal diagrams and causal graphical models have been introduced by \citet{pearl_1995_causal} as a powerful tool for causal inference, especially in observational studies.
Perhaps one of the most important and useful features of causal
graphs when dealing with causal inference problems is the clear illustration
of the causal relationships between the variables.
While a formal introduction to the topic of DAGs is beyond the scope of this chapter,
here we focus specifically on illustrating the power of DAGs to represent
and encode complex causal relationships between variables in an intuitive and
clear manner.
Interested readers can refer to \citet{pearl_causality_2000} for a thorough introduction.

Consider the causal graph represented in Figure \ref{fig:simple-graph}.
Suppose we're interested in the effect of a treatment Z on Y.
What the graph in Figure \ref{fig:simple-graph} implies is that Z is
independent of Y(Z) given X.
For more information on how to semantically parse a causal graph as just done, see \citet[Ch 11.1]{pearl_2009_causality}.
Continuing with interpretation, in causal jargon, we say the mechanism by which the treatment Z was assigned is ignorable, once we control for covariates X.
In such situations, it is sufficient to control for X to obtain an unbiased estimate of the causal effect of Z on Y.

\begin{figure}[h!]
   \centering
   \includegraphics[height=0.25\textwidth]{simple-graph.pdf}
   \caption{Simple DAG of the causal relationships between X, Y and Z}
   \label{fig:simple-graph}
\end{figure}

The graph in Figure \ref{fig:simple-graph} can also be summarized by the following set of structural equations:
\[Z = f_Z(X, \epsilon_Z)  \]

\[Y(z) = f_Y(X, z, \epsilon_Y)  \]

Now consider the case where there exists another latent confounding variable, U, which also affects both the treatment Z, as well as the outcome, Y.
Figure \ref{fig:simple-graph-confounded} illustrates this assumption in DAG form, and the equations below are the structural equation equivalent:

\[Z = f_Z(X, U, \epsilon_Z)  \]

\[Y(z) = f_Y(X, U, z, \epsilon_Y)  \]

\begin{figure}[h!]
   \centering
   \includegraphics[height=0.25\textwidth]{simple-graph-confounded.pdf}
   \caption{Simple DAG of the causal relationships between X, Y, Z and confounder U}
   \label{fig:simple-graph-confounded}
\end{figure}

Figure \ref{fig:simple-graph-confounded} shows an indirect connection between Z and Y that goes through U.
Therefore, even if we condition on X, omitting U will yield biased results of the relationship between Z and Y, due to the variation in both Z and Y caused by U.
The structural equations also show that the ignorability assumption of Z does not hold if we only control for X, and we risk obtaining biased estimates of the causal effect of Z on Y if we fail to account for U, sometimes referred to as a common cause.
While the set of structural equations and the DAGs illustrate the same points in theory, the clarity of structural equations scales poorly with a problem's complexity.
As the number of covariates in one's causal system increases, a graphical representation of the relationship between those covariates quickly becomes more intuitive and easier to follow.
More importantly, graphically illustrating our assumptions eases the communication of these assumptions to the reader.
This communication is key to avoiding the misuse of our models in real-world applications.
For example, consider travel demand modelling applications.
Here, the communication advantage of DAGs is of particular relevance, as travel demand models typically involve many variables with complex interdependencies.

Another benefit of DAGs is that they come with testable implications.
Incorporating these tests in any causal analysis adds robustness and defensibility to one's study.
In section 5, we discuss these implications further, and we illustrate how to test for them in one's analysis.

Lastly, it is important to note that the graphical approach to causality focuses
primarily on issues of identification of causal effects, as opposed to issues of estimation and inference. That is, given a
DAG that encodes an analyst's knowledge and assumptions about the data generation process of the problem at hand, one can determine algorithmically whether a causal effect of interest can be identified.
As such, we emphasize that DAGs are great tools
for a modeller to encode their assumptions about a problem.
Later on, we will use more familiar statistical tools to actually infer/estimate our causal effects of interest.

\subsection{Prior Uses of Causal Graphs in Choice Modelling}
\label{sec:choice-graphs}

In our last subsection, we reviewed the basics of causal graphs: what are they and why are they useful?
We targeted that subsection at choice modellers who are unfamiliar with these tools.
However, readers should be aware of the history of causal graphs in choice modelling.
Indeed, choice modellers have used causal graphs (in limited fashion) for years.
Here are three examples to illustrate our point.
First, consider the case of Random Utility Maximization (RUM) models.

\begin{figure}
   \centering
   \includegraphics[width=0.25\textwidth]{rum-causal-graph}
   \caption{Archetypical RUM causal diagram}
   \label{fig:example-graph-rum}
\end{figure}

For decades, choice modellers have used stylized DAGs to depict RUM models.
In particular, these diagrams illustrate an assumed choice-making (i.e., causal) process.
As an example, see Figure \ref{fig:example-graph-rum}, reproduced from Figure 1 of \citet{ben_2002_integration}.
\citeauthor{ben_2002_integration} draw a two-part process.
First, they assume that explanatory variables cause unobserved utilities.
Then, they assume that the unobserved utilities cause the choice.

Note that although RUM diagrams adequately show the choice process, we still call them stylized.
Specifically, these diagrams lack detail about the relationships between explanatory variables.
When speaking collectively, we cannot tell if one explanatory variable causes another.
Unfortunately, as shown in Section \ref{sec:graph-importance}, such knowledge is crucial.
Without more detailed causal knowledge, our inferences may be inconsistent and arbitrarily bad.

\begin{figure}
   \centering
   \includegraphics[width=0.35\textwidth]{iclv-causal-graph}
   \caption{Archetypical ICLV causal diagram}
   \label{fig:example-graph-iclv}
\end{figure}

Besides RUM models, causal graphs appear in the literature on Integrated Choice and Latent Variable (ICLV) models.
An example of an ICLV model is in Figure \ref{fig:example-graph-iclv}, based on Figure 5 of \citet{ben_2002_integration}.
As with RUM models, ICLV causal graphs represent assumptions about the choice process.
Here concern typically centers around unobserved mediators.
These unobserved variables cause the outcome and are caused by one's observed covariates.
Omitting such mediators leads to models that misrepresent the assumed choice process.
As a result, researchers care deeply about ICLV models that avoid these behavioral misrepresentations.

\begin{figure}
   \centering
   \includegraphics[width=0.5\textwidth]{daysim-model-graph}
   \caption{Archetypical causal diagram for activity-based models}
   \label{fig:example-graph-abm}
\end{figure}

Finally, consider activity based travel demand models (ABM).
These models often come with a causal diagram that depicts the interrelations between the outcomes in the model.
E.g., household location choice, destination choice, travel mode choice, departure time choice, route choice etc.
For example, see Figure \ref{fig:example-graph-abm}, from \citet[Fig. 2]{bradley_2010_sacsim}.
The purpose of these graphs is to explain the structure of the entire system of outcome models.

In particular, ABM diagrams highlight two sets of researcher assumptions.
They detail the researcher's beliefs about which choices, i.e. outcomes, precede others.
For instance, work location choice preceding auto-ownership choice.
ABM diagrams also detail which downstream choices partially cause which preceding ones.
For example, considerations about one's travel mode choice influences one's daily activity choices.
This happens even though the activity choices precede the travel mode choices.

In conclusion, choice modellers have long used causal graphs.
However, two main features distinguish RUM, ICLV, and ABM graphs from the causal graphs of \citet{pearl_1995_causal}.
First, causal graphs in choice modelling traditionally ignore relations between the explanatory variables.
Commonly, choice modellers present these variables together, as a monolith.
Even worse, researchers may tacitly treat the variables as if they are jointly independent.
In the language used by a large group of causal inference scholars, econometric causal diagrams ignore the treatment assignment mechanism.

Secondly, causal graphs in choice modelling papers are purely didactic.
They convey how choice modellers perceive the world and the choice generation process.
However, they are seldom treated as a model themselves.
In particular, choice modellers rarely test the predictions of causal graphs.
In doing so, we underuse causal implications such as statistical independence.
Worse, we possibly violate these implications with great frequency.
Such actions ignore the efforts from causal inference in computer science.
There, researchers stress using their data to test the implications of their causal graphs.

These two issues are opportunities for choice modelling to gain from the insights and methods of causal inference scholars.
The rest of the chapter will focus on the following three topics.
How we can construct causal diagrams that pay attention to explanatory and outcome variables?
How we can test a given causal graph against one's data?
How can we deal with ``real-world'' graphs featuring unobserved confounding?


%!TEX root = da2020-05.tex

\Chapter{5}{\tCONGEST{}~Model: Bandwidth~Limitations}

\noindent
In the previous chapter, we learned about the $\LOCAL$ model. We saw that with the help of unique identifiers, it is possible to gather the full information on a connected input graph in $O(\diam(G))$ rounds. To achieve this, we heavily abused the fact that we can send arbitrarily large messages. In this chapter we will see what can be done if we are only allowed to send small messages. With this restriction, we arrive at a model that is commonly known as the ``$\CONGEST$ model''.

\section{Definitions}\label{sec:congest}

Let $A$ be a distributed algorithm that solves a problem $\Pi$ on a graph family $\calF$ in the $\LOCAL$ model. Assume that $\Msg_A$ is a countable set; without loss of generality, we can then assume that
\[
    \Msg_A = \NN,
\]
that is, the messages are encoded as natural numbers. Now we say that $A$ solves problem $\Pi$ on graph family $\calF$ in the $\CONGEST$ model if the following holds for some constant $C$: for any graph $G = (V,E) \in \calF$, algorithm $A$ only sends messages from the set $\{0, 1, \dotsc, |V|^C\}$.

Put otherwise, we have the following \emph{bandwidth restriction}: in each communication round, over each edge, we only send $O(\log n)$-bit messages, where $n$ is the total number of nodes.

\section{Examples}

Assume that we have an algorithm $A$ that is designed for the $\LOCAL$ model. Moreover, assume that during the execution of $A$ on a graph $G = (V,E)$, in each communication round, we only need to send the following pieces of information over each edge:
\begin{itemize}[noitemsep]
    \item $O(1)$ node identifiers,
    \item $O(1)$ edges, encoded as a pair of node identifiers,
    \item $O(1)$ counters that take values from $0$ to $\diam(G)$,
    \item $O(1)$ counters that take values from $0$ to $|V|$,
    \item $O(1)$ counters that take values from $0$ to $|E|$.
\end{itemize}
Now it is easy to see that we can encode all of this as a binary string with $O(\log n)$ bits. Hence $A$ is not just an algorithm for the $\LOCAL$ model, but it is also an algorithm for the $\CONGEST$ model.

Many algorithms that we have encountered in this book so far are of the above form, and hence they are also $\CONGEST$ algorithms (see Exercise~\ref{ex:congest-prior}). However, there is a notable exception: the algorithm for gathering the entire network from Section~\longref{4.2}{sec:gather}. In this algorithm, we need to send messages of size up to $\Theta(n^2)$ bits:
\begin{itemize}
    \item To encode the set of nodes, we may need up to $\Theta(n \log n)$ bits (a list of $n$ identifiers, each of which is $\Theta(\log n)$ bits long).
    \item To encode the set of edges, we may need up to $\Theta(n^2)$ bits (the adjacency matrix).
\end{itemize}

While algorithms with a running time of $O(\diam(G))$ or $O(n)$ are trivial in the $\LOCAL$ model, this is no longer the case in the $\CONGEST$ model. Indeed, there are graph problems that \emph{cannot} be solved in time $O(n)$ in the $\CONGEST$ model (see Exercise~\ref{ex:congest-gather-lb}).

In this chapter, we will learn techniques that can be used to design efficient algorithms in the $\CONGEST$ model. We will use the all-pairs shortest path problem as the running example.

\section{All-Pairs Shortest Path Problem}

Throughout this chapter, we will assume that the input graph $G = (V,E)$ is connected, and as usual, we have $n = |V|$. In the \emph{all-pairs shortest path} problem (APSP in brief), the goal is to find the distances between all pairs of nodes. More precisely, the local output of node $v \in V$ is
\[
    f(v) = \bigl\{ (u, d) : u \in V,\ d = \dist_G(v, u) \bigr\}.
\]
That is, $v$ has to know the identities of all other nodes, as well as the shortest-path distance between itself and all other nodes.

Note that to represent the local output of a single node we need $\Theta(n \log n)$ bits, and just to transmit this information over a single edge we would need $\Theta(n)$ communication rounds. Indeed, we can prove that any algorithm that solves the APSP problem in the $\CONGEST$ model takes $\Omega(n)$ rounds\mydash see Exercise~\ref{ex:apsp-lb}.

In this chapter, we will present an optimal distributed algorithm for the APSP problem: it solves the problem in $O(n)$ rounds in the $\CONGEST$ model.

\newcommand{\BFS}{\algo{BFS}}
\newcommand{\Wave}{\algo{Wave}}

\section{Single-Source Shortest Paths}\label{sec:wave}

As a warm-up, we will start with a much simpler problem. Assume that we have elected a leader $s \in V$, that is, there is precisely one node $s$ with input $1$ and all other nodes have input $0$. We will design an algorithm such that each node $v \in V$ outputs
\[
    f(v) = \dist_G(s,v),
\]
i.e., its shortest-path distance to leader $s$.

The algorithm proceeds as follows. In the first round, the leader will send message \msg{wave} to all neighbors, switch to state $0$, and stop. In round $i$, each node $v$ proceeds as follows: if $v$ has not stopped, and if it receives message \msg{wave} from some ports, it will send message \msg{wave} to all other ports, switch to state $i$, and stop; otherwise it does nothing. See Figure~\ref{fig:wave}.

\begin{figure}
    \centering
    \includegraphics[page=\PWave]{figs.pdf}
    \caption{(a)~Graph $G$ and leader~$s$. (b)~Execution of algorithm $\Wave$ on graph $G$. The arrows denote \msg{wave} messages, and the dotted lines indicate the communication round during which these messages were sent.}\label{fig:wave}
\end{figure}

The analysis of the algorithm is simple. By induction, all nodes at distance $i$ from $s$ will receive message \msg{wave} from at least one port in round $i$, and they will hence output the correct value~$i$. The running time of the algorithm is $O(\diam(G))$ rounds in the $\CONGEST$ model.

\section{Breadth-First Search Tree}\label{sec:bfs}

Algorithm $\Wave$ finds the shortest-path distances from a single source $s$. Now we will do something slightly more demanding: calculate not just the distances but also the shortest paths.

More precisely, our goal is to construct a \emph{breadth-first search tree} (BFS tree) $T$ rooted at $s$. This is a spanning subgraph $T = (V,E')$ of $G$ such that $T$ is a tree, and for each node $v \in V$, the shortest path from $s$ to $v$ in tree $T$ is also a shortest path from $s$ to $v$ in graph $G$. We will also label each node $v \in V$ with a \emph{distance label} $d(v)$, so that for each node $v \in V$ we have
\[
    d(v) = \dist_T(s,v) = \dist_G(s,v).
\]
See Figure~\ref{fig:bfs} for an illustration. We will interpret $T$ as a directed graph, so that each edge is of form $(u,v)$, where $d(u) > d(v)$, that is, the edges point towards the root~$s$.

\begin{figure}
    \centering
    \includegraphics[page=\PBFS]{figs.pdf}
    \caption{(a)~Graph $G$ and leader~$s$. (b)~BFS tree $T$ (arrows) and distance labels $d(v)$ (numbers).}\label{fig:bfs}
\end{figure}

There is a simple centralized algorithm that constructs the BFS tree and distance labels: breadth-first search. We start with an empty tree and unlabeled nodes. First we label the leader $s$ with $d(s) = 0$. Then in step $i = 0, 1, \dotsc$, we visit each node $u$ with distance label $d(u) = i$, and check each neighbor $v$ of $u$. If we have not labeled $v$ yet, we will label it with $d(v) = i+1$, and add the edge $(v,u)$ to the BFS tree. This way all nodes that are at distance $i$ from $s$ in $G$ will be labeled with the distance label $i$, and they will also be at distance $i$ from $s$ in~$T$.

We can implement the same idea as a distributed algorithm in the CONGEST model. We will call this algorithm $\BFS$. In the algorithm, each node $v$ maintains the following variables:
\begin{itemize}
    \item $d(v)$: distance to the root.
    \item $p(v)$: pointer to the parent of node $v$ in tree $T$ (port number).
    \item $C(v)$: the set of children of node $v$ in tree $T$ (port numbers).
    \item $a(v)$: acknowledgment\mydash set to $1$ when the subtree rooted at $v$ has been constructed.
\end{itemize}
Here $a(v) = 1$ denotes a stopping state. When the algorithm stops, variables $d(v)$ will be distance labels, tree $T$ is encoded in variables $p(v)$ and $C(v)$, and all nodes will have $a(v) = 1$.

Initially, we set $d(v) \gets \bot$, $p(v) \gets \bot$, $C(v) \gets \bot$, and $a(v) \gets 0$ for each node $v$, except for the root which has $d(s) = 0$. We will grow tree $T$ from $s$ by iterating the following steps:
\begin{itemize}
    \item Each node $v$ with $d(v) \ne \bot$ and $C(v) = \bot$ will send a \emph{proposal} with value $d(v)$ to all neighbors.
    \item If a node $u$ with $d(u) = \bot$ receives some proposals with value $j$, it will \emph{accept} one of them and \emph{reject} all other proposals. It will set $p(u)$ to point to the node whose proposal it accepted, and it will set $d(u) \gets j+1$.
    \item Each node $v$ that sent some proposals will set $C(v)$ to be the set of neighbors that accepted proposals.
\end{itemize}
This way $T$ will grow towards the leaf nodes. Once we reach a leaf node, we will send acknowledgments back towards the root:
\begin{itemize}
    \item Each node $v$ with $a(v) = 1$ and $p(v) \ne \bot$ will send an \emph{acknowledgment} to port $p(v)$.
    \item Each node $v$ with $a(v) = 0$ and $C(v) \ne \bot$ will set $a(v) \gets 1$ when it has received acknowledgments from each port of $C(v)$. In particular, if a node has $C(v) = \emptyset$, it can set $a(v) \gets 1$ without waiting for any acknowledgments.
\end{itemize}
It is straightforward to verify that the algorithm works correctly and constructs a BFS tree in $O(\diam(G))$ rounds in the $\CONGEST$ model.

Note that the acknowledgments would not be strictly necessary in order to construct the tree. However, they will be very helpful in the next section when we use algorithm $\BFS$ as a subroutine.

\section{Leader Election}\label{sec:leader}

Algorithm $\BFS$ constructs a BFS tree rooted at a single leader, assuming that we have already elected a leader. Now we will show how to elect a leader. Surprisingly, we can use algorithm $\BFS$ to do it!

We will design an algorithm $\algo{Leader}$ that finds the node with the smallest identifier; this node will be the leader. The basic idea is very simple:
\begin{enumerate}
    \item We modify algorithm $\BFS$ so that we can run multiple copies of it in parallel, with different root nodes. We augment the messages with the identity of the root node, and each node keeps track of the variables $d$, $p$, $C$, and $a$ separately for each possible root.
    \item Then we pretend that all nodes are leaders and start running $\BFS$. In essence, we will run $n$ copies of $\BFS$ in parallel, and hence we will construct $n$ BFS trees, one rooted at each node. We will denote by $\BFS_v$ the $\BFS$ process rooted at node $v \in V$, and we will write $T_v$ for the output of this process.
\end{enumerate}
However, there are two problems: First, it is not yet obvious how all this would help with leader election. Second, we cannot implement this idea directly in the $\CONGEST$ model\mydash nodes would need to send up to $n$ distinct messages per communication round, one per each $\BFS$ process, and there is not enough bandwidth for all those messages.

\begin{figure}
    \centering
    \includegraphics[page=\PLeader]{figs.pdf}
    \caption{Leader election. Each node $v$ will launch a process $\BFS_v$ that attempts to construct a BFS tree $T_v$ rooted at $v$. Other nodes will happily follow $\BFS_v$ if $v$ is the smallest leader they have seen so far; otherwise they will start to ignore messages related to $\BFS_v$. Eventually, precisely one of the processes will complete successfully, while all other process will get stuck at some point. In this example, node $1$ will be the leader, as it has the smallest identifier. Process $\BFS_2$ will never succeed, as node $1$ (as well as all other nodes that are aware of node $1$) will ignore all messages related to $\BFS_2$. Node $1$ is the only root that will receive acknowledgments from every child.}\label{fig:leader}
\end{figure}

Fortunately, we can solve both of these issues very easily; see Figure~\ref{fig:leader}:
\begin{enumerate}[resume]
    \item Each node will only send messages related to the tree that has the \emph{smallest identifier as the root}. More precisely, for each node $v$, let $U(v) \subseteq V$ denote the set of nodes $u$ such that $v$ has received messages related to process $\BFS_u$, and let $\ell(v) = \min U(v)$ be the smallest of these nodes. Then $v$ will ignore messages related to process $\BFS_u$ for all $u \ne \ell(v)$, and it will only send messages related to process $\BFS_{\ell(v)}$.
\end{enumerate}
We make the following observations:
\begin{itemize}
    \item In each round, each node will only send messages related to at most one $\BFS$ process. Hence we have solved the second problem\mydash this algorithm can be implemented in the $\CONGEST$ model.
    \item Let $s = \min V$ be the node with the smallest identifier. When messages related to $\BFS_s$ reach a node $v$, it will set $\ell(v) = s$ and never change it again. Hence all nodes will follow process $\BFS_s$ from start to end, and thanks to the acknowledgments, node $s$ will eventually know that we have successfully constructed a BFS tree $T_s$ rooted at it.
    \item Let $u \ne \min V$ be any other node. Now there is at least one node, $s$, that will ignore all messages related to process $\BFS_u$. Hence $\BFS_u$ will never finish; node $u$ will never receive the acknowledgments related to tree $T_u$ from all neighbors.
\end{itemize}
That is, we now have an algorithm with the following properties: after $O(\diam(G))$ rounds, there is precisely one node $s$ that knows that it is the unique node $s = \min V$. To finish the leader election process, node $s$ will inform all other nodes that leader election is over; node $s$ will output $1$ and all other nodes will output $0$ and stop.

\section{All-Pairs Shortest Paths}\label{sec:apsp}

Now we are ready to design algorithm $\algo{APSP}$ that solves the all-pairs shortest path problem (APSP) in time $O(n)$.

We already know how to find the shortest-path distances from a single source; this is efficiently solved with algorithm $\Wave$. Just like we did with the $\BFS$ algorithm, we can also augment $\Wave$ with the root identifier and hence have a separate process $\Wave_v$ for each possible root $v \in V$. If we could somehow run all these processes in parallel, then each node would receive a wave from every other node, and hence each node would learn the distance to every other node, which is precisely what we need to do in the APSP problem. However, it is not obvious how to achieve a good performance in the $\CONGEST$ model:
\begin{itemize}
    \item If we try to run all $\Wave_v$ processes simultaneously in parallel, we may need to send messages related to several waves simultaneously over a single edge, and there is not enough bandwidth to do that.
    \item If we try to run all $\Wave_v$ processes sequentially, it will take a lot of time: the running time would be $O(n \diam(G))$ instead of $O(n)$.
\end{itemize}
The solution is to \emph{pipeline} the $\Wave_v$ processes so that we can have many of them running simultaneously in parallel, without congestion. In essence, we want to have multiple wavefronts active simultaneously so that they never collide with each other.

To achieve this, we start with the leader election and the construction of a BFS tree rooted at the leader; let $s$ be the leader, and let $T_s$ be the BFS tree. Then we do a \emph{depth-first traversal} of $T_s$. This is a walk $w_s$ in $T_s$ that starts at $s$, ends at $s$, and traverses each edge precisely twice; see Figure~\ref{fig:dfs}.

\begin{figure}
    \centering
    \includegraphics[page=\PDFS]{figs.pdf}
    \caption{(a)~BFS tree $T_s$ rooted at $s$. (b)~A depth-first traversal $w_s$ of $T_s$.}\label{fig:dfs}
\end{figure}

More concretely, we move a \emph{token} along walk $w_s$. We move the token \emph{slowly}: we always spend 2 communication rounds before we move the token to an adjacent node. Whenever the token reaches a new node $v$ that we have not encountered previously during the walk, we launch process $\Wave_v$. This is sufficient to avoid all congestion!

\begin{figure}
    \centering
    \includegraphics[page=\PPipeline]{figs.pdf}
    \caption{Algorithm $\algo{APSP}$: the token walks along the BFS tree at speed $0.5$ (thick arrows), while each $\Wave_v$ moves along the original graph at speed $1$ (dashed lines). The waves are strictly nested: if $\Wave_v$ was triggered after $\Wave_u$, it will never catch up with $\Wave_u$.}\label{fig:pipeline}
\end{figure}

The key observation here is that the token moves slower than the waves. The waves move at speed $1$ edge per round (along the edges of $G$), while the token moves at speed $0.5$ edges per round (along the edges of $T_s$, which is a subgraph of $G$). This guarantees that two waves never collide. To see this, consider two waves $\Wave_u$ and $\Wave_v$, so that $\Wave_u$ was launched before $\Wave_v$. Let $d = \dist_G(u,v)$. Then it will take at least $2d$ rounds to move the token from $u$ to $v$, but only $d$ rounds for $\Wave_u$ to reach node $v$. Hence $\Wave_u$ was already past $v$ before we triggered $\Wave_v$, and $\Wave_v$ will never catch up with $\Wave_u$ as both of them travel at the same speed. See Figure~\ref{fig:pipeline} for an illustration.

Hence we have an algorithm $\algo{APSP}$ that is able to trigger all $\Wave_v$ processes in $O(n)$ time, without collisions, and each of them completes $O(\diam(G))$ rounds after it was launched. Overall, it takes $O(n)$ rounds for all nodes to learn distances to all other nodes. Finally, the leader can inform everyone else when it is safe to stop and announce the local outputs (e.g., with the help of another wave).

\section{Quiz}

Give an example of a graph problem that \emph{can} be solved in $O(1)$ rounds in the $\LOCAL$ model and \emph{cannot} be solved in $O(n)$ rounds in the $\CONGEST$ model. The input has to be an unlabeled graph; that is, the nodes will not have any inputs (beyond their own degree and their unique identifier). The output can be anything; you are free to construct an artificial graph problem. It is enough to give a brief definition of the graph problem; no further explanations are needed.

\section{Exercises}

\begin{ex}[prior algorithms]\label{ex:congest-prior}
    In Chapters \chapterref{3} and \chapterref{4} we have seen examples of algorithms that were designed for the $\PN$ and $\LOCAL$ models. Many of these algorithms use only small messages\mydash they can be used directly in the $\CONGEST$ model. Give at least four concrete examples of such algorithms, and prove that they indeed use only small messages.
\end{ex}

\begin{ex}[edge counting]
    The \emph{edge counting} problem is defined as follows: each node has to output the value $|E|$, i.e., it has to indicate how many edges there are in the graph.

    Assume that the input graph is connected. Design an algorithm that solves the edge counting problem in the $\CONGEST$ model in time $O(\diam(G))$.
\end{ex}

\begin{ex}[detecting bipartite graphs]
    Assume that the input graph is connected. Design an algorithm that solves the following problem in the $\CONGEST$ model in time $O(\diam(G))$:
    \begin{itemize}[noitemsep]
        \item If the input graph is bipartite, all nodes output $1$.
        \item Otherwise all nodes output $0$.
    \end{itemize}
\end{ex}

\begin{ex}[detecting complete graphs]
    We say that a graph $G = (V,E)$ is \emph{complete} if for all nodes $u, v \in V$, $u \ne v$, there is an edge $\{u,v\} \in E$.

    Assume that the input graph is connected. Design an algorithm that solves the following problem in the $\CONGEST$ model in time $O(1)$:
    \begin{itemize}[noitemsep]
        \item If the input graph is a complete graph, all nodes output $1$.
        \item Otherwise all nodes output $0$.
    \end{itemize}
\end{ex}

\begin{ex}[gathering]
    Assume that the input graph is connected. In Section~\longref{4.2}{sec:gather} we saw how to gather full information on the input graph in time $O(\diam(G))$ in the $\LOCAL$ model. Design an algorithm that solves the problem in time $O(|E|)$ in the $\CONGEST$ model.
\end{ex}

\begin{exs}[gathering lower bounds]\label{ex:congest-gather-lb}
    Assume that the input graph is connected. Prove that there is no algorithm that gathers full information on the input graph in time $O(|V|)$ in the $\CONGEST$ model.

    \hint{To reach a contradiction, assume that $A$ is an algorithm that solves the problem. For each $n$, let $\calF(n)$ consists of all graphs with the following properties: there are $n$ nodes with unique identifiers $1,2,\dotsc,n$, the graph is connected, and the degree of node $1$ is $1$. Then compare the following two quantities as a function of~$n$:
    \begin{enumerate}
        \item $f(n) = {}$how many different graphs there are in family $\calF(n)$.
        \item $g(n) = {}$how many different message sequences node number $1$ may receive during the execution of algorithm~$A$ if we run it on any graph $G \in \calF(n)$.
    \end{enumerate}
    Argue that for a sufficiently large $n$, we will have $f(n) > g(n)$. Then there are at least two different graphs $G_1, G_2 \in \calF(n)$ such that node $1$ receives the same information when we run $A$ on either of these graphs.}
\end{exs}

\begin{exs}[APSP lower bounds]\label{ex:apsp-lb}
    Assume that the input graph is connected. Prove that there is no algorithm that solves the APSP problem in time $o(|V|)$ in the $\CONGEST$ model.
\end{exs}

\section{Bibliographic Notes}

The name $\CONGEST$ is from Peleg's~\cite{peleg00distributed} book. Algorithm $\algo{APSP}$ is due to Holzer and Wattenhofer \cite{holzer12apsp}\mydash surprisingly, it was published only as recently as in 2012.


\input{preamble}

\begin{document}
    \section{Subjects}
    \begin{itemize}
        \item Backpropagation
        \item Deep Nets
        \item PCA
        \item Autoencoder
    \end{itemize}

    \section{Notes}

    \subsection{Neurons and background}
    Neurons consist of a dendritic tree (the input, which branches a lot), the
    axon (the output, which branches a little bit) and the cell body with
    nucleus.

    Neurons work by having axons connected to dendritic trees, they give a
    binary ($0$ or $1$) output, after an axon has sent a $1$, it has to
    recharge.

    In order to model this, we've got binary input $x_1,\dots, x_n$ which is
    sent to the nucleus that sums the inputs together and outputs $1$ if the
    input exceeds some threshold.

    \begin{tikzpicture}
        \node[textbox] (x11) {$x_1$};
        \node[simplebox, right=of x11] (n1) {$x$};
        \node[textbox, above=of n1, halfvdistance] (w1) {$w_1$};
        \path[edgepath]
            (x11) edge node {} (n1)
            (w1) edge node {} (n1);

        \node[textbox, below=of x11] (x1m) {$x_m$};
        \node[simplebox, right=of x1m] (nm) {$x$};
        \node[textbox, between=nm and n1] (wm) {$w_m$};

        \node[simplebox, right=of n1, between=nm and n1] (sigma1) {$\Sigma$};
        \node[simplebox, right=of sigma1] (threshold1) {threshold};
        \node[textbox, right=of threshold1] (z) {z};

        \draw[edgepath]
            (x1m) edge node {} (nm)
            (wm) edge node {} (nm)
            (sigma1) edge node {} (threshold1)
            (threshold1) edge node {} (z);
        \draw[edgepath]
            (n1) -| ++(0.7cm,-\nodedv) -- (sigma1);
        \draw[edgepath]
            (nm) -| ++(0.7cm,\nodedv) -- (sigma1);
    \end{tikzpicture}

    This models the following properties:
    \begin{itemize}
        \item All or non
        \item Cumulative influence
        \item Synaptic weight
    \end{itemize}
    But there are more properties that we might like to model, like:
    \begin{itemize}
        \item Refractory period, recovery time of each neuron.
        \item Axonal bifurcation, each pulse will either go down one branch of
        the axon or the other.
        \item Time patterns, we don't know if the timing of when impulses hit
        neurons matters.
    \end{itemize}
    So we actually don't know if what we are modelling is the essence of how
    neurons work or not. But we will start with the simple model.

    If we look at what a neural net actually is, then it's a vector of input
    that goes through a ``box'' which uses some weights and threshold and then
    outputs some vector $z$. Formally: $z = f(x,w,t)$. So the neural network is
    just the function $f$, when we train the neural net, all we need to do is
    change the weights and threshold. We can also think of the neural net as a
    ``function approximator''.

    So, how do we measure the performance of the neural network? If we say the
    desired result of the neural net $d$ is the function $g$ on the input
    $x$, $d=g(x)$, then it would be natural to define the performance as:
    $P=\|d-z\|$, this turns out to be mathematically inconvenient so instead we
    will use the performance indicator: $P=\|d-z\|^2$.

    What we want to do, is of course to maximize the performance. We can do
    this using gradient descent, for example for the weights $w_1$ and $w_2$:
    \begin{equation*}
        \Delta w= \eta \left(\frac{\partial P}{\partial w_1}i+\frac{\partial
        P}{\partial w_2}j\right)
    \end{equation*}
    Unfortunately, the function is not linear, and thus gradient descent
    doesn't really work. This was an issue for a long time untill Paul Werbos
    figure it out.

    First of all, those thresholds are annoying as they are just extra baggage,
    so we would like to reduce $z$ to be a function of just the inputs and
    weights:
    \begin{equation*}
        z=f(x,w)
    \end{equation*}
    So what he proposed instead, was adding the bias input to each neuron $w_0$
    which is always set to $1$.

    \begin{tikzpicture}
    \node[textbox, right=of n1, abovebetween] (w0) {$w_0$};
    \node[textbox, right=of w0, yshift=0.5cm, xshift=-0.5cm] (b1) {$1$};

    \node[textbox] (x11) {$x_1$};
    \node[simplebox, right=of x11] (n1) {$x$};
    \node[textbox, above=of n1, halfvdistance] (w1) {$w_1$};
    \path[edgepath]
    (x11) edge node {} (n1)
    (w1) edge node {} (n1);

    \node[textbox, below=of x11] (x1m) {$x_m$};
    \node[simplebox, right=of x1m] (nm) {$x$};
    \node[textbox, between=nm and n1] (wm) {$w_m$};

    \node[simplebox, right=of n1, between=nm and n1] (sigma1)
    {$\Sigma$};
    \node[simplebox, right=of sigma1] (threshold1) {threshold};
    \node[textbox, right=of threshold1] (z) {z};

    \draw[edgepath]
    (x1m) edge node {} (nm)
    (wm) edge node {} (nm)
    (sigma1) edge node {} (threshold1)
    (threshold1) edge node {} (z)
    (b1) edge node {} (w0);
    \draw[edgepath]
        (n1) -| ++(\nodedh,-\nodedv) -- (sigma1);
    \draw[edgepath]
        (nm) -| ++(\nodedh,\nodedv) -- (sigma1);
    \draw[edgepath]
        (w0) -| ++(-\nodedh,-\nodedv) -- (sigma1);
    \end{tikzpicture}
    And then we let $w_0=\text{threshold}$, then the threshold is effectively
    $1$.\todo{revisit}

    Step two, is to smooth the threshold function, which we could do by
    applying the sigmoid function $\sigma(\alpha)=\frac{1}{1+e^{-\alpha}}$.
    Then it will be $1$ if $\alpha$ is very big, and $0$ if $\alpha$ is very
    small.
    \sigmoidPlot
    The generalized term ``activation function'' $\phi$ is in this case
    $\sigma$, our
    function is linear and we can take the partial derivatives to the
    function. Now, suppose we have a very simple neural network:

    \begin{tikzpicture}
    \node[textbox] (x11) {$x_1$};
    \node[simplebox, right=of x11] (n1) {$x$};
    \node[textbox, above=of n1, halfvdistance] (w1) {$w_1$};
    \node[simplebox, right=of n1] (sigmoid1) {sigmoid};

    \path[edgepath]
        (x11) edge node {} (n1)
        (w1) edge node {} (n1)
        (n1) edge node[above] {$P_1$} (sigmoid1);

    \node[simplebox, right=of sigmoid1] (n2) {$x$};
    \node[textbox, above=of n2, halfvdistance] (w2) {$w_2$};
    \node[simplebox, right=of n2] (sigmoid2) {sigmoid};

    \path[edgepath]
        (sigmoid1) edge node[above] {y} (n2)
        (w2) edge node {} (n2)
        (n2) edge node[above] {$P_2$} (sigmoid2);

    \node[textbox, right=of sigmoid2] (z) {z};

    \path[edgepath]
        (sigmoid2) edge node {} (z);
    \end{tikzpicture}

    Now we want to re-write the partial derivative using the chain rule:
    \begin{equation*}
        \frac{\partial P}{\partial w_2} = \frac{\partial P}{\partial z}
        \frac{\partial z}{\partial w_2} = \frac{\partial P}{\partial z}
        \frac{\partial z}{\partial p_2} \frac{\partial p_2}{\partial w_2}
    \end{equation*}
    We can do the same thing for $w_2$:
    \begin{equation*}
        \frac{\partial P}{\partial w_1} = \frac{\partial P}{\partial z}
        \frac{\partial z}{\partial p_2} \frac{\partial p_2}{\partial y}
        \frac{\partial y}{\partial p_1} \frac{\partial p_1}{\partial w_1}
    \end{equation*}
    We can rewrite these two products as:
    \begin{align*}
        \frac{\partial P}{\partial w_2}&= \frac{\partial p_2}{\partial w_2}
        \frac{\partial z}{\partial p_2} \frac{\partial P}{\partial z}\\
        \frac{\partial P}{\partial w_1}&=\frac{\partial p_1}{\partial w_1}
        \frac{\partial y}{\partial p_1} \frac{\partial p_2}{\partial y}
        \frac{\partial z}{\partial p_2} \frac{\partial P}{\partial z}
    \end{align*}
    Now we can compute the derivatives, in this example, the resulting
    performance $P$ is:
    \begin{equation*}
    P=\frac{1}{2}(d-z)^2
    \end{equation*}
    And thus we can compute
    \begin{equation*}
        \frac{\partial P}{\partial w_2}= \frac{\partial p_2}{\partial w_2}
        \frac{\partial z}{\partial p_2} (d-z)
    \end{equation*}
    $p_2$ is simply $p_2 w_2$ so we get:
    \begin{equation*}
        \frac{\partial P}{\partial w_2}= y
        \frac{\partial z}{\partial p_2} (d-z)
    \end{equation*}
    Finally the derivative of the sigmoid function is simple:
    \begin{equation*}
    \frac{\partial P}{\partial w_2}= y
    (1-\sigma(\alpha))\sigma(\alpha) (d-z)
    \end{equation*}
    In this case the output of the $\sigma$ is $z$ so:
    \begin{equation*}
    \frac{\partial P}{\partial w_2}= y
    (1-z)z (d-z)
    \end{equation*}
    Now if we look at the derivative for $\frac{\partial P}{\partial w_1}$, it
    turns out that the last two elements we needed to compute, was the same as
    the last two elements in the computation of $\frac{\partial P}{\partial
    w_2}$! So if we make a neural network, where each ``column of neurons'' or
    layer is densely connected, i.e. each output from the previous layer
    connects to each input from the next layer, then even though the amount of
    connections increases exponentially, the computation of the derivatives do
    not!

    The thing to note here, is that the output of layer $i$ has to go through
    layer $i+1$ in order to affect the performance indicator. So the derivative
    for layer $i$ can re-use computation from the derivative of $i+1$. So the
    amount of work we are gonna have to do will be:
    \begin{itemize}
        \item Linear in depth
        \item With respect to width it, it will be proportional to the number
        of connections and thus depends on $w^2$
    \end{itemize}

    This is the foundation for the backpropagation algorithm, and why neural
    networks can efficiently learn.

    \subsection{Notation}
    \begin{itemize}
        \item $n_l$ is the number of layers.
        \item $s_l$ is the number of nodes in layer $l$ (not counting the bias
        unit)
        \item $L_l$ is layer $l$.
        \item $L_0$ is the input layer, $L_{n_l}$ is the output layer
        \item $W_{ij}^{(l)}$ is the weight associated with the connection
        between unit $j$ in layer $l$ and unit $i$ in layer $l+1$
        \item $b_i^{(l)}$ is the bias associated with unit $i$ in layer $l+1$
        \item $a_i^{(l)}$ is the activation (or the output value) of unit $i$
        in layer $l$. For $l=1$ we use $a_i^{(1)}=x_i$
        \item $z_i^{(l)}=\sum_{j=1}^{n}W_{ij}^{(l)}x_j+b_i^{(l)}$ is a
        convenience notation such that $a_i^{(l)}=\phi(z_i^{(l)})$
    \end{itemize}

    Let's look at this example network:

    \begin{tikzpicture}
    \node[inputnode] (x1) {$x_1$};
    \node[inputnode, below=of x1] (x2) {$x_2$};
    \node[inputnode, below=of x2] (x3) {$x_3$};
    \node[inputnode, below=of x3] (b1) {$+1$};

    \node[hiddennode, right=of x1] (h1) {};
    \node[hiddennode, below=of h1] (h2) {};
    \node[hiddennode, below=of h2] (h3) {};
    \node[hiddennode, below=of h3] (b2) {$+1$};

    \node[outputnode, right=of h1, between=h2 and h3] (o1) {};
    \node[right=of o1] (o) {};

    \path[edgepath]
        (x1) edge node {} (h1)
        (x1) edge node {} (h2)
        (x1) edge node {} (h3)
        (x2) edge node {} (h1)
        (x2) edge node {} (h2)
        (x2) edge node {} (h3)
        (x3) edge node {} (h1)
        (x3) edge node {} (h2)
        (x3) edge node {} (h3)
        (b1) edge node {} (h1)
        (b1) edge node {} (h2)
        (b1) edge node {} (h3)
        (h1) edge node {$a_1^{(2)}$} (o1)
        (h2) edge node {$a_2^{(2)}$} (o1)
        (h3) edge node {$a_3^{(2)}$} (o1)
        (b2) edge node {} (o1)
        (o1) edge node[below] {$h_{W,b}(x)$} (o);
    \end{tikzpicture}

    This neural network represents the following computation:
    \begin{align*}
        a_1^{(2)} &=
        \phi\left(W_{11}^{(1)}x_1+W_{12}^{(1)}x_2+W_{13}^{(1)}x_3+b_1^{(1)}\right)\\
        a_2^{(2)} &=
        \phi\left(W_{21}^{(1)}x_1+W_{22}^{(1)}x_2+W_{23}^{(1)}x_3+b_2^{(1)}\right)\\
        a_3^{(2)} &=
        \phi\left(W_{31}^{(1)}x_1+W_{32}^{(1)}x_2+W_{33}^{(1)}x_3+b_3^{(1)}\right)\\
        h_{W,b}(x) &=
        a_1^{(3)}=\phi\left(W_{11}^{(2)}a_1^{(2)}+W_{12}^{(2)}a_2^{(2)}+W_{13}^{(2)}a_3^{(2)}\right)
    \end{align*}

    If we extend the activation function, to work on vectors such that
    $f([z_1,z_2,z_3])=\left[f(z_1), f(z_2), f(z_3)\right]$ then we can write
    the previous equation in a more compact fashion:
    \begin{align*}
        z^{(2)}&=W^{(1)}x+b^{(1)}\\
        a^{(2)}&=\phi\left(z^{(2)}\right)\\
        z^{(3)}&=W^{(2)}a^{(2)}+b^{(2)}\\
        h_{W,b}(x)&=a^{(3)}=\phi\left(z^{(3)}\right)
    \end{align*}
    Which can be generalized to:
    \begin{align*}
        z^{(l+1)}&=W^{(l)}a^{(l)}+b^{(l)}\\
        a^{(l+1)}&=\phi\left(z^{(l+1)}\right)
    \end{align*}

    The most common neural networks, are $n_l$-layered networks where $L_1$ is
    the input, $L_{n_l}$ the output and $L_i$ is densely connected to
    $L_{i+1}$. In order to compute the output, we would simply have to compute
    the activation of $L_2$, $L_3$ etc. up to layer $L_{n_l}$, this is an
    example of a \textbf{feedforward} neural network, as there are no loops or
    cycles.

    \subsection{Backpropagation}
    Suppose we have a training set $D = \{(x_1,y_1),\dots,(x_m,y_m)\}$, we can
    then train our neural network with batch gradient descent, with the cost
    function for single sample as:
    \begin{equation*}
        J(W,b;x,y)=\frac{1}{2}\|h_{W,b}(x)-y\|^2
    \end{equation*}
    This is simply a (one-half) squared-error cost function. The overall cost
    function is then defined to be:
    \begin{equation*}
        J(W,b) =
        \frac{1}{|D|}\sum_{i=1}^{|D|}\left(\frac{1}{2}\|h_{W,b}(x_i)-y_i\|^2\right)
    \end{equation*}
    If we add a weight decay term for regularization, then it becomes:
    \begin{equation*}
        J(W,b)=\frac{1}{|D|}\sum_{i=1}^{|D|}\left(\frac{1}{2}\|h_{W,b}(x_i)-y_i\|^2\right)
        + \frac{\lambda}{2}
        \sum_{l=1}^{n_l-1}\sum_{i=1}^{s_l}\sum_{j=1}^{s_{l+1}}\left(W_{ji}^{(l)}\right)^2
    \end{equation*}
    Our goal now, is to minimize $J(W,b)$. To train our neural network, we will
    initialize each parameter $W_{ij}^{(l)}$ and each $b_i^{(l)}$ to a small
    random value near zero. If they are all the same value (e.g. $0$) then they
    will end up learning the same function such that
    $a_1^{(2)}=a_2^{(2)}=a_3^{(2)}=\dots$ for any input $x$.

    Each iteration of gradient descent then updates the parameters $W,b$ as
    follows:
    \begin{align*}
        W_{ij}^{(l)}&=W_{ij}^{(l)}-\eta \frac{\partial}{\partial
        W_{ij}^{(l)}}J(W,b)\\
        b_i^{(l)}&=-\eta \frac{\partial}{\partial b_i^{(l)}}J(W,b)
    \end{align*}

    Seems simple right? But how do we do this efficiently? The backpropagation
    algorithm is the key. The intuition is that given some training example
    $(x,y)$, we want to do a ``forward pass'' which computes all the
    activations throughout the network, including the output value. Then for
    each node $i$ in layer $l$, we compute an ``error term'' $\delta_i^{(l)}$
    which measures how much that node was ``responsible'' for any errors in the
    output.

    For $\delta_i^{(nl)}$ we can simply compute the difference between the
    activation and the true target label $y$. For hidden units, we don't have
    the ``correct answer'' so instead we compute $\delta_i^{(l)}$ based on
    weighted average of the error terms of the nodes that uses $a_i^{(l)}$ as
    input. So in detail:

    \begin{enumerate}
        \item Perform a feedforward pass, computing the activations for layers
        $L_2,L_3,\dots,L_{n_l}$
        \item For each output unit $i$ in layer $n_l$, set
        \begin{align*}
            \delta_i^{(n_l)} &= \frac{\partial}{\partial z_i^{(n_l)}}
            \frac{1}{2}\|y-h_{W,b}(x)\|^2 = \frac{\partial}{\partial
            z_i^{(n_l)}} \frac{1}{2}
            \left(\sqrt{(y_1-z_1^{(n_l)})^2+\dots+
                (y_i-z_i^{(n_l)})^2+\dots+
                (y_{s_l}-z_{s_l}^{(n_l)})^2}\right)^2\\
            \delta_i^{(n_l)} &= -(y_i - a_i^{(n_l)}) \cdot
            \phi'(z_i^{(n_l)})\\
        \end{align*}
        \item For $l=n_l-1, n_l-2,\dots, 2$
            For each node $i$ in $L_l$ set
            \begin{equation*}
                \delta_i^{(l)}=\left(\sum_{j=1}^{s_{l+1}}W_{ji}^{(l)}\delta_j^{(l+1)}\phi'(z_i^{(l)})\right)
            \end{equation*}
        \item We can now compute the desired partial derivatives, as:
        \begin{align*}
            \frac{\partial}{\partial W_{ij}^{(l)}}J(W,b;x,y) &=
            a_j^{(l)}\delta_i^{(l)}\\
            \frac{\partial}{\partial b_i^{(l)}}J(W,b;x,y) &= \delta_i^{(l+1)}
        \end{align*}
    \end{enumerate}
    We can then compute the derivatives with respect to the whole data-set, for
    the overall cost function, as:
    \begin{align*}
        \frac{\partial}{\partial W_{ij}^{(l)}} J(W,b) &=
        \left[\frac{1}{|D|}\sum_{i=1}^{|D|}\frac{\partial}{\partial
        W_{ij}^{(l)}} J(W,b;x_i,y_i)\right] + \lambda W_{ij}^{(l)}\\
        \frac{\partial}{\partial b_i^{(l)}} J(W,b) &=
        \frac{1}{|D|}\sum_{i=1}^{|D|} \frac{\partial}{\partial b_i^{(l)}}
        J(W,b;x_i, y_i)
    \end{align*}

    Numerical differentiation takes one evaluation of the neural net per weight
    (\bigO{(\#weights)^2}), now with back propagation, we simply use each edge
    in each pass \bigO{\#weights}.

    \subsection{Deep Nets}
    Deep nets are when we have many layers of neurons. Each layer transforms
    the input into a ``better'' representation, and we will just repeat this
    procedure, and keep getting ``better'' representations. Historically, this
    has seemed to failed, this could either be because of a bad hypothesis or
    an issue with the learning algorithm.

    One of the approaches to fixing this is better architecture, for example
    the convolutional network. Others are much more data, or computing power or
    do find better weights before we do backpropagation or maybe the field just
    needs to mature.

    \subsubsection{Convolutional networks}
    These are useful for, for example image recognition. The building blocks of
    a convolutional network are as follow:
    \begin{enumerate}
        \item Convolution
        \item Non Linearity
        \item Pooling
        \item Classification (fully connected layer)
    \end{enumerate}

    \subsubsection{Convolution}
    First of all, convolution is like sliding a window over the image and
    applying some filter to it. For example we could have a simple ``filter'',
    ``kernel'' or ``feature detector'' which is simply a $m\times n$ matrix
    which slides over the image. For example the matrix:
    \begin{equation*}
        \begin{pmatrix}
        1 & 0\\
        0 & 1
        \end{pmatrix}
    \end{equation*}
    might slide over the input:
    \begin{equation*}
        \begin{pmatrix}
        1 & 0 & 1\\
        0 & 2 & 0\\
        1 & 0 & 3
        \end{pmatrix}
    \end{equation*}
    Since the feature detector is $2\times 2$, it will look at first top-left 4
    numbers, then it will slide $1$ pixel right and look at the top-right
    numbers, then the bottom-left and then the bottom right. Producing the
    following ``feature map'':
    \begin{equation*}
        \begin{pmatrix}
        1*1 + 0*0 + 0*0 + 1*2 & 1*0 + 0*1 + 0*2 + 1*0\\
        1*0 + 0*2 + 0*1 + 1*0 & 1*2 + 0*1 + 0*1 + 1*3
        \end{pmatrix} = \begin{pmatrix}
        3 & 0\\
        0 & 5
        \end{pmatrix}
    \end{equation*}
    The size of the ``feature map'' is controlled by three parameers:
    \begin{itemize}
        \item Depth corresponding to the number of filters
        \item Stride is the number of pixels or indices we move the feature
        detector with. In the above example it was 1
        \item Zero-padding, we might zero-pad in order to be able to slide the
        window around the  edges of the matrix as well. Zero-padded convolution
        is also called \textit{wide convolution} while non-zero-padded is
        called \textit{narrow convolution}
    \end{itemize}

    \subsubsection{Non-linearity}
    The non-linearity operation aims to introduce some non-linearity into the
    data, as it real world data is often non-linear. An example of such an
    operation is the ReLU opeartion, which simply sets any negative elements to
    $0$. The result is the Rectified Feature Map

    \subsubsection{Pooling}
    Pooling reduces the dimensionality of each feature map but retains the most
    important information. Pooling can be done in a number of different way
    like Max, Average, Sum etc. It runs a window over the rectified feature
    map, and runs some function on the entries in the window. It might, for
    example, output the maximum value in the window, resulting in a smaller
    feature map.

    \subsubsection{Fully connected layer}
    The result of one or more convolution and poolings, can then be put through
    the fully connected layer which will try to use the information to perform
    e.g. classification, as usual.

    \subsection{PCA}
    PCA tries to take some data in dimension $d$ in to some dimension $k$. A
    neural net autoencoder can come up with something very similar to this.

    \subsection{Autoencoder}

    This is an auto-encoder:
    \begin{tikzpicture}[node distance = 0.2cm and 1.5cm]
    \node[inputnode] (x1) {$x_1$};
    \node[inputnode, below=of x1] (x2) {$x_2$};
    \node[inputnode, below=of x2] (x3) {$x_3$};
    \node[inputnode, below=of x3] (x4) {$x_4$};
    \node[inputnode, below=of x4] (x5) {$x_5$};
    \node[inputnode, below=of x5] (x6) {$x_6$};
    \node[inputnode, below=of x6] (b1) {$+1$};

    \node[hiddennode, right=of x1, between=x2 and x3] (h1) {};
    \node[hiddennode, below=of h1] (h2) {};
    \node[hiddennode, below=of h2] (h3) {};
    \node[hiddennode, below=of h3] (b2) {$+1$};

    \node[right=of x1] (xx1) {};

    \node[outputnode, right=of xx1] (o1) {$\hat{x_1}$};
    \node[right=of o1] (oo1) {};

    \node[outputnode, below=of o1] (o2) {$\hat{x_2}$};
    \node[right=of o2] (oo2) {};

    \node[outputnode, below=of o2] (o3) {$\hat{x_3}$};
    \node[right=of o3] (oo3) {};

    \node[outputnode, below=of o3] (o4) {$\hat{x_4}$};
    \node[right=of o4] (oo4) {};

    \node[outputnode, below=of o4] (o5) {$\hat{x_5}$};
    \node[right=of o5] (oo5) {};

    \node[outputnode, below=of o5] (o6) {$\hat{x_6}$};
    \node[right=of o6] (oo6) {};

    \node[textbox, right=of oo1, between=o3 and o4] (t) {$h_{W,b}(x)$};

    \path[edgepath]
    (x1) edge node {} (h1)
    (x1) edge node {} (h2)
    (x1) edge node {} (h3)
    (x2) edge node {} (h1)
    (x2) edge node {} (h2)
    (x2) edge node {} (h3)
    (x3) edge node {} (h1)
    (x3) edge node {} (h2)
    (x3) edge node {} (h3)
    (x4) edge node {} (h1)
    (x4) edge node {} (h2)
    (x4) edge node {} (h3)
    (x5) edge node {} (h1)
    (x5) edge node {} (h2)
    (x5) edge node {} (h3)
    (x6) edge node {} (h1)
    (x6) edge node {} (h2)
    (x6) edge node {} (h3)
    (b1) edge node {} (h1)
    (b1) edge node {} (h2)
    (b1) edge node {} (h3)
    (h1) edge node {} (o1)
    (h2) edge node {} (o1)
    (h3) edge node {} (o1)
    (h1) edge node {} (o2)
    (h2) edge node {} (o2)
    (h3) edge node {} (o2)
    (h1) edge node {} (o3)
    (h2) edge node {} (o3)
    (h3) edge node {} (o3)
    (h1) edge node {} (o4)
    (h2) edge node {} (o4)
    (h3) edge node {} (o4)
    (h1) edge node {} (o5)
    (h2) edge node {} (o5)
    (h3) edge node {} (o5)
    (h1) edge node {} (o6)
    (h2) edge node {} (o6)
    (h3) edge node {} (o6)
    (b2) edge node {} (o1)
    (b2) edge node {} (o2)
    (b2) edge node {} (o3)
    (b2) edge node {} (o4)
    (b2) edge node {} (o5)
    (b2) edge node {} (o6)
    (o1) edge node {} (oo1)
    (o2) edge node {} (oo2)
    (o3) edge node {} (oo3)
    (o4) edge node {} (oo4)
    (o5) edge node {} (oo5)
    (o6) edge node {} (oo6);
    \end{tikzpicture}
    The autoencoder tries to learn a function $h_{W,b}\approx x$, i.e. it takes
    the input and tries to re-create it. This can be useful, as it effectively
    tries to learn what features in the data is important if I have to know the
    difference between two inputs but I can only do it in e.g. half the space.
    So if there some of the input features are correlated, then this algorithm
    will be able to discover some of those correlations. In fact it often ends
    up learning a low-dimensional representation very similar to PCA's.

    \subsection{Regularization}
    We introduced weight decay regularization already, another good way to
    regularize neural networks is the simple ``dropout'' technique, which
    simply means that we ignore the output of some random neurons. This forces
    the hidden neurons to be robust and approcimates averaging of exponentially
    many models.

    \subsection{Finding a good local minimum}
    Our neural network is riddled with local minimum, so in order to find a
    good local minimum, we will simply start the gradient descent from many
    different starting points, so we end up in different local minimums and
    choose the best one, hoping that it is a global minimum.
\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS630: Database Management Systems
% Copyright 2014 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 4}

Design a database for a bank, including information about customers and their accounts.
Information about customers includes their name, address, phone and SSN.
Accounts have numbers, types (e.g., savings/checking) and balances.
Also, record the customer(s) who own an account.

\begin{enumerate}[label=(\alph*)]
\item Draw the E/R diagram for this database, assuming no constraints hold other than what results from the schema.

\textbf{Solution:}

\begin{figure}[H]\centering
\begin{tikzpicture}[node distance=3cm, every edge/.style={link}]
\node[entity] (customers) {Customers};
\node[attribute] (ssn) [above of = customers] {\key{SSN}} edge (customers);
\node[attribute] (name) [above left of = customers] {Name} edge (customers);
\node[attribute] (phone) [left of = customers] {Phone} edge (customers);
\node[attribute] (address) [below left of = customers] {Address} edge (customers);
\node[relationship] (own) [right of = customers] {Own} edge (customers);
\node[entity] (accounts) [right of = own] {Accounts} edge (own);
\node[attribute] (numbers) [right of=accounts] {\key{Number}} edge (accounts);
\node[attribute] (type) [below right of =accounts] {Type} edge (accounts);
\node[attribute] (credits) [above right of=accounts] {Credits} edge (accounts);
\end{tikzpicture}
\caption{ER Diagram for Question 4$\left(a\right)$} \label{fig:ER1}
\end{figure}

\item Modify the E/R diagram from (a) to reflect the constraint that each customer must have at least one account.

\textbf{Solution:}

\begin{figure}[H]\centering
\begin{tikzpicture}[node distance=3cm, every edge/.style={link}]
\node[entity] (customers) {Customers};
\node[attribute] (ssn) [above of = customers] {\key{SSN}} edge (customers);
\node[attribute] (name) [above left of = customers] {Name} edge (customers);
\node[attribute] (phone) [left of = customers] {Phone} edge (customers);
\node[attribute] (address) [below left of = customers] {Address} edge (customers);

\node[relationship] (own) [right of = customers] {Own} edge [total] (customers);
\node[entity] (accounts) [right of = own] {Accounts} edge (own);
\node[attribute] (numbers) [right of=accounts] {\key{Number}} edge (accounts);
\node[attribute] (type) [below right of =accounts] {Type} edge (accounts);
\node[attribute] (credits) [above right of=accounts] {Credits} edge (accounts);
\end{tikzpicture}
\caption{ER Diagram for Question 4$\left(b\right)$} \label{fig:ER2}
\end{figure}

\item Modify the E/R diagram from (a) to reflect the constraint that an account must have only one customer.

\textbf{Solution:}

\begin{figure}[H]\centering
\begin{tikzpicture}[node distance=3cm, every edge/.style={link}]
\node[entity] (customers) {Customers};
\node[attribute] (ssn) [above of = customers] {\key{SSN}} edge (customers);
\node[attribute] (name) [above left of = customers] {Name} edge (customers);
\node[attribute] (phone) [left of = customers] {Phone} edge (customers);
\node[attribute] (address) [below left of = customers] {Address} edge (customers);

\node[relationship] (own) [right of = customers] {Own} edge node[auto,swap] {1} (customers);
\node[entity] (accounts) [right of = own] {Accounts} edge node[auto,swap] {m} (own);
\node[attribute] (numbers) [right of=accounts] {\key{Number}} edge (accounts);
\node[attribute] (type) [below right of =accounts] {Type} edge (accounts);
\node[attribute] (credits) [above right of=accounts] {Credits} edge (accounts);
\end{tikzpicture}
\caption{ER Diagram for Question 4$\left(c\right)$} \label{fig:ER3}
\end{figure}

\item Modify the diagram from (a) such that a customer can have a set of addresses(which are street-city-state triples) and a set of phones.
Recall that in the E/R model there can be only primitive data types (no sets).

\textbf{Solution:}

\begin{figure}[H]\centering
\begin{tikzpicture}[node distance=3cm, every edge/.style={link}]
\node[entity] (customers) {Customers};
\node[attribute] (ssn) [above of = customers] {\key{SSN}} edge (customers);
\node[attribute] (name) [above left of = customers] {Name} edge (customers);

\node[relationship] (own) [right of = customers] {Own} edge (customers);
\node[entity] (accounts) [right of = own] {Accounts} edge (own);
\node[attribute] (numbers) [above of=accounts] {\key{Numbers}} edge (accounts);
\node[attribute] (type) [above right of =accounts] {Type} edge (accounts);
\node[attribute] (credits) [right of=accounts] {Credits} edge (accounts);

\node[relationship] (lives) [below right of = customers] {Live} edge (customers);
\node[entity] (address) [right of = lives] {Address} edge (lives);
\node[attribute] (number) [right of=address] {\key{Number}} edge (address);
\node[attribute] (street) [below right of =address] {Street} edge (address);
\node[attribute] (city) [below of=address] {City} edge (address);
\node[attribute] (state) [below left of=address] {State} edge (address);

\node[relationship] (has) [left of = customers] {Has} edge (customers);
\node[entity] (phone) [below of = has] {Phone} edge (has);
\node[attribute] (number) [right of =phone] {\key{Number}} edge (phone);
\end{tikzpicture}
\caption{ER Diagram for Question 4$\left(d\right)$} \label{fig:ER4}
\end{figure}

\end{enumerate}