% C-c C-o to insert the block

% Individual equation: equation* block
% Inline equation \begin{math}\frac{sin(x)}{x}\end{math}
\documentclass{article}

\usepackage{amsmath,amssymb}

\ifdefined\ispreview
\usepackage[active,tightpage]{preview}
\PreviewEnvironment{math}
\PreviewEnvironment{equation*}
\fi

\DeclareMathOperator{\E}{\mathbb{E}}
\DeclareMathOperator*{\argmin}{arg\,min}

\begin{document}

\subsection{Page 1}

One of the simplest example of the method in this family is random search, when
you randomly sample the thing you’re looking for (in case of RL
it’s the policy \begin{math}\pi(a|s)\end{math}), then you check

\subsection{Page 2}

More formally, the method above could be expressed as this sequence of steps.
\begin{enumerate}
  \item Initialize learning rate \begin{math}\alpha\end{math}, noise standard
  deviation \begin{math}\sigma\end{math},
    initial policy parameters \begin{math}\theta_0\end{math}
  \item For t = 0, 1, 2, ... do
    \begin{enumerate}
    \item Sample batch of noise with a shape of the
      weights \begin{math}\epsilon_1, \ldots, \epsilon_n \sim \mathcal{N}(0, I)\end{math}
    \item Compute returns \begin{math}F_i=F(\theta_t + \sigma
      \epsilon_i)\end{math} for \begin{math}i = 1, \ldots, n\end{math}
    \item Update weights \begin{math}\theta_{t+1} \leftarrow
      \theta_t+\alpha\frac{1}{n\sigma}\sum_{i=1}^nF_i\epsilon_i
    \end{math}
    \end{enumerate}
\end{enumerate}

\subsection{Page 4}

The last and the central function of the method is train\_step which takes the
batch with noise and their respective rewards and calculates the update to the
network parameters by applying the formula
\begin{math}\theta_{t+1} \leftarrow \theta_t+\alpha\frac{1}{n\sigma}\sum_{i=1}^nF_i\epsilon_i\end{math}

\subsection{Page 12}

\begin{enumerate}
\item Initialize mutation power \begin{math}\sigma\end{math}, population size N,
  number of the selected individuals T, initial
  population \begin{math}P^0\end{math} with N randomly-initialized
  policies, their fitness \begin{math}F^0=\{F(P^0_i)|i=1 \ldots N\}\end{math}
\item For generation \begin{math}g = 1 \ldots G\end{math}

  \begin{enumerate}
    \item Sort generation \begin{math}P^{g-1}\end{math} by descending of fitness \begin{math}F^{g-1}\end{math}
    \item Copy elite \begin{math}P^g_1=P^{g-1}_1, F_1^g=F_1^{g-1}\end{math}
    \item For individual \begin{math}i = 2\ldots N\end{math}
      \begin{enumerate}
      \item k = randomly select parent from \begin{math}1\ldots T\end{math}
      \item Sample \begin{math}\epsilon \sim \mathcal{N}(0, I)\end{math}
      \item Mutate parent \begin{math}P_i^g=P_i^{g-1}+\sigma\epsilon\end{math}
      \item Get its fitness \begin{math}F_i^g=F(P_i^g)\end{math}
      \end{enumerate}
  \end{enumerate}
\end{enumerate}

\subsection{Page 16}
To implement the Novelty Search, we define so called Behaviour Characteristic BC(\begin{math}\pi\end{math}),
which describes the behaviour of the policy and a distance between two BCs.

\end{document}


%!TEX root = thesis.tex

\chapter{Background}
\label{ch:Background}
This chapter introduces the theory behind the implemented method and the terminology used in this report.

The notation used in the report is as follows. Upper-case letters are used for random variables, e.g. $O_t$ for the random variable of observations at time $t$. Lower-case letters are used to indicate specific values of the corresponding random variable, e.g. $o_t$ represents a specific value of the random variable $O_t$.

\section{Formulation and Terminology of POMDPs}
A Markov decision process (MDP) is a mathematical framework for modeling sequential decision making problems and the framework forms the basis for the POMDP framework, which is a generalization of MDPs. In the MDP framework the state of the world is directly observable whereas in the POMDP framework it is only partially observable. When the state of the world cannot be determined directly a probability distribution over all the possible states of the world must be maintained. More detailed information about POMDPs can be found in \cite{Kaelbling1998} and \cite{Aberdeen2003}.

\subsection{The POMDP World Model}
\label{sec:POMDPModel}
The acting entity in the POMDP world model, the learner, is called an \emph{agent}. The world model in the POMDP framework consists of the following:

\begin{itemize}
  \item $\mathcal{S} = \{1, \dotsc, |\mathcal{S}|\}$ -- States of the world.
  \item $\mathcal{A} = \{1, \dotsc, |\mathcal{A}|\}$ -- Actions available to the agent.
  \item $\mathcal{O} = \{1, \dotsc, |\mathcal{O}|\}$ -- Observations the agent can make.
  \item Reward $r(i) \in \mathds{R}$ for each state $i \in \mathcal{S}$.
\end{itemize}
At time $t$, in state $S_t$, the agent takes an action $A_t$ according to its policy, makes an observation $O_t$, transitions to a new state $S_{t+1}$ and collects a reward $r(S_{t+1})$ according to the reward function $r(i)$. The state transition is made according to the system dynamics that are modeled by the transition model discussed in \ref{sec:TransitionModel}. Figure \ref{fig:POMDPModel} illustrates this along with the relations between MDPs and POMDPs.

\begin{figure}[ht]
  \centering
  \includegraphics[width=0.5\textwidth]{figures/pomdp_model}
  \caption{POMDP diagram. In an MDP the agent would observe the state, $S_t$, directly. In POMDPs the agent makes an observation, $O_t$, that is mapped to the state $S_t$ by a stochastic process. Figure adapted from \cite{Aberdeen2003}.}
  \label{fig:POMDPModel}
\end{figure}

To put these concepts into context consider the problem of a robot that needs to find its way out of a maze. In the POMDP world model for that problem the robot is the agent and the state could be the location of the robot in the maze. The actions could be that the robot decides to move in a certain direction or even use one of its sensors. An observation could then be a reading from that sensor.

Since this example is easy to grasp it will be used throughout this report where deemed necessary.

\subsection{Transition Model}
\label{sec:TransitionModel}
The transition model defines the probabilities of moving from the current state to any other state of the world for any given action. For each state and action pair $(S_t = i, A_t = k)$ we write the probability of moving from state $i$ to state $j$ when action $A_t = k$ is taken as
\begin{equation}
  p(S_{t+1} = j|S_t = i, A_t = k)
\end{equation}
When a transition from state $i$ to state $j$ is for some reason impossible given action $k$, this probability is zero. An example of this would be when the agent takes an action that is intended to sense the environment and not move the agent. When the system dynamics prohibit all transitions, i.e. the state never changes, this probability is 1 for $i = j$ and zero otherwise, i.e.
\begin{equation}
\label{eq:IdentityTransitionModel}
  p(S_{t+1} = j|S_t = i, A_t = k) =
  \begin{cases}
    1 & \text{if $i = j$}\\
    0 & \text{otherwise}
  \end{cases}
\end{equation}
The transition model for a specific problem is generally known or estimated using either empirical data or simply intuition.

\subsection{Belief States}
Because of the partial observability the agent is unable to determine its current state. However, the observations the agent makes depend on the underlying state so the agent can use the observations to estimate the current state. This estimation is called the agent's belief, or the belief state.
\begin{equation*}
  \mathcal{B} = \{1, \dotsc, |\mathcal{B}|\} \text{ -- Belief states, the agent's belief of the state of the world.}
\end{equation*}
A single belief state at time $t$, $B_t$, is a probability distribution over all states $i \in \mathcal{S}$. The probability of the state at time $t$ being equal to $i$, given all actions that have been taken and observations that have been made, is defined as
\begin{equation}
  B_t^i \defeq p(S_t = i | A_{1:t}, O_{1:t})
\end{equation}
Each time the agent takes an action and makes a new observation it gains more information about the state of the world. The most obvious way to keep track of this information is to store all actions and observations the agent makes, but that results in an unnecessary use of memory and computing power. It is possible to store all the acquired information within the belief state and update it only when necessary, after taking actions and making observations, without storing the whole history of actions and observations.

Given the belief state $b_t$, current action $a_t$ and observation $o_t$ the belief for state $j$ can be updated using
\begin{equation}
  \label{eq:UpdateBelief}
  b_t^j = \frac{ p(o_t | S_t = j, a_t) \sum_{i} p(S_t = j | S_{t-1} = i, a_t) b_{t-1}^i } { p(o_t) }
\end{equation}
If the state never changes we can simplify equation \eqref{eq:UpdateBelief} to

\begin{equation}
  \label{eq:UpdateBeliefFixedTarget}
  b_t^j = \frac{ p(o_t | S_t = j, a_t) b_{t-1}^j }{ p(o_t) }
\end{equation}
where $p(o_t)$ can be treated as a normalization factor. The derivations of \eqref{eq:UpdateBelief} and \eqref{eq:UpdateBeliefFixedTarget} are presented in Appendix \ref{app:BeliefUpdates}.

\subsection{Reward Function}
\label{sec:RewardFunction}
After each state transition the agent gets a reward according to the reward function. These rewards are the agent's indication of its performance and they are generally extrinsic and task-specific. In the case of a robot finding its way out of a maze the reward function might return a small negative reward for all states the robot visits and a large positive reward for the end state, i.e. when the robot is out of the maze, as described in \ref{sec:ModelBasedvsModelFree}. Since the goal is to find a policy that maximizes the expected total rewards the robot will learn to find the shortest path out of the maze.

The reward function may depend on the current action and either the current or the resulting state, but it can be made dependent on the resulting state only without loss of generality \cite{Aberdeen2003}. In Section \ref{sec:InformationRewards} we show how a reward function can be made dependent on the belief state only.

\subsection{Policy}
When it is time for the agent to act it consults its policy. The agent's policy is a mapping from states to actions that defines the agent's behavior. Finding a solution to a POMDP problem is done by finding the optimal policy, i.e. a policy that provides the optimal action for every state the agent visits, or approximating it when finding an exact solution is infeasible.

We already know that in POMDPs the agent is unable to determine its current state and a mapping from states to actions is therefore useless for the agent. The straightforward solution to this is to make the policy depend on the belief state. That is typically done by parameterizing the policy as a function of the belief state. Finding or approximating the optimal policy therefore becomes a search for the optimal policy parameters.

\subsection{Logistic Policies}
\label{sec:LogisticPolicies}
One way of parameterizing a policy is to represent it as a function with policy parameters $\theta \in \mathds{R^{|\mathcal{A}| \times |\mathcal{S}|}}$, where $\mathcal{A}$ is the set of actions available to the agent and $\mathcal{S}$ the set of states of the world as presented in \ref{sec:POMDPModel}. Then $\theta$ forms a $|\mathcal{A}| \times |\mathcal{S}|$ matrix which in combination with the current belief state, $b_t$, forms a set of linear functions of the belief state. If $\theta^i$ represents the $i$th row of the matrix $\theta$ then we can select an action according to the policy with
\begin{equation}
  a_{t+1} = \argmax{i} \theta^i \cdot b_t
\end{equation}
This is a deterministic policy since it will always result in the same action for a particular belief state $b_t$. However, if we need a stochastic policy -- as is the case for the policy gradient method described in \ref{sec:GradientAscent} -- we can represent it as a logistic function using the policy parameters $\theta$ and the current belief state, $b_t$. Again, if $\theta^i$ represents the $i$th row of the matrix $\theta$ then the probability of taking action $k$ is given by
\begin{equation}
\label{eq:LogisticPolicy}
  p(A_{t+1} = k | b_t, \theta) = \frac{ \exp{(\theta^k \cdot b_t)} }{ \sum_{i=1}^{|\mathcal{A}|}{ \exp{( \theta^i \cdot b_t}) }}
\end{equation}
where $b_t$ is the agent's belief. This is a probability distribution that represents a stochastic policy. Taking an action based on this stochastic policy is done by sampling from the probability distribution.

The key to estimating the policy gradient in equation \eqref{eq:PolicyGradient} lies in calculating the gradient of equation \eqref{eq:LogisticPolicy} with respect to $\theta$, as shown in \cite{Butko2010b}.

\subsection{Observation Model}
\label{sec:ObservationModel}
When training a policy, the agent must be able to make observations. The sensors the agent uses to observe the world are in general imperfect and noisy which means that when the agent makes an observation it has to assume some uncertainty.

The observations available to the agent can be actual readings from its sensors -- and that is the case when the agent is put to work in the real world -- but during training, having a model that can generate observations is preferred. This is called an observation model and its purpose is to model the noisy behavior of the actual sensors.

The observation model needs to be estimated or trained using real data acquired by the agent's sensors before training the policy.

\subsection{Observation Likelihood}
\label{sec:ObservationLikelihood}
The observation likelihood is the probability of making observation $O_t$ after the agent takes action $A_t$ in state $S_{t}$,
\begin{equation}
  p(O_t|S_t, A_t)
\end{equation}
given the current observation model.

\subsection{Model-based vs. Model-free}
\label{sec:ModelBasedvsModelFree}
When the observation model, the transition model and the reward function $r(i)$ are all known, the POMDP is considered to be \emph{model-based}. Otherwise the POMDP is said to be \emph{model-free}.

A model-free POMDP can learn a policy without a full model of the world by sampling trajectories through the state space, either by interacting with the world or using a simulator. To give an example, consider our robot friend again that needs to find its way out of a maze. If the robot receives a small negative reward in each state it visits and a large positive reward when it gets out, it can learn a policy by exploring the maze without necessarily knowing the transition model or observation model.

The POMDPs considered in this report are always model-based.

\section{Policy Gradient}
\label{sec:PolicyGradient}
Methods for finding exact optimal solutions to POMDPs exist but they become infeasible for problems with large state- and action-spaces and they require full knowledge of the system dynamics, such as the transition model and the reward function. When either of those requirements are not fulfilled, reinforcement learning can be used to estimate the optimal policy.

The methods used, when exact methods are infeasible, are either value function methods or direct policy search. Value function methods require the agent to assign a value to each state and action pair and this value represents the long term expected reward for taking an action in a given state. In every state the agent chooses the action that has the highest value given the state, so the policy is essentially extracted from the value function. As the state-space grows larger the value function becomes more complex, making it more challenging to learn. However, the policy-space does not necessarily become more complex, making it more desirable to search the policy-space directly \cite{Aberdeen2003}.

An algorithm for a direct policy search using gradient ascent was presented in \cite{BaxterB2001} and used in \cite{Butko2010b}. The algorithm generates an estimate of the gradient of the average reward with respect to the policy and using gradient ascent the policy is improved iteratively. This requires a parameterized representation of the policy.

\subsection{Gradient Ascent}
\label{sec:GradientAscent}
If the policy is parameterized by $\theta$, for example as described in \ref{sec:LogisticPolicies}, the average reward of the policy can be written as
\begin{equation}
  \underbrace{\eta(\theta)}_{\mathclap{\substack{\text{Average} \\ \text{reward}}}} = \sum_{x \in X} \underbrace{r(x)}_{\text{reward}} \overbrace{p(x | \theta)}^{\substack{\text{prob. of $x$} \\ \text{given policy}}} = \Ex [r(x)]
\end{equation}
where $x$ represents the parameters that the reward function depends on. In our case $x$ is the agent's belief, $b$. The gradient of the average reward given the parameterized policy $\theta$ is therefore
\begin{align}
  \nabla \eta(\theta) &= \sum_{b \in \mathcal{B}} r(b) \nabla p(b | \theta) \\
                      &= \sum_{b \in \mathcal{B}} r(b) \frac{\nabla p(b | \theta)}{p(b | \theta)} p(b | \theta) \\
                      &= \Ex \bigg{[}r(b) \frac{\nabla p(b | \theta)}{p(b | \theta)}\bigg{]}
\end{align}
If we generate $N$ i.i.d. samples from $p(b | \theta)$ then we can estimate $\nabla \eta(\theta)$ by
\begin{equation}
  \widetilde{\nabla} \eta(\theta) = \frac{1}{N} \sum_{i=1}^N r(b_i) \frac{\nabla p(b_i | \theta)}{p(b_i | \theta)}
  \Bigg{\}} \substack{\text{given fixed $\theta$} \\ \text{we sample $b_i$} \\ \text{and we estimate} \\ \text{$\nabla \eta(\theta)$}}
\end{equation}
with $N \to \infty$, $\widetilde{\nabla} \eta(\theta) \to \nabla \eta(\theta)$.
\\\\
Assume a regenerative process, each i.i.d. $b_i$ is now a sequence $b_i^1 \dotsc b_i^{M_i}$.
We get
\begin{equation}
  \widetilde{\nabla} \eta(\theta) = \frac{1}{N} \sum_{i=1}^N r(b_i^1, \dotsc, b_i^{M_i}) \sum_{j=1}^{M_i} \frac{\nabla p(b_i^j | \theta)}{p(b_i^j | \theta)}
\end{equation}
If we assume
\begin{equation}
  r(b_i^1, \dotsc, b_i^{M_i}) = f\Big{(} \underbrace{r(b_i^1, \dotsc, b_i^{M_{i-1}})}_{r_i^{M_{i-1}}}, b_i^{M_i}\Big{)}
\end{equation}
and write
\begin{equation}
  \sum_{j=1}^{M_i} \frac{\nabla p(b_i^j | \theta)}{p(b_i^j | \theta)} = Z_i^{M_i} = Z_i^{M_{i-1}}  + \frac{\nabla p(b_i^{M_i} | \theta)}{p(b_i^{M_i} | \theta)}
\end{equation}
we have
\begin{equation}
  \widetilde{\nabla} \eta(\theta) = \frac{1}{N} \sum_{i=1}^N \underbrace{r_i^{M_i} Z_i^{M_i}}_{\substack{\text{Computed} \\ \text{iteratively}}} \bigg{\}} \frac{1}{N} \widetilde{\nabla}_N
\end{equation}
where
\begin{equation}
  r_i^{M_{i}} = r(b_i^1, \dotsc, b_i^{M_i})
\end{equation}
This gives an unbiased estimate of the policy gradient
\begin{equation}
  \label{eq:GradientEstimate}
  \underbrace{\widetilde{\nabla}_N}_{\substack{\text{Computed} \\ \text{iteratively}}} = \widetilde{\nabla}_{N-1} + r_N^{M_N} Z_N^{M_N}
\end{equation}
The estimate of the gradient of the average reward, $\widetilde{\nabla}_N$, can therefore be computed iteratively and the policy parameters $\theta$ updated by a simple update rule
\begin{equation}
\label{eq:PolicyGradient}
  \theta_{new} = \theta + \frac{\widetilde{\nabla}_N}{N}
\end{equation}
where $N$ is the number of samples used to estimate the gradient, i.e. the number of episodes.

When training a policy using gradient ascent the gradient of the average reward is estimated from a number of episodes, during which the policy parameters are kept fixed. When all episodes have finished the policy parameters are updated using the gradient estimation and that concludes one training iteration, a so called epoch.

The gradient estimate is what has been referred to as the policy gradient.

\section{Infomax Control}
Using the theory of optimal control in combination with information maximization is known as \emph{Infomax control} \cite{Movellan2005}. The idea is to treat the task of learning as a control problem where the goal is to derive a policy that seeks to maximize information, or equivalently, reduce the total uncertainty. The POMDP models that are used for this purpose are information gathering POMDPs and have been called Infomax POMDPs, or I-POMDPs \cite{Butko2010b}.

\section{Information Rewards}
\label{sec:InformationRewards}
Conceptually, the POMDP world issues rewards to the agent as it interacts with the world and these rewards are generally task-specific. In order to make an information-driven POMDP, an I-POMDP, the reward function must depend on the amount of information the agent gathers. The way this is done is to make the reward a function of the agent's belief of the world. The entropy of the agent's belief at each point in time becomes the reward the agent receives. As the agent becomes more and more certain about the state of the world, the higher reward the agent receives. This principle is what drives the learning of information gathering strategies. The policy gradient is used to adjust the current policy and the reward, i.e. the certainty of the agent's belief, controls the magnitude of the adjustment. This is shown in equation \eqref{eq:GradientEstimate}.

The information reward function becomes
\begin{equation}
\label{eq:InformationRewards}
  r_t(b_t) = \sum_{j=1}^{|\mathcal{S}|} b_t^j \log_2{b_t^j}
\end{equation}
where $\mathcal{S}$ is the set of states of the world and $b_t$ is the agent's belief at time $t$.

Care needs to be taken when calculating the logarithm of the belief in practice because as the agent becomes more certain about the state of the world, most of the values of the belief vector $b_t$ approach zero. As the precision of numbers stored in computers varies, we might have the situation where some of those values are actually stored as zero. Therefore, when calculating the rewards we define

\begin{equation}
  \log_2{0} \defeq 0
\end{equation}


\documentclass[12pt, letterpaper]{article}
\usepackage[english]{babel}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{accents}

  %%%%%%%%%%%%
  % PREAMBLE %
  %%%%%%%%%%%%
\begin{document}
\selectlanguage{english}

\section{Whiteboard 2013-09-10}

\begin{equation}
  f = \frac{1}{10} \sum_{i} \limits
    ( G  p I_{\mathrm{calc}} + K - I_{\mathrm{obs}})^{2}
\end{equation}
where $g_{i} = G p I_{\mathrm{calc}} + K - I_{\mathrm{obs}}$ is
\texttt{func\_callable}

\begin{equation}
  \frac{\partial g}{\partial \theta_{x}}
    = ( G I_{\mathrm{c}} ) \frac{\partial p}{\partial \theta_{x}}
\end{equation}
where $\partial g / \partial \theta_{x}$ is \texttt{jacobian\_callable}

\begin{equation}
  \frac{\partial f}{\partial \theta_{x}}
    = 2 \sum (G p I_{c} + K - I_{o}) (G I_{c})
        \frac{\partial p}{\partial \theta_{x}}
\end{equation}

\begin{equation}
  \frac{\partial f}{\partial K}
    = 2 \sum_{i} \limits g_{i} \frac{\partial g_{i}}{\partial K}
    = 2 \sum_{i} \limits g_{i}
\end{equation}

\begin{equation}
  \frac{\partial f}{\partial G}
    = 2 \sum_{i} \limits g_{i} \frac{\partial g_{i}}{\partial G}
    = 2 \sum_{i} \limits g_{i} p I_{\mathrm{calc}}
\end{equation}

\begin{equation}
  \frac{\partial g_{i}}{\partial \theta_{x}},
  \frac{\partial g_{i}}{\partial \xi},
  \ldots
\end{equation}

\begin{equation}
  p_{B} = \frac{r_{s}^{2}}{2 r_{h}^{2} + r_{s}^{2}}
\end{equation}

\begin{equation}
  \frac{\partial p_{B}}{\partial \theta_{x}}
    = \frac{-4 r_{s}^{2} r_{h}}{(2 r_{h}^{2} + r_{s}^{2})^{2}}
      \frac{\partial r_{h}}{\partial \theta_{x}}
\end{equation}

\begin{equation}
  r_{h} = | \underaccent{\tilde}{S} |  - \frac{1}{\lambda}
\end{equation}

\begin{equation}
  r_{h} = (\underaccent{\tilde}{S} \cdot \underaccent{\tilde}{S})^{1 / 2} -
          \frac{1}{\lambda}
\end{equation}

\begin{equation}
  \frac{\partial r_{h}}{\partial \theta_{x}}
    = \frac{1}{2}
      \frac{1}{\sqrt{\underaccent{\tilde}{S} \underaccent{\tilde}{S}}}
      \frac{\partial \underaccent{\tilde}{S} \underaccent{\tilde}{S}}
           {\partial \theta_{x}}
\end{equation}

\begin{equation}
  \frac{\partial \underaccent{\tilde}{S} \underaccent{\tilde}{S}}
       {\partial \theta_{x}}
    = 2 \underaccent{\tilde}{S}
      \frac{\partial \underaccent{\tilde}{S}}{\partial \theta_{x}}
\end{equation}

\begin{equation}
  \underaccent{\tilde}{S} = \underaccent{\tilde}{x} + S_{0}
\end{equation}

\begin{equation}
  \frac{\partial \underaccent{\tilde}{S}}{\partial \theta_{x}}
    = \frac{\partial \underaccent{\tilde}{x}}{\partial \theta_{x}}
\end{equation}

\begin{equation}
  \underaccent{\tilde}{x} = A^{*} h
\end{equation}

\begin{equation}
  A^{*} = \mathbb{R}_{y} ( \mathbb{R}_{x} ( A_{0}^{*} ) )
\end{equation}

Aside:
\begin{equation}
  \frac{\partial A h}{\partial u}
    = A \frac{\partial h}{\partial u} +
      \frac{\partial A}{\partial u} h
\end{equation}

\begin{equation}
  \frac{\partial \underaccent{\tilde}{x}}{\partial \theta_{x}}
    = A^{*} \frac{\partial}{\partial \theta_{x}} \underaccent{\tilde}{h} +
      \frac{\partial A^{*}}{\partial \theta_{x}} \underaccent{\tilde}{h}
\end{equation}
where $\partial \underaccent{\tilde}{h} / \partial \theta_{x} = 0$ and
$\partial A^{*} / \partial \theta_{x}$ corresponds to
\texttt{dA\_drotx = X}, $( \underaccent{\tilde}{h}, k, l )$

\begin{equation}
  \frac{\partial A^{*}}{\partial \theta_{x}}
    = \mathbb{R}_{y} \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{x}} +
      \frac{\partial R_{y}}{\partial \theta_{x}} R_{x} A_{0}^{*}
\end{equation}
where  $\partial R_{y} / \partial \theta_{x} = 0$

\begin{equation}
  \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{x}}
    = R_{x} \frac{\partial A_{0}^{*}}{\partial \theta_{x}} +
      \frac{\partial R_{x}}{\partial \theta_{x}} A_{0}^{*}
\end{equation}
where $\partial A_{0}^{*} / \partial \theta_{x} = 0$ and $\partial
R_{x} / \partial \theta_{x}$ in the code.

\begin{equation}
  \frac{\partial A^{*}}{\partial \theta_{y}}
    = \mathbb{R}_{y} \frac{\partial R_{x} A_{0}^{*}}{\partial \theta_{y}} +
      \frac{\partial R_{y}}{\partial \theta_{y}} (R_{x} A_{0}^{*})
\end{equation}
where $\partial R_{x} A_{0}^{*} / \partial \theta_{y} = 0$

\begin{equation}
  \frac{\partial A^{*}}{\partial \theta_{y}}
    = \frac{\partial R_{y}}{\partial \theta_{x}} (R_{x} A_{0}^{*})
\end{equation}

\begin{equation}
  \frac{\partial A^{*}}{\partial \theta_{x}}
    = [R_{y}] \left[ \frac{\partial R_{x}}{\partial \theta_{x}} A_{0}^{*} \right]
\end{equation}

\end{document}


% This LaTeX was auto-generated from an M-file by MATLAB.
% To make changes, update the M-file and republish this document.

      \subsection{getPlotDistr\_cdp.m}

\begin{par}
\textbf{Summary:} Compute means and covariances of the Cartesian coordinates of the tips both the inner and outer pendulum assuming that the joint state $x$ of the cart-double-pendulum system is Gaussian, i.e., $x\sim N(m, s)$
\end{par} \vspace{1em}

\begin{verbatim}   function [M1, S1, M2, S2] = getPlotDistr_cdp(m, s, ell1, ell2)\end{verbatim}
    \begin{par}
\textbf{Input arguments:}
\end{par} \vspace{1em}
\begin{verbatim}m       mean of full state                                    [6 x 1]
s       covariance of full state                              [6 x 6]
ell1    length of inner pendulum
ell2    length of outer pendulum\end{verbatim}
\begin{verbatim}Note: this code assumes that the following order of the state:
       1: cart pos.,
       2: cart vel.,
       3: pend1 angular velocity,
       4: pend2 angular velocity,
       5: pend1 angle,
       6: pend2 angle\end{verbatim}
\begin{par}
\textbf{Output arguments:}
\end{par} \vspace{1em}
\begin{verbatim}M1      mean of tip of inner pendulum                         [2 x 1]
S1      covariance of tip of inner pendulum                   [2 x 2]
M2      mean of tip of outer pendulum                         [2 x 1]
S2      covariance of tip of outer pendulum                   [2 x 2]\end{verbatim}
\begin{par}
Copyright (C) 2008-2013 by Marc Deisenroth, Andrew McHutchon, Joe Hall, and Carl Edward Rasmussen.
\end{par} \vspace{1em}
\begin{par}
Last modification: 2013-03-06
\end{par} \vspace{1em}

\subsection*{High-Level Steps}

\begin{enumerate}
\setlength{\itemsep}{-1ex}
   \item Augment input distribution to complex angle representation
   \item Compute means of tips of pendulums (in Cartesian coordinates)
   \item Compute covariances of tips of pendulums (in Cartesian coordinates)
\end{enumerate}

\begin{lstlisting}
function [M1, S1, M2, S2] = getPlotDistr_cdp(m, s, ell1, ell2)
\end{lstlisting}

\subsection*{Code}

\begin{lstlisting}
% 1. Augment input distribution (complex representation)
[m1 s1 c1] = gTrig(m, s, [5 6], [ell1, ell2]); % map input through sin/cos
m1 = [m; m1];        % mean of joint
c1 = s*c1;           % cross-covariance between input and prediction
s1 = [s c1; c1' s1]; % covariance of joint

% 2. Mean of the tips of the pendulums (Cart. coord.)
M1 = [m1(1) - m1(7); m1(8)];     % p2: E[x -l1\sin\theta_2]; E[l2\cos\theta_2]
M2 = [M1(1) - m1(9); M1(2) + m1(10)];               % p3: mean of cart. coord.

% 2. Put covariance matrices together (Cart. coord.)
% first set of coordinates (tip of 1st pendulum)
S1(1,1) = s1(1,1) + s1(7,7) -2*s1(1,7);
S1(2,2) = s1(8,8);
S1(1,2) = s1(1,8) - s1(7,8);
S1(2,1) = S1(1,2)';

% second set of coordinates (tip of 2nd pendulum)
S2(1,1) = S1(1,1) + s1(9,9)  + 2*(s1(1,9) - s1(7,9));
S2(2,2) = s1(8,8) + s1(10,10) + 2*s1(8,10);
S2(1,2) = s1(1,8) - s1(7,8) - s1(9,8) ...
        + s1(1,10) - s1(7,10) - s1(9,10);
S2(2,1) = S2(1,2)';

% make sure we have proper covariances (sometimes numerical problems occur)
try
  chol(S1);
catch
  warning('matrix S1 not pos.def. (getPlotDistr)');
  S1 = S1 + (1e-6 - min(eig(S1)))*eye(2);
end

try
  chol(S2);
catch
  warning('matrix S2 not pos.def. (getPlotDistr)');
  S2 = S2 + (1e-6 - min(eig(S2)))*eye(2);
end
\end{lstlisting}


\input{preamble}

\begin{document}
    \section{Subjects}
    \begin{itemize}
        \item Basic algorithms and applications
        \item Building models and selecting model parameters
    \end{itemize}

    \section{Notes}

    \subsection{Motivation for HMM}

    Our predictions are based on models of observed data, a simple model is
    that observations are assumed to be independent and identically distributed.

    \begin{tikzpicture}
    \node[circlebox] (x1) {$x_1$};
    \node[circlebox, right=of x1] (x2) {$x_2$};
    \node[circlebox, right=of x2] (x3) {$x_3$};
    \node[circlebox, right=of x3] (x4) {$x_4$};
    \node[textbox, right=of x4] (dots) {$\cdots$};
    \end{tikzpicture}

    But this assumption is not always the best, an example is measurements of
    weather patterns, daily value of stocks etc. In these cases, a model that
    shows how each observation depends on the previous observations might be
    better.

    \begin{equation*}
        p(x_n|x_1,\dots.x_{n-1})=p(x_n|x_{n-1})
    \end{equation*}
     This can be depicted as:

    \begin{tikzpicture}
    \node[circlebox] (x1) {$x_1$};
    \node[circlebox, right=of x1] (x2) {$x_2$};
    \node[circlebox, right=of x2] (x3) {$x_3$};
    \node[circlebox, right=of x3] (x4) {$x_4$};
    \node[textbox, right=of x4] (dots) {$\cdots$};

    \path[edgepath]
        (x1) edge node {} (x2)
        (x2) edge node {} (x3)
        (x3) edge node {} (x4)
        (x4) edge node {} (dots);
    \end{tikzpicture}

    This chain of observations is a 1st-order Markov chain. The joint
    probability of observing some sequence of $N$ observations is then:
    \begin{equation*}
        p(x_1,\dots,x_N)=\prod_{n=1}^{N}p(x_n|x_1,\dots,x_{n-1}) =
        p(x_1)\prod_{n=2}^{N}p(x_n|x_{n-1})
    \end{equation*}

    For example, suppose the weather has two states, sun and rain. It might be
    that if the weather is sunny, then with probability $\frac{6}{7}$ it will
    stay sunny and $\frac{1}{7}$ it will start to rain however if it's already
    raining then the probability that it will keep raining is not $\frac{1}{7}$
    but instead it might be $\frac{2}{3}$, with just $\frac{1}{3}$ chance it
    will turn sunny. So the probability of some observation, depends on the
    last observation, or ``what state are we currently in''.

    Now suppose the state that influences the weather is not necessarily what
    the last observation was, but rather some other state that is hidden to us?
    For example wether there is a high or low pressure. If the hidden variables
    are discrete and form a Markov chain, then we can model it as a hidden
    Markov model (HMM).

    \begin{tikzpicture}
    \node[circlebox] (z1) {$z_1$};
    \node[circlebox, right=of z1] (z2) {$z_2$};
    \node[circlebox, right=of z2] (z3) {$z_3$};
    \node[circlebox, right=of z3] (z4) {$z_4$};
    \node[textbox, right=of z4] (dots) {$\cdots$};
    \node[circlebox, below=of z1] (x1) {$x_1$};
    \node[circlebox, below=of z2] (x2) {$x_2$};
    \node[circlebox, below=of z3] (x3) {$x_3$};
    \node[circlebox, below=of z4] (x4) {$x_4$};

    \path[edgepath]
    (z1) edge node {} (z2)
    (z2) edge node {} (z3)
    (z3) edge node {} (z4)
    (z4) edge node {} (dots)
    (z1) edge node {} (x1)
    (z2) edge node {} (x2)
    (z3) edge node {} (x3)
    (z4) edge node {} (x4);
    \end{tikzpicture}

    The joint distribution is then:

    \begin{equation*}
        p(x_1,\dots,x_N,\,z_1,\dots,z_N) =
        p(z_1)\left[\prod_{n=2}^{N}p(z_n|z_{n-1})\right]
        \prod_{n=1}^{N}p(x_n|z_n)
    \end{equation*}
    Then here we can see that
    $p(z_1)\left[\prod_{n=2}^{N}p(z_n|z_{n-1})\right]$ models the probability
    that we are in state $z_n$ given states $z_1,\dots,z_{n-1}$. And the
    probability $\prod_{n=1}^{N}p(x_n|z_n)$ is the probability that, given we
    are in some state $z_n$, we make the observation $x_n$.

    \subsubsection{Modeling transmission probabilities}
    Notation: we write the hidden variables as positional vectors, i.e. if
    $z_n=(0,0,1)$ then the model in step $n$ is in state $k=3$.

    Given that the hidden variables had to be discrete in order to model them
    with a HMM, we know that the have $K$ states. Since we know that the amount
    of states are limited, we can model the transmission probabilities as a
    $K\times K$ table called $A$. And we can describe the initial state given
    by the probability distribution $p(z_1)$ as a $K$ vector $\pi$. We can
    describe this formally as:

    \begin{equation*}
        A_{jk} = p(z_{nk}=1|z_{n-1,j}=1), \quad \pi_k = p(z_{1k}=1)
    \end{equation*}
    We then have that:
    \begin{equation*}
        \sum_{k}A_{jk}=1,\quad \sum_k \pi_k=1
    \end{equation*}
    We can then write the math cleverly like this:
    \begin{equation*}
        p(z_n|z_{n-1},A)=\prod_{k=1}^{K}\prod_{j=1}^{K}A_{jk}^{z_{n-1,j}z_{nk}},
        \quad p(z_1|\pi)=\prod_{k=1}^{K}\pi_k^{z_{1k}}
    \end{equation*}
    Now, this may look confusing. but remember that the hidden variables are
    positional vectors. That is:
    \begin{equation*}
        z_1=\begin{pmatrix}
            0\\
            \vdots\\
            0\\
            1\\
            0\\
            \vdots\\
            0
        \end{pmatrix}
    \end{equation*}
    So when we say $\pi_k^{z_{1k}}$ then $z_{1k}$ will be equal to $0$ for all
    entries except one where it is $1$. Thus we will have a product of $\pi_0^0
    \cdots \pi_{i-1}^0 \cdot \pi_{i}^1 \cdot \pi_{i+1}^0 \cdots \pi_K^0$. So it
    is really just a way of extracting a single value out of $\pi$ and $A$.

    \subsubsection{Modeling emission probabilities}
    Similar to the transmission probabilities, if we assume that the observed
    values $x_n$ are discrete (e.g. $D$ different symbols), then the emission
    probabilities $\phi$ can be modeled as a $K\times D$ table of
    probabilities, which for each state $K$ specifies the probability of
    observing some symbol.

    \begin{equation*}
        p(x_n|z_n,\phi)=\prod_{k=1}^K p(x_n|\phi_k)^{z_{nk}}
        = \prod_{k=1}^K \prod_{d=1}^{D} \phi_{kd}^{x_{nd},z_{nk}}
    \end{equation*}

    We then get the joint probability distribution of the HMM:

    \begin{equation*}
        p(X,Z|\Theta) =
        p(z_1|\pi)\left[\prod_{n=2}^{N}p(z_n|z_{n-1},A)\right]
        \prod_{n=1}^{N}p(x_n|z_n,\phi)
    \end{equation*}

    With our observables being: $X=\{x_1,\dots,x_N\}$ the latent states being:
    $Z=\{z_1,\dots,z_N\}$ and our model parameters: $\Theta=\{\pi, A, \phi\}$
    \\
    The matrix $A$ looks like this:\\
    \\
    \includegraphics[width=7cm]{Figures/trans_prob_A.pdf}

    Where: $A_{jk}=p(z_{nk}=1|z_{n-1,j}=1)=\text{ probability of going from $j$
    to $k$}$

    The matrix $\pi$ looks like this:\\
    \\
    \includegraphics[width=4cm]{Figures/init_prob_PI.pdf}

    Where: $\pi_k=p(z_{nk}=1)$.

    The matrix $\phi$ looks like this:\\
    \\
    \includegraphics[width=7cm]{Figures/emis_prob_PHI.pdf}

    Where: $\phi_{kd}=p(x_{nd}=1|z_{nk}=1)=$ probability of seeing observation
    $d$ in state $k$.

    \subsubsection{HMM as a generative model}
    There are two goals we might have when we are using HMMs, first is to
    determine the likelihood of a sequence of observations. The second is to
    find a plausible underlying explanation (or decoding) of a sequence of
    observations.

    Determining the likelihood of a sequence of observations are computed like
    this:
    \begin{equation*}
        p(X|\Theta) = \sum_Z p(X,Z|\Theta)
    \end{equation*}
    This sum has $K^N$ terms, but it turns out that it can actually be computed
    in \bigO{K^2N} time. But let's look at decoding first.

    \subsubsection{Decoding using HMMs}
    Given a HMM $\Theta$ and a sequence of observations $X=x_1,\dots,x_N$, then
    find a plausible explanation $Z^*=z^*_1,\dots,z^*_N$ of values of the
    hidden variable.
    We will look at two types of decoding here, Viterbi deconding and Posterior
    decoding.

    \begin{description}
        \item[Viterbi decoding] $Z^*$ is the overall most likely explanation of
        $X$:
        \begin{equation*}
            Z^* = \arg\max_Z p(X,Z|\Theta)
        \end{equation*}
        \item[Posterior decoding] $z^*_n$ is the most likely state to be in the
        $n$'th step:
        \begin{equation*}
            z^*_n=\arg\max_{z_n} p(z_n|x_1,\dots,x_N)
        \end{equation*}
    \end{description}

    \subsubsection{Viterbi decoding}
    Given $X$, find $Z^*$ such that: $Z^*=\arg\max_Z p(X,Z|\Theta)$

    We can write the probability of $X$ given the optimal explanation $Z^*$ as
    follows:

    \begin{align*}
        p(X,Z^*) &= \max_Z p(X,Z)\\
            &= \max_{z_1,\dots,z_N} p(x_1,\dots,x_N,\, z_1,\dots,z_N)\\
            &= \max_{z_N}\max_{z_1,\dots,z_{N-1}} p(x_1,\dots,x_N, \, z_1,
            \dots, z_N)\\
            &= \max_{z_N}\omega(z_N)\\
            z^*_N &= \arg\max_{z_N} \omega(z_N)
    \end{align*}

    Here, $\omega(z_n)=\max\limits_{z_1,\dots,z_{n-1}}
    p(x_1,\dots,x_n,\,z_1,\dots,z_n)$ is the probability of the most likely
    sequence of states $z_1,\dots,z_n$ ending in $z_n$ generating the
    observations $x_1,\dots,x_n$.

    We can expand $\omega(z_n)$ as follows:
    \begin{align*}
        \omega(z_n)&=\max_{z_1,\dots,z_{n-1}}p(x_1,\dots,x_n,\,z_1,\dots,z_n)\\
            &=\max_{z_1,\dots,z_{n-1}} p(z_1) \prod_{i=2}^n p(z_i|z_{i-1})
            \prod_{i=1}^{n}p(x_i|z_i)\\
            &=p(x_n|z_n)\max_{z_1,\dots,z_{n-1}}
            p(z_1)\prod_{i=2}^{n}p(z_i|z_{i-1}) \prod_{i=1}^{n-1}p(x_i|z_i)\\
            &\vdots\\
            &=p(x_n|z_n) \max_{z_{n-1}} p(z_n|z_{n-1})\omega(z_{n-1})
    \end{align*}
    So the $\omega$ is recursive, we can sum up $\omega$ as follows:
    \begin{description}
        \item[Recursion]
        \begin{equation*}
            \omega(z_n)=p(x_n|z_n) \max_{z_{n-1}} \omega(z_{n-1})p(z_n|z_{n-1})
        \end{equation*}
        \item[Basis]
        \begin{equation*}
            \omega(z_1)=p(x_1,z_1)=p(z_1)p(x_1|z_1)
        \end{equation*}
    \end{description}
    We can describe the $\omega$ function as a $N\times K$ matrix,
    $\omega[k][n]=\omega(z_n)$ if $z_n$ is state $k$, and construct it as
    follows:
    \begin{enumerate}
        \item $\omega[k][n] = 0$
        \item if $p(x[n]|k) \neq 0:$
        \begin{enumerate}
            \item for $j=1$ to $K$
            \begin{enumerate}
                \item if $p(k|j) \neq 0$
                \begin{enumerate}
                    \item $\omega[k][n]=\max(\omega[k][n], p(x[n]|k) \cdot
                    \omega[j][n-1] \cdot p(k|j)$
                \end{enumerate}
            \end{enumerate}
        \end{enumerate}
    \end{enumerate}
    We can then compute $\omega$ in \bigO{K^2N} time, using \bigO{KN} space.
    Now that we have $\omega(z_n)$, which is the probability of the most likely
    sequence of states, that ends in $z_n$, we can find $Z^*$ by backtracking :
    \begin{align*}
        z^*_N &= \arg\max_{z_N}\omega(z_N) = \arg\max_{z_N}\max_{z_{N-1}}
        \left(p(x_N|z_N)\omega(z_{n-1})p(z_N|z_{N-1})\right)\\
        z^*_{N-1} &=
        \arg\max_{z_{N-1}}\left(p(x_N|z^*_N)\omega(z_{N-1}p(z^*_N|z_{N-1})\right)\\
        z^*_{N-2} &=
        \arg\max_{z_{N-2}}\left(p(x_{N-1}|z^*_{N-1})\omega(z_{N-2}p(z^*_{N-1}|z_{N-2})\right)\\
    \end{align*}
    So we obviously have all that we need, we can just backtrack as follows:
    \begin{enumerate}
        \item $z[1\dots N] = \text{undef}$
        \item $z[N]=\arg\max_k \omega[k][N]$
        \item $n=N-1$ to $1$:
        \begin{enumerate}
            \item
            \begin{equation*}
                z[n]=\arg\max_k\left(p(x[n+1]|z[n+1]) \cdot \omega[k][n] \cdot
                p(z[n+1]|k)\right)
            \end{equation*}
        \end{enumerate}
        \item print $z[1\dots N]$
    \end{enumerate}
    We can backtrack in time \bigO{KN} using space \bigO{KN} using $\omega$.

    We could also use ``posterior decoding'', but it only works if there is a
    transition from any state to any other state. In other cases it might not
    return syntactically correct results.

    \subsubsection{Computing the likelihood of a sequence of observations}
    Apart from the $\omega$ recursion, we have the forward and backward
    algorithms:
    \begin{description}
        \item[Forward algorithm] Computes $\alpha(z_n)$ which is the joint
        probability of observing $x_1,\dots,x_n$ and being in state $z_n$:
        \begin{equation*}
            \alpha(z_n)=p(x_1,\dots,x_n,z_n)
        \end{equation*}
        \item[Backward algorithm] Computes $\beta(z_n)$ which is the
        conditional probability of observing $x_{n+1},\dots,x_N$ in the future,
        assuming we are in state $z_n$
        \begin{equation*}
            \beta(z_n)=p(x_{n+1},\dots,x_N|z_n)
        \end{equation*}
    \end{description}
    Then, having $\alpha(z_n)$ and $\beta(z_n)$, we can get the likelihood of
    the observations as:
    \begin{equation*}
        p(X)=\sum_{z_n}\alpha(z_n)\beta(z_n), \quad p(X)\sum_{z_N}\alpha(z_N)
    \end{equation*}
    We can compute $\alpha$ as:
    \begin{description}
        \item[Recursion]
        \begin{equation*}
            \alpha(z_n)=p(x_n|z_n) \sum_{z_{n-1}} \alpha(z_{n-1})p(z_n|z_{n-1})
        \end{equation*}
        \item[Basis]
        \begin{equation*}
            \alpha(z_1)=p(x_1,z_1)=p(z_1)p(x_1|z_1)
        \end{equation*}
    \end{description}
    And we can compute $\beta$ as:
    \begin{description}
        \item[Recursion]
        \begin{equation*}
        \beta(z_n)=\sum_{z_{n+1}}
        \beta(z_{n+1})p(x_{n+1}|z_{n+1})p(z_{n+1}|z_{n})
        \end{equation*}
        \item[Basis]
        \begin{equation*}
        \beta(z_N)=1
        \end{equation*}
    \end{description}

    \subsubsection{Training HMMs}

\end{document}


\subsection{Undirected graphs}\label{subsec:undirected_graphs}

\begin{remark}\label{rem:graph_etymology}
  Unfortunately, the term \enquote{graph} has at least several distinct established meanings:
  \begin{itemize}
    \item The \hyperref[def:multi_valued_function/graph]{graph of a valued function} (or relation).
    \item An \hyperref[def:undirected_multigraph]{undirected graph}, with or without loops or parallel edges.
    \item A \hyperref[def:quiver]{directed graph}, which may be a simple directed graph or a quiver.
  \end{itemize}

  The graph of a function is fundamentally different from the others, which we call \enquote{combinatorial graphs}. Although in \fullref{def:undirected_multigraph} we define undirected graphs as functions, the concepts are distinct and the formalisms are related by accident.
\end{remark}

\begin{definition}\label{def:undirected_multigraph}
  An \term{undirected multigraph} is a \hyperref[def:hypergraph]{hypergraph} \( G = (V, E, \mscrE) \) in which all hyperedges are either edges or loops, i.e. every hyperedge \( e \) has either one or two endpoints. We will use the term \enquote{edge} to include loops, although this in general varies with the author.

  \Cref{fig:def:undirected_multigraph} illustrates the concepts from this definition. The figure is not merely illustrative --- it is a \hyperref[def:quiver_geometric_realization/embedding]{graph embedding}.

  \begin{figure}[h]
    \begin{equation}\label{eq:fig:def:undirected_multigraph}
      \begin{aligned}
        \includegraphics[page=1]{output/def__undirected_multigraph.pdf}
      \end{aligned}
    \end{equation}
    \caption{An undirected multigraph, which becomes simple after removing the dashed edges.}\label{fig:def:undirected_multigraph}
  \end{figure}

  \begin{thmenum}
    \thmitem{def:undirected_multigraph/simple} A common assumption is that an undirected multigraph has no loops and no parallel edges. In this case, an established terminology is \term{simple undirected graph}.

    For a simple undirected graph, we often write \( G = (V, E) \), where \( E \) is a set of unordered pairs \( \set{ u, v } \) of distinct vertices. We can also regard simple undirected graphs as symmetric directed graphs --- see \fullref{rem:symmetric_directed_graphs}.

    \thmitem{def:undirected_multigraph/submodel} The \term{subgraphs} of \( G \) are the precisely \hyperref[def:hypergraph/submodel]{sub-hypergraphs} of \( G \). In particular, the \term{full subgraphs} are the \hyperref[eq:def:hypergraph/submodel/full]{full sub-hypergraphs}, that are maximal among the subgraphs with the same vertices.

    \thmitem{def:undirected_multigraph/categories} For a Grothendieck universe \( \mscrU \), we denote the \hyperref[def:category]{category} of undirected (multi)graphs by \( \ucat{MultGph} \) and \( \ucat{SimpGph} \), respectively. Both are strict \hyperref[def:subcategory]{subcategories} of the category \hyperref[def:category_of_small_hypergraphs]{\( \cat{HypGph} \)} of hypergraphs.
  \end{thmenum}
\end{definition}

\begin{example}\label{ex:complete_graph}
  For every set \( V \), the \term{complete graph} \( K_V \) is the \hyperref[def:undirected_multigraph]{simple undirected graph} with an edge between any pair of distinct vertices.

  \begin{figure}
    \begin{equation}\label{eq:fig:ex:complete_graph}
      \begin{aligned}
        \includegraphics[page=1]{output/ex__complete_graph.pdf}
      \end{aligned}
    \end{equation}
    \caption{The complete graph \( K_4 \).}\label{fig:ex:complete_graph}
  \end{figure}

  \Fullref{thm:ordinal_is_set_of_smaller_ordinals} justifies using the notation \( K_n \) for complete graphs of \hyperref[def:hypergraph/order]{finite order}.
\end{example}

\begin{definition}\label{def:graph_adjacency_matrix}\mcite[ch. 1, sec. 2.3]{GondranMinoux1984Graphs}
  Let \( G = (V, E, \mscrE) \) be a \hyperref[def:undirected_multigraph]{simple undirected graph}. Its \term{adjacency matrix}
  \begin{equation*}
    M = \seq{ a_{uv} }_{u, v \in V}
  \end{equation*}
  has elements
  \begin{equation*}
    M_{uv} \coloneqq \begin{cases}
      1, &u \T{is adjacent to} v \\
      0, &\T{otherwise.}
    \end{cases}
  \end{equation*}
\end{definition}

\begin{example}\label{ex:def:graph_adjacency_matrix}
  The \hyperref[def:graph_adjacency_matrix]{adjacency matrix} of the complete graph \( K_4 \) (\cref{fig:ex:complete_graph}) is
  \begin{equation}\label{eq:ex:def:graph_adjacency_matrix}
    \begin{blockarray}{ccccc}
        & a & b & c & d \\
    \begin{block}{c(cccc)}
      a & 0 & 1 & 1 & 1 \\
      b & 1 & 0 & 1 & 1 \\
      c & 1 & 1 & 0 & 1 \\
      d & 1 & 1 & 1 & 0 \\
    \end{block}
    \end{blockarray}
  \end{equation}
\end{example}

\begin{definition}\label{def:undirected_multigraph_path}
  Let \( G = (V, E, \mscrE) \) be an \hyperref[def:undirected_multigraph]{undirected multigraph}. We will adapt the definition of a \term{directed path} from \fullref{def:quiver_path}. Different authors use similar terms like \term{walk} and \term{trail} and put different restrictions on them. We will not put as few restrictions as possible.

  \begin{figure}
    \begin{equation}\label{eq:fig:def:undirected_multigraph_path}
      \begin{aligned}
        \includegraphics[page=1]{output/def__undirected_multigraph_path.pdf}
      \end{aligned}
    \end{equation}
    \caption{The dashed solid lines indicate the path with head \( a \) and edges \( (e_2, e_4, e_6) \) in the graph \eqref{eq:fig:def:undirected_multigraph}.}\label{fig:def:undirected_multigraph_path}
  \end{figure}

  Analogously to \hyperref[def:quiver_path]{quiver path}, consider a sequence \( p = (v, e_1, e_2, \ldots) \), where \( e_1, e_2, \ldots \) are edges and \( v \) is a vertex incident to \( e_1 \).

  Unlike arcs in quivers, however, edges have no head and tail. Fortunately, we can define them via \hyperref[thm:structural_recursion]{mutual recursion}, with the caveat that every edge may have a different head and tail with respect to another path:
  \begin{itemize}
    \item If \( e_k \) is a loop at \( u \), then \( u \) is both the head and the tail of \( e_k \).
    \item If \( e_1 \) is not a loop, its head is \( v \) and the tail is the other endpoint of \( e_1 \)
    \item If \( e_k \neq e_1 \) is not a loop, if \( e_{k-1} \) has a tail \( t(e_{k-1}) \) and if \( t(e_{k-1}) \) is an endpoint of \( e_k \), we define the head of \( e_k \) to be \( t(e_{k-1}) \) and the tail of \( e_k \) is the other endpoint of \( e_k \).
  \end{itemize}

  If the head and tail is defined for every edge in \( p \), and if \( e_k \neq e_{k-1} \) but \( h(e_k) = t(e_{k-1}) \) for every index \( k \), we say that \( p \) is a \term{path}.

  Note that it is not sufficient for \( e_k \) and \( e_{k-1} \) to be adjacent. Consider the following scenario, in which \( (v, e_1, e_2, e_3) \) is not a path because the tail of \( e_2 \) is not incident to \( e_3 \):
  \begin{equation}\label{eq:def:undirected_multigraph_path/almost_directed}
    \begin{aligned}
      \includegraphics[page=2]{output/def__undirected_multigraph_path.pdf}
    \end{aligned}
  \end{equation}

  We introduce the following terminology:
  \begin{thmenum}
    \thmitem{def:undirected_multigraph_path/endpoints} The \term{head} of the entire path \( p = (v, e_1, \ldots, e_n) \) is \( h(p) \coloneqq v \) and the \term{tail} is \( t(p) \coloneqq t(e_n) \). Note that the tail is only defined for finite paths.

    \thmitem{def:undirected_multigraph_path/empty} The \term{empty path} at \( v \) is the path \( p = (v) \). We call it empty because it has no edges. It is not an empty sequence because we want it to have a head and tail. The primary motivation for different empty paths are the identity morphisms in the free category for a \hyperref[def:quiver]{quiver} defined in \fullref{def:quiver_free_category}.

    \thmitem{def:undirected_multigraph_path/length} The \term{length} \( \len(p) \) of a path \( p \) is simply the number of edges. Hence, empty paths have length zero.

    \thmitem{def:undirected_multigraph_path/domain} The \term{domain} \( \dom(p) \) of \( p \) is the set of all vertices that belong to at least one edge in \( p \). We say that the path \term{visits} each member of \( \dom(p) \).

    The domain of the path \eqref{eq:fig:def:undirected_multigraph_path} is the set \( \set{ e_2, e_4, e_6 } \).

    Note that
    \begin{equation}\label{eq:def:undirected_multigraph_path/length_and_domain}
      \card(\dom(p)) + 1 \leq \len(p)
    \end{equation}
    in general since a vertex can be an endpoint of many edges.

    \thmitem{def:undirected_multigraph_path/concatenation} Similarly to concatenation of words defined in \fullref{def:formal_language/concatenation}, the concatenation of the paths
    \begin{align*}
              p &= (v, e_1, \ldots, e_n) \\
      \T{and} q &= (t(p), e_{n+1}, e_{n+1}, \ldots)
    \end{align*}
    is simply the path
    \begin{equation*}
      p \cdot q \coloneqq (v, e_1, e_2, \ldots, e_n, e_{n+1}, e_{n+1}, \ldots).
    \end{equation*}

    Note that the head of \( q \) must be the tail of \( p \) and that \( p \) is necessarily finite.

    \thmitem{def:undirected_multigraph_path/subpath} We say that the \hyperref[def:sequence]{subsequence} \( q \) of \( p \) is a \term{subpath} of \( p \) if the edges of \( q \) are consecutive in \( p \), i.e. there exists some index \( n \geq 0 \) such that \( q_k = e_{n + k} \) for \( 0 < k < \len(q) \).

    In \eqref{eq:fig:def:undirected_multigraph_path}, the subsequence \( (a, e_2, e_4) \) is a subpath and \( (a, e_2, e_6) \) is not.

    \thmitem{def:undirected_multigraph_path/cycle} A nonempty path \( p \) is a \term{cycle} if \( h(p) = t(p) \). Cycles are also called \term{closed paths}.

    Consider the graph \eqref{eq:fig:def:undirected_multigraph_path}. The path \( (a, e_2, e_4, e_3, e_1) \) is a cycle and so is and so is \( (a, e_2, \widehat{e_2}) \). We have forbidden for the same edge to appear twice consecutively, which means that \( (a, e_2, e_2) \) is not a cycle because it is not a valid path.

    A graph without cycles is called \term{acyclic}.

    \thmitem{def:undirected_multigraph_path/simple} The path \( p \) is \term{simple} if no subpath is a cycle.

    An alternative characterization is that every vertex in \( \dom(p) \) is visited only once, i.e. for every vertex in \( \dom(p) \), there exist at most two edges of the path which are incident to it.

    Another characterization is that equality holds in \eqref{eq:def:undirected_multigraph_path/length_and_domain}.

    \thmitem{def:undirected_multigraph_path/converse} If the chain \eqref{eq:fig:def:undirected_multigraph_path} is finite of length \( n \), we define its \term{converse} as
    \begin{equation*}
      p^{-1} \coloneqq \parens[\Big]{ t(p), e_n, e_{n-1}, \cdots, e_2, e_1 }.
    \end{equation*}

    \thmitem{def:undirected_multigraph_path/characteristic_vector} The \term{characteristic vector} \( \vect{p} \) of a path \( p \) is defined as the characteristic vector of \( \dom(p) \) in the \hyperref[def:hypergraph_vector_spaces/edge]{edge space} \( \BbbF_2^E \).
  \end{thmenum}
\end{definition}

\begin{definition}\label{def:undirected_multigraph_connectedness}
  Let \( G = (V, E, \mscrE) \) be an undirected multigraph.

  \begin{thmenum}
    \thmitem{def:undirected_multigraph_connectedness/reachability} We say that the vertices \( u \) and \( v \) are \term{connected} if there exists a path from \( u \) to \( v \).

    In relation to \fullref{def:quiver_condensation}, we also say that \( v \) is \term{reachable} from \( v \) and vice versa.

    \thmitem{def:undirected_multigraph_connectedness/components} Connectedness is obviously an equivalence relation in \( E \). Denote it by \( {\sim} \). For each vertex \( v \), the equivalence class \( [v] \) of all vertices reachable from \( v \) is called a \term{connected component}.

    The \term{connectivity number} of \( G \) is the cardinality \( \card(V / {\sim}) \) of the quotient set.

    If \( G \) has only one connected component, we say that it itself is \term{connected}. Equivalently, \( G \) is connected if every vertex is reachable from every other vertex.

    \thmitem{def:undirected_multigraph_connectedness/condensation} The \term{condensation} of \( G \) is the edgeless graph with vertices \( V / {\sim} \).

    This concept is useless for undirected graphs but important for \hyperref[def:quiver]{quivers} --- see \fullref{def:quiver_condensation}.
  \end{thmenum}

  Compare this definition to \fullref{def:quiver_connectedness}.
\end{definition}


\chapter{Quantum Electrodynamics}

The Lagrangian for quantum electrodynamics is
\begin{equation}
\begin{aligned}
	\mathcal{L}_{\mathrm{QED}}
	&= \bar\psi \left(i\gamma^\mu \partial_\mu - m\right)\psi - \frac{1}{4}F_{\mu\nu}F^{\mu\nu} -eA_\mu \bar\psi\gamma^\mu  \psi \\
	&= \mathcal{L}_{\mathrm{Dirac}}+\mathcal{L}_{\mathrm{Maxwell}}+\mathcal{L}_{\mathrm{int}},
\end{aligned}
\end{equation}
where
\begin{equation}
	F_{\mu\nu} = \partial_\mu A_\nu - \partial_\nu A_\nu = (dA)_{\mu\nu}.
\end{equation}
The Lagrangian is invariant under the gauge transformation:
\begin{equation}
\begin{aligned}
	\psi(x) &\rightarrow e^{-ie\alpha(x)}\psi(x), \\
	A^\mu(x) &\rightarrow A^\mu(x) + \partial^\mu \alpha(x).
\end{aligned}
\end{equation}
It is convenient to rewrite Lagrangian as
\begin{equation}
	\mathcal{L}_{\mathrm{QED}}
	= \bar\psi \left(i\cancel{D} - m\right)\psi - \frac{1}{4}F_{\mu\nu}F^{\mu\nu},
\end{equation}
where we have define the covariant derivative as:
\begin{equation}
	\cancel{D} = \gamma^\mu D_\mu
	= \gamma^\mu [\partial_\mu + i e A_\mu(x)]
	= \cancel{\partial} + ie \cancel{A}.
\end{equation}
As with the scalar field, the partition function with source is defined as
\begin{equation}
	Z[\bar\eta,\eta,J] = \exp\left\{i\int d^dx \mathcal{L}_{\mathrm{int}}\left[\frac{\delta}{i\delta J},\frac{\delta}{i\delta \eta},\frac{i\delta}{\delta \bar\eta}\right]\right\} Z_0[\bar\eta,\eta,J].
\end{equation}
In the following, we will use the dimensional regularization by default.
Note that the mass dimensionality for the fermion and gauge field is
\begin{equation*}
	[\psi] = \left[\frac{d-1}{2}\right],\quad
	[A] = \left[\frac{d}{2}-1\right],
\end{equation*}
which lead to the coupling constant $e$ to have the dimension
\begin{equation*}
	[e] = \left[2-\frac{d}{2} \right].
\end{equation*}
When $d=4-\epsilon$, we make the replacement
\begin{equation*}
	e \rightarrow e \tilde{\mu}^{\epsilon/2},
\end{equation*}
so that to make the coupling constant $e$ dimensionless.

\section{Perturbative Renormalization}

The interaction will renormalize the field and coefficients to
\begin{equation}
\begin{aligned}
	\psi &= \sqrt{Z_\psi} \psi_R, & m &= Z_m m_R, \\
	A &= \sqrt{Z_A} A_R, & e &= Z_e e_R.
\end{aligned}
\end{equation}
The renormalized Lagrangian is
\begin{equation}
\begin{aligned}
	\mathcal L
		&= Z_{\psi} \bar\psi_R(i\gamma^\mu \partial_\mu)\psi_R
		-Z_m Z_\psi m_R \bar\psi_R\psi_R
		+ Z_{A} \frac{1}{4} F_{R,\mu\nu}F_R^{\mu\nu} - Z_e Z_\psi \sqrt{Z_A} e_R A_R^\mu \bar\psi_R\gamma^\mu \psi_R \\
		&= \mathcal{L}_0 + \mathcal{L}_{\mathrm{int}} + \mathcal{L}_{\mathrm{ct}}.
\end{aligned}
\end{equation}
The we define the coefficients
\begin{equation}
	\delta_{\psi} = Z_\phi - 1,\quad
	\delta_{m} = Z_m - 1,\quad
	\delta_A = Z_A - 1, \quad
	\delta_1 = Z_e Z_\psi \sqrt{Z_A} - 1.
\end{equation}
The counter term also contribute to the perturbative expansion like the interactions.
The counter terms for the fermion propagator come from the diagram expression:
\begin{equation}
\begin{aligned}
	iG_F^{\mathrm{(ct)}}(p)
	&= \includegraphics[align=c, width=0.25\linewidth]{pics/QED-ct-1.png} \\
	&= G_F^{(0)}(p) \left[ \delta_{\psi}\cancel{p}-(\delta_m + \delta_\psi) m_R\right] G^{(0)}_F(p) .
\end{aligned}
\end{equation}
The contribution to the electron self energy is
\begin{equation}
	i\Sigma^{(\mathrm{ct})}(p) \equiv \delta_{\psi}\cancel{p}-(\delta_m + \delta_\psi) m_R.
\end{equation}

Similarly, the counter term contribution to the photon self energy is
\begin{equation}
\begin{aligned}
	i \Pi^{(
	\mathrm{ct})}_{\mu\nu}(k)
	&= \includegraphics[align=c,width=0.25\linewidth]{pics/QED-ct-2.png} \\
	&= \delta_A [-p^2 g_{\mu\nu} + (1-\xi)p_\mu p_\nu].
\end{aligned}
\end{equation}
In Landau gauge, $\xi=1$ and we can ignore the $p_\mu p_\nu$ terms, and the photon self-energy is always proportional to $g_{\mu\nu}$.
Whenever other gauge choices are concerned, we can simply replace $g_{\mu\nu}$ with
$g_{\mu\nu} - (1-\xi)\frac{p_\mu p_\nu}{p^2}$.

The counter term contribution to the QED vertex is
\begin{equation}
	\Gamma^{\mathrm{(ct)}\mu}_{\alpha\beta}
	= \includegraphics[align=c, width=0.2\linewidth]{pics/QED-ct-3.png}
	= -\delta_1 \gamma^{\mu}_{\alpha\beta}.
\end{equation}

\subsection{Vacuum Polarization}

Consider the one-loop correction to the photon propagator:
\begin{equation}
\begin{aligned}
	\includegraphics[align=c,width=0.15\linewidth]{pics/QED-2.png}
	&\simeq (-ie_R)^2\wick{
		A_\mu \c2{\bar\psi}_\alpha \gamma^\mu_{\alpha\beta} \c1{\psi}_\beta
		A_\nu \c1{\bar\psi}_\gamma \gamma^\nu_{\gamma\tau} \c2{\psi}_\tau
	} \\
	&\equiv iA_\mu \Pi^{\mu\nu}(p) A_\nu.
\end{aligned}
\end{equation}
The self energy is:
\begin{equation}
\begin{aligned}
	i\Pi^{\mu\nu}(p)
	&= -e_R^2 \int\frac{d^4 k}{(2\pi)^4} \mathrm{Tr}\left[\gamma^\mu G^{(0)}_F(k-p) \gamma^\nu G^{(0)}_F(k) \right] \\
	&= -e_R^2 \int\frac{d^4 k}{(2\pi)^4}
	\frac{\mathrm{Tr}\left[\gamma^\mu (\cancel k- \cancel p +m_R) \gamma^\nu (\cancel k +m) \right]}{(k^2-m_R^2)[(p-k)^2-m_R^2]}.
\end{aligned}
\end{equation}

The trace of the Dirac matrices can be evaluated in \texttt{Mathematic} using the \texttt{FeynCalc} package:

\begin{lstlisting}[style=mathematicaFrameTB]
(*Dirac trace using FeynCalc`*)
res=DiracTrace[GA[\[Mu]].(GS[k-p]+m).GA[\[Nu]].(GS[k]+m)];
DiracSimplify[res]
\end{lstlisting}

The Dirac trace is:
\begin{equation}
\begin{aligned}
	&\ \mathrm{Tr}\left[\gamma^\mu (\cancel k- \cancel p +m_R) \gamma^\nu (\cancel k +m_R) \right] \\
	=&\ 4 \left[g^{\mu\nu} \left(k\cdot p-{k}^2+m_R^2\right)+ 2k^\mu k^\nu - k^\mu p^\nu - p^\mu k^\nu \right].
\end{aligned}
\end{equation}
Using the Feynman parameters, the denominator is:
\begin{equation}
\begin{aligned}
	\frac{1}{(k^2-m_R^2)[(p-k)^2-m_R^2]}
	&= \frac{1}{\left\{[k-p(1-x)]^2-[m_R^2+p^2 x(x-1)]\right\}^2} \\
	&\equiv \frac{1}{\left\{[k-p(1-x)]^2 - D_x \right\}^2}.
\end{aligned}
\end{equation}
Since the Ward identity requires that the $p^\mu$ term in the propagator do not contribute to any scattering process, we then shift $k \rightarrow k + p(1-x)$ and drop all $p^\mu$ linear term.
The final result is simplified to:
\begin{equation}
\begin{aligned}
	i\Pi^{\mu \nu}(p)
	&= -4 e_R^{2} \int_0^1 dx
		\int \frac{d^{4} k}{(2 \pi)^{4}}  \frac{2 k^{\mu} k^{\nu}-g^{\mu \nu}\left[k^{2}-x(1-x) p^{2}-m_R^{2}\right]}{\left[k^{2}-D_x \right]^{2}} \\
	&\simeq 4e_R^2 g^{\mu\nu} \int_0^1 dx \int\frac{d^4 k}{(2 \pi)^{4}}
		\frac{\frac{1}{2}k^{2}-x(1-x) p^{2}-m_R^{2}}{\left[k^{2}-D_x \right]^{2}} \\
	&\simeq -ie_R^2 g^{\mu\nu} \int_0^1 dx\ \frac{\Omega_d \tilde{\mu}^\varepsilon}{(2\pi)^d} \int_0^\infty dk\ k^{d-1} \frac{\left(4-\frac{2}{d} \right) k^{2}+4x(1-x) p^{2}+4m_R^{2}}{\left[k^{2}+D_x\right]^{2}}.
\end{aligned}
\end{equation}
where we have made the Wick rotation, shifted the dimensionality to $(d=4-\varepsilon)$, and made the substitution (since the self-energy $i\Pi^{\mu\nu} \propto g^{\mu\nu}$):
\begin{equation}
	k^\mu k^\nu \rightarrow \frac{1}{d} k^2 g^{\mu\nu}.
\end{equation}

The remaining problem is to regularize and renormalize the divergent integral
\begin{equation}
	I_\varepsilon(x) \equiv \frac{\Omega_{4-\varepsilon} \tilde{\mu}^\varepsilon}{(2\pi)^{4-\varepsilon}} \int_0^\infty dk\ k^{3-\varepsilon} \frac{\left(4-\frac{8}{4-\varepsilon} \right) k^{2}+4x(1-x) p^{2}+4m_R^{2}}{\left[k^{2}+D_x\right]^{2}}.
\end{equation}

\subsubsection{Regularization and Renormalization}
In $(4-\varepsilon)$-dimensional Euclidean space, the integral is convergent.
The $\varepsilon$-expansion is carried out in \texttt{Mathematica} using the following code:

\begin{lstlisting}[style=mathematicaFrameTB]
omg = (2*Pi^(d/2))/(Gamma[d/2]);
cof = \[Mu]^(4-d)*omg/(2*Pi)^d;
nom = k^(d-1)*((4-8/(4-\[Epsilon]))k^2+4x*(1-x)p^2+4m^2);
int = cof*Integrate[nom/(k^2+D)^2,{k,0,Infinity}][[1]];
map = D->m^2-p^2*x*(1-x);
ans = Series[int/.{d->4-\[Epsilon]},{\[Epsilon],0,0}];
ans /. map // Simplify
\end{lstlisting}

The result is
\begin{equation}
	I_\varepsilon(x) = \frac{p^2 x(1-x)}{2\pi^2} \left[\frac{2}{\varepsilon}+\ln\left(\frac{4\pi e^{-\gamma_E} \tilde\mu^2}{m_R^2-p^2 x(1-x)}\right)\right]
\end{equation}
So the photon self-energy is (also denote $\mu^2=4\pi e^{-\gamma_E} \tilde\mu^2$):
\begin{equation}
	\Pi^{\mu\nu}(p)
	= -\frac{e_R^2 p^2 g^{\mu\nu}}{6\pi^2 \varepsilon}
	-\frac{e_R^2 p^2 g^{\mu\nu}}{2\pi^2} \int_0^1 dx\ x(1-x)
	\ln\left[\frac{\mu^2}{m_R^2-p^2 x(1-x)}\right]
\end{equation}
The counter term coefficient can be chosen as
\begin{equation}
	\delta_A = -\frac{e_R^2}{6\pi^2 \epsilon}.
\end{equation}
The renormalized photon self-energy is then
\begin{equation}
\begin{aligned}
	\Pi^{\mu\nu}(p)
	&= -\frac{e_R^2 p^2 g^{\mu\nu}}{2\pi^2} \int_0^1 dx\ x(1-x)
	\ln\left[\frac{\mu^2}{m_R^2-p^2 x(1-x)}\right] \\
	&= \frac{e_R^2 p^2 g^{\mu\nu}}{2\pi^2} \left\{\frac{1}{3}\ln\left(\frac{m_R}{\mu}\right)
	 + \int_0^{1} dx\ x(1-x) \ln\left[1-\frac{p^2 x(1-x)}{m_R^2}\right] \right\}.
\end{aligned}
\end{equation}

\subsubsection{Physical Observable}

The photon self-energy has the form
\begin{equation}
	\Pi^{\mu\nu}(p) = -e_R^2 \left[g^{\mu\nu}-(1-\xi)p^\mu p^\nu\right] g^{\mu\nu} \Pi_2(p),
\end{equation}
where
\begin{equation}
	\Pi_2(p) = \frac{1}{2\pi^2} \int_0^1 dx\ x(1-x)
	\ln\left[\frac{\mu^2}{m_R^2-p^2 x(1-x)}\right]
\end{equation}
The one-loop correction to photon propagator is
\begin{equation}
\begin{aligned}
	iG^{\mu\nu}_\gamma(p)
	&= -i \frac{g^{\mu\nu}}{p^2} \left(1 + \sum_{n=1}^\infty (-e_R^2)^n \Pi_2^n(p) \right) \\
	&= -i\frac{g^{\mu\nu}}{p^2 \left[1+e_R^2\Pi_2(p) \right]}.
\end{aligned}
\end{equation}

We can choose the on-shell condition that the photon has no rest mass:
\begin{equation}
	\Pi_2(0) = 0 \quad \Longrightarrow \quad
	\mu = m_R.
\end{equation}
Note that the propagator is related to the Coulomb potential.\footnote{The Coulomb potential arises just like we derive the force (\ref{eq:field-to-force}), but the sources have additional charge $e_R$, and the photon is mass less, so $V(p) = \frac{e_R^2}{p^2}$ for free field.
}
To the second order,
\begin{equation}
\begin{aligned}
	V(p) &= e_R^2 \frac{1-e_R^2 \Pi_2(p)}{p^2} + O(e_R^6) \\
	&= \frac{e_R^2}{p^2}\left\{1 + \frac{e_R^2}{2\pi^2}\int_0^1 dx\ x(1-x)\ln\left[1-\frac{p^2 x(1-x)}{m_R^2}\right] + O(e_R^4)\right\}.
\end{aligned}
\end{equation}

Consider the small momentum limit, where the integral in approximated by
\begin{equation}
\begin{aligned}
	\int_0^1 dx\ x(1-x)\ln\left[1-\frac{p^2 x(1-x)}{m_R^2}\right]
	\approx -\frac{p^2}{m_R^2} \int_0^1 dx\ x^2(1-x)^2 = -\frac{p^2}{30 m_R^2}.
\end{aligned}
\end{equation}
This implies
\begin{equation}
	V(p) = \frac{e_R^2}{p^2} - \frac{e_R^4}{60\pi^2 m_R^2}.
\end{equation}
The Fourier transformation gives
\begin{equation}
	V(r) = -\frac{e_R^2}{4\pi r} - \frac{e_R^4}{60\pi^2 m_R^2}\delta^{(3)}(r).
\end{equation}
For atomic orbit, since only the ($L=0$)-orbit have support at $r=0$, this extra potential will shift the spectrum.
This effect is called the \textit{Lamb shift}.

On the other hand, the the large momentum limit,
\begin{equation}
	V(p) \approx \frac{e_R^2}{p^2}\left[ 1 + \frac{e_R^2}{12\pi^2} \ln \frac{-p^2}{m_R^2} \right],
\end{equation}
which predicts a \textit{Landau pole} beyond which perturbation theory breaks down.

\subsection{One-loop Correction to Electron Propagator}

Consider the one-loop correction to the particle propagator:
\begin{equation}
	\includegraphics[width=0.2\linewidth,align=c]{pics/QED-1.png} \simeq (-ie_R)^2\wick{
		\c1{A}_\mu \bar\psi_\alpha \gamma^\mu_{\alpha\gamma} \c2{\psi}_\gamma
		\c1{A}_\nu \c2{\bar\psi}_\tau \gamma^\nu_{\tau\beta} \psi_\beta}
	\equiv i\bar\psi_\alpha \Sigma^{\alpha\beta}(p) \psi_\beta.
\end{equation}
The self energy is
\begin{equation}
\begin{aligned}
	i\Sigma_{\alpha\beta}(p)
	&= e_R^2\int\frac{d^4 k}{(2\pi)^4} G^{\mu\nu}_\gamma(p-k) \left[\gamma_\mu G_F(k) \gamma_\nu\right]_{\alpha\beta} \\
	&= -e_R^2\int\frac{d^d k}{(2\pi)^d}\frac{\gamma^\mu(\cancel{k}+m_R)\gamma_\mu}{(p-k)^2(k^2-m_R^2)}.
\end{aligned}
\end{equation}
The seconded equality comes from the contraction:
\begin{equation}
	(-ie)^2\wick{
		\c1{A}_\mu \bar\psi_\alpha \gamma^\mu_{\alpha\gamma} \c2{\psi}_\gamma
		\c1{A}_\nu \c2{\bar\psi}_\tau \gamma^\nu_{\tau\beta} \psi_\beta}
\end{equation}
The nominator can be simplified using the Dirac matrix identities:
\begin{equation}
	\gamma^\mu \gamma_\mu = d, \quad
	\gamma^\mu \gamma^\nu \gamma_\mu = (2-d) \gamma^\nu
	\quad \Longrightarrow \quad
	\gamma^\mu(\cancel{k}+m_R)\gamma_\mu = d m_R+(2-d)\cancel{k}.
\end{equation}
The denominator can be simplify using the Feynman parameter:
\begin{equation}
\begin{aligned}
	\frac{1}{(p-k)^2(k^2-m_R^2)}
	&= \int_0^1 \frac{dx}{\left[(k-px)^2-(1-x)(m_R^2-p^2 x)\right]^2} \\
	&\rightarrow \int_0^1 \frac{dx}{\left(k^2-D_x\right)^2}
\end{aligned}
\end{equation}
where we have shifted $k \rightarrow k + px$ (note this shift also change the numerator).

The self energy becomes (including a $\tilde \mu$ mass scale):
\begin{equation}
\begin{aligned}
	i\Sigma(p)
	&= e_R^2\tilde{\mu}^{\varepsilon}
		\int_0^1 [(2-\varepsilon)x\cancel{p}-(4-\varepsilon)m_R] dx
		\int \frac{d^d k}{(2\pi)^d}
		\frac{1}{(k^2-D_x)^2} \\
	&= ie_R^2 \int_0^1 dx\ [(2-\varepsilon)x\cancel{p}-(4-\varepsilon)m_R]\
		\frac{\tilde{\mu}^{\varepsilon}\Omega_d}{(2\pi)^d}
		\int \frac{k^{d-1} dk}{(k^2+D_x)^2}.
\end{aligned}
\end{equation}

\subsubsection{Regularization and Renormalization}

The regularization procedure is carried out by the following code:

\begin{lstlisting}[style=mathematicaFrameTB]
omg=(2*Pi^(d/2))/(Gamma[d/2]);
cof=\[Mu]^(4-d)*omg/(2*Pi)^d;
int=cof*Integrate[q^(d-1)/(q^2+D)^2,{q,0,Infinity}][[1]];
map={e->Sqrt[4*Pi*\[Alpha]],EulerGamma->Subscript[\[Gamma],E]};
ans=Series[int/.{d->4-\[Epsilon]},{\[Epsilon],0,0}];
ans/.map//Simplify
\end{lstlisting}

The result is ($\mu^2 = 4\pi e^{-\gamma_E} \tilde\mu^2$)
\begin{equation}
\begin{aligned}
	\Sigma(p)
	&= \frac{e_R^2}{16\pi^2}\int_0^1 dx \left[(2-\varepsilon)x\cancel{p}-(4-\varepsilon)m_R \right]
	\left[\frac{2}{\varepsilon}+\ln\frac{\mu^2}{(1-x)(m_R^2-p^2 x)}\right] \\
	&= \frac{e_R^2}{16\pi^2} \left\{ \int_0^1 dx \left[2x\cancel{p}-4m_R \right]
	\left[\frac{2}{\varepsilon}+\ln\frac{\mu^2}{(1-x)(m_R^2-p^2 x)}\right]-\cancel p + 2m_R\right\}.
\end{aligned}
\end{equation}
The infinity comes from
\begin{equation}
	\frac{e_R^2}{4\pi^2\varepsilon}\int_0^1 dx(x\cancel{p}-2m_R)
	= \frac{e_R^2}{8\pi^2\varepsilon}\cancel{p} - \frac{e_R^2}{2\pi^2\varepsilon}m_R.
\end{equation}
Using the $\overline{\mathrm{MS}}$ subtraction scheme, we choose
\begin{equation}
	\delta_{\psi} = -\frac{e_R^2}{8\pi^2\varepsilon},\quad
	\delta_m = -\frac{3 e_R^2}{8\pi^2\varepsilon},
\end{equation}
and the self energy is
\begin{equation}
	\Sigma(p)
	= \frac{e_R^2}{16\pi^2} \left\{\int_0^1 dx(2x\cancel{p}-4m_R)\ln\left[\frac{\mu^2}{(1-x)(m_R^2-p^2 x)}\right]-\cancel p + 2m_R \right\}.
\end{equation}

\subsubsection{Physical Observables}

The Dyson series gives:
\begin{equation}
	iG_F(p) = \frac{i}{\cancel p - m_R + \Sigma(p)}
\end{equation}
Experimentally, for a given
The on-shell subtraction requires that the $m_R$ equals to the physical mass:
\begin{equation}
	\left. \Sigma(\cancel p) \right|_{\cancel p=m_R} = 0, \quad
	\left. \frac{d}{d \cancel p}\Sigma(\cancel p) \right|_{\cancel p=m_R} = 0.
\end{equation}
To implement the on-shell condition, we have to modify the subtraction scheme to
\begin{equation}
\begin{aligned}
	\delta_{\psi} &= -\frac{e_R^2}{8\pi^2}\left(\frac{1}{\varepsilon}+\ln\frac{\mu}{m_R} +A\right), \\
	\delta_m &= -\frac{e_R^2}{8\pi^2}\left(\frac{3}{\varepsilon} + 3\ln\frac{\mu}{m_R} + B\right),
\end{aligned}
\end{equation}
and the self energy is
\begin{equation}
\begin{aligned}
	\Sigma(\cancel p)
	=&\ -\frac{e_R^2}{8\pi^2} \int_0^1 dx(x\cancel{p}-2m_R)\ln\left[(1-x)\left(1- \frac{p^2}{m_R^2} x \right)\right]\\
	&\  -\frac{e_R^2}{8\pi^2} \left[\left(A+\frac{1}{2}\right)\cancel p - (A+B+1)m_R\right].
\end{aligned}
\end{equation}
The first condition
\begin{equation}
\begin{aligned}
	\left. \Sigma(\cancel p) \right|_{\cancel p=m_R}
	&= -\frac{e_R^2}{8\pi^2} m_R \left[\int_0^1 dx\ (x-2) \ln(1-x)^2 - B-\frac{1}{2}\right] \\
	&= -\frac{e_R^2}{8\pi^2} m_R \left(2 - B\right) = 0
\end{aligned}
\end{equation}
gives the mass renormalization coefficient
\begin{equation}
	\delta_m = -\frac{e_R^2}{8\pi^2}\left(\frac{3}{\varepsilon} + 3\ln\frac{\mu}{m_R} + 2\right).
\end{equation}
While in the derivative of the self-energy:
\begin{equation}
	\frac{d}{d \cancel p}\Sigma(m_R)
	= -\frac{e_R^2}{8\pi^2} \left\{\int_0^1 dx\ \left[ x\ln(1-x)^2 -\frac{2x(x-2)}{1-x}\ln(1-x)\right] + A + \frac{1}{2}\right\},
\end{equation}
there is a divergent integral:
\begin{equation}
	\int_0^1 dx\ \frac{2x(x-2)}{1-x}\ln(1-x),
\end{equation}
indicating an IR divergence.
We can never the less get rid of it by introducing a small mass $m_\gamma$ for photon (which will be set to zero).
This mass term change the denominator in the loop integral:
\begin{equation}
	\frac{1}{\left[(p-k)^2-m_\gamma^2\right](k^2-m_R^2)}
	= \int_0^1 \frac{dx}{\left[(k-px)^2-D_x -x m_\gamma^2\right]^2}.
\end{equation}
Most derivation remains the same, we just need to make a substitution in the finial result:
\begin{equation}
	D_x \rightarrow D_x + x m_\gamma^2.
\end{equation}
Especially, the introducing of the photon mass will not change the result of the mass renormalization factor we have computed.

The modified self-energy is then
\begin{equation}
\begin{aligned}
	\Sigma(\cancel p)
	=&\ -\frac{e_R^2}{8\pi^2} \int_0^1 dx(x\cancel{p}-2m_R)\ln\left[(1-x)\left(1- \frac{p^2}{m_R^2} x \right) + x \frac{m_\gamma^2}{m_R^2} \right]\\
	&\  -\frac{e_R^2}{8\pi^2} \left[\left(A+\frac{1}{2}\right)\cancel p - (A+B+1)m_R\right].
\end{aligned}
\end{equation}
The derivative is now
\begin{equation}
	\frac{d}{d \cancel p}\Sigma(m_R)
	= -\frac{e_R^2}{8\pi^2} \left\{ \int_0^1 dx\ \left[ x\ln(1-x)^2 +\frac{2x(2-x)(1-x)}{(1-x)^2 + x \frac{m_\gamma^2}{m_R^2}} \right] + A + \frac{1}{2}\right\},
\end{equation}
Note that in the ($m_\gamma \rightarrow 0$) limit, the asymptotic behavior of the originally divergent integral is
\begin{equation}
	\lim_{m_\gamma \rightarrow 0} \int_0^1 dx\ \frac{2x(2-x)(1-x)}{(1-x)^2 + x \frac{m_\gamma^2}{m_R^2}}
	= -1 - 2\ln \frac{m_\gamma}{m_R}.
\end{equation}
So the second subtraction condition is:
\begin{equation}
	\frac{d}{d \cancel p}\Sigma(m_R)
	= -\frac{e_R^2}{8\pi^2} \left(A -2 - 2\frac{m_\gamma}{m_R} \right) = 0.
\end{equation}
The field strength renormalization is
\begin{equation}
	\delta_\psi = -\frac{e_R^2}{8\pi^2}\left(\frac{1}{\varepsilon}+\ln\frac{\mu}{m_R} +2+2\ln\frac{m_\gamma}{m_R}\right).
\end{equation}
The final self energy is (shall take the $m_\gamma \rightarrow 0$ limit):
\begin{equation}
\begin{aligned}
	\Sigma(\cancel p)
	=&\ -\frac{e_R^2}{16\pi^2} \int_0^1 dx(2x\cancel{p}-4m_R)\ln\left[(1-x)\left(1- \frac{p^2}{m_R^2} x \right) + 2x \ln\frac{m_\gamma}{m_R} \right]\\
	&\  -\frac{e_R^2}{16\pi^2} \left[\left(5+4\ln\frac{m_\gamma}{m_R}\right)\cancel p - \left(10 +4\ln\frac{m_\gamma}{m_R}\right)m_R\right].
\end{aligned}
\end{equation}

\subsection{One-loop Correction to Vertex}
Consider the one-loop correction to interaction:
\begin{equation}
\begin{aligned}
	\includegraphics[align=c,width=0.23\linewidth]{pics/QED-3.png}
	& \simeq (-ie_R)^3\wick{
		\c1{A}_\nu \bar\psi_\alpha \gamma^\nu_{\alpha\beta} \c2{\psi}_\beta
		A_\mu \c2{\bar\psi}_\lambda \gamma^\mu_{\lambda\tau} \c3{\psi}_\tau
		\c1{A}_\xi \c3{\bar\psi}_\rho \gamma^\nu_{\rho\sigma} \psi_\sigma
	} \\ &\equiv -ie A_\mu \Gamma^\mu_{\alpha\beta}(q,p,p') \bar\psi_\alpha \psi_\beta.
\end{aligned}
\end{equation}
The vertex function is:
\begin{equation}
\begin{aligned}
	i\Gamma^\mu_{\alpha\beta}(q,p,p')
	&= -e_R^2 \int\frac{d^4 k}{(2\pi)^4}
	G_\gamma^{\nu\lambda}(p-k)
	[\gamma_\nu G_F(k') \gamma^\mu G_F(k) \gamma_\lambda]_{\alpha\beta} \\
	&= e_R^2 \int\frac{d^4 k}{(2\pi)^4}
	\frac{[\gamma^\nu(\cancel{k}'+m_R)\gamma^\mu (\cancel{k}+m_R)\gamma_\nu]_{\alpha\beta}}{(k^2-m_R^2)(k'^2-m_R^2)(p-k)^2}
\end{aligned}
	\label{eq:QED-loop-vertex}
\end{equation}
Using the following code
\begin{lstlisting}[style=mathematicaFrameTB]
(*numerator*)
den=Contract[GA[\[Nu]].(GS[kp]+m).GA[\[Mu]].(GS[k]+m).GA[\[Nu]]];
DiracSimplify[den]

(*Feynman parameter*)
A1=k^2-m^2;
A2=(k+q)^2-m^2;
A3=(p-k)^2;
{c,b,a}=CoefficientList[x*A1+y*A2+z*A3,{k}];
-b/2//Simplify
-c+b^2/4//Simplify
\end{lstlisting}
The numerator is
\begin{equation}
	-2\cancel{k}\gamma^\mu\cancel{k}'-2m_R^2\gamma^\mu + 4m_R(k + k')^\mu.
\end{equation}
The denominator is
\begin{equation}
	\int \frac{dF_3}{[(k+yq-zp)^2-D_{xyz}]^3},
\end{equation}
where
\begin{equation*}
\begin{aligned}
	D_{xyz} &= (x+y)m^2 - z(1-z)p^2- y(1-y)q^2-2yzpq \\
	&=(x+y)m_R^2 - xy q^2- yz p'^2 - xz p^2.
\end{aligned}
\end{equation*}
Shift $k^\mu \rightarrow k^\mu + z q_1^\mu - y p^\mu$, throw away all terms with linear $k^\mu$, and replace $k^\mu k^\nu$ with $\frac{1}{d}k^2 g^{\mu\nu}$, the result is
\begin{equation*}
	\frac{4}{d}k^2\gamma^\mu  -2(-y \cancel{q}+z \cancel{p}) \gamma^{\mu}[(1-y) \cancel q+z \cancel p] +4 m_R^{2} \gamma^{\mu}-2 m_R\left[(1-2 y) q^{\mu}+2 z p^{\mu}\right].
\end{equation*}
Note that only the quadratic term is divergent.
\begin{equation*}
	\Gamma^\mu(p,q_1,q_2) = -i\frac{4e^2\tilde{\mu}^{\epsilon} \gamma^\mu}{d}  \int dF_3 \int \frac{d^dk}{(2\pi)^d} \frac{k^2}{(k^2-D)^3} + \delta\Gamma^\mu(p,q_1,q_2).
\end{equation*}
where $\delta \Gamma^\mu$ stores all the finite part
\begin{equation*}
\begin{aligned}
	& \delta\Gamma^\mu(p,q_1,q_2) \\
	=& \int \frac{e^2 k^3 dk dF_3}{(2\pi)^2(k^2+D)^3} \left\{(-y \cancel{q}+z \cancel{p}) \gamma^{\mu}[(1-y) \cancel q+z \cancel p] -2 m_R^{2} \gamma^{\mu}+ m_R\left[(1-2 y) q^{\mu}+2 z p^{\mu}\right]\right\}.
\end{aligned}
\end{equation*}
The divergent part is
\begin{equation}
	\frac{4 e^2\tilde{\mu}^{\epsilon} \Omega_d \gamma^\mu}{d(2\pi)^d}\int dF_3 \int \frac{k^{d+1}dk}{(k^2+D)^3}
	= \frac{e_R^2}{16\pi^2} \gamma^\mu \int dF_3 \left(\frac{2}{\epsilon}+\ln\frac{\mu^2}{D_{xyz}}\right).
\end{equation}
Using the $\overline{\mathrm{MS}}$ scheme, the coefficient of the counter term is
\begin{equation}
	\delta_e = -\frac{e_R^2}{8\pi^2\varepsilon}.
\end{equation}

\section{Systematic Renormalization}

\subsection{Renormalization Group}
In summery, the renormalization factors are
\begin{equation}
\begin{aligned}
	Z_\psi &= 1 -\frac{e_R^2}{8\pi^2\epsilon} + O(e_R^3), \\
	Z_A &= 1 - \frac{e_R^2}{6\pi^2 \epsilon} + O(e_R^3), \\
	Z_m &= 1 - \frac{e_R^2}{2\pi^2\epsilon} + O(e_R^3), \\
	Z_e &= 1 - \frac{e_R^2}{8\pi^2\epsilon} + O(e_R^3),
\end{aligned}
\end{equation}
which means
\begin{equation}
\begin{aligned}
	\frac{d\ln Z_\phi}{d e_R} &= -\frac{e_R}{4\pi^2 \epsilon} + O(e_R^2), \\
	\frac{d\ln Z_A}{d e_R} &= -\frac{e_R}{3\pi^2 \epsilon} + O(e_R^2), \\
	\frac{d\ln Z_m}{d e_R} &= -\frac{e_R}{\pi^2 \epsilon} + O(e_R^2), \\
	\frac{d\ln Z_e}{d e_R} &= -\frac{e_R}{4\pi^2 \epsilon} + O(e_R^2).
\end{aligned}
\end{equation}
The bare parameters are
\begin{equation}
\begin{aligned}
	\psi_0 &= Z_\psi^{1/2}\psi_R, \\
	A_0 &= Z_A^{1/2} A_R, \\
	m_0 &= Z_m Z_\psi^{-1} m_R, \\
	e_0 &= Z_e Z_\psi^{-1} Z_A^{-1/2} e_R \tilde{\mu}^{\epsilon/2}.
\end{aligned}
\end{equation}
The RG equation for $e_0$ is
\begin{equation}
	\frac{d\ln e_0}{d\ln \mu}
	= \left(\frac{\ln Z_e}{d e_R} - \frac{\ln Z_\psi}{d e_R} - \frac{1}{2} \frac{\ln Z_A}{d e_R} + \frac{1}{e_R} \right)\frac{de_R}{d\ln \mu} + \frac{\epsilon}{2} = 0.
\end{equation}
The beta function is
\begin{equation}
	\beta(e_R) = \frac{de_R}{d\ln \mu} = -\frac{\epsilon}{2}e_R + \frac{e_R^3}{12\pi^2} + O(e_R^4).
\end{equation}
The RG equation for $m_0$ is
\begin{equation}
	\frac{d\ln m_0}{d\ln \mu}
	= \left(\frac{\ln Z_m}{d e_R} - \frac{\ln Z_\psi}{d e_R}\right)\frac{de_R}{d\ln \mu} + + \frac{1}{m_R}\frac{d m_R}{d\ln\mu} = 0.
\end{equation}
The anomalous dimension of mass is
\begin{equation}
	\gamma_m = \frac{d \ln m_R}{d\ln\mu} = -\frac{3e_R^2}{8\pi^2} + O(e_R^3).
\end{equation}


\documentclass{article}

\usepackage[T1]{fontenc}
\usepackage[osf]{libertine}
\usepackage[scaled=0.8]{beramono}
\usepackage[margin=1.5in]{geometry}
\usepackage{url}
\usepackage{booktabs}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{nicefrac}
\usepackage{microtype}
\usepackage{subcaption}
\usepackage{bm}

\usepackage{sectsty}
\sectionfont{\large}
\subsectionfont{\normalsize}

\usepackage{titlesec}
\titlespacing{\section}{0pt}{10pt plus 2pt minus 2pt}{0pt plus 2pt minus 0pt}
\titlespacing{\subsection}{0pt}{5pt plus 2pt minus 2pt}{0pt plus 2pt minus 0pt}

\usepackage{pgfplots}
\pgfplotsset{
  compat=newest,
  plot coordinates/math parser=false,
  tick label style={font=\footnotesize, /pgf/number format/fixed},
  label style={font=\small},
  legend style={font=\small},
  every axis/.append style={
    tick align=outside,
    clip mode=individual,
    scaled ticks=false,
    thick,
    tick style={semithick, black}
  }
}

\pgfkeys{/pgf/number format/.cd, set thousands separator={\,}}

\usepgfplotslibrary{external}
\tikzexternalize[prefix=tikz/]

\newlength\figurewidth
\newlength\figureheight

\setlength{\figurewidth}{8cm}
\setlength{\figureheight}{6cm}

\newlength\squarefigurewidth
\newlength\squarefigureheight

\setlength{\squarefigurewidth}{4cm}
\setlength{\squarefigureheight}{4cm}

\newlength\smallsquarefigurewidth
\newlength\smallsquarefigureheight

\setlength{\smallsquarefigurewidth}{3.25cm}
\setlength{\smallsquarefigureheight}{3.25cm}

\newlength\smallfigurewidth
\newlength\smallfigureheight

\setlength{\smallfigurewidth}{6.25cm}
\setlength{\smallfigureheight}{4cm}

\setlength{\parindent}{0pt}
\setlength{\parskip}{1ex}

\newcommand{\acro}[1]{\textsc{\MakeLowercase{#1}}}
\newcommand{\given}{\mid}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\data}{\mc{D}}
\newcommand{\intd}[1]{\,\mathrm{d}{#1}}
\newcommand{\inv}{^{-1}}
\newcommand{\trans}{^\top}
\newcommand{\mat}[1]{\bm{\mathrm{#1}}}
\renewcommand{\vec}[1]{\bm{\mathrm{#1}}}
\newcommand{\R}{\mathbb{R}}
\renewcommand{\epsilon}{\varepsilon}

\DeclareMathOperator{\var}{var}
\DeclareMathOperator{\cov}{cov}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\argmax}{arg\,max}

\begin{document}

\section*{Linear regression}

We are ready to consider our first machine-learning problem:
\emph{linear regression.}  Suppose that we are interested in the
values of a function $y(\vec{x})\colon \R^d \to \R$, where $\vec{x}$ is
a $d$-dimensional vector-valued input.  We assume that we have made
some observations of this mapping, $\data = \bigl\{ (\vec{x}_i, y_i)
\bigr\}_{i = 1}^N$, to serve as training data.  Given these examples,
the goal of regression is to be able to predict the value of $y$ at a
new input location $\vec{x}_\ast$.  In other words, we want to learn
from $\data$ a (hopefully accurate!) prediction function (also called
a \emph{regression function}) $\hat{y}(\vec{x}_\ast)$.

Without any further restrictions on $\hat{y}$, the problem is not
well-posed.  In general, I could choose any function $\hat{y}$.  How
do I choose a good function?  In particular, how do I effectively use
the training observations $\data$ to guide this choice?

The space of all possible regression functions $\hat{y}$ is enormous;
to restrict this space and make learning more tractable, \emph{linear
  regression} assumes the relationship between $\vec{x}$ and $y$ is
``mostly'' linear:
\begin{equation}
  \label{linear_assumption}
  y(\vec{x}) = \vec{x}\trans \vec{w} + \epsilon(\vec{x}),
\end{equation}
where $\vec{w} \in \R^d$ is a vector of parameters, and
$\epsilon(\vec{x})$ represents a departure from the underlying linear
function $\vec{x}\trans \vec{w}$ at $\vec{x}$.  The value of
$\epsilon(\vec{x})$ is also called the \emph{residual.}  If the
underlying linear trend closely matches the data, we can expect small
residuals.

Note that this model does not explicitly contain an intercept term.
Instead, we normally add a constant feature to the input vector
$\vec{x}$ as in $\vec{x}' = [1, \vec{x}]\trans$.  In this case, the value
$w_1$ may be seen as an intercept term, allowing the hyperplane to
avoid passing through the origin at $\vec{x} = 0$.  To avoid messy
notation, we will not explicitly write the prime on the inputs and
rather assume this expansion has already been done.

In general, we can imagine applying any function to $\vec{x}$ to serve
as the inputs to our linear regression model.  For example, to
accomplish polynomial regression of a given degree $k$, we might take
$\vec{x}' = [1, \vec{x}, \vec{x}^2, \dots, \vec{x}^k]\trans$, where
exponentiation is pointwise.  A transformation of this type may be
summarized by a map $\vec{x} \mapsto \phi(\vec{x})$; this is known as
a \emph{basis expansion.}  With a nonlinear basis expansion, we can
learn nonlinear regression functions.  We assume in the following that
such an expansion, if desired, has already been applied to the inputs.

Let us collect the $N$ training examples into a matrix of training
inputs $\vec{X} \in \R^{N \times d}$ and a vector of associated outputs
$\vec{y} \in \R^N$.  Then our assumed linear relationship in
\eqref{linear_assumption}, when applied to the training data, may be
written
\begin{equation*}
  \vec{y} = \mat{X}\vec{w} + \vec{\epsilon},
\end{equation*}
where we have also collected the residuals into the vector
$\vec{\epsilon}$.

There are many classical approaches for estimating the parameter
vector $\vec{w}$; below, we describe two.

\subsection*{Ordinary least squares}

As mentioned previously, a ``good'' linear mapping $\vec{x}\trans
\vec{w}$ should result in small residuals.  Our goal is to minimize
the magnitude of the residuals at every possible test input
$\vec{x}_\ast$.  Of course, this is impossible, so we settle for the
next best thing: we minimize the magnitude of the residuals on our
training data.

The classical approach is to estimate $\vec{w}$ by minimizing the sum of
the squared residuals on the training data:
\begin{equation}
  \label{ols}
  \hat{\vec{w}}
  =
  \argmin_{\vec{w}}
  \sum_{i = 1}^N
  (\vec{x}_i\trans \vec{w} - y_i)^2.
\end{equation}
This approach is called \emph{ordinary least squares.}  The underlying
assumption is that the magnitude of the residuals are all independent
from each other, so that we can simply sum up the squared residuals
and minimize.  (If the residuals were instead correlated, we would
have to consider the interaction between pairs of residuals.  This is
also possible and gives rise to \emph{generalized least squares.})

It turns out that the minimization problem in \eqref{ols} has an
exact, closed-form solution:
\begin{equation*}
  \hat{\vec{w}} = (\mat{X}\trans\mat{X})\inv \mat{X}\trans \vec{y}.
\end{equation*}

To predict the value of $y$ at a new input $\vec{x}_\ast$, we plug our
estimate of the parameter vector into \eqref{linear_assumption}:
\begin{equation*}
  \hat{y}(\vec{x}_\ast)
  =
  \vec{x}_\ast\trans \hat{\vec{w}} =
  \vec{x}_\ast\trans (\mat{X}\trans\mat{X})\inv \mat{X}\trans \vec{y}
  .
\end{equation*}

Ordinary least squares is by far the most commonly applied linear
regression procedure.

\subsection*{Ridge regression}

Occasionally, the ordinary least squares approach can lead to
problems.  In particular, when the training data lie extremely close
to a particular hyperplane (or when the number of observations is less
than the dimension, and we may find a hyperplane intersecting the data
exactly), the magnitudes of the estimated parameters $\hat{\vec{w}}$
can become ill-behaved and extremely large.  This is an example of
\emph{overfitting.}

\emph{A priori,} we might reasonably expect or desire the values of
$\vec{w}$ to be relatively small.  For this reason, so-called
\emph{penalization} or \emph{regularization methods} modify the
objective in \eqref{ols} to avoid such pathologies.  The most common
approach is to add a term to the objective encouraging small entries
of $\vec{w}$, for example
\begin{equation}
  \label{ridge}
  \hat{\vec{w}}
  =
  \argmin_{\vec{w}}
  \sum_{i = 1}^N
  (\vec{x}_i\trans \vec{w} - y_i)^2
  +
  \lambda
  \lVert \vec{w} \rVert_{2}^2,
\end{equation}
where we have added the squared 2-norm of $\vec{w}$ to the function to
be minimized, thereby penalizing large-magnitude entries of $\vec{w}$.
The scalar $\lambda \geq 0$ serves to trade off the contributions of
the residual term and the penalty term, and is a parameter of the
model that we may tune as desired.  As $\lambda \to 0$, we recover the
ordinary least squares objective.

The minimization problem in \eqref{ridge} may also be solved exactly,
giving the slightly modified estimator
\begin{equation*}
  \hat{\vec{w}}
  =
  (\mat{X}\trans\mat{X} + \lambda \mat{I})\inv \mat{X}\trans \vec{y}.
\end{equation*}
Again, as $\lambda \to 0$ (corresponding to very small penalties on
the magnitude of $\vec{w}$ entries), we see that the resulting
estimator converges to the ordinary least squares estimator.  As
$\lambda \to \infty$ (corresponding to very large penalties on the
magnitude of $\vec{w}$ entries), the estimator converges to $\vec{0}$.

This approach is known as \emph{ridge regression,} named for the
additional term $\lambda \mat{I}$ in the estimator, which when viewed
from ``above'' can whimsically be described as resembling a mountain
ridge.

\subsection*{Bayesian linear regression}

Here we consider a Bayesian approach to linear regression. Consider
again the assumed underlying linear relationship \eqref{linear_assumption}:
\begin{equation*}
  y(\vec{x}) = \vec{x}\trans \vec{w} + \epsilon(\vec{x}).
\end{equation*}
Here the vector $\vec{w}$ serves as the unknown parameter of the
model, which we will reason about using the Bayesian method.  For
convenience, we will select a multivariate Gaussian prior distribution
on $\vec{w}$:
\begin{equation*}
  p(\vec{w} \given \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{w}; \vec{\mu}, \mat{\Sigma}).
\end{equation*}
As before, we might expect \emph{a priori} that the entries of
$\vec{w}$ are small.  Furthermore, in the absence of any evidence
otherwise, we might reason that all potential directions of $\vec{w}$
are equally likely.  These two assumptions can be codified by choosing
a multivariate Gaussian distribution for $\vec{w}$ with zero mean and
isotropic diagonal covariance $s^2 \mat{I}$:
\begin{equation}
  \label{simple_prior}
  p(\vec{w} \given s^2)
  =
  \mc{N}(\vec{w}; \vec{0}, s^2\mat{I}).
\end{equation}

Given our observed data $\data = (\mat{X}, \vec{y})$, we wish to form
the posterior distribution $p(\vec{w} \given \data)$.  We will do so
by deriving the joint distribution between the weight vector $\vec{w}$
and the observed vector of values $\vec{y}$, given the inputs
$\mat{X}$.  We write again the assumed linear relationship in our
training data:
\begin{equation*}
  \vec{y} = \mat{X}\vec{w} + \vec{\epsilon}.
\end{equation*}
Given $\vec{w}$ and $\mat{X}$, the only uncertainty in the above is
the additive residuals $\vec{\epsilon}$.  We assume that the residuals
are independent, unbiased, and tend to be ``small.''  We may
accomplish this by modeling each entry of $\vec{\epsilon}$ as arising
from zero-mean, independent, identically distributed Gaussian noise
with variance $\sigma^2$:
\begin{equation*}
  p(\vec{\epsilon} \given \sigma^2)
  =
  \mc{N}(\vec{\epsilon}; \vec{0}, \sigma^2 \mat{I}).
\end{equation*}

Define $\vec{f} = \mat{X}\vec{w},$ and note that $\vec{f}$ is a linear
transformation of the multivariate-Gaussian distributed $\vec{w}$;
therefore, we may easily derive its distribution:
\begin{equation*}
  p(\vec{f} \given \mat{X}, \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{f}; \mat{X}\vec{\mu}, \mat{X}\mat{\Sigma}\mat{X}\trans).
\end{equation*}
Now the observation vector $\vec{y}$ is a sum of two independent
multivariate Gaussian distributed vectors $\vec{f}$ and
$\vec{\epsilon}$.  Therefore, we may derive its prior distribution as
well:
\begin{equation*}
  p(\vec{y} \given \vec{X}, \vec{\mu}, \mat{\Sigma}, \sigma^2)
  =
  \mc{N}(
  \vec{y};
  \mat{X}\vec{\mu},
  \mat{X}\mat{\Sigma}\mat{X}\trans + \sigma^2\mat{I}
  ).
\end{equation*}
Finally, we compute the covariance between $\vec{y}$ and $\vec{w}$:
\begin{equation*}
  \cov[\vec{y}, \vec{w}]
  =
  \cov[\mat{X}\vec{w} + \vec{\epsilon}, \vec{w}]
  =
  \cov[\mat{X}\vec{w}, \vec{w}]
  =
  \mat{X}\cov[\vec{w}, \vec{w}]
  =
  \mat{X}\mat{\Sigma},
\end{equation*}
where we have used the linearity of covariance and the fact that
the noise vector $\vec{\epsilon}$ is independent of $\vec{w}$.

Now the joint distribution of $\vec{w}$ and $\vec{y}$ is multivariate
Gaussian:
\begin{equation*}
  p\Biggl(
  \begin{bmatrix}
    \vec{w}
    \\
    \vec{y}
  \end{bmatrix}
  \given
  \vec{X}, \vec{\mu}, \mat{\Sigma}, \sigma^2
  \Biggr)
  =
  \mc{N}
  \Biggl(
  \begin{bmatrix}
    \vec{w}
    \\
    \vec{y}
  \end{bmatrix}
  ;
  \begin{bmatrix}
    \vec{\mu}
    \\
    \mat{X}\vec{\mu}
  \end{bmatrix}
  ,
  \begin{bmatrix}
    \mat{\Sigma} & \mat{\Sigma}\mat{X}\trans
    \\
    \mat{X}\mat{\Sigma} & \mat{X}\mat{\Sigma}\mat{X}\trans + \sigma^2\mat{I}
  \end{bmatrix}
  \Biggr).
\end{equation*}
We apply the formula for conditioning multivariate Gaussians on
subvectors to derive the posterior distribution of $\vec{w}$ given
the data $\data$:
\begin{equation*}
  p(\vec{w} \given \data, \vec{\mu}, \mat{\Sigma}, \sigma^2)
  =
  \mc{N}(\vec{w};
  \vec{\mu}_{\vec{w}\given\data},
  \mat{\Sigma}_{\vec{w}\given\data}
  ),
\end{equation*}
where
\begin{align*}
  \vec{\mu}_{\vec{w}\given\data}
  &=
  \vec{\mu}
  +
  \mat{\Sigma}
  \mat{X}\trans
  (\mat{X}\mat{\Sigma}\mat{X}\trans + \sigma^2 \mat{I})\inv
  (\vec{y} - \mat{X}\vec{\mu});
  \\
  \mat{\Sigma}_{\vec{w}\given\data}
  &=
  \mat{\Sigma}
  -
  \mat{\Sigma}
  \mat{X}\trans
  (\mat{X}\mat{\Sigma}\mat{X}\trans + \sigma^2 \mat{I})\inv
  \mat{X}
  \mat{\Sigma}.
\end{align*}

\subsection*{Making predictions}

Suppose we wish to use our model to predict the outputs $\vec{y}_\ast$
associated with a set of inputs $\mat{X}_\ast$.  Writing again
$\vec{y}^\ast = \mat{X}_\ast\vec{w} + \vec{\epsilon}$, we derive:
\begin{equation*}
  p(\vec{y}_\ast \given \data, \vec{\mu}, \mat{\Sigma}, \sigma^2)
  =
  \mc{N}(
  \vec{y}_\ast;
  \mat{X}_\ast \vec{\mu}_{\vec{w}\given\data},
  \mat{X}_\ast \mat{\Sigma}_{\vec{w}\given\data} \mat{X}_\ast\trans + \sigma^2 \mat{I}).
\end{equation*}
Therefore, we have a full, joint probability distribution over the
outputs $\vec{y}_\ast$.  If we must make point estimates, we choose a
loss function for our predictions and use Bayesian decision theory.
For example, to minimize expected squared loss, we predict the
posterior mean.

An equivalent way to derive this result is to use the sum rule as
follows:
\begin{align*}
  p(\vec{y}_\ast \given \data, \vec{\mu}, \mat{\Sigma}, \sigma^2)
  &=
  \int
  p(\vec{y}_\ast \given \vec{w}, \sigma^2)
  p(\vec{w} \given \data, \vec{\mu}, \mat{\Sigma}, \sigma^2)
  \intd{\vec{w}}
  \\
  &=
  \int
  \mc{N}(\vec{y}_\ast; \mat{X}_\ast \vec{w}, \sigma^2 \mat{I})
  \,
  \mc{N}(\vec{w}; \vec{\mu}_{\vec{w}\given\data}, \mat{\Sigma}_{\vec{w}\given\data})
  \intd{\vec{w}}
  \\
  &=
  \mc{N}(
  \vec{y}_\ast;
  \mat{X}_\ast \vec{\mu}_{\vec{w}\given\data},
  \mat{X}_\ast \mat{\Sigma}_{\vec{w}\given\data} \mat{X}_\ast\trans + \sigma^2 \mat{I}),
\end{align*}
where we have used a slightly more general form of the convolution
formula for multivariate Gaussians.  In this case, we view the
prediciton process as considering the predictions we would make with
every possible value of the parameters $\vec{w}$ and averaging them
according to their plausibility.

\subsection*{Connection to ridge regression}

For the simple prior \eqref{simple_prior}, the posterior mean
can be rewritten as:
\begin{equation*}
  \vec{\mu}_{\vec{w}\given\data}
  =
  s^2
  \mat{X}\trans
  (s^2\mat{X}\mat{X}\trans + \sigma^2 \mat{I})\inv
  \vec{y}
  =
  \biggl(\mat{X}\trans\mat{X} + \frac{\sigma^2}{s^2} \mat{I}\biggr)\inv
  \mat{X}\trans
  \vec{y},
\end{equation*}
where we have factored out the $s^2$ in the inverse and have applied
the matrix identity
\begin{equation*}
  (\mat{A}\mat{B} + c\mat{I})\inv \mat{A} =
  \mat{A}(\mat{B}\mat{A} + c \mat{I})\inv.
\end{equation*}
This is exactly equal to the ridge regression solution \eqref{ridge}
with $\lambda = \frac{\sigma^2}{s^2}$!  Why is this?

Consider finding the \emph{maximum a posteriori} estimator of
$\vec{w}$ using the simple prior \eqref{simple_prior}.
Bayes' theorem tells us that the posterior distribution $p(\vec{w}
\given \data)$ is proportional to the likelihood times the prior:
\begin{equation*}
  p(\vec{w} \given \mat{X}, \vec{y}, \sigma^2, s^2)
  \propto
  p(\vec{y} \given \mat{X}, \vec{w}, \sigma^2)
  p(\vec{w} \given s^2).
\end{equation*}
The distribution of $\vec{y}$ given $\vec{w}$ is simple:
\begin{equation*}
  p(\vec{y} \given \mat{X}, \vec{w}, \sigma^2)
  =
  \mc{N}(\vec{y}; \mat{X}\vec{w}, \sigma^2\mat{I})
  =
  \prod_{i = 1}^N
  \mc{N}(y_i; \vec{x}_i\trans\vec{w}, \sigma^2).
\end{equation*}

Rather than maximizing the posterior directly, we may instead maximize
its logarithm:
\begin{align*}
  \hat{\vec{w}}_{\text{\acro{MAP}}}
  &=
  \argmax_{\vec{w}}
  -\frac{1}{2\sigma^2}
  \sum_{i = 1}^N
  (\vec{x}_i\trans \vec{w} - y_i)^2
  -
  \frac{1}{2s^2}
  \sum_{i = 1}^d
  w_i^2
  \\
  &=
  \argmin_{\vec{w}}
  \sum_{i = 1}^N
  (\vec{x}_i\trans \vec{w} - y_i)^2
  +
  \frac{\sigma^2}{s^2}
  \lVert \vec{w} \rVert_{2}^{2},
\end{align*}
where we multiplied by $-2\sigma^2$ to derive the second line.  This
is exactly the ridge regression objective.  Therefore ridge regression
with $\ell^2$ regularization can be seen from the Bayesian perspective
as placing a Gaussian prior on $\vec{w}$ and finding the \acro{MAP}
estimator.  In general, many regularization methods can be interpreted
as implicitly choosing a particular likelihood for the data and prior
for the parameters of a model and maximizing the posterior density.
For example, $\ell^1$ regularization (as seen in \acro{LASSO}) is
equivalent to placing a Laplace distribution prior on the weight
parameters, encouraging a sparse solution.

Continuing with this argument, we can interpret the squared loss
function $\sum_{i=1}^N (\vec{x}\trans \vec{w} - y_i)^2$ as coming from
an iid Gaussian noise assumption.  In general, loss functions in
minimization problems such as \eqref{ridge} can often be interpreted
as negative log likelihoods arising from an implicit independent noise
assumption.

\end{document}