\documentclass{article}

\usepackage[T1]{fontenc}
\usepackage[osf]{libertine}
\usepackage[scaled=0.8]{beramono}
\usepackage[margin=1.5in]{geometry}
\usepackage{url}
\usepackage{booktabs}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{nicefrac}
\usepackage{microtype}
\usepackage{subcaption}
\usepackage{bm}

\usepackage{sectsty}
\sectionfont{\large}
\subsectionfont{\normalsize}

\usepackage{titlesec}
\titlespacing{\section}{0pt}{10pt plus 2pt minus 2pt}{0pt plus 2pt minus 0pt}
\titlespacing{\subsection}{0pt}{5pt plus 2pt minus 2pt}{0pt plus 2pt minus 0pt}

\usepackage{pgfplots}
\pgfplotsset{
  compat=newest,
  plot coordinates/math parser=false,
  tick label style={font=\footnotesize, /pgf/number format/fixed},
  label style={font=\small},
  legend style={font=\small},
  every axis/.append style={
    tick align=outside,
    clip mode=individual,
    scaled ticks=false,
    thick,
    tick style={semithick, black}
  }
}

\pgfkeys{/pgf/number format/.cd, set thousands separator={\,}}

\usepgfplotslibrary{external}
\tikzexternalize[prefix=tikz/]

\newlength\figurewidth
\newlength\figureheight

\setlength{\figurewidth}{8cm}
\setlength{\figureheight}{6cm}

\newlength\squarefigurewidth
\newlength\squarefigureheight

\setlength{\squarefigurewidth}{4cm}
\setlength{\squarefigureheight}{4cm}

\newlength\smallsquarefigurewidth
\newlength\smallsquarefigureheight

\setlength{\smallsquarefigurewidth}{3.25cm}
\setlength{\smallsquarefigureheight}{3.25cm}

\newlength\smallfigurewidth
\newlength\smallfigureheight

\setlength{\smallfigurewidth}{6.25cm}
\setlength{\smallfigureheight}{4cm}

\setlength{\parindent}{0pt}
\setlength{\parskip}{1ex}

\newcommand{\acro}[1]{\textsc{\MakeLowercase{#1}}}
\newcommand{\given}{\mid}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\data}{\mc{D}}
\newcommand{\intd}[1]{\,\mathrm{d}{#1}}
\newcommand{\inv}{^{-1}}
\newcommand{\trans}{^\top}
\newcommand{\mat}[1]{\bm{\mathrm{#1}}}
\renewcommand{\vec}[1]{\bm{\mathrm{#1}}}
\newcommand{\R}{\mathbb{R}}

\DeclareMathOperator{\var}{var}
\DeclareMathOperator{\cov}{cov}
\DeclareMathOperator{\diag}{diag}

\begin{document}

\section*{The Gaussian distribution}
Probably the most-important distribution in all of statistics is the
\emph{Gaussian distribution,} also called the \emph{normal
  distribution.}  The Gaussian distribution arises in many contexts
and is widely used for modeling continuous random variables.

The probability density function of the univariate (one-dimensional)
Gaussian distribution is
\begin{equation*}
  p(x \given \mu, \sigma^2)
  =
  \mc{N}(x; \mu, \sigma^2)
  =
  \frac{1}{Z}
  \exp\biggl(
  -\frac{(x - \mu)^2}{2 \sigma^2}
  \biggr).
\end{equation*}
The normalization constant $Z$ is
\begin{equation*}
  Z = \sqrt{2 \pi \sigma^2}.
\end{equation*}
The parameters $\mu$ and $\sigma^2$ specify the mean and variance
of the distribution, respectively:
\begin{equation*}
  \mu = \mathbb{E}[x]; \qquad
  \sigma^2 = \var[x].
\end{equation*}
Figure \ref{1d_examples} plots the probability density function for
several sets of parameters $(\mu, \sigma^2)$.  The distribution is
symmetric around the mean and most of the density ($\approx 99.7\%$)
is contained within $\pm 3 \sigma$ of the mean.

\begin{figure}
  \centering
  \input{figures/1d_gaussian_pdfs.tex}
  \caption{Examples of univariate Gaussian \acro{PDF}s $\mc{N}(x; \mu,
    \sigma^2)$.}
  \label{1d_examples}
\end{figure}

We may extend the univariate Gaussian distribution to a distribution
over $d$-dimensional vectors, producing a multivariate analog.  The
probablity density function of the multivariate Gaussian distribution
is
\begin{equation*}
  p(\vec{x} \given \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{x}; \vec{\mu}, \mat{\Sigma})
  =
  \frac{1}{Z}
  \exp\biggl(
  -\frac{1}{2}
  (\vec{x} - \vec{\mu})\trans
  \mat{\Sigma}\inv
  (\vec{x} - \vec{\mu})
  \biggr).
\end{equation*}
The normalization constant $Z$ is
\begin{equation*}
  Z = \sqrt{\det (2 \pi \mat{\Sigma})}
    = (2 \pi)^{\nicefrac{d}{2}} (\det \mat{\Sigma})^{\nicefrac{1}{2}}.
\end{equation*}
Examining these equations, we can see that the multivariate density
coincides with the univariate density in the special case
when $\mat{\Sigma}$ is the scalar $\sigma^2$.

Again, the vector $\vec{\mu}$ specifies the mean of the multivariate
Gaussian distribution.  The matrix $\mat{\Sigma}$ specifies the
\emph{covariance} between each pair of variables in $\vec{x}$:
\begin{equation*}
  \mat{\Sigma}
  =
  \cov(\vec{x}, \vec{x})
  =
  \mathbb{E}\bigl[(\vec{x} - \vec{\mu})(\vec{x} - \vec{\mu})\trans\bigr].
\end{equation*}
Covariance matrices are necessarily symmetric and \emph{positive
  semidefinite,} which means their eigenvalues are nonnegative.  Note
that the density function above requires that $\mat{\Sigma}$ be
\emph{positive definite,} or have strictly positive eigenvalues.  A
zero eigenvalue would result in a determinant of zero, making the
normalization impossible.

The dependence of the multivariate Gaussian density on $\vec{x}$ is
entirely through the value of the quadratic form
\begin{equation*}
  \Delta^2
  =
  (\vec{x} - \vec{\mu})\trans
  \mat{\Sigma}\inv
  (\vec{x} - \vec{\mu}).
\end{equation*}
The value $\Delta$ (obtained via a square root) is called the
\emph{Mahalanobis distance,} and can be seen as a generalization of
the $Z$ score $\frac{(x - \mu)}{\sigma}$, often encountered in
statistics.

To understand the behavior of the density geometrically, we can set
the Mahalanobis distance to a constant. The set of points in $\R^d$
satisfying $\Delta = c$ for any given value $c > 0$ is an ellipsoid
with the eigenvectors of $\mat{\Sigma}$ defining the directions of the
principal axes.

\begin{figure}
  \begin{subfigure}{0.33\textwidth}
    \input{figures/2d_gaussian_pdf_1}
    \caption{$\mat{\Sigma} = \begin{bmatrix} 1 & 0 \\ 0 & 1 \end{bmatrix}$}
    \label{2d_example_1}
  \end{subfigure}
  \begin{subfigure}{0.33\textwidth}
    \input{figures/2d_gaussian_pdf_2}
    \caption{$\mat{\Sigma} = \begin{bmatrix} 1 & \nicefrac{1}{2} \\ \nicefrac{1}{2} & 1 \end{bmatrix}$}
  \end{subfigure}
  \begin{subfigure}{0.33\textwidth}
    \input{figures/2d_gaussian_pdf_3}
    \caption{$\mat{\Sigma} = \begin{bmatrix} 1 & -1 \\ -1 & 3 \end{bmatrix}$}
    \label{2d_example_3}
  \end{subfigure}
  \caption{Contour plots for example bivariate Gaussian distributions.
    Here $\vec{\mu} = \vec{0}$ for all examples.}
  \label{2d_examples}
\end{figure}

Figure \ref{2d_examples} shows contour plots of the density of three
bivariate (two-dimensional) Gaussian distributions.  The elliptical
shape of the contours is clear.

The Gaussian distribution has a number of convenient analytic
properties, some of which we describe below.

\subsection*{Marginalization}

Often we will have a set of variables $\vec{x}$ with a joint
multivariate Gaussian distribution, but only be interested in
reasoning about a subset of these variables.  Suppose $\vec{x}$
has a multivariate Gaussian distribution:
\begin{equation*}
  p(\vec{x} \given \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{x}, \vec{\mu}, \mat{\Sigma}).
\end{equation*}

Let us partition the vector into two components:
\begin{equation*}
  \vec{x}
  =
  \begin{bmatrix}
    \vec{x}_1 \\
    \vec{x}_2
  \end{bmatrix}.
\end{equation*}
We partition the mean vector and covariance matrix in the same
way:
\begin{equation*}
  \vec{\mu}
  =
  \begin{bmatrix}
    \vec{\mu}_1 \\
    \vec{\mu}_2
  \end{bmatrix}
  \qquad
  \mat{\Sigma}
  =
  \begin{bmatrix}
    \mat{\Sigma}_{11} & \mat{\Sigma}_{12} \\
    \mat{\Sigma}_{21} & \mat{\Sigma}_{22}
  \end{bmatrix}.
\end{equation*}
Now the marginal distribution of the subvector $\vec{x}_1$
has a simple form:
\begin{equation*}
  p(\vec{x}_1 \given \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{x}_1, \vec{\mu}_1, \mat{\Sigma}_{11}),
\end{equation*}
so we simply pick out the entries of $\vec{\mu}$ and $\mat{\Sigma}$
corresponding to $\vec{x}_1$.

Figure \ref{marginal_example} illustrates the marginal distribution of
$x_1$ for the joint distribution shown in Figure
\ref{2d_examples}(\subref{2d_example_3}).

\begin{figure}
  \centering
  \begin{subfigure}[t]{0.49\textwidth}
    \input{figures/2d_gaussian_pdf_3_large}
    \caption{$p(\vec{x} \given \vec{\mu}, \mat{\Sigma})$}
    \label{marginal_2d_pdf}
  \end{subfigure}
  \begin{subfigure}[t]{0.49\textwidth}
    \input{figures/2d_marginal_pdf}
    \caption{$p(x_1 \given \mu_1, \Sigma_{11}) = \mc{N}(x_1; 0, 1)$}
    \label{marginal_pdf}
  \end{subfigure}
  \caption{Marginalization example.  (\subref{marginal_2d_pdf}) shows
    the joint density over $\vec{x} = [x_1, x_2]\trans$; this is the same
    density as in Figure \ref{2d_examples}(\subref{2d_example_3}).
    (\subref{marginal_pdf}) shows the marginal density of $x_1$.}
  \label{marginal_example}
\end{figure}

\subsection*{Conditioning}

Another common scenario will be when we have a set of variables
$\vec{x}$ with a joint multivariate Gaussian prior distribution, and
are then told the value of a subset of these variables.  We may then
condition our prior distribution on this observation, giving a
posterior distribution over the remaining variables.

Suppose again that $\vec{x}$ has a multivariate Gaussian distribution:
\begin{equation*}
  p(\vec{x} \given \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{x}, \vec{\mu}, \mat{\Sigma}),
\end{equation*}
and that we have partitioned as before: $\vec{x} = [\vec{x}_1,
  \vec{x}_2]\trans$.  Suppose now that we learn the exact value of the
subvector $\vec{x}_2$.  Remarkably, the posterior distribution
\begin{equation*}
  p(\vec{x}_1 \given \vec{x}_2, \vec{\mu}, \mat{\Sigma})
\end{equation*}
is a Gaussian distribution!  The formula is
\begin{equation*}
  p(\vec{x}_1 \given \vec{x}_2, \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{x}_1; \vec{\mu}_{1 \given 2}, \mat{\Sigma}_{11 \given 2}),
\end{equation*}
with
\begin{align*}
  \vec{\mu}_{1 \given 2}
  &=
  \vec{\mu}_1 + \mat{\Sigma}_{12}\mat{\Sigma}_{22}\inv (\vec{x}_2 - \vec{\mu}_2);
  \\
  \mat{\Sigma}_{11 \given 2}
  &=
  \mat{\Sigma}_{11} - \mat{\Sigma}_{12}\mat{\Sigma}_{22}\inv \mat{\Sigma}_{21}.
\end{align*}
So we adjust the mean by an amount dependent on: (1) the covariance
between $\vec{x}_1$ and $\vec{x}_2$, $\mat{\Sigma}_{12}$, (2) the
prior uncertainty in $\vec{x}_2$, $\mat{\Sigma}_{22}$, and (3) the
deviation of the observation from the prior mean, $(\vec{x}_2 -
\vec{\mu}_2)$.  Similarly, we reduce the uncertainty in $\vec{x}_1$,
$\mat{\Sigma}_{11}$, by an amount dependent on (1) and (2).  Notably,
the reduction of the covariance matrix does \emph{not} depend on the
values we observe.

Notice that if $\vec{x}_1$ and $\vec{x}_2$ are independent, then
$\mat{\Sigma}_{12} = \mat{0}$, and the conditioning operation does not
change the distribution of $\vec{x}_1$, as expected.

Figure \ref{conditional_example} illustrates the conditional
distribution of $x_1$ for the joint distribution shown in Figure
\ref{2d_examples}(\subref{2d_example_3}), after observing $x_2 = 2$.

\begin{figure}
  \centering
  \begin{subfigure}[t]{0.49\textwidth}
    \input{figures/2d_gaussian_pdf_conditional}
    \caption{$p(\vec{x} \given \vec{\mu}, \mat{\Sigma})$}
    \label{conditional_2d_pdf}
  \end{subfigure}
  \begin{subfigure}[t]{0.49\textwidth}
    \input{figures/2d_conditional_pdf}
    \caption{$p(x_1 \given x_2, \vec{\mu}, \mat{\Sigma}) =
      \mc{N}\bigl(x_1; -\nicefrac{2}{3}, (\nicefrac{2}{3})^2\bigr)$}
    \label{conditional_pdf}
  \end{subfigure}
  \caption{Conditioning example.  (\subref{conditional_2d_pdf}) shows
    the joint density over $\vec{x} = [x_1, x_2]\trans$, along with
    the observation value $x_2 = 2$; this is the same density as in
    Figure \ref{2d_examples}(\subref{2d_example_3}).
    (\subref{conditional_pdf}) shows the conditional density of
    $x_1$ given $x_2 = 2$.}
  \label{conditional_example}
\end{figure}

\subsection*{Pointwise multiplication}

Another remarkable fact about multivariate Gaussian density functions
is that pointwise multiplication gives another (unnormalized) Gaussian
\acro{PDF}:
\begin{equation*}
  \mc{N}(\vec{x}; \vec{\mu}, \mat{\Sigma})
  \,
  \mc{N}(\vec{x}; \vec{\nu}, \mat{P})
  =
  \frac{1}{Z}
  \mc{N}(\vec{x}; \vec{\omega}, \mat{T}),
\end{equation*}
where
\begin{align*}
  \mat{T}
  &=
  (\mat{\Sigma}\inv + \mat{P}\inv)\inv \\
  \vec{\omega}
  &=
  \mat{T}
  (\mat{\Sigma}\inv \vec{\mu} +
  \mat{P}\inv \vec{\nu}) \\
  Z^{-1}
  &=
  \mc{N}(\vec{\mu}; \vec{\nu}, \mat{\Sigma} + \mat{P})
  =
  \mc{N}(\vec{\nu}; \vec{\mu}, \mat{\Sigma} + \mat{P}).
\end{align*}

\subsection*{Convolutions}

Gaussian probability density functions are closed under convolutions.
Let $\vec{x}$ and $\vec{y}$ be $d$-dimensional vectors, with
distributions
\begin{equation*}
  p(\vec{x} \given \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{x}; \vec{\mu}, \mat{\Sigma});
  \qquad
  p(\vec{y} \given \vec{\nu}, \mat{P})
  =
  \mc{N}(\vec{y}; \vec{\nu}, \mat{P}).
\end{equation*}
Then the convolution of their density functions is another Gaussian
\acro{PDF}:
\begin{equation*}
  f(\vec{y}) =
  \int
  \mc{N}(\vec{y} - \vec{x}; \vec{\nu}, \mat{P})
  \,
  \mc{N}(\vec{x}; \vec{\mu}, \mat{\Sigma})
  \intd{\vec{x}}
  =
  \mc{N}(\vec{y}; \vec{\mu} + \vec{\nu}, \mat{\Sigma} + \mat{P}),
\end{equation*}
where the mean and covariances add in the result.

If we assume that $\vec{x}$ and $\vec{y}$ are independent, then the
distribution of their sum $\vec{z} = \vec{x} + \vec{y}$ will also have
a multivariate Gaussian distribution, whose density will precisely the
convolution of the individual densities:
\begin{equation*}
  p(\vec{z} \given \vec{\mu}, \vec{\nu}, \mat{\Sigma}, \mat{P})
  =
  \mc{N}(\vec{z}; \vec{\mu} + \vec{\nu}, \mat{\Sigma} + \mat{P}).
\end{equation*}

These results will often come in handy.

\subsection*{Affine transformations}

Consider a $d$-dimensional vector $\vec{x}$ with a multivariate
Gaussian distribution:
\begin{equation*}
  p(\vec{x} \given \vec{\mu}, \mat{\Sigma})
  =
  \mc{N}(\vec{x}, \vec{\mu}, \mat{\Sigma}).
\end{equation*}
Suppose we wish to reason about an affine transformation of $\vec{x}$
into $\R^D$, $\vec{y} = \mat{A}\vec{x} + \vec{b}$, where $\mat{A} \in
\R^{D \times d}$ and $\vec{b} \in \R^D$.  Then $\vec{y}$ has a
$D$-dimensional Gaussian distribution:
\begin{equation*}
  p(\vec{y} \given \vec{\mu}, \mat{\Sigma}, \mat{A}, \vec{b})
  =
  \mc{N}(\vec{y}, \mat{A}\vec{\mu} + \vec{b}, \mat{A}\mat{\Sigma}\mat{A}\trans).
\end{equation*}

\begin{figure}
  \centering
  \begin{subfigure}[t]{0.49\textwidth}
    \input{figures/2d_gaussian_pdf_3_large}
    \caption{$p(\vec{x} \given \vec{\mu}, \mat{\Sigma})$}
    \label{transformed_2d_pdf}
  \end{subfigure}
  \begin{subfigure}[t]{0.49\textwidth}
    \input{figures/2d_transformed_pdf}
    \caption{$p(\vec{y} \given \vec{\mu}, \mat{\Sigma}, \mat{A}, \vec{b})$}
    \label{transformed_pdf}
  \end{subfigure}
  \caption{Affine transformation example.
    (\subref{transformed_2d_pdf}) shows the joint density over
    $\vec{x} = [x_1, x_2]\trans$; this is the same density as in
    Figure \ref{2d_examples}(\subref{2d_example_3}).
    (\subref{transformed_pdf}) shows the density of $\vec{y} =
    \mat{A}\vec{x} + \vec{b}$. The values of $\mat{A}$ and $\vec{b}$
    are given in the text.  The density of the transformed vector is
    another Gaussian.}
  \label{transformed_example}
\end{figure}

Figure \ref{transformed_example} illustrates an affine transformation
of the vector $\vec{x}$ with the joint distribution shown in Figure
\ref{2d_examples}(\subref{2d_example_3}), for the values
\begin{equation*}
  \mat{A}
  =
  \begin{bmatrix}
    \nicefrac{1}{5} & -\nicefrac{3}{5} \\
    \nicefrac{1}{2} & \nicefrac{3}{10}
  \end{bmatrix};
  \qquad
  \vec{b}
  =
  \begin{bmatrix}
    1 \\
    -1
  \end{bmatrix}.
\end{equation*}
The density has been rotated and translated, but remains a Gaussian.

\section*{Selecting parameters}

The $d$-dimensional multivariate Gaussian distribution is specified by
the parameters $\vec{\mu}$ and $\mat{\Sigma}$.  Without any further
restrictions, specifying $\vec{\mu}$ requires $d$ parameters and
specifying $\mat{\Sigma}$ requires a further $\binom{d}{2} = \frac{d(d
  -1)}{2}$.  The number of parameters therefore grows quadratically in
the dimension, which can sometimes cause difficulty.  For this reason,
we sometimes restrict the covariance matrix $\vec{\Sigma}$ in some way
to reduce the number of parameters.

Common choices are to set $\mat{\Sigma} = \diag \vec{\tau}$, where
$\vec{\tau}$ is a vector of marginal variances, and $\mat{\Sigma} =
\sigma^2 \mat{I}$, a constant diagonal matrix.  Both of these options
assume independence between the variables in $\vec{x}$.  The former
case is more flexible, allowing a different scale parameter for each
entry, whereas the latter assumes an equal marginal variance of
$\sigma^2$ for each variable.  Geometrically, the densities are
axis-aligned, as in Figure \ref{2d_examples}(\subref{2d_example_1}),
and in the latter case, the isoprobability contours are spherical
(also as in Figure \ref{2d_examples}(\subref{2d_example_1})).

\end{document}


\graphicspath{{Chapter4/Figs/simulations/}{Chapter4/Figs/scrna/}{Chapter4/Figs/scmet/}{Chapter4/Figs/scnmt/}}

\chapter{MOFA+: a statistical framework for the integration of large-scale structured datasets}

In Chapter 3 we developed Multi-Omics Factor Analysis (MOFA), a statistical framework for the unsupervised integration of multi-omics data. MOFA addresses key challenges in data integration, including overfitting, noise reduction, handling of missing values and improved interpretation of the model output. However, when applied to increasingly-large single-cell genomics datasets, the variational inference scheme offers limited scalability. In addition, the increased experimental throughput has facilitated the simultaneous study of multiple conditions within the same experimental design \cite{Replogle2020}. However, MOFA makes strong independence assumptions about the dependencies across samples and it provides no principled strategy to model datasets where the samples are structured into multiple groups (i.e. batches, donors or even independent studies). In this Chapter we improve the model formulation with the aim of performing integrative analysis of large-scale datasets where the features are structured into multiple data modalities (views) and the samples (or cells) are structured into different groups.

The work discussed in this Chapter has been peer-reviewed and published in \cite{Argelaguet2020}. The project was conceived by Damien Arnol and me. The mathematical derivations and the implementation of the stochastic variational inference scheme were done together by Damien Arnol, Yonatan Deloro and me. I implemented the downstream analysis package, but with significant contributions from Danila Bredikhin. I generated most figures and I wrote the manuscript with feedback from all authors. John C. Marioni and Oliver Stegle supervised the project.

\section{Theoretical foundations}

\subsection{Exponential family distributions} \label{section:exponential_family}

Exponential family distributions are a parametric class of probability distributions that have characteristic mathematical properties which make them amenable for probabilistic modelling.\\
The majority of probability distributions that are commonly used in statistics belong to the exponential family, including the normal or Gaussian, Gamma, Poisson, Bernoulli, Exponential, etc.\\
Formally, exponential family distributions can be represented in the following form:
\begin{equation} \label{eq:exponential_family}
	p(\bfx|\btheta) = h(\bfx) \exp \{ \eta(\btheta) T(\bfx) - A(\btheta) \}
\end{equation}
where $\bfx$ is a multivariate random variable and $\btheta$ are the distribution's parameters. Each term has a common notation: T(\bfx): sufficient statistics; $\eta(\btheta)$: natural parameters; $h(\bfx)$: base measure; $A(\eta)$: the log-partition function (or the normaliser).

The exponential family form for the probability distributions that are frequently used in this thesis are shown below:

Univariate normal distribution:
\begin{align*}
	& \eta(\mu,\sigma) = \lbrack \frac{\mu}{\sigma^2}; -\frac{1}{2\sigma^2} \rbrack \\
	& h(x) = \frac{1}{\sqrt{2\pi}} \\
	& T(x) = \lbrack x; x^2 \rbrack \\
	& A(\mu,\sigma) = \frac{\mu^2}{2\sigma^2} + \log \| \sigma \|
\end{align*}

Multivariate normal distribution:
\begin{align*}
	& \eta(\bmu,\bSigma)  = \lbrack\Sigma^{-1} \mu; -0.5\Sigma^{-1} \rbrack \\
	& T(x) = \lbrack x; xx^T \rbrack \\
	& h(x) = (2\pi)^{-\frac{k}{2}} \\
	& A(\theta) = -0.25\eta_1^T \eta_2{-1} \eta_1 - 0.5\log(\|-2\eta_2\|)
\end{align*}

Gamma distribution:
\begin{align*}
	& \eta = \lbrack \alpha - 1; -\beta \rbrack \\
	& T(x) = \lbrack \log x; x \rbrack \\
	& h(x) = 1 \\
	& A(\theta) = \log(\Gamma(\eta_1 + 1)) - (\eta_1 + 1) \log(-\eta_2)
\end{align*}

Beta distribution:
\begin{align*}
	& \eta = [\alpha; \beta] \\
	& T(x) = [\log x; \log (1-x)] \\
	& h(x) = \frac{1}{x(1-x)} \\
	& A(\theta) = \log(\Gamma(\eta_1)) +\log(\Gamma(\eta_2)) - \log(\Gamma(\eta_1+\eta_2))
\end{align*}

In the context of Bayesian inference, the main property that make exponential family distributions indispensable is that they have conjugate priors. That is, the combination of likelihood and prior distributions ensure a closed-form posterior distribution which is of the same form as the prior. As we have discussed in Chapter 3, this property is essential for enabling efficient statistical inference, otherwise posterior distributions must be computed using expensive and approximate numerical methods.

\subsection{Gradient ascent} \label{section:gradient_ascent}

Gradient ascent is a first-order optimization algorithm for finding the maximum of a function \cite{Bishop2006,Murphy}. Formally, for a differentiable function $F(x)$, the iterative scheme of gradient ascent is:
\begin{equation} \label{gradient_ascent}
	\bfx^{(t+1)} = \bfx^{(t)} + \rho^{(t)} \nabla F(\bfx^{(t)})
\end{equation}
In short, the algorithm works by taking steps proportional to the gradient $\nabla F$ evaluated at each iteration $t$.
% This leads to a monotonic sequence:
% \[
% 	\bfx^{0} \leq \bfx^{1} \leq \bfx^{1} \cdots
% \]
Importantly, the step size $\rho^{(t)}$ is typically adjusted at each iteration $t$ such that it satisfies the Robbins-Monro conditions: $\sum_t \rho^{(t)} = \infty \text{ and } \sum_t (\rho^{(t)})^2 < \infty$. $F$ is guaranteed to converge to the \textit{global} maximum if the objective function is convex \cite{Robbins-Monro1951}. If $F$ is not convex, the algorithm is sensitive to the initialisation $\bfx^{t=0}$ and can converge to local optima.

\subsubsection{Stochastic gradient ascent} \label{section:stochastic_gradient_ascent}

Gradient ascent becomes prohibitively slow with large datasets, mainly because of the computational cost involved in the iterative calculation of gradients \cite{Spall2003}.\\
A simple strategy to speed up gradient ascent is to replace $\nabla F$ by an estimate $\hat{\nabla} F$ using a random subset of the data (minibatch).
The iterative scheme is then defined in the same way as in standard gradient ascent:
\begin{equation}
	\bfx^{(t+1)} = \bfx^{(t)} + \rho^{(t)} \hat{\nabla} F(\bfx^{(t)})
\end{equation}

%In practice, the stochastic nature of the algorithm makes the optimisation trajectory more wiggly and typically requires a larger number of iterations than standard gradient ascent. However, the reduced computational cost in computing the gradients yields an overall faster training time

% Copied from hoffman
% Under the right conditions, stochastic optimization algorithms provably converge to an optimum of the objective. Stochastic optimization is particularly attractive when the objective (and therefore its gradient) is a sum of many terms that can be computed independently. In that setting, we can cheaply compute noisy gradients by subsampling only a few of these terms.

\subsubsection{Natural gradient ascent} \label{section:natural_gradient_ascent}

Gradient ascent becomes problematic when applied to probabilistic models. To give the intuition, consider a probabilistic model with a hidden variable $x$ and corresponding parameter $\theta$, with a general objective function $\Lagr(\theta)$. From the definition of a derivative:
\[
	\nabla \Lagr(\theta) = \lim_{||h||\to0} \frac{\Lagr(\theta + h) - \Lagr(\theta)}{||h||}
\]
where $h$ represents an infinitesimally small positive step in the space of $\theta$.\\
To find the direction of steepest ascent, one would need to search over all possible directions $d$ in an infinitely small distance $h$, and select the $\hat{d}$ that gives the largest gradient:
\[
\nabla \Lagr(\theta) = \lim_{h\to0} \frac{1}{h}\argmax_{d \, s.t. \|d\|=h} \Lagr(\theta+d) - \Lagr(\theta)
\]
Importantly, this operation requires a distance metric to quantify what a \textit{small} distance $h$ means. In standard gradient ascent, this is measured using the Euclidean norm, and the direction of steepest ascent is hence dependent on the Euclidean geometry of the $\theta$ space. This problematic when doing probabilistic modelling because it does not consider the uncertainty that underlies probability distributions. When $\theta$ is a random variable with an underlying probability distribution, a small step from $\theta^{(t)}$ to $\theta^{(t+1)}$ does not guarantee an equivalently small change from $\Lagr(\theta^{(t)})$ to $\Lagr(\theta^{(t+1)})$. To illustrate this, consider the following example of four random variables:
\begin{equation}
	\begin{split}
		 \Ndist{\psi_1}{0,5} \\
		 \Ndist{\psi_2}{10,5}
	\end{split}
	\qquad
	\begin{split}
		\Ndist{\psi_3}{0,1} \\
		\Ndist{\psi_4}{10,1}
	\end{split}
\end{equation}

Using the Euclidean metric, the distance between $\psi_1$ and $\psi_2$ is the same as the distance between $\psi_3$ and $\psi_4$. However, the distance in distribution space (measured for example by the KL divergence) is much larger between $\psi_3$ and $\psi_4$ than between $\psi_1$ and $\psi_2$ (\Cref{fig:mofa2_euclidean_distributions}).

\begin{figure}[!h]
	\begin{center}
		\includegraphics[width=0.65\textwidth]{mofa2_euclidean_distributions}
		\caption{\textbf{Illustration of the problem of using the Euclidean norm as a distance measure between parameters of probability distributions}.\\
		In both plots, the red and blue distributions are separated by the same Euclidean distance of 100. Yet, the distance in probability space between the two distributions is higher in the right.
		}
		\label{fig:mofa2_euclidean_distributions}
	\end{center}
\end{figure}

This basic simulation suggests that replacing the Euclidean distance by the KL divergence as a distance metric may be more appropriate in the context of probabilistic modelling:
\[
	\nabla_{KL} \Lagr(\theta) = \lim_{h\to0} \frac{1}{h}\argmax_{d \, s.t. KL[p_\theta||p_{\theta+d}]=h} \Lagr(\theta+d) - \Lagr(\theta)
\]
The direction of steepest ascent measured by the KL divergence is called the natural gradient \cite{Amari1998,Martens2014}. To find the optimal $\hat{d}_{KL}$, one needs to solve the following optimisation problem:
\begin{equation*} \begin{aligned}
	&\argmin_{d} \Lagr(\theta+d) \qquad
	& \text{subject to}
	& \quad KL[p_\theta||p_{\theta+d}] < c
\end{aligned} \end{equation*}
where $c$ is an arbitrary constant. Previous works have shown that this can be solved by introducing Lagrange multipliers and Taylor expansions \cite{Amari1998,Kristiadi2019}. The solution corresponds to the standard (Euclidean) gradient pre-multiplied by the inverse of the Fisher Information Matrix of $q(x|\theta)$:
\begin{equation}\label{natural_gradient}
	\hat{d}_{KL} \propto \bfF^{-1}(\theta) \nabla_{\theta} \Lagr(\theta)
\end{equation}
where $\bfF(\theta)$ is defined as
\[
	\bfF(\theta) = \E_{q(x|\theta)}[(\nabla_\theta \log q(x|\theta)) (\nabla_\theta \log q(x|\theta))^T]
\]
%Effectively, the premultiplication by $\bfF^{-1}$ takes into account the local curvate of $q(\theta)$ in distribution space. \\

%Importantly, when $q(x|\theta)$ belongs to the exponential family, the Fisher Information matrix is simply the Hessian of the log normalizer.\\

In conclusion, while the standard gradient points to the direction of steepest ascent in Euclidean space, the natural gradient points to the direction of steepest ascent in a space where distances are defined by the KL divergence \cite{Kristiadi2019,Amari1998,Hoffman2012}.

\subsection{Stochastic variational inference} \label{section:stochastic_variational_inference}

In this section I will demonstrate how to derive a stochastic variational inference algorithm for general Bayesian models. This work is inspired and adapted from \cite{Hoffman2012}. A comprehensive mathematical derivation of the algorithm is not sought in this Chapter, instead I will describe a modified and simplified derivation that captures the gist of the original. For a complete mathematical derivation I refer the reader to \cite{Hoffman2012}.

This section builds upon three theoretical foundations that have been introduced before: Variational inference (\Cref{section:variational_inference}), exponential family distributions (\Cref{section:exponential_family}) and (natural) gradient ascent (\Cref{section:gradient_ascent}).

% COPIED
%The stochastic nature of the approach is interesting when one dimension of the matrix of observed variables is much larger than the others. In our case, it corresponds to $N$, the number of samples (or cells). \\

% Copoed from Hoffman
% In variational inference, we define a flexible family of distributions over the hidden variables, indexed by free parameters (Jordan et al., 1999; Wainwright and Jordan, 2008). We then find the setting of the parameters (i.e., the member of the family) that is closest to the posterior. Thus we solve the inference problem by solving an optimization problem.\\

\subsubsection{Model definition}

Consider a probabilistic model with a set of unobserved random variables, observations and (non-random) parameters. We begin by classifying the variables of the model into four different categories:

% what about the params for the local variables???
\begin{itemize}
	%\itemsep-1.5em
	\item observations ($\bfY$): $N$ different vectors $\bfy_{n}$, each one containing the observed variables for the $n$-th sample.
	\item local (hidden) variables ($\bfZ)$: $N$ different vectors $\bfz_{n}$, each one containing $K$ hidden variables associated with the $n$-th sample.
	\item \textit{global} (hidden) variables ($\bbeta$): one vector that contains $B$ hidden variables not indexed by $n$.
	\item parameters (non-random) for the \textit{global} variables ($\balpha_{\beta}$).
	\item parameters (non-random) for the \textit{local} variables ($\balpha_{z}$).
\end{itemize}

First, let us assume the following factorisation of the joint distribution:
\begin{equation}
	p(\bfY, \bfZ, \balpha_{\beta}, \balpha_{z}) = p(\bfZ|\balpha_{z}) p(\bbeta|\balpha_{\beta}) \prod_{n=1}^{N} p(\bfy_{n}|\bfz_{n},\bbeta)
\end{equation}
and the corresponding graphical model representation:
\begin{figure}[H]
	\centering
	\input{graphical_models/stochastic_general}
	\caption{\textbf{Graphical model for a general probabilistic model where unobserved variables are classified as \textit{global} and \textit{local}.}\\
	The dashed line indicates that the connection between \textit{global} and \textit{local} variables is optional, not used in the MOFA model.
	}
	\label{fig:graphical_model_stochastic}
\end{figure}

Notice that the difference between \textit{local} and \textit{global} variables lies on the conditional dependency assumptions. The \textit{local} variables for the $n$-th sample $\bfz_n$ are conditionally independent from any other observation $\bfy_{j}$ or \textit{local} variable $\bfz_{j}$ (where $j \neq n$), given that the \textit{global} variables $\bbeta$ are observed:
\[
	p(\bfy_n,\bfz_n| \bfy_j,\bfz_{nj},\bbeta,\balpha_{z_{n}},\balpha_{z_{j}}) = p(\bfy_n,\bfz_n|\bbeta,\balpha_{z_{n}})
\]
To relate this formulation to the MOFA model, the \textit{local} variables would contain the factors whereas the \textit{global} variables would contain the feature weights.

For simplicity in the derivation, we will assume the existence of a single \textit{global} variable $\beta$, a single parameter $\alpha_{\beta}$ for the \textit{global} variables and a single parameter $\alpha_{z_{nk}}$ for each \textit{local} variable.

The first assumption in the model is that the prior distributions of the \textit{local} and \textit{global} variables are members of the exponential family (see \Cref{eq:exponential_family})
\begin{align} \label{eq_priors}
	\begin{split}
	p(\beta|\alpha_{\beta}) = h(\beta) \exp\{ \eta_g(\alpha_{\beta}) t(\beta) - a_g(\alpha_{\beta}) \} \\
	p(z_{nk}|\alpha_{z}) = h(z_{nk}) \exp\{ \eta_l(\alpha_{z}) t(z_{nk}) - a_l(\alpha_{z}) \}
	\end{split}
\end{align}

The second assumption is that the complete conditionals of the unobserved variables are also members of the exponential family:
\begin{align} \label{eq_complete_conditionals}
	\begin{split}
	p(\beta|\bfY,\bfZ,\balpha) = h(\beta) \exp\{ \eta_g(\bfY,\bfZ,\balpha)^T t(\beta) - a_g(\eta_g(\bfY,\bfZ,\balpha)) \} \\
	p(\bfz_{n}|\bfy_{nj},\bfz_{nj},\beta) = h(\bfz_{n}) \exp\{ \eta_l(\bfy_{nj}, \bfz_{nj},\beta)^T t(\bfz_{n}) - a_l(\eta_l(\bfy_{nj},\bfz_{nj},\beta)) \}
	\end{split}
\end{align}

\subsubsection{Setting up the inference problem}

 First, we set up the variational distributions for both the \textit{local} variables and the \textit{global} variables. Here we are going to assume that all unobserved variables are independent (mean-field assumption)
\[
	q(\bfz,\beta) = q(\beta|\lambda) \prod_{n=1}^{N} \prod_{k=1}^{K} p(z_{nk}|\phi_{nk})
\]
and belong to the same exponential family as the corresponding prior distribution:
\begin{align} \label{eq_variational_distributions}
	q(\beta|\lambda) &= h(\beta) \exp\{ \eta_g(\lambda) t(\beta) - a_g(\lambda) \} \\
	q(z_{nk}|\phi_{nk}) &= h(z_{nk}) \exp \{ \eta_l(\phi_{n}) t(z_{nk}) - a_l(z_{nk}) \}
\end{align}
where $\lambda$ are the parameters governing the variational distribution for the \textit{global} variables and $\phi_{nk}$ are the parameters governing the variational distribution for the $k$-th \textit{local} variable and the $n$-th sample.

From the assumptions above, the ELBO (the objective function in variational inference, introduced in Chapter 3) factorises as:
\begin{align} \label{eq_elbo_factorised} \begin{split}
	\Lagr &= \E_{q(\bfZ,\beta)}[\log p(\bfY,\bfZ,\beta)] - \E_{q(\bfZ)}[\log q(\bfZ)] - \E_{q(\beta)}[\log q(\beta)] \\
	 &= \sum_{n=1}^{N} \E_{q(\bfz_n,\beta)}[\log p(\bfy_n,\bfz_n,\beta)] - \sum_{n=1}^{N} \sum_{k=1}^{K} \E_{q(z_{nk})}[\log q(z_{nk})] - \E_{q(\beta)}[\log q(\beta)]
\end{split} \end{align}

Notice that the objective decomposes into \textit{global} terms (not involving $N$) and \textit{local} terms (involving $N$). Importantly, the \textit{local} terms can be approximated using estimates of the gradient by subsampling the data set. Assuming a mini-batch of size $S$:
\[
	\hat{\Lagr} = \frac{N}{S} \sum_{n=1}^{S} \E_{q(\bfz_n,\beta)}[\log p(\bfy_n,\bfz_n,\beta)] - \frac{N}{S}\sum_{s=1}^{S} \sum_{k=1}^{K} \E_{q(z_{nk})}[\log q(z_{nk})] - \E_{q(\beta)}[\log q(\beta)]
\]
If the samples are independent then the expectation of this noisy gradient is equal to the true gradient. This is the main principle of stochastic optimisation. The next step is to derive an iterative algorithm to find the values of the variational parameters that maximise the ELBO.

\subsubsection{Calculating the gradient for the \textit{global} parameters}

To derive the updates for the \textit{global} parameters we first write the ELBO in terms of $\lambda$:
\[
	\Lagr(\lambda) = \E_{q(z,\beta)}[\log p(\beta|\bfY,\bfZ)] - \E_{q(\beta)}[\log q(\beta)] + \const
\]
where the constant term captures all quantities that do not depend on $\lambda$. Then, from the assumption that the complete conditionals and the variational distributions belong to the exponential family (\Crefrange{eq_complete_conditionals}{eq_variational_distributions}):
\baln
	\Lagr(\lambda) &= \E_{q(z,\beta)}[\eta_g(\bfY,\bfZ,\balpha)^T t(\beta)] - \E_{q(\beta)}[\lambda^T t(\beta) - a_g(\lambda) ] + \const \\
	&= \E_{q(z)}[\eta_g(\bfY,\bfZ,\balpha)^T] \nabla a(\lambda) - \lambda^T \nabla a_g(\lambda) - a_g(\lambda) + \const
\ealn
where we have used the exponential family identity $\E_{q(\beta)}[t(\beta)] = \nabla a_g(\lambda)$.

Taking the gradient with respect to $\lambda$:
\begin{equation} \label{gradient_global}
	\nabla_{\lambda} \Lagr(\lambda) = \nabla_{\lambda}^{2} a_g(\lambda)(\E_{q(z)}[\eta_g(\bfY,\bfZ,\balpha)] - \lambda)
\end{equation}
and setting it to zero leads to the solution:
\begin{equation} \label{solution_global}
	\lambda = \E_{q(z)}[\eta_g(\bfY,\bfZ,\balpha)]
\end{equation}

\subsubsection{Calculating the gradient for the \textit{local} parameters}

Turning to the \textit{local} parameters, as a function of $\phi_{nk}$ the ELBO becomes:
\[
	\Lagr(\phi_{nk}) = \E_{q(\beta,\bfz_{nj})}[\log p(\bfz_{nj}|\bfy_{n},\bfz_{nj}, \beta)] - \E_{q(z_{nk})}[\log q(z_{nk})] + \const
\]
Again, from the assumption that the complete conditionals and the variational distributions belong to the exponential family (\Crefrange{eq_complete_conditionals}{eq_variational_distributions}):
\begin{align*}
	\Lagr(\phi_{nk}) &= \E_{q(\beta,\bfz_{nj})}[\eta_l(\bfy_n,\bfz_{nj},\beta)^T t(\bfz_{nj})] - \E_{q(z_{nk})}[\phi_{nk} t(z_{nk}) - a_l(\phi_{nk}) ] + \const \\
	&= \E_{q(\beta,\bfz_{nj})}[\eta_l(\bfy_n,\bfz_{nj},\beta)]^T \nabla a_l(\phi_{nk}) - \phi_{nk} \nabla a_l(\phi_{nk}) - a_l(\phi_{nk}) + \const
\end{align*}

Taking the gradient with respect to $\phi_{nk}$:
\begin{equation} \label{gradient_local}
	\nabla_{\phi} \Lagr(\phi_{nk}) = \nabla_{\phi}^2 a_l(\phi_{nk}) (\E_{q(\beta,\bfz_{nj})}[\eta_l(\bfy_n,\bfz_{nj},\beta)] - \phi_{nk})
\end{equation}

and setting it to zero leads to the following solution:
\begin{equation} \label{solution_local}
	\phi_{nk} = \E_{q(\beta,\bfz_{nj})}[\eta_l(\bfy_{n},\bfz_{nj},\beta)]
\end{equation}

\subsubsection{Coordinate ascent variational inference algorithm}

Now that we have the gradients for both the \textit{local} and the \textit{global} parameters, we can define a gradient ascent algorithm to optimise the model:

\begin{algorithm*}
  \caption{Coordinate ascent variational inference algorithm}
  \begin{algorithmic}[1]
	\State Initialise the \textit{global} parameters $\blambda^{(t=0)}$
	\Repeat
		\For{\text{each \textit{local} variational parameter $\phi_{nk}$}}
			\State $ \phi_{nk}^{(t+1)} \gets \E_{q(\bbeta,\bfz_{nj})^t}[\eta_l(\bfy_{n},\bfz_{nj},\bbeta)] $
      	\EndFor
		\For{\text{each \textit{global} variational parameter $\lambda$}}
			\State $ \lambda^{(t+1)}= \E_{q(z)^t}[\eta_g(\bfY,\bfZ,\balpha)] $
      	\EndFor
	\Until{Convergence}
	\end{algorithmic}
\end{algorithm*}

However, as discussed in \Cref{section:natural_gradient_ascent}, the use of Euclidean-based gradients ignores important information about the geometry of the distribution and is thus not optimal for the optimisation of probabilistic models. Next, we will derive a similar coordinate ascent algorithm but using instead the natural gradient.

\subsubsection{Deriving the natural gradients for the \textit{global} variational parameters}

From \Cref{gradient_global}, the gradient of the ELBO with respect to the \textit{global} parameters $\lambda$ is:
\[
	\nabla_{\lambda} \Lagr(\lambda) = \nabla_{\lambda}^{2} a_g(\lambda)(\E_{q(z)}[\eta_g(\bfY,\bfZ,\balpha)] - \lambda)
\]
Premultiplying by $\bfF(\beta)^{-1}=\nabla_{\lambda}^{2} a_g(\lambda)$ gives the natural gradient for the \textit{global} parameters:
\[
	\hat{\nabla}_{\lambda} \Lagr(\lambda) = \E_{q(z)}[\eta_g(\bfY,\bfZ,\balpha)] - \lambda
\]

\subsubsection{Deriving the natural gradients for the \textit{local} variational parameters}

From \Cref{gradient_local}, the gradient of the ELBO with respect to the \textit{local} parameters $\bphi$ is:
\[
	\nabla_{\phi} \Lagr(\phi_{nk}) = \nabla_{\phi}^2 a_l(\phi_{nk}) (\E_{q(\beta,\bfz_{nj})}[\eta_l(\bfy_n,\bfz_{nj},\beta)] - \phi_{nk})
\]
Premultiplying by $\bfF(z_{nk})^{-1}=\nabla_{\phi}^{2} a_l(\phi_{nk})$ gives the natural gradient for the \textit{global} parameters:
\[
	\hat{\nabla}_{\phi} \Lagr(\phi_{nk}) = \E_{q(\beta,\bfz_{nj})}[\eta_l(\bfy_n,\bfz_{nj},\beta)] - \phi_{nk}
\]

Remarkably, the natural gradient for both the \textit{local} and \textit{global} variational parameters is simply the standard gradient subtracting the current value of the parameters. Thus, the Fisher Information matrix does \textit{not} need to be explicitly computed at each iteration, which leads to a considerable simplification of the problem.

\subsubsection{Stochastic variational inference algorithm using natural gradients}

After replacing the Euclidean gradient with the natural gradients, the model can be trained using the following stochastic algorithm based on gradient descent (\Cref{algorithm:svi}). Notice that the stochastic variational inference algorithm introduces additional hyperparameters:

\begin{itemize}
    \item \textbf{Batch size}: controls the number of samples that are used to compute the gradients at each iteration. A trade off exists where large batch sizes lead to a more expensive computation of the gradient but yield a less noisy estimate.

    \item \textbf{Learning rate}: The learning rate $p(t)$ controls the step size in the direction of the natural gradient, with high learning rates leading to higher steps. In the natural gradient setting, the learning rate also controls how much memory from previous iterations is translated to the current updates. The particular case of a constant learning rate of $1$ yields no memory from previous iterations (thus simplifies to standard gradient ascent). To ensure proper convergence, the learning rate has to be decayed during training. Several strategies exist \cite{Ranganath2013}, here we used the simple function $\rho(t) = \frac{\rho_0}{(1 + \kappa t)^{3/4}}$, which introduces two extra hyperparameters: (1) The forgetting rate $\kappa$, which controls the decay of the learning rate, and $\rho_0$ which determines the initial learning rate.

\end{itemize}

\begin{algorithm}
  \caption{Stochastic variational inference algorithm using natural gradients}\label{algorithm:svi}
  \begin{algorithmic}[1]
	\State Initialise the \textit{global} parameters $\blambda^{(t=0)}$.
	\State Initialise step size $\rho^{(t=0)}$
	\Repeat
	    \State \text{sample $\mathcal{B}$ a mini-batch of samples of size $S$}
		\For{\text{each \textit{local} variational parameter $\phi_{nk}$ such that $n$ is in batch $\mathcal{B}$}}
			\State $$ \phi_{nk}^{(t+1)} = \E_{q^{(t)}(\bbeta,\bfz_{nj})}[\eta_l(\bfy_{n},\bfz_{nj},\bbeta)] $$
      	\EndFor
		\For{\text{each \textit{global} variational parameter $\lambda$}}
		     \State
        		\begin{align*} \begin{split}
            	\lambda^{(t + 1)} &=  (1-\rho ^{(t)})\lambda^{(t)} +  \rho ^{(t)} \hat{\nabla}_{\lambda} \Lagr^S(\lambda) \\
            	 &=  (1-\rho ^{(t)})\lambda^{(t)} +  \rho ^{(t)} \E_{q^{(t+1)}(z)}\left[\frac{N}{S}\eta_g(\bfY_{[n \in \mathcal{B}],:},\bfZ_{[n \in \mathcal{B}],:},\balpha)\right]
            \end{split} \end{align*}
            \State where $[n \in \mathcal{B}]$ denotes the subset of indices corresponding to the samples in $\mathcal{B}$
      	\EndFor
	\Until{Convergence}
	\end{algorithmic}
\end{algorithm}


\documentclass[]{article}

%opening
\title{}
\author{}

\begin{document}

\maketitle

Example: f(x) = $ 3^{-x} $ on [0,1]

\section*{Closed Interval Method}

\begin{enumerate}
	\item Critical points are x = 0 \& x = 1
	\item f'(x) = $ -3^{-x} \cdot \ln 3 \ne 0 $
	\item f' always exists
	\item $ f(0) = 1 $
	\item $ f(1) = \frac{1}{3} $
	\item $ f : [0,1] -> [\frac{1}{3}, 1] \subset [0,1] $
	\item => There is a fixed point
	\item |f'(0)| = ln3 > 1
	\item No guarantee of uniqueness
\end{enumerate}

\section*{Interpolation and Polynomial Approximation}

Interpolation refers to determining a function that exactly represents a collection of data. We need to compute values for a tabulated function at a point not in the table.

We will find a polynomial that fits a selected set of points $ (x_{i}, f(x_i)) $ \& assumes the polynomial and function are exactly the same (1st degree polynomial is called linear interpolation).

Goal is to replace some times a function by a simpler one.

\section*{Applications}

\begin{itemize}
	\item Performance of a new rocket.
	Signals are received every 10 sec. Interpolation gives the position of the rocket as well as other information.
	\item Astronomy:
	When the motion of heavenly bodies is determined from periodic observations.
\end{itemize}

\section*{Lagrange Polynomials}

\begin{tabular}{c | c}
	 $ 	x  $ & $  f(x)   $ \\
$ 	x_0 $  &  $ f(x_0) $  \\
$ 	x_1 $  &  $ f(x_1)  $
\end{tabular}

$ P_1(x) = \frac{x-x_0}{x_0 - x_1}f_0 + \frac{x - x_1}{x_1 - x_0}f_1 $

\end{document}


\section{Project Aims}
The main aim of this project is to design a framework which acts as an intermediate stage in HDL synthesis from Fortran code onto
an FPGA. We restrict the framework to using a pure functional approach; representing a program as vector structures\cite{transformations} operated
on with a combination of element processing functions with map, fold, and zip operations.

This approach allows us to perform type transformations on the computations to tune the amount of parallelism
whilst still preserving the results of the computations.

We aim to model the framework as an EDSL (Embedded Domain Specific Language); this saves the tedious work
of having to implement a custom DSL (Domain Specific Language) for this specific purpose, allowing us to use the
language constructs of the host language used to implement the framework. This also has the added benefit of being able to interpret
these programs on a typical desktop computing platform which can be useful for testing purposes before transforming into a
HDL.

We also aim to implement a rudimentary cost model which should be able to be replaced with more complex
cost models in the future. Each given function has a given performance and space cost, represented as integers.
The lower the integer the lower the performance cost, and vice versa. The same goes for the space cost.
For example given a function with performance cost \textbf{p}, and space cost \textbf{s} which is being mapped on over
a 1 dimensional vector with \textbf{n} elements; performing the map entirely in parallel would incur a performance
cost of \textbf{p}, and space cost of \textbf{n * s}. Performing the map sequentially would incur a performance cost of
\textbf{n * p}, and a space cost of \textbf{s}. A higher amount of parallelization increases the space cost but reduces
the performance cost, a smaller amount of parallelization reduces the space cost but increases the
performance cost.

Another aim is to also be able to create an interface for converting the optimal program for the given cost model
into an Intermediate Representation(IR) format which is then itself processed and converted to a HDL.

The focus of this project isn't on translating Fortran code into the EDSL, or directly translating the optimal program
into a HDL.


\documentclass{subfile}

\begin{document}
	\section{BrNO}\label{sec:brmo}

		\begin{problem}[$2018$, problem $9$]
			Let $n\geq2$ be an integer. Prove the inequality
				\begin{align*}
					\dfrac{1}{2!}+\ldots+\dfrac{2^{n-2}}{n!}
						& \leq\dfrac{3}{2}
				\end{align*}
		\end{problem}

		\begin{problem}[$2015$, Day $2$, problem $1$]
			Find all real number $x\geq-1$ such that the inequality
				\begin{align*}
					\dfrac{a_{1}+x}{2}\cdots\dfrac{a_{n}+x}{2}
						& \leq \dfrac{a_{1}\cdots a_{n}+x}{2}
				\end{align*}
			holds for all positive integer $n\geq2$ and for all real numbers $a_{1},\ldots,a_{n}\geq1$.
		\end{problem}

		\begin{problem}[$2014$ Final Round]
			Find all values $\lambda$ such that the inequality
				\begin{align*}
					\dfrac{a+b}{2}
						& \geq\lambda\sqrt{ab}+(1-\lambda)\sqrt{\dfrac{a^{2}+b^{2}}{2}}
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Final Round]
			Prove that for all positive real numbers $x$ and $y$,
				\begin{align*}
					\dfrac{1}{x+y+1}-\dfrac{1}{(x+1)(y+1)}
						& < \dfrac{1}{11}
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Test $2$]
			Let $a,b,c$ be positive real numbers such that
				\begin{align*}
					ab+bc+ca
						& \geq a+b+c
				\end{align*}
			Prove that
				\begin{align*}
					(a+b+c)(ab+bc+ca)+3abc
						& \geq 4(ab+bc+ca)
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Test $4$]
			Let $a,b,c$ be positive real numbers such that $a+b+c=1$. Prove that
				\begin{align*}
					\dfrac{a^{2}}{(b+c)^{3}}+\dfrac{b^{2}}{(c+a)^{3}}+\dfrac{c^{3}}{(a+b)^{3}}
						& \geq \dfrac{9}{8}
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Test $6$]
			Let $a,b,c$ be real numbers in the interval $(0,2)$ such that $a+b+c=ab+bc+ca$. Prove that
				\begin{align*}
					\dfrac{a^{2}}{a^{2}-a+1}+\dfrac{b^{2}}{b^{2}-b+1}+\dfrac{c^{2}}{c^{2}-c+1}
						& \leq3
				\end{align*}
		\end{problem}

		\begin{problem}[$2013$ Test $3$]
			Let $n\geq3$ be an integer and $x_{1},\ldots,x_{n}$ be positive real numbers such that $x_{1}\cdots x_{n}=1$. Prove that
				\begin{align*}
					\dfrac{x_{1}^{8}}{(x_{1}^{4}+x_{2}^{4})x_{2}}+\dfrac{x_{n}^{8}}{(x_{n}^{4}+x_{1}^{4})x_{1}}
						& \geq\dfrac{n}{2}
				\end{align*}
		\end{problem}

		\begin{problem}[$2013$ Test $7$]
			Given positive real numbers $a,b,c$ such that
				\begin{align*}
					\dfrac{1}{ab}+\dfrac{1}{bc}+\dfrac{1}{ca}
						& = 1
				\end{align*}
			Prove that
				\begin{align*}
					\dfrac{a^{2}+b^{2}+c^{2}+ab+bc+ca-3}{5}
						& \geq \dfrac{a}{b}+\dfrac{b}{c}+\dfrac{c}{a}
				\end{align*}
		\end{problem}

		\begin{problem}[$2012$ Test $1$]
			Let $a,b,c$ be real numbers such that $0<a<b<c$. Prove that
				\begin{align*}
					a^{20}b^{12}+b^{20}c^{12}+c^{20}a^{12}
						& < b^{20}a^{12}+c^{20}b^{12}+b^{20}a^{12}
				\end{align*}
		\end{problem}

		\begin{problem}[$2011$ Round $3$]
			Let $a,b,x,y$ be positive real numbers such that
				\begin{align*}
					ab
						& \geq xa+yb
				\end{align*}
			Prove that
				\begin{align*}
					\sqrt{a+b}
						& \geq\sqrt{x}+\sqrt{y}
				\end{align*}
		\end{problem}

		\begin{problem}[$2011$ Round $3$]
			Let $a,b,c,k,l$ and $m$ be positive real numbers such that
				\begin{align*}
					abc
						& \geq ka+lb+mc
				\end{align*}
			Prove that
				\begin{align*}
					a+b+c
						& \geq \sqrt{3}(\sqrt{k}+\sqrt{l}+\sqrt{m})
				\end{align*}
		\end{problem}

		\begin{problem}[$2011$ Final Round]
			Let $a,b,c$ be positive real numbers such that
				\begin{align*}
					a^{2}+b^{2}+c^{2}
						& = 3
				\end{align*}
			Prove that
				\begin{align*}
					a+b+c
						& \geq ab+bc+ca
				\end{align*}
		\end{problem}

		\begin{problem}[$2011$ Test $5$]
			Let $a,b,c$ be positive real numbers such that
				\begin{align*}
					\dfrac{a}{b+c}+\dfrac{b}{c+a}+\dfrac{c}{a+b}
						& = 1+\dfrac{1}{6}\left(\dfrac{a}{c}+\dfrac{c}{b}+\dfrac{b}{a}\right)
				\end{align*}
			Prove that
				\begin{align*}
					\dfrac{a^{3}bc}{b+c}+\dfrac{b^{3}ca}{c+a}+\dfrac{c^{3}ab}{a+b}
						& \geq \dfrac{1}{6}(ab+bc+ca)^{2}
				\end{align*}
		\end{problem}

		\begin{problem}[$2010$ Final Round]
			Prove that
				\begin{align*}
					\dfrac{a^{2}}{a+b}+\dfrac{b^{2}}{b+c}
						& \geq \dfrac{3a+2b-c}{4}
				\end{align*}
			for all positive real numbers $a,b$ and $c$.
		\end{problem}

		\begin{problem}[$2010$ Test $1$]
			Given non-negative real numbers $a,b$ and $c$ such that $a+b+c=1$. Prove that
				\begin{align*}
					\left(a^{2}+b^{2}+c^{2}\right)^{2}+6abc
						& \geq ab+bc+ca
				\end{align*}
		\end{problem}

		\begin{problem}[$2010$ Test $7$]
			Prove that all positive real numbers $x,y$ and $z$ satisfy the inequality
				\begin{align*}
					x^{y}+y^{z}+z^{x}
						& > 1
				\end{align*}
		\end{problem}

		\begin{problem}[$2010$ Test $8$]
			Let $a,b,c$ be positive real numbers such that $abc=1$. Prove that
				\begin{align*}
					\dfrac{a}{b(a+b)}+\dfrac{b}{c(b+c)}+\dfrac{c}{a(c+a)}
						& \geq \dfrac{3}{2}
				\end{align*}
		\end{problem}

		\begin{problem}[$2009$ Selection and Training Session]
			Let $a,b$ and $c$ be positive real numbers. Prove that
				\begin{align*}
					\dfrac{1}{(a+b)b}+\dfrac{1}{(b+c)c}+\dfrac{1}{(c+a)c}
						& \geq \dfrac{9}{2(ab+bc+ca)}
				\end{align*}
		\end{problem}

		\begin{problem}[$2008$, Day $2$, problem $1$]
			Let $x_{1},\ldots,x_{n}$ be non-negative real numbers. Prove that
				\begin{align*}
					\dfrac{x_{1}(2x_{1}-x_{2}-x_{3})}{x_{2}+x_{3}}+\ldots+\dfrac{x_{n}(2x_{n}-x_{1}-x_{2})}{x_{1}+x_{2}}
						& \geq0
				\end{align*}
		\end{problem}

		\begin{problem}[$2002$]
			For any positive integers $a$ and $b$, prove that
				\begin{align*}
					|a\sqrt{2}-b|
						& > \dfrac{1}{2(a+b)}
				\end{align*}
		\end{problem}

		\begin{problem}[$2002$]
			Given positive real numbers $a,b,c$ and $d$. Prove that
				\begin{align*}
					\sqrt{(a+c)^{2}+(b+d)^{2}}
						& \leq \sqrt{a^{2}+b^{2}}+\sqrt{c^{2}+d^{2}}\leq \sqrt{(a+c)^{2}+(b+d)^{2}}+\dfrac{2|ad-bc|}{\sqrt{(a+c)^{2}+(b+d)^{2}}}
				\end{align*}
		\end{problem}
\end{document}


\documentclass[../main.tex]{subfiles}

\begin{document}

\setcounter{chapter}{0}
\chapter{Étale morphisms}

\oldpage{1}To simplify the exposition we assume that all preschemes in the following are locally Noetherian (at least, starting from section 2).

\section{Basics of differential calculus}

Let $X$ be a prescheme on $Y$, and $\Delta_{X/Y}$ the diagonal morphism $X\to X\times_Y X$.
This is an immersion, and thus a closed immersion of $X$ into an open subset $V$ of $X\times_Y X$.
Let $\ideal_X$ be the ideal of the closed sub-prescheme corresponding to the diagonal in $V$ (N.B. if one really wishes to do things intrinsically, without assuming that $X$ is separated over $Y$ — a \unsure{misleading} hypothesis — then one should consider the set-theoretic inverse image of $\O_{X\times X}$ in $X$ and denote by $\ideal_X$ the augmentation ideal in the above\ldots).
The sheaf $\ideal_X/\ideal_X^2$ can be thought of as a quasi-coherent sheaf on $X$, which we denote by $\Omega_{X/Y}^1$.
It is of finite type if $X\to Y$ is of finite type.
It behaves well with respect to a change of base $Y'\to Y$.
We also introduce the sheaves $\O_{X\times_Y X}/\ideal_X^{n+1}=\mathscr{P}^n_{X/Y}$, which are sheaves of \emph{rings} on $X$, giving us preschemes denoted by $\Delta_{X/Y}^n$ and called the \emph{$n$-th infinitesimal neighbourhood of $X/Y$}.
The polysyllogism is entirely trivial, even if rather long\footnote{cf. EGA~IV~16.3.}; it seems wise to not discuss it until we use it for something helpful, with smooth morphisms.

\section{Quasi-finite morphisms}

\begin{prop}
    Let $A\to B$ be a local homomorphism (N.B. all rings are now Noetherian) and $\mathfrak{m}$ the maximal ideal of $A$.
    Then the following conditions are equivalent:
    \begin{enumerate}[\normalfont(i)]
        \item $B/\mathfrak{m}B$ is of finite dimension over $k=A/\mathfrak{m}$.
        \item $\mathfrak{m}B$ is \unsure{[\ldots]} and $B/\mathfrak{r}(B)=k(B)$ is an extension of $k=k(A)$.
        \item The completion $\hat{B}$ of $B$ is finite over the completion $\hat{A}$ of $A$.
    \end{enumerate}
\end{prop}

\oldpage{2}We then say that $B$ \emph{is quasi-finite} over $A$.
A morphism $f\colon X\to Y$ is said to be quasi-finite at $x$ (or the $Y$-prescheme $f$ is said to be quasi-finite at $x$) if $\O_x$ is quasi-finite over $\O_{f(x)}$.
\completelyunsure
A morphism is said to be quasi-finite if it is quasi-finite in each point\footnote{In EGA~II~6.2.3 we further suppose that $f$ is of finite type.}.

\begin{cor}
    If $A$ is complete then quasi-finiteness is equivalent to finiteness.
\end{cor}

We could give the usual polysyllogism (i), (ii), (iii), (iv), (v) for quasi-finite morphisms, but that doesn't seem necessary here.

\section{Unramified morphisms}

\begin{prop}
    Let $f\colon X\to Y$ be a morphism of finite type, $x\in X$, and $y=f(x)$.
    Then the following conditions are equivalent:
    \begin{enumerate}[\normalfont(i)]
        \item $\O_x/\mathfrak{m}_y\O_x$ is a finite separable extension of $k(y)$.
        \item $\Omega_{X/Y}^1$ is \unsure{null} at $x$.
        \item The diagonal morphism $\Delta_{X/Y}$ is an open immersion on a neighbourhood of $x$.
    \end{enumerate}
\end{prop}

For the implication (i) $\implies$ (ii), we are brought by Nakayama to the case where $Y=\Spec(k)$ and $X=\Spec(k')$, where it is well known and otherwise trivial by the definition of separable; (ii) $\implies$ (iii) comes from a nice and easy characterisation of open immersions, using Krull; (iii) $\implies$ (i) follows as well from reducing to the case where $Y=\Spec(k)$ and the diagonal morphism is everywhere an open immersion.
One must then prove that $X$ is finite \completelyunsure and this leads us to consider the case where $k$ is algebraically closed.
But then every closed point of $X$ is isolated (since it is identical to the inverse image of the diagonal by the morphism $X\to X\times_k X$ defined by $x$), whence $X$ is finite.
We can thus suppose that $X$ reduces to a point, of the ring $A$, and so $A\otimes_k A\to A$ is an isomorphism, hence $A=k$.\qed

\begin{defn}
    \begin{enumerate}[\normalfont(a)]
        \item We then say that $f$ is \emph{unramified} at $x$, or that $X$ is unramified at $x$ on $Y$.
        \item Let $A\to B$ be a local homomorphism.
            We say that it is \emph{unramified}, or that $B$ is a local \emph{unramified} algebra on $A$, if $B/mathfrak{r}(A)B$ is a finite separable extension of $A/\mathfrak{r}(A)$, i.e. if $\mathfrak{r}(A)B=\mathfrak{r}(B)$ and $k(B)$ is a separable extension of $k(A)$\footnote{Cf. regrets in III~1.2.}.
    \end{enumerate}
\end{defn}

\oldpage{3}\begin{rem}
    The fact that $B$ is unramified over $A$ \completelyunsure.
    Unramified implies quasi-finite.
\end{rem}

\begin{cor}
    The set of points where $f$ is unramified is open.
\end{cor}

\begin{cor}
    Let $X'$ and $X$ be two preschemes of finite type over $Y$, and $g\colon X'\to X$ a $Y$-morphism.
    If $X$ is unramified over $Y$ then the graph morphism $\Gamma_g\colon X'\to X\times_Y X$ is an open immersion.
\end{cor}

In effect, this is the inverse image of the diagonal morphism $X\to X\times_Y X$ by
\begin{equation*}
    g\times_Y \id_{X'}\colon X'\times_Y X\to X\times_Y X.
\end{equation*}
One can also introduce the annihilator ideal $\mathfrak{d}_{X/Y}$ of $\Omega_{X/Y}^1$, called the \unsure{different} ideal of $X/Y$; it defines a closed sub-prescheme of $X$ which, set theoretically, is the set of point where $X/Y$ is ramified, i.e. not unramified.

\begin{prop}
    \begin{enumerate}[\normalfont(i)]
        \item An immersion is ramified.
        \item The composition of two ramified morphisms is also ramified.
        \item Base extension of a ramified morphisms is also ramified.
    \end{enumerate}
\end{prop}

\unsure{We don't care so much about (ii) or (iii)} (the second seems more \unsure{entertaining} to me).
We can also \completelyunsure by giving a few punctual comments; it is no more general than in appearance (except in the case of definition b) and also boring.
We obtain, as per usual, the corollaries:

\begin{cor}
    \begin{enumerate}[\normalfont(i)]
        \setcounter{enumi}{3}
        \item The cartesian product of two unramified morphisms is unramified.
        \item If $gf$ is unramified then so too is $f$.
        \item If $f$ is unramified then so too is $f_\text{red}$.
    \end{enumerate}
\end{cor}

\begin{prop}
    Let $A\to B$ be a local homomorphism and suppose that the residue extension $k(B)/k(A)$ is trivial with $k(A)$ algebraically closed.
    In order for $B/A$ to be unramified it is necessary and sufficient that $\hat{B}$ be (as an $\hat{A}$-algebra) a quotient of $\hat{A}$.
\end{prop}

\begin{rem}
    \begin{itemize}
        \item In the case where we don't suppose that the residue extension is trivial, we can bring ourselves to the case where it is by taking a suitable finite flat extension on $A$ which \unsure{destroys} the aforementioned extension.
        \item Consider the example where $A$ is the local ring of an ordinary double point of a curve, and $B$ a point of the \unsure{normalised curve}: then $A\subset B$, $B$ is unramified over $A$ with trivial residue extension,\oldpage{4} and $\hat{A}\to\hat{B}$ is surjective but \emph{not injective}.
        We are thus going to strengthen the notion of unramifiedness.
    \end{itemize}
\end{rem}

\section{Étale morphisms. Étale covers.}

We are going to suppose all that will be necessary concerning flat morphisms; these facts will be proved later, if there is time\footnote{Cf. Exp.~IV.}.

\begin{defn}
    \begin{enumerate}[\normalfont a)]
        \item Let $f\colon X\to Y$ be a morphism of finite type.
            We say that $f$ is \emph{étale} at $x$ if $f$ is both flat and unramified at $x$.
            We say that $f$ is étale if it is étale at all points.
            We say that $X$ is étale at $x$ over $Y$, or that it is a $Y$-prescheme étale at $x$ etc.
        \item Let $f\colon A\to B$ be a local homomorphism.
            We say that $f$ is étale, or that $B$ is étale over $A$, if $B$ is flat and unramified over $A$\footnote{Cf. regrets in III~1.2.}.
    \end{enumerate}
\end{defn}

\begin{prop}
    For $B/A$ to be étale, it is necessary and sufficient that $\hat{B}/\hat{A}$ be étale.
\end{prop}

\oldpage{5}In effect, this is true individually for both ``unramified'' and ``flat''.

\begin{cor}
    Let $f\colon X\to Y$ be of finite type, and $x\in X$.
    The property of $f$ being étale at $x$ depends only on the local homomorphism $\O_{f(x)}\to\O_x$, and in fact only on the corresponding homomorphism for the completions.
\end{cor}

\begin{cor}
    Suppose that the residue extension $k(A)\to k(B)$ is trivial, or that $k(A)$ is algebraically closed.
    Then $B/A$ is étale if and only if $\hat{A}\to\hat{B}$ is an isomorphism.
\end{cor}

We combine the platitude and 3.7.

\begin{prop}
    Let $f\colon X\to Y$ be a morphism of finite type.
    Then the set of points where $f$ is étale is open.
\end{prop}
Again, this is in fact true individually for both ``unramified'' and ``flat''.

This proposition shows that we can forget about the punctual comments in the study of morphisms of finite type that are somewhere étale.

\begin{prop}
    \begin{enumerate}[\normalfont(i)]
        \item An open immersion is étale.
        \item The composition of two étale morphisms is étale.
        \item The base extension of an étale morphism is étale.
    \end{enumerate}
\end{prop}

In effect, (i) is trivial, and for (ii) and (iii) it suffices to note that it is true for ``unramified'' and ``flat''.
As a matter of fact, there are also corresponding comments to make about local homomorphisms (without the finiteness condition), which in any case should figure in the \marginpar{\small Grothendieck's \emph{multiplodoque d'algèbre homologique} was the final version of his \emph{Tohoku paper} — see (2.1) in `Life and work of Alexander Grothendieck' by Ching-Li Chan and Frans Oort for more information. [trans.]}multiplodoque (starting with the case of unramified).

\begin{cor}
    A cartesian product of two étale morphisms is étale.
\end{cor}

\begin{cor}
    Let $X$ and $X'$ be of finite type over $Y$, and $g\colon X\to X'$ a $Y$-morphism.
    If $X'$ is unramified over $Y$ and $X$ is étale over $Y$, then $g$ is étale.
\end{cor}

In effect, $g$ is the composition of the graph morphism $\Gamma_g\colon X\to X\times_Y X'$ (which is an open immersion by 3.4) and the projection morphism, which is étale since it is just a ``change of base'' by $X'\to Y$ of the étale morphism $X\to Y$.

\begin{defn}
    We say that a cover of $Y$ is étale (resp. unramified) if it is a $Y$-scheme $X$ that is finite over $Y$ and étale (resp. unramified) over $Y$.
\end{defn}
The first condition means that $X$ is defined by a coherent sheaf of algebras $\mathscr{B}$ over $Y$.
The second means that $\mathscr{B}$ is locally free over $Y$ (resp. \unsure{absolutely nothing}) \emph{and} further that, for all $y\in Y$, the fibre $\mathscr{B}(y)=\mathscr{B}_y\otimes_{\O_y}k(y)$ is a separable algebra (i.e. a finite composition of finite separable extensions) over $k(y)$.

\begin{prop}
    Let $X$ be a flat cover of $Y$ of degree $n$ (the definition of this term deserved to figure in 4.9) defined by a locally free coherent sheaf $\mathscr{B}$ of algebras.
    We define, as usual, the trace homomorphism $\mathscr{B}\to\mathscr{A}$ (that is a homomorphism of $\mathscr{A}$-modules, where $\mathscr{A}=\O_Y$).
    For $X$ to be étale it is necessary and sufficient that the corresponding bilinear form $\tr_{\mathscr{B}/\mathscr{A}}xy$ defines an isomorphism of $\mathscr{B}$ over $\mathscr{B}$, or, equivalently, that the \emph{discriminant section}
    \begin{equation*}
        d_{X/Y}=d_{\mathscr{B}/\mathscr{A}}\in\Gamma(Y,\wedge^n\check{\mathscr{B}}\otimes_\mathscr{A}\wedge^n\check{\mathscr{B}})
    \end{equation*}
    is invertible, or that the discriminant ideal defined by this section is the unit ring.
\end{prop}

In effect, we can reduce to the case where $Y=\Spec(k)$, and then it is a well-known criterion of separability, and trivial by passing to the algebraic closure of $k$.

\begin{rem}
    We will have a less trivial statement to make later on, \oldpage{6}when we do not suppose a priori that $X$ is flat over $Y$, but instead require some normality hypothesis.
\end{rem}

\section{The fundamental property of étale morphisms}

\begin{thm}
    Let $f\colon X\to Y$ be a morphism of finite type.
    For $f$ to be an open immersions, it is necessary and sufficient for it to be \emph{étale and radical}.
\end{thm}

Recall what radical means:

\end{document}


\section{Properties of Power Spectral Density (PSD)}

\subsection*{Approximation in the definition of PSD}

Equivalence is shown by starting from equation (9), as indicated in the coursework instructions.

\begin{align}
P(w)    & = \lim_{N\to\infty} \E \bigg\{ \frac{1}{N} \bigg| \sum_{n=0}^{N-1} x(n) e^{-jwn} \bigg|^{2} \bigg\}
            \label{proof:periodogram:starter}\\
        & = \lim_{N\to\infty} \E \bigg\{\frac{1}{N}
            \sum_{m=0}^{N-1} x(m) e^{-jwm} \sum_{k=0}^{N-1} x^{*}(k) e^{jwk} \bigg\}\nonumber\\
        & = \lim_{N\to\infty} \frac{1}{N}
            \sum_{m=0}^{N-1} \sum_{k=0}^{N-1} \E \bigg\{ x(m) e^{-jwm} x^{*}(k) e^{jwk} \bigg\}\nonumber\\
        & = \lim_{N\to\infty} \frac{1}{N}
            \sum_{m=0}^{N-1} \sum_{k=0}^{N-1} \E \bigg\{ x(m) x^{*}(k) \bigg\} e^{-jw(m-k)} \nonumber\\
        & = \lim_{N\to\infty} \frac{1}{N}
            \sum_{m=0}^{N-1} \sum_{k=0}^{N-1} r_{xx}(m-k) e^{-jw(m-k)}
          = \lim_{N\to\infty} \frac{1}{N}
            \sum_{m=0}^{N-1} \sum_{k=0}^{N-1} g(m-k)
\label{proof:periodogram}
\end{align}
%
where $g(\tau) = r_{xx}(\tau) e^{-jw\tau}$. Converting the double into a single summation according to:
%
\begin{equation}
    \sum_{m=-N}^{N} \sum_{k=-N}^{N} g(m-k) = \sum_{\tau=-2N}^{2N}(2N + 1 - |\tau|)g(\tau)
\label{proof:periodogram:helper}
\end{equation}
%
(\ref{proof:periodogram}) can be written as:
%
\begin{align}
P(w)    & =         \lim_{N\to\infty} \frac{1}{N} \sum_{\tau=-(N-1)}^{N-1}(N - |\tau|)r_{xx}(\tau) e^{-jw\tau}\nonumber\\
        & =         \lim_{N\to\infty} \sum_{\tau=-(N-1)}^{N-1} r_{xx}(\tau) e^{-jw\tau} -
                    \lim_{N\to\infty} \frac{1}{N} |\tau| \sum_{\tau=-(N-1)}^{N-1} r_{xx}(\tau) e^{-jw\tau}\nonumber\\
        & \approx   \sum_{\tau=-\infty}^{\infty} r_{xx}(\tau) e^{-jw\tau}
\label{proof:periodogram:shown}
\end{align}
%
where the last equality in (\ref{proof:periodogram:shown}) is obtained under the mild assumption that the
covariance sequence $r_{xx}(k)$ decays rapidly, or equivalently:
%
\begin{equation}
    \lim_{N\to\infty} \frac{1}{N} \sum_{\tau=-(N-1)}^{N-1} |\tau| |r_{xx}(\tau)| = 0
\label{proof:periodogram:decaying-covariance}
\end{equation}
%
Thus under the mild assumption that the covariance sequence decays rapidly we have shown that (\ref{proof:periodogram:starter})
is equal to (\ref{proof:periodogram:shown}), which is the DTFT of the autocovariance function (ACF).
%
\begin{equation}
    P(w) = \sum_{t=-\infty}^{\infty} r_{xx}(t) e^{-jwt}
\end{equation}

\begin{enumerate}[label=\alph*), leftmargin=*]
%% a)
\item
%

Power Spectrum Estimation is performed by taking the Fourier Transform of the ACF, $r_{xx}(k)$. Due to the even symmetry of ACF,
its Fourier Transform, $P(w)$, is expected to be purely real, as well as non-negative since the positive semi-definiteness of the
covariance matrix, $xx^{T}$. These properties are preserved after zero-padding in the middle of the ACF, around zero, while at the
same time, frequency resolution is increased. Figure \ref{fig:1_2_a} verifies our claims in the case of the rectangular window signal,
whose width is controlled by varying $M$. Unsurprisingly, the increase of $M$ from $10$ to $128$ results in an increase of the rectangular
width and and inversely proportional decrease in the width of the mainlobe (frequency domain), while the power concentration is increased
around 0.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/a/zero-pad-L_20-M_10}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/a/zero-pad-L_256-M_10}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/a/zero-pad-L_256-M_128}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/a/zero-pad-L_1024-M_128}
    \end{subfigure}
    \caption{Spectrum estimation using symmetric ACF and zero-padding effect.}
    \label{fig:1_2_a}
\end{figure}

%% b)
\item
%

Because of the symmetry of ACF, $P(w)$ is expected to be purely real. Nonetheless, the numerical methods used for the implementation
of \texttt{fft} and the finite precision using in computers (64-bits in this case) lead to round-off errors, introducing a negligible
imaginary part \texttt{imag(xf)}. Figure \ref{fig:1_2_b_1} demonstrates this point, verifying how trivial (16 orders of magnitude smaller)
the imaginary part is, compared to the real part, \texttt{real(xf)}, while figure \ref{fig:1_2_b_2} depicts the PSD estimates with and
without the imaginary part.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/b/symmetric_real-M_10}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/b/symmetric_imag-M_10}
    \end{subfigure}
    \caption{Symmetric $r_{xx}(k)$: real and imaginary parts of $P(w)$ and round-off errors.}
    \label{fig:1_2_b_1}
\end{figure}

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/b/comparison-M_10}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/b/comparison-M_128}
    \end{subfigure}
    \caption{Symmetric $r_{xx}(k)$: $P(w)$ estimate using \texttt{abs} and \texttt{real}.}
    \label{fig:1_2_b_2}
\end{figure}

%% c)
\item
%

On the other hand, in case of non-symmetric ACF, $r_{zz}(k)$, the imaginary part is of the same order with the real part, as illustrated
in figure \ref{fig:1_2_c_1} and the PSD estimates at figure \ref{fig:1_2_c_2} have significant deviation. Moreover, it is also important
to highlight that when \texttt{real} is used instead of the \texttt{abs}, the Fourier Transform, $P_{z}(w)$, is not guaranteed to be positive,
resulting in a meaningless PSD estimate. Consequently, the magnitude (\texttt{abs}) should be used in case of non-symmetric ACF.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/c/non_symetric_real-M_10}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/c/non_symetric_imag-M_10}
    \end{subfigure}
    \caption{Non-Symmetric $r_{xx}(k)$: real and imaginary parts of $P(w)$.}
    \label{fig:1_2_c_1}
\end{figure}

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/c/comparison-M_10}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/properties-of-power-spectral-density/assets/c/comparison-M_128}
    \end{subfigure}
    \caption{Non-Symmetric $r_{xx}(k)$: $P(w)$ estimate using \texttt{abs} and \texttt{real}.}
    \label{fig:1_2_c_2}
\end{figure}

%% d)
\item

The figures in all the parts are generated, having previously centered the frequency axis via \texttt{fftshift}. Moreover, the signal length
is important when the frequency index, $w$, or the time index, $n$, are generated. Table \ref{tab:1_2_d} summarized the MATLAB commands
used for any case.

\begin{table}[h]
\centering
\begin{tabular}{|c||c|c|}
\hline
\textbf{Vector Length}, $L$ &
\textbf{Frequency Index}, $w$ &
\textbf{Time Index}, $n$ \\
\hline
\hline
\textbf{Even} &
$-\pi : \frac{2\pi}{L} : \pi - \frac{2\pi}{L}$ &
$-\frac{L}{2} : 1 : \frac{L}{2} - 1$ \\
\hline
\textbf{Odd} &
$-\pi + \frac{\pi}{L} : \frac{2\pi}{L - 1} : \pi - \frac{\pi}{L}$ &
$-\frac{L-1}{2} : 1 : \frac{L-1}{2} - 1$ \\
\hline
\end{tabular}
\caption{MATLAB commands for frequency $w$ and time $n$ indexes.}
\label{tab:1_2_d}
\end{table}

%
\end{enumerate}


\documentclass[ag.tex]{subfiles}

\begin{document}

\chapter{Schemes}

The central objects of our study are schemes,  which are mathematical structures that generalize the notion of algebraic varieties in a similar manner to how manifolds generalize subsets of Euclidean spaces.  An algebraic variety is often defined as a set of solutions of a system of polynomials over some field, which gives an algebraic description of geometric objects such as curves and surfaces.  It may be noted that varieties and appropriately defined maps between them form a category.

\section{From varieties to spectra}

We may briefly introduce affine algebraic varieties, a type of variety,  in order to motivate schemes.

\begin{definition}
An \textit{affine algebraic set} $X$ is the set of solutions in an algebraically closed field $k$ of a system of polynomials in $k[x_1,x_2,...,x_n]$ for some $n$.
\end{definition}

\begin{definition}
An \textit{affine (algebraic) variety} $X$ is an affine algebraic set which is not the union of two proper affine algebraic subsets.
\end{definition}

The important observation is that we can associate a "special" ring to any affine variety which reflects all properties of the variety.  This is the quotient of the polynomial ring $k[x_1,x_2,...,x_n]$ by the ideal $I$ of polynomials that are zero on $X$:

\begin{definition}
Let $X$ be an affine variety and $I$ be the ideal of $k[x_1,x_2,...,x_n]$ generated by all polynomials that are zero on $X$. Then we call the quotient ring $k[X] := k[x_1,x_2,...,x_n] / I$ the \textit{coordinate ring} of $X$.
\end{definition}

The idea is to go in the other direction i.e., to start with an arbitrary ring and associate a geometric object to it. This program will yield the notion of schemes, which is much more general than that of varieties.

A way to recover an affine variety $X$ from its coordinate ring $k[X]$ is to use that fact that there is a one-to-one correspondence between "points" $x$ of $X$ and maximal ideals $m_x$ of $k[X]$.  Therefore a first candidate for the geometric object associated with an arbitrary ring $A$ is the set of all maximal ideals of $A$, which we shall denote by $m$-Spec$A$.  But this assignment fails to obey functoriality for the following reason.

Suppose $f: A \to B$ is a ring homomorphism.  For our assignment to be functorial we must find a morphism between $m$-Spec$A$ and $m$-Spec$B$. But there is no natural way to do this since for maximal ideals $a \subset A$ and $b \subset B$, neither $f^{-1} (b)$ or $f(a)$ is necessarily maximal.

If we instead consider the set of all prime ideals of $A$, also called the \textit{spectrum} of $A$, Spec $A \supset$ $m$-Spec $A$, then this difficulty disappears owing to Proposition \ref{primes_map_to_primes}.  In other words, we have a contravariant functor from the category of (commutative) rings to the category of spectra (or the category of topological spaces,  as we shall see).  We derive a justification for the claim that Spec$A$ is "geometrical" by successfully assigning it a topology.

\begin{definition}
The \textit{Zariski topology} of Spec $A$ is given by setting the collection of closed sets of Spec $A$ as
\begin{equation*}
\text{\{$V_I$: $I$ is an ideal of $A$\}}
\end{equation*}
where $V_I$ is the set of all prime ideals containing $I$.
\end{definition}

It can be checked that this indeed defines a topology.

\section{Sheaves}

Along with Spec $A$,  the building blocks of a scheme is given by sheaves.  We first define a weaker notion, namely that of a presheaf.

\begin{definition}
Let $X$ be a topological space and $Open(X)$ be the category in which the objects are open sets of $X$ and morphisms are inclusion maps. A \textit{presheaf} is a contravariant functor

\begin{equation*}
\mathcal{F}: Open(X) \to \mathcal{C}
\end{equation*}
where $\mathcal{C}$ is some category.
\end{definition}

The maps $\mathcal{F}(V) \to \mathcal{F}(U)$ where $U \subset V$ are open sets of $X$ is denoted by $\rho_U^V$.  If $\mathcal{C}$ is the category of groups, rings or modules then we say $\mathcal{F}$ is a presheaf of groups, rings or modules respectively.

\begin{exmp}\label{sheaf_of_rings_of_cont_fun}
Presheaf of the rings of $C^0$ functions defined on open sets of $X$. Here the homomorphisms $\rho_U^V$ give restrictions of functions $f: V \to \mathbb{R}$ to $U$, i.e.  $\rho_U^V (f) = f|_{U}$.
\end{exmp}

The most important presheaf for us is the structure presheaf on Spec $A$:

\begin{definition}
The \textit{structure presheaf} of Spec $A$, denoted by $\mathcal{O}$,  is the contravariant functor given by

\begin{equation}
\mathcal{O}(U) = \varprojlim \mathcal{O}(D(f))
\end{equation}
and

\begin{equation}
\rho_U^V(\{v_\alpha\}) = \{v_\beta\}
\end{equation}
where
\end{definition}

It can be verified that $\mathcal{O}$ is a functor as claimed.

\begin{definition}
Suppose the category $\mathcal{C}$ has a terminal object $T$ (see Definition \ref{terminal object}).  Then a $\textit{section}$ of $\mathcal{F}(U)$ is a morphism $u: T \to \mathcal{F}(U)$.
\end{definition}

Note that if $\mathcal{F}(U)$ is a set then $u$ can be understood as "picking out" an element (say $t_u$) in $\mathcal{F}(U)$.  Thus for any morphism $v$ from $\mathcal{F}(U)$ to some object $C$ in $\mathcal{C}$,  we can regard the composition $v \circ u$ as a generalization of $v(t_u)$. With this given, we define a sheaf as a special case of presheaves:

\begin{definition}
A \textit{sheaf} is a presheaf that satisfies the following conditions
\begin{enumerate}
\item
\end{enumerate}
\end{definition}

\begin{exmp}
The presheaf in Example \ref{sheaf_of_rings_of_cont_fun} and the structure presheaf on Spec $A$ are sheaves.
\end{exmp}

\section{Schemes}

First we introduce the category of ringed spaces.

\begin{definition}
A \textit{ringed space} is a pair $(X, \mathcal{O})$ where $X$ is a topological space and $\mathcal{O}$ is a sheaf of rings. A \textit{morphism of ringed spaces} $\varphi: (X, \mathcal{O}_X) \to (Y, \mathcal{O}_Y)$ is a map $\varphi: X \to Y$ along with a natural transformation $\psi$ from $\mathcal{O}_Y$ to $\mathcal{O}_X \circ \varphi^{-1}$.
\end{definition}

Note that naturalness means $\psi$ is a collection of homomorphisms $\psi_U: \mathcal{O}_Y (U) \to \mathcal{O}_X (\varphi^{-1}(U))$ for open sets $U \subset Y$ such that the following diagram commutes for all open sets $U \subset V$ \\

It can be checked that for any ring $A$, (Spec $A$,  $\mathcal{O}_A$) where $\mathcal{O}_A$ is the structure sheaf on Spec $A$ is a ringed space.  We denote this simply as Spec $A$ from now on. Finally we define a scheme:

\begin{definition}
A \textit{scheme} is a ringed space $(X, \mathcal{O})$ for which every point has a neighborhood $U$ such that the ringed space $(U, \mathcal{O}|_{U})$ is isomorphic to Spec $A$ for some ring $A$.
\end{definition}

\end{document}


\section{Operators on Inner Product Spaces}
  Note that on the first page of this chapter, the author says that all the
  inner product spaces appeared in this chapter should be assumed to be
  finite-dimensional if not specified.

\subsection{Self-Adjoint and Normal Operators}
  \paragraph{2.}
  \begin{proof}
    $\nul(T-\lambda I) = (\range(T-\lambda I)^*)^\perp = (\range(T^*-
    \bar{\lambda}I))^\perp$. Hence, $\dim\nul(T-\lambda I)>0$ iff $\dim\range(
    T^*-\bar{\lambda}I)<\dim V$ iff $\dim\nul(T^*-\bar{\lambda}I)>0$. Thus,
    $\lambda$ is an eigenvalue of $T$ iff $\bar{\lambda}$ is an eigenvalue of
    $T^*$.
  \end{proof}

  \paragraph{4.}
  \begin{proof}
    If follows immediately from 7.7.
  \end{proof}

  \paragraph{8.}
  \begin{proof}
    Let $S,T\in\mathcal{L}(V)$ be self-adjoint and $a,b\in\mathbb{R}$. Then
    by 7.5,
    \[
      (aS+bT)^* = \bar{a}S^*+\bar{b}T^* = aS+bT.
    \]
    Hence, the set of self-adjoint operators on $V$ is a subspace of
    $\mathcal{L}(V)$.
  \end{proof}

  \paragraph{10.}
  \begin{proof}
    The $n$-by-$n$ matrices
    \[
      A=
      \begin{bmatrix}
        1 &        &        & \\
          & \ddots &        & \\
          &        & \ddots & \\
          &        &        & 1
      \end{bmatrix}
      \quad\text{and}\quad
      B=
      \begin{bmatrix}
        1 & 1      &        &   \\
          & \ddots & \ddots &   \\
          &        & \ddots & 1 \\
          &        &        & 1
      \end{bmatrix}
    \]
    are both normal, but $C=B-A$, which is nilpotent, is not.
  \end{proof}

  \paragraph{12.}
  \begin{proof}
    Since $T$ is normal and $3$ and $4$ are two eigenvalues of $T$, there exists
    an orthonormal list $u_1,u_2$ such that $Tu_1=3u_1$ and $Tu_2=4u_2$. Let $v
    =u_1+u_2$. Then $\|v\|=\sqrt{2}$ and
    \[
      \|Tv\| = \|3u_1+4u_2)\| = 5.
    \]
  \end{proof}

  \paragraph{16.}
  \begin{proof}
    7.20 implies $\nul T=\nul T^*$ and therefore, by 7.7, $\range T=\range T^*$.
  \end{proof}
% end

\subsection{The Spectral Theorem}
  \paragraph{4.}
  \begin{proof}
    If $T$ is normal, then by the complex spectral theorem, clear that all pairs
    of eigenvectors corresponding to distinct eigenvalues of $T$ are orthogonal
    and $V=E(\lambda_1,T)\oplus\cdots\oplus E(\lambda_m,T)$.\par
    Now we show the converse. Clear that the union of the orthonormal bases of
    each $E(\lambda_i,T)$ is an orthonormal basis consisting of eigenvectors of
    $T$.
  \end{proof}

  \paragraph{6.}
  \begin{proof}
    Suppose $T\in\mathbb{L}(V)$ is self-adjoint and $Tu=\lambda u$ for some $u
    \ne 0$. Then by 7.21, $\lambda u = Tu=T^*u=\bar{\lambda}u$. Hence, $\lambda
    =\bar{\lambda}$ and therefore $\lambda$ is real.\par
    By the complex spectral theorem, $T$ has a matrix $M=\diag(\lambda_1,\dots,
    \lambda_n)$ with respect to some orthonormal basis. If $\lambda_i$, the
    eigenvalues of $T$, are all real. Then the conjugate transpose of $M$ is
    still $M$. Hence, $T=T^*$. Namely, $T$ is self-adjoint.
  \end{proof}

  \paragraph{8.}
  \begin{proof}
    Let
    \[
      A = \begin{bmatrix}
        0 & 1      &        &   \\
          & \ddots & \ddots &   \\
          &        & \ddots & 1 \\
          &        &        & 0
      \end{bmatrix}
    \]
    be an $8$-by-$8$ matrix. $A^9=A^8=0$ but $A^2\ne A$.
  \end{proof}

  \paragraph{12.}
  \begin{proof}
    We argue by contradiction. Assume that for every eigenvalue $\lambda\hp$ of
    $T$, $|\lambda-\lambda\hp|>\vep$. Since $T$ is self-adjoint, there exists an
    orthonormal basis $e_1,\dots,e_n$ such that $Te_i=\lambda_ie_i$ for $i=1,
    \dots,n$. Then
    \[
      Tv-\lambda v = \sum_{i=1}^n\langle v,e_i\rangle(\lambda_i-\lambda)e_i.
    \]
    Hence,
    \[
      \|Tv-\lambda v\|^2
      = \sum_{i=1}^n |\langle v,e_i\rangle|^2|\lambda_i-\lambda|^2
      \ge \vep^2\sum_{i=1}^n |\langle v,e_i\rangle|^2 = \vep^2.
    \]
    Contradiction. Hence, $T$ has an eigenvalue $\lambda\hp$ such that $|\lambda
    -\lambda\hp|<\vep$.
  \end{proof}

  \paragraph{14.}
  \begin{proof}
    If there exists an inner product on $U$ which makes $T$ into a self-adjoint
    operator, then by the spectral theorem, $U$ has a basis consisting of
    eigenvectors of $T$. Now we suppose that such a basis, denoted by $u_1,
    \dots, u_n$, exists. Define $\langle e_i,e_j\rangle$ to be $0$ if $i\ne j$
    and $1$ if $i=j$. It is easy to verify that it does define an inner product
    and $u_1,\dots,u_n$ is an orthonormal basis.
  \end{proof}
% end

\subsection{Positive Operators and Isometries}
  \paragraph{2.}
  \begin{proof}
    The hypothesis implies $T^2v=v$ and therefore $1$ is an eigenvalue of $T^2$
    and $v$ is its associated eigenvector. Note that $T^2$ is still a positive
    operator and $T$ is its positive square root. Hence, $v$ is also an
    eigenvector of $T$ associated with eigenvalue $1$. Namely, $v=Tv=w$.
  \end{proof}

  \paragraph{4.}
  \begin{proof}
    $\langle T^*Tv,v\rangle=\langle Tv,Tv\rangle = \|Tv\|^2\ge 0$ for all $v\in
    V$. Hence, $T^*T$ is a positive operator on $V$. Similarly, $TT^*$ is a
    positive operator on $W$.
  \end{proof}

  \paragraph{6.}
  \begin{proof}
    If $T$ is positive, $T$ is self-adjoint and so does $T^k$. Meanwhile, $T$
    has a diagonal matrix $\diag(\lambda_1,\dots,\lambda_n)$ with respect to
    some orthonormal basis and $\lambda_i\ge 0$ for each $i=1,\dots,n$. With
    respect to the same basis, $T^k$ has the matrix $\diag(\lambda_1^k,\dots,
    \lambda_n^k)$. Since $\lambda_i^k\ge 0$, $T^k$ is positive by 7.35.
  \end{proof}

  \paragraph{8.}
  \begin{proof}
    Suppose $T$ is positive and invertible. Then $\langle v,v\rangle_T=\langle
    Tv,v\rangle\ge 0$. By 7.35, $T=R^*R$ for some $R\in\mathcal{L}(V)$. Since
    $T$ is invertible, both $R^*$ and $R$ are invertible. Hence, $0=\langle v,v
    \rangle_T=\langle Tv,v\rangle=\|Rv\|^2$ iff $Rv=0$ iff $v=0$. Meanwhile,
    for every $u,v\in V$ and scalar $a,b$
    \[
      \langle au+bv,w\rangle_T = a\langle Tu,w\rangle + b\langle Tv,w\rangle =
      a\langle v,w\rangle_T + b\langle v,w\rangle_T.
    \]
    Finally,
    \[
      \langle u,v\rangle_T = \langle Ru,Rv\rangle =
      \overline{\langle Rv,Ru\rangle} = \overline{\langle v,u\rangle_T}.
    \]
    Thus, $\langle\cdot,\cdot\rangle_T$ is an inner product on $V$.\par
    Now we suppose $\langle\cdot,\cdot\rangle_T$ is an inner product on $V$.
    Hence, by the positivity of inner product, $T$ is positive and by the
    previous discussion, $Tv=0$ iff $v=0$. Hence, $T$ is invertible.
  \end{proof}

  \paragraph{14.}
  \begin{proof}
    As $T$ is self-adjoint by Exercise 21 of Section 7.A, so does $-T$.
    Suppose that
    \[
      f(x)=x_0 + x_1\cos x +\cdots+ x_n\cos nx + y_1\sin x +\cdots+ y_n\sin nx.
    \]
    Then
    \[
      f^{\prime\prime}(x)
      = -\sum_{k=1}^nx_kk^2\cos kx - \sum_{k=1}^ny_kk^2\sin kx.
    \]
    Note that for all $i\ne j$, the integral of $(\cos ix\sin jx)$ is $0$. Thus,
    \begin{align*}
      \langle f,f^{\prime\prime}\rangle &=
      -\int_{-\pi}^\pi
      \left(x_0 + \sum_{k=1}^n(x_k\cos kx + y_k\sin kx) \right)
      \sum_{k=1}^n(k^2x_k\cos kx+ k^2y_k\sin kx)\rd x\\
      &= -\int_{-\pi}^\pi\left( \sum_{k=1}^n x_k^2k^2\cos^2kx +
      \sum_{k=1}^ny_k^2k^2\sin^2kx \right)\rd x \\
      &= -2\pi\sum_{k=1}^n k^2(x_k^2+y_k^2) \le 0.
    \end{align*}
    Therefore, $-T$ is a positive operator.
  \end{proof}
% end

\subsection{Polar Decomposition and Singular Value Decomposition}
  \paragraph{3.}
  \begin{proof}
    As $\sqrt{T^*T}$ is positive, $(\sqrt{T^*T})^*=\sqrt{T^*T}$. The polar
    decomposition asserts that there exists an isometry $S_1$ such that
    $T=S_1\sqrt{T^*T}$. Replacing $T$ with $T^*$ yields
    \[
      T^*=S_2\sqrt{TT^*} \quad\Rightarrow\quad
      T=\sqrt{TT^*}S_2^*,
    \]
    where $S_2^*$ is also an isometry.
  \end{proof}

  \paragraph{8.}
    This result can be used to prove Exercise 1 in a simple way.
  \begin{proof}
    Note that $R^2$ and $T^*T$ are both positive (and thereby self-adjoint).
    Hence, $R^2-T^*T$ is also self-adjoint. For all $v\in V$,
    \begin{align*}
      \langle(R^2-T^*T)v,v\rangle
      = \langle R^2v,v\rangle - \langle T^*Tv,v\rangle
      = \|Rv\|^2 - \|Tv\|^2.
    \end{align*}
    Since $S$ is an isometry, $\|Rv\| = \|SRv\| = \|Tv\|$. Therefore, $\langle
    (R^2-T^*T)v,v\rangle=0$. Thus, $R^2=T^*T$ and so does their unique positive
    square roots.
  \end{proof}

  \paragraph{9.}
  \begin{proof}
    As $T=S\sqrt{T^*T}$ for some isometry $S$ and every isometry is invertible,
    $T$ is invertible iff $\sqrt{T^*T}$ is invertible. If $T$ is invertible and
    $T=S_1\sqrt{T^*T}=S_2\sqrt{T^*T}$, multiplying $(\sqrt{T^*T})\inv$ on the
    both sides yields $S_1=S_2$. \par
    Now we suppose that such an isometry $S$ is not unique and show that
    $\sqrt{T^*T}$ is not invertible to complete the proof. Suppose $T=S_1
    \sqrt{T^*T}=S_2\sqrt{T^*T}$ where $S_1\ne S_2$. Then, since $S_1-S_2\ne 0$
    but $(S_1-S_2)\sqrt{T^*T}=0$, $\sqrt{T^*T}$ is not invertible and so is $T$.
  \end{proof}

  \paragraph{13.}
  \begin{proof}
    By the discussion in Exercise 9, $T$ is invertible iff $\sqrt{T^*T}$ is
    invertible iff $0$ is not an eigenvalue of $\sqrt{T^*T}$.
  \end{proof}
% end