\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\varepsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf Arena Analytics } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Escriba: #4}{Sesi\'{o}n #1}}

\newtheorem{theorem}{Teorema}
\newtheorem{corollary}[theorem]{Corolario}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observación}
\newtheorem{proposition}[theorem]{Proposición}
\newtheorem{definition}[theorem]{Definición}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Hecho}
\newtheorem{assumption}[theorem]{Suposición}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{NoSes\'{o}n: Tema, Fecha}{Invierno 2019}{Nombre del presentador}{Nombre del escriba}

\section{Visión general}

In the last lecture we \ldots.

In this lecture we \ldots.

\section{Definiciones y Teoremas}
Here is an equation:
\begin{equation*}
a^2 + b^2 = c^2 ,
\end{equation*}
or
$$
a^2 + b^2 = c^2 .
$$

Here's the same equation but that can be referenced:
\begin{equation}
a^2 + b^2 + c^2 \label{eqn:pythagoras}
\end{equation}
Equation~\ref{eqn:pythagoras} has been known for a long time. Sometimes you might want to align equations:
\begin{align*}
(x+y)^2 - (x-y)^2 &= (x^2 + 2xy + y^2) - (x^2 - 2xy - y^2) \\
{}&= 4xy
\end{align*}
or align them with numbers to be referenced:
\begin{align}
(x+y)^2 - (x-y)^2 &= (x^2 + 2xy + y^2) - (x^2 - 2xy - y^2) \label{eqn:1}\\
{}&= 4xy \label{eqn:2}
\end{align}
or where only some of them can be easily referenced:
\begin{align}
\nonumber (x+y)^2 - (x-y)^2 &= (x^2 + 2xy + y^2) - (x^2 - 2xy - y^2) \\
{}&= 4xy \label{eqn:3}
\end{align}

Here are some other math goodies: $\binom{n}{k}$, $\frac{n}{k}$, $x\gg y$, $x\ll y$, $x\lesssim y$, $x\gtrsim y$, $\inprod{x,y}$, $x\in\R^n$, $x\in S$, $S\ni x$, $x\notin S$, $x\neq y$, $x\oplus y$, $x\pm y$, $\sqrt{x}$, $x\equiv y$, $S\cap T$, $\S\cup T$, $S\subset T$, $S\subseteq T$, $S\supset T$, $\gamma$,  $\Gamma$, $\R$, $\mathbb{F}$
$$\sum_{i=1}^n i,\ \sum_{\substack{i=1\\i\ odd}}^n i, \prod_{i=1}^n i$$

\section{Main Section}

We begin by describing the problem \ldots.
Make sure to use sections and subsections.

\subsection{Blah blah blah}
Here is a subsection.

\subsubsection{Blah blah blah}
Here is a subsubsection. You can use these as well.

\subsection{Using Boldface}
Make sure to use \textbf{lots} of {\bf boldface}.

\paragraph{Question:}
How would you use boldface?

\paragraph{Example:}
This is an example showing how to use boldface to
help organize your lectures.

\paragraph{Some Formatting.}
Here is some formatting that you can use in your notes:
\begin{itemize}
\item {\em Item One} -- This is the first item.
\item {\em Item Two} -- This is the second item.
\item \dots and here are other items.
\end{itemize}

If you need to number things, you can use this style:
\begin{enumerate}
\item {\em Item One} -- Again, this is the first item.
\item {\em Item Two} -- Again, this is the second item.
\item \dots and here are other items.
\end{enumerate}

\paragraph{Bibliography.}
Please give real bibliographical citations for the papers that we
mention in class. See below for how to include a bibliography section.
If you use BibTeX, integrate the .bbl file into your .tex
source. You should reference papers like this: ``The tug of war sketch
originates in a paper by Alon, Matias and Szegedy \cite{AlonMS99}.''
In general, the name of the authors should appear in text at most once
(for the first citation); further citations look like: ``Our proof follows
that of \cite{AlonMS99}''.

Take a look at previous lectures (TeX files are available) to see the
details. A excellent source for bibliographical citations is
DBLP. Just Google DBLP and an author's name.

\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem{AlonMS99}
Noga~Alon, Yossi~Matias, Mario~Szegedy.
\newblock The Space Complexity of Approximating the Frequency Moments.
\newblock {\em J. Comput. Syst. Sci.}, 58(1):137--147, 1999.

\end{thebibliography}

\end{document}


\documentclass[a4paper,11pt]{article}
\usepackage[latin1]{inputenc}
\usepackage[english]{babel}
\usepackage{epsfig}
\usepackage{amsmath}
\usepackage{amsfonts}

\newcommand\R{{\mathbb R}}
\newcommand\x{{\mathbf x}}
\newcommand\X{{\mathbf X}}
\newcommand\K{{\mathbf K}}
\newcommand\J{{\mathbf J}}
\newcommand\LL{{\mathbf L}}
\newcommand\ELL{{\Ivec \ell}}
%%\newcommand\L{{\mathbf L}}
%%\DeclareMathOperator*{\argmax}{arg\,max}
%% \DeclareMathOperator*{\argmax}{argmax} %% (argmax wihtouth mid space)
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\dm}{dm}
\newcommand{\Rvec}[1]{{\bf #1}}
\newcommand{\Ivec}[1]{\mbox{\boldmath $#1$}}

\title{Kernels}
\author{Emanuele Olivetti}

\begin{document}

\maketitle

\section{Introduction}
This document gives a detailed description of kernels implemented in
PyMVPA together with derivation of their gradients. These kernels are
used mainly by Gaussian Process Regression but can be of use for other
kernel methods. Gradients are useful when trying to maximize the
marginal likelihood of a Gaussian Process, i.e., during model
selection. In some cases a further reparametrization in log scale is
provided in order to reduce numerical stability issues when computing
likelihoods.

The following notation and definitions are used:
\begin{itemize}
\item $\x \in \R^D$ : a $D$-dimensional column vector, $\x =
  (x_1,\ldots,x_D)$.
\item $\X = (\x_1^{\top},\ldots,\x_N^{\top})$ : a $N \times D$ matrix
  where each row is a $D$-dimensional vector. $\X$ is also called set
  of \emph{samples}. $\X_{* i}$ indicates the $i$-th column of $\X$
  and is a column vector. $\X_{j *}$ indicates the $j$-th row of $\X$
  and is a row vector.
\item $k: \R^D \times \R^D \rightarrow \R$ : a covariance (or kernel)
  function.
\item $\K(\X,\X')$ : the matrix extension of $k$, i.e., $\K_{pq} =
  k(\X_{*p},\X'_{*q})$. If $\X$ is a $N \times D$ matrix and $\X'$ is
  $N' \times D$ then $\K(\X,\X')$ is a $N \times N'$ matrix.
\item $\J_{n,m}$ : the $n \times m$ matrix of ones, i.e., a matrix
  where each element is 1.
\item $\|\mathbf{z}\|_p$ : the $p$-norm of vector $\mathbf{z}$ defined
  as $\|\mathbf{z}\|_p = (\sum_{i=1}^D \|z_i\|^p)^{\frac{1}{p}}$.
  Euclidean norm is $p=2$, then $\|\mathbf{z}\|_2 =
  \sqrt{\mathbf{z}^{\top}\mathbf{z}} = \sqrt{\sum_{i=1}^D z_i^2}$.
\item $\|\mathbf{z},\mathbf{w}\|_p$ : the \emph{weighted} $p$-norm of
  vector $\mathbf{z}$ defined as $\|\mathbf{z},\mathbf{w}\|_p =
  (\sum_{i=1}^D w_i|z_i|^p)^{\frac{1}{p}}$.  Euclidean norm is
  $p=2$, then $\|\mathbf{z},\mathbf{w}\|_2 = \sqrt{\mathbf{z}^{\top}
    \mathbf{W}^{-1} \mathbf{z}} = \sqrt{\sum_{i=1}^D w_i z_i^2}$,
  where $\mathbf{W} = diag(\mathbf{w})$.
\item $\dm(\X,\X')$ : the \emph{Euclidean distance matrix} between
  $\X$ and $\X'$ defined element by element as $\dm(\X,\X')_{pq} =
  \|\X_{p *} - \X'_{q *}\|_2 = \sqrt{\sum_{i=1}^D (\X_{p i} - \X'_{q
      i})^2}$. If $\X$ is a $N \times D$ matrix and $\X'$ is $N'
  \times D$ then $\dm(\X,\X1)$ is a $N \times N'$ matrix. Note that
  $\dm(\X,X')$ is the square root of what it is usually called
  ``distance matrix''.
\item $\dm(\X,\X',\mathbf{w})$ : the \emph{weighted} Euclidean
  distance matrix between $\X$ and $\X'$ defined element by element as
  $\dm(\X,\X',\mathbf{w})_{pq} = \|(\X_{p *} - \X'_{q *})^{\top},
  \mathbf{w}\|_2 = \sqrt{\sum_{i=1}^D w_i(\X_{p i} - \X'_{q i})^2}$
  through the weight vector $\mathbf{w} \in \R^D$. It is
  straightforward that $\dm(\X,\X') = \dm(\X,\X',\mathbf{J}_{D,1})$.
\item $\X \bullet \mathbf{Y}$ : the Hadamard (or Schur) matrix
  product, i.e. the entry-wise product between matrices of the same
  size. Let $\mathbf{Z} = \X \bullet \mathbf{Y}$, then $z_{ij} =
  x_{ij} y_{ij}$.
\item $\X^{\alpha}$ : $(\X^{\alpha})_{ij} = (X_{ij})^{\alpha}$.
\end{itemize}

\section{Constant kernel}
$$k(\x,\x') = \sigma_0^2$$
where $\sigma_0 \ge 0$ is the standard deviation of the Gaussian prior
probability $\mathcal{N}(0,\sigma_0^2)$ of the value of the constant.
$$\K(\X,\X') = \sigma_0^2 \J_{N,N'}$$
$$\mathbf{\Theta} = \{\sigma_0\}$$
$$\frac{\partial k}{\partial \sigma_0}(\x,\x') = 2\sigma_0$$
$$\frac{\partial \K}{\partial \sigma_0} = 2\sigma_0 \J_{N,N'}$$
$$A = \sigma_0^2$$
$$A \ge 0$$
$$\mathbf{\Theta}^* = \{A\}$$
$$k(\x,\x') = A$$
$$\K(\X,\X') = A \J_{N,N'}$$
$$\frac{\partial k}{\partial A} = 1$$
$$\nabla_A \K = \frac{\partial \K}{\partial A} = \J_{N,N'}$$
Note that using $A$ as hyperparameter the gradient becomes constant.

\section{Linear kernel}
Let $\Ivec{\Sigma}_p$ be the $D \times D$ covariance matrix of the Gaussian
prior probability $\mathcal{N}(\Ivec{0},\Ivec{\Sigma}_p)$ of the weights of
the Bayesian linear regression.
$$k(\x,\x') = \x^{\top} \Ivec{\Sigma}_p \x' + \sigma_2^2$$
$$\K(\X,\X') = \X \Ivec{\Sigma}_p \X'^{\top} + \Ivec{I}\sigma_0^2$$
In order to simplify formulas we assume $\Ivec{\Sigma}_p$ is diagonal, i.e.,
$\Ivec{\Sigma}_p = diag(\Ivec{\sigma}^2_p)$ where $\Ivec{\sigma}^2_p =
({\sigma^2_p}_1,\ldots,{\sigma^2_p}_D)$:
$$k(\x,\x') = \sum_{i=1}^D {\sigma^2_p}_i x_i x'_i + \sigma_0^2$$
$$\mathbf{\Theta} = \{{\sigma_p}_1,\ldots,{\sigma_p}_D,\sigma_0\}$$
$$\frac{\partial k}{\partial {\sigma_p}_i} = 2 {\sigma_p}_i x_i x'_i$$
$$A_i = {\sigma_p^2}_i$$
$$A_i \ge 0$$
$$\mathbf{A} = (A_1,\ldots,A_D)^{\top}$$
$$\frac{\partial k}{\partial \sigma_0} = 2 \sigma_0$$
$$B = \sigma_0^2$$
$$\mathbf{\Theta}^* = \{ \mathbf{A} , B\}$$
$$k(\x,\x') = \x^{\top} diag(\mathbf{A}) \x' + B$$
$$\K(\X,\X') = \X diag(\mathbf{A}) \X'^{\top} + \Ivec{I}B$$
$$\frac{\partial k}{\partial A_i} = x_i x'_i$$
$$\frac{\partial \K}{\partial A_i} = \X_{* i} {\X'_{* i}}^{\top}$$
$$\nabla_{\mathbf{A}} \K = ( \X_{* 1} {\X'_{* 1}}^{\top}, \ldots,
\X_{* D} {\X'_{* D}}^{\top})$$
$$\frac{\partial k}{\partial B} = 1$$
$$\frac{\partial \K}{\partial B} = \Ivec{I}$$
As expected the gradient is independent of the hyperparameters values
and can be computed once for all at the beginning.

Logscale:
$$A_i = \ln{\sigma_{p_i}^2}$$
$$\sigma_{p_i}^2 = e^{A_i}$$
$$\frac{\partial k}{\partial A_i} = \sigma_{p_i}^2 x_i x_j$$
$$B = \ln{\sigma_0}$$
$$\sigma_0^2 = e^{2B}$$
$$\frac{\partial k}{\partial B} = 2 e^{2B} = 2 \sigma_0^2$$
Another reparametrization:
$$C = \ln{\sigma_0^2}$$
$$\sigma_0^2 = e^C$$
$$\frac{\partial k}{\partial C} = \sigma_0^2$$

\section{Polynomial kernel}
$$k(\x,\x') = (\sigma_0^2 + \x^{\top} \Ivec{\Sigma}_p \x')^p =
(\sigma_0^2 + \sum_{i=1}^D {\sigma^2_p}_i x_i x'_i)^p$$
$$\K(\X,\X') = (\sigma_0^2 \mathbf{J}_{N,N'} + \X \Ivec{\Sigma}_p
\X'^{\top})^p$$
In order to simplify formulas we assume $\Ivec{\Sigma}_p$ is diagonal, i.e.,
$\Ivec{\Sigma}_p = diag(\Ivec{\sigma}^2_p)$ where $\Ivec{\sigma}^2_p =
({\sigma^2_p}_1,\ldots,{\sigma^2_p}_D)$.
$$\sigma_0 \ge 0$$
$$\Ivec{\sigma}_p = ({\sigma_p}_1,\ldots,{\sigma_p}_D)$$
$${\sigma_p}_i \ge 0$$
$$\mathbf{\Theta} = \{\sigma_0,\Ivec{\sigma}_p, p\}$$
$$A = \sigma_0^2$$
$$B_i = {\sigma^2_p}_i$$
$$\mathbf{B} = (B_1,\ldots,B_D)$$
$$\mathbf{\Theta}^* = \{A,\mathbf{B}, p\}$$
$$k(\x,\x') = (A + \x^{\top} diag(\mathbf{B}) \x')^p = (A +
\sum_{i=1}^D B_i x_i x'_i)^p$$
$$\frac{\partial k}{\partial A} = p(A + \sum_{i=1}^D B_i x_i
x'_i)^{p-1}$$
$$\frac{\partial \K}{\partial A} = p(A\mathbf{J}_{N,N'} + \sum_{i=1}^D
B_i \X_{*i} {\X'_{*i}}^{\top})^{p-1}$$
$$\frac{\partial k}{\partial B_i} = p(A + \sum_{i=1}^D B_i x_i
x'_i)^{p-1} x_i x'_i = \frac{\partial k}{\partial A} x_i x'_i$$
$$\frac{\partial \K}{\partial B_i} = \frac{\partial \K}{\partial A} \X_{*i} {\X'_{*i}}^{\top}$$
$$\frac{\partial k}{\partial p} = k(\x,\x') \ln(A + \sum_{i=1}^D B_i x_i
x'_i)$$
$$\frac{\partial \K}{\partial p} = \K(\X,\X') \bullet \ln(A \mathbf{J}_{N,N'}+ \sum_{i=1}^D B_i \X_{*i} \X'_{*i})$$
$$\mathbf{M} = A \mathbf{J}_{N,N'}+ \sum_{i=1}^D B_i \X_{*i} \X'_{*i}$$
$$\nabla_{A,\mathbf{B},p} \K = \left(p\mathbf{M}^{p-1},\left\{p\mathbf{M}^{p-1} \X_{*i} {\X'_{*i}}^{\top} \right\}_{i=1,\ldots,D}, \K(\X,\X') \bullet \ln(\mathbf{M}) \right)$$

\section{Exponential kernel}
\subsection{Scalar Length-scale $\ell$}
$$k(\x,\x') = \sigma_f^2 e^{-\frac{\|\x-\x'\|_2}{\ell}}$$
$$\ell > 0$$
$$\sigma_f \ge 0$$
$$\mathbf{\Theta} = \{ \sigma_f, \ell \}$$
$$\K(\X,\X') = \sigma_f^2 e^{-\frac{1}{\ell}\dm(\X,\X')}$$
$$A = \sigma_f^2$$
$$A \ge 0$$
$$B = -\frac{1}{\ell}$$
$$B < 0$$
$$\mathbf{\Theta}^* = \{ A, B \}$$
$$k(\x,\x') = A e^{B\|\x-\x'\|_2}$$
$$\K(\X,\X') = A e^{B \dm(\X-\X')}$$
$$\frac{\partial k}{\partial A} = e^{B\|\x-\x'\|_2} = \frac{1}{A}k(\x,\x')$$
$$\frac{\partial \K}{\partial A} = e^{B \dm(\X,\X')} = \frac{1}{A} \K(\X,\X')$$
$$\frac{\partial k}{\partial B} = A e^{B\|\x-\x'\|_2} \|\x-\x'\|_2 =
k(\x,\x') \|\x-\x'\|_2$$
$$\frac{\partial \K}{\partial B} = \K(\X,\X') \bullet \dm(\X-\X')$$
$$\nabla_{A,B} \K = (\frac{1}{A} \K(\X,\X'), \K(\X,\X') \bullet \dm(\X-\X'))$$

Note that if $\K(\X,\X')$ is precomputed, then the gradient consists
in just two element-by-element products, the second being against a
constant matrix independent of the hyperparameters.

\subsection{Vector of Length-Scales $\Ivec{\ell}$}
Given $\Ivec{\ell} = (\ell_1,\ldots,\ell_D)$, $\ell_i \ge 0$ and
$\ELL^{-1} = (1/\ell_1,\ldots,1/\ell_D)$
$$k(\x,\x') = \sigma_f^2 e^{-\|\x-\x',\ELL^{-2}\|_2}= \sigma_f^2
  e^{-\sqrt{\sum_{i=1}^D \left(\frac{x_i - x'_i}{\ell_i}\right)^2}}$$
$$K(\X,\X') = \sigma_f^2 e^{-\dm(\X,\X',\ELL^{-2})}$$
$$\mathbf{\Theta} = \{ \sigma_f, \ELL\}$$
$$\sigma_f \ge 0$$
$$\ell_i > 0$$
$$A = \sigma_f^2$$
$$\mathbf{B} = \ELL^{-2}$$
$$A \ge 0$$
$$B_i > 0$$
$$\mathbf{\Theta}^* = \{ A, \mathbf{B}\}$$
$$k(\x,\x') = A e^{-\|\x-\x',\mathbf{B}\|_2} = A e^{-\sqrt{\sum_{i=1}^D
    B_i(x_i - x'_i)^2}}$$
$$K(\X,\X') = A e^{-\dm(\X,\X',\mathbf{B})}$$
$$\frac{\partial k}{\partial A} = e^{-\sqrt{\sum_{i=1}^D B_i(x_i -
    x'_i)^2}} = \frac{k(\x,\x')}{A}$$
$$\frac{\partial \K}{\partial A} = e^{-\dm(\X,\X',\mathbf{B})} =
\frac{1}{A}\K(\X,\X')$$
$$\frac{\partial k}{\partial B_i} = A e^{-\|\x-\x',\mathbf{B}\|_2}
\left( -\frac{1}{2} \|\x-\x',\mathbf{B}\|_2^3 \right) (x_i - x'_i)^2 =
-\frac{1}{2} k(\x,\x') \|\x-\x',\mathbf{B}\|_2^3 (x_i - x'_i)^2 $$
$$\frac{\partial \K}{\partial B_i} = -\frac{1}{2} K(\X,\X')
\bullet \dm(\X-\X',\mathbf{B})^3 \bullet (\X_{*i}J_{1,N'}-
(\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{A,\mathbf{B}} \K = \left(\frac{1}{A}\K(\X,\X'), \left\{-\frac{1}{2}
    K(\X,\X') \bullet \dm(\X-\X',\mathbf{B})^3 \bullet
    (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right\}_{i=1 \ldots
    D} \right)$$

Note that $\frac{\partial \K}{\partial A}$ requires just the
multiplication of a constant by the kernel matrix $\K(\X,\X')$ whose
values usually already available. Instead $\frac{\partial \K}{\partial
  B_i}$ is a entry-wise product of 3 matrices: $\K(\X,\X')$ (usually
already available), $\dm(\X-\X',\mathbf{B})^3$ (which is, apart the
cube, part of the computation of $\K(\X,\X')$ so it can be stored in
advance), and $(\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$ which
does not depend upon $A$ and $\mathbf{B}$ so it can be computed once
for all. Note that in NumPy $(\X_{*i}J_{1,N'}-
(\X'_{*i}J_{1,N})^{\top})^2$ can be computed as {\ttfamily
  numpy.subtract.outer($X_{*i},X'_{*i}$)**2}.

\section{Squared Exponential kernel}

\subsection{Scalar Length-Scale $\ell$}
$$k(\x,\x') = \sigma_f^2 e^{-\frac{1}{2\ell^2} (\x-\x')^{\top}(\x-\x')}
  = \sigma_f^2 e^{-\frac{1}{2\ell^2} \sum_{i=1}^D (x_i - x'_i)^2} =
  \sigma_f^2 e^{-\frac{1}{2\ell^2} \|\x-\x'\|_2^2}$$
$$\K(\X,\X') = \sigma_f^2 e^{-\frac{1}{2\ell^2} \dm(\X,\X')^2}$$
$$\sigma_f \ge 0$$
$$\ell > 0$$
$$\mathbf{\Theta} = \{ \sigma_f, \ell \}$$
$$\frac{\partial k}{\partial \sigma_f} = \frac{2}{\sigma_f}k$$
$$\frac{\partial \K}{\partial \sigma_f} = \frac{2}{\sigma_f}\K$$
$$\frac{\partial k}{\partial \ell} = \ell^{-3} k \|\x-\x'\|_2^2 $$
$$\frac{\partial \K}{\partial \ell} = \ell^{-3} \K \bullet \dm(\X,\X')^2$$
$$\nabla_{\sigma_f,\ell} \K = \left(\frac{2}{\sigma_f}K(\X,\X'), \ell^{-3} \K \bullet \dm(\X-\X')^2 \right)$$
Logscale:
$$A = \ln{\sigma_f}$$
$$\sigma_f = e^A$$
$$\frac{\partial k}{\partial A} = 2k$$
$$\frac{\partial \K}{\partial \sigma_f} = 2\K$$
$$B = \ln{\ell}$$
$$\ell = e^B$$
$$\frac{\partial k}{\partial B} = \ell^{-2} k \|\x-\x'\|_2^2$$
$$\frac{\partial \K}{\partial B} = \ell^{-2} \K \bullet \dm(\X,\X')^2 = \K \bullet \dm(\X,\X',\ell^{-2}\J_N)^2$$
$$\nabla_{A,B} \K = \left( 2\K(\X,\X'), \ell^{-2} \K \bullet \dm(\X-\X')^2 \right)$$
Another mapping:
$$A = \sigma_f^2$$
$$B = -\frac{1}{\ell^2}$$
$$A \ge 0$$
$$B < 0$$
$$\mathbf{\Theta}^* = \{ A, B\}$$
$$k(\x,\x') = A e^{B (\x-\x')^{\top}(\x-\x')}$$
$$\K(\X,\X') = A e^{B \dm(\X,\X')^2}$$
$$\frac{\partial k}{\partial A} = \frac{k(\x,\x')}{A}$$
$$\frac{\partial \K}{\partial A} = \frac{1}{A}\K(\X,\X')$$
$$\frac{\partial k}{\partial B} = k(\x,\x') \|\x-\x'\|_2^2$$
$$\frac{\partial \K}{\partial B} = K(\X,\X') \bullet \dm(\X,\X')^2$$
$$\nabla_{A,B} \K = (\frac{1}{A} \K(\X,\X'), \dm(\X-\X')^2 \bullet
\K(\X,\X'))$$

Note that $\nabla_{A,B} \K$ is similar to that of the exponential
kernel so almost all comments made before applies here as well.

\subsection{Vector of Length-Scales $\ELL$}
Let $\mathbf{L} = diag(\ELL)$:
$$k(\x,\x') = \sigma_f^2 e^{-\frac{1}{2}(\x-\x')^{\top} \LL^{-2}
  (\x-\x')} = \sigma_f e^{-\frac{1}{2}\sum_{i=1}^D \frac{(x_i -
    x'_i)^2}{\ell_i^2}} = \sigma_f^2
e^{-\frac{1}{2}\|\x-\x',\ELL^{-2}\|_2^2}$$
$$\K(\X,\X') = \sigma_f^2 e^{-\frac{1}{2} \dm(\X,\X',\ELL^{-2})^2}$$
$$\sigma_f \ge 0$$
$$\ell_i > 0$$
$$\mathbf{\Theta} = \{ \sigma_f, \ELL\}$$
$$\frac{\partial k}{\partial \sigma_f} = \frac{2}{\sigma_f}k$$
$$\frac{\partial \K}{\partial \sigma_f} = \frac{2}{\sigma_f}\K$$
$$\frac{\partial k}{\partial \ell_i} = \ell_i^{-3} k \|\x_i-\x_i'\|_2^2 $$
$$\frac{\partial \K}{\partial \ell_i} = \ell_i^{-3} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{\sigma_f,\ell} \K = \left(\frac{2}{\sigma_f}K(\X,\X'), \ell_i^{-3} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right)$$
Logscale:
$$A = \ln{\sigma_f}$$
$$\sigma_f = e^A$$
$$\frac{\partial k}{\partial A} = 2k$$
$$\frac{\partial \K}{\partial \sigma_f} = 2\K$$
$$B_i = \ln{\ell_i}$$
$$\ell_i = e^B_i$$
$$\frac{\partial k}{\partial B_i} = \ell_i^{-2} k \|\x_i-\x'_i\|_2^2$$
$$\frac{\partial \K}{\partial B_i} = \ell^{-2} \K \bullet \dm(\X,\X')^2 = \ell_i^{-2} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{A,\mathbf{B}} \K = \left( 2\K(\X,\X'), \left\{ \ell_i^{-2} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right\}_{i=1 \ldots D} \right)$$
Another mapping:
$$A = \sigma_f$$
$$\mathbf{B} = -\frac{1}{2}\ELL^{-2} =
\left(-\frac{1}{2\ell_1^2},\ldots,-\frac{1}{2\ell_D^2} \right)$$
$$A \ge 0$$
$$B_i < 0$$
$$\mathbf{\Theta}^* = \{ A, \mathbf{B}\}$$
$$k(\x,\x') = A e^{(\x-\x')^{\top} diag(\mathbf{B}) (\x-\x')} = A
e^{\sum_{i=1}^D B_i (x_i - x'_i)^2}$$
$$\K(\X,\X') = A e^{\dm(\X,\X',\mathbf{B})}$$
$$\frac{\partial k}{\partial A} = \frac{k(\x,x')}{A}$$
$$\frac{\partial \K}{\partial A} = \frac{1}{A}\K(\X,X')$$
$$\frac{\partial k}{\partial B_i} = k(\x,\x') (x_i -x'_i)^2$$
$$\frac{\partial \K}{\partial B_i} = \K(\X,\X') \bullet
(\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{A,\mathbf{B}} \K = \left(\frac{\K(\X,\X')}{A}, \left\{ K(\X,\X')
  \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right\}_{i=1
  \ldots D} \right)$$

Note that $\nabla_{A,\mathbf{B}} \K$ requires to compute $K(\X,\X')$
(which is usually already available), and its entry-wise product with
$(\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$ which is independent
of the value of the hyperparameters and can be precomputed once for
all.

\section{$\gamma$-Exponential kernels}
$$k(\x,\x') = \sigma_f^2 e^{-(\frac{\x-\x'}{\ell})^\gamma}$$
$$k(\x,\x') = \sigma_f^2 e^{-(\frac{\x-\x'}{\ELL})^\gamma}$$

\section{Mat\'ern kernels}
%% $$k_{\mbox{Mat\'ern}}(\x,\x') = \frac{2^{1-\nu}}{\Gamma(\nu)} \left(\sqrt{2\nu} \frac{\|\x-\x'\|_2}{\ell}\right)^{\nu} \K_{\nu}\left(\sqrt{2}\nu \frac{\|\x-\x'\|_2}{\ell} \right)$$
%% $$\ell > 0$$
%% $$\nu > 0$$
%% where $K_{\nu}$ is a modified Bessel function (REMOVE?).
Let $\nu$ be half integer, i.e., $\nu = p + 1/2$ ($p \in
\mathbb{Z}^+$), then
$$k_{\nu=p+1/2}(\x,\x') = e^{-\sqrt{2\nu}\frac{\|\x-\x'\|}{\ell}}
\frac{\Gamma(p+1)}{\Gamma(2p+1)} \sum_{i=0}^p
\frac{(p+i)!}{i!(p-i)!}\left(\sqrt{8\nu}\frac{\|\x-\x'\|}{\ell}\right)^{p-i}$$
is the class of Mat\'ern covariance functions with half-integer $\nu$.
$$\nu > 0$$
$$\ell > 0$$
When $\nu \rightarrow \infty$ we obtain the squared exponential
covariance function. Most popular cases of the Mat\'ern functions are
$p=0$ (exponential kernel), $p=1$ and $p=2$:
$$k_{\nu=1/2}(\x,\x') = e^{-\frac{\|\x-\x'\|_2}{\ell}}$$
$$k_{\nu=3/2}(\x,\x') = \left(1+\sqrt{3}\frac{\|\x-\x'\|_2}{\ell}
\right) e^{-\sqrt{3}\frac{\|\x-\x'\|_2}{\ell}}$$
$$k_{\nu=5/2}(\x,\x') = \left(1+\sqrt{5}\frac{\|\x-\x'\|_2}{\ell}
 + \frac{5\|\x-\x'\|_2^2}{3\ell^2}\right)
e^{-\sqrt{5}\frac{\|\x-\x'\|_2}{\ell}}$$

\section{Rational Quadratic kernels}
$$k_{RQ} = \left(1+\frac{\|\x-\x'\|_2^2}{2\alpha\ell^2}
\right)^{-\alpha}$$
$$\alpha > 0$$
$$\ell > 0$$

\end{document}

% LocalWords:  Logscale


\SecDef{decomposition}{Decomposition of the 6-bit APN permutation}

Let $S_0$ be the APN permutation of $\field{6}$ proposed in \cite{DillonAPN}. The look-up table of $S_0$ is given in \TabRef{dillon}.

\FigTex{dillon.tex}

\subsection{TU-decomposition}

\FigTex{lats.tex}

The first step to the decomposition of $S_0$ completely resembles the reverse-engineering of the GOST S-Box. We start by looking at the visualization of the LAT of $S_0$ (see \FigRef{dillon-lat}). In the same way, there are 7 special columns and their indices (i.e., the linear masks defining the coordinates of $S_0$) together with $\hex{00}$ form a 3-dimensional linear subspace $V \subseteq \field{6}$:
$$
V = \pset{\hex{00},\hex{04},\hex{0a},\hex{0e},\hex{10},\hex{14},\hex{1a},\hex{1e}} = \Span(\hex{04},\hex{0a},\hex2{10}).
$$
Following the same path as in \ChapRef{kuz}, we compose $S_0$ with a linear map to ``move'' the special lines to the left. Let $\eta\in \linbij{6}$ be such that
\begin{align*}
& \eta(\hex{01}) \eqdef \hex{04}, \eta(\hex{02}) \eqdef \hex{0a}, \eta(\hex{04}) \eqdef \hex{10}, \\
& \eta(\hex{08}) \eqdef \hex{01}, \eta(\hex{10}) \eqdef \hex{02}, \eta(\hex{20}) \eqdef \hex{20},\\
\end{align*}
where the first 3 values correspond to the basis of $V$ and the last 3 values were chosen arbitrarily to complete the map to bijection. As a result, the special columns are aligned to the left in the LAT of $\eta^{\top} \circ S_0$, see \FigRef{eta-dillon-lat}. Furthermore, a clear square-based structure emerged in the LAT. Note that in the case of the GOST S-Box, the lines of the LAT had also to be reordered, and the inverse of the same linear mapping accidentally could be used to do it. Here, only the output of the S-Box has to be composed with a linear map. Indeed, already a white $8\times 8$ square is observed in the top-left corner. In the decomposition of the GOST S-Box, it suggested multiset properties which led to the high-level TU-decomposition. The same decomposition is obtained for $S_0$ as well.

\begin{proposition}[TU-decomposition of $\eta^{\top}\circ S_0$]
There exist 16 permutations $T_{\hex{0}},\ldots, T_{\hex{7}},U_{\hex{0}},\ldots, U_{\hex{7}}$ of $\field{3}$ such that for all $l,r \in \field{3}$
$$
\eta^{\top}\times S_0(l,r) = \pround{U_{T_r(l)}(r), T_r(l)}.
$$
The codebooks of the keyed permutations $T$ and $U$ are given in \FigRef{tu}.
\FigTex{tu.tex}
\end{proposition}

The algebraic degrees of the functions $(x,k) \mapsto T_k(x)$ and $(x,k) \mapsto T_k^{-1}(x)$ are 3 and 2 respectively. For $U$, the respective degrees are 2 and 3. This observation suggests that $U$ and $T^{-1}$ should be easier to decompose and that these keyed permutations may be related. We applied the linear equivalence algorithm from~\cite{LinAffEQ} to the mappings $(x,k) \mapsto (T_k^{-1}(x),k)$ and $(x,k) \mapsto (U_k(x),k)$ and found that they are linearly related.

\begin{proposition}[Linear Equivalence of $T^{-1}$ and $U$]
Let $T,U$ be permutations of $\field{6}$ given by
$$
T(x,k) \eqdef (T_k(x),k), U(x,k) \eqdef (U_k(x), k).
$$
Let $M_U,M_U' \in \linbij{8}$ be given by
$$
M_U \eqdef \matb{
0 & 1 & 1 & 0 & 1 & 0 \\
1 & 0 & 0 & 0 & 1 & 0 \\
0 & 0 & 1 & 0 & 1 & 1 \\
0 & 0 & 0 & 1 & 0 & 0 \\
0 & 0 & 0 & 0 & 1 & 0 \\
0 & 0 & 0 & 0 & 1 & 1 \\
},~~ M_U' \eqdef \matb{
1 & 1 & 0 & 0 & 0 & 0 \\
0 & 1 & 0 & 0 & 1 & 0 \\
1 & 0 & 1 & 1 & 1 & 1 \\
0 & 0 & 0 & 1 & 0 & 0 \\
0 & 0 & 0 & 0 & 1 & 0 \\
0 & 0 & 0 & 0 & 1 & 1 \\
}.
$$
Then
$$
U = M_U' \circ T^{-1} \circ M_U.
$$
\end{proposition}

Since $T^{-1}$ and $U$ are linear-equivalent, it is enough to decompose one of them. Since the linear layer $\eta^{\top}$ was applied at the output side, it could ``obfuscate'' some algebraic structure in $U$. Therefore, we choose to decompose $T^{-1}$. Afterwards, the two decompositions will be joined and the linear layer in between will be simplified.

\subsection{Decomposition of $T^{-1}$}

\FigTex{simplify-t.tex}

\textbf{Step 1.}
Similarly to the decomposition process of the GOST S-Box, we ``detach'' a Feistel function from $T$ in order to obtain a new keyed permutation $T'$ such that $T'_k(0) = 0$ for all $k \in \field{3}$. Detaching such function at the input and at the output of $T^{-1}$ leads to the Feistel functions $t$ and $\hat{t}$ respectively, a permutations of $\field{3}$ given by
$$
t(k) \eqdef T_k(0),~~  \hat{t}(k) \eqdef T_k^{-1}(0).
$$
$t$ is linear and $\hat{t}$ has algebraic degree 2. We conclude that detaching $t$ at the input of $T^{-1}$ leads to simpler decomposition. Let $T'_{\hex{0}},\ldots,T'_{\hex{7}}$ be permutations of $\field{3}$ given by, for any $k \in \field{3}$,
$$
T'_k(x) \eqdef T_k(x) \oplus t(k).
$$
The result of this step is shown in \FigRef{simplify-t-a}.

\textbf{Step 2.}
Due to the algebraic nature of the S-Box, we may expect an algebraic structure. We consider the field
$$
\fielde{3} \simeq \field{}[\WW]/(\WW^3+\WW+1),
$$
The primitive element $\WW$ will be used to express field elements. As was noted in \ChapRef{prelim}, from now on the isomorphism between $\field{3}$ and $\fielde{3}$ will be implicit.

\FigTex{tp-inter.tex}

We now apply Lagrange interpolation method to each permutation $T'^{-1}_k$ to obtain its polynomial representation, see \TabRef{tp-inter}. It is clear that the coefficients of the nonlinear part (i.e. the coefficients of $x^3, x^5, x^6$) are independent of $k$ for each $T'^{-1}_k$. This leads to the following proposition, illustrated in \FigRef{simplify-t-b}.

\begin{proposition}
Let $N$ be a permutation of $\field{3}$ given by its polynomial representation in $\fielde{3}$:
$N(x) \eqdef \WW^3x^6 + \WW^1x^5 + \WW^6x^3.$
Let $L_{\hex{0}},\dots,L_{\hex{7}}$ be permutations of $\field{3}$ given by
$$
L_k(x) \eqdef T'^{-1}_k(x) \oplus N(x).
$$
Then for any $k\in \field{3}$, $L_k$ is linear.
\end{proposition}

\textbf{Step 3.}
We continue by simplifying the nonlinear function $N$. We do so by composing a linear bijection of our choice with $N$ and $L_k$. Its inverse will then merge to the outer linear layer. It turns out that $N$ can be transformed into the finite field inverse function, which we denote $\inv\colon \field{3} \to \field{3}$. Its polynomial representation is
$$
\inv(x) = x^6.
$$

\begin{proposition}
Let $p \in \linbij{3}$ be given by its $\fielde{3}$-polynomial
$$
p(x) \eqdef \WW^2x^4 + x^2 + x.
$$
Then
$$
p(N(x)) = x^6 = \inv(x)
$$ is the inversion in the finite field, and also for all $k \in \field{3}$
$$
p(L_k(x)) = l_2(k+\WW)x^2 + l_4(k+\WW)x^4,
$$
where $l_2,l_4 \in \linbij{3}$ are given by
\begin{align*}
l_2(x) &\eqdef \WW x^4 + \WW^2x^2 + x, \\
l_4(x) &\eqdef x^4 + \WW^4x^2 + \WW x.
\end{align*}
\end{proposition}

Note that $k+\WW$ was used because $L_{\hex{2}} = 0$ ($\WW$ corresponds to $\hex{2}$), so that $l_2$ and $l_4$ are linear. The composition of $p$ with $N$ and $L_k$ is illustrated in \FigRef{simplify-t-c}.

\textbf{Step 4.}
The next step is to simplify the linear bijections $l_2$ and $l_4$. By composing $l_2$ with an arbitrary linear bijection $q \in \linbij{3}$, an arbitrary linear bijection may be obtained. However, $l_4$ has to be composed with the same linear map $q$. Therefore, $q$ should simplify both $l_2$ and $l_4$. By exhaustive search of $q$, we found $q$ such that
$l_2(q(x)) = x^4, ~~l_4(q(x)) = x^2$. These expressions lead to a simple expression for $p\circ L_k$, as it is shown by the following proposition.

\begin{proposition}
\PropLabel{t-eq}
Let $q \in \linbij{3}$ be given by its $\fielde{3}$-polynomial:
$$
q(x) \eqdef \WW^3x^4 + \WW^5x^2 + \WW^3x.
$$
Then, for all $x \in \fielde{3}$,
\begin{equation}
    \EqLabel{pinvTinv}
    p(T'^{-1}_{k}(x))
        = x^6 + x^2k'^4 + x^4k'^2
        = (x + k')^6 + k'^6,
\end{equation}
where $k' = q^{-1}(k + \WW)$.
\end{proposition}
\begin{proof}
Recall that
$$
p(T'^{-1}_{k}(x)) = x^6 + l_2(k+\WW)x^2 + l_4(k+\WW)x^4,
$$
where
$$
l_2(k+\WW) = l_2(q(q^{-1}(k+\WW))) = (q^{-1}(k+\WW))^4 = k'^4.
$$
Similarly,
$l_2(k+\WW) = k'^2$.
\end{proof}

The graphical representation of the effect of this proposition is shown in \FigRef{simplify-pl}.

\FigTex{simplify-pl}

\textbf{Step 5}.
The final step is to simplify the affine layers. The application of $q'$ and the addition of $\WW$ can be moved from the Feistel function into the input and output linear layers of $T'^{-1}$. The output affine layer of $T'^{-1}$ can be omitted, since it corresponds to an S-Box affine-equivalent to $S_0$. The simplification process is illustrated in \FigRef{simplify-t-linear}.

\FigTex{simplify-t-linear}

\begin{proposition}[Decomposition of $T'^{-1}$]
For all $x,k \in \fielde{3}$,
$$
T^{-1}_k(x) = p^{-1}(k'^6 + (x + k'')^6),
$$
where
\begin{align*}
    & k' =  q^{-1}(k) \oplus \WW^6,\\
    & k'' = z(q^{-1}(k)) \oplus \WW^6,\\
    & z\in \linbij{3}, z(y) \eqdef t(q(y)) + y.
    & q \in \linbij{3}, q(y) \eqdef \WW^3y^4 + \WW^5y^2 + \WW^3y.
\end{align*}
\end{proposition}
\begin{proof}
Recall that
$$
T^{-1}_k(x) = T'^{-1}_k(x \oplus t(k)).
$$
Together with \PropRef{t-eq}, it is enough to have
$$
k'' = k' \oplus t(k).
$$
This is true, because
\begin{align*}
k' &= q^{-1}(k \oplus \WW) = q^{-1}(k) \oplus \WW^6, \\
k'' &= z(q^{-1}(k))\oplus \WW^6 = t(k) \oplus q^{-1}{k} \oplus \WW^6.
\end{align*}
\end{proof}

\subsection{Combining $T$ and $U$}

We can now obtain a decomposition of full $S_0$. Recall that
$$
U = M_U' \circ T^{-1} \circ M_U.
$$
The decomposition of $T$ follows from the decomposition of $T^{-1}$ by inverting $q$ in the middle. We omit all outer affine maps and merge the inner linear maps into one linear transformation of $\field{6}$. The resulting structure is given in \FigRef{middle-affine-a}, \FigRef{middle-affine-b}.

\FigTex{middle-affine.tex}

\newcommand\TI{V}
\begin{proposition}
\PropLabel{middle-simplified}
The APN permutation $S_0$ is affine-equivalent to the structure given in \FigRef{middle-affine-b}. Formally, let $\TI$ be the permutation of $\field{3}\times\field{3}$ given by
$$
\TI(x,k) = (\inv(x \oplus \inv(k)), k),
$$
and let $M \in \linbij{6}$ be given by its matrix:
$$
M = \matb{
1 & 0 & 1 & 1 & 1 & 1 \\
1 & 1 & 0 & 0 & 1 & 0 \\
0 & 0 & 1 & 1 & 1 & 0 \\
1 & 1 & 0 & 1 & 0 & 1 \\
1 & 1 & 1 & 1 & 1 & 0 \\
1 & 0 & 1 & 0 & 0 & 1 \\
}.
$$
Then the permutation $S_M$ of $\field{6}$ defined as $S_M \eqdef \TI^{-1} \circ \xorf{5,5} \circ M \circ \xorf{5,5} \TI$ is APN and is affine-equivalent to $S_0$.
\end{proposition}
\begin{proof}
The proposition follows form the TU-decomposition of $S_0$, decomposition of $T^{-1}$ and linear relation between $T^{-1}$ and $U$. Furthermore, $M$ is obtained through composition including linear maps $q$, $q^{-1}$, $z$ and $M_U$.
\end{proof}

We further studied how the middle affine layer can be changed while preserving the APN property. It turns out that, for affine layers $a,b\colon \field{3} \to \field{3}$, applying $a$ to both branches before $M$ and applying $b$ to both branches after $M$ always leads to an APN permutation affine-equivalent to $S_0$. This is formally stated and proved in \SecRef{affine-propagation}.

We observe that removing the constant addition (i.e. $\xorf{5,5}$) from the structure does not break the APN property. Therefore, it is left to simplify the linear map $M$. By exhaustive search over linear maps $a,b \in \linbij{3}$ we found that $M$ can be transformed into a $2\times 2$ matrix over $\fielde{3}$.

\begin{proposition}
Let $\TI$ be defined as in \PropRef{middle-simplified}. Let $M' \in \linbije{2}{3}$ be given by:
$$
M' \eqdef \matb{
\WW, \WW^6, \\
1, \WW
}.
$$
Then the permutation $S_{M'}$ of $\field{6}$ defined by
$$
S_{M'} \eqdef \TI^{-1} \circ M \circ \TI
$$
is APN.
\end{proposition}
\begin{proof}
Let $a,b\in \linbij{3}$ be given by $\fielde{3}$-polynomials:
\begin{align*}
    a(x) &= \WW x^4 + \WW x^2 + \WW^2 x,\\
    b(x) &= \WW x^4 + \WW^3 x^2 + \WW x,\\
\end{align*}
Then,
$$
M' = \parf{b} \circ M \circ \parf{a}.
$$
According to \ThmRef{affine-propagation}, $S_{M'}$ is APN.
\end{proof}

Observe that $M'$ happens to be an involution, making $S_{M'}$ an involution too due to its symmetric structure. Furthermore, $M'$ can be decomposed as a two-round Feistel Network with multiplication by $\WW$ as the Feistel function. The following theorem finalizes our decomposition, which is illustrated in \FigRef{final-decomposition}.

\newcommand\TJ{W}
\begin{theorem}[Decomposition of $S_0$]
\ThmLabel{main}
Let $\TJ,\Swap$ be permutations of $\field{3}\times\field{3}$ given by their bivariate $\fielde{3}$-polynomials:
\begin{align*}
    \TJ(x,k) = ((x + \WW k)^6 + k^6, k),~~ \Swap(x,k) = (k, x).
\end{align*}
Then the permutation $S_{\inv}$ of $\field{6}$ given by
$$
S_{\inv} \eqdef \TJ \circ \Swap \circ \TJ^{-1}
$$
is an APN involution and is affine-equivalent to $S_0$:
$$
S_0(x) = B(S_{\inv}(A(x) \oplus \hex{9}) \oplus \hex{4},
$$
where $A,B \in \linbij{6}$ are given by
$$
A =
\begin{bmatrix}
1 & 1 & 0 & 1 & 0 & 1 \\
1 & 1 & 1 & 1 & 0 & 0 \\
1 & 0 & 0 & 0 & 0 & 0 \\
0 & 0 & 0 & 1 & 0 & 1 \\
0 & 0 & 0 & 1 & 0 & 0 \\
0 & 0 & 0 & 1 & 1 & 0 \\
\end{bmatrix},
\hspace{1em}
B =
\begin{bmatrix}
0 & 1 & 1 & 1 & 0 & 1 \\
0 & 0 & 0 & 0 & 0 & 1 \\
0 & 0 & 1 & 1 & 1 & 0 \\
0 & 0 & 0 & 1 & 1 & 1 \\
0 & 0 & 1 & 0 & 1 & 0 \\
1 & 0 & 1 & 1 & 0 & 1 \\
\end{bmatrix}.
$$
\end{theorem}

\FigTex{final-decomposition.tex}


\subsection{Calibration curve preparation}

\begin{enumerate}
\item Taje the stock solution concentrate at \py{final_concentration_DexFITC} \py{final_concentration_units_DexFITC}
\item Do the dilution following table \ref{Courbe-calibration}
\item Use a 96 well plates to do the dilution
\item Perform the dilution in waterfall and always starting with the lowest concentration
\begin{table}[H]
\caption{Dilution to create the calibration curve of \py{Dex_Mw} kDa Dextran-FITC}
\label{Courbe-calibration}
\begin{pycode}

from tabulate import tabulate
infos = [
    ["Initial Concentration ($\mu g/mL$)", "Initial Volume ($\mu L$)", "Final concentration ($\mu g/mL$)", "Diluant Volume ($\mu L$)",  "Total Volume($\mu L$)"],
    [2000, 62.5, 1250, 37.5, 100],
    [2000, 37.5, 750, 62.5, 100 ],
    [2000, 12.5, 250, 87.5 , 100],
    [1250, 10.0, 125, 90.0, 100 ],
    [750, 10.0, 75, 90.0, 100],
    [250, 10.0, 25, 90.0, 100 ],
    [125, 10.0, 12.5, 90.0, 100 ],
    [75, 10.0, 7.5, 90.0, 100],
    [25, 10.0, 2.5, 90.0, 100 ]
]
table = r"\resizebox{\textwidth}{!}{" + tabulate(infos, headers="firstrow", tablefmt="latex_raw", numalign="center") + "}"
print(table)
\end{pycode}
\label{table-ensemencement}
\end{table}

\item Put aluminium foil on the plate
\end{enumerate}


% Created 2016-04-28 tor 16:53
% Intended LaTeX compiler: pdflatex
\documentclass{scrartcl}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble}
\newcommand{\tustin}{\frac{2}{h}\frac{z-1}{z+1}}
\author{Kjartan Halvorsen}
\date{2015-11-24}
\title{Computerized control - Final Exam - modified from Fall 2015}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={Computerized control - Final Exam - modified from Fall 2015},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 24.5.1 (Org mode 8.3.4)},
 pdflang={English}}
\begin{document}

\maketitle
The dynamic model of a ship with input \(u\) being the rudder angle and the output \(y\) being the heading (see figure \ref{fig:tanker}) can be described as a continuous-time second order system with a pole in the origin
\[ G(s) = \frac{K}{s(s + a)}. \]
For fully loaded, large tankers this dynamics is often unstable, meaning that \(a<0\) \footnote{Fossen, Thor I. Handbook of marine craft hydrodynamics and motion control. John Wiley \& Sons, 2011.} .
\begin{figure}[h]
\begin{center}
\includegraphics[]{tanker}
\caption{Heading of a ship controlled by rudder input.}
\label{fig:tanker}
\end{center}
\end{figure}

Consider for this exam the normalized continuous-time model of the tanker
\[ G(s) = \frac{1}{s(s - 1)}. \]
with the discrete-time model obtained by zero-order hold
\begin{equation*}
 H(z) = \frac{(-1+\mexp{h} -h)z + 1 - (1-h)\mexp{h}}{(z-1)(z-\mexp{h})}.
\end{equation*}
Specifically, use sampling time \(h=0.2\), which gives the (approximate) model
\begin{equation}
 H(z) = \frac{0.02z + 0.02}{(z-1)(z-1.2)} = \frac{0.02z + 0.02}{z^2-2.2z+1.2}.
\label{eq:model}
\end{equation}

\textbf{All answers should be well motivated!}

\section*{Problem 1}
\label{sec:orgheadline1}
\begin{enumerate}
\item In figure \ref{fig:complex-plane} draw the poles (crosses) and zero (circle) for the  discrete-time pulse-transfer function in \eqref{eq:model}.
\begin{figure}[h]
\begin{center}
\includegraphics[]{complex-plane}
\caption{Problem 1: Plot the poles and zeros of the discrete-time system.}
\label{fig:complex-plane}
\end{center}
\end{figure}
\item Assume that the tanker with model \eqref{eq:model} is stabilized using error-feedback and a PD-controller. The  Bode-diagram of the resulting \textbf{closed-loop} system is  given in figure \ref{fig:bode}. What is the bandwidth of the closed-loop system? At what frequency is the resonance peak?
\begin{figure}[h]
\begin{center}
\includegraphics[]{bode-closed}
\caption{Problem 1: Bode diagram of closed-loop system with PD-control}
\label{fig:bode}
\end{center}
\end{figure}
\end{enumerate}

\section*{Problem 2}
\label{sec:orgheadline2}
Figure \ref{fig:rst} shows a system controlled with an RST controller. Note that the system includes an anti-aliasing filter modelled as a pure time-delay of two sampling periods. What is the closed-loop pulse-transfer function from the disturbance \(d\) to the output \(y\)? You do not need to multiply the polynomials. It is sufficient to state your answer in terms of \(A(z)\), \(B(z)\), \(R(z)\), \(S(z)\) and \(z^2\).
\begin{figure}[h]
\begin{center}
\includegraphics[]{rst-anti-aliasing}
\caption{Problem 2: Two-degree-of-freedom controller with anti-aliasing filter.}
\label{fig:rst}
\end{center}
\end{figure}

\section*{Problem 3}
\label{sec:orgheadline3}
When designing an RST-controller for the system in Problem 2, \(R(z)\) and \(S(z)\) are determined from a diophantine equation, based on the required placement of the closed-loop poles. Assume the following desired closed-loop denominator:
\begin{equation}
A_{cl} = \underbrace{(z-p_1)(z-p_2)z^2}_{A_c}\underbrace{(z-p_3)^3}_{A_o}
\end{equation}
\begin{enumerate}
\item Write the diophantine equation in terms of \(A_c(z)\), \(A_o(z)\), \(A(z)\), \(B(z)\), \(R(z)\), \(S(z)\) and \(z^2\).
\item Let the controller polynomials \(R(z)\) and \(S(z)\) have the same order. Determine this order, so that all the controller parameters can be determined from the diophantine equation. Note that you only need to determine the \textbf{order} of the controller. You do not need to write the equation for the controller parameters.
\end{enumerate}

\section*{Problem 4}
\label{sec:orgheadline4}
The controllable canonical state-space representation of \eqref{eq:model} is given by
\begin{equation}
\begin{split}
x(k+1) &= \bbm 2.2 & -1.2\\1 & 0\ebm \x(k) + \bbm 1\\0\ebm u(k)\\
y(k) &= \bbm 0.02 & 0.02 \ebm x(k),
\end{split}
\end{equation}
with
\[ x(k) = \bbm x_1(k)\\x_2(k)\ebm. \]
Introduce the state-feedback law \(u(k) = -l_1x_1(k) -l_2x_2(k)\) and determine \(l_1\) and \(l_2\) so that the closed-loop system has the characteristic polynomial
\[ (z-0.9+0.1i)(z-0.9-0.1i) = z^2 -1.8z + 0.82. \]

\section*{Solution}
\label{sec:orgheadline9}
\subsection*{Problem 1}
\label{sec:orgheadline5}
\begin{enumerate}
\item Poles and zeros
\begin{center}
\includegraphics[]{complex-plane-sol-final}
\end{center}
\item Bandwidth and resonance
\begin{center}
\includegraphics[]{bode-closed-sol}
\end{center}
\end{enumerate}

\subsection*{Problem 2}
\label{sec:orgheadline6}
To compute the pulse-transfer function from \(d\) to \(y\), assume \(u_c=0\). We get
\begin{equation*}
\begin{split}
Y &= D + \frac{B}{A} U = D - \frac{B}{A}\frac{S}{R}\frac{1}{z^2} Y\\
Y + \frac{BS}{ARz^2} Y &= D\\
Y &= \frac{1}{1 + \frac{BS}{ARz^2}} D\\
 &= \frac{A(z)R(z)z^2}{A(z)R(z)z^2 + B(z)S(z)} D
\end{split}
\end{equation*}

\subsection*{Problem 3}
\label{sec:orgheadline7}
\begin{enumerate}
\item The diophantine equation becomes
\[ A(z)R(z)z^2 + B(z)S(z) = A_c(z)A_o(z) \]
\item The right hand side of the diophantine equation has order 7, hence the left hand side must have the same order. Since \(A(z)z^2\) has order 4, then \(R(z)\) must have order 3. We choose \(R(z)\) and \(S(z)\) to have the same order (which is a smart choice because then the pulse-transfer function of the controller does not introduce a time-delay), we get
\[ \frac{S(z)}{R(z)} = \frac{s_0z^3 + s_1z^2 + s_2z + s_3}{z^3+r_1z^2 + r_2z + r_3} \]
which has 7 parameters. The diophantine equation gives 7 equations to determine uniquely the 7 control parameters. Note that the terms \(A(z)R(z)z^2\) and \(B(z)S(z)\) do \textbf{not} have to have the same order.
\end{enumerate}

\subsection*{Problem 4}
\label{sec:orgheadline8}
With the control law we get the closed-loop system
\begin{equation*}
\begin{split}
 x(k+1) &= \bbm 2.2 & -1.2\\1 & 0\ebm x(k) - \bbm l_1 & l_2\\ 0 & 0 \ebm x(k)\\
        &= \bbm 2.2-l_1 & -1.2-l_2\\1 & 0\ebm x(k)
\end{split}
\end{equation*}
which is also on controllable canonical form. Thus we can immediately write the characteristic polynomial of the closed-loop system as
\[ z^2 + (-2.2+l_1)z + (1.2 + l_2). \]
It is also straight-forward to write the characteristic polynomial using the formula
\begin{equation*}
\begin{split}
\det \left( zI - \bbm 2.2-l_1 & -1.2-l_2\\1 & 0\ebm \right)
        &= \det \bbm z -2.2+l_1 & 1.2+l_2\\-1 & z\ebm\\
	&= (z-2.2+l_1)z + (1.2+l_2) = z^2 + (-2.2+l_1)z + (1.2+l_2).
\end{split}
\end{equation*}

Comparing coefficients with the desired characteristic polynomial
\[ z^2 -1.8 + 0.82 \] gives the solution
\begin{align*}
l_1 &= -1.8 + 2.2 = 0.4\\
l_2 &= 0.82 - 1.2 = - 0.38
\end{align*}

With \[ m_0 = \frac{A_c(1)}{B(1)} = \frac{1-1.8+0.82}{0.02+ 0.02} = 0.5 \]
and
\[ u = -Lx + m_0u_c, \]
the Bode-diagram of the closed-loop system becomes
\begin{center}
\includegraphics{bode-statefb-closed-sol}
\end{center}
\end{document}


%!TEX root=report.tex
\subsection{Splines}
\label{section:result-splines}

Using nine knots with a year in between and letting the sine and cosine functions enter at each knot, one can allow for different seasons. The columns in $X$ can be visualized as in Figure \ref{fig:splines-x-columns}.
\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{figures/splines-x-columns}
	\caption{Constant, trend and acceleration in the three uppermost plots. Sine and cosine hinge functions in the bottom plot.}
	\label{fig:splines-x-columns}
\end{figure}

A RMSE diagnostic shows that the residuals are indeed smaller (max standard OLS was 0.135).

\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{figures/splines-rmse}
	\caption{RMSE for each position using basis expansion.}
	\label{fig:splines-rmse}
\end{figure}

Looking at the west coast of Greenland now using a basis expansion, the overall fit (Figure \ref{fig:splines-selected-0-fit}) looks improved. For some reason 2009 still causes some issues, which is partially seen in the residuals (Figure \label{fig:splines-selected-0-residual}). Another thing to notices is that the curves are a lot more smooth, but this is simply because only the first 3 frequencies (year, half year and quarter year) was used. This was to prevent the degrees of freedom to drop dramatically, dude to the otherwise huge amount of parameters.
\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{figures/splines-selected-0-fit}
	\caption{Measurements are blue, the OLS fit is red.}
	\label{fig:splines-selected-0-fit}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{figures/splines-selected-0-residual}
	\caption{The OLS residuals are blue.}
	\label{fig:splines-selected-0-residual}
\end{figure}

Unfortunately for the south pole, the resulting fits do display some cusps and overfitting behavior. This is particularly seen at 2006 and 2008. Here the residuals haven't improved much, however interestingly enough there now appear to be a two year seasonal trend, between 2003 and 2007.

\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{figures/splines-selected-1-fit}
	\caption{Measurements are blue, the OLS fit is red.}
	\label{fig:splines-selected-1-fit}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{figures/splines-selected-1-residual}
	\caption{The OLS residuals are blue.}
	\label{fig:splines-selected-1-residual}
\end{figure}


\documentclass{article}
\usepackage[margin = .7in]{geometry}
\usepackage[dvipdfmx]{graphicx}
\usepackage{listings}
\usepackage{amsmath}
\usepackage{bm}
\lstset{%
  language={python},
  basicstyle={\small},%
  identifierstyle={\small},%
  commentstyle={\small\itshape},%
  keywordstyle={\small\bfseries},%
  ndkeywordstyle={\small},%
  stringstyle={\small\ttfamily},
  frame={tb},
  breaklines=true,
  columns=[l]{fullflexible},%
  numbers=left,%
  xrightmargin=0zw,%
  xleftmargin=3zw,%
  numberstyle={\scriptsize},%
  stepnumber=1,
  numbersep=1zw,%
  lineskip=-0.5ex%
}

\begin{document}
\title{STAT6011/7611/6111/3317 \\
COMPUTATIONAL STATISTICS (2016 Fall)}
\author{Kei Ikegami (u3535947)}
\maketitle

\section{}
\subsection{}

	Consider model 1 in section 4.1. I calculate the marginal likelihood by the algorithm in 3.2.2.
	\par
	The prior distributions are as follows.
	\begin{align*}
		\alpha &\sim N(\mu_{\alpha}, \sigma^2_{\alpha}) \\
		\beta &\sim N(\mu_{\beta}, \sigma^2_{\beta})\\
		\sigma^2 &\sim IG(a, b)
	\end{align*}
	First, I calculate $\alpha$'s full conditional distribution as follows.
	\begin{align*}
		p(\alpha | {\bf y}, {\bf x}, t, \beta, \sigma^2) &\propto p({\bf y} | \alpha, \beta, \sigma^2)^t p(\alpha) \\
		&= \left\{ \Pi_{i=1}^{N} \left( \frac{1}{\sqrt{2\pi \sigma^2}} \exp \left( - \frac{(y_i - \alpha - \beta(x_i - \bar{x}))^2}{2\sigma^2}\right)^t \right) \right\} \frac{1}{\sqrt{2\pi \sigma_{\alpha}}} \exp \left( - \frac{(\alpha - \mu_{\alpha})^2}{2\sigma_{\alpha}^2}\right)\\
		&\propto \exp \left( - \frac{1}{2\sigma^2} \left( t \sum_i (y_i - \beta(x_i - \bar{x}))^2 - 2t \alpha \sum_i (y_i - \beta(x_i - \bar{x}))  + \alpha^2 Nt \right) + \frac{1}{2\sigma_{\alpha}^2} (\alpha^2 - 2\mu_{alpha} \alpha + \mu_{\alpha}^2)\right)\\
		&= \exp \left( - \left( \frac{\sigma_{\alpha}^2 Nt + \sigma^2}{\sigma^2 \sigma_{\alpha}^2} \alpha^2 - \left( \frac{\sigma_{\alpha}^2 t \sum_i (y_i - \alpha - \beta (x_i -\bar{x})) + \sigma^2 \mu_{\alpha}}{\sigma^2 \sigma_{\alpha} ^2} \alpha \right)\right)\right)\\
		&\propto \exp \left( -\frac{\sigma_{\alpha}^2 Nt + \sigma^2}{\sigma^2 \sigma_{\alpha}^2} \left( \alpha - \frac{\sigma_{\alpha}^2 t \sum_i (y_i - \beta(x_i -\bar{x})) + \sigma^2 \mu_{\alpha}}{\sigma_{\alpha}^2 Nt + \sigma^2} \right)^2 \right)\\
	\end{align*}
	This means $\alpha$'s full conditional distribution is $N \left(\frac{\sigma_{\alpha}^2 t \sum_i (y_i - \beta(x_i -\bar{x})) + \sigma^2 \mu_{\alpha}}{\sigma_{\alpha}^2 Nt + \sigma^2}, \frac{\sigma^2 \sigma_{\alpha}^2}{\sigma_{\alpha}^2 Nt + \sigma^2} \right)$.
	\par
	Secondly, I compute $\beta$'s full conditional distribution as follows.
	\begin{align*}
		p(\beta | {\bf y}, {\bf x}, t, \alpha, \sigma^2) &\propto p({\bf y} | \alpha, \beta, \sigma^2)^t p(\beta)\\
		&\propto \exp \left( - \left( \frac{1}{2\sigma^2} t\sum_i (y_i - \alpha - \beta(x_i -\bar{x}))^2 + \frac{1}{2\sigma_{\beta}^2} (\beta - \mu_{\beta})^2 \right)\right)\\
		&\propto \exp \left( - \left( \frac{\sigma_{\beta}^2 t \sum_i (x_i - \bar{x})^2 + \sigma^2}{2\sigma^2 \sigma_{\beta}^2} \beta^2 - \frac{\sigma_{\beta}^2 t \sum_i (x_i- \bar{x})(y_i - \alpha) + \sigma^2 \mu_{\beta}}{\sigma^2 \sigma_{\beta}^2} \beta \right) \right)\\
		&\propto \exp \left( - \frac{\sigma_{\beta}^2 t \sum_i (x_i - \bar{x})^2 + \sigma^2}{2\sigma^2 \sigma_{\beta}^2} \left( \beta - \frac{\sigma_{\beta}^2 t \sum_i (x_i- \bar{x})(y_i - \alpha) + \sigma^2 \mu_{\beta}}{\sigma_{\beta}^2 t \sum_i (x_i - \bar{x})^2 + \sigma^2}\right)^2 \right)\\
	\end{align*}
	This means $\beta$'s full conditional distribution is $N\left( \frac{\sigma_{\beta}^2 t \sum_i (x_i- \bar{x})(y_i - \alpha) + \sigma^2 \mu_{\beta}}{\sigma_{\beta}^2 t \sum_i (x_i - \bar{x})^2 + \sigma^2}, \frac{\sigma^2 \sigma_{\beta}^2}{\sigma_{\beta}^2 t \sum_i (x_i - \bar{x})^2 + \sigma^2} \right)$.
	\par
	Third, I derive $\sigma^2$'s full conditional distribution.
	\begin{align*}
		p(\sigma^2 | {\bf y}, {\bf x}, t, \alpha, \beta) &\propto p({\bf y} | \alpha, \beta, \sigma^2)^t p(\sigma^2) \\
		&= \left\{ \Pi_{i=1}^{N} \left( \frac{1}{\sqrt{2\pi \sigma^2}} \exp \left( - \frac{(y_i - \alpha - \beta(x_i - \bar{x}))^2}{2\sigma^2}\right)^t \right) \right\} \exp \left( -\frac{1}{b \sigma^2} \right) \frac{1}{\gamma(a)} b^{-a} (\sigma^2)^{-(a + 1)}\\
		&\propto (\sigma^2)^{-\frac{Nt}{2} + a + 1} \exp \left( - \frac{bt \sum_i (y_i - \alpha - \beta(x_i -\bar{x}))^2 + 2}{2b\sigma^2} \right)\\
	\end{align*}
	This implies that the full conditional distribution of $\sigma^2$ is $IG\left( \frac{Nt}{2} + a, \frac{2b}{bt \sum_i (y_i - \alpha - \beta(x_i -\bar{x}))^2 + 2} \right)$.
	\par
	I carry out Gibbs sampling by using these results.

\end{document}


%-----------------------------------------------------------------------
% Beginning of proc-l-template.tex
%-----------------------------------------------------------------------
%
%     This is a topmatter template file for PROC for use with AMS-LaTeX.
%
%     Templates for various common text, math and figure elements are
%     given following the \end{document} line.
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%     Remove any commented or uncommented macros you do not use.

\documentclass{proc-l}

%     If you need symbols beyond the basic set, uncomment this command.
%\usepackage{amssymb}

%     If your article includes graphics, uncomment this command.
%\usepackage{graphicx}

%     If the article includes commutative diagrams, ...
%\usepackage[cmtip,all]{xy}

%     Update the information and uncomment if AMS is not the copyright
%     holder.

\copyrightinfo{2020}{}

\newtheorem{theorem}{Theorem}[subsection]
\newtheorem{lemma}[theorem]{Lemma}

\theoremstyle{definition}
\newtheorem{definition}[theorem]{\bf Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{xca}[theorem]{Exercise}

\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}

\numberwithin{equation}{section}

%aliases

\newcommand{\comment}[1]{}
\newcommand{\R}{\mathbb{R}}
\newcommand{\rank}[1]{\textrm{rank}({#1})}
\newcommand{\im}[1]{\textrm{Im}({#1})}
\newcommand{\x}{\times}
\newcommand{\id}[1]{\textrm{Id}_{{#1}\times {#1}}}
\renewcommand{\ker}[1]{\textrm{ker}({#1})}
\renewcommand{\span}[1]{\textrm{span}\{ {#1} \}}

% set section default counter
\setcounter{section}{1}

% do not show page numbers
\pagenumbering{gobble}

\begin{document}

% \title[short text for running head]{full title}
\title{Understading linear maps\\ with Gauss-Jordan elimination}

%    Only \author and \address are required; other information is
%    optional.  Remove any unused author tags.

%    author one information
% \author[short version for running head]{name for top of paper}
% \author{Ramon Massoni}
\address{}
\curraddr{}
\email{}
\thanks{}

%    author two information
% \author{Ferran Mui\~nos}
\address{}
\curraddr{}
\email{}
\thanks{}

%    \subjclass is required.
%    \subjclass[2010]{Primary }

%     \date{\today}

%     \dedicatory{}

%    "Communicated by" -- provide editor's name; required.
%     \commby{}

%    Abstract is required.
\begin{abstract}
We provide a condensed proof of the fundamental theorem of linear algebra in a Gauss-Jordan elimination fashion.
\end{abstract}

\maketitle

\subsection{}
We denote $\R^{m \x n}$ the collection of all matrices of shape $m\x n$.

\subsection{}
Any matrix $A\in\R^{m\times n}$ is associated with a linear map that we can denote $f_A:\R^n\to\R^m$.

\subsection{}
Column-wise elementary operations can be of either of three kinds:
\begin{enumerate}
\item Permutation of columns.
\item Replacing a column $c_i$ with $\lambda c_i$ with $\lambda\neq 0$.
\item Replacing a column $c_i$ with $c_i + \mu c_j$ for any $\mu$.
\end{enumerate}
We denote $e(A)$ the transform of a matrix $A$ by a given elementary operation $e$.

\subsection{}
Hereinafter our discussion will be based entirely on column-wise operations, so we will drop ``column-wise'' and ``column'' from all our statements whenever possible.

\subsection{}
Each elementary operation of $A$ is equivalent to multiplying $A$ by an appropriate matrix $E \in \R^{n\times n}$. Hence, we say that $E$ is the matrix representing the elementary operation $e$ whenever $e(A) = AE$ for all possible matrices $A$.

\subsection{}
Note that elementary operations preserve the rank, i.e., $\rank{A} = \rank{e(A)} = \rank{AE}$. Can you see why?

\subsection{}
In Gauss-Jordan elimination we always keep track of two matrices, that we can simply represent as a tuple $(A, B)$ where $A\in\R^{m\x n}$ and $B\in\R^{n\x n}$. We convene $(A, B) \sim (A', B')$ to mean that $(A', B')$ results from $(A, B)$ by applying the same sequence of elementary operations to both matrices.

\subsection{}
Gauss-Jordan elimination is the method whereby the pair $(A, \id{n})$ is transformed, by means of applying a sequence of elementary operations, into another pair $(L, B)$ where $L$ has a canonical form known as ``reduced echelon form''. This means that $L$ satisfies the following requirements:
\begin{enumerate}
\item $L$ is a lower column echelon matrix.
\item The leading entries of $L$ are $1$.
\item The leading entries of $L$ are the only non-zero entries in their row.
\end{enumerate}
Following our notation: $(A, \id{n})\sim (L, B)$.

\subsection{}
Given an input matrix $A$, Gauss-Jordan elimination leads to a unique matrix $L$ in reduced echelon form. We can stress this fact by defining a Gauss-Jordan algorithm $\textrm{GJ}: A\mapsto L(A)$ that accepts $A$ as input and returns its unique reduced column echelon form.

\subsection{Proposition}
If $(A, \id{n}) \sim (L, B)$ then $L=AB$.
\begin{proof}
Let $E = E_1\cdots E_k$ the product of the matrices encoding the elementary operations that have been applied. By definition $L = AE$ and $B = \id{n} E = E$, so it is clear that $B=E$, i.e., $B$ keeps track of all the elementary operations that have been applied to A that give $L$. It follows that $L=AB$.
\end{proof}

\subsection{}
Observe that $\rank{B} = n$. Can you figure out why?

\subsection{Proposition}
If $A$ is a square, full-rank matrix, then $L(A) = \id{n}$ and $B=A^{-1}$.
\begin{proof}
If $(A, \id{n}) \sim (\id{n}, B)$ for some matrix $B$, then we know by the previous discussion that $\id{n} = AB$, so $B=A^{-1}$.
\end{proof}

\subsection{}
For the next discussion, let $(A, \id{n}) \sim (L, B)$, with $L$ in reduced echelon form. Let $B=[\;B_1 \;| \ldots |\; B_n\;]$ and $L=[\;L_1\;|\ldots|\; L_r\;|\;0\;|\ldots|\;0\;]$ specified by their respective columns, where $L_1,\ldots,L_r$ are non-zero column vectors.

\subsection{Theorem}
$\rank{A} = r$, where $r$ is the number of non-zero columns in $L$. Moreover, $L_1,\ldots,L_r$ form a basis of the column space of $A$, denoted $C(A)$. Therefore, $\dim{C(A)} = r$.

\subsection{}
Observe that the last $n - r$ columns of $L$ are zero. What does this mean?

\subsection{Theorem}
$B_1, \ldots, B_{n-r}$ form a basis of the null-space of $A$, denoted $N(A)$.
\begin{proof}
The following are known facts:
\begin{enumerate}
\item $B_1, \ldots, B_n$ is a basis of $\R^n$; in particular, any subset is linearly independent.
\item The vectors $L_i = AB_i$ for each $1\leq i\leq r$ form a linearly independent set.
\item The vectors $L_i=AB_i$ are zero for $r + 1\leq i\leq n$. Consequently,
\[
S=\span{B_i\;|\; r + 1\leq i \leq n} \} \subset N(A).
\]
\end{enumerate}
Let's check that $N(A)\subset S$. By (1) we can write any $v\in N(A)$ as a linear combination $v=\sum_{i=1}^n\lambda_i B_i$. From the fact that $v\in N(A)$ and (3), it follows that $0 = Av = \sum_{i=1}^r \lambda_i AB_i = \sum_{i=1}^r \lambda_i L_i$. By (2) this cannot hold unless $\lambda_i = 0$ for $i=1,\ldots, r$. Then $v\in S$.
\end{proof}

\subsection{}
It follows that $\dim{N(A)} = n - r$.

\subsection{Fundamental Theorem of Linear Algebra}
For any matrix $A\in\R^{m\times n}$ the following identity holds:
\[
\dim{N(A)} + \dim{C(A)} = n.
\]

%    Text of article.

%    Bibliographies can be prepared with BibTeX using amsplain,
%    amsalpha, or (for "historical" overviews) natbib style.
\bibliographystyle{amsplain}
%    Insert the bibliography data here.

\end{document}

\comment{

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%    Templates for common elements of a journal article; for additional
%    information, see the AMS-LaTeX instructions manual, instr-l.pdf,
%    included in the PROC author package, and the amsthm user's guide,
%    linked from http://www.ams.org/tex/amslatex.html .

%    Section headings
\section{}
\subsection{}

%    Ordinary theorem and proof
\begin{theorem}[Optional addition to theorem head]
% text of theorem
\end{theorem}

\begin{proof}[Optional replacement proof heading]
% text of proof
\end{proof}

%    Figure insertion; default placement is top; if the figure occupies
%    more than 75% of a page, the [p] option should be specified.
\begin{figure}
\includegraphics{filename}
\caption{text of caption}
\label{}
\end{figure}

%    Mathematical displays; for additional information, see the amsmath
%    user's guide, linked from http://www.ams.org/tex/amslatex.html .

% Numbered equation
\begin{equation}
\end{equation}

% Unnumbered equation
\begin{equation*}
\end{equation*}

% Aligned equations
\begin{align}
  &  \\
  &
\end{align}

%-----------------------------------------------------------------------
% End of proc-l-template.tex
%-----------------------------------------------------------------------
}