\documentclass{beamer}

\usepackage{verbatim}
\usepackage{fancyvrb}
\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{booktabs}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{calc}
\usepackage{color}
\usepackage{multicol}
\usepackage{wrapfig}
\usepackage{natbib}
\usepackage[ruled,vlined]{algorithm2e}
\usepackage{animate}
\usepackage{mathtools}
\usepackage{listings}

\usepackage{cmbright}
\fontencoding{OT1}\fontfamily{cmbr}\selectfont %to load ot1cmbr.fd
\DeclareFontShape{OT1}{cmbr}{bx}{n}{% change bx definition
<->cmbrbx10%
}{}
\normalfont % back to normalfont

% two col: two columns
\newenvironment{twocol}[4]{
\begin{columns}[c]
\column{#1\textwidth}
#3
\column{#2\textwidth}
#4
\end{columns}
}

\makeatletter
\setbeamertemplate{theorem begin}
{%
\begin{\inserttheoremblockenv}
  {}{\usebeamerfont*{block title}\usebeamercolor[fg]{block title}%
  \inserttheoremname
  %\inserttheoremnumber
  \ifx \inserttheoremaddition \empty \else\ (\inserttheoremaddition)\fi
  \inserttheorempunctuation}
  \normalfont
  }
  \setbeamertemplate{theorem end}{\end{\inserttheoremblockenv}}
\makeatother

\newcommand{\E}{\mathrm{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\sd}{\mathrm{sd}}
\newcommand{\se}{\mathrm{se}}
\newcommand{\Corr}{\mathrm{Corr}}
\newcommand{\rank}{\mathrm{rank}}
\newcommand{\trace}{\mathrm{trace}}
\newcommand{\nullspace}{\mathrm{null}}
\newcommand{\myspan}{\mathrm{span}}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\softmax}{softmax}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{exe}{Exercise}
\newtheorem{notation}{Notation}
\newtheorem{remark}{Remark}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\title{Matrix Approach to SLR}
\author{Zhenisbek Assylbekov}
\institute{Department of Mathematics}
\date{Regression Analysis}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\section{Random Vectors}

\begin{frame}{Random Vectors}
Let $\mathbf{Y}$ be a random vector in $\mathbb{R}^n$, i.e.
$$
\mathbf{Y}=\begin{bmatrix}
Y_1\\
\vdots\\
Y_n
\end{bmatrix}
$$
\pause The \textbf{pdf} of $\mathbf{Y}$ is the joint pdf of $Y_1,\ldots,Y_n$:
$$
f_{\mathbf{Y}}(\mathbf{y})=f_{Y_1,\ldots,Y_n}(y_1,\ldots,y_n)
$$
\pause It is a function s.t. for any (measurable) $B\subset\mathbb{R}^n$:
\begin{align*}
\Pr(\mathbf{Y}\in B)&=\int_{\mathbf{y}\in B}f_{Y_1,\ldots,Y_n}(y_1,\ldots,y_n)dy_1\cdots d y_n\\
\onslide<4->{&=\int_{\mathbf{y}\in B}f_\mathbf{Y}(\mathbf{y})d\mathbf{y}}
\end{align*}
\end{frame}

\begin{frame}{Expectation of a Random Vector}
The \textbf{expectation} of a random vector $\mathbf{Y}$ is
$$
\boldsymbol\mu=\E[\mathbf{Y}]=\begin{bmatrix}
\E[Y_1]\\
\vdots\\
\E[Y_n]
\end{bmatrix}
$$
\end{frame}

\begin{frame}{Random Vectors}
The \textbf{covariance matrix} of a random vector $\mathbf{Y}$ is an $n\times n$ matrix defined as
\begin{footnotesize}
\begin{align*}
\Var[\mathbf{Y}]&=\E[(\mathbf{Y}-\E[\mathbf{Y}])(\mathbf{Y}-\E[\mathbf{Y}])^\top]\\
\onslide<2->{&=\begin{bmatrix}
\Var[Y_1] & \Cov[Y_1,Y_2] & \cdots & \Cov[Y_1,Y_n] \\
\Cov[Y_2,Y_1] & \Var[Y_2] & \cdots & \Cov[Y_2,Y_n]\\
\vdots & \vdots & \ddots & \vdots \\
\Cov[Y_n,Y_1] & \Cov[Y_n,Y_2] & \cdots & \Var[Y_n]
\end{bmatrix}\\}
\onslide<3->{&=
\begin{bmatrix}
\sigma_1^2 & \sigma_{1,2} & \cdots & \sigma_{1,n}\\
\sigma_{2,1} & \sigma_{2}^2 & \cdots & \sigma_{2,n}\\
\vdots & \vdots & \ddots & \vdots \\
\sigma_{n,1} & \sigma_{n,2} & \cdots & \sigma_{n}^2
\end{bmatrix}\\}
\onslide<4->{&=\boldsymbol\Sigma\qquad\text{(symmetric)}}
\end{align*}
\end{footnotesize}
\end{frame}

\begin{frame}{SLR}
In the regression model, the only random term is $\boldsymbol\epsilon$ with
$$
\E[\boldsymbol\epsilon]=\pause\begin{bmatrix}
0\\
\vdots\\
0
\end{bmatrix}=\mathbf{0}\quad\text{and}\quad\Var[\boldsymbol\epsilon]\pause=\begin{bmatrix}
\sigma^2 & 0 & \cdots & 0\\
0 & \sigma^2 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & \sigma^2
\end{bmatrix}=\sigma^2\mathbf{I}_n
$$
\pause Hence for the model
$$
\mathbf{Y}=\mathbf{X}\boldsymbol\beta+\boldsymbol\epsilon
$$
\begin{itemize}
    \item\pause $\E[\mathbf{Y}]\pause=\E[\mathbf{X}\boldsymbol\beta]+\E[\boldsymbol\epsilon]=\mathbf{X}\boldsymbol\beta$
    \item\pause $\Var[\mathbf{Y}]\pause=\Var[\mathbf{X}\boldsymbol\beta+\boldsymbol\epsilon]=\Var[\boldsymbol\epsilon]=\sigma^2\mathbf{I}_n$
\end{itemize}
\end{frame}

\begin{frame}{Linear Transform of a Random Vector}
Let $\mathbf{A}\in\mathbb{R}^{m\times n}$ and $\mathbf{Y}$ be a random $n$-vector, then
\begin{align*}
\mathbf{AY}&=\onslide<2->{\begin{bmatrix}
a_{11} & a_{12} & \cdots & a_{1n}\\
a_{21} & a_{22} & \cdots & a_{2n}\\
\vdots & \vdots & \ddots & \vdots\\
a_{m1} & a_{m2} & \cdots & a_{mn}
\end{bmatrix}\begin{bmatrix}
Y_1\\
Y_2\\
\vdots\\
Y_n
\end{bmatrix}\\}
\onslide<3->{&=\begin{bmatrix}
a_{11}Y_1+a_{12}Y_2+\ldots+a_{1n}Y_n\\
a_{21}Y_1+a_{22}Y_2+\ldots+a_{2n}Y_n\\
\vdots\\
a_{m1}Y_1+a_{m2}Y_2+\ldots+a_{mn}Y_n
\end{bmatrix}}
\end{align*}
\end{frame}

\begin{frame}{Mean of a Linear Transform}
\begin{theorem}
Let $\mathbf{Y}$ be a random $n$-vector, and $\mathbf{A}\in\mathbb{R}^{m\times n}$. Then $\E[\mathbf{AY}]=\mathbf{A}\E[\mathbf{Y}]$.
\end{theorem}
\begin{proof}
\begin{footnotesize}
\begin{align*}
\E[\mathbf{AY}]&=\onslide<2->{\begin{bmatrix}
\E[a_{11}Y_1+a_{12}Y_2+\ldots+a_{1n}Y_n]\\
\vdots\\
\E[a_{m1}Y_1+a_{m2}Y_2+\ldots+a_{mn}Y_n]
\end{bmatrix}\\}
\onslide<3->{&=\begin{bmatrix}
a_{11}\E[Y_1]+a_{12}\E[Y_2]+\ldots+a_{1n}\E[Y_n]\\
\vdots\\
a_{m1}\E[Y_1]+a_{m2}\E[Y_2]+\ldots+a_{mn}\E[Y_n]
\end{bmatrix}\\}
\onslide<4->{&=\begin{bmatrix}
a_{11} & a_{12} & \cdots & a_{1n}\\
a_{21} & a_{22} & \cdots & a_{2n}\\
\vdots & \vdots & \ddots & \vdots\\
a_{m1} & a_{m2} & \cdots & a_{mn}
\end{bmatrix}\begin{bmatrix}
\E[Y_1]\\
\E[Y_2]\\
\vdots\\
\E[Y_n]
\end{bmatrix}\\}
\onslide<5->{&=\mathbf{A}\E[\mathbf{Y}]}
\end{align*}
\end{footnotesize}
\end{proof}
\end{frame}

\begin{frame}{Covariance Matrix of a Linear Transform}
\begin{theorem}
Let $\mathbf{Y}$ be a random $n$-vector, and $\mathbf{A}\in\mathbb{R}^{m\times n}$. Then $\Var[\mathbf{AY}]=\mathbf{A}\Var[\mathbf{Y}]\mathbf{A}^\top$.
\end{theorem}
\begin{proof}
\begin{align*}
\Var[\mathbf{AY}]&=\onslide<2->{\E[(\mathbf{AY}-\E[\mathbf{AY}])(\mathbf{AY}-\E[\mathbf{AY}])^\top]\\}
\onslide<3->{&=\E[(\mathbf{AY}-\mathbf{A}\E[\mathbf{Y}])(\mathbf{AY}-\mathbf{A}\E[\mathbf{Y}])^\top]\\}
\onslide<4->{&=\E[\mathbf{A}(\mathbf{Y}-\E[\mathbf{Y}])(\mathbf{Y}-\E[\mathbf{Y}])^\top\mathbf{A}^\top]\\}
\onslide<5->{&=\mathbf{A}\E[(\mathbf{Y}-\E[\mathbf{Y}])(\mathbf{Y}-\E[\mathbf{Y}])^\top]\mathbf{A}^\top\\}
\onslide<6->{&=\mathbf{A}\Var[\mathbf{Y}]\mathbf{A}^\top}
\end{align*}
\end{proof}
\end{frame}

\begin{frame}{Covariance Matrix is Positive Semi-definite}
\begin{theorem}
Let $\mathbf{Y}$ be a random vector in $\mathbb{R}^n$. Then $\Var[\mathbf{Y}]\succeq\mathbf{0}$.
\end{theorem}
\pause\begin{proof}
$\forall\mathbf{a}\in\mathbb{R}^n$:
$$
\mathbf{a}^\top\Var[\mathbf{Y}]\mathbf{a}\pause=\Var[\mathbf{a}^\top\mathbf{Y}]\pause\ge0,
$$
since $\mathbf{a}^\top\mathbf{Y}=\sum_i a_iY_i$ is a random variable with values in $\mathbb{R}^1$.
\end{proof}
\end{frame}

\begin{frame}{Pdf of a Linear Transform, $n=1$}
\begin{theorem}
Let $X$ be a continuous r.v. with the pdf $f_X(x)$, and $a\ne0$. Then the pdf of $Y=aX$ is given by
$$
f_{Y}(y)=\frac{1}{|a|}f_X\left(\frac{y}{a}\right)
$$
\end{theorem}
\begin{proof}
For $a>0$:
\begin{align*}
f_Y(y)&\onslide<2->{=\frac{d}{dy}[F_Y(y)]}\onslide<3->{=\frac{d}{dy}\left[\Pr(Y\le y)\right]}\onslide<4->{=\frac{d}{dy}\left[\Pr(aX\le y\right]\\}
\onslide<5->{&=\frac{d}{dy}\left[\Pr\left(X\le\frac{y}{a}\right)\right]}\onslide<6->{=\frac{d}{dy}\left[F_X\left(\frac{y}{a}\right)\right]}\onslide<7->{=\frac1a f_X\left(\frac{y}{a}\right)}
\end{align*}
\onslide<8->{For $a<0$ the proof is analogous and is left as \textit{exercise}.}
\end{proof}
\end{frame}

\begin{frame}{Pdf of a Linear Transform, General Case}
\begin{theorem}
Let $\mathbf{X}$ be a random vector with the pdf $f_{\mathbf{X}}(\mathbf{x})$, and $\mathbf{A}\in\mathbb{R}^{n\times n}$ with $\det\mathbf{A}\ne0$. Then the pdf of $\mathbf{Y}=\mathbf{AX}$ is given by
$$
f_{\mathbf{Y}}(\mathbf{y})=\frac{1}{|\det\mathbf{A}|}f_\mathbf{X}(\mathbf{A}^{-1}\mathbf{y})
$$
\end{theorem}
\pause\begin{proof} For any $B\subset \mathbb{R}^n$,
\begin{align*}
\onslide<3->{\int_{\mathbf{y}\in B}f_{\mathbf{Y}}(\mathbf{y})d\mathbf{y}=\Pr(\mathbf{Y}\in B)&}\onslide<4->{=\Pr(\mathbf{AX}\in B)}\onslide<5->{=\int_{\mathbf{Ax}\in B}f_\mathbf{X}(\mathbf{x})d\mathbf{x}\\}
\onslide<6->{&=\int_{\mathbf{y}\in B}f_{\mathbf{X}}(\mathbf{A}^{-1}\mathbf{y})|\det\mathbf{A}^{-1}|d\mathbf{y}}
\end{align*}
\onslide<7->{Noting that $\det\mathbf{A}^{-1}=\frac{1}{\det\mathbf{A}}$ concludes the proof.}
\end{proof}
\end{frame}

\begin{frame}{Pdf of an Affine Transform}
\begin{theorem}
Let $X$ be a continuous r.v. with the pdf $f_X(x)$, and $a\ne0$. Then the pdf of $Y=aX+b$ is given by
$$
f_{Y}(y)=\frac{1}{|a|}f_X\left(\frac{y-b}{a}\right)
$$
\end{theorem}

\pause\begin{theorem}
Let $\mathbf{X}$ be a random vector with the pdf $f_{\mathbf{X}}(\mathbf{x})$, $\mathbf{A}\in\mathbb{R}^{n\times n}$ with $\det\mathbf{A}\ne0$, and $\mathbf{b}\in\mathbb{R}^b$. Then the pdf of $\mathbf{Y}=\mathbf{AX}+\mathbf{b}$ is given by
$$
f_{\mathbf{Y}}(\mathbf{y})=\frac{1}{|\det\mathbf{A}|}f_\mathbf{X}(\mathbf{A}^{-1}(\mathbf{y}-\mathbf{b}))
$$
\end{theorem}

\pause \begin{proof}[Proofs are left as exercises]\end{proof}
\end{frame}

\section{Multivariate Gaussian Distribution}

\begin{frame}{Multivariate Normal Distribution}
\begin{definition}
A random vector $\mathbf{Y}=\begin{bmatrix}Y_1 & \ldots & Y_n\end{bmatrix}^\top$ is \textbf{multivariate normal (Gaussian)} if
it can be represented as
$$
\mathbf{Y}=\mathbf{AZ}+\boldsymbol{\mu},
$$
where
\begin{itemize}
    \item $\mathbf{Z}=\begin{bmatrix}Z_1 & \ldots & Z_k\end{bmatrix}^\top$ with $Z_i\,\,\stackrel{\text{iid}}{\sim}\,\,\mathcal{N}(0, 1)$,
    \item $\mathbf{A}\in\mathbb{R}^{n\times k}$ s.t. $\boldsymbol\Sigma=\mathbf{AA}^\top$%, where $\boldsymbol{\Sigma}=\E[(\mathbf{y}-\boldsymbol{\mu})(\mathbf{y}-\boldsymbol{\mu})^\top]$ is the {covariance matrix} of $\mathbf{y}$.
\end{itemize}
\end{definition}
\pause
\begin{columns}
\begin{column}{.5\textwidth}
\begin{notation}$\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu, \boldsymbol\Sigma)$
\end{notation}
\end{column}
\begin{column}{.5\textwidth}
\pause\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{Multivariate_Gaussian.png}
\end{figure}
\end{column}
\end{columns}
\end{frame}

\begin{frame}{Multivariate Normal Distribution}
\begin{theorem}
$\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu, \boldsymbol\Sigma)$, and $\rank\boldsymbol\Sigma=n$\quad$\Rightarrow$\quad The pdf of $\mathbf{Y}$ has the form
$$
f_{\mathbf{Y}}(\mathbf{y})=\frac{1}{\sqrt{\det\boldsymbol{\Sigma}}(\sqrt{2\pi})^n}e^{-\frac12(\mathbf{y}-\boldsymbol\mu)^\top\boldsymbol{\Sigma}^{-1}(\mathbf{y}-\boldsymbol{\mu})}.
$$
\end{theorem}
\begin{block}{Proof.}
$\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\boldsymbol\Sigma)$\pause\quad$\Rightarrow$\quad$\exists\mathbf{A}\in\mathbb{R}^{n\times k},\,\mathbf{Z}\in\mathcal{N}_k(\mathbf{0},\mathbf{I}_k)$:
\pause$$
\mathbf{Y}=\mathbf{AZ}+\boldsymbol\mu\quad\text{with }\mathbf{AA}^\top=\boldsymbol\Sigma
$$
Since $n=\rank\boldsymbol\Sigma\pause=\rank\mathbf{AA}^\top\pause=k$, we have $k=n$.\\~\\
Also, $\det\boldsymbol\Sigma=\det\mathbf{AA}^\top=\det\mathbf{A}\cdot\det\mathbf{A}^\top=(\det\mathbf{A})^2$. Thus,
$$\det\mathbf{A}=\sqrt{\det\boldsymbol\Sigma}.$$
\end{block}
\end{frame}

\begin{frame}{Multivariate Normal Distribution}
\begin{block}{Poof (cont'd).}
For the $\mathbf{Z}=\begin{bmatrix}
Z_1 & \cdots & Z_n
\end{bmatrix}$ with $Z_i\,\,{\stackrel{\text{iid}}{\sim}}\,\,\in\mathcal{N}(0,1)$,
\begin{align*}
f_{\mathbf{Z}}(\mathbf{z})\onslide<2->{&=f_{Z_1,\ldots,Z_n}(z_1,\ldots,z_n)}\onslide<3->{=\prod_{i=1}^n f_{Z_i}(z_i)}\onslide<4->{=\prod_{i=1}^n\frac{1}{\sqrt{2\pi}}e^{-\frac{z_i^2}{2}}\\}
\onslide<5->{&=\frac{1}{(\sqrt{2\pi})^n}e^{-\frac12(z_1^2+\ldots+z_n^2)}}\onslide<6->{=\frac{1}{(\sqrt{2\pi})^n}e^{-\frac12\mathbf{z}^\top\mathbf{z}}}
\end{align*}
\onslide<7->{Since $\mathbf{Y}=\mathbf{AZ}+\boldsymbol\mu$, and $\boldsymbol\Sigma=\mathbf{AA}^\top$, we have}
\begin{align*}
\onslide<8->{f_\mathbf{Y}(\mathbf{y})&=\frac{1}{\det\mathbf{A}}f_{\mathbf{Z}}(\mathbf{A}^{-1}(\mathbf{y}-\boldsymbol\mu))}\onslide<9->{=\frac{1}{\sqrt{\det\boldsymbol\Sigma}(\sqrt{2\pi})^n}e^{-\frac12(\mathbf{y}-\boldsymbol\mu)^\top(\mathbf{A}^{-1})^\top\mathbf{A}^{-1}(\mathbf{y}-\boldsymbol\mu)}\\}
\onslide<10->{&=\frac{1}{\sqrt{\det\boldsymbol\Sigma}(\sqrt{2\pi})^n}e^{-\frac12(\mathbf{y}-\boldsymbol\mu)^\top(\mathbf{A}^{\top})^{-1}\mathbf{A}^{-1}(\mathbf{y}-\boldsymbol\mu)}\\}
\onslide<11->{&=\frac{1}{\sqrt{\det\boldsymbol\Sigma|}(\sqrt{2\pi})^n}e^{-\frac12(\mathbf{y}-\boldsymbol\mu)^\top\boldsymbol\Sigma^{-1}(\mathbf{y}-\boldsymbol\mu)}}
\end{align*}
\end{block}
\end{frame}

\begin{frame}{Orthogonal Transform of a Multivariate Gaussian}
\begin{theorem}
Let $\mathbf{Z}\sim\mathcal{N}_n(\mathbf{0},\mathbf{I})$, and $\mathbf{Q}$ be an orthogonal $n\times n$ matrix ($\mathbf{Q}\in\mathcal{O}_n$). Then $\mathbf{QZ}\sim\mathcal{N}_n(\mathbf{0},\mathbf{I})$.
\end{theorem}
\begin{proof}
\onslide<2->{The pdf of $\mathbf{QZ}$ is}
\begin{align*}
\onslide<3->{f_{\mathbf{QZ}}(\mathbf{w})}\onslide<4->{&=\frac{1}{|\det\mathbf{Q}|}f_{\mathbf{Z}}(\mathbf{Q}^{-1}\mathbf{w})}\onslide<5->{=\frac{1}{1}\cdot\frac{1}{(\sqrt{2\pi})^n}e^{-\frac12(\mathbf{Q}^{-1}\mathbf{w})^\top(\mathbf{Q}^{-1}\mathbf{w})}\\}
\onslide<6->{&=\frac{1}{(\sqrt{2\pi})^n}e^{-\frac12(\mathbf{Q}^\top\mathbf{w})^\top(\mathbf{Q}^\top\mathbf{w})}}\onslide<7->{=\frac{1}{(\sqrt{2\pi})^n}e^{-\frac12(\mathbf{w}^\top\mathbf{Q})(\mathbf{Q}^\top\mathbf{w})}\\}
\onslide<8->{&=\frac{1}{(\sqrt{2\pi})^n}e^{-\frac12\mathbf{w}^\top(\mathbf{Q}\mathbf{Q}^\top)\mathbf{w}}}\onslide<9->{=\frac{1}{(\sqrt{2\pi})^n}e^{-\frac12\mathbf{w}^\top\mathbf{w}}}
\end{align*}
\onslide<10->{Hence, $\mathbf{QZ}\sim\mathcal{N}_n(\mathbf{0},\mathbf{I})$.}
\end{proof}
\end{frame}

\begin{frame}{Linear Transform of a Multivariate Gaussian}
\begin{theorem}
Let $\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\boldsymbol\Sigma)$. If $\mathbf{A}\in\mathbb{R}^{m\times n}$ with $\rank\mathbf{A}={\min}(m,n)$, then
$$
\mathbf{AY}\sim\mathcal{N}_m(\mathbf{A}\boldsymbol\mu,\mathbf{A}\boldsymbol\Sigma\mathbf{A}^\top)
$$
\end{theorem}
\begin{proof}
Since $\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\boldsymbol\Sigma)$, we can write
$$
\mathbf{Y}=\mathbf{BZ}+\boldsymbol\mu,
$$
\pause where $\mathbf{B}\in\mathbb{R}^{n\times k}$, $\mathbf{Z}\sim\mathcal{N}_{k}(\mathbf{0},\mathbf{I})$, and $\mathbf{BB}^\top=\boldsymbol\Sigma$. \pause Thus,
$$\mathbf{AY}=(\mathbf{AB})\mathbf{Z}+\mathbf{A}\boldsymbol\mu,$$
\pause where $\mathbf{AB}\in\mathbb{R}^{m\times k}$ s.t. $(\mathbf{AB})(\mathbf{AB})^\top=\mathbf{ABB}^\top\mathbf{A}=\mathbf{A}\boldsymbol\Sigma\mathbf{A}^\top$.
\end{proof}
\end{frame}

\iffalse
\begin{frame}{Multivariate Gaussian}{Uncorrelated Components are Independent}
\begin{theorem}
Let $\mathbf{y}\sim\mathcal{N}(\boldsymbol\mu,\boldsymbol\Sigma)$. Then $Y_i$ and $Y_j$ are independent\quad $\Leftrightarrow$ \quad $\Cov[Y_i,Y_j]=0$.
\end{theorem}
\begin{proof}
\vspace{150pt}
\end{proof}
\end{frame}
\fi

\section{Quadratic Forms of Multivariate Gaussians}

\begin{frame}{Non-central $\chi^2$ distribution}
\begin{definition}
Let $Y_i\,\,{\stackrel{\text{ind}}{\sim}}\,\,\mathcal{N}(\mu_i,1)$, \pause then the random variable
$$
\sum_{i=1}^n Y_i^2
$$
is said to have \textbf{noncentral chi-square} distribution with parameters $n$ and $\lambda=\sum_{i=1}^n\mu_i^2$.
\end{definition}
\vspace{-10pt}
\pause \begin{notation}
$\chi^2_{n}(\lambda)$
\end{notation}
\begin{itemize}
    \item\pause $n$ is called \textit{degrees of freedom}
    \item\pause $\lambda$ is called \textit{noncentrality parameter}
\end{itemize}
\vspace{-10pt}
\pause\begin{remark} Using matrix notation, if $\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\mathbf{I})$,\pause then $\mathbf{Y}^\top\mathbf{Y}\sim\chi^2_n(\lambda)$, with $\lambda=\boldsymbol\mu^\top\boldsymbol\mu$.
\end{remark}
\end{frame}

\begin{frame}{Quadratic Form of a Multivariate Gaussian}
\begin{theorem}
Let $\mathbf{Z}\sim\mathcal{N}_n(\mathbf{0},\mathbf{I})$, and $\mathbf{A}\in\mathbb{R}^{n\times n}$ be symmetric idempotent with $\rank\mathbf{A}=r$. Then
$\mathbf{Z}^\top\mathbf{A}\mathbf{Z}\sim\chi^2_r$.
\end{theorem}
\begin{proof}\begin{footnotesize}
\pause$\mathbf{A}^2=\mathbf{AA}=\mathbf{A}$. \pause Let $\mathbf{x}$ be an eigenvector of $\mathbf{A}$ and $\lambda$ is the corresponding eigenvalue. Then
$$
\mathbf{A}^2\mathbf{x}=\mathbf{AAx}\pause=\mathbf{A}\lambda\mathbf{x}=\lambda\mathbf{Ax}\pause=\lambda^2\mathbf{x}\pause=\lambda\mathbf{x}=\mathbf{Ax}
$$
\pause So, $(\lambda^2-\lambda)\mathbf{x}=\mathbf{0}\pause\quad\Rightarrow\quad\lambda^2-\lambda=0\quad\Rightarrow\quad\lambda\in\{0,1\}$.\\~\\

\pause Eigendecomposition of $\mathbf{A}$: \pause $\mathbf{A}=\mathbf{Q}^\top\boldsymbol\Lambda\mathbf{Q}$, \pause where $\boldsymbol\Lambda=\mathrm{diag}(\underbrace{1,\ldots,1}_{r},\underbrace{0,\ldots,0}_{n-r})$.\\~\\

Thus, $\mathbf{Z}^\top\mathbf{AZ}\pause=\mathbf{Z}^\top\mathbf{Q}^\top\boldsymbol\Lambda\mathbf{Q}\mathbf{Z}\pause=(\mathbf{QZ})^\top\boldsymbol\Lambda(\mathbf{QZ})\pause=\tilde{\mathbf{Z}}^\top\boldsymbol\Lambda\tilde{\mathbf{Z}}\pause=\sum_{i=1}^r\tilde{Z}_i^2\sim\chi^2_r$, \pause where $\tilde{\mathbf{Z}}=(\tilde{Z}_1,\ldots,\tilde{Z}_n)\sim\mathcal{N}_n(\mathbf{0},\mathbf{I})$.
\end{footnotesize}
\end{proof}
\end{frame}

\begin{frame}{Quadratic Form of a Multivariate Gaussian}
\begin{theorem}
Let $\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\mathbf{I})$, and $\mathbf{A}\in\mathbb{R}^{n\times n}$ be symmetric idempotent with $\rank\mathbf{A}=r$. Then
$\mathbf{Y}^\top\mathbf{A}\mathbf{Y}\sim\chi^2_r(\lambda)$, with $\lambda=\boldsymbol\mu^\top\mathbf{A}\boldsymbol\mu$
\end{theorem}
\begin{proof}
\pause $\mathbf{A}=\mathbf{Q}^\top\boldsymbol\Lambda\mathbf{Q}$, where $\boldsymbol\Lambda=\mathrm{diag}(1,\ldots,1,0,\ldots,0)$. \pause Thus,
$$
\mathbf{Y}^\top\mathbf{AY}=(\mathbf{QY})^\top\boldsymbol\Lambda(\mathbf{QY})
$$
\pause Notice that $\tilde{\mathbf{Y}}:=\mathbf{QY}\sim\mathcal{N}_n(\mathbf{Q}\boldsymbol\mu,\mathbf{I})$. (Why?) \pause Therefore,
$$
\mathbf{Y}^\top\mathbf{AY}=\tilde{\mathbf{Y}}^\top\boldsymbol\Lambda\tilde{\mathbf{Y}}=\sum_{i=1}^r\tilde{Y}_i^2\sim\chi^2_r(\lambda),
$$
\pause where $\lambda=(\boldsymbol\Lambda\mathbf{Q}\boldsymbol\mu)^\top(\boldsymbol\Lambda\mathbf{Q}\boldsymbol\mu)=\boldsymbol\mu^\top\mathbf{Q}^\top\boldsymbol\Lambda^\top\boldsymbol\Lambda\mathbf{Q}\boldsymbol\mu=\boldsymbol\mu^\top\mathbf{Q}^\top\boldsymbol\Lambda\mathbf{Q}\boldsymbol\mu$\\
$=\boldsymbol\mu^\top\mathbf{A}\boldsymbol\mu$.
\end{proof}
\end{frame}

\iffalse
\begin{frame}{Quadratic Form of a Multivariate Gaussian}
\begin{theorem}
Let $\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\boldsymbol\Sigma)$, $\boldsymbol\Sigma\succ\mathbf{0}$, $\mathbf{A}\boldsymbol\Sigma$ be idempotent matrix, and  $\rank(\mathbf{A})=r$. Then $\mathbf{Y}^\top\mathbf{AY}\sim\chi^2_r(\lambda)$ with $\lambda=\boldsymbol\mu^\top\mathbf{A}\boldsymbol\mu$.
\end{theorem}
\begin{proof}\begin{footnotesize}
\pause $\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\boldsymbol{\Sigma})\quad\Rightarrow\quad\tilde{\mathbf{Y}}:={\boldsymbol\Sigma}^{-1/2}\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\Sigma^{-1/2}\boldsymbol\mu,\mathbf{I})$\qquad (Why?).\pause\quad Also,
\begin{equation}
\mathbf{Y}^\top\mathbf{AY}=\mathbf{Y}^\top\boldsymbol\Sigma^{-1/2}\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma^{1/2}\boldsymbol\Sigma^{-1/2}\mathbf{Y}=\tilde{\mathbf{Y}}^\top(\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma^{1/2})\tilde{\mathbf{Y}}\label{eq:decor}
\end{equation}
\pause We are given that $\mathbf{A}\boldsymbol\Sigma\mathbf{A}\boldsymbol\Sigma=\mathbf{A}\boldsymbol\Sigma$, thus $\mathbf{A}\boldsymbol\Sigma\mathbf{A}=\mathbf{A}$, and
\pause $$(\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma^{1/2})(\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma^{1/2})\pause=\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma\mathbf{A}\boldsymbol\Sigma^{1/2}\pause=\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma^{1/2}$$%
\pause Thus, $\tilde{\mathbf{A}}:=\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma^{1/2}$ is idempotent, and its rank is $r$ (check this). \pause Combining this with \eqref{eq:decor}, we have
$$
\mathbf{Y}^\top\mathbf{AY}\pause=\tilde{\mathbf{Y}}^\top\tilde{\mathbf{A}}\tilde{\mathbf{Y}}\sim\chi^2_r(\lambda)
$$
\pause with $\lambda=(\boldsymbol\Sigma^{-1/2}\boldsymbol\mu)^\top(\boldsymbol\Sigma^{1/2}\mathbf{A}\boldsymbol\Sigma^{1/2})(\boldsymbol\Sigma^{-1/2}\boldsymbol\mu)=\boldsymbol\mu^\top\mathbf{A}\boldsymbol\mu$.
\end{footnotesize}
\end{proof}
\end{frame}
\fi

\begin{frame}{Matrix Calculus}
It is useful to have differentiation rules for matrix and vector expressions. \onslide<2->{The most important for our purposes are
\begin{align*}
&\boldsymbol{\nabla}_\mathbf{x}(\mathbf{a}^\top\mathbf{x})=\mathbf{a}\\
&\boldsymbol{\nabla}_\mathbf{x}(\mathbf{x}^\top\mathbf{Ax})=(\mathbf{A}+\mathbf{A}^\top)\mathbf{x}
\end{align*}
}%
\onslide<3->{Note that the second rule is defined only if $\mathbf{A}$ is square. Furthermore, if $\mathbf{A}$ is symmetric, we can simplify the result to $2\mathbf{Ax}$.}
\onslide<4->{\begin{exe}
Prove the rules above.
\end{exe}}
\end{frame}

\begin{frame}{Back to SLR}
Recall the SLR in matrix form:
$$
\mathbf{Y}\sim\mathcal{N}_n(\mathbf{X}\boldsymbol\beta,\sigma^2\mathbf{I})
$$
\pause Least squares objective is
\begin{align*}
\onslide<2->{Q(\beta_0,\beta_1)&=\sum_{i=1}^n[Y_i-(\beta_0+\beta_1 x_i)]^2}\onslide<3->{=(\mathbf{Y}-\mathbf{X}\boldsymbol\beta)^\top(\mathbf{Y}-\mathbf{X}\boldsymbol\beta)\\}
\onslide<4->{&=\mathbf{Y}^\top\mathbf{Y}-\boldsymbol\beta^\top\mathbf{X}^\top\mathbf{Y}-\mathbf{Y}^\top\mathbf{X}\boldsymbol\beta+\boldsymbol\beta^\top\mathbf{X}^\top\mathbf{X}\boldsymbol\beta\\}
\onslide<5->{\nabla_{\boldsymbol\beta}Q&=\mathbf{X}^\top\mathbf{Y}-\mathbf{X}^\top\mathbf{Y}+2\mathbf{X}^\top\mathbf{X}\boldsymbol\beta}\onslide<6->{=2\mathbf{X}^\top\mathbf{X}\boldsymbol\beta-2\mathbf{X}^\top\mathbf{Y}}\onslide<7->{=\mathbf{0}\\}
\onslide<8->{&\mathbf{X}^\top\mathbf{X}\boldsymbol\beta=\mathbf{X}^\top\mathbf{Y}\quad\Rightarrow\quad}\onslide<9->{\boxed{\hat{\boldsymbol\beta}=(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{Y}}}
\end{align*}
\onslide<10->{where we used  $\mathbf{X}^\top\mathbf{X}=\begin{bmatrix}
n & \sum_i x_i\\
\sum_i x_i & \sum_i x_i^2
\end{bmatrix}$ is non-singular (check).}\\~\\
\onslide<11->{The predictions are} $\onslide<11->{(\hat{Y}_1,\ldots,\hat{Y}_n)=\hat{\mathbf{Y}}=\mathbf{X}\hat{\boldsymbol\beta}}\onslide<12->{=\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{Y}}$
\end{frame}

\section{Distributions of SSE, SSR; Independence}

\begin{frame}{Distribution of SSE}
\begin{theorem}
$\frac{\text{SSE}}{\sigma^2}\sim\chi^2_{n-2}$
\end{theorem}
\begin{block}{Proof.}\begin{small}
\vspace{-20pt}
\begin{align*}
\onslide<2->{\text{SSE}&=\sum_{i=1}^n(Y_i-\hat{Y}_i)^2}\onslide<3->{=(\mathbf{Y}-\hat{\mathbf{Y}})^\top(\mathbf{Y}-\hat{\mathbf{Y}})\\}
\onslide<4->{&=[\mathbf{Y}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{Y}]^\top[\mathbf{Y}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{Y}]\\}
\onslide<5->{&=\mathbf{Y}^\top[\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]^\top[\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]\mathbf{Y}}
\end{align*}
\onslide<6->{The matrix $\mathbf{A}:=\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top$ is symmetric (check) and idempotent:}
\begin{align*}
\onslide<7->{\mathbf{A}^2&=[\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top][\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]\\}
\onslide<8->{&=\mathbf{I}-2\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top+\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\\}
\onslide<9->{&=\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top=\mathbf{A}}
\end{align*}
\onslide<10->{Hence, $\text{SSE}=\mathbf{Y}^\top\mathbf{A}^\top\mathbf{AY}=\mathbf{Y}^\top\mathbf{A}\mathbf{Y}$.}
\end{small}\end{block}
\end{frame}

\begin{frame}{Distribution of SSE}
\begin{proof}[Proof (cont'd).]
Recall, $\mathbf{Y}\sim\mathcal{N}_n(\mathbf{X}\boldsymbol\beta,\sigma^2\mathbf{I})$\pause\quad$\Rightarrow$\quad$\sigma^{-1}\mathbf{Y}\sim\mathcal{N}_n(\sigma^{-1}\mathbf{X}\boldsymbol\beta,\mathbf{I})$. \pause Hence,
\begin{align*}
\onslide<3->{\frac{\text{SSE}}{\sigma^2}&=(\sigma^{-1}\mathbf{Y})^\top\mathbf{A}(\sigma^{-1}\mathbf{Y})\pause}\onslide<4->{\sim\chi^2_{\rank\mathbf{A}}(\lambda),\\}
\onslide<5->{\text{where\quad }\rank\mathbf{A}&=\trace\mathbf{A}}\onslide<6->{=\trace[\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]\\}
\onslide<7->{&=\trace\mathbf{I}-\trace[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]\\}
\onslide<8->{&=n-\trace[(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{X}]}\onslide<9->{=n-2,\\}
\onslide<10->{\text{and\quad }\lambda&=(\sigma^{-1}\mathbf{X}\boldsymbol\beta)^\top\mathbf{A}(\sigma^{-1}\mathbf{X}\boldsymbol\beta)\\}
\onslide<11->{&=\frac1{\sigma^2}\boldsymbol\beta^\top\mathbf{X}^\top[\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]\mathbf{X}\boldsymbol\beta\\}
\onslide<12->{&=\frac{1}{\sigma^2}\boldsymbol\beta^\top[\mathbf{X}^\top\mathbf{X}-\mathbf{X}^\top\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{X}]\boldsymbol\beta}\onslide<13->{=0}
\end{align*}
\onslide<14->{Finally, $\frac{\text{SSE}}{\sigma^2}\sim\chi^2_{n-2}$.}
\end{proof}
\end{frame}

\begin{frame}{Distribution of SSR}
\begin{theorem}
$\frac{\text{SSR}}{\sigma^2}\sim\chi^2_1$ (non-central)
\end{theorem}
\begin{block}{Proof.}\begin{small}
\onslide<2->{Let $\mathbf{1}:=(1,\ldots,1)$, then $\mathbf{1}^\top\mathbf{1}=n$}\onslide<3->{, and $\bar{Y}=\frac{\mathbf{1}^\top\mathbf{Y}}{\mathbf{1}^\top\mathbf{1}}$.\\~\\}
\onslide<4->{Let $\bar{\mathbf{Y}}:=(\bar{Y},\ldots,\bar{Y})$. Then $\bar{\mathbf{Y}}=\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top\mathbf{Y}$}
\begin{align*}
\onslide<5->{\text{SSR}&=\sum_{i=1}^n(\hat{Y}_i-\bar{Y})^2}\onslide<6->{=(\hat{\mathbf{Y}}-\bar{\mathbf{Y}})^\top(\hat{\mathbf{Y}}-\bar{\mathbf{Y}})\\}
\onslide<7->{&=[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{Y}-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top\mathbf{Y}]^\top[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{Y}-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top\mathbf{Y}]\\}
\onslide<8->{&=\mathbf{Y}^\top[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top]^\top[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top]\mathbf{Y}}
\end{align*}
\onslide<9->{The matrix $\mathbf{B}:=\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top$ is symmetric and idempotent (check).}
\onslide<10->{Hence, $\text{SSR}=\mathbf{Y}^\top\mathbf{B}^\top\mathbf{BY}=\mathbf{Y}^\top\mathbf{B}\mathbf{Y}$.}
\end{small}\end{block}
\end{frame}

\begin{frame}{Distribution of SSR}
\begin{proof}[Proof (cont'd).]
Recall, $\mathbf{Y}\sim\mathcal{N}_n(\mathbf{X}\boldsymbol\beta,\sigma^2\mathbf{I})$\pause\quad$\Rightarrow$\quad$\sigma^{-1}\mathbf{Y}\sim\mathcal{N}_n(\sigma^{-1}\mathbf{X}\boldsymbol\beta,\mathbf{I})$. \pause Hence,
\begin{align*}
\onslide<3->{\frac{\text{SSR}}{\sigma^2}&=(\sigma^{-1}\mathbf{Y})^\top\mathbf{B}(\sigma^{-1}\mathbf{Y})\pause}\onslide<4->{\sim\chi^2_{\rank\mathbf{B}}(\lambda),\\}
\onslide<5->{\text{where\quad }\rank\mathbf{B}&=\trace\mathbf{B}}\onslide<6->{=\trace[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top]\\}
\onslide<7->{&=\trace[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]-\trace[\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top]\\}
\onslide<8->{&=\trace[(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top\mathbf{X}]-\trace[(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top\mathbf{1}]\\}
\onslide<9->{&=\trace\mathbf{I}_2-\trace1=2-1=1\\}
\end{align*}
\onslide<10->{Finally, $\frac{\text{SSR}}{\sigma^2}\sim\chi^2_{1}$ (non-central).}
\end{proof}
\end{frame}

\begin{frame}{Independence between SSR and SSE}
\begin{theorem}
Let $\mathbf{Y}\sim\mathcal{N}_n(\boldsymbol\mu,\boldsymbol\Sigma)$, and $\boldsymbol\Sigma\succ\mathbf{0}$. Then two quadratic forms $\mathbf{Y}^\top\mathbf{A}\mathbf{Y}$ and $\mathbf{Y}^\top\mathbf{BY}$ are independent \quad$\Leftrightarrow$\quad $\mathbf{A}\boldsymbol\Sigma\mathbf{B}=\mathbf{0}$.
\end{theorem}

\pause\begin{theorem}
SSR and SSE are independent.
\end{theorem}
\begin{proof}
\pause $\text{SSE}=\mathbf{Y}^\top\mathbf{A}\mathbf{Y}$ with $\mathbf{A}=\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top$\\
\pause $\text{SSR}=\mathbf{Y}^\top\mathbf{B}\mathbf{Y}$ with $\mathbf{B}=\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top$\\
\pause $\mathbf{Y}\sim\mathcal{N}_n(\mathbf{X}\boldsymbol\beta,\sigma^2\mathbf{I})$. \pause And now let's check
\begin{align*}
\onslide<6->{&\mathbf{A}\sigma^2\mathbf{IB}=\sigma^2\mathbf{AB}\\}
\onslide<7->{&=\sigma^2[\mathbf{I}-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top][\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top]\\}
\onslide<8->{&=\sigma^2[\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top-\mathbf{1}(\mathbf{1}^\top\mathbf{1})^{-1}\mathbf{1}^\top\mathbf{X}(\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top]\\}
\onslide<9->{&=\mathbf{0}}
\end{align*}
\onslide<10->{Hence, SSE and SSR are independent.}
\end{proof}

\end{frame}

\end{document}

Slides/05 Matrix Approach to SLR/main.tex


\subsection{Transient analysis of bimaterial plate with a notch}

\paragraph{}
In this last example, we study the transient response of a bimaterial plate with a notch terminating at the material interface.
The crack is assumed to be in material 1 and terminating at the interface between the two materials (see Fig.~\ref{iso_fig:bimaterial_plate_geo_bc}).
The length of the crack is assumed to be b/2 and the width of the notch angle is $\theta = 20^\circ$ .
The inputs and the results are non-dimensional.
The plate consists of two materials with different Young's modulus and the same mass density and Poisson's ratio $\nu = 1/3$.
A state of plane stress condition is assumed and the plate is fixed at the bottom.
    \begin{figure}[h!]
        \begin{subfigure}[b]{0.5\linewidth}
            \centering
            \scalebox{0.5}{
                \includegraphics{isogeometric_sbfem/images/bimaterial_plate_geo_bc.eps}
            }
        \end{subfigure}
        \begin{subfigure}[b]{0.5\linewidth}
            \centering
            \scalebox{0.5}{
                \includegraphics{isogeometric_sbfem/images/bimaterial_plate_load_history.png}
            }
        \end{subfigure}
        \caption{Bimaterial plate with a notch: geometry, boundary conditions and force history}
        \label{iso_fig:bimaterial_plate_geo_bc}
    \end{figure}

\paragraph{}
The main advantage of the proposed method when applied to this example is that, no special treatment is required to represent the weak material discontinuity or the strong discontinuity due to the notch.
The scaling center is placed at the point $O$, where the notch intersects the material interface.
The SBFEM does not require the faces of the notch to be discretized and only the boundary of the domain is discretized.
For this study, the vertical boundaries are discretized using four quartic NURBS functions and the horizontal boundaries are discretized using two quartic
NURBS functions.
Fig.~\ref{iso_fig:bimaterial_plate_natural_frequencies} shows the non-dimensional natural frequencies computed using the proposed method for different orders of continued fractions.
The reference solution is computed by using the commercial software ANSYS$^\circledR$ 14.0.
It is seen that the results from the present approach are in good agreement with the results from the FE software and for the transient analysis, the order of continued fractions is chosen as $M_{cf} = 6$.
    \begin{figure}[h!]
        \centering
        \scalebox{0.4}{
            \includegraphics{isogeometric_sbfem/images/bimaterial_plate_natural_frequencies.png}
        }
        \caption{Non-dimensional natural frequencies of a bimaterial plate with a notch}
        \label{iso_fig:bimaterial_plate_natural_frequencies}
    \end{figure}

\paragraph{}
The transient response of the bimaterial plate with a notch is evaluated considering a uniformly distributed load acting at the top horizontal boundary (see Fig.~\ref{iso_fig:bimaterial_plate_geo_bc}).
A triangular impulse load with an amplitude $\widetilde{p}$ with a duration of $2 \times \frac{b}{c_s}$ and peak value $\widetilde{p}$ at time $\widetilde{t}=1\frac{c_s}{b}$ is applied.
The time integration is carried out by employing Newmark's method with $\gamma = 1/2$ and $\beta = 1/4$.
The time step is chosen as $\Delta t = 0.05\frac{b}{c_s}$ and a total of 400 time steps are considered.
The vertical displacement responses at points $A$, $B$ and $C$ are plotted in Fig.~\ref{iso_fig:bimaterial_plate_uy} as functions of the dimensionless time $\widetilde{t} = c_s\frac{t}{b}$
The corresponding results of a finite element analysis are also shown for comparison and an excellent agreement is observed.
The main advantage of the proposed method is that internal discontinuities or material discontinuity does not require special treatment as in other approaches such as the XFEM or the conventional IGA.
Moreover, creating a scaled boundary mesh with NURBS or B-spline functions is straightforward as only the boundary information is required when compared to generating finite element mesh around the notch.
    \begin{figure}[htb]
        \begin{subfigure}[b]{1\linewidth}
            \centering
            \scalebox{0.5}{
                \includegraphics{isogeometric_sbfem/images/bimaterial_plate_uva.png}
            }
            \caption{Point A}
        \end{subfigure}

        \begin{subfigure}[b]{1\linewidth}
            \centering
            \scalebox{0.5}{
                \includegraphics{isogeometric_sbfem/images/bimaterial_plate_uvb.png}
            }
            \caption{Point B}
        \end{subfigure}
    \end{figure}
    \begin{figure}[htb]\ContinuedFloat
        \begin{subfigure}[b]{1\linewidth}
            \centering
            \scalebox{0.5}{
                \includegraphics{isogeometric_sbfem/images/bimaterial_plate_uvc.png}
            }
            \caption{Point C}
        \end{subfigure}
    \caption[Vertical displacements of bimaterial plate with a notch]{Vertical displacements of bimaterial plate with a notch at (a) point A; (b) point B; (c): point C}
    \label{iso_fig:bimaterial_plate_uy}
    \end{figure}


\documentclass[a4paper,11pt]{scrartcl}
\usepackage{fullpage}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{algorithmic}
\usepackage{algorithm}
\usepackage{listings}
\usepackage{graphicx}
\usepackage{cite}
%\usepackage{showlabels}

\usepackage[usenames,dvipsnames]{pstricks}
\usepackage{epsfig}
\usepackage{pst-grad} % For gradients
\usepackage{pst-plot} % For axes

\title{Seminararbeit Numerik}
\subtitle{Iterative Solvers - Algebraic Multi-grid}
\author{Bernd Schwarzenbacher, Daniel Herold}

\begin{document}
\maketitle
\tableofcontents
\pagebreak

\section{Motivation} \label{section:motiv}
\subsection{Iterative Solvers} \label{section:iter}
Iterative methods are used to solve big systems of linear equations in form of
matrices.
In practice the conjugate gradient (CG) method is the most often used iterative method for
the (often sparse and positive-definite) matrices arising from finite-element
discretization of partial differential equations.
This method is based on substituting the original problem by the fixed-point
equation:
$$b-Ax = 0 \iff x + \tau C^{-1} (b-Ax) = x$$
where $C$ is an invertible matrix.

From this formulation, one can immediately derive the
{\em Richardson Iteration}\/, which is a simple iterative method and is
used here to illustrate some essential aspects:

\begin{algorithm}
\caption{Richardson Iteration}
\begin{algorithmic}
  \STATE \text{start value} \: $x_{0}$
  \FOR{$j = 0, 1, 2, \dots$}
    \STATE $x_{j+1} = x_{j} + C^{-1} (b - Ax_{j})$
  \ENDFOR
\end{algorithmic}
\end{algorithm}

The matrix $C$ denotes the so-called preconditioner, which is used to
improve the condition of the problem to obtain faster convergence speed.
It should fulfill two major conditions:
\begin{itemize}
\item cheap matrix-vector multiplication with $C^{-1}$
\item good approximation of $A$
\end{itemize}

A first approach for $C$ is to take the diagonal of $A$ as the
preconditioner:
$$C = diag(A)$$
This constitutes the {Jacobi method}:
$$ x_j^{new} = x_j^{old} + \frac{1}{A_{jj}} \left(b_{j} -
               \sum_{k} A_{jk} x_k^{old}\right)
               \qquad j = 1 \dots n$$
It can be altered to use the newly computed entries as well. This leads to
the {Gauss-Seidel method}, which is described in more detail in
chapter~\ref{section:gs}.
\cite{iterative}

\subsection{Algebraic Multi-Grid}
The Multi-grid (MG) method now seeks to combine the smoothing of high
frequency errors by the Jacobi or Gauß-Seidel method with a second method which
better addresses lower frequency errors.

Therefore a prolongation matrix $P$ is introduced, which for a basic example
can look like:
$$P =
\begin{pmatrix}
  1 & 0 & 0 & 0 \\
  1 & 0 & 0 & 0 \\
  0 & 1 & 0 & 0 \\
  0 & 1 & 0 & 0 \\
  0 & 0 & 1 & 0 \\
  0 & 0 & 0 & 1 \\
  0 & 0 & 0 & 1 \\
  0 & 0 & 0 & 1 \\
\end{pmatrix}
$$

Now there are several approaches to come up with a prolongation matrix.
The Algebraic Multi-grid (AMG) method we adopted uses only the information
in the matrix $A$ to compute the coarsening and then the projection matrix.
It does so by weighting the relative strength of connecting vertices to
determine which vertices should collapse on the next level.

Once the prolongation matrix is computed it is used to compute the update in a
step of an iterative method similar to the preconditioners described in
chapter~\ref{section:iter}.
First the transposed prolongation matrix projects the residuum on a
coarser space, where the problem can be solved directly and then the computed
update gets prolongated back onto the fine space by multiplication with $P$.
An iterative step with two levels is thus described by:
$$x^{new} = x^{old} +\underbrace{P}_\text{refine}\cdot \underbrace{(P^{T} A P)^{-1}}_\text{solve the coarsed problem}\cdot \underbrace{P^{T}}_\text{coarse the residuum} (b - A x^{old})$$

The combination with the Gauß-Seidel method is provided by solving the problem
recursively on multiple levels and applying a forward and backward smoothing
step on each level. Additionally a transposed matrix vector multiplication
is needed to project onto the coarse space.
See Listing~\ref{lst:mult} for more details on the AMG algorithm.
Those three points are subject to performance improvements by parallelization
which is the main point we address.
\cite{multigrid}

\section{Sparse Matrices}
In our case the sparse matrices are stored in the Compressed Row Storage (CRS)
format. The commands {\em firsti[k]}\/ and {\em lasti[k]}\/ provide the
position of the first and last entry of the row k in the index vector.
An iteration through one row is very easy to provide:

\lstset{language=C++, numbers=none, captionpos=b,
        caption={Row-iteration of sparsematrices }}
\begin{lstlisting}
for (int j = firsti[k]; j < lasti[k]; ++j)
  {...}
\end{lstlisting}

The difficulty with sparse matrices is the iteration among any other order as
the next section will show.

\section{Matrix Vector Multiplication}
The multiplication of a sparse matrix and a given vector is very simple.
The entry k of the output vector is given by the summation of the products
of the matrix entries in the row k with the entries of the vector. In other
words, one entry of the output vector depends only on one row of the matrix
and vice versa.

One of the difficulties of parallelization is given by multiple writing
performances on a single variable at the same time. In the case of a simple
matrix vector multiplication this is no big deal to handle: each row has to be
executed by a single thread, there is no limitation in which order which thread
calculates its vector entry.

\lstset{language=C++, numbers=left, captionpos=b,
        caption={Matrix vector multiplication with parallel for section.}}
\begin{lstlisting}
void MultAddParallel w(double s, const BaseVector & x, BaseVector & y)
{
#pragma omp parallel for
    for (int i = 0; i < this->Height(); ++i) {
      int first = firsti [i];
      int last  = firsti [i+1];

      for (int j = first; j < last; ++j) {
        y(i) += s * data[j] * x(colnr[j]);
      }
    }
  }

\end{lstlisting}

\subsection{Transposed Matrix Vector Multiplication}\label{section:trans}
To provide the multiplication of a vector with the transposed matrix, a
sequential algorithm is written in a similar way.

\lstset{language=C++, numbers=none, captionpos=b,
        caption={Transposed vector multiplication algorithm.}}
\begin{lstlisting}
    for (int i = 0; i < this->Height(); ++i) {
      int first = firsti [i];
      int last  = firsti [i+1];

      for (int j = first; j < last; ++j) {
        y(colnr[j]) += s * data[j] * x(i);
      }
    }
\end{lstlisting}

The difference to the previous algorithm is due to the dependencies of a single
output vector entry. It is calculated by iterating through the entries
of one matrix column. This is not efficient due to the CRS format.
Therefore the algorithm has to iterate first through the matrix rows.
The access of distinct entries per row is no problem for the sequential
code, but simple parallelization causes issues.
Concurrent reading and writing operations to one variable can lead to incorrect
stored values as shown in figure~\ref{figure:parallelwriting}. Two
thread are trying to write at the same variable at almost the same time. The
second thread starts his procedure before thread 1 has finished and overwrites the
result of the first thread.

\begin{figure}[ht]
\includegraphics{graphic/parallel_writing_problem.pdf}
\caption{Problem with Parallelization of Writing Processes}
\label{figure:parallelwriting}
\end{figure}

\begin{samepage}
Some approaches to solve this problem are:
\begin{itemize}
\item atomic writing process (easy to program, but very inefficient)
\item atomic writing process with dynamic thread partitioning
\item atomic writing process with static thread separation
\item matrix coloring without any atomic processes
\end{itemize}
\end{samepage}

All those solutions efficiency depend on the number of entries in the matrix.
Less entries and a larger matrix result in higher amounts of calculations per
time and thus are suited for parallel algorithms.
Whereas a sequential algorithm fares better than a parallel solution for
matrices with a high amount of nonzero elements for this problem.

The \textbf{atomic} version is very straight forward. The idea is to avoid the
problem shown in figure~\ref{figure:parallelwriting} by making the operations
atomic. This means that a thread blocks the address it accesses till it is
finished with its read-modify-write operation.

\lstset{language=C++, numbers=none, captionpos=b,
        caption={Atomic section in the transposed vector multiplication.}}
\begin{lstlisting}
#pragma omp atomic
          fy(colnr[j]) += s * data[j] * fx(i);
\end{lstlisting}

This part is the bottleneck of the algorithm, because every thread has to has
pass this line and they block each other.

The \textbf{dynamic thread partitioning} solves another problem of this code.
{\em OMP for}\/ (as used in the atomic version) splits the for-loop into a
evenly distributed amount of iterations for each thread beforehand. Due to
differing amount of calculations each thread has to perform, some may finish
faster than other thread. This relates to the specific structure of matrices
resulting in a finite element discretization, the number of nonzero elements
per row can vary significantly.

Instead it is possible to split the jobs dynamically. Faster threads are
assigned new iterations while slower thread still work on the old iterations.
The dynamic assignment produces some overhead, which can be mitigated by
assigning chunks of iterations.

\lstset{language=C++, numbers=none, captionpos=b,
        caption={Dynamic for loop.}}
\begin{lstlisting}
#pragma omp for schedule (dynamic, 100)
\end{lstlisting}

The \textbf{balancing option} performs a similar assignment of iterations, but
does so before the for-loop starts instead of a dynamic assignment. The work
for the calculation of each row is approximated with the number of nonzero
elements in this row and split the loop accordingly. (see listing~\ref{lst:tranbal}

Every method still has the same problem with the atomic operation in its
essential part. We can avoid this by "\textbf{coloring}" each row of the matrix.
Rows are assigned to the same color if they don't write on the same entries.
Parallelization of this group is therefore possible, as no conflicting
concurrent operations will occur. Thus it is possible to iterate through the
colors sequentially and parallelize each color.

Not writing at the same entry is equivalent to not sharing any columns with
nonzero elements. Figure~\ref{figure:coloring} shows an example of the
coloring process.

\begin{figure}[ht]
\includegraphics[width=0.32\textwidth]{graphic/coloringT2.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringT3.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringT4.eps}
\includegraphics[width=0.32\textwidth]{graphic/coloringT5.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringT6.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringT7.eps}
\includegraphics[width=0.32\textwidth]{graphic/coloringT8.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringT9.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringT10.eps}
\caption{Coloring of a Simple Matrix}\label{figure:coloring}
\end{figure}

This is easy to perform for small matrices. To cope with larger matrices we
iterate through the matrix for every color and store the position of blocked
nonzero elements for the specific color in a mask.
Each row is compared to the mask and if no nonzero entry of the row interferes
with the already blocked entries, the row is added to the current color and the
mask gets updated accordingly.
As long as there are uncolored rows of the matrix new colors will be used.
To further optimize this we use a mask for 32 colors at once.
(see listing~\ref{lst:trancol}
and~\ref{lst:tranmul})

\section{Gauß-Seidel Method} \label{section:gs}
As mentioned in chapter~\ref{section:motiv} we use two Gauß-Seidel steps on
each level. The Gauß-Seidel method is similar to the Jacobi method.
Both methods need to compute the vector $Ax$. The difference is, that the
Gauß-Seidel method also uses the new information obtained by previous
computations of entries.
In particular a new entry is given by:
$$ x_j^{new} = x_j^{old} + \frac{1}{A_{jj}} \left(b_{j} -
  \sum_{k \in K^{new}}A_{jk} x_k^{new} - \sum_{k \in K^{old}}A_{jk} x_k^{old}
  \right) \qquad j = 1 \dots n$$
$K^{new}$ denotes the index set of newly calculated entries
of $x$ and $K^{old}$ denotes the index set of old entries.
\cite{iterative}

The calculation order in the forward Gauß-Seidel precondition is arbitrary,
but to achieve a symmetric preconditioner the backward smoothing needs to be
in exact reverse order.

Entries of the output vector $x$ must not be calculated concurrently if they
depend on each other. As in chapter~\ref{section:trans} described, this would
cause incorrect entries. The problems with parallelization are also very
similar to chapter~\ref{section:trans}, as are the ideas to cope with them.

A coloring algorithm can be used as well and is similar in its implementation.
As with the transposed matrix vector multiplication we group the rows which
may run parallel without influencing each other.

\subsection{Coloring}
To calculate the product $A_jx$ which modifies the entry $x_j$ (where $A_j$
denotes the j-th row of the matrix $A$), for every $k$ with $A_{jk} \neq 0$,
$x_k$ is used. According to this observation, every row $A_j$, which should be
added to an existing color must not have any nonzero elements on the positions $A_{jk}$
for all $k$ indices of rows marked with this color.

Instead of testing all remaining rows for this property, we use that $A$ is
symmetric: $A_{kj} = 0 \Leftrightarrow A_{jk} = 0$. Thus we only check
the specific row we added to one color to determine which rows can also
relate to it.

The main advantage of this method is, that the coloring process has to be
done only once, whereas it is used by GSSmooth and GSSmoothBack as well.
Also the order of colors is stored by the process itself.

For a small matrix the coloring process is demonstrated in
figure~\ref{figure:coloringGS}.

\begin{figure}
\includegraphics[width=0.32\textwidth]{graphic/coloringGS1.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringGS4.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringGS7.eps}
\includegraphics[width=0.32\textwidth]{graphic/coloringGS8.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringGS9.eps}\hfill\vline\hfill
\includegraphics[width=0.32\textwidth]{graphic/coloringGS10.eps}
\caption{Coloring of a Simple Matrix for the Gauß-Seidel Algorithm.}
\label{figure:coloringGS}
\end{figure}

\subsection{Speed Improvement}
As mentioned, the AMG preconditioner is a combination of the Gauß-Seidel
preconditioner and intermediate projections onto coarser spaces
by multiplying with the transposed prolongation matrix. These steps are
executed alternately until the matrix is small enough to invert it directly.
Then the correction is refined to its original size and backwards Gauß-Seidel is
used in between these steps.
This sums up to one step of the AMG-Preconditioner. Many of such steps are
executed as a part of the CG method until the solution is sufficiently exact.
(see listing~\ref{lst:mult})

Speed improvements by parallelization are possible within Gauß-Seidel steps
and the transposed matrix vector multiplication. The Gauß-Seidel coloring has to
be done separately for each level before the CG method
(see listing~\ref{lst:jacobi}), but can be used again
in each CG iteration (listing~\ref{lst:gss} and~\ref{lst:gssb}).
For the transposed matrix vector multiplication a balancing algorithm is used.

To measure and visualize the time of these steps a trace explorer can be used.
{\em Vite\footnote{http://vite.gforge.inria.fr}}\/ is a tool to visualize
execution traces in Pajé or OTF format for debugging and profiling parallel or
distributed applications.
The work of every thread is shown in a single row. Sections in the code have
different but specific colors, so it is obvious which parts of the algorithm
took most of the computing time and how the work is distributed among the
threads.

Figure~\ref{figure:sequ} shows a CG method with sequential AMG, whereas
figure~\ref{figure:par} shows it with parallel AMG\@.
Only one thread is working in the sequential version compared to 24 threads in
the parallel.

In figure~\ref{figure:par} and~\ref{figure:step} the red blocks indicate a
thread of forward Gauß-Seidel smoothing, the purple ones are projections and
prolongations and the green ones indicate backward Gauß-Seidel smoothing.
In figure~\ref{figure:step} a single step of the AMG is shown. The large matrix
needs more time to perform a Gauß-Seidel step.
This is indicated by the big red blocks at the beginning. Only one color is
calculated at the same time, but its rows can be processed by several threads
simultaneously.

\begin{figure}
    \includegraphics[width=1\textwidth]{seq.png}
    \caption{AMG sequential}\label{figure:sequ}
\end{figure}

\begin{figure}
    \includegraphics[width=1\textwidth]{iterations_trace.png}
    \caption{AMG parallel}\label{figure:par}
\end{figure}

\begin{figure}
    \includegraphics[width=1\textwidth]{undistributed_coloring_gs.png}
    \caption{AMG single parallel recursion}\label{figure:step}
\end{figure}

Due to the matrix being stored split in half on to the specific structure of
the {\em Vector}\/ server, its two nodes have different speed of writing and
reading at parts of the matrix.
Depending on the tread to node correspondence, some colors are faster
calculated by the first half of threads, and others by the other half.
This emerges from the coloring process. When iterating over the matrix rows,
the colors tend to cluster, as it is unlikely that rows have columns in common.
To mitigate this effect we iterate with an offset over the matrix rows in the
coloring process.

A comparison of figure~\ref{figure:step} and figure~\ref{figure:stepdis} shows
that most of the gray spaces between the blocks can be eliminated by minimizing the
correspondence of rows and colors. (see listing~\ref{lst:jacobi})

\begin{figure}
    \includegraphics[width=1\textwidth]{one_iteration.png}
    \caption{AMG single parallel recursion with distribution}\label{figure:stepdis}
\end{figure}

\pagebreak

\section{Code}

\lstset{language=C++, numbers=left, captionpos=t, breaklines=true,
        caption={Static balancing in the transposed vector multiplication.},label=lst:tranbal}
\begin{lstlisting}
// The separation of the matrix is calculated based on the number
// of non-zero elements in the matrix. Each thread has to do almost
// the same amount of work.
void TranMultBalance (double s, const BaseVector & x, BaseVector & y) const
{
  FlatVector<double> fx = x.FV<double> ();
  FlatVector<double> fy = y.FV<double> ();

  int height = this->Height();

  Array<int> thread_separation;
#pragma omp parallel
  {
    int num_threads = omp_get_num_threads();

#pragma omp single
    {
      thread_separation = Array<int>(num_threads+1);
      // load for one thread
      int separation_step = ceil(this->nze / num_threads);

      // find section of rows in the matrix for each thread
      int thread_i = 1;
      for (int row = 0; row < height; ++row)
      {
        if (firsti[row] >= thread_i * separation_step)
        {
          thread_separation[thread_i] = row;
          ++thread_i;
        }
      }
      thread_separation[0] = 0;
      thread_separation[num_threads] = height;
    }

// parallelization of the transposed matrix-vector-multiplication based on the
// previous calculated thread separation
#pragma omp for
    for (int thread_i = 1; thread_i <= num_threads; ++thread_i)
    {
      for (int i = thread_separation[thread_i-1];
            i < thread_separation[thread_i]; ++i)
      {
        int first = firsti [i];
        int last  = firsti [i+1];

        for (int j = first; j < last; ++j)
        {
#pragma omp atomic
          fy(colnr[j]) += s * data[j] * fx(i);
        }
      }
    }
  }
}

\end{lstlisting}

\lstset{language=C++, numbers=left, captionpos=t,
        caption={Coloring of a matrix for transposed vector multiplication},
        label=lst:trancol}
\begin{lstlisting}
void Coloring ()
{
  int height = this->Height();
  int width = this->Width();

  Array<int> row_color(height);  // storage of the colors for each row
  row_color = -1;

  int maxcolor = 0;
  int basecol = 0;

  Array<unsigned int> mask(width);  // 32-bit masks for each column

  int found = 0;

  do
  {
    mask = 0;

    for (int row = 0; row < height; ++row)
    {
      // next row if row already has a color
      if (row_color[row] >= 0) continue;

      int first = firsti [row];
      int last  = firsti [row+1];

      unsigned check = 0;

      // check is the union of all mask bits corresponding
      // to nze's in the row
      for (int i = first; i < last; ++i)  // iterate over entries of the row
        check |= mask[colnr[i]];

      // find next free color
      if (check != UINT_MAX)
      {
        found++;
        unsigned checkbit = 1;
        int color = basecol;
        while (check & checkbit)
        {
          color++;
          checkbit *= 2;
        }

        row_color[row] = color;  // set color to current row
        if (color > maxcolor) maxcolor = color;

        for (int i = first; i < last; ++i)
          mask[colnr[i]] |= checkbit;  // update mask
      }
    }

    basecol += 8*sizeof(unsigned int);
  }
  while (found < height);

  // count rows per color
  Array<int> cntcol(maxcolor+1);
  cntcol = 0;
  for (int row = 0; row < height; ++row)
    ++cntcol[row_color[row]];

  coloring_ = table<int>(cntcol);

  cntcol = 0;
  // construct mapping of rows to colors for the iteration
  for (int row = 0; row < height; ++row)
    coloring_[row_color[row]][cntcol[row_color[row]]++] = row;

  std::cout << "needed " << maxcolor+1 << " colors" << std::endl;
}

\end{lstlisting}

\lstset{language=c++, numbers=left, captionpos=t,
        caption={transposed vector multiplication by coloring a matrix},
        label=lst:tranmul}
\begin{lstlisting}
void TranMultAdd4 (double s, const BaseVector & x, BaseVector & y)
{
  FlatVector<double> fx = x.FV<double> ();
  FlatVector<double> fy = y.FV<double> ();

  Coloring();

#pragma omp parallel
  {
    // iterate sequential over colors
    for (auto color : coloring_)
    {
      // iterate parallel over corresponding rows
#pragma omp for
      for (int i = 0; i < color.Size(); ++i)
      {
        int first = firsti [color[i]];
        int last  = firsti [color[i]+1];

        for (int j = first; j < last; ++j)
        {
          fy(colnr[j]) += s * data[j] * fx(color[i]);
        }
      }
    }
  }
}

\end{lstlisting}

\lstset{language=C++, numbers=left, captionpos=t, breaklines=true,
        caption={Multiplication Method of the AMG Preconditioner},label=lst:mult}
\begin{lstlisting}
void my_AMG_H1 :: Mult (const ngla::BaseVector & b, ngla::BaseVector & x) const
{
  // direct solve at lowest level
  if (inv) {
    x = (*inv) * b;
    return;
  }

  auto residuum = pmat->CreateVector();
  auto coarse_x = coarsemat->CreateVector();
  auto coarse_residuum = coarsemat->CreateVector();

  x = 0;
  jacobi->GSSmooth (x, b);  // one Gauss-Seidel step

  if (recAMG) {
    residuum = b - (*pmat) * x;
    coarse_residuum = ngla::Transpose (*prol) * residuum;  // coarsening

    // recursive solving of coarser problem
    recAMG->Mult(coarse_residuum, coarse_x);

    x += (*prol) * coarse_x;  // prolongate back to fine grid
  }

  jacobi->GSSmoothBack (x, b);  // Gauss-Seidel back
}
\end{lstlisting}

\lstset{language=c++, numbers=left, captionpos=t,
        caption={Gauß-Seidel coloring and balancing},label=lst:jacobi}
\begin{lstlisting}

void JacobiPrecond<TM,TV_ROW,TV_COL> :: Coloring ()
{
  int height = mat.Height();
  int width = mat.Width();

  Array<int> row_color(height);  // storage of colors for each row
  row_color = -1;

  int maxcolor = 0;
  int basecol = 0;

  Array<unsigned int> mask(width);  // 32-bit masks for each column

  int found = 0;

  int num_threads = task_manager->GetNumThreads();

  do
  {
    mask = 0;

    int block_size = ceil((double)height / (double)num_threads);

    // iterate over blocks to get rows of the whole matrix
    // for every color
    for (int block_row = 0; block_row < block_size; ++block_row)
    {
      for (int threadi = 0; threadi < num_threads; ++threadi)
      {
        int row = block_row + threadi * block_size;
        if (row >= height) break;

        // next row if row already has a color
        if (row_color[row] >= 0) continue;

        int first = mat.firsti [row];
        int last  = mat.firsti [row+1];

        unsigned check = 0;

        // check is the union of all mask bits corresponding
        // to nze's in the row
        for (int i = first; i < last; ++i)  // iterate over entries of the row
          check |= mask[mat.colnr[i]];

        // find the next possible color
        if (check != UINT_MAX)
        {
          found++;
          unsigned checkbit = 1;
          int color = basecol;
          while (check & checkbit)
          {
            color++;
            checkbit *= 2;
          }

          row_color[row] = color;  // set the color to the current row
          if (color > maxcolor) maxcolor = color;

          mask[row] |= checkbit;  // update the mask
        }
      }
    }

    basecol += 8*sizeof(unsigned int);
  }
  while (found < height);

  Array<int> cntcol(maxcolor+1);
  cntcol = 0;
  for (int row = 0; row < height; ++row)
    ++cntcol[row_color[row]];  // count rows per color

  coloring_ = Table<int>(cntcol);

  cntcol = 0;
  // construct colors to row mapping
  for (int row = 0; row < height; ++row)
    coloring_[row_color[row]][cntcol[row_color[row]]++] = row;

  balancing_.SetSize (maxcolor+1);
  for (auto c : Range (balancing_))
  {
    balancing_[c].Calc (coloring_[c].Size(),
      [&] (int bi)
      {
        return 5 + mat.GetRowIndices(coloring_[c][bi]).Size();
      });
  }

  std::cout << "needed " << maxcolor+1 << " colors" << std::endl;
}

\end{lstlisting}

\lstset{language=c++, numbers=left, captionpos=t,
        caption={Gauß-Seidel Smooth},label=lst:gss}
\begin{lstlisting}
void JacobiPrecond<TM,TV_ROW,TV_COL> ::
GSSmooth (BaseVector & x, const BaseVector & b) const
{
  FlatVector<TV_ROW> fx = x.FV<TV_ROW> ();
  const FlatVector<TV_ROW> fb = b.FV<TV_ROW> ();

  // iterate sequential over colors
  for (int j = 0; j < this->coloring_.Size(); ++j)
  {
    auto color = this->coloring_[j];
    // task-manager based parallelization
    ParallelFor( this->balancing_[j],
      [this, fx, fb, color] (int i)
      {
        int row = color[i];
        if (!this->inner || this->inner->Test(row))
        {
          TV_ROW ax = mat.RowTimesVector (row, fx);
          fx(row) += invdiag[row] * (fb(row) - ax);
        }
      }, 4);
  }
}
\end{lstlisting}

\lstset{language=c++, numbers=left, captionpos=t,
        caption={Gauß-Seidel SmoothBack},label=lst:gssb}
\begin{lstlisting}
void JacobiPrecond<TM,TV_ROW,TV_COL> ::
GSSmoothBack (BaseVector & x, const BaseVector & b) const
{
  FlatVector<TV_ROW> fx = x.FV<TV_ROW> ();
  const FlatVector<TV_ROW> fb = b.FV<TV_ROW> ();

  // iterate sequential over colors
  for (int j = this->coloring_.Size()-1; j >= 0; --j)
  {
    auto color = this->coloring_[j];

    // task-manager based parallelization
    ParallelFor( this->balancing_[j],
      [this, fx, fb, color] (int i)
      {
        int row = color[i];
        if (!this->inner || this->inner->Test(row))
        {
          TV_ROW ax = mat.RowTimesVector (row, fx);
          fx(row) += invdiag[row] * (fb(row) - ax);
        }
      }, 4);
  }
}
\end{lstlisting}

\bibliography{doc}{}
\bibliographystyle{plain}
\end{document}


\documentclass[conference]{IEEEtran}
\IEEEoverridecommandlockouts
% The preceding line is only needed to identify funding in the first footnote. If that is unneeded, please comment it out.
\usepackage{acronym}

\usepackage{esvect}

\acrodef{PSO}[PSO]{\emph{particle swarm optimisation algorithm}}
\acrodef{NN}[NN]{\emph{neural network}}
\acrodef{CPSO}[CPSO]{\emph{chaotic particle swarm optimisation algorithm}}
\acrodef{RNG}[RNG]{\emph{random number generator}}
\acrodef{CPRNG}[CPRNG]{\emph{chaotic pseudo-random number generator}}
\acrodef{VN}[VN]{\emph{Von Neumann}}
\acrodef{MSE}[MSE]{\emph{mean squared error}}
\usepackage{cite}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{commath}
\usepackage{algorithmic}
\usepackage{array,multirow,graphicx}
\usepackage{textcomp}
\def\BibTeX{{\rm B\kern-.05em{\sc i\kern-.025em b}\kern-.08em
    T\kern-.1667em\lower.7ex\hbox{E}\kern-.125emX}}
\begin{document}

\title{Investigating Boundary Constraint Handling Mechanisms in Particle Swarm Optimization\\
}

\author{\IEEEauthorblockN{\\Quinton Weenink}
\IEEEauthorblockA{\textit{dept. Computer Science} \\
\textit{University of Pretoria}\\
Pretoria, South Africa \\
u13176545@tuks.co.za}
}

\maketitle

\section{Introduction}
This paper aims to investigate the effect of Boundary Constraint Handling Mechanisms on \ac{PSO}. There are many approaches that claim to improve exploration of the viable search space. In comparing these approaches to the a \ac{PSO} with no boundary constraints we could observe the changes such mechanisms are having on the performance of the \ac{PSO}.

\section{Particle Swarm Optimization}
A \ac{PSO} is a machine learning algorithm where each particle in the population represents a possible solution to the optimisation problem \cite{anna:meas-sat-nn}. Each particle moves around the search space attempting to find the optimal solution with the influence of its neighbouring particles. The position for each particle iteration is calculated as follows.

\begin{equation} \label{eq:pso:update-position}
\vec{x}_{i}^{\,t} = \vec{x}_{i}^{\,t-1} + \vec{v}_{i}^{\,t}
\end{equation}

\noindent Where:
\begin{itemize}
	\item $ \vec{x} $ is the particle position vector in n-dimensions
	\item $ \vec{v} $ is the particle velocity vector, used to calculate the particles next position
	\item $t$ is the time increment
	\item $i$ is the particle number
\end{itemize}
\vspace{5mm}
\noindent For each iteration of the algorithm, a new location of the particle is calculated based on its previous location and velocity vector as seen in (\ref{eq:pso:update-position}). The following describes a the velocity update algorithm.

\begin{equation} \label{eq:pso:update-velocity}
\vec{v}_{i}^{\,t} = w . \vec{v}_{i}^{\,t-1} + c_1 . \vec{r}_{1}^{\,t} . (\vec{x}_{pBest, i} - \vec{x}_{i}^{t-1}) + c_2 . \vec{r}_{2}^{\,t} . (\vec{x}_{nBest, i} - \vec{x}_{i}^{t-1})
\end{equation}

\noindent Where:
\begin{itemize}
	\item $c_1$ is the acceleration coefficient for the cognitive component
	\item $c_2$ is the acceleration coefficient for the social component
	\item $r_1$ and $r_2$ is a vector of random numbers in the range (0, 1)
	\item $\vec{x}_{pBest}$ is the personal best position of that particle
	\item $\vec{x}_{nBest}$ is the best position found in that particles neighbourhood
\end{itemize}
\vspace{5mm}
\noindent Particle's social component, the third term in (\ref{eq:pso:update-velocity}), requires $ nBest $, the best neighbouring particles position. The set of neighbouring particles is determined by the topology used in the \ac{PSO}. Different topologies could influence the performance of the \ac{PSO}.

Additionally, velocity max, $ V_{max} $, can be used as the limiter when calculating the particle's velocity in (\ref{eq:pso:update-velocity}), ensuring that the particle stays inside search space as well as reduce skipping over more optimal solutions. $ V_{max} $ could however prevent particles from exploring more optimal solutions.

    \subsection{Social Network Topologies}

    In the above mentioned \ac{PSO} the neighbourhood could be described in a variety of ways. One such method is the $ gBest $ topology, where each particle is a neighbour of every other particle in the network. This means that the particle's $ nBest $ is always the global best particle for the swarm.

    Another social topology called $ lBest $ connects particles in a ring topology, the neighbourhood is specified by a fixed size of $ n_s $. The $ nBest $ in this case is determined by the best particle within $ n_s $ neighbours from the current particle\cite{vanwyk:overfitting-psoffnn}.

    Finally, another social network topology exists called the \ac{VN} social network topology. This topology connects each particle to its neighbours in a lattice\cite{vanwyk:overfitting-psoffnn}. In order to determine the $ nBest $, the best error of its north, south, east and western, neighbours is used (\ref{eq:pso:update-velocity}).

	\subsection{Benchmark functions}

	No shifted, noisy or rotated version of the functions were used.
	In an effort to make things scaleable different types of benchmarks functions were used in order to determine the fitness of the functions.

	\begin{table}[htbp]
    \caption{Benchmark functions}
    \begin{center}
    \begin{tabular}{|c|c|c|c|c|}
    \hline
    \textbf{Function}& \textbf{\textit{Multimodal}}& \textbf{\textit{Inseparable}}\\
    \hline
    Absolute value& & \\
    \hline
    Ackley& Yes& Yes\\
    \hline
    Hyperellipsoid& & \\
    \hline
    Quartic& & \\
    \hline
    Salomon& Yes& Yes\\
    \hline
    Schaffer 6& Yes& Yes\\
    \hline
    \end{tabular}
    \label{tab:nn:configuration}
    \end{center}
    \end{table}

\section{Boundary Constraint Handling Mechanisms}

The following describes the boundary constraint handling mechanisms used in this research.

\subsubsection{\textbf{No boundary constraints}}
Particles were not constrained in any way.

\subsubsection{\textbf{Feasible position update}}
Update the personal best positions only if the new particle position is better than its current personal best position, and if the new particle position is feasible. That is, a new particle position can not become
a personal best position if it violates boundary constraints.

\subsubsection{\textbf{Clamping approach}}
If a particle violates a boundary constraint in a specific dimension, then clamp the corresponding decision variable at the boundary value.

\subsubsection{\textbf{Per element reinitialization}}
For any decision variable of any particle that violates a boundary constraint, reinitialize that decision variable to a random position that satisfies the boundary constraints.

\subsubsection{\textbf{Per element reinitialization and velocity = 0}}
Adapt the per element reinitialization approach above to also set the velocity of the decision variable that violates a boundary constraint to zero. The corresponding decision variable’s new position will therefore not be influenced by the momentum term.

\subsubsection{\textbf{Initialize to personal best position}}
Initialize the boundary violating decision variable to the corresponding personal best position.

\subsubsection{\textbf{Initialize to personal best position and velocity = 0}}
Adapt the intialize to personal best position strategy above to also set the corresponding velocity to 0.

\subsubsection{\textbf{Initialize to global best position}}
Initialize to global best position: As for the above, but rather the global best for the boundary violating decision variable.

\subsubsection{\textbf{Initialize to global best position velocity = 0}}
Initialize to global best position and set velocity to zero: Adapt the intialize to global best position strategy to also set the corresponding velocity to 0.

\subsubsection{\textbf{Reverse velocity}}
The velocity of the bounadry violating decision variable is simply reversed while that decision variable violates the boundary constraint.

\subsubsection{\textbf{Arithmetic average}}
Set the bounadry violating decision variable to an arithmetic average of the corresponding personal best and global best position.

\section{Experiments}
    \subsection{\ac{PSO} configuration}
    In order to achieve viable results, each experiment was run as a mean over 50 samples. $r_1$ and $r_2$ sampled from a uniform distribution $ (0, 1) $ as specified in equation \ref{eq:pso:update-velocity}.

    A memory based \ac{PSO} was used with a star or $ gBest $ social topologie was used for this study. 30 particles were used for all functions. 30 dimensions were used for all functions. $ c_1 = c_2 = 1.4 $ as well $ w = 0.7 $

    No $ V_{max} $ was used

\section{Results}

\begin{table}[htbp]
\caption{Results after 5000 iterations. Means are reported over 50 samples with standard deviations in parenthesis}
\begin{center}
\begin{tabular}{|c|c|c|c|}
\hline
& $ f_1 $ & $ f_2 $ & $ f_3 $\\\hline
\multirow{2}{*}{$ BC_1 $} &
12.576554695 & 4.89108089516 & 6.6095267e-05\\
&(16.285050) & (1.948212190) & (0.00038360)\\
\hline
\multirow{2}{*}{$ BC_2 $} &
5.1857243227 & 4.58116294703 & 2.786555e-06\\
&(9.6293110) & (1.565766830) & (7.7739e-06)\\
\hline
\multirow{2}{*}{$ BC_3 $} &
57.531250757 & 6.00606880913 & 54.526024907\\
&(65.651655) & (3.434289275) & (90.3297955)\\
\hline
\multirow{2}{*}{$ BC_4 $} &
8.2467633532 & 4.55418774247 & 1.01489e-05\\
&(11.461532) & (1.648445767) & (4.589539e-05)\\
\hline
\multirow{2}{*}{$ BC_5 $} &
5.1389339174 & 4.89810256890 & 9.465576e-05\\
&(9.0355906) & (1.720959805) & (0.00032185)\\
\hline
\multirow{2}{*}{$ BC_6 $} &
9.4929921868 & 4.97768223959 & 9.924092e-06\\
&(13.040311) & (2.130050755) & (3.56359e-05)\\
\hline
\multirow{2}{*}{$ BC_7 $} &
13.266182721 & 5.07560687421 & 2.096943e-05\\
&(22.017559) & (2.252312710) & (9.81292e-05)\\
\hline
\multirow{2}{*}{$ BC_8 $} &
15.164538450 & 6.02058451060 & 0.0004165906\\
&(23.109397) & (3.086329288) & (0.00160546)\\
\hline
\multirow{2}{*}{$ BC_9 $} &
20.925653391 & 6.42808810347 & 0.0009752216\\
&(26.146772) & (3.256848326) & (0.00390649)\\
\hline
\multirow{2}{*}{$ BC_10 $} &
599.58515676 & 17.9263162102 & 797.0981965\\
&(173.65772) & (1.816182460) & (379.309663) \\
\hline
\multirow{2}{*}{$ BC_11 $} &
20.710428245 & 5.73874408816 & 15.008693567\\
&(37.284614) & (3.127739837) & (28.25022732)\\
\hline
& $ f_4 $ & $ f_4 $ & $ f_6 $\\\hline
\multirow{2}{*}{$ BC_1 $} &
1.9239946e-21 & -0.259725655 & 9.913329509\\
&(1.08892e-20) & (0.14964295) & (0.656834)\\
\hline
\multirow{2}{*}{$ BC_2 $} &
1.7858202e-26 & -0.1334162 & 9.194013340\\
&(1.1080e-25) & (0.4525834) & (0.916300)\\
\hline
\multirow{2}{*}{$ BC_3 $} &
0.37580965 & 0.2911377709212 & 10.4460088\\
&(1.8605501) & (1.837814591) & (0.7901153)\\
\hline
\multirow{2}{*}{$ BC_4 $} &
1.699433e-25 & -0.136358848 & 7.8171530590\\
&(8.329e-25) & (0.32299095) & (0.97061129)\\
\hline
\multirow{2}{*}{$ BC_5 $} &
6.203499e-20 & -0.134392611 & 9.17659654534\\
&(4.2647e-19) & (0.4148771) & (1.1028635337)\\
\hline
\multirow{2}{*}{$ BC_6 $} &
8.175363e-21 & -0.210002336 & 8.74905186834\\
&(5.722e-20) & (0.28267089) & (0.9779485313)\\
\hline
\multirow{2}{*}{$ BC_7 $} &
2.418665e-16 & -0.0636568206 & 9.9507224427\\
&(1.324e-15) & (0.466326789) & (0.798154431)\\
\hline
\multirow{2}{*}{$ BC_8 $} &
7.26727e-16 & -0.2007856567 & 10.1430737966\\
&(5.086e-15) & (0.287498829) & (0.852829383)\\
\hline
\multirow{2}{*}{$ BC_9 $} &
1.907742e-18 & -0.053840835 & 9.899939307\\
&(1.335e-17) & (0.552251796) & (0.88794107)\\
\hline
\multirow{2}{*}{$ BC_10 $} &
26.79745200 & 15.5318252566 & 12.16846579\\
&(15.81222) & (2.559650039) & (0.50818004)\\
\hline
\multirow{2}{*}{$ BC_11 $} &
4.786350e-16 & -0.2006067722 & 10.158431877\\
&(3.2479e-15) & (0.288737979) & (0.69947078)\\
\hline
\end{tabular}
\label{tab:glass}
\end{center}
\end{table}

\begin{figure}[htbp]
\centerline{\includegraphics[width=90mm]{images/results/AbsoluteValue.png}}
\caption{$f_1$ performance}
\label{fig:absoluteValue}
\end{figure}

\begin{figure}[htbp]
\centerline{\includegraphics[width=90mm]{images/results/Ackley.png}}
\caption{$f_2$ performance}
\label{fig:ackley}
\end{figure}

\begin{figure}[htbp]
\centerline{\includegraphics[width=90mm]{images/results/Hyperellipsoid.png}}
\caption{$f_3$ performance}
\label{fig:hyperellipsoid}
\end{figure}

\begin{figure}[htbp]
\centerline{\includegraphics[width=90mm]{images/results/Quartic.png}}
\caption{$f_4$ performance}
\label{fig:quartic}
\end{figure}

\begin{figure}[htbp]
\centerline{\includegraphics[width=90mm]{images/results/Salomon.png}}
\caption{$f_5$ performance}
\label{fig:salomon}
\end{figure}

\begin{figure}[htbp]
\centerline{\includegraphics[width=90mm]{images/results/SchafferF6.png}}
\caption{$f_6$ performance}
\label{fig:schafferf6}
\end{figure}

\newpage
\section{Conclusion}
In conclusion one can observe that that reversing the velocity performs quite poorly and almost immediately stops exploring. Clamping which one could have been assumed to be a viable approach seemed to perform poorly on all benchmark functions.
Per-element initialisation to with out changing the velocity to zero introduces good results over all presumably due to the velocity allow it to still move towards its last found position.

\newpage

\nocite{*}
\noindent
\bibliographystyle{IEEEtran}
\indent
\bibliography{IEEEabrv,sbc-template}

\section{Appendix}

$f_1$, the absolute value function, with $x_j \in [-100, 100]$
\begin{equation} \label{eq:function:absolute}
    f(\mathbf{x}) = \sum_{j=1}^{n_x}{\abs{x_i}}
\end{equation}

$f_2$, the ackley function, with $x_j \in [-32.768, 32.768]$
\begin{equation} \label{eq:function:ackley}
    f(\mathbf{x}) = -20e^{-0.2\sqrt{\frac{1}{n}\sum_{j=1}^{n_x}{x_j^2}}}-e^{\frac{1}{n}\sum_{j=1}^{n_x}{cos(2\pi x_j)}} + 20 + e
\end{equation}

$f_n$, the hyperellipsoid function, with $x_j \in [−5.12, 5.12]$
\begin{equation} \label{eq:function:hyperellipsoid}
    f(\mathbf{x}) = \sum_{j=1}^{n_x}{jx_j^2}
\end{equation}

$f_n$, the quartic function, with $x_j \in [−5.12, 5.12]$
\begin{equation} \label{eq:function:hyperellipsoid}
    f(\mathbf{x}) = \sum_{j=1}^{n_x}{jx_j^4}
\end{equation}

$f_n$, the salomon function, with $x_j \in [-100, 100]$
\begin{equation} \label{eq:function:salomon}
    f(\mathbf x)=-cos(2\pi\sum_{j=1}^{n_x}x_j^2)+0.1\sqrt{\sum_{j=1}^{n_x}x_j^2}
\end{equation}

$f_n$, the schaffer 6 function, with $x_j \in [-100, 100]$
\begin{equation} \label{eq:function:schafferf6}
    f(x,y)=\sum_{j=1}^{n_x}{\left(0.5+\frac{sin^2(\sqrt{x^2 + y^2})-0.5}{[1+0.001 \cdot (x^2 + y^2)]^2}\right)}
\end{equation}

\end{document}


\section{Data Collection} \label{sec: data collection}

The data must be collected in the pattern illustrated in Figure \ref{fig:datacollection}.
%
First, the IMU must be static for $t_{init}$ seconds.
%
Then it must be placed in another aleatory position for $t_{wait}$ seconds, which causes the accelerometer reading to flip.
%
This process is repeated $n\times$, generating the accelerometer signals patterns illustrated in Figure \ref{fig:datacollection}.

\begin{figure}[h]
	\centering
	\includegraphics[width=0.7\linewidth]{figures/data_collection}
	\caption{Data collection pattern.}
	\label{fig:datacollection}
	\source{Reproduced from \cite{tedaldi2013imu}}
\end{figure}

\begin{important}
	The software will automatically detect the static intervals using the variance of the accelerometer readings as described in \cite{2014:Tedaldi}. The first static interval needs to be long (recommended $t_{init} \approx 50s$), which the other intervals are short (recommended $t_{wait}\approx 3s$). The tool will not perform the calibration if it fails to identify static intervals satisfying these conditions.
\end{important}

We implemented a module to record and save the IMU readings in a data within the \texttt{event-driven} library.
%
For installation follow:

\begin{enumerate}
	\item Check out the branch: \url{https://github.com/robotology/event-driven/tree/IMU_port}
	\item Compile with the \texttt{cmake} option \texttt{-DENABLE\_IMUCalibDumper=ON}
\end{enumerate}

For running, we recommend starting the module using the command line, which will prompt a timer to help the user to keep track of the $t_{init}$ and $t_{wait}$ times.
%
Use the following command, indicating where the accelerometer and gyroscope files must be saved (choose an existing folder):

\texttt{IMUCalibDumper --accFileName <full path for acc file> --gyrFileName <full path for gyr file>}

\begin{important}
	The integral to calculate the orientation displacement between two static intervals (Equation \ref{eq: gyr opt}) is susceptible to integration bias, the angular displacements between two static intervals during data acquisition must be short (try to keep is less than $90^\circ$).
\end{important}

The data will be written in the specified file only during \texttt{IMUCalibDumper}'s closure.


\chapter{Specification of Derived Instances}  \label{derived} \index{instance!derived}

A \emph{derived instance} is an instance that results from a \hyperref[derivedcl]{\texttt{derive} declaration}.
The body of a derived instance is derived syntactically from the definition of the associated type. Derived instances are possible only for certain classes known to the compiler.

\section{Derived Instances for Eq}

Instances of \texttt{Eq} are types whose values can be compared for equality. \texttt{Eq} instances can be derived for all algebraic data types.

\trans{
Let $P$ be a $n$-ary $(n \ge 0)$ type constructor for a product type with $k$-ary $(k \ge 0)$ data constructor $C$:
\begin{flushleft}
\textbf{data} $P$ $u_1$ $\cdots$ $u_n$ = $C$ $ct_1$ $\cdots$ $ct_k$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Eq} ($P$ $t_1$ $\cdots$ $t_n$)\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Eq}  ($Eq$ $t_1$, $\cdots$, $Eq$ $t_n$) \sym{=>} ($P$ $t_1$ $\cdots$ $t_n$) \textbf{where}\\
\hspace{0.3cm}$C$ $a_1$ $\cdots$ $a_k$ \texttt{==} $C$ $b_1$ $\cdots$ $b_k$ \texttt{=}\\
\hspace{0.6cm}\textbf{true} \texttt{\&\&} $a_1$\texttt{.==} $b_1$ \texttt{\&\&} $\cdots$ \texttt{\&\&} $a_k$\texttt{.==} $b_k$\\
\hspace{0.3cm}hashCode $x$ \texttt{=} $\cdots$\\
\end{flushleft}
}

The generated expression for the \sym{==} operator returns \textbf{true} if all subcomponents of the left operand are pairwise equal with the corresponding subcomponents of the right operand, otherwise the result is  \textbf{false}.

Note that the special case $k=0$ is trivial: such a type has only one value $C$ and the derived \texttt{==} returns always \textbf{true}.

The generated expression for \term{hashCode} computes a value of type \term{Int} suitable for use in hash tables and similar data structures. In the process of doing so, all sub-components of the value will be evaluated recursively. The result is undefined for infinite values.

The case gets only marginally more complex with sum types.
\trans{
Let $S$ be a $n$-ary $(n \ge 0)$ type constructor for a sum type with $m$  $(m \ge 2)$ data constructors $C_1, \cdots, C_m$ and arities $k_1, \cdots, k_m$:
\begin{flushleft}
\textbf{data} $S$ $u_1$ $\cdots$ $u_n$ = $C_1$ $ct_{1_1}$ $\cdots$ $ct_{k_1}$ $| \cdots |$ $C_m$ $ct_{m_1}$ $\cdots$ $ct_{k_m}$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Eq} ($S$ $t_1$ $\cdots$ $t_n$)\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Eq} ($Eq$ $t_1$, $\cdots$, $Eq$ $t_n$) \sym{=>} ($S$ $t_1$ $\cdots$ $t_n$) \textbf{where}\\
\hspace{0.33cm}$a$ \texttt{==} $b$ \texttt{=} \textbf{case} $(a, b)$ \textbf{of}\\
\hspace{0.66cm}$(C_1 a_1 \cdots a_{k_1}, C_1 b_1 \cdots b_{k_1})$\\
\hspace{2cm}$\rightarrow$ \textbf{true}  \texttt{\&\&} $a_1$\texttt{.==} $b_1$ \texttt{\&\&} $\cdots$ \texttt{\&\&} $a_{k_1}$\texttt{.==} $b_{k_1}$\\
\hspace{0.66cm}$\cdots$\\
\hspace{0.66cm}$(C_m a_1 \cdots a_{k_m}, C_m b_1 \cdots b_{k_m})$\\
\hspace{2cm}$\rightarrow$ \textbf{true} \texttt{\&\&} $a_1$\texttt{.==} $b_1$ \texttt{\&\&} $\cdots$ \texttt{\&\&} $a_{k_m}$\texttt{.==} $b_{k_m}$\\
\hspace{0.66cm}\_ $\rightarrow$ \textbf{false}\\
\hspace{0.3cm}hashCode $x$ \texttt{=} $\cdots$\\
\end{flushleft}
}

The expression $a$ \texttt{==} $b$ evaluates to \textbf{true} if both $a$ and $b$ were constrcuted with the same data constructor and their corresponding subcomponents are pairwise equal.

\subsubsection{Derived Instances for Ord}
The \texttt{Ord} class is used for totally ordered datatypes. It is a subclass of \texttt{Eq} and inherits the operations \texttt{==} and \texttt{!=}. It defines one new operation \texttt{<=>} that must be implemented by all instances, and operations \texttt{<}, \texttt{<=}, \texttt{>}, \texttt{>=}, \texttt{max} and \texttt{min} in terms of \texttt{<=>}.

The compare function \texttt{<=>}
\footnote{The alias \texttt{compare} is provided  for \haskell{} compatibility.}
compares two values and returns a result of type \texttt{Ordering}, which is defined as
\footnote{Aliases \texttt{LT},  \texttt{EQ} and \texttt{GT} are provided for \haskell{} compatibility.}
\begin{code}
    data Ordering = Lt | Eq | Gt
\end{code}

Instances of \texttt{Ord} can be derived for all algebraic data types that are either already instances of \texttt{Eq} or have an implementation for \texttt{hashCode}.

The translation shown here does not handle the case of the trivial product type. Such a type will have an implementation of \texttt{<=>} that always returns \texttt{Ordering.Eq}.

For product types, the generated expression compares the components $a_i$, $b_i$ from 1 to $k-1$; the first result $r_i$ that does not signify equality is the result of the overall comparison. Otherwise, if all component pairs up to ${k-1}$ compare equal, the result is the ordering of the last component pair, $a_k$\texttt{.<=>} $b_k$.

\trans{
Let $P$ be a $n$-ary $(n \ge 0)$ type constructor for a product type with $k$-ary $(k \ge 1)$ data constructor $C$:
\begin{flushleft}
\textbf{data} $P$ $u_1$ $\cdots$ $u_n$ = $C$ $ct_1$ $\cdots$ $ct_k$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Ord} ($P$ $t_1$ $\cdots$ $t_n$)\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Ord} ($Ord$ $t_1$, $\cdots$, $Ord$ $t_n$) \sym{=>} ($P$ $t_1$ $\cdots$ $t_n$) \textbf{where}\\
\hspace{0.33cm}$(C a_1 \cdots a_k)$ \texttt{<=>} $(C b_1 \cdots b_k)$ \texttt{=} \textbf{case} $a_1$\texttt{.<=>} $b_1$ \textbf{of}\\
\hspace{0.66cm}\texttt{Eq} $\rightarrow$\\
\hspace{1cm}$\cdots$\\
\hspace{1.33cm}\textbf{case} $a_{k-1}$\texttt{.<=>} $a_{k-1}$ \textbf{of}\\
\hspace{1.66cm}\texttt{Eq} $\rightarrow$ $a_k$\texttt{.<=>} $b_k$\\
\hspace{1.66cm}$r_{k-1}$ $\rightarrow$ $r_{k-1}$\\
\hspace{1cm}$\cdots$\\
\hspace{0.66cm}$r_1$ $\rightarrow$ $r_1$\\
\end{flushleft}
}

Derived instances for sum types make use of the Prelude function
\begin{quote}
\texttt{constructor} :: $any \rightarrow$ \texttt{Int}
\end{quote}
which returns the index of the constructor for algebraic data types.\footnote{The constructors are numbered starting from 0 in the order they appear in the \hyperref[algdcl]{data declaration}.}

The code for sum types first sorts out the cases where the constructors are not the same; the result in such a case is the ordering of the constructors. The remaining $m$ cases compare nullary constructors equal to themselves, values with unary constructors compare just like the components compare and values with constructors of higher arity compare like the tuples constructed from their components would compare when $k_i$-ary tuples had a derived instance of \texttt{Ord}.

\trans{
Let $S$ be a $n$-ary $(n \ge 0)$ type constructor for a sum type with $m$  $(m \ge 2)$ data constructors $C_1, \cdots, C_m$ and arities $k_1, \cdots, k_m$:
\begin{flushleft}
\textbf{data} $S$ $u_1$ $\cdots$ $u_n$ = $C_1$ $ct_{1_1}$ $\cdots$ $ct_{k_1}$ $| \cdots |$ $C_m$ $ct_{m_1}$ $\cdots$ $ct_{k_m}$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Ord} ($S$ $t_1$ $\cdots$ $t_n$)\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Ord} ($Ord$ $t_1$, $\cdots$, $Ord$ $t_n$) \sym{=>} ($S$ $t_1$ $\cdots$ $t_n$) \textbf{where}\\
\hspace{0.33cm}$a$ \texttt{<=>} $b$ \texttt{=} \textbf{case} $($\texttt{constructor} $a)$\texttt{<=>} $($\texttt{constructor} $b)$ \textbf{of}\\
\hspace{0.66cm}\texttt{Eq} $\rightarrow$ \textbf{case} $(a,b)$ \textbf{of}\\
\hspace{1cm}$alt_1$\\
\hspace{1cm}$\cdots$\\
\hspace{1cm}$alt_m$\\
\hspace{0.66cm}$r_0$ $\rightarrow$ $r_0$
\end{flushleft}
where each of the alternatives $alt_i$ has a form that depends on the arity of the constructor $C_i$:
\begin{flushleft}
\hspace{0.3cm}$(C_i, C_i) \rightarrow$ \texttt{Eq}\hspace{\fill}for nullary $C_i$\\
\hspace{0.3cm}$(C_i a_1, C_i b_1) \rightarrow a_1$\texttt{.<=>} $b_1$ \hspace{\fill}for unary $C_i$\\
\hspace{0.3cm}$(C_i a_1 \cdots a_{k_i}, C_i b_1 \cdots b_{k_i}) \rightarrow$ \hspace{\fill}for $C_i$ with arity $k_i \ge 2$\\
\hspace{1cm}$(a_1, \cdots, a_{k_i})$\texttt{.<=>} $(b_1, \cdots, b_{k_i})$
\end{flushleft}
}

\section{Derived Instances for Enum}

The \texttt{Enum} class can be derived for algebraic datatypes that have only nullary constructors.
It provides conversion from and to \texttt{Int} values, successor and predecessor functions and the operations \emph{enumFrom}, \emph{enumFromTo}, \emph{enumFromThen} and \emph{enumFromThenTo} to construct \hyperref[aseq]{arithmetic sequences}. \texttt{Enum} is a subclass of \texttt{Ord} and hence of \texttt{Eq}.

In addition to the above, derived instances also provide the operation \emph{hashCode} required for instances of \texttt{Eq}. In derived instances, \emph{hashCode} is always the same as \emph{ord} and not shown in the translations.

In all derived instances, the following holds for the successor and predecessor functions:

\trans{
\begin{flushleft}
\hspace{0.5cm}succ e = from (ord e + 1)\\
\hspace{0.5cm}pred e = from (ord e - 1)\\
\end{flushleft}
}
This implies that the successor of the last enumeration value as well as the predecessor of the first enumeration value are undefined.

\hasdiff{The functions \emph{toEnum} and \emph{fromEnum} are know as \emph{from} and \emph{ord} in \frege{}. Aliases are provided for compatibility.}

A trivial type can be an instance of \texttt{Enum}.
\trans{
Let $T$ be a trivial type:
\begin{flushleft}
\textbf{data} $T$ = $C$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Enum} $T$\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Enum} $T$ \textbf{where}\\
\hspace{0.5cm}ord $C$ = 0; from 0  = $C$; succ \_ = undefined; pred \_ = undefined;\\
\hspace{0.5cm}enumFrom \_ = [$C$]; enumFromTo \_ \_ = [$C$];\\
\hspace{0.5cm}enumFromThen \_ \_ = [$C$]; enumFromThenTo \_ \_ \_ = [$C$];
\end{flushleft}
}
Note that predecessor and successor are undefined, and all arithmetic sequences result in a list with just one element, $C$.

Product types with arity $k>0$ cannot be derived instances of \texttt{Enum}. It remains to show the translation for those sum types that can be instances of \texttt{Enum}.

\trans{
Let $S$ be a sum type with $m$ $(m \ge 2)$ nullary constructors $C_1$, $\cdots$, $C_{m-1}$:
\begin{flushleft}
\textbf{data} $S$ = $C_1 | \cdots | C_m$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Enum} $S$\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Enum} $S$ \textbf{where}\\
\hspace{0.5cm}ord $e$ = \textbf{case} $e$ \textbf{of}\\
\hspace{1.0cm}$C_1 \rightarrow$ 0\\
\hspace{1.0cm}$\cdots$\\
\hspace{1.0cm}$C_m \rightarrow$ $m-1$\\
\hspace{0.5cm}from $i$  = \textbf{case} $i$ \textbf{of}\\
\hspace{1.0cm}0 $\rightarrow C_1$\\
\hspace{1.0cm}$\cdots$\\
\hspace{1.0cm}$m-1$ $\rightarrow C_m$\\
\hspace{0.5cm}enumFromTo $a$ $b$ = \textbf{if} $a \le{} b$ \textbf{then} $a$:enumFromTo $($succ $a)$ $b$ \textbf{else} \bracka{}\brackz{}\\
\hspace{0.5cm}enumFrom $a$ = enumFromTo $a$ $C_m$\\
\hspace{0.5cm}enumFromThen $a$ $b$ = enumFromThenTo $a$ $b$ (\textbf{if} $a \le{} b$ \textbf{then} $C_m$ \textbf{else} $C_1$)\\
\hspace{0.5cm}enumFromThenTo $a$ $b$ $c$ = map from \\
\hspace{3.5cm}(Int.enumFromThenTo (ord $a$) (ord $b$) (ord $c$))\\
\end{flushleft}
}
Note that the construct $m-1$ will be substituted by the appropriate integer constant. The application ($S$\texttt{.from} $i$) is undefined for $(i<0)$ or $(i \ge m)$. For all $C_i$ it is the case that $S$\texttt{.from} $C_i$\texttt{.ord ==} $C_i$

\section{Derived instances for Bounded}

This type class defines two per type constants \emph{minBound} and \emph{maxBound} and can be derived for enumeration types.
\trans{
Let $S$ be a sum type with $m$ $(m \ge 2)$ nullary constructors:
\begin{flushleft}
\textbf{data} $S$ = $C_1 | \cdots | C_m$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Bounded} $S$\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Bounded} $S$ \textbf{where}\\
\hspace{0.5cm}minBound = $C_1$\\
\hspace{0.5cm}maxBound = $C_m$\\
\end{flushleft}
}

\section{Derived instances for Show}

The type class \texttt{Show} is for types whose values can be represented as character strings. It can be derived for any algebraic data type.

\trans{
Let $S$ be a $n$-ary $(n \ge 0)$ type constructor for a type with $m$  $(m \ge 1)$ data constructors $C_1, \cdots, C_m$ and arities $k_1, \cdots, k_m$:
\begin{flushleft}
\textbf{data} $S$ $u_1$ $\cdots$ $u_n$ = $C_1$ $ct_{1_1}$ $\cdots$ $ct_{k_1}$ $| \cdots |$ $C_m$ $ct_{m_1}$ $\cdots$ $ct_{k_m}$\\
\hspace{1cm}Then\\
\textbf{derive} \texttt{Show} ($S$ $t_1$ $\cdots$ $t_n$)\\
\hspace{1cm}is equivalent to:\\
\textbf{instance} \texttt{Show} ($Show$ $t_1$, $\cdots$, $Show$ $t_n$) \sym{=>} ($S$ $t_1$ $\cdots$ $t_n$) \textbf{where}\\
\hspace{0.5cm}\texttt{show v =} \textbf{case} \texttt{v} \textbf{of}\\
\hspace{1cm}$C_1 a_1 \cdots a_{k_1} \rightarrow$ \texttt{"}$C_i$\texttt{" ++ " "}\\
\hspace{2cm}\texttt{ ++} $a_1$\texttt{.showsub ++} $\cdots$ \texttt{" " ++} $a_k$ \texttt{.showsub}\\
\hspace{1cm}$\cdots$\\
\hspace{1cm}$C_m a_1 \cdots a_{k_m} \rightarrow \cdots$\\
\hspace{0.5cm}\texttt{showsub} $C_i$ \texttt{=} \texttt{"}$C_i$\texttt{"}\hspace{\fill}for each $i$ where $k_i=0$\\
\hspace{0.5cm}\texttt{showsub} $C_i a_1 \cdots a_{k_i}$ \texttt{=} \hspace{\fill}for each $i$ where $k_i>0$\\
\hspace{2cm}\texttt{"(" ++ show (} $C_i a_1 \cdots a_{k_i}$\texttt{) ++ ")"}
\end{flushleft}
}

The derived \texttt{show} functions create a textual representation of a value that will be syntactically reminiscent of a  constructor application if the \texttt{Show} instances of the subcomponents behave likewise. The \texttt{showsub} function shows the value enclosed in parenthesis if it is more complex than just a nullary constructor.

The translation above is equally valid for product and sum types. Types that enjoy special syntactic support (list types, tuples,  and the unit type) have also special \texttt{Show} instances whose translation is omitted for brevity. Suffice it to say that these instances will reproduce the familiar textual representations, i.e. the expression \texttt{show (1,2)} will produce \texttt{"(1, 2)"} and not \texttt{"(,) 1 2"}.


\section{Eligibility Traces}