\documentclass[notes]{beamer}          % print frame + notes
%\documentclass[notes=only]{beamer}     % only notes
%\documentclass{beamer}                 % only frames

\usecolortheme{beaver}

% Some commonly used packages
% (copied mainly from the Utrecht University theme: https://www.overleaf.com/project/5c900fa3bd9930036341116a)
\usepackage{ragged2e}  % `\justifying` text
\usepackage{booktabs}  % Tables
\usepackage{tabularx}
\usepackage{tikz}      % Diagrams
\usetikzlibrary{calc, shapes, backgrounds}
\usepackage{amsmath, amssymb, amsfonts, amsthm}
\usepackage{url}       % `\url`s
\usepackage{listings}  % Code listings
\usepackage{comment}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{subfig}
\usepackage{bm}

% Mainly math commands
\newcommand{\vect}[1]{\bm{#1}}
\usepackage{amsfonts}% to get the \mathbb alphabet
\newcommand{\field}[1]{\mathbb{#1}}
\newcommand{\C}{\field{C}}
\newcommand{\R}{\field{R}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\newcommand{\argmin}{\operatornamewithlimits{argmin}}
\providecommand{\abs}[1]{\lvert#1\rvert}
\providecommand{\norm}[1]{\lVert#1\rVert}

% A variable used to exclude slides from the lecture version
\newif\iffull
%\fullfalse
\fulltrue

% Bibliography
\usepackage[uniquename=init,giveninits=true,maxcitenames=1,style=authortitle-comp,backend=bibtex]{biblatex}
\bibliography{./references}

\setbeamertemplate{section in toc}{\inserttocsectionnumber.~\inserttocsection}

%Information to be included in the title page:
\title{Machine learning foundations I}
\author{Mitko Veta}
\institute{Eindhoven University of Technology

Department of Biomedical Engineering}
\date{2020}

\begin{document}

\frame{\titlepage}

\begin{frame}
\frametitle{Learning goals}

At the end of this lecture you will:
\begin{itemize}
    \item Have an understanding of the goal of machine learning (ML) models.
    \item Have a good understanding of basic mathematical concepts used in ML and be able to apply them in the design and implementation of ML methods.
\end{itemize}
\end{frame}

\begin{frame}{Overview}
Topics covered in this lecture:
    \tableofcontents
\end{frame}

\iffull
\begin{frame}{Note on the slides}
This set of slides is larger than the one used during the lectures. It includes some additional material that you can use as a guide when studying.
\end{frame}
\fi

\section{Linear algebra}

\begin{frame}
\frametitle{Linear algebra}
Materials:
\begin{itemize}
    \item Chapter I.2 from \cite{deeplearning}
    \item \cite{linearalgebra}
\end{itemize}

\end{frame}

\iffull
\begin{frame}
\frametitle{Scalars}
\begin{itemize}
    \item A scalar is a single number (integer, real, rational, ...).
    \item Denoted by italics $a, n, x$

\end{itemize}
\end{frame}
\fi

\iffull
\begin{frame}
\frametitle{Vectors}
\begin{itemize}
    \item A vector is a 1-D array of numbers (integer, real, rational, ...)
    $
    \vect{x} = \begin{bmatrix} x_1 \\ x_2 \\ \ldots \\ x_n \end{bmatrix}
    $
    \item{Example notation for type and size} \\
    $\vect{x} \in \R^n$
 \end{itemize}
\end{frame}
\fi

\begin{frame}
\frametitle{Matrices}
\begin{itemize}
    \item A matrix is a 2-D array of numbers \\
    $$
    \begin{bmatrix}
    a_{1,1} & a_{1,2} \\
    a_{2,1} & a_{2,2}
    \end{bmatrix}
    $$
    \item Example notation for type and shape \\
    $\mathbf{A} \in \R^{m \times n}$
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Tensors}
\begin{itemize}
    \item A tensor is an array of numbers that may have
    \begin{itemize}
        \item a zero dimensions and be a scalar,
        \item one dimension and be a vector,
        \item two dimensions and be a matrix,
        \item more dimensions ...
    \end{itemize}
\end{itemize}

{\bf Side note}: One of the most popular frameworks for implementing deep machine learning models is called TensorFlow (https://www.tensorflow.org/).

\end{frame}

\begin{frame}
\frametitle{Transpose matrix}
    %\begin{itemize}
         $$(\vect{A}^T)_{i,j} = \vect{A}_{j,i}$$

        $$ \vect{A} = \begin{bmatrix}
                        A_{1,1} & A_{1,2}  \\
                        A_{2,1} & A_{2,2}  \\
                        A_{3,1} & A_{3,2}
                      \end{bmatrix} \Rightarrow
                      \begin{bmatrix}
                        A_{1,1} & A_{2,1} & A_{3,1} \\
                        A_{1,2} & A_{2,2} & A_{3,2}
                      \end{bmatrix}
                      $$
        The transpose matrix is a mirror image with regard to the main diagonal \\
        $$(\vect{A}\vect{B})^T = \vect{B}^T\vect{A}^T$$
    %\end{itemize}

\end{frame}

\iffull
\begin{frame}
\frametitle{Identity matrix}
    \begin{itemize}
        \item Identity matrix $\vect{I}_3$
        $$\begin{bmatrix}
            1 & 0 & 0 \\
            0 & 1 & 0 \\
            0 & 0 & 1
          \end{bmatrix}$$

        \item The identity matrices are neutral elements in matrix-matrix and matrix-vector multiplication, e.g.
        $$\forall \vect{x} \in \R^n: \vect{I}_n \vect{x} = \vect{x} \vect{I}_n = \vect{x} $$
    \end{itemize}

\end{frame}
\fi

\begin{frame}
\frametitle{Matrix (dot) product}

    $$\vect{C} = \vect{A}\vect{B}$$
    The matrices must be compatible: an $m \times n$ matrix is multiplied with an $n \times r$ matrix and as a result an $m \times r$ matrix is obtained
    $$C_{i,j} = \sum_k A_{i,k}B_{k,j}$$

    $$
    \vect{A}
    \begin{bmatrix}
       1 & 2 & 3 \\
       4 & 5 & 6 \\
       7 & 8 & 9 \\
       10 & 11 & 12
    \end{bmatrix}
    \times
    \vect{B}
    \begin{bmatrix}
        1 & 2 & 3 & 4 & 5 \\
        6 & 7 & 8 & 9 & 10 \\
        11 & 12 & 13 & 14 & 15
    \end{bmatrix} =
    \vect{C}
    \begin{bmatrix}
       4 \times 5
    \end{bmatrix}
    $$
    $$ C_{2,5} = A_{2,1}B_{1,5} + A_{2,2}B_{2,5} + A_{2,3}B_{3,5} = 4 \cdot 5 + 5 \cdot 10 + 6 \cdot 15 = 160 $$

\end{frame}

\begin{frame}{Matrix (dot) product}
    \begin{itemize}
        \item In general matrix multiplication is not commutative, i.e., most of the time $\vect{A}\vect{B} \not = \vect{B}\vect{A}$.
        \item Depending on the dimensions sometimes $\vect{A} \vect{B}$ or $\vect{B} \vect{A}$ are not possible.
        \item As a special case the matrix can be a (column or row) vector; an $m \times n$ matrix is multiplied with a $n \times 1$ vector to obtain a $m \times 1$ vector.
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Systems of linear equations}

$$A_{1,1}x_1 + A_{1,2}x_2 + \ldots + A_{1,n}x_n = b_1$$
$$A_{2,1}x_1 + A_{2,2}x_2 + \ldots + A_{2,n}x_n = b_2$$
$$\ldots$$
$$A_{m,1}x_1 + A_{m,2}x_2 + \ldots + A_{m,n}x_n = b_m$$
\\~\

\begin{itemize}
  \item $A_{*,*}$ and $b_*$ are the knowns, $x_*$ are the unknowns.
  \item In matrix form: $\vect{A}\vect{x} = \vect{b}$
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Systems of linear equations}
    \begin{itemize}
        \item $\vect{A}\vect{x} = \vect{b}$ expands to
            $$\vect{A}_{1,:} \vect{x}_1 = \vect{b}_1 $$
            $$\vect{A}_{2,:} \vect{x}_2 = \vect{b}_2 $$
            $$\ldots $$
            $$\vect{A}_{m,:} \vect{x}_m = \vect{b}_m $$
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Solving systems of linear equations}
    \begin{itemize}
        \item A linear system of equations can have
            \begin{itemize}
                \item no solutions,
                \item many solutions,
                \item exactly one solution.
            \end{itemize}
        \item Only one solution implies that multiplication by a matrix is an invertible operation.
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Matrix inversion}
    \begin{itemize}
        \item Matrix inverse is defined with
            $$ \vect{A}^{-1}\vect{A} = \vect{I}_n $$
        \item A system of linear equations can be solved using inverse matrix
        \begin{center}
            $$ \vect{A} \vect{x} = \vect{b} $$
            $$ \vect{A}^{-1}\vect{A} \vect{x} = \vect{A}^{-1}\vect{b} $$
            $$ \vect{I}_n \vect{x} = \vect{A}^{-1}\vect{b} $$
            $$ \vect{x} = \vect{A}^{-1}\vect{b} $$
        \end{center}
	\iffull
        \item This is useful mostly for abstract analysis.
        \item From a numerical point of view there are much more efficient methods.
        \fi
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Invertibility}
    A matrix cannot be inverted if
    \begin{itemize}
        \item the number of rows and columns is not the same, or
        \item some rows and columns are "redundant" ("linearly dependent", "low rank").
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Moore-Penrose pseudoinverse}
    \begin{itemize}
        \item Matrix inversion is not defined on matrices that are no square.
        \item The {\bf Moore-Penrose pseudoinverse} is defined as
        $$
        \vect{A}^{+} = \lim_{\alpha \searrow 0}(\vect{A}^T\vect{A}+\alpha \vect{I})^{-1}\vect{A}^T
        $$

    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Moore-Penrose pseudoinverse}
    Now we can consider
    $$\vect{x} = \vect{A}^{+} \vect{y}$$
    \begin{itemize}
        \item If the equation has
            \begin{itemize}
                \item exactly one solution: this is the same as inverse,
                \item no solution: gives the solution with the smallest error, $\norm{\vect{A}\vect{x} - \vect{y}}_2$
                \item many solutions: gives the solution with the smallest norm of $\vect{x}$.
            \end{itemize}
    \end{itemize}

\end{frame}

\iffull

\begin{frame}
\frametitle{Singular value decomposition}
    \begin{itemize}
        \item Similar to eigenvalue decomposition
        \item More general: matrix need not be square
        $$\vect{A} = \vect{U} \vect{D} \vect{V}^T$$
        \item $\vect{U}$ and $\vect{V}$ are square matrices and are both orthogonal, $\vect{D}$ is diagonal.
        \item The diagonal elements of $D$ are called {\bf singular values} of matrix $\vect{A}$; the columns of $\vect{U}$ and $\vect{V}$ are {\bf left-singular} and {\bf right-singular vectors} of $\vect{A}$, respectively.
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Computing the pseudoinverse}
    \begin{itemize}
        \item Efficient implementations are based on the formula allowed by the singular decomposition
        $$
        \vect{A}^{+} = \vect{V}\vect{D}^{+}\vect{U}^T
        $$
        \item $\vect{U},\vect{D},\vect{V}$ are from the singular value decomposition of $\vect{A}$.
        \item The pseudoinverse $\vect{D}^{+}$ of $\vect{D}$ is obtained by taking the reciprocal non-zero elements and after that taking the transpose of the resulting matrix.
    \end{itemize}

\end{frame}

\fi

\begin{frame}
\frametitle{Norms}
    \begin{itemize}
        \item Norms are functions that measure how "large" a vector is.
        \item Similar to a distance between zero and the point represented by the vector
        \begin{itemize}
           \item $f(\vect{x}) = 0 \Rightarrow \vect{x} = 0$
           \item $f(\vect{x} + \vect{y}) \leq f(\vect{x}) + f(\vect{y})$ (\bf the triangle inequality)
           \item $\forall \alpha \in \R: f(\alpha \vect{x}) = \alpha f(\vect{x})$
        \end{itemize}
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Norms}
    \begin{itemize}
        \item $L^p$- norm
        $$\norm{\vect{x}}_p = \left( \sum_i |x_i|^p  \right) ^{\frac{1}{p}}$$
        \begin{itemize}
            \item Most popular $L^2$-norm (for $p=2$)
            \item $L_1$-norm (for $p = 1$): $\norm{\vect{x}}_1 = \sum_i |x_i|$
            \item Max norm (for infinite $p$): $\norm{\vect{x}}_\infty = \max_i |x_i|$
            \end{itemize}
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Special vectors and matrices}
    \begin{itemize}
        \item Unit vector $\norm{\vect{x}}_n = 1$
        \item Symmetric matrix $\vect{A} = \vect{A}^T$
        \item Orthogonal matrix $$\vect{A}\vect{A}^T = \vect{I} = \vect{A}^T \vect{A}$$
        \item It follows that for orthogonal matrices  $\vect{A}^T = \vect{A}^{-1}$
    \end{itemize}
\end{frame}

\begin{frame}
\iffull
\frametitle{Eigendecomposition}
\else
\frametitle{Eigendecomposition (self-study)}
\fi
    \begin{itemize}
        \item Eigenvector and eigenvalue
        $$ \vect{A}\vect{x} = \lambda \vect{x}$$
        \item Eigendomdecomposition of a matrix
        $$ \vect{A} = \vect{V} \mbox{diag}(\lambda) \vect{A}^{-1}$$
        where $\mbox{diag}(\lambda)$ is a diagonal matrix having the (scalar) eigenvalues $\lambda$ as diagonal elements.
    \end{itemize}

\end{frame}

\iffull
\begin{frame}
\frametitle{Eigendecomposition}
    \begin{itemize}
        \item Every real symmetric matrix has a real orthogonal eigendecomposition
        $$ \vect{A} = \vect{Q} \vect{\Lambda} \vect{Q}^T $$
        where $\vect{Q}$ is an orthogonal matrix composed of eigenvectors of $\vect{A}$ and $\vect{\Lambda}$ is a diagonal matrix.
        \item The eigenvalue $\Lambda_{ii}$ is associated with the eigenvector in column $i$ of $\vect{Q}$, denoted as $\vect{Q}_{:,i}$.
        \item We can think of $\vect{A}$ as scaling space by factor $\lambda_i$ in the direction of its corresponding eigenvector $v^{(i)}$ (represented by $\vect{Q}_{:,i}$).
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Effect of eigenvalues}

\begin{center}
\includegraphics[width=0.48\textwidth]{../figures/week_1/unit_circle_before.pdf}
\hfill
\includegraphics[width=0.48\textwidth]{../figures/week_1/unit_circle_after.pdf}
\end{center}

\end{frame}

\begin{frame}
\frametitle{Eigendecomposition}
    \begin{itemize}
        \item From the eigendecomposition we learn useful properties of the matrix.
        \item The eigendecomposition of a real symmetric matrix is used in optimization of quadratic expressions of the form $f(\vect{x}) = \vect{x}^T \vect{A} \vect{x}$ under the constraint $\norm{\vect{x}}_2 = 1$.
        \item For instance, if $\vect{x}=\vect{v}^{(i)}$, then $f(\vect{x}) = \lambda_i$, when $\vect{v}^{(i)}$ is an eigenvector of $A$ and $\lambda_i$ is its corresponding eigenvalue.
        \item The maximal (minimal) value of $f$ within the constraint region is equal to the maximal (minimal) eigenvalue.
    \end{itemize}

\end{frame}
\fi

\iffull
\begin{frame}
\frametitle{Trace}
    \begin{itemize}
        \item A {\bf trace} of a matrix is defined as
        $$Tr(\vect{A}) = \sum_{i} \vect{A}_{i,i} $$
        \item Expressions in terms of the trace operators allow to exploit many useful identities, e.g.
        $$Tr(\vect{A}\vect{B}\vect{C}) = Tr(\vect{B}\vect{C}\vect{A}) = Tr(\vect{C}\vect{A}\vect{B})$$
    \end{itemize}

\end{frame}
\fi

\section{Gradient-based optimization}

\begin{frame}
\frametitle{Gradient-based optimization}
Materials:
\begin{itemize}
    \item Chapters I.4 and I.5 from \cite{deeplearning}
    \item \cite{linearalgebra}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Gradient}
    \begin{itemize}
        \item Let $f: \R^{m \times n} \mapsto \R$ be a function that takes $m \times n$ matrix $\vect{A}$ as input and returns a real number (scalar).
        \item A {\bf gradient} of $f$ with respect to $A$ is the matrix
        $$
        \nabla_{\vect{A}} f(\vect{A}) =
        \begin{bmatrix}
         \frac{\partial f}{\partial A_{11}} & \frac{\partial f}{\partial A_{12}} & \ldots & \frac{\partial f}{\partial A_{1n}} \\
          \frac{\partial f}{\partial A_{21}} & \frac{\partial f}{\partial A_{22}} & \ldots & \frac{\partial f}{\partial A_{2n}} \\
          \vdots & \vdots & \ddots & \vdots \\
           \frac{\partial f}{\partial A_{m1}} & \frac{\partial f}{\partial A_{m2}} & \ldots & \frac{\partial f}{\partial A_{mn}}
        \end{bmatrix}
        $$
        \item i.e. an $m \times n$ matrix with  $$(\nabla_{\vect{A}} f(\vect{A}))_{ij} = \frac{\partial f}{\partial A_{ij}}  $$
        \item The size of the gradient of $\vect{A}$ is the same as the size of $A$.

    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Gradient}
    \begin{itemize}
        \item In the special case when $A$ is a vector we obtain the (possibly more familiar) gradient
        $$ \nabla_{\vect{x}} f(\vect{x}) =
        \begin{bmatrix}
         \frac{\partial f}{\partial x_{1}}  \\
          \frac{\partial f}{\partial x_{2}}  \\
          \vdots  \\
           \frac{\partial f}{\partial x_{m}}
        \end{bmatrix}$$
        \item In general to define a gradient we require that the function returns a {\bf real} value.
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Jacobian}

\begin{itemize}
    \item The Jacobian $\vect{J_f}$ is a generalization of the gradient for vector valued functions.
    \item Let $\vect{f}:\R^n \mapsto \R^m $ be a function that takes $n$-dimensional vector $\vect{x}$ as input and returns a $m$-dimensional vector as an output.
    \item The Jacobian $\vect{J_f}$ is defined as
    $$
        \vect{J_f} =
        \begin{bmatrix}
         \frac{\partial f_1}{\partial x_{1}} & \frac{\partial f_1}{\partial x_{2}} & \ldots & \frac{\partial f_1}{\partial x_{n}} \\
          \frac{\partial f_2}{\partial x_{1}} & \frac{\partial f_2}{\partial x_{2}} & \ldots & \frac{\partial f_2}{\partial x_{n}} \\
          \vdots & \vdots & \ddots & \vdots \\
           \frac{\partial f_m}{\partial x_{1}} & \frac{\partial f_m}{\partial x_{2}} & \ldots & \frac{\partial f_m}{\partial x_{n}}
        \end{bmatrix}
    $$
    \item Note that for the special case of a scalar-valued function, the Jacobian is the transpose of the gradient.
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Optimization}

\begin{itemize}
    \item Most machine learning methods involve some kind of optimization.
    \begin{itemize}
        \item One exception is the $k$-Nearest neighbour classifier introduced later.
    \end{itemize}
    \item Optimization means minimizing or maximizing some function $f(\vect{x})$, i.e. finding the values of $\vect{x}$ for which $f(\vect{x})$ has a minimum or a maximum.
    \item Notation: $\vect{x}^* = \argmin f(\vect{x})$
\end{itemize}
\end{frame}

\begin{frame}{Gradient-based optimization}

\begin{itemize}
    \item The derivative tells us how to change $x$ in order to make a small improvement of $f(x)$.
    \item Therefore, derivatives can be useful in optimization.

\end{itemize}
\end{frame}

\section{Two simple machine learning models}

\begin{frame}
\frametitle{Two simple machine learning models}
Materials:
\begin{itemize}
    \item Chapter 2.3 from \cite{elements}
\end{itemize}

\end{frame}

\iffull
\begin{frame}
\frametitle{Some notations}
    \begin{itemize}
        \item We denote an input variable with the symbol $x$ (scalar) or $\vect{x}$ (vector).
        \item The $i$-th component of a vector input $\vect{x}$ is denoted as $x_i$.
        \item Quantitative (numerical) outputs are denoted with $y$.
        \item Qualitative outputs are denoted with $g$ (from group) and take values from a set $\cal{G}$.
        \item Matrices are denoted with bold and uppercase letters $\vect{X}$\\
        for instance, a set of $N$ input $p$-vectors $\vect{x}_i$ ($1 \leq i \leq N)$ is "packed" in a $N \times p$ input matrix $\vect{X}$.
        \item Since by default vectors are assumed to be column vectors, the rows of $\vect{X}$ are the transposes $\vect{x}_i^T$.
    \end{itemize}
\end{frame}
\fi

\begin{frame}
\frametitle{The learning task}
    \begin{itemize}
        \item Given a value of the input vector $\vect{x}$ make a good prediction of the output $y$, denoted as $\hat{y}$.
        \item Both $y$ and $\hat{y}$ should take values from the same numerical set.
        \item Similarly, $g$ and $\hat{g}$ should both take values from the same set $\cal{G}$.
         \item We suppose that we have available a set of measurements $(\vect{x}_i,y_i)$ or $(\vect{x}_i,g_i)$ $(1 \leq i \leq N)$ called {\bf training data} \\
         (in matrix form: $(\vect{X},\vect{y})$ and/or $(\vect{X},\vect{g})$).
        \item Our task is to construct a prediction rule based on the training data.

    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{The learning task}
    Example:
    \begin{itemize}
         \item {\bf Variable values}: Let $g$ (and therefore also $\hat{g}$) be two valued (categorical), e.g. $\cal{G} = \{ \text{\textcolor{blue}{BLUE}, \textcolor{orange}{ORANGE}}\}$.
        \item {\bf Encoding of $g$s with $y$s}: Then each class can be encoded binary, i.e., with $y \in \{0,1\}$, e.g., \textcolor{blue}{BLUE} and \textcolor{orange}{ORANGE}, would correspond to $0$ and $1$, respectively.
        \item {\bf Predicted output values}: $\hat{y}$ ranges over the interval $[-\infty,+\infty]$ (of which $\{0,1\}$ is a subset).
        \item {\bf Prediction rule}: $\hat{g}$ is assigned a (class label) \textcolor{blue}{BLUE} if $\hat{y} < 0.5$ and \textcolor{orange}{ORANGE}, otherwise.

    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Two simple approaches to prediction}
\begin{itemize}
    \item Linear model fit
        \begin{itemize}
            \item strong assumptions about the structure of the decision boundary
            % \item stable but possibly inaccurate predictions
        \end{itemize}
    \item $k$-nearest neighbours
        \begin{itemize}
            \item weak assumptions about the structure of the decision boundary
            % \item predictions most often accurate but possibly unstable
        \end{itemize}
\end{itemize}

\end{frame}

\subsection{Linear model}

\begin{frame}
\frametitle{Linear model fit by least squares}
    \begin{itemize}
        \item Despite relative simplicity one of the most important statistical tools
        \item Input vector $\vect{x}^T = (x_1, x_2, \ldots, x_p)$
        \item Output $y$ predicted using the model \\
            \begin{center}
            $\hat{y} = \hat{w_0} + \sum_{j=1}^{p} x_j \hat{w_j}$
            \end{center}
        \item $\hat{w}_i$ $ (0 \leq i \leq p)$ are the parameters of the linear model
        \item In vector form
            \begin{center}
            $\hat{y} = \hat{\vect{w}}^T\vect{x} = \vect{x}^T \hat{\vect{w}}$
            \end{center}
            using the fact that the scalar (inner) product of two vectors is a commutative operation.
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Linear model fit by least squares}
    \begin{itemize}
        \item  We assume that $w_0$ is in $\vect{w}$ and $1$ is included in $\vect{x}$.
        \item $\hat{y}$ is a scalar, but in general can be a $k$-vector $\hat{\vect{y}}$, in which case $\vect{w}$ becomes a $p \times k$ matrix of coefficients.
    \end{itemize}
\end{frame}

\iffull
\begin{frame}
\frametitle{Linear model fit by least squares}
    Some hyper(space) terminology:
    \begin{itemize}
        \item Points $\vect{x},\hat{y}$ form a {\bf hyperplane} in the $(p+1)$-dimensional input-output hyperspace.
        \item If $\vect{x}$ is extended with constant $1$ then the hyperplane includes the origin and it forms a {\bf subspace}.
        \item If $1$ is not included then the hyperplane is an {\bf affine} set and it cuts the $y$-axis at the point $(\vect{0}, \hat{w_0})$, where the vector $\vect{0}$ has all $x_i$ coordinates equal to $0$.
        \item Reminder: from now on we assume that $1$ is included in $\vect{x}$ and $\hat{w_0}$ in $\hat{\vect{w}}$/
        \item The function $f(\vect{x}) =  \vect{w}^T \vect{x}$ defined on the $p$-dimensional (input) space is a {\bf linear} function (we omit the hats over the $w$s since now we consider them as free variables).
        \item The gradient $\nabla f(\vect{x})$ is a vector pointing along the direction of maximal change.
    \end{itemize}
\end{frame}
\fi

\begin{frame}
\frametitle{Linear model fit by least squares}
\frametitle{}
    \begin{itemize}
        \item There are many ways to fit a linear model to a training dataset.
        \item {\bf Least squares} method
            \begin{itemize}
                \item We need to find coefficients $\hat{w_i}$ which minimize the error estimated with the {\bf residual sum of squares}
                \begin{center}
                    $$\mbox{RSS}(\vect{w}) = \sum_{i = 1}^{N}(y_i - \vect{x}_i^T \vect{w})^2$$
                \end{center}
                 assuming $N$ input-output pairs.
            \end{itemize}
        \item $\mbox{RSS}(\vect{w})$ is a quadratic function.
        \item A minimum always exists though not necessarily a unique one.
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Linear model fit by least squares}
    \begin{itemize}
        \item We look for the solution $\hat{\vect{w}}$ using the matrix notation:
        \item $\vect{y} = [y_1, y_2, \ldots, y_N]^T$ is the vector formed from the $N$ output vectors and $\vect{X}$ is an $N \times p$ matrix  \\
        % (\vect{y} - \vect{X}^T \vect{w})^2 =
        $$\mbox{RSS}(\vect{w}) =  (\vect{y} - \vect{X} \vect{w})^T (\vect{y} - \vect{X} \vect{w})$$
        \item To find the minimum we differentiate with respect to $\vect{w}$ which gives
        $$(-\vect{X})^T(\vect{y}-\vect{X}\vect{w}) + (\vect{y}-\vect{X}\vect{w})^T (-\vect{X})$$
        using the rule $(\vect{A}\vect{B})^T = \vect{B}^T\vect{A}^T$ this is equivalent to
        $$ -2 \vect{X}^T(\vect{y}-\vect{X}\vect{w})$$
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Linear model fit by least squares}
    \begin{itemize}

    \item To find the minimum our derivative must be $\vect{0}$, hence:
        $$\vect{X}^T(\vect{y}-\vect{X}\vect{w}) =  \vect{0}$$
        $$\vect{X}^T \vect{y} -\vect{X}^T\vect{X}\vect{w} =  \vect{0}$$
        $$ \vect{X}^T \vect{y} = \vect{X}^T\vect{X}\vect{w}$$
    \item    If $\vect{X}^T\vect{X}$ is non-singular there exists a unique solution given by
        $$\hat{\vect{w}} = (\vect{X}^T\vect{X})^{-1}\vect{X}^T \vect{y}$$
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Discussion point}

Why we cannot simply solve for $\hat{\vect{w}}$ in the following way?

$$\vect{y} -\vect{X}\vect{w}=  \vect{0}$$
$$\vect{y} = \vect{X}\vect{w}$$
$$\hat{\vect{w}} = \vect{X}^{-1} \vect{y}$$

\end{frame}

\begin{frame}{Linear model fit by least squares}
    \begin{itemize}
        \item For each input $\vect{x}_i$ there corresponds the fitted output $$\hat{y}_i = \hat{y}_i(\vect{x}_i) = \hat{\vect{w}}^T\vect{x}_i$$.
        \item This is called ``making a prediction'' for $\vect{x}_i$.

        \item The entire fitted surface (hyperplane) is fully characterized by the parameter vector $\hat{\vect{w}}$.

        \item After fitting the model, we can ``discard'' the training dataset.
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Example: Linear model fit by least squares}
\begin{itemize}
    \item Scatter plot (on next slide) of training data on a pair of inputs $x_1$ and $x_2$
    \item Output class variable $g$ has two values \textcolor{blue}{BLUE} and \textcolor{orange}{ORANGE}.
    \item Linear regression model fitted with the response variable $y$ coded as $0$ for \textcolor{blue}{BLUE} and $1$ for \textcolor{orange}{ORANGE}.
    \item Fitted values $\hat{y}$ converted to a fitted class variable $\hat{g}$ as
    \[ \hat{g} = \begin{cases}
                    \text{\textcolor{blue}{BLUE}}  & \text{if $\hat{y} \leq 0.5$} \\
                    \text{\textcolor{orange}{ORANGE}} & \text{if $\hat{y} > 0.5$ }
                 \end{cases} \]

\end{itemize}
\end{frame}

\begin{frame}{Example: Linear model fit by least squares}
    \begin{center}
        \includegraphics[width=0.48\textwidth]{../figures/week_1/classification_problem.pdf}
        \hfill
        \includegraphics[width=0.48\textwidth]{../figures/week_1/linear_model.pdf}
        \end{center}
\end{frame}

\iffull
\begin{frame}
\frametitle{Example: Linear model fit by least squares}
\begin{itemize}
    \item Two classes separated in the plane ($\R^2$) by the decision boundary $\{ \vect{x} : \vect{w}^T\vect{x} = 0.5 \}$
    \item $\{ \vect{x} : \vect{w}^T\vect{x} < 0.5 \}$ set of \textcolor{blue}{BLUE} points
    \item $\{ \vect{x} : \vect{w}^T\vect{x} \geq 0.5 \}$ set of \textcolor{orange}{ORANGE} points

\end{itemize}
\end{frame}
\fi

\begin{frame}
\frametitle{Example: Linear model fit by least squares}
\begin{itemize}
    \item Wrong classifications on both sides of the boundary
    \item Are the errors caused by the model or are they unavoidable?
    \item Two possible scenarios
        \begin{itemize}
            \item {\bf Scenario 1}: data generated from bivariate Gaussian distribution
            \item {\bf Scenario 2}: data generated from 10 Gaussian distributions; the means of these distributions are also distributed as Gaussian
        \end{itemize}
    \item In Scenario 1 the linear boundary is the best we can do since the overlap is inevitable.
    \item In Scenario 2 the linear boundary is unlikely to be optimal \\
          (in fact the boundary is non-linear and disjoint).
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Discussion point}
What is the expression for the Euclidean distance between two vectors (points) $\mathbf{a}$ and $\mathbf{b}$ in vector form?

\end{frame}

\subsection{Nearest-neighbours model}

\begin{frame}
\frametitle{Nearest-neighbours model}
    \begin{itemize}
        \item In nearest-neignbour methods $\hat{y}(\vect{x})$ is determined based on the inputs (points) in the training set $\cal{T}$ which are "closest" to the input $\vect{x}$.
        \item $k$-nearest neighbour fit is defined as
        $$ \hat{y}(\vect{x}) = \frac{1}{k} \sum_{\vect{x}_i \in N_k(\vect{x})} y_i $$
        where $N_k(\vect{x})$ is the neighbourhood of $\vect{x}$ consisting of the $k$ "closest" points to $\vect{x}$.
        \item "Closeness" requires a definition of {\bf metrics}.
        \item For the moment we assume Euclidian distance (each $\vect{x}$ is a point in the hyperspace).
        \item An average of the classes of the $k$ closest points (but only for binary classification problem.
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Back to the \textcolor{blue}{BLUE} and \textcolor{orange}{ORANGE} example}
    \begin{itemize}
        \item We use the same training data as in the linear model example.
        \item New borderline between the classes generated with 15-nearest-neighbour model.
        \item Since \textcolor{orange}{ORANGE} is encoded as 1 $\hat{y}$ is the proportion of \textcolor{orange}{ORANGE} points in the 15-neighbourhood.
        \item Class \textcolor{orange}{ORANGE} assigned to $\vect{x}$ if $\hat{y}(\vect{x}) > 0.5$ (majority is \textcolor{orange}{ORANGE}).
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{15-Nearest neighbour classifier}
    \begin{center}
        \includegraphics[width=0.48\textwidth]{../figures/week_1/classification_problem.pdf}
        \hfill
        \includegraphics[width=0.48\textwidth]{../figures/week_1/15-nn.pdf}
    \end{center}
\end{frame}

\begin{frame}
\frametitle{Linear classifier vs. 15-Nearest neighbour}
    \begin{center}
        \includegraphics[width=0.48\textwidth]{../figures/week_1/linear_model.pdf}
        \hfill
        \includegraphics[width=0.48\textwidth]{../figures/week_1/15-nn.pdf}
    \end{center}
\end{frame}

\begin{frame}
\frametitle{1-Nearest neighbour vs. 15-Nearest neighbour}
    \begin{center}
        \includegraphics[width=0.48\textwidth]{../figures/week_1/1-nn.pdf}
        \hfill
        \includegraphics[width=0.48\textwidth]{../figures/week_1/15-nn.pdf}
    \end{center}
\end{frame}

\iffull
\begin{frame}
\frametitle{Comparison of the techniques}
    \begin{itemize}
        \item 15-NN seems to work better than the linear classifier since fewer points are missclassified.
        \item On the other hand, {\bf none} of the points in the 1-NN case was misclassified!?
        \item Actually with the 1-NN method the error on {\bf training data} is always 0.
        \item An independent test set needed to obtain a better comparison of the methods.
    \end{itemize}
\end{frame}
\fi

\begin{frame}
\frametitle{Comparison of techniques}

    \begin{itemize}
        \item At first sight it looks like k-NN has only one parameter, $k$ versus $p$ parameters (number of weights $w_i$) of the linear model.
        \item The {\bf effective}  number of parameters of k-NN is $N/k$ which is in general bigger than $p$ ($N$ is the size of the training set).
        \item For instance, assume non-overlapping neighbourhoods
            \begin{itemize}
                \item There will be $N/k$ neighbourhoods.
                \item To each neighbourhood there correspond one parameter (the mean of the elements of the neighbourhood).
            \end{itemize}
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Discussion point}
Assume that you are building a machine learning model to be used as an \textit{aid} by clinicians for decision making. The inputs to the model are a number of \textit{biomarkers} describing the condition of the patient.
\\~\

The clinician specifies that they are interested in a model that is \textit{interpretable}, i.e. a model that will not only output a prediction but also give an indication about which biomarkers are important when making the prediction.
\\~\

You can either use a linear model or a $k$-NN classifier. What is the better choice in your opinion?

\end{frame}

\iffull
\section{Probability theory}
\else
\section{Probability theory (self-study)}
\fi

\begin{frame}
\iffull
\frametitle{Probability theory}
\else
\frametitle{Probability theory (self-study)}
\fi
Materials:
\begin{itemize}
    \item Chapter I.3 from \cite{deeplearning}
\end{itemize}
\end{frame}

\iffull

\begin{frame}
\frametitle{Probability theory}
\begin{itemize}
    \item Probability theory is a mathematical framework for dealing with uncertainty, i.e., modeling and analyzing uncertain events and statements
    \item In AI probability theory is used in two major ways:
        \begin{itemize}
            \item To design AI systems, i.e., derive models and expressions and the corresponding algorithms.
            \item To analyze the behaviour of the AI systems.
        \end{itemize}

\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Probability theory}
    \begin{itemize}
        \item A {\bf random variable} is a variable that can take values randomly.
        \iffull
        \item We will denote random variables with plain (ordinary text) typeface and their values with standard math typeface\\
        for example, if the random variable is denoted as x its values can be $x_1$ and $x_2$.
        \item A vector-valued random variable is denoted with bold typeface, e.g. {\bf x}.
        \fi
        \item On its own a random variable just denotes the set of its possible values; to get its full meaning in needs to be coupled with a distribution.
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Probability theory}
    \begin{itemize}
        \item There are two types of random variables: {\bf discrete} and {\bf continuous}.
        \item Consequently there are two ways to describe probability distributions: {\bf probability mass functions} and {\bf probability density functions}.
    \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Probability mass function}
\begin{itemize}
    \item The domain of a probability mass function $P$ is the set of all possible states of the random variable x.
    \item $\forall x \in \text{x}: 0 \leq P(\text(x) \leq 1$
        \begin{itemize}
            \item An impossible event has probability 0 and no state can be less probable than that.
            \item An event that is guaranteed to happen has probability 1 and no state can have a greater chance of occurring.
        \end{itemize}
    \item $\sum_{x \in \text{x}} P(x) = 1$
        \begin{itemize}
            \item We say that \text{x} is {\bf normalized}.
        \end{itemize}
    \item Example: Uniform distribution: $P(\text{x} = x_i) = \frac{1}{k}$.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Probability density function}
\begin{itemize}
    \item The domain of the probability density function $p$ must be the set of all possible states of $\text{x}$.
    \item $\forall x \in \text{x}: p(x)\geq 0.$
    \item $$\int p(x)dx = 1$$
    \item Example: uniform distribution $u(x;a,b) = \frac{1}{b-a}$, for $x \in [a,b]$
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Conditional probability}
    \begin{itemize}
        \item {\bf Conditional probability} is the probability of some event provided that some other event has happened.
        \item
        Given two random variables x and y, the conditional probability that y has value $y$ provided that we know that x has value $x$ is given by
        $$
        P(\text{y}=y \mid \text{x} = x) = \frac{P(\text{x,y})}{P(\text{x}=x)}
        $$
        \item Another way to see this formula is $$P(\text{x,y}) = P(\text{x} = x)P(\text{y} = y \mid \text{x} = x)$$ i.e., the probability of $x$ and $y$ occurring together is equal to the probability of occurrence of $x$ times the probability of $y$ occurring provided $x$ has occurred.
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Expectation}
\begin{itemize}
    \item The {\bf expectation} or {\bf expected} value of a function $f(x)$ with respect to a probability distribution $P(x)$ is the average value of $f$ over all values $x$ assuming they are drawn from $P$
    \item $$\field{E}_{\text{x} \sim P} [f(x)] = \sum_{x} P(x) f(x)$$
    \item $$\field{E}_{\text{x} \sim P} [f(x)] = \int p(x) f(x) dx$$
    \item Linarity of expectations:
    $$\field{E}_{\text{x}} [\alpha f(x) + \beta g(x)] =  \alpha \field{E}_\text{x}[f(x)] + \beta \field{E}_\text{x}[g(x)]$$
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Variance and covariance}
\begin{itemize}
    \item The {\bf variance} gives a measure of variation of the values of a random variable x $$\mbox{Var}(f(x)) = \field{E}[(f(x) - E[f(x)])^2]$$
    Square root of the variance is called {\bf standard deviation}.

    \item The {\bf covariance} is a measure of linear relation as well as scale between $$\mbox{Cov}(f(x),g(x)) = \field{E}[f(x) - E[(f(x)])(g(x) - E[g(x)])]$$
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Covariance matrix}
    \begin{itemize}
        \item The {\bf covariance matrix} of a random vector $\vect{x} \in \R^n$ is a $n \times n$ matrix with elements
    $$
    \mbox{Cov}(\vect{x})_{i,j} = \mbox{Cov}(x_i,x_j)
    $$
    \item The diagonal elements of the matrix give the variance
    $$
      \mbox{Cov}(\text{x}_i,\text{x}_i) = \mbox{Var}(\text{x}_i)
    $$
    \end{itemize}

\end{frame}

\begin{frame}
\frametitle{Bernouli Distribution}
\begin{itemize}
    \item A distribution over a single binary random variable
    \item Controlled by a single parameter $\phi \in [0,1]$ which corresponds to the probability of the random variable taking the value 1
    \item Properties:
    \begin{eqnarray*}
    P({\rm x}) = 1) = \phi \\
    P({\rm x} = 0) = 1 - \phi \\
    P({\rm x} = x) = \phi^x (1-\phi)^{1-x} \\
    \field{E}_\text{x}[\text{x}] = \phi \\
    Var(\text{x}) = \phi(1-\phi)
    \end{eqnarray*}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Gaussian distribution}
\begin{itemize}
    \item The most commonly used distribution, also called {\bf normal distribution}.
    \item Controlled by two parameters $\mu \in \R$ (the {\bf mean}) and $\sigma \in (0, \infty)$, (the {\bf standard deviation})

    $$
    {\cal N}(x ; \mu, \sigma^2) = \sqrt{\frac{1}{2 \pi \sigma^2}} \mbox{exp} \left (-\frac{1}{2 \sigma^2}(x - \mu)^2 \right )
    $$

    $$
    {\cal N}(\vect{x} ; \vect{\mu}, \vect{\Sigma}) = \sqrt{\frac{1}{(2 \pi)^n \mbox{det}(\Sigma)}} \mbox{exp} \left (-\frac{1}{2} (\vect{x} - \vect{\mu})^T \vect{\Sigma}^{-1} (\vect{x} - \vect{\mu}) \right )
    $$

\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Gaussian distribution}
\begin{center}
\includegraphics[width=0.48\textwidth]{../figures/week_1/gaussian_1d.pdf}
\hfill
\includegraphics[width=0.48\textwidth]{../figures/week_1/gaussian_2d.pdf}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Logistic sigmoid}
\begin{itemize}
    \item A useful function that we are going to consider
    $$\sigma(x) = \frac{1}{1 + \exp{(-x)}}$$
    \item The Logistic (sigmoid) function is commonly used to parametrize Bernoulli distributions.
    \begin{center}
    \includegraphics[width=0.48\textwidth]{../figures/week_1/sigmoid.pdf}
    \end{center}

\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Bayes' rule}
\begin{itemize}
    \item Suppose know $P(\text{y} \mid \text{x})$, but we actually need $P(\text{x} \mid \text{y})$. If we know $P(\text{x})$ then we can compute
    $$
        P(\text{x} \mid \text{y}) = \frac{P(\text{y} \mid \text{x})P(\text{x})}{P(\text{y})}
    $$
    Although it appears in the formula prior knowledge $P(\text{y}$ is not needed since usually it can be computed as $\sum_{\text{x}} P(\text{y} \mid \text{x})P(\text{x})$
    \item It can be straightforwardly derived from the conditional probability formula.
    \item It could have be named also after Laplace who independently found it, generalized it, and introduced it in practice.
\end{itemize}
\end{frame}

\fi % probability theory

\begin{frame}
\frametitle{Acknowledgements}

The slides for this lecture were prepared by Mitko Veta and Dragan Bo{\v s}nacki.

Some of the slides are based on the accompanying lectures of \cite{deeplearning}.

\end{frame}

\begin{frame}
\frametitle{References}
\printbibliography
\end{frame}

\end{document}


\subsection{Polarization of Gravitational waves}

Gravitational waves can also be polarized. Since they are three dimensional waves their polarization can be restricted to two forms where the the amplitude tensor $A^{\mu\nu}$ has two forms $A^{\mu\nu}_{+}$ and $A^{\mu\nu}_{\times}$ which are orthogonal to each other \cite{Dirkes_2018}. They can be represented as

\begin{equation}
    A^{\mu\nu}_{+} = h_{+}\, \varepsilon^{\mu\nu}_{+}
\end{equation}

\begin{equation}
    A^{\mu\nu}_{\times} = h_{\times} \,\varepsilon^{\mu\nu}_{\times}
\end{equation}

\noindent where $\varepsilon^{\mu\nu}_{+}$ and $\varepsilon^{\mu\nu}_{\times}$ are unit polarization tensors.

\begin{equation}
\varepsilon^{\mu\nu}_{+} =
\begin{bmatrix}
0 & 0 & 0 & 0 \\
0 & +1 & 0 & 0 \\
0 & 0 & -1 & 0 \\
0 & 0 & 0 & 0 \\
\end{bmatrix}
\end{equation}
\\
\begin{equation}
\varepsilon^{\mu\nu}_{\times} =
\begin{bmatrix}
0 & 0 & 0 & 0 \\
0 & 0 & +1 & 0 \\
0 & +1 & 0 & 0 \\
0 & 0 & 0 & 0 \\
\end{bmatrix}
\end{equation}

\noindent In general relativity any tensor with indices $\mu\nu$ is a rank 2 tensor with 4 rows and 4 columns where each index can take values of space time coordinates which are $(t,x,y,z)$ , and position of each element is associated with any two coordinates. Thus in such tensors, the positions of elements are associated with space-time as follows:

\begin{equation*}
\mu\nu =
    \begin{bmatrix}
    tt & tx & ty & tz \\
    xt & xx & xy & xz \\
    yt & yx & yy & yz \\
    zt & zx & zy & zz \\
    \end{bmatrix}
\end{equation*}

\noindent So when we compare the unit polarization tensors $\varepsilon^{\mu\nu}_{+}$ and $\varepsilon^{\mu\nu}_{\times}$ with the above one, we see that in $\varepsilon^{\mu\nu}_{+}$ the non zero entries are +1 in $`xx$' direction and -1 in $`yy$' direction, hence the $A^{\mu\nu}_{+}$ amplitude is oriented only along X and Y axes, thus this gravitational wave which oscillates along X and Y axes is called as `PLUS' polarized wave because the vibration resembles `+' symbol. But in $\varepsilon^{\mu\nu}_{\times}$ the non zero entries are +1 in $`xy$' direction and -1 in $`yx$' direction, hence the $A^{\mu\nu}_{+}$ amplitude is oriented in the `XY' plane at a an angle of 45$\degree$ to the axes, thus this gravitational wave which oscillates in the `XY' plane at a an angle of 45$\degree$ to the axes is called as `CROSS' polarized wave because the vibration resembles `$\times$' symbol. So the equation of polarized gravitational waves are:-\\

\hspace{0.7cm} (+) wave $\Rightarrow $  $\tilde{h}_{\mu\nu} = h_{+}\, \varepsilon^{\mu\nu}_{+}\, e^{i(\omega t - k_{z}z)}$ \hspace{2mm} and \hspace{2mm} $(\times)$ wave $\Rightarrow $  $\tilde{h}_{\mu\nu} = h_{\times}\, \varepsilon^{\mu\nu}_{\times}\, e^{i(\omega t - k_{z}z)}$
\\

\noindent Here the position variable is just `$z$', assuming that the wave is travelling along z axis and space-time is oscillating in the X-Y plane, which helps us to visualize polarized GWs easily.

\begin{figure}[h]
    \centering
    \includegraphics[height=4.3cm, width = 9.5cm]{images.tex/polarization_simulation.jpeg}
    \caption{Simulation of Polarized Gravitational waves. Source:- \href{https://sudonull.com/post/7567-Einstein-Telescope-a-new-generation-gravitational-wave-detector}{Sudonull.com}}
\end{figure}

\pagebreak

gw-physics-and-ligo/3 Properties.tex/3.1 Polarization of gw.tex


\subsection{Convex Bottom}
\subsubsection{Description of the Algorithm}
Convex Bottom Algorithm uses the Convex Hull principle to create a convex
polygon. The algorithm starts from a random 2d point cloud.
\fref{cb:base}.
\\[12pt]
Following the description of the algorithm in detail:

\begin{enumerate}
  \item sort all points along the x-axis.
  \item choose the lowest and greatest point on x-axis and draw a line between them\fref{cb:line}
  \item create a list of all points below or on this line \fref{cb:below-line}
  \item create a convex hull from those points \fref{cb:hull}
  \item add points not on the convex bottom hull to the polygon \fref{cb:polygon-1}
  \item add points from the convex bottom hull in reverse order to the polygon
    \fref{cb:polygon-2}
\end{enumerate}

\subsubsection{Implementation description}
\begin{enumerate}
  \item the function std::sort from the std lib does the sorting
  \item the sorted point list gives as the possibility to use the first point
    and the last point as the point with the lowest and the highest x value
  \item the CGAL::orientation function in combination with a for loop checks if
    the point lies below the formed line
  \item the CGAL::convex\_hull\_2 creates the bottom convex hull
  \item add all points which does not lie on the convex bottom hull to the polygon
  \item with std::reverse and a for loop the points from the convex bottom hull
    were added to the final polygon
\end{enumerate}

\subsubsection{Complexity}
\begin{enumerate}
  \item sort all points along the x-axis. $\bigO(nlogn)$
  \item choose the lowest and greatest point on x-axis and draw a line between them $\bigO(1)$
  \item create a list of all points below or on this line $\bigO(n)$
  \item create a convex hull from those points $\bigO(nlogn)$
  \item add points not on the convex bottom hull to the polygon $\bigO(n)$
  \item add points from the convex bottom hull in reverse order to the polygon $\bigO(n)$
\end{enumerate}
In summarize this leads to $\bigO(nlogn)$.

\subsubsection{Parameters}
\begin{description}
  \item [--nodes] how many nodes the polygon have to have. [default: 100]
  \item [--sampling-grid] the area within the polygon could grow. [default: 1500x800]
\end{description}

\subsubsection{Examples}

\begin{figure}[ht]
  \centering

  \begin{minipage}[t]{0.4\textwidth}
    \begin{tikzpicture}[yscale=0.05,xscale=0.05]

      \setcounter{i}{0}

      \draw[->] (0,0) -- (80,0) node[below] {$x$};
      \draw[->] (0,0) -- (0,70) node[left] {$y$};

      \foreach \p in {(1,11),(5,40),(10,20),(21,10),(20,18),(20,41),(33,15),(39,22),(48,9),(55,17),(60,50),(68,19)} {
        \node[point] (\arabic{i}) at \p {};
        \stepcounter{i}
      }
    \end{tikzpicture}
    \caption{Base random point cloud}
    \label{fig:cb:base}
  \end{minipage}\hfill
  \begin{minipage}[t]{0.4\textwidth}
    \begin{tikzpicture}[yscale=0.05,xscale=0.05]

      \setcounter{i}{1}

      \draw[->] (0,0) -- (80,0) node[below] {$x$};
      \draw[->] (0,0) -- (0,70) node[left] {$y$};

      \foreach \p in {(1,11),(5,40),(10,20),(21,10),(20,18),(20,41),(33,15),(39,22),(48,9),(55,17),(60,50),(68,19)} {
        \node[point] (\arabic{i}) at \p {};
        \stepcounter{i}
      }

      \draw (1) -- (12);
    \end{tikzpicture}
    \caption{Line from lowest x to highest x}
    \label{fig:cb:line}
  \end{minipage}
\end{figure}

\begin{figure}[ht]
  \centering
  \begin{minipage}[t]{0.4\textwidth}
    \begin{tikzpicture}[yscale=0.05,xscale=0.05]

      \draw[->] (0,0) -- (80,0) node[below] {$x$};
      \draw[->] (0,0) -- (0,70) node[left] {$y$};

      \node[point, fill=red] (1) at (1,11) {};
      \node[point] (2) at (5,40) {};
      \node[point] (3) at (10,20) {};
      \node[point, fill=red] (4) at (21,10) {};
      \node[point] (5) at (20,18) {};
      \node[point] (6) at (20,41) {};
      \node[point] (7) at (33,15) {};
      \node[point] (8) at (39,22) {};
      \node[point, fill=red] (9) at (48,9) {};
      \node[point, fill=red] (10) at (55,17) {};
      \node[point] (11) at (60,50) {};
      \node[point, fill=red] (12) at (68,19) {};

      \draw (1) -- (12);
    \end{tikzpicture}
    \caption{Points on and below of the line}
    \label{fig:cb:below-line}
  \end{minipage}\hfill
  \begin{minipage}[t]{0.4\textwidth}
    \centering
    \begin{tikzpicture}[yscale=0.05,xscale=0.05]

      \setcounter{i}{1}

      \draw[->] (0,0) -- (80,0) node[below] {$x$};
      \draw[->] (0,0) -- (0,70) node[left] {$y$};

      \foreach \p in {(1,11),(5,40),(10,20),(21,10),(20,18),(20,41),(33,15),(39,22),(48,9),(55,17),(60,50),(68,19)} {
        \node[point] (\arabic{i}) at \p {};
        \stepcounter{i}
      }

      \draw (1) -- (12) -- (9) -- (4) -- (1);
    \end{tikzpicture}
    \caption{Convex bottom hull}
    \label{fig:cb:hull}
  \end{minipage}
\end{figure}

\begin{figure}[ht]
  \centering
  \begin{minipage}[t]{0.4\textwidth}
    \begin{tikzpicture}[yscale=0.05,xscale=0.05]
      \setcounter{i}{1}

      \draw[->] (0,0) -- (80,0) node[below] {$x$};
      \draw[->] (0,0) -- (0,70) node[left] {$y$};

      \foreach \p in {(1,11),(5,40),(10,20),(21,10),(20,18),(20,41),(33,15),(39,22),(48,9),(55,17),(60,50),(68,19)} {
        \node[point] (\arabic{i}) at \p {};
        \stepcounter{i}
      }

      \draw (1) -- (2) -- (3) -- (5) -- (6) -- (7) -- (8) -- (10) -- (11) -- (12);
    \end{tikzpicture}
    \caption{Polygon from the sorted values}
    \label{fig:cb:polygon-1}
  \end{minipage}\hfill
  \begin{minipage}[t]{0.4\textwidth}
    \centering
    \begin{tikzpicture}[yscale=0.05,xscale=0.05]

      \setcounter{i}{1}

      \draw[->] (0,0) -- (80,0) node[below] {$x$};
      \draw[->] (0,0) -- (0,70) node[left] {$y$};

      \foreach \p in {(1,11),(5,40),(10,20),(21,10),(20,18),(20,41),(33,15),(39,22),(48,9),(55,17),(60,50),(68,19)} {
        \node[point] (\arabic{i}) at \p {};
        \stepcounter{i}
      }
      \draw (1) -- (2) -- (3) -- (5) -- (6) -- (7) -- (8) -- (10) -- (11) -- (12) -- (9) -- (4) -- (1);
    \end{tikzpicture}
    \caption{Polygon from above combined with polygon from below}
    \label{fig:cb:polygon-2}
  \end{minipage}
\end{figure}

\FloatBarrier


\documentclass[12pt,a4paper]{article}

\usepackage{tikz}
\usetikzlibrary{automata, positioning, arrows}

\begin{document}
\title{Automata}
\maketitle

\section{Introduction}

\begin{equation}
    \sum = \{0,1\}
\end{equation}

\begin{equation}
    L(M_1) = \{\omega \in \sum^\ast | \omega .endsWith(00)\}
\end{equation}
\begin{tikzpicture}[shorten >=1pt,node distance=2cm,on grid,auto]
   \node[state, initial] (q_0) {$q_0$};
   \node[state, right=of q_0] (q_1) {$q_1$};
   \node[state, accepting, right=of q_1] (q_2) {$q_2$};
   \path[->]
    (q_0) edge [bend left, above] node {0} (q_1)
          edge [loop above] node {1} ()
    (q_1) edge [bend left, above] node {0} (q_2)
          edge [bend left, below] node {1} (q_0)
    (q_2) edge [loop above] node {0} ()
          edge [bend left, below] node {1} (q_0);
\end{tikzpicture}

\begin{equation}
    L(M_2) = \{\omega \in \sum^\ast | \omega .contains(000)\}
\end{equation}
\begin{tikzpicture}[shorten >=1pt,node distance=2cm,on grid,auto]
    \node[state, initial] (q_0) {$q_0$};
    \node[state, right=of q_0] (q_1) {$q_1$};
    \node[state, right=of q_1] (q_2) {$q_2$};
    \node[state, accepting, right=of q_2] (q_3) {$q_3$};
    \path[->]
        (q_0) edge [bend left, above] node {0} (q_1)
            edge [loop above] node {1} ()
        (q_1) edge [bend left, above] node {0} (q_2)
            edge [bend left, below] node {1} (q_0)
        (q_2) edge [bend left, above] node {0} (q_3)
            edge [bend left, below] node {1} (q_0)
        (q_3) edge [loop above] node {0,1} ();
 \end{tikzpicture}

\end{document}


%Fourier Transform and Circulant Mass Matrices - M. D. Sheppeard 2007
%
%Copyright (c) 2001 The American Physical Society.
% This is a template for producing manuscripts for use with REVTEX 4.0
% For Phys. Rev. appearance, change preprint to twocolumn.
% Choose pra, prb, prc, prd, pre, prl, prstab, or rmp for journal
% Add 'draft' option to mark overfull boxes with black boxes
% Add 'showpacs' option to make PACS codes appear
% Add 'showkeys' option to make keywords appear
%----------------------------------------------------------------
\documentclass[aps,prl,onecolumn,showpacs,address,11pt]{revtex4}
%\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}

\pagestyle{empty}

\begin{document}

\title{Mass matrix transforms in qubit field theory}

\author{M. D. Sheppeard}
\affiliation{Department of Physics and Astronomy \\ University of
Canterbury, Christchurch, New Zealand}

\begin{abstract}
Circulant mass matrices for triples of charged and neutral leptons
have been studied in the context of qubit quantum field theory.
This note describes the discrete Fourier transform behind such
matrices, and discusses a category theoretic interpretation of
these operators.
\end{abstract}

\pacs{03.67.-a, 03.67.Lx, 04.60.-m}

\maketitle
\thispagestyle{empty}

%----------------------------------------------------------------
\section{Introduction}

Using a measurement algebra approach to QFT, Brannen
\cite{Brannen} recently recovered the Koide \cite{Koide1} formula
\begin{equation} (\sqrt{m_{e}} + \sqrt{m_{\mu}} + \sqrt{m_{\tau}})^{2} = \frac{3}{2}
(m_{e} + m_{\mu} + m_{\tau}) \end{equation} for charged lepton
masses in the form of a $3 \times 3$ circulant complex matrix,
whose eigenvalues squared give the lepton masses to experimental
precision. This analysis was extended to a set of three neutrinos,
and the mass ratio predictions agree with preliminary neutrino
oscillation data.

Here it is observed that the discrete Fourier transform
\cite{AdrovBook} provides a further interpretation of the mass
matrices, both as a duality between operators and eigenvalues and
also as a link to the theory of quantum computation
\cite{NielsenChuang}.

It is expected that other triples of Standard Model particles,
namely baryons and mesons, will also be associated with $3 \times
3$ matrix operators of the same kind in accord with their preon
structure \cite{Brannen} and the association of spatial directions
to the number of particle generations, given by the three
primitive idempotents of the measurement algebra.

%----------------------------------------------------------------
\section{Fourier transforms and mass matrices}

A {\em circulant} matrix is built from its first row by adding
cyclic permutations. In particular, a $3 \times 3$ circulant takes
the form \begin{equation} \left( \begin{array}{ccc} A & B & C \\ C & A & B \\
B & C & A \end{array} \right) \end{equation} where $A$, $B$ and
$C$ will be complex numbers. Note that any such circulant is a
combination of the three permutations $(123)$, $(231)$ and
$(312)$. For real eigenvalues $\lambda_k$ it is essential that $A$
be real and $C = \overline{B}$. Thus a mass matrix \cite{Brannen}
takes the form
\begin{equation}\label{BrannenMat} C = \eta
\left( \begin{array}{ccc} 1 & r e^{i \theta}
& r e^{- i \theta} \\ r e^{- i \theta} & 1 & r e^{i \theta} \\ r
e^{i \theta} & r e^{- i \theta} & 1 \end{array} \right)
\end{equation} for real $\eta$, $r$ and $\theta$. In terms of
these parameters, the eigenvalues are given by
\[ \lambda_{k} = \eta (1 + 2r \textrm{cos}(\theta + \frac{2 \pi k}{3}))
\] The Koide formula (1) follows when $r^2 = \frac{1}{2}$ and this
choice may be applied also to the neutrino matrix.

In the $n \times n$ case, the discrete Fourier transform
\cite{AdrovBook}\cite{NielsenChuang} interchanges the set of
eigenvalues $\lambda_{k}$ (assumed distinct) and matrix entries
$A_1 , A_2, A_3 , \cdots , A_n$ via
\begin{eqnarray} \lambda_k = \sum_{j} e^{\frac{2 \pi i jk}{n}}
A_{j} \\ \nonumber A_j = \frac{1}{n} \sum_{k} e^{- \frac{2 \pi i
jk}{n}} \lambda_{k} \end{eqnarray} Viewing the eigenvalues as a
diagonal matrix, the transform interchanges the bases of
projection operators and cyclic permutations. For real eigenvalues
$(m_1 , m_2 , m_3)$ with $m_i = \lambda_{i}^{2}$ in the above, and
letting $\omega = e^{\frac{2 \pi i}{3}}$, the transform takes the
diagonal matrix to the circulant {\small
\[ \left( \begin{array}{ccc} m_1 + m_2 + m_3
& m_1 \omega + m_2 \omega^{2} + m_3 & m_1 \omega^{2} + m_2 \omega
+ m_3 \\ m_1 \omega^{2} + m_2 \omega
+ m_3 & m_1 + m_2 + m_3 & m_1 \omega + m_2 \omega^{2} + m_3 \\
m_1 \omega + m_2 \omega^{2} + m_3 & m_1 \omega^{2} + m_2 \omega +
m_3 & m_1 + m_2 + m_3
\end{array} \right) \] } which must be the square of
(\ref{BrannenMat}) since the square of a circulant is a circulant.
Thus a choice of scale is specified by $\eta = \frac{1}{3} (m_1 +
m_2 + m_3)$.

%Products and Quantum Torus
A $3 \times 3$ matrix is viewed as a function on the discrete
torus $\mathbb{Z}_{3} \times \mathbb{Z}_{3}$, which has a quantum
description and a convolution product for matrices
\cite{AdrovBook}. Letting $D_{ij} = \delta_{ij} \omega^{i}$ there is a Weyl rule \[ D \circ (312) = \omega (312)
\circ D \] where the phase $\frac{2 \pi}{3}$ is proportional to
$\hbar^{-1}$. This associates Planck's constant with a hierarchy
$\mathbb{N}$ determined by the size of the matrix, but the
continuum limit is obtained via $\hbar \rightarrow \infty$ rather
than $\hbar \rightarrow 0$.

If masses are to be thought of as quantum numbers, then why are
their values so awkward in comparison to, say, spin? For $2 \times
2$ circulants with entries $A$ and $B$, the eigenvectors are
$(1,1)$ and $(1,-1)$ with eigenvalues $(A + B)$ and $(A - B)$
respectively. For example, for the Pauli swap matrix $\sigma_{x}$,
with $A = 0$, the spin eigenvalues are $\pm 1$. Complexity in the
eigenvalue set only arises in dimension three or higher.

Degenerate eigenvalues $\frac{\lambda_{k}}{\eta} \in \{ 1 - r , 1
- r , 1 + 2r \}$ occur when $\theta = 0$ and all matrix entries
are real. Although this pattern does not describe the leptons, we
observe that it is the typical composition of masses for baryon
constituents. Since such mass operators arise in a preon model
that unifies particle structure, it is expected that all standard
model bound states and resonances may be arranged into mass
triples.

In quantum computation \cite{NielsenChuang} a Fourier transform is
also defined in this way, acting on a set of $n$ basis states. For example, an
$N$ qubit computation uses $n = 2^{N}$ basis states. The transform is
unitary and it may be built from unitary gates, namely the
Hadamard gate $H = \frac{1}{\sqrt{2}} (\sigma_{x} + \sigma_{z})$
and the series
\[ B_{k} = \left( \begin{array}{cc} 1 & 0 \\ 0 & e^{\frac{2 \pi i}{2^{k}}}
\end{array} \right) \] By analogy, a mass computation with $3^{N}$
basis states uses ternary digits, so the gates $B_{k}$ would be
replaced by gates \begin{equation} T_{k} = \left( \begin{array}{ccc} 1 & 0 & 0 \\
0 & e^{\frac{2 \pi i}{3^{k}}} & 0 \\ 0 & 0 & e^{\frac{4 \pi
i}{3^{k}}} \end{array} \right) \end{equation} which are also
unitary. In general, the Fourier operator entries $F_{ij}$ are given by $\omega^{ij}$, and the theory of {\em mutually unbiased bases} generalises the Pauli operator algebra in all prime power dimensions.

A basic time evolution operator exists for each dimension $n$. Note, however, that unlike in conventional constructions, this local evolution is not in any way associated with an emergent cosmic clock, the latter being more closely related to the scale $\hbar$, given here by the matrix dimension. That is, this approach does not assume a globally defined time for a nonsensical universal
observer.

%----------------------------------------------------------------
\section{Discussion}

The mass matrices arise from a one dimensional discrete transform,
which itself involves commutative variables. However, it is seen
that phase space variables satisfy the Weyl algebra of the quantum
plane. Is there a noncommutative transform that extends this
analysis to nonclassical underlying spaces? This is relevant to
the question of extending the perturbative rest mass computations
\cite{Brannen} to nonperturbative regimes.

Kapranov \cite{KapranovFT} has recently considered path spaces
approximated by cubical paths, each of which is represented by a
noncommutative monomial in the spatial directions. In dimension $d
> 1$ a noncommutative Fourier transform relates measures on the
space of paths to functions of the noncommuting variables. The
basic idea is that a path integral is just a map from a
noncommutative ring to a suitable commutative subring. In this
way, particle masses \cite{Brannen} could arise as path integral
invariants.

Taking T-duality seriously, one also expects to deal with
nonassociativity. From a category theoretic point of view, both
noncommutative and nonassociative structures can be dealt with in
a unified framework. The cohomological element of interest here is
the parity cube axiom, which describes the now familiar pentagon
law on five of its faces. In a sufficiently lax algebraic setting,
such as for tetracategories, the sixth face may break this law,
providing the deformation parameter that turns a pentagon into a
hexagon representing the permutation group $S_3$
\cite{BataninComb}.

The generation count by primitive idempotents \cite{Brannen} is
confirmed by the string theoretic index theorem argument applied
to the Riemann moduli space of the six punctured sphere, which has
an orbifold Euler characteristic \cite{MulasePenk} of -6. The six
punctures are associated to the six faces of a cube via a dual
vertex, which is thickened to a sphere. Note that cohomological
integrals for such moduli spaces commonly appear in QFT
computations as multiple zeta values and polylogarithms.

%----------------------------------------------------------------
\begin{acknowledgments} For helpful discussions I thank Carl
Brannen, Michael Rios, Matti Pitkanen, Tony Smith and Louise
Riofrio. \end{acknowledgments}

\begin{thebibliography}{7}
\expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi
\expandafter\ifx\csname bibnamefont\endcsname\relax
 \def\bibnamefont#1{#1}\fi
\expandafter\ifx\csname bibfnamefont\endcsname\relax
 \def\bibfnamefont#1{#1}\fi
\expandafter\ifx\csname citenamefont\endcsname\relax
 \def\citenamefont#1{#1}\fi
\expandafter\ifx\csname url\endcsname\relax
 \def\url#1{\texttt{#1}}\fi
\expandafter\ifx\csname urlprefix\endcsname\relax\def\urlprefix{URL }\fi
\providecommand{\bibinfo}[2]{#2}
\providecommand{\eprint}[2][]{\url{#2}}

\bibitem[{\citenamefont{Brannen}()}]{Brannen}
\bibinfo{author}{\bibfnamefont{C.~A.} \bibnamefont{Brannen}},
 \bibinfo{note}{http://brannenworks.com/dmaa.pdf}.

\bibitem[{\citenamefont{Koide}(1982)}]{Koide1}
\bibinfo{author}{\bibfnamefont{Y.}~\bibnamefont{Koide}},
 \bibinfo{journal}{Lett. Nuovo Cim.} \textbf{\bibinfo{volume}{34}},
 \bibinfo{pages}{201} (\bibinfo{year}{1982}).

\bibitem[{\citenamefont{Aldrovandi}(2001)}]{AdrovBook}
\bibinfo{author}{\bibfnamefont{R.}~\bibnamefont{Aldrovandi}},
 \emph{\bibinfo{title}{Special matrices of mathematical physics}}
 (\bibinfo{publisher}{World Scientific}, \bibinfo{year}{2001}).

\bibitem[{\citenamefont{Nielsen and Chuang}(2000)}]{NielsenChuang}
\bibinfo{author}{\bibfnamefont{M.~A.} \bibnamefont{Nielsen}} \bibnamefont{and}
 \bibinfo{author}{\bibfnamefont{I.~L.} \bibnamefont{Chuang}},
 \emph{\bibinfo{title}{Quantum computation and quantum information}}
 (\bibinfo{publisher}{Cambridge}, \bibinfo{year}{2000}).

\bibitem[{\citenamefont{Kapranov}()}]{KapranovFT}
\bibinfo{author}{\bibfnamefont{M.}~\bibnamefont{Kapranov}},
 \bibinfo{note}{math.QA/0612411}.

\bibitem[{\citenamefont{Batanin}()}]{BataninComb}
\bibinfo{author}{\bibfnamefont{M.}~\bibnamefont{Batanin}},
 \bibinfo{note}{math.CT/0301221}.

\bibitem[{\citenamefont{Mulase and Penkava}()}]{MulasePenk}
\bibinfo{author}{\bibfnamefont{M.}~\bibnamefont{Mulase}} \bibnamefont{and}
 \bibinfo{author}{\bibfnamefont{M.}~\bibnamefont{Penkava}},
 \bibinfo{note}{math-ph/9811024}.

\end{thebibliography}

\end{document}


%!TEX root = ../thesis.tex
% ******************************* Thesis Appendix A ****************************

\chapter{Sampling Algorithms}
\label{chapter:appendix_a}

\par
The rejection sampling algorithm presented in Algorithm
\ref{alg:harsha_rej_sampling} is due to \cite{harsha2007communication}.

\begin{algorithm}
  \caption{Rejection sampling presented in \cite{harsha2007communication}.}
  \label{alg:harsha_rej_sampling}
  \begin{algorithmic}[1]
    \Procedure{Rej-Sampler}{$P, Q, \langle x_i \sim Q \mid i \in \Nats \rangle$}
    \Comment $P$ is the prior
    \Statex
    \Comment $Q$ is the posterior
    \Statex
    \Comment $x_i$ are i.i.d. samples from $Q$
    \State $p_0(x) \gets 0 \quad \forall x \in \X$.
    \State $p_0^* \gets 0$.
    \For{$i \gets 1, \hdots \infty$}

    \State
    $\alpha_i(x) \gets \min{P(x) - p_{i - 1}(x), (1 - p_{i - 1}^*)Q(x)}\quad
    \forall x \in \X$

    \State $p_i(x) \gets p_{i - 1}(x) + \alpha_i(x)$

    \State $p_i^* \gets \sum_{x \in \X}p_i(x)$

    \State $\beta_i(x_i) \gets \frac{\alpha_i(x)}{(1 - p_i^*)Q(x)}$

    \State Draw $u \sim \Unif{0, 1}$

    \Statex

    \If{$u < \beta_i(x_i)$}

    \State\Return $i, x_i$

    \EndIf

    \EndFor
    \EndProcedure
  \end{algorithmic}
\end{algorithm}

\par The importance sampling algorihtm presented in Algorithm
\ref{alg:miracle_imp_samp} is due to \cite{havasi2018minimal}.

\begin{algorithm}
  \caption{Importance sampling algorithm proposed by \cite{havasi2018minimal}}
  \label{alg:miracle_imp_samp}
  \begin{algorithmic}
    \Procedure{Importance-Sampler}{$P, Q, \langle x_i \sim Q \mid i \in \Nats \rangle$}
    \Comment $P$ is the prior
    \Statex
    \Comment $Q$ is the posterior
    \Statex
    \Comment $x_i$ are i.i.d. samples from $Q$

    \State $K \gets \exp\{\KL{Q}{P}\}$

    \State $\tilde{w}_i \gets \frac{Q(x_i)}{P(x_i)} \quad \forall i =
    1,\hdots K$

    \State Sample $j \sim p(\tilde{w})$

    \Return $j, x_j$
    \Statex
    \EndProcedure
  \end{algorithmic}
\end{algorithm}


\documentclass{article}
\usepackage[utf8]{inputenc}

\usepackage{amsthm,amsmath,amssymb}

\newtheorem{theorem}{Theorem}[section]

\title{A Simpler Method for Finding Expected Value in Camel Up}
\author{kyjeckland }
\date{November 2021}

\begin{document}
\maketitle

\section{A New Formulation}
 Generally, the expected value of a discrete random variable $X$ is defined as $E[X] = \sum_{k = 1}^{r}kp(X=k)$, where $p(X = k)$ is the probability that $X = k$, and $r$ is the furthest distance a camel could move on that turn. If we were to try to apply this method to the expected value of a camel moving $k$ spaces in a given leg, we would have to first have to find each way the camel could move a distance of $k$, which can be difficult. This method is tedious and prone to errors, so instead we'll use a method that paints the same picture with more efficient brush strokes. To do this, we will switch our perspective: rather than looking at ways a camel can move a given distance, we will look at the number of ways a camel moves a given number of times within a leg. We'll denote the random variable that represents the number of times a camel moves in a given turn as $Z$. Then, the expected number of times a camel move is $\sum_{z=1}^{n}zp(z)$, where $n$ is the total number of times a camel can move in a turn. Notice that each time a camel moves requires a dice roll, and we know that the expected value of a single dice roll here is $2$. Since the value of an individual roll is independent from the number of dice rolled or the order they're rolled in, if we multiply the expected value of a dice roll by the expected value of the number of times a camel moves in a turn, we find the expected distance a camel moves in a turn. Thus, $E[X] = 2E[Z]$. This allows for a simpler way to find the expected value by means of calculating the probability of each individual distance.

\section{Analysis}

\begin{theorem}
  The expected value for the number of spaces moved for a camel on the top of a stack of 3 camels is $\displaystyle\frac{11}{3}$.
\end{theorem}

With our new tool, let's put it to the test and get some results. First, let's look at the top camel in a stack of three.

\subsection{Stack of 3}
 It is clear that this camel can move at most $3$ times. The total number of permutations of the dice rolls here is $3! = 6$. We find that there is one case where the camel moves once, and here the camel's die must be rolled before the other two, so $P(Z=1) = \frac{1}{3}$. There are two cases where the camel moves twice. The first, where the middle camel moves first and then the top camel moves, and since the yellow's movement afterwords doesn't affect this scenario it has probability $\frac{1}{3}$. The second has the bottom camel moving and then the top camel moving immediately after, with probability $\frac{1}{6}$, so $P(Z=2) = \frac{1}{3} + \frac{1}{6} = \frac{1}{2}$. Finally, there is only one way in which the top camel moves three times, so $P(Z = 3) = \frac{1}{6}$. Thus, we find that the expected number of times the top camel moves is $E[Z] = \sum{z=1}^{3}zp(z) =1*\frac{1}{3} + 2*\frac{1}{2} + 3*\frac{1}{6} = \frac{11}{6}$. We can then find that the expected value of the top camel's movement that turn is $E[X]=2*\frac{11}{6} = \frac{11}{3} \approx 3.67$.

\subsection{Stack of 4}
 Now for a more interesting example, finding the expected value of the top camel in a stack of $4$. As before, we will begin by finding the number of ways the camel moves $1$ to $4$ times. The total permutations of dice rolls here is $4! = 24$. There is one way the top camel moves once, and it occurs with probability $1/4$. Next we find the number of ways the top camel moves twice, which are broken down into $3$ types of scenarios.
The first is when the bottom camel moves and the top camel follows immediately after, which has $2$ different ways of occurring. The second way is when the second to bottom camel moves first, in which case the bottom camel's order of movement doesn't matter but the second to top's does. This type of scenario has $3$ possibilities. The third scenario type is when the second to top camel moves and then the top camel moves. In this case the very bottom camel and the second to bottom camel's movement order doesn't matter, and so we find that there are $6$ ways this scenario occurs. Thus, there are $2 + 3 6 = 11$ different independent ways the top camel moves twice, and so the probability the top camel moves twice is $\frac{11}{24}$ There are $2$ different scenario types in which the top camel moves $3$ times. The first is when the bottom camel moves first, and there are $3$ different variations here where the top camel moves $3$ times. The second type of scenario is when the second to bottom camel moves, and similarly there are $3$ variations here where the top camel moves $3$ times, and so there are $3 + 3 = 6$ ways the top camel moves $3$ times, and so it has probability $\frac{6}{24} = \frac{1}{4}$. Finally, there is one way the top camel moves $4$ times, so it has probability $\frac{1}{24}$. Finally, the expectation of the movement of the top camel here is:
$$E[X]=\sum_{k=1}^4 kp(k) = 2(1*\frac{1}{4} + 2*\frac{11}{24} + 3*\frac{1}{4} + 4*\frac{1}{24}) = 2(\frac{1}{4}+\frac{11}{12}+\frac{3}{4}+\frac{1}{6}) = \frac{25}{6} \approx 4.167$$

\subsection{Stack of 5}
 Although unlikely, the expected value of a stack of five can be found with this method as well. We find that $P(Z=1) = \frac{1}{5}$, $P(Z=2) = \frac{49}{120}$, $P(Z=3) = \frac{3}{10}$, $P(Z=4) = \frac{1}{12}$, and $P(Z=5+ = \frac{1}{120}$. Thus, we find that $E(Z) = \frac{55}{24}$, and so $E(X) = 2E(Z) = 2*\frac{55}{12} \approx 4.5833$
\end{document}


\section{First Sample Split}

\subsection{Testing for a Sample Split, Using GDP\_1960}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 GDP\_1960 \\
Threshold Estimate:                833 \\
LM-test for no threshold:          12.60184 \\
Bootstrap P-Value:                 0.091 \\

\subsection{Testing for a Sample Split, Using Literacy}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 Literacy \\
Threshold Estimate:                10 \\
LM-test for no threshold:          10.78627 \\
Bootstrap P-Value:                 0.208 \\

\subsection{Estimate Sample Split, Using GDP\_1960}

\subsubsection{Global OLS Estimation, Without Threshold}
Dependent Variable:      GDP\_Gwth \\
Heteroskedasticity Correction Used \\\\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Estimate     &   St Error \\
\midrule
Const   &    2.8262   &   0.72473 \\
Log\_GDP\_1960   &   -0.2816   &   0.05335 \\
Log\_Inv/GDP   &    0.4919   &   0.10435 \\
Log\_Pop\_Gwth   &   -0.5467   &   0.23278 \\
Log\_School   &    0.2395   &   0.06560 \\
\bottomrule
\end{tabular}
\bigskip \\
Observations:                       96 \\
Degrees of Freedom:                 91 \\
Sum of Squared Errors:              9.622743 \\
Residual Variance:                  0.1057444 \\
R-squared:                          0.4805041 \\
Adjusted R-squared:                 0.4576691 \\
AIC:                                -210.821 \\
BIC:                                -197.9993 \\
HQC:                                -205.6382 \\
Heteroskedasticity Test (P-Value):  0.2008698 \\

\subsubsection{Threshold Estimation}
Threshold Variable:                 GDP\_1960 \\
Threshold Estimate:                 863 \\
0.9 Confidence Interval:           [594, 1794] \\
0.95 Confidence Interval:           [594, 1794] \\
0.99 Confidence Interval:           [539, 4802] \\
Sum of Squared Errors:              8.024881 \\
Residual Variance:                  0.09331257 \\
Joint R-squared:                    0.5667667 \\
Joint Adjusted R-squared:           0.5214283 \\
Joint AIC:                          -216.2529 \\
Joint BIC:                          -188.0451 \\
Joint HQC:                          -204.8509 \\
Heteroskedasticity Test (P-Value):  0.1565995 \\

\subsubsection*{Regime 1: GDP\_1960$\leq$863 }
Parameter Estimates \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Estimate     &   St Error \\
\midrule
Const   &    4.31203   &   1.62680 \\
Log\_GDP\_1960   &   -0.65697   &   0.21762 \\
Log\_Inv/GDP   &    0.22774   &   0.07160 \\
Log\_Pop\_Gwth   &   -0.29487   &   0.33678 \\
Log\_School   &    0.01806   &   0.09686 \\
\bottomrule
\end{tabular}
\bigskip

0.9 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable   &    Low            &   High \\
\midrule
Const   &    1.18837   &    8.9645 \\
Log\_GDP\_1960   &   -1.20170   &   -0.2184 \\
Log\_Inv/GDP   &    0.06085   &    0.5363 \\
Log\_Pop\_Gwth   &   -1.37220   &    0.7700 \\
Log\_School   &   -0.21504   &    0.4116 \\
\bottomrule
\end{tabular}
\bigskip

0.95 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable   &    Low            &   High \\
\midrule
Const   &    0.68755   &    9.5624 \\
Log\_GDP\_1960   &   -1.25007   &   -0.1465 \\
Log\_Inv/GDP   &    0.02471   &    0.5740 \\
Log\_Pop\_Gwth   &   -1.51316   &    0.9225 \\
Log\_School   &   -0.24701   &    0.4397 \\
\bottomrule
\end{tabular}
\bigskip

0.99 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable   &    Low            &   High \\
\midrule
Const   &   -0.31408   &   10.75843 \\
Log\_GDP\_1960   &   -1.34682   &   -0.00266 \\
Log\_Inv/GDP   &   -0.04758   &    0.64950 \\
Log\_Pop\_Gwth   &   -1.79508   &    1.22748 \\
Log\_School   &   -0.31094   &    0.49604 \\
\bottomrule
\end{tabular}
\bigskip

Observations:                       18 \\
Degrees of Freedom:                 13 \\
Sum of Squared Errors:              0.6742722 \\
Residual Variance:                  0.05186709 \\
R-squared:                          0.5165107 \\
Adjusted R-squared:                 0.3677448 \\
AIC:                                -49.12088 \\
BIC:                                -44.66902 \\
HQC:                                -48.50703 \\

\subsubsection*{Regime 2: GDP\_1960$>$863 }
Parameter Estimates \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Estimate     &   St Error \\
\midrule
Const   &    3.6631   &   0.71905 \\
Log\_GDP\_1960   &   -0.3234   &   0.06144 \\
Log\_Inv/GDP   &    0.4957   &   0.14497 \\
Log\_Pop\_Gwth   &   -0.4877   &   0.25532 \\
Log\_School   &    0.3569   &   0.08997 \\
\bottomrule
\end{tabular}
\bigskip

0.9 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Low            &   High \\
\midrule
Const   &    2.15721   &    5.48302 \\
Log\_GDP\_1960   &   -0.50226   &   -0.20099 \\
Log\_Inv/GDP   &    0.22373   &    0.91018 \\
Log\_Pop\_Gwth   &   -0.98136   &   -0.05348 \\
Log\_School   &   -0.05371   &    0.52305 \\
\bottomrule
\end{tabular}
\bigskip

0.95 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Low            &   High \\
\midrule
Const   &    1.8448   &    5.79544 \\
Log\_GDP\_1960   &   -0.5230   &   -0.18203 \\
Log\_Inv/GDP   &    0.1823   &    0.95436 \\
Log\_Pop\_Gwth   &   -1.0685   &    0.03369 \\
Log\_School   &   -0.0848   &    0.54919 \\
\bottomrule
\end{tabular}
\bigskip

0.99 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Low            &   High \\
\midrule
Const   &    1.21994   &    6.4203 \\
Log\_GDP\_1960   &   -0.56451   &   -0.1441 \\
Log\_Inv/GDP   &    0.09581   &    1.0427 \\
Log\_Pop\_Gwth   &   -1.24285   &    0.2080 \\
Log\_School   &   -0.14698   &    0.6015 \\
\bottomrule
\end{tabular}
\bigskip

Observations:                       78 \\
Degrees of Freedom:                 73 \\
Sum of Squared Errors:              7.350609 \\
Residual Variance:                  0.1006933 \\
R-squared:                          0.5492007 \\
Adjusted R-squared:                 0.5244994 \\
AIC:                                -174.2302 \\
BIC:                                -162.4467 \\
HQC:                                -169.513 \\

\subsubsection*{Threshold Regression Table}
\begin{tabular}{l*{2}{S[table-format=2.3]r}}
\toprule
& \multicolumn{4}{c}{GDP\_Gwth} \\
\cmidrule(lr){2-5}
& \multicolumn{2}{c}{Regime 1} & \multicolumn{2}{c}{Regime 2} \\
\cmidrule(lr){2-3}
\cmidrule(lr){4-5}
& \multicolumn{2}{c}{GDP\_1960$^\ast$ $\leq$ 863} & \multicolumn{2}{c}{GDP\_1960$^\ast$ $>$ 863} \\
\cmidrule(lr){2-3}
\cmidrule(lr){4-5}
{Variable} & {Estimate} & {Std error} & {Estimate} & {Std error} \\
\midrule
Const &  4.312$^{\ast\ast}$ & (1.627) &  3.663$^{\ast\ast\ast}$ & (0.719) \\
Log\_GDP\_1960 & -0.657$^{\ast\ast\ast}$ & (0.218) & -0.323$^{\ast\ast\ast}$ & (0.061) \\
Log\_Inv/GDP &  0.228$^{\ast\ast}$ & (0.072) &  0.496$^{\ast\ast\ast}$ & (0.145) \\
Log\_Pop\_Gwth & -0.295 & (0.337) & -0.488$^\ast$ & (0.255) \\
Log\_School &  0.018 & (0.097) &  0.357 & (0.090) \\
\midrule
\#Obs & 18 & & 78 & \\
$R^2_\text{adj}$ & 0.368 & & 0.524 & \\
\bottomrule
\end{tabular}
\smallskip \\
$^\ast p<0.1$; $^{\ast\ast} p<0.05$; $^{\ast\ast\ast} p<0.01$
\section{Second Sample Split: Subsample, Incomes below 863}

\subsection{Testing for a Sample Split, Using GDP\_1960}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 GDP\_1960 \\
Threshold Estimate:                594 \\
LM-test for no threshold:          7.103181 \\
Bootstrap P-Value:                 0.401 \\

\subsection{Testing for a Sample Split, Using Literacy}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 Literacy \\
Threshold Estimate:                10 \\
LM-test for no threshold:          5.16476 \\
Bootstrap P-Value:                 0.844 \\

\section{Second Sample Split: Subsample, Incomes above 863}

\subsection{Testing for a Sample Split, Using GDP\_1960}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 GDP\_1960 \\
Threshold Estimate:                1410 \\
LM-test for no threshold:          11.00934 \\
Bootstrap P-Value:                 0.157 \\

\subsection{Testing for a Sample Split, Using Literacy}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 Literacy \\
Threshold Estimate:                57 \\
LM-test for no threshold:          12.09135 \\
Bootstrap P-Value:                 0.073 \\

\subsection{Estimate Sample Split, Using Literacy}

\subsubsection{Global OLS Estimation, Without Threshold}
Dependent Variable:      GDP\_Gwth \\
Heteroskedasticity Correction Used \\\\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Estimate     &   St Error \\
\midrule
Const   &    3.6631   &   0.71905 \\
Log\_GDP\_1960   &   -0.3234   &   0.06144 \\
Log\_Inv/GDP   &    0.4957   &   0.14497 \\
Log\_Pop\_Gwth   &   -0.4877   &   0.25532 \\
Log\_School   &    0.3569   &   0.08997 \\
\bottomrule
\end{tabular}
\bigskip \\
Observations:                       78 \\
Degrees of Freedom:                 73 \\
Sum of Squared Errors:              7.350609 \\
Residual Variance:                  0.1006933 \\
R-squared:                          0.5492007 \\
Adjusted R-squared:                 0.5244994 \\
AIC:                                -174.2302 \\
BIC:                                -162.4467 \\
HQC:                                -169.513 \\
Heteroskedasticity Test (P-Value):  0.362994 \\

\subsubsection{Threshold Estimation}
Threshold Variable:                 Literacy \\
Threshold Estimate:                 45 \\
0.9 Confidence Interval:           [29, 57] \\
0.95 Confidence Interval:           [19, 57] \\
0.99 Confidence Interval:           [14, 62] \\
Sum of Squared Errors:              6.198249 \\
Residual Variance:                  0.09115072 \\
Joint R-squared:                    0.6198728 \\
Joint Adjusted R-squared:           0.5695619 \\
Joint AIC:                          -175.5305 \\
Joint BIC:                          -149.6067 \\
Joint HQC:                          -165.1527 \\
Heteroskedasticity Test (P-Value):  0.551557 \\

\subsubsection*{Regime 1: Literacy$\leq$45 }
Parameter Estimates \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Estimate     &   St Error \\
\midrule
Const   &    2.0923   &   1.8702 \\
Log\_GDP\_1960   &   -0.1163   &   0.1649 \\
Log\_Inv/GDP   &    0.1726   &   0.2161 \\
Log\_Pop\_Gwth   &   -0.3902   &   0.5161 \\
Log\_School   &    0.4525   &   0.1168 \\
\bottomrule
\end{tabular}
\bigskip

0.9 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable   &    Low            &   High \\
\midrule
Const   &   -1.2736   &   5.5508 \\
Log\_GDP\_1960   &   -0.4185   &   0.2859 \\
Log\_Inv/GDP   &   -0.2427   &   0.5956 \\
Log\_Pop\_Gwth   &   -1.4324   &   0.7110 \\
Log\_School   &    0.2389   &   0.7024 \\
\bottomrule
\end{tabular}
\bigskip

0.95 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable   &    Low            &   High \\
\midrule
Const   &   -1.8538   &   6.1610 \\
Log\_GDP\_1960   &   -0.4709   &   0.3447 \\
Log\_Inv/GDP   &   -0.3182   &   0.6555 \\
Log\_Pop\_Gwth   &   -1.5932   &   0.8683 \\
Log\_School   &    0.1954   &   0.7460 \\
\bottomrule
\end{tabular}
\bigskip

0.99 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable   &    Low            &   High \\
\midrule
Const   &   -3.0144   &   7.3815 \\
Log\_GDP\_1960   &   -0.5756   &   0.4623 \\
Log\_Inv/GDP   &   -0.4692   &   0.7878 \\
Log\_Pop\_Gwth   &   -1.9149   &   1.1831 \\
Log\_School   &    0.1083   &   0.8331 \\
\bottomrule
\end{tabular}
\bigskip

Observations:                       30 \\
Degrees of Freedom:                 25 \\
Sum of Squared Errors:              2.609994 \\
Residual Variance:                  0.1043998 \\
R-squared:                          0.5780366 \\
Adjusted R-squared:                 0.5105224 \\
AIC:                                -63.25548 \\
BIC:                                -56.24949 \\
HQC:                                -61.0142 \\

\subsubsection*{Regime 2: Literacy$>$45 }
Parameter Estimates \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Estimate     &   St Error \\
\midrule
Const   &    4.31048   &   0.96516 \\
Log\_GDP\_1960   &   -0.39503   &   0.06103 \\
Log\_Inv/GDP   &    0.83364   &   0.13937 \\
Log\_Pop\_Gwth   &   -0.41801   &   0.26957 \\
Log\_School   &    0.09458   &   0.13489 \\
\bottomrule
\end{tabular}
\bigskip

0.9 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Low            &   High \\
\midrule
Const   &    1.9302   &    5.92850 \\
Log\_GDP\_1960   &   -0.5140   &   -0.26693 \\
Log\_Inv/GDP   &    0.4820   &    1.08602 \\
Log\_Pop\_Gwth   &   -1.0204   &    0.03267 \\
Log\_School   &   -0.2868   &    0.36155 \\
\bottomrule
\end{tabular}
\bigskip

0.95 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Low            &   High \\
\midrule
Const   &    1.5819   &    6.2292 \\
Log\_GDP\_1960   &   -0.5354   &   -0.2495 \\
Log\_Inv/GDP   &    0.4271   &    1.1319 \\
Log\_Pop\_Gwth   &   -1.1133   &    0.1146 \\
Log\_School   &   -0.3443   &    0.4021 \\
\bottomrule
\end{tabular}
\bigskip

0.99 Confidence Regions for Parameters \\
\begin{tabular}{l*{2}{r}}
\toprule
Variable    &   Low            &   High \\
\midrule
Const   &    0.8853   &    6.8305 \\
Log\_GDP\_1960   &   -0.5783   &   -0.2147 \\
Log\_Inv/GDP   &    0.3174   &    1.2307 \\
Log\_Pop\_Gwth   &   -1.2992   &    0.2784 \\
Log\_School   &   -0.4594   &    0.4985 \\
\bottomrule
\end{tabular}
\bigskip

Observations:                       48 \\
Degrees of Freedom:                 43 \\
Sum of Squared Errors:              3.588255 \\
Residual Variance:                  0.08344778 \\
R-squared:                          0.5821175 \\
Adjusted R-squared:                 0.5432447 \\
AIC:                                -114.4897 \\
BIC:                                -105.1337 \\
HQC:                                -110.954 \\

\subsubsection*{Threshold Regression Table}
\begin{tabular}{l*{2}{S[table-format=2.3]r}}
\toprule
& \multicolumn{4}{c}{GDP\_Gwth} \\
\cmidrule(lr){2-5}
& \multicolumn{2}{c}{Regime 1} & \multicolumn{2}{c}{Regime 2} \\
\cmidrule(lr){2-3}
\cmidrule(lr){4-5}
& \multicolumn{2}{c}{Literacy$^\ast$ $\leq$ 45} & \multicolumn{2}{c}{Literacy$^\ast$ $>$ 45} \\
\cmidrule(lr){2-3}
\cmidrule(lr){4-5}
{Variable} & {Estimate} & {Std error} & {Estimate} & {Std error} \\
\midrule
Const &  2.092 & (1.870) &  4.310$^{\ast\ast\ast}$ & (0.965) \\
Log\_GDP\_1960 & -0.116 & (0.165) & -0.395$^{\ast\ast\ast}$ & (0.061) \\
Log\_Inv/GDP &  0.173 & (0.216) &  0.834$^{\ast\ast\ast}$ & (0.139) \\
Log\_Pop\_Gwth & -0.390 & (0.516) & -0.418 & (0.270) \\
Log\_School &  0.453$^{\ast\ast\ast}$ & (0.117) &  0.095 & (0.135) \\
\midrule
\#Obs & 30 & & 48 & \\
$R^2_\text{adj}$ & 0.511 & & 0.543 & \\
\bottomrule
\end{tabular}
\smallskip \\
$^\ast p<0.1$; $^{\ast\ast} p<0.05$; $^{\ast\ast\ast} p<0.01$
\section{Third Sample Split: Subsample, Incomes above 863, Literacy below 45}

\subsection{Testing for a Sample Split, Using GDP1960}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 GDP1960 \\
Threshold Estimate:                1618 \\
LM-test for no threshold:          7.423359 \\
Bootstrap P-Value:                 0.529 \\

\subsection{Testing for a Sample Split, Using Literacy}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 Literacy \\
Threshold Estimate:                26 \\
LM-test for no threshold:          9.233321 \\
Bootstrap P-Value:                 0.174 \\

\section{Third Sample Split: Subsample, Incomes above 863, Literacy above 45}

\subsection{Testing for a Sample Split, Using GDP1960}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 GDP1960 \\
Threshold Estimate:                3493 \\
LM-test for no threshold:          9.415477 \\
Bootstrap P-Value:                 0.21 \\

\subsection{Testing for a Sample Split, Using Literacy}
\subsubsection*{Test of Null of No Threshold Against Alternative of Threshold}
Allowing Heteroskedastic Errors (White Corrected) \\\\
Number of Bootstrap Replications:  1000 \\
Trimming Percentage:               0.15 \\\\
Threshold Variable:                 Literacy \\
Threshold Estimate:                83 \\
LM-test for no threshold:          9.327215 \\
Bootstrap P-Value:                 0.183 \\


\documentclass{article}

\usepackage{siunitx,amsmath,amssymb}
\usepackage[margin=1in]{geometry}

\title{The Size of Ordinary Jupiters}
\author{Matthias J. Raives}

\begin{document}

  \maketitle{}

  \section{Electron Degeneracy Pressure}
    We start with the pressure integral
    \begin{equation}
      P_{e} = \frac{1}{3}\int_{0}^{\infty}pvn(p)\:\mathrm{d}p\label{eq:Pressure}
    \end{equation}
    with
    \begin{equation}
      n(p)\:\mathrm{d}p = \frac{8\pi}{h^{3}}p^{2}\:\mathrm{d}p\label{eq:Density}
    \end{equation}
    For a degenerate gas, all momentum states are filled up to the maximum momentum, the Fermi momentum $p_{F}$.  We can find $p_{F}$ as a function of $n_{e}$ by integrating \ref{eq:Density} up to $p_{F}$:
    \begin{align}
      n_{e} &= \int_{0}^{p_{F}}n(p)\:\mathrm{d}p\\
      n_{e} &= \int_{0}^{p_{F}}\frac{8\pi}{h^{3}}p^{2}\:\mathrm{d}p\\
      n_{e} &= \frac{8\pi}{3h^{3}}p_{F}^{3}\\
      p_{F} &= h\left(\frac{3}{8\pi}n_{e}\right)^{1/3}\approx \frac{h}{2}n_{e}^{1/3} = \pi\hbar n_{e}^{1/3}
    \end{align}
    For a non-relativistic gas, we take $v=p/m_{e}$.  Thus, the electron degeneracy pressure is:
    \begin{align}
      P_{e} &= \frac{8\pi}{3h^{3}m_{e}}\int_{0}^{p_{F}}p^{4}\:\mathrm{d}p\\
      P_{e} &\approx \frac{8}{5}\frac{1}{h^{3}m_{e}}\frac{h^{5}}{2^{5}}n_{e}^{5/3}\\
      P_{e} &\approx \frac{1}{20}\frac{h^{2}}{m_{e}}n_{e}^{5/3}
    \end{align}
    We can write this in terms of the total density by assuming $n_{e}=n_{p}=\frac{\rho}{m_{p}}$:
    \begin{equation}
      P_{e} \approx \frac{1}{20}\frac{h^{2}}{m_{e}m_{p}^{5/3}}\rho^{5/3}
    \end{equation}
    The electron degeneracy pressure must support the planet against gravity, in hydrostatic equilibrium:
    \begin{equation}
      \frac{\mathrm{d}P}{\mathrm{d}r} = -\rho\frac{GM}{r^{2}}
    \end{equation}
    At an order of magnitude level, we can simplify this:
    \begin{align}
      \frac{P(R)-P(0)}{R} &= -\rho\frac{GM}{R^{2}}\\
      P_{e} = P(0) &= \rho\frac{GM}{R}\\
      \frac{1}{20}\frac{h^{2}}{m_{e}m_{p}^{5/3}}\rho^{5/3} &= \rho\frac{GM}{R}\\
      \intertext{Using $\rho\sim M/R^{3}$:}
      \frac{1}{20}\frac{h^{2}}{m_{e}m_{p}^{5/3}}M^{5/3}R^{-5} &= GM^{2}R^{-4}\\
      RM^{1/3} &= \frac{1}{20}\frac{h^{2}}{Gm_{e}m_{p}^{5/3}}\\
      R &= \SI{1.2e10}{\cm}\:\left(\frac{M}{\si{M_{J}}}\right)^{-1/3} = \SI{1.7}{R_{J}}\:\left(\frac{M}{\si{M_{J}}}\right)^{-1/3}
    \end{align}

  \section{Electrostatic Pressure}
    The central density of a non-degenerate gas giant has an upper limit of about 1 hydrogen atom per Bohr radius cubed:
    \begin{equation}
      \rho = \frac{m_{p}}{a_{0}^{3}}
    \end{equation}
    Again taking $\rho\sim M/R^{3}$:
    \begin{align}
      \frac{M}{R^{3}} &= \frac{m_{p}}{a_{0}^{3}}\\
      R &= \frac{a_{0}}{m_{p}^{1/3}}M^{1/3}\\
      R &= \SI{5.5e9}{\cm}\left(\frac{M}{\si{M_{J}}}\right)^{1/3} = \SI{0.8}{R_{J}}\left(\frac{M}{\si{M_{J}}}\right)^{1/3}
    \end{align}

  \section{Maximum Radius}
    At large masses, where degeneracy pressure is important, the size of a gas giant \emph{decreases} with mass, as $R\propto M^{-1/3}$.  Whereas, at small masses, the size of the planet \emph{increases} with mass, as $R\propto M^{1/3}$.  The turnover mass is:
    \begin{equation}
      \SI{0.8}{R_{J}}\left(\frac{M}{\si{M_{J}}}\right)^{1/3} = \SI{1.7}{R_{J}}\:\left(\frac{M}{\si{M_{J}}}\right)^{-1/3}\:\longrightarrow M = \SI{3}{M_{J}}
    \end{equation}
    and thus the maximum radius of a gas giant is:
    \begin{equation}
      R = \SI{1.1}{R_{J}}
    \end{equation}

  \begin{thebibliography}{9}
    \bibitem{BurrowsOstriker}
      {{Burrows}, A.~S. and {Ostriker}, J.~P.},
      \textit{Astronomical reach of fundamental physics},
      {Proceedings of the National Academy of Science},
      2014,
      {http://adsabs.harvard.edu/abs/2014PNAS..111.2409B}

  \end{thebibliography}

\end{document}