\documentclass[10pt]{beamer}

\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{dsfont}
\usepackage{graphicx}
\usepackage{interval}
\usepackage{color}
\usepackage{xcolor,colortbl}
\usepackage[style=authoryear, maxcitenames=11, mincitenames=11, natbib=true, uniquename=false, uniquelist=false]{biblatex}
\addbibresource{main.bib}
\usepackage{multirow}
\usepackage{booktabs}
\usepackage{tikz}
\usetikzlibrary{patterns}
\usetikzlibrary{positioning}
\usetikzlibrary{decorations.pathreplacing,angles,quotes}
\usepgflibrary{arrows.meta,shapes.geometric}
\usepackage{algorithm}
\usepackage{algorithmicx}
\usepackage[noend]{algpseudocode}
\usepackage{dirtytalk}
\usepackage{hyperref}

\newcommand{\N}{\mathbf{N}}
\newcommand{\Z}{\mathbf{Z}}
\newcommand{\Q}{\mathbf{Q}}
\newcommand{\R}{\mathbf{R}}
\newcommand{\C}{\mathbf{C}}

\beamertemplatenavigationsymbolsempty%
\addtobeamertemplate{navigation symbols}{}{%
    \usebeamerfont{footline}%
    \usebeamercolor[fg]{footline}%
    \hspace{1em}%
    \insertframenumber/\inserttotalframenumber%
}

\author{Florian Fontan}
\title{Advanced Models and Methods in Operations Research \\ Column Generation Heuristics}
\date{November 23, 2021}

\begin{document}

\newcommand{\customcite}[1]{\citetitle{#1}, \citeauthor{#1}, \citeyear{#1}}

\AtBeginSection[]
{%
  \begin{frame}<beamer>
    \frametitle{Table of contents}
    \tableofcontents[currentsection]
  \end{frame}
}

\maketitle

\section{Exponential MILP formulations}

\begin{frame}
  \frametitle{Cutting Stock Problem, Description}

  Input:
  \begin{itemize}
    \item a capacity \alert{$C$}
    \item $n$ item types; for each item type $j = 1, \dots, n$, a weight \alert{$w_j$} and a demand \alert{$q_j$}
  \end{itemize}

  Problem:
  \begin{itemize}
    \item Pack all items such that the total weight of the items in a bin does not exceed the capacity.
  \end{itemize}

  Objective:
  \begin{itemize}
    \item Minimize the number of bin used.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Cutting Stock Problem, Formulation}

  Let us define the $K$ possible patterns such that $x_j^k = q$ iff pattern $k$, $k = 1 \dots K$ contains $q$ copies of item type $j$

  \begin{itemize}
    \item Variables:
      \begin{itemize}
        \item $y^k \in \N$,
          $\forall k = 1 \dots K$.

          $y^k = q$ iff $q$ copies of pattern $k$ are used
      \end{itemize}

    \item Objective:
      \begin{displaymath}
        \min \sum_{k = 1}^{K} y^k
      \end{displaymath}

    \item Constraints:
      \begin{align*}
        \sum_{k = 1}^{K} x_j^k y^k = q_j && \forall j = 1 \dots n \
      \end{align*}
  \end{itemize}

  \pause Why is this formulation good compared to the classical one?
  \begin{itemize}
    \item \pause No big-M constraint
    \item Better relaxation
    \item Easier to write
  \end{itemize}
\end{frame}

%\mystar{y}{r}{d-r}{t-r}{p}{nom}
\def\mystar#1#2#3#4#5#6#7{
  % 1: y
  % 2: r_s
  % 3: d_s - r_s
  % 4: t_s - r_s
  % 5: p_s
  % 6: nom
  % 7: opacity
  \begin{scope}[shift={(#2,#1)},opacity=#7,transparency group]
    \draw[line width=1pt,color=blue,arrows={Bracket-Bracket}] (0.4,0) -- +(#3,0);
    \draw[line width=5pt,color=red] (0.4,0) ++(#4,0) -- ++(#5,0);
    \def\tmp{#6}\if\tmp\empty\else
    \node[star, star point height=3mm, minimum size=5mm, color=yellow, fill=yellow, text=black,scale=.4] at (0,0) {#6};
    \fi
  \end{scope}
}

\def\mystaronly#1#2#3#4{
  % 1: y
  % 2: r_s
  % 3: nom
  % 4: opacity
  \begin{scope}[shift={(#2,#1)},opacity=#4,transparency group]
    \node[star, star point height=3mm, minimum size=5mm, color=yellow, fill=yellow, text=black,scale=.4] at (0,0) {#3};
  \end{scope}
}

\begin{frame}
  \frametitle{Star Observation Scheduling Problem, Description}

  Input: a set \alert{$\cal M$} of nights and a set \alert{$\cal N$} of stars; for each star $j \in\cal N$, a scientific interest \alert{$w_j$}, an observation duration \alert{$p_j^i$} and a visibility window \alert{$\interval[open right]{r_j^i}{d_j^i}$}, depending on the night $i$ of the observation.

  \vfill

  \only<1>{%
    \begin{tikzpicture}
      % \mystar{y}{r}{d-r}{t-r}{p}{nom}
      \mystaronly{2.5}{-.2}{$1$}{1}
      \mystaronly{2.0}{-.2}{$2$}{1}
      \mystaronly{1.5}{-.2}{$3$}{1}
      \mystaronly{1.0}{-.2}{$4$}{1}
      \mystaronly{0.5}{-.2}{$5$}{1}
      \mystaronly{0.0}{-.2}{$6$}{1}
      \mystar{2.5}{0}{2.5}{0.25}{2}{}{1}     \mystar{2.5}{5.3}{3.0}{0.5}{2}{}{1}
      \mystar{2.0}{3.0}{1.5}{.25}{1}{}{1}      \mystar{2.0}{7.5}{1.5}{.25}{1}{}{1}
      \mystar{1.5}{1.5}{3.0}{0.5}{2}{}{1}      \mystar{1.5}{5.8}{2.8}{0.4}{2}{}{1}
      \mystar{1.0}{0.5}{2.0}{0.45}{1.1}{}{1}   \mystar{1.0}{5.2}{2.0}{0.45}{1.1}{}{1}
      \mystar{0.5}{3.5}{1.25}{0.25}{0.75}{}{1} \mystar{0.5}{8.2}{1.25}{0.25}{0.75}{}{1}
      \mystar{0}{2}{2.5}{.5}{1.5}{}{1}    \mystar{0}{6.7}{2.75}{.625}{1.5}{}{1}

      \draw[color=gray,dotted,line width=3pt] (.2,3)--(.2,-1) (5.4,3)--(5.4,-1) (10.1,3)--(10.1,-1);

      \node [gray] at (2.5,-.8) {nuit 1};
      \node [gray] at (8.3,-.8) {nuit 2};
    \end{tikzpicture}
  }
  \only<2>{%
    \begin{tikzpicture}
      % \mystar{y}{r}{d-r}{t-r}{p}{nom}
      \mystaronly{2.5}{-.2}{$1$}{1}
      \mystaronly{2.0}{-.2}{$2$}{.2}
      \mystaronly{1.5}{-.2}{$3$}{1}
      \mystaronly{1.0}{-.2}{$4$}{1}
      \mystaronly{0.5}{-.2}{$5$}{1}
      \mystaronly{0.0}{-.2}{$6$}{1}
      \mystar{2.5}{0}{2.5}{0}{2}{}{1}        \mystar{2.5}{5.3}{3.0}{0.5}{2}{}{.2}
      \mystar{2.0}{3.0}{1.5}{.25}{1}{}{.2}     \mystar{2.0}{7.5}{1.5}{.25}{1}{}{.2}
      \mystar{1.5}{1.5}{3.0}{0.5}{2}{}{1}      \mystar{1.5}{5.8}{2.8}{0.4}{2}{}{.2}
      \mystar{1.0}{0.5}{2.0}{0.45}{1.1}{}{.2}  \mystar{1.0}{5.2}{2.0}{0}{1.1}{}{1}
      \mystar{0.5}{3.5}{1.25}{0.5}{0.75}{}{1}  \mystar{0.5}{8.2}{1.25}{0.25}{0.75}{}{.2}
      \mystar{0}{2}{2.5}{.5}{1.5}{}{.2}   \mystar{0}{6.7}{2.75}{0}{1.5}{}{1}

      \draw[color=gray,dotted,line width=3pt] (.2,3)--(.2,-1) (5.4,3)--(5.4,-1) (10.1,3)--(10.1,-1);

      \node [gray] at (2.5,-.8) {nuit 1};
      \node [gray] at (8.3,-.8) {nuit 2};
    \end{tikzpicture}
  }
\end{frame}

\begin{frame}
  \frametitle{Star Observation Scheduling Problem, Formulation}

  For each night $i$, $i = 1 \dots m$, let us define the $K_i$ possible schedules such that $x_{i, j}^k = 1$ iff schedule $k$, $k = 1 \dots K_i$ of night $i$ contains star $j$

  \begin{itemize}
    \item Variables:
      \begin{itemize}
        \item $y_i^k \in \{0, 1\}$,
          $\forall i = 1 \dots m$, $\forall k = 1 \dots K_i$.

          $y_i^k = 1$ iff scheduled $k$ of night $i$ is selected
      \end{itemize}

    \item Objective:
      \begin{displaymath}
        \max \sum_{i = 1}^m \sum_{k = 1}^{K_i} \sum_{j = 1}^n w_j x_{i, j}^k y_i^k
      \end{displaymath}

    \item Constraints:
      \begin{align*}
        \sum_{k = 1}^{K_i} y_i^k = 1 && \forall i = 1 \dots m \\
        \sum_{i = 1}^n \sum_{k = 1}^{K_i} x_{i, j}^k y_i^k \le 1 && \forall j = 1 \dots n \\
      \end{align*}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{2D Guillotine Variable-sized Bin Packing, Description}

  Input:
  \begin{itemize}
    \item $n$ item types; for each item type $j = 1, \dots, n$, a width \alert{$w_j$}, a height \alert{$h_j$} and a demand \alert{$q_j$}
    \item $m$ bin types; for each bin type $i = 1, \dots, m$, a wdith \alert{$W_i$}, a heigh \alert{$H_i$}, a lower bound \alert{$l_i$}, an upper bound \alert{$u_i$} and a cost \alert{$c_i$}
  \end{itemize}

  Problem:
  \begin{itemize}
    \item Find a subset of guillotine patterns such that all item type demands and bin type use bounds are satisfied
  \end{itemize}

  Objective:
  \begin{itemize}
    \item Minimize the cost of the selected bins.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{2D Guillotine Variable-sized Bin Packing, Formulation}

  For each bin type $i$, $i = 1 \dots m$, let us define the $K_i$ possible patterns such that $x_{i, j}^k = q$ iff pattern $k$, $k = 1 \dots K_i$ of bin type $i$ contains $q$ copies of item type $j$

  \begin{itemize}
    \item Variables:
      \begin{itemize}
        \item $y_i^k \in \N$,
          $\forall i = 1 \dots m$, $\forall k = 1 \dots K_i$.

          $y_i^k = q$ iff $q$ copies of pattern $k$ of bin type $i$ are used
      \end{itemize}

    \item Objective:
      \begin{displaymath}
        \min \sum_{i = 1}^m \sum_{k = 1}^{K_i} c_i y_i^k
      \end{displaymath}

    \item Constraints:
      \begin{align*}
        l_i \le \sum_{k = 1}^{K_i} y_i^k \le u_i && \forall i = 1 \dots m \\
        \sum_{i = 1}^n \sum_{k = 1}^{K_i} x_{i, j}^k y_i^k = q_j && \forall j = 1 \dots n \\
      \end{align*}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Other examples}

  Usually, variables represent:
  \begin{itemize}
    \item A bin/knapsack (for packing problems)
    \item The schedule of a machine (for parallel scheduling problems)
    \item The route of a vehicle (for vehicle routing problems)
    \item \dots
  \end{itemize}
\end{frame}

\section{Solving the relaxation by Column Generation}

\begin{frame}
  \frametitle{Introduction}

  \begin{itemize}
    \item With these formulations, generating all the variables is generally not possible since their number grows exponentially with the size of the problem.
    \item First we focus on solving the \alert{linear relaxation}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{The Column Generation procedure}

  \begin{itemize}
    \item We use the \alert{simplex algorithm}.
      \begin{itemize}
        \item At each iteration, it adds a variable of negative reduced cost to the current basis
        \begin{itemize}
          \item Objective:
            \begin{displaymath}
              \min \sum_{j = 1}^n c_j x_j
            \end{displaymath}
          \item Constraints:
          \begin{align*}
            \sum_{j = 1}^n a_{i, j} x_j \le b_j && \forall j = 1 \dots n \\
          \end{align*}
          \item Reduced cost of variable $x_j$:
            \begin{displaymath}
              c_j - \sum_{i = 1}^m a_{i, j} v_i
            \end{displaymath}
            with $v_i$ the dual value of constraint $i$.
        \end{itemize}
        \item It stops when there are no variable of negative reduced cost
      \end{itemize}
    \item The difference with the traditional simplex algorithm, is that here, it is not possible to loop through all the variables to find a variable of negative reduced cost, since they have not been all generated.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{The Column Generation procedure}
  \begin{itemize}
    \item Instead, finding a variable of negative reduced cost becomes an optimization problem

    \item Example with the Cutting Stock Problem
      \begin{itemize}
        \item Objective:
          \begin{displaymath}
            \min \sum_{k = 1}^{K} y^k
          \end{displaymath}

        \item Constraints:
          \begin{align*}
            \sum_{k = 1}^{K} x_j^k y^k = q_j && \forall j = 1 \dots n \\
          \end{align*}
      \end{itemize}

    \item Reduced cost of $y^k$:
      \begin{displaymath}
        1 - \sum_{j = 1}^n x_j^k v_j
      \end{displaymath}
      with $v_j$ the dual value of constraint $j$.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{The Column Generation procedure}

  \begin{itemize}
    \item We look for a variable $y^k$ such that
      \begin{displaymath}
        1 - \sum_{j = 1}^n x_j^k v_j < 0
      \end{displaymath}

    \item Finding a variable of negative reduced cost is equivalent to finding a pattern with total profit $\ge 1$ with the profit of item type $j$ being equal to $v_j$.

    \item In practice, we solve the problem as an optimization problem: we find the best solution of the Knapsack Problem and check if the reduced cost of the corresponding variable is negative.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{The Column Generation procedure}

  Summary:
  \begin{algorithmic}
    \Function{ColumnGeneration}{$P$}
    \State $Y \gets \text{initial set of columns}$
    \While{True}
    \State Solve the Linear Program $P'$ with variables from $Y$
    \State Look for a variable of negative reduced cost (\alert{Pricing Problem})
    \If {there is one}
    \State Add it to $Y$
    \Else
    \State \Return the solution of $P'$
    \EndIf
    \EndWhile
    \EndFunction
  \end{algorithmic}
\end{frame}

\begin{frame}
  \frametitle{Initial set of columns}

  \begin{itemize}
    \item To get dual values, the LP needs to be feasible
    \item With $0$ variable, the LP might be infeasible
      \begin{itemize}
        \item Example: Cutting Stock Problem, demand constraints are not satisfied
      \end{itemize}
    \item Therefore, we need to find a way to get an initial set of columns such that the LP is feasible
      \begin{itemize}
        \item Find a feasible solution and add the corresponding columns
          \begin{itemize}
            \item Example: Cutting Stock, Best Fit algorithm
            \item Drawback: Problem specific, additional work for the implementation of the heuristic
            \item Advantage: if the solution is good, it might speed up the column generation proceudre
          \end{itemize}
        \item Find manually a set of columns that ensures the LP to be feasible
          \begin{itemize}
            \item Example: create $n$ columns with only one item
          \end{itemize}
        \item Generate a dummy column with very high cost for each problematic constraint
          \begin{itemize}
            \item Advantage: not problem specific
            \item Drawback: numerical issue is the cost of the dummy columns is not well calibrated
          \end{itemize}
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Star Observation Scheduling Problem, Pricing}

  \begin{itemize}
    \item Objective:
      \begin{displaymath}
        \max \sum_{i = 1}^m \sum_{k = 1}^{K_i} \sum_{j = 1}^n w_j x_{i, j}^k y_i^k
      \end{displaymath}

    \item Constraints:
      \begin{align*}
        \sum_{k = 1}^{K_i} y_i^k = 1 && \forall i = 1 \dots m && \alert{\text{dual } u_i} \\
        \sum_{i = 1}^n \sum_{k = 1}^{K_i} x_{i, j}^k y_i^k \le 1 && \forall j = 1 \dots n && \alert{\text{dual } v_j} \\
      \end{align*}

    \item \pause Reduced cost of $y_i^k$:
      \begin{displaymath}
        \sum_{j = 1}^n w_j x_{i, j}^k - u_i - \sum_{j = 1}^n x_{i, j}^k v_j
        = \sum_{j = 1}^n (w_j - v_j) x_{i, j}^k - u_i
      \end{displaymath}

    \item \pause Finding a variable of maximum reduced cost reduces to solving $m$ Single Night Star Observation Scheduling Problems with targets with profit $w_j - v_j$.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{2D Guillotine Variable-sized Bin Packing, Pricing}

  \begin{itemize}
    \item Objective:
      \begin{displaymath}
        \min \sum_{i = 1}^m \sum_{k = 1}^{K_i} c_i y_i^k
      \end{displaymath}

    \item Constraints:
      \begin{align*}
        l_i \le \sum_{k = 1}^{K_i} y_i^k \le u_i && \forall i = 1 \dots m && \alert{\text{dual } u'_i} \\
        \sum_{i = 1}^n \sum_{k = 1}^{K_i} x_{i, j}^k y_i^k = q_j && \forall j = 1 \dots n && \alert{\text{dual } v_j} \\
      \end{align*}

    \item \pause Reduced cost of $y_i^k$:
      \begin{displaymath}
        c_i - u'_i - \sum_{j = 1}^n x_j^k v_j
      \end{displaymath}

    \item \pause Finding a variable of minium reduced cost reduces to solving $m$ 2D Guillotine Knapsack Problems with items with profit $v_j$ for each bin type.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Transition}

  \begin{itemize}
    \item The Column Generation procedure solves the relaxation of the exponential formulation
    \item \pause Thus, it provides a valid bound
    \item \pause But it generally does not provide a feasible solution
    \item \pause How to exploit the Column Generation to get feasible solutions?
  \end{itemize}
\end{frame}

\section{Finding solutions after the Column Generation}

\begin{frame}
  \frametitle{The Branch-and-Price algorithm}

  \begin{itemize}
    \item LP-based branch-and-bound, the relaxation is solved by the Column Generation procedure in each node
    \item How to branch?
      \begin{itemize}
        \item Branching on columns of the exponential formulation? No, the pricing problem becomes to difficult
          \item Branching on the variables of the primal formulation?
            \begin{itemize}
              \item Bin Packing: branch on whether item $j$ is packed in bin $i$ or not. If yes, then the available bins have now different capacities and a Knapsack Problem for each capacity needs to be computed
            \end{itemize}
          \item Best solution for the Bin Packing: branch on whether two items are packed in the same bin or not. If yes, then they are merged into a single item. If no, then the subproblem becomes a Knapsack Problem with Conflicts which is strongly NP-hard
      \end{itemize}
      \item $\implies$ Complex to implement.
      \item It can be combined with cuts (Branch-and-Price-and-Cut). The added cuts might also change the pricing problem $\implies$ even more complex to implement
      \item Only exact method based on Column Generation, state-of-the-art exact method for many Vehicle Routing and Parallel Machine Scheduling Problems
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Solving the restricted master}

  \begin{itemize}
    \item The Column Generation procedure is executed once
    \item Solve the exponential formulation with a MILP solver using only the columns generated during the Column Generation procedure
    \item No guarantee to find the optimal solution (or even a feasible solution)
    \item Solving the MILP is computationaly expensive if many columns have been generated. It can take some time before finding a first solution
    \item It requires a good MILP solver
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Heuristic Tree Search}

  Branching scheme:
  \begin{itemize}
    \item Root node: no column has been fixed
    \item Children: solve the relaxation by column generation, select the variable $y$ with the most integral value $v \neq 0$, for each possible value $v'$ of $y$ create one child.
    \item The discrepancy of a child is computed as:
      \begin{displaymath}
        \mathrm{disc}_\text{child} = \mathrm{disc}_\text{father} + | v' - v |
      \end{displaymath}
  \end{itemize}

  Algorithms:
  \begin{itemize}
    \item Greedy
    \item Limited Discrepancy Search
  \end{itemize}

  Note that the depth of the tree is of the order of the number of columns in a solution.
\end{frame}

\begin{frame}
  \frametitle{Additional tricks}

  \begin{itemize}
    \item Using a fast heuristic algorithm to solve the pricing problem. If the heuristic doesn't find a column of negative reduced cost:
      \begin{itemize}
        \item Case 1: Try with a more expensive exact algorithm
        \item Case 2: Stop the column generation procedure. The bound is not valid, therefore, it is not possible to use an exact Branch-and-price in this case. But the heuristics still work.
      \end{itemize}
    \item Generating columns without the simplex algorithm
      \begin{itemize}
        \item It might be faster than the column generation procedure
        \item It might be difficult to generate columns that fit well together
        \item No bound
        \item Then solve the restricted master or use a Heuristic Tree Search algorithm
      \end{itemize}
    \item Solving the restricted master with a heuristic algorithm
      \begin{itemize}
        \item Often, the master problem is a set covering or set packing problem for which heuristic algorithms have already been developed
        \item It might be faster than a MILP solver
      \end{itemize}
  \end{itemize}
\end{frame}

\section{\texttt{columngenerationsolverpy}}

\begin{frame}
  \frametitle{\texttt{columngenerationsolverpy}}

  \begin{itemize}
    \item A package that simplifies the implementation of Column Generation based algorithms
    \item Written in \texttt{Python3} (original version in \texttt{C++})
    \item \url{https://github.com/fontanf/columngenerationsolverpy}
    \item Install with: \texttt{pip3 install columngenerationsolverpy}
    \item It includes:
      \begin{itemize}
        \item The Column Generation algorithm
        \item The Greedy algorithm
        \item The Limited Discrepancy Search algorithm
      \end{itemize}
    \item To solve a problem, one needs to provide the exponential formulation and the solver for the Pricing Problem (able to take as input the currently fixed columns)
    \item The implementation of the Greedy algorithm and the Limited Discpreancy Search algorithm relies on the \texttt{treesearchsolverpy} package
  \end{itemize}
\end{frame}

\section{Conclusion}

\begin{frame}
  \frametitle{Conclusion}

  \begin{itemize}
    \item Column Generation: solving the relaxation of exponential formulations by generating the columns dynamically
    \item It can be embedded in a classical Branch-and-bound
      \begin{itemize}
        \item State-of-the art exact method for many Vehicle Routing and Parallel Machine Scheduling Problems
        \item Cumbersome to implement
      \end{itemize}
    \item It can be embedded in a Heuristic Tree Search framework
      \begin{itemize}
        \item Also state of the art heuristics for several problems
        \item Easier to implement
      \end{itemize}
    \item Works better when the number of elements in columns is small ($\le 20$)
  \end{itemize}
\end{frame}

\maketitle

\end{document}

2021-2022 - M2 ORCO - Advanced OR/3 - Column Generation Heuristics/slides/slides.tex


\documentclass[11pt]{article}
\newcommand\tab[1][1cm]{\hspace*{#1}}
\usepackage{graphicx}
\graphicspath{ {C:/Users/yedkk/Desktop/CS465/HW1} }
\begin{document}
\section{Homework 4}
Name: Kangdong Yuan

\subsection{problem1}
a).I did not work in a group.
\\b).I did not consult without anyone my group members
\\c).I did not consult any non-class materials.

\subsection{problem2}
1). First, divide the array into two same length array A1 and A2, then we get the majority of the sub-lists l and r. if l or r is the majority element of whole list, return l or r. And if the l and r is same, this element is the majority element of whole list. And we do this divide and combine process recursively to find whether the array has majority element.

The algorithm in  pseudo-code. \\
program majority element: \\
Input:  array A[1...n] \\
Output: Majority element of array \\
if n=None: \\
\tab return None \\
if n=1:\\
\tab return A[1]\\
half = n // 2 \\
left = majority(A[1...half+1]) \\
right = majority(A[half+1...n]) \\
if left == right: \\
\tab return left \\
if A.count(left)$>$half:\\
\tab return left\\
if A.count(right)$>$half:\\
\tab return right\\
return None\\
Count() return the number of specified element in given array. \\
b). If A has a majority element, the majority element m must also be a majority element of A1 or A2 or both. If the count of m is not majority in each sub array, it must not be the majority in total array. If both parts have the same majority element, it is the majority element for total array. If one of the sub array has a majority element, count occurrence of combined array (O(n) time) to see whether it is a majority element. If both parts have a majority, the algorithm check those two elements and return the result (O(n) running times). And this divide algorithm perform recursively, the running time in each level is O(n), and the recursive tree has nlog(n) levels. \\
c). this algorithm is every function call two another functions, and the time complexity in each level is O(n), because the count() function go through every element in array.\\
$T(n)=2T(n/2)+n$, using the master theorem, $a=2, \ , b=2, \ c=1, \ log_a(b)=log_2(2)=1=c$\\ The time complexity is $T(n)=O(n^{log_a(b)}*log(n))=O(n*log(n))$

\subsection{problem3}
a). For graph stored in adjacency list format, if we want to return the Reverse graph $G^R$, we need to do below algorithm. \\

The algorithm in  pseudo-code. \\
program  Reverse graph: \\
Input:  graph: G (adjacency list format) \\
Output: Reverse graph: $G^R$ (adjacency list format)\\
create an empty adjacency list $G^R$ \\
for each vertex u in V \\
\tab add the new vertex u to $G^R$\\
\tab for each edge (u, w) in E \\
\tab \tab add new edge (w,u) to $G^R$ \\
return changed graph $G^R$ \\
When add a vertex or edge, it take O(1 ) time. the total number of add is constant, since there are only $|V|+|E|$ vertices and edges in graph $G$, so the running time is also $O(|V|+|E|)$\\
\\
b)For graph stored in adjacency matrix format, if we want to return the Reverse graph $G^R$, we need to do below.\\
for the adjacency matrix format, the number of columns and rows are equal to number of vertices. We assume the number of vertices is $|V|$\\
The algorithm in  pseudo-code. \\
program  Reverse graph: \\
Input:  graph: G (adjacency matrix format) \\
Output: Reverse graph: $G^R$ (adjacency matrix format)\\
create a new matrix $G^R$ with dimension $|V|*|V|$\\
for i in rows of G\\
\tab for j in columns of G\\
\tab \tab if edge G[i,j] exist\\
\tab \tab \tab add edge to $G^R[j,i]$\\
return $G^R$\\
we assume deal with a condition and add a edge all take O(1) time. There are $|V|^2$ conditions and $|E|$ adds, so the running time of conditions and adds are $O(|V|^2+|E|)$. And create a empty matrix take $O(|V|^2)$ times. Adding all step together, $O(2*|V|^2+|E|)=O(|V|^2)$, so, the running time is $O(|V|^2)$

\subsection{problem4}
a).
the adjacency list for graph 1:\\
 $A->\{B,E\} \tab B->\{G,D\} \tab D->\{E\} \tab E->\{D\} \tab G->\{D,F\} $\\
the adjacency list for graph 2:\\
$C->\{H,I\} \tab H->\{I\} \tab I->\{\}$ \\
b). In undirected graph,  each edge connect to two vertices and no parallel edge, and the number of self loop edge is $|v|$. The number of edges is the number of subsets of size 2 chosen from the set of vertices. Since the set of vertices has size $|V|$, the number of such subsets is given by the binomial coefficient $C(|V|,2)=\frac{|V|*(|V|-1)}{2}$. The possible max number of edge in undirected graph is $\frac{|V|*(|V|-1)}{2}+|V|=\frac{|V|^2+|V|}{2}$ \\
c). for each undirected edge, it incident to two vertices, so the sum of $d_i$ cause by one edge is 2. We assume there are n edge in graph, those edges incident to the vertices in graph. Since one edge only can incident to two vertices, the sum of $d_i$ is $2n$. So, whatever the graph is, the sum of $d_i$ must be the even number.\\
d).The number of edge in this graph is inherited the result in question b,  the upper bound of  $|E|=\frac{|V|^2+|V|}{2}$, and lower bound of $|E|<|V|$\\
if $|E|>|V|log(|V|)$, then the $\Theta (|V|^2log(|V|))$ run faster than $\Theta(|V||E|)$ \\
if $|E|<|V|log(|V|)$, then the  $\Theta(|V||E|)$ run faster than $\Theta (|V|^2log(|V|))$ \\
e). for any undirected garph in question b, the upper bound of  $|E|=\frac{|V|^2+|V|}{2}$, and lower bound of $|E|<|V|$. \\ So,if $|E|>|V|$, the $\Theta(|E|log(|V|))$ run faster $\Theta(|E|log(|E|))$. \\But, if $|E|<|V|$, the $\Theta(|E|log(|E|))$ run faster $\Theta(|E|log(|V|))$.

\end{document}


\documentclass{report}

\usepackage{booktabs}
\usepackage{float}
\usepackage{comment}
\usepackage{graphicx}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{pdfpages}
\usepackage{siunitx}
\usepackage{mhchem}
\usepackage{subfloat}
\usepackage{fullpage}
\usepackage{array}
\usepackage[toc,page]{appendix}

\newcommand{\ecoli}{\textit{E. coli}}
\newcommand{\ecloni}{\textit{E. cloni 10g}}
\newcommand{\bsub}{\textit{B. subtilis}}
\newcommand{\pbad}{P$_{BAD}$}
\newcommand{\plux}{P$_{LUX}$}
\newcommand{\etal}{\textit{et al.}}
\newcommand{\invivo}{\textit{in vivo}}
\newcommand{\invitro}{\textit{in vitro}}
\newcommand{\csix}{3OC6HSL}

\newcommand{\tred}[1]{\textcolor{red}{#1}}
\newcommand{\tgr}[1]{\textcolor{green}{#1}}

% Units
\DeclareSIUnit{\molar}{M}
\newcommand{\ul}{\si{\micro\litre}}
\newcommand{\um}{\si{\micro\metre}}
\newcommand{\ml}{\si{\milli\litre}}
\newcommand{\nm}{\si{\nano\metre}}
\newcommand{\ugml}{\si{\micro\gram\per\milli\litre}}
\newcommand{\degC}{\si{\degreeCelsius}}

% Misc
\newcommand{\tenx}{\ensuremath{10\times}}
\newcommand{\fortyx}{\ensuremath{40\times}}
\newcommand{\sixtyx}{\ensuremath{60\times}}
\newcommand{\hundredx}{\ensuremath{100\times}}
\newcommand{\odsix}{\ensuremath{\textrm{OD}_{600}}}
\newcommand{\Pspo}{\ensuremath{\textrm{P}_{\textrm{\scriptsize SPO1-26}}}}

% Matrix/ vector shit
\let\oldhat\hat
\renewcommand{\vec}[1]{\mathbf{#1}}
\renewcommand{\hat}[1]{\oldhat{\mathbf{#1}}}
\let\oldcaption\caption
%\renewcommand{\caption}[2][]{\oldcaption[#1]{\small\textbf{#1.} #2\normalsize}}
\renewcommand{\caption}[2][]{\oldcaption[#1]{\textbf{#1.} #2}}

\newcommand{\mat}{\mathbf}
\newcommand{\invmat}[1]{\mat{#1}^{-1}}
\newcommand{\deltap}{\Delta \vec{p}}
\newcommand{\deltaL}{\Delta \vec{L}}
\newcommand{\deltag}{\Delta g}
\newcommand{\Iinv}{\mat{I}^{-1}}
\newcommand{\Minv}{\mat{M}^{-1}}
\newcommand{\crossmat}[1]{{\left[#1\right]}_{\times}}

\newtheorem{remark}{Remark}
\newtheorem*{note}{Note}

\begin{document}

\chapter{Biophysical modelling of rod-shaped bacterial cells}
\section{Biophysical model}
The measurements of individual cells in microcolonies of \ecoli\
presented above are consistent with the overview presented by
Koch~\cite{Koch:1995} that cells have fixed diameter and grow only along their
major axis, and the findings of Wang \etal~\cite{Wang:2010} showing that cells
are equally bisected on division.  The observation that cell diameter is
constant suggests that cells are \textit{rigid} except in the direction of
their growth axis. Following this suggestion, we formulated a simple model of
cellular growth on an agar substrate as follows:
\begin{enumerate}
\item Cells are rigid elongating capsules~\cite{Koch:1995}.
\item Viscous drag dominates inertia; cells are non-motile and move only when
subjected to a force~\cite{Purcell:1977}.
\item The distribution of cell lengths at division is constant, and each cell
divides in half~\cite{Wang:2010,Koch:1995}.
\item Each cell's unconstrained growth rate is proportional to its
length~\cite{Wang:2010}.
\item Growth is constrained by forces between cells and from viscous drag from
the substrate (figure \ref{fig:model}).  The ratio of the work required to
constrain growth to the work required to move a cell is defined by a parameter
$\Gamma$.
\end{enumerate}
In a rigid body mechanics model objects cannot be deformed and are completely
represented by their position and orientation. We extend this formulation by
allowing cell length to vary, adding an extra degree of freedom to accommodate
growth.

\section{Mathematical description of biophysical model}
\subsection{Motion of cells due to applied impulse}
Bacterial cells are modelled as capsule-shaped rigid bodies, that is cylinders
capped with hemispherical ends. Each cell is represented by its
position, orientation and length ($\vec{x}$, $\vec{\theta}$, $l$), and the
associated linear, angular, and growth velocity ($\vec{v}_\textrm{lin}$,
$\vec{w}$, $v_\textrm{growth}$). Here $v_\textrm{growth}$ is the
linear growth rate $dl/dt$. By extension of the usual rigid-body
formulation, we define the generalised position of a cell as
$\tilde{\vec{x}} =
\left(\begin{array}{lll}\vec{x} & \vec{\theta} & l \end{array}\right)^T$, and
the generalised velocity as $\tilde{\vec{v}} = d\tilde{\vec{x}}/dt =
\left(\begin{array}{lll}\vec{v}_\textrm{lin} & \vec{w} & v_\textrm{growth} \end{array}\right)^T$.
Similarly there is a generalised momentum associated with the motion of the cell
$\tilde{\vec{p}} =
\left(\begin{array}{lll}\vec{p} & \vec{L} & g \end{array}\right)^T$, with
linear, angular and growth components.
Cells grow by
acquiring growth momentum $g$ which causes extension in length.  For a physically
possible arrangement of cells, this extension in length cannot cause the cells
to overlap or intersect, leading to a system of constraints. It is the solution
of this system of constraints, which is derived below, that results in
computation of the correct arrangement of cells during colony growth.

In appendix \ref{app:eqmot} I show with the assumptions listed above, under a generalised impulse $\Delta
\tilde{\vec{p}}_i$ the change in generalised position of a cell $\Delta
\tilde{\vec{x}}_i$ is
determined by a matrix $\mat{M}_i$ such that:
\begin{align}
\Delta \tilde{\vec{ x}}_i &=
\invmat{M_i} \Delta \tilde{\vec{p}}_i \\
&=
\left[
\begin{array}{ccc}
\textrm{diag}(\frac{1}{\mu l_0}) & 0 & 0 \\
0 & \frac{12\mat{P_i}}{\mu l_0^3} & 0 \\
0 & 0 & \frac{1}{\gamma} \\
\end{array}
\right]
\left[
\begin{array}{l}
\deltap_i \\ \deltaL_i \\ \deltag_i
\end{array}
\right]%\\
%&=
%\mat{M}^{-1}\deltap
\label{eqn:motion2}
\end{align}
where $l_0$ is the length of cell $i$, and $\invmat{M_i}$ is written in block form - $\textrm{diag}(\frac{1}{\mu l_0})$
and $\frac{12\mat{P}}{\mu l_0^3}$ are $(3\times3)$, and $\frac{1}{\gamma}$ is
$(1\times1)$. The matrix $\mat{P}$ is the projection onto
the plane perpendicular to the cell axis $\hat{\vec{a}}$, i.e. $\mat{P}\deltaL
\equiv \deltaL -
\hat{\vec{a}}(\hat{\vec{a}} \cdot \deltaL)$.

\begin{note}
There are 7 degrees of freedom for each cell, hence generalised position,
velocity and impulse are $7\times1$ vectors. There are 3 components of centroid
position/velocity/impulse ($\vec{x}$, $\vec{v}_\mathrm{lin}$, $\Delta \vec{p}$),
3 components of angular position/velocity/impulse ($\vec{\theta}$, $\vec{w}$,
$\deltaL$), and 1 component of length/growth rate/growth impulse ($l$,
$v_\textrm{growth}$, $g$).
\end{note}

\begin{center}
\begin{figure}[p]\centering
\subfloat[]{\includegraphics[scale=0.75]{cellforces}}\qquad
\subfloat[]{\includegraphics{cellvel}}
\caption[Biophysical model of rod-shaped bacteria, forces and motion]
{Biophysical model of rod-shaped bacteria considers each cell as a capsule
(shown in two-dimensions). Forces on each cell (a) are generated by viscous drag
(red) and constraints at cell-cell contacts (black). The velocity of a point on
a cell consists of multiple components (b): the cell centre of mass (red circle)
moves with linear velocity $\vec{v}_\textrm{lin}$, the cell rotates with angular
velocity $\vec{w}$, and grows at instantaneous linear rate $v_\textrm{growth}$ causing all points to
move away from the centre of mass along the axis ($\hat{\vec{a}}$).}
\label{fig:model}
\end{figure}
\end{center}

\begin{center}
\begin{figure}[p]\centering
\includegraphics[scale=0.75]{celloverlap}
\caption{}
\label{fig:overlap}
\end{figure}
\end{center}

\subsection{Motion of points on cell surfaces}
We described above the change in generalised position of a cell due to a
generalised impulse, that is the change in its centroid location, angular
orientation, and length. Now consider the motion of a point on the
surface of this cell due to the change in generalised position. This point is
positioned according to vector $\vec{q}$ from the center
of the capsule $\vec{x}_i$, which has axis $\hat{\vec{a}}_i$.  Its original position is
$\vec{q}_w = \vec{x}_i + \vec{q}$.
For small displacments the cell axis $\hat{\vec{a}}_i$, the associated
projection matrix $\mat{P}_i$, and the cell length $l_0$ remain approximately constant, and
\[
\Delta \vec{q}_w \approx
\Delta \vec{ x_i} +
(\Delta \vec{\theta_i}\times\vec{q}) +
\frac{\Delta l_i}{l_0} \left(\hat{a}_i\cdot\vec{q}\right)\hat{a}_i
\]

And substituting from equation \ref{eqn:motion}:
\begin{align*}
\Delta \vec{ q}_w &\approx
\Delta \vec{ x_i} -
(\vec{q} \times \Delta \vec{\theta_i}) +
\frac{\left(\hat{a}_i\cdot\vec{q}\right)\hat{a}_i}{l_0} \Delta l_i \\
&\approx
\frac{1}{\mu l_0} \deltap_i -
(\vec{q} \times \frac{12\mat{P}_i}{\mu l_0^3}\deltaL_i) +
\frac{\left(\hat{a}_i\cdot\vec{q}\right)\hat{a}_i}{l_0}\frac{\deltag_i}{\gamma}
\end{align*}

Now project this vector along the direction vector $\hat{n}$ of interest to get
the position change in a particular direction:
\begin{align*}
\Delta q_{\hat{n}}
&=
\hat{n} \cdot
\left(
\frac{1}{\mu l_0} \deltap_i -
(\vec{q} \times \frac{12\mat{P}_i}{\mu l_0^3}\deltaL_i) +
\frac{\left(\hat{a}_i\cdot\vec{q}\right)\hat{a}_i}{l_0}\frac{\deltag_i}{\gamma}
\right) \\
&=
\frac{1}{\mu l_0}\hat{n}\cdot\deltap_i -
\frac{12}{\mu l_0^3}\left(\hat{n}\times\vec{q}\right)\cdot \mat{P}_i\deltaL_i +
\frac{\left(\hat{n}\cdot\hat{a}_i\right)\left(\hat{q}\cdot\hat{a}_i\right)}{l_0}\frac{\deltag_i}{\gamma} \\
\end{align*}

Which can be written in block matrix form as:
\begin{align*}
\Delta  q_{\hat{n}}
&=
\left[
\begin{array}{lll}
\hat{n}^{\textrm{T}} & -\left(\hat{n}\times\vec{q}\right)^{\textrm{T}} &
\frac{\left(\hat{n}\cdot\hat{a}_i\right)\left(\vec{q}\cdot\hat{a}_i\right)}{l_0} \\
\end{array}
\right]
\left[
\begin{array}{lll}
\frac{1}{\mu l_0} & 0 & 0 \\
0 & \frac{12\mat{P}_i}{\mu l_0^3} & 0 \\
0 & 0 & \frac{1}{\gamma} \\
\end{array}
\right]
\left[
\begin{array}{l}
\deltap_i \\ \deltaL_i \\ \deltag_i
\end{array}
\right]\\
&= \mat{B}_{\hat{n},\vec{q},i}\invmat{M}_i\Delta\tilde{\vec{p}}_i
\end{align*}

Here $\mat{B}_{\hat{n},\vec{q},i}$ is a matrix which represents
the geometric effect of a generalised position change on a point at $\vec{q}$
from the centre of cell $i$ along direction $\hat{n}$, and
matrix $\invmat{M}_i$ is the inverse drag matrix for cell $i$.
The expression for $\Delta q_{\hat{n}}$ represents the motion of the surface point
at position $\vec{q}$ relative to the capsule centre in the direction $\hat{n}$ that
results from application of generalised impulse $\Delta \tilde{\vec{p}}$. This quantity is central
to the formation of the constraints explained below.

\subsection{Effect of a set of applied impulses on cell-cell overlap distances}
Now consider two overlapping capsules $a$ and $b$, forming contact $k$. We find the contact points on
each cell $\vec{z}_{ak}$ and $\vec{z}_{bk}$ as described above (figure \ref{fig:overlap}). The
change in overlap distance $d_k$ is the difference in the movement of these
contact points along the contact normal $\hat{n}_k$. That is for generalised
impulses $\Delta\tilde{\vec{p}}_a$ and $\Delta\tilde{\vec{p}}_b$ applied to cell
$a$ and $b$,
\begin{align*}
\mat{B}_{ak} \Minv_a \Delta\tilde{\vec{p}}_a -
\mat{B}_{bk} \Minv_b \Delta\tilde{\vec{p}}_b
&=
\Delta d_k \\
\end{align*}
where $\mat{B}_{ak}$
and $\mat{B}_{bk}$ are as $\mat{B}$ matrices as defined above using the contact normal
$\hat{\vec{n}}_k$ as the direction vector, and the relative positions of
$\vec{z}_{ak}$ and $\vec{z}_{bk}$ from their respective cells.  Viewed in block matrix form:
\begin{align*}
\left[
\begin{array}{ll}
\mat{B}_{ak} \Minv_a &
-\mat{B}_{bk} \Minv_b \\
\end{array}
\right]
\left[
\begin{array}{l}
\Delta\tilde{\vec{p}}_a \\
\Delta\tilde{\vec{p}}_b \\
\end{array}
\right]
=
\left[
\begin{array}{l}
\Delta d_k
\end{array}
\right]
\end{align*}

In a system of multiple cells, there will be one such equation for each contact $k$ between two capsules.
To solve all these equations simultaneously, each equation becomes a
row in a matrix equation.  For example, with four cells and
three contacts, the
system might be:
\begin{align*}
\left[
\begin{array}{llll}
\mat{B}_{a1}\Minv_1 & -\mat{B}_{b1}\Minv_2 & 0 & 0 \\
\mat{B}_{a2}\Minv_1 & 0 & -\mat{B}_{c2}\Minv_3 & 0 \\
0 & \mat{B}_{b3}\Minv_1 & 0 & -\mat{B}_{d3}\Minv_4 \\
\end{array}
\right]
\left[
\begin{array}{c}
\Delta\tilde{\vec{p}}_a\\
\Delta\tilde{\vec{p}}_b\\
\Delta\tilde{\vec{p}}_c\\
\Delta\tilde{\vec{p}}_d\\
\end{array}
\right]
=
\left[
\begin{array}{c}
\Delta d_1\\
\Delta d_2\\
\Delta d_3\\
\end{array}
\right]
\end{align*}

\begin{align*}
\left[
\begin{array}{llll}
\mat{B}_{a1} & -\mat{B}_{b1} & 0 & 0 \\
\mat{B}_{a2} & 0 & -\mat{B}_{c2} & 0 \\
0 & \mat{B}_{b3} & 0 & -\mat{B}_{d3} \\
\end{array}
\right]
\left[
\begin{array}{llll}
\Minv_1 & 0 & 0 & 0 \\
0 & \Minv_2 & 0 & 0 \\
0 & 0 & \Minv_3 & 0 \\
0 & 0 & 0 & \Minv_1  \\
\end{array}
\right]
\left[
\begin{array}{c}
\Delta\tilde{\vec{p}}_a\\
\Delta\tilde{\vec{p}}_b\\
\Delta\tilde{\vec{p}}_c\\
\Delta\tilde{\vec{p}}_d\\
\end{array}
\right]
=
\left[
\begin{array}{c}
\Delta d_1\\
\Delta d_2\\
\Delta d_3\\
\end{array}
\right]\\
\end{align*}

and writing the system with single collected matrices and vectors,

\begin{equation*}
\mat{B} \invmat{M} \Delta \tilde{\vec{p}} = \Delta \vec{d}
\end{equation*}

\begin{note}
The elements $\mat{B}_{ij}\invmat{M}_i$ are computed by the \texttt{to\_ents}
(positive) and
\texttt{fr\_ents} (negative) arrays ($7\times1$ vectors stored in
\texttt{float8}) which are stored for each cell for each contact it
is involved in. We consider contacts to be directed from low to high index, and
so only the cell with the lowest index in each contact stores and calculates
these entries.
\end{note}

\begin{note}
In the analysis above I have split the matrices $\mat{B}$ and $\invmat{M}$, but
currently the combination $\mat{B}\invmat{M}$ is computed. It may be better to
split it so that we can more easily compute other forms of $\mat{B}$, e.g. for adhesion (see
below). Application of $\invmat{M}$ is already implemented by
\texttt{compute\_Minvx}.
\end{note}

\subsection{Growth of cells generates impulses to prevent overlap}
Growth of cells during a discrete time-step $\Delta t$ is modelled by simply
increasing the length of each cell by $v_\textrm{growth}\Delta t$. This
causes cells to overlap (figure \ref{fig:overlap}). For a physically correct
arrangement of cells, reaction or constraint forces must be generated to
eliminate this overlap.

A capsule of total length $l$ and radius $r$
is defined as the locus of points within a
radius $r$ of the line segment of length $l-2r$ along its axis.
When two such capsules overlap, we define the points of intersection on each cell
($\vec{z}$ in figure \ref{fig:overlap}) as follows. First we find the closest points
on each cell's axis line segment. The line joining these points then intersects
each cell surface at points labelled $\vec{z}$, where it is normal to the surface.
Hence we have defined contact points on each cell, and the normal to the contact
$\hat{\vec{n}}$. The distance of overlap $d$ for each contact is then determined
by the distance between the contact points along the contact normal direction
(figure \ref{fig:overlap}).

In order to constrain the system of cells to produce
a physically possible, non-overlapping arrangement, we must find the
impulse applied to each cell $\Delta \tilde{\vec{p}}$ that produces a change in
the overlap distance so that $d=0$ for each contact.
We have just described how these overlap distances are changed by a set of
applied impulses, and so we can now formulate the constraint system.

\subsection{Constraint formulation}
To form a physically possible arrangement of cells after increasing their length
we wish to apply constraint impulses to cells such that
\begin{eqnarray*}
\vec{d} + \Delta \vec{d} &=& 0\\
\Delta \vec{d} &=& -\vec{d}
\end{eqnarray*}
As shown above the change in overlaps $\Delta \vec{d}$ produced by a set of
generalised impulses $\Delta \tilde{\vec{p}}$ means that
\[
\mat{B}\invmat{M}\Delta\tilde{\vec{p}} = \mat{A}\Delta\tilde{\vec{p}} = \Delta
\vec{d}
\]
where $\mat{A}$ is a block matrix $n_{\textrm{contacts}} \times
n_{\textrm{cells}}$, $\Delta\tilde{\vec{p}}$ is a block vector $n_{\textrm{cells}} \times 1$ and
$\vec{d}$ is  $n_{\textrm{contacts}}\times 1$. Hence the constrained system is
given by:
\[
\mat{A}\Delta\tilde{\vec{p}} + \vec{d} = 0
\]
Hence we have formulated the system of constraints as a linear system. Given the
solution of this system, the change in generalised position of each cell is then $\Minv \Delta\tilde{\vec{p}}$.
However it is clear that in general this linear system does not have a unique solution.
To see this consider that cells may move perpendicular to the normal(s) of their
contact(s) without affecting $\Delta \vec{d}$.

\subsection{Energy minimising solution}
The least squares solution of the constraint system is:
\[\underset{\deltap}{\operatorname{argmin}}
\left\{
\left\|\mat{A}\Delta\tilde{\vec{p}} + \vec{d}\right\|^2 \right\} \]
%\[\mat{A}^{\textrm{T}}\mat{A}\deltap = -\mat{A}^{\textrm{T}}\vec{d}\]
where the expression $\mat{A}\Delta\tilde{\vec{p}} + \vec{d}$ is the vector of overlap
distances of each contact after application of impulses $\Delta \tilde{\vec{p}}$. If we interpret the least-squares
solution as an energy minimising solution, where the energy is
associated with the overlap of each contact. If the overlap at each
contact is $\vec{\epsilon}$, and if the energy is stored in an elastic
fashion, then this energy is:
\[\frac{1}{2}\vec{\epsilon}^T \mat{K} \vec{\epsilon}
 =  \frac{1}{2}\left\| \sqrt{\mat{K}}\vec{\epsilon} \right\|^2
 = \frac{1}{2}\left\| \sqrt{\mat{K}}\left(\mat{A}\Delta\tilde{\vec{p}} + \vec{d}\right) \right\|^2 \]
 where $\mat{K}$ is a diagonal stiffness coefficient matrix. So
it is implicit in the above minimisation that $\mat{K}=2\mat{I}$. If
all the stiffnesses are the same (i.e. $\mat{K} = K\mat{I}$), then
this makes no difference to the solution.  If some
contacts softer than others are required, the corresponding entry should be changed such that
$K_i<1$. We use this approach to approximate the
effect of a soft agarose substrate. This is a bit of distraction for now, so
lets just stick with $\mat{K}=\mat{I}$.

As explained above the matrix $\mat{A}$ will typically be poorly conditioned,
and the matrix $\mat{A}^T\mat{A}$ will be even worse. So we still can't get a
unique solution. The usual approach is to use regularisation to `select' a
solution.
Following the energy minimisation formalism the solution obtained should
minimise the work done or energy expended by the constraint impulses.
Given an impulse $\Delta \tilde{\vec{p}}$ and time step $\Delta t$, the average
force applied over the time step is $\vec{F}=\Delta \tilde{\vec{p}}/\Delta t$. The work
done by these forces is then
\begin{eqnarray*}
W &=& \vec{F}\Delta \tilde{\vec{x}} \\
&=& \vec{F} \invmat{M} \Delta \tilde{\vec{p}} \\
&=& \frac{1}{\Delta t} \Delta \tilde{\vec{p}}^\textrm{T} \invmat{M}\Delta \tilde{\vec{p}}
\end{eqnarray*}
Combining this
with the overlap energy described above leads to the problem
\[\underset{\deltap}{\operatorname{argmin}}
\left\{
\left\|\mat{A}\Delta\tilde{\vec{p}} + \vec{d}\right\|^2 + \frac{\alpha}{\Delta
t} \Delta \tilde{\vec{p}}^\textrm{T}\invmat{M}\Delta \tilde{\vec{p}} \right\} \]
where $\alpha$ is a regularisation parameter. In this case we can interpret
$\alpha$ physically as the relative energy cost of cell-cell overlap (e.g. due
to deformation of the cell wall) compared to the energy cost in overcoming drag.
Differentiating the above total energy, and setting to zero gives:
\begin{equation}
\left(\mat{A}^{\textrm{T}}\mat{A} + \frac{\alpha}{\Delta t}\Minv\right)\Delta \tilde{\vec{p}} = -\mat{A}^{\textrm{T}}\vec{d}
\label{eqn:finalsystem}
\end{equation}

It is now clear that \textit{any} quadratic energy term can be included in the
system by forming the appropriate matrix $\mat{H}$ such that the energy is given
by $E = \|\mat{H}\Delta \tilde{\vec{p}} \|^2$. Or maybe more usefully, the
energy can be defined directly in terms of change in generalised position
$\Delta \tilde{\vec{x}}$ as $E = \|\mat{G}\Delta \tilde{\vec{x}} \|^2 =
\|\mat{G}\invmat{M}\Delta \tilde{\vec{p}} \|^2$, and the linear system is then
given by
\begin{equation}
\left(\frac{1}{\alpha}\mat{A}^{\textrm{T}}\mat{A} + \frac{1}{\Delta t}\Minv +
\invmat{M}\mat{G}^\textrm{T} \mat{G}\invmat{M} \right) \Delta \tilde{\vec{p}} = -\mat{A}^{\textrm{T}}\vec{d}
\label{eqn:finalsystem}
\end{equation}
where I have moved the $\alpha$ to make clear its physical interpretation as the
relative cost of overlap. Note also I have used the fact that
$\mat{M}^{-\textrm{T}} \equiv \invmat{M}$ because it is diagonal except for terms
involving the projection matrices $\mat{P}_i$, which are themselves symmetric by
definition.

\begin{note}
Writing out the matrix $\mat{A}=\mat{B}\invmat{M}$ and $\Delta \tilde{\vec{p}} =
\mat{M}\Delta \tilde{\vec{x}}$ in the above system gives a
factor of $\invmat{M}$ which could be eliminated. However, this is actually
acting as a right-preconditioner. Similarly expanding
$\mat{A}^\textrm{T}=\invmat{M}\mat{B}^\textrm{T}$ gives us left-preconditioning.
Hence we have split-preconditioning.
I'm pretty sure we did some testing and the preconditioning did improve
the condition number.
\end{note}

\begin{note}
The factor of $1/\Delta t$ is not computed in the current code. At the
time we were considering a kind of ad hoc regularisation term rather than
explicitly minimising physical work. Adding the $1/\Delta t$ makes sense.
\end{note}

\begin{note}
In the current code the viscosity from substrate per unit length of cells $\mu$
is set at $\mu=1$ -
i.e. it is factored out of the $\invmat{M}$ matrices. This means that the
parameter $\alpha$ incorporates the value of $\mu$ into the ratio of drag from the substrate to
resistance to deformation of cells. It seems sensible that this should be the
same as the ratio $\Gamma = \mu/\gamma$, where $\gamma$ is the resistance to
change in length (see appendix). This would be nice because the whole model
would then be parameterised only by $\Gamma$. Need to think about this a bit more.
\end{note}

\section{Adhesion}
Now we want to form a matrix $\mat{G}$ that adds a quadratic energy to the
system to simulate adhesion. Consider adhesion that acts only in the plane of
the cell surface at each contact (tangent plane), such that $E_k = (\Delta
d^{\perp}_k)^2 = \|\Delta \vec{d}^\perp_k\|^2$ where $\Delta \vec{d}^\perp_k$ is
the movement vector in the tangent plane
between contact points on each cell involved in contact $k$. This is a
$3\times1$ vector that is the projection of the total movement into the tangent plane of
the contact. In this case
\begin{equation*}
\mat{G}\invmat{M}\Delta \tilde{\vec{p}} = \Delta \vec{d}^\perp
\end{equation*}
where the matrices and vectors are in block form, with one block per cell or
contact.

This matrix $\mat{G}$ is closely related to the above described $\mat{B}$
matrix, which projects the movement of each contact point along its respective contact
normal $\hat{n}_k$. The tangent plane is perpendicular to this normal vector.
For cell $i$ and contact $k$,
\begin{equation*}
\mat{B}_{ik} =
\left[
\begin{array}{lll}
\hat{n}_k^{\textrm{T}} & -\left(\hat{n}_k\times\vec{z}_{ik}\right)^{\textrm{T}} &
\frac{\left(\hat{n}_k\cdot\hat{a}_i\right)\left(\vec{z}_{ik}\cdot\hat{a}_i\right)}{l_0} \\
\end{array}
\right]
\end{equation*}
where $\mat{B}_{ik}$ is a $1\times7$ matrix. The matrix blocks $\mat{G}_{ik}$
are $3\times7$ and given by
\begin{eqnarray*}
\mat{G}_{ik} &=&
\left[
\begin{array}{lll}
\mat{Q} & \mat{Q}\mat{Z}^\times_{ik} &
\left[\hat{a}_i - \hat{n}_k\left(\hat{n}_k\cdot\hat{a}_i\right)\right] \frac{\left(\vec{z}_{ik}\cdot\hat{a}_i\right)}{l_0} \\
\end{array}
\right]\\
&=&
\left[
\begin{array}{lll}
\mat{Q} & \mat{Q}\mat{Z}^\times_{ik} &
\frac{\left(\vec{z}_{ik}\cdot\hat{a}_i\right)}{l_0} \mat{Q}\hat{a}_i \\
\end{array}
\right]\\
&=&
\mat{Q}
\left[
\begin{array}{lll}
\mat{I} & \mat{Z}^\times_{ik} &
\frac{\left(\vec{z}_{ik}\cdot\hat{a}_i\right)}{l_0} \hat{a}_i \\
\end{array}
\right]
\end{eqnarray*}
Here $\mat{Q}$ is the $3\times3$ matrix projecting into the tangent plane, so that for any vector
$\vec{u}$
\begin{equation}
\mat{Q}\vec{u} = \vec{u} - \hat{n}_k\left(\hat{n}_k \cdot \vec{u}\right)
\label{eqn:Q}
\end{equation}
and $\mat{Z}^\times_{ik}$ is the matrix representing the cross product
\begin{equation}
\mat{Z}^\times_{ik} \vec{u} = \vec{z}_{ik} \times \vec{u}
\label{eqn:Z}
\end{equation}
These first two blocks ($\mat{Q}$ and $\mat{Q}\mat{Z}^\times_{ik}$) of $\mat{G}_{ik}$ are $3\times3$ and the third
($\frac{\left(\vec{z}_{ik}\cdot\hat{a}_i\right)}{l_0} \mat{Q}\vec{a}_i$) is
$3\times1$.

\begin{note}
We could explicitly store the $3\times3$ blocks and apply them as matrix
multiplications. However, it is more efficient to do the computation directly
using equations \ref{eqn:Q} and \ref{eqn:Z}.
\end{note}

\begin{note}
The result of applying each block $\mat{G}_{ik}$ to the cells impulse $\Delta
\tilde{\vec{p}}_i$ is a $3\times1$ vector. Hence the array storing the result of
$\mat{G}\Delta \tilde{\vec{p}}$ must use \texttt{float4} types. This might seem
counterintuitive since the distances are projected onto a plane, but note that
the tangent plane of each contact is in general different.
\end{note}

\section{Viscous adhesion}
Because we recompute the closest points for contacts at each time step the above
described adhesion energy will be completely dissipated. This is effectively a
viscous model in which
\begin{equation*}
\vec{F} = C \frac{d\vec{x}}{dt} \approx C \frac{\Delta \vec{x}}{\Delta t}
\end{equation*}
and thus $E \approx \frac{1}{\Delta t}C\|\Delta \vec{x}\|^2$. So that using the
above analyis the energy from viscous adhesion is given by
\begin{equation*}
E = \left(\frac{1}{\Delta t}C \right) \|\mat{G}\invmat{M}\Delta \tilde{\vec{p}}\|^2
\end{equation*}
for the same viscosity coefficient at each contact. More generally,
\begin{equation*}
E = \left(\frac{1}{\Delta t} \right)
\Delta \tilde{\vec{p}}^\textrm{T}\invmat{M}\mat{G}^\textrm{T}\mat{C}\mat{G}\invmat{M}\Delta \tilde{\vec{p}}
\end{equation*}
where $\mat{C}$ is a diagonal matrix of viscosity coefficients for each contact.
In practice this matrix is best implemented by just scaling each block
$\mat{G}_{ik}$ by the appropriate coefficient.

\begin{note}
The scaling by $1/\Delta t$ is required to make this purely viscous model physically sensible.
To implement a purely elastic model of contact would require keeping track of
the original point of contact between each pair of cells.
\end{note}

\section{Implementation in CellModeller4}
The following table describes which functions or OpenCL kernels perform which
calculations and data arrays that contain the results, using the mathematical notation above.

\begin{table}[H]
\begin{tabular}{c c  p{4in}}
\toprule
Function/kernel & Maths & Description\\
\midrule
\texttt{find\_contacts} & $\vec{d}$, $\hat{\vec{n}}$ & Finds overlapping cells
and computes the overlap distance $d_k$ and surface normal $\hat{\vec{n}}_k$ for
each contact $k$.\\
\addlinespace
\texttt{rel\_dists} & $\vec{d}$ & Stores the vector of overlap distances, one
element (\texttt{float}) per contact.\\
\addlinespace
\texttt{compute\_Minvx} & $\invmat{M}$ & Applies the inverse drag matrix to some
given vector of length $n_\mathrm{cells}$ and type \texttt{float8} (only first 7
elements are used). It does this per cell, by applying its $\invmat{M}_i$ matrix
block.\\
\bottomrule
\end{tabular}
\end{table}

% Long maths in appendix for ease of reading
\begin{appendices}
\chapter{Equations of motion}
\label{app:eqmot}
Consider a capsule with main axis $\hat{\vec{a}}$, mass $m$, and length $l$.
When in motion the capsule is associated with linear ($\vec{v}_\textrm{lin}$),
angular ($\vec{w}$), and growth ($v_\textrm{growth}$) velocities (figure
\ref{fig:model}b).
Now consider a point on the
surface of the capsule with position $\vec{r}$ relative to the
capsule's center of mass.  The velocity of that point is given by:
\begin{align*}
\vec{v}_{r}
&=
\vec{v}_\textrm{lin}
+ \vec{w} \times \vec{r}
+ \frac{1}{2}\frac{\hat{a}\cdot\vec{r}}{l_0}\hat{a}v_\textrm{growth} \\
&=
\vec{v}_\textrm{lin}
- \vec{r} \times \vec{w}
+ \frac{1}{2}\frac{\hat{a}\cdot\vec{r}}{l_0}\hat{a}v_\textrm{growth}
\end{align*}
where $l_0$ is the current capsule length. The first term in this expression is
the linear velocity of the whole cell, the second term is the angular component,
and the final term is due to growth. Growth is assumed to occur by extension of
all parts of the cell wall along the axis, meaning that any point on the cell is
moving away from the centre of mass at a rate proportional to its distance from
the centre. The maximum velocity due to growth then occurs at the tips of the cell
which are each moving away from the centre of mass at rate
$\frac{1}{2}v_\textrm{growth}$ along the cell axis $\hat{\vec{a}}$.

Each point on the cell surface experiences viscous drag from the growth substrate due
to its motion.  The viscous drag force on a differential length $dr$ of the
capsule is approximately given by: \[d\vec{F} = -\mu dr \vec{v}_r\] where $\mu$
is a drag coefficient per unit length.  To understand the motion of the capsule
shape, initially at rest, after the application of a generalised impulse
$\Delta\tilde{\vec{p}}$, which can be divided into linear, angular and growth
components $\left(\begin{array}{lll}\deltap & \deltaL &
\deltag\end{array}\right)^T$, we must find the net force on the capsule due to
viscous drag at each point on its surface. Given this force, Newton's laws can
be applied to find the change in generalised position.

\section{Linear impulse $\deltap$}
Approximating the forces on the differential length element at the average
position, so that position $\vec{r}=r\hat{a}$ lies on the axis of the cell:

\begin{align*}
d\vec{F} &= -\mu dr \vec{v}_r\\
d\vec{F} &= -\mu dr
\left(
\vec{v}_\textrm{lin}
- \vec{r} \times \vec{w}
+ \frac{1}{2}\frac{\hat{a}\cdot\vec{r}}{l_0}\hat{a}v_\textrm{growth}
\right)
\\
d\vec{F} &= -\mu dr
\left(
\vec{v}_\textrm{lin}
- r\hat{a} \times \vec{w}
+ \frac{1}{2}\frac{r}{l_0}\hat{a}v_\textrm{growth}
\right)
\\
\end{align*}

The total force on the capsule is given by the integral over the length of
the capsule from $-\frac{l_0}{2}$ to $+\frac{l_0}{2}$. The
second two terms in the expression for velocity are odd in $r$ and
so will cancel when integrated so the total force on
the capsule is:
\begin{align*}
\vec{F} &= -\mu \vec{v}_\textrm{lin} \int_{-\frac{l_0}{2}}^{+\frac{l_0}{2}} dr \\
&= -\mu \vec{v}_\textrm{lin} l_0 \\
\end{align*}
And following Newton's laws:
\begin{align*}
m \frac{d\vec{v}_\textrm{lin}}{dt} &= -\mu  l_0 \vec{v}_\textrm{lin}\\
\vec{v}_\textrm{lin} &= v_0\exp{(-\frac{\mu l_0}{m}t)} \\
&= \frac{\deltap}{m}\exp{(-\frac{\mu l_0}{m}t)} \\
\end{align*}

The change in position of the centre of mass of the capsule $\vec{x}$ due to
linear impulse $\deltap$, and thus the change in position of any point on the
cell is then found as:
\begin{align*}
\frac{d\vec{x}}{dt} &= \frac{\deltap}{m}\exp{(-\frac{\mu l_0}{m}t)} \\
%x &= -\frac{m\deltap}{m\mu l}\exp{(-\frac{\mu l}{m}t)} + C \\
\Delta \vec{x} &= \frac{\deltap}{\mu l_0}\left(1-\exp{(-\frac{\mu l_0}{m}t)}\right) \\
\end{align*}
In the case of a bacterium, inertial effects are very small compared to drag
forces~\cite{Purcell:1977} meaning $m/\mu \approx 0$, so that the
exponential term is nearly zero and so:
\begin{equation}
\Delta \vec{x} \approx \frac{\deltap}{\mu l_0}
\label{eqn:deltax}
\end{equation}

\section{Angular impulse $\Delta\vec{L}$}
Now consider the rotational motion of the cell.  The torque applied at a
differential section of the cell at position $\vec{r}=r\hat{\vec{a}}$ is:
\begin{align*}
d\vec{\tau} &= \vec{r} \times d\vec{F} \\
&= \vec{r} \times -\mu dr \vec{v}_r \\
&= \vec{r} \times -\mu dr \left(
\vec{v}_\textrm{linear}
- r\hat{a} \times \vec{w}
+ \frac{1}{2}\frac{r}{l_0}\hat{a}v_\textrm{growth}
\right) \\
\end{align*}

The drag force due to linear velocity makes no total
contribution to torque (the contributions on the two sides of the
capsule's center of mass cancel) and neither does drag force due to
growth velocity (it is parallel to $\vec{r}$).  Note that the triple product:
\begin{align*}
\vec{r}\times(\vec{w}\times\vec{r}) &= \vec{w}(\vec{r}\cdot\vec{r}) - \vec{r}(\vec{r}\cdot\vec{w}) \\
&= \vec{w}r^2 - \vec{r}(\vec{r}\cdot\vec{w})
\end{align*}
and so:
\begin{align*}
d\tau &= -\mu \left( \vec{w}r^2 - r\hat{a}(r\hat{a}\cdot\vec{w}) \right) dr \\
    &= -\mu \left( \vec{w} - \hat{a}(\hat{a}\cdot\vec{w}) \right) r^2 dr \\
\tau &= -\mu \left( \vec{w} - \hat{a}(\hat{a}\cdot\vec{w}) \right) \int_{-\frac{l_0}{2}}^{\frac{l_0}{2}}r^2dr \\
&=-\frac{1}{12}\mu l_0^3 \left( \vec{w} - \hat{a}(\hat{a}\cdot\vec{w}) \right) = \mat{I}\frac{d\vec{w}}{dt} \\
\end{align*}

The expression $\vec{w} - \hat{a}(\hat{a}\cdot\vec{w})$ represents a projection of the vector $\vec{w}$ onto the
plane perpendicular to the axis $\hat{a}$, and can be written as $\mat{P}\vec{w}$ where $\mat{P}$ is the
appropriate projection matrix:

\begin{align*}
\tau &=-\frac{1}{12}\mu l_0^3 \mat{P}\vec{w}  = I\frac{d\vec{w}}{dt} \\
\vec{w} &= \vec{w}_0 \exp{\left( -\frac{\mu l_0^3 \mat{P}t}{12 I} \right)} \\
\end{align*}

For a projection matrix $\mat{P}$, an exponential of the form $\exp{(k\mat{P})}$, where $k$ is a scalar, is:
\[\exp{(k\mat{P})} = \mat{P}\exp{(k)} +  \left(\mat{I}-\mat{P}\right)\]
 so:
\[\vec{w} = \mat{P} \vec{w}_0 \exp{\left( -\frac{\mu l_0^3 t}{12 I} \right)} + \left(\mat{I}-\mat{P}\right) \vec{w}_0\]

In which the second term $\left(\mat{I}-\mat{P}\right) \vec{w}_0$ is the angular velocity about the cell axis $\hat{a}$.
Approximating the capsule as a `slender rod' with no rotational inertia about its axis
the second term can be ignored, and:

\begin{align*}
%\frac{dw}{w} &= -\frac{\mu l^3}{12I}dt \\
%\log{w} &= -\frac{\mu l^3}{12I}t + C \\
\vec{w} &= \mat{P} \vec{w}_0\exp{(-\frac{\mu l_0^3}{12I}t)} \\
&= \frac{\mat{P} \deltaL}{I}\exp{(-\frac{\mu l_0^3}{12I}t)} = \frac{d\vec{\theta}}{dt} \\
%\Delta \theta &= -\frac{12I \mat{P} \deltaL}{I\mu l_0^3}\exp{(-\frac{\mu l_0^3}{12I}t)} + C \\
%&= \frac{12 \mat{P} \deltaL}{\mu l_0^3}\left(1-\exp{(-\frac{\mu l_0^3}{12I}t)}\right) \\
\end{align*}

Making the same approximation for negligible inertia as above $I/\mu \approx
0$ gives:

\begin{equation}
\Delta \vec{\theta} \approx \frac{12 \mat{P} \deltaL}{\mu l_0^3}
\label{eqn:deltatheta}
\end{equation}

So, the effect of a rotational impulse is projected onto the plane of the cell axis - that is, it only causes rotation
about the other two axes.

\section{Growth impulse $\Delta g$}
A growth impulse tends to elongate the cell, and is resisted by compressive forces along the axis of the capsule due to drag:

\begin{align*}
    F_c &= \mu \frac{1}{2} \left[ \int_{0}^{\frac{l_0}{2}}{\frac{r}{l_0}v_\textrm{growth}} dr
    - \int_{-\frac{l_0}{2}}^{0}{\frac{r}{l_0}v_\textrm{growth}} dr \right] \\
    &= \frac{\mu}{2} v_\textrm{growth} \left[ \int_{0}^{\frac{l_0}{2}}{\frac{r}{l_0}} dr
    - \int_{-\frac{l_0}{2}}^{0}{\frac{r}{l_0}} dr \right] \\
    &= \frac{\mu l_0}{8} v_\textrm{growth}
\end{align*}

%Here we have written the same drag coefficient $\mu$. We incorporate forces from the cells growth,
% and resulting resistance to length change, as a drag term. Thus rewriting:
This is the drag due to interaction with the substrate. In order to
model the cell's resistance to change in length due to its own structural and
mechanical properties, I incorporate an additional
drag term:
\begin{align*}
    F_c = \left( \frac{\mu l_0}{8} + \gamma \right) v_\textrm{growth}
\end{align*}
where $\gamma$ describes cell growth drag. Following the same
reasoning as above, integrating and neglecting inertial effects, and assuming
that $\Delta l << l$ so that $l_0$ can be taken as constant, the effect of
growth impulse $\delta g$ on cell length is:
\begin{align*}
    \Delta l \approx \frac{8 \Delta g}{\mu l_0 + 8 \gamma}
\end{align*}
where $\Delta g$ is the applied growth impulse. Assuming that the resistance of
the cell to deformation dominates viscous drag from the substrate, then
this can be reduced to:
\begin{equation}
    \Delta l \approx \frac{\Delta g}{\gamma}
    \label{eqn:deltal}
\end{equation}

\section{Motion due to generalised impulse}
%We have shown above that the change in the capsule generalised position vector
Combining the results for each component of motion from equations
\ref{eqn:deltax}, \ref{eqn:deltatheta} and \ref{eqn:deltal}, the change in generalised
position $\tilde{\vec{x}} = \left(\begin{array}{lll}\vec{x} & \vec{\theta} & l
\end{array}\right)^T$ after the application of a general impulse $\Delta\tilde{\vec{p}} =
\left(\begin{array}{lll} \deltap & \deltaL & \deltag\end{array}\right)^T$
can be written in block matrix form as:
\begin{align}
\Delta \tilde{\vec{ x}} &=
\left[
\begin{array}{l}
\frac{\deltap}{\mu l_0} \\
\frac{12\mat{P}\deltaL}{\mu l_0^3} \\
\frac{\deltag}{\gamma} \\
\end{array}
\right] \\
&=
\left[
\begin{array}{lll}
\textrm{diag}(\frac{1}{\mu l_0}) & 0 & 0 \\
0 & \frac{12\mat{P}}{\mu l_0^3} & 0 \\
0 & 0 & \frac{1}{\gamma} \\
\end{array}
\right]
\left[
\begin{array}{l}
\deltap \\ \deltaL \\ \deltag
\end{array}
\right]%\\
%&=
%\mat{M}^{-1}\deltap
\label{eqn:motion}
\end{align}

\end{document}


\documentclass[12pt, a4paper]{article}
\usepackage[margin=0.5in]{geometry}

\usepackage{color}
\usepackage[dvipsnames]{xcolor}
\usepackage{hyperref}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    urlcolor=blue,
    linktoc=all
}

\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{amssymb}
\usepackage{cancel}
\usepackage{bm}
\usepackage{dsfont}
\usepackage{graphicx}
\usepackage{graphics}
\usepackage{xfrac}
\usepackage{array}
\setcounter{MaxMatrixCols}{40}

\usepackage{enumerate}
\usepackage{enumitem}
\usepackage{multirow}

%inclusions carried over from past class homework formats
\usepackage{units}
\usepackage{fullpage}
\usepackage{alltt}
\usepackage{mathrsfs}
\usepackage{xcolor}
\usepackage{soul}

\usepackage{pgfplots}

\DeclarePairedDelimiter{\abs}{\lvert}{\rvert}
\newcommand*{\fontCourier}{\fontfamily{pcr}\selectfont}
\newcommand*\mean[1]{\overline{#1}}
\newcommand\scalemath[2]{\scalebox{#1}{\mbox{\ensuremath{\displaystyle #2}}}}

\setcounter{tocdepth}{5}
\setcounter{secnumdepth}{5}

\usepackage{pdfpages}
\usepackage{Sweave}
\begin{document}
%\includepdf{TitlePage_MastersThesis}
%\includepdf{ThesisApprovalPage}
\input{Regression-concordance}

%\tableofcontents
%\newpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%INTRODUCTION
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Least Squares Estimation with Example (Hoff p. 149ff.)}

Regression modeling is concerned with describing how the sampling istribution of one random variable $Y$ varies with another variable or set of variables $\mathbf{x} = \left(x_1,...,x_p\right)$.  Specifically, a regression model postulates a form for $p(y|\mathbf{x})$, the conditional distribution of $Y$ given $\mathbf{x}$.  Estimation of $p(y|\mathbf{x})$ is made using data $y_1,...,y_n$ that are gathered under a variety of conditions $\mathbf{x}_1,...,\mathbf{x}_n$.

The normal linear regression model specifies that, in addition to $E[Y|\mathbf{x}]$ being linear, the sampling variability around the mean is i.i.d. normal:

\begin{flalign*}
    \epsilon_1,...,\epsilon_n &\overset{\text{i.i.d}}{\sim} \text{normal}\left(0,\sigma^2\right)\\
    Y_i &= \beta^T \mathbf{x}_i + \epsilon_i
\end{flalign*}

This model provides a complete specification of the joint probability density of observed data $y_1,...,y_n$ conditional upon $\mathbf{x}_1,...,\mathbf{x}_n$ and values of $\beta$ and $\sigma^2$:

\begin{flalign}
    p\left(y_1,...y_n|\mathbf{x}_1,...,\mathbf{x}_n,\beta,\sigma^2\right) &= \prod_{i=1}^n p\left(y_i|\mathbf{x}_i,\beta,\sigma^2\right) \nonumber\\
    &= \left(2\pi\sigma^2\right)^{-n/2}\text{exp}\left\{-\frac{1}{2\sigma^2}\sum_{i=1}^n\left(y_i - \beta^T\mathbf{x}_i\right)^2\right\} \label{conditional_density}
\end{flalign}

Another way to write this joint probability density is in terms of the multivariate normal distribution:  Let $\mathbf{y}$ be the $n$-dimensional column vector $\left(y_1,...,y_n\right)^T$ and let $\mathbf{X}$ be the $n \times p$ matrix whose $i$th row is $\mathbf{x}_i = \left\{ x_{i,1}, x_{i,2}, ..., x_{i,p} \right\}$.  Then the normal regression model is

$$\{\mathbf{y}|\mathbf{X},\beta,\sigma^2\} \sim \text{multivariate normal}\left(\mathbf{X}\beta,\sigma^2\mathbf{I}\right),$$

where $\mathbf{I}$ is the $p \times p$ identity matrix and

\begin{equation*}
    \mathbf{X}\beta =
    \begin{pmatrix}
        \mathbf{x}_1 \\
        \mathbf{x}_2 \\
        \vdots  \\
        \mathbf{x}_n
    \end{pmatrix}
    \begin{pmatrix}
        \beta_1 \\
        \beta_2 \\
        \vdots \\
        \beta_p
    \end{pmatrix}
    =
    \begin{pmatrix}
        \beta_1 x_{1,1} + \cdots + \beta_p x_{1,p} \\
        \vdots \\
        \beta_1 x_{n,1} + \cdots + \beta_p x_{n,p} \\
    \end{pmatrix}
    =
    \begin{pmatrix}
        E\left[Y_1|\mathbf{\beta},\mathbf{x}_1\right] \\
        \vdots \\
        E\left[Y_n|\mathbf{\beta},\mathbf{x}_n\right] \\
    \end{pmatrix}
\end{equation*}

The density (\ref{conditional_density}) depends on $\beta$ through the residuals $\left(y_i - \beta^T\mathbf{x}_i\right)$.  We compute the ordinary least squares estimates

$$\hat{\beta}_{ols} = \left(\mathbf{X}^T\mathbf{X}\right)^{-1}\mathbf{X}^T\mathbf{y}$$

and

$$\hat{\sigma}^2_{ols} = \frac{SSR\left(\hat{\beta}_{ols}\right)}{(n-p)} = \frac{\sum\left(y_i - \hat{\beta}_{ols}^T x_i\right)^2}{(n-p)}.$$

\clearpage

\textit{Example:  Oxygen uptake (from Kuehl (2000), Hoff p. 149ff)}

Twelve healthy men who did not exercise regularly were recruited to take part in a study of the effects of two different exercise regimens on oxygen uptake.  Six of the twelve men were randomly assigned to a 12-week flat-terrain running program, and the remaining six were assigned to a 12-week step aerobics program.  The maximum oxygen uptake of each subject was measured (in liters per minute) while running on an inclined treadmill, both before and after the 12-week program.  Of interest is how a subject's change in maximal oxygen uptake may depend on which program they were assigned to.  However, other factors, such as age, are expected to affect the change in maximal uptake as well.  The results are shown here:

\includegraphics{Regression-002}

Hoff's regression model:

% \begin{flalign*}
%     Y_i &= \beta_1x_{i,1} + \beta_2x_{i,2} + \beta_3x_{i,3} + \beta_4x_{i,4} + \epsilon_i, \text{ where}\\
%     x_{i,1} &= 1 \text{ for each subject } i\\
%     x_{i,2} &= 0 \text{ if subject } j \text{ is on the running program, } 1 \text{ if on aerobic}\\
%     x_{i,3} &= \text{ age of subject } i\\
%     x_{i,4} &= x_{i,2} \times x_{i,3}
% \end{flalign*}

%\begin{subequations}
    \begin{align}
        Y_i &= \beta_1x_{i,1} + \beta_2x_{i,2} + \beta_3x_{i,3} + \beta_4x_{i,4} + \epsilon_i, \text{ where} \label{example_model}\\
        x_{i,1} &= 1 \text{ for each subject } i \nonumber \\
        x_{i,2} &= 0 \text{ if subject } j \text{ is on the running program, } 1 \text{ if on aerobic} \nonumber \\
        x_{i,3} &= \text{ age of subject } i \nonumber \\
        x_{i,4} &= x_{i,2} \times x_{i,3} \nonumber
    \end{align}
%\end{subequations}

% \begin{subequations}
% \begin{align}
% \frac{\mathrm{d} x}{\mathrm{d} t} &=\sigma(y-x)\label{eqn:line-1} \\
% \frac{\mathrm{d} y}{\mathrm{d} t} &=x(\rho-z)-y\label{eqn:line-2} \\
% \frac{\mathrm{d} z}{\mathrm{d} t} &=x y-\beta z
% \end{align}
% \label{eqn:all-lines}
% \end{subequations}
%
% Look at the first line \ref{eqn:line-1}, and now look at the second line \ref{eqn:line-2}. They are both part of the whole system \ref{eqn:all-lines}.

Under this model the conditional expectations of $Y$ for the two different levels of $x_{i,1}$ are

\begin{flalign*}
    E[Y|\mathbf{x}] &= \beta_1 + \beta_3 \times \text{ age if } x_1 = 0, \text{ and}\\
    E[Y|\mathbf{x}] &= \left(\beta_1 + \beta_2\right) + \left(\beta_3 + \beta_4\right) \times \text{ age if } x_1 = 1
\end{flalign*}

In other words, the model assumes that the relationship is linear in age for both exercise groups, with the difference in intercepts given by $\beta_2$ and the difference in slopes given by $\beta_4$.  If we assumed that $\beta_2 = \beta_4 = 0$, then we would have identical lines for both groups.  If we assumed $\beta_4 =  0$ then we would have a different line for each group but they would be parallel.  Allowing all coefficients to be non-zero gives us two unrelated lines.  Some different possibilities are depicted graphically below:\\\\

\includegraphics{Regression-003}

Let's find the least squares regression estimates for the model in \ref{example_model}, and use the results to evaluate the differences between the two exercise groups.  The ages of the 12 subjects, along with their observed changes in maximal oxygen uptake, are

\begin{flalign*}
    \mathbf{x}_3 &= (23,22,22,25,27,20,31,23,27,28,22,24)\\
    \mathbf{y}   &= (-0.87,-10.74,-3.27,-1.97,7.50,-7.25,17.05,4.96,10.40,11.05,0.26,2.51),
\end{flalign*}

\noindent with the first six elements of each vector corresponding to the subjects in the running group and the latter six corresponding ot sujects in the aerobics group.  After constructing the $12 \time 4$ matrix $\mathbf{X}$ out of the vectors $\mathbf{x}_1,\mathbf{x}_2,\mathbf{x}_3,\mathbf{x}_4$ defined as in (\ref{example_model}), the matrices $\mathbf{X}^T\mathbf{X}$ and $\mathbf{X}^T\mathbf{y}$ can be computed, from which we get $\beta_{ols} = (-51.29,13.11,2.09,-0.32)^T$:\\

\begin{Schunk}
\begin{Sinput}
> n <- length(y)
> x1 <- rep(1,n)
> x4 <- x2*x3
> X = cbind(x1,x2,x3,x4)
> p = ncol(X)
> beta.ols<- solve(t(X)%*%X)%*%t(X)%*%y
> sig2.ols = (t(y-X%*%beta.ols)%*%(y-X%*%beta.ols)/(n-p))[1]
> sig2.ols
\end{Sinput}
\begin{Soutput}
[1] 8.542477
\end{Soutput}
\begin{Sinput}
> #sampling variance-covariance matrix of beta.ols:
> SIG2.ols = solve(t(X)%*%X)*sig2.ols
> #standard errors for the components of beta.ols:
> SE.ols = sqrt(diag(SIG2.ols))
> betadata = cbind(beta.ols,SE.ols)
> colnames(betadata) = c("beta.ols","SE.ols")
> betadata
\end{Sinput}
\begin{Soutput}
      beta.ols     SE.ols
x1 -51.2939459 12.2522126
x2  13.1070904 15.7619762
x3   2.0947027  0.5263585
x4  -0.3182438  0.6498086
\end{Soutput}
\end{Schunk}

This means that the estimated linear relationship between uptake and age has an intercept and slope of -51.29 and 2.09 for the running group, and -51.29 + 13.11 = -38.18 and 2.09 - 0.32 = 1.77 for the aerobics group.  These two lines are plotted in the fourth panel of Figure XX.  We obtain unbiased estimate $\sigma^2 = SSR(\hat{\beta}_{ols})/(n-p) = 8.54$, and use this to compute the standard error of the components of $\hat{\beta}_{ols}$, which are 12.25, 15.76, 0.53, and 0.65, respectively.  comparing the values of $\hat{\beta}_{ols}$ to their standard errors suggests that the evidence for differences between the two exercise regimens is not very strong.

\clearpage

\section{Bayesian Estimation for a Regression Model (Hoff p. 154ff)}

    \subsection{A semiconjugate prior distribution}
    A semiconjugate prior distribution for $\beta$ and $\sigma^2$ is used when there is information available about the parameters.  The sampling density of the data (Equation \ref{conditional_density}) is

    $$p(\mathbf{y}|\mathbf{X},\beta,\sigma^2) \propto \text{exp}\{-\frac{1}{2\sigma^2}\text{SSR}(\beta)\} = \text{exp}\{-\frac{1}{2\sigma^2}[\mathbf{y}^T\mathbf{y} - 2\beta^T\mathbf{X}^T\mathbf{y}+\beta^T\mathbf{X}^T\mathbf{X}\beta]\}.$$

    The role that $\beta$ plays in the exponent looks very similar to that played by $\mathbf{y}$, and the distribution of $\mathbf{y}$ is multivariate normal.  This suggests that a multivariate normal prior distribution for $\beta$ is conjugate:  if $\beta \sim \text{multivariate normal}(\beta_0,\Sigma_0)$, then

    \begin{flalign*}
        p&(\beta|\mathbf{y,X},\sigma^2)\\
        &\propto p(\mathbf{y}|\mathbf{X},\beta, \sigma^2) \times p(\beta)\\
        &\propto \text{exp}\{-\frac{1}{2}(-2\beta^T\mathbf{X}^T\mathbf{y}/\sigma^2 + \beta^T\mathbf{X}^T\mathbf{X}\beta/\sigma^2) - \frac{1}{2}(-2\beta^T\Sigma_0^{-1}\beta_0 + \beta^T\Sigma_0^{-1}\beta)\}\\
        &=\text{exp}\{\beta^T(\Sigma_0^{-1}\beta_0 + \mathbf{X}^T\mathbf{y}/\sigma^2) - \frac{1}{2}\beta^T(\Sigma_0^{-1} + \mathbf{X}^T\mathbf{X}/\sigma^2)\beta\}
    \end{flalign*}

    This is proportional to a multivariate normal density, with

    \begin{flalign}
        Var[\beta|\mathbf{y,X},\sigma^2] &= (\Sigma_0^{-1} + \mathbf{X}^T\mathbf{X}/\sigma^2)^{-1} \label{semiconj_var}\\
        \text{E}[\beta|\mathbf{y,X},\sigma^2] &= (\Sigma_0^{-1} + \mathbf{X}^T\mathbf{X}/\sigma^2)^{-1} (\Sigma_0^{-1}\beta_0 + \mathbf{X}^T\mathbf{y}/\sigma^2). \label{semiconj_expec}
    \end{flalign}

As usual, we can gain some understanding of these formulae by considering some limiting cases.  If the elements of the prior precision matrix $\Sigma_0^{-1}$ are small in magnitude, then the conditional expectation E$[\beta|\mathbf{y,X},\sigma^2]$ is approximately equal to $(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{y}$, the least squares estimate. On the other hand, if the measurement precision is very small ($\sigma^2$ is very large), then the expectation is approximately $\beta_0$, the prior expectation.\\

As in most normal sampling problems, the semiconjugate prior distribution for $\sigma^2$ is an inverse-gamma distriution.  Letting $\gamma = 1/\sigma^2$ be the measurement precision, if $\gamma \sim \text{gamma}(\nu_0/2,\nu_0\sigma^2_0/2)$, then

\begin{flalign*}
    p(\gamma|\mathbf{y,X},\beta) &\propto p(\gamma)p(\mathbf{y}|\mathbf{X},\beta,\gamma)\\
        &\propto \left[\gamma^{\nu_0/2-1}\text{exp}(-\gamma \times \nu_0\sigma^2_0/2)\right] \times
                 \left[\gamma^{n/2}\text{exp}(-\gamma \times \text{SSR}(\beta)/2)\right]\\
        &= \gamma^{(\nu_0+n)/2-1} \text{exp}(-\gamma[\nu_0\sigma^2_0 + \text{SSR}(\beta)]/2),
\end{flalign*}

\noindent which we recognize as a gamma density, so that

$$\{\sigma^2|\mathbf{y,X},\beta\} \sim \text{inverse-gamma}([\nu_0 + n]/2,[\nu_0\sigma^2_0 + \text{SSR}(\beta)]/2).$$

\noindent Constructing a Gibbs sampler to approximate the joint posterior distribution $p(\beta,\sigma^2|\mathbf{y,X})$ is then straightforward:  given current values $\{\beta^{(s)},\sigma^{2(s)}\}$, new values can be generated by

\begin{enumerate}
    \item updating $\beta$:
    \begin{enumerate}
        \item compute $\mathbf{V} = \text{Var}[\beta|\mathbf{y,X},\sigma^{2(s)}]$ and $\mathbf{m} = \text{E}[\beta|\mathbf{y,X},\sigma^{2(s)}]$
        \item sample $\beta^{(s+1)} \sim \text{multivariate normal}(\mathbf{m,V})$
    \end{enumerate}
    \item updating $\sigma^2$:
    \begin{enumerate}
        \item compute SSR$(\beta^{(s+1)})$
        \item sample $\sigma^{2(s+1)} \sim \text{inverse-gamma}([\nu_0 + n]/2,[\nu_0\sigma_0^2 + \text{SSR}(\beta^{(s+1)})]/2)$.
    \end{enumerate}
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% R FUNCTION FOR CARRYING OUT SEMICONJUGATE PRIOR DISTRIBUTION
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{Schunk}
\begin{Sinput}
> #### Bayesian estimation via MCMC
> x2<-c(0,0,0,0,0,0,1,1,1,1,1,1)
> x3<-c(23,22,22,25,27,20,31,23,27,28,22,24)
> y<-c(-0.87,-10.74,-3.27,-1.97,7.50,-7.25,17.05,4.96,10.40,11.05,0.26,2.51)
> x1 <- rep(1,n)
> x4 <- x2*x3
> X = cbind(x1,x2,x3,x4)
> n<-length(y)
> #X<-cbind(rep(1,n),x1,x2,x1*x2)
> p<-dim(X)[2]
> fit.ls<-lm(y~-1+ X) #linear regression fit omitting intercept term
> beta.0<-rep(0,p) ; Sigma.0<-diag(c(150,30,6,5)^2,p) # SOME INITIAL
> nu.0<-1 ; sigma2.0<- 15^2                           # VALUES
> beta.0<-fit.ls$coef                                 # beta.0 is the coefficients of the linear fit
> nu.0<-1  ; sigma2.0<-sum(fit.ls$res^2)/(n-p)        # nu.0 = # prior obs, sigma2.0 = ols estimator for var
> Sigma.0<- solve(t(X)%*%X)*sigma2.0*n                # Sigma.0 is sampling variance of BetaHat_ols TIMES n (why times n?)
> S<-5000
> rmvnorm<-function(n,mu,Sigma)
+ { # samples from the multivariate normal distribution
+   E<-matrix(rnorm(n*length(mu)),n,length(mu))
+   t(  t(E%*%chol(Sigma)) +c(mu))
+ }
> ## some convenient quantites
> n<-length(y)
> p<-length(beta.0)
> iSigma.0<-solve(Sigma.0)                            #iSigma.0 = inverse of Sigma.0
> XtX<-t(X)%*%X
> ## store mcmc samples in these objects
> beta.post<-matrix(nrow=S,ncol=p)                    #storage for S instances for each coefficient of the p predictors
> sigma2.post<-rep(NA,S)                              #storage for S instances of variance corresponding to the betas
> ## starting value
> set.seed(1)
> sigma2<- var( residuals(lm(y~0+X)) )                #starting with the variance of the residuals
> ## MCMC algorithm
> for( scan in 1:S) {
+
+ #update beta                                        #Formulas and steps Hoff p. 155
+ V.beta<- solve(  iSigma.0 + XtX/sigma2 )            #Conditional variance of the regression coefficients
+ E.beta<- V.beta%*%( iSigma.0%*%beta.0 + t(X)%*%y/sigma2 ) #Conditional mean of the variance coefficients
+ beta<-t(rmvnorm(1, E.beta,V.beta) )                 #Gibbs sampler step:  update beta
+
+ #update sigma2
+ nu.n<- nu.0+n                                       #numerator of 1st term of inverse gamma sample
+ ss.n<-nu.0*sigma2.0 + sum(  (y-X%*%beta)^2 )        #numerator of 2nd term of inverse gamma sample
+ sigma2<-1/rgamma(1,nu.n/2, ss.n/2)                  #Gibbs ampler step:  update sigma^2
+
+ #save results of this scan
+ beta.post[scan,]<-beta                              #Store updated beta in current row of beta matrix
+ sigma2.post[scan]<-sigma2                           #Store updated sigma^2 in current position in sigma^2 vector
+                         }
> round( apply(beta.post,2,mean), 3)                  #compute mean of Gibbs sampled betas (for check)
\end{Sinput}
\begin{Soutput}
[1] -50.943  12.656   2.079  -0.300
\end{Soutput}
\begin{Sinput}
>
\end{Sinput}
\end{Schunk}

    \subsection{Default and weakly informative prior distributions}
    In situations where prior information is unavailable or difficult to quantify, an alternative ``default" class of prior distributions is given. Specification of the prior parameters $(\beta_0, \Sigma_0)$ and $(\nu_0,\sigma^2_0)$ that represent actual prior information for a Bayesian analysis can be difficult.  For a prior distribution that is not going to represent real prior information about the parameters, we choose one that is as minimally informative as possible.  The resulting posterior distribution, then, will represent the posterior information of someone who began with little knowledge of the population being studied.  Here we will employ Zellner's ``$g$-prior" (Zellner, 1986).  We choose $\beta_0 = \mathbf{0}$ and $\Sigma_0 = k(\mathbf{X}^T\mathbf{X})^{-1}, k = g\sigma^2, g > 0$, which satisfies a desired condition that the regression parameter estimation be invariant to changes in the scale of the regressors.  With this, equations \ref{semiconj_var} and \ref{semiconj_expec} reduce to

\begin{flalign}
    \text{Var}[\beta|\mathbf{y,X},\sigma^2] &= [\mathbf{X^TX}/(g\sigma^2) + \mathbf{X^TX}/\sigma^2]^{-1} = \frac{g}{g+1}\sigma^2(\mathbf{X^TX})^{-1} \label{noninf_var}\\
    \text{E}[\beta|\mathbf{y,X},\sigma^2] &= [\mathbf{X^TX}/(g\sigma^2) + \mathbf{X^TX}/\sigma^2]^{-1}\mathbf{X^Ty}/\sigma^2 = \frac{g}{g+1}\sigma^2(\mathbf{X^TX})^{-1}\mathbf{X^Ty}.\label{noninf_expec}
\end{flalign}

Letting

$$\mathbf{V} = \frac{g}{g+1}\sigma^2(\mathbf{X^TX})^{-1} \text{ and } \mathbf{m} = \frac{g}{g+1}\sigma^2(\mathbf{X^TX})^{-1}\mathbf{X^Ty}$$

we arrive at posteriors

\begin{flalign}
    \{\sigma^2|\mathbf{y,X}\} &\sim \text{inverse-gamma}([\nu_0 + n]/2,[\nu_0\sigma^2_0 + \text{SSR}_g]/2) \label{noninf_sig2_post}\\
    \{\beta|\mathbf{y,X},\sigma^2\} &\sim \text{multivariate normal}\left(\frac{g}{g+1}\hat{\beta}_{ols},\frac{g}{g+1}\sigma^2[\mathbf{X^TX}]^{-1}\right).\label{noninf_beta_post}
\end{flalign}

Here $\text{SSR}_g = \mathbf{y^Ty - m^TV^{-1}m = y^T(I - }\frac{g}{g+1}\mathbf{X(X^TX)^{-1}X^T)y}$.\\

Simple Monte Carlo approximation can be used to sample from the joint posterior density $p(\sigma^2,\beta|\mathbf{y,X})$ as follows:

\begin{enumerate}
    \item sample $\sigma^2 \sim \text{inverse-gamma}([\nu_0 + n]/2,[\nu_0\sigma^2_0 + \text{SSR}_g]/2)$
    \item sample $\beta \sim \text{multivariate normal}\left(\frac{g}{g+1}\hat{\beta}_{ols},\frac{g}{g+1}\sigma^2[\mathbf{X^TX}]^{-1}\right)$.
\end{enumerate}

\end{document}


\section{Special Functions}
\subsection{Power Series}
Recall that if $\sum_nc_n$ converges then $c_n\to0$.
We also have a strong convergence test where if $|c_n|\le a_n$ for all $n$ and $\sum_na_n$ converges, then $\sum_nc_n$ converges.
We used it to porve the ratio test: suppose $\lim_{n\to\infty}|c_{n+1}/c_n|$ exists and equals $\alpha$, then $\alpha<1$ implies $\sum_nc_n$ converges and $\alpha>1$ implies it diverges.
\begin{definition}
    A power series centered at $a\in\mathbb C$ is a series that can be written in the form
    $$\sum_{n=0}^\infty c_n(z-a)^n,c_n\in\mathbb C$$
\end{definition}
\begin{example}
    The series
    $$\sum_{n=0}^\infty (z-1)^n,\sum_{n=0}\frac{z^n}{n!},\sum_{n=0}^\infty (-1)^n\frac{z^{2n}}{(2n)!}$$
    are power series.
    In general, any Taylor series around $a$ is a power series centered at $a$.
\end{example}
For a power series and each $z\in\mathbb C$, we can get a seperate series.
Then we want to ask for which $z\in\mathbb C$ does the series converges.
\begin{example}
    $\sum_n(z-1)^n$ converges in $|z-1|<1$ and diverges otherwise.
    On the other hand, $\sum_nz^n/n!,\sum_n(-1)^nz^{2n}/(2n)!$ converge absolutely.
    But $\sum_nn!z^n$ converges iff $z=0$.
\end{example}
Note that suppose $|c_{n+1}/c_n|\to\alpha$, then
$$\lim_{n\to\infty}\left|\frac{c_{n+1}(z-a)^{n+1}}{c_n(z-a)^n}\right|=\alpha|z-a|$$
So for $|z-a|<1/\alpha$, the series converges and $|z-a|>1/\alpha$ means it diverges.
But we do not know about the case $|z-a|=\alpha$.\\
We want to show similar result in general (i.e. $\alpha$ may not exist).
\begin{lemma}[Key Estimate]
    Suppose $p_z$ is the series
    $$\sum_{n=0}^\infty c_n(z-a)^n$$
    and that $p_{z_0}$ converges, the there is $M>0$ such that
    $$c_n\le\frac{M}{|z_0-a|^n}$$
\end{lemma}
\begin{proof}
    We have $|c_n||z_0-a|^n\to 0$, so there is some $N$ such that $|c_n||z_0-a|^n<1$ for $n>N$, so $|c_n|\le 1/|z_0-a|^n$ for $n>N$, but we can take care of the rest by taking maximum since there are only finitely many terms left.
\end{proof}
\begin{theorem}
    Suppose
    $$P(z_0)=\sum_{n=0}^\infty c_n(z-a)^n$$
    converges, then $P(z)$ converges absolutely whenever $|z-a|<|z_0-a|$.
\end{theorem}
\begin{proof}
    Given $z,|z-a|<|z_0-a|$, let $\alpha=(z-a)/(z_0-a)$, so $|\alpha|<1$, so choose $M$ as in the preceding lemma gives $c_n|z-a|^n\le M|\alpha|^n$, but $\sum_mM|\alpha|^n$ converges as a geometric series, so $P(z)$ converges absolutely.
\end{proof}
\begin{corollary}
    With $P(z)$ as above, either $P(z)$ converges for all $z\in\mathbb C$ or there exists some $R>0$ such that $P(z)$ converges absolutely for $|z|<R$ and diverges for $|z|>R$.
\end{corollary}
\begin{proof}
    Immediate.
\end{proof}
Such an $R$ is obviously unique, so we define
\begin{definition}
    Such an $R$ is called the radius of convergence of the power series.
\end{definition}
Note that the series may or may not converge on the circle $|z-a|=R$.
\begin{example}
    The following series have radius of convergence $1$:
    $$\sum_{n=0}^\infty nz^n,\sum_{n=1}^\infty\frac{z^n}{n^2},\sum_{n=0}^\infty\sum_{n=1}^\infty\frac{z^n}{n}$$
    But they exhibit different behaviours on $|z|=1$.
\end{example}
\begin{proposition}
    Suppose $P(z)$ has radius of convergence $R$, then\\
    1. $c_n(z-a)^n\to 0$ for $|z|<R$\\
    2. $c_n(z-a)^n$ is unbounded for $|z|>R$.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
What we really want to do is to differentiate them.
Suppose we have
$$P(z)=\sum_{n=0}^\infty c_n(z-a)^n$$
Then let $R>0$ be its radius of convergence, then $P:B_R(a)\to\mathbb C$ is a function.
If $P$ is differentiable, we obviously want it to be
$$Q(z)=\sum_{n=1}^\infty nc_n(z-a)^{n-1}$$
which is also a power series.
So we will want $Q$ to have radius of convergence $R$, $P$ to be differentiable and $P^\prime=Q$.
\begin{proposition}
    $Q$ has radius of convergence $R$.
\end{proposition}
\begin{proof}
    Suppose $P(z_0)$ converges with $z_0\neq a$, then there is some $M>0$ with $|c_n|\le M/|z_0-a|^n$ for all $n$.
    Then for any $z$ with $|z-a|<|z_0-a|$,
    $$|nc_n(z-a)^{n-1}|\le nC\alpha^{n-1},C=M/|z_0-a|,\alpha=|z-a|/|z_0-a|<1$$
    The rest is trivial.
\end{proof}
Since we can shift stuff around, we assume $a=0$ for the moment.
The preceding proposition then says
$$P(z)=\sum_{n=0}^\infty c_nz^n,Q(z)=\sum_{n=1}^\infty nc_nz^{n-1}$$
both have radius of convergence $R$.
What we want to show now is that $P$ is complex differentiable on $B_R(0)$ and its derivative is $Q$.
The optimal way to do this is to use the idea of uniform convergence, but we do not have that tool right now, so here goes the clumsy solution.
For $z\in B_R(0)$, consider
\begin{align*}
    P(z+h)-P(z)-hQ(z)&=\sum_{n=0}^\infty c_n(z+h)^n-\sum_{n=0}^\infty c_nz^n-h\sum_{n=1}^\infty nc_nz^{n-1}\\
    &=\sum_{n=0}^\infty c_n((z+h)^n-z^n-nhz^{n-1})\\
    &=\sum_{n=0}^\infty c_n\sum_{k=2}^n\binom{n}{k}h^kz^{n-k}\\
    &=\sum_{n=0}^\infty h^2c_n\sum_{k=2}^n\binom{n}{k}h^{k-2}z^{n-k}\\
    &=\sum_{n=0}^\infty h^2c_nr_n(h;z)
\end{align*}
But $r_n(h;z)$ is just a polynomial.
So intuitively we want to show that its growth is slow, so we claim
\begin{lemma}
    For any $z,h\in\mathbb C$,
    $$|r_n(h;z)|\le\binom{n}{2}(|h|+|z|)^{n-2}$$
\end{lemma}
\begin{proof}
    First suppose that $z,h\in\mathbb R_{\ge 0}$, then by using Taylor's Theorem on $f(z)=z^n$, we have
    $$f(z+h)=f(z)+hf^\prime(z)+\frac{h^2}{2}f^{\prime\prime}(y)$$
    for some $y\in [z,z+h]$, so
    $$r_n(h,z)=\binom{n}{2}y^{n-2}\le\binom{n}{2}(z+h)^{n-2}$$
    For the general case,
    \begin{align*}
        |r_n(h,z)|&=\left|\sum_{i=0}^n\binom{n}{2}h^{i-2}z^{n-i}\right|\\
        &\le\sum_{i=0}^n\binom{n}{2}|h|^{i-2}|z|^{n-i}\\
        &=r_n(|h|,|z|)\le\binom{n}{2}(|z|+|h|)^{n-2}
    \end{align*}
    by above.
\end{proof}
\begin{theorem}\label{tbt_diff}
    $P$ is complex differentiable on $B_R(a)$ and its derivative is $Q$.
\end{theorem}
\begin{proof}
    Again take WLOG $a=0$.
    Let $0<\rho<R-r_1=R-|z|$ and let $r_2=r_1+\rho<R$, then for $h\in B_\rho(0)$, $z+h\in B_\rho(z)\subset B_{r_2}(0)\subset B_R(0)$.
    Pick $r_3$ such that $r_2<r_3<R$, then $P(r_3)$ converges, so $\exists M>0$ such that $|c_n||r_3|^n\le M$ for any $n$.
    Then
    \begin{align*}
        |c_nr_n(h;z)|&\le\frac{M}{r_3^n}\binom{n}{2}(|h|+|z|)^{n-2}\\
        &\le\frac{M}{r_2^2}\binom{n}{2}\alpha^n,\alpha=\frac{r_2}{r_3}<1
    \end{align*}
    Now the series $\sum_n\binom{n}{2}\alpha^n$ converges due to the ratio test, consequently
    $$\sum_{n=0}^\infty c_nr_n(h;z)\to w$$
    For some complex number, which we assume has modulus at most $C>0$, then
    \begin{align*}
        P(z+h)-P(z)-hQ(z)=h^2\sum_{n=0}^\infty c_nr_n(h;z)=h^2\beta(h)
    \end{align*}
    Where $\beta(h)$ is the series sum.
    Let $\alpha(h)=h\beta(h)$, then $|\alpha(h)|\le C|h|$, so $\alpha(h)\to 0$ as $h\to 0$.
    But also $\alpha(0)=0$, so $\alpha$ is $0$ at $0$ and continuous at $0$ and we have $P(z+h)=P(z)+hQ(z)+h\alpha(h)$, so the proof is done.
\end{proof}
\begin{corollary}
    A power series
    $$P(z)=\sum_{n=0}^\infty c_n(z-a)^n$$
    with radius of convergence $R>0$ is infinitely differentiable on $B_R(a)$ and $n!c_n=f^{(n)}(a)$.
\end{corollary}
\begin{proof}
    Immediate.
\end{proof}
\begin{example}
    Consider the sum $\sum_nn^2/2^n$.
    We have $\sum_nz^n=1/(1-z)$ for $|z|<1$, so by differentiating and building stuff we have $\sum_nn^2z^n=z(1+z)/(1-z)^3$, so the series converges to $6$ by taking $z=1/2$.
\end{example}
\begin{remark}
    In general, it is false to say $f_n(z)\to f(z)$ implying $f_n^\prime(z)\to f^\prime(z)$.
    For example one can take $f_n(x)=\sin(nx)/n$ converges to $0$ (even uniformly), but $f_n^\prime(x)=\cos(nx)$ certainly does not converge to $0$.
\end{remark}
\subsection{The Exponentials, etc.}
\begin{definition}
    For $z\in\mathbb C$, the exponential function $\exp(z)$ is defined by
    $$\exp(z)=\sum_{n=0}^\infty\frac{z^n}{n!}$$
    which converges for any $z\in\mathbb C$ by the ratio test.
\end{definition}
We have $\exp^\prime=\exp$ by Theorem \ref{tbt_diff}.
Also $\exp(0)=1$.
\begin{proposition}
    $\forall z,w\in\mathbb C,\exp(z+w)=\exp(z)\exp(w)$.
\end{proposition}
\begin{proof}
    Fix $c\in\mathbb C$ and consider $f(x)=\exp(x)\exp(c-x)$, then by differentiation $f^\prime=0$, so $f$ is constant.
    So $\exp(x)\exp(c-x)=\exp(0)\exp(c-0)=\exp(c)$, so taking $x=z,c=z+w$ gives the result.
\end{proof}
In particular, $\exp(z)\exp(-z)=1$, so $\exp(z)$ is never zero.
And $\exp(-z)=1/\exp(z)$.
By induction, we can easily see that $\exp(nz)=\exp(z)^n$ for all $n\in\mathbb Z$.\\
For $x\in\mathbb R$, $\exp x$ is real since every term of it is real.
\begin{lemma}
    $\exp(\mathbb R)=(0,\infty)$ and it is a strictly increasing bijection $\mathbb R\to (0,\infty)$.
\end{lemma}
\begin{proof}
    Obvious.
\end{proof}
The lemma guarantees a function $\log:(0,\infty)\to\mathbb R$ as the inverse of $\exp:\mathbb R\to (0,\infty)$.
So we have $\log(1)=0,\log(ab)=\log(a)+\log(b)$ for $a,b>0$.
As for the derivative, by the inverse function theorem, $\log^\prime(x)=1/x$.
\begin{proposition}
    $$-\log(1-y)=\sum_{n=1}^\infty\frac{y^n}{n}$$
    for $|y|<1$.
\end{proposition}
\begin{proof}
    Let $p(y)$ be the series in the right hand side, then $p$ does converge for $|y|<1$ by the ratio test.
    Also $p^\prime(y)=1/(1-y)$ by differenating term-by-term.
    So $f(y)=p(y)+\log(1-y)$ has zero derivative, so $f$ is constant.
    Also when $y=0$, we have $f(0)=0$, so $f\equiv 0$, hence $-\log(1-y)=p(y)$.
\end{proof}
\begin{definition}
    For $a\in (0,\infty)$ and $x\in\mathbb R$, we define the power $a^x$ is defined by $\exp(x\log a)$.
\end{definition}
It follows instantly that the power is well-defined, differentiable, and has the series
$$a^x=\sum_{n=0}^\infty\frac{(\log a)^n}{n!}x^n$$
Also we have $a^{x+y}=a^xa^y$ and $(a^x)^y=a^{xy}$.
Define the constant $e=\exp(1)$, so $e^x=\exp(x)$.
\begin{definition}
    For $z\in\mathbb C$, we define
    $$\sin z=\frac{1}{2i}(e^z-e^{-z}),\cos z=\frac{1}{2}(e^z+e^{-z})$$
\end{definition}
Then it follows that they are both differentiable and $\sin^\prime=\cos,\cos^\prime=-\sin$
Also they have power series
$$\sin z=\sum_{n=0}^\infty(-1)^n\frac{1}{(2n+1)!}z^{2n+1},\cos z=\sum_{n=0}^\infty(-1)^n\frac{1}{(2n)!}z^{2n}$$
The familiar trigonometric identities all follows from definition.
In particular, $\sin^2z+\cos^2z=1$, so $\cos,\sin$ has range in $[0,1]$ whenever $z\in\mathbb R$.
We also have Euler's formula: $e^{iz}=\cos z+i\sin z$.\\
Now we recall the alternating series test: If $a_{n+1}\ge a_n\ge 0$ and $a_n\to 0$, then $\sum_n(-1)^na_n$ converges.
Furthermore, if we let $s_n$ be the $n^{th}$ partial sum of the alternating series and $s$ be the limit, then
\begin{proposition}
    For any $n\in\mathbb N$, $s_{2n}\ge s\ge s_{2n+1}$.
\end{proposition}
\begin{proof}
    $s_{2n}$ decreases to $s$ and $s_{2n+1}$ increases to $s$, so the proposition follows immediately.
\end{proof}
\begin{lemma}
    $\sin x>0$ for $x\in (0,2]$.
\end{lemma}
\begin{proof}
    For $x\in (0,2]$, $\sin x$ is an alternating series, so $\sin x\ge x-x^3/6=x(1-x^2/6)>0$.
\end{proof}
\begin{lemma}
    $\cos x>0$ for $x\in [0,1]$ and there is a unique $\alpha\in (1,2]$ with $\cos\alpha=0$.
\end{lemma}
\begin{proof}
    For $x\in [0,1]$, $\cos$ is an alternating series, so $\cos x\ge 1-x^2/2\ge 1/2>0$ for $x\in [0,1]$.\\
    Now the series $\cos 2$ is alternating from the third term, so $\cos 2\le 1-2^2/2+2^4/24=-1/3<0$.
    So there is some $\alpha\in (1,2)$ such that $\cos\alpha=0$ by the intermediate value theorem.
    To see uniqueness, observe that $\cos^\prime=-\sin<0$ in $[1,2]$ by the preceding lemma, so $\cos$ must be strictly monotone hence injective in $[1,2]$, therefore $\alpha$ is unique.
\end{proof}
\begin{definition}
    Let $\alpha$ be as in the preceding lemma, then we define $\pi=2\alpha$.
\end{definition}
Consequenly $\sin\alpha=1$ by $\sin^2z+\cos^2z=1$, and $\cos(x+\alpha)=-\sin x,\sin(x+\alpha)=\cos x$.
So $\sin,\cos$ are periodic with period $2\pi$, and we know the trends in between.