\def\p{\vec{\theta}}
\def\hatp{\vec{\hat{\theta}}}
\def\BiasField{\vec{\beta}}

\section{The Expectation Maximization Algorithm}

\subsection{Parameter Estimation}

\begin{frame}
  \frametitle{Parameter Estimation Methods}

  \structure{Goal:} Derivation of a parameter estimation technique that can deal with
  \begin{itemize}
    \item high dimensional parameter spaces and
    \item latent, hidden, incomplete data.
  \end{itemize}
  \pspread

  Parameter estimation techniques known from statistics: \\[0.25cm]

  \begin{enumerate}
    \item \structure{Maximum likelihood estimation (ML estimation)}
      \begin{itemize}
        \item All observations are assumed to be mutually statistically independent.
        \item The observations are kept fixed.
        \item The (log-)likelihood function is optimized regarding the parameters. \\[0.25cm] \pause
      \end{itemize}
    \item \structure{Maximum a-posteriori estimation (MAP estimation)}
      \begin{itemize}
        \item The probability density function of the parameters $p(\vec{\theta})$ to be estimated \\
          is known.
      \end{itemize}
  \end{enumerate}
\end{frame}

\begin{frame}
  \frametitle{Parameter Estimation}

  Let $X$ be the observed random variable and $\p$ the parameter set. \\
  The estimates of $\p$ are denoted by $\hatp$. \\
  Let $x$ be an event assigned to the random variable $X$. \\[.25cm] \pause
%
  \begin{itemize}
    \item \structure{ML estimation:} $\displaystyle \hatp= \argmax_{\p}\ p(x; \p) = \argmax_{\p}\ \log p(x; \p)$  \\[.25cm] \pause
    \item \structure{MAP estimation:}
      {\small
        \begin{eqnarray*}
          \hatp &=& \argmax_{\p}\ p(\p|{x})\\
                &=& \argmax_{\p}\ \frac{p(\p)\,p(x|\p)}
                                       {\sum_{\p} p(\p)\,p(x|\p)} \\
                &=& \argmax_{\p}\ \log p(\p) + \log p(x|\p)
        \end{eqnarray*}
      }
    \item[] Here $\p$ is considered as a random variable and its probability density function $p(\p)$ is known.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{ML Estimation: Example}

  \begin{ovalblock}{Example}
    Let us assume a Gaussian distributed random vector:
%
    \begin{displaymath}
      p(\vec{x};\vec{\mu}, \mat{\Sigma}) = \frac{1}{\sqrt{\det(2\pi\mat{\Sigma})}}
        e^{-\frac{1}{2}(\vec{x}-\vec{\mu})^T \mat{\Sigma}^{-1} (\vec{x}-\vec{\mu})} \pause
    \end{displaymath}
%
    \begin{itemize}
      \item We observe the random vectors $\vec{x}_1, \vec{x}_2, \dots, \vec{x}_m$ (training data). \pause
      \item Based on these training data, we have to estimate the mean vector $\vec{\mu}$ and the covariance matrix $\mat{\Sigma}$.
    \end{itemize}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{ML Estimation: Example \cont}

  \begin{ovalblock}{Example \cont}
    \small
    The ML estimator assumes \structure{mutually independent observations} and \\
    optimizes the pdf for the given set of training data:
%
    {\footnotesize
      \begin{eqnarray*}
        \{\hat{\vec{\mu}}, \hat{\mat{\Sigma}}\}
          &=& \argmax_{\vec{\mu},\mat{\Sigma}}\ \prod_{i=1}^m p(\vec{x}_i;\vec{\mu}, \mat{\Sigma}) \\ \pause
          &=& \argmax_{\vec{\mu},\mat{\Sigma}}\ \sum_{i=1}^m \log p(\vec{x}_i;\vec{\mu}, \mat{\Sigma}) \\ \pause
          &=& \argmax_{\vec{\mu},\mat{\Sigma}} L(\vec{x}_1, \vec{x}_2, \dots, \vec{x}_m; \vec{\mu}, \mat{\Sigma}) \pause
      \end{eqnarray*}
    }
%
    where the \structure{log-likelihood function} is defined by
%
    {\footnotesize
      \begin{displaymath}
        L:= L(\vec{x}_1, \vec{x}_2, \dots, \vec{x}_m; \vec{\mu}, \mat{\Sigma}) =
        \sum_{i=1}^m \log p(\vec{x}_i;\vec{\mu}, \mat{\Sigma})
      \end{displaymath}
    }
    \vspace{-.25cm}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{ML Estimation: Example \cont}

  \begin{ovalblock}{Example \cont}
     \small
     \structure{Necessary conditions} for the estimation of the parameters are:
%
     {\footnotesize
       \begin{displaymath}
         \frac{\partial L}{\partial \vec{\mu}} \stackrel{!}{=} \vec{0}
         \quad \quad \mbox{and} \quad \quad
         \frac{\partial L}{\partial \mat{\Sigma}} \stackrel{!}{=} \vec{0} \pause
       \end{displaymath}
     }
%
     Now we get for the mean vector:
%
     {\footnotesize
       \begin{displaymath}
         \frac{\partial L}{\partial \vec{\mu}} =
         \sum_{i=1}^m \mat{\Sigma}^{-1}(\vec{x_i}-\vec{\mu}) \stackrel{!}{=}
         \vec{0} \pause
       \end{displaymath}
     }
%
     and thus the \structure{ML estimate for the mean vector} meets our expectation:
%
     {\footnotesize
       \begin{displaymath}
         \vec{\hat{\mu}} = \frac{1}{m}\sum_{i=1}^m \vec{x}_i
      \end{displaymath}
    }
    \vspace{-.25cm}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{ML Estimation: Example \cont}

  \begin{ovalblock}{Example \cont}
    \small
    Along the same lines, we get the \structure{estimator of the covariance matrix} \\
    by computation of the zero crossings of the partial derivatives w.\,r.\,t.\ the components of the covariance matrix:

    \begin{displaymath}
      \hat{\mat{\Sigma}} = \frac{1}{m} \sum_{i=1}^m (\vec x_i-\hat{\vec{\mu}})
                                                    (\vec x_i-\hat{\vec{\mu}})^T
    \end{displaymath}
  \end{ovalblock}
\end{frame}

\subsection{Gaussian Mixture Models}

\begin{frame}
  \frametitle{Gaussian Mixture Models}

  So far, we have considered parameter estimation for statistical models with: \pause

  \begin{itemize}
    \item one class-dependent distribution component \pause
    \item uni- or multivariate feature vectors \pause
    \item the type was mostly Gaussian (normally distributed features)
  \end{itemize}
  \pspread

  Now we extend this model by representing the observations with a set of $K$ multivariate Gaussian distributions:\\

  \begin{center}
    \structure{Gaussian Mixture Model (GMM)}
  \end{center}
\end{frame}

\mode<handout>{
  \begin{frame}
    \frametitle{Gaussian Mixture Models \cont}

    \begin{center}
      \resizebox{.8\linewidth}{!}{
        \input{\texfigdir/koerpergroesse_gesamt.pstex_t}
      }
    \end{center}
  \end{frame}
}

\begin{frame}
  \frametitle{Gaussian Mixture Models \cont}

  \begin{center}
    \resizebox{.8\linewidth}{!}{
     \alt<3->{
       \input{\texfigdir/koerpergroesse_weiblich+maennlich.pstex_t}
     }{\alt<2>{
       \input{\texfigdir/koerpergroesse_weiblich.pstex_t}
     }{
        \input{\texfigdir/koerpergroesse_gesamt.pstex_t}
     }}
    }
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Gaussian Mixture Models \cont}

  \begin{center}
    \resizebox{.8\linewidth}{!}{
     \alt<2->{
       \input{\texfigdir/gmm2.pstex_t}
     }{
       \input{\texfigdir/gmm1.pstex_t}
     }
    }
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Gaussian Mixture Models \cont}

  \structure{Problem description:} \\[.5cm]

  Given $m$ feature vectors in an $d$ dimensional space, find a set of $K$ multivariate Gaussian distributions that best represent the observations.
  \pspread

  GMMs are an example of classification by \structure{\emph{unsupervised learning}}: \pause

  \begin{itemize}
    \item It is not known which feature vectors are generated by which of the $K$ Gaussians \pause
    \item The desired output is, for each feature vector, an estimate of the probability that it is generated by distribution $k$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Gaussian Mixture Models \cont}

  \structure{GMM parameter estimation:}
%
  \begin{eqnarray*}
    \vec\mu_k            &\phantom{=}& \mbox{the $K$ means} \\
    \mat\Sigma_k         &\phantom{=}& \mbox{the $K$ covariance matrices of size $d \times d$} \\
    p_k                  &\phantom{=}& \mbox{fraction of all features in component $k$} \\
    p(k|i) \equiv p_{ik} &\phantom{=}& \mbox{the $K$ probabilities for each of the $m$ feature vectors $\vec{x}_i$}
  \end{eqnarray*}
  \pspread

  \structure{Additional estimates:}
%
  \begin{eqnarray*}
    p(\vec x) &\phantom{=}& \mbox{probability distribution of observing a feature vector $\vec x$} \\
    L         &\phantom{=}& \mbox{overall log-likelihood function of the estimated parameter set}
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{GMM -- Expectation}

  The key to the estimation problem is the \structure{overall log-likelihood objective function $L$}:
  \begin{displaymath}
    L = \sum_{i = 1}^{m} \log p(\vec x_i) \pause
  \end{displaymath}

  Split $p(\vec x_i)$ % (mixture weight of $\vec x_i$)
  into its contributions from the $K$ Gaussians:
  \begin{displaymath}
    p(\vec x_i) = \sum_{k=1}^{K} p_k \, {\mathcal N}(\vec x_i | \vec \mu_k, \mat \Sigma_k) \pause
  \end{displaymath}

  Individual probabilities for the $K$ contributions:
  \begin{displaymath}
    p_{ik} \equiv p(k|i) = \frac{ p_k\,{\mathcal N}(\vec x_i | \vec \mu_k, \mat \Sigma_k)}{p(\vec x_i)}
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{GMM -- Maximization}

  \structure{Problem:} How do we get $\vec \mu_k, \mat \Sigma_k$ and $p_k$? \pause

  \begin{itemize}
    \item Similar to the ML estimate for the Gaussian, we maximize the log-likelihood by deriving w.\,r.\,t.\ the unknowns. \\[.25cm] \pause
    \item The \structure{ML estimates} are:
      \begin{eqnarray*}
        \hat{\vec \mu}_k
          &=& \frac{\sum_i p_{ik} \vec x_i}{\sum_i p_{ik}} \\
        \hat{\mat \Sigma}_k
          &=& \frac{\sum_i p_{ik} (\vec x_i - \hat{\vec \mu}_k)(\vec x_i - \hat{\vec \mu}_k)^T}{\sum_i p_{ik}} \\
        \hat{p}_k
          &=& \frac{1}{m} \sum_{i=1}^m p_{ik}
      \end{eqnarray*}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{GMM Parameter Estimation}

  \structure{Observations:}

  \begin{itemize}
    \item If we know the values for the parameters ($\vec \mu_k, \mat \Sigma_k, p_k$), we can compute the expectations (\structure{E-step}).\pause
    \item Once we have the expectations we can compute improved values for the parameters (\structure{M-step}).
  \end{itemize}
  \pspread

  We have found an \structure{iterative solution scheme} for the nonlinear GMM parameter estimation problem:

  \begin{itemize}
    \item \emph{Right at} the ML solution both E- and M-step relations hold. \pause
    \item The ML parameters are a stationary point for the E- and M-step. \pause
    \item Starting from any parameter values, an iteration of the E-step combined with an M-step will increase $L$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{GMM Parameter Estimation \cont}

  \structure{EM algorithm for GMM parameter estimation:}
%
  \begin{centernss}
    \begin{struktogramm}(100,50)
      \assign{Initialization: $\vec \mu_k^{(0)}, \mat \Sigma_k^{(0)}, p_k^{(0)}$}
      \assign{$j \gets 0$}
      \until{$L$ is no longer changing}
        \assign{\structure{Expectation step}: \\
          compute new values for $p_{ik}, L$ }
        \assign{\structure{Maximization step}: \\
          update values for $\vec \mu_k^{(j)}, \mat \Sigma_k^{(j)}, p_k^{(j)}$ }
        \assign{$j \gets j+1$}
      \untilend
      \assign{Output: estimates $\hat{\vec \mu}_k, \hat{\mat \Sigma}_k, \hat{p}_k$}
    \end{struktogramm}
  \end{centernss}
%
%   \begin{center}
%     \begin{struktogramm}{10cm}{0.7cm}
%       \BLOCK {Initialization: $\vec \mu_k^{(0)}, \mat \Sigma_k^{(0)}, p_k^{(0)}$}
%       \BLOCK {Set $j:=0$}
%       \REPEAT {
%         \BLOCK {\structure{Expectation Step}: \\
%           compute new values for $p_{ik}, L$ }
%         \BLOCK {\structure{Maximization Step}: \\
%           update values for $\vec \mu_k^{(j)}, \mat \Sigma_k^{(j)}, p_k^{(j)}$ }
%         \BLOCK {Set $j:=j+1$}
%       }
%       UNTIL{$L$ is no longer changing}
%       \BLOCK {Output: estimates $\hat{\vec \mu}_k, \hat{\mat \Sigma}_k, \hat{p}_k$}
%     \end{struktogramm}
%   \end{center}
\end{frame}

\input{nextTime.tex}

\subsection{Missing Information Principle}

\begin{frame}
  \frametitle{Missing Information Principle}

  A colloquial formulation of the \structure{missing information principle (MIP)} is \\
  as simple as:

  \begin{center}
    \tikz[baseline]{
      \node[fill=bl1!100, anchor=base, rounded corners=3pt, inner sep=3mm] (d1) {
        \color{bl3}
        observable information $=$ complete information $-$ hidden information
      };
    }
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Missing Information Principle \cont}

  \structure{Mathematical formalization of the MIP:}

  \begin{itemize}
    \item observable random variable: $X$
    \item hidden random variable: $Y$
    \item parameter set: $\p$ \\[0.25cm]
  \end{itemize}
  \pspread

  The joint probability density of the events $x$ (observation) and $y$ (hidden) is:
  \begin{displaymath}
    p(x,y; \p) = p(x; \p)\  p(y | x; \p) \pause
  \end{displaymath}
%
  and thus:
  \begin{displaymath}
    p(x; \p)= \frac{p(x, y; \p)}{p(y|x; \p)}
  \end{displaymath}
  \pause

  \vspace*{0.25cm}
  The mathematical formulation of the MIP is:
  \begin{displaymath}
    -\log p(x; \p) = -\log p(x, y; \p) -(-\log p(y|x; \p))
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Key Equation}

  We now consider the mathematical formulation of the key equation and derive an iterative parameter estimation scheme: \pause

  \begin{itemize}
   \item Let $i$ denote the iteration parameter. \pause
   \item Consider the key equation $(i+1)$-st iteration
     {\small
       \begin{displaymath}
         \log p\left(x    ; \hatp^{(i+1)} \right) =
         \log p\left(x, y ; \hatp^{(i+1)} \right) -
         \log p\left(y|x  ; \hatp^{(i+1)} \right)~,
       \end{displaymath}
     }
     where $\hatp^{(i+1)}$ denotes the estimation in iteration step $(i+1)$. \pause
   \item Now we multiply both sides with $p\left( y|x ; \hatp^{(i)} \right)$ and \\
     integrate over the hidden event $y$:
     \footnotesize
     \begin{eqnarray*}
             \int p\left(y|x ; \hatp^{(i)} \right) \log p\left( x ;   \hatp^{(i+1)} \right)\, \mathsf{d}y \pause
         &=& \int p\left(y|x ; \hatp^{(i)} \right) \log p\left(x, y ; \hatp^{(i+1)} \right)\, \mathsf{d}y -{} \\
         & & \int p\left(y|x ; \hatp^{(i)} \right) \log p\left(y|x  ; \hatp^{(i+1)} \right)\, \mathsf{d}y
     \end{eqnarray*}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Key Equation \cont}

  Now consider the left hand side of this equation:

  \begin{eqnarray*}
          & & \int p\left(y|x ; \hatp^{(i)}   \right) \, \log p\left( x ; \hatp^{(i+1)} \right)\ \mathsf{d}y ={} \\ \pause
          &=& \log p\left(x   ; \hatp^{(i+1)} \right) \, \int p\left( y|x ; \hatp^{(i)} \right)\ \mathsf{d}y = \\ \pause
          &=& \log p\left(x   ; \hatp^{(i+1)} \right) \pause
  \end{eqnarray*}

  \begin{itemize}
    \item \structure{Observation:} The left side of the key equation is the log likelihood function of observations. \\[.3cm] \pause
    \item \structure{Conclusion:} The maximization of the right hand side of the above key equation corresponds to a ML estimation
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Kullback-Leibler Statistics and Entropy}

  For the terms on the right hand side we introduce the following notation \\
  (formally this is incorrect due to the differences in the iteration index):

  \begin{itemize}
    \item \structure{Kullback-Leibler Statistics}
      \begin{displaymath}
        Q(\hatp^{(i)};\hatp^{(i+1)}) =
        \int p(y|x; \hatp^{(i)}) \log p(x, y;\hatp^{(i+1)}) \ \mathsf{d}y \pause
      \end{displaymath}
   \item \structure{Entropy:}
     \begin{displaymath}
       H(\hatp^{(i)};\hatp^{(i+1)}) =
       - \int p(y|x; \hatp^{(i)}) \log p(y|x; \hatp^{(i+1)})\ \mathsf{d}y
      \end{displaymath}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Kullback-Leibler Statistics}

  Let us first take a closer look at the Kullback-Leibler statistics:
  \begin{displaymath}
    Q(\p,\p') = \int p(y| x; \p)\ \log{p(x,y;\p')}\ \mathsf{d}y
  \end{displaymath}

  The Kullback-Leibler statistics (also called $Q$-function) w.\,r.\,t.\ $\p'$ given $\p$ is the \structure{conditional expectation}:
  \begin{displaymath}
    E[\log p(x,y;\p') \ | \ x, \p] =
    \int p(y| x; \p)\ \log{p(x,y;\p')}\ \mathsf{d}y
  \end{displaymath}
\end{frame}

\iffalse
\begin{frame}
 \frametitle{Kullback-Leibler Statistics}
 Using the argument that the position of a maximum of a function does
 not change by scaling the function value with a positive
 factor and by translation, we see that for give $X$ and $\p$::
 \begin{eqnarray*}
   \hat{\p}' &=& \argmax_{\p'} \int p(Y| X; \p)\ \log{p(X,Y;\p')} \ dy\\
            &=& \argmax_{\p'} \frac{1}{p(X;\p)}
                                 \int p(X, Y; \p)\ \log
                                 {p(X,Y;\p')} \ dy\\
            &=& \argmax_{\p'} \int p(X, Y; \p)\ \log{p(X,Y;\p')} \ dy\\
            &=& \argmax_{\p'} -\int p(X, Y; \p)\ \log{p(X,Y;\p')}\ dy
                                 +\int p(X, Y; \p)\ \log{p(X,Y;\p)}\ dy\\
            &=& \argmax_{\p'} \int p(X, Y; \p)\
                \log\frac{p(X,Y;\p')}{p(X,Y;\p)} \ dy
 \end{eqnarray*}
\end{frame}
\fi

\begin{frame}
  \frametitle{Key Equation}

  The \structure{key equation} of the Expectation Maximization algorithm (EM algorithm) can be rewritten:

  \begin{center}
    \tikz[baseline]{
      \node[fill=bl1!100, anchor=base, rounded corners=3pt, inner sep=3mm] (d1) {
        \color{bl3}
        $\log p\left( x; \hatp^{(i+1)} \right) =
        Q\left( \hatp^{(i)}; \hatp^{(i+1)} \right) +
        H\left( \hatp^{(i)}; \hatp^{(i+1)} \right)$
      };
    }
  \end{center}
  \spread

  \begin{itemize}
    \item Below we will motivate that the maximization of the Kullback-Leibler statistics can replace the optimization of the log-likelihood function. \\[.5cm]
    \item A complete proof can be found in the literature (see Further Readings).
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Entropy Changes with Iterations}

  For the entropy we get the inequality:
  \begin{displaymath}
    H(\p;\p') \geq H(\p; \p) \pause
  \end{displaymath}

  This is shown rather straightforward:

  \begin{eqnarray*}
    & & \hspace{-1cm}H({\p};{\p'}) - H(\p;{\p}) \\ \pause
    &=& - \int p(y|x;\p) \, \log p(y|x;{\p}')\ \mathsf{d}y
        + \int p(y|x;\p) \, \log p(y|x;{\p})\ \mathsf{d}y \\ \pause
    &=& - \int p(y|x;\p) \, \log \frac{p(y|x;{\p}')}{p(y|x;{\p})}\ \mathsf{d}y \\ \pause
    &=&   \int p(y|x;\p) \, \log \frac{p(y|x;{\p})}{p(y|x;{\p'})}\ \mathsf{d}y
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{Entropy Changes with Iterations \cont}

  The difference of the considered entropies

  \begin{eqnarray*}
    & & \hspace{-1cm}H({\p};{\p'})-H(\p;{\p}) = \\
    &=& \int p(y|x;{\p}) \, \log \frac{p(y|x;{\p})}{p(y|x;{\p'})}\ \mathsf{d}y \ge 0
  \end{eqnarray*}

  is thus the Kullback-Leibler divergence of the pdf's $p(y|x;{\p})$ and $p(y|x;{\p'})$, \\
  and the Kullback-Leibler divergence is known to be non-negative.
\end{frame}

\begin{frame}
  \frametitle{Entropy Changes with Iterations \cont}

  The best to see this is to make use of the inequality
  \begin{displaymath}
  \log(x) \leq x-1
  \end{displaymath}

  and conclude:
  \begin{eqnarray*}
    \int p(x) \, \log \frac{p(x)}{q(x)}\ \mathsf{d}x
      & =  & -\int p(x) \, \log \frac{q(x)}{p(x)}\ \mathsf{d}x \\ \pause
      &\geq& \int p(x) \left( 1 - \frac{q(x)}{p(x)} \right) \, \mathsf{d}x\\ \pause
      & =  & 1-1 = 0
  \end{eqnarray*}
\end{frame}

\subsection{Expectation Maximization Algorithm}

\begin{frame}
  \frametitle{Expectation Maximization Algorithm}

  \structure{The basic idea of the EM algorithm:} \\[0.5cm]

  Instead of maximizing the log-likelihood function on the left hand side \\
  of the key-equation, we maximize the Kullback-Leibler statistics iteratively \\
  while ignoring the entropy term.
\end{frame}

\begin{frame}
  \frametitle{Expectation Maximization Algorithm \cont}

  \begin{centernss}
    \resizebox{.85\linewidth}{!}{
      \begin{struktogramm}(100,50)
        \assign{Initialization: $\hatp^{(0)}$}
        \assign{$i \gets -1$}
        \until{$\hatp^{(i+1)}$ $=$ $\hatp^{(i)}$}
          \assign{$i \gets i+1$}
          \assign{ \structure{Expectation step:} \\[.25cm]
            \centerline{$Q\left( \hatp^{(i)}\, ; \,\p \right) := \int p\left( y|x; \hatp^{(i)} \right) \log p(x, y;{\p} )\ \mathsf{d}y$} }
          \assign{ \structure{Maximization step:} \\[.25cm]
            \centerline{$\displaystyle {\hatp}^{(i+1)} \gets \argmax_{{\p}} Q\left( \hatp^{(i)}\,;\, \p \right)$} }
        \untilend
        \assign {Output: estimate $\hatp \gets \hatp^{(i)}$}
      \end{struktogramm}
    }
  \end{centernss}
%
%   \begin{center}
%     \begin{struktogramm}{10cm}{0.7cm}
%       \BLOCK {Initialization: $\hatp^{(0)}$}
%       \BLOCK {Set $i:=-1$}
%       \REPEAT {
%         \BLOCK { Set $i:= i+1$ }
%         \BLOCK { \structure{Expectation Step:} \\
%           $Q\left( \hatp^{(i)}\, ; \,\p \right) := \int p\left( y|x; \hatp^{(i)} \right) \log p(x, y;{\p} )\ \mathsf{d}y$ }
%         \BLOCK { \structure{Maximization Step:} \\
%           $\displaystyle {\hatp}^{(i+1)} := \argmax_{{\p}} Q\left( \hatp^{(i)}\,;\, \p \right)$ }
%       }
%       UNTIL {$\hatp^{(i+1)}$ $=$ $\hatp^{(i)}$}
%       \BLOCK {Output: estimate $\hatp:= \hatp^{(i)}$}
%     \end{struktogramm}
%   \end{center}
\end{frame}

\subsection{Advantages of the EM Algorithm}

\begin{frame}
  \frametitle{Advantages of the EM Algorithm}

  A few \structure{practical positive aspects} regarding the EM algorithm:

  \begin{itemize}
    \item The maximum of the KL statistics is usually computed using zero crossings of the gradient. \\[.3cm] \pause
    \item Mostly we find \structure{closed form iteration schemes}. \\[.3cm] \pause
    \item Easy to implement closed form iteration formulas (if these exist). \\[.3cm] \pause
    \item Iteration scheme is numerically robust. \\[.3cm] \pause
    \item Closed form iterations have constant memory requirements. \\[.3cm] \pause
    \item If the argument in the logarithm can be factorized properly, \\
      we observe a decomposition of the parameter space \\
      (independent lower dimensional sub-spaces)
  \end{itemize}
\end{frame}

\subsection{Drawbacks of EM}

\begin{frame}
  \frametitle{Drawbacks of EM}

  The EM algorithm has a few \structure{major drawbacks}: \pause

  \begin{itemize}
    \item slow, slow, slow convergence \\
      (should not be used in run time critical applications) \pause
    \item local optimization method, i.\,e.\ the initialization $\hatp^{(0)}$ has to lie in the area of attraction of the global maximum.
  \end{itemize}

  \begin{center}
    \resizebox{.6\linewidth}{!}{
      \input{\texfigdir/em_local.pstex_t}
    }
  \end{center}
\end{frame}

\subsection{Constrained Optimization}

\begin{frame}
  \frametitle{Constrained Optimization}

  Many optimization problems in the context of the EM algorithm are \\
  of the following form: \\[.25cm]

  \begin{ovalblock}{Example}
    \small
    Optimize the multivariate function
    {\footnotesize
      \begin{displaymath}
        f_0(p_1,p_2,\dots,p_K) = \sum_{k=1}^{K} a_k \ \log p_k
      \end{displaymath}
    }
%
    subject to
    {\footnotesize
      \begin{eqnarray*}
        \sum_{k=1}^K p_k & =    & 1 \\
        p_k              & \geq & 0
      \end{eqnarray*}
    }
    \vspace{-.5cm}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{Constrained Optimization \cont}

  \begin{ovalblock}{Example}
    \small
    Application of the \structure{Lagrange multiplier} method:
    {\footnotesize
      \begin{displaymath}
        L(p_1,p_2,\dots,p_K) =
        \sum_{k=1}^K a_k\ \log p_k + \nu \left( \sum_{k=1}^K p_k-1 \right) \pause
      \end{displaymath}
    }

    The optimization can be done using the \structure{partial derivative}:
    {\footnotesize
      \begin{displaymath}
        \frac{\partial L(p_1,p_2,\dots,p_K)}{\partial p_k} =
        \frac{a_k}{p_k} + \nu \stackrel{!}{=} 0 \quad .
      \end{displaymath}
    }
    \vspace{-.25cm}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{Constrained Optimization \cont}

  \begin{ovalblock}{Example \cont}
    \small
    The \structure{Lagrange multiplier} is:
    {\footnotesize
    \begin{displaymath}
      a_k= -\nu p_k~. \pause
    \end{displaymath}
    }

    Due to the fact that the $p_k$'s are unknown, we have to apply a trick to get $\nu$. \\
    We just sum both sides of the above equation over all $k$ and get:
    {\footnotesize
      \begin{displaymath}
        \nu = -\sum_{k=1}^K a_k~. \pause
      \end{displaymath}
    }

    The estimator for $p_k$ now is:
    {\footnotesize
      \begin{displaymath}
        \hat{p}_k = \frac{a_k}{\sum_{l=1}^K a_l}
      \end{displaymath}
    }
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{EM Algorithm: Example}

  \begin{ovalblock}{Example}
    \small
    Estimate the priors $p_k$ of classes $k=1,2, \dots, K$ from the observation $x$ \\
    where the probability density function of observations is given by the marginal over all classes:
    {\footnotesize
    \begin{displaymath}
      p(x;\BiasField) = \sum_{k=1}^K p_k\,p(x|k; \BiasField) \pause
    \end{displaymath}
    }

    \structure{Application of the EM scheme:}
    \begin{itemize}
      \item observable random measurement: $x$
      \item hidden random measurement: $k$
      \item parameter set: $\p = \{p_k; k=1,\dots, K\}$
    \end{itemize}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{EM Algorithm: Example \cont}

  \begin{ovalblock}{Example}
    \small
    For illustration purposes let us consider three classes.
    If events, in this case 2-D points, are labeled by colors representing different classes, the priors are easily estimated by relative frequencies.

    \begin{center}
      \resizebox{.45\linewidth}{!}{
        \input{\texfigdir/gauss_em1.pstex_t}
      }
    \end{center}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{EM Algorithm: Example \cont}

  \begin{ovalblock}{Example \cont}
    \small
    The problem appears quite difficult, if the class (color) labels are missing.

    \begin{center}
      \resizebox{.45\linewidth}{!}{
        \input{\texfigdir/gauss_em2.pstex_t}
      }
    \end{center}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{EM Algorithm: Example \cont}

  \begin{ovalblock}{Example}
    \small
    The \structure{Kullback-Leibler statistics} results in:
    {\footnotesize
      \begin{eqnarray*}
        Q\left( \hatp^{(i)}; \hatp^{(i+1)} \right)
          &=& \sum_{k=1}^{K} a_k \log \left( \hat{p}_k^{(i+1)} \, p(x|k;\BiasField) \right) \\ \pause
          &=& \sum_{k=1}^{K} a_k \left( \log \hat{p}_k^{(i+1)} + \log p(x|k; \BiasField) \right) \\ \pause
          &=& \sum_{k=1}^{K} a_k \log \hat{p}_k^{(i+1)} +
              \sum_{k=1}^{K} a_k \log p(x|k;\BiasField)
      \end{eqnarray*}
    }
    where
    {\footnotesize
      \begin{eqnarray*}
        a_k
          &=& \frac{\hat{p}_k^{(i)} \, p(x|k; \BiasField)}
                   {\sum_{j}\hat{p}_j^{(i)} \, p(x|j; \BiasField)}
      \end{eqnarray*} \vspace{-.2cm}
    }
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{EM Algorithm: Example \cont}

  \begin{ovalblock}{Example \cont}
    \small
    Now we compute the gradient with respect to $\hat{p}_k^{(i+1)}$ and its zero crossing.\\
    The final estimator for priors now is a closed form iteration scheme:

   \begin{displaymath}
     \hat{p}_k^{(i+1)} =
     \frac{
       \frac{\hat{p}_k^{(i)} \, p(x|k; \BiasField)}
            {\sum_{j}\hat{p}_j^{(i)} \, p(x|j, \BiasField)}
     }{
       \sum_l \frac{\hat{p}_l^{(i)} \, p(x|l; \BiasField)}
                   {\sum_j \hat{p}_j^{(i)} \, p(x|j; \BiasField)}
     } \pause =
     \frac{\hat{p}_k^{(i)} \, p(x|k; \BiasField)}
          {\sum_j \hat{p}_j^{(i)} \, p(x|j; \BiasField)}
    \end{displaymath}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{Initialization of Priors:}

  \begin{itemize}
   \item Use prior medical knowledge about the frequency of tissue classes \\[.5cm]
   \item If no prior information is available, assume uniform distribution
  \end{itemize}
\end{frame}

\iffalse
\fi
\iffalse

\begin{figure}
  \centerline{\psscaleboxto(\linewidth,0){%
   \includegraphics[height=.4\textheight]{mr_wells0.eps}
   \includegraphics[height=.4\textheight]{mr_wells1.eps}
  }}
 \caption[MRI enhancement \cite{IMG-Wells}]
   {Original MRI (left) and the estimated bias field (right)
          using the EM technique
         (Courtesy of W. Wells)}\label{f:mr:enhance}
\end{figure}

\begin{figure}
  \centerline{\psscaleboxto(\linewidth,0){%
   \includegraphics[height=.4\textheight]{mr_wells2.eps}
   \includegraphics[height=.4\textheight]{mr_wells3.eps}
  }}
 \caption[MRI segmentation \cite{IMG-Wells}]
   {Original MRI (left) and the result of segmentation (right)
         (Courtesy of W. Wells)}\label{f:mr:segm}
\end{figure}

\begin{figure}
  \centerline{\psscaleboxto(\linewidth,0){%
   \includegraphics[height=.4\textheight]{mr_wells4.eps}
   \includegraphics[height=.4\textheight]{mr_wells5.eps}
  }}
 \caption{White matter surface computed by conventional and
          adaptive segmentation (Courtesy of W. Wells)}\label{f:ii:struct}
\end{figure}
\fi

\subsection{Lessons Learned}

\begin{frame}
  \frametitle{Lessons Learned}

  \begin{itemize}
    \item Standard parameter estimation method: ML estimation \\[.5cm]
    \item If the prior pdf of the parameters is known: MAP estimation \\[.5cm]
    \item In the presence of latent random variables: EM algorithm \\[.5cm]
    \item EM advantages: decomposition of search space, closed form iteration schemes \\[.5cm]
    \item EM disadvantage: slow convergence, local method
  \end{itemize}
\end{frame}

\input{nextTime.tex}

\subsection{Further Readings}

\begin{frame}
  \frametitle{Further Readings}

  \small
  \begin{itemize}
    \item Easy to understand tutorial on ML estimation: \\[.15cm]
      In Jae Myung: \\
      \point\href{http://faculty.psy.ohio-state.edu/myung/personal/mle.pdf}{\structure{Tutorial on maximum likelihood estimation}}, \\
      Journal of Mathematical Psychology, 47(1):90-100, 2003 \\[.25cm]
    \item The classics for an introduction to the EM algorithm is: \\[.15cm]
      A.\,P.\ Dempster, N.\,M.\ Laird, D.\,B.\ Rubin: \\
      \point\href{http://web.mit.edu/6.435/www/Dempster77.pdf}{\structure{Maximum Likelihood Estimation from Incomplete Data via the EM Algorithm}}, \\
      Journal of the Royal Statistical Society, Series B, 39(1):1-38. \\[.25cm]
    \item W.\,H.\ Press, S.\,A.\ Teukolsky, W.\,T.\ Vetterling, B.\,P.\ Flannery: \\
      \point\href{http://www.nr.com/}{\structure{Numerical Recipes}}, \\
      3rd Edition, Cambridge University Press, 2007.
  \end{itemize}
\end{frame}

\subsection{Comprehensive Questions}

\begin{frame}
  \frametitle{Comprehensive Questions}

  \begin{itemize}
    \item What is a Gaussian Mixture Model? \\[0.5cm]
    \item What is the missing information principle? \\[0.5cm]
    \item Write down the key equation for the EM algorithm: \\[0.5cm]
    \item Is the EM algorithm a local or a global parameter estimation method?
  \end{itemize}
\end{frame}


\documentclass[11pt]{amsart}
\usepackage{geometry}                % See geometry.pdf to learn the layout options. There are lots.
\geometry{letterpaper}                   % ... or a4paper or a5paper or ...
%\geometry{landscape}                % Activate for for rotated page geometry
\usepackage{amsmath}    % Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{epstopdf}
\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}

\title{Linearized Split Bregman}
\author{Wang Yu}
%\date{}                                           % Activate to display a given date or no date

\begin{document}
\maketitle
\section{proximity operator}\label{sec:prox}
Firstly, Consider the following problem

\begin{equation}
prox_{x}=\arg\min_{z}\|x-z\|^2_2 + \lambda \|z\|_1
\end{equation}
since all the functions in the problem are convex, the optimality condition is
\[
0\in\nabla(\|x-z\|^2_2)+\lambda \partial\|z\|_1
\]
Notice that the above equation is just summation of each component, it is equivalent to considering an arbitrary i-th component.  if $\|z_i\| \neq 0$, the subgradient of $|z_i|$ is just  the gradient, which is $sgn(z_i) $, where $z_i$ is one component of $z$, thus the above relation becomes the equality

\begin{align}
0=&2(z_i - x_i) + \lambda sgn(z_i) \\
z_i = &x_i - \frac{1}{2}\lambda sgn(z_i)
\end{align}
from above equation it is clear that if $z_i > 0$, $x_i$ must be greater then 0, and if $z_i <0$, $x_i < 0$ also holds. Hence $sgn(z_i) = sgn(x_i)$, in turn the following is true

 \begin{equation}
z_i=x_i - \frac{1}{2}\lambda sgn(x_i)
\end{equation}

For $z_i$ equals 0 the subgraidient of  $|z_i|$is the interval $[-1, 1]$,
\[
0\in2 ( z_i- x_i)+ \lambda [-1, 1] => x_i \in [-\frac{\lambda}{2}, \frac{\lambda}{2}]
\]
Combine two cases, for each $z_i$ the minimizer satisfy the following relation
\begin{equation}
z_i=\left\{
\begin{array}{lr}
0   & |x_i| \leq \frac{\lambda}{2} \\
x_i - \frac{1}{2}\lambda sgn(x_i) & |x_i| > \frac{\lambda}{2}
\end{array}
\right.
\end{equation}
%\subsection{}
\section{General linear $L_1$ minimization}
In many applications, the problem can be formulized as
\begin{equation}
\arg\min_{z}\|Az-x\|^2_2 + \lambda \|z\|_1
\end{equation}
where $A$ is a linear operator. Approximate the above equation by Taylor expansion at $z^k$, the linearized verion of the problme becomes an iterative shceme
\begin{equation}
z^{k+1}=\arg\min_{z} \lambda \|z\|_1 + \|Az^k-x\|^2_2 + (A^tAz^k-A^tx, z-z^k) + \frac{1}{2\delta}\|z-z^k\|^2
\end{equation}
The last penalty term is to make sure $z_{k+1}$ and $z_{k}$ not too far from each other so that Taylor expansion is still valid.
Throw away some constants, the scheme can be simplified as
\begin{equation}
z^{k+1}=\arg\min_{z} \lambda \|z\|_1 + \frac{1}{2\delta}\|z-(z^k-\delta(A^tAz^k-A^tx))\|^2
\end{equation}
Note that the right hand side is of the form derived in Section \ref{sec:prox}, hence $z^{k+1}$ can be solved as
\begin{equation}
z_i=\left\{
\begin{array}{lr}
0   & |y_i| \leq \lambda\delta\\
y_i - \frac{1}{2}\lambda sgn(y_i) & |y_i| > \lambda\delta
\end{array}
\right.
\end{equation}
where $y_i = z^k_i-\delta(A^tAz^k_i-A^tx_i)$
\section{Linearized split Bregman and Augmented Lagrange method}
\section{Appendix}
subderivative
\end{document}


\documentclass[12pt]{article}
\author{David Alves}

\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{dirtytalk}
\usepackage{forest}
\usepackage{skak}
\usepackage{tikz}
\usepackage{titling}

\title{Math 142 Problem Set 2}
\author{David Alves}
\date{2016-09-06}

\begin{document}
\pagenumbering{gobble}

\begin{center}
\Large \thetitle \\
\large \theauthor \\
\thedate
\end{center}

\subsection*{Sources}

    \begin{itemize}
    \item Notes from lecture and my own memory from Math 42 for solving the problems
    \item http://tex.stackexchange.com and https://www.sharelatex.com for help with \LaTeX
    \end{itemize}

\section{Chess Piece Ordering}

\subsection*{Problem Statement}
Suppose we play chess where the positions of all 16 pawns are the same, but each player permutes the 8 pieces behind his/her pawns in any way they want. How many possible starting positions of the game are there? (the two knights, two rooks, etc are indistinguishable). Optional: How would you choose to place your places?
\subsection*{Solution}

    There are eight pieces on each side, so if all pieces were distinct there would be $8!=40,320$ possible orderings for each player by the product principle. However, the two knights, two rooks, and two bishops are each interchangeable, so by the quotient principle we have $\frac{8!}{2!2!2!}=5,040$ possible orderings on each side. For each ordering that player 1 chooses, player 2 can choose any ordering, so by the product principle we have $5,040^2=25,401,600$ total board configurations.

    \paragraph{Optional}
    I would use this configuration:
    \newline
    \newgame
    \fenboard{8/8/8/8/8/8/PPPPPPPP/BNRRQNKB w - - 0 1}

    \begin{center}
    \showboard
    \end{center}

    Bishops are on the long diagonals to get control of the center. The rooks and queen are on the center files for the same reason. The king is already in \say{castled} position for safety and all three pawns in front of the king are protected by at least one piece other than the king.

\section{Subset Pairs}
\subsection*{Problem Statement}
How many pairs $(A_1, A_2)$ of subsets of $[n]$ are there such that $A_1 \cap A_2 = \emptyset$?
\subsection*{Solution}
An ordered pair $(A_1, A_2)$ such that $A_1 \cap A_2 = \emptyset$ consists of an assignment of each element in $[n]$ to either $A_1$, $A_2$, or neither. (An element cannot be in both since $A_1$ and $A_2$ are disjoint). This gives three possibilities for each element, for a total of $3^n$ ordered pairs.

\section{Function Counting}
\subsection*{Problem Statement}
Suppose the domain is $[m]$ and the codomain is $[n]$ for $n,m \in \mathbb{N}$. Count the number of injective, surjective, and bijective functions on $[m] \rightarrow [n]$.
\subsection*{Solution}
\subsubsection*{Injective}

For $n < m$, no injective functions exist by the pigeonhole principle since two values in $[m]$ would have to  map to the the same value in $[n]$, violating the definition of injective. This gives $n$ choices for $f(1)$, $n-1$ choices for the f(2), etc. Multiplying these choices due to the product principle for all $m$ elements of the domain gives
    $$(n) \times (n-1) \times (n-2) \times \ldots \times (n - m + 1)$$
    Which is equivalent to $$\frac{n!}{(n-m)!}$$

\subsubsection*{Surjective}
For $n > m$, no surjective functions exist by the pigeonhole principle (since each element of the domain corresponds with at most one element of the codomain, there must be 1 or more elements of the codomain which do not correspond with any elements of the domain if $n > m$). Unfortunately I haven't been able to figure out a way to count the surjective functions when $n \leq m$. \emph{(Note: This is defined as $S(n)$, but we didn't cover that until after this homework was due.)}

\subsubsection*{Bijective}

If $n < m$ or $n > m$, no bijective functions exist because of the pigeonhole principle (explained under injective / surjective above). For $n = m$, There are $n$ elements of both the domain and the codomain. We have $n$ choices for $f(1)$, $n-1$ choices for $f(2)$, \ldots, 1 choice for $f(n)$. This gives a total of $n!$ assignments, so there are $n!$ bijective functions on $[n] \rightarrow [n]$.

\section{Basketball Team Counting}

\subsection*{Problem Statement}
If you have 14 copies of Miguel Jordan (who can play all positions), two clones of Shackle O'Neil (who can only play center) and three clones of Stephan Curry (who can play either guard position) and you need a center, a power forward, a small forward, a point guard, and a shooting guard (all considered different positions) to make a team, how many teams can you make?
\subsection*{Solution}

The final answer is 473,928. The following is a tree representation of the calculation. Nodes labeled \say{OR} are disjoint and use the sum principle to add their children, while nodes labeled \say{AND} use the product principle and multiply their children. A number in parenthesis indicates the number of possible choices for that assignment.

\begin{center}
\begin{forest}
  for tree={
    grow'=0,
    child anchor=west,
    parent anchor=south,
    anchor=west,
    calign=first,
    edge path={
      \noexpand\path [draw, \forestoption{edge}]
      (!u.south west) +(7.5pt,0) |- node[fill,inner sep=1.25pt] {} (.child anchor)\forestoption{edge label};
    },
    inner ysep=1.75pt,
    before typesetting nodes={
      if n=1
        {insert before={[,phantom]}}
        {}
    },
    fit=band,
    before computing xy={l=15pt},
  }
[OR
    [AND
        [Center is an SN (2)]
        [OR
            [AND
                [SG is a SC (3)]
                [OR
                    [PG is an SC (2) AND SF is an MJ (14) AND PF is an MJ (13) \equal 364]
                    [PG is an MJ (14) AND SF is an MJ (13) AND PF is an MJ (12) \equal 2184]
                ]
            ]
            [AND
                [SG is an MJ (14)]
                [OR
                    [PG is an SC (3) AND SF is an MJ (13) AND PF is an MJ (12) \equal 468]
                    [PG is an MJ (13) AND SF is an MJ (12) AND PF is an MJ (11) \equal 1716]
                ]
            ]
        ]
    ]
    [AND
        [Center is an MJ (14)]
        [OR
            [AND
                [SG is an SC (3)]
                [OR
                    [PG is an SC (2) AND SF is an MJ (13) AND PF is an MJ (12) \equal 312]
                    [PG is an MJ (13) AND SF is an MJ (12) AND PF is an MJ (11) \equal 1716]
                ]
            ]
            [AND
                [SG is an MJ (13)]
                [OR
                    [PG is an SC (3) AND SF is an MJ (12) AND PF is an MJ (11) \equal 396]
                    [PG is an MJ (12) AND SF is an MJ (11) AND PF is an MJ (10) \equal 1320]
                ]
            ]
        ]
    ]
]
\end{forest}
\begin{tabular}{ |p{1cm}|p{3cm}|  }
 \hline
 \multicolumn{2}{|c|}{\textbf{Key}} \\
 \hline
 PG & Point Guard \\
 SG & Shooting Guard \\
 PF & Power Forward \\
 SF & Small Forward \\
 \hline
 SN & Shackle O'Neil \\
 MJ & Miguel Jordan \\
 SC & Stephan Curry \\
 \hline
\end{tabular}

\end{center}
Evaluating this tree gives
\begin{multline*}
2*\Biggl(3*(364 + 2184)+14*(468+1716)\Biggr)+\\
14*\Biggl(3*(312+1716)+13*(396+1320)\Biggr)= 473,928
\end{multline*}

\section{Function Classification}
\subsection*{Problem Statement}
Let $Y$ be the set of subsets of $[n]$. Is the following function an injection? Surjection? Bijection? For each subset $X \in [n]$, let $f(X)$ be the set of all elements in $[n]$ that are \emph{not} in $X$.
\subsection*{Solution}

$f$ is a bijection because it is both injective and surjective.
\subsubsection*{Injective Proof}

If $f$ is not injective, there exists some $X$ and $X'$ such that $X \neq X'$ and $f(X) = f(X')$. There must be some member m of Y for which $m \in X \neq m \in X'$ since $X \neq X'$. However, this means that $m \in f(X) \neq m \in f(X')$ (since membership on both sides is negated), which contradicts the statement that $f(X) = f(X')$. Therefore $f(X)$ is injective.

\subsubsection*{Surjective Proof}
If $f(X)$ is not surjective, there must be some $N \subset Y$ for which there is no $X \subset Y$ such that $f(X) = N$.
N consists of an assignment of each element in Y to be in the $Y$ or not in $Y$. We can construct $N'$ such that each element of $Y$ that is in $N$ is not in $N'$, and each element of $Y$ that is not in $N$ is in $N'$. This gives $N' \subset Y$ and $f(N') = N$, which contradicts our statement that there is no $X$ such that $X \subset Y$ and $f(X) = N$. Therefore $f(X)$ is surjective.

\section{Time Spent}
I spent about 12 hours on this homework assignment. Roughly eight of those were spent doing the problems while the rest were spent learning \LaTeX.
\end{document}


\chapter{Application Programming Interface (API)}

\verd\ was designed with a C interface so that it can be used in a variety of
applications.

Each metric has its own function.  For example, the Hex Condition Number metric
is:

\begin{verbatim}
double v_hex_condition(int num_nodes,
                       double node_coordiantes[][3])
\end{verbatim}

It may be used as follows:
\begin{verbatim}
double coords[8][3];
...
double condition_value = v_hex_condition(8, coords);
\end{verbatim}

The number of nodes is given for each element, and the implementation may be
expanded to include higher order elements.

Each type of element has one function for getting multiple metrics at the same
time.  The following is a prototype to get multiple metrics for a hexahedron:
\begin{verbatim}
double v_hex_quality(int num_nodes,
                     double node_coordiantes[][3],
                     unsigned int request_flag,
                     struct HexMetricVals *metric_vals)
\end{verbatim}

If one wants multiple metrics for an element, it is usually less computationally
expensive to use this approach because some metrics share the same computations.
For example, computing the Jacobian and shape metrics of a hexahedron both use the
Jacobian matrix.
It may be used as follows:
\begin{verbatim}
double coords[8][3];
HexMetricVals vals;
double jacobian_value;
double shape_value;
int request = V_HEX_JACOBIAN | V_HEX_SHAPE;
...
v_hex_quality(8, coords, request, &vals);
double jacobian_value = vals.jacobian;
double shape_value = vals.shape;
\end{verbatim}


\documentclass[aps,superscriptaddress,tightenlines,nofootinbib,floatfix,longbibliography,notitlepage]{revtex4-1}
\usepackage[left=18mm,right=19mm,top=23mm,bottom=16mm]{geometry}
\usepackage{amsmath,amssymb}
\usepackage{bm,bbm}
\usepackage{comment}
\usepackage{graphicx}
\usepackage{color}
\usepackage[dvipsnames]{xcolor}
\usepackage{slashed}
\usepackage[multidot]{grffile} %because multiple dots in filenames confuse latex
\usepackage[
    colorlinks=true,
    allcolors=blue
]{hyperref}
\usepackage{pgfplots}
%%%%
%%%%    Document preparation
%%%%

\begin{document}

\section{Why "truncated" dispersion Lüscher works}

Let's start with infinite volume continuum with a cutoff.
We know that for a contact interaction, the on-shell $T$-matrix is given by
\begin{equation}
	V(p', p) = c(\Lambda) \qquad \Rightarrow \qquad
	T(p, \Lambda) = \frac{c(\Lambda)}{1 - c(\Lambda) I(p, \Lambda)} \, .
\end{equation}
In three-d, we also find that
\begin{equation}
	I(p, \Lambda)
	=
	\frac{4 \pi}{(2\pi)^3} \lim_{\epsilon \to 0}\int_{0}^{\Lambda} d q q^2 \frac{2 \mu}{p^2 - q^2 + i \epsilon}
	=
	-\frac{\mu}{2\pi^2}   \left(2 \Lambda +p \log \left(1-\frac{2 p}{\Lambda +p}\right)\right) + \frac{i \mu p}{2 \pi}
	\, .
\end{equation}
and thus we know that
\begin{equation}
	p \cot \delta(p)
	= \frac{2 \pi}{\mu}\frac{1}{T(p, \Lambda)} + i p
	= \frac{2 \pi}{\mu} \frac{1}{c(\Lambda)} + \frac{2 \Lambda}{\pi} + \frac{2 p}{\pi} \log \left( \frac{\Lambda - p}{\Lambda + p}\right)
\end{equation}
So if you choose
\begin{equation}
	\frac{2 \pi}{\mu} \frac{1}{c(\Lambda)} + \frac{2 \Lambda}{\pi} = - \frac{1}{a_0} \, ,
\end{equation}
the limit of $\Lambda \to \infty$ is well defined and reproduces the scattering length $a_0$.

In other words, if you fix your contact interaction properly, you get
\begin{equation}
	- \frac{1}{a_0}
	=
	\lim\limits_{\Lambda \to \infty} \mathcal R \left( \frac{2 \pi}{\mu} \frac{1}{T(p, \Lambda)} \right)
	=
	\lim\limits_{\Lambda \to \infty} \mathcal R \left(\frac{2 \pi}{\mu}   \frac{1 - c(\Lambda) I(p, \Lambda)}{c(\Lambda)} \right)
	\, .
\end{equation}
Note that this expression is constant in $p$.

Now we play a few funny tricks.
First, let me identify $\Lambda$ with $\frac{\pi}{\epsilon}$.
This gives
\begin{equation}
	- \frac{1}{a_0}
	=
	\frac{2 \pi}{\mu}
	\lim\limits_{\epsilon \to 0} \mathcal R \left(\frac{1 - c\left(\frac{\pi}{\epsilon}\right) I\left(p, \frac{\pi}{\epsilon}\right)}{c\left(\frac{\pi}{\epsilon}\right)} \right)
	\, .
\end{equation}
Next, I introduce this parameter $n_s$ with deforms the $q^2$ inside $I(p, \epsilon/\pi)$ in such a way\footnote{Also I change the integration boundaries to be cubic but that's another story we found to not make a difference in the $\Lambda \to \infty$ limit.} that
\begin{equation}
	q^2_{n_s} = \frac{1}{\epsilon^2}\sum_{i=1}^3\sum_{n=0}^{n_s}  \gamma_n^{(n_s)} \cos(\epsilon q_i n \epsilon) \, ,
\end{equation}
with coefficients $\gamma_n^{(n_s)}$ such that $\lim\limits_{n_s \to \infty} q^2_{n_s} = q^2$ and $q^2_{n_s} = q^2(1 + \mathcal O(q\epsilon)^{2 n_s})$.
Note that because $p_\text{max} = \pi / \epsilon$, we actually have to do the $n_s \to \infty$ limit first.
Though probably one can find an argument why they commute.
Also, because I am particularly funny today, I also say that $c$ depends on this $n_s$ parameter as well and is smooth in the limit of $n_s \to \infty$.

So we have
\begin{equation}
	- \frac{1}{a_0}
	=
	\frac{2 \pi}{\mu}
	\lim\limits_{\epsilon \to 0} \lim\limits_{n_s \to \infty}\mathcal R \left(\frac{1 - c^{(n_s)}\left(\frac{\pi}{\epsilon}\right) I^{(n_s)}\left(p, \frac{\pi}{\epsilon}\right)}{c^{(n_s)}\left(\frac{\pi}{\epsilon}\right)} \right)
	\, .
\end{equation}
Furthermore, one can show that solving the Schrödinger equation on the lattice in finite volume is equal to
\begin{equation}
	0 = 1 - c^{(n_s)}\left(\frac{\pi}{\epsilon}\right) I^{(FV, n_s)}\left(\sqrt{2\mu E_i^{(FV, n_s)}}, \frac{\pi}{\epsilon}\right)
	\, ,
\end{equation}
where $E_i$ is an eigen energy solution to the Schrödinger equation which was solved for a given $c^{(n_s)}\left(\frac{\pi}{\epsilon}\right)$.
Note the the energies know about finite volume and is independent of the previous $p$!
The finite volume "integral" $I^{(FV, n_s)}$, now sums over discrete $q_i$ which range from $[0, N)$ where $N = L / \epsilon$ with a momentum lattice spacing of $2\pi/L$.

If one adds this delicate zero, we find that
\begin{align}
	- \frac{1}{a_0}
	&=
	\frac{2 \pi}{\mu}
	\lim\limits_{\epsilon \to 0} \lim\limits_{n_s \to \infty}\mathcal R \left(
		\frac{1 - c^{(n_s)}\left(\frac{\pi}{\epsilon}\right) I^{(n_s)}\left(p, \frac{\pi}{\epsilon}\right)
		-
			\left[1 - c^{(n_s)}\left(\frac{\pi}{\epsilon}\right) I^{(FV, n_s)}\left(\sqrt{2\mu E_i^{(FV, n_s)}}, \frac{\pi}{\epsilon}\right)\right]
		}
		{c^{(n_s)}\left(\frac{\pi}{\epsilon}\right)}
	\right)
	\\
	&=
	\frac{2 \pi}{\mu}
	\lim\limits_{\epsilon \to 0} \lim\limits_{n_s \to \infty}\mathcal R \left(
		I^{(FV, n_s)}\left(\sqrt{2\mu E_i^{(FV, n_s)}}, \frac{\pi}{\epsilon}\right)
		-
		I^{(n_s)}\left(p, \frac{\pi}{\epsilon}\right)
	\right)
\end{align}
Here comes the thing Evan and I have been discussing today.

If everything is convergent for particular values of $p$, then we also find that
\begin{align}
	- \frac{1}{a_0}
	&=
	\frac{2 \pi}{\mu}
	\lim\limits_{\epsilon \to 0} \lim\limits_{n_s \to \infty}\mathcal R \left(
		I^{(FV, n_s)}\left(\sqrt{2\mu E_i^{(FV, n_s)}}, \frac{\pi}{\epsilon}\right)
		-
		I^{(n_s)}\left(p=0, \frac{\pi}{\epsilon}\right)
	\right)
	\, ,
\end{align}
Which I will call (truncated) dispersion Lüscher zeta function from now on (modulo factors)
\begin{equation}
	\tilde S^{(FV, n_s)}(E_i^{(FV, n_s)}, \epsilon)
	\equiv
	\mathcal R \left(
		I^{(FV, n_s)}\left(\sqrt{2\mu E_i^{(FV, n_s)}}, \frac{\pi}{\epsilon}\right)
		-
		I^{(n_s)}\left(p=0, \frac{\pi}{\epsilon}\right)
	\right)
	\, .
\end{equation}

Because $I^{(n_s)}\left(p=0, \frac{\pi}{\epsilon}\right)$ is independent off $E_i^{(FV, n_s)}$ and since $I^{(FV, n_s)}\left(\sqrt{2\mu E_i^{(FV, n_s)}}, \frac{\pi}{\epsilon}\right)$ is a constant ($= - 1 / c(\pi / \epsilon)$) if evaluated at eigen energies of the Hamiltonian with contact interaction $c(\pi / \epsilon)$, we know that
\begin{equation}
	\tilde S^{(FV, n_s)}(E_i^{(FV, n_s)}, \epsilon)
	=
	- \frac{1}{c^{(n_s)}(\frac{\pi}{\epsilon})} - C^{(n_s)}(\epsilon)
	\,, \qquad
	C^{(n_s)}(\epsilon)
	=
	\mathcal R \left[(I^{(n_s)}\left(p=0, \frac{\pi}{\epsilon}\right)\right]
	\, .
\end{equation}
So in other words, if I choose
\begin{equation}
	\label{eq:Counterterm-vs-contact}
	- \frac{1}{c^{(n_s)}(\frac{\pi}{\epsilon})} - C^{(n_s)}(\epsilon) = - \frac{\mu}{2 \pi} \frac{1}{a_0} \, ,
\end{equation}
then everything works out.

More particularly, if I fit them in such a way, the dispersion Lüscher zeta function is constant in $\epsilon$ and in $n_s$ (for eigen energies) and thus I actually do not have to run the limits at all: I can simply convert my energy level in discrete finite space to the infinite volume continuum results.

Note that:
\begin{itemize}
	\item eq.~\ref{eq:Counterterm-vs-contact} is a prediction -- we can go back and check wether this relation is True!
	\item @Evan: Everything I said here translates to $p \neq 0$ (as long as not evaluated at poles).
		Thus you can see that non-truncated dispersion zeta (the above zeta evaluated at $p_i^{(FV, n_s)} = \sqrt{2 \mu E_i^{(FV, n_s)}}$) will cause an additional $f^{(n_s)}(p, \epsilon)$ dependence in the LHS of eq.~\ref{eq:Counterterm-vs-contact}.
		But since $c^{(n_s)}$ is $p$ independent, this will cause finite $p_\text{fit}^{(FV, n_s)}$ residue:
		$$
			- \frac{1}{c^{(n_s)}(\frac{\pi}{\epsilon})} - C^{(n_s)}(\epsilon)  - f^{(n_s)}(p_\text{fit}^{(FV, n_s)}, \epsilon)= - \frac{\mu}{2 \pi} \frac{1}{a_0} \, .
		$$
		Thus, evaluating for different energies will cause a shift in the dispersion zeta of kind
		$$
			\hat S^{(FV, n_s)}(E_i^{(FV, n_s)}, \epsilon)
			=
			\tilde S^{(FV, n_s)}(E_i^{(FV, n_s)}, \epsilon)
			+
			\Delta^{(FV, n_s)}(\epsilon, \text{fit}, i)
		$$
		with
		$$
			\Delta^{(FV, n_s)}(\epsilon, \text{fit}, i)
			=
			f^{(n_s)}(p_\text{fit}^{(FV, n_s)}, \epsilon)
			-
			f^{(n_s)}(p_i^{(FV, n_s)}, \epsilon)
		$$
		I don't see that this generally cancels out.
\end{itemize}

\end{document}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{bm}
\usepackage{amsfonts}

\usepackage{graphicx}
\graphicspath{ {/} }

\RequirePackage[dvipsnames]{xcolor}
\RequirePackage{amsfonts}

%%This parts handles the options passed to the package.

    %\DeclareOption{red}{\renewcommand{\wordcolour}{sharelatexcolour}}
    %\DeclareOption{blue}{\renewcommand{\wordcolour}{mybluecolour}}
    %\DeclareOption*{\PackageWarning{examplepackage}{Unknown ‘\CurrentOption’}}
    %\ProcessOptions\relax

%%Body of the package, most of the declarations appear here.

\newcommand{\comment}[1]{}%define a new command that effectively does nothing with the input

\newcommand{\bigO}[1]{\mathcal{O}(#1)} %define the bigO symbol

\newcommand{\questRef}[1]{\textcolor{ForestGreen}{{\textbf{($#1_?$)}}}} %creates a question reference
\newcommand{\titem}[1]{\item{}\textit{#1}} %creates a titled item, has an argument for title
\newcommand{\bitem}[1]{\item{}\textbf{#1}}

%number systems

\newcommand{\reals}{$\mathbb{R}$}
\newcommand{\rats}{$\mathbb{Q}$}
\newcommand{\ints}{$\mathbb{Z}$}
\newcommand{\naturals}{$\mathbb{N}$}

\newcommand{\defRef}[1]{\textcolor{Purple}{{\textbf{($#1_D$)}}}} %creates a question reference
%%Important words are added to the index and printed in different colour

%Need to add:%
% std def of DFA
% std def of a turing machine

\title{Solving Problem 47 with TMs}
\author{Matan Shtepel}
\date{August 13, 2021}

\newcommand{\K}{$\bm{K}$}
\newcommand{\aaa}{algorithm/automaton}

\begin{document}

\maketitle

\section{Relevant Definitions}
\begin{itemize}
    \titem{Problem 25:} "Axiomatically describe classes \K{} of algorithms or automata in which any algorithm/automaton has a totalling."
    \titem{Problem 26:} "Axiomatically describe classes \K{} of algorithms or automata in which any partial algorithm/automaton has an extension." Note this problem, as far as I can tell, is equivalent to problem 25, just stated from a different perspective.
    \titem{Definabillity Problem $R_d$:} "Find an algorithm/automaton $H$ that for an arbitrary algorithm/automaton $A$ from \K{} and an arbitrary element $x$ from $D^*(K)$, informs whether $A(x)$ is defined."
    \titem{Total definition:} An algo/automata A is called total $\iff$ $DD(A)=AD(A)$.
    \titem{Partial definition:} An algo/automata A is called partial $\iff$ $DD(A)\neq{}AD(A)$
    \titem{$AD$ vs $DD$:} $AD(A)$ is the acceptability domain, all the inputs $A$ can accept. $DD(A)$ are all the inputs for which $A$ gives an output. For example, if $A$ is a turing machine that runs forever on any input, $AD(A)$ contains any string of input symbols, yet, $DD(A)$ is the empty set.
\end{itemize}
I believe this is what the author (M. Burgin) intended, but for sake of being explicit, I will require that if an \aaa{} $T$ is a totaling for an \aaa{} $A$, $T$ has to have following properties:
\begin{enumerate}
    \item $A(x)=b$ $\rightarrow{}$ $T(x)=b$
    \item $A(x)=*$ (A is not defined on input x, i.e, $x\notin{}DD(A)$) $\rightarrow{}$ $T(x)=N$ where $N$ is a some element which symbolizes a null output and $N\notin{}C(A)$.
\end{enumerate}
This way, $T$ is useful for evaluating $A$ on some input $x$, allowing us to skip the infinite wait time in case $A(x)$ is not defined. Additionally, $N$ being a distinct element which $A$ never outputs allows us to discern between the valid outputs of $A$ and the outputs tacked on by $T$ to complete the mapping, also a valuable property.

\section{Proposed Reduction/Solution}
The following discussion addresses both question 25 and 26. For convenience, I will state most of my ideas in terms of problem of 25, but the claim and proof applies to both problems. \\

\noindent{}\textit{Claim:} Any automaton/algo in class \K{} has a totalling $\iff{}$ the definability problem is decidable in \K{}. Or, in simpler terms, any algorithm/automata $A$ may be totaled if and only if we can tell for what inputs it is not defined.\\

\noindent{}\textit{Proof:} \\
(If:) Assume that we can total arbitrary \aaa{} $A$ to \aaa{} $T$ as defined above. Then, we can construct $H$ which decides the definability problem as such: $H(c(A),x)=P_{sw}\cdot{}T(c(A),x)$ where $P_{sw}$ is the parallel composition of $A_{sw(N,0)}$ and $A_{sw(\lnot{}N, 1)}$. As can be seen, ($H(c(A), x)=1$ $\iff{}$ $A(x)=b\in{}C(A)$) $\wedge{}$ ($H(c(A), x)=0$ $\iff{}$ $A(x)=*$) . Hence, $H$ decides the definability problem.\\
(Only if:) Assume that $H'$ decides of the definability problem. We can construct $H$ that only corecognizes the decidability problem from $H'$ by making $H$ output nothing if $H'$ outputs a $1$ and output a $1$ if $H'$ outputs a $0$. Then, $H$ recognizes when \aaa{} $A$ in \K{} is not defined, that is, $H(c(A), x)=1$ $\iff{}$ $A(x)=*$. $T$, the totalling of $A$, will operate in the following way: $T(x)$ is a parallel composition of $A_{sw(1,N)}\cdot{}H$ and $A$. If the $H$ component gives a result first (which it must if it gives a result, since by def $A$ will never give a result) then $A(x)=*$ $\rightarrow{}$ $T(x)=N$ as desired. Meanwhile if $A$ gives a result, $b\in{}R(A)$, $H$ will never output and $T$ will output $b$ as desired.\\

Therefore, as long as the definabillity problem is decidable in \K{} any algorithm automata can be totalled and vice-versa. Note that the corecognizability of the definabillity problem is also a sufficient and necessary since corecognizable implies decidability (view next section). Lastly, note this nice property: for all classes of algorithms/automata \K{} in which all $A$ are already total, the definability problem is easily corecognizable by $H$, where $H$ is any \aaa{} s.t that $DD(H)=\emptyset{}$. It's decidable by $C_1$, the constant function $1$.

\section{Equivalency of Corecognisability and Decidability of $R_D$}
\textit{Claim:} $R_D$ is corecognisable in \K{}$\iff{}$ $R_D$ is decidable in \K{}.\\

\noindent{}\textit{Proof:}\\
(If:) Assume that $R_D$ is corecognisable in \K{}, that is, there exists $H\in{}\bm{K}$ s.t $H(c(A), x)=1$ $\iff{}$ $A(x)=*$. Construct $H'$ that decides $R_D$ as follows: $H'(c(A),x)$ is the parallal composition of $A_{sw(1,0)}\cdot{}H(c(A),x)$ and $C_1 \cdot{} U(c(A),x)$. Hence, $(H'(c(A),x)=1\iff{}A(x)=b\in{}R(A)$ $\wedge{}$ $(H'(c(A),x)=0\iff{}A(x)=*$.\\

Here's Edward Witten and Paul Dirac: \\

\includegraphics[width=4cm, height=4cm]{wittenDirac.jpg}\\

\noindent{}(Only If:) Assume that $R_D$ is decidable in \K{}, that is, there exists $H\in{}\bm{K}$ s.t $(H(c(A),x)=1\iff{}A(x)=b\in{}R(A)$ $\wedge{}$ $(H(c(A),x)=0\iff{}A(x)=*$. Construct $H'$ that corecognizes $R_D$ as follows: $H'=A_{sw(0,1)}\cdot{}H$ where $P_{sw(0,1)}$ switches a 0 by a 1 and is undefined for all other inputs. Hence, $H'(c(A), x)=1$ $\iff{}$ $A(x)=*$ and else $H'$ is undefined as desired.  \\

Ishaan's equation: $E=MC^2$ --- $\int_{\infty{}}^{b} x^2 \,dx$

\section{NOTING}

\end{document}


\section{Wang Mason}

\begin{frame}
\frametitle{Wang Mason Model \cite{wangmason}}
\begin{itemize}
    \item 2D Impact Model with Friction
    \item Inputs:
    \begin{itemize}
        \item Geometric and mass properties
        \item Pre-impact state
        \item Frictional coefficient ($\mu$)
        \item Coefficient of restitution ($\epsilon$)
    \end{itemize}
\end{itemize}
\begin{figure}
    \centering
    \includegraphics[scale=0.6]{figures/WangPlot.png}
    \caption{Two colliding rigid bodies in a plane\cite{wangmason}}
    \label{fig:WangPlot}
\end{figure}

\end{frame}

\subsection{Implementation}

\begin{frame}{Implementing the Wang Mason Model}
\begin{table}[h!]
    \centering
    \begin{tabular}{|c|c|c|}
        \hline
         & $\mu > |\mu_s|$ & $\mu < |\mu_s|$ \\
         \hline
         $P_d > (1+e)P_q$  & \multicolumn{2}{c|}{Sliding} \\
         \hline
         $P_q < P_d < (1+e)P_q$ & R-Sticking & R-Reversed Sliding \\
         \hline
         $P_d < P_q$ & C-Sticking & C-Reversed Sliding\\
         \hline
    \end{tabular}
    \caption{Analytical Method for Determining Contact Mode}
\end{table}

\begin{itemize}
    \item Each contact mode has their own unique two equations to solve for the impulses ($P_x$ and $P_y$)
    \item From this, we have a function that can take the pre-impact state and some properties from the system and predict a post-impact state
\end{itemize}

\end{frame}

\subsection{Optimizing Parameters ($\mu$ and $\epsilon$)}

\begin{frame}{Optimizing Parameters ($\mu$ and $\epsilon$)}
\begin{itemize}
    \item  Next, we optimized a grid-search over the two input parameters ($\mu$ and $\epsilon$) in an attempt to find the values which give out the most accurate post-impact state data.
    \item The optimization of the frictional coefficient $\mu$ leads to values which don't match traditional measured friction values between materials
    \item In this case,  maybe physically intuitive parameters are not ideal

    \item With this optimization we introduce a cost function \cite{nima2} (Normalized $\ell_2$ norm of velocity)
    \begin{align*}
        \mbox{error} = \frac{||v_{observed} - v_{predicted}||}{||v_{observed}||}
    \end{align*}

\end{itemize}

 %% Add our findings/plots from WANG
 % found an optimal mu of around 0.13 ?? and optimal epsilon of about 0.62 ??
 % minimal error was roughly 0.38 to 0.4
 % after a certain threshold for mu, the errors all became the same across the board, leading to vertical lines on the contour plot. This is to be expected, because at a certain mu value, you begin sticking and then the mu value becomes arbitrary??

\end{frame}

\begin{frame}{Optimal $\mu$ and $\epsilon$}
   \begin{itemize}
       \item Optimal $\mu$ : 0.1607 (compared to 0.057 \cite{nima1} and 0.120 \cite{nima2})
       \item Optimal $\epsilon$ : 0.6405 (compared to 0.612 \cite{nima1} and 0.537 \cite{nima2})
       \item \textit{NOTE: The optimal values were determined by taking the average of the best $\mu$ and $\epsilon$ pairings of 1000 random trials.}
   \end{itemize}

   \begin{figure}
       \centering
       \includegraphics[scale=0.37]{figures/Contour1000WM.jpg}
       \caption{Error contour plot for the Wang Model}
       \label{fig:ContourWang}
   \end{figure}

    %\begin{figure}
    %\centering
    %\begin{minipage}{.5\textwidth}
      %\centering
      %\includegraphics[width=1.1\linewidth]{figures/Contour1000WM.jpg}
    %   \captionof{figure}{Contour Plot of Errors}
      %\label{fig:contourWM}
    %\end{minipage}%
   % \begin{minipage}{.5\textwidth}
      %\centering
    %  \includegraphics[width=1.1\linewidth]{figures/HistError1000WM.jpg}
    %   \captionof{figure}{Histogram of Errors}
     % \label{fig:histWM}
    %\end{minipage}
    %\end{figure}

\end{frame}