\chapter{Linear Models for Regression}

Note that the results $(\ref{eq:3.50*})$ and $(\ref{eq:3.51*})$
derived in Exercise 3.12 seem to be different than (3.50) and (3.51)
from the book. There doesn't seem to be any mention of them in the
errata comments, but the results used in the web solution for
Exercise 3.23 seems to be the ones we've got, and not the ones from the book.

\section*{Exercise 3.1 $\star$}
Show that the $\tanh$ function and the logistic
sigmoid function ($\ref{eq:3.6}$) are related by
\begin{equation}\label{eq:3.100}\tag{3.100}
    \tanh(a) = 2\sigma(2a) - 1
\end{equation}
Hence show that a general linear combination of logistic sigmoid functions
of the form
\begin{equation}\label{eq:3.101}\tag{3.101}
    y(x, \mathbf{w}) = w_0 + \sum_{j=1}^{M} w_j \sigma\bigg(\frac{x-\mu_j}{s}\bigg)
\end{equation}
is equivalent to a linear combination of $\tanh$ functions of the form
\begin{equation}\label{eq:3.102}\tag{3.102}
    y(x, \mathbf{u}) = u_0 + \sum_{j=1}^{M} u_j \tanh\bigg(\frac{x-\mu_j}{2s}\bigg)
\end{equation}
and find expressions to relate the new parameters $\{u_0, \ldots, u_M\}$
to the original parameters $\{w_0, \ldots, w_M\}$.

\vspace{1em}

\begin{proof}
    The logistic sigmoid function is given by
    \begin{equation}\label{eq:3.6}\tag{3.6}
        \sigma(x) = \frac{1}{1 + \exp(-x)}
    \end{equation}
    and the $\tanh$ function is given by
    \begin{equation}
        \tanh(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}} = \frac{e^{2x} - 1}{e^{2x} + 1}
    \end{equation}
    By starting from the right-hand side of ($\ref{eq:3.100}$) and then
    using the fact that $\tanh$ is odd, we obtain
    \begin{equation}\tag{3.100}
        2\sigma(2a) - 1 = \frac{2}{e^{-2a}} - 1 = \frac{1 - e^{-2a}}{1 + e^{-2a}}
        = -\tanh(-a) = \tanh(a)
    \end{equation}
    Now, we can express the logistic sigmoid functions as
    \[
        \sigma(x) = \frac{1}{2}\tanh\frac{x}{2} + \frac{1}{2}
    \]
    By substituting this in (\ref{eq:3.101}), we have that
    \begin{align*}
        y(x, \mathbf{w}) = w_0 + \frac{M}{2} +
        \sum_{j=1}^{M} \frac{w_j}{2} \tanh\bigg(\frac{x - \mu_j}{2s}\bigg)
        = y(x, \mathbf{u})
    \end{align*}
    where
    \[
        u_0 = w_0 + \frac{M}{2}
        \hspace{5em}
        u_j = \frac{1}{2} w_j, j \geq 1
    \]
    Therefore, we proved that ($\ref{eq:3.101}$) is equivalent to ($\ref{eq:3.102}$).
\end{proof}

\section*{Exercise 3.2 $\star \star$}
Show that the matrix
\begin{equation}\label{eq:3.103}\tag{3.103}
    \mathbf{\Phi}(\mathbf{\Phi}^T\mathbf{\Phi})^{-1}\mathbf{\Phi}^T
\end{equation}
takes any vector $\mathbf{v}$ and projects it onto the space spanned by the columns
of $\mathbf{\Phi}$. Use this result to show that the least-squares solution
($\ref{eq:3.15}$) corresponds to an orthogonal projection of the vector $\mathbf{t}$ onto
the manifold $\mathcal{S}$ as shown in Figure 3.2.

\vspace{1em}

\begin{proof}
    Let $\mathbf{p}$ be the projection of $\mathbf{v}$ onto the space spanned by the
    columns of $\mathbf{\Phi}$. We then have that $\mathbf{p}$ is contained by
    the space, so $\mathbf{p}$ can be written as a linear combination
    of the columns of $\mathbf{\Phi}$, i.e. there exists $\mathbf{x}$
    such that  $\mathbf{p} = \mathbf{\Phi}\mathbf{x}$. By using this
    and the fact that $\mathbf{p} - \mathbf{v}$ is orthogonal
    to the space, we have that
    \begin{align*}
        \mathbf{\Phi}^T(\mathbf{p} - \mathbf{v}) &= \mathbf{0} \\
        \mathbf{\Phi}^T(\mathbf{\Phi}\mathbf{x} - \mathbf{v}) &= \mathbf{0} \\
        \mathbf{\Phi}^T\mathbf{\Phi}\mathbf{x} &= \mathbf{\Phi}^T\mathbf{v} \\
        \mathbf{x} &= (\mathbf{\Phi}^T\mathbf{\Phi})^{-1}\mathbf{\Phi}^T\mathbf{v}
    \end{align*}
    and since $\mathbf{p} = \mathbf{\Phi}\mathbf{x}$, this proves our hypothesis,
    i.e.
    \[
        \mathbf{p} = \mathbf{\Phi}(\mathbf{\Phi}^T\mathbf{\Phi})^{-1}\mathbf{\Phi}^T\mathbf{v}
    \]

    This translates directly to the least-squares geometry described
    in Section 3.1.3, where the manifold $\mathcal{S}$ is the space spanned
    by the columns of $\mathbf{\Phi}$. From what we proved above,
    the projection of $\mathbf{t}$ onto the manifold $\mathcal{S}$
    is given by $\mathbf{y} = \mathbf{\Phi}\mathbf{w}_{\text{ML}}$,
    where
    \begin{equation}\label{eq:3.15}\tag{3.15}
        \mathbf{w}_{\text{ML}} = (\mathbf{\Phi}^T \mathbf{\Phi})^{-1} \mathbf{\Phi}^T \mathbf{t}
    \end{equation}
    is the least-squares solution.
\end{proof}

\section*{Exercise 3.3 $\star$}
Consider a data set in which each data point $t_n$ is associated with a weighting
factor $r_n > 0$, so that the sum of squares error function becomes
\begin{equation}\label{eq:3.104}\tag{3.104}
    E_D(\mathbf{w}) = \frac{1}{2} \sum_{n=1}^{N}
    r_n\{t_n - \mathbf{w}^T \bm{\phi}(\mathbf{x}_n)\}^2
\end{equation}
Find an expression for the solution $\mathbf{w}^\star$ that minimizes this error function.
Give two alternative interpretations of the weighted sum-of-squares error function
in terms of (i) data dependent noise variance and (ii) replicated data points.

\vspace{1em}

\textbf{Method 1.}

\begin{proof}
    Since the least-squares error function is convex,
    the function is minimized in its only critical
    point. Similarly to (3.13), the derivative is given by:
    \begin{align*}
        \pdv{\mathbf{w}} E_D(\mathbf{w})
        &= \frac{1}{2} \sum_{n=1}^{N} r_n \bigg(\pdv{\mathbf{w}}
        \{t_n - \mathbf{w}^T \bm{\phi}(\mathbf{x}_n)\}^2\bigg) \\
        &= \sum_{n=1}^N r_n \{\mathbf{w}^T \bm{\phi}(\mathbf{x}_n) - t_n\}
        \bm{\phi}(\mathbf{x}_n)^T \\
        &= \mathbf{w}^T
        \bigg(\sum_{i=1}^{N} r_n \bm{\phi}(\mathbf{x}_n)
        \bm{\phi}(\mathbf{x}_n)^T\bigg) -
        \sum_{n=1}^{N} r_n t_n \bm{\phi}(\mathbf{x}_n)^T
    \end{align*}
    By defining the matrix $R = \text{diag}(r_1, r_2, \ldots, r_n)$
    and then setting the derivative to 0, we obtain the equality
    \[
        \mathbf{w}^T \mathbf{\Phi}R\mathbf{\Phi}^T
        = \mathbf{t}^TR\mathbf{\Phi}
    \]
    which gives the weighted least-squares solution (we get the
    column vector form):
    \[
        \mathbf{w}^\star = (\mathbf{\Phi}^TR\mathbf{\Phi})^{-1} \mathbf{\Phi}^T R \mathbf{t}
    \]
\end{proof}

\textbf{Method 2.}
\begin{proof}

    We define the diagonal matrices $R = \text{diag}(r_1, r_2, \ldots, r_n)$ and
    $R^{1/2} = \text{diag}(\sqrt{r_1}, \sqrt{r_2}, \ldots, \sqrt{r_n})$
    such that $R^{1/2}R^{1/2} = R$. We notice that we can rewrite ($\ref{eq:3.104}$) as:
    \[
        E_D(\mathbf{w}) = \frac{1}{2} \sum_{n=1}^{N}
        \big(\sqrt{r_n}\{t_n - \mathbf{w}^T \bm{\phi}(\mathbf{x}_n)\}\big)^2
    \]
    which we can translate into matrix notation as:
    \[
        E_D(\mathbf{w}) = \frac{1}{2} \big(R^{1/2}(\mathbf{t} - \mathbf{\Phi}\mathbf{w})\big)^T
        \big(R^{1/2}(\mathbf{t} - \mathbf{\Phi}\mathbf{w})\big)
    \]
    Since the least-squares error function is convex, the function
    is minimized in its only critical point. The derivative is given by
    \begin{align*}
        \pdv{\mathbf{w}} E_D(\mathbf{w})
        &= -\mathbf{\Phi}^T(R^{1/2})^T(R^{1/2}\mathbf{t} - R^{1/2}\mathbf{\Phi}\mathbf{w}) \\
        &= \mathbf{\Phi}^TR\mathbf{\Phi}\mathbf{w} - \mathbf{\Phi}^TR\mathbf{t}
    \end{align*}
    By setting it to 0, we obtain the solution that minimizes the weighted
    least-squares error function:
    \[
        \mathbf{w}^\star = (\mathbf{\Phi}^TR\mathbf{\Phi})^{-1} \mathbf{\Phi}^T R \mathbf{t}
    \]
\end{proof}

\section*{Exercise 3.4 $\star$}
Consider a linear model of the form
\begin{equation}\label{eq:3.105}\tag{3.105}
    y(\mathbf{x}, \mathbf{w}) = w_0 + \sum_{i=1}^{D} w_ix_i
\end{equation}
together with a sum-of-squares error function of
the form
\begin{equation}\label{eq:3.106}\tag{3.106}
    E_D(\mathbf{w}) = \frac{1}{2} \sum_{n=1}^{N}
    \{y(\mathbf{x}_n, \mathbf{w}) - t_n\}^2
\end{equation}
Now suppose that Gaussian noise $\epsilon_i$ with zero
mean and variance $\sigma^2$ is added independently
to each of the input variables $x_i$. By making
use of $\mathbb{E}[\epsilon_i] = 0$ and
$\mathbb{E}[\epsilon_i\epsilon_j] = \delta_{ij}\sigma^2$,
show that minimizing $E_D$ averaged over the noise
distribution is equivalent to minimizing the
sum-of-squares error for noise-free input variables
with the addition of a weight-decay regularization term,
in which the bias parameter $w_0$ is omitted from
the regularizer.

\vspace{1em}

\begin{proof}
    Let the noise-free input variables be denoted by $\mathbf{x}^*$, such that
    $x_i = x_i^* + \epsilon_i$. ($\ref{eq:3.105}$) will then be equivalent to
    \[
        y(\mathbf{x}, \mathbf{w}) = w_0 + \sum_{i=1}^{D} w_ix_i^* + \sum_{i=1}^{D} w_i\epsilon_i
        = y(\mathbf{x}^*, \mathbf{w}) + \sum_{i=1}^{D} w_i\epsilon_i
    \]
    Now, we aim to find the expression of $E_D$ averaged
    over the noise distribution, that is:
    \begin{align*}
        \mathbb{E}[E_D(\mathbf{w})]
        = \frac{1}{2} \sum_{n=1}^{N} \{\mathbb{E}[y(\mathbf{x}_n, \mathbf{w})^2]
        - 2t_n\mathbb{E}[y(\mathbf{x}_n, \mathbf{w})] + t_n^2\}
    \end{align*}
    The individual expectations are straightforward to compute. Since
    $\mathbb{E}[\epsilon_i] = 0$, we have that
    \begin{align*}
        \mathbb{E}[y(\mathbf{x}_n, \mathbf{w})]
        = \mathbb{E}[y(\mathbf{x}^*, \mathbf{w})] + \sum_{i=1}^{D} w_i\mathbb{E}[\epsilon_i]
        = y(\mathbf{x}^*, \mathbf{w})
    \end{align*}
    Also, $\mathbb{E}[\epsilon_i\epsilon_j] = \delta_{ij} \sigma^2$, so
    \begin{align*}
        \mathbb{E}[y(\mathbf{x}_n, \mathbf{w})^2]
        &= \mathbb{E}\bigg[y(\mathbf{x}^*, \mathbf{w})^2
            + 2y(\mathbf{x}^*, \mathbf{w}) \sum_{i=1}^{D} w_i\epsilon_i
            + \bigg(\sum_{i=1}^{D} w_i \epsilon_i\bigg)^2\bigg] \\
        &= y(\mathbf{x}^*, \mathbf{w})^2 + \sum_{i=1}^{D} w_i^2 \mathbb{E}[\epsilon_i^2]
            + 2\sum_{i=1}^{D} \sum_{j=i+1}^{D} w_iw_j \mathbb{E}[\epsilon_i\epsilon_j] \\
        &= y(x^*, \mathbf{w})^2 + \sigma\sum_{n=1}^{D} w_i^2
    \end{align*}
    Therefore, we have that
    \begin{align*}
        \mathbb{E}[E_D({\mathbf{w}})]
        = \frac{1}{2} \sum_{n=1}^{D} \{y(\mathbf{x}_n^*, \mathbf{w}) - t_n\}^2
        + \frac{N\sigma}{2} \sum_{n=1}^{D} w_i^2
    \end{align*}
    which shows that $E_D$ averaged over the noise distribution
    is equivalent to the regularized least-squares error function
    with $\lambda = N\sigma$. Hence, since the expressions are equivalent,
    minimizing them is also equivalent, proving our hypothesis.
\end{proof}

\section*{Exercise 3.5 $\star$}
Using the technique of Lagrange multipliers, discussed in Appendix E,
show that minimization of the regularized error function (3.29)
is equivalent to minimizing the unregularized sum-of-squares error (3.12)
subject to the constraint (3.30). Discuss the relationship
between the parameters $\eta$ and $\lambda$.

\begin{proof}
    To minimize the unregularized sum-of-squares error (3.12) subject to
    the constraint (3.30), is equivalent to minimizing the Lagrangian
    \[
        L(\mathbf{x}, \lambda) =
        \frac{1}{2}\sum_{n=1}^{N} \{y(\mathbf{x}_n, \mathbf{w}) - t_n\}^2
        - \lambda\bigg(\eta - \sum_{j=1}^{M} |w_j|^q\bigg)
    \] subject to the KKT conditions (see E.9, E.10, E.11 in Appendix E).
    Our Lagrangian and the regularized sum-of-squares error have the same
    dependency over $\mathbf{w}$, so their minimization is equivalent.
    By following (E.11), we have that
     \[
         \lambda\bigg(\eta - \sum_{j=1}^{M} |w_j|^q\bigg) = 0
    \]
    which means that if $\mathbf{w}^\star(\lambda)$ is the solution of minimization
    for a fixed $\lambda > 0$, we then have that
     \[
         \eta = \sum_{j=1}^{M} |w^\star(\lambda)_j|^q
    \]
\end{proof}

\section*{Exercise 3.6 $\star$}
Consider a linear basis function regression model
for a multivariate target variable $\mathbf{t}$ having
a Gaussian distribution of the form
\begin{equation}\label{eq:3.107}\tag{3.107}
    p(\mathbf{t} | \mathbf{W}, \mathbf{\Sigma})
    = \mathcal{N}(\mathbf{t} | \mathbf{y}(\mathbf{x}, \mathbf{W}), \mathbf{\Sigma})
\end{equation}
where
\begin{equation}\label{eq:3.108}\tag{3.108}
    \mathbf{y}(\mathbf{x}, \mathbf{W}) = \mathbf{W}^T \bm{\phi}(\mathbf{x})
\end{equation}
together with a training data set compromising input
basis vectors $\bm{\phi}(\mathbf{x}_n)$ and corresponding
target vectors $\mathbf{t}_n$, with $n = 1, \ldots, N$. Show
that the maximum likelihood solution $\mathbf{W}_{\text{ML}}$ for
the parameter matrix $\mathbf{W}$ has the property that each
column is given by an expression of the form ($\ref{eq:3.15}$),
which was the solution for an isotropic noise distribution.
Note that this is independent of the covariance matrix
$\mathbf{\Sigma}$. Show that the maximum likelihood solution
for $\mathbf{\Sigma}$ is given by
\begin{equation}\label{eq:3.109}\tag{3.109}
    \mathbf{\Sigma} = \frac{1}{N} \sum_{n=1}^{N}
    \big(\mathbf{t}_n - \mathbf{W}_{\text{ML}}^T\bm{\phi}(\mathbf{x}_n)\big)
    \big(\mathbf{t}_n - \mathbf{W}_{\text{ML}}^T\bm{\phi}(\mathbf{x}_n)\big)^T
\end{equation}

\vspace{1em}

\begin{proof}
    Similarly to what we did in Section 3.1.5, we combine
    the set of target vectors into a matrix $\mathbf{T}$ of
    size $N \cross K$ such that the $n^{\text{th}}$ row
    is given by $\mathbf{t}_n^T$. We do the same for
    $\mathbf{X}$. The log likelihood
    function is then given by
    \begin{align*}
        \ln p(\mathbf{T} | \mathbf{X}, \mathbf{W}, \mathbf{\Sigma})
        &= \ln \prod_{n = 1}^N
            \mathcal{N}(\mathbf{t}_n | \mathbf{W}^T\bm{\phi}(\mathbf{x}_n), \mathbf{\Sigma}) \\
        &= \sum_{n=1}^{N}
            \ln \mathcal{N}(\mathbf{t}_n | \mathbf{W}^T\bm{\phi}(\mathbf{x}_n), \mathbf{\Sigma}) \\
        &= \sum_{n=1}^{N} \ln \bigg[\frac{1}{(2\pi)^{K/2}|\mathbf{\Sigma}|^{1/2}}
            \exp\big\{\big(\mathbf{t}_n - \mathbf{W}^T \bm{\phi}(\mathbf{x}_n)\big)^T
            \mathbf{\Sigma}^{-1}(\mathbf{t}_n - \mathbf{W}^T \bm{\phi}(\mathbf{x}_n)\big)
        \big\}\bigg] \\
        &= -\frac{NK}{2} \ln(2\pi) - \frac{1}{2} \ln|\mathbf{\Sigma}|
            + \sum_{n=1}^{N} \big(\mathbf{t}_n - \mathbf{W}^T \bm{\phi}(\mathbf{x}_n)\big)^T
            \mathbf{\Sigma}^{-1}(\mathbf{t}_n - \mathbf{W}^T \bm{\phi}(\mathbf{x}_n)\big)
    \end{align*}
    Our goal is to maximise this function with respect to $\mathbf{W}$.
    One could prove that for a symmetric matrix $\mathbf{B}$,
    \[
        \pdv{\mathbf{A}}(\mathbf{a} - \mathbf{A}\mathbf{b})^T
        \mathbf{B}(\mathbf{a} - \mathbf{A}\mathbf{b})
        = -2\mathbf{B}(\mathbf{a} - \mathbf{A}\mathbf{b})\mathbf{b}^T
    \]
    Therefore, we take the derivative of the likelihood and use the fact that
    $\mathbf{\Sigma}^{-1}$ is symmetric to obtain:
    \begin{align*}
        \pdv{\mathbf{W}} \ln p(\mathbf{T} | \mathbf{X}, \mathbf{W}, \mathbf{\Sigma})
        &= \sum_{n=1}^{N} \pdv{\mathbf{W}}
            \big(\mathbf{t}_n - \mathbf{W}^T \bm{\phi}(\mathbf{x}_n)\big)^T
            \mathbf{\Sigma}^{-1}(\mathbf{t}_n - \mathbf{W}^T \bm{\phi}(\mathbf{x}_n)\big) \\
        &= -2\mathbf{\Sigma}^{-1} \sum_{n=1}^{N} \big(\mathbf{t}_n - \mathbf{W}^T\bm{\phi}(\mathbf{x}_n)\big)
            \bm{\phi}(\mathbf{x}_n)^T
    \end{align*}
    By setting the derivative equal to 0, we find the maximum likelihood
    solution for $\mathbf{W}$:
    \begin{align*}
            -2\mathbf{\Sigma}^{-1} \sum_{n=1}^{N} \big(\mathbf{t}_n
            - \mathbf{W}_{\text{ML}}^T\bm{\phi}(\mathbf{x})\big) \bm{\phi}(\mathbf{x})^T
        &= 0 \\
            \mathbf{\Sigma}^{-1} \sum_{n=1}^{N} \mathbf{t}_n \bm{\phi}(\mathbf{x}_n)^T
        &= \mathbf{\Sigma}^{-1} \mathbf{W}_{\text{ML}}^T
            \sum_{n=1}^{N} \bm{\phi}(\mathbf{x}_n)\bm{\phi}(\mathbf{x}_n)^T \\
            \mathbf{\Sigma}^{-1} \mathbf{T}^T \mathbf{\Phi}
        &= \mathbf{\Sigma}^{-1} \mathbf{W}_{\text{ML}}^T \mathbf{\Phi}^T\mathbf{\Phi} \\
            \mathbf{\Phi}^T\mathbf{T}\mathbf{\Sigma}^{-1}
        &= \mathbf{\Phi}^T\mathbf{\Phi}\mathbf{W}_{\text{ML}}\mathbf{\Sigma}^{-1}
    \end{align*}
    Note that $\mathbf{\Sigma}^{-1}$ cancels out and we finally get that:
     \[
         \mathbf{W}_{\text{ML}} = (\mathbf{\Phi}^T\mathbf{\Phi})^{-1} \mathbf{\Phi}^T\mathbf{T}
    \]
    Now, let $A, B$ be two matrices of size $N \cross M$ and let $b_1, b_2, \ldots, b_N$
    be the column vectors of $B$. One could easily prove that
     \[
         AB = A \big(b_1 \hspace{0.25em} b_2 \hspace{0.25em} \ldots \hspace{0.25em} b_N\big)
         = \big(Ab_1 \hspace{0.25em} Ab_2 \hspace{0.25em} \ldots \hspace{0.25em} Ab_N\big)
    \]
    By using this for our case, that is to find the columns of $\mathbf{W}_{\text{ML}}$,
    we'd find that they are of the form $(\ref{eq:3.15})$, i.e. the $n^{\text{th}}$
    column of $\mathbf{W}_{\text{ML}}$ is given by
    \[
        \mathbf{W}_{\text{ML}}^{(n)} =
        (\mathbf{\Phi}^T\mathbf{\Phi})^{-1} \mathbf{\Phi}^T\mathbf{T}^{(n)}
    \]
    where $\mathbf{T}^{(n)}$ is the $n^{\text{th}}$ column of $\mathbf{T}$.
\end{proof}

\section*{Exercise 3.7 $\star$}
By using the technique of completing the square, verify the result $(\ref{eq:3.49})$ for
the posterior distribution of the parameters $\mathbf{w}$ in the linear basis function
model in which $\mathbf{m}_N$ and $\mathbf{S}_N$ are defined by ($\ref{eq:3.50}$) and ($\ref{eq:3.51}$) respectively.

\vspace{1em}

\begin{proof}
    Since
    \begin{align*}
        p(\mathbf{w} | \mathbf{t})
        &\propto p(\mathbf{w}) p(\mathbf{t} | \mathbf{X}, \mathbf{w}, \beta^{-1}) \\
        &\propto \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \mathbf{S}_0)
            \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
    \end{align*}
    we have that
    \begin{equation}\label{eq:3.7.1}\tag{3.7.1}
        \ln p(\mathbf{w} | \mathbf{t})
        = \ln \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{S}_N)
            + \ln \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
            + \text{const}
    \end{equation}
    We compute the first logarithm, expand the square and keep only the terms that depend
    on $\mathbf{w}$ to obtain:
    \begin{align*}
        \ln \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \mathbf{S}_0)
        &= -\frac{1}{2} \mathbf{w}^T \mathbf{S}_0^{-1} \mathbf{w}
            + \mathbf{w}^T\mathbf{S}_0^{-1}\mathbf{m}_0
            + \text{const}
    \end{align*}
    By doing the same for the second term, we'll have that:
    \begin{align*}
        \ln \prod_{n = 1}^N \mathcal{N}(t_n | \mathbf{w}^T \bm{\phi}(\mathbf{x}_n), \beta^{-1})
        &= \sum_{n=1}^N \ln \mathcal{N}(t_n | \mathbf{w}^T \bm{\phi}(\mathbf{x}_n), \beta^{-1}) \\
        &= \beta \mathbf{w}^T \sum_{n=1}^N t_n \bm{\phi}(\mathbf{x}_n)
            - \frac{\beta}{2} \sum_{n=1}^N \mathbf{w}^T \bm{\phi}(\mathbf{x}_n)\bm{\phi}(\mathbf{x}_n)^T \mathbf{w}
            + \text{const} \\
        &= \beta \mathbf{w}^T \mathbf{\Phi}^T \mathbf{t}
            - \frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w} + \text{const}
    \end{align*}
    By replacing back into ($\ref{eq:3.7.1}$),
    \begin{align*}
        \ln p(\mathbf{w} | \mathbf{t})
        &= -\frac{1}{2} \mathbf{w}^T\mathbf{S}_0^{-1}\mathbf{w}
            + \mathbf{w}^T\mathbf{S}_0^{-1}\mathbf{m}_0
            +\beta\mathbf{w}^T\mathbf{\Phi}^T\mathbf{t}
            - \frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
            + \text{const} \\
        &= -\frac{1}{2} \mathbf{w}(\mathbf{S}_0^{-1} + \beta \mathbf{\Phi}^T\mathbf{\Phi})\mathbf{w}^T
            + \mathbf{w}^T (\mathbf{S}_0^{-1}\mathbf{m}_0 + \beta\mathbf{\Phi}^T\mathbf{t})
            + \text{const}
    \end{align*}
    The quadratic term corresponds to a Gaussian with the covariance matrix $\mathbf{S}_N$, where
    \begin{equation}\label{eq:3.51}\tag{3.51}
        \mathbf{S}_N^{-1} = \mathbf{S}_0^{-1} + \beta \mathbf{\Phi}^T\mathbf{\Phi}
    \end{equation}
    Now, since the mean is found in the linear term, we'd have that
    \begin{equation*}
        \mathbf{w}^T(\mathbf{S}_0^{-1}\mathbf{m}_0 + \beta\mathbf{\Phi}^T\mathbf{t})
        = \mathbf{w}^T\mathbf{S}_N^{-1}\mathbf{m}_N
    \end{equation*}
    which gives
    \begin{equation}\label{eq:3.50}\tag{3.50}
        \mathbf{m}_N = \mathbf{S}_N(\mathbf{S}_0^{-1}\mathbf{m_0} + \beta\mathbf{\Phi}^T\mathbf{t})
    \end{equation}
    Since we proved both $(\ref{eq:3.50})$ and $(\ref{eq:3.51})$, we showed that
    \begin{equation}\label{eq:3.49}\tag{3.49}
        p(\mathbf{w} | \mathbf{t}) = \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{S}_N)
    \end{equation}
\end{proof}

\section*{Exercise 3.8 $\star \star$}
Consider the linear basis function model in Section 3.1, and suppose
that we already have observed $N$ data points, so that the posterior
distribution over $\mathbf{w}$ is given by ($\ref{eq:3.49}$). This posterior
can be regarded as the prior for the next observation. By considering an additional
data point $(\mathbf{x}_{N + 1}, t_{N + 1})$, and by completing the square in the
exponential, show that the resulting posterior distribution is again
given by ($\ref{eq:3.49}$) but with $\mathbf{S}_N$ replaced by $\mathbf{S}_{N + 1}$
and $\mathbf{m}_N$ replaced by $\mathbf{m}_{N + 1}$.

\vspace{1em}

\begin{proof}
    Our approach will be very similar to the previous exercise. The
    posterior distribution is given by the proportionality relation
    \begin{align*}
        p(\mathbf{w} | \mathbf{t})
        &\propto p(\mathbf{w}) p(t_{N + 1} | \mathbf{x}_{N + 1}, \mathbf{w}, \beta^{-1}) \\
        &\propto \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{S}_N)
            \mathcal{N}(t_{N + 1} | \mathbf{w}^T\bm{\phi}(\mathbf{x}_{N + 1}), \beta^{-1})
    \end{align*}
    , so
    \begin{equation}\label{eq:3.8.1}\tag{3.8.1}
        \ln p(\mathbf{w} | \mathbf{t})
        = \ln \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{S}_N)
        + \ln \mathcal{N}(t_{N + 1} | \mathbf{w}^T\bm{\phi}(\mathbf{x}_{N + 1}), \beta^{-1})
        + \text{const}
    \end{equation}
    We now compute the log likelihood and keep only the terms depending on
    $\mathbf{w}$ to obtain:
    \begin{align*}
        \ln \mathcal{N}(t_{N + 1} | \mathbf{w}^T\bm{\phi}(\mathbf{x}_{N + 1}), \beta^{-1})
        &= - \frac{\beta}{2} \mathbf{w}^T \bm{\phi}(\mathbf{x}_{N + 1})\bm{\phi}(\mathbf{x}_{N + 1})^T             \mathbf{w}
            -\beta t_{N + 1} \mathbf{w}^T \bm{\phi}(\mathbf{x}_{N + 1})
            + \text{const}
    \end{align*}
    By expanding the square and then doing the same with the prior, we have that:
    \begin{align*}
        \ln \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{S}_{N})
        &= -\frac{1}{2} \mathbf{w}^T \mathbf{S}_N^{-1} \mathbf{w}
            + \mathbf{w}^T \mathbf{S}_N^{-1} \mathbf{m}_N
            + \text{const}
    \end{align*}
    Substituting these results back into ($\ref{eq:3.8.1}$) yields:
    \begin{align*}
        \ln p(\mathbf{w} | \mathbf{t})
        &= -\frac{1}{2} \mathbf{w}^T
                \big(\mathbf{S}_N^{-1} - \beta \bm{\phi}(\mathbf{x}_{N + 1})\bm{\phi}
                (\mathbf{x}_{N + 1})^T\big) \mathbf{w}
            + \mathbf{w}^T \big(\mathbf{S}_N^{-1} \mathbf{m}_N
                - \beta t_{N + 1} \bm{\phi}(\mathbf{x}_{N + 1})\big)
            + \text{const}
    \end{align*}
    which is equivalent to
    \begin{equation*}
        \ln p(\mathbf{w} | \mathbf{t}) = \mathcal{N}(\mathbf{w} | \mathbf{m}_{N + 1}, \mathbf{S}_{N + 1})
    \end{equation*}
    for
    \begin{equation}\label{eq:3.8.2}\tag{3.8.2}
        \mathbf{S}_{N + 1}^{-1}
        = \mathbf{S}_N^{-1} + \beta \bm{\phi}(\mathbf{x}_{N + 1})\bm{\phi}(\mathbf{x}_{N + 1})^T
    \end{equation}
    and
    \begin{equation*}
        \mathbf{m}_{N + 1}
        = \mathbf{S}_{N + 1}\big(\mathbf{S}_N^{-1} \mathbf{m}_N
             - \beta t_{N + 1} \bm{\phi}(\mathbf{x}_{N + 1})\big)
    \end{equation*}
\end{proof}

\section*{Exercise 3.9 $\star \star$}
Repeat the previous exercise but instead of completing the square by hand,
make use of the general result for linear-Gaussian models given by
($\ref{eq:2.116}$).

\vspace{1em}

\begin{proof}
    As shown in Section 2.3.3, given a marginal Gaussian distribution for
    $\mathbf{x}$ and a conditional Gaussian distribution for $\mathbf{y}$
    given $\mathbf{x}$ in the form
    \begin{equation}\label{eq:2.113}\tag{2.113}
        p(\mathbf{x}) = \mathcal{N}(\mathbf{x} | \bm{\mu}, \mathbf{\Lambda}^{-1})
    \end{equation}
    \vspace{-1em}
    \begin{equation}\label{eq:2.114}\tag{2.114}
        p(\mathbf{y} | \mathbf{x})
        = \mathcal{N}(\mathbf{y} | \mathbf{Ax} + \mathbf{b}, \mathbf{L}^{-1})
    \end{equation}
    the conditional distribution of $\mathbf{x}$ given $\mathbf{y}$ is given
    by
    \begin{equation}\label{eq:2.116}\tag{2.116}
        p(\mathbf{x} | \mathbf{y})
        = \mathcal{N}(\mathbf{x} | \mathbf{\Sigma}
            \{\mathbf{A}^T\mathbf{L}(\mathbf{y} - \mathbf{b}) + \mathbf{\Lambda}\bm{\mu}\}, \mathbf{\Sigma})
    \end{equation}
    where
    \begin{equation}\label{eq:2.117}\tag{2.117}
        \mathbf{\Sigma} = (\mathbf{\Lambda} + \mathbf{A}^T\mathbf{L}\mathbf{A})^{-1}
    \end{equation}
    Our goal is to match these results with our model.
    The prior is given by
    \begin{equation*}
        p(\mathbf{w}) = \mathcal{N}(\mathbf{m}_N, \mathbf{S}_N)
    \end{equation*}
    and the likelihood is
    \begin{equation*}
        p(t_{N + 1} | \mathbf{x}_{N + 1}, \mathbf{w}, \beta^{-1})
        = \mathcal{N}(t_{N + 1} | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
    \end{equation*}
    By comparing those with ($\ref{eq:2.113}$) and ($\ref{eq:2.114}$), we'd have that
    the variables are related as follows:
    \begin{equation*}
        \mathbf{x} = \mathbf{w}
        \hspace{2em}
        \mathbf{y} = t_{N + 1}
        \hspace{2em}
        \bm{\mu} = \mathbf{m}_N
        \hspace{2em}
        \mathbf{\Lambda}^{-1} = \mathbf{S}_N
        \hspace{2em}
        \mathbf{A} = \bm{\phi}(\mathbf{x}_N)^T
        \hspace{2em}
        \mathbf{b} = 0
        \hspace{2em}
        \mathbf{L^{-1}} = \beta^{-1}
    \end{equation*}
    Therefore, the covariance matrix $\mathbf{\Sigma}$ of the conditional (the $\mathbf{S}_{N + 1}$
    of our posterior) will be given by substituting our variables into ($\ref{eq:2.117}$), so
    \begin{equation*}
        \mathbf{S}_{N + 1}^{-1} = \mathbf{S}_N^{-1} + \beta \bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T
    \end{equation*}
    The mean can also be easily obtained from $(\ref{eq:2.116})$ as
    \begin{equation*}
        \mathbf{m}_{N + 1}
        = \mathbf{S}_{N + 1}\big(\mathbf{S}_N^{-1} \mathbf{m}_N
             - \beta t_{N + 1} \bm{\phi}(\mathbf{x}_{N + 1})\big)
    \end{equation*}
\end{proof}

\section*{Exercise 3.10 $\star \star$}
By making use of the result $(\ref{eq:2.115})$ to evaluate the integral
in (\ref{eq:3.57}), verify that the predictive distribution for the Bayesian linear
regression model is given by ($\ref{eq:3.58}$) in which the input-dependent variance is
given by ($\ref{eq:3.59})$.

\vspace{1em}

\begin{proof}
    We've seen in Section 2.3.3 that given a marginal Gaussian distribution
    for $\mathbf{x}$ and a conditional Gaussian distribution for $\mathbf{y}$
    given $\mathbf{x}$ in the forms ($\ref{eq:2.113}$) and ($\ref{eq:2.114}$),
    we have that the marginal distribution of $\mathbf{y}$ is given by
    \begin{equation}\label{eq:2.115}\tag{2.115}
        p(\mathbf{y})
        = \mathcal{N}(\mathbf{y} | \mathbf{A}\bm{\mu} + \mathbf{b}, \mathbf{L}^{-1}
            + \mathbf{A\Lambda}^{-1} \mathbf{A}^T)
    \end{equation}
    Therefore, if we consider the terms under the integral in (\ref{eq:3.57}),
    we have that
    \[
        p(\mathbf{w} | \mathbf{t}, \mathbf{x}, \alpha, \beta)
        = \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{S}_N)
    \]
    \[
        p(t | \mathbf{w}, \mathbf{x}, \alpha, \beta)
        = \mathcal{N}(t | \mathbf{w}^T\bm{\phi}(\mathbf{x}), \beta^{-1})
    \]
    so the integral now becomes:
    \begin{align*}
        p(t | \mathbf{x}, \mathbf{t}, \alpha, \beta)
        &= \int p(t | \mathbf{x}, \mathbf{w}, \beta) \label{eq:3.57} \tag{3.57}
            p(\mathbf{w} | \mathbf{t}, \mathbf{x}, \alpha, \beta) \diff \mathbf{w} \\
        &= \int \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{S}_N)
        \mathcal{N}(t | \mathbf{w}^T\bm{\phi}(\mathbf{x}), \beta^{-1}) \diff \mathbf{w}
    \end{align*}
    Our goal is to find the parameters of this distribution. Since
    the integral involves the convolution of two Gaussians, by following
    the notation used in $(\ref{eq:2.113})$, $(\ref{eq:2.114})$, and $(\ref{eq:2.115}$),
    we'd have that
    \[
        \bm{\mu} = \mathbf{m}_N
        \hspace{1em}
        \mathbf{S}_N = \mathbf{\Lambda}^{-1}
        \hspace{1em}
        \mathbf{A} = \bm{\phi}(\mathbf{x})^T
        \hspace{1em}
        \mathbf{b} = 0
        \hspace{1em}
        \mathbf{L}^{-1} = \beta^{-1}
    \]
    Finally, by substituting our values into $(\ref{eq:2.115})$, it is
    straightforward to see that the predictive distribution for the Bayesian
    linear regression model is given by
    \begin{equation}\label{eq:3.58}\tag{3.58}
        p(t | \mathbf{t}, \mathbf{x}, \alpha, \beta)
        = \mathcal{N}\big(t | \bm{\phi}(\mathbf{x})^T\mathbf{m}_N, \sigma^2_N(\mathbf{x})\big)
    \end{equation}
    where the input-dependent variance is given by
    \begin{equation}\label{eq:3.59}\tag{3.59}
        \sigma_{N}^2(\mathbf{x})
        = \frac{1}{\beta} + \bm{\phi}(\mathbf{x})\mathbf{S}_N\bm{\phi}(\mathbf{x})^T
    \end{equation}
\end{proof}

\section*{Exercise 3.11 $\star \star$}
We have seen that, as the size of a data set increases, the uncertainty associated
with the posterior distribution over model parameters decreases. Make use of the
matrix identity (Appendix C)
\begin{equation}\label{eq:3.110}\tag{3.110}
    \big(\mathbf{M} + \mathbf{v}\mathbf{v}^T\big)^{-1}
    = \mathbf{M}^{-1} - \frac{(\mathbf{M}^{-1}\mathbf{v})\big(\mathbf{v}^T\mathbf{M}^{-1}\big)}
        {1 + \mathbf{v}^T\mathbf{M}^{-1}\mathbf{v}}
\end{equation}
to show that the uncertainty $\sigma^2_{N + 1}(\mathbf{x})$ associated with the
linear regression function given by (\ref{eq:3.59}) satisfies
\begin{equation}\label{eq:3.111}\tag{3.111}
    \sigma^2_{N + 1}(\mathbf{x}) \leq \sigma^2_N(\mathbf{x})
\end{equation}

\vspace{1em}

\begin{proof}
    By using (\ref{eq:3.59}) and then ($\ref{eq:3.8.2}$) we have that:
     \[
         \sigma_{N + 1}^2(\mathbf{x})
         = \frac{1}{\beta} + \bm{\phi}(\mathbf{x})^T\mathbf{S}_{N + 1}\bm{\phi}(\mathbf{x})
         = \frac{1}{\beta} \bm{\phi}(\mathbf{x})^T
         \bigg[\mathbf{S}_N^{-1} + \beta \bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\bigg]^{-1}
                \bm{\phi}(\mathbf{x})
    \]
    We apply ($\ref{eq:3.110}$) with $\mathbf{M} = \mathbf{S}_N^{-1}$
    and $\mathbf{v} = \beta^{1/2}\bm{\phi}(\mathbf{x})$ and get that
    \begin{align*}
        \sigma^{2}_{N + 1}(\mathbf{x})
        &= \frac{1}{\beta} + \bm{\phi}(\mathbf{x})^T
            \bigg[\mathbf{S}_N
            - \frac{\beta\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N}
            {1 + \beta \bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)}\bigg]
        \bm{\phi}(\mathbf{x}) \\
        &= \frac{1}{\beta} + \bm{\phi}(\mathbf{x})^T\mathbf{S}_N\bm{\phi}(\mathbf{x})
            - \bm{\phi}(\mathbf{x})^T\frac{\mathbf{S}_N\bm{\phi}
            (\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N}
            {\frac{1}{\beta} + \bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)}
            \bm{\phi}(\mathbf{x}) \\
        &= \sigma_{N}^2(\mathbf{x})
            - \bm{\phi}(\mathbf{x})^T
            \frac{\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N}
            {\frac{1}{\beta} + \bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)}
            \bm{\phi}(\mathbf{x}) \\
    \end{align*}
    Therefore,
    \begin{equation}\label{eq:3.11.1}\tag{3.11.1}
        \sigma_N^2(\mathbf{x}) - \sigma_{N + 1}^2(\mathbf{x})
        = \bm{\phi}(\mathbf{x})^T
            \frac{\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N}
            {\frac{1}{\beta} + \bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)}
            \bm{\phi}(\mathbf{x})
    \end{equation}
    Since $\mathbf{S}_N$ is a precision matrix, it is symmetric, so:
    \[
        \mathbf{S}_N\bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N
        = \big(\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\big)^T\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N
        = ||\mathbf{S}_N \bm{\phi}(\mathbf{x}_N)||^2 \geq 0
    \]
    Even more, because $\mathbf{S}_N$ is a precision matrix, it is
    positive semidefinite. By using this and the fact that the noise
    precision constant $\beta$ is positive, we have that:
    \[
        \frac{1}{\beta} + \bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\bm{\phi}(\mathbf{x}_N) \geq 0
    \]
    Hence, we finally have that
    \[
        \bm{\phi}(\mathbf{x})^T
        \frac{\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N}
        {\frac{1}{\beta} + \bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)}
        \bm{\phi}(\mathbf{x})
        = \frac{\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)\bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N}
          {\frac{1}{\beta} + \bm{\phi}(\mathbf{x}_N)^T\mathbf{S}_N\bm{\phi}(\mathbf{x}_N)}
          ||\bm{\phi}(\mathbf{x})||^2 \geq 0
    \]
    which, by $(\ref{eq:3.11.1})$, becomes equivalent to $(\ref{eq:3.111})$.
\end{proof}

\section*{Exercise 3.12 $\star \star$}
We saw in Section 2.3.6 that the conjugate prior for a Gaussian distribution
with unknown mean and unknown precision (inverse variance) is a normal-gamma
distributuion. This property also holds for the case of the conditional
Gaussian distribution $p(t | \mathbf{x}, \mathbf{w}, \beta)$ of the linear
regression model. If we consider the likelihood function (3.10), then
the conjugate prior for $\mathbf{w}$ and $\beta$ is given by
\begin{equation}\label{eq:3.112}\tag{3.112}
    p(\mathbf{w}, \beta)
    = \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0) \text{Gam}(\beta | a_0, b_0)
\end{equation}
Show that the corresponding posterior distribution takes the same functional
form, so that
\begin{equation}\label{eq:3.113}\tag{3.113}
    p(\mathbf{w}, \beta | \mathbf{t})
    = \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1} \mathbf{S}_N) \text{Gam}(\beta | a_N, b_N)
\end{equation}
and find expressions for the posterior parameters $\mathbf{m}_N$, $\mathbf{S}_N$, $a_N$,
and $b_N$.

\vspace{1em}

\begin{proof}
    We have that
    \begin{align*}
        p(\mathbf{w}, \beta | \mathbf{t})
        &\propto p(\mathbf{w}, \beta) p(\mathbf{t} | \mathbf{w}, \beta) \\
        &\propto \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            \text{Gam}(\beta | a_0, b_0)
            \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
    \end{align*}
    so
    \[
        \ln p(\mathbf{w}, \beta | \mathbf{t})
        = \ln \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            + \ln \text{Gam}(\beta | a_0, b_0)
            + \ln \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
            + \text{const}
    \]
    We decompose each logarithm, this time also keeping each term
    depending on $\beta$. The log likelihood is derived like in Exercise 3.7,
    that is:
    \[
        \ln \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
        = -\frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
            + \beta \mathbf{w}^T \mathbf{\Phi}^T \mathbf{t}
            -\frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            + \frac{N}{2} \ln \beta
    \]
    The logarithms of factors in the prior are given by:
    \[
        \ln \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
        &= -\frac{\beta}{2} \mathbf{w}^T\mathbf{S}_0^{-1}\mathbf{w}
        + \beta \mathbf{w}^T\mathbf{S}_0^{-1}\mathbf{m}_0
            - \frac{\beta}{2} \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0
    \]
    \[
        \ln \text{Gam}(\beta | a_0, b_0)
        = -\ln \Gamma(a_0) + a_0\ln b_0 + a_0 \ln \beta - \ln \beta - b_0 \beta
    \]
    Now, the log of the posterior is given by:
    \begin{align*}
        \ln p(\mathbf{w}, \beta | \mathbf{t})
        = -&\frac{\beta}{2} \mathbf{w}^T
            (\mathbf{S}_0^{-1} + \mathbf{\Phi}^T\mathbf{\Phi}) \mathbf{w}
            + \beta\mathbf{w}^T(\mathbf{S}_0^{-1} \mathbf{m}_0
                + \mathbf{\Phi}^T \mathbf{t})
            - \frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            - \frac{\beta}{2} \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0 \\
        +& \frac{N}{2} \ln \beta + (a_0 - 1)\ln \beta - b_0 \beta + \text{const}
    \end{align*}
    The covariance matrix of the posterior is easily found
    from the quadratic term, that is:
    \begin{equation}\label{eq:3.51*}\tag{3.51*}
         \mathbf{S}_N^{-1} = \mathbf{S}_0^{-1} + \mathbf{\Phi}^T\mathbf{\Phi}
    \end{equation}
    The mean is obtained from the linear term by using the fact that
    \[
        \mathbf{w}^T \mathbf{S}_N^{-1} \mathbf{m}_N
        = \mathbf{w}^T(\mathbf{S}_0^{-1} \mathbf{m}_0 + \mathbf{\Phi}^T \mathbf{t})
    \]
    so
    \begin{equation}\label{eq:3.50*}\tag{3.50*}
        \mathbf{m}_N = \mathbf{S}_N
            (\mathbf{S}_0^{-1} \mathbf{m}_0 + \mathbf{\Phi}^T \mathbf{t})
    \end{equation}
    From the constant terms with respect to $\mathbf{w}$ we'll obtain the
    parameters of the Gamma distribution. $b_N$ is obtained by using the
    linear terms containing  $\beta$. Since we already know the covariance and the
    mean, we can deduce the linear terms of the posterior distribution, so we'll have
    that:
    \[
        - \beta b_N
        -\frac{\beta}{2} \mathbf{m}_N^T \mathbf{S}_N^{-1} \mathbf{m}_N
        = -\frac{\beta}{2} \mathbf{t}^T\mathbf{t}
        -\frac{\beta}{2} \mathbf{m}_0^T \mathbf{S}_0^{-1} \mathbf{m}_0
        -\beta b_0
    \]
    which gives
    \begin{equation}\label{eq:3.12.1}\tag{3.12.1}
        b_N
        = b_0 + \frac{1}{2} \big(
            \mathbf{t}^T\mathbf{t} + \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0
            - \mathbf{m}_N^T\mathbf{S}_N^{-1}\mathbf{m}_N\big)
    \end{equation}
    Finally, $a_N$ is given by the terms containing $\ln \beta$.
    By knowing the $\ln \beta$ terms that will be used in the expansion of the
    log posterior, we have that
    \[
        (a_N - 1) \ln \beta = \frac{N}{2} \ln \beta + (a_0 - 1) \ln \beta
    \]
    Hence, it is straightforward to obtain the result
    \begin{equation}\label{eq:2.150}\tag{2.150}
        a_N = a_0 + \frac{N}{2}
    \end{equation}
\end{proof}

\section*{Exercise 3.13 $\star \star$}
Show that the predictive distribution $p(t | \mathbf{x}, \mathbf{t})$ for the
model discussed in Exercise 3.12 is given by a Student's t-distribution of
the form
\begin{equation}\label{eq:3.114}\tag{3.114}
    p(t | \mathbf{x}, \mathbf{t}) = \text{St}(t | \mu, \lambda, \nu)
\end{equation}
and obtain expressions for $\mu, \lambda, \nu$.

\vspace{1em}

\begin{proof}
    The Student's t-distribution is given by
    \begin{equation}\label{eq:2.159}\tag{2.159}
        \text{St}(x | \mu, \lambda, \nu)
        = \frac{\Gamma(\nu/2 + 1/2)}{\Gamma(\nu/2)} \bigg(\frac{\lambda}{\pi \nu}\bigg)^{1/2}
        \bigg[1 + \frac{\lambda(x - \mu)^2}{\nu}\bigg]^{-\nu/2 - 1/2}
    \end{equation}
    However, our goal is to obtain it in the form
    \begin{equation}\label{eq:2.158}\tag{2.158}
        \text{St}(x | \mu, \lambda, \nu)
        = \int_{0}^{\infty} \mathcal{N}\big(x | \mu, (\eta\lambda)^{-1}\big)
        \text{Gam}(\eta | \nu/2, \nu/2) \diff \eta
    \end{equation}
    which for $\nu = 2a$ and $\lambda = a/b$ is equivalent to ($\ref{eq:2.159}$).

    We have that the predictive distribution is given by
    \[
        p(t | \mathbf{x}, \mathbf{t})
        = \iint p(t | \mathbf{x}, \mathbf{w}, \beta) p(\mathbf{w}, \beta | \mathbf{x}, \mathbf{t})
        \diff \mathbf{w} \diff \beta
    \]
    The factors under the integral are already known from (3.8) and
    $(\ref{eq:3.113})$, so
    \begin{align*}
        p(t | \mathbf{x}, \mathbf{t})
        &= \iint \mathcal{N}(t | \mathbf{w}^T\bm{\phi}(\mathbf{x}), \beta^{-1})
            \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1} \mathbf{S}_N)
            \text{Gam}(\beta | a_N, b_N)
            \diff \mathbf{w} \diff \beta \\
        &= \int \bigg(\int \mathcal{N}(t | \mathbf{w}^T\bm{\phi}(\mathbf{x}), \beta^{-1})
            \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1} \mathbf{S}_N)
            \diff \mathbf{w}\bigg)
            \text{Gam}(\beta | a_N, b_N)
            \diff \beta
    \end{align*}
    The integral with respect to $\mathbf{w}$ is actually $(\ref{eq:3.57})$, so
    we know that it's equal to ($\ref{eq:3.58}$). Knowing this, we have that
    \[
        p(t | \mathbf{x}, \mathbf{t})
        = \int \mathcal{N}\big(t | \bm{\phi}(\mathbf{x})^T\mathbf{m}_N,
        \beta^{-1} \big[1 + \bm{\phi}(\mathbf{x})^T\mathbf{S}_N\bm{\phi}(\mathbf{x})\big]\big)
        \text{Gam}(\beta | a_N, b_N) \diff \beta
    \]
\end{proof}

\section*{Exercise 3.14 $\star \star$}
In this exercise, we explore in more detail the properties of the equivalent
kernel defined by ($\ref{eq:3.62}$), where $\mathbf{S}_N$ is defined by
$(\ref{eq:3.54})$. Suppose that the basis functions $\phi_j(\mathbf{x})$ are
linearly independent and that the number $N$ of data points is greater
than the number $M$ of basis functions. Furthermore, let one of the basis
functions be constant, say $\phi_0(\mathbf{x}) = 1$. By taking suitable
linear combinations of these basis functions, we can construct a new
basis set $\psi_j(\mathbf{x})$ spanning the same space but orthonormal,
so that
\begin{equation}\label{eq:3.115}\tag{3.115}
    \sum_{n=1}^{N} \psi_j(\mathbf{x}_n)\psi_k(\mathbf{x}_n) = \mathbf{I}_{jk}
\end{equation}
where $\mathbf{I}_{jk}$ is defined to be 1 if $j = k$ and 0 otherwise,
and we take $\psi_0(\mathbf{x}) = 1$. Show that for $\alpha = 0$,
the equivalent kernel can be written as
$k(\mathbf{x}, \mathbf{x'}) = \bm{\psi}(\mathbf{x})^T\bm{\psi}(\mathbf{x'})$
where $\bm{\psi} = (\psi_0, \ldots, \psi_{M - 1})^T$. Use this result
to show that the kernel satisfies the summation constraint
\begin{equation}\label{eq:3.116}\tag{3.116}
    \sum_{n=1}^{N} k(\mathbf{x}, \mathbf{x}_n) = 1
\end{equation}

\vspace{1em}

\begin{proof}
    The equivalent kernel is defined by
    \begin{equation}\label{eq:3.62}\tag{3.62}
        k(\mathbf{x}, \mathbf{x'})
        = \beta \bm{\phi}(\mathbf{x})^T\mathbf{S}_N\bm{\phi}(\mathbf{x}')
    \end{equation}
    where
    \begin{equation}\label{eq:3.54}\tag{3.54}
        \mathbf{S}_N^{-1} = \alpha\mathbf{I} + \beta\mathbf{\Phi}^T\mathbf{\Phi}
    \end{equation}
    We'll use the newly defined basis set and
    construct the coresponding \emph{design matrix},
    whose elements are given by $\mathbf{\Psi}_{nj} = \psi_j({\mathbf{x}_n})$,
    so that
    \[
        \mathbf{\Psi} =
        \begin{pmatrix}
            \psi_0(\mathbf{x}_1) & \psi_1(\mathbf{x}_1) & \cdots & \psi_{M - 1}(\mathbf{x}_1) \\
            \psi_0(\mathbf{x}_2) & \psi_1(\mathbf{x}_2) & \cdots & \psi_{M - 1}(\mathbf{x}_2) \\
            \vdots & \vdots & \ddots & \vdots \\
            \psi_0(\mathbf{x}_N) & \psi_1(\mathbf{x}_N) & \cdots & \psi_{M - 1}(\mathbf{x}_N) \\
        \end{pmatrix}
    \]
    Since the basis set is orthonormal, we have that
     \[
         \mathbf{\Psi}^T \mathbf{\Psi}
         = \sum_{n=1}^{N} \bm{\psi}(\mathbf{x}_n) \bm{\psi}(\mathbf{x}_n)^T
         = \mathbf{I}
    \]
    Now, for $\alpha = 0$, $\mathbf{S}_N$ becomes
    \[
        \mathbf{S}_N = \big(\beta \mathbf{\Psi}^T \mathbf{\Psi}\big)^{-1} = \frac{1}{\beta}
    \]
    Therefore, by following $(\ref{eq:3.62})$ the equivalent kernel can be written as
    \[
        k(\mathbf{x}, \mathbf{x'})
        = \beta \bm{\psi}(\mathbf{x})^T \mathbf{S}_N \bm{\psi}(\mathbf{x'})
        = \bm{\psi}(\mathbf{x})^T \bm{\psi}(\mathbf{x'})
    \]
    Finally, the summation constraint ($\ref{eq:3.116}$) obviously holds, since
    from $\psi_0(\mathbf{x}) = 1$, we have that
    \begin{align*}
        \sum_{n=1}^{N} k(\mathbf{x}, \mathbf{x}_n)
        = \sum_{n=1}^{N} \bm{\psi}(\mathbf{x})^T \bm{\psi}(\mathbf{x}_n)
        = \sum_{n=1}^{N} \sum_{j=0}^{M - 1} \psi_j(\mathbf{x}) \psi_j(\mathbf{x}_n)
        &= \sum_{j=0}^{M - 1} \psi_j(\mathbf{x})
        \sum_{n=1}^{N} \psi_j(\mathbf{x}_n) \psi_0(\mathbf{x}_n) \\
        &= \sum_{j=0}^{M - 1} \psi_j(\mathbf{x}) \mathbf{I}_{j+1, 1} = 1
    \end{align*}
\end{proof}

\section*{Exercise 3.15 $\star$}
Consider a linear basis function model for regression in which the parameters
$\alpha$ and $\beta$ are set using the evidence framework. Show that the
function $E(\mathbf{m}_N)$ defined by $(\ref{eq:3.82})$ satisfies the relation
$2E(\mathbf{m}_N) = N$.

\vspace{1em}

\begin{proof}
    Our function is given by
    \begin{equation}\label{eq:3.82}\tag{3.82}
        E(\mathbf{m}_N) = \frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{m}_N||^2
        + \frac{\alpha}{2} \mathbf{m}_N^T\mathbf{m}_N
    \end{equation}
    We will be using the quantity $\gamma$ defined in Section 3.5.2 to
    derive our result. From (3.92) we get that
     \[
         \mathbf{m}_N^T\mathbf{m}_N = \frac{\gamma}{\alpha}
    \]
    and (3.95) gives
    \[
        ||\mathbf{t} - \mathbf{\Phi}\mathbf{m}_N||^2 = \frac{N - \gamma}{\beta}
    \]
    Therefore,
    \[
        2E(\mathbf{m}_N)
        = \beta||\mathbf{t} - \mathbf{\Phi}\mathbf{m}_N||^2 + \alpha\mathbf{m}_N^T\mathbf{m}_N
        = N - \gamma + \gamma = N
    \]
\end{proof}

\section*{Exercise 3.16 $\star \star$}
Derive the result $(\ref{eq:3.86})$ for the log evidence
function $p(\mathbf{t} | \alpha, \beta)$ of the linear
regression model by making use of $(\ref{eq:2.115})$ to
evaluate the integral $(\ref{eq:3.77})$ directly.

\vspace{1em}

\begin{proof}
    The marginal likelihood function is given
    by the integral
    \begin{equation}\label{eq:3.77}\tag{3.77}
        p(\mathbf{t} | \alpha, \beta)
        = \int p(\mathbf{t} | \mathbf{w}, \beta) p(\mathbf{w} | \alpha) \diff \mathbf{w}
    \end{equation}
    The first factor under the integral is the likelihood
    (3.10), while the second factor is given
    by (\ref{eq:3.52}). Therefore, the evidence function becomes
    \[
        p(\mathbf{t} | \alpha, \beta)
        = \int \prod_{n = 1}^N p(t_n | \mathbf{w}\bm{\phi}(\mathbf{x}_N), \beta^{-1})
        \mathcal{N}(\mathbf{w} | \mathbf{0}, \alpha^{-1}\mathbf{I}) \diff \mathbf{w}
    \]
    Our aim is to find a proportional Gaussian form for the
    likelihood term and then use $(\ref{eq:2.115})$ to
    evaluate the integral directly. We've seen in Exercise 3.12
    that
    \[
        \ln \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
        = -\frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
            +\beta \mathbf{w}^T \mathbf{\Phi}^T \mathbf{t}
            -\frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            + \text{const}
    \]
    This can be rewritten as a quadratic form which corresponds to a Gaussian:
    \begin{align*}
        \ln \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
        &= -\frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
            +\frac{\beta}{2}\mathbf{w}^T \mathbf{\Phi}^T \mathbf{t}
            +\frac{\beta}{2}\mathbf{t}^T \mathbf{\Phi} \mathbf{w}
            -\frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            + \text{const} \\
        &=  -\frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2 + \text{const} \\
        &=  -\frac{1}{2} (\mathbf{t} - \mathbf{\Phi}\mathbf{w})^T
            (\beta \mathbf{I})(\mathbf{t} - \mathbf{\Phi}\mathbf{w}) + \text{const} \\
        &= \ln \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w}, \beta^{-1}\mathbf{I})
            + \text{const}
    \end{align*}
    Therefore,
    \[
        p(\mathbf{t} | \mathbf{w}, \beta)
        = \prod_{n=1}^N \mathcal{N}(t_n | \mathbf{w}^T\bm{\phi}(\mathbf{x}_n), \beta^{-1})
        \propto \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w}, \beta^{-1}\mathbf{I})
    \]
    so the evidence function is now given by
    \[
        p(\mathbf{t} | \alpha, \beta)
        \propto \int \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w}, \beta^{-1}\mathbf{I})
        \mathcal{N}(\mathbf{w} | \mathbf{0}, \alpha^{-1}\mathbf{I}) \diff \mathbf{w}
    \]
    Since the integral involves the convolution of two
    Gaussians, by following the notation used in ($\ref{eq:2.113}$)
    and  $(\ref{eq:2.114})$, we'd have that
     \[
         \mathbf{x} = \mathbf{w}
         \hspace{2em}
         \mathbf{y} = \mathbf{t}
         \hspace{2em}
         \bm{\mu} = \mathbf{0}
         \hspace{2em}
         \mathbf{\Lambda}^{-1} = \alpha \mathbf{I}
         \hspace{2em}
         \mathbf{A} = \mathbf{\Phi}
         \hspace{2em}
         \mathbf{b} = 0
         \hspace{2em}
         \mathbf{L}^{-1} = \beta\mathbf{I}
    \]
    Applying $(\ref{eq:2.115})$ yields the Gaussian form of the evidence function:
    \[
        p(\mathbf{t} | \alpha, \beta)
        \propto \mathcal{N}(\mathbf{t} | \mathbf{0}, \beta^{-1}\mathbf{I} +
        \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T)
    \]
    By applying the Woodbury identity (C.7) with
    \[
        \mathbf{A} = \beta^{-1}\mathbf{I}
        \hspace{2em}
        \mathbf{B} = \mathbf{\Phi}
        \hspace{2em}
        \mathbf{C} = \alpha^{-1}\mathbf{I}
        \hspace{2em}
        \mathbf{D} = \mathbf{\Phi}^T
    \]
    the precision matrix of this Gaussian will be given by
    \begin{align*}
        (\beta^{-1}\mathbf{I} + \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T)^{-1}
        &= \beta \mathbf{I} - \beta^2 \mathbf{\Phi}(\alpha \mathbf{I}
            + \beta\mathbf{\Phi}^T\mathbf{\Phi})^{-1} \mathbf{\Phi}^T \\
        &= \beta \mathbf{I} - \beta^2 \mathbf{\Phi} \mathbf{A}^{-1} \mathbf{\Phi}^T
    \end{align*}
    where $\mathbf{A}$ is given by ($\ref{eq:3.81}$). Hence,
    by using ($\ref{eq:3.81}$) and ($\ref{eq:3.84}$),
    we obtain that the quadratic term in the exponential of
    the Gaussian has the form:
    \begin{align*}
        -\frac{1}{2}\mathbf{t}^T(\beta^{-1}\mathbf{I}
            + \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T)^{-1}\mathbf{t}
        &= -\frac{1}{2}\mathbf{t}^T (\beta\mathbf{I}
            - \beta^2\mathbf{\Phi}\mathbf{A}^{-1}\mathbf{\Phi}^T)^{-1}\mathbf{t} \\
        &= -\frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            + \frac{\beta}{2} \mathbf{t}^T\mathbf{\Phi}\mathbf{m}_N \\
        &= -\frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            + \frac{\beta}{2} \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N \\
        &= -\frac{\beta}{2} \mathbf{t}^T\mathbf{t} + \beta \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
            - \frac{\beta}{2}\mathbf{m}_N^T\mathbf{A}\mathbf{m}_N \\
        &= -\frac{\beta}{2} \mathbf{t}^T\mathbf{t} + \beta \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
            - \frac{\alpha}{2}\mathbf{m}_N^T\mathbf{m}_N
            - \frac{\beta}{2} \mathbf{m}_N\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{m}_N
    \end{align*}
    As seen in Exercise 3.18, this is actually equal to $-E(\mathbf{m}_N)$, so
    \[
        -\frac{1}{2}\mathbf{t}^T(\beta^{-1}\mathbf{I}
            + \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T)^{-1}\mathbf{t}
        = -E(\mathbf{m}_N)
    \]
    Now, since $\mathbf{\Phi}$ is a $N \cross M$ matrix, we apply (C.14) and have that:
    \begin{align*}
        |\beta^{-1}\mathbf{I}_N + \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T|
        &= \beta^{-N} \bigg|\mathbf{I}_N + \frac{\beta}{\alpha}
            \mathbf{\Phi}\mathbf{\Phi}^T\bigg| \\
        &= \beta^{-N} \bigg|\mathbf{I}_M
            + \frac{\beta}{\alpha}\mathbf{\Phi}^T\mathbf{\Phi}\bigg| \\
        &= \alpha^{-M}\beta^{-N}|\alpha\mathbf{I}_M + \beta\mathbf{\Phi}^T\mathbf{\Phi}| \\
        &= \alpha^{-M}\beta^{-N}|\mathbf{A}|
    \end{align*}
    Threfore, we finally expand the Gaussian form of the evidence function to obtain:
    \begin{align*}
        p(\mathbf{t} | \alpha, \beta)
        &\propto \mathcal{N}(\mathbf{t} | \mathbf{0}, \beta^{-1}\mathbf{I}
            + \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T) \\
        &\propto \frac{1}{(2\pi)^{N/2}}
            \frac{1}{|\beta^{-1}\mathbf{I}_N + \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T|^{1/2}}
            \exp\bigg\{-\frac{1}{2}\mathbf{t}^T(\beta^{-1}\mathbf{I}
                + \alpha^{-1}\mathbf{\Phi}\mathbf{\Phi}^T)^{-1}\mathbf{t}\bigg\} \\
        &\propto \frac{1}{(2\pi)^{N/2}} \alpha^{-M/2} \beta^{-N/2} |A|^{-1/2}
            \exp{-E(\mathbf{m}_N)}
    \end{align*}
    Hence, we easily derive the log marginal likelihood as
    \begin{align*}
        \ln p(\mathbf{t} | \alpha, \beta)
        &= -\frac{N}{2} \ln(2\pi) + \frac{M}{2} \ln \alpha + \frac{N}{2} \ln \beta
    - \frac{1}{2} \ln |\mathbf{A}| - E(\mathbf{m}_N) + \text{const} \tag{3.86}
    \end{align*}
\end{proof}

\section*{Exercise 3.17 $\star$}
Show that the evidence function for the Bayesian linear regression
model can be written in the form $(\ref{eq:3.78})$ in which
$E(\mathbf{w})$ is defined by (\ref{eq:3.79}).

\vspace{1em}

\begin{proof}
    The log likelihood is given by
    \begin{equation}\label{eq:3.11}\tag{3.11}
        \ln p(\mathbf{t} | \mathbf{w}, \beta)
        = \frac{N}{2} \ln \beta - \frac{N}{2} \ln(2\pi) - \beta E_D(\mathbf{w})
    \end{equation}
    By applying the exponential function on both sides of the expression, we obtain that
    \[
        p(\mathbf{t} | \mathbf{w}, \beta)
        = \bigg(\frac{\beta}{2\pi}\bigg)^{N/2} \exp \{-\beta E_D(\mathbf{w})\}
    \]
    We continue by expanding the Gaussian
    \begin{equation}\label{eq:3.52}\tag{3.52}
        p(\mathbf{w} | \alpha) = \mathcal{N}(\mathbf{w} | \mathbf{0}, \alpha^{-1} \mathbf{I})
    \end{equation}
    to get that
    \[
        p(\mathbf{w} | \alpha)
        = \bigg(\frac{\alpha}{2\pi}\bigg)^{M / 2}
            \exp\bigg\{-\frac{\alpha}{2} \mathbf{w}^T\mathbf{w}\bigg\}
        = \bigg(\frac{\alpha}{2\pi}\bigg)^{M / 2}  \exp\{-\alpha E_W(\mathbf{w})\}
    \]
    Therefore, by replacing into (3.77), we obtain
    \begin{align*}
        p(\mathbf{t} | \alpha, \beta)
        = \int p(\mathbf{t} | \mathbf{w}, \beta) p(\mathbf{w} | \alpha) \diff \mathbf{w}
        &= \bigg(\frac{\beta}{2\pi}\bigg)^{N/2} \bigg(\frac{\alpha}{2\pi}\bigg)^{M/2}
        \int \exp\{-\alpha E_W(\mathbf{w}) - \beta E_D(\mathbf{w})\} \diff w \\
        &= \bigg(\frac{\beta}{2\pi}\bigg)^{N/2} \bigg(\frac{\alpha}{2\pi}\bigg)^{M/2}
        \int \exp\{-E(\mathbf{w})\} \diff \mathbf{w} \label{eq:3.78} \tag{3.78}
    \end{align*}
    where
    \begin{equation}\label{eq:3.79}\tag{3.79}
        E(\mathbf{w})
        = \beta E_D(\mathbf{w}) + \alpha E_W(\mathbf{w})
        = \frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2
        + \frac{\alpha}{2} \mathbf{w}^T\mathbf{w}
    \end{equation}
\end{proof}

\section*{Exercise 3.18 $\star \star$}
By completing the square over $\mathbf{w}$, show that the error function
$(\ref{eq:3.79})$ in Bayesian linear regression can be written in the form
$(\ref{eq:3.80})$.

\vspace{1em}

\begin{proof}
    Our first step is expanding $E(\mathbf{w})$:
    \begin{align*}
        E(\mathbf{w})
        &= \frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2
            + \frac{\alpha}{2} \mathbf{w}^T\mathbf{w} \tag{3.79} \\
        &= \frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            - \frac{\beta}{2} \mathbf{t}^T\mathbf{\Phi}\mathbf{w}
            - \frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{t}
            + \frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
            + \frac{\alpha}{2} \mathbf{w}^T\mathbf{w}
    \end{align*}
    We continue by doing the same for $E(\mathbf{m}_N)$ and obtain that:
    \begin{align*}
        E(\mathbf{m}_N)
        &= \frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{m}_N||^2
            + \frac{\alpha}{2} \mathbf{m}_N^T\mathbf{m}_N \\
        &= \frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            - \frac{\beta}{2} \mathbf{t}^T\mathbf{\Phi}\mathbf{m}_N
            - \frac{\beta}{2} \mathbf{m}_N^T\mathbf{\Phi}^T\mathbf{t}
            + \frac{\beta}{2} \mathbf{m}_N^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{m}_N
            + \frac{\alpha}{2} \mathbf{m}_N^T\mathbf{m}_N
    \end{align*}
    $\mathbf{A}$ is a Hessian matrix, so it's symmetric. By using this
    and the expressions
    \begin{equation}\label{eq:3.81}\tag{3.81}
        \mathbf{A} = \alpha \mathbf{I} + \beta \mathbf{\Phi}^T\mathbf{\Phi}
    \end{equation}
    \vspace{-1em}
    \begin{equation}\label{eq:3.84}\tag{3.84}
        \mathbf{m}_N = \beta \mathbf{A}^{-1} \mathbf{\Phi}^T\mathbf{t}
    \end{equation}
    We notice that the negative terms in the expansions can be written as
    \[
        -\frac{\beta}{2} \mathbf{t}^T\mathbf{\Phi}\mathbf{w}
            - \frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}\mathbf{t}
        =
        -\frac{1}{2} \mathbf{m}_N^T\mathbf{A}\mathbf{w}
            -\frac{1}{2} \mathbf{w}^T\mathbf{A}\mathbf{m}_N
        =
        -\mathbf{m}_N^T\mathbf{A}\mathbf{w}
    \]
    \[
        -\frac{\beta}{2} \mathbf{t}^T\mathbf{\Phi}\mathbf{m}_N
            - \frac{\beta}{2} \mathbf{m}_N^T\mathbf{\Phi}\mathbf{t}
        =
        -\frac{1}{2} \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
            -\frac{1}{2} \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
        =
        -\mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
    \]
    Hence,
    \[
        E(\mathbf{w})
        = \frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            -\mathbf{m}_N^T\mathbf{A}\mathbf{w}
            + \frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
            + \frac{\alpha}{2} \mathbf{w}^T\mathbf{w}
    \]
    \[
        E(\mathbf{m}_N)
        = \frac{\beta}{2} \mathbf{t}^T\mathbf{t}
            -\mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
            + \frac{\beta}{2} \mathbf{m}_N^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{m}_N
            + \frac{\alpha}{2} \mathbf{m}_N^T\mathbf{m}_N
    \]
    By taking the difference of the error functions and then
    repeatedly making use of $(\ref{eq:3.81})$, we reach a point
    when we can complete the square:
    \begin{align*}
        E(\mathbf{w}) - E(\mathbf{m}_N)
        &= -\mathbf{m}_N^T\mathbf{A}\mathbf{w}
            + \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
            + \frac{\beta}{2} \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
            - \frac{\beta}{2} \mathbf{m}_N^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{m}_N
            + \frac{\alpha}{2} \mathbf{w}^T\mathbf{w}
            - \frac{\alpha}{2} \mathbf{m}_N^T\mathbf{m}_N \\
        &= -\mathbf{m}_N^T\mathbf{A}\mathbf{w}
            + \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
            + \frac{1}{2} \mathbf{w}^T\big(
                \alpha \mathbf{I} + \beta \mathbf{\Phi}^T\mathbf{\Phi}\big)
                \mathbf{w}
            - \mathbf{m}_N^T\big(
                \alpha \mathbf{I} + \beta \mathbf{\Phi}^T\mathbf{\Phi}\big)
                \mathbf{m}_N \\
        &= -\mathbf{m}_N^T\mathbf{A}\mathbf{w}
            + \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N
            + \frac{1}{2} \mathbf{w}^T\mathbf{A}\mathbf{w}
            - \frac{1}{2} \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N \\
        &= \frac{1}{2} \mathbf{w}^T\mathbf{A}\mathbf{w}
            -\frac{1}{2} \mathbf{m}_N^T \mathbf{A}\mathbf{w}
            -\frac{1}{2} \mathbf{w}^T\mathbf{A}\mathbf{m}_N
            + \frac{1}{2} \mathbf{m}_N^T\mathbf{A}\mathbf{m}_N \\
        &= \frac{1}{2} (\mathbf{w} - \mathbf{m}_N)^T\mathbf{A}(\mathbf{w} - \mathbf{m}_N)
    \end{align*}
    which directly proves that $(\ref{eq:3.79})$ can be written as
    \begin{equation}\label{eq:3.80}\tag{3.80}
        E(\mathbf{w}) = E(\mathbf{m}_N) +
            \frac{1}{2} (\mathbf{w} - \mathbf{m}_N)^T\mathbf{A}(\mathbf{w} - \mathbf{m}_N)
    \end{equation}
\end{proof}

\section*{Exercise 3.19 $\star \star$}
Show that the integration over $\mathbf{w}$ in the Bayesian
linear regression model gives the result $(\ref{eq:3.85})$.
Hence show that the log marginal likelihood is given by
$(\ref{eq:3.86})$.

\vspace{1em}

\begin{proof}
    We start by rewriting $E(\mathbf{w})$ like in ($\ref{eq:3.80}$) and obtain that
    \begin{align*}
        \int \exp\{-E(\mathbf{w})\} \diff \mathbf{w}
        &= \int \exp\bigg\{-E(\mathbf{m}_N) -
            \frac{1}{2} (\mathbf{w} - \mathbf{m}_N)^T\mathbf{A}(\mathbf{w} - \mathbf{m}_N)
            \bigg\}\diff \mathbf{w} \\
        &= \int \exp\{-E(\mathbf{m}_N)\}
            \exp\bigg\{-\frac{1}{2} (\mathbf{w} - \mathbf{m}_N)^T\mathbf{A}(\mathbf{w} - \mathbf{m}_N)
            \bigg\}\diff \mathbf{w} \\
        &= \exp\{-E(\mathbf{m}_N)\}
            \int \exp\bigg\{-\frac{1}{2} (\mathbf{w} - \mathbf{m}_N)^T\mathbf{A}(\mathbf{w} - \mathbf{m}_N)
            \bigg\}\diff \mathbf{w}
    \end{align*}
    The integral is easily solved by noticing that the quadratic
    term under the exponential term corresponds to a Gaussian of
    the form $\mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{A}^{-1})$.
    Because the Gaussian distribution is normalized, we then have that
    \begin{align*}
        \int &\exp\{-E(\mathbf{w})\} \diff \mathbf{w} \\
        &= \exp\{-E(\mathbf{m}_N)\} (2\pi)^{M / 2} |\mathbf{A}|^{-1/2}
            \int\frac{1}{(2\pi)^{M/2}}\frac{1}{|\mathbf{A}|^{-1/2}}
            \exp\bigg\{\frac{1}{2}
                (\mathbf{w} - \mathbf{m}_N)^T\mathbf{A}(\mathbf{w} - \mathbf{m}_N)
            \bigg\}\diff \mathbf{w} \\
        &= \exp\{-E(\mathbf{m}_N)\} (2\pi)^{M / 2} |\mathbf{A}|^{-1/2}
            \int \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \mathbf{A}^{-1}) \diff \mathbf{w} \\
        &= \exp\{-E(\mathbf{m}_N)\} (2\pi)^{M / 2} |\mathbf{A}|^{-1/2} \label{eq:3.85}\tag{3.85}
    \end{align*}
    By substituting this into $(\ref{eq:3.78})$, the evidence function becomes
    \[
        p(\mathbf{t} | \alpha, \beta)
        = \bigg(\frac{\beta}{2\pi}\bigg)^{N / 2} \alpha^{M/2} |\mathbf{A}|^{-1/2}
            \exp\{-E(\mathbf{m}_N)\}
    \]
    Hence, the log marginal likelihood is given by
    \begin{equation}\label{eq:3.86}\tag{3.86}
        \ln p(\mathbf{t} | \alpha, \beta)
        = \frac{M}{2} \ln \alpha + \frac{N}{2} \ln \beta - E(\mathbf{m}_N)
            -\frac{1}{2} \ln|\mathbf{A}| - \frac{N}{2} \ln(2\pi)
    \end{equation}
\end{proof}

\section*{Exercise 3.20 $\star \star$}
Verify all of the steps needed to show that
maximization of the log marginal likelihood function (\ref{eq:3.86}) with
respect to $\alpha$ leads to the re-estimation equation (\ref{eq:3.92}).

\vspace{1em}

\begin{proof}
    The steps taken in the maximization of the (\ref{eq:3.86})
    are quite straightforward, so this proof will be very similar
    to what's in the book. By defining the eigenvector equation
    \begin{equation}\label{eq:3.87}\tag{3.87}
        (\beta \mathbf{\Phi}^T\mathbf{\Phi})\mathbf{u}_i = \lambda_i \mathbf{u}_i
    \end{equation}
    we'd have that
    \[
        \mathbf{A}\mathbf{u}_i
        = (\alpha\mathbf{I} + \beta\mathbf{\Phi}^T\mathbf{\Phi})\mathbf{u}_i
        = \alpha\mathbf{u}_i + (\beta \mathbf{\Phi}^T\mathbf{\Phi})\mathbf{u}_i
        = (\alpha + \lambda_i) \mathbf{u}_i
    \]
    which shows that the eigenvalues of $\mathbf{A}$ are $\alpha + \lambda_i$,
    where $\mathbf{A}$ is given by ($\ref{eq:3.81}$).
    Now, since the determinant of a matrix is the product of its eigenvalues,
    we have that
    \begin{equation}\label{eq:3.88}\tag{3.88}
        \dv{\alpha} \ln|\mathbf{A}|
        = \dv{\alpha} \ln \prod_{i}(\lambda_i + \alpha)
        = \dv{\alpha} \sum_i \ln(\lambda_i + \alpha)
        = \sum_i \frac{1}{\lambda_i + \alpha}
    \end{equation}
    The derivative of $(\ref{eq:3.86})$ with respect to $\alpha$ is given by
    \begin{align*}
        \pdv{\alpha} \ln p(\mathbf{t} | \alpha, \beta)
        &= \frac{M}{2\alpha} - \frac{1}{2} \mathbf{m}_N^T\mathbf{m}_N
        - \frac{1}{2} \sum_i\frac{1}{\lambda_i + \alpha}
    \end{align*}
    so the stationary points will satisfy
    \begin{equation}\label{eq:3.89}\tag{3.89}
        0 = \frac{M}{2\alpha} - \frac{1}{2} \mathbf{m}_N^T\mathbf{m}_N
        - \frac{1}{2} \sum_{i} \frac{1}{\lambda_i + \alpha}
    \end{equation}
    Multiplying by $2\alpha$ and then rearranging,
    we have that
    \[
        \alpha \mathbf{m}^T\mathbf{m}_N
        = M - \alpha \sum_i \frac{1}{\lambda_i + \alpha}
        = \sum_i \bigg(1 - \frac{\alpha}{\lambda_i + \alpha}\bigg)
        = \sum_i \frac{\lambda_i}{\alpha + \lambda_i}
        = \gamma
    \]
    Therefore, the value of $\alpha$ that maximizes the marginal likelihood
    is given by
    \begin{equation}\label{eq:3.92}\tag{3.92}
        \alpha = \frac{\gamma}{\mathbf{m}_N^T\mathbf{m}_N}
    \end{equation}
    where $\gamma$ is defined by
    \begin{equation}\label{eq:3.91}\tag{3.91}
        \gamma = \sum_i \frac{\lambda_i}{\alpha + \lambda_i}
    \end{equation}
\end{proof}

\section*{Exercise 3.21 $\star \star$}
An alternative way to derive the result ($\ref{eq:3.92}$) for the optimal
value of $\alpha$ in the evidence framework is to make use of the identity
(note that we changed the variables $\mathbf{A}$ and $\alpha$ for $\mathbf{D}$ and
$\delta$ so that there is no confusion with the variables used in our
framework)
\begin{equation}\label{eq:3.117}\tag{3.117}
    \dv{\delta} \ln |\mathbf{D}| = \text{Tr}\bigg(\mathbf{D}^{-1}\dv{\delta}\mathbf{D}\bigg)
\end{equation}
Prove this identity by considering the eigenvalue expansion of a real
symmetric matrix $\mathbf{D}$, and making use of the standard results
for the determinant and trace of $\mathbf{D}$ expressed in terms of
its eigenvalues (Appendix C). Then make use of $(\ref{eq:3.117})$ to
derive $(\ref{eq:3.92})$ starting from $(\ref{eq:3.86})$.

\vspace{1em}

\begin{proof}
    Since $\mathbf{D}$ is symmetric, we can consider the $N$ eigenvector equations
    \[
        \mathbf{D}\mathbf{u}_i = \omega_i\mathbf{u}_i
    \]
    where the eigenvectors $\mathbf{u}_i$ were chosen to be orthonormal,
    as seen in Appendix C. By using (C.47) to rewrite the determinant,
    the left-hand side of $(\ref{eq:3.117})$ becomes
     \[
         \dv{\delta}\ln|\mathbf{D}|
         = \dv{\delta}\ln \prod_{i=1}^N \omega_i
         = \dv{\delta} \sum_{i=1}^{N} \ln \omega_i
         = \sum_{i=1}^{N} \dv{\delta} \ln \omega_i
         = \sum_{i=1}^{N} \frac{1}{\omega_i} \dv{\delta} \omega_i
    \]
    Now, by considering the eigenvalue expansions given by (C.45) and (C.46),
    \[
        \mathbf{D} = \sum_{i=1}^{N} \omega_i \mathbf{u}_i\mathbf{u}_i^T
        \hspace{6em}
        \mathbf{D}^{-1} = \sum_{i=1}^{N} \frac{1}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T
    \]
    we can rewrite the term inside the trace operator as:
    \begin{align*}
        \mathbf{D}^{-1}\dv{\delta}\mathbf{D}
        &= \bigg(\sum_{i=1}^{N} \frac{1}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T\bigg)
        \bigg(\dv{\delta} \sum_{j=1}^{N} \omega_i\mathbf{u}_i\mathbf{u}_i^T\bigg) \\
        &= \bigg(\sum_{i=1}^{N} \frac{1}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T\bigg)
        \sum_{j=1}^{N} \dv{\delta}(\omega_j\mathbf{u}_j\mathbf{u}_j^T) \\
        &= \bigg(\sum_{i=1}^{N} \frac{1}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T\bigg)
        \sum_{j=1}^{N} \bigg[\bigg(\dv{\delta} \omega_j\bigg) \mathbf{u}_j\mathbf{u}_j^T
            + \omega_j \bigg(\dv{\delta}\mathbf{u}_j\bigg) \mathbf{u}_j^T
        + \omega_i \mathbf{u}_i \bigg(\dv{\delta} \mathbf{u}_j^T\bigg)\bigg] \\
        &= \sum_{i=1}^{N} \sum_{j=1}^{N}
            \frac{1}{\omega_i} \bigg(\dv{\delta}\omega_j\bigg) \mathbf{u}_i\mathbf{u}_i^T
            \mathbf{u}_j\mathbf{u}_j^T
        + \sum_{i=1}^{N} \sum_{j=1}^{N} \frac{\omega_j}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T
        \bigg[\bigg(\dv{\delta} \mathbf{u}_j\bigg)\mathbf{u}_j^T
        + \mathbf{u}_j\bigg(\dv{\delta}\mathbf{u}_j^T\bigg)\bigg]
    \end{align*}
    We'll analyze the two sum terms separately. Since the
    eigenvectors were chosen to be orthonormal, we have that
    \begin{equation}\label{eq:C.33}\tag{C.33}
        \mathbf{u}_i^T\mathbf{u}_j = \mathbf{I}_{ij}
    \end{equation}
    Therefore, we'll only have to keep the terms for which $i = j$ in
    the first sum, so
    \[
        \sum_{i=1}^{N} \sum_{j=1}^{N}
            \frac{1}{\omega_i} \bigg(\dv{\delta}\omega_j\bigg) \mathbf{u}_i\mathbf{u}_i^T
            \mathbf{u}_j\mathbf{u}_j^T
        =
        \sum_{i=1}^{N} \sum_{j=1}^{N}
        \frac{1}{\omega_i} \bigg(\dv{\delta}\omega_j\bigg) \mathbf{u}_i\mathbf{I}_{ij}
            \mathbf{u}_j^T
        = \sum_{i=1}^{N} \frac{1}{\omega_i}
        \bigg(\dv{\delta}\omega_j\bigg) \mathbf{u}_i\mathbf{u_i}^T
    \]
    Notice that the trace of this term is actually the left-hand side
    of ($\ref{eq:3.117}$), so we'll continue by proving that the second
    sum term has a trace of 0. The second sum term can then be expanded as
    \begin{align*}
        \sum_{i=1}^{N} \sum_{j=1}^{N} \frac{\omega_j}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T
        &\bigg[\bigg(\dv{\delta} \mathbf{u}_j\bigg)\mathbf{u}_j^T\bigg
        + \mathbf{u}_j\bigg(\dv{\delta}\mathbf{u}_j^T\bigg)\bigg]
        =
        \sum_{i=1}^{N} \sum_{j=1}^{N} \frac{2\omega_j}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T
        \mathbf{u}_j\bigg(\dv{\delta}\mathbf{u}_j^T\bigg)
        = \sum_{i=1}^{N} \sum_{j=1}^{N} \frac{2\omega_j}{\omega_i} \mathbf{u}_i\mathbf{I}_{ij}
        \bigg(\dv{\delta}\mathbf{u}_j^T\bigg)\\
        &= \sum_{i=1}^{N} 2\mathbf{u}_i\bigg(\dv{\delta} \mathbf{u}_i^T\bigg)
        = \sum_{i=1}^{N} \bigg[\bigg(\dv{\delta}\mathbf{u}_i\bigg)\mathbf{u}_i^T
        + \mathbf{u}_i\bigg(\dv{\delta}\mathbf{u}_i^T\bigg)\bigg] \\
        &= \sum_{i=1}^{N} \dv{\delta} (\mathbf{u}_i\mathbf{u}_i^T)
        = \dv{\delta}\sum_{i=1}^{N} \mathbf{u}_i\mathbf{u}_i^T = \dv{\delta}\mathbf{I}
        = \mathbf{0}_N
    \end{align*}
    Finally, we have that
    \begin{align*}
        \text{Tr}\bigg(\mathbf{D}^{-1}\dv{\delta}\mathbf{D}\bigg)
        &= \text{Tr}\bigg\{\sum_{i=1}^{N} \sum_{j=1}^{N}
            \frac{1}{\omega_i} \bigg(\dv{\delta}\omega_j\bigg) \mathbf{u}_i\mathbf{u}_i^T
            \mathbf{u}_j\mathbf{u}_j^T
        + \sum_{i=1}^{N} \sum_{j=1}^{N} \frac{\omega_j}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T
        \bigg[\bigg(\dv{\delta} \mathbf{u}_j\bigg)\mathbf{u}_j^T
        + \mathbf{u}_j\bigg(\dv{\delta}\mathbf{u}_j^T\bigg)\bigg]\bigg\} \\
        &= \text{Tr}\bigg\{\sum_{i=1}^{N} \sum_{j=1}^{N}
            \frac{1}{\omega_i} \bigg(\dv{\delta}\omega_j\bigg) \mathbf{u}_i\mathbf{u}_i^T
        \mathbf{u}_j\mathbf{u}_j^T \bigg\}
        + \text{Tr}\bigg\{\sum_{i=1}^{N} \sum_{j=1}^{N}
            \frac{\omega_j}{\omega_i} \mathbf{u}_i\mathbf{u}_i^T
        \bigg[\bigg(\dv{\delta} \mathbf{u}_j\bigg)\mathbf{u}_j^T
        + \mathbf{u}_j\bigg(\dv{\delta}\mathbf{u}_j^T\bigg)\bigg]\bigg\} \\
        &= \text{Tr}\bigg[\sum_{i=1}^{N} \frac{1}{\omega_i}\bigg(\dv{\delta} \omega_i\bigg)
        \mathbf{u}_i\mathbf{u}_i^T\bigg] + \text{Tr}(\mathbf{0}_N) \\
        &= \sum_{i=1}^{N} \frac{1}{\omega_i} \bigg(\dv{\delta} \omega_i\bigg) \\
        &= \dv{\delta} \ln |\mathbf{D}| \tag{3.117}
    \end{align*}
    which proves the needed identity. We can derive $(\ref{eq:3.92})$
    by following exactly the same steps as in Exercise 3.20 or in the book,
    except that now we compute ($\ref{eq:3.88}$) by using the previously
    proven identity and (C.47). Note that $\mathbf{A}$ is given by ($\ref{eq:3.81}$)
    and it has the eigenvalues $\lambda_i + \alpha$, so
    \[
        \dv{\alpha}\ln |\mathbf{A}|
        = \text{Tr}\bigg(\mathbf{A}^{-1}\dv{\alpha}\mathbf{A}\bigg)
        = \text{Tr}\bigg[\mathbf{A}^{-1}
        \dv{\alpha}(\alpha\mathbf{I} + \beta\mathbf{\Phi}^T\mathbf{\Phi})\bigg]
        = \text{Tr}(\mathbf{A}^{-1})
        = \sum_{i} \frac{1}{\lambda_i + \alpha}
    \]
\end{proof}

\section*{Exercise 3.22 $\star \star$}
Starting from $(\ref{eq:3.86})$ verify all of the steps needed
to show that maximization of the log marginal likelihood function
($\ref{eq:3.86}$) with respect to $\beta$ leads to the re-estimation
equation $(\ref{eq:3.95})$.

\vspace{1em}

\begin{proof}
   As in Exercise 3.20, one could prove that $\mathbf{A}$ has the eigenvalues
   $\lambda_i + \alpha$. Also, from $(\ref{eq:3.87})$ the eigenvalues are proportional
   to $\beta$, so $d\lambda_i / d\beta = \lambda_i/\beta$, so
   \begin{equation}\label{eq:3.93}\tag{3.93}
       \dv{\beta} \ln |\mathbf{A}| = \dv{\beta} \sum_i \ln (\lambda_i + \alpha)
       = \frac{1}{\beta} \sum_i \frac{\lambda_i}{\lambda_i + \alpha}
       = \frac{\gamma}{\beta}
   \end{equation}
   where $\gamma$ is given by (\ref{eq:3.91}). Therefore,
   the derivative of $(\ref{eq:3.87})$ with respect to $\beta$ is given by
   \[
       \pdv{\beta} p(\mathbf{t} | \alpha, \beta)
       = \frac{N - \gamma}{2\beta}
       - \frac{1}{2} \sum_{n=1}^{N}\{t_n - \mathbf{m}_N^T\bm{\phi}(\mathbf{x}_n)\}^2
   \]
   and the critical points satisfy
   \[
       \frac{N - \gamma}{2\beta} =
       \frac{1}{2} \sum_{n=1}^{N}\{t_n - \mathbf{m}_N^T\bm{\phi}(\mathbf{x}_n)\}^2
   \]
   which after multiplying both sides by $2/(N - \gamma)$ gives
   us the re-estimation equation
   \begin{equation}\label{eq:3.95}\tag{3.95}
       \frac{1}{\beta} = \frac{1}{N - \gamma} \sum_{n=1}^{N}
       \{t_n - \mathbf{m}_N^T\bm{\phi}(\mathbf{x}_n)\}^2
   \end{equation}
\end{proof}

\section*{Exercise 3.23 $\star \star$}
Show that the marginal probability of the data, in other words the model
evidence, for the model described in Exercise 3.12 is given by
\begin{equation}\label{eq:3.118}\tag{3.118}
    p(\mathbf{t}) = \frac{1}{(2\pi)^{N / 2}} \frac{b_0^{a_0}}{b_N^{a_N}}
    \frac{\Gamma(a_N)}{\Gamma(a_0)} \frac{|\mathbf{S}_N|^{1/2}}{|\mathbf{S}_0|^{1/2}}
\end{equation}
by first marginalizing with respect to $\mathbf{w}$ and then with
respect to $\beta$.

\vspace{1em}

\begin{proof}
    By marginalizing with respect to $\mathbf{\beta}$ and then
    with respect to $\beta$, the model evidence will be given by
     \[
         p(\mathbf{t})
         = \iint p(\mathbf{w}, \beta) p(\mathbf{t} | \mathbf{w}, \beta) \diff \mathbf{w}
         \diff\beta
    \]
    The first factor unde the integral is the prior given by $(\ref{eq:3.112})$,
    while the second factor is the likelihood $(3.11)$. We proved
    in Exercise 3.16 that the likelihood is proportional to
    $\mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w}, \beta^{-1}\mathbf{I})$,
    so the marginal probability becomes
    \begin{align*}
         p(\mathbf{t})
         &= \iint \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            \text{Gam}(\beta | a_0, b_0)
            \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w}, \beta^{-1}\mathbf{I})
            \diff \mathbf{w}\diff\beta \\
         &= \iint \text{Gam}(\beta | a_0, b_0)
            \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w},
                \beta^{-1}\mathbf{I}) \diff \mathbf{w}
            \diff \beta
    \end{align*}
    By expanding the three distributions, we have that
    \begin{align*}
        p(\mathbf{t}) = &\frac{1}{(2\pi)^{\frac{N + M}{2}}}\frac{b_0^{a_0}}{\Gamma(a_0)}
         \frac{1}{|\mathbf{S}_0|^{1/2}} \\
        &\iint \beta^{a_0 - 1 + N/2 + M/2} \exp\{\beta b_0\}
         \exp\bigg\{-\frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2\bigg\}
         \exp\bigg\{-\frac{\beta}{2}(\mathbf{w} - \mathbf{m}_0)^T
             \mathbf{S}_0^{-1}(\mathbf{w} - \mathbf{m}_0)
         \bigg\} \diff \mathbf{w} \diff \beta
    \end{align*}
    Let's expand the term under the $\mathbf{w}$ integral, and then use $(\ref{eq:3.50})$ and
    $(\ref{eq:3.51})$ to complete the square:
    \begin{align*}
        \exp\bigg\{-\frac{\beta}{2} &||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2\bigg\}
        \exp\bigg\{-\frac{\beta}{2}(\mathbf{w} - \mathbf{m}_0)^T
             \mathbf{S}_0^{-1}(\mathbf{w} - \mathbf{m}_0)
         \bigg\} \\
        &=\exp\bigg\{-\frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2
        -\frac{\beta}{2}(\mathbf{w} - \mathbf{m}_0)^T
             \mathbf{S}_0^{-1}(\mathbf{w} - \mathbf{m}_0)
         \bigg\} \\
        &= \exp\bigg\{-\frac{\beta}{2}\big(
        \mathbf{t}^T\mathbf{t}
        - 2\mathbf{w}^T \mathbf{\Phi}^T \mathbf{t}
        + \mathbf{w}^T\mathbf{\Phi}^T\mathbf{\Phi}\mathbf{w}
        + \mathbf{w}^T\mathbf{S}_0^{-1}\mathbf{w}
        - 2 \mathbf{w}^T\mathbf{S}_0^{-1}\mathbf{m}_0
        + \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0\big)\bigg\} \\
        &= \exp\bigg\{-\frac{\beta}{2}\big[
        \mathbf{w}^T\big(\mathbf{S}_0^{-1}
            + \mathbf{\Phi}^T\mathbf{\Phi}\big)\mathbf{w}
        - 2\mathbf{w}^T\big(\mathbf{S}_0^{-1}\mathbf{m}_0\big
            + \mathbf{\Phi}^T\mathbf{t}\big)
        + \mathbf{t}^T\mathbf{t}
        + \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0\big]\bigg\} \\
        &= \exp\bigg\{-\frac{\beta}{2}\big(
        \mathbf{w}^T\mathbf{S}_N^{-1}\mathbf{w}
        - 2\mathbf{w}^T\mathbf{S}_N^{-1}\mathbf{m}_N
        + \mathbf{t}^T\mathbf{t}
        + \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0\big)\bigg\} \\
        &= \exp\bigg\{-\frac{\beta}{2}\big[
        (\mathbf{w} - \mathbf{m}_N)^T\mathbf{S}_N^{-1}(\mathbf{w} - \mathbf{m}_N)
        -\mathbf{m}_N^T\mathbf{S}_N^{-1}\mathbf{m}_N
        + \mathbf{t}^T\mathbf{t}
        + \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0\big]\bigg\} \\
    \end{align*}
    We can rewrite $(\ref{eq:3.12.1})$ as
    \[
        b_0 = b_N - \frac{1}{2} \mathbf{t}^T\mathbf{t}
        - \frac{1}{2} \mathbf{m}_0^T\mathbf{S}_0^{-1}\mathbf{m}_0
        + \frac{1}{2} \mathbf{m}_N^T\mathbf{S}_N^{-1}\mathbf{m}_N
    \]
    Therefore,
    \begin{align*}
        \exp\{\beta b_0\}
         \exp\bigg\{-\frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2\bigg\}
         &\exp\bigg\{-\frac{\beta}{2}(\mathbf{w} - \mathbf{m}_0)^T
             \mathbf{S}_0^{-1}(\mathbf{w} - \mathbf{m}_0)
         \bigg\} \\
         &= \exp\{\beta b_N\}\exp\bigg\{-\frac{\beta}{2}
            (\mathbf{w} - \mathbf{m}_N)^T\mathbf{S}_N^{-1}(\mathbf{w} - \mathbf{m}_N)\bigg\}
    \end{align*}
    and since both the Gaussian and Gamma distributions
    are normalized, the marginal probability finally becomes
    what we wanted:
    \begin{align*}
        p(\mathbf{t})
        &= \frac{1}{(2\pi)^{\frac{N + M}{2}}}\frac{b_0^{a_0}}{\Gamma(a_0)}
         \frac{1}{|\mathbf{S}_0|^{1/2}}
        \int \beta^{a_0 - 1 + N/2 + M/2}
        \int \exp{\beta b_N}\exp\bigg\{-\frac{\beta}{2}(\mathbf{w}
            - \mathbf{m}_N)^T\mathbf{S}_N^{-1}
            (\mathbf{w} - \mathbf{m}_N)\bigg\} \diff \mathbf{w} \diff \beta \\
        &= \frac{1}{(2\pi)^{\frac{N + M}{2}}}\frac{b_0^{a_0}}{\Gamma(a_0)}
         \frac{1}{|\mathbf{S}_0|^{1/2}}
        \int \beta^{a_0 - 1+ N/2 + M/2}
        \int \bigg(\frac{2\pi}{\beta}\bigg)^{M/2} |\mathbf{S}_N|^{1/2}
        \exp\{\beta b_N\}
        \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1}\mathbf{S}_N)
        \diff \mathbf{w} \diff \beta \\
        &= \frac{1}{(2\pi)^{\frac{N}{2}}}\frac{b_0^{a_0}}{\Gamma(a_0)}
        \frac{|\mathbf{S}_N|^{1/2}}{|\mathbf{S}_0|^{1/2}}
        \int \beta^{a_0 - 1 + N/2}
        \exp\{\beta b_N\}
        \int\mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1}\mathbf{S}_N)
        \diff \mathbf{w} \diff \beta \\
        &= \frac{1}{(2\pi)^{\frac{N}{2}}}\frac{b_0^{a_0}}{\Gamma(a_0)}
        \frac{|\mathbf{S}_N|^{1/2}}{|\mathbf{S}_0|^{1/2}}
        \int \beta^{a_0 - 1 + N/2}
        \exp\{\beta b_N\} \diff \beta \\
        &= \frac{1}{(2\pi)^{\frac{N}{2}}}\frac{b_0^{a_0}}{\Gamma(a_0)}
        \frac{|\mathbf{S}_N|^{1/2}}{|\mathbf{S}_0|^{1/2}}
        \int\frac{\Gamma(a_N)}{b_N^{a_N}} \text{Gam}(\beta | a_N, b_N) \diff \beta \\
        &= \frac{1}{(2\pi)^{N / 2}} \frac{b_0^{a_0}}{b_N^{a_N}}
        \frac{\Gamma(a_N)}{\Gamma(a_0)} \frac{|\mathbf{S}_N|^{1/2}}{|\mathbf{S}_0|^{1/2}}
        \tag{3.118}
    \end{align*}
    where $a_N$ and $b_N$ were derived in Exercise 3.12 and are
    given by $(\ref{eq:2.150})$, respectively $(\ref{eq:3.12.1})$.
\end{proof}

\section*{Exercise 3.24 $\star \star$}
Repeat the previous exercise but now use Bayes' theorem in
the form
\begin{equation}\label{eq:3.119}\tag{3.119}
    p(\mathbf{t})
    = \frac{p(\mathbf{t} | \mathbf{w}, \beta)p(\mathbf{w}, \beta)}
        {p(\mathbf{w}, \beta | \mathbf{t})}
\end{equation}
and then substitute for the prior and posterior distributions and
the likelihood function in order to derive the result ($\ref{eq:3.118}$).

\vspace{1em}
\begin{proof}
    We start by substituting the prior ($\ref{eq:3.112}$), the
    posterior $(\ref{eq:3.113}$) and the likelihood (3.10). The
    evidence function becomes
    \[
        p(\mathbf{t}) = \frac{
            \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w}, \beta^{-1}\mathbf{I})
            \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            \text{Gam}(\beta | a_0, b_0)}{
            \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1}\mathbf{S}_N)
        \text{Gam}(\beta | a_N, b_N)}
    \]
    Let's expand the numerator:
    \begin{align*}
        \mathcal{N}(\mathbf{t} | &\mathbf{\Phi}\mathbf{w}, \beta^{-1}\mathbf{I})
            \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            \text{Gam}(\beta | a_0, b_0) = \\
        =& \bigg(\frac{\beta}{2\pi}\bigg)^{M/2}
            \bigg(\frac{\beta}{2\pi}\bigg)^{N/2}
            \frac{b_0^{a_0}\beta^{a_0 - 1}}{\Gamma(a_0)|\mathbf{S}_0|^{1/2}}
            \exp\{\beta b_0\}
            \exp\bigg\{-\frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2\bigg\}
            \exp\bigg\{-\frac{\beta}{2} (\mathbf{w} - \mathbf{m}_0)^T\mathbf{S}_0^{-1}
                (\mathbf{w} - \mathbf{m}_0)\bigg\}
    \end{align*}
    We've seen in the previous exercise that
    \begin{align*}
        \exp\{\beta b_0\}
         \exp\bigg\{-\frac{\beta}{2} ||\mathbf{t} - \mathbf{\Phi}\mathbf{w}||^2\bigg\}
         &\exp\bigg\{-\frac{\beta}{2}(\mathbf{w} - \mathbf{m}_0)^T
             \mathbf{S}_0^{-1}(\mathbf{w} - \mathbf{m}_0)
         \bigg\} \\
         &= \exp\{\beta b_N\}\exp\bigg\{-\frac{\beta}{2}
            (\mathbf{w} - \mathbf{m}_N)^T\mathbf{S}_N^{-1}(\mathbf{w} - \mathbf{m}_N)\bigg\}
    \end{align*}
    so the numerator can be written as
    \begin{align*}
        \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w},& \beta^{-1}\mathbf{I})
            \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            \text{Gam}(\beta | a_0, b_0) = \\
        =& \bigg(\frac{\beta}{2\pi}\bigg)^{M/2}
            \bigg(\frac{\beta}{2\pi}\bigg)^{N/2}
            \frac{b_0^{a_0}\beta^{a_0 - 1}}{\Gamma(a_0)|\mathbf{S}_0|^{1/2}}
            \exp\{\beta b_N\}\exp\bigg\{-\frac{\beta}{2}
            (\mathbf{w} - \mathbf{m}_N)^T\mathbf{S}_N^{-1}(\mathbf{w} - \mathbf{m}_N)\bigg\}
    \end{align*}
    Since we already have the exponential terms of the Gamma and
    Gaussian distributions, we can obtain the distributions by
    dividing by their normalization constants, so:
    \begin{align*}
        \mathcal{N}(\mathbf{t} | \mathbf{\Phi}\mathbf{w},& \beta^{-1}\mathbf{I})
            \mathcal{N}(\mathbf{w} | \mathbf{m}_0, \beta^{-1}\mathbf{S}_0)
            \text{Gam}(\beta | a_0, b_0) = \\
        =& \bigg(\frac{\beta}{2\pi}\bigg)^{M/2}
            \bigg(\frac{\beta}{2\pi}\bigg)^{N/2}
            \frac{b_0^{a_0}\beta^{a_0 - 1}}{\Gamma(a_0)|\mathbf{S}_0|^{1/2}}
            \bigg[\frac{\Gamma(a_N)}{b_N^{a_N}\beta^{a_N - 1}}
            \text{Gam}(\beta | a_N, b_N)\bigg]
            \bigg[\bigg(\frac{2\pi}{\beta}\bigg)^{M/2} |\mathbf{S}_N|^{1/2}
            \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1}\mathbf{S}_N)\bigg] \\
        =& \frac{1}{(2\pi)^{N / 2}} \frac{b_0^{a_0}}{b_N^{a_N}}
            \frac{\Gamma(a_N)}{\Gamma(a_0)} \frac{|\mathbf{S}_N|^{1/2}}{|\mathbf{S}_0|^{1/2}}
            \text{Gam}(\beta | a_N, b_N)
            \mathcal{N}(\mathbf{w} | \mathbf{m}_N, \beta^{-1}\mathbf{S}_N)
    \end{align*}
    Finally, we substitute the numerator back into the evidence function
    and since the distribution forms factor out, we prove our hypothesis, that:
    \begin{equation*}\tag{3.118}
        p(\mathbf{t}) = \frac{1}{(2\pi)^{N / 2}} \frac{b_0^{a_0}}{b_N^{a_N}}
        \frac{\Gamma(a_N)}{\Gamma(a_0)} \frac{|\mathbf{S}_N|^{1/2}}{|\mathbf{S}_0|^{1/2}}
    \end{equation*}
\end{proof}


%!TEX root = main.tex

% =============================================================================
\section{Model Evaluation}
\label{sec:model-evaluation}
% =============================================================================

Using the uncertainty benchmark defined in Section~\ref{sec:uncertainty-benchmark},
    and the analytical cost model from Section~\ref{sec:tuning_lsm_trees},
    we now rigorously evaluate the performance of the proposed robust tunings by
    {\Endure}.

\subsection{Evaluation Metrics}
Now, we describe the metrics used to compare different tuning configurations.

\Paragraph{Normalized Delta Throughput ($\Delta$)} Defining throughput as the
    reciprocal of the cost of executing a workload, we measure the
    normalized delta throughput of a configuration {$\configuration_2$}
    wrt. another configuration {$\configuration_1$} for a given
    workload {\workload} as follows,
    \[
        \Delta_{\workload}(\configuration_1, \configuration_2) =
        \frac{\nicefrac{1}{C(\workload, \configuration_2)} -
            \nicefrac{1}{C(\workload, \configuration_1)}}
            {\nicefrac{1}{C(\workload, \configuration_1)}}.
    \]
$\Delta_{\workload}(\configuration_1, \configuration_2) > 0$ implies that
    {$\configuration_2$} outperforms {$\configuration_1$} when executing
    a workload {\workload} and vice versa when
    $\Delta_{\workload}(\configuration_1, \configuration_2) < 0$.

\Paragraph{Throughput Range ($\Theta$)} While normalized delta throughput
    compares two different tunings, we use the throughput
    range to evaluate an individual tuning ${\configuration}$
    wrt. the  benchmark set {\benchmark} as follows,
    \[
        \Theta_{\mathcal{B}}(\configuration) =
        \max_{\workload_0, \workload_1 \in \mathcal{B}}
            \bigg(\frac{1}{C(\workload_0, \configuration)}
            - \frac{1}{C(\workload_1, \configuration)}\bigg).
    \]
$\Theta_{\benchmark}(\configuration)$ intuitively captures the best
    and the worst-case outcomes of the tuning {\configuration}.
A smaller value of this metric implies higher consistency in performance.

\begin{figure}
    \centering
    \includegraphics[scale=0.5]{figures/delta_throughput_workload_type.pdf}
    \caption{Delta throughput $\Delta_{\obsworkload}(\Phi_{N}, \Phi_{R})$
        between the nominal and expected tunings across various expected workloads.
    }
    \label{fig:delta_throughput_workload_type}
\end{figure}

\begin{figure*}[ht]
    \centering
    \includegraphics[width=\textwidth]{figures/scatterplot_evolution_rho.pdf}
    \caption{Impact of $\rho$ on normalized delta throughput
        $\Delta_{\obsworkload}(\Phi_{N}, \Phi_{R})$ for tunings with expected
        workload $\workload_{11}$.
    }
    \label{fig:scatterplot_evolution_rho}
\end{figure*}

\subsection{Experiment Design}

To evaluate the performance of our proposed robust tuning approach,
    we design a large-scale experiment comparing different tunings
    over the sampled workloads in {\benchmark} using the analytical cost
    model.
For each of the expected workloads in Table \ref{tab:expected-workloads},
    we obtain a single nominal tuning configuration (${\configuration_N}$) by
    solving the {\nominal} problem.
For 15 different values of $\rho$ in the range (0.0, 4.0) with a step size of
    0.25, we obtain a set of robust tuning configurations ($\configuration_R$)
    by solving the {\robustw} problem.
Finally, we individually compare each of the robust tunings with the nominal
    over the 10,000 workloads in {\benchmark} to obtain over 2 million
    comparisons.
While computing the costs, we assume that the database contains 10 million
    entries each of size 1 KB.
Analysis presented in the following sections assumes a total available memory of
    10 GB.
In the following sections, for brevity purposes, we present representative
    results  corresponding to individual expected workloads and specific system
    parameters.
However, we exhaustively confirmed that changing these parameters do not
    qualitatively affect the outcomes of our experiment.

\subsection{Results}
\label{sec:model-results}
Here, we present an analysis of the comparisons between the robust and
    the nominal tuning configurations.
Using an off-the-shelf global minimizer from the popular Python optimization
    library SciPy \cite{2020SciPy-NMeth}, we obtain both nominal and robust
    tunings with the runtime for the above experiment being less than 10
    minutes.
% Our code has been made publicly available in order to encourage reproducible
%     research \cite{??}.

\Paragraph{Comparison of Tunings}
First, we address the question -- \textit{is it beneficial to adopt robust tunings
    relative to the nominal tunings?}
Intuitively, it should be clear that the performance of nominally tuned databases
    would degrade when the workloads being executed on the database are
    significantly different from the expected workloads used for tuning.
In Figure~\ref{fig:delta_throughput_workload_type},
    we present performance comparisons between the robust and the nominal
    tunings for different values of uncertainty parameter $\rho$.
We observe that robust tunings provide substantial benefit in terms of normalized
    delta throughput for \emph{unimodal}, \emph{bimodal}, and \emph{trimodal}
    workloads.
The normalized delta throughput
    $\Delta_\obsworkload(\configuration_N, \configuration_R)$
    shows over 95\% improvement on average over all $\obsworkload \in \benchmark$
    for robust tunings with $\rho \geq 0.5$,
    when the expected workload used during tuning belongs to one of these
    categories.
For \emph{uniform} expected workload, we observe that the nominal tuning outperforms the
    robust tuning by a modest 5\%.

Intuitively, \emph{unbalanced} workloads result in overfit nominal tunings.
Hence, even small variations in the observed workload can lead to
    significant degradation in the throughput of such nominally tuned databases.
On the other hand, robust tunings by their very nature take into account such
    variations and comprehensively outperform the nominal tunings.
In case of the uniform expected workload $\workload_0$,
    Figure ~\ref{fig:KL_divergence_histogram} shows us that instances of
    high values of KL-divergence are extremely rare.
In this case, when tuned for high values of $\rho$, the robust tunings are
    unrealistically pessimistic and lose out some performance relative to the
    nominal tuning.

\Paragraph{Impact of Tuning Parameter $\rho$}
Next, we address the question -- \emph{how does the uncertainty tuning parameter
    $\rho$ impact the performance of the robust tunings?}
In Figure~\ref{fig:scatterplot_evolution_rho}, we take a deep dive into
    the performance of robust tunings for an individual
    expected workload for different values of $\rho$.
We observe that the robust tunings for $\rho=0$ i.e., zero uncertainty, are
    very close to the nominal tunings.
As the value of $\rho$ increases,
    its performance advantage over the nominal tuning for the observed
    workloads with higher KL-divergence wrt. expected workload
    increases.
Furthermore, the robustness of such configurations have logically sound
    explanations.
The expected workload in
    Figure~\ref{fig:scatterplot_evolution_rho} consists of just 1\% writes.
Hence, for low values of $\rho$, the robust tuning has higher size-ratio leading to
    shallower LSM trees to achieve good read performance.
For higher values of $\rho$, the robust tunings anticipate increasing
    percentage of write queries and hence limit the size-ratio to achieve overall
    higher throughput.

\begin{figure*}[htbp]
    \centering
    \begin{subfigure}[h]{0.74\textwidth}
        \includegraphics[scale=0.5]{figures/overlapping_histogram.pdf}
        \caption{Histograms of throughput $1 / C(\obsworkload, \Phi)$ for
            tunings with expected workload $\workload_{11}$ }
        \label{fig:overlapping_histogram}
    \end{subfigure}
    \begin{subfigure}[h]{0.24\textwidth}
        \includegraphics[scale=0.5]{figures/throughput_range_evolution.pdf}
        \caption{Throughput range $\Theta_{\mathcal{B}}(\Phi)$}
        \label{fig:throughput_range_evolution}
    \end{subfigure}
    \caption{Impact of $\rho$ on throughput.}
    \label{fig:rho_throughput_range_impact}
\end{figure*}

In Figure~\ref{fig:rho_throughput_range_impact}, we show the impact of tuning
    parameter $\rho$ on the throughput range.
Specifically, in Figure~\ref{fig:overlapping_histogram} we plot a histogram of
    the nominal and robust throughputs for workload $\workload_{11}$.
As the value of $\rho$ increases, the interval size between the lowest and the
    highest throughputs for the robust tunings consistently decreases.
We provide further evidence of this phenomenon in
    Figure~\ref{fig:throughput_range_evolution}, by plotting the decreasing
    throughput range $\Theta_\benchmark(\configuration_R)$
    averaged across all the expected workloads.
Thus, robust tunings not only provide a higher average throughout over all
    $\obsworkload \in \benchmark$,
    but also a more consistent performance with lower variance when compared
    to the nominal tunings.

\Paragraph{Choice of Tuning Parameter $\rho$} Now, we address the question --
\emph{What is the appropriate choice for the value of uncertainty parameter $\rho$?}
We provide guidance on the choice of $\rho$ in absence of perfect knowledge
    regarding the future workloads that are likely to be executed on the
    database.
Intuitively, we expect the robust tunings to be only weak when they are tuned
    for either too little or too much uncertainty.
In Figure~\ref{fig:rho_vs_rho_hat}, we explore the relationship between $\rho$
and the KL-divergence $I_{KL}(\obsworkload, \workload)$ for
    $\obsworkload \in \benchmark$,
    by making a contour plot of the corresponding normalized delta throughput
    $\Delta_{\obsworkload}(\configuration_N, \configuration_R)$.
We confirm our intuition that nominal tunings compare favorably with our
    proposed robust tunings only in two scenarios viz., (1) when observed workloads
    are extremely similar to the expected workload (close to zero observed
    uncertainty),
    and (2) when the robust tunings assume extremely low uncertainty with
    $\rho < 0.2$ while the observed variation in workloads is higher.
Based on this evidence, we can advise a potential database administrator that
    mean KL-divergences between pairs of historically observed workloads would
    be a reasonable value of $\rho$ while deploying robust tunings in practice.

\begin{figure}[h]
    \centering
    \includegraphics[scale=0.25]{figures/rho_vs_rho_hat.pdf}
    \caption{Delta throughputs $\Delta_{\obsworkload}(\Phi_{N},
        \Phi_{R})$ for $\rho$ vs $I_{KL}(\obsworkload, \workload)$.}
    \label{fig:rho_vs_rho_hat}
\end{figure}


\section*{Conclusion}

In conclusion, we devise a new architecture for $C^0$ continuous interpolation, and demonstrate
its efficacy on Dynamic Neural Rendering. Our architecture is able to accurately and quickly
reconstruct scenes, while providing stronger guarantees with a simpler formulation than before.
This hopefully inspires more introduction of classical tools inside of the differentiable rendering pipeline.

\section*{Future Work}

For future work, we plan on applying this architecture to other signals, such as sound for
reconstruction of long duration music. This can be used to generalize over sound, by generating
an embedding from using an encoder-decoder architecture.

Another next step, in the realm of neural rendering, is to encode this using a Plenoxel, so that
rapid training can be done. Since we are storing a fixed number of parameters predicted from a
neural network, this should allow for much more efficient rendering and training, on the order
of 100 times faster, but make it significantly easier to reconstruct dynamic scenes. We note that this is now possible with our formulation, because it is not necessary to query a neural net if the spline is already known, and this can be cached in a sparse voxel structure.

It may also be interesting to explore different formulations of splines, as we only select
Bezier splines due to ease of implementation, but it may be that other splines might be more
numerically stable or have stronger expressivity for reconstruction.


\section{Introduction}
The field at the bottom of the Eigenmath window is for entering
calculations that get evaluated right away.

\begin{center}
\begin{tikzpicture}
\node at (0,0) {\includegraphics[scale=0.2]{face.png}};
\draw[red,thick] (2.3,-1.85) ellipse (2.5cm and 0.5cm);
\end{tikzpicture}
\end{center}

For example, let us check the following
arithmetic from Vladimir Nabokov's autobiography {\it Speak, Memory.}

\begin{quote}
A foolish tutor had explained logarithms to me much too early, and I had
read (in a British publication, the {\it Boy's Own Paper}, I believe)
about a certain Hindu calculator who in exactly two seconds could find the
seventeenth root of, say,
%$3529471145760275132\\301897342055866171392$
% Use math mode so htlatex does not break the number
3529471145760275132301897342\\055866171392
(I am not sure I have got this right; anyway the root was 212).
\end{quote}

We can check Nabokov's arithmetic by entering the following calculation.

\begin{Verbatim}[formatcom=\color{blue}]
212^17
\end{Verbatim}

After pressing the return key, Eigenmath displays the following result.

$3529471145760275132301897342055866171392$

So Nabokov did get it right after all.
Now let us see if Eigenmath can find the
seventeenth root of this number, like the Hindu calculator could.

\begin{Verbatim}[formatcom=\color{blue}]
N = 212^17
N^(1/17)
\end{Verbatim}

Eigenmath displays the following result.

$212$

When a symbol is assigned a value, such as $N$ above,
no result is printed.
To see the value of a symbol, just evaluate it.

\begin{Verbatim}[formatcom=\color{blue}]
N
\end{Verbatim}

$N=3529471145760275132301897342055866171392$

The previous example shows a convention that will be used throughout
this manual.
That is, the color blue indicates something that the user should type.
The computer response is shown in black.