\chapter{}
9不会, 15答案已经给出来了
\begin{ex}
    \rmk{1} this problem reinterprets a nonrandomized test as a randomized one.

    Given a family of probability measures
    \(\mathcal{P}\), consider the \(\phi_{1}(\mathbf{x}):=1_{R}(\mathbf{x}), R \in \mathcal{B}\left(\mathbb{R}^{n}\right)\), called ``test 1". Now consider the following procedure (called ``test 2" ): observe \(\mathbf{x}\), and
    \[\left\{\begin{array}{l}\text { if } \mathbf{x} \in R, \text { flip an independent coin } Y \sim B(1,1) \text { and reject } H_{0} \text { if and only if } Y=1 \\ \text { if } \mathbf{x} \in R^{c}, \text { flip an independent coin } Y \sim B(1,0) \text { and reject } H_{0} \text { if and only if } Y=1\end{array}\right.\]
    Conclude that the tests 1 and 2 are equal \(\mathcal{P}\)-a.s.
\end{ex}

\begin{solution}
    For nonrandomized test (tets 1),
    \[
        \phi(x)=\left\{\begin{matrix}
            1 & \text{if } x \in R \\
            0 & \text{if } x \in R^{c}
        \end{matrix}\right.
    \]
    And for tets 2,
    \[
        \left\{\begin{matrix}
            x\in R, \quad\text{reject } H_0 \text{ with probability } 1; \\
            x\in R^C, \quad\text{reject } H_0 \text{ with probability } 0.
        \end{matrix}\right.
    \]
    Apparently, they are equivalent.
\end{solution}

\begin{ex}
    \rmk{2} in this problem, we calculate the size of a randomized test.

    Assume that we want to decide between the following two distributions for a random variable \(X\) :
    \[
        H_{0}: \mathbb{P}_{0}(X=x)=\left\{\begin{array}{ll}
        0.05, & \text { if } x=2 ; \\
        0.10, & \text { if } x=1 ; \\
        0.85, & \text { if } x=0 .
        \end{array} \quad H_{1}: \mathbb{P}_{1}(X=x)= \begin{cases}0.40, & \text { if } x=2 \\
        0.50, & \text { if } x=1 \\
        0.10, & \text { if } x=0\end{cases}\right.
    \]
    Consider the (randomized) test \((n=1)\)
    \[
        \phi(x)=\left\{\begin{array}{cl}
        1, & \text { if } x=2 \\
        0.5, & \text { if } x=1 \\
        0, & \text { if } x=0
        \end{array}\right.
    \]
    This means that, once we observe \(x\), we flip an independent coin \(Y \sim B(1, \phi(x))\) and reject \(H_{0}\) if and only if \(Y=1\). Prove that \(\phi\) is a size \(0.10\) test, i.e.,
    \[
        \mathbb{E}_{0} \phi(X)=0.10
    \]
    (note that \(H_{1}\) plays no role).
\end{ex}

\begin{solution}
    \begin{align*}
        E_0\phi(X)&=\sum \phi(X) P_0(X=x)\\
        &=0.05+0.05+0\\
        &=0.10
    \end{align*}
\end{solution}

\begin{ex}
    Let \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\mu, \sigma_{0}^{2}\right)\), where \(\sigma_{0}^{2}\) is known. Find the MP level \(\alpha\) test for
    \[
        H_{0}: \mu=\mu_{0} \quad \text { vs } \quad H_{1}: \mu=\mu_{1}
    \]
    when \(\mu_{1}<\mu_{0}\).
\end{ex}

\begin{solution}
    $\bar{X}$ is UMVU for $\mu$, $\bar{X}\sim \mathcal{N}(\mu, \sigma_0^2/n)$.
    \[
        \begin{aligned}
            \frac{p_1(\bar{X})}{p_0\bar{X})}&=\exp\left(-\frac{n}{2\sigma_0^2}\left((\bar{X}-\mu_1)^2-(\bar{X}-\mu_0)^2\right)\right)\\
            &=\exp\left(-\frac{n}{2\sigma_0^2}(2\bar{X}-\mu_0-\mu_1)(\mu_0-\mu_1)\right).
        \end{aligned}
    \]
    This ratio is decreasing  for $\bar{X}$. So, $\frac{p_1(x)}{p_0(x)}>c$ equals to $\bar{X}<c'$.
    \[
        P_0(\bar{X}<c')=\alpha.
    \]
    \[
        P_0\left(\frac{\bar{X}-\mu}{\sigma/\sqrt{n}}<\frac{c'-\mu}{\sigma/\sqrt{n}}\right)=\alpha.
    \]
    \[
        c'=\frac{\sigma}{\sqrt{n}}z_\alpha+\mu_0.
    \]
    Finally, the MP test is: Reject $H_0$ if $\bar{X}<c'$.
\end{solution}

\begin{ex}
    A lot of items contains the proportion \(p\) of defectives. The lot is large enough so that it is reasonable to assume individual items selected at random without replacement are independent Bernoulli trials with parameter \(p\). Suppose \(n\) items are selected at random. We want to test
    \[
        H_{0}: p=0.1 \quad \text { vs } \quad H_{1}: p=0.2
    \]
    Considering only nonrandomized tests, prove that the MP level \(\alpha\) test rejects in the region
    \begin{equation}
        \label{eq:8.4}
        \left\{\mathbf{x}: \sum_{i=1}^{n} x_{i}>c(\alpha)\right\}.
    \end{equation}
    for appropriate values of \(\alpha\) (n.b.: there are only finitely many values of \(\alpha\) such that the nonrandomized test (\ref{eq:8.4}) is \(\mathrm{MP})\). Indicate how one calculates \(c\) as a function of \(\alpha\).
\end{ex}

\begin{solution}
    Because $x_i\sim Ber(p)$, then $X'=\sum x_i\sim B(n,p)$.
    \begin{align*}
        \frac{p_1(X')}{p_0(X')}&=2^x(8/9)^{n-x}.
    \end{align*}
    This ratio is increasing for $X'$. So, $\frac{p_1(x')}{p_0(x')}>c$ equals to $X'>c'$. Then
    \[
        P_0(X'>c')=\alpha;
    \]
    \[
        \sum_{i=c'+1}^{n} \binom{n}{i}p^i(1-p)^{n-i}=\alpha.
    \]
    From this equation, we can calculate \(c'\) as a function of \(\alpha\).
\end{solution}

\begin{ex}
    Let \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \exp (\lambda)\). For \(\lambda_{0}>0\), we want to test
    \[
        H_{0}: \lambda \leq \lambda_{0} \quad \text { vs } \quad H_{1}: \lambda>\lambda_{0} .
    \]
    Find the UMP level \(\alpha\) test.
\end{ex}

\begin{solution}
    \[
        f(x, \lambda)=\lambda^n\exp\left(-\lambda\sum x_i\right),
    \]
    For $\lambda'>\lambda$,
    \[
        \frac{\lambda'^n\exp\left(-\lambda'\sum x_i\right)}{\lambda^n\exp\left(-\lambda\sum x_i\right)}=\left(\frac{\lambda'}{\lambda}\right)^n\exp\left((\lambda-\lambda')\sum x_i\right).
    \]
    It is decreasing for $T(X)=\sum x_i$. So, this $H_0$ has UMP level $\alpha$ test:
    \[
        \phi(X)=\left\{\begin{matrix}
            1, & \text{if } T(X)<c\\
            0, & \text{if } T(X)>c
        \end{matrix}\right.
    \]
    \[
        E_{\lambda_0}(\phi(X))=\alpha.
    \]
    And $M_x(t)=(1-t/\lambda)^{-1}$, $M_{T(X)}(t)=(1-t/\lambda)^{-n}$. $M_{2n\lambda_0T(X)}(t)=(1-2t)^{-2n/2}$. So,
    \[
        2n\lambda_0T(X)\sim \chi^2(2n).
    \]
    \[
        P(2n\lambda_0T(X)<2n\lambda_0c)=\alpha\Rightarrow c=\chi^2_{\alpha}(2n)/(2n\lambda_0).
    \]
\end{solution}

\begin{ex}
    Let \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\mu, \sigma_{0}^{2}\right)\), where \(\sigma_{0}^{2}\) is known. Prove that the family of distributions has a MLR with respect to \(\bar{X}\).
\end{ex}

\begin{solution}
    \[
        f(x, \mu)=\left(\frac{1}{\sqrt{2\pi}\sigma_0}\right)^n\exp\left(-\frac{1}{2\sigma_{0}^{2}}\sum\left(x_i-\mu\right)^2\right)
    \]
    For $\mu'>\mu$,
    \[\begin{aligned}
        \frac{f(x,\mu')}{f(x,\mu)}&=\exp\left(\frac{1}{2\sigma_0^2}(\mu'-\mu)\left(-2\sum x_i+n(\mu'+\mu)\right)\right)\\
        &=\exp\left(\frac{1}{2\sigma_0^2}(\mu'-\mu)\left(-2n\bar{X}+n(\mu'+\mu)\right)\right)
    \end{aligned}
    \]
    For $T(X)=\bar{X}$, the ratio is nonincreasing. So, it has a MLR.
\end{solution}

\begin{ex}
    Let \(\mathcal{P}\) be a one-parameter canonical exponential family generated by \(T(x)\) and \(\mathcal{E}=\operatorname{int} \mathcal{E} \neq \emptyset\). Show that \(\mathcal{P}\) has a MLR with respect to \(T(x)\).
\end{ex}

\begin{solution}
    \[
        f(x, \eta)=h(x)\exp\left(\eta T(x)-A(\eta)\right),
    \]
    For $\eta'>\eta$,
    \[
        \begin{aligned}
            \frac{f(x,\eta')}{f(x,\eta)}&=\exp\left((\eta'-\eta)T(x)+A(\eta)-A(\eta')\right)
        \end{aligned}
    \]
    It is monotone increasing for $T(X)$, hence, it has a MLR.
\end{solution}

\begin{ex}
    Suppose that the assumptions of the Neyman-Pearson lemma hold for hypotheses of the form
    \[
        H_{0}: \theta=\theta_{0} \quad \text { vs } \quad H_{1}: \theta=\theta_{1} .
    \]
    For a given \(\alpha\), let \(\pi\left(\theta_{1}\right)\) be the power of the MP test. We would like to show that \(\alpha<\pi\left(\theta_{1}\right)\).
    \begin{enumerate}[(a)]
        \item Define the test \(\psi(\mathbf{x}) \equiv \alpha\), i.e., irrespective of \(\mathbf{x}\), it rejects \(H_{0}\) with probability \(\alpha\) (remark: this is a randomized test). Conclude that \(\alpha \leq \pi\left(\theta_{1}\right)\).
        \item Assume that \(\alpha=\pi\left(\theta_{1}\right)\). Conclude that \(\psi(\mathbf{x})\) is \(\mathrm{MP}\), and arrive at a contradiction with one of the assumptions of the Neyman-Pearson lemma.
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item \[\alpha=E_0(\psi(x))< E_1(\psi(x))\leqslant\pi(\theta_1). \]
        and $\pi(\theta)=\alpha$. So, $\alpha\leqslant\pi(\theta)$.
        \item If $\alpha=\pi(\theta_1)$, and because $\alpha=\sup_{\theta\in\Theta_0}\pi(\theta)$, which means no other test has large power than $\psi$, i.e., $\psi$ is MP. Then, $E_0(\psi)=E_1(\psi)\Rightarrow p_0=p_1$. Here we get the contradiction.
    \end{enumerate}
\end{solution}

\begin{ex}
    (in this problem, we show the Karlin-Rubin theorem)

    Suppose that the family \(\mathcal{P}=\) \(\{f(\mathbf{x} \mid \theta)\}_{\theta \in \Theta}\) has MLR with respect to \(T(\mathbf{x})\), and we are willing to test the hypothesis
    \[
        H_{0}: \theta \leq \theta_{0} \quad \text { vs } \quad H_{1}: \theta>\theta_{0}.
    \]
    \begin{enumerate}[(a)]
        \item Consider first the simple hypothesis
        \[
        H_{0}: \theta=\theta_{0} \quad \text { vs } \quad H_{1}: \theta=\theta_{1}>\theta_{0}
        \]
        Show that the test that rejects for large values of \(T(\mathbf{x})\) is MP.
        \item For a fixed, pre-chosen \(\alpha\), show that there are \(c\) and \(\gamma\) such that
        \[
        \phi(\mathbf{x})=\left\{\begin{array}{ll}
        1, & \text { if } T(\mathbf{x})>c \\
        0, & \text { if } T(\mathbf{x})<c ; \\
        \gamma, & \text { if } T(\mathbf{x})=c
        \end{array} \quad \mathbb{E}_{\theta_{0}} \phi(\mathbf{X})=\alpha\right.
        \]
        (hint: use the argument in the existence part of the Neyman-Pearson lemma).
        \item Conclude that the test obtained in (b) is also MP for testing \(\theta^{\prime}\) against \(\theta^{\prime \prime}, \theta^{\prime}<\theta^{\prime \prime}\), at level \(\alpha^{\prime}=\pi\left(\theta^{\prime}\right)\).
        \item Show that the power function
        \[
            \pi\left(\theta^{\prime}\right)=\mathbb{E}_{\theta^{\prime}} \phi(\mathbf{X})
        \]
        is strictly increasing (hint: use the previous problem).
        \item Let \(\psi(\mathbf{X})\) be any test, and consider the condition
        \item
        \begin{equation}
            \label{eq:8.9.e.1}
            \mathbb{E}_{\theta} \psi(\mathbf{X}) \leq \alpha, \quad \theta \leq \theta_{0}.
        \end{equation}
        Conclude that \(\psi(\mathbf{X})\) also satisfies
        \begin{equation}
            \label{eq:8.9.e.2}
            \mathbb{E}_{\theta_{0}} \psi(\mathbf{X}) \leq \alpha.
        \end{equation}
        \item Conclude that \(\phi(\mathbf{X})\) satisfies the condition (\ref{eq:8.9.e.1}) and that it is MP (with respect to \(\theta_{1}\) ) in the broader class of tests that satisfy (\ref{eq:8.9.e.2}).
        \item Conclude that \(\phi(\mathbf{X})\) is UMP against \(H_{1}\).

        (hint: use (f) and the fact that \(\phi(\mathbf{X})\) is independent of the particular \(\theta_{1}\) chosen).
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item \includegraphics[width=.9\textwidth]{Proof 1.jpeg}
        \item \includegraphics[width=.9\textwidth]{Proof 2.png}
        \item From NP-lemma existence, it is also MP.
        \item Since $\pi(\theta)$ is nondecreasing
    \end{enumerate}
\end{solution}

\begin{ex}
    Let \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\mu, \sigma^{2}\right)\), where both parameters are unknown. Prove that the GLR tests have the form
    \[
        \begin{array}{ccc}
        \hline H_{0} & H_{1} & R \\
        \hline \mu \leq \mu_{0} & \mu>\mu_{0} & \bar{x}>\mu_{0}+\frac{s}{\sqrt{n}} t_{\alpha} \\
        \mu \geq \mu_{0} & \mu<\mu_{0} & \bar{x}<\mu_{0}-\frac{s}{\sqrt{n}} t_{\alpha} \\
        \mu=\mu_{0} & \mu \neq \mu_{0} & \left|\frac{\bar{x}-\mu_{0}}{s / \sqrt{n}}\right|>t_{\alpha / 2} \\
        \hline
        \end{array}
    \]
\end{ex}

\begin{solution}
    \begin{itemize}
        \item \[
            \mathcal{L}=\left(\frac{1}{\sqrt{2\pi}\sigma}\right)^n \exp\left(-\frac{1}{2\sigma^2}\sum\left(x_i-\mu\right)^2\right).
        \]
        If there is no any restriction for $\mu$, then the MLE for $\mu$ is $\bar{X}$, and the MLE for $\sigma^2$ is $\frac{\sum(x_i-\bar{x})^2}{n}$. However, with $\mu<\mu_0$, the MLE for $\mu$ is $\bar{x}$, when $\bar{x}\leqslant\mu_0$; but when $\bar{x}>\mu_0$, the MLE for $\mu$ is $\mu_0$. Same with the MLE for $\sigma^2$. i.e., when $\bar{x}\leqslant\mu_0$, MLE is $\frac{\sum(x_i-\bar{x})^2}{n}$, and when $\bar{x}>\mu_0$, MLE is $\frac{\sum(x_i-\mu_0)^2}{n}$.

        Noticing that when $\bar{x}\leqslant\mu_0$, $\mathcal{L}_0=\mathcal{L}_1\Rightarrow\frac{\mathcal{L}_0}{\mathcal{L}_1}=1$. And in this case, we always accept $H_0$. When $\bar{x}>\mu_0$, the reject region is:
        \begin{align*}
            \frac{\mathcal{L}_0}{\mathcal{L}_1}&=\left(\frac{\sum(x_i-\bar{x})^2}{\sum(x_i-\mu_0)^2}\right)^{n/2}<c\\
            &\Rightarrow  \frac{\sum(x_i-\mu_0)^2}{\sum(x_i-\bar{x})^2}>c_1\\
            &\Rightarrow  \frac{\sum(x_i-\bar{x})^2+n(\bar{x}-\mu_0)^2}{\sum(x_i-\bar{x})^2}>c_2\\
            &\Rightarrow  \frac{n(\bar{x}-\mu_0)^2}{\sum(x_i-\bar{x})^2}>c_3\\
            &\Rightarrow  \sqrt{\frac{(\bar{x}-\mu_0)^2}{\sum(x_i-\bar{x})^2/n}}>c_4\\
            &\Rightarrow  \frac{\bar{x}-\mu_0}{S/\sqrt{n}}>c_5.
        \end{align*}
        where $\frac{\bar{x}-\mu_0}{S/\sqrt{n}}\sim t(n-1)$.
        So, the Reject Region is:
        \[
            \frac{\bar{x}-\mu_0}{S/\sqrt{n}}>t_{1-\alpha}(n-1)\Rightarrow\bar{x}>\mu_0+\frac{S}{\sqrt{n}} t_{1-\alpha}(n-1).
        \]
        \item For $H_0: \mu\geqslant \mu_0$, we just need to reverse the inequality sign, and get the Reject Region:
        \[
            \frac{\bar{x}-\mu_0}{S/\sqrt{n}}<t_{\alpha}(n-1)\Rightarrow\bar{x}<\mu_0+\frac{S}{\sqrt{n}} t_{\alpha}(n-1).
        \]
        \item Same, the Reject Region is:
        \[
            \left|\frac{\bar{x}-\mu_0}{S/\sqrt{n}}\right|>t_{1-\alpha/2}(n-1).
        \]
    \end{itemize}
\end{solution}

\begin{ex}
    Let \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\mu, \sigma^{2}\right)\), where both parameters are unknown. Prove that the GLR tests have the form
    \begin{center}
        \begin{tabular}{ccc}
            \hline\(H_{0}\) & \(H_{1}\) & \(R\) \\
            \hline\(\sigma^{2} \leq \sigma_{0}^{2}\) & \(\sigma^{2}>\sigma_{0}^{2}\) & \(\frac{(n-1) s^{2}}{\sigma_{0}^{2}}>\chi_{n-1}^{2}(\alpha)\) \\
            \(\sigma^{2} \geq \sigma_{0}^{2}\) & \(\sigma^{2}<\sigma_{0}^{2}\) & \(\frac{(n-1) s^{2}}{\sigma_{0}^{2}}<\chi_{n-1}^{2}(1-\alpha)\) \\
            \(\sigma^{2}=\sigma_{0}^{2}\) & \(\sigma^{2} \neq \sigma_{0}^{2}\) & \(\left\{\frac{(n-1) s^{2}}{\sigma_{0}^{2}}<\chi_{n-1}^{2}(1-\alpha / 2)\right\} \cup\left\{\frac{(n-1) s^{2}}{\sigma_{0}^{2}}>\chi_{n-1}^{2}(\alpha / 2)\right\}\) \\
            \hline
        \end{tabular}
    \end{center}
\end{ex}

\begin{solution}
    \begin{itemize}
        \item \[
            \mathcal{L}=\left(\frac{1}{\sqrt{2\pi}\sigma}\right)^n \exp\left(-\frac{1}{2\sigma^2}\sum\left(x_i-\mu\right)^2\right).
        \]
        When $\frac{\sum(x_i-\bar{x})^2}{n}>\sigma_0^2$,
        \begin{align*}
            \frac{\mathcal{L}_0}{\mathcal{L}_1}&=\left(\frac{\sigma^2}{\sigma_0^2}\right)^{n/2}\exp\left(-\frac{n}{2}\frac{\sigma^2}{\sigma_0^2}\right).
        \end{align*}
        This ratio first increase before $\sigma^2/\sigma_0^2=1$, then decrease. So, the Reject Region is:
        \[
            \sigma^2/\sigma_0^2<c_1<1, \text{ and } \sigma^2/\sigma_0^2>c_2>1.
        \]
        We accept $H_0$ when $\sigma^2/\sigma_0^2<1$,
        \[
            \begin{aligned}
                &\sigma^2/\sigma_0^2>c_2\\
                \Rightarrow &\frac{\sum(x_i-\bar{x})^2}{n\sigma_0^2}>c_2\\
                \Rightarrow &\frac{(n-1)s^2}{\sigma_0^2}>c_3.
            \end{aligned}
        \]
        And $\frac{(n-1)s^2}{\sigma_0^2}\sim\chi^2(n-1)$. So, the Reject Region is:
        \[
            \frac{(n-1)s^2}{\sigma_0^2}>\chi^2_{1-\alpha}(n-1).
        \]
        \item Same, the Reject Region is:
        \[
            \frac{(n-1)s^2}{\sigma_0^2}<\chi^2_{\alpha}(n-1).
        \]
        \item The Reject Region is:
        \[
            \left\{\frac{(n-1)s^2}{\sigma_0^2}<\chi^2_{\alpha/2}(n-1)\right\}\bigcup\left\{\frac{(n-1)s^2}{\sigma_0^2}>\chi^2_{1-\alpha/2}(n-1)\right\}
        \]
    \end{itemize}
\end{solution}

\begin{ex}
    Suppose we have two independent samples: \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \exp \left(\lambda_{1}\right), Y_{1}, \ldots, Y_{m} \stackrel{\text { i.i.d. }}{\sim} \exp \left(\lambda_{2}\right)\).
    \begin{enumerate}[(a)]
        \item Find the GLR test for the hypotheses
        \[
        H_{0}: \lambda_{1}=\lambda_{2} \quad \text { vs } \quad H_{1}: \lambda_{1} \neq \lambda_{2}.
        \]
        (remark: note that the definition of GLR tests does encompass this two-sample case).
        \item Show that the test in part (a) can be based on the statistic
        \[
        T(\mathbf{X}, \mathbf{Y})=\frac{\sum_{i=1}^{n} X_{i}}{\sum_{i=1}^{n} X_{i}+\sum_{j=1}^{m} Y_{j}}.
        \]
        \item Find the distribution of \(T(\mathbf{X}, \mathbf{Y})\) when \(H_{0}\) is true.
    \end{enumerate}
    (hint: for (b), express \(\Lambda(\mathbf{x}, \mathbf{y})\) as a function of \(T(\mathbf{x}, \mathbf{y})\), and note that \(\Lambda(\mathbf{x}, \mathbf{y})\) is unimodal with respect to \(T(\mathbf{x}, \mathbf{y})\). Now conclude that \(\Lambda(\mathbf{x}, \mathbf{y})\) is small if and only if \(T(\mathbf{x}, \mathbf{y})\) is small or large. For (c), \(\sum_{i} X_{i}\) and \(\sum_{j} Y_{j}\) follow Gamma distributions; there is a relation between Gamma and Beta random variables).
\end{ex}

\begin{solution}
        \[
            \mathcal{L}(\lambda_1, \lambda_2)=\lambda_1^n\lambda_2^m\exp\left(-\lambda_1\sum^nX_i-\lambda_2\sum^mY_i\right),
        \]
        $\hat{\lambda}_1=\frac{1}{\bar{X}}$, $\hat{\lambda}=\frac{n+m}{n\bar{X}+m\bar{Y}}$,
        \[
            \frac{\mathcal{L}_0}{\mathcal{L}_1}=\frac{(n+m)^{n+m}}{n^nm^m}\frac{(n\bar{X})^n(m\bar{Y})^m}{(n\bar{X}+m\bar{Y})^{n+m}}.
        \]
        Let $T(X)=n\bar{x}/(n\bar{x}+m\bar{y})$,
        \[
            \begin{aligned}
                \frac{\mathcal{L}_0}{\mathcal{L}_1}<c&\Rightarrow \frac{(n+m)^{n+m}}{n^nm^m}\frac{(n\bar{X})^n(m\bar{Y})^m}{(n\bar{X}+m\bar{Y})^{n+m}}<c\\
                &\Rightarrow  T(X)^m(1-T(X))^n<c_1.
            \end{aligned}
        \]
        this function of $T(X)$ will increase before $T(X)=m/(m+n)$, then. after $m/(m+n)$, it will decrease. So, the Reject Region is:
        \[
            T(X)<c_l, \text{ and } T(X)>c_r.
        \]
        $X\sim Exp(\lambda_1)$, $2n\lambda\bar{X}\sim \chi^2(2n)$. \[
            \frac{n\bar{x}}{n\bar{x}+m\bar{y}}=\frac{2\lambda n\bar{x}}{2\lambda n\bar{x}+2\lambda m\bar{y}}\sim\frac{\chi^2(2n)}{\chi^2(2n+2m)}=F(2n,2m+2n).
        \]
        So, the Reject Region is:
        \[
            \left\{T(X)<F_{\alpha/2}(2n,2m+2n)\right\} \bigcup \left\{T(X)>F_{1-\alpha/2}(2n,2m+2n)\right\}
        \]
\end{solution}

\begin{ex}
    Let \(X_{1}, \ldots, X_{n} \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\mu_{1}, \sigma^{2}\right)\) and \(Y_{1}, \ldots, Y_{m} \stackrel{\text { i.i.d. }}{\sim} \mathcal{N}\left(\mu_{2}, \sigma^{2}\right)\) be two independent samples. We are interested in testing
    \[
        H_{0}: \mu_{1}=\mu_{2} \quad \text { vs } \quad H_{0}: \mu_{1} \neq \mu_{2} .
    \]
    \begin{enumerate}[(a)]
        \item Derive the GLR test. Show that it can be based on the statistic
        \[
            T(\mathbf{x}, \mathbf{y})=\frac{\bar{X}-\bar{Y}}{\sqrt{S_{p}^{2}\left(\frac{1}{n}+\frac{1}{m}\right)}}, \quad S_{p}^{2}=\frac{1}{n+m-2}\left(\sum_{i=1}^{n}\left(X_{i}-\bar{X}\right)^{2}+\sum_{j=1}^{m}\left(Y_{j}-\bar{Y}\right)^{2}\right).
        \]
        \item Show that
        \[
            T(\mathbf{x}, \mathbf{y}) \sim t_{n+m-2} \quad \text { under } H_{0} .
        \]
    \end{enumerate}
    In other words, the GLR test is equivalent to the so-called two-sample \(t\) test (remark: the case where the variances from the two samples are potentially distinct is called the Behrens-Fisher problem, and it is surprisingly intricate).
\end{ex}

\begin{solution}
    If $H_0$ is true:
    $\hat{\mu}=(\sum^nx_i+\sum^my_i)/(m+n)$, $\hat{\sigma}^2=(\sum^n(x_i-\mu)^2+\sum^m(y_i-\mu)^2)/(m+n)$.

    \noindent If $\mu_1\neq\mu_2$, then $\hat{\mu}_1=\bar{X}$, $\hat{\mu}_2=\bar{Y}$, $\hat{\sigma}^2=(\sum^n(x_i-\mu_1)^2+\sum^m(y_i-\mu_2)^2)/(m+n)$.
    \[
        \frac{\mathcal{L}_0}{\mathcal{L}_1}=\left(1+\frac{(T(x,y))^2}{m+n-2}\right)^{(m+n)/2}.
    \]
    这个式子没算出来, 网上找的结果.
    \[
        \bar{X}\sim N(\mu_1, \sigma^2/n),
        \bar{Y}\sim N(\mu_2, \sigma^2/m), \bar{X}-\bar{Y}\sim N(\mu_1-\mu_2, \sigma^2(1/n+1/m)),
    \]
    \[
        \frac{\bar{X}-\bar{Y}}{S\sqrt{\frac{1}{n}+\frac{1}{m}}}\sim t(n+m-2), \quad S^2=\frac{1}{n+m-2}\left(\sum_{i=1}^{n}\left(X_{i}-\bar{X}\right)^{2}+\sum_{j=1}^{m}\left(Y_{j}-\bar{Y}\right)^{2}\right).
    \]
\end{solution}

\begin{ex}
    Consider the hypotheses
    \(H_{0}: f_{0}(\mathbf{x})\) is true vs \(H_{1}: f_{1}(\mathbf{x})\) is true,
    where \(f_{j}(\mathbf{x}), j=0,1\), are generalized densities with respect to some \(\sigma\)-finite measure \(G(d \mathbf{x})\) and satisfying the identifiability condition \(G\left(\left\{\mathbf{x}: f_{1}(\mathbf{x}) \neq f_{0}(\mathbf{x})\right\}\right)>0\) (cf. the NeymanPearson lemma). Assume the distribution of the likelihood ratio statistic \(\frac{f_{1}(\mathbf{X})}{f_{0}(\mathbf{X})}\) is continuous under \(H_{0}\) (i.e., the c.d.f. of \(\frac{f_{1}(\mathbf{X})}{f_{0}(\mathbf{X})}\) is continuous). Show that the rejection regions of the MP test are nested, i.e.,
    \[
        \alpha<\alpha^{\prime} \Rightarrow R_{\alpha} \subseteq R_{\alpha^{\prime}}.
    \]
\end{ex}

\begin{solution}
    From NP-lemma, for  $\alpha<\alpha'$, there are $c$, $c'$, such that the critical region are $f_1/f_0>c$, $f_1/f_0>c'$ respectively. And
    \[
        \alpha=P_0(f_1/f_0>c)<P_0(f_1/f_0>c')=\alpha'.
    \]
    So ,$c>c'$. And $\{x:f_1/f_0>c\}\subset\{x:f_1/f_0>c'\}$. i.e., $R_\alpha\subset R_{\alpha'}$.
\end{solution}

\begin{ex}
    Consider the hypotheses
    \[
        H_{0}: \theta \in \Theta_{0} \quad \text { vs } \quad H_{1}: \theta \in \Theta_{1},
    \]
    and assume the rejections regions \(\left\{R_{\alpha}\right\}_{\alpha \in(0,1)}\) for a given (family of) tests are nested.
    \begin{enumerate}[(a)]
        \item if \(\sup _{\theta \in \Theta_{0}} \mathbb{P}_{\theta}\left(X \in R_{\alpha}\right) \leq \alpha, \alpha \in(0,1)\), show that
        \[
            \mathbb{P}_{\theta}(\widehat{p} \leq u) \leq u, \quad 0 \leq u \leq 1, \quad \theta \in \Theta_{0}.
        \]
        \item if \(\mathbb{P}_{\theta}\left(X \in R_{\alpha}\right)=\alpha, \alpha \in(0,1)\), show that
        \[
            \mathbb{P}_{\theta}(\widehat{p} \leq u)=u, \quad 0 \leq u \leq 1, \quad \theta \in \Theta_{0}.
        \]
    \end{enumerate}
    {\bfseries Suggestion: }

    (a) note that \(\{\widehat{p} \leq u\} \subseteq\left\{X \in R_{v}\right\}, v>u\), by the assumption of "nestedness" (interpretation: if \(\widehat{p}\) is small with respect to \(u\), then \(X\) must be in all rejection regions at a level \(v>u\). Observe that this is not necessarily true for \(\left\{X \in R_{u}\right\}\). Consequently,
    \[
    \mathbb{P}_{\theta}(\widehat{p} \leq u) \leq \mathbb{P}_{\theta}\left(X \in R_{v}\right) \leq v
    \]
    and now take \(v \downarrow u\) and \(\sup _{\theta \in \Theta_{0}}\).

    (b) note that \(\left\{X \in R_{v}\right\} \subseteq\{\widehat{p} \leq u\}\). Now take \(\mathbb{P}_{\theta}\) and use (a) to conclude that equality holds.
\end{ex}

\begin{ex}
    \rmk{3} in this problem, we build a MP test with non-nested rejection regions.

    Consider the following two distributions.
    \begin{center}
        \begin{tabular}{cccc}
            \hline\(x\) & 1 & 2 & 3 \\
            \hline\(f_{0}(x)\) & \(0.85\) & \(0.10\) & \(0.05\) \\
            \(f_{1}(x)\) & \(0.70\) & \(0.20\) & \(0.10\) \\
            \hline\(f_{1}(x) / f_{0}(x)\) & \(0.82\) & 2 & 2 \\
            \hline
        \end{tabular}
    \end{center}
    \begin{enumerate}[(a)]
        \item Just to fix ideas, use the argument for existence in the Neyman-Pearson lemma to construct a randomized test \(\phi\) of size \(\alpha=0.10\) (hint: choose \(k=2\) ).
        \item Now define the test
        \[
            \psi(x)=\left\{\begin{array}{lc}
            1, & \text { if } x=2 \\
            0, & \text { if } x=1,3
            \end{array}\right.
        \]
        Conclude that \(\psi\) has size \(\alpha=0.10\). Use the Neyman-Pearson lemma to conclude that \(\psi\) is MP (n.b.: as expected, \(\psi\) and \(\phi\) do agree in the region \(\left\{x: f_{1}(x) \neq 2 f_{0}(x)\right\}\).
        \item We now redefine \(\psi(x)\) at a different size, namely, \(\alpha=0.05\) (n.b.: rejection regions do change as a function of \(\alpha\) ). Let
        \[
            \psi(x)=\left\{\begin{array}{lc}
            1, & \text { if } x=3 \\
            0, & \text { if } x=1,2
            \end{array}\right.
        \]
        Conclude that \(\psi\) has size \(\alpha=0.05\), indeed. Use the Neyman-Pearson lemma to conclude that \(\psi\) is MP.
        \item Conclude that \(R_{0.05} \nsubseteq R_{0.10}\).
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item \[
            \psi(x)=\left\{\begin{matrix}
            1, & \text { if } x=3 \\
            \gamma, & \text { if } x=2 \\
            0, & \text { if } x=1
            \end{matrix}\right.
        \]
        \[
            E_0(\psi)=0.05+0.10\gamma=0.1 \Rightarrow \gamma=0.5.
        \]
        \item $E_0(\psi)=0.1$. Let $k=2$, if $x=2$, $\lambda=\frac{L_1}{L_0}=2\geqslant k$. If $x\neq2$, $\lambda=\frac{L_1}{L_0}\leqslant k$. So, $\psi$ is MP.
        \item Same, let $k=2$, $\psi$ is MP for $\alpha=0.05$.
        \item $R_{0.05}=\{3\}\neq\{2\}=R_{0.1}$.
    \end{enumerate}
\end{solution}


\hypertarget{group__numpp__krylov}{}\section{Krylov Subspace methods}
\label{group__numpp__krylov}\index{Krylov Subspace methods@{Krylov Subspace methods}}

Group of methods used to iteratively solve linear equations of form $ Ax = b$.

\begin{DoxyCompactItemize}
\item
{\footnotesize template$<$typename T , std\+::size\+\_\+t Size$>$ }\\C\+O\+N\+S\+T\+E\+X\+PR auto \hyperlink{group__numpp__krylov_ga5e105c0002fbd19b7c8144540040550f}{numpp\+::krylov\+::conjugate\+\_\+gradient} (const \hyperlink{classnumpp_1_1matrix_1_1dense}{matrix\+::dense}$<$ T, Size, Size $>$ \&A, const \hyperlink{classnumpp_1_1vector}{vector}$<$ T, Size $>$ \&x, const \hyperlink{classnumpp_1_1vector}{vector}$<$ T, Size $>$ \&b, const double threshold=0.\+01, const std\+::size\+\_\+t iterations=Size $>$ 20 ? Size \+:20)
\end{DoxyCompactItemize}

\subsection{Detailed Description}
Group of methods used to iteratively solve linear equations of form $ Ax = b$.

\subsection{Function Documentation}
\mbox{\Hypertarget{group__numpp__krylov_ga5e105c0002fbd19b7c8144540040550f}\label{group__numpp__krylov_ga5e105c0002fbd19b7c8144540040550f}}
\index{Krylov Subspace methods@{Krylov Subspace methods}!conjugate\+\_\+gradient@{conjugate\+\_\+gradient}}
\index{conjugate\+\_\+gradient@{conjugate\+\_\+gradient}!Krylov Subspace methods@{Krylov Subspace methods}}
\subsubsection{\texorpdfstring{conjugate\+\_\+gradient()}{conjugate\_gradient()}}
{\footnotesize\ttfamily template$<$typename T , std\+::size\+\_\+t Size$>$ \\
C\+O\+N\+S\+T\+E\+X\+PR auto numpp\+::krylov\+::conjugate\+\_\+gradient (\begin{DoxyParamCaption}\item[{const \hyperlink{classnumpp_1_1matrix_1_1dense}{matrix\+::dense}$<$ T, Size, Size $>$ \&}]{A,  }\item[{const \hyperlink{classnumpp_1_1vector}{vector}$<$ T, Size $>$ \&}]{x,  }\item[{const \hyperlink{classnumpp_1_1vector}{vector}$<$ T, Size $>$ \&}]{b,  }\item[{const double}]{threshold = {\ttfamily 0.01},  }\item[{const std\+::size\+\_\+t}]{iterations = {\ttfamily Size},  }\item[{20 ? Size \+:20}]{ }\end{DoxyParamCaption})}

Calculates solution to linear equation $ Ax = b $ via conjugate gradient

\begin{DoxyTemplParams}{Template Parameters}
{\em T} & Argument type of the matrix (e.\+g. double) \\
\hline
{\em Size} & size of matrix row and column (have to be equal)\\
\hline
\end{DoxyTemplParams}

\begin{DoxyParams}{Parameters}
{\em A} & matrix A of the method {\bfseries }\\
\hline
\end{DoxyParams}
\begin{DoxyWarning}{Warning}
{\bfseries  Matrix A H\+AS TO be symmetric, it\textquotesingle{}s not enforced in anyway in this method}
\end{DoxyWarning}

\begin{DoxyParams}{Parameters}
{\em x} & Vector containing an initial guess of the solution (solution won\textquotesingle{}t be placed here!). {\bfseries }\\
\hline
\end{DoxyParams}
\begin{DoxyWarning}{Warning}
{\bfseries  If you are unsure about this parameter go with vector filled with 1\textquotesingle{}s.}
\end{DoxyWarning}

\begin{DoxyParams}{Parameters}
{\em b} & Solution vector of the linear equations\\
\hline
{\em threshold} & Tolerance of the algorithm. If the error (as an euclidean norm of residual)~\newline
is smaller than this value, the algorithm will stop\\
\hline
{\em iterations} & Maximum number of iterations performed by the algorithm\\
\hline
\end{DoxyParams}
Conjugate gradient is one of the algorithms from the krylov subspace method.

It may allow us to solve linear equations of the form $Ax = b$ in a more efficient manner~\newline
than popular direct methods like LU, Cholesky or similiar.

For exact use cases consult the professional literature.

\begin{DoxyReturn}{Returns}
vector of type identical to x filled with an answer
\end{DoxyReturn}


\chapter{Observational equality}

\section{Observational equality on $\N$}

\begin{defn}
A \define{(type valued) binary relation} on a type $A$ in context $\Gamma$ is a type family
\begin{equation*}
\Gamma,x:A,y:A\vdash R(x,y)~\mathrm{type}.
\end{equation*}
A \define{small binary relation} on a type $A$ is a term of type
\begin{equation*}
A\to (A\to\UU)
\end{equation*}
\end{defn}


\documentclass{beamer}

\usepackage{verbatim}
\usepackage{fancyvrb}
\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{booktabs}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{calc}
\usepackage{color}
\usepackage{multicol}
\usepackage{wrapfig}
\usepackage{natbib}
\usepackage[ruled,vlined]{algorithm2e}
\usepackage{animate}
\usepackage{mathtools}
\usepackage{listings}
\usepackage{xfrac}

% two col: two columns
\newenvironment{twocol}[4]{
\begin{columns}[c]
\column{#1\textwidth}
#3
\column{#2\textwidth}
#4
\end{columns}
}

\makeatletter
\setbeamertemplate{theorem begin}
{%
\begin{\inserttheoremblockenv}
  {}{\usebeamerfont*{block title}\usebeamercolor[fg]{block title}%
  \inserttheoremname
  %\inserttheoremnumber
  \ifx \inserttheoremaddition \empty \else\ (\inserttheoremaddition)\fi
  \inserttheorempunctuation}
  \normalfont
  }
  \setbeamertemplate{theorem end}{\end{\inserttheoremblockenv}}
\makeatother

\newcommand{\E}{\mathrm{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\s}{\mathrm{s}}
\newcommand{\Corr}{\mathrm{Corr}}
\newcommand{\rank}{\mathrm{rank}}
\newcommand{\nullspace}{\mathrm{null}}
\newcommand{\myspan}{\mathrm{span}}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\softmax}{softmax}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\newtheorem{prop}[theorem]{Proposition}
\newtheorem{exe}{Exercise}
\newtheorem{remark}{Remark}
\newtheorem{myproof}{Proof}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\title{Diagnostics and Remedial Measures}
\author{Zhenisbek Assylbekov}
\institute{Department of Mathematics}
\date{Regression Analysis}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\section{Diagnostics for Predictor Variable}

\begin{frame}{Diagnostics for Predictor Variable}
Goal: identify any outlying values that could affect the appropriateness of the linear model.
\vspace{10pt}

\pause Two main issues:
\begin{itemize}
    \item \pause Outliers (recall Extrapolation)
    \item \pause Dependence b/w $x_i$ and $i$
\end{itemize}
\vspace{10pt}

\pause To check these we use
\begin{itemize}
    \item Histogram and/or Boxplot
    \item Sequence plot
\end{itemize}
\end{frame}

\begin{frame}{Example}
{\color{blue}\url{https://raw.githubusercontent.com/zh3nis/MATH440/main/chp03/diag_x.R}}
\begin{center}
\includegraphics[width=.65\textwidth]{plots/hist_bp_x.pdf}\\
\includegraphics[width=.65\textwidth]{plots/seq_x.pdf}
\end{center}
\end{frame}

\section{Checking Assumptions}
\begin{frame}{Checking Assumptions}
Recall the SLR model:
\begin{align*}
&Y_i=\beta_0+\beta_1 x_i+\epsilon_i,\qquad i=1,\ldots,n\\
&\epsilon_i\,\,{\stackrel{\text{iid}}{\sim}}\,\,\mathcal{N}(0,\sigma^2)
\end{align*}
\pause After the model is fit, but \textit{before} any inference or conclusions are made, the assumptions of the model need to be checked.

\vspace{10pt}
\pause Assumptions:
\begin{enumerate}
    \item\pause Normality
    \item\pause Homogeneity of variance
    \item\pause Linearity
    \item\pause Independence
\end{enumerate}
\vspace{10pt}
\pause The assumptions are checked using the residuals $e_i:=y_i-\hat{y}_i$. This process is sometimes referred to as \textbf{residual analysis}.
\end{frame}

\subsection{Graphical Methods}
\begin{frame}[fragile]{Graphical Methods: Normality}{Normal Q-Q Plot}
\begin{itemize}
    \item The \textbf{empirical c.d.f.} for a r.s. $Y_1,\ldots,Y_n$ is given by $\hat{F}(y)=\frac{\#(Y_i\le y)}{n}=\frac1n\sum_i\mathbb{I}[Y_i\le y]$.
    \item \pause To compare an empirical distribution $\hat{F}(y)$ against some theoretical distribution $G(y)$, we can scatterplot quantiles $\hat{F}^{-1}(p)$ vs $G^{-1}(p)$ for $p\in[0,1]$. \pause This is called \textbf{Q-Q plot}.
\end{itemize}
\pause
\begin{columns}
\begin{column}{.5\textwidth}
Comparing the distribution of poverty rates vs normal distribution using a Q-Q Plot.\\~\\
\begin{footnotesize}
{\color{blue}\url{https://raw.githubusercontent.com/zh3nis/MATH440/main/chp03/qq.R}}
\end{footnotesize}
\end{column}
\begin{column}{.5\textwidth}
\includegraphics[width=\textwidth]{plots/qq_pov.pdf}
\end{column}
\end{columns}
\end{frame}

\begin{frame}{Graphical Methods: Normality}{Q-Q Plot for Residuals}

Recall, that by SLR assumtions, $\epsilon_i\,\,{\stackrel{\text{iid}}{\sim}}\,\,\mathcal{N}(0,\sigma^2)$. \pause In addition to checking normality of $y_i$'s, we should also check normality of the residuals $e_i$ once the model is fit.
\pause
\includegraphics[width=\textwidth]{plots/res_qq_hist.pdf}\\
\footnotesize{\color{blue}\url{raw.githubusercontent.com/zh3nis/MATH440/main/chp03/res_plots.R}}
\end{frame}

\begin{frame}{Graphical Methods: Homogeneity of Variance / Linearity}
\begin{itemize}
    \item SLR assumes that $\Var[\epsilon_i]=\sigma^2$ is constant.
    \item<2->In order to check this assumption a plot of the residuals ($e_i$) versus the fitted values ($\hat{y}_i$) is used.
    \item<3->We expect to see a constant spread/distance of the residuals to the 0 line across all the $\hat{y}_i$ values.

\onslide<4->{\centerline{\includegraphics[width=.8\textwidth]{plots/res_fit.pdf}}}
    \item<5->The same plot can be used to check the linearity assumption. \onslide<6->{If the linear model is a good fit, once expects to see the residuals evenly spread on either side of the 0 line.}
\end{itemize}
\end{frame}

\begin{frame}{Graphical Methods: Homogeneity of Variance / Linearity}{Examples of Violations}
\begin{columns}
\pause\begin{column}{.5\textwidth}
\begin{center}
Non-constant variance\\~\\

\includegraphics[width=.9\textwidth]{plots/heteroscedastic.pdf}
\end{center}
\end{column}
\pause\begin{column}{.5\textwidth}
\begin{center}
Non-linear relationship\\~\\

\includegraphics[width=.9\textwidth]{plots/nonlinear.pdf}
\end{center}
\end{column}
\end{columns}
\end{frame}

\begin{frame}{Graphical Methods: Independence}{Time Series Plot of the Residuals}
\begin{itemize}
    \item To check for independence b/w $\epsilon_i$'s, we plot $e_i$ vs $i$.
    \item\pause Independence is graphically checked if there is no discernible pattern in the
plot. \pause I.e., one cannot predict $e_i$ from $e_{<i}$.
\end{itemize}
\pause\begin{center}
    \includegraphics[width=.9\textwidth]{plots/res_ts.pdf}
\end{center}
\vspace{-10pt}
\footnotesize{\color{blue}\url{raw.githubusercontent.com/zh3nis/MATH440/main/chp03/res_plots.R}}
\end{frame}

\subsection{Significance Tests}
\begin{frame}{Runs Test for Independence}
\begin{itemize}
    \item Write out the sequence of $+/-$ signs of the residuals
    \item\pause Count $n_1=\#[e_i\ge0]$, $n_2=\#[e_i<0]$
    \item\pause Count $u=\#\text{runs}$ of positive and negative residuals. \pause What is a \textit{run}? \pause E.g., if we have the following 9 residuals:
    $$
    \underbrace{-}_{1}\underbrace{+++}_{2}\underbrace{--}_{3}\underbrace{+}_{4}\underbrace{--}_{5}
    $$
    then w have $u=5$ runs with $n_1=4$ and $n_2=5$.
    \item\pause Under $\mathrm{H}_0:\,e_i\text{'s are independent}$, the pmf of the r.v. $U$ is
    $$
    p(u)=\begin{cases}
    \frac{2{{n_1-1}\choose{k-1}}{{n_2-1}\choose{k-1}}}{{n_1+n_2\choose n_1}}, &u=2k,\,k\in\mathbb{N}\\
    \frac{{n_1-1\choose k}{n_2-1\choose k-1}+{n_2-1\choose k}{n_1-1\choose k-1}}{{n_1+n_2\choose n_1}},&u=2k+1,\,k\in\mathbb{N}
    \end{cases}
    $$
    \pause And $\text{p-value}=\Pr(U\le u)$. No need to do by hand.
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Runs Test in \texttt{R}}

\begin{small}
\begin{verbatim}
> library(lawstat)
> poverty = read.table("path/to/poverty.txt", h=T, sep="\t")
> my_model = lm(Poverty ~ Graduates, data=poverty)
> re = my_model$residuals
> runs.test(re, plot.it=TRUE)

	Runs Test - Two sided

data:  re
Standardized Runs Statistic = -0.13873, p-value = 0.8897
\end{verbatim}
\end{small}

\centerline{\includegraphics[width=.7\textwidth]{plots/runs.pdf}}

{\color{blue}\url{https://raw.githubusercontent.com/zh3nis/MATH440/main/chp03/runs_test.R}}
\end{frame}

\begin{frame}{Shapiro-Wilk Test for Normality}
\begin{itemize}
    \item $\mathrm{H}_0:\,e_1,\ldots,e_n\sim\mathcal{N}$
    \item<2->Test statistic
    $$
    W=\frac{\left(\sum_{i=1}^n a_i e_{(i)}\right)^2}{\sum_{i=1}^n(e_i-\bar{e})^2}
    $$
    where
    \begin{itemize}
        \item<3-> $e_{(i)}$ is the $i$th smallest number in the sample
        \item<4-> $(a_1,\ldots,a_n)^\top=\mathbf{a}=\frac{\mathbf{V}^{-1}\mathbf{m}}{\|\mathbf{V}^{-1}\mathbf{m}\|}$
        \item<5->$\mathbf{m}=(m_1,\ldots,m_n)^\top$ is a vector with $m_i=\E[Z_{(i)}]$ where $Z_1,\ldots,Z_n\,\,{\stackrel{\text{iid}}{\sim}}\,\,\mathcal{N}(0,1)$
        \item<6->$\mathbf{V}=\{\Cov[Z_{(i)},Z_{(j)}]\}$
    \end{itemize}
    \item<7->No name for the distribution of $W$ under $\mathrm{H}_0$. Its critical values are calculated through Monte-Carlo simulations.
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Shapiro-Wilk Test in \texttt{R}}
\begin{verbatim}
> poverty = read.table("path/to/poverty.txt",
                       h=T, sep="\t")
> my_model = lm(Poverty ~ Graduates, data=poverty)
> re = my_model$residuals
> shapiro.test(re)

	Shapiro-Wilk normality test

data:  re
W = 0.96804, p-value = 0.1831
\end{verbatim}
\pause Hence, FTR normality of the residuals in the Graduation--Poverty example.
\end{frame}

\begin{frame}{Levene's Test for Homogeneity of Variance}
\begin{itemize}
    \item If the response can be split into $t$ distinct groups, the we use Levene's Test for
    $\mathrm{H}_0:\,\sigma^2_1=\ldots=\sigma^2_t$
    \item<2-> If the response is numerical (which is what we have in regression), we can artificially split responses into groups bases on predictor values.
    \item<3-> Test statistic is tedious to calculate and left for software. However,
    $$
    \text{T.S.}\,\,{\stackrel{\mathrm{H}_0}{\sim}}\,\,F_{t-1,n-t}
    $$
    where $t=\#(\text{groups})$, and $n=\#(\text{observations})$.
    \item<4->The $\text{p-value}=\Pr(F_{t-1,n-t}\ge \text{T.S.})$.
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Levene's Test in \texttt{R}}{\url{https://raw.githubusercontent.com/zh3nis/MATH440/main/chp03/levene_test.R}}
\begin{small}
\begin{verbatim}
> library(lawstat)
> poverty = read.table("path/to/poverty.txt",
                       h = T, sep = "\t")
> my_model = lm(Poverty ~ Graduates, data=poverty)
> re = my_model$residuals
> breaks = c(0, 80, 90, 100)
> groups = cut(poverty$Graduates, breaks)
> levene.test(re, groups)

	Modified robust Brown-Forsythe Levene-type test based on the
	absolute deviations from the median

data:  re
Test Statistic = 0.39829, p-value = 0.6737
\end{verbatim}
\end{small}
\pause FTR homogeneity of variance.
\end{frame}

\section{Remedial Measures}

\begin{frame}{Remedial Measures}
\begin{itemize}
    \item Nonlinear Relation: \pause Add polynomials or transform $x$ and/or $y$ (more emphasis on $x$)
    \item\pause Non-Constant Variance: \pause Weighted Least Squares, transform $x$ and/or $y$, or fit Generalized Linear Model
    \item\pause Non-Independence of Errors: \pause Transform $y$ or use Generalized Least Squares, or fit Generalized Linear Model with correlated errors.
    \item\pause Non-Normality of Errors: \pause Box-Cox transformation, or fit Generalized Linear Model.
    \item\pause Outliers: \pause Robust Regression or Nonparametric Regression
\end{itemize}
\end{frame}

\begin{frame}{Box-Cox (Power) Transformation}
Transforms the variable $w$ as
$$
w^{(\lambda)}=\begin{cases}
\frac{w^\lambda-1}{\lambda}&\text{if }\lambda\ne0\\
\ln(w)&\text{if }\lambda=0
\end{cases}
$$
\pause It can be applied to the response:
$$
Y_i^{(\lambda)}=\beta_0+\beta_1x_i+\epsilon_i,\qquad\text{with }\epsilon_i\,\,{\stackrel{\text{iid}}{\sim}}\,\,\mathcal{N}(0,\sigma^2)
$$
\pause Or it can be applied to the predictor:
$$
Y_i=\beta_0+\beta_1x_i^{(\lambda)}+\epsilon_i,\qquad\text{with }\epsilon_i\,\,{\stackrel{\text{iid}}{\sim}}\,\,\mathcal{N}(0,\sigma^2)
$$
\pause Software is used to estimate the $\lambda$.
\end{frame}

\begin{frame}{Example: Recalling Items}
In an experiment 13 people are asked to memorize a list of
disconnected items. Asked to recall them at various times up to a week later.\pause
\begin{itemize}
    \item $Y$ --- proportion of items recalled correctly
    \item $x$ --- time, in minutes, since initially memorized the list.
\end{itemize}
\pause\begin{table}
    \centering
    \begin{tabular}{c c c c c c c c}
        \toprule
        $x$ & 1 & 5 & 15 & 30 & 60 & 120 & 240 \\
        $Y$ & 0.84 & 0.71 & 0.61 & 0.56 & 0.54 & 0.47 & .45\\
        \midrule
        $x$ & 480 & 720 & 1440 & 2880 & 5760 & 10080 \\
        $Y$ & 0.38 & 0.36 & 0.26 & 0.20 & 0.16 & 0.08\\
        \bottomrule
    \end{tabular}
\end{table}
\end{frame}

\begin{frame}[fragile]{Box-Cox Transformation in \texttt{R}}

\begin{footnotesize}
\begin{verbatim}
bcPower Transformation to Normality
            Est Power Rounded Pwr Wald Lwr Bnd Wald Upr Bnd
recall$time    0.0617           0      -0.1514       0.2748

Likelihood ratio test that transformation parameter is equal to 0
 (log transformation)
                           LRT df    pval
LR test, lambda = (0) 0.327992  1 0.56684
\end{verbatim}
\end{footnotesize}
\pause$\Rightarrow$ Log transformation of $x$ seems a good choice.
\pause \begin{figure}
    \centering
\includegraphics[width=.5\textwidth]{plots/recall_baseline.pdf}\includegraphics[width=.5\textwidth]{plots/recall_bc.pdf}
    \caption{Left: original data, Right: after log transformation of $x$}
\end{figure}
\end{frame}

\begin{frame}{Graphical Diagnostics for $\hat{Y}_i=\beta_0+\beta_1 x_i$}{\url{https://github.com/zh3nis/MATH440/blob/main/chp03/recall.R}}
\includegraphics[width=\textwidth]{plots/baseline_diag.pdf}
\end{frame}

\begin{frame}{Graphical Diagnostics for $\hat{Y}_i=\beta_0+\beta_1\ln(x_i)$}{\url{https://github.com/zh3nis/MATH440/blob/main/chp03/recall_boxcox.R}}
\includegraphics[width=\textwidth]{plots/recall_bc_diag.pdf}
\end{frame}

\end{document}

Slides/03 Diagnostics and Remedial Measures/main.tex


\section{Deep Residual Networks - ResNet}

This section will elaborate on the implementaiton of ResNet used for this project.

The ResNet implemented in the project consists of 34 layers, where a flatten layer and a softmax classifier layer is appended to the end of the ResNet. The first layer in the net is a normal convolutional layer. 25 of the layers are residual blocks, which also contains internal layers. After 5 residual blocks a pooling layer is added with a 3x3 filter and a stride with 2x2, meaning that a total of 5 pooling layers will be added to the net. At last a fully connected layer is added to the top with the flatten and softmax classifier. Adam is used as the activation function for the ResNet. The creation of the ResNet layers can be seen in listing \ref{lst:resloop}.

\begin{lstlisting}[language=Python, label=lst:resloop, caption=For loop that creates the layers in the ResNet]
layer1 = slim.conv2d(input_layer,64,[3,3],
	normalizer_fn=slim.batch_norm,scope='conv_'+str(0))
for i in range(5):
    for j in range(units_between_stride):
        layer1 = resUnit(layer1,j + (i*units_between_stride))
    layer1 = slim.conv2d(layer1,64,[3,3],stride=[2,2],
    	normalizer_fn=slim.batch_norm,scope='conv_s_'+str(i))

top = slim.conv2d(layer1,10,[3,3],
	normalizer_fn=slim.batch_norm,activation_fn=None,scope='conv_top')

output = slim.layers.softmax(slim.layers.flatten(top))
\end{lstlisting}

As seen in listing \ref{lst:resloop} the function resUnit is called to create a residual block. This function creates the residual block with the internal layers and returns the output from the block. The residual block consists of 2 batch normalization blocks and 2 convolutional layers. Between the convolutional layer and the batch layer a ReLU activation function is added. At last the output from the layers is added with the input, and this is how the shortcut connection is implemented. The code for creating a residual block can be seen in listing \ref{lst:resblock}.

\begin{lstlisting}[language=Python, label=lst:resblock, caption=resUnit function that creates the residual block]
def resUnit(input_layer,i):
    with tf.variable_scope("res_unit"+str(i)):
        part1 = slim.batch_norm(input_layer,activation_fn=None)
        part2 = tf.nn.relu(part1)
        part3 = slim.conv2d(part2,64,[3,3],activation_fn=None)
        part4 = slim.batch_norm(part3,activation_fn=None)
        part5 = tf.nn.relu(part4)
        part6 = slim.conv2d(part5,64,[3,3],activation_fn=None)
        output = input_layer + part6
        return output
\end{lstlisting}

When training neural networks the distribution of each layers input change, because of the parameters of the previous layers change\citep{BATCH}. This makes it hard to train models with nonlinearities as the training is slowed down because of a lower learning rate and careful paramter initialization. The solution to this problem is the batch normalization layers, which makes normalization a part of the nets architecture, by performing a normalization for each training batch. Batch normalization makes it possible to use a higher learning rate, and gives the freedom to care less about initialization. Batch normalization aslo acts as a regularier, and can in some cases eliminate the need for dropout.

The visualization of the entire residual block can be seen in figure \ref{fig:resblock_imp}.

\myFigure{ResNetBlock.png}{Visualization of the residual block shown in listing \ref{lst:resblock}.}{fig:resblock_imp}{0.35}


\documentclass{article}

\usepackage{multirow}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage[hidelinks]{hyperref}

\newcommand{\Invariant}{\textbf{\underline{Invariant:}}}
\newcommand{\topic}[1]{\textbf{#1:}}
\newcommand{\ls}{\\*[1ex]}
\newcommand{\timecase}[2]{
    \vspace{1em}
    \begin{center}
    \begin{tabular}{ |c|c|c| }
    \hline
    \textbf{Time} & \textbf{Best Case} & \textbf{Worst Case} \\ \cline{2-3}
    \textbf{Complexity} & $\Theta \left( #1 \right)$ & $\Theta \left( #2 \right)$ \\ \cline{2-3}
    \hline
    \end{tabular}
    \end{center}
}

\begin{document}

\tableofcontents

\pagebreak

\section{Insertion sort algorithm and proof}

\subsection{Algorithm}
\begin{algorithm}
    \caption{Insertion Sort}
    \begin{algorithmic}[1]
        \Require array of comparable elements
        \Function{insertion\_sort}{array} \Comment{0 indexed array assumed}
        \For {$j = 1$ \textbf{to} $array.length - 1$}
        \State $key = array[j]$;
        \State $i = j - 1$;
        \While{$i \ge 0$ and $array[i] > key$}
        \State $array[i + 1] = array[i]$;
        \State $i = i - 1$;
        \EndWhile
        \State $array[i + 1] = key$;
        \EndFor
        \EndFunction
    \end{algorithmic}
\end{algorithm}

\subsection{Proof}

\Invariant The subarray array[0..j-1], is always in sorted order.
\ls
\topic{Initialization} Initially we have j = 1, so the array[0] (a single element) is always sorted.
\ls
\topic{Maintenance} For loop works by moving forward from j = 1, in each iteration, it searches for the position of current key then places it there while shifting every value down the line by one.
\ls
\topic{Termination} when the loop terminates, j = array.length = n, so array[0..n-1] is in sorted order, which is the entire space value of array. So we conclude that entier array is sorted.

\timecase{n}{n^2}

\pagebreak

\section{Linear search algorithm and proof}

\subsection{Algorithm}
\begin{algorithm}
    \caption{Linear Search Algorithm}
    \begin{algorithmic}[1]
        \Require Array to search given element in and the value to search for
        \Function {linear\_search}{$array, value$}
        \For {$index = 0$ \textbf{to} $array.length - 1$}
        \If {$array[index] == value$}
        \State \Return $index$;
        \EndIf
        \EndFor
        \State \Return $index$;
        \EndFunction
    \end{algorithmic}
\end{algorithm}

\subsection{Proof}

\Invariant The value is not within the range
array[0..index-1]
\ls
\topic{Initialization} Initially we have index = 0, so the value isn't within the array range array[0..-1] which is empty arrray.
\ls
\topic{Maintenance} Within each for loop if the value pointed by index isn't equal to the value, we increment otherwise we return the index and terminate the function. In either case, the value will not be contained within the range array[0..index-1].
\ls
\topic{Termination} The return value of index could either point to the value itself, or point to the value one after the size limit of array. In former case, since the value was first found at that index, there is no possibility for the same value to exist within the sub-array, array[0..index-1]. In later case, the value of index is one plus the total size of array. This implies, that the value isn't contained within the array at all.
So, we conclude that linear search will always aware us about the presence or absence of the value in the given array.

\timecase{1}{n}

\pagebreak

\section{Selection sort algorithm and proof}

\subsection{Algorithm}

\begin{algorithm}
    \caption{Selection sort algorithm}
    \begin{algorithmic}[1]
        \Function{selection\_sort}{$array$}
        \For {$i = 0$ \textbf{to} $array.length - 2$}
        \State index = i;
        \For {$j = i + 1$ \textbf{to} $array.length - 1$}
        \If {$array[index] > array[j]$}
        \State index = j;
        \EndIf
        \EndFor
        \State \Call{swap}{$array[index], array[i]$};
        \EndFor
        \EndFunction
    \end{algorithmic}
\end{algorithm}

\subsection{Proof}

\Invariant The sub array array[0..i-1] is always in sorted order.
\ls
\topic{Initialization} Initially we have i = 1, so the array[0] (a single element) is always sorted.
\ls
\topic{Maintenance} In each iteration, we move in the array by one position. At each position we find the smallest value since that position then swap the value at given position with found smallest value. This ensures that the array[0..i-1] is always sorted in ascending order. We only need to move by n-1 position, as the final will be sorted implicitly.
\ls
\topic{Termination} When the loop ends, we have i = array.length, so array[0..i-1] which comprises of the entire array is sorted. Hence we can conclude that the entier array is sorted.

\timecase{n^2}{n^2}

\end{document}


\subsection{$A$-sequence and $A$-matrix}
\label{sec:back:to:the:basics:sequences}

In this section we rework the concept of $A$-sequence introduced by
\citeauthor{rogers:1977} and discussed in
\autoref{sec:back:to:the:basics:rogers}, using the \emph{Riordan group} to
characterize it for \emph{Riordan arrays}. Furthermore, we will see a
generalization of this concept, introducing $A$-matrices and some applications
are reported for the sake of clarity.

\begin{theorem}
    $\mathcal{M}$ is a Riordan array over a matrix of coefficients
    $\lbrace m_{n,k}\rbrace_{n,k\in\mathbb{N}}$ if and only if
    there exists a sequence $\vect{a}=\lbrace a_{n}\rbrace_{n\in\mathbb{N}}$,
    with $a_{0}\neq0$, such that:
    \begin{displaymath}
        m_{n+1,k+1}=a_{0}m_{n,k}+a_{1}m_{n,k+1}+a_{2}m_{n,k+2}+\ldots+a_{j}m_{n,k+j}
    \end{displaymath}
    where $n,k,j\in\mathbb{N}$ and $k+j=n$. Sequence $\vect{a}$ is called the $A$-sequence
    of Riordan array $\mathcal{M}$.
    \label{thm:merlini:A:sequence:characterization}
\end{theorem}
\marginpar{The idea behind this proof comes from \cite{he:sprugnoli:2009}}
\begin{proof}[Proof of ($\rightarrow$) direction]
    Let $\mathcal{M}=(d_{\mathcal{M}}(t),h_{\mathcal{M}}(t))$ be a Riordan array. Since it is requested
    to prove that such a sequence $\vect{a}$ \emph{exists}, we proceed to
    build a Riordan array $\mathcal{A}=(d_{\mathcal{A}}(t),h_{\mathcal{A}}(t))$ such
    that function $d_{\mathcal{A}}$ is the \ac{gf} for sequence $\vect{a}$, proving this direction.
    Riordan array $\mathcal{A}$ is defined as the solution of the following relation:
    \begin{displaymath}
        (d_{\mathcal{M}}(t),h_{\mathcal{M}}(t))\cdot(d_{\mathcal{A}}(t),h_{\mathcal{A}}(t))
            = \left(d_{\mathcal{M}}(t)\frac{h_{\mathcal{M}}(t)}{t},h_{\mathcal{M}}(t)\right)
    \end{displaymath}
    by definition of group operator $\cdot$, the \ac{lhs} can be rewritten as:
    \begin{displaymath}
        (d_{\mathcal{M}}(t)d_{\mathcal{A}}(h_{\mathcal{M}}(t)),h_{\mathcal{A}}(h_{\mathcal{M}}(t)))
            = \left(d_{\mathcal{M}}(t)\frac{h_{\mathcal{M}}(t)}{t},h_{\mathcal{M}}(t)\right)
    \end{displaymath}

    Looking at second components, the following relation defines function $h_{\mathcal{A}}$:
    \begin{displaymath}
         \left[h_{\mathcal{A}}(y)=y \mid y = h_{\mathcal{M}}(t)\right]
    \end{displaymath}
    while looking at the first component, the following relation defines function $d_{\mathcal{A}}$:
    \marginpar{a \ac{gf} over $A$-sequence $\vect{a}$ for Riordan arrays}
    \begin{displaymath}
         \left[d_{\mathcal{A}}(y)=\frac{y}{\hat{h}_{\mathcal{M}}(y)} \mid y = h_{\mathcal{M}}(t)\right]
    \end{displaymath}
    therefore it is possible to write the complete definition for array $\mathcal{A}$:
    \begin{displaymath}
         \mathcal{A}=(d_{\mathcal{A}}(y),h_{\mathcal{A}}(y))
            =\left(\frac{y}{\hat{h}_{\mathcal{M}}(y)}, y \right)
    \end{displaymath}

    \marginpar{although we are in the Riordan group, here we go back to
        matrix theory}
    Assume function $d_{\mathcal{A}}$ can be expanded as \ac{fps} over a sequence
    $\vect{a}=\lbrace a_{n}\rbrace_{n\in\mathbb{N}}$, therefore expanding array $\mathcal{A}$
    as a matrix $\lbrace a_{nk}\rbrace_{n,k\in\mathbb{N}}$ yield:
    \begin{displaymath}
        \mathcal{A} =
        \left[
            \begin{array}{cccccc}
                a_{0} & & & &  &\\
                a_{1} & a_{0} & & &  &\\
                a_{2} & a_{1}& a_{0}& &  &\\
                a_{3} & a_{2}& a_{1}& a_{0}&  &\\
                a_{4} & a_{3}& a_{2}& a_{1}& a_{0} &\\
                \vdots & \vdots& \vdots& \vdots& \vdots & \ddots\\
            \end{array}
        \right]
    \end{displaymath}
    so doing the \emph{matrix} product of array $\mathcal{M}$ with array $\mathcal{A}$ yield:
    \begin{displaymath}
        \mathcal{M}\mathcal{A}
        =
        \left[
            \begin{array}{cccccc}
                m_{00} & & & &  &\\
                m_{10} & m_{11} & & &  &\\
                m_{20} & m_{21}& m_{22}& &  &\\
                m_{30} & m_{31}& m_{32}& m_{33}&  &\\
                m_{40} & m_{41}& m_{42}& m_{43}& m_{44} &\\
                \vdots & \vdots& \vdots& \vdots& \vdots & \ddots\\
            \end{array}
        \right]
        \left[
            \begin{array}{cccccc}
                a_{0} & & & &  &\\
                a_{1} & a_{0} & & &  &\\
                a_{2} & a_{1}& a_{0}& &  &\\
                a_{3} & a_{2}& a_{1}& a_{0}&  &\\
                a_{4} & a_{3}& a_{2}& a_{1}& a_{0} &\\
                \vdots & \vdots& \vdots& \vdots& \vdots & \ddots\\
            \end{array}
        \right]
    \end{displaymath}
    Consider coefficient at row $n$ and column $k$ of $\mathcal{M}\mathcal{A}$, by definition
    of \emph{matrix} product:
    \begin{displaymath}
        \left(\mathcal{M}\mathcal{A}\right)_{nk}=
        \sum_{j=0}^{n}{m_{nj}{a_{jk}}}
            =\sum_{j=0}^{n-k}{m_{n,k+j}{a_{k+j,k}}}
            =\sum_{j=0}^{n-k}{m_{n,k+j}{a_{j}}}
    \end{displaymath}
    Do the same for coefficient at row $n$ and column $k$ of
    $\left(d_{\mathcal{M}}(t)\frac{h_{\mathcal{M}}(t)}{t},h_{\mathcal{M}}(t)\right)$:
    \begin{displaymath}
        [t^{n+1}]d_{\mathcal{M}}(t)\,h_{\mathcal{M}}(t)^{k+1} = m_{n+1,k+1}
    \end{displaymath}
    so equating this and the previous result yield:
    \begin{displaymath}
        m_{n+1,k+1}=\sum_{j=0}^{n}{m_{n,k+j}{a_{j}}}
    \end{displaymath}
    as required.
\end{proof}
The other direction is less interesting since it is not a \emph{constructive} one,
so we left it as an exercise.
\\\\
The above proof leave space for another result. Start again from the following equation:
\begin{displaymath}
    (d_{\mathcal{M}}(t)d_{\mathcal{A}}(h_{\mathcal{M}}(t)),h_{\mathcal{M}}(t))
        = \left(d_{\mathcal{M}}(t)\frac{h_{\mathcal{M}}(t)}{t},h_{\mathcal{M}}(t)\right)
\end{displaymath}
and consider coefficient at row $n$ and column $k$ in both arrays, as done before:
\begin{displaymath}
    [t^{n}]d_{\mathcal{A}}(h_{\mathcal{M}}(t))\left(d_{\mathcal{M}}(t)\,h_{\mathcal{M}}(t)^{k}\right)
        = [t^{n+1}]d_{\mathcal{M}}(t)\,h_{\mathcal{M}}(t)^{k+1}
\end{displaymath}
\marginpar{sequence $\vect{\hat{a}}$, with \ac{gf} $\hat{A}(t)$,
is yet another characterization}
on the \ac{lhs} appears a convolution of function $\hat{A}(t)=d_{\mathcal{A}}(h_{\mathcal{M}}(t))$,
which can be written as \ac{fps} over sequence $\vect{\hat{a}}=\lbrace\hat{a}_{i}\rbrace_{i\in\mathbb{N}}$,
with function $m_{k}(t)=d_{\mathcal{M}}(t)\,h_{\mathcal{M}}(t)^{k}$. Therefore the following identity
holds as well:
\begin{displaymath}
    \sum_{j=0}^{n}{\hat{a}_{j}\,m_{n-j,k}}
        = m_{n+1,k+1}
\end{displaymath}
combining this result with the one stated in the above theorem, the following holds:
\begin{displaymath}
    \sum_{j=0}^{n}{\hat{a}_{j}\,m_{n-j,k}}
        =\sum_{j=0}^{n}{m_{n,k+j}{a_{j}}}
\end{displaymath}

Yet another characterization for a coefficient $m_{nk}$ in a Riordan array $\mathcal{M}$
can be stated and is closely related to $\mathcal{M}$'s $A$-sequence.
\begin{theorem}
    Let $\mathcal{M}$ be a Riordan array over a matrix of coefficients
    $\lbrace m_{n,k}\rbrace_{n,k\in\mathbb{N}}$, then there exists a sequence
    $\vect{\omega}=\lbrace \omega_{n}\rbrace_{n\in\mathbb{N}}$ such that:
    \begin{displaymath}
        m_{nk}=\sum_{j=0}^{n-k}{\omega_{j}\,m_{n+1,k+1+j}}
    \end{displaymath}
    moreover, denoting with $\Omega$ a \ac{gf} over sequence $\vect{\omega}$ and
    with $A$ the \ac{gf} for $\mathcal{M}$'s $A$-sequence, then:
    \begin{displaymath}
        A(t)\,\Omega(t)=1
    \end{displaymath}
    \label{thm:characterization:next:row}
\end{theorem}
\begin{proof}
    Assume not, therefore for \emph{each} sequence $\vect{\omega}=\lbrace \omega_{n}\rbrace_{n\in\mathbb{N}}$,
    from:
    \begin{displaymath}
        m_{nk}\neq\sum_{j=0}^{n-k}{\omega_{j}\,m_{n+1,k+1+j}}
    \end{displaymath}
    we have to show a contraddiction. By $A$-sequence characterization,
    there exists a sequence $\vect{a}=\lbrace a_{n}\rbrace_{n\in\mathbb{N}}$
    such that allows us to rewrite terms $m_{n+1,k+1+j}$
    within the sum in the \ac{rhs}:
    \begin{displaymath}
        \begin{split}
            m_{nk} &\neq \omega_{0}\left(a_{0}m_{n,k}+a_{1}m_{n,k+1}+a_{2}m_{n,k+2}+a_{3}m_{n,k+3}+\ldots+a_{j_{0}}m_{n,k+j_{0}}\right)\\
                   &+ \omega_{1}\left(a_{0}m_{n,k+1}+a_{1}m_{n,k+2}+a_{2}m_{n,k+3}+a_{3}m_{n,k+4}+\ldots+a_{j_{1}}m_{n,k+j_{1}}\right)\\
                   &+ \omega_{2}\left(a_{0}m_{n,k+2}+a_{1}m_{n,k+3}+a_{2}m_{n,k+4}+a_{3}m_{n,k+5}+\ldots+a_{j_{2}}m_{n,k+j_{2}}\right)\\
                   &+ \omega_{3}\left(a_{0}m_{n,k+3}+a_{1}m_{n,k+4}+a_{2}m_{n,k+5}+a_{3}m_{n,k+6}+\ldots+a_{j_{3}}m_{n,k+j_{3}}\right)\\
                   &\ldots\\
                   &+ \omega_{n-k-1}\left(a_{0}m_{n,n-1}+a_{1}m_{n,n}\right)\\
                   &+ \omega_{n-k}\left(a_{0}m_{n,n}\right)\\
        \end{split}
    \end{displaymath}
    factoring coefficients $m_{nk}$, the \ac{rhs} can be rewritten as:
    \begin{displaymath}
        \begin{split}
            m_{nk}&\neq \omega_{0}a_{0}m_{n,k} + \left(\sum_{i_{1}+i_{2}=1}{\omega_{i_{1}}a_{i_{2}}}\right)m_{n,k+1}
                + \left(\sum_{i_{1}+i_{2}=2}{\omega_{i_{1}}a_{i_{2}}}\right)m_{n,k+2}\\
                &+ \left(\sum_{i_{1}+i_{2}=3}{\omega_{i_{1}}a_{i_{2}}}\right)m_{n,k+3}
                + \ldots
                + \left(\sum_{i_{1}+i_{2}=n-k}{\omega_{i_{1}}a_{i_{2}}}\right)m_{n,n}
        \end{split}
    \end{displaymath}
    which can be written more compactly as:
    \begin{displaymath}
        m_{nk} \neq \omega_{0}a_{0}m_{n,k} +
            \sum_{j=1}^{n-k}{\left(\sum_{i_{1}+i_{2}=j}{\omega_{i_{1}}a_{i_{2}}}\right)m_{n,k+j}}
    \end{displaymath}
    but if we take sequence $\vect{\omega}$ to be the \emph{inverse} sequence of
    $\mathcal{M}$'s $A$-sequence $\vect{a}$, then the previous equation is false,
    since $\omega_{0}a_{0}=1$ and $\sum_{i_{1}+i_{2}=j}{\omega_{i_{1}}a_{i_{2}}}=0$,
    for each $j\in\lbrace1,\ldots,n-k\rbrace$, equality \emph{holds}.
    A contraddiction occurs, as required.
\end{proof}

\subsubsection{$A$-matrix}

We are now ready to tackle the most general characterization for a coefficient
$m_{nk}$ in a Riordan array $\mathcal{M}$, the concept of $A$-matrix. Actually,
we will see only one formulation but there exist another two of them, yet more
general indeed. The following theorem and the proof idea comes from
\cite{merlini:some:alternative:characterizations:1997}, where additional
formulation of the $A$-matrix concept can be found.

\begin{theorem}
    $\mathcal{M}$ is a Riordan array, over a matrix of coefficients
    $\lbrace m_{n,k}\rbrace_{n,k\in\mathbb{N}}$, if there exists a matrix
    $\lbrace \sigma_{nk}\rbrace_{n,k\in\mathbb{N}}$ such that:
    \begin{displaymath}
        m_{n+1,k+1}=\sum_{i\in\mathbb{N}}{\sum_{j\in\mathbb{N}}{\sigma_{ij}m_{n-i,k+j}}}
    \end{displaymath}
\end{theorem}

\begin{proof}
    The idea underlying this proof is to show that a sequence
    $\vect{a}=\lbrace a_{n}\rbrace_{n\in\mathbb{N}}$ can be built from
    matrix $\lbrace \sigma_{nk}\rbrace_{n,k\in\mathbb{N}}$,
    such that $\vect{a}$ can be used to combine elements in matrix
    $\lbrace m_{n,k}\rbrace_{n,k\in\mathbb{N}}$ according to:
    \begin{displaymath}
        m_{n+1,k+1}=a_{0}m_{nk}+a_{1}m_{n,k+1}+\ldots+a_{j}m_{n,k+j}
    \end{displaymath}
    where $k+j=n$, as usual.  If we success, then sequence $\vect{a}$ is the $A$-sequence for
    $\lbrace m_{n,k}\rbrace_{n,k\in\mathbb{N}}$, therefore $\mathcal{M}$ is a Riordan array.
    \\\\
    Choose $n\in\mathbb{N}$, then proceed by natural induction on $n-k$:
    \begin{itemize}
        \item base case, $n-k=0$. By assumption the following holds:
            \begin{displaymath}
                m_{n+1,n+1}=\sigma_{00}\,m_{nn}
            \end{displaymath}
            which suggests to set $a_{0}=\sigma_{00}$;
        \item although not required, consider case $n-k=1$. By assumption the following holds:
            \begin{displaymath}
                m_{n+1,n}=\sigma_{00}m_{n,n-1}+\sigma_{01}m_{n,n}+\sigma_{10}m_{n-1,n-1}
            \end{displaymath}
            by \autoref{thm:characterization:next:row}, there exist a sequence $\vect{\omega}=
            \lbrace \omega_{n}\rbrace_{n\in\mathbb{N}}$ which allow us to rewrite $m_{n-1,n-1}$
            as a combination of $m_{nn}$:
            \begin{displaymath}
                m_{n+1,n}=\sigma_{00}m_{n,n-1}+\left(\sigma_{01}+\sigma_{10}\omega_{0}\right)m_{nn}
            \end{displaymath}
            which reinforces $a_{0}=\sigma_{00}$ and suggests to set
            $a_{1}=\sigma_{01}+\sigma_{10}\omega_{0}$; moreover, using sequence $\vect{\omega}$
            to rewrite terms in the \ac{rhs}:
            \begin{displaymath}
                m_{n+1,n}=\sigma_{00}\left(\omega_{0}m_{n+1,n}+\omega_{1}m_{n+1,n+1}\right)+
                    \left(\frac{\sigma_{01}+\sigma_{10}\omega_{0}}{\sigma_{00}}\right)m_{n+1,n+1}
            \end{displaymath}
            manipulating:
            \begin{displaymath}
                m_{n+1,n}=
                    \left(\frac{\sigma_{00}^{2}\omega_{1}+\sigma_{01}+\sigma_{10}\omega_{0}}
                    {(1-\sigma_{00}\omega_{0})\sigma_{00}}\right)m_{n+1,n+1}
            \end{displaymath}
        \item \emph{induction hp} assume that, for $n-k=n-1$, ie. $k=1$, if:
            \begin{displaymath}
                m_{n+1,2}=\sum_{i\in\mathbb{N}}{\sum_{j\in\mathbb{N}}{\sigma_{ij}m_{n-i,1+j}}}
            \end{displaymath}
            then there exists a sequence $\vect{\hat{a}}$ such that:
            \begin{displaymath}
                m_{n+1,2}=\hat{a}_{0}m_{n1}+\hat{a}_{1}m_{n,2}+\ldots+\hat{a}_{j}m_{n,1+j}
            \end{displaymath}
            where $1+j=n$;
        \item \emph{induction step} show that, for $n-k=n$, ie. $k=0$, if:
            \begin{displaymath}
                m_{n+1,1}=\sum_{i\in\mathbb{N}}{\sum_{j\in\mathbb{N}}{\sigma_{ij}m_{n-i,j}}}
            \end{displaymath}
            then there exists a sequence $\vect{a}$ such that:
            \begin{displaymath}
                m_{n+1,1}=a_{0}m_{n0}+a_{1}m_{n,1}+\ldots+a_{n}m_{n,n}
            \end{displaymath}

            Begin by expanding the assumption:
            \begin{displaymath}
                \hspace{-2cm}
                \begin{split}
                    m_{n+1,1}&=\sigma_{00}m_{n0}+\sigma_{01}m_{n1}+\sigma_{02}m_{n2}+\ldots
                                +\sigma_{0,n-2}m_{n,n-2}+\sigma_{0,n-1}m_{n,n-1}+\sigma_{0n}m_{nn}\\
                             &+\sigma_{10}m_{n-1,0}+\sigma_{11}m_{n-1,1}+\sigma_{12}m_{n-1,2}+\ldots
                                +\sigma_{1,n-2}m_{n-1,n-2}+\sigma_{1,n-1}m_{n-1,n-1}\\
                             &+\sigma_{20}m_{n-2,0}+\sigma_{21}m_{n-2,1}+\sigma_{22}m_{n-2,2}+\ldots+\sigma_{2,n-2}m_{n-2,n-2}\\
                             &\ldots\\
                             &+\sigma_{n-1,0}m_{10}+\sigma_{n-1,1}m_{11}\\
                             &+\sigma_{n0}m_{00}\\
                \end{split}
            \end{displaymath}
            from the bottom line of the previous sum expansion, keep applying
            \autoref{thm:characterization:next:row} to every coefficient $m_{rc}$, for
            $r\in\lbrace0,\ldots,n-1\rbrace$ and, consequently, $c\in\lbrace0,\ldots,r\rbrace$.
            When every coefficient $m_{n-1,c}$, for $c\in\lbrace0,\ldots,n-1\rbrace$, has been
            expanded, coefficient $m_{n+1,1}$ is a combination of coefficients
            $\lbrace m_{n0}\rbrace\cup\lbrace m_{n,1},m_{n,2}\ldots,m_{n,n}\rbrace$ using
            a sequence $\vect{\beta}$, namely:
            \begin{displaymath}
                m_{n+1,1}=\beta_{0}m_{n0}+\beta_{1}m_{n1}+\ldots+\beta_{n}m_{nn}
            \end{displaymath}
            By induction hypothesis, there exists a sequence $\vect{\hat{a}}$ such that:
            \begin{displaymath}
                m_{n+1,2}=\hat{a}_{0}m_{n1}+\hat{a}_{1}m_{n2}+\ldots+\hat{a}_{n-1}m_{nn}
            \end{displaymath}
            so, we can build a sequence $\vect{a}$ as follows:
            \begin{displaymath}
                \vect{a}=\left(\hat{a}_{0},\hat{a}_{1},\hat{a}_{2},\ldots,\hat{a}_{n-1},\beta_{n}\right)
            \end{displaymath}
            where $\beta_{n}$ can be possibly different from $\hat{a}_{n-1}$.
            Sequence $\vect{a}$ is the $A$-sequence
            for matrix $\lbrace m_{nk}\rbrace_{n,k\in\mathbb{N}}$, therefore $\mathcal{M}$ is a Riordan
            array, as required.
    \end{itemize}
\end{proof}


%!TEX root = /home/renaud/Documents/EPL/tfe/latex/tfe.tex
\section{Properties of the compartment model solution} \label{sec:prop_comp}
In section~\ref{sec:propcontinuous}, important properties of the continuous transport model have been derived. In this section, the counterparts of those properties are established in the case of a compartment model. The domain-averaged concentration can be expressed in terms of the compartments concentrations. Indeed, by equations \eqref{eq:Cmean} and \eqref{eq:C_i(t)}:
\begin{equation} \label{eq:Cmean_compartment}
	\bar{C}(t) = \frac{1}{|\Omega|} \int_\Omega C(\b x,t) \rm d\Omega = \frac{1}{|\Omega|} \sum_{i=1}^N \int_{\Omega_i} C(\b x,t) \rm d\Omega_i
		= \frac{1}{|\Omega|} \sum_{i=1}^N |\Omega_i| C_i(t).
\end{equation}
The variance of the concentration is approximated by the \textit{box variance}
\begin{equation}
	\sigma_{box}^2(t) = \frac{1}{|\Omega|} \sum_{i=1}^{N} |\Omega_i| \left[C_i(t) - \bar{C}(t)\right]^2.
\end{equation}

\begin{property} \label{prop1_comp}
	The mean tracer concentration $\bar C$ is not influenced by the internal advective and diffusive fluxes.
\end{property}
\begin{proof}
	By~\eqref{eq:Cmean_compartment},~\eqref{eq:compartmentmodel},~\eqref{eq:Uprop} and~\eqref{eq:Vprop}, we get successively
	\begin{subequations}
	\begin{align}
		\frac{d\bar{C}}{dt} &= \frac{1}{|\Omega|} \sum_{i=1}^N |\Omega_i| \frac{dC_i(t)}{dt}\\
			&= \frac{1}{|\Omega|} \sum_{i=1}^N \left[|\Omega_i| q_i - \sum_{j=1}^N \left[ U_{i,j}\frac{C_i + C_j}{2} - V_{i,j} (C_j-C_i)\right] - \phi_{i,e} \right]\\
			&= \frac{1}{|\Omega|} \sum_{i=1}^N(|\Omega_i| q_i + \phi_{e,i}). \label{eq:prop1deleersnijder}
	\end{align}
	\end{subequations}
\end{proof}

\begin{corollary}{prop1_comp} \label{cor:Cmeanconstant}
	For a passive tracer in an isolated domain, the mean tracer concentration within the whole domain $\bar C$ is a constant.
\end{corollary}
\begin{proof}
	The proof is straightforward by introducing $q_i = 0$ (passive tracer) and $\phi_{i,e} = 0$ (isolated domain) in equation~\eqref{eq:prop1deleersnijder}.
\end{proof}

\begin{property} \label{prop2_comp}
	If the tracer is passive, the domain isolated and the initial concentration is constant, then the concentration stays constant.
\end{property}
\begin{proof}
	If the initial concentration is constant, then $C_i(0) = C_0$ for all $i = 1,\dots,N$. At time $t=0$, equation~\eqref{eq:compartmentmodel} becomes
	\begin{equation}
		|\Omega_i| \frac{dC_i}{dt} = -C_0 \sum_{j=1}^N U_{i,j},
	\end{equation}
	which is equal to zero by~\eqref{eq:continuitycompartment}. Here we have used that $q_i = 0$ and $\phi_{i,e} = 0$ (and thus $U_{i,e}=0$) since the tracer is passive and the domain is isolated.
\end{proof}

\begin{property} \label{prop3_comp}
	For a passive tracer ($q_i = 0$) in an isolated domain ($\phi_{i,e} = 0$), the box variance of the concentration decreases monotonically until $C_i(t) = \bar{C}(0)$ for all $i = 1,\dots,N$ (i.e., until the tracer distribution is uniform and equal to $\bar C(0)$ everywhere).
\end{property}
\begin{proof}
	Let $\hat{C}_i(t)$ denote the deviation of the concentration in compartment $i$ with respect to the mean concentration over the whole domain:
	\begin{equation}
		\hat{C}_i(t) := C_i(t) - \bar C(t).
	\end{equation}
	By property~\ref{prop2_comp}, $\bar C(t) = \bar C(0) = \bar C$ since the domain is isolated and the tracer is passive. Hence, we can rewrite equation~\eqref{eq:compartmentmodel} as
	\begin{equation}
		|\Omega_i| \frac{d\hat C_i}{dt} = - \sum_{j=1}^N \left[ U_{i,j}\frac{\hat C_i + \hat C_j}{2} - V_{i,j} (\hat C_j - \hat C_i)\right] - \bar C \sum_{j=1}^N U_{i,j},
	\end{equation}
	which by~\eqref{eq:continuitycompartment} simplifies to
	\begin{equation} \label{eq:temp_comp1}
		|\Omega_i| \frac{d\hat C_i}{dt} = - \sum_{j=1}^N \left[ U_{i,j}\frac{\hat C_i + \hat C_j}{2} - V_{i,j} (\hat C_j - \hat C_i)\right].
	\end{equation}
	Multiplying~\eqref{eq:temp_comp1} by $\hat C_i(t)$ and summing over $i = 1,\dots,N$ yields after some calculations
	\begin{equation} \label{eq:temp_comp2}
		|\Omega| \frac{d \sigma^2_{box}}{dt} = - \sum_{i=1}^N \sum_{j=1}^N \left[ U_{i,j}(\hat C_i + \hat C_j)\hat C_i - 2 V_{i,j} (\hat C_j - \hat C_i) \hat C_i\right].
	\end{equation}
	Let us look at both terms in the right-hand side of~\eqref{eq:temp_comp2} separately. The first term is equal to zero:
	\begin{subequations}
	 	\begin{align}
	 		\sum_{i=1}^N \sum_{j=1}^N U_{i,j}(\hat C_i + \hat C_j)\hat C_i &= \sum_{i=1}^N \left(\hat C_i \right)^2 \underbrace{\sum_{j=1}^N U_{i,j}}_{= 0} + \sum_{i=1}^N \sum_{j=1}^N U_{i,j}\hat C_j\hat C_i\\
	 		&= \sum_{i=1}^N \sum_{j=1}^N \underbrace{\frac{U_{i,j} + U_{j,i}}{2}}_{=0}\hat C_j\hat C_i.
	 	\end{align}
	 \end{subequations}
	The second term is rewritten as
	\begin{equation}
 		2 \sum_{i=1}^N \sum_{j=1}^N V_{i,j} (\hat C_j - \hat C_i) \hat C_i = -2 \sum_{i=1}^N \sum_{j=1}^N V_{i,j} (\hat C_j - \hat C_i)^2 - 2\sum_{i=1}^N \sum_{j=1}^N V_{i,j} (\hat C_j - \hat C_i) \hat C_i,
	 \end{equation}
	 so that
	 \begin{equation}
	 	2 \sum_{i=1}^N \sum_{j=1}^N V_{i,j} (\hat C_j - \hat C_i) \hat C_i = - \sum_{i=1}^N \sum_{j=1}^N V_{i,j} (\hat C_j - \hat C_i)^2.
	 \end{equation}
	 Finally,
	 \begin{equation}
	 	|\Omega| \frac{d \sigma^2_{box}}{dt} = - \sum_{i=1}^N \sum_{j=1}^N V_{i,j} (\hat C_j - \hat C_i)^2,
	 \end{equation}
	 and thus
	 \begin{equation}
	 	\frac{d \sigma^2_{box}}{dt} = -\frac{1}{|\Omega|} \sum_{i=1}^N \sum_{j=1}^N V_{i,j} (C_j - C_i)^2,
	 \end{equation}
	 which proves the claim.  Interestingly, the advective part of the fluxes does not contribute to the homogenization of the concentration; only the diffusive part does.
\end{proof}