\documentclass[12pt,landscape]{article}
\usepackage[landscape]{geometry}
\input{../../MathHeader}

% Turn off header and footer
\pagestyle{empty}

% print only section numbers
\setcounter{secnumdepth}{1}

\setlength{\parindent}{0pt}
\setlength{\parskip}{0pt plus 0.5ex}

\begin{document}
    \raggedright
    \footnotesize
    \begin{multicols}{3}
    % multicol parameters
    % These lengths are set only within the two main columns
    %\setlength{\columnseprule}{0.25pt}
    \setlength{\premulticols}{1pt}
    \setlength{\postmulticols}{1pt}
    \setlength{\multicolsep}{1pt}
    \setlength{\columnsep}{2pt}

    \begin{flushleft}
        \large{
            \underline{MA1521 Cheat Sheet} \\
            \texttt{by Howard Liu} \\
            AY2019/20 Semester 1
        }
    \end{flushleft}

    \section{Basics of Probability}

    \subsection{1-34 Operation of Events}
    Refer to CS1231S page 2 column 2 \textbf{"Set Properties"}.

    \subsection{1-107 Binomial Coefficient}
    \begin{enumerate}
        \item ${n \choose r} = {n \choose n-r}$ for $r = 0, 1, \dots, n$
        \item ${n \choose r} = {n-1 \choose r} + {n-1 \choose r-1}$ for $1 \leq r \leq n$
        \item ${n \choose r} = 0$ for $r < 0$ or $r > n$
    \end{enumerate}

    \subsection{1-128 Axioms of Probability}
    For each event $A$ of the sample space $S$ we assume that a number $Pr(A)$, which is called the \textbf{probability} of the event $A$, is defined and satisfies the following three axioms:
    \begin{enumerate}
        \item $0 \leq Pr(A) \leq 1$
        \item $Pr(S) = 1$
        \item if $A_1$, $A_2$, $\dots$ are \textbf{mutually exclusive} events (that is, $A_i \cap A_j = \emptyset$ when $i \neq j$), then
            $$ Pr(\bigcup\limits_{i = 1}^{\infty} A_i) = \sum_{i = 1}^{\infty} Pr(A_i) $$
    \end{enumerate}

    \subsection{1-141 Probability Properties}
    For any two events $A$ and $B$,
        $$ Pr(A \cup B) = Pr(A) + Pr(B) - Pr(A \cap B) $$

    \subsection{1-144 The Inclusion-Exclusion Principle}
    $ Pr(\bigcup\limits_{i = 1}^{n} A_i) = \sum_{i=1}^{n} Pr(A_i) - \sum_{i=1}^{n-1} \sum_{j=i+1}^{n} Pr(A_i \cap A_j) + \sum_{i=1}^{n-2} \sum_{j=i+1}^{n-1} \sum_{k=j+1}^{n} Pr(A_i \cap A_j \cap A_k) - \dots + (-1)^{n+1} Pr(\bigcap\limits_{i = 1}^{n} A_i) $

    \subsection{1-194 Conditional Probability}
    The conditional probability of $B$ given $A$, is defined as
    $$ Pr(B|A) = \frac{Pr(A \cap B)}{Pr(A)} \text{ , if } Pr(A) \ne 0 $$

    If events $B_1, B_2$ are \textbf{mutually exclusive} events, we have
    $$ Pr(B_1 \cup B_2 | A) = Pr(B_1|A) + Pr(B_2|A) $$

    \subsection{1-216 Multiplication Rule}
    If we have 3 events $A$, $B$ and $C$, we have
    $$ Pr(A \cap B \cap C) = Pr(A) Pr(B|A) Pr(C|A \cap B) $$
    providing that $Pr(A \cap B) > 0$

    \subsection{1-243 Bayer's Theorem}
    $$ Pr(A|B) = \frac{Pr(A) Pr(B|A)}{Pr(B)} $$

    \subsection{1-271 Independent Events}
    Two events $A$ and $B$ are independent iff.
    $$ Pr(A \cap B) = Pr(A) Pr(B) $$

    \subsection{1-273 Properties of Independent Events}
    \begin{enumerate}
        \item $Pr(A|B) = Pr(A)$ and $Pr(B|A) = Pr(B)$
        \item When two events (each with probability greater than 0) are \textbf{independent}, they cannot be \textbf{mutually exclusive}. Vice versa.
        \item The sample space $S$ and the empty set $\emptyset$ are independent of any events.
        \item If $A \subset B$, then $A$ and $B$ are dependent unless $B = S$.
        \item Properties of independence cannot be shown on a Venn diagram.
        \item If $A$ and $B$ are independent, so are $A$ and $B'$, $A'$ and $B$, $A'$ and $B'$.
    \end{enumerate}

    \subsection{1-288 Pairwise and Mutually Independence}
    Mutually independence implies pairwise independence. However, the \textbf{reverse} does not hold - pairwise independence does not imply mutually independence.

    \section{Concepts of Random Variables}

    \subsection{2-12 Equivalent Events}
    Two events $A$ and $B$ are equivalent iff. \textbf{$A$ consists of all sample points, $s$, in $S$ for which $X(s) \in B$}.

    \subsection{2-22 Probability Function}
    \label{2-22}
    The probability of $X = x_i$ denoted by $f(x_i)$ must satisfy the following two conditions
    \begin{enumerate}
        \item $f(x_i) \geq 0$ for all $x_i$.
        \item $\sum_{i = 1}^{\infty} f(x_i) = 1$.
    \end{enumerate}

    \subsection{2-44 Probability Density Function}
    \label{2-44}
    For any $c$ and $d$ s.t. $c < d$,
        $$ Pr(c \leq X \leq d) = \int_{c}^{d}f(x)dx $$
    Note: for any specified value of $X$, say $x_0$, we have $Pr(X = x_0) = Pr(x_0 \leq X \leq x_0) = \int_{x_0}^{x_0} f(x)dx = 0$

    \subsection{2-60 Cumulative Distribution Function}
    For c.d.f, we have definition $F(x) = Pr(X \leq x)$.

    If it is discrete random variable, then
        $$ F(x) = \sum_{t \leq x} f(t) = \sum_{t \leq x} Pr(X = t) $$
    If it is continuous random variable, then
        $$ F(x) = \int_{-\infty}^{x} f(t)dt $$
    Remark: $F(x)$ is non-decreasing.

    \subsection{2-64 Derive p.f and p.d.f from c.d.f}
    For a continuous random variable,
        $$ f(x) = \frac{dF(x)}{dx} $$
    if the derivative exists.

    Also, we have $Pr(a \neq X \neq b) = Pr(a < X \neq b) = F(b) - F(a)$ for CRV.

    \subsection{2-87 Expected Values (Mean)}
    For DRV, we define the \textbf{mean or expected value} of $X$, denoted by $E(X)$ or $\mu_X$ as:
        $$ \mu_X = E(X) = \sum_{x} x f_X(x) $$
    Remark: $E(X)$ does not necessary to be a value of $X$.

    For CRV, it is defined as:
        $$ \mu_X = E(X) = \int_{-\infty}^{\infty} x f_X(x) dx $$
    Remark: Expectation of a RV exists provided the sum or integral exists.

    \subsection{2-103 Expectation of a Function of a RV}
    For DRV and CRV respectively,
    \begin{enumerate}
        \item $E[g(X)] = \sum_x g(x)f_X(x)$
        \item $E[g(X)] = \int_{\infty}^{\infty} g(x)f_X(x)dx$
    \end{enumerate}
    Special Case: $V(X)$ below and \textbf{k-th moment of X} which is $E(X^k)$

    \subsection{2-104 Variance}
    When $g(x) = (X - \mu_X)^2$, $E(g(x))$ is called the \textbf{variance} of X.
        $$ \sigma_X^2 = V(X) = E[(X - \mu_X)^2] $$

    \textbf{Remarks}:
    \begin{enumerate}
        \item $V(X) \geq 0$
        \item $V(X) = E(X^2) - [E(X)]^2$
        \item Its principle square root is called \textbf{standard deviation}.
    \end{enumerate}

    \subsection{2-122 Properties of Expectation}
    When $a$ and $b$ are constants,
    \begin{enumerate}
        \item $E(aX + b) = aE(X) + b$
        \item $V(aX + b) = a^2V(X)$
    \end{enumerate}

    \subsection{2-137 Chebyshev's Inequality}
    Let $X$ be a random variable (DRV or CRV) with $E(X) = \mu$ and $V(X) = \sigma^2$, for any positive
    number $k$, we have:
        $$ Pr(\abs{X - \mu} \geq k\sigma) \leq \frac{1}{k^2} $$
    Alternatively,
        $$ Pr(\abs{X - \mu} \leq k\sigma) \leq 1 - \frac{1}{k^2} $$
    Remarks: This applies for \textbf{all} distribution with finite mean and variance. Only a boundary
    is given and there is no guarantee that actual value is close to this boundary.

    \section{2D RV}

    \subsection{3-10 Joint p.f./p.d.f for DRVs}
    \begin{enumerate}
        \item $f_{X,Y}(x_i, y_i) \geq 0$
        \item $\sum_{i = 1}^{\infty} \sum_{j = 1}^{\infty} f_{X,Y}(x_i, y_i) = 1$
    \end{enumerate}

    \subsection{3-21 Joint p.f./p.d.f for CRVs}
    \begin{enumerate}
        \item $f_{X,Y}(x,y) \geq 0$ for all $(x, y) \in R_{X,Y}$
        \item $\int\int_{(x,y) \in R_{X,Y}} f_{X,Y}(x,y)dxdy = 1$
    \end{enumerate}

    \subsection{3-30 Marginal Probability}
    For DRV:
        $$ f_X(x) = \sum_y f_{X,Y}(x,y) \text{ and } f_Y(y) = \sum_x f_{X,Y}(x,y) $$
    For CRV:
        $$ f_X(x) = \int_{-\infty}^{\infty} f_{X,Y}(x,y)dy \text{ and } f_Y(x) = \int_{-\infty}^{\infty} f_{X,Y}(x,y)dx $$

    \subsection{3-41 Conditional Distribution of 2D RV}
    Conditional distribution of Y \textbf{fiven that $X = x$} is given by:
        $$ f_{Y|X}(y|x) = \frac{f_{X,Y}(x,y)}{f_X(x)} \text{ , if } f_X(x) > 0 $$
    Remarks: All requirements for 1D p.f/p.d.f still applies (see \nameref{2-22} and \nameref{2-44} above).

    \subsection{3-72 Uniformly Distributed}
    When we say $X$ and $Y$ are uniformly distributed over some area, it means that $f_{X,Y}$ is a constant
    within this boundary. We can let it be k, and use summation/integration in 3-10 or 3-21 to find this value.

    \subsection{3-84 Independent 2D RVs}
    Two RVs are said to be independent iff.
        $$ f_{X,Y}(x,y) = f_X(x)f_Y(y) \text{ , for all } x,y $$

    \subsection{3-108 Expectation for 2D RVs}
    The expectation of $g(X,Y)$ is defined as
        $$
            E[g(X,Y)] = \begin{cases}
                \sum_x \sum_y g(x,y)f_{X,Y}(x,y) \text{ , for DRVs} \\
                \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} g(x,y)f_{X,Y}(x,y) dxdy \text{ , for CRVs}
            \end{cases}
        $$

    \subsection{3-109 Covariance}
    When $g(X,Y) = (X - \mu_X)(Y - \mu_Y)$, it becomes the definition of \textbf{covariance} between two RVs.
        $$ Cov(X,Y) = E[(X - \mu_X)(Y - \mu_Y)] $$

    Remarks ($a,b,c,d$ are constants):
    \begin{enumerate}
        \item $Cov(X,Y) = E(XY) - \mu_X\mu_Y$
        \item If $X$ and $Y$ are \textbf{independent}, then $Cov(X,Y) = 0$. However, the reverse is not true.
        \item $Cov(aX+b, cX+d) = acCov(X,Y)$
        \item $V(aX+bY) = a^2V(X) + b^2V(Y) + 2abCov(X,Y)$
    \end{enumerate}

    \subsection{3-112 Correlation Coefficient}
    The \textbf{Correlation Coefficient} of $X$ and $Y$, denoted by $Cor(X,Y)$, $\rho_{X,Y}$ or $\rho$, is defined by:
        $$ \rho_{X,Y} = \frac{Cov(X,Y)}{\sqrt{V(X)}\sqrt{V(Y)}} $$

    Remarks:
    \begin{enumerate}
        \item $-1 \leq \rho \leq 1$
        \item $\rho$ is a measure of the \textbf{degree if linear relationship} between $X$ and $Y$.
        \item If $X$ and $Y$ are independent, then $\rho = 0$. However, the reverse is not true.
    \end{enumerate}

    \section{Special Probability Distributions}

    \subsection{4-4 Discrete Uniform Distribution}
    All $k$ random variables all have the same probability. Hence, we have the p.f.:
        $$ f_X(x) = \frac{1}{k} \text{ for } x = x_1, x_2, \dots, x_k $$
    and 0 otherwise.

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = \frac{1}{k} \sum_{i=1}^k x_i$
        \item $\sigma^2 = \frac{1}{k}\sum_{i=1}^k (x_i - \mu)^2$ or
            $\sigma^2 = \frac{1}{k}(\sum_{i=1}^k x_i^2) - \mu^2$
    \end{itemize}

    \subsection{4-10 Bernoulli Distribution}
    There are only two outcomes, 0 and 1. We have p.f.:
        $$ f_X(x) = p^x(1-p)^{1-x} \text{ , } x = 0,1 $$
    where $p$ is a parameters and $0 < p < 1$; 0 otherwise.

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = p$
        \item $\sigma^2 = p(1-p) = pq$
    \end{itemize}

    \subsection{4-20 Binomial Distribution}
    For an RV $X$ having a \textbf{binomial distribution}, it can be seen as the sum of
    $n$ \textbf{independent Bernoulli trials}:
        $$ X = Y_1 + Y_2 + \dots + Y_n \text{ , where $Y_i$ has Bern. Dist. with p} $$
    Then we have its p.f.:
        $$ f_X(x) = {n \choose x}p^x(1-p)^{n-x} = {n \choose q}p^x q^{n-x} $$

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = np$
        \item $\sigma^2 = p(1-p) = npq$
    \end{itemize}

    \subsection{4-39 Negative Binomial Distribution}
    NBD interests in the $k$-th success occurs on the $x$-th trial. We have p.f.:
        $$ Pr(X = x) = f_X(x) = {x-1 \choose k-1} p^k q^{x-k} $$
    For $x = k, k+1, k+2, \dots$

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = \frac{k}{p}$
        \item $\sigma^2 = \frac{(1-p)k}{p^2}$
    \end{itemize}

    \subsection{4-51 Poisson Distribution}
    This describes the number of success $X$ occuring \textbf{during a given time interval or in a speficied region}.

    \textbf{Properties}:
    \begin{enumerate}
        \item $X$ in one time interval or region is \textbf{independent} of those in other \textbf{disjoint} time
            interval or region of space.
        \item $f_X(x)$ during a \textbf{very short time or in a very small region} is proportional to the length
            of the time interval or the size of the region.
        \item $f_X(x)$ for $x > 1$ is negligible in the condition of (2) above.
    \end{enumerate}

    \textbf{P.F.}:
        $$ f_X(x) = \frac{e^{-\lambda}\lambda^x}{x!} \text{ for } x = 0, 1, 2, 3, \dots $$
    where $\lambda$ is the average number of successes occuring in the given time interval or specified region.

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = \lambda$
        \item $\sigma^2 = \lambda$
    \end{itemize}

    \subsection{4-73 Poisson Approximation to Binomial D.}
    If $X \sim B(n, p)$, when $n \to \infty$ and $p \to 0$ in such a way that $\lambda = np$ remains
    constant as $n \to \infty$, then $X$ will have a approximate Poisson distribution:
        $$ \lim_{p \to 0, n \to \infty} Pr(X = x) = \frac{e^{-np}(np)^x}{x!} $$
    Remark: If $p$ is close to 1, we interchange what we defined as success and failure to get a $p$ close to 0.

    \subsection{4-81 Continuous Uniform Distribution}
    When $X \sim Uniform(a, b)$, its p.d.f. graph will be a rectangle with base $a$ to $b$ inclusive and height $1/(b-a)$.

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = \frac{a+b}{2}$
        \item $\sigma^2 = \frac{1}{12}(b-a)^2$
    \end{itemize}

    \subsection{4-90 Exponential Distribution}
    P.D.F of $X$ having this distribution:
        $$ f_X(x) = \alpha e^{-ax} \text{ for } x > 0 $$
    Note: $\int_{-\infty}^{\infty} f(x)dx = 1$

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = \frac{1}{\alpha}$
        \item $\sigma^2 = \frac{1}{\alpha^2}$
    \end{itemize}

    \textbf{No Memory Property}:
        $$ Pr(X > s + t | X > s) = Pr(X > t) $$
    Example: Probability a bulb lasts for \textbf{next} 1 month after using it for 12 months is the
    same as that for it to last for the 1st 1 month as brand new.

    \subsection{4-105 Normal Distribution}
        $$ f_X(x) = \frac{1}{\sqrt{2\pi}\sigma} \exp(-\frac{(x-\mu)^2}{2\sigma^2}), -\infty < x < \infty $$

    \textbf{Properties}:
    \begin{enumerate}
        \item Its graph is bell-shaped and symmetrical about $x = \mu$
        \item Maximum point occurs at $x = \mu$ with $f(x) = \frac{1}{\sqrt{2\pi}\sigma}$
        \item The curve approaches the $x$ axis asymptotically when going to either direction
        \item Total area under the curve is 1
        \item Two curves with same $\sigma$ will have same shape (with different center if $\mu$ is different)
        \item When $\sigma$ increases, the curve flattens (reverse: the curve sharpens)
    \end{enumerate}

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $E(X) = \mu$
        \item $V(X) = \sigma^2$
    \end{itemize}
    where $\mu$ and $\sigma^2$ are the parameters of the distribution.

    \subsection{4-110 Std. Normal Distribution}
    If $X$ has a normal distribution,
        $$ Z = \frac{X-\mu}{\sigma} $$
    has a \textbf{standardized normal distribution}, where $E(Z) = 0$ and $V(Z) = 1$.

    \subsection{4-132 Normal Approx. to Binomial D.}
    If $X \sim B(n,p)$, we have $\mu = np$ and $\sigma^2 = np(1-p)$. Then as $n \to \infty$,
        $$ Z = \frac{X-np}{\sqrt{npq}} \text{ is approx. } \sim N(0, 1) $$

    \subsection{4-136 Continuity Correction}
    This applies to normal approx. to $B(n,p)$. Known, $0 \leq X \leq n$:
    \begin{enumerate}
        \item $Pr(X = k) \approx Pr(k - \frac{1}{2} < X < k + \frac{1}{2})$
        \item \begin{itemize}
            \item $Pr(a \leq X \leq b) \approx Pr(a - \frac{1}{2} < X < b + \frac{1}{2})$
            \item $Pr(a < X \leq b) \approx Pr(a + \frac{1}{2} < X < b + \frac{1}{2})$
            \item $Pr(a \leq X < b) \approx Pr(a - \frac{1}{2} < X < b - \frac{1}{2})$
            \item $Pr(a < X < b) \approx Pr(a + \frac{1}{2} < X < b - \frac{1}{2})$
        \end{itemize}
        \item $Pr(X \leq c) = Pr(0 \leq X \leq c) \approx Pr(-\frac{1}{2} < X < c + \frac{1}{2})$
        \item $Pr(X > c) = Pr(c < X \leq n) \approx Pr(c + \frac{1}{2} < X < n + \frac{1}{2})$
    \end{enumerate}

    \section{Sampling and Sampling Distributions}

    \subsection{5-12 Sampling from a Finite Population}
    From a population of size $N$, drawing $n$ samples. If no replacement, each sample has probability off
    $\frac{1}{_NC_n}$ being chosen. If there is replacement, $\frac{1}{N^n}$.

    \subsection{5-31 Sampling Distribution of $\bar{X}$}
    For random samples of size $n$ from infinite population or finite one with replacement having population
    mean $\mu$ and population standard deviation $\sigma$, sampling distribution of $\bar{X}$:
        $$ \mu_{\bar{X}} = \mu_X \text{ and } \sigma_{\bar{X}^2} = \frac{\sigma_X^2}{n} $$
    That is,
        $$ E(\bar{X}) = E(X) \text{ and } V(\bar{X}) = \frac{V(X)}{n} $$

    \textbf{Law of Large Number}:
    When population have a finite $\sigma^2$, as the sample size increases, probability that sample mean differs
    from population mean goes to 0.

    \subsection{5-37 Central Limit Theorem}
    The \textbf{sample distribution} of $\bar{X}$ is approximately normal with mean $\mu$ and variance $\sigma^2/n$
    if $n$ is sufficiently large (say, $\geq 30$).

    If $X$ is originally normally distributed, $\bar{X}$ is normally distributed regardless the size of $n$.

    \subsection{5-52 Sampling Distribution of $\bar{X_1} - \bar{X_2}$}
    If independent samples of sizes $n_1$ and $n_2$ (each $\geq 30$) are drawn from two populations, with means
    $\mu_1, \mu_2$ and variances $\sigma^2_1, \sigma^2_2$,
        $$ \bar{X_i} - \bar{X_2} \text{ approx. } \sim N(\mu_1 - \mu_2, \frac{\sigma_1^2}{n_1} + \frac{\sigma_2^2}{n_2}) $$

    \subsection{5-59 Chi-square Distribution}
    If $Y$ is an RV and it has p.d.f.:
        $$ f_Y(y) = \frac{1}{2^{n/2}\Gamma(n/2)} y^{n/2-1} e^{-y/2} \text{ , for } y > 0 $$
    and 0 otherwise, $Y$ is defined to have a \textbf{Chi-square distribution with n degrees of freedom}, denoted by
    $ Y \sim \chi^2(n)$. $n$ is a positive integer.

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $\mu = n$
        \item $\sigma^2 = 2n$
    \end{itemize}

    \textbf{Summation}:
    For independent $Y_1 \sim \chi^2(n_1), Y_2 \sim \chi^2(n_2), \dots$, $\sum_{i = 1}^k Y_i \sim \chi^2(\sum_{i = 1}^k n_i)$.

    \subsection{5-62 Conversion to Chi-Square D.}
    \begin{enumerate}
        \item If $X \sim N(0,1)$, then $X^2 = \chi^2(1)$.
        \item If $X \sim N(\mu, \sigma^2)$, then $[(X-\mu)/\sigma]^2 \sim \chi^2(1)$.
        \item When there are $n$ random samples from a normal population, define
            $$ Y = \sum_{i=1}^n \frac{(X_i - \mu)^2}{\sigma^2} $$
            then $Y \sim \chi^2(n)$.
    \end{enumerate}

    \subsection{5-67 Sample Variance}
        $$ S^2 = \frac{1}{n-1} \sum_{i=1}^n (X_i - \bar{X})^2 $$

    \subsection{5-68 Sample Variance and Chi-sq.}
    If $S^2$ is from samples from a \textbf{normal} population having variance $\sigma^2$, then
        $$ \frac{(n-1)S^2}{\sigma^2} \sim \chi^2 (n-1) $$

    \subsection{5-69 The T-distribution}
    Suppose \textbf{independent} RVs $Z \sim N(0,1)$ and $U \sim \chi^2(n)$, and let
        $$ T = \frac{Z}{\sqrt{U/n}} $$
    then the RV $T$ follows \textbf{the t-distribution with $n$ degrees of freedom}. That is,
        $$ T \sim t(n) $$

    \textbf{Properties}:
    \begin{enumerate}
        \item The graph of t-distribution is symmetrical about $y$-axis, and is very close to that
            of the standard normal distribution.
        \item $\lim_{n \to \infty} f_T(t) = f_Z(t)$
    \end{enumerate}

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $E(T) = 0$
        \item $V(T) = n/(n-2)$ for $n > 2$
    \end{itemize}

    \subsection{5-80 The F-distribution}
    Let $U \sim \chi^2(n_1)$ and $V \sim \chi^2(n_2)$, then
        $$ F = \frac{U/n_1}{V/n_2} $$
    is called a $F$ distribution with $(n_1,n_2)$ degrees of freedom.

    \textbf{Mean and variance}:
    \begin{itemize}
        \item $E(X) = n_2/(n_2 - 2)$ with $n_2 > 2$
        \item $V(X) = \frac{2n_2^2(n_1 + n_2 - 2)}{n_1(n_2-2)^2(n_2-4)}$ for $n_2 > 4$
    \end{itemize}

    Remark: If $F \sim F(n,m)$, then $1/F ~ F(m,n)$.

    \section{Estimation Based on Normal Distribution}

    \subsection{6-11 Interval Estimation of $\mu$}
    Suppose $\sigma^2$ is known. Let
        $$ \hat{\Theta}_L = \bar{X} - 2\frac{\sigma}{\sqrt{n}} \text{ and } \hat{\Theta}_U = \bar{X} + 2\frac{\sigma}{\sqrt{n}} $$
    then we have an interval estimator of $\mu$:
        $$ (\bar{X} - 2\frac{\sigma}{\sqrt{n}}, \bar{X} + 2\frac{\sigma}{\sqrt{n}}) $$

    \subsection{6-12 Unbiased Estimator}
    A statictic $\hat{\Theta}$ is said to be an \textbf{unbiased estimator} of the parameter $\theta$ if:
        $$ E(\hat{\Theta}) = \theta $$

    \textbf{Examples}:
    $\bar{X}$ is an unbiased estimator of $\mu$ and $S^2$ is an unbiased estimator of $\sigma^2$.

    \subsection{6-17 Interval Estimation}
    We seek a random interval $(\hat{\Theta}_L, \hat{\Theta}_U)$ containing $\theta$ with a given probability
    $1 - \alpha$. That is,
        $$ Pr(\hat{\Theta}_L < \theta < \hat{\Theta}_U) = 1-\alpha $$
    and this is called \textbf{a $(1-\alpha)100\%$ confidence interval} for $\theta$.

    \subsection{6-22 Known Variance Case}
    \label{6-22}
    When population
    \begin{enumerate}
        \item has known variance and,
        \item is normal or $n$ is sufficiently large (CLT)
    \end{enumerate}
    , we can have the interval given by
        $$ (\bar{X} - z_{\alpha/2} \frac{\sigma}{\sqrt{n}} < \mu < \bar{X} + z_{\alpha/2} \frac{\sigma}{\sqrt{n}}) $$
    and the size of error can be given by
        $$ Pr(\abs{\bar{X} - \mu} < z_{\alpha/2} \frac{\sigma}{\sqrt{n}}) = 1 - \alpha $$

    \subsection{6-27 Margin of Error}
    \label{6-27}
    Let $e$ denote the \textbf{margin of error}. We want:
        $$ Pr(\abs{\bar{X} - \mu} \leq e) \geq 1 - \alpha $$
    For a given margin of error $e$, the sample size is given by
        $$ n \geq (z_{\alpha/2} \frac{\sigma}{e}) $$

    \subsection{6-32 Unknown Variance Case}
    \label{6-32}
    It needs to satisfy:
    \begin{enumerate}
        \item unknown population variance
        \item the population is \textbf{normal or very closed to normal}
        \item the sample size is \textbf{small}
    \end{enumerate}
    , then we let
        $$ T = \frac{\bar{X} - \mu}{S / \sqrt{n}} \sim t_{n-1} $$
    Hence,
        $$ Pr(-t_{n-1;\alpha/2} < T < t_{n-1;\alpha/2}) = 1 - \alpha $$
    Or:
        $$ Pr(\bar{X} - t_{n-1;\alpha/2}\frac{S}{\sqrt{n}} < \mu < \bar{X} + t_{n-1;\alpha/2}\frac{S}{\sqrt{n}}) = 1 - \alpha $$

    \subsection{6-36 Unknown Variance with Large $n$}
    When $n$ is large, we simply replace $sigma$ with $S$ in the section~\nameref{6-22} above.

    \subsection{6-43 CI for independent $\bar{X}_1 - \bar{X}_2$}
    $X_1$ and $X_2$ have to be independent. We simply replace:
    \begin{itemize}
        \item $\bar{X}$ with $\bar{X}_1 - \bar{X}_2$
        \item $\mu$ with $\mu_1 - \mu_2$
        \item $\frac{\sigma}{\sqrt{n}}$ with $\sqrt{\frac{\sigma_1^2}{n_1} + \frac{\sigma_2^2}{n_2}}$
    \end{itemize}
    in the section~\nameref{6-22} and~\nameref{6-27} above.

    \subsection{6-56 Unknown but Equal Variance}
    Conditions are the same as~\nameref{6-32} above. Let $\sigma_1 = \sigma_2 = \sigma$. Then, $\sigma^2$ can be
    estimated by the pooled variance:
        $$ S_p^2 = \frac{(n_1-1)S_1^2 + (n_2-1)S_2^2}{n_1+n_2-2} $$
    Then, substituting $S_p^2$ for $\sigma^2$, we get this statistic:
        $$ T = \frac{(\bar{X}_1 - \bar{X}_2) - (\mu_1 - \mu_2)}{\sqrt{S_p^2(\frac{1}{n_1} + \frac{1}{n_2})}} \sim t_{n_1+n_2-2} $$
    Hence, let
        $$ d = t_{n_1+n_2-2;\alpha/2} S_p \sqrt{\frac{1}{n_1} + \frac{1}{n_2}} $$
    We have the confidence interval be:
        $$ (\bar{X}_1 - \bar{X}_2) - d < \mu_1 - \mu_2 < (\bar{X}_1 - \bar{X}_2) + d $$

    \subsection{6-64 Unknown but Equal Variance with Large $n$}
    Replace $t_{n_1+n_2-2;\alpha/2}$ by $z_{\alpha/2}$ in the above formula.

    \subsection{6-70 CI for paired data}
    Observations in two samples made from the \textbf{same individual} are related and hence form a pair.
    Consider $d_i = x_i - y_i$ of paired observations. We assume $d$ is normal and we then have:
        $$ \bar{d} = \frac{1}{n}\sum_{i=1}^n d_i $$
    and point estimate of $\sigma_D^2$:
        $$ S_D^2 = \frac{1}{n-1}\sum_{i=1}^n (d_i - \bar{d})^2 $$
    Hence we have the CI for $d$:
        $$ \bar{d} - t_{n-1;\alpha/2}(\frac{S_D}{\sqrt{n}}) < \mu_D < \bar{d} + t_{n-1;\alpha/2}(\frac{S_D}{\sqrt{n}}) $$

    \subsection{6-73 CI for paired data with large $n$}
    For \textbf{sufficiently large} sample, we can replace $t_{n-1;\alpha/2}$ by $z_{\alpha/2}$ above.

    \subsection{6-78 CI for variances}
    Consider $X_1, X_2, \dots, X_n$ from (approximate) $N(\mu, \sigma^2)$ distribution. Then a point estimator of $\sigma^2$:
        $$ S^2 = \frac{1}{n-1}\sum_{i=1}^{n}(X_i-\bar{X})^2 = \frac{1}{n-1}(\sum_{i=1}^{n}X_i^2-n\bar{X}^2) $$
    We have CI when $\mu$ is known:
        $$ \frac{\sum_{i=1}^n(X_i-\mu)^2}{\chi^2_{n;\alpha/2}} < \sigma^2 < \frac{\sum_{i=1}^n(X_i-\mu)^2}{\chi^2_{n;1-\alpha/2}} $$
    When $\mu$ is unknown:
        $$ \frac{(n-1)S^2}{\chi^2_{n-1;\alpha/2}} < \sigma^2 < \frac{(n-1)S^2}{\chi^2_{n-1;1-\alpha/2}} $$
    Remark: When we want to find CI for $\sigma$, we just square root both sides in the above inequalities.

    \subsection{6-90 CI for Ratio of Two Variances}
    Consider samples $X_1, X_2, \dots, X_{n_1}$ from a $N(\mu_1, \sigma_1^2)$ population, and
    samples $Y_1, Y_2, \dots, Y_{n_2}$ from a $N(\mu_2, \sigma_2^2)$ population, and $\mu_1, \mu_2$ are \textbf{unknown}.
    Then we have the CI:
        $$ \frac{S_1^2}{S_2^2}\frac{1}{F_{n_1-1,n_2-1;\alpha/2}} < \frac{\sigma_1^2}{\sigma_2^2} < \frac{S_1^2}{S_2^2}F_{n_2-1,n_1-1;\alpha/2} $$

    \end{multicols}
\end{document}


%\addcontentsline{toc}{chapter}{Development Process}
\chapter{Experiment Methods}

\section{Overview}

The technical outcome of this project was to produce an image analysis pipeline. Broadly speaking the pipeline can be broken into four distinct components. These are feature detection and extraction, dimensionality reduction, quality evaluation, and visualisation components. This section discusses the how the chosen techniques where implemented.

\section{Techniques}

\subsection{Preprocessing}
Very little preprocessing of the images has been used in this project. Each of the mammograms (real and synthetic) has an accompanying binary mask which is used to remove the background and pectoral muscle (see section \ref{sec:datasets}). The skin around the edge of the breast can cause issues with the blob and line detection due to the intense response during image acquisition. As we are only interested in structure within the parenchyma binary morphological erosion was performed on the breast masks using a disk shaped kernel with a radius of 30 pixels to remove the edge of the breast, preserving only the parenchyma.

\subsection{Features}
\label{sec:experiment-features}
In this project we have used four different types of image features to build a feature space from the mammogram datasets. We have used two different types of shape features. One to detect blobs and one to detect linear structure. Both of these features intrinsically define a ROI within the mammogram. From the ROIs defined by these features we can extract intensity features (based on the image histogram of the ROI) and texture features (in the case of this project use the GLCM of the patch).

\subsubsection{Blob Features}
\label{subsubsec:implementation-blob-features}
The blob detection was achieved by following an approach similar to that described by Chen et al. \cite{chen2013multiscale, chen2013mammographic}. This is a multi-scale approach based on building a Laplacian of Gaussian (LoG) pyramid over ten different scales.

A LoG pyramid is created by convolving an image with a two dimensional kernel that is the combination of a Gaussian kernel with the Laplace operator over a variety of different scales. The response from filter will show sharp peaks anywhere there is a sharp change of intensity. Performing this operation over multiple scales given the resulting response different levels of granularity relative to the width of the Gaussian used.

To obtain a multi-scale view but retain comparative performance (due to the very large size of mammographic images) instead of increasing the size of the kernel the sigma of the Gaussian is fixed to 8.0 and the image is smoothed and downscaled by a factor or $\sqrt{2}$ instead. Once the image has been convolved with the LoG kernel, the resulting image is upscaled to full size. Finally a peak peak detection algorithm is run to find the all maxima within resulting filtered image.

Each of the mammographic images in the real dataset come with a set of breast masks which segment the tissue of the breast from the result of the image (such as the pectoral muscle). This helps to ensure that we are detecting the only within the parenchyma but causes issues due to a large edge response produced from the LoG kernel around the edge of the breast. In Chen's thesis this effect was dealt with by means of a ``deformable" convolution. The image is convolved with a standard image kernel when the area of the kernel is entirely within the mask area. When the kernel being convolved falls outside of the mask the filter kernel is modified so that it returns zero outside of the mask and the LoG response normalised by the number of nonzero pixels otherwise.

For each scale image produced the local maxima are detected using maximum filter and a conservative threshold. The resulting position of the peak defines the location of the blob detected in the image while the effective sigma of the Gaussian for the scale of the image $\sigma k i$ (where $i$ is the scale and $k$ is the downscale factor equal to $\sqrt{2}$) is the radius of the blob.

\begin{figure}[H]
	\label{fig:blob-detection}
	\centering
	\includegraphics[width=0.8\textwidth]{Images/blob-detections.png}
	\caption{Example image showing the raw detections (left) and the results of merging the blobs (right).}
\end{figure}

This procedure returns a very large number of responses with many overlapping detections. Since we are only interested in the blob that best characterises the size of the detected peak, a blob merging and pruning strategy is employed to remove redundant blobs. As we are only concerned with blobs that characterise patterns within the parenchyma all blobs whose radius falls outside of the bounds of the image mask are removed.

Next a thresholding technique is used to remove blobs that fall below a certain level of intensity. The area of the image covered by the blob is categorised into 9 clusters. The average intensity of each cluster is computed and the top 5 clusters are selected. The threshold used is defined to be the average intensity of the top five most intense clusters across all blobs less the standard deviation of those same clusters.

In Chen's thesis the clustering is performed using the Fuzzy c-means algorithm while in my implementation I have used the k-means algorithm. The result of this is that the algorithm requires more clusters (5 compared to only the top 3 clusters in Chen's thesis) to achieve comparable results.

After these operations the number of blobs detected is significantly reduced but there are still a large number of blobs which significantly overlap one another in high density regions. To achieve a better representation of the distribution of high density tissue in the mammogram, blobs are merged according to how they interact one another. The intersections are classified into one of three categories:

\begin{itemize}
	\item External: $d \geq r_A + r_B$
	\item Intersection: $r_A - r_B < d < r_A + r_B$
	\item Internal: $d \leq r_A + r_B$
\end{itemize}

Where $d$ is the distance between the two blobs and the $r_A$ and $r_B$ are the radii of blobs $A$ and $B$ respectively. The above definitions assume that $r_A \geq r_B$.

Merging proceeds as follows: if a blob is external then it remains retained. If a blob is internal to a larger (coarser) blob it will be removed. If the blobs intersect one another and they are closely located ($d \leq r_A + \alpha r_B$ for $0 \leq \alpha \leq 1$, assuming $r_A \geq r_B$). In the experiments documented in the report overlap parameter $\alpha$ used was 0.01.

For each of the blobs detected, the coordinates of the blob and the radius (i.e. the sigma of the Gaussian associated with the scale the blob was detected at) are saved to a comma separated value (CSV) file. This file was then loaded into the IPython notebook system for further processing, dimensionality reduction, and visualisation. Using the analysis python module developed as part of the general API for this project the following features were calculated from the radius and position of the blobs for each image:

\begin{itemize}
	\item Number of blobs detected
	\item Average radius
	\item Standard deviation of the radius
	\item Min \& max radius
	\item Small/medium/large radius count: Each radius was binned into three separate equally sized bins across the range scales used to detect blobs.
	\item Density: the average distance between this blob and the $k$ nearest blobs. In all experiments $k$ was set to 4.
	\item 25/50/75 percentiles
	\item Count of blobs above the mean.
\end{itemize}

\subsubsection{Line Features}
Along with shape features based on blobs of high intensity a shape feature based on finding linear structure within a mammogram was used. Linear structure aims to try and characterise the ductal shapes visible in a typical mammogram. For the implementation of this feature we follow the work of Zwiggelaar et al \cite{zwiggelaar1996finding} and use an orientated bins method to pick out ROIs which correspond to ductal structure visible in mammograms.

The orientated bins method proposed in reference \cite{zwiggelaar1996finding} filters the local neighbourhood of an image by dividing the neighbourhood window into $n$ angular bins with an angular resolution of $\frac{2 \pi}{n}$. The line strength of the local neighbourhood is given by computing the difference between the maximum average intensity of opposing bins and the average intensity of the local neighbourhood as a whole. In the case of a well defined line only one set of opposing bins will give a high response. The orientation of the line can also be derived from the orientation of the bin with the largest intensity.

\begin{figure}[H]
	\label{fig:linear-structure}
	\centering
	\includegraphics[width=0.5\textwidth]{Images/linear-structure.png}
	\caption{Example image showing the orientated bins feature used to detect linear structure.}
\end{figure}

This operation results in two different filtered images. The first is the line strength image. Each pixel in the line strength image represents the average value from the maximum bin of the filtering. The second image represents the orientation and each pixel in the orientation image is a number representing the direction that the maximum bin was orientated in.

Once the line strength and orientation images have been generated the results can be enhanced by applying non-maximal suppression \cite{sonka2014image}. For each pixel in the line strength image non-maximal suppression selects the two pixels perpendicular to the current pixel based on the corresponding pixel in the line orientation image. If either of the two perpendicular neighbouring pixels are more intense than the current pixel the pixel is ignored.

After suppression the image is once thresholded using a conservative value to remove noise and the image is converted to a binary image. A morphological dilation operation is used to improve the connectivity of the line shapes detected from the image by increasing the size very slightly to connect any disconnected components. Any points which are within an 8-connectivity of one another are counted as being part of the same structure.

The resulting shape features can be measured by standard first order statistics to produce descriptive features about the blobs and lines detected from the mammogram. As with blob features the detections were exported to a CSV file and then loaded into a IPython notebook for further analysis (as with blob features). Features created from the distribution of areas detected by the line features were:

\begin{itemize}
	\item Number of lines detected
	\item Average area
	\item Standard deviation of the areas
	\item Min \& max area
	\item 25/50/75 percentiles
	\item Count of areas above the mean.
\end{itemize}

\begin{figure}[H]
	\label{fig:feature-diagram}
	\centering
	\includegraphics[width=0.9\textwidth]{Images/features-diagram.png}
	\caption{Diagram showing how features are generated in the pipeline. First blobs and lines are detected and features are generated from these. These ROIs defined by these detections are then used to produce intensity and texture features.}
\end{figure}

\subsubsection{Intensity}
The shape features detected using orientated bins and multi-scale blobs define regions of interest across the breast. From these ROIs the patch of the image which is covered by the area or radius of the shape feature can be extracted. The histogram of the intensity values of this image patch provide can provide additional discriminative information about ROI. Descriptive statistics derived from the histogram of the ROI were computed. The final features derived were:

\begin{itemize}
	\item Number of intensity values
	\item Mean
	\item Standard deviation.
	\item Min \& max values
	\item 25/50/75 percentiles
	\item Skew
	\item Kurtosis
\end{itemize}

\subsubsection{Texture}
As with intensity features, texture features can be extracted from the patches defined by the shapes features as well. In this project we have only used texture features derived from the grey-level co-occurrence matrix \cite{haralick1973textural}. The properties computed from the co-occurrence matrices were homogeneity, dissimilarity, energy, and contrast. The definitions of each are given as the following:

Homogeneity:
\begin{equation}
	\sum\limits_{i,j=0}^{levels-1} \frac{P_{i,j}}{1+(i-j)^2}
\end{equation}

Dissimilarity:
\begin{equation}
	\sum\limits_{i,j=0}^{levels-1} P_{i,j}|i-j|
\end{equation}

Energy (or the square root of the angular second moment):
\begin{equation}
	\sqrt{ \sum\limits_{i,j=0}^{levels-1} P_{i,j}^2 }
\end{equation}

Contrast:
\begin{equation}
	\sum\limits_{i,j=0}^{levels-1} P_{i,j}(i-j)^2
\end{equation}

The experiments performed in chapter \ref{subsec:results-texture} all used a distance of 1 and a combination of eight different orientations of angles 0.0, 22.5, 45.0, 67.5, 90.0, 112.5, 135.0, 157.5 degrees.

When deriving texture and intensity features from ROIs defined by blobs features are generated for every blob detected. Because all of the detected blobs vary in scale, the final texture/intensity features are grouped and then averaged according to the scale of the ROI. In all experiments reported there were 10 scales. There were four texture features used so the resulting feature space was $4 \times 10 = 40$ dimensions. A similar process is used for intensity features derived from blobs. Lines are detected over only one scale so the results for each are simply averaged.

Some blob based texture/intensity features will produce zero entries for blobs of larger scales because there were none of that scale detected in that particular image. This causes the projections produced to be split into clusters according to what the size of the largest blob detected was. To prevent this unwanted effect zero entires were replace by the mean value for the relevant scale.

\subsection{Dimensionality Reduction}
In this project the three types of dimensionality reduction algorithm have been used to produce the lower dimensional representations from the higher dimensional feature spaces. The implementation of all algorithms used in this project are standard implementations available through the scikit-learn library.

Before running the dimensionality reduction algorithm the input feature matrix is standardised by removing the mean from each feature and scaling to standard variance. This ensures that the all of the features will roughly on the same scale as one another and therefore that the distance metric used to compute neighbours shouldn't be heavily weighted in favour of feature orders of magnitude larger than the others.

These three algorithms were chosen because they exhibit a variety of different properties. t-SNE is a fairly recent technique that is quite popular and is well known to produce good visualisations of higher dimensional data and is good at preserving local structure but fails to produce good global representations. Isomap was chosen in sharp contrast to this because it is a well know algorithm that attempts to preserve global structure but may not produce visualisations that are as good as t-SNE. LLE, being an approach based on eigendecomposition is similar to Isomap, but also preserve local structure over global structure similar to t-SNE.

\subsection{Visualisation}
The visualisation aspects of this project have largely been handled by the inbuilt functionality that is available in the matplotlib and pandas Python libraries. However some additional visualisation techniques have also been implemented for and used for examining what the images look like for each point in the lower dimensional mapping.

\subsubsection{Visualisation of Images from Mapping}
In order to examine the how images change across the lower dimensional mapping and to try and understand why images are grouped closely together a small utility was created that would read in the mapping of the feature space and display a scatter plot of the mapping. When hovering over each point in the visualisation the image corresponding to that data point is displayed to the right of the scatter plot.

\begin{figure}[H]
	\label{fig:nd-viz}
	\centering
	\includegraphics[width=0.9\textwidth]{Images/nd-viz.png}
	\caption{Web based scatterplot visualisation tool. Hovering over points in the scatterplot on the left changes the image on the right to match the image corresponding to the data point.}
\end{figure}

This is the only part of the project that was not implemented in Python. Due to time constraints it was deemed not worth exerting a huge amount of time on developing this feature, but rather to just get something rapidly working. Ideally this would have been implemented in something like pyQt and utilised the matplotlib library. However, the actual implementation is written in JavaScript, HTML, and CSS. The main driving principle behind this is that it was quicker write some basic CSS to position the image and scatter plot next to one another than it was to write a proper Python GUI from scratch.

\subsubsection{Median Image Plot}

\begin{figure}[H]
	\label{fig:median-image}
	\centering
	\includegraphics[width=0.8\textwidth]{Images/median-image.png}
	\caption{The median image plotting routine takes a mapping and creates a 2D histogram. The ``median" image from each bin is then chosen to represent the whole bin.}
\end{figure}

Similarly to the previous visualisation this visualisation takes a projection of the feature space and creates a two dimensional histogram of the points. From each of the resulting bins the median point in both the $x$ and $y$ directions if found. The image which corresponds to this point is selected to be used as part of the visualisation. Each of the images is stitched together to form a matrix of images in the same shape as the lower dimensional mapping. Due to the huge size of mammographic images the images used in this visualisation are downscaled by a factor of 8. This keeps the visualisation generation performance reasonable at the expense of loosing some of the finer detail in the images.

\section{Datasets}
\label{sec:datasets}
Two different datasets were used as part of this project. One consisting of mammograms taken from a collection of real patients. The other dataset is a collection of artificially generated breast phantoms generously created by the University of Pennsylvania.

\subsection{Real Data}
The dataset of real mammograms was taken from a private dataset captured using a Hologic full field digital mammography system. The dataset contains images of 90 unique patients each with a craniocaudal and mediolateral oblique view of both the left and right breasts, resulting in 360 total images. Each image in the dataset also has a corresponding binary breast mask. This mask is used to segment the  background and the pectoral muscle from the breast parenchyma.

\subsection{Synthetic Data}
The synthetic breast phantoms were generated by the University of Pennsylvania using the techniques outlined by Bakic et al. \cite{bakic2002mammogram1, bakic2002mammogram2, bakic2003mammogram3}. Their simulation system consists of three major components: a breast tissue model, a compression model, and an acquisition model. Adipose tissue compartments within the breast are modelled using thin shells in areas of primarily adipose tissue and as blobs in areas of predominantly fibroglandular tissue. Ductal lobes are also simulated by the model using a randomly generated tree.

As the phantoms have not been formally assigned a BI-RADS class by an expert radiologist (as in the case of the real mammogram dataset) the ground truth for the risk associated with a particular mammogram is given by it's volumetric breast density (VBD). This information was supplied in the meta-data produced alongside the phantom mammograms.

None of the breast phantoms came with masks, but the masks are an essential part of blob detection. As none of the phantoms contain a pectoral muscle the masks were generated by simply using Otsu's method \cite{otsu1975threshold} to provide a threshold used to segment ``breast" from ``not breast" and create a binary mask for each phantom.

\section{Comparison of Features}
\label{sec:comparison-of-features}
A comparison of the features can be made using the Kolmogorov-Smirnov two sample test. This is a non parametric test used to compare the difference between two probability distributions. In the one sample version of the test an empirical distribution function is compared with a reference cumulative distribution function (such as the normal distribution). The test quantifies the distance between the two distributions. The two sample version of the test is used to measure if two empirical distribution functions differ. Formally, the two sample Kolmogorov-Smirnov test is given by \cite{realStatsTwoSampleKStest}

\begin{equation}
	D_{m,n} = \max\limits_x | F(x) - G(x) |
\end{equation}

Where $F$ and $G$ are cumulative distribution functions of size $m$ and $n$ respectively. The null hypothesis that both samples come from a population that is similarly distributed can be rejected if $D_{m,n}$ is greater than the critical value for the corresponding significance level $D_{m,n,a}$ which is given by

\begin{equation}
	D_{m,n,a} = c(a) \sqrt{ \frac{m+n}{mn} }
\end{equation}

Where $c(a)$ is the inverse Kolmogorov distribution at $a$. The Kolmogorov-Smirnov two sample test can be applied to each feature extracted from both real and phantom mammograms. This will be used to quantitively compare how well each of the features extracted compare between datasets.

\section{Evaluating the Quality of the Mapping}
As discussed in section \ref{sec:quality-measures} there are several quality metrics that can be used to measure the quality of a mapping produced from a dimensionality reduction algorithm which are independent of the technique being used.

In this project two different types of quality measures derived from the co-ranking matrix are implemented. The chosen metrics are trustworthiness and continuity and the local continuity meta-criteria. These metrics were chosen because together they can be used to measure both the false positives/negatives present in the mapping (for trustworthiness and continuity) and the true positives/negatives (LCMC).

\section{Implementation}
\label{sec:implementation}
This section provides a brief overview of the implementation details used in the project. All of the methods discussed in the preceding sections were implemented in Python. The technical output of the project is a Python library and command line tool which is used to create the methods discussed in the first part of this chapter.

The final implementation produced for this project is a Python library that includes a small command line interface. The feature detection part of the pipeline takes a long time to process and subsequently the commands for running this were made into a CLI that can be left to run overnight.

However the second half of the ``pipeline" has not been fixed in the same way. The subsequent analysis of the generated features was performed by means of a IPython notebooks. This was not made into a fixed pipeline because it is often desirable to explore the extracted features in an interactive way. Instead commonly performed operations were identified and moved the most appropriate module as the project progressed. This evolved over time into the final submitted API.

\subsection{Python Package}
The majority of the components used in the project are built upon the top of the scipy stack \cite{jones2014scipy}. Two major additional libraries (which also rely on the scipy stack) which have been used heavily in the project are the scikit image \cite{van2014scikit} and scikit learn \cite{pedregosa2011scikit} projects.

The library created as part of this project forms a complete python package. The package consists of five top level modules and a collection of submodules implementing specific functionality relating to the features detected by the system. The description of each of the modules is as follows:

\begin{itemize}
	\item {\bf analysis}: The analysis module implements commonly used functions used to analyse the images after feature extraction has been performed. These functions are typically called directly from the python interpreter to in a IPython notebook session.
	\item {\bf coranking} This module contains the code for computing the co-ranking matrix form two representations of the same dataset.
	\item {\bf plotting}: This module provides a collection of custom, convenience plotting functions that largely depend on the matplotlib library.
	\item {\bf io\_tools}: The io\_tools module implements functions for iterating over directories of images and there corresponding masks as well as image loadings and preprocessing functions.
	\item {\bf utils}: The utils module contains a collection of miscellaneous helper functions used in various places throughout the package.
\end{itemize}

In order to automatically detect features across a whole dataset of images a sub-package called reduction was created. This package handles the loading and multiprocessing of multiple images in a dataset. While most of the implementation of done as a purely functional API sub-package was developed using classes to make it more easily extensible for new image reductions.

\begin{itemize}
	\item {\bf reduction}: Contains an abstract class Reduction which defines a generic reducer interface. This is a super class to all processes that require iterating over a dataset to carry out an operation on each image (i.e. feature detection).
	\item {\bf multi\_processed\_reduction}: Derived from the Reducer class in the reduction module this adds multiprocessing support to the original reducer base class. (see subsection \ref{subsec:multiprocessing}
	\item {\bf reducers}: This is a collection of classes that implement the Reduction and MultiProcessedReduction abstract classes to provide specific functionality for each feature detection process.
\end{itemize}

An additional sub-package contains the modules used by the reduction sub-package to perform feature extraction. The package includes the following modules:

\begin{itemize}
	\item {\bf blobs}: Contains the code implementing blob detection using the approach outlined in section \ref{sec:features}. This also uses an additional private module which provides the code for the graph built as part of the blob merging scheme.
	\item {\bf linear\_structure}: Contains the code implementing line feature detection using the approach outlined in section \ref{sec:features}. This also uses a couple of additional private modules within the sub-package which provide the code for non-maximal suppression and the orientated bins feature.
	\item {\bf texture}: Contains the code for computing the GLCM and texture features from ROIs.
	\item {\bf intensity}: Contains the code for computing first order statistics from ROIs.
\end{itemize}

The last part of the library is the deformable convolution module. This implements the deformable convolution approach outlined by Chen et al \cite{chen2013multiscale}. Due to the performance considerations associated with convolution I chose not to write this module directly in Python but to implement it as a C function which is compiled directly using the Python C API.

\subsection{Command Line Interface}
The command line tools offered by the program are built using the Click library \cite{clickLibrary}. The CLI provides a thin wrapper to the some of the higher level library functions available in the package. The most important commands offered by the CLI are those concerned with image processing to collect features from the images. These functions involve iterating over a folder of images and applying the feature extraction techniques outlined in section  \ref{sec:features}. These operations can take in the order of several hours to complete and so it is useful to have them exposed on the command so they can be run and left until complete.

These commands for image reduction also add the ability to automatically dump the output of the reduction to a CSV file named by the user upon completion. Typically these commands take two directories (one for the images and one for the masks) and the name of the output file. In the case of intensity and texture features they also take an additional file containing the ROIs from which to extract features.

\subsection{Multiprocessing}
\label{subsec:multiprocessing}
The image processing component of the pipeline created for this project is extremely time consuming. The two shape features used in this project take up to a minute to process a single image. The biggest contribution to the running time is the filtering and convolution of the images. Sadly this is an operation that is intrinsically complex and therefore slow. However, since the processing of an image is completely separate from all other images multiprocessing support becomes an option.

The feature detection routines produces as part of the Python package all support multiprocessing through the Python multiprocessing module. Each routine has support to use an unlimited number of threads although in practice the physical limitations of the computer hardware used limited the number to just two. Despite only using one additional process this still managed to speed up the feature detection routines for the whole set of images by several hours.


\chapter*{Appendix B: Wavelet Packets}
\addcontentsline{toc}{chapter}{Appendix B: Wavelet Packets}

Wavelet packets were introduced by Coifman, Meyer and Wickerhauser
\cite{wave:coifman92}. Instead of dividing only the approximation space,
as in the standard orthogonal wavelet transform,
detail spaces are also divided.

\begin{figure}[htb]
\centerline{
\hbox{
\psfig{figure=fig_packet.ps,bbllx=1cm,bblly=5cm,bburx=17cm,bbury=23cm,width=12cm,height=15cm,clip=}}
}
\caption{Wavelet packet scheme: (a) forward direction, (b) reconstruction.}
\label{fig_packet}
\end{figure}

Figure~\ref{fig_packet} shows the wavelet packet scheme.


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{November 24, 2014}
\maketitle
\section*{5.1\#7}
$1=(1-a)^n=(1-a)(1+a+a^2+\dots+a^(n-1)$

$u-a=a(1-u^{-1}a)$ and note that $u^{-1}a=b$ is nilpotent and so $(1-b)$ is invertible and then $u-a$ is invertible
\section*{5.1\#8}
$a+a=(a+a)^2=a^2+a^2+a^2+a^2=a+a+a+a\to0=a+a$

\section*{ring homomorphisms}
for $R,S$ commutative rings, $\varphi:R\to S$ is a ring homomorphism if
\begin{enumerate}
\item
$\varphi(a+b)=\varphi(a)+\varphi(b)$
\item
$\varphi(ab)=\varphi(a)\varphi(b)$
\item
$\varphi(1_R)=1_S$ (not always defined with this property, but are in this class)
\end{enumerate}

\subsection*{prop}
given that $\varphi:R\to S$ is a ring homomorphism then
\begin{enumerate}
\item
$\varphi(0_R)=0_S$
\item
$\varphi(-a)=-\varphi(a)$

because $\varphi(0+(-1)a)=\varphi(0)+\varphi(-1)\varphi(a)$?
\end{enumerate}

\subsection*{def}
we say the $\varphi$ is a isomorphism if it is bijective

\section*{exercises}
if $\varphi$ is an isomorphism then $\varphi^{-1}$ is also an isomorphism

also composition of isomorphisms are isomorphisms (transitivity)
\section*{examples}
$i:\mathbb{C}\to\mathbb{C}[x]$ where $i(a)=a$ is a ring homomorphism

$\mathbb{Z}\to\mathbb{Z}_n$ is not injective because it is an infinite set onto a finite set (pigeonhole principle)

$K\to K[x]/\langle f(x)\rangle$ where $\varphi(a)=[a]$

$K[x]\to K$. Fix $\alpha\in K$ and for each $\alpha$ we define $\varphi_\alpha:K[x]\to K$ and so $\varphi_{\alpha}(f(x))=f(\alpha)$ is called evaluation function.

\subsection*{def}
given a ring homomorphism where $\varphi:R\to S$ then $\ker \varphi=\{x\in R:\varphi(x)=0\}$

\section*{proposition}
if $R$ and $S$ are commutative rings and we have $\varphi$ a ring homomorphism, then
\begin{enumerate}
\item
for every $a,b\in\ker\varphi$ we have that $a-b$ and $a+b$ are also elements in $\ker\varphi$.
\item
for every $r\in R$ and every $a\in\ker\varphi$ we have $ra\in\ker\varphi$
\end{enumerate}
\subsubsection*{proof}
1 follows because $\ker\varphi$ is an additive subgroup of the abelian group $(R,+)$.

2 folows because $\varphi(ra)=\varphi(r)\varphi(a)=\varphi(r)\cdot0=0$

\section*{construction}
two rings and a ring homomorphism $\varphi:R\to S$ and on $R$ we define an equivalence relation $x\sim_\varphi y\Leftrightarrow \varphi(x)=\varphi(y)$ where $[x]$ is the equivalence class of $x$ where $R/\ker\varphi=\{[x]:x\in R\}$. On $R/\ker\varphi$ we have the well defined operation $[x]+[y]=[x+y]$

now we define a new operation on the set $[x][y]=[xy]$

is this well defined?

$[x]=[x']\to\varphi(x)=\varphi(x')$ and similarly with $y$ and so $[x'][y']=\varphi(x')\varphi(y')=\varphi(x)\varphi(y)=\varphi(xy)=[xy]=[x'y']$
\end{document}


% This LaTeX was auto-generated from an M-file by MATLAB.
% To make changes, update the M-file and republish this document.

%%% \documentclass{article}
%%% \usepackage{graphicx}
%%% \usepackage{color}

%%% \sloppy
%%% \definecolor{lightgray}{gray}{0.5}
\setlength{\parindent}{0pt}

%%% \begin{document}

\subsection*{Phase Sensitive Frequency Estimator}

\begin{par}
Example for algorithm PSFE.
\end{par} \vspace{1em}
\begin{par}
PSFE is an algorithm for estimating the frequency, amplitude, and phase of the fundamental component in harmonically distorted waveforms. The algorithm minimizes the phase difference between the sine model and the sampled waveform by effectively minimizing the influence of the harmonic components. It uses a three-parameter sine-fitting algorithm for all phase calculations. The resulting estimates show up to two orders of magnitude smaller sensitivity to harmonic distortions than the results of the four-parameter sine fitting algorithm.
\end{par} \vspace{1em}
\begin{par}
See also Lapuh, R., "Estimating the Fundamental Component of Harmonically Distorted Signals From Noncoherently Sampled Data," Instrumentation and Measurement, IEEE Transactions on , vol.64, no.6, pp.1419,1424, June 2015, doi: 10.1109/TIM.2015.2401211, URL: \url{http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7061456&isnumber=7104190}'
\end{par} \vspace{1em}

\subsubsection*{Contents}

\begin{itemize}
\setlength{\itemsep}{-1ex}
   \item Generate sample data
   \item Call algorithm
   \item Display results
\end{itemize}

\subsubsection*{Generate sample data}

\begin{par}
Two quantities are prepared: \lstinline{Ts} and \lstinline{y}, representing 1 second of sinus waveform of nominal frequency 100 Hz, nominal amplitude 1 V and nominal phase 1 rad, sampled with sampling time 0.1 ms.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI = [];
Anom = 1; fnom = 100; phnom = 1;
DI.Ts.v = 1e-4;
t = [0:DI.Ts.v:1-DI.Ts.v];
DI.y.v = Anom*sin(2*pi*fnom*t + phnom);
\end{lstlisting}
\begin{par}
Add noise:
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI.y.v = DI.y.v + 1e-3.*randn(size(DI.y.v));
\end{lstlisting}

\subsubsection*{Call algorithm}

\begin{par}
Use QWTB to apply algorithm \lstinline{PSFE} to data \lstinline{DI}.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DO = qwtb('PSFE', DI);
\end{lstlisting}

        \begin{lstlisting}[style=output]
QWTB: no uncertainty calculation
\end{lstlisting} \color{black}

\subsubsection*{Display results}

\begin{par}
Results is the amplitude, frequency and phase of sampled waveform.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
f = DO.f.v
A = DO.A.v
ph = DO.ph.v
\end{lstlisting}

        \begin{lstlisting}[style=output]

f =

  100.0000

A =

    1.0000

ph =

    1.0000

\end{lstlisting} \color{black}
    \begin{par}
Errors of estimation in parts per milion:
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
ferrppm = (DO.f.v - fnom)/fnom .* 1e6
Aerrppm = (DO.A.v - Anom)/Anom .* 1e6
pherrppm = (DO.ph.v - phnom)/phnom .* 1e6
\end{lstlisting}

        \begin{lstlisting}[style=output]

ferrppm =

    0.0082

Aerrppm =

   -4.5238

pherrppm =

   12.4392

\end{lstlisting} \color{black}

%%% \end{document}


%!TEX root=report.tex
\subsection{GMM clustering}
\label{section:result-gmm}

Like with K-means clustering, a matrix $X$ with dimensions ($64800 \times 341$) is used. Due to the high number of variables in GMM, the dimensionality was reduced using PCA and only selecting the first 10 PCs.
The results were not ideal, so instead a kernel version of PCA was used.

\begin{itemize}
\item The first 10 principal components were used
\item The ``Radial basis function'' was used as a kernel function
\item The amount of degree is 3
\item The gamma coefficient is set to $\sfrac{1}{341}$
\end{itemize}

\begin{figure}[H]
	\center
	\includegraphics[width=\textwidth]{figures/gmm-pca-explaned}
	\caption{Variance explained for the first 25 PCs, calculated as eigenvalues in $K = H(X) H(X)^T$.}
	\label{fig:gmm-pca-explaned}
\end{figure}

As seen in Figure \ref{fig:gmm-pca-explaned}, by selecting the first 10 PCs 93.18\% of the variance in the data (in the kernel space) is explained.

Next the GMM with 6 clusters was computed using the projected data.
\begin{figure}[H]
	\center
	\includegraphics[width=\textwidth]{figures/gmm-world}
	\caption{Each position is a point, the color then matches a given cluster.}
\end{figure}

\begin{figure}[H]
	\center
	\includegraphics[width=\textwidth]{figures/gmm-centroids}
	\caption{The mean vectors for each Gaussian component, projected back in the original space, using the inverse kernel PCA transformation.}
\end{figure}

Comparing the two plots above:
\begin{itemize}
	\item Light green have a slight mass increase, this was also seen in K-means clustering. Since this is observed in North America, it is likely caused by post glacial rebound.
	\item  Red and light blue correlates with extreme and regular seasonality, also very much like K-means.
	\item Blue and orange correspond with trending mass loss. With the same locations as in K-means.
	\item The big difference between GMM and K-means, is that the world graph is a lot less noisy. Only those positions with a significant different behavior than normal (green) have been caught by GMM.
	\item Finally the pink cluster is almost not noticeable and could be considered redundant.
\end{itemize}


\chapter{Rotor-dynamic analysis}

\noindent
At this point, the tail-rotor behaviour has been investigated implementing a simplified FEM model and taking advantage of the Rotordynamics capabilities of Ansys. \\
Rotor-dynamics is the study of vibrational behaviour in axially symmetric rotating structures. At high rotational speeds, such as in a helicopter's tail rotor, the inertia effects of the rotating parts must be consistently represented in order to accurately predict the rotor behaviour. An important part of the inertia effects is the \underline{gyroscopic moment} introduced by the precession motion of the rotor which is function of the spin velocity. Hence, the velocity term in the equation of motion as well as the support flexibility and damping behaviour cannot be neglected and they are important factors in enhancing the stability of the vibrating rotor.

\section*{Modal analysis of rotating structures}
\addcontentsline{toc}{section}{Modal analysis of rotating structures}
\noindent
The modal analysis allows for the calculation of natural frequencies and critical speeds (Campbell diagram) of the rotor. \\
Dinamically speaking, the equations of motion of a generic rotating structure is:

\begin{equation*}
\left[ M \right] \left\lbrace \ddot{u} \right\rbrace + \left( \left[ C \right] + \left[ G \right] \right) \left\lbrace \dot{u} \right\rbrace + \left( \left[ K \right] - \left[ K_c \right] \right) \left\lbrace u \right\rbrace = \left\lbrace 0 \right\rbrace
\end{equation*}

\noindent
where [G] is the gyroscopic matrix that depends on the rotational velocity and is the major contributor to tailboom's rotor, while [$K_c$], the spin softnening matrix, also depends upon the rotational velocity and it modifies the apparent stiffness of the structure. \\
This equation holds when motion is described in a stationary reference frame.

\noindent
\underline{STEPS FOR MODAL ANALYSIS IN ANSYS}: \\
1) Model implementation; \
2) Boundary conditions; \
3) Solution includung rotational effects (centrifugal and Coriolis); \
4) Postprocessing.

\subsection*{Tail rotor simplified model}
\addcontentsline{toc}{subsection}{Tail rotor simplified model}
\noindent
A simplified model of the tail rotor has been defined, and it consists in the following parts:
\begin{itemize}
	\item \textbf{SHAFT}: elastically supported shaft modelled with BEAM 188 elements;
	\item \textbf{ROTOR'S HUB}: modelled with a lumped mass and inertia concentrated in the center of the rotor attached to a master node;
	\item \textbf{ROTOR}: modelled with a circular ring of SHELL 181 elements with radius passing through to the center of mass of the blades. CERIG elements have been introduced in order to connect the master node (hub) to the slave nodes of the ring.
\end{itemize}

\medskip
\begin{figure}[h]
	\begin{center}
		\centering
		\includegraphics[width=0.7\linewidth]{PICTURES/5_Rotordynamics/scheme.png}
	\end{center}
	\caption {Scheme of the simplified tail rotor model}
\end{figure}
%\vspace{0.5cm}

\medskip
\begin{figure}[h]
	\begin{center}
		\centering
		\includegraphics[width=0.9\linewidth]{PICTURES/5_Rotordynamics/2.png}
	\end{center}
	\caption {Tail rotor simplified model}
\end{figure}
%\vspace{0.5cm}

\subsection*{Model assumptions}
\addcontentsline{toc}{subsection}{Model assumptions}
\begin{itemize}
	\item Axial-symmetric structure (requirement for rotor-dynamics);
	\item Linear elastic material properties;
	\item Rotor elastically supported (2 bearings);
	\item Aerodynamic loads not considered;
	\item Rigid rotor and connections (no hinges or flexible joints).
\end{itemize}

\subsection*{Applied boundary conditions}
\addcontentsline{toc}{subsection}{Applied boundary conditions}
\noindent
\begin{itemize}
	\item Only fixed constraints are allowed for rotor-dynamic analysis;
	\item Support elasticity modelled using COMBIN14 elements to represent bearings;
\end{itemize}

\noindent
\textbf{COMBIN14} \\
Bearings have been modelled with COMBIN14 element whose properties simulate the effect of a longitudinal spring-damper as a uniaxial tension-compression element.
The element is created between two nodes which, in our case, are overlapped. One of the nodes is rigidly attached to the shaft while the other one is constrained on the ground. These elements allows the elastic movement of the shaft in the Y and Z directions. \\

\medskip
\begin{table}[h!]
	\centering

	\begin{tabular}{c c}
		\toprule
		\multicolumn{2}{c}{Bearing's properties}\\
		\midrule
		Type & Stiffness K (N/m)  \\
		\midrule
		7 balls bearing & $378e+7$  \\
		\bottomrule
	\end{tabular}
\end{table}

\clearpage
\subsection*{Solution including rotational effects (centrifugal and Coriolis)}
\addcontentsline{toc}{subsection}{Solution including rotational effects (centrifugal and Coriolis)}
\noindent
The modal analysis must be solved using an algorithm for damped modal analysis (complex Eigenvalues and Eigenvectors). We have chosen the \textbf{QRDAMP} solver including the rotational effects (\textbf{CORIOLIS, ON, , , ON}). \\ The rotation speed (2000 RPM) has been divided in several load-steps. 10 modes have been extracted from each step. \\

\lstinputlisting[firstline=133, lastline=146, language=apdl-modified]{./COMMAND_LISTS/SOLO_ROTOR.txt}

\subsection*{Postprocessing}
\addcontentsline{toc}{subsection}{Postprocessing}
\noindent
Rotor's natural frequencies have been calculated for each value of the rotation speed (hence for each load step) and assigned to a substep. Resulting natural frequencies vary with the rotor speed as it is displayed in the campbell diagram below. \\
A \textbf{critical speed} appears when the natural frequency is equal to the excitation frequency, and excitation may come from unbalance that is synchronous with the rotational velocity. \\
Critical speeds are directly determined by solving a new eigenvalue problem or by performing a Campbell diagram analysis, where the intersection points between the frequency curves and the excitation line are calculated.

\noindent
The rotational velocity of the rotor is specified via the \textbf{CMOMEGA} command which requires to specify which is the rotating component, previously defined and selected and to input the velocity vector (magnitude and direction). \\
Then, we can set the \textbf{CAMPBELL, ON} Ansys' command. \\

\noindent
\textbf{NOTE:} \\
Bearing stiffness has an important effect on the critical speeds. When analyzing a rotor, it is important to understand the effect of the bearing stiffness on the critical speeds and this can be done drawing the "Critical Speed Map" (here neglected).

\clearpage
\medskip
\begin{figure}[h]
	\begin{center}
		\centering
		\includegraphics[width=0.95\linewidth]{PICTURES/5_Rotordynamics/campbell1.png}
	\end{center}

\end{figure}
%\vspace{0.5cm}

\begin{table}[h!]
	\centering
	\pgfplotstableset{
		% global config, for example in the preamble
		% these columns/<colname>/.style={<options>} things define a style
		% which applies to <colname> only.
		every head row/.style={before row=\hline,after row=\hline},
		every last row/.style={after row=\hline},
		display columns/0/.style={column name =Num, int detect,column type=r},
		display columns/1/.style={column name =Critical Speed [rpm], column type=r,
			fixed,fixed zerofill,precision=5,set thousands separator={\,}},
		%other style option
	}
	\pgfplotstabletypeset[col sep=space]{VelocCritic-ModalAnalisys.txt}
	\caption{Natural frequencies for the simple model}
	\label{tab:ModalFreq-Shellmodel}
\end{table}

\clearpage
\smallskip
\begin{figure}[h!]
	\begin{center}
		\centering
		\includegraphics[width=0.9\linewidth]{PICTURES/2_Lama_truss/PNG/model2/hqdefault2}
	\end{center}
	\caption{Tail rotor assembly}
\end{figure}
%\vspace{0.5cm}


% Author: Jonathan Siegel
% This chapter aims to give a basic introduction to convex optimization.
% To stay focused, we will only consider smooth convex functions and
% first-order (i.e. gradient-based) methods.

\chapter{Convex Optimization}
\section{Introduction}
\subsection{Basic Notions}
\section{Gradient Descent}
\subsection{Strongly Convex Case}
\subsection{Convex Case}
\section{Accelerated Gradient Descent}
\subsection{Strongly Convex Case}
\subsection{Convex Case}
\section{Mirror Descent}
\subsection{Bonus: Accelerated Mirror Descent}
\section{Stochastic Gradient Descent}
\subsection{Strongly Convex Case}
\subsection{Convex Case}


\documentclass[letterpaper]{article}

\usepackage[utf8]{inputenc}
\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{January 16, 2015}
\maketitle
irrationals $\subseteq\bigcup_{i=1}^\infty(-i,i)=\mathbb{R}$

$\{0\}\subseteq$

\section*{outer measure of a sum}
$E\subseteq\mathbb{R}$ then $m*(E)=\inf\{\sum\limits_{i=1}^\infty{b_i=a_i}:E\subseteq\bigcup_{i=1}^{\infty}(a_i,b_i)\}$

\subsection*{example}
$m*(\{x\})=0$

$\{x\}\subseteq(x-\epsilon,x+\varepsilon$ for any $\varepsilon>0$, $m*(t)\le 2\varepsilon$ for any $\epsilon>0\to 0$

\begin{enumerate}
\item
note that $m*(E)\subseteq[0,\infty]$, no negative
\item
the empty set has measure zero

$m*(\emptyset)=0$

$\emptyset\subseteq\bigcup_{i=1}^\infty(a_i,b_i)$

\item
$A\subseteq B\Rightarrow m*(A)\le m*(B)$
\subsubsection*{proof}
if $B\subseteq\bigcup_{i=1}^\infty(a_i,b_i)$ then $A\subseteq\bigcup_{i=1}^\infty(a_i,b_i)$

$\inf\{\sum b_i-a_i | A\subseteq\bigcup(a_i,b_i\}\le \inf\{\sum c_i-d_i |B\subseteq \bigcup(c_i,d_i\}$

cant do this with contradiction, that's the ``actual way''
\subsubsection*{example}
$m*([a,b])=b-a$?

$\inf\{\sum(b_i-a_i):[a,b]\subseteq\bigcup(a_i,b_i\}$

{\bfseries proof}

\begin{align*}
  [a,b]&\subseteq(a-\epsilon,b+\epsilon)\text{ for all }\epsilon>0\\
  m*([a,b])&\subseteq(b+\epsilon)-(a-\epsilon)\text{ for all }\epsilon>0\\
  &\subseteq(b-a)+2\epsilon
\end{align*}

so $m*([a,b])\subseteq b-a$

heine-borel theorem (HBT) compact set is closed and bounded

compact:every open cover has finite subcovers? check this

if $[a,b]\subseteq\bigcup(a_i,b_i)$

wlog $[a,b]\subseteq\bigcup^n(a_i,b_i)$

\begin{align*}
  a_1&<a\\
  a_2&<b_1\\
  a_3&<b_2\\
  a_4&<b_3\\
  &\vdots
  a_n&<b_{n-1}\\
  b&<b_n\\
\end{align*}
these are overlapping covers, they make a sequence, it is important that they overlap

sum to infinity is bigger than sum to $n$.
$\sum(b_1-a_i)+(b_2-a_2)+\dots+(b_n-a_n)\ge(a_2-a_1)+(a_3-a_2)\dots(a_n-a_n-1)+(b_n-a_n)=b_n-a_1\ge b-a$

and so $m*([a,b])\ge b-a$ and we already have less than so it's equal
\subsubsection*{example 2}
$m*((a,b))=b-a$ second part of limhof? thm, uniqueness

notice that the different sets are the same size
\subsubsection*{example 3}
$m*([a,\infty))=\infty$ notice that the measure of any subset is greater than or equal and so $[a,a+k]\subseteq[a,\infty)$ for all $k$ and so $k=m*[a,a+k]\le m*[a,\infty)\forall k$

\subsubsection*{example 4,5}
$m*\mathbb{Q},m*\mathbb{C}-\mathbb{R}$
\item
$m*(\bigcup_{i=1}^\infty A_i)\le\sum\limits_{i=1}^\infty{m*(A_i)}$

{\bfseries proof}

let $\epsilon>0$ then for all $i$ there is $\{(a_i^j,b_i^j\}_{i=1}^\infty$ such that $m*A_i\le\sum\limits{b_i^j-a_i^j}\le m*A_j+\frac{\epsilon}{2^j}$

notice that $\{\{a_i^j,b_i^j\}_{i=1}^\infty\}_{j=1}^\infty$ is countable collection of intervals with $\bigcup A_j\subseteq \bigcup_{j=1}(\bigcup_{i=1}(a_i^j,b_i^j))$
$m* A_j\le \sum_{j=1}(\sum_{i=1}(a_i^j,b_i^j))$

this is called countable subadditivity
\end{enumerate}

\end{document}