\documentclass[openany,11pt]{homework}

\coursename{ELEN 4903 Machine Learning (Spring 2018)} % DON'T CHANGE THIS

\studname{Pratyus Pati}    % YOUR NAME GOES HERE
\studmail{pp2636@columbia.edu}% YOUR UNI GOES HERE
\hwNo{2}                   % THE HOMEWORK NUMBER GOES HERE

% Uncomment the next line if you want to use \includegraphics.
\usepackage{graphicx}

\begin{document}
\maketitle

\section*{Problem 1(a)}

\begin{align}
\hat{\pi}_{ML} & = \operatornamewithlimits{arg\,max}_{\pi} p(y_1, y_2, ..., y_n \mid \pi) \\
			   & = \operatornamewithlimits{arg\,max}_{\pi} \pi^{n\mu_y}(1-\pi)^{n-n\mu_y} \\
			   & = \operatornamewithlimits{arg\,max}_{\pi} \ln(\pi^{n\mu_y}(1-\pi)^{n-n\mu_y}) \\
			   & = \operatornamewithlimits{arg\,max}_{\pi} [(n\mu_y) (\ln \pi)] + [(n - n\mu_y)(\ln (1-\pi))]
\end{align}

On taking the derivative w.r.t. $\pi$\\
\begin{align}
\frac{\partial }{\partial \pi} [(n\mu_y) (\ln \pi)] + [(n - n\mu_y)(\ln (1-\pi))]
& = \left[n\mu_y \frac{\partial \ln \pi}{\partial \pi}\right] + \left[(n - n\mu_y)\left(\frac{\partial \ln(1-\pi)}{\partial \pi}\right)\right] \\
& = \frac{n\mu_y}{\pi} -\frac{n-n\mu_y}{1-\pi}
\end{align}

On setting the partial derivative w.r.t $\pi$ as 0 to get $\hat{\pi}_{ML}$,
\begin{align}
\frac{n\mu_y}{\hat{\pi}_{ML}} -\frac{n-n\mu_y}{1-\hat{\pi}_{ML}} & = 0 \\
\Rightarrow n\mu_y(1-\hat{\pi}_{ML}) - (n-n\mu_y)(\hat{\pi}_{ML}) & = 0 \\
\Rightarrow n\mu_y - n\mu_y\hat{\pi}_{ML} - n\hat{\pi}_{ML} + n\mu_y\hat{\pi}_{ML} & = 0 \\
\Rightarrow n\mu_y - n\hat{\pi}_{ML} & = 0 \\
\Rightarrow \hat{\pi}_{ML} & = \mu_y = \frac{\sum_{i=1}^{n} y_i}{n}
\end{align}

\section*{Problem 1(b)}

\begin{align}
\hat{\theta_y^{1}}_{ML} & = \operatornamewithlimits{arg\,max}_{\theta_y^{1}} p(x_1, x_2, ..., x_n \mid \theta_y^{1}) \\
\end{align}

Splitting the data into the two classes: \\
\begin{align}
I_0 = \{i \mid y_i = 0\} \\
I_1 = \{i \mid y_i = 1\} \\
I_0
\end{align}

Such that:
\mid I_0 \mid = n_0
\mid I_1 \mid = n_1

Such that &$n_0 + n_1 = n$

On taking the derivative w.r.t. $\theta_{0}^{1}$\\
\begin{align}
\frac{\partial }{\partial \pi} [(n\mu_y) (\ln \pi)] + [(n - n\mu_y)(\ln (1-\pi))]
& = \left[n\mu_y \frac{\partial \ln \pi}{\partial \pi}\right] + \left[(n - n\mu_y)\left(\frac{\partial \ln(1-\pi)}{\partial \pi}\right)\right] \\
& = \frac{n\mu_y}{\pi} -\frac{n-n\mu_y}{1-\pi}
\end{align}

On setting the partial derivative w.r.t $\pi$ as 0 to get $\hat{\pi}_{ML}$,
\begin{align}
\frac{n\mu_y}{\hat{\pi}_{ML}} -\frac{n-n\mu_y}{1-\hat{\pi}_{ML}} & = 0 \\
\Rightarrow n\mu_y(1-\hat{\pi}_{ML}) - (n-n\mu_y)(\hat{\pi}_{ML}) & = 0 \\
\Rightarrow n\mu_y - n\mu_y\hat{\pi}_{ML} - n\hat{\pi}_{ML} + n\mu_y\hat{\pi}_{ML} & = 0 \\
\Rightarrow n\mu_y - n\hat{\pi}_{ML} & = 0 \\
\Rightarrow \hat{\pi}_{ML} & = \mu_y = \frac{\sum_{i=1}^{n} y_i}{n}
\end{align}

On taking the derivative w.r.t. $\theta$\\
\begin{align}
\frac{\partial }{\partial \pi} [(n\mu_y) (\ln \pi)] + [(n - n\mu_y)(\ln (1-\pi))]
& = \left[n\mu_y \frac{\partial \ln \pi}{\partial \pi}\right] + \left[(n - n\mu_y)\left(\frac{\partial \ln(1-\pi)}{\partial \pi}\right)\right] \\
& = \frac{n\mu_y}{\pi} -\frac{n-n\mu_y}{1-\pi}
\end{align}

On setting the partial derivative w.r.t $\pi$ as 0 to get $\hat{\pi}_{ML}$,
\begin{align}
\frac{n\mu_y}{\hat{\pi}_{ML}} -\frac{n-n\mu_y}{1-\hat{\pi}_{ML}} & = 0 \\
\Rightarrow n\mu_y(1-\hat{\pi}_{ML}) - (n-n\mu_y)(\hat{\pi}_{ML}) & = 0 \\
\Rightarrow n\mu_y - n\mu_y\hat{\pi}_{ML} - n\hat{\pi}_{ML} + n\mu_y\hat{\pi}_{ML} & = 0 \\
\Rightarrow n\mu_y - n\hat{\pi}_{ML} & = 0 \\
\Rightarrow \hat{\pi}_{ML} & = \mu_y = \frac{\sum_{i=1}^{n} y_i}{n}
\end{align}

\section*{Problem 1(c)}

\section*{Problem 2(a)}

\section*{Problem 2(b)}

\section*{Problem 2(c)}

\section*{Problem 1(d)}

\section*{Problem 1(e)}

\end{document}


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{titletoc}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\expec}{E}

\begin{document}
\title{\textsf{Homework 1 for Bayesian Data Analysis}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Question 2.1}
{
    The likelihood goes like $$p{(y\le 3 | \theta)} = \sum_{j=0}^{3} {\binom{n}{j} \theta^{j} (1-\theta)^{n-j}}$$

    Thus, the posterior distribution is $$p{(\theta | y\le 3)} \propto p{(\theta)} p{(y\le 3 | \theta)} \propto \sum_{j=0}^{3} {\binom{n}{j} \theta^{j+3} (1-\theta)^{n-j+3}}$$, where $n=10$. Here is its sketch.

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.7\linewidth]{1.png}
        \caption{Posterior density of $\theta$}
    \end{figure}
}

\section*{Question 2.2}
{
    Denote $A$ as the event that the first two spins are tail. We have the likelihood $$p{(A | C_1)} = 0.16$$ and $$p{(A | C_2)} = 0.36$$

    So, the posterior probabilities are $$p{(C_1 | A)} \propto p{(C_1)} p{(A | C_1)} = 0.08$$ and $$p{(C_2 | A)} \propto p{(C_2)} p{(A | C_2)} = 0.18$$

    Denote $y$ as the number of additional spins until a head shows up. We have $$E{(y | A)} = \frac{p{(C_1 | A)} E{(y | A, C_1)} + p{(C_2 | A)} E{(y | A, C_2)}}{p{(C_1 | A)} + p{(C_2 | A)}} $$ $$= \frac{p{(C_1 | A)} E{(y | C_1)} + p{(C_2 | A)} E{(y | C_2)}}{p{(C_1 | A)} + p{(C_2 | A)}} $$ $$= \frac{0.08 * (1/0.6) + 0.18 * (1/0.4)}{0.08 + 0.18} = 2.2436$$
}

\section*{Question 2.3b}
{
    $$y \sim \mathrm{Bin}(n, p),$$ where $n=1000$ and $p=1/6$. Since $n=1000$ is large and $p=1/6$ is not close to $0$ or $1$, we can apply the normal approximation of binomial distribution here: $$y \sim \mathrm{Norm}(np, np(1-p))$$

    Using the normal distribution table, we have the quantiles as follows.

    \begin{table}[!hbp]
        \begin{tabular}{|c|c|}
        \hline
        Quantile & Point \\
        \hline
        0.05 & 147.2819 \\
        \hline
        0.25 & 158.7177 \\
        \hline
        0.50 & 166.6667 \\
        \hline
        0.75 & 174.6156 \\
        \hline
        0.95 & 186.0515 \\
        \hline
        \end{tabular}
    \end{table}
}

\section*{Question 2.5a}
{
    $$\Pr{(y=k)} = \int_0^1 {\Pr{(y=k | \theta)}\mathrm{d}\theta}$$
    $$= \int_0^1 {\binom{n}{k} \theta^{k} (1-\theta)^{n-k}\mathrm{d}\theta}$$
    $$= \binom{n}{k} \int_0^1 {\theta^{k} (1-\theta)^{n-k}\mathrm{d}\theta}$$
    $$= \binom{n}{k} \mathrm{Beta}(k+1, n-k+1) = \frac{1}{n+1}$$
}

\section*{Question 2.5b}
{
    $$p(\theta | y) \propto \Pr{(y | \theta)} \Pr{(\theta)} \propto \theta^{y} (1-\theta)^{n-y} \theta^{\alpha-1} (1-\theta)^{\beta-1}$$
    $$\propto \theta^{y+\alpha-1} (1-\theta)^{n-y+\beta-1}$$
    Therefore, the posterior distribution of $\theta$ is $\mathrm{Beta}(y+\alpha, n-y+\beta)$, with its mean
    $$E(\theta | y) = \frac{y+\alpha}{n+\alpha+\beta}$$
    Since
    $$\frac{y+\alpha}{n+\alpha+\beta} - \frac{\alpha}{\alpha+\beta} = \frac{\beta y - \alpha (n-y)}{(n+\alpha+\beta)(\alpha+\beta)}$$
    and
    $$\frac{y+\alpha}{n+\alpha+\beta} - \frac{y}{n} = \frac{-\beta y + \alpha (n-y)}{(n+\alpha+\beta)n}$$
    have opposite sign, the posterior mean of $\theta$ must be between $\frac{\alpha}{\alpha+\beta}$ and $\frac{y}{n}$.

}

\section*{Question 2.5c}
{
    The prior variance is $1/12$, and the posterior variance is $\frac{(y+1)(n-y+1)}{(n+2)^2 (n+3)}$ (by the property of Beta distribution).
    $$\frac{(y+1)(n-y+1)}{(n+2)^2 (n+3)} - \frac{1}{12}$$
    $$= \frac{12 (y+1)(n-y+1) - (n+2)^2 (n+3)}{12 (n+2)^2 (n+3)}$$
    $$\le \frac{12 (n/2+1)(n-n/2+1) - (n+2)^2 (n+3)}{12 (n+2)^2 (n+3)}$$
    $$= \frac{-n}{12(n+3)} \le 0$$
}

\section*{Question 2.5d}
{
    Note that the variance of $\mathrm{Beta}(\alpha, \beta)$ distribution is $$\frac{\alpha \beta}{(\alpha+\beta)^2 (\alpha+\beta+1)}$$ Try many parameters in R, and we find that the variance increases after observation for $\alpha=1$, $\beta=5$, $n=2$, and $y=1$.
}

\section*{Question 2.8a}
{
    $$p(\theta | \bar{y}=150) \propto \exp{((\frac{\theta-180}{40})^2)} \cdot \exp{((\frac{150-\theta}{20/\sqrt{n}})^2)}$$
    $$= \exp{((\frac{\theta-180}{40})^2 + (\frac{150-\theta}{20/\sqrt{n}})^2)}$$
    $$\propto \exp{((\frac{\theta-\mu}{\sigma})^2)},$$
    which is normalized to a normal distribution with mean $\mu$ and variance $\sigma^2$, where
    $$\frac{1}{\sigma^2} = \frac{1}{40^2} + n \frac{1}{20^2},$$
    $$\mu = \sigma^2 (\frac{180}{40^2} + n \frac{150}{20^2}).$$
}

\section*{Question 2.8b}
{
    The posterior predictive distribution of $\widetilde{y}$ is a normal distribution with mean $\mu$ and variance $\sigma^2 + 20^2$, where $\mu$ and $\sigma^2$ is defined in Question 2.8a.
}

\section*{Question 2.8c}
{
    For $n=10$, calculate $\mu = 150.7317$ and $\sigma^2 = 39.02439$. the posterior predictive distribution of $\widetilde{y}$ is $\mathrm{Normal}(150.7317, 439.02439)$.
}

\section*{Question 2.8d}
{
    For $n=100$, calculate $\mu = 150.0748$ and $\sigma^2 = 3.990025$. the posterior predictive distribution of $\widetilde{y}$ is $\mathrm{Normal}(150.0748, 403.990025)$.
}

\section*{Question 2.19a}
{
    Assume the likelihood is proportional to $\theta \exp{(-\theta y)}$, and prior distribution of $\theta$ is $\mathrm{Gamma}(\alpha, \beta)$, that is
    $$p(\theta) \propto \theta^{\alpha-1} \exp{(-\beta \theta)}$$
    So, the posterior distribution is
    $$p(\theta | y) \propto \theta^{\alpha} \exp{(-(\beta + y) \theta)},$$
    which follows a $\mathrm{Gamma}(\alpha+1, \beta+y)$. Thus, the gamma distribution is the conjugate distribution for the exponential likelihood.
}

\section*{Question 2.19b}
{
    $$p(\theta) \propto \theta^{\alpha-1} \exp{(-\beta \theta)}$$
    By chain rule, one has
    $$p(\phi) \propto (\frac{1}{\phi^2}) \cdot (\frac{1}{\phi})^{\alpha-1} \exp{(-\beta (\frac{1}{\phi}))} = \phi^{-\alpha-1} \exp{(-\frac{\beta}{\phi})}$$
    This is an inverse-gamma distribution.
}

\section*{Question 2.19c}
{
    From Question 2.19a, after $n$ experiments, $$\theta|y_1,\cdots,y_n \sim \mathrm{Gamma}(\alpha+n, \beta+t(n))$$
    For the $\mathrm{Gamma}(\alpha, \beta)$ distribution, the coefficient of variation is $1/\sqrt{\alpha}$. So, we have
    $$\frac{1}{\sqrt{\alpha}} = 0.5$$ and $$\frac{1}{\sqrt{\alpha+n}} = 0.1$$
    That goes to $\alpha = 4$ and $n=96$.
}

\section*{Question 2.19d}
{
    The likelihood of $y$ about $\phi$, after $n$ experiments, is $$\frac{1}{\phi^n} \exp{(-\frac{y_1 + \cdots + y_n}{\phi})}$$
    Thus, the posterior distribution is $$\frac{1}{\phi^n} \exp{(-\frac{y_1 + \cdots + y_n}{\phi})} \cdot \phi^{-\alpha-1} \exp{(-\frac{\beta}{\phi})}$$
    $$= \phi^{-(\alpha+n)-1} \exp{(-\frac{\beta + y_1 + \cdots + y_n}{\phi})},$$
    which is an Inverse-gamma$(\alpha+n, \beta+t(n))$ distribution.

    For the Inverse-gamma$(\alpha, \beta)$ distribution, the coefficient of variation is $1/\sqrt{\alpha-2}$. So, we have
    $$\frac{1}{\sqrt{\alpha-2}} = 0.5$$ and $$\frac{1}{\sqrt{\alpha+n-2}} = 0.1$$
    That goes to $\alpha = 6$ and $n=96$.

    The answer does not change, despite that they have different $\alpha$s.
}

\section*{Question 2.20a}
{
    $$p(y\geq 100 | \theta) = \int_{100}^{+\infty} {\theta \exp{(-\theta y)} \mathrm{d}y} = \exp{(-100\theta)}$$
    $$p(\theta | y\geq 100) \propto \theta^{\alpha-1} \exp{(-\beta \theta)} \cdot \exp{(-100\theta)} = \theta^{\alpha-1} \exp{(-(\beta+100)\theta)}$$
    The posterior distribution $$\theta | y\geq 100 \sim \mathrm{Gamma}(\alpha, \beta+100),$$ with mean $\frac{\alpha}{\beta+100}$ and variance $\frac{\alpha}{(\beta+100)^2}$.
}

\section*{Question 2.20b}
{
    $$p(y=100 | \theta) = \theta \exp{(-100\theta)}$$
    $$p(\theta | y\geq 100) \propto \theta^{\alpha-1} \exp{(-\beta \theta)} \cdot \theta\exp{(-100\theta)} = \theta^{\alpha+1-1} \exp{(-(\beta+100)\theta)}$$
    The posterior distribution $$\theta | y\geq 100 \sim \mathrm{Gamma}(\alpha+1, \beta+100),$$ with mean $\frac{\alpha+1}{\beta+100}$ and variance $\frac{\alpha+1}{(\beta+100)^2}$.
}

\section*{Question 2.20c}
{
    In formula 2.8, we have $$\var{(\theta)} = \expec{(\var{(\theta|y)})} + \var{(\expec{(\theta|y)})} \geq \expec{(\var{(\theta|y)})}.$$

    The means the prior variance is, \emph{on average}, greater than the posterior variance. The second term here $\expec{(\var{(\theta|y)})}$ is the expectation of posterior variance with respect to $y$, which is different from the posterior variance itself:
    $$\expec{(\var{(\theta|y)})} \neq \var{(\theta|y)}$$

    In this case, however, we focus on specific $y$s ($y=100$ and $y\geq 100$). Therefore, the formula 2.8 does not apply here since it holds for the expectation of all possible $y$s.
}

\clearpage
\end{document}


% Created 2021-06-30 Wed 07:58
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble}
\usepackage{amssymb}
\usepgfplotslibrary{groupplots}
\DeclareMathOperator{\shift}{q}
\DeclareMathOperator{\diff}{p}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{2021-06-30}
\title{Sampling and aliasing}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={Sampling and aliasing},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.4.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Intro}
\label{sec:org685c9c4}

\begin{frame}[label={sec:orgcd3f3cb}]{Computer-controlled system}
\begin{center}
\includegraphics[width=0.7\linewidth]{../../figures/fig7-2.png}
\end{center}
{\footnotesize Source: Åström \& Wittenmark}
\end{frame}

\section{The sampling theorem}
\label{sec:orgf503a06}
\begin{frame}[label={sec:org881e394}]{Challenges with computerized control - aliasing}
\begin{columns}
\begin{column}{0.6\columnwidth}
\begin{center}
  \begin{tikzpicture}
    \node {\includegraphics[width=0.99\linewidth]{../../figures/comp-contr-sys.png}};
    \node[pin=145:{60Hz mains hum}] at (2.7,2.4) {};
    \node[pin=-60:{90Hz sampling freq}] at (0.5,-1.4) {};
  \end{tikzpicture}
\end{center}
\end{column}
\begin{column}{0.4\columnwidth}
\includegraphics[width=0.99\linewidth]{../../figures/aliasing-example-60Hz}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:orgc438ec7}]{Spatial aliasing}
\begin{center}
\includegraphics[width=0.45\linewidth]{../../figures/Moire_pattern_of_bricks.png}
\includegraphics[width=0.45\linewidth]{../../figures/Moire_pattern_of_bricks_small.png}
\end{center}
\end{frame}

\begin{frame}[label={sec:org77a96ac}]{The sampling theorem}
Shannon and Nyquist:

A continuous-time signal with Fourier transform that is zero outside the interval \((-\omega_0, \omega_0)\)  can be completely reconstructed from equidistant samples of the signal, as long as the sampling frequency is at least \(2\omega_0\).

\begin{center}
  \begin{tikzpicture}[scale=1.2]
    \draw[->] (-3,0) -- (3,0) node[below] {$\omega$};
    \draw[->] (0,0) -- (0,1.5);
    \draw[red, thick] (0,1) to (1,0);
    \draw[red, thick] (0,1) to (-1,0);
    \node at (1,-0.3) {$\omega_0$};
    \node at (-1,-0.3) {$-\omega_0$};
    \node at (0,-0.3) {$0$};
    \node[coordinate, pin=-90:{$2\omega_0$}] at (2,0) {};

  \end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}[label={sec:org1c94273}]{The impulse modulation model}
The \alert{impulse train}, a.k.a the \alert{Dirac comb}:
\begin{center}
\(m(t) = \sum_{k=-\infty}^{\infty} \delta(t-kh)\)\hspace*{10mm}
 \includegraphics[width=0.4\linewidth]{../../../figures/modulation-model-blocks}
\end{center}

\[f_s(t) = f(t)m(t) = f(t) \sum_{k=-\infty}^{\infty} \delta(t-kh) = \sum_{k=-\infty}^{\infty} f(t)\delta(t-kh) = \sum_{k=-\infty}^{\infty} f(kh) \delta(t-kh) \]

\begin{center}
\includegraphics[width=0.8\linewidth]{../../figures/modulation-model-timeseries}
\end{center}
\end{frame}

\begin{frame}[label={sec:org5b6339a}]{Fourier transform of the sampled signal}
The Fourier transform of \(f_s\) and the Fourier transform of \(f\) are related as
\[ F_s(\omega) = \frac{1}{h} \sum_n F(\omega + n\omega_s). \]

Because the Fourier transform of the sampled signal equals the Fourier transform of the continuous-time signal repeated at every multiple of the sampling frequency and added, we get \emph{frequency-folding} or \emph{aliasing}.

\begin{center}
\includegraphics[width=0.28\linewidth]{../../figures/frequency-folding.png}
\end{center}
\end{frame}

\section{Exercises using line spectra}
\label{sec:org57afb82}
\begin{frame}[label={sec:org45627d9}]{Fourier transform of a complex exponential}
The function \(x(t) = \mathrm{e}^{i\omega_1 t}\)
\begin{center}
  \begin{tikzpicture}[scale=2]
    \draw[->] (-1.2, 0) -- (1.2,0) node[below] {Re};
    \draw[->] (0, -1.2) -- (0,1.2) node[left] {Im};
    \draw[domain=0:360, samples=361, dashed] plot ({cos(\x)}, {sin(\x)});
    \node[circle, fill, inner sep=2pt, red] (pnt) at (0.868, 0.5) {};
    \draw[dashed, blue] (0,0) to (0.868, 0.5);
    \draw[domain=0:30, samples=20, ->] plot ({0.6*cos(\x)}, {0.6*sin(\x)});
    \node at (0.7, 0.2) {$\omega_1 t$};
    \node[pin=-135:{1}, coordinate] at (1, 0) {};
    \node[right of=pnt, node distance=3mm, anchor=west] {$x(t) = \mathrm{e}^{i\omega_1 t} = \cos(\omega_1 t) + i\sin(\omega_1 t)$};
  \end{tikzpicture}
\end{center}
has Fourier transform
\[X(i\omega) = \int_{-\infty}^{\infty} x(t) \mathrm{e}^{-i\omega t}dt = \int_{-\infty}^{\infty} \mathrm{e}^{i(\omega_1 - \omega) t}dt = \delta(\omega_1 - \omega)\]
\end{frame}

\begin{frame}[label={sec:orgded89dc}]{Fourier transform of a sinusoid}
A sinusoidal signal \(y(t) = \sin(\omega_1 t)\) has all its power concentrated at one single frequency, \(\omega=\unit{\omega_1}{rad\per\second}\).
\begin{center}
\begin{tikzpicture}
\small
\pgfmathsetmacro{\ww}{1}
\pgfmathsetmacro{\TT}{2*pi/\ww}
\begin{axis}[
width=14cm,
height=2.5cm,
xlabel={$t$},
ylabel={$y(t)$},
xmin=0.,
xmax=30.5,
ytick = {0},
xtick = {0, \TT},
xticklabels={0, $T=\frac{2\pi}{\omega_1}$},
]
\addplot+[black, thick,no marks, domain=0:30, samples=400,variable=t] { sin(deg(\ww*t)) };
\end{axis}
\end{tikzpicture}
\end{center}
Since \[y(t) = \sin(\omega_1 t) = \frac{1}{2i} \left( \mathrm{e}^{i\omega_1 t} - \mathrm{e}^{-i \omega_1 t} \right)\]
the Fourier transform of a sinusoid becomes
\[ Y(i\omega) = \frac{1}{2i} \left( \delta(\omega_1 - \omega) - \delta(\omega_1 + \omega) \right)\]
\end{frame}

\begin{frame}[label={sec:org419e9da}]{Exercise 1: Fourier transform of a sinusoid}
Consider the signal below

\begin{center}
\begin{tikzpicture}
\small
\pgfmathsetmacro{\ww}{1}
\pgfmathsetmacro{\TT}{2*pi/\ww}
\begin{axis}[
width=14cm,
height=2.5cm,
xlabel={$t$},
ylabel={$y(t)$},
xmin=0.,
xmax=30.5,
ytick = {0},
xtick = {0, \TT},
xticklabels={0, $T$},
]
\addplot+[black, thick,no marks, domain=0:30, samples=400,variable=t] { sin(deg(\ww*t)) };
\end{axis}
\end{tikzpicture}
\end{center}

Which of the below is the correct Fourier transform (magnitude plot shown)?

\pgfplotsset{
dirac/.style={
mark=triangle*,
mark options={scale=0.6},
ycomb,
scatter,
visualization depends on={y/abs(y)-1 \as \sign},
scatter/@pre marker code/.code={\scope[rotate=90*\sign,yshift=-2pt]}
}
}
  \begin{tikzpicture}
  \footnotesize

  \pgfmathsetmacro{\ww}{1}
  \pgfmathsetmacro{\TT}{2*pi/\ww}
  \pgfmathsetmacro{\omegaone}{2/\TT}
  \pgfmathsetmacro{\omegatwo}{pi/\TT}
  \pgfmathsetmacro{\omegathree}{1/\TT}
  \pgfmathsetmacro{\omegafour}{2*pi/\TT}

  \begin{groupplot}[group style={group size=2 by 2, vertical sep=1.2cm, horizontal sep=1.3cm},
  width=7cm,
  height=2.5cm,
  xlabel={$\omega$ [rad/s]},
  ylabel={$|Y(i\omega)|$},
  xmin=-1.5,
  xmax=1.5,
  ytick = \empty,
  xtick = \empty,
  ]
  \nextgroupplot[xtick={-\omegaone, 0, \omegaone},
  xticklabels={$-\frac{2}{T}$, 0, $\frac{2}{T}$}]
  \addplot[red, thick, dirac] coordinates {(-\omegaone, 1) (\omegaone, 1)};

  \nextgroupplot[xtick={-\omegatwo, 0, \omegatwo},
  xticklabels={$-\frac{\pi}{T}$, 0, $\frac{\pi}{T}$}]
  \addplot[red, thick, dirac] coordinates {(-\omegatwo, 1) (\omegatwo, 1)};

  \nextgroupplot[xtick={-\omegathree, 0, \omegathree},
  xticklabels={$-\frac{1}{T}$, 0, $\frac{1}{T}$}]
  \addplot[red, thick, dirac] coordinates {(-\omegathree, 1) (\omegathree, 1)};

  \nextgroupplot[xtick={-\omegafour, 0, \omegafour},
  xticklabels={$-\frac{2\pi}{T}$, 0, $\frac{2\pi}{T}$}]
  \addplot[red, thick, dirac] coordinates {(-\omegafour, 1) (\omegafour, 1)};
  \end{groupplot}

  \node[red] at (group c1r1.center) {\huge 1};
  \node[red] at (group c2r1.center) {\huge 2};
  \node[red] at (group c1r2.center) {\huge 3};
  \node[red] at (group c2r2.center) {\huge 4};
  \end{tikzpicture}
\end{frame}
\begin{frame}[label={sec:org3cbc6fb}]{Exercise 2: Two sinusoids}
Consider a signal with Fourier transform

\pgfplotsset{
dirac/.style={
mark=triangle*,
mark options={scale=0.6},
ycomb,
scatter,
visualization depends on={y/abs(y)-1 \as \sign},
scatter/@pre marker code/.code={\scope[rotate=90*\sign,yshift=-2pt]}
}}
\begin{center}
\begin{tikzpicture}
\small
\pgfmathsetmacro{\wwone}{1}
\pgfmathsetmacro{\wwtwo}{5*\wwone}
\begin{axis}[
width=14cm,
height=2.5cm,
xlabel={$\omega$ [rad/s]},
ylabel={$|Y(i\omega)|$},
xmin=-7,
xmax=7,
ymin=-0.5,
ytick=\empty,
xtick = {-\wwtwo, -\wwone, 0, \wwone, \wwtwo},
% ticklabels={$-5\omega_1$, $-\omega_1$, 0, $\omega_1$, $5\omega_1$},
]
\addplot[black, thick, dirac] coordinates {(-\wwtwo, 0.3) (-\wwone, 1) (\wwone, 1) (\wwtwo, 0.3)};
\end{axis}
\end{tikzpicture}
\end{center}

Which of the below time series could this Fourier transform correspond to?

\begin{tikzpicture}
\footnotesize

\pgfmathsetmacro{\wwone}{1}
\pgfmathsetmacro{\wwtwo}{5*\wwone}

\begin{groupplot}[group style={group size=2 by 2, vertical sep=1.2cm, horizontal sep=1.3cm},
width=7cm,
height=2.5cm,
xlabel={$t$ [s]},
ylabel={$y(t)$},
xmin=0,
xmax=15,
ytick = \empty,
xtick = \empty,
domain=0:20,
samples=600,
variable=t,
]

\nextgroupplot[xtick={0, 6.28, 12.56}, xticklabels={0, $2\pi$, $4\pi$},]
 \addplot[red, thick, no marks] { sin(deg(\wwone*t)) + 0.3*cos(deg(\wwtwo*t))};

\nextgroupplot[xtick={0, 6.28, 12.56}, xticklabels={0, $2\pi$, $4\pi$},]
 \addplot[red, thick, no marks] { 0.3*cos(deg(\wwone*t)) + sin(deg(\wwtwo*t))};

\nextgroupplot[xtick={0, 6.28, 12.56}, xticklabels={0, $2\pi$, $4\pi$},]
 \addplot[red, thick, no marks] { cos(deg(0.5*\wwone*t)) + 0.3*cos(deg(0.5*\wwtwo*t))};

\nextgroupplot[xtick={0, 6.28, 12.56}, xticklabels={0, $2\pi$, $4\pi$},]
 \addplot[red, thick, no marks] { cos(deg(\wwone*t)) + 0.3*cos(deg(2*\wwtwo*t))};

\end{groupplot}

\node[red] at (group c1r1.center) {\huge 1};
\node[red] at (group c2r1.center) {\huge 2};
\node[red] at (group c1r2.center) {\huge 3};
\node[red] at (group c2r2.center) {\huge 4};
\end{tikzpicture}
\end{frame}
\begin{frame}[label={sec:orgda14f17}]{Exercise 3: Fourier transform of a sampled sinusoid}
Consider the continuous and sampled signals with sampling period \(h=\frac{2}{3}T\)

 \pgfplotsset{
dirac/.style={
mark=triangle*,
mark options={scale=0.6},
ycomb,
scatter,
visualization depends on={y/abs(y)-1 \as \sign},
scatter/@pre marker code/.code={\scope[rotate=90*\sign,yshift=-2pt]}
}
}
\begin{center}
\begin{tikzpicture}
\small
\pgfmathsetmacro{\ww}{1}
\pgfmathsetmacro{\TT}{2*pi/\ww}
\pgfmathsetmacro{\TTT}{2*\TT}
\pgfmathsetmacro{\wws}{3*\ww/2}
\pgfmathsetmacro{\hh}{2*pi/\wws}
\pgfmathsetmacro{\Ttot}{60}
\pgfmathsetmacro{\Nsamples}{floor(\Ttot/\hh)}

\begin{axis}[
clip=false,
width=14cm,
height=3.5cm,
xlabel={$t$},
ylabel={$y(t)$},
xmin=0.,
xmax=\Ttot,
ytick = {0},
xtick = {0, \hh, \TT, \TTT},
xticklabels={0, $h$, $T$, $2T$},
]
\addplot+[black, thick,no marks, domain=0:\Ttot, samples=400,variable=t] { sin(deg(\ww*t)) }
       node [coordinate, pos=0.87, pin=45:{$y(t)$}] {};
\addplot+[color=blue!80!red!90, thick,dirac, domain=0:\Ttot, samples=\Nsamples+1,variable=t] { sin(deg(\ww*t))} node [coordinate, pos=0.93, pin=-45:{$y_s(t)$}] {};

\draw[blue!80!red!90, thick] (axis cs: 0,0) -- (axis cs: \Ttot, 0);

\end{axis}
\end{tikzpicture}
\end{center}

What is the frequency of the sinusoid? What is the sampling frequency \(\omega_s\) and the Nyquist frequency \(\omega_N\)?
\end{frame}
\begin{frame}[label={sec:org390eac2}]{Exercise 3: Fourier transform of a sampled sinusoid}
Consider the continuous and sampled signals with sampling period \(h=\frac{2}{3}T\)

\pgfplotsset{
dirac/.style={
mark=triangle*,
mark options={scale=0.6},
ycomb,
scatter,
visualization depends on={y/abs(y)-1 \as \sign},
scatter/@pre marker code/.code={\scope[rotate=90*\sign,yshift=-2pt]}
}
}
\begin{center}
\begin{tikzpicture}
\small
\pgfmathsetmacro{\ww}{1}
\pgfmathsetmacro{\TT}{2*pi/\ww}
\pgfmathsetmacro{\TTT}{2*\TT}
\pgfmathsetmacro{\wws}{3*\ww/2}
\pgfmathsetmacro{\hh}{2*pi/\wws}
\pgfmathsetmacro{\Ttot}{60}
\pgfmathsetmacro{\Nsamples}{floor(\Ttot/\hh)}

\begin{axis}[
clip=false,
width=14cm,
height=2.2cm,
xlabel={$t$},
ylabel={$y(t)$},
xmin=0.,
xmax=\Ttot,
ytick = {0},
xtick = {0, \hh, \TT, \TTT},
xticklabels={0, $h$, $T$, $2T$},
]
\addplot+[black, thick,no marks, domain=0:\Ttot, samples=400,variable=t] { sin(deg(\ww*t)) }
       node [coordinate, pos=0.87, pin=45:{$y(t)$}] {};
\addplot+[color=blue!80!red!90, thick,dirac, domain=0:\Ttot, samples=\Nsamples+1,variable=t] { sin(deg(\ww*t))} node [coordinate, pos=0.93, pin=-45:{$y_s(t)$}] {};

\draw[blue!80!red!90, thick] (axis cs: 0,0) -- (axis cs: \Ttot, 0);

\end{axis}
\end{tikzpicture}
\end{center}
Which of the below corresponds to the Fourier transform of the \alert{sampled signal}?

\pgfplotsset{
dirac/.style={
mark=triangle*,
mark options={scale=0.6},
ycomb,
scatter,
visualization depends on={y/abs(y)-1 \as \sign},
scatter/@pre marker code/.code={\scope[rotate=90*\sign,yshift=-2pt]}
}
}
  \begin{tikzpicture}
  \scriptsize

  \pgfmathsetmacro{\ww}{1}
  \pgfmathsetmacro{\TT}{2*pi/\ww}
  \pgfmathsetmacro{\wws}{3*\ww/2}
  \pgfmathsetmacro{\wwN}{\wws/2}

  \pgfmathsetmacro{\omegaone}{\ww-\wwN}
  \pgfmathsetmacro{\omegathree}{\wws - \ww}
  \pgfmathsetmacro{\omegafour}{\wwN/2}

  \begin{groupplot}[group style={group size=2 by 2, vertical sep=1.2cm, horizontal sep=1.3cm},
  width=8cm,
  height=2.5cm,
  xlabel={$\omega$ [rad/s]},
  ylabel={$|Y_s(i\omega)|$},
  xmin=-1.8,
  xmax=1.8,
  ymax=1.2,
  ytick = \empty,
  xtick = \empty,
  ]
  \nextgroupplot[xtick={-\wws, -\ww, -\omegaone, 0, \omegaone, \ww, \wws},
  xticklabels={$-\frac{3\pi}{T}$, $-\frac{2\pi}{T}$, $-\frac{\pi}{2T}$, $$,$\frac{\pi}{2T}$, $\frac{2\pi}{T}$, $\frac{3\pi}{T}$},]
  \addplot[red, thick, dirac] coordinates {(-\ww, 1) (-\omegaone, 1) (\omegaone, 1) (\ww, 1)};
  \addplot+[black, dotted, no marks] coordinates { (-\wwN, 0) (-\wwN, 2) };
  \addplot+[black, dotted, no marks] coordinates { (\wwN, 0) (\wwN, 2) };

  \nextgroupplot[xtick={-\wws, -\ww,  0,  \ww, \wws},
  xticklabels={$-\frac{3\pi}{T}$, $-\frac{2\pi}{T}$,  $$, $-\frac{2\pi}{T}$, $\frac{3\pi}{T}$},]
  \addplot[red, thick, dirac] coordinates {(-\ww, 1) (\ww, 1)};
  \addplot+[black, dotted, no marks] coordinates { (-\wwN, 0) (-\wwN, 2) };
  \addplot+[black, dotted, no marks] coordinates { (\wwN, 0) (\wwN, 2) };

  \nextgroupplot[xtick={-\wws, -\ww, -\omegathree, 0, \omegathree, \ww, \wws},
  xticklabels={$-\frac{3\pi}{T}$, $-\frac{2\pi}{T}$, $-\frac{\pi}{T}$, $$,$\frac{\pi}{T}$, $\frac{2\pi}{T}$, $\frac{3\pi}{T}$},]
  \addplot[red, thick, dirac] coordinates {(-\ww, 1) (-\omegathree, 1) (\omegathree, 1) (\ww, 1)};
  \addplot+[black, dotted, no marks] coordinates { (-\wwN, 0) (-\wwN, 2) };
  \addplot+[black, dotted, no marks] coordinates { (\wwN, 0) (\wwN, 2) };

  \nextgroupplot[xtick={-\wws, -\ww, -\omegafour, 0, \omegafour, \ww, \wws},
  xticklabels={$-\frac{3\pi}{T}$, $-\frac{2\pi}{T}$, $-\frac{3\pi}{4T}$, $$,$\frac{3\pi}{4T}$, $\frac{2\pi}{T}$, $\frac{3\pi}{T}$},]
  \addplot[red, thick, dirac] coordinates {(-\ww, 1) (-\omegafour, 1) (\omegafour, 1) (\ww, 1)};
  \addplot+[black, dotted, no marks] coordinates { (-\wwN, 0) (-\wwN, 2) };
  \addplot+[black, dotted, no marks] coordinates { (\wwN, 0) (\wwN, 2) };

  \end{groupplot}

  \node[red] at (group c1r1.center) {\huge 1};
  \node[red] at (group c2r1.center) {\huge 2};
  \node[red] at (group c1r2.center) {\huge 3};
  \node[red] at (group c2r2.center) {\huge 4};
  \end{tikzpicture}
\end{frame}
\begin{frame}[label={sec:org676d5be}]{Alias frequency}
   To find the low frequency alias \(\omega_a<\omega_N\) of a high freqency sinusoid \(\omega_1\), The expression
\[ \omega_a = \left| \big( (\omega_1 + \omega_N) \, \text{mod}\, \omega_s\big) - \omega_N\right|\]
can be used.
\end{frame}

\begin{frame}[label={sec:orgf12cdf1}]{Aliasing example}
If a continuous-time signal with frequency content (bandwidth) \(\omega_B\) is sampled at too low sampling rate ( \(\omega_s < 2\omega_B\) ), then the energy at higher frequencies is folded onto lower frequencies.

\begin{center}
\includegraphics[width=0.6\linewidth]{../../figures/aliasing-example.png}
\end{center}
A high-frequency sinusoid ( \(\omega_1 = 1800\pi\) rad/s ) masquerading as a lower frequency sinusoid ( \(200 \pi\) rad/s ) due to aliasing when sampled with \(h=10^{-3}\) s.

\alert{Draw the spectrum (lines) of the two sinusoids. Mark the Nyquist frequency and verify that the alias frequency is obtained by folding about the Nyquist frequency.}
\end{frame}

\section{antialiasing}
\label{sec:org9130c47}

\begin{frame}[label={sec:org95929b6}]{Noisy measurements}
\begin{columns}
\begin{column}{0.6\columnwidth}
\begin{center}
  \begin{tikzpicture}
    \node {\includegraphics[width=0.99\linewidth]{../../figures/comp-contr-sys.png}};
    \node[pin=145:{60Hz mains hum}] at (2.7,2.4) {};
    \node[pin=-60:{90Hz sampling freq}] at (0.5,-1.4) {};
  \end{tikzpicture}
\end{center}
\end{column}
\begin{column}{0.4\columnwidth}
\includegraphics[width=0.99\linewidth]{../../figures/aliasing-example-60Hz}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:orgba81902}]{Antialiasing filter}
The \alert{Bessel filter} is often used. From wikipedia:
\begin{quote}
In electronics and signal processing, a Bessel filter is a type of analog linear filter with a maximally flat group/phase delay (maximally linear phase response), which preserves the wave shape of filtered signals in the passband. Bessel filters are often used in audio systems.
\end{quote}

Why use a Bessel filter as antialiasing filter?
\end{frame}

\begin{frame}[label={sec:org0f7beb3}]{Antialiasing filter}
The \alert{Bessel filter} is often used. From wikipedia:
\begin{quote}
In electronics and signal processing, a Bessel filter is a type of analog linear filter with a maximally flat group/phase delay (maximally linear phase response), which preserves the wave shape of filtered signals in the passband. Bessel filters are often used in audio crossover systems.
\end{quote}

Why use as antialiasing filter?
\begin{itemize}
\item Preserves wave shapes \(\Rightarrow\) very little distortion of signals in the passband
\item Maximally linear phase response \(\Rightarrow\) \(\arg H \approx -T\omega\),  Can be modelled as a pure delay
\end{itemize}
\end{frame}
\end{document}


%
% STAT 100: Chance and Data Analysis - A Course Overview
% Section: Analysis of Single-Variable Data
%
% Author: Jeffrey Leung
%

\section{Analysis of Single-Variable Data}
	\label{sec:analysis-of-single-variable-data}
\subsection{Categorical Data}
	\label{subsec:analysis-of-single-variable-data:categorical-data}
\begin{easylist}

	& Analysis using statistics, tables, and graphs:
		&& Identify all options for the categorical variable(s)
		&& Create a frequency table
			&&& Total the number of each option chosen
			&&& Use a two-way frequency table if necessary (see \emph{Two-way frequency table}, below)
		&& Calculate the percentage/proportion of each category
			&&& $\frac{frequency}{total\ sample\ size}$
		&& Use graphs to visually display the data
			&&& E.g. Pie chart, line chart, bar graph
		&& Summarize the data and draw conclusions
			&&& Several sentences
			&&& Describe which options are the most/least frequent
			&&& Do not simply reproduce the numbers

		\medskip
		&& E.g. A study was conducted on SFU students to determine their opinion (agree/neutral/disagree) on purchasing an iPad as a substitution of textbooks. See table~\ref{tab:ipad-textbook-replacement} for the results.

		\Deactivate
		\begin{table}[!htb]
			\centering
			\caption{Opinions of Students on iPads as Textbook Substitutions}
			\label{tab:ipad-textbook-replacement}
			\begin{tabular}{ r l }
				Student ID & Opinion \\
				\hline
				 1 & Disagree \\
				 2 & Disagree \\
				 3 & Neutral \\
				 4 & Neutral \\
				 5 & Neutral \\
				 6 & Agree \\
				 7 & Disagree \\
				 8 & Neutral \\
				 9 & Disagree \\
				10 & Disagree \\
				11 & Agree \\
				12 & Neutral \\
				13 & Agree \\
				14 & Disagree \\
				15 & Neutral \\
				16 & Neutral \\
				17 & Agree \\
				18 & Disagree \\
				19 & Agree \\
				20 & Neutral
			\end{tabular}
		\end{table}
		\Activate

		\smallskip
		Variable: Opinion on purchasing an iPad as a substitution for textbooks \\
		Categories: Agree, neutral, disagree

		\smallskip
		For the frequency and percentage table, see table~\ref{tab:ipad-textbook-replacement-freq-table}.

		\Deactivate
		\begin{table}[!htb]
			\centering
			\caption{Frequency and Percentage Table of Opinions of Students on iPads as Textbook Substitutions}
			\label{tab:ipad-textbook-replacement-freq-table}
			\begin{tabular}{ l r r }
				Opinion:	& Frequency:	& Percentage: \\
				\hline
				Agree		& 5 			& 25 \\
				Neutral		& 8 			& 40 \\
				Disagree	& 7 			& 35 \\
				\hline
				Total:		& 20			& 100
			\end{tabular}
		\end{table}
		\Activate

		\smallskip
		For the bar graph, see figure~\ref{fig:ipad-textbook-replacement-bar-graph}.

		\begin{figure}[!htb]
			\centering
			\begin{tikzpicture}
				\begin{axis}
					[
						enlarge x limits=0.3,
						%axis lines*=left, % Removes the top and right borders
						width=6cm,
						height=7cm,
						xtick=data,
						ybar,
						ymin=0,
						ymax=50,
						ytick={0, 25, 50},
						%bar width=\textwidth*.1,
						bar shift=0pt,
						symbolic x coords={Agree, Neutral, Disagree},
						nodes near coords,
						nodes near coords align = vertical,
						xlabel={\emph{Opinion}},
						ylabel={\emph{Percentage}}
					]
					\addplot+
						[
							draw=black
						]
						coordinates
						{
							(Agree, 25)
							(Neutral, 40)
							(Disagree, 35)
						};
				\end{axis}
			\end{tikzpicture}

			\caption{Bar Graph of Opinions of Students on iPads as Textbook Substitutions}
			\label{fig:ipad-textbook-replacement-bar-graph}
		\end{figure}

		\smallskip
		Conclusion: A large portion of students (40\%) stay neutral on purchasing an iPad as a substitution of textbooks. A small portion of students (25\%) agree on purchasing an iPad as a substitution of textbooks.

	& \emph{Two-way frequency table:} Table which allows the comparison of the frequency of two variables
		&& I.e. See table~\ref{tab:freq-table-example}

		\Deactivate
		\begin{table}[!htb]
			\centering
			\caption{Example of a two-way frequency table}
			\label{tab:freq-table-example}
			\begin{tabular}{ l l | r r | r }
				\multicolumn{2}{ c | }{} & \multicolumn{2}{ c | }{\emph{Var2}} & \\
				\multicolumn{2}{ c | }{} & Option1 & Option2 & \emph{Total:} \\
				\hline
				\multirow{2}{*}{\emph{Var1}}& Option1 & & & \\
				& Option2 & & & \\
				\hline
				\emph{Total:} & & & &
			\end{tabular}
		\end{table}
		\Activate

		&& \emph{Side-by-side bar chart:} Display of multiple objects of study, each with multiple variable data (all objects sharing the same variables)
			&&& See figure~\ref{fig:side-by-side-bar-chart-example}

			\begin{figure}[!htb]
				\centering
				\begin{tikzpicture}
					\begin{axis}
						[
							enlarge x limits=0.7,
							%axis lines*=left, % Removes the top and right borders
							width=7cm,
							height=10cm,
							compat=newest,
							ybar,
							xtick=data,
							ymin=0,
							ymax=100,
							ytick={0, 25, 50, 75, 100},
							bar width=\textwidth*.03,
							symbolic x coords={Male, Female},
							nodes near coords,
							%nodes near coords align = vertical,z
							xlabel={Gender},
							ylabel={Percentage},
						]
						\addplot+
							[
								draw=black
							]
							coordinates
							{
								(Male, 25)
								(Female, 50)
							};
						\addplot+
							[
								draw=black
							]
							coordinates
							{
								(Male, 75)
								(Female, 50)
							};
						\legend{Agree, Disagree}
					\end{axis}
				\end{tikzpicture}

				\caption{Distribution of Agreement by Gender}
				\label{fig:side-by-side-bar-chart-example}
			\end{figure}

		&& Create $x$ bar charts to isolate analysis of each variable, where $x =$ number of variables per object of study
			&&& Compare the magnitude of the same choice in different sample groups

		&& Example: A study was conducted on the distribution of smokers by gender. The results were as follows: \\
		50\% of males smoked; 50\% of males did not smoke. \\
		20\% of females smoked; 80\% of females did not smoke.

		\medskip
		For the side-by-side bar graph, see figure~\ref{fig:distribution-of-smokers-by-gender}.

		\begin{figure}[!htb]
			\centering
			\begin{tikzpicture}
				\begin{axis}
					[
						% GRAPH:
						width=7cm,
						height=9cm,
						%axis lines*=left, % Removes the top and right borders
						% BARS:
						ybar, % Bar chart
						%bar width=\textwidth*.03,
						enlarge x limits=0.7, % Decreases distance between bars
						nodes near coords,
						nodes near coords align = vertical,
						% AXES:
						xtick=data,
						ytick={0, 25, 50, 75, 100},
						symbolic x coords={Male, Female},
						xlabel={Gender},
						ylabel={Percentage},
						ymin=0,
						ymax=100,
						%LEGEND:
						legend style={at={(1.1,0.5)},anchor=west},
					]
					\addplot+
						[
							draw=black
						]
						coordinates
						{
							(Male, 50)
							(Female, 20)
						};
					\addplot+
						[
							draw=black
						]
						coordinates
						{
							(Male, 50)
							(Female, 80)
						};
					\legend{Smoke, Do not smoke}
				\end{axis}
			\end{tikzpicture}

			\caption{Distribution of Smokers by Gender}
			\label{fig:distribution-of-smokers-by-gender}
		\end{figure}

		From figure~\ref{fig:distribution-of-smokers-by-gender:comparison-of-smokers}, the percentage of smokers is higher in the male group than the female group by 30\%.

		\begin{figure}[!htb]
			\centering
			\begin{tikzpicture}
				\begin{axis}
					[
						% GRAPH:
						width=5cm,
						height=9cm,
						%axis lines*=left, % Removes the top and right borders
						% BARS:
						ybar, % Bar chart
						%bar width=\textwidth*.1,
						enlarge x limits=0.7, % Decreases distance between bars
						nodes near coords,
						nodes near coords align = vertical,
						% AXES:
						xtick=data,
						ytick={0, 25, 50, 75, 100},
						symbolic x coords={Male, Female},
						xlabel={Gender},
						ylabel={Percentage},
						ymin=0,
						ymax=100
					]
					\addplot+
						[
							draw=black
						]
						coordinates
						{
							(Male, 50)
							(Female, 20)
						};
				\end{axis}
			\end{tikzpicture}

			\caption{Comparison of Male and Female Smokers in a Distribution of Smokers by Gender}
			\label{fig:distribution-of-smokers-by-gender:comparison-of-smokers}
		\end{figure}

		From figure~\ref{fig:distribution-of-smokers-by-gender:comparison-of-nonsmokers}, the percentage of nonsmokers is higher in the female group than the male group by 30\%.

		\begin{figure}[!htb]
			\centering
			\begin{tikzpicture}
				\begin{axis}
					[
						% GRAPH:
						width=5cm,
						height=9cm,
						%axis lines*=left, % Removes the top and right borders
						% BARS:
						ybar, % Bar chart
						%bar width=\textwidth*.1,
						enlarge x limits=0.7, % Decreases distance between bars
						nodes near coords,
						nodes near coords align = vertical,
						% AXES:
						xtick=data,
						ytick={0, 25, 50, 75, 100},
						symbolic x coords={Male, Female},
						xlabel={Gender},
						ylabel={Percentage},
						ymin=0,
						ymax=100
					]
					\addplot+
						[
							draw=black
						]
						coordinates
						{
							(Male, 50)
							(Female, 80)
						};
				\end{axis}
			\end{tikzpicture}

			\caption{Comparison of Male and Female Non-Smokers in a Distribution of Smokers by Gender}
			\label{fig:distribution-of-smokers-by-gender:comparison-of-nonsmokers}
		\end{figure}

\end{easylist}
\subsection{Quantitative Data}
	\label{subsec:analysis-of-single-variable-data:quantitative-data}
\begin{easylist}

	& Analysis using statistics, tables, and graphs:
		&& Divide the range of data into even classes
			&&& \emph{Class:} Range of values equal in length to all other classes, with no values between classes and no overlapping of classes
				&&&& I.e. Equal-length adjacent ranges of values
				&&&& E.g.  0-10, 10-20, 20-30
				&&&& All data must fall into exactly one class (i.e. there must be no value between classes, and the same value cannot be in multiple classes)
		&& Create a frequency table comparing the number of values in each class (see table~\ref{tab:freq-table-classes})

		\Deactivate
		\begin{table}[!htb]
			\centering
			\caption{Frequency Table with Classes}
			\label{tab:freq-table-classes}
			\begin{tabular}{ l r }
				Class Range: & Frequency: \\
				\hline
				$ 0 \leq value < 10$ & 5 \\
				$10 \leq value < 20$ & 4 \\
				$20 \leq value < 30$ & 3 \\
				$30 \leq value < 40$ & 2 \\
				$40 \leq value < 50$ & 1 \\
				\hline
				Total: & 15
			\end{tabular}
		\end{table}
		\Activate

		&& Create a histogram:
			&&& \emph{Histogram:} Visual display of classes of values against the frequency of data in each class (see figure~\ref{fig:histogram})

			\begin{figure}[!htb]
				\centering
				\begin{tikzpicture}[scale=1]
					\begin{axis}
					[
						ybar,
						ymin=0,
						xtick={-100, -50, 0, 50, 100},
						ytick={0, 1, 2, 3},
						xlabel=X axis (Classes of Values),
						ylabel=Y axis (Frequency of Values per Class)
					]
						\addplot +
						[
							hist=
							{
								bins=4,
								data min=-100,
								data max=100
							}
						] table [y index = 0]
						{
							-75
							-75
							-75
							-25
							25
							25
							75
						};
					\end{axis}
				\end{tikzpicture}

				\caption{Example of a Histogram}
				\label{fig:histogram}
			\end{figure}

			\medskip
			&& E.g. The annual salary (in thousands of dollars) of 10 random UBC graduates was found to be 16, 18, 25, 26, 28, 32, 38, 42, 55, and 80. \smallskip \\
			For the frequency table, see table~\ref{tab:ubc-graduates-salaries-freq-table}.

			\Deactivate
			\begin{table}[!htb]
				\centering
				\caption{Frequency Table of UBC Graduates' Salaries}
				\label{tab:ubc-graduates-salaries-freq-table}
				\begin{tabular}{ l r }
					Classes: & Frequency: \\
					\hline
					$15 \leq salary < 30$ & 5 \\
					$30 \leq salary < 45$ & 3 \\
					$45 \leq salary < 60$ & 1 \\
					$60 \leq salary < 75$ & 0 \\
					$75 \leq salary < 90$ & 1
				\end{tabular}
			\end{table}
			\Activate

			For the histogram, see figure~\ref{fig:ubc-graduates-salaries-histogram}.

			\begin{figure}[!htb]
				\centering
				\begin{tikzpicture}[scale=1]
					\begin{axis} %TODO
					[
						ybar,
						ymin=0,
						xtick={15, 30, 45, 60, 75, 90},
						ytick={0, 1, 2, 3, 4, 5},
						xlabel=Salary (\$ thousands),
						ylabel=Frequency
					]
						\addplot +
						[
							hist=
							{
								bins=5,
								data min=15,
								data max=90
							}
						] table [y index = 0]
						{
							16
							18
							25
							26
							28
							32
							38
							42
							55
							80
						};
					\end{axis}
				\end{tikzpicture}

				\caption{Histogram of UBC Graduates' Salary}
				\label{fig:ubc-graduates-salaries-histogram}
			\end{figure}

\end{easylist}
\subsection{Statistics of Quantitative Data}
	\label{subsec:analysis-of-single-variable-data:statistics-of-quantitative-data}
\begin{easylist}

	& \emph{Minimum:} Least value in a set of data
	& \emph{Maximum:} Greatest value in a set of data
	& \emph{Range:} Area of values over which the data is spread; expressed in terms of the minimum and maximum
	& \emph{Median:} Number which 50\% of values are less than, and which 50\% of values are greater than
		&& Location in a table of ascending order: $\frac{n + 1}{2}$ where $n =$ the number of data points
			&&& If there are two middle numbers, the median is their average
		&& E.g. The median of values \{1, 3, 5, 7\} is located at $\frac{4+1}{2} = 2.5$. Therefore, the median is the average of the two middle values, which is $\frac{3+5}{2} = 4$.
		&& Example of interpretation: About 50\% of UBC graduates earn less than \$30,000 and the other 50\% of UBC graduates earn greater than \$30,000 per year.

	\medskip
	& \emph{Quartiles:} 3 points which divide the data into 4 groups, with the same amount of values in each group
		&& \emph{First quartile (Q1)}: Value which 25\% of the data is less than
			&&& I.e. Median of the lesser half of the data (the median itself is excluded)
		&& \emph{Third quartile (Q3)}: Value which 25\% of the data is greater than
			&&& I.e. Median of the greater half of the data (the median itself is excluded)
		&& Example of interpretation: \\
		About 25\% of UBC graduates earned less than \$25,000. \\
		About 25\% of UBC graduates earned more than \$42,000. \\
		Furthermore, about 50\% of UBC graduates earned between \$25,000 and \$42,000.

	\medskip
	& E.g. Given the 11 values \{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11\}: (see table~\ref{tab:quantitative-data-statistics-example})

	\Deactivate
	\begin{table}[!htb]
		\centering
		\caption{11 Values as an Example of Quantitative Data Statistics}
		\label{tab:quantitative-data-statistics-example}
		\begin{tabular}{ l *{11}{ c } }
			Minimum = 1		& \begin{bfseries} 1 \end{bfseries} & 2 & 3 & 4 & 5 & 6 & 7 & 8 & 9 & 10 & 11 \\
			Maximum = 11	& 1 & 2 & 3 & 4 & 5 & 6 & 7 & 8 & 9 & 10 & \begin{bfseries} 11 \end{bfseries} \\
			Median = 6		& 1 & 2 & 3 & 4 & 5 & \begin{bfseries} 6 \end{bfseries} & 7 & 8 & 9 & 10 & 11 \\
			Q1 = 3			& 1 & 2 & \begin{bfseries} 3 \end{bfseries} & 4 & 5 & 6 & 7 & 8 & 9 & 10 & 11 \\
			Q3 = 9			& 1 & 2 & 3 & 4 & 5 & 6 & 7 & 8 & \begin{bfseries} 9 \end{bfseries} & 10 & 11
		\end{tabular}
	\end{table}
	\Activate

	& \emph{Five-number summary:} Description of data using 5 specific values (minimum, first quartile, median, third quartile, maximum)
		&& About 25\% of data falls between:
			&&& The minimum and Q1
			&&& Q1 and the median
			&&& The median and Q3
			&&& Q3 and the maximum

		&& \emph{Boxplot:} Visual display of the five-number summary (see figure~\ref{fig:boxplot-example})
			&&& Consists of a box bordered by Q1 and Q3, a vertical line in the box at the median, and two `tails' to the left and right of the box at the minimum and maximum

			\begin{figure}[!htb]
				\centering
				\begin{tikzpicture}
					\begin{axis}
					[
						xscale=1.7, % Size of figure
						yscale=.7,
						ytick=0,
						xlabel=X axis
					]
						\addplot+
						[
							boxplot prepared=
							{
								lower whisker=0,
								lower quartile=1,
								median=2,
								upper quartile=3,
								upper whisker=4
							},
						]
						coordinates {} ;
					\end{axis}
				\end{tikzpicture}

				\caption{Example of a Boxplot}
				\label{fig:boxplot-example}
			\end{figure}

			&&& \emph{Inter-quartile range:} Width of the box; difference between Q1 and Q3

	& \emph{Outlier:} Data point which falls outside the overall pattern
		&& E.g. 10 students write a final exam. 9 students received a mark below 60\%; 1 student received a mark of 100\%. The 1 student is an outlier.
		&& May occur due to chance or error
		&& Can be detected by:
			&&& A boxplot generated by statistical software (displayed as a plot outside the boxplot range)
			&&& Using the 1.5 IQR rule:
				&&&& \emph{Lower limit:} Q1 - 1.5 $\cdot$ IQR
				&&&& \emph{Upper limit:} Q3 + 1.5 $\cdot$ IQR
				&&&& Any point less than the lower limit or greater than the upper limit is an outlier
		&& E.g. The annual salary (in thousands) of 10 random UBC graduates was found to be 16, 18, 25, 26, 28, 32, 38, 42, 55, and 80. Detect any outliers. \smallskip \\
		Q1 = 25 \\
		Q3 = 38 \\
		IQR = Q3 - Q1 = 38 - 25 = 13 \\
		Lower limit = Q1 - 1.5 $\cdot$ IQR = 25 - 1.5 $\cdot$ 13 = 5.5 \\
		Upper limit = Q3 + 1.5 $\cdot$ IQR = 38 + 1.5 $\cdot$ 13 = 57.5 \smallskip \\
		The outlier is 80, because it is the only data point less than the lower limit or greater than the upper limit.

\end{easylist}
\subsection{Shape, Center, and Spread of a Distribution}
	\label{subsec:analysis-of-single-variable-data:shape-center-spread-of-a-distribution}
\begin{easylist}

	& \emph{Shape (distribution):} Skew of data points or lack thereof
		&& Calculated through the distances between Q1 and the median, and the median and Q3

		&& \emph{Skewed to the left:} Most data points are mainly on the right side of the distribution
			&&& Tail of data on the left; bulge of data on the right
			&&& $median - Q1 > Q3 - median$
			&&& E.g. See figure~\ref{fig:distribution-skewed-to-the-left}

			\begin{figure}[!htb]
				\centering
				\begin{tikzpicture}
					\begin{axis}
					[
						ybar,
						ymin=0,
						xtick={0, 20, 40, 60, 80, 100},
						ytick={0, 5},
						xlabel=X axis,
						ylabel=Y axis
					]
						\addplot +
						[
							hist=
							{
								bins=5,
								data min=0,
								data max=100
							}
						] table [y index = 0]
						{
							10
							30
							30
							50
							50
							50
							70
							70
							70
							70
							70
							90
							90
							90
						};
					\end{axis}
				\end{tikzpicture}

				\caption{Example of a Distribution Skewed to the Left}
				\label{fig:distribution-skewed-to-the-left}
			\end{figure}

		&& \emph{Normal distribution:} Most data points are mainly in the centre of the distribution
			&&& Tails of data on the right and left; bulge of data in the centre
			&&& $median - Q1 \approx Q3 - median$
			&&& See subsection~\ref{subsec:analysis-of-single-variable-data:normal-distribution}
			&&& E.g. See figure~\ref{fig:distribution-normal}

			\begin{figure}[!htb]
				\centering
				\begin{tikzpicture}[scale=1]
					\begin{axis}
					[
						ybar,
						ymin=0,
						xtick={0, 20, 40, 60, 80, 100},
						ytick={0, 5},
						xlabel=X axis,
						ylabel=Y axis
					]
						\addplot +
						[
							hist=
							{
								bins=5,
								data min=0,
								data max=100
							}
						] table [y index = 0]
						{
							10
							30
							30
							30
							50
							50
							50
							50
							50
							70
							70
							70
							90
						};
					\end{axis}
				\end{tikzpicture}

				\caption{Example of a Normal Distribution Skewed to the Right}
				\label{fig:distribution-normal}
			\end{figure}

		&& \emph{Skewed to the right:} Most data points are mainly on the left side of the distribution
			&&& Tail of data on the right; bulge of data on the left
			&&& $median - Q1 < Q3 - median$
			&&& E.g. See figure~\ref{fig:distribution-skewed-to-the-right}

			\begin{figure}[!htb]
				\centering
				\begin{tikzpicture}[scale=1]
					\begin{axis}
					[
						ybar,
						ymin=0,
						xtick={0, 20, 40, 60, 80, 100},
						ytick={0, 5},
						xlabel=X axis,
						ylabel=Y axis
					]
						\addplot +
						[
							hist=
							{
								bins=5,
								data min=0,
								data max=100
							}
						] table [y index = 0]
						{
							10
							10
							10
							30
							30
							30
							30
							30
							50
							50
							50
							70
							70
							90
						};
					\end{axis}
				\end{tikzpicture}

				\caption{Example of a Distribution Skewed to the Right}
				\label{fig:distribution-skewed-to-the-right}
			\end{figure}

		&& Correspond directly to the boxplot
		&& Extreme values affect the mean more than the median:
			&&& Skewed to the left: Mean is less than the median
			&&& Normal distribution: Mean is roughly equal to the median
			&&& Skewed to the right: Mean is greater than the median

		\medskip
		&& E.g. The annual salary (in thousands) of 10 random UBC graduates was recorded and analyzed. See table~\ref{tab:ubc-graduates-five-number-summary} for the five-number summary.

		\Deactivate
		\begin{table}[!htb]
			\centering
			\caption{Five-Number Summary of UBC Graduates' Annual Salaries}
			\label{tab:ubc-graduates-five-number-summary}
			\begin{tabular}{ l r }
				Minimum & 16 \\
				Q1 & 25 \\
				Median & 30 \\
				Q3 & 42 \\
				Maximum & 80
			\end{tabular}
		\end{table}
		\Activate

		Median - Q1 = 30 - 25 = 5 \\
		Q3 - median = 42 - 30 = 12 \\
		Since the distance between the median and Q3 is greater than the distance between Q1 and the median, the distribution is skewed to the right.

		%TODO Weeks 1/2, Document 03, Page 3

	& \emph{Center:} Median of a distribution
		&& Used for general comparisons of magnitude

	& \emph{Spread/variability:} How data points are distributed across the range
		&& Often measured by IQR (see \emph{inter-quartile range}, subsection~\ref{subsec:analysis-of-single-variable-data:categorical-data})
			&&& The greater the IQR, the greater the spread
		&& Not often measured by range because it is affected greatly by outliers
			&&& Only used when a conclusion cannot be derived using the IQR

	& UBC study %TODO

	& E.g. Boxplot and summary statistics \smallskip \\ %TODO
	The shape is skewed to the right. \\
	The median exam score is 60\%. Therefore, about 50\% of students scored less than 60\%, while the other 50\% of students scored higher than 60\%. \\
	The spread: Ranges from 55\% min to 92\% max. The middle 50\% of exam scores are between 58\% at Q1 and 66.5\% at Q3. \\
	There are 2 outliers; therefore, 2 students received abnormally high exam scores.

	& \emph{Mean/Average:} Sum of a set of values divided by the number of values
		&& Denoted by an overline ($\overline{x}$)

	& \emph{Standard deviation:}
		&& Denoted by $\sigma$
		&& Formula:
		\begin{displaymath}
		\sigma =
		\sqrt
		{
			\frac
			{
				\sum \limits_{i=0}^n (x_{i} - \overline{x})^2
			}
			{
				n - 1
			}
		}
		\end{displaymath}
		\Deactivate
		\begin{center}
			\begin{tabular}{ l r @{ = } l }
				where: & $x$ & variable data \\
				& $\overline{x}$ & mean of the variable data \\
				& $n$ & total number of sample data points
			\end{tabular}
		\end{center}
		\Activate

	& %TODO example, other

\end{easylist}
\subsection{Normal Distributions}
	\label{subsec:analysis-of-single-variable-data:normal-distribution}
\begin{easylist}

	& Use a mean and standard deviation to analyze a normal distribution

	& %TODO

	& \emph{Standard (z-) score:}
		&& Formula: $z = \frac{x - mean}{s}$ %TODO LEGEND - s = standard deviation
			&&& $x = mean + z \cdot s$
		&& Shade the region of interest and use the standard normal table to find the corresponding percentage
			&&& Only calculate the z-score up to one significant digit
			&&& Standard normal table gives area to the left of the z-score; if the z-score is positive ( is greater than the mean), subtract it from 100\%
		&& Example: %TODO

\end{easylist}
\clearpage


%% grundlagen.tex
%% $Id: grundlagen.tex 61 2012-05-03 13:58:03Z bless $
%%

\chapter{Principles of compression}
\label{ch:Principles of compression}
%% ==============================
\par{
Throughout this work a finite alphabet $\Sigma$ is assumed with a linear order $\leq$ over its elements. A word $w$ is a sequence $a_1,...,a_n$ of letters $a_i \in \Sigma$, $1 \leq i \leq n$. The set of all such sequences is denoted by $\Sigma^*$ which is the monoid over $\Sigma$, with concatenation as composition and with the empty word $\varepsilon$ as neutral element. Supplementary the set $\Sigma^+ = \Sigma^*$ \textbackslash $\: \varepsilon$ consists of all non-empty words. Each word induces a labeling function $\lambda_w$ as a labeled linear order where position $i$ of $w$ is labeled by $a_i \in \Sigma$ and we write $\lambda_w (i) = a_i$. If, for words $u, \: v$, $u = v$ or $u$ is lexicographically smaller than $v$ we write $u \leq v$ in the context of the order over the alphabet. We say that two words $u$, $v$ are conjugate if $u = st$ and $v = ts$ for some words $s$ and $t$ and $u$ and $v$ are cyclic shifts of one another. The ordered conjugacy class of a word $w \in \Sigma^n$ denoted as $[w] = (w_1,...,w_n)$ represents the lexicographically distinct ordered cyclic rotations of that word. A \emph{Lyndon word} is the unique minimal element within its conjugacy class. More formally, let $[w] = (w,w_2,...w_n)$, then $w \: \in \Sigma^+$ is a Lyndon word if $w<w_i$ for all $i \: \in \{2,...,n\}$.
}

%% ==============================
\section{Compression and Encoding Fundamentals}
%% ==============================
\label{ch:Principles of compression:sec:Compression}
\par{
	The basic idea of compression is to remove redundancy in data, since all non random data contains redundant information. Pattern or structure identification and exploitation enables storing the original data in less space. Compression can be broken down into two broad categories: Lossless and lossy compression. Lossless compression makes it possible to reproduce the original data exactly while lossy compression allows some degradation in the encoded data to gain even higher compression at the cost of losing some of the original information. To understand the applied methods, one first has to understand some basic principles of information theory like entropy and different approaches to compress different types of data with different encoding. We will also show the key differences between probability coding and dictionary coding.}

\subsection{Information Theory and Entropy}
\par{
As Shannon described his analysis about the English language \cite{entropy}, he used the term entropy closely aligned with its definition in classical physics, where it is defined as the disorder of a system. Specifically speaking, it is assumed that a system has a set of states $s$ of which it can be in and it exists a probability distribution over those states $P(s)$. Shannon defined the term self information $i(s)$ as:
\[
i(S) = log_{2} \: \frac{1}{P(s)}
\]
where $S$ describes all possible states, $P(s_i)$ is the likelihood of the system being in state $s_i$, indicating that with higher probability of a state, less information can be contained. As an example, the statement \enquote{The criminal is smaller than 2 meters.} is very likely but doesn't contain much information, whereas the statement \enquote{The criminal is larger than 2 meters.} is not very likely but contains more information. In addition to that, Shannon then defines the entropy as:
\[
H(S) = \sum_{s \in S} P(s) \: i(s)
\]
 So generally speaking it means that evenly distributed probabilities imply a higher entropy and vice versa as seen in Table \ref{tab:t1:entropy} It also implies that, given a source of information $S$, its average information content per message from $S$ is also described by this formula. With these definitions in mind, we can analyze some properties for the English language from an information theory point of view.
 \begin{table}[h]
 	\centering
 	\begin{tabular}{l|l|l|l|l|l|l}
 		& $P(a)$ & $P(b)$ & $P(c)$ & $P(d)$ & $P(e)$ & $H$ \\
 		\hline
 		1. & 0.2 & 0.2 & 0.2 & 0.2 & 0.2 & 2.322 \\
 		2. & 0.94 & 0.01 & 0.01 & 0.01 & 0.01 & 0.322
 	\end{tabular}
  	\label{tab:t1:entropy}
 	\caption{Entropy in relation to the probability of symbols.}
 \end{table}
}

\par{
Combining those approaches, you will find that $\Sigma$ being a finite alphabet, and $P(s_i)$ describing the likelihood of $s_i$ and $i(s_i)$ its self information, $i(s_i)$ also describes the amount of bits needed for this symbol.
\[
	H(\Sigma) = \sum_{i = 1}^{n} P(s_i) \: \cdot \: log_{2} \: \frac{1}{P(s_i) }
\]
So the entropy 	also describes the theoretical amount of bits needed to persist a message from $\Sigma$, calculated in $bps$ as bits per symbol.
}
\subsection{General Analysis}
\par{
To evaluate the efficiency of a specific compression technique, we have to determine how much information the raw data contains. We are looking at ASCII encoded English language sentences as examples for text compression. There have been broad analysis of ASCII entropy, consisting of 96 different printable symbols for the English language, generating approaches for calculating the entropy \cite{entropy-fernau}.
}

\par {
If we assume a 96 symbol alphabet and a uniform probability distribution, we find an entropy of $log_2 (96) = 6.6 \: bps$. An empirically distribution generated by text analysis, the entropy comes down to $4.5 \: bps$. Using an encoding which encodes each symbol separately instead of the whole input at once, like the Huffman Coding, we can achieve an entropy of $4.7 \: bps$ which is only slightly worse than the assumed entropy. By changing the assumption to blocks of symbols of length 8, we get $96^8$ different blocks. Although the probability distribution of the English language implies an entropy as low as $1.3 \: bps$ which leads to a possible reduction of symbols to 3 printable characters without generating longer texts. Consulting newer sources we will find that, up to 500 character long symbol based text analysis, with around $20.3 \: Mio.$ different symbols, results in an entropy of around $1,58 \: bps$ \cite{entropy-new}. This gives a vague limit of how much compression we can theoretically expect from English text.}

\subsection{Probabilistic Coding}
\par{
The general idea behind Probability Coding is to analyze probabilities for messages and encode them in variable length bit strings according to their probability. This way, messages that are more likely and will repeat more often can be encoded in a smaller bit string representation. The generation of those probability distributions is considered part of the analyzer module by the algorithm and will be discussed later on in Section \ref{ch:Conceptual Design}. Probability coding can be further distinguished into variable and fixed unique coding, which will represent each message with a bit string with an amount of $n$ bits $n \in \mathbb{N}$, for example Huffman coding as type of prefix coding which will be explained in greater detail in Section \ref{ch:Principles of compression:sec:Huffman Coding}. In contrast to Huffman coding there is also arithmetic coding which can represent a set of messages as a single floating point number $q$ where $0.0 < q < 1.0$. By encoding them in nestend intervals in $[0,1]$ determined by a probability distribution, they can \enquote{fuse} messages together and need less space to represent a set of messages than encoding every message separately. Arithmetic coding is usually stronger than Huffman coding but also slower. In 2014 a new probabilistic approach was produced by Jarek Duda called Asymmetric Numeral Systems (ANS) \cite{DBLP:journals/corr/Duda13}. This method encodes a string of symbols into a single natural number and combines the strength of arithmetic coding with the speed of Huffman coding.
}

\subsection{Dictionary Coding}
\par{
Dictionary Coding is best suited for data with a small amount of repeating patterns like a text containing repeated words. In this case it is very effective to save the patterns just once and refer to them if they are used later on in the text. If we know a lot about the text itself in advance, a static dictionary is sufficient but in general we want to compress an arbitrary text. This requires a more general approach as used by the well known LZ77 and LZ88 algorithms described by Jacob Ziv and Abraham Lempel \cite{lz}. They use a so called sliding window principle, a buffer moving across the input in the case of LZ77 or a dynamic dictionary in the implementation of LZ78. Both of them are still used in modified versions in real world applications as described in Section \ref{ch:Principles of compression:sec:SOTA}, although mostly implemented in newer variants or derivatives.
}
\par{
The main difference between probabilistic coding and dictionary coding is, that the so far presented probability based methods like RLE or Huffman Coding are working on single characters or bytes whereas the dictionary based methods encode groups of characters of varying length. This is a clear advantage in terms of compression performance on every day data because of its repeating patterns most textual data has, as shown in Section \ref{ch:Analysis:sec:Initial Findings}, and has been overcome by sophisticated probabilistic methods like arithmetic coding or ANS.
}

\subsection{Irreversible Compression}
\par{
Irreversible Compression or also called \enquote{Lossy Compression} is a type of compression which looses information in the compression process and is therefore not completely reversible, hence the name. There are a lot of use-cases for these type of compression algorithms, mostly used for images, video and audio files. These files typically contain information almost not perceptible for an average human like really small color differences between two pixels of an image or very high frequencies in an audio file. By using approximations to store the information and accept the loss of some information while retaining most of it, lossy image compression algorithms can get twice the compression performance compared to lossless algorithms. Due to the fact that most lossy compression techniques are not suited for text compression which is the main use case for this work, we will not elaborate any further on this topic.
}

%% ==============================
\section{Run Length Encoding}
%% ==============================
\label{ch:Principles of compression:sec:Run Length Encoding}
\par{
	Run length coding might count as the simplest probability coding scheme that makes use of context and repeating occurrences as described in the Section \ref{ch:Introduction:sec:Motivation}. While the example made earlier was in textual representation, it is best suited and mostly used for pallet based bitmap images \cite{palette-image} such as fax transmissions or computer icons.
}

\subsection{History}
\label{ch:Principles of compression:sec:Run Length Encoding:subSec:History}
\par{
 The ITU-T T4 (Group 3) standard for Facsimile (fax) machines \cite{ITU} is still in force for all devices used over regular phone lines. Each transmission sends a black and white image, where each pixel is called a \textit{pel} and with a horizontal resolution of $8.05 \: \frac{pels}{mm}$ and the vertical resolution depending on the mode. To encode each sequence of black and white pixels, the T4 standard uses RLE to encode each sequence of black and white pixels and since there are only two values, only the run length itself has to be encoded. It is assumed that the first run is always a white run, so there is a dummy white pel at the beginning of each sequence. For example, the sequence $bbbbwwbbbbb$ can be encoded as 1,4,2,5 with the leading white dummy pixel.
}
\par{
To further reduce the RLE encoded sequence, a probability coding procedure like Huffman coding (explained in Section \ref{ch:Principles of compression:sec:Huffman Coding}) can be used to code the sequence, since shorter run lengths are generally more common than very long runs, which is also done by the T4 standard. Based on a broad analysis of the average run lengths counted, the T4 defined a set of static Huffman codes, using different codes for the black and white pixels. This way for example 20 consecutive white pels  are a white run with length 20 and so the message can be encoded and transmitted as the code-word \enquote{0001000}, requiring only 7 bit instead of 20, seen in row 6 column 2 in Table \ref{tab:t1:static huffman codes}. Since a fax typically contains more white, the white runs are more frequent and thus get shorter code-words which is explained in greater detail in Section \ref{ch:Principles of compression:sec:Huffman Coding}. Since the appearance of text usually only needs a few consecutive black pels, short black runs of length 2 or 3 appear to be the most frequent seen runs so to save space they are encoded in the overall shortest codes. Runs of more than 64 have to be encoded in multiple Huffman codes, for example a run of 150 has to be encoded with the Huffman code of 128 followed by the code for 22.
}
\begin{table}
	\centering
	\begin{tabular}[p]{r|c|c}
		run length &  white run Huffman codes & black run Huffman code\\
		\hline
		0 &  00110101 & 0000110111\\
		1 & 000111 & 010\\
		2 & 0111 & 11\\
		3 & 1000 & 10\\
		4 & 1011 & 011\\
		... &  & \\
		20 & 0001000 & 00001101000\\
		... & & \\
		64+ & 11011 & 0000001111\\
		128+ & 10010 & 000011001000
		\label{tab:t1:static huffman codes}
	\end{tabular}
	\caption{T4 static Huffman codes.}
\end{table}
\par{
By combining RLE with a probabilistic approach, the compression ratio increases because we no longer have to encode a run of length 1 or 2 with a fixed size of bits each instead we can write a recognizable Huffman code of varying length, which will be explored in grater detail later on as well as the decoding of the shown Huffman codes.}

\subsection{Limitations}
\par{
As mentioned in Section \ref{ch:Introduction:sec:Problem statement}, run length encoding is rarely used for regular text or continuous tone images because its potentially increase in size, due to non repetitive characters or bytes. A detailed analysis of this issue is performed in Section \ref{ch:Analysis:sec:Initial Findings}. Several approaches to mitigate this issue have been implemented in this scope, some of the (e.g. Burrows-Wheeler-Transformation) in multiple variants. While plain RLE is still in use by fax transmission or other highly specific tasks, it is mostly implemented in combination with other approaches \cite{rle-dna},\cite{fax-rle}.
}
%% ==============================
\section{Prefix Coding}
%% ==============================
\label{ch:Principles of compression:sec:Huffman Coding}
\par{
Prefix codes make use of an untypical idea in computer science. Usually we deal with fixed length codes like 7 bit ASCII or 32 bit Integer representations which map each possible value into that fixed amount of bits. For compression it would be a benefit if we could write codes of variable length. The problem with these varying length codes is that as soon as they are part of a sequence, it becomes very hard to tell where one code word starts and finishes, resulting in ambiguous encoding. For example the set of codes $ C= \{(a,1),(b,01),(c,101),(d,011)\}$ and the encoded message is $1011$ do not generate distinct decodeable code words because there is no way to tell which message was encoded. From now on a code $C$ for a set of messages $S$ is considered to be in the form of $ C= \{(s_1,w_1),(s_2,w_2),...,(s_m,w_w)\}$.
}
\par{
One way to address this problem is by adding extra stop symbols or encoding a length before each code but those just add additional data. Another approach is to use so called \emph{prefix codes} and make no code which is prefix of another code, which makes them distinct and uniquely decodable. A Huffman coding is one way of generating a prefix code, which is uniquely decodable. When no code is prefix of another one, an encoded message is always decodable and yields a unique result because once a matching codeword is read, no other longer codeword could also match. Huffman codes have another property, as they are call optimal prefix codes. To understand this property we first have to define the average length $l_a$ of a code $C$ as:
\[
l_a(C) = \sum_{(s,w)\in C} \: p(s) \: l(w)
\]
We say that a prefix code is optimal, if $l_a(C)$ is minimized, so there is no other prefix code for a given probability distribution that has a lower average length. The existence of a relation between the average length of a prefix code to the entropy of a set of messages can be shown by making use of the Kraft McMillan Inequality \cite{mcKillan}. For a uniquely decodeable code $C$ where $l(w)$ is the length of the code word $C$ we have:
\[
\sum_{(s,w)\in C} 2^{-l(w)} \leq 1
\]
And it is also proven that the Huffman algorithm generates optimal prefix codes \cite{compressionIntroduction}. The algorithm	 in general is rather simple and the generation of the prefix codes will be demonstrated using an example. Assume we want to generate a prefix code for the message \enquote{accbccac}. To generate a Huffman tree, first all occurrences of source symbols are counted and then a leaf node, with its frequency, is added to a priority queue for every symbol. Then, the two nodes with the lowest frequency are removed from the list, combined into one node with the two original nodes as children and this combined node is then added back to the list with the sum of the child nodes frequencies. This step is repeated while there is more than one nodes left in the queue. For our example the counts are $a = 2$, $b = 1$ and $c = 5$. So the first two nodes are $b$ and $a$ which are combined and become a new node, then this node is combined with $c$ into another node. Since there is no further node, the last created node is the root node of the tree. This is resembled in Figure \ref{fig:M1:example Huffman tree} with the tree leaf nodes in green and the two generated nodes in red with the combined frequency. To get the actual mapping between a symbol and its code, simply follow the tree from its root to each symbol, the path from the root is its code. Obviously the most frequent symbol is assigned the shortest code and all codes are prefix free. For decoding it is required to persists the mapping as well.

\begin{figure}[h]
	\centering
\begin{tikzpicture}
\begin{scope}[every node/.style={circle,thick,draw=red}]
\node (B) at (1.25,2) {3};
\node (C) at (2.5,4) {8};
\end{scope}
\begin{scope}[every node/.style={circle,thick,draw=green}]
\node (A) at (0,0) {a,2};
\node (D) at (2.5,0) {b,1};
\node (F) at (5,0) {c,5} ;
\end{scope}

\begin{scope}[>={Stealth[black]},
every node/.style={fill=white,circle}]
\path [->] (A) edge node {$0$} (B);
\path [->] (B) edge node {$0$} (C);
\path [->] (D) edge node {$1$} (B);
\path [->] (F) edge node {$1$} (C);

\end{scope}
\end{tikzpicture}
	\caption{Example Huffman tree with 3 leaf nodes.} \label{fig:M1:example Huffman tree}
\end{figure}
}
\par{
Decoding is very trivial due to the prefix codes used by this algorithm. Reading bit by bit one can always see if a code matches one of the mappings and if not, another bit is read. As soon as the parsed content matches a mapped code, we can decode it to the original symbol because there cannot be a longer matching mapping. Due to the implementation via a priority queue, which needs $O(log \: n)$ time per insertion and a tree with $n$ leaves has $2n-1$ nodes let this algorithm perform in $O(n \: log \: n)$ time where $n$ is the number of symbols.
}

\section{Other Methods}
\label{ch:Principles of compression:sec:Other}
\par{
There are other well known methods like Move-to-Front coding or arithmetic coding also applied in different compression algorithms with different methods and therefore their own pros and cons. But other than Huffman coding, no further probability coding approach will be discussed nor implemented in this scope. But its worth mentioning that real world applications are mostly combinations of methods, like Deflate\cite{deutsch1996rfc1951} which uses LZSS\cite{10.1145/322344.322346} (derivative of LZ77) and Huffman encoding or more recent ones like the PAQ family which is using a context mixing algorithm \cite{mahoney2009data}. Some combinations and PAQ implementations can be found in table \ref{tab:t20 stat of the art} where the state of the art compression methods and algorithms are discussed.
}

\subsection{Burrows-Wheeler-Transformation}
\label{ch:Principles of compression:sec:Other:subSec:bwt}

\par{
The Burrows-Wheeler-Transformation is not a compression technique but rather a method to prepare text to increase its compression potential, described by M. Burrows and D. J. Wheeler \cite{Burrows94}. It is a transformation of a string $w$ of $n$ characters by forming the $n$ cyclic rotations of $w$, sorting them lexicographically, and extracting the last character of each of the rotations or in other words, the conjugacy class of $w$ which is $[w]$. Each rotation is induced by a right shift of the original word. The right shift of a word $w=a_1 ... a_n$ is denoted as $r(w) = a_na_1...a_{n-1}$ and the $i$-fold right shift $r^i(w)$ is defined inductively by $r^0(w) = w$ and $r^{i+1}(w) = r(r^i(w))) $ which is also defined for $n \leq i$ as $r^i(w)=r^j(w)$ where $j = i \: mod \: n$. The result of this transformation is a string $L$, consisting of these last characters of each sorted rotation. The basic algorithm also has to add special termination symbols or compute the index $I$ of the original string $S$ in the sorted list of rotations to be able to revert this transformation. Surprisingly, there is an efficient algorithm to invert the transformation back to the original string $S$ given only $L$ and the Index $I$ \cite{Burrows-linear-time} and also a modified version, which is not depended on additional information, described in Section \ref{ch:Principles of compression:sec:Other:subSec:bwtscott}.
}

\par {
As an example of the creation of $L$, given the input string $w = abcabr$, $n = 6$, and the alphabet $\Sigma = \{a,b,c,r\}$ with $a \textless b \textless c \textless r $. Create a $n \times n$ matrix $M$ whose elements are characters, and with all cyclic rotations of $w$, which assembles the conjugacy class $[w]$ of word $w$. All rotations are sorted in lexicographical order, in this example, the index is $I = 1$ and the matrix $M$ is:

\begin{table}[h]
	\centering
	\begin{tabular}{r|l|l|l|l|l|l}
		row 1 & a & b & c & a & b & r\\
		\hline
		row 2 & a & b & r & a & b & c\\
		\hline
		row 3 & b & c & a & b & r & a\\
		\hline
		row 4 & b & r & a & b & c & a\\
		\hline
		row 5 & c & a & b & r & a & b\\
		\hline
		row 6 & r & a & b & c & a & b		\label{tab:t10 bwt-example}
	\end{tabular}
	\caption{Burrows Wheeler Transformation Matrix (all cyclic rotations).}
\end{table}
The resulting string $BWT(w)=L$ corresponds to the last column of $M$, with characters $M[0,n -1],\: \dots \: ,M[n - 1, n - 1]$. The output of the transformation is the pair $(L, i)$, in the example, $L = rcaabb$ and $i = 1$. Obviously, the string $L$ contains consecutive identical characters, which results in better compressibility as shown in Section \ref{ch:Analysis:sec:Improvements by Preprocessing:subSec:bwt}. Longer words with more identical characters in general tend to have more equal characters in series. Also this depicts that the word w is coincidentally in row one of $M$ and is therefore a Lyndon word, because it is the smallest in its conjugacy class.
}

\subsection{Inverse Burrows-Wheeler-Transformation}
\label{ch:Principles of compression:sec:Other:subSec:bwtInverse}
\par{
 The BWT is invertible and given $(L, i)$ where $i$ is the index of $w$ in $M(w)$ one can reconstruct the word $w$. There is a formal proof \cite{kufleitner2009bijective} available for this as well as the bijective variant described in the next section. To abstract the idea of inverting the permutation using only $L$ and $i$ is described by the following process by carrying on with the example. We now reconstruct the original word from the pair $(L,i)$ by first calculating the standard permutation of $L$ by writing $L$ column-wise, add positions, and then sort the pairs lexicographically. Mapping each index with its sorted neighbor results in the standard permutation.
 \begin{table}[h]
 	\centering
 	\begin{tabular}{l|l|l}
		word & word with position & sorted\\
		\hline
		r & (r,1) & (a,3) \\
		c & (c,2) & (a,4) \\
		a & (a,3) & (b,5) \\
		a & (a,4) & (b,6) \\
		b & (b,5) & (c,2) \\
		b & (b,6) & (r,1)
 		\label{tab:t11 standard permutation}
 	\end{tabular}
 	\caption{Standard permutation generation of the word $L$.}
 \end{table}
}
\par{
This yields a standard permutation of:
\begin{gather}
\pi_L =
\begin{pmatrix} 1 & 2 & 3 & 4 & 5 & 6\\ 3 & 4 & 5 & 6 & 2 & 1 \end{pmatrix}
\end{gather}
Following $\pi^1_L(1)$ to $\pi^6_L(1)$ gives us a sequence of positions with $\pi_L(1)=3$ as a beginning, described by:
\begin{gather}
3 \xrightarrow[]{\pi_L} 5 \xrightarrow[]{\pi_L} 2 \xrightarrow[]{\pi_L} 4 \xrightarrow[]{\pi_L} 6 \xrightarrow[]{\pi_L} 1
\end{gather}
If this sequence is applied to the labeling function of the word $L$ it results in:
\begin{gather}
\lambda_L(3) \: \lambda_L(5) \: \lambda_L(2) \: \lambda_L(4) \: \lambda_L(6) \: \lambda_L(1)
= abcabr = w
\end{gather}
This shows that it is possible to reconstruct the word $w$ given the transformation $BWT(w)=L$ and the index $i$.
}

\subsection{Bijective Burrows-Wheeler-Scott Transformation}
\label{ch:Principles of compression:sec:Other:subSec:bwtscott}
\par{
Although, the bijective variant (abbreviated with BWTS, sometimes BBWT) is mostly used further on. It its rather complicated because it maps a word $w$ with $l(w)$ to another word with equal length without any start or stop symbols and no additional index. It strongly relies on the Lyndon factorization, because calculating the BWT of a Lyndon word $v$ always returns 1 as index since we know that $v$ is always the first element in $M$. Then, one can calculate the BWT over all Lyndon factors of the input data, without additional information needed, a proof for this is also mentioned in \cite{kufleitner2009bijective}. The bijective transform is generated by slitting the input into a sequence of Lyndon words. The Chen-Fox-Lyndon \cite{chenFoxlyndonTheorem} theorem states that such a factorization exists and can be found in linear time \cite{lyndonFactorLinearTime}. Afterwards all rotations of all words are sorted as in the Burrows–Wheeler-Transformation which results in a sequence of sorted strings. Again, the final character of each string in this sorted list represents the transformation result. Decoding is also similar up to the last step of the inverse BWT. But instead of giving rotations of words, it gives rotations of Lyndon words , which are sorted into reverse order and then concatenated to get the original result.
}

%% ==============================
\section{State of the Art}
%% ==============================
\label{ch:Principles of compression:sec:SOTA}
\par{
The current state of the art at the time of writing is depicted in the Table \ref{tab:t20 stat of the art}, all algorithms used highest possible compression scheme available. In general there is always a balance between compressed size and compression or decompression speed, where the faster algorithms use mostly some form of dictionary coding sometimes in combination with a Huffman coder to be able to output variable length codes. The more advanced ones like PPMd or ZPAQ use complex context modeling or context mixing approaches where they generate a probability distribution for the next symbol based on just read symbols. In general these complex methods achieve better compression at the expense of using more space or time. The currently modern and most used algorithms have been executed with the Calgary Corpus, all files were processed separately. Algorithms like \emph{compress}, \emph{gzip} and \emph{ZIP} use a dictionary based method like LZ77 or some derivative and achieve reasonable compression at very fast rates. \emph{bzip2} and \emph{p7zip} use a Burrows-Wheeler-Transformation (described in \ref{ch:Principles of compression:sec:Other:subSec:bwt}) and then a combination of different techniques to improve performance. They sometimes even offer explicit choosing of a method, like \emph{7zip} which achieves even better compression when using an advanced probabilistic method called \emph{prediction by partial matching (PPM)}, also implemented but with even better results by \emph{ZPAQ} which performs even better while still maintaining reasonable speeds. In the last decade methods like \emph{zstandard} developed by Facebook or \emph{brotli} developed by Google arose, mostly implementing ANS and other methods arose but they are either desired to be just faster or designed for a different task like \emph{brotli} for HTML and JSON compression and do not achieve best results on this Corpus. In the context of the \href{http://prize.hutter1.net/}{hutter prize}, a competitive compression challenge, other advanced probabilistic methods were developed like \emph{paq8hp1} to \emph{paq8hp12}, a series of algorithms by Alexander Rhatushnyak \cite{mahoney2011large} who won the challenge. A more recent fork of \emph{paq8} from 2014 is the current leader, \emph{cmix}. It is still the best performing algorithm on this list and achieved by far the best results on the Calgary Corpus at expense of almost 3 hours of computing and 32GB of needed ram, displayed in Section \ref{tab:t100benchmark}. Unfortunately it was not possible to run the paq8hp* variants due to an major bug in the only available releases. Most algorithms are available from the default repositories of your Linux distribution, or in the case of Fedora and CentOS, pre-installed with your operating system. The paq8hp* algorithms can be found on the Hutter price website and cmix is developed on \href{https://github.com/byronknoll/cmix}{github} \cite{cmix}.
	\begin{table}[h]
		\begin{tabular}{r|r|r|r|r}
			method & options &  size in bytes & compression & \textit{bps}\\
			\hline
			uncompressed & & 3,145,718 & 100.0\% & 8.00 \\
			compress 4.2.4 & & 1,250,382 & 40.4\% & 3.24 \\
			gzip v1.10 & -9 & 1,021,720 & 32.4\% & 2.60\\
			ZIP v3.0 &-9& 1,019,783 & 32.4\% & 2.59 \\
			zstandard 1.4.2& --ultra-23 -long=30 & 887,004 & 28.1\% & 2.25\\
			bzip2 v1.0.8 & --best & 832,443 & 26.4\% & 2.11 \\
			brotli 1.0.7 & -q 11 -w 24 & 826,638 & 26.3\%& 2.10\\
			p7zip 16.02 (deflate) & a -mx10 & 821,873 & 26.1\% & 2.08 \\
			p7zip 16.02 (PPMd) & a -mm=ppmd o=32 & 763,067& 24.2\% & 1.93 \\
			ZPAQ v7.15 & -m5 & 659,700 & 20.9\% & 1.67  \\
			paq8hp* & - & - & - & - \\
			cmix v18 & -c -d & 554,983 & 17.6\% & 1.41
		\end{tabular}
				\label{tab:t20 stat of the art}
			\caption{State of the art compression ratios on the Calgary Corpus.}
	\end{table}
}

%% ==============================
\section{Limits of Compression}
%% ==============================
\label{ch:Principles of compression:sec:Limits of Conpression}
\par{
All so far introduced methods and techniques work by removing redundant information but applied to random data without any structure or patterns they will fail to compress. This is explained by the Kolmogorov complexity\cite{kolmogorov} off a string $s$ which is the shortest possible program with $s$ as output. Consider the following two strings of equal length \textit{ababababab} and \textit{4c1j5b20fg}. While the first string can simply be denoted as $5 \times ab$, the other one has obviously no simple representation hence it has a higher Kolmogorov complexity. Using the pigeonhole principle it is also possible to trivially prove this by assuming $s$ is a binary string and $l(s) \: = \: n$. This implies $2^n$ possible string, all unequal. Assuming we can reduce all of them by 1 character, they only need $m = n - 1$ space and can only have $2^m$ possible unequal strings. This results in a contradiction because with lossless compression we need a distinct reverse operation and we cannot decode $2^{n-1}$ distinctly into $2^n$ strings. So to recap, no algorithm can compress all data of a given length, even by just one byte. If this was not true it would result in algorithm that could compress every string, so it could be applied recursively until any string has length 0. Therefore, it is obvious that there exist some hard limits to compression.
}
%%% Local Variables:
%%% mode: latex
%%% TeX-master: "thesis"
%%% End: