\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{August 27, 2014}
\maketitle
\section*{real numbers}
we define a real number as $\sum\limits_{i=0}^{\inf}{a_i\cdot 10^{-i}}=a_0.a_1a_2a_3a_4, 0\le a_i\le 9, a_i\in \mathbb{Z}$
\subsection*{2.2e}
\begin{align*}
  x&=2.1357
\end{align*}
\subsection*{example}
$0.99999...=\sum\limits_{i=0}^{\inf}{9\cdot 10^{-i}}=9\left(\sum\limits_{i=1}^{\inf}{10^{-1}}\right)=9\left(\frac{\frac{1}{10}}{1-\frac{1}{10}}\right)=\frac{9}{9}=1$
\section*{not in book}
\subsection*{the real line axiomatic definition}
the real numbers are the set $\mathbb{R}$ with two operations $+,\cdot$ multiplication and addition that are closed in $\mathbb{R}$. if $a,b\in \mathbb{R}$ then $a+b\in \mathbb{R}$ and $a+b\in \mathbb{R}$

and verify the following axioms:
\begin{enumerate}
\item
for all a,b in R, a+b=b+a and a*b=b*a
\item
for all a,b,c in R, (a+b)+c=a+(b+c) and a(bc)=(ab)c
\item
for all a,b,c in R, a(b+c)=a*b+a*c
\item
give a,b in R, there exists c in R such that a+c=b (each c is denoted by (b-a)

in particular the real number a-a is independent of a and is denoted by 0 and the real number 0-a is denoted by -a and called the negative of a
\item
\begin{enumerate}
\item
there is a non-zero element in R
\item
given x,y in R, x not 0 then there exists z in R such that xz=y. such z is denoted by y/x

in particular, x/x is independant of x in R, x not 0 and denoted by 1, and 1/x is called the reciprocal of x ($x^{-1}$.
\end{enumerate}
\end{enumerate}
actually these are properties of fields, not just reals. \emph{field axioms}. this includes rationals.

now for the things that differentiate from rationals
\subsection*{order axioms}
no order in complex numbers, unlike reals and rationals
\begin{enumerate}
\item
for all x,y in R, exactly one of these hold:
\[x<y, x=y, x>y\]
\item
if $x<y$ then for all z in R $x+z<y+z$
\item
if $x>0$ and $y>0$ then $xy>0$
\item
if $x>y$ and $y>z$ then $x>z$
\end{enumerate}
now we have an ordered field defined, both reals and rationals (Q) are ordered fields
\subsection*{completeness axiom}
every non-empty set $S\subset R$ (not proper subset) that is bounded above has a least upper bound
\section*{moving on}
prove that if $x<y$ and $z<0$ then $xz<yz$
\subsection*{lemma}
if $z<0$ then $-z>0$

assume that $z<0$. by 6 we know that $-z$ and 0 we get that exactly one of these happen:

\[-z<0, -z=0,-z>0\]
we know that $z<0$ and $z+(-z)=0$. use 7. $z<0$ so $z+(-z)<0+(-z)$ and $0<-z$
\subsection*{case 1, x is positive}
then by transitivity property (9) y is also positive and by 8 $x(-z)<y(-z)$.
by 7 $x(-z)+xz<y(-z)+xz$ and $0<y(-z)+xz$ and similarly $yz<xz$

\section*{2.3.1 definition}
a set S contained by R is bounded above if there exists M in R such that $x\le M$ for all $x\in S$. M is called an upper bound for S.

A set S contained by R is bounded below if there exists m in R such that x is greater than or equal to m for all x in S. m is called a lower bound for S

a set S contained by R is bounded if it is bounded above and below

Let S be a bounded subset of R

L is the supremum or least upper bound of S if 1) L is an upper bound of S and 2) if M is another upper bound of S then L is less than or equal to M

l is the infimum or greatest lower bound of S if 1) l is a lower bound of S and 2) if m is another lower bound of
\subsection*{examples}
\begin{align*}
  S&=(1,5]\\
  sup(S)&=5\\
  inf(S)&=1\\
\end{align*}
6 is an upper bound and 0 or -4324 are lower bounds for s.

the sup and inf need not necessarily be elements of s

let s be contained in R and bounded. if sup(s) is in s then we call it the maximum of s. if inf(s) is in s then we call it the minimum of s.
\subsection*{examples}
find sup, inf, max and min(if they  exist)

if S=\{2/n,n in N, $n\ge 1$\} then sup(S)=2=max S and inf(S)=0

if S=\{$\frac{(-1)^nn}{n+1}$,$n\in N$\} then sup(S)=1, inf(S)=-1, no max or min

if S=\{$x\in Q:x^2<2$\} then sup(S)=$\sqrt{2}$

if we view s as a subset only of q and forget about R, s has no supremum. any q in Q, q squared greater than 2 is an upper bound for s but there is not a smallest such q in Q
\end{document}


\input{preamble}

\begin{document}
    \section{Subjects}
    \begin{itemize}
        \item Representation based
        \item Density based
        \item Subspace
    \end{itemize}

    \section{Notes}

    \subsection{Representation based}
    Here are some of the main algorithms within representation based clustering:
    \begin{itemize}
        \item $K$-Means
        \item $K$-Medoids
        \todo{Any more?}
    \end{itemize}
    Common for all of these algorithms, is that they all use some point which
    summarizes or ``represents'' the cluster, a common choice being the mean
    (or the \textit{centroid}) $\mu_i$. If we were to make this partitioning in
    an exhaustive way, we would simply find all possible partitions (i.e. all
    possible means) and pick the best. Doing this brute-force approach, results
    in \bigO{k^n/k!} clusterings of $n$ points into $k$ groups, so this is not
    practically feasible at all. Let's instead look at some smarter algorithms
    for solving this problem.

    \subsubsection{$K$-Means}
    The idea of $k$-means is for some $k$ to form $k$ groups so that the sum of
    the (squared) distances between the mean of the groups and their elements
    is minimal. In other words we want to minimize the squared error measure as
    we often do for linear regression.

    Given that our points $p_i=(p_{i1},\dots,p_{id})$ are a point in a
    $d$-dimensional vector space, where the mean of a set of points is defined
    (e.g. euclidean). We then have that the centroid $\mu_C$ for some cluster
    $C$ is defined as:
    \begin{equation*}
        \mu_C = \frac{1}{|C|} \sum_{p_i\in C}p_i
    \end{equation*}
    We can then compute the \textit{sum of squared errors} as:
    \begin{equation*}
        SSE(C)=\sum_{i=1}^{k}\sum_{x_j\in C_i}\|x_j-\mu_i\|^2
    \end{equation*}
    We then want to find the clustering that minimizes SSE:
    \begin{equation*}
        C^*=\arg\min_C SSE(C)
    \end{equation*}
    We can outline the $k$-means algorithm as:
    \begin{enumerate}
        \item Partition the objects into $k$ non-empty subsets
        \item Compute the centroids of the clusters of the current partition.
        The centroid is the center (mean point) of the cluster.
        \item Assign each object to the cluster with the nearest representative.
        \item Go back to Step 2, stop when representatives do not change.
    \end{enumerate}
    Which gives us the following pseudocode:
    \begin{algorithm}
        \caption{$k$-means}\label{alg:k-means}
        \begin{algorithmic}
            \Procedure{$k$-means}{$D,k,\epsilon$}\Comment{asd}
            \State $t \gets 0$
            \State Randomly initialize $k$ centroids
            $\mu_{t1},\mu_{t2},\dots,\mu_{tk} \in \Re^d$
            \Repeat
                \State $t \gets t+1$
                \State $C_j \gets \emptyset$ for all $j = 1,\dots,k$
                \State \textit{// Cluster assignment step}
                \ForAll{$x_j\in D$}
                    \State $j^* \gets \arg\min\limits_{i=1,\dots,k} \|x_j -
                    \mu_{ti}\|^2$
                    \Comment{\textit{Assign $x_j$ to closest centroid}}
                    \State $C_{j^*} \gets C_{j^*} \cup \{x_j\}$
                \EndFor
                \State \textit{// Centroid Update Step}
                \ForAll{$i=1$ to $k$}
                    \State $\mu_{ti} \gets \frac{1}{|C_i|}\sum_{x_j \in C_i}
                    x_j$
                \EndFor
            \Until{$\sum_{i=1}^{k}\|\mu_{ti}-\mu_{t-1,i}\|^2 \leq \epsilon$}
            \EndProcedure
        \end{algorithmic}
    \end{algorithm}
    Then, if we are using euclidean distance (the manhattan distance could look
    differently), the voronoi diagram (a visual representation of the clusters)
    will look like this image:\\

    \voronoi{4}{2}\\

    \textbf{Strengths}
    \begin{itemize}
        \item Relatively efficient \bigO{tkn} where $n$ is \#objects, $k$ is
        \#clusters and $t$ is \#iterations
        \item Normally $k$ and $t$ are both much smaller than $n$ so in
        practice it's only $n$ that matters.
        \item It's easy to implement
    \end{itemize}
    \textbf{Weaknesses}
    \begin{itemize}
        \item Applicable only in vector spaces where the mean is defined
        \item We need to specify the number of clusters $k$ in advanced (it's a
        hyper-parameter)
        \item It's sensitive to noisy data and outliers
        \item Clusters are forced to have convex shapes, for example here is a
        voronois diagram with many clusters:\\
        \voronoi{30}{4}
        \item Both results and running time are very dependent on the initial
        selection of $k$-means, as it often terminates at a \textit{local
        optimum}, however there do exist methods for good initialization.
    \end{itemize}
    There are several variant of $k$-means, e.g. ISODATA which extends
    $k$-means by merging and splitting clusters to eliminate very small
    clusters, at a cost of more hyper-parameters.

    \subsubsection{$K$-Medoids}
    $k$-means implicitly assume euclidean distance, since we need to minimize
    the distance to the mean, there are variations for other distance functions.

    The $k$-medoid algorithm is more general, it's motivated by the $L_1$ norm
    (Manhattan) distance and also works in spaces where a mean is not defined.
    We just need to be able to compute pairwise distances. Furthermore
    $k$-medoid will turn out to be more robust to noise. The general $L_p$
    distance metric (Minkowski-Distance) is defined as:
    \begin{equation*}
        d_p(x,y)=\sqrt[p]{\sum_{i=1}^{d}|x_i-y_i|^p}
    \end{equation*}
    Where Euclidean is the $L_2$ metric and Manhattan is the $L_1$ metric.
    Other examples of distance functions is the maximum metric:
    \begin{equation*}
        d_\infty(x,y)=\max_{1\leq i\leq d} |x_i-y_i|
    \end{equation*}
    Or for two sets we could define it as:
    \begin{equation*}
        d_{set}(x,y)=\frac{|x\cup y| - |x\cap y|}{|x\cup y|}
    \end{equation*}
    Or we could use the Hamming distance etc.

    Now for the basic idea of $k$-medoid, we start by defining some notions.
    First of all, the medoid $m_C$ is the representative object for a cluster
    $C$. The compactness of a clustering $C$ is measured as:
    \begin{equation*}
        TD=\sum_{i=1}^{k}\sum_{p\in C}\dist(p, m_C)
    \end{equation*}
    We can then comput the $k$-medoids, using PAM as follows:
    \begin{enumerate}
        \item Select $k$ objects arbitrarily as medoids, then assign each
        remaining (non-medoid) object to the cluster with the nearest
        representative. We will then compute the $TD$ and name it $TD_{current}$
        \item For each pair (medoid $M$, non-medoid $N$), exhaustively compute
        the $TD$ value for the partition if $N$ was the medoid instead of $M$,
        $TD_{NM}$.
        \item Select the non-medoid $N$ for which the $TD_{NM}$ value was
        minimal, if the $TD$ value is smaller than $TD_{current}$ then:
        \begin{enumerate}
            \item Swap $N$ with $M$
            \item Set $TD_{current} \gets TD_{NM}$
            \item Go back to Step 2
        \end{enumerate}
        \item Stop
    \end{enumerate}

    \textbf{Strength}
    \begin{itemize}
        \item We can use it on arbitrary objects (e.g. points or sets) and
        arbitrary distance measures
        \item Not quite as sensitive to noisy data and outliers as $k$-means
    \end{itemize}
    \textbf{Weaknesses}
    \begin{itemize}
        \item Inefficient
        \item We need to specify number of clusters $k$ in advance and clusters
        need to have convex shapes
    \end{itemize}

    \subsubsection{Expectation Maximization (EM)}
    In $k$-means, each point could only belong to precisely one cluster, EM is
    soft-assignment of points to clusters, so each point has a probability of
    belonging to each cluster. We represent a cluster by a probability
    distribution, usually we will represent it as center point $\mu_C$ and a
    $d\times d$ covariance matrix $\Sigma_C$ for the points in the cluster $C$.
    If we assume that the each cluster $C_i$ is characterized by a multivariate
    normal distribution, then we can define the density function for cluster
    $C$ as:
    \begin{equation*}
        P(x|C)=\frac{1}{\sqrt{(2\pi)^d|\Sigma_C|}} \cdot
        e^{-\frac{1}{2}(x-\mu_C)^T \cdot (\Sigma_C)^{-1} \cdot (x-\mu_C)}
    \end{equation*}
    We can then estimate the a-priori probability of class $C_i$, $P(C_i)$ as
    the relative frequency $W_i$. I.e. we define the $P(C_i)$ as the fraction
    of contribution from the entire data-set $D$ to class $C_i$:
    \begin{equation*}
        P(x)=\sum_{i=1}^{k}W_i \cdot P(x|C_i)
    \end{equation*}
    We can then compute the probability that $x$ belongs to cluster $C_i$ as:
    \begin{equation*}
        P(C_i|x)=W_i \cdot \frac{P(x|C_i)}{P(x)}
    \end{equation*}
    Now, we can compute a measure of the quality of a clustering $M$ as $E(M)$
    which indicates the probability that the data $D$ has been generated by the
    distribution model $M$:
    \begin{equation*}
        E(M)=\sum_{x\in D} \log(P(x))
    \end{equation*}
    Which is what we will try to maximize in EM. Which brings us to our
    algorithm:
    \begin{algorithm}
        \caption{Expectation Maximization}\label{alg:EM}
        \begin{algorithmic}
            \Procedure{EM}{$D,k$}
                \State Generate an initial model $M'=(C_1',\dots,C_k')$
                \Repeat
                    \State \textit{// (Re-) assign points to clusters -
                    expectation step}
                    \ForAll{$x \in D$}
                        \ForAll{$C_i,\, i=1,\dots,k$}
                            \State Compute $P(x|C_i)$, $P(x)$ and $P(C_i|x)$
                        \EndFor
                    \EndFor
                    \State \textit{// (Re-) compute the model - maximization
                    step}
                    \ForAll{$C_i,\, i=1,\dots,k$}
                        \State Recompute $W_i$, $\mu_C$ and $\Sigma_C$
                        \State Compute a new model $M=\{C_1,\dots,C_k\}$ using
                        $W_i$, $\mu_C$ and $\Sigma_C$
                        \State Replace $M'$ with $M$
                    \EndFor
                \Until{$|E(M)-E(M')| < \epsilon$}
            \EndProcedure
        \end{algorithmic}
    \end{algorithm}

    During the maximization step we will need to compute the following values:
    The weight $W_i$ of cluster $C_i$ which is the estimate for the previous
    probability of each cluster. The center $\mu_i$ of cluster $C_i$ and the
    covariance matrix $\Sigma_i$ of cluster $C_i$.

    We can estimate the weight as the fraction of weights that contribute to
    the cluster:
    \begin{equation*}
        W_i=\frac{1}{|D|}\sum_{x\in D} P(C_i|x)
    \end{equation*}

    We can estimate the mean as the weighted average of all points:
    \begin{equation*}
        \mu_i=\frac{\sum_{x\in D}x \cdot P(C_i|x)}{\sum_{x\in D}P(C_i|x)}
    \end{equation*}

    And lastly we can re-estimate the covariance matrix as the weighted
    covariance over all combinations of dimensions:
    \begin{equation*}
        \Sigma_i=\frac{\sum_{x \in D}P(C_i|x)(x-\mu_i)(x-\mu_i)^T}{\sum_{x \in
        D}P(C_i|x)}
    \end{equation*}

    Currently the covariance matrix is a $d \times d$ matrix which can be a
    quite costly as we have to estimate $d^2$ parameters and often we don't
    have enough data for a reliable estimation. An optimization we could make
    here would be to assume that all dimensions are independent and only
    estimate the $d$ parameters that make up the diagonal of the matrix.

    \textbf{Strengths}
    \begin{itemize}
        \item We converge to a minimum (which might be local)
        \item Rather efficient \bigO{n\cdot k \cdot \#iterations}
        \item However \#iterations is quite high in many cases unlike $k$-means
    \end{itemize}
    \textbf{Weaknesses}
    \begin{itemize}
        \item Both result and runtime depends highly on the initial assignment
        \item Also depends strongly on a proper choice of parameter $k$
    \end{itemize}
    Furthermore, with EM, objects may belong to several clusters. If we want it
    to be hard-assignment, then we can assign each object to the cluster which
    it has highest probability of belonging to.

    A last note, a good initialization of EM, is often to run $k$-means first
    to return a crude estimate of the means and the finetune the clustering
    with EM.

    \subsubsection{Initialization of representative based clustering}
    Here is an approach sugested by [Fayyad, Reina and Bradley 1998]:
    \begin{itemize}
        \item Draw $M$ different (small) samples of the dataset
        $S_1,S_2,\dots,S_M$
        \item Cluster each sample, such that we get $M$ estimates for $k$
        representatives:
        \begin{equation*}
            S_i=(S_{i1}, S_{i2},\dots, S_{ik})
        \end{equation*}
        \item We then cluster the unioned set: $DS=S_1 \cup S_2 \cup \dots \cup
        S_M$, one time for each of our $M$ estimates for the $k$
        representatives, leaving us with $M$ different clusterings of $DS$.
        \item Now use the best of these $M$ clusterings as initialization for
        the partitioning clustering of the whole data-set.
    \end{itemize}

    \subsection{Density based clustering}
    Remember how $k$-means worked by seperating into a voronoid diagram?\\
    \voronoi{3}{3}\\
    In general representation-based clustering only work for convex clusters,
    Density-based clustering attempts to cluster more complex shapes, like a
    S-shaped cluster. The basic concept is as follows:

    \begin{itemize}
        \item For any point in a cluster, the local point density around that
        point has to exceed some threshold
        \item The set of points from one cluster is spatially connected
    \end{itemize}
    Local point density at a point $p$ is defined by two parameters:
    \begin{itemize}
        \item $\epsilon$ - radius for the neighbourhood of point $q$:
        \begin{equation*}
            N_\epsilon(q) = \{p \text{ in data-set } D| \dist(p,q)\leq\epsilon\}
        \end{equation*}
        \item MinPts - minimum number of points in the given neighbourhood
        $N(p)$
    \end{itemize}
    We call an object $q$ a core object (or core point) w.r.t. $\epsilon$ if
    $|N_\epsilon(q)| \geq MinPts$ i.e. if there are at least $MinPts$ objects
    within a radius of $\epsilon$ of $q$.

    We now define the following terms:
    \begin{itemize}
        \item $p$ is \textbf{\textit{directly density-reachable}} from $q$
        w.r.t. $\epsilon$ and $MinPts$ if:
        \begin{enumerate}
            \item $p \in N_\epsilon(q)$
            \item $q$ is a core object w.r.t. $\epsilon$ and $MinPts$
        \end{enumerate}
        \item \textbf{\textit{density-reachable}} is a transitive closure of
        \textit{directly} density-reachable
        \item $p$ is \textbf{\textit{density-connected}} to a point $q$ w.r.t.
        $\epsilon$ and $MinPts$, if there is a point $o$ such that both $p$ and
        $q$ are density-reachable from $o$ w.r.t. $\epsilon$ and $MinPts$.
        \item A \textbf{\textit{density-based cluster}} is a non-empty subset
        of $S$ of dataset $D$ satisfying:
        \begin{enumerate}
            \item Maximality: if $p$ is in $S$ and $q$ is density-reachable
            from $p$ then $q$ is in $S$
            \item Connectivity: each object in $S$ is density-connected to all
            other objects
        \end{enumerate}
        \item A \textbf{\textit{density-based clustering}} of a dataset is
        $D:\{S_1,\dots,S_n;N\}$ where:
        \begin{itemize}
            \item $S_1,\dots, S_n$ is all density-based clusters in the
            data-set $D$
            \item $N=D \setminus \{S_1,\dots,S_n\}$ is called the
            \textbf{\textit{noise}} (the objects that are not in any cluster)
        \end{itemize}
    \end{itemize}
    One clarification, imagine two non-core points $p$ and $q$ with a core
    point $c$ between them. Since neither $p$ nor $q$ are core points, they
    cannot be density-reachable, but they can be density-connected because of
    $c$.

    \subsubsection{DBSCAN}

    The basic idea of DBSCAN is that:
    \begin{itemize}
        \item Each object in a density-based cluster $C$ is density-reachable
        from any of its core-objects
        \item Nothing else is density-reachable from core objects
    \end{itemize}
    We can then set up the following algorithm:
    \begin{algorithm}
        \caption{DBSCAN}
        \begin{algorithmic}
            \ForAll{$o \in D$}
                \If{$o$ is not yet classified}
                    \If{$o$ is a core-object}
                        \State Collect all objects density-reachable from $o$
                        and assign them to a new cluster
                    \Else
                        \State Assign $o$ to NOISE
                    \EndIf
                \EndIf
            \EndFor
        \end{algorithmic}
        Running time: \bigO{n * N_\epsilon}\\
        Running time without support: \bigO{n^2}\\
        Running time with tree-based support: \bigO{n\cdot \log(n)}\\
        Running time with direct access to neighbourhood \bigO{n}
    \end{algorithm}

    The issue here is selecting the parameters $\epsilon$ and $MinPts$. An idea
    would be to use the point density of the least dense cluster in the
    data-set as parameters. Then it would find both the least and the more
    dense clusters, but how do we determine these least-dense parameters? A
    heuristic is to look at the distances to the $k$-nearest neighbours.

    The main issue with DBSCAN is that if there are clusters with more than one
    density, then a single $\epsilon$ wont suffice.

    \subsubsection{DENCLUE}
    DenClue works by estimating the density function and find an arbitrarily
    shaped cluster such that each point $x \in C$ is attracted to some
    attractor $x_i^*$. Each density attractor has density above $\xi$, that is
    $\hat{f}(x_i^*) \geq \xi$. And any such two density attractors are
    \textit{density reachable} (such that for all points between them,
    $\hat{f}(y) \geq \xi$).
    % Not gonna go into depths with this here, as I'm not going to be able to
    %learn it in time for exam.

    %TODO \subsection{Hierarchical clustering}

    \subsection{Subspace clustering}
    In high dimensional spaces, a phenomenon called \textit{Curse of
    Dimensionality} occurs. Consider a $d$-dimensional query point $Q$ and $N$
    $d$-dimensional sample points $X_1,\cdots,X_N$. We then have that:
    \begin{align*}
        DMIN_d&=\min\{dist_2(x_i,Q) | 1 \leq i \leq N\}\\
        DMAX_d&=\max\{dist_2(x_i,Q) | 1 \leq i \leq N\}\\
        \lim_{d\rightarrow \infty} \Pr[DMAX_d \leq (1+\epsilon)DMIN_d] = 1
    \end{align*}
    That is, the probability that $DMAX$ is $\epsilon$ close to $DMIN$
    converges to $1$. In other words, distance functions use their
    functionality.

    Subspace clustering is about identifying clusters in any subspace of the
    full dimensionality.

    \subsubsection{CLIQUE}
    One of the first approaches to the subspace clustering problem is the
    CLIQUE method. CLIQUE is a grid-based algorithm which recursively navigates
    through the set of possible subspaces in a bottom-up way.

    In CLIQUE, each subset is split into blocks of width $\xi$, called units.
    All units which exceed a threshold $\tau$ are retained. $\xi$ and $\tau$
    are the input parameters of CLIQUE. We start with 1-dimensional dense
    units, and then the step from $(k-1)$ to $k$ can be described as taking the
    $(k-1)$-dimensional units as candidates and then all the candidates which
    have the first $(k-2)$ dimensions in common.

    The big drawback is the high dependency of grid placement. It runs linear
    in $n$ superlinear in $d$.

    \subsubsection{SUBCLU}
    SUBCLU is an extension of DBSCAN. We want to use the density-connected
    properties in individual subspaces, in order to detect clusters in high
    dimensional spaces as well. One straightforward approach would be to run
    DBSCAN in all possible subspaces, however there is $2^d$ subspaces.

    A more efficient approach would be if we could use the clustering
    information of previous subspaces in the process of generating all clusters
    and drop all subspaces that cannot contain density-connected clusters.
    However, density connected clusters are not monotonic, i.e. if a cluster
    $C$ is density connected in $S \subseteq A$ then $C$ need not be a
    density-connected cluster in $T \subseteq S$, as it may not be the maximal
    cluster w.r.t. density-reachability anymore.

    However, density-connected sets are monotonic, i.e. if $C$ is density
    connected in $S$, then it is also density-connected in $T$. We get that for
    all $C \subseteq D$ which is a cluster in $S \subseteq A$ and $T \subseteq
    S$:

    \begin{itemize}
        \item $o$ is a core object in $S$ $\implies$ $o$ is a core object in
        $T$.
        \item $o$ is directly density reachable from $q$ in $S$ $\implies$ $o$
        is directly density reachable from $q$ in $T$.
        \item $o$ is density-reachable from $in$ in $s$ $\implies$ $o$ is
        density reachable from $q$ in $T$.
        \item $o$ is density-connected to $q$ in $S$ $\implies$ $o$ is
        density-connected to $q$ in $T$.
        \item $o$ and $q$ is part of a connected set in $S$ $\implies$ $o$ and
        $q$ is part of a connected in $T$.
    \end{itemize}

    We can inverse this, and say that if we e.g. have a set in space $AB$ then
    then if $p$ and $q$ are connected in $A$ but not in $B$ then they wont be
    connected in $AB$. The key idea here, is that we do not have to examine any
    subspace $S$ if at least on $T_i \subset S$ contains no cluster (i.e. a
    density connected set). reversely, if all $T_i \subset S$ contain clusters,
    then we have to test $S$.

    SUBCLU is based on based on a bottom-up, greedy algorithm to detect
    density-connected clusters in all subspaces of high dimensional data.

    The SUBCLU algorithm has automatic detection of subspaces with clusters,
    automatic detection of clusters and no assumptions regarding data
    distributions. The drawbacks are that the parameter settings highly affect
    clustering results, they are challenged by large differences in densities
    and computation is at least \bigO{n^2} and typically much greater. However
    the results are much better than that of CLIQUE.

    \subsection{Measuring the quality of a clustering}
    For choosing $k$, one idea could be to determine the clustering for
    $k=2,3,\dots,n-1$ and then choose the ``best'' clustering. But how do we
    actually measure the quality of a clustering? If we need to use it to
    select the best $k$, it has to be independent of $k$, and the measures for
    compactness of a clustering (TD) are decreasing with increasing values of
    $k$.

    So we define the silhouette coefficient, with the basic idea that:
    \begin{itemize}
        \item Quality of clustering = how appropriate is the mapping of objects
        to clusters.
        \item Elements in a cluster should be ``similar'' to their
        representative, so we should measure the distance of objects to their
        representative $a$.
        \item Elements in different clusters should be different, so we measure
        the average distance of objects to an alternative cluster (second
        closest) $b$.
    \end{itemize}
    We can define $a(o)$ as the average distance between object $o$ and the
    objects in its cluster $A$:
    \begin{equation*}
        a(o)=\frac{1}{|C(o)|} \sum_{p\in C(o)} \dist(o,p)
    \end{equation*}
    And we can define $b(o)$ as the average distance between object $o$ in its
    ``second closest'' cluster $B$
    \begin{equation*}
        b(o)=\min_{C_i \neq C(o)} \frac{1}{|C_i|} \sum_{p\in C_i} \dist(o,p)
    \end{equation*}
    Then we can compute the silhouette of $o$ as:
    \begin{equation*}
        s(o)=\frac{b(o)-a(o)}{\max\{a(o),b(o)\}}
    \end{equation*}
    The values of the silhouette coefficient range from $-1$ to $+1$ and we can
    read the values as:
    \begin{itemize}
        \item $s(o)=-1$: bad, on average $o$ is closer to members of $B$.
        \item $s(o)=0$: $o$ is somewhere in-between $A$ and $B$
        \item $s(o)=1$: good, $o$ is on average closest to its cluster $A$
    \end{itemize}
    The silhouette coefficient $s_C$ of a clustering is simply the average
    silhouette of all objects and we can read the value as:
    \begin{itemize}
        \item $0.7 < s_C \leq 1.0$ strong structure
        \item $0.5 < s_C \leq 0.7$ medium structure
        \item $0.25 < s_C \leq 0.5$ weak structure
        \item $s_C \leq 0.25$ no structure
    \end{itemize}
\end{document}


\chapter{\label{chap:inference}Statistical models:
  learning, estimation, inference and prediction}

% If we know that we have a fair coin,
% we can calculate the probability of obtaining
% a certain number of heads in a sequence of coin tosses
% using probability theory.
% What if, we do not know if the coin is fair or not?
% Can we \emph{infer} whether the coin is fair or not
% from a finite number of outcomes?
% Similarly,
% if we know the election results in a country,
% probability theory can tells us
% the probability of a randomly picked the voters
% having voted a particular political party.
% But what can we say about the election results,
% given we know which parties a limited number of voters will vote?
% In general,
% given a finite sample of data,
% we want to know, or infer,
% some properties of the data source (e.g., population)
% that the data comes from.
% These questions fall into the area of \emph{statistical inference}.

Modeling natural phenomena is one of the basic activities in science.
We build models of a natural phenomenon to gain a better insight
into the phenomenon being modeled,
and predict aspects that we do not know about it.
Models are used in many fields in science and engineering.
Just to name a few well-known examples,
we note Galilean model of solar system in astronomy;
Bohr model of atom in physics;
animal models in medicine that are used for studying human diseases
on non-human animals;
econometric models in economics;
the formal models of atmospheric conditions used for weather forecasts;
scaled physical models of bridges, cars, and other objects
that are used frequently in design and engineering of these objects.

Whether they are abstract (mathematical) models,
or physical ones,
these models allow us to study the phenomenon being modeled
in a convenient way.
They allow us to test hypotheses and make \emph{inferences}
or \emph{predictions} that we cannot do directly
on the real object being studied.

In this lecture, we are interested in
\emph{statistical model}s
which constitute a very broad range of models used in many disciplines.
In a statistical model,
part of the phenomenon being modeled is subject to uncertainties.
As a result,
the data we collect or observe does not lead to fully deterministic model.
Statistical models are the main subject of study
in statistics and machine learning,
and naturally, has a very important role in natural language processing.

Before this general introduction to statistical models,
we remind that ``all models are wrong, some are useful.''%
\autocite[p.~424]{box1986}
In any modeling effort,
one has to make simplifications or assumptions that do not match
with the object or the process being modeled.
However,
as long as we are aware of its shortcomings,
we can build useful models.

\section*{Where to go from here}

\textcite{wasserman2004}


\newpage
\section{Elementary Functions}

\begin{Operator}[acos]{ACOS}
\index{arccosine}
The \name{acos} operator returns the arccosine of its argument.

\begin{Syntax}
\name{acos}\(\meta{expression}\) or \name{acos} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
acos(ab);                  &          ACOS(AB) \\
acos 15;                   &          ACOS(15) \\
df(acos(x*y),x);           &         \rfrac{SQRT( - X^{2}*Y^{2} + 1)*Y}{X^{2}*Y^{2} - 1} \\
on rounded; \\
res := acos(sqrt(2)/2);    &          RES := 0.785398163397 \\
res-pi/4;                  &          0
\end{Examples}

\begin{Comments}
An explicit numeric value is not given unless the switch \nameref{rounded} is
on and the argument has an absolute numeric value less than or equal to 1.
\end{Comments}
\end{Operator}

\begin{Operator}[acosh]{ACOSH}
\index{hyperbolic arccosine}
\name{acosh} represents the hyperbolic arccosine of its argument.  It takes
an arbitrary scalar expression as its argument. The derivative of
\name{acosh} is known to the system.  Numerical values may also be found by
turning on the switch \nameref{rounded}.

\begin{Syntax}
\name{acosh}\(\meta{expression}\) or \name{acosh} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
acosh a;                   &       ACOSH(A) \\
acosh(0);                  &       ACOSH(0) \\
df(acosh(a**2),a);         &       \rfrac{2*SQRT(A^{4} - 1)*A}{A^{4} - 1} \\
int(acosh(x),x);           &       INT(ACOSH(X),X)
\end{Examples}
\begin{Comments}
You may attach functionality by defining \name{acosh} to be the inverse of
\name{cosh}.  This is done by the commands
\begin{verbatim}
        put('cosh,'inverse,'acosh);
        put('acosh,'inverse,'cosh);
\end{verbatim}
You can write a procedure to attach integrals or other
functions to \name{acosh}.  You may wish to add a check to see that its
argument is properly restricted.
\end{Comments}
\end{Operator}

\begin{Operator}[acot]{ACOT}
\index{arccotangent}
\name{acot} represents the arccotangent of its argument.  It takes
an arbitrary scalar expression as its argument. The derivative of
\name{acot} is known to the system.  Numerical values may also be found by
turning on the switch \nameref{rounded}.

\begin{Syntax}
\name{acot}\(\meta{expression}\) or \name{acot} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.
You can add functionality yourself with \name{let} and procedures.

\end{Operator}

\begin{Operator}[acoth]{ACOTH}
\index{hyperbolic cotangent}
\name{acoth} represents the inverse hyperbolic cotangent of its argument.
It takes an arbitrary scalar expression as its argument.  The derivative
of \name{acoth} is known to the system.  Numerical values may also be found
by turning on the switch \nameref{rounded}.

\begin{Syntax}
\name{acoth}\(\meta{expression}\) or \name{acoth} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array,
matrix or vector expression. \meta{simple\_expression} must be a single
identifier or begin with a prefix operator name.  You can add
functionality yourself with \name{let} and procedures.
\end{Operator}

\begin{Operator}[acsc]{ACSC}
\index{arccosecant}
The \name{acsc} operator returns the arccosecant of its argument.

\begin{Syntax}
\name{acsc}\(\meta{expression}\) or \name{acsc} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
acsc(ab);                  &          ACSC(AB) \\
acsc 15;                   &          ACSC(15) \\
df(acsc(x*y),x);           &         \rfrac{-SQRT(X^{2}*Y^{2} - 1)}{X*(X^{2}*Y^{2} - 1)} \\
on rounded; \\
res := acsc(2/sqrt(3));    &          RES := 1.0471975512 \\
res-pi/3;                  &          0
\end{Examples}

\begin{Comments}
An explicit numeric value is not given unless the switch \name{rounded} is
on and the argument has an absolute numeric value less than or equal to 1.
\end{Comments}
\end{Operator}

\begin{Operator}[acsch]{ACSCH}
\index{arccosecant}
The \name{acsch} operator returns the hyperbolic arccosecant of its argument.

\begin{Syntax}
\name{acsch}\(\meta{expression}\) or \name{acsch} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
acsch(ab);                 &          ACSCH(AB) \\
acsch 15;                  &          ACSCH(15) \\
df(acsch(x*y),x);           &         \rfrac{-SQRT(X^{2}*Y^{2} + 1)}{X*(X^{2}*Y^{2} + 1)} \\
on rounded; \\
res := acsch(3);           &          RES := 0.327450150237
\end{Examples}

\begin{Comments}
An explicit numeric value is not given unless the switch \name{rounded} is
on and the argument has an absolute numeric value less than or equal to 1.
\end{Comments}
\end{Operator}

\begin{Operator}[asec]{ASEC}
\index{arccosecant}
The \name{asec} operator returns the arccosecant of its argument.

\begin{Syntax}
\name{asec}\(\meta{expression}\) or \name{asec} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
asec(ab);                  &          ASEC(AB) \\
asec 15;                   &          ASEC(15) \\
df(asec(x*y),x);           &         \rfrac{SQRT(X^{2}*Y^{2} - 1)}{X*(X^{2}*Y^{2} - 1)} \\
on rounded; \\
res := asec sqrt(2);       &          RES := 0.785398163397 \\
res-pi/4;                  &          0
\end{Examples}

\begin{Comments}
An explicit numeric value is not given unless the switch \name{rounded} is
on and the argument has an absolute numeric value greater or equal to 1.
\end{Comments}
\end{Operator}

\begin{Operator}[asech]{ASECH}
\index{hyperbolic arccosecant}
\name{asech} represents the hyperbolic arccosecant of its argument.  It takes
an arbitrary scalar expression as its argument. The derivative of
\name{asech} is known to the system.  Numerical values may also be found by
turning on the switch \nameref{rounded}.

\begin{Syntax}
\name{asech}\(\meta{expression}\) or \name{asech} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
asech a;                   &       ASECH(A) \\
asech(1);                  &       0 \\
df(acosh(a**2),a);         &       \rfrac{2*SQRT(- A^{4} + 1)}{A*(A^{4} - 1)} \\
int(asech(x),x);           &       INT(ASECH(X),X)
\end{Examples}
\begin{Comments}
You may attach functionality by defining \name{asech} to be the inverse of
\name{sech}.  This is done by the commands
\begin{verbatim}
        put('sech,'inverse,'asech);
        put('asech,'inverse,'sech);
\end{verbatim}
You can write a procedure to attach integrals or other
functions to \name{asech}.  You may wish to add a check to see that its
argument is properly restricted.
\end{Comments}
\end{Operator}

\begin{Operator}[asin]{ASIN}
\index{arcsine}
The \name{asin} operator returns the arcsine of its argument.

\begin{Syntax}
\name{asin}\(\meta{expression}\) or \name{asin} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
asin(givenangle);          &      ASIN(GIVENANGLE) \\
asin(5);                   &      ASIN(5) \\
df(asin(2*x),x);           &      - \rfrac{2*SQRT( - 4*X^{2} + 1))}{4*X^{2} - 1} \\
on rounded; \\
asin .5;                   &      0.523598775598 \\
asin(sqrt(3));             &      ASIN(1.73205080757) \\
asin(sqrt(3)/2);           &      1.04719755120 \\
\end{Examples}

\begin{Comments}
A numeric value is not returned by \name{asin} unless the switch
\name{rounded} is on and its argument has an absolute value less than or
equal to 1.
\end{Comments}
\end{Operator}

\begin{Operator}[asinh]{ASINH}
\index{hyperbolic arcsine}
The \name{asinh} operator returns the hyperbolic arcsine of its argument.
The derivative of \name{asinh} and some simple transformations are known
to the system.

\begin{Syntax}
\name{asinh}\(\meta{expression}\) or \name{asinh} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
asinh d;                   &       ASINH(D) \\
asinh(1);                  &       ASINH(1) \\
df(asinh(2*x),x);          &       \rfrac{2*SQRT(4*X^{2} + 1))}{4*X^{2} + 1} \\
\end{Examples}

\begin{Comments}
You may attach further functionality by defining \name{asinh} to be the
inverse of \name{sinh}.  This is done by the commands
\begin{verbatim}
        put('sinh,'inverse,'asinh);
        put('asinh,'inverse,'sinh);
\end{verbatim}
A numeric value is not returned by \name{asinh} unless the switch
\name{rounded} is on and its argument evaluates to a number.
\end{Comments}
\end{Operator}

\begin{Operator}[atan]{ATAN}
\index{arctangent}
The \name{atan} operator returns the arctangent of its argument.

\begin{Syntax}
\name{atan}\(\meta{expression}\) or \name{atan} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
atan(middle);              &      ATAN(MIDDLE) \\
on rounded; \\
atan 45;                   &      1.54857776147 \\
off rounded; \\
int(atan(x),x);            &      \rfrac{2*ATAN(X)*X - LOG(X^{2} + 1)}{2} \\
df(atan(y**2),y);          &      \rfrac{2*Y}{Y^{4} + 1}
\end{Examples}

\begin{Comments}
A numeric value is not returned by \name{atan} unless the switch
\nameref{rounded} is on and its argument evaluates to a number.
\end{Comments}
\end{Operator}

\begin{Operator}[atanh]{ATANH}
\index{hyperbolic arctangent}
The \name{atanh} operator returns the hyperbolic arctangent of its argument.
The derivative of \name{asinh} and some simple transformations are known
to the system.

\begin{Syntax}
\name{atanh}\(\meta{expression}\) or \name{atanh} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression.  \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
atanh aa;                  &       ATANH(AA) \\
atanh(1);                  &       ATANH(1) \\
df(atanh(x*y),y);          &       \rfrac{- X}{X^{2}*Y^{2} - 1}
\end{Examples}

\begin{Comments}
A numeric value is not returned by \name{asinh} unless the switch
\name{rounded} is on and its argument evaluates to a number.
You may attach additional functionality by defining \name{atanh} to be the
inverse of \name{tanh}.  This is done by the commands

\begin{verbatim}
        put('tanh,'inverse,'atanh);
        put('atanh,'inverse,'tanh);
\end{verbatim}
\end{Comments}
\end{Operator}

\begin{Operator}[atan2]{ATAN2}

\begin{Syntax}
\name{atan2}\(\meta{expression},\meta{expression}\)
\end{Syntax}

\meta{expression} is any valid scalar REDUCE expression.  In
\nameref{rounded} mode, if a numerical value exists, \name{atan2} returns
the principal value of the arc tangent of the second argument divided by
the first in the range [-pi,+pi] radians, using the signs of both
arguments to determine the quadrant of the return value.  An expression in
terms of \name{atan2} is returned in other cases.

\begin{Examples}
atan2(3,2);                  &        ATAN2(3,2); \\
on rounded; \\
atan2(3,2);                  &        0.982793723247 \\
atan2(a,b);                  &        ATAN2(A,B); \\
atan2(1,0);                  &        1.57079632679
\end{Examples}
\begin{Comments}
\name{atan2} returns a numeric value only if \nameref{rounded} is on. Then
\name{atan2} is calculated to the current degree of floating point precision.

\end{Comments}
\end{Operator}

\begin{Operator}[cos]{COS}
The \name{cos} operator returns the cosine of its argument.

\begin{Syntax}
\name{cos}\(\meta{expression}\) or \name{cos} \meta{simple\_expression}
\end{Syntax}

\meta{expression} is any valid scalar REDUCE expression,
\meta{simple\_expression} is a single identifier or begins with a prefix
operator name.

\begin{Examples}

cos abc;                     &        COS(ABC) \\

cos(pi);                     &        -1 \\

cos 4;                       &        COS(4) \\

on rounded; \\

cos(4);                      &        - 0.653643620864 \\

cos log 5;                   &        - 0.0386319699339
\end{Examples}
\begin{Comments}
\name{cos} returns a numeric value only if \nameref{rounded} is on. Then the
cosine is calculated to the current degree of floating point precision.

\end{Comments}
\end{Operator}

\begin{Operator}[cosh]{COSH}
\index{hyperbolic cosine}
The \name{cosh} operator returns the hyperbolic cosine of its argument.
The derivative of \name{cosh} and some simple transformations are known
to the system.

\begin{Syntax}
\name{cosh}\(\meta{expression}\) or \name{cosh} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression. \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}

cosh b;                      &             COSH(B) \\

cosh(0);                     &             1 \\

df(cosh(x*y),x);             &             SINH(X*Y)*Y \\

int(cosh(x),x);              &             SINH(X)
\end{Examples}

\begin{Comments}
You may attach further functionality by defining its inverse (see
\nameref{acosh}).
A numeric value is not returned by \name{cosh} unless the switch
\nameref{rounded} is on and its argument evaluates to a number.

\end{Comments}
\end{Operator}

\begin{Operator}[cot]{COT}
\name{cot} represents the cotangent of its argument.  It takes an arbitrary
scalar expression as its argument. The derivative of \name{acot} and some
simple properties are known to the system.

\begin{Syntax}
\name{cot}\(\meta{expression}\) or \name{cot} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression.  \meta{simple\_expression}
must be a single identifier or begin with a prefix operator name.

\begin{Examples}
cot(a)*tan(a);               &         COT(A)*TAN(A)) \\
cot(1);                      &         COT(1) \\
df(cot(2*x),x);              &         - 2*(COT(2*X)^{2}  + 1)
\end{Examples}
\begin{Comments}
Numerical values of expressions involving \name{cot} may be found by
turning on the switch \nameref{rounded}.

\end{Comments}
\end{Operator}

\begin{Operator}[coth]{COTH}
\index{hyperbolic cotangent}
The \name{coth} operator returns the hyperbolic cotangent of its argument.
The derivative of \name{coth} and some simple transformations are known
to the system.

\begin{Syntax}
\name{coth}\(\meta{expression}\) or \name{coth} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression.  \meta{simple\_expression}
must be a single identifier or begin with a prefix operator name.

\begin{Examples}
df(coth(x*y),x);             &      - Y*(COTH(X*Y)^{2}  - 1) \\

coth acoth z;                &      Z
\end{Examples}

\begin{Comments}
You can write \nameref{let} statements and procedures to add further
functionality to \name{coth} if you wish.  Numerical values of expressions
involving \name{coth} may also be found by turning on the switch
\nameref{rounded}.

\end{Comments}
\end{Operator}

\begin{Operator}[csc]{CSC}
\index{cosecant}
The \name{csc} operator returns the cosecant of its argument.
The derivative of \name{csc} and some simple transformations are known
to the system.

\begin{Syntax}
\name{csc}\(\meta{expression}\) or \name{csc} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression.  \meta{simple\_expression}
must be a single identifier or begin with a prefix operator name.

\begin{Examples}

csc(q)*sin(q);               &       CSC(Q)*SIN(Q) \\

df(csc(x*y),x);              &       -COT(X*Y)*CSC(X*Y)*Y
\end{Examples}

\begin{Comments}
You can write \nameref{let} statements and procedures to add further
functionality to \name{csc} if you wish.  Numerical values of expressions
involving \name{csc} may also be found by turning on the switch
\nameref{rounded}.

\end{Comments}
\end{Operator}

\begin{Operator}[csch]{CSCH}
\index{hyperbolic cosecan}
The \name{cosh} operator returns the hyperbolic cosecant of its argument.
The derivative of \name{csch} and some simple transformations are known
to the system.

\begin{Syntax}
\name{csch}\(\meta{expression}\) or \name{csch} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression. \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}

csch b;                      &             CSCH(B) \\

csch(0);                     &             0 \\

df(csch(x*y),x);             &             - COTH(X*Y)*CSCH(X*Y)*Y \\

int(csch(x),x);              &             INT(CSCH(X),X)
\end{Examples}

\begin{Comments}
% You may attach further functionality by defining its inverse (see
% \nameref{acsch}).
A numeric value is not returned by \name{csch} unless the switch
\nameref{rounded} is on and its argument evaluates to a number.
\end{Comments}
\end{Operator}

\begin{Operator}[erf]{ERF}
\index{error function}
The \name{erf} operator represents the error function, defined by
\begin{TEX}
\begin{displaymath}
erf(x) = {2\over \sqrt{\pi}} \int{e^{-x^2}\,dx}
\end{displaymath}
\end{TEX}
\begin{INFO}
erf(x) = (2/sqrt(pi))*int(e^(-x^2),x)
\end{INFO}

A limited number of its properties are known to the system, including the
fact that it is an odd function.  Its derivative is known, and from this,
some integrals may be computed.  However, a complete integration procedure
for this operator is not currently included.

\begin{Examples}
erf(0);                      &    0 \\
erf(-a);                     &    - ERF(A) \\
df(erf(x**2),x);             &    \rfrac{4*SQRT(PI)*X}{E^{X^{4}}*PI} \\\\
int(erf(x),x);               &    \rfrac{E^{X^{2}}*ERF(X)*PI*X + SQRT(PI)}{E^{X^{2}}*PI}
\end{Examples}

\end{Operator}

\begin{Operator}[exp]{EXP}
\index{exponential function}
The \name{exp} operator returns \name{e} raised to the power of its argument.
\begin{Syntax}
\name{exp}\(\meta{expression}\) or \name{exp} \meta{simple\_expression}
\end{Syntax}

\meta{expression} can be any valid REDUCE scalar expression.
\meta{simple\_expression} must be a single identifier or begin with a
prefix operator.

\begin{Examples}
exp(sin(x));                 &   E^{SIN X} \\
exp(11);                     &   E^{11} \\
on rounded; \\
exp sin(pi/3);               &   2.37744267524
\end{Examples}
\begin{Comments}
Numeric values are returned only when \name{rounded} is on.
The single letter \name{e} with the exponential operator \name{^} or
\name{**} may be substituted for \name{exp} without change of function.
\end{Comments}
\end{Operator}

\begin{Operator}[sec]{SEC}
The \name{sec} operator returns the secant of its argument.

\begin{Syntax}
\name{sec}\(\meta{expression}\) or \name{sec} \meta{simple\_expression}
\end{Syntax}

\meta{expression} is any valid scalar REDUCE expression,
\meta{simple\_expression} is a single identifier or begins with a prefix
operator name.

\begin{Examples}

sec abc;                     &        SEC(ABC) \\

sec(pi);                     &        -1 \\

sec 4;                       &        SEC(4) \\

on rounded; \\

sec(4);                      &        - 1.52988565647 \\

sec log 5;                   &        - 25.8852966005
\end{Examples}
\begin{Comments}
\name{sec} returns a numeric value only if \nameref{rounded} is on. Then the
secant is calculated to the current degree of floating point precision.
\end{Comments}
\end{Operator}

\begin{Operator}[sech]{SECH}
\index{hyperbolic secant}
The \name{sech} operator returns the hyperbolic secant of its argument.

\begin{Syntax}
\name{sech}\(\meta{expression}\) or \name{sech} \meta{simple\_expression}
\end{Syntax}

\meta{expression} is any valid scalar REDUCE expression,
\meta{simple\_expression} is a single identifier or begins with a prefix
operator name.

\begin{Examples}
sech abc;                    &        SECH(ABC) \\

sech(0);                     &        1 \\

sech 4;                      &        SECH(4) \\

on rounded; \\

sech(4);                     &        0.0366189934737 \\

sech log 5;                  &        0.384615384615
\end{Examples}
\begin{Comments}
\name{sech} returns a numeric value only if \nameref{rounded} is on. Then the
expression is calculated to the current degree of floating point precision.
\end{Comments}
\end{Operator}

\begin{Operator}[sin]{SIN}
\index{sine}
The \name{sin} operator returns the sine of its argument.
\begin{Syntax}
\name{sin}\(\meta{expression}\) or \name{sin} \meta{simple\_expression}
\end{Syntax}

\meta{expression} is any valid scalar REDUCE expression,
\meta{simple\_expression} is a single identifier or begins with a prefix
operator name.

\begin{Examples}
sin aa;                      &        SIN(AA) \\
sin(pi/2);                   &        1 \\
on rounded; \\
sin 3;                       &        0.14112000806 \\
sin(pi/2);                   &        1.0
\end{Examples}
\begin{Comments}
\name{sin} returns a numeric value only if \name{rounded} is on.
Then the sine is calculated to the current degree of floating point precision.
The argument in this case is assumed to be in radians.
\end{Comments}
\end{Operator}

\begin{Operator}[sinh]{SINH}
\index{hyperbolic sine}
The \name{sinh} operator returns the hyperbolic sine of its argument.
The derivative of \name{sinh} and some simple transformations are known
to the system.

\begin{Syntax}
\name{sinh}\(\meta{expression}\) or \name{sinh} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression. \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}

sinh b;                      &             SINH(B) \\

sinh(0);                     &             0 \\
df(sinh(x**2),x);            &          2*COSH(X^{2})*X \\
int(sinh(4*x),x);            &          \rfrac{COSH(4*X)}{4} \\
on rounded; \\
sinh 4; & 27.2899171971
\end{Examples}

\begin{Comments}
You may attach further functionality by defining its inverse (see
\nameref{asinh}).
A numeric value is not returned by \name{sinh} unless the switch
\nameref{rounded} is on and its argument evaluates to a number.
\end{Comments}
\end{Operator}

\begin{Operator}[tan]{TAN}
The \name{tan} operator returns the tangent of its argument.
\begin{Syntax}
\name{tan}\(\meta{expression}\) or \name{tan} \meta{simple\_expression}

\end{Syntax}

\meta{expression} is any valid scalar REDUCE expression,
\meta{simple\_expression} is a single identifier or begins with a prefix
operator name.

\begin{Examples}
tan a;                       &        TAN(A) \\
tan(pi/5);                   &        TAN(\rfrac{PI}{5}) \\
on rounded;
tan(pi/5);                   &        0.726542528005
\end{Examples}
\begin{Comments}
\name{tan} returns a numeric value only if \name{rounded} is on.  Then the
tangent is calculated to the current degree of floating point accuracy.

When \nameref{rounded} is on,
no check is made to see if the argument of \name{tan} is a multiple of
$\pi/2$, for which the tangent goes to positive or negative infinity.
(Of course, since REDUCE uses a fixed-point representation of $\pi/2$,
it produces a large but not infinite number.)  You need to make a check for
multiples of $\pi/2$ in any program you use that might possibly ask
for the tangent of such a quantity.
\end{Comments}
\end{Operator}

\begin{Operator}[tanh]{TANH}
\index{hyperbolic tangent}
The \name{tanh} operator returns the hyperbolic tangent of its argument.
The derivative of \name{tanh} and some simple transformations are known
to the system.

\begin{Syntax}
\name{tanh}\(\meta{expression}\) or \name{tanh} \meta{simple\_expression}
\end{Syntax}

\meta{expression} may be any scalar REDUCE expression, not an array, matrix or
vector expression. \meta{simple\_expression} must be a single identifier or
begin with a prefix operator name.

\begin{Examples}
tanh b;                      &          TANH(B) \\
tanh(0);                     &          0 \\
df(tanh(x*y),x);             &          Y*( - TANH(X*Y)^{2} + 1) \\
int(tanh(x),x);              &          LOG(E^{2*X} + 1) - X \\
on rounded; tanh 2;          &          0.964027580076
\end{Examples}

\begin{Comments}
You may attach further functionality by defining its inverse (see
\nameref{atanh}).
A numeric value is not returned by \name{tanh} unless the switch
\nameref{rounded} is on and its argument evaluates to a number.
\end{Comments}
\end{Operator}


\newpage
\section{Data Augmentation with Explicit Regularization Formula}

\paragraph{Data Augmentation}
I think data augmentation is some kind of regularization.
First we have,
\begin{equation}\label{key}
D_1 = \text{training set}, \quad D_2 = \text{test set},
\end{equation}
and the standard loss function:
\begin{equation}\label{key}
L(\theta) = \sum_{i=1}^N \ell( f(x_i; \theta), y_i) = N \mathbb{E}_{(x,y)\in D_1} \ell( f(x; \theta), y).
\end{equation}

Let me write the FAA training process:
\begin{itemize}
	\item For k = 1:200 (epoch iteration)
	\item Shuffle the data set, and for each training data $x_i \in D_1$, randomly choose one policy $\tau^k_i \in \mathcal T$, and then the training data becomes
	\begin{equation}\label{key}
	\tilde \tau^{k}(D_1) = \{ \tau^k_i(x_i): x_i \in D_1 \},
	\end{equation}
	\item SGD and train.
\end{itemize}

Here, for CIFAR100, we have the next properties of augmentation policies:
\begin{equation}\label{key}
T = \# \mathcal T = 500,
\end{equation}
for each policy $\tau_i \in \mathcal T$,
\begin{equation}\label{key}
\tau_i(x) = \begin{cases}
x, \quad \text{with probability}~(1-p^i_1)(1-p^i_2) \\
T^i_1(x), \quad \text{with probability}~p^i_1(1-p^i_2) \\
T^i_2(x), \quad \text{with probability}~(1-p^i_1)p^i_2 \\
T^i_2(T^i_1(x)), \quad \text{with probability}~ p^i_1p^i_2
\end{cases}
\end{equation}

A key observation is the expectation of the SGD with FAA or general data augmentation. Think about a mini-batch step:
\begin{equation}\label{key}
\theta^{k+1} = \theta^k - \eta \nabla \sum_{x\in \mathcal B_k} \ell(f(\tau(x)), y),
\end{equation}
where \blue{for every $x$, the transfer $\tau$ is a sample} from the next distribution:
\begin{equation}\label{key}
\mathbb P(\tau = \tau^i) = \frac{1}{T}, \forall i = 1, 2, \cdots, T,
\end{equation}
and
\begin{equation}\label{key}
\tau^i = \begin{cases}
id, \quad \text{with probability}~(1-p^i_1)(1-p^i_2) \\
T^i_1, \quad \text{with probability}~p^i_1(1-p^i_2) \\
T^i_2, \quad \text{with probability}~(1-p^i_1)p^i_2 \\
T^i_2\circ T^i_1, \quad \text{with probability}~ p^i_1p^i_2
\end{cases}.
\end{equation}

Thus, we know the ``real'' loss function is:
\begin{equation}\label{key}
\tilde L(\theta)  = N T \mathbb{E}_{(x,y) \in D_1} \mathbb{E}_{\mathcal T} \mathbb{E}_{\tau} \left( \frac{\ell \left( f(\tau(x);\theta), y\right)}{P(\tau)}  \right)
\end{equation}
Our we can write them with details:
\begin{equation}\label{key}
\begin{aligned}
\tilde L(\theta) = N T \frac{1}{N} \sum_{i=1}^N \big( \frac{1}{T}\sum_{t=1}^T \big( &(1-p^t_1)(1-p^t_2)\frac{\ell(f(x_i;\theta),y_i)}{\mathbb P(\tau= id)}  \\
&+ p^t_1(1-p^t_2)  \frac{\ell(f(T^t_1(x_i);\theta),y_i)}{\mathbb P(\tau = T^t_1)} + (1-p^t_1)p^t_2 \frac{\ell(f(T^t_2(x_i);\theta),y_i)}{\mathbb P(\tau = T^t_2)}  \\
&+  p^t_1p^t_2\frac{\ell(f(T^t_2 (T^t_1(x_i) );\theta),y_i)}{\mathbb P(\tau = T^t_2 \circ T^t_1)}   \big) \big).
\end{aligned}
\end{equation}

Then, the FAA data augmentation can be understood as a ``direct'' SGD method to solve the above problem by sampling
\begin{equation}\label{key}
\mathbb{E}_{(x,y) \in D_1} \quad  \mathbb{E}_{ \mathcal T} \quad \text{and} \quad \mathbb{E}_{\tau},
\end{equation}
one-by-one.

So, the main difference here is we have a new loss function. For simplicity, if we think $p^t_1 \approx p_1$ and $p^t_1 \approx p_2$.
We will have the new loss function as:
\begin{equation}\label{key}
\begin{aligned}
\tilde L(\theta) &=  NT \frac{1}{N}\sum_{i=1}^N \left(\ell(f(x_i;\theta),y_i) + \frac{1}{T}\left(\sum_1 + \sum_2 + \sum_{12}\right)  \right) \\
&= T\sum_{i=1}^N \ell(f(x_i;\theta),y_i)  +  NT \frac{1}{N} \sum_{i=1}^N  \frac{1}{T} \sum_{t=1}^T \left(\sum_tL_1 + \sum_tL_2 +  \sum_{t}L_{12}\right) \\
&=  T L(\theta) +NT(1-(1-p_1)(1-p_2)) \frac{1}{N} \sum_{i=1}^N  \frac{1}{T} \sum_{t=1}^T \left( \tilde P_1 \sum_tL_1 + \tilde P_2 \sum_tL_2 +\tilde P_{12} \sum_{t}L_{12}\right)  \\
&= T L(\theta) +NT(1-(1-p_1)(1-p_2)) \mathbb{E}_{(x,y) \in D_1} \mathbb{E}_{ \bar{\mathcal T}} \mathbb{E}_{\bar \tau} \left( \frac{\ell \left( f(\bar T(x);\theta), y\right)}{\tilde{\mathbb P}(\bar \tau)}  \right)
\end{aligned}.
\end{equation}
Here we have the new $\bar \tau^t$ defined as:
\begin{equation}\label{key}
\bar \tau^t(x) = \begin{cases}
T^t_1(x), \quad \text{with probability}~\tilde{\mathbb{ P}}_1 = \frac{p_1(1-p_2)}{1-(1-p_1)(1-p_2)} \\
T^t_2(x), \quad \text{with probability}~\tilde{\mathbb{ P}}_2 = \frac{(1-p_1)p_2}{1-(1-p_1)(1-p_2)}\\
T^t_2(T^t_1(x)), \quad \text{with probability}~ \tilde{\mathbb{ P}}_{12} = \frac{p_1p_2}{1-(1-p_1)(1-p_2)}
\end{cases}.
\end{equation}

Finally, we have the equivalent loss function as:
\begin{equation}\label{key}
\begin{aligned}
\tilde L(\theta) &= L(\theta) + \frac{(1-(1-p_1)(1-p_2))}{T} NT\mathbb{E}_{(x,y) \in D_1} \mathbb{E}_{ \bar{\mathcal T}} \mathbb{E}_{\bar \tau} \left( \frac{\ell \left( f(\bar T(x);\theta), y\right)}{\tilde{\mathbb P}(\bar \tau)}  \right) \\
&= L(\theta) + \blue{\frac{(1-(1-p_1)(1-p_2))}{T} } R(\theta; \blue{\bar{\mathcal T}}).
\end{aligned}
\end{equation}

Then, we can understand these data augmentation as a regularization method with a ``direct'' SGD method which likes this batch normalization methods...

Then we have the next observation:
\begin{itemize}
	\item This can help to explain why this can help to improve the generalization accuracy.
	\item This also explain why the training accuracy dose not have good improvement.
	\item This help to explain why FAA have a bigger gap than basic augmentation. (The parameter $T$ for FAA is bigger than basic one.)
	\item Some possible numerical experiments:
	\begin{itemize}
		\item Different $T$ may have different scale of gap;
		\item Modify $p_1$ and $p_2$ will have the similar result for $T$...
	\end{itemize}
	\item Because of the special form of regularization, I am trying to give some mathematical analysis on why the test accuracy
	with FAA will drop. And how to design some numerical experiments to verify.
\end{itemize}

\subsection{Preliminary numerical results}

\subsubsection{Jianhong's results}
There are $p_1$ and $p_2$. I set $p_1=p_2$ and choose different values of $p_1$ to investigate the effect of the augmentation.

Jianhong trained with basic and FAA augmentation and saved the model. Then Jianhong evaluated the training and test accuracy with different augmentation techniques for this model.

\subsection{CIFAR-100}
\begin{table}[!htbp]
	\centering
	\caption{Training and test accuracy with different augmentation techniques on CIFAR-100}
	\scalebox{0.7}{	\begin{tabular}{|c|c|c|c|c|c|c|c|}
			\hline
			p1	&	lambda	&	training accuracy	&	training+aug accuracy	&	test accuracy	&	test+faa accuracy	&	test+basic accuracy	&	test+basic+faa accuracy	\\\hline
			0	&	0.00000	&	99.98	&	99.98	&	74.54	&	74.54	&	73.81	&	73.93	\\\hline
			0.1	&	0.00042	&	99.96	&	98.49	&	75.40	&	74.13	&	74.79	&	73.46	\\\hline
			0.2	&	0.00080	&	99.97	&	97.90	&	75.58	&	73.86	&	74.97	&	73.40	\\\hline
			0.3	&	0.00113	&	99.97	&	97.57	&	76.59	&	74.51	&	75.80	&	73.21	\\\hline
			0.4	&	0.00142	&	99.92	&	96.86	&	77.00	&	74.56	&	76.49	&	73.20	\\\hline
			0.5	&	0.00167	&	99.93	&	96.46	&	76.79	&	73.77	&	76.27	&	72.43	\\\hline
			0.6	&	0.00187	&	99.92	&	95.86	&	76.99	&	73.25	&	76.51	&	72.31	\\\hline
			0.7	&	0.00202	&	99.92	&	95.27	&	77.24	&	73.38	&	76.68	&	72.04	\\\hline
			0.8	&	0.00213	&	99.93	&	94.77	&	77.61	&	72.82	&	76.58	&	71.76	\\\hline
			0.9	&	0.00220	&	99.82	&	94.20	&	77.67	&	72.73	&	77.27	&	71.07	\\\hline
			1	&	0.00222	&	99.76	&	92.95	&	77.25	&	72.31	&	76.61	&	70.28	\\\hline
	\end{tabular}}
\end{table}

\begin{table}[!htbp]
	\centering
	\caption{Training and test loss with different augmentation techniques on CIFAR-100}
	\scalebox{0.8}{	\begin{tabular}{|c|c|c|c|c|c|c|c|}
			\hline
			p1	&	lambda	&	training loss	&	training+aug loss	&	test loss	&	test+faa loss	&	test+basic loss	&	test+basic+faa loss	\\\hline
			0	&	0.00000	&	0.0043	&	0.2092	&	1.1758	&	1.1758	&	1.1376	&	1.1457	\\\hline
			0.1	&	0.00042	&	0.0078	&	0.3749	&	1.0719	&	1.1314	&	1.0498	&	1.1153	\\\hline
			0.2	&	0.00080	&	0.0068	&	0.4241	&	1.0650	&	1.1287	&	1.0409	&	1.1151	\\\hline
			0.3	&	0.00113	&	0.0072	&	0.4607	&	1.0069	&	1.0998	&	0.9718	&	1.0948	\\\hline
			0.4	&	0.00142	&	0.0087	&	0.5078	&	0.9931	&	1.0801	&	0.9637	&	1.0829	\\\hline
			0.5	&	0.00167	&	0.0086	&	0.5362	&	0.9687	&	1.0964	&	0.9657	&	1.1228	\\\hline
			0.6	&	0.00187	&	0.0106	&	0.5701	&	0.9569	&	1.1089	&	0.9605	&	1.1087	\\\hline
			0.7	&	0.00202	&	0.0117	&	0.5989	&	0.9480	&	1.0921	&	0.9522	&	1.1137	\\\hline
			0.8	&	0.00213	&	0.0126	&	0.6270	&	0.9602	&	1.1343	&	0.9400	&	1.1482	\\\hline
			0.9	&	0.00220	&	0.0141	&	0.6526	&	0.9295	&	1.1231	&	0.9273	&	1.1637	\\\hline
			1	&	0.00222	&	0.0199	&	0.7041	&	0.9240	&	1.1157	&	0.9159	&	1.1749	\\\hline
	\end{tabular}}
\end{table}

\newpage
\subsection{CIFAR-10}
\begin{table}[!htbp]
	\centering
	\caption{Training and test accuracy with different augmentation techniques on CIFAR-10}
	\scalebox{0.7}{	\begin{tabular}{|c|c|c|c|c|c|c|c|}
			\hline
			p1	&	lambda	&	training accuracy	&	training+aug accuracy	&	test accuracy	&	test+faa accuracy	&	test+basic accuracy	&	test+basic+faa accuracy	\\\hline
			0	&	0.00000	&	100.00	&	99.99	&	94.37	&	94.37	&	94.17	&	94.34	\\\hline
			0.1	&	0.00042	&	100.00	&	99.56	&	95.39	&	94.88	&	94.98	&	94.43	\\\hline
			0.2	&	0.00080	&	100.00	&	99.18	&	95.73	&	94.88	&	95.03	&	94.04	\\\hline
			0.3	&	0.00113	&	99.99	&	99.09	&	95.44	&	94.26	&	95.54	&	94.14	\\\hline
			0.4	&	0.00142	&	100.00	&	98.93	&	95.56	&	94.43	&	95.61	&	93.95	\\\hline
			0.5	&	0.00167	&	100.00	&	98.70	&	95.95	&	93.73	&	95.53	&	93.85	\\\hline
			0.6	&	0.00187	&	100.00	&	98.63	&	95.65	&	93.83	&	95.46	&	93.26	\\\hline
			0.7	&	0.00202	&	99.99	&	98.26	&	95.93	&	94.06	&	95.66	&	93.27	\\\hline
			0.8	&	0.00213	&	99.99	&	98.17	&	95.77	&	93.03	&	95.68	&	92.99	\\\hline
			0.9	&	0.00220	&	99.99	&	97.94	&	96.08	&	93.48	&	95.51	&	92.80	\\\hline
			1	&	0.00222	&	99.98	&	97.76	&	95.98	&	92.80	&	95.49	&	91.99	\\\hline
	\end{tabular}}
\end{table}

\begin{table}[!htbp]
	\centering
	\caption{Training and test loss with different augmentation techniques on CIFAR-10}
	\scalebox{0.8}{	\begin{tabular}{|c|c|c|c|c|c|c|c|}
			\hline
			p1	&	lambda	&	training loss	&	training+aug loss	&	test loss	&	test+faa loss	&	test+basic loss	&	test+basic+faa loss	\\\hline
			0	&	0.00000	&	0.0006	&	0.0833	&	0.2655	&	0.2655	&	0.2491	&	0.2463	\\\hline
			0.1	&	0.00042	&	0.0010	&	0.1182	&	0.1748	&	0.1915	&	0.1802	&	0.1997	\\\hline
			0.2	&	0.00080	&	0.0012	&	0.1409	&	0.1623	&	0.1841	&	0.1797	&	0.2036	\\\hline
			0.3	&	0.00113	&	0.0011	&	0.1487	&	0.1714	&	0.2030	&	0.1668	&	0.2054	\\\hline
			0.4	&	0.00142	&	0.0010	&	0.1630	&	0.1696	&	0.2086	&	0.1619	&	0.2075	\\\hline
			0.5	&	0.00167	&	0.0011	&	0.1762	&	0.1583	&	0.2191	&	0.1543	&	0.2122	\\\hline
			0.6	&	0.00187	&	0.0012	&	0.1840	&	0.1666	&	0.2230	&	0.1687	&	0.2278	\\\hline
			0.7	&	0.00202	&	0.0011	&	0.1969	&	0.1587	&	0.2117	&	0.1578	&	0.2277	\\\hline
			0.8	&	0.00213	&	0.0013	&	0.2028	&	0.1595	&	0.2324	&	0.1564	&	0.2311	\\\hline
			0.9	&	0.00220	&	0.0013	&	0.2149	&	0.1540	&	0.2303	&	0.1552	&	0.2386	\\\hline
			1	&	0.00222	&	0.0018	&	0.2268	&	0.1506	&	0.2424	&	0.1554	&	0.2559	\\\hline	\end{tabular}}
\end{table}

\subsubsection{Jianqing's results}
The next table shows Jianqing's results for $p$ version for FAA with 450 sub-policies with
the basic data augmentation before standard FAA augmentation is applied.
\begin{table}[!htbp]
\centering
\begin{tabular}{| l | c | c | c | c | r |}
	\hline
	p          & train acc & train+aug acc & test accuracy  &  test+aug acc & test acc drop\\
	\hline
	0.0          &  99.97   & 99.97       &  74.78         &   74.11            &   0.67  \\
	\hline
	0.1          &  99.97   & 98.53       &  76.44         &   74.56            &   1.88  \\
	\hline
	0.2          &  99.96   & 97.83       &  76.72         &   73.68            &   3.04  \\
	\hline
	0.3          &  99.96   & 97.35       &  76.75         &   73.92            &   2.83  \\
	\hline
	0.4          &  99.96   & 96.83       &  76.78         &   73.45            &   3.33  \\
	\hline
	0.5          &  99.96   & 96.66       &  77.07         &   72.61            &   4.46  \\
	\hline
	0.6          &  99.95   & 95.91       &  77.29         &   72.75            &   4.54  \\
	\hline
	0.7          &  99.93   & 95.51       &  77.67         &   72.59            &   5.08  \\
	\hline
	0.8          &  99.92   & 95.13       &  77.67         &   72.05            &   5.62  \\
	\hline
	0.9          &  99.91   & 94.27       &  77.68         &   71.25            &   6.43  \\
	\hline
	1.0          &  99.75   & 93.22       &  77.67         &   70.31            &   7.36  \\
	\hline
\end{tabular}
\end{table}


% declare document class and geometry
\documentclass[12pt]{article} % use larger type; default would be 10pt
\usepackage[english]{babel} % for hyphenation dictionary
%\setdefaultlanguage{english} % polyglossia command for use with XeTeX / LuaTeX
\usepackage[margin=1in]{geometry} % handle page geometry

% import packages and commands
\input{../header2.tex}

% title information
\title{Phys 221A -- Quantum Mechanics -- Lec17}
\author{UCLA, Fall 2014}
\date{\formatdate{1}{12}{2014}} % Activate to display a given date or no date (if empty),
         % otherwise the current date is printed
         % format: formatdate{dd}{mm}{yyyy}

\begin{document}
\maketitle

\section{Angular momentum}

In general, the total angular momentum has two components
\begin{eqn}
\v J = \v L + \v S,
\end{eqn}
where $\v L = \v r \times \v p$ is the orbital angular momentum and $\v S$ is the spin angular momentum. The spin angular momentum acts in an abstract space, spin space, with dimensionality $2s+1$ where $s$ is the spin quantum number (i.e. $S_i$ are $(2s+1) \times (2s+1)$ Hermitian matrices). The spin quantum number is integer-valued for bosons and half-integer-valued for fermions. (In QFT, bosons can be modeled by complex-valued spinors but to model fermions we must introduce Grassman numbers, i.e. anticommuting numbers.) Since $\v L$ and $\v S$ act in different spaces, we find commutation relations
\begin{eqn}
[L_i, L_j] = i \hbar \epsilon_{ijk} L_k, \qquad
[S_i, S_j] = i \hbar \epsilon_{ijk} S_k, \qquad
[L_i, S_j] = 0,
\end{eqn}
and thus
\begin{eqn}
[J_i, J_j] = i \hbar \epsilon_{ijk} J_k.
\end{eqn}

Quantum mechanically, we will represent a general ket, including angular momentum, as
\begin{eqn}
\ket{\alpha} = \ket{\psi} \otimes \ket{\sigma}
\end{eqn}
where $\ket{\psi}$ is the usual complex-valued $L_2$ wavefunction, and $\ket{\sigma}$ is the spin state living in $(2s+1)$-dimensional spin space. We will represent the spin state $\ket{\sigma}$ in an orthonormal basis as a column vector called a ``spinor'',
\begin{eqn}
\ket{\sigma} \rightarrow \chi_\sigma = \pmat{\alpha \\ \beta}, \quad
\alpha, \beta \in \C
\end{eqn}
with inner product
\begin{eqn}
\braket{\sigma}{\sigma'} = \chi^\dagger \chi' = \alpha^* \alpha' + \beta^* \beta'.
\end{eqn}
Generally, we will demand that the spin state be normalized so that
\begin{eqn}
1 = \braket{\sigma}{\sigma} = \chi^\dagger \chi = \abs{\alpha}^2 + \abs{\beta}^2.
\end{eqn}
Furthermore, the general inner product is written
\begin{eqn}
\braket{\alpha}{\alpha'} = \braket{\psi}{\psi'} \braket{\sigma}{\sigma'}.
\end{eqn}

\subsection{Rotations}

What happens to the general ket under a general rotation $R : \ket{\alpha} \mapsto \ket{\alpha'}$? For the wavefunction we have as always
\begin{eqn}
\ket{\psi} \mapsto \ket{\psi'} = e^{-i \phi \uv \cdot \v L / \hbar} \ket{\psi}.
\end{eqn}
On the other hand, the spin state does not depend on spatial coordinates, so instead it transforms as
\begin{eqn}
\ket{\sigma} \mapsto \ket{\sigma'} = D(R) \ket{\sigma}
\end{eqn}
where $D(R)$ is some representation of the rotation $R$, i.e. a homomorphism
\begin{eqn}
D(R_1 R_2) = D(R_1) D(R_2).
\end{eqn}
Since it must preserve the norm, the representation must be unitary, $D^\dagger = D\inv$, and unimodular, i.e. $\det D = 1$, so that
\begin{eqn}
D(R) \in SU(2s+1).
\end{eqn}

Recall that under the Euclidean rotations $R_i(\theta)$ we have
\begin{eqn}
[R_x(\epsilon), R_y(\epsilon)] = R_z(\epsilon^2) - 1 + \bigO(\epsilon^3).
\end{eqn}
Since the representation is a homomorphism, this commutation relation must pass through to the representation. So we will define $S_k$ by
\begin{eqn}
D(R_k(\epsilon)) = 1 - \frac{i}{\hbar} \, \epsilon S_k + \bigO(\epsilon^2).
\end{eqn}
Then the commutation relation tells us that
\begin{eqn}
[S_x, S_y] = i\hbar S_z,
\end{eqn}
as expected. Thus $\v S$ is the generator of $SU(2s+1)$ rotations in spin space for a real rotation $R$ parameterized by $\phi, \uv n$, i.e.
\begin{eqn}
D(R) = U_S(\uv n, \phi) = e^{i \phi \uv n \cdot \v S / \hbar} \in SU(2s+1).
\end{eqn}
We can define a total rotation operator
\begin{eqn}
U = U_L \otimes U_S.
\end{eqn}
Finally, we can parameterize any orientational configuration by Euler angles
\begin{eqn}
U_S (\phi, \theta, \psi) = e^{-i \phi S_z / \hbar} e^{-i \theta S_y / \hbar} e^{-i \psi S_z / \hbar}.
\end{eqn}

A useful identity is given by
\begin{eqn}
e^{-i \phi J_i / \hbar} J_j e^{i \phi J_i / \hbar} = e^{-i \phi \opname{ad}_{J_i} / \hbar} J_j = J_j \cos \phi + \epsilon_{ijk} J_k \sin \phi,
\end{eqn}
where $\opname{ad}_X Y \equiv [X,Y]$. Then for any vector $\v a$ we have
\begin{eqn}
e^{-i \phi \uv n \cdot \v J / \hbar} (\v a \cdot \v J) e^{i \phi \uv n \cdot \v J / \hbar} = (R \v a) \cdot \v J,
\end{eqn}
but we can also pull out the vector $\v a$ so that
\begin{eqn}
\v a \cdot \left[ e^{-i \phi \uv n \cdot \v J / \hbar} \v J e^{i \phi \uv n \cdot \v J / \hbar} \right] = \v a \cdot (R\inv \v J),
\end{eqn}
where we obtained the RHS from
\begin{eqn}
(R \v a) \cdot \v J = (R \v a) \cdot (R (R\inv \v J)) = \v a \cdot (R\inv \v J),
\end{eqn}
the last equality following from the fact that for a rotation we have
\begin{eqn}
(R \v a) \cdot (R \v b) = \v a \cdot \v b.
\end{eqn}
Then since $R(\phi)\inv = R(-\phi)$ we find the useful relation
\begin{eqn}
e^{i \phi \uv n \cdot \v J / \hbar} \v J e^{-i \phi \uv n \cdot \v J / \hbar} = R \v J.
\end{eqn}
Thus under a rotation the matrix element changes by
\begin{eqn}
\matrixel{\alpha}{\v J}{\alpha} \overset{R}{\longrightarrow} \matrixel{\alpha}{e^{i \phi \uv n \cdot \v J / \hbar} \v J e^{-i \phi \uv n \cdot \v J / \hbar}}{\alpha} = R \avg{\v J}.
\end{eqn}
Finally, we have
\begin{eqn}
\v S = \frac{\hbar}{2} \v \sigma.
\end{eqn}

\end{document}


\chapter{Kinetics}
\section{Introduction}
\subsection{Topics}
\begin{enumerate}
  \item Definition of Rate
  \item Related rates and stoichiometry
  \item Rates and KMT
  \item Reaction Order
  \item Temperature and rate
  \item Mechanisms of reaction
\end{enumerate}

\section{Rates and Reactions}
To illustrate the rates for a simple reaction, an example is shown:

\ce{2NO2(g) -> N2O4(g)}

\begin{tikzpicture}
  \begin{axis}[
      xlabel=$x$, ylabel=$M$,
      xmin=0, xmax=4,
      ymin=0, ymax=4,
      legend entries={$M_{\ce{NO2}}$, $M_{\ce{N2O4}}$}
    ]
    \addplot[blue, samples=1000] {-ln(x/4)};
    \addplot[red, samples=1000] {ln(x/4)+4};
  \end{axis}
\end{tikzpicture}

\begin{equation}
  \frac{d}{dt}\left[\ce{N2O4}\right] = \frac{-1}{2}\frac{d}{dt}\left[\ce{NO2}\right]
\end{equation}

Rates are given to have coefficients of the inverse of their stoichiometric
coefficient, and have a negative sign only if they are a reactant.

\section{Changing the Reaction Rate}
On a molecular level, the following occur during a reaction:
\begin{enumerate}
  \item Reacting particles must come into contact with each other.
  \item The particles must have some amount of energy available to break
    existing bonds.
  \item The orientntation of the reacting species may be important in the
    reacting process.
\end{enumerate}

As such, the rate at which a reaction occurs may be altered in the following
ways:
\begin{description}
  \item[Change the concentration of the reactants] The concentration of the
    reactants is directly proportional to the amount of times the reacting
    species will come into contact with each other.
  \item[Temperature] temperature not only determines molecular speed but also
    the amount of energy available to break bonds. If we increase temperature,
    not only will we get more collisions, but we'll have more collisions that
    break bonds.
  \item[Catalysts] A catalyst is a substance that is not used up in the
    reaction, but fits with the reacting species as a "key" to orient them
    together to react more effectively.
\end{description}

\section{Rate Law}
\begin{equation}
  \text{rate}=k[R_1]^m[R_2]^n \ldots
\end{equation}

To calculate the rate law for an equation involving two products, you must
eliminate all exponents except for one and then solve. Once that information is
obtained, you can go back, determine the other exponent, and then the value of
$k$. An example is shown below:\\

\textit{Both A and B are reactants. We look at the rate of reaction in several
different experiments.}

\begin{table}[]
\centering
\begin{tabular}{l|l|l|l|}
\cline{2-4}
                        & {[}A{]} & {[}B{]} & Initial rate (M/sec) \\ \hline
\multicolumn{1}{|l|}{1} & 0.1     & 0.1     & 0.014                \\ \hline
\multicolumn{1}{|l|}{2} & 0.1     & 0.2     & 0.056                \\ \hline
\multicolumn{1}{|l|}{3} & 0.5     & 0.2     & 0.280                \\ \hline
\multicolumn{1}{|l|}{4} & 1.0     & 0.4     & 2.24                 \\ \hline
\end{tabular}
\end{table}

\begin{equation}
  \begin{split}
    \frac{\SI{.056}{\Molar\per\second}}{\SI{.014}{\Molar\per\second}} &=
    \frac{k[0.1]^m[0.2]^n}{k[0.1]^m[0.1]^n}\\
    4 &= 2^n \Rightarrow n = 2
  \end{split}
\end{equation}

\begin{equation}
  \begin{split}
    \frac{\SI{.280}{\Molar\per\second}}{\SI{.056}{\Molar\per\second}} &=
    \frac{k[0.5]^m[0.2]^n}{k[0.1]^m[0.2]^n}\\
    5 &= 5^m \Rightarrow m = 5
  \end{split}
\end{equation}

\begin{equation}
  \SI{.014}{\Molar\per\second}=k[0.1][0.1]^2\\
  k=\SI{14}{\Molar\squared\per\second}
\end{equation}

\section{First Order Reactions}
First order reactions proceed according to one reactant. We know, therefore,
that the following is true:

\begin{equation}
  \text{rate}=k[R]^1
\end{equation}

Therefore...

\begin{equation}
  \begin{split}
    \text{rate} &= \frac{-d[R]}{dt}\\
                &= k[R]
  \end{split}
\end{equation}

\begin{equation}
  \frac{1}{[R]}d[R]=-k dt
\end{equation}

\begin{equation}
  \int_{[R]_0}^{[R]_t} \frac{1}{[R]} d[R] = -k\int_{0}^{1} dt
\end{equation}

Which yields the integrated rate-law (1st order):

\begin{equation}
  \ln([R]_t) - \ln([R]_0) = -kt
\end{equation}

This produces a graph which looks like the following:

\begin{tikzpicture}
  \begin{axis}[
    xlabel={$t$}, ylabel={$\ln[A]$},
     xmin=0, xmax=5
  ]

    \addplot[blue] {-x};
  \end{axis}
\end{tikzpicture}

\subsection{$n^{\text{th}}$-life}
To calculate the $n^{\text{th}}$ life of a reactant, you simply use the
integrated rate law and the following equation:

\begin{equation}
  \frac{[R]_1}{[R]_0} = \frac{1}{n}
\end{equation}


\section{Floquet theory}

Since we describe the lifetime of an electron in certain Landau level using conventianal perturbation theeory, now we can apply the Floquet theory to identify the difference of these methods.

\noindent
First we need to identify the \textit{quasienergies} and periodic \textit{Floquet modes} for derived wavefunctions \eqref{1.52} for a 2DEG system with both stationary magnetic field and strong dressing filed.

\noindent
Let's consider the following paramter which is lineraly increasing in time
\begin{equation} \label{3.1}
  \Delta_E t \equiv \frac{t}{T} \int_0^T dt' \; L(\zeta,\dot{\zeta},t')
\end{equation}
where we can calculate this using Eq. \eqref{1.42} and \eqref{1.51} as follows
\begin{equation} \label{3.2}
  \begin{aligned}
    \Delta_E t =
    \frac{t}{T} \int_0^T dt' \;
    \frac{1}{2}m_e \frac{(eE\omega)^2}{m_e^2(\omega_0^2 - \omega^2)^2} \cos[2](\omega t')
    & - \frac{1}{2}m_e\omega_0^2 \frac{(eE)^2}{m_e^2(\omega_0^2 - \omega^2)^2} \sin[2](\omega t') \\
    & +
    \frac{eE}{m_e(\omega_0^2 - \omega^2)} \sin(\omega t') eE \sin(\omega t')
  \end{aligned}
\end{equation}
\begin{equation} \label{3.3}
  \begin{aligned}
    \Delta_E t  =
    \frac{t\omega}{2\pi} \times
    \frac{(eE)^2}{2m_e(\omega_0^2 - \omega^2)^2}
    \bigg[
    \omega^2
    \int_0^T dt' \; \cos[2](\omega t')
    & -
    \omega_0^2
    \int_0^T dt'\; \sin[2](\omega t') \\
    & +
    2(\omega_0^2 - \omega^2) \int_0^T dt'\; \sin[2](\omega t')
    \bigg]
  \end{aligned}
\end{equation}
\begin{equation} \label{3.4}
  \begin{aligned}
    \Delta_E t  =
    \frac{t\omega}{2\pi} \times
    \frac{(eE)^2}{2m_e(\omega_0^2 - \omega^2)^2}
    \bigg[
    \omega^2
    \frac{\pi}{\omega}
    -
    \omega_0^2
    \frac{\pi}{\omega}
    +
    2(\omega_0^2 - \omega^2) \frac{\pi}{\omega}
    \bigg]
  \end{aligned}
\end{equation}
\begin{equation} \label{3.5}
  \begin{aligned}
    \Delta_E t  =
    \frac{t\omega}{2} \times
    \frac{(eE)^2}{2m_e(\omega_0^2 - \omega^2)^2}
    (\omega_0^2 - \omega^2) =
    \frac{(eE)^2}{4m_e(\omega_0^2 - \omega^2)} t
  \end{aligned}
\end{equation}
Since this is the continuous increasing part of the Laggrangian integral in Eq. \eqref{1.52} we can make this as $2\omega$ periodic function as follows
\begin{equation} \label{3.6}
    \Lambda\equiv
    \int_0^t dt' \; L(\zeta,\dot{\zeta},t') -
    \frac{t}{T} \int_0^T dt' \; L(\zeta,\dot{\zeta},t')
\end{equation}
which can be proved as follows. First consider the first term of the $\Lambda$
\begin{equation} \label{3.7}
  \begin{aligned}
    \int_0^t dt' \; L(\zeta,\dot{\zeta},t') =
    \frac{(eE)^2}{2m_e(\omega_0^2 - \omega^2)^2}
    \bigg[
    \omega^2
    \int_0^t dt' \; \cos[2](\omega t')
    & -
    \omega_0^2
    \int_0^t dt'\; \sin[2](\omega t') \\
    & +
    2(\omega_0^2 - \omega^2) \int_0^t dt'\; \sin[2](\omega t')
    \bigg]
  \end{aligned}
\end{equation}
\begin{equation} \label{3.8}
  \begin{aligned}
    \int_0^t dt' \; L(\zeta,\dot{\zeta},t') =
    \frac{(eE)^2}{2m_e(\omega_0^2 - \omega^2)^2}
    \bigg[
    \omega^2
    \qty[\frac{t}{2} + \frac{\sin(2\omega t)}{4\omega}]
    & -
    \omega_0^2
    \qty[\frac{t}{2} - \frac{\sin(2\omega t)}{4\omega}] \\
    & +
    2(\omega_0^2 - \omega^2)
    \qty[\frac{t}{2} - \frac{\sin(2\omega t)}{4\omega}]
    \bigg]
  \end{aligned}
\end{equation}
\begin{equation} \label{3.9}
  \begin{aligned}
    \int_0^t dt' \; L(\zeta,\dot{\zeta},t')  =
    \frac{(eE)^2}{2m_e(\omega_0^2 - \omega^2)^2}
    \bigg[&
    \frac{t}{2}
    \qty[\omega^2 - \omega_0^2 + 2\omega_0^2 - 2\omega^2] \\
    & +
    \frac{\sin(2\omega t)}{4\omega}
    \qty[\omega^2 + \omega_0^2 - 2\omega_0^2 + 2\omega^2]
    \bigg]
  \end{aligned}
\end{equation}
\begin{equation} \label{3.10}
    \int_0^t dt' \; L(\zeta,\dot{\zeta},t')  =
    \frac{(eE)^2}{4m_e(\omega_0^2 - \omega^2)}t
    +
    \frac{(eE)^2\qty(3\omega^2 - \omega_0^2)}{8m_e\omega(\omega_0^2 - \omega^2)^2}
    \sin(2\omega t)
\end{equation}
then using Eq.\eqref{3.5} we can write this as
\begin{equation} \label{3.11}
    \int_0^t dt' \; L(\zeta,\dot{\zeta},t')  =
    \Delta_E t
    +
    \frac{(eE)^2\qty(3\omega^2 - \omega_0^2)}{8m_e\omega(\omega_0^2 - \omega^2)^2}
    \sin(2\omega t).
\end{equation}
Now we can express
\begin{equation} \label{3.12}
    \Lambda  =
    \Delta_E t
    +
    \frac{(eE)^2\qty(3\omega^2 - \omega_0^2)}{8m_e\omega(\omega_0^2 - \omega^2)^2}
    \sin(2\omega t) - \Delta_E t
    =
    \frac{(eE)^2\qty(3\omega^2 - \omega_0^2)}{8m_e\omega(\omega_0^2 - \omega^2)^2} \sin(2\omega t)
\end{equation}
which is a periodic function in time with $2\omega$ frequency.

\noindent
Now using this parmaters we can factorize the wavefunction \eqref{1.52} as linearly time dependend part and periodic time dependend part as follows
\begin{equation} \label{3.13}
  \begin{aligned}
    \psi_{\alpha}(x,y,t)  = &
    \exp(\frac{i}{\hbar}\qty[-E_nt + \Delta_E t ])
    \frac{1}{\sqrt{L_x}} \chi_n\big(y - \zeta(t)\big)
    \\
    & \times
    \exp(
     \frac{i}{\hbar}\bigg[
     p_x x +
     \frac{eEy}{\omega}\cos(\omega t)+
     m_e\dot{\zeta(t)}\big[y-\zeta(t)\big]
     + \int_0^{t}dt'L(\zeta,\dot{\zeta},t') - \Delta_E t  \bigg])
  \end{aligned}
\end{equation}
where we can identify(let $\alpha \rightarrow (n,m)$) the \textit{quasienergies} as
\begin{equation} \label{3.14}
  \varepsilon_{\alpha} \equiv
  \varepsilon_{n} =
  \hbar \omega_0\qty(n + \frac{1}{2}) - \Delta_E \quad \text{where} \quad
  n = 0,1,2, ... \quad \text{for any given } m
\end{equation}
which is only depend on one quantum number ($n$) and \textit{Floquet modes} as
\begin{equation} \label{3.15}
  \begin{aligned}
    \phi_{\alpha}(x,\tilde{y},t) \equiv &
    \frac{1}{\sqrt{L_x}} \chi_{n}\big(\tilde{y} - \zeta(t)\big)
    \exp(
     \frac{i}{\hbar}\bigg[
     p_x x +
     \frac{eE\tilde{y}}{\omega}\cos(\omega t)+
     m_e\dot{\zeta}(t)\big[\tilde{y}-\zeta(t)\big]
     + \Lambda \bigg])
  \end{aligned}
\end{equation}
with
\begin{equation} \label{3.16}
  \zeta(t) = \frac{eE}{m_e(\omega_0^2 - \omega^2)}\sin(\omega t) \quad \text{and} \quad
  \dot{\zeta}(t) = \frac{eE\omega}{m_e(\omega_0^2 - \omega^2)}\cos(\omega t)
\end{equation}
where  \textit{Floquet modes} are time-periodic functions that also create a complete orthonormal set.
\hfill$\blacksquare$
\begin{figure}[ht!]
  \centering
  \includegraphics[scale=0.9]{figures/fig03.pdf}
  \caption{Quasienergies for each Landau levels against magnitude of momentum.}
  \label{fig:3.0}
\end{figure}

\vspace{5mm}
\noindent
Therefore using Floquet theory, the solutions (Floquet states) for the periodic Hamiltonian \eqref{1.5} can be written in position space as
\begin{equation} \label{3.17}
  \psi_{\alpha}(x,\tilde{y},t) =
  \exp(-\frac{i}{\hbar}\varepsilon_{\alpha}t)   \phi_{\alpha} (x,\tilde{y},t)
\end{equation}
where
\begin{equation} \label{3.18}
  \varepsilon_{\alpha} \equiv
  \qty(\frac{eB\hbar }{m_e})\qty(n + \frac{1}{2}) -
  \frac{(eE)^2}{4m_e(\omega_0^2 - \omega^2)}
  \quad \text{where} \quad
  n = 0,1,2, ...
\end{equation}
and
\begin{equation} \label{3.19}
  \begin{aligned}
    \phi_{\alpha}(x,\tilde{y},t) \equiv &
    \frac{1}{\sqrt{L_x}}
    \chi_{n}\qty(\tilde{y} - \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)} ) \\
    & \times
    \exp(
     \frac{i}{\hbar}\qty[
     p_x x +
     \frac{eE\tilde{y}}{\omega}\cos(\omega t)+
     \frac{eE\omega \tilde{y} }{(\omega_0^2 - \omega^2)}\cos(\omega t)
     ])\\
     & \times
     \exp(\frac{i}{\hbar}\qty[
     -\frac{(eE)^2\omega }{2m_e(\omega^2 - \omega_0^2)^2}\sin(2\omega t)
     + \frac{(eE)^2\qty(3\omega_0^2 - \omega^2)}{8m_e\omega(\omega_0^2 - \omega^2)^2} \sin(2\omega t)
  ])
  \end{aligned}
\end{equation}

\noindent
Now we can write this by more simplying and considering spacial dependencies and using previous subtituting done in Eq. \eqref{1.16} and now $\chi$ function depend on both quantum numbers because $y_0$ gives the $p_x$ dependence and we can present as
\begin{equation} \label{3.20}
  \begin{aligned}
    \phi_{\alpha}(x,y,t)  \equiv
    \frac{1}{\sqrt{L_x}}
    \chi_{n}\qty(y - y_0 - \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)} ) &
    \exp( \frac{i p_x }{\hbar}x )
    \exp(
     \frac{i}{\hbar}
     \qty[
     \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]
     \qty(y - y_0 ))\\
     & \times
     \exp(\frac{-i}{\hbar}
     \qty[
     \frac{(eE)^2\qty(\omega_0^2 + \omega^2)}{8\omega m_e(\omega_0^2 - \omega^2)^2}
     ]\sin(2\omega t)
     )
  \end{aligned}
\end{equation}

\noindent
Now we can transform this solution in spacial variable into the momentum space using Fourier trasform over the considering confined space $A=L_xL_y$.
\begin{equation} \label{3.21}
  \begin{aligned}
    \phi_{\alpha}(k_x,k_y,t)  &=
    \int_{-L_y/2}^{L_y/2} dy\; \exp(-ik_y y)
    \qty[
    \frac{1}{\sqrt{L_x}}
    \chi_{n}\qty(y - y_0 - \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)} )
    \exp(
     \frac{i}{\hbar}
     \qty[
     \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]
     y)] \\
     & \times
     \int_{-L_x/2}^{L_x/2} dx\; \exp(-ik_x x)
     \qty[
     \exp( \frac{i p_x }{\hbar}x )
     ] \\
     &
     \times
     \exp(
      \frac{-i}{\hbar}
      \qty[
      \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]y_0)
      \times
     \exp(\frac{-i}{\hbar}
     \qty[
     \frac{(eE)^2\qty(\omega_0^2 + \omega^2)}{8\omega m_e(\omega_0^2 - \omega^2)^2}
     ]\sin(2\omega t)
     )
  \end{aligned}
\end{equation}
Then this can be re-write as follows
\begin{equation} \label{3.22}
    \phi_{\alpha}(k_x,k_y,t)  =
    \exp(
     \frac{-i}{\hbar}
     \qty[
     \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]y_0)
    \exp(\frac{-i}{\hbar}
    \qty[
    \frac{(eE)^2\qty(\omega_0^2 + \omega^2)}{8\omega m_e(\omega_0^2 - \omega^2)^2}
    ]\sin(2\omega t))
    \Theta_{\alpha}(k_y,t)
    \delta_{k_x,\frac{p_x}{\hbar}}
\end{equation}
where we used
\begin{equation} \label{3.23}
  \int_{L_x} dx\;
  \exp( -ik_x x + \frac{i p_x }{\hbar}x ) =
  L_x \delta_{k_x,\frac{p_x}{\hbar}}
\end{equation}
and
\begin{equation} \label{3.24}
  \Theta_{\alpha}(k_y,t) \equiv
  \int_{-L_y/2}^{L_y/2}  dy\; \exp(-ik_y y)
  \qty[
  \sqrt{L_x}
  \chi_{n}\qty(y -y_0 - \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)} )
  \exp(
   \frac{i}{\hbar}
   \qty[
   \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]
   y)]
\end{equation}
and this can be simplied as
\begin{equation} \label{3.25}
  \Theta_{\alpha}(k_y,t) =
  \sqrt{L_x}
  \int_{-L_y/2}^{L_y/2} dy\;
  \chi_{n}\qty(y -y_0 - \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)} )
  \exp(
    -ik_y y +
   \frac{i}{\hbar}
   \qty[
   \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}] y).
\end{equation}
Then by defining
\begin{equation} \label{3.26}
  \mu(t) \equiv \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)} + y_0
\end{equation}
and
\begin{equation} \label{3.27}
  \gamma(t) \equiv
  \frac{eE\omega_0^2\cos(\omega t)}{\hbar\omega(\omega_0^2 - \omega^2)}
\end{equation}
we can re-write this by neglecting time dependencies as
\begin{equation} \label{3.28}
  \Theta_{\alpha}(k_y,t) =
  \sqrt{L_x}
  \int_{-\infty}^{\infty} dy\;
  \chi_{n}\qty(y - \mu )
  \exp(
    -i\qty(k_y - \gamma)
    y).
\end{equation}
We can subtitute following variables
\begin{equation} \label{3.29}
  {k_y}' = k_y -\gamma \quad \text{and} \quad y' = y -\mu
\end{equation}
and for $L_y \rightarrow \infty$ this leads to
\begin{equation} \label{3.30}
  \Theta_{\alpha}({k_y}' ,t) =
  {\sqrt{L_x}} e^{-i {k_y}'\mu}
  \int_{-\infty}^{\infty} dy'\;
  \chi_{n}\qty(y')
  \exp(-i{k_y}' y')
  =
  {\sqrt{L_x}} e^{-i {k_y}'\mu}
  \sqrt{\kappa}
  \int_{-\infty}^{\infty} dy'\;
  \vartheta_n \qty(\kappa y')
  \exp(-i{k_y}' y')
\end{equation}
We know that $\{\chi_{\alpha}\}$ are well-known harmonic eigenfunctions(with Gauss-Hermite functions) as given in the Eq. \eqref{1.47}. However, the equation in \eqref{3.30} represents the Fourier transform of the these Gauss-Hermite functions. Due to the symmetric condition [*Ref:E.Celeghini] the Fourier transform of these functions can be represent as
\begin{equation} \label{3.31}
  \mathcal{FT}[\vartheta_n(\kappa x),x,k] = \frac{i^n}{|\kappa|}\vartheta_n(k/\kappa)
\end{equation}
Therefore
\begin{equation} \label{3.32}
  \Theta_{\alpha}({k_y}' ,t) =
  \sqrt{L_x}e^{-i {k_y}'\mu} \times \frac{i^n}{\sqrt{\kappa}} \vartheta_n \qty(\frac{{k_y}'}{\kappa}) =
    \sqrt{L_x}e^{-i {k_y}'\mu}
    \tilde{\chi}_{n}\qty({k_y}')
\end{equation}
where
\begin{equation} \label{3.33}
  \tilde{\chi}_{n}\qty(k) =
  \frac{i^n}{\sqrt{2^{n} n! \sqrt{\pi}}}
  \qty(\frac{1}{\kappa})^{1/2}
  e^{-\frac{k^2}{2 \kappa^2}}
  \mathcal{H}_{\alpha} \qty(\frac{k}{\kappa}).
\end{equation}
Using Eq. \eqref{3.32} and Eq. \eqref{3.22} we can derive that
\begin{equation} \label{3.34}
  \begin{aligned}
    \phi_{\alpha}(k_y,t)  =
    \exp(
      \frac{-i}{\hbar}
      \qty[
      \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]y_0
    )&
    \exp(
      \frac{-i}{\hbar}
      \qty[
      \frac{(eE)^2\qty(\omega_0^2 + \omega^2)}{8\omega m_e(\omega_0^2 - \omega^2)^2}
      ]\sin(2\omega t)
    )
    \\
    & \times
    {\sqrt{L_x}} e^{-i \qty(k_y -\gamma)\mu}
    \tilde{\chi}_{n}\qty(k_y -\gamma)
  \end{aligned}
\end{equation}
where we included the $k_x$ dependence into $\alpha$ quantum number using $m$ value and this can be re-write subtituting $\mu$ and $\gamma$ values as follows
\begin{equation} \label{3.35}
  \begin{aligned}
    \phi_{\alpha}(k_y,t) & =
    {\sqrt{L_x}}
    \exp(
      \frac{-i}{\hbar}
      \qty[
      \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]y_0
    )
    \exp(
      \frac{-i}{\hbar}
      \qty[
      \frac{(eE)^2\qty(\omega_0^2 + \omega^2)}{8\omega m_e(\omega_0^2 - \omega^2)^2}
      ]\sin(2\omega t)
    ) \\
    & \times
    \exp(
      -ik_y  \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)}
    )
    \exp(
      \frac{i}{\hbar}
      \qty[ \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]
      \frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)}
    )
    \exp(
      -i k_y y_0
    ) \\
    & \times
    \exp(
      i \frac{1}{\hbar}
      \qty[ \frac{eE\omega_0^2\cos(\omega t)}{\omega(\omega_0^2 - \omega^2)}]
      y_0
    )
    \tilde{\chi}_{n}\qty(k_y -\gamma)
  \end{aligned}
\end{equation}
and
\begin{equation} \label{3.36}
  \begin{aligned}
    \phi_{\alpha}(k_y,t) & =
    {\sqrt{L_x}}
    \exp(
      \frac{i}{\hbar}
      \qty[
      \frac{(eE)^2\qty(3\omega_0^2 - \omega^2)}{8\omega m_e(\omega_0^2 - \omega^2)^2}
      ]\sin(2\omega t)
    ) \\
    & \times
    \exp(
      -ik_y  \qty[\frac{eE\sin(\omega t)}{m_e(\omega_0^2 - \omega^2)} + y_0]
    )
    \tilde{\chi}_{n}\qty(k_y -\gamma).
  \end{aligned}
\end{equation}
For notation convinient we can introduce few constant as follows
\begin{equation} \label{3.37}
  b \equiv
  \frac{(eE)^2\qty(3\omega_0^2 - \omega^2)}{8\hbar \omega m_e(\omega_0^2 - \omega^2)^2}
\end{equation}
and
\begin{equation} \label{3.38}
  d \equiv
  \frac{eE}{m_e(\omega_0^2 - \omega^2)}
\end{equation}
with
\begin{equation} \label{3.39}
  g \equiv
  \frac{eE\omega_0^2}{\hbar\omega(\omega_0^2 - \omega^2)}.
\end{equation}
Therefore we can write Eq. \eqref{3.36} as
\begin{equation} \label{3.40}
  \begin{aligned}
    \phi_{\alpha}(k_y,t) & =
    {\sqrt{L_x}}
    e^{
      ib\sin(2\omega t)
    }
    e^{
      -ik_y  \qty[d\sin(\omega t) + y_0]
    }
    \tilde{\chi}_{n}\qty(k_y -g\cos(\omega t)).
  \end{aligned}
\end{equation}