\documentclass[11pt,oneside]{article}	%use"amsart"insteadof"article"forAMSLaTeXformat
\usepackage{geometry}		%Seegeometry.pdftolearnthelayoutoptions.Therearelots.
\geometry{letterpaper}		%...ora4paperora5paperor...
%\geometry{landscape}		%Activateforforrotatedpagegeometry
%\usepackage[parfill]{parskip}		%Activatetobeginparagraphswithanemptylineratherthananindent
\usepackage{graphicx}				%Usepdf,png,jpg,orepsßwithpdflatex;useepsinDVImode
								%TeXwillautomaticallyconverteps-->pdfinpdflatex
\usepackage{amssymb}
\usepackage[colorlinks]{hyperref}

\usepackage{framed}
\usepackage{amsthm}
\newtheorem{remark}{Remark}
\newtheorem{definition}{Definition}

%----macros begin---------------------------------------------------------------
\usepackage{color}
\usepackage{amsthm}

\def\conv{\mbox{\textrm{conv}\,}}
\def\aff{\mbox{\textrm{aff}\,}}
\def\E{\mathbb{E}}
\def\R{\mathbb{R}}
\def\Z{\mathbb{Z}}
\def\tex{\TeX}
\def\latex{\LaTeX}
\def\v#1{{\bf #1}}
\def\p#1{{\bf #1}}
\def\T#1{{\bf #1}}

\def\vet#1{{\left(\begin{array}{cccccccccccccccccccc}#1\end{array}\right)}}
\def\mat#1{{\left(\begin{array}{cccccccccccccccccccc}#1\end{array}\right)}}

\def\lin{\mbox{\rm lin}\,}
\def\aff{\mbox{\rm aff}\,}
\def\pos{\mbox{\rm pos}\,}
\def\cone{\mbox{\rm cone}\,}
\def\conv{\mbox{\rm conv}\,}
\newcommand{\homog}[0]{\mbox{\rm homog}\,}
\newcommand{\relint}[0]{\mbox{\rm relint}\,}

%----macros end-----------------------------------------------------------------

\title{Imaging Morphology with LAR
\footnote{This document is part of the \emph{Linear Algebraic Representation with CoChains} (LAR-CC) framework~\cite{cclar-proj:2013:00}. \today}
}
\author{Alberto Paoluzzi}
%\date{}							%Activatetodisplayagivendateornodate

\begin{document}
\maketitle
\nonstopmode

\begin{abstract}
In this module we aim to implement the four operators of mathematical morphology, i.e.~the \emph{dilation}, \emph{erosion}, \emph{opening} and \emph{closing} operators, by the way of matrix operations representing the linear operators---\emph{boundary} and \emph{coboundary}---over LAR.
According to the multidimensional character of LAR, our implementation is dimension-independent.
In few words, it works as follows: (a)  the input is (the coordinate representation of) a $d$-chain $\gamma$; (b) compute its boundary $\partial_d(\gamma)$; (c) extract the maximal $(d-2)$-chain $\epsilon \subset \partial_d(\gamma)$; (d) consider the $(d-1)$-chain returned from its coboundary $\delta_{d-2}(\epsilon)$; (e) compute the $d$-chain $\eta := \delta_{d-1}(\delta_{d-2}(\epsilon)) \subset C_d$ \emph{without} performing the  $\mbox{mod\ 2}$ final transformation on the resulting coordinate vector, that would provide a zero result, according to the standard algebraic constraint $\delta\circ\delta=0$. It is easy to show that $\eta \equiv (\oplus \gamma) - (\ominus \gamma)$ provides the \emph{morphological gradient} operator. The four standard morphological operators are therefore  consequently computable.
\end{abstract}

\tableofcontents

\section{Test image generation}

Various methods for the input or the generation of a test image  are developed in the subsections of this section. The aim is to prepare a set of controlled test beds, used to check both the implementation and the working properties of our topological implementation of morphological operators.

\subsection{Random binary multidimensional image}

A multidimensional binary image is generated here by using a random approach, both for the bulk structure and the small artefacts of the image.

%-------------------------------------------------------------------------------
@d Generation of random image
@{
def randomImage(shape, structure, noiseFraction=0.0):
	""" Generation of random image of given shape and structure.
		Return scipy.ndarray(shape)
	"""
	print "noiseFraction =",noiseFraction
	@< Generation of bulk array structure @>
	@< Generation of random artifacts @>
	return image_array
@}
%-------------------------------------------------------------------------------

\paragraph{Generation of the gross image}
First we generate a 2D grid of squares by Cartesian product, and produce the bulk of the random image then used to test our approach to morphological operators via topological ones.

%import scipy
%scipy.misc.imsave('outfile.jpg', image_array)
%
%scipy.ndimage.imread(fname, flatten=False, mode=None)[source]
%
%rand(10, 10)
%

%-------------------------------------------------------------------------------
@d Generation of bulk array structure
@{ranges = [shape[k]/structure[k] for k in range(len(shape))]
random_array = randint(0, 255, size=structure)
image_array = numpy.zeros(shape)
for index in scipy.array(CART(AA(range)(shape))):
	block = index/ranges
	if random_array[tuple(block)] < 127:
		image_array[tuple(index)] = 0
	else:
		image_array[tuple(index)] = 255
@}
%-------------------------------------------------------------------------------

\paragraph{Generation of random artefacts upon the image}

Then random noise is added to the previously generated image, in order to produce artifacts at the pixel scale.

%-------------------------------------------------------------------------------
@d Generation of random artifacts
@{noiseQuantity = PROD(list(shape))*noiseFraction
k = 0
while k < noiseQuantity:
	index = tuple(AA(randint)(list(shape)))
	if image_array[index] == 0: image_array[index] = 255
	else: image_array[index] = 0
	k += 1
if len(shape)==3:
	for k in range(shape[0]):
		scipy.misc.imsave('tmp/outfile'+str(k).zfill(3)+'.png', image_array[k])
else:
	scipy.misc.imsave('tmp/outfile'+str(k).zfill(3)+'.png', image_array)
@}
%-------------------------------------------------------------------------------

\section{Selection of an image segment}

In this section we implement several methods for image segmentation and segment selection.

\subsection{Selection of a test chain}

The first and simplest method is the selection of the portion of a binary image contained within a masking window.
Here we select the (white) sub-image contained in a given window, and compute the coordinate representation of the (chain) sub-image.

\paragraph{Mask definition}

A \emph{window} within a $d$-image is defined by $2\times d$ integer numbers (2 multi-indices), corresponding to the window  \texttt{minPoint} (minimum indices) and to the window \texttt{maxPoint} (maximum indices). A list of multi-index tuples, contained in the \texttt{window} variable, is generated by the function \texttt{setMaskWindow} below.

%-------------------------------------------------------------------------------
@d Generation of a masking window
@{def setMaskWindow(window,image_array):
	minPoint, maxPoint = window
	imageShape = list(image_array.shape)
	@< Generation of multi-index window @>
	@< Window-to-chain mapping @>
	@< Change chain color to grey @>
	return segmentChain
@}
%-------------------------------------------------------------------------------

The set of tuples of indices contained in a (multidimensional) window is given below.

%-------------------------------------------------------------------------------
@d Generation of multi-index window
@{indexRanges = zip(minPoint,maxPoint)
tuples = CART([range(min,max) for min,max in indexRanges])
@}
%-------------------------------------------------------------------------------

\subsection{Mapping of integer tuples to integers}

In order to produce the coordinate representation of a chain in a multidimensional image (or $d$-image) we need: (a) to choose a basis of image elements, i.e.~of $d$-cells, and in particular to fix an ordering of them; (b) to map the multidimensional index, selecting a single $d$-cell of the image, to a single integer mapping the cell to its linear position within the chosen basis ordering.

\paragraph{Grid of hyper-cubes of unit size}
Let $S_i=(0,1,...,n_i-1)$ be ordered integer sets with $n_i$ elements, and
\[
S= S_0 \times S_1 \times \cdots \times S_{d-1}
\]
the set of indices of elements of a $d$-image.

\begin{definition}[$d$-image shape]
The \emph{shape} of a $d$-image with $n_0\times n_1 \times\cdots\times n_{d-1}$ elements (here called \emph{voxels}) is the ordered set $(n_0, n_1, \ldots, n_{d-1})$.
\end{definition}

\paragraph{$d$-dimensional row-major order}

Given a $d$-image with shape $S=(n_0,n_1,...,n_{d-1})$ and number of elements $n=\prod n_i$,
the mapping
\[
S_0 \times S_1 \times \cdots \times S_{d-1} \to \{ 0, 1, \ldots, n-1\}
\]
 is a {linear combination} with integer {weights}  $(w_0,w_1,...,w_{d-2},1)$, such that:
\[
(i_0,i_1,...,i_{d-1}) \mapsto i_0 w_0 +i_1 w_1 +\cdots +i_{d-1} w_{d-1},
\]
where
\[
w_k = n_{k+1}  n_{k+2} \cdots  n_{d-1}, \qquad 0\leq k\leq d-2.
\]

\paragraph{Implementation}
A functional implementation of the \emph{Tuples to integers mapping} is given by the second-order  \texttt{mapTupleToInt} function, that  accepts in a first application the \texttt{shape} of the image (to compute the tuple space of indices of $d$-cells), and then takes a single tuple in the second application. Of course, the function  returns the cell address in the linear address space associated to the given \texttt{shape}.

%-------------------------------------------------------------------------------
@d Tuples to integers mapping
@{def mapTupleToInt(shape):
	d = len(shape)
	weights = [PROD(shape[(k+1):]) for k in range(d-1)]+[1]

	def mapTupleToInt0(tuple):
		return INNERPROD([tuple,weights])
	return mapTupleToInt0
@}
%-------------------------------------------------------------------------------

\paragraph{From tuples multi-indices to chain coordinates}

The set of address \texttt{tuples} of $d-cells$ ($d$-dimensional image elements) within the \emph{mask} is here mapped to the corresponding set of (single) integers associated to the low-level image elements (pixels or voxels, depending on the image dimension and shape), denoted \texttt{windowChain}. Such total chain of the mask \texttt{window} is then filtered to contain the only coordinates of \emph{white} image elements within the window, and returned as the set of integer cell indices \texttt{segmentChain}.

%-------------------------------------------------------------------------------
@d Window-to-chain mapping
@{imageCochain = image_array.reshape(PROD(imageShape))
mapping = mapTupleToInt(imageShape)
windowChain = [mapping(tuple) for tuple in tuples]
segmentChain = [cell for cell in windowChain if imageCochain[cell]==255]
@}
%-------------------------------------------------------------------------------

\subsection{Show segment chain on binary image}

Now we need to show visually the selected \texttt{segmentChain}, by change the color of its cells from white (255) to middle grey (127). Just remember that \texttt{imageCochain} is the linear representation of the image, with number of cells equal to \texttt{PROD(imageShape)}. Then the modified image is restored within \texttt{image\_array}, and is finally exported to a \texttt{.png} image file.

%-------------------------------------------------------------------------------
@d Change chain color to grey
@{for cell in segmentChain: imageCochain[cell] = 127
image_array = imageCochain.reshape(imageShape)
#for k in range(shape[0]):
#	scipy.misc.imsave('tmp/outfile'+str(k).zfill(3)+'.png', image_array[k])
@}
%-------------------------------------------------------------------------------

\section{Construction of (co)boundary operators}

A $d$-image is a \emph{cellular $d$-complex} where cells are $k$-cuboids ($0\leq k\leq d$), i.e.~Cartesian products of a number $k$ of 1D intervals, embedded in $d$-dimensional Euclidean space.

\subsection{Reading and writing LAR of image from disk}
Since the construction of the \emph{chain complex} supported by a cellular complex with $O(n^d)$ $d$-cells---and $n=O(10^3)$---may be really time-consuming, it is unquestionably useful to store on disk, once and for all, the topological model of a multidimensional image, i.e.~the LAR of its chain complex,  and just restore it (or the needed part of it --- TODO) when necessary, since \emph{it only depends on the \texttt{shape} of a $d$-image}, i.e.~from the array arrangement of its $d$-cells (``hypervoxels'').

%-------------------------------------------------------------------------------
@d Save and restore data object from file
@{def dump(object,filename):
	with open(filename, 'wb') as f:
	    pickle.dump(object, f)

def load(filename):
	with open(filename, 'rb') as f:
	    object = pickle.load(f)
	return object
@}
%-------------------------------------------------------------------------------

%-------------------------------------------------------------------------------
@d Save or restore the chain complex of multidimensional image
@{def imageChainComplex (shape):
	tokens = str(shape)[1:-1].split(',')
	tokens = [token.strip() for token in tokens]
	filename = "tmp/larimage-" + "-".join(tokens) + ".pickle"

	if os.path.isfile(filename):
		shape, skeletons, operators = loadImageLAR(filename)
	else:
		skeletons = gridSkeletons(list(shape))
		operators = boundaryOps(skeletons)
		imageLAR = (shape, skeletons, operators)
		dump(imageLAR,filename)
		print "filename =",filename
	return shape, skeletons, operators

def loadImageLAR(filename):
	object = load(filename)
	shape, skeletons, operators = object
	return shape, skeletons, operators
@}
%-------------------------------------------------------------------------------

\subsection{LAR chain complex construction}

In our first multidimensional implementation of morphological operators through algebraic topology of the image seen as a cellular complex, we compute the whole sequence of characteristic matrices $M_k$ ($0\leq k \leq d$) in \texttt{BRC} form, and the whole sequence of matrices $[\partial_k]$ ($0\leq k \leq d$) in \texttt{CSR} form.

\begin{figure}[htbp] %  figure placement: here, top, bottom, or page
   \centering
   \includegraphics[width=0.7\linewidth]{images/larcomplex}
   \caption{The LAR definition of  a chain complex: a sequence of characteristic matrices \emph{and} a sequence of boundary operators.}
   \label{fig:example}
\end{figure}

\paragraph{Array of characteristic matrices}

A direct construction of cuboidal complexes is offered, within the \texttt{larcc} package, by the \texttt{largrid.larCuboids} function.

%-------------------------------------------------------------------------------
@d Characteristic matrices of multidimensional image
@{def larImage(shape):
	""" Compute vertices and skeletons of an image of given shape """
	imageVerts = larImageVerts(shape)
	skeletons = gridSkeletons(list(shape))
	return imageVerts, skeletons
@}
%-------------------------------------------------------------------------------

\paragraph{Example}
Consider a (very!) small 3D image of \texttt{shape=(2,2,2)}. The data structures returned by the \texttt{larImage} function are shown below, where \texttt{imageVerts} gives the integer coordinates of vertices of the 3D (image) complex, and \texttt{skeletons} is the list of characteristic matrices $M_k$ ($0\leq k\leq d$) in \texttt{BRC} form.

%-------------------------------------------------------------------------------
@D Example of characteristic matrices (and vertices) of multidimensional image
@{imageVerts, skeletons = larImage((2,2,2))

print imageVerts,
>>> [[0,0,0],[0,0,1],[0,0,2],[0,1,0],[0,1,1],[0,1,2],[0,2,0],[0,2,1],[0,2,2],
[1,0,0],[1,0,1],[1,0,2],[1,1,0],[1,1,1],[1,1,2],[1,2,0],[1,2,1],[1,2,2],[2,0,
0],[2,0,1],[2,0,2],[2,1,0],[2,1,1],[2,1,2],[2,2,0],[2,2,1],[2,2,2]]

print skeletons[1:],
>>> [
[[0,1],[1,2],[3,4],[4,5],[6,7],[7,8],[9,10],[10,11],[12,13],[13,14],[15,
16],[16,17],[18,19],[19,20],[21,22],[22,23],[24,25],[25,26],[0,3],[1,4],[2,
5],[3,6],[4,7],[5,8],[9,12],[10,13],[11,14],[12,15],[13,16],[14,17],[18,21],
[19,22],[20,23],[21,24],[22,25],[23,26],[0,9],[1,10],[2,11],[3,12],[4,13],[5,
14],[6,15],[7,16],[8,17],[9,18],[10,19],[11,20],[12,21],[13,22],[14,23],[15,
24],[16,25],[17,26]],
[[0,1,3,4],[1,2,4,5],[3,4,6,7],[4,5,7,8],[9,10,12,13],[10,11,13,14],[12,13,15,
16],[13,14,16,17],[18,19,21,22],[19,20,22,23],[21,22,24,25],[22,23,25,26],[0,
1,9,10],[1,2,10,11],[3,4,12,13],[4,5,13,14],[6,7,15,16],[7,8,16,17],[9,10,18,
19],[10,11,19,20],[12,13,21,22],[13,14,22,23],[15,16,24,25],[16,17,25,26],[0,
3,9,12],[1,4,10,13],[2,5,11,14],[3,6,12,15],[4,7,13,16],[5,8,14,17],[9,12,18,
21],[10,13,19,22],[11,14,20,23],[12,15,21,24],[13,16,22,25],[14,17,23,26]],
[[0,1,3,4,9,10,12,13],[1,2,4,5,10,11,13,14],[3,4,6,7,12,13,15,16],[4,5,7,8,13,
14,16,17],[9,10,12,13,18,19,21,22],[10,11,13,14,19,20,22,23],[12,13,15,16,21,
22,24,25],[13,14,16,17,22,23,25,26]]
]
@}
%-------------------------------------------------------------------------------

\paragraph{Array of matrices of boundary operators}

The function \texttt{boundaryOps} takes the array of \texttt{BRC} reprs of characteristic matrices, and returns the array of \texttt{CSR} matrix reprs of boundary operators $\partial_k$ ($1\leq k\leq d$).

%-------------------------------------------------------------------------------
@d CSR matrices of boundary operators
@{def boundaryOps(skeletons):
	""" CSR matrices of boundary operators from list of skeletons """
	return [larBoundary(skeletons[k+1],faces)
		for k,faces in enumerate(skeletons[:-1])]
@}
%-------------------------------------------------------------------------------

\paragraph{Boundary chain of a $k$-chain of a $d$-image}

%-------------------------------------------------------------------------------
@d Boundary of image chain computation
@{def imageChainBoundary(shape, operators):
	imageVerts, skeletons = larImage(shape)
	# operators = boundaryOps(skeletons)
	cellNumber = PROD(list(shape))

	def imageChainBoundary0(k):
		csrBoundaryMat = operators[-1]
		facets = skeletons[k-1]

		def imageChainBoundary1(chain):
			@< Boundary*chain product and interpretation @>
			boundaryChainModel = imageVerts, [facets[h] for h in boundaryChain]
			return boundaryChainModel,boundaryChain

		return imageChainBoundary1
	return imageChainBoundary0
@}
%-------------------------------------------------------------------------------

\paragraph{Low-level SpMSpV matrix-chain product}

A low-level implementation of the product of boundary matrix times the coordinate representation of the image chain under consideration is given below. It was enveloped and protected in this macro because it is strongly dependent on the CSR structures and tools provided by the sparse matrix module of the \texttt{scipy} library.

%-------------------------------------------------------------------------------
@d Boundary*chain product and interpretation
@{csrChain = scipy.sparse.csr_matrix((cellNumber,1))
for h in chain: csrChain[h,0] = 1
csrBoundaryChain = matrixProduct(csrBoundaryMat, csrChain)
for h,value in enumerate(csrBoundaryChain.data):
	if MOD([value,2]) == 0: csrBoundaryChain.data[h] = 0
cooBoundaryChain = csrBoundaryChain.tocoo()
boundaryChain = [cooBoundaryChain.row[h]
	for h,val in enumerate(cooBoundaryChain.data) if val == 1]
@}
%-------------------------------------------------------------------------------

\subsection{Visualisation of an image chain and its boundary}

\paragraph{$d$-Chain visualisation}

The \texttt{visImageChain} function given by the macro \emph{Visualisation of an image chain} below.

%-------------------------------------------------------------------------------
@d Pyplasm visualisation of an image chain
@{def visImageChain (shape,chain, imageVerts, skeletons):
	# imageVerts, skeletons = larImage(shape)
	chainLAR = [cell for k,cell in enumerate(skeletons[-1]) if k in chain]
	return imageVerts,chainLAR
@}
%-------------------------------------------------------------------------------

\begin{figure}[htbp] %  figure placement: here, top, bottom, or page
   \centering
   \includegraphics[height=0.3\linewidth,width=0.16\linewidth]{images/morph-chain2D1}
   \includegraphics[height=0.3\linewidth,width=0.16\linewidth]{images/morph-chain1D1}
   \includegraphics[height=0.3\linewidth,width=0.16\linewidth]{images/morph-chain2D2}
   \includegraphics[height=0.3\linewidth,width=0.16\linewidth]{images/morph-chain1D2}
   \includegraphics[height=0.3\linewidth,width=0.16\linewidth]{images/morph-chain2D3}
   \includegraphics[height=0.3\linewidth,width=0.16\linewidth]{images/morph-chain1D3}
   \caption{example caption}
   \label{fig:morph}
\end{figure}

\section{The bulk of morphological imaging}

The main part of the implementation of our algebraic construction of morphological operators on $d$-images is given in this section. First we introduce two generalized incidence operators, respectively named \emph{$\mathcal{U}_d$}, which stands for \emph{Upper}, and \emph{$\mathcal{L}_d$}, which stands for \emph{Lower}. Then we implement the standard morphological operators of Dilation (\emph{$\mathcal{D}_d$}), Erosion (\emph{$\mathcal{E}_d$}), Opening (\emph{$\mathcal{O}_d$}), and Closing (\emph{$\mathcal{C}_d$}).

\subsection{``Down'' and ``Up'' operators on Chains}

The Up and Down incidence operators are defined as mapping $d$-cells to $(d+1)$- and $(d+1)$-cells that, respectively, share vertices with them:
\[
\mathcal{U}_d : C_d \to C_{d+1}, \qquad \mathcal{D}_d : C_d \to C_{d-1}.
\]

Remembering that the characteristic matrix $M_d$ corresponds to the mapping from vertices to $d$-cells, we have
\[
[\mathcal{U}_d] = M_{d+1}\,M_d^t, \qquad [\mathcal{D}_d] = M_{d-1}\,M_d^t.
\]

\paragraph{Down and Up operators} Just consider that the characteristic matrices $M_{d-1},M_{d},M_{d+1}$,  stored in position $d-1$, $d$, and $d+1$ of the \texttt{skeletons} array, respectively provide the subsets of vertices of proper dimensions incising on each cell. They are used to compute the topological incidence operators, according to the paper~\cite{Dicarlo:2014:TNL:2543138.2543294}.

%-------------------------------------------------------------------------------
@d Down and Up operators
@{def larDown(skeletons,d):
	""" Down operator, to multiply a d-chain and return the incident (d-1)-chain """
	csrMd = csrCreate(skeletons[d])
	csrMinus = csrCreate(skeletons[d-1])
	csrDown = matrixProduct(csrMinus,csrTranspose(csrMd))
	return csrDown

def larUp(skeletons,d):
	""" Up operator, to multiply a d-chain and return the incident (d+1)-chain """
	csrMd = csrCreate(skeletons[d])
	csrPlus = csrCreate(skeletons[d+1])
	csrUp = matrixProduct(csrPlus,csrTranspose(csrMd))
	return csrUp
@}
%-------------------------------------------------------------------------------

\paragraph{The $\mathcal{UUD}$ operator}
A generic $\mathcal{UUD}: C_{d-1} \to C_{d}$ operator is given to be associated with the boundary $\partial_2$, in order to return the $\mathcal{ERO}\cup\mathcal{DIL}$ (TODO:correct) of a $d$-chain.

%-------------------------------------------------------------------------------
@d UUD operator: maps Down-UP-UP its input chains
@{def UUD(skeletons,d):
	""" Compute the morphological operator UUP.
		Return a CSR matrix to be applied to the coordinate representation of a chain
	"""
	D = larDown(skeletons,d)
	U1 = larUp(skeletons,d-1)
	U2 = larUp(skeletons,d)
	UUDout = matrixProduct(U2,matrixProduct(U1,D))
	return D,U1,U2,UUDout
@}
%-------------------------------------------------------------------------------

A pair of similar function \texttt{testAlgebraicMorphology} and \texttt{testAlgebraicMorphologyStepByStep} are given here, both accepting the same (redundant) inputs, in order to test our algebraic approach to mathematical morphology. In particular:

%\item
%\texttt{solid} is the LAR model (i.e.~the pair (\texttt{V},\texttt{CV})) of the original image;
%\item
%\texttt{b_rep} is the LAR model of its boundary (\texttt{V},\texttt{BV});
%\item
%\texttt{chain} is the $d$-chain of the image, given as the list of indices of $(d-1)$-cells within the boundary's $(d-1)$-chain;
%\item
%\texttt{imageVerts}
%\item
%\texttt{skeletons}

%-------------------------------------------------------------------------------
@d Testing the algebraic morphology
@{def testAlgebraicMorphology (solid, b_rep, chain, imageVerts, skeletons):
	d = len(skeletons)-1
	D,U1,U2,csrMorphOp = UUD(skeletons,d-1)
	outputChain = chainTransform(skeletons,1,chain,csrMorphOp,d)
	chainLAR = [cell for k,cell in enumerate(skeletons[d]) if k in outputChain]
	model2 = (imageVerts,chainLAR)
	return imageVerts,chainLAR
@}
%-------------------------------------------------------------------------------

%-------------------------------------------------------------------------------
@d Testing the algebraic morphology method step-by-step
@{def testAlgebraicMorphologyStepByStep (solid, b_rep, chain, imageVerts, skeletons):
	d = len(skeletons)-1
	D,U1,U2,csrMorphOp = UUD(skeletons,d-1)

	VIEW(EXPLODE(1.5,1.5,1)(MKPOLS(solid)))
	VIEW(COLOR(MAGENTA)(STRUCT(MKPOLS(b_rep))))

	outputChain = chainTransform(skeletons,d-1,chain,D,d-1)
	chainLAR = [cell for k,cell in enumerate(skeletons[0]) if k in outputChain]
	model0 = (imageVerts,chainLAR)
	VIEW(COLOR(RED)(STRUCT(MKPOLS(model0))))

	M2 = csrCreate(skeletons[d])

	outputChain = chainTransform(skeletons,0,outputChain,M2,d)
	chainLAR = [cell for k,cell in enumerate(skeletons[d]) if k in outputChain]
	model2 = (imageVerts,chainLAR)
	VIEW(COLOR(YELLOW)(STRUCT(MKPOLS(model2))))

	return imageVerts,chainLAR
@}
%-------------------------------------------------------------------------------

\paragraph{Boundary chain mapping}
The function \texttt{chainTransform} takes as input a \texttt{chain} of dimension \texttt{d} (actually the boundary of a chain of dimension $d+1$), the \texttt{skeletons} array of \texttt{BRC} representations of matrices $M_k$ ($0\leq k\leq d+1$), and the number $n$ that $(d+1)$-cells resulting by multiplication with the operator's matrix \texttt{csrMatrix} must ahare with they incident $d$-cells in order to be included in the output of the considered algebraic morphology operator.

%-------------------------------------------------------------------------------
@d csrMatrix*chain product and interpretation
@{
def chainTransform(skeletons,d,chain,csrMatrix,n):
	# n: number of vertices shared in the incidence relation
	cellNumber = len(skeletons[d])
	csrChain = scipy.sparse.csr_matrix((cellNumber,1))
	for h in chain: csrChain[h,0] = 1
	cooOutChain = matrixProduct(csrMatrix, csrChain).tocoo()
	outChain = [cooOutChain.row[h]
		for h,val in enumerate(cooOutChain.data) if int(val) >= n]
	return outChain
@}
%-------------------------------------------------------------------------------

\paragraph{Example}

%-------------------------------------------------------------------------------
@d Algebraic dilation minus erosion
@{D1 = larDown(skeletons,1)
U0 = larUp(skeletons,0)
U1 = larUp(skeletons,1)
UUD1 = UUD(skeletons,1)
@}
%-------------------------------------------------------------------------------

\subsection{Maximal ($d-2$)-chain extraction}

%-------------------------------------------------------------------------------
@d Extract the maximal ($d-2$)-chain from a ($d-1$)-chain
@{

@}
%-------------------------------------------------------------------------------

\subsection{($d-1$)-Star of a ($d-2$)-chain computation}

%-------------------------------------------------------------------------------
@d Compute the ($d-1$)-star of a ($d-2$)-chain
@{

@}
%-------------------------------------------------------------------------------

\subsection{$d$-Star of a $(d-1)$-chain computation}

%-------------------------------------------------------------------------------
@d Compute the d-star of a ($d-1$)-chain
@{

@}
%-------------------------------------------------------------------------------

\subsection{Dilation and erosion computation}

\subsection{Opening and closing computation}

\section{Exporting the \texttt{morph} module}

\paragraph{Exporting the morph module}
%-------------------------------------------------------------------------------
@O larlib/larlib/morph.py
@{""" LAR implementation of morphological operators on multidimensional images."""
@< Initial import of modules @>
@< Generation of random image @>
@< Save and restore data object from file @>
@< Save or restore the chain complex of multidimensional image @>
@< Tuples to integers mapping @>
@< Generation of a masking window @>
@< Characteristic matrices of multidimensional image @>
@< CSR matrices of boundary operators @>
@< csrMatrix*chain product and interpretation @>
@< Pyplasm visualisation of an image chain @>
@< Boundary of image chain computation @>
@< Down and Up operators @>
@< UUD operator: maps Down-UP-UP its input chains @>
@< Testing the algebraic morphology method step-by-step @>
@< Testing the algebraic morphology @>
@}
%-------------------------------------------------------------------------------

The set of importing commends needed by test files in this module is given in the macro below.

%-------------------------------------------------------------------------------
@d Initial import of modules
@{""" Initial import of modules """
from larlib import *
import scipy.misc, numpy, pickle
from numpy.random import randint
@}
%-------------------------------------------------------------------------------

\section{Morphological operations examples}

\subsection{2D image masking and boundary computation}

\paragraph{Test example}

The \texttt{larcc.morph} API is used here to generate a random black and white image, with an \emph{image segment} selected and extracted by masking, then colored in middle grey, and exported to an image file.
The \texttt{shape} and \texttt{structure} variables must contain two tuples, of equal \texttt{len}, of powers of 2, used toy define the size of random blocks in the generated image. The \texttt{window} variable is used to define the portion of the image where the \emph{segmentChain} of white pixels (or voxels, for 3-images) is computed.

%------------------------------------------------------------------
@o test/py/morph/test01.py
@{
@< Initial import of modules @>

shape = 64,64
structure = 8,8
assert len(shape) == len(structure)
imageVerts = larImageVerts(shape)
_, skeletons, operators = imageChainComplex (shape)
image_array = randomImage(shape, structure, 0.05)
minPoint, maxPoint = (0,0), (64,64)
window = minPoint, maxPoint
segmentChain = setMaskWindow(window,image_array)

solid = visImageChain (shape,segmentChain, imageVerts, skeletons)
b_rep,boundaryChain = imageChainBoundary(shape, operators)(2)(segmentChain)

stepwiseTest = True
if stepwiseTest:
	model = testAlgebraicMorphologyStepByStep(solid, b_rep,
				boundaryChain, imageVerts, skeletons)
else:
	model = testAlgebraicMorphology(solid, b_rep,
				boundaryChain, imageVerts, skeletons)
VIEW(STRUCT(MKPOLS(model)))

V = model[0]
M = AA(tuple)(model[1])
S = AA(tuple)(solid[1])
B = AA(tuple)(b_rep[1])
D = list(set(S).union(M))
E = list(set(S).difference(M))
M,S,B,D,E = (AA(AA(list)))([M,S,B,D,E])
M2 = STRUCT(MKPOLS((V,M)))
S2 = STRUCT(MKPOLS((V,S)))

S2 = COLOR(CYAN)(STRUCT(MKPOLS((V,S))))
B1 = COLOR(MAGENTA)(STRUCT(MKPOLS((V,B))))
D2 = COLOR(YELLOW)(STRUCT(MKPOLS((V,D))))
E2 = COLOR(WHITE)(STRUCT(MKPOLS((V,E))))
VIEW(STRUCT([D2,S2,E2,B1]))

VIEW(STRUCT([D2,S2,B1]))
VIEW(STRUCT([S2,E2,B1]))

@}
%------------------------------------------------------------------

%A0 = matrix([
%[0,2,0],
%[4,0,0],
%[0,0,2],
%])
%B0 = matrix([
%[0,1,0],
%[1,0,0],
%[0,0,1],
%])
%A1 = matrix([
%[0,10,0],
%[0,11,0],
%[5,0,12],
%])
%B1 = matrix([
%[0,1,0],
%[0,1,0],
%[1,0,1],
%])

%===============================================================================
\appendix
\section{Utilities}

\subsection{Importing a generic module}
First we define a parametric macro to allow the importing of \texttt{larcc} modules from the project repository \texttt{lib/py/}. When the user needs to import some project's module, she may call this macro as done in Section~\ref{sec:lar2psm}.
%------------------------------------------------------------------
@d Import the module
@{import @1
from @1 import *
@}
%------------------------------------------------------------------

\paragraph{Importing a module} A function used to import a generic \texttt{lacccc} module within the current environment is also useful.
%------------------------------------------------------------------
@d Function to import a generic module
@{def importModule(moduleName):
	@< Import the module @(moduleName@) @>
@| importModule @}
%------------------------------------------------------------------

\bibliographystyle{amsalpha}
\bibliography{morph}

\end{document}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{multicol}
\usepackage[formats]{listings}
\lstloadaspects{formats}
\usepackage{verbatim}
\usepackage{color}
\usepackage{geometry}
\usepackage{float}
\usepackage{amsmath}
\usepackage{caption}
\usepackage{pdflscape}
\usepackage{caption}
\usepackage[font={color=black},figurename=Fig.,labelfont={it}]{caption}
\usepackage{hyperref}
\setlength{\belowcaptionskip}{-10pt}
\setlength{\abovecaptionskip}{-30pt}
\floatstyle{boxed}
\restylefloat{figure}
\usepackage{graphicx}
\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.58,0,0.82}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\lstdefinestyle{mystyle}{
	backgroundcolor=\color{backcolour},
	commentstyle=\color{codegreen},
	keywordstyle=\color{blue},
	numberstyle=\tiny\color{codegray},
	stringstyle=\color{codepurple},
	basicstyle=\footnotesize,
	breakatwhitespace=false,
	breaklines=true,
	captionpos=b,
	keepspaces=true,
	numbers=left,
	numbersep=5pt,
	showspaces=false,
	showstringspaces=false,
	showtabs=false,
	tabsize=2
}
\lstset{style=mystyle}
\title{Data Mining\\
		Home work 11\\ML, Clustering, projects... }
\author{Aqeel Labash\\ \textbf{Lecturer:} Jaak Vilo}
\date{21 April 2016}
\geometry{
	a4paper,
	total={170mm,257mm},
	left=10mm,
	top=5mm,
}
\begin{document}
	\maketitle
\section*{First Question}
For this question I simply created a new point depending on the mean value of all \(Xs ,Ys\) and then calculated the distance between all the points and that point as the \(Z\) value. Here is the code I used :
\begin{lstlisting}[language=Python]
import pandas as pd
import numpy as np
get_ipython().magic(u'matplotlib inline')
import matplotlib.pyplot as plt
from IPython.display import display, HTML
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm

data =pd.read_csv('Linear.csv',sep=',')
plt.plot(data[data.Class==1].X, data[data.Class==1].Y, 'ro')
plt.plot(data[data.Class==0].X, data[data.Class==0].Y, 'bo')
(mX,mY) = (np.mean(data.X),np.mean(data.Y))
plt.plot(mX,mY,'go')
plt.axis([min(data.Y)-1, max(data.X)+1, min(data.Y)-1, max(data.Y)+1])
plt.show()

# #The three levels is :
# - X
# - Y
# - Z :distance from the mean point
lst =[]
for i in range(len(data.X)):
    lst.append(np.linalg.norm(np.array((data.X[i],data.Y[i]))-np.array((mX,mY))))

data['Z'] = pd.Series(np.array(lst), index=data.index)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(data[data.Class==1].X, data[data.Class==1].X, data[data.Class==1].Z, c='r', marker='o')
ax.scatter(data[data.Class==0].X, data[data.Class==0].X, data[data.Class==0].Z, c='b', marker='o')

xx, yy = np.meshgrid(range(6), range(6))
z1 = np.reshape(np.repeat(1.65,36),(6,6))

ax.plot_surface(xx,yy,z1, color='blue',                rstride=3,
                cstride=3,
                alpha=0.3,            # transparency of the surface
                cmap=cm.coolwarm)

ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')

plt.show()
\end{lstlisting}

The previous code output The following figures :
\begin{figure}[H]
\includegraphics[scale=0.7]{points.png}
\caption{Show the points in the plane with the mean point}
\end{figure}
In the previous figure we can see the green point which represent the mean point between all the points.
\begin{figure}[H]
\includegraphics[trim={12cm 5cm 12cm 5cm},clip,scale=0.6]{horizntal.png}
\caption{The hyper plan in 2d prospective}
\end{figure}

\begin{figure}[H]
\includegraphics[trim={1cm 0 10cm 2cm},clip,scale=0.4]{3d.png}
\caption{The hyper plane in 3d prospective}
\end{figure}
Please notice that we can add angle to the hyper plane which would give a bigger margin even.
\section*{Second Question}
For this question I used python code to calculate the distances and make things easier (Just to print the distance list and draw a picture of the points). Here is the code :
\begin{lstlisting}[language=Python]
import pandas as pd
import numpy as np
get_ipython().magic(u'matplotlib inline')
import matplotlib.pyplot as plt
from IPython.display import display, HTML
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from operator import attrgetter

points = pd.read_csv('Clustering.csv',sep=',')

plt.plot(points.X, points.Y, 'ro')
plt.axis([min(points.X)-1, max(points.X)+1, min(points.Y)-1, max(points.Y)+1])
for i, txt in enumerate(points.index):
    plt.annotate(txt+1, (points.X[i],points.Y[i]))
plt.show()

class Dist:
    def __init__(self,p1indx,p2indx,dist):
        self.p1 = p1indx
        self.p2 = p2indx
        self.dist=dist
distances={}
distancelst = []
for i in range(len(points.index)):
    for j in range(len(points.index)):
        if i==j:
            continue
        if (i,j) not in distances.keys():
            dist = np.linalg.norm(np.array([points.X[i],points.Y[i]])-np.array([points.X[j],points.Y[j]]))
            distances[(i,j)]=dist
            distances[(j,i)]=dist
            distancelst.append(Dist(i,j,dist))

distancelst.sort(key=lambda x: x.dist, reverse=False)

for i in distancelst:
    print i.dist,i.p1+1,i.p2+1
\end{lstlisting}
In the previous code I just calculate the distance between all the points and order them min to max so I can pick what points to connect first.Here is the outcome from the previous code.
\begin{figure}[H]
\includegraphics[scale=0.7]{q2points.png}
\caption{Points in the plane numbered}
\end{figure}
The previous code as I said prints the list of ordered points by distance here is the table of it :\\
\begin{tabular}{|c|c|c|c|}
\hline
Distance&Point 1 & Point 2&Notes\\ \hline
0.424264068712&9&18&C1\\ \hline
 0.583095189485&6&11&C2\\ \hline
 0.707106781187&3&12&C3\\ \hline
 0.707106781187&3&13&C3 Extended\\ \hline
 0.707106781187&12&19&C3 Extended\\ \hline
 0.761577310586&12&14&C3 Extended\\ \hline
 0.824621125124&2&14&C3 Extended\\ \hline
 0.860232526704&10&11&C2 Extended\\ \hline
 1.11803398875&7&15&C4\\ \hline
 1.11803398875&18&20&C1 Extended\\ \hline
 1.1401754251&9&11&C1\&C2 merged\(\rightarrow\) C1\\ \hline
 1.39283882772&1&11&C1 Extended\\ \hline
 1.41421356237&3&4&C3 Extended\\ \hline
 1.41421356237&4&8&C3 Extended\\ \hline
 1.41421356237&6&19&C1\&C3 merged\(\rightarrow\) C1\\ \hline
 1.58113883008&7&17&C4 Extended\\ \hline
 1.58113883008&8&17&C1\&C4 merged\(\rightarrow\) C1\\ \hline
 1.80277563773&15&16&C1 Extended\\ \hline
 2.5&5&20&C1 Extended\\ \hline
\end{tabular}
From the previous table I could see better the shortest distance.And here is what I got on papers:
\begin{figure}[H]
\includegraphics[scale=0.15,angle=90,trim={10cm 5cm 10cm 10cm},clip]{single_link.jpg}
\caption{The clusters build}
\end{figure}
Please notice that the previous plot may not be accurate due to many points (I might mixed some points) but the tree is correct  I believe because  I depended on automated calculation for it.I tried to automate the drawing but didn't continue it not to take more time.
\begin{figure}[H]
\includegraphics[scale=0.7,trim={15cm 2cm 15cm 2cm},clip]{groups.png}
\caption{Show some groups}
\end{figure}
Now for the \textbf{Complete link}, the first stage is the same when we create join the first clusters (at the beginning  each point is a cluster).And here is the distance table (New one ):\\
\begin{tabular}{|c|c|c|c|}
\hline
Distance&Point 1 & Point 2&Notes\\ \hline
0.424264068712&9&18&C1\\ \hline
 0.583095189485&6&11&C2\\ \hline
 0.707106781187&3&12&C3\\ \hline
 0.824621125124&2&14&C4\\ \hline
 1.0&3&19&C3 Extended\\ \hline
 1.0&6&10&C2 Extended\\ \hline
 1.11803398875&7&15&C5\\ \hline
 1.3152946438&9&20&C1 Extended\\ \hline
 1.41421356237&4&8&C6\\ \hline
 1.58113883008&13&19&C3 Extended\\ \hline
 1.80277563773&15&17&C5 Extended\\ \hline
 2.2360679775&1&10&C2 Extended\\ \hline
 2.2360679775&2&3&C4\&C3 merged \(\rightarrow\)C3\\ \hline
 2.69258240357&6&20&C1\&C2 merged \(\rightarrow\)C1\\ \hline
 2.82842712475&7&16&C5 Extended\\ \hline
 5.0&2&8&C3\&C6 merged \(\rightarrow\)C3\\ \hline
 5.09901951359&5&6&C1 Expanded\\ \hline
 7.07106781187&10&16&C1\&C5 merged \(\rightarrow\)C1\\ \hline
 8.0622577483&2&16&C1\&C3 merged \(\rightarrow\)C1\\ \hline
\end{tabular}
And here is the images:
\begin{figure}[H]
\includegraphics[angle=90,scale=0.25,trim={10cm 30cm 10cm 30cm},clip]{complete_link.jpg}
\caption{Joining points map}
\end{figure}
\begin{figure}[H]
\includegraphics[angle=90,scale=0.23,trim={15cm 15cm 25cm 37cm},clip]{complete_tree.jpg}
\caption{The tree}
\end{figure}
\section*{Third Question}
For this question I wrote a python code,
\begin{enumerate}
\item  First calculate the distance between the four centers and assign the point to the closest center.
\item Plot the data
\item Calculate the new centers
\item Repeat
\end{enumerate} Here it is :
\begin{lstlisting}[language=Python]
import pandas as pd
import numpy as np
#%matplotlib inline
import matplotlib.pyplot as plt
from IPython.display import display, HTML
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from operator import attrgetter
from matplotlib.patches import Ellipse
from math import atan2,degrees
import numpy.random as rnd

points = pd.read_csv('Clustering.csv',sep=',')

class CenterValue:
    def __init__ (self,p,val,lab):
        self.value=val
        self.point=p
        self.label=lab
    def __str__(self):
        return 'point:{},Value:{},index:{}'.format(self.point,self.value,self.label)

def Distance(p1,p2indx):
    p1 = np.array(p1)
    p2 = np.array((points.X[p2indx],points.Y[p2indx]))
    return np.linalg.norm(p1-p2)

def K_mean(centerpoints):
    centers={}
    for i in centerpoints:
        centers[i]=[]
    for i in range(len(points.index)):
        values=[]
        values.append(CenterValue(centerpoints[0],Distance(centerpoints[0],i),i))
        values.append(CenterValue(centerpoints[1],Distance(centerpoints[1],i),i))
        values.append(CenterValue(centerpoints[2],Distance(centerpoints[2],i),i))
        values.append(CenterValue(centerpoints[3],Distance(centerpoints[3],i),i))
        values.sort(key=lambda x: x.value, reverse=False)
        centers[values[0].point].append(GetPointsList([i])[0])
    return centers

def GetPointsList(Indexs):
    lst=[]
    for i in Indexs:
        lst.append((points.X[i],points.Y[i]))
    return lst

def Get_Means(dictionary_of_lists):
    lst=[]
    for i in dictionary_of_lists.keys():
        mX=0
        mY=0
        for p in dictionary_of_lists[i]:
            mX+=p[0]
            mY+=p[1]
        lst.append((mX/len(dictionary_of_lists[i]),mY/len(dictionary_of_lists[i])))
    return lst

def Plot(k):
    keys = k.keys()
    fig = plt.figure(0)
    ax = fig.add_subplot(111, aspect='equal')
    ax.plot([t[0] for t in k[keys[0]]], [t[1] for t in k[keys[0]]], 'rs')
    ax.plot([t[0] for t in k[keys[1]]], [t[1] for t in k[keys[1]]], 'bs')
    ax.plot([t[0] for t in k[keys[2]]], [t[1] for t in k[keys[2]]], 'gs')
    ax.plot([t[0] for t in k[keys[3]]], [t[1] for t in k[keys[3]]], 'ms')
    for k in keys:
        ax.plot(k[0],k[1],'k*')
        ax.annotate('Cntr',(k[0],k[1]))
    ax.axis([0, 10, 0,10])
    plt.show()

#First Iteration
k = K_mean(GetPointsList(range(4)))
Plot(k)

#Each Time new Iteration
k = Get_Means(k)
k = K_mean(k)
Plot(k)
\end{lstlisting}
The previous code output the following :
\begin{figure}[H]
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{1.png}
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{2.png}\\
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{3.png}
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{4.png}\\
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{5.png}
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{6.png}\\
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{7.png}
\includegraphics[scale=0.5,trim={3cm 1cm 3cm 1cm},clip]{8.png}
\caption{Shows how centers change counting(1,2),(3,4)}
\end{figure}
In the previous figures we can see that after the 5th iteration the centers got stable
\section*{Fourth Question}
The Slide already added.
\section*{Fifth Question}
\href{https://www.kaggle.com/dsoreo/home-depot-product-search-relevance/testing-r/notebook}{Analysis}\\
To Find something different than the previous information I stemmed the text using the following code :
\begin{lstlisting}[language=Python]

# coding: utf-8

# In[1]:

import time
start_time = time.time()

import numpy as np
import pandas as pd
from nltk.stem.porter import *
stemmer = PorterStemmer()
#from nltk.stem.snowball import SnowballStemmer #0.003 improvement but takes twice as long as PorterStemmer
import re
import random
random.seed(1301)

# In[2]:

df_train = pd.read_csv('Data Warehouse/train.csv', encoding="ISO-8859-1")
df_test = pd.read_csv('Data Warehouse/test.csv', encoding="ISO-8859-1")
df_pro_desc = pd.read_csv('Data Warehouse/product_descriptions.csv')
df_attr = pd.read_csv('Data Warehouse/attributes.csv', encoding="ISO-8859-1")
df_brand = df_attr[df_attr.name == "MFG Brand Name"][["product_uid", "value"]].rename(columns={"value": "brand"})
num_train = df_train.shape[0]
df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)
df_all = pd.merge(df_all, df_pro_desc, how='left', on='product_uid')
df_all = pd.merge(df_all, df_brand, how='left', on='product_uid')

# In[3]:

def stem_sentence(sentence):
    sentence = [stemmer.stem_word(x) for x in sentence.split()]
    return sentence

# In[4]:

df_all['search_term'] = df_all['search_term'].map(lambda x:stem_sentence(x))
df_all['product_title'] = df_all['product_title'].map(lambda x:stem_sentence(x))
df_all['product_description'] = df_all['product_description'].map(lambda x:stem_sentence(x))

# In[9]:

words={}
def AddToWords(sentence):
    for i in sentence:
        words[i]=True

# In[10]:

df_all['search_term'] = df_all['search_term'].map(lambda x:AddToWords(x))
df_all['product_title'] = df_all['product_title'].map(lambda x:AddToWords(x))
df_all['product_description'] = df_all['product_description'].map(lambda x:AddToWords(x))

# In[11]:

len(words.keys())
\end{lstlisting}
The Total number of distinct words is :
\textbf{Note:}All code,pythhon ,tex,pdf,etc... files exist in \href{https://github.com/aqeel13932/DM/tree/master/HW11}{github}
\begin{center}
\textbf{E.O.F}
\end{center}
\end{document}


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[utf8]{inputenc}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{December 8, 2014}
\maketitle
$f_n$ converges pointwise to $f$ if $\lim f_n(x)=f(x)$ for all $x$ in the domain of $f_n$

the $L^\infty$-norm is $||f||_\infty=\sup_{x\in\text{dom} f}|f(x)|$

\subsubsection*{lebesque}
$f_n$ converges to $f$ uniformally if $\lim||f_n-f||$

\section*{maybe in exam}
if $f_n:[a,b]\to\mathbb{R}$ is continuous and $f_n\to f$ uniformally then $f$ is continuous. counterexample is $f_n(x)=x^n$ if $x\in[0,1]$ does not converge uniformally

\section*{Lebesque}

\section*{uniform convergence and integrals}
\subsection*{thrm 8.3.1}
if$f_n\to f$ is uniform and $f_n$ are Riemann integrablethen $f$ is Riemann and $\lim\int_a^b{f_n}=\int_a^b{fkk}$

\section*{corollary 8.2.2}
$C(K)=\{f:K\to  \mathbb{R}\text{ that are continuous}\}$. $C(K)$ is a complete space with the uniform norm,ie every cauchy sequence $f_n$ of functions in $C(K)$ converges to a function in $C(K)$.

\section*{example}
let $f_n(x)=f(x)+\frac{1}{n}\sin(\frac{x}{n}\in[a,b]$ where $a\ne 0$ error gets smaller and smaller but sin function wildly varies. so $f_n\to f$ is uniform all are continuous. but the$f_n'$ goes nuts. if we want $f_n\to f$ to converge and to preserve derivitives then we need to explicitly require $f_n'\to f'$ be uniform

\section*{8.4 series}
we say a series of functions converges pointwise (or uniformally) ifthe partial sums converge pointwise (or uniformly)

\subsection*{power series}
a special type of series is the power series. $\sum\limits_{n=0}^\infty{a_nx^n}$. these are easy to study because the ratio test works for them.

$\lim_{n\to\infty}\frac{|a_{n+1}x^{n+1}|}{|a_nx^n|}=\lim_{n\to\infty}\left\lvert\frac{a_{n+1}}{a_n}\right\rvert|x|$ and so it converges if $|x|<\frac{1}{\lim_{n\to \infty}\frac{a_{n+1}}{a_n}}$. check $\frac{x^n}{n!}$

\section*{weierstrass-m}

\section*{power series}
\subsection*{hadamard's thrm}
given a power series $\sum{a_nx^n}$ there is $R$ in $[,+\infty)\cup{+\infty}$ so that the series converges for all $x$ with $|x|<R$ and diverges for all $x$ with $|x|>R$. moreover the series converges uniformally oneach closedinterval $[a,b]\subset(-R,R)$

$R=\frac{1}{\lim_{n\to\infty}\sqrt[n]{|a_n|}}$ root test?
\end{document}


% $Id$

Unstructured grids are commonly used in the computational solution of partial differential
equations.  These are especially useful for problems that involve complex geometry, where
using the less flexible structured grids can result in grid representation of regions
where no computation is needed.  Finite element and finite volume methods map naturally
to unstructured grids and are used commonly in hydrology, ocean modeling, and many other
applications.

In order to provide support for application codes using unstructured grids, the ESMF library
provides a class for representing unstructured grids called the {\bf Mesh}. Fields can be
created on a Mesh to hold data. Fields created on a Mesh can also be used as either the
source or destination or both of an interpolation (i.e. an {\tt ESMF\_FieldRegridStore()} call)
which allows data to be moved between unstructured grids. This section describes the Mesh
and how to create and use them in ESMF.

\subsubsection{Mesh representation in ESMF}\label{sec:meshrep}

A Mesh in ESMF is constructed of {\bf nodes} and {\bf elements}.

A {\bf node}, also known as a vertex or corner, is a part of a Mesh which represents a single point. Coordinate information is
set in a node.

An {\bf element}, also known as a cell, is a part of a mesh which represents a small
region of space. Elements are described in terms of a connected set of nodes which represent locations along their boundaries.

Field data may be located on either the nodes or elements of a Mesh.

\medskip

The dimension of a Mesh in ESMF is specified with two parameters: the {\bf parametric dimension} and the {\bf spatial dimension}.

The {\bf parametric dimension} of a Mesh is the dimension of the topology of the Mesh. This can be thought of as the dimension of
the elements which make up the Mesh. For example, a Mesh composed of triangles would have a parametric dimension of 2, whereas
a Mesh composed of tetrahedra would have a parametric dimension of 3.

The {\bf spatial dimension} of a Mesh is the dimension of the space the Mesh is embedded in. In other words, it is the number of coordinate dimensions needed to describe the location of the nodes making up the Mesh.

For example, a Mesh constructed of squares on a plane would have a parametric dimension of 2 and a spatial dimension of 2.
If that same Mesh were used to represent the 2D surface of a sphere, then the Mesh would still have a parametric dimension
of 2, but now its spatial dimension would be 3.

\subsubsection{Supported Meshes}

The range of Meshes supported by ESMF are defined by several factors: dimension, element
types, and distribution.

ESMF currently only supports Meshes whose number of coordinate dimensions (spatial dimension)
is 2 or 3. The dimension of the elements in a Mesh (parametric dimension) must be less than
or equal to the spatial dimension, but also must be either 2 or 3. This means that a Mesh may
be either 2D elements in 2D space, 3D elements in 3D space, or a manifold constructed of 2D
elements embedded in 3D space.

ESMF supports a range of elements for each Mesh parametric dimension. For a
parametric dimension of 2, the native supported element types are triangles and quadrilaterals.
In addition to these, ESMF supports 2D polygons with any number of sides. Internally these
are represented as sets of triangles, but to the user should behave like any other element.
For a parametric dimension of 3, the supported element types are tetrahedrons
and hexahedrons. See Section~\ref{const:meshelemtype} for diagrams of these. The Mesh
supports any combination of element types within a particular dimension, but types from
different dimensions may not be mixed.  For example, a Mesh cannot be constructed of both
quadrilaterals and tetrahedra.

ESMF currently only supports distributions where every node on a PET must be a part of an
element on that PET. In other words, there must not be nodes without a corresponding element
on any PET.


\section{The effects of camera calibration}
\label{sec:calib-model}
% Original: Error propagation in calibration parameters

During the description of our model, in many cases we considered the contribution of some parameters as null, such as focal length, principal point and distortion coefficients. In Section \ref{sec:teo-calibration}, we briefly discussed about two algorithms that can be used in our scenarios. Like any other calibration algorithm, the two proposed determine the parameters of the camera using some heuristics that could introduce some evaluation errors. Therefore, at the end of our analysis, we wonder if it is necessary to introduce the effects of camera calibration into the model. \\

Focusing on Tsai, we found several researches that aim to reach our same goal:
\cite{Brauer2017}
\cite{fujimoto-teo-err-an}
\cite{7153104}
\cite{Kopparapu:2001:ENC:569876.569877}
\cite{SALVI20021617}
\cite{Samper2013}
\cite{4129503} and
\cite{159901}.
We grouped these references into two different sets: in the first one we put articles that use a more empirical approach, while in the second the most theoretical one. \\

The articles grouped into the ``empirical set'' try to improve the accuracy assessment originally proposed by Tsai in \cite{TsaiTvLenses}. What he suggested were three type of measures, described bellow:
  \begin{itemize}
    \item \textit{Type I} \\
          This type of measures evaluate the accuracy of the calibrated parameters comparing the (theoretic) known 3D points with a the set of 3D points obtained by converting the 2D input grid, using the calibrated parameters themselves. In this way we can realize a statistic on the committed error. An example of this type of measures is the \textit{Normalized Calibration Error} (\acs{NCE})\cite{159901}. Let $p = (x_i, y_i, z_i)$ be the true 3D coordinates of the point $p$, $\hat{p} = (\hat{x}_i, \hat{y}_i, \hat{z}_i)$ be the reconstructed coordinates of $p$, and let $\left( f_u, f_v \right)$ the components of the focal length along the axis of our reference system, we can define the $NCE$ as:
          \begin{equation*}
            NCE = \frac{1}{n}\sum_{i=1}^n \sqrt{\frac{(\hat{x}_i - x_i)^2 + (\hat{y}_i - y_i)^2}{\hat{z}_i^2(f_u^{-2} + f_v^{-2})/12}}
          \end{equation*}

    \item \textit{Type II} \\
    These measures estimate the calibration error by evaluating the distance between two ray of light. As discussed in Section \ref{sec:pinhole_camera} the projection of a 3D point $P$ in the image plane is performed solving the intersection between this plane and a line that connects $P$ with the principal point $O$. In the reality, rays of light don't pass exactly from $O$. The difference from the theoretical and real lines form in the image plane is a circle of radius $r$. The statistics over $r$ allow to estimate the calibration error.

    \item \textit{Type III} \\
Measures belonging to this set are similar to the ones of type I. However, unlike them, type III measures evaluate the calibration error by measuring a known target object, or performing dimensional measures.
  \end{itemize}

We observed that all these errors are strongly related to the set-up used, and in particular to the working conditions. This means that several executions of the calibration process over several grids acquisitions, could change from each other, because of minimum changes in the working conditions. Thus, what we need is a model that allows to assess the quality of the calibration theoretically. If we could do this, we would have a complete view on the design of triangulation systems. Therefore, we focused on the second set, but also in this case we didn't find what we needed. The main issue in evaluating calibration processes is due by the non-linear optimizations that many calibration algorithm, like Tsai and Zhang, use. The problem is not the fact that the problem is non-linear, but the fact that this algorithms use heuristics based on statistics evaluated on the input points. Some studies developed mathematical models, but all of them are limited on a specific case, and can't be used in the general one. \\

Another aspect that we have to consider is that all calibration algorithm uses different strategies to evaluate parameters of the camera: this means that each algorithm is based on a different mathematical model. All this implies that we should design a different error propagation model for each considered mathematical model.
We could consider the pinhole general model, shown in Equation \ref{eq:perspective_projection}. Accordingly with \cite{andersson2008calibration} it is possible to write the Equation \ref{eq:perspective_projection} as:
  \begin{equation}
    u = ID(EX)
  \end{equation}
where $E$ and $I$ are the extrinsic and intrinsic matrix respectively, and $D$ is a matrix that allows to consider the lens distortion, as described in \cite{DBLP:journals/corr/cs-CV-0308003}. From this point, the article continues describing a more general (and may be robust) algorithm for camera calibration but, in the end, we can consider this approach as the most general one. Even if all the proposal models tend to converge to this last one, we continue to remain perplexed about the possibility of generalize the calibration process. \\

Therefore we concluded that an analysis of this type was too taxing for our purposes, and that this type of work may be superfluous \cite{5944307}. Thus, we decided to continue to consider the contribution of the camera parameters as zero.


\documentclass[12pt]{article}
\usepackage[margin=2cm]{geometry}
\usepackage[utf8]{inputenc}

\title{Mutual Information between Categorical and Gaussian data}
\author{David Atienza}
\date{\today}

\usepackage{amsmath}
\usepackage{tikz}
\usetikzlibrary{positioning}
\usetikzlibrary{arrows.meta}

\usepackage{subfig}

\newcommand{\zd}{\mathbf{Z}_D}
\newcommand{\zc}{\mathbf{Z}_C}
\newcommand{\z}{\mathbf{Z}_D, \mathbf{Z}_C}

\begin{document}

\maketitle

\section{Introduction}

This documents shows how to calculate the (conditional) mutual information between categorical and Gaussian data. The mutual information is always calculated between two variables $X$ and $Y$, and can be conditioned on a set of variables $\mathbf{Z}$. Any of this variables can be discrete (categorical) or continuous (Gaussian). When needed, the subscript $D$ and $C$ are used to specify that the variable is discrete or continuous, e.g. $X_{D}$ is a discrete $X$ variable, $\mathbf{Z}_C$ is a set of conditioning continuous variables. The instantations for a given variable is denoted using lowercase, e.g. $x_d$, $y_c$, etc. The number of categories of the variables $X$ and $Y$ are denoted $\text{llx}$ and $\text{lly}$, respectively. The total number of categories for all the $\zd$ variables is denoted $\text{llz} = \prod_i \text{llz}_{i}$.

\section{Mutual Information}

\subsection{Mutual Information between Two Discrete Variables}

First we will calculate the mutual information between two discrete variables:

\begin{equation}
\begin{aligned}
I(X_D; Y_D \mid \z) & = H(X_D, \z) + H(Y_D, \z) - H(X_D, Y_D, \z) - H(\z)\\
& = H(\zc \mid X_D, \zd) + H(X_D, \zd) + H(\zc\mid Y_D, \zd) + H(Y_D, \zd)\\
&\phantom{ = }{}- H(\zc \mid X_D, Y_D, \zd) - H(X_D, Y_D, \zd) - H(\zc \mid \zd) - H(\zd)
\end{aligned}
\end{equation}

Note that:

\begin{equation}
I(X_D; Y_D \mid \zd) = H(X_D, \zd) + H(Y_D, \zd) - H(X_D, Y_D, \zd) - H(\zd)
\end{equation}

Thus:

\begin{equation}
\begin{aligned}
I(X_D; Y_D \mid \z) & = I(X_D; Y_D \mid \zd)\\
&\phantom{=}{} + H(\zc \mid X_D, \zd) + H(\zc\mid Y_D, \zd) - H(\zc \mid X_D, Y_D, \zd) - H(\zc \mid \zd)
\end{aligned}
\end{equation}

where $H(\zc \mid X_D, \zd)$ is the entropy of the Gaussian variables conditioned on $X_D, \zd$. This can be easily calculated:

\begin{equation}
\begin{aligned}
H(\zc \mid X_D, \zd) & = - \sum\limits_{x_d \in X_D, \mathbf{z}_d \in \zd} \int f(\zc, x_d, \mathbf{z}_d)\log\left(\frac{f(\zc, x_d, \mathbf{z}_d)}{f(x_d, \mathbf{z}_d)}\right)d\zc\\
& = - \sum\limits_{x_d \in X_D, \mathbf{z}_d \in \zd} p(x_d, \mathbf{z}_d) \int f(\zc \mid x_d, \mathbf{z}_d)\log\left(f(\zc \mid x_d, \mathbf{z}_d)\right)d\zc\\
& = \sum\limits_{x_d \in X_D, \mathbf{z}_d \in \zd} p(x_d, \mathbf{z}_d) H(\zc \mid x_d, \mathbf{z}_d)
\end{aligned}
\label{eq:conditional_entropy}
\end{equation}

The last integral in (\ref{eq:conditional_entropy}) is the entropy of the Gaussian distribution trained with the $x_d, \mathbf{z}_d$ configuration.

For a Gaussian distribution, this integral can be solved with a closed-form formula:

\begin{equation}
H(\zc \mid x_d, \mathbf{z}_d) = \frac{k}{2} + \frac{k}{2}\log(2\pi) + \frac{1}{2}\log(\lvert\boldsymbol{\Sigma}_{x_d, \mathbf{z}_d}\rvert)
\end{equation}
where $k = \lvert\zc\rvert$ is the dimensionality of the Gaussian distribution and $\boldsymbol{\Sigma}_{x_d, \mathbf{z}_d}$ is the covariance of the data with the discrete configuration $x_d, \mathbf{z}_d$.

The remaining terms $H(\zc \mid Y_D, \zd)$, $H(\zc \mid X_D, Y_D, \zd)$ and $H(\zc \mid \zd)$ can be calculated similarly.

\subsection{Mutual Information between a Discrete and Continuous Variable}

\begin{equation}
\begin{aligned}
I(X_D; Y_C \mid \z) & = H(X_D, \z) + H(Y_C, \z) - H(X_D, Y_C, \z) - H(\z)\\
& = H(\zc \mid X_D, \zd) + H(X_D, \zd) + H(Y_C, \zc\mid \zd) + H(\zd)\\
&\phantom{ = }{}- H(Y_C, \zc \mid X_D, \zd) - H(X_D, \zd) - H(\zc \mid \zd) - H(\zd)\\
& = H(\zc \mid X_D, \zd) + H(Y_C, \zc\mid \zd) - H(Y_C, \zc \mid X_D, \zd) - H(\zc \mid \zd)
\end{aligned}
\end{equation}
where the entropy terms can be calculated as in (\ref{eq:conditional_entropy}), but in this case the variable $Y$ is added in some of the estimated multivariate Gaussian distributions.

\subsection{Mutual Information between Two Continuous Variable}

For an unconditional mutual information, the mutual information can be calculated with the correlation coefficient:

\begin{equation}
I(X_C; Y_C) = -\frac{1}{2}\log\left(1-\rho^2\right)
\end{equation}
where $\rho$ is the linear correlation coefficient between $X$ and $Y$.

For the general case:

\begin{equation}
\begin{aligned}
I(X_C; Y_C \mid \z) & = H(X_C, \z) + H(Y_C, \z) - H(X_C, Y_C, \z) - H(\z)\\
& = H(X_C, \zc \mid \zd) + H(\zd) + H(Y_C, \zc\mid \zd) + H(\zd)\\
&\phantom{ = }{}- H(X_C, Y_C, \zc \mid \zd) - H(\zd) - H(\zc \mid \zd) - H(\zd)\\
& = H(X_C, \zc \mid \zd) + H(Y_C, \zc\mid \zd) - H(X_C, Y_C, \zc \mid \zd) - H(\zc \mid \zd)
\end{aligned}
\end{equation}
where the entropy terms can be calculated as in (\ref{eq:conditional_entropy}), but in this case the variables $X$ and $Y$ are added in some of the estimated multivariate Gaussian distributions.

\section{Empirical Degrees of Freedom}

This section shows the empirical degrees of freedom by running a simulation over 1000 datasets of 100000 instances that are compatible with the null hypothesis (conditional independence). The empirical degrees of freedom have been rounded to the nearest integer number.

\subsection{Empirical Degrees of Freedom between Two Discrete Variables}

\begin{table}[h]
\begin{center}
\begin{tabular}{ccccccccccccccccccc}
\multicolumn{4}{c}{1 cont. parents} & &\multicolumn{4}{c}{2 cont. parents} & &\multicolumn{4}{c}{3 cont. parents} & & \multicolumn{4}{c}{4 cont. parents}\\
llx & lly & llz & df & & llx & lly  & llz & df & & llx & lly & llz & df & & llx & lly  & llz & df\\
2 & 4 & 3 & 18 & & 2 & 4 & 3 & 36 & & 2 & 4 & 3 & 63 & & 2 & 4 & 3 & 99\\
2 & 3 & 4 & 16 & & 2 & 3 & 4 & 32 & & 2 & 3 & 4 & 56 & & 2 & 3 & 4 & 88\\
3 & 4 & 2 & 24 & & 3 & 4 & 2 & 48 & & 3 & 4 & 2 & 84 & & 3 & 4 & 2 & 132
\end{tabular}
\end{center}
\end{table}

Inducted formula:

\begin{equation}
\text{df} = (\text{llx} - 1)\cdot(\text{lly} - 1)\cdot\text{llz}\cdot\left[1 + \frac{\lvert\zc\rvert\cdot(\lvert\zc\rvert + 1)}{2}\right]
\end{equation}

\subsection{Empirical Degrees of Freedom between a Discrete and Continuous Variable}

\begin{table}[h]
\begin{center}
\begin{tabular}{ccccccccccccccccccc}
\multicolumn{4}{c}{1 cont. parents} & &\multicolumn{4}{c}{2 cont. parents} & &\multicolumn{4}{c}{3 cont. parents} & & \multicolumn{4}{c}{4 cont. parents}\\
llx & conty & llz & df & & llx & conty  & llz & df & & llx & conty & llz & df & & llx & conty  & llz & df\\
2 & & 3 & 6 & & 2 & & 3 & 9 & & 2 & & 3 & 12 & & 2 & & 3 & 15\\
2 & & 4 & 8 & & 2 & & 4 & 12 & & 2 & & 4 & 16 & & 2 & & 4 & 20\\
3 & & 2 & 8 & & 3 & & 2 & 12 & & 3 & & 2 & 16 & & 3 & & 2 & 20\\
3 & & 4 & 16 & & 3 & & 4 & 24 & & 3 & & 4 & 32 & & 3 & & 4 & 40\\
4 & & 2 & 12 & & 4 & & 2 & 18 & & 4 & & 2 & 24 & & 4 & & 2 & 30\\
4 & & 3 & 18 & & 4 & & 3 & 27 & & 4 & & 3 & 36 & & 4 & & 3 & 45
\end{tabular}
\end{center}
\end{table}

Inducted formula:

\begin{equation}
\text{df} = (\text{llx} - 1)\cdot\text{llz}\cdot\left[1 + \lvert\zc\rvert\right]
\end{equation}

\subsection{Empirical Degrees of Freedom between Two Continuous Variables}

\begin{table}[h]
\begin{center}
\begin{tabular}{ccccccccccccccccccc}
\multicolumn{4}{c}{1 cont. parents} & &\multicolumn{4}{c}{2 cont. parents} & &\multicolumn{4}{c}{3 cont. parents} & & \multicolumn{4}{c}{4 cont. parents}\\
contx & conty & llz & df & & contx & conty  & llz & df & & contx & conty & llz & df & & contx & conty  & llz & df\\
 & & 2 & 2 & & & & 2 & 2 & & & & 2 & 2 & & & & 2 & 2\\
 & & 3 & 3 & & & & 3 & 3 & & & & 3 & 3 & & & & 3 & 3\\
 & & 4 & 4 & & & & 4 & 4 & & & & 4 & 4 & & & & 4 & 4
\end{tabular}
\end{center}
\end{table}

Inducted formula:

\begin{equation}
\text{df} = \text{llz}
\end{equation}

\section{Asymptotic Degrees of Freedom}

\subsection{Asymptotic Degrees of Freedom between Two Discrete Variables}

There is a direct relationship between mutual information and a likelihood ratio test ($G$-test):

\begin{equation}
G = 2\cdot N\cdot I(X_D; Y_D \mid \z)
\end{equation}

The $G$ statistic is distributed as a $\chi^2$ if the null hypothesis is true. The degrees of freedom of the $\chi^2$ distribution is the difference in the number of free parameters between a model where there is conditional dependence between $X_D$ and $Y_D$ (Figure~\ref{fig:h1_discrete}), and a model where there is no conditional dependence between $X_D$ and $Y_D$  (Figure~\ref{fig:h0_discrete}).

\begin{figure}
\begin{center}
\subfloat[$H_{0}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_D$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_D$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
\end{tikzpicture}
\label{fig:h0_discrete}
}
\hspace{3cm}
\subfloat[$H_{1}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_D$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_D$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (X_D.east) edge (Y_D.west);
\end{tikzpicture}
\label{fig:h1_discrete}
}
\end{center}
\caption{Null hypothesis (left) and alternative (right) models for two discrete variables.}
\end{figure}

The only node that contains a different number of parameters is the conditional distribution of $Y_D$. So we must analyze that distribution to find the degrees of freedom of the $\chi^2$ distribution.

For the $H_0$ model, the distribution $f(Y_D \mid \z)$ can be defined using the Bayes rule (as in a conditional linear Gaussian networks the discrete nodes do not have continuous parents):

\begin{equation}
f(Y_D \mid \z) = \frac{f(\zc \mid Y_D, \zd) f(Y_D \mid \zd)}{f(\zc \mid \zd)}
\end{equation}

Note that only $\text{lly} - 1$ models are needed to be fitted because the probabilities $f(Y_D \mid \z)$ must sum to 1, so the probability for the last category can be defined as:

\begin{equation}
f(Y_D = \text{lly}\mid \z) = 1 - \sum\limits_{i = 1}^{\text{lly} - 1}\frac{f(\zc \mid Y_D = i, \zd) f(Y_D = i \mid \zd)}{f(\zc \mid \zd)}
\end{equation}

$f(\zc \mid Y_D, \zd)$ has  a number of free parameters equal to:

\begin{equation}
(\text{lly} - 1)\cdot\text{llz}\cdot\frac{\lvert\zc\rvert\cdot(\lvert\zc\rvert + 3)}{2}
\end{equation}

Note that $\frac{\lvert\zc\rvert\cdot(\lvert\zc\rvert + 3)}{2}$ is the number of free parameters of a multivariate Gaussian distribution of $\lvert\zc\rvert$ dimensions.

$f(Y_D \mid \zd)$ has a number of free parameters equal to:

\begin{equation}
(\text{lly} - 1)\cdot \text{llz}
\end{equation}

$f(\zc \mid \zd)$ does not contain free parameters because it can be represented using the previous functions:

\begin{equation}
f(\zc \mid \zd) = \sum\limits_{y_d \in Y_D} f(\zc \mid Y_D = y_d, \zd) f(Y_D = y_d \mid \zd)
\end{equation}

For the $H_1$ model, the distribution $f(Y_D \mid X_D, \z)$ can be defined using the Bayes rule (as in a conditional linear Gaussian networks the discrete nodes do not have continuous parents):

\begin{equation}
f(Y_D \mid X_D, \z) = \frac{f(\zc \mid X_D, Y_D, \zd) f(Y_D \mid X_D, \zd)}{f(\zc \mid X_D, \zd)}
\end{equation}

Note that only $\text{lly} - 1$ models are needed to be fitted because the probabilities $f(Y_D \mid X_D, \z)$ must sum to 1, so the probability for the last category can be defined as:

\begin{equation}
f(Y_D = \text{lly}\mid X_D, \z) = 1 - \sum\limits_{i = 1}^{\text{lly} - 1}\frac{f(\zc \mid X_D, Y_D = i, \zd) f(Y_D = i \mid X_D, \zd)}{f(\zc \mid X_D, \zd)}
\end{equation}

$f(\zc \mid X_D, Y_D, \zd)$ has  a number of free parameters equal to:

\begin{equation}
(\text{lly} - 1)\cdot\text{llx}\cdot\text{llz}\cdot\frac{\lvert\zc\rvert\cdot(\lvert\zc\rvert + 3)}{2}
\end{equation}

$f(Y_D \mid X_D, \zd)$ has a number of free parameters equal to:

\begin{equation}
(\text{lly} - 1)\cdot\text{llx}\cdot\text{llz}
\end{equation}

$f(\zc \mid X_D, \zd)$ does not contain free parameters because it can be represented using the previous functions:

\begin{equation}
f(\zc \mid X_D, \zd) = \sum\limits_{y_d \in Y_D} f(\zc \mid X_D, Y_D = y_d, \zd) f(Y_D = y_d \mid X_D, \zd)
\end{equation}

The difference in parameters (and the degrees of freedom of the $\chi^2$) is equal to:

\begin{equation}
\begin{aligned}
\text{df} & = \text{lly} - 1)\cdot(\text{llx} - 1)\cdot\text{llz}\cdot\frac{\lvert\zc\rvert\cdot(\lvert\zc\rvert + 3)}{2} + (\text{lly} - 1)\cdot(\text{llx}-1)\cdot\text{llz}\\
& = \boxed{(\text{lly} - 1)\cdot(\text{llx}-1)\cdot\text{llz}\left[1 + \frac{\lvert\zc\rvert\cdot(\lvert\zc\rvert + 3)}{2}\right]}
\end{aligned}
\end{equation}

\subsection{Asymptotic Degrees of Freedom between a Discrete and Continuous Variable}

The $H_{0}$ model is shown in Figure~\ref{fig:h0_mixed} and the $H_{1}$ model is shown in Figure~\ref{fig:h1_mixed}.

\begin{figure}
\begin{center}
\subfloat[$H_{0}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_D$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_C$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
\end{tikzpicture}
\label{fig:h0_mixed}
}
\hspace{3cm}
\subfloat[$H_{1}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_D$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_C$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (X_D.east) edge (Y_D.west);
\end{tikzpicture}
\label{fig:h1_mixed}
}
\end{center}
\caption{Null hypothesis (left) and alternative (right) models for a continuous and discrete variable.}
\end{figure}

The distribution $f(Y_C \mid \z)$ has a number of free parameters equal to:

\begin{equation}
\text{llz}\cdot\left(\lvert\zc\rvert + 2\right)
\end{equation}

The distribution $f(Y_C \mid X_D, \z)$ has a number of free parameters equal to:

\begin{equation}
\text{llx}\cdot\text{llz}\cdot\left(\lvert\zc\rvert + 2\right)
\end{equation}

The difference in parameters (and the degrees of freedom of the $\chi^2$) is equal to:

\begin{equation}
\boxed{\text{df} = (\text{llx}-1)\cdot\text{llz}\cdot\left(\lvert\zc\rvert + 2\right)}
\end{equation}

The same result can be derived using a different $H_0$ model (Figure~\ref{fig:h0_mixed2}) and $H_1$ model (Figure~\ref{fig:h1_mixed2}). In this case, the difference in the number of parameters happens to be in the conditional distribution of $X_D$.

\begin{figure}
\begin{center}
\subfloat[$H_{0}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_D$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_C$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
\end{tikzpicture}
\label{fig:h0_mixed2}
}
\hspace{3cm}
\subfloat[$H_{1}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_D$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_C$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Y_D.west) edge (X_D.east);
\end{tikzpicture}
\label{fig:h1_mixed2}
}
\end{center}
\caption{Variation of the null hypothesis (left) and alternative (right) models for a continuous and discrete variable.}
\end{figure}

The $f(X_D\mid \z)$ can be defined using the Bayes rule:

\begin{equation}
f(X_D\mid\z) = \frac{f(\zc\mid X_D, \zd)f(X_D\mid \zd)}{f(\zc\mid\zd)}
\end{equation}

Note that only $\text{llx} - 1$ models are needed to be fitted because the probabilities $f(X_D \mid \z)$ must sum to 1, so the probability for the last category can be defined as:

\begin{equation}
f(X_D = \text{llx}\mid \z) = 1 - \sum\limits_{i = 1}^{\text{llx} - 1}\frac{f(\zc \mid X_D = i, \zd) f(X_D = i \mid \zd)}{f(\zc \mid\zd)}
\end{equation}

The distribution $f(\zc\mid X_D, \zd)$ has a number of free parameters equal to:

\begin{equation}
(\text{llx} - 1)\cdot\text{llz}\cdot\frac{\lvert\zc\rvert\cdot(\lvert\zc\rvert + 3)}{2}
\end{equation}

The distribution $f(X_D\mid \zd)$ has a number of free parameters equal to:

\begin{equation}
(\text{llx} - 1)\cdot\text{llz}
\end{equation}

$f(\zc \mid\zd)$ does not contain free parameters because it can be represented using the previous functions:

\begin{equation}
f(\zc \mid \zd) = \sum\limits_{x_d \in X_D} f(\zc \mid X_D = x_d, \zd) f(X_D = x_d \mid \zd)
\end{equation}

The $f(X_D\mid \z)$ can be defined using the Bayes rule:

\begin{equation}
f(X_D\mid Y_C, \z) = \frac{f(Y_C, \zc\mid X_D, \zd)f(X_D\mid \zd)}{f(Y_C, \zc\mid\zd)}
\end{equation}

Note that only $\text{llx} - 1$ models are needed to be fitted because the probabilities $f(X_D \mid Y_C, \z)$ must sum to 1, so the probability for the last category can be defined as:

\begin{equation}
f(X_D = \text{llx}\mid Y_C, \z) = 1 - \sum\limits_{i = 1}^{\text{llx} - 1}\frac{f(Y_C, \zc \mid X_D = i, \zd) f(X_D = i \mid \zd)}{f(Y_C,\zc \mid\zd)}
\end{equation}

The distribution $f(Y_C, \zc\mid X_D, \zd)$ has a number of free parameters equal to:

\begin{equation}
(\text{llx} - 1)\cdot\text{llz}\cdot\frac{\left(\lvert\zc\rvert + 1\right)\cdot\left(\lvert\zc\rvert + 4\right)}{2}
\end{equation}

The distribution $f(X_D\mid \zd)$ has a number of free parameters equal to:

\begin{equation}
(\text{llx} - 1)\cdot\text{llz}
\end{equation}

$f(\zc \mid\zd)$ does not contain free parameters because it can be represented using the previous functions:

\begin{equation}
f(Y_C, \zc \mid \zd) = \sum\limits_{x_d \in X_D} f(Y_C, \zc \mid X_D = x_d, \zd) f(X_D = x_d \mid \zd)
\end{equation}

The difference in parameters (and the degrees of freedom of the $\chi^2$) is equal to:

\begin{equation}
\begin{aligned}
\text{df} & = (\text{llx}-1)\cdot\text{llz}\cdot\left(\frac{\lvert\zc\rvert^{2} + 5\lvert\zc\rvert + 4 - \lvert\zc\rvert^{2} - 3\lvert\zc\rvert}{2}\right)\\
& = (\text{llx}-1)\cdot\text{llz}\cdot\left(\frac{2\lvert\zc\rvert + 4}{2}\right)\\
& = \boxed{(\text{llx}-1)\cdot\text{llz}\cdot\left(\lvert\zc\rvert + 2\right)}
\end{aligned}
\end{equation}

\subsection{Asymptotic Degrees of Freedom between Two Continuous Variables}

The $H_{0}$ model is shown in Figure~\ref{fig:h0_cont} and the $H_{1}$ model is shown in Figure~\ref{fig:h1_cont}.

\begin{figure}
\begin{center}
\subfloat[$H_{0}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_C$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_C$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
\end{tikzpicture}
\label{fig:h0_cont}
}
\hspace{3cm}
\subfloat[$H_{1}$ model] {
\begin{tikzpicture}
    \node[shape=circle,draw=black] (Z_D) at (0,3) {$\zd$};
    \node[shape=circle,draw=black] (Z_C) at (3,3) {$\zc$};
    \node[shape=circle,draw=black] (X_D) at (0,0) {$X_C$};
    \node[shape=circle,draw=black] (Y_D) at (3,0) {$Y_C$};

    \draw [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_D) edge (Y_D.north);

    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (X_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (Z_C) edge (Y_D.north);
    \path [-{Stealth[length=3mm, width=2mm]}] (X_D.east) edge (Y_D.west);
\end{tikzpicture}
\label{fig:h1_cont}
}
\end{center}
\caption{Null hypothesis (left) and alternative (right) models for two continuous variables.}
\end{figure}

For the $H_0$ model, the distribution $f(Y_C\mid \z)$ has a number of free parameters equal to:

\begin{equation}
\text{llz}\cdot\left(\lvert\zc\rvert + 2\right)
\end{equation}

For the $H_1$ model, the distribution $f(Y_C\mid X_C, \z)$ has a number of free parameters equal to:

\begin{equation}
\text{llz}\cdot\left(\lvert\zc\rvert + 3\right)
\end{equation}

The difference in parameters (and the degrees of freedom of the $\chi^2$) is equal to:

\begin{equation}
\boxed{\text{llz}}
\end{equation}

\end{document}


% This LaTeX was auto-generated from an M-file by MATLAB.
% To make changes, update the M-file and republish this document.

\subsection*{sq\_dist.m}

\begin{par}
sq\_dist - a function to compute a matrix of all pairwise squared distances between two sets of vectors, stored in the columns of the two matrices, a (of size D by n) and b (of size D by m). If only a single argument is given or the second matrix is empty, the missing matrix is taken to be identical to the first.
\end{par} \vspace{1em}
\begin{par}
Special functionality: If an optional third matrix argument Q is given, it must be of size n by m, and in this case a vector of the traces of the product of Q' and the coordinatewise squared distances is returned.
\end{par} \vspace{1em}
\begin{par}
NOTE: The program code is written in the C language for efficiency and is contained in the file sq\_dist.c, and should be compiled using matlabs mex facility. However, this file also contains a (less efficient) matlab implementation, supplied only as a help to people unfamiliar with mex. If the C code has been properly compiled and is avaiable, it automatically takes precendence over the matlab code in this file.
\end{par} \vspace{1em}
\begin{par}
Usage: C = sq\_dist(a, b)    or: C = sq\_dist(a)  or equiv.: C = sq\_dist(a, [])    or: c = sq\_dist(a, b, Q) where the b matrix may be empty.
\end{par} \vspace{1em}
\begin{par}
where a is of size D by n, b is of size D by m (or empty), C and Q are of size n by m and c is of size D by 1.
\end{par} \vspace{1em}
\begin{par}
Copyright (c) 2003, 2004, 2005 and 2006 Carl Edward Rasmussen. 2006-03-09.
\end{par} \vspace{1em}

\begin{lstlisting}
function C = sq_dist(a, b, Q);
\end{lstlisting}

\subsection*{Code}

\begin{lstlisting}
if nargin < 1 | nargin > 3 | nargout > 1
  error('Wrong number of arguments.');
end

if nargin == 1 | isempty(b)                   % input arguments are taken to be
  b = a;                                   % identical if b is missing or empty
end

[D, n] = size(a);
[d, m] = size(b);
if d ~= D
  error('Error: column lengths must agree.');
end

if nargin < 3
  C = zeros(n,m);
  for d = 1:D
    C = C + (repmat(b(d,:), n, 1) - repmat(a(d,:)', 1, m)).^2;
  end
  % C = repmat(sum(a.*a)',1,m)+repmat(sum(b.*b),n,1)-2*a'*b could be used to
  % replace the 3 lines above; it would be faster, but numerically less stable.
else
  if [n m] == size(Q)
    C = zeros(D,1);
    for d = 1:D
      C(d) = sum(sum((repmat(b(d,:), n, 1) - repmat(a(d,:)', 1, m)).^2.*Q));
    end
  else
    error('Third argument has wrong size.');
  end
end
\end{lstlisting}


% declare document class and geometry
\documentclass[12pt]{article} % use larger type; default would be 10pt
\usepackage[margin=1in]{geometry} % handle page geometry

% import packages and commands
\input{../header.tex}

% title information
\title{Phys 221A -- Quantum Mechanics -- Lec04}
\author{UCLA, Fall 2014}
\date{\formatdate{15}{10}{2014}} % Activate to display a given date or no date (if empty),
         % otherwise the current date is printed

\begin{document}
\maketitle

\section{Continuous variables}

It's possible to choose an orthonormal basis $\set{\ket{\xi}}$ such that
\begin{equation}
\braket{\xi}{\xi'} = \delta(\xi - \xi').
\end{equation}
Then for any $\ket{\psi} \in G$ we have
\begin{equation}
\ket{\psi} = \int d\xi ~ c_\xi \ket{\xi}.
\end{equation}
This immediately implies the closure relation
\begin{equation}
\int d\xi \ket{\xi} \bra{\xi} = 1,
\end{equation}
and furthermore we have
\begin{equation}
\braket{\xi'}{\psi} = \int d\xi c_\xi \braket{\xi'}{\xi} = c_{\xi'}.
\end{equation}
We can check the closure relation:
\begin{equation}
\int d\xi \ket{\xi} \braket{\xi}{\psi} = \int d\xi ~ c_\xi \ket{\xi} = \ket{\psi},
\end{equation}
which is what we wanted.

Now, given an operator $O$, we can write $O(\xi, \xi') = \bra{\xi} O \ket{\xi'}$. If we have $O\ket{\psi} = \ket{\psi'}$ and
\begin{equation}
\int d\xi ~ c_\xi \ket{\xi}, \qquad \int d\xi ~ b_\xi \ket{\xi},
\end{equation}
then we have
\begin{eqn}
b_\xi = \int d\xi' ~ O(\xi, \xi') c_{\xi'}.
\end{eqn}

In the position representation we have $\xi \rightarrow \v{r}$ in $n$-dimensional space. So
\begin{eqn}
\ket{\psi} \rightarrow c_{\v{r}} = \braket{\v{r}}{\psi} \equiv \psi(\v{r})
\end{eqn}
or we can write
\begin{eqn}
\ket{\psi} = \int d^3r ~ \psi(\v{r}) \ket{\v{r}}.
\end{eqn}
In the momentum representation we would have
\begin{eqn}
\ket{\psi} = \int d^3p ~ \psi(\v{p}) \ket{\v{p}}.
\end{eqn}
Then if we measure the position, the probability density is given by $\abs{\psi(\v{r})}^2$ and the probability to find the particle in a given volume $dV$ is given by $\abs{\psi(\v{r})}^2 dV$. This of course assumes normalization,
\begin{eqn}
\int d^3r \abs{\psi(\v{r})}^2 = 1, \qquad \text{or} \qquad \braket{\psi}{\psi} = 1.
\end{eqn}

\section{Spinless particle in 1D space}

Classically we would have the Lagrangian $\mathcal{L}(x, \dot{x}, t)$. Quantum mechanically, we will have a Hilbert space $L_2$ over $c$-valued function $f(x)$. That is, functions in the space have the property
\begin{eqn}
\int dx \abs{f(x)}^2 < \infty,
\end{eqn}
with inner product given by
\begin{eqn}
\braket{f}{g} = \int dx f^*(x) g(x).
\end{eqn}
Physical states will be normalized in this space,
\begin{eqn}
\braket{\psi}{\psi} = 1, \qquad \text{or} \qquad \int dx \abs{\psi(x)}^2 = 1.
\end{eqn}

Let's look at some operators in this space. We find that $x$ is a Hermitian operator:
\begin{eqn}
\int dx (x f(x))^* g(x) \int dx f^*(x) (x g(x)).
\end{eqn}
We find that $d/dx$ is anti-Hermitian:
\begin{eqn}
\int dx (\frac{d}{dx} f(x))^* g(x) = \int dx f^*(x) (-\frac{d}{dx}) g(x)
\end{eqn}
so that $(d/dx)^\dagger = -d/dx$ and $i d/dx$ is Hermitian.

We will, for now, \textit{define} the linear momentum: $p = -i \hbar d/dx$. Later we will see that this reduces to the classical momentum. Furthermore, the momentum part of the Hamiltonian will typically be given by
\begin{eqn}
H = \frac{p^2}{2m} = -\frac{\hbar}{2m} \partial_x^2.
\end{eqn}
In general $H(\v{r},\v{p})$ will be the same in classical and quantum mechanics.

We also have the commutation relation
\begin{eqn}
[x,p] = i\hbar.
\end{eqn}
This gives us the Heisenberg uncertainy principle,
\begin{eqn}
\avg{(\Delta x)^2} \avg{(\Delta p)^2} \geq \hbar^2 / 4.
\end{eqn}
A Gaussian wave packet will take the form
\begin{eqn}
\psi(x) = \frac{1}{\pi^{1/4} \sqrt{d}} ~ e^{ikx - x^2 / 2d}
\end{eqn}
for all $k, d \in \reals$ such that $d > 0$. Gaussian wave packets minimize the uncertainty,
\begin{eqn}
\avg{(\Delta x)^2} \avg{(\Delta p)^2} = \hbar^2 / 4.
\end{eqn}

In the position representation we might have an equation
\begin{eqn}
x g_y(x) = y g_y(x)
\end{eqn}
for any fixed $y \in \reals$. Then $y$ is the eigenvalue of $x$ and $g_y(x)$ is the eigenfunction of $x$, and we have
\begin{eqn}
g_y(x) = \delta(x-y) \equiv \ket{y}
\end{eqn}
and
\begin{eqn}
\braket{y}{y'} = \int dx \delta(x-y) \delta(x-y') = \delta(y-y').
\end{eqn}

Next we have the eigenvalue equation
\begin{eqn}
(-i \hbar \frac{d}{dx}) \ket{f_p} = p \ket{f_p}.
\end{eqn}
The eigenstate will be given by
\begin{eqn}
f_p (x) = e^{ipx / \hbar} / \sqrt{2\pi \hbar}
\end{eqn}
where we can write a wavelength $\lambda = 2 \pi \hbar / p$ which is just the de Broglie wavelength. Furthermore we have
\begin{eqn}
\braket{f_p}{f_{p'}} = \int dx \frac{1}{2\pi \hbar} e^{i(p-p') x / \hbar} = \delta(p - p').
\end{eqn}
Recall that we can write
\begin{eqn}
\delta(k) = \int \frac{dx}{2\pi} e^{ikx}.
\end{eqn}

We can write down a translation operator in $L_2$,
\begin{eqn}
U(a) = e^{-a \partial_x} = 1 - a \partial_x + (a/2) \partial_x^2 + \dots
\end{eqn}
which acts on functions by translating the position by a constant,
\begin{eqn}
U(a) f(x) = f(x-a).
\end{eqn}
We can write
\begin{eqn}
\partial_a U(a) \Big|_{a \rightarrow 0} = -\partial_x
\end{eqn}
so we say that $\partial_x$ is the \textit{generator} of translations. Up to a constant, we can say that the momentum $p$ is the generator of translation,
\begin{eqn}
p = i \hbar \partial_a U(a) \Big|_{a \rightarrow 0}.
\end{eqn}

\section{Quantum Dynamics}

Temporal evolution will take us from $\ket{\psi(0)} \mapsto \ket{\psi(t)}$. We will consider $t=0$ the present, $t<0$ the past, and $t>0$ the future, and denote the present state $\ket{\psi(0)} = \ket{\psi}$. We will be able to write time evolution as an operator
\begin{eqn}
\ket{\psi} \mapsto \ket{\psi(t)} \equiv U(t) \ket{\psi}.
\end{eqn}
Because the state should be physical and thus normalized at all times, we have
\begin{eqn}
1 = \braket{U(t) \psi}{U(t) \psi} = \bra{\psi} U^\dagger U \ket{\psi}
\end{eqn}
so $U(t)$ is a unitary operator.

Finally, we can write the Schroedinger equation,
\begin{eqn}
i \hbar \frac{d}{dt} \ket{\psi(t)} = H(t) \ket{\psi(t)}
\end{eqn}
for any physical state $\ket{\psi}$, where $H(t)$ is a Hermitian operator called the Hamiltonian (aka the energy operator). We can rewrite this as
\begin{eqn}
\left[ i \hbar \frac{d}{dt} U(t) \right] \ket{\psi} = \left[ H(t) U(t) \right] \ket{psi},
\end{eqn}
which is true for any $\ket{\psi}$, so we can write the Schroedinger equation instead as an equation for $U(t)$,
\begin{eqn}
i \hbar \frac{d}{dt} U(t) = H(t) U(t), \quad \text{where} \quad U(0) = 1.
\end{eqn}
We can solve this pretty easily -- consider the differential equation
\begin{eqn}
\frac{d}{dt} f(t) = h(t) f(t).
\end{eqn}
We can solve this by rewriting
\begin{eqn}
df / f = h ~ dt
\end{eqn}
which has solution
\begin{eqn}
\log(f(t) / f(0)) = \int_0^t h(t') dt'.
\end{eqn}
Thus applying this to the equation for $U(t)$ we find the time evolution operator,
\begin{eqn}
U(t) = \exp \left( -\frac{i}{\hbar} \int_0^t H(t') dt' \right).
\end{eqn}

\end{document}


\numberwithin{definition}{section}
\numberwithin{example}{section}
\numberwithin{equation}{section}
\numberwithin{figure}{section}

\addbibresource{../bib/taiga.bib}
\addbibresource{../bib/tex.bib}
%-----------------------------------------------------------------
\title{Regression costs for decision trees}
\author{\textsc{John Alan McDonald }}
\date{\today}
%-----------------------------------------------------------------
\begin{document}
\maketitle
%-----------------------------------------------------------------

The purpose of this document is work thru an alternative to $L_2$
cost that is a bit more efficient to compute, and gives the same
results when choosing split predicates in decision tree growing.

\section{\label{sub:Decision-trees}Greedy decision trees}

A general binary decision tree consists of
\begin{itemize}
\item internal \emph{split} nodes, each containing a predicate
that determines
whether a record goes to the left or right child of that node.
\item terminal \emph{leaf} nodes, each containing a leaf model
 function whose value is the tree's prediction for any record
 that ends up in that node.
\end{itemize}

Greedy split optimization --- choose the best out of all
feasible splits, and repeat on the resulting child nodes until
there are no feasible splits --- is the most common way of
growing decision trees.
It depends on several things:
\begin{enumerate}
  \item A cost function $c$ used to define 'best'.
  \item An enumeration of splits to consider. Pure greedy
  splitting
  considers all 'feasible' splits on all attributes.
  \begin{enumerate}
    \item For categorical attributes, that, in general, means
    considering every partition of the categories into $2$
    subsets. However, for some important cost functions
    (eg 2-class Gini, $L_2$), it can be shown that the optimal
    split can be found by sorting the categories by the
    corresponding score function (eg the response mean for $L_2$
    cost), and then considering only splits by score.
    \item For numerical attributes, the most general split would
    come from treating the distinct values like the categories
    of a categorical variable. However, no one does that, mostly
    because there are usually too many distinct values. Instead,
    only splits by
    $\leq$ vs $>$ one of the distinct values are considered.
\end{enumerate}
\item A feasibility test that determines whether a given split
on a attribute is allowed. The most common case here is to
require both children of the split contain some minimum number of
training records.
\end{enumerate}

\section{\label{sec:numerical}Cost functions for $L_2$ numerical
regression}

Let $\mathcal{T} = \{ \left( y,\mathbf{x} \right) \}$ be the
training data in the node to be split.
It is a set of pairs of predictor record $\mathbf{x}$ and
ground truth response $y$, where $y\in\mathbb{R}$ for numerical
regression. We are considering splits on some particular
predictor field $x_k$, which might be numerical or categorical.

The cost function for $L_2$ regression is the sum of squared
deviations from the mean:

$L_{2}\left(\mathcal{T}\right)
= \sum_{y\in\mathcal{T}}\,
\left(y-\bar{y}_{\mathcal{T}}\right)^{2}$,
where $\bar{y}_{\mathcal{T}}
= \frac{1}{\#\mathcal{T}}\sum_{y\in\mathcal{T}}\,y$.

Note that computing this \textit{accurately},
in an online fashion, for
moderate $\#\mathcal{T}$, the number of records in $\mathcal{T}$,
allowing for
the updating/downdating needed for fast split optimization,
requires some care.

However, a little bit of algebra will let us use a simpler
alternative to get the same splits.

Any split partitions the training y-values
$\mathcal{T=}\left\{ y\right\} $ into left and right subsets:
$\mathcal{T}=\mathcal{L}\uplus\mathcal{R}$.
The split cost is:

\begin{align*}
c\left(\mathcal{L},\mathcal{R}\right)= &
L_{2}\left(\mathcal{L}\right)
+ L_{2}\left(\mathcal{R}\right)\\
= & \sum_{y\in\mathcal{L}}\,
\left(y-\bar{y}_{\mathcal{L}}\right)^{2}
+
\sum_{y\in\mathcal{R}}\,\left(y-\bar{y}_{\mathcal{R}}\right)^{2}\\
= & \sum_{y\in\mathcal{L}}\left[y^{2}-2\bar{y}_{\mathcal{L}}y
+
\bar{y}_{\mathcal{L}}^{2}\right]
+
\sum_{y\in\mathcal{R}}\left[y^{2}-2\bar{y}_{\mathcal{R}}y
+
\bar{y}_{\mathcal{R}}^{2}\right]\\
= & \sum_{\mathcal{L}\uplus\mathcal{R}}y^{2}
-
\frac{\left(\sum_{\mathcal{L}}y\right)^{2}}{\#\mathcal{L}}
-
\frac{\left(\sum_{\mathcal{R}}y\right)^{2}}{\#\mathcal{R}}
\end{align*}

Since $\sum_{\mathcal{L}\uplus\mathcal{R}}y^{2}$ doesn't
depend on the split, minimizing
$c\left(\mathcal{L},\mathcal{R}\right)$ is
equivalent to minimizing
$-\left[\frac{\left(\sum_{\mathcal{L}}y\right)^{2}}{\#\mathcal{L}}
+
\frac{\left(\sum_{\mathcal{R}}y\right)^{2}}{\#\mathcal{R}}\right]$,
so we can use
$\frac{-\left(\sum_{\mathcal{T}}y\right)^{2}}{\#\mathcal{T}}$
as our cost function in split optimization.

\section{Cost functions for $L_2$ vector-valued regression}

Let $\mathcal{T} = \{ \left( \mathbf{y}, \mathbf{x} \right) \}$
be the training data in the node to be split.
Here the ground truth response $\mathbf{y}$ is a vector,
$\mathbf{y}\in\mathbb{R}^m$, rather than a single number.

The cost function for $L_2$ vector-valued regression is the sum of
squared $L_2$ distances from the mean vector:
\begin{align*}
L_{2}\left(\mathcal{T}\right)
= &
\sum_{y\in\mathcal{T}}\,
\|\mathbf{y}-\bar{\mathbf{y}}_{\mathcal{T}}\|_{2}^{2}\\
= &
\sum_{y\in\mathcal{T}}\sum_{i=0}^{m-1}\,
\left(y_i-\bar{y_i}_{\mathcal{T}}\right)^{2}
\end{align*}

Following the same reasoning as in section~\ref{sec:numerical},
we get for a simpler cost:
$$
c\left(\mathcal{L},\mathcal{R}\right) =
-\left[
\frac{
\sum_{i=0}^{m-1} \, \left(\sum_{\mathcal{L}}y_i\right)^{2}}
{\#\mathcal{L}}
\; + \;
\frac{\sum_{i=0}^{m-1} \,
\left(\sum_{\mathcal{R}}y_i\right)^{2}}{\#\mathcal{R}}\right]
$$

\section{Optimizing categorical splits}

Sorting by score:
\cite{breiman-friedman-olshen-stone-1984}
\cite{fisher-1958}
\cite{ripley-1996}

Approximations \cite{loh-vanichesetakul-1988}.

\newpage{}
\input{../tex/typesetting}
\input{../tex/tail}
\end{document}