% ----------------------------------------------------------------------
\lecture{Heuristic-driven solving}{hsolving}
% ----------------------------------------------------------------------
\part{Heuristic-driven solving}
\newcommand{\data}[2]{&#1&#2} % STRANGE PLACING but only way
% ----------------------------------------------------------------------
\section{Motivation}
% ----------------------------------------------------------------------
\input{hsolving/motivation}
% ----------------------------------------------------------------------
\input{hsolving/cdcl}
% ----------------------------------------------------------------------
% \begin{frame}{Inside \alert<1>{\textit{decide}}}
%   \bigskip
%   \begin{itemize}
%   \item<2-> \structure{Basic concepts}
%     \begin{itemize}
%     \item Atoms, $\mathcal{A}$
%     \item Assignments, $\ass: \mathcal{A}\to\{\true,\false\}$
%       \begin{itemize}
%       \item[]
%         $\tlits{\ass}=\{a\in\mathcal{A}\mid\Tsigned{a}\in\ass\}$ \ and \
%         $\flits{\ass}=\,\{a\in\mathcal{A}\mid\Fsigned{a}\in\ass\}$
%       \end{itemize}
%     \end{itemize}
%   \item<3-> \structure{Heuristic functions}
%     \[
%     h: \mathcal{A}\to[0,+\infty)
%     \quad\text{and}\quad
%     s: \mathcal{A}\to\{\true,\false\}
%     \]
%   \item<4-> \structure{Algorithmic scheme}
%   \medskip
%     \begin{enumerate}
%     \item $h(a) := \alpha\times h(a) + \beta(a)$ \hfill for each $a\in\mathcal{A}\qquad$
%     \item $U:=\mathcal{A}\setminus (\tlits{\ass}\cup\flits{\ass})$
%     \item $C:= \textit{argmax}_{a\in U}h(a)$
%     \item $a:= \tau(C)$
%     \item $\ass := \ass\cup\{a\mapsto s(a)\}$
%     \end{enumerate}
%   \end{itemize}
% \end{frame}
% ----------------------------------------------------------------------
\section{Heuristically modified ASP}
% ----------------------------------------------------------------------
\input{hsolving/hlanguage}
% ----------------------------------------------------------------------
\input{hsolving/hstrips}
% ----------------------------------------------------------------------
\input{hsolving/hoptions}
% ----------------------------------------------------------------------
\input{hsolving/hminimality}
% ----------------------------------------------------------------------
% \begin{frame}{Heuristic~modifications~to~functions~$h$~and~$s$}
%   \medskip
%   \begin{itemize}
%   \item<2-> $\nu_{a,m}(\ass)$ --- ``{\em value for modifier $m$ on atom $a$ wrt assignment \ass}''
%     \medskip
%   \item<3-> \structure{\texttt{init}}\uncover<3->{ and \structure{\texttt{factor}}}
%     \begin{align*}
%       d_0(a)&=\quad\;\nu_{a,\mathtt{init}}(\ass_0)+h_0(a)
%       % \text{ and for} i\geq 1
%       \\
%       \uncover<3->{d_i(a)}&\uncover<3->{=
%         \left\{
%           \begin{array}{rl}
%             \nu_{a,\mathtt{factor}}(\ass_i)\times h_i(a)&\text{if } V_{a,\mathtt{factor}}(\ass_i)\neq\emptyset
%             \\
%             h_i(a)&\text{otherwise}
%           \end{array}
%         \right.}
%     \end{align*}
%   \item<3-> \structure{\texttt{sign}}
%     \begin{align*}
%       t_i(a)&=
%       \left\{
%         \begin{array}{rl}
%           \true &\text{if }
%           \nu_{a,\mathtt{sign}}(\ass_i)>0
%           \\
%           \false&\text{if }
%           \nu_{a,\mathtt{sign}}(\ass_i)<0
%           \\
%           s_i(a)&\text{otherwise}
%         \end{array}
%       \right.\qquad\qquad\qquad\quad\;
%     \end{align*}
%   \item<3-> \structure{\texttt{level}} \quad
%     \(
%     \ell_{\ass_i}(\mathcal{A}')=\textit{argmax}_{a\in\mathcal{A}'}\nu_{a,\mathtt{level}}(\ass_i)
%     \)
%     \qquad
%     $\mathcal{A}'\subseteq\mathcal{A}$
%   \end{itemize}
% \end{frame}
% % ----------------------------------------------------------------------
% \begin{frame}[c]{Inside \alert<1>{\textit{decide}}, {heuristically modified}}
%   \pause
%   \begin{enumerate}\addtocounter{enumi}{-1}
%   \item $\alert<3>{h(a) := d(a)}$                         \hfill for each $a\in\mathcal{A}\qquad$
%   \item $h(a) := \alpha\times h(a) + \beta(a)$ \hfill for each $a\in\mathcal{A}\qquad$
%   \item $U:=\alert<3>{\ell_{\ass}}(\mathcal{A}\setminus (\tlits{\ass}\cup\flits{\ass}))$
%   \item $C:= \textit{argmax}_{a\in U}\alert<3>{d}(a)$
%   \item $a:= \tau(C)$
%   \item $\ass := \ass\cup\{a\mapsto\alert<3>{t}(a)\}$
%   \end{enumerate}
% \end{frame}
% ----------------------------------------------------------------------
\section{Experimental results}
% ----------------------------------------------------------------------
\begin{frame}[c]{Abductive problems with optimization}
\small
\begin{center}
  \begin{tabular}{|@{\;}l@{\;}|r@{s\,}@{\,(}r@{)\,}|r@{s\,}@{\,(}r@{)\,}|r@{s\,}@{\,(}r@{)\ }r@{s\,}@{\,(}r@{)\,}|}
    \hline
    \multicolumn{1}{|c|}{Setting} & \multicolumn{2}{c|}{\textit{Diagnosis}} & \multicolumn{2}{c|}{\textit{Expansion}} & \multicolumn{2}{c}{\textit{Repair (H)}} & \multicolumn{2}{c|}{\textit{Repair (S)}}\\
    \hline
         \textit{base configuration}            \data{         111.1 }{         115 }\data{        161.5 }{        100 }\data{        101.3 }{        113 }\data{         33.3 }{         27 }\\
    \hline
         \texttt{sign,-1}                       \data{         324.5 }{         407 }\data{          7.6 }{          3 }\data{          8.4 }{          5 }\data{          3.1 }{\alert<2>{0}}\\
         \texttt{sign,-1}  \, \texttt{factor,2} \data{         310.1 }{         387 }\data{          7.4 }{\alert<2>{2}}\data{          3.5 }{\alert<2>{0}}\data{          3.2 }{          1 }\\
         \texttt{sign,-1}  \, \texttt{factor,8} \data{         305.9 }{         376 }\data{          7.7 }{\alert<2>{2}}\data{          3.1 }{\alert<2>{0}}\data{          2.9 }{\alert<2>{0}}\\
         \texttt{sign,-1}  \, \texttt{level,1}  \data{\alert<2>{76.1}}{\alert<2>{83}}\data{\alert<2>{6.6}}{\alert<2>{2}}\data{\alert<2>{0.8}}{\alert<2>{0}}\data{          2.2 }{          1 }\\
\phantom{\texttt{sign,-1}} \, \texttt{level,1}  \data{          77.3 }{          86 }\data{         12.9 }{          5 }\data{          3.4 }{\alert<2>{0}}\data{\alert<2>{2.1}}{\alert<2>{0}}\\
    \hline
  \end{tabular}
\end{center}
\bigskip
\ (abducibles subject to optimization via \texttt{\#minimize} statements)
\end{frame}
% ----------------------------------------------------------------------
\begin{frame}[c,fragile]{Planning  benchmarks}
%
\begin{semiverbatim}
#heuristic holds(F,T-1) :     holds(F,T). [t-T+1, true]
#heuristic holds(F,T-1) : not holds(F,T), fluent(F),time(T).
                                          [t-T+1, false]
\end{semiverbatim}
\pause
\smallskip
\begin{center}\scriptsize
  \newcommand{\pdata}[3]{&\,\ignorespaces#1$s$&(\ignorespaces#2/\ignorespaces#3)}
  \newcommand{\sdata}[3]{&\,\ignorespaces#1$s$&(\ignorespaces#3)}
  \begin{tabular}{|@{\,}r@{\,}|@{\,}r@{\,}@{\,}r@{\,}|@{\,}r@{\,}@{\,}r@{\,}|@{\,}r@{\,}@{\,}r@{\,}|@{\,}r@{\,}@{\,}r@{\,}|}
    \hline
   \multicolumn{1}{|@{\,}c@{\,}|@{\,}}{Problem} & \multicolumn{2}{c@{\,}|@{\,}}{\textit{base configuration}} &
   \multicolumn{2}{c@{\,}|@{\,}}{\texttt{\#heuristic}} & \multicolumn{2}{c@{\,}|@{\,}}{\textit{base config.} (\alert<3>{SAT})} & \multicolumn{2}{c@{\,}|}{\texttt{\#heu.} (\alert<3>{SAT})} \\
    \hline
    \textit{Blocks'00}    \pdata{134.4}{ 180}{  61}\pdata{  9.2}{ 239}{  3}\sdata{163.2}{180}{59}\sdata{ 2.6}{239}{0}\\
    \textit{Elevator'00}  \pdata{  3.1}{ 279}{   0}\pdata{  0.0}{ 279}{  0}\sdata{  3.4}{279}{ 0}\sdata{ 0.0}{279}{0}\\
    \textit{Freecell'00}  \pdata{288.7}{ 147}{ 115}\pdata{184.2}{ 194}{ 74}\sdata{226.4}{147}{47}\sdata{52.0}{194}{0}\\
    \textit{Logistics'00} \pdata{145.8}{ 148}{  61}\pdata{115.3}{ 168}{ 52}\sdata{113.9}{148}{23}\sdata{15.5}{168}{3}\\
    \hline
    \textit{Depots'02}    \pdata{400.3}{  51}{ 184}\pdata{297.4}{ 115}{135}\sdata{389.0}{ 51}{64}\sdata{61.6}{115}{0}\\
    \textit{Driverlog'02} \pdata{308.3}{ 108}{ 143}\pdata{189.6}{ 169}{ 92}\sdata{245.8}{108}{61}\sdata{ 6.1}{169}{0}\\
    \textit{Rovers'02}    \pdata{245.8}{ 138}{ 112}\pdata{165.7}{ 179}{ 79}\sdata{162.9}{138}{41}\sdata{ 5.7}{179}{0}\\
    \textit{Satellite'02} \pdata{398.4}{  73}{ 186}\pdata{229.9}{ 155}{106}\sdata{364.6}{ 73}{82}\sdata{30.8}{155}{0}\\
    \textit{Zenotravel'02}\pdata{350.7}{ 101}{ 169}\pdata{239.0}{ 154}{116}\sdata{224.5}{101}{53}\sdata{ 6.3}{154}{0}\\
    \hline
    \textit{Total}        \pdata{252.8}{\alert<3>{1225}}{1031}\pdata{158.9}{\alert<3>{1652}}{657}\sdata{\alert<3>{187.2}}{1225}{\alert<3>{430}}\sdata{\alert<3>{17.1}}{1652}{\alert<3>{3}}\\
    \hline
  \end{tabular}
\end{center}
\bigskip
\end{frame}
%------------------------------------------------------------
%
%%% Local Variables:
%%% mode: latex
%%% TeX-master: "asp"
%%% End:


\section{Scheduling: Non-Preemptive, Non-Size-Based}
\label{sec:Scheduling-Non-Preemptive-Non-Size-Based}

The most common non-preemptive non-size-based scheduling policies are

\begin{description}

	\item [Random Order (RO)] the server chooses the job in queue at random.

	\item [First-Come First-Served (FCFS)] the server chooses the job at the head of the queue.

	\item [Last-Come First-Served (LCFS)] the server chooses the job at the tail of the queue.

\end{description}

\begin{theorem}
\label{thm:Scheduling--Non-Preemptive-Non-Size-Based}
All non-preemptive service orders that do not make use of job sizes have the same $\expected{N}$ and the same $\expected{T}$.
\end{theorem}

In particular, we have that

\begin{equation}
\expected{N^{\mathit{FCFS}}}=\expected{N^{\mathit{LCFS}}}=\expected{N^{\mathit{RO}}}
\end{equation}

\begin{equation}
\expected{T^{\mathit{FCFS}}}=\expected{T^{\mathit{LCFS}}}=\expected{T^{\mathit{RO}}}
\end{equation}

\begin{equation}
\variance{T^{\mathit{FCFS}}}<\variance{T^{\mathit{RO}}}=\variance{T^{\mathit{LCFS}}}
\end{equation}

The latter holds because LCFS can produce high response time, since the first come job has to wait for the queue to be empty.

Thus, all non-preemptive non-size-based scheduling policies for $M/G/1$ have the same $\expected{T}$ as $M/M/1$, namely

\begin{equation}
\label{eqn:Non-Preemptive-Non-Size-Based-Time}
\expected{T}=\expected{S}+\frac{\lambda\expected{S^{2}}}{2(1-\varrho)}
\end{equation}

The problem is that under high-variability job sizes the mean response time gets very high.
In particular, the mean slowdown is high for small sized jobs.


\chapter{Additional calculations}\label{chap:AppB}
For the sake of completeness, we present some auxiliary calculations and important steps, that were used to obtain the results presented in the scope of this work, but were in general too long or unsuitable to be included in the main part.
\subsection*{Fermion part}
In the part on fermions, we are confronted with the Dirac operator $\slashed{\nabla}=\gamma^{\mu}\nabla_{\mu}$, that involves a contraction with gamma matrices. Here, we present the proof of an identity we used in the derivation of the fermion two-point function:
\begin{equation}
	\begin{aligned}
		\slashed{\nabla}^2 &= \nabla_{\mu}\nabla_{\nu}\gamma^{\mu}\gamma^{\nu} \\
		&= \frac{1}{2}\nabla_{\mu}\nabla_{\nu}\left(\gamma^{\mu}\gamma^{\nu}+\gamma^{\nu}\gamma^{\mu}\right) \\
		&=  g^{\mu\nu}\nabla_{\mu}\nabla_{\nu}\\
		&= \nabla^2
	\end{aligned}
\end{equation}
The third line follows from the definition of the Clifford algebra $\left\{\gamma^{\mu},\gamma^{\nu}\right\} = 2g^{\mu\nu}\mathbbm{1}$. \\

We solved the functional trace for the fermions once again using the heat-kernel techniques introduced in appendix \ref{chap:AppA}, but in contrast to the other calculations, we could not use the rather simple expressions for the threshold functions evaluated for the Litim cutoff (\ref{eqn:Litim}). We expressed the fermion regulator in terms of the scalar regulator which means, that the $Q$-functionals looked different and therefore we had to solve these \enquote{new} integrals analytically. We start by simplifying the trace expression as much as possible:
\begin{equation}
\begin{aligned}
	-\tr{G_{k, \bar{\psi}\psi}\ \partial_t R_{k,D}} &=-\tr{\frac{Z_D\cdot\bar{\Delta}_{\left(\sfrac{1}{2}\right)}\cdot\left(\frac{\partial_t r_{k,S}}{2\sqrt{1+r_{k, S}}} - \eta_D\left(\sqrt{1+r_{k,S}} -1\right)\right)}{Z_D\cdot\bar{\Delta}_{\left(\sfrac{1}{2}\right)}\sqrt{1 + r_{k,S}}}\mathbbm{1}_D} \\[10pt]
	&= -N_D\tr{\frac{\theta\left(1-\chi\right)}{\chi\left(r_{k,S}(\chi)+1\right)} + \eta_D\left(\frac{1}{\sqrt{r_{k,S}(\chi) + 1}}-1\right)},
	\end{aligned}
\end{equation}
We substituted $\chi := \sfrac{\bar{\Delta}_{\left(\sfrac{1}{2}\right)}}{k^2}$. Since the trace is linear, we can separate both terms and compute them independently.
\paragraph{First term:}
\begin{align}
		-N_D\tr{\frac{\theta(1-\chi)}{\chi\left(1+r_{k,S}(\chi)\right)}} \overset{(\mathrm{\ref{eqn:master-eqn}})}{=} -N_D\frac{1}{(4\pi)^2}\int_x\sqrt{\bar{g}}&\left[\operatorname{Tr}\mathbf{b}_0\int_0^{\infty}\dd\chi\ \chi\frac{\theta(1-\chi)}{\chi\left(1+r_{k,S}(\chi)\right)}\right.\nonumber \\
		\phantom{.}\\
 &+ \left.\operatorname{Tr}\mathbf{b}_2\int_0^{\infty}\dd\chi\ \frac{\theta(1-\chi)}{\chi\left(1+r_{k,S}(\chi)\right)}\right].\nonumber
\end{align}
We directly set $\Gamma(n)=1$, since $n$ equals $1$ or $\ 2$ in both cases. With the definition of the Heaviside function, we can set the upper integration bound for the $\dd\chi$ integration to $1$. We use the heat-kernel coefficients for the spin-$\frac{1}{2}$ Laplacian given in (\ref{eqn:fermion_coefficients}) and the definition of the Litim shape function (\ref{eqn:Litim}) and obtain:
\begin{equation}
\begin{aligned}
		-N_D\tr{\frac{\theta(1-\chi)}{\chi\left(1+r_{k,S}(\chi)\right)}} &= -N_D\frac{1}{(4\pi)^2}\int_x\sqrt{\bar{g}}\left[4\int_0^1\dd\chi \ \chi + \frac{5}{12}\bar{\Ricci}\int_0^1\dd\chi\  1\right]\\[10pt]
&= -N_D\frac{1}{(4\pi)^2}\int_x\sqrt{\bar{g}}\left[2 + \frac{5}{12}\bar{\Ricci}\right].
\end{aligned}
\end{equation}
\paragraph{Second term:}
\begin{equation}
\begin{aligned}
	-N_D\tr{\eta_D\left(\frac{1}{\sqrt{1+r_{k,S}(\chi)}}-1\right)} &= -N_D\frac{1}{(4\pi)^2} \int_x\sqrt{\bar{g}}\left[4\eta_D\int_0^1\dd\chi \ \chi(\chi-1)\right. \\[10pt]
 &\left. \qquad\qquad\qquad\qquad\quad+ \frac{5\eta_D}{12}\bar{\Ricci}\int_0^1\dd\chi\left(\chi-1\right)\right]\\[10pt]
	&= -N_D\frac{1}{(4\pi)^2} \int_x\sqrt{\bar{g}} \left[\eta_D\left(-\frac{2}{3} - \frac{5}{24}\bar{\Ricci}\right)\right].
\end{aligned}
\end{equation}
Altogether we find the following result for the functional trace of the fermion contribution:
\begin{equation}
	-\operatorname{Tr}\left[G_{k, \bar{\psi} \psi} \partial_{t} R_{k, D}\right]= -N_D\frac{1}{(4\pi)^2}\int_x\sqrt{\bar{g}}\left[2\left(1-\frac{\eta_D}{3}\right) + \frac{5}{12}\bar{\Ricci}\left(1-\frac{\eta_D}{2}\right)\right].
	\label{eqn:trace_fermions}
\end{equation}
\vfill
\subsection*{Gauge field part}
During the computation of the gauge field contribution to the running of $G$ and $\Lambda$, we encounter the following term, which can be simplified a lot after a few manipulations:
\begin{align}\label{eqn:FF2}
\int g^{\mu\nu}g^{\kappa\lambda} F_{\mu\kappa}F_{\nu\lambda} = \int F_{\mu}^{\phantom{\mu}\lambda}F_{\phantom{\mu}\lambda}^{\mu}	&= \int  F_{\mu\lambda}F^{\mu\lambda} \nonumber	\\
&= \int \left(\partial_{\mu}A_{\lambda} - \partial_{\lambda}A_{\mu}\right)F^{\mu\lambda} + \mathcal{O}\left(A^3\right)\nonumber \\
&\overset{(\star)}{=} \int 2\partial_{\mu}A_{\lambda} F^{\mu\lambda}\nonumber  \\
&=  \int 2\partial_{\mu}A_{\lambda}\left(\partial^{\mu}A^{\lambda} - \partial^{\lambda}A^{\mu}\right) \\
&= \int 2\left(\partial_{\mu}A_{\lambda}\partial^{\mu}A^{\lambda} - \partial_{\mu}A_{\lambda}\partial^{\lambda}A^{\mu}\right) \nonumber\\
&\overset{(\dagger)}{=} -\int 2\left( A_{\lambda}\partial^2A^{\lambda} - A_{\lambda}\partial_{\mu}\partial^{\lambda}A^{\mu}\right) \nonumber\\
&= \int 2A_{\lambda}\left[\partial^{\mu}\partial^{\lambda} - g^{\mu\lambda}\partial^2\right]A_{\mu}\nonumber
\end{align}
For the first non-trivial step ($\star$) we use that $2\partial_{\mu}A_{\lambda} = \partial_{(\mu}A_{\lambda)} + \partial_{[\mu}A_{\lambda]}$, where $(\cdots)$ and $[\cdots]$ denote symmetrization and antisymmetrization w.\,r.\,t. the respective indices. The symmetric part vanishes due to the fact, that $F^{\mu\lambda}$ is antisymmetric under $\mu \rightleftharpoons \lambda$. This allows us to write $2\partial_{\mu}A_{\lambda}F^{\mu\lambda} = \partial_{[\mu}A_{\lambda]}F^{\mu\lambda} = \left(\partial_{\mu}A_{\lambda} - \partial_{\lambda}A_{\mu}\right)F^{\mu\lambda}$. The  second non-trivial step ($\dagger$) results from integrating by parts and assuming vanishing boundary terms. \\

Later on in the gauge field calculation, after specifying the gauge parameter $\xi=1$, we encounter a commutator of covariant derivatives acting on $A_{\mu}$. With the definition of the curvature tensor, given in equation (\ref{eqn:Riemann}), we find
\begin{equation}
\begin{aligned}
\left[\nabla^{\mu}, \nabla^{\lambda}\right]A_{\mu} &= R_{\mu}^{\phantom{\mu}\rho\mu\lambda}A_{\rho} \\
&= R^{\rho\lambda}A_{\rho} \label{eqn:RiemannB}
\end{aligned}
\end{equation}
Now, one simply has to rename the dummy indices $\rho \rightleftharpoons \mu$ to find the wanted expression. We encounter the same term also in the computation of the Faddeev-Popov ghosts for the graviton sector.


\documentclass[10pt]{article}

\usepackage{enumerate}
\usepackage{amsmath,amsthm,amssymb,mathtools}
\usepackage{graphicx}
\usepackage{mycommands}
\usepackage{hyperref}

\newcommand{\eV}{\mtx{eV}}
\newcommand{\nm}{\mtx{nm}}
\newcommand{\varep}{\varepsilon}

\setlength{\oddsidemargin}{-0.25in}
\setlength{\evensidemargin}{-0.25in}
\setlength{\topmargin}{-0.25in}
\setlength{\headheight}{-0.25in}
\setlength{\headsep}{0in}
\setlength{\textwidth}{7.0in}
\setlength{\textheight}{9.5in}
\setlength{\parindent}{.25in}
\setlength{\parskip}{0in}

\begin{document}

\section{Doing MO excitation for a hamiltonian in a localized basis. }

I first came across this when trying to form singles excitations for a embedded hamiltonian for AFQMC.
I figured I'd write down the notes so I wouldn't need to rederive this again.

\subsection{Definitions}

The elements of a localized basis will be $\ket{\alpha} = a^\dag_\alpha \ket{0}$.
The MO basis for a SCF calculation is $\ket{\phi_i} = c_i^\dag \ket{0}$.

\subsection{Occupying the LOs for a representation of the ground state SCF determinant}

The localized orbitals are defined in terms of occupied and unoccupied MOs, so the first $N_\mtx{elec}$ LO orbitals, $\ket{\alpha}$, are rotations of occupied MOs.
Therefore the SCF ground state is equivilent to a determinant that is constructed from occupying the occupied local orbitals.
This state would look something like:
\begin{align}
  \ket{\Phi^{(0)}}
  &\sim
  \begin{pmatrix}
    1 &   &        &  &  &   \\
      & 1 &        &  &  &   \\
      &   & \ddots &  &  &   \\
      &   &  & \ddots  &  &   \\
      &   &        &  &  & 1  \\
    0 & \cdots &  &  &  & 0 \\
    \vdots & \ddots & \ddots & \ddots & & \vdots \\
    0 & \cdots & & & & 0
  \end{pmatrix}
  =
  \Phi_{\alpha \alpha'}
  =
  \braket{\alpha}{\alpha'}
\end{align}
The shape of the matrix is $N_\mtx{basis} \times N_\mtx{occ}$.
There's one like that for each spin.
The python command is \\\verb|np.concatenate( (np.eye(maxN),np.zeros((nBasis-maxN,maxN))) )|.
In this case the ``MOs" \textbf{are} the LOs.

This construction of the ground state is more convenient than rotating the SCF ground state into a localized basis, because when freezing the localized basis by location, many occupied MOs will overlap with a single LO.
You know which LO are active, but each MO has some overlap with an occupied LO.
When you freeze the system, you can truncate the hamiltonian, but it is difficult to know which rows or columns to remove from a SCF determinent in the MO basis.

\subsection{Performing an excitation on a different basis}

Essentially a HOMO-LUMO excitation would be something like:
\begin{align}
  \ket{\Phi^{(1)}}
  &=
  c_{i+1}^\dag c_i \ket{\Phi^{(0)}}
\end{align}
Changing basis from the MOs to the LOs can be found:
\begin{align}
  c_i^\dag \ket{0}
  &=
  \ket{\phi_i}
  =
  \sum_\alpha
  \ket{\alpha} \braket{\alpha}{\phi_i}
  =
  \sum_\alpha
  a_\alpha^\dag \ket{0} \braket{\alpha}{\phi_i}
  \\
  &=
  \lt(
    \sum_\alpha
    a_\alpha^\dag \braket{\alpha}{\phi_i}
  \rt)
  \ket{0}
\end{align}
So:
\begin{align}
  & c_i^\dag = \sum_\alpha a_\alpha^\dag \braket{\alpha}{\phi_i} &
  & c_i = \sum_\alpha \braket{\phi_i}{\alpha} a_\alpha &
  \\
\end{align}
Then:
\begin{align}
  c_{i+1}^\dag c_i
  \ket{\Phi^{(0)}}
  &=
  \sum_{\alpha \alpha'}
  \braket{\alpha}{\phi_{i+1}}
  \braket{\phi_i}{\alpha'}
  \lt(
    a_\alpha^\dag a_{\alpha'} \ket{\Phi^{(0)}}
  \rt)
\end{align}
Then $ a_\alpha^\dag a_{\alpha'} \ket{\Phi^{(0)}} $ represents a determinant where the diagonal of the $\alpha'$ column is zeroed and row $\alpha$ is 1.
Columns $\alpha' > N_\mtx{occ}$ are not affected, which is good because they don't exist.

The values of the off-diagonal are easily $
  \braket{\alpha}{\phi_{i+1}}
  \braket{\phi_i}{\alpha'}
$, because these are the only contributions to it.
So:
\begin{align}
  \label{eq:offdiag}
  \Phi^{(1)}_{\alpha \alpha'}
  &=
  \braket{\alpha}{\phi_{i+1}}
  \braket{\phi_i}{\alpha'}
  \qquad
  :\alpha \ne \alpha'
\end{align}
The values on the diagonal are harder to compute directly, because they get some weight for every $
    a_\alpha^\dag a_{\alpha'} \ket{\Phi^{(0)}}
$ that don't touch that column.
Probably the easiest way to get the diagonal is to enforce normalization of the columns, which represents the normalization of the orbitals that are occupied:
\begin{align}
  \label{eq:diag}
  \Phi_{\alpha \alpha}
  &=
  1
  -
  \sum_{\alpha' \ne \alpha}
  \Phi_{\alpha' \alpha}^2
\end{align}
Eqns. (\ref{eq:offdiag}) and (\ref{eq:diag}) are sufficient to compute the singles excitation in the LO basis.

\subsection{Checking that the excitation can be done safely inside an active region}

The above works out well only if the active space LOs can represent the ground and excited state.
One simple check for this is to check the normalization in the active space for both orbitals.
\begin{align}
  &
  \sum_{\alpha \in \mca{A}}
  \braket{\alpha}{\phi_i}^2
  = 1
  &
  &
  \sum_{\alpha \in \mca A}
  \braket{\alpha}{\phi_{i+1}}^2
  = 1,
  &
\end{align}
where $\alpha \in \mca{A}$ means LOs only in the active space.
If this doesn't hold, then the excitation requires orbitals that are frozen, and hence cannot be expressed correctly within the active space you've chosen.

\end{document}


\documentclass[english,11pt]{article}
\usepackage[T1]{fontenc}
\usepackage[latin9]{inputenc}
\usepackage{geometry}
\usepackage{graphicx}
\usepackage{float}
\usepackage{subcaption} % to plot subfigure
\usepackage[linesnumbered,ruled,vlined]{algorithm2e}
\geometry{verbose,tmargin=2.5cm,bmargin=2.5cm,lmargin=2.5cm,rmargin=2.5cm}
\usepackage{amsmath}
\usepackage{babel}
\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%% part 0 %%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\part*{Bonus question:}
\subsection*{ 4.1 Regularized Linear Regression}
Figure~\ref{fig:4_1}  plots the variation in training/testing error with $\lambda$ for regularized linear model.
\begin{itemize}
\item Best $\lambda=10$:
\item training error: 11.605155
\item test error: 12.6566689644
\end{itemize}

\begin{figure}[ht]
\centering
\includegraphics[width=.5\textwidth]{../hw1/part2/fig4_linear_lambda_10.png}
\caption{Variation in training/testing error with $\lambda$ for regularized linear model}
\label{fig:4_1}
\end{figure}

\subsection*{ 4.2 Selecting $\lambda$ with quadratic features}
Figure~\ref{fig:4_2} plots the variation in training/testing error with $\lambda$ for regularized linear model with quadratic features.
\begin{itemize}
\item Best $\lambda=0.3$:
\item training error:  4.256794
\item test error: 4.82815820863
\end{itemize}

\begin{figure}[ht]
\centering
\includegraphics[width=.5\textwidth]{../hw1/part2/fig4_quadratic_lambda_03.png}
\caption{Variation in training/testing error with $\lambda$ for regularized linear model with quadratic features}
\label{fig:4_2}
\end{figure}

\subsection*{ 4.3 Selecting $\lambda$ with cubic features}
\begin{figure}[h]
\centering
\includegraphics[width=.5\textwidth]{../hw1/part2/fig4_cubic_lambda_3.png}
\caption{Variation in training/testing error with $\lambda$ for regularized linear model with quadratic features}
\label{fig:4_3}
\end{figure}

Figure~\ref{fig:4_3} plots the variation in training/testing error with $\lambda$ for regularized linear model with quadratic features.
\begin{itemize}
\item Best $\lambda=3$:
\item training error:  3.645624
\item test error: 4.73217608015
\end{itemize}

\subsection*{ Summary}
In above sections, we trained linear regression model with linear/quadratic/cubic features to predict Boston house price. We can observe that all three models we built embodied the power of regulation in that appropriate value of $\lambda$ gives the "sweet region" --- and even though the number of features grow dramatically as we square and even cube the features for more variation, the regulation term in the loss function successfully constrains the model from overfitting. The fact that models with linear, quadratic, cubic features are successively one better than the other shows that: given more features (variations) and proper control of overfitting, the more delicate model has better performance in prediction.

\end{document}


\documentclass{beamer}
\usetheme{CambridgeUS}
\title[Process]{The Koopman operator identification algorithm}
\subtitle{So far}
\institute[Polimi]{Politecnico di Milano}
\author{Sergio Vanegas}
\date{\today}

\usepackage{listings}
\usepackage[framed,numbered,autolinebreaks,useliterate]{mcode/mcode}

\usepackage{xcolor}

\usepackage{siunitx}

\usepackage{caption}
\usepackage{subcaption}

\begin{document}

\begin{frame}[plain,noframenumbering]
    \maketitle
\end{frame}

\begin{frame}{Table of Contents}
    \tableofcontents
\end{frame}

\section{Robust Koopman}

\begin{frame}{Frobenius regularization - Formulation}
    We start by following the proposed reformulation of the minimization problem proposed in \textit{Robust tube-based model predictive control with Koopman operators} as follows:

    \begin{equation*}
        \overline{\mathcal{K}}_N =
        \min_{\mathcal{K} \in \mathbb{R}^{\tilde{N} \times N}}
        \sum_{k=1}^K \left[
            \left|\left|
                \mathbf{g}\left(\mathbf{y}\left[k\right]\right) -
                \mathcal{K}
                \begin{pmatrix}
                \mathbf{g}\left(\mathbf{x}\left[k\right]\right)
                \\
                \mathbf{u}\left[k\right]
                \end{pmatrix}
            \right|\right|^2
            {
                \color{red}
                + \alpha\left|\left|\mathcal{K}\right|\right|_F^2
            }
        \right],
    \end{equation*}

    where

    \begin{itemize}
        \item $\mathbf{g}$ is the observable vector,
        \item $K$ is the total number of snapshots,
        \item $\left(\mathbf{x},\mathbf{y}\right)$ are the snapshot pairs,
        \item $\mathbf{u}$ is the input vector,
        \item $\left|\left|\cdot\right|\right|_F$ is the element-wise 2-norm (Frobenius norm).
    \end{itemize}
\end{frame}

\begin{frame}[fragile]{Frobenius regularization - MATLAB implementation}
    In order to properly minimize the regularized formulation, an external Toolbox that exploited its convexity had to be integrated. In our case, CVX was used.

    \begin{lstlisting}[language=Matlab,basicstyle=\tiny]
function [A,B] = Koopman(X,Y,U,alpha)
    % Following "Practical Considerations" from Korda-Mezic + Williams

    A = zeros(size(Y,1),size(X,1));
    B = zeros(size(Y,1),size(U,1));

    for i=1:size(Y,1)
        % In the real implementation, a skip is called when alpha=0
        cvx_begin
            variable m(length)
            minimize (norm(m*[X;U]-Y(i,:)) + alpha*norm(m))
        cvx_end

        A(i,:) = m(1:size(X,1));
        B(i,:) = m(size(X,1)+1:end);
    end
end
    \end{lstlisting}
\end{frame}

\begin{frame}{Frobenius regularization - Results I}
    The regularization technique was put to the test with both a \textit{clean} and a \textit{noisy} dataset, using a clean simulation as reference during the benchmarks.

    \begin{figure}
        \centering
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Clean_Training.png}
            \caption{Clean training dataset}
            \label{fig:Koopman_clean}
        \end{subfigure}
        \hfill
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Noisy_Training.png}
            \caption{Noisy training dataset}
            \label{fig:Koopman_noisy}
        \end{subfigure}
        \caption{Frobenius regularized Koopman operator}
        \label{fig:Koopman_Frobenius}
    \end{figure}
\end{frame}

\begin{frame}{Frobenius regularization - Results II}
    \begin{figure}
        \centering
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Clean_Eigen.png}
            \caption{Clean training dataset}
            \label{fig:Eigen_clean}
        \end{subfigure}
        \hfill
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Noisy_Eigen.png}
            \caption{Noisy training dataset}
            \label{fig:Eigen_noisy}
        \end{subfigure}
        \caption{Frobenius regularized eigenvalues}
        \label{fig:Eigen_Frobenius}
    \end{figure}
\end{frame}

\section{From prototyping to implementation}

\begin{frame}{Language alternatives - Benchmark formulation}
    Considering the performance limitations of a high-level interpreted languages, some alternatives had to be considered. The evaluated candidates were:

    \begin{itemize}
        \item Python: still high-level, but further optimized because of its open-source nature. Both an open-source clone of CVX (cvxpy) and a mat file utility are included in the scipy official package.
        \item C++: compiled language, and a classical choice for scientific programming. Even if complex, the development process is facilitated by the availability of the CPLEX API.
        \item Rust: memory-safe compiled language that pretends to replace C/C++ in the following years. High development complexity given it's in its early stages.
    \end{itemize}

    A simple benchmark based on the performance of ODE data generation (which evaluates both matrix and nonlinear implementations) was set up.
\end{frame}

\begin{frame}{Language alternatives - Benchmark results}

    \begin{itemize}
        \item MATLAB

        \begin{itemize}
            \item User time: $\SI{26.05}{\second}$
            \item Wall clock time: $\SI{28.62}{\second}$
            \item Maximum resident set size: $782348$ kb
        \end{itemize}

        \item Python

        \begin{itemize}
            \item User time: $\SI{25.44}{\second}$
            \item Wall clock time: $\SI{26.21}{\second}$
            \item Maximum resident set size: $59880$ kb
        \end{itemize}

        \item C++

        \begin{itemize}
            \item User time: $\SI{0.70}{\second}$
            \item Wall clock time: $\SI{1.76}{\second}$
            \item Maximum resident set size: $4320$ kb
        \end{itemize}

        \item Rust

        \begin{itemize}
            \item User time: $\SI{0.54}{\second}$
            \item Wall clock time: $\SI{6.44}{\second}$
            \item Maximum resident set size: $2736$ kb
        \end{itemize}
    \end{itemize}

\end{frame}

\begin{frame}{Language alternatives - Accuracy verification}
    \begin{figure}
        \centering
        \begin{subfigure}[b]{0.3\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Python_Sim.png}
            \caption{Python - Simulated trajectory}
            \label{fig:sim_python}
        \end{subfigure}
        \hfill
        \begin{subfigure}[b]{0.3\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Cpp_Sim.png}
            \caption{C++ - Simulated trajectory}
            \label{fig:sim_cpp}
        \end{subfigure}
        \hfill
        \begin{subfigure}[b]{0.3\textwidth}
            \centering
            \includegraphics[width=\textwidth]{Rust_Sim.png}
            \caption{Rust - Simulated trajectory}
            \label{fig:sim_rust}
        \end{subfigure}
        \caption{Simulated trajectories (MATLAB simulation as reference)}
        \label{fig:implementations}
    \end{figure}
\end{frame}

\section{Extended Dynamic Mode Decomposition}

\begin{frame}{EDMD - Theory I}
    We derive the procedure from Williams et Al.: having calculated the Koopman matrix projection, we find the Koopman eigenvalue/eigenvector pairs for the $A$ matrix s.t.

    \begin{equation*}
        A^T \mathbf{\xi}_j = \mu_j \mathbf{\xi}_j \quad , \quad j \in \left\{1,\dots,\tilde{N}\right\}
    \end{equation*}

    The reason why we find the eigenvectors of the transposed is because each row of the $A$ matrix is associated to a single observable, but in the Eigen procedure we need the opposite.

    Having obtained all the aforementioned pairs, we define an eigenfunction as the projection of an observable vector over each eigenvector, just as follows

    \begin{equation*}
        \phi_j\left(\mathbf{x}\right) = \mathbf{g}\left(\mathbf{x}\right)^T \mathbf{\xi}_j
    \end{equation*}
\end{frame}

\begin{frame}{EDMD - Theory II}
    Exploiting the linearity of the operator and the Eigenvalue theory, we express the Koopman operator as follows:

    \begin{align*}
        \mathbf(\phi_j)\left(\mathbf{y}\right)
        &=
        \mathbf{g}\left(\mathbf{y}\right)^T \mathbf{\xi}_j \\
        &=
        \left(\overline{\mathcal{K}}_N
        \begin{pmatrix}
            \mathbf{g}\left(\mathbf{x}\right)
            \\
            \mathbf{u}
        \end{pmatrix}\right)^T \mathbf{\xi}_j \\
        &=
        \begin{pmatrix}
            \mathbf{g}\left(\mathbf{x}\right)^T
            &
            \mathbf{u}^T
        \end{pmatrix}
        \left(
            \begin{pmatrix}
                A^T
                \\
                B^T
            \end{pmatrix}
            \mathbf{\xi}_j
        \right) \\
        &=
        \mu_j \phi_j\left(\mathbf{x}\right)
        +
        \mathbf{u}^T B^T \mathbf{\xi}_j \\
        &=
        \mu_j \phi_j\left(\mathbf{x}\right)
        +
        \mathbf{\xi}_j^T B \mathbf{u}
    \end{align*}

    The last equivalence is possible because, at that point, we are dealing with a scalar value.
\end{frame}

\begin{frame}{EDMD - Theory III}
    From this form, we can derive the full eigenspace decomposition by concatenating the $\phi$-projections vertically as follows

    \begin{equation*}
        \Phi\left(\mathbf{y}\right) = M \Phi\left(\mathbf{x}\right) + \Xi^T B \mathbf{u}
    \end{equation*}

    where $M$ is a diagonal matrix containing the eigenvalues ordered w.r.t. the horizontal concatenation of the vertical vectors $\xi_j$ in $\Xi$.

    At this point, it is easy to see that the original observables are recoverable at any point through the expression

    \begin{align*}
        \Phi\left(\mathbf{x}\right)^T &= \mathbf{g}\left(\mathbf{x}\right)^T \Xi \\
        \implies \left[\Xi^T\right]^{-1} \Phi\left(\mathbf{x}\right) &= \mathbf{g}\left(\mathbf{x}\right)
    \end{align*}
\end{frame}

\begin{frame}[fragile]{Implementation - Eigenpair extraction and discrimination}
    We start by extracting the eigenpairs with the following line: \mcode{[~,Mu,Xi] = eig(A);}. For numerical reasons, we request the left eigenvectors of the original matrix instead of the right eigenvectors of the transposed matrix.

    Then, we discriminate irrelevant eigenvectors by setting a magnitude threshold as follows:

    \begin{lstlisting}[language=Matlab]
thr = 0.1;
idx_r = abs(diag(Mu))>thr;
Xi_r = Xi(:,idx_r);
Mu_r = Mu(idx_r,idx_r);
    \end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Implementation - Projection tracking and observable reconstruction}
    Finally, we track the trajectory exclusively in the eigenspace with the filtered pairs, and then reconstruct the original observables (and from those, the states) as follows:

    \begin{lstlisting}[language=Matlab]
Phi = zeros(length(Mu_r),L);
Phi(:,1) = Xi_r'*g_p(:,1);

for i=1:L-1
    Phi(:,i+1) = Mu_r*Phi(:,i) + Xi_r'*B*u(:,i);
end

g_eig = Xi_r'\Phi;
z_eig = C*g_eig;
    \end{lstlisting}
\end{frame}

\begin{frame}{EDMD decomposition - Results}
    \begin{figure}
        \centering
        \begin{subfigure}[b]{0.3\textwidth}
            \centering
            \includegraphics[width=\textwidth]{EDMD_Ref.png}
            \caption{Eigenvalues - M=300}
            \label{fig:EDMD_Ref}
        \end{subfigure}
        \begin{subfigure}[b]{0.3\textwidth}
            \centering
            \includegraphics[width=\textwidth]{EDMD_00.png}
            \caption{EDMD - Unfiltered}
            \label{fig:EDMD_00}
        \end{subfigure}
        \\
        \begin{subfigure}[b]{0.3\textwidth}
            \centering
            \includegraphics[width=\textwidth]{EDMD_50.png}
            \caption{EDMD - Thr = 0.5}
            \label{fig:EDMD_50}
        \end{subfigure}
        \begin{subfigure}[b]{0.3\textwidth}
            \centering
            \includegraphics[width=\textwidth]{EDMD_70.png}
            \caption{EDMD - Thr = 0.70}
            \label{fig:EDMD_70}
        \end{subfigure}
        \label{fig:EDMD}
   \end{figure}
\end{frame}

\section{Conclusions}

\begin{frame}[allowframebreaks]{Conclusions}
    \begin{itemize}
        \item The Frobenius regularization does not visibly blind the identification algorithm from relatively low measurement noise; nevertheless, it does make it more resistant to instabilities, which we can easily verify by observing the eigenvalues moving towards the origin when increasing the value of $\alpha$.
        \item For larger datasets/higher dimension systems, MATLAB is no longer a viable framework to calculate the matrix approximation of the Koopman operator. C++ stands as the preferred language given its standardization and low memory consumption; Python, however, already improves significantly the memory efficiency on its own (even though the execution time does not change to a measurable degree).
        \item Nevertheless, both MPC and eigenvalue decomposition can be applied to the operator in MATLAB once the projection has been calculated through a lower-level implementation. The data management method now has to be switched from a proprietary \textit{mat file} approach to a cross-compatible \textit{csv} approach.
        \item EDMD does present an opportunity to reduce computational load in real scenarios, since eigenvectors associated to numerically insignificant eigenvectors produced by measurement noise can be removed when handling a high-dimensional observable/state space. Integration with the regularized method should provide a solid identification framework, although a more realistic scenario is required in order to verify that hypothesis.
    \end{itemize}
\end{frame}

\end{document}


\documentclass{report}

\usepackage{hyperref}
\usepackage{fancyhdr}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{enumerate}

% start meta data
\title{Python Machine Learning\\ Equation Reference}
\author{Sebastian Raschka \\ \texttt{mail@sebastianraschka.com}}
\date{05\slash 04\slash 2015 \\ (last updated: 06\slash 15\slash 2016) \\ \href{https://github.com/rasbt/python-machine-learning-book}{https://github.com/rasbt/python-machine-learning-book }}

% end meta data

% start header and footer
\pagestyle{fancy}
\lhead{Sebastian Raschka}
\rhead{Python Machine Learning -- Equation Reference -- Ch. \thechapter}
\cfoot{\thepage} % centered footer
\renewcommand{\headrulewidth}{0.4pt}
\renewcommand{\footrulewidth}{0.4pt}
\renewcommand{\chaptermark}[1]{%
{}}

% end header and footer

\begin{document} % start main document

\maketitle % makes the title page from meta data

%%%%%%%%%%%%%%%
% CHAPTER 1
%%%%%%%%%%%%%%%

\chapter{Giving Computers the Ability to Learn from Data}

\section{Building intelligent machines to transform data into knowledge}
\section{The three different types of machine learning}
\section{Making predictions about the future with supervised learning}
\subsection{Classification for predicting class labels}
\subsection{Regression for predicting continuous outcomes}
\section{Solving interactive problems with reinforcement learning}
\section{Discovering hidden structures with unsupervised learning}
\subsection{Finding subgroups with clustering}
\subsection{Dimensionality reduction for data compression}
\section{An introduction to the basic terminology and notations}

\newpage

The Iris dataset, consisting of 150 samples and 4 features, can then be written as a $150 \times 4$ matrix $\mathbf{X} \in \mathbb{R}^{150 \times 4}:$

\[
\begin{bmatrix}
    x_{1}^{(1)} & x_{2}^{(1)} & x_{3}^{(1)} & \dots  & x_{4}^{(1)} \\
    x_{1}^{(2)} & x_{2}^{(2)} & x_{3}^{(2)} & \dots  & x_{4}^{(2)} \\
    \vdots & \vdots & \vdots & \ddots & \vdots \\
    x_{1}^{(150)} & x_{2}^{(150)} & x_{3}^{(150)} & \dots  & x_{4}^{(150)}
\end{bmatrix}
\]

For the rest of this book, unless noted otherwise, we will use the superscript $(i)$ to refer to the $i$th training sample, and the subscript $j$ to refer to the $j$th dimension of the training dataset.

We use lower-case, bold-face letters to refer to vectors ($\mathbf{x} \in \mathbb{R}^{n \times 1}$) and upper-case, bold-face letters to refer to matrices, respectively ($\mathbf{X} \in \mathbb{R}^{n \times m}$), where $n$ refers to the number of rows, and $m$ refers to the number of columns, respectively. To refer to single elements in a vector or matrix, we write the letters in italics ($x^{(n)}) or x^{(n)}_{m}$, respectively). For example, $x^{150}_1$ refers to the refers to the first dimension of the flower sample 150, the sepal length. Thus, each row in this feature matrix represents one flower instance and can be written as four-dimensional row vector $\mathbf{x}^{(i)} \in \mathbb{R}^{1 \times 4}$

\[ \mathbf{x}^{(i)} = \bigg[x^{(i)}_1 \; x^{(i)}_2 \; x^{(i)}_3 \; x^{(i)}_4 \bigg]. \]

Each feature dimension is a 150-dimensional column vector $\mathbf{x}_{j} \in \mathbb{R}^{150 \times 1}$, for example

\[
\mathbf{x_j} = \begin{bmatrix}
    x_{j}^{(1)}  \\
    x_{j}^{(2)}  \\
    \vdots  \\
    x_{j}^{(150)}
\end{bmatrix}
.\]

Similarly, we store the target variables (here: class labels) as a 150-dimensional column vector

\[
\mathbf{y} = \begin{bmatrix}
    y^{(1)}  \\
    y^{(2)}  \\
    \vdots  \\
    y^{(150)}
\end{bmatrix}
, (y \in \{ \text{Setosa, Versicolor, Virginica \}}).\]

\newpage

\section{A roadmap for building machine learning systems}
\subsection{Preprocessing -- getting data into shape}
\subsection{Training and selecting a predictive model}
\subsection{Evaluating models and predicting unseen data instances}
\section{Using Python for machine learning}
\subsection{Installing Python packages}
\section{Summary}

%%%%%%%%%%%%%%%
% CHAPTER 2
%%%%%%%%%%%%%%%

\chapter{Training Machine Learning Algorithms for Classification}

\section{Artificial neurons -- a brief glimpse into the early history of machine learning}

We can then define an activation function $\phi(z)$ that takes a linear combination of certain
input values $\mathbf{x}$ and a corresponding weight vector $\mathbf{w}$ where $z$ is the so-called net
input ($z = w_1 x_1 + \dots + w_m x_m$):

\[
\mathbf{w} = \begin{bmatrix}
    w^{(1)}  \\
    w^{(2)}  \\
    \vdots  \\
    w^{(m)}
\end{bmatrix},
\mathbf{w} = \begin{bmatrix}
    x^{(1)}  \\
    x^{(2)}  \\
    \vdots  \\
    x^{(m)}
\end{bmatrix}.
\]

Now, if the activation of a particular sample $x^{(i)}$, that is, the output of $\phi(z)$, is greater than a defined threshold $\theta$, we predict class 1 and class -1, otherwise. In the perceptron algorithm, the activation function $\phi(\cdot)$ is a simple \textit{unit step function}, which is sometimes also called the \textit{Heaviside step function}:

\[ \phi(z) = \begin{cases}
      1 & \text{ if } z \ge \theta \\
      -1 & \text{ otherwise }.
   \end{cases}
\]

For simplicity, we can bring the threshold $\theta$ to the left side of the equation and define a weight-zero as $w_0 = -\theta$ and $x_0=1$, so that we write $\mathbf{z}$ in a more compact form

\[
z  = w_0 x_0 + w_1 x_1 + \dots + w_m x_m = \mathbf{w^T x}
\]

and

\[ \phi(z) = \begin{cases}
      1 & \text{ if } z \ge 0 \\
      -1 & \text{ otherwise }.
   \end{cases}
\]

In the following sections, we will often make use of basic notations from linear algebra. For example, we will abbreviate the sum of the products of the values in $\mathbf{x}$ and $\mathbf{w}$ using a \textit{vector dot product}, whereas superscript $T$ stands for \textit{transpose}, which is an operation that transforms a column vector into a row vector and vice versa:

\[
z  = w_0 x_0 + w_1 x_1 + \dots + w_m x_m = \mathbf{w^T x} = \sum_{j=0}^{m} \mathbf{w_j} \mathbf{x_j} = \mathbf{w}^T \mathbf{x}.
\]

For example:

\[
\big[1 \quad 2 \quad 3 \big] \times \begin{bmatrix}
    4  \\
    5  \\
    6
\end{bmatrix} = 1 \times 4 + 2 \times 5 + 3 \times 6 = 32.
\]

Furthermore, the transpose operation can also be applied to a matrix to
reflect it over its diagonal, for example:

\[
 \begin{bmatrix}
    1 & 2  \\
    3 & 4  \\
    5 & 6
\end{bmatrix}^T =  \begin{bmatrix}
    1 & 3 & 5 \\
    2 & 4 & 6
\end{bmatrix}
\]

Rosenblatt's initial perceptron rule is fairly simple and can be summarized by the following steps:

\begin{enumerate}
\item Initialize the weights to 0 or small random numbers.
\item For each training sample $\mathbf{x}^{(i)}$, perform the following steps:
\begin{enumerate}
\item Compute the output value $\hat{y}$.
\item Update the weights.
\end{enumerate}
\end{enumerate}

Here, the output value is the class label predicted by the unit step function that we de ned earlier, and the simultaneous update of each weight $w_j$ in the weight vector $\mathbf{w}$ can be more formally written as:

\[
w_j := w_j + \Delta w_j
\]

The value of $\Delta w_j$, which is used to update the weight $w_j$, is calculated by the perceptron rule:

\[
\Delta w_j = \eta \bigg( y^{(i)} - \hat{y}^{(i)} \bigg)x_{j}^{(i)}
\]

Where $\eta$ is the learning rate (a constant between 0.0 and 1.0), $y^{(i)}$ is the true class label of the $i$th training sample, and $\hat{y}^{(i)}$ is the predicted class label. It is important to note that all weights in the weight vector are being updated simultaneously, which means that we don't recompute $\hat{y}^{(i)}$ before all of the weights $\Delta w_j$ were updated. Concretely, for a 2D dataset, we would write the update as follows:

\[
\Delta w_0 = \eta \bigg(  y^{(i)} - \hat{y}^{(i)} \bigg)
\]

\[
\Delta w_1 = \eta \bigg(  y^{(i)} - \hat{y}^{(i)} \bigg) x_{1}^{(i)}
\]

\[
\Delta w_2 = \eta \bigg(  y^{(i)} - \hat{y}^{(i)} \bigg) x_{2}^{(i)}
\]

Before we implement the perceptron rule in Python, let us make a simple thought experiment to illustrate how beautifully simple this learning rule really is. In the two scenarios where the perceptron predicts the class label correctly, the weights remain unchanged:

\[
\Delta w_j = \eta \bigg( -1 -- 1 \bigg)x_{j}^{(i)} = 0
\]

\[
\Delta w_j = \eta \bigg( 1-1 \bigg)x_{j}^{(i)} = 0
\]

However, in the case of a wrong prediction, the weights are being pushed towards the direction of the positive or negative target class, respectively:

\[
\Delta w_j = \eta \bigg( 1 -- 1 \bigg)x_{j}^{(i)} = \eta(2)x_{j}^{(i)}
\]

\[
\Delta w_j = \eta \bigg( -1-1 \bigg)x_{j}^{(i)} = \eta(-2)x_{j}^{(i)}
\]

To get a better intuition for the multiplicative factor $x_{j}^{(i)}$, let us go through another
simple example, where:

\[
y^{(i)} = +1, \quad \hat{y}^{(i)} = -1, \quad \eta = 1
 \]

Let's assume that $x_{j}^{(i)}=0.5$ and we misclassify this sample as $-1$. In this case, we would increase the corresponding weight by $1$ so that the activation $x_{j}^{i} \times w_{j}^{(i)}$ will be more positive the next time we encounter this sample and thus will be more likely to be above the threshold of the unit step function to classify the sample as  $+1$:

\[
\Delta w_{j}^{(i)} = (1--1)0.5 = (2)0.5 = 1
\]

The weight update is proportional to the value of $x_{j}^{(i)}$. For example, if we have another sample $x_{j}^{(i)}=2$ that is incorrectly classified as $-1$, we'd push the decision boundary by an even larger extent to classify this sample correctly the next time:

\[
\Delta w_{j}^{(i)} = (1--1)2 = (2)2 = 4.
\]

\section{Implementing a perceptron learning algorithm in Python}

\subsection{Training a perceptron model on the Iris dataset}

\section{Adaptive linear neurons and the convergence of learning}

The key difference between the Adaline rule (also known as the Widrow-Hoff rule) and Rosenblatt's perceptron is that the weights are updated based on a linear activation function rather than a unit step function like in the perceptron. In Adaline, this linear activation function $\phi{z}$ is simply the identity function of the net input so that

\[
\phi \big( \mathbf{w}^T \mathbf{x} \big) = \mathbf{w}^T \mathbf{x}
\]

\subsection{Minimizing cost functions with gradient descent}

One of the key ingredients of supervised machine learning algorithms is to define an objective function that is to be optimized during the learning process. This objective function is often a cost function that we want to minimize. In the case of Adaline, we can define the cost function $J(\cdot)$ to learn the weights as the Sum of Squared Errors (SSE) between the calculated outcomes and the true class labels

\[
J(\mathbf{w}) = \frac{1}{2} \sum_i \bigg(y^{(i)}  - \phi \big(z^{(i)} \big) \bigg)^2.
\]

Using gradient descent, we can now update the weights by taking a step away from the gradient $\nabla J(\mathbf{w})$ of our cost function $J(\mathbf{\cdot})$:

\[
\mathbf{w} := \mathbf{w} + \Delta \mathbf{w}.
\]

To compute the gradient of the cost function, we need to compute the partial derivative of the cost function with respect to each weight $w_j$,

\[
\frac{\partial J}{\partial w_j} = - \sum_i \bigg( y^{(i)} - \phi \big(z^{(i)} \big) \bigg) x_{j}^{(i)},
\]

so that we can write the update of weight $w_j$ as

\[
\Delta w_j = - \eta \frac{\partial J}{\partial w_j} = \eta  \sum_i \bigg( y^{(i)} - \phi \big(z^{(i)} \big) \bigg) x_{j}^{(i)}
\]

Since we update all weights simultaneously, our Adaline learning rule becomes

\[
\mathbf{w} := \mathbf{w} + \Delta \mathbf{w}.
\]

For those who are familiar with calculus, the partial derivative of the SSE cost function with respect to the $j$th weight in can be obtained as follows:

\begin{equation}
\begin{split}
& \frac{\partial J}{\partial w_j} = \frac{\partial}{\partial w_j} \frac{1}{2} \sum_i \bigg(  y^{(i)} - \phi \big( z^{(i)} \big)  \bigg)^2 \\
& = \frac{1}{2} \frac{\partial}{\partial w_j} \sum_i \bigg(  y^{(i)} - \phi \big( z^{(i)} \big)  \bigg)^2 \\
& = \frac{1}{2} \sum_i 2 \bigg(  y^{(i)} - \phi \big( z^{(i)} \big)  \bigg) \frac{\partial J}{\partial w_j} \Bigg( y^{(i)} - \sum_i \bigg( w_{j}^{(i)} x_{j}^{(i)} \bigg)\Bigg) \\
& = \sum_i \bigg(  y^{(i)} - \phi \big( z^{(i)} \big)  \bigg) \bigg( - x_{j}^{(i)} \bigg) \\
& = - \sum_i \bigg(  y^{(i)} - \phi \big( z^{(i)} \big)  \bigg) x_{j}^{(i)}  \\
\end{split}
\end{equation}

Performing a matrix-vector multiplication is similar to calculating a vector dot product where each row in the matrix is treated as a single row vector. This vectorized approach represents a more compact notation and results in a more efficient computation using NumPy. For example:

\[
 \begin{bmatrix}
    1 & 2  & 3\\
    4 & 5  & 6
\end{bmatrix} \times  \begin{bmatrix}
    7 \\
    8 \\
    9
\end{bmatrix} =  \begin{bmatrix}
    1 \times 7 + 2 \times 8 + 3 \times 9 \\
    4 \times 7 + 5 \times 8 + 6 \times 9
\end{bmatrix} = \begin{bmatrix}
    50 \\
    122
\end{bmatrix}
\]

\subsection{Implementing an Adaptive Linear Neuron in Python}

Here, we will use a feature scaling method called standardization, which gives our data the property of a standard normal distribution. The mean of each feature
is centered at value 0 and the feature column has a standard deviation of 1. For example, to standardize the $j$th feature, we simply need to subtract the sample mean $\mu_j$ from every training sample and divide it by its standard deviation $\sigma_j$:

\[
\mathbf{x'}_j = \frac{\mathbf{x} - \mathbf{\mathbf{\mu_j}}}{\sigma_j}.
\]

Here $\mathbf{x}_j$ is a vector consisting of the $j$th feature values of all training samples $n$.

\subsection{Large scale machine learning and stochastic gradient descent}

A popular alternative to the batch gradient descent algorithm is stochastic gradient descent, sometimes also called iterative or on-line gradient descent. Instead of updating the weights based on the sum of the accumulated errors over all samples $\mathbf{x}^{(i)}$:

\[
\Delta \mathbf{w} = \eta \sum_i \bigg( y^{(i)} - \phi \big( z^{(i)}\big) \bigg) \mathbf{x}^{(i)}.
\]

We update the weights incrementally for each training sample:

\[
\Delta \mathbf{w} = \eta  \bigg( y^{(i)} - \phi \big( z^{(i)}\big) \bigg) \mathbf{x}^{(i)}.
\]

\section{Summary}

%%%%%%%%%%%%%%%
% CHAPTER 3
%%%%%%%%%%%%%%%

\chapter{A Tour of Machine Learning Classifiers Using Scikit-learn}

\section{Choosing a classification algorithm}
\section{First steps with scikit-learn}
\subsection{Training a perceptron via scikit-learn}
\section{Modeling class probabilities via logistic regression}
\subsection{Logistic regression intuition and conditional probabilities}

The odds ratio can be written as

\[
\frac{p}{(1-p)},
\]

where $p$ stands for the probability of the positive (1? p)
 event. The term positive event does not necessarily mean good, but refers to the event that we want to predict, for example, the probability that a patient has a certain disease; we can think of the positive event as class label $y =1$. We can then further define the logit function, which is simply the logarithm of the odds ratio (log-odds):

\[
logit(p) = log \frac{p}{1-p}
\]

The logit function takes input values in the range 0 to 1 and transforms them to values over the entire real number range, which we can use to express a linear relationship between feature values and the log-odds:

\[
logit ( p (y=1 | \mathbf{x})) = w_0 x_0 + w_1 x_1 + \cdots + x_m w_m = \sum^{m}_{i=0} w_i x_i = \mathbf{w}^T \mathbf{x}.
\]

Here, $p(y=1 | \mathbf{x})$ s the conditional probability that a particular sample belongs to class 1 given its features $\mathbf{x}$. Now what we are actually interested in is predicting the probability that a certain sample belongs to a particular class, which is the inverse form of the logit function. It is also called the logistic function, sometimes simply abbreviated as sigmoid function due to its characteristic S-shape

\[
\phi(z) = \frac{1}{1+e^{-z}}.
\]

The output of the sigmoid function is then interpreted as the probability of particular sample belonging to class 1

\[
\phi(z) = P(y=1 | \mathbf{x}; \mathbf{w})
\]

given its features $x$ parameterized by the weights $w$. For example, if we compute $\phi(z) = 0.8$ for a particular flower sample, it means that the chance that this sample is an Iris-Versicolor  ower is 80 percent. Similarly, the probability that this  ower is an Iris-Setosa  ower can be calculated as $P(y=0 | \mathbf{x};\mathbf{w})=1 - P (y=1 | \mathbf{x}; \mathbf{w}) = 0.2 or 20 percent.$ The predicted probability can then simply be converted into a binary outcome via a quantizer (unit step function):

\[ \hat{y}= \begin{cases}
      1 & \text{ if } \phi(z) \ge 0.5 \\
      0 & \text{ otherwise }.
   \end{cases}
\]

If we look at the preceding sigmoid plot, this is equivalent to the following:

\[ \hat{y}= \begin{cases}
      1 & \text{ if } \phi(z) \ge 0.0 \\
      0 & \text{ otherwise }.
   \end{cases}
\]

\subsection{Learning the weights of the logistic cost function}

In the previous chapter, we de ned the sum-squared-error cost function:

\[
J(\mathbf{w}) = \frac{1}{2} \sum_i \bigg( \phi \big( z^{(i)} \big) - y^{(i)}  \bigg)^2.
\]

We minimized this in order to learn the weights w for our Adaline classification model. To explain how we can derive the cost function for logistic regression, let's  first define the likelihood L that we want to maximize when we build a logistic regression model, assuming that the individual samples in our dataset are independent of one another. The formula is as follows:

\[
L(\mathbf{w}) = P(\mathbf{y} | \mathbf{x}; \mathbf{w}) = \prod_{i=1}^{n} P \big( y^{(i)} | x^{(i)}; \mathbf{w} \big) =  \prod_{i=1}^{n} \bigg( \phi \big(z^{(i)} \big) \bigg) ^ {y^{(i)}} \bigg( 1 - \phi \big( z^{(i)} \big) \bigg)^{1-y^{(i)}}
\]

In practice, it is easier to maximize the (natural) log of this equation, which is called
the log-likelihood function:

\[
l(\mathbf{w}) = \log L(\mathbf{w}) = \sum_{i=1}^{n} \Bigg[ y^{(i)} \log \bigg(\phi \big( z^{(i)} \big) \bigg) + \bigg(1 - y^{(i)} \bigg) \log \bigg( 1 - \phi \big( z^{i()} \big) \bigg)  \Bigg]
\]

Firstly, applying the log function reduces the potential for numerical under ow, which can occur if the likelihoods are very small. Secondly, we can convert the product of factors into a summation of factors, which makes it easier to obtain the derivative of this function via the addition trick, as you may remember
from calculus.

Now we could use an optimization algorithm such as gradient ascent to maximize this log-likelihood function. Alternatively, let's rewrite the log-likelihood as a cost function $J(\cdot)$ that can be minimized using gradient descent as in \textit{Chapter 2, Training Machine Learning Algorithms for Classification}:

\[
J(\mathbf{w}) = \sum_{i=1}^{n} \Bigg[- y^{(i)} \log \bigg(\phi \big( z^{(i)} \big) \bigg) - \bigg(1 - y^{(i)} \bigg) \log \bigg( 1 - \phi \big( z^{i()} \big) \bigg)  \Bigg]
\]

To get a better grasp on this cost function, let's take a look at the cost that we
calculate for one single-sample instance:

\[
J\big( \phi(z), y; \mathbf{w}   \big) = -y \log \big( \phi(z) \big) - (1-y) \log \big(1 - \phi(z) \big).
\]

Looking at the preceding equation, we can see that the  rst term becomes zero if
$y = 0$ , and the second term becomes zero if $y = 1$, respectively:

\[
J \big( \phi(z), y; \mathbf{w} \big)= \begin{cases}
      - \log \big( \phi(z) \big) \text{ if } y=1\\
      - \log \big( 1 - \phi(z) \big)  \text{ if }  y=0
   \end{cases}
\]

\subsection{Training a logistic regression model with scikit-learn}

If we were to implement logistic regression ourselves, we could simply substitute the cost function $J(\cdot)$ in our Adaline implementation from \textit{Chapter 2, Training Machine Learning Algorithms for Classification}, by the new cost function:

\[
J(\mathbf{w}) = \sum_{i=1}^{n} \Bigg[- y^{(i)} \log \bigg(\phi \big( z^{(i)} \big) \bigg) - \bigg(1 - y^{(i)} \bigg) \log \bigg( 1 - \phi \big( z^{i()} \big) \bigg)  \Bigg]
\]

We can show that the weight update in logistic regression via gradient descent is indeed equal to the equation that we used in Adaline in \textit{Chapter 2, Training Machine Learning Algorithms for Classification}. Let's start by calculating the partial derivative of the log-likelihood function with respect to the $j$th weight:

\[
\frac{\partial}{\partial w_j} l(\mathbf{w}) = \Bigg( y \frac{1}{\phi(z)}  - (1-y) \frac{1}{1-\phi(z)}   \Bigg)   \frac{\partial}{\partial w_j} \phi(z)
\]

Before we continue, let's calculate the partial derivative of the sigmoid function  first:

\[
\frac{\partial}{\partial z} \phi(z) = \frac{\partial}{\partial z} \frac{1}{1 + e^{-1}} \frac{1}{\big( 1 + e^{-z}\big)^2} e^{-z} = \frac{1}{1 + e^{-z}} = \frac{1}{1 + e^{-z}} \bigg( 1 - \frac{1}{1 + e^{-z}} \bigg) \\\\
\]
\[
= \phi(z)(1-\phi(z)).
\]

Now we can resubstitute $\frac{\partial}{\partial z} \phi(z) = \phi(z)(1-\phi(z))$ in our first equation to obtain the following:

\begin{equation}
\begin{split}
& \Bigg( y \frac{1}{\phi(z)} - (1-y) \frac{1}{1-\phi(z)} \Bigg) \frac{\partial}{\partial w_j} \phi(z) \\
& = \Bigg( y \frac{1}{\phi(z)} - (1-y) \frac{1}{1-\phi(z)} \Bigg) \phi(z) \big(1 - \phi(z)\big) \frac{\partial}{\partial w_j} z \\
& = \bigg(  y \big( 1 - \phi(z)   \big) - (1-y) \phi(z)  \bigg) x_j \\
& = \big( y - \phi(z)  \big) x_j
\end{split}
\end{equation}

Remember that the goal is to find the weights that maximize the log-likelihood so that we would perform the update for each weight as follows:

\[
w_j := w_j + \eta \sum_{i=1}^{n} \bigg( y^{(i)} - \phi(z^{(i)}) \bigg) x_{j}^{(i)}
\]

Since we update all weights simultaneously, we can write the general update rule as follows:

\[
\mathbf{w} := \mathbf{w} + \Delta \mathbf{w}
\]

We define $\Delta \mathbf{w}$ as follows:

\[
\Delta \mathbf{w} = \eta \nabla l (\mathbf{w})
\]

Since maximizing the log-likelihood is equal to minimizing the cost function $J(\cdot)$ that we defined earlier, we can write the gradient descent update rule as follows:

\[
\Delta w_j = - \eta \frac{\partial J}{\partial w_j} = \eta \sum_{i=1}^{n} \bigg( y^{(i)} - \phi(z^{(i)}) \bigg)x_{j}^{(i)}
\]

\[
\mathbf{w} := \mathbf{w} + \Delta \mathbf{w}, \; \Delta \mathbf{w} = - \eta \nabla J(\mathbf{w})
\]

This is equal to the gradient descent rule in Adaline in \textit{Chapter 2, Training Machine Learning Algorithms for Classification}.

\subsection{Tackling overfitting via regularization}

The most common form of regularization is the so-called L2 regularization (sometimes also called L2 shrinkage or weight decay), which can be written as follows:

\[
\frac{\lambda}{2} \lVert \mathbf{w} \rVert^2 = \frac{\lambda}{2} \sum_{j=1}^m w_{j}^{2}
\]

Here, $\lambda$ is the so-called regularization parameter.

In order to apply regularization, we just need to add the regularization term to the cost function that we de ned for logistic regression to shrink the weights:

\[
J(\mathbf{w}) = \sum_{i=1}^{n} \bigg[ - y^{(i)} \log \big(  \phi(z^{(i)})  \big)  - \big( 1 - y ^{(i)} \big)  \log \big( 1 - \phi(z^{(i)})   \big)   \bigg] + \frac{\lambda}{2} \lVert \mathbf{w}\rVert^2
\]

Via the regularization parameter $\lambda$, we can then control how well we  t the training data while keeping the weights small. By increasing the value of $\lambda$, we increase the regularization strength.

The parameter \textit{C} that is implemented for the \textit{LogisticRegression} class in scikit-learn comes from a convention in support vector machines, which will be the topic of the next section. \textit{C} is directly related to the regularization parameter $\lambda$ , which is its inverse:

\[
C = \frac{1}{\lambda}
\]

So, we can rewrite the regularized cost function of logistic regression as follows:

\[
J(\mathbf{w}) = C \Bigg[  \sum_{i=1}^{n} \Big(   -y^{(i)} \log \big( \phi(z^{(i)} \big) - \big(  1 - y^{(i)} \big)    \Big) \log \bigg( 1 - \phi(z^{(i)}) \bigg)         \Bigg] + \frac{1}{2} \lVert \mathbf{w} \rVert^2
\]

\section{Maximum margin classification with support vector machines}

\subsection{Maximum margin intuition}

To get an intuition for the margin maximization, let's take a closer look at those \textit{positive} and \textit{negative} hyperplanes that are parallel to the decision boundary, which can be expressed as follows:

\[
w_0 + \mathbf{w}^T \mathbf{x}_{pos} = 1 \quad (1)
\]

\[
w_0 + \mathbf{w}^T \mathbf{x}_{neg} = -1 \quad (2)
\]

If we subtract those two linear equations (1) and (2) from each other, we get:

\[
\Rightarrow \mathbf{w}^T \big(  \mathbf{x}_{pos} - \mathbf{x}_{neg} \big) = 2
\]

We can normalize this by the length of the vector $\mathbf{w}$, which is de ned as follows:

\[
\lVert \mathbf{w} \rVert = \sqrt{\sum_{j=1}^{m} w_{j}^{2}}
\]

So we arrive at the following equation:

\[
\frac{\mathbf{w}^T ( \mathbf{x}_{pos} - \mathbf{x}_{neg}  )}{\lVert \mathbf{w} \rVert} = \frac{2}{\lVert \mathbf{w} \rVert}
\]

The left side of the preceding equation can then be interpreted as the distance between the positive and negative hyperplane, which is the so-called margin that we want to maximize.

Now the objective function of the SVM becomes the maximization of this margin by maximizing $\frac{2}{\lVert \mathbf{w} \rVert}$ under the constraint that the samples are classi ed correctly, which can be written as follows:

\[
w_0 + \mathbf{w}^T \mathbf{x}^{(i)} \ge 1 \text{ if } y^{(i)} = 1
\]

\[
w_0 + \mathbf{w}^T \mathbf{x}^{(i)}  < -1 \text{ if } y^{(i)} = -1
\]

These two equations basically say that all negative samples should fall on one side of the negative hyperplane, whereas all the positive samples should fall behind the positive hyperplane. This can also be written more compactly as follows:

\[
y^{(i)} \big(  w_0 + \mathbf{w}^T \mathbf{x}^{(i)} \big) \ge 1 \quad \forall_i
\]

In practice, though, it is easier to minimize the reciprocal term $\frac{1}{2} \lVert \mathbf{w} \rVert^2$, which can be solved by quadratic programming.

\subsection{Dealing with the nonlinearly separable case using slack variables}

The motivation for introducing the slack variable $\xi$ was that the linear constraints need to be relaxed for nonlinearly separable data to allow convergence of the optimization in the presence of misclassifications under the appropriate cost penalization. The positive-values slack variable is simply added to the linear constraints:

\[
\mathbf{w}^T \mathbf{x}^{(i)} \ge 1 - \xi^{(i)} \text{ if } y^{(i)} = 1
\]

\[
\mathbf{w}^T \mathbf{x}^{(i)} < -1 + \xi^{(i)} \text{ if } y^{(i)} = -1
\]

So the new objective to be minimized (subject to the preceding constraints) becomes:

\[
\frac{1}{2} \lVert \mathbf{w} \rVert^2 + C \Big(\sum_i \xi^{(i)} \Big)
\]

\subsection{Alternative implementations in scikit-learn}
\section{Solving nonlinear problems using a kernel SVM}

As shown in the next  figure, we can transform a two-dimensional dataset onto a new three-dimensional feature space where the classes become separable via the following projection:

\[
\phi(x_1, x_2) = (z_1, z_2, z_3) = (x_1, x_2, x_{1}^{2} + x_{2}^{2})
\]

\subsection{Using the kernel trick to find separating hyperplanes in higher dimensional space}

To solve a nonlinear problem using an SVM, we transform the training data onto a higher dimensional feature space via a mapping function $\phi(\cdot)$ and train a linear SVM model to classify the data in this new feature space. Then we can use the same mapping function $\phi(\cdot)$ to transform new, unseen data to classify it using the linear SVM model.

However, one problem with this mapping approach is that the construction of the new features is computationally very expensive, especially if we are dealing with high-dimensional data. This is where the so-called kernel trick comes into play. Although we didn't go into much detail about how to solve the quadratic programming task to train an SVM, in practice all we need is to replace the dot product

\[
\mathbf{x}^{(i) \; T} \mathbf{x}^{(j)} \text{ by } \phi \big( \mathbf{x}^{(i)} \big)^T \phi \big( \mathbf{x}^{(j)} \big)
\]

In order to save the expensive step of calculating this dot product between two points explicitly, we de define a so-called kernel function:

\[
k \big( \mathbf{x}^{(i)}, \mathbf{x}^{(j)}  \big) = \phi \big( \mathbf{x}^{(i)} \big)^T \phi \big( \mathbf{x}^{(j)} \big)
\]

One of the most widely used kernels is the \textit{Radial Basis Function kernel} (RBF kernel) or Gaussian kernel:

\[
k \big( \mathbf{x}^{(i)}, \mathbf{x}^{(j)}  \big) = \exp \Bigg( - \frac{ \lVert \mathbf{x}^{(i)} - \mathbf{x}^{(j)} \rVert^2  }{2 \sigma^2} \Bigg)
\]

This is often simplified to:

\[
k \big( \mathbf{x}^{(i)}, \mathbf{x}^{(j)}  \big) = \exp \bigg(  -\gamma\ \lVert \mathbf{x}^{(i)} - \mathbf{x}^{(j)} \rVert^2  \bigg)
\]

Here, $\gamma = \frac{1}{2 \sigma^2}$ is a free parameter that is to be optimized.

\section{Decision tree learning}

In order to split the nodes at the most informative features, we need to define an objective function that we want to optimize via the tree learning algorithm. Here, our objective function is to maximize the information gain at each split, which we define as follows:

\[
IG(D_p, f) = I(D_p) - \sum_{j=1}^{m} \frac{N_j}{N_p} I(D_j)
\]

Here, $f$ is the feature to perform the split, $D_p$ and $D_j$ are the dataset of the parent $p$ and $j$th child node; $I$ is our impurity measure, $N_p$ is the total number of samples at the parent node, and $N_j$ is the number of samples at the $j$th child node.  As we can see, the information gain is simply the difference between the impurity of the parent node and the sum of the child node impurities?the lower the impurity of the child nodes, the larger the information gain. However, for simplicity and to reduce the combinatorial search space, most libraries (including scikit-learn) implement binary decision trees. This means that each parent node is split into two child nodes, $D_{left}$ and $D_{right}$:

\[
IG(D_p, f) = 1 (D_p) - \frac{N_{left}}{N_p} I(D_{left}) - \frac{N_{right}}{N_p} I (D_{right})
\]

Now, the three impurity measures or splitting criteria that are commonly used in
binary decision trees are \textit{Gini impurity} ($I_G$), \textit{Entropy} ($I_H$) and the \textit{classification error} ($I_E$). Let's start with the definition of Entropy for all non-empty classes $p(i | t) \neq 0$:

\[
I_H(t) = - \sum_{i=1}^{c} p(i | t) \log_2 p(i|t)
\]

Here, $p(i | t)$ is the proportion of the samples that belongs to class $i$ for a particular node $t$. The entropy is therefore 0 if all samples at a node belong to the same class, and the entropy is maximal if we have a uniform class distribution. For example, in a binary class setting, the entropy is 0 if $p(i=1 | t) = 1$ or $p(i=0| t)=0$. If the classes are distributed uniformly with $p(i=1|t)=0.5$ and $p(i=0|t)=0.5$, the entropy is 1. Therefore, we can say that the entropy criterion attempts to maximize the mutual information in the tree.

Intuitively, the Gini impurity can be understood as a criterion to minimize the probability of misclassification:

\[
I_G(t) = \sum_{i=1}^{c} p(i | t) (1 - p (i | t)) = 1 - \sum_{i=1}^{c} p(i|t)^2
\]

Similar to entropy, the Gini impurity is maximal if the classes are perfectly mixed, for example, in a binary class setting ($c = 2 $):

\[
1 - \sum_{i=1}^{c} 0.5^2 = 0.5.
\]

...

Another impurity measure is the classification error:

\[
I_E = 1 - max \{ p(i|t) \}
\]

\subsection{Maximizing information gain -- getting the most bang for the buck}
\subsection{Building a decision tree}
\subsection{Combining weak to strong learners via random forests}
\section{K-nearest neighbors -- a lazy learning algorithm}

The \textit{minkowski} distance that we used in the previous code example is just a generalization of the Euclidean and Manhattan distances that can be written as follows:

\[
d \big(\mathbf{x}^{(i)}, \mathbf{x}^{(j)}\big) =  \sqrt[p]{\sum_k \big| x_{k}^{(i)} - x_{k}^{(j)} \big|^p }
\]

\section{Summary}

\chapter{Building Good Training Sets -- Data Pre-Processing}

\section{Dealing with missing data}
\subsection{Eliminating samples or features with missing values}
\subsection{Imputing missing values}
\subsection{Understanding the scikit-learn estimator API}
\section{Handling categorical data}
\subsection{Mapping ordinal features}
\subsection{Encoding class labels}
\subsection{Performing one-hot encoding on nominal features}
\section{Partitioning a dataset in training and test sets}
\section{Bringing features onto the same scale}

Now, there are two common approaches to bringing different features onto the same
scale: \textit{normalization} and \textit{standardization}. Those terms are often used quite loosely
in different fields, and the meaning has to be derived from the context. Most often,
\textit{normalization} refers to the rescaling of the features to a range of [0, 1], which is a
special case of min-max scaling.  To normalize our data, we can simply apply the
min-max scaling to each feature column, where the new value $x_{norm}^{(i)}$ of a sample  $x^{(i)}$:

\[
x_{norm}^{(i)} = \frac{x^{(i)} - \mathbf{x}_{min}}{\mathbf{x}_{max} - \mathbf{x}_{min}}
\]

Here, $x^{(i)}$ is a particular sample, $x_{min}$ is the smallest value in a feature column, and $x_{max}$ the largest value, respectively.

[...]  Furthermore, standardization maintains useful information about outliers and makes the algorithm less sensitive to them in contrast to min-max scaling, which scales
the data to a limited range of values.

The procedure of standardization can be expressed by the following equation:

\[
x_{std}^{(i)} = \frac{x^{(i)} - \mu_{x}}{\sigma_{x}}
\]

Here, $\mu_{x}$ is the sample mean of a particular feature column and $\sigma_{x}$ the corresponding standard deviation, respectively.

\section{Selecting meaningful features}
\subsection{Sparse solutions with L1 regularization}

We recall from \textit{Chapter 3, A Tour of Machine Learning Classfiers Using Scikit-learn}, that L2 regularization is one approach to reduce the complexity of a model by penalizing large individual weights, where we de ned the L2 norm of our weight vector w as follows:

\[
L2: \lVert \mathbf{w} \rVert^{2}_{2} = \sum_{j=1}^{m} w^{2}_{j}
\]

Another approach to reduce the model complexity is the related \textit{L1 regularization}:

\[
L1: \lVert \mathbf{w} \rVert_{1} = \sum_{j=1}^{m} |w_j|
\]

\subsection{Sequential feature selection algorithms}

Based on the preceding definition of SBS, we can outline the algorithm in 4 simple steps:

\begin{enumerate}
\item Initialize the algorithm with $k=d$, where $d$  is the dimensionality of the full feature space $\mathbf{X}_d$
\item Determine the feature $x^{-}$ that maximizes the criterion $x^{-} = \text{arg max} J(\mathbf{X_k} - x)$, where  $x \in \mathbf{X}_k$.
\item Remove the feature $x^-$ from the feature set: $\mathbf{X}_{k-l} := \mathbf{X}_k - x^{-}; \quad k:= k-1$.
\item Terminate if $k$ equals the number of desired features, if not, got to step 2.

\end{enumerate}

\section{Assessing feature importance with random forests}
\section{Summary}

\newpage

... to be continued ...

\newpage

\end{document} % end main document