% If problems with the headers:
% get headings in appendix etc. right with
% \markboth{\spacedlowsmallcaps{Appendix}}{\spacedlowsmallcaps{Appendix}}
%==============================================================================
\chapter{Validation of GPE-based GSA and HM techniques}\label{cha:chapterA}
%==============================================================================

%
%
%
\section[GPE-based GSA technique validation using analytic solutions for Sobol' indices of test functions]{GPE-based GSA technique validation using analytic\\ solutions for Sobol' indices of test functions}\label{sec:chAGPE-based_GSA_technique_validation_using_analytic_solutions_for_Sobol'_indices_of_test_functions}
We validated the GPE-based GSA technique used throughout the entire work (second emulation-based approach presented in equations~\eqref{eq:emulpostsamplesgsa1}--\eqref{eq:emulpostsamplesgsa2}, Section~\ref{sec:ch3emulatorbasedestimates}) by emulating test functions whose Sobol' sensitivity indices could be calculated analytically. After training, the emulators were used to estimate the Sobol' indices and the estimates were compared with the corresponding known theoretical values.

%
%
%
\subsection{Ishigami function}
The Ishigami function~\cite{Ishigami:1990} is commonly used in sensitivity analysis and uncertainty quantification studies~\cite{Saltelli:2007}, because it exhibits strong nonlinearity and nonmonotonicity. This function is regulated by a vector of $D=3$ input variables $\mathbf{X}=(X_1,\,X_2,\,X_3)$, which are assumed to be independent and uniformly distributed in $[-\pi,\,\pi]$:
%
\begin{align}\label{eq:ishigamifun}
	& Y = \sin{X_1} + a\,\sin^2{X_2} + b\,X_3^4\sin{X_1}\,,\quad\text{with} \\
	& X_i\,\,\sim\,\,\mathcal{U}([-\pi,\,\pi])\quad \text{for}\quad i=1,2,3\quad \text{and}\quad a,b>0
\end{align}

\vspace{0.2cm}\noindent
The function output total variance is given in terms of the $a$ and $b$ parameters:
%
\begin{equation}\label{eq:ifun_total_var}
	V = \frac{1}{2} + a^2\,\frac{1}{8} + b\,\frac{\pi^4}{5} + b^2\,\frac{\pi^8}{18}
\end{equation}

\vspace{0.2cm}\noindent
Sobol' sensitivity indices are then calculated by dividing the partial variances (which are also given in terms of the $a$ and $b$ parameters) by the total variance of equation~\eqref{eq:ifun_total_var}:
%
\begin{align}
	& S_{1} = \frac{V_{1}}{V},\quad V_{1} = \frac{1}{2}\left(1 + b\frac{\pi^4}{5}\right)^2 \\
	& S_{2} = \frac{V_{2}}{V},\quad V_{2} = a^2\frac{1}{8} \\
	& S_{3} = \frac{V_{3}}{V},\quad V_{3} = 0
\end{align}

\begin{align}
	& S_{T1} = \frac{V_{1}}{V},\quad V_{T1} = \frac{1}{2}\left(1 + b\frac{\pi^4}{5}\right)^2 + b^2\frac{8\pi^8}{225} \\
	& S_{T2} = \frac{V_{T2}}{V},\quad V_{T2} = V_2 \\
	& S_{T3} = \frac{V_{T3}}{V},\quad V_{T3} = b^2\frac{8\pi^8}{225}
\end{align}

%
%
%
\subsection{G*-function}
The G*-function proposed by Saltelli~\cite{Saltelli:2010} is a modified version of the classic Sobol' G-function~\cite{Archer:1997} which is commonly used for sensitivity analysis and uncertainty quantification purposes as its input variables' importance can be directly controlled by tuning specific function parameters (along with being strongly nonlinear and nonmonotonic). Saltelli's G*-function is a shifted and curved G-function, and overcomes the issue when low-discrepancy quasi-random Sobol' sequences (whose initial sequence point is $1/2$) are used to estimate the Sobol' sensitivity indices, since the classic G-function has got a singularity in each $i$-th dimension exactly at the point $X_{i}=1/2$. The G*-function is regulated by an arbitrary positive integer number $D$ of input variables $\mathbf{X}=(X_1,\,\dots,\,X_D)$, assumed to be independent and uniformly distributed in $[0,\,1]$:
%
\begin{align}\label{eq:gstarfun}
	& Y = \prod_{i=1}^{D}g_i(X_i),\quad\text{with} \\
	& g_i(X_i)=\frac{(1+\alpha_i)\vert 2(X_i+\delta_i-I[X_i+\delta_i])-1\vert^{\alpha_i}+a_i}{1+a_i}\quad\text{and} \\
	& X_i\,\,\sim\,\,\mathcal{U}([0,\,1]),\,\,\alpha_i > 0,\,\,\delta_i\in [0,\,1],\,\,a_i>0\quad\text{for}\,\,i=1,\dots,D
\end{align}

\vspace{0.2cm}\noindent
where $I[X_i+\delta_i]$ indicates the integer part of $X_i+\delta_i$. Parameter $a_i$ controls the importance of input factor $X_i$ into affecting the total variance of the function output. A low $a_i$ means an important $X_i$ factor, while a high $a_i$ indicates a negligible $X_i$ effect. If more than one $a_i$ parameters is low, higher order interaction effects will be present as well. $\delta_i$ is the parameter through which the classic G-function is shifted, while the $\alpha_i$ parameter changes the curvature of the function. The choice of $\delta_i=0$ and $\alpha_i=1,\,\forall\,i=1,\dots,D$ gives back the original G-function formulation.

\vspace{0.2cm}
The partial variances of the first order are given in terms of the $a_i$s parameters and of the curvature parameters $\alpha_i$s, while the partial total variances are calculated as a function of the first-order ones:
%
\begin{align}
	& V_i = \frac{\alpha_i^2}{(1+2\alpha_i)(1 + a_i)^2}\,,\quad\text{for}\quad i=1,\dots,D \label{eq:gfun_partial_var_first}\\
	& V_{Ti} = V_i \prod_{j=1,\,j\neq i}^{D}(1 + V_j)\,,\quad\text{for}\quad i=1,\dots,D \label{eq:gfun_partial_var_total}
\end{align}

\vspace{0.2cm}\noindent
Sobol' sensitivity indices are then calculated by dividing the partial variances of equations~\eqref{eq:gfun_partial_var_first}--\eqref{eq:gfun_partial_var_total} by the total variance, which is again given as a function of the first-order partial variances as:
%
\begin{align}
	& S_i = \frac{V_i}{V} \quad\text{and}\quad S_{Ti} = \frac{V_{Ti}}{V} \quad\text{for}\quad i=1,\dots,k,\,\quad\text{with} \\
	& V = -1 + \prod_{i=1}^{k}(1 + V_i)
\end{align}

%
%
%
\subsection{Simulator estimates}\label{sec:chAsimulator_estimates}
We first calculated the Sobol' first-order and total sensitivity indices using the forward models (the functions themselves, or simulators) and the estimators presented in equations~\eqref{eq:gsaestimators}--\eqref{eq:varianceestimate}, Section~\ref{sec:ch3estimatingsobolsensitivityindices}. In particular, we used the Ishigami function (equation~\eqref{eq:ishigamifun}) with $a=7,\,b=0.1$, and we used the G*-function (equation~\eqref{eq:gstarfun}) with $D=10$ and $\alpha_i=1$, $\delta_i\,\,\sim\,\,\mathcal{U}([0,\,1])$, $a_i=i-1$, for $i=1,\dots,D$. This particular particular choice of the $a_i$ coefficients coupled with the chosen problem dimension of $D=10$ ensured that at least one ``very-important'' and one ``non-important'' input factors were present~\cite{Saltelli:2010,Marrel:2008}, modelled using parameter values $a_1=0$ and $a_{10}=9$, respectively. We calculated the Sobol' sensitivity indices by testing $5$ different values (spaced evenly on a log scale) in the range $[10^2,\,10^4]$ for the number of samples $N$ (introduced in Section~\ref{sec:ch3estimatingsobolsensitivityindices}) to be used when approximating the partial and total variances' integrals. We recall that the cost of calculating all the main and total effects is $N\times(D+2)$ model evaluations.

\vspace{0.2cm}
The theoretical Sobol' indices' values are summarised in Tables~\ref{tab:ifun_theo_vals}--\ref{tab:gfun_theo_vals} for the Ishigami and G*- functions, respectively. In Figures~\ref{fig:ishigami_simulator_estimates}--\ref{fig:gstar_simulator_estimates}, the estimated Sobol' sensitivity indices are compared with the respective theoretical values. Bootstrap (number of resamples $=100$) $\SI{95}{\percent}$ confidance intervals for the estimates are also provided. We can see that the estimates perfectly match the theoratical values, with better matching as the number of quadrature formulae samples increases. Although an absolute statement cannot be derived about the optimal choice of $N$ by simply looking at the two presented example test functions, we decided to fix $N=10^3$ in the next analysis for emulation-based estimates, to have a balance between small enough bootstrap confidence intervals, good match in the theoretical values and not too high number of model evaluations to be performed (although the latter will not be an issue anymore when using emulators).

\begin{table}[ht!]
    \myfloatalign
    \begin{tabularx}{\textwidth}{XXX}
    \toprule
    \tableheadline{Input factor} & \multicolumn{2}{c}{\spacedlowsmallcaps{Sobol' sensitivity index}} \\
    \midrule
    & \tableheadline{$S_{i}$} & \tableheadline{$S_{Ti}$} \\
    \cmidrule{2-3}
    $X_{1}$ & $0.3139$ & $0.5576$ \\
    $X_{2}$ & $0.4424$ & $0.4424$ \\
    $X_{3}$ & $0.0000$ & $0.2437$ \\
    \bottomrule
    \end{tabularx}
    \caption{Ishigami function Sobol' first-order and total effects' theoretical values for $a=7,\,b=0.1$.}
    \label{tab:ifun_theo_vals}
\end{table}

\begin{figure}[ht!]
    \myfloatalign
    \subfloat[Main effects.]
    {\label{fig:ishigami_s1_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/Ishigami_S1.pdf}}\quad
    \subfloat[Total effects.]
    {\label{fig:ishigami_st_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/Ishigami_ST.pdf}}
    \caption{Ishigami function Sobol' first-order (main) and total sensitivity indices estimated using the simulator. Indices are given as pointwise estimates with bootstrap $\SI{95}{\percent}$ confidance intervals.}
    \label{fig:ishigami_simulator_estimates}
\end{figure}

\begin{table}[ht!]
    \myfloatalign
    \begin{tabularx}{\textwidth}{XXX}
    \toprule
    \tableheadline{Input factor} & \multicolumn{2}{c}{\spacedlowsmallcaps{Sobol' sensitivity index}} \\
    \midrule
    & \tableheadline{$S_{i}$} & \tableheadline{$S_{Ti}$} \\
    \cmidrule{2-3}
    $X_{1}$   & $0.5607$ & $0.6705$ \\
    $X_{2}$   & $0.1402$ & $0.2063$ \\
    $X_{3}$   & $0.0623$ & $0.0958$ \\
    $X_{4}$   & $0.0350$ & $0.0547$ \\
    $X_{5}$   & $0.0224$ & $0.0353$ \\
    $X_{6}$   & $0.0156$ & $0.0246$ \\
    $X_{7}$   & $0.0114$ & $0.0181$ \\
    $X_{8}$   & $0.0088$ & $0.0139$ \\
    $X_{9}$   & $0.0069$ & $0.0110$ \\
    $X_{10}$  & $0.0056$ & $0.0089$ \\
    \bottomrule
    \end{tabularx}
    \caption{G*-function Sobol' first-order and total effects' theoretical values for $D=10$ and $\alpha_i=1$, $\delta_i\,\,\sim\,\,\mathcal{U}([0,\,1])$, $a_i=i-1$, for $i=1,\dots,D$.}
    \label{tab:gfun_theo_vals}
\end{table}

\begin{figure}[ht!]
    \myfloatalign
    \subfloat[Main effects.]
    {\label{fig:gstar_s1_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/GStar_S1.pdf}}\quad
    \subfloat[Total effects.]
    {\label{fig:gstar_st_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/GStar_ST.pdf}}
    \caption{G*-function Sobol' first-order (main) and total sensitivity indices estimated using the simulator. Indices are given as pointwise estimates with bootstrap $\SI{95}{\percent}$ confidance intervals.}
    \label{fig:gstar_simulator_estimates}
\end{figure}

%
%
%
\clearpage
\subsection{Emulator estimates}\label{sec:chAemulator_estimates}
We wanted to test whether we could obtain reliable estimates of the Sobol' sensitivity indices this time by using emulators of the above presented test functions.

\vspace{0.2cm}
For this purpose, we sampled points $\mathbf{X}$ from a LHD over the hypercubes $[-\pi,\,\pi]^3$ and $[0,\,1]^{10}$ for the Ishigami and the G*- functions, respectively. We then simulated these points $Y=f_{simul}(\mathbf{X})$ to form a dataset $(\mathbf{X},\,Y)$ to be used as learning sample. The leaning sample size was given by $\vert\mathbf{X}\vert=\text{factor}\times D$, being $D$ the problem dimension ($D=3$ and $D=10$ for the Ishigami and G*- functions, respectively). Common practice for computer model experiments is to take at least $10\times D$ learning samples in Gaussian process regression settings~\cite{Rasmussen:2006}. In this study, we tested different multiplicative factors, taken from the set $\{10,\,20,\,30,\dots,\,100\}$. We also simulated an additional set of points (size $=\SI{20}{\percent}$ of the biggest learning sample used) to be used as a testing dataset for evaluating each of the trained emulators predictivity, as described by the coefficient of determination ($R^2$ score). After training and for each emulator, $1,000$ emulator posterior distribution's samples were used to obtain Sobol' sensitivity indices' distributions as described in Section~\ref{sec:ch3emulatorbasedestimates}. In this case, we only investigated the uncertainty in the estimates arising from the use of the emulators, while the quadrature formulae numerical errors were neglected (and not estimated using the bootstrap technique as done in Section~\ref{sec:chAsimulator_estimates}).

\vspace{0.2cm}
The obtained $R^2$ scores are summarised in Tables~\ref{tab:ifun_scores}--\ref{tab:gfun_scores} for the Ishigami and G*- functions, respectively. We can see that the emulator predictivity generally increases as the number of learning samples increases, although this behaviour eventually saturates, meaning that ``the more points'' does not always imply ``the better accuracy''. In Figures~\ref{fig:ishigami_emulator_estimates}--\ref{fig:gstar_emulator_estimates}, the estimated Sobol' sensitivity indices are compared with the respective theoretical values. The estimated values are given as distributions, and each distribution is plotted against the multiplicative factor used to build the training dataset of the corresponding emulator which was used to obtain the distribution itself. We can see that the distributions progressively center around (i.e. their mean values match) the theoretical values as the number of learning samples used to train the emulator increases. In particular, for both the Ishigami and G*- functions we can see that at least $40\times D$ learning samples were necessary to achieve a satisfactory match with theoretical values. Again, we cannot derive absolute statements concerning what is the optimal choice of the multiplicative factor, and investigating this goes beyond the scope of the project. Nevertheless, we have demonstrated the feasibility of estimating Sobol' sensitivity indices with good accuracy by using emulators of both example $3$D and $10$D nonlinear and nonmonotonic computer codes.

\newpage
\begin{table}[ht!]
    \myfloatalign
    \begin{tabularx}{0.5\textwidth}{XX}
    \toprule
    \tableheadline{Factor} & \tableheadline{$R^2$ score} \\
    \midrule
    $10\times$   & $0.3796$ \\
    $20\times$   & $0.5187$ \\
    $30\times$   & $0.5438$ \\
    $40\times$   & $0.8563$ \\
    $50\times$   & $0.9526$ \\
    $60\times$   & $0.9770$ \\
    $70\times$   & $0.9806$ \\
    $80\times$   & $0.9862$ \\
    $90\times$   & $0.9876$ \\
    $100\times$  & $0.9941$ \\
    \bottomrule
    \end{tabularx}
    \caption{Ishigami function emulators' predictivity as described by the $R^2$ score. Predictivity is tested against the same set of points for each of emulators trained on differently large training datasets.}
    \label{tab:ifun_scores}
\end{table}

\begin{figure}[ht!]
    \myfloatalign
    \subfloat[Main effects.]
    {\label{fig:ishigami_s1_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/Ishigami_S1_gpe_based.pdf}}\quad
    \subfloat[Total effects.]
    {\label{fig:ishigami_st_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/Ishigami_ST_gpe_based.pdf}}
    \caption{Ishigami function Sobol' first-order (main) and total sensitivity indices estimated using the emulator. Indices are given as different estimates (forming entire distributions) corresponding to different samples from the full emulators' posterior distributions.}
    \label{fig:ishigami_emulator_estimates}
\end{figure}

\begin{table}[ht!]
    \myfloatalign
    \begin{tabularx}{0.5\textwidth}{XX}
    \toprule
    \tableheadline{Factor} & \tableheadline{$R^2$ score} \\
    \midrule
    $10\times$   & $0.6470$ \\
    $20\times$   & $0.7689$ \\
    $30\times$   & $0.7534$ \\
    $40\times$   & $0.7881$ \\
    $50\times$   & $0.8332$ \\
    $60\times$   & $0.8396$ \\
    $70\times$   & $0.8427$ \\
    $80\times$   & $0.8569$ \\
    $90\times$   & $0.8805$ \\
    $100\times$  & $0.8896$ \\
    \bottomrule
    \end{tabularx}
    \caption{G*-function emulators' predictivity as described by the $R^2$ score. Predictivity is tested against the same set of points for each of emulators trained on differently large training datasets.}
    \label{tab:gfun_scores}
\end{table}

\begin{figure}[ht!]
    \myfloatalign
    \subfloat[Main effects.]
    {\label{fig:gstar_s1_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/GStar_S1_gpe_based.pdf}}\quad
    \subfloat[Total effects.]
    {\label{fig:gstar_st_theo}
    \includegraphics[width=\linewidth]{figures/chapterA/GStar_ST_gpe_based.pdf}}
    \caption{G*-function Sobol' first-order (main) and total sensitivity indices estimated using the emulator. Indices are given as different estimates (forming entire distributions) corresponding to different samples from the full emulators' posterior distributions.}
    \label{fig:gstar_emulator_estimates}
\end{figure}

%
%
%
\cleardoublepage
\section{HM validation using synthetic data}\label{sec:chAHM_validation_using_synthetic_data}
We validated HM technique against \textit{synthetic data}, generated using the \textit{in silico} model of SHAM rat heart contraction mechanics (Section~\ref{sec:ch4ratheartcontractionmodel}).

\vspace{0.2cm}
For this purpose, we started from the performed HM on the SHAM model (Section~\ref{sec:ch4modelfitting}, Figure~\ref{fig:shamhm}). We selected an intermediate wave, specifically wave $4$, among the total $8$ waves the full process took to complete (Figure~\ref{fig:hm1}). We then sampled $256$ points in wave $4$ that did not belong to the final wave, namely wave $8$ (Figures~\ref{fig:hm2}--\ref{fig:hm3}). We then simulated all these points and computed the LV features' mean and standard deviation values for the ones that led to a converging simulation ($48$ points). We further used these mean and standard deviation values as synthetic data to be matched. The LV features' synthetic variability is summarised in Table~\ref{tab:values2match4synt}.

\begin{table}[ht!]
    \myfloatalign
    \begin{tabularx}{\textwidth}{lXX}
    \toprule
    \tableheadline{LV feature} & \tableheadline{Units}                  & \tableheadline{Synthetic variability} \\
    \midrule
    $\textrm{EDV}^{*}$         & $\SI{}{\micro\liter}$                  & $493.10 \pm 16.85$ \\
    $\textrm{ESV}^{*}$         & $\SI{}{\micro\liter}$                  & $204.29 \pm  7.93$ \\
    $\textrm{EF}$              & $\SI{}{\percent}$                      & $ 58.54 \pm  1.74$ \\
    $\textrm{IVCT}$            & $\SI{}{\milli\second}$                 & $140.86 \pm  0.78$ \\
    $\textrm{ET}^{*}$          & $\SI{}{\milli\second}$                 & $ 56.77 \pm  1.53$ \\
    $\textrm{IVRT}^{*}$        & $\SI{}{\milli\second}$                 & $ 20.03 \pm  2.43$ \\
    $\textrm{Tdiast}$          & $\SI{}{\milli\second}$                 & $ 94.07 \pm  1.11$ \\
    $\textrm{PeakP}^{*}$       & $\SI{}{\kilo\pascal}$                  & $ 18.56 \pm  1.14$ \\
    $\textrm{Tpeak}$           & $\SI{}{\milli\second}$                 & $ 39.39 \pm  1.36$ \\
    $\textrm{ESP}$             & $\SI{}{\kilo\pascal}$                  & $ 8.74  \pm  0.21$ \\
    $\textrm{maxdP}^{*}$       & $\SI{}{\kilo\pascal\per\milli\second}$ & $ 1.08  \pm  0.06$ \\
    $\textrm{mindP}^{*}$       & $\SI{}{\kilo\pascal\per\milli\second}$ & $-0.54  \pm  0.06$ \\
    \bottomrule
    \end{tabularx}
    \caption{Synthetic data LV features' variability given as mean $\pm$ standard deviation values to match for HM validation. Only the LV features with an asterisk $(\,\,)^*$ were used as targets for HM.}
    \label{tab:values2match4synt}
\end{table}

\vspace{0.2cm}
We started the HM process from the same initial space-filling design (lightest blue variant coloured points in Figure~\ref{fig:shamhm}) used for the SHAM model HM, this time trying to match the synthetic data instead of the experimental data. To replicate exactly the same process, we tried to match the same $7$ features out of the total $12$ features of interest, although we had mean and standard deviation values for all the LV features. Figure~\ref{fig:syntfeatmatch} shows that all the LV features were perfectly matched at the end of the HM process (which took $9$ waves to complete), including the $5$ features we did not directly selected as targets for HM.

\begin{figure}[ht!]
    \myfloatalign
    \includegraphics[width=1\textwidth]{figures/chapterA/hm_vs_synt_data_features.pdf}
    \caption{Simulated LV features’ distributions around synthetic mean values. These are obtained by evaluating the simulator at input parameter points belonging to the first (w$1$) up to the last (w$9$) waves of HM performed on synthetic data. Simulated features are shown as empty dots, while the related synthetic mean values are displayed as full dots. $2$ STD confidence intervals are also shown as vertical straight lines centred around synthetic mean values. All the displayed values (including confidence intervals) are normalised by their respective synthetic mean values.}
    \label{fig:syntfeatmatch}
\end{figure}

\vspace{0.2cm}\noindent
Moreover, we can see that the input parameters' ranges characterising the non-implausible $X_{NIMP}$ space of the last HM wave were most of the time restricted to ranges which have non-empty intersection with the ranges where the synthetic data were sampled from (Figure~\ref{fig:syntintmatch}). This shows that HM technique not only matched the target features' values but also recovered the initial sample space.

\begin{figure}[ht!]
    \myfloatalign
    \includegraphics[width=1\textwidth]{figures/chapterA/hm_vs_synt_data_parameters.pdf}
    \caption{Input parameters' ranges at each wave during SHAM rat heart model HM on synthetic data (blue). The parameters' ranges where the synthetic data were sampled from are also shown (red).}
    \label{fig:syntintmatch}
\end{figure}

\begin{figure}[ht!]
    \myfloatalign
    \subfloat[The initial space with wave $4$ and wave $8$ $X_{NIMP}$ spaces is highlighted.]
    {\label{fig:hm1}
    \includegraphics[width=\linewidth]{figures/chapterA/1_initial_w4_w8.png}}\quad
    \subfloat[Wave $8$ $X_{NIMP}$ space is cut out of the full space.]
    {\label{fig:hm2}
    \includegraphics[width=\linewidth]{figures/chapterA/2_initial_w4minusw8.png}}\quad
    \subfloat[Points are sampled within wave $4$ $X_{NIMP}$ but not within wave $8$ $X_{NIMP}$ space.]
    {\label{fig:hm3}
    \includegraphics[width=\linewidth]{figures/chapterA/3_initial_w4minusw8_sampledpoints.png}}
    \caption{The process of points' selection for synthetic data generation to be used for HM technique validation.}\label{fig:ratrepimag}
\end{figure}


\chapter{Weight initialization, batch normalization and dropout}
In this chapter, we will discuss several basic techniques used in the training of neural networks, including
\begin{enumerate}
\item Weights initialization
\item Batch normalization
\item Dropout
\end{enumerate}
\section{Weights Initialization}
They adopt the strategy from Glorot \& Bengio 2010 ``Understanding the difficulty of training deep feedforward neural networks". Basically speaking, Bengio's results come from the idea to analyse the statistical properties for totally linear deep neural network cases i.e take nonlinear activation function as identity.

\section{Batch Normolization}
This method is closed related to SGD with mini-batch. For example, in every step to update, the real objective function is
\begin{equation}
\sum_{i \in S} \|f^J(x_i) - y_i\|^2,
\end{equation}
with $S$ is a mini-batch of the whole data.

Generally speaking, we need do some initialization for whole data $\{x_i, y_i\}$ like:
\begin{equation}
\tilde x_{i} = \frac{x_i - \mu}{\sigma},
\end{equation}
with $\mu$ is the average of data $x_i$ and $\sigma$ is the standard deviation of data, i.e the square of variance. Here we can extend this normolization method like the deep learning idea i.e stack this operation.

The batch normolization operation can be seen as do this initialization operation for every layers w.r.t the mini-batch. In our notation, this skill can be explained as do one step gradient descent for the next model:
\begin{equation}
\sum_{i \in S} \|\tilde {f}^J(x_i) - y_i\|^2,
\end{equation}
with
\begin{equation}
\tilde f^j = \theta^j \circ g^j \circ (\gamma^j BN_{S} (\tilde f^{j-1}) + \beta^j).
\end{equation}
%\footnote{The following order seems to be more reasonable:
%\begin{equation}
%\tilde f^j = \theta^j \circ g^j \circ BN_{S} (\tilde f^{j-1}).
%\end{equation}
%Please see \S3.2 (page 5) of the original paper by Ioffe and Szegedy.
%}
Here
\begin{equation}
BN_{S}( \tilde f^{j-1}(x_i) )= \frac{\tilde f^{j-1}(x_i) - \mu^j}{\sigma^j},
\end{equation}
where
\begin{equation}
\mu^j = \frac{\sum_{i\in S}  \tilde f^{j-1}(x_i) }{|S|},
\end{equation}
and
\begin{equation}
\sigma^j  = \sqrt{ \delta + \frac{ \sum_{i\in S} (  \tilde f^{j-1}(x_i) - \mu^j)^2}{|S|} }.
\end{equation}
Here $(  \tilde f^{j-1}(x_i) - \mu^j)^2$ means element-wise operation, i.e $\rm{dim}(\tilde f^{j-1}) = \rm{dim}(\mu^j) = \rm{dim}(\sigma^j)$.
{\bf{Here the have a very important assumption: $\tilde f^{(j-1)}$ is an random vector with elements are independent.} }

What's more, here $\gamma^j$ and $\eta^j$ are two new parameters to be trained which helps to change the mean and stander deviation for the out put after BN. This seems making no sense, but this change some really because now the mean and stander deviation can be chosen adaptively not just have a very complex dependence with previous layers parameters.

\begin{remark}
Here to apply $BN_S$ just after $\tilde f^{j-1}$ or $g^j(\tilde f^{j-1})$ makes difference. In the first paper about BN, the author suggest to apply $BN_S$ just after $\tilde f^{j-1}$ because generally speaking,
\begin{equation}
\tilde f^{j-1}(X_i) = WX_i + b,
\end{equation}
if note
\begin{equation}
X_i = g^{j-1}(\tilde f^{j-2}(x_i)),
\end{equation}
then we have
\begin{equation}
BN_{S}(WX_i + b) = BN_S(WX_i),
\end{equation}
i.e we can ignore those bais term.

\end{remark}

Here we need to mention that, the $BN$ function will influence taking gradient in training process. For example:
\begin{align}
\frac{\partial \tilde f^j}{ \partial \theta^{j-1}} &= \theta^j \circ (g^j)' \circ \frac{\partial BN_{S} (\tilde f^{j-1})}{ \partial \theta^{j-1}}, \\
&=  \theta^j \circ (g^j)' \circ (\frac{\partial BN_{S} (\tilde f^{j-1})}{ \partial \mu^j}\frac{\partial \mu^j}{\partial \theta^j} + \frac{\partial BN_{S} (\tilde f^{j-1})}{ \partial \sigma^j}\frac{\partial \sigma^j}{\partial \theta^j} + \frac{\partial BN_{S} (\tilde f^{j-1})}{ \partial \tilde f^j}\frac{\partial \tilde f^j}{\partial \theta^j}  )
\end{align}

\subsection{One question}
\begin{quote}
  \bf Since the normalization is carried out in each batch, what is
  the final outcome of the trained neural network (which should be
  independent of batches)?
\end{quote}
{\bf For test phase}, to the $\hat \mu^j$ and $\hat \sigma^j$ are computed
by averaging those $\mu^j$ and $\sigma^j$ in training phase. In training phase, suppose we have those next training process with mini-batch $S_t$ for $t = 1:T$. So, after $T$ steps gradient descent, those $\theta^j$ have been trained well, and we get:
\begin{equation}
\mu^j_{t}, \sigma^j_t, \quad t = 1:T,
\end{equation}
So the test model is
\begin{equation}
f(x) = \hat{f}^J(x),
\end{equation}
with
\begin{equation}
 \hat{f}^j(x) = \theta^j \circ g^j \circ \hat{BN}^j(\hat f^{j-1})
\end{equation}
for
\begin{equation}
\hat{BN}^j( \hat f^{j-1}(x_i) )= \frac{\hat f^{j-1}(x_i) - \hat \mu^j}{\hat \sigma^j},
\end{equation}
where
\begin{equation}
\hat \mu^j =\frac{\sum_{i=1}^T \mu^j_t }{T},
\end{equation}
and
\begin{equation}
\hat \sigma^j  = \frac{\sum_{i=1}^T \sigma^j_t }{T}.
\end{equation}

In some situation, they may use the similar strategy like the momentum for SGD, then may get $\hat \mu^j$ and $\hat \sigma^j$ by: For $t= 1 :T$
\begin{equation}
\hat \mu^j = 0.9 \hat \mu^j + 0.1 \mu^j_t,
\end{equation}
and the similar operation for $\hat \sigma^j$

\subsection{An example in DL book}
Now we consider a very simple case, i.e
\begin{itemize}
\item No nonlinear activation function, i.e $g = id$,
\item All layers just have one neuron and no bais, i.e $\theta(x) = wx$.
\end{itemize}
Suppose we have a network with $J$ layers, i.e
\begin{equation}
f(x) = w_J w_{J-1} \cdots w_2 w_1 x
\end{equation}
And the loss function is:
\begin{equation}
L(w_1, \cdots, w_J) = \sum_{i = 1}^N\frac{1}{2} (f(x_i) - y_i)^2.
\end{equation}

Now we have
\begin{equation}
\frac{\partial L}{\partial f(x_i)} = 1,
\end{equation}
this means that when we update $\bm{w} = (x_1, \cdots, w_J)$ in this steps, we just need to compute
\begin{equation}
\bm{g} = \frac{\partial L}{\partial \bm{w}} = \sum_{i=1}^J \nabla_{\bm{w}} f(x_i).
\end{equation}

Now we suppose to decrease the value for $f$ by $0.1$ at $x$. With the first order approximation of $f$, we know that $f$ will decrease about
\begin{equation}
\epsilon \|\bm{g}\|^2,
\end{equation}
with
\begin{equation}
\bm{w} \leftarrow \bm{w} - \epsilon \bm{g}.
\end{equation}

However, for really situation, we have
\begin{equation}
f(x) = (w_J - \epsilon g_J)\cdots(w_1 - \epsilon g_1)x.
\end{equation}
For one term in seconder approximation like:
\begin{equation}
\epsilon^2 g_1 g_2 \Pi_{i=3}^J w_i x.
\end{equation}
Because we connot control the value of $\Pi_{i=3}^J w_i x$, it can be very small or extremely large so {\bf it it very difficult to choose suitable learning rate.}

But, if we adopt the BN operation, which means our model has been changed to
\begin{equation}
\tilde f(x) = w_J \tilde f^{J-1}(x),
\end{equation}
with $\tilde f^{J-1}$ has the stander statistics( or $\gamma$ and $\eta$ by adaptivity). If we don't use BN, we know that for any $w_j$ with $j = 1:J-1$ can effect the $f(x)$ remarkably like $w_j = 0$ makes this network degeneration or $w_j = -w_j$ changes the prediction of $f(x)$ totally. Without normalization, nearly every update would have an extreme effect on the statistics of $f^{J-1}$.

Now suppose that $x$ is drawn from a unit Gaussian i.e:
\begin{equation}
x \sim \mathcal{N}(0,1).
\end{equation}
 Then the original method $f$ is also Gaussian but not unit it depended w.r.t $\bm{w}$:
\begin{equation}
f \sim \mathcal{N}(\mu({\bm{w}}),\sigma(\bm{w})).
\end{equation}
But if we use BN, here $\tilde h^{J-1}$ becomes unit Gaussian again, i.e
\begin{equation}
\tilde f^{J-1} \sim \mathcal{N}(0,1).
\end{equation}  So $\tilde f$ is an Gaussian only depends on $\omega_J$,
\begin{equation}
\tilde f \sim \mathcal{N}(\mu(w_J),\sigma(w_J)).
\end{equation}  which means we just need to trained $w_J$ all $w_j$ for $j=1:J-1$ make no sense for this model. So, what we only need to train is only $w_J$, this make the training process very fast.

Those results comes from those expression from this book:
``Batch normalization has thus made this model significantly easier to learn. In this example, the ease of learning of course came at the cost of making the lower layers useless. In our linear example, the lower layers no longer have any harmful effect, but they also no longer have any beneficial effect. This is because we have normalized out the first and second order statistics, which is all that a linear network can influence. In a deep neural network with nonlinear activation functions, the lower layers can perform nonlinear transformations of the data, so they remain useful. Batch normalization acts to standardize only the mean and variance of each unit in order to stabilize learning, but allows the relationships between units and the nonlinear statistics of a single unit to change."

But to understand this exactly, we need some basic idea for statistics I think.

%\section{Dropout}
%\include{6DL/dropout}

\section{Introduction}
Consider the (fully connected) neural network
\begin{equation}\label{nn}
\begin{aligned}
f(x)=W_2g(W_1x+b_1)+b_2,\\
W_2\in\mathbb{R}^{c\times n_1},
W_1\in\mathbb{R}^{n_1\times n},
\end{aligned}
\end{equation}
where $x\in\mathbb{R}^n$ is the input vector, $g$ is a non-linear function (activation).

Every update of $f$ is done with a (randomly chosen) mini-batch \[X=\{x^i|i=1,...,m\}.\]
Denote $\theta=(W_2,W_1,b_2,b_1)$, each (SGD) update can be written as
\begin{equation}\label{SGD}
\theta=\theta-\eta\nabla_\theta \frac{1}{m}\sum_{i=1}^m\|f(x^i)-p^i\|_2^2,
\end{equation}
where $\eta$ is called the step size, or the learning rate.

To improve the performance of \eqref{SGD}, there are many different normalization methods, among which are
\begin{enumerate}
	\item batch normalization (BN), 2015 \cite{ioffe2015batch},
	\item layer normalization (LN), 2016 \cite{2016arXiv160706450L},
	\item weight normalization (WN), 2016 \cite{2016arXiv160207868S},
	\item cosine normalization (CN), 2017 \cite{2017arXiv170205870L},
\end{enumerate}
We also consider an additional spectral norm regularization (SN), 2017 \cite{2017arXiv170510941Y}.

\newpage
%All of them, denoted as a function $h(x,X)$,
%are added to \eqref{nn} like
%\begin{equation}
%	f(x)=W_2g(h(W_1x+b_1))+b_2,
%\end{equation}
%and have the same scheme
%\begin{equation}
%	\begin{aligned}
%		h(y)=\gamma\cdot\frac{y-\mu}{\sigma}+\beta,\\
%		y=Wx+b,\quad x\in X,
%	\end{aligned}
%\end{equation}
%(we also denote $y^i=Wx^i+b$) where,
%\begin{enumerate}
%	\item for BN,
%		\begin{equation}
%			\mu=\frac{1}{m}\sum_{i=1}^m y^i,\quad \sigma=\sqrt{\frac{1}{m}\sum_{i=1}^m(y^i-\mu)^2}.
%		\end{equation}
%		$\mu,\sigma\in\mathbb{R}^n$, and the division by $\sigma$ is element-wise.
%
%		$\gamma,\beta$ are trainable parameters.
%	\item for LN, recall that $W_1\in\mathbb{R}^{n_1\times n}$, and denote $\mathbf{1}=(1,...,1)^T\in\mathbb{R}^{n_1}$,
%		\begin{equation}
%		\mu=\frac{1}{n_1}\sum_{i=1}^{n_1}y_i,\quad \sigma=\sqrt{\frac{1}{n_1}\sum_{i=1}^{n_1}(y-\mu)^2},
%		\end{equation}
%		where $y_i$ is the $i$th element of $y$, and $\mu,\beta\in\mathbb{R}$.
%
%		$\gamma=1,\beta=0$.
%	\item for WN,
%		\begin{equation}
%		\mu=0,\quad
%		\sigma\in\mathbb{R}^{n_1},\ \sigma_j=\|w_j\|_2,\ w_j\text{ is the $j$th row of $W$}.
%		\end{equation}
%		The division by $\sigma$ is element-wise.
%
%		$\gamma$ is trainable, $\beta=0$.
%	\item for CN,
%		\begin{equation}
%		\mu=0,\quad
%		\sigma\in\mathbb{R}^{n_1},\ \sigma_j=\|x\|_2\|w_j\|_2,\ w_j\text{ is the $j$th row of $W$}.
%		\end{equation}
%		$\gamma=1,\beta=0$.
%	\item for SN,
%		\begin{equation}
%			\mu=0,\sigma=1,\gamma=1,\beta=0,
%		\end{equation}
%		i.e. $h(x)=x$. SN in fact only changes the loss function $L(X)$ to
%		\begin{equation}
%			L_\text{SN}(X)=\frac{1}{m}\sum_{i=1}^m\|f(x^i)-p^i\|_2^2+\frac{\lambda}{2}\sum_{l=1}^{n_L}\sigma(W_l)^2,
%		\end{equation}
%		where $\lambda$ is a hyperparameter, $n_L=2$ is the number of layers, and $\sigma(W)$ is the spectral of $W$,
%		\begin{equation}
%			\sigma(W)=\max_{x\neq0}\frac{\|Wx\|_2}{\|x\|_2}.
%		\end{equation}
%\end{enumerate}

\newpage
\section{Ideas behind the normalizations}
\subsection{Batch Normalization}
Batch normalization aims at solving the so called \textbf{internal covariate shift} (ICS) problem.

By definition, ICS means the change in the distribution of network activations, due to the change in network parameters during training.

In neural networks, the output of the first layer feeds into the second layer, the output of the second layer feeds into the third, and so on. When the parameters of a layer change, so does the distribution of inputs to subsequent layers\footnote{See \href{this}{https://machinelearning.wtf/terms/internal-covariate-shift/}.}.

That is to say, the distribution of the random variable
\begin{equation}
z(x)=Wx+b,x\in X
\end{equation}
changes with $W$ and $b$ changing.

Notice that $y$ is the input of the activation $g$.

BN tries to keep the distribution of each output in each layer, or at least to keep some statistics of them. It in fact keeps the mean and variance, with a possible scale $\gamma$ and shift $\beta$ to the distribution.

As a result, BN can be written as
\begin{equation}
h_{\text{BN}}(x,X,W,b,\gamma,\beta)=\gamma\cdot\frac{z-\mu}{\sigma}+\beta,
\end{equation}
where
\begin{equation}
\begin{aligned}
z=Wx+b,\quad x\in X,\\
z^i=Wx^i+b,\quad x^i\in X,\\
\mu=\frac{1}{m}\sum_{i=1}^m z^i,\quad \sigma=\sqrt{\frac{1}{m}\sum_{i=1}^m(z^i-\mu)^2}.\\
\end{aligned}
\end{equation}

Many people argue that the variance should use Bessel's correction.
\newpage
\subsection{Layer Normalization}
Layer normalization considers another distribution, the one of the random variable for any given input $x$,
\begin{equation}
z_i=(Wx+b)_i,\quad i\in I.
\end{equation}
Like BN, It also tries to keep the mean and the variance. It can be written as
\begin{equation}
h(z)=\frac{z-\mu}{\sigma},\quad z=Wx+b,
\end{equation}
where
\begin{equation}
\begin{aligned}
\mu=\frac{1}{n_1}\sum_{i=1}^{n_1}z_i,\quad \sigma=\sqrt{\frac{1}{n_1}\sum_{i=1}^{n_1}(z_i-\mu)^2},\\
\end{aligned}
\end{equation}
where $z_i$ is the $i$th element of $z$.

\subsection{Weight Normalization}
Weight normalization considers the distribution of the random variable
\begin{equation}
w_i, i\in I,
\end{equation}
where $w_i$ is the $i$th row of $W$.

It tries to normalize $w_i$, i.e. keep the 2-norm of $w_i$ to $1$, with an extra scale $\gamma_i\in\mathbb{R}$ on it. WN finally can be written as
\begin{equation}
h(x,W,b,\gamma),\quad(h(x,W,b,\gamma))_i=\gamma_i\frac{w_ix+b_i}{\|(w_i,b_i)\|_2}.
\end{equation}

\subsection{Cosine Normalization}
Cosine normalization normalizes the random variable
\begin{equation}
u_i=(Wx+b)_i,\quad i\in I,
\end{equation}
for any given $x$. It can be written as
\begin{equation}
h(x,W,b),\quad (h(x,W,b))_i=\frac{w_ix+b_i}{\|(x;1)\|_2\|(w_i,b_i)\|_2},
\end{equation}
where $(x;1)\in\mathbb{R}^{n+1}$, $(w_i,b_i)^T\in\mathbb{R}^{n+1}$.

\subsection{Spectral Norm Regularization}
Spectral normalization devotes to normalize the parameters $W$, i.e. $W_1$ and $W_2$ in this case. It chooses to control the spectral of $W$,
\begin{equation}
\sigma(W)=\max_{x\neq0}\frac{\|Wx\|_2}{\|x\|_2}.
\end{equation}

The authors of SN argues that an advantage of smaller $\sigma(W)$ is that $Wx$ is less sensitive to the perturbation $\xi$ on $x$, because
\begin{equation}
\frac{\|(W(x+\xi)+b)-(Wx+b)\|}{\|\xi\|}=\frac{\|W\xi\|}{\|\xi\|}\leq\sigma(W).
\end{equation}
$\sigma(W)$ is thus added to the loss function, and gives
\begin{equation}
L_\text{SN}(X)=\frac{1}{m}\sum_{i=1}^m\|f(x_i)-y_i\|_2^2+\frac{\lambda}{2}\sum_{l=1}^{n_L}\sigma(W_l)^2.
\end{equation}


\documentclass[a4paper,12pt]{scrartcl}

\usepackage{bm,amsmath,url,graphicx}
\usepackage{palatino}
\usepackage{color, xcolor}
\usepackage{listings}

\newcommand{\n}{\mathbf{ n}}
\newcommand{\h}{\mathbf{ h}}
\newcommand{\x}{\mathbf{ x}}
\newcommand{\y}{\mathbf{ y}}
\newcommand{\w}{\mathbf{ w}}
\newcommand{\HH}{\mathbf{ H}}
\newcommand{\R}{\mathbf{ R}}
\newcommand{\C}{\mathbf{ C}}
\newcommand{\thb}{{\boldsymbol{\theta}}}
\newcommand{\mub}{{\boldsymbol{\mu}}}
\newcommand{\python}{{\fbox{\texttt{\bfseries python}}\quad}}
\newcommand{\pen}{{\fbox{\texttt{\bfseries pen\&paper}}\quad}}

\renewcommand{\familydefault}{\rmdefault}

\begin{document}
\section*{SGN-41007 Pattern Recognition and Machine Learning}
\emph{Exercise Set 6: February 15--February 17, 2017}
\bigskip
\sloppy

\lstdefinestyle{mystyle}{
  belowcaptionskip=1\baselineskip,
  breaklines=true,
  frame=single,
  xleftmargin=\parindent,
  language=Python,
  showstringspaces=false,
  basicstyle=\ttfamily,
  keywordstyle=\bfseries\color{green!40!black},
  commentstyle=\itshape\color{purple!40!black},
  identifierstyle=\color{blue},
  stringstyle=\color{orange},
  moredelim=**[is][\color{red}]{@}{@},
}

\lstset{language=Python,style=mystyle}

\noindent
Exercises consist of both pen\&paper and computer assignments.
Pen\&paper questions are solved at home before exercises, while
computer assignments are solved during exercise hours. The
computer assignments are marked by  \python and
Pen\&paper questions by  \pen
\medskip

\noindent\textbf{Note: Python exercises can not be executed in Pinni ML1084.
Try to bring your own laptop with Anaconda installed. We will form
groups around those machines that have Keras.}

\begin{enumerate}

\item \pen \emph{Derive an estimate for the variance of the random forest.}

The efficiency of the random forest relies on the variability of the
individual weak trees. However, this results in randomness of the
results, as well: If you run training 10 times, you may have 10
different results.

Increasing the number of trees helps to reduce the variation of outputs,
as seen in Figure~\ref{fig:variances}. More trees mean less variation.
However, the question remains: how many trees is enough.

The number of trees can be assessed theoretically via the following
theorem in a regression context (output of the RF is the average of
individual trees).

Consider the prediction of trees $1,2,\ldots, N$ as random variables
$T_1, T_2,\ldots, T_N$, each with identical variance $\sigma^2$.
If the predictions were independent, their average would have variance
$\sigma^2/N$. However, if the RV's have pairwise correlation coefficient
$\rho$, show that the variance of the average is given as
\[
\rho\sigma^2 + \frac{1-\rho}{N}\sigma^2
\]

\begin{figure}[b]%
\centerline{\includegraphics[width=0.6\columnwidth]{Variances.pdf}}%
\caption{Variances of random forest with different number of trees.}%
\label{fig:variances}%
\end{figure}

\eject

\item \pen {Count the number of parameters in a neural network}

Consider the traditional shallow neural network architecture
of Figure~\ref{fig:net}. Suppose our inputs are $64\times 64$
bitmaps of two categories of traffic signs.
\begin{enumerate}
\item Let the network structure be the following:
\begin{itemize}
	\item The input is $3\times 64\times 64 = 12288$-dimensional
	\item On the 1st layer there are 100 nodes (marked in blue)
	\item On the 2nd layer there are 100 nodes (marked in blue)
	\item On the 3rd (output) layer there are 10 nodes (marked in blue; one for each class)
\end{itemize}
Compute the number of parameters (coefficients) in the net.

\item Consider the following code defining a convolutional architecture
and compute the number of coefficients in this case.
\begin{lstlisting}
N = 10        # Number of feature maps
w, h = 3, 3   # Conv. window size

model = Sequential()

model.add(Convolution2D(nb_filter = N,
                        nb_col = w,
                        nb_row = h,
                        activation = 'relu',
                        border_mode = 'same',
                        input_shape = (3,64,64)))
model.add(MaxPooling2D((2,2)))

model.add(Convolution2D(nb_filter = N,
                        nb_col = w,
                        nb_row = h,
                        border_mode = 'same',
                        activation = 'relu'))
model.add(MaxPooling2D((2,2)))

model.add(Flatten())
model.add(Dense(2, activation = 'sigmoid'))
\end{lstlisting}

\item An old rule of thumb states that the number of training samples should be at least
5 times the number of coefficients. Compute the desired sample size based on this rule for (a) and (b).
\end{enumerate}

\begin{figure}%
\centerline{\includegraphics[width=0.6\columnwidth]{VanillaNeuralNet.pdf}}%
\caption{Vanilla neural network.}%
\label{fig:net}%
\end{figure}

\item \python \emph{Load Traffic sign data for deep neural network processing.}

Download an extended version of the two class German Traffic Sign
Recognition Benchmark (GTSRB) dataset from

\url{http://www.cs.tut.fi/courses/SGN-41006/GTSRB_subset_2.zip}

This time, images are in color and there are about 400 from both classes.
Edit your previous script (exercise set 4) such that the image array will
have dimension (660, 3, 64, 64): 660 images of size 64x64 and 3 color channels.
You will need \verb+numpy.transpose+ to reorder dimensions: (64, 64, 3) $\rightarrow$ (3, 64, 64).

After collecting the data, normalize all samples into range [0,1];
\emph{i.e.,} subtract \verb+numpy.min(X)+ and divide the result
by \verb+numpy.max(X)+. Make sure that you are using float (not integer)
division.

Finally, split the data to training and testing (80\% / 20\%)
using \verb+sklearn.cross_validation.train_test_split+.

\item \python \emph{Define the network in Keras.}

Define the network of Question 2b in your code.

\item \python \emph{Compile and train the net.}

\textbf{Note:} You need to set up Keras to use the Theano backend instead of
Tensorflow (the default). To this aim, your config file \verb+~/.keras/keras.json+
has to look like this:
\begin{verbatim}
{
    "image_dim_ordering": "th",
    "epsilon": 1e-07,
    "floatx": "float32",
    "backend": "theano",
    "optimizer": "None"
}
\end{verbatim}

Compile and train the network as described in

\url{https://keras.io/models/sequential/}

Use the following parameters:
\begin{itemize}
	\item \textbf{Loss:} binary crossentropy (same thing as log loss; see previous exercises)
	\item \textbf{Optimizer:} stochastic gradient descent
	\item \textbf{Minibatch size:} 32
	\item \textbf{Number of epochs:} 20
\end{itemize}
Also add the parameter \verb+metrics=['accuracy']+ as an argument of
\verb+model.compile+ and give the test data to training algorithm
\verb+model.fit(..., validation_data = [X_test, y_test])+
Then, the optimizer will report the test error every epoch.
\end{enumerate}

\end{document}


\documentclass[10pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{hyperref}

\usepackage{graphicx}
\graphicspath{ {./images/} }

\usepackage{amsmath}
\DeclareMathOperator*{\argmax}{arg\,max}

\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,
    urlcolor=cyan,
    pdftitle={Overleaf Example},
    pdfpagemode=FullScreen,
    }

\title{Chapter 2 - Multi-armed Bandits Answers}
\author{Stéphane Liem NGUYEN}
\begin{document}
\maketitle

Exercises with (\textbf{\textit{corrected}}) were corrected based on the \href{http://incompleteideas.net/book/errata.html}{Errata}.

These are my own answers and mistakes or errors are possible.

\paragraph{\textit{Exercise 2.1} (p. 28)} In $\epsilon$-greedy action selection, for the case of two actions and $\epsilon = 0.5$, what is
the probability that the greedy action is selected?

\bigskip
Recall that at each time step, $\epsilon$-greedy action selection methods select with probability $\epsilon$ a random action uniformly out of $k$ actions (if there's $k$ actions) and with probability $1-\epsilon$ one of the greedy actions with ties broken arbitrarily (for instance randomly).

From the recall, we can say that the probability of the greedy action to be selected   is $1-\epsilon = 0.5$.

\paragraph{\textit{Exercise 2.2: Bandit example} (p. 30)} Consider a $k$-armed bandit problem with $k = 4$ actions,
denoted $1$, $2$, $3$, and $4$. Consider applying to this problem a bandit algorithm using
$\epsilon$-greedy action selection, sample-average action-value estimates, and initial estimates
of $Q_1(a) = 0$, for all a. Suppose the initial sequence of actions and rewards is $A_1 = 1,
R_1 = -1, A_2 = 2, R_2 = 1, A_3 = 2, R_3 = -2, A_4 = 2, R_4 = 2, A_5 = 3, R_5 = 0$. On some of these time steps the $\epsilon$ case may have occurred, causing an action to be selected at
random. On which time steps did this definitely occur? On which time steps could this
possibly have occurred?

\bigskip
We already recalled what were the $\epsilon$-greedy action selection methods. We can recall quickly that sample-average action-value estimates compute estimates $Q_n(a)$ based on an empirical mean of the rewards received when the action $a$ is taken. The empirical means can be computed incrementally as explained in the book.

To know in what time step did the $\epsilon$ case definitely occur, we use the information that the decision-maker only picks a non-greedy action (not actions with highest current estimates) when exploring.

For this reason, we also compute estimates after each action-selection:
\begin{enumerate}
\item Take action $1$, receive reward $-1$ and update estimates with $Q_2(1)=-1$ and keep $Q_1(a)=0,\, \forall a \neq 1$. For the next action selection, the action $1$ cannot be selected from greedy selection because its estimate is lower than the rest. The action $1$ can only be picked from exploration for the next action selection.

\item Take action $2$, receive reward $1$ and update estimates with $Q_2(2)=1$ and keep $Q_2(1)=-1,\, Q_1(a)=0,\, \forall a \notin \{1, 2\}$. For the next time step, if we pick another action than $2$ then it would be due to exploration.

\item Take action $2$, receive reward $-2$ and update estimates with $Q_3(2)=\frac{1 + (-2)}{2} = -0.5$ and keep $Q_2(1)=-1,\, Q_1(a)=0,\, \forall a \notin \{1, 2\}$. For the next action selection, the action $2$ can only be picked from exploration because its estimate is lower than the estimates of actions $3$ and $4$.

\item Take action $2$, receive reward $2$ and update estimates with $Q_4(2)=\frac{1 + (-2) + 2}{3} = \frac{1}{3}$ and keep $Q_2(1)=-1,\, Q_1(a)=0,\, \forall a \notin \{1, 2\}$. For the next time step, if we pick another action than $2$ then it would be due to exploration.

\item Take action $3$, receive reward $0$ and update estimates with $Q_2(3)=0$ and keep $Q_4(2) = \frac{1}{3},\, Q_2(1)=-1,\, Q_1(4)=0$.

\end{enumerate}

The $\epsilon$ case could have occurred at all time steps but it definitely occurred  for time step $4$ and time step $5$.

\paragraph{\textit{Exercise 2.3} (p. 30)} In the comparison shown in Figure $2.2$, which method will perform best in
the long run in terms of cumulative reward and probability of selecting the best action?
How much better will it be? Express your answer quantitatively.

\bigskip
Action-value method with $\epsilon=0.01$ will perform best in the long run in terms of cumulative reward and probability of selecting the best action.

The probability of selecting the best action $a_*$ for $\epsilon$-greedy methods with $\epsilon > 0$ (with sample-averages and on stationary problems) can be written as the probability of taking an optimal action when exploring ($\epsilon$ case, action selected at random) plus the probability of taking a greedy action (we can add them because in one single action selection, we cannot behave greedily at the same time as non-greedily. Moreover the second part with greedy action assumes that the estimates of the optimal actions converge to the action values).

\begin{equation}
\lim_{t\to\infty}\mathbb{P}[A_t = a_*] = \epsilon\cdot\frac{1}{k}+ 1 - \epsilon
\end{equation}

This formula does not work for $\epsilon = 0$ (greedy) because if $\epsilon=0$, some actions can be selected a finite number of times asymptotically. As consequence, some estimates might not converge to the true action values.

For $10$-armed testbed, $k=10$ and by applying the formula with $\epsilon=0.1$ we get $0.01 + 1 - 0.1 = 0.91$. If we apply the formula with $\epsilon=0.01$ then we get $0.001 + 1 - 0.01 = 0.991$.

\clearpage
\paragraph{\textit{Exercise 2.4} (p. 33)} If the step-size parameters, $\alpha_n$, are not constant, then the estimate $Q_n$ is
a weighted average of previously received rewards with a weighting different from that
given by ($2.6$). What is the weighting on each prior reward for the general case, analogous
to ($2.6$), in terms of the sequence of step-size parameters?

\bigskip
Recall that in this section, the book focused on estimating/tracking incrementally the value of a single action for nonstationary problem and this was done by using a constant-step-size parameter in the incremental update rule that we rewrite below
\begin{equation}
Q_{n+1} \doteq Q_n + \alpha_n [R_n - Q_n]
\end{equation}
where $R_n$ is the reward received after the $n$-th selection of the action and $\alpha_n=\frac{1}{n}$ for the incremental implementation of sample-averages (empirical mean) or $\alpha_n=\alpha \in (0, 1]$ for the exponential recency-weighted average.

Equation $(2.6)$ showed that if we update estimates using exponential recency-weighted average then $Q_{n+1}$ was a weighted average of past rewards and the initial estimate $Q_1$.

We can try to do the same but for the general case where $\alpha_n$ is not necessarily $\frac{1}{n}$ or a constant $\alpha \in (0,1]$.
\begin{equation}
\begin{split}
Q_{n+1} &= Q_n + \alpha_n [R_n - Q_n] = \alpha_n R_n + (1-\alpha_n) Q_n\\
&= \alpha_n R_n + (1-\alpha_n) [\alpha_{n-1} R_{n-1} + (1-\alpha_{n-1})Q_{n-1}]\\
&= \alpha_n R_n + (1-\alpha_n) \alpha_{n-1} R_{n-1} + (1-\alpha_n)\cdot (1-\alpha_{n-1})Q_{n-1}\\
&= Q_1 \prod_{k=1}^n (1-\alpha_k) + \sum_{i=1}^n \alpha_i R_i \prod_{j=i+1}^n (1-\alpha_j)
\end{split}
\end{equation}

Each reward $R_i$ is weighted by $\alpha_i \prod_{j=i+1}^n (1-\alpha_j)$.

\bigskip
We can verify by recurrence that it's a weighted sum, i.e $\prod_{k=1}^n (1-\alpha_k) + \sum_{i=1}^n \alpha_i \prod_{j=i+1}^n (1-\alpha_j) = 1$. For $n=1$ we have
\begin{equation}
1-\alpha_1 + \alpha_1 = 1
\end{equation}
Suppose the formula is true for $n$, let's prove that it holds for $n+1$. Let $H_n = \prod_{k=1}^n (1-\alpha_k) + \sum_{i=1}^n \alpha_i \prod_{j=i+1}^n (1-\alpha_j) = 1$

\begin{equation}
\prod_{k=1}^{n+1} (1-\alpha_k) + \sum_{i=1}^{n+1} \alpha_i \prod_{j=i+1}^{n+1} (1-\alpha_j) = H_n \cdot (1-\alpha_{n+1}) + \alpha_{n+1} = 1
\end{equation}

From the general formula, we can obtain equation $(2.6)$ of the book for exponential recency-weighted average as well as the sample-average methods where $R_i$'s are weighted by $\frac{1}{i} \cdot \prod_{j=i+1}^n \left(1-\frac{1}{j}\right) = \frac{1}{i} \cdot \frac{n-1}{n} \cdot \frac{n-2}{n-1} \cdot \hdots \cdot \frac{i}{i+1} = \frac{1}{n}$

\clearpage
\paragraph{\textit{Exercise 2.5 (programming)} (p. 33)} Design and conduct an experiment to demonstrate the
difficulties that sample-average methods have for nonstationary problems. Use a modified version of the $10$-armed testbed in which all the $q_*(a)$ start out equal and then take independent random walks (say by adding a normally distributed increment with mean $0$ and standard deviation $0.01$ to all the $q_*(a)$ on each step). Prepare plots like Figure $2.2$
for an action-value method using sample averages, incrementally computed, and another action-value method using a constant step-size parameter, $\alpha = 0.1$. Use $\epsilon = 0.1$ and
longer runs, say of 10,000 steps.

\bigskip
\begin{figure}[h]
\centering
\includegraphics[width=\textwidth]{./non-stationary-testbed/images/avg_reward.png}
\includegraphics[width=\textwidth]{./non-stationary-testbed/images/percent_optimal_action.png}
\caption{Average performance of $\epsilon$-greedy action-value methods with different action-value estimates on a modified version of the 10-armed testbed with nonstationary reward distributions.
These data are averages over $2000$ runs with different nonstationary bandit problems and with longer runs of $10000$ steps. All methods used $\epsilon=0.1$ and we compared the sample-average methods with the exponential recency-weighted average method for estimating values.}
\end{figure}
We can observe that the sample-average methods struggle to track the optimal action that can change over time compared to exponential recency-weighted average that gives more weight to recent rewards than old rewards. The code for the exercise can be found \href{https://github.com/Zenchiyu/learning-rl/tree/develop/Intro_RL_Sutton_book/Chap2-Multi-Armed-Bandits/non-stationary-testbed#non-stationary-testbed}{here}.

\paragraph{\textit{Exercise 2.6: Mysterious Spikes} (p. 35)} The results shown in Figure $2.3$ should be quite reliable because they are averages over $2000$ individual, randomly chosen $10$-armed bandit tasks.
Why, then, are there oscillations and spikes in the early part of the curve for the optimistic method? In other words, what might make this method perform particularly better or worse, on average, on particular early steps?

\bigskip
In the book, optimistic initial action-value estimates method was applied on stationary problems (reward distributions do not change over time) and we track action-values by exponential recency-weighted average (constant step-size parameter) to use the bias caused by initial estimates (if we used sample-averages, the bias disappears once all actions are selected at least once).

For the first $k$ action-selections, the method (with greedy selections) explored most of the time all the existing actions because the initial estimates are optimistic compared to the reward distributions with means given by a standard normal distribution (rewards received are most of the time lower than $5$). Consequently, the percentage of action selections out of $2000$ runs that are optimal are initially around $10$ percent ($1/k=0.1$ for $k=10$).

After the $k$ first action-selections, all action selections are based on estimates computed from a weighted sum of past rewards and initial estimates of $5$ where the bias decreases over time as well as the exploration.

The mysterious spikes come from the frequent selection of the optimal action just after the first $k$ action selections. This might be due the first reward of the optimal action (obtained most of the time from the $k$ first action selections) that can be in average higher sometimes than the rest and this implies a higher $Q_2(a)$ for the optimal action. After the selection of the optimal action, the impact of the initial estimate and the estimates of the action value for the action are both reduced. This gives more chance for the other non optimal actions to be selected.

\paragraph{\textit{Exercise 2.7: Unbiased Constant-Step-Size Trick} (\textit{corrected}) (p. 35)} In most of this chapter we have used
sample averages to estimate action values because sample averages do not produce the
initial bias that constant step sizes do (see the analysis leading to ($2.6$)). However, sample averages are not a completely satisfactory solution because they may perform poorly on nonstationary problems. Is it possible to avoid the bias of constant step sizes while retaining their advantages on nonstationary problems? One way is to use a step size of
\begin{equation}
\beta_n \doteq \alpha/\bar{o}_n
\end{equation}

to process the $n$th reward for a particular action, where $\alpha > 0$ is a conventional constant step size, and $\bar{o}_n$ is a trace of one that starts at $0$:
\begin{equation}
\bar{o}_n \doteq \bar{o}_{n-1} + \alpha \cdot (1 - \bar{o}_{n-1}),\quad \textrm{for } n > 0,\quad \textrm{with } \bar{o}_0 \doteq 0
\end{equation}

Carry out an analysis like that in ($2.6$) to show that $Q_n$ is an exponential recency-weighted average \textit{without initial bias}.

\bigskip
With the unbiased constant-step-size trick, the step size parameter $\beta_n$ starts at $1$ and decreases until it reaches the constant step size $\alpha \in (0, 1]$. Because it starts at $1$, the initial bias disappears once all actions are selected at least once. Moreover, because the step size parameter doesn't decrease below $\alpha$, we obtain some advantages of exponential recency-weighted average for tracking nonstationary problems.

We start to plugging the new step size into the incremental update rule and then carry out an analysis like ($2.6$) of the book.
\begin{equation}\label{ex_2_7_incr_update_rule}
\begin{split}
Q_{n+1} &\doteq Q_n + \beta_n [R_n - Q_n],\quad \beta_n \doteq \alpha/\bar{o}_n\\
&=Q_1 \prod_{k=1}^n (1-\beta_k) + \sum_{i=1}^n \beta_i R_i \prod_{j=i+1}^n (1-\beta_j)\\
&=Q_1 \cdot (1 - \beta_1) \prod_{k=2}^n (1-\beta_k) + \sum_{i=1}^n \beta_i R_i \prod_{j=i+1}^n (1-\beta_j)\\
&= \sum_{i=1}^n \beta_i R_i \prod_{j=i+1}^n (1-\beta_j) = \sum_{i=1}^n \frac{\alpha}{\bar{o}_i} R_i \prod_{j=i+1}^n (1-\frac{\alpha}{\bar{o}_j})
\end{split}
\end{equation}
by using $\beta_1 = \alpha/\alpha = 1$ because $\bar{o}_1 = \alpha$. Each reward $R_i$ is weighted by $\frac{\alpha}{\bar{o}_i} \prod_{j=i+1}^n (1-\frac{\alpha}{\bar{o}_j})$.

If we try to apply the same procedure (or substitute in ($2.6$) of the book) but on the trace of one we get
\begin{equation}
\begin{split}
\bar{o}_n = \sum_{k=0}^{n-1} (1-\alpha)^k \alpha,\quad \textrm{for } n > 0,\quad \textrm{with } \bar{o}_0 \doteq 0
\end{split}
\end{equation}
and this converges to $1$ (geometric series). So as $n \to \infty$, the weights on each reward are like exponential recency-weighted average.

%Substituting that back into our equation \ref{ex_2_7_incr_update_rule} gives
%\begin{equation}
%Q_{n + 1} = \sum_{i=1}^n \frac{\alpha}{\bar{o}_i} R_i \prod_{j=i+1}^n (1-\frac{\alpha}{\bar{o}_j})
%\end{equation}

\paragraph{\textit{Exercise 2.8: UCB Spikes} (p. 36)} In Figure $2.4$ the UCB algorithm shows a distinct spike
in performance on the $11$th step. Why is this? Note that for your answer to be fully satisfactory it must explain both why the reward increases on the $11$th step and why it decreases on the subsequent steps. Hint: If $c = 1$, then the spike is less prominent.

\bigskip
Upper-Confidence-Bound action selection method selects action $A_t$ at time step $t$ based on
\begin{equation}\label{UCB}
A_t \doteq \argmax\limits_{a}\left[Q_t(a) + c \sqrt{\frac{\ln t}{N_t(a)}}\right]
\end{equation}

where $c > 0$ is the degree of exploration, $N_t(a) \geq 0$ the number of times we selected the action $a$ prior to time $t$ and $Q_t(a)$ the estimated action value for action $a$.
The book explains that if $N_t(a) = 0$, then the action $a$ is considered to be a maximizing action. If we interprete this as always randomly choosing between actions that were never selected when they still exist and using \ref{UCB} when they no longer exist, then the method will try all the $k$ actions at least once in the $k$ first action selections before using \ref{UCB}. Combined with how we generated the reward distributions for the $10$-armed testbed, it can explain why the average reward (out of $1000$ runs) is around $0$ for the $10$ first actions selections. The $11$th action selection is then based on \ref{UCB} where the measure of the uncertainty or variance (on the right of the $+$) is the same for all actions. Consequently, choosing an action for that time step is just based on the first rewards sampled for each action from their reward distributions and for this reason, the best action is often picked.

After the spike of the $11$th step, the uncertainty term of the action that was selected is reduced, giving a bit more chance for the other actions to be selected.
This causes the average reward to drop if the $11$th action selection was often the best action.

If the degree of exploration $c$ is $1$ instead of $2$, then the effect of the uncertainty term is lower. Action selections for $c=1$ would rely more on the estimated action values and because of this, the best action would still be often picked just after the $11$th step (still less than the $11$th step but the drop is less pronounced than for $c=2$). The average reward for $c=1$ on steps after the spike does not decrease as much as for $c=2$ as explained in the hint.

\paragraph{\textit{Exercise 2.9} (p. 37)} Show that in the case of two actions, the soft-max distribution is the same
as that given by the logistic, or sigmoid function often used in statistics and artificial
neural networks.

\bigskip
We can first recall the formula for the soft-max distribution based on action preferences
\begin{equation}
\mathbb{P}[A_t = a] \doteq \frac{e^{H_t(a)}}{\sum_{b=1}^k H_t(b)} \doteq \pi_t(a)
\end{equation}
where $H_t(a)$ is the numerical \textit{preference} of action $a$.

And for two actions $1$ and $2$, we have
\begin{equation}
\mathbb{P}[A_t = 1] = \frac{e^{H_t(1)}}{ e^{H_t(1)} + e^{H_t(2)}} = \frac{1}{1 +  e^{H_t(2) -  H_t(1)}} = \frac{1}{1 +  e^{-x}}
\end{equation}
with $x = H_t(1)- H_t(2)$

\paragraph{\textit{Exercise 2.10} (p. 41)} Suppose you face a $2$-armed bandit task whose true action values change randomly from time step to time step. Specifically, suppose that, for any time step, the true values of actions $1$ and $2$ are respectively $10$ and $20$ with probability $0.5$ (case
A), and $90$ and $80$ with probability $0.5$ (case B). If you are not able to tell which case you face at any step, what is the best expected reward you can achieve and how should you behave to achieve it?

Now suppose that on each step you are told whether you are facing case A or case B (although you still don't know the true action values). This is an associative search task. What is the best expected reward you can achieve in this task, and how should you behave to achieve it?

\bigskip
If we're not able to tell which case we face at any step, then the problem can be converted into a single stationary $2$-armed bandit task where the true values of the actions would be $(10 + 90)/2 = 50$ for action $1$ and $(20 + 80)/2 = 50$ for action $2$. The best expected reward we can achieve would be $50$. We can behave either randomly by selecting actions uniformly or always pick the first action or always pick the second action.

If we know what case we face at each step, the best expected reward we can achieve would be $(20 + 90)/2=55$ by taking action $2$ when we face case A and by taking action $1$ when we face case B.

\paragraph{\textit{Exercise 2.11 (programming)} (p. 44)} Make a figure analogous to Figure $2.6$ (parameter study) for the nonstationary case outlined in \textbf{\textit{Exercise 2.5}}. Include the constant-step-size $\epsilon$-greedy algorithm with
$\alpha=0.1$. Use runs of 200,000 steps and, as a performance measure for each algorithm and parameter setting, use the average reward over the last 100,000 steps.

\bigskip
%\begin{figure}[h]
%\centering
%\includegraphics[width=\textwidth]{./non-stationary-testbed/images/avg_reward.png}
%\caption{}
%\end{figure}
The code for the exercise can be found \href{https://github.com/Zenchiyu/learning-rl/blob/develop/Intro_RL_Sutton_book/Chap2-Multi-Armed-Bandits/non-stationary-testbed/parameter_study_non_stationary.py}{here}.

\end{document}


%!TEX root = vorlage.tex

\subsection{Local Models}\label{sec:ap4}

A slightly better model than the baseline used the coordinate of the pixel in
the image as two additional features ($x, y \in \mathbb{R}$) as well as
$\SI{3}{\pixel}$ erosion and dilation. This is known as a morphological
opening. This model achieved an higher accuracy and a higher precision, but the
recall improved most. As one can see in the results
in~\cref{table:results-local-model-experiments}, most of the improvement is due
to the coordinate features. The confusion matrix is given
in~\cref{table:cm-model-303}.

\begin{table}[ht]
    \centering
    \begin{tabular}{lrrr}
    \toprule
    Model              & Accuracy & Precision & Recall \\\midrule
    Baseline (BL)      & $\SI{92.88}{\percent}$  & $\SI{76.13}{\percent}$ & $\SI{32.94}{\percent}$\\
    BL + CFs           & $\SI{93.42}{\percent}$  & $\underline{\SI{76.98}{\percent}}$ & $\SI{40.65}{\percent}$\\
    BL + Opening       & $\SI{91.07}{\percent}$  & $\SI{74.76}{\percent}$ & $\SI{4.50 }{\percent}$\\
    BL + CFs + Opening & $\underline{\SI{93.51}{\percent}}$  & $\SI{76.76}{\percent}$ & $\underline{\SI{42.30}{\percent}}$\\
    \bottomrule
    \end{tabular}
    \caption{Results of the local model experiments. The coordinate features (CFs)
             improved the quality.}
    \label{table:results-local-model-experiments}
\end{table}

An example for the segmentation can be seen
in~\cref{fig:model-303-segmentation}. The model fits some of the edges of the
medical instruments very well, but has severe problems with specular
highlights. It is also not able to deal with the red reflection on the medical
instruments head and very dark, but not black areas with little image
information.

\begin{figure}[t]
    \centering
    \includegraphics[width=\linewidth]{images/model-303-img_06_raw-overlay.png}
    \caption{Example segmentation by the local model with opening.}
    \label{fig:model-303-segmentation}
\end{figure}

Astonishingly, the accuracy of models which made use of a tiny patch around the
pixel which was to classify decreased to about $\SI{92.11}{\percent}$, the
precision is $\SI{42.30}{\percent}$ and the recall is $\SI{76.74}{\percent}$.
The classification of one image took about $\SI{1.54}{\second}$. In theory, the
model could simply set the weights to 0 and thus ignoring the other features.
This means such a model should never be worse than a model without the
surrounding pixel information. Besides programming errors and numerical
problems, the problem might be the increased number of parameters. It is
possible that the optimization was stuck in a local minimum.