\selectlanguage{american}

%%%%%%%%%%%%%%%%%%%%%%%%%% BASE MATH %%%%%%%%%%%%%

\newcommand*{\natrange}[3]{ #2 \le #1 \le #3 }   % #1 in {#2 .. #3} or #2 \le #1 \le #3
\newcommand*{\natrangeex}[3]{ #2 \le #1 < #3 } % #2 \le #1 < #3
\newcommand*{\natbelow}[2]{{  #1 \le #2}}
\newcommand*{\natabove}[2]{{ #1 \ge #2}}
\newcommand*{\natzero}[1]{{ #1 \ge 0}} % 1 \leq i
\newcommand*{\posnats}{ \mathbb{Z}^{+} } % 1,2,3,...
\newcommand*{\intabove}[2]{{ #1 \ge #2}} % 1 \leq i

\newcommand*{\bigdjunion}{\bigcup} % union of disjoint sets \bigsqcup
\newcommand*{\djunion}{\cup} % union of disjoint sets \sqcup

\newcommand{\init}{\mathit{init}}

%Proposition Logic
\newcommand*{\false}{\mathrm{false}}
\newcommand*{\true}{\mathrm{true}}

%Temporal Logic
\newcommand*{\TlFinally}{\mathbf{F}\,}    % \operatorname
\newcommand*{\TlUntil}{\,\mathbf{U}\,}
\newcommand*{\TlFinallyBounded}[1]{\mathbf{F}_{\le #1}\,}
\newcommand*{\TlUntilBounded}[1]{\,\mathbf{U}_{\le #1}\,}
\newcommand*{\TlOnce}{\,\mathbf{O}\,}

%Probabilistic
\newcommand*{\prob}{p}
\newcommand*{\ProbDomain}{\mathbb{Q}} %or Interval {q\in \mathbb{Q}| 0\le q\le 1}
\newcommand*{\ProbRange}{[0,1]}
\newcommand*{\ProbNonZeroRange}{(0,1]}
\newcommand*{\Prob}{\mathit{Pr}}
\newcommand*{\ProbOf}[1]{\Prob^{#1}}
\newcommand*{\ProbInState}[1]{\Prob_{#1}}
\newcommand*{\ProbOfInState}[2]{\ProbInState{#2}^{#1}}
\newcommand*{\ProbSampleSpace}{\Omega}
\newcommand*{\ProbSetOfEvents}{\mathfrak{E}} %\mathit{Events}
\newcommand*{\ExpectedProb}{E}

%Generic Transition systems
\newcommand{\InfPathName}{InfPaths}

\newcommand{\emptysequence}{\scalebox{0.8}{$\varnothing$}}
\newcommand{\state}{s}
\newcommand{\AtProps}{P}
\newcommand{\States}{S}
\newcommand*{\Labels}{L}
\newcommand*{\lbl}{\ell}
\newcommand*{\infpath}{\widehat{\pi}}
\newcommand*{\initinfpath}{\widehat{\pi}_\init}
\newcommand*{\ancpath}{\infpath_{\anchor}}
\newcommand*{\dotspath}{\dotso}
\newcommand*{\finpath}{\pi}
\newcommand*{\initfinpath}{\pi_\init}
\newcommand*{\ancfinpath}{\finpath_{\anchor}}
\newcommand*{\anctuple}[2]{[#1,#2]}
\newcommand*{\cpath}{\varrho}
\newcommand*{\finobspath}{\finpath_o}
\newcommand*{\infobspath}{\infpath_o}
\newcommand{\pathprefix}[1]{\infpath[..#1]}
\newcommand{\ancpathprefix}[1]{\ancpath[..#1]}
\newcommand{\Paths}{\mathit{\InfPathName}}
\newcommand{\InitPaths}{\mathit{\InfPathName}_\init}
\newcommand{\AncPaths}{\mathit{\InfPathName}_{\anchor}}
\newcommand{\AncPathsStartingFrom}[1]{\mathit{\InfPathName}_{#1}}
\newcommand*{\firstStateOfAncpath}[1]{\mathit{first}(#1)}
\newcommand{\FinPaths}{\mathit{FinPaths}}
\newcommand{\InitFinPaths}{\mathit{FinPaths}_\init}
\newcommand{\AncEmptyPaths}{\mathit{EmptyPaths}}
\newcommand{\AncFinPaths}{\mathit{FinPaths}_{\anchor}}
\newcommand{\AncFinPathsStartingFrom}[1]{\mathit{FinPaths}_{#1}}
\newcommand*{\AncFinPathsOfInterest}{\Pi}
\newcommand*{\AncFinPathsOfInterestStartingFrom}[1]{{\restrictTo{\AncFinPathsOfInterest}{}{#1}}}
\newcommand*{\FinObservablePathsOf}[1]{\mathit{\InfPathName}_{\mathit{fobs}}^{#1}}
\newcommand*{\ObservablePathsOf}[1]{\mathit{\InfPathName}_{\mathit{obs}}^{#1}}
\newcommand*{\ChoicePathsOf}[1]{\mathit{Paths}_{\mathit{choice}}^{#1}}
\newcommand*{\AncPathsOfInterest}{\hat{\Pi}}
\newcommand*{\AncPathsOfInterestStartingFrom}[1]{{\restrictTo{\Pi}{}{#1}}}
\newcommand*{\SetOfAncPaths}{\AncPathsOfInterest_{\anchor}}
\newcommand*{\LtLabel}{\Lambda}
\newcommand*{\Trans}{R}
\newcommand*{\Targets}{\Theta}
\newcommand*{\target}{\theta}
\newcommand*{\targetstate}{\tau}
\newcommand*{\righttargetstate}{\tau}
\newcommand*{\Prefixes}{\mathit{Pref}}
\newcommand*{\AncPrefixes}{\mathit{Pref}}
\newcommand*{\StateRootChoice}{C}
\newcommand*{\initialRootChoice}{\cmdpChoice_\init}
\newcommand*{\choiceTrans}{\mathit{succ}}
\newcommand*{\tsleadsto}{\rightsquigarrow}
\newcommand*{\pathlength}[1]{\left| #1 \right|}
\newcommand*{\pathconcat}[2]{#1{}#2}
\newcommand*{\pathjoin}[2]{#1{}#2}

%Both, Mdps and Dtmcs
\newcommand*{\Dists}{\mathit{Dists}}
\newcommand*{\dist}{\mu}
\newcommand*{\initialdist}{\mu_\init}
\newcommand*{\Scheduler}{\mathfrak{s}}
\newcommand*{\Schedulers}{\mathfrak{S}}
\newcommand*{\Cylinder}[1]{#1\raisebox{\depth}{\scalebox{0.8}{$\uparrow$}}}
\newcommand*{\restrictTo}[3]{#1\mathop{\hspace{-0.2em}\upharpoonright^{#2}_{#3}}}
\newcommand*{\CylinderOf}[2]{\Cylinder{#2}^{#1}}

%Labeled Transition system structures
\newcommand{\Lts}{\mathit{LTS}}
\newcommand{\LtsTrans}{\mathit{R}}
\newcommand{\LtsInit}{\mathit{I}}

%Markov Chains
\newcommand*{\Mc}{\mathcal{M}}
\newcommand*{\McStates}{\States^{\Mc}}
\newcommand*{\McTargets}{\Targets^{\Mc}}
\newcommand*{\McDists}{\mathit{Dists}}
\newcommand*{\mcdiststarget}{\theta}
\newcommand*{\Mctargetstate}{\targetstate^{\Mc}}
\newcommand*{\Mcinitialdist}{\initialdist^{\Mc}}
\newcommand*{\McCylinder}[1]{\CylinderOf{\Mc}{#1}}
\newcommand*{\McSamples}{\ProbSampleSpace^{\Mc}}
\newcommand*{\McSamplesInState}[1]{\ProbSampleSpace_{#1}^{\Mc}}
\newcommand*{\McEvents}{\ProbSetOfEvents^{\Mc}}
\newcommand*{\McEventsInState}[1]{\ProbSetOfEvents_{#1}^{\Mc}}
\newcommand*{\McProb}{\ProbOf{\Mc}}
\newcommand*{\McProbInState}[1]{\ProbOfInState{\Mc}{#1}}

\newcommand{\mcdist}{\dist}
\newcommand*{\distOfState}[1]{\dist_{#1}}
\newcommand{\McTrans}{\Trans^{\Mc}}
\newcommand*{\McPaths}{\Paths^{\Mc}}
\newcommand*{\McInitPaths}{\InitPaths^{\Mc}}
\newcommand*{\McAncPaths}{\AncPaths^{\Mc}}
\newcommand*{\McAncPathsStartingFrom}[1]{\AncPathsStartingFrom{#1}^{\Mc}}
\newcommand*{\McFinPaths}{\FinPaths^{\Mc}}
\newcommand*{\McAncEmptyPaths}{\AncEmptyPaths^{\Mc}}
\newcommand*{\McInitFinPaths}{\InitFinPaths^{\Mc}}
\newcommand*{\McAncFinPaths}{\AncFinPaths^{\Mc}}
\newcommand*{\McAncFinPathsStartingFrom}[1]{\AncFinPathsStartingFrom{#1}^{\Mc}}

%Mdps
\newcommand*{\Mdp}{\mathcal{M}}
\newcommand*{\MdpDists}{\McDists}
\newcommand*{\MdpSteps}{\mathit{Steps}}  % \mathitcmr

%Choice-aware Markov Decision Processes
\newcommand*{\Choices}{\mathcal{C}}
\newcommand*{\ChoiceScheduler}{\mathfrak{c}}
\newcommand*{\choiceSchedulerWithChoiceSchedulerDistOfState}[2]{\dist^{#1}_{#2}}
\newcommand*{\choiceSchedulerDistOfState}[1]{\choiceSchedulerWithChoiceSchedulerDistOfState{\ChoiceScheduler}{#1}}
\newcommand*{\choiceSchedulerInitialDist}{\initialdist^{\ChoiceScheduler}}
\newcommand*{\StepScheduler}{\Scheduler} %
\newcommand*{\MaxStepScheduler}{\mathrm{max}}
\newcommand*{\MinStepScheduler}{\mathrm{min}}
\newcommand*{\ChoiceSchedulers}{\mathfrak{C}}
\newcommand*{\StepSchedulers}{\Schedulers}
\newcommand*{\choicepathtarget}{\righttargetstate}
\newcommand*{\choiceschedulerLeadsWithProbTo}[2]{\overset{#2}{\tsleadsto}_{#1}}

\newcommand*{\AncStepSchedulerPaths}{\AncPaths^\StepScheduler}
\newcommand*{\AncStepSchedulerPathsStartingFrom}[1]{\AncPathsStartingFrom{#1}^\StepScheduler}
\newcommand*{\AncFinStepSchedulerPathsStartingFrom}[1]{\AncFinPathsStartingFrom{#1}^\StepScheduler}
\newcommand*{\AncFinStepSchedulerPaths}{\AncFinPaths^\StepScheduler}
\newcommand*{\AncStepSchedulerPathsOfInterest}{{\restrictTo{\AncPathsOfInterest}{\StepScheduler}{}}}
\newcommand*{\AncStepSchedulerPathsOfInterestStartingFrom}[1]{{\restrictTo{\AncPathsOfInterest}{\StepScheduler}{#1}}}
\newcommand*{\AncFinStepSchedulerPathsOfInterest}{{\restrictTo{\AncFinPathsOfInterest}{\StepScheduler}{}}}
\newcommand*{\AncFinStepSchedulerPathsOfInterestStartingFrom}[1]{{\restrictTo{\AncFinPathsOfInterest}{\StepScheduler}{#1}}}
\newcommand*{\MidwayStepScheduler}[1]{{\Scheduler[#1]}}
\newcommand*{\InitFinStepSchedulerPathsOfInterest}{{\restrictTo{\AncFinPathsOfInterest}{\StepScheduler,\ChoiceScheduler_\init}{}}}

\newcommand*{\Cmdp}{\mathcal{M}}
\newcommand*{\CmdpChoices}{\Choices^{\Cmdp}}
\newcommand*{\cmdpChoice}{c}
\newcommand*{\CmdpStateRootChoice}{\StateRootChoice^{\Cmdp}}
\newcommand*{\cmdpInitialRootChoice}{\initialRootChoice^{\Cmdp}}
\newcommand*{\cmdpChoiceTrans}{\choiceTrans^{\Cmdp}}
\newcommand*{\cmdpLeadsto}{\tsleadsto_{\Cmdp}}

\newcommand*{\CmdpStates}{\States^{\Cmdp}}
\newcommand*{\CmdpTargets}{\Targets^{\Cmdp}}

\newcommand*{\CmdpScheduler}{\Scheduler^{\Cmdp}}
\newcommand*{\CmdpFinPaths}{\FinPaths^{\Cmdp}}
\newcommand*{\CmdpFinObservablePaths}{\FinObservablePathsOf{\Cmdp}}
\newcommand*{\CmdpObservablePaths}{\ObservablePathsOf{\Cmdp}}
\newcommand*{\CmdpChoicePaths}{\ChoicePathsOf{\Cmdp}}
\newcommand*{\cmdpdist}{\mu}
\newcommand*{\CmdpPaths}{\Paths^{\Cmdp}}
\newcommand*{\CmdpInitPaths}{\InitPaths^{\Cmdp}}
\newcommand*{\CmdpInitFinPaths}{\InitFinPaths^{\Cmdp}}
\newcommand*{\CmdpAncPaths}{\AncPaths^{\Cmdp}}
\newcommand*{\CmdpAncPathsStartingFrom}[1]{\AncPathsStartingFrom{#1}^{\Cmdp}}
\newcommand*{\CmdpAncFinPaths}{\AncFinPaths^{\Cmdp}}
\newcommand*{\CmdpAncEmptyPaths}{\AncEmptyPaths^{\Cmdp}}
\newcommand*{\CmdpAncFinPathsStartingFrom}[1]{\AncFinPathsStartingFrom{#1}^{\Cmdp}}
\newcommand*{\CmdpAncStepSchedulerPaths}{\AncPaths^{\Cmdp,\StepScheduler}}
\newcommand*{\CmdpAncFinStepSchedulerPaths}{\AncFinPaths^{\Cmdp,\StepScheduler}}
\newcommand*{\CmdpAncStepSchedulerPathsStartingFrom}[1]{\AncPathsStartingFrom{#1}^{\Cmdp,\StepScheduler}}
\newcommand*{\CmdpAncFinStepSchedulerPathsStartingFrom}[1]{\AncFinPathsStartingFrom{#1}^{\Cmdp,\StepScheduler}}

% First parameter here is \StepScheduler
\newcommand*{\cmdpLeadstoAdheringScheduler}[1]{\tsleadsto_{#1}} %\Cmdp,#1
\newcommand*{\CmdpFinPathsAdheringScheduler}[1]{\FinPaths^{\Cmdp,#1}}
\newcommand*{\CmdpChoicePathsAdheringScheduler}[1]{\ChoicePathsOf{\Cmdp,#1}}
\newcommand*{\CmdpCylinder}[2]{\CylinderOf{\Cmdp,#1}{#2}}
\newcommand*{\CmdpProb}[1]{\ProbOf{\Cmdp,#1}}
\newcommand*{\CmdpProbInState}[2]{\ProbOfInState{\Cmdp,#1}{#2}}
\newcommand*{\CmdpSamples}[1]{\ProbSampleSpace^{\Cmdp,#1}}
\newcommand*{\CmdpSamplesInState}[2]{\ProbSampleSpace_{#2}^{\Cmdp,#1}}
\newcommand*{\CmdpEvents}[1]{\ProbSetOfEvents^{\Cmdp,#1}}
\newcommand*{\CmdpEventsInState}[2]{\ProbSetOfEvents_{#2}^{\Cmdp,#1}}
\newcommand*{\correspondingOfCmdp}{{}}
\newcommand*{\correspondingOfMc}{{\stixleftarrowaccent{c}}}

% For Expressions
\newcommand*{\expr}{e}
\newcommand*{\dsformula}[1]{\expr{}#1}
\newcommand*{\dsvariablevaluation}[2]{$\textrm{\code{#1}}=#2$}

%=============   NAME MACROS   =============%
\let\depth\relax
\newcommand{\CSharp}{{\protect\settoheight{\dimen0}{C}C\kern-.05em \protect\resizebox{!}{\dimen0}{\protect\raisebox{\depth}{\#}}}\xspace}
% https://tex.stackexchange.com/questions/44528/how-to-make-the-correct-hash-symbol-in-c-sharp-c
\newcommand{\SSharp}{{\protect\settoheight{\dimen0}{S}S\kern-.05em \protect\resizebox{!}{\dimen0}{\protect\raisebox{\depth}{\#}}}\xspace}

\chapter{Probabilistic Systems}\label{ch:FormalFoundations}

In the literature, various formal underpinnings have been proposed to reason about the probability of specific events in a dynamically behaving system.
In this document, such formal underpinnings are summarized under the term \emph{probabilistic systems}.
This chapter introduces two probabilistic systems, namely \emph{generic Markov chains} and \emph{choice-aware Markov decision processes}.
These are later used for the probabilistic safety analysis of executable models.

Roughly speaking, a Markov chain (MC) contains for each state a probability distribution over potential successor states.
MCs can be used for the analysis of purely probabilistic models, i.e., models containing no nondeterministic choice.
\Cref{sec:FormalFoundations:GMC} describes generic Markov chains, a slightly generalized version of MCs that have been developed to better suit to executable models than standard MCs.

Markov decision processes (MDPs) combine nondeterministic choice and probabilistic choice:
for each state, a MDP contains a nondeterministic choice between different probability distributions over potential successor states.
An intuitive way to look at a transition in a MDP is that \emph{first} one probability distribution of the active state is selected \emph{nondeterministically}, and \emph{secondly}, this probability distribution is used to select a successor state \emph{probabilistically}.
Choice-aware Markov decision processes (CMDPs) are a major extension of the well-known MDPs:
to define the transitions of a state, there may be multiple consecutive nondeterministic and probabilistic choices in any order, in contrast to MDPs, which are limited to exactly one nondeterministic choice and afterwards exactly one probabilistic choice per transition.
In CMDPs, the choices are ``saved'' explicitly in the structure (hence ``choice-aware'').
CMDPs enable the probabilistic analysis of executable models that contain both probabilistic and nondeterministic choices, and are described in more detail in \cref{sec:FormalFoundations:CMDP}.

Both \cref{sec:FormalFoundations:GMC} (discussing generic MCs) and \cref{sec:FormalFoundations:CMDP} (discussing generic CMDPs) have the same outline.
This approach is useful because the definitions and proofs of the generic CMDPs are based on those of the generic MCs.
First, the probabilistic systems are introduced formally.
Afterwards, paths and a probability measure based on that paths are defined for the respective system.

Generic MCs have been published in \cite{Leupolz2017}.
Descriptions about standard Markov chains and standard Markov decision processes have been discussed by many authors before, e.g., in~\cite{BaierKatoen,BaierHabil}.
The definitions of generic MCs and CMDPs are structurally close to their standard counterparts.
This chapter is an excerpt from the thesis ``Probabilistic Safety Analysis of Executable models''~\cite{LeupolzPhD}.
More details can be found there.

\section{Generic Markov Chains}\label{sec:FormalFoundations:GMC}

As the first probabilistic system, generic Markov chains (generic MCs/GMCs), which are suited to model purely probabilistic systems, are introduced.

\begin{definition}[Generic Markov Chain] A generic Markov chain $\Mc$ is represented by the tuple $(\States, \Targets, \targetstate, \Trans)$ consisting of
\label{def:mc}
\begin{compactitem}
\item a countable set $\States$ of \emph{states}, and
\item a countable set $\Targets$ of \emph{targets} with a \emph{target state} function $\targetstate : \Targets \to \States$, and
\item a \emph{transition distribution} function $\Trans : \States \to \Dists(\Targets)$
\end{compactitem}
where $\Dists(\Targets) = \{ \dist : \Targets \to \ProbRange \mid \sum_{\target \in \Targets} \mu(\target) = 1 \}$.
\end{definition}
Let $\distOfState{\state}$ be a shorthand for $\Trans(\state)$.
By convention $\McStates$ stands for $\Mc$'s states, $\McTargets$ for $\Mc$'s targets, and so on.
This is used in the remainder, when the corresponding generic Markov chain is not clear from the context.

By choosing the states as targets, i.e., $\Targets = \States$, and the identity function as $\targetstate$, a probabilistic system is obtained that is commonly known as Markov chains.
Such Markov chains are identified as \emph{standard Markov chains} in the remainder.

This work introduces another useful manifestation of generic Markov chains, the so called labeled Markov chains.
\defAmRand{$\Labels$-labeled MC}
An \emph{$\Labels$-labeled Markov chain} is a generic Markov chain where $\Targets$ is chosen to be $\Labels \times \States$ for some set of \emph{labels} $\Labels$ with $\targetstate(\lbl, \state) = \state$.
In contrast to approaches where states are labeled, the labeling belongs to the transitions.
Shifting the labeling on the transitions is one reason for the efficiency of the implemented algorithms.

{
\newcommand{\introExampleLmcTargets}{
	\begin{tabular}{@{}cl@{}}
	\toprule
		target		&	label	\\
	\midrule
		$\target_1$		& $\dsformula{0}$ \\
		$\target_2$		& $\dsformula{0}$ \\
		$\target_3$		& $\dsformula{0}, \dsformula{1}$ \\  %\emph{f, t}. 3,4,5 must not be equal
		$\target_4$		&  \\
		$\target_5$		& $\dsformula{0}$ \\
	\bottomrule
	\end{tabular}
}
\begin{figure}[h]
	\centering
	\sbox{\temporaryimage}{\includegraphics*{intro_lmc}}
	\begin{minipage}[c]{\probabilisticsystemscalefactor\wd\temporaryimage}
		\resizebox{\textwidth}{!}{\usebox{\temporaryimage}}
	\end{minipage}%
	\hspace{.8cm}
	\begin{minipage}[c]{2cm}
		\resizebox{2cm}{!}{\introExampleLmcTargets}
	\end{minipage}
	\caption{Labeled Markov chain with 3 states}
	\label{fig:introLmcExample}
\end{figure}
}

\Cref{fig:introLmcExample} depicts an example of a $2^{\{\dsformula{0},\dsformula{1}\}}$-labeled MC.
The squared nodes represent the states, e.g., node $1$ represents $\state_1$.
The distribution function of a state is depicted by the white-headed outgoing arcs of its corresponding node.
Each arc is associated with a target, e.g., the reflexive arc of state $1$ is associated with $\target_1$.
The target node of an arc corresponds to the target state of its target, e.g., $\targetstate(\target_{1})=\state_2$.
The labels of each target are given in the table.
This labeled Markov chain can be used to calculate the probability that eventually a label like $\dsformula{0}$ is reached from a specific state.
How this can be formally expressed and calculated is topic of the remainder of this subsection.

\subsection{Paths and Probability Measure}

Let $\Mc=(\States, \Targets, \targetstate, \Trans)$ be a generic MC.

A finite path\defAmRand{$\finpath$} $\finpath = \target_1\dotspath\,\target_n \in \Targets^*$ of $\Mc$ is a sequence of targets where $\distOfState{\targetstate(\target_i)}(\target_{i+1}) > 0$ for all $\natrangeex{i}{1}{n}$.
%
The empty sequence is also a valid finite path;
$\emptysequence$ denotes the empty sequence.
$\McFinPaths$ is the set of all finite paths of $\Mc$.
%
\defAmRand{$\ancfinpath$}
An anchored finite path $\ancfinpath = \anctuple{\state}{\target_1\,\target_2\dotspath\,\target_n} \in \States \times \McFinPaths$ of $\Mc$ is a pair of a state and a finite path where $\distOfState{\state}(\target_1) > 0$.
\defAmRand{$\pathlength{\ancfinpath}$}
Let $\pathlength{\ancfinpath}=n$ be the path length of $\ancfinpath=\anctuple{\state}{\target_1\,\target_2\dotspath\,\target_n}$.
$\McAncFinPaths$ is the set of all anchored finite paths of $\Mc$.
$\McAncFinPathsStartingFrom{\state} = (\{\state\} \times \McFinPaths) \cap \McAncFinPaths$ is the set of all anchored finite paths starting from $\state$.
%
\defAmRand{$\righttargetstate$}
Let $\righttargetstate : \McAncFinPaths \rightarrow \States$ be a function that returns the last state of an anchored finite path, i.e., $\righttargetstate(\anctuple{\state}{\emptysequence})=\state$, and $\righttargetstate(\anctuple{\state}{\target_1\dotspath\target_n})=\targetstate(\target_n)$ otherwise.
%
Given an anchored finite path $\ancfinpath=\anctuple{\state}{\target_1\dotspath\target_n}$ and a target $\target$, let $\pathconcat{\ancfinpath}{\target} = \anctuple{\state}{\target_1\dotspath\target_n\,\target}$ be the concatenation of the finite path with the target.
Sometimes, a subset $\AncFinPathsOfInterest\subseteq\McAncFinPaths$ is from interest; then, such a subset $\AncFinPathsOfInterest$ is called \emph{paths of interest}.

An infinite path\defAmRand{$\infpath$} $\infpath = \target_1\,\target_2\,\target_3\dotspath \in \Targets^{\omega}$ of $\Mc$ is a sequence of targets where  for all $\intabove{i}{1}$, $\distOfState{\targetstate(\target_i)}(\target_{i+1}) > 0$.
%
$\McPaths$ is the set of all infinite paths of $\Mc$.
%
\defAmRand{$\ancpath$}
An anchored infinite path $\ancpath = \anctuple{\state}{\target_1\,\target_2\,\target_3\dotspath} \in \States \times \McPaths$ of $\Mc$ is a pair of a state and an infinite path where $\distOfState{\state}(\target_1) > 0$.
$\McAncPaths$ is the set of all infinite anchored infinite paths of $\Mc$.
$\McAncPathsStartingFrom{\state} = (\{\state\} \times \McPaths) \cap \McAncPaths$ is the set of all anchored infinite paths starting from $\state$.
%
\defAmRand{$\pathprefix{k}$}
Let $\pathprefix{k}=\mcdiststarget_1\dots\mcdiststarget_k$ denote the $k$-th prefix of $\infpath = \target_1\,\target_2\,\target_3\dotspath$, and let $\Prefixes(\infpath)=\{ \finpath' \mid \finpath' \ \text{is prefix of}\ \infpath \}$ denote the set of all prefixes of $\infpath$.
Note that $\emptysequence$ is a prefix of all infinite paths.
Also, let $\AncPrefixes(\anctuple{\state}{\infpath})=\{\anctuple{\state}{\finpath} \mid \finpath \in \Prefixes(\infpath) \}$ denote the set of all anchored prefixes of $\anctuple{\state}{\infpath}$.

In the remainder, both finite and infinite paths are sometimes just called paths when it is clear from the context which one is meant.

The \emph{cylinder set} $\McCylinder{\ancfinpath}$ of the anchored finite path $\ancfinpath \in \McAncFinPaths$ contains all anchored infinite paths of $\Mc$ that start with $\ancfinpath$.
\defAmRand{$\McCylinder{\ancfinpath}$}
More formally, $\McCylinder{\ancfinpath}=\{ \ancpath' \in \McAncPaths \mid \ancfinpath \in \Prefixes(\ancpath') \}$.
Let $\AncFinPathsOfInterest\subseteq\McFinPaths$, then $\McCylinder{\AncFinPathsOfInterest} = \bigcup_{\ancfinpath\in\AncFinPathsOfInterest} \McCylinder{\ancfinpath}$.
\defAmRand{$\McCylinder{\AncFinPathsOfInterest}$}
If $\Mc$ is clear from the context, $\Cylinder{\ancfinpath}$ abbreviates $\McCylinder{\ancfinpath}$,
\defAmRand{$\Cylinder{\ancfinpath}$, $\Cylinder{\AncFinPathsOfInterest}$}
and $\Cylinder{\AncFinPathsOfInterest}$ abbreviates $\McCylinder{\AncFinPathsOfInterest}$.

The cylinder sets of a generic Markov chain are the basic elements to define its probability measure.
\begin{definition}[Probability Space of a Generic Markov Chain starting in a certain state]
\label{def:probspace_mc}
The probability space $(\McSamplesInState{\state},\McEventsInState{\state},\McProbInState{\state})$ of a generic Markov chain $\Mc=(\States, \Targets, \targetstate, \Trans)$ starting in state $\state\in\McStates$ consists of
\begin{compactitem}
\item the sample space $\McSamplesInState{\state} = \McAncPathsStartingFrom{\state}$, and
\item the events $\McEventsInState{\state}$ as the smallest $\sigma$-algebra on $\McSamplesInState{\state}$ that contains
        \begin{equation*}
         \{ \Cylinder{\ancfinpath}\mid \ancfinpath \in \McAncFinPathsStartingFrom{\state} \}
         \ \text{, and}
      \end{equation*}
\item the probability measure $\McProbInState{\state}$ with
		\begin{equation*}%\textstyle
  			\McProbInState{\state}(\Cylinder{\anctuple{\state}{\target_1\dotspath\,\target_n}}) =
  			\distOfState{\state}(\target_1) \cdot \prod_{\natrangeex{i}{1}{n}} \distOfState{\targetstate(\target_i)}(\target_{i+1}) \ \text{.}
		\end{equation*}
\end{compactitem}
\end{definition}

Using this probability measure, the probability of a countable set of anchored finite paths $\AncFinPathsOfInterest$ starting from state $\state$ which are non-overlapping, i.e., $\Cylinder{\ancfinpath} \cap \Cylinder{\ancfinpath'} = \emptyset$ for all $\ancfinpath \neq \ancfinpath' \in \AncFinPathsOfInterest$, can be calculated.
This can be achieved by summing up the probabilities of each anchored finite path in the set, i.e.
\begin{equation*}
	\McProbInState{\state}(\Cylinder{\AncFinPathsOfInterest}) =
	\McProbInState{\state}(\bigdjunion_{\ancfinpath\in\AncFinPathsOfInterest} \Cylinder{\ancfinpath}) =
	\sum_{\ancfinpath \in \AncFinPathsOfInterest} \McProbInState{\state}(\Cylinder{\ancfinpath})
	\ \text{.}
\end{equation*}
The non-overlapping property ensures that each path is counted exactly once.
%
To allow this simple summation, $\AncFinPathsOfInterest$ needs to be a subset of $\McAncFinPathsStartingFrom{\state}$.
Otherwise, this set may contain paths starting at different states for which the probabilities are defined in other probability spaces.
In general, summing up probabilities from different probability spaces is not well defined.
Such sums may even have values greater than $1$.
%
For that reason, given an arbitrary $\AncFinPathsOfInterest \subseteq \McAncFinPaths$, the probability of that paths cannot be calculated.
But the probability of those paths in $\AncFinPathsOfInterest$ that start from state $\state$ can be calculated.
\defAmRand{$\AncFinPathsOfInterestStartingFrom{\state}$}
Therefore, the \emph{restriction} to those paths is defined as $\AncFinPathsOfInterestStartingFrom{\state} = \AncFinPathsOfInterest \cap \McAncFinPathsStartingFrom{\state}$.

\newpage
\section{Choice-Aware Markov Decision Processes}\label{sec:FormalFoundations:CMDP}

After the explanations of the generic Markov chains, choice-aware Markov decision process (CMDPs) are introduced as the second probabilistic system.
CMDPs are well-suited for models that comprise both probabilistic and nondeterministic choice.
Before CMDPs are defined, its choices and its choice transition function are introduced, which enable CMDPs to have multiple consecutive nondeterministic and probabilistic choices in any order.

Let $\Targets$ be a finite set of targets, and
$\Choices$ a finite set of choices, and
$\Dists(\Choices) = \{ \dist_{\Choices} : \Choices \to \ProbRange \mid \sum_{\cmdpChoice \in \Choices} \dist_{\Choices}(\cmdpChoice) = 1 \}$ be the probability distributions for the choices $\Choices$.
Furthermore, let $\choiceTrans : \Choices \rightarrow \Targets + \Dists(\Choices) + 2^{\Choices}$ be the choice transition function that maps each choice to either a target, a probability distribution, or a non-empty set of choices.
A set of choices is used to model nondeterminism.
\defAmRand{$\cpath$}
The sequence of choices $\cpath = \cmdpChoice_0\,\cmdpChoice_1\dotspath\,\cmdpChoice_n \in \Choices^*$ is a choice path of the choice transition function $\choiceTrans$ if for all non-final choices,
$\choiceTrans$ returns a probability distribution with a probability greater 0 to its successor (probabilism),
or $\choiceTrans$ returns a set of choices that includes the successor (nondeterminism);
more formally,
for all $\natrangeex{i}{0}{n}$ either $\dist_{\Choices}=\choiceTrans(\cmdpChoice_{i})$ with $\dist_{\Choices}(\cmdpChoice_{i+1})>0$ or $\cmdpChoice_{i+1}\in\choiceTrans(\cmdpChoice_{i})$.
A choice path $\cmdpChoice_0\,\cmdpChoice_1\dotspath\,\cmdpChoice_n$ with $\intabove{n}{1}$ forms a cycle if the $\cmdpChoice_0=\cmdpChoice_n$.
A choice transition function $\choiceTrans$ is acyclic, if there exists no choice path of $\choiceTrans$ that forms a cycle.
A choice path $\cmdpChoice_0\,\cmdpChoice_1\dotspath\,\cmdpChoice_n$ is terminal if $\choiceTrans(\cmdpChoice_n)\in\Targets$.
Note that it makes sense to call an acyclic choice transition function also terminal or finite, because applying the function iteratively on its result finally terminates.

\begin{definition}[Choice-aware Markov Decision Process]
\label{def:CMDP}
A choice-aware Markov decision process $\Cmdp$ is represented by the tuple $(\States, \Targets, \targetstate, \Choices, \choiceTrans, \StateRootChoice)$ consisting of
\begin{compactitem}
\item a finite set $\States$ of \emph{states}, and
\item a finite set $\Targets$ of \emph{targets} with a \emph{target state} function $\targetstate : \Targets \to \States$, and
\item a finite set $\Choices$ of \emph{choices} with an acyclic \emph{choice transition} function $\choiceTrans : \Choices \rightarrow \Targets + \Dists(\Choices) + 2^{\Choices}$, and
\item a function $\StateRootChoice : \States \rightarrow \Choices$ that determines the root choice of each state,
\end{compactitem}
where $\Dists(\Choices) = \{ \dist_{\Choices} : \Choices \to \ProbRange \mid \sum_{\cmdpChoice \in \Choices} \dist_{\Choices}(\cmdpChoice) = 1 \}$.
\end{definition}

By convention $\CmdpStates$ stands for $\Cmdp$'s states, $\CmdpTargets$ for $\Cmdp$'s targets, and so on.
This is used in the remainder, when the corresponding choice-aware Markov decision process is not clear from the context.

This work introduces another useful manifestation of choice-aware Markov decision processes, the so called labeled choice-aware Markov decision processes.
\defAmRand{$\Labels$-labeled CMDP}
An \emph{$\Labels$-labeled choice-aware Markov decision processes} is a choice-aware Markov decision process where $\Targets$ is chosen to be $\Labels \times \States$ for some set of \emph{labels} $\Labels$ with $\targetstate(\lbl, \state) = \state$.
{
\newcommand{\introExampleLmcTargets}{
	\begin{tabular}{@{}cl@{}}
	\toprule
		target		&	label	\\
	\midrule
		$\target_1$		& $\dsformula{0}$ \\
		$\target_2$		& $\dsformula{0}$ \\
		$\target_3$		& $\dsformula{0}, \dsformula{1}$ \\  %\emph{f, t}. 3,4,5 must not be equal
		$\target_4$		&  \\
		$\target_5$		& $\dsformula{0}$ \\
	\bottomrule
	\end{tabular}
}
\begin{figure}[t]
	\centering
	\sbox{\temporaryimage}{\includegraphics*{intro_cmdp}}
	\begin{minipage}[c]{\probabilisticsystemscalefactor\wd\temporaryimage}
		\resizebox{\textwidth}{!}{\includegraphics*{intro_cmdp}}
	\end{minipage}%
	\hspace{.8cm}
	\begin{minipage}[c]{2cm}
		\resizebox{2cm}{!}{\introExampleLmcTargets}
	\end{minipage}
	\caption{Example of labeled choice-aware Markov decision process}
	\label{fig:cmdp_example1b}
\end{figure}
}

\Cref{fig:cmdp_example1b} depicts an example of a $2^{\{\dsformula{0},\dsformula{1}\}}$-labeled CMDP.
The round-bordered smaller squared nodes represent choices,
and the larger squared nodes with a number in the brackets represent the states, e.g., state node $[1]$ represents $\state_1$.
The brackets in state nodes make it easier to distinguish them from choice nodes.
The root choice of a state is depicted by a sole arc from the corresponding state node to a choice node.
If the choice transition function returns a probabilistic distribution for a choice, the outgoing, white-headed arcs from the choice depict the distribution (with the probabilities as labels),
e.g., the outgoing arcs from choice node $1$ to choice nodes $2$ and $3$.
Black-headed arcs without labels are used for nondeterministic choices, e.g., $2$ to choice nodes $4$ and $5$.
Black-headed arcs with targets as labels are used for target choices; the destination node of such an arc is the corresponding state node of the target's target state, e.g., choice node $4$ to state node $[1]$.
The labels of each target are given in the table.
This labeled CMDP can be used to calculate the worst case and the best case probability that eventually a label like $\dsformula{0}$ is reached from a specific state.
How this can be formally expressed and calculated is topic of the remainder of this subsection.

\subsection{Paths and Probability Measure}

Let $\Cmdp=(\States, \Targets, \targetstate, \Choices, \choiceTrans, \StateRootChoice)$ be a CMDP with choice transition function $\choiceTrans$.

A choice path $\cpath = \cmdpChoice_0\,\cmdpChoice_1\dotspath\,\cmdpChoice_n \in \Choices^*$ of $\Cmdp$ is a choice path of $\choiceTrans$.
$\CmdpChoicePaths$ is the set of all choice paths of $\Cmdp$. \defAmRand{$\CmdpChoicePaths$}
Given a choice path $\cpath = \cmdpChoice_0\,\cmdpChoice_1\dotspath\,\cmdpChoice_n$,
\defAmRand{$\cpath[i]$}
let $\cpath[i]=\cmdpChoice_i$ denote the $i$-th choice for all $\natrange{i}{0}{n}$.
Given a terminal choice path $\cpath = \cmdpChoice_0\,\cmdpChoice_1\dotspath\,\cmdpChoice_n$, let  $\choicepathtarget(\cpath)$ be the choice path target of $\cpath$, i.e.,  $\choicepathtarget(\cpath)=\choiceTrans(\cmdpChoice_n)$; this is properly defined, because in a terminal choice path $\choiceTrans(\cmdpChoice_n)\in\Targets$.
%
Let $\state \cmdpLeadsto \target$ (read as `$\state$ leads to $\target$') be satisfied, if and only if there exists a terminal choice path $\cpath$ with $\StateRootChoice(\state)=\cpath[0]$ and $\choicepathtarget(\cpath)=\target$.

A finite path $\finpath = \target_1\,\target_2\dotspath\,\target_n \in \Targets^*$ of $\Cmdp$ is a sequence of targets
where $\targetstate(\target_i) \cmdpLeadsto \target_{i+1}$ for all $\natrangeex{i}{1}{n}$.
\defAmRand{$\finpath$}
The empty sequence $\emptysequence$ is also a valid finite path.
$\CmdpFinPaths$ is the set of all finite paths of $\Cmdp$.
%
\defAmRand{$\ancfinpath$}
An anchored finite path $\ancfinpath = \anctuple{\state}{\target_1\,\target_2\dotspath\,\target_n} \in \States \times \CmdpFinPaths$ of $\Cmdp$ is a pair of a state and a finite path
where $\state \cmdpLeadsto \target_1$.
%
\defAmRand{$\pathlength{\ancfinpath}$}
Let $\pathlength{\ancfinpath}=n$ be the path length of $\ancfinpath=\anctuple{\state}{\target_1\,\target_2\dotspath\,\target_n}$.
%
$\CmdpAncFinPaths$ is the set of all anchored finite paths of $\Cmdp$.
$\firstStateOfAncpath{\ancfinpath}=\state$ be the anchor or the anchored finite path.
$\CmdpAncFinPathsStartingFrom{\state} = (\{\state\} \times \CmdpFinPaths) \cap \CmdpAncFinPaths$ is the set of all anchored finite paths starting from $\state$.
%
Let $\righttargetstate : \CmdpAncFinPaths \rightarrow \CmdpStates$ be the last state of an anchored finite path $\anctuple{\state}{\target_1\dotspath\target_n}$, i.e., $\righttargetstate(\anctuple{\state}{\emptysequence})=\state$ and $\righttargetstate(\anctuple{\state}{\target_1\dotspath\target_n})=\targetstate(\target_n)$ otherwise.
%
Given an anchored finite path $\ancfinpath=\anctuple{\state}{\target_1\dotspath\target_n}$ and a target $\target$, let $\pathconcat{\ancfinpath}{\target} = \anctuple{\state}{\target_1\dotspath\target_n\,\target}$ be the concatenation of the finite path with the target.

An infinite path $\infpath = \target_1\,\target_2\,\target_3\dotspath \in \Targets^{\omega}$ of $\Cmdp$ is a sequence of choice paths
where $\targetstate(\target_i) \cmdpLeadsto \target_{i+1}$ for all $\intabove{i}{1}$.
\defAmRand{$\infpath$}
$\CmdpPaths$ is the set of all infinite paths of $\Cmdp$.
%
\defAmRand{$\ancpath$}
An anchored infinite path $\ancpath = \anctuple{\state}{\target_1\,\target_2\,\target_3\dotspath} \in \States \times \CmdpPaths$ of $\Cmdp$ is a pair of a state and an infinite path
where $\state \cmdpLeadsto \target_1$.
$\CmdpAncPaths$ is the set of all anchored infinite paths of $\Cmdp$.
$\CmdpAncPathsStartingFrom{\state} = (\{\state\} \times \CmdpPaths) \cap \CmdpAncPaths$ is the set of all anchored infinite paths starting from $\state$.
%
\defAmRand{$\pathprefix{k}$}
Let $\pathprefix{k}=\target_1\,\target_2\dotspath\,\target_k$ denote the $k$-th prefix of $\infpath = \target_1\,\target_2\,\target_3\dotspath$ and let $\Prefixes(\infpath)=\{ \finpath' \mid \finpath' \ \text{is prefix of}\ \infpath \}$ denote the set of all prefixes of $\infpath$.
Also, let $\AncPrefixes(\anctuple{\state}{\infpath})=\{\anctuple{\state}{\finpath} \mid \finpath \in \Prefixes(\infpath) \}$ denote the set of all anchored prefixes of $\anctuple{\state}{\infpath}$.

In the remainder, both finite and infinite paths are sometimes just called paths when it is clear from the context which one is meant.

A probability measure cannot be defined directly for a CMDP, because a nondeterministic choice is not a ``probabilistic construct''.
With a so called scheduler, the nondeterminism in a CMDP can be resolved, which results in a GMC with a properly defined probability measure.
GMCs that are the result of such schedulers provide a view onto the CMDP from the schedulers' perspective.
By using different schedulers, a broader view onto the CMDP can be obtained.
By consulting all schedulers, minimum and maximum probabilities can be derived.
There are two kinds of schedulers in CMDPs: \emph{choice scheduler}, which resolve the nondeterministic choices of choice paths, and \emph{step scheduler}, which select the choice scheduler for each step between two states.

\begin{definition}[Choice Scheduler of a Choice-aware Markov Decision Process]
\label{def:CMDP_choicescheduler}
Let $\Cmdp=(\States, \Targets, \targetstate, \Choices, \choiceTrans, \StateRootChoice)$ be a CMDP. A \emph{choice scheduler} for $\Cmdp$ is a partial function $\ChoiceScheduler : \Choices \rightharpoonup \Choices$ such that for all $\cmdpChoice \in \Choices$, $\choiceTrans(\cmdpChoice)\in 2^{\Choices}$ implies $\ChoiceScheduler(\cmdpChoice) \in \choiceTrans(\cmdpChoice)$.
\end{definition}
%
Let $\ChoiceSchedulers(\Cmdp)$ be the finite set of all possible choice schedulers for $\Cmdp$;
\defAmRand{$\ChoiceSchedulers$}
let $\ChoiceSchedulers$ denote $\ChoiceSchedulers(\Cmdp)$ if $\Cmdp$ is clear from the context.
Note that the numbers of $\ChoiceSchedulers(\Cmdp)$ is finite, because $\Choices$ is finite and there is only a limited number of combinations of possible successors for the nondeterministic choices.
%
%
The choice path $\cpath = \cmdpChoice_0\,\cmdpChoice_1\dotspath\,\cmdpChoice_n$ of $\Cmdp$ is a $\ChoiceScheduler$-path if and only if the choice scheduler $\ChoiceScheduler$ agrees with all its nondeterministic choices, i.e., if and only if for all $\natrangeex{i}{0}{n}$, $\choiceTrans(\cmdpChoice_i)\in 2^{\Choices}$ implies $\ChoiceScheduler(\cmdpChoice_i)=\cmdpChoice_{i+1}$.

\begin{definition}[Markov Chain of a Choice Scheduler]
\label{def:mc_of_choicescheduler}
Let $\Cmdp=(\States, \Targets, \targetstate, \Choices, \choiceTrans, \StateRootChoice)$ be a CMDP with choice transition function $\choiceTrans$, and $\ChoiceScheduler$ a \emph{choice scheduler} for $\Cmdp$. The standard Markov Chain $\Mc^\ChoiceScheduler$ induced by $\ChoiceScheduler$ is given by
$\Mc^\ChoiceScheduler=(\Choices \cup \Targets, \Trans)$
where
\begin{compactitem}
\item       $\Trans(\cmdpChoice)(\choiceTrans(\cmdpChoice))=1$ if $\choiceTrans(\cmdpChoice) \in \Targets$, and
\item       $\Trans(\cmdpChoice)=\choiceTrans(\cmdpChoice)$ if $\choiceTrans(\cmdpChoice) \in \Dists(\Choices)$, and
\item       $\Trans(\cmdpChoice)(\cmdpChoice')=1$ if $\choiceTrans(\cmdpChoice)\in 2^{\Choices}$ and $\ChoiceScheduler(\cmdpChoice)=\cmdpChoice'$, and
\item       $\Trans(\target)(\target)=1$, and
\item       $\Trans(\cdot)(\cdot)=0$ otherwise.
\end{compactitem}
\end{definition}

For state $\state$ and choice scheduler $\ChoiceScheduler$, let $\choiceSchedulerDistOfState{\state}$ be the choice distribution with $\choiceSchedulerDistOfState{\state}(\target)=\ProbInState{\StateRootChoice(\state)}^{\Mc^\ChoiceScheduler}(\TlFinally \target)$.

\begin{definition}[Step Scheduler of a Choice-aware Markov Decision Process]
\label{def:CMDP_stepscheduler}
Let $\Cmdp=(\States, \Targets, \targetstate, \Choices, \choiceTrans, \StateRootChoice)$ be a CMDP. A \emph{step scheduler} for $\Cmdp$ is a function $\StepScheduler : \CmdpAncFinPaths \rightarrow (\Choices \rightharpoonup \Choices)$ such that for all anchored finite paths $\ancfinpath \in \CmdpAncFinPaths$, $\StepScheduler(\ancfinpath)$ is a choice scheduler for $\Cmdp$.
\end{definition}
Let $\StepSchedulers(\Cmdp)$ be the countable set of all possible step schedulers for $\Cmdp$;
\defAmRand{$\StepSchedulers$}
let $\StepSchedulers$ denote $\StepSchedulers(\Cmdp)$ if $\Cmdp$ is clear from the context.
Let $\state \cmdpLeadstoAdheringScheduler{\ChoiceScheduler} \target$ (read as `by adhering to $\ChoiceScheduler$, $\state$ leads to $\target$') be satisfied, if and only if there exists a terminal choice path $\cpath$ with $\CmdpStateRootChoice(\state)=\cpath[0]$ and $\choicepathtarget(\cpath)=\target$ and for all $1 \le i < n$, $\choiceTrans(\cmdpChoice_i)\in 2^{\Choices}$ implies $\ChoiceScheduler(\cmdpChoice_i)=\cmdpChoice_{i+1}$.

The finite anchored path $\anctuple{\state}{\target_1\,\target_2\dotspath\,\target_n}$ of $\Cmdp$ is a finite $\StepScheduler$-path if the scheduler $\StepScheduler$ agrees with all its nondeterministic choices,
i.e., iff $\state \cmdpLeadstoAdheringScheduler{\ChoiceScheduler_0} \target_1$ with $\ChoiceScheduler_0=\StepScheduler({\anctuple{\state}{\emptysequence}})$,
     and for all $1 \le i < n$, $\targetstate(\target_i) \cmdpLeadstoAdheringScheduler{\ChoiceScheduler_{i}} \target_{i+1}$ with $\ChoiceScheduler_{i}=\StepScheduler(\anctuple{\state}{\target_1\,\target_2\dotspath\,\target_{i}})$.
Let $\CmdpAncFinStepSchedulerPaths$ denote the set of all finite $\StepScheduler$-paths of $\Cmdp$, and
$\CmdpAncFinStepSchedulerPathsStartingFrom{\state}=\CmdpAncFinPathsStartingFrom{\state} \cap \CmdpAncFinStepSchedulerPaths$ denote the set of all finite $\StepScheduler$-paths of $\Cmdp$ starting from $\state$.

Analogously,
the anchored infinite path $\ancpath = \anctuple{\state}{\target_1\,\target_2\,\target_3\dotspath}$ of $\Cmdp$ is a $\StepScheduler$-path if the scheduler $\StepScheduler$ agrees with all its nondeterministic choices,
i.e., iff $\state \cmdpLeadstoAdheringScheduler{\ChoiceScheduler_0} \target_0$ with $\ChoiceScheduler_0=\StepScheduler({\anctuple{\state}{\emptysequence}})$,
     and for all $1 \le i$, $\targetstate(\target_i) \cmdpLeadstoAdheringScheduler{\ChoiceScheduler_{i}} \target_{i+1}$ with $\ChoiceScheduler_{i}=\StepScheduler(\anctuple{\state}{\target_1\,\target_2\dotspath\,\target_{i}})$.
Let $\CmdpAncStepSchedulerPaths$ denote the set of all infinite $\StepScheduler$-paths of $\Cmdp$, and
$\CmdpAncStepSchedulerPathsStartingFrom{\state}=\CmdpAncPathsStartingFrom{\state} \cap \CmdpAncStepSchedulerPaths$ denote the set of all infinite $\StepScheduler$-paths of $\Cmdp$ starting from $\state$.

\begin{definition}[Markov Chain of a Step Scheduler]
\label{def:mc_of_CMDP_inducedby_stepscheduler}
Let $\Cmdp=(\States, \Targets, \targetstate, \Choices, \choiceTrans, \StateRootChoice)$ be a CMDP and $\StepScheduler$ a \emph{step scheduler} for $\Cmdp$. The standard Markov Chain $\Mc^\StepScheduler$ induced by $\StepScheduler$ is given by
$(\CmdpAncFinStepSchedulerPaths, \Trans^\StepScheduler)$
with
%the \emph{target state} function is the identity function,
%and
the \emph{transition distribution} function $\Trans^\StepScheduler : \CmdpAncFinStepSchedulerPaths \to \Dists(\CmdpAncFinStepSchedulerPaths)$
	where $\Trans^\StepScheduler(\ancfinpath) = \dist$
	and $\ChoiceScheduler=\StepScheduler(\ancfinpath)$
	such that $\dist( \pathconcat{\ancfinpath}{\target_{n+1}} ) = \choiceSchedulerDistOfState{\righttargetstate(\ancfinpath)}(\target_{n+1})$.
	%when $\targetstate(\target_i) \cmdpLeadstoAdheringScheduler{\ChoiceScheduler_{i}} \target_{i+1}$ and $\dist( \target_0\,\target_1\dotspath\,\target_n\,\target_{n+1} )=0$ otherwise.
\end{definition}

Each finite $\StepScheduler$-path $\anctuple{\state}{\target_1\,\target_2\dotspath\,\target_n} \in \CmdpAncFinStepSchedulerPaths$ of $\Cmdp$ has a corresponding anchored finite path
%\ancfinpath^\correspondingOfMc =
$\anctuple{\anctuple{\state}{\emptysequence}}{\anctuple{\state}{\target_1}\,\allowbreak{}\anctuple{\state}{\target_1\,\target_2}\dotspath\,\allowbreak{}\anctuple{\state}{\target_1\,\target_2\dotspath\,\target_n}}  \in \AncFinPaths^{\Mc^\StepScheduler}$ in the MC $\Mc^{\StepScheduler}$, and vice versa.
%
Analogously, each infinite $\StepScheduler$-path $\ancpath^\correspondingOfCmdp \in \CmdpAncFinStepSchedulerPaths$ of the CMDP $\Cmdp$
has a corresponding anchored infinite path $\ancpath^\correspondingOfMc$ in the generic Markov chain $\Mc^\StepScheduler$, and the other way around.

This correspondence makes it possible to use the probability measure $\ProbOfInState{\Mc^\StepScheduler}{\state}$ induced by MC $\Mc^\StepScheduler$ to define the probability space of the choice-aware Markov decision process $\Mc$ with its associated step scheduler $\StepScheduler$.

The \emph{cylinder set} $\CmdpCylinder{\StepScheduler}{\ancfinpath}$
\defAmRand{$\CmdpCylinder{\StepScheduler}{\ancfinpath}$}
of the anchored finite path $\ancfinpath$ contains all anchored paths of $\Cmdp$ that start with $\ancfinpath$ and adhere to $\StepScheduler$. More formally, $\CmdpCylinder{\StepScheduler}{\ancfinpath}=\{ \ancpath' \in \CmdpAncStepSchedulerPaths \mid \ancfinpath \in \Prefixes(\ancpath') \}$.
Let $\AncFinPathsOfInterest\subseteq\CmdpAncFinStepSchedulerPathsStartingFrom{\state}$, then $\CmdpCylinder{\StepScheduler}{\AncFinPathsOfInterest} = \bigcup_{\ancfinpath\in\AncFinPathsOfInterest} \CmdpCylinder{\StepScheduler}{\ancfinpath}$.

\begin{definition}[Probability Space of a Choice-aware Markov Decision Process induced by a Step Scheduler starting in a certain state]
\label{def:probspace_CMDP_stepscheduler}
%Let $\Cmdp=(\States, \Targets, \targetstate, \Choices, \choiceTrans, \StateRootChoice)$ be a CMDP and $\StepScheduler$ a \emph{step scheduler} for $\Cmdp$.
The probability space $(\CmdpSamplesInState{\StepScheduler}{\state},\CmdpEventsInState{\StepScheduler}{\state},\CmdpProbInState{\StepScheduler}{\state})$ of a Choice-aware Markov Decision Process $\Cmdp$ induced by a Step Scheduler $\StepScheduler$ starting in state $\state\in\CmdpStates$ consists of
\begin{compactitem}
\item the sample space $\CmdpSamplesInState{\StepScheduler}{\state} = \CmdpAncStepSchedulerPathsStartingFrom{\state}$, and
\item the events $\CmdpEventsInState{\StepScheduler}{\state}$ as the smallest $\sigma$-algebra on $\CmdpSamplesInState{\StepScheduler}{\state}$ that contains $\{ \CmdpCylinder{\StepScheduler}{\ancfinpath}\mid \ancfinpath \in \CmdpAncFinStepSchedulerPathsStartingFrom{\state} \}$, and
\item the probability measure $\CmdpProbInState{\StepScheduler}{\state}$ with $\CmdpProbInState{\StepScheduler}{\state}(\CmdpCylinder{\StepScheduler}{\ancfinpath^\correspondingOfCmdp})=\ProbOfInState{\Mc^\StepScheduler}{\anctuple{\state}{\emptysequence}}(\CylinderOf{\Mc^\StepScheduler}{\ancfinpath^c})$ where $\ancfinpath^c$ is the corresponding path of $\ancfinpath$.
\end{compactitem}
\end{definition}

Note that the probability measure cannot generally be used on the union of paths of different step schedulers, even if the paths all start in the same state.
Thus, given an arbitrary $\AncFinPathsOfInterest \subseteq \CmdpAncFinPaths$, the probability of that paths cannot be calculated.
But the probability of the $\StepScheduler$-paths in $\AncFinPathsOfInterest$ that start from state $\state$ can be calculated.
\defAmRand{$\AncFinStepSchedulerPathsOfInterestStartingFrom{\state}$}
Therefore, the \emph{restriction} to those paths is defined as $\AncFinStepSchedulerPathsOfInterestStartingFrom{\state} = \AncFinPathsOfInterest \cap \CmdpAncFinStepSchedulerPathsStartingFrom{\state}$.


\section{Introduction}
\label{sec:introduction}

\subsection{Motivation}
Many institutions around the world use central, automated matching schemes to assign agents to resources based on their preferences. For instance, the National Resident Matching Program (NRMP) in the United States uses such a matching mechanism to pair graduating medical students to residency positions at hospitals every year \cite{Roth-NRMP}. There are different goals for these matching problems, which can range from efficiency to fairness, and finding a mechanism that fulfills these goals is an important task to efficiently design such matching markets.
There are also many variants of matching problems that include, but are not limited to, one-to-one, one-to-many and many-to-many matchings. Additionally, the distribution of preferences needs to be considered as well. For instance, in the hospital-residents problem, both the hospitals and residents supply preferences over the other set. In that case, it is also important to consider if incomplete preference lists or ties should be allowed in the instances.

We will describe, examine and analyze the problem of one-to-many matching mechanisms with one-sided preferences. While there are many use cases for this problem, we will focus on the problem of matching students to seminars, which is of high interest to many universities, who often require their students to participate in a seminar. Students typically have a choice between a handful of different seminars; however, there are capacity constraints that make it hard to match all students to their first choice. Let us consider the following example: 100 students have to be assigned to one of six seminars, wherein each seminar has a capacity of 20 students. The students express their preferences by supplying a \emph{strict}, meaning without ties, but possibly incomplete preference list of the six seminars. The goal for the school's administration is to find an assignment of students to seminars that fulfills their requirements, which can range from assigning as many students as possible to being as fair as possible with the assignments. We will see that there is no obvious choice in picking an algorithm, because there are many trade-offs between existing mechanisms that make it necessary to prioritize the requirements.
What makes this problem harder is that the students' preferences are not necessarily equally distributed. Oftentimes, a majority of students prefer one seminar which conflicts with other students getting their first choice. At the same time, it can occur that students go unmatched when their preference lists are short and primarily consist of seminars that are of high popularity among the student body.

The goal of this thesis is to formally model the one-to-many matching problem with one-sided preferences, while presenting different algorithms for finding matchings and finally to evaluate these algorithms using a set of metrics that makes sense for the one-to-many case. For the purpose of this thesis, we will use the example of student-seminar matching to analyze the problem, but it is important to note that it can just as well be generalized to any other one-to-many matching problem with one-sided preferences. Lastly, an interactive web system is presented, which allows a school's administration to find a student-seminar matching using one of the presented algorithms.

\subsection{Formal Definition: Capacitated House Allocation Problem}\label{intro:cha}
Many economists and game theorists \cite{FEKETE2003219} study variants of the \emph{House Allocation Problem (HA)}, wherein a set of indivisible items $H$, the houses, needs to be divided among a set $A$ of applicants. Each applicant may have a strict preference order over a subset of $H$. Formally this means that an instance $I$ of the problem consists of two disjoint sets, where $H := \{h_1, h_2, ..., h_n\}$ is the set of houses and $A := \{a_1, a_2, ..., a_m\}$ is the set of applicants. Each applicant $a_i \in A$ ranks a subset of the houses in $H$ using a preference list. The houses, on the other hand, do not have any preferences over applicants. A matching or assignment $M$ is map with $M: A \rightarrow H$, so that for every applicant $a_i$, the house $M(a_i)$ is on the applicants preference list \cite{SngThesis}.

The house allocation problem is essentially an alias for a matching problem on bipartite graphs with one-sided preferences. There are many applications, including matching clients to servers, professors to offices and also students to seminars. For the latter, some generalizations have to be made to the problem; specifically, one seminar should now be matched to more than one student, whereas the houses in HA are matched to one and only one applicant. In the literature, the latter, one-to-many, variant of the problem is also referred to as the \emph{Capacitated House Allocation Problem}, denoted by CHA \cite{algorithmics}. The following chapters of this thesis explore performance indicators and algorithms for finding matchings in the context of students and seminars, which is semantically equivalent to the CHA problem.

\subsection{Formal Definition: Student-to-Seminar Matching}
Similarly to CHA, we can describe the problem of assigning students to seminars as a many-to-one matching problem, with a set of students $S:= \{s_1, s_2, ...,s_n\}$ and a set of seminars $T:= \{t_1, t_2, ..., t_m\}$. Every student $s_i \in S$ provides a strict preference order over a subset of $T$, and every seminar $t_j \in T$ has a capacity of $c_j \in \mathbb{N}$ students. The goal is to find a matching $M: S \rightarrow T$, that assigns students to one of their preferred seminars while respecting the capacity of the seminars. That means that for every seminar $t_j$ the following is satisfied: $|M^{-1}(t_j)| \leq c_j$. Using this definition, we can describe the problem as a many-to-one matching with one-sided, incomplete preferences. In the case of students and seminars, this means that one student is matched to one seminar, but one seminar is matched to a number of students, while only students express their preferences over the seminars. Later, we discuss different optimality criteria which can be used for defining an objective function for the problem.

\subsection{Outline}
Given this problem definition, we will review the literature for optimality criteria and, based on that criteria, present algorithms that produce matchings with those characteristics. Many of those characteristics are quite obvious, such as maximum-cardinality; however, we will also review terms such as \emph{Pareto-efficiency} or \emph{popularity} that are commonly used in the context of market design and game theory.
Using these criteria, we will review matching algorithms for CHA and then analyze and compare their runtime-complexity and optimality-properties. We will see that no algorithm is known that fulfills all desirable properties, which makes it necessary to understand the trade-offs and relationships between the algorithms.

After completing a theoretical comparison of the algorithms, we will proceed to analyze matchings computed using real and synthetic data to better understand the theoretical observations and trade-offs discussed before. Additionally, a web-interface will be presented that allows a university's administration to compute matchings according to their requirements, using data that can be created and managed as part of the system. Lastly, we will briefly explore extensions to the problem, such as the many-to-many case, and finally summarize the results and make recommendations for the implementation and usage of such a matching system, based on the current state of research on these matching mechanisms.

Therefore, the contribution of this thesis is a practically motivated survey and analysis of one-to-many matching mechanisms with one-sided preferences. We use the problem of matching students to seminars as a motivating example to understand commonly used algorithms and their performance characteristics, by performing both a theoretical and a practical analysis using real and synthesized data. This will conclude with recommendations for building such systems based on the theoretical and practical results obtained in the previous sections.

\subsection{Related Problems}
There are many related matching problems that can be classified as follows:
\begin{enumerate}
    \item Bipartite matching problems
    \begin{enumerate}
        \item One-sided preferences (see Section \ref{intro:cha})
        \begin{enumerate}
            \item One-to-one (e.g. House Allocation problem)
            \item \textbf{One-to-many (e.g. Capacitated House Allocation problem)}
        \end{enumerate}
        \item Two-sided preferences
        \begin{enumerate}
            \item One-to-one (e.g. Stable-Marriage problem) (see Section \ref{intro:stable-marriage})
            \item One-to-many (e.g. Hospital-Residents problem) (see Sections \ref{intro:hospital-residents}, \ref{extensions:two-sided})
        \end{enumerate}
    \end{enumerate}
    \item Non-bipartite matching problems
    \begin{enumerate}
        \item One-to-one (e.g. Stable-Roommates problem)
    \end{enumerate}
\end{enumerate}
Even though these problems have some differences, key mechanisms (Section \ref{chapter:algorithms}) and optimality criteria (Section \ref{sec:optimality}) used for them can nonetheless be similar or even identical. For instance, in the case of one-sided preferences, mechanisms for the one-to-one case can often easily be extended to the one-to-many case \cite{Gusfield}. It is also important to note that the problem can be extended to using incomplete preference lists and ties in the preference lists. In fact, real-world settings often necessitate at least incomplete preference lists, as students often want to classify a seminar as unacceptable. The following subsections will briefly present some of the listed matching problems and present some key results.

\subsubsection{Stable Marriage Problem}\label{intro:stable-marriage}
The stable marriage problem was one of the first matching problems to be researched \cite{GaleShapleyOrig}, and consequently motivated further research in the field of matching under preferences.

The problem is stated as follows: A set of men $M$ and of women $W$ shall be matched one-to-one, where each men and women provide a complete and strict preference order over the agents of the other set. The \emph{Deferred Acceptance Algorithm} presented in Gale and Shapley's paper \cite{GaleShapleyOrig} finds a \emph{stable}, maximum-cardinality matching in polynomial time. Stability is defined as follows: given a women $w \in W$ and any man $m \in M$ that she was not matched to, $w$ does not prefer $m$ more than her current partner, and $m$ does not prefer $w$ more than his current partner. Therefore, in a stable matching, no pair of applicants $(m, w)$ should exist, so that $m$ and $w$ would benefit from leaving their assigned partner to be with each other.

The Deferred Acceptance Algorithm picks one of the sets of applicants, for example the men, and proceeds by letting every man propose to their most preferred woman. The woman will tentatively accept the proposal, if she is not engaged at that point, or prefers the new proposal over her previous one. Then the algorithm repeats this process with men proposing to their most-preferred, but not yet proposed to woman until they are either matched or have no more women to propose to \cite{Gusfield}.

It is important to note that the algorithm produces different results if the women are the proposers: it has been shown that all executions of the algorithm with all possible permutations of the input with men as proposers yield the same stable matching. That matching is men-optimal, which means that every man has the best partner that he can have in any stable matching \cite{Gusfield}. Additionally, with men proposing, the produced matchings have also been shown to be women-pessimal, meaning that every woman is matched to the worst partner that she can have in any matching \cite{Gusfield}.

While the scenario of matching women to men like described is presumably not a common real-world occurrence, Gale and Shapley's paper inspired several new approaches to other similar problems.

\subsubsection{Hospitals/Residents Problem}\label{intro:hospital-residents}
A few years after the publication of Gale and Shapley's original paper, it was determined that the deferred acceptance mechanism was essentially the same mechanism used by the National Resident Matching Program (NRMP) in the United States to match graduating medical students to residency positions in hospitals \cite{Gusfield}. As a matter of fact, Gale and Shapley's paper described an algorithm for the so-called \emph{College Admissions Problem} \cite{GaleShapleyOrig}, which is essentially the same problem. Just like in the stable-marriage problem, there is a solution that can be hospital-optimal or resident-pessimal.

The problem can also be described as finding a one-to-many matching with two-sided, incomplete, but strict preferences. Essentially, a hospital can offer multiple spots and both parties can mark entities from the other set as unacceptable by not including them in their preference lists \cite{RePEc:ris:nobelp:2012_005}. Therefore, the main difference to the stable marriage problem is that one and only one of the sets allows their entities to have a capacity, i.e. accept more than one applicant. At the same time, however, entities in both sets still express preferences, which makes this problem different from the CHA-problem (Section \ref{intro:cha}) where only entities from one set express preferences.

In reality, the problem solved by the NRMP is more complex, as it permits couples of residents to submit preferences together. It has been shown that a stable solution does not always exist and that finding one if it exists, or showing that it does not exist is NP-complete \cite{RONN1990285}. The revised algorithm used by the NRMP utilizes findings about stability and simple matching markets to find a good approximation, while minimizing opportunities for strategic manipulation, which was possible before \cite{NBERw6963}.

\subsubsection{Assignment Problem}\label{intro_assignment}
The problem of matching students to seminars can also be defined as an assignment problem. The goal of the assignment problem is to find a minimum-weight, maximum-cardinality matching in a bipartite graph. In this case the goal of the problem is, given the set of students $S$, seminars $T$, and a cost function $W: S \times T \rightarrow \mathbb{R}$, to find a map $M: S \rightarrow T$, which minimizes the following objective function:
$\sum_{s \in S} W(s, M(s))$.

One of the first algorithms used for solving this problem was the Hungarian algorithm by Munkres, which finds a maximum-cardinality, minimum-weight matching in polynomial time \cite{Munkres}.
Alternatively, the problem can be transformed into an instance of the minimum-cost flow problem to determine a minimum weight matching. Section \ref{algo:assignment} will further investigate using this algorithm for the problem of student-seminar matching.

\subsubsection{The Student-Project Allocation Problem}
A similar problem to the CHA or student-seminar assignment problem is the \emph{Student-Project Allocation Problem (SPA)}. This problem considers three entities instead of two, which are students, projects and lecturers. In most variations of the problem, students have preferences over courses, while courses again have capacities. However, lecturers' preferences are also considered, where lecturers can have preferences over students and/or courses. Algorithmic approaches used for this problem are mostly based on the deferred acceptance mechanism and find stable matchings that can either favor the students or lecturers \cite{algorithmics}.


\documentclass[12pt]{article}
\usepackage[scaled]{helvet}
\renewcommand\familydefault{\sfdefault}
\usepackage[T1]{fontenc}

\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{bm}
\usepackage{parskip}
\usepackage{hyperref}
\usepackage{graphicx}
\usepackage{listings}
\usepackage{xcolor}

\definecolor{Brown}{cmyk}{0,0.81,1,0.60}
\definecolor{OliveGreen}{cmyk}{0.64,0,0.95,0.40}
\definecolor{CadetBlue}{cmyk}{0.62,0.57,0.23,0}
\definecolor{lightlightgray}{gray}{0.9}

\lstset { %
    language=C++,
    %backgroundcolor=\color{black!5}, % set backgroundcolor
    %basicstyle=\footnotesize,% basic font setting
	basicstyle=\ttfamily,                   % Code font, Examples: \footnotesize, \ttfamily
	keywordstyle=\color{OliveGreen},        % Keywords font ('*' = uppercase)
	commentstyle=\color{gray},              % Comments font
	backgroundcolor=\color{lightlightgray},
	tabsize=4,
	frame=single,
}

\title{\textbf{Practical 7: Rigid Bodies, part 2 }}
\author{Babis Koniaris}
\date{}

\begin{document}
\maketitle

\begin{center}
\includegraphics[width=\textwidth]{p7-teaser.png}
\end{center}
\pagebreak

\section*{Introduction}

The goals of this practical are to:

\begin{itemize}
\item Assign and maintain correct inertia tensors for cuboids
\item Apply impulses to rigid bodies
\item Apply correct bounce impulse to rigid body colliding with a plane
\end{itemize}

\section*{Tasks}

\subsection*{Task 1: Inertia tensor}

As we’ve seen in the lecture, the inertia tensor is to torque what mass is to acceleration. Unfortunately, it is not as simple to work with as mass is. Not only is it a matrix rather than a scalar, it depends on the geometry of the solid, and it also depends on its orientation. Computing the inertia tensor of a random solid model isn’t trivial, but luckily, the inertia tensors for common primitives are simple and well-documented. Please refer to the following page reference: \url{https://en.wikipedia.org/wiki/List_of_moments_of_inertia}. Since we are working with cuboids for the time being, here’s the inertia tensor of a cuboid of width $w$ (along the $x$ axis), height $h$ (along the $y$ axis), depth $d$ (along the $z$ axis) and mass $m$:

\begin{align}
I_0 = \begin{bmatrix}
\frac{1}{12}m(h^2+d^2) & 0 & 0\\
0 & \frac{1}{12}m(w^2+d^2) & 0\\
0 & 0 & \frac{1}{12}m(w^2+h^2)\\
\end{bmatrix}
\end{align}

\textbf{Task 1: \emph{Create} and \emph{maintain} a correct inverse inertia tensor for a cuboid}

You will have noticed that there are two parts to this task: creating and maintaining. Also notice that we will not store the tensor, but its inverse, as it is what we need in our simulation. Creating a valid tensor is easy, you can do so using the above formula. However, it must be updated any time the mass and dimension of the solids are modified. There are two design options to consider to achieve this:

\begin{itemize}
\item Calculate the tensor dynamically. In other words, you don’t need to store the inverse tensor in a variable like m\_invInertia, instead the accessor method InverseInertia() computes the matrix any time it is called. The benefit of this approach is that this single method ensures that the inertia is always correct without having to modify any other part of our code. The drawback is cost: InverseInertia() will get called at every step of the simulation, so this would be unwelcome computation.
\item Make sure the tensor (or in our case inverse tensor) is updated whenever the mass or dimensions of the body are updated. This approach requires the modification of more code, but it is computationally cheaper. This is the one I recommend that you implement.
\end{itemize}

Also remember that the inertia (and inverse inertia) also depend on the orientation (i.e., rotation matrix) of the rigid body:

\begin{align}
I(t) = R(t)I_0R(t)^T \\
I^{-1}(t) = R(t)I_0^{-1}R(t)^T
\end{align}

\subsection*{Task 2: Applying impulses}

The goal of this task is to allow you to understand the effect of applying impulses at various points on a solid. In what follows, you will work with a cuboid named rb that has a mass $m = 1$ and the following dimensions:
\begin{itemize}
\item width = 2
\item height = 6
\item depth = 2
\end{itemize}
This corresponds to a default cubic rigid body that has been scaled by a vector (1,3,1).

\textbf{Task 2: Experiment with the application of impulses to a rigid body}

Here are examples of tests you should carry out:
\begin{itemize}
\item Give rb an initial velocity of (0,0,0) and an angular velocity of (0,0,0). In other words, rb is immobile! No force applies to rb. After 2 seconds:
	\begin{itemize}
	\item Apply an impulse that will make rb move to the left with a velocity of (-1,0,0)
	\item Apply an impulse that will make rb move to the left with a velocity of (-1,0,0) and start spinning anticlockwise around the z axis
	\end{itemize}
\item Give rb an initial velocity of (3,0,0) and an angular velocity of (0,0,0). No force applies to rb. After 2 seconds:
	\begin{itemize}
	\item Apply an impulse that will make rb stop moving
	\item Apply an impulse that will make rb change direction and travel with a velocity of (-3,0,0).
	\item Apply an impulse that will make rb stop translating and start spinning clockwise
	\end{itemize}
\item Consider that rb is traveling with a constant velocity of (3,0,0) and angular velocity of (0,0,1). No force applies to rb. After 2 seconds:
	\begin{itemize}
	\item Apply an impulse that will make rb stop moving and rotating
	\end{itemize}
\end{itemize}

\subsection*{Task 3: Collision response}

\textbf{Task 3: Simulate the collision between a rigid body (cuboid) and a fixed horizontal plane using impulse-based collision response.}

Collision response can be managed in different ways. We are going to implement an impulse-based response in which some coarse approximations will be made. The work involves these two parts:

\begin{itemize}
\item Once you have detected that a component (vertex, edge or face) overlaps with the ground plane, translate the body up so that the component lies exactly on the ground. This is a rough way of managing collisions, but it is simple to implement and should work reasonably well.
\item Apply the correct collision impulse at the correct point on the rigid body (as described in the lecture).
\end{itemize}

\subsection{Further tasks}

Collisions without friction look very unnatural: a solid that falls down on the ground will always bounce along the same vertical axis, regardless of the way it hits the ground. To make the bounces look more believable, friction must be simulated. Try to come up with a friction model that make bounces look more believable if you have time. You will learn how friction can be simulated next week...

\section*{Deliverables}

Nothing has to be handed in this week but the work you have completed will be a part of the assessed deliverables of next week’s practical. Make sure you complete this assignment this week or you will run out of time if you try to do everything next week.

\end{document}


\documentclass[a4paper]{article}

\usepackage{mathtools}
\usepackage{amsthm}
\usepackage{amsmath}
\usepackage{amssymb}
\newtheorem{theorem}{Theorem}
\usepackage{tikz-cd}
\newcommand*{\N}{{\mathbb{N}}}

\renewcommand{\c}[1]{\mathbf{#1}}
\newcommand{\Set}{{\c{Set}}}

\title{A unified view towards diagonal arguments}
\author{Qiangru Kuang}
\date{February 25, 2019}

\begin{document}

\maketitle

\begin{abstract}
  Cantor's diagonal argument is a simple yet deep theorem. Perhaps less known is that the same idea in the proof powers many famous results, including but not limited to: Russell's paradox, halting theorem, undefinability theorem and Gödel incompleteness theorem. This talk aims to state and prove an abstract form of diagonal argument and derive all of the above as corollaries if time permits. The talk will be self-contained. IA Numbers and Sets or knowing Cantor's diagonal argument will be helpful but not necessary.
\end{abstract}

\section{Acknowledgement}

I would like to thank Archimedeans for organising this talk. My gratitude goes especially to Jeremy, who invited me to give a talk at an early stage of the event, and whose continual checking of my progress eventuates in this article, instead of another draft in the limbo of unfinished blog posts.

I am also grateful to Rob, Linshu and Jenny for listening to an earlier version of the talk and for kindly providing advice which substantially improved the talk.

\section{Exponential}

We start by staring at the notation \(Y^Z\) where \(Y\) and \(Z\) are sets, which is by definition the set of all functions \(Z \to Y\). There are two things worth remarking: firstly \(Y^Z\) is a set itself. This statement does not hold in a general category. For example given two manifolds \(M\) and \(N\), the set of all smooth maps \(M \to N\) is not obviously a manifold. Given two groups \(G\) and \(H\), it is not clear how we can endow the space of all homomorphisms \(G \to H\) a group structure (however if \(G\) and \(H\) are abelian groups then \(\operatorname{Hom}(G, H)\) is obviously an abelian group, although pursuing in this direction will lead us astray from today's topic).

The second fact is a consequence of our definition. Consider another set \(X\). There is a canonical bijection
\[
  \{X \times Z \to Y\} \leftrightarrow \{X \to Y^Z\}
\]
given by what compskis call ``currying'' and ``uncurrying'': given a map \(f: X \times Z \to Y\), we may plug in some \(x \in X\) to get
\[
  g(-) := f(x, -): Z \to Y.
\]
In this case we say \(g\) is \emph{represented} by \(x\). Conversely, ``uncurrying'' is the process of, given \(X \to Y^Z\), i.e.\ a family of maps \(Z \to Y\) parameterised by \(X\), we turn it into a function \(X \times Z \to Y\).

We may ask the question: when does \(X\) fully represent \(Y^Z\)? Obviously we can just take \(X\) to be \(Y^Z\). A more meaningful question would be when \(X\) fully represents \(Y^X\). Cantor's theorem says that if \(Y\) has more than one element then there can be no surjection \(X \to Y^X\), so there is always an element that is not represented.

\section{Lawvere fixed point theorem}

We need to develop a little bit category theory to state Lawvere fixed point theorem in full. If you are not familiar with category theory, you can safely take \(\c C\) to be \(\Set\), every instance of \(X, Y, Z\) to be a set, a point \(x: 1 \to X\) to be an element of \(X\) and operations to be the obvious set theoretical operations.

A category \(\c C\) is \emph{cartesian closed} if there are finite products and for every object \(Z\) there is a right adjoint to \((-) \times Z: \c C \to \c C\). The adjunction is usually written as \((-) \times Z \dashv (-)^Z\), ergo the name exponential. Let the unit and counit of this adjunction be
\begin{align*}
  \eta^Z_Y &: Y \to (Y \times Z)^Z \\
  \varepsilon^Z_Y &: Y^Z \times Z \to Y
\end{align*}
respectively. The counit is usually called \(\operatorname{ev}\), which is the internal language version of taking an ordered pair \((f, x) \in Y^Z \times Z\) and sending it to \(f(x)\).

There is an bijection between \(f: X \times Z \to Y\) and \(\tilde f: X \to Y^Z\). More specifically, given \(f: X \times Z \to Y\), \(\tilde f\) is the composition
\[
  \begin{tikzcd}
    X \ar[r, "\eta_Y^Z"] & (X \times Z)^Z \ar[r, "f^Z"] & Y^Z
  \end{tikzcd}
\]
and given \(\tilde f: X \to Y^Z\), \(f\) is the composition
\[
  \begin{tikzcd}
    X \times Z \ar[r, "\tilde f \times 1_Z"] & Y_Z \times Z \ar[r, "\varepsilon^Z_Y"] & Y.
  \end{tikzcd}
\]

By abuse of notation we take the liberty to use \(f\) to denote both morphisms. In particular, we identify a morphism \(f: 1 \times Z \cong Z \to Y\) with \(f: 1 \to Y^Z\), i.e.\ an element of \(Y^Z\).

Finally, a morphism \(f: A \to B\) is \emph{point-surjective} if for every point \(b: 1 \to B\) there exists \(a: 1 \to A\) such that \(fa = b\). The corresponding notion in \(\Set\) is surjectivity.

\begin{theorem}[Lawvere fixed point theorem]
  In a cartesian closed category \(\c C\), if \(f: X \to Y\) is point-surjective then every \(\alpha: Y \to Y\) has a fixed point \(1 \to Y\).
\end{theorem}

This is proven by William Lawvere in 1970s, an influential category theorist who made contibutions to category theory and foundation of maths, as well as philosophy. He is credited with introducing, for example, elementary topos and ETCS.

\begin{proof}
  Define \(q: X \to Y\) to be the following composition
  \[
    \begin{tikzcd}
      X \ar[r, "\Delta"] & X \times X \ar[r, "f \times 1_X"] & Y^X \times X \ar[r, "\operatorname{ev}"] & Y \ar[r, "\alpha"] & Y
    \end{tikzcd}
  \]
  By point-surjectivity of \(f\) exists \(x: 1 \to X\) that represents \(q\), i.e.\ \(f(x_0, -) = q\). Now if we plug \(x_0\) back in we get
  \begin{align*}
    q(x_0)
    &= \alpha(f(x_0, x_0)) \quad \text{by definition of } q \\
    &= \alpha(q(x_0)) \quad \text{by definition of } f(x_0, -) \\
  \end{align*}
  so \(q(x_0) \in Y\) is a fixed point of \(\alpha\).
\end{proof}

It might be instructive to consider the contrapositive:

\begin{theorem}
  If there exists \(\alpha: Y \to Y\) with no fixed point then \(f: X \to Y^X\) is not point-surjective.
\end{theorem}

By the way this is named by Lawvere ``Cantor's theorem'', which we'll justify in a second.

\begin{proof}
  The same argument as above will exhibit a function \(q: X \to Y\) that is not represented by \(X\): suppose for contradiction it is represented by \(x_0 \in X\), i.e.\ \(f(x_0, -) = q\). But
  \[
    q(x_0) = f(x_0, x_0) = \alpha(q(x_0)) \neq q(x_0),
  \]
  contradiction.
\end{proof}

The idea is that if there exists a point-surjection \(X \to Y^X\), i.e.\ every map \(X \to Y\) can be represented by an element of \(X\) then \(Y\) has a particularly simple ``internal structure'' in the sense that every endomorphism as a fixed point. When we are in the setting of sets, the only such \(Y\) is singletons.

\section{Cantor's diagonal argument}

By the trivial observation that \(2 = \{0, 1\}\) admits a self map with no fixed point, it follows from Lawvere fixed point theorem that a nonempty set \(X\) cannot surject its power set \(2^X\), which is Cantor's diagonal argument. As the proof is constructive, we may well apply it and see what nonrepresentable element we get.

We will use as example the classical Cantor's diagonal argument which proves the nonexistence of surjection \(\N \to 2^\N\). Any attempted enumeration of \(2^\N\) can be seen as a map
\begin{align*}
  f: \N \times \N &\to 2 \\
  (n, k) &\mapsto \text{\(k\)th digit of \(n\)th binary sequence in the enumeration}
\end{align*}
so suppose \(f\) looks like
\[
  \begin{array}{c|ccccc}
    n, k & 1 & 2 & 3 & \cdots \\ \hline
    1 & 1 & 0 & 1 & \cdots \\
    2 & 0 & 0 & 1 & \cdots \\
    3 & 1 & 1 & 1 & \cdots \\
    \vdots
  \end{array}
\]
then \(q \in 2^\N\) is obtained by negating elements along the diagonals.

\section{Russell's paradox}

Recall Russell's paradox: we cannot have a set of all sets as otherwise consider the set \(\{A: A \notin A\}\).

%(Digression for those familiar with set theory: point of Russell's paradox is that naive set theory fails, namely not every statement we write down defines a set. It might worth noting how we settled Russell's paradox using axiomatic set theory. It is not because (or not directly because) we added axioms such as axiom of foundation. Note that adding axioms does not pick out those ``bad sets'' from the model. Instead, it is because we banned unrestricted comprehension.)

Suppose we have the set of all sets \(\Set\). We have a membership function
\begin{align*}
  f = \in: \Set \times \Set &\to 2 \\
  (A, B) &\mapsto 1_{A \in B}
\end{align*}
so the nonrepresentable element \(q \in 2^\Set\) we get is the indicator function \(1_{A \neq A}\). By Yoneda lemma this is the same as the set \(\{A: A \notin A\}\), which is what we get in classical proof of Russell's paradox.

\section{Entertainment maths}

Many self-referential paradoxes find their way into entertainment maths. You might have heard of this one. Some adjectives in English describe themselves. For example ``English'' is an English word, ``polysyllabic'' is itself polysyllabic, but ``monosyllabic'' is not\footnote{Let's just say for the moment that this notion is well-defined so we don't have to fight to decide if ``short'' is indeed short.}. Call such adjectives describing themselves ``autological'' and ``heterological'' otherwise. But then you will find that heterological is heterological if and only if it is not.

It is mildly interesting on an entertainment level that we can produce an example witnessing the paradox. However, it has the disadvantage of being overly constructive and more importantly, depending on the semantics of the adjective ``heterological'' in English language. A more convincing presentation of the paradox can be given by Lawvere fixed point theorem: consider the function
\begin{align*}
  f: \c{Adj} \times \c{Adj} &\to 2 \\
  (A, B) &\mapsto 1_{A \text{ describes } B}
\end{align*}
Then the nonrepresentable \(q\) is the indicator function of the set of adjectives whose element has the property that it describes itself if and only if it doesn't. ``Heterological'' is an element of this set under ordinary semantics.

Exercise: exhibit Liar's paradox as an instance of Lawvere fixed point theorem.

\section{Gödel incompleteness theorem}

There is actually a moral in the paradox. The function \(f: \c{Adj} \times \c{Adj} \to 2\) comes for free with the category \(\c{Adj}\). By constrast, in Cantor's diagonal argument we rely on extra data to define \(f\) (membership function in Russell's paradox can also be seen as ``free'' if we allow unrestricted comprehension, which is granted since we assume for contradiction there is a set of all sets). In \(\c{Adj}\), an object \(A\) serves an additional syntactical role, namely the ability to form the predicate ``\(B\) is described by \(A\)''.

More generally, consider \(X\), a collection of syntactical entities such as nouns, formulae, proofs, programs etc. We can define a function
\[
  f: X \times X \to \Omega
\]
where \(\Omega\) should be thought as values predicate can take, that describe all describable properties \(X \to \Omega\).

The innovation of Gödel to introduce Gödel number is an example of this, which is roughly speaking a way to name formulae in PA. This allows us to talk about predicates of the form ``the natural number \(3\) satisfies the formula with Gödel number \(5\), which is \(t^3 = 3t\)''. We denote by \(\ulcorner A \urcorner\) the Gödel number of fomula \(A\).

We claim that for \(E(x) \in L^1\), there exists a formula \(C\) such that
\[
  \vdash (C \iff E(\ulcorner C \urcorner)).
\]
Thus if we take \(E\) to be
\[
  E(x) := (\forall y)\neg \text{Prov}(y, x)
\]
where \(\text{Prov}(y, x)\) is the predicate ``\(y\) is the Gödel number of the proof of a formula whose Gödel number is \(x\)'', then by the assertion we can find closed \(C\) such that
\[
  \vdash (C \iff E(C)),
\]
i.e.\ \(C\) is equivalent to its own unprovability. This is the famous Gödel incompleteness theorem.

\begin{proof}[Proof of claim]
  We define an auxillary function \(D: \N \to \N\): for every formula \(A(x) \in L^1\),\footnote{\(L^1\) is the Lindebaum algebra of 1 free variable, defined as the set of all formula with 1 free variable with provably equivalent formulas identified.} where \(x\) is its only free variable, we have
  \begin{align*}
    D: \N &\to \N \\
    \ulcorner A(x) \urcorner &\mapsto \ulcorner A(\ulcorner A(x) \urcorner) \urcorner
  \end{align*}

  We let
  \begin{align*}
    f: L^1 \times L^1 &\to L^0 \\
    (A(x), B(x)) &\mapsto \ulcorner A(\ulcorner B(x) \urcorner) \urcorner
  \end{align*}
  and
  \begin{align*}
    \alpha: L^0 &\to L^0 \\
    A &\mapsto E(A)
  \end{align*}
  Then we consider the usual composite \(q\) which is given by
  \begin{align*}
    q: L^1 &\to L^0 \\
    A(x) &\mapsto E(\ulcorner A(\ulcorner A(x) \urcorner) \urcorner)
  \end{align*}
  We claim that \(q\) is represented by \(G(x) = E(D(x))\): indeed
  \[
    f(G(x), A(x)) = G(\ulcorner A(x) \urcorner) = E(\ulcorner A(\ulcorner A(x) \urcorner) \urcorner)
  \]
  which is the same as \(q(A(x))\). It follows from general Lawvere fixed point argument that \(C = q(G(x)) = G(\ulcorner G(x) \urcorner)\) is a fixed point of \(E\), i.e.\ \(C = E(C)\).
\end{proof}

\begin{thebibliography}{9}
\bibitem{diagonal arguments}
  F.\ W.\ Lawvere.
  \textit{Diagonal arguments and Cartesian closed categories}.
  Reprints in Theory and Applications of Categories, No. 15, 2006
\bibitem{self-referential paradoxes}
  N.\ S.\ Yanofsky.
  \textit{A Universal Approach to Self-Referential Paradoxes, Incompleteness and Fixed Points}.
  arXiv:math/0305282 [math.LO]
\bibitem{conceptual mathematics}
  F.\ W.\ Lawvere, S.\ H.\ Schanuel.
  \textit{Conceptual Mathematics: A first introduction to categories}.
  Cambridge University Press, 2011
\end{thebibliography}
\end{document}