\documentclass{article}
\input{preamble}
\myexternaldocument[main:]{main}
\title{Appendix of Paper \#6340}
\begin{document}
\maketitle

\begin{abstract}
In this appendix, we provide the proof of the main theorem, the detailed experimental settings for reproducibility, and additional experimental results.
\end{abstract}

\appendix
\section{Proof of \texorpdfstring{\cref{main:thm:risk}}{theorem 2}}\label{sec:proof-of-main-theorem}
As we commented in the main body of the paper, we can regard $\bm b (\bm t) - \hat{\bm b} (\bm t | S_N)$ as a new B\'ezier simplex, and we call its control points as $\bm p'_{\bm d}$.
By using it, the risk $R_N = \E_{S_N} \sqbra{\E_{\bm t} \sqbra{\norm{\bm b (\bm t) - \hat{\bm b} (\bm t | S_N)}^2}}$ can be rewritten as
\begin{align}
R_N
&=
\E_{S_N}
\sqbra{
\E_{\bm t}
\sqbra{
\norm{
\sum_{\bm d \in \N_D^M}
\binom{D}{\bm d}
\bm t^{\bm d} \bm p_{\bm d}'
}^2
}
}
\notag \\
&=
\E_{S_N}
\sqbra{
\E_{\bm t}
\sqbra{
\sum_{\bm d_A, \bm d_B \in \N_D^M}
\binom{D}{\bm d_A}
\binom{D}{\bm d_B}
\bm t^{\bm d_A + \bm d_A }
\paren{
\bm p_{\bm d_A}'
\cdot
\bm p_{\bm d_B}'
}
}
}
\notag \\
&= %%%%
\sum_{\bm d_A, \bm d_B \in \N_D^M}
\binom{D}{\bm d_A}
\binom{D}{\bm d_B}
\E_{\bm t}
\sqbra{
\bm t^{\bm d_A + \bm d_A }
}
\E_{S_N}
\sqbra{
\paren{
\bm p_{\bm d_A}'
\cdot
\bm p_{\bm d_B}'
}
}
.
\label{eqn:RN_in_supp}
\end{align}
The inner product in the expectation over $S_N$ is equivalent to $\paren{\bm P \bm P^\top}_{\bm d_A \bm d_B}$ where $\bm P$ is the control point matrix defined in main body.
So we need to massage the expectation over $\bm t \sim U(\Delta^{M-1})$.
To do it, the following proposition is useful.
\begin{prop}\label{thm:integ_formula}
Suppose $\sum_{i=1}^M q_i = Q \in \N$ and $r \in \R$, then the following integral formula is satisfied.
\begin{align}
I_{\bm q} (r)
&:=
\paren{
\prod_{i=1}^M
\int_0^\infty
 dt_i t_i^{q_i}
}
\delta \paren{
r - \sum_{i=1}^M t_i
}
= %%%%
r^{Q+M-1} \frac{Q!}{(Q+M-1)! } %\prod_{i=1}^M q_i !
\binom{Q}{\bm q}^{-1}
,
\end{align}
where $\delta (r)$ is the Dirac's delta.
\end{prop}
%
\begin{proof}
One of the easiest proofs is done by making the use of the Laplace transform $\mathcal{L}$ and its inverse $\mathcal{L}^{-1}$, i.e. the nature of $I_{\bm q}(r) = \mathcal{L}^{-1} \sqbra{\mathcal{L} \sqbra{I_{\bm q}}} (r)$.
First, the Laplace transform of $I_{\bm q}$ is
\begin{align}
\mathcal{L} [ I_{\bm q} ] (z)
=
\int_0^\infty dr \ e^{-zr} I_{\bm q} (r)
&=
\Big(
\prod_{i=1}^M
\int_0^\infty
 dt_i t_i^{q_i}
 e^{-zt_i}
\Big)
\notag \\
&=
\frac{Q!}{z^{Q+M}}
\binom{Q}{\bm q}^{-1}.
\end{align}
The inverse Laplace transform of a function $f(z)$ is defined by picking up the residue, the coefficient of $1/z$, of $e^{-zr} f(z)$.
One can calculate the residue of $e^{-zr} \mathcal{L} [ I_{\bm q} ] (z)$ by expanding $e^{-zr}$ with respect to $z$.
It provides $z^{Q+M-1}r^{Q+M-1}/{(Q+M-1)!}$, so we get
\[
\mathcal{L}^{-1} [\mathcal{L} [ I_{\bm q} ]] (r)
=
r^{Q+M-1} \frac{Q!}{(Q+M-1)! } %\prod_{i=1}^M q_i !
\binom{Q}{\bm q}^{-1}.
\]
\end{proof}
%%%%%
Now, let us get back to the proof of the theorem.
The expectation value $\E_{\bm t}[ f(\bm t) ]$ can be represented by the following integral
\[
\E_{\bm t} [f(\bm t)]
=
(M-1)!
\Big(
\prod_{i=1}^M
\int_0^\infty
 dt_i
\Big)
\delta \Big(
1 - \sum_{i=1}^M t_i
\Big)
f(\bm t)
\]
The multiplication of $(M-1)!$ is necessary because we need $\E_{\bm t}[1] = 1$.
This is easily checked by using the above proposition by taking $q_i =0$ for all $i$ and $r=1$.
We can also calculate the value $\E_{\bm t}[ \bm t^{\bm d_A + \bm d_B} ]$ by using the proposition with $r=1$ as
\[
\E_{\bm t}[ \bm t^{\bm d_A + \bm d_B} ]
=
\frac{(2D)!(M-1)!}{(2D + M -1)!}
\binom{2D}{\bm d_A + \bm d_B}^{-1}.
\]
By substituting it to \cref{eqn:RN_in_supp}, we get what we want
\begin{align}
R_N
=
\sum_{\bm d_A, \bm d_B \in \N_D^M}
\frac{(2D)!(M-1)!}{(2D + M -1)!}
\binom{D}{\bm d_A}
\binom{D}{\bm d_B}
\binom{2D}{\bm d_A + \bm d_B}^{-1}
\E_{S_N}
\Bigg[
(
\bm P \bm P^\top
)_{\bm d_A \bm d_B}
\Bigg].
\end{align}

\section{Derivation of \texorpdfstring{$\sum_{i, j = 1}^N (A^{-1} \circ A)_{ij} = N$}{sum of Hadamard products}}\label{sec:derivation-of-sum}
\begin{prop}
Let $A = (a_{ij})$ be a regular matrix of size $N$.
Then
\[
    \sum_{i, j = 1}^N (A^{-1} \circ A)_{ij} = N,
\]
where $A \circ B$ is Hadamard product of $A$ and $B$.
\end{prop}
\begin{proof}
Let $\tilde{A}$ be a cofactor matrix and $C_{ij}$ be a $(i, j)$ minor of $A$.
\begin{equation*}
\begin{split}
\sum_{i, j = 1}^N (A^{-1} \circ A)_{ij}
    &= \sum_{i, j = 1}^N \left(\frac{1}{\det(A)}\tilde{A} \circ A\right)_{ij} \\
    &= \frac{1}{\det(A)}\sum_{i, j = 1}^N (\tilde{A}\circ A)_{ij} \\
    &= \frac{1}{\det(A)}\sum_{i, j = 1}^N a_{ij}C_{ij} \\
    &= \frac{1}{\det(A)}\sum_{i = 1}^N \sum_{j = 1}^N a_{ij}C_{ij}\\
    &= \frac{1}{\det(A)}\sum_{i = 1}^N \det(A) \\
    &= N
\end{split}
\end{equation*}
\end{proof}

\section{Proof of \texorpdfstring{\cref{main:thm:sigma-is-non-singular}}{Theorem 3}}\label{sec:proof-of-regularity}
Put
\begin{align}
\ZZ_{d_A d_B}
:=
\frac{(2D)!(M - 1)!}{(2D + M - 1)!}
 \begin{pmatrix}
  D \\
  \bm{d}_A
 \end{pmatrix}
 \begin{pmatrix}
  D \\
  {\bm d}_B
 \end{pmatrix}
 \begin{pmatrix}
  2D \\
  {\bm d}_A + {\bm d}_B
 \end{pmatrix}^{-1}.
\end{align}
We prove $\ZZ = (\ZZ_{AB})$ is a regular matrix by considering the representation of appropriate spaces.
We start to give some review on bilinear form.
\begin{defn}
Let $V$ be a real vector space. A map $B\colon V\times V \rightarrow \R$ is said to be bininear form if for any $\bm{u}, \bm{v}, \bm{w} \in V$ and $\lambda \in \R$,
\begin{eqnarray*}
\bullet B(\bm{u}+ \bm{v}, \bm{w}) = B(\bm{u}+\bm{w}) + B(\bm{v}, \bm{w})\ \text{and}\  B(\lambda \bm{u}, \bm{v}) = \lambda B(\bm{u}, \bm{v}) \\
\bullet B(\bm{u}, \bm{v}+\bm{w})  = B(\bm{u}+\bm{v}) + B(\bm{u}, \bm{w})\ \text{and}\  B( \bm{u},\lambda \bm{v}) = \lambda B(\bm{u}, \bm{v})
\end{eqnarray*}
holds. Moreover, a bilinear form $B$ is called symmetric if $B(\bm{u}, \bm{v})=B(\bm{v}, \bm{u})$ holds for any $\bm{u}, \bm{v} \in V$

\end{defn}

\begin{thm}
Let $V$ be an $n$-dimensional real vector space and $\set{\bm{e}_1, \dots, \bm{e}_n}$ be a basis of $V$. Let  $B\colon V\times V \rightarrow \R$ be a bilinear form. Put $A = (B(\bm{e}_i, \bm{e_j}))$. Then the map

\[
\begin{array}{ccc}
B':V \times V  & \longrightarrow & \R\\
\ \ \ \ \  \rotatebox{90}{$\in$}    &                 & \rotatebox{90}{$\in$} \\
\ \ \ \ \ (\bm{u}, \bm{v})                   & \longmapsto     & \bm{x}^{\top} A\bm{y}
\end{array},
\]
is a bilinear form which is equal to $B$, where $\bm{u}=\sum_{i=1}^n x_i e_i$ and $\bm{v}=\sum_{i=1}^n y_i e_i$.
\end{thm}
We call the matrix $A$ the representation matrix of $B$ with respect to a basis  $\set{\bm{e}_1, \dots, \bm{e}_n}$ .  By the construction of $A$, we have the following proposition.
\begin{prop}
Let $V$ be an $n$-dimensional real vector space and   $B\colon V\times V \rightarrow \R$ be a bilinear form. Then the representation matrix of $B$ with respect to a basis of $V$ is a symmetric matrix if and only if $B$ is a symmetric bilinear form.
\end{prop}
\begin{thm}\label{thm:Sigma-is-regular}
$\ZZ$ is a regular matrix.
\end{thm}
\begin{proof}
Let $V_{M, D}$ is the vector space of the homogenous polynomials of degree $D$ with $M$ variables $\bm{x}$.
We define the symmetric bilinear form  by
\[
\begin{array}{ccc}
L:V_{M, D} \times V_{M, D} & \longrightarrow & \R\\
\ \ \ \  \rotatebox{90}{$\in$}    &                 & \rotatebox{90}{$\in$} \\
\ \ \ \  (P, Q)                   & \longmapsto     & \int_{\Delta} P(\bm{x}) Q(\bm{x}) d\bm{x}
\end{array},
\]
where $\Delta = \Set{\bm{x} = (x_1, \dots, x_M) \in \R^M | x_i >0,  x_1 + \dots + x_M = 1}$.

$L$ is clearly symmetric and bilinear.
Let
\[
    Z = \Set{\frac{\bm{x}^{\bm{d_A}}}{\bm{d_A}!} | \bm{d_A} = (d_A^{(1)}, \dots, d_A^{(M)}) \in \N^M, d_A^{(1)} + \dots + d_A^{(M)} = D}
\]
be a set of polynomials of degree $D$ with $M$ variables.
Then $Z$ is a basis of $V_{M, D}$.
We claim that the representation matrix $G$ of the symmetric bilinear form $L$ with respect to $Z$ is equal to $\ZZ$ up to scalar multiplication.

\begin{claim}\label{thm:rep}
Let $\ZZ$, $Z$, $L$ as above.
Then the representation matrix of $L$ with respect to $Z$ is equal to $r\cdot \ZZ$ for some $r \neq 0 \in \R$.
\end{claim}
\begin{proof}[Proof of \cref{thm:rep}]
We have
\begin{equation*}
\begin{split}
L\paren{\frac{\bm{x}^{\bm{d_A}}}{\bm{d_A}!}, \frac{\bm{x}^{\bm{d_B}}}{\bm{d_B}!}}
  & = \frac{1}{\bm{d_A}!\bm{d_B}!}\int_{\Delta} \bm{x}^{\bm{d_A} + \bm{d_B}}d\bm{x}\\
  & = \frac{1}{\bm{d_A}!\bm{d_B}!} \frac{\Gamma(d_A^{(1)} + d_B^{(1)}) \dots \Gamma(d_A^{(M)} + d_B^{(M)}) }{\Gamma(2D + M)} \\
  & = \frac{(\bm{d_A} + \bm{d_B})!}{\bm{d_A}!\bm{d_B}!}\frac{1}{(2D + M - 1)!}\\
  & = \frac{1}{(D!)^2(M - 1)!} \frac{D!}{\bm{d_A}!} \frac{D!}{\bm{d_B}!} \paren{\frac{2D!}{(\bm{d_A + \bm{d_B}})!}}^{-1} \frac{(2D)!(M - 1)!}{(2D + M - 1)!} \\
  & = \frac{1}{(D!)^2(M - 1)!} \ZZ_{\bm{d_A}\bm{d_B}}.
\end{split}
\end{equation*}
Hence, the claim holds.
\end{proof}
Since $G$ is a symmetric matrix, $G$ is diagonalizable.
Hence, for some basis $Z'$, the representation matrix $H$ of the symmetric bilinear form $L$ with respect to $Z'$ is a diagonal matrix.
In this case, the diagonal components of $H$ is equal to the eigenvalues of $G$ and is equal to $L(P, P)$ for some $P \in Z'$.
Hence, for proving our theorem, it is enough to show that $L(P, P)$ is positive.
To see the positivity of $L(P, P)$, we claim the following.

\begin{claim}\label{thm:hom}
Let $P(\bm{x})$ be a homogenous polynomial of degree $D$ with $M$ variables.
Put $\Delta = \Set{\bm{x} = (x_1, \dots, x_M) \in \R^M | x_i > 0,\ x_1 + \dots + x_M = 1}$.
Assume $\int_{\Delta} P(\bm{x})^2 d \bm{x} = 0$.
Then $P(\bm{x}) = 0$.
\end{claim}

\begin{proof}[Proof of \cref{thm:hom}]
Assume $P(\bm{x}) \neq 0$.
Since $\int_{\Delta} P(\bm{x})^2 d \bm{x} = 0$, we see $P(\bm{x}) = 0$ for any $\bm{x} \in \Delta$.
Since $P(\bm{x}) \neq 0$, we have a point $\bm{a} = (a_1, \dots, a_M) \in \R^M_+$ such that $P(\bm{a}) \neq 0$.
Put $\lambda = \sum a_i$. Then, since $P(\bm{x})$ is a homogeneous polynomial of degree $D$, we have
\[
    P(\bm{a}) = \lambda^D P(\frac{1}{\lambda} \bm{a}) \neq 0.
\]
The non equality holds because $\sum 1 / \lambda \cdot a_i \in \Delta$.
\end{proof}

Since $P$ is a nonzero polynomial, we have
\[
    (P, P) = \int_{\Delta} P(\bm{x})^2 d \bm{x} > 0
\]
by \cref{thm:hom}.
Therefore, any eigenvalue of $G$ is positive.
This implies that $\ZZ$ is a regular matrix.
\end{proof}
%%%%%%
\section{Complete derivation of the risk asymptotics in inductive skeleton fitting}\label{sec:risk-derivation}
Let us begin with how control points on $(m-1)$-subsimplices from minimizing the OLS error
\begin{align}
\frac{1}{N^{(m)}}
\sum_{n=1}^{N^{(m)}}
\norm{
\bm x_n^{(m)}
-
\hat{\bm b} (\bm t_n^{(m)})
}^2
.
\end{align}
In this paper, we assume that sample $\bm x_n$ always represented by sum of a certain B\'esier simplex and noise:
\begin{align}
\bm x_n^{(m)}
=
\bm b (\bm t_n^{(m)})
+
\bm \epsilon_n^{(m)},
\quad
\epsilon_n^{(m)} \sim \mathcal{N}(\bm 0, \sigma^2 \bm I_L)
\end{align}
%%
\paragraph{all-at-once fitting}
All-at-once fitting in our paper can be regarded $m=M$ case.
In this case, we omit the superscript $(m=M)$ and consider minimization of
\begin{align}
\frac{1}{N}
\sum_{n=1}^{N}
\norm{
\bm x_n
-
\hat{\bm b} (\bm t_n)
}^2
&=
\frac{1}{N}
\sum_{n=1}^{N}
\Big| \Big|
{\bm b} (\bm t_n)
-
\hat{\bm b} (\bm t_n)
+
\bm \epsilon_n
\Big| \Big|^2
\notag \\
&=
\frac{1}{N}
\sum_{n=1}^{N}
\Big| \Big|
\sum_{\bm d \in \N_D^M}
\binom{D}{\bm d} \bm t^{\bm d}_n
\bm p_{\bm d}'
+
\bm \epsilon_n
\Big| \Big|^2
\notag \\
&=
\frac{1}{N}
\norm{
\bm Z \bm P
+
\bm Y
}_\mathrm{F}^2,
\end{align}
where the $\norm{\cdot}_\mathrm{F}$ means Frobenius norm of the matrix, and the optimum is determined as
$\bm P_\mathrm{OLS} = - (\bm Z^\top \bm Z)^{-1} \bm Z^\top \bm Y$ by using usual argument.
%%
\paragraph{inductive skeleton fitting}
On the other hand, the above argument should be modified when we consider the inductive skeleton fitting.
First of all, we can reduce the problem to minimization of
\begin{align}
\frac{1}{N^{(m)}}
\sum_{n=1}^{N^{(m)}}
\Big| \Big|
\sum_{\bm d \in \N_D^M}
\binom{D}{\bm d} (\bm t_n^{(m)})^{\bm d}
\bm p_{\bm d}'
+
\bm \epsilon_n^{(m)}
\Big| \Big|^2.
\label{eqn:ols_inductive_supp}
\end{align}
Note that $\bm t_n^{(m)}$ is on $(m-1)$-subsimplices.
It means that the vector
\begin{align}
\bm t_n^{(m)}
=
[
(\bm t_n)_1,
(\bm t_n)_2,
\dots ,
(\bm t_n)_M
]
\end{align}
has $(M-m)$ zero components.
For example, if it is on a subsimplex labelled by
\begin{align}
\bm I = [1, 1, \dots, 1, 0, 0, \dots, 0],
\end{align}
with $m$ ones and $(M-m)$ zeros, then $\bm t_n^{(m)}$ takes
\begin{align}
\bm t_n^{(m)}
=
[
\text{non-zero},
\text{non-zero},
\dots ,
\text{non-zero},
0,
0,
\dots,
0
].
\end{align}
As a result, there is no contribution from $\bm p_{\bm d}'$ which is not on the simplex because such point is labelled by
\begin{align}
\bm d
=
[
d_1,
d_2,
\dots,
d_m,
\text{non-zero},
\text{non-zero},
\dots,
\text{non-zero}
]
\end{align}
because
\begin{align}
(t_n^{(m)})^{\bm d}
=
(\text{non-zero})_1^{d_1}
(\text{non-zero})_2^{d_2}
\dots
(\text{non-zero})_m^{d_m}
(0)^{\text{non-zero}}
(0)^{\text{non-zero}}
\dots
(0)^{\text{non-zero}}
\end{align}
vanishes inside the summation of \cref{eqn:ols_inductive_supp}.
In summary, we should restrict range of the summation in \cref{eqn:ols_inductive_supp} as
\begin{align}
&
\frac{1}{N^{(m)}}
\sum_{n=1}^{N^{(m)}}
\Big| \Big|
\sum_{\bm d \in (\N_D^M)^{(m)}}
\binom{D}{\bm d} (\bm t_n^{(m)})^{\bm d}
\bm p_{\bm d}'
+
\sum_{k < m}
\sum_{\bm d \in (\N_D^M)^{(k)}}
\binom{D}{\bm d} (\bm t_n^{(m)})^{\bm d}
\bm p_{\bm d}'
+
\bm \epsilon_n^{(m)}
\Big| \Big|^2
\notag \\
&=
\frac{1}{N^{(m)}}
\norm{
\bm Z^{(m)} \bm P^{(m)}
+
\sum_{k<m}
\bm Z^{(m)[k]} \bm P^{(k)}
+
\bm Y^{(m)}
}^2_\mathrm{F}.
\label{eqn:ols_ind_supp2}
\end{align}
In the inductive skeleton fitting, we regard all $\bm P^{(k<m)}$ are already determined and fixed in the optimization process of $\bm P^{(m)}$, so the last 2 terms in \cref{eqn:ols_ind_supp2} play a role of $\bm Y$ in all-at-once fitting, and the optimal $\bm P^{(m)}$ is determined as
\begin{align}
{\bm P}_\mathrm{OLS}^{(m)}
= %%%%
-
[({\bm Z}^{(m)})^\top {\bm Z}^{(m)}]^{-1}
({\bm Z}^{(m)})^\top
\Big(
{\bm Y}^{(m)}
+
\sum_{k<m}
{\bm Z}^{(m) [k]}
{\bm P}_\mathrm{OLS}^{(k)}
\Big).
\label{eqn:rec_supp}
\end{align}

To derive the asymptotics of the risk, we have to consider $\E_\text{samples} [ \bm P \bm P^\top ]$.
In the all-at-once fitting, it is simple as explained in the main body of the paper.
The key idea is calculating the expectation over the noise $\bm Y$ first and making use of the law of large numbers.
On the inductive skeleton fitting, we take same strategy.
%
Let us call the control point matrix $\bm P$ determined by the inductive skeleton fitting as $\bm P_\text{ISK}$.
It decomposes to ``direct sum'' along row of the matrix as
\begin{align}
\bm P_\text{ISK}
=
\begin{pmatrix}
{\bm P}^{(1)}_\mathrm{OLS}
\\
{\bm P}^{(2) }_\mathrm{OLS}
\\
\vdots
\\
{\bm P}^{(M)}_\mathrm{OLS}
\end{pmatrix}
.
\end{align}
If the degree of the B\'ezier simplex is less than $m$, there may be no control point on $(m+k)$-subsimplices.
In such case, we regard $\bm P_\mathrm{OLS}^{(m+k)} = \bm 0$.
By using the decomposition, we can get the ``direct sum'' of the matrix $\bm P_\text{ISK} \bm P_\text{ISK}^\top$ as follows.
\begin{align}
\bm P_\text{ISK} \bm P_\text{ISK}^\top
=
\begin{pmatrix}
{\bm P}^{(1)}_\mathrm{OLS}
({\bm P}^{(1)}_\mathrm{OLS})^\top
&
{\bm P}^{(1)}_\mathrm{OLS}
({\bm P}^{(2)}_\mathrm{OLS})^\top
&
\cdots
&
{\bm P}^{(1)}_\mathrm{OLS}
({\bm P}^{(M)}_\mathrm{OLS})^\top
\\
{\bm P}^{(2)}_\mathrm{OLS}
({\bm P}^{(1)}_\mathrm{OLS})^\top
&
{\bm P}^{(2)}_\mathrm{OLS}
({\bm P}^{(2)}_\mathrm{OLS})^\top
&
\cdots
&
{\bm P}^{(2)}_\mathrm{OLS}
({\bm P}^{(M)}_\mathrm{OLS})^\top
\\
\vdots & \vdots & \ & \vdots
\\
{\bm P}^{(M)}_\mathrm{OLS}
({\bm P}^{(1)}_\mathrm{OLS})^\top
&
{\bm P}^{(M)}_\mathrm{OLS}
({\bm P}^{(2)}_\mathrm{OLS})^\top
&
\cdots
&
{\bm P}^{(M)}_\mathrm{OLS}
({\bm P}^{(M)}_\mathrm{OLS})^\top
\end{pmatrix}
.
\end{align}
Therefore, to get the asymptotic form of the risk, it is sufficient to consider $\E_\text{samples}[{\bm P}^{(i)}_\mathrm{OLS}
({\bm P}^{(j)}_\mathrm{OLS})^\top]$ for $i, j = 1, 2, \dots, M$.
To get a grasp of how it calculated, let us take $(i,j) = (1,1)$ and $(i, j) =  (2, 1)$ here.
First, $(i,j) = (1,1)$ is the simplest case because there is no lower-dimensional subsimplex.
So we do not need 2nd term in \cref{eqn:rec_supp} and get
\begin{align}
&\E_\text{samples}
\Big[
\bm P_\mathrm{OLS}^{(1)}
(\bm P_\mathrm{OLS}^{(1)})^\top
\Big]
\notag \\
&=
\E_\text{samples}
\Bigg[
\Big[
-
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
({\bm Z}^{(1)})^\top
{\bm Y}^{(1)}
\Big]
\Big[
-
({\bm Y}^{(1)})^\top
{\bm Z}^{(1)}
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
\Big]
\Bigg]
\notag \\
&=
\E_{\text{samples}}
\Bigg[
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
({\bm Z}^{(1)})^\top
\E_{\bm Y^{(1)}}
\Big[
{\bm Y}^{(1)}
({\bm Y}^{(1)})^\top
\Big]
{\bm Z}^{(1)}
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
\Bigg]
\notag \\
&=
\E_{\text{samples}}
\Bigg[
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
({\bm Z}^{(1)})^\top
\Big[
\sigma^2 L 1_{N^{(1)}}
\Big]
{\bm Z}^{(1)}
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
\Bigg]
\notag \\
&=
\sigma^2 L
\E_{\text{samples}}
\Bigg[
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
\Bigg].
\label{eqn:P1P1}
\end{align}
In fact, this simplification occurs in calculation with all-at-once fitting.
So $(i, j) = (1,1)$ case is in same situation of the calculation of the asymptotic risk of the all-at-once fitting.
How about $(i, j) = (2, 1)$ ?
This is also simple because we can reduce the calculation to lower-dimensional object by using \cref{eqn:rec_supp}:
\begin{align}
&\E_\text{samples}
\Big[
\bm P_\mathrm{OLS}^{(2)}
(\bm P_\mathrm{OLS}^{(1)})^\top
\Big]
\notag \\
&=
\E_\text{samples}
\Bigg[
\Big[
-
[({\bm Z}^{(2)})^\top {\bm Z}^{(2)}]^{-1}
({\bm Z}^{(2)})^\top
\Big(
{\bm Y}^{(2)}
+
\bm Z^{(2)[1]}
\bm P_\mathrm{OLS}^{(1)}
\Big)
\Big]
(\bm P_\mathrm{OLS}^{(1)})^\top
\Bigg]
\notag \\
&=
\E_\text{samples}
\Bigg[
\Big[
-
[({\bm Z}^{(2)})^\top {\bm Z}^{(2)}]^{-1}
({\bm Z}^{(2)})^\top
\Big(
\E_{\bm Y^{(2)}}
[
{\bm Y}^{(2)}
]
+
\bm Z^{(2)[1]}
\bm P_\mathrm{OLS}^{(1)}
\Big)
\Big]
(\bm P_\mathrm{OLS}^{(1)})^\top
\Bigg]
\notag \\
&=
\E_\text{samples}
\Bigg[
\Big[
-
[({\bm Z}^{(2)})^\top {\bm Z}^{(2)}]^{-1}
({\bm Z}^{(2)})^\top
\Big(
[
0
]
+
\bm Z^{(2)[1]}
\bm P_\mathrm{OLS}^{(1)}
\Big)
\Big]
(\bm P_\mathrm{OLS}^{(1)})^\top
\Bigg]
\notag \\
&=
- \sigma^2 L
\E_\text{samples}
\Bigg[
[({\bm Z}^{(2)})^\top {\bm Z}^{(2)}]^{-1}
[({\bm Z}^{(2)})^\top \bm Z^{(2)[1]}]
[({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
\Bigg].
\label{eqn:P2P1}
\end{align}
With generic $(i, j)$, we can perform same procedure.

Now, let us make next step.
To calculate the asymptotic form, it is important to notice that the following explicit forms are averages on sampling from $\bm t_n^{(m)} \sim U(\cup_{|\bm I|=m} \Delta^{\bm I})$.
\begin{align}
&
\frac{1}{N^{(m)}}
\Big(
(\bm Z^{(m)})^\top
\bm Z^{(m)[k]}
\Big)_{\bm d_m \bm d_k}
=
\binom{D}{\bm d_m}
\binom{D}{\bm d_k}
\sum_{n=1}^{N^{(m)}}
\frac{1}{N^{(m)}}
(\bm t_n^{(m)})^{\bm d_m + \bm d_k}
\label{eqn:Zmk}
\\
&
\frac{1}{N^{(m)}}
\Big(
(\bm Z^{(m)})^\top
\bm Z^{(m)}
\Big)_{\bm d_A \bm d_B}
=
\binom{D}{\bm d_A}
\binom{D}{\bm d_B}
\sum_{n=1}^{N^{(m)}}
\frac{1}{N^{(m)}}
(\bm t_n^{(m)})^{\bm d_A + \bm d_B}
\label{eqn:Zmm}
\end{align}
By applying the law of large numbers, they converge to exact values of expectation:
\begin{align}
&
\cref{eqn:Zmk}
\to
\binom{D}{\bm d_m}
\binom{D}{\bm d_k}
\E_{\bm t^{(m)} \sim U(\cup_{|\bm I|=m} \Delta^{\bm I})} [
(\bm t^{(m)})^{\bm d_m + \bm d_k}
]
\label{eqn:Lmk}
\\
&
\cref{eqn:Zmm}
\to
\binom{D}{\bm d_A}
\binom{D}{\bm d_B}
\E_{\bm t^{(m)} \sim U(\cup_{|\bm I|=m} \Delta^{\bm I})} [
(\bm t^{(m)})^{\bm d_A + \bm d_B}
]
\label{eqn:Lmm}
\end{align}
To compute expectation values over $U(\cup_{|\bm I|=m} \Delta^{\bm I})$, the next proposition is useful.
\begin{prop}
\begin{align}
\E_{\bm t \sim U(\cup_{|\bm I|=m} \Delta^{\bm I})} [
\bm t^{\bm d^{(m)} + \bm d^{(k)}}
]
=
\frac{(m-1)!}{\ _M C_m}
\sum_{|\bm I| = m}
\frac{
1_{{\bm I}=(\bm d ^{(m)} +  \bm d ^{(k)} )_{01} }
\prod_{I_i = 1}( \bm d ^{(m)} +  \bm d ^{(k)}  )_i !}{
[\sum_{I_i = 1} ( \bm d ^{(m)} +  \bm d ^{(k)}  )_i
+ m - 1]!
},
\end{align}
where $\bm I = [I_1, I_2, \dots, I_M]$ represents vector with binary components, $I_i \in \set{0, 1}$ and $\card{\bm I} = \sum_{i=1}^M I_i$.
\end{prop}
%%%
\begin{proof}
The expectation value with the uniform distribution on $(m-1)$-subsimplices should be expressed by integral
\begin{align}
\E_{\bm t \sim U(\cup_{|\bm I|=m} \Delta^{\bm I})} [ f(\bm t) ]
=
(\text{const.})
\int_{0}^1 \Big(
\prod_{i=1}^M dt_i
\Big)
\ f(\bm t)
\sum_{|\bm I| = m}
\delta \Big(
1 - \sum_{I_i =1} t_i
\Big)
\prod_{I_i =0}
\delta (t_i)
\label{eqn:exp_sub}
\end{align}
because it can be regarded integral over $\Delta^{M}$ restricted to the set of subsimplices.
The normalization constant can be calculated by the integral $f(\bm t) = 1$ inserted.
In this situation, there is no special direction on $\bm t$, and all contributions from the sum $\sum_{|\bm I|=m}$ should be identical.
Therefore, we do not need to calculate all of them but just select one configuration $\bm I$ easy to calculate and multiply the number of combination giving $|\bm I|=m$, i.e. $\ _M C_m$.
Let us take
\begin{align}
\bm I = [\underbrace{1,1, \dots, 1}_{m}, 0, 0, \dots, 0]
\end{align}
then the integral reduces to
\begin{align}
\int_{0}^1 \Big(
\prod_{i=1}^m dt_i
\Big)
\delta \Big(
1 - \sum_{i=1}^m t_i
\Big)
=
\frac{1}{(m-1)!}
\end{align}
where we use the integral formula derived in \ref{thm:integ_formula}.
It determines the constant in \cref{eqn:exp_sub},
\begin{align}
(\text{const.})
=
\frac{(m-1)!}{\ _M C_m}
.
\end{align}
Next task is performing the integral $f(\bm t) = \bm t^{\bm d^{(m)} + \bm d^{(k)}}$ inserted.
It is also not so difficult.
To get nonzero value from $\bm I$-th contribution, $\bm d^{(m)} + \bm d^{(k)}$ should be on subsimplex labelled by $\bm I$.
If not so, the integrand includes
\begin{align}
\int dt \
t^\text{non-zero}
\delta(t)
=
0^\text{non-zero}
=0.
\end{align}
To represent this condition, let us introduce the notation
\begin{align}
(\bm d)_{01}
=
[d^\mathrm{bin}_1, d^\mathrm{bin}_2, \dots, d^\mathrm{bin}_M],
\quad
d^\mathrm{bin}_i
=
\begin{cases}
0 & \text{if $d_i = 0$}\\
1 & \text{otherwise}
\end{cases}
\end{align}
By using this notation, the nonzero contribution condition is represented by the insertion
\begin{align}
1_{\bm I = (\bm d^{(m)} + \bm d^{(k)})_{01} }.
\end{align}
If index $\bm I$ in the summation $\sum_{|\bm I| = m}$ enjoys this condition, the remaining part is integral on an $(m-1)$-subsimplex labelled by $\bm I$, and the contribution is
\begin{align}
\frac{\prod_{I_i=1} (\bm d^{(m)} + \bm d^{(k)})_i ! }{
[
\sum_{I_i=1}(\bm d^{(m)} + \bm d^{(k)})_i + m -1 ]!
}
\end{align}
which is derived in the same way of the integral formula shown in \ref{thm:integ_formula}.
\end{proof}
Applying this proposition to \cref{eqn:Lmk,eqn:Lmm}, we get the asymptotic formula of $
\E_\text{samples}[
\bm P_\text{ISK} \bm P_\text{ISK}^\top]$, or equivalently, $\E_\text{samples}[\bm P_\mathrm{OLS}^{(i)} (\bm P_\mathrm{OLS}^{(j)})^\top]$.
For example,
\begin{align}
\E_\text{samples}[\bm P_\mathrm{OLS}^{(1)} (\bm P_\mathrm{OLS}^{(1)})^\top]
&=
\cref{eqn:P1P1}
=
\frac{\sigma^2 L}{N^{(1)}} \E_\text{samples}
\Big[
[\frac{1}{N^{(1)}} (\bm Z^{(1)})^\top \bm Z^{(1)} ]^{-1}
\Big]
\notag \\
& \to
\frac{\sigma^2 L}{N^{(1)}} \E_\text{samples}
\Big[
\bm \Lambda_{(1)}
\Big]
=
\frac{\sigma^2 L}{N^{(1)}} \bm \Lambda_{(1)},
\end{align}
and
\begin{align}
&\E_\text{samples}[\bm P_\mathrm{OLS}^{(2)} (\bm P_\mathrm{OLS}^{(1)})^\top]
\notag \\
&=
\cref{eqn:P2P1}
=
- \frac{\sigma^2 L}{N^{(1)}}
\E_\text{samples}
\Bigg[
[\frac{1}{N^{(2)}} ({\bm Z}^{(2)})^\top {\bm Z}^{(2)}]^{-1}
[\frac{1}{N^{(2)}}({\bm Z}^{(2)})^\top \bm Z^{(2)[1]}]
[\frac{1}{N^{(1)}}({\bm Z}^{(1)})^\top {\bm Z}^{(1)}]^{-1}
\Bigg]
\notag \\
& \to
- \frac{\sigma^2 L}{N^{(1)}}
\E_\text{samples}
\Bigg[
\bm \Lambda_{(2)}
\bm \Lambda^{(2)[1]}
\bm \Lambda_{(1)}
\Bigg]
=
- \frac{\sigma^2 L}{N^{(1)}}
\bm \Lambda_{(2)}
\bm \Lambda^{(2)[1]}
\bm \Lambda_{(1)},
\end{align}
where $\bm \Lambda$s are defined in the main body of the paper.
As one can see, to complete our argument, it is sufficient to show the next proposition:
\begin{prop}
\begin{align}
&\E_\text{all noises}
\Big[
\bm P^{(i)}_\mathrm{OLS}
(\bm P^{(j)}_\mathrm{OLS})^\top
\Big]
\notag \\
&=
\sigma^2 L
\sum_{
\substack{
m \leq i
\\
m \leq j
}}
\sum_{
\substack{
m\leq k_1<\cdots<k_{\heartsuit}< i
\\
m \leq l_1<\cdots<l_{\spadesuit}<j
}
}
\frac{(-1)^{\heartsuit + \spadesuit}}{N^{(m)}}
%\notag \\ \nline \quad \times %\qquad %\qquad
\hat{\bm \Lambda}_{(i)}
\hat{\bm \Lambda}^{(i)[k_{\heartsuit}]}
\hat{\bm \Lambda}_{(k_{\heartsuit} )}
%\hat{\bm \Lambda}^{(k_{\#_i})[k_{\#_i -1}]}
\cdots
\hat{\bm \Lambda}^{(k_1)[m]}
\hat{\bm \Lambda}_{(m)}
\hat{\bm \Lambda}^{[m](l_1)}
\cdots
\hat{\bm \Lambda}_{(l_{\spadesuit} )}
\hat{\bm \Lambda}^{[l_{\spadesuit}](j)}
\hat{\bm \Lambda}_{(j)},
\label{eqn:31}
\end{align}
where
\begin{align}
&\hat{\bm \Lambda}^{(m)[k]}
=
\frac{1}{N^{(m)}}
({\bm Z}^{(m)})^\top
{\bm Z}^{(m)[k]}
\\
&
\hat{\bm \Lambda}^{[k](m)}
=
(\hat{\bm \Lambda}^{(m)[k]})^\top
\\
&
\hat{\bm \Lambda}_{(m)}
=
[\frac{1}{N^{(m)}}({\bm Z}^{(m)})^\top
{\bm Z}^{(m)}]^{-1}
\end{align}
\end{prop}
%%%
\begin{proof}
We show it by induction.
For $(i, j) = (1, 1)$, the only possible contribution of the summation in \cref{eqn:31} is $m=1$ and the (RHS) reduces to
\begin{align}
\sigma^2 L
\frac{(-1)^{0+0}}{N^{(1)}}
\hat{\bm \Lambda}_{(1)}
\hat{\bm \Lambda}^{(1)[1]}
\hat{\bm \Lambda}_{(1)}
\hat{\bm \Lambda}^{[1](1)}
\hat{\bm \Lambda}_{(1)}
=
\frac{\sigma^2 L}{N^{(1)}}
\hat{\bm \Lambda}_{(1)},
\end{align}
which is satisfied as we shown already.
Now let us assume \cref{eqn:31} with $(k, j)$ for all $k< i+1$, then, by using the recursive relation \cref{eqn:rec_supp}, we get
\begin{align}
&\E_\text{all noises}
\Big[
\bm P_\mathrm{OLS}^{(i+1)}
(\bm P_\mathrm{OLS}^{(j)})^\top
\Big]
\notag \\
&=
\E_\text{all noises}
\Big[
\Big\{
-
[(\bm Z^{(i+1)})^\top \bm Z^{(i+1)} ]^{-1}
(\bm Z^{(i+1)})^\top
\Big(
\bm Y^{(i+1)}
+
\sum_{k<i+1}
\bm Z^{(i+1)[k]}
\bm P_\mathrm{OLS}^{(k)}
\Big)
\Big\}
(\bm P_\mathrm{OLS}^{(j)})^\top
\Big].
\label{eqn:41}
\end{align}
1st term gives
\begin{align}
&\E_\text{all noises}
\Big[
-
[(\bm Z^{(i+1)})^\top \bm Z^{(i+1)} ]^{-1}
(\bm Z^{(i+1)})^\top
\bm Y^{(i+1)}
(\bm P_\mathrm{OLS}^{(j)})^\top
\Bigg]
\notag \\
&= %%%%
\E_\text{all noises}
\Big[
-
[(\bm Z^{(i+1)})^\top \bm Z^{(i+1)} ]^{-1}
(\bm Z^{(i+1)})^\top
\bm Y^{(i+1)}
\Big\{
-
(\bm Y^{(j)})^\top
\notag \\ & \hspace{180pt}
-
\sum_{m < j}
(\bm P_\mathrm{OLS}^{(m)})^\top
\bm Z^{[m](j)}
\Big\}
\bm Z^{(j)}
[(\bm Z^{(j)})^\top \bm Z^{(j)} ]^{-1}
\Big]
\notag \\
&= %%%%
\E_\text{all noises}
\Big[
\underbrace{
[(\bm Z^{(i+1)})^\top \bm Z^{(i+1)} ]^{-1}
(\bm Z^{(i+1)})^\top
\bm Y^{(i+1)}
(\bm Y^{(j)})^\top
\bm Z^{(j)}
[(\bm Z^{(j)})^\top \bm Z^{(j)} ]^{-1}
}_{(*1)}
\notag \\ &
+
\underbrace{
[(\bm Z^{(i+1)})^\top \bm Z^{(i+1)} ]^{-1}
(\bm Z^{(i+1)})^\top
\bm Y^{(i+1)}
\sum_{m < j}
(\bm P_\mathrm{OLS}^{(m)})^\top
\bm Z^{[m](j)}
\bm Z^{(j)}
[(\bm Z^{(j)})^\top \bm Z^{(j)} ]^{-1}
}_{(*2)}
\Big].
\end{align}
If $i+1 = j$, the first term only contribute because there is no $(\bm Y^{(i+1)})^\top$ in the summation $\sum_{m < j=i+1}$ of the second term, and it gives
\begin{align}
\E (*1)
=
[(\bm Z^{(i+1)})^\top \bm Z^{(i+1)} ]^{-1}
=
\frac{1}{N^{(i+1)}}
\hat{\bm \Lambda}_{(i+1)}
\label{eqn:43}
\end{align}
If $i+1 \neq j$, the first term vanish in the same reason.
In this case, the second term possibly contribute if $i+1 < j$.
By applying the recursive formula \cref{eqn:rec_supp} to $\bm P_\mathrm{OLS}^{(m)}$ repeatedly until $(\bm Y^{(i+1)})^\top$ appears , we get
\begin{align}
\E (*2)
=
\sigma^2 L
\sum_{m < j}
\sum_{m \leq l_1 < \dots < l_\spadesuit <j }
\frac{(-1)^{\spadesuit}}{N^{(i+1)}}
\hat{\bm \Lambda}_{(i+1)}
\hat{\bm \Lambda}^{[i+1](l_1)}
\hat{\bm \Lambda}_{(l_1)}
\dots
\hat{\bm \Lambda}_{(l_\spadesuit)}
\hat{\bm \Lambda}^{[l_\spadesuit](j)}
\hat{\bm \Lambda}_{(j)}.
\label{eqn:44}
\end{align}

On the other hand, the 2nd term in \cref{eqn:41} is
\begin{align}
&
\E_\text{all noises}
\Big[
-
[(\bm Z^{(i+1)})^\top \bm Z^{(i+1)}]^{-1}
(\bm Z ^{(i+1)})^\top
\sum_{k<i+1}
\bm Z^{(i+1)[k]}
\bm P_\mathrm{OLS}^{(k)}
(\bm P_\mathrm{OLS}^{(j)})^\top
\Big]
\notag \\
&=
\sum_{k<i+1}
(-1)
\hat{\bm \Lambda}_{(i+1)}
\hat{\bm \Lambda}^{(i+1)[k]}
\sigma^2 L
\sum_{
\substack{
m \leq k
\\
m \leq j
}}
\sum_{
\substack{
m\leq k_1<\cdots<k_{\heartsuit}<k
\\
m \leq l_1<\cdots<l_{\spadesuit}<j
}
}
\notag \\ & \qquad\qquad\qquad
\frac{(-1)^{\heartsuit + \spadesuit}}{N^{(m)}}
%\notag \\ \nline \quad \times %\qquad %\qquad
\hat{\bm \Lambda}_{(k)}
\hat{\bm \Lambda}^{(k)[k_{\heartsuit}]}
\hat{\bm \Lambda}_{(k_{\heartsuit} )}
%\hat{\bm \Lambda}^{(k_{\#_i})[k_{\#_i -1}]}
\cdots
\hat{\bm \Lambda}^{(k_1)[m]}
\hat{\bm \Lambda}_{(m)}
\hat{\bm \Lambda}^{[m](l_1)}
\cdots
\hat{\bm \Lambda}_{(l_{\spadesuit} )}
\hat{\bm \Lambda}^{[l_{\spadesuit}](j)}
\hat{\bm \Lambda}_{(j)}
\notag \\
&=
\sigma^2 L
\sum_{
\substack{
m \leq k
\\
m \leq j
}}
\sum_{
\substack{
m\leq k_1<\cdots<k_{\heartsuit}<k<i+1
\\
m \leq l_1<\cdots<l_{\spadesuit}<j
}
}
\hat{\bm \Lambda}_{(i+1)}
\hat{\bm \Lambda}^{(i+1)[k]}
\notag \\ & \qquad\qquad\qquad
\frac{(-1)^{\heartsuit+1 + \spadesuit}}{N^{(m)}}
%\notag \\ \nline \quad \times %\qquad %\qquad
\hat{\bm \Lambda}_{(k)}
\hat{\bm \Lambda}^{(k)[k_{\heartsuit}]}
\hat{\bm \Lambda}_{(k_{\heartsuit} )}
%\hat{\bm \Lambda}^{(k_{\#_i})[k_{\#_i -1}]}
\cdots
\hat{\bm \Lambda}^{(k_1)[m]}
\hat{\bm \Lambda}_{(m)}
\hat{\bm \Lambda}^{[m](l_1)}
\cdots
\hat{\bm \Lambda}_{(l_{\spadesuit} )}
\hat{\bm \Lambda}^{[l_{\spadesuit}](j)}
\hat{\bm \Lambda}_{(j)}
\notag \\
&= %%%%
\sigma^2 L
\sum_{
\substack{
m \leq k
\\
m \leq j
}}
\sum_{
\substack{
m\leq k_1<\cdots<k_{\heartsuit}<k<i+1
\\
m \leq l_1<\cdots<l_{\spadesuit}<j
}
}
\notag \\ &
\frac{(-1)^{\heartsuit+1 + \spadesuit}}{N^{(m)}}
\hat{\bm \Lambda}_{(i+1)}
\hat{\bm \Lambda}^{(i+1)[k]}
%\notag \\ \nline \quad \times %\qquad %\qquad
\hat{\bm \Lambda}_{(k)}
\hat{\bm \Lambda}^{(k)[k_{\heartsuit}]}
\hat{\bm \Lambda}_{(k_{\heartsuit} )}
%\hat{\bm \Lambda}^{(k_{\#_i})[k_{\#_i -1}]}
\cdots
\hat{\bm \Lambda}^{(k_1)[m]}
\hat{\bm \Lambda}_{(m)}
\hat{\bm \Lambda}^{[m](l_1)}
\cdots
\hat{\bm \Lambda}_{(l_{\spadesuit} )}
\hat{\bm \Lambda}^{[l_{\spadesuit}](j)}
\hat{\bm \Lambda}_{(j)}
\end{align}
This is close to what we want to show by renaming $k = k_{\heartsuit + 1}$, but it lacks contribution from $m = i+1$:
\begin{align}
\sigma^2 L
\sum_{m \leq j}
\sum_{m \leq l_1 < \dots < l_\spadesuit <j }
\frac{(-1)^{\spadesuit}}{N^{(i+1)}}
\hat{\bm \Lambda}_{(i+1)}
\hat{\bm \Lambda}^{[i+1](l_1)}
\hat{\bm \Lambda}_{(l_1)}
\dots
\hat{\bm \Lambda}_{(l_\spadesuit)}
\hat{\bm \Lambda}^{[l_\spadesuit](j)}
\hat{\bm \Lambda}_{(j)}
.
\end{align}
But is is compensated by \cref{eqn:43,eqn:44}, if $j=i+1$ and $j \neq i+1$ respectively, so we succeed in deriving the equation with $(i+1, j)$.

On $(i, j+1)$ case, we can show it by repeating the above argument in transposed version.
\end{proof}
Now we complete the derivation of the asymptotic form of $\E_\text{samples}[\bm P_\text{ISK} \bm P_\text{ISK}^\top]$.

%%%%%%
The following proposition gives the optimal value of the risk of inductive skeleton fitting in the case of $D=2$.
\begin{prop}
Let $N$ be a natural number and $a, b$ be positive real numbers.
Let $f(x) = \frac{a}{x} + \frac{b}{N - x}$.
Assume $a > b$ and $\frac{a - \sqrt{ab}}{a - b} < 1$.
Then $\min \Set{f(x) | x \in \R, 0 < x < N} = \frac{a + b + 2 \sqrt{ab}}{N}$.
\end{prop}

\begin{proof}
At first, we consider the differential of $f(x)$.
We have
\[
    f'(x) = -\frac{(a - b) x^2 - 2aNx + aN^2}{x^2 (N - x)^2}.
\]
$f'(x) = 0$ if and only if $x = \frac{a \pm \sqrt{ab}}{a - b}N$.
By the assumptions, $f(x)$ takes the minimal value at $x = \frac{a - \sqrt{ab}}{a - b}N$.
Hence, we have
\[
    \min \Set{f(x) | x \in \R, 0 < x < N} = f(\frac{a - \sqrt{ab}}{a - b} N) = \frac{a + b + 2 \sqrt{ab}}{N}.
\]
\end{proof}

%How to generate data sets, sample size, etc.
\section{All experiments}\label{sec:numerical-experiments}
This section provides all experiments we conducted and their detailed settings for completeness and reproducibility.

\subsection{Synthetic instances}
First of all, we present detailed experimental settings and results of synthetic instances described in \cref{main:sec:synthetic-instances} of the main paper.

\subsubsection{Data generation}
We consider the following data generating process to create synthetic instances.
Given parameters $(L, M, N)$, we first define true control points $\bm p_{\bm d}\in \R^L~(\bm d \in \N^M_D)$ as follows:
\begin{equation}\label{eqn:def_control_points}
    \bm p_{\bm d} := \sum_{j=1}^M \frac{d_j}{D} \bm e_j,
\end{equation}
where $\bm e_j \in \R^L~(j = 1, \dots, M)$ is a unit vector whose $j$-th element is one and the others are zeros.
The B\'ezier simplex defined by \cref{eqn:def_control_points} is a unit $(M-1)$-simplex on $\R^L$ and their $M$ vertices are $\bm e_j~(j = 1, \dots, M)$.

Next, we randomly generated $N$ training points $\set{(\bm t_n, \bm x_n)}_{n = 1}^N$%$S_N = \Set{(\bm t_1, \bm x_1), \dots, (\bm t_N, \bm x_N)}$ and $S_{N(0)}, S_{N(1)}$
for the all-at-once fitting and the inductive skeleton fitting respectively.
For the all-at-once fitting, we generated parameters $\bm t_n~(n = 1, \dots, N)$ randomly from an uniform distribution $U(\Delta^{M-1})$, and set up $\bm x_n$ as follows:
\begin{equation}\label{eqn:exp_add_noise}
    \bm x_n = \sum_{\bm d \in \N^M_D} \binom{D}{\bm d} \bm t_n^{\bm d} \bm p_{\bm d} + \bm \varepsilon_n \quad (n = 1, \dots, N),
\end{equation}
where $\bm \varepsilon_n~(n = 1, \dots, N)$ is a noise generated from a normal distribution $N(\bm 0, 0.1^2 \bm I)$.

For the inductive skeleton on the other hand, we set up training points for each dimensional subsimplex of $\Delta^{(m-1)} = \cup_{\card{\bm I} = m} \Delta^{\bm{I}}~(m = 1, \dots, M)$ to be learned.
First, we decoupled $N$ into $N^{(m)}~(m = 0, \dots, M-1)$.
To obtain parameters $\bm{t}^{(m)}_n~(n = 1, \dots, N^{(m)})$ from $\Delta^{(m)} = \cup_{\card{I} = m} \Delta^{\bm{I}}$, we further divided $N^{(m)}$ by the number of $m$-subsimplices $\Delta^{\bm{I}}~(\card{I} = m)$, and generated parameters of equal size from each uniform distribution $U(\Delta^{\bm{I}})$.
%generated parameters $\bm{t}^{(m)}_n~(n = 1, \dots, N^{(m)}$ for each subsimplex from an uniform distribution $U(\Delta^m)$.
Then we produced training points $\bm x^{(m)}_n~(n = 1, \dots, N^{(m)})$ in the same way as \cref{eqn:exp_add_noise} and obtained training points $\set{(\bm x_n^{(m)}, \bm t_n^{(m)})}_{n=1}^{N^{(m)}}$ for each $(m)$-skeleton.

%In the experiments,
\subsubsection{Experimental settings}
Experiments were conducted on the following tuple $(N, M, L)$ with $D \in \Set{2, 3}$:
\begin{itemize}
    \item  $(M, L) = (8, 100)$ and $N \in \Set{250, 500, 1000, 2000}$;
    \item  $(N, L) = (1000, 100)$ and $M \in \Set{3, 4, 5, 6, 7, 8}$;
    \item  $(M, N) = (8, 1000)$ and $ M \in \Set{8, 25, 50, 100}$.
\end{itemize}

To verify the asymptotic risks derived in Section 3.1 and Section 3.2, we compared the following three methods:
\begin{description}
    \item[all-at-once] the all-at-once fitting (Section 3.1);
    \item[inductive skeleton (non-optimal)] the inductive skeleton fitting (Section 3.2) with $N^{(0)} = \dots = N^{(M-1)} = N / M$, which is not the minimizer of the risk of the inductive skeleton fitting;
    \item[inductive skeleton (optimal)] the inductive skeleton fitting (Section 3.2) where $N^{(0)}, N^{(1)}, \dots, N^{(M-1)}$ are determined by minimizing the risk of the inductive skeleton fitting under the constraints $\sum_{m = 0}^{M-1} N^{(m)} = N$ and $N^{(m)}\geq 0~(m = 0, \dots, M-1)$. The actual sample sizes $N^{(m)}$ for each $(M,D)$ are described in \cref{tab:optimal-subsample-ratio}.
    \Cref{tab:optimal-subsample-ratio} shows the optimal solutions of $N^{(m)}$ for each pairs $(M, D)$.
\end{description}
%For the inductive skeleton fitting (optimal), we decoupled $N$ into $N(m)$ according to the following table:
\begin{table}[ht]
    \centering
    \caption{Optimal subsample ratio for inductive skeleton fitting ($D$: degree of B\'ezier simplex, $M$: dimension of B\'ezier simplex, $N$: total sample size, $N^{(m)}$: sample size of $m$-skeleton).}
    \label{tab:optimal-subsample-ratio}
    \begin{tabular}{cccccccc}
        \toprule
        $D$     &$N^{(m)}$ &$M = 3$  &$M = 4$  &$M = 5$  &$M = 6$  &$M = 7$  &$M = 8$ \\ \midrule
        $2$     &$N^{(0)}$ &0.262$N$ &0.229$N$ &0.228$N$ &0.233$N$ &0.238$N$ &0.242$N$\\
                &$N^{(1)}$ &0.738$N$ &0.771$N$ &0.772$N$ &0.767$N$ &0.762$N$ &0.758$N$\\  \hline
        $3$     &$N^{(0)}$ &0.118$N$ &0.083$N$ &0.066$N$ &0.058$N$ &0.052$N$ &0.051$N$\\
                &$N^{(1)}$ &0.576$N$ &0.444$N$ &0.547$N$ &0.577$N$ &0.589$N$ &0.613$N$\\
                &$N^{(2)}$ &0.306$N$ &0.473$N$ &0.387$N$ &0.365$N$ &0.359$N$ &0.336$N$\\
        \bottomrule
    \end{tabular}
\end{table}

In this experiment, we estimated the B\'ezier simplex with degree $D = 2$ and 3 respectively.

\subsubsection{All results}\label{sec:results}
\Cref{fig:mse_vsN} shows box plots of MSEs over 20 trials and our theoretical risks for both the all-at-once fitting and the inductive skeleton fitting for each $N \in \Set{250, 500, 1000, 2000}$ with $(L, M) = (100, 8)$ and $D \in \Set{2, 3}$.
We observed that these figures empirically show that our theoretical risks are correct for both $D = 2$ and 3, and the gap between the actual MSEs and the risks are sufficiently small at $N = 1000$.
For both $D = 2$ and 3, the inductive skeleton (optimal) always achieved lower MSEs than that of the inductive skeleton (non-optimal).
This result suggests the efficiency of optimizing the risk of the inductive skeleton fitting with respect to the sample sizes $N^{(m)}$ of each dimension.
In addition, the inductive skeleton fitting (optimal) also outperformed the all-at-once fitting in the case of $D = 2$.
This result supports the discussion described in Section 3.3.
\begin{figure}[ht]
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/D=2_M=8_L=100.pdf}
    \subcaption{D=2}
    \label{fig:mse_vsN_d2}
 \end{minipage}
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/D=3_M=8_L=100.pdf}
    \subcaption{D=3}
    \label{fig:mse_vsN_d3}
 \end{minipage}
 \caption{Sample size $N$ vs. MSE with $(L,M)=(0.1,100,8)$ (boxplots over 20 trials and theoretical risks).}
 \label{fig:mse_vsN}
\end{figure}

\Cref{fig:mse_vsM} shows box plots of MSEs over 20 trials and our theoretical risks for each $M \in \Set{3, 4, 5, 6, 7, 8}$ with $(L, N) = (100, 1000)$.
As well as \cref{fig:mse_vsN}, the inductive skeleton fitting always outperformed the all-at-once fitting in the case of $D = 2$.
Furthermore, the difference of MSEs between the inductive skeleton fitting and the all-at-once fitting gets wider as $M$ grows.
This suggests that the inductive skeleton fitting with $D = 2$ approximates more effectively than the all-at-once fitting does for a high-dimensional $M$.
\begin{figure}[ht]
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/D=2_L=100_N=1000.pdf}
    \subcaption{$D = 2$}    \label{fig:mse_vsM_d2}
 \end{minipage}
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/D=3_L=100_N=1000.pdf}
    \subcaption{$D = 3$}    \label{fig:mse_vsM_d3}
 \end{minipage}
 \caption{Dimension of a simplex $M$ vs. MSE with $(L, N) = (100, 1000)$ (boxplots over 20 trials and theoretical risks).}
 \label{fig:mse_vsM}
\end{figure}

\Cref{fig:mse_vsL} shows box plots of MSEs over 20 trials and our theoretical risks for each $L\in \Set{8, 25, 50, 100}$ and $D = 2, 3$ with $(M, N)=(8, 1000)$.
As well as \cref{fig:mse_vsN,fig:mse_vsM}, the inductive skeleton fitting always outperform the all-at-once fitting in the case of $D = 2$.

\begin{figure}[ht]
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/D=2_M=8_N=1000.pdf}
    \subcaption{$D = 2$}
    \label{fig:mse_vsL_d2}
 \end{minipage}
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/D=3_M=8_N=1000.pdf}
    \subcaption{$D = 3$}
    \label{fig:mse_vsL_d3}
 \end{minipage}
 \caption{Dimension of control points $L$ vs. MSE with sample size $(M, N) = (3, 8, 1000)$ (boxplots over 20 trials and theoretical risks).}
\label{fig:mse_vsL}
\end{figure}

\subsection{Multi-objective optimization instances}\label{sec:MOP-instances}
Next, we describe the experiment protocol for the multi-objective optimization instances used in \cref{main:sec:MOP-instances} of the main paper.
We only present the data generation process since the fitting and evaluation part are the same as the synthetic cases.

\subsubsection{A generalized location problem}\label{sec:location-problem}
We generalized the multi-objective location problem \cite{Kuhn1967} to a higher dimension:
\begin{equation}
\begin{split}
\text{minimize } & f(x) = (f_1(x), f_2(x), f_3(x)) \text{ subject to }x \in \R^4\\
\text{where }    & f_m(x) = \norm{x - e_m}^2 \quad (m = 1, \dots, 3)\\
                 & e_1 = (1,0,0,0),\ e_2 = (0,1,0,0),\ e_3 = (0,0,1,0).
\end{split}
\end{equation}
Note that this is a special case of the MED benchmark problem \cite{Hamada2010}.
The MED problem is simplicial \cite{Hamada2017} and its Pareto set is known to be the convex hull of the minimizers of separate objective functions, i.e., the 2-simplex spanned by $e_1, e_2, e_3$.
For each vertex, edge, face of this simplex, which is the Pareto set of each 1-, 2-, 3-objective subproblem, we generate a subsample according to the uniform distribution on it.

\subsubsection{The group lasso}\label{sec:group-lasso}
We applied the B\'ezier simplex fittings to estimation of the hyper-parameter space of a sparse regression method.
The dataset used in this experiment was \texttt{Birthwt} in the R-package \texttt{MASS}, which contains 189 births at the Baystate Medical Centre, Springfield, Massachusetts during 1986 \cite{Hosmer1989,Venables2002}.
From the dataset, we adopted six continuous features \texttt{age1}, \texttt{age2}, \texttt{age3}, \texttt{lwt1}, \texttt{lwt2}, \texttt{lwt3} as predictors and one continuous feature \texttt{bwt} as a response for regression analysis.
Since the predictors are classified into two groups, \texttt{age} and \texttt{lwt}, the group lasso~\cite{Yuan2006} was employed.

Put $N=189$ and $M=6$.
Let $A$ be an $N \times M$ matrix of observations of the predictors, $x \in \R^M$ be a row vector of the predictor coefficients to be estimated, separated into two groups $x_\text{age} = (x_1, x_2, x_3)^\top$ and $x_\text{lwt} = (x_4, x_5, x_6)^\top$, and $y \in \R^N$ be a row vector of observations of the response.
The group lasso regressor is the solution to the following problem:
\begin{equation}\label{eqn:group-lasso}
\text{minimize } \frac{1}{2N} \norm{Ax - y}^2 + \frac{\lambda}{\sqrt{3}} \paren{\norm{x_\text{age}} + \norm{x_\text{lwt}}} \text{ subject to } x \in \R^6
\end{equation}
where $\norm{\cdot}$ is the Euclidean norm, and $\lambda$ is a positive number to be tuned by users.
This original form suffers from two drawbacks:
\begin{itemize}
    \item Choosing an appropriate value for $\lambda$ involves a grid search on an unbounded domain.
    \item Since two groups have physically different units of measurement, same weights are not always appropriate even if their values are normalized.
\end{itemize}

Instead, we consider each term in \cref{eqn:group-lasso} as a separate objective function:
\begin{equation}\label{eqn:group-lasso-mop}
\begin{split}
    \text{minimize } & f(x)   = (f_1(x), f_2(x), f_3(x)) \text{ subject to } x \in \R^6\\
    \text{where }    & f_1(x) = \norm{Ax - y}^2,\ f_2(x) = \norm{x_\text{age}}^2,\ f_3(x) = \norm{x_\text{lwt}}^2.
\end{split}
\end{equation}
Notice that the use of the squared norm in $f_2$ and $f_3$ does not change their solutions.
It is easy to see that every objective function in \cref{eqn:group-lasso-mop} is convex but not strongly convex.
We make them strongly convex by the following perturbation:
\begin{align*}
    \tilde f_1 &= f_1 + \varepsilon \norm{x}^2,\\
    \tilde f_2 &= f_2 + \varepsilon \norm{x}^2,\\
    \tilde f_3 &= f_3 + \varepsilon \norm{x}^2
\end{align*}
where $\varepsilon$ is an arbitrarily small positive number (we set $\varepsilon=10^{-4}$).
Now the problem minimizing a mapping $\tilde f= (\tilde f_1, \tilde f_2, \tilde f_3)$ is strongly convex.
By \cite[Theorems 1.1 and 3.1]{Hamada2019}, this problem is weakly simplicial and the mapping
\begin{equation}\label{eqn:group-lasso-sop}
    x^*(w) = \arg\min_x \inprod{w}{f(x)}
\end{equation}
is well-defined and continuous on $\Delta^2$, satisfying $x^*(\Delta^2_I) = X^*(\tilde f_I)$ for all $I \subseteq \set{1,2,3}$.

Then, we obtained subsamples by solving \cref{eqn:group-lasso-sop} repeatedly with varying $w \in \Delta^2_I$ for each $I \subseteq \set{1, 2, 3}$.
For each such $I$, the weight $w$ was drawn from the uniform distribution on $\Delta^2_I$ and the problem \cref{eqn:group-lasso-sop} was solved by the steepest descent method.

The same idea can be applied to a broad range of sparse learning methods, including the original lasso \cite{Tibshirani1996}, the fused lasso \cite{Tibshirani2005}, the smooth lasso \cite{Hebiri2011}, and the elastic net \cite{Zou2005}.
For those methods, their group-wise regularization terms can be considered as separate objectives, and the resulting problems would be many-objective (four-objective or more) where the all-at-once fitting will much outperform over the inductive skeleton fitting.
We however remark that the bridge regression \cite{Frank1993} is not the case since its regularization term using a nonconvex $\ell_p$-norm (i.e., $p < 1$) cannot change into a strongly convex function via perturbations.

\subsubsection{The Pareto fronts of each multi-objective optimization problem}\label{sec:Pareto-fronts}
\Cref{fig:Pareto-fronts} shows the Pareto fronts of the location problem and the group lasso.
From \cref{fig:Pareto-fronts}, we can see that the Pareto front of the location problem can be represented by a B\'ezier simplex of degree $D=2$.
For the group lasso on the other hand, its Pareto front cannot be represented by a a B\'ezier simplex of degree $D=2$ but of $D=3$.
\begin{figure}[ht]
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/pareto_med.pdf}
    \subcaption{Location problem}
    \label{fig:Pareto-front-location-problem}
 \end{minipage}
 \begin{minipage}{0.49\hsize}
        \centering
    \includegraphics[width=1\textwidth]{neurips2019/fig/pareto_grouplasso.pdf}
    \subcaption{Group lasso}
    \label{fig:Pareto-front-group-lasso}
 \end{minipage}
 \caption{The Pareto fronts of the location problem and the group lasso.}
\label{fig:Pareto-fronts}
\end{figure}

% \section{Application to Hyper-parameter Tuning of Elastic Net}\label{sec:application}
% The elastic net~\cite{Zou2005} is a regression method for linear models with multicollinearity and sparsity.
% Given a matrix $A$ with $m$ rows of observations and $n$ columns of features (may be $m \ll n$), a row vector $y$ of $m$ responses, the elastic net regression is the solution to the following problem:
% \begin{equation}\label{eqn:elastic-net}
% \text{minimize } \frac{1}{2N} \norm{Ax - y}^2 + \alpha \paren{\rho \abs{x} + \frac{1 - \rho}{2} \norm{x}^2} \text{ subject to } x \in \R^n
% \end{equation}
% where $\norm{\cdot}$ is the Euclidean norm, $\abs{\cdot}$ is the $\ell_1$-norm, and $\alpha, \rho \geq 0$ are fixed coefficients for regularization.
% Note that with $\alpha = 0$, it is the ordinary least squares regression; with $\alpha > 0$ and $\rho = 0$, it becomes the ridge regression \cite{Hoerl1970} which stabilizes a solution against multicollinear variables; with $\alpha > 0$ and $\rho = 1$, it turns into the lasso regression \cite{Tibshirani1996} which finds a sparse solution from a lot of features; when $\alpha > 0$ and $0 < \rho < 1$, it thus inherits both of their properties.
% Choosing appropriate values for $\alpha$ and $\rho$ involves a 2-D grid search on an unbounded domain, which often requires heavy computational effort.

% We consider each term in \cref{eqn:elastic-net} as an objective function:
% \begin{equation}\label{eqn:elastic-net-mop}
% \begin{split}
%     \text{minimize } & f(x)   = (f_1(x), f_2(x), f_3(x)) \text{ subject to } x \in \R^n\\
%     \text{where }    & f_1(x) = \norm{Ax - y}^2,\ f_2(x) = \abs{x},\ f_3(x) = \norm{x}^2.
% \end{split}
% \end{equation}
% While $f_3$ is strongly convex, $f_1$ and $f_2$ are (not strongly) convex.
% We make them strongly convex by the following perturbation:
% \begin{align}
% \tilde f_1 &= f_1 + \varepsilon_1 f_3,\\
% \tilde f_2 &= f_2 + \varepsilon_2 f_3
% \end{align}
% where $\varepsilon_1, \varepsilon_2 > 0$ are arbitrarily small numbers.
% Now the problem minimizing a mapping $\tilde f= (\tilde f_1, \tilde f_2, f_3)$ is strongly convex.
% This problem is weakly simplicial and the mapping
% \[
%  x^*(w) = \arg\min_x \inprod{w}{f(x)}
% \]
% is well-defined and continuous on $\Delta^2$ with $x^*(\Delta^2_I) = X^*(\tilde f_I)$ for all $I \subseteq \Set{1,2,3}$.

% We can obtain subsamples by solving each subproblem and can fit a B\'ezier triangle of degree two.
% By setting the weight as
% \[
% w = (1,0,0),(0,1,0),(0,0,1),(\frac 1 2, \frac 1 2, 0),(\frac 1 2, 0, \frac 1 2), (0, \frac 1 2, \frac 1 2),
% \]
% we obtain seven models and their objective values evaluated with test data.
% We obtain an approximate trade-off triangle between error, sparsity, and multicollinearity.

% We convert weights $(w_1, w_2, w_3) \in \Delta^2$ to regularization coefficients $(\alpha, \rho) \in \R_{\geq 0}^2$ for the \texttt{scikit-learn}'s elastic net as follows:
% \[
% \alpha = \frac{w_2 + w_3 + 2 \varepsilon}{w_1 + \varepsilon},\quad
% \rho = \frac{w_2 + \varepsilon}{w_2 + w_3 + 2 \varepsilon}
% \]
% where $\varepsilon > 0$ is a small number to avoid numerical instability (we set $\varepsilon=10^{-4}$).

% The same idea can be applied to a broad range of sparse learning methods, including the group LASSO \cite{Yuan2006}, the fused LASSO \cite{Tibshirani2005}, the smooth LASSO \cite{Hebiri2011}, and their elastic net counterparts.
% For those methods, their group-wise regularization terms can be considered as separate objectives, and the resulting problems would be many-objective (four-objective or more) where the inductive skeleton will much outperform over the all-at-once fitting.
% We however remark that the bridge regression \cite{Frank1993} is not the case since its regularization term using a nonconvex $\ell_p$-norm (i.e., $p < 1$) cannot change into a strongly convex function via perturbations.

\bibliographystyle{plain}
\bibliography{reference}
\end{document}


%!TEX root = ../../copatterns-thesis.tex
% Thomas
%###########
% Hvad er guarded recursion?
% Hvorfor guarded recursion?
% Klassisk guarded recursion vs. Atkey-McBride (clock variables)
% Guarded recursion og dependent types (topos of trees)
% Eksempler

% Ny viden: Hvad er guarded recursion?, Hvorfor guarded recursion?, Clock variables
%###########
\section{Guarded Recursion}
\label{sec:guarded-recursion}

Guarded recursion provides a typing discipline which ensures that all programs adhering to their type specification must be productive. In a system with a syntactic guardedness condition, the productivity of a given program follows from its syntactic structure. With guarded recursion, the productivity of a program follows from its type. Consequently, it is impossible to write a well-typed guarded recursive program which is not productive.

\subsection{The Guardedness Type Constructor}
The use of guarded recursion was pioneered by Nakano\,\citep{Nakano:2000}, who
presented a type system $\lambda\!\bullet\!\mu$ (read ``lambda-later-mu'', based
on $\lambda \mu$) where no well-typed terms diverge. He achieves this by
introducing modality into the type system through the addition of a guardedness
type constructor for a type $A$, $\later A$ (``later A''). This allows him to
give type definitions where
recursive occurrences are only allowed in the context of the $\later$
modality. Combined with a modal Y-combinator (see
Figure~\ref{fig:nakano_Y}), he arrives at a system in which only non-diverging
programs can be written.

\begin{figure}[H]
\[
\vdash \lambda f. (\lambda x. f (x x)) (\lambda x. f (x x)) : (\bullet X \to X) \to X
\]
\caption{The Y-combinator as derived in Nakano's $\lambda\!\bullet\!\mu$ system\,\citep{Nakano:2000}.}
\label{fig:nakano_Y}
\end{figure}
% Although
% Nakano does not call it so explicitly, we recommend that the guardedness type
% constructor $\later$ is read as ``later''.

The addition of the $\later$ operator leads to a type system where there is a distinction between an inhabitant of a type $A$, which must be available now (i.e. at any point in time), and an inhabitant of $\later A$, which is available later (i.e. not now). Using this intuition, a stream of elements of type $A$ can be defined as described in Figure\,\ref{fig:guarded_recursion_stream}. The idea is that we can access the head of the stream now, but we cannot access the recursive reference in the tail until later.

\begin{figure}
\[
Stream\,A = \mu X. A \times \later X
\]
\[
\frac{\Gamma\vdash x : A \quad \quad \Gamma\vdash s : \later Stream\,A}{\Gamma\vdash StreamCons(x,s) : Stream\,A} StreamCons
\]
\[
\frac{\Gamma\vdash s : Stream\,A}{\Gamma\vdash hd(s) : A} hd
\]
\[
\frac{\Gamma\vdash s : Stream\,A}{\Gamma\vdash tl(s) : \later Stream\,A} tl
\]
\caption{A guarded recursive definition of a stream type, along with its introduction and elimination rules.}
\label{fig:guarded_recursion_stream}
\end{figure}

%At this point, we can draw a parallel to syntactic guardedness. Since any manipulation of recursive references is disallowed, syntactic guardedness essentially requires that recursive references are also available now, because there is no guarantee that a result can be given at any later point in time. In contrast, any guarded recursive definition must guarantee that even though the recursive reference is manipulated, a result will always be available later.

\subsection{Constructing Guarded Recursive Programs}
\label{sec:constr-guard-recurs}
To obtain the guarantees of productivity provided by guarded recursion, guarded
recursive programs must be constructed in a special way. For this purpose,
Nakano defines a guarded fixed point operator, shown in
Figure\,\ref{fig:guarded_recursion_fixpoint}. As the recursive reference of the
fixed point is only available later, we cannot inadvertently construct a well-typed program where the recursive reference is used in a way that would make the program diverge.
\begin{figure}
\[
\frac { \Gamma \vdash f : \later A \rightarrow A }{ \Gamma \vdash fix(f) : A } fix
\]
\caption{Fixpoint rule for guarded recursive programs.}
\label{fig:guarded_recursion_fixpoint}
\end{figure}
As an illustration, consider the example of an infinite stream of zeros given in Figure\,\ref{fig:guarded_recursion_zeros}. The function \texttt{zeros} is only well-typed because the recursive reference has type $\later Stream\,Nat$, since a $Stream\,Nat$ could not have been given as an argument to \texttt{StreamCons}. Productivity is thus ensured because the type system forces us to preserve the levels of guardedness required by the type of the program.
\begin{figure}
\input{figures/guarded_zeros}
\caption{A guarded recursive definition of an infinite stream of zeros.}
\label{fig:guarded_recursion_zeros}
\end{figure}

\begin{figure}
\[
\frac { \Gamma \vdash f: \later (A\rightarrow B)\quad \quad \Gamma \vdash e : \later A }{ \Gamma \vdash f \tensor e : \later B } { \tensor }_{ I }
\]
\[
\frac { \Gamma \vdash e:A }{ \Gamma \vdash Next(e):{ \later A } } Next
\]
\caption{Later application ($\tensor$) and $next$ introduction rules for guarded recursive programs.}
\label{fig:guarded_recursion_rules}
\end{figure}
To define guarded recursive programs with a more interesting structure than
\texttt{zeros}, we can apply the $\tensor$ operator shown in
Figure\,\ref{fig:guarded_recursion_rules} defined by Atkey and
McBride\,\citep{Atkey:2013}. We choose to read this operator as ``later
application''.  The later application operator facilitates the composition of
guarded recursive programs by allowing us to apply functions which are available
later to values which are available later. As hinted at by its type, later
application is also an applicative functor in the sense introduced by McBride
and Paterson\,\citep{Mcbride:2008}, and the standard laws for these apply here
as well. Also shown in Figure~\ref{fig:guarded_recursion_rules} is the $Next$
rule, which lifts a value available now into a into a later context. Since
$Next$ can be applied multiple times, any value available now can be made
available at any later point in time.

% \begin{figure}
% \begin{lstlisting}[mathescape]
% map : (a $\rightarrow$ b) $\rightarrow$ Stream a $\rightarrow$ Stream b
% map = fix($\lambda$m.$\lambda$f.$\lambda$s. StreamCons$\,$(f (hd s))
%                               $\;$(m $\tensor$ (next f) $\tensor$ (tl s)))
% nats : Stream Nat
% nats = fix($\lambda$n. StreamCons 0 ((next map $\tensor$ next S) $\tensor$ n))
% \end{lstlisting}
% \caption{A guarded recursive definition of a function \texttt{map} for streams.}
% \label{fig:guarded_recursion_map}
% \end{figure}

\begin{figure}
\begin{lstlisting}[mathescape]
map : (a $\rightarrow$ b) $\rightarrow$ Stream a $\rightarrow$ Stream b
map = fix($\lambda$m.$\lambda$f.$\lambda$s. StreamCons$\,$(f (hd s))
                              $\;$((m $\tensor$ (Next f)) $\tensor$ (tl s)))

nats : Stream Nat
nats = fix($\lambda$n. StreamCons Z (((Next map) $\tensor$ (Next S)) $\tensor$ n))
\end{lstlisting}
%\input{figures/guarded_nats}
\caption{A guarded recursive definition of the natural numbers, using the \texttt{map} function. The function \texttt{S} is the successor function for the natural numbers, which has the standard definition without any added guardedness information.}
\label{fig:guarded_recursion_nats}
\end{figure}

Both the later application rule and the $Next$ rule are used in the example of
\texttt{nats} in Figure~\ref{fig:guarded_recursion_nats}. In order to obtain a
value of type $\later Stream\,Nat$ for the second argument of
\texttt{StreamCons}, we proceed as follows:

\begin{enumerate}
\item We apply $Next$ to \texttt{map} and \texttt{S}, obtaining
  \texttt{Next~map} of type ${\later((a~\rightarrow~b)~\rightarrow~Stream~a~\rightarrow~Stream~b)}$
    and \texttt{Next~S} of type ${\later(Nat~\rightarrow~Nat)}$
\item Applying the later application operator, $\tensor$, we get
  (\texttt{Next~map} $\tensor$ \texttt{Next~S}) of type
    ${\later(Stream\,Nat \rightarrow Stream\,Nat)}$

\item Applying this to the recursive reference, \texttt{n}, leaves us with an element of ${\later Stream\,Nat}$, as desired.
\end{enumerate}

 % first apply $Next$ to \texttt{map} and \texttt{S}, obtaining \texttt{next map
 %   :} $\later((a~\rightarrow~b)~\rightarrow~Stream~a~\rightarrow~Stream~b)$ and
 % \texttt{next~S~:}~$\later(Nat~\rightarrow~Nat)$. Applying the later application
 % operator, $\tensor$, we get (\texttt{Next map} $\tensor$ \texttt{Next S}) : $\later(Stream\,Nat \rightarrow Stream\,Nat)$, which when applied to the recursive reference \texttt{n} leaves us with an element of $\later Stream\,Nat$, as desired.

\subsection{Clock Variables}
While the system of guarded recursion presented so far is useful for manipulating values that we know will be available later, it is still impossible to leave the time constraints behind (go from $\later A$ to $A$), even when it is safe to do so. This puts some limitations on the practical use of the system. To alleviate this situation, Atkey and McBride\,\citep{Atkey:2013} introduce the idea of clock variables. A clock variable $\kappa$ represents a clock with a fixed amount of time remaining. Clocks can be associated with types, such that when $x : \onk{A}$, we know that at least the first $\kappa$ elements of $x$ can be provided. This intuition is then extended with universal quantification over clocks, such that when $x : \forall\kappa. A^\kappa$ we know that $x$ is productive, since for any $\kappa$ the first $\kappa$ elements of $x$ can be provided.
\begin{figure}
\input{figures/mogel_rules}
% \input{figures/guarded_tensor_intro_rule}
% \input{figures/guarded_next_rule}
% \input{figures/guarded_fix_rule}
% \input{figures/guarded_applyfresh}
% \input{figures/guarded_force}
% \input{figures/guarded_forall_intro_rule}
% \input{figures/guarded_forall_elim_rule}
% \[
% \frac { \Delta;\Gamma\vdash \later^{\!\!\!\kappa'}\forall\kappa.A \quad \quad \kappa \neq \kappa' } { \Delta;\Gamma\vdash \forall\kappa.\later^{\!\!\!\kappa'}A} switch
% \]
\caption{Rules for guarded recursive definitions with clocks, as defined by
  M\o gelberg\,\citep{Mogelberg:2014}.}
\label{fig:guarded_recursion_rules_clocks}
\end{figure}
\begin{figure}
\[
Stream\,A = \mu X. A \times \laterk{X}
\]
\[
\frac{\Delta;\Gamma\vdash x : A \quad \quad \Delta;\Gamma\vdash s : \laterk{Stream}\,A}{\Delta;\Gamma\vdash StreamCons(x,s) : \onk{Stream}\,A} StreamCons
\]
\[
\frac{\Delta;\Gamma\vdash s : \onk{Stream}\,A}{\Delta;\Gamma\vdash hd(s) : A} hd
\]
\[
\frac{\Delta;\Gamma\vdash s : \onk{Stream}\,A}{\Delta;\Gamma\vdash tl(s) : \laterk{Stream}\,A} tl
\]
\caption{The guarded recursive definition of streams from
  Figure\,\ref{fig:guarded_recursion_stream} extended with clocks.}
\label{fig:guarded_recursion_stream_clocks}
\end{figure}
The introduction of clock variables and quantification over these requires some
changes to the rules presented above. In
Figure~\ref{fig:guarded_recursion_rules_clocks}, the extended rules, defined for
a dependently typed system by M\o gelberg\,\citep{Mogelberg:2014}, are
shown. Addtionally, a stream definition extended with clocks is shown in
Figure~\ref{fig:guarded_recursion_stream_clocks}. To exemplify the use of clock
variables, we have extended the example of \texttt{nats} from
Figure~\ref{fig:guarded_recursion_nats} with clock variables and clock
quantification in Figure~\ref{fig:guarded_recursion_nats_with_clocks}. Notably,
we must use clock abstraction ($\Lambda\kappa$) in order to obtain types
quantified over clocks. Furthermore, clock application
(e.g. \texttt{map[$\kappa$]}) ensures that \texttt{map} operates on the same
clock as \texttt{nats}. However, not all examples of interesting programs are as
simple as \texttt{nats}. If we wish to write programs such as
\texttt{evens} in Figure~\ref{fig:guarded_recursion_evens}, where the
input and output are individually quantified over a clock, we must define it using an
indexed fixed point. The use of the indexed fixed point ($\mathit{pfix}$ in
Figure~\ref{fig:pfix}) ensures that the arguments to the recursive reference are
not placed under a later ($\laterkappa$). Note that the indexed fixed point can
be defined in terms of the original fixed point (defined in
Figure~\ref{fig:guarded_recursion_rules_clocks}), so it can be added to a
guarded recursive system without introducing any additional axioms.
\begin{figure}
\begin{lstlisting}[mathescape]
map : $\forall\kappa.$ (a $\rightarrow$ b) $\rightarrow$ Stream a $\rightarrow$ Stream b
map = $\Lambda\kappa.$ fix$^\kappa$($\lambda$m.$\lambda$f.$\lambda$s. StreamCons$\,$(f (hd s))
                                   ((m $\tensor$ (Next f)) $\tensor$ (tl s)))

nats : $\forall\kappa.$ Stream Nat
nats = $\Lambda\kappa.$ fix$^\kappa$($\lambda$n. StreamCons Z (((Next (map[$\kappa$]) $\tensor$ (Next S)) $\tensor$ n))
\end{lstlisting}
%\input{figures/guarded_nats}
\caption{A guarded recursive definition of the natural numbers using clock
  variables and clock quantification.}
\label{fig:guarded_recursion_nats_with_clocks}
\end{figure}

\begin{figure}
\begin{lstlisting}[mathescape]
evens$'$ : ($\forall\kappa.$ Stream$^\kappa$ a $\to$ $\laterkappa$(Stream$^\kappa$ a)) $\to$ ($\forall\kappa.$ Stream$^\kappa$ a) $\to$ Stream$^\kappa$ a
evens$'$ = $\lambda$rec.$\lambda$s. StreamCons (head s) (rec (tail (tail s)))

evens : ($\forall\kappa.$ Stream$^\kappa$ a) $\to$ ($\forall\kappa.$ Stream$^\kappa$ a)
evens = $\Lambda\kappa.$ pfix$^{\kappa}$(evens$'$)
\end{lstlisting}
\caption{A guarded recursive definition of a function that removes every second element from a stream.}
\label{fig:guarded_recursion_evens}
\end{figure}

\begin{figure}
%\begin{prooftree}
\input{figures/pfix}
\caption{An indexed fixed point used to define functions where both the input
  and the output have types quantified over clocks.}
\label{fig:pfix}
\end{figure}

% The most important change is the addition of the $Force$ rule, which says that whenever it is possible to universally quantify over a clock on a type $\laterk{A}$, the time constraints imposed by the $\later$ operator can be removed. Universal quantification over clocks is introduced by adding a clock abstraction form $\Lambda\kappa$, as indicated by the rules $\forall_{I}$ and $\forall_{E}$.
% \begin{figure}
% \begin{lstlisting}[mathescape]
% evens : $\forall\kappa.$ Stream$^\kappa$ $\to$ $\forall\kappa.$ Stream$^\kappa$
% evens = fix^{\kappa}(evens$'$)
% \end{lstlisting}
% \caption{A guarded recursive definition of a function that removes every second element from a stream.}
% \label{fig:guarded_recursion_evens}
% \end{figure}
% The $force$ rule is necessary for the \texttt{evens} function in Figure~\ref{fig:guarded_recursion_evens} to be well-typed, since \texttt{StreamCons} requires as its second argument a $\laterk{Stream} A$. Seeing as we need to have two calls to \texttt{tl} on the input stream \texttt{s} to obtain the desired semantics, we have a situation in which \texttt{(next tl $\tensor$ (tl s[$\kappa'$])) :} $\laterkappap\laterkp{Stream} A$, i.e. the result is available too late. We can recover from this situation by forcing the result to be available earlier. Forcing is only possible because the type of \texttt{evens} requires the input stream to be universally quantified over all clocks, so we could not have defined this function in a guarded recursive type system without clock variables. Since the $force$ rule only works on values that are universally quantified over all clocks, we abstract over $\kappa'$ in the expression, getting \texttt{($\Lambda\kappa'$.(next tl $\tensor$ (tl s[$\kappa'$]))) :} $\forall\kappa'.\laterkappap\laterkp{Stream} A$. The universal quantification can now be handled with $force$, although according to the type of \texttt{evens} this leaves us with the clock quanfication of the wrong side of the $\laterkappap$ operator. Therefore the $switch$ rule must be used to get the desired type. For the first argument to \texttt{StreamCons} in \texttt{evens}, the universal quantification can be eliminated by the $applyFresh$ rule since no $\laterkappap$ constructors are involved.
% \begin{figure}
% \begin{lstlisting}[mathescape]
% take : Nat $\rightarrow$ ($\forall\kappa.$Stream$^\kappa$ a) $\rightarrow$ List a
% take Z     s = []
% take (S n) s = applyFresh($\Lambda\kappa$.(hd(s[$\kappa$])))
%                :: take n (force($\Lambda\kappa$.(tl(s[$\kappa$]))))
% \end{lstlisting}
% \caption{A function that takes the first \texttt{n} elements of the input stream \texttt{s}. The definition of the \texttt{List} type is standard, without any $\later$ operations or clocks.}
% \label{fig:guarded_recursion_take}
% \end{figure}

% A different example of the use of clock variables is shown in Figure~\ref{fig:guarded_recursion_take}. The \texttt{take} function is not defined in terms of the $fix$, since elements of its result type, \texttt{List}, should always be available. To be able to make the recursive call well-typed, however, we need to rid ourselves of the $\laterkappa$ constraint imposed on the stream argument \texttt{s} by the use of \texttt{tl}. The solution is to use the $force$ rule, which requires that \texttt{s} is known to be productive.

% Note that the examples in Figure~\ref{fig:guarded_recursion_zeros} and Figure~\ref{fig:guarded_recursion_nats} are all still valid in a setting with clock variables. Their types simply have to be annotated with a top-level clock quantification, as shown in Figure~\ref{fig:guarded_recursion_quantified_examples}.

% \begin{figure}
% \begin{lstlisting}[mathescape]
% zeros : $\forall\kappa$.Stream$^\kappa$$\,$Nat
% map   : $\forall\kappa$.(A $\rightarrow$ B) $\rightarrow$ Stream$^\kappa$$\,$A $\rightarrow$ Stream$^\kappa$$\,$A
% nats  : $\forall\kappa$.Stream$^\kappa$$\,$Nat
% \end{lstlisting}
% \caption{The types of the examples from Section~\ref{sec:constr-guard-recurs} extended with clock variables.}
% \label{fig:guarded_recursion_quantified_examples}
% \end{figure}
%\todo{Introduce indexed fixed point}
%\todo{Explanation of topos of trees goes here.}

% \begin{landscape}
% \begin{figure}
% \[
% \cfrac { \cfrac { \cfrac { \cfrac {  }{ \Gamma ,\, n\, \vdash \, 0\, :\, \mathbb{N} } \, \, \cfrac { \cfrac { \cfrac { \cfrac { \cfrac {  }{ \Gamma ,\, n\, \vdash \, map\, :\, (\mathbb{N}\, \rightarrow \, \mathbb{N})\, \rightarrow \, Stream\, \mathbb{N}\, \rightarrow \, Stream\, \mathbb{N} } \, \, \cfrac {  }{ \Gamma ,\, n\, \vdash \, S\, :\, \mathbb{N}\, \rightarrow \, \mathbb{N} }  }{ \Gamma ,\, n\, \vdash \, map\, S\, :\, Stream\, \mathbb{N}\, \rightarrow \, Stream\, \mathbb{N} }  }{ \Gamma ,\, n\, \vdash \, next\, (map\, S)\, :\, \rhd \, (Stream\, \mathbb{N}\, \rightarrow \, Stream\, \mathbb{N}) } \, \, \cfrac {  }{ \Gamma ,\, n\, \vdash \, n\, :\, \rhd \, Stream\, \mathbb{N} }  }{ \Gamma ,\, n\, \vdash \, next\, (map\, S)\, \circledast \, n\, :\, \rhd \, Stream\, \mathbb{N}\, \,  }  }{ \Gamma ,\, n\, \vdash \, map\, S\, n\, :\, \rhd \, Stream\, \mathbb{N} }  }{ \Gamma ,\, n\, :\, \rhd \, Stream\, \mathbb{N}\, \vdash \, StreamCons\, 0\, (map\, S\, n)\, :\, Stream\, \mathbb{N} }  }{ \Gamma \, \vdash \, fix\, (\lambda n.\, StreamCons\, 0\, (map\, S\, n))\, :\, Stream\, \mathbb{N}\,  }  }{ \Gamma \, \vdash \, 0\, ::\, map\, S\, nats\, :\, Stream\, \mathbb{N} }
% \]
% \caption{An inference tree for a program \texttt{nats} defining the natural numbers.}
% \end{figure}
% \end{landscape}

% \begin{figure}[h]
%   Møgelberg rules here!
%   \caption{The rules for guarded recursion in a dependently typed system.}
%   \label{fig:guarded_recursion_dependent_rules}
% \end{figure}

%\paragraph{}
%Note that guardedness type constructors can be nested, meaning that $\later A$ and $\later \later A$ are both valid and distinct types. However, an element of type $\later A$ can never be given where an element of type $\later \later A$ is expected, and vice versa, since guardedness levels cannot be collapsed.

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../../copatterns-thesis"
%%% End:


%###############################################################
\section{Analysis}\label{sec:Analysis}
%###############################################################
This section analyzes the travel distance and space required for a factory and gives simulation results. Algorithms  \ref{alg:FindBuildPath}---\ref{alg:FactoryAddTile}  were coded in {\sc Matlab} and are available at \cite{Manzoor2017gitAssemply}.

%###############################################################
\subsection{Maximum Distance Travelled}\label{sec:runningTime}
%###############################################################
Running a factory simulation has three phases: ramp up, production, and wind down.
During the $n-1$ \emph{ramp up}  cycles, the first polyomino is being constructed one tile at a time and no polyominoes are produced.
Clever design of delays in the part hoppers ensures no unconnected tiles are released.
During \emph{production} cycles, one  polyomino is finished each cycle.
Once the first part hopper empties, the $n-1$ \emph{wind down}  cycles each produce a complete polyomino as each successive hopper empties.
 This section analyzes maximum distance, defined as the maximum distance any tile must move.
 There are two results, \emph{construction distance}, the maximum distance required to assemble a single polyomino from scratch, and
  \emph{cycle distance}, the maximum distance required during production cycles to advance all partial assemblies one cycle.
 Since a polyomino contains $n$ tiles, the \emph{construction distance} during production cycles is $n\cdot$(\emph{cycle distance}).

Cycle distance is the sum of the maximum distances moved in each direction.
 As shown in Fig.~\ref{fig:timeplot}, polyominoes shaped as a $n\times 1$ row require the longest distance of $4n+16$.
Polyominoes shaped as a $1\times n$ column require the least distance of $2n+16$.
 Construction distance therefore requires $O(n^2)$ distance.
 \begin{figure}
   \centering
\begin{overpic}[width =1\columnwidth]{Distanceplot.pdf}
\end{overpic}
\vspace{-2em}
\caption{\label{fig:timeplot}Worst-case cycle distance plotted as a function of polyomino size $n$.  The cycle distance is the sum of distances to move during the $r,d,l,u$ moves each cycle. Cycle distance increases linearly with polyomino size and is upper bounded by row parts and lower bounded by column parts.  Total construction distance for a particle is $n\cdot$cycle distance.
}
\end{figure}
%###############################################################
\subsection{Space Required}\label{sec:requiredSpace}
%###############################################################
The space required by a factory is a function of the widths of individual sub-factories and height of the last sub-factory.

The first sub-factory is constructed separately and it does not have any delay.
Beginning from the second sub-factory, height can be computed as a function of the number of copies $n_c$ of the polyomino, width of the hopper $w$, position of the sub-factory $i$, and rows of the sub-assembled polyomino $\mathbf{b}_y$ as in \eqref{eq:1}. If a tile is added before the top row of $\mathbf{b}$, then an additional row is added to the height.
The width of the sub-factory can be calculated similarly as in \eqref{eq:2} and \eqref{eq:3}.
In a case where twice of $\mathbf{b}_x$ is greater than $width_{\mathrm{hopper}+\mathrm{delays}}$ then additional columns are added to the left of the sub-factory.
When a tile is added to $\mathbf{b}$ using a down move, width also depends on the location of the column, $column_{\mathrm{loc}}$, to which the tile is added.
\begin{align}
height(i) &= \left \lceil{ \frac{n_c}{w}}\right \rceil+2\left(\left \lceil{\frac{i}{2}}\right \rceil+\mathbf{b}_y\right)+ \nonumber \\
&\begin{cases}
  4,&\text{for } m = l \text{ or } d, i \geq 2\\
7,&\text{for } m = u \text{ or } r, i \geq 2
\end{cases}\label{eq:1} \\
%\end{align}
%\begin{align}
width_{\mathrm{hopper}+\mathrm{delays}}&=w+2\left \lceil{ \frac{i}{2}}\right \rceil+8,  i \geq 2 \label{eq:2} \end{align}
\begin{align}
width(i)=&~width_{\mathrm{hopper}+\mathrm{delays}} +\nonumber\\
&\begin{cases}
(\mathbf{b}_x-column_{\mathrm{loc}}),&\text{for } m = d \\
0 &\text{for } m \neq d  \label{eq:3}
\end{cases}
\end{align}

Because a factory requires $O(n)$ rows and $O(n)$ columns, the total required space is $O(n^2)$.
As shown in Fig.~\ref{fig:sizeplot}, the required size is  upper bounded by column-shaped polyominoes and lower bounded by row-shaped polyominoes, and is $O(n^2)$.

\begin{figure}
   \centering
   \vspace{.2em}
\begin{overpic}[width =1\columnwidth]{FactorySizePlot.pdf}
\end{overpic}\\
\vspace{-1em}
\caption{\label{fig:sizeplot}
Factory size grows quadratically with the number of tiles. % and is upper bounded by column-shaped polyominoes and lower bounded by row-shaped polyominoes.
}
\end{figure}


  \documentclass{mynotes}

%\geometry{showframe}% for debugging purposes -- displays the margins

\newcommand{\E}{\mbox{E}}
\newcommand{\MSE}{\mbox{MSE}}
\newcommand{\var}{\mbox{var}}
\newcommand{\bx}{\mathbf{x}}

\usepackage{amsmath}
%\usepackage[garamond]{mathdesign}
\usepackage{url}

% Set up the images/graphics package
\usepackage{graphicx}
\setkeys{Gin}{width=\linewidth,totalheight=\textheight,keepaspectratio}
\graphicspath{{graphics/}}

\title[Exercises 4 $\cdot$ SDS 383D]{Exercises 4: Intro to Hierarchical Models}
%\author[ ]{ }
\date{}  % if the \date{} command is left out, the current date will be used

% The following package makes prettier tables.  We're all about the bling!
\usepackage{booktabs}

% The units package provides nice, non-stacked fractions and better spacing
% for units.
\usepackage{units}

% The fancyvrb package lets us customize the formatting of verbatim
% environments.  We use a slightly smaller font.
\usepackage{fancyvrb}
\fvset{fontsize=\normalsize}

% Small sections of multiple columns
\usepackage{multicol}

% Provides paragraphs of dummy text
\usepackage{lipsum}

% These commands are used to pretty-print LaTeX commands
\newcommand{\doccmd}[1]{\texttt{\textbackslash#1}}% command name -- adds backslash automatically
\newcommand{\docopt}[1]{\ensuremath{\langle}\textrm{\textit{#1}}\ensuremath{\rangle}}% optional command argument
\newcommand{\docarg}[1]{\textrm{\textit{#1}}}% (required) command argument
\newenvironment{docspec}{\begin{quote}\noindent}{\end{quote}}% command specification environment
\newcommand{\docenv}[1]{\textsf{#1}}% environment name
\newcommand{\docpkg}[1]{\texttt{#1}}% package name
\newcommand{\doccls}[1]{\texttt{#1}}% document class name
\newcommand{\docclsopt}[1]{\texttt{#1}}% document class option name

\newcommand{\N}{\mbox{N}}
\newcommand{\thetahat}{\hat{\theta}}
\newcommand{\sigmahat}{\hat{\sigma}}
\newcommand{\betahat}{\hat{\beta}}

\begin{document}

\maketitle% this prints the handout title, author, and date

\section{Math tests}

The data set in ``mathtest.csv'' shows the scores on a standardized math test from a sample of 10th-grade students at $P=100$ different U.S.~urban high schools, all having enrollment of at least 400 10th-grade students.  (A lot of educational research involves ``survey tests'' of this sort, with tests administered to all students being the rare exception.)

Let $\theta_i$ be the underlying mean test score for school $i$, and let $y_{ij}$ be the score for the $j$th student in school $i$.  Our main parameter of interest is $\theta$, the vector of school-level means.  We'll assume that each student's test score is normally distributed around the corresponding school-level mean: $y_{ij} \sim \mbox{N}(\theta_i, \sigma^2)$ for $i=1, \ldots, P$ and $j = 1, \ldots, N_i$.

\begin{enumerate}[(A)]
\item Show (somewhat trivially) that the maximum likelihood estimate for $\theta$ is just the vector of sample means: $\hat{\theta}_{\mathrm{MLE}} = (\bar{y}_1, \ldots, \bar{y}_P)$.
\item Make a plot that illustrates the following fact: extreme school-level averages $\bar{y}_i$ (both high and low) tend to be at schools where fewer students were sampled.  Explain briefly why this would be.
\item Fit the following two-level hierarchical model to these data via Gibbs sampling:
\begin{eqnarray*}
(y_{ij} \mid \theta_i, \sigma^2) &\sim& \mbox{N}(\theta_i, \sigma^2) \\
(\theta_i \mid \tau^2, \sigma^2) &\sim& \mbox{N}(\mu, \tau^2 \sigma^2)
\end{eqnarray*}

As a starting point, use a flat prior on $\mu$, Jeffreys' prior on $\sigma^2$, and an inverse-gamma(1/2, 1/2) prior on $\tau^2$.  Your Gibbs sampler should cycle between the complete conditional posterior distributions for each of these parameters in turn, as well as $\theta$ (the vector of means).  While you could update each $\theta_i$ individually, I encourage you to think about it as a vector whose conditional distribution is multivariate normal, and whose covariance matrix happens to be diagonal.  This view will generalize more readily to future problems.

\item Express the conditional posterior mean for $\theta_i$ in the following form:
$$
E(\theta_i \mid y, \tau^2, \sigma^2, \mu) = \kappa_i \mu + (1-\kappa_i) \bar{y}_i \, ,
$$
i.e. a convex combination of prior mean and data mean.  Here $\kappa_i$ is a \textit{shrinkage coefficient} whose form you should express in terms of the model hyperparameters.  In the extreme case where $\kappa_i$ is 1, then the data ($\bar{y}_i$) are essentially ignored, and the posterior mean is ``shrunk'' all the way back to the prior mean.  In the other extreme where $\kappa_i$ is 0, the prior mean is ignored, and the posterior mean is entirely ``un-shrunk'' compared to the MLE for $\theta_i$.

For each draw of your MCMC, calculate $\kappa_i$ for each school, and save the posterior draws.  Average these MCMC samples to calculate $\bar{\kappa}_i$, the posterior mean of this shrinkage coefficient.   Plot $\bar{\kappa}_i$ for each school as a function of that school's sample size, and comment on what you see.

\item Observe that an equivalent way to write your model involves the following decomposition:
$$
y_{ij} = \mu + \delta_i + e_{ij}
$$
where $\delta_i \sim N(0, \tau^2 \sigma^2)$ and $e_{ij} \sim N(0, \sigma^2)$.  (In the paper by Gelman that I've asked you to read, he writes it this way, where the school-level ``offsets'' are centered at zero, although he doesn't scale these offsets by $\sigma$ the way I prefer to do.)  To translate between the two parameterizations, just observe that in the previous version, $\theta_i = \mu + \delta_i$.

Conditional on the ``grand mean'' $\mu$, but \emph{marginally} over both $\delta_i$ and $e_{ij}$, compute the following two covariances:

\begin{compactitem}
\item $\mbox{cov}(y_{i,j}, y_{i,k})$, $j \neq k$
\item $\mbox{cov}(y_{i,j}, y_{i', k})$, $i \neq i'$ and $j \neq k$
\end{compactitem}

Does this make sense to you?  Why or why not?

\item Does the assumption that $\sigma^2$ is common to all schools look justified in light of the data?

\end{enumerate}

\section{Blood pressure}

The data set in ``bloodpressure.csv'' contains data on repeated blood pressure measurements on 20 different subjects, ten of whom received a control medication (treatment=1) and ten of whom received an experimental medication (treatment = 2).  Patients were randomized to receive the two treatments.  The columns are the data of measurement, a numerical ID for the subject, the blood pressure measurement (systolic), and which treatment the patient received.

\begin{enumerate}[(A)]

\item Is the experimental medication effective at reducing blood pressure?  Do the naive thing and perform a t-test for a difference of means, pooling all the data from treatment 1 into group 1, and all the data from treatment 2 into group 2.  What does this t-test say about the difference between these two group means, and the standard error for the difference?  Why is the t-test (badly) wrong?

\item Now do something better, but still less than ideal.  Calculate $\bar{y}_i$, the mean blood pressure measurement for each patient.  Now treat each person-level mean as if it were just a single data point, and conduct a different t-test for mean blood pressure between treatment 1 and treatment 2.  (If you're doing this correctly, you should have only ten "observations" in each group, where each observation is actually a person-level mean.) What does this t-test say about the difference between these two group means, and the standard error for the difference?  Why is the standard error so much bigger, and why is this appropriate?  Even so, why is this approach (subtly) wrong?

\item Now fit a two-level hierarchical model to this data, of the following form:
\begin{eqnarray*}
(y_{ij} \mid \theta_i, \sigma^2) &\sim& \mbox{N}(\theta_i, \sigma^2) \\
(\theta_i \mid \tau^2, \sigma^2) &\sim& \mbox{N}(\mu + \beta x_i, \tau^2 \sigma^2)
\end{eqnarray*}
where $y_{ij}$ is blood pressure measurement $j$ on person $i$, and $x_i$ is a dummy (0/1) variable indicating whether a patient received treatment 2, the experimental medication.   Apply what you learned on the previous problem about sampling, hyperparameters, etc, but account for the extra wrinkle here, i.e. the presence of the $\beta x_i$ term that shifts the mean between the treatment and control groups.

Write our your model's complete conditional distributions, and fit it.  Make a histogram of the posterior distribution for $\beta$, which represents the treatment effect here.  In particular, what are the posterior mean and standard deviation of $\beta$?  How do these compare to the estimates and standard errors from the approaches in (A) and (B)?

\item Your two-level model assumes that, conditional on $\theta_i$, the $y_{ij}$ are independent.  Written concisely: $(y_{ij} \perp y_{ik} \mid \theta_i)$ for $j \neq k$.

There are many ways this assumption could break down.  So check!  Does this assumption look (approximately) sensible in light of the data?  Provide evidence one way or another.

\end{enumerate}

%
%\subsection{Price elasticity of demand}
%
%The data in ``cheese.csv'' are about sales volume, price, and advertisting display activity for packages of Borden sliced ``cheese.'' The data are taken from Rossi, Allenby, and McCulloch's textbook on \textit{Bayesian Statistics and Marketing.} For each of 88 stores (store) in different US cities, we have repeated observations of the weekly sales volume (vol, in terms of packages sold), unit price (price), and whether the product was advertised with an in-store display during that week (disp = 1 for display).
%
%Your goal is to estimate, on a store-by-store basis, the effect of display ads on the demand curve for cheese.  A standard form of a demand curve in economics is of the form $Q = \alpha P^\beta$, where $Q$ is quantity demanded (i.e.~sales volume), $P$ is price, and $\alpha$ and $\beta$ are parameters to be estimated.  You'll notice that this is linear on a log-log scale,
%$$
%\log Q = \log \alpha + \beta \log P \,
%$$
%which you should feel free to assume here.  Economists would refer to $\beta$ as the price elasticity of demand (PED).  Notice that on a log-log scale, the errors enter multiplicatively.
%
%There are several things for you to consider in analyzing this data set.
%\begin{compactenum}
%\item The demand curve might shift (different $\alpha$) and also change shape (different $\beta$) depending on whether there is a display ad or not in the store.
%\item Different stores will have very different typical volumes, and your model should account for this.
%\item Do different stores have different PEDs?  If so, do you really want to estimate a separate, unrelated $\beta$ for each store?
%\item If there is an effect on the demand curve due to showing a display ad, does this effect differ store by store, or does it look relatively stable across stores?
%\item Once you build the best model you can using the log-log specification, do see you any evidence of major model mis-fit?
%\end{compactenum}
%Propose an appropriate hierarchical model that allows you to address these issues, and use Gibbs sampling to fit your model.
%
%
\end{document}


\documentclass[10pt,a4paper]{report}
\usepackage[latin1]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{hyperref}

\input{def}

\title{Adapting ETDRK4 (Kassam \& Trefethen 2003) for integrating
	   \KS\ equation in the first Fourier mode slice}
\author{Nazmi Burak Budanur}

\begin{document}

\maketitle

\section*{Introduction}
In \BCDS, we adapted Exponential time differencing fourth-order Runge-Kutta
(ETDRK4) method for integrating 1D \KS\ system within the first Fourier mode
slice hyperplane. In this short document, we present details of this
implementation which are not immediately related to the content of \BCDS.

\section*{Implementation}

ETDRK4 method is used to solve nonlinear ODEs of the following form:
\begin{equation}
	\dot{\sspC} = \vel (\sspC) = L \sspC + N(\sspC) \, ,
	\label{e-ODE}
\end{equation}
where $\sspC \in \man$ is a complex valued state space vector, $L$ is a matrix
and $N(\sspC)$ is a nonlinear function. \KS\ equation is of this form with
$L_{kl} = (q_k^2 - q_k^4) \delta_{kl}$ (diagonal), $N_k (a) = - i \frac{q_k}{2}
\sum_{m=-\infty}^{\infty} \Fu_m \Fu_{k - m}$ and $\sspC = (\Fu_1, \Fu_2, \Fu_3,
\ldots)^T\,\mbox{where, } \Fu (\zeit) = \Fourier \{u(x, \zeit)\} $. Numerical
integration of \KS\ equation is explained in detail, with a Matlab code in
\KasTre . This is the code that our implementation \texttt{ksETDRK4red.m} is
based on. In the complex representation, dynamical equations with respect to
the first Fourier mode slice time yields
\begin{eqnarray}
	\frac{d \sspRed}{d \zeitRed}  &=& Re[\sspRed_1] \vel(\sspRed)
	- Im[\vel(\sspRed)_1] \, \groupTan(\sspRed) \label{e-ffSliceStatsp}
	\\
	\frac{d \gSpace}{d \zeitRed} &=& Im[\vel(\sspRed)_1] \label{e-ffSlicePhase}
	\,.
\end{eqnarray}
where, $d \zeitRed = Re[\sspRed_1]^{-1} d \zeit$ is the in-slice time,
$\groupTan(\sspRed) = T \sspRed $ is the group tangent. In the complex
representation, $SO(2)$ symmetry becomes $U(1)$, hence $T_{kl} = i \,
k\delta_{kl}$. Linear and nonlinear parts of equation \ref{e-ffSliceStatsp} are
not immediately separable, however, we can add and subtract a constant $\alpha$
to $Re[\sspRed_1]$ in \ref{e-ffSliceStatsp} and rewrite it as follows
\begin{equation}
	\frac{d \sspRed}{d \zeitRed} = \tilde{L} \sspRed + \tilde{N}(\sspRed) .
\end{equation}
where,
\begin{eqnarray}
	\tilde{L} &=& \alpha L ,
	  \\
	\tilde{N}(\sspRed) &=& N(\sspRed) + (Re[\sspRed_1] - \alpha) \vel (\sspRed)
	  			- Im[\vel(\sspRed)_1] \, \groupTan(\sspRed) .
\end{eqnarray}
In our implementation, we set the parameter $\alpha$ experimentally to $1$.
\end{document}