\section{Frobenius Groups and Permutations}
{\bf Theorem 10:}
Let $G$ be a transitive permutation group acting on $\Omega$ and $H= G_{\alpha}, \alpha \in \Omega$,
then (1) $G$ is $2$-transitive iff $H$ acts transitively on $\Omega \setminus \{ \alpha \}$;
(2) $G$ is $2$-transitive iff $G= HTH$, $|T|= 2$ and $T \nsubseteq H$;
(3) If $G$ is $2$-transitive and $|G:H|= n$ then $G= d(n-1)n$.
\begin{quote}
\emph{Proof:}
Standard.
\end{quote}
{\bf Theorem 11:}
Let $G$ be a transitive permutation group acting on $\Omega$ then
(1) $\sum_{g \in G} |Fix_{\Omega}(g)| = |G|$; and,
(2) $G$ is $2$-transitive iff $\sum_{g \in G} |Fix_{\Omega}(g)|^2 = 2|G|$.
\begin{quote}
\emph{Proof:}
(1) Let $\beta(i)$ be the number of elements of $G$ fixing $i$, this is $|G_i|$.
$\sum_{x \in G} Fix(x) = \sum_{i=1}^n \beta(i)= \sum_{i=1}^n |G_i|= |G|$.
For (2), note that $G$ is $2$-transitive pn $\Omega$ iff $G_{\alpha}$ is transitive on
$\Omega- \{ \alpha \}$.
$\sum_{x \in G_{\alpha}} Fix_i(x) = G_{\alpha}$ where $Fix_i(x)$ is the fixed points in
$\Omega_i$,
$\Omega_1= \{ \alpha \}$ and
$\Omega_2= \Omega -\{ \alpha \}$.  So
$\sum_{i=1}^n \sum_{x \in G_{\alpha}} Fix(x) = 2 \sum_{i=1}^n G_{\alpha} =2|G|$.
\end{quote}
{\bf Definition 10:} The class of transitive permutation groups in which only the identity fixes
more than one element but $|G_a| > \{ 1 \}$ is called a \emph{Frobenius group}.
\\
\\
{\bf Theorem 12:}
Let $G$ be a Frobenius group acting on $\Omega$ and $H= G_{\alpha}, \alpha \in \Omega$ and
put $K= \{ 1 \} \cup \{X \in G: |Fix_{\Omega}(x)|= 0 \}$.  Then $K$ is a normal subgroup
of $G$ and $|K|= |G:H|$.  $K$ is called the \emph{Frobenius kernel} of $G$ and $H$ is the
\emph{Frobenius complement}.  $G= K \rtimes H$.
\begin{quote}
The proof uses character theory.
\end{quote}
{\bf Theorem 13:}
Let $G$ be a transitive permutation group acting on $\Omega$ and $H= G_{\alpha}, \alpha \in \Omega$.
Then $G$ is a Frobenius group iff $\forall x \in G \setminus H: H \cap H^x = \{ 1 \}$
(``TI Property'') and $N_G(H)= H$.
\begin{quote}
\emph{Proof:}
Suppose $G$ is Frobenius and put $H=G_a$. $1 < H < G$.   If $g \in G-H$ then
$H^g= G_a^g= G_{a^g}= G_b$ where $b= a^g \ne a$.   Thus $H^g \cap H = G_a \cap G_b= G_{a,b}=1$.
Hence $H$ is a TI set and $N(H)=H$.  Conversely,
given $G, H$ then $G$ permutes the right cosets of $H$ transitively by right multiplication.
Suppose $\langle x_1, x_2, \ldots , x_n \rangle$ is a set of coset representatives of $H$ then
we have a homomorphism $\varphi: G \rightarrow Sym(\langle Hx_i \rangle)$.  Let
$a- H x_i$, $b= Hx_j$ with $i \ne j$.  $G_{a,b}= H^{x_i} \cap H^{x_j}$ and
$x_j x_i^{-1} \notin H=N(H)$ and so $H^{x_i} \ne H^{x_j}$.   Since $H$ is a TI set,
$G_{a,b} = 1$.  Thus the representation is faithful and $G$ is a Frobenius group.
\end{quote}
{\bf Theorem 14:}
$G= L_2(q)$ is $2$-transitive on points in $V= E_{q^2}$ and only the identity fixes
$3$ points.  Put $N= G_a$; then, in addition,
(1) if $q>3$, $N$ is a Frobenius group with complement $H$ of order $\epsilon (q-1)$ and
elementary abelian kernel $K$ of order $q$ disjoint from its conjugates;
(2) $H$ is the subgroup fixing $2$ points and $H$ is inverted by an element of order $2$
in $G$.
\begin{quote}
\emph{Proof:}
Standard.
\end{quote}


%!TEX root = fastZKP.tex
\section{GKR Protocol with Linear Prover Time}
\label{sec::gkrlin}

In this section we present a new algorithm (see Algorithm~\ref{final_alg}) for the prover of the GKR protocol~\cite{GKR} that runs in linear time for \emph{arbitrary layered circuits}. Before that, we present some necessary building blocks.
%We start with presenting a new linear time algorithm for the prover on certain type of sumcheck protocols, and then show how to apply it to build a linear time GRK prover.

\subsection{Linear-time sumcheck for a multilinear function~\cite{JT_Thesis}}
In~\cite{JT_Thesis}, Thaler proposed a linear-time algorithm for the prover of the sumcheck protocol on a multilinear function $f$ on $\ell$ variables (the algorithm runs in $O(2^\ell)$ time). We review this algorithm here. Recall that in the $i$-th round of the sumcheck protocol the prover sends the verifier the univariate polynomial on $x_i$ $$\sum_{b_{i+1}, \ldots,b_{\ell, }\in\binary}f(r_1,\ldots, r_{i-1}, x_{i}, b_{i+1},\ldots, b_{\ell})\, ,$$
where $r_1, \ldots, r_{i-1}$ are random values chosen by the verifier in previous rounds. Since $f$ is multilinear, it suffices for the prover to send two evaluations of the polynomial at points $t = 0$ and $t=1$, namely the evaluations
\begin{equation}\label{0-message}
\sum_{b_{i+1}, \ldots,b_{\ell, }\in\binary}f(r_1,\ldots, r_{i-1}, 0, b_{i+1},\ldots, b_{\ell})
\end{equation} and
\begin{equation}\label{1-message}
\sum_{b_{i+1}, \ldots,b_{\ell, }\in\binary}f(r_1,\ldots, r_{i-1}, 1, b_{i+1},\ldots, b_{\ell})\,.
\end{equation}

To compute the above sums the prover maintains a \emph{bookkeeping table} $\textbf{A}$ for $f$. This table, at round $i$, has $2^{\ell-i+1}$ entries storing the values $$f(r_1,\ldots, r_{i-1}, b_{i}, b_{i+1},\ldots, b_{\ell})$$ for all $b_i,\ldots,b_\ell\in\binary$ and is initialized with evaluations of $f$ on the hypercube. For every entry of $\textbf{A}$, the prover subsequently computes, as in Step~\ref{alg:func_eval} of Algorithm~\ref{alg:evaluate} \textsf{FunctionEvaluations}\footnote{To be compatible with other protocols later, we use three values $t=0,1,2$ in our evaluations instead of just two.}  two values
$$
f(r_1,\ldots, r_{i-1}, 0, b_{i+1},\ldots, b_{\ell})
\text{ and } f(r_1,\ldots, r_{i-1}, 1, b_{i+1},\ldots, b_{\ell})\,.$$
Once these function evaluations are in place, the prover can easily sum over them and compute the required sumcheck messages as reguired by Relations~\ref{0-message} and~\ref{1-message}. This is done in Algorithm~\ref{alg:sumcheck} \textsf{SumCheck}\footnote{We note here that although these two steps  can be performed together in a single algorithm and without the need to store function evaluations, we explicitly decouple them with two different algorithms (\textsf{FunctionEvaluations} and \textsf{SumCheck}) for facilitating the presentation of more advanced protocols later. }.

\paragraph{Complexity analysis.}
Both Algorithms~\ref{alg:evaluate} and~\ref{alg:sumcheck} run in $O(2^\ell)$ time: The first iteration takes $O(2^\ell)$, the second $O(2^{\ell-1})$ and so on, and therefore the bound holds.

\begin{figure}
\begin{algorithm}[H]
		\caption{$\mathcal{F}\leftarrow \textsf{FunctionEvaluations}(f,\textbf{A},r_1,\ldots,r_\ell)$}\label{alg:evaluate}
		 \textbf{Input:} Multilinear $f$ on $\ell$ variables, initial bookkeeping table $\textbf{A}$, random $r_1,\ldots,r_\ell$;  \\
    \textbf{Output:} All function evaluations $f(r_1,\ldots, r_{i-1}, t, b_{i+1},\ldots, b_{\ell})$;

		\begin{algorithmic}[1]

%\State \label{bookkeeping_step}Initialize the bookeeping table $\textbf{A}$  storing the evaluations of $f$ on the hypecube
					\For{$i = 1, \ldots, \ell$}
				\For{$b\in\binary^{\ell-i}$} \Comment{$b$ is both a number and its binary representation.} \label{step:consider_b}
	\For{$t= 0,1,2$}
					 \State \label{alg:func_eval} Let $f(r_1,\ldots, r_{i-1}, t, b) = \textbf{A}[b]\cdot (1-t)+\textbf{A}[b+2^{\ell-i}]\cdot t$
					\EndFor

					\State\label{alg::dynamic::update} $\textbf{A}[b]=\textbf{A}[b]\cdot(1-r_i)+\textbf{A}[b+2^{\ell-i}]\cdot r_i$
				\EndFor
%					\EndFor
%\For{$i = 1, \ldots, \ell$}

		\EndFor
		\State Let $\mathcal{F}$ contain all function evaluations $f(.)$ computed at Step~\ref{alg:func_eval}
		\State \textbf{return} $\mathcal{F}$
		\end{algorithmic}
	\end{algorithm}

\end{figure}

\begin{figure}[t!]

	\begin{algorithm}[H]
		\caption{$\{a_1,\ldots,a_\ell \}\leftarrow \textsf{SumCheck}(f,\textbf{A},r_1,\ldots,r_\ell)$}\label{alg:sumcheck}
		 \textbf{Input:} Multilinear $f$ on $\ell$ variables, initial bookkeeping table $\textbf{A}$, random $r_1,\ldots,r_\ell$;  \\
    \textbf{Output:}
    $\ell$ sumcheck messages for $\sum_{x\in\binary^\ell}f(x)$. Each message $a_i$ consists of 3 elements $(a_{i0},a_{i1},a_{i2})$;

		\begin{algorithmic}[1]

\State $\mathcal{F}\leftarrow \textsf{FunctionEvaluations}(f,\textbf{A},r_1,\ldots,r_\ell)$
					\For{$i = 1, \ldots, \ell$}
								\For{$t\in\{0,1,2\}$}
					\label{alg::dynamic::sum0} \State $a_{it}=\sum_{b\in\binary^{\ell-i}}f(r_1,\ldots, r_{i-1}, t, b)$ \Comment{All evaluations needed are in $\mathcal{F}$.}

	\EndFor

		\EndFor
		\State \textbf{return} $\{a_1,\ldots,a_\ell \}$;
		\end{algorithmic}
	\end{algorithm}

\end{figure}
\begin{figure}
\begin{algorithm}[H]
		\caption{$\{a_1,\ldots,a_\ell \}\leftarrow \textsf{SumCheckProduct}(f,\textbf{A}_f,g,\textbf{A}_g, r_1,\ldots,r_\ell)$}\label{alg:sumcheckproduct}
		 \textbf{Input:} Multilinear $f$ and $g$, initial bookkeeping tables $\textbf{A}_f$ and $\textbf{A}_g$, random $r_1,\ldots,r_\ell$;  \\
    \textbf{Output:}
    $\ell$ sumcheck messages for $\sum_{x\in\binary^\ell}f(x)g(x)$. Each message $a_i$ consists of 3 elements $(a_{i0},a_{i1},a_{i2})$;

		\begin{algorithmic}[1]

\State $\mathcal{F}\leftarrow \textsf{FunctionEvaluations}(f,\textbf{A}_f, r_1,\ldots,r_\ell)$
\State $\mathcal{G}\leftarrow \textsf{FunctionEvaluations}(g,\textbf{A}_g,r_1,\ldots,r_\ell)$

					\For{$i = 1, \ldots, \ell$}
								\For{$t\in\{0,1,2\}$}
					\label{alg::dynamic::sum0} \State $a_{it}=\sum_{b\in\binary^{\ell-i}}f(r_1,\ldots, r_{i-1}, t, b)\cdot g(r_1,\ldots, r_{i-1}, t, b)$ \Comment{All evaluations needed are in $\mathcal{F}$ and $\mathcal{G}$.}

	\EndFor

		\EndFor
		\State \textbf{return} $\{a_1,\ldots,a_\ell \}$;
		\end{algorithmic}
	\end{algorithm}

\end{figure}
\subsection{Linear-time sumcheck for products of multilinear functions~\cite{JT_Thesis}\label{particular}}
The linear-time sumcheck in the previous section can be generalized to a product of two multilinear functions. Let now $f$ and $g$ be two multilinear functions on $\ell$ variables each, we describe a linear-time algorithm to compute the messages of the prover for the sumcheck on the product $f\cdot g$, as proposed in~\cite{JT_Thesis}. Note that we cannot use Algorithm~\ref{alg:sumcheck} here since $f\cdot g$ is not multilinear. However, similarly with the single-function case, the prover must now send, at round $i$, the following evaluations at points $t = 0$, $t=1$ and $t=2$

$$\sum_{b_{i+1}, \ldots,b_{\ell, }\in\binary}f(r_1,\ldots, r_{i-1}, t, b_{i+1},\ldots, b_{\ell})\cdot g(r_1,\ldots, r_{i-1}, t, b_{i+1},\ldots, b_{\ell}) $$
The above can be easily computed by computing evaluations for functions $f$ and $g$ \emph{separately} using Algorithm~\ref{alg:evaluate} and the combining the results using our new Algorithm~\ref{alg:sumcheckproduct} $\textsf{SumCheckProduct}$. We now have the following lemma:
\begin{lemma}\label{main_lemma_complexity}
Algorithm $\textsf{\emph{SumCheckProduct}}$ runs in time $O(2^\ell)$\end{lemma}
\begin{proof}
All loops in \textsf{SumCheckProduct} require time $2^\ell+2^{\ell-1}+\ldots=O(2^\ell)$. Also $\textsf{SumCheckProduct}$ calls $\textsf{FunctionEvaluations}$ twice (one for $f$ and one for $g$) and each such call takes $O(2^\ell)$ time. \end{proof}

\subsection{Linear-time sumcheck for GKR functions}
Let us now consider the sumcheck problem on a particular class of functions that are relevant for the GKR protocol (that is why we call them GKR functions). In particular we want to compute the sumcheck
\begin{equation}\label{eq:linearsc}
\sum\nolimits_{x,y\in\binary^\ell}f_1(g,x,y)f_2(x)f_3(y)\, ,
\end{equation}
for a fixed point $g\in\mathbb{F}^\ell$, where $f_2(x),f_3(x): \F^\ell\rightarrow \F$ are multilinear extensions of arrays $\textbf{A}_{f_2},\textbf{A}_{f_3}$ of size $2^\ell$, and function $f_1:\mathbb{F}^{3\ell}\rightarrow\mathbb{F}$ is the multilinear extension of a sparse array with $O(2^\ell)$ (out of $2^{3\ell}$ possible) nonzero elements.
It is not hard to see that the sumcheck polynomials in GKR given by Equations~\ref{eq:GKR} and~\ref{eq:randomGKR} satisfy these properties.

% \begin{figure}[b!]
%     \vspace{-.5in}
% 	\begin{algorithm}[H]
% 		\caption{sumcheck on $\sum_{x\in\binary^\ell}f(x)$}\label{alg::dynamic}
% 		\begin{algorithmic}[1]
% 			\Procedure{{\sf Init}}{$f(x)$}
% 			\For{$b\in\binary^{\ell}$}
% 			\State $\textbf{A}[b] = f(b)$ \Comment{$b$ represents both a number and its binary representation.}
% 			\EndFor
% 			\EndProcedure
% 			\Procedure{\sf Update}{$\textbf{A}, r$} \Comment{This is an interactive protocol with the verifier.}
% 			\For{$i = 1, \ldots, \ell$}
% 				\For{$b\in\binary^{\ell-i}$}
% 					\For{$t\in\{0,1\}$}
% 					\State\label{alg::dynamic::compute} Compute $f(r_1,\ldots, r_{i-1}, t, b) = \textbf{A}[b]\cdot(1-t)+\textbf{A}[b+2^{\ell-i}]\cdot t$
% 					\EndFor

% 					\State\label{alg::dynamic::update} $\textbf{A}[b]=\textbf{A}[b]\cdot(1-r_i)+\textbf{A}[b+2^{\ell-i}]\cdot r_i$
% 				\EndFor
% 				\For{$t\in\{0,1\}$}
% 					\State\label{alg::dynamic::sum} Compute $\sum_{b\in\binary^{\ell-i}}f(r_1,\ldots, r_{i-1}, t, b)$ and send to $\V$.
% 				\EndFor
% 			\EndFor
% 			\EndProcedure
% 		\end{algorithmic}
% 	\end{algorithm}
% %	\vspace{-.5in}
% \end{figure}

We note here that applying Algorithm~\ref{alg:evaluate} \textsf{FunctionEvaluations} for this particular class of polynomials would lead to quadratic prover time. This is because $f_1$ has $2^{2\ell}$ variables to sum on yielding $O(2^{2\ell})$ complexity. However, one could take advantage of the sparsity of $f_1$: the prover can store only the $O(2^\ell)$ non-zero values of the bookkeeping table \textbf{A}. This is exactly the approach used in many prior work~\cite{CMT,wahby2017full,vram}. However, with this approach, the number of nonzero values that must be considered in Step~\ref{step:consider_b} is always at most $2^{\ell}$, since it is not guaranteed that this number will reduce to half (i.e., to $2^{\ell-i}$) after every update in Step~\ref{alg::dynamic::update} of Algorithm~\ref{alg:evaluate} because it is sparse. Therefore, the overall complexity becomes $O(\ell \cdot 2^\ell)$.

In this section we effectively reduce this bound to $O(2^\ell)$. Our protocol divides the sumcheck into two phases: the first $\ell$ rounds bounding the variables of $x$ to a random point $u$, and the last $\ell$ rounds bounding the variables of $y$ to a random point $v$. The central idea lies in rewriting Equation~\ref{eq:linearsc} as follows

\begin{eqnarray}
 \sum\nolimits_{x,y\in\binary^\ell}f_1(g,x,y)f_2(x)f_3(y)&=&\sum\nolimits_{x\in\binary^\ell}f_2(x)\sum\nolimits_{y\in\binary^\ell}f_1(g,x,y) f_3(y)\nonumber\\
&=& \sum\nolimits_{x\in\binary^\ell}f_2(x)   h_g(x)\nonumber\,,
\end{eqnarray}
where $h_g(x) = \sum_{y\in\binary^\ell}f_1(g,x,y) f_3(y)$.

\subsubsection{Phase one.} With the formula above, in the first $\ell$ rounds, the prover and the verifier are running exactly a sumcheck on a product of two multilinear functions $f_2\cdot h_g$, since functions $f_2$ and $h_g$ can be viewed as functions only in $x$---$y$ can be considered constant (it is always summed on the hypercube). To compute the sumcheck messages for the first $\ell$ rounds, given their bookkeeping tables, we can call $$\textsf{SumCheckProduct}(h_g(x), \textbf{A}_{h_g},f_2(x),\textbf{A}_{f_2}, u_1,\ldots,u_\ell)$$ in Algorithm~\ref{alg:sumcheckproduct}. By Lemma~\ref{main_lemma_complexity} this will take $O(2^\ell)$ time. We now show how to initialize the bookkeeping tables in linear time.

\smallskip
\noindent{\underline{\emph{Initializing the bookkeeping tables:}} \\Initializing the bookkeeping table for $f_2$  in $O(2^\ell)$ time is trivial, since $f_2$ is a multilinear extension of an array and therefore the evaluations on the hypercube are known. Initializing the bookkeeping table for $h_g$ in $O(2^\ell)$ time is more challenging but we can leverage the sparsity of $f_1$. Consider the following lemma.

\begin{lemma}\label{lemma_h_g}
	Let $\mathcal{N}_x$ be the set of $(z,y)\in \binary^{2\ell}$ such that $f_1(z,x,y)$ is non-zero. Then for all $x\in\binary^\ell$, it is
	$h_g(x) = \sum_{(z,y)\in \mathcal{N}_x}I(g,z)\cdot f_1(z,x,y)\cdot f_3(y)$, where $I(g,z) = \prod_{i=1}^\ell ((1-g_i)(1-z_i)+g_iz_i))$.
\end{lemma}
\begin{proof}
	As $f_1$ is a multilinear extension, as shown in~\cite{JT_Thesis}, we have $f_1(g,x,y) = \sum_{z\in\binary^\ell}I(g,z)f_1(z,x,y)$, where $I$ is the multilinear extension of the identity polynomial, i.e., $I(w,z) = 1$ iff $w=z$ for all $w,z\in\binary^\ell$. Therefore, we have
	\[
	h_g(x) = \sum_{y\in\binary^\ell}f_1(g,x,y) f_3(y) = \sum_{z,y\in\binary^\ell}I(g,z)f_1(z,x,y) f_3(y)
	\]
	\[
	=\sum_{(z,y)\in \mathcal{N}_x}I(g,z)\cdot f_1(z,x,y)\cdot f_3(y)
	\]
	Moreover, $I(w,z) = \prod_{i=1}^\ell ((1-w_i)(1-z_i)+w_iz_i))$ is the unique polynomial that evaluates to 1 iff $w=z$ for all $w,z\in\binary^\ell$. As the multilinear extension is unique, we have $I(g,z) = \prod_{i=1}^\ell ((1-g_i)(1-z_i)+g_iz_i))$.
\end{proof}

\begin{lemma}\label{lemma_h_g_alg}
	The bookkeeping table $\textbf{A}_{h_g}$ can be initialized in time $O(2^\ell)$.
\end{lemma}
\begin{proof}
	As $f_1$ is sparse, $\sum_{x\in\binary^\ell}|\mathcal{N}_x| = O(2^\ell)$. From Lemma~\ref{lemma_h_g}, given the evaluations of $I(g,z)$ for all $z\in\binary^\ell$, the prover can iterate all $(z,y)\in\mathcal{N}_x$ for all $x$ to compute $\textbf{A}_{h_g}$. The full algorithm is presented in Algorithm~\ref{alg:init_h_g}.

	Procedure $\mathsf{Precompute}(g)$ is to evaluate $\textbf{G}[z]=I(g,z)=\prod_{i=1}^\ell ((1-g_i)(1-z_i)+g_iz_i))$ for $z\in\binary^\ell$. By the closed-form of $I(g,z)$, the procedure iterates each bit of $z$, and multiples $1-g_i$ for $z_i=0$ and multiples $g_i$ for $z_i = 1$. In this way, the size of $\textbf{G}$ doubles in each iteration, and the total complexity is $O(2^\ell)$.

	Step 8-9 computes $h_g(x)$ using Lemma~\ref{lemma_h_g}. When $f_1$ is represented as a map of $(z,x,y), f_1(z,x,y)$ for non-zero values, the complexity of these steps is $O(2^\ell)$. In the GKR protocol, this is exactly the representation of a gate in the circuit, where $z,x,y$ are labels of the gate, its left input and its right input, and $f_1(z,x,y)=1$.

\begin{figure}[t!]
\begin{algorithm}[H]
	\caption{$\textbf{A}_{h_g}\leftarrow \textsf{Initialize\_PhaseOne}(f_1, f_3,\textbf{A}_{f_3}, g)$}\label{alg:init_h_g}
	\textbf{Input:} Multilinear $f_1$ and $f_3$, initial bookkeeping tables $\textbf{A}_{f_3}$, random $g = g_1,\ldots,g_\ell$;  \\
	\textbf{Output:} Bookkeeping table $\textbf{A}_{h_g}$;
	\begin{algorithmic}[1]
		\Procedure{{$\textbf{G}\leftarrow$  \sf Precompute}}{$g$}    \Comment{$\textbf{G}$ is an array of size $2^\ell$.}
		\State Set $\textbf{G}[0] = 1$
		\For{$i=0,\ldots,\ell-1$}
		\For{$b\in\binary^i$}
		\State $\textbf{G}[b,0] = \textbf{G}[b]\cdot(1-g_{i+1})$
		\State $\textbf{G}[b,1] = \textbf{G}[b]\cdot g_{i+1}$
		\EndFor
		\EndFor
		\EndProcedure

		\State{$\forall x\in\binary^\ell$, set $\textbf{A}_{h_g}[x] = 0$}
		\For{every $(z,x,y)$ such that $f_1(z,x,y)$ is non-zero}
		\State{$\textbf{A}_{h_g}[x] = \textbf{A}_{h_g}[x]+\textbf{G}[z]\cdot f_1(z,x,y) \cdot \textbf{A}_{f_3}[y]$}
		\EndFor
		\State{\textbf{return} $\textbf{A}_{h_g}$;}
	\end{algorithmic}
	\end{algorithm}

\end{figure}

\end{proof}

%	Before we present our main result (see Theorem~\ref{final_time}), let us define array $\textbf{G}[z]$ for $z\in \binary^\ell$, with respect to the fixed point $g$, as
%$$\textbf{G}[z]=\prod_{i=1}^\ell ((1-g_i)(1-z_i)+g_iz_i))\,.$$

%We now have the following lemmata:

%\babis{pls write formal proofs of the lemmata below}
%
%\begin{lemma}\label{lemma_g}
%There exists an algorithm that runs in $O(2^\ell)$ time that computes $\textbf{\emph{G}}[z]$ for all $z\in \binary^\ell$.
%\begin{proof}
%
%
%		\begin{algorithmic}[1]
%			\Procedure{{$\textbf{G}\leftarrow$  \sf Precompute}}{$g$}    \Comment{$g\in\F^\ell$, $\textbf{G}$ is an array of size $2^\ell$.}
%			\State Set $\textbf{G}[0] = 1$
%			\For{$i=0,\ldots,\ell-1$}
%			\For{$b\in\binary^i$}
%			\State $\textbf{G}[b,0] = \textbf{G}[b]\cdot(1-g_{i+1})$ \Comment{$g_{i+1}$ is the $(i+1)$-th element in $g$.}
%			\State $\textbf{G}[b,1] = \textbf{G}[b]\cdot g_{i+1}$
%			\EndFor
%			\EndFor
%			\EndProcedure
%		\end{algorithmic}
%
%\babis{pls provide alg here and proof}
%\end{proof}
%\end{lemma}

%\begin{lemma}\label{lemma_h_g}
%Let $\mathcal{N}_x$ be the set of $(z,x,y)\in \binary^{3\ell}$ such that $f_1(z,x,y)$ is non-zero. Then it is
%$h_g(x) = \sum_{(z,x,y)\in \mathcal{N}_x} \textbf{\emph{G}}[z]\cdot f_3(y)$.
%\end{lemma}
%\begin{proof}
%\babis{pls provide proof here}
%\end{proof}

%\begin{lemma}\label{final_time}
%When using the formula from Lemma~\ref{lemma_h_g} to initialize the bookkeeping table for $h_g$, Algorithm $\textsf{\emph{SumCheckProduct}}(h_g(x),f_2(x),u_1,\ldots,u_\ell)$ runs in $O(2^\ell)$ time.
%\end{lemma}
%\begin{proof}
%Since $f_1$ has $O(2^\ell)$ non-zero terms it follows that $\sum |N_x|=O(2^\ell)$. Also since by Lemma~\ref{lemma_g}, $\textbf{G}[z]$ can be pre-computed in $O(2^\ell)$ time, it follows that the bookkeeping table for $h_g$ can be computed in $O(2^\ell)$ time.
%\end{proof}

With the bookkeeping tables, the prover runs $\textsf{SumCheckProduct}(h_g(x), \textbf{A}_{h_g},f_2(x),\textbf{A}_{f_2}, u_1,\ldots,u_\ell)$ in Algorithm~\ref{alg:sumcheckproduct} and the total complexity for phase one is $O(2^\ell)$.

\subsubsection{Phase two.} At this point, all variables in $x$ have been bounded to random numbers $u$. In the second phase, the equation to sum on becomes
\[
\sum\nolimits_{y\in\{0,1\}^\ell}f_1(g,u,y)f_2(u)f_3(y)
\]
Note here that $f_2(u)$ is merely a single value which we already computed in phase one. Both $f_1(g,u,y)$ and $f_3(y)$ are polynomials on $y$ with $\ell$ variables. Similar to phase one, to compute the messages for the last $\ell$ rounds we can call
$$\textsf{SumCheckProduct}(f_1(g,u,y),\textbf{A}_{f_1} ,f_3(y)\cdot f_2(u),\textbf{A}_{f_3}\cdot f_2(u), ,v_1,\ldots,v_\ell)\,.$$
Note here that $\textbf{A}_{f_1}$ is the bookkeeping table for $f_1(g,u,y)$, not the original sparse function $f_1(g,x,y)$.

\smallskip
\noindent{\underline{\emph{Initializing the bookkeeping table for $f_1$:}} \\
It now remains to initialize the bookkeeping table for $f_1(g,u,y)$ efficiently. Similar to phase one, we have the following lemma:
\begin{lemma}\label{lemma_f_1}
	Let $\mathcal{N}_y$ be the set of $(z,x)\in \binary^{2\ell}$ such that $f_1(z,x,y)$ is non-zero. Then for all $y\in\binary^\ell$, it is
	$f_1(g,u,y) = \sum_{(z,x)\in \mathcal{N}_y}I(g,z)\cdot I(u,x) \cdot f_1(z,x,y)$.
\end{lemma}

\begin{proof}
This immediately follows from the fact that $f_1$ is a multilinear extension. We have $f_1(g,u,y) = \sum_{z,y\in\binary^\ell}I(g,z)\cdot I(u,x) \cdot f_1(z,x,y)$, where the closed from of $I$ is given in Lemma~\ref{lemma_h_g}.
\end{proof}

\begin{lemma}\label{lemma_f_1_alg}
	The bookkeeping table $\textbf{A}_{f_1}$ can be initialized in time $O(2^\ell)$.
\end{lemma}
\begin{proof}
	Similar to Algorithm~\ref{alg:init_h_g}, he prover again iterates all non-zero indices of $f_1$ to compute it using Lemma~\ref{lemma_f_1}.The full algorithm is presented in Algorithm~\ref{alg:init_f_1}.

	\begin{figure}[t!]
		\begin{algorithm}[H]
			\caption{$\textbf{A}_{f_1}\leftarrow \textsf{Initialize\_PhaseTwo}(f_1, g, u)$}\label{alg:init_f_1}
			\textbf{Input:} Multilinear $f_1$, random $g = g_1,\ldots,g_\ell$ and $u = u_1, \ldots, u_\ell$;  \\
			\textbf{Output:} Bookkeeping table $\textbf{A}_{f_1}$;
			\begin{algorithmic}[1]
				\State{$\textbf{G}\leftarrow\mathsf{Precompute}(g)$}
				\State{$\textbf{U}\leftarrow\mathsf{Precompute}(u)$}

				\State{$\forall y\in\binary^\ell$, set $\textbf{A}_{f_1}[y] = 0$}
				\For{every $(z,x,y)$ such that $f_1(z,x,y)$ is non-zero}
				\State{$\textbf{A}_{f_1}[y] = \textbf{A}_{f_1}[y]+\textbf{G}[z]\cdot\textbf{U}[x]\cdot f_1(z,x,y)$}
				\EndFor
				\State{\textbf{return} $\textbf{A}_{f_1}$;}
			\end{algorithmic}
		\end{algorithm}

	\end{figure}

\end{proof}

%Again, we define $\textbf{U}[x]$ for $x\in \binary^\ell$, with respect to the fixed point $u$ (previous randomness), as $\prod_{i=1}^\ell ((1-u_i)(1-x_i)+u_ix_i)$. Note that by Lemma~\ref{lemma_g}, $\textbf{U}[x]$ for $x\in \binary^\ell$ can be computed in $O(2^\ell)$ time. Similarly we now have the following lemma.
%\begin{lemma}\label{lemma_f_1}
%Let $\mathcal{N}_y$ be the set of $(z,x,y)\in \binary^{3\ell}$ such that $f_1(z,x,y)$ is non-zero. Then it is
%$f_1(z,x,y) = \sum_{(z,x,y)\in \mathcal{N}_y} \textbf{\emph{G}}[z] \cdot \textbf{\emph{U}}[x]$.
%\end{lemma}
%\begin{proof}
%\babis{pls provide proof}
%\end{proof}
%\begin{lemma}\label{final_time_2}
%When using the formula from Lemma~\ref{lemma_f_1} to initialize the bookkeeping table for $f_1$, Algorithm $\textsf{\emph{SumCheckProduct}}(f_1(g,u,y) ,f_3(y)\cdot f_2(u) ,v_1,\ldots,v_\ell)$ runs in $O(2^\ell)$ time.
%\end{lemma}
%\begin{proof}
%Follows from Lemma~\ref{lemma_g} and Lemma~\ref{lemma_f_1}---as in Lemma~\ref{final_time}.
%\end{proof}

We now summarize the final linear-time algorithm for computing the prover messages for the sumcheck protocol on GKR functions. See Algorithm~\ref{final_alg} $\textsf{SumCheckGKR}$.
\begin{theorem}
Algorithm $\textsf{\emph{SumCheckGKR}}$ runs in $O(2^\ell)$ time.
\end{theorem}
\begin{proof}
Follows from Lemmata~\ref{main_lemma_complexity},~\ref{lemma_h_g_alg} and~\ref{lemma_f_1_alg}.
\end{proof}

\begin{figure}
\begin{algorithm}[H]
		\caption{\label{final_alg}$\{a_1,\ldots,a_{2\ell}\}\leftarrow \textsf{SumCheckGKR}(f_1,f_2,f_3,u_1\ldots,u_\ell,v_1,\ldots,v_\ell,g)$}\label{alg:sumcheckgkr}
		 \textbf{Input:} Multilinear extensions $f_1(z,x,y)$ (with $O(2^\ell)$ non-zero entries), $f_2(x), f_3(y)$ and their bookkeeping tables $\textbf{A}_{f_2},\textbf{A}_{f_3}$, randomness $u = u_1,\ldots,u_\ell$ and $v = v_1,\ldots,v_\ell$ and point $g$;\\
    \textbf{Output:}
    $2\ell$ sumcheck messages for $\sum\nolimits_{x,y\in\binary^\ell}f_1(g,x,y)f_2(x)f_3(y)$;

		\begin{algorithmic}[1]
			\State{$\textbf{A}_{h_g}\leftarrow \textsf{Initialize\_PhaseOne}(f_1, f_3,\textbf{A}_{f_3}, g)$}
\State $\{a_1,\ldots,a_\ell\}\leftarrow \textsf{SumCheckProduct}(\sum_{y\in\binary^\ell}f_1(g,x,y) f_3(y), \textbf{A}_{h_g},f_2, \textbf{A}_{f_2},u_1,\ldots,u_\ell)$
\State{$\textbf{A}_{f_1}\leftarrow \textsf{Initialize\_PhaseTwo}(f_1, g, u)$}
\State $\{a_{\ell+1},\ldots,a_{2\ell}\}\leftarrow \textsf{SumCheckProduct}(f_1(g,u,y),\textbf{A}_{f_1} ,f_3(y)\cdot f_2(u), \textbf{A}_{f_3}\cdot f_2(u) ,v_1,\ldots,v_\ell)$
%\State $\textsf{SumCheckProduct}(h_g,f_2,u_1,\ldots,u_\ell)$
%\mathcal{F}\leftarrow \textsf{FunctionEvaluations}(f,r_1,\ldots,r_\ell)$$
%\State $\mathcal{G}\leftarrow \textsf{FunctionEvaluations}(g,r_1,\ldots,r_\ell)$

							\State \textbf{return} $\{a_1,\ldots,a_{2\ell} \}$
		\end{algorithmic}
	\end{algorithm}

\end{figure}

\subsubsection{Generalizations of our technique.}

Our technique can be extended to sumchecks of the general type

\[
\sum\nolimits_{x_1, x_2,\ldots, x_c\in\binary^c}f_0(g,x_1, x_2,\ldots, x_c)f_1(x_1)f_2(x_2)...f_c(x_c)\, ,
\]
where $c$ is a constant, functions $f_i$ are multilinear and $f_0()$ is sparse and consists of linearly-many nonzero monomials. We divide the protocol into $c$ phases similarly as above. This generalization captures the sumcheck in the original GKR paper with identity polynomials (see~\cite{GKR}), and our new algorithms also improve the prover time of this to linear.

\subsection{Putting everything together}

The sumcheck protocol in GKR given by Equation~\ref{eq:GKR} can be decomposed into several instances that has the form of Equation~\ref{eq:linearsc} presented in the previous section. The term $$\sum_{x,y\in\binary^{s_{i+1}}}\tmult_{i+1}(g,x,y)(\tV_{i+1}(x)\tV_{i+1}(y))$$ is exactly the same as Equation~\ref{eq:linearsc}.  The term $\sum_{x,y\in\binary^{s_{i+1}}}\tadd_{i+1}(g,x,y)(\tV_{i+1}(x)+\tV_{i+1}(y))$ can be viewed as:
\[
\sum_{x,y\in\binary^{s_{i+1}}}\tadd_{i+1}(g,x,y)\tV_{i+1}(x)+\sum_{x,y\in\binary^{s_{i+1}}}\tadd_{i+1}(g,x,y)\tV_{i+1}(y)
\]
The first sum can be computed using the same protocol in Algorithm~\ref{final_alg} without $f_3(y)$, and the second sum can be computed without $f_2(x)$. The complexity for both cases remains linear. Due to linearity of the sumcheck protocol, the prover can execute these 3 instances simultaneously in every round, and sum up the individual messages and send them to the veriifer.

\paragraph{Combining two claims.} After the sumcheck in the GKR protocol is completed, as described in Section~\ref{subsec::GKR}, the prover and the verifier need to combine the two claims about $\tV_{i+1}$ received at the end of the sumcheck protocol to one to avoid the exponential blow-up. There are two ways to combine the two claims and we show how to do each of them in linear time.

The second approach using random linear combinations is rather straight forward. After the output layers, $\P$ and $\V$ execute sumcheck protocol on Equation~\ref{eq:randomGKR} instead of Equations~\ref{eq:GKR}, which still satisfies the properties of Equation~\ref{eq:linearsc}. One could view it as 6 instances of Equation~\ref{eq:linearsc} and the prover time is still linear. Moreover, there is a better way to further improve the efficiency. Taking $\sum_{x, y \in\binary^{s_{i+1}}}(\alpha_i\tmult_{i+1}(u,x,y)+\beta_i\tmult_{i+1}(v,x,y))\tV_{i+1}(x)\tV_{i+1}(y)$ as an example, in Algorithm~\ref{alg::phase1}, the prover runs $\mathsf{Precompute}$ twice on $u$ and $v$ to generate two arrays ($\textbf{G}_1$ and $\textbf{G}_2$), and sets $\textbf{G}[b]=\alpha_i \textbf{G}_1[b] + \beta_i\textbf{G}_2[b]$ for all $b$. The rest of the algorithms remains the same. This only incurs a small overhead in practice in our implementation, compared to the original algorithm on Equation~\ref{eq:linearsc}.

Though with the approach above we already have a linear prover GKR protocol, the technique to condense two points to one proposed in the original GKR protocol~\cite{GKR} may still be interesting in some scenarios (e.g., in our implementation, we use this approach in the last layer and only make one query to the multi-linear extension of the input, which is more efficient practice). We present an algorithm to reduce the prover time of this approach to linear in Appendix~\ref{app:onepoint}.

\ignore{
	\subsubsection{Linear Prover Time Sumcheck}
	We will follow the CMT protocol, and improve the performance of sumcheck and the performance of combining two points. We will break the protocol into two parts, the first part deal with queries about $u$, and the second part will deal with queries about $v$.
	\subsubsection{Part One} In this part, we will deal with queries in first $s_{i+1}$ rounds.
	\paragraph{Review the sumcheck}
	In this subsection, we focus on answering the sumcheck queries. In the first round, the verifier asks the prover to give a univariate function:

	$$f_{i, 1}(x_1)=\sum_{u_{k} \in \{0, 1\}, v \in \{0, 1\}^{s_{i+1}}}\tilde{mult}_{i}(g, \{x_1, u_{2},..., u_{s_{i+1}}\}, v)(\tV_{i+1}(x_1, u_{2},..., u_{|u|})\tV_{i+1}(v)),$$
	where $v$ is a binary vector in $\{0, 1\}^{s_{i+1}}$, $\{x_1, u_{2},..., u_{s_{i+1}}\}$ is the representation of $u$.

	In the $k$-th round where $2\le k\le s_{i+1}$, the verifier asks the prover to give a univariate function:
	\begin{align*}
	f_{i,k}(x_k)=\sum_{u_k \in \{0, 1\}, v \in \{0, 1\}^{s_{i+1}}}\tilde{mult}_{i}(g, \{r_1,r_2,...,r_{k-1},x_k, u_{k+1},..., u_{s_{i+1}}\}, v)\times\\
	(\tV_{i+1}(r_1,r_2,...,r_{k-1},x_k, u_{k+1},..., u_{s_{i+1}})\tV_{i+1}(v))
	\end{align*}
	where $v$ is a binary vector in $\{0, 1\}^{s_{i+1}}$, $\{r_1,r_2,...,r_{k-1},x_k, u_{k+1},..., u_{s_{i+1}}\}$ is the representation of $u$.

	\paragraph{The Definition of Building Block}
	In the first round, we calculate two arrays to help speed up calculation, where each array element is a pair $(a, b)$ representing a linear function $ax+b$.

	The first array is $mult_1^1$, where $mult_1^1[p]$ is a pair $(a, b)$ representing a linear function as follows:
	$${mult}_1^1[p]\overset{def}{=}\sum_{v\in \{0,1\}^{s_{i+1}}}\tilde{mult}_i(g, \{x_1, p\}, v)\tV_{i+1}(v),$$
	here $p$ is a $s_{i+1}-1$ bits binary number. $\{x_1, p\}$ is the representation of $u$. Similarly we define our second array ${V}_1^1[p]$ as follows:
	$$V_1^1[p]\overset{def}{=}\tV_i(\{x_1, p\})$$

	We will treat both arrays as a linear function or a pair of field elements interchangeably, for example ${mult}_1^1[p](x)$ means we treat it as a linear function, and ${mult}_1^1[p].a, {mult}_1^1[p].b$ means we treat it as a pair.

	Similarly we define arrays that used in $k$-th round as follows:
	\begin{definition}[Linear Function Array Part One]

		The definition for $k=1$ is the statement above, for $k\ge 2$
		\begin{enumerate}
			\item ${mult}_k^1[p]\overset{def}{=}\sum_{v\in \{0, 1\}^{s_{i+1}}}\tilde{mult}_i(g, \{r_1, ..., r_{k-1}, x_k, p\}, v)\tV_{i+1}(v)$
			\item ${V}_k^1[p]\overset{def}{=}\tV_i(\{r_1, ..., r_{k-1}, x_k, p\})$
			\item Where $p$ is a $s_{i+1}-k$ bits binary number and $\{r_1, ..., r_{k-1}, x_k, p\}$ is the representation of $u$.
		\end{enumerate}
	\end{definition}

	\paragraph{Initialize Arrays}
	We will try to initialize ${mult}_1^1, {V}_1^1$. And we will calculate next arrays using these arrays.

	For ${V}_1^1$, we can scan through the circuit and get all values of $\tV_{i+1}(u)$ when $u \in \{0, 1\}^{s_{i+1}}$, and we have the following equations:
	\begin{align*}
	{V}_1^1[p](0)=\tV_{i+1}(0||p)\\
	{V}_1^1[p](1)=\tV_{i+1}(1||p)
	\end{align*}
	We can solve for ${V}_1^1[p]$ easily.

	For ${mult}_1^1$, the idea is the same as ${V}_1^1$, but it's non-trivial to calculate $\sum_{v\in \{0,1\}^{s_{i+1}}}\tilde{mult}_{i+1}(g, u, v)\tV_{i+1}(v)$ for every binary $u$ in linear time, since we need to calculate $\tilde{\beta}_{s_{i}}(g, g')$ part in $\tilde{mult}_{i+1}(g, u, v)$. Instead enumerate every binary $u$ to initialize the array, we can enumerate $g'$ instead. And accumulate the contribution of $(g', u', v')$ to $\sum_{v\in \{0,1\}^{s_{i+1}}}\tilde{mult}_{i+1}(g, u', v)\tV_{i+1}(v)$. We will use a depth first search to enumerate every $g'$ bit by bit. The detailed algorithm is shown in Algorithm \ref{alg::premult}.
	\begin{figure}[p]
		\begin{algorithm}[H]
			\label{alg::premult}
			\caption{Preprocess ${mult}_1^1, {V}_1^1$}
			\begin{algorithmic}[1]
				\parState{Assuming we have global access to $g$, $s_i$, $s_{i+1}$, which is reasonable since prover knows these values, and $g[k]$ means the $k$-th field element of $g$, starting from $0$ since we are writing pseudo-code}
				\Procedure{{\sf DFS}}{$A$, $g'$, $depth$, $Value$}
				\parState{Note $A$, $g'$ is the pointer to the array, any change to $A$ or $g'$ will change it globally.}
				\parState{In this function, we will recurse on the bits of $g$}
				\If {$depth=s_{i}$} \Comment{We have enumerate enough bits}
				\State Let $u', v'$ is input gates to $g'$
				\State Find $\tV_{i+1}(v')$ in circuit.
				\State $A[u'] = A[u'] + \tV_{i+1}(v')\times Value$
				\Else
				\State set $g'[depth]=0$
				\State Call {\sf DFS}($A$, $g'$, $depth+1$, $Value\times(1-g[depth])$)
				\State set $g'[depth]=1$
				\State Call {\sf DFS}($A$, $g'$, $depth+1$, $Value\times g[depth]$)
				\EndIf
				\EndProcedure
				\Procedure{{\sf InitializeMult}}{}
				\State Let array ${mult}_1^1$ be filled with zeros.
				\parState{Since calculate $\sum_{v\in \{0,1\}^{s_{i+1}}}\tilde{mult}_{i+1}(g, u, v)\tV_{i+1}(v)$ for every binary $u$ is enough to initialize array ${mult}_1^1$, we will call a subroutine to initialize every binary $u$, and store the result into array $A$.}
				\State Initialize array $A$ with zeros.
				\State let $g'$ be a vector of $s_{i+1}$ bits.
				\State Call {\sf DFS}($A$, $g'$, $0$, $1$)
				\For{each $p \in \{0, 1\}^{s_{i+1}-1}$}
				\parState{By solving the linear equations, we can get the following statements}
				\State ${mult}_1^1[p].b=A[0||p]$
				\State ${mult}_1^1[p].a=A[1||p]-A[0||p]$
				\EndFor
				\EndProcedure
			\end{algorithmic}
		\end{algorithm}
	\end{figure}

	\paragraph{Answer Queries}
	The answer to the first query is the following univariate function:

	$$\sum_{p}{mult}_1^1[p](x)\times{V}_1^1[p](x),$$

	It's easy to get this result by doing some transformation on the original equation.

	We reduce these arrays to the second round using the following way

	\begin{enumerate}
		\item By definition, ${mult}_2^1[p'](0)={mult}_1^1[0||p'](r_1)$ and ${mult}_2^1[p'](1)={mult}_1^1[1||p'](r_1)$
		\item By definition, ${V}_2^1[p'](0)={V}_1^1[0||p'](r_1)$ and ${V}_2^1[p'](1)={V}_1^1[1||p'](r_1)$

		\item Since both ${mult}_2^1, {V}_2^1$ are linear equations, we can solve the ${mult}_2^1, {V}_2^1$ by solving linear equations.
		\item By solving linear equations we found that ${mult}_2^1[p'].b={mult}_1^1[0||p'](r_1)$ and ${mult}_2^1[p'].a={mult}_1^1[1||p'](r_1)-{mult}_1^1[0||p'](r_1)$
	\end{enumerate}

	The answer to the second round is

	$$\sum_{p'}{mult}_2^1[p']\times{V}_2^1[p']$$

	We define arrays for the $k$-th round ($k\ge 2$) in the similarly using the result from $k-1$-th round, and solving the linear equation.

	The answer to the $k$-th round query is
	$$\sum_{p}{mult}_k^1[p]\times{V}_k^1[p]$$

	\begin{theorem} For every round, the algorithm above correctly answered the query. And the running time of the algorithm runs in time $O(\max(2^{s_i}, 2^{s_{i+1}}))$.
	\end{theorem}

	\begin{proof}
		It's trivial to see that ${mult}_k^1, {V}_k^1$ are computed correctly.

		For time complexity part, ${mult}_k^1, {V}_k^1$ are computed in linear time since Algorithm \ref{alg::premult} runs in linear time, and for $k$-th layer, the size of ${mult}_k^1$ is half size of ${mult}_{k-1}^1$, by total size of these arrays is linear to the circuit size.

		The answer to the verifier is correct by the definition of ${mult}_k^1, {V}_k^1$.
	\end{proof}

	\subsubsection{Part two}
	In this part, we will deal with next $s_{i+1}$ rounds. In these rounds, $g, u$ are fixed random vector with $s_{i}, s_{i+1}$ random field elements respectively.

	\paragraph*{Initialize Arrays}
	We define two arrays ${mult}_k^2, {V}_k^2$ analogue to ${mult}_k^1, {V}_k^1$.

	\begin{definition}[Linear Function Array Part Two]
		For $k=1$, we define
		\begin{enumerate}
			\item ${mult}_1^2[p] = \tilde{mult}_i(g, u, \{x_1, p\})\tV_{i+1}(u)$
			\item ${V}_1^2[p] = \tV_{i+1}(\{x_1, p\})$
		\end{enumerate}
		For $k\ge 2$, we define
		\begin{enumerate}
			\item ${mult}_k^2[p]\overset{def}{=}\tilde{mult}_i(g, u, \{r_1, ..., r_{k-1}, x_k, p\})\tV_{i+1}(u)$
			\item ${V}_k^2[p]\overset{def}{=}\tV_i(\{r_1, ..., r_{k-1}, x_k, p\})$
			\item Where $p$ is a $s_{i+1}-k$ bits binary number and $\{r_1, ..., r_{k-1}, x_k, p\}$ is the representation of $v$.
		\end{enumerate}
	\end{definition}

	As we discussed in Part One, we only need to calculate all the values of ${mult}_k^2, {V}_k^2$. By the definition of $\tilde{mult}_i(g, u, v)\overset{def}{=}\sum_{g', u', v' \in G_{i, mult}}\tilde{\beta_{s_i}}(g, g')\tilde{\beta_{s_{i+1}}}(u, u')\tilde{\beta_{s_{i+1}}}(v, v')$. We found that $\tilde{mult}_i$ is composed by three separate functions. Since $g, u$ are fixed random vector, we can preprocess $\tilde{\beta}_{s_i}(g, g')$ for every binary $g'$ and preprocess $\tilde{\beta}_{s_{i+1}}(u, u')$ for every binary $u'$. See the detailed algorithm in Algorithm \ref{alg::parttwo}.

	\begin{figure}[p]
		\begin{algorithm}[H]
			\label{alg::parttwo}
			\caption{Preprocess $mult_1^2, V_1^2$}
			\begin{algorithmic}[1]
				\parState{Array/Vector parameters are passed by pointer, any change to the array will effect the array globally}
				\Procedure{{\sf DFS}}{$A, x, randVec, depth, maxDepth, value$}
				\If{$depth=maxDepth$}
				\State $A[x]=value$
				\Else
				\State set $x[depth]=0$
				\State Call DFS($A$, $x$, $depth + 1$, $value \times (1-randVec[depth])$)
				\State set $x[depth]=1$
				\State Call DFS($A$, $x$, $depth + 1$, $value \times randVec[depth]$)
				\EndIf
				\EndProcedure
				\Procedure{{\sf PreprocessBeta}}{$BetaMult, BetaV$}
				\State let $g', u'$ be zero $s_{i}, s_{i+1}$ length vector
				\State Call {\sf DFS}($BetaMult$, $g'$, $g$, $0$, $s_{i}$, $1$)
				\State Call {\sf DFS}($BetaV$, $u'$, $u$, $0$, $s_{i + 1}$, $1$)
				\EndProcedure
				\Procedure{{\sf Preprocess}}{${Mult}_1^2, {V}_1^2$}
				\State Let $BetaMult, BetaV$ be a array with length $2^{s_{i}}, 2^{s_{i+1}}$ respectively
				\State Call {\sf PreprocessBeta}($BetaMult$, $BetaV$)
				\State Let $MultBinaryU$ be a array with $2^{s_{i+1}}$ field elements, initially set to $0$.
				\parState{The array $MultBinaryU[v'] \overset{def}{=}\tilde{mult}_i(g, u, v')\tV_{i+1}(u)$}, we will show that how to compute this array in this For loop.
				\For{Each $v' \in \{0, 1\}^{s_{i+1}}$}
				\For{Each $(g', u')$ that is connected to $v'$ in the circuit}
				\State $MultBinaryU[v'] += BetaMult[g'] \times BetaV[u'] \times \tV_{i+1}[u]$
				\parState{Since $u$ is a constant, $\tV_{i+1}[u]$ is a constant all the time as well, we can obtain the value from the result of Part One (It equals to $V_{s_{i+1}-1}^1[0](r_{s_{i+1}})$ by the definition of $V_{k}^1$).}
				\EndFor
				\EndFor

				\For {each $p \in \{0, 1\}^{s_{i+1}-1}$}
				\parState{By solving the linear equations , we can get the following statements}
				\State $Mult_1^2[p].b = MultBinaryU[0||p]$
				\State $Mult_1^2[p].a = MultBinaryU[1||p]-MultBinaryU[0||p]$
				\parState{$\tV_{s_{i+1}}[v']$ is the value directly from the circuit, we assume it's preprocessed into a array.}
				\State $V_1^2[p].b = \tV_{s_{i+1}}[0||p]$
				\State $V_1^2[p].a = \tV_{s_{i+1}}[1||p]-\tV_{s_{i+1}}[0||p]$
				\EndFor
				\EndProcedure
			\end{algorithmic}
		\end{algorithm}
	\end{figure}

	\paragraph*{Answer Queries}
	The answer to the first query is the following univariate function:

	$$\sum_{p}mult_{1}^2[p](x)\times V_1^2[p](x)$$

	It's easy to check that this summation is equals to the original sumcheck summation.

	To answer $k$-th query ($k\ge 2$), we suppose we have answered the $k-1$-th query and generated arrays $mult_{k-1}^2, V_{k-1}^2$, we generate $mult_{k}^2, V_{k}^2$ and answer the $k$-th query in the following way:

	\begin{enumerate}
		\item By definition, $\forall p \in {\{0, 1\}^{s_{i+1}-k}}$, $mult_{k}^2[p](0)=mult_{k-1}^2[0||p](r_{k-1}), mult_{k}^2[p](1)=mult_{k-1}^2[1||p](r_{k-1})$
		\item By definition, $\forall p \in {\{0, 1\}^{s_{i+1}-k}}$, $V_{k}^2[p](0)=V_{k-1}^2[0||p](r_{k-1}), V_{k}^2[p](1)=V_{k-1}^2[1||p](r_{k-1})$
		\item Since $mult_{k}^2[p], V_{k}^2[p]$ are linear function, we can solve the equations above. We have the following results.
		\item $\forall p \in {\{0, 1\}^{s_{i+1}-k}}$, $mult_{k}^2[p].b = mult_{k-1}^2[0||p](r_{k-1}), mult_{k}^2[p].a = mult_{k-1}^2[1||p](r_{k-1})-mult_{k-1}^2[0||p](r_{k-1})$
		\item $\forall p \in {\{0, 1\}^{s_{i+1}-k}}$, $V_{k}^2[p].b = V_{k-1}^2[0||p](r_{k-1}), V_{k}^2[p].a = V_{k-1}^2[1||p](r_{k-1})-V_{k-1}^2[0||p](r_{k-1})$
		\item For $k$-th round, the answer to the query is $\sum_{p} mult_{k}^2[p](x)\times V_{k}^2[p](x)$
	\end{enumerate}

	\begin{theorem} The algorithm above correctly answered queries of last $s_{i+1}$ rounds, and runs in time $O(\max(2^{s_{i}}, 2^{s_{i+1}}))$.
	\end{theorem}

	\begin{proof}
		Since we only need to scan through the circuit, the time complexity is linear for initialize $mult_{1}^2, V_1^2$. The size of $mult_{k}^2, V_k^2$ is half of $mult_{k-1}^2, V_{k-1}^2$. The total size of $mult_{k}^2, V_k^2$ is linear to the size of $mult_1^2, V_1^2$, which is the circuit size. And to answer the query, we only need to scan through these arrays. The total time complexity is linear to the circuit size.

		If $mult_k^2, V_k^2$ are generated correctly, the answer to the query is correct by definition.

		$V_1^2$ is correct since it's equal to the circuit value. $mult_1^2$ is correct if and only if $MultBinaryU$ is generated correctly. $\forall g'$, $BetaMult[g']$ is correct because we calculated it bit by bit according to definition in $DFS$ part. And so does $BetaV[u']$. By definition
		\begin{align*}
		MultBinaryU[v']&\overset{def}{=}\tilde{mult}_{i}(g, u, v')\tV_{i+1}(u)
		\\&=\sum_{v=v', (g', u', v) \in G_{i, mult}}\tilde{\beta}_{s_{i}}(g, g')\tilde{\beta}_{s_{i+1}}(u, u')\tilde{\beta}_{s_{i+1}}(v, v')\tV_{i+1}(u)
		\\&=\sum_{v=v', (g', u', v) \in G_{i, mult}}\tilde{\beta}_{s_{i}}(g, g')\tilde{\beta}_{s_{i+1}}(u, u')\tV_{i+1}(u)
		\end{align*}

		Since $BetaMult[g'], BetaV[u']$ are correct, we have $MultBinaryU$ is correct. Thus $mult_1^2$ is correct.

		To prove $mult_k^2, V_k^2$ is correct ($k \ge 2$). We assume $mult_{k-1}^2, V_{k-1}^2$ is correct. According to the definition of $mult_k^2$, $mult_k^2[p](0)=\tilde{mult}_i(g, u, {r_1, r_2,...,r_{k-1}, 0, p})=mult_{k-1}^2[0||p](r_{k-1})$, $mult_k^2[p](1)=\tilde{mult}_i(g, u, {r_1, r_2,...,r_{k-1}, 1, p})=mult_{k-1}^2[1||p](r_{k-1})$, by solving the linear equation, we can get the right value of $mult_k^2[p]$. And we have a similar argument of $V_k^2$.
	\end{proof}
}


% declare document class and geometry
\documentclass[12pt]{article} % use larger type; default would be 10pt
\usepackage[margin=1in]{geometry} % handle page geometry

\input{../header2.tex}

\title{Phys 220A -- Classical Mechanics -- Lec13}
\author{UCLA, Fall 2014}
\date{\formatdate{20}{11}{2014}} % Activate to display a given date or no date (if empty),
         % otherwise the current date is printed

\begin{document}
\setlength{\unitlength}{1mm}
\maketitle

\section{Rigid bodies}

Rigid bodies are great because they give us interesting differential equations and provide a good example of when coordinate transformations are useful. What is a rigid body? It's a set of $3N$ degrees of freedom $\v x^1, \dots, \v x^N$ ($N$ mass points) where all the distances are constant, i.e.
\begin{eqn}
\abs{\v x^i - \v x^j} = \text{const}
\end{eqn}
for all $i,j$. We can think of rigid bodies from a different perspective by considering translations of the center of mass $\v X$,
\begin{eqn}
\v x^i \rightarrow \v x^i + \v X,
\end{eqn}
which gives us three degrees of freedom, and rotations of the body
\begin{eqn}
\v x^i \rightarrow R \cdot \v x^i, \qquad
R \in SO(3),
\end{eqn}
which give us another 3 degrees of freedom. So the $3N$ degrees of freedom reduce down to 6.

If we look at the rotations in terms of the Lie algebra, we have
\begin{eqn}
R = 1 + \epsilon T
\end{eqn}
where $T$ is a generator of rotations. We find that
\begin{eqn}
R^\top R =1 \qquad
\implies \qquad
T^\top + T = 0,
\end{eqn}
so the generators $T$ are $3 \times 3$ antisymmetric matrices,
\begin{eqn}
T =
\begin{pmatrix}
0 & a & b \\
-a & 0 & c \\
-b & -c & 0
\end{pmatrix}.
%\begin{pmatrix}
%0 & \omega_3 & -\omega_2 \\
%-\omega_3 & 0 & \omega_1 \\
%\omega_2 & -\omega_1 & 0
%\end{pmatrix}.
\end{eqn}
For $b=c=0$ we get rotations about the $z$-axis, similarly about the $x$-axis for $a=b=0$ and about the $y$-axis for $a=c=0$. So we can rewrite the generators as
\begin{eqn}
T_{ij} = \sum_k \epsilon_{ijk} \omega_k
\end{eqn}
which gives us rotation about the vector $\v \omega_k$.

Generally it will be useful to change from the lab frame $\Gamma$ into the body frame $\Gamma'$. The relation between points $\v x$ and $\v x'$ in these frames can be written
\begin{eqn}
x_i(t) = X_i(t) + R_{ij}(t) x'_j(t),
\label{eq:changeFrame}
\end{eqn}
where $\v x'$ is a constant vector and we need to find the rotation $R_{ij}(t)$.

\subsection{Euler Angles}

It turns out that for an arbitrary rotation $R \in SO(3)$ we can write
\begin{eqn}
R = R_3(\psi) R_1(\theta) R_3(\phi)
\end{eqn}
where $R_1$ and $R_3$ are rotations
\begin{eqn}
R_3(\phi) =
\begin{pmatrix}
\cos \phi & \sin\phi & 0 \\
-\sin\phi & \cos\phi & 0 \\
0 & 0 & 1,
\end{pmatrix}, \qquad
R_1(\theta) =
\begin{pmatrix}
1 & 0 & 0 \\
0 & \cos\theta & \sin\theta \\
0 & -\sin\theta & \cos\theta
\end{pmatrix}.
\end{eqn}
Note that we perform these rotations in right-left order, i.e. first perform the rotation $R_3(\phi)$, next the rotation $R_1(\theta)$, and finally the rotation $R_3(\psi)$.

\subsection{Velocity}

If we take the time derivative of our coordinate transformation \eqref{eq:changeFrame}, we obtain
\begin{eqn}
\dot x_i = \dot X_i + \dot R_{ij} x'_j.
\label{eq:velocity0}
\end{eqn}
Now consider for a moment the derivative of $R^\top R = 1$,
\begin{eqn}
\dot R^\top R + R^\top \dot R = 0,
\end{eqn}
which if we write $M = R^\top \dot R$ is just
\begin{eqn}
M^\top + M = 0.
\end{eqn}
Then we can write $M$ as
\begin{eqn}
M_{ij} = (R^\top \dot R)_{ij} = \sum_k \epsilon_{ijk} \omega_k
\end{eqn}
where $\v \omega$ is the angular velocity, so we can rewrite the velocity equation \eqref{eq:velocity0} as
\begin{eqn}
\vd x = \vd X + R \cdot (\v \omega \times \v x'),
\label{eq:velocity1}
\end{eqn}
which is our major result. One more thing to note is that if we shift coordinates in the body frame $\v x' \rightarrow \v x' + \v a$, our velocity changes by
\begin{eqn}
\vd x \rightarrow \vd x + R \cdot (\v \omega \times \v a).
\end{eqn}

Now we can work out the kinetic energy with our new velocity equation \eqref{eq:velocity1},
\begin{align}
T &= \frac{1}{2} \sum_i m_i \vd x^{i\top} \vd x^i \\
	&= \frac{1}{2} \sum_i m_i \vd X^\top \vd X
		\; + \; \vd X^\top R \cdot (\v \omega \times \cancelto{0}{\sum_i m_i  \v x'^i}) \\
		&\qquad + \; \frac{1}{2} \sum_i m_i (\v \omega \times \v x'^i)^\top \underbrace{R^\top R}_{=1} \cdot (\v \omega \times \v x'^i),
\end{align}
where the second term cancels because it is just the center of mass in the body frame, which is zero. So we can rewrite the remaining two terms as
\begin{eqn}
T = T_\mathrm{CM} + \frac{1}{2} \sum_{ij} \omega_i I_{ij} \omega_j,
\end{eqn}
where $I_{ij}$ is the inertia tensor
\begin{eqn}
I_{ij} = \sum_k m_k \left[ \delta_{ij} (\v x'^k)^2 - x'^k_i x'^k_j \right].
\end{eqn}
We've been working with discrete point masses, what if we more generally have a mass density $\rho(\v x)$? Then the sum over masses becomes an integral over the mass density. Explicitly, the matrix takes the form
\begin{eqn}
I = \int \dif^3{x} \, \rho(\v x)
\begin{pmatrix}
y^ + z^2 & -xy & -xz \\
-xy & x^2 + z^2 & -yz \\
-xz & -yz & x^2 + y^2
\end{pmatrix}.
\end{eqn}

We can immediately notice some nice properties of $I_{ij}$:
\begin{itemize}
\item it's positive definite,
\item it's symmetric, and thus
\item it can be diagonalized, i.e. we can rotate so that $I = \diag\set{I_1, I_2, I_3}$.
\end{itemize}
The inertia tensor's eigenvectors define a set of ``principal axes'' determined by the eigenvalues $I_k$. We can summarize the symmetry of an object by
\begin{itemize}
\item $I_1 = I_2 = I_3$ --- completely spherically symmetric object, e.g. point mass;
\item $I_1 = I_2 \neq I_3$ --- radially symmetric, e.g. symmetric top;
\item $I_1 \neq I_2 \neq I_3$ --- asymmetric, e.g. asymmetric top.
\end{itemize}
Finally we have the following theorem.
\begin{theorem}[Parallel Axis theorem]
Suppose we shift the axis from the center of mass to the point $\v a$, then the inertia tensor shifts by
\begin{eqn}
I_{ij}' = I_{ij}^\mathrm{CM} + M_\mathrm{tot} (\v a^2 \delta_{ij} - a_i a_j).
\end{eqn}
\end{theorem}

How can we write the angular momentum in this notation? We have
\begin{align}
\v L_\mathrm{tot} &= \sum_k m_k \v x x^k \times \vd x^k \\
	&= \sum_k m_k (\v X + R \v x'^k) \times (\vd X + R \cdot (\omega \times \v x'^k)) \\
	&= \underbrace{\sum_k m_k \v X \times \vd X}_{\v L_\mathrm{CM}} \; + \; \cancelto{0}{\sum_k m_k \v x'^k} \\
		&\qquad\quad + \; \sum_k m_k (R \v x'^k) \times R (\v \omega \times \v x'^k).
\end{align}
Using the identity
\begin{eqn}
\sum_{ijk} \epsilon_{ijk} R_{ii'} R_{jj'} R_{kk'} = \epsilon_{i'j'k'},
\end{eqn}
one can show that this results in
\begin{eqn}
\v L_\mathrm{tot} = \v L_\mathrm{CM} + R \cdot (I \cdot \v \omega).
\end{eqn}
This informs us that the angular momentum in the body frame is $\v L' = I \cdot \v \omega$.

\subsection{Free rigid body}

Suppose there are no external forces and therefore no torques. Then we have $\vd L_\mathrm{inertial} = 0$ so that
\begin{eqn}
0 = \od{}{t} (R I \v \omega) = \dot R I \v \omega + R I \vd \omega.
\end{eqn}
If we multiply the first term on the left by $R R^\top = 1$ we find
\begin{eqn}
R \underbrace{(\v \omega \times (I \v \omega) + I \vd \omega)}_{=0} = 0.
\end{eqn}
Explicitly, working in the diagonalized frame where our axes are the principal axes, we obtain \textit{Euler's equations},
\begin{align}
0 &= I_1 \dot \omega_1 + \omega_2 \omega_3 (I_3 - I_2), \\
0 &= I_2 \dot \omega_2 + \omega_3 \omega_1 (I_1 - I_3), \\
0 &= I_3 \dot \omega_3 + \omega_1 \omega_2 (I_2 - I_1).
\end{align}
If we further assume that $\v \omega$ is constant and we have a symmetric top, $I_1 = I_2$, we find
\begin{eqn}
\dot \omega_1 = \omega_2 \Omega, \qquad
\dot \omega_2 = - \omega_1 \Omega,
\end{eqn}
where
\begin{eqn}
\Omega = \omega_3 \, \frac{I_1 - I_3}{I_1}.
\end{eqn}
The general case is very difficult and must be solved using elliptic functions.

\end{document}


\section{Introduction}
When studying intelligence, insects, reptiles, and humans have been found to possess neurons with the capacity to hold integers, real numbers, and perform arithmetic operations \cite{nieder-neuronal-number,rugani-arithmetic-chicks,gallistel-numbers-in-brain}.
In our quest to mimic intelligence, we have put much faith in neural networks, which in turn has provided unparalleled and often superhuman performance in tasks requiring high cognitive abilities \cite{natureGo,bert,openai-learning-dexterous}.
However, when using neural networks to solve simple arithmetic problems, such as counting, multiplication, or comparison, they systematically fail to extrapolate onto unseen ranges \cite{stillNotSystematic,suzgun2019evaluating,trask-nalu}.
The absence of inductive bias makes it difficult for neural networks to extrapolate well on arithmetic tasks as they lack the underlying logic to represent the required operations.

A neural component that can solve arithmetic problems should be able to: take an arbitrary hidden input, learn to select the appropriate elements, and apply the desired arithmetic operation.
A recent attempt to achieve this goal is the Neural Arithmetic Logic Unit (NALU) by \citet{trask-nalu}.

The NALU models the inductive bias explicitly via two sub-units: the $\text{NAC}_{+}$ for addition/subtraction and the $\text{NAC}_{\bullet}$ for multiplication/division.
The sub-units are softly gated between, using a sigmoid function, to exclusively select one of the sub-units.
However, we find that the soft gating-mechanism and the $\text{NAC}_{\bullet}$ are fragile and hard to learn.

In this paper, we analyze and improve upon the $\text{NAC}_{+}$ and $\text{NAC}_{\bullet}$ with respect to addition, subtraction, and multiplication.
Our proposed improvements, namely the Neural Addition Unit (NAU) and Neural Multiplication Unit (NMU), are more theoretically founded and improve performance regarding stability, speed of convergence, and interpretability of weights.
Most importantly, the NMU supports both negative and small numbers and a large hidden input-size, which is paramount as neural networks are overparameterized and hidden values are often unbounded.

The improvements, which are based on a theoretical analysis of the NALU and its components, are achieved by a simplification of the parameter matrix for a better gradient signal, a sparsity regularizer, and a new multiplication unit that can be optimally initialized.
The NMU does not support division.
However, we find that the $\text{NAC}_{\bullet}$ in practice also only supports multiplication and cannot learn division (theoretical analysis on division discussed in section \ref{sssec:nac-mul}).

To analyze the impact of each improvement, we introduce several variants of the $\text{NAC}_{\bullet}$.
We find that allowing division makes optimization for multiplication harder, linear and regularized weights improve convergence, and the NMU way of multiplying is critical when increasing the hidden size.

Furthermore, we improve upon existing benchmarks in \citet{trask-nalu} by expanding the ``simple function task'', expanding ``MNIST Counting and Arithmetic Tasks'' with a multiplicative task, and using an improved success-criterion \citet{maep-madsen-johansen-2019}.
This success-criterion is important because the arithmetic layers are solving a logical problem.
We propose the MNIST multiplication variant as we want to test the NMU’s and $\text{NAC}_{\bullet}$’s ability to learn from real data and extrapolate.

\begin{figure}[t]
\centering
\vspace{-1em}
\includegraphics[scale=0.65]{graphics/nmu.pdf}
\caption{Visualization of NMU for a single output scalar $z_1$, this construction repeats for every element in the output vector $\mathbf{z}$.}
\end{figure}

\subsection{Learning a 10 parameter function}
Consider the static function $t = (x_1 + x_2) \cdot (x_1 + x_2 + x_3 + x_4)$ for $x \in \mathbb{R}^4$. To illustrate the ability of $\mathrm{NAC}_{\bullet}$, NALU, and our proposed NMU, we conduct 100 experiments for each model, where we attempt to fit this function. Table \ref{tab:very-simple-function-results} shows that NMU has a higher success rate and converge faster.
\input{results/simple_mul.tex}


\section{Probability Generating Functions}
\subsection{Definitions and Examples}
\begin{definition}
    Let $X$ random variable with $\Omega_X\subset\mathbb N$, the Probability distribution function as defined previously would be $p_r=\mathbb P(X=r),\forall r\in\mathbb N$.
    The probability generating function (PGF) is defined to be
    $$p(z)=\sum_{r=0}^\infty p_rz^r=\mathbb E[z^X]$$
\end{definition}
For $|z|\le1$, the series converges absolutely by comparison test, so this function is well defined in the interval $[-1,1]$.
\begin{theorem}
    The PGF uniquely determines the distribution of the random variable.
\end{theorem}
\begin{proof}
    Suppose
    $$\sum_{r=0}^\infty p_rz^r=\sum_{r=0}^\infty q_rz^r$$
    Plugging in $z=0$ gives $p_0=q_0$.
    Suppose $p_r=q_r$ for any $r\in\{1,2,\ldots,n\}$, we can get
    $$\sum_{r=n+1}^\infty p_rz^r=\sum_{r=n+1}^\infty q_rz^r$$
    dividing both sides by $z^{n+1}$ and sending $z\to0$ shows $p_{n+1}=q_{n+1}$.
\end{proof}
\begin{theorem}
    $$\lim_{z\to 1^-}p^\prime(z)=p^\prime(1-)=\mathbb E[X]$$
\end{theorem}
\begin{proof}
    Assume first that $\mathbb E[X]<\infty$.
    For $0<z<1$ we have
    $$p^\prime(z)=\sum_{r=0}^\infty rp_rz^{r-1}\le\sum_{r=0}^\infty rp_r=\mathbb E[X]\implies \lim_{z\to 1^-}p^\prime(z)\le \mathbb E[X]$$
    The limit exists since $p^\prime$ is increasing.
    Also for any $\epsilon>0$, for large enough $N$ we have
    $$\sum_{r=0}^Nrp_r\ge\mathbb E[X]-\epsilon$$
    Now since $p^\prime(z)\ge\sum_{r=0}^Nrp_rz^{r-1}$, we have
    $$\lim_{z\to 1^-}p^\prime(z)\ge\sum_{r=0}^Nrp_r\ge\mathbb E[X]-\epsilon$$
    Hence we must have $p^\prime(1-)=\mathbb E[X]$.\\
    Now if $\mathbb E[X]=\infty$, then $\forall M\in\mathbb N$, for large enough $N$ we have
    $$\sum_{r=0}^Nrp_r>M$$
    Thus by the same argument $p^\prime(1-)\ge M$, hence $p^\prime(1-)=\infty$.
\end{proof}
Similarly and by induction, we have
$$p^{(k)}(1-)=\mathbb E[X(X-1)\cdots (X-k+1)]$$
In particular
$$\operatorname{Var}(X)=p^{\prime\prime}(1-)+p^\prime(1-)-(p^\prime(1-))^2$$
Also by simply differentiating we get $p_n=\mathbb P(X=n)=P^{(n)}(0)/n!$.
\begin{proposition}
    Let $X_1,\ldots,X_n$ be independent random variables with PGFs $q_i$, then the PGF $p$ of $X_1+\cdots +X_n$ satisfies $p=\prod_iq_i$.
\end{proposition}
\begin{proof}
    Obvious.
\end{proof}
In particular, if $X_i$'s are i.i.d., then $p=q^n$.
\begin{example}
    1. Let $X\sim\operatorname{Bin}(n,p)$, then
    $$p(z)=\mathbb E[z^X]=\sum_{k=0}^n\binom{n}{k}z^kp^k(1-p)^{n-k}=(1-p+zp)^n$$
    Therefore if we have independent random binomial variables $X\sim\operatorname{Bin}(n,p),Y\sim\operatorname{Bin}(m,p)$, then $\mathbb E[z^{X+Y}]=(1-p+pz)^{n+m}$, hence $X+Y\sim\operatorname{Bin}(n+m,p)$ as expected.\\
    2. Let $X\sim\operatorname{Geom}(p)$, then
    $$p(z)=\mathbb E[z^X]=\sum_{r=0}^\infty z^r(1-p)^rp=\frac{p}{1-z(1-p)}$$
    3. Suppose $X\sim\operatorname{Pois}(\lambda)$, then
    $$p(z)=\sum_{r=0}^\infty z^re^{-\lambda}\frac{\lambda^r}{r!}=e^{(z-1)\lambda}$$
    So for independent $X\sim\operatorname{Pois}(\lambda),Y\sim\operatorname{Pois}(\mu)$, we have $\mathbb E[z^{X+Y}]=e^{(z-1)(\lambda+\mu)}$, so $X+Y\sim\operatorname{Pois}(\lambda+\mu)$.
\end{example}
\subsection{Summing up a Random Number of Random Variables}
Suppose $X_1,X_2,\ldots$ be a sequence of i.i.d. random variables and $N$ be a random variable, independent of all $X_i$, taking value in $\mathbb N$.
Let $S_n=X_1+\cdots +X_n$ and we consider the random variable $S_N$ with $S_N(\omega)=X_1(\omega)+\cdots+X_{N(\omega)}(\omega)$.
Legt $q$ be the PGF of $N$ and $p$ the PGF of $X_1$ and $r$ be the PGF of $S_N$, then we have
\begin{align*}
    r(z)&=\mathbb E[z^{S_N}]\\
    &=\mathbb E[z^{X_1+\cdots+X_N}]\\
    &=\mathbb E\left[ \sum_{n=0}^\infty z^{X_1+\cdots+X_n}1_{(N=n)} \right]\\
    &=\sum_{n=0}^\infty\mathbb E[z^{X_1+\cdots+X_n}1_{(N=n)}]\\
    &=\sum_{n=0}^\infty\mathbb E[z^{X_1+\cdots+X_n}]\mathbb P(N=n)\\
    &=\sum_{n=0}^\infty p(z)^n\mathbb P(N=n)\\
    &=\mathbb E[p(z)^N]\\
    &=q(p(z))
\end{align*}
We can do it another way using conditional expectations.
We have
$$r(z)=\mathbb E[z^{S_N}]=\mathbb E[\mathbb E[z^{X_1+\cdots+X_N}|N]]$$
We have $\mathbb E[z^{X_1+\cdots+X_N}|N=n]=\mathbb E[z^{X_1}]^n=p(z)^n$, therefore $\mathbb E[z^{X_1+\cdots+X_N}|N]=p(z)^N$, hence $r(z)=\mathbb E[p(z)^N]=q(p(z))$.\\
In particular, $\mathbb [S_N]=\lim_{z\to 1^-}r^\prime(z)=q^\prime(p(1-))p^\prime(1-)=\mathbb E[N]\mathbb E[X_1]$.
Similarly $\operatorname{Var}(S_N)=\mathbb E[N]\operatorname{Var}(X_1)+\operatorname{Var}(N)\mathbb E[X_1]^2$.
\subsection{Branching Processes}
The branching processes is a family of stochastic process that is developed by Bienaym\'e, Galton and Watson (therefore also called the Bienaym\'e-Galton-Watson processes).\\
Suppose $(X_n)_{n\ge 0}$ is a random process with $X_0=1$ and $X_n$ equals the number of individials in generation $n$.
The individual at time $0$ produces a random number of offspring with probability distribution $\mathbb P(X_1=k)=g_k,k=0,1,\ldots$, and every individual in the first generation produces an independent number of offspring (which is independent for different individials in generation $n$) with the same distribution.
We call the distribution of $X_1$ the offspring distribution.\\
Continuing this way, we consider a sequence $(Y_{k,n})_{k\ge 1,n\ge 0}$ be i.i.d. as $X_1$, then we can write
$$X_{n+1}=\begin{cases}
    Y_{1,n}+\cdots+Y_{X_n,n}\text{, if $X_n\ge 1$}\\
    0\text{, otherwise}
\end{cases}$$
So we can say $Y_{k,n}$ is the number of offspring of $k^{th}$ individual in generation $n$.
\begin{theorem}
    $\mathbb E[X_n]=(\mathbb E[X_1])^n,\forall n\ge 1$.
\end{theorem}
\begin{proof}
    We proceed by induction.
    $n=1$ is obvious.
    We set $\mu=\mathbb E[X_1]$.
    Suppose $\mathbb E[X_n]=\mu^n$, then for $n+1$, $\mathbb E[X_{n+1}]=\mathbb E[\mathbb E[X_{n+1}|X_n]]$.
    But note that for any $m$, we have
    $$\mathbb E[X_{n+1}|X_n=m]=\mathbb E[Y_{1,n}+\cdots+Y_{m,n}|X_n=m]=m\mu$$
    SO $\mathbb E[X_{n+1}|X_n]=X_n\mu$, therefore $\mathbb E[X_{n+1}]=\mu^n\mu=\mu^{n+1}$.
\end{proof}
\begin{theorem}
    Set $G(z)=\mathbb E[z^{X_1}]$ and $G_n(z)=\mathbb E[z^{X_n}]$, then
    $$G_n(z)=G_{n-1}(G(z))=G^n(z)$$
\end{theorem}
\begin{proof}
    Induction again.
    $n=1$ is from definition.
    Now assuming the previous cases, we have
    $$G_{n+1}(z)=\mathbb E[z^{X_{n+1}}]=\mathbb E[\mathbb E[z^{X_{n+1}}|X_n]]$$
    But now we have $\mathbb E[z^{X_{n+1}}|X_n=m]=\mathbb E[z^{X_1}]^m$ by independence, therefore $\mathbb E[z^{X_{n+1}}|X_n]=G(z)^{X_n}$, hence
    $$\mathbb E[z^{X_{n+1}}]=\mathbb E[G(z)^{X_n}]=G_n(G(z))$$
    As wanted.
\end{proof}
Certainly we would want to calculate the probability of extinction.
Write $q$ as the probability that $X_n=0$ for some $n\ge 1$, and define $q_n$ be the probability of $X_n=0$.
Let $A_n=\{X_n=0\}$, then $A_n\subset A_{n+1}$ and by continuity of probability measure we get
$$q=\mathbb P\left( \bigcup_{n\in\mathbb N}A_n \right)=\lim_{n\to\infty}\mathbb P(A_n)=\lim_{n\to\infty}q_n$$
But we know that for any $n$, $q_n=\mathbb P(X_n=0)=G_n(0)$, therefore $q_{n+1}=G(G_n(0))=G(q_n)$.
So since $q_n$ does tends to a limit and $G$ is continuous, we have $q=G(q)$.\\
Another way to get $q_{n+1}=G(q_n)$ is by observing that we can also condition on $X_1$.
Conditioning on $X_1=m$, we can write $X_{n+1}=X_n^{(1)}+\cdots+X_n^{(m)}$ where $X_n^{(i)}$ are i.i.d. branching processes corresponding to the $i^{th}$ individual in the first generation.
So
\begin{align*}
    q_{n+1}&=\mathbb P(X_{n+1}=0)\\
    &=\sum_m\mathbb P(X_{n+1}=0|X_1=m)\mathbb P(X_1=m)\\
    &=\sum_m\mathbb P(X_n^{(1)}+\cdots+X_n^{(m)}|X_1=m)\mathbb P(X_1=m)\\
    &=\sum_m\mathbb P(X_n^{(1)}=\cdots=X_n^{(m)}=0|X_1=m)\mathbb P(X_1=m)\\
    &=\sum_mq_n^m\mathbb P(X_1=m)\\
    &=G(q_n)
\end{align*}
\begin{theorem}
    The extinction probability $q$ is the smallest non-negative solution to the equation $q=G(q)$.
    Also provided $\mathbb P(X_1=1)\le1$ we have $q\le1\iff \mu\ge1$.
\end{theorem}
\begin{proof}
    Let $t$ be the smallest nonnegative solution to $q=G(q)$, we will show $q=t$.
    Note that $q_0=0\le t$, and if $q_n\le t$, $q_{n+1}=G(q_n)\le G(t)=t$ since $G$ is increasing in $[0,1]$.
    Hence $0\le q\le t$ by Squeeze Theorem, so $q=t$ by minimality of $t$.\\
    Now consider $H(z)=G(z)-z$, then $H^{\prime\prime}(z)=\sum_{r=2}^\infty r(r-1)g_rz^{r-2}$.
    If we assume $g_0+g_1<1$ (trivial otherwise), then $H^{\prime\prime}(z)>0$ for any $z\in (0,1)$, therefore $H^\prime$ is strictly increasing in $[0,1]$, which means (by Rolle's Theorem) that $H$ can have at most one solution other than $1$ in $[0,1]$.\\
    If $H$ has no root in $[0,1)$, then since $H(0)=G(0)\ge 0$, $\forall z\in [0,1],H(z)\ge 0$.
    Now
    $$H^\prime(1-)=\lim_{z\to 1^-}\frac{H(1)-H(z)}{1-z}=\lim_{z\to 1^-}-\frac{H(z)}{1-z}\le 0$$
    So $\mu=G^\prime(1-)=H^\prime(1-)+1\le1$.\\
    Otherwise, let $r$ be a root of $H$ in $[0,1)$, then $G(r)=r$.
    But $G^\prime$ is strictly increasing, so $H^\prime$ must have a root $z$ in $[r,1)$ by Rolle's.
    Hence $G^\prime(z)=1$, but $G^\prime$ is increasing, so $\mu=G^\prime(1-)>G^\prime(z)=1$.
\end{proof}


\documentclass{article}

\usepackage[utf8x]{inputenc}
\usepackage[english]{babel}
\usepackage{hyperref}

\usepackage{amsmath,amsfonts,amsthm,amssymb,enumerate,fullpage,tikz}
\setlength{\parindent}{0pt}
\usepackage{listings}
\usepackage{color}
\newcommand{\mb}[1]{\mathbb{#1}}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\Z}{\mb{Z}}
\newcommand{\N}{\mb{N}}
\newcommand{\R}{\mb{R}}
\newcommand{\C}{\mb{C}}
\newcommand{\cproj}{\C\textup{P}^\infty}
\DeclareMathOperator{\myker}{ker}
\DeclareMathOperator{\im}{im}
\DeclareMathOperator{\Tor}{Tor}
\usetikzlibrary{arrows}

%% Theorem environment declarations (using amsthm):
\newtheorem{theorem}{Theorem}[section]
\newtheorem{axiom}[theorem]{Axiom}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}

\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{convention}[theorem]{Convention}
\newtheorem{examplex}[theorem]{Example}
\newenvironment{example}
  {\pushQED{\qed}\renewcommand{\qedsymbol}{$\triangle$}\examplex}
  {\popQED\endexamplex}
\newtheorem{examples}[theorem]{Examples}
\newtheorem{notation}[theorem]{Notation}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{idea}[theorem]{Idea}

\begin{document}

\title{Applications of the Serre Spectral Sequence}
\author{Floris van Doorn}
\date{November 10, 2015}
\maketitle

\section{Serre Spectral Sequence}

\begin{definition}
A \emph{Spectral Sequence} is a sequence $(E_{p,q}^r,d_r)$ consisting of
\begin{itemize}
\item An $R$-module $E^r_{p,q}$ for $p,q\in \Z$ and $r\geq 0$.
\item Differentials $d_{p,q}^r : E_{p,q}^r\to E_{p-r,q+r-1}^r$ such that $d_r^2=0$
\end{itemize}
where $E^{r+1}$ is defined to be the homology of $(E^r,d^r)$. That is,
$E_{p,q}^{r+1}=\myker(d_{p,q}^r)/\im(d_{p+r,q-r+1}^r)$. The variable $r$ is called the \emph{page}, $p$ the \emph{filtration degree}, $q$ the \emph{complementary degree} and $p+q$ the \emph{total degree}.
\end{definition}

\begin{theorem}[Serre Spectral Sequence]
Let $F \to X \twoheadrightarrow B$ be a fibration such that $B$ is path-connected and $\pi_1(B)$
acts trivially on $H_*(F;G)$. Then
$$H_p(B;H_q(F;G))\ \Longrightarrow\ H_{p+q}(X;G).$$ This means that there is a spectral sequence
$(E_{p,q}^r,d_r)$ where $E_{p,q}^2\simeq H_p(B;H_q(F;G))$ and there is a filtration $0 \subseteq
F_{p+q}^0 \subseteq \cdots \subseteq F_{p+q}^{p+q}=H_{p+q}(X;G)$ such that $E_{p,q}^\infty\simeq
F_{p+q}^p/F_{p+q}^{p-1}$.
\end{theorem}

Note that if $B$ is simply connected, then conditions of the theorem are satisfied.

\subsection{Examples}

\begin{example}
Suppose $X=B\times F$, where $B$ is path-connected, and suppose that $G$ is a field. Then $\pi_1(B)$
acts trivially on $H_*(F;G)$ and we have
\begin{align*}
H_n(X;G) &= \bigoplus_{p+q=n}H_p(B;G)\otimes H_q(F;G)&&\text{(K\"unneth formula)}\\
&=\bigoplus_{p+q=n}H_p(B;H_q(F;G))&&\text{(Univ. Coeff. Th. for homology)}\\
\end{align*}
This means that all entries in the second page survive until page infinity. The other extreme is if
$X$ is contractible, where almost nothing will survive, as we will see in the next examples.
\end{example}

In the next example, we will use that $S^1=K(\Z,1)$ and $\cproj=K(\Z,2)$.

\begin{example}
Consider the path space fibration of $B=\cproj$, that is $\Omega B\to PB \twoheadrightarrow B$ and
note that $S^1=\Omega B$. Since $B$ is simply connected, we can apply the Serre Spectral Sequence
with coefficients in $\Z$. We know that $E_{p,q}^2\simeq H_p(B;H_q(S^1))$ and $H_q(S^1)=0$ for $q>1$
and $\Z$ for $q=0,1$. This means that the page $E^2$ looks like this.

\begin{center}
\begin{tikzpicture}[>=stealth',auto,node distance=1.5cm,
  main node/.style={font=\sffamily\bfseries},text height=1.5ex]
  \node[main node]   (-11)at (-1.2,0.8)   {\mbox{}};
  \node[main node]   (00) at (0,0)      {$\Z$};
  \node[main node]   (01) at (0,1)      {$\Z$};
  \node[main node]   (10) at (1.5,0)    {$H_1(B)$};
  \node[main node]   (11) at (1.5,1)    {$H_1(B)$};
  \node[main node]   (20) at (3,0)      {$H_2(B)$};
  \node[main node]   (21) at (3,1)      {$H_2(B)$};
  \node[main node]   (30) at (4.5,0)    {$H_3(B)$};
  \node[main node]   (31) at (4.5,1)    {$H_3(B)$};
  \node[main node]   (40) at (6,0)      {$H_4(B)$};
  \node[main node]   (41) at (6,1)      {$H_4(B)$};
  \node[main node]   (50) at (7.5,0)    {$\cdots$};
  \node[main node]   (51) at (7.5,1)    {$\cdots$};
  \node at (-1.2,1) {1};
  \node at (-1.2,0) {0};
  \node at (0,-1)   {0};
  \node at (1.5,-1) {1};
  \node at (3,-1)   {2};
  \node at (4.5,-1) {3};
  \node at (6,-1)   {4};
  \node at (7.3,-0.7)   {$p$};
  \node at (-0.8,1.5)   {$q$};
  \draw (-0.5,-0.5) -- (8,-0.5);
  \draw (-0.5,-0.5) -- (-0.5,1.5);
  \path[every node/.style={font=\sffamily\small}]
%    (10) edge [->] (-11)
    (20) edge [->] (01)
    (30) edge [->] (11)
    (40) edge [->] (21)
    (50) edge [->] (31);
\end{tikzpicture}
\end{center}

Moreover, we have $E_{p,q}^\infty=\Z$ for $p=q=0$ and $0$ otherwise. From this we can conclude that
$$H_i(\cproj,\Z)=\begin{cases} \Z & \text{if $i$ even} \\ 0 & \text{if $i$ odd.}\end{cases}$$
\end{example}

\begin{example}
In this example we will compute the homology groups of the loop space of the sphere, $\Omega S^n$
for $n\geq 2$. We use the fibration $\Omega S^n \to PS^n \twoheadrightarrow S^n$ and we can apply the Serre
Spectral Sequence, since $S^n$ is simply connected. Now $H_p(S^n;G)=G$ for $p=0,n$ and $0$
otherwise. This means that only the $0$ and the $n$ column can be nonzero.

\begin{center}
\begin{tikzpicture}[>=stealth',auto,node distance=1.5cm,
  main node/.style={font=\sffamily\bfseries},text height=1.5ex]
  \node[main node]   (00) at (0,0)      {$\Z$};
  \node[main node]   (10) at (2.5,0)    {$\Z$};
  \node[main node]   (01) at (0,1)      {$H_{n-1}(\Omega S^n)$};
  \node[main node]   (11) at (2.5,1)    {$H_{n-1}(\Omega S^n)$};
  \node[main node]   (02) at (0,2)      {$H_{2n-2}(\Omega S^n)$};
  \node[main node]   (12) at (2.5,2)    {$H_{2n-2}(\Omega S^n)$};
  \node[main node]   (03) at (0,3)      {$H_{3n-3}(\Omega S^n)$};
  \node[main node]   (13) at (2.5,3)    {$H_{3n-3}(\Omega S^n)$};
  \node[main node]   (04) at (0,4)      {$\vdots$};
  \node[main node]   (14) at (2.5,4)    {$\vdots$};
  \node at (-1.9,0) {0};
  \node at (-1.9,1) {$n-1$};
  \node at (-1.9,2) {$2n-2$};
  \node at (-1.9,3) {$3n-3$};
  \node at (0,-1)   {0};
  \node at (2.5,-1) {$n$};
  \node at (2.8,-0.7)   {$p$};
  \node at (-1.5,4.5)   {$q$};
  \draw (-1.2,-0.5) -- (3,-0.5);
  \draw (-1.2,-0.5) -- (-1.2,5);
  \path[every node/.style={font=\sffamily\small}]
    (10) edge [->] (01)
    (11) edge [->] (02)
    (12) edge [->] (03)
    (13) edge [->] (04);
\end{tikzpicture}
\end{center}
After some reasoning, we get that $H_i(\Omega S^n,\Z)=\begin{cases} \Z & \text{for $n-1\mid i$}\\ 0 & \text{otherwise.}\end{cases}$
\end{example}

\section{Serre Class Theorem}

\begin{definition}
We say that a space $X$ is \emph{abelian} if the action of $\pi_1(X)$ on $\pi_n(X)$ is trivial for all $n\geq 1$.
\end{definition}
Note that every simply connected space is abelian.
\begin{definition}
  A \emph{Serre Class} is a class $\mc C$ of abelian groups containing the trivial group such that for every SES $0\to A\to B\to C\to 0$ we have $B\in\mc C$ iff $A,C\in \mc C$. In this document I call a Serre class \emph{nice} if for every $A,B\in\mc C$ also $A\otimes B$ and $\Tor(A,B)$ are in $\mc C$. (this name is made up by me)
\end{definition}
\begin{lemma}
  The following classes are nice Serre classes.
\begin{itemize}
\item $\mc {FG}$, the class of finitely generated abelian groups
\item $\mc{T}_P$ for some set $P$ of primes. This is the class of torsion abelian groups whose
  elements have orders divisible only by primes in $P$.
\item $\mc{F}_P$, the finite groups in $\mc{T}_P$.
\end{itemize}
\end{lemma}
Note that $P$ is the set of all primes $\mc{T}_P$ becomes the class of all torsion abelian groups
and $\mc{F}_P$ becomes the class of all finite abelian groups.

\begin{theorem}
  Let $X$ be a path-connected and abelian space, and let $\mc C$ be a nice Serre class. Then
  $$\forall(n>0)(\pi_n(X)\in\mc{C})\quad \longleftrightarrow\quad \forall(n>0)(H_n(X)\in\mc{C})$$
\end{theorem}

\begin{corollary}
The homotopy groups of a finite simply connected CW-complex are finitely generated. In particular, the homotopy groups of spheres are finitely generated.
\end{corollary}

Recall the following definition and theorem.
\begin{definition}
The \emph{Hurewicz homomorphism} is the homomorphism $h : \pi_n(X) \to H_n(X)$ defined by $h([f])=f_*(\gamma)$, where $\gamma$ is a generator of $H_n(S^n)\simeq\Z$.
\end{definition}

\begin{theorem}[Hurewicz]
Let $n\geq2$ and $X$ a $(n-1)$-connected space. Then $\widetilde H_i(X)=0$ for $i<n$ and the Hurewicz homomorphism $h : \pi_n(X) \to H_n(X)$ is an isomorphism.
\end{theorem}

We will now generalize this theorem.
\begin{theorem}
  Let $X$ be a path-connected and abelian space, and let $\mc C$ be a nice Serre class. Suppose that $\pi_i(X)\in\mc C$ for $i<n$. Then the Hurewicz homomorphism $h:\pi_n(X)\to H_n(X)$ is an isomorphism modulo $\mc{C}$, which means that its kernel and cokernel are in $\mc{C}$.
\end{theorem}

\section{Cohomology Serre Spectral Sequence}

There is a Serre Spectral Sequence for cohomology which is completely analogous. Of course, the
arrows in these spectral sequences are reversed.

\begin{theorem}
Let $F \to X \twoheadrightarrow B$ be a fibration such that $B$ is path-connected and $\pi_1(B)$
acts trivially on $H_*(F;G)$. Then
$$H^p(B;H^q(F;G))\ \Longrightarrow\ H^{p+q}(X;G).$$
\end{theorem}

However, we can now also use the cup product if the underlying group is a ring.

\begin{theorem}
  There is a bilinear product $E_r^{p,q}\times E_r^{s,t}\to E_r^{p+s,q+t}$ for $1\leq r\leq \infty$
  (written as concatenation) satisfying
  \begin{itemize}
  \item For $x\in E^{p,q}_r$ we define $|x|=p+q$. Then we have
    $d_r(xy)=(d_rx)y+(-1)^{|x|}x(d_ry)$. This means that the product on level $r$ induces a product
    on level $r+1$, which coincides with the given bilinear product at level $r+1$. The product in
    $E_\infty$ is induces from the products at the finite levels.
  \item At page 2, the product is up to a factor $(-1)^{qs}$ induced from the cup product under the
    correspondence $E_2^{p,q}\simeq H^p(B;H^q(F;R))$. In the RHS the cup product sends $(\phi,\psi)$
    to $\phi\smallsmile\psi$ and coefficients are also multiplied via the cup product.
  \item The cup product in $H^*(X;R)$ restricts to maps on the filtrations $F_p^m\times F_s^n\to
    F_{p+s}^{m+n}$ which induce quotient maps $F_p^m/F_{p+1}^m\times F_s^n/F_{s+1}^n\to
    F_{p+s}^{m+n}/F_{p+s+1}^{m+n}$. Under the correspondence $E^{p,q}_\infty\simeq
    F^{p+q}_p/F^{p+q}_{p+1}$ the product in the LHS corresponds to these maps in the RHS.
  \item $ab=(-1)^{|a||b|}ba$ and $d(x^n)=nx^{n-1}dx$ if $|x|$ is even.
  \end{itemize}
\end{theorem}

In particular, if we apply the cup $E_2^{p,0}\times E_2^{0,q}\to E_2^{p,q}$ to a pair of units, we
get a unit.

\subsection{Examples}

\begin{example}
Consider the path space fibration of $B=\cproj$ again. By the universal coefficient theorem, we know
that the cohomology groups are the same as the homology groups. Now let's compute the cup product
structure. Let $x_{2i}$ be the generator of $E_2^{2i,0}$ and $a$ the generator of $E_2^{0,1}$. Then
$x_0$ is the unit for multiplication, and $ax_{2i}$ are generators of $E_2^{2i,1}$.
\begin{center}
\begin{tikzpicture}[>=stealth',auto,node distance=1.5cm,
  main node/.style={font=\sffamily\bfseries},text height=1.5ex]
  \node[main node]   (-11)at (-1.2,0.8)   {\mbox{}};
  \node[main node]   (00) at (0,0)      {$\Z x_0$};
  \node[main node]   (01) at (0,1)      {$\Z a$};
  \node[main node]   (10) at (1.5,0)    {$0$};
  \node[main node]   (11) at (1.5,1)    {$0$};
  \node[main node]   (20) at (3,0)      {$\Z x_2$};
  \node[main node]   (21) at (3,1)      {$\Z ax_2$};
  \node[main node]   (30) at (4.5,0)    {$0$};
  \node[main node]   (31) at (4.5,1)    {$0$};
  \node[main node]   (40) at (6,0)      {$\Z x_4$};
  \node[main node]   (41) at (6,1)      {$\Z ax_4$};
  \node[main node]   (50) at (7.5,0)    {$\cdots$};
  \node[main node]   (51) at (7.5,1)    {$\cdots$};
  \node at (-1.2,1) {1};
  \node at (-1.2,0) {0};
  \node at (0,-1)   {0};
  \node at (1.5,-1) {1};
  \node at (3,-1)   {2};
  \node at (4.5,-1) {3};
  \node at (6,-1)   {4};
  \node at (7.3,-0.7)   {$p$};
  \node at (-0.8,1.5)   {$q$};
  \draw (-0.5,-0.5) -- (8,-0.5);
  \draw (-0.5,-0.5) -- (-0.5,1.5);
  \path[every node/.style={font=\sffamily\small}]
%    (10) edge [->] (-11)
    (01) edge [->] (20)
    (21) edge [->] (40);
\end{tikzpicture}
\end{center}
All arrows are isomorphisms. We may assume that $d_2a=x_2$. Then we compute $d_2(ax_{2i})=x_2x_{2i}$ so we may assume that $x_2x_{2i}=x_{2i+2}$. This gives $x_{2i}=x_2^i$. Hence $H^*(\cproj,\Z)\simeq \Z[x_2]$.
\end{example}

\begin{example}
We will compute the cup product structure of $H^*(\Omega S^n;\Z)$ using the path space fibration of $S^n$ for $n\geq2$. The additive structure is the same as for homology, and we can name the generators as in the figure, where $a_0=1$.
\begin{center}
\begin{tikzpicture}[>=stealth',auto,node distance=1.5cm,
  main node/.style={font=\sffamily\bfseries},text height=1.5ex]
  \node[main node]   (00) at (0,0)      {$\Z a_0$};
  \node[main node]   (10) at (1.5,0)    {$\Z a_0x$};
  \node[main node]   (01) at (0,1)      {$\Z a_1$};
  \node[main node]   (11) at (1.5,1)    {$\Z a_1x$};
  \node[main node]   (02) at (0,2)      {$\Z a_2$};
  \node[main node]   (12) at (1.5,2)    {$\Z a_2x$};
  \node[main node]   (03) at (0,3)      {$\Z a_3$};
  \node[main node]   (13) at (1.5,3)    {$\Z a_3x$};
  \node[main node]   (04) at (0,4)      {$\vdots$};
  \node[main node]   (14) at (1.5,4)    {$\vdots$};
  \node at (-1.4,0) {0};
  \node at (-1.4,1) {$n-1$};
  \node at (-1.4,2) {$2n-2$};
  \node at (-1.4,3) {$3n-3$};
  \node at (0,-1)   {0};
  \node at (1.5,-1) {$n$};
  \node at (1.8,-0.7)   {$p$};
  \node at (-1,4.5)   {$q$};
  \draw (-0.7,-0.5) -- (2,-0.5);
  \draw (-0.7,-0.5) -- (-0.7,5);
  \path[every node/.style={font=\sffamily\small}]
    (01) edge [->] (10)
    (02) edge [->] (11)
    (03) edge [->] (12);
\end{tikzpicture}
\end{center}
We may assume that $d(a_{k+1})=a_kx$ and note that $a_kx=xa_k$.

We distinguish two cases.

\emph{If $n$ is odd} we compute by induction to $i+j$ that $a_ia_j={i+j \choose i}a_{i+j}$. Hence $H^*(\Omega S^n,\Z)\simeq \Gamma_{\Z}[a_1]$, where the \emph{divided polynomial algebra} $\Gamma_R[\alpha]$ is the quotient of the free $R$-algebra $R[\alpha_1,\alpha_2,\ldots]$ by the relations $\alpha_i\alpha_j={i+j \choose i}\alpha_{i+j}$.

\emph{If $n$ is even}, then we compute $a_1^2=0$ and by induction on $k$ we compute $a_1a_{2k}=a_{2k+1}$ and $a_1a_{2k+1}=0$ and $a_2^k=k!a_{2k}$.

Now $H^*(\Omega S^n,\Z)\simeq \Lambda_\Z[a_1]\otimes \Gamma_\Z[a_2]$ where the \emph{exterior algebra} $\Lambda_R[\alpha_1,\alpha_2,\ldots]$ is the free $R$-module with basis finite products $\alpha_{i_1}\cdots\alpha_{i_k}$ for $i_1<\cdots<i_k$ where multiplication is defined as $\alpha_i\alpha_j=-\alpha_j\alpha_i$ and $\alpha_i^2=0$.
\end{example}

For the next example, we use the following.

\begin{remark}
We can factor any map $f:A\to B$ as a homotopy equivalence followed by a fibration: $A \stackrel{\sim}{\to} E_f \to B$. Here $E_f=\{(a,\gamma)\in A\times B^I\mid \gamma(0)=f(a)\}$. In HoTT we would have $E_f=\Sigma(x:A)\Sigma(y:B), f(a) = y$.
\end{remark}
\begin{example}
In this example we will proof that the $p$-torsion subgroups of $\pi_i(S^3)$ is $0$ for $i<2p$ and $\Z_p$ for $i=2p$. Start with a map $S^3\to K(\Z,3)$ inducing an isomorphism on $\pi_3$. Turn this into a fibration with fiber $F$. By the LES of homotopy groups of a fibration we get that $F$ is 3-connected and $\pi_i(F)=\pi_i(S^3)$ for $i>3$. Now convert the map $F\to S^3$ into a fibration. By the LES we see that the fiber is $K(\Z,2)=\cproj$.

\begin{center}
\begin{tikzpicture}[auto,node distance=1.5cm,
  main node/.style={font=\sffamily\bfseries},text height=1.5ex]
  \node[main node]   (S3) at (0,0)      {$S^3$};
  \node[main node]   (Z)  [above of=S3] {$Z$};
  \node[main node]   (K3) [right of=Z]  {$K(\Z,3)$};
  \node[main node]   (F)  [left  of=Z]  {$F$};
  \node[main node]   (X)  [left  of=S3] {$X$};
  \node              (K2) [left  of=X]  {$\cproj$};
  \path[every node/.style={font=\sffamily\small}]
    (F) edge [->] (Z)
    (F) edge [->] (S3)
    (F) edge [->] node [right] {$\wr$} (X)
    (S3) edge [->] node [right] {$\wr$} (Z)
    (Z) edge [->>] (K3)
    (K2) edge [->] (X)
    (X) edge [->>] (S3)
    (S3) edge [->] node [below right] {$f$} (K3);
\end{tikzpicture}
\end{center}

We now use the Serre Spectral Sequence of this last fibration. We know the homology groups of $S^3$ and $\cproj$, so we know the second page looks like this. Here the arrows are \emph{not} all isomorphisms.

\begin{center}
\begin{tikzpicture}[>=stealth',auto,node distance=1.5cm,
  text height=1.5ex]
  \node (00) at (0,0)      {$\Z 1$};
  \node (10) at (1.5,0)    {$\Z x$};
  \node (01) at (0,1)      {$\Z a$};
  \node (11) at (1.5,1)    {$\Z ax$};
  \node (02) at (0,2)      {$\Z a^2$};
  \node (12) at (1.5,2)    {$\Z a^2x$};
  \node (03) at (0,3)      {$\Z a^3$};
  \node (13) at (1.5,3)    {$\Z a^3x$};
  \node (04) at (0,4)      {$\vdots$};
  \node (14) at (1.5,4)    {$\vdots$};
  \node at (-1.4,0) {0};
  \node at (-1.4,1) {$2$};
  \node at (-1.4,2) {$4$};
  \node at (-1.4,3) {$6$};
  \node at (0,-1)   {0};
  \node at (1.5,-1) {$3$};
  \node at (1.8,-0.7)   {$p$};
  \node at (-1,4.5)   {$q$};
  \draw (-0.7,-0.5) -- (2,-0.5);
  \draw (-0.7,-0.5) -- (-0.7,5);
  \path[every node/.style={font=\sffamily\small}]
    (01) edge [->] (10)
    (02) edge [->] (11)
    (03) edge [->] (12);
\end{tikzpicture}
\end{center}
\end{example}

Since $X$ is 3-connected, $d:\Z a \to \Z x$ must be an iso, so we may assume $da=x$. Then $d(a^n)=na^{n-1}x$. Now we know what groups survive until $E_\infty$, to compute
\begin{align*}
H^i(X;\Z)&=\begin{cases} \Z_n & \text{if $i=2n+1$} \\ 0 & \text{if $i=2n$,} \end{cases} && \text{hence}&
H_i(X;\Z)&=\begin{cases} \Z_n & \text{if $i=2n>0$} \\ 0 & \text{if $i=2n-1$.} \end{cases}
\end{align*}
The Hurewicz Theorem modulo $\mc C$ now implies that the first $p$-torsion in $\pi_*(X)$, hence also in $\pi_*(S^3)$ is a $\Z_p$ in $\pi_{2p}$. For $p=2$ we get the stronger result that $\pi_4(S^3)=\Z^2$, hence also $\pi_{n+1}(S^n)=\Z_2$ for $n\geq3$.

\end{document}


\documentclass[12pt, letterpaper]{report}
\usepackage[margin=1in]{geometry}
\usepackage[utf8]{inputenc}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{float}
\usepackage{subfig}
\graphicspath{ {./img/} }
\setlength\parindent{0pt}
\renewcommand{\thesection}{\Roman{section}.}
\renewcommand{\thesubsection}{\alph{subsection}.}

\title{CS1675 - Assignment 9}
\author{Zachary M. Mattis}

\begin{document}

\maketitle

\section{Problem 1 - K-Means Clustering}

S = \{(0,0)(0,5)(7,0)(6,7)\}

\[ d(\vec{p},\vec{q}) = \sqrt{\sum_{i=1}^{n} (q_i-p_i)^2} \]

% A
\subsection{Start: $\mu_1=(0,0), \mu_2=(7,0)$}

After Convergence w/ Euclidean Algorithm:
\\
\begin{table}[H]
	\centering
	\begin{tabular}{ l c l }
		$\mu_1=(0,2.5)$ & \qquad &  $\mu_2=(6.5,3.5)$ \\ \\
		$S_1 = \{(0,0),(0,5)\}$ & \qquad & $S_2 = \{(7,0),(6,7)\}$
	\end{tabular}
\end{table}

% B
\subsection{Start: $\mu_1=(3,3), \mu_2=(7,0)$}

After Convergence w/ Euclidean Algorithm:
\\
\begin{table}[H]
	\centering
	\begin{tabular}{ l c l }
		$\mu_1=(2,4)$ & \qquad &  $\mu_2=(7,0)$ \\ \\
		$S_1 = \{(0,0),(0,5),(6,7)\}$ & \qquad & $S_2 = \{(7,0)\}$
	\end{tabular}
\end{table}

\pagebreak
\section{Problem 2 - K-Means Clustering Experiments}

% A
\subsection{K = 3}

\begin{table}[H]
	\centering
	\begin{tabular}{ |r|r|r|r| }
		\hline
		& \textbf{$S_1$} & \textbf{$S_2$} & \textbf{$S_3$} \\
		\hline
		\textbf{$\mu$} & (3.94, 4.04) & (2.94, -4.97) & (0.86, 2.03) \\
		\hline
		\textbf{Total} & 66 & 36 & 98 \\
		\hline
		\textbf{Plot Color} & green & red & blue \\
		\hline
	\end{tabular}
	\caption{clustering\_data.txt}
\end{table}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\columnwidth]{cluster_k3.png}
	\caption{clustering\_data.txt}
\end{figure}

\pagebreak
% B
\subsection{K = 4}

NOTE: Given the clustering of the data, running the algorithm on this dataset often yields different clusterings due to the starting points for the means. This concept is illustrated above from the differing resulting sets of Problem I, Parts A and B.
\\
\begin{table}[H]
	\centering
	\begin{tabular}{ |r|r|r|r|r| }
		\hline
		& \textbf{$S_1$} & \textbf{$S_2$} & \textbf{$S_3$} & \textbf{$S_4$} \\
		\hline
		\textbf{$\mu$} & (4.04, 4.03) & (0.68, 2.73) & (2.94, -4.97) & (1.23, 1.04) \\
		\hline
		\textbf{Total} & 63 & 61 & 36 & 40  \\
		\hline
		\textbf{Plot Color} & red & green & blue & cyan \\
		\hline
	\end{tabular}
	\caption{clustering\_data.txt}
\end{table}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\columnwidth]{cluster_k4_alpha.png}
	\caption{clustering\_data.txt}
\end{figure}

% C
\subsection{Different Starting Means}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\columnwidth]{cluster_k4_delta.png}
	\caption{clustering\_data.txt}
\end{figure}

% D
\subsection{K-Means Optimizations}

Since the K-Means algorithm optimizes a distance metric, such as Euclidean Distance, it makes sense to compare the effectiveness of two different clustering via this measurement. For instance, the total distance between n data points for each given $k^{\text{th}}$ set is given by eq. \ref{eq:EuclideanDistance}. By summing individual distance for each set $S \in k$, the overall effectiveness of each algorithm instance can be compared, as seen in eq. \ref{eq:KMean}. Whichever algorithm instance produces the minimum total distance can be viewed as most effective.
\\ \\
\begin{equation}
\label{eq:EuclideanDistance}
d(\vec{x},\mu_k) = \sqrt{\sum_{i=1}^{n} (\mu_k-x_i)^2}
\end{equation}
\\
\begin{equation}
\label{eq:KMean}
\text{total distance}  = \sum_{j=1}^{k} d(\vec{x},\mu_k)
\end{equation}

% E
\subsection{Random K-Means Initialization}

\begin{table}[H]
	\centering
	\begin{tabular}{ |r|r|r|r| }
		\hline
		& \textbf{Run 1} & \textbf{Run 2} & \textbf{Run 3} \\
		\hline
		\textbf{Cluster Sizes} & 40,36,63,61 & 38,36,63,63 & 63,52,36,49 \\
		\hline
		\textbf{Total Distance} & 281.9109 & 281.9388 & 282.4565 \\
		\hline
	\end{tabular}
	\caption{Random Initialization Analysis}
\end{table}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\columnwidth]{cluster_k4_opt.png}
	\caption{Optimal K=4}
\end{figure}

\section{Hierarchical Clustering}

% A
\subsection{Hierarchy of Clusters}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.6\columnwidth]{dendrogram.png}
	\caption{Dendrogram}
\end{figure}

% B
\subsection{Cluster Tree}

The clusters for both my K-Means algorithm implementation and the built-in Matlab \mbox{linkage()} and \mbox{cluster()} functions produced the same clustering of data. This makes sense, as each technique optimizes the a distance metric to produce the best result set.

\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\columnwidth]{cluster_hierarchical.png}
	\caption{Cluster Tree}
\end{figure}

\end{document}


In Nyx a stochastic force field can be applied. To make sure this option is chosen correctly, we must always set \\

\noindent {\bf USE\_FORCING = TRUE} \\

\noindent in the GNUmakefile and \\

\noindent {\bf nyx.do\_forcing} = 1  \\

\noindent in the inputs file.  \\

The external forcing term in the momentum equation~(\ref{eq:momt}) is then given by
\begin{equation}
  {\bf S}_{\rho \Ub} = \rho_b \fb
\end{equation}
where the acceleration field $\mathbf{f}(\mathbf{x},t)$ is computed as inverse Fourier transform of the forcing spectrum $\widehat{\mathbf{f}}(\mathbf{k},t$). The time evolution of each wave mode is given by an Ornstein-Uhlenbeck process (see \cite{SchmHille06,Schmidt14} for details). Since the real space forcing acts on large scales $L$, non-zero modes are confined to a narrow window of small wave numbers with a prescribed shape (the forcing profile). The resulting flow reaches a statistically stationary and isotropic state with a root-mean-square velocity of the order $V=L/T$, where the integral time scale $T$ (also known as large-eddy turn-over time) is usually set equal to the autocorrelation time of the forcing. It is possible to vary the force field from solenoidal (divergence-free) if the weight parameter $\zeta=1$ to dilational (rotation-free) if $\zeta=0$.

To maintain a nearly constant root-mean-square Mach number, a simple model for radiative heating and cooling around a given equilibrium temperature $T_0$ is applied in the energy equation~(\ref{eq:energy}):
\begin{equation}
  S_{\rho E} = S_{\rho e} + \Ub \cdot {\bf S}_{\rho \Ub} = -\frac{\alpha k_{\rm B}(T-T_0)}{\mu m_{\rm H}(\gamma-1)} + \rho_b\Ub\cdot\fb
\end{equation}
The parameters $T_0$ and $\alpha$ correspond to temp0 and alpha, respectively, in the probin file (along with rho0 for the mean density, which is unity by default). While the gas is adiabatic for $\alpha=0$, it becomes
nearly isothermal if the cooling time scale given by $1/\alpha$ is chosen sufficiently short compared to $T$. For performance reasons, a constant composition (corresponding to constant molecular weight $\mu$) is assumed.

\section{List of Parameters}

\begin{table*}[h]
\begin{scriptsize}
\begin{tabular}{|l|l|l|l|} \hline
Parameter & Definition & Acceptable Values & Default \\
\hline
{\bf forcing.seed} & seed of the random number generator & Integer $>0$ & 27011974 \\
{\bf forcing.profile} & shape of forcing spectrum & 1 (plane), 2 (band), 3 (parabolic) & 3\\
{\bf forcing.alpha} & ratio of domain size $X$ to integral length $L=X/\alpha$ & Integer $>0$ & 2 2 2\\
{\bf forcing.band\_width} & band width of the forcing spectrum relative to alpha & Real $\ge 0$ and $\le 1$ & 1.0 1.0 1.0\\
{\bf forcing.intgr\_vel} & characteristic velocity $V$ & Real $> 0$ & must be set\\
{\bf forcing.auto\_corrl} & autocorrelation time in units of $T=L/V$ & Real $> 0$ & 1.0 1.0 1.0\\
{\bf forcing.soln\_weight} & weight $\zeta$ of solenoidal relative to dilatational modes & Real $\ge 0$ and $\le 1$ & 1.0\\
\hline
\end{tabular}
\label{Table:Geometry}
\end{scriptsize}
\end{table*}

Triples for forcing.alpha, forcing.band\_width, forcing.intgr\_vel, and forcing.auto\_corrl correspond to the three spatial dimensions.