\chapter{Matters related to tensor product}

These consideration on (possibly infinite) indexed families of
join-semilattices is based on \cite{nforum-todd-tensor} (for the finite case).

Let $\mathfrak{A}$ be an indexed family of join-semilattices with least
elements. Let $T$ also be a join-semilattice.

Let $F (X)$ mean free join-semilattice for a set $X$.

\begin{defn}
$\mathbf{SepJoin}(\prod\mathfrak{A},T)$ is the set of maps from $\prod\mathfrak{A}$
to~$T$, preserving joins in every argument $i\in\dom\mathfrak{A}$.
\end{defn}

\begin{obvious}
The set of free join-semilattices $F (X)$ is order-isomorphic to the set of
subsets $X$ of a ``universal'' set $\mho$.{\hspace*{\fill}}{\medskip}
\end{obvious}

Let $i : \prod \mathfrak{A} \rightarrow F \left( \prod \mathfrak{A} \right)$
be the universal embedding.

Let $\sim$ be defined as the smallest equivalence relation on $F \left( \prod
\mathfrak{A} \right)$ that for every $k \in \dom \mathfrak{A}$, $L \in
\prod_{i \in (\dom \mathfrak{A}) \setminus \{ k \}} \mathfrak{A}_i$:
\begin{enumerate}
  \item $i (L \cup \{ (k , g \sqcup h) \}) \sim i (L \cup \{ (k , g) \})
  \sqcup i (L \cup \{ (k , h) \})$;

  \item $\bot \sim i (L \cup \{ (k , \bot) \})$;

  \item $x \sim y \wedge x' \sim y' \Rightarrow x \sqcup x' \sim y \sqcup y'$
  for all $x, y, x', y' \in F \left( \prod \mathfrak{A} \right)$.
\end{enumerate}

\begin{obvious}
Some function $h : X \rightarrow Y$ induces a well defined map $\psi : X / E
\rightarrow Y$ on equivalence classes, if $E \subseteq F$ where $x \mathrel{F}
y \Leftrightarrow h x = h y$.{\hspace*{\fill}}{\medskip}
\end{obvious}

\begin{lem}
  The set of join-homomorphisms $\psi : F \left( \prod \mathfrak{A} \right) / \sim
  \rightarrow T$ is isomorphic to the set of maps $\phi :
  \prod \mathfrak{A} \rightarrow T$ preserving finite joins in separate arguments.
\end{lem}

\begin{proof}
  The quotient map $q : F \left( \prod \mathfrak{A} \right) \rightarrow F
  \left( \prod \mathfrak{A} \right) / \sim$ which takes an element $x$ to its
  equivalence class $[x]$ map is well defined because
  \[ x \sim y \wedge x' \sim y' \Rightarrow x \sqcup x' \sim y \sqcup y' . \]
  The map $q$ \ preserves join. $F \left( \prod \mathfrak{A} \right) / \sim$
  is associative, commutative, and idempotent since it is so on $F \left(
  \prod \mathfrak{A} \right)$ and thus is a join-semilattice.

  Let join-preserving map $\psi : F \left( \prod \mathfrak{A} \right) / \sim
  \rightarrow T$. It is easy to show that $\psi \circ q \circ i$ preserves
  joins in separate arguments.

  Let now $\phi : \prod \mathfrak{A} \rightarrow T$ preserves joins in
  separate arguments. There is a unique join-preserving map $\tilde{\phi} : F
  \left( \prod \mathfrak{A} \right) \rightarrow T$ such that $\tilde{\phi}
  \circ i = \phi$. We must show that this induces a well-defined
  join-preserving map $\psi : F \left( \prod \mathfrak{A} \right) / \sim
  \rightarrow T$ such that $\psi (q (x)) = \tilde{\phi} (x)$ for all $x \in F
  \left( \prod \mathfrak{A} \right)$ (clearly at most one function $\psi$ can
  satisfy this equation since $q$ is surjective). This will show that $\psi$
  bijectively correspond to $\tilde{\phi}$ and thus bijectively correspond to
  $\phi$. (This will finish the proof as that this bijection is monotone is
  obvious.)

  Using the ``obvious'' above, it's enough (taking into account that $\sim$ is
  the minimal equivalence relation subject to the above formulas) to prove
  that:
  \begin{enumerate}
    \item $\tilde{\phi} (i (L \cup \{ (k , g \sqcup h) \})) = \tilde{\phi} (i
    (L \cup \{ (k , g) \}) \sqcup i (L \cup \{ (k , h) \}))$;

    \item $\tilde{\phi} (\bot) = \tilde{\phi} (i (L \cup \{ (k , \bot) \}))$;

    \item $\tilde{\phi} (x) = \tilde{\phi} (y) \wedge \tilde{\phi} (x') =
    \tilde{\phi} (y') \Rightarrow \tilde{\phi} (x \sqcup x') = \tilde{\phi} (y
    \sqcup y')$
  \end{enumerate}
  The first easily follows from $\tilde{\phi} \circ i = \phi$ and the fact
  that $\tilde{\phi}$ preserves binary joins.

  The second easily follows from $\tilde{\phi} \circ i = \phi$ and that $\phi$
  preserves $\bot$.

  The third follows from the fact that $\tilde{\phi}$ preserves joins.
\end{proof}

\begin{cor}
  The poset of prestaroids
  $\mathsf{preStrd} (\mathfrak{A})$ is isomorphic to an ideal
  (on a join-semilattice), provided that $\mathfrak{A}$ is an indexed family
  of join-semilattices.
\end{cor}

\begin{proof}
  $\mathsf{preStrd} (\mathfrak{A}) \cong \mathbf{SepJoin} (\mathfrak{A}, 2)
  \cong F \left( \prod \mathfrak{A} \right) / \sim \rightarrow 2 \cong
  \mathfrak{I} \left( F \left( \prod \mathfrak{A} \right) / \sim \right)$.
\end{proof}

\fxwarning{Check below (especially posets vs dual posets) for errors.}

\begin{cor}
$\mathsf{preStrd}$ is a complete lattice.
\end{cor}

\begin{proof}
Corollary~\bookref{filt-is-complete}.
\end{proof}

\begin{cor}
$\mathsf{preStrd}$ is a filtered filtrator.
\end{cor}

\begin{proof}
Theorem~\bookref{semifilt-joinclosed}.
\end{proof}

\fxnote{Try to prove that $\mathsf{preStrd}$ is atomic and moreover atomistic (under certain conditions). Other properties?}


% This LaTeX was auto-generated from an M-file by MATLAB.
% To make changes, update the M-file and republish this document.

%%% \documentclass{article}
%%% \usepackage{graphicx}
%%% \usepackage{color}

%%% \sloppy
%%% \definecolor{lightgray}{gray}{0.5}
\setlength{\parindent}{0pt}

%%% \begin{document}

\subsection*{Simple example of the QWTB use}

\begin{par}
Sample data are simulated. QWTB is used to apply two different algorithms on the same data. Uncertainty of the results is calculated by means of Monte Carlo Method.
\end{par} \vspace{1em}

\subsubsection*{Contents}

\begin{itemize}
\setlength{\itemsep}{-1ex}
   \item Generate sample data
   \item Analyzing data
   \item Uncertainties
\end{itemize}

\subsubsection*{Generate sample data}

\begin{par}
Two quantities are prepared: \lstinline{t} and \lstinline{y}, representing 0.5 second of sinus waveform of nominal frequency 1 kHz, nominal amplitude 1 V and nominal phase 1 rad, sampled at sampling frequency \lstinline{fsnom} 10 kHz.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI = [];
Anom = 1; fnom = 1e3; phnom = 1; fsnom = 1e4;
DI.t.v = [0:1/fsnom:0.5];
DI.y.v = Anom*sin(2*pi*fnom*DI.t.v + phnom);
\end{lstlisting}
\begin{par}
Add noise of standard deviation 1 mV:
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI.y.v = DI.y.v + 1e-3.*randn(size(DI.y.v));
\end{lstlisting}

\subsubsection*{Analyzing data}

\begin{par}
To get a frequency spectrum, algorithm \lstinline{SP-FFT} can be used. This algorithm requires sampling frequency, so third quantity \lstinline{fs} is added.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI.fs.v = fsnom;
DO = qwtb('SP-FFT', DI);
plot(DO.f.v, DO.A.v, '-xb'); xlim([980 1020])
\end{lstlisting}

        \begin{lstlisting}[style=output]
QWTB: no uncertainty calculation
\end{lstlisting} \color{black}

\begin{center}
\includegraphics[width=0.7\textwidth]{qwtb_examples_published/qwtb_example_1_01.pdf}
\end{center}
\begin{par}
One can see it is not a coherent measurement. Therefore to get 'unknown' amplitude and frequency of the signal algorithm \lstinline{PSFE} can be used:
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DO = qwtb('PSFE', DI);
f = DO.f.v
A = DO.A.v
\end{lstlisting}

        \begin{lstlisting}[style=output]
QWTB: no uncertainty calculation
QWTB: PSFE wrapper: sampling time was calculated from sampling frequency

f =

 1000.0000

A =

    1.0000

\end{lstlisting} \color{black}

\subsubsection*{Uncertainties}

\begin{par}
Uncertainties are added to the \lstinline{t} (time stamps) and \lstinline{y} (sampled data) structures.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI.t.u = zeros(size(DI.t.v)) + 1e-5;
DI.y.u = zeros(size(DI.y.v)) + 1e-4;
\end{lstlisting}
\begin{par}
Calculations settings is created with Monte Carlo uncertainty calculation method, 1000 repeats and singlecore calculation. The output of messages is supressed to increase calculation speed.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
CS.unc = 'mcm';
CS.mcm.repeats = 1000;
CS.mcm.method = 'singlecore';
CS.verbose = 0;
\end{lstlisting}
\begin{par}
An uncertainty of sampling frequency has to be added. Let suppose the value:
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DI.fs.u = 1e-3;
\end{lstlisting}
\begin{par}
Run PSFE algorithm on input data \lstinline{DI} and with calculattion settings \lstinline{CS}.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
DO = qwtb('PSFE',DI,CS);
\end{lstlisting}
\begin{par}
Result is displayed as a histogram of calculated frequency.
\end{par} \vspace{1em}
\begin{lstlisting}[style=mcode]
figure; hist(DO.f.r,50);
\end{lstlisting}

\begin{center}
\includegraphics[width=0.7\textwidth]{qwtb_examples_published/qwtb_example_1_02.pdf}
\end{center}
\begin{par}
One can see the histogram is not Gaussian function. To get correct uncertainties, a shortest covariant interval has to be used.
\end{par} \vspace{1em}

%%% \end{document}


\section{Conclusion}\label{sec:abstractrefinements:conclusion}
We presented \emph{abstract refinement types} which enable
quantification over the refinements of data- and
function-types. Our key insight is that we
can avail of quantification while preserving SMT-based
decidability, simply by encoding refinement parameters
as \emph{uninterpreted} propositions within the
refinement logic.
%
We showed how this mechanism yields a variety
of sophisticated means for reasoning about programs, including:
\emph{parametric} refinements for reasoning with
type classes,
\emph{index-dependent} refinements for reasoning about
key-value maps,
\emph{recursive} refinements for reasoning about
recursive data types, and
\emph{inductive} refinements for reasoning about
higher-order traversal routines.
%
We implemented our approach in \toolname and present
experiments using our tool to verify correctness invariants
of various programs.

As discussed in~\ref{sec:experiments},
verification many times required code modifications
and definition of ``ghost'' variables (\eg to verify @append@),
that is, extra arguments not used at run time
but required for specifications.
%
In next chapter we raise this limitation by
introducing Bounded Refinement Types,
that impose constrains in the abstract refinements
to further increase the expressiveness of decidable specifications.
%
Abstract and Bounded Refinement Types lead to a relatively complete~\cite{TerauchiPOPL13}
specification system,
that is the system can express any specification (relatively to the underlying logic)
without the requirement of code modifications and ``ghost'' variables.


\documentclass[../algorithms.tex]{subfiles}
\begin{document}
In this chapter, we talk about recursive function and how to write a recursive function that work properly. Then we give an algorithm theory: divide and conquer, which is a recursive function and divide problems, and solve problems separely, the final result is merged from the result of each subproblem. Also we need to understand the space and time complexity.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% Recursive Programming
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Recursive Programming}
For recursive function, we can draw recursive tree to denote the state transfer graph. With recursive function, it can simplify the programming of certain programs, once we mastered recursive function, we would feel it is a lot simpler than iterative implementation. However, it does take extra space complexity compared with the iterative implementation.

\textbf{Recursive Function}: for recusrive program, we need to figure out the recursive transfer function between $f(i)$ and the next level $f(i+1)$. For example, the fibonacci number we could have $f(i) = f(i-1) + f(i-2)$. For the merge sort, we cant use a math function to represent this operation, we have $T(n) = 2*T(n/2) + O(n)$. $f(i) = merge(f(left), f(right))$. Some recursive function would have redundancy which we can improve the efficiency and avoid to compute the same subproblem twice by memoization which saves the result, and the iterative peer of the recursive implementation is called \textit{dynamic programming}. We will discuss the dynamic programming in details in the following chapter ~\ref{dynamic-programming}. Some other recursive function, there is no overlaps between subproblems which are the \textit{divide and conquer} cases, which will be discussed in detain in chapter~\ref{divide-conquer}. Or we have the universal \textit{Depth-first-searching} which can be implemented with recursive function, we will include this in chapter~\ref{searching}.

\textbf{Recursive Function and Tree Structure}:

\textbf{Stack Overflow and Iterative Implementation}: According to Wikepedia, in software, a stack overflow occurs if the call stack pointer exceeds the stack bound. The call stack may consist of a limited amount of address space, often determined at the start of the program depending on many factors, including the programming language, machine architecture, multi-threading, and amount of available memory. When a program attemps to use more space than is available on the call stack, the stack is said to \textit{overflow}, typically resulting in a program crash. The very deep recursive function is faced with the threat of stack overflow. And the only way we can fix it is by transforming the recursion into a loop and storing the function arguments in an explicit stack data structure, this is often called the iterative implementation which corresponds to the recursive implementation.

We need to follow these points:
\begin{enumerate}
    \item End condition, Base Cases and Return Values: either return an answer for base cases or None, and used to end the recursive calls.
    \item Parameters: parameters include: data needed to implement the function, current paths, the global answers and so on.
    \item Variables: What the \textbf{local} and {global} variables. In Python any pointer type of data can be used as global variable global result putting in the parameters.
    \item Construct current result: when to collect the results from subtree and combine to get the result for current node.
    \item Check the depth: if the program will lead to the heap stack overflow.
\end{enumerate}

% 递归函数关注以下几个因素
% ·退出条件
% ·参数有哪些
% ·返回值是什么
% ·局部变量有哪些
% ·全局变量有哪些
% ·何时输出
% ·会不会导致堆栈溢出
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% Divide and Conquer
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Divide and Conquer}
Divide and conquer partitions the problems into smaller subproblems and solves the problems recursively, and then combine the solutions to solve the original problem. It includes three steps at each recursive call:
\begin{enumerate}
    \item Divide: divide one problems into a series of subproblems that are smaller instances of the same problem;
    \item Conquer: recursively solve each subproblem. When the subproblem is small enough, it will be solved directly, which is the end condition in the recursive function;
    \item Combine: combine the result from each subproblem into the solution to the current problem.
\end{enumerate}

For divide and conquer, we have two cases, 1) our subproblems are disjoint with each other; 2) our smaller subproblem can include another smaller subproblem. We can use the the following two recurrence equation to generalize.
\begin{equation} \label{bt_time}
\begin{split}
T(n) & = aT(n/b) + f(n)\\
\end{split}
\end{equation}
where $f(n)$ denotes the time and operation need to divide the problem into disjoint and independent subproblems and combine the solutions of the subproblems to solve the current problem.
\begin{equation} \label{dp_equation}
\begin{split}
T(n) &= T(n-1) + T(n-2) +...+T(1) + f(n)\\
\end{split}
\end{equation}
 We can have any combination of terms $T(k), k = [1, n-1]$ on the right side of this recurrence equation. Here the problem is divided into subproblems, however these problem are not disjoint and always depend on other. And the subproblems overlap, which is to say, when subproblems share subproblems.

 Because of the different relations between these two situations' subproblems, Eq.~\ref{bt_time} when we use recursive programming to solve the problem directly, we get the best time complexity since there is no overlap between subproblems. However, for the second case in Eq.~\ref{dp_equation}, programming them recursivly would end up with redundancy in time complexity becase their subproblems share subproblems. This also means they can be further optimized, and these type of algorithms fall in the category of dynamic programming which we will discuss in the next chapter in details. So, commonly divide and conquer refers to methods in the first category.

 Now, enough of the concepts, let us look at an example:

 Example 1: Maximum Subarray (53. medium)
\begin{lstlisting}

Find the contiguous subarray within an array (containing at least one number) which has the largest sum.

For example, given the array [-2,1,-3,4,-1,2,1,-5,4],
 the contiguous subarray [4,-1,2,1] has the largest sum = 6.
\end{lstlisting}
divide and conquer solution: $T(n) = max(T(left),T(right), T(cross))$, max is for merging and the T(cross) is for the case that the potential subarray across the mid point. For the complexity, $T(n)=2T(n/2)+n$, if we use the master method, it would give us $O(nlgn)$. We write the following Python code
\begin{lstlisting}[language = Python]
def maxSubArray(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        def getCrossMax(low,mid,high):
            left_sum,right_sum =0,0
            left_max,  right_max = -maxint, -maxint
            left_i,right_j=-1,-1
            for i in xrange(mid,low-1,-1): #[)
                left_sum+=nums[i]
                if left_sum>left_max:
                    left_max= left_sum
                    left_i = i
            for j in xrange(mid+1,high+1):
                right_sum+=nums[j]
                if right_sum>right_max:
                    right_max= right_sum
                    right_j = j
            return (left_i,right_j,left_max+right_max)

        def maxSubarray(low,high):
            if low==high:
                return (low,high, nums[low])
            mid = (low+high)//2
            rslt=[]
            #left_low, left_high, left_sum = maxSubarray(low,mid) #[low,mid]
            rslt.append(maxSubarray(low,mid)) #[low,mid]
            #right_low,right_high,right_sum = maxSubarray(mid+1,high)#[mid+1,high]
            rslt.append(maxSubarray(mid+1,high))
            #cross_low,cross_high,cross_sum = getCrossMax(low, mid, high)
            rslt.append(getCrossMax(low, mid, high))
            return max(rslt, key=lambda x: x[2])
        return maxSubarray(0,len(nums)-1)[2]
\end{lstlisting}
Also, we does not necessarily to use divide and conquer, we can be more creative and try harder to make the time complexity goes to $O(n)$. We can convert this problem to best time to buy and sell stock problem.[0, -2, -1, -4, 0, -1, 1, 2, -3, 1], => O(n), then we use prefix\_sum, the difference is we set prefix\_sum to 0 when it is smaller than 0, O(n)
\begin{lstlisting}[language = Python]
from   sys import maxint
class Solution(object):
    def maxSubArray(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        max_so_far = -maxint - 1
        prefix_sum= 0
        for i in range(0, len(nums)):
            prefix_sum+= nums[i]
            if (max_so_far < prefix_sum):
                max_so_far = prefix_sum

            if prefix_sum< 0:
                prefix_sum= 0
        return max_so_far
\end{lstlisting}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% Complexity Analysis
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Complexity Analysis}
The complexity analysis incldudes the time and space complexity. While for the divide and conquer methods, because of the usage of the recursion function, it could be slightly tricky and we need to learn how to attack this complexity. Also it is usually required and asked by interviewers in real interview.

\subsection{Time Complexity}
The core to analyze the time complexity of the divide and conquer methodology is by characterizing the recurrence relation shown in Eq.~\ref{bt_time}. In general we have three ways to do this, 1) substitution method; 2) recursion-tree method; 3) master method. However, in real situation, it depends on XX to choose which one to use. Each one has its own limitation. In this book, we prove enough theory about how to compute the time complexity for the recurrence equation for practical coding or interview situation. If you want to learn more, it is a good choice to refer the book (Introduction to Algorithms). Here, we wont detail on the substitution method, because in real interviews, we need some quick and more straightforward ways to get the computational cost. And the substitution method is more used to prove the cost in a very rigorous way. Recursion tree and the master therorems are the main ways we rely on to answer the time complexity for a divide and conquer method shown in Eq.~\ref{bt_time}.

\subsubsection{Recursion Tree Method}
Drawing out a recursion tree serves as a straightforward way to come up with a good guess. Normally we can tolerate a small amount of "sloppiness", because later on, we can prove the complexity with substitution method discussed in the last section. However, when we are drawing the recursion tree, if we are careful enough and summing up the costs from each level and each node, we can use is as a direct proof of the solution to the recurrence.

In the corresponding recursion tree for recurrence equation in divide and conquer, each node represents the cost of a single subproblem somewhere in the set of recursive function invocations. We sum the costs within each level of the tree to obtain a set of per-level costs, and then we sum all the per-level costs to determine the total cost of all levels of the recursion. Let's look at one example for given recursion $T(n) = 3T(\floor*{n/4}) + \Theta(n^2)$. We replace $\Theta(n^2) = cn^2$, where $c>0$. $cn^2$ is the cost we pay to divide a problem with $n$ input size to three problems each with $n/4$ input size and combine the solution of the subproblems to solve the current problem.  We first expand $T(n)$, and put the cost $cn^2$ at the root, and with three children each noted with $T(n/4)$. Then we recursively replace $T(n/4)$ with the cost and its subproblem till the size of each subproblem to be 1, which means we get to the leaves. The computational complexity for this recursion would be the sum of all layers's costs. And we assume $T(1)=1$.
\begin{figure}[h]
    \centering
    \includegraphics[width=0.8\columnwidth]{fig/recursive_tree_1.png}
    \includegraphics[width=0.8\columnwidth]{fig/recursive_tree_2.png}
    \caption{The process to construct a recursive tree for $T(n) = 3T(\floor*{n/4}) + cn^2$}
    \label{fig:recursive_tree}
\end{figure}
\begin{equation} \label{eg_recurrence_5}
\begin{split}
T(n) & = cn^2+\frac{3}{16}cn^2+(\frac{3}{16})^2cn^2+...+(\frac{3}{16})^{\log_4 {n-1}}cn^2+\Theta(n^{\log_4 3})\\
& = \sum_{i=0}^{\log_4 {n-1}}(\frac{3}{16})^{i}cn^2+\Theta(n^{\log_4 3})\\
&< \sum_{i=0}^{\infty}(\frac{3}{16})^{i}cn^2+\Theta(n^{\log_4 3})\\
& = \frac{1}{1-(3/16)} cn^2+\Theta(n^{\log_4 3})\\
& = O(n^2).
\end{split}
\end{equation}
\subsubsection{Master Method}
The master method is probably the easiest way to come up with the computational complexity analysis. It is a theorem that are proved by researchers, and we just need to learn how to use them. The master theorem goes:

For Eq.~\ref{bt_time}, let $a\geq1, b>1$, we first compute $n^{\log_b a}$,
\begin{enumerate}
    \item If $f(n) = O(n^{\log_b a - \epsilon}$ for constant $\epsilon>0$, then we get $T(n) = \Theta(n^{\log_b a})$.
    \item If $f(n) = \Theta(n^{\log_b a }$, then we get $T(n) = \Theta(n^{\log_b a} \log n)$.
    \item If $f(n) = \Omega(n^{\log_b a + \epsilon}$ for constant $\epsilon>0$, and if $a f(n/b)\leq cf(n)$ for constant $c<1$ and all sufficiently large $n$, then we get $T(n) = \Theta(f(n))$.
\end{enumerate}
\subsection{Space Complexity}
The space the recursive function occupies is rational to the depth of the recursive calls, $O(h)$, $h$ is the height of the recursive tree.

\subsection{Summary}
For your convenience, we prove a table that shows the frequent used recurrence equations' time complexity.
\begin{figure}[h]
    \centering
    \includegraphics[width=1\columnwidth] {fig/complexity_cheatsheet.png}
    \caption{The cheat sheet for time and space complexity with recurrence function.}
    \label{fig:cheat_sheet}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%% Exercises
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Exercises}
\begin{enumerate}
    \item Pow(x, n) (50)

    Solution: T(n)= T(n/2)+O(1), the complexity is the same as the binary search, $O(logn)$.
    \begin{lstlisting}[language=Python]
    def myPow(self, x, n):
        """
        :type x: float
        :type n: int
        :rtype: float
        """
        if n==0:
            return 1
        if n<0:
            n=-n
            x=1.0/x
        def helper(n):
            if n==1:
                return x

            h = n//2
            r = n-h
            value = helper(h) #T(n/2), then we have O(1)
            if r==h:
                return value*value
            else: #r is going to be 1 bigger
                return value*value*x
        return helper(n)
    \end{lstlisting}

    \item House Robber (198)

    Solution: If we use brute force is $O(2^n)$. Use divide and conquer, here because we use half and half. Which we need to get rid of.
\begin{lstlisting}[language = Python]
def rob(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        memo=[[-1 for _ in range(len(nums))] for _ in range(len(nums))]

        def dp(l,r):
            nonlocal memo
            if l==r:
                return nums[l]
            if l>r:
                return 0
            if l<r:
                if memo[l][r]==-1:
                    mid=l+(r-l)//2
                    value = nums[mid] +dp(l,mid-2)+dp(mid+2,r)#take this value
                    not_value =dp(l,mid-1)+dp(mid+1,r) #not take
                    memo[l][r]=max(value, not_value)
                return memo[l][r]
        return dp(0,len(nums)-1)
\end{lstlisting}

\item Create Maximum Number (321)

Given two arrays of length m and n with digits 0-9 representing two numbers. Create the maximum number of length k <= m + n from digits of the two. The relative order of the digits from the same array must be preserved. Return an array of the k digits. You should try to optimize your time and space complexity.
\begin{lstlisting}
Example 1:

nums1 = [3, 4, 6, 5]
 nums2 = [9, 1, 2, 5, 8, 3]
 k = 5
 return [9, 8, 6, 5, 3]

Example 2:

nums1 = [6, 7]
 nums2 = [6, 0, 4]
 k = 5
 return [6, 7, 6, 0, 4]

Example 3:

nums1 = [3, 9]
 nums2 = [8, 9]
 k = 3
 return [9, 8, 9]

\end{lstlisting}

Solution: First use DP + memo: LTE, 70/120 passed. We should use greedy and solve each array to find i,j, i+j = k, then we combine them.
\begin{lstlisting}[language=Python]
def maxNumber(self, nums1, nums2, k):
        """
        :type nums1: List[int]
        :type nums2: List[int]
        :type k: int
        :rtype: List[int]
        """
        '''i, j T(i,j,k)= max(nums[i]*10^k-1+T(i+1,j,k-1), j, T(i+1,j+1,k)), for the third iterm, the restriction is we need the left       elements>=k'''
        n1=len(nums1)
        n2=len(nums2)
        memo=[[[None for k in range(k+1)] for col in range(n2+1) ] for row in range(n1+1)]
        def dp(i,j,k):
            if k==0:
                return 0
            if memo[i][j][k] is None:
                max1,max2,max3=-1,-1,-1
                if i<len(nums1):
                    tmp =dp(i+1, j, k-1) #take i
                    tmp2 = dp(i+1,j,k) #neglect i in nums1
                    if tmp!=-1:
                        max1 = nums1[i]*(10**(k-1))+tmp
                    max1=max(max1,tmp2)
                if j<len(nums2):
                    tmp = dp(i, j+1, k-1) #take j
                    tmp2=dp(i,j+1,k) #neglect j in nums2
                    if tmp!=-1:
                        max2 = nums2[j]*(10**(k-1))+tmp
                    max2=max(max2,tmp2)
                memo[i][j][k] = max(max1,max2)
            return memo[i][j][k]
        r = dp(0,0,k)
        rst=[]
        for i in range(k-1,-1,-1):
            rst.append(r//(10**i))
            r = r%(10**i)

        return rst
\end{lstlisting}

\item Best Time to Buy and Sell Stock (121)

Say you have an array for which the ith element is the price of a given stock on day i.

If you were only permitted to complete at most one transaction (ie, buy one and sell one share of the stock), design an algorithm to find the maximum profit.
\begin{lstlisting}[language=Python]
Example 1:

Input: [7, 1, 5, 3, 6, 4]
Output: 5

max. difference = 6-1 = 5 (not 7-1 = 6, as selling price needs to be larger than buying price)

Example 2:

Input: [7, 6, 4, 3, 1]
Output: 0

In this case, no transaction is done, i.e. max profit = 0.
\end{lstlisting}

Solution: Step 1: convert it to the maximum subarray problem. The first example is, [7–7=0,1–7=-6,5–1=4,3–5=-2,6–3=3, 4–6=-2],set the first element to 0, then the second is nums[i]-[nums[i-1], [0,-6,4,-2,3,-2] =>[0+6, 6–6=0, 0+4=4, 4–2=2, ] Set the first to 0+6, nums[i-1]+nums[i].

r = max(left\_subarray, right\_subarry, max(right\_subarry)-min(left\_subarray)), Thus, the real operation is max(right\_subarry)-min(left\_subarray). The time complexity would be decreased to $O(nlgn)$ from the brute force $O(n^2)$. So this example shows the divide and conquer. However, it might not be the best solution. Try the BCR with $O(n)$.
\begin{lstlisting}[language=Python]
class Solution(object):
    def maxProfit(self, prices):
        """
        :type prices: List[int]
        :rtype: int
        """
        if len(prices)<=1:
            return 0

        r = -maxint
        min_price = maxint
        for price in prices:
            if price<min_price:
                min_price = price
            else:
                r = max(r,price-min_price)

        return 0 if r<0 else r
\end{lstlisting}
\end{enumerate}
\end{document}


\section{\texorpdfstring{Polynomial Hierarchy}{Polynomial Hierarchy}}
\vspace{5mm}
\large

\subsection{Simplified PH}

\begin{definition}[Simplified PH]
	Consider a sequence
	\[ \Sigma_0^P, \Sigma_1^P, \Sigma_2^P, \ldots \]
	Where $\Sigma_0^P = \TP, \Sigma_{i + 1}^P = \TNP(\Sigma_i^P)$.

	And Polynomial hierarchy(simplified) is:
	\[ PH = \bigcup_{i \geq 1} \Sigma_i^P \]
\end{definition}

\begin{theorem}[Polynomial hierarchy(simplified)]
	\[ PH \subseteq PS \]

	In plain words, PH is only smth in between NP and PS.
\end{theorem}
\begin{proof}
	By induction prove $\forall i: \Sigma_i \subseteq PS$.

	\begin{itemize}
		\item $i = 0 \Rightarrow \TP \subseteq PS$
		\item $i \to i + 1$, assume $\Sigma_i \subseteq PS$.
	\end{itemize}

	By definition: $\Sigma_{i + 1} = \TNP(\Sigma_i)$
	Then
	\[ \TNP(\Sigma_i) \subseteq \TNP(PS) \]
	We made set of oracles larger, set of recognized languages cannot shrink.

	Now we use $\forall C: \TNP(C) \subseteq PS(C)$.\\
	Therefore
	\[ \TNP(C) \subseteq PS(PS) \subseteq PS \]

	Last inclusion is up to prove (similar to $\TP = \TP(\TP)$):\\
	\[ B \in PS(PS) \iff \exists A \in PS \exists DTM M: B = L(M, A) \]
	Where $M$ works in poly space.

	\[ A \in PS \iff \exists DTM \ M_A: A = L(M_A) \]
	Where $M_A$ works in poly space.
	Same in $\TP = \TP(\TP)$ proof \cref{p_eq_pp} replace oracle QUERY by DTM computation (which accepts or rejects)

	The last thing is to check that used space is polynomial.
	Which is true since $\forall t \in QUERY: |t| \leq p(|x|)$ for some polynomial $p$.
	Space taken by DTM $M_A$ that computes the QUERY is $p_t(|t|) \leq p_t(p(|x|))$.
	Also, we can ask exponentially many QUERies, however the space is reused, therefore space is bounded by the largest QUERY.
\end{proof}

% 3rd lecture
\subsection{Time and Space classes relation}

\begin{reminder}[Space classes]
	\begin{equation}
	\begin{split}
		LOG = DS(\log n) \\
		NLOG = NS(\log n) \\
		POLYLOG = \bigcup_{i \geq 0} DS(\log^i n )\\
		PS = \bigcup_{i \geq 0} DS(n^i)\\
		NSSP = \bigcup_{i \geq 0} NS(n^i)\\
		EXPSPACE = \bigcup_{i \geq 0} DS(2^{ni})\\
	\end{split}
	\end{equation}
\end{reminder}

\begin{reminder}[Time classes]
	No way to define LOG class, as we have to read the input.
	\begin{equation}
	\begin{split}
		\TP = \bigcup_{i \geq 0} DT(n^i)\\
		\TNP = \bigcup_{i \geq 0} NT(n^i)\\
		DEXT = \bigcup_{i \geq 0} DT(2^{ni})\\
		NEXT = \bigcup_{i \geq 0} NT(2^{ni})\\
		EXPTIME = \bigcup_{i \geq 0} DT(2^{n^i})\\
		NEXPTIME = \bigcup_{i \geq 0} NT(2^{n^i})\\
	\end{split}
	\end{equation}
\end{reminder}

\begin{theorem}[Complexity classes relations]
	Complexity classes
	\begin{enumerate}[label=\alph*)]
		\item $NLOG \subseteq \TP$
		\item $PS = NPS$
		\item $\TNP \subseteq PS$
		\item $PS = EXPTIME$
		\item $NLOG \subsetneq PS \subsetneq EXPSPACE$
		\item $\TP \subsetneq DEXT \subsetneq EXPTIME$
	\end{enumerate}
\end{theorem}
\begin{proof}
	a)
	\[ L \in NS(\log n) \stackrel{\cref{ns_dt}}{\Rightarrow} \exists c_L: L \in DT(2^{c_L \log n}) = DT((2^{\log n})^{c_L}) = DT(n^{c_L}) \in \TP \]
	b) $PS \subseteq NPS$ trivial, as deterministic computation is a special case of non-deterministic.

	$NPS \subseteq PS$
	\[ L \in NS \Rightarrow \exists i: L \in NS(n^i) \]
	by Savic \cref{savic}
	\[ L \in DS(n^{2i}) \Rightarrow L \in PS \]

	c)
	\[ \forall L \in \TNP \stackrel{\cref{nt_ds_rel}}{\Rightarrow} \exists i: L \in NT(n^i) \Rightarrow L \in DS(n^i) \Rightarrow \TNP \subseteq PS \]
	d)
	\[ L \in PS \Rightarrow \exists i: L \in DS(n^i) \subseteq NS(n^i) \stackrel{\cref{ns_dt}}{\Rightarrow}\exists c_L: L \in DT(2^{c_L \log n}) \subseteq DT(2^{n^{i + 1}}) \subseteq EXPTIME \]
	e) by Savic \cref{savic}
	\[ NS(\log n) \subseteq DS(\log^2 n) \]
	by space hierarchy \cref{s_hier}
	\[ \log^2 n \in o(n) \Rightarrow DS(\log^2 n) \subsetneq DS(n) \]
	\[ L \in PS \Rightarrow \exists i: L \in DS(n^i) \subseteq DS(2^n) \]
	by space hierarchy \cref{s_hier}
	\[ DS(2^n) \subsetneq DS(2^{2n}) \]

	f)
	by time hierarchy \cref{t_hier}
	\[ \TP \subseteq DT(2^n) \subsetneq DT(2^{2n}) \subseteq DEXT \]
	as
	\[ 2^n \log(2^n) = n 2^n \in o((2^n)^2) \]
	Then
	\[ DEXT \subseteq DT(2^{n^2}) \subsetneq DT(2^{n^3}) \subseteq EXPTIME \]
	as
	\[ n^2 (2^{n^2}) \in o (2^{n^3}) \]
\end{proof}

\begin{note}
	\[ NLOG \subseteq \TP \subseteq \TNP \subseteq PS \]
	Also
	\[ NLOG \subsetneq PS \]
	Therefore one of the inclusions in first relation should be strict, or all of them. Still open question.

	Moreover, if we prove strict equalities in leftmost and rightmost inclusions $\Rightarrow \TP \neq \TNP$.
\end{note}

\subsection{Full Polynomial hierarchy}

\begin{definition}[Polynomial hierarchy]
	Consider a 3 sequences of classes
	\[ \Sigma_k, \Pi_k, \Delta_k \]
	Where
	\begin{enumerate}
		\item $\Sigma_0 = \Pi_0 = \Delta_0 = \TP$.
		\item $\Sigma_{k + 1} = \TNP(\Sigma_k)$.
		\item $\Pi_{k + 1} = co-\TNP(\Sigma_k)$.
		\item $\Delta_{k + 1} = \TP(\Sigma_k)$.
	\end{enumerate}

	And Polynomial hierarchy is:
	\[ PH = \bigcup_{i \geq 1} \Sigma_i^P ( = \bigcup_{i \geq 1} \Pi_i^P = \bigcup_{i \geq 1} \Delta_i^P) \]

\end{definition}

\begin{definition}[Language complement]
	$L$ is a language over alphabet $\tau$
	\[ \overline{L} = \{ x \in \tau^{\ast} |\ x \notin L \} \]

	If $\C$ is a class of languages, then
	\[ L \in \C \iff \overline{L} \in co-\C \]
\end{definition}

\begin{theorem}[Polynomial Hierarchy] Relations
	\begin{enumerate}[label=\alph*)]
		\item $\Sigma_1 = \TNP $.
		\item $\Pi_k = co-\Sigma_k \land \Sigma_k = co-\Pi_k$.
		\item $\Sigma_{k + 1} = \TNP(\Pi_k)$.
		\item $\Delta_{k + 1} = \TP(\Pi_k)$.
		\item $\Pi_{k + 1} = co-\TNP(\Pi_k)$.
		\item $\Sigma_{k + 1} = \TNP(\Delta_{k + 1})$.
		\item $\Pi_{k + 1} = co-\TNP(\Delta_{k + 1})$.
	\end{enumerate}
\end{theorem}
\begin{proof}~
\begin{enumerate}[label=\alph*)]
	\item $\TNP(\TP) = \TNP$ also $\TP(\TP) = \TP$.\\
	Proof by embedding oracle computation via DTM simulation.

	\item by definition, for $k \geq 1$
	\[ \Pi_k = co-\TNP(\Sigma_{k - 1}) = co-\Sigma_k \]
	same for $\Sigma_k$ by symmetry.

	\item we can deduce $\overline{B}$ from questions to $B$ by negating every request.
	In other words, swap YES/NO answers after queries.

	\item swap YES/NO answers after queries.

	\item follows from b), c).

	\item
	\[ \Sigma_{k + 1} = \TNP(\Sigma_k) \stackrel{\text{by (*)}}{=} \TNP(\TP(\Sigma_k)) = \TNP(\Delta_{k + 1}) \]
	Where (*) follows from
	\[ \TNP(\Sigma_k) \subseteq \TNP(\TP(\Sigma_k)) \]
	as we made class of query languages larger.

	"$\supseteq$"\\
	\[ L \in \TNP(\TP(\Sigma_k)) \Rightarrow \exists NTM\ M_n \exists E \in \TP(\Sigma_k): L = L(M_n, E) \]
	Also
	\[ E \in \TP(\Sigma_k) \iff \exists DTM\ M_d, \exists Q \in \Sigma_k: E = L(M_d, Q) \]
	We want $L \in \TNP(\Sigma_k)$.
	We proceed similarly as in $\TP(\TP) = \TP$ by simulating $M_n$ and every time it ask a query, use $M_d$ to decide.

	\item follows from f) by taking the complements
\end{enumerate}

\end{proof}

\begin{theorem}[Polynomial Hierarchy - 2] Relations 2.
	\begin{enumerate}[label=\alph*)]
		\item $\Delta_k = co-\Delta_k$
		\item $\TP(\Delta_k) = \Delta_k$
		\item $\Sigma_k \cup \Pi_k = \Delta_{k + 1}$
		\item $\Delta_k = \Sigma_k \cap \Pi_k$
		\item if $\Sigma_k \subseteq \Pi_k \lor \Pi_k \subseteq \Sigma_k \Rightarrow \Pi_k = \Sigma_k$
	\end{enumerate}
\end{theorem}
\begin{proof}
	a) for deterministic computation we can negate every answer.
	Specifically this rule means $\TP = co-\TP$

	b) $\Delta_k \subseteq \TP(\Delta_k)$ trivially. Since we can use same language as oracle with 1 query??

	$\TP(\Delta_k) \subseteq \Delta_k$. Proof by induction on $k$:
	\[ k = 0, \TP(\TP) = \TP \]

	$k \geq 1$ we have $\TP(\Delta_{k - 1}) \subseteq \Delta_{k - 1}$
	By definition $\TP(\Delta_k) \subseteq \Delta_k$ is equivalent to
	\[ \TP(\TP(\Delta_{k - 1})) \subseteq \TP(\Delta_{k - 1}) \]

	Both c) and d) use the fact, that deterministic computation is a special case of non-deterministic.

	c) proof consists of 2 steps
	\begin{enumerate}
		\item $\Sigma_k \subseteq \Delta_{k + 1} = \TP(\Sigma_k)$\\
			Same argument as in first inclusion in b).
			Ask single query to the same language.
		\item $\Pi_k \subseteq \Delta_{k + 1} = \TP(\Sigma_k)$\\
			Since we can negate queries
			\[ \TP(\Sigma_k) = \TP(\Pi_k) \]
			Therefore using same argument as above with single query
			\[ \Pi_k \subseteq \TP(\Pi_k) \]
	\end{enumerate}

	d) proof consists of 2 steps
	\begin{enumerate}
		\item
			\[ \Delta_k \stackrel{def}{=} \TP(\Sigma_{k - 1}) \subseteq \TNP(\Sigma_{k - 1}) = \Sigma_k \]
		\item
			\[ \Delta_k \stackrel{\text{a)}}{=} co-\Delta_k = co-\TP(\Sigma_{k - 1}) = co-\TP(\Pi_{k - 1}) \subseteq co-\TNP(\Pi_{k - 1}) = \Pi_k \]
	\end{enumerate}

	e) Assume $\Sigma_k \subseteq \Pi_k$ we need to prove reverse.

	\[ L \in \Pi_k \iff \overline{L} \in co-\Pi_k = \Sigma_k \xRightarrow{assumption} \overline{L} \in co-\Sigma_k = \Pi_k \iff L \in \Sigma_k  \]
\end{proof}

\begin{theorem}[NP = co-NP]
	if $A \in \TNP$-complete $\land A \in co-\TNP$-complete $\Rightarrow \TNP = co-\TNP$.
\end{theorem}
\begin{proof}
	Let $B \in \TNP$ arbitrary.
	By the $\TNP$-completeness of A $\exists DTM$ transducer $M_t$ st
	\[ x \in B \iff M_t(x) \in A \]

	\[ A \in co-\TNP \iff \overline{A} \in \TNP \Rightarrow \exists NTM\ M_n: \overline{A} = L(M_n) \]
	Also by the properties of polynomial reduction
	\[ x \in \overline{B} \iff M_t(x) \in \overline{A} \]
	$\overline{A}$ can be recognized by $M_n$, therefore we can recognize $\overline{B}$ by NTM which is a concatenation of $M_t$ and $M_n$.
	Therefore $\overline{B} \in \TNP \iff B \in co-\TNP $.

	Same proof is valid for oracle TM.
\end{proof}

%3th lecture 01:50:00
\begin{theorem}[NP $\neq$ co-NP]
	\[ \TNP \ne co-\TNP \Rightarrow \TNP \cup\ co-\TNP \subsetneq \Delta_2 \]
\end{theorem}
\begin{proof}
	Let
	\[ L = \{ (F, F^{\prime}) |\ F \in SAT \land F^{\prime} \in \overline{SAT} \} \]
	We can construct DTM with following algorithm for L

	%todo add identation for readbility
	\begin{enumerate}
		\item if($F \in SAT$)
		\item \tab if($\overline{F} \in SAT$)
		\item \tab \tab reject
		\item \tab else
		\item \tab \tab accept
		\item else
		\item \tab reject
	\end{enumerate}

	Therefore $L \in \TP(SAT) \subseteq \Delta_2 = \TP(\Sigma_1) = \TP(\Pi_1)$.

	Define
	\[ L_2 = \{ (F, 0)|\ F \in SAT \} \subseteq L \]
	where $0$ is e.g. $(x \land \neg x)$.

	Trivially $ L_2 \simeq SAT$ therefore L contains $\TNP$-complete language.

	Now assume by contradiction $L \in co-\TNP$.
	Using previous proof we can deduce $\TNP = co-\TNP$.
\end{proof}

\subsection{Constrained quantifiers}
\begin{definition}[Constrained $\exists$]
	\[ \exists^{p(n)} x: R(x) := \exists x: |x| \leq p(n) \land R(x) \]
\end{definition}
\begin{definition}[Constrained $\forall$]
	\[ \forall^{p(n)} x: R(x) := \forall x: |x| \leq p(n) \land R(x) \]
\end{definition}

\begin{definition}[$\exists \C$]
	Let $\C$ be a class of languages, we define class of languages $\exists \C$ as following:
	\[ A \in (\exists \C) \stackrel{\text{def}}{\iff} \exists B \in \C, \exists p: x \in A \iff \exists^{p(|x|)} y: \langle x, y \rangle \in B \]

	Note that if $\C = \TP \Rightarrow \exists \TP = \TNP $.
	Since $y$ is a branch of NTM (certificate)
\end{definition}

\begin{definition}[$\forall \C$]
	Let $\C$ be a class of languages, we define class of languages $\forall \C$ as following:
	\[ A \in (\forall \C) \stackrel{\text{def}}{\iff} \exists B \in \C, \forall p: x \in A \iff \forall^{p(|x|)} y: \langle x, y \rangle \in B \]

	Note that if $\C = \TP \Rightarrow \forall \TP = co-\TNP $.
\end{definition}

\begin{lemma}[$\exists \C, \forall \C, co-\C$]\label{quant_class_rel}
	Let $\C$ be an arbitrary class of languages. Then
	\begin{equation}
		co-\exists \C = \forall(co-\C)
	\end{equation}
	As
	\[ A \in (\exists \C) \iff \overline{A} \in (\forall(co-\C)) \]

	\begin{equation}
		\C \subseteq \exists \C \land \C \subseteq \forall \C
	\end{equation}
\end{lemma}
\begin{proof}
	Equation (3): \\
	Negating the definition
	\[ A \in (\exists \C) \iff \exists B \in \C, \exists p: x \in A \iff \exists^{p(|x|)} y: \langle x, y \rangle \in B \]
	we get
	\[ x \in \overline{A} \iff \forall^{p(|x|)} y: \langle x, y \rangle \in \overline{B} \]
	Which is the same as
	\[ \overline{A} \in \forall (co-\C) \iff \exists \overline{B} \in co-\C, \exists p: \forall^{p(|x|)} y: \langle x, y \rangle \in \overline{B} \]

	Equation (4): \\
	For $\exists \C$ take $B = A \land y = \emptyset$.
	Clearly $\langle x, \emptyset \rangle \simeq x$.

	For $\forall \C$ take polynomial with degree 0, only $y = \emptyset$ is accepted.
\end{proof}

\begin{theorem}[Polynomial Hierarchy with quantifiers]\label{polyn_q}
	\begin{enumerate}[label=\alph*)]
		\item $\exists \TP = \TNP$
		\item $\forall \TP = co-\TNP $.
		\item $\forall k > 0: \exists \Sigma_k = \Sigma_k$.
		\item $\forall k > 0: \forall \Pi_k= \Pi_k$.
		\item $\forall k \geq 0: \exists \Pi_k = \Sigma_{k + 1}$.
		\item $\forall k \geq 0: \forall \Sigma_k = \Pi_{k + 1}$.
	\end{enumerate}
\end{theorem}
\begin{proof}
	a) "$\exists \TP \subseteq \TNP$"
	\[ A \in (\exists \TP) \iff \exists B \in \TP, \exists p: x \in A \iff \exists^{p(|x|)} y: \langle x, y \rangle \in B \]
	Construct an NTM that accepts A ($L(M) = A$):

	\begin{enumerate}
		\item guess y: $|y| \leq p(|x|)$
		\item run deterministic verification $\langle x, y \rangle \in B$
	\end{enumerate}
	B is in $\TP$ therefore verification is deterministic.
	As $|y|$ is bounded by $p(|x|)$ total time complexity is polynomial.

	"$\TNP \subseteq \exists \TP$"
	\[ A \in \TNP \iff \exists NTM\ M: L(M) = A \]
	Also
	\[ x \in A \iff \exists^{p(|x|)} y \]
	where $y$ encodes the accepting branch of M on input x.

	// todo why??\\
	Consequently any language $B$ of pairs $\langle x, y \rangle$ is in $\TP$.
	So, by definition
	\[ A \in \exists \TP \]

	b) using lemma \cref{quant_class_rel}
	\[ \forall \TP \xlongequal{\TP = co-\TP} \forall(co-\TP) \xlongequal{\cref{quant_class_rel}} co-\exists \TP \xlongequal{a)} co-\TNP \]

	c) trivially $\Sigma_k \subseteq (\exists \Sigma_k)$. Now reversed.

	% todo finish
	\[ A \in (\exists \Sigma_k) \iff \exists B \in \Sigma_k, \exists p: x \in A \iff \exists^{p(|x|)} y: \langle x, y \rangle \in B \]
	We want
	\[ A \in \TNP(\Sigma_{k - 1}) = \Sigma_k \]
	So, we construct an NTM M with oracle $C \in \Sigma_{k - 1}: L(M, C) = A$, we also use
	\[ B \in \TNP(\Sigma_{k - 1}) \iff \exists M_B: B = L(M_B, C) \]
	algorithm of M:
	\begin{enumerate}
		\item guess y: $|y| \leq p(|x|)$
		\item run $M_B$ on $\langle x, y \rangle$.
	\end{enumerate}

	Note that
	\[ \exists y \exists z \iff \exists \langle y, z \rangle \]

	d)
	\[ \forall \Pi_k = \forall(co-\Sigma_k) \xlongequal{\cref{quant_class_rel}} co-\exists \Sigma_k \xlongequal{c)} co-\Sigma_k = \Pi_k \]

	Note that now we cannot prove e)

	f) follows from e)
	\[ \forall \Sigma_k = \forall(co-\Pi_k) \xlongequal{\cref{quant_class_rel}} co-\exists \Pi_k \xlongequal{e)} co-\Sigma_{k + 1} = \Pi_{k + 1} \]
\end{proof}


\subsection{Complete preferences}

\subsubsection{Axiom 2: An economic agent has preferences across all pairs of elements of the choice set}

Where an agent prefers one option to another, we say this choice is preferred to the other. We can show this formulaically. If \(a\) is preferred to \(b\) then:

\(a\succ b\)

Alternatively \(b\) may be preferred to \(a\).

\(a\prec b\)

Finally an agent may be indifferent to the choices.

\(a\sim b\)

We use can additional symbols to denote an agent prefers or is indifferent between options:

\(a\succeq b\)

\(a\preceq b\)

Note that if:

\(a\succeq b\)

And:

\(a\preceq b\)

Then:

\(a\sim b\)

For a set of choice, we want the agent to have preferences defined across possible subsets – the power set.

Add symbols for choice across a set. include symbols for choice of a set, x =c({a,b},utility)

a>b etc only for pairwise, but preferences are across more than that

Discussion of repeated. agents don't choose the same thing each time, interpretation?


\section{\texorpdfstring{TM with advice}{TM with advice}}
\vspace{5mm}
\large
% 8th lecture

\begin{definition}[$DT(T(n))|_{a(n)}$]
	Let $T, s: \N \to \N$. Then
	\[ DT(T(n))|_{a(n)} \]
	is a class of languages recognizable by DTM in time $T(n)$ with $s(n)$ bit advice $\alpha_n$.
	\[ L \in DT(T(n))|_{s(n)} \iff \exists DTM M \& \exists \{\alpha_n\}_{n \in \N}, \alpha_n \in \{ 0, 1 \}^{s(n)}: \]
	\[ \forall x \in \{ 0, 1 \}^n: x \in L \iff M(x, \alpha_n) = 1 \]
	And M makes $\bigO(T(n))$ steps on the input $(x, \alpha_n)$.
\end{definition}

\begin{note}
	TM with advice is an non-uniform computation as advice size depends on input size.
\end{note}

\begin{theorem}[$\Ppoly$]
	$\Ppoly = \bigcup_{c, d} DT(n^c)|_{n^d} $
\end{theorem}
\begin{proof}
	"$\subseteq$".
	Let $L \in \Ppoly$ then by definition exists polynomial time circuit $\{ C_n \}$ that recognizes $L$.
	We set $\{ \alpha(n) \} = \{ C_n \}$ (via encoding) and the DTM M simulates $C_n$ on $x$.

	As circuit is of polynomial size, M works in polynomial time.
	Therefore
	\[ L \in \bigcup_{c, d} DT(n^c)|_{n^d} \]

	"$\supseteq$". Let $L \in \bigcup_{c, d} DT(n^c)|_{n^d} \Rightarrow \exists DTM M$ which works in time $T(n)$, uses advice $\{ \alpha(n) \}$ of size $a(n)$ and $L = M(x, \alpha_n)$.

	Like in proof $\TP \subseteq \Ppoly$ \cref{tp_ppoly} there exists a polynomial size family of $\{ C_n \}$:
	\[ \forall x \in \{ 0, 1 \}^n \forall \alpha \in \{ 0, 1 \}^{a(n)}: M(x, \alpha) = C_m(x, \alpha), m = n + a(n) \]

	Now we hardcode $\{ \alpha_n \}$ into the circuit.
	Define
	\[ C_m^{hard}(x) = C_m(x, \alpha_n) \]
	Then
	\[ x \in L \iff M(x, \alpha_n) = 1 \iff C_m(x, \alpha_n) = 1 \iff C_m^{hard}(x) = 1 \Rightarrow L \in \Ppoly \]
\end{proof}

\begin{note}
	Not every boolean function can be computed by a polynomial size circuit.
\end{note}

\begin{theorem}[$\exists f$ no poly circuit]
	\[ \forall n > 1, \exists f: \szo^n \to \szo \]
	such that $f$ cannot be computed by circuit $|C| = \frac{2^n}{10n}$.
\end{theorem}
\begin{proof}
	Idea: show the mismatch between \# of functions and \# of circuits.

	Let $C$ be a circuit, $|C| = m$.
	Since every elementary gate has at most 2 inputs, $C$ has $\leq 2m$ edges.
	Therefore $C$ can be represented by a binary string of length $3m \log m$.
	So
	\[ |\{ C |\ |C| = m \}| = 2^{3m \log m} \]
	However, there are $2^{2^n}$ functions.

	Setting $m = \frac{2^n}{10n}$ we have at most
	\[ 2^{3 \frac{2^n}{10n} \log (\frac{2^n}{10n})} \leq 2^{3 \frac{2^n}{10n} n} = 2^{\frac{3}{10} 2^n} < 2^{2^n} \]
	circuits.
\end{proof}

\begin{consequence}
	With growing $n$ more and more functions are not computable by polynomial circuits.
\end{consequence}

\begin{note}[Open Problem]
	If we can find a family of functions $\{ f_n\}$ that are not computable by polynomial size circuit such that $L \in \TNP$.
	Would imply $\TNP \nsubseteq \Ppoly \Rightarrow \TP \neq \TNP$.
\end{note}

\begin{definition}[$\Pi_n, \Sigma_n$-SAT]
	\[ \Sigma_n-SAT = \exists x_1 \forall x_2 \ldots Q x_n \varphi(x_1, \ldots, x_n) = 1 \]
	\[ \Pi_n-SAT = \forall x_1 \exists x_2 \ldots Q x_n \varphi(x_1, \ldots, x_n) = 1 \]
	Where $Q$ is either $\exists$ of $\forall$ depending on parity of $n$, $\varphi \in CNF$.
\end{definition}

\begin{lemma}[$\Pi_n, \Sigma_n$-complete language]
	$\Sigma_n-SAT$ is $\Sigma_n$-complete.
	$\Pi_n-SAT$ is $\Pi_n$-complete.
\end{lemma}
\begin{proof}
	For $\Sigma_n$ only as proof for $\Pi_n$ is the same.

	Let $L \in \Sigma_n$ arbitrary.
	By the PH definition via alternating quantifiers \cref{ph_altern}:
	\[ \exists p \exists L_{po} \in \TP: x \in L \iff \exists^{p(n)} y_1, \forall^{p(n)} \ldots Q^{p(n)} y_n: (x, y_1, \ldots, y_n) \in L_{po} \]
	Let $M$ be a DTM which accepts $L_{po}$.
	From Cook-Levin theorem we can encode the computation of $M$ by a CNF $\varphi_n$:
	\[ x \in L \iff \exists^{q(|x|)} t_1, \forall^{q(|x|)} \ldots Q^{q(|x|)} t_n: \varphi_n(v, t_1, \ldots, t_n) = 1 \]
	where $v$ encodes input $x$ and $t_i$ encodes $y_i$ (both as binary string).
	All encodings are of size $q(|x|)$.

	$L$ can be a language over any alphabet, however the blowup of binary encoding would be logarithmic with respect to alphabet size.
\end{proof}

\begin{theorem}[Karp-Lipton(1980)]
	$\TNP \subseteq \Ppoly \Rightarrow PH = \Sigma_2^p$.
\end{theorem}
\begin{proof}
	% todo check
	Suffices to prove $\Pi_2 \subseteq \Sigma_2$ rest follows from \cref{ph_collapse}???
	To do that we need to prove
	\[\exists L \in \Pi_2\text{-complete}\ \& L \in \Sigma_2 \]

	Idea: show $\Pi_2$-SAT $\in \Sigma_2$.

	$\Pi_2$-SAT is a language of following CNF formulas:
	\begin{equation}
		\forall x \in \szo^n \exists y \in \szo^n: \varphi(x, y) = 1
		\tag{$*$}
	\end{equation}
	If we fix $x$ we get a language
	\[ L_x = \{ (\varphi, x) |\ \exists y \in \szo^n: \varphi(x, y) = 1 \} \]
	Then
	\[ \forall x: L_x \in \Sigma_1 = \TNP \stackrel{\text{assumption}}{\subseteq} \Ppoly \]
	Therefore $\exists \{ C_k \}: |C_k| = p(k)$ such that
	\[ \forall \varphi_{n + m} \forall x \in \szo^n: C_k(\varphi_{n + m}, x) = 1 \iff \forall y \in \szo^n \varphi(x, y) = 1 \]
	Where $k$ is \# of bits to encode $(\varphi, x)$.

	Main idea: change $\{ C_k \}$ to a new family $\{ C_k^{out} \}$ which also outputs the satisfiable assignment.
	Meaning if $C_k$ outputs 1 then $C_k^{out}$ outputs $y \in \szo^n: \varphi(x, y) = 1$.

	$C_k^{out}$ outputs such $y$ bit by bit by chaining circuits that do $\varphi \land y[i]$ where $i$ is the bit position in $y$.
	We assume $|C_k^{out}| \leq q(n)$.
	We can encode $C_k^{out}$ into a bit string of length $r(k) \leq \bigO(q^2(n))$.

	Then,
	\begin{equation}
		\exists y \in \szo^{r(k)} \forall x \in \szo^n: \varphi(x, y) = C_k^{out}(x, (\varphi, x)) = 1
	\tag{$**$}
	\end{equation}

	Claim $(*) \iff (**)$ since if $(*)$ is true than $\exists$ circuit that on input $(\varphi, x)$ outputs $y$ such that $\varphi(x, y) = 1$.
	If $(*)$ is false $\Rightarrow \exists x: \forall y: \varphi(x, y) = 0$ then $(**)$ is also false.

	Also $(**)$ is a $\Sigma_2$ formula, therefore $\Pi_2$-SAT $\in \Sigma_2$.

\end{proof}

\subsection{Classes AC, NC}
% 9th lecture

\begin{definition}[Class NC]
	$L \in NC^i$ if there exists polynomial size family of circuits $\{ C_n \}_{n \in \N}$ accepting $L$ such that $\forall n$ depth of $C_n \in \bigO(\log^i n)$.

	\[ NC = \bigcup NC^i \]
\end{definition}

\begin{definition}[Class AC]
	$AC^i$ is defined similarly to $NC^i$ but gates may have arbitrary \# of input and output edges.
\end{definition}

\begin{theorem}[NC and AC]\label{ac_dc}
	\[ \forall i: NC^i \subseteq AC^i \subseteq NC^{i + 1} \]
\end{theorem}
\begin{proof}
	By definition:
	\[ \forall i: NC^i \subseteq AC^i \]

	Let $L \in AC^i$ arbitrary, then there is a circuit $\{ C_n \}_{n \in \N}$ with depth $\bigO(\log^i n)$ that recognizes $L$.
	$\{ C_n \}_{n \in \N}$ is a polynomial size family of circuits $\Rightarrow \exists p: |C_n| \leq p(n)$.
	Then, every gate in $C_n$ has fan-in $\leq p(n)$.
	Replace every gate in $C_n$ by a binary tree of fanin-2 gates with depth $\log p(n) \in \bigO(\log n)$.
	% todo picture lecture 9 12:00
	Therefore, we can construct $C_n^{new}$ with fanin-2 of depth $\bigO(\log^{i + 1} n)$ that simulates $C_n \Rightarrow L \in NC^{i + 1}$.
\end{proof}

% book chapter 6.5.1
\subsection{Connection between NC and parallel computing}

\begin{definition}[Parallel computer]
	Has $n$ processors, communication between them with global clock.
	Maximum "distance" (shortest path in graph of processors) between processors is $\bigO(\log n)$.

	For example hypercube architecture: labels of processors are binary numbers with $\log n$ bits.
	Processors are connected if their labels differ by 1 bit, Hamming distance is 1.
\end{definition}

\begin{definition}[Efficient parallel algorithm]
	A computational task is said to have \emph{efficient parallel algorithms} if inputs of size $n$ can be solved using a parallel computer with $n^{\bigO(1)}$ processors and in time $\log^{\bigO(1)} n$.

	Source ~\cite[p.~109]{arora2009computational}.
\end{definition}

\begin{example}[Transitive closure]
	Problem of finding a transitive closure of directed graph can be solved by efficient parallel algorithm.
	Transitive closure adds an $(a, b)$ edge for every $Path(a, b)$.

	Instead of running multiple instances of BFS in parallel, we can multiple the adjacency matrix of a graph.
	The reachability matrix can be constructed in 1 multiplication $\bigO(1)$ using $n^3$ processors in time $\bigO(\log n)$.
\end{example}

Q: Do we assume that every processor outputs 1 bit?\\
A: No, each processor is a TM with output tape. As each of them works in $\log$ time, is uses at most $\log$ space which yields a polynomial size output.

\begin{theorem}[NC and parallel computing]
	$L$ has an efficient parallel algorithm (which recognizes $L$) $\iff L \in NC$.
\end{theorem}
\begin{proof}
	"$\Rightarrow$". Let efficient parallel algorithm for $L$ has $n^c$ processors and works in time $\bigO(\log^d n)$.
	Design a circuit $\{ C_n \}$ with $\bigO(n^c \cdot \log^d n)$ processors.

	"Gates" we construct (more precicely, fixed size circuits) are organized in $\bigO(\log^d n)$ levels.
	Gate at $j$-th level simulates processor in $j$-th step of the computation.
	The input edges comes from the previous levels and corresponds to the communication channels between the processors.

	Then $L \in AC^d$, by \cref{ac_dc} $L \in NC^{d + 1} \Rightarrow L \in NC$.

	"$\Leftarrow$" we have a polynomial size circuit with $\log$ depth.
	Replace every gate by a processor and interconnect them in some standard way.
	Sumilating $C_n$ getting all signals from level $j$ to $(j + 1)$ may take $\bigO(\log n)$ steps of the parallel computing.
	Therefore simulation takes $\bigO(\log^{d + 1} n)$.
\end{proof}

Q: what is the relationship between $NC$ and $\Ppoly$?\\
A: $NC, AC \subseteq \Ppoly$, these classes allow fast computaton. Similar to linear or quadratic subclasses in $\TP$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\chapter{Time Series Analysis}
\label{chap:time_series}
% https://www.youtube.com/playlist?list=PLvcbYUQ5t0UHOLnBzl46_Q6QKtFgfMGc3

Time series analysis is the study of time-dependent data such as price over time.
Similar to regression models we aim to fit a model
to the data in order to make future predictions.
However, instead of interpolating within a range of previously observed data points,
time series aims to extrapolate from our current data into the future,
as we are not interested in making predictions about past dates.
As we extrapolate further and further away from observed data
the uncertainty of our predictions will naturally grow,
making time series a harder problem to address than normal regression.
There are a variety of potential models to choose from,
each with their own assumptions and use cases.
The problem can be attacked from two general directions,
frequency-domain methods and time-domain methods,
but we shall focus primarily on the time-domain here.
The \texttt{statsmodels} \href{https://www.statsmodels.org}{python module}
is very useful for implementing these modeling methods,
which will be referenced as appropriate.

\subsubsection{Lag Notation}
\label{time_series:L}

When working with time series data we often index time steps as $t$, $t-1$, $t-2$,\,\ldots\,
where $t$ is the current time.
The value of the variable in question, price, quantity, \etc,
at time $t$ is then $y_{t}$, with $y_{t-1}$, $y_{t-2}$,\,\ldots\, at subsequent times.
Instead of writing subscripts, sometimes it is more convenient
to switch to a polynomial based notation using
the lag operator $L$, also known as the backshift operator $B$ in some texts:

\begin{equation}\label{eq:time_series:L}
L y_{t} = y_{t-1},\quad L^{2} y_{t} = y_{t-2},\,\ldots \quad L^{k} y_{t} = y_{t-k}.
\end{equation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Correlation in Time Series}
\label{time_series:correlation}

In time series analysis $y_{t}$ can depend, to varying degree,
on the previous values $L^{k} y_{t},\, \forall k$.
There is a direct component to this dependence,
$y_{t} \sim f\left(y_{t-1}\right)$,
but also an indirect or recursive component,
$y_{t} \sim f\left(y_{t-1}\right) \sim f\left(g\left(y_{t-2}\right)\right)$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Auto-Correlation Function (ACF)}
\label{time_series:ACF}

Neglecting the multiple components of the correlation of $y_{t}$ with prior times,
we can commute the overall auto-correlation function (ACF) of $y_{t}$
with itself at some amount of lag $k$ as

\begin{equation}\label{eq:time_series:ACF}
\text{ACF}\left(y,k\right) = \corr{y_{t}}{y_{t-k}},
\end{equation}

\noindent using the regular Pearson correlation coefficient \cref{eq:stats:corr:pearson}.
The auto-correlation shows the total correlation and is simple to compute,
but mixes the direct and indirect effects of prior values together.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Partial Auto-Correlation Function (PACF)}
\label{time_series:PACF}

In contrast, partial auto-correlation function
aims to isolate the direct influence of $y_{t-k}$ on $y_{t}$ for some lag $k$.
We can get at this component by fitting $y_{t}$ as a linear function of $L^{n} y_{t}$,

\begin{equation}\label{eq:time_series:PACF}
y_{t} = \sum_{i=1}^{k} \phi_{k,i}\, y_{t-i} + \epsilon_{k},
\end{equation}

\noindent and identifying $\text{PACF}\left(y,k\right) = \phi_{k,k}$
as the partial auto-correlation function for $k$.
Note that for each $k$, we need to do a new fit of \cref{eq:time_series:PACF},
and after fitting should check that $\phi_{k,k}$ is statistically significant,
\ie we reject $H_{0}$ that $\phi_{k,k}=0$, by looking at the coefficient's confidence interval from the fit.
We can generate plots of ACF and PACF vs $k$ to explore a time series
with the \texttt{plot\_acf} and \texttt{plot\_pacf}
\href{https://www.statsmodels.org/stable/graphics.html\#time-series-plots}{functions}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Stationarity}
\label{time_series:stationarity}

Stationarity is an important assumption for many parametric time series models,
which combines three properties:

\begin{enumerate}[noitemsep]
  \item The mean $\expval{y}$ of the time series is constant, \ie the average is not moving.\label{item:time_series:stationarity:constant_mean}
  \item The variance $\variance{y}$ of the time series is constant.\label{item:time_series:stationarity:constant_var}
  \item The time series does not exhibit seasonality, \ie no periodic fluctuations in $y\left(t\right)$.\label{item:time_series:stationarity:seasonality}
\end{enumerate}

If the mean or variance is growing with time
we can look at the first difference, $\widetilde{y}_{t} = y_{t} - y_{t-1}$,
and see if the time series of $\widetilde{y}_{t}$ is stationary,
see \cref{time_series:I} for more.
Similarly, we can try to remove seasonality
by subtracting a deterministic time-dependent function $f\left(t\right)$
or by using the SARIMA model of \cref{time_series:SARIMA}.
We can also try normalizing with a \Zscore to address a changing mean or variance;
subtracting or dividing as needed to make the series more stationary.
This even works with seasonality if we subtract the mean value per medium-sized time unit, \eg month, quarter, \etc.
When cleaning the data in this way,
we will need to invert any preprocessing steps
while making predictions to return to the original units.

Simply looking at the graph of a time series can give us
a good intuition on if it is stationary or not.
However, we can formally test for stationarity
with the augmented Dickey--Fuller test (ADF) of \cref{time_series:ADF}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{White Noise}
\label{time_series:white_noise}

In the context of time series analysis, white noise is random fluctuations which
have mean zero, $\expval{y} = 0$,
constant variance with respect to time, $\partial_{t} \variance{y} = 0$,
and no correlation between lags, $\corr{y_{t}}{y_{t-k}} = 0,\,\, \forall k$.
Ideally after fitting a time series, we will capture all of the signal
in the model and the resulting error term $\epsilon$ will only contain white noise.
We can test for white noise by looking at the moving average and ACF plot,
or more formally with the Ljung--Box test of \cref{time_series:ljung_box}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Autoregressive (AR) Models}
\label{time_series:AR}
Building on the PACF, we can construct an autoregressive (AR) model $\text{AR}\left(p\right)$
from multiple lag components of the time series:

\begin{equation}\label{eq:time_series:AR}
\yhat_{t} = \phi_{0} + \sum_{i=1}^{p} \phi_{i}\, y_{t-i} + \epsilon_{t}.
\end{equation}

Note that we should look at the PACF plot before fitting \cref{eq:time_series:AR}
to determine which lags to include,
\ie set $p$ and decide which lags to skip\footnote{It may not be possible
to specify individual lags to include in some software packages,
but excluding non-significant coefficients and reducing model complexity
is still a good idea if possible.} by setting $\phi_{i} = 0$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Moving Average (MA) Models}
\label{time_series:MA}

Instead of looking at the auto-correlation $y_{y-i}$,
we can also build a time series model around
correcting for prior errors in the prediction,
\ie fitting to the moving average (MA).
The $\text{MA}\left(q\right)$ model
is then:

\begin{subequations}\label{eq:time_series:MA}
\begin{align}
\yhat_{t} &= \theta_{0} + \sum_{i=1}^{q} \theta_{i}\, \epsilon_{t-i} + \epsilon_{t}, \label{eq:time_series:MA_y} \\
\epsilon_{t-1} &= y_{t-1} - \yhat_{t-1}. \label{eq:time_series:MA_epsilon}
\end{align}
\end{subequations}

Note that a $\text{MA}\left(q\right)$ model will fluctuate around the initial average $\theta_{0}$,
and can only predict $q$ steps into the future before constantly returning $\theta_{0}$.
We can select $q$ by looking at the ACF plot
and identifying the last non-zero lag.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Invertibility of Time Series}
\label{time_series:invert}
% https://youtu.be/QU_VNu3rJKY
% https://youtu.be/q0vz7dGlZL0

One interesting property of AR and MA time series models is their invertibility:

\begin{subequations}\label{eq:time_series:invert}
\begin{align}
\text{MA}\left(1\right) &= \text{AR}\left(\infty\right), \label{eq:time_series:invert:MA_1} \\
\text{AR}\left(1\right) &= \text{MA}\left(\infty\right). \label{eq:time_series:invert:AR_1}
\end{align}
\end{subequations}

For a proof of \cref{eq:time_series:invert:MA_1}, see \cref{eq:time_series:invert_proof}.
Letting $\theta = \phi < 0$ for convenience and
assuming\footnote{Here we are assuming the time series is stationary, \ie no unit roots $\abs{\phi} < 1$.} $\abs{\phi} < 1$
we can construct the time series $C_{t}$,

\begin{subequations}\label{eq:time_series:invert_proof}
\begin{align}
C_{t} \equiv \text{MA}\left(1\right) - \theta_{0} &= -\phi\, \epsilon_{t-1} + \epsilon_{t} = \left(1 - \phi L \right) \epsilon_{t}, \label{eq:time_series:invert_proof:a} \\
\implies \epsilon_{t} &= \frac{C_{t}}{1 - \phi L}, \label{eq:time_series:invert_proof:b} \\
&= \left(\,\sum_{i=0}^{\infty} \phi^{i} L^{i} \right) C_{t} = \sum_{i=0}^{\infty} \phi^{i} C_{t-i}, \label{eq:time_series:invert_proof:c} \\
\implies C_{t} &= \sum_{i=1}^{\infty} -\phi^{i} C_{t-i} + \epsilon_{t} = \text{AR}\left(\infty\right) - \phi_{0}, \label{eq:time_series:invert_proof:d}
\end{align}
\end{subequations}

\noindent where in \cref{eq:time_series:invert_proof:c} we have used
the sum of a geometric series \cref{eq:misc:math:geometric}.
The proof of \cref{eq:time_series:invert:AR_1} is similar and also relies on the geometric series.
Essentially we are making the recursive nature of these time series explicit and using it to transform from one to the other.
The $\text{AR}\left(\infty\right)$ expansion of $\text{MA}\left(1\right)$
is particularly useful as it is possible in practice to truncate the series at $\order{\phi^{\,p}}$
where $\phi^{\,p} \approx 0$ and represent $\text{MA}\left(1\right)$ as
a pure function of the time series itself, \ie without recursion.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{ARMA Model}
\label{time_series:ARMA}

While we can represent AR $\Leftrightarrow$ MA with infinite series,
it is more practical to include both in one model with a finite number of terms.
The $\text{ARMA}\left(p,q\right)$ model is a simple combination of
both approaches:

\begin{equation}\label{eq:time_series:ARMA}
\yhat_{t} = \mu_{0} + \sum_{i=1}^{p} \phi_{i}\, y_{t-i} + \sum_{i=1}^{q} \theta_{i}\, \epsilon_{t-i} + \epsilon_{t}.
\end{equation}

Using lag notation we can write \cref{eq:time_series:ARMA} more
compactly\footnote{We drop
the constant intercept terms for convenience, as do many other sources.
However a constant term can still be included if desired,
or we can construct $\widetilde{y}_{t} = y_{t} - \mu_{0}$ with $\expval{\widetilde{y}_{t}} = 0$.}\footnote{If we set
$\phi_{0}=-1$, $\theta_{0} = 1$ and start the sums at $i=0$
we can simplify \cref{eq:time_series:ARMA_lag} even further.} as

\begin{subequations}\label{eq:time_series:ARMA_lag}
\begin{align}
\phi\left(L,p\right) y_{t} &= \theta\left(L,q\right) \epsilon_{t}, \label{eq:time_series:ARMA_lag:def} \\
\phi\left(L,p\right) &= 1 - \sum_{i=1}^{p} \phi_{i}\, L^{i}, \label{eq:time_series:ARMA_lag:phi} \\
\theta\left(L,q\right) &= 1 + \sum_{i=1}^{q} \theta_{i}\, L^{i}, \label{eq:time_series:ARMA_lag:theta}
\end{align}
\end{subequations}

\noindent where \cref{eq:time_series:ARMA_lag:phi,eq:time_series:ARMA_lag:theta} are polynomials of the lag operator $L$.
To avoid parameter redundancy, \ie multiple $\phi$, $\theta$ coefficients for $\order{L^{n}}$,
we require the polynomials $\phi\left(L,p\right)$ and $\theta\left(L,q\right)$ to not share any common factors.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Integrated (I) Models}
\label{time_series:I}

The previous models have worked well for stationary time series with constant means,
however we may wish to model a quantity with a mean that linearly changes over time.
One method of incorporating this behavior in a time series model is by
recursively taking the difference $d$ times between lags of the original time series $y_{t}$
to construct a new time series $\widetilde{y}_{t}$ \cref{eq:time_series:I:forward}.
With the appropriate choice of $d$, typically $d \leq 2$,
$\widetilde{y}_{t}$ should have a constant mean,
allowing other models to be subsequently used.
This process is known as the integrated model $\text{I}\left(d\right)$,
though the name is misleading as what we are really doing is differentiating $y$ to find a constant slope, curvature, \etc.
Note, when making predictions we will need to
transform back to the original $y_{t}$ \cref{eq:time_series:I:backward1,eq:time_series:I:backward2},
at the cost of $d$ initial data points where no predictions can be made.

\begin{subequations}\label{eq:time_series:I}
\begin{align}
\widetilde{y}_{t} &= \left(1-L\right)^{d} y_{t}, \label{eq:time_series:I:forward} \\
y_{t} &= \widetilde{y}_{t} + y_{t-1},\,\, \text{where}\,\, d=1, \label{eq:time_series:I:backward1} \\
y_{t} &= \widetilde{y}_{t} + 2 y_{t-1} - y_{t-2},\,\, \text{where}\,\, d=2. \label{eq:time_series:I:backward2}
\end{align}
\end{subequations}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{ARIMA Model}
\label{time_series:ARIMA}

Putting everything together, we can construct the $\text{ARIMA}\left(p,d,q\right)$ model
by including \cref{eq:time_series:I:forward} in \cref{eq:time_series:ARMA_lag}:

\begin{equation}\label{eq:time_series:ARIMA}
\phi\left(L,p\right) \left(1-L\right)^{d} y_{t} = \theta\left(L,q\right) \epsilon_{t}.
\end{equation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{ARIMA Model Example}
\label{time_series:ARIMA:ex}

% TODO include example plots of y_{t}, PACF, ACF, predictions, and residuals; and link to the example notebook

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Seasonal ARIMA Model (SARIMA)}
\label{time_series:SARIMA}

Extending the ideas of the integrated model from \cref{time_series:I},
we can attempt to control for seasonality by
taking lags across $m$ time steps at once, \ie using $L^{m}$ in place of $L$.
For example, if $m=12$ corresponds to 1 year,
we will be doing the normal extrapolation of ARIMA on $y_{t-1-1\,\text{Year}}$ instead of $y_{t-1}$,
thereby canceling out any yearly periodic trends.
The resulting $\text{SARIMA}\left(p,d,q\right)\left(P,D,Q\right)_{m}$ model
has the normal $\text{ARIMA}\left(p,d,q\right)$ model components,
plus an additional set\footnote{Details around
coefficient conditions and parameter redundancy are omitted for brevity.} of uppercase parameters
controlling a $L^{m}$ based ARIMA model:

\begin{equation}\label{eq:time_series:SARMA}
\phi\left(L,p\right) \Phi\left(L^{m},P\right) \left(1-L\right)^{d} \left(1-L^{m}\right)^{D} y_{t} = \theta\left(L,q\right) \theta\left(L^{m},q\right) \epsilon_{t}.
\end{equation}

The $\text{SARIMA}\left(p,d,q\right)\left(P,D,Q\right)_{m}$ family of models
is available via the \texttt{ARIMA}
\href{https://www.statsmodels.org/stable/generated/statsmodels.tsa.arima.model.ARIMA.html}{class},
with a wide variety of attributes and methods
enabling us to implement all of the above time series models.
Computationally, \texttt{ARIMA} preforms fits via maximum likelihood estimation (MLE),
but AR models can also be fit with ordinary least squares (OLS).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{ARMA Model with Exogenous Inputs (ARMAX)}
\label{time_series:ARMAX}

The $\text{ARMAX}\left(p,q,b\right)$ model
extends ARMA by incorporating exogenous (X) inputs
via another known time series $d_{t}$:

\begin{equation}\label{eq:time_series:ARMAX}
\yhat_{t} = \mu_{0} + \sum_{i=1}^{p} \phi_{i}\, y_{t-i} + \sum_{i=1}^{q} \theta_{i}\, \epsilon_{t-i} + \sum_{i=1}^{b} \eta_{i}\, d_{t-i} + \epsilon_{t}.
\end{equation}

We can think of the $d_{t}$ time series like the driving force of a driven harmonic oscillator.
In practice, ARMAX usually is implemented as ARMA on
a transformed $y_{t}$ which takes into account different lags of $d_{t}$,
so careful interpretation of a given software package's outputs and documentation is required.
The \texttt{ARIMA}
\href{https://www.statsmodels.org/stable/generated/statsmodels.tsa.arima.model.ARIMA.html}{class}
can incorporate an exogenous time series,
as well as deterministic polynomials of $t$,
\ie the $f\left(t\right)$ mentioned in \cref{time_series:stationarity}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Other Models (VAR, ARCH, GARCH, Bayesian Approaches)}
\label{time_series:other_models}

In addition to the SARIMAX family of models detailed above,
there are a few other useful time series models to note here.
The vector autoregression (VAR) model is a matrix based
generalization of $\text{AR}\left(p\right)$ to multiple dimensions.
See the \texttt{vector\_ar}
\href{https://www.statsmodels.org/stable/generated/statsmodels.tsa.vector_ar.var_model.VAR.html}{class} for more.

The autoregressive conditional heteroskedasticity (ARCH) and generalized ARCH (GARCH) models,
popular in finance, are appropriate when the underlying time series
exhibits periodic episodes of increased variance\footnote{Conditional heteroskedasticity meaning periodically differing variances.}.
These models predict the volatility $\sigma_{t}^{2}$ as a function of
the prior errors $\epsilon_{t-1}$ and variances $\sigma_{t-1}^{2}$
which allow them to cope with ``bursty'' periods of increased volatility.
See the \texttt{arch}
\href{https://arch.readthedocs.io/en/stable/univariate/introduction.html}{package}
for one implementation.

We can also look at time series analysis within a Bayesian framework;
assuming prior distributions for the model parameters $\vb*{\beta}$
then updating based on the observed data,
thereby producing distributions of our predictions and parameters instead of only MLE point estimates.
The \pymcThree \href{https://docs.pymc.io}{module}
has \href{https://docs.pymc.io/api/distributions/timeseries.html}{timeseries functions}
for implementing this approach, including AR and GARCH models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Evaluating Time Series Models}
\label{time_series:eval}

Like other machine learning problems, we must split our data into train and test sets
when fitting and evaluating time series models.
As we are extrapolating we use the first
$\approx \SI{70}{\percent}$ of the data by time for the training set,
and the reminding $\approx \SI{30}{\percent}$ for the test set,
instead of randomly splitting the data as is done for interpolative models.
To improve on this approach even further, we can use
rolling forecast origin techniques to evaluate the model.
In short, we use the first $k$ data points to train the model,
evaluate it on the next $k + l$ data points, increment $k$ and reiterate.
Performance metrics can then be averaged across the iterations.
In this way the model's near term predictions at each iteration are as accurate as possible,
as if time had continued and we received more data,
giving a better estimate of the predictive performance in practice.
There are a few different methods for how to exactly roll the origin,
see \cref{fig:rolling_forecast_origin} for details.

\begin{figure}
\centering
\includegraphics[width=0.6\textwidth]{figures/stats/rolling_forecast_origin}
\caption{
Illustration of multiple test train split methods for time series models \cite{bergmeir_dissertation}.
Note that $t$ is increasing along the $x$-axis.
In the fixed origin method, we do not do any retraining and the prediction quality
can be expected to drop as we go further into the future.
In the rolling origin update method, we do not retrain the model parameters $\vb*{\beta}$
with each iteration, but do update the $y_{i}$ lags.
In the rolling origin recalibration method we fully retrain the model on each iteration.
Lastly, in the rolling window method we remove data from the beginning of the training period
as we iterate to keep the training set a constant size.
This can help with some statistical interpretations of the model,
but also helps when working with slowly changing non-stationary data.
}
\label{fig:rolling_forecast_origin}
\end{figure}

Regardless of the test train split method, we can get a sense of the model's performance
qualitatively by looking at residual plots of $y_{t} - \yhat_{t}$.
The residuals will show any remaining systematic trends that the model did not learn,
as ideally they would be white noise.
Quantitatively, we can test
if the residuals are white noise with the Ljung--Box test of \cref{time_series:ljung_box},
as well as can compute the
root mean squared error\footnote{Also known as the root mean squared deviation (RMSD).} (RMSE)
and mean absolute percent error (MAPE)
between the predictions and observed values:

\begin{subequations}\label{eq:time_series:RMSE_MAPE}
\begin{align}
\text{RMSE}\left(\yhat\right) &= \sqrt{\expval{\left(y - \yhat\right)^{2}}} = \sqrt{\frac{1}{n}\sum_{t=1}^{n} \left(y_{t} - \yhat_{t}\right)^{2}}, \label{eq:time_series:RMSE} \\
\text{MAPE}\left(\yhat\right) &= \frac{\SI{100}{\percent}}{n} \sum_{t=1}^{n} \abs{\frac{y_{t} - \yhat_{t}}{y_{t}}}\,. \label{eq:time_series:MAPE}
\end{align}
\end{subequations}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Box--Jenkins Method (Expanded)}
\label{time_series:box_jenkins}

The Box--Jenkins method, popularized in their book \cite{boxjen76},
is a recipe for fitting an ARMA-based model to a given time series $y_{t}$.
The steps have been expanded here to make the entire process
for working with time series data more explicit:

\begin{enumerate}[noitemsep]
  \item Do any necessary pre-cleaning of $y_{t}$ to normalize the data.\label{item:time_series:box_jenkins:cleaning0}
  \begin{enumerate}[noitemsep]
    \item \Zscore normalization, deterministic transforms $f\left(t\right)$, \etc.\label{item:time_series:box_jenkins:cleaning0:detail}
    \item Also, split the training and test sets.\label{item:time_series:box_jenkins:cleaning0:train_test_split}
  \end{enumerate}
  \item Check the stationarity and seasonality of the cleaned $y_{t}$.\label{item:time_series:box_jenkins:assumptions}
  \begin{enumerate}[noitemsep]
    \item Look at $y_{t}$ and the MA plot for obvious problems.\label{item:time_series:box_jenkins:assumptions:eye}
    \item Use the augmented Dickey--Fuller test (ADF) for stationary.\label{item:time_series:box_jenkins:assumptions:ADF}
    \item Correct any remaining problems with a ARIMA, $1 \leq d$, or SARIMA model.\label{item:time_series:box_jenkins:assumptions:cleaning1}
  \end{enumerate}
  \item Select the appropriate ARMA model, \ie choose $p$ and $q$.\label{item:time_series:box_jenkins:order}
  \begin{enumerate}[noitemsep]
    \item The PACF plot gives clues to the $\text{AR}\left(p\right)$ order.\label{item:time_series:box_jenkins:order:PACF}
    \item The ACF plot gives clues to the $\text{MA}\left(q\right)$ order.\label{item:time_series:box_jenkins:order:ACF}
  \end{enumerate}
  \item Fit the chosen model.\label{item:time_series:box_jenkins:fit}
  \item Verify the fitted model by checking the residuals. Return to \cref{item:time_series:box_jenkins:cleaning0} if needed.\label{item:time_series:box_jenkins:verify}
  \begin{enumerate}[noitemsep]
    \item Use the Ljung--Box test for white noise on the residuals.\label{item:time_series:box_jenkins:verify:Ljung_Box}
    \item Measure performance with RMSE and MAPE.\label{item:time_series:box_jenkins:verify:RMSE_MAPE}
    \item Measure the relative performance of multiple models with AIC and BIC.\label{item:time_series:box_jenkins:verify:AIC_BIC}
  \end{enumerate}
  \item Invert any cleaning steps to return to the original units of $y_{t}$ for predictions.\label{item:time_series:box_jenkins:invert}
\end{enumerate}

In \cref{item:time_series:box_jenkins:order} a simple approach is to
go out to the last significant non-zero order of $p$ ($q$) in the PACF (ACF) plot.
However, interpreting the PACF and ACF plots is a developed skill
and there are many different pieces of advice
available\footnote{See
\href{https://people.duke.edu/~rnau/411arim3.htm}{here}
and the Box--Jenkins method
\href{https://en.wikipedia.org/wiki/Box\%E2\%80\%93Jenkins\_method\#Autocorrelation\_and\_partial\_autocorrelation\_plots}{wiki page}
for just two examples.}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Unit Roots}
\label{time_series:unit_root}

A time series with $\abs{\phi} = 1$ for the leading AR coefficient
is said to have a unit root and is non-stationary.
More formally, we can construct the characteristic
equation\footnote{As if we were solving a
homogeneous linear ordinary differential equation with constant coefficients.} \cref{eq:time_series:unit_roots:characteristic}
for a $\text{AR}\left(p\right)$ time series $y_{t}$ and look for roots $m$.
If $m=1$ is one of the roots, the time series then has a unit root.

\begin{subequations}\label{eq:time_series:unit_roots}
\begin{gather}
y_{t} = \sum_{i=1}^{p} \phi_{i}\, y_{t-i} + \epsilon_{t}\,, \label{eq:time_series:unit_roots:y} \\
\implies m^{p} - \left(\, \sum_{i=1}^{p} \phi_{i}\, m^{i-1} \right) = 0. \label{eq:time_series:unit_roots:characteristic}
\end{gather}
\end{subequations}

Time series with unit roots are non-stationary,
as can be illustrated with the $\text{AR}\left(1\right)$ series\footnote{The expansion
in \cref{eq:time_series:unit_root_ex:expand} is really
$\text{AR}\left(1\right) = \text{MA}\left(\infty\right)$ \cref{eq:time_series:invert:AR_1} written out.}:

\begin{subequations}\label{eq:time_series:unit_root_ex}
\begin{align}
y_{t}
= \phi\, y_{t-1} + \epsilon_{t}
&= \phi \left(\phi\, y_{t-2} + \epsilon_{t-1}\right) + \epsilon_{t}
= \cdots
= \phi^{t} y_{0} + \sum_{i=0}^{t-1} \phi^{i} \epsilon_{t-i}\,, \label{eq:time_series:unit_root_ex:expand} \\
\expval{y_{t}}
&= \phi^{t} \expval{y_{0}} + \sum_{i=0}^{t-1} \phi^{i} \expval{\epsilon_{t-i}}
= \phi^{t} y_{0}\,, \label{eq:time_series:unit_root_ex:expval} \\
\variance{y_{t}}
&= \phi^{2t}\, \variance{y_{0}} + \sum_{i=0}^{t-1} \phi^{2i}\, \variance{\epsilon_{t-i}}
= \sigma^{2} \sum_{i=0}^{t-1} \phi^{2i}. \label{eq:time_series:unit_root_ex:var}
\end{align}
\end{subequations}

\noindent Note that we are assuming the errors only contain white noise,
and thus are uncorrelated with constant variance $\variance{\epsilon} = \sigma^{2}$.
When $\abs{\phi} < 1$, $\expval{y_{t}} \to 0$ and
$\variance{y_{t}} \to \sigma^{2}/\left(1-\phi^{2}\right)$ are constant
as $t \to \infty$, and the time series is stationary.
When $1 < \abs{\phi}$, $\expval{y_{t}} = \phi^{t} y_{0}$
will diverge as $t \to \infty$, and the time series is clearly non-stationary\footnote{Technically,
the time series is an ``explosive process'', more than just non-stationary.}.
When we have a unit root $\abs{\phi} = 1$,
$\expval{y_{t}} = y_{0}$ is constant\footnote{Making these unit roots harder to identify visually.},
but $\variance{y_{t}} = t \sigma^{2}$ is not constant and the time series is non-stationary.

Lastly, a time series with unit roots can be made stationary through
differencing\footnote{$d_{t} \equiv y_{t} - y_{t-1} = \epsilon_{t}$, $\expval{d_{t}} = 0$, $\variance{d_{t}} = \sigma^{2}$.}.
In particular, if the $m=1$ unit root of the characteristic equation has multiplicity $d$,
a $\text{I}\left(d\right)$ model will make the time series stationary.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Augmented Dickey--Fuller (ADF) Test for Stationarity}
\label{time_series:ADF}

The augmented\footnote{The ADF is an extension of the normal Dickey--Fuller test where $p = 1$.} Dickey--Fuller (ADF) test
is a hypothesis tests for the stationarity of a time series.
We begin by assuming the time series for $y$ is a $\text{AR}\left(p\right)$ model \cref{eq:time_series:AR}
and construct the difference:

\begin{subequations}\label{eq:time_series:ADF}
\begin{align}
\Delta y_{t} &= y_{t} - y_{t-1} = \phi_{0} + \delta y_{t-1} + \sum_{i=2}^{p} \phi_{i}\, y_{t-i} + \epsilon_{t}, \label{eq:time_series:ADF:Delta_y} \\
\delta &= \phi_{1} - 1. \label{eq:time_series:ADF:delta}
\end{align}
\end{subequations}

Now we test $H_{0}$ that $0 \leq \delta$ and the time series is
non-stationary\footnote{As $\delta = 0 \implies \phi_{1} = 1$,
\ie the time series has a unit root and is non-stationary.
Note that it is possible to have a non-stationary time series without a unit root,
thus the ADF test is not fully comprehensive.} vs
the alternative hypothesis $H_{a}$ that $\delta < 0$, \ie time series is stationary.
We fit the data and then perform the hypothesis testing with test statistic
$t_{\hat{\delta}} = \hat{\delta} / \text{Standard Error}\left(\hat{\delta}\right)$
by comparing to the Dickey--Fuller distribution\footnote{We can not use the standard \tdist as $y_{t-1}$ is still assumed to be non-stationary under $H_{0}$.}.
If we can reject $H_{0}$ we therefore have shown that $y_{t}$ is stationary.
Note we can further extend the ADF test by replacing the constant $\phi_{0}$
with a deterministic, quadratic, time trend $\phi_{0} + a_{1} t + a_{2} t^{2}$.
The ADF test is available via the \texttt{adfuller}
\href{https://www.statsmodels.org/stable/generated/statsmodels.tsa.stattools.adfuller.html}{function}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Ljung--Box Test for White Noise}
\label{time_series:ljung_box}

The Ljung--Box test tests if a time series $y$ solely consists of white noise
by looking at multiple lags of the ACF.
The test statistic is

\begin{equation}\label{eq:time_series:ljung_box:Q}
Q = n \left(n+2\right) \sum_{i=1}^{h} \frac{\corr{y_{t}}{y_{t-i}}^{2}}{n-i}
\end{equation}

\noindent where $n$ is the sample size and $h$ is the number of lags to be tested.
Using the \chiSqdist with $\nu = h$ degrees of freedom we find the critical $Q_{c}$
and reject $H_{0}$, the data is white noise, if $Q_{c} < Q$.
If we fail to reject $H_{0}$, \ie $Q \leq Q_{c}$, we have shown $y$ is white noise.
Note that if we are performing the Ljung--Box test on the remainders of a fitted model, such as $\text{ARIMA}\left(p,d,q\right)$,
we must adjust the degrees of freedom accordingly, $\nu = h - p - d - q$.
The Ljung--Box test can be performed with the \texttt{acorr\_ljungbox}
\href{https://www.statsmodels.org/stable/generated/statsmodels.stats.diagnostic.acorr_ljungbox.html}{function}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Granger Causality}
\label{time_series:granger_causality}

When one time series, $x_{t}$, appears to
predict the behavior of another time series, $y_{t}$,
we can test for Granger causality \cite{10.2307/1912791}.
We begin by constructing two $\text{AR}\left(p\right)$ models:

\begin{subequations}\label{eq:time_series:granger_causality}
\begin{align}
\text{AR}_{y}\left(p_{y}\right) &= \phi_{y}\left(L,p_{y}\right) y_{t} + \epsilon_{t}\,, \label{eq:time_series:granger_causality:y} \\
\text{AR}_{y,x}\left(p_{y}, p_{x}\right) &= \phi_{y}\left(L,p_{y}\right) y_{t} + \phi_{x}\left(L,p_{x}\right) x_{t} + \epsilon_{t}\,, \label{eq:time_series:granger_causality:yx}
\end{align}
\end{subequations}

\noindent where $\text{AR}_{y}$ is only a function of the $y_{t}$ time series,
and $\text{AR}_{y,x}$ is a function of both the $y_{t}$ and $x_{t}$ time series.
For both models we test the $\phi$ coefficients for significance\footnote{The
individual coefficient significances are determined via \ttest methods,
while a \Ftest of lack-of-fit sum of squares, \cref{hypo:F_test_fit},
is used to test the model fit as a whole.} individually,
removing any non-significant lags, and collectively for the whole model.
After this process, if $\text{AR}_{y,x}$ still contains $x_{t}$ terms
and is found to fit $y$ better than $\text{AR}_{y}$,
we can say that $x_{t}$ Granger-causes $y_{t}$.
More formally, the null hypothesis $H_{0}$ that $x_{t}$ does not Granger-cause $y_{t}$,
is accepted if and only if all $\phi_{x}$ terms have been removed from the final model.

A confirmation of Granger causality does not imply ``true causality'' in the general sense,
it is merely a statement that a linear model of $x_{t}$ forecasts, or predicts, $y_{t}$.
For example, there maybe latent confounding effects behind both $x_{t}$ and $y_{t}$
which would require a more sophisticated analysis to detect or rule out.
Even with this limited causal interpretation,
there are still challenges \cite{GRASSMANN2020} to the validity of Granger causality.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Anomaly Detection}
\label{time_series:anomaly_detection}

% TODO
% https://youtu.be/XPwCo4cqqt0

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Kalman Filters}
\label{time_series:kalman_filters}
% TODO


\section{Definition}
\label{sec:definition}

ARX describes a family of lightweight cryptographic schemes. In particular, an
instance of ARX cipher consists of only three kinds of primitives: modular addition,
circular bit-shifting and bitwise xor. Formally, the ARX cipher family is defined
as follow:

\begin{definition}
A scheme $\Pi=(\mathit{Gen},\mathit{Enc},\mathit{Dec})$, with plaintext
space and ciphertext space $\mathcal{P}=\mathcal{C}\subseteq\mathbb{Z}_{2^n}^*$, is an
ARX cipher of security parameter $n$, if and only if all the three functions are
composites of the following three primitives:
\begin{itemize}
\setlength\partopsep{0em}
\setlength\topsep{0em}
\setlength\itemsep{0em}
\setlength\parskip{0em}
\item {\bf Addition}: binary operator of modular addition on $\mathbb{Z}_{2^n}$,
whose operands $x,y\in\mathbb{Z}_{2^n}$, denoted by $x+y$
\item {\bf Rotation}: intraword \textbf{left} rotation by $b$ bits to a quantity $x$
such that $b\in\mathbb{Z}_n$ and $x\in\mathbb{Z}_{2^n}$, denoted by $\mathrm{r}^bx$
\item {\bf Xor}: bitwise xor operation on $\mathbb{Z}_{2^n}$, denoted by $x\oplus y$
\end{itemize}
Denote this scheme by $\Pi\in\textit{ARX}(n)$. When there are no confusions, we can
also say the function $\mathit{Enc}$ and $\mathit{Dec}$ are $\textit{ARX}(n)$, denote
by $\mathit{Enc},\mathit{Dec}\in\textit{ARX}(n)$.
\end{definition}

The definition above does not distinguish the operands for addition and xor. In other
word, for operation $x+y$, $x$ can either be a word in the ciphertext, or a constant,
the same as $y$. The operands for $\oplus$ can also be ciphers or constants. As for
the operation $\mathrm{r}^bx$, the shift offset $b$ must be a constant in $\mathbb{Z}_n$.

The security parameter $n$ here does not correspond to the key size or block size of
an encryption scheme. Typically a block cipher involves a key scheduling function
which derives the round keys for each iteration from a fixed-sized key (the
\textit{seed}). The value of $n$ specifies how the round keys are calculated regarding
the quantity's word length, while the key size is the length of the seed. The block
size can also distinct from $n$, as DES (the Feistel construction) encrypts 64-bits
cipher blocks while the round function is a permutation on 32-bits strings.

As most lightweight ciphers adopt S-boxes as the nonlinear components, though ARX does
not provide such component. ARX obtains its nonlinearity solely from the modular
additions, hence may be vulnerable to rotational cryptanalysis. The trade-off here is,
ARX has more compact implementation, while S-box generally consumes more memory and
persistent storage. This property means ARX is suitable for most constrained platforms,
which are designed to equip with scarce resources. For example, a processing unit in a
wireless sensor network typically has several KBs memory for code and data, while their
OS code occupies nearly half of the available space \cite{akyildiz2002wireless}.

ARX can be realized by in-place operations, while each one can be run in constant time.
These two properties bring high performance to ARX ciphers on both software and hardware
levels. Table \ref{tab:arx} shows how these operations can be realized by in-place
Python statement and single MIPS32 instruction.

\begin{table*}[t]
\caption{Implementing in-place ARX operations by Python statements and MIPS32 instructions}
\label{tab:arx}
\medskip
\begin{center}
\begin{tabular}{c|l|l|l}
\hline
             & notation & Python statement & MIPS32 instruction \\ \hline
\textbf{Add} & $x=x+y$ & \texttt{x = (x + y) \% 2**n} & \texttt{add \$x, \$x, \$y} \\
\textbf{Rot} & $x=\mathrm{r}^bx$ & \texttt{x = ((x>>(n-b))|(x<<b)) \% 2**n} & \texttt{rol \$x, \$x, \#b} \\
\textbf{Xor} & $x=x\oplus y$ & \texttt{x = x \textasciicircum\ y} & \texttt{xor \$x, \$x, \$y} \\ \hline
\end{tabular}
\end{center}
\end{table*}

Another important note is that, running single operation in constant time mitigates
the possibility from suffering timing side-channel attacks. The abandon of lookup tables
(such as S-box) also prevents the system from \textit{cache-timing attacks}
\cite{bonneau2006cache}. The properties of ARX cipher show that that it is suitable on
most constrained computing environments, where the resources are limited but the secrecy
is still essential for various tasks. Also, it is efficient at software level hence
it is able to provide secure transmission for IO-intensive channels. Later in Section
\ref{sec:instances}, two such instances of ARX ciphers will be introduced.