\documentclass[a4paper]{article}

\def\npart{II}

\def\ntitle{Probability and Measure}
\def\nlecturer{E.\ Breuillard}

\def\nterm{Michaelmas}
\def\nyear{2018}

\input{header}

\renewcommand{\P}{\prob} % probability measure
\DeclareMathOperator{\var}{Var} % variance
\DeclareMathOperator{\essup}{essup} % essential supremum
\newcommand*{\ip}{\innerproduct} % inner product
\DeclareMathOperator{\cov}{Cov} % variance

\begin{document}

\input{titlepage}

\tableofcontents

\section{Lebesgue measure}

\subsection{Boolean algebra}

\begin{definition}[Boolean algebra]\index{Boolean algebra}
  Let \(X\) be a set. A \emph{Boolean algebra} on \(X\) is a family of subsets of \(X\) which
  \begin{enumerate}
  \item contains \(\emptyset\),
  \item is stable under finite unions and complementation.
  \end{enumerate}
\end{definition}

\begin{eg}\leavevmode
  \begin{itemize}
  \item The \emph{trivial Boolean algebra} \(\mathcal B = \{\emptyset, X\}\).
  \item The \emph{discrete Boolean algebra} \(\mathcal B = 2^X\), the family of all subsets of \(X\).
  \item Less trivially, if \(X\) is a topological space, the family of \emph{constructible sets} forms a Boolean algebra, where a constructible set is the finite union of locally closed set, i.e.\ a set \(E = U \cap F\) where \(U\) is open and \(F\) is closed.
  \end{itemize}
\end{eg}

\begin{definition}[finitely additive measure]\index{measure!finitely additive}
  Let \(X\) be a set and \(\mathcal B\) a Boolean algebra on \(X\). A \emph{finitely additive measure} on \((X, \mathcal B)\) is a function \(m: \mathcal B \to [0, +\infty]\) such that
  \begin{enumerate}
  \item \(m(\emptyset) = 0\),
  \item \(m(E \cup F) = m(E) + m(F)\) where \(E \cap F = \emptyset\).
  \end{enumerate}
\end{definition}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Counting measure: \(m(E) = \#E\), the cardinality of \(E\) where \(\mathcal B\) is the discrete Boolean algebra of \(X\).
  \item More generally, given \(f: X \to [0, +\infty]\), define for \(E \subseteq X\),
    \[
      m(E) = \sum_{e \in E} f(e).
    \]
  \item Suppose \(X = \coprod_{i = 1}^N X_i\), then define \(\mathcal B(X)\) to be the unions of \(X_i\)'s. Assign a weight \(a_i \geq 0\) to each \(X_i\) and define \(m(E) = \sum_{i: X_i \subseteq E} a_i\) for \(E \in \mathcal B\).
  \end{enumerate}
\end{eg}

\subsection{Jordan measure}

This section is a historic review and provides intuition for Lebesgue measure theory. We'll gloss over details of proofs in this section.

\begin{definition}
  A subset of \(\R^d\) is called \emph{elementary} if it is a finite union of \emph{boxes}, where a box is a set \(B = I_1 \times \dots \times I_d\) where each \(I_i\) is a finite interval of \(\R\).
\end{definition}

\begin{proposition}
  Let \(B \subseteq \R^d\) be a box. Let \(\mathcal E(B)\) be the family of elementary subsets of \(B\). Then
  \begin{enumerate}
  \item \(\mathcal E(B)\) is a Boolean algebra on \(B\),
  \item every \(E \in \mathcal E(B)\) is a disjoint finite union of boxes,
  \item if \(E \in \mathcal E(B)\) can be written as disjoint finite union in two ways, \(E = \bigcup_{i = 1}^n B_i = \bigcup_{j = 1}^m B_j'\), then \(\sum_{i = 1}^n |B_i| = \sum_{j = 1}^m |B_j'|\) where \(|B| = \prod_{i = 1}^d |b_i - a_i|\) if \(B = I_1 \times \dots \times I_d\) and \(I_i\) has endpoints \(a_i, b_i\).
  \end{enumerate}
\end{proposition}

Following this, we can define a finitely additive measure correponding to our intuition of length, area, volume etc:

\begin{proposition}
  Define \(m(E) = \sum_{i = 1}^n |B_i|\) if \(E\) is any elementary set and is the disjoint union of boxes \(B_i \subseteq \R^d\). Then \(m\) is a finitely additive measure on \(\mathcal E(B)\) for any box \(B\).
\end{proposition}

\begin{definition}
  A subset \(E \subseteq \R^d\) is \emph{Jordan measurable} if for any \(\varepsilon > 0\) there are elementary sets \(A, B\), \(A \subseteq E \subseteq B\) and \(m(B \setminus A) < \varepsilon\).
\end{definition}

\begin{remark}
  Jordan measurable sets are bounded.
\end{remark}

\begin{proposition}
  If a set \(E \subseteq \R^d\) is Jordan measurable, then
  \[
    \sup_{A \subseteq E \text{ elementary}} \{m(A)\} = \inf_{B \supseteq E \text{ elementary}} \{m(B)\}.
  \]
  In which case we define the \emph{Jordan measure} of \(E\) as
  \[
    m(E) = \sup_{A \subseteq E} \{m(A)\}.
  \]
\end{proposition}

\begin{proof}
  Take \(A_n \subseteq E\) such that \(m(A_n) \uparrow \sup\) and \(B_n \supseteq E\) such that \(m(B_n) \downarrow \inf\). Note that
  \[
    \inf \leq m(B_n) = m(A_n) + m(B_n \setminus A_n) \leq \sup + m(B_n \setminus A_n) \leq \sup + \varepsilon
  \]
  for arbitrary \(\varepsilon > 0\) so they are equal.
\end{proof}

\begin{ex}\leavevmode
  \begin{enumerate}
  \item If \(B\) is a box, the family \(\mathcal J(B)\) of Jordan measurable subsets of \(B\) is a Boolean algebra.
  \item A subset \(E \subseteq [0, 1]\) is Jordan measurable if and only if \(\mathbf 1_E\), the indicator function on \(E\), is Riemann integrable.
  \end{enumerate}
\end{ex}

\subsection{Lebesgue measure}

Although Jordan measure corresponds to the intuition of length, area and volume, it suffer from a few severe problems and issues:
\begin{enumerate}
\item unbounded sets in \(\R^d\) are not Jordan measurable.
\item \(\mathbf 1_{\Q \cap [0, 1]}\) is not Riemann integrable, and therefore \(\Q \cap [0, 1]\) is not Jordan measurable.
\item pointwise limits of Riemann integrable function \(f_n := \mathbf 1_{\frac{1}{n!} \Z \cap [0, 1]} \to \mathbf 1_{\Q \cap [0, 1]}\) is not Riemann integrable.
\end{enumerate}

The idea of Lebesgue is to use countable covers by boxes.

\begin{definition}
  A subset \(E \subseteq \R^d\) is \emph{Lebesgue measurable} if for all \(\varepsilon > 0\), there exists a countable union of boxes \(C\) with \(E \subseteq C\) and \(m^*(C \setminus E) < \varepsilon\), where \(m^*\), the \emph{Lebesgue outer measure}, is defined as
  \[
    m^*(E) = \inf \{\sum_{i \geq 1} |B_i|: E \subseteq \bigcup_{i \geq 1} B_i, B_i \text{ boxes}\}
  \]
  for \emph{every} subset \(E \subseteq \R^d\).
\end{definition}

\begin{remark}
  wlog in these definitions we may assume that boxes are open.
\end{remark}

% We are going to show that the family of Lebesgue measurable subsets is not only a Boolean algebra, but also stable under countable union. Next we are going to define the Lebesgue measure on the family, with the additive property (which is not possessed by Lebesgue outer measure). In fact, we can show that we cannot define a measure for \emph{all} subsets of a set.

\begin{proposition}
  \label{prop:Lebesgue measurable subset is Boolean algebra}
  The family \(\mathcal L\) of Lebesgue measurable subsets of \(\R^d\) is a Boolean algebra stable under countable unions.
\end{proposition}

\begin{lemma}\leavevmode
  \begin{enumerate}
  \item \(m^*\) is monotone: if \(E \subseteq F\) then \(m^*(E) \subseteq m^*(F)\).
  \item \(m^*\) is countably subadditive: if \(E = \bigcup_{n \geq 1} E_n\) where \(E_n \subseteq \R^d\) then
    \[
      m^*(E) \leq \sum_{n \geq 1} m^*(E_n).
    \]
  \end{enumerate}
\end{lemma}

\begin{proof}
  Monotonicity is obvious. For countable subadditivity, pick \(\varepsilon > 0\) and let \(C_n = \bigcup_{i \geq 1} C_{n, i}\) where \(C_{n, i}\) are boxes such that \(E_n \subseteq C_n\) and
  \[
    \sum_{i \geq 1} |C_{n, i}| \leq m^*(E_n) + \frac{\varepsilon}{2^n}.
  \]
  Then
  \[
    \sum_{n \geq 1} \sum_{i \geq 1} |C_{n, i}|
    \leq \sum_{n \geq 1} (m^*(E_n) + \frac{\varepsilon}{2^n})
    = \varepsilon + \sum_{n \geq 1} m^*(E_n)
  \]
  and \(E \subseteq \bigcup_{n \geq 1} C_n = \bigcup_{n \geq 1} \bigcup_{i \geq 1} C_{n, i}\) so
  \[
    m^*(E) \leq \varepsilon + \sum_{n \geq 1} m^*(E_n)
  \]
  for all \(\varepsilon > 0\).
\end{proof}

\begin{remark}
  Note that \(m^*\) is \emph{not} additive on the family of all subsets of \(\R^d\). However, it will be on \(\mathcal L\), as we will show later.
  % eg?
\end{remark}

\begin{lemma}
  If \(A, B\) are disjoint compact subsets of \(\R^d\) then
  \[
    m^*(A \cup B) = m^*(A) + m^*(B).
  \]
\end{lemma}

\begin{proof}
  \(\leq\) by the previous lemma so need to show \(\geq\). Pick \(\varepsilon > 0\). Let \(A \cup B \subseteq \bigcup_{n \geq 1} B_n\) where \(B_n\) are open boxes such that
  \[
    \sum_{n \geq 1} |B_n| \leq m^*(A \cup B) + \varepsilon.
  \]
  wlog we may assume that the side lengths of each \(B_n\) are \(< \frac{\alpha}{2}\), where
  \[
    \alpha = \inf \{\norm{x - y}_1: x \in A, y \in B\} > 0.
  \]
  where the inequality comes from the fact that \(A\) and \(B\) are compact and thus closed. wlog we may discard the \(B_n\)'s that do not interesect \(A \cup B\). Then by construction
  \[
    \sum_{n \geq 1} |B_n| = \sum_{n \geq 1, B_n \cap A = \emptyset} |B_n| + \sum_{n \geq 1, B_n \cap B = \emptyset} |B_n|
    \geq m^*(A) + m^*(B)
  \]
  so
  \[
    \varepsilon + m^*(A \cup B) \geq m^*(A) + m^*(B)
  \]
  for all \(\varepsilon\).
\end{proof}

\begin{lemma}
  If \(E \subseteq \R^d\) has \(m^*(E) = 0\) then \(E \in \mathcal L\).
\end{lemma}

\begin{definition}[null set]\index{null set}
  A set \(E \subseteq \R^d\) such that \(m^*(E) = 0\) is called a \emph{null set}.
\end{definition}

\begin{proof}
  For all \(\varepsilon > 0\), there exist \(C = \bigcup_{n \geq 1} B_n\) where \(B_n\) are boxes such that \(E \subseteq C\) and \(\sum_{n \geq 1} |B_n| \leq \varepsilon\). But
  \[
    m^*(C \setminus E) \leq m^*(C) \leq \varepsilon.
  \]
\end{proof}

\begin{lemma}
  \label{lem:open and closed sets are Lebesgue measurable}
  Every open subset of \(\R^d\) and every closed subset of \(\R^d\) is in \(\mathcal L\).
\end{lemma}

We will prove the lemma using the fact that the family of Lebesgue measurable subsets is stable under countable union, which itself \emph{does not} use this lemma. This lemma, however, will be used to show the stability under complementation. Since the proof is quite technical (it has more to do with general topology than measure theory), for brevity and fluency of ideas we present the proof the main proposition first.

\begin{proof}[Proof of \Cref{prop:Lebesgue measurable subset is Boolean algebra}]
  It is obvious that \(\emptyset \in \mathcal L\). To show it is stable under countable unions, start with \(E_n \in \mathcal L\) for \(n \geq 1\). Need to show \(E := \bigcup_{n \geq 1} E_n \in \mathcal L\).

  Pick \(\varepsilon > 0\). By assumption there exist \(C_n = \bigcup_{i \geq 1} B_{n, i}\) where \(B_{n, i}\) are boxes such that \(E_n \subseteq C_n\) and
  \[
    m^*(C_n \setminus E_n) < \frac{\varepsilon}{2^n}.
  \]
  Now
  \[
    E = \bigcup_{n \geq 1} E_n \subseteq \bigcup_{n \geq 1} C_n =: C
  \]
  so \(C\) is again a countable union of boxes and \(C \setminus E \subseteq \bigcup_{n \geq 1} C_n \setminus E_n\).
  so
  \[
    m^*(C \setminus E) \leq \sum_{n \geq 1} m^*(C_n \setminus E_n) \leq \sum_{n \geq 1} \frac{\varepsilon}{2^n} = \varepsilon
  \]
  by countable subadditivity so \(E \in \mathcal L\).

  To show it is stable under complementation, suppose \(E \in \mathcal L\). By assumption there exist \(C_n\) a countable union of boxes with \(E \subseteq C_n\) and \(m^*(C_n \setminus E) \leq \frac{1}{n}\). wlog we may assume the boxes are open so \(C_n\) is open, \(C_n^c\) is closed so \(C_n^c \in \mathcal L\). Thus \(\bigcup_{n \geq 1} C_n^c \in \mathcal L\) by first part of the proof.

  But
  \[
    m^*(E^c \setminus \bigcup_{n \geq 1} C_n^c)
    \leq m^*(E^c \setminus C_n^c)
    = m^*(C_n \setminus E)
    \leq \frac{1}{n}
  \]
  so \(m^*(E^c \setminus \bigcup_{n \geq 1} C_n^c) = 0\) so \(E^c \setminus \bigcup_{n \geq 1} C_n^c \in \mathcal L\) since it is a null set. But
  \[
    E^c = (E^c \setminus \bigcup_{n \geq 1} C_n^c) \cup \bigcup_{n \geq 1} C_n^c,
  \]
  both of which are in \(\mathcal L\) so \(E^c \in \mathcal L\).
\end{proof}

\begin{proof}[Proof of \Cref{lem:open and closed sets are Lebesgue measurable}]
  Every open set in \(\R^d\) is a countable union of boxes so is in \(\mathcal L\). It is more subtle for closed sets. %(in fact it is the key difference between this and Jordan measure).
  The key observation is that every closed set is the countable union of compact subsets so we are left to show compact sets of \(\R^d\) are in \(\mathcal L\).

  Let \(F \subseteq \R^d\) be compact. For all \(k \geq 1\), there exist \(O_k\) a countable union of open sets such that \(F \subseteq O_k := \bigcup_{i \geq 1} O_{k, i}\) where \(O_{k, i}\) are open boxes such that
  \[
    \sum_{i \geq 1} |O_{k, i}| \leq m^*(F) + \frac{1}{2^k}.
  \]
  By compactness there exist a finite subcover so we can assume \(O_k\) is a finite union of open boxes. Moreover, wlog assume that
  \begin{enumerate}
  \item the side lengths of \(O_{k, i}\) are \(\leq \frac{1}{2^k}\).
  \item for each \(i\), \(O_{k, i}\) intersects \(F\).
  \item \(O_{k + 1} \subseteq O_k\) (by replacing \(O_{k + 1}\) with \(O_{k + 1} \cap O_k\) iteratively).
  \end{enumerate}
  Then \(F = \bigcap_{k \geq 1} O_k\) and we are left to show \(m^*(O_k \setminus F) \to 0\). By additivity on disjoint compact sets,
  \[
    m^*(F) + m^*(\cl O_i \setminus O_{i + 1}) = m^*(F \cup (\cl O_i \setminus O_{i + 1}))
  \]
  so
  \[
    m^*(F) + m^*(\cl O_i \setminus O_{i + 1}) \leq m^*(\cl O_i)
    \leq \sum_{j \geq 1} |O_{i, j}|
    \leq m^*(F) + \frac{1}{2^i}
  \]
  so \(m^*(\cl O_i \setminus O_{i + 1}) \leq \frac{1}{2^i}\). Finally,
  \[
    m^*(O_k \setminus F)
    = m^*(\bigcup_{i \geq k} (O_i \setminus O_{i + 1}))
    \leq \sum_{i \geq k} m^*(O_i \setminus O_{i + 1})
    \leq \sum_{i \geq k} \frac{1}{2^i}
    = \frac{1}{2^{k - 1}}.
  \]
\end{proof}

The result we're working towards is

\begin{proposition}
  \label{prop:additivity of Lebesgue measure}
  \(m^*\) is countably additive on \(\mathcal L\), i.e.\ if \((E_n)_{n \geq 1}\) where \(E_n \in \mathcal L\) are pairwise disjoint then
  \[
    m^*(\bigcup_{n \geq 1} E_n) = \sum_{n \geq 1} m^*(E_n).
  \]
\end{proposition}

\begin{lemma}
  If \(E \in \mathcal L\) then for all \(\varepsilon > 0\) there exists \(U\) open, \(F\) closed, \(F \subseteq E \subseteq U\) such that \(m^*(U \setminus E) < \varepsilon\) and \(m^*(E \setminus F) < \varepsilon\).
\end{lemma}

\begin{proof}
  By definition of \(\mathcal L\), there exists a countable union of open boxes \(E \subseteq \bigcup_{n \geq 1} B_n\) such that \(m^*(\bigcup_{n \geq 1} B_n \setminus E) < \varepsilon\). Just take \(U = \bigcup_{n \geq 1} B_n\) which is open.

  For \(F\) do the same with \(E^c = \R^d \setminus E\) in place of \(E\).
\end{proof}

\begin{proof}[Proof of \Cref{prop:additivity of Lebesgue measure}]
  First we assume each \(E_n\) is compact. By a previous lemma \(m^*\) is additive on compact sets so for all \(N \in \N\),
  \[
    m^*(\bigcup_{n = 1}^N E_n) = \sum_{n = 1}^N m^*(E_n).
  \]
  In particular
  \[
    \sum_{n = 1}^N m^*(E_n) \leq m^*(\bigcup_{n \geq 1} E_n)
  \]
  since \(m^*\) is monotone. Take \(N \to \infty\) to get one inequality. The other direction holds by countable subadditivity of \(m^*\).

  Now assume that each \(E_n\) is a bounded subset in \(\mathcal L\). By the lemma there exists \(K_n \subseteq E_n\) closed, so compact, such that \(m^*(E_n \setminus K_n) \leq \frac{\varepsilon}{2^n}\). Since \(K_n\)'s are disjoint, by the previous case
  \[
    m^*(\bigcup_{n \geq 1} K_n) = \sum_{n \geq 1} m^*(K_n)
  \]
  then
  \begin{align*}
    &\sum_{n \geq 1} m^*(E_n) \\
    \leq& \sum_{n \geq 1} m^*(K_n) + m^*(E_n \setminus K_n) \\
    \leq& m^*(\bigcup_{n \geq 1} K_n) + \sum_{n \geq 1} \frac{\varepsilon}{2^n} \\
    \leq& m^*(\bigcup_{n \geq 1} E_n) + \varepsilon
  \end{align*}
  so one direction of inequality. Similarly the other direction holds by countable subadditivity of \(m^*\).

  For the general case, note that \(\R^d = \bigcup_{n \in \Z^d} A_n\) where \(A_n\) is bounded and in \(\mathcal L\), for example by taking \(A_n\) to be product of half open intervals of unit length. Write \(E_n\) as \(\bigcup_{m \in \Z^d} E_n \cap A_m\) so just apply the previous results to \((E_n \cap A_m)_{n\geq 1, m \in \Z^d}\).
\end{proof}

\begin{definition}[Lebesgue measure]\index{Lebesgue measure}
  \(m^*\) when restricted to \(\mathcal L\) is called the \emph{Lebesgue measure} and is simply denoted by \(m\).
\end{definition}

\begin{eg}[Vitali counterexample]
  Althought \(\mathcal L\) is pretty big (it includes all open and closed sets, countable unions and intersections of them, and has cardinality at least \(2^{\mathfrak c}\) where \(\mathfrak c\) is the continuum, by considering a null set with cardinality \(\mathfrak c\), and each subset thereof), it does not include every subset of \(\R^d\).

  Consider \((\Q, +)\), the additive subgroup of \((\R, +)\). Pick a set of representative \(E\) of the cosets of \((\Q, +)\). Choose it inside \([0, 1]\). For each \(x \in \R\), there exists a unique \(e \in E\) such that \(x - e \in \Q\) (here we require axiom of choice). Claim that \(E \notin \mathcal L\) and \(m^*\) is not additive on the family of all subsets of \(\R^d\).

  \begin{proof}
    Pick distinct rationals \(p_1, \dots, p_N\) in \([0, 1]\). The sets \(p_i + E\) are pairwise disjoint so if \(m^*\) were additive then we would have
    \[
      m^*(\bigcup_{i = 1}^N p_i + E)
      = \sum_{i = 1}^N m^*(p_i + E)
      = N m^*(E)
    \]
    by translation invariance of \(m^*\). But then
    \[
      \bigcup_{i = 1}^N p_i + E \subseteq [0, 2]
    \]
    since \(E \subseteq [0, 1]\) so by monotonicity of \(m^*\) have
    \[
      m^*(\bigcup_{i = 1}^N p_i + E) \leq 2
    \]
    so for all \(N m^*(E) \leq 2\) so \(m^*(E) = 0\). But
    \[
      [0, 1] \subseteq \bigcup_{q \in \Q} E + q = \R,
    \]
    by countable subadditivity of \(m^*\),
    \[
      1 = m^*([0, 1]) \leq \sum_{q \in \Q} m^*(E + q) = 0.
    \]
    Absurd.

    In particular \(E \notin \mathcal L\) as \(m^*\) is additive on \(\mathcal L\).
  \end{proof}
\end{eg}

\section{Abstract measure theory}

In this chapter we extend measure theory to arbitrary set. Most part of the theory is developed by Fréchet and Carathéodory.

\begin{definition}[\(\sigma\)-algebra]\index{\(\sigma\)-algebra}
  A \emph{\(\sigma\)-algebra} on a set \(X\) is a Boolean algebra stable under countable unions.
\end{definition}

\begin{definition}[measurable space]\index{measurable space}
  A \emph{measurable space} is a couple \((X, \mathcal A)\) where \(X\) is a set and \(\mathcal A\) is a \(\sigma\)-algebra on \(X\).
\end{definition}

\begin{definition}[measure]\index{measure}
  A \emph{measure} on \((X, \mathcal A)\) is a map \(\mu: \mathcal A \to [0, \infty]\) such that
  \begin{enumerate}
  \item \(\mu(\emptyset) = 0\),
  \item \(\mu\) is countably additive (also known as \(\sigma\)-additive), i.e.\ for every family \((A_n)_{n \geq 1}\) of disjoint subsets in \(\mathcal A\), have
    \[
      \mu (\bigcup_{n \geq 1} A_n) = \sum_{n \geq 1} \mu(A_n).
    \]
  \end{enumerate}

  The triple \((X, \mathcal A, \mu)\) is called a measure space.
\end{definition}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item \((\R^d, \mathcal L, m)\) is a measure space.
  \item \((X, 2^X, \#)\) where \(\#\) is the counting measure.
  \end{enumerate}
\end{eg}

\begin{proposition}
  Let \((X, \mathcal A, \mu)\) be a measure space. Then
  \begin{enumerate}
  \item \(\mu\) is monotone: \(A \subseteq B\) implies \(\mu(A) \subseteq \mu(B)\),
  \item \(\mu\) is countably subadditive: \(\mu (\bigcup_{n \geq 1} A_n) \leq \sum_{n \geq 1} \mu(A_n)\),
  \item upward monotone convergence: if
    \[
      E_1 \subseteq E_2 \subseteq \dots \subseteq E_n \subseteq \dots
    \]
    then
    \[
      \mu (\bigcup_{n \geq 1} E_n) = \lim_{n \to \infty} \mu(E_n) = \sup_{n \geq 1} \mu(E_n).
    \]
  \item downard monotone convergence: if
    \[
      E_1 \supseteq E_2 \supseteq \dots \supseteq E_n \supseteq \dots
    \]
    and \(\mu(E_1) < \infty\) then
    \[
      \mu (\bigcap_{n \geq 1} E_n) = \lim_{n \to \infty} \mu(E_n) = \inf_{n \geq 1} \mu(E_n).
    \]
  \end{enumerate}
\end{proposition}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item
    \[
      \mu(B) = \mu(A) + \underbrace{\mu(B \setminus A)}_{\geq 0}
    \]
    by additivity of \(\mu\).
  \item See example sheet. The idea is that every countable union \(\bigcup_{n \geq 1} A_n\) is a disjoint countable union \(\bigcup_{n \geq 1} B_n\) where for each \(n\), \(B_n \subseteq A_n\). It then follows by \(\sigma\)-additivity.
  \item Let \(E_0 = \emptyset\) so
    \[
      \bigcup_{n \geq 1} E_n = \bigcup_{n \geq 1} (E_n \setminus E_{n - 1}),
    \]
    a disjoint union. By \(\sigma\)-additivity,
    \[
      \mu(\bigcup_{n \geq 1} E_n) = \sum_{n \geq 1} \mu(E_n \setminus E_{n - 1})
    \]
    but for all \(N\), by additivity of \(\mu\),
    \[
      \sum_{n = 1}^N \mu(E_n \setminus E_{n - 1}) = \mu(E_N)
    \]
    so take limit. The supremum part is obvious.
  \item Apply the previous result to \(E_1 \setminus E_n\).
  \end{enumerate}
\end{proof}

\begin{remark}
  Note the \(\mu(E_1) < \infty\) condition in the last part. Counterexample: \(E_n = [n, \infty) \subseteq \R\).
\end{remark}

\begin{definition}[\(\sigma\)-algebra generated by a family]
  Let \(X\) be a set and \(\mathcal F\) be some family of subsets of \(X\). The the intersection of all \(\sigma\)-algebras on \(X\) containing \(\mathcal F\) is a \(\sigma\)-algebra, called the \(\sigma\)-algebra \emph{generated} by \(\mathcal F\) and is denoted by \(\sigma(\mathcal F)\).
\end{definition}

\begin{proof}
  Easy check. See example sheet.
\end{proof}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Suppose \(X = \coprod_{i = 1}^N X_i\), i.e.\ \(X\) admits a finite partition. Let \(\mathcal F = \{X_1, \dots, X_n\}\), then \(\sigma(\mathcal F)\) consists of all subsets that are unions of \(X_i\)'s.
  \item Suppose \(X\) is countable and let \(\mathcal F\) be the collection of all singletons. Then \(\sigma(\mathcal F) = 2^X\).
  \end{enumerate}
\end{eg}

\begin{definition}[Borel \(\sigma\)-algebra]\index{Borel \(\sigma\)-algebra}
  Let \(X\) be a topological space. The \(\sigma\)-algebra generated by open subsets of \(X\) is called the \emph{Borel \(\sigma\)-algebra} of \(X\), denoted by \(\mathcal B(X)\).
\end{definition}

\begin{proposition}
  If \(X = \R^d\) then \(\mathcal B(X) \subseteq \mathcal L\). Moreover every \(A \in \mathcal L\) can be written as a disjoint union \(A = B \cup N\) where \(B \in \mathcal B(X)\) and \(N\) is a null set.
\end{proposition}

%This gives an alternative way to define Lebesgue measure:

\begin{proof}
  We've shown that \(\mathcal L\) is a \(\sigma\)-algebra and contains all open sets so \(\mathcal B(X) \subseteq \mathcal L\). Given \(A \in \mathcal L\), \(A^c \in \mathcal L\) so for all \(n \geq 1\) there exists \(C_n\) countable unions of (open) boxes such that \(A^c \subseteq C_n\) and \(m^*(C_n \setminus A^c) \leq \frac{1}{n}\). Take \(C = \bigcap_{n \geq 1} C_n \in \mathcal B(X)\). Thus \(B := C^c \in \mathcal B(X)\) and \(m(A \setminus B) = 0\) because \(A \setminus B = C \setminus A^c\).
\end{proof}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item It can be shown that \(\mathcal B(\R^d) \subsetneq \mathcal L\). In fact \(|\mathcal L| \geq 2^{\mathfrak c}\) and \(|\mathcal B(\R^d)| = \mathfrak c\).
  \item If \(\mathcal F\) is a family of subsets of a set \(X\), the Boolean algebra generated by \(\mathcal F\) can be explicitly described as
    \[
      \mathcal B(\mathcal F) = \{\text{finite unions of } F_1 \cap \dots \cap F_N: F_i \in \mathcal F \text{ or } F_i^c \in \mathcal F\}.
    \]
  \item However, this is not so for \(\sigma(\mathcal F)\). There is no ``simple'' description of \(\sigma\)-algebra generated by \(\mathcal F\). (c.f.\ Borel hierarchy in descriptive set theory and transfinite induction)
  \end{enumerate}
\end{remark}

\begin{definition}[\(\pi\)-system]\index{\(\pi\)-system}
  A family \(\mathcal F\) of subsets of a set \(X\) is called a \emph{\(\pi\)-system} if it contains \(\emptyset\) and it is closed under finite intersection.
\end{definition}

\begin{proposition}[measure uniqueness]
  Let \((X, \mathcal A)\) be a measurable space. Assume \(\mu_1\) and \(\mu_2\) are two finite measures (i.e.\ \(\mu_i(X) < \infty\)) such that \(\mu_1(F) = \mu_2(F)\) for every \(F \in \mathcal F\) where \(\mathcal F\) is a \(\pi\)-system with \(\sigma(\mathcal F) = \mathcal A\). Then \(\mu_1 = \mu_2\).
\end{proposition}

For \(\R^d\), we only have to check open boxes.

\begin{proof}
We state first the following lemma:
\begin{lemma}[Dynkin lemma]\index{Dynkin lemma}
  If \(\mathcal F\) is a \(\pi\)-system on \(X\) and \(\mathcal C\) is a family of subsets of \(X\) such that \(\mathcal F \subseteq \mathcal C\) and \(\mathcal C\) is stable under complementation and disjoint countable unions. Then \(\sigma(\mathcal F) \subseteq \mathcal C\).
\end{lemma}

Let \(\mathcal C = \{A \in \mathcal A: \mu_1(A) = \mu_2(A)\}\). Then \(\mathcal C\) is clearly stable under complementation as
\[
  \mu_i (A^c) = \mu_i(X \setminus A) = \mu_i(X) - \mu_i(A).
\]
\(\mathcal C\) is also clearly stable under countable disjoint unions by \(\sigma\)-additivity. Thus by Dynkin lemma, \(\mathcal C \supseteq \sigma(\mathcal F) = \mathcal A\).

\begin{proof}[Proof of Dynkin lemma]
  Let \(\mathcal M\) be the smallest family of subsets of \(X\) containing \(\mathcal F\) and stable under complementation and countable disjoint union (\(2^X\) is such a family and taking intersection). Sufficient to show that \(\mathcal M\) is a \(\sigma\)-algebra, as then \(\mathcal M \subseteq \mathcal C\) implies \(\sigma(\mathcal F) \subseteq \mathcal C\).

  It suffices to show \(\mathcal M\) is a Boolean algebra. Let
  \[
    \mathcal M' = \{A \in \mathcal M: A \cap B \in \mathcal M \text{ for all } B \in \mathcal F\}.
  \]
  \(\mathcal M'\) again is stable under countable disjoint unions and complementation because
  \[
    A^c \cap B = (B^c \cup (A \cap B))^c
  \]
  as a disjoint union so is in \(\mathcal M\).

  As \(\mathcal M' \supseteq \mathcal F\), by minimality of \(\mathcal M\), have \(\mathcal M = \mathcal M'\). Now let
  \[
    \mathcal M'' = \{A \in \mathcal M': A \cap B \in \mathcal M \text{ for all } B \in \mathcal M\}.
  \]
  The same argument shows that \(\mathcal M'' = \mathcal M\). Thus \(\mathcal M\) is a Boolean algebra and a \(\sigma\)-algebra.
\end{proof}
\end{proof}

\begin{proposition}[uniqueness of Lebesgue measure]
  Lebesgue measure is the unique translation invariant measure \(\mu\) on \((\R^d, \mathcal B(\R^d))\) such that
  \[
    \mu([0, 1]^d) = 1.
  \]
\end{proposition}

\begin{proof}
  Exercise. Hint: use the \(\pi\)-system \(\mathcal F\) made of all boxes in \(\R^d\) and dissect a cube into dyadic pieces. Then approximate and use monotone.
\end{proof}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item There is \emph{no} countably additive translation invariant measure on \(\R\) defined on all subsets of \(\R\). (c.f.\ Vitali's counterexample).
  \item However, the Lebesgue measure can be extended to a finitely additive measure on all subsets of \(\R\) (proof requires Hahn-Banach theorem. See IID Linear Analysis).
  \end{enumerate}
\end{remark}

Recall the construction of Lebesgue measure: we take boxes in \(\R^d\), and define elementary sets, which is the Boolean algebra generated by boxes. Then we can define Jordan measure which is finitely additive. However, this is not countably additive but analysis craves limits so we define Lebesgue measurable sets, by introducing the outer measure \(m^*\), which is built from the Jordan measure. Finally we restrict this outer measure to \(\mathcal L\). We also define the Borel \(\sigma\)-algebra, which is the same as the \(\sigma\)-algebra generated by the boxes. We show that the Borel \(\sigma\)-algebra is contained in \(\mathcal L\), and every element in \(\mathcal L\) can be written as a disjoint union of an element in the Borel \(\sigma\)-algebra and a measure zero set.

Suppose \(\mathcal B\) is a Boolean algebra on a set \(X\). Let \(\mu\) be a finitely additive measure on \(\mathcal B\). We are going to construct a measure on \(\sigma(\mathcal B)\).

\begin{theorem}[Carathéodory extension theorem]\index{Carathéodory extension theorem}
  Assume that \(\mu\) is countably additive on \(\mathcal B\), i.e.\ if \(B_n \in \mathcal B\) disjoint is such that \(\bigcup_{n \geq 1} B_n \in \mathcal B\) then \(\mu(\bigcup_{n \geq 1} B_n) = \sum_{n \geq 1} \mu(B_n)\) and assume that \(\mu\) is \(\sigma\)-finite, i.e.\ there exists \(X_m \in \mathcal B\) such that \(X = \bigcup_{m \geq 1} X_m\) and \(\mu(X_m) < \infty\), then \(\mu\) extends uniquely to a measure on \(\sigma(\mathcal B)\).
\end{theorem}

\begin{proof}
  For any \(E \subseteq X\), let
  \[
    \mu^*(E) = \inf \{\sum_{n \geq 1} \mu(B_n): E \subseteq \bigcup_{n \geq 1} B_n, B_n \in \mathcal B\}
  \]
  and call it the outer measure associated to \(\mu\). Define a subset \(E \subseteq X\) to be \(\mu^*\)-measurable if for all \(\varepsilon > 0\) there exists \(C = \bigcup_{n \geq 1} B_n\) with \(B_n \in \mathcal B\) such that \(E \subseteq C\) and
  \[
    \mu^*(C \setminus E) \leq \varepsilon.
  \]
  We denote by \(\mathcal B^*\) the set of \(\mu^*\)-measurable subsets. Claim that
  \begin{enumerate}
  \item \(\mu^*\) is countably subadditive and monotone.
  \item \(\mu^*(B) = \mu(B)\) for all \(B \in \mathcal B\).
  \item \(\mathcal B^*\) is a \(\sigma\)-algebra and contains all \(\mu^*\)-null sets and \(\mathcal B\).
  \item \(\mu^*\) is \(\sigma\)-additive on \(\mathcal B^*\).
  \end{enumerate}

  Then existence follows from the proposition as \(\mathcal B^* \supseteq \sigma(\mathcal B)\): \(\mu^*\) will be a measure on \(\mathcal B^*\) and thus on \(\sigma(\mathcal B)\). Uniqueness follows from a similar proof for Lebesgue measure via Dynkin lemma.

  \begin{proof}
    This will be very easy as we only need to adapt our previous work to the general case. Note that in a few occassion we used properties of \(\R^d\), such as openness of some sets, so be careful.
    \begin{enumerate}
    \item Same.
    \item \(\mu^*(B) \leq \mu(B)\) for all \(B \in \mathcal B\) by definition of \(\mu^*\). For the other direction, for all \(\varepsilon > 0\), there exist \(B_n \in \mathcal B\) such that \(B \subseteq \bigcup_{n \geq 1} B_n\) and \(\sum_{n \geq 1} \mu(B_n) \leq \mu^*(B) + \varepsilon\). But
      \[
        B = \bigcup_{n \geq 1} B_n \cap B = \bigcup_{n \geq 1} C_n
      \]
      where \(C_n := B_n \cap B \setminus \bigcup_{i < n} B\cap B_i\) and so \(C_n \in \mathcal B\). Thus by countable additivity
      \[
        \mu(B)
        = \sum_{n \geq 1} \mu(C_n)
        \leq \sum_{n \geq 1} \mu(B_n)
        \leq \mu^*(B) + \varepsilon
      \]
    \item \(\mu^*\)-null sets and \(\mathcal B\) are obviously in \(\mathcal B^*\). Thus it is left to show that \(\mathcal B^*\) is a \(\sigma\)-algebra. Stability under countable union is exactly the same and then we claim that \(\mathcal B^*\) is stable under complementation. This is the bit where we used closed/open sets in \(\R^d\) in the original proof. Here we use a lemma as a substitute.

      \begin{lemma}
        %\(\mathcal B^*\) is stable under countable unions and countable intersections.
        Suppose \(B_n \in \mathcal B\) then \(\bigcap_{n \geq 1} B_n \in \mathcal B^*\).
      \end{lemma}

      \begin{proof}
        First claim that if \(E = \bigcap_{n \geq 1} I_n\) where \(I_{n + 1} \subseteq I_n\) and \(I_n \in \mathcal B\) such that \(\mu(I_1) < \infty\) then \(\mu^*(E) = \lim_{n \to \infty} \mu(I_n)\) and \(E \in \mathcal B^*\): by additivity of \(\mu\) on \(\mathcal B\),
        \[
          \sum_{n = 1}^N \mu(I_n \setminus I_{n + 1})
          = \mu(I_1) - \mu(I_N)
        \]
        which converges as \(N \to \infty\) (because \(\mu(I_{n + 1}) \leq \mu(I_n)\)), so
        \[
          \sum_{n \geq N} \mu(I_n \setminus I_{n + 1}) \to 0
        \]
        as \(N \to \infty\). But LHS is greater than \(\mu^*(I_N \setminus E)\) because \(I_N \setminus E = \bigcup_{n \geq N} I_n \setminus I_{n + 1}\). Therefore \(E \in \mathcal B^*\) and
          \[
            \mu(I_n) \leq \underbrace{\mu^*(I_n \setminus E)}_{\to 0} + \underbrace{\mu^*(E)}_{\leq \mu(I_n)}
          \]
          so
          \[
            \lim_{n \to \infty} \mu(I_n) = \mu^*(E).
          \]

          Now for the actual lemma, let \(E = \bigcap_{n \geq 1} I_n\) where \(I_n \in \mathcal B\). wlog we may assume \(I_{n + 1} \subseteq I_n\). By \(\sigma\)-finiteness assumption, \(X = \bigcup_{m \geq 1} X_m\) where \(X_m \in \mathcal B\) with \(\mu(X_m) < \infty\) so
          \[
            E = \bigcup_{m \geq 1} E \cap X_m.
          \]
          By the claim for all \(m\), \(E \cap X_m \in \mathcal B^*\) so \(E \in \mathcal B^*\).
      \end{proof}
      From the lemma we can derive that \(\mathcal B^*\) is also stable under complementation: given \(E \in \mathcal B^*\), for all \(n\) there exist \(C_n = \bigcup_{i \geq 1} B_{n, i}\) where \(B_{n, i} \in \mathcal B\) such that \(E \subseteq C_n\) and \(\mu^*(C_n \setminus E) \leq \frac{1}{n}\). Now
      \[
        E^c = (\bigcup_{n \geq 1} C_n^c) \cup (E^c \setminus \bigcup_{n \geq 1} C_n^c)
      \]
      but \(C_n^c\) is a countable intersection \(\bigcap_{i \geq 1} B_{n, i}^c\) and \(E^c \setminus \bigcup_{n \geq 1} C_n^c\) is \(\mu^*\)-null so by the lemma, \(C_n^c \in \mathcal B^*\). Therefore their union is also in \(\mathcal B^*\). Since we've shown that null sets are in \(\mathcal B^*\), \(E^c \in \mathcal B^*\).
    \item We want to show \(\mu^*\) is countably additive on \(\mathcal B^*\). Recall that \(\mu\) is \(\sigma\)-finite: there exists \(X_m \in \mathcal B\) such that \(X = \bigcup_{m \geq 1} X_m\), \(\mu(X_m) < \infty\). We say \(E \subseteq X\) is \emph{bounded} if there exists \(m\) such that \(E \subseteq X_m\). It is then enough to show countable additivity for bounded sets by the same argument as before: write \(X = \bigcup_{m \geq 1} \tilde X_m\) where \(\tilde X_m = X_m \setminus \bigcup_{i < m} X_i \in \mathcal B\) so this is a disjoint union. Then if \(E = \bigcup_{n \geq 1} E_n\) as a disjoint union then
      \[
        E = \bigcup_{n \geq 1} \bigcup_{m \geq 1} (E_n \cap \tilde X_m)
      \]
      which is also a countable disjoint union.

      Given \(E\), if we can show finite additivity then
      \[
        \sum_{n = 1}^N \mu^*(E_n)
        = \mu^*(\bigcup_{n = 1}^N E_n)
        \leq \mu^*(E)
        \leq \sum_{n \geq 1} \mu^*(E_n)
      \]
      take limit as \(N \to \infty\) to have equality throughout.

      It suffices to prove finite additivity when \(E\) and \(F\) are countable intersections of sets from \(\mathcal B\): \(E,F \in \mathcal B^*\) so for \(\varepsilon > 0\) there exists \(C, D\) countable intersections of sets from \(\mathcal B\) such that \(C \subseteq E, D \subseteq F\) and
      \begin{align*}
        \mu^*(E) &\leq \mu^*(C) + \varepsilon \\
        \mu^*(F) &\leq \mu^*(D) + \varepsilon
      \end{align*}
      As \(E \cap F = \emptyset\) and \(C \subseteq E, D \subseteq F\), \(C \cap D = \emptyset\) so by finite additivity,
      \[
        \mu^*(E) +\mu^*(F) \leq 2\varepsilon + \mu^*(C \cup D) \leq 2 \varepsilon + \mu^*(E \cup F).
      \]
      As usual, reverse holds by subadditivity.

      Finally for \(E = \bigcap_{n \geq 1} I_n, F = \bigcap_{n \geq 1} J_n\) bounded, wlog assume \(I_{n + 1} \subseteq I_n, J_{n + 1} \subseteq J_n\). \(\mu(I_n), \mu(J_n) < \infty\). Now use claim 3,
      \begin{align*}
        \mu^*(E) &= \lim_{n \to \infty} \mu^*(I_n) \\
        \mu^*(F) &= \lim_{n \to \infty} \mu^*(J_n)
      \end{align*}
      so
      \[
        \mu^*(E) + \mu^*(F)
        = \lim_{n \to \infty} \mu(I_n) + \mu(J_n)
        = \lim_{n \to \infty} (\mu(I_n \cup J_n) + \mu(I_n \cap J_n))
      \]
      But
      \begin{align*}
        \bigcap_{n \geq 1} (I_n \cap J_n) &= E \cap F = \emptyset \\
        \bigcap_{n \geq 1} (I_n \cup J_n) &= E \cup F
      \end{align*}
      so by claim 3
      \begin{align*}
        \lim_{n \to \infty} \mu(I_n \cap J_n) &= 0 \\
        \lim_{n \to \infty} \mu(I_n \cup J_n) &= \mu^*(E \cup F)
      \end{align*}
      which finishes the proof.
    \end{enumerate}
  \end{proof}
\end{proof}

\begin{remark}
  We prove that every set in \(\mathcal B^*\) is a disjoint union \(E = F \cup N\) where \(F \in \sigma(\mathcal B)\) and \(N\) is \(\mu^*\)-null.
\end{remark}

\begin{definition}[completion]\index{completion}
  We say that \(\mathcal B^*\) is the \emph{completion} of \(\sigma(\mathcal B)\) with respect to \(\mu\).
\end{definition}

\begin{eg}
  \(\mathcal L\) is the completion of \(\mathcal B(\R^d)\) in \(\R^d\).
\end{eg}

\section{Integration and measurable functions}

\begin{definition}[measurable function]\index{measurable}
  Let \((X, \mathcal A)\) be a measurable space. A function \(X \to \R\) is called \emph{measurable} or \emph{\(\mathcal A\)-measurable} if for all \(t \in \R\),
  \[
    \{x \in X: f(x) < t\} \in \mathcal A.
  \]
\end{definition}

\begin{remark}
  The \(\sigma\)-algebra generated by intervals \((-\infty, t)\) where \(t \in \R\) is the Borel \(\sigma\)-algebra of \(\R\), denote \(\mathcal B(\R)\). Thus for every measurable function \(f: X \to \R\), the preimage \(f^{-1}(B) \in \mathcal A\) for all \(B \in \mathcal B(\R)\). However, it is \emph{not} true that \(f^{-1}(L) \in \mathcal A\) for any \(L \in \mathcal L\).
\end{remark}

\begin{remark}
  If \(f\) is allowed to take the values \(+\infty\) and \(-\infty\) we will say that \(f\) is measurable if additionally \(f^{-1}(\{+\infty\}) \in \mathcal A\) and \(f^{-1}(\{-\infty\}) \in \mathcal A\).
\end{remark}

More generally,

\begin{definition}[measurable map]\index{measurable}
  Suppose \((X, \mathcal A)\) and \((Y, \mathcal B)\) are measurable spaces. A map \(f: X \to Y\) is \emph{measurable} if for all \(B \in \mathcal B\), \(f^{-1}(B) \in \mathcal A\).
\end{definition}

\begin{proposition}\leavevmode
  \begin{enumerate}
  \item The composition of measurable maps is measurable.
  \item If \(f, g: (X, \mathcal A) \to \R\) are measurable functions then \(f + g, fg\) and \(\lambda f\) for \(\lambda \in \R\) are also measurable.
  \item If \((f_n)_{n \geq 1}\) is a sequence of measurable functions on \((X, \mathcal A)\) then so are \(\sup_n f_n, \inf_n f_n, \limsup_n f_n\) and \(\liminf_n f_n\).
  \end{enumerate}
\end{proposition}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item Obvious.
  \item Follow from 1 once it's shown that \(+: \R^2 \to \R\) and \(\times: \R^2 \to \R\) are measurable (with respect to Borel sets). The sets
    \begin{align*}
      &\{(x, y): x + y < t\} \\
      &\{(x, y): xy < t\}
    \end{align*}
    are open in \(\R^2\) and hence Borel.
  \item \(\inf_n f_n(x) < t\) if and only if
    \[
      x \in \bigcup_n \{x: f_n(x) < t\}
    \]
    and similar for \(\sup\). Similarly \(\limsup_n f_n(x) < t\) if and only if
    \[
      x \in \bigcup_{m \geq 1} \bigcap_{k \geq 1} \bigcup_{n \geq k} \{x: f_n(x) < t - \frac{1}{m}\}.
    \]
  \end{enumerate}
\end{proof}

\begin{proposition}
  \(f = (f_1, \dots, f_d): (X, \mathcal A) \to (\R^d, \mathcal B(\R^d))\) where \(d \geq 1\) is measurable if and only if each \(f_i: X \to \R\) is measurable.
\end{proposition}

\begin{proof}
  One direction is easy: suppose \(f\) is measurable then
  \[
    \{x: f_i(x) < t\} = f^{-1}(\{y \in \R^d: y_i < t\}),
  \]
  which is open so \(f_i\) is measurable.

  Conversely, suppose \(f_i\) is measurable. Then
  \[
    f^{-1}( \prod_{i = 1}^d [a_i, b_i] )
    = \bigcap_{i = 1}^d \{x: a_i \leq f_i(x) \leq b_i\}
  \]
  As the boxes generate the Borel sets, done.
\end{proof}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Let \((X, \mathcal A)\) be a measurable space and \(E \subseteq X\). Then \(E \in \mathcal A\) if and only if \(\mathbf 1_E\), the indicator function on \(E\), is \(\mathcal A\)-measurable.
  \item If \(X = \coprod_{i = 1}^N X_i\) and \(\mathcal A\) is the Boolean algebra generated by the \(X_i\)'s. A function \(f: (X, \mathcal A) \to \R\) is measurable if and only if \(f\) is constant on each \(X_i\). In this case the vector space of measurable functions has dimension \(N\).
  \item Every continuous function \(f: \R^d \to \R\) is measurable.
  \end{enumerate}
\end{eg}

\begin{definition}[Borel measurable]\index{Borel measurable}
  If \(X\) is a topological space, \(f: X \to \R\) is \emph{Borel} or \emph{Borel measurable} if it is \(\mathcal B(X)\)-measurable.
\end{definition}

\begin{definition}[simple function]\index{simple function}
  A function \(f\) on \((X, \mathcal A)\) is called \emph{simple} if
  \[
    f = \sum_{i = 1}^n a_i \mathbf 1_{A_i}
  \]
  for some \(a_i \geq 0\) and \(A_i \in \mathcal A\).
\end{definition}

Of course simple functions are measurable.

\begin{lemma}
  If a simple function can be written in two ways
  \[
    f = \sum_{i = 1}^n a_i \mathbf 1_{A_i} = \sum_{j = 1}^s b_j \mathbf 1_{B_j}
  \]
  then
  \[
    \sum_{i = 1}^n a_i \mu(A_i) = \sum_{j = 1}^s b_j \mu(B_j)
  \]
  for any measure \(\mu\) on \((X, \mathcal A)\).
\end{lemma}

\begin{proof}
  Example sheet 1.
\end{proof}

\begin{definition}[integral of a simple function with respect to a measure]\index{integral with respect to a measure}
  The \emph{\(\mu\)-integral} of \(f\) is defined by
  \[
    \mu(f) := \sum_{i = 1}^n a_i \mu(A_i).
  \]
\end{definition}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item The lemma says that the integral is well-defined.
  \item We also use the notation \(\int_X f d\mu\) to denote \(\mu(f)\).
  \end{enumerate}
\end{remark}

\begin{proposition}
  \(\mu\)-integral satisfies, for all simple functions \(f\) and \(g\),
\begin{enumerate}
\item linearity: for all \(\alpha, \beta \geq 0\), \(\mu(\alpha f + \beta g) = \alpha \mu(f) + \beta \mu(g)\).
\item positivity: if \(g \leq f\) then \(\mu(g) \leq \mu(f)\).
\item if \(\mu(f) = 0\) then \(f = 0\) \(\mu\)-almost everywhere, i.e.\ \(\{x \in X: f(x) \neq 0\}\) is a \(\mu\)-null set.
\end{enumerate}
\end{proposition}

\begin{proof}
  Obvious from definition and lemma.
\end{proof}

\begin{definition}
  If \(f \geq 0\) and measurable on \((X, \mathcal A)\), define
  \[
    \mu(f) = \sup \{\mu(g): g \text{ simple }, g \leq f\} \in [0, +\infty].
  \]
\end{definition}

\begin{remark}
  This is consistent with the definition for \(f\) simple, due to positivity.
\end{remark}

\begin{definition}[integrable]\index{integrable}
  If \(f\) is an arbitrary measurable function on \((X, \mathcal A)\) we say \(f\) is \emph{\(\mu\)-integrable} if
  \[
    \mu(|f|) < \infty.
  \]
\end{definition}

\begin{definition}[integral with respect to a measure]\index{integral with respect to a measure}
  If \(f\) is \(\mu\)-integrable, then we define its \emph{\(\mu\)-integral} by
  \[
    \mu(f) = \mu(f^+) - \mu(f^-)
  \]
  where \(f^+ = \max\{0, f\}\) and \(f^- = (-f)^+\).
\end{definition}

\begin{note}
  \begin{align*}
    |f| &= f^+ + f^- \\
    f &= f^+ - f^-
  \end{align*}
\end{note}

\begin{theorem}[monotone convergence theorem]\index{monotone convergence theorem}
  \label{thm:monotone convergence theorem}
  Let \((f_n)_{n \geq 1}\) be a sequence of measurable functions on a measure space \((X, \mathcal A, \mu)\) such that
  \[
    0 \leq f_1 \leq f_2 \leq \dots \leq f_n \leq \dots
  \]
  Let \(f = \lim_{n \to \infty} f_n\). Then
  \[
    \mu(f) = \lim_{n \to \infty} \mu(f_n).
  \]
\end{theorem}

\begin{lemma}
  If \(g\) is a simple function on \((X, \mathcal A, \mu)\), the map
  \begin{align*}
    m_g: \mathcal A &\to [0, \infty] \\
    E &\mapsto \mu(\mathbf 1_E g)
  \end{align*}
  is a measure on \((X, \mathcal A)\).
\end{lemma}

\begin{proof}
  Write \(g = \sum_{i = 1}^r a_i \mathbf 1_{A_i}\) so \(g \mathbf 1_E = \sum_{i = 1}^r a_i \mathbf 1_{A_i \cap E}\) so
  \[
    \mu(\mathbf 1_E g) = \sum_{i = 1}^r a_i \mu(A_i \cap E).
  \]
  By a question on example sheet this is well-defined. Then \(\sigma\)-additivity follows immediately from \(\sigma\)-additivity of \(\mu\).
\end{proof}

\begin{proof}[Proof of \nameref{thm:monotone convergence theorem}]
  \(f_n \leq f_{n + 1} \leq f\) by assumption so
  \[
    \mu(f_n) \leq \mu(f_{n + 1}) \leq \mu(f)
  \]
  by definition of integral so
  \[
    \lim_{n \to \infty} \mu(f_n) \leq \mu(f),
  \]
  although RHS may be infinite.

  Let \(g\) be any simple function with \(g \leq f\). Need to show that \(\mu(g) \leq \lim_{n \to \infty} \mu(f_n)\). Pick \(\varepsilon > 0\). Let
  \[
    E_n = \{x \in X: f_n(x) \geq (1 - \varepsilon) g(x) \}.
  \]
  Then \(X = \bigcup_{n \geq 1} E_n\) and \(E_n \subseteq E_{n + 1}\). So we may apply upward monotone convergence for sets to measure \(m_g\) and get
  \[
    \lim_{n \to \infty} m_g(E_n) = m_g(X) = \mu(g \mathbf 1_X) = \mu(g).
  \]
  But
  \[
    (1 - \varepsilon) m_g(E_n) = \mu((1 - \varepsilon) g \mathbf 1_{E_n})) \leq \mu(f_n)
  \]
  because \((1 - \varepsilon) g \mathbf 1_{E_n}\) is a simple function smaller than \(f_n\). Taking limit,
  \[
    (1 - \varepsilon) \mu(g) \leq \lim_{n \to \infty} \mu(f_n)
  \]
  which holds for all \(\varepsilon\). So
  \[
    \mu(g) \leq \lim_{n \to \infty} \mu(f_n).
  \]
\end{proof}

\begin{lemma}
  \label{lem:approximation by simple functions}
  If \(f \geq 0\) is a measurable function on \((X, \mathcal A)\) then there is a sequence of simple functions \((g_n)_{n \geq 1}\)
  \[
    0 \leq g_n \leq g_{n + 1} \leq f
  \]
  such that for all \(x \in X\), \(g_n(x) \uparrow f(x)\).
\end{lemma}

\begin{notation}
  \(g_n \uparrow f\) means that \(\lim_{n \to \infty} g_n(x) = f(x)\) and \(g_{n + 1} \geq g_n\).
\end{notation}

\begin{proof}
  We can take
  \[
    g_n = \frac{1}{2^n} \floor*{2^n \min \{f, n\}}
  \]
  pointwise. Check that \(\floor{2y} \geq 2 \floor{y}\) for all \(y \geq 0\).
\end{proof}

\begin{proposition}
  Basic properties of the integral (for positive functions): suppose \(f, g \geq 0\) are measurable on \((X, \mathcal A, \mu)\).
  \begin{enumerate}
  \item linearity: for all \(\alpha, \beta \geq 0\), \(\mu(\alpha f + \beta y) = \alpha \mu(f) + \beta \mu(g)\).
  \item positivity: if \(0 \leq f \leq g\) then \(\mu(f) \leq \mu(g)\).
  \item if \(\mu(f) = 0\) then \(f = 0\) \(\mu\)-almost everywhere.
  \item if \(f = g\) \(\mu\)-almost everywhere then \(\mu(f) = \mu(g)\).
  \end{enumerate}
\end{proposition}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item Follows from the same property for simple functions and from \Cref{lem:approximation by simple functions} combined with monotone convergence theorem.
  \item Obvious from definition.
  \item
    \[
      \{x \in X: f(x) \neq 0\} = \bigcup_{n \geq 0} \{x \in X: f(x) > \frac{1}{n}\}
    \]
    set \(g_n = \frac{1}{n} \mathbf 1_{\{x \in X: f(x) > 1/n\}}\) which is simple and \(g_n \leq f\) so by definition of integral \(\mu(g_n) \leq \mu(f)\) so \(\mu(g_n) = 0\), i.e.\ \(\mu(\{x: f(x) > \frac{1}{n}\}) = 0\).
  \item Note that if \(E \in \mathcal A\), \(\mu(E^c) = 0\) then
    \[
      \mu(h \mathbf 1_E) = \mu(h)
    \]
    for all \(h\) simple. Thus it holds for all \(h \geq 0\) measurable. Now take \(E = \{x: f(x) = g(x)\}\).
  \end{enumerate}
\end{proof}

\begin{proposition}[linearity of integral]
  Suppose \(f, g\) are \(\mu\)-integrable functions and \(\alpha, \beta \in \R\). Then \(\alpha f + \beta g\) is \(\mu\)-integrable and
  \[
    \mu(\alpha f + \beta g) = \alpha \mu(f) + \beta \mu(g).
  \]
\end{proposition}

\begin{proof}
  We have shown the case when \(\alpha, \beta \geq 0\) and \(f, g \geq 0\). In the general case, use the positive and negative parts.
\end{proof}

\begin{lemma}[Fatou's lemma]\index{Fatou's lemma}
  Suppose \((f_n)_{n \geq 1}\) is a sequence of measurable functions on \((X, \mathcal A, \mu)\) such that \(f_n \geq 0\) for all \(n\). Then
  \[
    \mu(\liminf_{n \to \infty} f_n) \leq \liminf_{n \to \infty} \mu(f_n).
  \]
\end{lemma}

\begin{remark}
  We may not have equality: let \(f_n = \mathbf 1_{[n, n + 1]}\) on \((\R, \mathcal L, m)\). Then \(\mu(f_n) = 1\) but \(\lim_{n \to \infty} f_n = 0\).
\end{remark}

\begin{proof}
  Let \(g_n := \inf_{k \geq n} f_k\). Then \(g_{n + 1} \geq g_n \geq 0\) so by monotone convergence theorem, \(\mu(g_n) \uparrow \mu(g)\) as \(n \to \infty\) where \(g = \lim_{n \to \infty} g = \liminf_{n \to \infty} f_n\) and \(g_n \leq f_n\) so \(\mu(g_n) \leq \mu(f_n)\) for all \(n\). Take \(n \to \infty\),
  \[
    \mu(g) \leq \liminf_{n \to \infty} \mu(f_n).
  \]
\end{proof}

In both monotone convergence theorem and Fatou's lemma we assumed that the sequence of functions is nonnegative. There is another version of convergence theorem where we replace nonnegativity by domination:

\begin{theorem}[Lebesgue's dominated convergence theorem]\index{Lebesgue's dominated convergence theorem}
  Let \((f_n)_{n \geq 1}\) be a sequence of measurable functions on \((X, \mathcal A, \mu)\) and \(g\) a \(\mu\)-integrable function on \(X\). Assume \(|f_n| \leq g\) for all \(n\) (domination assumption) and assume for all \(x \in X\), \(\lim_{n \to \infty} f_n(x) = f(x)\). Then \(f\) is \(\mu\)-integrable and
  \[
    \mu(f) = \lim_{n \to \infty} \mu(f_n).
  \]
\end{theorem}

This allows us to swap limit and integral.

\begin{proof}
  \(|f_n| \leq g\) so \(|f| \leq g\) so \(\mu(|f|) \leq \mu(g) < \infty\) and \(f\) is integrable. Note that \(g + f_n \geq 0\) so by Fatou's lemma,
  \[
    \mu(\liminf_{n \to \infty} (g + f_n)) \leq \liminf_{n \to \infty} \mu(g + f_n).
  \]
  But \(\liminf_{n \to \infty} (g + f_n) = g + f\) and by linearity \(\mu(g + f_n) = \mu(g) + \mu(f_n)\), so
  \[
    \mu(g) + \mu(f) \leq \mu(g) + \liminf_{n \to \infty} \mu(f_n),
  \]
  i.e.
  \[
    \mu(f) \leq \liminf_{n \to \infty} \mu(f_n).
  \]
  Do the same with \(g - f_n\) in place of \(g + f_n\), get
  \[
    \mu(-f) \leq \liminf_{n \to \infty} \mu(-f_n) = - \limsup_{n \to \infty} \mu(f_n)
  \]
  so
  \[
    \mu(f) = \lim_{n \to \infty} \mu(f_n).
  \]
\end{proof}

\begin{corollary}[exchanging integral and summation]
  Let \((X, \mathcal A, \mu)\) be a measure space and let \((f_n)_{n \geq 1}\) be a sequence of measurable functions on \(X\).
  \begin{enumerate}
  \item If \(f_n \geq 0\) then
    \[
      \mu(\sum_{n \geq 1} f_n) = \sum_{n \geq 1} \mu(f_n).
    \]
  \item If \(\sum_{n \geq 1} |f_n|\) is \(\mu\)-integrable then \(\sum_{n \geq 1} f_n\) is \(\mu\)-integrable and
    \[
      \mu(\sum_{n \geq 1} f_n) = \sum_{n \geq 1} \mu(f_n).
    \]
  \end{enumerate}
\end{corollary}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item Let \(g_N = \sum_{n = 1}^N f_n\), then \(g_N \uparrow \sum_{n \geq 1} f_n\) as \(N \to \infty\) so the result follows from monotone convergence theorem.
  \item Let \(g = \sum_{n \geq 1} |f_n|\) and \(g_N\) as above. Then \(|g_N| \leq g\) for all \(N\) so the domination assumption holds. The result thus follows from dominated convergence theorem.
  \end{enumerate}
\end{proof}

\begin{corollary}[differentiation under integral sign]
  Let \((X, \mathcal A, \mu)\) be a measure space. Let \(U \subseteq \R\) be an open set and let \(f: U \times X \to \R\) be such that
  \begin{enumerate}
  \item \(x \mapsto f(t, x)\) is \(\mu\)-integrable for all \(t \in U\),
  \item \(t \mapsto f(t, x)\) is differentiable for all \(x \in X\),
  \item domination: there exists \(g: X \to \R\) \(\mu\)-integrable such that for all \(t \in U, x \in X\),
    \[
      \frac{\partial f}{\partial t} (t, x) \leq g(x).
    \]
  \end{enumerate}
  Then \(x \mapsto \frac{\partial f}{\partial t}(t, x)\) is \(\mu\)-integrable for all \(t \in U\) and if we set \(F(t) = \int_X f(t, x) d\mu\) then \(F\) is differentiable and
  \[
    F'(t) = \int_X \frac{\partial f}{\partial t}(t, x) d\mu.
  \]
\end{corollary}

\begin{proof}
  Pick \(h_n > 0, h_n \to 0\) and define
  \[
    g_n(t, x) := \frac{1}{h_n} (f(t + h_n, x) - f(t, x)).
  \]
  Then
  \[
    \lim_{n \to \infty} g_n(t, x) = \frac{\partial f}{\partial t}(t, x).
  \]
  By mean value theorem, there exists \(\theta_{t, n, x} \in [t, t + h_n]\) such that
  \[
    g_n(t, x) = \frac{\partial f}{\partial t}(\theta, x)
  \]
  so
  \[
    |g_n(t, x)| \leq g(x)
  \]
  by domination assumption. Now apply dominated convergence theorem.
\end{proof}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item If \(f: [a, b] \to \R\) is continuous where \(a < b\) in \(\R\), then \(f\) is \(m\)-integrable (where \(m\) is the Lebesgue measure) and \(m(f) = \int_a^b f(x)dx\) is the Riemann integral. In general if \(f\) is only assumed to be bounded, then \(f\) will be Riemann integrable if and only if the points of discontinuity of \(f\) is an \(m\)-null set. See example sheet 2.
  \item If \(g \in \GL_d(\R)\) and \(f \geq 0\) is Borel measurable on \(\R^d\), then
    \[
      m(f \compose g) = \frac{1}{|\det g|} m(f).
    \]
    See example sheet 2. In particular \(m\) is invariant under linear transformation whose determinant has absolute value \(1\), e.g.\ rotation.
  \end{enumerate}
\end{remark}

\begin{remark}
  In each of monotone convergence theorem, Fatou's lemma and dominated convergence theorem, we can replace pointwise assumption by the corresponding \(\mu\)-almost everywhere. The same conclusion holds. Indeed, let
  \[
    E = \{x \in X: \text{ assumptions hold at } x\}
  \]
  so \(E^c\) is a \(\mu\)-null set. Replace each \(f_n\) (and similarly \(g\) etc) by \(\mathbf 1_E f_n\). Then assumptions then hold everywhere as \(\mu(f \mathbf 1_E) = \mu(f)\) for all \(f\) measurable.
\end{remark}

\section{Product measures}

\begin{definition}[product \(\sigma\)-algebra]\index{product \(\sigma\)-algebra}
  Let \((X, \mathcal A)\) and \((Y, \mathcal B)\) be measurable spaces. The \(\sigma\)-algebra of subsets of \(X \times Y\) generated by the product sets \(E \times F\) where \(E \in \mathcal A, F \in \mathcal B\) is called the \emph{product \(\sigma\)-algebra} of \(\mathcal A\) and \(\mathcal B\) and is denoted by \(\mathcal A \otimes B\).
\end{definition}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item By analogy with the notion of product topology, \(\mathcal A \otimes \mathcal B\) is the smallest \(\sigma\)-algebra of subsets of \(X \times Y\) making the two projection maps measurable.
  \item \(\mathcal B(\R^{d_1}) \otimes \mathcal B(\R^{d_2}) = \mathcal B(\R^{d_1 + d_2})\). See example sheet. However this is not so for \(\mathcal L(\R^d)\).
  \end{enumerate}
\end{remark}

\begin{lemma}
  If \(E \subseteq X \times Y\) is \(\mathcal A \otimes \mathcal B\)-measurable then for all \(x \in X\), the slice
  \[
    E_x = \{y \in Y: (x, y) \in E\}
  \]
  is in \(\mathcal B\).
\end{lemma}

\begin{proof}
  Let
  \[
    \mathcal E = \{E \subseteq X \times Y: E_x \in \mathcal B \text{ for all } x \in X\}.
  \]
  Note that \(\mathcal E\) contains all product sets \(A \times B\) where \(A \in \mathcal A, B \in \mathcal B\). \(\mathcal E\) is a \(\sigma\)-algebra: if \(E \in \mathcal E\) then \(E^c \in \mathcal E\) and if \(E_n \in \mathcal E\) then \(\bigcup E_n \in \mathcal E\) since \((E^c)_x = (E_x)^c\) and \((\bigcup E_n)_x = \bigcup (E_n)_x\).
\end{proof}

\begin{lemma}
  Assume \((X, \mathcal A, \mu)\) and \((Y, \mathcal B, \nu)\) are \(\sigma\)-finite measure spaces. Let \(f: X \times Y \to [0, +\infty]\) be \(\mathcal A \otimes \mathcal B\)-measurable. Then
  \begin{enumerate}
  \item for all \(x \in X\), the function \(y \mapsto f(x, y)\) is \(\mathcal B\)-measurable.
  \item for all \(x \in X\), the map \(x \mapsto \int_Y f(x, y) d\nu(y)\) is \(\mathcal A\)-measurable.
  \end{enumerate}
\end{lemma}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item In case \(f = \mathbf 1_E\) for \(E \in \mathcal A \otimes \mathcal B\) the function \(y \mapsto f(x, y)\) is just \(y \mapsto \mathbf 1_{E_x}(y)\), which is measurable by the previous lemma.

    More generally, the result is true for simple functions and thus for all measurable functions by taking pointwise limit.
  \item By the same reduction we may assume \(f = \mathbf 1_E\) for some \(E \in \mathcal A \otimes \mathcal B\). Now let \(Y = \bigcup_{m \geq 1} Y_m\) with \(\nu(Y_m) < \infty\). Let
    \[
      \mathcal E = \{E \in \mathcal A \otimes \mathcal B: x \mapsto \nu(E_x \cap Y_m) \text{ is \(\mathcal A\)-measurable for all } m\}.
    \]
    \(\mathcal E\) contains all product sets \(E = A \times B\) where \(A \in \mathcal A, B \in \mathcal B\) because \(\nu(E_x \cap Y_m) = \mathbf 1_{x \in \mathcal A} \nu(B \cap Y_m)\). \(\mathcal E\) is stable under complementation:
    \[
      \nu((E^c)_x \cap Y_m) = \nu(Y_m) - \nu(Y_m \cap E_x)
    \]
    where LHS is \(\nu\)-measurable. \(\mathcal E\) is stable under disjoint countable union: let \(E = \bigcup_{n \geq 1} E_n\) where \(E_n \in \mathcal E\) disjoint. Then by \(\sigma\)-additivity
    \[
      \nu(E_x \cap Y_m) = \sum_{n \geq 1} \nu((E_n)_x \cap Y_m)
    \]
    which is \(\mathcal A\)-measurable.

    The product sets form a \(\pi\)-system and generates the product measure so by Dynkin lemma \(\mathcal E = \mathcal A \otimes \mathcal B\).
  \end{enumerate}
\end{proof}

\begin{definition}[product measure]\index{product measure}
  Let \((X, \mathcal A, \mu)\) and \((Y, \mathcal B, \nu)\) be measure spaces and \(\mu, \nu\) \(\sigma\)-finite. Then there exists a unique \emph{product measure}, denoted by \(\mu \otimes \nu\), on \(\mathcal A \otimes \mathcal B\) such that for all \(A \in \mathcal A, B \in \mathcal B\),
  \[
    (\mu \otimes \nu) (A \times B) = \mu(A)\nu(B).
  \]
\end{definition}

\begin{proof}
  Uniqueness follows from Dynkin lemma. For existence, set
  \[
    \sigma(E) = \int_X \nu(E_x) d\mu(x).
  \]
  \(\sigma\) is well-defined because \(x \mapsto \nu(E_x)\) is \(\mathcal A\)-measurable by lemma 2. \(\sigma\) is countably-additive: suppose \(E = \bigcup_{n \geq 1} E_n\) where \(E_n \in \mathcal A \otimes \mathcal B\) disjoint, then
  \[
    \sigma(E)
    = \int_X \nu(E_x) d\mu(x)
    = \int_X \sum_{n \geq 1} \nu((E_n)_x) d\mu x
    = \sum_{n \geq 1} \int_X \nu((E_n)_x) d\mu(x)
    = \sum_{n \geq 2} \sigma(E_n)
  \]
  by a corollary of MCT.
\end{proof}

\begin{theorem}[Tonelli-Fubini]\index{Tonelli-Fubini theorem}
  Let \((X, \mathcal A, \mu)\) and \((Y, \mathcal B, \nu)\) be \(\sigma\)-finite measure spaces.
  \begin{enumerate}
  \item Let \(f: X \times Y \to [0, +\infty]\) be \(\mathcal A \otimes \mathcal B\)-measurable. Then
    \[
      \int_{X \times Y} f(x, y) d(\mu \otimes \nu)
      = \int_X \int_Y f(x, y) d\nu(y) d\mu(x)
      = \int_X \int_X f(x, y) d\mu(x) d\nu(y).
    \]
  \item If \(f: X \times Y \to \R\) is \(\mu \otimes \nu\)-integrable then for \(\mu\)-almost everywhere \(x\), \(y \mapsto f(x, y)\) is \(\nu\)-integrable, and for \(\nu\)-almost everywhere \(y\), \(x \mapsto f(x, y)\) is \(\mu\)-integrable and
    \[
      \int_{X \times Y} f(x, y) d(\mu \otimes \nu)
      = \int_X \int_Y f(x, y) d\nu(y) d\mu(x)
      = \int_X \int_X f(x, y) d\mu(x) d\nu(y).
    \]
  \end{enumerate}
\end{theorem}

Without the nonnegativity or integrability assumption, the result is false in general. For example for \(X = Y = \N\), let \(\mathcal A = \mathcal B\) be discrete \(\sigma\)-algebras and \(\mu = \nu\) counting measure. Let \(f(n, m) = \mathbf 1_{n = m} - \mathbf 1_{n = m + 1}\). Check that
\begin{align*}
  \sum_{n \geq 1} f(n, m) &= 0 \\
  \sum_{m \geq 1} f(n, m) &=
                            \begin{cases}
                              0 & n \geq 2 \\
                              1 & n = 1
                            \end{cases}
\end{align*}
so
\[
  \sum_{n \geq 1} \sum_{m \geq 1} f(n, m) \neq \sum_{m \geq 1} \sum_{n \geq 1} f(n, m).
\]

\begin{proof}\leavevmode
  \begin{enumerate}
  \item The result holds for \(f = \mathbf 1_E\) where \(E \in \mathcal A \otimes \mathcal B\) by the definition of product measure and lemma 2, so it holds for all simple functions. Now take limits and apply MCT.
  \item Write \(f = f^+ - f^-\) and apply 1.
  \end{enumerate}
\end{proof}

\begin{note}\leavevmode
  \begin{enumerate}
  \item The Lebesgue measure \(m_d\) on \(\R^d\) is equal to \(m_1 \otimes \dots \otimes m_1\), because it is true on boxes and extend by uniqueness of measure.
  \item \(E \in \mathcal A \otimes \mathcal B\) if is \(\mu \otimes \nu\)-null if and only if for \(\mu\)-almost every \(x\), \(\nu(E_x) = 0\).
  \end{enumerate}
\end{note}

\section{Foundations of probability theory}

Modern probability theory was founded by Kolmogorov, who formulated the axioms of probability theory in 1933 in his thesis \emph{Foundations on the Theory of Probability}. He defined a probability space to be a measure space \((\Omega, \mathcal F, \P)\). The interpretation is as follow: \(\Omega\) is the universe of possible \emph{outcomes}. However, we wouldn't be able to assign probability to every single outcome unless the space is discrete. Instead we are interested in studying some subsets of \(\Omega\), which are called \emph{events} and contained in \(\mathcal F\). Finally \(\P\) is a probability measure with \(\P(\Omega) = 1\). Thus for \(A \in \mathcal F\), \(\P(A \text{ occurs}) \in [0, 1]\). Thus finite additivity of \(\P\) says that if \(A\) and \(B\) never occurs simultaneously then \(\P(A \text{ or } B = \P(A) + \P(B)\). \(\sigma\)-additivity is slightly more difficult to justify and it is perhaps to see the equivalent notion \emph{continuity}: if \(A_{n + 1} \supseteq A_n\) and \(\bigcap_{n \geq 1} A_n = \emptyset\) then \(\P(A_n) \to 0\) as \(n \to \infty\).

\begin{definition}[probability measure, probability space]\index{probability measure}\index{probability space}
  Let \(\Omega\) be a set and \(\mathcal F\) a \(\sigma\)-algebra on \(\Omega\). A measure \(\mu\) on \((\Omega, \mathcal F)\) is called a \emph{probability measure} if \(\mu(\Omega) = 1\) and the measure space \((\Omega, \mathcal F, \mu)\) is called a \emph{probability space}.
\end{definition}

\begin{definition}[random variable]\index{random variable}
  A measurable function \(X: \Omega \to \R\) is called a \emph{random variable}.
\end{definition}

We usually use a capital letter to denote a random variable.

\begin{definition}[expectation]\index{expectation}
  If \((\Omega, \mathcal F, \P)\) is a probability space then the \(\P\)-integral is called \emph{expectation}, denoted \(\E\).
\end{definition}

\begin{definition}[distribution/law]\index{distribution}\index{law}
  A random variable \(X: \Omega \to \R\) on a probability space \((\Omega, \mathcal F, \P)\) determines a Borel measure \(\mu_X\) on \(\R\) defined by
  \[
    \mu_X((-\infty, t]) = \P(X \leq t) = \P(\{\omega \in \Omega: X(\omega) \leq t\})
  \]
  and \(\mu_X\) is called the \emph{distribution} of \(X\), or the \emph{law} of \(X\).
\end{definition}

\begin{note}
  \(\mu_X\) is the image of \(\P\) under
  \begin{align*}
    \Omega &\to \R \\
    \omega &\mapsto X(\omega)
  \end{align*}
\end{note}

\begin{definition}[distribution function]\index{distribution function}
  The function
  \begin{align*}
    F_X: \R &\to [0, 1] \\
    t &\mapsto \P(X \leq t)
  \end{align*}
  is called the \emph{distribution function} of \(X\).
\end{definition}

\begin{proposition}\leavevmode
  If \((\Omega, \mathcal F, \P)\) is a probability space and \(X: \Omega \to \R\) is a random variable then \(F_X\) is non-decreasing, right-continuous and it determines \(\mu_X\) uniquely.
\end{proposition}

\begin{proof}
  Given \(t_n \downarrow t\),
  \[
    F_X(t_n) = \P(X \leq t_n) \to \P(\bigcap_{n \geq 1} \{X \leq t_n\}) = \P(\{X \leq t\}) = F_X(t)
  \]
  by downward monotone convergence for sets. Uniqueness follows from Dynkin lemma applied to the \(\pi\)-system \(\{\emptyset\} \cup \{(-\infty, t]\}_{t \in \R}\).
\end{proof}

Conversely,
\begin{proposition}
  If \(F: \R \to [0, 1]\) is a non-decreasing right-continuous function with
  \begin{align*}
    \lim_{t \to -\infty} F(t) &= 0 \\
    \lim_{t \to +\infty} F(t) &= 1
  \end{align*}
  then there exists a unique probability measure \(\mu\) on \(\R\) such that
  \[
    F(t) = \mu((-\infty, t])
  \]
  for all \(t \in \R\).
\end{proposition}

\begin{remark}
  The measure \(\mu\) is called the Lebesgue-Stieltjes measure on \(\R\) associated to \(F\). Furthermore for all \(a, b \in \R\),
  \[
    \mu((a, b]) = F(b) - F(a).
  \]
  We can also construct Lebesgue measure this way.
\end{remark}

\begin{proof}
  Uniqueness is the same as above. For existence, we use the lemma
  \begin{lemma}
    Let
    \begin{align*}
      g: (0, 1) &\to \R \\
      y &\mapsto \inf \{x \in \R: F(x) \geq y\}
    \end{align*}
    then \(g\) is non-decreasing, left-continuous and for all \(x \in \R, y \in (0, 1)\), \(g(y) \leq x\) if and only if \(F(x) \geq y\).
  \end{lemma}

  \begin{proof}
    Let
    \[
      I_y = \{x \in \R: F(x) \geq y\}.
    \]
    Clearly if \(y_1 \geq y_2\) then \(I_{y_1} \subseteq I_{y_2}\) so \(g(y_2) \leq g(y_1)\) so \(g\) is non-decreasing. \(I_y\) is an interval of \(\R\) because if \(x > x_1\) and \(x_1 \in I_y\) then \(F(x) \geq F(x_1) \geq y\) so \(x \in I_y\). So \(I_y\) is an interval with endpoints \(g(y)\) and \(+ \infty\). But \(F\) is right-continuous so \(g(y) = \min I_y\) and the minimum is obtained. Thus \(I_y = [g(y), + \infty)\).

    This means that \(x \geq g(y)\) if and only if \(x \in I_y\) if and only if \(F(x) \geq y\).

    Finally for left-continuity, suppose \(y_n \uparrow y\) then \(\bigcap_{n \geq 1} I_{y_n} = I_y\) by definition of \(I_y\) so \(g(y_n) \to g(y)\).
  \end{proof}
  \begin{remark}
    If \(F\) is continuous and strictly increasing then \(g = F^{-1}\).
  \end{remark}

  Now back to the proposition. Set \(\mu = g_* m\) where \(m\) is the Lebesgue measure on \((0, 1)\). \(\mu\) is a probability measure as \(g\) is Borel-measurable. By the lemma
  \[
    \mu((a, b]) = m(g^{-1}(a, b]) = m((F(a), F(b))) = F(b) - F(a).
  \]
\end{proof}

\begin{proposition}
  If \(\mu\) is a Borel probability measure on \(\R\) then there exists some probability space \((\Omega, \mathcal F, \P)\) and a random variable \(X\) on \(\Omega\) such that \(\mu_X = \mu\).

  In fact, one can even pick \(\Omega = (0, 1)\), \(\mathcal F = \mathcal B(0, 1)\) and \(\P = m\), the Lebesgue measure.
\end{proposition}

\begin{proof}
  For the first claim set \(\Omega = \R\), \(\mathcal F = \mathcal B(\R)\), \(\P = \mu\) and \(X(x) = x\).

  For the second claim, set \(F(t) = \mu((-\infty, t])\) and take \(X = g\) where \(g\) is the auxillary function defined in the previous lemma, namely
  \[
    X(\omega) = \inf\{x: F(x) \geq \omega\}.
  \]
  Check that \(\mu_X = \mu\):
  \begin{align*}
    \mu_X((a, b])
    &= \P(X \in (a, b]) \\
    &= m(\{\omega \in (0, 1): a < X(w) \leq b\}) \\
    &= m(\{\omega \in (0, 1): F(a) < \omega < F(b)\})
  \end{align*}
\end{proof}

\begin{remark}
  If \(\mu\) is a Borel probability measure on \(\R\) such that \(\mu = f dt\) for some \(f \geq 0\) measurable, we say that \(\mu\) has a \emph{density}\index{density} (with respect to Lebesgue measure) and \(f\) is called the \emph{density of \(\mu\)}. Here \(\mu = f dt\) means that \(\mu((a, b]) = \int_a^b f(t) dt\).
\end{remark}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item uniform distribution on \([0, 1]\):
    \begin{align*}
      f(t) &= \mathbf 1_{[0, 1]}(t) \\
      F(t) &= \mu((-\infty, t] \cap [0, 1])
    \end{align*}
  \item exponential distribution of rate \(\lambda\):
    \begin{align*}
      f_\lambda(t) &= \lambda e^{-\lambda t} \mathbf 1_{t \geq 0} \\
      F_\lambda(t) &= \int_{-\infty}^t f_\lambda(s) ds = \mathbf 1_{t \geq 0} (1 - e^{-\lambda t}) \\
      \end{align*}
    \item Gaussian distribution with standard deviation \(\sigma\) and mean \(m\):
      \begin{align*}
        f_{\sigma, m}(t) &= \frac{1}{\sqrt{2\pi \sigma^2}} \exp(- \frac{(t - m)^2}{2\sigma^2}) \\
        F_{\sigma, m}(t) & = \int_{-\infty}^t \frac{1}{\sqrt{2\pi \sigma^2}} \exp (- \frac{(s - m)^2}{2\sigma^2}) ds
      \end{align*}
  \end{enumerate}
\end{eg}

\begin{definition}[mean, moment, variance]\index{mean}\index{moment}\index{variance}
  If \(X\) is a random variable then
  \begin{enumerate}
  \item \(\E(X)\) is called the \emph{mean},
  \item \(\E(X^k)\) is called the \emph{\(k\)th-moment} of \(X\),
  \item \(\var(X) = \E((X - \E X)^2) = \E(X^2) - \E(X)^2\) is called the \emph{variance}.
  \end{enumerate}
\end{definition}

\begin{remark}
  Suppose \(f \geq 0\) is measurable and \(X\) is a random variable. Then
  \[
    \E(f(X)) = \int_\R f(x) d \mu_X(x)
  \]
  where by definition of \(\mu_X = X_* \P\).
\end{remark}

\section{Independence}

Independence is the key notion that makes probability theory distinct from (abstract) measure theory.

\begin{definition}[independence]\index{independence}
  Let \((\Omega, \mathcal F, \P)\) be a probability space. A sequence of events \((A_n)_{n \geq 1}\) is called \emph{independent} or \emph{mutually independent} if for all \(F \subseteq \N\) finite,
  \[
    \P(\bigcap_{i \in F} A_i) = \prod_{i \in F} \P(A_i).
  \]
\end{definition}

\begin{definition}[independent \(\sigma\)-algebra]\index{\(\sigma\)-algebra!independence}
  A sequence of \(\sigma\)-algebras \((\mathcal A_n)_{n \geq 1}\) where \(\mathcal A_n \subseteq \mathcal F\) is called \emph{independent} if for all \(A_n \in \mathcal A_n\), the family \((A_n)_{n \geq 1}\) is independent.
\end{definition}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item To prove that \((\mathcal A_n)_{n \geq 1}\) is an independent family, it is enough to check the independence condition for all \(A_n\)'s with \(A_n \in \Pi_n\) where \(\Pi_n\) is a \(\pi\)-system generating \(\mathcal A_n\). The proof is an application of Dynkin lemma. For example for \(\sigma\)-algebras \(\mathcal A_1, \mathcal A_2\), suffices to check
    \[
      \P(A_1 \cap A_2) = \P(A_1) \P(A_2)
    \]
    for all \(A_1 \in \Pi_1, A_2 \in \Pi_2\). Fix \(A_2 \in \Pi_2\), look at the measures
    \begin{align*}
      A_1 &\mapsto \P(A_1 \cap A_2) \\
      A_1 &\mapsto \P(A_1) \P(A_2)
    \end{align*}
    on \(\mathcal A_1\). They coincide on \(\Pi_1\) by assumption and hence everywhere on \(\mathcal A_1\). Subsequently consider \(\mathcal A_2\).
  \end{enumerate}
\end{remark}

\begin{notation}
  Suppose \(X\) is a random variable. Denote by \(\sigma(X)\) the smallest \(\sigma\)-subalgebra \(\mathcal A\) of \(\mathcal F\) such that \(X\) is \(\mathcal A\)-measurable, i.e.
  \[
    \sigma(X) = \sigma(\{\omega \in \Omega: X(\omega) \leq t\}_{t \in \R}).
  \]
\end{notation}
% terminal object?

\begin{definition}[independence]\index{random variable!independence}
  A sequence of random variables \((X_i)_{i \geq 1}\) is called \emph{independent} if the sequence of \(\sigma\)-subalgebras \((\sigma(X_i))_{i \geq 1}\) is independent.
\end{definition}

\begin{remark}
  This is equivalent to the condition that for all \((t_i)_{i \geq 1}\), for all \(n\),
  \[
    \P((X_1 \leq t_1) \cap \dots \cap (X_n \leq t_n)) = \prod_{i = 1}^n \P(X_i \leq t_i).
  \]

  Yet another equivalent formulation is
  \[
    \mu_{(X_1, \dots, X_n)} = \bigotimes_{i = 1}^n \mu_{X_i}
  \]
  as Borel probability measures on \(\R^n\). ``The joint law is the same as the product of individual laws''.
\end{remark}

\begin{note}
  Note that independence is a property of a \emph{family} so pairwise independence is necessary but not sufficient for independence. A famous counterexample is \emph{Berstein's example}: take \(X\) and \(Y\) to be random variables for two independent fair coins flips. Set \(Z = |X - Y|\). Then \(Z = 0\) if and only if \(X = Y\). Check that
  \[
    \P(Z = 0) = \P(Z = 1) = \frac{1}{2}
  \]
  and each pair \((X, Y), (X, Z)\) and \((Y, Z)\) is independent. But \((X, Y, Z)\) is not independent.
\end{note}

\begin{proposition}
  If \(X\) and \(Y\) are independent random variables, \(X \geq 0, Y \geq 0\) then
  \[
    \E(XY) = \E(X) \E(Y).
  \]
\end{proposition}

\begin{proof}
  Essentially Tonelli-Fubini:
  \begin{align*}
    \E(XY) &= \int_{\R^2} xy d\mu_{X, Y} (x, y) = \int_{\R^2} d \mu_X(x) d \mu_Y(y) \\
           &= \left(\int_\R x d \mu_X(x) \right) \left(\int_\R y d \mu_Y(y) \right) \\
           &= \E(X) \E(Y)
  \end{align*}
\end{proof}

\begin{remark}
  As in Tonelli-Fubini, we may require \(XY\) to be integrable instead and the same conclusion holds.
\end{remark}

\begin{eg}
  Let \(\Omega = (0, 1), \mathcal F = \mathcal B(0, 1), \P = m\) the Lebesgue measure. Write the decimal expansion of \(\omega \in (0, 1)\) as
  \[
    \omega = 0. \varepsilon_1 \varepsilon_2 \dots
  \]
  where \(\varepsilon_i(\omega) \in \{0, \dots, 9\}\). Choose a convention so that each \(\omega\) has a well-defined expansion (to avoid things like \(0.099 \dots = 0.100\dots\)). Now let \(X_n(\omega) = \varepsilon_n(\omega)\). Claim that the \((X_n)_{n \geq 1}\) are iid.\ random variables uniformly distributed on \(\{0, \dots, 9\}\), where ``iid.'' stands for \emph{independently and identically distributed}\index{iid.}.

  \begin{proof}
    Easy check. For example \(X_1(\omega) = \floor{10 \omega}\) so
    \[
      \P(X_1 = i_1) = \frac{1}{10}.
    \]
    Similarly for all \(n\)
    \[
      \P(X_1 = i_1, \dots, X_n = i_n) = \frac{1}{10^n}, \quad \P(X_n = i_n) = \frac{1}{10}
    \]
    so
    \[
      \P(X_1 = i_1, \dots, X_n = i_n) = \prod_{i = 1}^n \P(X_k = i_k).
    \]
  \end{proof}
\end{eg}

\begin{remark}
  \[
    \omega = \sum_{n \geq 1} \frac{X_n(\omega)}{10^n}
  \]
  is distributed according to Lebesgue measure so if we want we can construct Lebesgue measure as the law of this random variable.
\end{remark}

\begin{proposition}[infinite product of product measure]\index{product measure!infinite}
  Let \((\Omega_i, \mathcal F_i, \mu_i)_{i \geq 1}\) be a sequence of probability spaces, \(\Omega = \prod_{i \geq 1} \Omega_i\) and \(\mathcal E\) be the Boolean algebra of cylinder sets, i.e.\ sets of the form
  \[
    A \times \prod_{i \geq n} \Omega_i
  \]
  for some \(A \in \bigotimes_{i = 1}^n \mathcal F_i\). Set \(\mathcal F = \sigma(\mathcal E)\), the \emph{infinite product \(\sigma\)-algebra}\index{infinite product \(\sigma\)-algebra}\index{product \(\sigma\)-algebra!infinite}. Then there is a unique probability measure \(\mu\) on \((\Omega, \mathcal F)\) such that it agrees with product measures on all cylinder sets, i.e.
  \[
    \mu(A \times \prod_{i > n} \Omega_i) = (\bigotimes_{i = 1}^n \mu_i)(A)
  \]
  for all \(A \in \bigotimes_{i = 1}^n \mathcal F_i\).
\end{proposition}

\begin{proof}
  Omitted. See example sheet 3.
\end{proof}

\begin{lemma}[Borel-Cantelli]\index{Borel-Cantelli lemma}
  Let \((\Omega, \mathcal F, \P)\) be a probability space and \((A_n)_{n \geq 1}\) a sequence of events.
  \begin{enumerate}
  \item If \(\sum_{n \geq 1} \P(A_n) < \infty\) then
    \[
      \P(\limsup_n A_n) = 0.
    \]
  \item Conversely, if \((A_n)_{n \geq 1}\) are independent and \(\sum_{n \geq 1} \P(A_n) = \infty\) then
    \[
      \P(\limsup_n A_n) = 1.
    \]
  \end{enumerate}
\end{lemma}

Note that \(\limsup_n A_n\) is also called \(A_n\) io. meaning ``infinitely often''\index{io.}.

\begin{proof}\leavevmode
  \begin{enumerate}
  \item Let \(Y = \sum_{n \geq 1} \mathbf 1_{A_n}\) be a random variable. Then
    \[
      \E(Y) = \sum_{n \geq 1} \E(\mathbf 1_{A_n}) = \sum_{n \geq 1} \P(A_n).
    \]
    Since \(Y \geq 0\), recall that we prove that \(\E(Y) < \infty\) implies that \(Y < \infty\) almost surely, i.e.\ \(\P\)-almost everywhere.
  \item Note that
    \[
      (\limsup_n A_n)^c = \bigcup_N \bigcap_{n \geq N} A_n^c
    \]
    so
    \begin{align*}
      \P(\bigcap_{n \geq N} A_n^c)
      &\leq \P(\bigcap_{n = N}^M A_n^c) \\
      &= \prod_{n = N}^M \P(A_n^c)
      = \prod_{n = N}^M (1 - \P(A_n)) \\
      &\leq \prod_{n = N}^M \exp (-\P(A_n)) \\
      &\leq \exp (- \sum_{n = N}^M \P(A_n)) \\
      &\to 0
    \end{align*}
    as \(M \to \infty\). Thus
    \[
      \P(\bigcap_{n \geq N} A_n^c) = 0
    \]
    for all \(N\) so
    \[
      \P(\bigcup_N \bigcap_{n \geq N} A_n^c) = 0.
    \]
  \end{enumerate}
\end{proof}

\begin{definition}[random/stochastic process, filtration, tail \(\sigma\)-algebra, tail event]\index{random process}\index{random process}\index{filtration}\index{\(\sigma\)-algebra!tail}\index{tail event}
  Let \((\Omega, \mathcal F, \P)\) be a probability space and \((X_n)_{n \geq 1}\) a sequence of random variables.
  \begin{enumerate}
  \item \((X_n)_{n \geq 1}\) is sometimes called a \emph{random process} or \emph{stochastic process}.
  \item
    \[
      \mathcal F_n = \sigma(X_1, \dots, X_n) \subseteq \mathcal F
    \]
    is called the associated \emph{filtration}. \(\mathcal F_n \subseteq \mathcal F_{n + 1}\).
  \item
    \[
      \mathcal C = \bigcap_{n \geq 1} \sigma(X_n, X_{n + 1}, \dots)
    \]
    is called the \emph{tail \(\sigma\)-algebra} of the process. Its elements are called \emph{tail events}.
  \end{enumerate}
\end{definition}

\begin{eg}
  Tail events are those not affected by the first few terms in the sequence of random variables. For example,
  \[
    \{\omega \in \Omega: \lim_n X_n(\omega) \text{ exists}\}
  \]
  is a tail event, so is
  \[
    \{\omega \in \Omega: \limsup_n X_n(\omega) \geq T\}.
  \]
\end{eg}

\begin{theorem}[Kolmogorov \(0 - 1\) law]\index{Kolmogorov \(0 - 1\) law}
  If \((X_n)_{n \geq 1}\) is a sequence of mutually independent random variables then for all \(A \in \mathcal C\),
  \[
    \P(A) \in \{0, 1\}.
  \]
\end{theorem}

\begin{proof}
  Pick \(A \in \mathcal C\). Fix \(n\). For all \(B \in \sigma(X_1, \dots, X_n)\),
  \[
    \P(A \cap B) = \P(A) \P(B)
  \]
  as \(\mathcal C\) is independent of \(\sigma(X_1, \dots, X_n)\). The measures \(B \mapsto \P(A) \P(B)\) and \(B \mapsto \P(A \cap B)\) coincide on each \(\mathcal F_n\) so on \(\bigcup_{n \geq 1} \mathcal F_n\). Hence they coincide on \(\sigma(\bigcup_{n \geq 1} \mathcal F_n) \supseteq \mathcal C\) so
  \[
    \P(A) = \P(A \cap A) = \P(A) \P(A)
  \]
  so
  \[
    \P(A) \in \{0, 1\}.
  \]
\end{proof}

\subsection{Useful inequalities}

\begin{proposition}[Cauchy-Schwarz]
  Suppose \(X, Y\) are random variables then
  \[
    \E(|XY|) \leq \sqrt{\E(X^2) \cdot \E(Y^2)}.
  \]
\end{proposition}

\begin{proof}
  For all \(t \in \R\),
  \[
    0 \leq \E((|X| + t|Y|)^2) = \E(X^2) + 2t \E(|XY|) + t^2\E(Y^2)
  \]
  so viewed as a quadratic in \(t\), the discriminant is nonpositive, i.e.
  \[
    (\E(|XY|)^2 - \E(X)^2 - \E(Y)^2 \leq 0.
  \]
\end{proof}

\begin{proposition}[Markov]
  Let \(X \geq 0\) be a random variable. Then for all \(t \geq 0\),
  \[
    t \P(X \geq t) \leq \E(X).
  \]
\end{proposition}

\begin{proof}
  \[
    \E(X)
    \geq \E(X \mathbf 1_{X \geq t})
    \geq \E(t \mathbf 1_{X \geq t})
    = t \P(X \geq t)
  \]
\end{proof}

\begin{proposition}[Chebyshev]
  Let \(Y\) be a random variable with \(\E(Y^2) < \infty\), then for all \(t \in \R\),
  \[
    t^2 \P(|Y - \E(Y)| \geq t) \leq \var Y.
  \]
\end{proposition}

\(\E(Y^2) < \infty\) implies that \(\E(|Y|) < \infty\) by Cauchy-Schwarz, so \(\var Y < \infty\). The converse is more subtle.

\begin{proof}
  Apply Markov to \(X = |Y - \E(Y)|^2\).
\end{proof}

\begin{theorem}[strong law of large numbers]\index{strong law of large numbers}\index{law of large numbers}
  Let \((X_n)_{n \geq 1}\) be a sequence of iid.\ random variables. Assume \(\E(|X_1|) < \infty\). Let
  \[
    S_n = \sum_{k = 1}^n X_k,
  \]
  then \(\frac{1}{n} S_n\) converges almost surely to \(\E(X_1)\).
\end{theorem}

\begin{proof}
  We prove the theorem under a stonger condition: we assume \(\E(X_1^4) < \infty\). This implies, by Cauchy-Schwarz, \(\E(X_1^2), \E(|X_1|) < \infty\). Subsequently \(\E(|X_1|^3) < \infty\). The full proof is much harder but will be given later when we have developed enough machinery.

  wlog we may assume \(\E(X_1) = 0\) by replacing \(X_n\) with \(X_n - \E(X_1)\). Have
  \begin{align*}
    \E(S_n^4)
    &= \sum_{i,j, k, \ell} \E(X_iX_jX_kX_\ell).
  \end{align*}
  All terms vanish because \(\E(X_i) = 0\) and \((X_i)_{i \geq 1}\) are independent, except for \(\E(X_i^4)\) and \(\E(X_i^2X_j^2)\) for \(i \neq j\). For example,
  \[
    \E(X_iX_j^3) = \E(X_i) \cdot \E(X_j^3) = 0
  \]
  for \(i \neq j\). Thus
  \[
    \E(S_n^4) = \sum_{i = 1}^n \E (X_i^4) + 6 \sum_{i < j} \E(X_i^2 X_j^2).
  \]
  By Cauchy-Schwarz,
  \[
    \E(X_i^2X_j^2) \leq \sqrt{\E X_i^4 \cdot \E X_j^4} = \E X_1^4
  \]
  so
  \[
    \E(S_n^4) \leq (n + 6 \cdot \frac{n(n - 1)}{2}) \E X_1^4
  \]
  and asymptotically,
  \[
    \E (\frac{S_n}{n})^4 = O(\frac{1}{n^2})
  \]
  so
  \[
    \E (\sum_{n \geq 1} (\frac{S_n}{n})^4) = \sum_{n \geq 1} \E(\frac{S_n}{n})^4 < \infty.
  \]
  Hence \(\sum (\frac{S_n}{n})^4 < \infty\) almost surely and it follows that
  \[
    \lim_{n \to \infty} \frac{S_n}{n} = 0
  \]
  almost surely.
\end{proof}

Strong law of large numbers has a very important statistical implication: we can sample the mean of larger number of iid.\ to detect an unknown law, at least the mean.

\section{Convergence of random variables}

\begin{definition}[weak convergence]\index{weak convergence}
  A sequence of probability measures \((\mu_n)_{n \geq 1}\) on \((\R^d, \mathcal B(\R^d))\) is said to \emph{converge weakly} to a measure \(\mu\) if for all \(f \in C_b(\R^d)\), the set of continuous bounded functions on \(\R^d\),
  \[
    \lim_{n \to \infty} \mu_n(f) = \mu(f).
  \]
\end{definition}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Let \(\mu_n = \delta_{1/n}\) be the \emph{Dirac mass}\index{Dirac mass} on \(\R^d\), i.e.\ for \(x \in \R^d\), \(\delta_x\) is the Borel probability measure on \(\R^d\) such that
    \[
      \delta_x(A) =
      \begin{cases}
        1 & x \in A \\
        0 & x \notin A
      \end{cases}
    \]
    then \(\mu_n \to \delta_0\).
  \item Let \(\mu_n = \mathcal N(0, \sigma_n^2)\), Gaussian distribution with standard deviation \(\sigma_n\), where \(\sigma_n \to 0\), then again \(\mu_n \to \delta_0\). Indeed,
    \begin{align*}
      \mu_n(f)
      &= \int f(x) d\mu_n(x) \\
      &= \int f(x) \frac{1}{\sqrt{2\pi \sigma_n^2}} \exp (- \frac{x^2}{2\sigma_n^2}) dx \\
      &= \int f(x\sigma_n) \frac{1}{\sqrt{2\pi}} \exp (- \frac{x^2}{2}) dx
    \end{align*}
    \(\sigma_n \to 0\) so \(f(x\sigma_n) \to f(0)\) so by dominated convergence theorem, \(\mu_n(f) \to f(0) = \delta_0(f)\).
  \end{enumerate}
\end{eg}

\begin{definition}[convergence of random variable]\index{convergence!almost surely}\index{convergence!in probability}\index{convergence!in measure}\index{convergence!in distribution}
  A sequence \((X_n)_{n \geq 1}\) of \(\R^d\)-valued random variables on \((\Omega, \mathcal F, \P)\) is said to converge to a random variable \(X\)
  \begin{enumerate}
  \item \emph{almost surely} if
    \[
      \lim_{n \to \infty} X_n(\omega) = X(\omega)
    \]
    for \(\P\)-almost every \(\omega\).
  \item \emph{in probability} or \emph{in measure} if for all \(\varepsilon > 0\),
    \[
      \lim_{n \to \infty} \P(\norm{X_n - X} > \varepsilon) = 0.
    \]
    Note that all norms on \(\R^d\) are equivalent so we don't have to specify one.
  \item \emph{in distribution} or \emph{in law} if \(\mu_{X_n} \to \mu_X\) weakly, where \(\mu_X = X_*\P\) is the law of \(X\), a Borel probability measure on \(\R^d\). Equivalently, for all \(f \in C_b(\R^d)\), \(\E(f(X_n)) \to \E(f(X))\).
  \end{enumerate}
\end{definition}

\begin{proposition}
  \(1 \implies 2 \implies 3\).
\end{proposition}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item \(1 \implies 2\):
    \[
      \P(\norm{X_n - X} > \varepsilon) = \E(\mathbf 1_{\norm{X_n - X} > \varepsilon})
    \]
    so if \(X_n \to X\) almost surely then
    \[
      \mathbf 1_{\norm{X_n - X} > \varepsilon} \to 0
    \]
    \(\P\)-almost everywhere so by dominated convergence theorem \(\P(\norm{X_n - X} > \varepsilon) \to 0\).
  \item \(2 \implies 3\): given \(f \in C_b(\R^d)\), need to show that \(\mu_{X_n}(f) \to \mu_X(f)\). But
    \[
      \mu_{X_n}(f) - \mu_X(f) = \E(f(X_n) - f(X)).
    \]
    To bound this, note that \(f\) is continuous and \(\R^d\) is locally compact so it is locally uniformly continuous. In particular for all \(\varepsilon > 0\) exists \(\delta > 0\) such that if \(\norm x < \frac{1}{\varepsilon}\) and \(\norm{y - x} < \delta\) then \(|f(y) - f(x)| < \varepsilon\). Thus
    \begin{align*}
      |\E(f(X_n)  - f(X))|
      &\leq \E(\mathbf 1_{\norm{X_n - X} < \delta} \mathbf 1_{\norm X < 1/\varepsilon} \underbrace{|f(X_n) - f(X)|}_{< \varepsilon}) \\
      &\quad + 2 \underbrace{\norm f_\infty}_{< \infty} (\P(\norm{X_n - X} \geq \delta) + \P(\norm X \geq \frac{1}{\varepsilon}))
    \end{align*}
    so
    \[
      \limsup_{n \to \infty} |\E(f(X_n)  - f(X))|
      \leq \varepsilon + 2 \norm f_\infty \underbrace{\P(\norm X > \frac{1}{\varepsilon})}_{\to 0 \text{ as } \varepsilon \to 0}.
    \]
    which is \(0\) as \(\varepsilon\) is arbitrary.
  \end{enumerate}
\end{proof}

\begin{remark}
  When \(d = 1\), 3 is equivalent to \(F_{X_n}(x) \to F_X(x)\) for all \(x\) as \(n \to \infty\) where \(F_X\) is continuous. See example sheet 3.
\end{remark}

The strict converses do not hold but we can say something weaker.

\begin{proposition}
  If \(X_n \to X\) in probability then there is a subsequence \((n_k)_k\) such that \(X_{n_k} \to X\) almost surely as \(k \to \infty\).
\end{proposition}

\begin{proof}
  We know for all \(\varepsilon > 0\), \(\P(\norm{X_n - X} > \varepsilon) \to 0\) as \(n \to \infty\). So for all \(k\) exists \(n_k\) such that
  \[
    \P(\norm{X_{n_k} - X} > \frac{1}{k}) \leq \frac{1}{2^k}
  \]
  so
  \[
    \sum_{k \geq 1} \P(\norm{X_{n_k} - X} > \frac{1}{k}) < \infty
  \]
  so by the first Borel-Cantelli lemma
  \[
    \P(\norm{X_{n_k} - X} > \frac{1}{k} \text{ io.}) = 0.
  \]
  This means that with probability \(1\), \(\norm{X_{n_k} - X} \to 0\) as \(k \to \infty\).
\end{proof}

\begin{definition}[convergence in mean]\index{convergence!in mean}\index{convergence!in \(L^1\)}
  Let \((X_n)_{n \geq 1}\) and \(X\) be \(\R^d\)-valued integrable random variables. We say that \((X_n)_n\) \emph{converges in mean} or \emph{in \(L^1\)} to \(X\) if
  \[
    \lim_{n \to \infty} \E(\norm{X_n - X}) = 0.
  \]
\end{definition}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item If \(X_n \to X\) in mean then \(X_n \to X\) in probability by Markov inequality:
    \[
      \varepsilon \cdot \P(\norm{X_n - X} > \varepsilon) \leq \E(\norm{X_n - X}).
    \]
  \item The converse is false. For example take \(\Omega = (0, 1)\), \(\mathcal F = \mathcal B(\Sigma)\) and \(\P\) Lebesgue measure. Let \(X_n = n \mathbf 1_{[0, \frac{1}{n}]}\). \(X_n \to 0\) almost surely but \(\E X_n = 1\).
  \end{enumerate}
\end{remark}

When does convergence in probability imply convergence in mean? We need some kind of domination assumption.

\begin{definition}[uniformly integrable]\index{uniformly integrable}
  A sequence of random variables \((X_n)_{n \geq 1}\) is \emph{uniformly integrable} if
  \[
    \lim_{M \to \infty} \limsup_{n \to \infty} \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M}) = 0.
  \]
\end{definition}

\begin{remark}
  If \((X_n)_{n \geq 1}\) are \emph{dominated}, namely exists an integrable random variable \(Y \geq 0\) such that \(\norm{X_n} \leq Y\) for all \(n\) then \((X_n)_n\) is uniformly integrable by dominated convergence theorem:
  \[
    \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M}) \leq \E(Y \mathbf 1_{Y \geq M}) \to 0
  \]
  %???
  as \(M \to \infty\).
\end{remark}

\begin{theorem}
  Let \((X_n)_{n \geq 1}\) be a sequence of \(\R^d\)-valued integrable random variable. Let \(X\) be another random variable. Then TFAE:
  \begin{enumerate}
  \item \(X\) is integrable and \(X_n \to X\) in mean,
  \item \((X_n)_{n \geq 1}\) is uniformly integrable and \(X_n \to X\) in probability.
  \end{enumerate}
\end{theorem}

\begin{proof}\leavevmode
  \begin{itemize}
  \item \(1 \implies 2\): Left to show uniform integrability:
    \begin{align*}
      \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M})
      &\leq \E(\norm{X_n - X} \mathbf 1_{\norm{X_n} \geq M}) + \E(\norm{X} \mathbf 1_{\norm{X_n} \geq M}) \\
      &\leq \E(\norm{X_n - X}) + \E(\norm{X} \mathbf 1_{\norm{X_n} \geq M} (\mathbf 1_{\norm X \leq \frac{M}{2}} + \mathbf 1_{\norm X > \frac{M}{2}})) \\
      &\leq \E(\norm{X_n - X}) + \E(\norm X \mathbf 1_{\norm{X_n - X} \geq \frac{M}{2}} \mathbf 1_{\norm X \leq \frac{M}{2}}) + \E(\norm X \mathbf 1_{\norm X \geq \frac{M}{2}}) \\
      &\leq \E(\norm{X_n - X}) + \frac{M}{2} \P(\norm{X_n - X} \geq \frac{M}{2}) + \E(\norm X \mathbf 1_{\norm X \geq \frac{M}{2}})
    \end{align*}
    Take \(\limsup\),
    \[
      \limsup_{n \to \infty} \E(\norm{X_n} \mathbf 1_{\norm{X_M} \geq n})
      \leq 0 + 0 + \E(\norm X \mathbf 1_{\norm X \geq \frac{M}{2}})
      \to 0
    \]
    by dominated convergence theorem.
  \item \(2 \implies 1\): Prove first that \(X\) is integrable. By the previous proposition, we can find a subsequence \((n_k)_k\) such that \(X_{n_k} \to X\) almost surely. By Fatou's lemma,
    \[
      \E(\norm X \mathbf 1_{\norm X \geq M}) \leq \liminf_{k \to 0} \E(\norm{X_{n_k}} \mathbf 1_{\norm{X_{n_k}} \geq M})
    \]
    which goes to \(0\) as \(M \to \infty\) by uniform integrability assumption. Thus
    \[
      \E(\norm X) \leq M + \E(\norm X \mathbf 1_{\norm X \geq M}) < \infty
    \]
    for \(M\) sufficiently large. Thus \(X\) is integrable.

    To show convergence in mean, we use the same trick of spliting into small and big parts.
    \begin{align*}
      \E(\norm{X_n - X})
      &= \E((\mathbf 1_{\norm{X_n - X} \leq \varepsilon} + \mathbf 1_{\norm{X_n - X} > \varepsilon}) \norm{X_n - X}) \\
      &\leq \varepsilon + \E(\mathbf 1_{\norm{X_n - X} > \varepsilon} \norm{X_n - X} (\mathbf 1_{\norm{X_n} \leq M} + \mathbf 1_{\norm{X_n} > M})) \\
      &\leq \varepsilon + \dagger + \ddagger
    \end{align*}
    where
    \begin{align*}
      \dagger
      &= \E(\norm{X_n - X} \mathbf 1_{\norm{X_n - X} > \varepsilon} \mathbf 1_{\norm{X_n} \leq M}) \\
      &\leq \E((M + \norm X) \mathbf 1_{\norm{X_n - X} > \varepsilon} (\mathbf 1_{\norm X \leq M} + \mathbf 1_{\norm X > M}) \\
      &\leq 2M \P(\norm{X_n - X} > \varepsilon) + 2 \E(\norm X \mathbf 1_{\norm X > M})
    \end{align*}
    so
    \[
      \limsup_{n \to \infty} \dagger \leq 2 \E(\norm X \mathbf 1_{\norm X > M}) \to 0
    \]
    as \(M \to \infty\). On the other hand
    \begin{align*}
      \ddagger
      &= \E(\norm{X_n - X} \mathbf 1_{\norm{X_n - X} > \varepsilon} \mathbf 1_{\norm{X_n} > M}) \\
      &\leq \E((\norm{X_n} + \norm X) \mathbf 1_{\norm{X_n} \geq M}) \\
      &\leq \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M} + \norm X \mathbf 1_{\norm X > M} + \norm X \mathbf 1_{\norm{X_n} \geq M} \mathbf 1_{\norm X \leq M}) \\
      &\leq 2 \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M}) + \E(\norm X \mathbf 1_{\norm X > M})
    \end{align*}
    taking \(\limsup\),
    \[
      \limsup_{n \to \infty} \ddagger
      \leq 2 \limsup_{n \to \infty} \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M}) + \E(\norm X \mathbf 1_{\norm X > M}) \to 0 + 0
    \]
    as \(M \to \infty\).
  \end{itemize}
\end{proof}

\begin{definition}
  We say that a sequence of random variables \((X_n)_{n \geq 1}\) is \emph{bounded} in \(L^p\) if there exists \(C > 0\) such that \(\E(\norm{X_n}^p) \leq C\) for all \(n\).
\end{definition}

\begin{proposition}
  If \(p > 1\) and \((X_n)_{n \geq 1}\) is bounded in \(L^p\) then \((X_n)_{n \geq 1}\) is uniformly integrable.
\end{proposition}

\begin{proof}
  \[
    M^{p - 1} \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M})
    \leq \E(\norm{X_n}^p \mathbf 1_{\norm{X_n} \geq M})
    \leq \E( \norm{X_n}^p) \leq C
  \]
  so
  \[
    \limsup_{n \to \infty} \E(\norm{X_n} \mathbf 1_{\norm{X_n} \geq M}) \leq \frac{C}{M^{p - 1}} \to 0
  \]
  as \(M \to \infty\).
\end{proof}
This provides a sufficient condition for uniform integrability and thus convergnce in mean.

\section{\(L^p\) spaces}

Recall that \(\varphi: I \to \R\) is convex means that for all \(x, y \in I\), for all \(t \in [0, 1]\),
\[
  \varphi(tx + (1 - t) y) \leq t \varphi(x) + (1 - t) \varphi(y).
\]

\begin{proposition}[Jensen inequality]\index{Jensen inequality}
  \label{prop:Jensen inequality}
  Let \(I\) be an open interval of \(\R\) and \(\varphi: I \to \R\) a \emph{convex} function. Let \(X\) be a random variable \((\Omega, \mathcal F, \P)\). Assume \(X\) is integrable and takes values in \(I\). Then
  \[
    \E(\varphi(X)) \geq \varphi(\E(X)).
  \]
\end{proposition}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item As \(X \in I\) almost surely and \(I\) is an interval, have \(\E(X) \in I\).
  \item We'll show that \(\varphi(X)^-\) is integrable so
    \[
      \E(\varphi(X)) = \E(\varphi(X)^+) - \E(\varphi(X)^-)
    \]
    with the possibility that both sides are infinity.
  \end{enumerate}
\end{remark}

\begin{lemma}
  TFAE:
  \begin{enumerate}
  \item \(\varphi\) is convex,
  \item there exists a family \(\mathcal F\) of affine functions (\(x \mapsto ax + b\)) such that \(\varphi = \sup_{\ell \in \mathcal F} \ell\) on \(I\).
  \end{enumerate}
\end{lemma}

\begin{proof}\leavevmode
  \begin{enumerate}
  \item \(2 \implies 1\): every \(\ell\) is convex and the supremum of \(\ell\) is convex:
    \[
      \ell(t x + (1 - t) y) = t \ell(x) + (1 - t) \ell(y) \leq t \sup_{\ell \in \mathcal F} \ell(x) + (1 - t) \sup_{\ell \in \mathcal F} \ell(y)
    \]
    so
    \begin{align*}
      \varphi(tx + (1 - t)y)
      &= \sup_{\ell \in \mathcal F} \ell( t x + (1 - t) y) \\
      &\leq t \sup_{\ell \in \mathcal F} \ell(x) + (1 - t) \sup_{\ell \in \mathcal F} \ell(y) \\
      &= t \varphi(x) + (1 - t) \varphi(y)
    \end{align*}
  \item We need to show that for all \(x_0 \in I\) we can find an affine function
    \[
      \ell_{x_0}(x) = \theta_{x_0} (x - x_0) + \varphi(x_0),
    \]
    where \(\theta_{x_0}\) is morally the slope at \(x_0\), such that \(\varphi(x) \geq \ell_{x_0}(x)\) for all \(x \in I\). Then have \(\varphi = \sup_{x_0 \in I} \ell_{x_0}\).

    To find \(\theta_{x_0}\) observe that for all \(x < x_0 < y\) where \(x, y \in I\), have
    \[
      \frac{\varphi(x_0) - \varphi(x)}{x_0 - x} \leq \frac{\varphi(y) - \varphi(x_0)}{y - x_0}.
    \]
    Indeed this is the convexity of \(\varphi\) on \([x, y]\) with \(t = \frac{x_0 - x}{y - x}\). This holds for all \(x < x_0, y > x_0\) so there exists \(\theta \in \R\) such that
    \[
      \frac{\varphi(x_0) - \varphi(x)}{x_0 - x} \leq \theta \leq \frac{\varphi(y) - \varphi(x_0)}{y - x_0}.
    \]
    Then just set \(\ell_{x_0}(x) = \theta(x - x_0) + \varphi(x_0)\). By construction \(\varphi(x) \geq \ell_{x_0}(x)\) for all \(x \in I\).
  \end{enumerate}
\end{proof}

\begin{proof}[Proof of \nameref{prop:Jensen inequality}]
  Let \(\varphi(x) = \sup_{\ell \in \mathcal F} \ell(x)\) where \(\ell\) affine. then
    \[
      \E(\varphi(X)) \geq \E(\ell(X)) = \ell(\E(X))
    \]
    for all \(\ell \in \mathcal F\) so take supremum,
    \[
      \E(\varphi(X)) \geq \sup_{\ell \in \mathcal F} \ell(\E(X)) = \varphi(\E(X)).
    \]

    Also for the remark,
    \[
      - \varphi = - \sup_{\ell \in \mathcal F} \ell = \inf_{\ell \in \mathcal F} (-\ell)
    \]
    so \(\varphi^- = (-\varphi)^+ \leq |\ell|\) for all \(\ell \in \mathcal F\). Then
    \[
      \varphi(X)^- \leq |\ell(X)| \leq |a| |X| + |b|.
    \]
    As \(X\) is integrable, \(\varphi(X)^-\) is integrable.
\end{proof}

Jensen inequality is for probability space only. The following applies to all measure spaces.

\begin{proposition}[Minkowski inquality]\index{Minkowski inequality}
  Let \((X, \mathcal A, \mu)\) be a measure space and \(f, g\) measurable functions on \(X\). Let \(p \in [1, \infty)\) and define the \(p\)-norm
  \[
    \norm f_p = \left( \int_X |f|^p d \mu \right)^{1/p}.
  \]
  Then
  \[
    \norm{f + g}_p \leq \norm f_p + \norm g_p.
  \]
\end{proposition}

\begin{proof}
  wlog assume \(\norm f_p, \norm f_p \neq 0\). Need to show
  \[
    \norm*{\frac{\norm f_p}{\norm f_p + \norm g_p} \frac{f}{\norm f}_p + \frac{\norm g_p}{\norm f_p + \norm g_p} \frac{g}{\norm g_p}}_p \leq 1.
  \]
  Suffice to show for all \(t \in [0, 1]\), for all \(F, G\) measurable such that \(\norm F_p = \norm G_p = 1\), have
  \[
    \norm{tF + (1 - t) G}_p \leq 1
  \]
  ``the unit ball is convex''. For this note that
  \begin{align*}
    [0, +\infty) &\to [0, +\infty) \\
    x &\mapsto x^p
  \end{align*}
  is convex if \(p \geq 1\) so
  \[
    |t F + (1 - t) G|^p \leq t |F|^p + (1 - t) |G|^p
  \]
  and
  \[
    \int_X |tF + (1 - t)G|^p d\mu \leq t \underbrace{\int_X |F|^p d\mu}_{= 1} + (1 - t) \underbrace{\int_X |G|^p d\mu}_{= 1} = 1.
  \]
\end{proof}

\begin{proposition}[Hölder inequality]\index{Hölder inequality}
  \label{prop:Hölder inequality}
  Suppose \((X, \mathcal A, \mu)\) is a measure space and let \(f, g\) be measurable functions on \(X\). Given \(p, q \in (1, \infty)\) such that \(\frac{1}{p} + \frac{1}{q} = 1\),
  \[
    \int_X |fg| d\mu \leq \left( \int_X |f|^p d\mu \right)^{1/p} \left( \int_X |g|^q d\mu \right)^{1/q}
  \]
  with equality if and only if there exists \((\alpha, \beta) \neq (0, 0)\) such that \(\alpha |f|^p = \beta |g|^q\) \(\mu\)-almost everywhere.
\end{proposition}

\begin{lemma}[Young inequality]\index{Young inequality}
  For all \(p, q \in (1, \infty)\) such that \(\frac{1}{p} + \frac{1}{q} = 1\), for all \(a, b \geq 0\), have
  \[
    ab \leq \frac{a^p}{p} + \frac{b^q}{q}.
  \]
\end{lemma}

\begin{proof}[Proof of \nameref{prop:Hölder inequality}]
  wlog assume \(\norm f_p, \norm g_q \neq 0\). By scaling by factors \((\alpha, \beta) \neq (0, 0)\) wlog \(\norm f_p = \norm g_q = 1\). Then by Young inequality,
  \[
    |fg| \leq \frac{1}{p} |f|^p + \frac{1}{q} |g|^q
  \]
  so
  \[
    \int_X |fg| d\mu \leq \frac{1}{p} \int_X |f|^p d\mu + \frac{1}{q} \int_X |g|^q d\mu = \frac{1}{p} + \frac{1}{q} = 1.
  \]
\end{proof}

\begin{remark}
  Apply Jensen inequality to \(\varphi(x) = x^{p'/p}\) for \(p' > p\), we have
  \[
    \E(|X|^p)^{1/p} \leq \E(|X|^{p'})^{1/p'},
  \]
  so the function \(p \mapsto \E(|X|^p)^{1/p}\) is non-decreasing. This can be used, for example, to show that if \(X\) has finite \(p'\)th moment then it has finite \(p\)th moment for \(p' \geq p\).
\end{remark}

\begin{definition}
  Let \((X, \mathcal A, \mu)\) be a measure space.
  \begin{itemize}
  \item For \(p \geq 1\),
    \[
      \mathcal L^p(X, \mathcal A, \mu) = \{f: X \to \R \text{ measurable such that \(|f|^p\) is \(\mu\)-integrable}\}.
    \]
  \item For \(p = \infty\),
    \[
      \mathcal L^\infty(X, \mathcal A, \mu) = \{f: X \to \R \text{ measurable such that \(\essup |f| < \infty\)}\}
    \]
    where
    \[
      \essup |f| = \inf \{t: |f(x)| \leq t \text{ for \(\mu\)-almost every } x\}.
    \]
  \end{itemize}
\end{definition}

\begin{lemma}
  \(\mathcal L^p(X, \mathcal A, \mu)\) is an \(\R\)-vector space.
\end{lemma}

\begin{proof}
  For \(p < \infty\) use Minkowski inequality. Similar we can check that \(\norm{f + g}_\infty \leq \norm f_\infty + \norm g_\infty\) for all \(f, g\).
\end{proof}

\begin{definition}
  We say \(f\) and \(g\) are \emph{\(\mu\)-equivalent} and write \(f \equiv_\mu g\) if for \(\mu\)-almost every \(x\), \(f(x) = g(x)\).
\end{definition}

Check that this is an equvalence relation stable under addition and multiplication.

\begin{definition}[\(L^p\)-space]\index{\(L^p\)-space}
  Define
  \[
    L^p(X, \mathcal A, \mu) = \mathcal L^p(X, \mathcal A, \mu) / \equiv_\mu
  \]
  and if \([f]\) denotes the equivalence class of \(f\) under \(\equiv_\mu\) we define
  \[
    \norm{[f]}_p = \norm f_p.
  \]
\end{definition}

\begin{proposition}
  For \(p \in [1, \infty]\), \(L^p(X, \mathcal A, \mu)\) is a normed vector space under \(\norm \cdot_p\) and it is complete, i.e.\ it is a Banach space.
\end{proposition}

\begin{proof}
  If \(f \equiv_\mu g\) then \(\norm f_p = \norm g_p\) so \(\norm \cdot_p\) on \(L^p\) is well-defined. Triangle inequality follows from Minkowski inequality and linearity is obvious so \(\norm \cdot_p\) is indeed a norm.

  For completeness, pick \((f_n)_n\) a Cauchy sequence in \(\mathcal L^p(X, \mathcal A, \mu)\). Need to show that there exists \(f \in \mathcal L^p\) such that \(\norm{f_n - f}_p \to 0\) as \(n \to \infty\). This then implies that \([f_n] \to [f]\) in \(L^p\).

  We can extract a subsequence \(n_k \uparrow \infty\) such that \(\norm{f_{n_{k + 1}} - f_{n_k}}_p \leq 2^{-k}\). Let
  \[
    S_K = \sum_{k = 1}^K |f_{n_{k + 1}} - f_{n_k}|
  \]
  then
  \[
    \norm{S_K}_p
    \leq \sum_{k = 1}^K \norm{f_{n_{k + 1}} - f_{n_k}}_p
    \leq \sum_{k = 1}^K 2^{-k}
    \leq 1
  \]
  so by monotone convergence,
  \[
    \lim_{K \to \infty} \int_X |S_K|^p d\mu = \int_X |S_\infty|^p d\mu,
  \]
  i.e.\ \(S_\infty \in \mathcal L^p\). In particular for \(\mu\)-almost everywhere \(x\), \(|S_\infty(x)| < \infty\), i.e.
  \[
    \sum_{k \geq 1} |f_{n_{k + 1}}(x) - f_{n_k}(x)| < \infty.
  \]
  Hence \((f_{n_k}(x))_k\) is a Cauchy sequence in \(\R\). By completeness of \(\R\), the limit exists and set \(f(x)\) to be it. When this limit does not exist set \(f(x) = 0\).

  We then have, in case \(p < \infty\), by Fatou's lemma
  \[
    \norm{f_n - f}_p \leq \liminf_{k \to \infty} \norm{f_n - f_{n_k}}_p \leq \varepsilon
  \]
  for any \(\varepsilon\) for \(n\) sufficiently large. Thus
  \[
    \lim_{n \to \infty} \norm{f_n - f}_p = 0.
  \]
  When \(p = \infty\), we use the fact that if \(f_n \to f\) \(\mu\)-almost everywhere then
  \[
    \norm f_\infty \leq \liminf_{n \to \infty} \norm{f_n}_\infty.
  \]

  \begin{proof}
    Let \(t > \limsup_{n \to \infty} \norm{f_n}_\infty\). Then exists \(n_k \uparrow \infty\) such that
    \[
      \norm{f_{n_k}}_\infty = \essup |f_{n_k}| = \inf \{s \geq 0: |f_{n_k}(x)| \leq s \text{ for \(\mu\)-almost every } x\} < t
    \]
    for all \(k\). Thus for all \(k\), for \(\mu\)-almost every \(x\), \(|f_{n_k}(x)| < t\). But by \(\sigma\)-additivity of \(\mu\) we can \emph{swap the quantifiers}, i.e.\ for \(\mu\)-almost every \(x\), for all \(x\), \(|f_{n_k}(x)| < t\). Thus for \(\mu\)-almost every \(x\), \(\norm{f(x)}_\infty \leq t\).
  \end{proof}
\end{proof}

\begin{proposition}[approximation by simple functions]
  Let \(p \in [1, \infty)\). Let \(V\) be the linear span of all simple functions on \(X\). Then \(V \cap \mathcal L^p\) is dense in \(\mathcal L^p\).
\end{proposition}

\begin{proof}
  Note that \(g \in \mathcal L^p\) implies \(g^+, g^- \in \mathcal L^p\). Thus by writing \(f = f^+ - f^-\) and using Minkowski inequality, suffice to show \(f \geq 0\) is the limit of a sequence of simple functions.

  Recall there exist simple functions \(0 \leq g_n \leq f\) such that \(g_n(x) \uparrow f(x)\) for \(\mu\)-almost every \(x\). Then
  \[
    \lim_{n \to \infty} \norm{g_n - f}_p^p = \lim_{n \to \infty} \int_X |g_n - f|^p d\mu = 0
  \]
  by dominated convergence theorem (\(|g_n - f| \leq g_n + f \leq 2f\) so \(g_n - f\) is \(\mu\)-integrable).
\end{proof}

\begin{remark}
  When \(X = \R^d, \mathcal A = \mathcal B(\R^d)\) and \(\mu\) is the Lebesgue measure, \(C_c(\R^d)\), the space of continuous functions with compact support is dense in \(\mathcal L^p(X, \mathcal A, \mu)\) when \(p \in [1, \infty)\) (this does not hold for \(p = \infty\): a constant nonzero function has no noncompact support). See example sheet. In fact, \(C_c^\infty(\R^d)\) suffices.
\end{remark}

\section{Hilbert space and \(L^2\)-methods}

\begin{definition}[inner product]\index{inner product}
  Let \(V\) be a complex vector space. A \emph{Hermitian inner product} on \(V\) is a map
  \begin{align*}
    V \times V &\to \C \\
    (x, y) &\mapsto \ip{x, y}
  \end{align*}
  such that
  \begin{enumerate}
  \item \(\ip{\alpha x + \beta y, z} = \alpha \ip{x, z} + \beta \ip{y, z}\) for all \(\alpha, \beta \in \C\), for all \(x, y, z \in V\).
  \item \(\ip{y, x} = \conj{\ip{x, y}}\).
  \item \(\ip{x, x} \in \R\) and \(\ip{x, x} \geq 0\), with equality if and only if \(x = 0\).
  \end{enumerate}
\end{definition}

\begin{definition}[Hermitian norm]\index{Hermitian norm}
  The \emph{Hermitian norm} is defined as \(\norm x = \sqrt{\ip{x, x}}\).
\end{definition}

\begin{lemma}
  Properties of norm:
  \begin{enumerate}
  \item linearity: \(\norm{\lambda x} = |\lambda| \norm x\) for all \(\lambda \in \C, x \in V\),
  \item Cauchy-Schwarz: \(|\ip{x, y}| \leq \norm x \cdot \norm \psi\),
  \item triangle inequality: \(\norm{x + y} \leq \norm x + \norm y\)
  \item parallelogram identity: \(\norm{x + y}^2 + \norm{x - y}^2 = 2(\norm x^2 + \norm y^2)\)
  \end{enumerate}
\end{lemma}

\begin{proof}
  Exercise. For reference see author's notes on IID Linear Analysis.
\end{proof}

\begin{corollary}
  \((V, \norm \cdot)\) is a normed vector space.
\end{corollary}

\begin{definition}[Hilbert space]\index{Hilbert space}
  We say \((V, \norm \cdot)\) is a \emph{Hilbert space} if it is complete.
\end{definition}

\begin{eg}
  Let \(V = L^2(X, \mathcal A, \mu)\) where \((X, \mathcal A, \mu)\) is a measure space. Then we can define
  \[
    \ip{f, g} = \int_X f \conj g d\mu
  \]
  which is well-defined (i.e.\ finite) by Cauchy-Schwarz. The axioms are easy to check, with positive-definiteness given by
  \[
    0 = \ip{f, f} = \int_X |f|^2 d\mu
  \]
  if and only if \(f = 0\) \(\mu\)-almost everywhere so \(f = 0\).
\end{eg}

\begin{proposition}\index{orthogonal projection}
  Let \(H\) be a Hilbert space and let \(\mathcal C\) be a closed convex subset of \(H\). Then for all \(x \in H\), there exists unique \(y \in \mathcal C\) such that
  \[
    \norm{x - y} = d(x, \mathcal C)
  \]
  where by definition
  \[
    d(x, \mathcal C) = \inf_{c \in \mathcal C} \norm{x - c}.
  \]
  This \(y\) is called the \emph{orthogonal projection} of \(x\) on \(\mathcal C\).
\end{proposition}

\begin{proof}
  Let \(c_n \in \mathcal C\) be a sequence such that \(\norm{x - c_n} \to d(x, \mathcal C)\). Let's show that \((c_n)_n\) is a Cauchy sequence. By parallelogram identity,
  \[
    \norm*{\frac{x - c_n}{2} + \frac{x - c_m}{2}}^2 + \norm*{\frac{x - c_n}{2} - \frac{x - c_m}{2}}^2
    = \frac{1}{2} (\norm{x - c_n}^2 + \norm{x - c_m}^2)
  \]
  so
  \[
    \underbrace{\norm*{x - \underbrace{\frac{c_n + c_m}{2}}_{\in \mathcal C}}^2}_{\geq d(x, \mathcal C)^2} + \frac{1}{4} \norm{c_n - c_m}^2
    = \underbrace{\frac{1}{2} (\norm{x - c_n}^2 + \norm{x - c_m}^2)}_{\to d(x, \mathcal C)^2}
  \]
  so
  \[
    \lim_{n, m \to \infty} \norm{c_n - c_m} = 0
  \]
  i.e.\ \((c_n)_n\) is Cauchy. By completeness exist \(\lim_{n \to \infty} c_n = y \in H\). As \(\mathcal C\) is closed, \(y \in \mathcal C\). As \(\norm{x - c_n} \to d(x, \mathcal C)\), \(\norm{x - y} \to d(x, \mathcal C)\). This shows existence of \(y\).

  For uniqueness, use parallelogram identity
  \[
    \underbrace{\norm*{x - \frac{y + y'}{2}}^2}_{\geq d(x, \mathcal C)} + \frac{1}{4} \norm{y - y'}^2 = \frac{1}{2} (\norm{x - y}^2 + \norm{x - y'}^2) = d(x, \mathcal C)^2
  \]
  so \(\norm{y - y'} = 0\).
\end{proof}

\begin{corollary}
  Suppose \(V \leq H\) is a closed subspace of a Hilbert space \(H\). Then
  \[
    H = V \oplus V^\perp
  \]
  where
  \[
    V^\perp = \{x \in H: \ip{x, v} = 0 \text{ for all } v \in V\}
  \]
  is the \emph{orthogonal} of \(V\).
\end{corollary}

\begin{proof}
  \(V \cap V^\perp = 0\) by positivity of inner product. If \(x \in H\) then there exists a unique \(y \in V\) such that \(\norm{x - y} = d(x, y)\). Need to show that \(x - y \in V^\perp\).

  For all \(z \in V\),
  \[
    \norm{x - y - z} \geq \norm{x - y}
  \]
  as \(y + z \in V\). Thus
  \[
    \norm{x - y}^2 + \norm z - 2 \Re \ip{x - y, z} \geq \norm{x - y}^2.
  \]
  Rearrange,
  \[
    2\Re \ip{x - y, z} \leq \norm z^2
  \]
  for all \(z \in V\). Now substitute \(tz\) for \(z\) where \(t \in \R_+\), have
  \[
    t \cdot 2 \Re{x - y, z} \leq t^2 \norm z^2.
  \]
  For \(t = 0\),
  \[
    \Re \ip{x - y, z} \leq 0
  \]
  Similarly replace \(z\) by \(-z\) to conclude \(\Re \ip{x - y, z} = 0\). Finally replace \(z\) by \(e^{i\theta}z\) to have \(\ip{x - y, z} = 0\) for all \(z\). Thus \(x - y \in V^\perp\).
\end{proof}

\begin{definition}[bounded linear form]
  A linear form \(\ell: H \to \C\) is \emph{bounded} if there exists \(C > 0\) such that \(|\ell(x)| \leq C\norm x\) for all \(x \in H\).
\end{definition}

\begin{remark}
  \(\ell\) bounded is equivalent to \(\ell\) continuous.
\end{remark}

\begin{theorem}[Riesz representation theorem]\index{Riesz representation theorem}
  Let \(H\) be a Hilbert space. For every bounded linear form \(\ell\) there exists \(v_0 \in H\) such that
  \[
    \ell(x) = \ip{x, v_0}
  \]
  for all \(x \in H\).
\end{theorem}

\begin{proof}
  By boundedness of \(\ell\), \(\ker \ell\) is closed so write
  \[
    H = \ker \ell \oplus (\ker \ell)^\perp.
  \]
  If \(\ell = 0\) then just pick \(v_0 = 0\). Otherwise pick \(x_0 \in (\ker \ell)^\perp \setminus \{0\}\). But \((\ker \ell)^\perp\) is spanned by \(x_0\): indeed for any \(x \in (\ker \ell)^\perp\),
  \[
    \ell(x) = \frac{\ell(x)}{\ell(x_0)} \ell(x_0)
  \]
  so
  \[
    \ell(x - \frac{\ell(x)}{\ell(x_0)} x_0) = 0
  \]
  so \(x - \frac{\ell(x)}{\ell(x_0)} \in \ker \ell \cap (\ker \ell)^\perp = 0\). Now let
  \[
    v_0 = \frac{\conj{\ell(x_0)}}{\norm{x_0}^2} x_0
  \]
  and observe that \(\ell(x) - \ip{x, v_0}\) vanishes on \(\ker \ell\) and on \((\ker \ell)^\perp = \C x_0\). Thus it is identically zero.
\end{proof}

\begin{definition}[absolutely continuous, singular measure]\index{measure!absolutely continuous}\index{measure!singular}
  Let \((X, \mathcal A)\) be a measurable space and let \(\mu, \nu\) be two measures on \((X, \mathcal A)\).
  \begin{enumerate}
  \item \(\mu\) is \emph{absolutely continuous} with respect to \(\nu\), denoted \(\mu \ll \nu\), if for every \(A \in \mathcal A\), \(\nu(A) = 0\) implies \(\mu(A) = 0\).
  \item \(\mu\) is \emph{singular}, denoted \(\mu \perp \nu\), if exists \(\Omega \in \mathcal A\) such that \(\mu(\Omega) = 0\) and \(\nu(\Omega^c) = 0\).
  \end{enumerate}
\end{definition}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Let \(\nu\) be the Lebesgue measure on \((\R, \mathcal B(\R))\) and \(d\mu = fd\nu\) where \(f \geq 0\) is a Borel function then \(\mu \ll \nu\).
  \item If \(\mu = \delta_{x_0}\), the Dirac mass at \(x_0 \in \R\), then \(\mu \perp v\).
  \end{enumerate}
\end{eg}

Non-examinable theorem and proof:

\begin{theorem}[Radon-Nikodym]\index{Radon-Nikodym theorem}
  Assume \(\mu\) and \(\nu\) are \(\sigma\)-finite measures on \((X, \mathcal A)\).
  \begin{enumerate}
  \item If \(\mu \ll \nu\) then there exists \(g \geq 0\) measurable such that \(d \mu = g d \nu\), namely
    \[
      \mu(A) = \int_A g(x) d \nu(x)
    \]
    for all \(A \in \mathcal A\). \(g\) is called the \emph{density}\index{density} of \(\mu\) with respect to \(\nu\) or \emph{Radon-Nikodym derivative}\index{Radon-Nikodym derivative}, sometimes denoted by \(g = \frac{d\mu}{d\nu}\).
  \item For any \(\mu, \nu\) \(\sigma\)-finite, \(\mu\) decomposes as
    \[
      \mu = \mu_a + \mu_s
    \]
    where \(\mu_a \ll \nu\) and \(\mu_s \perp \nu\). Moreover this decomposition is unique.
  \end{enumerate}
\end{theorem}

\begin{proof}
  Consider \(H = L^2(X, \mathcal A, \mu + \nu)\), which is a Hilbert space. First assume \(\mu\) and \(\nu\) are finite. Consider the linear form
  \begin{align*}
    \ell: H &\to \R \\
    f &\mapsto \mu(f) = \int_X f d\mu
  \end{align*}
  \(\ell\) is bounded by Cauchy-Schwarz and finiteness of the measures:
  \[
    |\mu(f)|
    \leq \mu(|f|)
    \leq (\mu + \nu)(|f|)
    \leq \sqrt{(\mu + \nu)(X)} \cdot \sqrt{\int_X |f|^2 d(\mu + \nu)}
    = C \cdot \norm f_H
  \]
  so by Riesz representation theorem, there exists \(g_0 \in L^2(X, \mathcal A, \mu + \nu)\) such that
  \[
    \label{eqn:g0}
    \mu(f) = \int_X fg_0 d(\mu + \nu).
    \tag{\ast}
  \]
  Claim that for \((\mu + \nu)\)-almost every \(x\), \(0 \leq g_0(x) \leq 1\): take \(f = \mathbf 1_{\{g_0 < 0\}}\) and plug it into \eqref{eqn:g0},
  \[
    0
    \leq \mu(\{g_0 < 0\})
    = \int_X \underbrace{\mathbf 1_{\{g_0 < 0\}} g_0}_{\leq 0} d(\mu + \nu)
    \leq 0
  \]
  so equality throughout. Thus \(g_0 \geq 0\) \((\mu + \nu)\)-almost everywhere. Similarly take \(f = \mathbf 1_{\{g_0 > 1 + \varepsilon\}}\) for \(\varepsilon > 0\) and plug it into \eqref{eqn:g0},
  \begin{align*}
    (\mu + \nu) (\{g_0 > 1 + \varepsilon\})
    &\geq \mu(\{g_0 > 1 + \varepsilon\}) \\
    &= \int_X \mathbf 1_{\{g_0 > 1 + \varepsilon\}} g_0 d(\mu + \nu) \\
    &\geq (1 + \varepsilon) (\mu + \nu)(\{g_0 > 1 + \varepsilon\})
  \end{align*}
  so must have
  \[
    (\mu + \nu) (\{g_0 > 1 + \varepsilon\}) = 0
  \]
  i.e.\ \(g_0 \leq 1\) \((\mu + \nu)\)-almost everywhere.

  Now set \(\Omega = \{x \in X: g_0 \in [0, 1)\}\) so on \(\Omega^c\), \(g_0 = 1\) \((\mu + \nu)\)-almost everywhere. Then \eqref{eqn:g0} is equivalent to
  \[
    \int_X f(1 - g_0) d\mu = \int_X fg_0 d\nu
  \]
  for all \(f \in L^2(X, \mathcal A, \mu + \nu)\). Hence this holds for all \(f \geq 0\). Now \(f\) be \(\frac{f}{1 - g_0} \mathbf 1_\Omega\), get
  \[
    \label{eqn:Radon-Nikodym}
    \int_\Omega f d\mu = \int_\Omega f \frac{g_0}{1 - g_0} d\nu.
    \tag{\ast\ast}
  \]
  Set
  \begin{align*}
    \mu_a(A) &= \mu(A \cap \Omega) \\
    \mu_s(A) &= \mu(A \cap \Omega^c)
  \end{align*}
  Clearly \(\mu = \mu_a + \mu_s\). Claim that this is the required result, i.e.
  \begin{enumerate}
  \item \(\mu_a \ll \nu\),
  \item \(\mu_s \perp \nu\),
  \item \(d\mu_a = gd\nu\) where \(g = \frac{g_0}{1 - g_0} \mathbf 1_\Omega\).
  \end{enumerate}

  \begin{proof}\leavevmode
    \begin{enumerate}
    \item If \(\nu(A) = 0\) set \(f = \mathbf 1_A\) and plug into \eqref{eqn:Radon-Nikodym} to get \(\mu(A \cap \Omega) = 0\), namely \(\mu_a(A) = 0\).
    \item Set \(f = \mathbf 1_{\Omega^c}\). On \(\Omega^c\), \(g_0 = 1\) \((\mu + \nu)\)-almost everywhere. Plug this into \eqref{eqn:g0} to get \(\nu(\Omega^c) = 0\). But \(\mu_s(\Omega) = 0\) so \(\mu_s \perp \nu\).
    \item \eqref{eqn:Radon-Nikodym} is equivalent to \(d\mu_a = gd\nu\) where \(g = \frac{g_0}{1 - g_0} \mathbf 1_\Omega\).
    \end{enumerate}
  \end{proof}

  This settles part 2 of the theorem, and also part 1 as if \(\mu \ll \nu\) then \(\mu = \mu_a\).

  If \(\mu\) are \(\nu\) are not finite but only \(\sigma\)-finite, use the old trick of partition \(X\) into countably many \(\mu\)- and \(\nu\)-finite sets and take their intersections. Suppose we get a disjoint countable union \(X = \bigcup_n X_n\). Then \(\mu = \sum_n \mu|_{X_n}\) where for each \(n\) we can write
  \[
    \mu|_{X_n} = (\mu|_{X_n})_a + (\mu|_{X_n})_s.
  \]
  Then set
  \begin{align*}
    \mu_a &= \sum_n (\mu|_{X_n})_a \\
    \mu_s &= \sum_n (\mu|_{X_n})_s \\
  \end{align*}

  Remains to check uniqueness of decomposition. Suppose \(\mu\) can be decomposed in two ways
  \[
    \mu = \mu_a + \mu_s = \mu_a' + \mu_s'.
  \]
  As \(\mu_s, \mu_s' \perp \nu\) there exists \(\Omega_0, \Omega_0' \in \mathcal A\) such that
  \begin{align*}
    \mu_s(\Omega_0) &= 0, \nu(\Omega_0^c) = 0 \\
    \mu_s'(\Omega_0') &= 0, \nu((\Omega_0')^c) = 0
  \end{align*}
  Set \(\Omega_1 = \Omega_0 \cap \Omega_0'\). Check that
  \begin{align*}
    \mu_s(\Omega_1) &= \mu_s'(\Omega_1) = 0 \\
    \nu(\Omega_1^c) &= \nu(\Omega_0^c \cup \Omega_0'^c) = 0
  \end{align*}
  Now \(\mu_a, \mu_a' \ll \nu\) so
  \[
    \mu_a(\Omega_1^c) = \mu_a'(\Omega_1^c) = 0.
  \]
  Hence for all \(A \in \mathcal A\),
  \begin{align*}
    \mu_a(A)
    = \mu_a(A \cap \Omega_1)
    = \mu(A \cap \Omega_1)
    = \mu_a'(A \cap \Omega_1)
    = \mu_a'(A)
  \end{align*}
  so \(\mu_a = \mu_a'\) and hence \(\mu_s = \mu_s'\).
\end{proof}

\begin{proposition}
  Let \((\Omega, \mathcal F, \P)\) be a probability space. Let \(\mathcal G\) be a \(\sigma\)-subalgbebra of \(\mathcal F\) and \(X\) a random variable on \((\Omega, \mathcal F, \P)\). Assume \(X\) is integrable, then there exists a random variable \(Y\) on \((\Omega, \mathcal G, \P)\) such that
  \[
    \E(\mathbf 1_A X) = \E(\mathbf 1_AY)
  \]
  for all \(A \in \mathcal G\). Moreover \(Y\) is unique almost surely.
\end{proposition}

If you are perplexed by why it is nontrivial to recover a random variable on \(\mathcal G\) from one on \(\mathcal F\), as \(\mathcal G \subseteq \mathcal F\) so it seems that we can easily restrict to a ``sub-random variable''. But this reasoning makes absolutely no sense as random variables are functions \emph{from} the space. In other words, \(\id: (\Omega, \mathcal F, \P) \to (\Omega, \mathcal G, \P)\) is measurable but its inverse is not, and it is not obvious that \(X\) has a pushforward.

\begin{definition}[conditional expectation]\index{conditional expectation}
  \(Y\) as above is called the \emph{conditional expectation} of \(X\) with respect to \(\mathcal G\), denote by
  \[
    Y = \E(X | \mathcal G).
  \]
\end{definition}

\begin{proof}
  wlog assume \(X \geq 0\). Set \(\mu(A) = \E(\mathbf 1_A X)\) for all \(A \in \mathcal G\). \(\mu\) is finite by integrability of \(X\) and is a measure on \((\Omega, \mathcal G)\). Moreover \(\mu \ll \P\). Thus by Radon-Nikodym there exists \(g \geq 0\) \(\mathcal G\)-measurable such that
  \[
    \mu(A) = \int_A g d\P = \E(\mathbf 1_A g).
  \]
  Set \(Y = g\).

  Uniqueness is shown in example sheet 3.
\end{proof}

\begin{remark}
  In case \(X \in L^2(\Omega, \mathcal F, \P)\) then \(Y\) is the orthogonal projection of \(X\) onto \(L^2(\Omega, \mathcal G, \P)\). It is well-defined since \(L^2(\Omega, \mathcal F, \P)\) is a Hilbert space and \(L^2(\Omega, \mathcal G, \P)\) is a closed subspace. In this case TFAE:
  \begin{enumerate}
  \item \(\E((X - Y) \mathbf 1_A) = 0\) for all \(A \in \mathcal G\),
  \item \(\E((X - Y) h) = 0\) for all \(h\) simple on \((\Omega, \mathcal G, \P)\),
  \item \(\E((X - Y) h) = 0\) for all \(h \geq 0\) \(\mathcal G\)-measurable,
  \item \(\E((X - Y) h) = 0\) for all \(h \in L^2(\Omega, \mathcal G, \P)\).
  \end{enumerate}
\end{remark}

\begin{remark}
  Special case when \(\mathcal G = \{\emptyset, B, B^c, \Omega\}\) where \(B \in \mathcal F\):
  \[
    \E(\mathbf 1_A | \mathcal G)(\omega) =
    \begin{cases}
      \P(A|B) & \omega \in B \\
      \P(A|B^c) & \omega \in B^c
    \end{cases}
  \]
  where
  \[
    \P(A|B) = \frac{\P(A \cap B)}{\P(B)}.
  \]
\end{remark}

\begin{proposition}[non-examinable]
  Properties of conditional expectation:
  \begin{enumerate}
  \item linearity: \(\E(\alpha X + \beta Y | \mathcal G) = \alpha \E(X | \mathcal G) + \beta \E(Y | \mathcal G)\).
  \item if \(X\) is \(\mathcal G\)-measurable then \(\E(X | \mathcal G) = \E(X)\).
  \item positivity: if \(X \geq 0\) then \(\E(X | \mathcal G) \geq 0\).
  \item \(\E(\E(X | \mathcal G) | \mathcal H) = \E(X | \mathcal H)\) if \(\mathcal H \subseteq \mathcal G\).
  \item if \(Z\) is \(\mathcal G\)-measurable and bounded then \(\E(XZ| \mathcal G) = Z \cdot \E(X| \mathcal G)\).
  \end{enumerate}
\end{proposition}

\section{Fourier transform}

\begin{definition}[Fourier transform]\index{Fourier transform}
  Let \(f \in L^1(\R^d, \mathcal B(\R^d), dx)\) where \(dx\) is the Lebesgue measure. The function
  \begin{align*}
    \hat f: \R^d &\to \C \\
    u &\mapsto \int_{\R^d} f(x) e^{i \ip{u, x}} dx
  \end{align*}
  where \(\ip{u, x} = u_1x_1 + \dots + u_dx_d\), is called the \emph{Fourier transform} of \(f\).
\end{definition}

\begin{proposition}\leavevmode
  \begin{enumerate}
  \item \(|\hat f(u)| \leq \norm f_1\).
  \item \(\hat f\) is continuous.
  \end{enumerate}
\end{proposition}

\begin{proof}
  1 is clear. 2 follows from dominated convergence theorem.
\end{proof}

\begin{definition}
  Given a finite Borel measure \(\mu\) on \(\R^d\), its \emph{Fourier transform} is given by
  \[
    \hat \mu(u) = \int_{\R^d} e^{i \ip{u, x}} d \mu(x).
  \]
\end{definition}

Again \(|\hat \mu(u)| \leq \mu (\R^d)\) and \(\hat \mu\) is continuous.

\begin{remark}
  If \(X\) is an \(\R^d\)-valued random variable with law \(\mu_X\) then \(\hat \mu_X\) is called the \emph{characteristic function}\index{characteristic function} of \(X\).
\end{remark}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Normalised Gaussian distribution on \(\R\): \(\mu = \mathcal N(0, 1)\). \(d\mu = g dx\) where
    \[
      g(x) = \frac{e^{-x^2/2}}{\sqrt{2\pi}}.
    \]
    Claim that
    \[
      \hat \mu (u) = \hat g(u) = e^{-u^2/2},
    \]
    i.e.\ \(\hat g = \sqrt{2\pi} g\). This is the defining characteristic of Gaussian distribution.

    \begin{proof}
      Since \((u \mapsto |iu \cdot e^{iux} e^{-x^2/2}|) \in L^1(\R)\), we can differentiate under integral sign to get
      \begin{align*}
        \frac{d}{du} \hat g(u)
        &= \frac{d}{du} \int_\R e^{iux} e^{-x^2/2} \frac{dx}{\sqrt{2\pi}} \\
        &= \int_\R ix e^{iux} e^{-x^2/2} \frac{dx}{\sqrt{2\pi}} \\
        &= -\int_\R i e^{iux} g'(x) dx \\
        &= i \int_\R (e^{iux})' g(x) dx \\
        &= -u \int_\R e^{iux} g(x) dx \\
        &= -u \hat g(u)
      \end{align*}
      Thus
      \[
        \frac{d}{du} (\hat g(u) e^{u^2/2}) = 0
      \]
      so
      \[
        \hat g(u) = \hat g(0) e^{-u^2/2}.
      \]
      But
      \[
        \hat g(0) = \int_\R g(x) dx = 1
      \]
      so \(\hat g(u) = e^{-u^2/2}\) as required.
    \end{proof}
  \item \(d\)-dimensional version: \(\mu = \mathcal N(0, I_d)\). \(d\mu(x) = G(x) dx\) where \(dx = dx_1\cdots dx_d\) and
    \[
      G(x) = \prod_{i = 1}^d g(x_i)= \frac{e^{-\norm x^2/2}}{(\sqrt{2\pi})^d}.
    \]
    Then
    \begin{align*}
      \hat G(u)
      &= \int_{\R^d} e^{i\ip{u, x}} G(x) dx \\
      &= \prod_{i = 1}^d \int_\R g(x_i) e^{iu_i x_i} dx_i \\
      &= \prod_{i = 1}^d \hat g(u_i) \\
      &= e^{-\norm u^2/2}
    \end{align*}
  \end{enumerate}
\end{eg}

\begin{theorem}[Fourier inversion formula]\leavevmode
  \begin{enumerate}
  \item If \(f \in L^1(\R^d)\) is such that \(\hat f \in L^1(\R^d)\) then \(f\) is continuous (i.e.\ \(f\) equals to a continuous function almost everywhere) and
    \[
      f(x) = \frac{1}{(2\pi)^d} \hat{\hat f} (-x).
    \]
  \item If \(\mu\) is a finite Borel measure on \(\R^d\) such that \(\hat \mu \in L^1(\R^d)\) then \(\mu\) has a continuous density with respect to Lebesgue measure, i.e.\ \(d\mu = f dx\) with
    \[
      f(x) = \frac{1}{(2\pi)^d} \hat{\hat \mu} (-x).
    \]
  \end{enumerate}
\end{theorem}

\begin{remark}
  In other words
  \[
    f(x) = \frac{1}{(2\pi)^d} \int_{\R^d} \hat f(u) e^{-i \ip{u, x}} du
  \]
  where \(\hat f(u)\) are \emph{Fourier coefficients} and \(e^{-i\ip{u, x}}\) are called \emph{Fourier modes}, which are \emph{characters} \(e^{i\ip{u, -}}: \R^d \to \{z \in \C: |z| = 1\}\). Informally this says that every \(f\) can be written as an ``infinite linear combination'' of Fourier modes.
\end{remark}

\begin{proof}
  (!UPDATE: 1 does not quite reduce to 2 as \(\hat f = \hat f^+ - \hat f^-\) does not quite hold. Instead write \(\hat f = a \hat \mu_X - b \hat \mu_Y\) where \(a = \norm{f^+}_1, d\mu_x = \frac{f}{\norm{f^+}_1}dx\))

  1 reduces to 2 by considering \(f = f^+ - f^-\). In 2 we may assume wlog \(\mu\) is a probability measure so is the law of some random variable \(X\). Let \(f(x) = \frac{1}{(2\pi)^d} \hat{\hat \mu}(-x)\). We need to show \(\mu = fdx\), which is equivalent to for all \(A \in \mathcal B(\R^d)\),
  \[
    \mu(A) = \int_{\R^d} f \mathbf 1_A dx.
  \]
  Let \(h = \mathbf 1_A\) and wlog assume \(A\) is a bounded Borel set. The trick is to introduce an independent Gaussian random variable \(N \sim \mathcal N(0, I_d)\) with law \(G dx\). We have
  \[
    \int_{\R^d} h(x) d\mu(x) = \E(h(X))) = \lim_{\sigma \to 0} \E(h(X + \sigma N))
  \]
  by dominated convergence theorem. But
  \begin{align*}
    \E(h(X + \sigma N))
    &= \E \left(\int_{\R^d} h(X + \sigma x) G(x) dx \right) \\
    &= \E \left(\int_{\R^d} \int_{\R^d} h(X + \sigma x) e^{i \ip{u, x}} G(u) \frac{du dx}{(\sqrt{2\pi})^d} \right) \\
  \end{align*}
  as
  \[
    G(x) = \frac{1}{(\sqrt{2\pi})^d} \hat G(x) = \frac{1}{(\sqrt{2\pi})^d} \int_{\R^d} G(u) e^{i \ip{u, x}} du.
  \]
  So by a change of variable \(y = \sigma x\),
  \begin{align*}
    \E(h(X + \sigma N))
    &= \E \left(\int \int h(X + y) e^{i \ip{u, y/\sigma}} G(u) \frac{du}{(\sqrt{2\pi})^d} \frac{dy}{\sigma^d} \right) \\
    &= \E \left(\int \int h(z) e^{i \ip{u/\sigma, z - x}} G(u) \frac{du}{(\sqrt{2\pi \sigma^2})^d} dz \right) \\
    &= \int \int h(z) e^{i \ip{u/\sigma, z}} \hat \mu_X(- \frac{u}{\sigma}) G(u) \frac{du}{(\sqrt{2\pi \sigma^2})^d} dz \quad \text{ Tonelli-Fubini} \\
    &= \frac{1}{(2\pi)^d} \int \int \hat \mu_X(u) e^{-i \ip{u, z}} h(z) e^{-\sigma^2 \norm u^2/2} dudz
  \end{align*}
\end{proof}

We want a condition to ensure \(\hat f \in L^1(\R^d)\). Clearly continuity is necessary. Here we use a generally principle in Fourier analysis: Fourier transform converts decay at infinity to smoothness. In Fourier inversion formula, if \(u\) is large then the Fourier character has fast oscillation. Thus if \(\hat f\) decays fast at infinity then the Fourier coefficients also decays fast, and the resulting transfrom is smoother.

\begin{proposition}
  If \(f, f'\) and \(f''\) exists (for example if \(f\) is \(C^2\)) and are in \(L^1\) then \(\hat f \in L^1\).
\end{proposition}

\begin{proof}
  We prove the case \(d = 1\). The general case follows from Tonelli-Fubini. We show first that \(f, f' \in L^1\) implies that \(\hat f(u) = \frac{i}{u} \hat f'(u)\). This easily follows from integration by parts:
  \begin{align*}
    \hat f(u)
    &= \int f(x) e^{iux} dx \\
    &= \frac{1}{iu} \int f(x) (e^{iux})' dx \\
    &= -\frac{1}{iu} \int f'(x) e^{iux} dx
  \end{align*}
  so in particular \(|\hat f(u)| \leq \frac{1}{|u|} \norm{f'}_1\).

  Thus if \(f, f', f'' \in L^1\) then \(\hat f(u) = - \frac{1}{u^2} \hat f''(u)\) so
  \[
    |\hat f(u)| \leq \frac{\norm{f''}_1}{|u^2|}.
  \]
  As \(\int_1^\infty \frac{1}{|u|^2} du < \infty\), \(\hat f \in L^1\).
\end{proof}

\begin{definition}[convolution]\index{convolution}
  Given two Borel measures \(\mu\) and \(\nu\) on \(\R^d\), we define their \emph{convolution} \(\mu * \nu\) as the image of \(\mu \otimes \nu\) under the addition map
  \begin{align*}
    \Phi: \R^d \times \R^d &\to \R^d \\
    (x, y) &\mapsto x + y
  \end{align*}
  i.e.\ \(\mu * \nu = \Phi_*(\mu \otimes \nu)\)
\end{definition}

Thus given \(A \in \mathcal B(\R^d)\),
\[
  \Phi_*(\mu \otimes v)(A) = \mu \otimes \nu (\{(x, y): x + y \in A\}).
\]

\begin{eg}
  Given \(X, Y\) independent random variables and \(\mu, \nu\) be laws of \(X, Y\) respectively, then \(\mu * \nu\) is the law of \(X + Y\).
\end{eg}

\begin{definition}[convolution]\index{convolution}
  If \(f, g \in L^1(\R^d)\) define their \emph{convolution} \(f * g\) by
  \[
    (f * g) (x) = \int_{\R^d} f(x - t) g(t) dt.
  \]
\end{definition}

This is well defined by Fubini: \(f, g \in L^1\) so
\[
  \int_{\R^d} \int_{\R^d} |f(x - t) g(t)| dt dx < \infty
\]
and
\[
  \norm{f * g}_1
  = \int \Big| \int f(x - t) g(t) dt \Big| dx
  \leq \int \int |f(x - t) g(t)| dt dx
  \leq \norm f_1 \cdot \norm g_1
\]

Therefore \((L^1(\R^d), *)\) forms a \emph{Banach algebra}.

\begin{remark}
  If \(\mu, \nu\) are two finite Borel measures on \(\R^d\) and if \(\mu, \nu \ll dx\), i.e.\ absolutely continuous, then by Radon-Nikodym there exist \(f, g \in L^1(\R^d)\) such that
  \begin{align*}
    d\mu &= f dx \\
    d\nu &= g dx
  \end{align*}
  then \(\mu * \nu \ll dx\)  and
  \[
    d(\mu * \nu) = (f * g) dx.
  \]
\end{remark}

\begin{proposition}[Gaussian approximation]\index{Gaussian approximation}
  \label{prop:Gaussian approximation}
  If \(f \in L^p(\R^d)\) where \(p \in [1, \infty)\) then
  \[
    \lim_{\sigma \to 0} \norm{f * G_\sigma - f}_p = 0
  \]
  where \(G_\sigma = \mathcal N(0, \sigma^2 I_d)\), i.e.
  \[
    G_\sigma(x) = \frac{1}{(\sqrt{2\pi \sigma^2})^d} e^{- \frac{\norm x^2}{2\sigma^2}}.
  \]
\end{proposition}

\begin{lemma}[continuity of translation in \(L^p\)]
  Suppose \(p \in [1, \infty)\)  %maybe \infty?
  and \(f \in L^p\). Then
  \[
    \lim_{t \to 0} \norm{\tau_t(f) - f}_p = 0
  \]
  where \(\tau_t(f)(x) = f(x + t)\), \(t \in \R^d\).
\end{lemma}

\begin{proof}
  Example sheet.
\end{proof}

\begin{proof}[Proof of \nameref{prop:Gaussian approximation}]
  \[
    (f * G_\sigma - f)(x)
    = \int_{\R^d} G_\sigma(t) (f(x - t) - f(x)) dt
    = \E(f(x - \sigma N) - f(x))
  \]
  where \(N \sim \mathcal N(0, I_d)\) is Gaussian with density \(G_1\). Then
  \[
    \norm{f *G_\sigma - f}_p^p
    \leq \E(\norm{f(x + \sigma N) - f(x)}_p^p)
    = \E(\norm{\tau_{\sigma N}(f) - f}_p^p)
  \]
  by Jensen's inequality and convexity of \(x \mapsto x^p\). By the lemma,
  \[
    \lim_{\sigma \to 0} \norm{\tau_{\sigma N}(f) - f}_p = 0.
  \]
  As \(\norm{\tau_{\sigma N}(f) - f}_p \leq 2 \norm f_p\), apply dominated convergence theorem to get the required result.
\end{proof}

\begin{proposition}\leavevmode
  \begin{itemize}
  \item If \(f, g \in L^1(\R^d)\) then
    \[
      \widehat{f * g} = \hat f \cdot \hat g.
    \]
  \item If \(\mu, \nu\) are finite Borel measure then
    \[
      \widehat{\mu * \nu} = \hat \mu \cdot \hat \nu.
    \]
  \end{itemize}
\end{proposition}

\begin{proof}
  1 reduces to 2 by writing
  \[
    f(x) dx = f^+(x) dx - f^-(x) dx = a d\mu - b d\nu
  \]
  for some probability measure \(\mu, \nu\).

  wlog we may assume \(\mu\) and \(\nu\) are laws of independent random variables \(X\) and \(Y\). Then by a previous result \(\mu * \nu\) is just the law of \(X + Y\) so
  \begin{align*}
    \widehat{\mu * \nu}(u)
    &= \int e^{i \langle u, x + y \rangle} d\mu(x) d\nu(y) \\
    &= \E(e^{i \langle u, X + Y \rangle}) = \E(e^{i \langle u, X \rangle} e^{i \langle u, Y \rangle}) \quad \text{ homomorphism} \\
    &= \E(e^{i \langle u, X \rangle}) \E(e^{i \langle u, Y \rangle}) \quad \text{ as \(X, Y\) are independent} \\
    &= \hat \mu(u) \cdot \hat \nu(u).
  \end{align*}

  In short, this is precisely because \(e^{i \ip{u, -}}\) are characters.
\end{proof}

\begin{theorem}[Lévy criterion]\index{Lévy criterion}
  Let \((X_n)_{n \geq 1}\) and \(X\) be \(\R^d\)-valued random variables. Then TFAE:
  \begin{enumerate}
  \item \(X_n \to X\) in law,
  \item For all \(u \in \R^d\), \(\hat \mu_{X_n}(u) \to \hat \mu_X(u)\).
  \end{enumerate}
  In particular if \(\hat \mu_X = \hat \mu_Y\) for two random variables \(X\) and \(Y\) then \(X = Y\) in law, i.e.\ \(\mu_X = \mu_Y\).
\end{theorem}

Thus Fourier transform is an injection from Borel measure to certain function space.

\begin{proof}\leavevmode
  \begin{itemize}
  \item \(1 \implies 2\): Clear by defintion as \(f(x) = e^{i \langle u, x \rangle}\) is continuous and bounded for all \(u \in \R^d\).
  \item \(2 \implies 1\): Need to show that for all \(g \in C_b(\R^d)\),
    \[
      \E(g(X_n)) \to \E(g(X)).
    \]
    wlog it's enough to check this for all \(g \in C_c^\infty(\R^d)\). For the sufficiency see example sheet.

    Note that for all \(g \in C_c^\infty(\R^d)\), \(\hat g \in L^1\) so by Fourier inversion formula
    \[
      g(x) = \int \hat g(u) e^{-i \langle u, x\rangle} \frac{du}{(2\pi)^d}.
    \]
    Hence
    \begin{align*}
      \E(g(X_n))
      &= \int \hat g(u) \E(e^{-i \langle u, X_n \rangle}) du \\
      &= \int \hat g(u) \hat \mu_{X_n} (-u) \frac{du}{(2\pi)^d} \\
      &\to \int \hat g(u) \hat m\u_X(-u) \frac{du}{(2\pi)^d} \\
      &= \E(g(X))
    \end{align*}
    by dominated convergence theorem.
  \end{itemize}
\end{proof}

% think of \(f * g = \int \tau_{it}(f) g(t) dt\) so the mean of translation with respect to \(g\)

\begin{theorem}[Plancherel formula]\index{Plancherel formula}\leavevmode
  \begin{enumerate}
  \item If \(f \in L^1(\R^d) \cap L^2(\R^d)\) then \(\hat f \in L^2(\R^d)\) and
    \[
      \norm{\hat f}^2_2 = (2\pi)^d \norm f^2_2.
    \]
  \item If \(f, g \in L^1(\R^d) \cap L^2(\R^d)\) then
    \[
      \ip{\hat f, \hat g}_{L^2} = (2\pi)^d \ip{f, g}_{L^2}.
    \]
  \item The Fourier transform
    \begin{align*}
      \mathcal F: L^1(\R^d) \cap L^2(\R^d) &\to L^2(\R^d) \\
      f &\mapsto \frac{1}{(\sqrt{2\pi})^d} \hat f
    \end{align*}
    extends uniquely to a linear operator on \(L^2(\R^d)\) which is an isometry. Moreover
    \[
      \mathcal F \compose \mathcal F (f) = \check f
    \]
    where \(\check f(x) = f(-x)\), for all \(f \in L^2(\R^d)\).
  \end{enumerate}
\end{theorem}

\begin{proof}
  First we prove 1 and 2 assuming \(\hat f, \hat g \in L^1(\R^d)\). By Fourier inversion formula,
  \begin{align*}
    \norm{\hat f}_2^2
    &= \int_{\R^d} |\hat f(u)|^2 du \\
    &= \int \hat f(u) \conj{\hat f(u)} du \\
    &= \int \left(\int f(x) e^{i \ip{u, x}} dx \right) \conj{\hat f(u)} du \\
    &= \int \int f(x) \conj{\hat f(u) e^{-i \ip{u, x}}} du dx \\
    &= \int f(x) \conj{f(x)} (2\pi)^d dx \\
    &= (2\pi)^d \norm f_2^2
  \end{align*}
  and in particular \(\hat f \in L^2(\R^d)\).

  Similarly for 2,
  \begin{align*}
    \ip{\hat f, \hat g}_{L^2}
    &= \int \hat f(u) \conj{\hat g(u)} du \\
    &= \int \left(\int f(x) e^{i \ip{u, x}} dx\right) \conj{\hat g(u)} du \\
    &= \int \int f(x) \conj{\hat g(u) e^{-i \ip{u, x}}} du dx \\
    &= \int f(x) \conj{g(x)} dx (2\pi)^d \\
    &= (2\pi)^d \ip{f, g}_{L^2}
  \end{align*}

  Now for the general case we use Gaussian as a mollifier. Consider
  \begin{align*}
    f_\sigma &= f * G_\sigma \\
    g_\sigma &= g * G_\sigma
  \end{align*}
  and based on results and computations before,
  \[
    \hat f_\sigma
    = \hat f \cdot \hat G_\sigma
    = \hat f e^{-\sigma \norm u^2/2}.
  \]
  As \(\norm{\hat f}_\infty \leq \norm f_1\), \(\hat f_\sigma \in L^1(\R^d)\). Thus \(\hat f_\sigma \in L^2(\R^d)\) and \(\norm{\hat f_\sigma}_2^2 = (2\pi)^d \norm{f_\sigma}^2_2\). But by Gaussian approximation we know that \(f_\sigma \to f\) in \(L^2(\R^d)\) as \(\sigma \to 0\). Hence \(\norm{f_\sigma}_2 \to \norm f_2\). Then
  \[
    \norm{\hat f_\sigma}_2^2
    = \norm{\hat f \cdot \hat G_\sigma}_2^2
    = \int |\hat f(u)|^2 e^{-\sigma \norm u^2/2} du
    \to \norm f^2_2
  \]
  as \(\sigma \to 0\) by monotone convergence theorem. Thus
  \[
    \norm{\hat f}_2^2 = (2\pi)^d \norm f_2^2.
  \]

  For 2,
  \[
    \ip{\hat f_\sigma, \hat g_\sigma}
    = \int \hat f \conj{\hat g} e^{-\sigma \norm u^2} du
    \to \int \hat f \conj{\hat g} du
  \]
  as \(\sigma \to 0\) by dominated convergence theorem as \(\hat f \conj{\hat g} \in L^1\). The result follows from Gaussian approximation.

  For 3, \(L^1(\R^d) \cap L^2(\R^d)\) is dense in \(L^2(\R^d)\) because it contains \(C_c(\R^d)\). Then extend by completeness: given \(f \in L^2(\R^d)\), pick a sequence \(f_n \in L^1(\R^d) \cap L^2(\R^d)\) such that \(f_n \to f\) in \(L_2(\R^d)\). Then define
  \[
    \mathcal F f = \lim_{n \to \infty} \mathcal F f_n.
  \]
  The limit exists as \(L^2(\R^d)\) is complete. \(\mathcal F\) is well-defined as
  \[
    \norm{\mathcal F f_n - \mathcal F f_m}_2 = \norm{f_n - f_m}_2
  \]
  by 1. Finally,
  \[
    \norm{\mathcal F f}_2 = \norm f_2
  \]
  for all \(f \in L^2(\R^d)\) and
  \[
    \mathcal F \compose \mathcal F(f) = \check f
  \]
  for all \(f\) such that \(f, \hat f \in L^1(\R^d)\). Thus by continuity this holds for \(L^2(\R^d)\).
\end{proof}

\section{Gaussians}

\begin{definition}[Gaussian]\index{Gaussian}
  An \(\R^d\)-valued random variable \(X\) is called \emph{Gaussian} if for all \(u \in \R^d\), \(\ip{X, u}\) is Gaussian, namely its law has the form \(\mathcal N(m, \sigma^2)\) for some \(m \in \R, \sigma > 0\).
\end{definition}

\begin{proposition}
  The law of a Gaussian vector \(X = (X_1, \dots, X_d) \in \R^d\) is uniquely determined by
  \begin{enumerate}
  \item its \emph{mean}\index{mean} \(\E X = (\E X_1, \dots, \E X_d)\),
  \item its \emph{covariance matrix}\index{covariance matrix} \((\cov(X_i, X_j))_{ij}\) where
    \[
      \cov(X_i, X_j) = \E((X_i - \E X_i) (X_j - \E X_j)).
    \]
  \end{enumerate}
\end{proposition}

\begin{proof}
  If \(d = 1\) then this just says that it is determined by its mean \(m\) and covariance \(\sigma^2\), which is obviously true. For \(d > 1\), compute the characteristic function
  \[
    \hat \mu_X(u) = \E(e^{i \ip{X, u}})
  \]
  but by assumption \(\ip{X, u}\) is Gaussian in \(d = 1\) so its law is determined
  \begin{enumerate}
  \item the mean \(\E(\ip{X, u}) = \ip{\E X, u}\),
  \item the variance \(\var \ip{X, u}\). But
    \[
      \var \ip{X, u}
      = \E((\ip{X, u} - \E \ip{X, u})^2)
      = \sum_{i, j} u_iu_j \cov (X_i, X_j).
    \]
  \end{enumerate}
\end{proof}

In particular this shows that \((\cov(X_i, X_j))_{ij}\) is a non-negative semidefinite symmetric matrix.

\begin{proposition}
  If \(X\) is a Gaussian vector then exists \(A \in \matrixring_d(\R), b \in \R^d\) such that \(X\) has the same law as \(A N + b\) where \(N = (N_1, \dots, N_d)\), \((N_i)_{i = 1}^d\) are iid.\ \(\mathcal N(0, 1)\).
\end{proposition}

\begin{proof}
  Take \(A\) such that
  \[
    AA^* = (\cov(X_i, X_j))_{ij}
  \]
  where \(A^*\) is the adjoint/transpose of \(A\), and
  \[
    b = (\E X_1, \dots, \E X_d).
  \]
  Check that for all \(u \in \R^d\),
  \begin{align*}
    \E(\ip{X, u}) &= \ip{b, u} \\
    \var(\ip{X, u}) &= \ip{AA^* u, u} = \norm{A^* u}_2^2 = \var(\ip{AN + b, u})
  \end{align*}
\end{proof}

\begin{proposition}
  If \((X_1, \dots, X_d)\) is a Gaussian vector then TFAE:
  \begin{enumerate}
  \item \(X_i\)'s are independent.
  \item \(X_i\)'s are pairwise independent.
  \item \((\cov(X_i, X_j))_{ij}\) is a diagonal matrix.
  \end{enumerate}
\end{proposition}

\begin{proof}
  \(1 \implies 2 \implies 3\) is obvious. \(3 \implies 1\) as we can choose \(A\) to be diagonal. Thus \(X\) has the same law as \((a_1N_1, \dots, a_dN_d) + b\).
\end{proof}

\begin{theorem}[central limit theorem]\index{central limit theorem}
  Let \((X_i)_{i \geq 1}\) be \(\R^d\)-valued iid.\ random variables with law \(\mu\). Assume they have second moment, i.e.\ \(\E(\norm{X_1}^2) < \infty\). Let \(\V m = \E(X_1) \in \R^d\) and
  \[
    Y_n = \frac{X_1 + \dots + X_n - n \cdot \V m}{\sqrt n}.
  \]
  Then \(Y_n\) converges in law to a central Gaussian on \(\R^d\) with law \(\mathcal N(0, K)\) where
  \[
    K_{ij} = (\cov(X_1))_{ij} = \left[ \int_{\R^d} (x_i - \E(X_1)) (x_j - \E(X_1)) d \mu(x) \right]_{ij}.
  \]
\end{theorem}

%second order correction to law of large numbers?

\begin{proof}
  The proof is an application of Lévy criterion. Need to show \(\hat \mu_{Y_n}(u) \to \hat \mu_Y(u)\) as \(n \to \infty\) for all \(u\), where \(Y \sim \mathcal N(0, K)\). As
  \[
    \hat \mu_{Y_n}(u) = \E( e^{i \ip{Y_n, u}}),
  \]
  this is equivalent to show that for all \(u\), \(\ip{Y_n, u}\) converges in law to \(\ip{Y, u}\). But
  \[
    \ip{Y_n, u} = \frac{\ip{X_1, u} + \dots + \ip{X_n, u} - n \ip{\V m, u}}{\sqrt n}
  \]
  so we reduce the problem to \(1\)-dimension case. By rescaling wlog \(\E(X_1) = 0, \E(X_1^2) = 1\).

  Now
  \begin{align*}
    \hat \mu_{Y_n}(u)
    &= \E(e^{iuY_n}) \\
    &= \E(\exp (iu \frac{X_1 + \dots + X_n}{\sqrt n})) \\
    &= \prod_{i = 1}^n \E(\exp (iu \frac{X_i}{\sqrt n})) \\
    &= (\E(\exp (iu \frac{X_i}{\sqrt n})))^n \\
    &= (\hat \mu (\frac{u}{\sqrt n}))^n
  \end{align*}
  But \(\E(X_1) = 0, \E(X_1^2) = 1\) so we can differentiate \(\hat \mu\) under the integral sign
  \begin{align*}
    \hat \mu(u) &= \int_\R e^{iux} d \mu(x) \\
    \frac{d}{du} \hat \mu(u) &= \int_\R ix e^{iux} d\mu(x) = i \E(X_1) \\
    \frac{d^2}{du^2} \hat \mu(u) &= \int_\R -x^2 e^{iux} d\mu(x) = -\E(X_1^2)
  \end{align*}
  Taylor expand \(\hat \mu\) around \(0\) to \(2\)nd order,
  \begin{align*}
    \hat \mu (u)
    &= \hat \mu(0) + u \hat \mu'(0) + \frac{u^2}{2} \hat \mu''(u) + o(u^2) \\
    &= 1 + 0 \cdot u - \frac{u^2}{2} + o(u^2)
  \end{align*}
  so
  \[
    \hat \mu_{Y_n}(u)
    = (1 - \frac{u^2}{2n} + o(\frac{u^2}{n}))^n
    \to e^{-u^2/2}
    = \hat g(u)
  \]
  as \(n \to \infty\) where \(g\) is the law of \(Y\).
\end{proof}

\section{Ergodic theory}

Let \((X, \mathcal A, \mu)\) be a measure space. Let \(T: X \to X\) be an \(\mathcal A\)-measurable map. We are interested in the trajectories of \(T^n x\) for \(n \geq 0\) and their statistical behaviour. In particular we are interested in those \(T\) \emph{preserving measure \(\mu\)}.

\begin{definition}[measure-preserving]\index{measure-preserving}\index{measure-preserving dynamical system}
  \(T: X \to X\) is \emph{measure-preserving} if \(T_*\mu = \mu\). \((X, \mathcal A, \mu, T)\) is called a \emph{measure-preserving dynamical system}.
\end{definition}

\begin{definition}[invariant function, invariant set, invariant \(\sigma\)-algebra]\index{invariant function}\index{invariant set}\index{invariant \(\sigma\)-algebra}\index{\(\sigma\)-algebra!invariant}\leavevmode
  \begin{itemize}
  \item A measurable function \(f: X \to \R\) is called \emph{\(T\)-invariant} if \(f = f \compose T\).
  \item A set \(A \in \mathcal A\) is \emph{\(T\)-invariant} if \(\mathbf 1_A\) is \(T\)-invariant.
  \item
    \[
      \mathcal T = \{A \in \mathcal A: A \text{ is \(T\)-invariant}\}
    \]
    % TODO: decide whether this should be I or T
    is called the \emph{\(T\)-invariant \(\sigma\)-algebra}.
  \end{itemize}
\end{definition}

\begin{lemma}
  \(f\) is \(T\)-invariant if and only if \(f\) is \(\mathcal T\)-measurable.
\end{lemma}

\begin{proof}
  Indeed for all \(t \in \R\),
  \[
    \{x \in X: f(x) < t\}
    = \{x \in X: f \compose T (x) < t\}
    = T^{-1}(\{x \in X: f(x) < t\}).
  \]
\end{proof}

\begin{definition}[ergodic]\index{ergodic}
  \(T\) is \emph{ergodic} with respect to \(\mu\), or that \(\mu\) is \emph{ergodic} with respect to \(T\), if for all \(A \in \mathcal T\), \(\mu(A) = 0\) or \(\mu(A^c) = 0\).
\end{definition}

This condition asserts that \(\mathcal T\) is \emph{trivial}, i.e.\ its elements are either null or conull.

\begin{lemma}
  \(T\) is ergodic with respect to \(\mu\) if and only if  every invariant function \(f\) is almost everywhere constant.
\end{lemma}

%All (sub)level sets of \(f\) belong to \(\mathcal T\).

\begin{proof}
  Exercise.
\end{proof}

\begin{eg}\leavevmode
  \begin{enumerate}
  \item Let \(X\) be a finite space, \(T: X \to X\) a map and \(\mu = \#\) the counting measure, then \(T\) is measure preserving is equivalent to \(T\) being a bijection, and \(T\) is ergodic is equivalent to there does not exists a partition \(X = X_1 \cup X_2\) such that both \(X_1\) and \(X_2\) are \(T\)-invariant, which is equivalent to for all \(x, y \in X\), there exists \(n\) such that \(T^n x = y\).
  \item Let \(X = \R^d/\Z^d\), \(\mathcal A\) the Borel \(\sigma\)-algebra and \(\mu\) the Lebesgue measure. Given \(a \in \R^d\), translation \(T_a: x \mapsto x + a\) is measure-preserving. \(T_a\) is ergodic with respect to \(\mu\) if and only if \((1, a_1, \dots, a_d)\), where \(a_i\)'s are coordinates of \(a\), are linearly independent. See example sheet. (hint: Fourier transform)
  \item Let \(X = \R/\Z\) and again \(\mathcal A\) Borel \(\sigma\)-algebra and \(\mu\) the Lebesgue measure. The doubling map \(T: x \mapsto 2x - \floor{2x}\) is ergodic with respect to \(\mu\). (hint: again consider Fourier coefficeints). Intuitively in the graph of this function the preimage \(\varepsilon\) is two segments each of length \(\varepsilon/2\), so measure-preserving.
  \item Furstenberg conjecture\index{Furstenberg conjecture}: every ergodic measure \(\mu\) on \(\R/\Z\) invariant under \(T_2, T_3\) must be either Lebesgue or finitely supported.
  \end{enumerate}
\end{eg}

\subsection{The canonical model}

Let \((X_n)_{n \geq 1}\) be an \(\R^d\)-valued stochastic process\index{stochastic process} on \((\Omega, \mathcal F, \P)\). Let \(X = (\R^d)^\N\) and define the \emph{sample path map}\index{sample path map}
\begin{align*}
  \Phi: \Omega &\to X \\
  \omega &\mapsto (X_n(\omega))_{n \geq 1}
\end{align*}
Let
\begin{align*}
  T: X &\to X \\
  (x_n)_{n \geq 1} &\mapsto (x_{n + 1})_{n \geq 1}
\end{align*}
be the \emph{shift map}. Let \(x_n: X \to \R^d\) be the \emph{\(n\)th coordinate function} and let \(\mathcal A = \sigma(x_n: n \geq 1)\).

\begin{note}
  \(\mathcal A\) is the infinite product \(\sigma\)-algebra\index{infinite product \(\sigma\)-algebra}\index{product \(\sigma\)-algebra!infinite} \(\mathcal B(\R^d)^{\otimes \N}\) of \(\mathcal B(\R^d)^\N\).
\end{note}

Let \(\mu = \Phi_*\P\), a probability measure on \((X, \mathcal A)\). This \(\mu\) is called the \emph{law} of the process \((X_n)_{n \geq 1}\). Now \((X, \mathcal A, \mu, T)\) is called the \emph{canonical model}\index{canonical model} associated to \((X_n)_{n \geq 1}\).

\begin{proposition}[stationary process]\index{stationary process}
  TFAE:
  \begin{enumerate}
  \item \((X, \mathcal A, \mu, T)\) is measure-preserving.
  \item For all \(k \geq 1\), the law of \((X_n, X_{n + 1}, \dots, X_{n + k})\) on \((\R^d)^k\) is independent of \(n\).
  \end{enumerate}
  In this case we say that \((X_n)_{n \geq 1}\) is a \emph{stationary process}.
\end{proposition}

\begin{proof}\leavevmode
  \begin{itemize}
  \item \(1 \implies 2\): \(\mu = T_*\mu\) implies \(\mu = T^n_*\mu\) for all \(\mu\) and this says law of \((X_i)_{i \geq 1}\) is the same as that of \((X_{i + n})_{i \geq 1}\).
  \item \(1 \impliedby 2\): \(\mu\) and \(T_*^n\mu\) agree on cylinders \(A \times (\R^d)^{\N \setminus F}\) for \(F \subseteq \N\) finite, \(A \in \mathcal B((\R^d)^F)\).
  \end{itemize}
\end{proof}

In some sense ergodic system is the study of stationary process.

\begin{proposition}[Bernoulli shift]\index{Bernoulli shift}
  If \((X_n)_{n \geq 1}\) are iid.\ then \((X, \mathcal A, \mu, T)\) is ergodic. It is called the \emph{Bernoulli shift} associated to the law \(\nu\) of \(X_1\). We have
  \[
    \mu = \nu^{\otimes \N}.
  \]
\end{proposition}

\begin{proof}
  Claim that \(\Phi^{-1}(\mathcal T) \subseteq \mathcal C\), the tail \(\sigma\)-algebra of \((X_n)_{n \geq 1}\)\index{tail event}. But Kolmogorov \(0\)-\(1\) law\index{Kolmogorov \(0 - 1\) law} says that if \(A \in \mathcal T\) then \(\P(\Phi^{-1}(A)) = 0\) or \(1\), so \(\mu(A) = 0\) or \(1\), thus \(\mu\) is \(T\)-ergodic.

  Given \(A \in \mathcal T\), \(T^{-1}A = A\) so
  \begin{align*}
    \Phi^{-1}(A)
    &= \{\omega \in \Omega: (X_n(\omega))_{n \geq 1} \in A\} \\
    &= \{\omega \in \Omega: (X_n(\omega))_{n \geq 1} \in T^{-1}A\} \\
    &= \{\omega \in \Omega: (X_{n + 1}(\omega))_{n \geq 1} \in A\} \\
    &= \{\omega \in \Omega: (X_{n + k}(\omega))_{n \geq 1} \in A\} \quad \text{ for all } k \\
    &\in \sigma(X_k, X_{k + 1}, \dots)
  \end{align*}
  for \(\mu\)-almost every \(x\).
\end{proof}

\begin{theorem}[von Neumann mean ergodic theorem]\index{von Neumann mean ergodic theorem}
  Let \((X, \mathcal A, \mu, T)\) be a measure-preserving system. Let \(f \in L^2(X, \mathcal A, \mu)\). Then the ergodic average
  \[
    S_nf = \frac{1}{n} \sum_{i = 0}^{n - 1} f \compose T^i
  \]
  converges in \(L^2\) to \(\overline f\), a \(T\)-invariant function. In fact \(\overline f\) is the orthogonal projection of \(f\) onto \(L^2(X, \mathcal T, \mu)\).
\end{theorem}

The intuition is as follow: if \(f\) is the indicator function of a set, then \(S_nf\) is exactly the average of the time each orbit spending in \(A\).

\begin{proof}
  Hilbert space argument. Let \(H = L^2(X, \mathcal A, \mu)\) and define
  \begin{align*}
    U: H &\to H \\
    f &\mapsto f \compose T
  \end{align*}
  which is an isometry: because \(\mu\) is \(T\)-invariant, \(\int |f \compose T|^2 d\mu = \int |f|^2 d\mu\). Then by Riesz representation theorem\index{Riesz representation theorem} it has an adjoint
  \begin{align*}
    U^*: H &\to H \\
    x &\mapsto U^* x
  \end{align*}
  which satisfies \(\ip{U^*x, y} = \ip{x, Uy}\) for all \(y \in H\). Let
  \[
    W = \{\varphi- \varphi \compose T: \varphi \in H\}
  \]
  be the \emph{coboundaries}\index{coboundary}. Let \(f \in W\). Then
  \[
    S_nf = \frac{1}{n} \sum_{i = 0}^{n - 1} (\varphi \compose T^i - \varphi \compose T^{i + 1}) = \frac{\varphi - \varphi \compose T^n}{n} \to 0
  \]
  as \(n \to \infty\).

  Let \(f \in \cl W\) then again \(S_nf \to 0\) because for all \(\varepsilon\) exists \(g \in W\) such that \(\norm{f - g} < \varepsilon\). Then
  \[
    \norm{S_nf - S_ng}
    = \norm{S_n(f - g)}
    \leq \norm{f - g}
    \leq \varepsilon
  \]
  so \(\limsup_n \norm{S_nf} \leq \varepsilon\).

  Have \(H = \cl W \oplus \cl W^\perp\) and \(\cl W^\perp = W^\perp\). Claim \(W^\perp\) is exactly the \(T\)-invariant functions. The theorem then follows because if \(f \compose T = f\) then \(S_nf = f\) for all \(f\).

  \begin{proof}[Proof of claim]
    \begin{align*}
      W^\perp
      &= \{g \in H: \ip{g, \varphi - U\varphi} = 0 \text{ for all } \varphi \in H\} \\
      &= \{g: \ip{g, \varphi} = \ip{g, U\varphi} \text{ for all } \varphi\} \\
      &= \{g: \ip{g, \varphi} = \ip{U^*g, \varphi} \text{ for all } \varphi\} \\
      &= \{g: U^*g = g\} \\
      &= \{g: Ug = g\}
    \end{align*}
    where the last equality is by
    \[
      \norm{Ug - g}^2 = 2\norm g^2 - 2\Re \ip{g, Ug} = 2 \norm g^2 - 2 \Re \ip{U^*g, g},
    \]
    and this shows that \(W^\perp\) are exactly \(T\)-invariant functions.
  \end{proof}
\end{proof}

In fact we can do better:

\begin{theorem}[Birkhoff (pointwise) ergodic theorem]\index{Birkhoff ergodic theorem}
  \label{thm:Birkhoff ergodic theorem}
  Let \((X, \mathcal A, \mu, T)\) be a measure-preserving system. Assume \(\mu\) is finite (actually \(\sigma\)-finite suffices) and let \(f \in L^1(X, \mathcal A, \mu)\). Then
  \[
    S_nf = \frac{1}{n} \sum_{i = 0}^{n - 1} f \compose T^i
  \]
  converges \(\mu\)-almost everywhere to a \(T\)-invariant function \(\overline f \in L^1\). Moreover \(S_n f \to \overline f\) in \(L^1\).
\end{theorem}

% story: Birkhoff was pulished in Dec 1931 and von Neumann was published in Feb 1932. Needless to say he's not very happy

\begin{corollary}[strong law of large numbers]\index{strong law of large numbers}\index{law of larger numbers}
  Let \((X_n)_{n \geq 1}\) be a sequence of iid.\ random variables. Assume \(\E(|X_1|) < \infty\). Let
  \[
    S_n = \sum_{k = 1}^n X_k,
  \]
  then \(\frac{1}{n} S_n\) converges almost surely to \(\E(X_1)\).
\end{corollary}

\begin{proof}
  Let \((X, \mathcal A, \mu, T)\) be the canonical model associated to \((X_n)_{n \geq 1}\), where \(X = \R^\N, \mathcal A = \mathcal B(\R)^{\otimes \N}\), \(T\) the shift operator and \(\mu = \nu^{\otimes \N}\) where \(\nu\) is the law of \(X_1\). It is a Bernoulli shift. Let
  \begin{align*}
    f: X &\to \R \\
    x &\mapsto x_1
  \end{align*}
  the first coodinate. Then \(f \compose T^i(x) = x_{i + 1}\) so
  \[
    \frac{1}{n}(X_1 + \dots + X_n) (\omega) = S_nf(x)
  \]
  where \(x = (X_n(\omega))_{n \geq 1}\). Hence by Birkhoff ergodic theorem
  \[
    \frac{1}{n}(X_1 + \dots + X_n)
    \to \overline f = \int f d\mu = \int x_1 d\mu = \E(X_1)
  \]
  almost surely.
\end{proof}

\begin{remark}
  If \(T\) is ergodic then \(\overline f\) is almost everywhere constant. Hence \(\overline f = \int f d\mu\).
\end{remark}

\begin{lemma}[maximal ergodic lemma]\index{maximal ergodic lemma}
  \label{lem:maximal ergodic lemma}
  Let \(f \in L^1(X, \mathcal A, \mu)\) and \(\alpha \in \R\). Let
  \[
    E_\alpha = \{x \in X: \sup_{n \geq 1} S_nf(x) > \alpha\}
  \]
  then
  \[
    \alpha \mu(E_\alpha) \leq \int_{E_\alpha} f d\mu.
  \]
\end{lemma}

\begin{lemma}[maximal inequality]\index{maximal inequality}
  \label{lem:maximal inequality}
  Let
  \begin{align*}
    f_0 &= 0 \\
    f_n &= n S_nf = \sum_{i = 0}^{n - 1} f \compose T^i \quad n \geq 1
  \end{align*}
  Let
  \[
    P_N = \{x \in X: \max_{0 \leq n \leq N} f_n(x) > 0\}.
  \]
  Then
  \[
    \int_{P_N} f d\mu \geq 0.
  \]
\end{lemma}

\begin{proof}[Proof of \nameref{lem:maximal inequality}]
  Set \(F_N = \max_{0 \leq n \leq N} f_n\). Observe that for all \(n \leq N\), \(F_N \geq f_n\) and hence
  \[
    F_N \compose T + f \geq f_n \compose T + f = f_{n + 1}.
  \]
  Now if \(x \in P_n\) then
  \[
    F_N(x)
    \leq \max_{0 \leq n \leq N} f_{n + 1} % ???
    \leq F_N \compose T + f
  \]
  Integrate to get
  \[
    \int_{P_N} F_N d\mu \leq \int_{P_N} F_N \compose T d \mu + \int_{P_N} f d\mu.
  \]
  Note that \(F_N(x) = 0\) if \(x \notin P_N\) because \(f_0 = 0\) so
  \[
    \int_{P_N} F_N
    = \int_X F_N
    \leq \int_X F_N \compose T + \int_{P_n} f.
  \]
  As \(\mu\) is \(T\)-invariant, \(\int F_N \compose T = \int F_N\) so
  \[
    \int_{P_N} f d\mu \geq 0.
  \]
\end{proof}

\begin{proof}[Proof of \nameref{lem:maximal ergodic lemma}]
  Apply the maximal inequality to \(g = f - \alpha\). Observe that
  \[
    E_\alpha(f) = \bigcup_{N \geq 1} P_N(g)
  \]
  and \(S_n g = S_n f - \alpha\). Thus
  \[
    \int_{E_\alpha(f)} (f - \alpha) d\mu \geq 0,
  \]
  which is equivalent to
  \[
    \alpha \mu(E_\alpha) \leq \int_{E_\alpha} f d\mu.
  \]
\end{proof}

\begin{proof}[Proof of \nameref{thm:Birkhoff ergodic theorem}]
  Let
  \begin{align*}
    \overline f &= \limsup_n S_nf \\
    \underline f &= \liminf_n S_nf \\
  \end{align*}
  Observe that \(\overline f = \overline f \compose T, \underline f \compose T = \underline f\): indeed
  \[
    S_nf \compose T
    = \frac{1}{n}(f \compose T + \dots + f \compose T^n)
  = \frac{1}{n} ((n + 1)S_{n + 1}f - f).
  \]

  Need to show that \(\overline f = \underline f\) \(\mu\)-almost everywhere. This is equivalent to for all \(\alpha, \beta \in \Q, \alpha > \beta\), the set
  \[
    E_{\alpha, \beta}(f) = \{x \in X: \underline f(x) < \beta, \overline f(x) > \alpha\}
  \]
  is \(\mu\)-null, as then
  \[
    \{x: \overline f(x) \neq \underline f(x)\} = \bigcup_{\alpha > \beta} E_{\alpha, \beta}
  \]
  is \(\mu\)-null by subadditivity. Observe that \(E_{\alpha, \beta}\) is \(T\)-invariant. Apply the maximal ergodic theorem to \(E_{\alpha, \beta}\) to get
  \[
    \alpha \mu(E_{\alpha, \beta}(f)) \leq \int_{E_{\alpha, \beta}} f d\mu.
  \]
  Dually
  \[
    -\beta \mu(E_{\alpha, \beta}(f)) \leq \int_{E_{\alpha, \beta}} -f d\mu
  \]
  so
  \[
    \alpha \mu(E_{\alpha, \beta}) \leq \beta \mu(E_{\alpha, \beta}).\]
  But \(\alpha > \beta\) so \(\mu(E_{\alpha, \beta}) = 0\).

  We have proved that the limit \(\lim_n S_nf\) exists almost everywhere, which we now define to be \(\overline f\), and left to show \(\overline f \in L^1\) and \(\lim_n \norm{S_nf - \overline f}_1 = 0\). This is an application of Fatou's lemma:
  \begin{align*}
    \int |\overline f| d\mu
    &= \int \liminf_n |S_n f| d\mu \\
    &\leq \liminf_n \int |S_n f| d\mu \\
    &\leq \liminf_n \norm{S_n f}_1 \\
    &\leq \norm f_1
  \end{align*}
  so \(\overline f \in L^1\) where the last inequality is because
  \[
    \norm{S_nf} \leq \frac{1}{n}(\norm f_1 + \dots + \norm{f \compose T^{n - 1}}_1) = \norm f_1.
  \]
  Now to show \(\norm{S_n f - \overline f}_1 \to 0\), we truncate \(f\). Let \(M > 0\) and set \(\varphi_M = f \mathbf 1_{|f| < M}\). Note that
  \begin{itemize}
  \item \(|\varphi_M| \leq M\) so \(|S_n \varphi_M| \leq M\). Hence by dominated convergence theorem \(\norm{S_n \varphi_M - \overline\varphi_M}_1 \to 0\).
  \item \(\varphi_M \to f\) \(\mu\)-almost everywhere and also in \(L^1\) by dominated convergence theorem.
  \end{itemize}
  Thus by Fatou's lemma,
  \[
    \norm{\overline \varphi_M - \overline f}_1
    \leq \liminf_n \norm{S_n \varphi_M - S_n f}_1
    \leq \norm{\varphi_M - f}_1
  \]
  Finally
  \begin{align*}
    \norm{S_nf - \overline f}_1
    &\leq \norm{S_n f - S_n \varphi_M}_1 + \norm{S_n \varphi_M - \overline \varphi_M}_1 + \norm{\overline \varphi_M - \overline f}_1 \\
    &\leq \norm{f - \varphi_M}_1 + \norm{S_n \varphi_M - \overline \varphi_M}_1 + \norm{f - \varphi_M}_1
  \end{align*}
  so
  \[
    \limsup \norm{S_nf - \overline f}_1 \leq 2 \norm{f - \varphi_M}_1
  \]
  for all \(M\), so goes to \(0\) as \(M \to \infty\).
\end{proof}

\begin{remark}\leavevmode
  \begin{enumerate}
  \item The theorem holds if \(\mu\) is only assumed to be \(\sigma\)-finite.
  \item The theorem holds if \(f \in L^p\) for \(p \in [1, \infty)\). The \(S_nf \to f\) in \(L^p\).
  \end{enumerate}
\end{remark}

\printindex
\end{document}

% http://www.statslab.cam.ac.uk/~james/Lectures/pm.pdf

% Books: books listed on course handbook, as well as
% Intorduction to measure theory, T. Tao
% Real and Complex analysis, W. Rudin

% related courses: IID Linear Analysis

% schedule
% wk 1: Lebesgue measure on \R^d
% wk 2: abstract measure theory
% wk 3: integration
% wk 4: measure theoretical foundations of probability theory
% wk 5: modes of convergence of random variables
% wk 6: Hilbert space techniques, L^p spaces
% wk 7: Fourier analysis, Gaussian random variables, Central Limit Theorem, Law of Large Number Theory
% wk 8: Ergodic Theorem