text:
\section{The general construction used in the proof of Theorem~\ref{cor:complexity-fg}}
\label{sec:construction}
\subsection{The construction for the iteration complexity result}
\sa{Based on \cref{Lemma: graident of ME}, the key step for establishing \xz{\texttt{SAPD+}} convergence is to bound} $\|x^{t}_{0}-\prox{\lambda\phi}(x^{t}_{0})\|$, \sa{where $\phi(x) \triangleq \max_{y\in\mathcal{Y}}\mathcal{L}(x,y)$ for {every $x\in\mathcal{X}$}
and $\lambda = (\gamma+\mu_x)^{-1}$. To achieve this, we first give a bound on the gap function $\mathcal{G}^{t}$ at the $t$-th outer iteration.}
\begin{lemma}\label{lemma: Gap bound with fg}
Suppose \cref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption} hold. \sa{
Given $\{N_t\}_{t\geq 0}\subset\mathbb{Z}_+$, let $\{x^{t}_{0},y^{t}_{0}\}_{t\geq 1}$ be generated by \xz{\texttt{SAPD+}}, stated in~algorithm~\ref{Alg: SAPD-plus}, when \texttt{VR-flag}=\textbf{false}, initialized from {$(x_0^0,y_0^0)\in\mathop{\bf dom} f\times\mathop{\bf dom} g$} and using} $\tau,\sigma,\theta,\mu_x$>0 that satisfy {\small
\begin{equation}
    \label{eq: SCSC SAPD LMI}
  \sa{G\triangleq}\begin{pmatrix}
    \frac{1}{\tau}(1-\frac{1}{\rho})+\frac{\mu_x}{\rho} & 0 & 0 & 0 & 0\\
  0 & \frac{1}{\sigma}(1-\frac{1}{\rho})+\mu_y & (\frac{\theta}{\rho} - 1)L_{yx} & (\frac{\theta}{\rho} - 1)L_{yy}  & 0\\
  0 & (\frac{\theta}{\rho} - 1)L_{yx} & \tfrac{1}{\tau} - \sa{\bar{L}_{xx}} & 0 & -  \frac{\theta}{\rho}L_{yx}\\
  0& (\frac{\theta}{\rho} - 1)L_{yy} & 0 & \frac{1}{\sigma} - \alpha & -  \frac{\theta}{\rho}L_{yy}\\
  0 & 0 & - \frac{\theta}{\rho}L_{yx} & -  \frac{\theta}{\rho}L_{yy}  & \frac{\alpha}{\rho}
\end{pmatrix}\succeq 0
\end{equation}}%
{for some} $\alpha \in [0,  {\tfrac{1}{\sigma}\sa{)}}$ and $\rho \in (0,1\sa{)}$, \sa{where $\bar{L}_{xx}\triangleq L_{xx}+\mu_x+\gamma$}.  Then for all $t\geq {0}$, \sa{it holds that}
\begin{equation}\label{eq:Gap bound with fg}
  \mathbb{E}\left[\mathcal{G}^{t}(x^{t+1}_{0}, y^{t+1}_{0})\right] \leq \frac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\left(
\frac{\mu_x}{4}\ED{x^{t}_{*}(y^{t+1}_{0}) - x^{t}_{0}} + \frac{\mu_y}{4}\ED{y_*(x^{t+1}_{0}) - y^{t}_{0}}
\right) + \Xi_{\tau,\sigma,\theta},
\end{equation}
where $K_{N_t}(\rho) = \sum_{k=0}^{N_t-1}\rho^{-k}$ and $M_{\tau,\sigma,\theta} \triangleq \max \{ \frac{4}{\mu_x\tau}, \frac{4+4\theta}{\mu_y\sigma}\}$,
\begin{align*}
    & \Xi_{\tau,\sigma,\theta} \triangleq \tau\left( \Xi^x_{\tau,\sigma,\theta} + \tfrac{1}{2}\right)\delta_x^2
     + \sigma\left( \Xi^y_{\tau,\sigma,\theta} +  \tfrac{1+2\theta}{2}\right)\delta_y^2,\\
     &\Xi^x_{\tau,\sigma,\theta} \triangleq \left( 1+ \tfrac{\sigma\theta(1+\theta)L_{yx}}{2}\right),\\
     &\Xi^y_{\tau,\sigma,\theta} \triangleq \left(1+3\theta + \sigma\theta(1+\theta)L_{yy}  +  \tau\sigma\theta(1+\theta)L_{yx}L_{xy} \right)(1+2\theta) + \tfrac{\tau\theta(1+\theta)L_{yx}}{2}.
\end{align*}
\end{lemma}
\begin{proof}
For easier readability, we provide the proof in a separate subsection, see~\cref{PF: lemma Gap bound with fg}.
\end{proof}
Next, the following lemma provides \sa{a relation between $\mathcal{G}^{t}(x^{t}_{0},y^{t}_{0})$ and $\mathcal{G}^{t}(x^{t+1}_{0},y^{t+1}_{0})$.}
\begin{lemma}
\sa{Under the premise of Lemma~\ref{lemma: Gap bound with fg}, for all $t\geq 0$,}
\label{Lemma: gap convergence}
$$\left(1-\frac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right)\mathcal{G}^{t}(x^{t+1}_{0}, y^{t+1}_{0}) \leq \frac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\mathcal{G}^{t}(x^{t}_{0},y^{t}_{0})  + \Xi_{\tau,\sigma,\theta}.
$$
\end{lemma}
\begin{proof}
\sa{It is shown in~\cite[Lemma 1]{yan2020optimal} that
$$\frac{\mu_x}{4}\|x^{t}_{*}(y) - \sa{x'} \|^{\sa{2}} + \frac{\mu_y}{4} \|y_*(x) - \sa{y'} \|^{\sa{2}}\leq \mathcal{G}^{t}(x,y) + \mathcal{G}^{t}(x',y')$$
holds {for all $(x,y),(x',y')\in \mathop{\bf dom} f \times \mathop{\bf dom} g$.}
\mg{Applying this inequality to the right-hand side of \eqref{eq:Gap bound with fg}
with the choice of $x = x_0^{t+1}$, $y=y_0^{t+1}$, $x' = x_0^t$, $y' = y_0^t$ yields the desired result}. \sa{It is important to note that since $\mathcal{L}^t$ is SCSC, given $(x_0^{t+1}, y_0^{t+1})$, $\mathcal{G}^{t}(x^{t+1}_0,y^{t+1}_0)$ is finite; hence, one can subtract $\frac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\mathcal{G}^{t}(x^{t+1}_{0}, y^{t+1}_{0})$ from both sides.}
}
\end{proof}

\sa{For the sake of completeness, we state \cite[Lemma 8]{yan2020optimal} below, which will be used in our analysis.}

\begin{lemma}
\label{lemma: different bound of gap}
\sa{\textbf{\cite[Lemma 8]{yan2020optimal}.}} Under the premise of \cref{lemma: Gap bound with fg}, for any $\beta_{1},\beta_{2}\in(0,1)$ and $t\geq 0$,
\begin{equation}
    \begin{aligned}
        &\mathcal{G}^{t}(x^{t+1}_{0},y^{t+1}_{0})\geq \left(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_1}-1\Big)\mathcal{G}_{t+1}(x^{t+1}_{0},y^{t+1}_{0})\right) - \frac{\gamma+\mu_x}{2}\frac{\beta_1}{1-\beta_1}\|x^{t+1}_{0}-x^{t}_{0}\|^2,\\
        &\mathcal{G}^{t}(x^{t+1}_{0},y^{t+1}_{0})\geq \phi(x^{t+1}_{0})-\phi(x^{t}_{0}) + \frac{\gamma+\mu_x}{2}\|x^{t+1}_{0}-x^{t}_{0}\|^2,\\
        &\mathcal{G}^{t}(x^{t+1}_{0},y^{t+1}_{0})\geq \frac{\gamma \beta_2}{2}\|x^{t}_{0}-x^{t}_{*}\|^2 - \frac{\gamma\beta_2}{2(1-\beta_2)}\|x^{t+1}_{0}-x^{t}_{0}\|^2,
        \end{aligned}
\end{equation}
\mg{where $x^{t}_{*}=\prox{\lambda\phi}(x^{t}_{0})$.}
\end{lemma}
\sa{Recall that we aim to control $x^{t}_{0} - \prox{\lambda\phi}(x^{t}_{0})$ as it directly determines $\nabla\phi_\lambda(x^{t}_{0})$, and we also have $\|x^{t}_{0} - \prox{\lambda\phi}(x^{t}_{0})\|=\|x^{t}_{0} - x^{t}_{*}\|$. Thus, in the following result, we bound $\mathbb{E}[\|x^{t}_{0} - x^{t}_{*}\|^2]$. Moreover, this result will also help us construct a telescoping sum for analyzing the convergence of $\{x_0^t\}_{t\geq 0}$ to a stationary point.}
\begin{lemma}
\label{Lemma: telescope sum chpt2}
\sa{Under the premise of Lemma~\ref{lemma: Gap bound with fg}, for any $\beta_{1},\beta_{2}\in(0,1)$, and $p_1,p_2,p_3>0$ such that $p_1+p_2+p_3=1$, it holds for all $t\geq 0$ that}
{\small
\begin{equation}
\label{eq: WCSC step inequality chpt2}
    \begin{aligned}
        &\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right)\frac{\gamma p_3\beta_2}{2}\ED{x^{t}_{0} - x^{t}_{*}}
        \\
        \leq & \tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\mathbb{E}\left[\mathcal{G}^{t}(x^{t}_{0},y^{t}_{0})\right] -
        \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right) p_1\Big(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\Big)
        \mathbb{E}\left[\mathcal{G}_{t+1}(x^{t+1}_{0},\sa{y^{t+1}_{0}}) \right]
        \\
        & + \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right)p_2\mathbb{E}\left[\phi(x^{t}_{0}) - \phi(x^{t+1}_{0})\right]
        \\
        & \sa{+\frac{1}{2}\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right)\left(p_1(\gamma+\mu_x) \frac{\beta_1}{1-\beta_{1}} - p_2(\gamma+\mu_x) + p_3\gamma\frac{\beta_2}{1-\beta_{2}}\right)}\ED{x^{t+1}_{0} - x^{t}_{0}} +\Xi_{\tau,\sigma,\theta}.
    \end{aligned}
\end{equation}}%
\end{lemma}
\begin{proof}
Using \cref{lemma: different bound of gap}
and
$\mathcal{G}^{t}(x^{t+1}_{0},y^{t+1}_{0}) = (p_1+p_2+p_3)\mathcal{G}^{t}(x^{t+1}_{0},y^{t+1}_{0})$
leads to
\begin{equation*}
    \begin{aligned}
    \mathbb{E}\left[\mathcal{G}^{t}(x^{t+1}_{0}, y^{t+1}_{0})\right]\geq & - \left(p_1\frac{\gamma+\mu_x}{2} \frac{\beta_1}{1-\beta_{1}} - p_2\frac{\gamma+\mu_x}{2} + p_3\frac{\gamma\beta_2}{2(1-\beta_2)}\right)\ED{x^{t+1}_{0} - x^{t}_{0}} \\
    & + p_1\left(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\right)\mathbb{E}\left[\mathcal{G}_{t+1}(x^{t+1}_{0},y^{t+1}_{0})\right]
        + p_2\mathbb{E}\left[\phi(x^{t+1}_{0}) - \phi(x^{t}_{0})\right] \\
        & + p_3\frac{\gamma\beta_{2}}{2}\ED{x^{t}_{0} - x^{t}_{*}}.
    \end{aligned}
\end{equation*}
Then, combining \cref{Lemma: gap convergence} yields the desired result.
\end{proof}

\sa{Finally, in the following result, we establish a preliminary convergence result for \xz{\texttt{SAPD+}} under compactness assumption stated in \cref{assump:compact}.}
\begin{theorem}
\label{thm:WC_SP}
Under the premise of Lemma~\ref{lemma: Gap bound with fg}, given $T\in\mathbb{Z}_+$, suppose $N_t=N$ for all $t=\sa{0},\ldots T$ for some \sa{$N\in\mathbb{Z}_+$} such that $K_{N}(\rho) \geq \sa{(1+\zeta)} M_{\tau,\sigma,\theta}$ for some $\zeta>0$, and the inequality system,
\begin{subequations}
\label{eq: additional stepsize condition WCSC}
\begin{align}
        & \tfrac{M_{\tau,\sigma,\theta}}{K_{N}(\rho)} -
        \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N}(\rho)}\right) p_1\Big(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\Big) \leq 0, \label{eq: additional stepsize condition WCSC-1}
        \\&
        p_1(\gamma+\mu_x)\frac{\beta_1}{1-\beta_{1}} - p_2(\gamma+\mu_x) + p_3\gamma\frac{\beta_2}{1-\beta_2} \leq 0,\label{eq: additional stepsize condition WCSC-2}
\end{align}
\end{subequations}
has a solution for some $\beta_{1},\beta_{2}\in(0,1)$ and $p_1,p_2,p_3>0$ such that $p_1+p_2+p_3=1$. Then, for $\lambda=(\gamma+\mu_x)^{-1}$, \sa{under \cref{assump:compact},} the following bound holds for all $T\geq 1$:
\begin{equation}\label{eq: sketch convergence result}
    \begin{aligned}
        \frac{1}{T+1}\sum_{t=0}^{T}\mathbb{E}\left[\|\nabla\phi_{\lambda}(x^{t}_{0}) \|^2 \right]
        \leq
        &       \frac{2(1+\zeta)(\gamma+\mu_x)^2}{\zeta\gamma p_3 \beta_2}\left(\frac{1}{T+1}
        \mathcal{G}(x_0^0,y_0^0)
        +\Xi_{\tau,\sigma,\theta} \right).
    \end{aligned}
\end{equation}
\end{theorem}
\begin{proof}
\sa{Since $\mathop{\bf dom} f$ and $\mathop{\bf dom} g$ are compact sets, $\mathbb{E}[\mathcal{G}^{t}(x_0^t,y_0^t)]\in\mathbb{R}$ exist for $t=0,\ldots,T$, i.e., $-\infty<\mathbb{E}[\mathcal{G}^{t}(x_0^t,y_0^t)]<\infty$ for all $t$. Therefore,} if we sum up equation \eqref{eq: WCSC step inequality chpt2} from $0$ to T,
we get
\begin{equation}\label{eq: proof step A}
    \begin{aligned}
        &\sum_{t=\sa{0}}^{T}\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right)
        \frac{\gamma p_3 \beta_2}{2}
       \mathbb{E}\left[\|x^{t}_{0} - x^{t}_{*} \|^2 \right]
        \\
        & \leq  \tfrac{M_{\tau,\sigma,\theta}}{K_{N_0}(\rho)}\mathbb{E}\left[\mathcal{G}^0(x_0^0,y_0^0)\right] -
        \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_T}(\rho)}\right) p_1\left(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\right)
        \mathbb{E}\left[\mathcal{G}^{T+1}(\sa{x^{T+1}_{0},y^{T+1}_{0}}) \right]
        \\
        &
        + \sum_{t=0}^{T-1}\left(\tfrac{M_{\tau,\sigma,\theta}}{K_{N_{t+1}}(\rho)} -
        \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right) p_1\left(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\right)\right)
        \mathbb{E}\left[\mathcal{G}^{t+1}(x^{\sa{t+1}}_{0},y^{\sa{t+1}}_{0})\right]
        \\
        & +\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_{\sa{0}}}(\rho)}\right)p_2\mathbb{E}\left[\phi(x^{\sa{0}}_{0})\right]  - \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_{\sa{T}}(\rho)}}\right)p_2\mathbb{E}\left[\phi(x^{\sa{T+1}}_{0})\right] + p_2\sum_{t=0}^{T-1}\underbrace{\left(\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)} -\tfrac{M_{\tau,\sigma,\theta}}{K_{N_{t+1}}(\rho)} \right)}_{\textbf{part 1}}\mathbb{E}\left[\phi(x^{\sa{t+1}}_{0}) \right]
        \\
        &
        +  \sum_{t=0}^T \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N_t}(\rho)}\right)\left(p_1\frac{\gamma+\mu_x}{2} \frac{\beta_{1}}{1-\beta_{1}} - p_2\frac{\gamma+\mu_x}{2}
        +
        p_3\gamma\frac{\beta_2}{2(1-\beta_2)}\right)\mathbb{E}\left[\|x^{t+1}_{0} - x^{t}_{0}\|^2\right]+(T+1)\Xi_{\tau,\sigma,\theta}
    \end{aligned}
\end{equation}
\sa{Thus, using $N_t=N$ for $t=0,\ldots,N$,} it follows from \sa{the conditions in} \eqref{eq: additional stepsize condition WCSC} that
\begin{equation}
    \begin{aligned}
        &\frac{1}{T+1}\sum_{t=0}^{T}\left(1-\sa{\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)}}\right) \frac{\gamma p_3 \beta_2}{2}\mathbb{E}\left[\|x^{t}_{0} - x^{t}_{*} \|^2 \right]
        \\
        \leq & \frac{1}{T+1}\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)}\mathbb{E}\left[\mathcal{G}^0(x_0^0,y_0^0) \right]-
       \frac{1}{T+1} \left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)}\right) p_1\left(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\right)
       \mathbb{E}\left[ \mathcal{G}^{T+1}(x^{T+1}_{0},\sa{y^{T+1}_{0}}) \right]
        \\
        &
        + \frac{p_2\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)}\right)}{T+1}\mathbb{E}\left[\phi(x^0_0)  - \phi(x^{\sa{T}+1}_{0})\right]
        +\Xi_{\tau,\sigma,\theta}
        \\
        \leq
        & \frac{1}{T+1} \tfrac{M_{\tau,\sigma,\theta}}{K_{N}(\rho)}\mathbb{E}\left[\mathcal{G}^0(x_0^0,y_0^0)\right]
         + \frac{p_2\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)}\right)}{T+1}\mathbb{E}\left[\mathcal{G}(x_0^0,y_0^0)\right]
        +\Xi_{\tau,\sigma,\theta},
    \end{aligned}
\end{equation}
\sa{which follows from $(i)$} $\mathcal{G}_{T+1}(x^{T+1}_{0},y^{T+1}_{0})\geq 0$, $(ii)$ $\phi(x^0_0)  - \phi(x^{T+1}_{0}) = \mathcal{L}(x^0_0, y_*(x^0_0)) -\mathcal{L}(x^{T+1}_{0}, y_*(x^{\sa{T}+1}_{0}))\leq \mathcal{L}(x^0_0, y_*(x^0_0)) -\mathcal{L}(x^{T+1}_{0}, y^0_0)\leq  \sa{\sup_{y'\in\mathcal{Y}}\mathcal{L}(x^0_0, y')} -\inf_{x'\in\mathcal{X}}\mathcal{L}(x', y^0_0)=\mathcal{G}(x^0_0,y^0_0)$, and also from the fact that \eqref{eq: additional stepsize condition WCSC-1} implies  $\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_{N}(\rho)}\right) p_1\Big(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\Big)\geq 0$. Then dividing both sides by $\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)}\right) \frac{\gamma p_3 \beta_2}{2}$ gives us
{\small
\begin{eqnarray}
        \lefteqn{\frac{1}{T+1}\sum_{t=1}^{T}\|x^{t}_{0} - x^{t}_{*} \|^2}\\
        && \leq \frac{2}{(1-\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)})\gamma p_3\beta_{2}}\Big(
        \frac{1}{T+1} \tfrac{M_{\tau,\sigma,\theta}}{K_{N}(\rho)}\mathbb{E}\left[\mathcal{G}^0(x_0^0,y_0^0)\right]+ \frac{p_2\left(1-\tfrac{M_{\tau,\sigma,\theta}}{K_N(\rho)}\right)}{T\sa{+1}}\mathbb{E}\left[\mathcal{G}(x_0^0,y_0^0)\right]
        +\Xi_{\tau,\sigma,\theta}\Big),\nonumber
        \\
        && \leq \frac{2(1+\zeta)}{\zeta\gamma p_3 \beta_2}\left(\frac{1}{T+1}\mathbb{E}\left[\mathcal{G}(x_0^0,y_0^0)\right]
        +\Xi_{\tau,\sigma,\theta} \right),\nonumber
\end{eqnarray}}%
where the second inequality
\sa{follows from} $\mathcal{G}(x_0^0,y_0^0)\geq \mathcal{G}^0(x_0^0,y_0^0)$, \sa{and for $p_2\in(0,1)$, we have} $K_N(\rho)\geq (1+\zeta)M_{\tau,\sigma,\theta}$. Then using Lemma \ref{Lemma: graident of ME} completes the proof.
\end{proof}
\subsection{Proof of Lemma~\ref{lemma: Gap bound with fg} and preliminary technical results}
\sa{Recall that given some {$x^t_0\in\mathop{\bf dom} f$} and $\mu_x>0$, we define
\begin{subequations}
\begin{align}
    \mathcal{L}^{t}(x,y)&\triangleq f(x)+\Phi^t(x,y)-g(y),\\
    \Phi^t(x,y) &\triangleq\Phi(x,y) + \frac{\mu_x+\gamma}{2}\|x-\xz{x^t_0}\|^2,\label{eq:Phi_t}
\end{align}
\end{subequations}
where $\gamma>0$ is the weak-convexity constant of $\Phi(\cdot,y)$ for any $y\in\mathop{\bf dom} \xz{g}$. It follows from \cref{ASPT: lipshiz gradient} that $\nabla_y \Phi^t$ and $\nabla_x \Phi^t$ are Lipschitz such that
\begin{align}
    \norm{\nabla_y \Phi^t(x,y)-\nabla_y \Phi^t(x',y')}\leq L_{yx}\norm{x-x'}+L_{yy}\norm{y-y'},\\
    \norm{\nabla_x \Phi^t(x,y)-\nabla_x \Phi^t(x',y')}\leq L'_{xx}\norm{x-x'}+L_{xy}\norm{y-y'}
\end{align}
such that $L'_{xx}\triangleq L_{xx}+\mu_x+\gamma$. Furthermore, \eqref{eq:Phi_t} implies that for any $y\in\mathop{\bf dom} \xz{g}$, $\Phi(\cdot,y)$ is strongly convex with modulus $\mu_x>0$.}%

We will derive some key inequalities below for \xz{\texttt{SAPD}} iterates \sa{$\{x_k^t,y_k^t\}_{k\geq 0}$} generated
by algorithm~\ref{ALG: SAPD} \sa{to solve $\min_x\max_y\mathcal{L}^{t}(x,y)$}. Let $x_{-1}^t = x_0^t$, $y_{-1}^t = y_0^t$, and for $k\geq 0$, define
{\small
\begin{align}
\label{eq:qksk}
{q}_k^t \triangleq \nabla_y \sa{\Phi^t}(x_k^t,y_k^t) - \nabla_y\sa{\Phi^t}(x_{k-1}^t,y_{k-1}^t),\qquad s^{t}_{k} \triangleq \nabla_y \sa{\Phi^t}(x_k^t, y_k^t) + \theta q_k^t.
\end{align}}
\sa{Thus $q_0^t=\mathbf{0}$; and for $k\geq 0$, \cref{ASPT: lipshiz gradient} implies that
{\small
\begin{equation}
           \norm{q_{k+1}^t} \leq  L_{yx} \|x_{k+1}^t - x_{k}^t \|+  L_{yy} \|y_{k+1}^t - y_{k}^t \|. \label{INEQ: Cauchy Ineqaulity 1}
\end{equation}}}%
\begin{lemma}\label{lem: basic lemma for SCSC}
\xz{Suppose \cref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption} hold.} Let $\{x_k^t,y_k^t\}_{k\geq 0}$ be \xz{\texttt{SAPD}} iterates generated according to algorithm~\ref{ALG: SAPD} \sa{for solving $\min_x\max_y\mathcal{L}^{t}(x,y)$}.
Then
for all $x\in\mathop{\bf dom} f\subset \mathcal{X}$, $y\in\mathop{\bf dom} g\subset \mathcal{Y}$, and $k\geq 0$,
{\small
\begin{equation}\label{D1}
    \begin{aligned}
       \mathcal{L}^{t}( & x^{t}_{k+1}, y)  - \mathcal{L}^{t}(x, y^{t}_{k+1})
        \\
    \leq
    &-\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle + \theta \langle q^{t}_{k}, y^{t}_{k} - y \rangle
    + \Lambda^{t}_k(x,y) - \Sigma^{t}_{k+1}(x,y)+ \Gamma^{t}_{k+1}+\sa{\varepsilon^{t,x}_{k}(x)+\varepsilon^{t,y}_{k}(y)},
\end{aligned}
\end{equation}}%
where $\sa{\varepsilon^{t,x}_k(x)}\triangleq
       \langle  \tilde{\nabla}_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1};\omega_k^x) - \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}),~x-x^{t}_{k+1} \rangle$, $\sa{\varepsilon^{t,y}_{k}(y)}\triangleq\langle \tilde{s}^{t}_{k} -s^{t}_{k}, y^{t}_{k+1} - y \rangle$, $q^{t}_{k}$ and $s^{t}_{k}$ are defined as in \eqref{eq:qksk}, and
{\small
\begin{align*}
\Lambda^{t}_{k}(x,y) &\triangleq  (\frac{1}{2\tau}-\frac{\mu_x}{2}) \|x - x^{t}_{k}\|^2 + \frac{1}{2\sigma}\|y-y^{t}_{k}\|^2,\\
\Sigma^{t}_{k+1}(x,y) &\triangleq  \frac{1}{2\tau} \|x - x^{t}_{k+1}\|^2 + (\frac{1}{2\sigma} + \frac{\mu_y}{2})\|y-y^{t}_{k+1}\|^2,\\ \Gamma^{t}_{k+1} &\triangleq
     (\frac{ L'_{xx} }{2} - \frac{1}{2\tau}) \| x^{t}_{k+1} - x^{t}_{k} \|^2 - \frac{1}{2\sigma}\|y^{t}_{k+1}- y^{t}_{k}\|^2 \\ &\quad + \theta L_{yx} \|x^{t}_{k} - x^{t}_{k-1} \|\| y^{t}_{k+1} - y^{t}_{k} \| +   \theta   L_{yy} \|y^{t}_{k} - y^{t}_{k-1} \| \| y^{t}_{k+1} - y^{t}_{k} \|.
\end{align*}}%
\end{lemma}
\begin{proof}
Fix $x\in\mathop{\bf dom} f$ and $y\in\mathop{\bf dom} g$.
Using Lemma 7.1 from \cite{hamedani2018primal} for the $y-$ and $x-$subproblems in algorithm~\ref{ALG: SAPD}, we get
{\footnotesize
    $f(x^{t}_{k+1}) + \langle \tilde{\nabla}_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1};\omega^x_k) , x^{t}_{k+1} - x \rangle
       \leq
       f(x) + \frac{1}{2\tau} \left[  \|x- x^{t}_{k}\|^2 -\|x- x^{t}_{k+1}\|^2 - \|x^{t}_{k+1}- x^{t}_{k}\|^2
       \right]$,
    $g(y^{t}_{k+1}) - \langle \tilde{s}^{t}_k,
       y^{t}_{k+1} - y \rangle
       \leq
       g(y) + \frac{1}{2\sigma} \left[  \|y-y^{t}_{k}\|^2- \|y -  y^{t}_{k+1}\|^2 -
      \|y^{t}_{k+1}- y^{t}_{k}\|^2
       \right]$}.
Thus, we get
{\footnotesize
\begin{subequations}
\begin{equation} \label{IX}
    \begin{aligned}
 f(x^{t}_{k+1}) &+ \langle {\nabla}_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}
 ),~x^{t}_{k+1} - x \rangle\\
       \leq &  f(x) + \tfrac{1}{2\tau} (  \|x- x^{t}_{k}\|^2 - \|x- x^{t}_{k+1}\|^2 - \|x^{t}_{k+1}- x^{t}_{k}\|^2
       )+\varepsilon^{t,x}_k(x),
    \end{aligned}
\end{equation}
\begin{equation}\label{IY1}
    \begin{aligned}
    g(y^{t}_{k+1})  &- \langle s^{t}_{k},
       y^{t}_{k+1} - y \rangle
       \\
       \leq & g(y) + \tfrac{1}{2\sigma} (  \|y- y^{t}_{k}\|^2 - \|y- y^{t}_{k+1}\|^2 -
       \|y^{t}_{k+1}- y^{t}_{k}\|^2
       )+\varepsilon^{t,y}_k(y).
    \end{aligned}
\end{equation}
\end{subequations}}
Rearranging the terms in \eqref{IY1}, we get
{\footnotesize
\begin{equation}\label{IY2}
\begin{aligned}
    -g(y)&+ g(y^{t}_{k+1})
       \\
       \leq &\langle s^{t}_{k},  y^{t}_{k+1} - y \rangle + \tfrac{1}{2\sigma} \left[  \|y-y^{t}_{k}\|^2- \|y -  y^{t}_{k+1}\|^2 -
      \|y^{t}_{k+1}- y^{t}_{k}\|^2
       \right] +\varepsilon^{t,y}_k(y).
\end{aligned}
\end{equation}}
Since $y^{t}_{k+1}\in \mathop{\bf dom} g$, the inner product in (\ref{IX}) can be lower bounded using convexity of $\Phi^{t}(\cdot, y^{t}_{k+1})$ in \cref{ASPT: lipshiz gradient} as follows:
{\footnotesize
\begin{eqnarray*}
    \lefteqn{\langle \nabla_x  \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}), x^{t}_{k+1} - x \rangle
     =
    \langle \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}), x^{t}_{k} - x \rangle
    +
    \langle \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}), x^{t}_{k+1} - x^{t}_{k} \rangle}
    \\
    &&\geq
    \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}) - \Phi^{t}(x, y^{t}_{k+1})  +\frac{\mu_x}{2}\|x-x^{t}_{k}\|^2
    + \langle \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}),~x^{t}_{k+1} - x^{t}_{k} \rangle.
\end{eqnarray*}}
Using this inequality after adding $\Phi^{t}(x^{t}_{k+1}, y^{t}_{k+1})$ to both sides of \eqref{IX}, we get
{\footnotesize
\begin{equation}
\begin{aligned}
   \Phi^{t}( & x^{t}_{k+1}, y^{t}_{k+1}) + f(x^{t}_{k+1}) \\
    \leq &  \Phi^{t}(x, y^{t}_{k+1}) + f(x)
     + \Phi^{t}(x^{t}_{k+1}, y^{t}_{k+1}) - \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}) - \langle \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}), x^{t}_{k+1} - x^{t}_{k} \rangle
   \\
   &
   +  \tfrac{1}{2\tau} \left[ \| x - x^{t}_{k} \| ^ 2 - \| x - x^{t}_{k+1}  \| ^ 2 -
       \|x^{t}_{k+1}- x^{t}_{k}\|^2\right] - \tfrac{\mu_x}{2}\|x - x^{t}_{k}\|^2+\varepsilon^{t,x}_k(x)
   \\
   \leq & \Phi^{t}(x, y^{t}_{k+1}) +  f(x) + \frac{L'_{xx}}{2} \| x^{t}_{k+1} - x^{t}_{k} \|^2
   \\
   &    +  \tfrac{1}{2\tau} \left[ \| x - x^{t}_{k} \| ^ 2 - \|x-x^{t}_{k+1}\| ^ 2 -
       \|x^{t}_{k+1}- x^{t}_{k}\|^2\right]- \tfrac{\mu_x}{2}\|x - x^{t}_{k}\|^2+\sa{\varepsilon^{t,x}_k(x)},
\end{aligned}
\end{equation}}
where the last step uses
\cref{ASPT: lipshiz gradient}. Rearranging the terms gives us
{\footnotesize
\begin{equation}\label{IX2}
\begin{aligned}
     f(&x^{t}_{k+1}) - f(x) - \Phi^{t}(x,y^{t}_{k+1})
    \leq -\Phi^{t}(x^{t}_{k+1},y^{t}_{k+1}) + \frac{L'_{xx}}{2} \| x^{t}_{k+1} - x^{t}_{k} \|^2
    \\
    &    +  \frac{1}{\sa{2}\tau} \left[ \| x - x^{t}_{k} \| ^ 2 - \| x - x^{t}_{k+1}  \| ^ 2 -
       \|x^{t}_{k+1}- x^{t}_{k}\|^2\right]- \frac{\mu_x}{2}\|x - x^{t}_{k}\|^2+\sa{\varepsilon^{t,x}_k(x)}.
\end{aligned}
\end{equation}}
Then, for $k \geq 0$, by summing \eqref{IY2} and \eqref{IX2}, we obtain
{\footnotesize
\begin{equation}
\label{eq:one-step-aux1}
\begin{aligned}
   \mathcal{L}( & x^{t}_{k+1}, y)  - \mathcal{L}(x, y^{t}_{k+1})
   =
   f(x^{t}_{k+1}) + \Phi^{t}(x^{t}_{k+1},y) - g(y) - f(x) - \Phi^{t}(x,y^{t}_{k+1}) + g(y^{t}_{k+1})
   \\
   \leq & \Phi^{t}(x^{t}_{k+1}, y) - \Phi^{t}(x^{t}_{k+1}, y_{k+1}) + \langle s^{t}_{k}, y^{t}_{k+1} - y \rangle + \frac{L'_{xx}}{2} \| x^{t}_{k+1} - x^{t}_{k} \|^2
   \\
   &  +  \frac{1}{2\sigma} \left[  \|y-y^{t}_{k}\|^2- \|y -  y^{t}_{k+1}\|^2 -
      \|y^{t}_{k+1}- y^{t}_{k}\|^2 \right]+ \sa{\varepsilon_k^{t,y}(y)}
   \\
   & +  \frac{1}{2\tau} \left[ \| x - x^{t}_{k} \| ^ 2 - \| x - x^{t}_{k+1}\| ^ 2 -
       \|x^{t}_{k+1}- x^{t}_{k}\|^2\right]
       - \frac{\mu_x}{2}\|x - x^{t}_{k}\|^2 + \sa{\varepsilon_k^{t,x}(x)}.
\end{aligned}
\end{equation}}
From \cref{ASPT: lipshiz gradient},
the $\mu_y$-strongly concavity of $\Phi^t(x, \cdot)$ for fixed $x \in \mathop{\bf dom} f\subset \mathcal{X}$ implies
{\footnotesize
\begin{equation*}
    \begin{aligned}
         \Phi^{t}( & x^{t}_{k+1}, y) -  \Phi^{t}(x^{t}_{k+1}, y^{t}_{k+1}) + \langle s^{t}_{k}, y^{t}_{k+1} - y \rangle
         \\
         \leq & \langle \nabla_y\Phi^{t}(x^{t}_{k+1},y^{t}_{k+1}), y - y^{t}_{k+1} \rangle - \frac{\mu_y}{2}\|y - y^{t}_{k+1}\|^2 + \langle \nabla_y \Phi^{t}(x^{t}_{k}, y^{t}_{k}) + \theta q^{t}_{k}, y^{t}_{k+1} - y \rangle
         \\
         = & -\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle - \frac{\mu_y}{2}\|y - y^{t}_{k+1}\|^2 + \theta \langle q^{t}_{k}, y^{t}_{k} - y \rangle +\theta \langle q^{t}_{k}, y^{t}_{k+1}-y^{t}_{k}\rangle.
    \end{aligned}
\end{equation*}}
Thus, using the above inequality within~\eqref{eq:one-step-aux1}, we get
{\footnotesize
\begin{equation*}
\begin{aligned}
    \mathcal{L}^{t}( & x^{t}_{k+1}, y)  - \mathcal{L}^{t}(x, y^{t}_{k+1}) \leq
    -\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle + \theta \langle q^{t}_{k}, y^{t}_{k} - y+y^{t}_{k+1}-y^{t}_{k} \rangle
    + \frac{L'_{xx}}{2} \| x^{t}_{k+1} - x^{t}_{k} \|^2
    \\
    & +  \frac{1}{2\sigma} \left[  \|y-y^{t}_{k}\|^2- \|y -  y^{t}_{k+1}\|^2 -
      \|y^{t}_{k+1}- y^{t}_{k}\|^2
       \right]- \frac{\mu_y}{2}\|y - y^{t}_{k+1}\|^2
   \\
   & +  \frac{1}{2\tau} \left[ \| x - x^{t}_{k} \| ^ 2 - \| x - x^{t}_{k+1}  \| ^ 2 -
       \|x^{t}_{k+1}- x^{t}_{k}\|^2\right]
       - \frac{\mu_x}{2}\|x - x^{t}_{k}\|^2+\varepsilon^{t,x}_k(x)+\sa{\varepsilon^{t,y}_k(y).}
\end{aligned}
\end{equation*}}
Finally, \eqref{D1} follows from using Cauchy-Schwarz
for $\fprod{q_k,y_{k+1}-y_k}$ and \eqref{INEQ: Cauchy Ineqaulity 1}.
\end{proof}
\begin{lemma}\label{lemma: proximal ineq}
\cite[Theorem 6.42]{beck2017first} Let $f$ be proper, closed and convex function. Then for any {$x,x'\in \mathcal{X}$},
we get
{\small
$
\|\prox{f}(x) - \prox{f}(x') \| \leq \|  x- x' \|.
$}
\end{lemma}
Next, based on the above inequality, we prove an intermediate result, \sa{which we use later} to bound the variance of the \xz{\texttt{SAPD}} iterate sequence.

\begin{lemma}\label{lemma: intermediate noisy bound}
\xz{Suppose \cref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption} hold.} Let $\{x_k^t,y_k^t\}_{k\geq 0}$ be \xz{\texttt{SAPD}}  iterates generated according to algorithm~\ref{ALG: SAPD} \sa{for solving $\min_x\max_y\mathcal{L}^{t}(x,y)$}.
\sa{For $k\geq 0$, let $q^{t}_{k}$ and $s^{t}_{k}$ be defined as in \eqref{eq:qksk}, and
let}
{\footnotesize
\begin{equation*}
    \begin{aligned}
       &{\hat{x}^{t}_{k+1}} \triangleq   \prox{\tau f}\left({x^{t}_{k}- \tau{\nabla}_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}})\right), \quad
       \hat{y}^{t}_{k+1} \triangleq  \prox{\sigma g}\left(y^{t}_{k} + \sigma s^{t}_{k}\right),\\
       &{\hat{\hat{x}}^{t}_{k+1}  \triangleq   \prox{\tau f}\left({x^{t}_{k}- \tau{\nabla}_x \Phi^{t}(x^{t}_{k}, \hat{y}^{t}_{k+1}})\right)}, \quad \hat{\hat{y}}^{t}_{k+1}\triangleq \prox{\sigma \sa{g}}\left( \hat{y}^{t}_{k} + \sigma(1+\theta){\nabla}_y \Phi^{t}(\hat{\hat{x}}^{t}_{k}, \hat{y}^{t}_{k}) - \sigma\theta{\nabla}_y \Phi^{t}(x^{t}_{k-1}, y^{t}_{k-1})  \right),
    \end{aligned}
\end{equation*}}%
then the following inequalities hold for $k\geq 0$:
{\footnotesize
\begin{subequations}
\begin{align}
    \|x^{t}_{k+1} - \hat{x}^{t}_{k+1} \| &\leq \tau\| \Delta_k^{\Phi, x}\|
    \qquad \|y^{t}_{k+1} - \hat{y}^{t}_{k+1}\| \leq \sigma\left((1+\theta)\|\Delta^{\Phi^t,y}_k\| + \theta\|\Delta^{\Phi^t,y}_{k-1}\|\right),\label{eq:y_k - y_hat_k}\\
    \|{y}^{t}_{k+1} - \hat{\hat{y}}^{t}_{k+1}\| &\leq  \sigma\left(
    (1+\theta)\|\Delta^{\Phi^t,y}_k\| + \theta\|\Delta^{\Phi^t,y}_{k-1}\|
    +
   \tau(1+\theta)L_{yx}\|\Delta^{\Phi^t,x}_{k-1}\|\right)
    \label{eq:y_k - y_hat_hat_k}\\
    &
    \quad + \sigma\left(1 + \sigma(1+\theta)L_{yy} +  \tau\sigma(1+\theta)L_{yx}L_{xy} \right)\left((1+\theta)\|\Delta^{\Phi^t,y}_{k-1}\| + \theta\|\Delta^{\Phi^t,y}_{k-2}\|\right), \nonumber
    \end{align}
\end{subequations}}%
where
{\small
    $\Delta^{\Phi^t,x}_k \sa{\triangleq} \tilde{\nabla}_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1};\omega_k^x) - \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1})$, and  $\Delta^{\Phi^t,y}_k \sa{\triangleq} \sa{\tilde\nabla_y}\Phi^{t}(x^{t}_{k}, y^{t}_{k};\omega_k^y) - \nabla_y \Phi^{t}(x^{t}_{k}, y^{t}_{k})$.
}
\end{lemma}
\begin{proof}
The first inequality in~\cref{eq:y_k - y_hat_k} is
from \cref{lemma: proximal ineq}; for the second, we have
{\footnotesize
\begin{equation*}
\begin{aligned}
    \|y^{t}_{k+1} - \hat{y}^{t}_{k+1}\| \leq \sigma\|\tilde{s}^{t}_{k} - s^{t}_{k} \| \leq \sigma\left((1+\theta)\|\Delta^{\Phi^t,y}_k\| + \theta\|\Delta^{\Phi^t,y}_{k-1}\|\right),
\end{aligned}
\end{equation*}}%
\sa{which follows from \cref{lemma: proximal ineq} and the
triangle inequality.
To show \cref{eq:y_k - y_hat_hat_k},
we bound $\| y^{t}_{k+1} - \hat{y}^{t}_{k+1}\|$ and $\| \hat{y}^{t}_{k+1}- \hat{\hat{y}}^{t}_{k+1}\|$ separately.} It follows from \cref{lemma: proximal ineq} that
    $\|x^{t}_{k+1} - \hat{\hat{x}}^{t}_{k+1} \| \leq \tau\|\tilde{\nabla}_x\Phi^{t}(x^{t}_{k},y^{t}_{k+1};\omega_k^x) - \nabla_x\Phi^{t}(x^{t}_{k},\hat{y}_{k+1})\|$.
\sa{After adding and subtracting $\nabla_x\Phi^{t}(x^{t}_{k},y^{t}_{k+1})$, \cref{ASPT: lipshiz gradient} implies that}
{\small
\begin{equation}\label{ineq: x_k - x_hat_hat_k}
    \begin{aligned}
    \|x^{t}_{k+1} - \hat{\hat{x}}^{t}_{k+1} \| \leq \tau\left(\|\Delta_k^{\Phi,x}\| + L_{xy}\|y^{t}_{k+1} - \hat{y}^{t}_{k+1}\|\right).
    \end{aligned}
\end{equation}}
\sa{We will use this relation to bound $\|\hat{y}^{t}_{k+1} - \hat{\hat{y}}^{t}_{k+1}\|$.
Indeed,} using \cref{lemma: proximal ineq}, we have
{\footnotesize
\begin{align*}
    \|\hat{y}^{t}_{k+1} - \hat{\hat{y}}^{t}_{k+1}\| \leq & \|y^{t}_{k} - \hat{y}^{t}_{k} + \sigma(1+\theta) \left( \nabla_y\Phi^{t}(x^{t}_{k},y^{t}_{k}) - \nabla_y\Phi^{t}(\hat{\hat{x}}^{t}_{k},\hat{y}^{t}_{k}) \right) \|\\
    \leq &(1 + \sigma(1+\theta)L_{yy}) \|y^{t}_{k} - \hat{y}^{t}_{k}\| + \sigma(1+\theta)L_{yx}\|x^{t}_{k} - \hat{\hat{x}}^{t}_{k} \|\\
    \leq & \Big(1 + \sigma(1+\theta)L_{yy} + \tau\sigma(1+\theta)L_{yx}L_{xy}\Big) \|y^{t}_{k} - \hat{y}^{t}_{k}\|
    + \tau\sigma(1+\theta)L_{yx}\| \Delta_{k-1}^{\Phi, x}\|\\
    \leq & \sigma\left(1 + \sigma(1+\theta)L_{yy} +  \tau\sigma(1+\theta)L_{yx}L_{xy} \right) \cdot \left((1+\theta)\|\Delta^{\Phi^t,y}_{k-1}\| + \theta\|\Delta^{\Phi^t,y}_{k-2}\|\right)\\
    &
    + \sigma\tau(1+\theta)L_{yx}\|\Delta^{\Phi^t,x}_{k-1}\|,
\end{align*}}%
\sa{where the second, third and fourth inequalities follow from \cref{ASPT: lipshiz gradient}, \cref{ineq: x_k - x_hat_hat_k} and the second inequality in~\cref{eq:y_k - y_hat_k}, respectively.  Combining this
with
    $\| y^{t}_{k+1} - \hat{\hat{y}}^{t}_{k+1}\|
    \leq \| y^{t}_{k+1} - \hat{y}^{t}_{k+1}\| + \| \hat{y}^{t}_{k+1}- \hat{\hat{y}}^{t}_{k+1}\|$,
and the second inequality in~\cref{eq:y_k - y_hat_k}
give us the desired bound.}
\end{proof}

\sa{Next, we provide some inequalities to bound the \xz{\texttt{SAPD}} variance term later} in our analysis.

\begin{lemma}\label{Lemma:noisy bound with fg}
\xz{Suppose \cref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption} hold.} Let $\{x_k^t,y_k^t\}_{k\geq 0}$ be \xz{\texttt{SAPD}} iterates generated according to algorithm~\ref{ALG: SAPD} for solving $\min_x\max_y\mathcal{L}^{t}(x,y)$.
The following inequality holds for all $k\geq 0$:
{\footnotesize
\begin{eqnarray*}
\lefteqn{\mathbb{E}\left[\langle \Delta^{\Phi^t,x}_k,  \hat{x}^{t}_{k+1}-x^{t}_{k+1} \rangle\right] \leq \tau\delta_x^2,\qquad
     \mathbb{E}\left[\langle \Delta^{\Phi^t,y}_k,  y^{t}_{k+1} - \hat{y}^{t}_{k+1}\rangle\right] \leq \sigma(1+2\theta)\delta_y^2,}\\
    \lefteqn{\mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_{k-1},  \hat{\hat{y}}^{t}_{k+1}-y^{t}_{k+1} \rangle \right]}\\
    &&\leq\sigma \left[  \left( \left(2 + \sigma(1+\theta)L_{yy}  +  \tau\sigma(1+\theta)L_{yx}L_{xy} \right)\cdot
    (1+2\theta)
    + \frac{\tau(1+\theta)L_{yx}}{
     \sa{2}}\right)\delta_y^2 + \frac{\tau(1+\theta)L_{yx}}{
     \sa{2}}\delta_x^2\right].
\end{eqnarray*}}%
\end{lemma}
\begin{proof}
With the convention that $y^{t}_{-2} = y^{t}_{-1} = y^{t}_0$, and $x^{t}_{-2} = x^{t}_{-1} = x^{t}_0$, \cref{lemma: intermediate noisy bound} and Cauchy-Schwarz inequality
imply for all $k\geq 0$ that
{\footnotesize
\begin{equation*}
    \begin{aligned}
    & \langle \Delta^{\Phi^t,x}_k, x^{t}_{k+1} - \hat{x}^{t}_{k+1} \rangle \leq \tau\|\Delta^{\Phi^t,x}_k \|^2, \\
    & \langle \Delta^{\Phi^t,y}_k,  y^{t}_{k+1} - \hat{y}^{t}_{k+1}\rangle \leq \sigma\left((1+\theta)\|\Delta^{\Phi^t,y}_k\|^2 + \theta\|\Delta^{\Phi^t,y}_{k-1}\|\|\Delta^{\Phi^t,y}_k\|\right),\\
    &  \langle \Delta^{\Phi^t,y}_{k-1},  y^{t}_{k+1} - \hat{\hat{y}}^{t}_{k+1}\rangle
    \leq
    \sigma\Bigg(
    (1+\theta)\|\Delta^{\Phi^t,y}_k\|\|\Delta^{\Phi^t,y}_{k-1}\|
    +
    \theta\|\Delta^{\Phi^t,y}_{k-1}\|^2+
    \tau(1+\theta)L_{yx}\|\Delta^{\Phi^t,x}_{k-1}\|
    \|\Delta^{\Phi^t,y}_{k-1}\|\\
    &
    +
    \Big(1 + \sigma(1+\theta)L_{yy} +  \tau\sigma(1+\theta)L_{yx}L_{xy} \Big)\cdot
    \Big((1+\theta)\|\Delta^{\Phi^t,y}_{k-1}\|^2
    +
    \theta\|\Delta^{\Phi^t,y}_{k-2}\|\|\Delta^{\Phi^t,y}_{k-1}\|\Big)
    \Bigg).
    \end{aligned}
\end{equation*}}%
Next, using \cref{ASPT: unbiased noise assumption} and
$\| a\|\|b\|\leq  \frac{1}{2}\| b\|^2
 + \frac{1}{2}\| b\|^2$,
which holds for $a,b \in \mathbb{R}^n$, and taking the expectation leads to the desired result.
\end{proof}
Before we move on to prove our intermediate result in~\cref{lemma: Gap bound with fg}, we give two technical lemmas that help us simplify the \xz{\texttt{SAPD}} parameter selection rule \sa{and lead to} the LMI in \eqref{eq: SCSC SAPD LMI}.
\begin{lemma}
\label{lem:equivalent_systems}
Given $\tau,\sigma>0$, $\theta,\alpha\geq 0$, and $\rho\in(0,1)$, let
{\scriptsize
\begin{equation}
\label{eq:G-def}
G\triangleq
  \begin{pmatrix}
    \frac{1}{\tau}(1-\frac{1}{\rho})+\frac{\mu_x}{\rho} & 0 & 0 & 0 & 0\\
  0 & \frac{1}{\sigma}(1-\frac{1}{\rho})+\mu_y & (\frac{\theta}{\rho} - 1)L_{yx} & (\frac{\theta}{\rho} - 1)L_{yy}  & 0\\
  0 & (\frac{\theta}{\rho} - 1)L_{yx} & \tfrac{1}{\tau} - \sa{L'_{xx}} & 0 & -  \frac{\theta}{\rho}L_{yx}\\
  0& (\frac{\theta}{\rho} - 1)L_{yy} & 0 & \frac{1}{\sigma} - \alpha & -  \frac{\theta}{\rho}L_{yy}\\
  0 & 0 & - \frac{\theta}{\rho}L_{yx} & -  \frac{\theta}{\rho}L_{yy}  & \frac{\alpha}{\rho}
\end{pmatrix},
\end{equation}}%
{\scriptsize
\begin{equation}
\label{eq:Gp-def}
 \sa{G'} \triangleq
 \begin{pmatrix}
    \frac{1}{\tau}(1-\frac{1}{\rho})+\frac{\mu_x}{\rho} & 0 & 0 & 0 & 0\\
  0 & \frac{1}{\sigma}(1-\frac{1}{\rho})+\mu_y & \sa{-|1-\frac{\theta}{\rho}|}~L_{yx} & \sa{-|1-\frac{\theta}{\rho}|}~L_{yy} & 0\\
  0 & \sa{-|1-\frac{\theta}{\rho}|}~L_{yx} & \tfrac{1}{\tau} - \sa{L'_{xx}} & 0 & -  \frac{\theta}{\rho}L_{yx}\\
  0& \sa{-|1-\frac{\theta}{\rho}|}~L_{yy} & 0 & \frac{1}{\sigma} - \alpha & -  \frac{\theta}{\rho}L_{yy}\\
  0 & 0 & - \frac{\theta}{\rho}L_{yx} & -  \frac{\theta}{\rho}L_{yy} & \frac{\alpha}{\rho}
\end{pmatrix},
\end{equation}}%
then \sa{$G\succeq 0$ if and only if $G'\succeq 0$}.
\end{lemma}
\begin{proof}
$\forall ~ \mathbf{y} = (y_1,y_2,y_3,y_4,y_5)^\top\in \mathbb{R}^5$, letting $\tilde{\mathbf{y}}= (y_1,-y_2,y_3,y_4,y_5)^\top$, we have
{\footnotesize
\begin{equation*}
    \mathbf{y}^\top G' \mathbf{y} =
    \begin{cases}
    \mathbf{y}^\top \sa{G} \mathbf{y} & \text{if} ~\sa{\theta\leq \rho,}\\
    \tilde{\mathbf{y}}^\top G \tilde{\mathbf{y}} & \text{else;}
    \end{cases}\quad
    \mathbf{y}^\top G \mathbf{y} =
    \begin{cases}
    \mathbf{y}^\top G' \mathbf{y} & \text{if} ~\sa{\theta\leq \rho},\\
    \tilde{\mathbf{y}}^\top G' \tilde{\mathbf{y}} & \text{else.}
    \end{cases}
\end{equation*}}%
\sa{Thus, $G\succeq 0$ is equivalent to $G'\succeq 0$.}
\end{proof}
\begin{lemma}\label{lemma: sub positive matrix}
\sa{Given $\tau,\sigma>0$, $\theta,\alpha\geq 0$, and $\rho\in(0,1)$, consider $G$ defined in \eqref{eq:G-def}. If $G\succeq 0$, then $G''\succeq 0$, where}
{\footnotesize
\begin{equation}
    \label{eq: sub matrix of general SAPD LMI}
  \sa{G''} \triangleq \begin{pmatrix}
   \frac{1}{\sigma}(1-\frac{1}{\rho})+\mu_y +\frac{\alpha}{\rho} & (-| 1 - \tfrac{\theta}{\rho} | - \tfrac{\theta}{\rho})L_{yx} & (-| 1 - \tfrac{\theta}{\rho} | - \tfrac{\theta}{\rho})L_{yy} \\
 (-| 1 - \tfrac{\theta}{\rho} | - \tfrac{\theta}{\rho}) L_{yx} & \tfrac{1}{\tau} - \sa{L'_{xx}} & 0 \\
 (-| 1 - \tfrac{\theta}{\rho} | - \tfrac{\theta}{\rho})L_{yy} & 0 & \frac{1}{\sigma} - \alpha \\
\end{pmatrix}\sa{\succeq 0}.
\end{equation}}%
\end{lemma}
\begin{proof}
\sa{Note that}
$\mathbf{x}}  %{\mbox{\boldmath $\lambda$}^\top
G''
\mathbf{x}}  %{\mbox{\boldmath $\lambda$}
=
{\mathbf{x}}  %{\mbox{\boldmath $\lambda$}'}^\top
G'
\mathbf{x}}  %{\mbox{\boldmath $\lambda$}'\geq 0$ for all $\mathbf{x}}  %{\mbox{\boldmath $\lambda$}=[x_1~x_2~x_3]^\top\in \mathbb{R}^3$, where  $\mathbf{x}}  %{\mbox{\boldmath $\lambda$}'=[0~x_1~x_2~x_3~x_1]^\top$ and $G'$ is defined in \eqref{eq:Gp-def}. Then the desired result follows from \cref{lem:equivalent_systems}.
\end{proof}

\sa{Finally, with the following observation, we will be ready to proceed to the proof of \cref{lemma: Gap bound with fg}. Let $\{\mathcal{F}_k^{t,x}\}$ and $\{\mathcal{F}_k^{t,y}\}$ be the filtrations such that $\mathcal{F}^{t,x}_k \triangleq \mathcal{F}(\{x^t_i\}_{i=0}^{k},\{y^t_i\}_{i=0}^{k+1})$ and $\mathcal{F}^{t,y}_k \triangleq \mathcal{F}(\{x^t_i\}_{i=0}^{k},\{y^t_i\}_{i=0}^{k})$ denote the $\sigma$-algebras generated by the random variables in their arguments.}
A consequence of Assumption~\ref{ASPT: unbiased noise assumption} is that
\sa{for $\mathcal{F}^{t,x}_k$-measurable random variable $v$, i.e.,} $v\in\mathcal{F}^{t,x}_k$, we have that
$\mathbb{E}\left[\langle \tilde{\nabla}\Phi_x(x^{t}_{k},y^{t}_{k+1};\omega_k^x) - \nabla\Phi_x(x^{t}_{k},y^{t}_{k+1}),v \rangle \right]= 0$; similarly,
for \sa{$v\in\mathcal{F}^{t,y}_k$}, it holds that
$\mathbb{E}\left[\langle \tilde{\nabla}\Phi_y(x^{t}_{k},y^{t}_{k};\omega_k^y) - \nabla\Phi_y(x^{t}_{k},y^{t}_{k}),v \rangle \right]= 0$.

\subsubsection{Proof of Lemma~\ref{lemma: Gap bound with fg}}\label{PF: lemma Gap bound with fg}
\begin{proof}
Fix arbitrary $(x,y)\in\mathop{\bf dom} f\times \mathop{\bf dom} g$. Since $(x^{t}_{k+1},y^{t}_{k+1})\in\mathop{\bf dom} f\times \mathop{\bf dom} g$, using the concavity
of $\mathcal{L}^{t}(x^{t}_{k+1}, \cdot)$
and the convexity
of $\mathcal{L}^{t}(\cdot, y^{t}_{k+1})$, Jensen's lemma immediately implies that
{\small
\begin{align}
\label{eq:jensen _bounded version}
 K_{N}(\rho)  \left(\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}^{t}(x, \bar{y}^{t}_{N}) \right)
            \leq
            \sum_{k=0}^{N-1}\rho^{-k} \left( \mathcal{L}^{t}(x^{t}_{k+1}, y) - \mathcal{L}^{t}(x, y^{t}_{k+1}) \right),
            \;\sa{\forall}\rho\in \sa{(0,1]},
\end{align}}%
where $\bar{x}^{t}_N=\frac{1}{K_N(\rho)}\sum_{k=0}^{N-1}\sa{\rho^{-k}}x^{t}_{k+1},~\bar{y}^{t}_N=\frac{1}{K_N(\rho)}\sum_{k=0}^{N-1}\sa{\rho^{-k}}y^{t}_{k+1}$. Thus, if we multiply both sides of
\eqref{D1} \sa{by $\rho^{-k}$} and \sa{sum the resulting inequality from $k=0$ to $N-1$, then using \eqref{eq:jensen _bounded version} we get}
{\footnotesize
\begin{equation}\label{INEQ: difference of gap  _bounded version}
    \begin{aligned}
            K_{N} &(\rho)  \left(\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}(x, \bar{y}^{t}_{N}) \right)  \\
            \leq &
             \sum_{k=0}^{N-1}\rho^{-k} \Big(
                \underbrace{ -\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle + \theta \langle q^{t}_{k}, y^{t}_{k} - y \rangle}_{\text{\bf part 1}} + \Lambda^{t}_{k} \sa{(x,y)} - \Sigma^{t}_{k+1} \sa{(x,y)}+ \Gamma^{t}_{k+1}
            \\
             & \quad  \underbrace{-
           \langle  \tilde{\nabla}_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1};\omega_k^x) - \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}) , x^{t}_{k+1} - x \rangle}_{\text{\bf part 2} }
            +
           \underbrace{\langle \tilde{s}^{t}_k -s^{t}_k,
          y^{t}_{k+1} - y \rangle}_{ \text{\bf part 3} }
          \Big).
    \end{aligned}
\end{equation}}%
\sa{Using Cauchy–Schwarz inequality and \eqref{INEQ: Cauchy Ineqaulity 1}} leads to
{\small
\begin{equation}\label{def: Q_k}
    |\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle|\leq  \sa{S^{t}_{k+1}} \triangleq L_{yx}\|x^{t}_{k+1}-x^{t}_{k}\| \|y^{t}_{k+1}-y\|+L_{yy}\|y^{t}_{k+1}-y^{t}_{k}\| \|y^{t}_{k+1}-y\|
\end{equation}}%
for $k \geq -1$.
Recall $x^{t}_{-1} = x^{t}_0$ and $y^{t}_{-1} = y^{t}_0$, thus \sa{$q_0^t=\mathbf{0}$; therefore, for \sa{\bf part\;1},}
{\footnotesize
\begin{align}
\label{eq:noisy-rate-part1}
         \sum_{k=0}^{N-1}&\rho^{-k}
                 ( \theta \langle q^{t}_k, y^{t}_{k} - y \rangle-\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle )
                 = \sum_{k=0}^{N-2}\rho^{-k}\Big(\frac{\theta}{\rho}-1\Big)\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle - \rho^{-N+1}\langle q^{t}_{N}, y^{t}_{N} - y \rangle
                 \nonumber
                 \\
                 \leq & \sum_{k=0}^{N-2}\rho^{-k}\sa{|1-\frac{\theta}{\rho}|~S^{t}_{k+1}} + \rho^{-N+1}\sa{S^{t}_{N}}
                 =  \sum_{k=0}^{N-1}\rho^{-k}\sa{|1-\frac{\theta}{\rho}|~S^{t}_{k+1}}
                 + \rho^{-N+1}(1-|1-\frac{\theta}{\rho}|)\sa{S^{t}_{N}},
\end{align}}%
where \sa{the first inequality follows from
\cref{def: Q_k}.}
Next we consider \textbf{part 2},
\sa{let $\Delta_k^{\Phi,x}$ be defined as in
\cref{lemma: intermediate noisy bound}.} \sa{For some arbitrary $\eta_x> 0$}, define \sa{$\{\tilde{x}_k\}$} sequence as follows:
{\small
         $\tilde{x}_0 \triangleq x^{t}_0$,
         and
         $\tilde{x}_{k+1} \triangleq \argmin_{x'\in \sa{\mathcal{X}}}  - \langle \Delta^{\Phi^t,x}_k , x' \rangle + \frac{\sa{\eta_x}}{2}\| x' - \tilde{x}_k\|^2,\quad \forall~k\geq 0$.}
Then by \sa{\cite[Lemma 2.1]{nemirovski2009robust}}, \sa{for all $k\geq 0$ and {$x\in \mathcal{X}$}}, we have that
       $\langle \Delta^{\Phi^t,x}_k , x -  \tilde{x}_{k} \rangle   \leq \frac{\eta_x}{2}\| x - \tilde{x}_{k}\|^2 -\frac{\eta_x}{2}\| x - \tilde{x}_{k+1}\|^2  +\frac{1}{2\eta_x}\| \Delta^{\Phi^t,x}_k\|^2$.
\sa{Thus, using $\tilde{x}_0 = x^{t}_{0}$ we get}
{\footnotesize
\begin{equation}\label{INEQ: bound of x - tilde x }
    \begin{aligned}
           \sum_{k=0}^{N-1} &\rho^{-k} \langle \Delta^{\Phi^t,x}_k , x -  \tilde{x}_{k} \rangle   \leq
           \sum_{k=0}^{N-1}\rho^{-k} \Big(\frac{\eta_x}{2}\| x - \tilde{x}_{k}\|^2 -\frac{\eta_x}{2}\| x - \tilde{x}_{k+1}\|^2  +\frac{1}{2\eta_x}\| \Delta^{\Phi^t,x}_k\|^2\Big) \\
            = & \frac{\eta_x}{2}(\| x - {{x}^{t}_{0}}\|^2 -\rho^{-N+1}\| x - \tilde{x}_{N}\|^2)+ \sum_{k=0}^{N-1}\frac{\rho^{-k}}{2\eta_x} \| \Delta^{\Phi^t,x}_k\|^2 - \sum_{k=1}^{N-1}\frac{\eta_x}{2}\sa{\rho^{-k+1}}(1-\rho)\| x - \tilde{x}_{k}\|^2
            \\
            \leq & {\frac{\eta_x}{2}} \sa{\norm{x-x^{t}_{0}}^2}
            + {\frac{1}{2\eta_x}}\sum_{k=0}^{N-1}\rho^{-k}\| \Delta^{\Phi^t,x}_k\|^2,
     \end{aligned}
\end{equation}}
where
\sa{for the last inequality
we use $0<\rho\leq 1$;
hence,} \textbf{part 2} becomes
{\footnotesize
\begin{equation}\label{eq:noisy-rate-part2}
    \begin{aligned}
            & \sum_{k=0}^{N-1} -\rho^{-k} \langle \Delta^{\Phi^t,x}_k , x^{t}_{k+1} -  x \rangle\\
            =&    \sum_{k=0}^{N-1} \rho^{-k}
            \Big(\langle \Delta^{\Phi^t,x}_k , {\hat{x}_{k+1}}^{t}-x^{t}_{k+1} \rangle  - \langle \Delta^{\Phi^t,x}_k , {\hat{x}^{t}_{k+1}} -  \tilde{x}_k \rangle
            + \langle \Delta^{\Phi^t,x}_k , x -\tilde{x}_k   \rangle\Big)
            \\
            \leq&   \sum_{k=0}^{N-1} \rho^{-k}
            \Big(\langle \Delta^{\Phi^t,x}_k , {\hat{x}^{t}_{k+1}}-x^{t}_{k+1} \rangle  - \langle \Delta^{\Phi^t,x}_k , {\hat{x}^{t}_{k+1}} -  \tilde{x}_k \rangle
           +\frac{1}{2\sa{\eta_x}}\| \Delta^{\Phi^t,x}_k\|^2\Big) + \frac{\sa{\eta_x}}{2}\sa{\norm{x-x^{t}_0}^2},
    \end{aligned}
\end{equation}}%
\sa{which follows} from \eqref{INEQ: bound of x - tilde x }, and $\hat{x}_{k+1}$ is defined in \cref{lemma: intermediate noisy bound}. {When $\delta_x=0$, clearly $\Delta^{\Phi^t,x}_k=\mathbf{0}$; thus, \textbf{part 2} is equal to $0$ and we can set $\eta_x=0$ for which \eqref{eq:noisy-rate-part2}
becomes $0\leq 0$. Next, we consider \textbf{part 3}, let $\Delta_k^{\Phi^t,y}$ be defined as in
\cref{lemma: intermediate noisy bound}.}
{For some arbitrary $\eta_y> 0$, we construct two auxiliary sequences:
let $\tilde{y}_0^+  = \tilde{y}_0^- = y^{t}_{0}$, and we define}
{\small
           $\tilde{y}^+_{k+1} \triangleq \argmin_{y'\in \mathcal{Y}}   \langle \Delta^{\Phi^t,y}_k , y' \rangle + \frac{\sa{\eta_y}}{2}\| y' - \tilde{y}^+_k\|^2$, and
           $\tilde{y}^-_{k+1} \triangleq \argmin_{y'\in \mathcal{Y}}   -\langle \Delta^{\Phi^t,y}_k , y' \rangle + \frac{\sa{\eta_y}}{2}\| y' - \tilde{y}^-_k\|^2$,}
{for $k\geq 0$. Thus, it follows from \cite[Lemma 2.1]{nemirovski2009robust} that for $y\in \mathcal{Y}$,}
   $\langle \Delta^{\Phi^t,y}_k ,   \tilde{y}_{k}^+ - y  \rangle   \leq \frac{\eta_y}{2}\| y - \tilde{y}_{k}^+\|^2 -\frac{\eta_y}{2}\| y - \tilde{y}_{k+1}^+\|^2  +\frac{1}{2\eta_y}\| \Delta^{\Phi^t,y}_k\|^2$,
   and
       $\langle \Delta^{\Phi^t,y}_k ,   y-\tilde{y}_{k}^-  \rangle   \leq \frac{\eta_y}{2}\| y - \tilde{y}_{k}^-\|^2 -\frac{\eta_y}{2}\| y - \tilde{y}_{k+1}^-\|^2  +\frac{1}{2\eta_y}\| \Delta^{\Phi^t,y}_k\|^2$.
\sa{Therefore,
as in~\eqref{INEQ: bound of x - tilde x },
we get\footnote{\sa{As in~\textbf{part 2}, when $\delta_y=0$, we can set $\eta_y=0$.}}
{\footnotesize
\begin{equation}\label{INEQ:bound of y-tilde y}
\begin{aligned}
            &\sum_{k=0}^{N-1} \rho^{-k} \Big[ (1+\theta)\langle \Delta^{\Phi^t,y}_k , \tilde{y}_{k}^+ -  y \rangle+ \theta \langle \Delta^{\Phi^t,y}_{k-1} ,y-\tilde{y}_{k-1}^-   \rangle \Big]
            \\
              \leq&
            \frac{\eta_y}{2}(1+2\theta) \sa{\norm{y-y^{t}_{0}}^2}
            + \frac{1}{2\eta_y}\sum_{k=0}^{N-1}\rho^{-k}\Big( (1+\theta)\| \Delta^{\Phi^t,y}_k\|^2 + \theta\| \Delta^{\Phi^t,y}_{k-1}\|^2\Big).
\end{aligned}
\end{equation}}}%
Next, using \cref{INEQ:bound of y-tilde y}, we can bound \textbf{part 3} as follows:
{\footnotesize
\begin{equation}\label{eq:noisy-rate-part3}
    \begin{aligned}
        &\sum_{k=0}^{N-1} \rho^{-k} \langle \tilde{s}^{t}_{k} -s^{t}_{k},
       y^{t}_{k+1} - y \rangle
       \\
      = &
         \sum_{k=0}^{N-1} \rho^{-k}\Big[(1 +\theta) \langle \Delta^{\Phi^t,y}_k,
       y^{t}_{k+1} - {\hat{y}^{t}_{k+1}}  + {\hat{y}^{t}_{k+1}} - \tilde{y}_k^+ + \tilde{y}_k^+ - y \rangle
       - \theta \langle \Delta^{\Phi^t,y}_{k-1},
       y^{t}_{k+1} - {\hat{\hat{y}}^{t}_{k+1}} + {\hat{\hat{y}}^{t}_{k+1}}  - \tilde{y}_{k-1}^- + \tilde{y}_{k-1}^- - y \rangle\Big]
       \\
       \leq &
        \sum_{k=0}^{N-1} \rho^{-k}\Big[(1 +\theta) \langle \Delta^{\Phi^t,y}_k,
       y^{t}_{k+1} -{\hat{y}^{t}_{k+1}} + {\hat{y}^{t}_{k+1}} - \tilde{y}_k^+  \rangle
       - \theta \langle \Delta^{\Phi^t,y}_{k-1},
       y^{t}_{k+1} - {\hat{\hat{y}}^{t}_{k+1}} + {\hat{\hat{y}}^{t}_{k+1}} - \tilde{y}_{k-1}^-  \rangle \Big]
       \\
       & + \frac{1}{2{\eta_y}}\sum_{k=0}^{N-1} \rho^{-k}\Big((1+\theta)\| \Delta^{\Phi^t,y}_k\|^2 +  \theta\| \Delta^{\Phi^t,y}_{k-1}\|^2 \Big) +\frac{\sa{\eta_y}}{2}(1+2\theta) \sa{\norm{y-y^t_0}^2},
    \end{aligned}
\end{equation}}%
where $\hat{y}^{t}_{k+1}$ and $\hat{\hat{y}}^{t}_{k+1}$ are defined in \cref{lemma: intermediate noisy bound}.
\sa{Let $\Delta^{t}_N(x,y)\triangleq\frac{1}{2\rho}(\frac{1}{\tau}-\mu_x) \|x^{t}_{N}-x\|^2 +  \frac{1}{2}{\left(\frac{1}{\rho\sigma} - \frac{\alpha}{\rho} \right)}\|y^{t}_{N}-y\|^2$;}
adding $\rho^{-N+1}\Delta^{t}_N(x,y)$ to both sides of \eqref{INEQ: difference of gap  _bounded version}, then using \eqref{eq:noisy-rate-part1}, \eqref{eq:noisy-rate-part2} and \eqref{eq:noisy-rate-part3},
{for any fixed $(x,y)\in \mathop{\bf dom} f\times \mathop{\bf dom} g$, we get}
{\footnotesize
\begin{equation}
\label{INEQ: ergordic gap  bounded version}
\begin{aligned}
        K_{N} ({\rho}) & \left(\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}^{t}(x, \bar{y}^{t}_{N}) \right)
        + {\rho}^{-N+1}\Delta^{t}_N(x,y)\\
        &\leq
      U^{t}_N(x,y)   + \frac{\sa{\eta_x}}{2} \norm{x-x^{t}_0}^2
      +\frac{\sa{\eta_y}}{2}(1+2\theta)\sa{\norm{y-y^{t}_0}^2}
       + \sum_{k=0}^{N-1}{\rho}^{-k}(P^{t}_{k}+ Q^{t}_{k}),
\end{aligned}
\end{equation}}%
where \sa{{$U^{t}_N(x,y)$}, $P^{t}_{k}$ and $Q^{t}_{k}$ for $k=0,\ldots,N-1$ are defined as}
{\scriptsize
\begin{subequations}
\label{eq:UPQ}
\begin{align}
    \sa{U^{t}_N(x,y)}   \triangleq
      &  {\sum_{k=0}^{N-1}{\rho}^{-k}
      \left(\Gamma^{t}_{k+1} + \Lambda^{t}_k{(x,y)} - \Sigma^{t}_{k+1} \sa{(x,y)} + {\sa{|1-\frac{\theta}{\rho}|~S^{t}_{k+1}}} \right)
      - {\rho}^{-N+1}( - \Delta^{t}_N(x,y) - {\frac{\theta}{\rho}\sa{S^{t}_{N}}}),}
      \\
      P^{t}_k  \triangleq
      & -\langle \Delta^{\Phi^t,x}_k , {\hat{x}^{t}_{k+1}} -  \tilde{x}_k \rangle
     +    (1+\theta)\langle \Delta^{\Phi^t,y}_k, {\hat{y}^{t}_{k+1}} - \sa{\tilde{y}_k^+}  \rangle
     -\theta \langle \Delta^{\Phi^t,y}_{k-1}, {\hat{\hat{y}}^{t}_{k+1}} - \sa{\tilde{y}_{k-1}^-}  \rangle,
     \\
     Q^{t}_k  \triangleq &
     \langle \Delta^{\Phi^t,x}_k , {\hat{x}^{t}_{k+1} -  x^{t}_{k+1}} \rangle
     +
     (1+\theta)\langle \Delta^{\Phi^t,y}_k, {y^{t}_{k+1} - \hat{y}^{t}_{k+1}}  \rangle
     -\theta \langle \Delta^{\Phi^t,y}_{k-1}, {y^{t}_{k+1} - \hat{\hat{y}}^{t}_{k+1}}  \rangle
     \nonumber\\
     & +
     \frac{1}{2{\eta_x}}\|\Delta^{\Phi^t,x}_k \|^2
     +
     \frac{1+\theta}{2{\eta_y}}\| \Delta^{\Phi^t,y}_k\|^2
     +
     \frac{\theta}{2{\eta_y}}\|\Delta^{\Phi^t,y}_{k-1}\|^2.
     \label{eq:Qk-def}
    \end{align}
\end{subequations}}%
{For any fixed $(x,y)\in \mathcal{X} \times \mathcal{Y}$}, we first analyze $U^{t}_N(x,y)$. For any $\alpha> 0$, after adding and subtracting $\tfrac{\alpha}{2}\|y^{t}_{k+1}-y^{t}_{k}\|^2$,  and rearranging the terms, we get
{\footnotesize
\begin{equation}
\label{eq:U_def _bounded version}
    \begin{aligned}
           U^{t}_N(x,y) =  &
           \frac{1}{2}\sum_{k=0}^{N-1}{\rho}^{-k}\Big(\xi_k^\top A\xi_k-\xi_{k+1}^\top B \xi_{k+1}\Big)
           -{\rho}^{-N+1}( - \Delta^{t}_N(x,y) - \frac{\theta}{\rho}\sa{S^{t}_{N}})\\
           =  &
           \frac{1}{2}\xi_0^\top A\xi_0- \frac{1}{2}\sum_{k=1}^{N-1}{\rho}^{-k+1}[\xi_k^\top{( B
           - \tfrac{1}{\rho}A)}\xi_k]
           -\rho^{-N + 1 }( \frac{1}{2}\xi_N^\top   B \xi_N  - \Delta^{t}_N(x,y) -\frac{\theta}{\rho}S^{t}_{N}),
    \end{aligned}
\end{equation}}%
\sa{
where $A, B \in\mathbb{R}^{5\times 5}$
and $\xi_k\in\mathbb{R}^5$ are defined for $k\geq 0$ as follows:}

{\scriptsize
$\xi_k \triangleq \left(
    \begin{array}{*{20}{c}}
  \|x^{t}_{k} - x\| \\
  \|y^{t}_{k} - y\| \\
  \|x^{t}_{k} - x^{t}_{k-1}\| \\
  \|y^{t}_{k} - y^{t}_{k-1}\| \\
  \|y^{t}_{k+1} - y^{t}_{k}\|
   \end{array}
   \right)$ {\normalsize such that $x^{t}_{-1} = x^{t}_{0}$, $y^{t}_{-1} = y^{t}_{0}$, and}
$A \triangleq
  \begin{pmatrix}
    \frac{1}{\tau}-\mu_x & 0 & 0 & 0 & 0\\
  0 & \frac{1}{\sigma} & 0 & 0 & 0\\
  0 & 0 & 0 & 0 & {\theta L_{yx}}\\
  0& 0 & 0 & 0 & {\theta L_{yy}}\\
  0 & 0 & \theta L_{yx} & \theta L_{yy} & -\alpha
\end{pmatrix}$,
{\normalsize and}

   $B \triangleq
  \begin{pmatrix}
    \frac{1}{\tau} & 0 & 0 & 0 & 0\\
  0 & \frac{1}{\sigma}+\mu_y & \sa{-|1-\frac{\theta}{\rho}|}~L_{yx} & \sa{-|1-\frac{\theta}{\rho}|}~L_{yy} & 0\\
  0 & \sa{-|1-\frac{\theta}{\rho}|}~L_{yx} & \tfrac{1}{\tau} - L'_{xx} & 0 & 0\\
  0& \sa{-|1-\frac{\theta}{\rho}|}~L_{yy} & 0 & \frac{1}{\sigma} - \alpha & 0\\
  0 & 0 & 0 & 0 & 0
\end{pmatrix}$.}
\sa{In Lemma~\ref{lem:equivalent_systems} we show that \cref{eq: SCSC SAPD LMI}
is equivalent to $B -\tfrac{1}{\rho}A\succeq 0$;} therefore, it follows from \eqref{eq:U_def _bounded version} that
    $U^{t}_N(x,y) \leq  \sa{\frac{1}{2}\xi_0^\top A\xi_0}- {\rho}^{-N + 1 } ( \tfrac{1}{2}\xi_N^\top  B \xi_N  -  \Delta^{t}_N(x,y) -\frac{\theta}{\rho}\sa{S^{t}_{N}})$
holds w.p. 1 for {any given $(x,y)\in \mathcal{X}\times \mathcal{Y}$.} Furthermore, recall that  $\Delta^{t}_N(x,y)=\frac{1}{2\rho}(\frac{1}{\tau}-\mu_x) \|x^{t}_{N}-x\|^2 +  \frac{1}{2}{\left(\frac{1}{\rho\sigma} - \frac{\alpha}{\rho} \right)}\|y^{t}_{N}-y\|^2$; thus, we have
{\footnotesize
$\frac{1}{2}\xi_N^\top    B  \xi_N  -\Delta^{t}_N(x,y)-\frac{\theta}{\rho}\sa{S^{t}_{N}}=
\frac{1}{2}\xi_N^\top
     \begin{pmatrix}
       \sa{\frac{1}{\tau}(1-\frac{1}{\rho})}+\frac{\mu_x}{\rho}
       & \mathbf{0}_{1\times3} & 0\\
       \mathbf{0}_{3\times 1} & \sa{G''} & \mathbf{0}_{3\times 1} \\
       0 & \mathbf{0}_{1\times3} & 0\\
     \end{pmatrix}
\xi_N
  \geq
0,$}
which
follows from \cref{eq: SCSC SAPD LMI,lemma: sub positive matrix}, and \sa{$G''$}, defined in \cref{eq: sub matrix of general SAPD LMI}.
Finally,
$\frac{1}{2}\xi_0^\top A\xi_0\sa{\leq} \frac{1}{2}(\frac{1}{\tau}-\mu_x)\norm{x-x^{t}_{0}}^2+\frac{1}{2\sigma}\norm{y-y^{t}_{0}}^2$. \xzd{Thus, for any $(x,y)\in\mathcal{X}\times \mathcal{Y}$,}
\begin{align}
\label{eq:U-bound}
    U^{t}_N(x,y)\leq \frac{1}{2}\Big(\frac{1}{\tau}-\mu_x\Big)\norm{x-x^{t}_{0}}^2+\frac{1}{2\sigma}\norm{y-y^{t}_{0}}^2,\quad \text{w.p.}~1.
\end{align}%

Now, we are ready to show \cref{eq:Gap bound with fg}. \xz{It follows from \cref{INEQ: ergordic gap  bounded version} and the above analysis that}
{\footnotesize
\begin{equation}\label{eq:gap_bound_fg_2}
\begin{aligned}
K_{N}& ({\rho})   \sup_{(x,y)\in\mathcal{X}\times\mathcal{Y}}\left\{\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}^{t}(x, \bar{y}^{t}_{N}) \right\} +  \xz{{\rho}^{-N+1}\Delta^{t}_N(x^{t}_*(\bar{y}^{t}_N),y_{*}(\bar{x}^{t}_{N}))} \\
&\leq
      \left(\frac{1}{2\tau}+\frac{\eta_x}{2}\right)\|x^{t}_*(\bar{y}^{t}_N)-x^{t}_{0}\|^2
      +\left(\frac{1}{2\sigma}+\frac{\eta_y(1+2\theta)}{2}\right)\|y_{*}(\bar{x}^{t}_{N})-y^{t}_{0}\|^2
       +\sum_{k=0}^{N-1}{\rho}^{-k}(P^{t}_k+ Q^{t}_k).
\end{aligned}
\end{equation}}%

From \cref{ASPT: unbiased noise assumption}, for
$k\geq-1$, we have
$\mathbb{E}\left[ \langle \Delta^{\Phi^t,x}_k , {\hat{x}^{t}_{k+1}} -  \tilde{x}_k \rangle \right] = 0$,
$\mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_k, {\hat{y}^{t}_{k+1}} - \tilde{y}_k^{\pm}  \rangle \right] = 0$,
$\mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_{k-1},{\hat{\hat{y}}^{t}_{k+1}} - \tilde{y}_{k-1}^{-}  \rangle \right] = 0$.
Thus,
$\mathbb{E}[P^{t}_k]=0$.
Also, from \cref{ASPT: unbiased noise assumption}, for $k\geq -1$, we have
$
\mathbb{E}\left[ \|\Delta^{\Phi^t,x}_k\|^2 \right] \leq \delta_x^2,\quad \mathbb{E}\left[ \|\Delta^{\Phi^t,y}_k\|^2 \right] \leq \delta_y^2.
$
Next, we uniformly upper bound $\mathbb{E}\left[ Q^{t}_k \right]$ for $k\geq 0$ using \cref{Lemma:noisy bound with fg}, i.e.,
     $\mathbb{E}[\sum_{k=0}^{N-1}{\rho}^{-k} Q^{t}_k]  \leq
     \Big[\Big(\tau\Xi_{\tau,\sigma,\theta}^x + \tfrac{1}{2\eta_x}\Big)\delta_x^2  + \Big(     \sigma\Xi_{\tau,\sigma,\theta}^y +  \sa{\tfrac{1+2\theta}{2\eta_y}}\Big)\delta_y^2\Big] \sum_{k=0}^{N-1}{\rho}^{-k}$.
Therefore, combining this result with $\mathbb{E}[P^{t}_k]=0$ for any $k\in\{0,\ldots, N-1\}$, we get
{\small
\begin{equation}
\label{eq:variance-bound}
   \mathbb{E}[ \sum_{k=0}^{N-1}{\rho}^{-k}(P^{t}_k+ Q^{t}_k)]\leq K_N({\rho})~\Xi_{\tau,\sigma,\theta}.
\end{equation}}
Finally, because \xz{$\Delta^{t}_N(x^{t}_*(\bar{y}^{t}_N),y_{*}(\bar{x}^{t}_{N}))\geq0$}, setting $\eta_x=\frac{1}{\tau}$, $\eta_y=\frac{1}{\sigma}$, $x^{t+1}_{0}=\bar{x}^{t}_{N}$, and $y^{t+1}_{0}=\bar{y}^{t}_{N}$, the desired result in~\eqref{eq:Gap bound with fg} follows from \eqref{eq:gap_bound_fg_2} and \eqref{eq:variance-bound}.
\end{proof}
\section{Proof of Theorem~\ref{cor:complexity-fg} and preliminary technical results}
We employ the
matrix inequality~(MI) in~\cref{eq: SCSC SAPD LMI} to describe the admissible set of algorithm parameters that guarantee convergence of inner loop \xz{of} \cref{ALG: SAPD}.
We  compute a particular solution
 \sa{by exploiting
 the structure of MI in~\cref{eq: SCSC SAPD LMI}.}

First, in \cref{LEMMA: Noise LMI after young's ineq}, we give an intermediate condition to help us construct the particular solution subsequently provided in \cref{Coroallary: explicit solution to noisy LMI} for \sa{solving the SCSC subproblems in \cref{eq: SCSC problem chpt2}.}
\begin{lemma}\label{LEMMA: Noise LMI after young's ineq}
For any $\mu_x>0$, let $L'_{xx}= L_{xx}+\gamma+\mu_x$. Suppose $\rho = \theta$, and $\tau, \sigma>0$, $\theta\in(0,1\sa{)}$ \sa{satisfy}
{\small
\begin{equation}
\label{eq:sufficient_cond_noisy_LMI}
\tau \geq \frac{1-\theta}{\mu_x}, \quad \sigma\geq \frac{1-\theta}{\mu_y\theta},\quad \frac{1}{\tau} \geq \xz{L'_{xx}} + \pi_1  L_{yx},
\quad \frac{1}{\sigma} \geq \frac{ \theta L_{yx}}{\pi_1} + (\frac{\theta}{\pi_2} +\pi_2)  L_{yy},
\end{equation}}
for some $\pi_1,\pi_2>0$.
Then $\{\tau, \sigma,\theta,\alpha\}$ is a solution to \eqref{eq: SCSC SAPD LMI} for $\alpha = \frac{\theta L_{yx}}{\pi_1} + \frac{\theta L_{yy}}{\pi_2}$.
\end{lemma}
\begin{proof}
\sa{It follows from the choice of $\tau$ and $\sigma$ in \eqref{eq:sufficient_cond_noisy_LMI} and $\theta=\rho$ that a sufficient condition for \eqref{eq: SCSC SAPD LMI} is given by the following smaller matrix inequality for $\alpha = \frac{\theta L_{yx}}{\pi_1} + \frac{\theta L_{yy}}{\pi_2}$,}
{\footnotesize
\begin{equation*}
\begin{aligned}
\sa{\mathbf{0}\preceq}
\begin{pmatrix}
  \tfrac{1}{\tau} - \xz{L'_{xx}} & 0 & - L_{yx}\\
 0 &\tfrac{1}{\sigma}- \alpha & - L_{yy}\\
 - L_{yx} & - L_{yy} & \tfrac{\alpha}{ \theta}
\end{pmatrix}
= &
    \begin{pmatrix}
  \tfrac{1}{\tau} - \xz{L'_{xx}} & 0 & - L_{yx}\\
 0 & \tfrac{1}{\sigma}-  \tfrac{\theta L_{yx}}{\pi_1} - \tfrac{\theta L_{yy}}{\pi_2} & - L_{yy}\\
 - L_{yx} & - L_{yy} & \tfrac{L_{yx}}{\pi_1} + \tfrac{ L_{yy}}{\pi_2}
\end{pmatrix}
\triangleq &
M_1 + M_2,
\end{aligned}
\end{equation*}}
where
${\small M_1
 \triangleq
 \begin{pmatrix}
\tfrac{1}{\tau} - \xz{L'_{xx}} & 0 & - L_{yx}\\
 0 & 0 & 0 \\
 - L_{yx} & 0 & \tfrac{L_{yx}}{\pi_1}
\end{pmatrix}}$ and
${\small
M_2 \triangleq
    \begin{pmatrix}
  0 & 0 & 0\\
 0 & \tfrac{1}{\sigma}-  \tfrac{\theta L_{yx}}{\pi_1} - \tfrac{\theta L_{yy}}{\pi_2} & - L_{yy}\\
 0 & - L_{yy} & \tfrac{ L_{yy}}{\pi_2}
\end{pmatrix}}$.
Therefore, since $\pi_1,\pi_2>0$, the Schur complement conditions in~\eqref{eq:sufficient_cond_noisy_LMI}, i.e., \sa{the third and the fourth} inequalities,
 imply $M_1
 \succeq 0$ and $M_2 \succeq 0$, respectively.
Thus, $M_1+M_2\succeq 0$.
\end{proof}

Lemma~\ref{LEMMA: Noise LMI after young's ineq} shows
 that every solution to
\eqref{eq:sufficient_cond_noisy_LMI} can be converted to a solution to
\eqref{eq: SCSC SAPD LMI}. Next, based on
this lemma, we will give an explicit parameter choice for algorithm~\ref{ALG: SAPD}.
\begin{lemma}\label{Coroallary: explicit solution to noisy LMI}
\sa{For any $\mu_x>0$, let} $\xz{L'_{xx}=} L_{xx}+\gamma+\mu_x$. If $L_{yy}>0$, then for any given $\beta\in(0,1]$, let $\tau, \sigma>0$ and $\theta\in (0,1)$ be chosen satisfying
{\small
\begin{equation}
\label{Condition: SP solution to noisy LMI}
\tau = \frac{1-\theta}{\mu_x },\quad \sigma = \frac{1-\theta}{\mu_y\theta},
\quad
\theta  \geq \bar{\theta}(\beta),
\end{equation}}%
\sa{where $\bar{\theta}(\beta)$ is defined in \cref{cor:complexity-fg}.}
Then, $\{\tau, \sigma,\theta,\alpha,\xz{\rho}\}$ \sa{with $\alpha = \frac{1}{\sigma}-\sqrt{\theta}L_{yy}>0$}, $\xz{\rho=\theta}$ is a solution to \eqref{eq: SCSC SAPD LMI}.
\end{lemma}
\begin{proof}
Consider arbitrary $\tau,\sigma,\pi_1,\pi_2>0$ and $\theta\in(0,1)$. By a straightforward
calculation,
$\{\tau, \sigma,\theta,\pi_1,\pi_2\}$ is a solution to \eqref{eq:sufficient_cond_noisy_LMI}
if and only if
{\small
\begin{subequations}
\label{Condition: SP solution sufficient}
\begin{gather}
        \tau\geq \frac{1-\theta}{\mu_x},\quad \sigma\geq \frac{1-\theta}{\theta\mu_y},\quad
    \pi_1 \geq  \frac{\sigma\theta L_{yx}}{1- \sigma(\pi_2+\frac{\theta}{\pi_2})L_{yy}},\label{Condition: SP solution sufficient 123}\\
    \sigma(\pi_2+\frac{\theta}{\pi_2})L_{yy} < 1,\quad
    \frac{1}{\tau}-\xz{L'_{xx}} \geq\pi_1 L_{yx}. \label{Condition: SP solution sufficient 4}
\end{gather}
\end{subequations}}
In the remainder of the proof, we fix $(\pi_1,\pi_2)$ as follows:
{\small
\begin{align}
        \pi_1 =   \frac{\sigma\theta L_{yx}}{1-\sigma\left(\pi_2+\frac{\theta}{\pi_2}\right)L_{yy}},\quad \pi_2 = \sqrt{\theta}.
        \label{Condition: SP solution to noisy LMI 3}
\end{align}}
Note the definition of $\xz{\bar{\theta}(\beta)}$ implies that $\xz{\bar{\theta}(\beta)}\in (0,1)$. Next, we show that $\theta\in [\xz{\bar{\theta}(\beta)},1)$ implies $\pi_1,\pi_2>0$; furthermore, we also show that $\tau,\sigma>0$ defined as in \eqref{Condition: SP solution to noisy LMI} for $\theta\in[\xz{\bar{\theta}(\beta)}, 1)$ together with $(\pi_1, \pi_2)$ as in \eqref{Condition: SP solution to noisy LMI 3} is a solution to \eqref{Condition: SP solution sufficient}.

First, setting $\tau,\sigma$ as in~\eqref{Condition: SP solution to noisy LMI} and $\pi_1,\pi_2$ as in~\cref{Condition: SP solution to noisy LMI 3} imply that \eqref{Condition: SP solution sufficient 123}
is trivially satisfied. Next, by substituting $\{\tau,\sigma,\pi_1,\pi_2\}$, chosen as in \eqref{Condition: SP solution to noisy LMI} and \cref{Condition: SP solution to noisy LMI 3}, into \eqref{Condition: SP solution sufficient 4}, we conclude that $\{\tau,\sigma,\theta,\pi_1,\pi_2\}$ satisfies \eqref{Condition: SP solution sufficient} for any $\theta\in (0,1)$ such that
{\small
\begin{gather}
         \frac{2L_{yy}}{\mu_y}\cdot\frac{1-\theta}{\sqrt{\theta}}  \leq 1-\beta,
         \label{Condition: SP solution from sigma 2}\\
         \frac{\mu_x}{1-\theta} - \xz{L'_{xx}} \geq (1-\theta)\frac{L_{yx}^2}{\mu_y}\cdot\Big(1- \frac{2L_{yy}}{\mu_y}\cdot\frac{1-\theta}{\sqrt{\theta}}\Big)^{-1}, \label{Condition: SP solution from tau}
\end{gather}}%
for some $\beta\in(0,1]$.
Clearly, a sufficient condition for \eqref{Condition: SP solution from tau} is
{\small
\begin{equation}\label{Condition: SP solution from tau 2}
            \frac{\mu_x}{1-\theta} - \xz{L'_{xx}} \geq (1-\theta)\frac{L_{yx}^2}{\mu_y}\cdot\frac{1}{\beta}.
\end{equation}}%
Note that \eqref{Condition: SP solution from sigma 2} implies that $\pi_1>0$. We also have $\pi_2=\sqrt{\theta}>0$ trivially.

When $L_{yy}>0$, given any $\beta\in(0,1)$, solving \cref{Condition: SP solution from sigma 2,Condition: SP solution from tau 2} for $\theta\in(0,1)$, we get the third condition in \eqref{Condition: SP solution to noisy LMI}. Indeed, it can be checked that $\theta\in[\bar{\theta}_2\xz{(\beta)},1)$ satisfies \eqref{Condition: SP solution from sigma 2} and $\theta\in[\bar{\theta}_1\xz{(\beta)},1)$ satisfies \eqref{Condition: SP solution from tau 2}; thus, $\theta\in[\bar{\theta}\xz{(\beta)},1)$ satisfies \eqref{Condition: SP solution from sigma 2} and  \eqref{Condition: SP solution from tau 2} simultaneously. Moreover, when $L_{yy}=0$, one does not need to solve \cref{Condition: SP solution from sigma 2} as the first inequality in \eqref{Condition: SP solution sufficient 4} holds trivially; thus, the only condition on $\theta$ comes from \eqref{Condition: SP solution from tau} which is equivalent to \eqref{Condition: SP solution from tau 2} with $\beta=1$.
The rest follows from  Lemma~\ref{LEMMA: Noise LMI after young's ineq} by setting $\alpha = \frac{\theta L_{yx}}{\pi_1} + \frac{\theta L_{yy}}{\pi_2}$. Indeed, the particular choice of $(\pi_1,\pi_2)$ in \eqref{Condition: SP solution to noisy LMI 3} gives us $\alpha = \frac{1}{\sigma}-\sqrt{\theta}L_{yy}$.
\end{proof}

\subsection{Proof of Theorem~\ref{cor:complexity-fg}}
\begin{proof}
\sa{It follows from \cref{Coroallary: explicit solution to noisy LMI} that this choice of $\{\tau,\sigma,\theta\}$ \xz{together with $\rho=\theta$} for the inner loop
ensures that \cref{eq: SCSC SAPD LMI} hold.}
For the outer loop, if we \sa{set $N$ as in \cref{Condition: SP solution to noisy LMI label 2} and}
\begin{equation}\label{eq: speical paramater}
    \begin{aligned}
          p_1 = \frac{1}{16},~p_2 = \sa{\frac{19}{32}},~p_3=\sa{\frac{11}{32}},~\beta_1 = \frac{4}{5},~\beta_2 = \frac{1}{2},~\zeta=32,
    \end{aligned}
\end{equation}
\sa{all assumptions of \cref{thm:WC_SP} are satisfied, i.e., both the inequality system \cref{eq: additional stepsize condition WCSC} and $K_{N}(\theta) \geq \sa{(1+\zeta)} M_{\tau,\sigma,\theta}$ hold.} \xz{Specifically, recall $K_N(\theta)=\sum_{k=0}^{N-1}\theta^{-k} = \frac{1-\theta^N}{1-\theta}\frac{1}{\theta^{\sa{N-1}}}$ and $M_{\tau,\sigma,\theta} = \max \{ \frac{4}{\mu_x\tau}, \frac{4+4\theta}{\mu_y\sigma}\}$. \sa{Since we set $\mu_x=\gamma$,} if \sa{we choose $\tau$ and $\sigma$
as in \cref{Condition: SP solution to noisy LMI label 2}, then we get $M_{\tau,\sigma,\theta}=\frac{4}{1-\theta}\max\{1,~(1+\theta)\theta\}$. Therefore,} a sufficient condition for $\frac{M_{\tau,\sigma,\theta}}{K_N(\theta)} \leq \frac{1}{(1+\zeta)}$ is}
\xz{
$$
\frac{4\max\{\frac{1}{\theta},1+\theta\}\theta^N}{1-\theta^N}\leq \frac{1}{(1+\zeta)}.
$$}

Because \sa{$\theta\in(0,1)$}, the above inequality is \sa{satisfied when}
\begin{equation}\label{eq: N bound 1}
    N\geq \frac{\ln \big(4\max\{ \frac{1}{\theta},1+\theta\}(1+\zeta)+1\big)}{\ln (\frac{1}{\theta})}.
\end{equation}
Now,
\sa{to get a sufficient condition, we consider two cases for \cref{eq: N bound 1}.}

\xz{\sa{\textbf{CASE 1:} Suppose $\frac{1}{\theta}\geq 1+\theta$. Since we have}
$$
\ln \big(\frac{4(1+\zeta)}{\theta}+1\big) \sa{\leq} \ln \big(\frac{4(1+\zeta)+1}{\theta}\big) = \ln \big(4(1+\zeta)+1\big) +\ln(\frac{1}{\theta}),
$$
\sa{it follows that \eqref{eq: N bound 1} holds for}
$$
N\geq \frac{\ln \big(4(1+\zeta)+1\big)}{\ln (\frac{1}{\theta})}+1.
$$
\sa{\textbf{CASE 2:} Now consider $1+\theta\geq \frac{1}{\theta}$. Since $\theta\in(0,1)$, \eqref{eq: N bound 1} holds for}
$$
\sa{N\geq \frac{\ln \big(8(1+\zeta)+1\big)}{\ln (\frac{1}{\theta})}\geq \frac{\ln \big(4(1+\theta)(1+\zeta)+1\big)}{\ln (\frac{1}{\theta})}.}
$$
\sa{Combining the two exhaustive cases, we conclude that \cref{eq: N bound 1} holds when}
$$
N\geq  \frac{\ln \big(8(1+\zeta)+1\big)}{\ln (\frac{1}{\theta})} + 1.
$$
\sa{Thus, choosing $\zeta=32$ and $N = \frac{\ln(265)}{\ln(\frac{1}{\theta})}+1$
ensures that
$K_{N}(\theta) \geq (1+\zeta) M_{\tau,\sigma,\theta}$.}}

Next, we continue to show the inequality system \cref{eq: additional stepsize condition WCSC} holds. \sa{Recall that for our choice of $N$ and $\zeta$ as above, we have $M_{\tau,\sigma,\theta}/K_N(\theta)\leq (1+\zeta)^{-1}$; hence, it follows that \cref{eq: additional stepsize condition WCSC-1} holds for $\mu_x=\gamma$, $p_1 = \frac{1}{16}$ and $\beta_1 = \frac{4}{5}$, i.e.,}
$$
 \sa{\frac{M_{\tau,\sigma,\theta}}{K_{N}(\theta)} -
        \left(1-\frac{M_{\tau,\sigma,\theta}}{K_{N}(\theta)}\right) p_1\Big(1-\frac{\gamma+\mu_x}{\gamma}\Big(\frac{1}{\beta_{1}}-1\Big)\Big) =\frac{33}{32} \tfrac{M_{\tau,\sigma,\theta}}{K_{N}(\rho)} - \frac{1}{32}
        \leq \frac{33}{32}\frac{1}{1+\zeta}-\frac{1}{32}=0.}
$$
\sa{Moreover, it is trivial to check that \cref{eq: additional stepsize condition WCSC-2} holds for the parameter values given in \cref{eq: speical paramater}.}

Since all assumptions of \cref{thm:WC_SP} are satisfied for parameters chosen as in \cref{Condition: SP solution to noisy LMI label 2} and \cref{eq: speical paramater},
if we substitute \cref{eq: speical paramater} into \cref{eq: sketch convergence result}, if follows that
\begin{equation}
    \begin{aligned}
        \frac{1}{T+1}\sum_{t=0}^{T} \mathbb{E}\left[\|\nabla\phi_{\lambda}(x^t_0) \|^2 \right]
        \leq
        &
        \sa{48\gamma}
        \left(\frac{1}{T+1}
        \sa{\mathcal{G}(x_0^0,y_0^0)}

        +\Xi_{\tau,\sigma,\theta} \right).
    \end{aligned}
\end{equation}
Thus, for any $\epsilon>0$,
the right side of the above inequality can be bounded by $\epsilon^2$ when
\begin{equation}\label{eq: split bound}

    \frac{\sa{48\gamma}}{T+1}

    \mathcal{G}(x_0^0,y_0^0)

    \leq \frac{\epsilon^2}{2},\qquad
    \sa{48\gamma}\Xi_{\tau,\sigma,\theta}\leq\frac{\epsilon^2}{2}.
\end{equation}
The first inequality in \cref{eq: split bound} implies that $\|\nabla\phi_{\lambda}(x_0^T)\|\leq\epsilon$ for
{\small
\begin{align}
    T \geq \sa{96}\mathcal{G}(x_0^0,y_0^0)  \cdot\frac{\gamma}{\epsilon^2}+1.
    \label{eq:T-bound}
\end{align}}%
Furthermore, \xz{because
    $\Xi_{\tau,\sigma,\theta} = \tau\left( \Xi^x_{\tau,\sigma,\theta} + \tfrac{1}{2}\right)\delta_x^2
     + \sigma\left( \Xi^y_{\tau,\sigma,\theta} +  \tfrac{1+2\theta}{2}\right)\delta_y^2$}, a sufficient condition for the second inequality in \cref{eq: split bound} is that
\begin{equation}\label{eq: sufficient condition for variance complexity}

    \sa{24\gamma}\tau(1+2\Xi^x_{\tau,\sigma,\theta})\delta^2_x\leq\frac{\epsilon^2}{4},
    \qquad

    \sa{24\gamma}\sigma(1+2\theta+2\Xi^{\sa{y}}_{\tau,\sigma,\theta})\delta_y^2\leq\frac{\epsilon^2}{4}.
\end{equation}
\sa{For the upcoming analysis below, define $\bar{\Xi}^x(\beta)\triangleq 3+ 2\Psi(\beta)$ and $\bar{\Xi}^y(\beta) \triangleq 33 +6\beta\left(2\frac{L_{xy}}{L_{yx}}-1\right) +2\frac{\mu_y}{\gamma}\Psi(\beta)$
 with $\Psi(\beta)\triangleq \min\left\{\sqrt{\frac{\beta\gamma}{\mu_y}},\frac{1-\beta}{2}\frac{L_{yx}}{L_{yy}}\right\}$.} {Moreover, recall that $\Xi^x_{\tau,\sigma,\theta}$, $\Xi^y_{\tau,\sigma,\theta}$ are defined in \cref{lemma: Gap bound with fg} as follows
 \begin{align*}
     &\Xi^x_{\tau,\sigma,\theta} = \left( 1+ \frac{\sigma\theta(1+\theta)L_{yx}}{2}\right),\\
     &\Xi^y_{\tau,\sigma,\theta} = \left(1+3\theta + \sigma\theta(1+\theta)L_{yy}  +  \tau\sigma\theta(1+\theta)L_{yx}L_{xy} \right)(1+2\theta) + \frac{\tau\theta(1+\theta)L_{yx}}{2}.
\end{align*}}%
Thus, the choice of $\tau$ and $\sigma$ further implies that $\Xi^x_{\tau,\sigma,\theta} = \sa{1 +
(1-\theta^2)\frac{L_{yx}}{2\mu_y}}$,
and
{\footnotesize
$$
     \Xi^y_{\tau,\sigma,\theta} = \Big( 1 + 3\theta + (1-\theta^2)\frac{L_{yy}}{\mu_y} + (1+\theta)(1-\theta)^2\frac{ L_{yx}L_{xy}}{\sa{\gamma}\mu_y}\Big)(1+2\theta) + \sa{\theta}(1-\theta^2)\frac{L_{yx}}{2\sa{\gamma}}.
$$}%
Since $0<\theta < 1$ and $1-\theta^2\leq 2(1-\theta)$, we have
$1+2\Xi^x_{\tau,\sigma,\theta} \leq 3+ 2(1-\theta)\frac{L_{yx}}{\mu_y}$ and
{\small
\begin{equation}
\label{eq:Xi_y_1}
    1+2\theta+2\Xi^y_{\tau,\sigma,\theta} \leq  3\Big(\sa{9} + 4(1-\theta)\frac{L_{yy}}{\mu_y} + 4(1-\theta)^2\frac{ L_{yx}L_{xy}}{\gamma\mu_y}\Big) + 2(1-\theta)\frac{L_{yx}}{\gamma}.
\end{equation}}%
On the other hand, since $\theta\geq \bar{\theta}=\max\{\bar{\theta}_1,\bar{\theta}_2\}$, the inequality $\sqrt{a+b} \leq \sqrt{a} + \sqrt{b}$ for all $a,b\geq 0$, and the definition of $\bar{\theta}(\beta)$
  together imply that
$1-\theta\leq \min\{\frac{\sqrt{\beta\gamma\mu_y}}{L_{yx}}, (1-\beta)\frac{\mu_y}{2L_{yy}}\}$. Thus,
{\small
\begin{equation}\label{eq: Xi_x bound}
    1+2\Xi^x_{\tau,\sigma,\theta} \leq 3+2(1-\theta)\frac{L_{yx}}{\mu_y}\leq \bar{\Xi}^x(\beta),
\end{equation}}%
and within \eqref{eq:Xi_y_1} bounding $(1-\theta)\frac{L_{yx}}{\mu_x}$ similarly and using $(1-\theta)^2 \leq \frac{\beta\gamma\mu_y}{L_{yx}^2}$, we get
{\small
\begin{equation*}
    1+2\theta+2\Xi^y_{\tau,\sigma,\theta} \leq  3\Big(9 + 4(1-\theta)\frac{L_{yy}}{\mu_y} + \frac{4\beta L_{xy}}{L_{yx}}\Big) + \frac{2\mu_y}{\gamma}\min\Big\{\sqrt{\tfrac{\beta\gamma}{\mu_y}},\tfrac{1-\beta}{2}\tfrac{L_{yx}}{L_{yy}}\Big\}.
\end{equation*}}%
\sa{Thus, $1-\theta\leq (1-\beta)\frac{\mu_y}{2L_{yy}}$ implies that}
{\small
\begin{align}\label{eq: Xi_y bound}
    1+2\theta+2\Xi^y_{\tau,\sigma,\theta} \leq
    \bar{\Xi}^y(\beta).
\end{align}}%
\sa{Note that $\beta=\min\{\frac{1}{2},~\frac{\mu_y}{4\gamma},~\frac{\gamma}{4\mu_y},~\frac{L_{yx}}{2L_{xy}}\}\in(0,1)$ implies that $\Psi(\beta)\leq \frac{1}{2}$; hence, $\bar{\Xi}^x(\beta)\leq 4$. Furthermore, our choice of $\beta$ also implies that $2\frac{\mu_y}{\gamma}\Psi(\beta)\leq 1$ and $2\beta\frac{L_{xy}}{L_{yx}}\leq 1$; therefore, $\bar{\Xi}^y(\beta)\leq 40$.
Using these upper bounds on $\bar{\Xi}^x(\beta)$ and $\bar{\Xi}^y(\beta)$} together with \cref{eq: Xi_x bound,eq: sufficient condition for variance complexity} and
\sa{$\theta\geq\{\overline{\overline{\theta}}_1,~\overline{\overline{\theta}}_2\}$ implied by \cref{Condition: SP solution to noisy LMI label 2},} we obtain that
\sa{48\gamma}\Xi_{\tau,\sigma,\theta}<\frac{\epsilon^2}{2}$.

\sa{Moreover, let $\overline{\theta}_1\triangleq\overline{\theta}_1(\beta)$ and $\overline{\theta}_2\triangleq\overline{\theta}_2(\beta)$ for $\beta=\min\{\frac{1}{2},~\frac{\mu_y}{4\gamma},~\frac{\gamma}{4\mu_y},~\frac{L_{yx}}{2L_{xy}}\}\in(0,1)$.} Consider $\theta=\max\{\overline{\theta}_1,~\overline{\theta}_2,~\overline{\overline{\theta}}_1,~\overline{\overline{\theta}}_2\}$. Since
$N = \frac{\ln(265)}{\ln(\frac{1}{\theta})}+1$,
the inequality $\frac{1}{\ln(\frac{1}{\theta})}\leq(1-\theta)^{-1}$ for $\theta\in(0,1)$ implies that
$$
N = \mathcal{O}\big(\max\{(1-\overline{\theta}_1)^{-1},(1-\overline{\theta}_2)^{-1},(1-\overline{\overline{\theta}}_1)^{-1},(1-\overline{\overline{\theta}}_2)^{-1}\}\big).
$$
First, we equivalently rewrite $(1-\overline{\theta}_1)^{-1}$
and $(1-\overline{\theta}_2)^{-1}$
as follows:
{\footnotesize
\begin{equation*}
         (1-\overline{\theta}_1)^{-1}
         =  \frac{1}{2}\frac{\xz{L'_{xx}}}{\gamma} + \sqrt{\frac{1}{4}\frac{\xz{{L'_{xx}}^2}}{\gamma^2}+ \frac{L_{yx}^2}{\beta \gamma\mu_y}},\quad
         (1-\overline{\theta}_2)^{-1}
     = \frac{1}{2} + \sqrt{\frac{1}{4} +\frac{4L_{yy}^2}{\left(1-\beta\right)^2\mu_y^2}};
\end{equation*}}%
\sa{thus, $(1-\overline{\theta}_1)^{-1}\leq \frac{\xz{L'_{xx}}}{\gamma}+\frac{L_{yx}}{\sqrt{\beta\gamma\mu_y}}$ and $(1-\overline{\theta}_2)^{-1}\leq 1+\frac{2}{1-\beta}\cdot\frac{L_{yy}}{\mu_y}$.} Finally, $(1-\overline{\overline{\theta}}_1)^{-1}=96\bar{\Xi}^x(\beta)\frac{\delta_x^2}{\epsilon^2}=\mathcal{O}(\frac{\delta_x^2}{\epsilon^2})$ and $(1-\overline{\overline{\theta}}_2)^{-1}=96\bar{\Xi}^y(\beta)\frac{\gamma}{\mu_y}\cdot\frac{\delta_y^2}{\epsilon^2}+1=\mathcal{O}(\frac{\gamma}{\mu_y}\cdot\frac{\delta_y^2}{\epsilon^2}+1)$. Recall that $\xz{L'_{xx}} = 2\gamma+L_{xx}$, using the above four identities we derive that
{\footnotesize
\begin{equation}
\label{eq:N-bound}
N = \mathcal{O}\Big(
\frac{\max\{L_{xx},L_{yx}\}}{\gamma} + \frac{\max\{L_{yx},L_{xy}\}}{\sqrt{\gamma\mu_y}} + \frac{\max\{L_{yy},L_{yx}\}}{\mu_y}
+ \Big(
\frac{\delta_x^2}{\gamma} +
\frac{\delta_y^2}{\mu_y} \Big)
\frac{\gamma}{\epsilon^2}\Big).
\end{equation}}
\sa{Since \xz{\texttt{SAPD+}} requires $NT$ oracle calls in total combining \eqref{eq:T-bound} with \eqref{eq:N-bound} leads to $\mathcal{O}(\epsilon^{-4})$ bound on $C_\epsilon$, which completes the proof.}
\end{proof}
\section{Proof of Theorem~\ref{thm:metric-equivalance}}
\sa{We first provide two technical result that will be used later when we give the proof of \cref{thm:metric-equivalance}.}
\begin{lemma}\label{lemma: SAPD single call complexity}
\sa{Consider $\mathcal{L}$ defined in \eqref{eq:main problem}. \xz{Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption} hold}. Given arbitrary $\bar{x}$, let $\hat x=\prox{\lambda\phi}(\bar x)$, where $\phi(\cdot)=\max_{y\in\mathcal{Y}}\mathcal{L}(\cdot,y)$ and $\lambda=(2\gamma)^{-1}$.
For any given $\hat{\epsilon}>0$, \texttt{SAPD} can generate $\tilde x_*\in\mathcal{X}$ such that $\mathbb{E}\left[\|\tilde x_* - \hat{x}\|\right]\leq\hat{\epsilon}$
within $\mathcal{\tilde{O}}(\frac{1}{\hat{\epsilon}^2})$ stochastic first-order oracle calls.}
\end{lemma}
\begin{proof}
Recall that $y_*(x)=\argmax_{y\in\mathcal{Y}}\mathcal{L}(x,y)$ for {$x\in\mathop{\bf dom} f$}.
Hence,
$(\hat{x},y_*(\hat{x}))$ is the unique saddle point to the SCSC problem:
\begin{align}
\label{eq:additional_SAPD label 2}
\min_{x\in\mathcal{X}} \max_{y\in\mathcal{Y}} \bar{\mathcal{L}}(x,y) \triangleq f(x) + \Phi(x,y) +
\sa{\gamma}\|x- \bar{x}\|^2-g(y).
\end{align}
Let $\{(x_N,y_N)\}$ be the iterate sequence generated by \texttt{SAPD} running on \eqref{eq:additional_SAPD label 2}, initialized from an arbitrary point $(x_0,y_0)$,
with parameters $\{\tau,\sigma,\theta\}$ chosen as follows:
{\small
\begin{equation}
\label{Condition: SP solution to noisy LMI label 3}
\tau = \frac{1-\theta}{
\gamma},\quad \sigma = \frac{1-\theta}{\mu_y\theta},
\quad
\theta  \sa{=} \max\{\bar{\theta}(\beta),~\doublehat{\theta}_1,~\doublehat{\theta}_2\},
\end{equation}}
for $\beta=\min\{\frac{1}{2},~\frac{\mu_y}{4\gamma},~\frac{\gamma}{4\mu_y},~\frac{L_{yx}}{2L_{xy}}\}$, where $\bar{\theta}(\beta)$ is \sa{defined in
\cref{cor:complexity-fg}} and
{\footnotesize
\begin{equation}
\label{eq:theta_bound single call SAPD}
    \doublehat{\theta}_1  \triangleq \max\Big\{0,1 - \frac{1}{16}\cdot\gamma^2\cdot\frac{\hat{\epsilon}^2}{\delta_x^2}\Big\},
\quad
\doublehat{\theta}_2 \triangleq \left(1+ \frac{1}{16}\cdot\frac{\mu_y\gamma}{10}\cdot\frac{\hat{\epsilon}^2}{\delta_y^2}\right)^{-1}.
\end{equation}}%
\sa{The problem in~\cref{eq:additional_SAPD label 2} is a generic form of the SCSC subproblems given in~\cref{eq: SCSC problem chpt2}; furthermore, $(\tau,\sigma,\theta)$ chosen as in~\eqref{Condition: SP solution to noisy LMI label 3} satisfies \eqref{eq: SCSC SAPD LMI} with $\rho=\theta$, $\mu_x=\gamma$ and $\alpha = \frac{1}{\sigma}-\sqrt{\theta}L_{yy}>0$. Therefore,
the arguments
in the proof of \cref{lemma: Gap bound with fg} establishing \eqref{INEQ: ergordic gap  bounded version} also apply to our setting here.
Indeed, if we customize \eqref{INEQ: ergordic gap  bounded version} for \eqref{eq:additional_SAPD label 2} and use \eqref{eq:U-bound} and \eqref{eq:variance-bound} together with $\eta_x=\frac{1}{\tau}$ and $\eta_y=\frac{1}{\sigma}$, then we get}
{\footnotesize
\begin{eqnarray}
\lefteqn{K_{N} ({\theta}) \sa{\big(\bar{\mathcal{L}}(\bar{x}_N,y_*(\hat{x}))  - \bar{\mathcal{L}}(\hat{x},\bar{y}_N)\big)} + \sa{\theta^{-N+1}}\Delta_N(\hat{x},y_*(\hat{x}))} \nonumber  \\
&&\leq

      \frac{1}{\tau}\|\hat{x}-x_{0}\|^2
      +

      \frac{1+\theta}{\sigma}~\|y_*(\hat{x})-y_{0}\|^2
       + K_{N} ({\theta})~\Xi_{\tau,\sigma\theta},

       \label{eq:gap_bound single SAPD}
\end{eqnarray}}%
where \sa{$(\bar{x}_N,\bar{y}_N)=\sum_{k=0}^{N-1}\theta^{-k}(x_{k+1},y_{k+1})$, $K_N(\theta)=\sum_{k=0}^{N-1}\theta^{-k}\leq\frac{\theta^{-N+1}}{1-\theta}$ and}
$$
\Delta_N(\hat{x},y_*(\hat{x}))=\frac{1}{2\theta}\Big(\frac{1}{\tau}-\gamma\Big) \|x_{N}-\hat{x}\|^2 +  \frac{1}{2\theta}{\Big(\frac{1}{\sigma} - \alpha \Big)}\|y_{N}-y_*(\hat{x})\|^2.
$$
Since $(\hat x, y_*(\hat x))$ is the saddle point of $\bar\mathcal{L}$, we have $\bar{\mathcal{L}}(\bar{x}_N,y_*(\hat{x}))  - \bar{\mathcal{L}}(\hat{x},\bar{y}_N)\geq 0$. Moreover, $\tau=\frac{1-\theta}{\gamma}$ implies that $\frac{1}{\tau}-\gamma=\frac{\gamma\theta}{1-\theta}$; hence, \eqref{eq:gap_bound single SAPD} implies that
{\small
\begin{align}
    \frac{\theta^{-N+1}}{2\theta}\frac{\gamma\theta}{1-\theta}\norm{x_N-\hat x}^2\leq \frac{\gamma}{1-\theta}\norm{x_0-\hat x}^2+\mu_y\theta\frac{1+\theta}{1-\theta}\norm{y_0-y_*(\hat x)}^2+\frac{1}{1-\theta}\theta^{-N+1}\Xi_{\tau,\sigma,\theta}
\end{align}}%
Thus, using $\theta\in(0,1)$, we get
\begin{equation*}
    \mathbb{E}\left[ \|x_{N}-\hat{x}\|^2 \right] \leq \theta^{N-1}\Big(2\|\hat{x}-x_{0}\|^2 + \frac{4\mu_y}{\gamma}\|y_*(\hat{x})-y_{0}\|^2\Big) + \frac{2}{\gamma} \Xi_{\tau,\sigma,\theta}.
\end{equation*}
Then, by Jensen's inequality, it follows that
\begin{equation*}
    \Big(\mathbb{E}\left[ \|x_{N}-\hat x\| \right]\Big)^2 \leq \mathbb{E}\left[ \|x_{N}-\hat x\|^2 \right] \leq \theta^{N-1}4\max\Big\{1,\frac{\mu_y}{\gamma}\Big\}\mathcal{D}_0^2 + \frac{2}{\gamma} \Xi_{\tau,\sigma,\theta},
\end{equation*}
where $\mathcal{D}_0 \triangleq \big(\|\hat{x}-x_{0}\|^2 + \|y_*(\hat{x})-y_{0}\|^2\big)^{1/2}$.
For given $\hat{\epsilon}>0$, the distance metric $\mathbb{E}\left[ \|x_{N}-\hat{x}\| \right]$ can be bounded by $\hat{\epsilon}$ when
\begin{subequations}
\label{eq: split bound of distance metric}
\begin{align}
    &\frac{2}{\gamma} \Xi_{\tau,\sigma,\theta}\leq \frac{\hat{\epsilon}^2}{2},
    \label{eq: split bound of distance metric-xi}\\
    &\theta^{N-1}4\max\Big\{1,\frac{\mu_y}{\gamma}\Big\}\mathcal{D}_0^2 \leq\frac{\hat{\epsilon}^2}{2}.
    \label{eq: split bound of distance metric-N}
\end{align}
\end{subequations}
Recall that in the proof of \cref{cor:complexity-fg}, we showed that for any given $\epsilon>0$,
\begin{equation}
\label{eq:cor-2-xi-bound}
\sa{48\gamma}\Xi_{\tau,\sigma,\theta}\leq\frac{\epsilon^2}{2}
\end{equation}
holds for $\theta\in(0,1)$ satisfying $\theta\geq\max\{\overline{\overline{\theta}}_1,~\overline{\overline{\theta}}_2\}$, \xz{where $\overline{\overline{\theta}}_1$ and $\overline{\overline{\theta}}_2$ are defined in \eqref{eq:theta_double_bar_bound}.} Note for $\epsilon=2\sqrt{6}\gamma\hat\epsilon$, \cref{eq:cor-2-xi-bound} is equivalent to \eqref{eq: split bound of distance metric-xi}. If we substitute $2\sqrt{6}\gamma\hat\epsilon$ for $\epsilon$ within the definitions of $\overline{\overline{\theta}}_1$ and $\overline{\overline{\theta}}_2$, we get $\doublehat{\theta}_1$ and $\doublehat{\theta}_2$, respectively, as in \eqref{eq:theta_bound single call SAPD}. Therefore, requiring $\theta\geq\max\{\doublehat{\theta}_1,~\doublehat{\theta}_2\}$ implies \eqref{eq: split bound of distance metric-xi}.

Furthermore, \eqref{eq: split bound of distance metric-N} holds when $N\geq \ln\Big( \frac{8\max\{1,\mu_y/\gamma\}\mathcal{D}_0^2}{\hat{\epsilon}^2}\Big)/\ln\Big(\frac{1}{\theta}\Big) + 1$. Thus, we conclude that for any $\hat{\epsilon}>0$, \texttt{SAPD}, stated in \cref{ALG: SAPD}, can generate $x_N$ such that $\mathbb{E}\left[ \|x_{N}-\hat{x}\| \right]\leq\hat{\epsilon}$ within $N_{\hat\epsilon}$ iterations for $\theta  \sa{=} \max\{\bar{\theta}(\beta),~\doublehat{\theta}_1,~\doublehat{\theta}_2\}$, where
\begin{equation}\label{eq: N bound with theta single SAPD}
    N_{\hat\epsilon} = \mathcal{O}\Big(\ln\Big( \frac{\max\{1,\mu_y/\gamma\}}{\hat{\epsilon}}\Big)/\ln\Big(\frac{1}{\theta}\Big) + 1 \Big).
\end{equation}
Note $\frac{1}{\ln(\frac{1}{\theta})}\leq(1-\theta)^{-1}$ for $\theta\in(0,1)$ implies that
$$
\frac{1}{\ln(\frac{1}{\theta})} \leq \mathcal{O}\Big(\max\{(1-\overline{\theta}_1(\beta))^{-1},(1-\overline{\theta}_2(\beta))^{-1},(1-\doublehat{\theta}_1)^{-1},(1-\doublehat{\theta}_2)^{-1}\}\Big).
$$
Therefore, \eqref{Condition: SP solution to noisy LMI}, \eqref{eq:theta_bound single call SAPD} and our choice of $\beta=\min\{\frac{1}{2},~\frac{\mu_y}{4\gamma},~\frac{\gamma}{4\mu_y},~\frac{L_{yx}}{2L_{xy}}\}$ imply that
{\footnotesize
\begin{equation*}
\frac{1}{\ln(\frac{1}{\theta})}= \mathcal{O}\Big(
\frac{\max\{L_{xx},L_{yx}\}}{\gamma} + \frac{\max\{L_{yx},L_{xy}\}}{\sqrt{\gamma\mu_y}} + \frac{\max\{L_{yy},L_{yx}\}}{\mu_y}
+ \Big(
\frac{\delta_x^2}{\gamma} +
\frac{\delta_y^2}{\mu_y} \Big)
\frac{1}{\gamma\hat{\epsilon}^2}\Big),
\end{equation*}}%
which can also be verified through plugging $\epsilon= 2\sqrt{6}\gamma\hat{\epsilon}$ into \cref{eq:N-bound}. From \eqref{eq: N bound with theta single SAPD}, we conclude that
{\footnotesize
\begin{equation*}
N_{\hat\epsilon} = \mathcal{O}\left(\Big(
\frac{\max\{L_{xx},L_{yx}\}}{\gamma} + \frac{\max\{L_{yx},L_{xy}\}}{\sqrt{\gamma\mu_y}} + \frac{\max\{L_{yy},L_{yx}\}}{\mu_y}
+ \Big(
\frac{\delta_x^2}{\gamma} +
\frac{\delta_y^2}{\mu_y} \Big)
\frac{1}{\gamma\hat{\epsilon}^2}\Big)\ln\Big( \frac{\max\{1,\mu_y/\gamma\}}{\hat{\epsilon}}\Big)\right),
\end{equation*}}
which completes the proof.
\end{proof}

\begin{lemma}\label{lemma: proximal point lemma v2}
\sa{Suppose $f:\mathcal{X}\to\mathbb{R}\cup\{+\infty\}$ is closed convex, and $V$ is a strictly convex function on $\mathop{\bf dom} f$ and differentiable on an open set containing $\mathop{\bf dom} f$.} Let $x_* = \argmin_{x\in\mathcal{X}} f(x) + V(x)$. Then, $x_* =  \prox{\alpha f}(x_* - \alpha \nabla V(x_*))$
for any $\alpha>0$.
\end{lemma}
\begin{proof}
\sa{From the first-order optimality condition}, we have
\begin{equation}\label{eq: optimality condition lemma v2}
   0 \in \partial f(x_*) + \nabla V(x_*).
\end{equation}
\sa{Moreover, from the definition of $\prox{\alpha f}(\cdot)$ operator, it follows that}
\begin{align}
\label{eq:prox-condition}
\prox{\alpha f}(x_* - \alpha \nabla V(x_*)) = \argmin_{x\in\mathcal{X}} f(x) + \nabla V(x_*)(x-x_*) + \frac{1}{2\alpha}\|x-x_*\|^2.
\end{align}
\sa{Finally, \eqref{eq: optimality condition lemma v2} implies that $x_*$ is the unique minimizer of the problem on the rhs of \eqref{eq:prox-condition}. Therefore, $x_* =\prox{\alpha f}(x_* - \alpha \nabla V(x_*))$, which completes the proof.}
\end{proof}
\sa{We are now ready to prove \cref{thm:metric-equivalance}.}
\begin{proof}
Let $\hat{x}= \prox{\lambda\phi}(\sa{x_\epsilon})$, and $\phi^s$ be the smooth part of $\phi$, i.e., $\phi=f+\phi^s$.
Moreover, since $\Phi(x,\cdot)-g(\cdot)$ is strongly concave and $\Phi(\cdot,y)$ is differentiable, we have that $\phi^s$ is differentiable; hence, for any \sa{$x\in\mathop{\bf dom} f$},
$$
\nabla \phi^s(x) = \nabla_x \Phi(x,y_*(x)), \quad \text{where} \quad y_*(x) = \argmax_{y\in\mathcal{Y}} \Phi(x,y)-g(y).
$$
Then we can explicitly write $\hat{x}$ as
\begin{equation*}
    \hat{x} = \argmin_{x\in\mathcal{X}} f(x) + \phi^s(x) + \frac{1}{2\lambda}\|x- x_\epsilon\|^2.
\end{equation*}
\sa{Since $\phi^s(\cdot) + \frac{1}{2\lambda}\|\cdot- x_\epsilon\|^2$ is smooth and strongly convex,} for any $\alpha>0$, \cref{lemma: proximal point lemma v2} implies that
$$
\hat{x} = \prox{\alpha f}\left(\hat{x} - \alpha\big(\nabla\phi^s(\hat{x})\xz{+}\frac{1}{\lambda}(\hat{x}-x_\epsilon) \big)\right).
$$
If we let $\alpha=\lambda$, it follows that
$$
\hat{x} = \prox{\lambda f}\big(x_\epsilon - \lambda\nabla_x\phi^s(\hat{x}) \big).
$$
\sa{Moreover, since $f$ is closed convex, $\prox{f}(\cdot)$ is nonexpansive; hence,}
\begin{equation}
    \begin{aligned}
 \|\hat{x} - \prox{\lambda f}\big(
\hat{x} - \lambda\nabla_x\phi^s(\hat{x})\big) \|
\leq  \|x_\epsilon-\hat{x}\|\leq \frac{\lambda \epsilon}{2},
    \end{aligned}
\end{equation}
where we  used  \cref{Lemma: graident of ME} for the last inequality, i.e.,
$
\|x_\epsilon - \hat{x}\|=\lambda\|\nabla \phi_{\lambda}(x_\epsilon)\|.
$
On the other hand, for any $\tilde{x}\in\sa{\mathop{\bf dom} f}$,
\begin{equation}\label{eq: gradient bound v2}
    \begin{aligned}
   & \|
\tilde{x} -
\prox{\lambda f}
\big(
\tilde{x} - \lambda\nabla_x\phi^s(\tilde{x})
\big)  \|
 \\
\leq &            \|
\tilde{x} -
\prox{\lambda f}
\big(
\tilde{x} - \lambda\nabla_x\phi^s(\tilde{x})
\big)  -  \hat{x} +
\prox{\lambda f}
\big(
\hat{x} - \lambda\nabla_x\phi^s(\hat{x})
\big)\| + \frac{\lambda\epsilon }{2}
\\
\leq & 2\|\tilde{x}-\hat{x}\| + \lambda
\norm{\nabla_x \Phi(\tilde x,y_*(\tilde x))-\nabla_x \Phi(\hat x,y_*(\hat x))}+ \frac{\lambda\epsilon}{2}.
\end{aligned}
\end{equation}
According to
\cite[Proposition 1]{chen2021proximal}, $y_*(\cdot)$ is Lipschitz with constant \xz{$\kappa_{yx}=\frac{L_{yx}}{\mu_y}$}.
Therefore, we get
$$
\norm{\nabla_x \Phi(\tilde x,y_*(\tilde x))-\nabla_x \Phi(\hat x,y_*(\hat x))}
\leq L_{xx}\|\tilde x-\hat x\| + L_{xy}\|y_*(\tilde x)-y_*(\hat x)\|\leq \big(L_{xx}+L_{xy}\xz{\kappa_{yx}}\big)\|\tilde x-\hat x\|,
$$
which together with \cref{eq: gradient bound v2} implies that
\begin{equation}\label{eq:gradient bound 2 v2}
    \frac{1}{\lambda}\norm{\tilde{x} -
\prox{\lambda f}
\big(
\tilde{x} - \lambda\nabla_x\phi^s(\tilde{x})
\big) }
    \leq  \big(\frac{2}{\lambda}+L_{xx}+L_{xy}\xz{\kappa_{yx}}\big)\|\tilde{x}-\hat{x}\| + \frac{\epsilon}{2}.
\end{equation}
Let $\lambda^{-1}=2\gamma$, and $C\triangleq (4\gamma+ L_{xx}+L_{xy}\xz{\kappa_{yx}})^{-1}/2$. Thus, for any $\tilde x\in\sa{\mathop{\bf dom} f}$ such that {$\norm{\tilde x-\hat x}^2\leq C^2\epsilon^2$}, we have
$$
 \frac{1}{\lambda}\norm{\tilde{x} -
\prox{\lambda f}
\big(
\tilde{x} - \lambda\nabla_x\phi^s(\tilde{x})
\big) }\leq\epsilon.
$$
Indeed, when $f(x)=0$ for all $x\in\mathcal{X}$, we get $\phi(x)=\phi^s(x)$ and the above inequality implies that
$$
\|\nabla\phi(\tilde{x})\|\leq\epsilon.
$$
The rest directly follows from invoking \cref{lemma: SAPD single call complexity} with $\hat\epsilon=C\epsilon$ and $\bar{x}=x_\epsilon$.
\end{proof}
\section{Proof of Theorem~\ref{thm:unbounded-domains}}
Next, \sa{for completeness,} we provide a technical lemma establishing Lipschitz continuity of the best response functions (see also~\cite[Lemma 2.5]{zhang2021robust} and \cite[Lemma B.2(a)]{lin-near-optimal}).

\begin{lemma}\sa{\cite[Proposition 1]{chen2021proximal}}
\label{lem:implicit_function}
\sa{Suppose Assumptions~\ref{ASPT: fg} and~\ref{ASPT: lipshiz gradient} hold.} {For any given $y\in\mathop{\bf dom} g$}, let $x^t_*(y)\triangleq\argmin_{x\in\mathcal{X}}\mathcal{L}^{t}(x,y)$; and {for any given $x\in\mathop{\bf dom} f$}, let $y_*(x)\triangleq\argmax_{y\in\mathcal{Y}}\mathcal{L}^{t}(x,y)=\argmax_{y\in\mathcal{Y}}\mathcal{L}(x,y)$. Then $x^t_*(\cdot)$ and $y^t_*(\cdot)$ are Lipschitz maps on {$\mathop{\bf dom} g$ and $\mathop{\bf dom} f$}, with constants
\sa{$\kappa_{xy}$ and $\kappa_{yx}$}, respectively, where $\kappa_{xy}\triangleq L_{xy}/\mu_x$ and $\kappa_{yx}\triangleq L_{yx}/\mu_y$.
\end{lemma}

\begin{lemma}\label{lemma: bounded Gap and disance}
 \sa{For any $t\geq 0$, let $z^t_*\triangleq (x^t_*,y^t_*)$ be the unique saddle point of $\mathcal{L}^{t}$ defined in \cref{eq: SCSC problem chpt2}, and let $\{z_k^t\}_{k=0}^{N_t}$ be generated by running \texttt{SAPD} on $\min_{x\in\mathcal{X}}\max_{y\in\mathcal{Y}}\mathcal{L}^{t}(x,y)$ for $N_t\in\mathbb{Z}_+$ iterations, where $z^t_k\triangleq (x^t_k,y^t_k)$; and define $z_0^{t+1} \triangleq \tfrac{1}{K_{N_t}(\rho)}\sum_{i=0}^{N_t-1}\rho^{-i}z^t_{i+1}$.} Under the setting of \cref{lemma: Gap bound with fg},

\begin{equation}\label{eq: bounded Gap and disance}
     \max\Big\{\mathbb{E}\left[\mathcal{G}^{t}(z_0^{t+1})\right],~\mathbb{E}\left[\norm{z_0^{t+1}-z_*^t}^2\right]\Big\}\leq \rho^{N_t}N_t C_{\tau,\sigma,\theta} \mathbb{E}[\norm{z_0^{t}-z_*^t}^2] + C'_{\tau,\sigma,\theta}
\end{equation}
holds for all {$t\geq 0$ and $N_t\geq 1$}, for some \sa{positive constants} $C_{\tau,\sigma,\theta}$ and $C'_{\tau,\sigma,\theta}$.
\end{lemma}
\begin{proof}
For simplicity we assume $N_t=N$ for all $t\geq 0$ --the proof still holds for arbitrary $\{N_t\}_{t\geq 0}\subset\mathbb{Z}_+$. The proof mainly follows the proof of \cref{lemma: Gap bound with fg}.
\sa{We first show that the rhs of \cref{eq: bounded Gap and disance}  is a bound for
$\mathbb{E}\left[\norm{z_0^{t+1}-z_*^t}^2\right]$;
then,
we show it for $\mathbb{E}\left[\mathcal{G}^{t}(z_0^{t+1})\right]$.}

\sa{Now, we start with analyzing $\mathbb{E}\left[\norm{z_0^{t+1}-z_*^t}^2\right]$.
The analysis below mainly relies on the proof of \cref{lemma: Gap bound with fg}. Indeed,} given $z^t_0=(x^t_0,y^t_0)$ for $t\geq0$, substituting $x=x^{t}_*$ and $y = y^{t}_*$ within \eqref{INEQ: ergordic gap  bounded version} and then using \cref{eq:U-bound}, we obtain that
{\footnotesize
\begin{equation}
    \begin{aligned}
        \label{eq: custom gap bound}
\lefteqn{K_{N} ({\rho})   \big(\mathcal{L}^{t}(\bar{x}^t_{N}, y^t_*)  - \mathcal{L}^{t}(x^t_*, \bar{y}^t_{N}) \big) +  {\rho}^{-N+1}\Delta^{t}_N(x^{t}_*,y^t_{*})}\\
&\leq
      \left(\frac{1}{2\tau}+\frac{\eta_x}{2}\right)\|x^{t}_*-x^{t}_{0}\|^2
      +\left(\frac{1}{2\sigma}+\frac{\eta_y(1+2\theta)}{2}\right)\|y^t_{*}-y^{t}_{0}\|^2
       +\sum_{k=0}^{N-1}{\rho}^{-k}(P^{t}_k+ Q^{t}_k), \nonumber
    \end{aligned}
\end{equation}}%
where \sa{$\Delta_N^t(x,y) = \frac{1}{2\rho}(\frac{1}{\tau}-\mu_x)  \|x-x^{t}_{N}\|^2 +  \frac{1}{2\rho}(\frac{1}{\sigma} - \alpha ) \|y-y^{t}_{N}\|^2$}.
\sa{Since} $\mathcal{L}^{t}(\bar{x}^t_{N}, y^t_*)  - \mathcal{L}^{t}(x^t_*, \bar{y}^t_{N}) \geq0$, if we let $\eta_x = \frac{1}{\tau}$ and $\eta_y = \frac{1}{\sigma}$, \sa{then
it follows from \eqref{eq:variance-bound} that}
\begin{equation}
{\rho}^{-N+1}\mathbb{E}\left[\Delta^{t}_N(x^{t}_*,y^t_{*})\right] \leq \mathbb{E}\left[\overline{U}^t(x^t_*,y^t_*)\right] + \sa{K_N(\rho){\Xi}_{\tau,\sigma,\theta}},
\end{equation}
where \sa{${\Xi_{\tau,\sigma,\theta}}$} is defined in \cref{lemma: Gap bound with fg}  and
\begin{align}
\label{eq:bar_U_t}
    \overline{U}^t(x,y) \triangleq \frac{1}{\tau}\|x-x^{t}_{0}\|^2
+\frac{1+\theta}{\sigma}\|y-y^{t}_{0}\|^2.
\end{align}

\sa{Recall $K_N(\rho) = \sum_{i=0}^{N-1}\rho^{-i} =\rho^{-N+1}\frac{1-\rho^N}{1-\rho}$ for $\rho\in(0,1)$. Since $N\geq 1$ is arbitrary, for all $k\in\mathbb{Z}_+$ such that $k\leq N$, we get}
{\small
\begin{equation}\label{eq: custom distance bound}
    \mathbb{E}\left[
    \frac{1}{2}\Big(\frac{1}{\tau}-\mu_x\Big)  \|x^{t}_{k}-x^t_*\|^2 +  \frac{1}{2}\Big(\frac{1}{\sigma} - \alpha\Big) \|y^{t}_{k}-y^t_*\|^2
    \right] \leq \rho^{k}\mathbb{E}\left[\overline{U}^t(x^t_*,y^t_*)\right] + \frac{\rho}{1-\rho}\Xi_{\tau,\sigma,\theta}.
\end{equation}}%
\sa{Recall that $z_0^{t+1}=(x^{t+1}_{0},y^{t+1}_{0})$ such that $z_0^{t+1} = \tfrac{1}{K_{N}(\rho)}\sum_{k=1}^{N}\rho^{-k+1}z^t_{k}$, i.e.,
$z_0^{t+1}$ is
a weighted average of $t$-stage iterates $\{z^t_k\}_{k=1}^N$.} Therefore, we can use Jensen's inequality and \cref{eq: custom distance bound} to bound $\mathbb{E}[\norm{z^{t+1}_{0}-z^{t}_*}^2]$. Indeed, since $\norm{\cdot}^2$ is convex, if we multiply \cref{eq: custom distance bound} by $\rho^{-k+1}$ and sum it from $k=1$ to $N$, using Jensen's inequality we obtain that
{\small
\begin{equation}\label{eq: custom gap bound 2}
    \begin{aligned}
     &K_{N}(\rho)\mathbb{E}\left[
    \frac{1}{2}(\frac{1}{\tau}-\mu_x)  \|x^{t+1}_{0}-x^t_*\|^2 +  \frac{1}{2}(\frac{1}{\sigma} - \alpha ) \|y^{t+1}_{0}-y^t_*\|^2
    \right]  \\
    \leq & \sum_{k=1}^{N}
    \rho^{-k+1}\mathbb{E}\left[
    \frac{1}{2}(\frac{1}{\tau}-\mu_x)  \|x^{t}_{k}-x^t_*\|^2 +  \frac{1}{2}(\frac{1}{\sigma} - \alpha ) \|y^{t}_{k}-y^t_*\|^2
    \right]   \\
    \leq  & \sum_{k=1}^{N}\rho^{-k+1}\left(\rho^{k}\mathbb{E}\left[\overline{U}^t(x^t_*,y^t_*)\right] + \frac{\rho}{1-\rho}\Xi_{\tau,\sigma,\theta}
    \right)
    = \rho N\mathbb{E}\left[\overline{U}^t(x^t_*,y^t_*)\right] + \frac{\rho}{1-\rho} K_N(\rho)\Xi_{\tau,\sigma,\theta}.
    \end{aligned}
\end{equation}}
Thus,
{\small
$$
\mathbb{E}\left[
    \frac{1}{2}(\frac{1}{\tau}-\mu_x)  \|x^{t+1}_{0}-x^t_*\|^2 +  \frac{1}{2}(\frac{1}{\sigma} - \alpha ) \|y^{t+1}_{0}-y^t_*\|^2
    \right]  \leq \rho^{N}N\mathbb{E}\left[\overline{U}^t(x^t_*,y^t_*)\right] + \frac{\rho}{1-\rho}\Xi_{\tau,\sigma,\theta}.
$$}
Therefore, we conclude that
\begin{equation}\label{eq: distance metric bound}
    \mathbb{E}\left[\|z^{t+1}_0 -z^t_*\|^2 \right]\leq \rho^{N}N\overline{C}_{\tau,\sigma,\theta}\mathbb{E}\left[\|z^t_0-z^t_*\|^2\right] + \overline{C}'_{\tau,\sigma,\theta},\quad \forall N\geq 1,
\end{equation}
where
$$
\overline{C}_{\tau,\sigma,\theta} = \frac{2\max\{\frac{1}{\tau},\frac{1+\theta}{\sigma} \}}{\min\{\frac{1}{\tau}-\mu_x, \frac{1}{\sigma}-\alpha\}},\quad \overline{C}'_{\tau,\sigma,\theta} = \frac{2}{\min\{\frac{1}{\tau}-\mu_x, \frac{1}{\sigma}-\alpha\}}\frac{\rho}{1-\rho}\Xi_{\tau,\sigma,\theta}.
$$
\sa{This completes the first part of the proof. Next, we will bound $\mathbb{E}[\mathcal{G}^{t}(z^{t+1}_0)]$ using the bound on $\mathbb{E}[\|z^{t+1}_{0}-z^{t}_{*}\|^2]$ we derived in the first part.}

Given $z^t_0$, using \cref{eq:gap_bound_fg_2} and \cref{eq:variance-bound} in the proof \cref{lemma: Gap bound with fg} for $\eta_x = \frac{1}{\tau}$ and $\eta_y = \frac{1}{\sigma}$ as above, we obtain that
\begin{equation}\label{eq: simplified gap bound}
    \mathbb{E}\left[\mathcal{G}^{t}(z_0^{t+1})\right]
\leq \frac{1}{K_{N}(\rho)}\mathbb{E}\left[\overline{U}^t\Big(x^{t}_*(y^{t+1}_0), y_{*}(x^{t+1}_0)\Big)\right]
      +\Xi_{\tau,\sigma,\theta},
\end{equation}
where $\overline{U}^t(x,y)$ is defined in~\eqref{eq:bar_U_t} and \sa{${\Xi_{\tau,\sigma,\theta}}$} is defined in \cref{lemma: Gap bound with fg}; furhermore, $x^t_*(\cdot)$ and $y_*(\cdot)$ are defined in \cref{eq: x* y*}. Next, we will use \cref{eq: distance metric bound} to \sa{derive} an upper bound for the right hand side of \cref{eq: simplified gap bound}.

\sa{Since $z_*^t$ is the unique saddle point for $\mathcal{L}^t$, we have $x^t_*(y^t_*)=x^t_*$ and $y_*(x^t_*)=y^t_*$. Moreover,} according to
\cref{lem:implicit_function}, $x^t_*(\cdot)$, $y_*(\cdot)$ is Lipschitz with \sa{constants} ${\kappa_{xy}=\frac{L_{xy}}{\mu_x}}$ and ${\kappa_{yx}=\frac{L_{yx}}{\mu_y}}$, respectively. Without loss of generality, we assume $\kappa_{yx}\geq1$ and $\kappa_{xy}\geq1$.
Therefore, \cref{lem:implicit_function} and
$$
\overline{U}^t(x^{t}_*(y^{t+1}_0), y_{*}(x^{t+1}_0)) \leq \frac{2}{\tau}\|x^t_*-x^{t}_*(y^{t+1}_0)\|^2
      +\frac{2+2\theta}{\sigma}\|y^t_*-y_{*}(x^{t+1}_0)\|^2 + 2\overline{U}^t(x^t_*,y^t_*),\quad \text{w.p.}~1,
$$
together imply that
\begin{equation*}
    \begin{aligned}
            \mathbb{E}& \left[\overline{U}^t(x^{t}_*(y^{t+1}_0), y_{*}(x^{t+1}_0))\right]
            \\
            \leq&  \mathbb{E}\left[\frac{2\kappa_{xy}^2}{\tau}\|y^t_*-y^{t+1}_0\|^2
      +\frac{(2+2\theta)  \kappa_{yx}^2}{\sigma}\|x^t_*-x^{t+1}_0\|^2 + 2\overline{U}^t(x^t_*,y^t_*) \right]\\
      \leq &
      \mathbb{E}\left[ \max\Big\{
      \frac{2}{\tau},
      \frac{(2+2\theta)}{\sigma}
      \Big\} \Big(\sa{\max\{\kappa_{xy}^2,~\kappa_{yx}^2\}}\|  z^{t+1}_0 -z^t_*\|^2  +\| z^{t}_0 -z^t_*\|^2 \Big)\right]
      \\
        \leq & \mathbb{E}\left[ \max\{
      \frac{2}{\tau},
      \frac{(2+2\theta)}{\sigma}
      \} \left(\Big(\rho^{N}N\sa{\max\{\kappa_{xy}^2,~\kappa_{yx}^2\}}+1\Big)\overline{C}_{\tau,\sigma,\theta}\|z^t_0-z^t_*\|^2 + \overline{C}'_{\tau,\sigma,\theta}\right)\right],
    \end{aligned}
\end{equation*}
where we use \cref{eq: distance metric bound} for the last inequality. Then, if we use the above inequality within \cref{eq: simplified gap bound}, it follows that
\sa{
\begin{equation*}
    \begin{aligned}
         \mathbb{E}\left[\mathcal{G}^{t}(z_0^{t+1})\right] & \leq\frac{1}{K_N(\rho)}\Big(\rho^{N}N \max\{\kappa_{xy}^2,~\kappa_{yx}^2\}+1\Big) \overline{\overline{C}}_{\tau,\sigma,\theta}\mathbb{E}\left[\|z^t_0-z^t_*\|^2 \right]+ \frac{\overline{\overline{C}}'_{\tau,\sigma,\theta}}{K_{N}(\rho)}  +\Xi_{\tau,\sigma,\theta}\\
         & = \rho^N N \frac{1-\rho}{1-\rho^N}\Big(\rho^{N-1} \max\{\kappa_{xy}^2,~\kappa_{yx}^2\} + \frac{1}{\rho N}\Big) \overline{\overline{C}}_{\tau,\sigma,\theta}\mathbb{E}\left[\|z^t_0-z^t_*\|^2 \right]+ \frac{\overline{\overline{C}}'_{\tau,\sigma,\theta}}{K_{N}(\rho)}  +\Xi_{\tau,\sigma,\theta},\\
         &\leq  \rho^{N}N \Big(\rho^{N-1} \max\{\kappa_{xy}^2,~\kappa_{yx}^2\} + \frac{1}{\rho N}\Big) \overline{\overline{C}}_{\tau,\sigma,\theta}\mathbb{E}\left[\|z^t_0-z^t_*\|^2 \right]+ \rho^{N-1}\overline{\overline{C}}'_{\tau,\sigma,\theta}  +\Xi_{\tau,\sigma,\theta},
    \end{aligned}
\end{equation*}}%
where \sa{we used the fact that $(1-\rho)/(1-\rho^{N})\leq 1$ and}
\sa{\begin{align*}
\overline{\overline{C}}_{\tau,\sigma,\theta} =  \max\Big\{
      \frac{2}{\tau},
      \frac{(2+2\theta)}{\sigma}
      \Big\} \overline{C}_{\tau,\sigma,\theta},
      \qquad
      \overline{\overline{C}}'_{\tau,\sigma,\theta} =  \max\Big\{
      \frac{2}{\tau},
      \frac{(2+2\theta)}{\sigma}
      \Big\}\overline{C}'_{\tau,\sigma,\theta}.
\end{align*}
Thus, for $C_{\tau,\sigma,\theta} \triangleq \max\{\overline{C}_{\tau,\sigma,\theta},~ \big(\rho^{N-1} \max\{\kappa_{xy}^2,~\kappa_{yx}^2\} + \frac{1}{\rho N}\big)\overline{\overline{C}}_{\tau,\sigma,\theta}\}$ and $C'_{\tau,\sigma,\theta} \triangleq \max\{\overline{C}'_{\tau,\sigma,\theta},~\rho^{N-1}\overline{\overline{C}}'_{\tau,\sigma,\theta} + \Xi_{\tau,\sigma,\theta}\}$, we get the desired result in~\eqref{eq: bounded Gap and disance}.}
\end{proof}

\begin{lemma}
\sa{Under the premise of \cref{lemma: Gap bound with fg}, $\mathbb{E}[\norm{z_0^{t}-z_*^t}^2]$, $\mathbb{E}\left[\mathcal{G}^{t}(z_0^{t})\right]$ and $\mathbb{E}\left[\mathcal{G}^{t}(z_0^{t+1})\right]$ are finite for any $t\geq 0$ when either \cref{assump:compact} or \cref{assump:bounded-subdifferential} holds.}
\end{lemma}
\begin{proof}
\sa{In~\cref{lemma: bounded Gap and disance}, we show that}
\begin{align}
    \label{eq:induction-key}
    \max\Big\{\mathbb{E}\left[\mathcal{G}^{t}(z_0^{t+1})\right],~\mathbb{E}\left[\norm{z_0^{t+1}-z_*^t}^2\right]\Big\}\leq \rho^{N_t} \sa{N_t} C_{\tau,\sigma,\theta} \mathbb{E}[\norm{z_0^{t}-z_*^t}^2] + C'_{\tau,\sigma,\theta},
\end{align}
for some $C_{\tau,\sigma,\theta},C'_{\tau,\sigma,\theta}\in\mathbb{R}_+$ constants, dependent on the SAPD parameters. Next, we show that $\{\mathbb{E}\left[\norm{z_0^t-z_*^t}^2\right]\}<\infty$ for all $t\geq 0$ by induction. This is trivially true for $t=0$, i.e., $\mathbb{E}\left[\norm{z_0^0-z_*^0}^2\right]=\norm{z_0^0-z_*^0}^2<\infty$. Next, for some $t\geq 0$, suppose $\mathbb{E}\left[\norm{z_0^t-z_*^t}^2\right]<\infty$, \eqref{eq:induction-key} implies that
\begin{align}
    \mathbb{E}\left[\norm{z_0^{t+1}-z_*^t}^2\right]<\infty.
    \label{eq:first-bound-induction}
\end{align}
\sa{The inductive assumption  $\mathbb{E}\left[\norm{z_0^t-z_*^t}^2\right]<\infty$ and \eqref{eq:first-bound-induction} imply that
\begin{align}
    \mathbb{E}\left[\norm{z_0^{t+1}-z_0^t}^2\right]\leq 2\mathbb{E}\left[\norm{z_0^{t+1}-z_*^t}^2\right]+2\mathbb{E}\left[\norm{z_0^{t}-z_*^t}^2\right]<\infty.
    \label{eq:intermediate-induction}
\end{align}}%
For any $\mu_x>0$, fix $\lambda=(\mu_x+\gamma)^{-1}$; since we have $x_*^\ell=\prox{\lambda\phi}(x_0^\ell)$ for $\ell=t, t+1$ and $\prox{\lambda\phi}(\cdot)$ is non-expansive, we have
$\mathbb{E}[\norm{x_*^{t+1}-x_*^t}^2]\leq \mathbb{E}[\norm{x_0^{t+1}-x_0^t}^2]$. Moreover,
\sa{
\cref{lem:implicit_function}
implies that $\mathbb{E}[\norm{y_*^{t+1}-y_*^t}^2]\leq \kappa_{yx}^2\mathbb{E}[\norm{x_*^{t+1}-x_*^t}^2]$ for $\kappa_{yx}=\frac{L_{yx}}{\mu_y}$;} thus, \sa{using \eqref{eq:intermediate-induction}, we get}
\begin{align}
    \mathbb{E}[\norm{z_*^{t+1}-z_*^t}^2]
\leq (\kappa_{yx}^2+1)\mathbb{E}[\norm{x_0^{t+1}-x_0^t}^2]\sa{\leq(\kappa_{yx}^2+1)\mathbb{E}[\norm{z_0^{t+1}-z_0^t}^2] <\infty.}
\label{eq:second-bound-induction}
\end{align}
Therefore, we can conclude that $\mathbb{E}[\norm{z_0^{t+1}-z_*^{t+1}}^2]\leq 2\mathbb{E}[\norm{z_0^{t+1}-z_*^{t}}^2] + 2 \mathbb{E}[\norm{z_*^{t+1}-z_*^{t}}^2]<\infty$, which follows from \eqref{eq:first-bound-induction} and \eqref{eq:second-bound-induction}. This completes induction, providing us with $\mathbb{E}[\norm{z_0^{t}-z_*^t}^2]<\infty$ for all $t\geq 0$. Note that using this result together with the definition of $\mathcal{G}^{t}$ and \eqref{eq:induction-key} implies that $0\leq \mathbb{E}[\mathcal{G}^{t}(z_0^{t+1})]<\infty$ for $t\geq 0$.

Next, we will argue that $\mathbb{E}[\mathcal{G}^{t}(z_0^{t})]<\infty$ for all $t\geq 0$ as well. Recall that $\mathcal{G}^{t}(z_0^t)=\sup_{y\in\mathcal{Y}}\mathcal{L}^{t}(x_0^t,y)-\inf_{x\in\mathcal{X}}\mathcal{L}^{t}(x,y_0^t)$; furthermore, note that $\mathcal{L}^{t}(x_0^t,y)=\mathcal{L}(x_0^t,y)$ for all $y\in\mathcal{Y}$, and given $z_0^t$, we have $\mathcal{L}^{t}(\cdot,y_0^t)$ strongly convex with modulus $\mu_x$ and $\mathcal{L}(x_0^t,\cdot)$ strongly concave with modulus $\mu_y$. Therefore, we have
{\small
\begin{align}
    \mathcal{L}(x_0^t,y)&\leq \mathcal{L}(x_0^t,y_0^t)+\fprod{\sa{\nabla_y\Phi(x_0^t,y_0^t)-s_g(y_0^t)},~y-y_0^t}-\frac{\mu_y}{2}\norm{y-y_0^t}^2\nonumber\\
    &\leq \mathcal{L}(x_0^t,y_0^t)+\frac{1}{2\mu_y}\norm{\sa{\nabla_y\Phi(x_0^t,y_0^t)-s_g(y_0^t)}}^2,\label{eq:G-upper}\\
    \mathcal{L}^{t}(x,y_0^t)&\geq \mathcal{L}(x_0^t,y_0^t)+\fprod{\sa{\nabla_x\Phi(x_0^t,y_0^t)+s_f(x_0^t)},~x-x_0^t}\sa{+}\frac{\mu_x}{2}\norm{x-x_0^t}^2\nonumber\\
    &\geq \mathcal{L}(x_0^t,y_0^t)-\frac{1}{2\mu_x}\norm{\sa{\nabla_x\Phi(x_0^t,y_0^t)+s_f(x_0^t)}}^2,\label{eq:G-lower}
\end{align}}%
where \sa{$s_f(x_0^t)\in\partial f(x_0^t)$ and $s_g(y_0^t)\in\partial g(y_0^t)$ such that $\norm{s_f(x_0^t)}\leq B_f$ and $\norm{s_g(y_0^t)}\leq B_g$ --see \cref{assump:bounded-subdifferential}; moreover,} we have used the fact that $\mathcal{L}^{t}(x_0^t,y_0^t)=\mathcal{L}(x_0^t,y_0^t)$ and \sa{$\partial_x\mathcal{L}^{t}(x_0^t,y_0^t)=\nabla_x\Phi(x_0^t,y_0^t)+\partial f(x_0^t)$}. Thus, \eqref{eq:G-upper} and \eqref{eq:G-lower} imply that
\begin{align*}
    \mathcal{G}^{t}(z_0^t)
    &=\sup_{\substack{x\in\mathcal{X}, y\in\mathcal{Y}}}\{\mathcal{L}(x_0^t,y)-\mathcal{L}^{t}(x,y_0^t)\}\leq \sa{\left(\norm{\nabla\Phi(z_0^t)}^2+\norm{s_f(x_0^t)}^2+\norm{s_g(y_0^t)}^2\right)}/\min\{\mu_x,\mu_y\}\\
    &\leq \sa{\frac{1}{\mu}\left(\norm{\nabla\Phi(z_0^t)-\nabla\Phi(z_0^0)}^2+\norm{\nabla\Phi(z_0^0)}^2+B_f^2+B_g^2\right)}\\
    &\leq \frac{L}{\mu}\norm{z_0^t-z_0^0}^2+\sa{\frac{1}{\mu}\left(\norm{\nabla\Phi(z_0^0)}^2+B_f^2+B_g^2\right)},
\end{align*}
where $L=\max\{L_{xx},L_{yy},L_{yx},L_{xy}\}$ and $\mu=\min\{\mu_x,\mu_y\}$. \sa{Finally,  \eqref{eq:intermediate-induction} implies that $\mathbb{E}[\norm{z_0^t-z_0^0}^2]<\infty$; therefore, we can conclude that $\mathbb{E}[\mathcal{G}^{t}(z_0^t)]<\infty$ for all $t\geq 0$.}
\end{proof}

\section{Bounds for smooth WCSC problems}
\label{sec:smooth-problems}
\begin{assumption}\label{ASPT: non-fg}
\sa{$f\equiv 0$, $g\equiv0$; hence, $\mathcal{L}(x,y)=\Phi(x,y)$ for all $x\in\mathcal{X}$ and $y\in\mathcal{Y}$.}
\end{assumption}
\begin{lemma}\label{lemma: Gap bound non-fg}
\sa{Consider the setting of \cref{lemma: Gap bound with fg}, and suppose Assumption~\ref{ASPT: fg} is replaced with Assumption~\ref{ASPT: non-fg}. Then \eqref{eq:Gap bound with fg} holds with ${\Xi}_{\tau,\sigma,\theta}=\overline{\Xi}_{\tau,\sigma,\theta}$,}
where  $\overline{\Xi}_{\tau,\sigma,\theta} ~\triangleq \tau~\frac{3}{2}\delta_x^2 + \sigma\big(\frac{3}{2}+4\theta+3\theta^2\big)\delta_y^2$.
\end{lemma}
\begin{proof}
See \cref{PF:Gap bound non-fg}.
\end{proof}
\begin{theorem}
\label{thm:WC_SP non-fg}
Under the premise of Lemma~\ref{lemma: Gap bound non-fg}, given $T\in\mathbb{Z}_+$, suppose $N_t=N$ for all $t=0,\ldots T$ for some $N\in\mathbb{Z}_+$ such that $K_{N}(\rho) \geq (1+\zeta) M_{\tau,\sigma,\theta}$ for some $\zeta>0$, \eqref{eq: additional stepsize condition WCSC}
has a solution for some $\beta_{1},\beta_{2}\in(0,1)$ and $p_1,p_2,p_3>0$ such that $p_1+p_2+p_3=1$. Then, \eqref{eq: sketch convergence result} holds with ${\Xi}_{\tau,\sigma,\theta}=\overline{\Xi}_{\tau,\sigma,\theta}$ for $\lambda=(\gamma+\mu_x)^{-1}$ and for all $T\geq 1$.
\end{theorem}
\begin{proof}
The proof is omitted as it is essentially the same with the proof of \cref{thm:WC_SP}.
\end{proof}
\begin{theorem}
\sa{Suppose \cref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold. Let $\mu_x=\gamma$, $\{\tau,\sigma,\theta\}$ and $N$ are set} as in \cref{cor:complexity-fg} for $\overline{\overline{\theta}}_1  = \max\{0,1 - \frac{1}{\sa{216}}\cdot\frac{\epsilon^2}{\delta_x^2}
\}$ and $\overline{\overline{\theta}}_2 = \big(1+ \frac{1}{306}\cdot\frac{\hat{\mu}_y}{4\gamma}\cdot\frac{\epsilon^2}{\delta_y^2}\big)^{-1}$.
Then,
the conclusion of \cref{cor:complexity-fg} continue to hold.
\end{theorem}
\begin{proof}
\sa{The proof is omitted as it is essentially the same with the proof of \cref{cor:complexity-fg}.}
\end{proof}

\subsection{Proof of Lemma~\ref{lemma: Gap bound non-fg}}\label{PF:Gap bound non-fg}
\sa{In order to} prove \cref{lemma: Gap bound non-fg}, we first provide
\sa{a technical result.} In fact, suppose \cref{ASPT: non-fg} hold, and for any $t\geq 0$, let $\{x_k^t,y_k^t\}_{k\geq 0}$ be \xz{\texttt{SAPD}}  iterates generated according to algorithm~\ref{ALG: SAPD} for solving $\min_x\max_y\mathcal{L}^{t}(x,y)$, i.e., for $k\geq 0$,
$$
y^{t}_{k+1} = y^{t}_{k} + \sigma \tilde{s}^{t}_{k},\quad x^{t}_{k+1} = x^{t}_{k} - \tau \tilde{\nabla}\Phi(x^{t}_{k},y^{t}_{k+1}).
$$
Using the above
simple update rule --due to \cref{ASPT: non-fg}, we
obtain a refined variance bound.
\begin{lemma}\label{Lemma:noisy bound non-fg}
\xz{Suppose \cref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold.} Let $\{x_k^t,y_k^t\}_{k\geq 0}$ be \xz{\texttt{SAPD}}  iterates generated according to algorithm~\ref{ALG: SAPD} for solving $\min_x\max_y\mathcal{L}^{t}(x,y)$.
The following inequalities hold for all $k\geq 0$:
{\footnotesize
\begin{equation*}
   \begin{aligned}
    \mathbb{E}\left[ \langle \Delta^{\Phi^t,x}_k ,    x^{t}_{k+1} \rangle \right] \leq \tau\delta_x^2,
    \quad
    \mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_k ,   y^{t}_{k+1}\rangle \right] \leq \sigma(1+\theta)\delta_y^2,
     \quad
    \mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_{k-1} ,   y^{t}_{k+1}\rangle \right]
    \leq
    \sigma(1+2\theta)\delta_y^2,
    \end{aligned}
\end{equation*}}
where
{\small
    $\Delta^{\Phi^t,x}_k$, $\Delta^{\Phi^t,y}_k$ are defined in \cref{lemma: intermediate noisy bound}.
}
\end{lemma}
\begin{proof}
With the convention that $y_{-2} = y_{-1} = y_0$, and $x_{-2} = x_{-1} = x_0$, \cref{ASPT: unbiased noise assumption}
implies that
\begin{equation*}
    \begin{aligned}
        &\mathbb{E}\left[ \langle \Delta^{\Phi^t,x}_k ,    x^{t}_{k+1} \rangle \right] = \mathbb{E}\left[ \langle \Delta^{\Phi^t,x}_k ,   x^{t}_{k} - \tau\tilde{\nabla}\Phi(x^{t}_{k},y^{t}_{k+1}) \rangle \right] \leq \tau \mathbb{E}\left[\|  \Delta^{\Phi^t,x}_k\|^2\right]\leq \tau\delta_x^2,
        \\
        &\mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_k ,   y^{t}_{k+1}\rangle \right] = \mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_k ,   y^{t}_{k} + \sigma \tilde{s}^{t}_k \rangle \right] \leq \sigma(1+\theta)\mathbb{E}\left[\|  \Delta^{\Phi^t,y}_k\|^2\right] \leq \sigma(1+\theta)\delta_y^2
        \\
        & \mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_{k-1} ,   y^{t}_{k+1}\rangle \right] = \mathbb{E}\left[ \langle \Delta^{\Phi^t,y}_{k-1} ,   y^{t}_{k-1}+\sigma\tilde{s}^{t}_{k-1} + \sigma \tilde{s}^{t}_k \rangle \right] \leq  \sigma(1+2\theta)\mathbb{E}\left[\| \Delta^{\Phi^t,y}_{k-1}\|^2\right]
        \leq
        \sigma(1+2\theta)\delta_y^2,
    \end{aligned}
\end{equation*}
hold for all $k\geq 0$, where we use the fact $\mathbb{E}\left[ \langle \Delta^{\Phi,y}_{k_1} ,    \Delta^{\Phi,y}_{k_2} \rangle \right]=0$ for any $k_1>k_2\geq0$.
\end{proof}
Now, we are ready to prove \cref{lemma: Gap bound non-fg}. The proof \sa{directly follows from} the proof of Lemma~\ref{lemma: Gap bound with fg} provided in \cref{PF: lemma Gap bound with fg}, \sa{except for the  way we bound $\mathbb{E}\left[Q^{t}_k\right]$, which is used in} \cref{eq:variance-bound}. \sa{Recall the definition of $Q_k^t$ given in~\eqref{eq:Qk-def}, i.e.,}
\begin{equation}
\begin{aligned}
            Q^{t}_k & =
     \langle \Delta^{\Phi^t,x}_k , {\hat{x}^{t}_{k+1} -  x^{t}_{k+1}} \rangle
     +
     (1+\theta)\langle \Delta^{\Phi^t,y}_k, {y^{t}_{k+1} - \hat{y}^{t}_{k+1}}  \rangle
     -\theta \langle \Delta^{\Phi^t,y}_{k-1}, {y^{t}_{k+1} - \hat{\hat{y}}^{t}_{k+1}}  \rangle
   \\
     & +
     \frac{1}{2{\eta_x}}\|\Delta^{\Phi^t,x}_k \|^2
     +
     \frac{1+\theta}{2{\eta_y}}\| \Delta^{\Phi^t,y}_k\|^2
     +
     \frac{\theta}{2{\eta_y}}\|\Delta^{\Phi^t,y}_{k-1}\|^2.
\end{aligned}
\end{equation}
From \cref{ASPT: unbiased noise assumption}, for
$k\geq-1$, we obtain that $\mathbb{E}\left[\langle \Delta^{\Phi^t,x}_k , \hat{x}^{t}_{k+1} \rangle \right] = 0$, $\mathbb{E}\left[\langle \Delta^{\Phi^t,y}_k,  \hat{y}^{t}_{k+1} \rangle \right] = 0$, and $\mathbb{E}\left[\langle \Delta^{\Phi^t,y}_{k-1}, \hat{\hat{y}}^{t}_{k+1} \rangle \right] = 0$; \sa{furthermore, again from} \cref{ASPT: unbiased noise assumption}, we have
$
\mathbb{E}\left[ \|\Delta^{\Phi^t,x}_k\|^2 \right] \leq \delta_x^2,\quad \mathbb{E}\left[ \|\Delta^{\Phi^t,y}_k\|^2 \right] \leq \delta_y^2,$
\sa{for $k\geq -1$.}
Then using \cref{Lemma:noisy bound non-fg} and the above relations and setting $\eta_x=\frac{1}{\tau},\eta_y = \frac{1}{\sigma}$, we obtain that
$$\mathbb{E}\left[Q^{t}_k\right]\leq \overline{\Xi}_{\tau,\sigma,\theta}= \frac{3\tau}{2}\delta_x^2 + \sigma\big(\frac{3}{2}+4\theta+3\theta^2\big)\delta_y^2.$$
\sa{The rest of proof} is the same with \cref{PF: lemma Gap bound with fg}.

\section{\sa{Proof of Theorem~\ref{cor:complexity VR} and preliminary technical results}}
\label{sec:VR-proofs}
\subsection{Construction for the iteration complexity result}
\begin{lemma}\label{lemma: Gap bound with fg SPIDER}
Suppose \cref{ASPT: fg},\ref{ASPT: unbiased noise assumption},\ref{ASPT: lipshiz gradient VR} hold.
\sa{Given $\{N_t\}_{t\geq 0}\subset\mathbb{Z}_+$, let $\{x^{t}_{0},y^{t}_{0}\}_{t\geq 0}$ be generated by \xz{\texttt{SAPD+}}, stated in~algorithm~\ref{Alg: SAPD-plus}, when \texttt{VR-flag}=\textbf{true}, initialized from {$(x_0^0,y_0^0)\in\mathop{\bf dom} f\times\mathop{\bf dom} g$}
and using} $\tau,\sigma,\theta,\mu_x$>0 that satisfy below inequality {for some} $\alpha \in [0,  {\tfrac{1}{\sigma})}$, $\rho \in (0,1]$ and \sa{$\pi_x,\pi_y>0$,}

\begin{equation}
    \label{eq: SCSC SAPD LMI SPIDER}
    \sa{G-\mathop{\bf diag}(g)\succeq 0,}
\end{equation}}%
\sa{where $G$ is defined in~\eqref{eq: SCSC SAPD LMI}, $g\triangleq [\pi_x,\pi_y,L_x',L_y',0]^\top$} and
\begin{align*}
L_x' \triangleq \sa{c(\rho)}~\big( \frac{ \xz{{L'_{xx}}^2}}{\xz{\pi_x}b_x} + \frac{\xz{2}(1+2\theta+2\theta^2)\rho^{-1} L^2_{yx}}{\xz{\pi_y}b_y}\big), \;
L_y' \triangleq c(\rho)~\big(\frac{\rho L^2_{xy}}{\xz{\pi_x}b_x} +  \frac{\xz{2}(1+2\theta+2\theta^2)\rho^{-1} L^2_{yy}}{\xz{\pi_y}b_y}\big),

\end{align*}
such that $c(\rho)=\frac{2}{1-\rho}(\rho^{-q+1}-1)$ for $\rho\in(0,1)$ and $c(\rho)=2(q-1)$ for $\rho=1$.
Then for all $t\geq {0}$, \eqref{eq:Gap bound with fg} holds with ${\Xi}_{\tau,\sigma,\theta}=\Xi^{\texttt{VR}}$ and \sa{$M_{\tau,\sigma,\theta}=M^{\texttt{VR}}$},
where $\Xi^{\texttt{VR}}\triangleq \frac{\delta^2_x}{2\pi_x b}+(1+2\theta+2\theta^2)\frac{\delta^2_y}{\pi_y b}$ and $M^{\texttt{VR}}\triangleq\max\{\frac{2}{\mu_x}(\frac{1}{\tau}-\mu_x),\frac{2}{\mu_y\sigma}\}$.
\end{lemma}
\begin{proof}
For easier readability, we provide the proof in a separate subsection, see~\cref{sec:spider-proof}.
\end{proof}

\begin{theorem}
\label{thm:WC_SP SPIDER}
\sa{Under the premise of Lemma~\ref{lemma: Gap bound with fg SPIDER}, given an arbitrary $\zeta>0$ and $T\in\mathbb{Z}_+$, suppose $N_t=N$ for all $t=0,\ldots T$ for some $N\in\mathbb{Z}_+$ such that $K_{N}(\rho) \geq (1+\zeta) M^{\texttt{VR}}$, and \eqref{eq: additional stepsize condition WCSC}
has a solution for some $\beta_{1},\beta_{2}\in(0,1)$ and $p_1,p_2,p_3>0$ such that $p_1+p_2+p_3=1$. If either \cref{assump:compact} or \cref{assump:bounded-subdifferential} holds, then \eqref{eq: sketch convergence result} holds with ${\Xi}_{\tau,\sigma,\theta}=\Xi^{\texttt{VR}}$ for $\lambda=(\gamma+\mu_x)^{-1}$ and for all $T\geq 1$.}
\end{theorem}
\begin{proof}
\sa{The proof is omitted as it is essentially the same with the proof of \cref{thm:WC_SP}.}
\end{proof}
\subsection{Proof of Lemma~\ref{lemma: Gap bound with fg SPIDER} and preliminary technical results}
\label{sec:spider-proof}
\sa{In this section we prove \cref{lemma: Gap bound with fg SPIDER}.} We first state a technical lemma that will be used in our analysis.

\begin{lemma}\label{Lemma: SPIDER}
Suppose \cref{ASPT: fg}, \ref{ASPT: unbiased noise assumption},  \ref{ASPT: lipshiz gradient VR} hold. Let $\{x_k^t,y_k^t\}_{k\geq 0}$ be \xz{\texttt{VR-SAPD}} iterates generated according to algorithm~\ref{ALG: SAPD-VR} for solving $\min_x\max_y\mathcal{L}^{t}(x,y)$.
\sa{Then,
}
{\small
\begin{subequations}
\label{eq:variance-bound-VR}
    \begin{align}
        &\ED{v_{k}^{t}- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}\leq \frac{\delta^2_x}{b}+ \sum_{i=(n_k-1)q+1}^{k}\frac{2\xz{{L'_{xx}}^2}}{\xz{b_x}}\ED{x^{t}_{i}-x^{t}_{i-1}}+
        \frac{2L^2_{xy}}{\xz{b_x}}\ED{y^{t}_{i+1}-y^{t}_{i}}
        \\
        & \ED{w_{k}^{t}- \nabla_y\Phi^t(x^{t}_{k},y^{t}_{k})}\leq  \frac{\delta^2_y}{b}+\sum_{i=(n_k-1)q+1}^{k}\frac{2L^2_{yx}}{\xz{b_y}}\ED{x^{t}_{i}-x^{t}_{i-1}}+
        \frac{2L^2_{yy}}{\xz{b_y}}\ED{y^{t}_{i}-y^{t}_{i-1}},\label{ineq: basic for wk}
    \end{align}
\end{subequations}}%
\sa{for all $k\geq 0$ such that $\mo(k,q)\neq 0$, where $n_k \triangleq \lceil k/q \rceil$.} Moreover, if $\mo(k,q)=0$, then
\begin{equation}
        \ED{v_{k}^{t}- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}\leq   \frac{\delta^2_x}{b},\quad
         \ED{w_{k}^{t}- \nabla_y\Phi^t(x^{t}_{k},y^{t}_{k})}\leq \frac{\delta^2_y}{b}.
\end{equation}
\end{lemma}
\begin{proof}
Recall that $\tilde\nabla_x\Phi_{I^x_k}^t(x^{t}_{k},y^{t}_{k+1}) \sa{\triangleq} \frac{1}{|I^x_k|} \sum_{\omega_{k}^{x,i}\in  I^x_k} \tilde\nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1};\omega_{k}^{x,i})$, where $I_{k}^{x}=\{\omega_{k}^{x,i}\}_{i=1}^{b_x}$ is \sa{a randomly generated batch with $|I_{k}^{x}|=b_x$ independent elements which are also independent of $(x^t_{k-1},y^t_{k})$ and $(x^t_k,y^t_{k+1})$.}
According to the definition of \xz{$v_k$} in \cref{ALG: SAPD-VR}, for $\mo(k,q)>0$,
\begin{equation}
    v_{k}^{t}  =  v_{k-1}^{t} + \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k},y^{t}_{k+1})-\tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k-1},y^{t}_{k}).
\end{equation}
\sa{Therefore,}
{\small
\begin{equation}\label{ineq: basic for vk}
    \begin{aligned}
        &\ED{v_{k}^{t}- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}
        \\
        &=  \ED{v_{k-1}^{t} + \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k},y^{t}_{k+1})-\tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k-1},y^{t}_{k})- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}
        \\
        &=  \ED{v_{k-1}^{t}- \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k})  + \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k}) - \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k-1},y^{t}_{k})+ \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k},y^{t}_{k+1})- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}
        \\
        &=  \ED{v_{k-1}^{t}- \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k})}\\
        &\quad+\ED{ \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k}) - \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k-1},y^{t}_{k})+ \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k},y^{t}_{k+1})- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})},

    \end{aligned}
\end{equation}}
where for the last equality we used $$\mathbb{E}\left[\nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k}) - \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k-1},y^{t}_{k})+ \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k},y^{t}_{k+1})- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})\right]=0.$$
Next, we bound the second expectation on the rhs of \eqref{ineq: basic for vk}. It follows that
{\small
\begin{equation}\label{ineq: basic for vk 2}
    \begin{aligned}
           & \ED{ \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k}) - \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k-1},y^{t}_{k})+ \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k},y^{t}_{k+1})- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}
           \\
           = & \frac{1}{b_x^2}\mathbb{E}\left[
           \|
           \sum_{i=1}^{b_x}\big(
           \tilde\nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1};\omega_{k}^{x,i})- \tilde\nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k};\omega_{k}^{x,i}) -\nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})
           + \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k})
           \big)
           \|^2
           \right]
           \\
           = & \frac{1}{b_x^2}\sum_{i=1}^{b_x}\mathbb{E}\left[
           \|
           \tilde\nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1};\omega_{k}^{x,i})- \tilde\nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k};\omega_{k}^{x,i}) -\nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})
           + \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k})
           \|^2
           \right]
           \\
           \leq & \frac{1}{b_x^2}\sum_{i=1}^{b_x}\big(\mathbb{E}\left[
           \|
           \tilde\nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1};\omega_{k}^{x,i})- \tilde\nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k};\omega_{k}^{x,i}) \|^2\right]

           \big)
           \\
            \leq &  \frac{2\xz{{L'_{xx}}^2}}{b_x}\ED{x^{t}_{k}-x^{t}_{k-1}}+\frac{2L^2_{xy}}{b_x}\ED{y^{t}_{k+1}-y^{t}_{k}},
     \end{aligned}
\end{equation}}%
where the second equality \sa{follows from the stochastic oracle being unbiased --see \cref{ASPT: unbiased noise assumption}, which implies $$\mathbb{E}\left[\nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k}) - \tilde\nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k};\omega_{k}^{x,i})+ \tilde\nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1};\omega_{k}^{x,i})- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})\right]=0,$$
for all $i=1,\ldots,b_x$ and $\{\omega_i^k\}_{i=1}^{b_x}$ being independent}; the first inequality is because $\ED{\zeta-\mathbb{E}[\zeta]}\leq \ED{\zeta}$ for \sa{any} given random variable $\zeta$ with finite second order moment \sa{--we invoke this inequality for} $\zeta = \tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k},y^{t}_{k+1};\omega_{k}^{x,i})-\tilde\nabla_x\Phi_{I_{k}^{x}}^t(x^{t}_{k-1},y^{t}_{k};\omega_{k}^{x,i})$; \sa{and finally, the last inequality follows from \cref{ASPT: lipshiz gradient VR} and the inequality $(a+b)^2\leq 2a^2+2b^2$ for any $a,b\in\mathbb{R}$.}
Next, if we combine \cref{ineq: basic for vk} and \cref{ineq: basic for vk 2}, we get
\begin{equation*}
    \begin{aligned}
            &\ED{v_{k}^{t}- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}
            \\
          \leq & \ED{v_{k-1}^{t}- \nabla_x\Phi^t(x^{t}_{k-1},y^{t}_{k})} +\frac{2\xz{{L'_{xx}}^2}}{b_x}\ED{x^{t}_{k}-x^{t}_{k-1}}+\frac{2L^2_{xy}}{b_x}\ED{y^{t}_{k+1}-y^{t}_{k}}.
    \end{aligned}
\end{equation*}
\sa{Hence, if we sum the above inequality from $(n_k-1)q+1$ to $k$, we get a telescoping sum:}
\begin{equation}\label{ineq: basic for vk 3}
    \begin{aligned}
        &\ED{v_{k}^{t}- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}
        \\
        & \leq    \sum_{i=(n_k-1)q+1}^{k}\frac{2\xz{{L'_{xx}}^2}}{\xz{b_x}}\ED{x^{t}_{i}-x^{t}_{i-1}}+ \sum_{i=(n_k-1)q+1}^{k}\frac{2L^2_{xy}}{\xz{b_x}}\ED{y^{t}_{i+1}-y^{t}_{i}}
        \\
        & +
        \ED{v_{(n_k-1)q}-\nabla_x\Phi^t(x^{t}_{(n_k-1)q}, y^{t}_{(n_k-1)q+1})}
        \\
        & \leq  \sum_{i=(n_k-1)q+1}^{k}\frac{2\xz{{L'_{xx}}^2}}{\xz{b_x}}\ED{x^{t}_{i}-x^{t}_{i-1}}+ \sum_{i=(n_k-1)q+1}^{k}\frac{2L^2_{xy}}{\xz{b_x}}\ED{y^{t}_{i+1}-y^{t}_{i}} + \frac{\delta^2_x}{b},
    \end{aligned}
\end{equation}
where \sa{the last inequality follows from \cref{ASPT: unbiased noise assumption} since $\mo((n_k-1) q,q)=0$ and for $\ell\in\mathbb{Z}_+$ such that $\mo(\ell,q)=0$, we have $v_{\ell} = \tilde\nabla_x\Phi^{t}_{\mathcal{B}_\ell}(x^{t}_{\ell},y^{t}_{\ell+1})=\frac{1}{|\mathcal{B}_\ell|} \sum_{\omega_\ell^i\in  \mathcal{B}_\ell} \tilde\nabla_x\Phi^t(x^{t}_{\ell},y^{t}_{\ell+1};\omega^i_\ell)$, where $\mathcal{B}_\ell=\{\omega^i_\ell\}$ is a randomly generated batch with $|B_\ell|=b$ independent elements which are also independent of $(x^t_{\ell},y^t_{\ell+1})$. This completes the proof of the case for $k$ such that $\mo(k,q)>0$.}

When $\mo(k,q) = 0$, it follows from \cref{ALG: SAPD-VR} that $v_{k} = \tilde\nabla_x\Phi^{t}_{\mathcal{B}_k}(x^{t}_{k},y^{t}_{k+1})$.
\sa{Hence, above discussion yields}
\begin{equation}
    \ED{v_{k}^{t}- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})} \leq \frac{\delta^2_x}{b}.
\end{equation}
\sa{Finally, the second inequality in~\eqref{eq:variance-bound-VR} can be shown similarly.}
\end{proof}

Next, we will modify \cref{lem: basic lemma for SCSC} for \xz{\texttt{VR-SAPD}}, stated in algorithm~\ref{ALG: SAPD-VR}.
Specifically, instead of using the stochastic oracles $\tilde{\nabla}_x\Phi^t(x^{t}_{k},y^{t}_{k+1};\omega_k^x)$ and $\tilde{\nabla}_y\Phi^t(x^{t}_{k},y^{t}_{k};\omega_k^y)$ as in \cref{lem: basic lemma for SCSC}, we adopt $v_{k}^{t}$ and $w_{k}^{t}$ to estimate ${\nabla}_x\Phi^t(x^{t}_{k},y^{t}_{k+1})$ and ${\nabla}_y\Phi^t(x^{t}_{k},y^{t}_{k})$, respectively.
\begin{lemma}\label{lem: basic lemma for SCSC VR}
\xz{Suppose \cref{ASPT: fg}, \ref{ASPT: unbiased noise assumption},  \ref{ASPT: lipshiz gradient VR} hold.} Let $\{x_k^t,y_k^t\}_{k\geq 0}$ be \xz{\texttt{VR-SAPD}} iterates generated according to algorithm~\ref{ALG: SAPD-VR} \sa{for solving $\min_x\max_y\mathcal{L}^{t}(x,y)$}.
Then
for all $x\in\mathop{\bf dom} f\subset \mathcal{X}$, $y\in\mathop{\bf dom} g\subset \mathcal{Y}$, and $k\geq 0$,
{\small
\begin{equation}\label{D1 VR}
    \begin{aligned}
       \mathcal{L}^{t}( & x^{t}_{k+1}, y)  - \mathcal{L}^{t}(x, y^{t}_{k+1})
        \\
    \leq
    &-\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle + \theta \langle q^{t}_{k}, y^{t}_{k} - y \rangle
    + \Lambda^{t}_k(x,y) - \Sigma^{t}_{k+1}(x,y)+ \Gamma^{t}_{k+1}+\sa{\varepsilon^{t,x}_{k}(x)+\varepsilon^{t,y}_{k}(y)},
\end{aligned}
\end{equation}}%
where $\sa{\varepsilon^{t,x}_k(x)}\triangleq
       \langle v_{k}^{t} - \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}),~x-x^{t}_{k+1} \rangle$ and $\sa{\varepsilon^{t,y}_{k}(y)}\triangleq\langle \tilde{s}^{t}_k -s^{t}_{k}, y^{t}_{k+1} - y \rangle$ for \sa{$\tilde{s}_k^t = (1+\theta) w_{k}^{t} - \theta w_{k-1}^t$} as defined in algorithm~\ref{ALG: SAPD-VR},
       $q^{t}_{k}$ and $s^{t}_{k}$ are defined as in \eqref{eq:qksk}, and $\Lambda^{t}_{k}(x,y)$, $\Sigma^{t}_{k+1}(x,y)$, $\Gamma^{t}_{k+1}$ are the same with those in \cref{lem: basic lemma for SCSC}.
\end{lemma}
\begin{proof}
\sa{The proof uses the same arguments as the proof of \cref{lem: basic lemma for SCSC}. One only needs to replace $\tilde{\nabla}_x\Phi^t(x^{t}_{k},y^{t}_{k+1})$ and $\tilde{\nabla}_y\Phi^t(x^{t}_{k},y^{t}_{k})$ in the proof of \cref{lem: basic lemma for SCSC} with $v_{k}^{t},w_{k}^{t}$, respectively.}
\end{proof}

\subsection{Proof of Lemma~\ref{lemma: Gap bound with fg SPIDER}}\label{PF: lemma Gap bound with fg SPIDER}
\begin{proof}
Fix arbitrary $(x,y)\in\mathop{\bf dom} f\times \mathop{\bf dom} g$. Since $(x^{t}_{k+1},y^{t}_{k+1})\in\mathop{\bf dom} f\times \mathop{\bf dom} g$, using the concavity
of $\mathcal{L}^{t}(x^{t}_{k+1}, \cdot)$
and the convexity
of $\mathcal{L}^{t}(\cdot, y^{t}_{k+1})$, Jensen's lemma immediately implies that
{\small
\begin{align}
\label{eq:jensen bounded version SPIDER}
 K_{N}(\rho)  \left(\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}^{t}(x, \bar{y}^{t}_{N}) \right)
            \leq
            \sum_{k=0}^{N-1}\rho^{-k} \left( \mathcal{L}^{t}(x^{t}_{k+1}, y) - \mathcal{L}^{t}(x, y^{t}_{k+1}) \right),
            \;\sa{\forall}\rho\in \sa{(0,1]},
\end{align}}%
where $\bar{x}^{t}_N=\frac{1}{K_N(\rho)}\sum_{k=0}^{N-1}\sa{\rho^{-k}}x^{t}_{k+1},~\bar{y}^{t}_N=\frac{1}{K_N(\rho)}\sum_{k=0}^{N-1}\sa{\rho^{-k}}y^{t}_{k+1}$. Thus, if we multiply both sides of
\eqref{D1 VR} by $\rho^{-k}$ and sum the resulting inequality from $k=0$ to $N-1$, then using \eqref{eq:jensen bounded version SPIDER} we get
{\footnotesize
\begin{equation}\label{INEQ: difference of gap SPIDER}
    \begin{aligned}
            K_{N} &(\rho)  \left(\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}(x, \bar{y}^{t}_{N}) \right)  \\
            \leq &
         \sum_{k=0}^{N-1}\rho^{-k} \Big(
                -\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle + \theta \langle q^{t}_{k}, y^{t}_{k} - y \rangle + \Lambda^{t}_{k} \sa{(x,y)} - \Sigma^{t}_{k+1} \sa{(x,y)}+ \Gamma^{t}_{k+1}
            \\
             & \quad -
           \langle  v_{k}^{t} - \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}) , x^{t}_{k+1} - x \rangle
            +
           \langle \tilde{s}^{t}_k -s^{t}_k,
          y^{t}_{k+1} - y \rangle
          \Big)
          \\
        \leq &
         \sum_{k=0}^{N-1}\rho^{-k} \Big(
                -\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle + \theta \langle q^{t}_{k}, y^{t}_{k} - y \rangle + \Lambda^{t}_{k} \sa{(x,y)} - \Sigma^{t}_{k+1} \sa{(x,y)}+ \Gamma^{t}_{k+1}
            \\
             & \quad \sa{+}
           \frac{1}{2\pi_x}\| v_{k}^{t} - \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}) \|^2 + \frac{\pi_x}{2} \|x^{t}_{k+1} - x \|^2
            + \frac{1}{2\pi_y}\|\tilde{s}^t_k-s^t_k\|^2 + \frac{\pi_y}{2}\|y^{t}_{k+1}-y\|^2
          \Big)
          \\
        = &
         \sum_{k=0}^{N-1}\rho^{-k} \Big(
                \underbrace{-\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle + \theta \langle q^{t}_{k}, y^{t}_{k} - y \rangle}_{\textbf{part 1}} +
                \Lambda^{t}_{k} \sa{(x,y)} - \Sigma^{t}_{k+1} \sa{(x,y)}
                + \Gamma^{t}_{k+1}
            \\
             & \quad  + \frac{\pi_x}{2} \|x^{t}_{k+1} - x \|^2 + \frac{\pi_y}{2}\|y^{t}_{k+1}-y\|^2 +
          \underbrace{\frac{1}{2\pi_x}\| v_{k}^{t} - \nabla_x \Phi^{t}(x^{t}_{k}, y^{t}_{k+1}) \|^2
            + \frac{1}{2\pi_y}\|\tilde{s}^t_k-s^t_k\|^2 }_{\textbf{part 2}}
          \Big).
    \end{aligned}
\end{equation}}%
The second inequality follows from Young's inequality for some constants $\pi_x,\pi_y>0$.

\sa{The following bound for \textbf{part1} can be obtained from \eqref{eq:noisy-rate-part1}. Indeed, for any $k \geq -1$, we get}
{\small
\begin{align}
\label{eq:noisy-rate-part1 SPIDER}
         \sum_{k=0}^{N-1}&\rho^{-k}
                 ( \theta \langle q^{t}_k, y^{t}_{k} - y \rangle-\langle q^{t}_{k+1}, y^{t}_{k+1} - y \rangle )

             \leq \sum_{k=0}^{N-1}\rho^{-k}|1-\frac{\theta}{\rho}|~S^{t}_{k+1}
                 + \rho^{-N+1}\frac{\theta}{\rho}S^{t}_{N}.
\end{align}}%
where \sa{$S_{k+1}^t$ is defined in \eqref{def: Q_k}.}

Next we consider \textbf{part 2}, recall that $n_k=\lceil k / q \rceil$ such that $(n_k-1)q+1\leq k \leq n_k q$, it follows from \cref{Lemma: SPIDER} that
\begin{equation}\label{eq:noisy-rate-part2.1 SPIDER}
    \begin{aligned}
        &\sum_{k=\sa{0}}^{N-1}\rho^{-k}\ED{v_{k}^{t}- \nabla_x\Phi^t(x^{t}_{k},y^{t}_{k+1})}
        \\&\leq\sum_{\substack{k\in\{1,\ldots,N-1\}\\\text{s.t.}\ \mo(k,q)\neq 0}}\rho^{-k} \sum_{i=(n_k-1)q+1}^{k}\big(\frac{2\xz{{L'_{xx}}^2}}{\xz{b_x}}\ED{x^{t}_{i}-x^{t}_{i-1}}+ \frac{2L^2_{xy}}{\xz{b_x}}\ED{y^{t}_{i+1}-y^{t}_{i}}\big) + \frac{\delta^2_x}{b}\sum_{k=0}^{N-1}\rho^{-k}
        \\ &\sa{=} \sum_{k=1}^{N-1} \rho^{-k} \big(\frac{2\xz{{L'_{xx}}^2}}{\xz{b_x}}\ED{x^{t}_{k}-x^{t}_{k-1}}+  \frac{2L^2_{xy}}{\xz{b_x}}\ED{y^{t}_{k+1}-y^{t}_{k}} \big) \sa{\sum_{i=0}^{n_kq-k-1}\rho^{-i}}+ \frac{\delta^2_x}{b}\sum_{k=0}^{N-1}\rho^{-k}
        \\ &\sa{=} \sum_{k=0}^{N-1} \rho^{-k}\cdot\frac{\rho}{1-\rho}(\rho^{-n_kq+k}-1)\big(\frac{2\xz{{L'_{xx}}^2}}{\xz{b_x}}\ED{x^{t}_{k}-x^{t}_{k-1}}+ \frac{2L^2_{xy}}{\xz{b_x}}\ED{y^{t}_{k+1}-y^{t}_{k}}\big) + \frac{\delta^2_x}{b}\sum_{k=0}^{N-1} \rho^{-k}
        \\ &\leq \sum_{k=0}^{N-1} \rho^{-k}\cdot\frac{\rho}{1-\rho}(\rho^{-q+1}-1)\big(\frac{2\xz{{L'_{xx}}^2}}{\xz{b_x}}\ED{x^{t}_{k}-x^{t}_{k-1}}+ \frac{2L^2_{xy}}{\xz{b_x}}\ED{y^{t}_{k+1}-y^{t}_{k}}\big) + \frac{\delta^2_x}{b}\sum_{k=0}^{N-1} \rho^{-k}
    \end{aligned}
\end{equation}
\sa{where the first inequality \sa{follows from} \cref{Lemma: SPIDER}, the following equality is by rearranging terms, and for the last inequality we used the following bound: $n_k=\lceil k / q \rceil\leq k/q + (q-1)/q$; hence, $-n_k q+k\geq -q+1$.}
\sa{To bound \textbf{part\;2} in \cref{INEQ: difference of gap SPIDER}, we next consider $\|\tilde s^{t}_k-s^{t}_k\|^2$. {For $k> 0$},}
\begin{equation}\label{eq: bound of tilde s VR}
    \begin{aligned}
         & \|\tilde s^{t}_k-s^{t}_k\|^2 = \|(1+\theta)w_{k}^{t} - (1+\theta)\nabla_y\Phi^{t}(x^{t}_k,y^{t}_{k})  -\theta w_{k-1}^{t} +\theta\nabla_y\Phi^{t}(x^{t}_{k-1},y^{t}_{k-1})\|^2  \\
         &\leq \sa{2}(1+\theta)^2\|w_{k}^{t} - \nabla_y\Phi^{t}(x^{t}_k,y^{t}_{k}) \|^2+ \sa{2}\theta^2\|  w_{k-1}^{t} -\nabla_y\Phi^{t}(x^{t}_{k-1},y^{t}_{k-1})\|^2.
    \end{aligned}
\end{equation}
First, $x^{t}_{-1}=x^{t}_{0}$, $y^{t}_{-1}=y^{t}_{0}$ and
\eqref{eq:qksk} imply that $s^t_{0} = \nabla_y\Phi^{t}(x^{t}_0,y^{t}_{0})$, and recall that in \cref{ALG: SAPD-VR}, we set $\tilde{s}^{t}_{0} = w^t_0$; hence,
$$
\|\tilde s^{t}_0-s^{t}_0\|^2 = \|w_{0}^{t} - \nabla_y\Phi^{t}(x^{t}_0,y^{t}_{0}) \|^2,
$$
and \cref{eq: bound of tilde s VR} holds for $k\geq 0$ with \sa{$w_{-1}^t\triangleq\nabla_y\Phi^{t}(x^{t}_0,y^{t}_{0})$.}
Then, \cref{Lemma: SPIDER} implies that
\begin{equation}
    \begin{aligned}
           & \sum_{k=\sa{0}}^{N-1}\rho^{-k}\ED{\tilde s^{t}_{k}-  s^{t}_{k}} \\
           & \leq  \xz{2}(1+\theta)^2\sum_{\sa{\substack{k\in\{1,\ldots,N-1\}\\\text{s.t.}\ \mo(k,q)\neq 0}}}\rho^{-k}\sum_{i=(n_k-1)q+1}^{k}\big(\frac{2L^2_{yx}}{\xz{b_y}}\ED{x^{t}_{i}-x^{t}_{i-1}}+ \frac{2L^2_{yy}}{\xz{b_y}}\ED{y^{t}_{i}-y^{t}_{i-1}}\big)
           \\
           & \quad + \sa{\frac{\xz{2}\theta^2}{\rho}}\sum_{\sa{\substack{k\in\{1,\ldots,N-2\}\\\text{s.t.}\ \mo(k,q)\neq 0}}}\rho^{-k}\sum_{i=(\sa{n_{k}}-1)q+1}^{\sa{k}}\big(\frac{2L^2_{yx}}{\xz{b_y}}\ED{x^{t}_{i}-x^{t}_{i-1}}+ \frac{2L^2_{yy}}{\xz{b_y}}\ED{y^{t}_{i}-y^{t}_{i-1}}
           \big)
           \\
           & \quad +  2(1+2\theta+2\theta^2)\frac{\delta^2_y}{b}\sum_{k=0}^{N-1}\rho^{-k}
           \\

           & \leq \xz{\frac{2}{\rho}}(1+2\theta+2\theta^2)\sum_{\sa{\substack{k\in\{1,\ldots,N-1\}\\\text{s.t.}\ \mo(k,q)\neq 0}}}\rho^{-k}\sum_{i=(n_k-1)q+1}^{k}\big(\frac{2L^2_{yx}}{\xz{b_y}}\ED{x^{t}_{i}-x^{t}_{i-1}}+ \frac{2L^2_{yy}}{\xz{b_y}}\ED{y^{t}_{i}-y^{t}_{i-1}}\big)
           \\
           & \quad +  \xz{2}(1+2\theta+2\theta^2)\frac{\delta^2_y}{b}\sum_{k=0}^{N-1}\rho^{-k},
    \end{aligned}
\end{equation}
where the first inequality follows from \cref{Lemma: SPIDER};
in the last inequality we used $\rho\leq 1$ and combined the two sums. Next, as in \cref{eq:noisy-rate-part2.1 SPIDER}, we can further obtain that
\begin{equation}\label{eq:noisy-rate-part2.2 SPIDER}
    \begin{aligned}
           & \sum_{k=0}^{N-1}\rho^{-k}\ED{\tilde s^{t}_{k}-  s^{t}_{k}} \\
           & \leq \xz{2}(1+2\theta+2\theta^2)\sum_{k=0}^{N-1} \rho^{-k}\cdot\frac{1}{1-\rho}(\rho^{-q+1}-1)\big(\frac{2L^2_{yx}}{\xz{b_y}}\ED{x^{t}_{k}-x^{t}_{k-1}}+ \frac{2L^2_{yy}}{\xz{b_y}}\ED{y^{t}_{k}-y^{t}_{k-1}}\big)
           \\& +\xz{2}(1+2\theta+2\theta^2)\frac{\delta^2_y}{b}\sum_{k=0}^{N-1}\rho^{-k}.
    \end{aligned}
\end{equation}
Now we can bound \textbf{part\;2} in \cref{INEQ: difference of gap SPIDER} using \cref{eq:noisy-rate-part2.1 SPIDER} and \cref{eq:noisy-rate-part2.2 SPIDER}.

Let $\Delta^{t}_N(x,y)\triangleq\frac{1}{2\rho}(\frac{1}{\tau}-\mu_x) \|x^{t}_{N}-x\|^2 +  \frac{1}{2}{\left(\frac{1}{\rho\sigma} - \frac{\alpha}{\rho} \right)}\|y^{t}_{N}-y\|^2$;
adding $\rho^{-N+1}\Delta^{t}_N(x,y)$ to both sides of \eqref{INEQ: difference of gap SPIDER} and taking the expectation of the new inequality, then using \cref{eq:noisy-rate-part1 SPIDER}, \cref{eq:noisy-rate-part2.1 SPIDER} and \cref{eq:noisy-rate-part2.2 SPIDER} to bound \textbf{part\;1} and \textbf{part\;2}, we obtain
{\footnotesize
\begin{eqnarray}
\label{INEQ: ergordic gap  SPIDER}
\lefteqn{\mathbb{E}\big[K_{N} ({\rho})  \left(\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}^{t}(x, \bar{y}^{t}_{N}) \right)+ {\rho}^{-N+1}\Delta^{t}_N(x,y) \big]}\nonumber\\
&&\leq
      \mathbb{E}\left[U^{t}_N(x,y)\right] + \Big(\frac{\delta^2_x}{2\pi_x b}+(1+2\theta+2\theta^2)\frac{\delta^2_y}{\xz{\pi_y b}}\Big)\sum_{k=0}^{N-1}{\rho}^{-k},
\end{eqnarray}}%
{for any fixed $(x,y)\in \mathop{\bf dom} f\times \mathop{\bf dom} g$}, where $U^{t}_N(x,y)$ is defined as
{\scriptsize
\begin{equation}\label{eq:UPQ SPIDER}
\begin{aligned}
    & U^{t}_N(x,y) \triangleq
        \sum_{k=0}^{N-1}{\rho}^{-k}
      \left(\Gamma^{t}_{k+1} + \Lambda^{t}_k{(x,y)} - \Sigma^{t}_{k+1} \sa{(x,y)} + {\sa{|1-\frac{\theta}{\rho}|~S^{t}_{k+1}}}
      + \frac{\pi_x}{2} \|x^{t}_{k+1} - x \|^2 + \frac{\pi_y}{2}\|y^{t}_{k+1}-y\|^2
      \right) \\
      & +\sum_{k=0}^{N-1}\frac{\rho^{-k+1}}{1-\rho}(\rho^{-q+1}-1)\left(
      \left(\xz{\frac{ \xz{{L'_{xx}}^2}}{\xz{\pi_x}b_x}
      +\frac{2(1+2\theta+2\theta^2)\rho^{-1}L^2_{yx}}{\pi_y b_y}}
      \right)
      \|x^{t}_{k}-x^{t}_{k-1}\|^2
      +
      \frac{ L^2_{xy}}{\xz{\pi_x}\xz{b_x}}\|y^{t}_{k+1}-y^{t}_{k}\|^2
      \right)
      \\&
      +
      \sum_{k=0}^{N-1}\frac{\rho^{-k+1}}{1-\rho}(\rho^{-q+1}-1)\frac{\xz{2}(1+2\theta+2\theta^2)\rho^{-1} L^2_{yy}}{\pi_y\xz{b_y}}\|y^{t}_{k}-y^{t}_{k-1}\|^2
      -
      \rho^{-N+1}(-\Delta^{t}_N(x,y) - \frac{\theta}{\rho}S^{t}_{N}).
    \end{aligned}
\end{equation}}%
{For any fixed $(x,y)\in \mathcal{X} \times \mathcal{Y}$}, we can analyze $U^{t}_N(x,y)$ \sa{through writing it as a telescoping sum}. For any $\alpha> 0$, after adding and subtracting $\tfrac{\alpha}{2}\|y^{t}_{k+1}-y^{t}_{k}\|^2$,  and rearranging the terms, we get
{\footnotesize
\begin{equation}
\label{eq:U_def SPIDER}
    \begin{aligned}
           U^{t}_N(x,y) =  &
           \frac{1}{2}\sum_{k=0}^{N-1}{\rho}^{-k}\Big(\xi_k^\top A\xi_k-\xi_{k+1}^\top B \xi_{k+1}\Big)
           -{\rho}^{-N+1}( - \Delta^{t}_N(x,y) - \frac{\theta}{\rho}\sa{S^{t}_{N}})\\
           =  &
           \frac{1}{2}\xi_0^\top A\xi_0- \frac{1}{2}\sum_{k=1}^{N-1}{\rho}^{-k+1}[\xi_k^\top{( B
           - \tfrac{1}{\rho}A)}\xi_k]
           -\rho^{-N + 1 }( \frac{1}{2}\xi_N^\top   B \xi_N  - \Delta^{t}_N(x,y) -\frac{\theta}{\rho}S^{t}_{N}),
    \end{aligned}
\end{equation}}%
where $\xi_k\in\mathbb{R}^5$ is defined for $k\geq 0$ as :
{\scriptsize
$\xi_k \triangleq \left(
    \begin{array}{*{20}{c}}
  \|x^{t}_{k} - x\| \\
  \|y^{t}_{k} - y\| \\
  \|x^{t}_{k} - x^{t}_{k-1}\| \\
  \|y^{t}_{k} - y^{t}_{k-1}\| \\
  \|y^{t}_{k+1} - y^{t}_{k}\|
   \end{array}
   \right)$ {\normalsize such that $x^{t}_{-1} = x^{t}_{0}$, $y^{t}_{-1} = y^{t}_{0}$; and $A, B \in\mathbb{R}^{5\times 5}$ are defined as}:
$$A \triangleq
  \begin{pmatrix}
    \frac{1}{\tau}-\mu_x & 0 & 0 & 0 & 0\\
  0 & \frac{1}{\sigma} & 0 & 0 & 0\\
  0 & 0 & \rho \sa{L_x'} & 0 & {\theta L_{yx}}\\
  0& 0 & 0 &  \rho \sa{L_y^+} & {\theta L_{yy}}\\
  0 & 0 & \theta L_{yx} & \theta L_{yy} & -\alpha
\end{pmatrix},
\quad
   B \triangleq
  \begin{pmatrix}
    \frac{1}{\tau} -\pi_x& 0 & 0 & 0 & 0\\
  0 & \frac{1}{\sigma}+\mu_y-\pi_y & \sa{-|1-\frac{\theta}{\rho}|}~L_{yx} & \sa{-|1-\frac{\theta}{\rho}|}~L_{yy} & 0\\
  0 & \sa{-|1-\frac{\theta}{\rho}|}~L_{yx} & \tfrac{1}{\tau} - L'_{xx} & 0 & 0\\
  0& \sa{-|1-\frac{\theta}{\rho}|}~L_{yy} & 0 & \frac{1}{\sigma} - \alpha - \sa{L_y^-} & 0\\
  0 & 0 & 0 & 0 & 0
\end{pmatrix},$$}
with
\begin{align*}
       & \sa{L_x'} \triangleq \frac{2}{1-\rho}(\rho^{-q+1}-1)
       \big(
       \frac{ \xz{{L'_{xx}}^2}}{\pi_x\xz{b_x}}
       +
       \frac{\xz{2}(1+2\theta+2\theta^2)\rho^{-1} L^2_{yx}}{\pi_y\xz{b_y}}
       \big),
    \\
   & \sa{L_y^+} \triangleq \frac{2}{1-\rho}(\rho^{-q+1}-1)\frac{\xz{2}(1+2\theta+2\theta^2)\rho^{-1} L^2_{yy}}{\pi_y\xz{b_y}},
   \\
   & \sa{L_y^-} \triangleq\frac{2\rho}{1-\rho}(\rho^{-q+1}-1)\frac{ L^2_{xy}}{\xz{\pi_x b_x}}.
\end{align*}
\sa{Using the same argument as in the proof of Lemma~\ref{lem:equivalent_systems}, and noticing that $L'_y$ in \cref{eq: SCSC SAPD LMI SPIDER} can be written as $L'_y=L_y^+ + L_y^-$, one can show that \cref{eq: SCSC SAPD LMI SPIDER} holds if and only if
$B -\tfrac{1}{\rho}A\succeq 0$.} Therefore, it follows from \eqref{eq:U_def SPIDER} that
    $U^{t}_N(x,y) \leq  \sa{\frac{1}{2}\xi_0^\top A\xi_0}- {\rho}^{-N + 1 } ( \tfrac{1}{2}\xi_N^\top  B \xi_N  -  \Delta^{t}_N(x,y) -\frac{\theta}{\rho}\sa{S^{t}_{N}})$
holds w.p. 1 for {any given $(x,y)\in \mathcal{X}\times \mathcal{Y}$.}
Furthermore, define
$$
G''' \triangleq \begin{pmatrix}
     \frac{1}{\sigma}(1-\frac{1}{\rho})+\mu_y-\pi_y+\frac{\alpha}{\rho} & \big(\sa{-|1-\frac{\theta}{\rho}|}-\frac{\theta}{\rho}\big)L_{yx} & \big(\sa{-|1-\frac{\theta}{\rho}|}-\frac{\theta}{\rho}\big)L_{yy} \\
   \big(\sa{-|1-\frac{\theta}{\rho}|}-\frac{\theta}{\rho}\big)L_{yx} & \tfrac{1}{\tau} - L'_{xx} & 0 \\
   \big(\sa{-|1-\frac{\theta}{\rho}|}-\frac{\theta}{\rho}\big)L_{yy} & 0 & \frac{1}{\sigma} - \alpha - \sa{L_y^{-}}
\end{pmatrix},
$$
and recall that  $\Delta^{t}_N(x,y)=\frac{1}{2\rho}(\frac{1}{\tau}-\mu_x) \|x^{t}_{N}-x\|^2 +  \frac{1}{2}{\left(\frac{1}{\rho\sigma} - \frac{\alpha}{\rho} \right)}\|y^{t}_{N}-y\|^2$. \sa{Using a similar argument as in the proof of \cref{lemma: sub positive matrix}, we can show that \cref{eq: SCSC SAPD LMI SPIDER} implies}
{\footnotesize
$\frac{1}{2}\xi_N^\top B  \xi_N  - \Delta^{t}_N(x,y) -\frac{\theta}{\rho}\sa{S^{t}_{N}}=
\frac{1}{2}\xi_N^\top
     \begin{pmatrix}
       \frac{1}{\tau}(1-\frac{1}{\rho}) + \frac{\mu_x}{\rho}-\pi_x
       & \mathbf{0}_{1\times3} & 0\\
       \mathbf{0}_{3\times 1} & G''' & \mathbf{0}_{3\times 1} \\
       0 & \mathbf{0}_{1\times3} & 0\\
     \end{pmatrix}
\xi_N
  \geq
0$}
{for $(x,y)\in \mathcal{X}\times \mathcal{Y}$}.
Finally, \sa{since $x^{t}_{-1} = x^{t}_{0}$, $y^{t}_{-1} = y^{t}_{0}$, we have}
$\frac{1}{2}\xi_0^\top A\xi_0\sa{\leq} (\frac{1}{2\tau}-\frac{\mu_x}{2})\norm{x-x^{t}_{0}}^2+\frac{1}{2\sigma}\norm{y-y^{t}_{0}}^2$.

\sa{Now, we are ready to show the desired result of \cref{lemma: Gap bound with fg SPIDER}.}
Given {$(\bar{x}^{t}_N,\bar{y}^{t}_N)$}, $(x^{t}_{*}(\bar{y}^{t}_N),y_{*}(\bar{x}^{t}_N))\triangleq\argmax_{(x,y)\in\mathcal{X}\times\mathcal{Y}}\mathcal{L}^{t}(\bar{x}^{t}_{N},y)-\mathcal{L}^{t}(x,\bar{y}^{t}_{N})$ uniquely exists. Since $\Delta^{t}_N(x,y)\geq 0$ and $U^{t}_N(x,y)\leq (\frac{1}{2\tau}-\frac{\mu_x}{2})\norm{x-x^{t}_{0}}^2+\frac{1}{2\sigma}\norm{y-y^{t}_{0}}^2$ with probability (w.p.) 1 {for all $(x,y)\in\mathcal{X}\times \mathcal{Y}$}, it follows from \eqref{INEQ: ergordic gap  SPIDER} that
{\small
\begin{eqnarray*}
\lefteqn{K_{N} ({\rho})\mathbb{E}\big[  \sup_{(x,y)\in\mathcal{X}\times\mathcal{Y}}\left\{\mathcal{L}^{t}(\bar{x}^{t}_{N}, y)  - \mathcal{L}^{t}(x, \bar{y}^{t}_{N}) \right\}\big] =K_N(\rho)\mathbb{E}\big[ \mathcal{L}^{t}(\bar{x}^{t}_{N}, y_{*}(\bar{x}^{t}_N))  - \mathcal{L}^{t}(x_{*}(\bar{y}^{t}_N), \bar{y}^{t}_{N})\big]}\nonumber \\
& & \leq
      \mathbb{E}\Big[\left(\frac{1}{2\tau}-\frac{\mu_x}{2}\right)\|x^{t}_*(\bar{y}^{t}_N)-x^{t}_{0}\|^2
      +\frac{1}{2\sigma}\|y_{*}(\bar{x}^{t}_{N})-y^{t}_{0}\|^2 \Big]
       +{K_{N} ({\rho}) } \Big(\frac{\delta^2_x}{2\pi_x b}+(1+2\theta+2\theta^2)\frac{\delta^2_y}{\sa{\pi_y b}}\Big).
\end{eqnarray*}}%
Then dividing both side by $K_N(\rho)$ completes the proof.
\end{proof}
\section{Proof of Theorem~\ref{cor:complexity VR} and preliminary technical results}
We employ the
matrix inequality~(MI) in~\cref{eq: SCSC SAPD LMI SPIDER} to describe the admissible set of \texttt{VR-SAPD} parameters that guarantee convergence.
We  compute a particular solution to it
 by exploiting its
 structure.
Next, in \cref{LEMMA: sp parameter VR}, we give an intermediate condition to help us construct the particular solution provided in \cref{cor:complexity VR}.
\begin{lemma}\label{LEMMA: sp parameter VR}
\sa{For any $\mu_x>0$, let $\xz{L'_{xx}=} L_{xx}+\gamma+\mu_x$. Let $\theta\in(0,1]$ and $\tau, \sigma>0$ \sa{be chosen as}
{\small
\begin{equation}
\label{eq:sufficient_cond_noisy_LMI-2}
\theta=1,\quad \tau = \frac{1}{L_{yx}+\sa{L'_{xx}}+\xz{L'_x}}, \quad \sigma = \frac{1}{2L_{yy}+L_{yx}+\xz{L'_y}},
\end{equation}}%
where $L'_x$ and $L'_y$ are defined in \cref{lemma: Gap bound with fg SPIDER}. Then $\{\tau, \sigma,\theta,\alpha,\rho, \pi_x,\pi_y\}$ is a solution to \eqref{eq: SCSC SAPD LMI SPIDER}  for $\rho=1$, $\pi_x=\mu_x$, $\pi_y=\mu_y$ and $\alpha =  L_{yx}+L_{yy}$.}
\end{lemma}
\begin{proof}
Define ${\everymath={\scriptscriptstyle}
M_1
 \triangleq
 \begin{pmatrix}
 \frac{1}{\tau}  - \sa{L'_{xx}} -\xz{L'_x} & 0 & - L_{yx}\\
 0 & 0 & 0 \\
 - L_{yx} & 0 &L_{yx}
\end{pmatrix}}$ and
${\everymath={\scriptscriptstyle}
M_2 \triangleq
    \begin{pmatrix}
  0 & 0 & 0\\
 0 & \frac{1}{\sigma} \sa{-\alpha} -\xz{L'_y} & - L_{yy}\\
 0 & - L_{yy} & L_{yy}
\end{pmatrix}}$.
\sa{Our choice of $\{\rho,\pi_x,\pi_y\}$ implies that \eqref{eq: SCSC SAPD LMI SPIDER} holds whenever}
{\footnotesize
\begin{equation*}
\begin{aligned}
M_1 + M_2=
\begin{pmatrix}
  \tfrac{1}{\tau} - \sa{L'_{xx}}-\xz{L'_x} & 0 & - L_{yx}\\
 0 &\tfrac{1}{\sigma}- \alpha - \xz{L'_y}& - L_{yy}\\
 - L_{yx} & - L_{yy} & L_{yx} +L_{yy}
\end{pmatrix}\succeq \mathbf{0}.
\end{aligned}
\end{equation*}}%
\sa{Our choice of $\alpha=L_{yx}+L_{yy}$, and $\tau,\sigma>0$ as in \eqref{eq:sufficient_cond_noisy_LMI-2} implies that $M_1
 \succeq 0$ and $M_2 \succeq 0$.} Thus, $M_1+M_2\succeq 0$.
\end{proof}
Next, based on
this lemma, we will give an explicit parameter choice for algorithm~\ref{ALG: SAPD-VR}.
\subsection{Proof of Theorem~\ref{cor:complexity VR}}
\begin{proof}
\cref{LEMMA: sp parameter VR} implies \sa{that our choice of $\{\tau,\sigma,\theta,\alpha,\rho,\pi_x,\pi_y\}$ ensures that \cref{eq: SCSC SAPD LMI SPIDER}
holds. For the outer loop, if we set N as in~\eqref{eq: speical paramater VR innner loop} and}
\begin{equation}\label{eq: speical paramater VR outerloop}
    \begin{aligned}
        p_1 = \frac{1}{16},~p_2 = \frac{19}{32},~p_3=\frac{11}{32},~\beta_1 = \frac{4}{5},~\beta_2 = \frac{1}{2},~\zeta=32,
    \end{aligned}
\end{equation}
all assumptions of \cref{thm:WC_SP SPIDER} are satisfied, i.e., both the \sa{inequality system in~\eqref{eq: additional stepsize condition WCSC} and $K_N(\theta)\geq (1+\zeta)M^{\texttt{VR}}$ hold. Specifically, $K_N(\theta)=N$ for $\theta=1$ and $M^{\texttt{VR}}=2\max\{\frac{1}{\gamma\tau}-1,~\frac{1}{\mu_y\sigma}\}$; thus, $K_N(\theta)\geq (1+\zeta)M^{\texttt{VR}}$ trivially holds by our choice of $N$ in \eqref{eq: speical paramater VR innner loop}. The proof of \cref{eq: additional stepsize condition WCSC} holding for parameters in~\eqref{eq: speical paramater VR outerloop} follows directly from the proof of \cref{cor:complexity-fg}.}

Since all assumptions of \cref{thm:WC_SP SPIDER} are satisfied \sa{for $\mu_x = \gamma$, $\{\tau,\sigma,\theta\}$ as in \eqref{eq:sufficient_cond_noisy_LMI-2}, $N$ and $b$ as in \cref{eq: speical paramater VR innner loop} and other parameters chosen as in \cref{eq: speical paramater VR outerloop}, if
we substitute $\mu_x=\gamma$ and \cref{eq: speical paramater VR outerloop} into} \cref{eq: sketch convergence result} with $\sa{\Xi_{\tau,\sigma,\theta}=\Xi^{\texttt{VR}}=} \frac{\delta^2_x}{2\gamma b}+\sa{5}\frac{\delta^2_y}{\sa{\mu_y}b}$, it follows that
\begin{equation}
    \begin{aligned}
        \frac{1}{T+1}\sum_{t=0}^{T}\mathbb{E}\left[\|\nabla\phi_{\lambda}(x^t_0) \|^2 \right]
        \leq
        &
        48\gamma\left(\frac{1}{T+1}\mathcal{G}(x_0^0,y_0^0)
        +\frac{\delta^2_x}{2\gamma b}+\frac{5\delta^2_y}{\mu_y b} \right).

    \end{aligned}
\end{equation}
Thus, for any $\epsilon>0$,
the right side of the above inequality can be bounded by $\epsilon^2$ when
\begin{equation}\label{eq: split bound VR}
    \frac{48\gamma}{T+1}\mathcal{G}(x_0^0,y_0^0)\leq \frac{\epsilon^2}{6},\quad \frac{24\delta_x^2}{ b}\leq\frac{\epsilon^2}{6},\quad \frac{240\gamma\delta_y^2}{ \mu_y b}\leq\frac{2\epsilon^2}{3}.
\end{equation}
\sa{Our choice of $b$ \sa{in~\eqref{eq: speical paramater VR innner loop}} and $T\geq 288\mathcal{G}(x_0^0,y_0^0)\frac{\gamma}{\epsilon^2}$ ensures that all the inequalities in~\eqref{eq: split bound VR} hold.}
\sa{Moreover, our choice of $N$ in \eqref{eq: speical paramater VR innner loop}, $(\tau,\sigma,\theta)$ in \eqref{eq:sufficient_cond_noisy_LMI-2} and $\rho=1$ together with the definitions of $L_x'$ and $L_y'$ in \cref{lemma: Gap bound with fg SPIDER} implies \eqref{eq:N-bound-VR}. Furthermore, it follows from the statement of \cref{ALG: SAPD-VR} that}
the total computation complexity is $T(Nb/q+N(b_x+b_y))$, which completes the proof.
\end{proof}
\section{Proof of Theorem~\ref{Thm:WCMC} and preliminary technical results}
For any $x\in\mathcal{X}$, define $\phi(x) = \max_{y\in\mathcal{Y}} \mathcal{L}(x,y)$ and $\hat{\phi}(x) = \max_{y\in\mathcal{Y}} \hat{\mathcal{L}}(x,y)$; moreover, let $\phi_\lambda(\cdot)$ and $\hat{\phi}_{\lambda}(\cdot)$ be respective Moreau envelopes for some $\lambda\in(0,\gamma^{-1})$.

\sa{We first show that one can obtain an $\epsilon$-stationary point for the WCMC problem in the form of \eqref{eq:main problem} such that $f(\cdot)=0$, $\mu_y=0$ and $\mathcal{D}_y<\infty$ by computing an $\epsilon$-stationary point for \cref{eq:weakly convex-merely concave problem approx} with $\hat{\mu}_y=\Theta(\epsilon^2/(\gamma\mathcal{D}_y^2))$.
Indeed, in \cref{lem:WCMC} below, we extend \cite[Corollary A.8]{lin2020near} from $g$ being an indicator function of a closed convex set to a closed convex function.}

\begin{lemma}
\label{lem:WCMC}
Under the premise of \cref{Thm:WCMC},
for some fixed $\hat{\mu}_y=\Theta(\epsilon^2/(\gamma\mathcal{D}_y^2))$, let $x_\epsilon\in \mathcal{X}$ be such that $ \|\nabla\hat{\phi}(x_\epsilon)\|\leq \epsilon/(2\sqrt{6})$,
where $\hat{\phi}(x)\triangleq\max_{y\in\mathcal{Y}} \hat{\mathcal{L}}(x,y)$. Then, $x_\epsilon$ is an $\epsilon$-stationary point of $\phi(\cdot)$, i.e., $\|\nabla\phi_{\lambda}(x_\epsilon)\|\leq\epsilon$ for $\lambda\in(0,\gamma^{-1})$, where $\phi(x) \triangleq \max_{y\in\mathcal{Y}}\mathcal{L}(x,y)$.
\end{lemma}
\begin{proof}
Below We state some useful relations that will be used later in the proof. Since $f(\cdot)=0$,
\cref{eq:weakly convex-merely concave problem approx} implies that for all $(x,y)\in\sa{\mathcal{X}} \times \mathop{\bf dom} g$,
\begin{equation}\label{eq: gradient bound WCMC}
  \nabla_x \mathcal{L}(x,y) = \nabla_x \hat{\mathcal{L}}(x,y),\quad \|\nabla_y \Phi(x,y) - \nabla_y \hat{\Phi}(x,y) \|\leq \hat{\mu}_y\mathcal{D}_y.
\end{equation}
We define $\hat{y}_{*}(\cdot)\triangleq \argmax_{y\in\mathcal{Y}} \hat{\mathcal{L}}(\cdot,y)$.
It follows that from \cref{lemma: proximal point lemma v2} that
\begin{equation}
    \hat{y}_{*}(x_\epsilon)=\prox{\alpha g}\big(\hat{y}_{*}(x_\epsilon) + \alpha\nabla_y\hat{\Phi}(x_\epsilon,\hat{y}_{*}(x_\epsilon))\big).\label{eq: G_y}
\end{equation}
for \sa{any} $\alpha>0$. We are now ready for the proof of Lemma~\ref{lem:WCMC}.

Let $y^+ \triangleq \prox{\alpha g}\big(\hat{y}_{*}(x_\epsilon) + \alpha\nabla_y\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon))\big)$, then we have
\begin{equation}\label{eq: y bound WCMC}
    \begin{aligned}
            & \|y^+  - \hat{y}_{*}(x_\epsilon) \|
            \\
            = & \|\prox{\alpha g}\big(\hat{y}_{*}(x_\epsilon) + \alpha\nabla_y\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon))\big) - \prox{\alpha g}\big(\hat{y}_{*}(x_\epsilon)
             + \alpha\nabla_y\hat{\Phi}(x_\epsilon,\hat{y}_{*}(x_\epsilon))\big) \|
             \\
             \leq & \alpha \|  \nabla_y\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon)) - \nabla_y\hat{\Phi}(x_\epsilon,\hat{y}_{*}(x_\epsilon))\| \leq \alpha\hat{\mu}_y\mathcal{D}_y
    \end{aligned}
\end{equation}
where the first equality is by \cref{eq: G_y}; the second inequality is by $\|\prox{\alpha g}(y_1) - \prox{\alpha g}(y_2)\|\leq \| y_1 - y_2\|$ for all $y_1,y_2\in\mathcal{Y}$ and \cref{eq: gradient bound WCMC}.
Moreover, using \cref{ASPT: lipshiz gradient} and  the above inequalities, we have
\begin{equation*}
\begin{aligned}
             &\|\nabla_x \mathcal{L}(x_\epsilon,y^+)\| \leq
            \|\nabla_x \mathcal{L}(x_\epsilon,y^+)
             - \nabla\hat{\phi}(x_\epsilon)
             \|
             +
             \|
             \nabla\hat{\phi}(x_\epsilon)
             \|
             \\
             \leq &\|\nabla_x \mathcal{L}(x_\epsilon,y^+) - \nabla_x \mathcal{L}(x_\epsilon,\hat{y}_{*}(x_\epsilon)) \|
             + \frac{\epsilon}{2\sqrt{6}}

            \leq L_{xy}\alpha\hat{\mu}_y D_y +\frac{\epsilon}{2\sqrt{6}},
\end{aligned}
\end{equation*}
where
\sa{the second inequality follows from Danskin's theorem and the fact that $\|\nabla\hat{\phi}(x_\epsilon)\|\leq \epsilon/(2\sqrt{6})$; finally, the last inequality use \cref{ASPT: lipshiz gradient} and \eqref{eq: y bound WCMC}. Thus, using $(a+b)^2\leq 2(a^2+b^2)$ for any $a,b\in\mathbb{R}$, we get}
\begin{equation}\label{eq: x bound WCMC}
    \|\nabla_x \mathcal{L}(x_\epsilon,y^+)\|^2  \leq \frac{\epsilon^2}{12} + 2L^2_{xy}\alpha^2\hat{\mu}_y^2 D_y^2.
\end{equation}
Later in the proof, \cref{eq: y bound WCMC} and \cref{eq: x bound WCMC} will be useful when we further analyze $y^+$.

\sa{Recall that our ultimate goal is to show that $\|\nabla \phi_{\lambda}(x_\epsilon)\|\leq\epsilon$. Now, for some arbitrary $\mu_x>0$, consider} $\prox{\lambda \phi}(x_\epsilon)=\argmin_{v\in\mathcal{X}} \phi(v) + \frac{1}{2\lambda}\|v-x_\epsilon\|^{\sa{2}}$, where $\lambda = (\mu_x+\gamma)^{-1}$. It follows from \cref{Lemma: graident of ME}  that
$$\|\nabla \phi_{\lambda}(x_\epsilon)\|^2 = \frac{1}{\lambda^2} \|x_\epsilon - \prox{\lambda \phi}(x_\epsilon)\|^2
.$$
Since $\lambda = (\mu_x+\gamma)^{-1}$, $\phi(\cdot)+\frac{1}{2\lambda}\|\cdot - x_\epsilon\|^2$ is $\mu_x$-strongly convex with the unique minimizer $\prox{\lambda \phi}(x_\epsilon)$; therefore,
\begin{equation}
    \begin{aligned}\label{eq: lower bound 1}
            &\max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) - \max_{y\in\mathcal{Y}} \mathcal{L}(\prox{\lambda \phi}(x_\epsilon),y) - \frac{1}{2\lambda}\|\prox{\lambda \phi}(x_\epsilon)-x_\epsilon\|^2 \\
            & = \phi(x_\epsilon) - \phi(\prox{\lambda \phi}(x_\epsilon))  - \frac{1}{2\lambda}\|\prox{\lambda \phi}(x_\epsilon)-x_\epsilon\|^2\\
            &\geq \frac{\mu_x}{2}\| x_\epsilon - \prox{\lambda \phi}(x_\epsilon) \|^2 = \lambda^2\frac{\mu_x}{2}\|\nabla \phi_{\lambda}(x_\epsilon) \|^2.
    \end{aligned}
\end{equation}
\sa{In the following analysis, we will continue to polish the upper bound on $\|\nabla \phi_{\lambda}(x_\epsilon) \|^2$ on the left hand side of \cref{eq: lower bound 1}. Indeed,}
\begin{equation}
    \begin{aligned}\label{eq: upper bound 1}
            &\max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) - \max_{y\in\mathcal{Y}} \mathcal{L}(\prox{\lambda \phi}(x_\epsilon),y) - \frac{1}{2\lambda}\|\prox{\lambda \phi}(x_\epsilon)-x_\epsilon\|^2 \\
            & = \max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) -\mathcal{L}(x_\epsilon,y^+)+ \mathcal{L}(x_\epsilon,y^+) - \max_{y\in\mathcal{Y}} \mathcal{L}(\prox{\lambda \phi}(x_\epsilon),y) - \frac{1}{2\lambda}\|\prox{\lambda \phi}(x_\epsilon)-x_\epsilon\|^2
            \\
            & \leq \max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) -\mathcal{L}(x_\epsilon,y^+)+ \mathcal{L}(x_\epsilon,y^+) - \mathcal{L}(\prox{\lambda \phi}(x_\epsilon),y^+) - \frac{1}{2\lambda}\|\prox{\lambda \phi}(x_\epsilon)-x_\epsilon\|^2
            \\
            & \leq \max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) -\mathcal{L}(x_\epsilon,y^+)
            + \|x_\epsilon - \prox{\lambda \phi}(x_\epsilon) \|\|\nabla_x \mathcal{L}(x_\epsilon, y^+)\| - \frac{\mu_x}{2}\| x_\epsilon - \prox{\lambda \phi}(x_\epsilon)\|^2
            \\
            & \leq \max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) -\mathcal{L}(x_\epsilon,y^+)
            + \frac{\|\nabla_x \mathcal{L}(x_\epsilon, y^+)\|^2}{2\mu_x},
    \end{aligned}
\end{equation}
where the second inequality \sa{follows from} the $\mu_x$-strongly convexity of $\mathcal{L}(\cdot,y^+) + \frac{1}{2\lambda}\|\cdot-x_\epsilon\|^2$ and \sa{Cauchy-Schwarz} inequality. \sa{Next, we continue to derive an appropriate upper bound on} $ \max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) -\mathcal{L}(x_\epsilon,y^+)$. Recall that $y^+ = \prox{\alpha g}\big(\hat{y}_{*}(x_\epsilon) + \alpha\sa{\nabla_y}\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon))\big)$; \sa{hence, the first-order optimality condition} yields that
$$
-\frac{1}{\alpha}\left(y^+ - \hat{y}_{*}(x_\epsilon) - \alpha\sa{\nabla_y}\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon))  \right) \in \partial g(y^+).
$$
Therefore, for any $y\in\mathcal{Y}$, we have that
$$
g(y) - g(y^+) \geq \langle y - y^+, -\frac{1}{\alpha}\left(y^+ - \hat{y}_{*}(x_\epsilon) - \alpha\sa{\nabla_y}\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon))  \right) \rangle ,
$$
which is equivalent to
\begin{equation}\label{eq:subgraident of g}
     g(y^+) - g(y)\leq \frac{1}{\alpha}\langle y - y^+, y^+ - \hat{y}_{*}(x_\epsilon)\rangle -\langle \sa{\nabla_y}\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon)),~ y - y^+  \rangle.
\end{equation}
Now, we ready to \sa{provide a useful} upper bound on  $ \max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) -\mathcal{L}(x_\epsilon,y^+)$. Indeed, given \sa{any} $\tilde{y}\in \argmax_{y\in\mathcal{Y}}\mathcal{L}(x_\epsilon,y)$, we have
\begin{equation}\label{eq: upper bound 2}
    \begin{aligned}
             & \max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) -\mathcal{L}(x_\epsilon,y^+)
              =\mathcal{L}(x_\epsilon,\tilde{y}) - \mathcal{L}(x_\epsilon,\hat{y}_{*}(x_\epsilon))+ \mathcal{L}(x_\epsilon,\hat{y}_{*}(x_\epsilon)) -\mathcal{L}(x_\epsilon,y^+)
              \\
              & = \underbrace{\Phi(x_\epsilon,\tilde{y}) - \Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon))}_{\textbf{part 1}}- g(\tilde{y}) + g(\hat{y}_{*}(x_\epsilon)) +
              \underbrace{\Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon)) -\Phi(x_\epsilon,y^+)}_{\textbf{part 2}}
              - g(\hat{y}_{*}(x_\epsilon)) + g(y^+)
              \\
              & \leq
             \langle \nabla_y \Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon)),\tilde{y} - \hat{y}_{*}(x_\epsilon)  \rangle \sa{-g(\tilde{y}) + g(y^+)}

             \\
             & \quad + \langle \nabla_y \Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon)), \hat{y}_{*}(x_\epsilon) -y^+ \rangle + \frac{L_{yy}}{2}\|\hat{y}_{*}(x_\epsilon)-y^+\|^2

             \\
             & = \langle \nabla_y \Phi(x_\epsilon,\hat{y}_{*}(x_\epsilon)),\tilde{y} - y^+  \rangle - g(\tilde{y})+g(y^+) + \frac{L_{yy}}{2}\|\hat{y}_{*}(x_\epsilon)-y^+\|^2
            \\
              & \leq \frac{1}{\alpha}\langle \tilde{y} - y^+, y^+ - \hat{y}_{*}(x_\epsilon)\rangle + \frac{L_{yy}}{2}\|\hat{y}_{*}(x_\epsilon)-y^+\|^2 \\

              & =  -\frac{L_{yy}}{2}
              \| \hat{y}_{*}(x_\epsilon) -y^+ \|^2  + L_{yy} \langle \tilde{y}-\hat{y}_{*}(x_\epsilon), y^+-\hat{y}_{*}(x_\epsilon)\rangle
              \\
              &\leq L_{yy} \mathcal{D}_{\mathcal{Y}}\|y^+-\hat{y}_{*}(x_\epsilon)\|,
    \end{aligned}
\end{equation}
where in the first inequality, \sa{we use concavity and smoothness of $\Phi(x_\epsilon,\cdot)$ for \textbf{part 1} and \textbf{part 2}, respectively;} in the second inequality, we use \cref{eq:subgraident of g}; in the last equality, we set $\alpha=L_{yy}^{-1}$; and in the last inequality, we use Cauchy-Schwarz inequality and the fact that $\sup_{y_1,y_2\in\mathop{\bf dom} g}\|y_1-y_2\|\leq \mathcal{D}_{\mathcal{Y}}$. Next, if we use \cref{eq: upper bound 2} within \cref{eq: upper bound 1}, it follows that
\begin{equation}
    \begin{aligned}\label{eq: bound 3}
            &\max_{y\in\mathcal{Y}} \mathcal{L}(x_\epsilon,y) - \max_{y\in\mathcal{Y}} \mathcal{L}(\prox{\lambda \phi}(x_\epsilon),y) - \frac{1}{2\lambda}\|\prox{\lambda \phi}(x_\epsilon)-x_\epsilon\|^2 \\
            & \leq L_{yy} \mathcal{D}_{\mathcal{Y}}\|y^+-\hat{y}_{*}(x_\epsilon)\|
            + \frac{\|\nabla_x \mathcal{L}(x_\epsilon, y^+)\|^2}{2\mu_x}
            \\
            & \leq\hat{\mu}_y\mathcal{D}_y^2  +\frac{\epsilon^2}{24\mu_x} + \frac{L^2_{xy}}{L^2_{yy}}\cdot\frac{\hat{\mu}_y^2}{\mu_x}\cdot D_y^2,
    \end{aligned}
\end{equation}
where the last inequality follows from \cref{eq: y bound WCMC} and \cref{eq: x bound WCMC} with $\alpha = L_{yy}^{-1}$.

\sa{Finally,} if we use \cref{eq: bound 3} within \cref{eq: lower bound 1} and substitute $\lambda = (\gamma+\mu_x)^{-1}$, it follows that
\begin{equation}
    \frac{\mu_x}{2(\gamma+\mu_x)^2}\|\nabla \phi_{\lambda}(x_\epsilon) \|^2 \leq\sa{\hat{\mu}_y\mathcal{D}_y^2}  +\frac{\epsilon^2}{24\mu_x} + \frac{L^2_{xy}}{L^2_{yy}}\cdot\frac{\hat{\mu}_y^2}{\mu_x}\cdot D_y^2.
\end{equation}
\sa{Thus, choosing the free parameter $\mu_x = \gamma$ implies that}
\begin{equation}
    \|\nabla \phi_{\lambda}(x_\epsilon) \|^2 \leq\sa{8\gamma\hat{\mu}_y\mathcal{D}_y^2}  +\frac{\epsilon^2}{3} + 8\frac{L^2_{xy}}{L^2_{yy}}\cdot\hat{\mu}_y^2 \cdot D_y^2.
\end{equation}
\sa{Thus, we get $\|\nabla \phi_{\lambda}(x_\epsilon) \| \leq \epsilon$ for}
$\hat{\mu}_y = \min\left\{ \frac{\epsilon^2}{\sa{24}\gamma\mathcal{D}_y^2},\;\frac{L_{yy}}{L_{xy}}\cdot\frac{\epsilon}{2\sqrt{6}\mathcal{D}_y}\right\}$.
\end{proof}
\subsection{Proof of Theorem~\ref{Thm:WCMC}}
\begin{proof}
 To get a worst-case complexity, as in the previous sections, let
\begin{equation*}
   L \triangleq \max\{L_{xy},L_{yx},L_{xx},L_{yy}\},~\delta \triangleq \max\{\delta_x,\delta_y\},~\gamma = L.
\end{equation*}
\cref{ASPT: lipshiz gradient} implies that $\nabla_y \hat{\Phi}$ and $\nabla_x \hat{\Phi}$ are Lipschitz such that for all $x,x'\in\mathcal{X}$ and $y,y'\in\mathop{\bf dom} g$,
\begin{align*}
    \norm{\nabla_y \hat{\Phi}(x,y)-\nabla_y \hat{\Phi}(x',y')}\leq L_{yx}\norm{x-x'}+\hat{L}_{yy}\norm{y-y'},\\
    \norm{\nabla_x \hat{\Phi}(x,y)-\nabla_x \hat{\Phi}(x',y')}\leq L_{xx}\norm{x-x'}+L_{xy}\norm{y-y'},
\end{align*}
where $\hat{L}_{yy}= L_{yy}+\hat{\mu}_y$. Therefore,
the proof immediately follows from \cref{lem:WCMC} and \cref{cor:complexity-fg}, considering \texttt{SAPD+} with $\texttt{VR-flag}=\textbf{false}$ is applied on \eqref{eq:weakly convex-merely concave problem approx} with $\hat{\mu}_y = \min\left\{ \frac{\epsilon^2}{\sa{24}\gamma\mathcal{D}_y^2},\;\frac{L_{yy}}{L_{xy}}\cdot\frac{\epsilon}{2\sqrt{6}\mathcal{D}_y}\right\}$.
\end{proof}

\section{Addition analyses for related works}
\label{sec:existing_work}
\sa{In some of the existing works on WCSC problems, particularly~\cite{huang2021efficient,huang2022accelerated,luo2020stochastic,xu2020enhanced}, except for $\kappa_y=L/\mu_y$, the individual effects of $L$ or $\mu_y$ are not explicitly stated in the final complexity bounds.}
\sa{We emphasize that to better compare existing bounds with ours,} it is necessary to consider those effects. For an example, Huang~\emph{et al.}~\cite{huang2021efficient,huang2022accelerated} assume that {$\frac{1}{\mu_y}\leq L$}, that is equivalent to {$L\geq \sqrt{\kappa_y}$}; \sa{however, a constant factor depending on $L$ was ignored} in their oracle complexity result. Moreover, Huang \emph{et al.}\cite{huang2021efficient} employ a different convergence metric and claim that they obtain a competitive result. It turns out that their convergence metric is scaled by \sa{an algorithmic constant and when their results are converted into GNP metric, i.e., $\|\nabla\phi(\cdot)\|$, this constant adversely affects their complexity bounds}. \sa{A similar issue with the claimed complexity bounds also exists in~\cite{huang2022accelerated}, where the complexity bound are computed after the objective function is rescaled.}
\sa{In this section, to provide a fair comparison,}
\begin{itemize}
    \item we give an explicit oracle complexity bound for the related works \sa{in~\cite{huang2021efficient,huang2022accelerated,luo2020stochastic,xu2020enhanced};}
    \item \sa{we discuss those parts in their analysis that are not convincing, and try our best to provide the corrected and optimized complexity bounds} based on their analysis.
\end{itemize}
 Without loss of generality, \sa{for the sake of easier comparison, we consider the \emph{smooth minimax} problems,} i.e., $\min_{x\in\mathcal{X}}\max_{y\in\mathcal{Y}}\mathcal{L}(x,y)=\Phi(x,y)$.
 \sa{We first briefly go over some important notions in order to fix notation to unify the following discussion for the WCSC setting, \sa{i.e., $\mathcal{L}(x,y)$ is weakly convex in $x$ and strongly concave in $y$.}}

 Recall that $\phi(x)\triangleq\max_{y\in\mathcal{Y}}\mathcal{L}(x,y)$; thus, $\phi(\cdot)$ is differentiable and we use $\|\nabla \phi(\cdot)\|$ as the convergence metric. In addition, we let $\phi_*\triangleq\inf_{x\in\mathcal{X}}  \phi(x)$ and recall that $y_*(\cdot)=\argmax_{y\in\mathcal{Y}}\mathcal{L}(\cdot,y)$.
 Moreover, for simplicity of the notation, we consider the  worst-case complexity bounds using $L$, i.e.,
\begin{equation}
\label{eq:worst-case-parameters}
   L = \max\{L_{xy},L_{yx},L_{xx},L_{yy}\},\quad\kappa_y = \frac{L}{\mu_y},\quad\delta= \max\{\delta_x,\delta_y\},\quad\gamma = L.
\end{equation}
\subsection{Revisit of \cite[Theorem 1]{huang2021efficient}}
In this section, we provide the oracle complexity of Huang~\emph{et al.}\cite[Theorem 1]{huang2021efficient} using the metric $\|\nabla\phi(\cdot)\|$ for the Stochastic Mirror Descent Algorithm~(\texttt{SMDA}), stated in \cite[algorithm 1]{huang2021efficient}. Let $\tau,\sigma$ be the primal and dual stepsizes, respectively, $\eta$ be the momentum parameter, $b$ be the batchsize, and $u$ be convexity modulus of the Bregman distance generating function. We also list our notational convention in \cref{table:Important notations comparison NIPS} for reader's convenience.
\begin{table}[]
    \centering
    \begin{tabular}{c c c}
    \hline
    \textbf{Notation in \cite{huang2021efficient}} & \textbf{Notation in our paper} & \textbf{Meaning}\\
    \hline
    $\gamma$ & $\tau$  & primal stepsize \\
        $\lambda$ & $\sigma $ & dual stepsize \\
       $L_f$ & $L$ & Lipschitz constant as in~\eqref{eq:worst-case-parameters} \\
        $\mu$ & $\mu_y$ & concavity modulus of $\mathcal{L}(x,\cdot)$\\
        $\kappa$ & $\kappa_y$ & condition number \\
        $\sigma$ & $\delta$ & variance bound for the SFO \\
        $b_1$ & $b'$ & small batch size for VR methods\\
        \hdashline
        $\rho$ & $u$ & \thead{convexity modulus of\\ Bregman distance generating function}\\
        \hline
    \end{tabular}
    \caption{Important notation for \cite{huang2021efficient} and this paper.\newline
    \textbf{Table notes.} (1) SFO: stochastic first-order oracle. (2) $u$ is only used in the analysis provided in this section.}
    \label{table:Important notations comparison NIPS}
\end{table}

Below, we restate the convergence result of \texttt{SMDA} for the class of Bregman distance functions such that $D_t(x,x')=(x-x')^\top H_t (x-x')/2$ for some $H_t\succ 0$ --this class of Bregman functions are used for all the numerical experiments reported in~\cite{huang2021efficient}.
\begin{theorem}\label{NIPS THM1} \cite[Thoerem 1]{huang2021efficient}
Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold with $f(\cdot)=g(\cdot)=0$. Let $\{x_t,y_t\}_{t=1}^{T}$ be generated by \texttt{SMDA}, stated in \cite[Algorithm 1]{huang2021efficient}, employing a stochastic first-order oracle to sample stochastic partial derivatives. For parameters chosen as $\eta\in(0,1]$, $\tau\in(0, \min\{ \tfrac{3u}{4L(1+\kappa_y)}, \tfrac{9\eta u \mu_y \sigma}{800\kappa_y^2}, \tfrac{2\eta \mu_y u \sigma}{25L^2}\}\Big]$ and $\sigma\in(0,
\frac{1}{6L}]$, let $\eta_t=\eta$, $\tau_t=\tau$ and $\sigma_t=\sigma$ for $t\geq 0$. Then, for any given initial point $(x_0,y_0)$, we have
\begin{equation}\label{eq: convergence result Huang NIPS 1}
\frac{1}{T}\sum_{t=1}^{T}\mathbb{E}[\|\mathbf{G}^{t}\|] \leq \frac{4\sqrt{2( \phi(x_0)-\phi_*)}}{\sqrt{3T\tau u}} + \frac{4\sqrt{2}\Delta_0}{\sqrt{3T\tau u}} + \frac{10\delta}{\sqrt{3b} u} + \frac{20\delta\sqrt{\eta\sigma}}{3\sqrt{\tau u\mu_y b}},
\end{equation}
where $\phi(x) = \max_{y}\mathcal{L}(x,y)$, $\phi_* = \inf_{x\in\mathcal{X}} \phi(x)$,  $\Delta_0 = \|y_0 - y_*(x_0)\|$, $y_*(x_0) = \argmax_{y\in\mathcal{Y}} \mathcal{L}(x_0,y)$, $\mathbf{G}^{t} =  H_t^{-1} \nabla\phi(x_t)$, and $H_t$ is a diagonal matrix such that $H_t\succeq u\mathbf{I}$ for some $u>0$.
\end{theorem}
\begin{remark}
When $f(\cdot)=g(\cdot)=0$, it follows from the update rules and the definition of Bregman distance function in  \cite[eq.(12-13),~eq.(22-23)]{huang2021efficient} that
$$
\mathbf{G}^{t} =  H_t^{-1} \nabla\phi(x_t),
$$
where $H_t$ is a diagonal matrix such that $H_t\succeq  u \mathbf{I}$. Note that
$$
\mathbf{G}^{t} =  \nabla\phi(x_t) \iff H_t = \mathbf{I}.
$$
We noticed that the authors set the value of $u$ to improve their bounds; but, without addressing its effect on $\mathbf{G}^{t}$. More precisely, they still use $\|\mathbf{G}^{t}\|$ as the convergence metric and compare their complexity results with those papers using $\|\nabla\phi(x_t)\|$ as the convergence metric.
\end{remark}
In the following corollary, we will provide the optimal complexity for \texttt{SMDA} based the result in~\cref{eq: convergence result Huang NIPS 1}, i.e., \cite[Thoerem 1]{huang2021efficient}.
\begin{corollary}
Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold with $f(\cdot)=g(\cdot)=0$, and $\frac{1}{\mu_y}\leq L$ hold\footnote{The assumption $\frac{1}{\mu_y}\leq L$ is also made in \cite{huang2021efficient}.}.
Consider the setting of \cref{NIPS THM1}, then \texttt{SMDA}~\cite[Algorithm 1]{huang2021efficient} can generate $x_\epsilon$ such that $\mathbb{E}\left[\|\nabla\phi(x_\epsilon)\| \right]\leq\epsilon$ by requiring at most $\mathcal{O}(\frac{\kappa_y^5\delta^2}{\mu_y ^2\epsilon^4})$ stochastic first-order oracle calls.
\end{corollary}
\begin{proof}
Recall that $H_t\succeq u\mathbf{I}$, $\mathbf{G}^{t} =  H_t^{-1} \nabla\phi(x_t)$ and $H_t$ is a diagonal matrix; therefore, we can obtain a tight upper bound on $\mathbb{E}[\|\nabla\phi(x_t)\|]$ using \cref{eq: convergence result Huang NIPS 1} as follows:
\begin{equation}\label{eq: convergence result Huang NIPS 2}
\begin{aligned}
\frac{1}{T}\sum_{t=1}^{T} \mathbb{E}[\|\nabla\phi(x_t)\|] \leq
\frac{4\sqrt{2( \phi(x_0)-\phi_*)}}{\sqrt{3T}}\sqrt{\frac{ u}{\tau}} + \frac{4\sqrt{2}\Delta_0}{\sqrt{3T}}\sqrt{\frac{ u}{\tau}} + \frac{ 10\delta}{\sqrt{3b}} + \frac{ 20\delta\sqrt{\eta\sigma}}{3\sqrt{\mu_y  b}}\sqrt{\frac{ u}{\tau}}.
\end{aligned}
\end{equation}
If we use their parameter choices, i.e., $\eta\in(0,1]$
\begin{equation}
\label{eq:Huang-param-choice}
    \sigma= \mathcal{O}\Big(\frac{1}{L}\Big),\quad \tau = u~\min\Big\{\frac{3 }{4L(1+\kappa_y)},~\frac{9\eta \mu_y\sigma}{800\kappa_y^2},~\frac{2\eta\mu_y  \sigma}{25L^2}\Big\},\quad u = \mathcal{O}(L^{\nu}),
\end{equation}
for some free design parameter $\nu\geq0$, then we get
\begin{equation}
\label{eq:u-effect}
    \frac{ u}{\tau} = \max\Big\{\frac{4L(1+\kappa_y)}{3},\frac{800\kappa_y^2}{9\eta\mu_y\sigma},\frac{25L^2}{2\eta\mu_y\sigma}\Big\} =\Omega(\kappa_y^3),
\end{equation}
where the second term leads to $\kappa_y^3$. It is essential to note that $\tau$ choice in \eqref{eq:Huang-param-choice} implies that $u/\tau$ ratio is independent of $u$; hence, the parameter $u$ indeed does not affect the bound on the right-hand-side of \cref{eq: convergence result Huang NIPS 2}. Therefore, contrary to what is suggested in \cite{huang2021efficient}, choosing different values for $u$ through picking different $\nu\geq 0$ values indeed is not useful for proving tighter bounds in GNP metric $\norm{\nabla\phi(x_k)}$ in this simple scenario using their parameter choices.

Note \cref{eq: convergence result Huang NIPS 2} can be simplified as
\begin{equation*}
    \frac{1}{T}\sum_{t=1}^{T}\mathbb{E}[ \|\nabla\phi(x_t)\|] \leq \mathcal{O}\Big( \sqrt{\frac{\kappa_y^3(\phi(x_0)-\phi_*)}{T}}+\frac{\delta}{\sqrt{b}} + \frac{\delta\kappa_y^2}{\sqrt{b}L}\Big).
\end{equation*}
Thus, for any $\epsilon>0$, to find point $x_t$ such that $\mathbb{E}[\|\nabla\phi(x_t)\|]\leq\epsilon$, one should choose $t\geq T$ for
$$
T = \mathcal{O}\Big(\frac{\kappa_y^3}{\epsilon^2}(\phi(x_0)-\phi_*)\Big),\quad b = \mathcal{O}\Big(\frac{\kappa_y^4\delta^2}{L^2\epsilon^2}\Big),
$$
which leads to the oracles complexity of $$2bT=\mathcal{O}\Big(\frac{\kappa_y^7\delta^2}{L^2\epsilon^4}\Big)=\mathcal{O}\Big(\frac{\kappa_y^5\delta^2}{\mu_y ^2\epsilon^4}\Big).$$
\end{proof}

\subsection{Revisit of \cite[Theorem 3]{huang2021efficient}}

 In this section, we provide the oracle complexity of Huang~\emph{et al.}~\cite[Theorem 3]{huang2021efficient} using the metric $\|\nabla\phi(\cdot)\|$ for the Stochastic Mirror Descent Algorithm with variance reduction~(\texttt{SMDA-VR}), stated in \cite[algorithm 2]{huang2021efficient}. Let $\tau,\sigma$ be the primal and dual stepsizes, respectively, $\eta$ be the momentum parameter, $b$ be the large batchsize, $b'$ be the small batchsize, $q$ be the period for sampling large batch size (i.e., once every $q$ batches is large), and $u$ be the strongly-convex constant of the Bregman distance generating function. We also list our notational convention in \cref{table:Important notations comparison NIPS} for reader's convenience.

 Below, as we did in the previous section for \texttt{SMDA}, we restate the convergence result of \texttt{SMDA-VR} for the class of Bregman distance functions such that $D_t(x,x')=(x-x')^\top H_t (x-x')/2$ for some $H_t\succ 0$ --this class of Bregman functions are used for all the numerical experiments reported in~\cite{huang2021efficient}.
\begin{theorem}\label{NIPS THM2} \cite[Thoerem 3]{huang2021efficient}
Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold with $f(\cdot)=g(\cdot)=0$. Let $\{x_t,y_t\}_{t=1}^{T}$ be generated by \texttt{SMDA-VR}, stated in \cite[Algorithm 2]{huang2021efficient}, employing a stochastic first-order oracle to sample stochastic partial derivatives. For parameters chosen as $\eta\in(0,1]$, $\tau=(0,~\min\Big\{
\tfrac{3u}{4L(1+\kappa_y)},
\tfrac{\eta \mu_y  \sigma u}{38L^2},
\frac{3u}{19L^2\eta},
\frac{u\eta}{8},
\tfrac{9 u \eta\mu_y \sigma}{400\kappa_y^2},
\Big\}\Big]$ and $\sigma\in(0,~\min\Big\{\frac{1}{6L}, \frac{9\mu_y}{100\eta^2L^2}\Big\}\Big]$, let $\eta_t=\eta$, $\tau_t=\tau$ and $\sigma_t=\sigma$ for $t\geq 0$ and $b'=q$. Then, for any given initial point $(x_0,y_0)$, we have
\begin{equation}\label{eq: convergence result Huang NIPS 3}
\frac{1}{T}\sum_{t=1}^{T}\mathbb{E}[\|\mathbf{G}^{t}\|]
\leq
\frac{4\sqrt{2(\phi(x_0)-\phi_*)}}{\sqrt{3T\tau u}}
+ \frac{4\sqrt{2}\Delta_0}{\sqrt{3T\tau u}}
+ \frac{2\sqrt{2}\delta}{\sqrt{\tau u\eta b}L}.
\end{equation}
where $\phi(x) = \max_{y}\mathcal{L}(x,y)$, $\phi_* = \inf_{x\in\mathcal{X}} \phi(x)$,  $\Delta_0 = \|y_0 - y_*(x_0)\|$, $y_*(x_0) = \argmax_{y\in\mathcal{Y}} \mathcal{L}(x_0,y)$,  $\mathbf{G}^{t} =  H_t^{-1} \nabla\phi(x_t)$, and $H_t$ is a diagonal matrix such that $H_t\succeq u\mathbf{I}$ for some $u>0$.
\end{theorem}
In the following corollary, we will provide the optimal complexity for \texttt{SMDA-VR} based the result in~\cref{eq: convergence result Huang NIPS 3}, i.e., \cite[Thoerem 3]{huang2021efficient}.
\begin{corollary}
Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold with $f(\cdot)=g(\cdot)=0$, and $\frac{1}{\mu_y}\leq L$ hold\footnote{The assumption $\frac{1}{\mu_y}\leq L$ is also made in \cite{huang2021efficient}.}.  Consider the setting of \cref{NIPS THM2}, then \texttt{SMDA-VR}~\cite[Algorithm 2]{huang2021efficient} can generate $x_\epsilon$ such that $\mathbb{E}\left[\|\nabla\phi(x_\epsilon)\| \right]\leq\epsilon$ by requiring at most $\mathcal{O}(\frac{\kappa_y^5\delta^2}{\mu_y\epsilon^3})$ stochastic first-order oracle calls.
\end{corollary}

\begin{proof}
Recall that $H_t\succeq u\mathbf{I}$, $\mathbf{G}^{t} =  H_t^{-1} \nabla\phi(x_t)$ and $H_t$ is a diagonal matrix; therefore, we can obtain a tight upper bound on $\mathbb{E}[\|\nabla\phi(x_t)\|]$ using \cref{eq: convergence result Huang NIPS 3} as follows:
\begin{equation}\label{eq: convergence result Huang NIPS 4}
\begin{aligned}
\frac{1}{T}\sum_{t=1}^{T} \mathbb{E}[\|\nabla\phi(x_t)\|]\leq \frac{4\sqrt{2(\phi(x_0)-\phi_*)}}{\sqrt{3T}}\sqrt{\frac{ u}{\tau}} + \frac{4\sqrt{2}\Delta_0}{\sqrt{3T}}\sqrt{\frac{ u}{\tau}}  + \frac{2\sqrt{2}\delta}{\sqrt{\eta b}L}\sqrt{\frac{ u}{\tau}}.
\end{aligned}
\end{equation}
If we use their parameter choices, i.e., $\eta\in(0,1]$,
\begin{equation}
\label{eq:Huang-param-choice-vr}
    \sigma = \mathcal{O}\Big(\frac{1}{\kappa_y L}\Big),~\tau = u~\min\Big\{\frac{3 }{4L(1+\kappa_y)},
    \frac{\eta\mu_y\sigma}{38L^2},
    \frac{3}{19L^2\eta},
    \frac{\eta}{8},
    \frac{9\eta\mu_y\sigma}{400\kappa_y^2}\Big\},~ u = \mathcal{O}(L^{1+\nu}),
\end{equation}
for some free design parameter $\nu\geq0$, then we get
\begin{equation*}
    \frac{ u}{\tau} = \max\Big\{\frac{4L(1+\kappa_y)}{3},\frac{38 L^2}{\eta\mu_y\sigma},
    \frac{19L^2\eta}{3},
    \frac{8}{\eta},
    \frac{400\kappa_y^2}{9\eta\mu_y\sigma}\Big\} =\Omega(\kappa_y^4),
\end{equation*}
 where the last term leads to $\kappa_y^4$. It is essential to note that $\tau$ choice in \eqref{eq:Huang-param-choice-vr} implies that $u/\tau$ ratio is independent of $u$; hence, the parameter $u$ indeed does not affect the bound on the right-hand-side of \cref{eq: convergence result Huang NIPS 4}. Therefore, contrary to what is suggested in \cite{huang2021efficient}, choosing different values for $u$ through picking different $\nu\geq 0$ values indeed is not useful for proving tighter bounds in GNP metric $\norm{\nabla\phi(x_k)}$ in this simple scenario using their parameter choices.

 Note \cref{eq: convergence result Huang NIPS 4} can be simplified as
\begin{equation*}
    \frac{1}{T}\sum_{t=1}^{T} \mathbb{E}[\|\nabla\phi(x_t)\|] \leq \mathcal{O}\Big(\sqrt{\frac{\kappa_y^4(\phi(x_0)-\phi_*)}{T}} + \delta\frac{\kappa_y^2}{\sqrt{b}L}\Big).
\end{equation*}
Thus, for any $\epsilon>0$, to find point $x_t$ such that $\mathbb{E}[\|\nabla\phi(x_t)\|]\leq\epsilon$, one should choose $t\geq T$ for
$$
T = \mathcal{O}\Big(\frac{\kappa_y^4(\phi(x_0)-\phi_*)}{\epsilon^2}\Big),\quad b = \mathcal{O}\Big(\frac{\kappa_y^4\delta^2}{L^2\epsilon^2}\Big),
$$
which leads to the oracle complexity of
$$4b'T + 2bT/q=\mathcal{O}\Big(\frac{b'\kappa_y^4}{\epsilon^2} + \frac{\kappa_y^8\delta^2}{L^2\epsilon^4}/q\Big).$$
Since their parameter choice requires $b'=q$, to optimize the above bound, we let $b'=q=\mathcal{O}\left(\frac{\kappa_y^2}{L\epsilon}\right)$, which leads to
$$
\mathcal{O}\left(\frac{\kappa_y^6\delta^2}{L\epsilon^3}\right) =\mathcal{O}\left(\frac{\kappa_y^5\delta^2}{\mu_y\epsilon^3}\right).
$$
\end{proof}

 \subsection{
 Revisit of \cite[Theorem~1]{luo2020stochastic}
 }
Recall that $\phi(x)=\max_{y}\mathcal{L}(x,y)$ and $\phi_* = \inf_{x\in\mathcal{X}} \phi(x)$.
In this paper, the total oracle complexity to find point $x_\epsilon$ such that $\mathbb{E}[\|\nabla\phi(x_\epsilon)\|]\leq\epsilon$  is given by
\begin{equation}\label{eq: complexity luo2020 1}
    \mathcal{O}(\kappa_y^2\epsilon^{-2}\log(\kappa_y/\epsilon)) + \mathcal{O}(T/q\cdot \sa{b}) + \mathcal{O}(T\cdot \sa{b'} \cdot m)
\end{equation}
where\footnote{In fact, they write $b = \lceil \frac{2250}{19}\delta^2\kappa_y^{-2}\epsilon^{2}\rceil$ which is a obvious typo and we provide the correct one.}
\begin{equation}
\label{eq:SREDA-parameters}
    \begin{aligned}
            T = \Big\lceil \frac{100\kappa_y L\Delta_f}{9\epsilon^{2}}\Big\rceil,~q = \lceil \epsilon^{-1} \rceil,\ b = \lceil \frac{2250}{19}\delta^2\kappa_y^{2}\epsilon^{-2}\rceil,\ b' = \Big\lceil \frac{3687}{76}\kappa_y q \Big\rceil,\ m = \lceil 1024\kappa_y \rceil.
    \end{aligned}
\end{equation}
Given an arbitrary initial point $x_0$, let $y_0$ be obtained by inexactly solving $\max_y \mathcal{L}(x_0,y)$, and they define $\Delta_f = \mathcal{L}(x_0,y_0) - \frac{134\epsilon^2}{\kappa_yL} - \phi_*$.
In~\eqref{eq:SREDA-parameters}, the other parameters are defined as follows:
$b$ is the large batchsize, $b'$ is the small batchsize, $q$ is the period such that once every $q$ outer iterations, \texttt{SREDA} calls for a large batchsize,
$T$ is the number of the outer iterations and $m$ is the number of the inner iterations --each outer iteration requires $m$ inner iterations and each inner iteration calls for a small batchsize. Then \cref{eq: complexity luo2020 1} becomes $\mathcal{O}(\frac{L\kappa_y^3}{\epsilon^3})$.

\subsection{
Revisit of \cite[Theorem~1]{xu2020enhanced}
}

Recall that $\phi(x)=\max_{y}\mathcal{L}(x,y)$ and $\phi_* = \inf_{x\in\mathcal{X}} \phi(x)$. In this paper, the precise parameter selection for \cite[Theorem~1]{xu2020enhanced} is provided in \cite[Theorem~3]{xu2020enhanced} of the supplementary material. Using these parameter choice implies that the total oracle complexity to find point $x_\epsilon$ such that $\mathbb{E}[\|\nabla\phi(x_\epsilon)\|]\leq\epsilon$ is given by
\begin{equation}\label{eq: complexity xu2020 1}
    T\cdot b' \cdot m + \Big\lceil \frac{T}{q} \Big\rceil \cdot b + T_0,
\end{equation}
for an arbitrary initial point $x_0$, where the number of outer iterations, $T$,  the number of the inner iterations per each outer iteration, $m$, are set as follows:
\begin{equation*}
\begin{aligned}
    &T = \max\Big\{\frac{3345\kappa_y}{\epsilon^2},\ 6600(1+\kappa_y)L\frac{(\phi(x_0)-\phi_*)}{\epsilon^2}\Big\},\quad b = \frac{9366\delta^2\kappa_y^2}{\epsilon^2},
    \\&b' = \frac{\kappa_y}{\epsilon},\quad m = 52\kappa_y-1,\quad q = \frac{2}{13(1+\kappa_y)}\frac{\kappa_y}{\epsilon},\quad T_0=\mathcal{O}(\kappa_y\log(\kappa_y)).
\end{aligned}
\end{equation*}
Above $b$ is the large batchsize, $b'$ is the small batchsize, $q$ is the period such that once every $q$ outer iterations, a large batch size is sampled rather than a small batch size. Then \cref{eq: complexity xu2020 1} becomes $\mathcal{O}(\frac{L\kappa_y^3}{\epsilon^3})$.

 \subsection{
 Revisit of
\cite[Theorem~12]{huang2022accelerated}
 }
In this section, we provide the oracle complexity of \cite[Theorem 12]{huang2022accelerated} using the metric $\|\nabla\phi(\cdot)\|$ for the Accelerated first-order Momentum Descent Ascent~(\texttt{ACC-MDA}) algorithm, stated in \cite[algorithm 3]{huang2022accelerated}. Let $\tau,\sigma$ be the primal and dual stepsizes, respectively, $\{\eta_t\}$ be the momentum parameter sequence, and $b$ be the batchsize. We also list our notational convention in \cref{table:Important notations comparison JMLR} for reader's convenience.

\begin{table}[h]
    \centering
    \begin{tabular}{c c c}
    \hline
    Notations in \cite{huang2022accelerated} & Notations in our paper & Meaning\\
    \hline
    $\gamma$ & $\tau$  & primal stepsize \\
        $\lambda$ & $\sigma $ & dual stepsize \\
       $L_f$ & $L$ & Lipschitz constant as in~\eqref{eq:worst-case-parameters}
       \\
       $L_g$ & $L(1+\kappa_y)$ &
       $L$-smooth constant of $\phi(x)$
       \\
        $\tau$ & $\mu_y$ &  concavity modulus of $\mathcal{L}(x,\cdot)$\\
        \hline
    \end{tabular}
    \caption{Important notations for \cite{huang2022accelerated} and this paper.}
    \label{table:Important notations comparison JMLR}
\end{table}

Below, we restate the convergence result of \texttt{ACC-MDA} reported in \cite{huang2022accelerated}.
\begin{theorem}\label{JMRL THM12} \cite[Thoerem 12]{huang2022accelerated}
Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold with $f(\cdot)=g(\cdot)=0$. Let  $\{x_t,y_t\}_{t=1}^{T}$ be generated by \texttt{ACC-MDA} algorithm, stated in~\cite[Algorithm 3]{huang2022accelerated}, when applied to the \emph{smooth minimax} problem $\min_{x\in\mathcal{X}}\max_{y\in\mathcal{Y}}\mathcal{L}(x,y)=\Phi(x,y)$. For some given $p>0$, let
$\eta_t = \tfrac{p}{(\psi+t)^{1/3}}$ for all $t\geq 0$, $\tau\in (0,\ \min\{ \tfrac{\sigma\mu_y}{2L}\sqrt{\tfrac{2b}{8\sigma^2+75\kappa_y^2b}},\tfrac{\psi^{1/3}}{2L(1+\kappa_y)p}\}]$ and $\sigma\in(0, \min\{\tfrac{1}{6L}, \tfrac{27b\mu_y}{16}\}]$
such that $\psi\geq \max\{2, p^3,(c_1 p)^3, (c_2 p)^3\}$ for some $c_1\geq \tfrac{2}{3p^3} + \tfrac{9\mu_y^2}{4}$
and
$c_2\geq \tfrac{2}{3p^3} + \tfrac{75L^2}{2}$. Then for any given $x_0$, we have
\begin{equation}\label{eq: convergence result JMLR 1}
\frac{1}{T}\sum_{t=1}^{T}\mathbb{E}[\|\nabla\phi(x_t)\|]
\leq
\frac{\sqrt{2M''}\psi^{1/6}}{T^{1/2}} + \frac{\sqrt{2M''}}{T^{1/3}},
\end{equation}
where $\phi(x) = \max_{y}\mathcal{L}(x,y)$, $\Delta_0 = \|y_0 - y_*(x_0)\|^2$, $y_*(x_0) = \argmax_{y\in\mathcal{Y}} \mathcal{L}(x_0,y)$, $M''=\frac{\phi(x_0)-\phi_*}{\tau p} + \frac{9L^2\Delta_0}{p\sigma\mu_y} + \frac{2\psi^{1/3}\delta^2}{b\mu_y^2 p^2} + \frac{2(c_1^2+c_2^2)\delta^2p^2}{b\mu_y^2}\ln(\psi+T)$, and $\phi_* = \inf_{x\in\mathcal{X}} \phi(x)$.
\end{theorem}

\begin{remark}\cite[Remarks 13 and 14]{huang2022accelerated}
When $b=\mathcal{O}\big(\kappa_y^{\nu}\big)$ for $\nu>0$ and $\kappa_y^{\nu}\leq \tfrac{8}{81 L\mu_y}$, they claim that they can obtain the gradient complexity of $\tilde{\mathcal{O}}(\kappa_y^3\epsilon^{-3})$ if $\nu=3$, and $\tilde{\mathcal{O}}(\kappa_y^{2.5}\epsilon^{-3})$ if $\nu=4$. However, for the assumption $\kappa_y^\nu\leq \frac{8}{81 L \mu_y}$ to hold in general, one needs to rescale the original objective function $\mathcal{L}(x,y)$ with some $s\in(0,1]$ to define
\begin{align}
\label{eq:Ls}
\mathcal{L}_s(x,y) \triangleq s\cdot\mathcal{L}(x,y).
\end{align}
Then the Lipschitz constant of $\nabla \mathcal{L}_s$, strongly concavity modulus of $\mathcal{L}(x,\cdot)$ for any $x\in\mathcal{X}$ and the variance bound of the stochastic oracle for $\nabla_x\mathcal{L}_s$ and $\nabla_y\mathcal{L}_s$ can be written as $sL$, $s \mu_y$, and $s^2\delta^2$, respectively. We notice that the effect of scaling $\mathcal{L}$ on the problem parameters is not discussed in \cite{huang2022accelerated} and \cref{eq: convergence result JMLR 1} is directly used to derive the convergence result assuming $\kappa_y^\nu\leq \tfrac{8}{81L\mu_y}$. As a consequence, the complexity results of $\tilde{\mathcal{O}}(\kappa_y^3\epsilon^{-3})$, $\tilde{\mathcal{O}}(\kappa_y^{2.5}\epsilon^{-3})$ do not hold without loss of generality unless the original function $\mathcal{L}$ satisfies the restrictive assumption of $\kappa_y^\nu\leq \tfrac{8}{81L\mu_y}$.
\end{remark}
In the following discussion, we analyze the effect of scaling $\mathcal{L}$ on the complexity bounds whenever $\kappa_y^\nu\leq \tfrac{8}{81L\mu_y}$ is not satisfied for the original objective function $\mathcal{L}$, and we provide complexity bounds holding without loss of generality that are optimized by choosing $\nu>0$ properly. Now, consider implementing \texttt{ACC-MDA} on an appropriately scaled problem $\min_x\max_y\mathcal{L}_s(x,y)$ where $\mathcal{L}_s$ is defined in~\eqref{eq:Ls}. Let
\begin{equation}\label{eq: rescale constants}
    L_s \triangleq sL,\quad \mu_s \triangleq s\mu_y,\quad \delta_s \triangleq s\delta.
\end{equation}
Note that the condition numbers of $\mathcal{L}_s$ and $\mathcal{L}$ are the same, and are equal to $\kappa_y$, i.e., $\kappa_y=\frac{L}{\mu_y}=\frac{L_s}{\mu_s}$. In the upcoming discussion, suppose that $s\in(0,1]$ is chosen such that $\kappa_y^\nu\leq \tfrac{8}{81 L_s\mu_s}$.

To facilitate the complexity analysis and make the upcoming discussion easier, first we restate \cref{JMRL THM12} for the function  $\mathcal{L}_s$, where we used the relation $\nabla \phi$ and the derivative of $\max_y\mathcal{L}_s(\cdot,y)$; indeed, the derivative of $\max_y\mathcal{L}_s(\cdot,y)$ is equal to $s \nabla \phi(\cdot)$, where $\phi(x) = \max_{y}\mathcal{L}(x,y)$.
\begin{theorem}\label{JMRL THM12 v2} \cite[Thoerem 12]{huang2022accelerated}
Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold with $f(\cdot)=g(\cdot)=0$. Let  $\{x_t,y_t\}_{t=1}^{T}$ be generated by \texttt{ACC-MDA} algorithm, stated in~\cite[Algorithm 3]{huang2022accelerated}, when applied to the \emph{smooth minimax} problem $\min_{x\in\mathcal{X}}\max_{y\in\mathcal{Y}}\mathcal{L}_s(x,y)=s\cdot\mathcal{L}(x,y)$.
For some given $p\geq0$,
let
$\eta_t = \tfrac{p}{({\psi}+t)^{1/3}}$ for all $t\geq 0$,
$\tau\in (0, \min\{ \tfrac{\sigma\mu_s}{2L_s}\sqrt{\tfrac{2b}{8\sigma^2+75\kappa_y^2b}},\tfrac{{\psi}^{1/3}}{2L_s(1+\kappa_y)p} \}]$
and
$\sigma\in(0, \min\{\tfrac{1}{6L_s}, \tfrac{27b\mu_s}{16}\}]$ such that
${\psi}\geq \max\{2, p^3,(c'_{1} p)^3, (c'_{2} p)^3 \}$ for some
$c'_{1}\geq \tfrac{2}{3p^3} + \tfrac{9\mu_s^2}{4}$
and
$c'_{2}\geq \tfrac{2}{3p^3} + \tfrac{75L_s^2}{2}$.
Then for any given $x_0$,
we have
\begin{equation}\label{eq: convergence result JMLR v2}
\frac{1}{T}\sum_{t=1}^{T}\mathbb{E}[\|\nabla\phi(x_t)\|]
\leq
\frac{1}{s}\left(\frac{\sqrt{2M_s''}{\psi}^{1/6}}{T^{1/2}} + \frac{\sqrt{2M_s''}}{T^{1/3}}\right),
\end{equation}
where $\phi(x) = \max_{y}\mathcal{L}(x,y)$, $\Delta_0 = \|y_0 - y_*(x_0)\|^2$, $y_*(x_0) = \argmax_{y\in\mathcal{Y}} \mathcal{L}(x_0,y)$, $M_s''=\frac{s(\phi(x_0)-\phi_*)}{\tau p} + \frac{9L_s^2\Delta_0}{p\sigma\mu_s} + \frac{2{\psi}^{1/3}\delta_s^2}{b\mu_s^2 p^2} + \frac{2({c'_{1}}^2+{c'_{2}}^2)\delta_s^2p^2}{b\mu_s^2}\ln({\psi}+T)$, and $\phi_* = \inf_{x\in\mathcal{X}} \phi(x)$.
\end{theorem}
Next, following the analysis in \cite[Remarks 10 and 13]{huang2022accelerated}, we provide a particular parameter choice for \texttt{ACC-MDA} so that it is applicable to the setting of \cref{JMRL THM12 v2}.
\begin{lemma}\label{eq: parameter JMLR}
Under the premise of \cref{JMRL THM12 v2}. Suppose  $\kappa_y^{\nu}\leq\frac{8}{81 L_s\mu_s}$, $b=\kappa_y^{\nu}$ for some $\nu>0$, and
\begin{equation}
    \label{eq: suggested choice}
    \frac{\sigma\mu_s}{2L_s}\sqrt{\frac{2b}{8\sigma^2+75\kappa_y^2b}}\leq\frac{{\psi}^{1/3}}{2L_s(1+\kappa_y)p}.
\end{equation}
If
$\sigma = \min\{\tfrac{1}{6L_s}, \tfrac{27b\mu_s}{16}\}$
and
$\tau = \min\{ \tfrac{\sigma\mu_s}{2L_s}\sqrt{\tfrac{2b}{8\sigma^2+75\kappa_y^2b}},\tfrac{{\psi}^{1/3}}{2L_s(1+\kappa_y)p} \}$, then ${\psi} = \Theta(\max\{1, L_s^6\})$ satisfies the condition in \cref{JMRL THM12 v2} and
\begin{equation}
        \sigma=\Theta(b\mu_s),\quad
        \tau^{-1} = \Theta \Big(\frac{\kappa_y^3}{bL_s }\Big).
\end{equation}
\end{lemma}
\begin{remark}
The conditions $\kappa_y^{\nu}\leq \frac{8}{81L_s\mu_s}$ and \cref{eq: suggested choice},
and the choice $b=\kappa_y^{\nu}$ are as suggested in \cite{huang2022accelerated}.
\end{remark}
\begin{proof}
Since $\kappa_y^{\nu}\leq \frac{8}{81L_s\mu_s}$, we have $\sigma=\frac{27b\mu_s}{16}=\Theta(b\mu_s)$. Furthermore, \cref{eq: suggested choice} implies that we can simplify $\tau$ as
$$
\tau= \frac{\sigma\mu_s}{2L_s}\sqrt{\frac{2b}{8\sigma^2+75\kappa_y^2b}}.
$$
Then it follows that,
\begin{equation*}
    \label{eq: tau bound JMLR during proof}
    \tau^{-1} = \Theta \Big(\frac{\kappa_y}{b\mu_s} \sqrt{b\mu_s^2 + \kappa_y^2} \Big) =\Theta \big(\frac{\kappa_y^2}{b L_s}(\kappa_y^{\nu/2}\mu_y +\kappa_y) \Big) = \Theta \Big(\frac{\kappa_y^2}{b }\Big(\kappa_y^{\nu/2-1} +\frac{\kappa_y}{L_s}\Big) \Big)= \Theta \Big(\frac{\kappa_y^3}{bL_s }\Big),
\end{equation*}
where we use the relation $\kappa^{\nu}_y\leq \frac{8}{81L_s\mu_s}\Rightarrow L_s^2\leq\frac{8}{81} \kappa_y^{1-\nu}$ for the last equality.
Next, from \cref{eq: suggested choice} and the requirement on $\psi$ in \cref{JMRL THM12 v2}, a sufficient condition $\psi$ is
\begin{equation}\label{eq: m bound 1}
    {\psi} \geq  \max\left\{2,\ p^3,\ (c'_{1} p)^3,\ (c'_{2} p)^3,\  \left(\sigma\mu_s(1+\kappa_y)p \sqrt{\frac{2b}{8\sigma^2+75\kappa_y^2b}}\right)^3 \right\},
\end{equation}
Now we consider the components of $\max$ operator in \eqref{eq: m bound 1}. Note positive constant $p$ can be chosen independent of other problem parameters, e.g., $p=1$. Furthermore, the requirement on $c_1',c_2'$ can be satisfied for
\begin{equation}
\label{eq:c-choice}
    \begin{aligned}
    c'_{1} = \Theta\big( \mu_s^2 \big),\quad c'_{2} = \Theta(L_s^2).
    \end{aligned}
\end{equation}
Finally, using $\sigma=\frac{27b\mu_s}{16}$ together with $b=\kappa_y^{\nu}$ yields that
\begin{equation*}
    \begin{aligned}
    & \sigma\mu_s(1+\kappa_y)p \sqrt{\frac{2b}{8\sigma^2+75\kappa_y^2b}}  = \Theta \Big(\kappa_y b\mu_s^2 \sqrt{\frac{1}{b\mu_s^2 + \kappa_y^2}}\Big) = \Theta \Big( L_s^2\kappa_y^{\nu-1}\sqrt{\frac{1}{\kappa_y^{\nu-2}L_s^2 + \kappa_y^2}}\Big)
    \\
    & = \Theta \Big( L_s^2\kappa_y^{\nu-1}\sqrt{\frac{1}{\kappa_y^2}}\Big) \leq \Theta(1),
    \end{aligned}
\end{equation*}
where we use the relation $\kappa^{\nu}_y\leq \frac{8}{81L_s\mu_s}\Rightarrow L_s^2\leq\frac{8}{81} \kappa_y^{1-\nu}$ for the last equality and the last inequality. Therefore, using the above relations within \cref{eq: m bound 1}, we observe that one can set
\begin{equation}
\label{eq:psi-choice}
    {\psi}^{1/3} = \Theta(\max\{1, L_s^2\}),
\end{equation}
which completes the proof.
\end{proof}

Next, we will use the parameters in \cref{eq: parameter JMLR} to provide an optimized complexity for \texttt{ACC-MDA} \cite[Algorithm 12]{huang2022accelerated} to generate $x_\epsilon$ such that $\mathbb{E}\left[\|\nabla\phi(x_\epsilon)\| \right]\leq\epsilon$.
\begin{corollary}
Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption},~\ref{ASPT: non-fg} hold with $f(\cdot)=g(\cdot)=0$, and  $\kappa_y^\nu> \tfrac{8}{81L\mu_y}$ for the original function $\mathcal{L}$. Running \texttt{ACC-MDA} on $\min_x\max_y\mathcal{L}_s(x,y)$ for
\begin{equation}\label{eq: choice of s}
    s = \frac{2\sqrt{2}}{9}\frac{1}{L}\kappa_y^{(1-\nu)/2},
\end{equation}
and $b=\kappa_y^{\nu}$, one can generate $x_\epsilon$ such that $\mathbb{E}\left[\|\nabla\phi(x_\epsilon)\| \right]\leq\epsilon$ requiring at most  $\tilde{O}(\frac{L^{1.5}\kappa_y^{3.5}}{\epsilon^3})$ stochastic first-order oracle calls.
\end{corollary}

\begin{proof}
It follows from \cref{JMRL THM12 v2} that
that
\begin{equation}\label{eq: convergence result JMLR 2}
\frac{1}{T}\sum_{t=1}^{T}\mathbb{E}[\|\nabla\phi(x_t)\| ]
\leq
\frac{\sqrt{2M_s''}}{s}\Big(\frac{{\psi}^{1/6}}{T^{1/2}} + \frac{1}{T^{1/3}}\Big).
\end{equation}
Based on \cref{eq: parameter JMLR}, let ${\psi}=\Theta(\max\{1, L_s^6\})$; thus, $\frac{{\psi}^{1/6}}{T^{1/2}} \leq \frac{1}{T^{1/3}}$ when $T$ is large enough. Therefore, for all sufficiently small $\epsilon>0$, $\frac{1}{T^{1/3}}\leq \frac{s}{\sqrt{2M_s''}}\cdot \frac{\epsilon}{2}$ implies that $\frac{\psi^{1/6}}{T^{1/2}}\leq \frac{s}{\sqrt{2M_s''}}\cdot\frac{\epsilon}{2}$, and we get
\begin{equation}\label{eq: convergence eq JMLR}
    \frac{1}{s}\frac{\sqrt{2M_s''}}{T^{1/3}}\leq\frac{\epsilon}{2} \implies \min_{t\in\{0,\ldots,T\}}\mathbb{E}[\|\nabla\phi(x_t)\|]=\epsilon.
\end{equation}
Moreover, note that \cref{eq: choice of s} implies $\kappa_y\leq \frac{8}{81 L_s\mu_s}$; thus, we can  choose $\tau,\sigma,b$ as in \cref{eq: parameter JMLR} which satisfy
\begin{equation*}
    \begin{aligned}
    &\sigma=\Theta(b\mu_s),\quad
        \tau^{-1} = \Theta \big(\frac{\kappa_y^3}{bL_s }\big), \quad
    b = \kappa_y^\nu.
\end{aligned}
\end{equation*}
Recall that $c'_{1}$, $c'_{2}$ chosen as in \eqref{eq:c-choice} and $\psi$ chosen as in \eqref{eq:psi-choice} satisfy all the required conditions in \cref{JMRL THM12 v2}; hence,
$M_s''=\frac{s(\phi(x_0)-\phi_*)}{\tau p} + \frac{9L_s^2\Delta_0}{p\sigma\mu_s} + \frac{2{\psi}^{1/3}\delta_s^2}{b\mu_s^2 p^2} + \frac{2({c'_{1}}^2+{c'_{2}}^2)\delta_s^2p^2}{b\mu_s^2}\ln({\psi}+T)$ implies that
\begin{equation}
\begin{aligned}
    \frac{1}{s^2}M_s''  &= \Theta \Big(
    \frac{\kappa_y^3}{sbL_s}
    +
    \frac{\kappa_y^2}{s^2b} + \frac{\delta_s^2}{s^2b\mu_s^2}\max\{1, L_s^2\} +  \frac{\kappa_y^2L_s^2\delta_s^2}{s^2b}\ln({\psi}+T)
    \Big) \\
    & = \tilde{\Theta} \Big(
    \frac{\kappa_y^3}{s^2bL}
    +
    \frac{\kappa_y^2}{s^2b} + \frac{\delta^2}{s^2b\mu_y^2}\max\{1, s^2L^2\} +  \frac{s^2\kappa_y^2L^2\delta^2}{b}
    \Big).
\end{aligned}
\end{equation}
Moreover, to satisfy \cref{eq: convergence eq JMLR}, one needs to choose $T\geq \Theta(\big(\frac{1}{s^2}M''_s\big)^{3/2} \frac{1}{\epsilon^3})$.  Since the total oracle complexity is $bT$, we obtain that
\begin{equation}\label{eq: bT bound with s JMLR}
\begin{aligned}
         & bT \geq\tilde{\Theta}\left(\frac{1}{\epsilon^3} \big(
    \frac{\kappa_y^{9/2-\nu/2}}{s^3L^{3/2}}
    +
    \frac{\kappa_y^{3-\nu/2}}{s^3} + \frac{\delta^3\kappa^{-\nu/2}}{s^3\mu_y^3}\max\{1, s^3L^3\} +  s^3\kappa_y^{3-\nu/2}L^3\delta^{3}
    \big) \right).
\end{aligned}
\end{equation}
From \cref{eq: choice of s}, i.e., $s^2 = \frac{8}{81}\frac{1}{L^2}\kappa_y^{1-\nu}$, it follows that
\begin{equation}
    \label{eq: bT bound JMLR}
   bT \geq \tilde{\Theta}\left(\frac{1}{\epsilon^3} \big(
    L^{3/2}\kappa_y^{\nu+3}
    +
    L^{3}\kappa_y^{\nu+3/2} + \delta^3\kappa^{\nu+3/2}\max\{1, \kappa_y^{\frac{3-3\nu}{2}}\} +  \kappa_y^{9/2-2\nu}\delta^{3}
    \big) \right).
\end{equation}
When $\nu\geq1$,  we have
\begin{equation*}
    bT \geq \tilde{\Theta}\big(L^{3/2}\kappa_y^{\nu+3}
    +
    L^{3}\kappa_y^{\nu+3/2} + \delta^3\kappa_y^{3-\nu/2} +  \kappa_y^{9/2-2\nu}\delta^{3}
    \big),
\end{equation*}
the optimal value is achieved at $\nu=1$ and $bT \geq \Theta(\frac{L^{1.5}\kappa_y^4}{\epsilon^3})$; when $\nu<1$, we have
\begin{equation*}
    bT \geq \tilde{\Theta}\big( L^{3/2}\kappa_y^{\nu+3}
    +
    L^{3}\kappa_y^{\nu+3/2} + \delta^3\kappa_y^{\nu+3/2} +  \kappa_y^{9/2-2\nu}\delta^{3}
    \big),
\end{equation*}
the optimal value is achieved at $\nu=\frac{1}{2}$ and $bT \geq\tilde{\Theta}(\frac{L^{1.5}\kappa_y^{3.5}}{\epsilon^3})$, which completes the proof.
\end{proof}
\begin{remark}
In \cite{huang2022accelerated}, Huang \emph{et al.} claims the oracle complexity of $\tilde{\mathcal{O}}(\kappa_y^3\epsilon^{-3})$ for $\nu=3$, and $\tilde{\mathcal{O}}(\kappa_y^{2.5}\epsilon^{-3})$ for $\nu=4$. However, our analysis leading to \cref{eq: bT bound JMLR} demonstrates that the complexities would be ${\tilde{\mathcal{O}}(\frac{L^{1.5}\kappa^6_y}{\epsilon^3})}$ for $\nu =3$ and  $\tilde{\mathcal{O}}(\frac{L^{1.5}\kappa^7_y}{\epsilon^3})$ for $\nu=4$.
\end{remark}
\section{Introduction}\label{section: weakly covex algorithms}
\vspace*{-2mm}
We consider the following saddle-point~\sa{(SP) problem}:
\begin{equation}\label{eq:main problem}
    \min_{x\in \mathcal{X}} \max_{y\in \mathcal{Y}} \mathcal{L}(x,y)\triangleq f(x)+\Phi(x,y)-g(y),
\end{equation}
where $\mathcal{X}$ and $\mathcal{Y}$ are, \sa{$n$ and $m$ dimensional
Euclidean spaces,}
the function $\Phi(x,y):\mathcal{X}\times\mathcal{Y} \rightarrow \mathbb{R}$ is smooth and
\sa{nonconvex} in $x\in\mathcal{X}$ and $\mu_y$-strongly
concave in $y\in\mathcal{Y}$ \sa{for some $\mu_y\geq 0$} (\mg{with the convention that for $\mu_y=0$, $\Phi$ is merely
\sa{concave} (MC) in $y$}), and the functions $f$ and $g$ are \sa{closed}, convex and possibly nonsmooth. In this \xz{paper}, we consider a particular case of nonconvexity, i.e.,  we assume that \sa{$\Phi(\cdot,y)$} is weakly convex~(WC) for any fixed \sa{$y\in\mathop{\bf dom} g\subset\mathcal{Y}$}. Weakly convex functions
\sa{constitute} a rich class of non-convex functions and arise
\sa{naturally} in many \sa{practical} settings
\sa{for} machine learning~\sa{(ML)} applications \cite{davis2019stochastic,rafique1810non}, precise definitions will be given later in Section \ref{sec-preliminaries}. \sa{In practice, WC assumption is widely satisfied, e.g., under smoothness --see~\cref{rem:smooth-WC}, \mg{in fact it can be argued that most of the} work \sa{in related literature} considering nonconvex-(strongly) concave
SP problems provide their analyses under the premise of weak convexity.}
\mg{The problem \eqref{eq:main problem} with $\mu_y> 0$ is called a weakly convex-strongly concave (WCSC) saddle-point
problem, whereas for $\mu_y=0$, it is called a weakly convex-merely concave (WCMC) saddle-point problem.
Both problems arise frequently in many
\sa{ML} problems including constrained optimization of
\sa{WC} objectives based on Lagrangian duality \cite{li2021augmented}, Generative Adversarial Networks (GAN)
(where
\sa{$x$
denotes} the parameters of the generator network whereas \sa{$y$
represents} the parameters of the discriminator network \cite{goodfellow2014generative}), distributional robust learning with weakly convex loss functions such as those arising in deep learning \cite{gurbuzbalaban2020stochastic,rafique1810non} and learning problems with non-decomposable losses \cite{rafique1810non}.}

\mg{There are
important two settings
for \eqref{eq:main problem}: (i) the \emph{deterministic setting}, where the \sa{partial} gradients of
$\Phi$
are exactly available, (ii) the \emph{stochastic setting}, where
only stochastic estimates of the gradients are available. Although, recent years have witnessed significant advances in the deterministic setting \cite{chen2021accelerated,huang2021efficient,jin2020local,lin2020gradient,lin-near-optimal,lu2020hybrid,ostrovskii2021efficient, thekumparampil2019efficient,xu2020unified}; our focus in this paper will be mainly the \emph{stochastic setting} which is more relevant and more applicable to
\sa{ML} problems. Indeed, due to large-dimensions and the \sa{sheer} size of the modern datasets, computing gradients exactly is either infeasible or impractical in
\sa{ML} practice and gradients are often estimated stochastically based on mini-batches (randomly sampled subset of data points) as in the case of stochastic gradient-type algorithms.}

\mg{There is a growing literature on the WCSC and WCMC problems in the stochastic setting. Several metrics for quantifying the quality of an approximate solution to \eqref{eq:main problem} have been proposed in the literature. A common way to assess the performance is to define the \emph{primal function} $\phi (\cdot) \sa{\triangleq} \max_{y\in \mathcal{Y}} \mathcal{L}(\cdot,y)$ and measure the \sa{violation of} first-order
\sa{necessary conditions for}}
the non-convex problem $\min_{x\in \mathcal{X}} \phi (x)$.  \mg{
\sa{Given the primal iterate sequence $\{x_k\}_{k\geq 0}$ of a stochastic SP algorithm and a threshold $\epsilon>0$, a commonly used \sa{metric} is the \emph{gradient norm of
the Moreau envelope}~\sa{(GNME)}; indeed, the objective is to provide a bound $K_\epsilon$ such that $\mathbb{E}[\norm{\phi_\lambda(x_k)}]\leq\epsilon$ for all $k\geq K_\epsilon$,} where $\phi_\lambda$ denotes the Moreau envelope of the primal function $\phi$ \sa{--see Definitions}~\ref{Def: Moreau envelope},~\ref{def:primal-function} and \ref{Def: stationary point}. Another commonly used natural metric is the \emph{gradient norm of the primal function} $\phi(\cdot)$~\cite{boct2020alternating,huang2021efficient,huang2022accelerated,luo2020stochastic,xu2020enhanced}, abbreviated as GNP,
\sa{where the aim is to derive $K_\epsilon$ such that $\mathbb{E}[ \|\nabla\phi(x_k)\|] \leq \epsilon$ for all $k\geq K_\epsilon$.}  Other metrics such as the notion of $\epsilon$-first-order Nash equilibrium (FNE) and its generalized versions also exist in the literature \cite{nouiehed2019solving,ostrovskii2021efficient}. \sa{When using any of these metrics, the ultimate goal is to establish a bound on the oracle (sampling) complexity, i.e., $\sum_{k=0}^{K_\epsilon}b_k$, where $b_k$ denotes the batch-size for iteration $k\geq 0$.}
\sa{It crucial to note for the WCSC setting that GNME, GNP and FNE metrics are
all} equivalent in the sense that convergence in either of them implies convergence in the other for WCSC problems~\cite{lin2020gradient}. In this paper, \sa{for the WCSC setting,} we adopt \sa{both} GNME \mg{and GNP} as \mg{the main performance metrics} to analyze our algorithms; \sa{indeed, in \cref{thm:metric-equivalance} we show that}, {\color{black} when the non-smooth part $f(\cdot)=0$},
we can convert a GNME guarantee to a GNP guarantee by incurring only little additional cost compared to the cost of obtaining a GNME guarantee \sa{such that the overall worst-case complexity \mg{(in terms of worst-case dependency to the target accuracy $\epsilon$)} remains the same for both metrics}}. {\color{black} When the non-smooth part $f(\cdot)\neq 0$, we also obtain similar guarantees and \sa{show equivalence between the metrics based on GNME and the generalized gradient mapping.}}
\sa{On the other hand, for the WCMC setting, we provide our guarantees in GNME metric as $\phi$ is not necessarily differentiable for this scenario.}
Moreover, our work accounts for the individual effects of $L_{xx}$, $L_{xy}$, $L_{yx}$ and $L_{yy}$, i.e., the Lipschitz constants of $\nabla_x\Phi(\cdot, y)$, $\nabla_x\Phi(x, \cdot)$, $\nabla_y\Phi(x, \cdot)$ and $\nabla_y\Phi(\cdot, y)$~(see \cref{ASPT: lipshiz gradient}), respectively, instead of using the worst-case
{parameters} $L\triangleq\max\{L_{xx}, L_{xy}, L_{yx}, L_{yy}\}$, while
\sa{the majority of related work} ignore the influence of
\sa{these block Lipschitz constants to be able to analyze their algorithms.} We emphasize that using the worst-case parameters will lead to a theoretically conservative step sizes, and this phenomenon has been validated in the work~\cite{zhang2021robust}.
\vspace*{-6mm}
\paragraph{Contributions.} Table~\ref{table-one} summarizes the relevant existing work for WCSC and WCMC problems closest to our setting and our results.
\sa{More specifically, in Table \ref{table-one}, for the stochastic setting, we report the (oracle) complexity with respect to the GNP and GNME as the performance metrics for WCSC and WCMC problems, respectively, and the batch-size (number of data points in the mini-batches) required at every iteration.}
We also report whether the method is based on a variance-reduction (VR) technique. VR-based methods mentioned in Table~\ref{table-one} use a small batch-size \sa{$b'$} in general, while using a large batch-size \sa{$b\geq b'$ once in} every $q$ iterations for reducing the variance where the period $q$ \sa{is the number of times small batches are sampled consecutively plus one,}  is \sa{also an
algorithm} parameter. Therefore, for VR-methods, we report the batch size as a triplet $(b', b, q)$.
In the column ``Compactness", we list whether achieving the specific complexity requires assuming compactness of the primal \sa{and/or} dual domains. \sa{To make the
comparison of our results with the existing work easier, we provide the results in the table for the worst-case setting, where} $\kappa_y \triangleq \frac{L}{\mu_y}$, and we
\sa{report the $\epsilon$-, $\kappa_y$- and $L$-dependency
of the complexity results for the existing} algorithms. That being said, our results have finer granularity in terms of their dependence to the individual effects of $L_{xx}$, $L_{xy}$, $L_{yx}$ and $L_{yy}$ as we mentioned earlier. Our contributions (\sa{also} summarized in \cref{table-one}) are as follows:
\begin{itemize}[noitemsep,topsep=0pt,leftmargin=*,align=left]
    \item We propose a new stochastic

    method, \texttt{SAPD+},

    based on the inexact proximal point method (iPPM). \sa{In this framework, one inexactly solves strongly convex-strongly concave (SCSC) saddle point sub-problems

    using an accelerated primal-dual method, SAPD~\cite{zhang2021robust}.}

    In \cref{thm:unbounded-domains}, \sa{we

    establish an oracle complexity of $\mathcal{O}(L\kappa_y\epsilon^{-4})$} for WCSC problems, and unlike the majority of  existing work we do not require \sa{compactness for neither the primal nor the dual domain}. To our knowledge,
    our bound has \sa{the best $\kappa$ dependence

    in the literature; indeed, prior to this work, without using variance reduction, the best known complexity was $\mathcal{O}(L\kappa^3\epsilon^{-4})$; hence, we establish a $\mathcal{O}(\kappa^2)$ improvement.}
    \item We propose a variance-reduced version of

    \texttt{SAPD+} in \cref{cor:complexity VR}. \sa{For WCSC setting,

    \texttt{SAPD+} using variance reduction achieves an
    oracle} complexity of $\mathcal{O}(L\kappa_y^{2}\epsilon^{-3})$ \sa{--this bound has} the best $\epsilon$-dependency in the literature to our knowledge, and among all the methods with the $\mathcal{O}(\epsilon^{-3})$ complexity, \sa{our approach has the best condition number, $\kappa_y$, dependency; indeed, prior to this work, the best known complexity was $\mathcal{O}(L\kappa_y^3\epsilon^{-3})$; hence, we establish $\mathcal{O}(\kappa_y)$ factor improvement.}

    \item For the WCMC case, our proposed algorithm \texttt{SAPD+} results in $\mathcal{O}(L^3\epsilon^{-6})$ complexity, which is the best
    to our knowledge, \sa{improving the best known complexity by $\log^3(L/\epsilon^2)$ factor.}
    \item Finally, we demonstrate the efficiency of

    \texttt{SAPD+} on a distributionally robust learning problem with a weakly convex cost and \sa{also} on a (worst-case) multi-class classification problem in deep learning. \vspace*{-2mm}
\end{itemize}
\vspace*{-2.5mm}

\renewcommand{\arraystretch}{1.3}
\begin{table}[]
\label{table-one}
{\scriptsize
\begin{center}
\begin{tabular}{c l c c c }
\hline
Ref. &  Complexity & Compactness & VR-based & \multicolumn{1}{c}{Batchsize}  \\
 \cline{2-5}
 \multicolumn{5}{c}{\textbf{Weakly Convex-Strongly Concave (WCSC) problems}}
\\
\hline
$\mathbf{{}^*}$Rafique \emph{et al.}~\cite{rafique1810non} & $\mathcal{O}(\epsilon^{-4}\log(\epsilon^{-1}))$ & (n, n)  &
\XSolidBrush  & $\mathcal{O}(1)$ \\
$\mathbf{{}^{\dag}}$Yan \emph{et al.}~\cite{yan2020optimal} & $\mathcal{O}(\epsilon^{-4}\log(\epsilon^{-1}))$ &   (y, y) & \XSolidBrush & $\mathcal{O}(1)$ \\

Lin \emph{et al.}~\cite{lin2020gradient} &  $\mathcal{O}(L\kappa^3_y\epsilon^{-4})$ &  (n, y)  & \XSolidBrush & $\mathcal{O}(\kappa_y\epsilon^{-2})$ \\

Bot and \"{B}ohm~~\cite{boct2020alternating} &  $\mathcal{O}(L\kappa^3_y\epsilon^{-4})$ &  (n, n)   & \XSolidBrush & $\mathcal{O}(\kappa_y\epsilon^{-2})$ \\

$\mathbf{{}^\ddagger}$Huang \emph{et al.}~\cite{huang2021efficient}&    $\mathcal{O}(\kappa_y^5\mu_y^{-1}\epsilon^{-3})$ & (n, n)   &
\Checkmark & $\mathcal{O}(\kappa_y\epsilon^{-1}),\;\mathcal{O}(\kappa_y^2\epsilon^{-2}),\; \mathcal{O}(\kappa_y\epsilon^{-1})$ \\

$\mathbf{{}^{\boldsymbol{\Sigma}}}$Huang \emph{et al.}~\cite{huang2022accelerated}& $\tilde{\mathcal{O}}(L^{1.5}\kappa_y^{3.5}\epsilon^{-3})$ & (y, y)  &
\Checkmark & $\mathcal{O}(\sqrt{\kappa_y})$ \\

Luo \emph{et al.}~\cite{luo2020stochastic}& $\mathcal{O}(L\kappa_y^{3}\epsilon^{-3})$ &   (y, y) & \Checkmark & $\mathcal{O}(\kappa_y\epsilon^{-1}),\;\mathcal{O}(\kappa_y^2\epsilon^{-2}),\;\mathcal{O}(\kappa_y\epsilon^{-1}) $ \\

Xu \emph{et al.}~\cite{xu2020enhanced}& $\mathcal{O}(L\kappa_y^{3}\epsilon^{-3})$ &  (y, y)  & \Checkmark & $\mathcal{O}(\kappa_y\epsilon^{-1}),\;\mathcal{O}(\kappa_y^2\epsilon^{-2}),\; \mathcal{O}(\kappa_y\epsilon^{-1})$  \\

\hdashline
\texttt{SAPD+}, \cref{thm:unbounded-domains} & $\mathcal{O}(L\kappa_y\epsilon^{-4})$ &  (n, n) & \XSolidBrush &  $\mathcal{O}(1) $ \\

\texttt{SAPD+}, \cref{cor:complexity VR} &  $\mathcal{O}(L\kappa_y^{2}\epsilon^{-3})$ &  (n, n)   &  \Checkmark &  $\mathcal{O}(\kappa_y\epsilon^{-1}),\; \mathcal{O}(\kappa_y\epsilon^{-2}),\; \mathcal{O}(\epsilon^{-1})$  \\

\hline
 \multicolumn{5}{c}{\textbf{Weakly Convex-Merely Concave (WCMC) problems}}\\
\hline
Rafique \emph{et al.}~\cite{rafique1810non} &  $\sa{\mathcal{O}(L^3\epsilon^{-6}\log^3(L\epsilon^{-2}))}$  &  (y, y)  &
- & $\mathcal{O}(1)$ \\

Bot and \"{B}ohm~~\cite{boct2020alternating} &  $\mathcal{O}(L^5\epsilon^{-8})$ &  (n, y)  & - & $\mathcal{O}(1)$\\

Lin \emph{et al.}~\cite{lin2020gradient} &  $\mathcal{O}(L^{3}\epsilon^{-8})$ &  (n, y)  & - & $\mathcal{O}(1)$ \\
\hdashline
\texttt{SAPD+}, \cref{Thm:WCMC} &  $\mathcal{O}(\sa{L^3}\epsilon^{-6})$ &  (n, y) &  - &  $\mathcal{O}(1)$ \\
\hline
\end{tabular}
\caption{{\footnotesize Summary of relevant work \sa{for WCSC and WCMC} problems. For \mg{the column} ``Compactness", we use y,n to represent requiring and not requiring compactness, respectively; the first argument is for primal domain and the second is for dual domain.
For batchsize, \sa{we use \sa{$(b',b,q)$} format for VR-based methods; from left to right, \emph{small batch} $(b')$, \emph{large batch} $(b)$, \emph{frequency} $(q)$}.\newline
\textbf{Table notes:} \sa{$\mathbf{{}^*}$For WCSC setting, \cite{rafique1810non} assumes
$\Phi(\cdot,y) \triangleq c^{\top}(\cdot)y$ is weakly convex and $g(\cdot)$ is strongly convex. ${}^\dag$
In~\cite{yan2020optimal}, $\mathcal{L}=\Phi$ and $\Phi$ need not be smooth, rather second moment of stochastic subgradients is assumed to be uniformly bounded. If $ \Phi$ is $L$-smooth, naively assuming $\Phi(\cdot,y)$ and $\Phi(x,\cdot)$ are $L_\Phi$-Lipschitz, then their result implies $\mathcal{O}(L_\Phi^2\kappa_y^2\epsilon^{-4}\log^2(\sqrt{\kappa_y}L_\Phi/\epsilon))$. ${}^{\ddagger,{\boldsymbol{\Sigma}}}$ The complexity result reported here are different than those in~\cite{huang2021efficient,huang2022accelerated}. The issues in their proofs leading to the wrong complexity results are explained in appendix~I
.}
}}
\end{center}}
\vspace*{-4mm}
\end{table}

\paragraph{Notation.}
Throughout the paper, $\|\cdot\|$ denotes the Euclidean norm.
Given $f:\mathbb{R}^n\to\mathbb{R}\cup\{\infty\}$ a closed convex function,
$\prox{\lambda f}(x)\triangleq\argmin_{w} f(w)+\frac{1}{2\lambda}\|w-x\|^2$ denotes the proximal map of $f$. Given \mg{random} $\omega$, we let $\tilde{\nabla}_x\Phi(x,y;\omega)$ and $\tilde{\nabla}_y\Phi(x,y;\omega)$ to be \mg{unbiased} estimators of $\nabla\Phi_x(x,y)$ and $\nabla\Phi_y(x,y)$. Moreover,
given a random mini-batch $\mathcal{B}=\{\omega_i\}_{i=1}^b$, we let $\tilde{\nabla}_x\Phi_{\mathcal{B}}(x,y)=\frac{1}{b}\sum_{i=1}^b\tilde{\nabla}_x\Phi(x,y;\omega_i)$ to denote the stochastic gradient estimate based on the batch $\mathcal{B}$, \sa{and we define $\tilde{\nabla}_y\Phi_{\mathcal{B}}(\cdot,\cdot)$ similarly.}
\vspace*{-2mm}
\section{Preliminaries}\label{sec-preliminaries}
\vspace*{-2mm}
\sa{ \mg{We start with describing the notion of weak convexity.}}
\begin{definition}\label{def: weakly convex}
\sa{$f:\mathbb{R}^d\to\mathbb{R}\cup\{+\infty\}$ is $\gamma$-weakly convex if $x\mapsto f(x)+\frac{\gamma}{2}\|x\|^2$ is convex.}
\end{definition}
\begin{definition}\label{def: l-smooth}
A \sa{differentiable function $f:\mathbb{R}^d\to\mathbb{R}\cup\{+\infty\}$} is $L$-smooth if \sa{$\exists L>0$ such that} for $\forall x,x'\in\mathop{\bf dom} f$, $\|\nabla f(x) - \nabla f(x')\|\leq L\|x-x'\|$.
\end{definition}
\begin{remark}
\label{rem:smooth-WC}
\sa{If a function is $L$-smooth, then it is also $L$-weakly convex.}
\end{remark}
Remark \ref{rem:smooth-WC} shows that weak convexity is a rich class containing the class of smooth functions.
\mg{In the rest of the paper, we consider the SP problem in~\eqref{eq:main problem}. Next, we introduce our assumptions.}

\begin{assumption}
\label{ASPT: fg}
$f:\mathcal{X}\rightarrow \mathbb{R}\cup\{\infty\}$ and $g:\mathcal{Y}\rightarrow \mathbb{R}\cup\{\infty\}$ are proper, closed, convex functions.
Let $\Phi:\mathcal{X}\times\mathcal{Y}\to\mathbb{R}$ be such that

    (i) for any $y\in \mathop{\bf dom} g\subset \mathcal{Y}$,
    $\Phi(\cdot,y)$ is $\gamma$-weakly convex and bounded from below;

    (ii) for any $x\in\mathop{\bf dom} f\subset \mathcal{X}$,
    $\Phi(x, \cdot)$ is $\mu_y$-strongly concave \sa{for some $\mu_y\geq 0$}; \sa{(iii) $\Phi$ is differentiable on an open set containing $\mathop{\bf dom} f\times \mathop{\bf dom} g$.}
\end{assumption}

\begin{assumption}\label{ASPT: lipshiz gradient}
There exist $L_{xx},L_{yy} \geq 0$, $L_{xy},L_{yx} > 0$ such that

        $\| \nabla_x \Phi(x,y) - \nabla_x \Phi(\bar{x},\bar{y})\|
        \leq
        L_{xx}\|x- \bar{x}\|+L_{xy}\|y-\bar{y}\|$, and $\| \nabla_y \Phi(x,y) - \nabla_y \Phi(\bar{x},\bar{y})\|
        \leq
        L_{yx}\|x- \bar{x}\|
        + L_{yy}\|y- \bar{y}\|$ for all $x,\bar{x}\in \mathop{\bf dom} f\subset \mathcal{X}$, and $y,\bar{y}\in \mathop{\bf dom} g\subset \mathcal{Y}$.

\end{assumption}

{Assumption \ref{ASPT: fg} allows non-convexity in $x$ while requiring \sa{(strong)} concavity in the $y$ variable.}
Assumption~\ref{ASPT: lipshiz gradient} is standard in the analysis of first-order methods \sa{for solving SP problems}. It should be noticed that when $L_{yx}=L_{xy}=0$, the problem \sa{in \eqref{eq:main problem}} can be solved separately for the primal and dual variables; hence, it is natural to assume $L_{yx}, L_{xy}>0$.

\begin{wrapfigure}{R}{0.57\textwidth}
\vspace{-0.9cm}
\begin{minipage}{0.57\textwidth}
\begin{algorithm}[H]
\caption{\texttt{SAPD} Algorithm}
{\footnotesize
\begin{algorithmic}[1]
\State {\textbf{Input:} $\tau,\sigma,\theta,\mu_x,x_0, y_0,N$}
\State \sa{$\tilde q_0\gets 0$}
\For{$k\geq 0$}
\State $\tilde{s}_k \leftarrow \sa{\tilde\nabla_y} \Phi(x_k,y_k;\sa{\omega_k^y}) + \theta \tilde{q}_k$
\State $y_{k+1}\leftarrow\prox{\sigma g}(y_k+\sigma \tilde{s}_k)$
\State $x_{k+1}\leftarrow\prox{\tau f}(x_k-\tau\sa{\tilde\nabla_x}\Phi(x_k,y_{k+1};\sa{\omega_k^x}))$
\State \sa{$\tilde{q}_{k+1}\leftarrow\sa{\tilde\nabla_y} \Phi(x_{k+1},y_{k+1};\sa{\omega_{k+1}^y}) -\sa{\tilde\nabla_y}\Phi(x_{k},y_{k};\sa{\omega_{k}^y})$}
\EndFor
\State {\textbf{Output}:}{$(\bar{x}_N,\bar y_N)=\frac{1}{K_N(\sa{\theta})}\sum_{i=0}^{N-1}\sa{\theta^{-k}}(x_{k+1},y_{k+1})$}
\end{algorithmic}}%

\label{ALG: SAPD}
\end{algorithm}
\end{minipage}
\end{wrapfigure}
\sa{Suppose that we implement \texttt{SAPD}, stated in \mg{Algorithm} \ref{ALG: SAPD},
on the SCSC problem
{\small
\begin{align}
\label{eq:WCSC-subproblem-generic}
\min_{x\in\mathcal{X}}\max_{y\in\mathcal{Y}}\mathcal{L}(x,y)+\frac{\mu_x+\gamma}{2}\norm{x-\bar{x}}^2
\end{align}}%
for some given $\mu_x>0$ and $\bar{x}\in\mathcal{X}$ --strong convexity follows from $\mathcal{L}(\cdot,y)$ being $\gamma$-weakly convex.
We
make the following assumption on the statistical nature of the gradient noise as in, e.g., \cite{cga-sapd,fallah2020optimal,zhang2021robust}}.
\hspace*{5cm}

\begin{assumption}\label{ASPT: unbiased noise assumption}
\sa{Given arbitrary $\bar{x}\in\mathcal{X}$ and $\mu_x>0$, let $\{(x_k,y_k)\}$ be generated by \texttt{SAPD} running on \eqref{eq:WCSC-subproblem-generic}.} There exist $\delta_x,\delta_y\geq 0$ such that for
all \sa{$k\geq 0$},
the stochastic gradients $\tilde{\nabla}_x\Phi(x_k,y_{\sa{k+1}};\omega_k^x)$, $\tilde{\nabla}_y\Phi(x_k,y_k;\omega_k^y)$ and random sequences $\{\omega_k^x\}_k$, $\{\omega_k^y\}_k$
satisfy the following conditions:
\begin{enumerate}[label=(\roman*),itemsep=-1ex,topsep=0pt]
    \item $\{\omega_k^x\}_k$ and $\{\omega_k^y\}_k$ are
    independent sequences, also independent from each other;
    \item $\mathbb{E}[\tilde{\nabla}_x\Phi(x_k,y_{k+1};\omega_k^x)|x_k,y_{k+1} ] = \nabla_x\Phi(x_k,y_{k+1})$;
    \item $\mathbb{E}[\tilde{\nabla}_y\Phi(x_k,y_{k};\omega_k^y)|x_k,y_k ] = \nabla_y\Phi(x_k,y_k)$;
    \item $\mathbb{E}[\|\tilde{\nabla}_x\Phi(x_k,y_{k+1};\omega_k^x) - \nabla_x\Phi(x_k,y_{k+1})\|^2 |x_k,y_{k+1} ]\leq \delta_x^2$;
    \item $\mathbb{E}[\|\tilde{\nabla}_y\Phi(x_k,y_{k};\omega_k^y) - \nabla_y\Phi(x_k,y_k)\|^2 |x_k,y_k ]\leq \delta_y^2$.
\end{enumerate}
\end{assumption}
{\cref{ASPT: unbiased noise assumption} says that the gradient noise conditioned
\sa{on} the iterates is unbiased with a finite variance, such assumptions are common in the literature, e.g., \cite{cga-sapd,fallah2020optimal,zhang2021robust}, and can be satisfied when gradients are estimated from randomly sampled data points with replacement.} For \sa{WCSC minimax problems}, \sa{a commonly adopted definition for $\epsilon$-stationary is based} on Moreau envelope, \sa{e.g., see~\cite{lin2020gradient,yan2020optimal}}. It is inspired by Davis and \sa{Drusvyatskiy's work}~\cite{davis2019stochastic} for solving weakly convex minimization problems. \sa{For the sake of completeness, we briefly review this idea below.}
\begin{definition}\label{Def: Moreau envelope}
\sa{Let $\phi:\mathbb{R}^d\rightarrow\mathbb{R}\cup\{+\infty\}$ be $\gamma$-weakly convex. Then, for any  $\lambda\in(0,\gamma^{-1})$, Moreau envelope of $\phi$ is defined as $\phi_{\lambda}:\mathbb{R}^d\rightarrow\mathbb{R}$ \sa{such that} $\phi_{\lambda}(x)\triangleq\min_{w\in\mathcal{X}} \phi(w) + \frac{1}{2\lambda}\|w-x \|^2$.}
\end{definition}
\begin{lemma}\label{Lemma: graident of ME}
\sa{Let $\phi:\mathbb{R}^d\rightarrow\mathbb{R}\cup\{+\infty\}$ be a $\gamma$-weakly convex function. For any given $\lambda\in(0,\gamma^{-1})$, $\phi_{\lambda}(\cdot)$} is {well-defined} on $\mathcal{X}$. Moreover, \sa{$\nabla \phi_{\lambda}(x) = \lambda^{-1}(x - \prox{\lambda\phi}(x))$ for $x\in\mathcal{X}$; hence, $\phi_\lambda$ is $\lambda^{-1}$-smooth, where $\prox{\lambda\phi}(\sa{{x}}) = \argmin_{\sa{w}\in \mathcal{X}} \{ \phi(w)+\tfrac{1}{2\lambda}\|\sa{w} - \sa{{x}}\|^2\}$.}%
\end{lemma}
\begin{definition}\label{def:primal-function}
\sa{Under \cref{ASPT: fg}, define $\phi:\mathbb{R}^d\to\mathbb{R}\cup\{+\infty\}$ such that $\phi(x) \triangleq \max_{y\in\mathcal{Y}}\mathcal{L}(x,y)$
for {$x\in\mathcal{X}$}, i.e., $\phi(x)=f(x)+\phi^s(x)$, where $\phi^s(x)\triangleq\max_{y\in\mathcal{Y}}\Phi(x,y)-g(y)$ for $x\in\mathop{\bf dom} f$.}
\end{definition}
\begin{remark}
\sa{Under \cref{ASPT: fg},  since \sa{$\Phi(\cdot,y)$ is $\gamma$-weakly convex for any $y\in\mathop{\bf dom} g$, $\phi^s$ is $\gamma$-weakly convex}}\footnote{One can argue that $\phi^s(\cdot) + \frac{\gamma}{2}\|\cdot\|^2$ is \sa{a pointwise supremum} of convex functions.}; hence, $\phi$ is also $\gamma$-weakly convex. Note that
\begin{equation}
\label{eq:prox-SCSC}
    \prox{\lambda\phi}(\sa{{x}}) = \argmin_{\sa{w}\in \mathcal{X}} \{ \phi(w)+\tfrac{1}{2\lambda}\|\sa{w} - \sa{{x}}\|^2\}=\argmin_{\sa{w}\in \mathcal{X}} \max_{y\in\mathcal{Y}}  \mathcal{L}(\sa{w},y)+\tfrac{1}{2\lambda}\|\sa{w} - \sa{{x}}\|^2.
\end{equation}
Furthermore, when $\mu_y>0$, $\phi^s$ is differentiable \sa{on $\mathop{\bf dom} f$}.
\end{remark}

\mg{In the following definition, we introduce the notion of $\epsilon$-stationary with respect to the GNME metric.}
\begin{definition}\label{Def: stationary point}
A point \sa{$x_\epsilon$ is an $\epsilon$-stationary point of a $\gamma$-weakly convex function $\phi$} if $\|\nabla\phi_{\lambda}(x_\epsilon)\|\leq \epsilon$ for some $\lambda\in(0,\gamma^{-1})$. If $\epsilon=0$, then \sa{$x_\epsilon$} is a stationary point \sa{of $\phi$.}
\end{definition}
\sa{Thus, \mg{from Lemma \ref{Lemma: graident of ME},} computing an $\epsilon$-stationary point $x_\epsilon$ for $\phi$ is equivalent to searching for $x_\epsilon$ such that $\|x_\epsilon-\prox{\lambda\phi}(x_\epsilon) \|$ is small.}
\sa{Recall that for any $\lambda\in(0,\gamma^{-1})$,
$\prox{\lambda\phi}(x)$ is well-defined and unique.
\mg{We also \sa{observe} from} \eqref{eq:prox-SCSC} that  $\prox{\lambda\phi}(\cdot)$ computation is indeed an SCSC SP problem.
To compute $x_\epsilon$ such that $\|x_\epsilon-\prox{\lambda\phi}(x_\epsilon) \|$ is small,
it is natural to consider the \mg{iPPM algorithm} (see e.g. \cite{iusem2003inexact}), which computes $x^{t+1}_{0}\approx \prox{\lambda\phi}(x^{t}_{0})$\mg{, i.e. proximal steps are} ``inexactly'' computed for $t\geq 0$, starting from an arbitrary given point $x^0_0\in\mathcal{X}$. \mg
\sa{Next,} we
\sa{describe}
\texttt{SAPD+}, \sa{an iPPM algorithm employing \texttt{SAPD} to \sa{\emph{inexactly}} solve the SCSC subproblems arising in the iPPM iterations.}}
}

\newpage
\subsection{\sa{The proposed algorithm \xz{\texttt{SAPD+}} and its analysis}}
\label{subsection: Algorithm and analysis}

\begin{wrapfigure}{R}{0.57\textwidth}
\vspace{-0.25in}
\begin{minipage}[H]{0.57\textwidth}
\begin{algorithm}[H]
\caption{\sa{\texttt{SAPD+}} Algorithm}
{\footnotesize
\begin{algorithmic}[1]
\State \textbf{Input:} $\{\tau,\sigma,\theta,\mu_x\}$, $(x^0_0, y^0_0) \in \mathcal{X} \times \mathcal{Y}$, $\{N_t\}_{t\geq 0}\in \mathbb{Z}^+$
\For{$t=0,{1},2,...,T$}
\If{\texttt{VR-flag} == \textbf{false}}
\State $(x^{t+1}_{0},y^{t+1}_{0})\gets \texttt{SAPD}(\tau,\sigma,\theta,\mu_x,x^{t}_{0},y^{t}_{0},N_t)$ \label{SAPD step chpt 2}
\Else
\State $(x^{t+1}_{0},y^{t+1}_{0})\gets \texttt{VR-SAPD}(\tau,\sigma,\theta,\mu_x,x^{t}_{0},y^{t}_{0},N_t)$
\EndIf
\EndFor
\end{algorithmic}}%
\label{Alg: SAPD-plus}
\end{algorithm}
\end{minipage}
 \end{wrapfigure}
\sa{The convergence and robustness properties of \texttt{SAPD} for SCSC
SP problems are analyzed in~\cite{zhang2021robust}. For the WCSC
SP~problems, as we explained in the previous subsection, the main idea is to apply \sa{the iPPM framework as stated in~\texttt{SAPD+}~(see~\cref{Alg: SAPD-plus})} which requires successively solving SCSC SP problems. In the rest, the counter for iPPM is denoted with $t\in\mathbb{Z}_+$. At each outer iteration $t\geq 1$, we inexactly compute the prox map, i.e., $x^{t+1}_{0}\approx \prox{\lambda\phi}(x^{t}_{0})$, which is well-defined for $\lambda\in(0,\gamma^{-1})$; hence, to derive our preliminary results, we fix $\lambda=(\mu_x+\gamma)^{-1}$ for some given $\mu_x>0$ -- thus, $\mathcal{L}(x,y)+\frac{\mu_x+\gamma}{2}\norm{x-x^{t}_{0}}^2$ is SCSC in $(x,y)$ with moduli $(\mu_x,\mu_y)$ and has a unique saddle point.}
\sa{To summarize, we will construct $\{x^{t}_{0}\}_{t=1}^{T}\subset \mathop{\bf dom} f$
by inexactly solving the following SCSC SP problem in \eqref{eq: SCSC problem chpt2} at each outer iteration $t\in\mathbb{Z}_+$ through running \texttt{SAPD} for $N_t\in\mathbb{Z}_+$ iterations:}
\begin{equation}\label{eq: SCSC problem chpt2}
  \min_{x\in\mathcal{X}} \max_{y\in\mathcal{Y}}
  \mathcal{L}^{t}(x,y)\triangleq f(x)+\Phi^{t}(x,y)-g(y),~\text{where}~\Phi^{t}(x,y) =\Phi(x,y) + \frac{\mu_x+\gamma}{2}\|x-x^{t}_{0}\|^2,
\end{equation}
\mg{where we will specify $N_t\in\mathbb{Z}_+$ later}. Hence, $\Phi^{t}$ is \sa{an SCSC}
function with moduli $(\mu_x,\mu_y)$.

\sa{Next, we
briefly explain the main step of \texttt{SAPD+} with \texttt{VR-flag}=\textbf{false}.}
The statement $(x^{t+1}_{0},y^{t+1}_{0})\gets \texttt{SAPD}(\tau,\sigma,\theta,x^{t}_{0}, y^{t}_{0}, N_t)$ in line \ref{SAPD step chpt 2} of~\cref{Alg: SAPD-plus} means that $(x^{t+1}_{0},y^{t+1}_{0})$ is generated using \texttt{SAPD}, as stated in~\cref{ALG: SAPD} --indeed, \texttt{SAPD} is run on \eqref{eq: SCSC problem chpt2} for $N_t$ iterations with \texttt{SAPD} parameters $(\tau,\sigma,\theta)$ and starting from the initial point $(x^{t}_{0},y^{t}_{0})$.

To analyze the convergence of \xz{\texttt{SAPD+}}, we first define the gap function $\mathcal{G}^{t}$ for $t$-th \xz{\texttt{SAPD+}} iteration:
\begin{equation}
    \mathcal{G}^{t}(x,y) \triangleq \max_{y'\in\mathcal{Y}} \mathcal{L}^{t}(x,y') -  \min_{x'\in\mathcal{X}} \mathcal{L}^{t}(x',y).
\end{equation}
\sa{Recall that $\mathcal{L}^{t}$ is an SCSC function; therefore, \emph{i)} it has a unique saddle point denoted by $(x^{t}_{*},y^{t}_{*})$, and \sa{it is important to note that $x^{t}_{*}=\prox{\lambda\phi}(x^{t}_{0})$ for $\phi(x)=\max_{y\in\mathcal{Y}}\mathcal{L}(x,y)$ and $\lambda=(\gamma+\mu_x)^{-1}$;} \emph{ii)} {for any $(x,y)\in\mathop{\bf dom} f\times\mathop{\bf dom} g$},
the following quantities are well-defined:}
\begin{equation}\label{eq: x* y*}
    x^{t}_{*}(y) \triangleq\argmin_{x'\in\mathcal{X}} \mathcal{L}^{t}(x',y),\quad y_{*}(x) \triangleq \argmax_{y'\in\mathcal{Y}} \mathcal{L}^{t}(x,y')\sa{=\argmax_{y'\in\mathcal{Y}} \mathcal{L}(x,y').}
\end{equation}
Thus, it follows that
$\mathcal{G}^{t}(x,y) = \mathcal{L}^{t}(x,y_{*}(x)) - \mathcal{L}^{t}(x^{t}_{*}(y), y)$.
Moreover, we also define
    $\mathcal{G}(x,y) \triangleq \sup_{y'\in\mathcal{Y}} \mathcal{L}(x,y') -  \inf_{x'\in\mathcal{X}} \mathcal{L}(x',y)$.

\sa{\cref{ASPT: fg}} ensures that $\mathcal{G}$
is well defined.
\sa{We first provide our oracle complexity
\mg{in the GNME metric} under the compactness assumption of the primal-dual domains; later, in
\cref{sec:compactness}, we show that under \mg{a} particular subdifferentiability assumption compactness requirement can be avoided.}
\begin{assumption}
\label{assump:compact}
\sa{$\mathop{\bf dom} f$ and $\mathop{\bf dom} g$ are compact sets.}
\end{assumption}
\begin{theorem}
\label{cor:complexity-fg}
\sa{Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption}, and~\ref{assump:compact} hold. Let $\mu_x=\gamma$, $\{\tau,\sigma,\theta\}$ and $N$ be chosen as
{\small
\begin{equation}
\label{Condition: SP solution to noisy LMI label 2}
N = \frac{\ln(265)}{\ln(\frac{1}{\theta})}+1,
 \quad \tau = \frac{1-\theta}{
\gamma},\quad \sigma = \frac{1-\theta}{\mu_y\theta},
\quad
\theta  \geq \max\{\bar{\theta}\sa{(\beta)},~\overline{\overline{\theta}}_1,~\overline{\overline{\theta}}_2\},
\end{equation}}
\sa{for $\beta=\min\{\frac{1}{2},~\frac{\mu_y}{4\gamma},~\frac{\gamma}{4\mu_y},~\frac{L_{yx}}{2L_{xy}}\}$,} where $\bar{\theta}(\beta)\triangleq\max\{\bar{\theta}_1(\beta),~\bar{\theta}_2(\beta)\}\in(0,1)$ such that}
\sa{\footnotesize
\begin{equation*}
        \bar{\theta}_1(\beta)\triangleq 1
         - \tfrac{\beta\mu_y\xz{L'_{xx}}}{2L^2_{yx}}\big(\sqrt{1 + \tfrac{4L^2_{yx}\mu_x}{\beta \xz{{L'_{xx}}^2}\mu_y}} - 1 \big)

        ,\quad \bar{\theta}_2(\beta)\triangleq
        \begin{cases}
        1 - \tfrac{(1-\beta)^2}{8}\tfrac{\mu_y^2}{L_{yy}^2} \Big( \sqrt{1+\tfrac{16L_{yy}^2}{(1-\beta)^2\mu_y^2}}-1\Big) & L_{yy}>0\\
        0 & L_{yy}=0,
        \end{cases}
\end{equation*}}%
{\footnotesize
\begin{equation}\label{eq:theta_double_bar_bound}
\begin{aligned}

    \overline{\overline{\theta}}_1  = \max\Big\{0,1 -
\frac{1}{\sa{384
}\cdot
\frac{\epsilon^2}{\delta_x^2}\Big\},\qquad \overline{\overline{\theta}}_2 = \sa{\Big(1 +
\frac{1}{\sa{384}
}\cdot
\frac{\mu_y}{10\gamma}
\frac{\epsilon^2}{\delta_y^2}\Big)^{-1}.}
\end{aligned}
\end{equation}}%
For $\epsilon>0$, \xz{when \texttt{VR-flag}=\textbf{false}},
\sa{\xz{\texttt{SAPD+}} iterate $x_0^T$ is $\epsilon$-stationary, i.e., $\mathbb{E}\left[\|\nabla\phi_{\lambda}(x_0^T)\|\right]\leq \epsilon$, for $T \geq \sa{96}\mathcal{G}(x_0^0,y_0^0)  \cdot\frac{\gamma}{\epsilon^2}+1$, which requires $C_\epsilon$ oracle calls in total where}
{\footnotesize
\begin{align*}
\sa{C_\epsilon=}\mathcal{O}\Big(
\Big(\tfrac{\max\{L_{xx},L_{yx}\}}{\gamma} + \tfrac{\max\{L_{yx},L_{xy}\}}{\sqrt{\gamma\mu_y}} + \tfrac{\max\{L_{yy},L_{yx}\}}{\mu_y}\Big)
\gamma\cdot\epsilon^{-2}+ \Big(
\tfrac{\delta_x^2}{\gamma} +
\tfrac{\delta_y^2}{\mu_y} \Big)
\sa{\gamma^2}\cdot\epsilon^{-4}\Big)\mathcal{G}(x_0^0,y_0^0).
\end{align*}}%
\end{theorem}

\begin{remark}
\sa{For any $y\in\mathop{\bf dom} g$, since $\Phi(\cdot,y)$ $L_{xx}$-smooth, it is necessarily $L_{xx}$-weakly convex; hence, $\gamma\leq L_{xx}$. To get a worst-case complexity, let}
\begin{equation}\label{eq:uniform setting}
   L \triangleq \max\{L_{xy},L_{yx},L_{xx},L_{yy}\},~\kappa_y \triangleq L/\mu_y,~\delta \triangleq \max\{\delta_x,\delta_y\},~\gamma = L.
\end{equation}
\sa{Our oracle complexity \small{$C_\epsilon$} in \cref{cor:complexity-fg} can be simplified as}
\small{$\sa{C_\epsilon}=\mathcal{O}\left(\max\{1, \tfrac{\delta^2}{\epsilon^2}\}\tfrac{\kappa_y L\mathcal{G}(x_0^0,y_0^0)}{\epsilon^2} \right)$}.

In fact, Li et al.~\cite{li2021complexity} (see also~\cite{zhang2021complexity}) provide a lower complexity bound for a class of first-order stochastic algorithms that do not use variance reduction. The lower bound for finding $\epsilon$-stationary \sa{points of smooth WCSC problems}
\sa{in GNP metric} is  $\Omega(L\Delta_{\phi}(\sqrt{\kappa_y}\epsilon^{-2}+\kappa_y^{\frac{1}{3}}\epsilon^{-4}))$, where $\Delta_{\phi} = \phi(x_0) - \min_{x\in\mathcal{X}}\phi(x)$ with  $x_0$ the initial point.
\end{remark}

\sa{Consider $\phi=f+\phi^s$ as given in \cref{def:primal-function}.} \mg{For $\lambda>0$, the map $G_\lambda: \mathbb{R}^d \to \mathbb{R}^d$ defined as
{\small
\begin{equation*}G_\lambda(\tilde{x})\triangleq \frac{1}{\lambda} [\tilde{x} -
\prox{\lambda f}
\big(
\tilde{x} - \lambda\nabla\phi^s(\xz{\tilde{x}})
\big)]
\end{equation*}}%
is called the\ \emph{generalized gradient mapping} and its norm is frequently used in optimization for assessing stationarity (see e.g. \cite{drusvyatskiy2018error}).
\cref{cor:complexity-fg} provides guarantees in the GNME metric. \sa
\cref{thm:metric-equivalance} shows that given $x_\epsilon$, an $\epsilon$-stationary point in GNME metric (see~\cref{Def: stationary point}), we can generate $\tilde{x}$ such that $\norm{G_\lambda(\tilde{x})}\leq \epsilon$ for some $\lambda>0$, i.e.,
an $\epsilon$-stationary point in
\emph{generalized gradient mapping} metric,
within $\tilde{\mathcal{O}}(1/\epsilon^2)$ \texttt{SAPD}
iterations. When $f(\cdot)=0$, this metric becomes equivalent to the GNP metric.}}
\begin{theorem}
\label{thm:metric-equivalance}
\xz{Suppose Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption} hold}. Given \sa{$x_\epsilon$}, an $\epsilon$-stationary point for the $\gamma$-weakly convex function $\phi(\cdot)=\max_{y\in\mathcal{Y}} \mathcal{L}(\cdot,y)$, i.e., $\|\nabla\phi_{\lambda}(\sa{x_\epsilon})\|\leq\frac{\epsilon}{2}$ for
\sa{some fixed} $\lambda\in(0,\gamma^{-1})$. \sa{Let $\phi^s(\cdot)=\max_{y\in\mathcal{Y}}\Phi(\cdot,y)-g(y)$ so that $\phi=f+\phi^s$.} Then, initialized from $x_\epsilon$, within
$\tilde{\mathcal{O}}(\frac{1}{\xz{\epsilon^2}})$ stochastic first-order oracle calls,
\texttt{SAPD}, stated in~\cref{ALG: SAPD}, can generate \sa{$\tilde x$} such that
\mg{$\mathbb{E}\big[ \norm{G_\lambda(\tilde{x})} \leq \epsilon$}.
\end{theorem}

\begin{remark}
\sa{Suppose \texttt{SAPD+}
requires $T_\epsilon$ iterations to output $x_\epsilon$ such that $\|\nabla\phi_{\lambda}(\sa{x_\epsilon})\|\leq\frac{\epsilon}{2}$. Then
\mg{generating}
$\tilde{x}$ from \mg{$x_\epsilon$ such that $\mathbb{E}\big[ \norm{G_\lambda(\tilde{x})} \leq \epsilon$} requires
an additional \texttt{SAPD+} iteration, i.e., $T_\epsilon+1$ iterations in total, where in the last iteration we run \texttt{SAPD} for $\tilde\mathcal{O}(\epsilon^{-2})$ iterations rather than $N$.}
\end{remark}
\subsection{\sa{Relaxing the compactness assumption}}
\label{sec:compactness}
In \mg{the} previous section, we assume that $\mathop{\bf dom} f$ and $\mathop{\bf dom} g$ are compact sets, e.g., $f(\cdot)=\mathbbm{1}_X(\cdot)$ and $g(\cdot)=\mathbbm{1}_Y(\cdot)$, where $X\subset\mathcal{X}$ and $Y\subset\mathcal{Y}$ are compact convex sets. In this section, we show that \texttt{SAPD+} can also handle unbounded domains under the following assumption.
\begin{assumption}
\label{assump:bounded-subdifferential}
\sa{For $f$ and $g$ closed convex, suppose $\exists B_f,~B_g>0$ such that $\inf\{\norm{s_f}:\ s_f\in\partial f(x)\}\leq B_f$ for all $x\in\mathop{\bf dom} f$ and $\inf\{\norm{s_g}:\ s_g\in\partial g(y)\}\leq B_g$ for all $y\in\mathop{\bf dom} g$.}
\end{assumption}
\begin{remark}
Assumption~\ref{assump:bounded-subdifferential} holds when $f$ is an indicator function of a closed convex set (not necessarily bounded) or for $f:\mathbb{R}^d\to\mathbb{R}\cup\{\infty\}$ such that $\mathop{\bf dom} f$ is open and $f$ is Lipschitz. Two important examples for this scenario: (i) $f(\cdot)=0$,
(ii) $f$ is a norm, e.g., $\ell_1$-, $\ell_2$-, or the Nuclear norms.
\end{remark}

\sa{The existing work based on iPPM framework either require compactness, e.g.,~\cite{yan2020optimal}, or some special structure on $\mathcal{L}$, e.g.,~\cite{rafique1810non}. This is also true for VR-based methods, e.g.,\cite{huang2022accelerated,luo2020stochastic,xu2020enhanced}. To our knowledge, ours is the first one to overcome this difficulty and strictly improve the complexity bounds for WCSC and WCMC settings without compactness assumption; moreover, the same idea also works simultaneously with a variance reduction technique that will be discussed later (see \cref{sec:VR}). Finally, same trick for removing compactness assumption for WCSC setting also helps removing compactness assumption for the  primal domain in WCMC setting as well (see~\cref{sec:wcmc}).}
\xz{
\begin{remark}
In \cite{lin2020gradient}, when $f=g=0$,
\mg{boundedness} of dual space is required while
\cref{assump:bounded-subdifferential} is a weaker
requirement. \sa{Furthermore, based on the discussion with the authors of \cite{yan2020optimal}, one also needs compactness of the domain for their proof to hold.}
In
\cite{huang2021efficient}, the authors require the sub-level set $\{x:\phi(x)+f(x)\leq \alpha\}$ \mg{to be}
compact for all $\alpha>0$.
\mg{There are simple convex functions that do not satisfy this condition} such as
$f(x)=\max\{0,x\}$.
Bot and \"{B}ohm~~\cite{boct2020alternating} use \mg{milder assumptions than \cite{lin2020gradient}} without requiring compactness; however, \mg{their complexity is the same as the complexity} of \cite{lin2020gradient}.
\end{remark}
}

\begin{theorem}
\label{thm:unbounded-domains}
\sa{The result of \cref{cor:complexity-fg} continues to hold, if one replaces the compact domain assumption, i.e., \cref{assump:compact}, with \cref{assump:bounded-subdifferential}.}

\end{theorem}
\sa{Finally, it is crucial to note that for smooth problems, i.e., $f(\cdot)=g(\cdot)=0$, the variance term for \texttt{SAPD} error bounds can be greatly simplified. Due to limited space, we provide the details in~\cref{sec:smooth-problems}.}
\newpage

\section{Variance reduction}
\label{sec:VR}
\vspace*{-3mm}
\begin{wrapfigure}[23]{r}{0.6\textwidth}
\vspace*{-1.3cm}
\begin{minipage}{0.6\textwidth}
\begin{algorithm}[H]
\caption{\texttt{VR-SAPD} Algorithm}
\label{ALG: SAPD-VR}
{\small
\begin{algorithmic}[1]
\State {\textbf{Input:} $\tau,\sigma,\theta,\mu_x,x_0, y_0,N,\sa{b,b_x,b_y,q}$}
\State \sa{Let $\mathcal{B}_{0}$ be random mini-batch sample  with $|\mathcal{B}_{0}|=b$}
\State \sa{$\tilde w_0\gets \tilde\nabla_y \Phi_{\mathcal{B}_0}(x_0,y_0)$,\quad $\tilde s_0\gets w_0$}
\For{$k\geq 0$}
\State $y_{k+1} \leftarrow \prox{\sigma g}(y_k+\sigma \tilde{s}_k)$
\If{$\mo(k,q)$ == $0$}
\State $v_k\leftarrow\tilde\nabla_x \Phi_{\mathcal{B}_k}(x_k,y_{k+1})$
\Else
\State Let $\mathcal{I}_k^x$ be random mini-batch sample  with $|\mathcal{I}_k^x|=\sa{b_x}$
\State $v_k\leftarrow\tilde\nabla_x \Phi_{\mathcal{I}_k^x}(x_k,y_{k+1})-\tilde\nabla_x\Phi_{\mathcal{I}_k^x}(x_{k-1},y_{k}) + v_{k-1}$
\EndIf
\State $x_{k+1} \leftarrow \prox{\tau f}(x_k-\tau v_k)$
\State Let $\mathcal{B}_{k+1}$ be mini-batch sample  with $|\mathcal{B}_{k+1}|=b$
\If{$\mo(\sa{k+1},q)$ == $0$}
\State \sa{$w_{k+1}\leftarrow\tilde\nabla_y \Phi_{\mathcal{B}_{k+1}}(x_{k+1},y_{k+1})$}
\Else
\State Let $\mathcal{I}_{k+1}^y$ be mini-batch sample  with $|\mathcal{I}_{k+1}^y|=\sa{b_y}$
\State $\tilde{q}_{k+1}\gets\tilde\nabla_y \Phi_{\mathcal{I}_{k+1}^y}(x_{k+1},y_{k+1})-\nabla_y\tilde\Phi_{\mathcal{I}_{k+1}^y}(x_{k},y_{k})$
\State \sa{$w_{k+1}\gets w_{k}+\tilde{q}_{k+1}$}
\EndIf
\State \sa{$\tilde{s}_{k+1} \leftarrow (1+\theta) w_{k+1} - \theta w_{k}$}
\EndFor
\State {\textbf{Output}:} $(\bar{x}_N,\bar{y}_N)=\frac{1}{K_N(\sa{\theta})}\sum_{i=0}^{N-1}\sa{\theta^{-k}}(x_{k+1},y_{k+1})$
\end{algorithmic}}
\end{algorithm}
\vspace{0.45in}
\end{minipage}
\end{wrapfigure}
In this section, we further strength our algorithm with \sa{SPIDER variance reduction technique~\cite{fang2018spider}, a variant of SARAH~\cite{nguyen2017sarah,nguyen2017stochastic},} as stated in \cref{ALG: SAPD-VR}. Indeed, we use a large batchsize \sa{of $b$ in} every $q$ iterations and use small \sa{batchsizes of $b_x$ and $b_y$ for the rest.}
We prove that \texttt{SAPD+}
using variance reduction, \sa{i.e., with \texttt{VR-flag}=\textbf{true}, achieves an oracle complexity of $\mathcal{O}(L\kappa_y^{2}\epsilon^{-3})$; hence, we provide an $\mathcal{O}(\kappa_y)$ factor improvement over the best known complexity in the literature to our knowledge.}

Here, we use
{\small $\tilde\nabla_y\Phi_{\mathcal{B}_k}^t(x_k,y_{k})$} to represent {\small $\frac{1}{|\mathcal{B}_k|} \sum_{\omega_k^i\in  \mathcal{B}_k} \tilde\nabla_y\Phi(x_k,y_y;\omega^i_k)$}, where $\mathcal{B}_k=\{\omega^i_k\}_{i=1}^b$ is the mini-batch with $|\mathcal{B}_k|=b$ and we define $\tilde\nabla_x\Phi_{\mathcal{B}_k}^t(x_k,y_{k+1})$ similarly; in addition, $\mathcal{I}^x_k=\{\omega_k^{x,i}\}$ and $\mathcal{I}^y_k=\{\omega_k^{y,i}\}$ with $|\mathcal{I}_k^x|=b_x$ and $|\mathcal{I}_k^y|=b_y$ denote the small mini-batches we use for generating $\tilde\nabla_y\Phi_{\mathcal{I}^y_k}^t(x_k,y_{k})$ and $\tilde\nabla_x\Phi_{\mathcal{I}^x_k}^t(x_k,y_{k+1})$. Moreover,
Throughout this section we make the following continuity assumption on the stochastic oracle similar
 to~\cite{huang2021efficient,huang2022accelerated,luo2020stochastic,xu2020enhanced}.

\begin{assumption}\label{ASPT: lipshiz gradient VR}
     $\exists L_{xx}, L_{xy},L_{yx},L_{yy} \sa{\geq} 0$ such that $\forall x,\bar{x}\in \mathop{\bf dom} f\subset\mathcal{X}$ and $\forall y,\bar{y}\in \mathop{\bf dom} g\subset \mathcal{Y}$,
     \vspace*{-0.5mm}
    \begin{equation}\label{LGX1 VR}
    \begin{aligned}
           \| \tilde\nabla_y \Phi(x,y;\omega) -\tilde \nabla_y \Phi(\bar{x},\bar{y};\omega)\|
        \leq
        L_{yx}\|x- \bar{x}\|
        + L_{yy}\|y- \bar{y}\|,\quad \sa{w.p.~1}, \\                \| \tilde\nabla_x \Phi(x,y;\omega) - \tilde\nabla_x \Phi(\bar{x},\bar{y};\omega)\|
        \leq
        L_{xx}\|x- \bar{x}\|+L_{xy}\|y-\bar{y}\|,\quad \sa{w.p.~1}.
    \end{aligned}
    \end{equation}
\end{assumption}
\begin{assumption}\label{ASPT: independence} \sa{Consider $\texttt{SAPD+}$ with $\texttt{VR-flag}=\textbf{true}$.}
\sa{For any $k\geq 0$, $\mathcal{I}_k^x$ is independent of $(x_i,y_{i+1})$ and $\mathcal{I}_k^y$ is independent of $(x_i,y_{i})$ for $i\in\{k-1,k\}$.}
\end{assumption}
\begin{remark}
\sa{For finite-sum type problems of the form $\min_x\max_y \mg{\frac{1}{n}}\sum_{i=1}^n\Phi_i(x,y)$, \mg{we can set the stochastic gradient according to
$\tilde \nabla_x \Phi(x,y;\omega) =  \nabla_x \Phi_\omega(x,y)$ and
$\tilde \nabla_y \Phi(x,y;\omega) =  \nabla_y \Phi_\omega(x,y)$ where $\omega$ is uniformly drawn at random from $\{1,\ldots,n\}$}. Therefore, if batch samples $\mathcal{I}^{x}_k,\mathcal{I}^{y}_k$
are drawn from $\{1,\ldots,n\}$ uniformly at random with replacement; batches will be independent of the past iterates satisfying \cref{ASPT: independence}.}
\end{remark}

\begin{theorem}\label{cor:complexity VR}
\sa{Suppose \cref{ASPT: fg},\ref{ASPT: unbiased noise assumption},\ref{ASPT: lipshiz gradient VR} and~\ref{ASPT: independence} hold. Moreover, either \cref{assump:compact} or \cref{assump:bounded-subdifferential} holds. Fix $\theta=1$, and let
$\tau$, $\sigma$,
$b$ and $N$ be chosen as follows:}
\vspace*{-2mm}
{\footnotesize
\begin{align}
    &\tau = \left(L_{yx}+L'_{xx} + 2(q-1)\Big(\frac{{L'_{xx}}^2}{\gamma b_x} + \frac{10L^2_{yx}}{\mu_y b_y}\Big)\right)^{-1},\;
    \sigma = \left(2L_{yy}+L_{yx} + 2(q-1)\Big(\frac{L^2_{xy}}{\gamma b_x} + \frac{10L^2_{yy}}{\mu_y b_y}\Big)\right)^{-1},\nonumber
    \\

    &
    N = 2(1+\zeta)\max\Big\{\frac{1}{\gamma\tau}-1,~\frac{1}{\mu_y\sigma}\Big\},\quad

    b \geq\Big\lceil \max\Big\{144\delta_x^2,~360\delta_y^2\frac{\gamma}{\mu_y}\Big\}\frac{1}{\epsilon^2}\Big\rceil,\label{eq: speical paramater VR innner loop}
\end{align}
}%
where $L'_{xx}=L_{xx}+2\gamma$. For $\epsilon>0$, and any $b_x,b_y,q\in \mathbb{N}^+$,
\sa{$x_0^T$ generated by $\texttt{SAPD+}$ with $\texttt{VR-flag}=\textbf{true}$} is $\epsilon$-stationary, i.e., $\mathbb{E}\left[\|\nabla\phi_{\lambda}(x_0^T)\|\right]\leq \epsilon$, for \sa{$T \geq 288\mathcal{G}(x_0^0,y_0^0)  \cdot\frac{\gamma}{\epsilon^2}$}, which requires $T(Nb/q+N(b_x+b_y))$
oracle calls in total, where
\vspace*{-2mm}
{\footnotesize
\begin{align}
\label{eq:N-bound-VR}
N = \mathcal{O}\Big(\max\Big\{
\frac{L_{yx}+L_{xx}}{\gamma} +\frac{q}{b_x}\frac{L_{xx}^2}{\gamma^2} + \frac{q}{b_y}\frac{L^2_{yx}}{\gamma\mu_y},\
\frac{L_{yy}+L_{yx}}{\mu_y}+
\frac{q}{b_y}\frac{L^2_{yy}}{\mu_y^2} + \frac{q}{b_x}\frac{L^2_{xy}}{\gamma\mu_y}
\Big\}
\Big).
\end{align}}%
\end{theorem}
\begin{remark}
For any $y\in\mathop{\bf dom} g$, since $\Phi(\cdot,y)$ $L_{xx}$-smooth, it is necessarily $L_{xx}$-weakly convex; hence, $\gamma\leq L_{xx}$. \sa{To get a worst-case complexity, consider the setting in \eqref{eq:uniform setting},}
and let
$b_x = b_y=\sa{b'}$. Then, \cref{cor:complexity VR} implies that setting
{\small$b = \mathcal{O}\Big(\kappa_y\tfrac{\delta^2}{\epsilon^2}\Big)$}, {\small$N = \mathcal{O}\big(\kappa_y+\kappa_y^2\tfrac{q}{\sa{b'}}\big)$}, and {\small$T = \mathcal{O}\Big(\tfrac{L\mathcal{G}(x_0^0,y_0^0)}{\epsilon^2}\Big)$} leads to
\small{$Nb/q+N\sa{b'}= \mathcal{O} \Big(\kappa_y \tfrac{b}{q} + \kappa_y^2\tfrac{b}{\sa{b'}} + \kappa_y \sa{b'} + \kappa_y^2 q \Big)$}. Thus, \sa{setting $q=\sqrt{\tfrac{ b}{\kappa_y}}$ and $\sa{b'} = \sqrt{b \kappa_y}$ leads to the oracle complexity of }
{\small
$
T(Nb/q+N\sa{b'})=\mathcal{O}\Big(\max\Big\{1,\tfrac{\delta}{\epsilon}\Big\}~\kappa^2_y\cdot\tfrac{L\mathcal{G}(x_0^0,y_0^0)}{\epsilon^2}\Big).
$}%

\end{remark}

\vspace*{-3mm}
\section{Weakly convex-\sa{merely concave~(WCMC)} problems}
\label{sec:wcmc}
\vspace*{-3mm}
In this section, \sa{we state the convergence guarantees of \texttt{SAPD+} for solving WCMC problems. In particular, we will consider \eqref{eq:main problem} such that $f(\cdot)=0$ and $\mu_y=0$, i.e., $\Phi(x,\cdot)$ is merely concave for all $x\in\mathcal{X}$. Instead of directly solving \eqref{eq:main problem} in WCMC setting,}
we will solve an approximate model obtained by smoothing the primal problem in a similar spirit to
the technique in \cite{nesterov2005smooth}.
More precisely, we approximate \eqref{eq:main problem} with the following
WCSC problem: \sa{given an arbitrary $\hat y\in\mathop{\bf dom} g$, consider}
\vspace*{-2mm}
{\small
\begin{equation}\label{eq:weakly convex-merely concave problem approx}
    \min_{x\in \mathcal{X}} \max_{y\in \mathcal{Y}} \hat{\mathcal{L}}(x,y)\triangleq \hat{\Phi}(x,y) -g(y),\quad\mbox{where}\quad \sa{\hat{\Phi}(x,y) \triangleq \Phi(x,y)-\frac{\hat{\mu}_y}{2}\|y -\hat{y} \|^2.}
\end{equation}}%
\vspace*{-4mm}

\begin{theorem}
\label{Thm:WCMC}
Under Assumptions~\ref{ASPT: fg},~\ref{ASPT: lipshiz gradient},~\ref{ASPT: unbiased noise assumption}, consider \eqref{eq:main problem} such that $f(\cdot)\equiv 0$, $\mu_y=0$,
and $\mathcal{D}_{\mathcal{Y}} \triangleq\sup_{y_1,y_2\in\mathop{\bf dom} g}\|y_1-y_2\|\sa{<\infty}$.
When either \cref{assump:compact} or \cref{assump:bounded-subdifferential} holds, for any given $\epsilon>0$, \texttt{SAPD+} with $\texttt{VR-flag}=\textbf{false}$, applied to~\eqref{eq:weakly convex-merely concave problem approx} with $\hat{\mu}_y=\Theta(\epsilon^2/(L\mathcal{D}_y^2))$,
is guaranteed to generate $x_\epsilon\in\mathcal{X}$ such that $\mathbb{E}\left[\|\nabla\phi_{\lambda}(x_\epsilon)\|\right]\leq\epsilon$ for $\lambda=1/(2\gamma)$ within $\mathcal{O}(L^{3}\epsilon^{-6})$ stochastic first-order oracle calls.
\end{theorem}

\vspace*{-3.5mm}
\section{\mg{Numerical experiments}}
\vspace*{-3mm}
The experiments are conducted on \sa{a PC with 3.6 GHz Intel Core i7 CPU and NVIDIA
RTX2070 GPU.} We consider distributionally robust optimization and fair classification. \sa{In the rest, $n$ and $d$ represent the number of samples in the dataset and the dimension of each data point, respectively.}

\paragraph{Distributionally Robust Optimization~\sa{(DRO)}.}
\vspace*{-2.5mm}
First, we consider nonconvex-regularized variant of
\sa{DRO} problem \cite{abadi2016deep,namkoong2016stochastic,kohler2017sub,luo2020stochastic,zhang2021robust,yan2019stochastic} \mg{which arises in distributionally robust learning}. Let $\{\mathbf{a}_i,b_i\}^{n}_{i=1}$ be the dataset where $\mathbf{a}_i\in\mathbb{R}^d$ \mg{are the features} and $b_i\in\{-1,1\}$ are labels. The DRO problem is
\vspace*{-2mm}
{\small
\begin{equation}\label{eq: experiment2 problem}
    \text{(DRO): } \min_{x\in \mathcal{R}^d} \max_{y \in Y}\frac{1}{n}\sum_{i=1}^n y_i \ell_i(x) + f(x) - g(y),\vspace*{-1mm}
\end{equation}}%
where $\ell_i(x)=\log(1+\exp{-b_i\mathbf{a}^\top_i\mathbf{x}})$, $f(x) = \sa{\eta_1}\sum_{i=1}^d \frac{\alpha x_i^2}{1+\alpha x_i^2}$ is a nonconvex regularizer~\cite{antoniadis2011penalized},
$g(y)=\frac{1}{2}\sa{\eta_2}\|n {y}-\mathbf{1}\|^2$, and $Y \triangleq \{{y}\in \mathbb{R}^d_+:~\mathbf{1}^\top {y} = 1 \}$ -- here, $\mathbf{1}$ denotes the vector with all entries equal to one. \mg{This problem can be viewed as a robust formulation of empirical risk minimization where the weights $y_i$ are allowed to deviate from $1/n$; and the aim is to minimize the worst-case empirical risk.}
We perform experiments on three data sets: $i)$ \verb+a9a+ with $n = 32561$; $d = 123$; $ii)$ \verb+gisette+ with $n = 6000$; $d = 5000$; $iii)$ \verb+sido0+ with $n = 12678$; $d = 4932$.  The dataset \verb+sido0+ is obtained from
Causality Workbench\footnote{http://www.causality.inf.ethz.ch/challenge.php?page=datasets} while the others can be downloaded from LIBSVM repository\footnote{https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary.html}.

\emph{Parameter tuning.} We set the parameters \mg{according to} \cite{yan2019stochastic, luo2020stochastic,kohler2017sub}, i.e., , \sa{$\alpha=10$, $\eta_1=10^{-3}$, $\eta_2=1/n^2$}.
\mg{We compare SAPD, SAPD-VR against PASGDA~\cite{boct2020alternating}, SREDA~\cite{luo2020stochastic}, SMDA, SMDA-VR~\cite{huang2021efficient} algorithms.} As suggested in \cite{luo2020stochastic}, we tune the primal stepsizes of all the algorithms \mg{based on a grid-search over the set} $\{10^{-3},10^{-2},10^{-1}\}$ and \mg{the ratio of the primal stepsize versus dual stepsize is varied to take values in the set $\{10,10^{2},10^{3},10^{4}\}$}. For all variance reduction-based algorithms, i.e., for SAPD-VR, SREDA, SMDA-VR, we tune the large batch size $\sa{b}\triangleq|\mathcal{B}|$ from the set $\{3000,6000\}$, and the small batch size $\sa{b'\triangleq}|I|$ from grid search over the set $\{10,100,200\}$. For the frequency parameter $q$, we let $q=b'=|I|$ \sa{for SAPD-VR and SMDA-VR (as suggested in \cite{huang2021efficient}); for SREDA, when we set $q$ and $m$ (SREDA's inner loop iteration number) to $\mathcal{O}(n/|I|)$ as suggested in~\cite{luo2020stochastic}, we noticed that SREDA does not perform well against SAPD-VR and SMDA-VR. Therefore,} to optimize the performance of SREDA further, we
tune $q,m$ from a grid search over $\{10,100,200\}$. For methods without variance reduction, i.e., for SAPD, SMDA and PASGDA, we tune the batch size from $\{10,100,200\}$ as well. For SAPD and SAPD-VR, we tune the momentum $\theta$ from $\{0.8,0.85,0.9\}$ and let the inner iteration numbers $N = \tfrac{\ln(265)}{\ln(\frac{1}{\theta})}$ according to \cref{Condition: SP solution to noisy LMI label 2}.

\emph{Results.} In \cref{fig: DRO}, we plot the \mg{average} loss against the epoch number \mg{based on 30 simulations (runs)}. \mg{The standard deviations of the runs are also illustrated around
\sa{the average} in lighter color \sa{as shaded regions}.}
We observe that SAPD and SAPD-VR consistently outperforms over other algorithms. For \verb+a9a+, \verb+gisette+, \verb+sido0+ datasets, the average training accuracy of SAPD are $84.06\%$, $95.41\%$, $96.43\%$, and of SAPD-VR are $84.33\%$, $97.69\%$, $97.46\%$, respectively. The best performance for \verb+a9a+, \verb+gisette+, \verb+sido0+ among all the other algorithms are $75.92\%$, $93.07\%$, $96.43\%$, respectively. More importantly, \mg{we observe that} \sa{as an accelerated method, SAPD-VR enjoys fast convergence properties while still being robust to gradient noise.}

\begin{figure}[t]
\centering
\caption{
Comparison of SAPD, SAPD-VR against PASGDA~\cite{boct2020alternating}, SREDA~\cite{luo2020stochastic}, SMDA, SMDA-VR~\cite{huang2021efficient} on real-data for solving \cref{eq: experiment2 problem} with $30$ times simulation.}
\label{fig: DRO}
\includegraphics[width = 0.32\textwidth]{a9a.png}
\includegraphics[width = 0.32\textwidth]{gisette.png}
\includegraphics[width = 0.32\textwidth]{sido0.png}
\centering
\vspace*{-3mm}
\end{figure}

\vspace*{-3mm}
\paragraph{Fair Classification.}
\mg{In the context of multi-class classification, Mohri \emph{et al.}~\cite{mohri2019agnostic} propose training a fair classifier
\sa{thorough minimizing} the worst-case loss over the classification categories.}
In the spirit of \cite{nouiehed2019solving,huang2021efficient}, we adopt a nonconvex \mg{convolutional neural network} (CNN) model as a classifier and set the number of categories to $3$, \sa{resulting in a minimax problem of the form:}
\vspace*{-2mm}
{\small
\begin{equation}
    \label{eq: experiment CNN}
    \min_{x\in\mathcal{X}} \max_{y\in\mathcal{Y}}\sum_{i=1}^{3}y_i \sa{\ell_i(x)} -g(y),
\end{equation}}%
where
\sa{$x\in\mathbb{R}^p$} represents the parameters of the CNN, and \mg{\sa{$\ell_1,\ell_2,\ell_3$} correspond to the loss of three categories, $g(y)=\frac{\sa{\eta}}{2}\|y\|_2^2$ is a regularizer with $\sa{\eta}>0$}. \sa{We
train \eqref{eq: experiment CNN} on the datasets to classify: $i)$ gray-scale hand-written digits $\{0,2,3\}$ from MNIST;
$ii)$ fashion images
with target classes \{T-shirt/top, Sandal, Ankle boot\} from F-MNIST;
$iii)$ RBG colored images with target classes \{Plane, Truck, Deer\} from CIFAR10. For both MNIST and F-MNIST $p=43831$, $n=18000$ and $d=28\times28\times 1$, and for CIFAR10 $p=61411$, $n=15000$, and  $d=32\times 32\times 3$.}

We let the regularization parameter $\sa{\eta}=0.1$
as suggested in \cite{huang2021efficient}. \mg{We compare SAPD-VR \sa{against the other} VR-based algorithms SREDA and SMDA-VR over 30 runs.
We tune the primal stepsizes of SAPD-VR and SREDA \mg{by a grid search over the set} $\{10^{-2},5\times 10^{-3},10^{-3}\}$ and the ratio of primal \sa{to dual stepsizes, i.e., $\tau/\sigma$, is chosen from}
$\{10,10^2,5\times 10^2,10^3\}$. For SMDA-VR, the primal and dual stepsizes are $10^{-3}$ and $10^{-5}$ as suggested in \cite{huang2021efficient} \sa{--we also tried stepsizes bigger than the suggested; but, it caused convergence issues in the experiments.}
We set the large batchsize $|\mathcal{B}|=3000$ and the small batchsize $|\mathcal{I}|=200$ for all algorithms and data sets; the frequency $q=200$ is used for SAPD-VR and SMDA-VR, and \sa{we tune $q$ for SREDA taking values from $\{10,50,100,200\}$.} The momentum $\theta$ for SAPD-VR is tuned taking values from  $\{0.8,0.85,0.9\}$ and inner iteration number $N = \frac{\ln(265)}{\ln(\frac{1}{\theta})}$ is set according to \cref{Condition: SP solution to noisy LMI label 2}.
For SREDA, we tune the inner loop iteration from $\{10,50,100\}$. \mg{Fig. \ref{fig: CNN WCSC} shows that SAPD-VR outperforms the other VR-based algorithms clearly in terms of both the average loss and the standard deviation of the loss.}

\begin{figure}[t]
\centering
\caption{
Comparison of SAPD-VR against other Variance Reduction algorithms, SREDA~\cite{luo2020stochastic}, SMDA-VR~\cite{huang2021efficient} on real-data for solving \cref{eq: experiment CNN} with $30$ times simulation.}
\label{fig: CNN WCSC}
\includegraphics[width = 0.32\textwidth]{MNIST.png}
\includegraphics[width = 0.32\textwidth]{F-MNIST.png}
\includegraphics[width = 0.32\textwidth]{CIFAR10.png}
\centering
\vspace*{-3mm}
\end{figure}
\vspace*{-3mm}
\section{Conclusion}
\vspace*{-3mm}
\mg{In this paper, we \sa{considered both WCSC and WCMC saddle-point problems assuming we only have an access to an unbiased stochastic first-oracle with a finite variance. This setting arises} in many applications \sa{ranging} from distributionally robust learning to GANs. We proposed a new method \texttt{SAPD+}, which achieves an improved complexity in terms of target accuracy $\epsilon$ for both WCSC and WCMC problems; \sa{moreover, our bound for \texttt{SAPD+} has a better dependency to the condition number $\kappa_y$ for the WCSC scenario.} \sa{We also showed that} our algorithm \texttt{SAPD+} can also support the
\sa{SPIDER} variance-reduction technique. \sa{Finally,} we provided numerical experiments
\sa{demonstrating that \texttt{SAPD+} can achieve a} state-of-the-art performance on distributionally robust learning and on multi-class classification \sa{problems arising in ML}.\looseness=-1}

\clearpage

redpajama_set_name: RedPajamaArXiv