text:
\section{Introduction}\label{sec:intro}
\input{sections/intro.tex}

\section{Background}\label{sec:background}
\input{sections/background.tex}

\section{Policy-based versus Value-based Updates}\label{sec:connect}
\input{sections/connect.tex}

\section{Update Variations: The Scale-Axis}\label{sec:scale_axis}
\input{sections/scale.tex}

\section{A Parametrized Class of Updates }\label{sec:unified}
\input{sections/unified.tex}

\section{Empirical Evaluation}\label{sec:eval}
\input{sections/experiments.tex}

\section{Related Work}\label{sec:related}
\input{sections/related.tex}

\section{Conclusion}\label{sec:conclusion}
\input{sections/conclusion.tex}

\section{Acknowledgements}\label{sec:ack}
We would like to thank Minmin Chen, Yingjie Miao, Yundi Qian and anonymous reviewers for constructive feedback.

\typeout{}

\subsection{Policy Gradient with Policy Baseline (PGPB)}\label{sec:pgpb}
In
(\ref{eqn:gpg}), $\hat{G}_{PG}$ can often be impractical for non-trivial problems due to its high variance, which motivates baselines, which are generally parameterized functions that can depend on both the state and the action.
The parameters that define baselines are typically independent of the policy parameters, however.
We now consider a particular form of state-action baseline
that uses the policy logits as the baseline under the softmax parameterization of
(\ref{eqn:softmax}).
The primary motivation for introducing this
is to clarify the relationship between policy-based and value-based gradients.
We begin with a result that characterizes this form of the policy gradient estimator,
whose relation to value-based gradients would be evident later in Section \ref{sec:pg_av_connect} by contrasting the terms in their respective gradients.
All proofs are provided in the Appendix.
\begin{restatable}{theorem}{pgpb}[Policy Gradient with Policy Baseline]\label{thm:pgpb}
Given $q_\theta(s, a): \mathcal{S} \times \mathcal{A} \mapsto \mathbb{R}$, let $\pi_\theta(a|s)$ be defined by
(\ref{eqn:softmax}). Consider the  sample based gradient estimator for
$\theta$:
\begin{align}\label{eqn:pgpb}
	& \hat{G}_{PGPB}(s, a, \theta) \triangleq  & \\
		 & \Big(\hat{T}(s, a) - q_\theta(s, a)\Big)\nabla_\theta \log \pi_\theta(a|s)	- \nabla_\theta H(\pi_\theta(.|s)) & \nonumber
\end{align}
where $H(\pi_\theta(.|s)) = \expectation_{a \sim \pi_\theta(.|s)} \left[- \log \pi_\theta(a|s) \right]$ denotes the entropy of
$\pi_\theta(.|s)$. Then, $\hat{G}_{PGPB}$ is an unbiased estimate of the policy gradient if $\hat{T}$ is an unbiased estimate of return:
\begin{equation}\label{eqn:exp_pgpb}
	\nabla_\theta J^\pi_\mu(\theta) = \expectation_{(s, a) \sim D^\pi_\mu} \left[ \hat{G}_{PGPB}(s, a, \theta) \right]
\end{equation}
\end{restatable}
\begin{remark}[\textbf{Entropy Term}]
Note that unlike MaxEnt-RL~\citep{haarnoja2017reinforcement}, the objective $J^\pi_\mu(\theta)$ does not include  entropy regularization in its formulation, yet an entropy term intriguingly shows up  in
(\ref{eqn:pgpb}).
\end{remark}
\subsection{Connecting Policy-based and Value-based Updates}\label{sec:pg_av_connect}
Consider two gradient estimators
that can be viewed as fitting an action-value function by minimizing an objective w.r.t.\ a target estimator $\hat{T}(s, a)$,
where
the first objective is the common Mean Squared Error (MSE), while the second is the less common Mean Variance Error (MVE) \citep{flet21}.
\begin{restatable}{theorem}{msemve}\label{thm:msemve}
Consider the value-based gradients following the MSE and MVE objectives respectively defined as:
\begin{align}
	G_{MSE}(\theta) &= - \nabla_\theta \frac{1}{2} \expectation_{(s, a)} \left[ ( \hat{T}(s, a) -  q_\theta(s, a) )^2 \right] \label{eqn:jmse} \\
	G_{MVE}(\theta) &= - \nabla_\theta \frac{1}{2} \expectation_{s} \variance_{a}\Big[\hat{T}(s, a) - q_\theta(s, a) \Big], \label{eqn:jmve}
\end{align}
where,
$\variance_{a}$ denotes the variance of the distribution over $a$, conditioned on the state $s$.
Then, $\hat{G}_{MSE}$ and $\hat{G}_{MVE}$  below are unbiased gradient\footnote{These are technically semi-gradients because any dependence of $\hat{D}^\pi_\mu$, the assumed on-policy sampling distribution for $(s, a)$ and $\hat{T}(s, a)$ on $\theta$ are ignored, similar to most TD learning methods.}
estimators for $G_{MSE}$ and $G_{MVE}$.
\begin{align}
	\hat{G}_{MSE}(s, a, \theta) &\triangleq  \left( \hat{T}(s, a) - q_\theta(s, a)\right) \nabla_\theta q_\theta(s, a)  \label{eqn:gmse} \\
	\hat{G}_{MVE}(s, a, \theta) &\triangleq  \Big(\hat{T}(s, a) - q_\theta(s, a)\Big) \nabla_\theta \log \pi_\theta(a|s) \label{eqn:gmve}
\end{align}
\end{restatable}
Note that value-based methods typically do not specify the replay buffer sampling distribution, in contrast
to PG, which comes with a specific on-policy sampling distribution, $D^\pi_\mu$ defined in (\ref{eqn:dpimu}). However, this distinction is not relevant when
considering the updates conditioned on a given $(s, a)$ unless off-policy corrections are considered.
We are now ready to relate the policy gradient $\hat{G}_{PGPB}$ with $\hat{G}_{MSE}$ and $\hat{G}_{MVE}$.
\begin{corollary}\label{cor:rel}
Given any fixed target return estimator, $\hat{T}(s, a)$,
the following relations hold between the sample-based gradient estimators at every sample $(s, a)$ and $\theta$.
\begin{align}
	\hat{G}_{MVE} - \hat{G}_{PGPB} &= \nabla_\theta H(\pi_\theta(.|s)) \label{eqn:rel1} \\
	\hat{G}_{MSE} - \hat{G}_{MVE} &\propto \sum_u \pi_\theta(u|s) \nabla_\theta q_\theta(s, u) \label{eqn:rel2} \\
	\hat{G}_{MSE} - \hat{G}_{PGPB} -& \nabla_\theta H(\pi_\theta(.|s)) \label{eqn:rel3} \\
					&\propto \sum_u \pi_\theta(u|s) \nabla_\theta q_\theta(s, u) \nonumber
\end{align}
\end{corollary}
\begin{proof}
Equation (\ref{eqn:rel1}) follows directly from combining
(\ref{eqn:pgpb}) with
(\ref{eqn:gmve}).
Equation (\ref{eqn:rel2}) follows from relating $\nabla_\theta \log \pi_\theta(a|s)$ with $\nabla_\theta q_\theta(s, a)$ using Equation (\ref{eqn:softmax}).
Finally, Equation
(\ref{eqn:rel3}) follows from the previous two observations.
\end{proof}
The significance of Corollary \ref{cor:rel} is that Equations
(\ref{eqn:rel1}), (\ref{eqn:rel2}), (\ref{eqn:rel3}) express the relationship between the three gradient forms in terms of quantities that have \emph{no dependence on the action  $a$ or the return estimate $\hat{T}(s, a)$}. The main  signal indicating quality of the chosen action $a$, quantified by $\hat{T}(s, a)$, is aggregated identically for all three updates via:
\begin{equation} \label{eqn:delta_r}
	\Delta_R \triangleq \hat{T}(s, a) - q_\theta(s, a)
\end{equation}
To summarize the three forms of updates above more explicitly in terms of the underlying parametric model $q_\theta(s, a)$, we now state a helper theorem:
\begin{restatable}{theorem}{entropy}\label{lemma:entropy}
	Let $\hat{q}_\theta$ indicate a stop-gradient operator on $q_\theta$ with respect to $\theta$. Given the softmax parameterization of $\pi_\theta$ (Equation (\ref{eqn:softmax})),
	we have:
\begin{equation}\label{eqn:entropy_lemma}
 \nabla_\theta H(\pi_\theta(.|s) +  \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] = 0
\end{equation}
\end{restatable}
Theorem \ref{lemma:entropy}, Equation
(\ref{eqn:softmax}) and Equation (\ref{eqn:delta_r}), together allow for a summary of the three updates directly in terms of $q_\theta(s, a)$ as:
\begin{align}
	\hat{G}_{PGPB} =& ~\Delta_R \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big) \nonumber \\
		       &~+ \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] \label{eqn:pgpb2} \\
	\hat{G}_{MVE} =&~ \Delta_R \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big) \label{eqn:mve2} \\
	\hat{G}_{MSE}  =&~ \Delta_R \Big( \nabla_\theta q_\theta(s, a) \Big) \label{eqn:mse2}
\end{align}
We refer to the gradients listed in Equations
(\ref{eqn:pgpb2}), (\ref{eqn:mve2}), (\ref{eqn:mse2}) as update variations along the \emph{form-axis} since the key changes required to span these updates have to do with what are effectively the \emph{bias} vectors, $\expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u)\right]$ and $\nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]$, that can be computed exclusively from the model $q_\theta$ without any query to the data sample, and in particular, $\Delta_R$.

\iffalse
\textbf{Target Estimate Variations:} Another axis of variation in practice also includes how $\Delta_R$,
or more precisely, $\hat{T}(s, a)$, is estimated.
In addition to Monte-Carlo estimates of the returns, this can include the TD bootstrap estimates, or TD($\lambda$), GAE($\lambda$)-style estimates which can smoothly interpolate between multi-step bootstrap and Monte-Carlo targets.
However, such variations on the target
are orthogonal to
our analysis, which focuses on how the updates differ with respect to their dependence on $\Delta_R$ once $\hat{T}(s, a)$ has been fixed.
\fi

\subsection{Error Loss Function Updates}\label{sec:error_loss}
Consider value fitting objectives arising from aggregating individual state action prediction errors via an \emph{error loss function}, $l$ which is assumed to be smooth and convex, $l: \mathbb{R} \mapsto \mathbb{R}_+$ such that $l(0) = 0$. Under these assumptions, the derivative of $l$ is given by a non-decreasing function $\lambda : \mathbb{R} \mapsto \mathbb{R}$ with $\lambda(0) = 0$.
Given any such loss criterion $l$ on the prediction error $\Delta_R$, let $G_{E(l)}(\theta)$ denote a generalization of $G_{MSE}(\theta)$ in Equation (\ref{eqn:jmse}) that generalizes to the error loss $l(.)$ from the MSE:
\begin{equation}\label{eqn:jef}
	G_{E(l)}(\theta) = - \nabla_\theta \expectation_{(s, a)} \left[l \Big ( \hat{T}(s, a) -  q_\theta(s, a) \Big ) \right]
\end{equation}
The sample gradient estimator, $\hat{G}_\lambda$ for the scaling function $\lambda$ (or equivalently, its loss $l$) is:
\begin{align}
	\hat{G}_\lambda(s, a, \theta) &= - \nabla_\theta ~l\Big(\hat{T}(s, a) - q_\theta(s, a) \Big ) \\
				&= \lambda(\Delta_R) \nabla_\theta q_\theta(s, a)
\end{align}
Unlike the \emph{form-axis} variations which involve the previously discussed sample independent \emph{bias} vectors, we refer to this type of variation as the \emph{scale-axis}.
The MSE update is equivalent to $l(x) = \frac{1}{2} x^2$ and $\lambda(x) = x$.
An alternative, the Huber loss \citep{huber64}, has an $l(x)$ parameterized by a hyperparameter $\delta > 0$ as:
\begin{equation} \nonumber
	l(x) =
	\begin{cases}
		\frac{1}{2} x^2, & \text{if } |x| < \delta \\
		\delta(|x| - \frac{1}{2} \delta), & \text{ otherwise}
	\end{cases}
\end{equation}
In this case, $\lambda(x) = \text{clip} (x, -\delta, \delta)$ is a clipped version of the identity scaling function that arises from the MSE loss.
Clipping the TD-errors is a common heuristic in deep RL  \citep{mnih2015human}, and corresponds to a simple scale variation as shown above.

\subsection{A Maximum-Likelihood Update}\label{sec:mlu}
Next, we derive a novel scale-axis update that is inspired by a maximum likelihood objective.
Given a target estimator, $\hat{T}$,
consider an action target distribution
$\hat{p}_{T}(a|s) \triangleq \exp{\Big(\hat{T}(s, a) - F(\hat{T})(s)\Big)}$, where $F(T)(s) \triangleq \log{\sum_{a} \exp{T(s, a)}}$.
The gradient of the log-likelihood for action conditionals given this target distribution is:
\begin{align}
	& G_{ML}(\theta) \triangleq \expectation_{s} \left[ \expectation_{a|s \sim \hat{p}_T} \nabla_\theta \log{\pi_\theta(a|s)} \right] \nonumber \\
	=& \expectation_{s} \left[ \expectation_{a|s \sim \hat{p}_T} \Big[ \nabla_\theta q_\theta(s, a) \Big] - \sum_u \pi_\theta(u|s) \nabla_\theta q_\theta(s, u) \right] \nonumber \\
	       = &\expectation_{s} \left[ \expectation_{a|s \sim \pi} \left[ \frac{\hat{p}_T(a|s)}{\pi_\theta(a|s)} \nabla_\theta q_\theta(s, a) \right] - \expectation_{u|s \sim \pi} \Big[ \nabla_\theta q_\theta(s, u) \Big] \right] \nonumber \\
	       = &\expectation_{(s, a) \sim \pi}  \left[ \left(\frac{\hat{p}_T(a|s)}{\pi_\theta(a|s)} - 1\right) \nabla_\theta q_\theta(s, a) \right] \label{eqn:deriv_ml}
\end{align}
Note that the log-likelihood ratio between $\hat{p}_T$ and $\pi_\theta$ is:
\begin{align*}
	& \log \hat{p}_T(a|s) - \log \pi_\theta(a|s) \\
	& = \hat{T}(s, a) - F(\hat{T})(s) - q_\theta(s, a) + F(q_\theta)(s) \\
	& = \Delta_R + \xi(s), \text { where } \xi(s) = F(q_\theta)(s) - F(\hat{T})(s)
	,
\end{align*}
which gives an unbiased sample estimator for
(\ref{eqn:deriv_ml}) in the form
$(e^{\Delta_R + \xi(s)} - 1) \nabla_\theta q_\theta(s, a)$. However, computing $F(\hat{T})(s)$
explicitly requires defining $\hat{T}(s,a)$ for all possible actions and not just the chosen action.
To avoid this issue, we make an approximation that $\xi(s) \approxeq 0$, which may also be justified as an
implicit constraint on the estimated target values.
This motivates an alternate update:
\begin{equation}\label{eqn:ml}
\hat{G}_{ML}(s, a, \theta) \triangleq \Big(e^{\Delta_R}- 1 \Big) \nabla_\theta q_\theta(s, a)
\end{equation}
This update can be recovered by $\lambda(x) = e^x - 1$, which also satisfies $\lambda(0) = 0$ and is increasing.

In \citet{sil}, a surrogate objective derived from modifying the MSE loss for Q-learning by clipping the prediction error below at 0 is proposed, which is referred to as self imitation learning (SIL).
We can write the gradient expression corresponding to this objective as
\begin{equation}\label{eqn:silo}
	\hat{G}_{SIL}(s, a, \theta) =  \max(\Delta_R, 0) \nabla_\theta q_\theta(s, a)
\end{equation}
which reveals that it could be considered a piece-wise linear approximation to the Maximum Likelihood gradient $\hat{G}_{ML}$ derived above since $\lambda(x) = e^x - 1 \approxeq \max (x, 0) $.

\subsection{Off-Policy Corrected Updates}\label{sec:off_policy}
In Section \ref{sec:connect}, we made an implicit assumption that the sampled actions for each update are chosen from the
policy encoded by the current parameters, which is
an on-policy assumption that is typically not satisfied in practice.
Accounting for a different behavior policy requires
re-weighting the updates with off-policy importance ratio for each sample, which is an action dependent scalar signals, similar to $\Delta_R$.
Let $\pi_b$ denote the behavior policy. Denote the log probability ratio at a given $(s,a)$ sample as:
\begin{equation}\label{eqn:delta_o}
	\Delta_O \triangleq  \log \frac{\pi_\theta(a|s)}{\pi_b(a|s)}
\end{equation}
For practical reasons, we only consider importance correction of the action conditional
distribution
at each state and implicitly assume that the state marginals match, an assumption which is also common in prior practical methods which make use of off-policy corrections \citep{chen19, ppo}.
Note that the two scalar quantities $\Delta_O$ and $\Delta_R$, defined in Equations
(\ref{eqn:delta_o}) and (\ref{eqn:delta_r})  summarize all the action dependent \emph{learning signals} needed for gradient estimation on any given $(s,a)$ sample.
Accounting for importance weight correction, the updates in Equations
(\ref{eqn:pgpb2}), (\ref{eqn:mve2}), (\ref{eqn:mse2}), (\ref{eqn:ml}) can be generalized to:
\begin{align}
	\hat{G}_{PGPB} =& ~e^{\Delta_O} \Delta_R \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)
	\nonumber

	\\
		       &~+ \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] \label{eqn:pgpb3} \\
	\hat{G}_{MVE} =&~ e^{\Delta_O}\Delta_R \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big) \label{eqn:mve3} \\

	\hat{G}_{\lambda}      =&~ e^{\Delta_O} \lambda(\Delta_R)\Big( \nabla_\theta q_\theta(s, a) \Big) \label{eqn:ml3}
\end{align}
\begin{remark}
Equation (\ref{eqn:ml3}) spans an entire class of updates by choosing $\lambda: \mathbb{R} \mapsto \mathbb{R}$
to be any particular non-decreasing function which vanishes at $0$. Setting $\Delta_O = 0$ recovers the on-policy special case that
ignores the importance weights.
\end{remark}
\begin{remark}
For
(\ref{eqn:pgpb3})
the  term $\nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]$ does not include an off-policy correction because it can be computed explicitly without estimating the return for the chosen action sample.
This is in contrast to the  terms involving $\Delta_R$, which are specific to the sampled action and need  importance correction whenever the actions are chosen by a divergent behavior policy.
\end{remark}
With off-policy corrections, the scale-axis variation can now include a joint dependence of the gradient estimates on $\Delta_O$ and $\Delta_R$ in a single expression, $f(\Delta_O, \Delta_R)$.
Note that the form-axis variations considered in Section \ref{sec:connect} can  be systematically generalized by replacing $\Delta_R$ with
 $f(\Delta_O, \Delta_R) = e^{\Delta_O} \Delta_R$.
So far, we have only demonstrated cases of $f(\Delta_O, \Delta_R)$ that are multiplicatively factored into terms that depend only on $\Delta_O$ and $\Delta_R$.
However, this need not be a universal assumption, as shown next for  PPO.

\subsection{The Proximal Policy Optimization (PPO) update}\label{sec:ppo}
PPO \citep{ppo} optimizes a surrogate objective derived from policy gradient to learn a policy parameterization directly without an explicit state-action value function.
To relate to our framework, we need to reconstruct an explicit state-action preference $q_\theta$ from $\pi_\theta$. While this can be done using the softmax assumption in the discrete action case, the more general continuous action setting is not straightforward,
as $\pi_\theta$ may only support sampling the conditional actions and estimating the conditional probability for a given state-action pair (e.g. from a closed form Gaussian).
Nevertheless, if we have an estimate for $V^\pi(s)$, the state value function, then \citet{pgqcombine} suggests that we can consider an implied estimate of the state-action values for some constant $\alpha$ of the form $\hat{q}(s, a) \approxeq \alpha (\log \pi_\theta(a|s) + H(\pi(.|s)) + \hat{V}^\pi(s)$.
Using the notation for the advantage function, $\hat{A} = \hat{T} - V^\pi(s)$, we define:
\begin{equation} \label{eqn:qhat_dest}
	\Delta_R = \hat{T} - \hat{q} = \hat{A} - \alpha \big(\log \pi(a|s) + H(\pi(.|s))\big)
\end{equation}
Given any advantage estimator (e.g., GAE($\lambda$)~\citep{schulman2016high}), this extends our generalized update rule starting from PGPB using the value notation to a policy parameterized update rule that also supports continuous actions as:
\begin{equation}\label{update:cts}
	\hat{G}_{f, \pi} \triangleq f(\Delta_O, \Delta_R) \nabla_\theta \log \pi_\theta(a|s) + \beta \nabla_\theta H(\pi_\theta(.|s))
\end{equation}
In the above equation, $\beta$ is an additional entropy  hyperparameter similar to the final term in $\hat{G}_{PGPB}$ for the special case of discrete actions from Section \ref{sec:connect}.

\begin{restatable}{theorem}{ppo}\label{thm:ppo}
The PPO surrogate objective gradient can be recovered as $\hat{G}_{f, \pi}$ in
(\ref{update:cts}) for the gradient scaling function, $f(x, y) = e^x y ~\tau_\epsilon(x, y)$, where $\tau_\epsilon(x, y) = \mathbbm{1}_{y > 0} \mathbbm{1}_{x < \log(1 + \epsilon)} + \mathbbm{1}_{y <0} \mathbbm{1}_{x > \log(1 - \epsilon)}$ while choosing $\alpha = 0$ in
(\ref{eqn:qhat_dest}).
	Note that $\tau_\epsilon(x, y) \in \{0, 1\} ~ \forall x, y, \epsilon$.
\end{restatable}

\subsection{$f_{MLA}$: A numerically stable approximation}

\subsection{MLA-PPO updates}

\subsection{A Parametric Class of Scaling Functions}
\label{subsec:parametric_scale}
We now leverage the unified view including the constraints of Assumption \ref{assumption:constraints} to  identify a numerically stable approximation, $f_{MLA}$ to the maximum likelihood scaling function, which was derived as $f_{ML}(\Delta_O, \Delta_R) = e^{\Delta_O} (e^{\Delta_R} - 1)$, but suffers from an exponential dependence on both $\Delta_O$ and $\Delta_R$.
This new approximation, derived in the appendix, is:
\begin{align}\label{eqn:ml1}
f_{MLA}(x, y) & = \\
	& \begin{cases}
		-\frac{1}{2}(1 + x)^2, \text{ if } y \le -(1 + x) \le 0  \\
		y \max \left(1 + x + \frac{y}{2}, 0\right)  \text{ otherwise}
	\end{cases} \nonumber
\end{align}
\begin{remark}[Qualitative similarity of $f_{ML}$ to $f_{MLA}$]
When highly off-policy ($\Delta_O \ll 0$), $f_{MLA}(\Delta_O, \Delta_R)\! =\! 0$ for all values of $\Delta_R$ up to some positive threshold that depends on $\Delta_O$. This threshold increases as the sample becomes more off-policy (i.e.\  $\Delta_O \!\rightarrow\! -\infty$). For large values of $\Delta_R$, the update strength is quadratic in $\Delta_R$.  Conversely, it saturates to a negative value as $\Delta_R\! \rightarrow\! -\infty$.
\end{remark}
We now introduce a  class of  scaling functions that is derived by further approximations to $f_{MLA}$.
This class spans a range of behaviors, all satisfying the constraints of Assumption \ref{assumption:constraints}.
\begin{align}\label{eqn:ml2}
	f_{MLA(\alpha_o, \alpha_r)}(x, y) &=  \\
	     & y \max \left( 1 + \alpha_o x + \alpha_r y , \frac{(1 + \alpha_o x)_+}{2}\right) \nonumber
\end{align}
In
Equation (\ref{eqn:ml2}), the constants
$\alpha_o, \alpha_r$ denote hyper-parameters\footnote{$(x)_+$ denotes $\max(x, 0)$} that can be used to span a class of update rules corresponding to a two-dimensional search space of scaling functions.
\begin{observation}\label{obs:cases}[Recovery of Specific Scale Functions]
	With particular values of $\alpha_o, \alpha_r$, $f_{MLA(\alpha_o, \alpha_r)}$ includes\footnote{In a non-vanishing neighborhood of $(0, 0)$} the following important special cases of scaling functions that were explicitly considered during the unification.
	\begin{enumerate}
	    \item $\alpha_o = 1, \alpha_r=0.5$ approximately recovers $f_{MLA}$, and therefore $f_{ML}$.

		\item $\alpha_o = 1, \alpha_r=0$ approximately recovers the baseline scaling function for policy gradient with importance weight corrections.
		\item $\alpha_o = 0, \alpha_r=0$ recovers the baseline identity scaling function for action value methods without importance weight corrections (e.g. Q-learning).
	\end{enumerate}
\end{observation}
\begin{proof}[Justification of Observation \ref{obs:cases}]
		For claim 1, a visualization for the projections of $f_{ML}$ and $f_{ML(1, 0.5)}$ along various slices of $\Delta_O$ and $\Delta_R$ is shown in Figures ~\ref{fig:ml1} and \ref{fig:ml2} in the Appendix.\\
		To justify claim 2, note that $f_{MLA(1, 0)}(x, y) = y \max(1 + x, \frac{(1+x)_+}{2}) = y \max(1+x, 0) \approxeq y e^x$. \\
		Claim 3 follows trivially as $f_{MLA(0, 0)}(x, y) = y$.
\end{proof}

\section{Connection Between Various Objective Gradients}\label{sec:unification}
\subsection{Target Estimates}
All methods implicitly or explicitly have some form of estimating a target discounted return. This could be max at next state for DQN or the Monte-Carlo estimate for PG or Td-lambda version.

In this section, we derive sample estimates for the gradients of four different objectives grounded under the set of common assumptions from Section \ref{sec:setting}. For simplicity, we only consider the on-policy expressions in this Section, which will be extended to the offline scenario in Section \ref{sec:scaling}.
\subsection{Q-learning Gradient}\label{sec:q}
We first consider the squared error minimization objective for $Q-$learning, which fits a parameterized $q_\theta(s, a)$ to the target return, $\hat{T}(s, a)$ on a dataset $\mathcal{D}$.
Note that both $\mathcal{D}$ as well as the target sample, $\hat{T}(s, a)$ may implicitly even depend on the policy $\pi$, but these dependencies are
never considered for the gradient calculation while minimizing the squared prediction error in $Q-$learning.
Under the assumptions from Section \ref{sec:setting}, we can write the gradient of the negative MSE objective as:
\begin{align}
	G_{QL} &= - \nabla_\theta \frac{1}{2} \expectation_{(s, a) \sim \mathcal{D}} \left[ ( \hat{T}(s, a) -  q_\theta(s, a) )^2 \right] \\
	       &= \expectation_{(s, a) \sim \mathcal{D}} (\hat{T}(s, a) - q_\theta(s, a)) \nabla_\theta q_\theta(s, a) \\
	       &= \expectation_{(s, a) \sim D_\pi} \left[ (Q^\pi(s, a) - q_\theta(s, a)) \nabla_\theta q_\theta(s, a)\right] \text{ using Equation \ref{eqn:onpolicy} and assuming $\hat{T} \equiv Q^\pi$ }
\end{align}
Written as a single sample stochastic gradient estimate of the maximization objective, we have:
\begin{align}
	G_{QL} &= \expectation_{(s, a) \sim D_\pi} \hat{G}_{QL}(s, a) \\
	\mbox{ where, }~~~ \hat{G}_{QL}(s, a, \theta) &\triangleq \left(Q^\pi(s, a) - q_\theta(s, a) \right) \nabla_\theta q_\theta(s, a) \label{eqn:ql}
\end{align}

\subsection{Policy Gradient with Policy Baseline (PGPB)}\label{sec:pg}
From the standard policy gradient theorem \citep{pgoriginal}:
\begin{align}
	G_{PG}(\theta) &= \sum_s d^\pi_\mu(s) \sum_a Q^\pi(s, a) \nabla_\theta \pi_\theta(a | s) \\
		       &= \sum_s d^\pi_\mu(s) \sum_a \pi_\theta(a | s) Q^\pi(s, a) \nabla_\theta \log{\pi_\theta(a | s)} \\
		       &= \expectation_{(s, a) \sim D_\pi } \left[ Q^\pi(s, a) \nabla_\theta  \log{\pi_\theta(a | s)} \right] \label{eqn:exact_pg}
\end{align}
To derive an unbiased sample estimate for Equation \ref{eqn:exact_pg}, there are several versions corresponding to particular choices of control variates, also commonly referred to as baselines.
In order to relate to the $Q-$learning gradients, consider a state-action baseline equal to the policy logits themselves, i.e. $q_\theta(s, a)$. Note that Assumption \ref{assumption:softmax} implies the following identity:
\begin{equation}\label{eqn:softmax_grad}
\nabla_\theta \log{\pi_\theta(a|s)} = \nabla_\theta q_\theta(s, a) - \sum_u \pi_\theta(u|s) \nabla_\theta q_\theta(s, u)
\end{equation}
Using this, we can write\footnote{We suppress $\pi$'s dependence on $\theta$ when there are no gradients through that term.}:
\begin{align}
	&G_{PG}(\theta) = \expectation_{(s, a) \sim D_\pi} \left[ Q^\pi(s, a) \nabla_\theta \log{\pi_\theta(a|s)} \right] \\
		       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \pi} \left[ (Q^\pi(s, a) - q_\theta(s, a))\nabla_\theta \log{\pi_\theta(a|s)}  \right]
		       + \expectation_{u | s \sim \pi} \left[ q_\theta(s, u) \nabla_\theta \log{\pi_\theta(u|s)} \right] \right]\\
		       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a | s \sim \pi} \left[ (Q^\pi(s, a) - q_\theta(s, a))\nabla_\theta \log{\pi_\theta(a|s)}  \right]
		       + \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] \right]\\ \label{eqn:hat}
		       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \pi} \left[ \Big(Q^\pi(s, a) - q_\theta(s, a)\Big)\Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)
\right]  + \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] \right]
\end{align}
In the last term in Equation \ref{eqn:hat}, we use $\hat{q}_\theta$ rather than $q_\theta$ to denote a partial derivative that does not consider $\hat{q}_\theta$ for the gradient with respect to $\theta$ (i.e., a `stop gradient').
We can therefore define a sample gradient estimate for $G_{PG}(\theta) = \expectation_{(s, a) \sim D_\pi} \hat{G}_{PGPB}(s, a, \theta)$ as:
\begin{equation}\label{eqn:pgpb}
	\hat{G}_{PGPB}(s, a, \theta) \triangleq  \Big(Q^\pi(s, a) - q_\theta(s, a)\Big)\Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)
+ \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]
\end{equation}
The gradient estimate $\hat{G}_{PGPB}(s, a, \theta)$ is unbiased for the expected reward objective. Notice that there are two terms, $\expectation_{u|s \sim \pi}\left[ \nabla_\theta q_\theta(s, u) \right]$ and  $\nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]$
in Equation \ref{eqn:pgpb} which do not depend on the sampled action, $a$ or the reward estimate, $Q^\pi(s, a)$. Therefore,
both these terms can be computed easily (with zero variance in case of discrete actions, or low variance with the reparameterization trick in case of continuous actions for suitable parameterization; and without querying the environment or the dataset).
Comparing $\hat{G}_{PGPB}(s, a, \theta)$ with $\hat{G}_{QL}(s, a, \theta)$ from Equation \ref{eqn:ql}, we can interpret the $Q-$learning gradient as a biased version of policy gradient obtained by dropping the two aforementioned terms in Equation \ref{eqn:pgpb}.
\subsection{Variance Minimization Gradient}\label{sec:v}
By contrast, consider an alternate gradient estimate that results from dropping only one of the two additional terms in $\hat{G}_{PGPB}$ compared to $\hat{G}_Q$, $\nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]$. Denote this by $\hat{G}_{V}$:
\begin{equation}\label{eqn:var}
\hat{G}_{V}(s, a, \theta) \triangleq \Big(Q^\pi(s, a) - q_\theta(s, a)\Big)\Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)
\end{equation}
As shown below, this is equivalent to the gradient of an objective which minimizes the variance, instead of the expectation of the squared error across actions.
\begin{align}
	& \expectation \left[\hat{G}_{V}(s, a, \theta)\right] = \expectation_{(s, a) \sim D_\pi}\left[ \Big(Q^\pi(s, a) - q_\theta(s, a)\Big)\Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)
\right] \\
									 &= \expectation_{s \sim d^{\pi}_\mu} \left[ \expectation_{a|s \sim \pi} \Big[ \Big(Q^\pi(s, a) - q_\theta(s, a)\Big)\nabla_\theta q_\theta(s, a)\Big] - \expectation_{a|s \sim \pi} \Big[Q^\pi(s, a) - q_\theta(s, a) \Big] \expectation_{u|s \sim \pi} \Big[ \nabla_\theta q_\theta(s, u) \Big] \right] \\
									 &= -\frac{1}{2} \expectation_{s \sim d^{\pi}_\mu} \left[ \nabla_\theta \expectation_{a|s \sim \hat{\pi}} \left[ (Q^\pi(s, a) - q_\theta(s, a))^2 \right] - \nabla_\theta \left( \expectation_{a|s \sim \hat{\pi}} \left[Q^\pi(s, a) - q_\theta(s, a)\right]\right)^2 \right] \\
									 &= -\frac{1}{2} \expectation_{s \sim d^{\pi}_\mu} \nabla_\theta \variance_{a|s \sim \hat{\pi}}\Big[Q^\pi(s, a) - q_\theta(s, a) \Big] \mbox{ where } \variance \mbox{denotes variance.}
\end{align}
In the last two equalities above, we use $\hat{\pi}$ rather than $\pi$ to indicate that the gradient operator can pass through the expectations without considering the implicit dependency of $\pi$ on $\theta$.

\subsection{Imitation Learning}\label{sec:il}
Next, consider the log-likelihood objective with respect to a target distribution defined by $\hat{p}_T(a|s) \triangleq \exp{\Big(Q^\pi(s, a) - F(Q^\pi)(s)\Big)}$, where $F(q)(s) \triangleq \log{\sum_{a} \exp{q(s, a)}}$ is used to denote the LogSumExp reduction over actions for an arbitrary state-action value function $q(s, a)$. Below, we derive an expression for the gradient of a Maximum Likelihood objective, which highlights it's differences with respect to the squared error objective of $Q-$learning.
\begin{align}
	G_{IL} &\triangleq \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \hat{p}_T} \nabla_\theta \log{\pi_\theta(a|s)} \right] \\
	       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \hat{p}_T} \left[ \nabla_\theta q_\theta(s, a) - \sum_u \pi_\theta(u|s) \nabla_\theta q_\theta(s, u) \right] \right] \\
	       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \hat{p}_T} \Big[ \nabla_\theta q_\theta(s, a) \Big] - \sum_u \pi_\theta(u|s) \nabla_\theta q_\theta(s, u) \right] \\
	       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \pi} \left[ \frac{\hat{p}_T(a|s)}{\pi_\theta(a|s)} \nabla_\theta q_\theta(s, a) \right] - \expectation_{u|s \sim \pi} \Big[ \nabla_\theta q_\theta(s, u) \Big] \right] \\
	       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \pi} \left[ \left(\frac{\hat{p}_T(a|s)}{\pi_\theta(a|s)} - 1\right) \nabla_\theta q_\theta(s, a) \right] \right] \\

	       &= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \pi} \left[ \left(\exp{\Big(Q^\pi(s, a) - q_\theta(s, a) + F(q_\theta)(s) - F(Q^\pi)(s)\Big)} - 1\right) \nabla_\theta q_\theta(s, a) \right] \right]
\end{align}
Now consider a biased gradient estimate which drops the log-normalizer difference term, $F(Q^\pi)(s) - F(q_\theta)(s)$, as $\hat{G}_{IL}$.
\begin{equation}\label{eqn:ilo}
	\hat{G}_{IL}(s, a, \theta) \triangleq \Big(\exp\big(Q^\pi(s, a) - q_\theta(s, a)\big) - 1 \Big) \nabla_\theta q_\theta(s, a)
\end{equation}

In \citet{sil}, the authors propose a novel objective derived from modifying $Q-$learning by clipping the prediction error below at 0, which is referred to as self imitation learning.
We write the gradient expression corresponding to the self imitation learning objective as $\hat{G}_{SIL}$ next.
\begin{equation}\label{eqn:silo}
	\hat{G}_{SIL}(s, a, \theta) =  \max(Q^\pi(s, a) - q_\theta(s, a), 0) \nabla_\theta q_\theta(s, a)
\end{equation}
This suggests that an alternate way to interpret the SIL gradient is by considering $\max (x, 0)$ as an approximation to $e^x - 1$.

\section{Gradient Scaling Functions with Off-Policy Correction}\label{sec:scaling}
Until now, we made an assumption that the sampling distribution for states and actions matches the policy $\pi_\theta$. In this section, that assumption is relaxed. Let $\pi_b$ denote a behavior policy that is different from the current policy. Note that $\pi_b$ may not be explicitly available, but can be estimated using conditional maximum-likelihood as:
\begin{equation}
b = \arg \max_\theta \sum_{(s, a) \sim \mathcal{D} } \log \pi_\theta(a|s)
\end{equation}
Consider off-policy importance weights for action conditionals\footnote{We consider exploration issues outside the scope of
this work and ignore any mismatch in the state-marginals while computing the importance weights.} between the behavior policy and the current policy, $\frac{\pi_\theta(a|s)}{\pi_b(a|s)}$.
To simplify the expressions, denote the off-policy log density ratio and the prediction error as the following two sample dependent scalar `learning signals':
\begin{equation}\label{eq:DeltaO} \Delta_O \triangleq \log
\frac{\pi_\theta(a|s)}{\pi_b(a|s)} \end{equation}
\begin{equation}\label{eq:DeltaR} \Delta_R \triangleq Q^\pi(s, a) - q_\theta(s, a) \end{equation}
$\Delta_O$ is the logarithm of the off-policy importance weight ratio, and can also be viewed as
a single sample unbiased estimate of the KL divergence between the behavior and current policies. When the dataset
is on-policy, this implies $\Delta_O = 0$ for all samples, $(s, a)$.
By contrast, $\Delta_R = 0$ when the value prediction error is zero at the current sample, $(s, a)$.
With this notation, the five gradient expressions in Equations \ref{eqn:ql}, \ref{eqn:pgpb}, \ref{eqn:var},
\ref{eqn:ilo}, \ref{eqn:silo} can be extended to the off-policy case where the dependence on $\Delta_O$ and $\Delta_R$ can be captured as a function $f: \mathbb{R}^2 \mapsto \mathbb{R}$.
\begin{align}
	\hat{G}_{QL}(s, a, \theta) &= f_{SQ}(\Delta_O, \Delta_R) \Big( \nabla_\theta q_\theta(s, a) \Big) \label{eqn:q} \\
	\hat{G}_{IL}(s, a, \theta) &= f_{IL}(\Delta_O, \Delta_R)\Big( \nabla_\theta q_\theta(s, a) \Big) \label{eqn:il} \\
	\hat{G}_{SIL}(s, a, \theta) &= f_{SIL}(\Delta_O, \Delta_R)\Big( \nabla_\theta q_\theta(s, a) \Big) \label{eqn:sil} \\
	\hat{G}_{PGPB}(s, a, \theta) &=  f_{SQ}(\Delta_O, \Delta_R)\Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)
	+ \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]  \label{eqn:pg} \\
	\hat{G}_{V}(s, a, \theta) &= f_{SQ}(\Delta_O, \Delta_R)  \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big) \label{eqn:v}
\end{align}
The scaling functions, $f_{SQ}, f_{IL}, f_{SIL}$ used above are:
\begin{align}
	f_{SQ}(\Delta_O, \Delta_R) &= e^{\Delta_O}  \Delta_R\\
	f_{IL}(\Delta_O, \Delta_R) &= e^{\Delta_O} ( e^{\Delta_R} - 1) \\
	f_{SIL}(\Delta_O, \Delta_R)  &= e^{\Delta_O} \max(\Delta_R, 0)
\end{align}
In all cases above, it is easy to verify that $f \in \{f_{SQ}, f_{IL}, f_{SIL}\}$ satisfies the properties listed below.
While we do not currently have a formal theoretical justification for why these properties might be important, they seem like reasonable constraints to use in constructing closely related, but slightly more general and improved alternative update rules.
\begin{assumption}\label{assumption:constraints}
	Let $f: \mathbb{R}^2 \mapsto \mathbb{R}$ denote a function that captures the dependence of a gradient update on $\Delta_O, \Delta_R$. Then, we require that $f$ satisfies the following constraints:

\begin{enumerate}
	\item $f(\Delta_O, 0) = 0, ~~\forall ~~ \Delta_O \in \mathbb{R}$.
	\item $f(-\infty, \Delta_R) = 0, ~~ \forall ~~ \Delta_R \in \mathbb{R}$.
	\item $f(\Delta_O, \Delta_R)$ is non-decreasing and convex in $\Delta_R$ for any fixed $\Delta_O$.
	\item $|f(\Delta_O, \Delta_R)|$ is non-decreasing and convex in $\Delta_O$ for any fixed $\Delta_R$. (~$|.|$ denotes absolute value.)
\end{enumerate}
\end{assumption}

\subsection{A practical approximation to the imitation learning gradient}\label{sec:approx}
We now construct a piece-wise polynomial second order approximation\footnote{ILA is used as an abbreviation for `Imitation Learning Approximation'.}, $f_{ILA}$, to $f_{IL}(\Delta_O, \Delta_R) = e^{\Delta_O} (e^{\Delta_R}-1)$ used
in $\hat{G}_{IL}$ from Equation \ref{eqn:il} that is consistent with the 4 constraints listed in Assumption \ref{assumption:constraints}.
\begin{equation}\label{eqn:ila}
	f_{ILA}(\Delta_O, \Delta_R)=
	\begin{cases}
		-\frac{1}{2}(1 + \Delta_O)^2 , & \text{if } 1 + \Delta_O + \Delta_R \le 0 < 1 + \Delta_O \\
		\Delta_R \max \left(1 + \Delta_O + \frac{\Delta_R}{2}, 0\right) ,              & \text{otherwise}
	\end{cases}
\end{equation}
Using this, we define the approximate gradient estimate $\hat{G}_{ILA}$ as:
\begin{equation}\label{eqn:gila}
\hat{G}_{ILA}(s, a, \theta) = f_{ILA}(\Delta_O, \Delta_R)\Big( \nabla_\theta q_\theta(s, a) \Big)
\end{equation}
A visualization for the projection of $f_{ILA}$ along various slices of $\Delta_O$ and $\Delta_R$ is shown in Figures ~\ref{fig:slice_r} and \ref{fig:slice_o}.
When highly off-policy ($\Delta_O \ll 0$), note that $f_{ILA}(\Delta_O, \Delta_R) = 0$ for $\Delta_R$ less than a positive threshold. This threshold increases as the sample becomes more off-policy (ie. as $\Delta_O \rightarrow -\infty$). For large values of $\Delta_R$, the update strength is quadratic in $\Delta_R$. Conversely, it saturates to a small non-positive constant as $\Delta_R \rightarrow -\infty$.
\begin{figure}
	\centering
	\begin{subfigure}{.5\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{slice_r}
		\caption{Projection of $f_{ILA}$ onto $\Delta_R$ for varying $\Delta_O$}
		\label{fig:slice_r}
	\end{subfigure}%
	\begin{subfigure}{.5\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{slice_o}
		\caption{Projection of $f_{ILA}$ onto $\Delta_O$ for varying $\Delta_R$}
		\label{fig:slice_o}
	\end{subfigure}
	\caption{One-dimensional projections for $f_{ILA}(\Delta_O, \Delta_R)$ from Equation \ref{eqn:ila}.}
	\label{fig:test}
\end{figure}

The full range of gradients characterized are summarized in Table~\ref{tab:table1}, which mixes the three types of updates (squared error minimization, variance minimzation and PGPB), with various scaling functions across the rows.  In addition to the six types of updates derived in Equations \ref{eqn:q}, \ref{eqn:il}, \ref{eqn:sil}, \ref{eqn:pg},
\ref{eqn:v}, \ref{eqn:gila}, the table also lists six natural additional variations for a total of twelve update rules being
considered in the numerical evaluations in Section \ref{sec:eval}.
\begin{table}[!ht]
\begin{center}
\caption{A summary of the various gradients derived and their relationships.}
	\renewcommand{\arraystretch}{1.5}
\label{tab:table1}
	\begin{tabular}{|c|c|c|c|}
		\hline
		\multicolumn{4}{|l|}{$\mathcal{U}_{Q}(f) \equiv f(\Delta_O,\Delta_R)\Big( \nabla_\theta q_\theta(s, a) \Big)$} \\
		\multicolumn{4}{|l|}{$\mathcal{U}_{V}(f) \equiv f(\Delta_O,\Delta_R) \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)$ }\\
		\multicolumn{4}{|l|}{$\mathcal{U}_{P}(f) \equiv f(\Delta_O,\Delta_R) \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big) + \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]$} \\
		\hline
		\hline
		$f$ & $\mathcal{U}_Q(f)$ & $\mathcal{U}_V(f)$ & $\mathcal{U}_P(f)$\\
		\hline
		$f_{SQ}(x, y) \triangleq e^x y$ & $\hat{G}_{Q}$& $\hat{G}_{V}$& $\hat{G}_{PGPB}$ \\
		\hline
		$f_{IL}(x, y) \triangleq e^x (e^y - 1) $ & $\hat{G}_{IL}$ & $\hat{G}_{IL, V}$ & $\hat{G}_{IL, PGPB}$ \\
		\hline
		$f_{SIL}(x, y) \triangleq e^{x} \max \left(y, 0\right)$ & $\hat{G}_{SIL}$ & $\hat{G}_{SIL, V}$ & $\hat{G}_{SIL, PGPB}$ \\
		\hline
		$f_{ILA}(x, y) \triangleq
		 \begin{cases}
			-\frac{1}{2}(1 + x)^2 , & \text{if }1 + x + y \le 0 < 1 + x \\
		y \max \left(1 + x + \frac{y}{2}, 0\right) ,              & \text{otherwise}
	\end{cases}$ & $\hat{G}_{ILA}$ & $\hat{G}_{ILA, V}$ & $\hat{G}_{ILA, PGPB}$\\
		\hline
 \end{tabular}
\end{center}
\end{table}

\subsection{A Synthetic 2D Contextual Bandit}\label{sec:2db}
\input{sections/synthetic.tex}

\subsection{FourRoom Environment}\label{sec:tabular}
\input{sections/tabular.tex}

\subsection{Continuous Control Experiments}\label{sec:cce}
\input{sections/cce.tex}

\section{Proofs} \label{app:proofs}
\pgpb*
\begin{proof}
	Under the given assumptions on $\hat{T}(s, a)$, we have:
\begin{align}
	&\nabla_\theta J^\pi_\mu(\theta)  = \expectation_{(s, a) \sim D_\pi} \left[ \hat{T}(s, a) \nabla_\theta \log{\pi_\theta(a|s)} \right] \\
	&= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a|s \sim \pi} \left[ (\hat{T}(s, a) - q_\theta(s, a))\nabla_\theta \log{\pi_\theta(a|s)}  \right]
		       + \expectation_{u | s \sim \pi} \left[ q_\theta(s, u) \nabla_\theta \log{\pi_\theta(u|s)} \right] \right]\\
	&= \expectation_{s \sim d^\pi_\mu} \left[ \expectation_{a | s \sim \pi} \left[ (\hat{T}(s, a) - q_\theta(s, a))\nabla_\theta \log{\pi_\theta(a|s)}  \right]
		       + \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] \right]\\ \label{eqn:hat}
	&= \expectation_{(s, a) \sim D_\pi} \left[ \Big(\hat{T}(s, a) - q_\theta(s, a)\Big) \nabla_\theta \log{\pi_\theta(u|s)} + \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] \right]  \\
	&= \expectation_{(s, a) \sim D_\pi} \left[ \Big(\hat{T}(s, a) - q_\theta(s, a)\Big) \nabla_\theta \log{\pi_\theta(u|s)} - \nabla_\theta H(\pi_\theta(.|s)  \right]  \text{ using Theorem \ref{lemma:entropy}} \\
	&= \expectation_{(s, a) \sim D_\pi} \left[ \hat{G}_{PGPB}(s, a, \theta) \right]
\end{align}
\end{proof}

\entropy*
\begin{proof}
\begin{align*}
&	\nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right] = \sum_u q_\theta(s, u) \nabla_{\theta} \pi_\theta(u|s) \\
											     & = \sum_u \pi_\theta(u|s) q_\theta(s, u) \nabla_\theta \log \pi_\theta(u|s) \\
											     &= \sum_u \pi_\theta(u|s) (\log \pi_\theta(u|s) + F(q_\theta(s))) \nabla_\theta \log \pi_\theta(u|s) \\
											     &= \sum_u \pi_\theta(u|s) \log \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s)  + F(q_\theta(s))  \sum_u \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s)  \\
											     &= \sum_u \pi_\theta(u|s) \log \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s)  + F(q_\theta(s))  \sum_u \nabla_\theta \pi_\theta(u|s) \\
											     &= \sum_u \pi_\theta(u|s) \log \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s)  + F(q_\theta(s))  \nabla_\theta \sum_u \pi_\theta(u|s) \\
											     &= \sum_u \pi_\theta(u|s) \log \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s)  + F(q_\theta(s))  \nabla_\theta 1 \\
											     &= \sum_u \pi_\theta(u|s) \log \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s)  + \nabla_\theta 1 \\
											     &= \sum_u \pi_\theta(u|s) \log \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s)  + \sum_u \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s) \\
											     &= \sum_u \log \pi_\theta(u|s) \nabla_\theta \pi_\theta(u|s)  + \sum_u \pi_\theta(u|s) \nabla_\theta \log \pi_\theta(u|s) \\
											     &= \sum_u \nabla_\theta \left( \pi_\theta(u|s) \log \pi_\theta(u|s) \right) \\
											     & = - \nabla_\theta H(\pi_\theta(.|s)
\end{align*}
\end{proof}

\msemve*
\begin{proof}
\begin{align}
	G_{MSE}(\theta) &= -  \frac{1}{2} \expectation_{(s, a) \sim \hat{D}^\pi_\mu}
	\nabla_\theta \left[ ( \hat{T}(s, a) -  q_\theta(s, a) )^2 \right] \nonumber \\
	&= \expectation_{(s, a) \sim \hat{D}^\pi_\mu}
	\left[ \left(\hat{T}(s, a) - q_\theta(s, a)\right) \nabla_\theta q_\theta(s, a) \right] \nonumber \\
	&= \expectation_{(s, a) \sim \hat{D}^\pi_\mu} \left[ \hat{G}_{MSE}(s, a, \theta) \right] \label{eqn:gmse_eq}
\end{align}
This proves the statement about $\hat{G}_{MSE}$. For the  claim regarding $\hat{G}_{MVE}$:
\begin{align}
	G_{MVE}(\theta) &= - \nabla_\theta \frac{1}{2} \expectation_{s} \variance_{a}\Big[\hat{T}(s, a) - q_\theta(s, a) \Big] \\
			&= - \frac{1}{2} \expectation_{s} \nabla_\theta \variance_{a|s \sim \hat{\pi}}\Big[\hat{T}(s, a) - q_\theta(s, a) \Big] \\
			&= -\frac{1}{2} \expectation_{s} \left[ \nabla_\theta \expectation_{a|s \sim \hat{\pi}} \left[ (\hat{T}(s, a) - q_\theta(s, a))^2 \right] - \nabla_\theta \left( \expectation_{a|s \sim \hat{\pi}} \left[\hat{T}(s, a) - q_\theta(s, a)\right]\right)^2 \right] \\
			&= \expectation_{s} \left[ \expectation_{a|s \sim \pi} \Big[ \Big(\hat{T}(s, a) - q_\theta(s, a)\Big)\nabla_\theta q_\theta(s, a)\Big] - \expectation_{a|s \sim \pi} \Big[\hat{T}(s, a) - q_\theta(s, a) \Big] \expectation_{u|s \sim \pi} \Big[ \nabla_\theta q_\theta(s, u) \Big] \right] \\
			&= \expectation_{(s, a)}\left[ \Big(\hat{T}(s, a) - q_\theta(s, a)\Big)\Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big) \right] \\
			&= \expectation_{(s, a)}\left[ \Big(\hat{T}(s, a) - q_\theta(s, a)\Big) \nabla_\theta \log \pi_\theta(a|s) \right] \\
			&= \expectation_{(s, a)} \left[ \hat{G}_{MVE}(s, a, \theta) \right]
\end{align}
\end{proof}

\ppo*

\begin{proof}
Assume that the batch of sample data is generated from a policy $\pi_b$ with parameters $b$.
Let $\hat{G}_{PPO}$ denote the PPO surrogate objective sample gradient. This gradient can be simplified as follows to recover the scaling function equivalence.
The PPO surrogate objective is defined for a given clipping parameter $\epsilon$ as:
$$J_{PPO}(s, a, \theta) = \left(\frac{\pi_\theta(a|s)}{\pi_b(a|s)}\hat{A}, \text{clip}\left(\frac{\pi_\theta(a|s)}{\pi_b(a|s)}, 1-\epsilon, 1 + \epsilon\right) \hat{A} \right)  $$
We will focus on computing the gradient, $\hat{G}_{PPO}(s, a, \theta) = \nabla_\theta J_{PPO}(s, a, \theta)$ below.
Note that $\Delta_O = \log \frac{\pi_\theta(a|s)}{\pi_b(a|s)}$, and $\Delta_R = \hat{A}$ under the specified assumptions. Therefore,
\begin{align}
	& \hat{G}_{PPO}(s, a, \theta) = \nabla_\theta \left( \min \left(\frac{\pi_\theta(a|s)}{\pi_b(a|s)}\Delta_R, \text{clip}\left(\frac{\pi_\theta(a|s)}{\pi_b(a|s)}, 1-\epsilon, 1 + \epsilon\right) \Delta_R \right) \right) \\
	&= \nabla_\theta \left( \mathbbm{1}_{\Delta_R > 0} \min \left( \frac{\pi_\theta(a|s)}{\pi_b(a|s)}, 1 + \epsilon \right) \Delta_R + \mathbbm{1}_{\Delta_R < 0} \max \left( 1- \epsilon, \frac{\pi_\theta(a|s)}{\pi_b(a|s)} \right) \Delta_R \right) \\
	 &= \nabla_\theta \Big( \left( \mathbbm{1}_{\Delta_R > 0} \mathbbm{1}_{\Delta_O < \log(1+\epsilon)} + \mathbbm{1}_{\Delta_R < 0} \mathbbm{1}_{\Delta_O > \log(1 + \epsilon)}\right) \frac{\pi_\theta(a|s)}{\pi_b(a|s)} \Delta_R ~~ + \nonumber \\
	 & \hspace{1in} (1+ \epsilon) \mathbbm{1}_{\Delta_R > 0} \mathbbm{1}_{\Delta_O \ge \log(1+\epsilon)} \Delta_R + (1 - \epsilon) \mathbbm{1}_{\Delta_R < 0} \mathbbm{1}_{\Delta_O \le \log(1 - \epsilon)} \Delta_R \Big) \\
	 &= \left( \mathbbm{1}_{\Delta_R > 0} \mathbbm{1}_{\Delta_O < \log(1+\epsilon)} + \mathbbm{1}_{\Delta_R < 0} \mathbbm{1}_{\Delta_O > \log(1 + \epsilon)}\right) \nabla_\theta \frac{\pi_\theta(a|s)}{\pi_b(a|s)} \Delta_R + \textbf{0}  ~~~(\because \text{ no gradient from the second term.}) \\
	 &= \tau_\epsilon(\Delta_O, \Delta_R) \frac{\pi_\theta(a|s)}{\pi_b(a|s)} \nabla_\theta \log \pi_\theta(a|s)  \Delta_R \\
	 &= \tau_\epsilon(\Delta_O, \Delta_R) e^{\Delta_O} \Delta_R \nabla_\theta \log \pi_\theta(a|s)
\end{align}
\end{proof}

\subsection{Design for the parametric scaling function approximations}
Consider the following lower bound to $e^{\Delta_O}(e^{\Delta_R} - 1)$ which is also exact upto second order around $(\Delta_O, \Delta_R) = (0, 0)$. Let $X \sim U[\Delta_O, \Delta_O + \Delta_R]$ be a continuous RV with uniform density. Then:
$$\expectation[e^X] = \int_{\Delta_O}^{\Delta_O + \Delta_R}
\frac{e^x}{\Delta_R} dx = \frac{e^{\Delta_O + \Delta_R} -
e^{\Delta_O}}{\Delta_R} \mbox{ and } e^{\expectation[X]} = e^{\Delta_O + \frac{\Delta_R}{2}}$$
Jensen's inequality implies that
$$0 < e^{\Delta_O + \frac{\Delta_R}{2}} \le \frac{e^{\Delta_O + \Delta_R} - e^{\Delta_O}}{\Delta_R} ~~ \forall ~~ \Delta_O, \Delta_R \in \mathbb{R} $$
Consider the following approximation inspired by the above inequality:
\begin{equation} \label{eqn:japprox}
e^{\Delta_O + \frac{\Delta_R}{2}} \approx \frac{e^{\Delta_O + \Delta_R} - e^{\Delta_O}}{\Delta_R}
\end{equation}
We then consider the following sequence of approximations:
\begin{align}
	e^{\Delta_O}(e^{\Delta_R} - 1) &= e^{\Delta_O + \Delta_R} - e^{\Delta_O} \\
				       &\approx \Delta_R e^{\Delta_O + \frac{\Delta_R}{2}} ~~ \mbox{ Using Eq } \ref{eqn:japprox}\\
				       &\ge \Delta_R \max \left(1 + \Delta_O + \frac{\Delta_R}{2}, 0\right) ~~ \mbox{ since $e^x \ge \max(1 + x, 0) ~~ \forall x \in \mathbb{R}$ } \label{eqn:approxd1}
\end{align}
Unfortunately, $f_0(\Delta_O, \Delta_R) \triangleq \Delta_R \max \left(1 + \Delta_O + \frac{\Delta_R}{2}, 0\right)$ does not satisfy the monotonicity constraint in $\Delta_R$ listed in \ref{assumption:constraints} in all cases.
To check this, we can verify that it is non-decreasing (and also convex) in $\Delta_R$ if and only if $1 + \Delta_O \le 0$.
Equation (\ref{eqn:ml1}) makes a fix to $f_0$ to ensure that the monotonicity constraints from Assumption \ref{assumption:constraints} hold while also remaining a good second order approximation when $\Delta_O, \Delta_R$ are small.

\textbf{Fix for non-monotonicity($f_{MLA}$):} First notice that the non-monotonicity only happens when $1 + \Delta_O > 0$.
In that case, the minimum of the quadratic in $y$, $q(x, y) = y (1 + x + y/2)$ occurs for $y_{min}(x) \triangleq - (1 + x) < 0$.
However, in this case $1 + x + y_{min}(x)/ 2 = (1 + x)/2 > 0$.
So $f(x, y) = q(x, y)$ in a left neighborhood of $y_{min}(x) = -(1 + x)$, which means it must be decreasing in $y$ in that range.
To avoid this, we make a fix $f(x, y) \triangleq q(x, y_{min}(x)) = -(1 + x)^2/2$, when $y < y_{min}(x) = - (1 + x)$ and $1 + x > 0$.  The resulting $f(x, y)$ is the definition of $f_{MLA}(x, y)$ in Equation (\ref{eqn:ml1}).

\textbf{Derivation of $f_{MLA(\alpha_0, \alpha_r)}$:}
In the policy parameterization, we did not include any temperature parameter for simplicity. However, by considering a generalization of these as free parameters,
we may consider scaling functions where the inputs are replaced with $\alpha_o \Delta_O$ and $\alpha_r \Delta_R$ respectively.
The same sequence of approximations resulting in Equation (\ref{eqn:approxd1}) now result in the constraint violating function\footnote{$(x)_+$ denotes $\max(x, 0)$} $f_1(x, y) = \alpha_r y(1 + \alpha_o x + \alpha_r y / 2)_+$.
Without loss of generality, this is equivalent to $f_1(x, y) = y(1 + \alpha_o x + \alpha_r y)_+$,
because (1) the constant $\alpha_r$ scaling the outer $y$ has no impact on the overall update, and (2) we can redefine the remaining $\alpha_r$ inside to include the 1/2 for simplicity.
We are now left with fixing the non-monotonicity of $f_1(x, y) = y\max(1 + \alpha_0 x + \alpha_r y, 0)$ for certain ranges of $y$ when $1 + \alpha_0 x < 0$.
However, we do this slightly differently compared to how it was done for $f_{MLA}$ above, in order to preserve the linear component  in $y$,
as this allows for a more intuitive understanding as being the component that matches the identity scaling function. \\

\textbf{Fix for non-monotonicity:}  As before, consider the quadratic in $y$, $q(x, y) \triangleq y(1 + \alpha_o x + \alpha_r y)$
and its argmin
$y_{min}(x) = \frac{-(1 + \alpha_o x)}{2\alpha_r}$ in the problematic case when $1 + \alpha_o x > 0$. Observe that when $y=y_{min}$, we have $1 + \alpha_o x + \alpha_r y = \frac{1+ \alpha_o x}{2}$. Recall that the problem
occurs due to the quadratic dependence on $y$ from the second term for $y < y_{min}$.
Instead of completely eliminating the dependence on $y$, we can continue a monotonic linear relationship by simply clipping the second term below by the value it takes at $y_{min}$ which is
$1 + \alpha_o x + \alpha_r y = \frac{1+ \alpha_o x}{2}$ .
However, recall that we should only perform this clipping of the second term below by $\frac{1 + \alpha_o x}{2}$
in the case when $1 + \alpha_o x > 0$.
This can be done simply by clipping the second term by $\frac{(1 + \alpha_o x)_+}{2}$ as this reduces to the original floor of $0$ in the case where no fix was needed.
This results in the proposed $f_{MLA(\alpha_o, \alpha_r)}(x, y) = y \max \left(1 + \alpha_o x + \alpha_r y, \frac{(1 + \alpha_o x)_+}{2}\right)$.

\begin{figure}
	\centering
	\begin{subfigure}{.45\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{r_slice_1}
		\caption{Projections on $\Delta_R$}
		\label{fig:slice_r_1}
	\end{subfigure}%
	\begin{subfigure}{.45\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{o_slice_1}
		\caption{Projections on $\Delta_O$}
		\label{fig:slice_o_1}
	\end{subfigure}
	\caption{Visualization for $f_{MLA}(\Delta_O, \Delta_R)$ (Equation \ref{eqn:ml1}) }
	\label{fig:ml1}
\end{figure}
\begin{figure}
	\centering
	\begin{subfigure}{.45\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{r_slice_2.pdf}
		\caption{Projections on $\Delta_R$}
		\label{fig:slice_r_2}
	\end{subfigure}%
	\begin{subfigure}{.45\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{o_slice_2}
		\caption{Projections on $\Delta_O$}
		\label{fig:slice_o_2}
	\end{subfigure}
	\caption{Visualization for $f_{MLA(1, 0.5)}(\Delta_O, \Delta_R)$ (Equation \ref{eqn:ml2} for $\alpha_o = 1, \alpha_r=0.5$.)}
	\label{fig:ml2}
\end{figure}

\begin{figure}
	\centering
	\begin{subfigure}{0.45\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{gap_q.pdf}
		\caption{Learning curves for $\mathcal{U}_Q(f)$: Col 1, Table \ref{tab:table1}}
		\label{fig:gap_all}
	\end{subfigure}%
	\begin{subfigure}{.45\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{gap_v.pdf}
		\caption{Learning curves for $\mathcal{U}_V(f)$: Col 2, Table \ref{tab:table1}}
		\label{fig:gap_sq}
	\end{subfigure}
	\begin{subfigure}{.45\textwidth}
		\centering
		\includegraphics[width=.9\linewidth]{gap_pgpb.pdf}
		\caption{Learning curves for $\mathcal{U}_{P}(f)$: Col 3, Table \ref{tab:table1}}
		\label{fig:gap_pgpb}
	\end{subfigure}
	\caption{Gap to the optimal policy reward for the twelve updates listed in Table \ref{tab:table1} grouped by the update type in each column, and compared across scaling functions $f \in \{f_{SQ}, f_{ML}, f_{SIL}, f_{MLA}\}$}
	\label{fig:gap}
\end{figure}

\begin{figure}
	\centering
	\begin{minipage}{.6\textwidth}
		\centering
		\includegraphics[width=0.9\linewidth]{err_all.pdf}
		\caption{Final Euclidean distance to the optimal parameters, $\theta^* = (1, 1)$ for the tweleve update rules considered.}
		\label{fig:err}
	\end{minipage}%
\end{figure}

\section{Table of update rule combinations evaluated for the 2D synthetic experiment}
The full range of gradients characterized are summarized in Table~\ref{tab:table1}, which considers the form-axis along the columns for the three types of updates (squared error minimization, variance minimzation and PGPB), in conjunction with several scaling functions along the row axis.
\begin{table}[!ht]
\begin{center}
\caption{A summary of the various updates evaluated in the synthetic 2D experiment.}
	\renewcommand{\arraystretch}{1.5}
\label{tab:table1}
	\begin{tabular}{|c|c|c|c|}
		\hline
		\multicolumn{4}{|l|}{$\mathcal{U}_{Q}(f) \equiv f(\Delta_O,\Delta_R)\Big( \nabla_\theta q_\theta(s, a) \Big)$} \\
		\multicolumn{4}{|l|}{$\mathcal{U}_{V}(f) \equiv f(\Delta_O,\Delta_R) \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big)$ }\\
		\multicolumn{4}{|l|}{$\mathcal{U}_{P}(f) \equiv f(\Delta_O,\Delta_R) \Big(\nabla_\theta q_\theta(s, a) - \expectation_{u|s \sim \pi} \left[ \nabla_\theta q_\theta(s, u) \right]\Big) + \nabla_\theta \expectation_{u|s \sim \pi_\theta} \left[ \hat{q}_\theta(s, u) \right]$} \\
		\hline
		\hline
		$f$ & $\mathcal{U}_Q(f)$ & $\mathcal{U}_V(f)$ & $\mathcal{U}_P(f)$\\
		\hline
		$f_{SQ}(x, y) \triangleq e^x y$ & $\hat{G}_{Q}$& $\hat{G}_{V}$& $\hat{G}_{PGPB}$ \\
		\hline
		$f_{ML}(x, y) \triangleq e^x (e^y - 1) $ & $\hat{G}_{ML}$ & $\hat{G}_{ML, V}$ & $\hat{G}_{ML, PGPB}$ \\
		\hline
		$f_{SIL}(x, y) \triangleq e^{x} \max \left(y, 0\right)$ & $\hat{G}_{SIL}$ & $\hat{G}_{SIL, V}$ & $\hat{G}_{SIL, PGPB}$ \\
		\hline
		$f_{MLA}(x, y) \triangleq
		 \begin{cases}
			-\frac{1}{2}(1 + x)^2 , & \text{if }1 + x + y \le 0 < 1 + x \\
		y \max \left(1 + x + \frac{y}{2}, 0\right) ,              & \text{otherwise}
	\end{cases}$ & $\hat{G}_{MLA}$ & $\hat{G}_{MLA, V}$ & $\hat{G}_{MLA, PGPB}$\\
		\hline
 \end{tabular}
\end{center}
\end{table}

\begin{figure}
	\centering

		\includegraphics[width=.5\linewidth]{policy_eval.png}
		\caption{Objective landscape}
		\label{fig:objective}

\end{figure}

\section{Experiment Details}

\subsection{Tabular Experiments}

\begin{figure}
	\centering
	\includegraphics[width=0.4\linewidth]{FourRoomEnv.pdf}
	\caption{The FourRoom Environment}
	\label{fig:fourroom-env}
\end{figure}

The FourRoom environment is shown in Figure~\ref{fig:fourroom-env}.
Both policy gradient (PG) and $Q$-learning (QL) use SGD as the optimizer.
The learning rate for PG is 0.1 for both the actor and the critic, while the learning rate for QL is 0.01.
Figure~\ref{fig:fourroom-results} reports the exponential running average return with exponent discount of 0.01.

\subsection{Continuous Control Experiments}\label{appendix:cce}
The full set of update equations used are summarized below:
\begin{align*}\label{eqn:mla_ppo}
	\hat{G}_{MLA-PPO}(s, a, \theta) &= f_{MLA-PPO}(\Delta_O, \Delta_R) \nabla_\theta \log \pi_\theta(a|s) + \beta \nabla_\theta H(\pi_\theta(.|s)), \text{ where:} \\
	f_{MLA-PPO}(x, y) &= f_{MLA(\alpha_o, \alpha_r)}(x, y) \tau_\epsilon(x, y), \text{ where:}  \\
	f_{MLA(\alpha_o, \alpha_r)}(x, y) &=  y \max \left( 1 + \alpha_o x + \alpha_r y , \frac{(1 + \alpha_o x)_+}{2}\right) \nonumber  \\
	\tau_\epsilon(x, y) &= \mathbbm{1}_{y > 0} \mathbbm{1}_{x < \log(1 + \epsilon)} + \mathbbm{1}_{y <0} \mathbbm{1}_{x > \log(1 - \epsilon)} \\
	\Delta_R &= \hat{A} - \alpha \big(\log \pi(a|s) + H(\pi(.|s))\big)
\end{align*}

\begin{table}[!ht]
\begin{center}
\caption{Optimal hyper-parameter configurations for PPO - MLA on the MuJoCo Tasks. Note that the performance reported for the baseline PPO implementation uses its own independently tuned clipping $\epsilon$.}
	\renewcommand{\arraystretch}{1.5}
\label{tab:table2}
	\begin{tabular}{|c|c|c|c|c|}
		\hline
		Environment & $\alpha_r$ & $\alpha_o$ & $\alpha$ & $\epsilon$\\
		\hline
		Walker2d-v2 & $0.05$ & $0.1$ & $0.0$ & $0.2$ \\
		\hline
		Hopper-v2 & $1.0$ & $0.1$ & $0.01$ & $0.2$ \\
		\hline
		HalfCheetah-v2 & $0.5$ & $0.1$ & $0.1$ & $0.2$ \\
		\hline
		Humanoid-v2 & $1.0$ & $1.0$ & $0.0$ & $0.3$ \\
		\hline
 \end{tabular}
\end{center}
\end{table}