Was interested about that myself, below seems to work:
import torch
from torch import autograd, nn

class F(autograd.Function):
	
	@staticmethod
	def forward(ctx, x, g):
		with torch.enable_grad():
			x = x.clone().requires_grad_(True)
			z = g(x)
			ctx.save_for_backward(x, z)
			return z.detach() #infinite backprop without detach()
	
	@staticmethod
	def backward(ctx, output_grad):
		x, z = ctx.saved_tensors
		g_x, = autograd.grad(z, x, output_grad)
		return g_x, None
		
x0 = torch.ones(3, requires_grad=True)
g = nn.Softplus()
y = F.apply(x0, g)
y.mean().backward()
print(x0.grad)
x2 = x0.detach().requires_grad_()
y2 = nn.functional.softplus(x2)
y2.mean().backward()
print(x1.grad)