Actually it seems to be surprisingly easy, once you work with the state_dict instead of the optimizer. To test my idea I created a very small neural network:
class small_perceptron(nn.Module):
    def __init__(self, inp=3, mid=2, out=2):
        super(small_perceptron, self).__init__()
        self.lin1 = nn.Linear(inp, mid, bias=True)
        #self.lin2 = nn.Linear(mid, out, bias=True)
        
    def forward(self, x):
        x = self.lin1(x)
        #x = self.lin2(x)
        return x

And then performed a one step optimization process:
lr=0.01
neuronet = small_perceptron()
neuronet.cuda()
loss=torch.nn.MSELoss().cuda()

print('copy initial state to k')
k = copy.deepcopy(neuronet.state_dict()) #k contains the neuronet state before optim
print(k)

optim = torch.optim.Adam(neuronet.parameters(), lr=lr, weight_decay=0.1)

A = Variable(torch.randn(3).cuda()) #random number, after all the goal is not convergence
B = Variable(torch.randn(2).cuda())
C = neuronet(A)
print('C before optim')
print(C)

L = loss(C, B)
optim.zero_grad()
L.backward(retain_graph=True) 
optim.step()
C = neuronet(A)
print('C after optim')
print(C)#C should be different

neuronet.load_state_dict(k)
C = neuronet(A)
print('C state_dict')
print(C)#C should be the same again

And to my great pleasure
C before optim is (0.1776, 0.1483)
C after optim is (0.1989, 0.1696)
and C state_dict is (0.1776, 0.1483), the same as before optim.
So even though you can’t do optim.step_backwards(), you can do as if optim.step() never happened which is pretty much the same!