Constructing new distribution instances is cheap, and you can use the constraints library to automatically handle the transformation of the std parameter to (0, inf), as below.
Code:
import torch
import torch.distributions as dd
from torch.distributions import biject_to

params = {'loc': torch.tensor([1.], requires_grad=True),
          'scale': torch.tensor([1.], requires_grad=True)}

optim = torch.optim.Adam(params.values(), lr=0.001)
target = dd.Normal(5, 5)


def get_dist():
    return dd.Normal(**get_constrained_params())


def get_constrained_params():
    # apply automatic constraints to distribution parameters.
    constraints = target.arg_constraints
    constrained_params = {}
    for k, v in params.items():
        constrained_params[k] = biject_to(constraints[k])(v)
    return constrained_params


for i in range(10000):
    optim.zero_grad()
    dist = get_dist()
    samples = dist.rsample((1000,))
    cost = -(target.log_prob(samples) - dist.log_prob(samples)).sum()
    cost.backward()
    optim.step()
    print(i, get_constrained_params())


Does this satisfy what you had in mind?