Thanks!
For anyone else, this is the working code:
import torch

class GmmNet(torch.nn.Module):
    def __init__(self):
        super(GmmNet, self).__init__()
        self.cat = torch.nn.parameter.Parameter(torch.ones(5,))
        self.cov = torch.nn.parameter.Parameter(torch.ones(5,))

    def forward(self, x):
        gmm = D.MixtureSameFamily(D.Categorical(self.cat), D.Normal(self.cov, self.cov))
        return gmm.log_prob(x)
        
print(GmmNet().state_dict())
# --> OrderedDict([('cat', tensor([1., 1., 1., 1., 1.])), ('cov', tensor([1., 1., 1., 1., 1.]))])