It looks like it’s the for loop causing the problem – the first time through things are fine, but the second time causes an issue.
class Net(nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.T = nn.Parameter(torch.Tensor(5, 5))        
    self.Q = torch.mm(self.T, self.T.t())
      
  def forward(self, x0):
    rhat = x0.unsqueeze(1).matmul(self.Q).matmul(x0.unsqueeze(2)).squeeze(2)
    return rhat
      
loop = tqdm.tqdm(range(EPOCHS))
model = Net()
optimizer = optim.Adam(model.parameters(), lr=1e-2)

for i in range(2):
  r = Variable(torch.from_numpy(np.random.randn(32, 1).astype(np.float32)))
  x0 = Variable(torch.from_numpy(np.random.randn(32, 5).astype(np.float32)))
  
  optimizer.zero_grad()
  rhat = model(x0)
  loss = F.mse_loss(rhat, r)
  
  loss.backward()  
  optimizer.step()