You could try this code and see if it’s faster than your current approach:
t = torch.tensor([[20, 4, 5, 9, 22, 5, 50, 9, 45, 5]]).float()
i = torch.tensor([[0, 2, 3, 1, 1, 3, 1, 3, 0, 0]]).long()

z = torch.zeros(4, t.size()[1])
z.scatter_(0, i, t)

tensors = []
for z_ in z:
    tensors.append(z_[z_.nonzero()])