I now changed the loss function to this, so I only use torch functions:
def custom_loss(p, t):
    total = torch.zeros(1, dtype=torch.float, requires_grad=True)
    for pair_p, pair_t in zip(torch.combinations(p, r=2), torch.combinations(t, r=2)):
        g1 = torch.eq(torch.gather(pair_p, 0, torch.tensor([0])), torch.gather(pair_p, 0, torch.tensor([1]))).type(torch.uint8)
        g2 = torch.eq(torch.gather(pair_t, 0, torch.tensor([0])), torch.gather(pair_t, 0, torch.tensor([1]))).type(torch.uint8)
        total += (g2 - g1)**2
    return total

model = nn.Linear(3, 5)
output = model(x)

loss = custom_loss(output, t)
loss.backward()
print('loss = ', loss)

And the loss printed now prints this:loss =  tensor([10.], device='cuda:0', dtype=torch.float32, grad_fn=<AddBackward0>)
But when I check the gradients it is still None. Do you know why this?
print(model.weight.grad)
# None
print([x.grad for x in model.parameters()])
# [None, None]