Hello,
Thanks for the reply. I am not using backward before cat.
I was trying something like this:
placeholder={x: torch.empty((0,1),device=device) for x in ["l1","l2","l3", (...... etc) , "l18"]}
for batch_idx, (inputs, labels,filenames) in loop:
   
    inputs = inputs.to(device)
    labels = labels.to(device)

    # zero the parameter gradients
    optimizer.zero_grad()
    
    # forward
    with torch.set_grad_enabled(phase == 'train'):
        
        outputs = model(inputs)
        outputs=torch.squeeze(outputs,1)
        loss_raw=criterion(outputs_prob.float(),labels.float())
        loss= loss_raw * mask

#for each example in batch
for k in range(loss_masked.size(0)):
    
    #add the loss of each label to its placeholder
    placeholder["l1"]=torch.cat((placeholder["l1"],torch.unsqueeze(
        loss_masked[k,1],0)),0)

    placeholder["l2"]=torch.cat((placeholder["l2"],torch.unsqueeze(
        loss_masked[k,2],0)),0)

    placeholder["l3"]=torch.cat((placeholder["l3"],torch.unsqueeze(
        loss_masked[k,3],0)),0)

   placeholder ...... (etc) ............

    for l in ["l1","l2","l3", .... (etc) .... "l18"]:
        #if the number of non-zero elements of the placeholder is =4, do a backward pass
        # and clear the placeholder
        if placeholder[l][placeholder[l].sum(1)!=0].size(0)==4:      
            print("------    backprop of branch", l, "    ------")
            placeholder[l].clone().mean().backward()
            placeholder[l]=torch.empty((0,placeholder[l].size(1)),device=device)

#the loss of label 0 is always !=0             
loss_masked[:,0].mean().backward()

optimizer.step()

PS- I know the unsqueezing would be unnecessary in this small working example but in reality the 18 labels are distributed by four groups which I want to cluster together.
Any ideas?
Thanks