I am training an ensemble of Resnets on CIFAR10.
I have created a list of optimizers, to store each optimizer for each individual model since the model.parameters() must match:
names = []
optimizers = []
for i in range(ensemble_size):
    
    model = ResNet20()
    names.append(model.name + '_' + str(i+1))
    opt = optim.SGD(model.parameters(), learning_rate, momentum, weight_decay)
    optimizers.append(opt)
    
    model.to(device)
    if gpus: model = nn.DataParallel(model)
    ensemble.append(model)

My question is if I need also to create a list of loss, to apply the loss.backward() separately:
for m in models: 
    m.train()
            
for epoch in range(1, epochs+1):
                
# Training
# --------
for i, (images, labels) in enumerate(trainloader):
            
        # Load images and labels
            
        outs = [] # Initialize a list to store the output of each individial model to further averate
        for n, m in enumerate(models):
                
            ## Individual forward pass
                
            # Calculate loss for individual                
            m.zero_grad()                                     # Zero grad each individual model
            output = m(images)                            # Forward pass that model
            outs.append(output)                           # Append to list of outputs
                        
            ## Individual backwad pass
                
            loss.backward()                             #### Does loss.backward wich model should backprop? ####
            optimizers[n].step()        
                
            ## Ensemble foward pass
            
            outputs = torch.mean(torch.stack(outs), dim=0) # Average estimates
            
            # Calculate loss for ensemble
            # Calculate accuracy for ensemble

Thank you!