My code first exec model’s forward, then use model’s output calculate loss on GPU0. Such as
import torch.nn as nn
from torchvision.models import resnet50

def classify_loss_fn(output, label):
  loss = torch.nn.CrossEntropyLoss()(output, label.long())
  return loss

model = resnet50()
model = nn.DataParallel(model, device_ids=[1, 2, 3, 4]) # or
model = nn.parallel.DistributedDataParallel(model, device_ids=[1, 2, 3, 4],
                                            find_unused_parameters=True)

for (img, label) in dataloader:
  output = model(img)
  loss = classify_loss_fn(output, label)
  loss.backward()