@ptrblck Thank you so much. I think it will work.
My original code is:
for epoch_index in range(cfg.epoch):
for train_batch_index, (img_batch, label_batch) in enumerate(train_loader):
img_batch = variable(img_batch)
label_batch = variable(label_batch).unsqueeze(dim=1)
predict, reconstruct_img = net(img_batch, label_batch, train=True)
acc = net.calc_acc(predict, label_batch)
margin_loss = net.margin_loss(predict, label_batch)
reconstruct_loss = net.reconstruction_loss(img_batch, reconstruct_img)
loss = margin_loss + reconstruct_loss
net.zero_grad()
loss.backward()
opt.step()
My new code is:
for epoch_index in range(cfg.epoch):
for train_batch_index, (img_batch, label_batch) in enumerate (zip (train_loader, valid_loader )):
img_batch, label_batch = img_batch[0], img_batch[1]
img_batch_v, label_batch_v = label_batch[0], label_batch[1]
 img_batch = variable(img_batch)
 label_batch = variable(label_batch).unsqueeze(dim=1)

 img_batch_v = variable(img_batch_v)
 label_batch_v = variable(label_batch_v).unsqueeze(dim=1)

 predict, reconstruct_img = net(img_batch, label_batch, train=True)

 acc = net.calc_acc(predict, label_batch)
 margin_loss = net.margin_loss(predict, label_batch)
 reconstruct_loss = net.reconstruction_loss(img_batch, reconstruct_img) 
 Custom_loss = custom_loss(img_batch, img_batch_v)  # img_batch from 1 source and img_batch_v for another

 loss = margin_loss + reconstruct_loss + Custom_loss
 net.zero_grad()
 loss.backward()
 opt.step()

But got an error.