Actually this is what I do but wanted my above code to be more readable such that I something like that:
for epoch in epochs:
   train1(dataloader1, dataloader2, model, optimizer, writer, epoch)
   eval(...)
   metric(...)

def train1(dataloader1, dataloader2, model, optimizer, writer, epoch):
   for iter1, data1 in enumerate(dataloader1): # batch size of dataloader1 is small because no DataParallel
      optimizer.zero_grad()
      class_loss1, reg_loss1 = model(input1, output1) # model is retinanet
      total1 = class_loss1 + reg_loss1
      total1.backward()
      writer.add_scalars('Loss', {'Loss1': total1.item()}, iter1)
      del total1
      if do_train2:
          train2(dataloader2, model, writer)
      optimizer.step()

def train2(dataloader2, model, writer):
   acc_loss = 0
   for iter2, data2 in enumerate(dataloader2): size dl2 >> dl1 -> batch size = 1 and we accumulate the gradient manually
       with torch.no_grad():
          scores, classification, predicted = model(input2) # inference
       class_loss2, reg_loss2 = model(input2, output2) # model is retinanet
       total2 = class_loss2 + reg_loss2
       total2.backward()
       acc_loss += total2.detach()
       del total2

   writer.add_scalars('Loss', {'Loss2': acc_loss}, iter1)