Thanks for your response.
Yes. Doing a single backward call, takes approximately an hour.
**Instantiating the Network, Optimizer,and Loss Criterion:**
model=ResidualLearningNet()
model.apply(weights_init)
model.to(device)
optimizer=torch.optim.SGD(model.parameters(),lr=0.00001)
criterion=nn.MSELoss()

**Training**
start_time = datetime.now()
n_epochs=10
print_every=1
val_loss_min=np.Inf
for epoch in range(n_epochs):
  train_loss=0
  model.train()
  for iteration,(Xstage1,X_batch) in enumerate(Train_Loader):
    optimizer.zero_grad()
    output=model(Xstage1.to(device))
    loss=criterion(output,X_batch.to(device))
    retain_graph=True
    loss.backward(retain_graph=True)
    optimizer.step()
    train_loss+=loss.item()*X_batch.shape[0]
    if iteration % print_every==0:
      is_training=model.training
      val_loss=0
      model.eval()
      for (Xstage1_valid,X_valid_batch) in Valid_Loader:
        output=model(Xstage1_valid.to(device))
        loss=criterion(output,X_valid.to(device))
        val_loss+=loss.item()*data.shape[0]
      val_loss=val_loss/len(Valid_Loader.sampler)
      if val_loss<=val_loss_min:
        print('\tValidation loss decreased ({:.6f}-->{:.6f}). Saving Model...'.format(val_loss_min,val_loss))
        torch.save(model.state_dict(),'model_second_step.pt')
        val_loss_min=val_loss
      model.train(mode=is_training)   
  train_loss=train_loss/len(Train_Loader.sampler)
  print('Epoch: {} \tTraining loss:{:.6f}'.format(epoch+1,train_loss))
end_time = datetime.now()
print('Duration: {}'.format(end_time - start_time))

did you mean this part?