Hi ptrblck.It’s me again.I hope you are okay.Okay i just learned that there is a parameter  torch.backends.cudnn.benchmark=True.It will increase speed of training.But i didn’t found any example on this even in pytorch documentation.So here is my training code.Can you tell me where to use this parameter.Thank you.
class Train:
  def __init__(self, model, dataloader, optimizer, stats, scheduler=None, L1lambda = 0,criterion=None,use_amp=True):
    self.model = model
    self.dataloader = dataloader
    self.optimizer = optimizer
    self.scheduler = scheduler
    self.stats = stats
    self.L1lambda = L1lambda
    self.criterion=criterion
    self.loss1=0.0
    self.loss2=0.0
    self.loss=0.0
    self.use_amp=True
  def run(self):
    self.model.train()
    torch.cuda.empty_cache()
    pbar = tqdm_notebook(self.dataloader)
    for data1,data2,target1,target2 in pbar:
      # get samples
      data1,data2 = data1.to(self.model.device), data2.to(self.model.device)
      target1, target2 = target2.to(self.model.device), target2.to(self.model.device)
      self.optimizer.zero_grad()
      output1,output2 = self.model(data1,data2)
      #print(target.shape)
      target1=target1.unsqueeze(1)
      target2=target2.unsqueeze(1)
      self.loss1=self.criterion[0](output1.float(), target1.half())
      self.loss2=self.criterion[1](output2.float(), target2.half())
      #print("loss1 {}".format(self.loss1))
      #print("loss2 {}".format(self.loss2))
      self.loss=(self.loss1+self.loss2)/2.0
      
      # In PyTorch, we need to set the gradients to zero before starting to do backpropragation because PyTorch accumulates the gradients on subsequent backward passes. 
      # Because of this, when you start your training loop, ideally you should zero out the gradients so that you do the parameter update correctly.

      # Predict
      
      #Implementing L1 regularization
      #print(self.loss)
      if self.L1lambda > 0:
        reg_loss = 0.
        for param in self.model.parameters():
          reg_loss += torch.sum(param.abs())
        self.loss += self.L1lambda * reg_loss

      #print(self.loss)
      # Backpropagation
      if self.use_amp:
        with amp.scale_loss(self.loss, self.optimizer) as scaled_loss:
          scaled_loss.backward()
      else:
        self.loss.backward()
      #self.loss.backward()
      #self.optimizer.step()

      # Update pbar-tqdm
      #pred = y_pred.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
      '''n_digits=1
      output = (output * 10**n_digits).round() / (10**n_digits)
      target = (target * 10**n_digits).round() / (10**n_digits)
      correct = output.eq(target).sum().item()
      print("correct is {}".format(correct))'''
      correct=0
      lr = 0
      if self.scheduler:
        lr = self.scheduler.get_last_lr()[0]
      else:
        # not recalling why i used sekf.optimizer.lr_scheduler.get_last_lr[0]
        lr = self.optimizer.param_groups[0]['lr']
      
      #lr =  if self.scheduler else (self.optimizer.lr_scheduler.get_last_lr()[0] if self.optimizer.lr_scheduler else self.optimizer.param_groups[0]['lr'])
      
      self.stats.add_batch_train_stats(self.loss.item(),correct, len(data1), lr)
      pbar.set_description(self.stats.get_latest_batch_desc())
      if self.scheduler:
        self.scheduler.step()

My calling function of train module
import torch.optim as optim
import torch.nn as nn
import time
EPOCHS =1
#max_lr=1
optimizer = optim.SGD(model.parameters(), lr=0.01)
model, optimizer = amp.initialize(
   model, optimizer, opt_level="O2", 
   keep_batchnorm_fp32=True, loss_scale="dynamic"
)
#criterion=nn.L1Loss()
criterion=[nn.BCEWithLogitsLoss(),nn.L1Loss()]
#criterion = nn.BCEWithLogitsLoss()
start=time.time()
model.gotrain(optimizer, train_loader, test_loader, EPOCHS, "/content/gdrive/My Drive",criterion,use_amp=True)
print(time.time()-start)

Thank you sir