Thank you very much for the reply! Yes, I was expecting forward time increases with larger batch size, but backward and step time keeps the same. Here is the code
...
            torch.cuda.synchronize()
            time0 = time.time()
            images, labels = images.cuda(), labels.cuda()
            target = labels
            torch.cuda.synchronize()
            cuda_time.update(time.time() - time0)
            
            # soft target
            if args.kd_ratio > 0:
                args.teacher_model.train()
                with torch.no_grad():
                    soft_logits = args.teacher_model(images).detach()
                    soft_label = F.softmax(soft_logits, dim=1)

            # clear gradients
            optimizer.zero_grad()
            
            torch.cuda.synchronize()
            time1 = time.time()
            output = run_manager.net(images)
            loss = run_manager.train_criterion(output, labels)
            torch.cuda.synchronize()
            forward_time.update(time.time() - time1)

            torch.cuda.synchronize()
            time2 = time.time()
            loss.backward()
            torch.cuda.synchronize()
            backward_time.update(time.time() - time2)

            torch.cuda.synchronize()
            time3 = time.time()
            optimizer.step()
            torch.cuda.synchronize()
            step_time.update(time.time() - time3)
...