Hi @ptrblck   Thanks a lot for your response!
I have a minimal script to show this problem at the end.
I am using Python 3.6.9, Pytorch version 1.7.1, I’m using RTX 2080, CUDA version is 11.0, driver version 450.80.02
Script:
import torchvision.models as models
import torch
import numpy as np
import time


if __name__ == '__main__':
    device=0
    net = models.resnet50(pretrained=True).to(device)
    net.eval()

    data = np.random.uniform(0, 1, (200, 3, 480, 854))
    label = np.random.randint(0, 100, (200,))
    tensor_imgs = torch.Tensor(data.astype(np.float32))
    tensor_labels = torch.Tensor(label.astype(np.float32))
    dataset = torch.utils.data.dataset.TensorDataset(tensor_imgs, tensor_labels)
    data_loader = torch.utils.data.dataloader.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=10)

    for i, (batch_input, batch_target) in enumerate(data_loader):
        if i >= 100:
            break
        batch_input_var = batch_input.to(device)
        output = net(batch_input_var)
    print('Warm-up finished')


    for i, (batch_input, batch_target) in enumerate(data_loader):
        batch_input_var = batch_input.to(device)
        torch.cuda.synchronize()

        time.sleep(0.2)
        start = torch.cuda.Event(enable_timing=True)
        end = torch.cuda.Event(enable_timing=True)
        start.record()
        with torch.no_grad():
            output = net(batch_input_var)
        end.record()
        torch.cuda.synchronize()
        print(start.elapsed_time(end))