We’re having the same problem here. No parallel execution of kernels or in parallel with memory transfer even when pinned memory and non_blocking are used.
Here is my minimal test code:
import torch
from torch import nn
import torch.nn.functional as F
from torch.cuda import Event
from torch.cuda.nvtx import range_push, range_pop, mark
import logging
import time


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv = nn.Conv3d(64, 64, 5, padding=2)

    def forward(self, x):
        for _ in range(5):
            x = F.relu(self.conv(x))
        return x


if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.DEBUG)

    logging.info("Data setup")
    range_push("Data setup")
    net = Net().to("cuda")
    gpu_data = torch.rand((1, 64, 128, 128, 128), dtype=torch.float32, device="cuda:0")
    cpu_data = torch.rand((1, 64, 128, 128, 128), dtype=torch.float32, device="cpu")
    torch.cuda.synchronize()
    range_pop()
    time.sleep(1)

    logging.info("Pinning memory")
    cpu_data_pinned = cpu_data.pin_memory()

    logging.info("Pinned")

    print(cpu_data.is_pinned())
    print(cpu_data_pinned.is_pinned())

    conv_op = torch.nn.Conv3d(64, 64, 5)
    torch.cuda.synchronize()

    range_push("Transfer")
    logging.info("Transfer")
    cpu_to_gpu_data = cpu_data_pinned.to("cuda:0", non_blocking=True)
    range_pop()

    logging.info("Net start")
    range_push("Net")
    conv_result = net(gpu_data)
    range_pop()
    logging.info("Net done")

    torch.cuda.current_stream().record_event(Event(enable_timing=True))
    time.sleep(1)
    range_push("Getting Results")
    print(conv_result[0, 0, 0, 0, 0])
    print(cpu_to_gpu_data[0, 0, 0, 0, 0])
    range_pop()
    logging.info("End")