Hello, I found that torch.cat runs slower on GPU than on CPU. Does anyone know the reason?
Result on CPU
time cost for autograd: -0.01325
time cost for cat: -0.00016

Result on GPU
time cost for autograd: -0.00249
time cost for cat: -0.00131


Here is the code. I ran it on a Tesla M40. Pytorch1.1.0
import time
import torch
import torch.nn as nn
import torch.autograd as autograd


class netG(nn.Module):
    def __init__(self):
        super(netG, self).__init__()
        self.net = nn.Sequential(
            nn.Linear(1, 2, bias=False),
            nn.Linear(2, 2, bias=True)
        )
        self.weight_init()

    def forward(self, x):
        return self.net(x)

    def weight_init(self):
        self.net[0].weight.data = torch.Tensor([[0.5], [1.0]])
        self.net[1].weight.data = torch.Tensor([[1.0, 0.0],
                                                [2.0, 1.0]])
        self.net[1].bias.data = torch.Tensor([-0.5, 1.0])


class netD(nn.Module):
    def __init__(self):
        super(netD, self).__init__()
        self.net = nn.Linear(2, 1, bias=True)
        self.weight_init()

    def forward(self, x):
        return self.net(x)

    def weight_init(self):
        self.net.weight.data = torch.Tensor([[3.0, 0.0]])
        self.net.bias.data = torch.Tensor([0.0])


if __name__ == '__main__':
    device = torch.device('cuda:0') if torch.cuda.is_available() else 'cpu'
    torch.cuda.init()
    z = torch.tensor([2.0], device=device)
    G = netG().to(device)
    D = netD().to(device)
    loss = D(G(z))
    torch.cuda.synchronize(device=device)
    start = time.time()
    grad_g = autograd.grad(loss, list(G.parameters()), create_graph=True, retain_graph=True)
    grad_d = autograd.grad(loss, list(D.parameters()), create_graph=True, retain_graph=True)
    torch.cuda.synchronize(device=device)
    end = time.time()
    grad_time = start - end
    print('time cost for autograd: {:.5f}'.format(grad_time))

    torch.cuda.synchronize(device=device)
    start = time.time()
    grad_g_vec = torch.cat([g.contiguous().view(-1) for g in grad_g])
    grad_d_vec = torch.cat([g.contiguous().view(-1) for g in grad_d])
    torch.cuda.synchronize(device=device)
    end = time.time()
    grad_time = start - end
    print('time cost for cat: {:.5f}'.format(grad_time))