Hi,
Thanks for the repro code !
I ran this on cpu (I don’t have a gpu install at hand) with the following modifications:
import time
def hvptest(D, G, z):
    criterion = nn.BCEWithLogitsLoss()
    d_fake = D(G(z))
    loss = criterion(d_fake, torch.zeros(d_fake.shape, device=device))

    start = time.time()
    grad_d = torch.autograd.grad(loss, D.parameters(), create_graph=True, retain_graph=True)
    end = time.time()
    print('Discriminator backward :  %.7fms' % (end - start))
    grad_d_vec = torch.cat([g.contiguous().view(-1) for g in grad_d])

    start = time.time()
    grad_g = torch.autograd.grad(loss, G.parameters(), create_graph=True, retain_graph=True)
    end = time.time()
    print('Generator backward :  %.7fms' % (end - start))

    grad_g_vec = torch.cat([g.contiguous().view(-1) for g in grad_g])
    # print('Discriminator parameter number: %d' % grad_d_vec.numel())
    # print('Generator parameter number: %d' % grad_g_vec.numel())
    vec_d = grad_d_vec.clone().detach()

    start = time.time()
    hvp_d = torch.autograd.grad(grad_d_vec, G.parameters(), grad_outputs=vec_d, retain_graph=True)
    end = time.time()
    print('Hessian vector product :  %.7fms' % (end - start))


if __name__ == '__main__':
    device = torch.device('cpu')
    batch_size = 128
    print('=====Test1=====')
    D = dc_D().to(device)
    G = dc_G(z_dim=96).to(device)
    img_cifar = (torch.rand((batch_size, 3, 32, 32), device=device) - 0.5) / 0.5
    z_mnist = torch.randn((batch_size, 96), device=device)
    hvptest(D=D, G=G, z=z_mnist)

    print('=====Test2=====')
    D = DC_discriminator(channel_num=3, feature_num=64).to(device)
    G = DC_generator(z_dim=100, channel_num=3, feature_num=64).to(device)
    z_celeba = torch.randn((batch_size, 100, 1, 1), device=device)
    hvptest(D=D, G=G, z=z_celeba)

And I get`
=====Test1=====
Discriminator backward :  0.2947176ms
Generator backward :  0.4657667ms
Hessian vector product :  1.2660155ms
=====Test2=====
Discriminator backward :  0.1807017ms
Generator backward :  0.9121838ms
Hessian vector product :  2.6689870ms

Do you see similar results on cpu?
If you still see the problem, which version of pytorch are you using?