Solved by soulslicer in post #2 
              
              
                test = torch.nn.Conv2d(in_channels=320, out_channels=320, groups=320, kernel_size= (5,5), padding=5//2, bias=False).cuda()
    test.weight.requires_grad = False
    test.weight[:,:,:,:] = kernel
    out = test(dpv_permuted)