I tried your code, but it gives me completely different results. Here are my two functions:
spread_kernel = None
def spread_dpv(dpv, N=5):
    global spread_kernel
    dpv_permuted = dpv.permute(0, 3, 2, 1)

    kernel = torch.Tensor(np.zeros((N, N)).astype(np.float32))
    kernel[int(N / 2), :] = 1 / float(N)
    kernel = kernel.repeat((dpv_permuted.shape[1], dpv_permuted.shape[1], 1, 1)).to(dpv_permuted.device)

    dpv_permuted = F.conv2d(input=dpv_permuted, weight=kernel, padding=N // 2)

    dpv = dpv_permuted.permute(0, 3, 2, 1)
    tofuse_dpv = dpv / torch.sum(dpv, dim=1).unsqueeze(1)
    return tofuse_dpv

spread_kernel = None
def spread_dpv_hack(dpv, N=5):
    # torch.Size([128, 384])
    # torch.Size([1, 1, 5, 5])
    global spread_kernel
    dpv_permuted = dpv.permute(0, 3, 2, 1)
    if spread_kernel is None:
        kernel = torch.Tensor(np.zeros((N, N)).astype(np.float32))
        kernel[int(N / 2), :] = 1.
        print(kernel)
        # kernel[2,2] = 1.
        kernel = kernel.unsqueeze(0).unsqueeze(0)
        kernel = kernel.repeat((1, 1, 1, 1))
        print(kernel.shape)
        kernel = {'weight': kernel.to(dpv_permuted.device), 'padding': N // 2}
        spread_kernel = kernel.copy()

    for b in range(0, dpv_permuted.shape[0]):
        for c in range(0, dpv_permuted.shape[1]):
            dpv_permuted[b,c,:,:] = F.conv2d(dpv_permuted[b,c].unsqueeze(0).unsqueeze(0), **spread_kernel).squeeze(0).squeeze(0)

    dpv = dpv_permuted.permute(0, 3, 2, 1)
    tofuse_dpv = dpv / torch.sum(dpv, dim=1).unsqueeze(1)
    return tofuse_dpv

They are suppose to be equivalent in theory. But both give completely different output