This seems to be a serialize error. Do you get the same error with these hooks:
import torch
import os
import uuid

tmp_dir = "temp"

class quadratic_fun(torch.nn.Module):
    def __init__(self):
        super(quadratic_fun, self).__init__()
        self.x = torch.nn.Parameter(torch.ones(5, requires_grad=True))
        self.y = torch.nn.Parameter(torch.ones(5, requires_grad=True))

    def forward(self):
        loss = torch.norm(self.x) ** 2 + torch.norm(self.y) ** 2
        return loss

class SelfDeletingTempFile():
    def __init__(self):
        self.name = os.path.join(tmp_dir, str(uuid.uuid4()))

    def __del__(self):
        os.remove(self.name)

def pack_hook(tensor):
    temp_file = SelfDeletingTempFile()
    torch.save(tensor, temp_file.name)
    return temp_file

def unpack_hook(temp_file):
    return torch.load(temp_file.name)


torch.autograd.graph.set_saved_tensors_default_hooks(pack_hook, unpack_hook)

model = quadratic_fun()
loss_quad = model.forward()
grad_ft = torch.autograd.grad(loss_quad, model.parameters(), create_graph=True)
flat_grad = torch.cat([g.contiguous().view(-1) for g in grad_ft])

# generate the constant matrix V, and compute the matrix-gradient product
torch.manual_seed(0)
V = torch.randn((10, 3))
h = torch.matmul(flat_grad, V)

# compute the matrix-Jacobian product by iterating over the columns of the constant matrix
for i in range(3):
    hvp = torch.autograd.grad(h[i], model.parameters(), retain_graph=True)
    hvp_flat = torch.cat([g.contiguous().view(-1) for g in hvp])
    print(hvp_flat)

torch.autograd.graph.reset_saved_tensors_default_hooks()