Hi, thank you for your answer.
I didn’t use load_dict to copy parameters. Instead, I use:
def assign_model_parameters(model, temp_parameters):
    idx = 0
    assert len(temp_parameters) == len(list(model.parameters()))
    for param in model.parameters():
        param.data.copy_(temp_parameters[idx].data)
        idx += 1

And also, if I use torch.autograd.grad instead of backward,
        assign_model_parameters(model, initial_parameters)
        model_parameters = list(model.parameters())
        grads = torch.autograd.grad(second_loss, model_parameters, allow_unused=True)

where second_loss is the loss in the third step aforementioned. And I manually update the gradient,
def update_parameters(temp_parameters, loss, alpha, clip_grad=5.0):
    weights = temp_parameters
    grads = torch.autograd.grad(loss, weights, allow_unused=True)
    direct_clip_grad_norm_(grads, clip_grad)
    new_weights = []
    idx = 0
    for w, g in zip(weights, grads):
        if g is not None:
            #print (g)
            #print (idx)
            new_weights.append(w - alpha * g.detach())
        else:
            new_weights.append(w)
        idx += 1
    assert len(new_weights) == len(weights)
    return new_weights

Can it work as I expected?