Perhaps, you could try adding the following code before updating the gradient:
with torch.no_grad():

and it works:
N, D_in, H, D_out = 64, 1000, 100, 10

X = torch.randn(N, D_in).cuda()
y = torch.randn(N, D_out).cuda()

# device = torch.cuda.device('cuda:0')
device = torch.device('cuda:0')
w1 = torch.randn(D_in, H, requires_grad=True, device=device)
w2 = torch.randn(H, D_out, requires_grad=True, device=device)

learning_rate = 1e-6
for it in range(10):
    # 1.forward pass
    y_pred = X.mm(w1).clamp(min=0).mm(w2).cuda()
    # 2.compute loss
    loss = (y_pred - y).pow(2).sum().cuda()
    print(f'iter {it}, loss {loss}')
    # Backward pass
    loss.backward()
    # update weights of w1 and w2
    with torch.no_grad():
        w1 -= learning_rate * w1.grad
        w2 -= learning_rate * w2.grad
        w1.grad.zero_()
        w2.grad.zero_()