Hello, rasbt

reply.png1379×734 53.2 KB

here is another check code :
import torch
import numpy as np

np.random.seed(123)
np.set_printoptions(8, suppress=True)

x_numpy = np.random.random((3, 4)).astype(np.double)
x_torch = torch.tensor(x_numpy, requires_grad=True)
x_torch2 = torch.tensor(x_numpy, requires_grad=True)

w_numpy = np.random.random((4, 5)).astype(np.double)
w_torch = torch.tensor(w_numpy, requires_grad=True)
w_torch2 = torch.tensor(w_numpy, requires_grad=True)

lr = 0.1
weight_decay = 0.9
sgd = torch.optim.SGD([w_torch], lr=lr, weight_decay=0)
sgd2 = torch.optim.SGD([w_torch2], lr=lr, weight_decay=weight_decay)

y_torch = torch.matmul(x_torch, w_torch)
y_torch2 = torch.matmul(x_torch2, w_torch2)

loss = y_torch.sum()
loss2 = y_torch2.sum()

sgd.zero_grad()
sgd2.zero_grad()

loss.backward()
loss2.backward()

sgd.step()
sgd2.step()

w_grad = w_torch.grad.data.numpy()
w_grad2 = w_torch2.grad.data.numpy()

print("check_grad")
print(w_grad + weight_decay * w_numpy)
print(w_grad2)

"""
check_grad
[[ 2.29158508  1.95058016  2.25510989  2.56106592  2.06111261]
 [ 1.25926989  1.57975955  1.58000814  1.67232418  1.86585193]
 [ 2.20280346  2.10071483  2.20099271  1.84145669  1.87640346]
 [ 2.17063112  2.22953686  2.53307273  2.04808866  2.35552527]]
[[ 2.29158508  1.95058016  2.25510989  2.56106592  2.06111261]
 [ 1.25926989  1.57975955  1.58000814  1.67232418  1.86585193]
 [ 2.20280346  2.10071483  2.20099271  1.84145669  1.87640346]
 [ 2.17063112  2.22953686  2.53307273  2.04808866  2.35552527]]
 """