>>> import torch
>>>
>>> lr = 1e-2
>>> torch.manual_seed(666)
<torch._C.Generator object at 0x7f278fb0dc10>
>>> a = torch.randn(3,3)
>>> a.grad = torch.randn(3,3)
>>>
>>> b = torch.add(a, -lr, a.grad)
>>> c = torch.add(a, -lr*a.grad)
>>> print(torch.abs(b-c).sum())
tensor(1.8626e-09)

Aren’t they same? 1e-9 ~= 0.