import torch
import torch.nn as nn
N,D_in,H,D_out=64,1000,100,10
model=torch.nn.Sequential(
    torch.nn.Linear(D_in,H,bias=True),
    torch.nn.ReLU(),
    torch.nn.Linear(H,D_out,bias=True)
)
#torch.nn.init.normal_(model[0].weight)
#torch.nn.init.normal_(model[2].weight)
loss_fn=torch.nn.MSELoss(reduction='sum')
x=torch.randn(N,D_in)
y=torch.randn(N,D_out)
learning_rate=1e-3
for i in range(1000000):

    y_pred=model(x)
    loss=loss_fn(y,y_pred)
    if(i%2000==0):
        print(i,loss.item())
    loss.backward()
    #with torch.no_grad():
    for param in model.parameters():
        param=param-param.grad*learning_rate

output:
0 654.9241333007812
2000 654.9241333007812
4000 654.9241333007812
6000 654.9241333007812
Loss is not reducing…