import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        self.fc = nn.Linear(3, 1)
        self.a_head = nn.Linear(1, 2)
        self.v_head = nn.Linear(1, 1)
        nn.init.uniform_(self.fc.weight,1,1)
        nn.init.uniform_(self.fc.bias,0,0)
        nn.init.uniform_(self.a_head.weight,1,1)
        nn.init.uniform_(self.a_head.bias,0,0)
        nn.init.uniform_(self.v_head.weight,1,1)
        nn.init.uniform_(self.v_head.bias,0,0)

    def forward(self, x):
        x = F.tanh(self.fc(x))
        a = self.a_head(x) - self.a_head(x).mean(1, keepdim=True)
        v = self.v_head(x)
        action_scores = a + v
        return action_scores

class Model():

    max_grad_norm = 0.5
    def __init__(self):

        self.eval_net, self.target_net = Net().float(), Net().float()
        self.optimizer = optim.Adam(self.eval_net.parameters(), lr=1e-3)
    
    def update(self):

        x1 = torch.tensor([[1,1,1]], dtype=torch.float)
        y1 = torch.tensor([[1]], dtype=torch.long).view(-1, 1)
        x2 = torch.tensor([[1,1,1]], dtype=torch.float)
                
        with torch.no_grad():
            y2 = self.eval_net(s_).max(1, keepdim=True)[1]
            q_target = self.target_net(x2).gather(1, y2)

        q_eval = self.eval_net(x1).gather(1, y1)
        print("__________________x1","_________",y1,"_")
        print("_fc_w",self.eval_net.fc.weight,"fc_b",self.eval_net.fc.bias,"_fc_wgrads",self.eval_net.fc.weight.grad,"_fc_bgrads",self.eval_net.fc.bias.grad)
        print("_a_w",self.eval_net.a_head.weight,"a_b",self.eval_net.a_head.bias,"_a_wgrads",self.eval_net.a_head.weight.grad,"_a_bgrads",self.eval_net.a_head.bias.grad)
        print("_v_w",self.eval_net.v_head.weight,"v_b",self.eval_net.v_head.bias,"_v_wgrads",self.eval_net.v_head.weight.grad,"_v_bgrads",self.eval_net.v_head.bias.grad)

        print("loss",q_eval-q_target)

        self.optimizer.zero_grad() 
        loss = F.smooth_l1_loss(q_eval, 2*q_target)

        loss.backward() # that's where the derivatives are calculated    

        print("loss__",loss)

        #nn.utils.clip_grad_norm_(self.eval_net.parameters(), self.max_grad_norm)
        #self.optimizer.step()

        print("_fc_w",self.eval_net.fc.weight,"fc_b",self.eval_net.fc.bias,"_fc_wgrads",self.eval_net.fc.weight.grad,"_fc_bgrads",self.eval_net.fc.bias.grad)
        print("_a_w",self.eval_net.a_head.weight,"a_b",self.eval_net.a_head.bias,"_a_wgrads",self.eval_net.a_head.weight.grad,"_a_bgrads",self.eval_net.a_head.bias.grad)
        print("_v_w",self.eval_net.v_head.weight,"v_b",self.eval_net.v_head.bias,"_v_wgrads",self.eval_net.v_head.weight.grad,"_v_bgrads",self.eval_net.v_head.bias.grad)

def main():
    model = Model()    
    model.update()
                 
if __name__ == '__main__':
    main()

well I wrote a little model that that has two networks, eval_net and target_net, with biase 0 and weight 1; data x1 y1 and x2 with 1 as value, just to try to recalculate manually using my calculator and check with pytorch gradients results, but i’m always wrong, so how pytorch computed these grads in this specific model ?
this is the output
_fc_weight.grads =  tensor[[-0.0098, -0.0098, -0.0098]]
_fc_bias.grads =  tensor[-0.0098]
_a_w grads tensor[[ 0.4951],[-0.4951]]
_a_b grads tensor([ 0.4975, -0.4975])
_v_wgrads tensor[[-0.9901]]
_v_bgrads tensor[-0.9951]
manually calculating, but got wrong values gradients, so what is the secret formula for this specific case ?
that’s what i’ve done for a_w_grads:
print("mycalculation_______",p*(1-math.tanh(3)**2)2(p-loss))