Hi All,
I’m opening this draft because I’m developing a model which computes gradients of some outputs with respect to some inputs but I’m having different results based on the library used.
I have PyTorch and Tensorflow implementations with same initializations and they compute slightly different gradients.
The models are fully-connected with tanh nonlinearities
PyTorch (1.5.0):
# x_train, y_train and t_train are numpy arrays with shapes (5,)

x_t = torch.from_numpy(x_train).float()
y_t = torch.from_numpy(y_train).float()
t_t = torch.from_numpy(t_train).float()
x_t.requires_grad = True
y_t.requires_grad = True
t_t.requires_grad = True

X = torch.stack([x_t, y_t, t_t], dim=1)
lb, _ = X.min(0)
ub, _ = X.max(0)
H = 2.0*(X - lb)/(ub - lb) - 1.0

psi_and_p = model(H)
psi = psi_and_p.select(dim=1, index=0)
p = psi_and_p.select(dim=1, index=1)

dummy = torch.ones(len(X), requires_grad=False)
u = torch.autograd.grad(psi, y_t, grad_outputs=dummy, create_graph=True, only_inputs=True)[0]
v = - torch.autograd.grad(psi, x_t, grad_outputs=dummy, create_graph=True, only_inputs=True)[0]

The outcomes are
psi (first column of the output of the NN):
tensor([ 0.0577,  0.0858, -0.1348, -0.0789, -0.0031], grad_fn=<SelectBackward>)

u (gradient of psi wrt y):
tensor([ 0.0489, -0.0578,  0.0303, -0.0616,  0.0401], grad_fn=<UnbindBackward>)

v (negative gradient of psi wrt x):
tensor([-0.0788,  0.1062,  0.1445, -0.0767, -0.0951], grad_fn=<NegBackward>)

Tensorflow (1.15.3):
# function that takes the input and returns the outputs of the network
def neural_net(self, X, weights, biases):
    num_layers = len(weights) + 1
    
    H = 2.0*(X - self.lb)/(self.ub - self.lb) - 1.0
    for l in range(0,num_layers-2):
        W = weights[l]
        b = biases[l]
        H = tf.tanh(tf.add(tf.matmul(H, W), b))
    W = weights[-1]
    b = biases[-1]
    Y = tf.add(tf.matmul(H, W), b)
    return Y

# function that computes the gradients
def net_NS(self, x, y, t):
    psi_and_p = self.neural_net(tf.concat([x,y,t], 1), self.weights, self.biases)
    psi = psi_and_p[:,0:1]
    p = psi_and_p[:,1:2]
    
    u = tf.gradients(psi, y)[0]
    v = -tf.gradients(psi, x)[0]  

The outcomes are
psi (first column of the output of the NN):
[[ 0.05774451]
 [ 0.08581609]
 [-0.13483115]
 [-0.07894167]
 [-0.00312973]]

u (gradient of psi wrt y):
[[0.048948  ]
 [0.05040099]
 [0.03029619]
 [0.04972227]
 [0.04007644]]

v (negative gradient of psi wrt x):
[[-0.0788489 ]
 [-0.07497928]
 [-0.07125   ]
 [-0.07668343]
 [-0.0951325 ]]

The output is exactly the same while the gradients are different just for some entries