Why isn't my backprop matching autograd?
Am attempting to implement backpropagation for a deep learning course but my backprop gradients don't seem to be matching the gradients you get from autograd. Here's the code:
Is my math incorrect somewhere or is something not acting the way you'd expect in python?
``` from collections import OrderedDict import torch import torch.nn as nn import torch.nn.functional as F import torch
class MLP:
def __init__(
self,
linear_1_in_features,
linear_1_out_features,
f_function,
linear_2_in_features,
linear_2_out_features,
g_function
):
"""
Args:
linear_1_in_features: the in features of first linear layer
linear_1_out_features: the out features of first linear layer
linear_2_in_features: the in features of second linear layer
linear_2_out_features: the out features of second linear layer
f_function: string for the f function: relu | sigmoid | identity
g_function: string for the g function: relu | sigmoid | identity
"""
self.f_function = f_function
self.g_function = g_function
self.parameters = dict(
W1 = torch.randn(linear_1_out_features, linear_1_in_features),
b1 = torch.randn(linear_1_out_features,1),
W2 = torch.randn(linear_2_out_features, linear_2_in_features),
b2 = torch.randn(linear_2_out_features,1),
)
self.grads = dict(
dJdW1 = torch.zeros(linear_1_out_features, linear_1_in_features),
dJdb1 = torch.zeros(linear_1_out_features,1),
dJdW2 = torch.zeros(linear_2_out_features, linear_2_in_features),
dJdb2 = torch.zeros(linear_2_out_features,1),
)
self.functions = dict(
relu = torch.relu, #torch.nn.functional.relu(),
sigmoid = torch.sigmoid, #torch.nn.functional.sigmoid(),
identity = self.identity_function #torch.identity #torch.nn.Identity()
)
self.cache = dict()
def identity_function(self, x):
return x
def forward(self, x):
"""
Args:
x: tensor shape (batch_size, linear_1_in_features)
"""
# TODO: Implement the forward function
self.x = x
self.xt = x.T
# print(self.parameters['W1'].shape, self.x.shape)
z1 = torch.matmul(self.parameters['W1'],x.T) + self.parameters['b1']
a1 = self.functions[self.f_function](z1)
# print(self.parameters['W2'].shape, a1.shape)
z2 = torch.matmul(self.parameters['W2'],a1) + self.parameters['b2']
a2 = self.functions[self.g_function](z2)
# a = activation_fn(z)
self.cache['z1']=z1
self.cache['a1']=a1
self.cache['z2']=z2
self.cache['a2']=a2
return a2
def backward(self, dJdy_hat):
"""
Args:
dJdy_hat: The gradient tensor of shape (batch_size, linear_2_out_features)
"""
#alpha = .01
#print('N samples:', len(dJdy_hat))
self.dJdy_hat = dJdy_hat
if self.g_function == 'relu':
dA2dZ2=torch.heaviside(self.cache['z2'], values = torch.tensor([0.]))
dA1dZ1=torch.heaviside(self.cache['z1'], values = torch.tensor([0.]))
elif self. 338