Understanding how torch.nn.Module works
Essentially, I want to reproduce the results I get when I do it “manually:” <SCODE>from torch.autograd import Variable
import torch


x = Variable(torch.Tensor([[1.0, 1.0], 
                           [1.0, 2.1], 
                           [1.0, 3.6], 
                           [1.0, 4.2], 
                           [1.0, 6.0], 
                           [1.0, 7.0]]))
y = Variable(torch.Tensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0]))
weights = Variable(torch.zeros(2, 1), requires_grad=True)


for i in range(5000):

    net_input = x.mm(weights)
    loss = torch.mean((net_input - y)**2)
    loss.backward()
    weights.data.add_(-0.0001 * weights.grad.data)
    
    if loss.data[0] < 1e-3:
        break

print('n_iter', i)
print(loss.data[0])
<ECODE> Output: <SCODE>n_iter 1188
0.0004487129335757345
<ECODE> Now, running the following <SCODE>import torch.nn.functional as F

class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.weights = Variable(torch.zeros(2, 1), 
                                requires_grad=True)
    
    def forward(self, x):
        net_input = x.mm(self.weights)
        return net_input
        
model = Model()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
<ECODE> raises an error: <SCODE>    ---------------------------------------------------------------------------
    IndexError                                Traceback (most recent call last)
    <ipython-input-258-3bcb3a8408d2> in <module>()
         15 model = Model()
         16 criterion = torch.nn.MSELoss()
    ---> 17 optimizer = torch.optim.SGD(model.parameters(), lr=0.001)

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/optim/sgd.py in __init__(self, params, lr, momentum, dampening, weight_decay)
         24         defaults = dict(lr=lr, momentum=momentum, dampening=dampening,
         25                         weight_decay=weight_decay)
    ---> 26         super(SGD, self).__init__(params, defaults)
         27 
         28     def step(self, closure=None):

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/optim/optimizer.py in __init__(self, params, defaults)
         25         self.state = defaultdict(dict)
         26         self.param_groups = list(params)
    ---> 27         if not isinstance(self.param_groups[0], dict):
         28             self.param_groups = [{'params': self.param_groups}]
         29 

    IndexError: list index out of range
<ECODE> I tried that: <SCODE>class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.weights = Variable(torch.zeros(2, 1), 
                        requires_grad=True)
        self.fc = torch.nn.Linear(2, 1)
        
    def forward(self, x):
        return x
        
model = Model()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.0001)

for i in range(5000):
    optimizer.zero_grad()
    outputs = model(x)
    
    loss = criterion(outputs, y)
    loss.backward()        

    optimizer.step()
    
print(loss.data[0])
<ECODE> but now, I am getting an error about the dimensions of the input: <SCODE>    ---------------------------------------------------------------------------
    RuntimeError                              Traceback (most recent call last)
    <ipython-input-259-c6bb483f3953> in <module>()
         28     outputs = model(x)
         29 
    ---> 30     loss = criterion(outputs, y)
         31     loss.backward()
         32 

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
        208 
        209     def __call__(self, *input, **kwargs):
    --> 210         result = self.forward(*input, **kwargs)
        211         for hook in self._forward_hooks.values():
        212             hook_result = hook(self, input, result)

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
         21         _assert_no_grad(target)
         22         backend_fn = getattr(self._backend, type(self).__name__)
    ---> 23         return backend_fn(self.size_average)(input, target)
         24 
         25 

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/nn/_functions/thnn/auto.py in forward(self, input, target)
         39         output = input.new(1)
         40         getattr(self._backend, update_output.name)(self._backend.library_state, input, target,
    ---> 41             output, *self.additional_args)
         42         return output
         43 

    RuntimeError: input and target have different number of elements: input[6 x 2] has 12 elements, while target[6] has 6 elements at /Users/soumith/anaconda/conda-bld/pytorch-0.1.6_1484801351127/work/torch/lib/THNN/generic/MSECriterion.c:12
<ECODE>
rasbt
Both your examples have small errors inside: You have a bug in forward - your model always just returns the input. Try this: <SCODE>class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.fc = torch.nn.Linear(2, 1)
        
    def forward(self, x):
        return self.fc(x) # it was just x there
<ECODE>
apaszke
rasbt
Huh, I think we’re missing the docs on Parameters, I’ll note that and make sure it’s added soon. Sorry for the confusion.
apaszke
Hi, thanks for clarifying the difference between Variable and Parameter in this thread. It might be slightly off the topic, but I have a question about the codes used in here. The very first code written by rasbt, which implemented OLS regression manually using only Variable, looks perfectly fine to me, and I got exactly same output. However, the loss is not monotonically decreasing for some unknown reason. Am I missing something? By the way, by replaced Variable used in the second code by rasbt to Parameter, the loss decreases monotonically as usual.
tanemaki
Yeah, same issue here. The first example has some issues with the minimization: <SCODE>from torch.autograd import Variable
import torch


x = Variable(torch.Tensor([[1.0, 1.0], 
                           [1.0, 2.1], 
                           [1.0, 3.6], 
                           [1.0, 4.2], 
                           [1.0, 6.0], 
                           [1.0, 7.0]]))
y = Variable(torch.Tensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0]))
weights = Variable(torch.zeros(2, 1), requires_grad=True)

loss1 = []

for i in range(5000):

    net_input = x.mm(weights)
    loss = torch.mean((net_input - y)**2)
    loss.backward()
    weights.data.add_(-0.0001 * weights.grad.data)
    
    loss1.append(loss.data[0])

print('n_iter', i)
print(loss.data[0])

plt.plot(range(5000), loss1)
<ECODE> The 2nd example works well, though: <SCODE>import torch.nn.functional as F
from torch.nn import Parameter

class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.weights = Parameter(torch.zeros(2, 1), 
                                 requires_grad=True)
    
    def forward(self, x):
        net_input = x.mm(self.weights)
        return net_input
        
model = Model()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
loss2 = []

for i in range(5000):
    optimizer.zero_grad()
    outputs = model(x)
    
    loss = criterion(outputs, y)
    loss2.append(loss.data[0])
    loss.backward()        

    optimizer.step()
    
plt.plot(range(5000), loss2)
<ECODE> After standardizing ex1, and lowering the learning rate further, it would somewhat work though: <SCODE>from torch.autograd import Variable
import torch


x = torch.Tensor([[1.0, 1.0], 
                  [1.0, 2.1], 
                  [1.0, 3.6], 
                  [1.0, 4.2], 
                  [1.0, 6.0], 
                  [1.0, 7.0]])

x = (x - x.mean()) / x.max()
x = Variable(x)

y = torch.Tensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0])
y = (y - y.mean()) / y.max()
y = Variable(y)

weights = Variable(torch.zeros(2, 1), requires_grad=True)

loss1 = []

for i in range(5000):

    net_input = x.mm(weights)
    loss = torch.mean((net_input - y)**2)
    loss.backward()
    weights.data.add_(-0.00000001 * weights.grad.data)
    
    loss1.append(loss.data[0])

print('n_iter', i)
print(loss.data[0])

plt.plot(range(5000), loss1)
<ECODE>
rasbt
apaszke
rasbt
Good questions - Ive been trying to grok it too!
Kalamaya
tanemaki
Atcold
Regarding the symmetry: Yeah, in this case zero weights wouldn’t matter like you said
rasbt
Hmm, well why do you have to zero them out? You are just computing new gradients, so shouldn’t the old ones just be over-written?
Kalamaya
No, they’re accumulated.
apaszke
hmishfaq
Yes, that’s what I meant.
apaszke
Alex_Choy
its hard to follow what ur question is cuz ur title is vague and question details is too long.
Brando_Miranda
Hi, I would like to come back to this example and understand why there is a difference between two loss functions: loss1 = torch.mean((y - y_pred)**2.0) 
loss2 = mse_loss(y_pred, y), where mse_loss = nn.MSELoss()
 Here is my complete code: <SCODE>x = Variable(torch.FloatTensor([[1.0, 1.0], 
                                 [1.0, 2.1], 
                                 [1.0, 3.6], 
                                 [1.0, 4.2], 
                                 [1.0, 6.0], 
                                 [1.0, 7.0]]))
y = Variable(torch.FloatTensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0]))

mse_loss = nn.MSELoss()
weights = Variable(torch.zeros((2, 1)).float(), requires_grad=True)

n = 500
loss1 = []
loss2 = []
weights_grads_history = np.zeros((n, 2))
weights_history = np.zeros((n, 2))
learning_rate = 0.0001

for i in range(n):

    y_pred = x.mm(weights)
    loss = torch.mean((y - y_pred)**2.0)
    loss_ = mse_loss(y_pred, y)    
    loss1.append(loss.data[0])
    loss2.append(loss_.data[0])
    # Compute gradients
    loss.backward()
    
    # Update parameters
    weights_grads_history[i, :] = weights.grad.data.numpy()[:, 0]
    weights_history[i, :] = weights.data.numpy()[:, 0]
    weights.data.sub_(weights.grad.data * learning_rate)

    # You need to clear the existing gradients though, else gradients will be accumulated to existing gradients         
    weights.grad.data.zero_()

    
print("n_iter", i)
print(loss1[-1], loss2[-1])
print("y_pred: ", y_pred)
print("Weights : ", weights)

plt.figure(figsize=(12, 4))
plt.subplot(131)
plt.plot(range(n), loss1, label='loss1')
plt.plot(range(n), loss2, label='loss2')
_ = plt.legend()
plt.subplot(132)
plt.plot(range(n), weights_grads_history[:, 0], label='W_grad_1')
plt.plot(range(n), weights_grads_history[:, 1], label='W_grad_2')
plt.legend()
plt.subplot(133)
plt.plot(range(n), weights_history[:, 0], label='W_1')
plt.plot(range(n), weights_history[:, 1], label='W_2')
_ = plt.legend()
<ECODE> Thanks
vfdev-5
Hi, Your code should work when you change the following line: <SCODE>loss = torch.mean((y - y_pred.view(-1))**2.0)<ECODE>
ptrblck