Repost of this Stackoverflow question 1
I can’t seem to get broadcasting to work with autograd in pytorch 0.4.0! Below is a minimal code example that reproduces my problem. I would like to find a single value “bias”, which minimizes the loss over the dataset. The understand the error message as it wants to backpropagate a vector with 5 entries into a scalar, which it cannot figure out. However, this is the whole idea of broadcasting. The behavior I expected was that it would propagate the mean of the error back to the broadcasted scalar value (here bias).
Please advice.
Code:
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
from torch.utils.data import Dataset

print(torch.__version__)

class AddBias(torch.autograd.Function):
    @staticmethod
    def forward(ctx, input, bias):
        ctx.save_for_backward(input, bias)
        return input - bias
    @staticmethod
    def backward(ctx, grad_out):
        input, bias = ctx.saved_tensors
        grad_in = grad_bias = None
        len_grad = len(ctx.needs_input_grad)
        assert len_grad in {0, 1, 2}
        if ctx.needs_input_grad[0]: grad_in = grad_out
        if len_grad == 2: grad_bias = -1 * grad_out 
        return grad_in, grad_bias

class BiasModel(nn.Module):
    def __init__(self, size):
        super(BiasModel, self).__init__()
        self.bias_model = AddBias.apply
        self.bias = nn.Parameter(torch.tensor(0.5, dtype=torch.float, requires_grad=True))
    def forward(self, arr): return self.bias_model(arr[:], self.bias).unsqueeze(-1)

class MyData(Dataset):
    def __init__(self, data): self.data = data
    def __len__(self): return len(self.data)
    def __getitem__(self, i): 
        arr = torch.tensor(data[i], dtype=torch.float)
        target = torch.tensor(arr > 0.5, dtype=torch.float).unsqueeze(-1)
        return arr, target

m = 5
data = np.random.random((100, m))
model = BiasModel(m)
my_data = MyData(data)

loss_func = F.binary_cross_entropy_with_logits
with torch.no_grad():
    loss = 0.
    for arr, target in my_data: loss += loss_func(model(arr), target)
    print('loss before', loss / len(my_data))

optimizer = torch.optim.SGD(model.parameters(), lr=0.1)

loss_tot = 0.
for arr, target in my_data:
    model.zero_grad()
    loss = loss_func(model(arr), target)
    loss_tot += loss
    loss.backward()
    optimizer.step()

Output:
0.4.0
loss before tensor(0.5735)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-27bce65b553b> in <module>()
     56     loss_tot += loss
     57     loss.backward()
---> 58     optimizer.step()

~/miniconda3/envs/myproject/lib/python3.6/site-packages/torch/optim/sgd.py in step(self, closure)
    105                         d_p = buf
    106 
--> 107                 p.data.add_(-group['lr'], d_p)
    108 
    109         return loss

RuntimeError: expand(torch.FloatTensor{[5]}, size=[]): the number of sizes provided (0) must be greater or equal to the number of dimensions in the tensor (1)