Hi, I’m trying to do some tutorial for myself with my recent research topic.
I have sequential images, classify the labels using LSTM.
trying to extract CNN features from the sequential images and put them into LSTM for classification, and facing some problem.
here is my code.
    class CNN_RNN(nn.Module):
        def __init__(self, input_size, hidden_size, num_layers, num_classes):
            super(CNN_RNN, self).__init__()
            self.hidden_size = hidden_size
            self.num_layers = num_layers
            self.conv1 = nn.Conv2d(3,6,5)
            self.pool = nn.MaxPool2d(2,2)
            self.conv2 = nn.Conv2d(6,16,5)
            
            self.relu = nn.ReLU()
            self.bfc1 = nn.Linear(53,120)
            
            self.lstm = nn.LSTM(120, hidden_size, num_layers, batch_first=True)
            self.fc = nn.Linear(hidden_size, num_classes)
            
        def forward(self, x):
            #CNN 
            x = self.conv1(x)
            x = F.relu(x)
            x = self.pool(x)
            x = self.conv2(x)
            x = F.relu(x)
            x = self.pool(x)

            x = self.bfc1(x)
            h0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))
            c0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))
            
            #forward propagate rnn
            out,_ = self.lstm(x, (h0, c0))
            
            #decode hidden state of last time step
            out = self.fc(out[:,-1, :])
            return out


    input_size = [244,244]
    hidden_size = 6
    output_size = len(loaded.classes)
    num_layers = 3

    model = CNN_RNN(input_size, hidden_size, num_layers, output_size)

    loss_function = nn.NLLLoss()
    optimizer = optim.SGD(model.parameters(), lr=0.1)

    # See what the scores are before training

    print trainloader
    dataiter = iter(trainloader)
    images, labels = dataiter.next()
    print images.size()

    inputs = (images[0][0], labels)
    print inputs
    print '-----------------------'
    tag_scores = model(Variable(images))
    print(tag_scores)


What I just simply want to see is it running, but it isn’t.
here is the error message.
    ---------------------------------------------------------------------------
    RuntimeError                              Traceback (most recent call last)
    <ipython-input-225-35a06775e1f4> in <module>()
         21 print inputs
         22 print '-----------------------'
    ---> 23 tag_scores = model(Variable(images))
         24 print(tag_scores)
         25 

    /usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
        222         for hook in self._forward_pre_hooks.values():
        223             hook(self, input)
    --> 224         result = self.forward(*input, **kwargs)
        225         for hook in self._forward_hooks.values():
        226             hook_result = hook(self, input, result)

    <ipython-input-224-d5df1538679a> in forward(self, x)
         39 
         40         #forward propagate rnn
    ---> 41         out,_ = self.lstm(x, (h0, c0))
         42 
         43         #decode hidden state of last time step

    /usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
        222         for hook in self._forward_pre_hooks.values():
        223             hook(self, input)
    --> 224         result = self.forward(*input, **kwargs)
        225         for hook in self._forward_hooks.values():
        226             hook_result = hook(self, input, result)

    /usr/local/lib/python2.7/dist-packages/torch/nn/modules/rnn.pyc in forward(self, input, hx)
        160             flat_weight=flat_weight
        161         )
    --> 162         output, hidden = func(input, self.all_weights, hx)
        163         if is_packed:
        164             output = PackedSequence(output, batch_sizes)

    /usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, *fargs, **fkwargs)
        349         else:
        350             func = AutogradRNN(*args, **kwargs)
    --> 351         return func(input, *fargs, **fkwargs)
        352 
        353     return forward

    /usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, weight, hidden)
        242             input = input.transpose(0, 1)
        243 
    --> 244         nexth, output = func(input, hidden, weight)
        245 
        246         if batch_first and batch_sizes is None:

    /usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, hidden, weight)
         82                 l = i * num_directions + j
         83 
    ---> 84                 hy, output = inner(input, hidden[l], weight[l])
         85                 next_hidden.append(hy)
         86                 all_output.append(output)

    /usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, hidden, weight)
        111         steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
        112         for i in steps:
    --> 113             hidden = inner(input[i], hidden, *weight)
        114             # hack to handle LSTM
        115             output.append(hidden[0] if isinstance(hidden, tuple) else hidden)

    /usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in LSTMCell(input, hidden, w_ih, w_hh, b_ih, b_hh)
         38     outgate = F.sigmoid(outgate)
         39 
    ---> 40     cy = (forgetgate * cx) + (ingate * cellgate)
         41     hy = outgate * F.tanh(cy)
         42 

    /usr/local/lib/python2.7/dist-packages/torch/autograd/variable.pyc in __mul__(self, other)
        827 
        828     def __mul__(self, other):
    --> 829         return self.mul(other)
        830     __rmul__ = __mul__
        831 

    /usr/local/lib/python2.7/dist-packages/torch/autograd/variable.pyc in mul(self, other)
        337     def mul(self, other):
        338         if isinstance(other, Variable):
    --> 339             return Mul.apply(self, other)
        340         else:
        341             assert not torch.is_tensor(other)

    /usr/local/lib/python2.7/dist-packages/torch/autograd/_functions/basic_ops.pyc in forward(ctx, a, b)
         46         ctx.b_size = b.size()
         47         ctx.save_for_backward(a, b)
    ---> 48         return a.mul(b)
         49 
         50     @staticmethod

    RuntimeError: inconsistent tensor size, expected r_ [1 x 14 x 24], t [1 x 14 x 24] and src [1 x 6] to have the same number of elements, but got 336, 336 and 6 elements respectively at /pytorch/torch/lib/TH/generic/THTensorMath.c:902

It seems like I had a mistake about tensor size, but [1*14*24] is equal to 336. So the error message is quite awkward for me (I’m new to python language)
I’m guessing I made plenty of errors while writing this code, but I don’t know where and how.
I’m waiting for your kind reply.