Hey all,
I am trying to implement a fully connected multilayer RNN using torch.nn.RNNCell.
I have implemented it, but it looks like it is not working.
Here is the code for reference:
class MultilayerRNN(nn.Module):
    def __init__(self, input_size, output_size, hidden_layers = [100, 100, 100]):
        super(NormalDeepRNN, self).__init__()
        assert len(hidden_layers) > 0

        self.hidden_layers = hidden_layers
        self.num_layers = len(hidden_layers)
        self.input_size = input_size

        self.embedding = nn.Embedding(self.input_size, self.hidden_layers[0])
        self.layer_out = nn.Linear(hidden_layers[-1], num_classes)
        
    def forward(self, input):
        # input.shape: [BATCH_SIZE, INPUT_SIZE], e.g. [16, 128]

        # After trnaspose,
        # input.shape: [INPUT_SIZE, BATCH_SIZE], e.g. [128, 16]
        input = input.t()

        # After embedding,
        # input.shape: [INPUT_SIZE, BATCH_SIZE, EMBEDDING_DIM], e.g. [128, 16, 100]
        input = self.embedding(input)

        for i, out_size in enumerate(self.hidden_layers):
            if i == 0:
               # For first hidden layer, input_size == embedding_dim
                in_size = input.size(2)
            else:
                in_size = self.hidden_layers[i-1]

            rnn = nn.RNNCell(in_size, out_size, 'tanh')
            hx = self.__init_hidden(input.size(1), in_size)

            input = self.__cell(rnn, input, hx)

        output = self.layer_out(input[-1])
        return output, input[-1]
            

    def __cell(self, cell, input, hx):
        inputs = input.unbind(0)
        outputs= []

        for i in range(len(inputs)):
            hx = cell(inputs[i], hx)
            outputs += [hx]

        return torch.stack(outputs)

    def __init_hidden(self, batch_size, hidden_size):
        hidden = torch.zeros(batch_size, hidden_size)
        return Variable(hidden)

It always gives accuracy around 50%, which is baseline considering I am trying binary classification.
The same dataset gives around 85% accuracy with standard RNN modules (e.g. nn.RNNBase or nn.RNN).
I am pretty sure there is something wrong with my forward method, but I am unable to point it out.
Any help is appreciated.
Thank you.
Best,
Harshil