@nitish I just updated my forward funcion above. The final forward is:
def forward(self, dict_index, features, prev_hidden, seq_sizes, original_index):
    i2e = self.embedding(dict_index)
    data = torch.cat((i2e, features), 2)

    packed = pack_padded_sequence(data, list(seq_sizes.data), batch_first=True)
    output, _ = self.rnn(packed, prev_hidden)

    output, _ = pad_packed_sequence(output, batch_first=True)
    # get the last time step for each sequence
    idx = (seq_sizes - 1).view(-1, 1).expand(output.size(0), output.size(2)).unsqueeze(1)
    decoded = output.gather(1, idx).squeeze()


    # restore the sorting
    odx = original_index.view(-1, 1).expand(args.batch_size, output.size(-1))
    decoded = decoded.gather(0, Variable(odx))
    return decoded

    return decoded