I was following the excellent tutorials on pytorch’s website. I modified the code for An LSTM for Part-of-Speech Tagging to implement the exercise which requires to add another LSTM to get char level representation of words concatenate it with word embedding and train for learning tags.
My network code is as follows:
    class LSTMTaggerWithChar(nn.Module):
        def __init__(self, embedding_dim, hidden_dim, embedding_dim_char, hidden_dim_char, vocab_size, vocab_size_char, target_size):
            super(LSTMTaggerWithChar, self).__init__()
            self.hidden_dim = hidden_dim
            self.hidden_dim_char = hidden_dim_char
            
            self.embedding_char = nn.Embedding(vocab_size_char, embedding_dim_char)
            self.lstm_char = nn.LSTM(embedding_dim_char, hidden_dim_char)
            
            self.embedding = nn.Embedding(vocab_size, embedding_dim)
            self.lstm = nn.LSTM(embedding_dim, hidden_dim)
            self.hidden2tag = nn.Linear(hidden_dim, target_size)
            
            self.hidden = self.init_hidden()
            self.hidden_char = self.init_hidden_char()
            
        def init_hidden(self):
            return (autograd.Variable(torch.zeros(1, 1, self.hidden_dim)), autograd.Variable(torch.zeros(1, 1, self.hidden_dim)))
        
        def init_hidden_char(self):
            return (autograd.Variable(torch.zeros(1, 1, self.hidden_dim_char)), autograd.Variable(torch.zeros(1, 1, self.hidden_dim_char)))
            
        def forward(self, sentence, words):
            
            for ix, word in enumerate(sentence):
                chars = words[ix]
    #             self.hidden_char = self.init_hidden_char() Should I re-initialize hidden_char tensor here?
                char_embeds = self.embedding_char(chars).view(len(chars), 1, -1)
                lstm_char_out, self.hidden_char = self.lstm_char(char_embeds, self.hidden_char)
                
                char_rep = lstm_char_out[-1]

                embeds = self.embedding(word).view(1, 1, -1)
                lstm_out, self.hidden = self.lstm(embeds, self.hidden)
                
                tag_score = F.log_softmax(self.hidden2tag(lstm_out.view(1, -1)))
                
                if ix == 0:
                    tag_scores = tag_score
                else:
                    tag_scores = torch.cat((tag_scores, tag_score), 0)
            return tag_scores



Here even if I uncomment the line self.hidden_char = self.init_hidden_char() in forward function, I get the same results. I don’t understand why this should happen?


Also I think that character LSTM’s hidden state should be reset after it spits out representation for a word, assuming representation of next word is unrelated to previous word. But if I do that I am not clear how back-propagation will happen for the character LSTM on calling loss.backward() upon training?


Is it OK to have for loops in the forward function? How can it be avoided?