I wrote some LSTM based code for language modeling:
 def forward(self, input, hidden):
         emb = self.encoder(input)
         h, c = hidden
         h.data.squeeze_(0)
         c.data.squeeze_(0)

         seq_len = input.size(0)
         batch_size = input.size(1)
         output_dim = h.size(1)

         output = [] 
         for i in range(seq_len):        
             h, c = self.rnncell(emb[i], (h, c))
             # self.hiddens: time * batch * nhid
             if i == 0:
                 self.hiddens = h.unsqueeze(0)
             else:
                 self.hiddens = torch.cat([self.hiddens, h.unsqueeze(0)])
             # h: batch * nhid
             #self.att = h.unsqueeze(0).expand_as(self.hiddens)

             self.hiddens = self.hiddens.view(-1, self.nhid)
             b = torch.mm(self.hiddens, self.U).view(-1, batch_size, 1)
             a = torch.mm(h, self.W).unsqueeze(0).expand_as(b)
             att = torch.tanh(a + b).view(-1, batch_size)
             att = self.softmax(att.t()).t()
             self.hiddens = self.hiddens.view(-1, batch_size, self.nhid)
             att = att.unsqueeze(2).expand_as(self.hiddens)
             output.append(torch.sum(att * self.hiddens, 0)) #hidden.data

         output = torch.cat(output)

         decoded = self.decoder(output.view(output.size(0)*output.size(1), output.size(2)))
         decoded = self.logsoftmax(decoded)
         output = decoded.view(output.size(0), output.size(1), decoded.size(1)) 
         return output, (h, c)

And I got error in backward():

RuntimeError: cuda runtime error (2) : out of memory at /data/users/soumith/miniconda2/conda-bld/pytorch-0.1.9_1487343590888/work/torch/lib/THC/generic/THCStorage.cu:66

Any ideas why it might happen?
The memory goes to 5800MB very quickly in the first 10 batches, and then it keeps running with this much memory occupied for another several hundred batches, and then it runs out of memory.