Hello, first of all I would like to say that i like PyTorch so far and eager to see what it do in the future.
I train a custom Module char-RNN because i want to save the last hidden state. but it seems that every step my memory (RAM) usage keep getting bigger and bigger. I don’t know where or what that caused memory leak.
Thanks in advance.
my model:
class CharRNN(torch.nn.Module):
    def __init__(...):
        ...
        self.hidden_state = None

    def forward(self, x):
        rnn_out, hidden = self.gru(x.view(1, 1, -1), self.hidden_state)
        logits = self.dense(rnn_out.view(1, -1))
        self.hidden_state = repackage_hidden(hidden)

        return logits

def repackage_hidden(h):
    """Wraps hidden states in new Variables, to detach them from their history."""
    if type(h) == Variable:
        return Variable(h.data)
    else:
        return tuple(repackage_hidden(v) for v in h)

my train loop:
# init model
char_RNN = CharRNN(n_feature, n_hidden, n_layer, n_feature, dropout)
loss_fn = torch.nn.CrossEntropyLoss()]
char_RNN.double()
char_RNN.cuda()
print char_RNN
optimizer = torch.optim.Adam(char_RNN.parameters(), lr=1e-4)


# init dataset
.....

losses = []

for i_epoch in range(n_epoch):
    rand_idx = range(len(dataset))
    random.shuffle(rand_idx)

    for i_minibatch in range(len(dataset)):
        x, y = dataset[rand_idx[i_minibatch]]
        x = Variable(torch.LongTensor(x), requires_grad=False).cuda()
        y = Variable(torch.LongTensor(y), requires_grad=False).cuda()
        char_RNN.zero_grad()

        loss = 0
        for i_char in range(len(x)):
            out = char_RNN(x[i_char])
            loss = loss + loss_fn(out, y[i_char])
        losses.append(loss.data[0] / float(len(x)))

        loss.backward()
        optimizer.step()

        if i_minibatch % 500 == 0:
            # print summary every 500 step
            summary = 'epoch {} batch {} avg. loss: {}'.format(i_epoch, i_minibatch, sum(losses) / float(500))
            losses = []