I don’t think I have inited the hidden states in between forward passes.
    optimizer = optim.LBFGS(seq.parameters(), lr=lr)
    pkle_file = result_file_path + "tr_loss"
    for epoch in range(num_epochs):
        print('EPOCH: ', epoch)
        tr_loss = None

        def closure():
            global tr_loss
            optimizer.zero_grad()
            out = seq(iput)
            l_train = criteria(out, target)
            tr_loss = l_train.item()
            print('loss:', tr_loss)
            with open(pkle_file, 'wb') as file:
                pickle.dump(tr_loss, file)
            l_train.backward()
            return l_train

        optimizer.step(closure)

Let me try that and revert back.