Thanks for quick reply.
I though it was an issue too. First, init_hidden was called once per training and saved with rest of the model. Later, I changed it to every epoch , just like in seq2seq tutorial.
This is my training routine :
    print('Starting loop')
    losses = []
        
    for epoch in range(ep):
        ep_start = time.time()
        total_loss = 0
        enc_optimizer.zero_grad()
        dec_optimizer.zero_grad()
        enc_hidden = enc.iniHidden()
        
        for context, target in loader:
            for con in context[0]:
                enc_out, enc_hidden = enc(con, enc_hidden)
        
            dec_hidden = enc_hidden
            dec_input = torch.tensor(9)
            
            for iii in range(len(target)):
                dec_out, dec_hidden = dec(dec_input, dec_hidden)
                top_val, top_idx = dec_out.topk(1)
                dec_input = top_idx.squeeze().detach()  # .detach() - Remove from gradient calculation
                total_loss += criterion(dec_out, target[iii].view(-1), )
        total_loss.backward()
        losses.append(total_loss.item()/len(Y))
        
        enc_optimizer.step()
        dec_optimizer.step()
        
        dec_hidden = dec_hidden.detach()
        enc_hidden = enc_hidden.detach()
        
        ep_end = time.time()
        ep_del = ep_end - ep_start 
        (mi,sec) = divmod(ep_del*(ep - epoch - 1),60)
        if epoch  == 0 :
            print('Estimated time is {} minutes.'.format(int(mi)))
            
        print('Epoch',epoch + 1 ,'/', ep, ', Loss :',round(total_loss.item()/len(Y), 4))
        if (epoch + 1) % 5 == 0:
            SaveModel(epoch + 1, enc, dec, enc_optimizer, dec_optimizer, losses, str(time.localtime(time.time()).tm_hour) + '_'
                      + str(time.localtime(time.time()).tm_min) + '_' + str(time.localtime(time.time()).tm_sec))

Thank you!