Isolating the issue is one of the key problems here, as the model being a sequence to sequence network is pretty big, and hence difficult to debug.
class Decoder(nn.Module):
    def __init__(self, emb_dim, persona_dim, hid_dim, trg_vocab_size, dropout):
        super().__init__() 
        self.output_dim = trg_vocab_size 
        self.word_embedding = nn.Embedding.from_pretrained(weights_matrix, freeze = True)
        self.rnn = nn.GRU(emb_dim + hid_dim, hid_dim)
        self.out = nn.Linear(emb_dim + hid_dim*2, trg_vocab_size)
        self.dropout = nn.Dropout(dropout)
    
    def forward(self, x, hidden, context):
        """
        ------------------------------------------------------
        Decoder for Seq2seq persona embedding creation 
        Variable    ----    Size
        x                   [1, batch size]
        embedded            [1, batch size, emb dim]
        
        context             [1, batch size, hid dim]
        emb_con             [1, batch size, hid dim + emb dim]
        
        hidden              [1, batch size, hid dim]
        output              [1, batch size, hid dim]

        output              [batch size, hid dim*2 + emb dim]
        prediction          [batch size, trg vocab size]
        ------------------------------------------------------
        """
        x = x.unsqueeze(0)

        embedded = self.dropout(self.word_embedding(x))
        emb_con = torch.cat((embedded, context), dim = 2)

        output, hidden = self.rnn(emb_con, hidden)

        output = torch.cat((embedded.squeeze(0), output.squeeze(0), context.squeeze(0)), dim = 1)
        prediction = self.out(output)

        return prediction, hidden

^the decoder part.
class Seq2Seq(nn.Module):
    def __init__(self,encoder, decoder, personas, hid_dim ,  device):
        super().__init__()
        self.encoder = encoder 
        self.decoder = decoder 
        self.device = device 
        self.fc = nn.Linear(hid_dim, personas+1)
    def forward(self, src,src_speaker,trg, trg_speaker, teacher_forcing_ratio  = 0.5):
        """
        ------------------------------------------------------
        End-to-end network for Seq2seq persona embedding creation 
        Variable    ----    Size    
        trg                 [batch size, sequence length]   
        src                 [batch size, sequence length]
        outputs             [sequence length, batch size, trg vocab size]
        hidden              [1, batch size, hid dim]
        input               [batch size]
        ------------------------------------------------------
        """
        batch_size = trg.shape[0]
        trg = trg.permute(1,0)
        max_len = trg.shape[0] 
        trg_vocab_size = self.decoder.output_dim 
        
        outputs = torch.zeros(max_len, batch_size, trg_vocab_size).to(self.device)
        
        context = self.encoder(src, src_speaker)
        
        hidden = context
        input = torch.LongTensor(np.ones(batch_size))
        
        for t in range(1,max_len):
            input = Variable(input).cuda()
            hidden = Variable(hidden).cuda()
            context = Variable(context).cuda()
            output, hidden = self.decoder(input, hidden, context)
            outputs[t] = output
            
            teacher_force = random.random() < teacher_forcing_ratio
            
            top1 = output.argmax(1) 
            
            input = trg[t] if teacher_force else top1
        
        persona_out = self.fc(context).squeeze(0)
        
        return outputs, persona_out

^ The main sequence to sequence network
model.encoder.persona_embedding.weight.detach().cpu()

^ I print this before and after training, and it is exactly the same in both the cases.