Got it. I had to just fix the embedding layer. @ptrblck Thanks it’s solved now.
Last time my vocab was create by enumerating from 1. So if I just enumerate from 0 I can keep the same embedding otherwise if I had insisted on keeping enumeration from 1. then all I had to do was:
self.encoder = nn.Embedding(self.input_size+1, self.hidden_size) #[57,20] but still 56 dimensions as 0th index is still empty.

Solution :
class CharMaker(nn.Module):
    def __init__(self, input_size, hidden_size, output_size,n_layers=1):
        super(CharMaker,self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        print("Input & Output Size", input_size,output_size)
        print("Hidden Size ", hidden_size )
        
        self.encoder = nn.Embedding(self.input_size, self.hidden_size) #[56,20]
        
        self.rnn = nn.RNN(self.hidden_size,self.hidden_size, num_layers=1,batch_first=True) #[20,20]
        self.linear = nn.Linear(self.hidden_size, self.output_size) #[20,56]
        
        
    def forward(self, inputs, hidden):
        batch_size = inputs.size(0)
        
        if hidden == None:
            hidden = torch.zeros(1,batch_size,self.hidden_size)
        #print("Original Input : ",inputs.shape)
        encoded = self.encoder(inputs)
        #print("Encoded Input : ",encoded.shape)
        output, hidden = self.rnn(encoded, hidden)
        output = self.linear(output)
        
        return output,hidden