Hello, I made the following modification to my code. I initialized hn and cn once when creating the neural network. Then I passed their value to h0,c0 everytime a forward pass is performed. I have run the code several times but I am never able to get equally good results. On the other hand, the rolled lstm always provides the roughly similar results over the epochs.
class Net(nn.Module):
    def __init__(self, feature_dim, hidden_dim, batch_size):
        super(Net, self).__init__()
        
        # lstm architecture
        self.hidden_size=hidden_dim
        self.input_size=feature_dim  
        self.batch_size=batch_size
        self.num_layers=1
        
        # initialize hidden and cell
        self.hn = Variable(torch.randn(self.num_layers, self.batch_size, self.hidden_size))
        self.cn = Variable(torch.randn(self.num_layers, self.batch_size, self.hidden_size))
        
        # lstm
        self.lstm = nn.LSTM(feature_dim, hidden_size=self.hidden_size, num_layers=self.num_layers, batch_first=True)
        
        # fc layers
        self.fc1 = nn.Linear(hidden_dim, 2)    
                
    def forward(self, x, mode=False):
              
        h0 = self.hn
        c0 = self.cn
        
        # step through the sequence one timestep at a time        
        for (i,xt) in enumerate(torch.t(x)): 
                output, (h0,c0) = self.lstm(xt[:,None,:], (h0,c0))
         
        output = self.fc1(output[:,-1,:])
        return output


epoch 0 tr loss 54.92 te loss 17.42 tr err 158/316 te err 56/100
epoch 20 tr loss 49.03 te loss 17.09 tr err 85/316 te err 36/100
epoch 40 tr loss 34.17 te loss 16.82 tr err 70/316 te err 26/100
epoch 60 tr loss 30.62 te loss 24.70 tr err 57/316 te err 31/100
epoch 80 tr loss 26.15 te loss 26.07 tr err 41/316 te err 32/100
epoch 100 tr loss 22.72 te loss 39.18 tr err 41/316 te err 33/100
epoch 120 tr loss 21.97 te loss 44.00 tr err 49/316 te err 34/100
epoch 140 tr loss 18.72 te loss 46.30 tr err 29/316 te err 32/100
epoch 160 tr loss 18.30 te loss 47.71 tr err 33/316 te err 35/100
epoch 180 tr loss 13.59 te loss 51.09 tr err 22/316 te err 36/100
epoch 200 tr loss 10.30 te loss 72.76 tr err 11/316 te err 40/100
epoch 220 tr loss 11.10 te loss 71.32 tr err 23/316 te err 37/100
epoch 240 tr loss 7.85 te loss 71.26 tr err 8/316 te err 36/100
epoch 260 tr loss 8.96 te loss 60.27 tr err 21/316 te err 32/100
epoch 280 tr loss 6.97 te loss 63.88 tr err 10/316 te err 36/100
epoch 300 tr loss 10.76 te loss 65.86 tr err 8/316 te err 36/100
epoch 320 tr loss 4.51 te loss 62.41 tr err 9/316 te err 35/100
epoch 340 tr loss 4.13 te loss 60.39 tr err 8/316 te err 33/100
epoch 360 tr loss 15.63 te loss 65.40 tr err 16/316 te err 36/100
epoch 380 tr loss 14.48 te loss 73.36 tr err 81/316 te err 36/100
epoch 400 tr loss 9.04 te loss 62.02 tr err 5/316 te err 37/100
epoch 420 tr loss 3.63 te loss 55.84 tr err 16/316 te err 29/100
epoch 440 tr loss 1.13 te loss 74.18 tr err 0/316 te err 39/100
epoch 460 tr loss 0.07 te loss 101.76 tr err 0/316 te err 45/100
epoch 480 tr loss 0.02 te loss 112.72 tr err 0/316 te err 44/100