This is a classic result of using LSTM for time series analysis. LSTM is simply using the hidden state to relay back an earlier input without actually learning any patterns. In order to trick the LSTM into learning patterns, you can do the following

Reduce step size
Increase HiddenDim size

%matplotlib inline
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
from torch.autograd import Variable

class LSTMSimple(nn.Module):
    def __init__(self,inputDim,hiddenDim,batchSize,outputDim):
        super(LSTMSimple,self).__init__()
        torch.manual_seed(1)
        self.lstm=nn.LSTM(inputDim,hiddenDim,1).cuda()
        # Hidden state is a tuple of two states, so we will have to initialize two tuples
        self.state_h = torch.randn(1,batchSize,hiddenDim).cuda()
        self.state_c = torch.rand(1,batchSize,hiddenDim).cuda()
        self.linearModel=nn.Linear(hiddenDim,outputDim).cuda()
        
    def forward(self,inputs):
        # LSTM
        output, self.hidden = self.lstm(inputs, (self.state_h,self.state_c) )
        self.state_h=self.state_h.detach()
        self.state_c=self.state_c.detach()
        # LINEAR MODEL
        output=self.linearModel(output).cuda()
        return output

def lossCalc(x,y):
    return torch.sum(torch.add(x,-y))
    
# Model Object
batchSize=5
inputDim=1
outputDim=1
stepSize=5
hiddenDim=20
model=LSTMSimple(inputDim,hiddenDim,batchSize,outputDim).cuda()
loss = torch.nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.00001)

# Input Data
dataInput = np.random.randn(stepSize*batchSize,inputDim)
dataY=np.insert(dataInput[1:],len(dataInput)-2,0)
dataInput=Variable(torch.from_numpy(dataInput.reshape(stepSize,batchSize,inputDim).astype(np.float32))).cuda()
dataY=Variable(torch.from_numpy(dataY.reshape(stepSize,batchSize,inputDim).astype(np.float32))).cuda()
for epoch in range(10000):
    optimizer.zero_grad()
    dataOutput=model(dataInput).cuda()
    curLoss=loss(dataOutput.view(batchSize*stepSize,outputDim),dataY.view(batchSize*stepSize,outputDim))
    curLoss.backward()
    optimizer.step()
    if(epoch % 1000==0):
        print("For epoch {}, the loss is {}".format(epoch,curLoss))

plt.plot(dataInput.cpu().detach().numpy().reshape(-1),color="red")
plt.plot(dataOutput.cpu().detach().numpy().reshape(-1),color="orange")
plt.plot(dataY.cpu().detach().numpy().reshape(-1),color="green")
plt.figure()