I guess the problem is caused by self.hidden. You should build a Variable of the hidden data and feed it to the network in every forward step. For example:
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
import torch.optim as optim

import numpy as np
import matplotlib.pyplot as plt

batch_size = 10
n_examples = 200
sequence_len = 15  # This is equavlent to time steps of the sequence in keras
sequence_size = 13
hidden_size = 30
class my_rnn(nn.Module):
    def __init__(self, input_size=2, hidden_size=20, num_layers=3, output_size=1,
                 batch_size=10):
        super(my_rnn, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.output_size = output_size
        self.batch_size = batch_size
        
        self.rnn = nn.RNN(input_size=self.input_size, hidden_size=self.hidden_size, 
                          num_layers=self.num_layers, batch_first=True)
        # The last layer is applied on the last output only of the RNN (not like
        # TimeDistributedDense in Keras)
        self.linear_layer = nn.Linear(self.hidden_size, self.output_size)
        
        self.hidden = Variable(torch.zeros((self.num_layers, self.batch_size, self.hidden_size)))
        
    def forward(self, input_sequence):
        out_rnn, self.hidden = self.rnn(input_sequence, self.hidden)
        in_linear = out_rnn[:, -1, :]
        final_output = self.linear_layer(in_linear)
        return final_output
    
    def init_hidden(self):
        self.hidden = Variable(torch.zeros((self.num_layers, self.batch_size, self.hidden_size)))
        
        
rnn = my_rnn(input_size=sequence_size, hidden_size=hidden_size, num_layers=3, batch_size=batch_size)
#input_data = Variable(torch.randn((batch_size, sequence_len, sequence_size)))
h0 = Variable(torch.zeros((3, batch_size, hidden_size)))
demo_target = Variable(torch.randn((batch_size, 1)))

loss_fn = nn.MSELoss()
optimizer = optim.SGD(rnn.parameters(), lr=0.001, momentum=0.9)
for i in range(10):
    input_data = Variable(torch.randn((batch_size, sequence_len, sequence_size)))
    rnn.init_hidden()  # init hidden Variable before every forward step
    output = rnn(input_data)
    #print output
    optimizer.zero_grad()
    loss = loss_fn(output, demo_target)
    loss.backward()
    #loss.backward(retain_variables=True)
    optimizer.step()
    print loss.data[0]

However, the better way is keep the hidden data and rebuild a Variable containing this data and then feed it to the forward function again. Like this:

  
      github.com
  
  
    Yugnaynehc/pytorch-fun/blob/master/word_language_model/main.py#L52 12

return min(1, clip / (totalnorm + 1e-6))




def train():
total_loss = 0
start = time.time()
hidden = model.init_hidden(batch_size)
for batch, i in enumerate(range(0, train_data.size(0)-1, bptt_len)):
    source, target = get_batch(train_data, i)
    model.zero_grad()
    output, hidden = model(source, hidden)
    hidden = repackage_hidden(hidden)
    loss = loss_fn(output, target)
    loss.backward()


    clip_lr = lr * clip_gradient(model, clip_coefficient)
    for p in model.parameters():
        p.data.sub_(clip_lr, p.grad.data)


    total_loss += loss.data[0]