After searching, I found the solution to my question using nn.Parameter 62 as follows:
class EncoderRNN(nn.Module):
def __init__(self, input_size, hidden_size, n_layers=1):
    super(EncoderRNN, self).__init__()
    self.n_layers = n_layers
    self.hidden_size = hidden_size

    hidden0 = torch.zeros(1, 1, hidden_size)
    if use_cuda:
        hidden0 = hidden0.cuda()
    else:
        hidden0 = hidden0

    self.hidden0 = nn.Parameter(hidden0, requires_grad=True)
    self.embedding = nn.Linear(input_size, hidden_size)
    self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)

def forward(self, input, hidden):
    output = self.embedding(input).view((1, 1, -1))
    for i in range(self.n_layers):
        output = F.relu(output)
        output, hidden = self.gru(output, hidden)
    return output, hidden

Of course, we can make initialization a bit prettier.