Here is my network:
import torch.nn as nn
from torch.autograd import Variable

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super().__init__()
        self.hidden_size = hidden_size
        self.i2h = nn.Linear(input_size , hidden_size)
        self.h2o = nn.Linear(hidden_size, output_size)
        self.h2h = nn.Linear(hidden_size, hidden_size)
        self.Relu = nn.ReLU()
        self.softmax = nn.LogSoftmax(dim = 1)

    def forward(self, input, hidden):
        h = self.Relu(self.h2h(hidden)+  self.i2h(input))
        o = self.softmax(self.h2o(h))
        return o, h

    def init_hidden(self):
        return Variable(torch.zeros(1, self.hidden_size), requires_grad=True)

rnn = RNN(n_chars, 90, n_chars)
criterion = nn.L1Loss()
learning_rate = 0.05
optimizer = torch.optim.Adam(rnn.parameters(), lr = learning_rate)
hidden = rnn.init_hidden()
epochs = 5

rnn.cuda()
for epoch in range(epochs):
    for i in range(len(X)):
        for ele in X[i]:
            output, hidden = rnn(Variable(ele.t()).cuda(), hidden.cuda())
        loss = criterion(output, Variable(Y[i]).cuda())
        
        loss.backward(retain_graph=True)
        optimizer.zero_grad()
        optimizer.step()
        if (i%10 == 0):
            print(loss)

The loss I get is approximately constant at 4.513. Why is the loss not changing?