This is what i tried.  It’s also mostly gibberish, but perhaps I need to train it longer, because the loss is going down although now very slowly
class SimpleGRU(nn.Module):
    def __init__(self, vocab_size, emb_size, hid_size, batch_size, seq_len, n_layers=1):
        super(SimpleGRU, self).__init__()
        self.vocab_size = vocab_size
        self.emb_size = emb_size
        self.hid_size = hid_size
        self.n_layers = n_layers
        self.batch_size = batch_size
        self.seq_len = seq_len
        self.emb = nn.Embedding(vocab_size, emb_size)
        self.batchnorm = nn.BatchNorm1d(emb_size-1)
        self.gru = nn.GRU(emb_size, hid_size, batch_first=True)
        self.fc1 = nn.Linear(seq_len * hid_size, vocab_size)
        self.selu = nn.SELU()
        self.logsoftmax = nn.LogSoftmax()
    def forward(self, input, hidden):
        x = self.emb(input)
        x = self.batchnorm(x)
        x, hidden = self.gru(x, hidden)
        x = x.contiguous().view(self.batch_size, -1)
        x = self.selu(self.fc1(x))
        x = self.logsoftmax(x)
        return x, hidden