So here’s the model I’ve been able to make:
#!/usr/bin/env python

import torch
import torch.nn as nn
import torch.optim as opt
import numpy as np

from torch.autograd import Variable
import pandas as pd

torch.manual_seed(1)

num_epochs = 1
hidden_size = 100

def var(tensor):
    if torch.cuda.is_available():
        return Variable(tensor.cuda())
    else:
        return Variable(tensor)

class Model(nn.Module):
    def __init__(self, input_size=1, hidden_size=hidden_size, output_size=1,
                 num_layers=2):
        super(Model, self).__init__()
        self.num_layers = num_layers
        self.hidden_size = hidden_size

        self.lstm = nn.LSTM(input_size, hidden_size, num_layers=num_layers)
        self.linear = nn.Linear(hidden_size, hidden_size)
        self.relu = nn.ReLU()
        self.out = nn.Linear(hidden_size, output_size)

    def forward(self, x, hidden):
        lstm_out, hidden = self.lstm(x, hidden)
        linear_out = self.linear(lstm_out)
        relu_out = self.relu(linear_out)
        out = self.out(relu_out)

        return out, hidden

    def init_hidden(self):
        return (var(torch.zeros(self.num_layers, 1, self.hidden_size)),
                var(torch.zeros(self.num_layers, 1, self.hidden_size)))

def load_data(n):
    ix = np.arange(n)
    data = np.sin(2*np.pi*ix/float(n/2))

    return (torch.from_numpy(data[:-1]).float(),
            torch.from_numpy(data[1:]).float())

def train(ds):
    model = Model()
    # criterion = nn.MSELoss()
    criterion = nn.L1Loss()
    optimizer = opt.Adam(model.parameters(), lr=0.0002)

    if torch.cuda.is_available():
        print("Using GPU")
        model.cuda()

    lsum = 0.0
    lcount = 0.0

    hidden = model.init_hidden()

    losses = []

    for epoch in range(num_epochs):
        for i, (_x, _y) in enumerate(zip(ds[0], ds[1])):
            x = torch.zeros(1, 1, 1)
            y = torch.zeros(1, 1, 1)

            x[0][0][0] = _x
            y[0][0][0] = _y

            out, hidden = model(var(x), hidden)
            loss = criterion(out, var(y))
            loss.backward()
            hidden = (var(hidden[0].data), var(hidden[1].data))

            lsum += loss.data[0]
            lcount += 1
            losses.append(loss.data[0])

            optimizer.step()
            optimizer.zero_grad()

            if i % 1000 == 0:
                l = lsum / lcount
                print('Epoch: %i, Loss: %.7f' % (epoch, l))
                lsum = 0.0
                lcount = 0.0

if __name__ == "__main__":
    train(load_data(100000))

It does not seem to be able to learn even the simple sine function, here’s the plot of losses:

loss.png853×640 18.2 KB

The loss just seems to oscillate along with the input.
I’d really appreciate any hint on what is it I’m doing wrong.