I’m trying to create a model that is able to learn any lower polynomial function. (Can this even be done?) So far I’ve got code that is able to learn the y=6x^2 + 2x - 4 quadratic equation.
I noticed that by providing [x, x^2] instead of [x] as the input the model performs much better. However, when I use [x, x^2, x^3] as the input the model doesn’t work anymore (huge test set loss). Why does providing x^3 to the model result in such a poor result?
Below is the code that I’m using. I’m quite new to Pytorch so general tips/critique is also appriciated. Thanks!
import torch
from torch import Tensor
from torch.nn import Linear, MSELoss, functional as F
from torch.autograd import Variable
import numpy as np

def our_function(x):
    # calculate the y value using the function 6x^2 + 2x - 4
    return 6 * x * x + 2 * x - 4

def data_generator(data_size=1000):
    x = np.random.randint(-1000, 1000, size=data_size)
    y = our_function(x)

    # Adding x^2 enables the model to find quadratic function much better.
    inputs = np.column_stack([x, x ** 2])
    # inputs = np.column_stack([x, x ** 2, x ** 3])
    labels = y.reshape(-1, 1)  # without the reshape we get wrong results

    return inputs, labels


class Net(torch.nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = Linear(2, 50)
        # self.fc1 = Linear(3, 50)
        self.fc2 = Linear(50, 50)
        self.fc3 = Linear(50, 1)

        self.criterion = MSELoss()
        self.optimizer = torch.optim.Adam(self.parameters(), lr=0.01)

    def forward(self, x):
        # Shoud i add a dropout?
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

    def train_model(self, epochs=1000, data_size=1000):
        x_train, y_train = data_generator(data_size)

        for epoch in range(epochs):
            y_pred = model(Variable(Tensor(x_train)))
            y_train = Variable(Tensor(y_train))
            loss = self.criterion(y_pred, y_train)
            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

            print(f"Epoch: {epoch} Loss: {loss / data_size:,.10f}")

model = Net()
model.train_model()