Hi Ömer!



 jarvico:

I decided to build the same model using Pytorch. However, I got very poor results in Pytorch.


You don’t show your scikit-learn code nor your pytorch results, so its
hard to know the the difference might lie.
However, I don’t see anything wrong with your code (other than it being
garbled by the forum because you didn’t enclose it in a triple-backtick
code block).
You code seems to run fine for me, and gives good results if I increase
the learning rate, get rid of weight-decay (l2 regularization), and add
“momentum” to the SGD optimizer.
Note, I am using pytorch version 0.3.0, so I had to wrap the Tensors
in autograd.Variables.
Here is your code, as slightly modified by me:
import torch
import torch.nn as nn
import torch.nn.functional as F
torch.__version__

import numpy as np
import sklearn.datasets

torch.manual_seed (2020)
np.random.seed (2020)

x,y = sklearn.datasets.make_moons(n_samples=200,noise=0.05)

x = torch.autograd.Variable (torch.FloatTensor(x))   # using pytorch version 0.3.0
y = torch.autograd.Variable (torch.LongTensor(y))    # using pytorch version 0.3.0

input_neurons = 2
hidden_neurons = 5
output_neurons = 2
# learning_rate = 0.001
learning_rate = 0.1   # increase learning rate
# lambda_reg = 0.01
lambda_reg = 0.00     # turn off weight decay

momentum = 0.9        # add momentum to SGD

class FeedForward(torch.nn.Module):
    def __init__(self, input_neurons, hidden_neurons, output_neurons):
        super(FeedForward,self).__init__()
        self.hidden = nn.Linear(input_neurons, hidden_neurons)
        self.out = nn.Linear(hidden_neurons,output_neurons)
    def forward(self, x):
        x = self.hidden(x)
        x = F.tanh(x)
        x = self.out(x)
        return x

def init_weights(m):
    if type(m) == nn.Linear:
        torch.nn.init.xavier_uniform(m.weight)
        # m.bias.data.fill_(0.01)

network = FeedForward(input_neurons = input_neurons, hidden_neurons = hidden_neurons, output_neurons = output_neurons)
network.apply(init_weights)

# optimizer = torch.optim.SGD(network.parameters(), lr = learning_rate,weight_decay=lambda_reg)
optimizer = torch.optim.SGD(network.parameters(), lr = learning_rate, weight_decay = lambda_reg, momentum = momentum)
loss_function = torch.nn.CrossEntropyLoss()

for epoch in range(1000):
    out = network(x)
    loss = loss_function(out, y)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    if epoch % 50 == 0:
        max_value, prediction = torch.max(out, 1)
        predicted_y = prediction.data.numpy()
        target_y = y.data.numpy()
        accuracy = (predicted_y == target_y).sum() / target_y.size
        print('Accuracy = {:.3f} Loss = {:.6f}'.format(accuracy,loss.data[0]))


Here is the output, showing rapid convergence to 100% accuracy:
Accuracy = 0.500 Loss = 0.761427
Accuracy = 0.885 Loss = 0.255683
Accuracy = 0.885 Loss = 0.241214
Accuracy = 0.905 Loss = 0.218220
Accuracy = 0.955 Loss = 0.119875
Accuracy = 0.990 Loss = 0.048679
Accuracy = 1.000 Loss = 0.027161
Accuracy = 1.000 Loss = 0.018272
Accuracy = 1.000 Loss = 0.013567
Accuracy = 1.000 Loss = 0.010696
Accuracy = 1.000 Loss = 0.008776
Accuracy = 1.000 Loss = 0.007410
Accuracy = 1.000 Loss = 0.006392
Accuracy = 1.000 Loss = 0.005607
Accuracy = 1.000 Loss = 0.004983
Accuracy = 1.000 Loss = 0.004478
Accuracy = 1.000 Loss = 0.004060
Accuracy = 1.000 Loss = 0.003710
Accuracy = 1.000 Loss = 0.003412
Accuracy = 1.000 Loss = 0.003156

Good luck.
K. Frank