Thank you Frank! Appreciated…
After setting weight decay parameter to 0, altering learning rate to 0.1 and adding momentum, I got similar results as yours which is pretty good.
The thing I still did not understand is that in pure python implementation, I used a learning rate of 0.001 and lambda value of 0.01 for L2 weight normalization. And no momentum applied. How come pure python implementation resulted in a much better performance than my first Pytorch model? I really don’t know.
By the way below is the pure python implementation code:
import numpy as np
import matplotlib.pyplot as plt
import sklearn.datasets
np.random.seed(3)
x,y = sklearn.datasets.make_moons(n_samples=200,noise=0.05)
input_neurons = 2
hidden_neurons = 5
output_neurons = 2
samples = x.shape[0]
learning_rate = 0.001
lambda_reg = 0.01
Define Initial Weights
def init_network(input_dim, hidden_dim, output_dim):
model = {}
# Xavier Initialization
W1 = np.random.randn(input_dim, hidden_dim) / np.sqrt(input_dim)
b1 = np.zeros((1, hidden_dim))
W2 = np.random.randn(hidden_dim, output_dim) / np.sqrt(hidden_dim)
b2 = np.zeros((1, output_dim))
model[‘W1’] = W1
model[‘b1’] = b1
model[‘W2’] = W2
model[‘b2’] = b2
return model
def retreive(model_dict):
W1 = model_dict[‘W1’]
b1 = model_dict[‘b1’]
W2 = model_dict[‘W2’]
b2 = model_dict[‘b2’]
return W1, b1, W2, b2
def forward(x, model_dict):
W1, b1, W2, b2 = retreive(model_dict)
z1 = x.dot(W1) + b1
a1 = np.tanh(z1)
z2 = a1.dot(W2) + b2
exp_scores = np.exp(z2)
softmax = exp_scores / np.sum(exp_scores, axis = 1, keepdims = True)
return z1, a1, softmax
def loss(softmax, y, model_dict):
W1, b1, W2, b2 = retreive(model_dict)
m = np.zeros(200)
for i,correct_index in enumerate(y):
predicted = softmax[i][correct_index]
m[i] = predicted
log_prob = -np.log(m)
loss = np.sum(log_prob)
reg_loss = lambda_reg / 2 * (np.sum(np.square(W1)) + np.sum(np.square(W2)))
loss+= reg_loss
return float(loss / y.shape[0])
def backpropagation(x, y, model_dict, epochs):
for i in range(epochs):
W1, b1, W2, b2 = retreive(model_dict)
z1, a1, probs = forward(x, model_dict)  # a1: (200,3), probs: (200,2)
delta3 = np.copy(probs)
delta3[range(x.shape[0]), y] -= 1  # (200,2)
dW2 = (a1.T).dot(delta3)  # (3,2)
db2 = np.sum(delta3, axis=0, keepdims=True)  # (1,2)
delta2 = delta3.dot(W2.T) * (1 - np.power(np.tanh(z1), 2))
dW1 = np.dot(x.T, delta2)
db1 = np.sum(delta2, axis=0)
# Add regularization terms
dW2 += lambda_reg * np.sum(W2)
dW1 += lambda_reg * np.sum(W1)
# Update Weights: W = W + (-lrgradient) = W - lrgradient
W1 += -learning_rate * dW1
b1 += -learning_rate * db1
W2 += -learning_rate * dW2
b2 += -learning_rate * db2
# Update the model dictionary
model_dict = {‘W1’: W1, ‘b1’: b1, ‘W2’: W2, ‘b2’: b2}
# Print the loss every 50 epochs
if i % 50 == 0:
prediction = np.argmax(probs, 1)
accuracy = (prediction == y).sum() / y.size
print(“Loss at epoch {} is: {:.3f}, accuracy is {:.3f}”.format(i, loss(probs, y, model_dict),accuracy))
return model_dict

def predict(model_dict, x):
W1, b1, W2, b2 = retreive(model_dict)
z1 = x.dot(W1) + b1
a1 = np.tanh(z1)
z2 = a1.dot(W2) + b2
exp_scores = np.exp(z2)
softmax = exp_scores / np.sum(exp_scores, axis = 1, keepdims = True)   # (200,2)
return np.argmax(softmax, axis = 1)    # (200,)
model_dict = init_network(input_dim = input_neurons , hidden_dim = hidden_neurons, output_dim = output_neurons)
model = backpropagation(x, y, model_dict, 1000)