text_plain_output_1.png
import numpy as np

def params(num_neurons):
    W = [np.random.randn(y, x) for x, y in zip(num_neurons[:-1], num_neurons[1:])]
    b = [0.01 * np.random.randn(x, 1) for x in num_neurons[1:]]
    return (W, b)

def sigmoid(z):
    sig = 1.0 / (1.0 + np.exp(-z))
    return sig

def feedforward(X, W, b):
    n = W.shape[0]
    m = X.shape[0]
    activation = np.zeros([m, n])
    for i in range(n):
        a = sigmoid(np.dot(X, W[i, :]) + b[i])
        activation[:, i] = a
    return activation

def cost(yhat, y):
    err = -sum(y * np.log(yhat) + (1 - y) * np.log(1 - yhat)) / y.shape[0]
    return err

def one_hot(i):
    e = np.zeros(10)
    e[i] = 1.0
    return e

y_train = np.array([one_hot(y) for y in train_data[1]])

def gradient(yhat, y, x2):
    common_on = (yhat - y) * (yhat * (1 - yhat))
    gw_l = np.zeros([yhat.shape[1], x2.shape[1]])
    for i in range(yhat.shape[1]):
        gw = common_on[:, i] * x2.T
        gw_l[i, :] = sum(gw.T) / y.shape[0]
    gb = sum(common_on) / y.shape[0]
    return (gw_l, gb.reshape(10, 1))

def gradient_l2(yhat, y, a1, X, w2):
    commo = (yhat - y) * (yhat * (1 - yhat))
    gb_l = np.zeros([a1.shape[1], 1])
    gw_l = np.zeros([a1.shape[1], X.shape[0]])
    for i in range(a1.shape[1]):
        gd_b = np.dot(commo, w2[:, i]) * (a1[:, i] * (1 - a1[:, i]))
        gw = gd_b * X
        gw_l[i, :] = sum(gw.T) / y.shape[0]
        gb_l[i] = sum(gd_b) / y.shape[0]
    return (gw_l, gb_l)

def gradient_descent(y, X, wo, bo, wh, bh, lr, epoch):
    costs = []
    cost_batch = []
    indexes = []
    for i in range(epoch):
        v1 = feedforward(train_data[0], wh, bh)
        v2 = feedforward(v1, wo, bo)
        err = cost(v2, y_train)
        costs.append(err)
        gwh, gbh = gradient_l2(v2, y, v1, X, wo)
        gwo, gbo = gradient(v2, y, v1)
        wh = wh - lr * gwh
        bh = bh - lr * gbh
        wo = wo - lr * gwo
        bo = bo - lr * gbo
        weight_params = {'out_layer': [wo, bo], 'hid_layer': [wh, bh]}
    return (weight_params, costs)

v1 = feedforward(train_data[0], parameters['hid_layer'][0], parameters['hid_layer'][1])
v2 = feedforward(v1, parameters['out_layer'][0], parameters['out_layer'][1])

yhat = v2.tolist()
ineed = np.array([yhat[i].index(max(yhat[i])) for i in range(len(yhat))])
compaire = np.array([ineed, train_data[1]])
compaire = compaire.T
compaires = compaire[compaire[:, 0] == compaire[:, 1]]
compaires.shape