text_plain_output_1.png
from sklearn import datasets

from sklearn.linear_model import LinearRegression

from sklearn.metrics import mean_squared_error

from sklearn.model_selection import train_test_split

import numpy as np

import pandas as pd

import plotly.express as px

class CustomLinearRegression:
    def __init__(self, learning_rate=0.1, n_iters=100, regularization=None, regularization_alpha=0.1):
        self.lr = learning_rate
        self.n_iters = n_iters
        self.weights = None
        self.bias = None
        self.regularization = regularization
        self.regularization_alpha = regularization_alpha
    def fit(self, X, y):
        n_samples, n_features = X.shape
        self.weights = np.zeros(n_features)
        self.bias = 0
        for _ in range(self.n_iters):
            y_predicted = self.predict(X)
            dw = (X.T * (y_predicted - y)).T.mean(axis=0)
            db = (y_predicted - y).mean(axis=0)
            if self.regularization == 'L1':
                if self.weights.any() != 0:
                    dw += self.regularization_alpha * (self.weights / np.absolute(self.weights))
            elif self.regularization == 'L2':
                dw += self.regularization_alpha * (2 * self.weights)
            self.weights -= self.lr * dw
            self.bias -= self.lr * db
    def predict(self, X):
        return X @ self.weights + self.bias

X, y = datasets.load_diabetes(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

df = pd.DataFrame(columns=['Number of iterations', 'MSE', 'Regularization', 'Average Weights'])
for n_iters in range(1, 51):
    model = CustomLinearRegression(n_iters=n_iters, regularization=None)
    model.fit(X_train, y_train)
    MSE = mean_squared_error(y_test, model.predict(X_test))
    average_weights = model.weights.mean()
    df = df.append({'Number of iterations': n_iters, 'MSE': MSE, 'Regularization': 'No', 'Average Weights': average_weights}, ignore_index=True)
    model = CustomLinearRegression(n_iters=n_iters, regularization='L1', regularization_alpha=1)
    model.fit(X_train, y_train)
    MSE = mean_squared_error(y_test, model.predict(X_test))
    average_weights = model.weights.mean()
    df = df.append({'Number of iterations': n_iters, 'MSE': MSE, 'Regularization': 'L1, alpha: 0.1', 'Average Weights': average_weights}, ignore_index=True)
    model = CustomLinearRegression(n_iters=n_iters, regularization='L2', regularization_alpha=1)
    model.fit(X_train, y_train)
    MSE = mean_squared_error(y_test, model.predict(X_test))
    average_weights = model.weights.mean()
    df = df.append({'Number of iterations': n_iters, 'MSE': MSE, 'Regularization': 'L2, alpha: 0.1', 'Average Weights': average_weights}, ignore_index=True)
    model = CustomLinearRegression(n_iters=n_iters, regularization='L2', regularization_alpha=11)
    model.fit(X_train, y_train)
    MSE = mean_squared_error(y_test, model.predict(X_test))
    average_weights = model.weights.mean()
    df = df.append({'Number of iterations': n_iters, 'MSE': MSE, 'Regularization': 'L2, alpha: 11', 'Average Weights': average_weights}, ignore_index=True)
average_weight = round(model.weights[1:].mean(), 2)
fig = px.line(df, x='Number of iterations', y='MSE', color='Regularization', title='Linear Regression models with regularization VS without regularization')

X, y = datasets.load_diabetes(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
model = CustomLinearRegression(n_iters=10000)
model.fit(X_train, y_train)
MSE = mean_squared_error(y_test, model.predict(X_test))
print('MSE for our model: {}'.format(MSE))
model = LinearRegression()
model.fit(X_train, y_train)
MSE = mean_squared_error(y_test, model.predict(X_test))
print('MSE for Sklearn model: {}'.format(MSE))