text_plain_output_1.png
from sklearn.ensemble import RandomForestRegressor

from sklearn.linear_model import LinearRegression

from sklearn.metrics import mean_squared_error

from sklearn.metrics import mean_squared_error, r2_score, make_scorer

from sklearn.model_selection import GridSearchCV

from sklearn.model_selection import RandomizedSearchCV

from sklearn.model_selection import learning_curve

from sklearn.model_selection import train_test_split

from sklearn.neighbors import KNeighborsRegressor

from sklearn.preprocessing import RobustScaler

from xgboost import XGBRegressor

import matplotlib.pyplot as plt

import matplotlib.pyplot as plt

import numpy as np

import numpy as np

import pandas as pd

import seaborn as sns

concat_california_array = np.concatenate((california.data, np.reshape(california.target, (california.target.shape[0], 1))), axis=1)
california_df = pd.DataFrame(concat_california_array, columns=california.feature_names + ['price'])

plt.colorbar()

california_df.isnull().sum()

# Our goal here is to extract the numeric columns so we can boxplot them in order to detect outliers.

# Numeric column extraction

numeric_columns = california_df.select_dtypes(include = ['float64', 'int']).columns
len_numeric_columns = len(numeric_columns)

# Boxplotting

fig = plt.figure(figsize = (15,10))

# Set number of columns you want to plot

n_cols = 3

n_plot_rows = len_numeric_columns//n_cols
n_plot_rows

for i, column in enumerate(numeric_columns):
    ax = fig.add_subplot(n_plot_rows, n_cols, i+1)
    sns.boxplot(y = california_df[column], orient = 'h', ax = ax)

fig.tight_layout()

X = california.data
y = california.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

rs = RobustScaler()
X_train_rs = rs.fit_transform(X_train)
X_test_rs = rs.transform(X_test)

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
lg = LinearRegression()
lg.fit(X_train, y_train)
lg_rs = LinearRegression()
lg_rs.fit(X_train_rs, y_train)
y_est = lg.predict(X_test)
y_est_rs = lg_rs.predict(X_test_rs)
MSE_tst = mean_squared_error(y_test, y_est)
R2_coeff = lg.score(X_test, y_test)
MSE_tst_rs = mean_squared_error(y_test, y_est_rs)
R2_coeff_rs = lg_rs.score(X_test_rs, y_test)
MSE_train = mean_squared_error(y_train, lg.predict(X_train))
R2_train_coeff = lg.score(X_train, y_train)

from sklearn.neighbors import KNeighborsRegressor
k_max = 20
rang_K = np.arange(1, k_max + 1)
tuned_parameters = [{'n_neighbors': rang_K}]
nfold = 5
neigh_CV = GridSearchCV(KNeighborsRegressor(), tuned_parameters, cv=nfold, scoring={'MSE': make_scorer(mean_squared_error), 'R2': make_scorer(r2_score)}, return_train_score=True, refit='R2', n_jobs=-1).fit(X_train_rs, y_train)
y_est = neigh_CV.predict(X_test_rs)
MSE_tst = mean_squared_error(y_test, y_est)
R2_coeff = neigh_CV.score(X_test_rs, y_test)
K_CV = neigh_CV.best_params_['n_neighbors']

fig = plt.figure(figsize = (15,10))

print("Cross validation results:")
cv_results = pd.DataFrame(neigh_CV.cv_results_)
accs = pd.DataFrame(columns=["Neighbors"])

# Mostramos los resultados
melted_accs = accs.assign(**{'Neighbors': pd.DataFrame(neigh_CV.cv_results_['params']).unstack().values,
                             "Training R2": cv_results.mean_train_R2,
                             "Validation R2": cv_results.mean_test_R2,
                             "Traning MSE": cv_results.mean_train_MSE,
                             "Validation MSE": cv_results.mean_test_MSE}) \
                            .melt('Neighbors', value_vars = ['Traning MSE', 'Validation MSE'], var_name="Type", value_name="MSE")
g = sns.lineplot(x="Neighbors", y="MSE", hue='Type', data=melted_accs)

from sklearn.ensemble import RandomForestRegressor
nfold = 5
param_grid = [{'n_estimators': [3, 10, 30, 100, 150], 'max_features': [2, 4, 6, 8]}, {'bootstrap': [False], 'n_estimators': [3, 10, 30, 100], 'max_features': [2, 3, 4]}]
grid_search = GridSearchCV(RandomForestRegressor(), param_grid, cv=nfold, scoring={'MSE': make_scorer(mean_squared_error), 'R2': make_scorer(r2_score)}, return_train_score=True, refit='R2', n_jobs=-1).fit(X_train_rs, y_train)

y_est = grid_search.predict(X_test_rs)
y_est_train = grid_search.predict(X_train_rs)
MSE_tst = mean_squared_error(y_test, y_est)
R2_coeff = grid_search.score(X_test_rs, y_test)
MSE_train = mean_squared_error(y_train, y_est_train)
R2_coeff_train = grid_search.score(X_train_rs, y_train)
RF_best_params = grid_search.best_params_

df_metrics = pd.concat( [pd.DataFrame(grid_search.cv_results_['params']),
            pd.DataFrame({'train_MSE': grid_search.cv_results_['mean_train_MSE']}),
            pd.DataFrame({'test_MSE': grid_search.cv_results_['mean_test_MSE']})
            ], axis = 1
            )
m = ((df_metrics.max_features == 3) & (df_metrics.bootstrap == False))
df_metrics.loc[:, ['n_estimators', 'test_MSE']]
#df_metrics = df_metrics.melt('n_estimators', value_vars = ['train_MSE', 'test_MSE'], var_name = 'Type')

fig = plt.figure(figsize = (15,10))

g = sns.lineplot(x="n_estimators", y="test_MSE", data=df_metrics, err_style = None)

# We use this function made by Sklearn (https://scikit-learn.org/stable/auto_examples/model_selection/plot_learning_curve.html)

# Graphics "Scalabilty of the model" and "Performance of the model" have been removed.

import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import learning_curve
from sklearn.model_selection import ShuffleSplit

def plot_learning_curve(estimator, title, X, y, axes=None, ylim=None, cv=None,
                        n_jobs=None, train_sizes=np.linspace(.1, 1.0, 5)):
    """
    Generate 3 plots: the test and training learning curve, the training
    samples vs fit times curve, the fit times vs score curve.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    title : string
        Title for the chart.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    axes : array of 3 axes, optional (default=None)
        Axes to use for plotting the curves.

    ylim : tuple, shape (ymin, ymax), optional
        Defines minimum and maximum yvalues plotted.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 5-fold cross-validation,
          - integer, to specify the number of folds.
          - :term:`CV splitter`,
          - An iterable yielding (train, test) splits as arrays of indices.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`StratifiedKFold` used. If the estimator is not a classifier
        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validators that can be used here.

    n_jobs : int or None, optional (default=None)
        Number of jobs to run in parallel.
        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
        for more details.

    train_sizes : array-like, shape (n_ticks,), dtype float or int
        Relative or absolute numbers of training examples that will be used to
        generate the learning curve. If the dtype is float, it is regarded as a
        fraction of the maximum size of the training set (that is determined
        by the selected validation method), i.e. it has to be within (0, 1].
        Otherwise it is interpreted as absolute sizes of the training sets.
        Note that for classification the number of samples usually have to
        be big enough to contain at least one sample from each class.
        (default: np.linspace(0.1, 1.0, 5))
    """
    if axes is None:
        _, axes = plt.subplots(1, 1, figsize=(15, 10))

    axes.set_title(title)
    if ylim is not None:
        axes.set_ylim(*ylim)
    axes.set_xlabel("Training examples")
    axes.set_ylabel("Score")

    train_sizes, train_scores, test_scores = \
        learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs,
                       train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    

    # Plot learning curve
    axes.grid()
    axes.fill_between(train_sizes, train_scores_mean - train_scores_std,
                         train_scores_mean + train_scores_std, alpha=0.1,
                         color="r")
    axes.fill_between(train_sizes, test_scores_mean - test_scores_std,
                         test_scores_mean + test_scores_std, alpha=0.1,
                         color="g")
    axes.plot(train_sizes, train_scores_mean, 'o-', color="r",
                 label="Training score")
    axes.plot(train_sizes, test_scores_mean, 'o-', color="g",
                 label="Cross-validation score")
    axes.legend(loc="best")

    return plt

params = {'n_estimators': [100, 150, 200, 300, 350], 'learning_rate': np.linspace(0.1, 1.0, 10), 'min_child_weight': [1, 5, 10], 'gamma': [0, 0.5, 1, 1.5, 2, 5], 'subsample': [0.6, 0.8, 1.0], 'colsample_bytree': [0.6, 0.8, 1.0], 'max_depth': np.arange(3, 11, 1)}

from sklearn.model_selection import RandomizedSearchCV
xgb_reg = XGBRegressor(silent=True)
xgb_random = RandomizedSearchCV(estimator=xgb_reg, param_distributions=params, n_iter=200, cv=5, verbose=0, scoring={'MSE': make_scorer(mean_squared_error), 'R2': make_scorer(r2_score)}, return_train_score=True, refit='R2', n_jobs=-1).fit(X_train_rs, y_train)