application_vnd.jupyter.stderr_output_1.png
from sklearn.metrics import fbeta_score,accuracy_score

from time import time

import numpy as np # linear algebra

from sklearn.metrics import fbeta_score, accuracy_score
from time import time
def train_predict(learner, sample_size, X_train, y_train, X_test, y_test):
    """
    inputs:
       - learner: the learning algorithm to be trained and predicted on
       - sample_size: the size of samples (number) to be drawn from training set
       - X_train: features training set
       - y_train: income training set
       - X_test: features testing set
       - y_test: income testing set
    """
    results = {}
    start = time()
    learner = learner.fit(X_train[:sample_size], y_train[:sample_size])
    end = time()
    results['train_time'] = end - start
    start = time()
    predictions_test = learner.predict(X_test)
    predictions_train = learner.predict(X_train[:300])
    end = time()
    results['pred_time'] = end - start
    results['acc_train'] = accuracy_score(y_train[:300], predictions_train)
    results['acc_test'] = accuracy_score(y_test, predictions_test)
    print('{} trained on {} samples.'.format(learner.__class__.__name__, sample_size))
    return results
def evaluate(results):
    """
    Visualization code to display results of various learners.
    inputs:
      - learners: a list of supervised learners
      - stats: a list of dictionaries of the statistic results from 'train_predict()'
      - accuracy: The score for the naive predictor
      - f1: The score for the naive predictor
    """
    fig, ax = pl.subplots(2, 3, figsize=(11, 7))
    bar_width = 0.23
    colors = ['#A00000', '#00A0A0', '#00A000', '#ffffff']
    for k, learner in enumerate(results.keys()):
        for j, metric in enumerate(['train_time', 'acc_train', 'f_train', 'pred_time', 'acc_test', 'f_test']):
            for i in np.arange(3):
                try:
                    ax[j // 3, j % 3].bar(i + k * bar_width, results[learner][i][metric], width=bar_width, color=colors[k])
                    ax[j // 3, j % 3].set_xticks([0.45, 1.45, 2.45])
                    ax[j // 3, j % 3].set_xticklabels(['1%', '10%', '100%'])
                    ax[j // 3, j % 3].set_xlabel('Training Set Size')
                    ax[j // 3, j % 3].set_xlim((-0.1, 3.0))
                except:
                    print('+++', learner, i, metric, '+++')
    ax[0, 0].set_ylabel('Time (in seconds)')
    ax[0, 1].set_ylabel('Accuracy Score')
    ax[0, 2].set_ylabel('F-score')
    ax[1, 0].set_ylabel('Time (in seconds)')
    ax[1, 1].set_ylabel('Accuracy Score')
    ax[1, 2].set_ylabel('F-score')
    ax[0, 0].set_title('Model Training')
    ax[0, 1].set_title('Accuracy Score on Training Subset')
    ax[0, 2].set_title('F-score on Training Subset')
    ax[1, 0].set_title('Model Predicting')
    ax[1, 1].set_title('Accuracy Score on Testing Set')
    ax[1, 2].set_title('F-score on Testing Set')
    ax[0, 1].axhline(y=accuracy, xmin=-0.1, xmax=3.0, linewidth=1, color='k', linestyle='dashed')
    ax[1, 1].axhline(y=accuracy, xmin=-0.1, xmax=3.0, linewidth=1, color='k', linestyle='dashed')
    ax[0, 2].axhline(y=f1, xmin=-0.1, xmax=3.0, linewidth=1, color='k', linestyle='dashed')
    ax[1, 2].axhline(y=f1, xmin=-0.1, xmax=3.0, linewidth=1, color='k', linestyle='dashed')
    ax[0, 1].set_ylim((0, 1))
    ax[0, 2].set_ylim((0, 1))
    ax[1, 1].set_ylim((0, 1))
    ax[1, 2].set_ylim((0, 1))
    patches = []
    for i, learner in enumerate(results.keys()):
        patches.append(mpatches.Patch(color=colors[i], label=learner))
    pl.legend(handles=patches, bbox_to_anchor=(-0.8, 2.53), loc='upper center', borderaxespad=0.0, ncol=3, fontsize='x-large')
    pl.suptitle('Performance Metrics for Three Supervised Learning Models', fontsize=16, y=1.1)
    pl.tight_layout()
    pl.show()