text_plain_output_1.png
import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

data = pd.read_csv('/kaggle/input/refined-data/data_hard_refined.csv')
data_full = pd.read_csv('/kaggle/input/refined-data/data_filtered.csv')
objectColumns = list(data.dtypes[data.dtypes == np.object].index)
numericColumns = list(data.dtypes[data.dtypes != np.object].index)
objectColumns = list(data.dtypes[data.dtypes == np.object].index)
numericColumns = list(data.dtypes[data.dtypes != np.object].index)

data['Energy/Surface'] = data['SiteEnergyUse(kBtu)'] / data['PropertyGFATotal']
data['GHG/Surface'] = data['TotalGHGEmissions'] / data['PropertyGFATotal']
columns_to_boxplot = ['SiteEnergyUse(kBtu)', 'Energy/Surface', 'TotalGHGEmissions', 'GHG/Surface']
for column in columns_to_boxplot:
    n_bins = 12
    subset = data[column]
    subset.plot.box(vert=False)

for column in columns_to_boxplot:
    i = 0
    quartile_1 = data[column].quantile(0.25)
    quartile_3 = data[column].quantile(0.75)
    iqr = quartile_3 - quartile_1
    min_value = quartile_1 - 1.5 * iqr
    max_value = quartile_3 + 1.5 * iqr
    outliers = data[(data[column] > max_value) | (data[column] < min_value)]
    i += outliers.shape[0]

# analyse des variables quantitatives continues: pour les groupes de PrimaryPropertyType)
groups = data['PrimaryPropertyType'].unique()

for column in columns_to_boxplot:
    print('column: {}'.format(column))
    n_bins = 12
    values=[]
    assessment =  []
    subset = data[column]
    # var = variance empirique, std = ecart type empirique
    print('all datas: \n\tmean: {}\n\tmed: {}\n\tmod: {}\n\tvar: {}\n\tstd: {}\n\tskew: {}\n\tkur: {}'
          .format(subset.mean(),subset.median(), subset.mode()[0],subset.var(ddof=0),subset.std(ddof=0),subset.skew(),subset.kurtosis()))
    plt.hist(subset,n_bins,label='all datas', alpha=0.5)
    plt.show()
    subset.plot.box(vert=False)
    plt.title(column)
    plt.show()
    
    fig = plt.figure(figsize=(36,18))
    for group in  groups:
        subset = data[data.PrimaryPropertyType == group][column] # Création du sous-échantillon
        values.append(subset)
        assessment.append([group,subset.mean(),subset.median(), subset.mode()[0],subset.var(ddof=0),subset.std(ddof=0),subset.skew(),subset.kurtosis()])
        plt.hist(subset,n_bins,label=group, alpha=0.3) 
        
    plt.legend(loc='best')
    plt.title(column)
    plt.show() # Affiche l'histogramme
    fig = plt.figure(figsize=(36,18))
    plt.hist(values, n_bins, histtype='bar', label=groups)
    plt.legend(loc='best')
    plt.title(column)
    plt.show() # Affiche l'histogramme
    
    df_assessment = pd.DataFrame(assessment,columns=['group', 'moy', 'med', 'mod','var','std','skew','kur'])
    display(df_assessment)
            
    # https://towardsdatascience.com/create-and-customize-boxplots-with-pythons-matplotlib-to-get-lots-of-insights-from-your-data-d561c9883643 
    
    fig, ax = plt.subplots(figsize=(36,10))
    ax.boxplot(values, labels=groups)
    plt.show()
    
    print('='*120)

data_refined = data.copy()
columns_outliers_to_delete = ['Energy/Surface', 'GHG/Surface']
for column in columns_outliers_to_delete:
    i = 0
    for group in groups:
        j = 0
        subset = data_refined[data_refined.PrimaryPropertyType == group]
        quartile_1 = subset[column].quantile(0.25)
        quartile_3 = subset[column].quantile(0.75)
        iqr = quartile_3 - quartile_1
        min_value = quartile_1 - 1.5 * iqr
        max_value = quartile_3 + 1.5 * iqr
        outliers = subset[(subset[column] > max_value) | (subset[column] < min_value)]
        j += outliers.shape[0]
        i += j
        data_refined.drop(outliers.index.values, inplace=True)

data_refined.to_csv('data_without_outliers.csv', index=False)

data_refine = data.copy()
columns_outliers_to_delete == ['SiteEnergyUse(kBtu)', 'TotalGHGEmissions']
for column in columns_outliers_to_delete:
    i = 0
    quartile_1 = data_refine[column].quantile(0.25)
    quartile_3 = data_refine[column].quantile(0.75)
    iqr = quartile_3 - quartile_1
    min_value = quartile_1 - 1.5 * iqr
    max_value = quartile_3 + 1.5 * iqr
    outliers = data_refine[(data_refine[column] > max_value) | (data_refine[column] < min_value)]
    i += outliers.shape[0]
    data_refine.drop(outliers.index.values, inplace=True)

print(data.shape)
print(data_refine.shape)