text_plain_output_1.png
from scipy import stats

from scipy.stats import norm, skew

import matplotlib.pyplot as plt

import numpy as np

import os

import pandas as pd

import seaborn as sns

import warnings

import os
import pandas as pd
import numpy as np
import seaborn as sns
color = sns.color_palette()
sns.set_style('darkgrid')
import math
import matplotlib.pyplot as plt
from scipy.stats import skew
import warnings
def ignore_warn(*args, **kwargs):
    pass
warnings.warn = ignore_warn
from scipy import stats
from scipy.stats import norm, skew

training_dataframe = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
testing_dataframe = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')

fig, ax = plt.subplots()
ax.scatter(x = training_dataframe['GrLivArea'], y = training_dataframe['SalePrice'])
plt.ylabel('SalePrice', fontsize=13)
plt.xlabel('GrLivArea', fontsize=13)
plt.show()

#Deleting outliers
training_dataframe= training_dataframe.drop(training_dataframe[(training_dataframe['GrLivArea']>4000) & (training_dataframe['SalePrice']<300000)].index)

#Check the scatterplot again
fig, ax = plt.subplots()
ax.scatter(training_dataframe['GrLivArea'], training_dataframe['SalePrice'])
plt.ylabel('SalePrice', fontsize=13)
plt.xlabel('GrLivArea', fontsize=13)
plt.show()

#Plotting histogram
sns.distplot(training_dataframe['SalePrice'] , fit=norm);

# Fitted Parameters used by function
(mu, sigma) = norm.fit(training_dataframe['SalePrice'])
print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))

#Plotting the distribution
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
            loc='best')
plt.ylabel('Frequency')
plt.title('SalePrice distribution')

#Plotting QQ-plot
fig = plt.figure()
res = stats.probplot(training_dataframe['SalePrice'], plot=plt)
plt.show()

#using log1p which  applies log(1+x) to all elements of the column
training_dataframe["SalePrice"] = np.log1p(training_dataframe["SalePrice"])

#Checking the new distribution
sns.distplot(training_dataframe['SalePrice'] , fit=norm);

# Geting fitted parameters used by the function
(mu, sigma) = norm.fit(training_dataframe['SalePrice'])
print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))

#Plotting Distribution
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
            loc='best')
plt.ylabel('Frequency')
plt.title('SalePrice distribution')

#Get also the QQ-plot
fig = plt.figure()
res = stats.probplot(training_dataframe['SalePrice'], plot=plt)
plt.show()

corealtion_matrix = training_dataframe.corr()
best_cor_feature = corealtion_matrix.index[abs(corealtion_matrix['SalePrice']) > 0.5]
best_cor_feature
def plotColor(*args):
    a = 3
    b = int(len(args) / a) + 1
    c = 1
    for i in args:
        plt.axis('off')
        plt.text(0, 0.04, i, color='k', fontsize=11)
        plt.hlines(0, 0, 10, color=i, linestyles='solid', linewidth=25)
        c = c + 1
    plt.tight_layout()
    return

ntrain = training_dataframe.shape[0]
ntest = testing_dataframe.shape[0]
y_train = training_dataframe.SalePrice.values
print('y_train shape is : {}'.format(y_train.shape))
all_data = pd.concat((training_dataframe, testing_dataframe)).reset_index(drop=True)
all_data.drop(['SalePrice'], axis=1, inplace=True)
print('all_data size is : {}'.format(all_data.shape))
total = all_data.isnull().sum().sort_values(ascending=False)
percent = (all_data.isnull().sum() / all_data.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.head(30)