image_output_1.png
from scipy import stats

from scipy.stats import norm

import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

import warnings

train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')

total = train.isnull().sum().sort_values(ascending=False)
percent = (train.isnull().sum() / train.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])

to_drop = ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu']
train = train.drop(to_drop, axis=1)
test = test.drop(to_drop, axis=1)

#histogram and normal probability plot
from scipy.stats import norm
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats

import warnings
warnings.filterwarnings('ignore')

sns.distplot(train['SalePrice'], fit=norm);
fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)

quantitative = [f for f in train.columns if train.dtypes[f] != 'object']
quantitative.remove('SalePrice')
quantitative.remove('Id')
qualitative = [f for f in train.columns if train.dtypes[f] == 'object']
f = pd.melt(train, value_vars=quantitative)
g = sns.FacetGrid(f, col='variable', col_wrap=2, sharex=False, sharey=False)
g = g.map(sns.distplot, 'value')

def anova(frame):
    anv = pd.DataFrame()
    anv['feature'] = qualitative
    pvals = []
    for c in qualitative:
        samples = []
        for cls in frame[c].unique():
            s = frame[frame[c] == cls]['SalePrice'].values
            samples.append(s)
        pval = stats.f_oneway(*samples)[1]
        pvals.append(pval)
    anv['pval'] = pvals
    return anv.sort_values('pval')

a = anova(train)
a['disparity'] = np.log(1.0 / a['pval'].values)
x = plt.xticks(rotation=90)

def nova(frame):
    anv = pd.DataFrame()
    anv['feature'] = quantitative
    pvals = []
    for c in quantitative:
        samples = []
        for cls in frame[c].unique():
            s = frame[frame[c] == cls]['SalePrice'].values
            samples.append(s)
        pval = stats.f_oneway(*samples)[1]
        pvals.append(pval)
    anv['pval'] = pvals
    return anv.sort_values('pval')

a = nova(train)
a['disparity'] = np.log(1.0 / a['pval'].values)
x = plt.xticks(rotation=90)

quant_features = ['OverallQual', 'GarageCars', 'GarageArea', 'FullBath', 'YearBuilt', 'TotRmsAbvGrd', 'GarageYrBlt', 'GrLivArea', 'TotalBsmtSF', 'Fireplaces', 'MSSubClass', 'YearRemodAdd', 'LotFrontage', '1stFlrSF']

qual_features = ['Neighborhood', 'ExterQual', 'BsmtQual', 'GarageFinish', 'Foundation', 'GarageType', 'BsmtFinType1', 'HeatingQC', 'MasVnrType', 'BsmtExposure']
target = ['SalePrice']
need_id = ['Id']
train = train[quant_features + qual_features + target + need_id]
test = test[quant_features + qual_features + need_id]
train

sns.catplot(x='Neighborhood', y='SalePrice', kind='bar', data=train)
x = plt.xticks(rotation=90)