image_output_1.png
from statsmodels.graphics.gofplots import ProbPlot

import matplotlib.pyplot as plt

import pandas as pd

import seaborn as sns

import statsmodels.api as sm

df = pd.read_csv('../input/multipleChoiceResponses.csv', encoding='ISO-8859-1')

df = df[['CompensationAmount', 'Age']]

df['CompensationAmount'] = df['CompensationAmount'].str.replace('[^\\w\\s]', '')
df['CompensationAmount'].fillna(0, inplace=True)
df['Age'].fillna(0, inplace=True)
df['CompensationAmount'] = pd.to_numeric(df['CompensationAmount'])
df = df[df['CompensationAmount'] > 0]

y = df['CompensationAmount']
x = df['Age']
x = sm.add_constant(x)
model = sm.GLM(y, x, family=sm.families.Poisson()).fit()

# statsmodels Q-Q plot on model residuals
QQ = ProbPlot(model.resid_deviance)
fig = QQ.qqplot(alpha=0.5, markersize=5);

model_g = sm.OLS(y, x).fit()
model_leverage = model_g.get_influence().hat_matrix_diag
model_cooks = model_g.get_influence().cooks_distance[0]
plt.xlim(xmin=-0.005, xmax=0.02)

df = df[df['CompensationAmount'] <= 150000]

y = df['CompensationAmount']
x = df['Age']
x = sm.add_constant(x)
model = sm.GLM(y, x, family=sm.families.Poisson()).fit()
model_g = sm.OLS(y, x).fit()

# statsmodels Q-Q plot on model residuals
QQ = ProbPlot(model.resid_deviance)
fig = QQ.qqplot(alpha=0.5, markersize=5);

model_leverage = model_g.get_influence().hat_matrix_diag
model_cooks = model_g.get_influence().cooks_distance[0]
plt.xlim(xmin=-0.005, xmax=0.02)

df['reg_fit'] = model.fittedvalues
df.sort_values('Age', inplace=True)

sns.regplot(df['Age'], df['CompensationAmount'], fit_reg=False)
plt.plot(df['Age'], df['reg_fit'])