image_output_1.png
import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

sns.set_style('white')
plt.rc('axes', titlesize=13)
plt.rc('axes', labelsize=12)
plt.rc('xtick', labelsize=11)
plt.rc('ytick', labelsize=11)
plt.rc('legend', fontsize=11)
plt.rc('font', size=10)

path = '../input/mobile-games-ab-testing/cookie_cats.csv'
df = pd.read_csv(path)

df.shape

df.userid = df.userid.astype(str)

df.userid.dtypes

df.version.unique()

treatment = df[df['version'] == 'gate_40']

control = df[df['version'] == 'gate_30']

treatment.shape[0]

control.shape[0]

contr_d1 = '{0:.2f}%'.format(control.retention_1.mean() * 100)
trtm_d1 = '{0:.2f}%'.format(treatment.retention_1.mean() * 100)

contr_d7 = '{0:.2f}%'.format(control.retention_7.mean() * 100)
trtm_d7 = '{0:.2f}%'.format(treatment.retention_7.mean() * 100)

retention_1_30 = control.retention_1.mean()
retention_1_40 = treatment.retention_1.mean()
obs_diff_1 = retention_1_40 - retention_1_30

diffs_1 = []
for i in range(10000):
    boot_sample = df.sample(df.shape[0], replace=True)
    gate30_df_1 = boot_sample.query('version == "gate_30"')['retention_1'].mean()
    gate40_df_1 = boot_sample.query('version == "gate_40"')['retention_1'].mean()
    diffs_1.append(gate40_df_1 - gate30_df_1)

#plot the histogram of difference in 1 day retention
plt.figure(figsize=(16,5), tight_layout=True)
ax = sns.histplot(diffs_1, kde = True, kde_kws = {'bw_method':0.4})
ax.lines[0].set_color('black')
plt.ylim(0,600);
plt.title('Sampling Distribution for difference between control group\'s Day 1 Retention and treatment group\'s Day 1 Retention')
plt.ylabel('Frequency')
plt.xlabel('Difference');

null_vals_1 = np.random.normal(0, np.std(diffs_1), len(diffs_1))

p_val_1 = (null_vals_1 > obs_diff_1).mean()
p_form_1 = '{0:.2f}%'.format(p_val_1 * 100)

plt.figure(figsize=(16, 5), tight_layout=True)
ax = sns.histplot(null_vals_1, kde=True, kde_kws={'bw_method': 0.4})
ax.lines[0].set_color('black')
plt.axvline(obs_diff_1, color='r')
plt.text(0.008, 500, 'P-value = {}'.format(p_form_1), color='black', size=15)
plt.ylim(0, 600)
plt.title('Simulated Distribution under the Null Hypothesis')
plt.ylabel('Frequency')