from scipy.stats import mannwhitneyu

from statsmodels.sandbox.stats.multicomp import multipletests

from statsmodels.stats.weightstats import zconfint

import pandas as pd

posts = pd.read_csv('../input/post.csv', parse_dates=['timeStamp'])

comments = pd.read_csv('../input/comment.csv')

com_count = comments.groupby('pid').count()['cid']
data = posts.join(com_count, on='pid', rsuffix='c')[['msg', 'likes', 'shares', 'cid', 'gid']]
data.columns = ['msg', 'likes', 'shares', 'comments', 'gid']
data['msg_len'] = data.msg.apply(len)

data.gid = data.gid.map({117291968282998: 1, 25160801076: 2, 1443890352589739: 3})

data.fillna(0, inplace=True)

park = data[data.gid == 1]
town = data[data.gid == 2]
free = data[data.gid == 3]
def conf_interval(field):
    """"
    Calculate confidence interval for given field
    """

def compare_means(field):
    """
    Mann–Whitney test to compare mean values level
    """
    mapping = {1: 'EPH', 2: 'UCT', 3: 'FSZ'}
    comparison = pd.DataFrame(columns=['group1', 'group2', 'p_value'])
    for i in range(1, 4):
        for j in range(1, 4):
            if i >= j:
                continue
            p = mannwhitneyu(data[data.gid == i][field], data[data.gid == j][field])[1]
            comparison = comparison.append({'group1': mapping[i], 'group2': mapping[j], 'p_value': p}, ignore_index=True)
    (rejected, p_corrected, a1, a2) = multipletests(comparison.p_value, alpha=0.05, method='holm')
    comparison['p_value_corrected'] = p_corrected
    comparison['rejected'] = rejected
    return comparison

shared = data[data.shares > data.shares.quantile(0.98)][data.shares > data.likes * 10][['msg', 'shares']]
top = 10
print('top %d out of %d' % (top, shared.shape[0]))
sorted_data = shared.sort_values(by='shares', ascending=False)[:top]
for i in sorted_data.index.values:
    print('shares:', sorted_data.shares[i], '\n', 'message:', sorted_data.msg[i][:200], '\n')