image_output_1.png
import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

districts_df = pd.read_csv('../input/learnplatform-covid19-impact-on-digital-learning/districts_info.csv')
products_df = pd.read_csv('../input/learnplatform-covid19-impact-on-digital-learning/products_info.csv')

class GetDfForPreprocessing:
    """
    this function will prepare data form dataframe for preprocessing
    Return
    ------
    dataframe
    """
    def __init__(self, df: pd.DataFrame):
        self.df = df
    def get_info(self):
        missing = self.df.isnull().sum().sum()
        total_cells = np.product(self.df.shape)
        total_missing_count = self.df.isnull().sum().sum()
        missing_columns = self.df.columns[self.df.isnull().any()]
    def get_percentage_missing_columns(self):
        """a function to check for missing values count and percentage missing"""
        count_missing = self.df.isnull().sum()
        count_missing_percentage = round(self.df.isnull().sum() * 100 / len(self.df))
        missing_column_name = self.df.columns
        missing_df = pd.DataFrame(zip(count_missing, count_missing_percentage, missing_column_name), columns=['Missing Count', '%Missing', 'ColumnName'])
        missing_df = missing_df.set_index('ColumnName')
        return missing_df
    def get_column_with_null(self):
        """
        Return List of Columns which contain more than 30% of null values
        """
        number_of_rows, number_of_columns = self.df.shape
        df_size = number_of_rows * number_of_columns
        df_size = self.df.shape[0]
        columns_list = self.df.columns
        bad_columns = []
        for column in columns_list:
            null_per_column = self.df[column].isnull().sum()
            percentage = round(null_per_column / df_size * 100, 2)
            if percentage > 30 or percentage == 30:
                bad_columns.append(column)
        return bad_columns

prep_dist = GetDfForPreprocessing(districts_df)

prep_dist.get_percentage_missing_columns()

district_null_columns = prep_dist.get_column_with_null()

districts_df = districts_df[districts_df.state.notna()].reset_index(drop=True)

districts_df.shape

districts_df.isnull().sum()

districts_df = districts_df.copy()
districts_df = districts_df[districts_df['state'].notna()]
for col in district_null_columns:
    freq = districts_df[col].mode()[0]
    districts_df[col] = districts_df[col].fillna(freq)

products_df = products_df[products_df['Sector(s)'].notna()]

products_df['primary_function_main'] = products_df['Primary Essential Function'].apply(lambda x: x.split(' - ')[0] if x == x else x)
products_df['primary_function_sub'] = products_df['Primary Essential Function'].apply(lambda x: x.split(' - ')[1] if x == x else x)
products_df.drop('Primary Essential Function', axis=1, inplace=True)

PATH = '../input/learnplatform-covid19-impact-on-digital-learning/engagement_data'
temp = []
for district in districts_df.district_id.unique():
    df = pd.read_csv(f'{PATH}/{district}.csv', index_col=None, header=0)
    df['district_id'] = district
    if df.time.nunique() == 366:
        temp.append(df)
engagement_df = pd.concat(temp)
engagement_df = engagement_df.reset_index(drop=True)

# Ploting functions
def bar_plot(df, x_col, y_col, title=''):
    plt.figure(figsize=(20, 7))
    sns.barplot(data = df, x=x_col, y=y_col, palette="Set3")
    plt.title(title, size=16)
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    plt.show()
    
def count_plot(df, col, title):
    plt.figure(figsize=(20, 7))
    sns.countplot(data = df,y = col, order=df[col].value_counts().index, palette="Set3")
    plt.title(title, size=16)
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    plt.show()
    
def pie_plot(df, col, title=''):
    count = df[col].value_counts()
    plt.figure(figsize=(20, 7))
    plt.pie(list(count), labels=count.index, pallete ='cubehelix',autopct='%1.2f%%')
    plt.title(title, size=16)
    plt.legend()
    plt.show()
    
def count_plot2(df,col,hue,title):
    plt.figure(figsize=(20,10))
    ax=sns.countplot(data=df,x=col,hue= hue,palette='Set3')
    plt.xticks(rotation=90)
    plt.title(title, size=16)
    plt.show()

def time_plot(df, x_col, y_col, title=''):
    plt.figure(figsize=(20, 7))
    sns.cubehelix_palette(as_cmap=True)
    sns.lineplot(data=df, x=x_col, y=y_col) #pallete='cubehelix_r')
    plt.title(title, size=16)
    plt.xticks(rotation=90, fontsize=14)
    plt.yticks( fontsize=14)
    plt.xlabel(x_col, fontsize=16)
    plt.ylabel(y_col, fontsize=16)
    plt.show()

count_plot(products_df, 'primary_function_sub', 'Sub-categories in Primary Function')