text_html_output_1.png
import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

districts_df = pd.read_csv('../input/learnplatform-covid19-impact-on-digital-learning/districts_info.csv')
products_df = pd.read_csv('../input/learnplatform-covid19-impact-on-digital-learning/products_info.csv')

class GetDfForPreprocessing:
    """
    this function will prepare data form dataframe for preprocessing
    Return
    ------
    dataframe
    """
    def __init__(self, df: pd.DataFrame):
        self.df = df
    def get_info(self):
        missing = self.df.isnull().sum().sum()
        total_cells = np.product(self.df.shape)
        total_missing_count = self.df.isnull().sum().sum()
        missing_columns = self.df.columns[self.df.isnull().any()]
    def get_percentage_missing_columns(self):
        """a function to check for missing values count and percentage missing"""
        count_missing = self.df.isnull().sum()
        count_missing_percentage = round(self.df.isnull().sum() * 100 / len(self.df))
        missing_column_name = self.df.columns
        missing_df = pd.DataFrame(zip(count_missing, count_missing_percentage, missing_column_name), columns=['Missing Count', '%Missing', 'ColumnName'])
        missing_df = missing_df.set_index('ColumnName')
        return missing_df
    def get_column_with_null(self):
        """
        Return List of Columns which contain more than 30% of null values
        """
        number_of_rows, number_of_columns = self.df.shape
        df_size = number_of_rows * number_of_columns
        df_size = self.df.shape[0]
        columns_list = self.df.columns
        bad_columns = []
        for column in columns_list:
            null_per_column = self.df[column].isnull().sum()
            percentage = round(null_per_column / df_size * 100, 2)
            if percentage > 30 or percentage == 30:
                bad_columns.append(column)
        return bad_columns

prep_dist = GetDfForPreprocessing(districts_df)

prep_dist.get_percentage_missing_columns()

district_null_columns = prep_dist.get_column_with_null()

districts_df = districts_df[districts_df.state.notna()].reset_index(drop=True)

districts_df.shape

districts_df.isnull().sum()

districts_df = districts_df.copy()
districts_df = districts_df[districts_df['state'].notna()]
for col in district_null_columns:
    freq = districts_df[col].mode()[0]
    districts_df[col] = districts_df[col].fillna(freq)

PATH = '../input/learnplatform-covid19-impact-on-digital-learning/engagement_data'
temp = []
for district in districts_df.district_id.unique():
    df = pd.read_csv(f'{PATH}/{district}.csv', index_col=None, header=0)
    df['district_id'] = district
    if df.time.nunique() == 366:
        temp.append(df)
engagement_df = pd.concat(temp)
engagement_df = engagement_df.reset_index(drop=True)

districts_df.head()