text_html_output_1.png
import os

import pandas as  pd

dataset_path = '../input/learnplatform-covid19-impact-on-digital-learning/'
engagement_path = '../input/learnplatform-covid19-impact-on-digital-learning/engagement_data'
district_path = '../input/learnplatform-covid19-impact-on-digital-learning/districts_info.csv'
products_path = '../input/learnplatform-covid19-impact-on-digital-learning/products_info.csv'

def create_dataset_from_engagement(engagement_path):
    dataset = pd.DataFrame()
    for path, dirc, files in os.walk(engagement_path):
        for file in files:
            if dataset.empty:
                dataset = pd.read_csv(os.path.join(path, file))
                dataset['dis_id'] = file.split('.')[0]
            else:
                next_frame = pd.read_csv(os.path.join(path, file))
                next_frame['dis_id'] = file.split('.')[0]
                dataset = pd.concat([dataset[:], next_frame[:]])
    return dataset

district_df = pd.read_csv(district_path)
products_df = pd.read_csv(products_path)

print('Before removing null values')
district_df.isnull().sum() * 100 / district_df.shape[0]
print('After removing null values')
district_df.dropna(subset=['state'], inplace=True)
district_df.isnull().sum() * 100 / district_df.shape[0]