text_html_output_1.png
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

data = pd.read_csv('/kaggle/input/covid-cases-and-deaths-worldwide/covid_worldwide.csv')
data

column_change = ['serial number', 'country', 'total_cases', 'total_deaths', 'total_recovered', 'active_cases', 'total_test', 'population']
data.columns = column_change
data['total_cases'] = data['total_cases'].str.replace(',', '').astype('float64')
data['total_deaths'] = data['total_deaths'].str.replace(',', '').astype('float64')
data['total_recovered'] = data['total_recovered'].str.replace(',', '').astype('float64')
data['active_cases'] = data['active_cases'].str.replace(',', '').astype('float64')
data['total_test'] = data['total_test'].str.replace(',', '').astype('float64')
data['population'] = data['population'].str.replace(',', '').astype('float64')

missing_c = [col for col in data.columns if data[col].isnull().sum() > 0]
missing_c

data.select_dtypes(include=['object']).columns.tolist()

def get_numerical_summary(df):
    total = df.shape[0]
    missing_columns = [col for col in df.columns if df[col].isnull().sum() > 0]
    missing_percent = {}
    for col in missing_columns:
        null_count = df[col].isnull().sum()
        per = null_count / total * 100
        missing_percent[col] = per
    return missing_percent

missing_percent = get_numerical_summary(data)

data_temp = data.copy()
features_thread = 25
for col, per in missing_percent.items():
    if per > features_thread:
        data_temp.drop(col, axis=1, inplace=True)

df_temp = data.copy()
for idx in range(df_temp.shape[0]):
    df_temp.loc[idx, 'missing_count'] = df_temp.iloc[idx, :].isnull().sum()

SAMPLE_THRESHOLD = 5
df_temp.drop(df_temp[df_temp['missing_count'] > SAMPLE_THRESHOLD].index, axis=0, inplace=True)

from sklearn.impute import KNNImputer
df_temp = data.copy()

df_temp