text_plain_output_1.png
import numpy as np

import os

import pandas as pd

import os
import math
import pandas as pd
import numpy as np
import scipy as sp
import scipy.stats as stats
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import TimeSeriesSplit
from sklearn.model_selection import cross_val_score
from sklearn.metrics import make_scorer, mean_squared_error
import lightgbm as lgb
from lightgbm import LGBMRegressor
from hyperopt import hp, tpe
from hyperopt.fmin import fmin

df_env = pd.read_csv('/kaggle/input/global-environmental-factors/env.csv')
df_airpol = pd.read_csv('/kaggle/input/pm25-global-air-pollution/pm25-global-air-pollution-2017.csv')
df_pop = pd.read_csv('/kaggle/input/world-population-by-country-state/country_population.csv')

df_env.isna().sum()

df_env.loc[df_env['Country_Region'].isin(['Norway', 'Finland', 'Iceland', 'Estonia']), 'wind'] = 4.689151
df_env.loc[df_env['Country_Region'].isin(['Maldives']), 'wind'] = np.mean([2.698925, 3.494908])
df_env.loc[df_env['Country_Region'].isin(['Bahrain']), 'wind'] = np.mean([3.728877, 3.173667, 4.525724])
df_env.loc[df_env['Country_Region'].isin(['Antigua and Barbuda']), 'wind'] = np.mean([3.586282, 3.378886, 2.749947])
df_env.loc[df_env['Country_Region'].isin(['Saint Vincent and the Grenadines']), 'wind'] = 3.515223
df_env.loc[df_env['Country_Region'].isin(['Malta']), 'wind'] = np.mean([3.078635, 2.648621])
df_env.loc[df_env['Country_Region'].isin(['Seychelles']), 'wind'] = 2.736786
df_env.loc[df_env['Country_Region'].isin(['Saint Kitts and Nevis']), 'wind'] = 3.515223
df_env.loc[df_env['Country_Region'].isin(['Grenada']), 'wind'] = 3.515223
df_env.loc[df_env['Country_Region'].isin(['Saint Lucia']), 'wind'] = 3.515223
df_env.loc[df_env['Country_Region'].isin(['Barbados']), 'wind'] = 3.515223
df_env.loc[df_env['Country_Region'].isin(['Monaco']), 'wind'] = np.mean([5.745106, 3.369222])
cols_na = ['accessibility_to_cities', 'elevation', 'aspect', 'slope', 'tree_canopy_cover', 'isothermality', 'rain_coldestQuart', 'rain_driestMonth', 'rain_driestQuart', 'rain_mean_annual', 'rain_seasonailty', 'rain_warmestQuart', 'rain_wettestMonth', 'rain_wettestQuart', 'temp_annual_range', 'temp_coldestQuart', 'temp_diurnal_range', 'temp_driestQuart', 'temp_max_warmestMonth', 'temp_mean_annual', 'temp_min_coldestMonth', 'temp_seasonality', 'temp_warmestQuart', 'temp_wettestQuart', 'cloudiness']
for c in cols_na:
    country = df_env.loc[df_env[c].isna(), 'Country_Region'].unique()
    if 'Maldives' in country:
        df_env.loc[df_env[c].isna(), c] = df_env[df_env['Country_Region'] == 'India'][c].mean()
    else:
        df_env.loc[df_env[c].isna(), c] = df_env[df_env['Country_Region'] == 'Denmark'][c].mean()

df_pop = df_pop[~df_pop['Country_Region'].isna()]
df_pop.drop(['quarantine', 'schools', 'restrictions'], axis=1, inplace=True)
df_pop.isna().sum()

df_pop.dtypes

cols_na = ['pop', 'tests', 'testpop', 'density', 'medianage', 'urbanpop', 'hospibed', 'smokers', 'sex0', 'sex14', 'sex25', 'sex54', 'sex64', 'sex65plus', 'sexratio', 'lung', 'femalelung', 'malelung']
for c in cols_na:
    df_pop[c] = df_pop.groupby(['Country_Region'])[c].transform(lambda x: x.fillna(x.mean()))
for c in cols_na:
    df_pop[c].fillna(df_pop[c].mean(), inplace=True)

dirname = '/kaggle/input'
train_enriched_filename = 'covid19-forecasting-data-with-containment-measures/train-enriched-with-containment_v4.csv'
train_filename = 'covid19-global-forecasting-week-4/train.csv'
test_filename = 'covid19-global-forecasting-week-4/test.csv'
df_train_enriched = pd.read_csv(os.path.join(dirname, train_enriched_filename))
df_train_raw = pd.read_csv(os.path.join(dirname, train_filename))
df_test_raw = pd.read_csv(os.path.join(dirname, test_filename))
df_train_raw.shape

df_train_clean = df_train_enriched.drop(['Id'], axis=1)
df_test_clean = df_test_raw[~df_test_raw['Date'].isin(df_train_enriched['Date'])]
df_test_clean = df_test_clean.drop(['ForecastId'], axis=1)

base_date = pd.to_datetime('2020-01-01')
base_date

df_test_clean['days_since'] = (pd.to_datetime(df_test_clean['Date']) - base_date).dt.days
df_test_clean['days_since'].unique()

df = pd.concat([df_train_clean, df_test_clean], sort=False).reset_index(drop=True)

df = pd.merge(df, df_airpol.drop_duplicates(subset=['Country_Region']), how='left')
df.shape

df = pd.merge(df, df_env.drop_duplicates(subset=['Country_Region']), how='left')
df.shape

df.dtypes

df_pop.columns.values

df = pd.merge(df, df_pop.drop_duplicates(subset=['Country_Region', 'Province_State']), how='left')
df.shape