text_plain_output_1.png
PATH = '/kaggle/input/covid19-global-forecasting-week-4/'
train_df = pd.read_csv(PATH + 'train.csv', parse_dates=['Date'])
test_df = pd.read_csv(PATH + 'test.csv', parse_dates=['Date'])
add_datepart(train_df, 'Date', drop=False)
add_datepart(test_df, 'Date', drop=False)
train_df.shape

PATH1 = '/kaggle/input/covid19-country-data-wk3-release/'
meta_convert_fun = lambda x: np.float32(x) if x not in ['N.A.', '#N/A', '#NULL!'] else np.nan
meta_df = pd.read_csv(PATH1 + 'Data Join - RELEASE.csv', thousands=',', converters={' TFR ': meta_convert_fun, 'Personality_uai': meta_convert_fun}).rename(columns=lambda x: x.strip())
PATH2 = '/kaggle/input/countryinfo/'
countryinfo = pd.read_csv(PATH2 + 'covid19countryinfo.csv', thousands=',', parse_dates=['quarantine', 'schools', 'publicplace', 'gathering', 'nonessential'])
testinfo = pd.read_csv(PATH2 + 'covid19tests.csv', thousands=',')
countryinfo.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
testinfo.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
testinfo = testinfo.drop(['alpha3code', 'alpha2code', 'date'], axis=1)
PATH3 = '/kaggle/input/covid19-forecasting-metadata/'
continent_meta = pd.read_csv(PATH3 + 'region_metadata.csv').rename(columns={'density': 'pop_density'})
continent_meta = continent_meta[['Country_Region', 'Province_State', 'continent', 'lat', 'lon', 'pop_density']]
recoveries_meta = pd.read_csv(PATH3 + 'region_date_metadata.csv', parse_dates=['Date'])
def fill_unknown_state(df):
    df.fillna({'Province_State': 'Unknown'}, inplace=True)
for d in [train_df, test_df, meta_df, countryinfo, testinfo, continent_meta, recoveries_meta]:
    fill_unknown_state(d)

outliars = ['China']
out_inputs = {}
for out in outliars:
    out_inputs[out] = train_df[train_df['Country_Region'] == out]
    train_df.drop(train_df.index[train_df['Country_Region'] == out], inplace=True)

test_ori = test_df.copy()
merge_cols = ['Province_State', 'Country_Region', 'Date']
test_hlp = test_df[merge_cols + ['ForecastId']]
fst_date = test_hlp.Date.min()
outlier_dfs = []
for out, in_df in out_inputs.items():
    last_date = in_df.Date.max()
    merged = in_df[in_df['Date'] >= fst_date].merge(test_hlp, on=merge_cols, how='left')[['ForecastId', 'ConfirmedCases', 'Fatalities']]
    future_test = test_hlp[(test_hlp['Country_Region'] == out) & (test_hlp['Date'] > last_date)]
    to_add = in_df.groupby(['Province_State', 'Country_Region']).last().reset_index()[['Province_State', 'Country_Region', 'ConfirmedCases', 'Fatalities']]
    merged_future = future_test.merge(to_add, on=['Province_State', 'Country_Region'], how='left')[['ForecastId', 'ConfirmedCases', 'Fatalities']]
    merged = pd.concat([merged, merged_future], sort=True)
    test_df.drop(test_df[test_df['ForecastId'].isin(merged.ForecastId)].index, inplace=True)
    outlier_dfs.append(merged)
outlier_df = pd.concat(outlier_dfs, sort=True)

outlier_df.index = outlier_df['ForecastId']
outlier_df.drop('ForecastId', axis=1, inplace=True)

train_max_date = train_df.Date.max()
outlier_all = test_df[test_df['Date'] <= train_max_date].merge(train_df, on=merge_cols, how='left')[['ForecastId', 'ConfirmedCases', 'Fatalities']]
outlier_all.index = outlier_all.ForecastId
test_df.drop(test_df[test_df['ForecastId'].isin(outlier_all.ForecastId)].index, inplace=True)
outlier_all.drop('ForecastId', axis=1, inplace=True)

outlier_df = pd.concat([outlier_df, outlier_all], sort=True)
outlier_df