text_html_output_29.png text_html_output_27.png text_html_output_28.png text_html_output_10.png text_html_output_22.png text_html_output_16.png text_html_output_4.png text_html_output_6.png text_html_output_26.png text_html_output_2.png text_html_output_15.png text_html_output_5.png text_html_output_14.png text_html_output_23.png text_html_output_19.png text_html_output_9.png text_html_output_13.png text_html_output_20.png text_html_output_21.png text_html_output_1.png text_html_output_17.png text_html_output_30.png text_html_output_18.png text_html_output_12.png text_html_output_11.png text_html_output_24.png text_html_output_31.png text_html_output_8.png text_html_output_25.png text_html_output_3.png text_html_output_7.png
from keras.layers import Dense

from keras.layers import LSTM

from keras.models import Sequential

from numpy import array

import datetime

import datetime

import numpy as np # linear algebra

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import plotly.express as px

import plotly.io as pio

import plotly.offline as py

import numpy as np
import pandas as pd
from plotly import tools, subplots
import plotly.offline as py
py.init_notebook_mode(connected=True)
import plotly.graph_objs as go
import plotly.express as px
from plotly.subplots import make_subplots
import plotly.figure_factory as ff
import plotly.io as pio
pio.templates.default = 'plotly_dark'
import os
import datetime

train_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
test_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
submission_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
train_df = train_df.drop(['Id'], axis=1)
train_df.rename(columns={'Country_Region': 'Country'}, inplace=True)
test_df.rename(columns={'Country_Region': 'Country'}, inplace=True)
train_df['Province_State'].fillna('None*', inplace=True)
test_df['Province_State'].fillna('None*', inplace=True)
renameCountryNames = {'Congo (Brazzaville)': 'Congo1', 'Congo (Kinshasa)': 'Congo2', "Cote d'Ivoire": "Côte d'Ivoire", 'Czechia': 'Czech Republic (Czechia)', 'Korea, South': 'South Korea', 'Saint Kitts and Nevis': 'Saint Kitts & Nevis', 'Saint Vincent and the Grenadines': 'St. Vincent & Grenadines', 'Taiwan*': 'Taiwan', 'US': 'United States'}
train_df.replace({'Country': renameCountryNames}, inplace=True)
test_df.replace({'Country': renameCountryNames}, inplace=True)
specific_countries = ['United States', 'United Kingdom', 'Netherlands']
days_df = train_df['Date'].apply(lambda dt: datetime.datetime.strptime(dt, '%Y-%m-%d') - datetime.datetime.strptime('2020-01-21', '%Y-%m-%d')).apply(lambda x: str(x).split()[0]).astype(int)
train_df['Days'] = days_df
days_df = test_df['Date'].apply(lambda dt: datetime.datetime.strptime(dt, '%Y-%m-%d') - datetime.datetime.strptime('2020-01-21', '%Y-%m-%d')).apply(lambda x: str(x).split()[0]).astype(int)
test_df['Days'] = days_df
all_countries = train_df['Country'].unique()
ww_df = train_df.groupby('Date')[['ConfirmedCases', 'Fatalities']].sum().reset_index()
ww_df['new_case'] = ww_df['ConfirmedCases'] - ww_df['ConfirmedCases'].shift(1)
ww_df['new_deaths'] = ww_df['Fatalities'] - ww_df['Fatalities'].shift(1)
country_df = train_df.groupby(['Date', 'Country'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
target_date = country_df['Date'].max()
train_end_day = train_df['Days'].max()
test_start_day = test_df['Days'].min()
test_end_day = test_df['Days'].max()

py.init_notebook_mode()
top_country_df = country_df.query('(Date == @target_date) & (ConfirmedCases > 2000)').sort_values('ConfirmedCases', ascending=False)
print(len(top_country_df))
top_country_melt_df = pd.melt(top_country_df, id_vars='Country', value_vars=['ConfirmedCases', 'Fatalities'])
display(top_country_df.head())
display(top_country_melt_df.head())
fig = px.bar(top_country_melt_df.iloc[::-1],
             x='value', y='Country', color='variable', barmode='group',
             title=f'Confirmed Cases/Deaths on {target_date}', text='value', height=1500, orientation='h')
fig.show()

country_province_df = train_df[train_df['Country'] == 'United States'].groupby(['Date', 'Province_State'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
top_province_df = country_province_df.query('(Date == @target_date)').sort_values('ConfirmedCases', ascending=False)
top30_provinces = top_province_df.sort_values('ConfirmedCases', ascending=False).iloc[:30]['Province_State'].unique()
country_province_df['prev_cases'] = country_province_df.groupby('Province_State')['ConfirmedCases'].shift(1)
country_province_df['New Case'] = country_province_df['ConfirmedCases'] - country_province_df['prev_cases']
country_province_df['New Case'].fillna(0, inplace=True)
country_province_df['prev_deaths'] = country_province_df.groupby('Province_State')['Fatalities'].shift(1)
country_province_df['New Death'] = country_province_df['Fatalities'] - country_province_df['prev_deaths']
country_province_df['New Death'].fillna(0, inplace=True)
for province in top30_provinces:
    present_country_df = country_province_df[country_province_df['Province_State'] == province]

top30_countries = top_country_df.sort_values('ConfirmedCases', ascending=False).iloc[:30]['Country'].unique()
country_df['prev_cases'] = country_df.groupby('Country')['ConfirmedCases'].shift(1)
country_df['New Case'] = country_df['ConfirmedCases'] - country_df['prev_cases']
country_df['New Case'].fillna(0, inplace=True)
country_df['prev_deaths'] = country_df.groupby('Country')['Fatalities'].shift(1)
country_df['New Death'] = country_df['Fatalities'] - country_df['prev_deaths']
country_df['New Death'].fillna(0, inplace=True)
top30_country_df = country_df[country_df['Country'].isin(top30_countries)]
for country in top30_countries:
    present_country_df = top30_country_df[top30_country_df['Country'] == country]

def get_time_series(df, country_name, insert=False):
    if df[df['Country'] == country_name]['Province_State'].nunique() > 1:
        country_table = df[df['Country'] == country_name]
        if insert:
            country_df = pd.DataFrame(pd.pivot_table(country_table, values=['ConfirmedCases', 'Fatalities', 'Days'], index='Date', aggfunc=sum).to_records())
            return country_df.set_index('Date')[['ConfirmedCases', 'Fatalities']]
        return country_table.set_index('Date')[['Province_State', 'ConfirmedCases', 'Fatalities', 'Days']]
    df = df[df['Country'] == country_name]
    return df.set_index('Date')[['ConfirmedCases', 'Fatalities', 'Days']]
def get_time_series_province(province):
    df = full_table[full_table['Province_State'] == province]
    return df.set_index('Date')[['ConfirmedCases', 'Fatalities']]
province_country_dfs = {}
no_province_country_dfs = {}
absent_country_in_age_data_dfs = {}
province_country_dfs_list = []
no_province_country_dfs_list = []
absent_country_in_age_data_dfs_list = []
province_countries = train_df[train_df['Province_State'] != 'None*']['Country'].unique()
no_province_countries = train_df[train_df['Province_State'] == 'None*']['Country'].unique()
no_province_countries = [x for x in no_province_countries if x not in province_countries]
for country in province_countries:
    province_country_dfs[country] = get_time_series(train_df, country)
for country in no_province_countries:
    no_province_country_dfs[country] = get_time_series(train_df, country)
assert len([x for x in all_countries if x not in list(no_province_countries) + list(province_countries)]) == 0

train_df['Province_State'].fillna('None*', inplace=True)
test_df['Province_State'].fillna('None*', inplace=True)
test_df['ConfirmedCases'] = np.nan
test_df['Fatalities'] = np.nan
test_df = test_df.set_index('Date')
prediction = {}

def build_model(n_steps):
    model = Sequential()
    model.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(n_steps, 1)))
    model.add(LSTM(50, activation='relu'))
    model.add(Dense(1))
    model.compile(optimizer='RMSprop', loss='mse')
    return model
def split_sequence(sequence, n_steps):
    X, y = (list(), list())
    for i in range(len(sequence)):
        end_ix = i + n_steps
        if end_ix > len(sequence) - 1:
            break
        seq_x, seq_y = (sequence[i:end_ix], sequence[end_ix])
        X.append(seq_x)
        y.append(seq_y)
    return (array(X), array(y))

for country in province_countries:
    current_country_provinces = province_country_dfs[country]['Province_State'].unique()
    for province in current_country_provinces:
        current_considered_country_df = province_country_dfs[country][province_country_dfs[country]['Province_State'] == province][['ConfirmedCases', 'Fatalities', 'Days']].reset_index()
        current_considered_country_df_copy = current_considered_country_df
        for i in range(train_end_day - test_start_day + 1):
            test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == i + test_start_day), 'ConfirmedCases'] = current_considered_country_df.loc[current_considered_country_df['Days'] == i + test_start_day, 'ConfirmedCases'].values[0]
            test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == i + test_start_day), 'Fatalities'] = current_considered_country_df.loc[current_considered_country_df['Days'] == i + test_start_day, 'Fatalities'].values[0]
        indexNames = current_considered_country_df[current_considered_country_df['ConfirmedCases'] == 0].index
        current_considered_country_df.drop(indexNames, inplace=True)
        cases_train = np.diff(current_considered_country_df['ConfirmedCases'].to_numpy())
        fatalities_train = np.diff(current_considered_country_df['Fatalities'].to_numpy())
        cases_train[cases_train < 0] = 0
        fatalities_train[fatalities_train < 0] = 0
        fatal_rate = 0.0
        if current_considered_country_df['Fatalities'].to_numpy()[-1] > 0:
            fatal_rate = current_considered_country_df['Fatalities'].to_numpy()[-1] / current_considered_country_df['ConfirmedCases'].to_numpy()[-1]
        cases_increase_avg = 0
        days = 0
        for i in range(len(cases_train) - 1):
            cases_increase_avg += cases_train[i + 1] - cases_train[i]
            days += 1
        if days > 0:
            cases_increase_avg = int(cases_increase_avg / days)
        days = 0
        fatal_increase_avg = 0
        for i in range(len(fatalities_train) - 1):
            fatal_increase_avg += fatalities_train[i + 1] - fatalities_train[i]
            days += 1
        if days > 0:
            fatal_increase_avg = int(fatal_increase_avg / days)
        del current_considered_country_df
        n_steps = max(int(len(cases_train) * 0.1), 3)
        avg_weekly_per_day_case = []
        avg_window = 4
        avg_step = 2
        if int(len(cases_train) / avg_window) > avg_step:
            for i in range(int(len(cases_train) / avg_window)):
                temp_list = cases_train[i * avg_window:i * avg_window + avg_window]
                avg_weekly_per_day_case.append(np.sum(temp_list) / len(temp_list))
            avg_weekly_per_day_case = np.array(avg_weekly_per_day_case)
            X_weekly_avg_val, y_weekly_avg_val = split_sequence(avg_weekly_per_day_case, avg_step)
            X_weekly_avg_val = np.reshape(X_weekly_avg_val, (X_weekly_avg_val.shape[0], X_weekly_avg_val.shape[1], 1))
            model_weekly_avg = build_model(avg_step)
            model_weekly_avg.fit(X_weekly_avg_val, y_weekly_avg_val, epochs=50, verbose=0)
            new_entry_avg = X_weekly_avg_val[len(X_weekly_avg_val) - 1]
            for i in range(int(30 / avg_window) + 1):
                weekly_avg_predict_next = model_weekly_avg.predict(np.reshape(new_entry_avg, (1, avg_step, 1)), verbose=0).astype(int)
                avg_weekly_per_day_case = np.append(avg_weekly_per_day_case, weekly_avg_predict_next[0])
                last_series = np.reshape(new_entry_avg, (1, avg_step, 1))
                new_entry_avg = np.delete(last_series, [0])
                new_entry_avg = np.insert(new_entry_avg, avg_step - 1, weekly_avg_predict_next[0])
        X_cases_val, y_cases_val = split_sequence(cases_train, n_steps)
        X_cases_val = np.reshape(X_cases_val, (X_cases_val.shape[0], X_cases_val.shape[1], 1))
        X_fatal_val, y_fatal_val = split_sequence(fatalities_train, n_steps)
        X_fatal_val = np.reshape(X_fatal_val, (X_fatal_val.shape[0], X_fatal_val.shape[1], 1))
        assert len(X_fatal_val) == len(X_cases_val)
        assert len(y_fatal_val) == len(y_cases_val)
        model_cases = build_model(n_steps)
        model_cases.fit(X_cases_val, y_cases_val, epochs=50, verbose=0)
        cases_predict_next = model_cases.predict(np.reshape(X_cases_val[len(X_cases_val) - 1], (1, n_steps, 1)), verbose=0).astype(int)
        cases_predict_next[0] = np.array([max(0, cases_predict_next[0])])
        model_fatalities = build_model(n_steps)
        model_fatalities.fit(X_fatal_val, y_fatal_val, epochs=50, verbose=0)
        fatality_predict_next = model_fatalities.predict(np.reshape(X_fatal_val[len(X_fatal_val) - 1], (1, n_steps, 1)), verbose=0).astype(int)
        fatality_predict_next[0] = np.array([max(0, fatality_predict_next[0])])
        fatality_predict_next[0] = np.array([max(fatality_predict_next[0], cases_predict_next[0] * fatal_rate)])
        test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == train_end_day + 1), 'ConfirmedCases'] = test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == train_end_day), 'ConfirmedCases'].values[0] + cases_predict_next[0]
        test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == train_end_day + 1), 'Fatalities'] = test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == train_end_day), 'Fatalities'].values[0] + fatality_predict_next[0]
        new_entry_cases = X_cases_val[len(X_cases_val) - 1]
        new_entry_fatal = X_fatal_val[len(X_fatal_val) - 1]
        for i in range(test_end_day - train_end_day - 1):
            last_series = np.reshape(new_entry_cases, (1, n_steps, 1))
            new_entry_cases = np.delete(last_series, [0])
            new_entry_cases = np.insert(new_entry_cases, n_steps - 1, cases_predict_next[0])
            cases_predict_next = model_cases.predict(np.reshape(new_entry_cases, (1, n_steps, 1)), verbose=0).astype(int)
            if cases_predict_next[0] - new_entry_cases[n_steps - 1] > cases_increase_avg:
                cases_predict_next = np.array([max(0, new_entry_cases[n_steps - 1] + cases_increase_avg)])
            if province in ['Kentucky', 'New Mexico', 'Sint Maarten', 'Cayman Islands', 'Isle of Man']:
                cases_predict_next[0] = avg_weekly_per_day_case[-int(30 / avg_window) - 1 + int(i / avg_window)]
            cases_predict_next[0] = np.array([max(0, cases_predict_next[0])])
            last_series = np.reshape(new_entry_fatal, (1, n_steps, 1))
            new_entry_fatal = np.delete(last_series, [0])
            new_entry_fatal = np.insert(new_entry_fatal, n_steps - 1, fatality_predict_next[0])
            fatality_predict_next = model_fatalities.predict(np.reshape(new_entry_fatal, (1, n_steps, 1)), verbose=0).astype(int)
            if fatality_predict_next[0] - new_entry_fatal[n_steps - 1] > fatal_increase_avg:
                fatality_predict_next[0] = max(0, new_entry_fatal[n_steps - 1] + fatal_increase_avg)
            fatality_predict_next[0] = np.array([max(0, fatality_predict_next[0])])
            fatality_predict_next[0] = np.array([max(fatality_predict_next[0], int(cases_predict_next[0] * fatal_rate))])
            test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == i + train_end_day + 2), 'ConfirmedCases'] = test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == i + train_end_day + 1), 'ConfirmedCases'].values[0] + cases_predict_next[0]
            test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == i + train_end_day + 2), 'Fatalities'] = test_df.loc[(test_df['Country'] == country) & (test_df['Province_State'] == province) & (test_df['Days'] == i + train_end_day + 1), 'Fatalities'].values[0] + fatality_predict_next[0]
        del model_fatalities
        del model_cases

country_province_df = test_df[test_df['Country'] == 'United States'].groupby(['Date', 'Province_State'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
country_province_df['prev_cases'] = country_province_df.groupby('Province_State')['ConfirmedCases'].shift(1)
country_province_df['New Case'] = country_province_df['ConfirmedCases'] - country_province_df['prev_cases']
country_province_df['New Case'].fillna(0, inplace=True)
country_province_df['prev_deaths'] = country_province_df.groupby('Province_State')['Fatalities'].shift(1)
country_province_df['New Death'] = country_province_df['Fatalities'] - country_province_df['prev_deaths']
country_province_df['New Death'].fillna(0, inplace=True)
display(country_province_df.head())
for province in top30_provinces:
    present_country_df = country_province_df[country_province_df['Province_State'] == province]
    px.bar(present_country_df, x='Date', y='New Case', color='Province_State', title=f'United States : DAILY NEW Confirmed cases in ' + province).show()