application_vnd.jupyter.stderr_output_2.png text_plain_output_3.png text_plain_output_1.png
import matplotlib.pyplot as plt

import numpy as np

import numpy as np # linear algebra

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import sys

data_path = '/kaggle/input/covid19-global-forecasting-week-4/train.csv'
data_train = pd.read_csv(data_path)

confirmed_total_date_Italy = data_train[data_train['Country_Region'] == 'Italy'].groupby(['Date']).agg({'ConfirmedCases': ['sum']})
fatalities_total_date_Italy = data_train[data_train['Country_Region'] == 'Italy'].groupby(['Date']).agg({'Fatalities': ['sum']})
total_date_Italy = confirmed_total_date_Italy.join(fatalities_total_date_Italy)
confirmed_total_date_US = data_train[data_train['Country_Region'] == 'US'].groupby(['Date']).agg({'ConfirmedCases': ['sum']})
fatalities_total_date_US = data_train[data_train['Country_Region'] == 'US'].groupby(['Date']).agg({'Fatalities': ['sum']})
total_date_US = confirmed_total_date_US.join(fatalities_total_date_US)
confirmed_total_date_Spain = data_train[data_train['Country_Region'] == 'Spain'].groupby(['Date']).agg({'ConfirmedCases': ['sum']})
fatalities_total_date_Spain = data_train[data_train['Country_Region'] == 'Spain'].groupby(['Date']).agg({'Fatalities': ['sum']})
total_date_Spain = confirmed_total_date_Spain.join(fatalities_total_date_Spain)
confirmed_total_date_Korea = data_train[data_train['Country_Region'] == 'Korea, South'].groupby(['Date']).agg({'ConfirmedCases': ['sum']})
fatalities_total_date_Korea = data_train[data_train['Country_Region'] == 'Korea, South'].groupby(['Date']).agg({'Fatalities': ['sum']})
total_date_Korea = confirmed_total_date_Korea.join(fatalities_total_date_Korea)
plt.ylim(0, 180000)
plt.ylim(0, 23000)

def add_daily_measures(df):
    df.loc[0, 'Daily Cases'] = df.loc[0, 'ConfirmedCases']
    df.loc[0, 'Daily Deaths'] = df.loc[0, 'Fatalities']
    for i in range(1, len(df)):
        df.loc[i, 'Daily Cases'] = df.loc[i, 'ConfirmedCases'] - df.loc[i - 1, 'ConfirmedCases']
        df.loc[i, 'Daily Deaths'] = df.loc[i, 'Fatalities'] - df.loc[i - 1, 'Fatalities']
    df.loc[0, 'Daily Cases'] = 0
    df.loc[0, 'Daily Deaths'] = 0
    return df

df_world = data_train.copy()
df_world = df_world.groupby('Date', as_index=False)['ConfirmedCases', 'Fatalities'].sum()
df_world = add_daily_measures(df_world)

df_usa = data_train.query("Country_Region=='US'")
df_usa = df_usa.groupby('Date', as_index=False)['ConfirmedCases', 'Fatalities'].sum()
df_usa = add_daily_measures(df_usa)
df_italy = data_train.query("Country_Region=='Italy'")
df_italy = df_italy.groupby('Date', as_index=False)['ConfirmedCases', 'Fatalities'].sum()
df_italy = add_daily_measures(df_italy)
df_spain = data_train.query("Country_Region=='Spain'")
df_spain = df_spain.groupby('Date', as_index=False)['ConfirmedCases', 'Fatalities'].sum()
df_spain = add_daily_measures(df_spain)
df_korea = data_train.query("Country_Region=='Korea, South'")
df_korea = df_korea.groupby('Date', as_index=False)['ConfirmedCases', 'Fatalities'].sum()
df_korea = add_daily_measures(df_korea)

data_flight = pd.read_csv('/kaggle/input/covid19/covid19_flight_countries_mod.csv')

data_daily_tested = pd.read_csv('/kaggle/input/covid19/full-list-covid-19-tests-per-day.csv')

df_usa_tested = data_daily_tested.query("Code=='USA'")
df_italy_tested = data_daily_tested.query("Entity=='Italy'")
df_spain_tested = data_daily_tested.query("Entity=='Spain'")
df_korea_tested = data_daily_tested.query("Entity=='South Korea'")

df_usa_merge = pd.merge(df_usa, df_usa_tested)
df_usa_merge = pd.merge(df_usa_merge, data_flight)

df_usa_data = df_usa_merge.drop(['Date', 'Entity', 'Code', 'US <-> Latin America', 'US <-> China', 'Canada <-> Canada', 'Canada <-> NON Canada', 'Europe <-> Europe', 'Europe <-> UK', 'Europe <-> Latin America', 'UK <-> UK', 'UK <-> NON UK', 'Italy <-> Italy', 'China <-> China', 'Brazil <-> Brazil', 'Brazil <-> NON Brazil', 'India <-> India', 'India <-> NON India', 'Iran <-> Iran'], axis=1)

quant_features = ['Daily Cases', 'Daily change in cumulative total tests', 'US <-> US', 'US <-> NON US', 'US <-> Europe', 'ConfirmedCases']
scaled_features = {}
for each in quant_features:
    mean, std = (df_usa_data[each].mean(), df_usa_data[each].std())
    scaled_features[each] = [mean, std]
    df_usa_data.loc[:, each] = (df_usa_data[each] - mean) / std

test_data = df_usa_data[-15:]
data = df_usa_data[:-15]
target_fields = ['Daily Cases', 'Daily change in cumulative total tests', 'ConfirmedCases']
features, targets = (df_usa_data.drop(target_fields, axis=1), df_usa_data[target_fields])
test_features, test_targets = (test_data.drop(target_fields, axis=1), test_data[target_fields])

class NeuralNetwork(object):
    def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
        self.input_nodes = input_nodes
        self.hidden_nodes = hidden_nodes
        self.output_nodes = output_nodes
        self.weights_input_to_hidden = np.random.normal(0.0, self.input_nodes ** (-0.5), (self.input_nodes, self.hidden_nodes))
        self.weights_hidden_to_output = np.random.normal(0.0, self.hidden_nodes ** (-0.5), (self.hidden_nodes, self.output_nodes))
        self.lr = learning_rate
        self.activation_function = lambda x: 1 / (1 + np.exp(-x))
    def train(self, features, targets):
        n_records = features.shape[0]
        delta_weights_i_h = np.zeros(self.weights_input_to_hidden.shape)
        delta_weights_h_o = np.zeros(self.weights_hidden_to_output.shape)
        for X, y in zip(features, targets):
            final_outputs, hidden_outputs = self.forward_pass_train(X)
            delta_weights_i_h, delta_weights_h_o = self.backpropagation(final_outputs, hidden_outputs, X, y, delta_weights_i_h, delta_weights_h_o)
        self.update_weights(delta_weights_i_h, delta_weights_h_o, n_records)
    def forward_pass_train(self, X):
        hidden_inputs = np.dot(X, self.weights_input_to_hidden)
        hidden_outputs = self.activation_function(hidden_inputs)
        final_inputs = np.dot(hidden_outputs, self.weights_hidden_to_output)
        final_outputs = final_inputs
        return (final_outputs, hidden_outputs)
    def backpropagation(self, final_outputs, hidden_outputs, X, y, delta_weights_i_h, delta_weights_h_o):
        error = y - final_outputs
        hidden_error = np.dot(self.weights_hidden_to_output, error)
        output_error_term = error * 1
        hidden_error_term = hidden_error * hidden_outputs * (1 - hidden_outputs)
        delta_weights_i_h += hidden_error_term * X[:, None]
        delta_weights_h_o += output_error_term * hidden_outputs[:, None]
        return (delta_weights_i_h, delta_weights_h_o)
    def update_weights(self, delta_weights_i_h, delta_weights_h_o, n_records):
        self.weights_hidden_to_output += self.lr * delta_weights_h_o / n_records
        self.weights_input_to_hidden += self.lr * delta_weights_i_h / n_records
    def run(self, features):
        hidden_inputs = np.dot(features, self.weights_input_to_hidden)
        hidden_outputs = self.activation_function(hidden_inputs)
        final_inputs = np.dot(hidden_outputs, self.weights_hidden_to_output)
        final_outputs = final_inputs
        return final_outputs

def MSE(y, Y):
    return np.mean((y - Y) ** 2)

iterations = 1000
learning_rate = 0.3
hidden_nodes = 7
output_nodes = 1

import sys
N_i = train_features.shape[1]
network = NeuralNetwork(N_i, hidden_nodes, output_nodes, learning_rate)
losses = {'train': [], 'validation': []}
for ii in range(iterations):
    batch = np.random.choice(train_features.index, size=128)
    X, y = (train_features.iloc[batch].values, train_targets.iloc[batch]['Daily Cases'])
    network.train(X, y)
    train_loss = MSE(network.run(train_features).T, train_targets['Daily Cases'].values)
    val_loss = MSE(network.run(val_features).T, val_targets['Daily Cases'].values)
    sys.stdout.write('\rProgress: {:2.1f}'.format(100 * ii / float(iterations)) + '% ... Training loss: ' + str(train_loss)[:5] + ' ... Validation loss: ' + str(val_loss)[:5])
    sys.stdout.flush()
    losses['train'].append(train_loss)
    losses['validation'].append(val_loss)

_ = plt.ylim()

fig, ax = plt.subplots(figsize=(16, 6))
mean, std = scaled_features['Daily Cases']
predictions = network.run(test_features).T * std + mean
ax.plot(predictions[0], label='Prediction')
ax.plot((test_targets['Daily Cases'] * std + mean).values, label='Daily Cases')
ax.set_xlim(right=len(predictions))
ax.legend()
dates = pd.to_datetime(df_usa_merge.iloc[test_data.index]['Date'])
dates = dates.apply(lambda d: d.strftime('%b %d'))
ax.set_xticks(np.arange(len(dates)))
_ = ax.set_xticklabels(dates, rotation=45)