import numpy as np
import pandas as pd
import os
import pickle
# from sklearn.externals import joblib
import matplotlib.pyplot as plt
from collections import Counter
from sklearn.model_selection import train_test_split
from datetime import timedelta
import scipy as sp
from SolarForecasting.ModulesProcessing import collect_data, clean_data
from SolarForecasting.ModulesLearning import preprocessing as preprocess
from SolarForecasting.ModulesLearning import postprocessing as postprocess
from SolarForecasting.ModulesLearning.ModuleLSTM import train as tranformers
from SolarForecasting.ModulesLearning.ModulesCNN import train as cnn
from SolarForecasting.ModulesLearning.ModulesCNN.Model import crps_score
from matplotlib import cm

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

# city
city = 'Sioux_Falls_SD'#'Boulder_CO'#'Goodwin_Creek_MS'#'Desert_Rock_NV''Penn_State_PA'

# lead time
lead_times = [24*7] #7days in advance

# season
seasons =['year']
res = '1hour' #15min

# file locations
path_local = "/Users/saumya/Desktop/SolarProject/"
path_cluster = "/pl/active/machinelearning/Solar_forecasting_project/"
path_project = path_local
path = path_project+"Data/"
folder_saving = path_project + city+"/Models/"
folder_plots = path_project + city+"/Plots/"
clearsky_file_path = path+'clear-sky/'+city+'_1min_original.csv'

# scan all the features
features = ['year','month','day','hour','min','zen','dw_solar','uw_solar','direct_n','diffuse','dw_ir','dw_casetemp','dw_dometemp','uw_ir','uw_casetemp','uw_dometemp','uvb','par','netsolar','netir','totalnet','temp','rh','windspd','winddir','pressure']

## ## selected features for the study
final_features = ['year','month','day','hour','zen','dw_solar','uw_solar','direct_n','dw_ir','uw_ir','temp','rh','windspd','winddir','pressure', 'clear_ghi']
ensmeble_col_list = ["ensemble_" + str(ensemble_num) for ensemble_num in range(51)]

# target or Y
target_feature = 'clearness_index'

# start and end month+year
startyear = 2016
endyear = 2018
startmonth = 1
endmonth = 12

testyear = 2018

# hyperparameters
n_timesteps = 24#48 #1day (can be 12hrs or 48hrs for a few models)
n_features = 16 #+51 #+ 1
quantile = False #False

#hyperparameters for LSTM/Transformer/CNNs
n_layers = 1 #2 #3
num_heads = 2 #4
d_model = 96#64 #128

hidden_size= 50
batch_size = 16 #32 #16 #16

epochs = 300 #250
lr = 1e-4 #1e-5 #1e-4

alphas = np.arange(0.05, 1.0, 0.05) #range of alpha (for quantile forecast)
q50 = 9  #median index in 'alphas'

def get_data():

    ## collect raw data
    years = [2015, 2016, 2017, 2018] #[2005, 2006, 2007, 2008, 2009]
    object = collect_data.SurfradDataCollector(years, [city], path)

    object.download_data()

    ## cleanse data to get processed version
    for year in years:
        object = clean_data.SurfradDataCleaner(city, year, path)
        object.process(path_to_column_names='ModulesProcessing/column_names.pkl')

def include_previous_features(X, col_to_indices_mapping):
    '''
    Features at time t includes features from previous n_timesteps
    '''
    # y_list = []
    # previous_time_periods = list(range(1, n_timesteps+1))
    # # indices_except_ensmebles = sorted([feature_ind for feature, feature_ind in col_to_indices_mapping.items() if not feature.startswith('ensemble')])
    # # print(indices_except_ensmebles)
    # # Xsub = X[:,indices_except_ensmebles]
    #
    # for l in previous_time_periods:
    #     # print("rolling by: ", l)
    #     X_train_shifted = np.roll(X, l, axis=0)
    #
    #     y_list.append(X_train_shifted)
    #     # y_list.append(dw_solar_rolled)
    # y_list = y_list[::-1]
    #
    # previous_time_periods_columns = np.column_stack(y_list)
    # X = np.column_stack([previous_time_periods_columns, X])
    #
    # print("X shape after adding prev features: ", X.shape)
    # return X

    X = np.expand_dims(X, axis=2)
    # indices_except_ensmebles = sorted(
    #     [feature_ind for feature, feature_ind in col_to_indices_mapping.items() if not feature.startswith('ensemble')])
    # # print(X.shape)
    X_with_prev_timesteps = []
    data_size = X.shape[0]
    for i in range(n_timesteps,data_size):
        prev_list =[]
        for j in range(n_timesteps,0,-1):
            prev = X[i-j,:,:]
            # prev_sub = prev[indices_except_ensmebles,:]
            # print(prev.shape)
            prev_list.append(prev)

        curr = X[i,:,:]
        prev_list.append(curr)
        temp = np.concatenate(prev_list, axis=1)

        # print(temp.shape)
        X_with_prev_timesteps.append(temp)

    # print(len(X_with_prev_timesteps))

    X = np.array(X_with_prev_timesteps)
    # print(X.shape)
    return X

def include_nwp_features(df):

    missing_hour_dates = {2016:[263,265], 2017:[283], 2018:[144]}

    fnwp = path + "NWP/" + city + "/"

    #Alreadt sorted
    # df = dfraw.sort_values(by=['year','month','day','hour'])
    # print("inside nwp")
    # print(dfraw.equals(df))

    for ensemble in ensmeble_col_list:
        df[ensemble] = ""

    nwp_new_columns = []
    for year in [2016,2017,2018]:
        print(len(df[df['year'] == year]))
        year_list = []
        fnwp_yr = fnwp+"nwp_"+str(year)+".npy"
        nwp_yr = np.load(fnwp_yr)
        nwp_yr = nwp_yr/(168*3600)
        days = nwp_yr.shape[0]
        for day in range(days):
            if day not in missing_hour_dates[year]:
                nwp_day = nwp_yr[day]
                print(nwp_day.shape)
                x_1_12 = np.tile(nwp_day[0], (12,1))
                x_13_24 = np.tile(nwp_day[1], (12,1))
                x_day = np.concatenate((x_1_12, x_13_24), axis=0)
                print(x_day.shape)
                year_list.append(x_day)

        year_nwp = np.concatenate(year_list)
        print(year_nwp)
        print(year_nwp.shape)
        nwp_new_columns.append(year_nwp)

    nwp_new_columns = np.concatenate(nwp_new_columns)
    print(np.max(nwp_new_columns), np.min(nwp_new_columns))

    df.loc[:,ensmeble_col_list] = nwp_new_columns

    return df

def get_crps_for_ngboost(model, X, y):
    '''
    CRPS is the evaluation metric to be used for probabilsitic forecasting
    '''

    params = model.pred_dist(X)._params
    loc = params[0]
    scale = np.exp(params[1])
    Z = (y - loc) / scale

    score = scale * (
            Z * (2 * sp.stats.norm.cdf(Z) - 1)
            + 2 * sp.stats.norm.pdf(Z)
            - 1 / np.sqrt(np.pi)
    )

    return np.average(score)

def get_crps_for_ngboost_scaled(model, X, y,X_before_normalized,index_clearghi,lead):
    '''
    CRPS is the evaluation metric to be used for probabilsitic forecasting
    '''

    params = model.pred_dist(X)._params
    loc = params[0]
    scale = np.exp(params[1])

    clearsky = X_before_normalized[:, index_clearghi]
    y = np.roll(y, lead)
    loc = np.roll(loc, lead)
    scale = np.roll(scale, lead)

    print(clearsky.shape, y.shape, loc.shape, scale.shape)
    y = np.multiply(y, clearsky)
    loc = np.multiply(loc, clearsky)
    scale = np.multiply(scale, clearsky)

    Z = (y - loc) / scale

    score = scale * (
            Z * (2 * sp.stats.norm.cdf(Z) - 1)
            + 2 * sp.stats.norm.pdf(Z)
            - 1 / np.sqrt(np.pi)
    )
    print(score.shape)
    score = score[2 * lead:]

    return np.average(score)

def get_CH_PeEN_baseline_crps(X_test, X_train, col_to_indices_mapping, y_test):
    index_hour = col_to_indices_mapping['hour']
    index_clearness = col_to_indices_mapping['clearness_index_input']

    current_hour_list = X_test[:, index_hour]

    CH_PeEN_baseline = []
    CH_PeEN_point_baseline = []

    # X_train_only_last_timestep = X_train[:, :, -1]

    for i in range(len(X_test)):
        current_hour = current_hour_list[i]
        # print(current_hour,current_month)
        X_train_for_hour = X_train[X_train[:,index_hour]==current_hour]

        hour_dist = X_train_for_hour[:, index_clearness]#X_train_for_hour[:, index_ghi]/X_train_for_hour[:, index_clearghi]

        CH_PeEN_point_baseline.append(np.average(hour_dist))

        quantiles = np.quantile(hour_dist, alphas)
        CH_PeEN_baseline.append(quantiles)

    CH_PeEN_baseline_quantiles = np.asarray(CH_PeEN_baseline)
    CH_PeEN_point_baseline = np.reshape(CH_PeEN_point_baseline, (len(CH_PeEN_point_baseline), 1))

    clearsky = y_test[:, 1].reshape(y_test.shape[0], 1)
    true = y_test[:, 0].reshape(y_test.shape[0], 1)  # np.roll(y_test, lead)
    y_test = np.multiply(true, clearsky)

    CH_PeEN_baseline_quantiles = np.multiply(CH_PeEN_baseline_quantiles, clearsky)
    CH_PeEN_point_baseline = np.multiply(CH_PeEN_point_baseline, clearsky)

    crps_CH_PeEN = crps_score(CH_PeEN_baseline_quantiles, y_test, alphas)

    return CH_PeEN_point_baseline, CH_PeEN_baseline_quantiles, crps_CH_PeEN

def get_climatology_baseline_crps(X_test, X_train, col_to_indices_mapping, y_test):

    index_hour = col_to_indices_mapping['hour']
    index_ghi = col_to_indices_mapping['dw_solar']

    current_hour_list = X_test[:, index_hour]
    print(current_hour_list.shape)

    climatology_baseline = []
    climatology_point_baseline = []

    # X_train_only_last_timestep = X_train[:,:,-1]
    # print(X_train_only_last_timestep.shape)

    for i in range(len(X_test)):
        current_hour = current_hour_list[i]
        # print(current_hour,current_month)
        X_train_for_hour = X_train[X_train[:, index_hour] == current_hour]
        hour_dist = X_train_for_hour[:, index_ghi]

        climatology_point_baseline.append(np.average(hour_dist))

        quantiles = np.quantile(hour_dist, alphas)
        climatology_baseline.append(quantiles)

    climatology_baseline_quantiles = np.asarray(climatology_baseline)
    climatology_point_baseline = np.reshape(climatology_point_baseline, (len(climatology_point_baseline), 1))

    clearsky = y_test[:, 1].reshape(y_test.shape[0], 1)
    true = y_test[:, 0].reshape(y_test.shape[0], 1)  # np.roll(y_test, lead)
    y_test = np.multiply(true, clearsky)

    crps_climatology = crps_score(climatology_baseline_quantiles, y_test, alphas)

    return climatology_point_baseline, climatology_baseline_quantiles, crps_climatology

# def reliability_diagrams(alphas, outputs, outputs_ch_pn, outputs_climatology, target, folder_saving):
#
#
#     outputs_tcn_attn = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_attn_quantile/tcn_attention.npy")
#     outputs_transformers = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_transformers_2day_2heads_less_dmodel_lag_week_ahead_1hr_res_quantile/transformer.npy")
#     print(outputs_tcn_attn.shape, outputs_transformers.shape)
#
#     ## had to add the following for boulder, since timesteps were different for tcn/transformers
#     # target = target[24:, :]
#     total_obs = len(target)
#     # outputs = outputs[24:,:]
#     # outputs_tcn_attn = outputs_tcn_attn[24:,:]
#     # outputs_ch_pn = outputs_ch_pn[24:,:]
#     # outputs_climatology = outputs_climatology[24:,:]
#     # print(outputs_tcn_attn.shape, outputs_transformers.shape)
#
#
#     obs_proportion_list = []
#     obs_proportion_list_chpn = []
#     obs_proportion_list_climatology = []
#     obs_proportion_list_tcn_attn =[]
#     obs_proportion_list_transformers = []
#     for i, alpha in zip(range(len(alphas)), alphas):
#         output = outputs[:, i].reshape((-1, 1))
#         output_ch_pn = outputs_ch_pn[:, i].reshape((-1, 1))
#         output_climatology = outputs_climatology[:, i].reshape((-1, 1))
#         output_transformers = outputs_transformers[:, i].reshape((-1, 1))
#         output_tcn_attn = outputs_tcn_attn[:, i].reshape((-1, 1))
#
#         obs_proportion_list.append(np.sum(target<=output)/total_obs)
#         obs_proportion_list_chpn.append(np.sum(target <= output_ch_pn) / total_obs)
#         obs_proportion_list_climatology.append(np.sum(target <= output_climatology) / total_obs)
#         obs_proportion_list_tcn_attn.append(np.sum(target <= output_tcn_attn) / total_obs)
#         obs_proportion_list_transformers.append(np.sum(target <= output_transformers) / total_obs)
#
#     print(obs_proportion_list == obs_proportion_list_climatology)
#
#     fig, ax = plt.subplots()
#     # ax.plot(alphas, alphas)
#     ax.plot(alphas, obs_proportion_list, marker='o', color ='g',label="TCN")
#     ax.plot(alphas, obs_proportion_list_tcn_attn, marker='d', color='y', label="TCN_with_attention")
#     ax.plot(alphas, obs_proportion_list_transformers, marker='h', color='m', label="Transformers")
#     ax.plot(alphas, obs_proportion_list_chpn, marker="s", color ='b',label="CH_PeEN")
#     ax.plot(alphas, obs_proportion_list_climatology, marker="*",color ='k', label="Climatology")
#     ax.set_xlabel('Nominal proportion')
#     ax.set_ylabel('Observed proportion')
#     ax.legend(loc='upper left')
#
#
#     plt.savefig(folder_saving + "reliability_with_all_plots")
#     plt.close()
#
#
# def sharpness_diagrams(alphas, outputs, outputs_ch_pn, outputs_climatology,folder_saving):
#     outputs_tcn_attn = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_attn_quantile/tcn_attention.npy")
#     outputs_transformers = np.load(
#         "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_transformers_2day_2heads_less_dmodel_lag_week_ahead_1hr_res_quantile/transformer.npy")
#
#
#     print(outputs_tcn_attn.shape, outputs_transformers.shape)
#
#
#     intervals = np.array([0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9])
#     sharpness = []
#     sharpness_ch_pn = []
#     sharpness_climatology = []
#     sharpness_tcn_attn = []
#     sharpness_transformers = []
#     for i in intervals:
#
#         h = 0.5+(i/2)
#         l = 0.5-(i/2)
#
#         h = np.where(np.around(alphas,2) == np.around(h,2))[0][0]
#         l = np.where(np.around(alphas,2) == np.around(l,2))[0][0]
#
#         sharpness.append(np.mean(outputs[:,h] - outputs[:,l]))
#         sharpness_ch_pn.append(np.mean(outputs_ch_pn[:, h] - outputs_ch_pn[:, l]))
#         sharpness_climatology.append(np.mean(outputs_climatology[:, h] - outputs_climatology[:, l]))
#         sharpness_tcn_attn.append(np.mean(outputs_tcn_attn[:, h] - outputs_tcn_attn[:, l]))
#         sharpness_transformers.append(np.mean(outputs_transformers[:, h] - outputs_transformers[:, l]))
#
#
#     print(sharpness==sharpness_climatology)
#
#     fig, ax = plt.subplots()
#     # plt.plot(alphas, alphas)
#     ax.plot(intervals, sharpness, marker='o', color ='g',label = "TCN")
#     ax.plot(intervals, sharpness_tcn_attn, marker='d', color='y', label="TCN_with_attention")
#     ax.plot(intervals, sharpness_transformers, marker='h', color='m', label="Transformers")
#     ax.plot(intervals, sharpness_ch_pn, marker="s", color ='b',label="CH_PeEN")
#     ax.plot(intervals, sharpness_climatology, marker="*", color ='k',label=" Hourly Climatology")
#     ax.set_xlabel('intervals')
#     ax.set_ylabel('Average width')
#     ax.legend(loc = 'upper left')
#
#     plt.savefig(folder_saving + "sharpness_with_all_plots")
#     plt.close()
#
#
# def probabilistic_prediction_plots(X_test,outputs,y_test, col_to_indices_mapping,folder_saving):
#     # outputs_tcn_attn = np.load(
#     #     "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_attn_quantile/tcn_attention.npy")
#     # outputs_tcn = np.load(
#     #     "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_tcn_week_ahead_1days_lag_small_kernel_1hr_res_quantile/tcn.npy")
#     # outputs_lstm = np.load(
#     #     "/Users/saumya/Desktop/SolarProject/Boulder_CO/Models/year/ML_models_2018/cnn/1hour/final_lstm_50_week_ahead_2day_lag_1hr_res_quantile/lstm.npy")
#     #
#     # sub_outputs_tcn_attn = outputs_tcn_attn[7*24:8*24,:]
#     # sub_outputs_tcn = outputs_tcn[7*24:8*24,:]
#     # sub_outputs_lstm = outputs_lstm[6 * 24:7 * 24, :]
#     index_hour = -n_features + col_to_indices_mapping['hour']
#     index_month = -n_features + col_to_indices_mapping['month']
#     print(X_test[X_test[:,index_month]==3][:,index_hour])
#     print(X_test[:,index_month]==3)
#     index_ghi = -n_features + col_to_indices_mapping['dw_solar']
#     true = y_test[X_test[:,index_month]==3][:10*3,:] #X_test[X_test[:,index_month]==3][:10*5,index_ghi]
#     sub_outputs = outputs[X_test[:,index_month]==3][:10*3,:]
#     true  = np.reshape(true,-1)
#     print(sub_outputs.shape, true.shape)
#
#     true_full_day=[]
#     for i in range(3):
#         true_before = [0 for _ in range(13)]
#         print(true_before+true[10*i:10*(i+1)].tolist()+[0])
#         true_full_day.extend(true_before+true[10*i:10*(i+1)].tolist()+[0])
#
#
#
#
#     x_full_day = list(range(len(true_full_day)))
#     # x_only_day = x_full_day[13:23] + x_full_day[24+13:24+23] + x_full_day[48+13:48+23]
#     # x_only_day = list(range(13,23)) + list(range(24+13,24+23)) + list(range(48+13,48+23))
#     # print(x_full_day,x_only_day)
#
#
#     # x = range(len(true))
#     fig, ax = plt.subplots(1)
#     ax.plot(x_full_day, true_full_day, lw=1, label='true', color='g')
#     ax.plot(x_full_day[13:23], sub_outputs[:10,9], lw=1, label='predicted median', color='blue')
#     ax.plot(x_full_day[24+13:24+23], sub_outputs[10:20, 9], lw=1, color='blue')
#     ax.plot(x_full_day[48+13:48+23], sub_outputs[20:30, 9], lw=1, color='blue')
#     # ax.fill_between(x, sub_outputs[:,0],sub_outputs[:,-1], facecolor='gray', alpha=0.5,
#     #                 label='prediction')
#
#     total_quantiles = sub_outputs.shape[1]
#     for i in range(int(total_quantiles/2)):
#         # label = ""
#         print(i,total_quantiles-i-1)
#         print(alphas[i], alphas[total_quantiles - i - 1])
#         c = plt.cm.Blues(0.2 + .6 * (float(i) / total_quantiles * 2))
#         ax.fill_between(x_full_day[13:23], sub_outputs[:10,i], sub_outputs[:10,total_quantiles-i-1], color=c)
#         ax.fill_between(x_full_day[24+13:24+23], sub_outputs[10:20, i], sub_outputs[10:20, total_quantiles - i - 1], color=c)
#         ax.fill_between(x_full_day[48+13:48+23], sub_outputs[20:30, i], sub_outputs[20:30, total_quantiles - i - 1], color=c)
#
#     # plt.legend(framealpha=1)
#     # plt.show()
#
#     # ax.fill_between(x_test[6 * 24:7 * 24], sub_outputs_lstm[:, 0], sub_outputs_lstm[:, -1], facecolor='green', alpha=0.5,
#     #                 label='prediction lstm')
#     # ax.fill_between(x_test[6 * 24:7 * 24], sub_outputs_tcn[:, 0], sub_outputs_tcn[:, -1], facecolor='yellow', alpha=0.5,
#     #                 label='prediction tcn ')
#     # ax.fill_between(x_test[6 * 24:7 * 24], sub_outputs_tcn_attn[:, 0], sub_outputs_tcn_attn[:, -1], facecolor='red', alpha=0.5,
#     #                 label='prediction tcn+attention')
#     ax.set_xticks([])
#     ax.legend(loc='upper left')
#     ax.set_xlabel('Time: 3 days in March 2018')
#     ax.set_ylabel('Irradiance')
#
#     plt.savefig(folder_saving + "probabilistic_prediction_best_plots_include_night")
#     plt.close()

def main():

    processed_file_path = path + 'processed/' + city + "/"
    # df.to_pickle(processed_file_path + "final_data_At_" + res + "_resolution_2016-2018_updated.pkl")
    df = pd.read_pickle(processed_file_path + "final_data_At_" + res + "_resolution_2016-2018_updated.pkl") ##this files inlcudes the day times - I'll be dropping them later!!

    # final_features.extend(ensmeble_col_list)#["ensemble_0"]) #ensmeble_col_list)
    ## name of the regression model (this helps to distinguish the models within a "CNN"/"LSTM"/"Transformer" super-folder)
    # reg = "baselines"
    reg = "tcn_no_nwp_week_ahead_24_seq_lag_small_kernel_1hr_res"
    # reg = "final_transformers_no_nwp_d64_week_ahead_24_lag_1hr_res_2layers_quantile"
    for season_flag in seasons:
        ## ML_models_2018 is the folder to save results on testyear 2018
        os.makedirs(
            folder_saving + season_flag + "/final_ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"/",
            exist_ok=True)
        f = open(folder_saving + season_flag + "/final_ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"/results.txt", 'a')

        for lead in lead_times:
            # create dataset with lead
            df_lead = preprocess.create_lead_dataset(df, lead, final_features, target_feature)
            df_lead = df_lead[:len(df_lead)-lead]
            print(df_lead.describe())
            df_lead_2018 = df_lead[df_lead.year == 2018]
            print("len 2018 df lead", len(df_lead_2018))
            ## dropping rows with 0 clear_ghi and taking only daytimes
            df_lead = df_lead[(df_lead['clear_ghi'] > 0) & (df_lead['clear_ghi_target'] > 0) ] #0]
            # print(len(df_lead))
            df_lead = df_lead[(df_lead['zen'] < 85) & (df_lead['zen_target'] < 85)]
            df_lead.reset_index(drop=True, inplace=True)
            print("after removing data points with 0 clear_ghi and selecting daytimes", len(df_lead))

            ## get the yearly/seasonal data
            df, test_startdate, test_enddate = preprocess.get_yearly_or_season_data(df_lead, season_flag, testyear)
            print("\n\n after getting seasonal data (test_startdate; test_enddate)", test_startdate, test_enddate)
            print(df.tail)

            # dividing into training and test set
            df_train, df_heldout = preprocess.train_test_spilt(df, season_flag, testyear)
            print("\n\n after dividing_training_test")
            print("train_set\n", len(df_train))
            print("test_set\n", len(df_heldout))

            if len(df_train) > 0 and len(df_heldout) > 0:
                # extract the X_train, y_train, X_test, y_test
                X_train, y_train, X_heldout, y_heldout, col_to_indices_mapping = preprocess.get_train_test_data(
                    df_train, df_heldout, final_features, target_feature)
                print("\n\n train and test df shapes ")
                print(X_train.shape, y_train.shape, X_heldout.shape, y_heldout.shape)

                # #including features from prev imestamps
                X_train = include_previous_features(X_train, col_to_indices_mapping)
                X_heldout = include_previous_features(X_heldout, col_to_indices_mapping)

                # X_train = X_train[n_timesteps:,:]
                # X_heldout = X_heldout[n_timesteps:, :]
                y_train = y_train[n_timesteps:, :]
                y_heldout = y_heldout[n_timesteps:, :]

                print("Final train size: ", X_train.shape, y_train.shape)
                print("Final heldout size: ", X_heldout.shape, y_heldout.shape)

                ## dividing the X_train data into train(70%)/valid(30%) the heldout data is kept hidden
                X_train, X_valid, y_train, y_valid = train_test_split(
                    X_train, y_train, test_size=0.3, random_state=42)#42

                X_test_before_normalized = X_heldout.copy()
                X_train_before_normalized = X_train.copy()

                ## normalizing the heldout with the X_train used for training
                X_train, X_valid, X_test = preprocess.standardize_from_train(X_train, X_valid, X_heldout)

                print("valid size", X_valid.shape, y_valid.shape)
                print("heldout size:", X_test.shape, y_heldout.shape)

                y_test = y_heldout
                y_true = y_test
                y_pred = None

                f.write("\n" + city + " at Lead " + str(lead) + " and " + season_flag + " Season"+"\n")
                #
                # with open(folder_saving + season_flag + "/ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"//model_at_lead_" + str(lead) + ".pkl", 'rb') as file:
                #     model = pickle.load(file)

                ## If doing probabilsitic prediction, just use the test function to scale the ys and crps score too
                ## In case of deterministic use the postprocessing modules

                y_pred, y_valid_pred, valid_crps, test_crps_scaled = cnn.test_DCNN_with_attention(quantile, X_valid,y_valid,
                                                                                X_test, y_test,
                                                                                n_timesteps + 1, n_features,
                                                                                folder_saving + season_flag + "/final_ML_models_" + str(
                                                                                testyear) + "/cnn/" + str(
                                                                                res) + "/" + reg + "/",
                                                                                "dcnn_lag_for_lead_" + str(
                                                                                lead),  X_test_before_normalized, lead)  # "multi_horizon_dcnn", n_outputs=n_output_steps)

                # y_pred, y_valid_pred, valid_crps, test_crps_scaled  = tranformers.test_LSTM(quantile, X_valid, y_valid,X_test, y_test, n_timesteps+1, n_features,hidden_size,alphas, q50,
                #                                folder_saving + season_flag + "/final_ML_models_"+str(testyear)+"/cnn/"+str(res)+"/"+reg+"/","model_lag_for_lead_" + str(lead),X_test_before_normalized, lead)#"multi_horizon_dcnn", n_outputs=n_output_steps)

                # y_pred, y_valid_pred, valid_crps, test_crps_scaled = tranformers.test_transformer(quantile, X_valid, y_valid,
                #                                                                           X_test, y_test, n_timesteps + 1,
                #                                                                            n_features, n_layers,
                #                                                                            num_heads, d_model, alphas,
                #                                                                            q50,
                #                                                                            folder_saving + season_flag + "/final_ML_models_" + str(
                #                                                                                testyear) + "/cnn/" + str(
                #                                                                                res) + "/" + reg + "/",
                #                                                                           "dcnn_lag_for_lead_" + str(
                #                                                                                lead),X_test_before_normalized, lead)  # "multi_horizon_dcnn", n_outputs=n_output_steps)
                #
                #

                print("\n" + city + " at Lead " + str(lead) + " and " + season_flag + " Season")

            #     print("#####TEST#################")
            #     # postprocessing on test and model's RMSE
                y_true, y_pred = postprocess.postprocessing_target(y_pred, y_true)

                rmse_our, mae_our, mb_our, sd_our, r2_our = postprocess.evaluation_metrics(y_true, y_pred)
                #
                print("Performance of our model (rmse, mae, mb, sd, r2): \n\n", round(rmse_our, 2), round(mae_our, 2),
                      round(mb_our, 2), round(sd_our, 2), round(r2_our, 2))

            #     #
                # # baseliens and plots
                # index_clearness = col_to_indices_mapping['clearness_index_input']
                # y_sp = postprocess.smart_persistence_model(X_test_before_normalized, y_test, index_clearness, lead)
                #
                #
                # rmse_sp, mae_sp, mb_sp, sd_sp, r2_sp = postprocess.evaluation_metrics(y_true, y_sp)
                # print("Performance of smart persistence model (rmse, mae, mb, sd, r2): \n\n", round(rmse_sp, 2),
                #       round(mae_sp, 2),
                #       round(mb_sp, 2), round(sd_sp, 2), round(r2_sp, 2))
                #
                #
                #
                # y_CH_PeEN, CH_PeEN_baseline_quantiles, crps_CH_PeEN = get_CH_PeEN_baseline_crps(X_test_before_normalized, X_train_before_normalized, col_to_indices_mapping, y_test)
                # y_climatology, climatology_probabilistic_baseline, crps_climatology = get_climatology_baseline_crps(X_test_before_normalized, X_train_before_normalized, col_to_indices_mapping, y_test)
                #
                # rmse_clm, mae_clm, mb_clm, sd_clm, r2_clm = postprocess.evaluation_metrics(y_true, y_climatology)
                # print("Performance of hourly climatology baseline model (rmse, mae, mb, sd, r2): \n\n",
                #       round(rmse_clm, 2),
                #       round(mae_clm, 2),
                #       round(mb_clm, 2), round(sd_clm, 2), round(r2_clm, 2))
                #
                # rmse_chp, mae_chp, mb_chp, sd_chp, r2_chp = postprocess.evaluation_metrics(y_true, y_CH_PeEN)
                # print("Performance of CH_PeEN baseline model (rmse, mae, mb, sd, r2): \n\n", round(rmse_chp, 2),
                #       round(mae_chp, 2),
                #       round(mb_chp, 2), round(sd_chp, 2), round(r2_chp, 2))
                #
                #     # reliability_diagrams(np.arange(0.05, 1.0, 0.05), y_pred, CH_PeEN_baseline_quantiles, climatology_probabilistic_baseline, y_test,
                # #                          folder_saving + season_flag + "/ML_models_" + str(
                # #                              testyear) + "/cnn/" + str(
                # #                              res) + "/")
                # #
                # #     sharpness_diagrams(np.arange(0.05, 1.0, 0.05),  y_pred, CH_PeEN_baseline_quantiles, climatology_probabilistic_baseline,
                # #                        folder_saving + season_flag + "/ML_models_" + str(
                # #                            testyear) + "/cnn/" + str(
                # #                            res) + "/")
                #
                # # probabilistic_prediction_plots(X_test_before_normalized,y_pred, y_test, col_to_indices_mapping, folder_saving + season_flag + "/ML_models_" + str(
                # #                                                                              testyear) + "/cnn/" + str(
                # #                                                                             res) + "/" )
                # # # #
                # print('CRPS CH_PeEN scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
                #     round(crps_CH_PeEN, 2)) + '\n')
                # f.write('CRPS CH_PeEN scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
                #     round(crps_CH_PeEN, 2)) + '\n')
                #
                # print('CRPS hourly Climatology scaled score on heldout data for year 2018 for lead' + str(
                #     lead) + '=' + str(
                #     round(crps_climatology, 2)) + '\n')
                # f.write('CRPS hourly Climatology scaled score on heldout data for year 2018 for lead' + str(
                #     lead) + '=' + str(
                #     round(crps_climatology, 2)) + '\n')
                #

                # valid_crps = get_crps_for_ngboost(model, X_valid, y_valid)
                # test_crps = get_crps_for_ngboost(model, X_test, y_test)
            #     # print("before crps",y_test.shape)
            #     test_crps_scaled = get_crps_for_ngboost_scaled(model, X_test, y_test, X_test_before_normalized, index_clearghi, lead)
            #
            #     print('CRPS score on valid data for lead' + str(lead) + '=' + str(
            #         round(valid_crps, 2)) + '\n')
            #
            #     print('CRPS scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
            #         round(test_crps_scaled, 2)) + '\n')
            #     f.write('CRPS scaled score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
            #         round(test_crps_scaled, 2)) + '\n')
                # #
            #
            #
            #     ## saving predictions with and without night times (makes sense with deterministic forecasts, otherwise it the 50th percentile predictions)
                test_with_predictions_array = X_test_before_normalized[:,[col_to_indices_mapping['month'], col_to_indices_mapping['day'], col_to_indices_mapping['hour']],-1]
                # test_with_predictions_array['true_GHI'] = y_test
                test_with_predictions_array = pd.DataFrame(test_with_predictions_array, columns=['month', 'day', 'hour'])

                test_with_predictions_array['predictions_GHI'] = y_pred

                test_with_predictions_array_with_night_times = df_lead_2018[
                    ['year', 'month', 'day', 'hour', 'clearness_index']]

                test_with_predictions_array_with_night_times = test_with_predictions_array_with_night_times.merge(
                    test_with_predictions_array, on=['month', 'day', 'hour'], how='left')

                print(test_with_predictions_array_with_night_times.describe())

                print(np.nanmax(test_with_predictions_array_with_night_times.predictions_GHI.values))
                np.save(folder_saving + season_flag + "/final_ML_models_" + str(testyear) + "/cnn/" + str(
                    res) + "/" + reg + "/" + 'model_predictions_with_all_info_including_night_times.npy',
                        test_with_predictions_array_with_night_times)

            else:
                print("not enough data for the season: ", season_flag, "and lead: ", lead)

        f.close()

if __name__=='__main__':
    main()

# In case you want to find skill score
#     # calculate the skill score of our model over persistence model
#     skill_sp = postprocess.skill_score(rmse_our, rmse_sp)
#     print("\nSkill rmse of our model over smart persistence: ", round(skill_sp, 2))
#
#     skill_np = postprocess.skill_score(rmse_our, rmse_np)
#     print("\nSkill rmse of our model over normal persistence: ", round(skill_np, 2))
#
#     skill_clm = postprocess.skill_score(rmse_our, rmse_clm)
#     print("\nSkill rmse of our model over hourly climatology baseline: ", round(skill_clm, 2))
#
#     skill_chp = postprocess.skill_score(rmse_our, rmse_chp)
#     print("\nSkill rmse of our model over CH_PeEN baseline: ", round(skill_chp, 2))
#
#     f.write('skill rmse score on heldout data for year 2018 for lead' + str(lead) + '=' + str(round(skill_sp, 2)) + '\n')
#     f.write('skill wrt hourly climatology rmse score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#         round(skill_clm, 2)) + '\n')
#     f.write(
#         'skill wrt CH_PeEN rmse score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#             round(skill_chp, 2)) + '\n')
#
#     skill_sp = postprocess.skill_score(mae_our, mae_sp)
#     print("\nSkill mae of our model over smart persistence: ", round(skill_sp, 2))
#
#     skill_np = postprocess.skill_score(mae_our, mae_np)
#     print("\nSkill mae of our model over normal persistence: ", round(skill_np, 2))
#
#     skill_clm = postprocess.skill_score(mae_our, mae_clm)
#     print("\nSkill mae of our model over climatology baseline: ", round(skill_clm, 2))
#
#     skill_chp = postprocess.skill_score(mae_our, mae_chp)
#     print("\nSkill mae of our model over climatology baseline: ", round(skill_chp, 2))
#
#     f.write('skill mae score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#         round(skill_sp, 2)) + '\n')
#     f.write(
#         'skill wrt hourly climatology mae score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#             round(skill_clm, 2)) + '\n')
#     f.write(
#         'skill wrt CH_PeEN mae score on heldout data for year 2018 for lead' + str(lead) + '=' + str(
#             round(skill_chp, 2)) + '\n')


def quicksort(alist, beg, end):
    # alist 表示数组
    # beg为开始位置
    # end 为结束位置

    # 递归结束条件,即只有一个元素时，默认为有序数组
    if beg >= end:
        return

    # 定义中间元素 mid_value为起始位置值
    mid_value = alist[beg]
    # 定义左侧右侧指针 left right
    left = beg
    right = end

    while left < right:
        # 若右侧指针对应元素大于mide_value，则继续将指针向左移动直到遇到小于mide_value的位置
        while left < right and alist[right] >= mid_value:
            right -= 1
        # 若左侧指针对应元素小于mide_value，则继续将指针向右移动直到遇到大于mide_value的位置
        while left < right and alist[left] <= mid_value:
            left += 1

        alist[left], alist[right] = alist[right], alist[left]

    # 循环结束时，此时right，left指针位置重叠,交换起始位置元素与当前指针位置元素

    alist[beg], alist[right] = alist[right], alist[beg]

    quicksort(alist, beg, right-1)
    quicksort(alist, right + 1, end)

if __name__ == "__main__":
    n = input("输入")
    n=int(n)
    q = list()
    for i in range(n):
        q.append(int(input()))

    quicksort(q, 0, n - 1)
    print(q)


## IMDb 페이지에서 현재 상영 중인 영화정보 수집하기  https://www.imdb.com/list/ls016522954/?ref_=nv_tvv_dvd

## 제목, 평점(metascore), 감독, 배우 찾기
# 제목: h3.lister-item-header>a
# 평점: div.ipl-rating-widget div.small > span.ipl-rating-star__rating
# 감독: p.text-muted.text-small>a
# 배우: p.text-muted.text-small>a
# 장르: p.text-muted.text-small>span.genre

## 코드를 통해 데이터 수집하기(select함수 이용)
'''
import requests
from bs4 import BeautifulSoup

raw = requests.get("https://www.imdb.com/list/ls016522954/?ref_=nv_tvv_dvd", headers={"User-Agent" : "Mozilla/5.0"})
html = BeautifulSoup(raw.text, 'html.parser')

container = html.select("div.lister-item-content")
for c in container :
    title = c.select_one("h3.lister-item-header>a").text
    metascore = c.select_one(" div.ipl-rating-widget div.small > span.ipl-rating-star__rating")
    info = c.select("p.text-muted.text-small")
    director = info[1].select("a")
    genre = info[0].select("span.genre")

    print("제목: ", title)
    print("평점: ", metascore.text)
    print("감독 및 배우: ")
    for d in director:
        print(d.text)
    for g in genre:
        print(g.text)
    print("="*50)

##### 평점 안적힌 영화 오류 해결 방법은???
'''

## 액션 장르의 영화만 출력하기(선택자 사용)

import requests
from bs4 import BeautifulSoup

raw = requests.get("https://www.imdb.com/list/ls016522954/?ref_=nv_tvv_dvd", headers={"User-Agent" : "Mozilla/5.0"})
html = BeautifulSoup(raw.text, 'html.parser')

container = html.select("div.lister-item-content")
for c in container :
    title = c.select_one("h3.lister-item-header>a").text
    metascore = c.select_one(" div.ipl-rating-widget div.small > span.ipl-rating-star__rating")
    director = c.select("div.lister-item-content p:nth-of-type(3) a")
    genre = c.select_one("div.lister-item-content p:nth-of-type(1) span.genre")

    if "Action" not in genre.text:
        continue

    print("제목: ", title)

    if metascore is not None:
        print( "\n평점:", metascore.text.strip())
    else:
        print("\n평점이 없습니다.")
        #
        # print("평점: ", metascore.text)

    print("\n장르: ")
    for g in genre:
        print(g.strip())       #g는 .text 붙이면 에러뜸.

    print("\n감독 및 배우: ")
    for d in director:
        print(d.text)

    print("="*20)


'''
This module runs the flask webpage, running scheduled
events and accepting and proccesing data entered on
the webpage by the user. It sends data (from files or
processed within the code) to the webpage for users to
see.
'''
import os
import logging
from flask import Flask
from flask import render_template
from flask import request
from flask import Markup
from werkzeug.utils import redirect
from app import read_config_file
from covid_data_handler import *
from covid_news_handling import *

app = Flask(__name__)
#array to store deleted items

config_file=read_config_file()
API_KEY= config_file["API key:"]
LOCATION=config_file["Location:"]
NATION=config_file["Nation:"]

logging.basicConfig(filename='sys.log',level=logging.DEBUG)

if os.path.isfile('updates.json'):
    with open('updates.json', 'r', encoding="UTF-8") as updates_file:
        update_data=json.loads(updates_file.read())
else:
    update_data=[]

if os.path.isfile('repeated_updates.json'):
    with open('repeated_updates.json', 'r', encoding="UTF-8") as repeated_updates_file:
        list_of_repeated_updates=json.loads(repeated_updates_file.read())
else:
    list_of_repeated_updates=[]

for update in update_data:
    if update["type"]=="covid":
        addition_to_update=schedule_covid_updates(update["content"],update["title"])
        update["event"]=addition_to_update["event"]
    elif update["type"]=="news":
        addition_to_update= update_news(update["title"],update["content"])
        update["event"]=addition_to_update["event"]

for update in list_of_repeated_updates:
    if update["type"]=="covid":
        addition_to_update=repeat_scheduled_covid_update(update["content"],update["title"],schedule_covid_updates)
        update["events"]=addition_to_update["events"]
    elif update["type"]=="news":
        addition_to_update= repeat_scheduled_news_update(update["content"],update["title"],update_news)
        update["events"]=addition_to_update["events"]

@app.route('/index')
def main_page():
    '''
    Loads and processes all data and sends it to the webpage.
    Also runs the schedule for the data updates and manages user
    inputs and edits data structures accordingly
    Returns:
        render_template(...): A HTML template called index.html
        rendered with all necessary data for display on the website
    '''
    API_schedule_covid.run(blocking=False)
    API_schedule_news.run(blocking=False)

    title=request.args.get("notif")
    if title: #remove news widgets
        remove_news_article(title)
        return redirect("/index")

    #reading data from files t o send to webpage
    with open('news_data.json', 'r', encoding="UTF-8") as news_file:
        news=json.loads(news_file.read())

    #constructing a list to send news data to web page
    update_name=request.args.get("two")
    news_data=[]
    #inserting links to article source sites
    for item in news:
        news_url=Markup(''+" Read full article here" + "")
        content=Markup(item["description"])+news_url
        news_data.append({"title":item["title"],"content":content})
        if len(news_data)>=8: # limits article widgets
            break
    #reading the covid data  to send to website
    #data is read from a file to avoid polling the API
    if os.path.isfile('national_covid_data.json'):
        with open('national_covid_data.json', 'r', encoding="UTF-8") as covid_file:
            nation_data=json.loads(covid_file.read())
    else:
        logging.info("can't find national_covid_data.json ")

    if os.path.isfile('covid_data.json'):
        with open('covid_data.json', 'r', encoding="UTF-8") as covid_file:
            local_data=json.loads(covid_file.read())
    else:
        logging.info("can't find covid_data.json ")

    #re adds a repeated scheduled event so it can be displayed as a widget
    queue_covid=API_schedule_covid.queue
    queue_news=API_schedule_news.queue

    for item in queue_covid:
        names=item[3]
        if item[2] == schedule_covid_updates:
            #if its a repeated covid update
            try:
                chronology=names[0]
                update_title=names[1]
                event=item
                update_dict={"event":event,"title":update_title,"content":chronology,"type":"covid"}
                if update_dict not in update_data:
                    logging.info('re-added a covid repeated update to the list of updates')
                    update_data.append(update_dict)
            except:
                logging.info('re-adding a covid repeated update to the list of updates failed')

    for item in queue_news:
        names=item[3]
        if item[2] == update_news:
            #if its a repeated news update
            try:
                chronology=names[1]
                update_title=names[0]
                event=item
                update_dict={"event":event,"title":update_title,"content":chronology,"type":"news"}
                if update_dict not in update_data:
                    logging.info('re-added a news repeated update to the list of updates')
                    update_data.append(update_dict)
            except:
                logging.info('re-adding a covid repeated update to the list of updates failed')

    #removing events

    for entry in update_data:
        if entry["event"] not in API_schedule_news.queue and entry["event"] not in API_schedule_covid.queue:
            update_data.remove(entry)
            logging.info('removed an update that had occured')

    cancel_covid_update,cancel_news_update=False,False

    remove_event=request.args.get("update_item") #removes event from event list +schedule
    if remove_event:
        for entry in update_data:
            if entry["title"]==remove_event:
                #cancelling event from schedule
                try:
                    API_schedule_covid.cancel(entry["event"])
                    cancel_covid_update=entry
                    logging.info("removed "+entry["title"]+(" from update schedule"))
                except:
                    try:
                        API_schedule_news.cancel(entry["event"])
                        cancel_news_update=entry
                        logging.info("removed "+entry["title"]+(" from update schedule"))
                    except:
                        logging.info("removing "+entry["title"]+(" from update schedule failed"))
        if cancel_covid_update:
            update_data.remove(cancel_covid_update)
            #remove from widget list
            logging.info("removed a covid entry from update_data")
        if cancel_news_update:
            update_data.remove(cancel_news_update)
            #remove from widget list
            logging.info("removed a news entry from update_data")
        for entry in list_of_repeated_updates: #REMOVES repeated updates from both queues
            if entry["title"]==remove_event:

                try:
                    API_schedule_covid.cancel(entry["events"][0])
                    API_schedule_covid.cancel(entry["events"][1])
                    #events field has 2 events- this cancels both
                    #one event is to schedule an update
                    #the other is to schedule the reschedule function
                    logging.info("removed repeated event from the covid schedule")
                except ValueError:
                    logging.info(" failed to remove repeated event from the covid API schedule")

                try:
                    API_schedule_news.cancel(entry["events"][0])
                    API_schedule_news.cancel(entry["events"][1])
                    logging.info("removed repeated event from the news API schedule")
                except ValueError:
                    logging.info("failed to remove repeated event from the news API schedule")
        return redirect("/index")

    if len(update_data)==0 and len(list_of_repeated_updates)!=0:
        #in case there schould be no updates, but repeated
        #updates haven't been cancelled yet
        logging.info("wiped list_of_repeated_updates and cancelled all events")
        list_of_repeated_updates.clear()
        #cancels all items in the schedules
        for item in API_schedule_news.queue:
            API_schedule_news.cancel(item)

        for item in API_schedule_covid.queue:
            API_schedule_covid.cancel(item)

    #schedules events
    if update_name:
        valid_name=True
        #this ensures the same title can't be used
        for item in update_data:
            if update_name == item["title"]:
                valid_name=False
        if valid_name:
            time_input = request.args.get('update')
            if time_input:
                repeat_update=request.args.get("repeat")
                update_covid_data=request.args.get("covid-data")
                update_news_data=request.args.get("news")

                if update_covid_data:
                    #adding to the list of update events
                    event_dict=schedule_covid_updates(time_input,update_name)
                    logging.info("scheduled a covid update")
                    event_dict["content"]=time_input
                    update_data.append(event_dict)
                    if repeat_update:
                        list_of_repeated_updates.append(repeat_scheduled_covid_update(time_input,
                        update_name,schedule_covid_updates))
                        logging.info("scheduled a repeated covid update")

                if update_news_data:
                    #adding to the list of update events
                    event_dict=update_news(update_name,time_input)
                    logging.info("scheduled a news update")
                    event_dict["content"]=time_input
                    update_data.append(event_dict)
                    if repeat_update:
                        list_of_repeated_updates.append(repeat_scheduled_news_update(time_input,
                        update_name,update_news))
                        logging.info("scheduled a repeated news update")
        return redirect("/index")

    update_list_for_web=[]
    #converting into one list - removes excess widgets caused by 2 schedulers
    #also converting to json dumpable lists
    for item in update_data:
        if item == update_data[0]:
            update_list_for_web.append({"title":item["title"],
            "type":item["type"],"content":item["content"]})
        for value in update_list_for_web:
            if value["title"] != item["title"]:
                update_list_for_web.append({"title":item["title"],
            "type":item["type"],"content":item["content"]})

    #writing updates to file so they can be used if the code
    #stops running
    repeated_updates_to_file=[]
    for item in list_of_repeated_updates:
        repeated_updates_to_file.append({"title":item["title"],
            "type":item["type"],"content":item["content"]})

    with open('updates.json', 'w', encoding="UTF-8") as updates_file:
        json.dump(update_list_for_web, updates_file, indent=6)

    with open('repeated_updates.json', 'w', encoding="UTF-8") as repeated_updates_file:
        json.dump(repeated_updates_to_file, repeated_updates_file, indent=6)

    #processing data
    last7days_cases,current_hospital_cases,total_deaths= process_covid_API_data(local_data)
    last7days_cases_nation, current_hospital_cases_nation, total_deaths_nation=process_covid_API_data(nation_data)
    current_hospital_cases_nation="Current Hospital Cases: "+str(current_hospital_cases_nation)
    total_deaths_nation="Current Total Deaths: "+str(total_deaths_nation)
    logging.info("processed API data")
    #sending data to webpage
    return render_template("index.html", location=LOCATION, local_7day_infections=last7days_cases,
    nation_location=NATION,national_7day_infections=last7days_cases_nation,
    hospital_cases=current_hospital_cases_nation,deaths_total=total_deaths_nation,
    news_articles=news_data, updates=update_list_for_web, image="boon.png")

if __name__=="__main__":
    app.run()


import secrets

from starlette.background import BackgroundTask
from asyncpg.exceptions import UniqueViolationError
from starlette.responses import JSONResponse

from admin.audit_logs import send_audit_log, AuditColour
from admin.discord_api import get_user
from admin.route import Route
from admin.models import APIKey
from admin.utils import is_admin, is_authorized, is_json

class AdminUserRoute(Route):
    """
    Route for fetching and creating new users.
    """
    name = "all_users"
    path = "/users"

    @is_authorized
    @is_admin
    async def get(self, request):
        users = await APIKey.query.where(
            APIKey.creator.isnot(None)
        ).order_by(
            APIKey.is_admin.desc()
        ).gino.all()

        response = []

        for user in users:
            user_data = user.__dict__["__values__"]

            user_data["creator"] = str(user_data["creator"])

            response.append(user_data)

        return JSONResponse(response)

    @is_authorized
    @is_admin
    @is_json
    async def post(self, request):
        data = await request.json()

        try:
            get_user(int(data["creator"]))
        except KeyError:
            return JSONResponse({
                "status": "error",
                "message": "Invalid user specified, upstream returned 404"
            }, status_code=404)

        new_key = APIKey(
            key=secrets.token_hex(32),
            is_admin=data["is_admin"],
            creator=int(data["creator"])
        )

        try:
            await new_key.create()
        except UniqueViolationError:
            return JSONResponse({
                "status": "error",
                "message": "Users can only have one API key per Discord ID"
            }, status_code=400)

        task = BackgroundTask(
            send_audit_log,
            title="New user added",
            body=f"Added by <@{request.state.api_key.creator}>",
            inline_fields={
                "User": f"<@{data['creator']}>",
                "Administrator": data["is_admin"]
            },
            colour=AuditColour.SUCCESS
        )

        return JSONResponse({
            "status": "success",
            "new_key": new_key.key
        }, background=task)


#!/usr/bin/env python3
#@author: Jorge III Altamirano-Astorga
import tensorflow as tf
import re, os, sys, shelve, time, dill, io, logging
import argparse #args from cli
from pickle import PicklingError
from dill import Pickler, Unpickler
shelve.Pickler = Pickler
shelve.Unpickler = Unpickler
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
tf.get_logger().setLevel('ERROR')

def evaluate_model(
    input_dataset,
    #output_datastore,
    model_file,
    mse=True,
    mae=True,
    debug=False):

    if debug:
        logging.info("Entering evaluate_model()")

    if not model_file.startswith('/'):
        logging.error('Model file should be on local storage. No URL gs:// supported!')
        return 255
    if mse == False and mae == False:
        logging.error('Stopping as no models need to be evaluated!')
        return 255

    base_dir = re.sub('^(.*/)[^.]+\.h5$', r'\1', model_file)
    output_datastore = base_dir
    modelname = re.sub('^.*/([^.]+)\.h5$', r'\1', model_file)
    logging.info("Processing base input dir: %s"%base_dir)
    logging.info("Processing base name: %s"%modelname)

    tsparams = get_tsparams(base_dir, modelname, debug)

    test3_iaq = get_timeseries_dataset_from_array(input_dataset, tsparams, debug)

    logging.info('Loading H5 model...')
    model = tf.keras.models.load_model(os.path.join(base_dir, f"{modelname}.h5"))
    logging.info('Done loading H5 model!')
    logging.info('Predicting... (this may take a while)')
    Y_hat = model.predict(test3_iaq, batch_size=tsparams["batch_size"])
    logging.info('Y_hat.shape = %s'%str(Y_hat.shape))
    logging.info('Done predictions!')

    logging.info('Creating Ground Truth Y...')
    Y_gt = np.array([])
    for batch in test3_iaq:
      _, targets = batch
      Y_gt = np.append(Y_gt,targets.numpy())
    logging.info('Y_gt.shape = %s'%str(Y_gt.shape))
    logging.info('Done creating Ground Truth Y!')

    returnValue = 0
    if mse:
        logging.info('Getting MSE...')
        metric_filename = os.path.join(
            output_datastore,
            f'{modelname}.mse.pickle.gz'
        )
        mse = get_mse(Y_gt, Y_hat, metric_filename, debug)
        logging.info('Done MSE!')
    if mae:
        logging.info('Getting MAE...')
        metric_filename = os.path.join(
            output_datastore,
            f'{modelname}.mae.pickle.gz'
        )
        mae = get_mae(Y_gt, Y_hat, metric_filename, debug)
        logging.info('Done MAE!')

    #### Start Section: Hyperparameter Tuning
    try:

        import hypertune

        hpt = hypertune.HyperTune()

        if mse:
            hpt.report_hyperparameter_tuning_metric(
                  hyperparameter_metric_tag='mse',
                  metric_value=mse,
                  global_step=1
                  )
        if mae:
            hpt.report_hyperparameter_tuning_metric(
                  hyperparameter_metric_tag='mae',
                  metric_value=mae,
                  global_step=1
                  )
    except Exception as e:
      logging.error("Start error of hypertune")
      logging.error(e)
      logging.error("End error of hypertune")
      pass
    #### End Section: Hyperparameter Tuning

    return returnValue

def get_mse(Y_gt, Y_hat, metric_filename, debug=False):
    if debug:
        logging.info("Entering get_mse()")
    A = []
    m = tf.keras.metrics.MeanSquaredError()

    for i in range(0, Y_hat.shape[1]):
      m.update_state(Y_gt, Y_hat[:,i])
      A.append(m.result().numpy())
      m.reset_states()

    logging.info("Saving MSE metrics to file: %s"%metric_filename)
    with io.open(metric_filename, 'wb') as metric_file:
        dill.dump(A, metric_file)
    logging.info("Saved %s"%metric_filename)
    A = np.array(A).mean()

    logging.info("Returning MSE mean: %f"%A)
    return A

def get_mae(Y_gt, Y_hat, metric_filename, debug=False):
    if debug:
        logging.info("Entering get_mae()")
    A = []
    m = tf.keras.metrics.MeanAbsoluteError()

    for i in range(0, Y_hat.shape[1]):
      m.update_state(Y_gt, Y_hat[:,i])
      A.append(m.result().numpy())
      m.reset_states()

    logging.info("Saving MAE metrics to file: %s"%metric_filename)
    with io.open(metric_filename, 'wb') as metric_file:
        dill.dump(A, metric_file)
    logging.info("Saved %s"%metric_filename)
    A = np.array(A).mean()

    logging.info("Returning MAE mean: %f"%A)
    return A

def get_timeseries_dataset_from_array(input_dataset, tsparams,
                                      debug=True):
    if debug:
        logging.info("Entering get_timeseries_dataset_from_array()")

    logging.info('Loading dataset: %s'%input_dataset)
    data =  pd.read_pickle(input_dataset)
    data = data[~data.isna().any(axis=1)]
    excluded_columns = ["iaqAccuracy", "wind_speed", "wind_deg"]
    if debug:
        logging.info('Splitting input dataset into Train and Test datasets.')
    train, test = train_test_split(data[[x
                                            for x in data.columns
                                            if x not in excluded_columns]],
                                   train_size=0.7, random_state=175904, shuffle=False)
    scaler = None
    scaler = MinMaxScaler()
    if debug:
        logging.info('MinMaxScale-ing data.')
    scaler_f = scaler.fit(train)
    test2 = scaler_f.transform(test)
    #with io.open(os.path.join("data/output-hyper05min-w07-stride2-samplingrate2/", "scaler.dill"), 'rb') as scalerfile:
    #  scaler = dill.load(scalerfile)
    X_cols = [i for i, x in enumerate(train.columns)
              if x not in ["IAQ", "gasResistance"]]
    Y_cols = [i for i, x in enumerate(train.columns)
              if x in ["IAQ", "gasResistance"]]
    X_test  = test2[:, X_cols]
    Y_test = test2[:, Y_cols]

    if debug:
        logging.info('Executing timeseries_dataset_from array...')
    test3_iaq = tf.keras.preprocessing.timeseries_dataset_from_array(
      X_test,
      Y_test[:, 1],
      sequence_length=tsparams["sequence_length"],
      sampling_rate=tsparams["sampling_rate"],
      sequence_stride=tsparams["stride"],
      batch_size=tsparams["batch_size"],
      seed=175904
    )
    if debug:
        logging.info('Done timeseries_dataset_from array...')

    return test3_iaq

def get_tsparams(base_dir, modelname, debug=False):
    if debug:
        logging.info("Entering get_tsparams()")
    tsparamsfilename = os.path.join(base_dir, f"{modelname}.tsparams.dill")
    with io.open(tsparamsfilename, 'rb') as tsparamsfile:
        tsparams = dill.load(tsparamsfile)
    logging.info("Time series parameters:")
    logging.info("{ %s }"%tsparams)

    if "sequence_length" not in tsparams.keys():
        logging.error("Sequence Length not found! Doing our best guesses :)")
        resample_min = re.sub("^.*output-hyper([0-9]*)min.*$", r"\1",
                            base_dir)
        logging.info("Resample Min: %s"%resample_min)
        logging.info("Window Size Days: %s"%tsparams["window_size_days"])
        tsparams["sequence_length"] = (
            24 * tsparams["window_size_days"] * int(resample_min)
        )
        logging.error("Sequence Length not found! Guessed value: %d"%(
            tsparams["sequence_length"]))

    return tsparams

def setup_logger(logname="logger_jorge3a", loglevel=logging.INFO):
    # Define the log format
    log_format = '[%(asctime)s] %(levelname)-8s %(name)-12s %(message)s'

    # Define basic configuration
    logging.basicConfig(
        # Define logging level
        level=loglevel,
        # Declare the object we created to format the log messages
        format=log_format,
        # Declare handlers
        handlers=[
            logging.StreamHandler()
        ]
    )

    # Define your own logger name
    logger = logging.getLogger(logname)

def usage(argv):
    parser = argparse.ArgumentParser(
        description="""
        Creates a files saved into a Datastore with the
        Mean Squared Errors and Mean Absolute Error using
        an H5 Model File evaluated with the input dataset.""",
        epilog="Example: %(prog)s -m trained/model.h5 https://data.example.com/data/data.pickle.gz",
        prefix_chars='-')

    parser.add_argument('--nomse', '-S', action="store_true",  default=False,
                        help='Avoid calculating Mean Squared Error (MSE).')
    parser.add_argument('--nomae', '-A', action="store_true",  default=False,
                        help='Avoid calculating Mean Squared Error (MSE).')
    parser.add_argument('--debug', '-d', action="store_true",  default=False,
                        help='Enable debugging.')
    parser.add_argument('--model', '-m', nargs=1, required=True,
                        help="""H5 **trained** model file.""")
    parser.add_argument('input_dataset', nargs=1, default="",
                        help="""Input dataset URL or Location of the File.""")
    #parser.add_argument('output_datastore', nargs=1, default="",
    #                    help="""Output dataset URL or Location of the File.
    #                    """)
    return parser.parse_args()

def main(argv):

    ### read cli arguments
    args = usage(argv)

    if args.debug:
        setup_logger(argv[0], logging.DEBUG)
    else:
        setup_logger(argv[0], logging.INFO)

    logging.info("MSE: %s"%(not args.nomse))
    logging.info("MAE: %s"%(not args.nomae))
    logging.info("Debug: %s"%args.debug)
    logging.info("Model Filename: %s"%args.model[0].strip())
    logging.info("Input Dataset: %s"%args.input_dataset[0].strip())
    logging.info("Output Datastore: %s"%re.sub('^(.*/)[^.]+\.h5$', r'\1', args.model[0].strip()))
    logging.info("")
    #print(args)

    result = evaluate_model(
        input_dataset=args.input_dataset[0].strip(),
        #output_datastore=args.output_datastore[0].strip(),
        model_file=args.model[0].strip(),
        mse=(not args.nomse),
        mae=(not args.nomae),
        debug=args.debug
    )

    exit(result)

if __name__ == "__main__":
    main(sys.argv)


from django.conf.urls import url,include
from django.contrib import admin
from. import views
urlpatterns=[
    url(r'^$',view=views.lb,name="logbook"),
    url(r'^login$',view=views.login,name="login"),
    url(r'^add$',view=views.add,name='add'),
    url(r'^view$',view=views.view,name='view'),
    url(r'^view/(?P<username>[A-Za-z0-9]+)$',view=views.viewbyusername,name='view'),
    url(r'^requestid$',view=views.requestid,name='requestid')

]


"""
   * author - kajol
   * date - 12/24/2020
   * time - 1:21 PM
   * package - com.bridgelabz.basicprograms
   * Title - Determine Leap Year or not
"""
try:
    year = int(input("Enter the Year Number: "))
    count = 0
    tempYear = year
    # Check if number of digits in a year is equal to 4 or not.
    while tempYear > 0:
        count += 1
        tempYear = tempYear // 10
    if count != 4:
        print("Not a valid year")
        exit(1)
    # Leap Year logic
    if (year % 400 == 0) or ((year % 4 == 0) and (year % 100 != 0)):
        print("%d is a Leap Year" % year)
    else:
        print("%d is Not the Leap Year" % year)
except Exception:
    print("Exception occured")


import unittest

from math import ceil

from gui.gui import GUI
from gui.score import Score
from logic.ball import Ball
from logic.ball_colors import BallColors
from logic.field import Field
from logic.point import Point

class LinesTestCase(unittest.TestCase):

    def test_raises_exception(self):
        with self.assertRaises(ValueError):
            board = Field(2)
            board = Field(-1)
            gui = GUI(1, Field(9), Score())
            gui = GUI(-1, Field(9), Score())

    def test_find_lines(self):
        sizes = [size for size in range(4, 20)]
        for field_size in sizes:
            field = Field(field_size)
            field.field_objects = [[None] * field_size for _
                                   in range(field_size)]
            for component in range(field.min_line_size):
                field.field_objects[component][0] = Ball(BallColors.blue,
                                                         Point(component, 0))
                field.field_objects[field_size - 1][component] = Ball(
                    BallColors.green, Point(field_size - 1, component))
                field.field_objects[component][field_size - component - 1] = \
                    Ball(BallColors.red, Point(component,
                                               field_size - component - 1))
            field.remove_lines_and_rectangles()
            balls = [(field.field_objects[component][0], field.field_objects
                     [field_size - 1][component],
                      field.field_objects[component]
                      [field_size - component - 1])
                     for component in range(field.min_line_size)]
            self.assertFalse(any(any(ball) for ball in balls))

    def test_find_rectangles(self):
        sizes = [size for size in range(4, 20)]
        for field_size in sizes:
            field = Field(field_size)
            field.field_objects = [[None] * field_size for _
                                   in range(field_size)]
            for x in range(ceil((field.min_line_size + 1) / 2)):
                for y in range(ceil((field.min_line_size + 1) / 2)):
                    field.field_objects[x][y] = Ball(BallColors.blue,
                                                     Point(x, y))
            field.remove_lines_and_rectangles()
            balls = (field.field_objects[x][y] for x in range(
                ceil((field.min_line_size + 1) / 2)
                    ) for y in range(ceil((field.min_line_size + 1) / 2)))
            self.assertFalse(any(balls))

if __name__ == '__main__':
    unittest.main()


from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import Optional
from enum import Enum

app = FastAPI(title="FastAPI App")

class Status(Enum):
    TO_DO = "todo"
    IN_PROGRESS = "in progress"
    DONE = "done"

class Task(BaseModel):
    id: int
    title: str
    description: Optional[str]
    status: Status

class TaskModel(BaseModel):
    title: str
    description: Optional[str]
    status: Status

tasks = [Task(id=1, title="title", description=None, status=Status.DONE)]

@app.get("/")
async def get_tasks():
    return {"message": "FastAPI works!"}

@app.get("/tasks", response_model=list[Task])
async def get_tasks():
    return tasks

@app.get("/tasks/{task_id}", response_model=Task)
async def get_item_by_id(task_id: int):
    task = [task for task in tasks if task.id == task_id]
    if task:
        return task[0]
    raise HTTPException(status_code=404, detail="Task not found")

@app.post("/tasks")
async def create_task(task: TaskModel):
    next_id = max(tasks, key=lambda x: x.id).id + 1
    new_task = Task(id=next_id, title=task.title, description=task.description, status=task.status)
    tasks.append(new_task)

@app.put("/tasks/{task_id}")
async def change_task(task_id: int, task: TaskModel):
    update_task = [t for t in tasks if t.id == task_id]
    if not update_task:
        raise HTTPException(status_code=404, detail="Task not found")
    update_task[0].title = task.title
    update_task[0].description = task.description
    update_task[0].status = task.status
    return update_task[0]

@app.delete("/tasks/{task_id}")
async def get_item_by_id(task_id: int):
    task = [task for task in tasks if task.id == task_id]
    if task:
        tasks.remove(task[0])
        return task_id
    raise HTTPException(status_code=404, detail="Task not found")


# coding=utf8

"""
剑指 Offer 32 - II. 从上到下打印二叉树 II
https://leetcode-cn.com/problems/cong-shang-dao-xia-da-yin-er-cha-shu-ii-lcof/
"""
import collections
from typing import List

class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution:
    def levelOrder(self, root: TreeNode) -> List[List[int]]:
        if not root:
            return []
        res = []
        q = collections.deque()
        q.append(root)
        while q:
            arr = []
            for i in range(len(q)):
                node = q.popleft()
                arr.append(node.val)
                if node.left:
                    q.append(node.left)
                if node.right:
                    q.append(node.right)
            res.append(arr)
        return res


class Node:
    def __init__(self,data):
        self.data = data
        self.next = None

class LinkedList:
    def __init__(self):
        self.head = None

    def push(self,new_data):

        new_node = Node(new_data)
        new_node.next = self.head
        self.head = new_node

    def search(self,key):
        while self.head != None:
            if self.head.data == key:
                return True
            self.head = self.head.next

        return False

if __name__ == '__main__':

    # Start with the empty list
    llist = LinkedList()
    llist.push(10)
    llist.push(30)
    llist.push(11)
    llist.push(21)
    llist.push(14)

    # Function call
    if llist.search(15):
        print("Yes")
    else:
        print("No")


# -*- coding = 'utf-8' -*-
'''
#@Time: 2022/7/2 17:52
#@Author: TephrocactusHC
#@File: ceo.py
#@Project: NKUSpider
#@Software: PyCharm
'''

import pandas as pd
from bs4 import BeautifulSoup as bs
import requests
import re

urls = ['https://ceo.nankai.edu.cn/szll/xdgxyjs.htm', 'https://ceo.nankai.edu.cn/szll/gdzbmqjyjsyjs.htm',
        'https://ceo.nankai.edu.cn/szll/wdzgcx.htm', 'https://ceo.nankai.edu.cn/szll/dzkxygcx.htm',
        'https://ceo.nankai.edu.cn/szll/dzxxgcx.htm', 'https://ceo.nankai.edu.cn/szll/txgcx.htm',
        'https://ceo.nankai.edu.cn/szll/dzxxsyjxzx.htm']
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
root_url = 'https://ceo.nankai.edu.cn/szll/'

def find_teacher(url):
    response = requests.get(root_url + url, headers=headers)
    response.encoding = 'utf-8'
    result = re.findall('邮箱.*?\s*(.*?)\s*', response.text, re.S)
    return result

for url in urls:
    thelist = []
    table = pd.read_html(url)[0]
    response = requests.get(url, headers=headers)
    response.encoding = 'utf-8'
    page = bs(response.text, 'html.parser')
    teacher_list = page.select('tbody tr td a')
    for teacher in teacher_list:
        email_list = find_teacher(teacher['href'])
        email_list = list(set(email_list))
        thelist.append(email_list)
    table['邮箱'] = thelist
    print(table)
    if url == 'https://ceo.nankai.edu.cn/szll/xdgxyjs.htm':
        table.to_csv('ceo.csv', mode='a', index=False)
    else:
        table.to_csv('ceo.csv', mode='a', index=False, header=False)


import torch
from pygranso.pygransoStruct import pygransoStruct

def user_fn(X_struct,A,d,device,folding_type):
    assert folding_type in [None, "l2"], "unidentified folding type for PyGranso."
    V = X_struct.V
    # objective function
    f = -torch.trace(V.T@A@V)
    # inequality constraint, matrix form
    ci = None
    # equality constraint
    ce = pygransoStruct()
    constr_vec = (V.T@V - torch.eye(d).to(device=device, dtype=torch.double)).reshape(d**2,1)
    # if folding_type == 'l1':
    #     ce.c1 = torch.sum(torch.abs(constr_vec))
    # elif folding_type == 'l2':
    #     ce.c1 = torch.sum(constr_vec**2)**0.5
    # elif folding_type == 'linf':
    #     ce.c1 = torch.amax(torch.abs(constr_vec))
    if folding_type == 'l2':
        ce.c1 = torch.sum(constr_vec**2)**0.5
    else: # unfolded
        ce.c1 = V.T@V - torch.eye(d).to(device=device, dtype=torch.double)

    return [f,ci,ce]

def opts_init(device,pygranso_config,final_obj_list,data_idx,n,d):
    opts = pygransoStruct()
    opts.torch_device = device
    opts.print_frequency = 10
    opts.maxit = pygranso_config["maxit"]
    opts.print_use_orange = False
    opts.print_ascii = True
    opts.quadprog_info_msg  = False
    opts.opt_tol = pygranso_config["opt_tol"]
    opts.viol_eq_tol = pygranso_config["viol_eq_tol"]
    opts.maxclocktime = pygranso_config["maxclocktime"]
    opts.mu0 = pygranso_config["mu0"]
    opts.fvalquit = final_obj_list[data_idx]*pygranso_config["threshold"]
    opts.x0 =  torch.randn((n*d,1)).to(device=device, dtype=torch.double)
    opts.x0 = opts.x0/torch.linalg.norm(opts.x0)
    return opts


import os

from flask.ext.wtf import Form
from wtforms import StringField, TextAreaField, BooleanField, IntegerField, SubmitField
from wtforms.validators import DataRequired, Optional, ValidationError

from .gitapi import GitAPI

def repo_path_validate(form, field):
    repo_path = field.data
    if not os.path.isdir(repo_path):
        raise ValidationError('Absolute path to the repository is not exist')
    API = GitAPI(repo_path)
    if not API.is_repo():
        raise ValidationError('There is not GIT repo in {}'.format(repo_path))

def ignore_pathes_validate(form, field):
    repo_path = form.repo_path.data
    ignore_pathes = field.data.splitlines()
    if not ignore_pathes:
        return
    for path in ignore_pathes:
        if path[0] == '/':
            raise ValidationError('Path %s is not relative' % path)

def min_days_validate(form, field):
    min_days = field.data
    if min_days < 0:
        raise ValidationError('Minimal working days value should be more then zero')

def min_loc_validate(form, field):
    min_loc = field.data
    if min_loc < 0:
        raise ValidationError('Minimal LOC value should be more then zero')

class ConfigForm(Form):
    repo_path = StringField('Absolute path to the repository', [DataRequired('Field is required'), repo_path_validate])
    ignore_pathes = TextAreaField('List of ignored folders and files', [ignore_pathes_validate])
    min_days = IntegerField('Minimal working days', [Optional(), min_days_validate])
    min_loc = IntegerField('Minimal LOC', [Optional(), min_loc_validate])
    anonym = BooleanField('Anonym mode')
    submit = SubmitField('Get Rate')


from random import randint
from scipy.ndimage import gaussian_filter as gauss_filt
import matplotlib.pyplot as plt
import numpy as np
from glob import glob
import os

def readFloatRandomPatches(fileName, width=1, num_sample=1, patch_size=1, rows=None, cols=None, height=None):
    with open(fileName, "rb") as fin:
        if rows is None:
            rows = np.random.randint(0, high=(height - patch_size), size=num_sample)
            cols = np.random.randint(0, high=(width - patch_size), size=num_sample)
            size_of_file = os.path.getsize(fileName)
            height = size_of_file / 4 / width
        patches = []
        for i in range(len(rows)):
            row = rows[i]
            col = cols[i]
            img = []
            for p_row in range(patch_size):
                fin.seek(4 * (width * (row + p_row) + col))
                img.append(np.frombuffer(fin.read(4 * patch_size), dtype=">f4").astype(np.float))
            patches.append(np.reshape(img, [patch_size, patch_size]))
    return patches, rows, cols, height

def wrap(x):
  return np.angle(np.exp(1j*x))

def rotate_grid(x,y,theta=0,p1=[0,0]):
  c = np.cos(theta)
  s = np.sin(theta)
  x_prime = (x-p1[0])*c-(y-p1[1])*s
  y_prime = (x-p1[0])*s+(y-p1[1])*c
  return x_prime, y_prime

def eval_2d_gauss(x,y,params):
  amp,xm,ym,sx,sy,theta = params
  a = np.cos(theta)**2./2./sx/sx+np.sin(theta)**2./2./sy/sy
  b = -np.sin(2*theta)/4./sx/sx+np.sin(2*theta)/4./sy/sy
  c = np.sin(theta)**2./2./sx/sx+np.cos(theta)**2./2./sy/sy
  return amp*np.exp(-(a*(x-xm)**2.+2.*b*(x-xm)*(y-ym)+c*(y-ym)**2.))

def eval_3d_ellipsoid(x,y,params):
  a, b, c, x_off, y_off = params
  x1 = x-x_off
  y1 = y-y_off
  goods =  (x1**2./b**2. + y1**2./c**2.) <= 1.0
  ellipse = np.zeros_like(x1)
  ellipse[goods] = a*np.sqrt(1 - x1[goods]**2./b**2. - y1[goods]**2./c**2.)
  return ellipse

def eval_3d_polygon(x,y,params):
  x_off, y_off, Ps, n, a, angels = params
  x1 = x-x_off
  y1 = y-y_off
  angels_0 = angels[0]
  angles = wrap(np.arctan2(y1,x1)-angels_0)
  angles[angles<0] = 2.*np.pi + angles[angles<0]
  for i in range(len(angels)):
    angels[i] -= angels_0
    #print angels[i]
  polygon = np.zeros_like(x1)
  zeres = np.zeros_like(x1)
  for f in range(1,n):
    x2, y2 = Ps[f-1][0], Ps[f-1][1]
    x3, y3 = Ps[f][0], Ps[f][1]
    ang1 = angels[f-1]
    ang2 = angels[f]
    if ang2-ang1<np.pi:
      goods = (angles>=ang1) & (angles<ang2)
      polygon[goods] += np.maximum(zeres[goods],a-((-y2*a+y3*a)*x1[goods]+(x2*a-x3*a)*y1[goods])/(x2*y3-x3*y2))
  x2, y2 = Ps[-1][0], Ps[-1][1]
  x3, y3 = Ps[0][0], Ps[0][1]
  ang1 = angels[-1]
  ang2 = 2.*np.pi
  if ang2-ang1<np.pi:
    goods = (angles>=ang1) & (angles<ang2)
    polygon[goods] += np.maximum(zeres[goods],a-((-y2*a+y3*a)*x1[goods]+(x2*a-x3*a)*y1[goods])/(x2*y3-x3*y2))
  return polygon

def eval_2d_building(x,y,input_mask,params):
  w,h,d,px,py = params
  x1 = x-px
  y1 = y-py
  wedge_mask = (np.abs(x1) <= w/2.) & (np.abs(y1) <= h/2.) & (input_mask)
  wedge = np.zeros_like(x1)
  wedge[wedge_mask] = -d/w*x1[wedge_mask] + d/2.
  return wedge, wedge_mask

def generate_band_mask(width,height,thickness=1):
  screen = gauss_filt(np.random.normal(0, 500., (height, width)), 12.)
  return (screen<thickness) & (screen>-thickness)

class IfgSim():
  """stores simulated data with and without noise and allows to add specific types of signals

  Attributes:
    width and height:
    rayleigh_scale: all amplitudes are randomly drawn according to this parameter
    signal_gauss_bubbles: the phase signal
    signal: the phase signal
    signal_buildings: the phase signal model parameters
    signal_faults: the phase signal model parameters
    signal: the phase signal model parameters
    amp1: the underlying amplitude of slc1
    amp2: the underlying amplitude of slc2
    slc1: a phasor of zero phase and amp1 amplitude
    slc2: a phasor of signal phase and amp2 amplitude
    noise1: complex normal gaussian added to slc1
    noise2: complex normal gaussian added to slc2
    ifg: ifg = (slc1+noise1)*np.conj(slc2+noise2)
    x,y: 2D arrays with the x and y indices
  """

  def __init__(self,width,height,rayleigh_scale=1.0):
    np.random.seed(np.random.randint(1,1000000+1))
    self.width = width
    self.height = height
    self.rayleigh_scale = rayleigh_scale
    self.x, self.y = np.meshgrid(range(self.width),range(self.height))
    self.x = self.x.astype(np.float)
    self.y = self.y.astype(np.float)
    self.signal = np.zeros((height,width))
    self.signal_gauss_bubbles = []
    self.signal_ellipses = []
    self.signal_polygons = []
    self.signal_buildings = []
    self.signal_faults = []
    self.signal_band = []
    amp = gauss_filt(np.random.rayleigh(self.rayleigh_scale,(self.height,self.width)),10)
    self.amp1 = amp.copy()
    self.amp2 = amp.copy()
    self.slc1 = np.exp(1j*np.zeros((height,width)))
    self.slc2 = np.zeros((height,width)).astype(np.complex)
    self.noise1 = np.zeros((height,width)).astype(np.complex)
    self.noise2 = np.zeros((height,width)).astype(np.complex)
    self.ifg = np.zeros((height,width)).astype(np.complex)
    self.noisy_ifg = np.zeros((height,width)).astype(np.complex)
    self.add_random_dem_signal_flag = False
    self.dem_scale = 1.0

  def add_random_dem_signal(self, scale=1.0):
    self.add_random_dem_signal_flag = True
    self.dem_scale = scale

  def add_gauss_bubble(self, sigma_range=[20,300], amp_range=[-1,1]):
    """
    :param sigma_range: the range of spatial scales for the gaussians
    :param amp_range: the range of amplitudes for the gaussians
    """
    amp = (np.random.random()*(amp_range[1]-amp_range[0])+amp_range[0])
    x_mean = float(np.random.randint(int(0),int(self.width-1)))
    y_mean = float(np.random.randint(int(0),int(self.height-1)))
    x_std = (np.random.random()*(sigma_range[1]-sigma_range[0])+sigma_range[0])
    y_std = (np.random.random()*(sigma_range[1]-sigma_range[0])+sigma_range[0])
    theta = np.random.random()*2.*np.pi-np.pi # rotate the gaussian by a random angle
    self.signal_gauss_bubbles.append((amp, x_mean, y_mean, x_std, y_std, theta))

  def add_n_gauss_bubbles(self, sigma_range=[20,300], amp_range=[-1,1], nps=100):
    """
    :param sigma_range: the range of spatial scales for the gaussians
    :param amp_range: the range of amplitudes for the gaussians
    :param nps: number of random gaussians
    """
    for i in range(nps):
      self.add_gauss_bubble(sigma_range, amp_range)

  def add_ellipse(self, z_range=[1,10], x_range=[1,10], y_range=[1,10]):
    """
    :param z_range: the range of heights for the ellipses
    :param x_range: the range of radii for the first axis of the ellipses
    :param y_range: the range of radii for the second axis of the ellipses
    """
    a = (np.random.random()*(z_range[1]-z_range[0])+z_range[0])
    b = (np.random.random()*(x_range[1]-x_range[0])+x_range[0])
    c = (np.random.random()*(y_range[1]-y_range[0])+y_range[0])
    x_off = float(np.random.randint(int(0),int(self.width-1)))
    y_off = float(np.random.randint(int(0),int(self.height-1)))
    self.signal_ellipses.append((a, b, c, x_off, y_off))

  def add_n_ellipses(self, z_range=[1,10], x_range=[1,10], y_range=[1,10], nps=100):
    """
    :param z_range: the range of heights for the ellipses
    :param x_range: the range of radii for the first axis of the ellipses
    :param y_range: the range of radii for the second axis of the ellipses
    :param nps: number of random gaussians
    """
    for i in range(nps):
      self.add_ellipse(z_range, x_range, y_range)

  def add_polygon(self, z_range=[1,10], r_range=[0,100], n_range=[3,10]):
    """
    :param z_range: the range of heights for the polygons
    :param r_range: the range of radii for the polygon edges
    :param n_range: the range of number of edges for the polygons
    """
    angel = np.random.rand()*2.*np.pi
    n = np.random.randint(n_range[0],n_range[1]+1)
    x_off = float(np.random.randint(0,self.width))
    y_off = float(np.random.randint(0,self.height))
    Ps = []
    angels = []
    for j in range(n):
      r = np.random.random()*(r_range[1]-r_range[0])+r_range[0]
      Ps.append([np.cos(angel)*r,np.sin(angel)*r])
      angels.append(angel)
      angel = np.min([angels[0]+2.*np.pi,angel+np.random.rand()*(2.*np.pi/(n-1.))])
    angels[-1] = np.max([angels[0]+2.*np.pi-np.random.rand()*np.pi,angels[-1]])
    Ps[-1] = [np.cos(angels[-1])*r,np.sin(angels[-1])*r]
    a = (np.random.random()*(z_range[1]-z_range[0])+z_range[0])
    self.signal_polygons.append((x_off,y_off,Ps,n,a,angels))

  def add_n_polygons(self, z_range=[1,10], r_range=[0,100], n_range=[3,10], nps=100):
    """
    :param z_range: the range of heights for the polygons
    :param r_range: the range of radii for the polygon edges
    :param n_range: the range of number of edges for the polygons
    :param nps: number of random gaussians
    """
    for i in range(nps):
      self.add_polygon(z_range, r_range, n_range)

  def add_building(self, width_range=[10,100], height_range=[10,100], depth_factor=0.2):
    """
    :param width_range: range of wedge widths
    :param height_range: range of wedge heights
    :param depth_factor: the height of the building is proportional to the width of the wedge by this factor
    """
    w = (np.random.random()*(width_range[1]-width_range[0])+width_range[0])
    h = (np.random.random()*(height_range[1]-height_range[0])+height_range[0])
    d = w*depth_factor
    px = float(randint(int(0),int(self.width-1)))
    py = float(randint(int(0),int(self.height-1)))
    amp = np.random.rayleigh(self.rayleigh_scale)
    self.signal_buildings.append((-px+w/2,amp,w,h,d,px,py))

  def add_n_buildings(self, width_range=[10,100], height_range=[10,100], depth_factor=0.2, nps=100):
    """
    :param width_range: range of wedge widths
    :param height_range: range of wedge heights
    :param depth_factor: the height of the building is proportional to the width of the wedge by this factor
    :param nps: number of buildings to add
    """
    for i in range(nps):
      self.add_building(width_range, height_range, depth_factor)

  def add_amp_stripe(self, thickness=1):
    """ alters the amplitude in a band region (excluding buildings)
    :param thickness: approximate thickness of the bands
    """
    amplitude = np.random.rayleigh(self.rayleigh_scale)
    mask = generate_band_mask(self.width,self.height,thickness)
    self.amp1[mask] = amplitude
    self.amp2[mask] = amplitude

  def add_phase_strip(self, amp_scale=1, thickness=1):
    mask = generate_band_mask(self.width,self.height,thickness)
    self.signal_band.append((mask, amp_scale))

  def add_n_amp_stripes(self, thickness=1, nps=5):
    """
    :param thickness: approximate thickness of the bands
    :param amplitude: new amplitude in the bands
    :param nps: number of bands to add
    """
    for i in range(nps):
      self.add_amp_stripe(thickness)

  def add_n_phase_stripes(self, thickness=1, nps=5, amp_range=[-1, 1]):
    amps = np.random.uniform(amp_range[0], amp_range[1], nps)
    for i in range(nps):
      self.add_phase_strip(amps[i], thickness)

  def compile(self):
    """ takes all the model parameters and generates the signals in the amplitude and phase based on them
    """

    # first add the gaussian bubbles
    self.signal = np.zeros((self.height,self.width))

    if self.add_random_dem_signal_flag:
      assert(self.width==self.height)
      dems = glob('/disk/tembofallback/c003/scratch/azimmer/rdc_dems/*')
      idx = np.random.randint(len(dems))
      rdc_dem = dems[idx]
      dem_width = int(rdc_dem.split('.')[-2])
      dem_height = int(rdc_dem.split('.')[-1])
      dem, rs, cs, h = readFloatRandomPatches(rdc_dem, width=dem_width, patch_size=self.width, num_sample=1, height=dem_height)
      self.signal += dem[0] * np.random.rand() * self.dem_scale
      #TODO: do the same thing with the average rmli for the amplitudes

    for params in self.signal_gauss_bubbles:
      self.signal += eval_2d_gauss(self.x, self.y, params)

    for params in self.signal_ellipses:
      self.signal += eval_3d_ellipsoid(self.x, self.y, params)*(-1 if np.random.rand()>0.5 else 1)

    for params in self.signal_polygons:
      tmp = eval_3d_polygon(self.x, self.y, params)*(-1 if np.random.rand()>0.5 else 1)
      if np.random.rand()>0.5:
        tmp = np.transpose(tmp)
      self.signal += tmp

    for params in self.signal_band:
      self.signal[params[0]] += params[1]

    # then add the buildings
    vacant_lots = np.ones((self.height,self.width)).astype(np.bool)
    for params in sorted(self.signal_buildings):
      _,amp,w,h,d,px,py = params
      #print params
      cur_building, cur_building_mask = eval_2d_building(self.x,self.y,vacant_lots,(w,h,d,px,py))
      self.signal[cur_building_mask] += cur_building[cur_building_mask]
      self.amp1[cur_building_mask] = amp
      self.amp2[cur_building_mask] = amp
      vacant_lots = (vacant_lots) & (cur_building_mask==False)

  def update(self, sigma=0.2, sigma_correlated = 0.0):
    """ combines the simulated amplitudes and phases with a constant amount of noise to be added to each slc pixel
    """
    self.compile()
    self.slc1 = self.amp1*self.slc1
    self.slc2 = self.amp2*np.exp(1j*(self.signal))
    self.noise1 = np.random.normal(0, sigma, (self.height, self.width)) + 1j*np.random.normal(0, sigma, (self.height, self.width))
    self.noise2 = np.random.normal(0, sigma, (self.height, self.width)) + 1j*np.random.normal(0, sigma, (self.height, self.width))
    self.noise12 = 0.0 if sigma_correlated==0 else np.random.normal(0, sigma_correlated, (self.height, self.width)) + 1j*np.random.normal(0, sigma_correlated, (self.height, self.width))
    self.ifg = (self.slc1)*np.conj(self.slc2)
    self.ifg_noisy = (self.slc1+self.noise1+self.noise12)*np.conj(self.slc2+self.noise2+self.noise12)

def get_coh_dict(amp_range=[0.0,10.0,10000],sigma=0.2,sigma_correlated=0.0,N=100000):
  indices = np.arange(amp_range[2])
  amps = indices*(amp_range[1]-amp_range[0])/(amp_range[2]-1.)+amp_range[0]
  cohs = []
  for amp in amps:
    noise1 = np.random.normal(0, sigma, N) + 1j*np.random.normal(0, sigma, N)
    noise2 = np.random.normal(0, sigma, N) + 1j*np.random.normal(0, sigma, N)
    noise12 = 0.0 if sigma_correlated==0 else np.random.normal(0, sigma_correlated, N) + 1j*np.random.normal(0, sigma_correlated, N)
    slc1 = amp*np.exp(1j*1) + noise1 + noise12
    slc2 = amp*np.exp(1j*1) + noise2 + noise12
    cohs.append(np.abs(np.sum(slc1*np.conj(slc2))/np.sqrt(np.sum(np.abs(slc1)**2.)*np.sum(np.abs(slc2)**2.))))
  cohs = np.array(cohs)
  return cohs, amps

def get_coherence(amps,cohs,amp_range=[0.0,10.0,10000]):
  indices = ((amps - amp_range[0])*(amp_range[2] - 1.)/(amp_range[1]-amp_range[0])).astype(np.int)
  return cohs[indices]

def example_1():
  sim = IfgSim(width=300, height=300, rayleigh_scale=0.9)
  sim.add_n_buildings(width_range=[10,100], height_range=[1,40], depth_factor=0.35, nps=25)
  sim.add_n_gauss_bubbles(sigma_range=[10,150], amp_range=[-4.5,4.5], nps=110)
  sim.add_n_amp_stripes(thickness=9, nps=5)
  sim.add_n_amp_stripes(thickness=3, nps=50)
  sim.update(sigma=0.5)
  plt.figure(); plt.imshow(np.angle(sim.ifg),interpolation="None"); plt.colorbar()
  plt.figure(); plt.imshow(np.angle(sim.ifg_noisy),interpolation="None"); plt.colorbar()
  plt.figure(); plt.imshow(np.abs(sim.ifg),interpolation="None"); plt.colorbar()
  plt.figure(); plt.imshow(np.abs(sim.ifg_noisy),interpolation="None"); plt.colorbar()

def example_coh():
  sim = IfgSim(width=300, height=300, rayleigh_scale=0.9)
  sim.add_n_buildings(width_range=[10,100], height_range=[1,40], depth_factor=0.35, nps=25)
  sim.add_n_gauss_bubbles(sigma_range=[10,150], amp_range=[-4.5,4.5], nps=110)
  sim.add_n_amp_stripes(thickness=9, nps=5)
  sim.add_n_amp_stripes(thickness=3, nps=50)
  sim.update(sigma=0.5)
  amp_range=[np.min(sim.amp1),np.max(sim.amp1),1000]
  coh_array, amp_array = get_coh_dict(amp_range=amp_range,sigma=0.5,N=100000)
  sim_cohs = get_coherence(sim.amp1,coh_array,amp_range=amp_range)
  plt.figure(); plt.imshow(np.angle(sim.ifg),interpolation="None"); plt.colorbar()
  plt.figure(); plt.imshow(np.angle(sim.ifg_noisy),interpolation="None"); plt.colorbar()
  plt.figure(); plt.imshow(np.abs(sim.ifg),interpolation="None"); plt.colorbar()
  plt.figure(); plt.imshow(np.abs(sim.ifg_noisy),interpolation="None"); plt.colorbar()
  plt.figure(); plt.imshow(sim_cohs,interpolation="None",cmap="Greys_r"); plt.colorbar()


def findNthSpokenWord(starter, n):
  lastSeenIndexes = dict();

  for i in range(0, len(starter) - 1):
    lastSeenIndexes[starter[i]] = i;

  print(lastSeenIndexes)

  lastNumber = starter[len(starter) - 1]
  for i in range(len(starter), n):
    lastSeenAt = lastSeenIndexes[lastNumber] if lastNumber in lastSeenIndexes else -1
    currentValue = i - 1 - lastSeenAt if lastSeenAt >= 0 else 0
    lastSeenIndexes[lastNumber] = i - 1
    lastNumber = currentValue

  return lastNumber

def main():
  input = [14, 3, 1, 0, 9, 5]

  print('--Part 1--')
  number2020 = findNthSpokenWord(input, 2020)
  print(number2020)

  print("--Part 2--")
  number30000000 = findNthSpokenWord(input, 30000000)
  print(number30000000)

main()


class Solution:
    def insert(self,head,data):
            p = Node(data)
            if head==None:
                head=p
            elif head.next==None:
                head.next=p
            else:
                start=head
                while(start.next!=None):
                    start=start.next
                start.next=p
            return head
    def display(self,head):
        current = head
        while current:
            print(current.data,end=' ')
            current = current.next

    def removeDuplicates(self,head):
        #Write your code here
        node = head
        follower=head
        while(node.next!=None):
            if(node.data == node.next.data):
                node.next=node.next.next
            else:
                node=node.next
        return head