import os
import sys
from functools import reduce

import numpy as np
import pandas as pd
from colorama import Fore
from matplotlib import pyplot as plt
from tqdm import tqdm

# from Data.TcrDataset import TCRDataset
# from TCR_Ofek import TcrDataset
for path_name in [os.path.join(os.path.dirname(__file__)),
                  os.path.join(os.path.dirname(__file__), 'Data'),
                  os.path.join(os.path.dirname(__file__), 'Missions')]:
    sys.path.append(path_name)
sys.path.append(os.path.join(os.path.dirname(__file__), 'TCR_Ofek'))

sys.path.append(os.path.join(os.path.dirname(__file__), 'TCR_Ofek', "TcrDataset"))
from TcrDataset import TCRDataset
from scipy.stats import norm

from GraphDataset import GraphDataset
# from graph_measures.features_algorithms.vertices.average_neighbor_degree import AverageNeighborDegreeCalculator
# from graph_measures.features_algorithms.vertices.betweenness_centrality import BetweennessCentralityCalculator
# from graph_measures.features_algorithms.vertices.closeness_centrality import ClosenessCentralityCalculator
# from graph_measures.features_algorithms.vertices.clustering_coefficient import ClusteringCoefficientCalculator
# from graph_measures.features_algorithms.vertices.general import GeneralCalculator
# from graph_measures.features_algorithms.vertices.load_centrality import LoadCentralityCalculator
# from graph_measures.features_algorithms.vertices.louvain import LouvainCalculator
# from graph_measures.features_algorithms.vertices.motifs import MotifsNodeCalculator
# from graph_measures.features_infra.feature_calculators import FeatureMeta
# from graph_measures.features_infra.graph_features import GraphFeatures
# from graph_measures.loggers import PrintLogger
import seaborn as sns
from networkx import closeness_centrality, average_neighbor_degree, core_number, betweenness_centrality

plt.rcParams["font.family"] = "Times New Roman"

import networkx as nx

# def add_nodes_attributes(graphs_list):
#     logger = PrintLogger("MyLogger")
#     for graph in graphs_list:
#         features_meta = {
#             "general": FeatureMeta(GeneralCalculator, {"general"}),
#             "average_neighbor_degree": FeatureMeta(AverageNeighborDegreeCalculator, {"nd_avg"}),
#             "louvain": FeatureMeta(LouvainCalculator, {"lov"}),
#             "closeness_centrality": FeatureMeta(ClosenessCentralityCalculator, {"closeness"}),
#             "betweenness_centrality": FeatureMeta(BetweennessCentralityCalculator, {"betweenness"})
#             # ,"motifs": FeatureMeta(MotifsNodeCalculator, {"motifs"}),
#             # "clustering_coefficient": FeatureMeta(ClusteringCoefficientCalculator, {"clustering"})
#         }
#
#         features = GraphFeatures(graph, features_meta, dir_path="stamdir", logger=logger)
#         # _feature_to_dict
#         features.build()
#         mx_dict = features.to_dict()
#         for node, graph_feature_matrix in mx_dict.items():
#             feature_matrix_0 = graph_feature_matrix.tolist()[0]  # the first row in graph_feature_matrix
#             for ind, feature in enumerate(feature_matrix_0):
#                 cur_feature_name = f"feature{ind}"
#                 graph.nodes[node][cur_feature_name] = feature  # add node attributes

def calc_attributes_by_myself(graphs_list):
    graphs_attributes_mean = {}
    for i, graph in enumerate(graphs_list):
        # edge number
        edges_number = graph.number_of_edges()
        # closeness_centrality
        closeness_centrality_dict = closeness_centrality(graph)
        closeness_centrality_df = pd.DataFrame.from_dict(closeness_centrality_dict, orient='index',
                                                         columns=["closeness_centrality"])
        # degree
        degree_dict = {node: val for (node, val) in graph.degree()}
        degree_df = pd.DataFrame.from_dict(degree_dict, orient='index', columns=["degree"])
        # average_neighbor_degree
        average_neighbor_degree_dict = average_neighbor_degree(graph)
        average_neighbor_degree_df = pd.DataFrame.from_dict(average_neighbor_degree_dict, orient='index',
                                                     columns=["average_neighbor_degree"])
        # core_number
        core_number_dict = core_number(graph)
        core_number_df = pd.DataFrame.from_dict(core_number_dict, orient='index',
                                                        columns=["core_number"])
        # betweenness_centrality
        betweenness_centrality_dict = betweenness_centrality(graph)
        betweenness_centrality_df = pd.DataFrame.from_dict(betweenness_centrality_dict, orient='index',
                                            columns=["betweenness_centrality"])
        graphs_attributes_mean[i] = {"closeness_centrality": closeness_centrality_df.mean(axis=0).values[0],
                                     "degree": degree_df.mean(axis=0).values[0],
                                     "average_neighbor_degree": average_neighbor_degree_df.mean(axis=0).values[0],
                                     "core_number": core_number_df.mean(axis=0).values[0],
                                     "betweenness_centrality": betweenness_centrality_df.mean(axis=0).values[0],
                                     "edges_number": edges_number}

    graphs_attributes_mean_df = pd.DataFrame.from_dict(graphs_attributes_mean, orient='index')
    return graphs_attributes_mean_df

def calc_attributes_by_myself_degree_only(dataset_dict, index_list):
    graphs_attributes_mean = {}
    for i, ind in enumerate(index_list):
        # degree
        graph = dataset_dict[ind]["graph"]
        degree_dict = {node: val for (node, val) in graph.degree()}
        degree_df = pd.DataFrame.from_dict(degree_dict, orient='index', columns=["degree"])
        graphs_attributes_mean[i] = {"degree": degree_df.mean(axis=0).values[0]}
    graphs_attributes_mean_df = pd.DataFrame.from_dict(graphs_attributes_mean, orient='index')
    return graphs_attributes_mean_df

# def merge_df_attributes_labels(graphs_attributes_mean_df_0, graphs_attributes_mean_df_1):
#     merged = pd.merge(graphs_attributes_mean_df_0, graphs_attributes_mean_df_1, left_index=True, right_index=True,
#                       how='inner', suffixes=("_0", "_1"))
#     merged = merged.reindex(sorted(merged.columns), axis=1)
#     return merged

def get_graphs_list(dataset_dict):
    graphs_list = []
    for k, v in dataset_dict.items():
        graphs_list.append(v['graph'])
    return graphs_list

def seperate_graphs_list_according_to_its_label(dataset_dict):
    index_list_0, index_list_1 = [], []
    for k, v in dataset_dict.items():
        graph = v['graph']
        label = v['label']
        if label == 0:
            index_list_0.append(k)
        else:
            index_list_1.append(k)
    return index_list_0, index_list_1

def arrange_new_tcr_dataset():
    def load_or_create_tcr_network(subject_list, adj_mat):
        networks_dict = {}
        values_dict = {}
        for i, subject in tqdm(enumerate(subject_list), desc='Create TCR Networks', total=len(subject_list),
                               bar_format="{l_bar}%s{bar}%s{r_bar}" % (Fore.LIGHTGREEN_EX, Fore.RESET)):
            # file_path = os.path.join(self.data_path, f"final_{subject}.csv")
            # tcr_sample_df = pd.read_csv(file_path, index_col=0)
            file_path = os.path.join("TCR_Dataset2", "Train", subject + ".csv")
            samples_df = pd.read_csv(file_path, usecols=["combined", "frequency"])
            no_rep_sample_df = samples_df.groupby("combined").sum()  # sum the repetitions
            golden_tcrs = set(list(adj_mat.index))
            cur_sample_tcrs = set(list(no_rep_sample_df.index))
            intersec_golden_and_sample_tcrs = list(golden_tcrs & cur_sample_tcrs)
            network = adj_mat.copy(deep=True)
            for tcr in list(golden_tcrs):
                if tcr not in intersec_golden_and_sample_tcrs:
                    network[tcr] = np.zeros(network.shape[1])
            for tcr in intersec_golden_and_sample_tcrs:
                network.loc[tcr] = network[tcr]
            networks_dict[subject] = network
            no_rep_sample_df['frequency'] = np.log(no_rep_sample_df['frequency'] + 1e-300)
            tcr_sample_dict = {}
            for tcr in golden_tcrs:
                if tcr in intersec_golden_and_sample_tcrs:
                    tcr_sample_dict[tcr] = no_rep_sample_df.loc[tcr]['frequency']
                else:
                    tcr_sample_dict[tcr] = 0
            values_list = []
            # To make all values among all samples the same order
            for tcr in golden_tcrs:
                values_list.append(tcr_sample_dict[tcr])
            values_dict[subject] = values_list
        return networks_dict, values_dict

    train_data_file_path = os.path.join("TCR_Dataset2", "Train")
    train_tag_file_path = os.path.join("TCR_dataset", "samples.csv")

    label_df = pd.read_csv(train_tag_file_path, usecols=["sample", 'status'])
    label_df["sample"] = label_df["sample"] + "_" + label_df['status']
    label_df.set_index("sample", inplace=True)
    label_df["status"] = label_df["status"].map({"negative": 0, "positive": 1})
    label_dict = label_df.to_dict()['status']

def plot_comparison_between_graph_and_labels(graphs_attributes_mean_df_0, graphs_attributes_mean_df_1, save_fig):
    a, b = graphs_attributes_mean_df_0.shape
    fig, axs = plt.subplots(b, 1, figsize=(25, 25))
    ind = [(i, j) for i in range(b) for j in [0]]
    p = 0
    for col in list(graphs_attributes_mean_df_0.columns):
        # for ax1
        ax1 = axs[p]
        x_0 = graphs_attributes_mean_df_0[col]
        mu_0 = x_0.mean()
        median_0 = np.median(x_0)
        sigma_0 = x_0.std()

        x_1 = graphs_attributes_mean_df_1[col]
        mu_1 = x_1.mean()
        median_1 = np.median(x_1)
        sigma_1 = x_1.std()
        textstr = '\n'.join((
            r'$\mu_0=%.4f$' % (mu_0, ),
            r'$\mu_1=%.4f$' % (mu_1,),
            r'$\sigma_0=%.4f$' % (sigma_0,),
            r'$\sigma_1=%.4f$' % (sigma_1,),
            r'$\mathrm{median}_0=%.4f$' % (median_0, ),
            r'$\mathrm{median}_1=%.4f$' % (median_1,)))

        props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
        ax1.text(0.88, 0.95, textstr, transform=ax1.transAxes, fontsize=15, verticalalignment='top', bbox=props)
        sns.histplot(x_0, stat='probability', ax=ax1, color="k")
        sns.histplot(x_1, stat='probability', ax=ax1, color="y")

        ax1.set_title(col, size=18)
        ax1.set(xlabel=None)
        # for ax2
        # ax2 = axs[ind[p + 1]]
        # x = graphs_attributes_mean_df_1[col]
        # mu = x.mean()
        # median = np.median(x)
        # sigma = x.std()
        #
        # textstr = '\n'.join((
        #     r'$\mu=%.4f$' % (mu, ),
        #     r'$\mathrm{median}=%.4f$' % (median, ),
        #     r'$\sigma=%.4f$' % (sigma, )))
        # props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
        # ax2.text(0.70, 0.95, textstr, transform=ax2.transAxes, fontsize=15, verticalalignment='top', bbox=props)
        # sns.histplot(graphs_attributes_mean_df_1[col], stat='probability', ax=ax2)
        # ax2.set_title(col+"_1", size=18)
        # ax2.set(xlabel=None)
        p += 1
    plt.tight_layout()
    plt.savefig(save_fig + ".jpeg")
    plt.show()

def plot_comparison_between_graph_and_labels_all_datasets(datasets, save_fig):
    fig, axs = plt.subplots(len(datasets), 1, figsize=(25, 45))
    p = 0

    for dataset_name in datasets.keys():
        graphs_attributes_mean_df_0, graphs_attributes_mean_df_1 = load_microbiome_dataset(dataset_name)
        # ind = [(i, j) for i in range(len_datasets) for j in [0]]
        for col in list(graphs_attributes_mean_df_0.columns):
            # for ax1
            print(p)
            ax1 = axs[p]
            x_0 = graphs_attributes_mean_df_0[col]
            mu_0 = x_0.mean()
            median_0 = np.median(x_0)
            sigma_0 = x_0.sem()

            x_1 = graphs_attributes_mean_df_1[col]
            mu_1 = x_1.mean()
            median_1 = np.median(x_1)
            sigma_1 = x_1.sem()
            # textstr = '\n'.join((
            #     r'$\mu_0=%.4f$' % (mu_0, ),
            #     r'$\mu_1=%.4f$' % (mu_1,),
            #     r'$\sigma_0=%.4f$' % (sigma_0,),
            #     r'$\sigma_1=%.4f$' % (sigma_1,)))
            #     # r'$\mathrm{median}_0=%.4f$' % (median_0, ),
            #     # r'$\mathrm{median}_1=%.4f$' % (median_1,)))
            textstr = '\n'.join((f"Class 0: {mu_0:.4f} ± {sigma_0:.4f}", f"Class 1: {mu_1:.4f} ± {sigma_1:.4f}"))
            # _, bins, _ = ax1.hist(x_0, bins=50, range=[0, 1])
            props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
            ax1.text(0.75, 0.95, textstr, transform=ax1.transAxes, fontsize=32, verticalalignment='top', bbox=props)
            # sns.histplot(x_0, stat='probability', ax=ax1, color="k", binrange=(0,1), bins=8)
            # sns.histplot(x_1, stat='probability', ax=ax1, color="y", binrange=(0,1), bins=8)
            bins = np.histogram_bin_edges(x_0, bins=30, range=(0, 1))
            sns.histplot(x_0, stat='probability',bins=bins, ax=ax1, color="r")
            sns.histplot(x_1, stat='probability', bins=bins, ax=ax1, color="b")
            # _ = ax1.hist(x_1, bins=bins, alpha=0.5)

            ax1.tick_params(axis='both', which='major', labelsize=30)

            ax1.set_title(datasets[dataset_name], size=35)
            ax1.set(xlabel=None)
            ax1.set(ylabel=None)

            p += 1
    plt.tight_layout()
    plt.savefig(save_fig + ".jpeg")
    plt.show()

def get_tcr_files():
    train_data_file_path = os.path.join("TCR_Dataset2", "Train")
    train_tag_file_path = os.path.join("TCR_dataset", "samples.csv")
    test_data_file_path = os.path.join("TCR_Dataset2", "Test")
    test_tag_file_path = os.path.join("TCR_dataset", "samples.csv")
    label_df = pd.read_csv(train_tag_file_path)
    label_df["sample"] = label_df["sample"] + "_" + label_df['status']
    label_df.set_index("sample", inplace=True)
    train_subject_list = list(label_df[label_df["test/train"] == "train"].index)
    test_subject_list = list(label_df[label_df["test/train"] == "test"].index)
    return train_data_file_path, train_tag_file_path, train_subject_list

def get_ISB_files():
    train_val_test_data_file_path = os.path.join("covid", "new_ISB")
    train_val_test_label_file_path = os.path.join("covid", "ISB_samples.csv")
    label_df = pd.read_csv(train_val_test_label_file_path)
    label_df["sample"] = label_df["sample"] + "_" + label_df['status']
    label_df.set_index("sample", inplace=True)
    subject_list = list(label_df.index)
    return train_val_test_data_file_path, train_val_test_label_file_path, subject_list

def get_NIH_files():
    train_val_test_data_file_path = os.path.join("covid", "new_NIH")
    train_val_test_label_file_path = os.path.join("covid", "NIH_samples.csv")
    label_df = pd.read_csv(train_val_test_label_file_path)
    label_df["sample"] = label_df["sample"] + "_" + label_df['status']
    label_df.set_index("sample", inplace=True)
    subject_list = list(label_df.index)
    return train_val_test_data_file_path, train_val_test_label_file_path, subject_list

def load_microbiome_dataset(dataset_name):
    mission = "just_values"

    if dataset_name != "TCR":
        origin_dir = os.path.join("Data", "split_datasets_new")
        train_data_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
                                            f'train_val_set_{dataset_name}_microbiome.csv')
        train_tag_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
                                           f'train_val_set_{dataset_name}_tags.csv')

        test_data_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
                                           f'test_set_{dataset_name}_microbiome.csv')
        test_tag_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
                                          f'test_set_{dataset_name}_tags.csv')
        data_path = train_data_file_path
        label_path = train_tag_file_path

        add_attributes, geometric_mode = False, False
        cur_dataset = GraphDataset(data_path, label_path, mission, add_attributes, geometric_mode)
    else:
        train_data_file_path, train_tag_file_path, train_subject_list = get_tcr_files()
        cur_dataset = TCRDataset(dataset_name, train_data_file_path, train_tag_file_path, train_subject_list, mission)
        adj_mat_path = "new_new_graph_type_corr_tcr_corr_mat_125_with_sample_size_547_run_number_1_mission_concat_graph_and_values"
        cur_dataset.calc_golden_tcrs(adj_mat_path=adj_mat_path)

    cur_dataset.update_graphs()
    graphs_list_0, graphs_list_1 = seperate_graphs_list_according_to_its_label(cur_dataset.dataset_dict)
    graphs_attributes_mean_df_0 = calc_attributes_by_myself_degree_only(graphs_list_0)
    graphs_attributes_mean_df_1 = calc_attributes_by_myself_degree_only(graphs_list_1)
    return graphs_attributes_mean_df_0, graphs_attributes_mean_df_1

def df_values(dataset_dict, index_list):
    # For tcr dataset only
    dataframes_list = []
    for ind in index_list:
        cur_values = dataset_dict[ind]["values"]
        cur_values = [(a, b) for a, b in enumerate(cur_values)]
        cur_df = pd.DataFrame(cur_values, columns=['bacteria', f'val_{ind}'])
        cur_df.set_index("bacteria", inplace=True)
        dataframes_list.append(cur_df)
    df_merged = reduce(lambda left, right: pd.merge(left, right, left_index=True, right_index=True, how='inner'),
                       dataframes_list)
    df_merged.sort_index(inplace=True)
    return df_merged

def find_epsilon_sigma(df_values_0, df_values_1, df_degrees_0, df_degrees_1):
    mu1, std1 = norm.fit(df_degrees_0.mean(axis=1).values)
    mu2, std2 = norm.fit(df_degrees_1.mean(axis=1).values)
    epsilon = (mu1 - mu2) / ((mu1 + mu2) / 2)
    # mu1, std1 = norm.fit(df_values_0.mean(axis=1).values)
    # mu2, std2 = norm.fit(df_values_1.mean(axis=1).values)
    print("Check if the features are aligned", list(df_values_0.index) == list(df_values_1.index))
    sigma = np.std(df_values_0.mean(axis=1) - df_values_1.mean(axis=1))
    print("epsilon", epsilon)
    print("sigma", sigma)
    return sigma, epsilon

def sigma_epsilon_for_tcr():
    train_data_file_path, train_tag_file_path, train_subject_list = get_tcr_files()
    cur_dataset = TCRDataset("TCR", train_data_file_path, train_tag_file_path, train_subject_list, "just_values")
    adj_mat_path = "new_new_graph_type_corr_tcr_corr_mat_125_with_sample_size_547_run_number_1_mission_concat_graph_and_values"
    cur_dataset.calc_golden_tcrs(adj_mat_path=adj_mat_path)

    cur_dataset.update_graphs()
    index_list_0, index_list_1 = seperate_graphs_list_according_to_its_label(cur_dataset.dataset_dict)
    df_values_0 = df_values(cur_dataset.dataset_dict, index_list_0)
    df_values_1 = df_values(cur_dataset.dataset_dict, index_list_1)
    graphs_attributes_mean_df_0 = calc_attributes_by_myself_degree_only(cur_dataset.dataset_dict, index_list_0)
    graphs_attributes_mean_df_1 = calc_attributes_by_myself_degree_only(cur_dataset.dataset_dict, index_list_1)
    sigma, epsilon = find_epsilon_sigma(df_values_0, df_values_1, graphs_attributes_mean_df_0, graphs_attributes_mean_df_1)
    print(sigma, epsilon)

if __name__ == "__main__":
    # dataset_name = "bw"
    # origin_dir = os.path.join("Data", "split_datasets_new")
    # train_data_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
    #                                     f'train_val_set_{dataset_name}_microbiome.csv')
    # train_tag_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
    #                                    f'train_val_set_{dataset_name}_tags.csv')
    #
    # test_data_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
    #                                    f'test_set_{dataset_name}_microbiome.csv')
    # test_tag_file_path = os.path.join(origin_dir, f'{dataset_name}_split_dataset',
    #                                   f'test_set_{dataset_name}_tags.csv')
    #
    # # train_data_file_path = os.path.join("covid", f"new_{dataset_name}")
    # # train_tag_file_path = os.path.join("covid", f"{dataset_name}_samples.csv")
    #
    # data_path = train_data_file_path
    # label_path = train_tag_file_path
    # # data_path, label_path, subject_list = get_tcr_files()
    # # data_path, label_path, subject_list = get_ISB_files()
    # # data_path, label_path, subject_list = get_NIH_files()
    #
    # add_attributes, geometric_mode = False, False
    # mission = "just_values"
    # cur_dataset = GraphDataset(data_path, label_path, mission, add_attributes, geometric_mode)
    # # label_df = pd.read_csv(train_tag_file_path)
    # # label_df["sample"] = label_df["sample"] + "_" + label_df['status']
    # # label_df.set_index("sample", inplace=True)
    # # subject_list = list(label_df.index)
    #
    # # cur_dataset = TCRDataset(dataset_name, data_path, label_path, subject_list, mission)
    # # adj_mat_path = "dist_mat_with_sample_size_135_run_number_0"
    # # cur_dataset.calc_golden_tcrs(adj_mat_path=adj_mat_path)
    # cur_dataset.update_graphs()
    # graphs_list_0, graphs_list_1 = seperate_graphs_list_according_to_its_label(cur_dataset.dataset_dict)
    # graphs_attributes_mean_df_0 = calc_attributes_by_myself(graphs_list_0)
    # # graphs_attributes_mean_df_0.to_csv("graphs_attributes_mean_df_0.csv")
    # graphs_attributes_mean_df_1 = calc_attributes_by_myself(graphs_list_1)
    # # graphs_attributes_mean_df_1.to_csv("graphs_attributes_mean_df_1.csv")
    #
    # # graphs_attributes_mean_df_0 = pd.read_csv("graphs_attributes_mean_df_0.csv", index_col=0)
    # # graphs_attributes_mean_df_1 = pd.read_csv("graphs_attributes_mean_df_1.csv", index_col=0)
    # save_fig = f"{dataset_name}_comparison_graphs_attributes"
    # plot_comparison_between_graph_and_labels(graphs_attributes_mean_df_0, graphs_attributes_mean_df_1, save_fig)
    # # add_nodes_attributes(graphs_list)

    datasets = ["Cirrhosis", "IBD", "bw", "IBD_Chrone", "male_female", "nugent", "milk", "nut", "peanut", "TCR"]
    datasets_dict = {"Cirrhosis":"Cirrhosis", "IBD":"IBD", "bw":"CA", "IBD_Chrone":"IBD_Chrone",
                     "male_female":"Male Female", "nugent":"Nugent", "milk":"Milk", "nut":"Nut", "peanut":"Peanut", "TCR":"TCR"}

    # datasets = ["Cirrhosis", "IBD"]
    # datasets_dict = {"Cirrhosis":"Cirrhosis", "IBD":"IBD"}
    # datasets = ["TCR", "Cirrhosis"]
    # datasets_dict = {"TCR":"TCR", "Cirrhosis":"Cirrhosis"}

    save_fig = "degree_distribution_compare_tcr"
    # plot_comparison_between_graph_and_labels_all_datasets(datasets_dict, save_fig)
    # corr_df_between_golden_tcrs = pd.read_csv("new_graph_type_corr_tcr_corr_mat_125_with_sample_size_547_run_number_1_mission_concat_graph_and_values.csv",
    #                                           index_col=0)
    # corr_df_between_golden_tcrs.values[[np.arange(corr_df_between_golden_tcrs.shape[0])] * 2] = 0
    # new_corr_df_between_golden_tcrs = (np.abs(corr_df_between_golden_tcrs) >= 0.2).astype(int)
    # new_corr_df_between_golden_tcrs.to_csv("new_new_graph_type_corr_tcr_corr_mat_125_with_sample_size_547_run_number_1_mission_concat_graph_and_values.csv")
    sigma_epsilon_for_tcr()


# build.py
# pre-build script, setting up build environment and fetch hal file for user's board

import sys
import os
import os.path
import requests
from os.path import basename
from platformio import util
from SCons.Script import DefaultEnvironment

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

# get platformio environment variables
env = DefaultEnvironment()
config = configparser.ConfigParser()
config.read("platformio.ini")

# get platformio project directory
prjdir = env.get("PROJECT_DIR")

# get platformio shared path
shareddir = os.path.join (prjdir, "shared")

# get hal path
haldir = os.path.join (shareddir, "hal")

# check if hal file is present in source directory
halconfig = config.get("board", "halfile")

# check if hal file is set by ENV CI_HALFILE
if os.getenv("CI_HALFILE"):
    # if set, use ENV CI_HALFILE
    print("CI_HALFILE ENV FOUND")
    # override halconfig
    halconfig = os.getenv("CI_HALFILE")

halconfigfile = os.path.join (haldir, halconfig)

if os.path.isfile(halconfigfile) and os.access(halconfigfile, os.R_OK):
    print("Parsing hardware configuration from " + halconfigfile)
else:
    sys.exit("Missing file " + halconfigfile + ", please create it! Aborting.")

# check if lmic config file is present in shared directory
lmicconfig = config.get("common", "lmicconfigfile")
lmicconfigfile = os.path.join (shareddir, lmicconfig)
if os.path.isfile(lmicconfigfile) and os.access(lmicconfigfile, os.R_OK):
    print("Parsing LMIC configuration from " + lmicconfigfile)
else:
    sys.exit("Missing file " + lmicconfigfile + ", please create it! Aborting.")

# check if lora key file is present in shared directory
lorakeyfile = os.path.join (shareddir, config.get("common", "lorakeyfile"))
if os.path.isfile(lorakeyfile) and os.access(lorakeyfile, os.R_OK):
    print("Parsing LORAWAN keys from " + lorakeyfile)
else:
    sys.exit("Missing file " + lorakeyfile + ", please create it! Aborting.")

# check if ota key file is present in shared directory
otakeyfile = os.path.join (shareddir, config.get("common", "otakeyfile"))
if os.path.isfile(otakeyfile) and os.access(otakeyfile, os.R_OK):
    print("Parsing OTA keys from " + otakeyfile)
else:
    sys.exit("Missing file " + otakeyfile + ", please create it! Aborting.")

# parse hal file
mykeys = {}
with open(halconfigfile) as myfile:
    for line in myfile:
        line2 = line.strip("// ")
        key, value = line2.partition(" ")[::2]
        mykeys[key.strip()] = str(value).strip()
myboard = mykeys["board"]
myuploadspeed = mykeys["upload_speed"]
env.Replace(BOARD=myboard)
env.Replace(UPLOAD_SPEED=myuploadspeed)
print('\033[94m' + "Target board: " + myboard + " @ " + myuploadspeed + "bps" + '\033[0m')

# re-set partition table
mypartitiontable = config.get("env", "board_build.partitions")
board = env.BoardConfig(myboard)
board.manifest['build']['partitions'] = mypartitiontable
print('\033[94m' + "Partition table: " + mypartitiontable + '\033[0m')

# parse ota key file
with open(otakeyfile) as myfile:
    for line in myfile:
        key, value = line.partition("=")[::2]
        mykeys[key.strip()] = str(value).strip()

# usage of paxexpress: see https://github.com/paxexpress/docs

# get paxexpress credentials from ota key file
user = mykeys["PAXEXPRESS_USER"]
repository = mykeys["PAXEXPRESS_REPO"]
apitoken = mykeys["PAXEXPRESS_API_TOKEN"]

# get paxexpress upload parameters from platformio environment
version = config.get("common", "release_version")
package, dummy = halconfig.partition(".")[::2]

# put paxexpress user credentials to platformio environment
env.Replace(PAXEXPRESS_USER=user)
env.Replace(PAXEXPRESS_REPO=repository)
env.Replace(PAXEXPRESS_API_TOKEN=apitoken)

# get runtime credentials and put them to compiler directive
env.Append(BUILD_FLAGS=[
    u'-DWIFI_SSID=\\"' + mykeys["OTA_WIFI_SSID"] + '\\"',
    u'-DWIFI_PASS=\\"' + mykeys["OTA_WIFI_PASS"] + '\\"',
    u'-DPAXEXPRESS_USER=\\"' + mykeys["PAXEXPRESS_USER"] + '\\"',
    u'-DPAXEXPRESS_REPO=\\"' + mykeys["PAXEXPRESS_REPO"] + '\\"',
    u'-DPAXEXPRESS_PACKAGE=\\"' + package + '\\"',
    u'-DARDUINO_LMIC_PROJECT_CONFIG_H=' + lmicconfig,
    u'-I \"' + shareddir + '\"'
    ])

# function for pushing new firmware to paxexpress storage using API
def publish_paxexpress(source, target, env):
    firmware_path = str(source[0])
    firmware_name = basename(firmware_path)
    url = "/".join([
        "https://pax.express", "content",
        user, repository, package, version, firmware_name
    ])

    print("Uploading {0} to PAX.express. Version: {1}".format(
        firmware_name, version))
    print(url)

    headers = {
        "Content-type": "application/octet-stream",
        "X-Bintray-Publish": "1",
        "X-Bintray-Override": "1"
    }

    r = None

    try:
        r = requests.put(url,
                         data=open(firmware_path, "rb"),
                         headers=headers,
                         auth=(user,apitoken))
        r.raise_for_status()
    except requests.exceptions.RequestException as e:
        sys.stderr.write("Failed to submit package: %s\n" %
                         ("%s\n%s" % (r.status_code, r.text) if r else str(e)))
        env.Exit(1)

    print("Firmware has been successfully published at PAX.express!")

# put build file name and upload command to platformio environment
env.Replace(
    PROGNAME="firmware_" + package + "_v%s" % version,
    UPLOADCMD=publish_paxexpress)


import os, config
import requests, sys, json, os
import numpy as np
from routines.sendData import sendImg

"""
End Device Routine to test Edge device and cloud server
this file sends an image
"""

try:
	filePath = os.path.join("landmarks", "landmarks", "Query")
	file_list = os.listdir(filePath)
	zipf_idx = np.random.zipf(config.ZIPF_HIGH_PARAMETER, size=1)

	# choice an random file to send to the edge server
	# Choose
	#fileName = file_list[np.random.choice(range(len(file_list)))]
	fileName = file_list[zipf_idx]
	filePath = os.path.join(filePath, fileName)

	url = config.URL_EDGE + "/api/edge/recognition_cache"
	sendImg(url, filePath, fileName)

except Exception as e:
	print(e.args)


def placements(n, k):
    placement = [0 for i in range(k)]
    while True:
        yield (placement)
        for i in range(k - 1, -1, -1):
            if placement[i] < n - 1:
                break
        else:
            return
        placement[i] += 1
        for j in range(i + 1, k):
            placement[j] = 0


import logging
from copy import deepcopy
import ipaddress
import functools
from typing import List, Union, Any, Iterable, Tuple
from pathlib import Path

from .vendor import attr
from .errors import UnknownParamError, SWUpdateRepoSourceError
from .pillar import (
    KeyPath, PillarKeyAPI, PillarKey, PillarItemsAPI
)
from .param import Param, ParamDictItem
from .api_spec import param_spec
from .values import (
    UNDEFINED, UNCHANGED, NONE, value_from_str,
    is_special
)
from .serialize import PrvsnrType, loads

METADATA_PARAM_GROUP_KEY = '_param_group_key'
METADATA_ARGPARSER = '_param_argparser'

logger = logging.getLogger(__name__)

# TODO IMPROVE use some attr api to copy spec
# TODO TEST
def copy_attr(_attr, name=None, **changes):
    attr_kw = {}
    for arg in (
        'default', 'validator', 'repr', 'hash',
        'init', 'metadata', 'type', 'converter', 'kw_only'
    ):
        attr_kw[arg] = (
            changes[arg] if arg in changes else getattr(_attr, arg)
        )

    if not name:
        name = _attr.name

    _UtilityClass = attr.make_class(
        "_UtilityClass", {
            name: attr.ib(**attr_kw)
        }
    )

    return attr.fields_dict(_UtilityClass)[name]

@attr.s(auto_attribs=True)
class AttrParserArgs:
    _attr: Any  # TODO typing
    prefix: str = attr.ib(default=None)

    name: str = attr.ib(init=False, default=None)
    action: str = attr.ib(init=False, default='store')
    metavar: str = attr.ib(init=False, default=None)
    dest: str = attr.ib(init=False, default=None)
    default: str = attr.ib(init=False, default=None)
    const: str = attr.ib(init=False, default=None)
    choices: List = attr.ib(init=False, default=None)
    help: str = attr.ib(init=False, default='')
    type: Any = attr.ib(init=False, default=None)  # TODO typing
    # TODO TEST EOS-8473
    nargs: str = attr.ib(init=False, default=None)

    def __attrs_post_init__(self):  # noqa: C901 FIXME
        self.name = self._attr.name

        if self.prefix:
            self.name = self.prefix + self.name

        parser_args = self._attr.metadata.get(
            METADATA_ARGPARSER, {}
        )

        if parser_args.get('choices'):
            self.choices = parser_args.get('choices')

        if parser_args.get('action'):
            self.action = parser_args.get('action')
        elif self._attr.type is bool:
            self.action = 'store_true'

        self.type = parser_args.get(
            'type',
            functools.partial(
                self.value_from_str, v_type=self._attr.type
            )
        )

        for arg in ('help', 'dest', 'const'):
            if arg in parser_args:
                setattr(self, arg, parser_args.get(arg))

        if self.choices:
            self.help = (
                '{} [choices: {}]'
                .format(self.help, ', '.join(self.choices))
            )

        # TODO TEST EOS-8473
        if parser_args.get('nargs'):
            self.nargs = parser_args.get('nargs')

        if self._attr.default is not attr.NOTHING:
            # optional argument
            self.name = '--' + self.name.replace('_', '-')
            self.default = self._attr.default
            self.metavar = (
                parser_args.get('metavar')
                or (self._attr.type.__name__ if self._attr.type else None)
            )
            if self.metavar:
                self.metavar = self.metavar.upper()

    @property
    def kwargs(self):
        def _filter(_attr, value):
            not_filter = ['_attr', 'name', 'prefix']
            if self.action in ('store_true', 'store_false'):
                not_filter.extend(['metavar', 'type', 'default'])
            if self.action in ('store_const',):
                not_filter.extend(['type'])
            # TODO TEST EOS-8473 nargs
            for arg in ('choices', 'dest', 'const', 'nargs'):
                if getattr(self, arg) is None:
                    not_filter.append(arg)
            return _attr.name not in not_filter

        return attr.asdict(self, filter=_filter)

    @classmethod
    def value_from_str(cls, value, v_type=None):
        _value = value_from_str(value)
        if _value is NONE:
            _value = None
        elif isinstance(_value, str):
            if (v_type is List) or (v_type == 'json'):
                _value = loads(value)
        return _value

class InputAttrParserArgs(AttrParserArgs):
    @classmethod
    def value_from_str(cls, value, v_type=None):
        _value = super().value_from_str(value, v_type=v_type)
        return UNCHANGED if _value is None else _value

# TODO test
class ParserFiller:
    @staticmethod
    def fill_parser(cls, parser, attr_parser_cls=AttrParserArgs):
        for _attr in attr.fields(cls):
            if METADATA_ARGPARSER in _attr.metadata:
                parser_prefix = getattr(cls, 'parser_prefix', None)
                metadata = _attr.metadata[METADATA_ARGPARSER]

                # TODO TEST
                if metadata.get('action') == 'store_bool':
                    for name, default, m_changes in (
                        (_attr.name, _attr.default, {
                            'help': f"enable {metadata['help']}",
                            'action': 'store_const',
                            'const': True,
                            'dest': _attr.name,
                        }), (f'no{_attr.name}', not _attr.default, {
                            'help': f"disable {metadata['help']}",
                            'action': 'store_const',
                            'const': False,
                            'dest': _attr.name,
                        })
                    ):
                        metadata_copy = deepcopy(metadata)
                        metadata_copy.update(m_changes)
                        attr_copy = copy_attr(
                            _attr, name=name, default=default, metadata={
                                METADATA_ARGPARSER: metadata_copy
                            }
                        )
                        args = attr_parser_cls(attr_copy, prefix=parser_prefix)
                        parser.add_argument(args.name, **args.kwargs)
                else:
                    args = attr_parser_cls(_attr, prefix=parser_prefix)
                    parser.add_argument(args.name, **args.kwargs)

    @staticmethod
    def extract_positional_args(cls, kwargs):
        _args = []
        for _attr in attr.fields(cls):
            if METADATA_ARGPARSER in _attr.metadata:
                if _attr.default is attr.NOTHING and _attr.name in kwargs:
                    _args.append(kwargs.pop(_attr.name))
        return _args, kwargs

@attr.s(auto_attribs=True)
class NoParams:
    @classmethod
    def fill_parser(cls, parser):
        pass

    @classmethod
    def extract_positional_args(cls, kwargs):
        return (), kwargs

@attr.s(auto_attribs=True, frozen=True)
class PillarKeysList:
    _keys: List[PillarKey] = attr.Factory(list)

    def __iter__(self):
        return iter(self._keys)

    def __len__(self):
        return len(self._keys)

    @classmethod
    def from_args(
        cls,
        *args: Tuple[Union[str, Tuple[str, str]], ...]
    ):
        pi_keys = []
        for arg in args:
            if type(arg) is str:
                pi_keys.append(PillarKey(arg))
            elif type(arg) is tuple:
                # TODO IMPROVE more checks for tuple types and len
                pi_keys.append(PillarKey(*arg))
            else:
                raise TypeError(f"Unexpected type {type(arg)} of args {arg}")
        return cls(pi_keys)

    @classmethod
    def fill_parser(cls, parser):
        parser.add_argument(
            'args', metavar='keypath', type=str, nargs='*',
            help='a pillar key path'
        )

    @classmethod
    def extract_positional_args(cls, kwargs):
        return (), kwargs

@attr.s(auto_attribs=True, frozen=True)
class PillarInputBase(PillarItemsAPI):
    keypath: str = attr.ib(
        metadata={
            METADATA_ARGPARSER: {
                'help': 'pillar key path',
                # 'metavar': 'value'
            }
        }
    )
    # TODO IMPROVE use some constant for json type
    value: Any = attr.ib(
        metadata={
            METADATA_ARGPARSER: {
                'help': 'pillar value',
                'type': functools.partial(
                    AttrParserArgs.value_from_str, v_type='json'
                )
                # 'metavar': 'value'
            }
        }
    )
    fpath: str = attr.ib(
        default=None,
        metadata={
            METADATA_ARGPARSER: {
                'help': (
                    'file path relative to pillar roots, '
                    'if not specified <key-path-top-level-part>.sls is used'
                ),
                # 'metavar': 'value'
            }
        }
    )

    def pillar_items(self) -> Iterable[Tuple[PillarKeyAPI, Any]]:
        return (
            (PillarKey(self.keypath, self.fpath), self.value),
        )

    @classmethod
    def from_args(cls, *args, **kwargs):
        return cls(*args, **kwargs)

    @classmethod
    def fill_parser(cls, parser):
        ParserFiller.fill_parser(cls, parser, AttrParserArgs)

    @classmethod
    def extract_positional_args(cls, kwargs):
        return ParserFiller.extract_positional_args(cls, kwargs)

@attr.s(auto_attribs=True)
class ParamsList:
    params: List[Param]

    def __iter__(self):
        return iter(self.params)

    @classmethod
    def from_args(cls, *args: List[Union[str, Param]]):
        params = []
        for param in args:
            key_path = KeyPath(str(param))
            param = param_spec.get(str(key_path))
            if param is None:
                param_di = param_spec.get(str(key_path.parent))
                if isinstance(param_di, ParamDictItem):
                    param = Param(
                        key_path,
                        (param_di.keypath / key_path.leaf, param_di.fpath)
                    )
                else:
                    logger.error(
                        "Unknown param {}".format(key_path)
                    )
                    raise UnknownParamError(str(key_path))
            params.append(param)
        return cls(params)

    @classmethod
    def fill_parser(cls, parser):
        parser.add_argument(
            'args', metavar='param', type=str, nargs='+',
            help='a param name to get'
        )

    @classmethod
    def extract_positional_args(cls, kwargs):
        return (), kwargs

class ParamGroupInputBase(PillarItemsAPI):
    _param_group = None
    _spec = None

    def pillar_items(self):  # TODO return type
        res = {}
        for attr_name in attr.fields_dict(type(self)):
            res[self.param_spec(attr_name)] = getattr(self, attr_name)
        return iter(res.items())

    @classmethod
    def param_spec(cls, attr_name: str):
        if cls._spec is None:
            cls._spec = {}
        if attr_name not in cls._spec:
            try:
                _attr = attr.fields_dict(cls)[attr_name]
            except KeyError:
                logger.error("unknown attr {}".format(attr_name))
                raise ValueError('unknown attr {}'.format(attr_name))
            else:
                # TODO TEST
                param_group = _attr.metadata.get(METADATA_PARAM_GROUP_KEY, '')
                full_path = (
                    "{}/{}".format(param_group, attr_name) if param_group
                    else attr_name
                )
                cls._spec[attr_name] = param_spec[full_path]
        return cls._spec[attr_name]

    @classmethod
    def from_args(cls, *args, **kwargs):
        return cls(*args, **kwargs)

    @classmethod
    def fill_parser(cls, parser):
        ParserFiller.fill_parser(cls, parser, InputAttrParserArgs)

    @classmethod
    def extract_positional_args(cls, kwargs):
        return ParserFiller.extract_positional_args(cls, kwargs)

    @staticmethod
    def _attr_ib(
        param_group='', default=UNCHANGED, descr='', metavar=None, **kwargs
    ):
        return attr.ib(
            default=default,
            metadata={
                METADATA_PARAM_GROUP_KEY: param_group,
                METADATA_ARGPARSER: {
                    'help': descr,
                    'metavar': metavar
                }
            },
            **kwargs
        )

class Validation():
    @staticmethod
    def check_ip4(instace, attribute, value):
        try:
            if value and value != UNCHANGED and value != 'None':  # FIXME JBOD
                ipaddress.IPv4Address(value)
        except ValueError:
            raise ValueError(
                f"{attribute.name}: invalid ip4 address {value}"
            )

@attr.s(auto_attribs=True)
class NTP(ParamGroupInputBase):
    _param_group = 'ntp'
    # TODO some trick to avoid passing that value
    server: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="ntp server ip"
    )
    timezone: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="ntp server timezone"
    )

class ReleaseParams():
    _param_group = 'release'
    target_build: str = ParamGroupInputBase._attr_ib(
        _param_group, descr=" Cortx deployment build"
    )

@attr.s(auto_attribs=True)
class Release(ParamGroupInputBase):
    target_build: str = ReleaseParams.target_build

class StorageEnclosureParams():
    _param_group = 'storage_enclosure'
    type: str = ParamGroupInputBase._attr_ib(
        _param_group, descr=" Type of storage"
    )
    primary_mc_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr=" Controller A IP",
        validator=Validation.check_ip4
    )
    secondary_mc_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr=" Controller B IP",
        validator=Validation.check_ip4
    )
    controller_user: str = ParamGroupInputBase._attr_ib(
        _param_group, descr=" Controller user"
    )
    controller_secret: str = ParamGroupInputBase._attr_ib(
        _param_group, descr=" Controller password"
    )

@attr.s(auto_attribs=True)
class StorageEnclosure(ParamGroupInputBase):
    controller_a_ip: str = StorageEnclosureParams.primary_mc_ip
    controller_b_ip: str = StorageEnclosureParams.secondary_mc_ip
    controller_user: str = StorageEnclosureParams.controller_user
    controller_secret: str = StorageEnclosureParams.controller_secret

class NodeNetworkParams():
    _param_group = 'node'
    hostname: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node hostname"
    )
    is_primary: bool = ParamGroupInputBase._attr_ib(
        _param_group, descr="mark node as a primary",
        converter=bool
    )
    data_nw_iface: List = ParamGroupInputBase._attr_ib(
        _param_group, descr="node data network iface"
    )
    bmc_user: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node BMC User"
    )
    bmc_secret: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node BMC password"
    )
    mgmt_nw_gateway: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node mgmt gateway IP",
        validator=Validation.check_ip4
    )
    mgmt_nw_public_ip_addr: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node management iface IP",
        validator=Validation.check_ip4
    )
    mgmt_nw_netmask: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node management iface netmask",
        validator=Validation.check_ip4
    )
    mgmt_nw_iface: List = ParamGroupInputBase._attr_ib(
        _param_group, descr="node management network iface"
    )
    data_nw_public_ip_addr: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node data iface IP", default=UNCHANGED,
        validator=Validation.check_ip4
    )
    data_nw_netmask: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node data iface netmask",
        validator=Validation.check_ip4
    )
    data_nw_gateway: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node data gateway IP",
        validator=Validation.check_ip4
    )
    pvt_ip_addr: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node data iface private IP",
        validator=Validation.check_ip4
    )
    bmc_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="node BMC  IP", default=UNCHANGED,
        validator=Validation.check_ip4
    )

class NetworkParams():
    _param_group = 'network'
    cluster_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="cluster ip address for public data network",
        validator=Validation.check_ip4
    )
    mgmt_vip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="virtual ip address for management network",
        validator=Validation.check_ip4
    )
    dns_servers: List = ParamGroupInputBase._attr_ib(
        _param_group, descr="list of dns servers as json"
    )
    search_domains: List = ParamGroupInputBase._attr_ib(
        _param_group, descr="list of search domains as json"
    )
    primary_hostname: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node hostname"
    )
    primary_floating_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node floating IP"
    )
    primary_data_gateway: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node data gateway IP"
    )
    primary_mgmt_gateway: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node mgmt gateway IP"
    )
    primary_mgmt_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node management iface IP"
    )
    primary_mgmt_netmask: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node management iface netmask"
    )
    primary_data_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node data iface IP",
        validator=Validation.check_ip4
    )
    primary_data_netmask: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node data iface netmask"
    )
    primary_data_network_iface: List = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node data network iface"
    )
    primary_bmc_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node BMC  IP",
        validator=Validation.check_ip4
    )
    primary_bmc_user: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node BMC User"
    )
    primary_bmc_secret: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="primary node BMC password"
    )
    secondary_hostname: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node hostname"
    )
    secondary_floating_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node floating IP"
    )
    secondary_data_gateway: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node data gateway IP"
    )
    secondary_mgmt_gateway: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node mgmt gateway IP"
    )
    secondary_mgmt_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node management iface IP",
        validator=Validation.check_ip4
    )
    secondary_mgmt_netmask: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node management iface netmask"
    )
    secondary_data_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node node data iface IP",
        validator=Validation.check_ip4
    )
    secondary_data_netmask: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node data iface netmask"
    )
    secondary_data_network_iface: List = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node data network iface"
    )
    secondary_bmc_ip: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node BMC  IP",
        validator=Validation.check_ip4
    )
    secondary_bmc_user: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node BMC User"
    )
    secondary_bmc_secret: str = ParamGroupInputBase._attr_ib(
        _param_group, descr="secondary node BMC password"
    )

@attr.s(auto_attribs=True)
class Network(ParamGroupInputBase):
    cluster_ip: str = NetworkParams.cluster_ip
    mgmt_vip: str = NetworkParams.mgmt_vip
    dns_servers: List = NetworkParams.dns_servers
    search_domains: List = NetworkParams.search_domains
    primary_hostname: str = NetworkParams.primary_hostname
    primary_floating_ip: str = NetworkParams.primary_floating_ip
    primary_mgmt_ip: str = NetworkParams.primary_mgmt_ip
    primary_mgmt_netmask: str = NetworkParams.primary_mgmt_netmask
    primary_mgmt_gateway: str = NetworkParams.primary_mgmt_gateway
    primary_data_netmask: str = NetworkParams.primary_data_netmask
    primary_data_gateway: str = NetworkParams.primary_data_gateway
    primary_data_network_iface: List = NetworkParams.primary_data_network_iface
    primary_data_ip: str = NetworkParams.primary_data_ip
    primary_bmc_ip: str = NetworkParams.primary_bmc_ip
    primary_bmc_user: str = NetworkParams.primary_bmc_user
    primary_bmc_secret: str = NetworkParams.primary_bmc_secret
    secondary_hostname: str = NetworkParams.secondary_hostname
    secondary_floating_ip: str = NetworkParams.secondary_floating_ip
    secondary_mgmt_ip: str = NetworkParams.secondary_mgmt_ip
    secondary_mgmt_netmask: str = NetworkParams.secondary_mgmt_netmask
    secondary_data_gateway: str = NetworkParams.secondary_data_gateway
    secondary_mgmt_gateway: str = NetworkParams.secondary_mgmt_gateway
    secondary_data_netmask: str = NetworkParams.secondary_data_netmask
    secondary_data_network_iface: List = NetworkParams.secondary_data_network_iface  # noqa: E501
    secondary_bmc_ip: str = NetworkParams.secondary_bmc_ip
    secondary_bmc_user: str = NetworkParams.secondary_bmc_user
    secondary_bmc_secret: str = NetworkParams.secondary_bmc_secret
    secondary_data_ip: str = NetworkParams.secondary_data_ip

# # TODO TEST
# @attr.s(auto_attribs=True)
# class ClusterIP(ParamGroupInputBase):
#     _param_group = 'network'
#     # dns_server: str = ParamGroupInputBase._attr_ib(_param_group)
#     cluster_ip: str = ParamGroupInputBase._attr_ib(
#         _param_group, descr="cluster ip address for public data network",
#         default=attr.NOTHING
#     )

# # TODO TEST
# @attr.s(auto_attribs=True)
# class MgmtVIP(ParamGroupInputBase):
#     _param_group = 'network'
#     # dns_server: str = ParamGroupInputBase._attr_ib(_param_group)
#     mgmt_vip: str = ParamGroupInputBase._attr_ib(
#         _param_group, descr="virtual ip address for management network",
#         default=attr.NOTHING
#     )

# TODO
# verify that attributes match _param_di during class declaration:
#   - both attributes should satisfy _param_di
#   - is_key might be replaced with checking attr name against _param_di.key
class ParamDictItemInputBase(PrvsnrType, PillarItemsAPI):
    _param_di = None
    _param = None

    def pillar_items(self):  # TODO return type
        return iter({
            self.param_spec(): getattr(self, self._param_di.value)
        }.items())

    def param_spec(self):
        if self._param is None:
            key = getattr(self, self._param_di.key)
            self._param = Param(
                self._param_di.name / key,
                (self._param_di.keypath / key, self._param_di.fpath)
            )
        return self._param

    @classmethod
    def from_args(cls, *args, **kwargs):
        return cls(*args, **kwargs)

    @classmethod
    def fill_parser(cls, parser):
        ParserFiller.fill_parser(cls, parser, InputAttrParserArgs)

    @classmethod
    def extract_positional_args(cls, kwargs):
        return ParserFiller.extract_positional_args(cls, kwargs)

    @staticmethod
    def _attr_ib(
        is_key=False, default=UNCHANGED, descr='', metavar=None, **kwargs
    ):
        return attr.ib(
            default=attr.NOTHING if is_key else default,
            metadata={
                METADATA_ARGPARSER: {
                    'help': descr,
                    'metavar': metavar
                }
            },
            **kwargs
        )

@attr.s(auto_attribs=True)
class SWUpdateRepo(ParamDictItemInputBase):
    _param_di = param_spec['swupdate/repo']
    release: str = ParamDictItemInputBase._attr_ib(
        is_key=True,
        descr=("release version")
    )
    source: Union[str, Path] = ParamDictItemInputBase._attr_ib(
        descr=(
            "repo source, might be a local path to a repo folder or iso file"
            " or an url to a remote repo, "
            "{} might be used to remove the repo"
            .format(UNDEFINED)
        ),
        metavar='str',
        converter=lambda v: (
            UNCHANGED if v is None else (
                v if is_special(v) or isinstance(v, Path) else str(v)
            )
        )
    )

    @source.validator
    def _check_source(self, attribute, value):
        if is_special(value):
            return  # TODO does any special is expected here

        if (
            type(self.source) is str
            and value.startswith(('http://', 'https://'))
        ):
            return

        reason = None
        _value = Path(str(value))
        if _value.exists():
            if _value.is_file():
                if _value.suffix != '.iso':
                    reason = 'not an iso file'
            elif not _value.is_dir():
                reason = 'not a file or directory'
        else:
            reason = 'unexpected type of source'

        if reason:
            logger.error(
                "Invalid source {}: {}"
                .format(str(value), reason)
            )
            raise SWUpdateRepoSourceError(str(value), reason)

    def __attrs_post_init__(self):
        if (
            type(self.source) is str
            and not self.source.startswith(('http://', 'https://'))
        ):
            self.source = Path(self.source)

        if isinstance(self.source, Path):
            self.source = self.source.resolve()

    @property
    def pillar_key(self):
        return self.release

    @property
    def pillar_value(self):
        if self.is_special() or self.is_remote():
            return self.source
        else:
            return 'iso' if self.source.is_file() else 'dir'

    def is_special(self):
        return is_special(self.source)

    def is_local(self):
        return not self.is_special() and isinstance(self.source, Path)

    def is_remote(self):
        return not (self.is_special() or self.is_local())

    def is_dir(self):
        return self.is_local() and self.source.is_dir()

    def is_iso(self):
        return self.is_local() and self.source.is_file()


from fastapi import FastAPI
from recommend import Recommend

app = FastAPI(docs_url="/api_doc", redoc_url=None)

@app.get("/bits/popular/")
def get_recommendations():
    try:
        print('Popularity Based')
        recommended = Recommend.popular_recommend()
        return {'data': recommended}
    except:
        return {'data': 'Error ❗❗❗'}

@app.get("/bits/user/{user_id}")
def get_recommendations(user_id: int):
    try:
        print('User Based')
        recommended = Recommend.user_based_recommend(user_id)
        return {'data': recommended}
    except:
        print('User not found, popularity called')
        recommended = Recommend.popular_recommend()
        return {'data': recommended}

@app.get("/bits/item/{bits_id}")
def get_recommendations(bits_id: int):
    try:
        print('Item Based')
        recommended = Recommend.item_based_recommend(bits_id)
        return {'data': recommended}
    except:
        print('Item not found, popularity called')
        recommended = Recommend.popular_recommend()
        return {'data': recommended}


from datetime import datetime

import pytest
from assertpy import assert_that
from geojson_pydantic import Polygon

from src.configs.discount_options import PARKING_ZONE_ARTICLE, REUSE_ARTICLE
from src.rate_policies.domain.models import Fee, DeerUsage, UsageTime, AreaFee, Deer, areas
from src.rate_policies.domain.models.areas import Location
from src.rate_policies.domain.models.policies import AmountDiscountPolicy
from tests.unit.fixtures.unit_of_work import FakeCalculatorUnitOfWork, FakeUsageRepository

class TestAmountDiscountPolicy:
    @pytest.fixture
    def deer_area(self):
        return areas.Area(
            area_id=1,
            area_boundary=Polygon(coordinates=[
                [(37.543272, 127.07655), (37.541734, 127.074072), (37.539088, 127.07449), (37.538928, 127.078034),
                 (37.540417, 127.080781), (37.542416, 127.080599), (37.543272, 127.07655)]]),
            area_center=Location(lat=37.541302, lng=127.077852),
        )

    @pytest.fixture
    def calculator_uow(self, deer_area):
        uow = FakeCalculatorUnitOfWork()
        uow.usages = FakeUsageRepository({
            1: DeerUsage(
                user_id=1,
                use_deer=Deer(deer_name=1, deer_area=deer_area),
                end_location=Location(
                    lat=37.541983,
                    lng=127.078179
                ),
                usage_time=UsageTime(
                    start=datetime(2021, 11, 18, 8, 20, 0),
                    end=datetime(2021, 11, 18, 8, 50, 0)
                )
            )
        })
        return uow

    @pytest.fixture
    def amount_discount_policy(self):
        return AmountDiscountPolicy

    class TestCalculate:

        class TestInKunKukUniv:
            @pytest.fixture
            def end_location(self):
                return Location(
                    lat=37.541983,
                    lng=127.078179
                )

            @pytest.fixture
            def area_fee(self):
                return AreaFee(
                    area_id=1,
                    base=Fee(amount=790, currency="KRW"),
                    rate_per_minute=Fee(amount=150, currency="KRW")
                )

            class TestWithUse10Minutes:
                @pytest.fixture
                def usage_time(self):
                    return UsageTime(
                        start=datetime(2021, 11, 18, 9, 0, 0),
                        end=datetime(2021, 11, 18, 9, 10, 0)
                    )

                @pytest.fixture
                def usage(self, deer_area, end_location, usage_time):
                    return DeerUsage(
                        user_id=1,
                        use_deer=Deer(deer_name=1, deer_area=deer_area),
                        end_location=end_location,
                        usage_time=usage_time
                    )

                @pytest.fixture
                def policy(self, amount_discount_policy, usage, area_fee, calculator_uow):
                    options = {
                        "parking_zone": PARKING_ZONE_ARTICLE,
                        "reuse": REUSE_ARTICLE
                    }
                    return amount_discount_policy(usage, area_fee, calculator_uow, options)

                def test_it_returns_basic_fee(self, policy, area_fee):
                    basic_fee = Fee(amount=2290, currency="KRW")
                    rate_discounted_fee = policy.apply_on(basic_fee)
                    expected = basic_fee - area_fee.base

                    assert_that(rate_discounted_fee).is_equal_to(expected)


# coding=utf-8
"""
    @project: Everyday_LeetCode
    @Author：Charles
    @file： 134.加油站.py
    @date：2023/1/10 15:35
"""

class Solution(object):
    def canCompleteCircuit(self, gas, cost):
        """
        :type gas: List[int]
        :type cost: List[int]
        :rtype: int
        """
        # 贪心算法
        # 1.如果sum(gas) < sum(cost)，说明加油满足不了消耗，所以无法跑完一圈
        # 2.为了跑完一圈，则再前往一个加油站时要有油
        #   所以cur_sum += gas[i] - cost[i]，必须一直为正数
        #   若出现负数则表示无法跑到下一个加油站
        #   题目说明有唯一的解，所以当cur_sum一直大于0的起始点，就为出发点

        if sum(gas) < sum(cost):
            return -1

        start = 0
        cur_sum = 0
        for i in range(len(gas)):
            print(gas[i] - cost[i])
            cur_sum += gas[i] - cost[i]
            print("cur sun", cur_sum)
            if cur_sum < 0:
                cur_sum = 0
                start = i + 1

        return start


import numpy as np
import matplotlib
#matplotlib.use('Agg')
import matplotlib.pyplot as plt

alpha = 10**-10
T = 10
batch = 100

data = np.load(f"results/stoppingTimes_{alpha}_{T}_{batch}.npy")

print(f"Data for {data.shape[0]} constituencies for {T} runs each with alpha = {alpha}")

avgStoppingTimes = np.clip(np.mean(data, axis = 1), 0, 10000)

avgStoppingTimes.sort()

print(avgStoppingTimes)

plt.hist(avgStoppingTimes, bins = 20)
plt.show()


'''
    These are the tokens for every language.
    This is because we convert everything to this
    standard set of tokens
'''

reserved = {
    "auto" : "AUTO",
    "break" :"BREAK",
    "case" : "CASE",
    "char" : "CHAR",
    "const" : "CONST",
    "continue" : "CONTINUE",
    "default" : "DEFAULT",
    "do" : "DO",
    "double" : "DOUBLE",
    "else" : "ELSE",
    "end" : "END",
    "enum" : "ENUM",
    "extern" : "EXTERM",
    "float" : "FLOAT",
    "for" : "FOR",
    "function" : "FUNCTION",
    "goto" : "GOTO",
    "if" : "IF",
    "int" : "INT",
    "long" : "LONG",
    "print" : "PRINT",
    "register" : "REGISTER",
    "return" : "RETURN",
    "short" : "SHORT",
    "signed" : "SIGNED",
    "sizeof" : "SIZEOF",
    "static" : "STATIC",
    "struct" : "STRUCT",
    "switch" :"SWITCH",
    "typedef" : "TYPEDEF",
    "union" : "UNION",
    "unsigned" : "UNSIGNED",
    "variable" : "VARIABLE",
    "void" : "VOID",
    "volatile" : "VOLATILE",
    "while": "WHILE"
}

tokens = [
    'PLUS',
    'MINUS',
    'DIV',
    'MUL',
    'MOD',
    'EXP',
    'INC',
    'DEC',
    'GT',
    'LT',
    'EQ',
    'NEQ',
    'GTE',
    'AS',
    'LTE',
    'BAND',
    'BOR',
    'BNOT',
    'BXOR',
    'BLS',
    'BRS',
    'AND',
    'OR',
    'NOT',
    'COND_TERN',
    'ID',
    'DOT',
    'COMMA',
    'SM',
    'COLON',
    'LPAREN',
    'RPAREN',
    'LBRACE',
    'RBRACE',
    'LBRACKET',
    'RBRACKET',
    'LITNUM',
    'LITCHAR',
    'LITSTRING',
    'COMMENT',
    'KEY',
    'REST'

] + list(reserved.values())

'''
    REGEX RULES FOR TOKENS
'''

# operators : arithmetic
t_PLUS = r'\+'
t_MINUS = r'-'
t_MUL = r'\*'
t_DIV = r'/'
t_MOD = r'%'    # todo: check if op
t_EXP = r'\^'
t_INC = r'\+\+'
t_DEC = r'\-\-'

# operators : relational
t_GT = r'>'
t_LT = r'<'
t_EQ = r'=='
t_NEQ = r'!='
t_GTE = r'>='
t_LTE = r'<='

# operators : bitwise
t_BAND = r'&'
t_BOR = r'\|'
t_BNOT = r'~'
t_BXOR = r'\$'   # todo: what about caret?
t_BLS = r'<<'
t_BRS = r'>>'

# operators : logical
t_AND = r'&&'
t_OR = r'\|\|'
t_NOT = r'!'

t_AS = r'\='

# operators : ternary
t_COND_TERN = r'\?'

# separators
t_DOT = r'\.'
t_COMMA = r','
t_SM = r'\;'
t_COLON = r':'

t_LPAREN  = r'\('
t_RPAREN  = r'\)'

t_LBRACE  = r'{'
t_RBRACE  = r'}'

t_LBRACKET  = r'\['
t_RBRACKET  = r'\]'

# literals
'LITNUM', 'LITCHAR', 'LITSTRING'
# literals
def t_LITNUM(t):
    r'\d+'
    t.value = int(t.value)
    return t

def t_LITCHAR(t):
    r'\'.\''
    t.value = str(t.value)
    return t

def t_LITSTRING(t):
    r'\".*\"'
    t.value = str(t.value)
    return t

# identifier
def t_ID(t):
    r'id[0-9]+'
    t.type = reserved.get(t.value,'ID')    # Check for reserved words
    return t

def t_KEY(t):
    r'[a-zA-Z_][a-zA-Z_0-9]*'
    t.type = reserved.get(t.value)
    return t

# ignore comments
def t_COMMENT(t):
    r'\#.*'
    pass

# to track line numbers
def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

# ignore spaces and tabs
t_ignore  = ' \t'
# t_REST = r'.+'

# Error handling rule
def t_error(t):
    print("Illegal character '%s'" % t.value[0])
    print(t.lexer.lineno)
    exit()


#  用户输入两个正整数，求他们的最小公倍数。
num1 = eval(input("请输入第一个数字： "))
num2 = eval(input("请输入第二个数字： "))

if num1<= 0 or num2 <= 0:
    print("两个数必须是正整数")
    exit(0)

if num1>num2:
    max=num1
    min=num2
else:
    max=num2
    min=num1

for i in range(1,min+1):
    numtemp=max*i
    if numtemp % min == 0:
        numresult=numtemp
        break

print("最小公倍数是：",numresult)


# -*- mode: python ; coding: utf-8 -*-

block_cipher = None

a = Analysis(['starter.py'],
             pathex=[],
             binaries=[],
             datas=[('templates', 'templates')],
             hiddenimports=['flask'],
             hookspath=[],
             hooksconfig={},
             runtime_hooks=[],
             excludes=[],
             win_no_prefer_redirects=False,
             win_private_assemblies=False,
             cipher=block_cipher,
             noarchive=False)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)

exe = EXE(pyz,
          a.scripts,
          a.binaries,
          a.zipfiles,
          a.datas,
          [],
          name='annotator',
          debug=False,
          bootloader_ignore_signals=False,
          strip=False,
          upx=True,
          upx_exclude=[],
          runtime_tmpdir=None,
          console=True,
          disable_windowed_traceback=False,
          target_arch=None,
          codesign_identity=None,
          entitlements_file=None )
app = BUNDLE(exe,
             name='annotator.app',
             icon=None,
             bundle_identifier=None)

# Only x86_64 app will be built for macOS since rosetta2 is equipped on M1 chip macOS.
# This is too avoid the case that some packages only support x86_64 and fail to run on arm64.
# https://pyinstaller.readthedocs.io/en/stable/feature-notes.html#macos-multi-arch-support


import numpy as np
import pandas as pd
import os
import pickle
import sys
obama_df = pd.read_csv("data/obama_csv_test.csv", sep='\t', encoding='latin1')
obama_df = obama_df[pd.notnull(obama_df['label'])]
obama_df['label'] = obama_df['label'].astype(np.int)

texts = []  # list of text samples
labels_index = {-1: 0,
				0: 1,
				1: 2}  # dictionary mapping label name to numeric id
labels = []  # list of label ids

obama_df = obama_df[obama_df['label'] != 2]
nb_rows = len(obama_df)
for i in range(nb_rows):
	row = obama_df.iloc[i]
	texts.append(str(row['tweet']))
	labels.append(labels_index[int(row['label'])])

texts = np.asarray(texts)
labels = np.asarray(labels)

def tokenize(texts):
    '''
    For SKLearn models / XGBoost / Ensemble, use CountVectorizer to generate
    n-gram vectorized texts efficiently.

    Args:
        texts: input text sentences list

    Returns:
        the n-gram text
    '''
    if os.path.exists('data/vectorizer.pkl'):
        with open('data/vectorizer.pkl', 'rb') as f:
            vectorizer = pickle.load(f)
            x_counts = vectorizer.transform(texts)
    else:
        vectorizer = CountVectorizer(ngram_range=(1, 2))
        x_counts = vectorizer.fit_transform(texts)

        with open('data/vectorizer.pkl', 'wb') as f:
            pickle.dump(vectorizer, f)

    print('Shape of tokenizer counts : ', x_counts.shape)
    return x_counts

def tfidf(x_counts):
    '''
    Perform TF-IDF transform to normalize the dataset

    Args:
        x_counts: the n-gram tokenized sentences

    Returns:
        the TF-IDF transformed dataset
    '''
    if os.path.exists('data/tfidf.pkl'):
        with open('data/tfidf.pkl', 'rb') as f:
            transformer = pickle.load(f)
            x_tfidf = transformer.transform(x_counts)
    else:
        transformer = TfidfTransformer()
        x_tfidf = transformer.fit_transform(x_counts)

        with open('data/tfidf.pkl', 'wb') as f:
            pickle.dump(transformer, f)

    return x_tfidf

print(['hello and Hi hello'])
x_counts = tokenize(['hello and Hi hello'])
# print(obama_df)
# print(len(obama_df))
print(x_counts)
# sys.exit("bye")
data = tfidf(x_counts)
print(data)
print('-' * 80)


import Parser
import copy

# Do the calculation for the current knowledge base
def kb_calculate(opr):
    # If it is leaf, return the boolean value
    if not opr.children:
        return opr.tf

    # If the opr is negation(only has one child)
    if len(opr.children) == 1:
        child = opr.children[0]
        kb_calculate(child)
        # Negation
        if child.tf is False:
            opr.set_value(True)
        else:
            opr.set_value(False)
        return opr.tf

    # if the opr is another operation
    else:
        # Two children of the current node
        left = opr.children[0]
        right = opr.children[1]
        # Find the bool value(leaf) in the tree
        # It means you find the minimal propositional logic formula
        if right.children:
            kb_calculate(right)
        if left.children:
            kb_calculate(left)

        # True table of each operation(except negation) in the KB tree
        if opr.label == 'v':
            if left.tf is False and right.tf is False:
                opr.set_value(False)
            else:
                opr.set_value(True)
        elif opr.label == '^':
            if left.tf is True and right.tf is True:
                opr.set_value(True)
            else:
                opr.set_value(False)
        elif opr.label == '=>':
            if left.tf is True and right.tf is False:
                opr.set_value(False)
            else:
                opr.set_value(True)
        elif opr.label == '<=':
            if left.tf is False and right.tf is True:
                opr.set_value(False)
            else:
                opr.set_value(True)
        elif opr.label == '<=>':
            if left.tf is False and right.tf is False:
                opr.set_value(True)
            elif left.tf is True and right.tf is True:
                opr.set_value(True)
            else:
                opr.set_value(False)
        return opr.tf

# Change the symbols(leaves) in the tree into bool value based on the current model
def kb_leaf_bool(node, model):
    # Set up the bool value of the leaf node
    if not node.children:
        node.set_value(model[node.label])

    # If the node is negation (only has one child)
    elif len(node.children) == 1:
        child = node.children[0]
        kb_leaf_bool(child, model)

    # Normal case: do the deep first search
    else:
        # DFS, Left subtree first,then right subtree
        left = node.children[0]
        right = node.children[1]
        kb_leaf_bool(left, model)
        kb_leaf_bool(right, model)

# Return true is a sentence holds within a model
def pl_true(node, model):
    kb_leaf_bool(node, model)
    return kb_calculate(node)

# Return true if KB entails alpha
def tt_entails(kb, alpha):
    # Get the symbols from KB and alpha and add to the symbol list
    symbol = []
    Parser.get_symbol(kb, symbol)
    Parser.get_symbol(alpha, symbol)
    # Initial the model(partial model as an assignment to some of the symbols)
    model = {}
    check_all = tt_check_all(kb, alpha, symbol, model)
    check_one = tt_check_one(kb, alpha, symbol, model)
    if check_all and check_one:
        return True
    elif not check_all and not check_one:
        return False
    else:
        return "Maybe"

# Check all the possible truth table model
# and return true if every model of KB is also a model of alpha
def tt_check_all(kb, alpha, symbol, model):
    # If all the symbol has added to the model
    if not symbol:
        if pl_true(kb, model):  # Check if sentence holds within a model
            return pl_true(alpha, model)
        else:
            return True  # When KB is false, always return true to neglect the current model

    else:
        # Create two models based on the current symbol (False & True)
        symbol0 = copy.deepcopy(symbol)
        p = symbol0[0]
        del symbol0[0]
        rest = symbol0
        model1 = copy.deepcopy(model)
        model2 = copy.deepcopy(model)
        model1[p] = True
        model2[p] = False
        return tt_check_all(kb, alpha, rest, model1) and tt_check_all(kb, alpha, rest, model2)

# Check all the possible truth table model
# and return true if one model of KB is also a model of alpha
def tt_check_one(kb, alpha, symbol, model):
    # If all the symbol has added to the model
    if not symbol:
        if pl_true(kb, model):  # Check if sentence holds within a model
            return pl_true(alpha, model)
        else:
            return False  # When KB is false, always return False to neglect the current model

    else:
        # Create two models based on the current symbol (False & True)
        symbol0 = copy.deepcopy(symbol)
        p = symbol0[0]
        del symbol0[0]
        rest = symbol0
        model1 = copy.deepcopy(model)
        model2 = copy.deepcopy(model)
        model1[p] = True
        model2[p] = False
        return tt_check_one(kb, alpha, rest, model1) or tt_check_one(kb, alpha, rest, model2)


import pandas as pd
import numpy as np
import pvlib
import datetime
from pvlib import irradiance
from pvlib import location
from pvlib.pvsystem import PVSystem
#from pvlib.location import Location
from pvlib.modelchain import ModelChain
from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
import weatherData

latitude, longitude, tz = 63.42024, 10.40122, 'Europe/Oslo' # specify location
start = pd.Timestamp(datetime.date.today(), tz=tz)
end = start + pd.Timedelta(days=1)
irrad_vars = ['ghi', 'dni', 'dhi']

# Create location object to store lat, lon, timezone
#location = Location(latitude=63.42024, longitude=10.40122) # Bredd/lengde-grad for Trondheim
site = location.Location(latitude, longitude, tz=tz)

temperature_model_parameters = TEMPERATURE_MODEL_PARAMETERS['sapm']['open_rack_glass_glass']
# Bestemmer hvilke moduler og invertere vi ønsker å modelere.
sandia_modules = pvlib.pvsystem.retrieve_sam('SandiaMod')
cec_inverters = pvlib.pvsystem.retrieve_sam('cecinverter')
sandia_module = sandia_modules['Canadian_Solar_CS5P_220M___2009_']
cec_inverter = cec_inverters['ABB__MICRO_0_25_I_OUTD_US_208__208V_']

# Calculate clear-sky GHI and transpose to plane of array
def get_irradiance(site_location, date, tilt, surface_azimuth):
    # Creates one day's worth of 10 min intervals
    times = pd.date_range(date, freq='10min', periods=6*24,
                          tz=site_location.tz)
    # Generate clearsky data using the Ineichen model, which is the default
    # The get_clearsky method returns a dataframe with values for GHI, DNI,
    # and DHI

    clearsky = site_location.get_clearsky(times)
    # Get solar azimuth and zenith to pass to the transposition function
    solar_position = site_location.get_solarposition(times=times)
    # Use the get_total_irradiance function to transpose the GHI to POA
    POA_irradiance = irradiance.get_total_irradiance(
        surface_tilt=tilt,
        surface_azimuth=surface_azimuth,
        dni=clearsky['dni'],
        ghi=clearsky['ghi'],
        dhi=clearsky['dhi'],
        solar_zenith=solar_position['apparent_zenith'],
        solar_azimuth=solar_position['azimuth'])
    # Return DataFrame with only GHI and POA
    return pd.DataFrame({'GHI': clearsky['ghi'],
                         'DNI' : clearsky['dni'],
                         'DHI' : clearsky['dhi'],
                         'POA': POA_irradiance['poa_global']})

system = PVSystem(surface_tilt=20, surface_azimuth=180, # vinkel og vridning for panel. Sør = 180
        module_parameters = sandia_module,
        inverter_parameters = cec_inverter,
        temperature_model_parameters = temperature_model_parameters)

###____ Kjør ____ ###

def getIndexIntoDay():
    now = datetime.datetime.now()
    start = datetime.datetime(now.year, now.month, now.day)
    diff = now - start
    seconds_in_day = 24 * 60 * 60
    return int((144 / seconds_in_day) * diff.seconds) # converts seconds to intervals of 144 (10 min) and throws away decimal.

def solarPanelPower(date, index):
    """
    Returns solarPower generation in Watt
     """

    irradiance = get_irradiance(site, date, 20, 180) # irradiance i Trondheim, i dag
    wattPerSquareMeter = irradiance["POA"][index]
    solarPanelArrayPower = wattPerSquareMeter * 150 * 0.19 # times square meters * solar panel efficiency.
    cloudCover = weatherData.Get_data_now().variables["cloud_area_fraction"]
    cloudCover = float(str(cloudCover).split()[1][:-1])/100
    solarPanelArrayPower = solarPanelArrayPower * (1 - cloudCover) # power * 1 - cloud coverage
    return solarPanelArrayPower

#power = solarPanelPower('14-05-2021', getIndexIntoDay())
#print(power)
''

"""
weatherList = [irradiance["GHI"], irradiance["DHI"], irradiance["DNI"], 30, 5]

#print(weatherList)

weather = pd.DataFrame([[irradiance["GHI"], irradiance["DHI"], irradiance["DNI"], 30, 5]],
                         columns=['ghi', 'dni', 'dhi', 'temp_air', 'wind_speed'],
                         index=[pd.Timestamp('20210401 1200', tz='Europe/Oslo')])

print(weather)

#MÅ FIKSES MER PÅ. Må få tatt innn riktige verdier i weather DF og ikke arrays.
mc = ModelChain(system, site)

mc.run_model(weather)
print(mc.ac)
"""


import json
import os
import re
import threading
import time
from pathlib import Path

import cv2
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image

from nonverbal_communication_analysis.environment import (DATASET_SYNC,
                                                          VALID_VIDEO_TYPES,
                                                          VIDEO_OUTPUT_DIR)
from nonverbal_communication_analysis.m0_Classes.Experiment import Experiment
from nonverbal_communication_analysis.utils import log

class Worker(threading.Thread):
    def __init__(self, worker_id, camera, video_cap, output_directory, verbose: bool = False, prettify: bool = False, display: bool = False):
        threading.Thread.__init__(self)
        self.worker_id = worker_id
        self.camera = camera
        self.video_cap = video_cap
        self.output_directory = output_directory

        self.verbose = verbose
        self.prettify = prettify
        self.display = display

        self.previous_frame = None
        self.current_frame = None
        self.frame_idx = 0

        self.cumulative_energy_frame = None

    def frame_energy(self, current_frame):
        frame_energy = np.zeros(current_frame.shape[:2], np.int)

        if self.previous_frame is None:
            return frame_energy

        frame_diff = cv2.absdiff(self.previous_frame, current_frame)
        gs_frame_diff = cv2.cvtColor(frame_diff, cv2.COLOR_BGR2GRAY)
        _, frame_energy = cv2.threshold(
            gs_frame_diff, 127, 1, cv2.THRESH_BINARY)
        return frame_energy

    def save_frame(self, frame_data, output_directory):
        os.makedirs(output_directory, exist_ok=True)
        output_frame_file = output_directory / \
            ("frame_%.12d_processed.json" % (self.frame_idx))

        metrics_dict = dict()
        metrics_dict["energy"] = int(np.sum(frame_data))

        obj = dict()
        obj["frame"] = self.frame_idx
        obj["group"] = metrics_dict

        if self.prettify:
            json.dump(obj, open(output_frame_file, 'w'), indent=2)
        else:
            json.dump(obj, open(output_frame_file, 'w'))

    def run(self):
        cap = self.video_cap
        cumulative_energy_frame = self.cumulative_energy_frame

        while(cap.isOpened()):
            ret, current_frame = cap.read()
            if ret == True:
                if self.display:
                    cv2.imshow('Camera %s' % self.camera, current_frame)
                    if cv2.waitKey(25) & 0xFF == ord('q'):
                        break

                if cumulative_energy_frame is None:
                    cumulative_energy_frame = np.zeros(
                        current_frame.shape[:2], np.int)

                frame_energy = self.frame_energy(
                    current_frame)
                cumulative_energy_frame += frame_energy

                output_path = self.output_directory / ('pc%s' % self.camera)
                self.save_frame(frame_energy, output_path)

                self.previous_frame = current_frame
                self.frame_idx += 1
            else:
                break

        cap.release()
        self.cumulative_energy_frame = cumulative_energy_frame
        # print("Worker %s finished with: %s" %
        #       (self.worker_id, np.unique(cumulative_energy_frame)))

class VideoProcess(object):

    def __init__(self, group_id: str, prettify: bool = False, verbose: bool = False):
        self.group_id = group_id
        self.prettify = prettify
        self.verbose = verbose

        self.experiment = Experiment(group_id)
        self.output_group_dir = VIDEO_OUTPUT_DIR / \
            group_id / (group_id + '_processed')
        os.makedirs(self.output_group_dir, exist_ok=True)
        json.dump(self.experiment.to_json(),
                  open(self.output_group_dir / (self.group_id + '.json'), 'w'))

    def save_output(self, metrics, task, camera):

        energy_heatmap_path = self.output_group_dir / task / \
            ('htmp_energy_pc%s.png' % camera)
        energy_heatmap = metrics['energy_htmp']

        masked_data = np.ma.masked_where(energy_heatmap == 0, energy_heatmap)

        scaled_heatmap = masked_data.astype('float64')
        scaled_heatmap *= 255.0/scaled_heatmap.max()
        scaled_heatmap = scaled_heatmap.astype(int)

        # cmap = plt.get_cmap('Reds')
        cmap = plt.get_cmap("Reds")
        cmap.set_under('k', alpha=0)

        if self.verbose:
            plt.imshow(scaled_heatmap, cmap=cmap)
            plt.show()

        plt.imsave(str(energy_heatmap_path), scaled_heatmap, cmap=cmap)

    def process(self, tasks_directories: dict, specific_frame: int = None, display: bool = False):

        if specific_frame is not None:
            log('WARN', 'Impossible to calculate energy in a single specific frame. Skipping video processing step')
        else:
            for task in tasks_directories:
                output_directory = self.output_group_dir / task.name
                os.makedirs(output_directory, exist_ok=True)
                if not output_directory.is_dir():
                    log('ERROR', 'Directory does not exist')

                task_video_files = [
                    x for x in task.iterdir() if x.suffix in VALID_VIDEO_TYPES]
                video_caps = {int(re.search(r'(?<=Videopc)(\d{1})(?=\d{12})', x.name).group(
                    0)): cv2.VideoCapture(str(x)) for x in task_video_files}

                thread_list = list()
                for cap_cam, video_cap in video_caps.items():
                    thread = Worker(cap_cam, cap_cam, video_cap, output_directory,
                                    verbose=self.verbose, prettify=self.prettify, display=display)
                    thread.start()
                    thread_list.append(thread)

                for thread in thread_list:
                    thread.join()

                metrics = dict()
                for thread in thread_list:
                    metrics['energy_htmp'] = thread.cumulative_energy_frame
                    self.save_output(metrics, task.name, thread.camera)


from db_helpers import create_age_table, create_connection, query1, query2, query3, query4, query5
from people_helpers import retrieve_people_data
import yaml

"""
To begin, we must extract data form the RandomUser API. Once extracted, we supplement the data in 2 ways:

1) We use the Agify.io API to predict a person's age based on their first name.
    NOTE: If we already know a person's predicted age from a previous API call, we will NOT re-call
    the API to save time and (theoretical) money.
2) We use haversine package to calculate the distance of each person's latitude and longitude from Ford Field in miles.
    NOTE: Unfortunately, the latitude and longitude values from RandomUser do not correspond to the person's listed
    address it provides, so haversine produces unusual results often.

"""

# Produce and/or append to two tables: people and names
num_of_people = 200

conn = create_connection('db.sqlite')
people = retrieve_people_data(num_of_people, conn)
create_age_table(conn)

"""
Now, let's analyze some of our data using SQL. We have a file (sql_queries.yml) that contains our queries.
As we cycle through these queries, we will print a description of the query being run, and insights/results gathered in terminal.

"""

# Open and read the yml file of queries to pass to our db_helpers
with open('sql_queries.yml', 'r') as f:
    doc = yaml.load(f, Loader=yaml.FullLoader)

for i in range(1,6):
    successful = False
    while not successful:
        if input(doc[f'query{i}']['prompt']).lower() in ['y','yes']:
            if i == 1:
                query1(conn, doc)
            elif i ==2:
                query2(conn, doc)
            elif i == 3:
                query3(conn, doc)
            elif i == 4:
                query4(conn, doc)
            elif i == 5:
                query5(conn, doc)
            successful = True
    continue

conn.close()