import torch
import numpy as np
from sklearn.metrics import roc_curve, auc, precision_recall_curve
import math
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
#将预测概率转换为预测标签。
def Prediction_label(pred):
    a,b,c=pred.size()
    pred_arry=pred.detach().numpy()
    # print(str(a)+' '+str(b)+' '+str(c))
    predlabels=np.zeros((a,b,c))
    i=0
    while i<a:
        j=0
        while j<b:
            k=0
            while k<c:
                if pred_arry[i][j][k]>0.5:
                    predlabels[i][j][k]=1.0
                else:
                    predlabels[i][j][k]=0.0
                k+=1
            j+=1
        i+=1
    return torch.from_numpy(predlabels)

#attention mask
def Zero_One_Mask(pred):
    a,b,c=pred.size()
    pred_arry=pred.detach().numpy()
    # print(str(a)+' '+str(b)+' '+str(c))
    i=0
    while i<a:
        j=0
        while j<b:
            k=0
            while k<c:
                if pred_arry[i][j][k]>0.5:
                    pred_arry[i][j][k]=1.0
                else:
                    pred_arry[i][j][k]=0.0
                k+=1
            j+=1
        i+=1
    return torch.from_numpy(pred_arry)

#index mask
def Index_Mask(pred,inputs_site,scale=0.5):
    a, b, c = pred.size()
    # print(type(scale))
    d=math.ceil(c*scale)

    a1,b1,c1 = inputs_site.size()
    pred_arry = pred.detach().numpy()
    inputs_site = inputs_site.detach().numpy()
    # 获得mask后的训练概率和训练标签
    pred_mask = np.zeros((a, b, d))
    pred_mask_index = np.zeros((a, b, d))
    inputs_site_mask = np.zeros((a1, b1, d))

    i=0
    while i < a:
        j = 0
        while j < b:
            k = 0
            counter=0
            index=0
            while k < c:
                #根据scale对预测序列和标签序列进行裁切
                if inputs_site[i][j][k] == 1.0 and counter<=3:
                    counter+=1
                    pred_mask_index[i][j][index] = k #记录正样本位置
                    pred_mask[i][j][index]=pred_arry[i][j][k]
                    inputs_site_mask[i][j][index]=inputs_site[i][j][k]
                    index+=1
                k += 1
            # print(str(inputs_site[i][j])+' '+str(counter))
            while index < d:
                n=0
                while n < len(inputs_site[i][j]):
                    if inputs_site[i][j][n]==0.0:
                        pred_mask_index[i][j][index] = n  # 记录正样本位置
                        pred_mask[i][j][index] = pred_arry[i][j][n]
                        inputs_site_mask[i][j][index] = inputs_site[i][j][n]
                        index += 1
                        break
                    n+=1

            j += 1
        i += 1

    return torch.from_numpy(pred_mask_index).float(),torch.from_numpy(pred_mask).float(),torch.from_numpy(inputs_site_mask) .float()#下标，概率，位点与否

def Sequence_reduction(pred,inputs_site,ID_len): #将窗口中的转态全部转换为二维列表，左边为状态，右边为标签
    a, b, c = pred.size()
    pred_arry = pred.detach().numpy()
    inputs_site = inputs_site.detach().numpy()
    ID_len_array=ID_len.detach().numpy()
    # 获得mask后的训练概率和训练标签

    #用于指标计算
    prediction=[]
    label=[]
    #用于序列还原
    ID_prediction=[]
    ID_label=[]

    i=0
    while i < a:
        j = 0
        while j < b:
            k = 0

            while k < int(ID_len_array[i][j][1]):
                temp_id_pre = []
                temp_id_lab = []
                #根据scale对预测序列和标签序列进行裁切
                prediction.append(float(pred_arry[i][j][k]))
                label.append(float(inputs_site[i][j][k]))

                temp_id_pre.append(ID_len_array[i][j][0])
                temp_id_pre.append(float(pred_arry[i][j][k]))

                temp_id_lab.append(ID_len_array[i][j][0])
                temp_id_lab.append(float(inputs_site[i][j][k]))

                ID_prediction.append(temp_id_pre)
                ID_label.append(temp_id_lab)

                k += 1
            j += 1
        i += 1

    return prediction,label,ID_prediction,ID_label

def Sequence_reduction1(pred,inputs_site,ID_len): #将窗口中的转态全部转换为二维列表，左边为状态，右边为标签
    a, b, c = pred.size()
    pred_arry = pred.detach().numpy()
    inputs_site = inputs_site.detach().numpy()
    ID_len_array=ID_len.detach().numpy()
    # 获得mask后的训练概率和训练标签

    #用于指标计算
    prediction=[]
    label=[]
    #用于序列还原
    ID_prediction=[]
    ID_label=[]

    i=0
    while i < a:
        j = 0
        while j < b:
            k = 0
            while k < c:
                temp_id_pre = []
                temp_id_lab = []
                #根据scale对预测序列和标签序列进行裁切
                prediction.append(float(pred_arry[i][j][k]))
                label.append(float(inputs_site[i][j][k]))

                temp_id_pre.append(ID_len_array[i][j][0])
                temp_id_pre.append(float(pred_arry[i][j][k]))

                temp_id_lab.append(ID_len_array[i][j][0])
                temp_id_lab.append(float(inputs_site[i][j][k]))

                ID_prediction.append(temp_id_pre)
                ID_label.append(temp_id_lab)

                k += 1
            j += 1
        i += 1

    return prediction,label,ID_prediction,ID_label

#多维张量指定维度对角化
def tensor_diag(input):
    a,b=input.size()
    input_arry=input.detach().numpy()
    input_diag_array=np.zeros((a,b,b))
    # print(str(a)+' '+str(b)+' '+str(c))
    i=0
    while i<a:
        j=0
        while j<b:
            input_diag_array[i][j][j]=input_arry[i][j]
            j+=1
        i+=1
    return torch.from_numpy(input_diag_array).float()

#最大最小归一化以及硬标签转换
def max_min(input):
    a, b, c = input.size()
    threahold=0.6
    input_array = input.detach().numpy()
    i = 0
    while i < a:
        j = 0
        while j < b:
            k=0
            max_x=max(input_array[i][j])
            min_x=min(input_array[i][j])
            # print('最大值 '+str(max_x)+' 最小值 '+str(min_x))
            diff=max_x-min_x
            while k<c:
                input_array[i][j][k]=((input_array[i][j][k])-min_x)/diff
                if input_array[i][j][k]<threahold:
                    input_array[i][j][k]=0.0
                k+=1
            j += 1

        i += 1
    return torch.from_numpy(input_array).float()

def max_min_2D(input):
    a, b = input.size()
    threahold=0.5
    input_array = input.detach().numpy()
    i = 0
    while i < a:
        j = 0
        max_x=max(input_array[i])
        min_x=min(input_array[i])
        diff=max_x-min_x
        while j<b:
            input_array[i][j]=((input_array[i][j])-min_x)/diff
            if input_array[i][j]<threahold:
                input_array[i][j]=0.0

            j += 1
        i += 1
    return torch.from_numpy(input_array).float()

def pos_neg_mask(input,pos_mask,neg_mask):
    a, b = input.size()
    input_array = input.detach().numpy()
    pos_mask = pos_mask.detach().numpy()
    neg_mask = neg_mask.detach().numpy()
    i = 0
    while i < a:
        j = 0
        while j < b:
            if input_array[i][j]<0:
                neg_mask[i][j]=0
            else:
                pos_mask[i][j]=0
            j += 1
        i += 1
    return torch.from_numpy(pos_mask).float(),torch.from_numpy(neg_mask).float()

def find_duplicates(lst):
    indices = {}
    for i, x in enumerate(lst):
        if x not in indices:
            indices[x] = [i]
        else:
            indices[x].append(i)
    return [v for v in indices.values() if len(v) > 1]

def normalize_motif(ids, tensor):

    for row in ids:
        idxs = row
        slices = [tensor[:, idx, :] for idx in idxs]
        mean_ = torch.stack(slices, axis=0).mean()
        for slice_ in slices:
            if mean_ != 0:
                slice_ /= mean_
            else:
                slice_.fill_(0)

    return tensor

def auroc(prob, label):
    y_true = label.data.numpy().flatten()
    y_scores = prob.data.numpy().flatten()
    y_scores=np.nan_to_num(y_scores)
    fpr, tpr, thresholds = roc_curve(y_true, y_scores)
    auroc_score = auc(fpr, tpr)
    return auroc_score, fpr, tpr

def AUC(prob, label):

    fpr, tpr, thresholds = roc_curve(label, prob)
    auroc_score = auc(fpr, tpr)
    return auroc_score

def auprc(prob,label):
    y_true=label.data.numpy().flatten()
    y_scores=prob.data.numpy().flatten()
    y_scores = np.nan_to_num(y_scores)
    precision,recall,thresholds=precision_recall_curve(y_true,y_scores)
    auprc_score=auc(recall,precision)
    return auprc_score,precision,recall

def AUPR(prob,label):
    precision,recall,thresholds=precision_recall_curve(prob,label)
    auprc_score=auc(recall,precision)
    return auprc_score

def MyRealAndPrediction(Real,prediction):
    RealAndPrediction = []
    counter = 0
    while counter < len(prediction):
        pair = []
        pair.append(Real[counter])
        pair.append(prediction[counter])
        RealAndPrediction.append(pair)
        counter = counter + 1
    return RealAndPrediction
def MyRealAndPredictionProb(Real,prediction):
    RealAndPredictionProb = []
    counter = 0
    while counter < len(prediction):
        pair = []
        pair.append(Real[counter])
        pair.append(prediction[counter])
        RealAndPredictionProb.append(pair)
        counter = counter + 1
    return RealAndPredictionProb

def Indicator(y_real,y_predict):
    from sklearn.metrics import confusion_matrix
    y_real=y_real.data.numpy().flatten()
    y_predict=y_predict.data.numpy().flatten()
    x=[]
    y=[]
    z=[]
    for ele in y_real:
        ele=int(ele)
        x.append(ele)

    for ele in y_predict:
        ele=float(ele)
        z.append(ele)
        if ele >0.5:
            y.append(1)
        else:
            y.append(0)

    RealAndPrediction = MyRealAndPrediction(x, y)
    RealAndPredictionProb = MyRealAndPredictionProb(x, z)

    CM = confusion_matrix(x, y)
    print(CM)
    CM = CM.tolist()
    TN = CM[0][0]
    FP = CM[0][1]
    FN = CM[1][0]
    TP = CM[1][1]

    Acc = (TN + TP) / (TN + TP + FN + FP)
    if (TP+FN)!=0:
        Sen = TP / (TP + FN)
    else:
        Sen=np.inf
    if (TN + FP)!=0:
        Spec = TN / (TN + FP)
    else:
        Spec=np.inf
    if (TP + FP)!=0:
        Prec = TP / (TP + FP)
    else:
        Prec=np.inf
    if math.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))!=0:
        MCC = (TP * TN - FP * FN) / math.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))
    else:
        MCC=np.inf
    f1 = f1_score(y, x)
    Auc,_,_= auroc(torch.tensor(y_predict),torch.tensor(x))
    Aupr,_,_=auprc(torch.tensor(y_predict),torch.tensor(x))

    Result = []
    Result.append(round(MCC, 4))
    Result.append(round(Auc, 4))
    Result.append(round(Aupr, 4))
    Result.append(round(f1, 4))
    Result.append(round(Acc, 4))
    Result.append(round(Sen, 4))
    Result.append(round(Spec, 4))
    Result.append(round(Prec, 4))

    return Result,RealAndPrediction,RealAndPredictionProb

def MyConfusionMatrix(y_real, y_predict):
    from sklearn.metrics import confusion_matrix
    y_real = y_real
    y_predict = y_predict

    x = []
    y = []
    z = []
    for ele in y_real:
        ele = int(ele)
        x.append(ele)

    for ele in y_predict:
        ele = float(ele)
        z.append(ele)
        if ele > 0.5:
            y.append(1)
        else:
            y.append(0)

    CM = confusion_matrix(x, y)
    print(CM)
    CM = CM.tolist()
    TN = CM[0][0]
    FP = CM[0][1]
    FN = CM[1][0]
    TP = CM[1][1]
    print('TN:%d, FP:%d, FN:%d, TP:%d' % (TN, FP, FN, TP))
    Acc = (TN + TP) / (TN + TP + FN + FP)
    if (TP + FN) != 0:
        Sen = TP / (TP + FN)
    else:
        Sen = np.inf
    if (TN + FP) != 0:
        Spec = TN / (TN + FP)
    else:
        Spec = np.inf
    if (TP + FP) != 0:
        Prec = TP / (TP + FP)
    else:
        Prec = np.inf
    if math.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)) != 0:
        MCC = (TP * TN - FP * FN) / math.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))
    else:
        MCC = np.inf
    f1 = f1_score(y, x)
    Auc = AUC(torch.tensor(y_predict), torch.tensor(x))
    Aupr = AUPR(torch.tensor(y), torch.tensor(x))

    print('MCC:'+str(round(MCC, 4))+' AUC:'+str(round(Auc, 4))+' AUPR:',str(round(Aupr, 4))+
          ' F1:'+str(round(f1, 4))+' Acc:'+str(round(Acc, 4)) + ' Sen:'+ str(round(Sen, 4))+' Spec:', str(round(Spec, 4))+' Prec:'+str(round(Prec, 4)))

    Result = [] #len=8
    Result.append(round(MCC, 4))
    Result.append(round(Auc, 4))
    Result.append(round(Aupr, 4))
    Result.append(round(f1, 4))
    Result.append(round(Acc, 4))
    Result.append(round(Sen, 4))
    Result.append(round(Spec, 4))
    Result.append(round(Prec, 4))
    return Result

#标准化
def normalized_input(input):
    a,b,c=input.size()
    input=input.detach().numpy()
    # print(str(a)+' '+str(b)+' '+str(c))
    new_input=np.zeros((a,b,c))
    i=0
    while i<a:
        j=0
        while j<b:
            k=0
            while k<c:
                new_input[i][j][k]=(input[i][j][k]-min(input[i][j]))/(max(input[i][j])-min(input[i][j]))
                k+=1
            j+=1
        i+=1
    return torch.from_numpy(new_input).float()


import os
import json
import copy
import logging
import functools
import numpy as np
from typing import List, Optional, Union, Tuple
from seqeval.metrics import classification_report
from seqeval.scheme import IOB2

import torch
from torch.utils.data import DataLoader
from seqlbtoolkit.data import entity_to_bio_labels, one_hot, probs_to_lbs
from seqlbtoolkit.embs import build_bert_token_embeddings

from .args import CHMMConfig
from ..utils.io import load_data_from_json, load_data_from_pt

logger = logging.getLogger(__name__)

# noinspection PyBroadException
class CHMMBaseDataset(torch.utils.data.Dataset):
    def __init__(self,
                 text: Optional[List[List[str]]] = None,
                 embs: Optional[List[torch.Tensor]] = None,
                 obs: Optional[List[torch.Tensor]] = None,  # batch, src, token
                 lbs: Optional[List[List[str]]] = None,
                 src: Optional[List[str]] = None,
                 ents: Optional[List[str]] = None):
        super().__init__()
        self._embs = embs
        self._obs = obs
        self._text = text
        self._lbs = lbs
        self._src = src
        self._ents = ents
        self._src_metrics = None

    @property
    def n_insts(self):
        return len(self.obs)

    @property
    def embs(self):
        return self._embs if self._embs else list()

    @property
    def text(self):
        return self._text if self._text else list()

    @property
    def lbs(self):
        return self._lbs if self._lbs else list()

    @property
    def obs(self):
        return self._obs if self._obs else list()

    @property
    def src(self):
        return self._src

    @property
    def ents(self):
        return self._ents

    @property
    def src_metrics(self):
        if self._src_metrics is None:
            self._src_metrics = self._get_src_metrics()
        return self._src_metrics

    @text.setter
    def text(self, value):
        logger.warning(f'{type(self)}: text has been changed')
        self._text = value

    @obs.setter
    def obs(self, value):
        logger.warning(f'{type(self)}: observations have been changed')
        self._obs = value

    @lbs.setter
    def lbs(self, value):
        logger.warning(f'{type(self)}: labels have been changed')
        self._lbs = value

    @embs.setter
    def embs(self, value):
        logger.warning(f'{type(self)}: embeddings have been changed')
        self._embs = value

    @src.setter
    def src(self, value):
        logger.warning(f'{type(self)}: sources have been changed')
        self._src = value

    @ents.setter
    def ents(self, value):
        logger.warning(f'{type(self)}: entity types have been changed')
        self._ents = value

    @src_metrics.setter
    def src_metrics(self, value):
        self._src_metrics = value

    def __len__(self):
        return self.n_insts

    def __getitem__(self, idx):
        if self._lbs is not None and len(self._lbs) > 0:
            return self._text[idx], self._embs[idx], self._obs[idx], self._lbs[idx]
        else:
            return self._text[idx], self._embs[idx], self._obs[idx]

    def __add__(self, other: "CHMMBaseDataset") -> "CHMMBaseDataset":
        assert self.src and other.src and self.src == other.src, ValueError("Sources not matched!")
        assert self.ents and other.ents and self.ents == other.ents, ValueError("Entity types not matched!")

        return CHMMBaseDataset(
            text=copy.deepcopy(self.text + other.text),
            embs=copy.deepcopy(self.embs + other.embs),
            obs=copy.deepcopy(self.obs + other.obs),
            lbs=copy.deepcopy(self.lbs + other.lbs),
            ents=copy.deepcopy(self.ents),
            src=copy.deepcopy(self.src)
        )

    def __iadd__(self, other: "CHMMBaseDataset") -> "CHMMBaseDataset":
        if self.src:
            assert other.src and self.src == other.src, ValueError("Sources do not match!")
        else:
            assert other.src, ValueError("Attribute `src` not found!")

        if self.ents:
            assert other.ents and self.ents == other.ents, ValueError("Entity types do not match!")
        else:
            assert other.ents, ValueError("Attribute `ents` not found!")

        self.text = copy.deepcopy(self.text + other.text)
        self.embs = copy.deepcopy(self.embs + other.embs)
        self.obs = copy.deepcopy(self.obs + other.obs)
        self.lbs = copy.deepcopy(self.lbs + other.lbs)
        self.ents = copy.deepcopy(other.ents)
        self.src = copy.deepcopy(other.src)
        return self

    def save(self, file_dir: str, dataset_type: str, config: CHMMConfig, force_save: Optional[bool] = False):
        """
        Save dataset for future usage

        Parameters
        ----------
        file_dir: the folder which the dataset will be stored in.
        dataset_type: decides if the dataset is training, validation or test set
        config: configuration file
        force_save: force to save the file even if a file of the same path exists.

        Returns
        -------
        None
        """
        assert dataset_type in ['train', 'valid', 'test']
        output_path = os.path.join(file_dir, f"{dataset_type}.chmmdp")
        if os.path.exists(output_path) and not force_save:
            return None

        chmm_data_dict = {
            'text': self.text,
            'lbs': self.lbs,
            'obs': self.obs,
            'src': self.src,
            'ents': self.ents,
            'embs': self.embs,
            'src_priors': config.src_priors
        }
        torch.save(chmm_data_dict, output_path)
        return None

    def load(self, file_dir: str, dataset_type: str, config: CHMMConfig):
        """
        Load saved datasets and configurations

        Parameters
        ----------
        file_dir: the folder which the dataset is stored in.
        dataset_type: decides if the dataset is training, validation or test set
        config: configuration file

        Returns
        -------
        self
        """
        assert dataset_type in ['train', 'valid', 'test']
        if os.path.isdir(file_dir):
            file_path = os.path.join(file_dir, f'{dataset_type}.chmmdp')
            assert os.path.isfile(file_path), FileNotFoundError(f"{file_path} does not exist!")
        else:
            raise FileNotFoundError(f"{file_dir} does not exist!")

        chmm_data_dict = torch.load(file_path)
        for attr, value in chmm_data_dict.items():
            if attr == 'src_priors':
                continue
            try:
                setattr(self, f'_{attr}', value)
            except AttributeError as err:
                logger.exception(f"Failed to set attribute {attr}: {err}")
                raise err

        config.sources = copy.deepcopy(self.src)
        config.entity_types = copy.deepcopy(self.ents)
        config.bio_label_types = entity_to_bio_labels(self.ents)
        config.src_priors = chmm_data_dict['src_priors']
        config.d_emb = self._embs[0].shape[-1]
        return self

    def load_file(self,
                  file_path: str,
                  config: Optional[CHMMConfig] = None) \
            -> Union["CHMMBaseDataset", Tuple["CHMMBaseDataset", "CHMMConfig"]]:
        """
        Load data from disk

        Parameters
        ----------
        file_path: the directory of the file. In JSON or PT
        config: chmm configuration; Optional to make function testing easier.

        Returns
        -------
        self (MultiSrcNERDataset)
        """
        if config is None:
            has_config_input = False
            config = CHMMConfig()
        else:
            has_config_input = True

        bert_model = getattr(config, "bert_model_name_or_path", 'bert-base-uncased')
        device = getattr(config, "device", torch.device('cpu'))

        file_path = os.path.normpath(file_path)
        logger.info(f'Loading data from {file_path}')

        file_dir, file_name = os.path.split(file_path)
        if file_path.endswith('.json'):
            sentence_list, label_list, weak_label_list = load_data_from_json(file_path, config)
            # get embedding directory
            emb_name = f"{'.'.join(file_name.split('.')[:-1])}-emb.pt"
            emb_dir = os.path.join(file_dir, emb_name)
        # for backward compatibility
        elif file_path.endswith('.pt'):
            sentence_list, label_list, weak_label_list = load_data_from_pt(file_path, config)
            # get embedding directory
            emb_dir = file_path.replace('linked', 'emb')
        else:
            logger.error(f"Unsupported data type: {file_path}")
            raise TypeError(f"Unsupported data type: {file_path}")

        if getattr(config, 'load_src_metrics', False):
            try:
                metrics = load_src_metrics(os.path.join(file_dir, 'src_metrics.json'))
                metrics = {k: metrics[k] for k in config.sources}
                self.src_metrics = metrics
            except Exception as e:
                logger.exception(f"Failed to load source metrics: {e}.")

        self._text = sentence_list
        self._lbs = label_list
        self._obs = weak_label_list
        logger.info(f'Data loaded from {file_path}.')

        logger.info(f'Searching for corresponding BERT embeddings...')
        if os.path.isfile(emb_dir):
            logger.info(f"Found embedding file: {emb_dir}. Loading to memory...")
            embs = torch.load(emb_dir)
            if isinstance(embs[0], torch.Tensor):
                self._embs = embs
            elif isinstance(embs[0], np.ndarray):
                self._embs = [torch.from_numpy(emb).to(torch.float) for emb in embs]
            else:
                logger.error(f"Unknown embedding type: {type(embs[0])}")
                raise RuntimeError
        else:
            logger.info(f"{emb_dir} does not exist. Building embeddings instead...")

            if not has_config_input:
                logger.warning(f"No configuration found. Using default bert model `bert-base_model-uncased` "
                               f"and default device `cpu`.")
            self.build_embs(bert_model, device, emb_dir)

        self._src = copy.deepcopy(config.sources)
        self._ents = config.entity_types
        config.d_emb = self._embs[0].shape[-1]
        if getattr(config, 'debug_mode', False):
            self._embs = self._embs[:100]

        # append dummy token/labels in front of the text/lbs/obs
        logger.info("Appending dummy token/labels in front of the text/lbs/obs for CHMM compatibility")
        self._text = [['[CLS]'] + txt for txt in self._text]
        self._lbs = [['O'] + lb for lb in self._lbs]
        prefix = torch.zeros([1, self._obs[0].shape[-2], self._obs[0].shape[-1]])  # shape: 1, n_src, d_obs
        prefix[:, :, 0] = 1
        self._obs = [torch.cat([prefix, inst]) for inst in self._obs]

        if has_config_input:
            return self
        else:
            return self, config

    def build_embs(self,
                   bert_model,
                   device: Optional[torch.device] = torch.device('cpu'),
                   save_dir: Optional[str] = None) -> "CHMMBaseDataset":
        """
        build bert embeddings

        Parameters
        ----------
        bert_model: the location/name of the bert model to use
        device: device
        save_dir: location to update/store the BERT embeddings. Leave None if do not want to save

        Returns
        -------
        self (MultiSrcNERDataset)
        """
        assert bert_model is not None, AssertionError('Please specify BERT model to build embeddings')
        logger.info(f'Building BERT embeddings with {bert_model} on {device}')
        self._embs = build_bert_token_embeddings(
            self._text, bert_model, bert_model, device=device, prepend_cls_embs=True
        )
        if save_dir:
            save_dir = os.path.normpath(save_dir)
            logger.info(f'Saving embeddings to {save_dir}...')
            embs = [emb.numpy().astype(np.float32) for emb in self.embs]
            torch.save(embs, save_dir)
        return self

    def update_obs(self,
                   obs: List[List[Union[int, str]]],
                   src_name: str,
                   config: CHMMConfig):
        """
        update weak labels (chmm observations)

        Parameters
        ----------
        obs: input observations (week annotations)
        src_name: source name
        config: configuration file

        Returns
        -------
        self (MultiSrcNERDataset)
        """
        if isinstance(obs[0][0], str):
            lb2ids = {lb: i for i, lb in enumerate(config.bio_label_types)}
            np_map = np.vectorize(lambda lb: lb2ids[lb])
            obs = [np_map(np.asarray(weak_lbs)).tolist() for weak_lbs in obs]

        if len(obs[0]) == len(self.text[0]):
            weak_lbs_one_hot = [one_hot(np.asarray(weak_lbs), n_class=config.n_lbs) for weak_lbs in obs]
        elif len(obs[0]) == len(self.text[0]) - 1:
            weak_lbs_one_hot = [one_hot(np.asarray([0] + weak_lbs), n_class=config.n_lbs) for weak_lbs in obs]
        else:
            logger.error("The length of the input observation does not match the dataset sentences!")
            raise ValueError("The length of the input observation does not match the dataset sentences!")

        if src_name in self._src:
            src_idx = self._src.index(src_name)
            for i in range(len(self._obs)):
                self._obs[i][:, src_idx, :] = torch.tensor(weak_lbs_one_hot[i])
        else:
            self._src.append(src_name)
            for i in range(len(self._obs)):
                self._obs[i] = torch.cat([self._obs[i], torch.tensor(weak_lbs_one_hot[i]).unsqueeze(1)], dim=1)
            # add the source into config and give a heuristic source prior
            if src_name not in config.sources:
                config.sources.append(src_name)
                config.src_priors[src_name] = {ent: (0.7, 0.7) for ent in config.entity_types}

            # remove the cached property
            try:
                delattr(self, "src_metrics")
            except Exception:
                pass

        return self

    def remove_src(self,
                   src_name: str,
                   config: CHMMConfig):
        """
        remove a source and its observations from the dataset

        Parameters
        ----------
        src_name: source name
        config: configuration file

        Returns
        -------
        self (MultiSrcNERDataset)
        """

        if src_name in config.sources:
            config.sources.remove(src_name)
        if src_name in config.src_priors.keys():
            config.src_priors.pop(src_name, None)

        if src_name not in self._src:
            logger.warning(f"Labeling function {src_name} is not presented in dataset. Nothing is changed!")
            return self

        # remove source name
        src_idx = self._src.index(src_name)
        other_idx = list(range(len(self.src)))
        other_idx.remove(src_idx)

        # remove the corresponding observation
        self._src.remove(src_name)
        for i in range(len(self._obs)):
            self._obs[i] = self._obs[i][:, other_idx, :]

        # remove the cached property
        try:
            delattr(self, "src_metrics")
        except Exception:
            pass

        return self

    @functools.cache
    def _get_src_metrics(self):
        src_record_list = [list() for _ in range(len(self.src))]
        for obs in self._obs:
            src_lbs_list = probs_to_lbs(obs, entity_to_bio_labels(self.ents)).T.tolist()
            for src_lbs, src_record in zip(src_lbs_list, src_record_list):
                src_record.append(src_lbs)
        metric_dict = dict()
        for src, record in zip(self.src, src_record_list):
            report = classification_report(
                self.lbs, record, output_dict=True, mode='strict', zero_division=0, scheme=IOB2
            )
            report_dict = dict()
            for ent in self.ents:
                report_dict[ent] = {
                    'precision': report[ent]['precision'],
                    'recall': report[ent]['recall'],
                    'f1': report[ent]['f1-score']
                }
                report_dict['micro-avg'] = {
                    'precision': report['micro avg']['precision'],
                    'recall': report['micro avg']['recall'],
                    'f1': report['micro avg']['f1-score']
                }
            metric_dict[src] = report_dict
        return metric_dict

def batch_prep(emb_list: List[torch.Tensor],
               obs_list: List[torch.Tensor],
               txt_list: Optional[List[List[str]]] = None,
               lbs_list: Optional[List[dict]] = None):
    """
    Pad the instance to the max seq max_seq_length in batch

    All input should already have the dummy element appended to the beginning of the sequence
    """
    for emb, obs, txt, lbs in zip(emb_list, obs_list, txt_list, lbs_list):
        assert len(obs) == len(emb) == len(txt) == len(lbs)
    d_emb = emb_list[0].shape[-1]
    _, n_src, n_obs = obs_list[0].size()
    seq_lens = [len(obs) for obs in obs_list]
    max_seq_len = np.max(seq_lens)

    emb_batch = torch.stack([
        torch.cat([inst, torch.zeros([max_seq_len-len(inst), d_emb])], dim=-2) for inst in emb_list
    ])

    prefix = torch.zeros([1, n_src, n_obs])
    prefix[:, :, 0] = 1
    obs_batch = torch.stack([
        torch.cat([inst, prefix.repeat([max_seq_len-len(inst), 1, 1])]) for inst in obs_list
    ])
    obs_batch /= obs_batch.sum(dim=-1, keepdim=True)

    seq_lens = torch.tensor(seq_lens, dtype=torch.long)

    # we don't need to append the length of txt_list and lbs_list
    return emb_batch, obs_batch, seq_lens, txt_list, lbs_list

def collate_fn(insts):
    """
    Principle used to construct dataloader

    :param insts: original instances
    :return: padded instances
    """
    all_insts = list(zip(*insts))
    if len(all_insts) == 4:
        txt, embs, obs, lbs = all_insts
        batch = batch_prep(emb_list=embs, obs_list=obs, txt_list=txt, lbs_list=lbs)
    elif len(all_insts) == 3:
        txt, embs, obs = all_insts
        batch = batch_prep(emb_list=embs, obs_list=obs, txt_list=txt)
    else:
        logger.error('Unsupported number of instances')
        raise ValueError('Unsupported number of instances')
    return batch

def load_src_metrics(file_path: str):
    """
    Load the file that contains the performance of each source

    Parameters
    ----------
    file_path: file path

    Returns
    -------
    source performance: dict[dict]
    """
    with open(file_path, 'r', encoding='UTF-8') as f:
        metrics = json.load(f)
    return metrics


import seldom
from seldom.utils import cache
from seldom import depend

class TestSafetyIndex(seldom.TestCase):

    def start(self):
        self.store_id = cache.get("store_id")
        self.store_name = cache.get("store_name")
        self.oper_id = cache.get("oper_id")

    def test_01_GetFoodSafetyIndex_Store_out(self):    # 门店安康指数
        url = f"http://dss.iuoooo.com/CockpitMobile/GetFoodSafetyIndex_Store_out"
        data_ = {
                "AppId": "2fd5f092-d360-4783-b95c-6d47ee5cac44",
                "IndexCode": 4,
                "StoreId": self.store_id,
            }
        self.post(url, json=data_)
        self.assertStatusCode(200)
        self.assertPath('SCater.Data.FoodSafetyIndex_StoreList[0].IndexNameText', "当前企业安康指数")

    # 企业信息
    def test_02_Company_html(self):
        url = "http://dss.iuoooo.com/Html/map/Company/Company.html?newpage=1&curChangeOrg=6a0376d0-9196-4792-a2ca-f6b6e2d6fc8b&changeOrg=6a0376d0-9196-4792-a2ca-f6b6e2d6fc8b&sessionId=9f900416-882b-4996-9956-d944e3c6f74c&storeId=ff223c14-aacb-4a3d-be72-548000c617d6&userId=3c73f483-0a2f-4838-b04b-58a2eb64ebd0&jhWebView=1&orgId=6a0376d0-9196-4792-a2ca-f6b6e2d6fc8b&shopOrgId=ab9d121e-0145-41d8-b787-460480290d42&isLogin=1&hidjhnavigation=1&appId=2fd5f092-d360-4783-b95c-6d47ee5cac44&IndexCode=4&roleName=%E7%AE%A1%E7%90%86%E5%91%98&RiskType=0&account=18800000064&relationOId=6a0376d0-9196-4792-a2ca-f6b6e2d6fc8b"
        self.get(url)
        self.assertStatusCode(200)

    def test_03_PermissionHuzhou(self):
        url = "http://dss.iuoooo.com/CockpitMobile/PermissionHuzhou"
        data_ = {"AppId": "2fd5f092-d360-4783-b95c-6d47ee5cac44",
                 "StoreId": self.store_id
                 }
        self.post(url, json=data_)
        self.assertStatusCode(200)

    def test_04_OperateFoodSafety(self):
        url = "http://dss.iuoooo.com/CockpitMobile/OperateFoodSafety"
        data_ = {"IndexCode": 4,
                 "StoreId": self.store_id,
                 "AppId": "2fd5f092-d360-4783-b95c-6d47ee5cac44",
                 "EBCOrganizationId": "6a0376d0-9196-4792-a2ca-f6b6e2d6fc8b"
                 }
        self.post(url, json=data_)
        self.assertStatusCode(200)
        self.assertPath("SCater.Code", 200)

    def test_05_GetStorDataInfo(self):
        url = "http://dss.iuoooo.com/CockpitMobile/GetStorDataInfo"
        data_ = {
                 "StoreId": self.store_id
                 }
        self.post(url, json=data_)
        self.assertStatusCode(200)
        self.assertPath("RM.Code", 200)

    def test_06_RankDistance(self):
        url = "http://dss.iuoooo.com/CockpitMobile/RankDistance"
        data_ = {"IndexCode": 4,
                 "StoreId": self.store_id,
                 "AppId": "2fd5f092-d360-4783-b95c-6d47ee5cac44",
                 "EBCOrganizationId": "6a0376d0-9196-4792-a2ca-f6b6e2d6fc8b"
                 }
        self.post(url, json=data_)
        self.assertStatusCode(200)
        self.assertPath("SCater.Code", 200)

    def test_07_OperateFoodDiagnostic(self):
        url = "http://dss.iuoooo.com/CockpitMobile/OperateFoodDiagnostic"
        data_ = {"IndexCode": 4,
                 "StoreId": self.store_id,
                 "AppId": "2fd5f092-d360-4783-b95c-6d47ee5cac44",
                 "EBCOrganizationId": "6a0376d0-9196-4792-a2ca-f6b6e2d6fc8b"
                 }
        self.post(url, json=data_)
        self.assertStatusCode(200)
        self.assertPath("SCater.Code", 200)

    # 动态排行
    def test_08_RankDistanceOut(self):
        url = "https://dss.iuoooo.com/CockpitMobile/RankDistanceOut"
        data_ = {
                "IndexCode": 4,
                "appid": "2fd5f092-d360-4783-b95c-6d47ee5cac44",
                "storeid": self.store_id
            }
        self.post(url, json=data_)
        try:
            self.assertStatusCode(200)
            self.assertPath("SCater.Code", 200)
            count = len(self.jsonpath("$..Id"))
            if count < 0:
                print("获取门店数据失败")
                raise AssertionError
        except AssertionError as e:
            print('执行结果:Failed')
            raise e


import tensorflow as tf
tfe = tf.contrib.eager
tfs = tf.contrib.summary
import matplotlib.pyplot as plt
import argparse
import os

from tqdm import tqdm

from utils import is_valid_file
from vae import VAE

tf.enable_eager_execution()

def mnist_input_fn(data, batch_size=256, shuffle_samples=5000):
    dataset = tf.data.Dataset.from_tensor_slices(data)
    dataset = dataset.shuffle(shuffle_samples)
    dataset = dataset.map(mnist_parse_fn)
    dataset = dataset.batch(batch_size)

    return dataset

def mnist_parse_fn(data):
    return tf.cast(data, tf.float32) / 255.

def run(args):

    # ==========================================================================
    # Configuration
    # ==========================================================================

    config = {
        "num_training_examples": 60000,
        "batch_size": 250,
        "num_epochs": 10,
        "learning_rate": 1e-3,
        "checkpoint_postfix": "ckpt",
        "log_freq": 100,
    }

    num_batches = config["num_training_examples"] // config["batch_size"] + 1

    # ==========================================================================
    # Load dataset
    # ==========================================================================

    ((train_data, _),
     (eval_data, _)) = tf.keras.datasets.mnist.load_data()

    # ==========================================================================
    # Create model
    # ==========================================================================

    vae = VAE(input_shape=(28, 28),
              num_units=400)
    vae(tf.zeros((1, 28, 28)))

    optimizer = tf.train.RMSPropOptimizer(learning_rate=config["learning_rate"])

    # ==========================================================================
    # Define Checkpoints
    # ==========================================================================

    global_step = tf.train.get_or_create_global_step()

    ckpt_prefix = os.path.join(args.model_dir, config["checkpoint_postfix"])

    checkpoint = tf.train.Checkpoint(**{v.name: v for v in vae.get_all_variables()})

    checkpoint.restore(tf.train.latest_checkpoint(ckpt_prefix))

    # ==========================================================================
    # Define Tensorboard Summary writer
    # ==========================================================================

    logdir = os.path.join(args.model_dir, "log")
    writer = tfs.create_file_writer(logdir)
    writer.set_as_default()

    # ==========================================================================
    # Train the model
    # ==========================================================================

    for epoch in range(1, config["num_epochs"] + 1):

        dataset = mnist_input_fn(data=train_data,
                                batch_size=config["batch_size"])

        with tqdm(total=num_batches) as pbar:
            for batch in dataset:
                # Increment global step
                global_step.assign_add(1)

                # Record gradients of the forward pass
                with tf.GradientTape() as tape, tfs.record_summaries_every_n_global_steps(config["log_freq"]):

                    output = vae(batch)

                    # negative ELBO
                    loss = vae.kl_divergence() - vae.input_log_prob()

                    output = tf.cast(tf.expand_dims(output, axis=-1), tf.float32)

                    # Add tensorboard summaries
                    tfs.scalar("Loss", loss)
                    tfs.image("Reconstruction", output)

                # Backprop
                grads = tape.gradient(loss, vae.get_all_variables())
                optimizer.apply_gradients(zip(grads, vae.get_all_variables()))

                # Update the progress bar
                pbar.update(1)
                pbar.set_description("Epoch {}, ELBO: {:.2f}".format(epoch, loss))

        checkpoint.save(ckpt_prefix)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='VAEs')

    parser.add_argument('--no_training', action="store_false", dest="is_training", default=True,
                    help='Should we just evaluate?')

    parser.add_argument('--model_dir', type=lambda x: is_valid_file(parser, x), default='/tmp/vae',
                    help='The model directory.')

    args = parser.parse_args()

    run(args)


# 풀이1. 재귀구조로 연결
# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next
class Solution:
    def mergeTwoLists(self, list1: Optional[ListNode], list2: Optional[ListNode]) -> Optional[ListNode]:
            if (not list1) or (list2 and list1.val > list2.val):    # 마지막 노드일 때, 더 큰 값을 찾았을 때
                list1, list2 = list2, list1
            if list1:   # 재귀가 종료될 조건(마지막 노드가 None임을 이용)
                list1.next = self.mergeTwoLists(list1.next, list2)

            return list1

Data Structure/Linear Data Structure/Linked List/21. Merge Two Sorted Lists.py


n = int(input())
k = int(input())

start = 1
end = k
answer = 0

while start <=end:
    mid = (start+end)//2
    count = 0
    for i in range(1,n+1):
        temp = min(mid//i,n)
        count+=temp
    if count < k:
        start = mid + 1
    else:
        answer = mid
        end = mid-1

print(answer)


# -*- coding: utf-8 -*-
"""
Created on Fri Nov 25 15:59:05 2022

@author: spitaler.t
"""

import numpy as np
import matplotlib.pyplot as plt

with open('input.txt', 'r') as f:
    A_input = f.read()
A = A_input.split('\n')

Xs = []
X = 1
for line in A:
    words = line.split()
    if words[0] == 'noop':
        Xs.append(X)
    elif words[0] == 'addx':
        Xs += [X,X]
        X+= int(words[1])

# Part 1
#Calculate the signal strenght
s_tot = 0
for i in range(20, 241, 40):
    s_tot += Xs[i-1]*i
print(s_tot)

##% part 2
X2 = np.array(Xs).reshape([6,40])
pixel_numbers = np.repeat(np.atleast_2d(np.arange(40)), 6, axis=0)
plt.imshow(np.abs(pixel_numbers-X2) <=1)


_ = int(input())
numbers = input().split()

numbers = list(map(str, numbers))

numbers.sort(reverse=True)

largest = int("".join(numbers))
largest_2 = list(map(int, str(largest)))
largest_2.sort(reverse=True)

print("Numbers: ", numbers)
print("Largest: ", largest)
print("Largest_2: ", largest_2)


import unittest
from typing import List, Any
from collections import defaultdict
import math
import heapq

class Dijkstra(unittest.TestCase):
    def dijkstra(self, edges: List[List[Any]]) -> int:
        graph = defaultdict(list)
        for u, v, c in edges:
            graph[u].append([v, c])
            graph[v].append([u, c])

        visited = {}
        for vertex in graph:
            visited[vertex] = math.inf

        heap = [(0, 'a')]
        while heap:
            cost, vertex = heapq.heappop(heap)

            if cost < visited[vertex]:
                visited[vertex] = cost
                for v, c in graph[vertex]:
                    heapq.heappush(heap, (cost + c, v))

        return visited['c']

    def test_1(self):
        edges = [['a', 'd', 1], ['a', 'b', 6], ['d', 'b', 2], ['d', 'e', 1], ['b', 'e', 2], ['b', 'c', 5], ['e', 'c', 5]]
        self.assertEqual(self.dijkstra(edges), 7)

def main():
    unittest.main()

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt
from ODESolver import *

class Region:
    def __init__(self, name, S_0, E2_0):
        self.name = name
        self.S_0  = S_0; self.E1_0 = 0; self.E2_0 = E2_0;
        self.I_0 = 0; self.Ia_0 = 0; self.R_0 = 0;

        self.population = sum([self.S_0, self.E2_0,
        self.I_0, self.Ia_0, self.R_0])

    def set_SEIR_values(self, u, t):
       self.S = u[:,0]; self.E1 = u[:,1]; self.E2 = u[:,2];
       self.I = u[:,3]; self.Ia = u[:,4]; self.R = u[:,5]
       self.t = t

    def plot(self):
        S = self.S; I = self.I;
        Ia = self.Ia; R = self.R;
        t = self.t

        plt.plot(t, S, label = "Susceptible")
        plt.plot(t, I, label = "Symtomatically infected ")
        plt.plot(t, Ia, label = "Asymptomatically infected")
        plt.plot(t, R, label = "Resistant")
        plt.title(self.name)
        plt.xlabel("Time(days")
        plt.ylabel("Population")

class RegionInteraction(Region):

    def __init__(self, name, S_0, E2_0, lat, long):
        self.S_0  = S_0; self.E1_0 = 0; self.E2_0 = E2_0;
        self.I_0 = 0; self.Ia_0 = 0; self.R_0 = 0;

        self.lat = lat*(np.pi/180)
        self.long = long*(np.pi/180)
        self.name = name

        self.population = sum([self.S_0, self.E2_0,
        self.I_0, self.Ia_0, self.R_0])

    def distance(self, other):

        lat = self.lat; long = self.long
        R_earth = 64
        phi_i = lat; phi_j = other.lat;
        lambda_i = long; lambda_j = other.long
        arg = np.sin(phi_i)*np.sin(phi_j)+np.cos(phi_i)*np.cos(phi_j)*np.cos(abs(lambda_i-lambda_j))
        if not 0<=abs(arg)<=1:
            print(f"the argument: {arg} is not between 0 and 1")
        sigma_ij = np.arccos(arg)
        d = R_earth*sigma_ij
        return d

class ProblemSEIR():
    def __init__(self, region, beta, r_ia = 0.1, r_e2=1.25,lmbda_1=0.33, lmbda_2=0.5, p_a=0.4, mu=0.2):
        if isinstance(beta, (float, int)):
            self.beta = lambda t: beta
        elif callable(beta):
            self.beta = beta
        self.region = region
        self.r_ia = r_ia; self.r_e2 = r_e2;
        self.lmbda_1 = lmbda_1; self.lmbda_2 = lmbda_2;
        self.p_a = p_a; self.mu = mu
        self.set_initial_condition()

    def set_initial_condition(self):
        S_0 = self.region.S_0
        E2_0 = self.region.E2_0
        self.initial_condition = [S_0, 0,
                                  E2_0,
                                  0, 0, 0]

    def get_population(self):
        return self.region.population

    def solution(self, u, t):
        self.region.set_SEIR_values(u,t)

    def __call__(self, u, t):
         beta = self.beta(t); r_ia = self.r_ia; r_e2=self.r_e2;
         lmbda_1=self.lmbda_1; lmbda_2=self.lmbda_2;
         p_a=self.p_a; mu=self.mu;
         S, E1, E2, I, Ia, R = u
         N =sum(u)
         dS  = -beta*S*I/N - r_ia*beta*S*Ia/N - r_e2*beta*S*E2/N

         dE1 = beta*S*I/N + r_ia*beta*S*Ia/N + r_e2*beta*S*E2/N - lmbda_1*E1
         dE2 = lmbda_1*(1-p_a)*E1 - lmbda_2*E2
         dI  = lmbda_2*E2 - mu*I
         dIa = lmbda_1*p_a*E1 - mu*Ia
         dR  = mu*(I + Ia)
         return[dS, dE1, dE2, dI, dIa, dR]

class ProblemInteraction(ProblemSEIR):
    def __init__(self, region, area_name, beta, r_ia = 0.1, r_e2=1.25,lmbda_1=0.33, lmbda_2=0.5, p_a=0.4, mu=0.2):
        if isinstance(beta, (float, int)):
            self.beta = lambda t: beta
        elif callable(beta):
            self.beta = beta

        self.area_name = area_name
        self.region = region
        self.r_ia = r_ia; self.r_e2 = r_e2;
        self.lmbda_1 = lmbda_1; self.lmbda_2 = lmbda_2;
        self.p_a = p_a; self.mu = mu
        self.set_initial_condition()

    def get_population(self):
        region = self.region
        pop = []
        for k in range(len(region)):
            val = region[k].population
            pop.append(val)
        return sum(pop)

    def set_initial_condition(self):
        region = self.region
        M = len(region)
        self.initial_condition = []
        for i in range(M):
            self.initial_condition += [region[i].S_0,
                                      region[i].E1_0,
                                      region[i].E2_0,
                                      region[i].I_0,
                                      region[i].Ia_0,
                                      region[i].R_0]

    def __call__(self, u, t):

        beta = self.beta(t)
        #print(beta)
        r_ia = self.r_ia; r_e2=self.r_e2;
        lmbda_1=self.lmbda_1; lmbda_2=self.lmbda_2;
        p_a=self.p_a; mu=self.mu;

        n = len(self.region)

        SEIR_list = [u[i:i+6] for i in range(0, len(u), 6)]
        E2_list = [u[i] for i in range(2, len(u), 6)]
        Ia_list = [u[i] for i in range(4, len(u), 6)]
        derivative = []

        for i in range(n):
            S, E1, E2, I, Ia, R = SEIR_list[i]
            dS = 0;
            N_i = self.region[i].population

            for j in range(n):
                E2_other = E2_list[j]
                Ia_other = Ia_list[j]
                N_j = self.region[j].population
                dij = self.region[i].distance(self.region[j])

                dS  += (-beta*S*I)/N_i - (r_ia*beta*S)*(Ia_other/N_j)*np.exp(-dij) - r_e2*beta*S*(E2_other/N_j)*np.exp(-dij)

            dE1 = -dS - lmbda_1*E1
            dE2 = lmbda_1*(1-p_a)*E1 - lmbda_2*E2
            dI  = lmbda_2*E2 - mu*I
            dIa = lmbda_1*p_a*E1 - mu*Ia
            dR  = mu*(I + Ia)

            derivative += [dS, dE1, dE2, dI, dIa, dR]

        return derivative

    def solution(self, u, t):
            n = len(t)
            n_reg = len(self.region)
            self.t = t
            self.S = np.zeros(n)
            self.E1 = np.zeros(n)
            self.E2 = np.zeros(n)
            self.I = np.zeros(n)
            self.Ia = np.zeros(n)
            self.R = np.zeros(n)
            SEIR_list = [u[:, i:i+6] for i in range(0, n_reg*6, 6)]
            for part, SEIR in zip(self.region, SEIR_list):
                part.set_SEIR_values(SEIR, t)
                self.S += SEIR[:,0]
                self.E1 += SEIR[:,1]
                self.E2 += SEIR[:,2]
                self.I += SEIR[:,3]
                self.Ia += SEIR[:,4]
                self.R += SEIR[:,5]

    def plot(self):
        S = self.S; I = self.I;
        Ia = self.Ia; R = self.R;
        t = self.t

        plt.plot(t, S, label = "Susceptible")
        plt.plot(t, I, label = "Symtomatically infected ")
        plt.plot(t, Ia, label = "Asymptomatically infected")
        plt.plot(t, R, label = "Resistant")
        plt.title(self.area_name)
        plt.xlabel("Time(days")
        plt.ylabel("Population")

class SolverSEIR:
    def __init__(self, problem, T, dt):
        self.problem = problem
        self.T = T
        self.dt = dt
        self.total_populatiion = problem.get_population

    def solve(self, method = RungeKutta4):
        solver = method(self.problem)
        solver.set_initial_condition(self.problem.initial_condition)
        t = np.linspace(0,self.T, self.T*int(self.dt))
        u, t = solver.solve(t)
        self.problem.solution(u, t)

def fileread(filename):
    word = []
    lst = []

    RegionList = []
    lat = []
    long = []
    infile = open(filename, "r")
    for line in infile:
        line = line.strip('\n')
        words = line.split(";")
        for i in range(len(words)):
            word.append(words[i])
    infile.close()
    for k in word:
        linebreak = k.strip("\t")
        lst.append(linebreak)

    county   = [lst[i] for i in range(1, len(lst), 6)]
    S_0      = [float(lst[i]) for i in range(2, len(lst), 6)]
    E2_0     = [float(lst[i]) for i in range(3, len(lst), 6)]
    lat      = [float(lst[i]) for i in range(4, len(lst), 6)]
    long     = [float(lst[i]) for i in range(5, len(lst), 6)]

    for i in range(len(S_0)):
        RegionName = RegionInteraction(county[i], S_0[i], E2_0[i], lat[i], long[i])
        RegionList.append(RegionName)
    return RegionList

def covid19_Norway(beta, filename, num_days, dt):
    # read file and create list of RegionInteraction instances
    RegionList = fileread(filename)
    # create problem, an instance of ProblemInteraction
    problem = ProblemInteraction(RegionList, "Norway", beta)
    problem.get_population()
    problem.set_initial_condition()
    u = problem.initial_condition
    # create the solver, an instance of SolverSEIR
    solver = SolverSEIR(problem,num_days,dt)
    # call the method solve
    solver.solve()

    plt.figure(figsize=(9, 12))
    index = 1
    for i in problem.region:
        plt.subplot(4,3,index)
        # Call plot method from current part
        i.plot()
        index += 1
    plt.legend(fontsize = 6)
    plt.subplot(4,3, index)
    plt.subplots_adjust(hspace = 0.75, wspace=0.5)
    # Call plot method from problem
    problem.plot()
    plt.show()

def beta(t):
    r_ia = 0.1; r_e2=1.25; lmbda_1=0.33; lmbda_2=0.5; p_a=0.4; mu=0.2

    if t < 29:
        x =4.0/(r_e2/lmbda_2 + r_ia/mu + 1/mu)
    if 66 > t >= 29:
        x =  0.5/(r_e2/lmbda_2 + r_ia/mu + 1/mu)

    if 86 > t >= 66:
        x = 0.4/(r_e2/lmbda_2 + r_ia/mu + 1/mu)

    if 137 > t >= 86:
        x = 0.8/(r_e2/lmbda_2 + r_ia/mu + 1/mu)

    if 168 > t >= 137:
        x = 0.9/(r_e2/lmbda_2 + r_ia/mu + 1/mu)

    if 199 > t >= 168:
        x = 1.0/(r_e2/lmbda_2 + r_ia/mu + 1/mu)

    if 281 > t >= 199:
        x = 1.1/(r_e2/lmbda_2 + r_ia/mu + 1/mu)

    if t >= 281:
        x = 4.0/(r_e2/lmbda_2 + r_ia/mu + 1/mu)

    return x

#beta = 0.5
num_days =100; dt =1.0; filename = "fylker.txt"
covid19_Norway(beta, filename, num_days, dt)

"""
% python3 covid19.py
"""


from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report,confusion_matrix
import numpy as np
import pandas as pd
def preprocess_feature(feature,data):
    ohe =OneHotEncoder()
    categories = np.array(
        list(set(data[feature].astype(str).values))).reshape(-1, 1)
    print(categories)
    print(data[feature].astype(str).values.reshape(-1,1))
    ohe.fit(data[feature].astype(str).values.reshape(-1, 1))
    OneHotEncoder(categories=categories,
                  dtype= np.float64 , handle_unknown='error', sparse=True)
    print(ohe.transform(data[feature].values.reshape(-1,1)))
    return ohe.transform(data[feature].values.reshape(-1,1))
def one_hot_pandas(features,df):
    for feature in features:
        df = pd.concat([df, pd.get_dummies(
            df[feature], prefix=feature)], axis=1)
    return df
data=pd.read_csv('train_data2.csv')
target_column=['result','gender','age','verified']
data=one_hot_pandas(['gender', 'age', 'verified'],data)
print(data)
predictors = list(set(list(data.columns))-set(target_column))
data[predictors]=(data[predictors]-data[predictors].mean())/data[predictors].max()
# data = data.sample(frac=1)
# preprocess_feature
y=data['result']
X = data.drop(target_column,axis=1)
X_train, X_test, y_train, y_test = train_test_split(X, y)
mlp = MLPClassifier(hidden_layer_sizes=(20, 20, 20,20,20,20,20), max_iter=10000,verbose=True)
mlp.fit(X, y)
predictions = mlp.predict(X_test)
print(classification_report(y_test,predictions))


### Parallel Code

import dask.bag as db

b = db.from_sequence(filenames)
series = b.map(lambda fn: pd.read_hdf(fn)['close'])

corr = (series.product(series)
              .filter(lambda ab: not (ab[0] == ab[1]).all())
              .map(lambda ab: ab[0].corr(ab[1]))
              .max())

result = corr.compute()


from common.dataobj import DataObject
from common.ascensions.dataobj import Ascensions
from items.dataobj import Dish

class CharTalentStats(DataObject) :
	def __init__(self) :
		self.names: "list[str]" = None
		self.values: "list[list[str]]" = None

class CharTalent(DataObject) :
	def __init__(self) :
		self.name_hash: int = 0
		self.desc_hash: int = 0
		self.name: str = None
		self.desc: str = None
		self.icon: str = None
		self.charge_num: int = 0
		self.costs: "list[dict[str,int]]" = None
		self.stats: CharTalentStats = None

class CharTalents(DataObject) :
	def __init__(self) :
		self.normal_attack:    CharTalent = None
		self.elemental_skill:  CharTalent = None
		self.elemental_burst:  CharTalent = None
		self.alternate_sprint: CharTalent = None

class CharPassive(DataObject) :
	def __init__(self) :
		self.name_hash: int = 0
		self.desc_hash: int = 0
		self.name: str = None
		self.desc: str = None
		self.icon: str = None
		self.ascension: int = 0

class CharConstellation(DataObject) :
	def __init__(self) :
		self.name_hash: int = 0
		self.desc_hash: int = 0
		self.name: str = None
		self.desc: str = None
		self.icon: str = None

class CharSpecialDish(Dish) :
	def __init__(self):
		self.original_recipe: str = None
	def fromDish(dish: Dish) -> "CharSpecialDish" :
		res = CharSpecialDish()
		res.__dict__.update(dish.__dict__)
		return res

class CharSkin(DataObject) :
    def __init__(self) :
        # Identifiers
        self.hoyo_id: int = 0
        # General data
        self.name_hash: int = 0
        self.desc_hash: int = 0
        self.name: str = None
        self.desc: str = None
        self.icon: str = None
        self.restricted: bool = False

class CharSkins(DataObject) :
    def __init__(self) :
        self.default: CharSkin = None
        self.alt: "dict[str,CharSkin]" = {}

class Character(DataObject) :
	def __init__(self) :
		# Identifiers
		self.hoyo_id:        int = 0
		self.promote_id:     int = 0
		self.skill_depot_id: int = 0
		self.fetter_id:      int = 0
		# General data
		self.weapon: str = None
		self.name_hash: int = 0
		self.desc_hash: int = 0
		self.name: str = None
		self.desc: str = None
		self.icon: str = None
		self.rarity: int = 0
		self.special: bool = False
		self.body: str = None
		self.birthday: "list[int]" = None
		self.region: str = None
		self.vision_hash:     "list[int]" = None
		self.astrolabe_hash:  "list[int]" = None
		self.allegiance_hash:  int        = 0
		self.vision:     str = None
		self.astrolabe:  str = None
		self.allegiance: str = None
		self.default_weapon: str = None
		self.increase_skill: int = 0
		self.increase_burst: int = 0
		# Stats
		self.base_stats = {}
		self.curves = {}
		# Ascensions
		self.ascensions: Ascensions = None
		# Talents, passives, constellations
		self.talents:        CharTalents = CharTalents()
		self.passives:       "list[CharPassive]" = None
		self.constellations: "list[CharConstellation]" = None
		# Skins
		self.skins: CharSkins = CharSkins()
		# Special dish
		self.special_dish: CharSpecialDish = None


from galaxy_analysis.plot.plot_styles import *

import numpy as np
import matplotlib.pyplot as plt
import deepdish as dd
import h5py, glob, sys

from galaxy_analysis.utilities import utilities
from galaxy_analysis.analysis import Galaxy

from mpl_toolkits.axes_grid1 import make_axes_locatable
from scipy.stats import binned_statistic_2d

# temporary
import time as cpu_time

def load_abundance_data(data, data_list,
                        fields,
                        property_list,
                        phases = ['CNM','WNM','WIM','HIM'],
                        field_types = None):

    time_data = {}
    for i, field in enumerate(fields):
        time_data[field] = {}

        if field_types is None:
            if '_Fraction' in field or '_over_' in field:
                ft = 'mass_fraction'
         #   elif '_over_' in field:
         #       ft = 'abundance'
            else:
                print("Cannot properly determing field type for " + field)
                raise RunTimeError

        elif hasattr(field_types, 'keys') :
            ft = field_types[field]
        else:
            ft = field_types[i]

        for phase in phases:
            time_data[field][phase] = {}

            for property in property_list:

                if property == 'IQR' or property == 'inner_quartile_range':
                    for p in ['Q1','Q3']:
                        field_path = [phase,ft,field,p]
                        # need to load the data here into a time array
                        time_data[field][phase][p] = utilities.get_property(field_path,
                                                                               file_list = data.filename,
                                                                               data_list = data_list,
                                                                               self_contained = True)
                        time_data[field][phase][p] = np.array(time_data[field][phase][p], dtype = np.float)

                    if '_Fraction' in field: # do not do for abundance ratios
                        time_data[field][phase][property] = np.log10(time_data[field][phase]['Q3']) -\
                                                            np.log10(time_data[field][phase]['Q1'])
                    else:
                        time_data[field][phase][property] = time_data[field][phase]['Q3'] -\
                                                            time_data[field][phase]['Q1']

                elif property == 'mean_median_distance':
                    for p in ['median','mean']:
                        if not p in list(time_data[field][phase].keys()):
                            field_path = [phase,ft,field,p]
                            time_data[field][phase][p] = utilities.get_property(field_path,
                                                                                file_list = data.filename,
                                                                                data_list = data_list,
                                                                                self_contained = True)
                            time_data[field][phase][p] = np.array(time_data[field][phase][p], dtype = np.float)
                    if '_Fraction' in field: # do not do for abundance ratios
                        time_data[field][phase]['mean_median_distance'] =\
                                                   np.log10(time_data[field][phase]['mean']) -\
                                                   time_data[field][phase]['median']
                    else:
                        time_data[field][phase]['mean_median_distance'] = time_data[field][phase]['mean']-\
                                                                          time_data[field][phase]['median']

                elif property == 'inner_decile_range' or property == 'd9_d1_range':
                    for p in ['decile_1','decile_9']:
                        field_path = [phase,ft,field,p]
                        # need to load the data here into a time array
                        time_data[field][phase][p] = utilities.get_property(field_path,
                                                                               file_list = data.filename,
                                                                               data_list = data_list,
                                                                               self_contained = True)
                        time_data[field][phase][p] = np.array(time_data[field][phase][p], dtype = np.float)

                    if '_Fraction' in field: # do not do for abundance ratios
                        time_data[field][phase][property] = np.log10(time_data[field][phase]['decile_9']) -\
                                                            np.log10(time_data[field][phase]['decile_1'])
                    else:
                        time_data[field][phase][property] = time_data[field][phase]['decile_9']-\
                                                            time_data[field][phase]['decile_1']
                else:
                    field_path = [phase,ft,field,property]
                    time_data[field][phase][property] = utilities.get_property(field_path,
                                                                               file_list = data.filename,
                                                                               data_list = data_list,
                                                                               self_contained = True)
                    time_data[field][phase][property] = np.array(time_data[field][phase][property], dtype = np.float)

                    if '_Fraction' in field: # do not do for abundance ratios
                        if property in ['median','mean']:
                            time_data[field][phase][property] = np.log10(time_data[field][phase][property])

    return time_data

def plot_stellar_2d_hist(galaxy, field, time_bins = np.arange(0.0,20.0,0.2),
                         ybins = np.arange(-20,-6,0.1)):

    if '_Fraction' in field:
        yval = np.log10(galaxy.df[('io','particle_' + field.strip('_Fraction') + '_fraction')].value)
    else:
        yval = galaxy.df[('io','particle_' + field)].value

    creation_time = galaxy.df['creation_time'].to('Myr').value

    fig, ax = plt.subplots()
    fig.set_size_inches(8,8)
    statistic_data = np.ones(np.size(creation_time))
    N, x_edge, y_edge, binnum = binned_statistic_2d(creation_time - np.min(creation_time), yval, statistic_data,
                                                    statistic = 'count', bins =  (time_bins,ybins))

    fraction = N / (1.0 * np.size(creation_time))
    fraction[fraction <= 0] = -99
    fraction[fraction >  0] = np.log10(fraction[fraction > 0])
    fraction = np.log10(N / (1.0 * np.size(creation_time)))
    plot_val = fraction

    xmesh, ymesh = np.meshgrid(x_edge, y_edge)
    img1 = ax.pcolormesh(xmesh, ymesh, plot_val.T,
                              cmap = 'magma', vmin = -4, vmax = -1)

    divider = make_axes_locatable(ax)
    cax1 = divider.append_axes('right', size = '5%', pad = 0.05)
    fig.colorbar(img1, cax=cax1, label = "Fraction")
    ax.set_xlabel(r'Time (Myr)')

    plt.minorticks_on()
    plt.tight_layout()
    fig.savefig('stellar_O_2d_hist.png')

    return

def plot_stellar_separation(time, data, galaxy,
                            field    = 'O_Fraction',
                            property = 'median',
                            phases   = ['CNM'],
                            figdim=None,
                            field_types = None,
                            labels = None, line_styles = {},
                            xlim = None, ylim = None,
                            annotate_text = None, ylabels = None):
    if labels is None:
        labels = {}

    for k in phases + [field] + [property]:
        if not (k in list(labels.keys())):
            labels[k] = k

    if figdim is None:
        if len(phases) == 1:
            nrow, ncol = 1, 1
        else:
            ncol = 3
            nrow = 2
    else:
        nrow, ncol = figdim

    data_list = np.array(np.sort([x for x in list(data.keys()) if 'DD' in x]))
    data_list = data_list[:len(time)]

    time_data = load_abundance_data(data, data_list, [field], [property],
                                    phases = phases, field_types = field_types)

    # now need to load the data output to get the stellar values

    #
    if nrow*ncol > 1:
        fig, all_axes = plt.subplots(nrow,ncol,sharex=True,sharey=True)
        fig.set_size_inches(ncol*5, nrow*5)
        fig.subplots_adjust(hspace=0.0,wspace=0.0)
    else:
        fig, ax = plt.subplots()
        fig.set_size_inches(6,6)

    axi = axj = 0

    creation_time = galaxy.df['creation_time'].convert_to_units('Myr').value

    for i, phase in enumerate(phases):
        if nrow*ncol > 1:
            ax = all_axes[(axi,axj)]

        property_value = time_data[field][phase][property]

        if '_Fraction' in field:
            star_val = np.log10(galaxy.df['particle_' + field.strip('_Fraction') + '_fraction'])
        else:
            star_val = galaxy.df['particle_' + field]

        select = creation_time > 0.0 #do all for now (np.min(creation_time) + 100.0)

        distance = star_val[select] - np.interp(creation_time[select],time,property_value)

        ax.scatter(creation_time[select] - np.min(creation_time),
                   distance, alpha = 0.75, color = 'black', s = 20)
        ax.set_xlim(0.0, 500.0)
        ax.set_ylim(-2.5,2.5)
        ax.plot( ax.get_xlim(), [0.0,0.0], color = 'black', lw = line_width, ls = '--')
        plt.minorticks_on()

        xy = (200.0, ax.get_ylim()[1] - 0.35)
        #q3dist = q3 - median_distance
        #q1dist = median_distance - q1
#        ax.annotate(labels[phase] + " : %.2f + %.2f - %.2f"%(median_distance,q3dist,q1dist), xy=xy,xytext=xy)
        #d1, d9 = np.percentile(distance, [0.1,0.9])
        select = creation_time > (np.min(creation_time) + 120.0)

        ax.annotate("Median Dist. = %.2f"%(np.median(np.abs(distance[select]))), xy=xy,xytext=xy)
        xy = (200.0, ax.get_ylim()[1] - 0.7)
        q1, q3 = np.percentile(np.abs(distance[select]), [25,75])
        print(q3, q1, q3-q1, np.size(distance[select][distance[select]<0])/(1.0*np.size(distance[select])), np.size(distance[select][distance[select]>0]))
        ax.annotate("IQR = %.2f"%(q3-q1), xy=xy,xytext=xy)

        axj = axj + 1
        if axj >= ncol:
            axj = 0
            axi = axi + 1

    #plt.tight_layout()
    if nrow*ncol>1:
        for i in np.arange(ncol):
            all_axes[(nrow-1,i)].set_xlabel(r'Time (Myr)')
        for i in np.arange(nrow):
            all_axes[(i,0)].set_ylabel(r'Distance From ' + labels[property] + '[dex]')
    else:

        ax.set_xlabel(r'Time (Myr)')
        if ylabels is None:
            ax.set_ylabel(r'Distance to ' + labels[phase] + ' ' + labels[property] + ' [dex]')
        else:
            ax.set_ylabel(ylabels[0])
    if nrow*ncol == 1:
        plt.tight_layout()
    fig.savefig("stellar_distance_to_median.png")

    return

def plot_abundace_resolution_study():

    fields = ['O_Fraction','Ba_Fraction']
    property_list = ['d9_d1_range']
    phases = ['CNM','WNM','WIM','HIM']

    simulations = {'3pcH2' : './3pc_H2/abundances/gas_abundances.h5',
                   '6pcH2' : './6pc_H2/abundances/gas_abundances.h5'}

    fig, ax = plt.subplots(2,2,sharex=True,sharey=True)
    fig.set_size_inches(12,12)
    fig.subplots_adjust(hspace=0.0,wspace=0.0)

    all_time_data = {}
    for sim in simulations:
        data = h5py.File(simulations[sim])

        data_list = np.sort([x for x in list(data.keys()) if 'DD' in x])
        times     = np.array([float(x.strip('DD')) for x in data_list])

        all_time_data[sim] = load_abundance_data(data, data_list,
                                                 fields, property_list)
        all_time_data[sim]['time'] = times - times[0]

    ls = {'CNM':'-','WNM':'-','WIM':'-','HIM':':'}

    for phase in phases:

        ax[(0,0)].plot(all_time_data['3pcH2']['time'],
                       all_time_data['3pcH2']['O_Fraction'][phase]['d9_d1_range'],
                       color = color_dict[phase], ls = ls[phase], lw = line_width)
        ax[(0,1)].plot(all_time_data['6pcH2']['time'],
                       all_time_data['6pcH2']['O_Fraction'][phase]['d9_d1_range'],
                       color = color_dict[phase], ls = ls[phase], lw = line_width)
        ax[(1,0)].plot(all_time_data['3pcH2']['time'],
                       all_time_data['3pcH2']['Ba_Fraction'][phase]['d9_d1_range'],
                       color = color_dict[phase], ls = ls[phase], lw = line_width)
        ax[(1,1)].plot(all_time_data['6pcH2']['time'],
                       all_time_data['6pcH2']['Ba_Fraction'][phase]['d9_d1_range'],
                       color = color_dict[phase], ls = ls[phase], lw = line_width)

    for a1 in ax:
        for a2 in a1:
            a2.set_xlim(0, 500)
            a2.set_ylim(0, 3)
            a2.minorticks_on()

    for i in [0,1]:
        ax[(i,0)].set_ylabel(r'Inner Decile Range [dex]')
        ax[(1,i)].set_xlabel(r'Time (Myr)')

    x = 300
    y = 2.7
    size = 20
    ax[(0,0)].text(x, y, r'O  - 3.6 pc', color = 'black', size = size)
    ax[(1,0)].text(x, y, r'Ba - 3.6 pc', color = 'black', size = size)
    ax[(0,1)].text(x, y, r'O  - 7.2 pc', color = 'black', size = size)
    ax[(1,1)].text(x, y, r'Ba - 7.2 pc', color = 'black', size = size)

    plt.minorticks_on()
    fig.savefig('O_Ba_resolution_study.png')

    return

def plot_abundance_evolution(time, data,
                             fields = ['O_Fraction','Ba_Fraction'],
                             property_list = ['median','IQR'],
                             phases = ['CNM','WNM','WIM','HIM'],
                             figdim=None,
                             field_types = None,
                             labels = None, line_styles = None,
                             xlim = None, ylim = None,
                             annotate_text = None, fsize=5):

    if figdim is None:
        ncol = len(property_list)
        nrow = len(fields)
    else:
        nrow, ncol = figdim

    data_list = np.array(np.sort([x for x in list(data.keys()) if 'DD' in x]))
    data_list = data_list[:len(time)]

    start = cpu_time.time()

    time_data = load_abundance_data(data, data_list, fields, property_list,
                                    phases = phases, field_types = field_types)

    print("DATA LOADING TOOK %2.2E"%(cpu_time.time()- start))

    fig, ax = plt.subplots(nrow,ncol)
    fig.set_size_inches(fsize*ncol, fsize*nrow)

    axi = 0
    axj = 0
    xval = np.array(time)
    xval = xval - xval[0]
    for field in fields:

        for property in property_list:
            if len(fields) == 1:
                axindex = axj
            else:
                axindex = (axi,axj)

            for phase in phases:
                yval = time_data[field][phase][property] # - time_data[field]['CNM'][property]

                if phase in list(line_styles.keys()):
                    ls = line_styles[phase]
                else:
                    ls = '-'

                ax[axindex].plot(xval, yval, lw = line_width,
                                 ls = ls, color = color_dict[phase],
                                 label = phase)

            ax[axindex].set_xlabel(r'Time (Myr)')

            if hasattr(labels[property],"keys"):
                ax[axindex].set_ylabel(labels[property][field])
            else:
                ax[axindex].set_ylabel(labels[property])

            if xlim is None:
                ax[axindex].set_xlim(xval[0],xval[-1])
            else:
                ax[axindex].set_xlim(xlim)

            if not (ylim is None):
                if np.size(fields) == 1:
                    ymin,ymax = ylim[axindex]
                else:
                    ymin,ymax = ylim[axindex[0]][axindex[1]]
                ax[axindex].set_ylim(ymin,ymax)

            axj = axj + 1
            if axj >= ncol:
                axj = 0
                axi = axi + 1

    if len(fields) == 1:
        indexzero=0
    else:
        indexzero=(0,0)

    ax[indexzero].legend(loc='lower right', ncol=2)

    if not annotate_text is None:
        for axi in len(fields):
            if len(fields) == 1:
                axindex0 = axi
                axindex1 = axi
            else:
                axindex0 = (axi,1)
                axindex1 = (axi,0)

            xy = annotate_text[axindex0]
            ax[axindex1].annotate( annotate_text[axindex0], xy,xy)

    plt.minorticks_on()
    #plt.tight_layout()
    fig.savefig( '_'.join(fields + property_list) + '_evolution.png')

    return fig, ax

if __name__ == "__main__":

#    plot_abundace_resolution_study()

    if len(sys.argv) > 1:
        filename = sys.argv[1]
    else:
        filename = './gas_abundances.h5'

    data = h5py.File(filename)

    #time = np.arange(0.0, 10.0*(len( [x for x in data.keys() if 'DD' in x])-1) + 0.1, 10.0)

    time = [float(x.strip('DD')) for x in np.sort(list(data.keys())) if 'DD' in x]

    galaxy = Galaxy('DD0619')

    plot_stellar_2d_hist(galaxy, 'O_Fraction')

    plot_stellar_separation(time, data, galaxy,
                             field    = 'O_Fraction',
                           property = 'median',
                            phases   = ['CNM'],
                            labels = {'median' : 'Median'},
                            ylabels = [r"[O/H] - [O/H]$_{\rm CNM}$"])

    plot_abundance_evolution(time, data,
                             fields = ['O_Fraction','Ba_Fraction'],
                             phases = ['CNM','WNM','WIM','HIM'],
                             property_list = ['median','mean_median_distance','IQR','d9_d1_range'],
                             labels = {'median' : r'log(Median)',
                                       'IQR'    : r'Inner Quartile Range [dex]',
                                       'mean_median_distance' : r'log(Mean) - log(Median) [dex]',
                                       'd9_d1_range' : r'Inner Decile Range [dex]'},
                             line_styles = {'HIM' : ':'},
                              ylim = [ [(-8,-2),(0.,1.5),(0.0,1.5),(0.0,3.0)], [(-16,-10),(0.,1.5),(0.0,1.5),(0.0,3.0)]],
                            # ylim = [ [(-1,8),(-3,3),(-3,3.0)], [(-1,8),(-3,3),(-3,3.0)]],
                             annotate_text = [ ('Oxygen', (20.0,-2.5)),   ('Barium',(20,-10.5))],
                             xlim = (0.0, 500.0))

    plot_abundance_evolution(time, data,
                             fields = ['O_Fraction','N_Fraction'],
                             phases = ['CNM','WNM','WIM','HIM'],
                             property_list = ['median','IQR','d9_d1_range'],
                             labels = {'median' : r'log(Median)',
                                       'IQR'    : r'Inner Quartile Range [dex]',
                                       'd9_d1_range' : r'Inner Decile Range [dex]'},
                             line_styles = {'HIM' : ':'},
                              ylim = [ [(-8,-2),(0.0,1.5),(0.0,3.0)], [(-9,-3),(0.0,1.5),(0.0,3.0)]],
                           # ylim = [ [(-1,8),(-3,3),(-3,3.0)], [(-1,8),(-3,3),(-3,3.0)]],
                             annotate_text = [ ('Oxygen', (20.0,-2.5)),   ('Nitrogen',(-8.5,-10.5))])


# This script receives a bpjson file and tokenizes the
# span strings and segment texts.

import argparse
import json
import os

import tokenizations
from sacremoses import MosesTokenizer

mt = MosesTokenizer(lang="en")

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--bpjson", type=str, required=True, help="input .bp.json file."
    )
    parser.add_argument(
        "--output", type=str, required=True, help="output .bp.json file."
    )

    args = parser.parse_args()

    return args

def _first_non_empty(tokens):
    for tok in tokens:
        if len(tok) > 0:
            return tok[0]

def find_index(text_tokens, span_tokens, char2tok, char_offset_start, char_offset_end):
    overlapping_tokens = char2tok[char_offset_start:char_offset_end]
    # Ignore mappings to non-tokens (e.g., spaces), which have value `[]`
    # Get the first and last "non empty" alignments
    predicted_token_offset_start = _first_non_empty(overlapping_tokens)
    predicted_token_offset_end = _first_non_empty(overlapping_tokens[::-1])

    match = (predicted_token_offset_start, predicted_token_offset_end)

    assert "".join(span_tokens) in "".join(
        text_tokens[match[0] : match[1] + 1]
    ), "Spans did not match"

    if match[1] - match[0] + 1 != len(span_tokens):
        # Slight offset issue due to character offsets, keep start of span and modify end of span
        match = (match[0], match[0] + len(span_tokens) - 1)

    assert match[1] - match[0] + 1 == len(
        span_tokens
    ), "lens did not match"  # length should match

    return match

def _findall(string, span):
    idx = string.find(span)
    while idx != -1:
        yield idx
        idx = string.find(span, idx + 1)
    return

def align(sentence, span):
    """align. Searches for span in sentence, to create character offsets
    Uses heuristics if the span happens more than once in a sentence.
    Prefers the earliest span that also ends on a word boundary, which
    is defined as a non-alpha numeric character (e.g. a space character)

    returns a tuple (start, end)

    :param sentence: A string sentence
    :param span: The desired text span
    """
    start_idxs = list(_findall(sentence, span))
    idxs = [(i, i + len(span)) for i in start_idxs]

    if len(idxs) == 1:
        return idxs[0]
    elif len(idxs) > 1:
        # heuristic!
        # get the spans that also end on word boundaries
        good_spans = []
        for start, end in idxs:
            end_ok = (end >= len(sentence)) or (not sentence[end].isalpha())
            start_ok = (start - 1 < 0) or (not sentence[start - 1].isalpha())
            if start_ok and end_ok:
                good_spans.append((start, end))

        return good_spans[0]
    else:
        raise Exception("No matches found for this span")

def tokenize_str(text):
    text = f"DUMMYWORD {text} DUMMYWORD"
    text_tok_dir = mt.tokenize(text, escape=False)
    text_tok = []
    for c in text_tok_dir:
        if c not in ["\u202b", "\u202c"]:
            text_tok.append(c)
    return text_tok[1:-1]

def tokenize_data(DATA_FILE, args):

    if not os.path.isfile(DATA_FILE):
        raise FileNotFoundError(f"Could not find {DATA_FILE}")

    with open(DATA_FILE, "r") as f:
        data = json.load(f)

    entries = data["entries"]
    dirty_docs = []
    for entry_id, entry in entries.items():

        entry["segment-text-tok"] = tokenize_str(entry["segment-text"])

        if "annotation-sets" not in entry:
            entry["annotation-sets"] = {
                "abstract-events": {"events": {}, "span-sets": {}}
            }

        annotation_sets = entry["annotation-sets"]

        events = annotation_sets["abstract-events"]

        tok2char, char2tok = tokenizations.get_alignments(
            entry["segment-text-tok"], [char for char in entry["segment-text"]]
        )

        for span_set_id, span_set in events["span-sets"].items():
            spans = span_set["spans"]
            for span in spans:
                span["string-tok"] = tokenize_str(span["string"])
                if "hstring" in span:
                    span["hstring-tok"] = tokenize_str(span["hstring"])

                try:
                    # if this is abstract, we need to extract offsets
                    char_start, char_end = align(entry["segment-text"], span["string"])
                    span["start"] = char_start
                    span["end"] = char_end

                    # now get the token offsets
                    span_indices = find_index(
                        entry["segment-text-tok"],
                        span["string-tok"],
                        char2tok,
                        span["start"],
                        span["end"],
                    )

                    start, end = span_indices
                    span["start-token"] = start
                    span["end-token"] = end
                except Exception:
                    dirty_docs.append(entry["doc-id"])

    dropped = 0
    # original_len = len(data["entries"])
    for doc_id in dirty_docs:
        for entry_id, entry in list(data["entries"].items()):
            if entry["doc-id"] == doc_id:
                dropped += 1
                del data["entries"][entry_id]
    return data

def main():
    args = parse_args()

    # tokenize segments and span strings in input json file
    tok_data = tokenize_data(args.bpjson, args)

    # write tokenization results into a new bpjson file
    with open(args.output, "w", encoding="utf8") as outfile:
        json.dump(tok_data, outfile, indent=2, ensure_ascii=False)

if __name__ == "__main__":
    main()


from flask import Flask, request, render_template, redirect, flash, session
from flask_debugtoolbar import DebugToolbarExtension
from surveys import satisfaction_survey as survey

app = Flask(__name__)
app.config['SECRET_KEY'] = "superSecret"

@app.route('/')
def home_page():
    instructions = survey.instructions
    title = survey.title
    return render_template('home.html', instructions=instructions, title=title)

@app.route('/clear', methods=["POST"])
def clear_responses():
    session['responses'] = []
    return redirect('/questions/0')

@app.route('/answer', methods=["POST"])
def add_answer():
    answer = request.form['answer']
    responses = session['responses']
    responses.append(answer)
    session['responses'] = responses

    return redirect(f'/questions/{len(responses)}')

@app.route('/questions/<int:question_num>')
def show_question(question_num):

    responses = session['responses']

    if question_num != len(responses):
        flash(f"Invalid question id: {question_num}.")
        return redirect(f'/questions/{len(responses)}')

    if len(survey.questions) == len(responses):
        return redirect('/thanks')

    question = survey.questions[question_num].question
    choices = survey.questions[question_num].choices
    return render_template(
        'questions.html', question_num=question_num, question=question, choices=choices)

@app.route('/thanks')
def thanks():
    responses = session['responses']
    return render_template('thanks.html', responses=responses)


import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore
import importlib
data = importlib.import_module('data')

products = data.getProducts()
types_id = data.getTypesID()
types_index = data.getTypesIndex()
series_id = data.getSeriesID()
series_name = data.getSeriesName()
image = data.getImage()
thumbnail = data.getThumbnail()
pdf = data.getPDF()
desc = data.getDesc()
customType = data.getCustomType()

def main():

	data = []

	types = []
	series = []

	d = {}

	# Untuk Types
	for p in products.keys():
		d['_id'] = types_id[p]
		d['name'] = p
		d['index'] = types_index[p]
		d['description'] = ''
		d['isCustom'] = types_id[p] in customType
		types += [d]
		d = {}

		# print(d)

	# Untuk Series
	# d = {}
	# for p in products.keys():
	# 	for s in products[p]:
	# 		d['_id'] = series_id[s]
	# 		d['code'] = s
	# 		d['name'] = series_name[s]
	# 		d['type'] = types_id[p]
	# 		d['descriptions'] = {
	# 		'description': desc[s]
	# 		}
	# 		d['image'] = image[s]
	# 		d['thumbnail'] = thumbnail[s]
	# 		d['pdf'] = pdf[s]
	# 		d['param'] = series_id[s][0]
	# 		series += [d]
	# 		d = {}
			# print(d)

	cred = credentials.Certificate("./serviceAccountKey.json")
	firebase_admin.initialize_app(cred)
	db = firestore.client()

	for t in types:
		ref = db.collection(u'types').document(t['_id']).set(t, merge=True)
		print('[+] Types {}'.format(t['_id']))

	# for s in series:
	# 	ref = db.collection(u'series').document(s['_id']).set(s, merge=True)
	# 	print('[+] Series {}'.format(s['_id']))

	print('[+] Data stored.')

if __name__ == '__main__':
	main()


from sklearn.base import BaseEstimator,TransformerMixin;
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import RobustScaler,OneHotEncoder
from sklearn.impute import SimpleImputer
import pandas as pd

'''
En Scikit-learn, un Pipeline y un ColumnTransformer
son dos clases que se utilizan para preprocesar los datos
antes de ajustar un modelo de aprendizaje automático.

La principal diferencia entre un Pipeline y un
ColumnTransformer es que un Pipeline se utiliza para
encadenar varios pasos de preprocesamiento en una única
tubería, mientras que un ColumnTransformer se utiliza
para aplicar diferentes transformaciones a diferentes
columnas en un solo paso.

Prepocesamiento de la data(
    Eliminacion de valores nulos(SimpleImputer),
    Regularacion (Normalizacion o estandarizacion)
    Convertir datos categoricos a numericos(One hot enconder)
)
'''

class DataFramePreparer(BaseEstimator,TransformerMixin):

    def __init__(self):
        self._full_pipeline = None;
        self._columns = None;
        self._numPipeline = Pipeline([
            #Valores nulos
            ("imputer",SimpleImputer(strategy="median")),
            #Escalar valores (estandarizacion con rango intercualitoco para eliminar outliners)
            ("rbst_scaler",RobustScaler())
        ]);

    def transformData(self,X_train:pd.DataFrame,Y=None):
        num_attribs = list(
            X_train.select_dtypes(exclude = ["object"])
        );
        cat_attribs = list(
            X_train.select_dtypes(include = ["object"])
        );
        self._full_pipeline = ColumnTransformer([
            ("num",self._numPipeline,num_attribs),
            ("cat",OneHotEncoder(),cat_attribs)
        ]);
        X_train_prep = self._full_pipeline.fit_transform(X_train);
        self._columns = pd.get_dummies(X_train);
        X_train_prep = pd.DataFrame(
            X_train_prep,
            columns=self._columns,
            # index=X_train.index
        );

        return X_train_prep;


def fizzBuzz(first , second , UpperLimit):

    for num in range(1,UpperLimit):
        result = ""
        if num % first == 0 : result += "fizz"
        if num % second == 0 : result += "Buzz"
        print( result or num)

fizzBuzz(3,5,100)

#advanced Fizz lets you pass a dictonary with custom words as keys with respective numbers so you can check more than 2 numbers at a time
#Advanced Fizz also lets you set a custom upper and lower limit number range , ex 50 and 250 instead of the base 1 - 100
def advancedFizzBuzz(dic , lowerLimit , upperLimit):

    for num in range(lowerLimit,upperLimit+1):
        advResult = ""
        for key in dic:
            if num % dic[key] == 0 : advResult += key
        print(advResult or num)

#checks 3,5,and 8 in a range or 50 to 250
advancedFizzBuzz({'Fizz ':3 , 'Buzz ':5 , 'DOC Oct ':8} , 50 , 250)


sizeofset = int(input())
myset = set(map(int,input().split()))
queries = int(input())
i = 0
while i < queries:
    string=input()
    if "remove" in string:
        idx = string.index(" ")
        idx += 1
        num = ""
        while idx < len(string):
            num = num + string[idx]
            idx += 1
        num = int(num)
        myset.remove(num)
    elif "discard" in string:
        idx = string.index(" ")
        idx += 1
        num = ""
        while idx < len(string):
            num = num + string[idx]
            idx += 1
        num = int(num)
        myset.discard(num)
    else:
        myset.pop()
    i+=1
myset=list(myset)
print(sum(myset))


import os
from datetime import date, timedelta

from tests_e2e.__specs.utils.hexagon_spec import as_a_user
from tests_e2e.__specs.utils.path import e2e_test_folder_path

LAST_UPDATE_DATE_FORMAT = "%Y%m%d"

test_folder_path = e2e_test_folder_path(__file__)
storage_path = os.path.realpath(os.path.join(test_folder_path, "storage"))
last_checked_storage_path = os.path.join(
    storage_path, "hexagon", "last-update-check.txt"
)

def _write_storage_path():
    if not os.path.exists(os.path.dirname(last_checked_storage_path)):
        os.makedirs(os.path.dirname(last_checked_storage_path))

def _write_last_check(last_check_date):
    _write_storage_path()

    with open(last_checked_storage_path, "w") as file:
        file.write(last_check_date.strftime(LAST_UPDATE_DATE_FORMAT))

def _clear_last_check():
    _write_storage_path()
    if os.path.exists(last_checked_storage_path):
        os.remove(last_checked_storage_path)

base_os_env_vars = {
    "HEXAGON_UPDATE_DISABLED": "false",
    "HEXAGON_TEST_VERSION_OVERRIDE": "0.1.0",
    "HEXAGON_STORAGE_PATH": storage_path,
}

no_changelog_env_vars = {**base_os_env_vars, "HEXAGON_UPDATE_SHOW_CHANGELOG": ""}

def test_new_hexagon_version_available():
    _clear_last_check()

    (
        as_a_user(__file__)
        .run_hexagon(["my-module"], os_env_vars=no_changelog_env_vars)
        .write("n")
        .then_output_should_be(
            [["Would you like to update?", "No"]], discard_until_first_match=True
        )
        .then_output_should_be(["my-module"])
        .exit()
    )

def test_prompt_to_update_hexagon_only_once():
    _clear_last_check()

    (
        as_a_user(__file__)
        .run_hexagon(["my-module"], os_env_vars=no_changelog_env_vars)
        .write("n")
        .then_output_should_be(
            [["Would you like to update?", "No"]], discard_until_first_match=True
        )
        .then_output_should_be(["my-module"])
        .exit()
    )

    (
        as_a_user(__file__)
        .run_hexagon(["my-module"], os_env_vars=no_changelog_env_vars)
        .then_output_should_be(["my-module"])
        .exit()
    )

def test_prompt_to_update_hexagon_once_a_day():
    _write_last_check(date.today())

    (
        as_a_user(__file__)
        .run_hexagon(["my-module"], os_env_vars=no_changelog_env_vars)
        .then_output_should_be(["my-module"])
        .exit()
    )

def test_prompt_to_update_hexagon_again_next_day():
    _write_last_check(date.today() - timedelta(days=1))

    (
        as_a_user(__file__)
        .run_hexagon(["my-module"], os_env_vars=no_changelog_env_vars)
        .write("n")
        .then_output_should_be(
            [["Would you like to update?", "No"]], discard_until_first_match=True
        )
        .then_output_should_be(["my-module"])
        .exit()
    )

def test_show_changelog():
    _clear_last_check()

    (
        as_a_user(__file__)
        .run_hexagon(
            ["my-module"],
            os_env_vars={
                **base_os_env_vars,
                "HEXAGON_CHANGELOG_FILE_PATH_TEST_OVERRIDE": os.path.join(
                    test_folder_path, "CHANGELOG.md"
                ),
                "HEXAGON_THEME": "default",
            },
        )
        .then_output_should_be(["New hexagon version available"], True)
        .then_output_should_be(
            [
                "- Feature 1",
                "- Feature 2",
                "- Feature 3",
                "- Feature 4",
                "- Fix 1",
                "- Fix 2",
                "- Fix 3",
                "- Fix 4",
                "- Documentation 1",
                "- Documentation 2",
                "and much more!",
            ],
            True,
        )
        .write("n")
        .then_output_should_be(
            [["Would you like to update?", "No"]],
            discard_until_first_match=True,
        )
        .then_output_should_be(["my-module"])
        .exit()
    )


from intpy.intpy import deterministic, initialize_intpy
import sys
from time import perf_counter

dic = {1:2, 2:1, 3:0}
@deterministic
def rec_1(n, begin, end):
    aux = dic[begin+end]
    if n != 1:
        return rec_1(n-1, begin, aux) + [(begin, end)] + rec_1(n-1, aux, end)
    else:
        return [(begin, end)]

@initialize_intpy(__file__)
def main(n):
    moves = rec_1(n, 0, 2)

if __name__ == '__main__':
    n = int(sys.argv[1])

    start = perf_counter()
    main(n)
    t = perf_counter() - start

    print(t)


'''
Created on 13 Feb 2018

@author: Ghadah

By computing distance to the Kth nearest neighbour, we can find the range of
possible values, for DBSCAN epsilon parameter. This class uses values produced
by PCA_Freq and PCA_BiGram, to plot the graphs.

Note: we used this class with both frequency vectors and Bi-grams
'''
from sklearn.neighbors import NearestNeighbors
from matplotlib import pyplot as plt
import PCA_BiGram
import PCA_Freq

# get PCA data
pca_bi = PCA_BiGram.returnPCA()
pca_freq = PCA_Freq.returnPCA()

# number of neighbours
n = 8

# compute Nearest Neighbour (Bi-Grams)
nbrs = NearestNeighbors(n_neighbors=n, algorithm='ball_tree').fit(pca_bi)
distances, indices = nbrs.kneighbors(pca_bi)

# compute Nearest Neighbour (Frequency)
nbrs2 = NearestNeighbors(n_neighbors=n, algorithm='ball_tree').fit(pca_freq)
freq_distances, freq_indices = nbrs2.kneighbors(pca_freq)

# sort distances
desc_distances = sorted(distances[:, n-1], reverse=True)
freq_desc_distances = sorted(freq_distances[:, n-1], reverse=True)

# plot data
with plt.style.context('seaborn-whitegrid'):
    plt.subplot(211)
    plt.plot(list(range(0, len(desc_distances))), desc_distances, label='KNN')
plt.title("K-distance graph (Bi-Grams)")
plt.xlabel("Points in Descending Order (Eps value)")
plt.ylabel(" KNN distance")
plt.legend()
plt.tight_layout()

# second plot
with plt.style.context('seaborn-whitegrid'):
    plt.subplot(212)
    plt.plot(list(range(0, len(freq_desc_distances))), freq_desc_distances,
             label='KNN', c='r')
plt.title("K-distance graph (Frequency)")
plt.xlabel("Points in Descending Order (Eps value)")
plt.ylabel(" KNN distance")
plt.legend()
plt.tight_layout()

plt.show()


"""Unit test module for recipie_reader.py"""
import unittest
import os
import sys
import json
import mock

SRC_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SRC_DIR + "/..")
import recipie_reader  # pylint: disable=import-error
import mealmaker_constants as MMK  # pylint: disable=import-error

T_JSON_RECIPIE_FILENAME = os.path.join(SRC_DIR, '../recipies/salt_water.json')
T_JSON_RECIPIE_STR = open(T_JSON_RECIPIE_FILENAME).read()
T_JSON_RECIPIE = json.loads(T_JSON_RECIPIE_STR)

class TestRecipieReader(unittest.TestCase):
    """Unit Test Class"""

    def setUp(self):
        """setup before each test"""
        self.reader = recipie_reader.RecipieReader(T_JSON_RECIPIE_FILENAME)

    def test_init_success(self):
        """Test the __init__ function for success"""
        # Validate necessary keys in recipie
        for key in MMK.KEY_INGREDIENTS, MMK.KEY_STEPS, MMK.KEY_PAN:
            self.assertIn(key, self.reader.recipie)
        # Validate ingredients
        for key in MMK.ING_SALT, MMK.ING_WATER:
            self.assertIn(key, self.reader.recipie[MMK.KEY_INGREDIENTS])
        # Verify that at least one step exists
        self.assertIn("1", self.reader.recipie[MMK.KEY_STEPS])

    def test_init_no_recipie_file(self):
        """Test the __init__ function when no recipie_file found"""
        with self.assertRaises(IOError) as io_exc:
            recipie_reader.RecipieReader("")
        self.assertTrue('File not found' in str(io_exc.exception))

    @mock.patch('json.load')
    @mock.patch('__builtin__.open')
    def test_init_ker_error(self, mock_open, mock_json_load):
        """Test the __init__ function when recipie_file is missing keys"""
        mock_open.return_value = mock.MagicMock()
        mock_json_load.return_value = json.loads("""{}""")
        with self.assertRaises(KeyError) as key_err:
            recipie_reader.RecipieReader("")
        self.assertTrue("ingredients' not found in recipie" in str(key_err.exception))

    def test_get_ingredients(self):
        """Test get_ingredients"""
        for key in MMK.ING_SALT, MMK.ING_WATER:
            self.assertIn(key, self.reader.get_ingredients())

    def test_get_steps(self):
        """Test get_steps"""
        self.assertIn("1", self.reader.get_steps())

    def test_get_pan(self):
        """Test get_pan"""
        self.assertEquals(0, self.reader.get_pans())

if __name__ == '__main__':
    SUITE = unittest.TestLoader().loadTestsFromTestCase(TestRecipieReader)
    unittest.TextTestRunner(verbosity=2).run(SUITE)
    unittest.main()


'''Given a string S that only contains "I" (increase) or "D" (decrease), let N = S.length.

Return any permutation A of [0, 1, ..., N] such that for all i = 0, ..., N-1:

If S[i] == "I", then A[i] < A[i+1]
If S[i] == "D", then A[i] > A[i+1]'''

class Solution(object):
    def diStringMatch(self, S):
        """
        :type S: str
        :rtype: List[int]
        """
        result_list = list()
        small = 0
        large = len(S)
        for char in S:
            if char == 'I':
                result_list.append(small)
                small += 1
            else:
                result_list.append(large)
                large -= 1
        result_list.append(large)
        return result_list