# -*- coding: utf-8 -*-
import os
import sys
import time
import numpy as np
import pickle as pkl
import multiprocessing
from itertools import product
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score

try:
    sys.path.append(os.getcwd())
    import sparse_module

    try:
        from sparse_module import c_algo_solam
        from sparse_module import c_algo_spam
        from sparse_module import c_algo_sht_auc
        from sparse_module import c_algo_opauc
        from sparse_module import c_algo_sto_iht
        from sparse_module import c_algo_hsg_ht
        from sparse_module import c_algo_fsauc
    except ImportError:
        print('cannot find some function(s) in sparse_module')
        pass
except ImportError:
    print('cannot find the module: sparse_module')
    pass

"""
Related genes are found by the following paper:
Agarwal, Shivani, and Shiladitya Sengupta.
"Ranking genes by relevance to a disease."
Proceedings of the 8th annual international
conference on computational systems bioinformatics. 2009.
"""
related_genes = {
    # markers for AML
    # 01: Myeloperoxidase
    1778: '"773","1779","MPO Myeloperoxidase","M19507_at"',
    # 02: CD13
    1816: '"792","1817","ANPEP Alanyl (membrane) aminopeptidase (aminopeptidase N, '
          'aminopeptidase M, microsomal aminopeptidase, CD13)","M22324_at"',
    # 03: CD33
    1833: '"808","1834","CD33 CD33 antigen (differentiation antigen)","M23197_at"',
    # 04: HOXA9 Homeo box A9
    3188: '"1391","3189","HOXA9 Homeo box A9","U41813_at"',
    # 05: MYBL2
    4129: '"1788","4130","MYBL2 V-myb avian myeloblastosis viral oncogene homolog-like 2","X13293_at"',
    # markers for ALL
    # 06: CD19
    6224: '"2673","6225","CD19 gene","M84371_rna1_s_at"',
    # 07: CD10 (CALLA)
    1085: '"493","1086","MME Membrane metallo-endopeptidase '
          '(neutral endopeptidase, enkephalinase, CALLA, CD10)","J03779_at"',
    # 08: TCL1 (T cell leukemia)
    4679: '"2065","4680","TCL1 gene (T cell leukemia) extracted from '
          'H.sapiens mRNA for Tcell leukemia/lymphoma 1","X82240_rna1_at"',
    # 09: C-myb
    5771: '"2489","5772","C-myb gene extracted from Human (c-myb) gene, complete primary cds, '
          'and five complete alternatively spliced cds","U22376_cds2_s_at"',
    # 10: Deoxyhypusine synthase
    6514: '"2801","6515","DHPS Deoxyhypusine synthase","U26266_s_at"',
    # 10: Deoxyhypusine synthase
    3763: '"1633","3764","DHPS Deoxyhypusine synthase","U79262_at"',
    # 12: G-gamma globin
    2344: '"1034","2345","G-gamma globin gene extracted from H.sapiens '
          'G-gamma globin and A-gamma globin genes","M91036_rna1_at",',
    # 13: Delta-globin
    6883: '"2945","6884","Delta-globin gene extracted from Human beta'
          ' globin region on chromosome 11","U01317_cds4_at"',
    # 14: Brain-expressed HHCPA78 homolog
    2481: '"1103","2482","Brain-expressed HHCPA78 homolog [human, HL-60 acute '
          'promyelocytic leukemia cells, mRNA, 2704 nt]","S73591_at"',
    # 15:
    6214: '"2669","6215","MPO from  Human myeloperoxidase gene, '
          'exons 1-4./ntype=DNA /annot=exon","M19508_xpt3_s_at"',
    # 16: Probable protein disulfide isomerase ER-60 precursor
    535: '"257","536","PROBABLE PROTEIN DISULFIDE ISOMERASE ER-60 PRECURSOR","D63878_at"',
    # 16: Probable protein disulfide isomerase ER-60 precursor
    5577: '"2415","5578","PROBABLE PROTEIN DISULFIDE ISOMERASE ER-60 PRECURSOR","Z49835_s_at"',
    # 16: Probable protein disulfide isomerase ER-60 precursor
    6167: '"2646","6168","PROBABLE PROTEIN DISULFIDE ISOMERASE ER-60 PRECURSOR","M13560_s_at"',
    # 17: NPM1 Nucleophosmin
    3577: '"1549","3578","NPM1 Nucleophosmin (nucleolar phosphoprotein B23, numatrin)","U66559_at"',
    # 18
    2441: '"1087","2442","CD34 CD34 antigen (hemopoietic progenitor cell antigen)","S53911_at"',
    # 20
    5687: '"2459","5688","CD24 signal transducer mRNA and 3 region","L33930_s_at"',
    # 21
    281: '"124","282","60S RIBOSOMAL PROTEIN L23","D21260_at"',
    # 22
    5224: '"2273","5225","5-aminolevulinic acid synthase gene extracted from Human DNA sequence '
          'from PAC 296K21 on chromosome X contains cytokeratin exon, delta-aminolevulinate synthase '
          '(erythroid); 5-aminolevulinic acid synthase.(EC 2.3.1.37). '
          '6-phosphofructo-2-kinase/fructose-2,6-bisphosphatase (EC 2.7.1.105, EC 3.1.3.46),'
          ' ESTs and STS","Z83821_cds2_at"',
    # 23
    4016: '"1736","4017","HLA CLASS II HISTOCOMPATIBILITY ANTIGEN, DR ALPHA CHAIN PRECURSOR","X00274_at"',
    # 24
    2878: '"1257","2879","Epstein-Barr virus-induced protein mRNA","U19261_at"',
    # 25
    4629: '"2042","4630","HNRPA1 Heterogeneous nuclear ribonucleoprotein A1","X79536_at"',
    # 26
    2401: '"1069","2402","Azurocidin gene","M96326_rna1_at"',
    # 27
    4594: '"2026","4595","Red cell anion exchanger (EPB3, AE1, Band 3) 3 non-coding region","X77737_at"',
    # 28
    5500: '"2386","5501","TOP2B Topoisomerase (DNA) II beta (180kD)","Z15115_at"',
    # 30
    5551: '"2402","5552","PROBABLE G PROTEIN-COUPLED RECEPTOR LCR1 HOMOLOG","L06797_s_at"',
    # 32
    3520: '"1527","3521","Int-6 mRNA","U62962_at"',
    # 33
    1173: '"534","1174","Alpha-tubulin isotype H2-alpha gene, last exon","K03460_at"',
    # 33
    4467: '"1957","4468","Alpha-tubulin mRNA","X68277_at"',
    # 33
    4909: '"2156","4910","Alpha-tubulin mRNA","X99325_at"',
    # 33
    6914: '"2957","6915","Alpha-tubulin mRNA","X01703_at"',
    # 34
    1684: '"738","1685","Terminal transferase mRNA","M11722_at"',
    # 35:
    5951: '"2561","5952","GLYCOPHORIN B PRECURSOR","U05255_s_at"'
}

def process_data_21_leukemia():
    """
    https://github.com/ramhiser/datamicroarray
    http://genomics-pubs.princeton.edu/oncology/affydata/index.html
    :return:
    """
    data_path = '/enter/your/directory/to/21_leukemia/'
    data = {'feature_ids': None, 'x_tr': [], 'y_tr': [], 'feature_names': []}
    import csv
    with open(data_path + 'golub_x.csv') as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        for row in csv_reader:
            if line_count == 0:
                data['feature_ids'] = [str(_) for _ in row[1:]]
                line_count += 1
            elif 1 <= line_count <= 72:
                data['x_tr'].append([float(_) for _ in row[1:]])
                line_count += 1
        data['x_tr'] = np.asarray(data['x_tr'])
        for i in range(len(data['x_tr'])):
            data['x_tr'][i] = data['x_tr'][i] / np.linalg.norm(data['x_tr'][i])
    # AML: 急性粒细胞白血病 ALL:急性淋巴细胞白血病
    with open(data_path + 'golub_y.csv') as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        for row in csv_reader:
            if line_count == 0:
                line_count += 1
                continue
            elif 1 <= line_count <= 72:
                line_count += 1
                if row[1] == 'ALL':
                    data['y_tr'].append(1.)
                else:
                    data['y_tr'].append(-1.)
        data['y_tr'] = np.asarray(data['y_tr'])
    data['n'] = 72
    data['p'] = 7129
    data['num_trials'] = 20
    data['num_posi'] = len([_ for _ in data['y_tr'] if _ == 1.])
    data['num_nega'] = len([_ for _ in data['y_tr'] if _ == -1.])
    trial_i = 0
    while True:
        # since original data is ordered, we need to shuffle it!
        rand_perm = np.random.permutation(data['n'])
        train_ind, test_ind = rand_perm[:50], rand_perm[50:]
        if len([_ for _ in data['y_tr'][train_ind] if _ == 1.]) == 33 or \
                len([_ for _ in data['y_tr'][train_ind] if _ == 1.]) == 32:
            data['trial_%d' % trial_i] = {'tr_index': rand_perm[train_ind], 'te_index': rand_perm[test_ind]}
            print(len([_ for _ in data['y_tr'][train_ind] if _ == 1.]),
                  len([_ for _ in data['y_tr'][train_ind] if _ == -1.]),
                  len([_ for _ in data['y_tr'][test_ind] if _ == 1.]),
                  len([_ for _ in data['y_tr'][test_ind] if _ == -1.])),
            success = True
            kf = KFold(n_splits=5, shuffle=False)
            for fold_index, (train_index, test_index) in enumerate(kf.split(range(len(train_ind)))):
                if len([_ for _ in data['y_tr'][train_ind[test_index]] if _ == -1.]) < 3:
                    success = False
                    break
                data['trial_%d_fold_%d' % (trial_i, fold_index)] = {'tr_index': train_ind[train_index],
                                                                    'te_index': train_ind[test_index]}
                print(len([_ for _ in data['y_tr'][train_ind[train_index]] if _ == 1.]),
                      len([_ for _ in data['y_tr'][train_ind[train_index]] if _ == -1.]),
                      len([_ for _ in data['y_tr'][train_ind[test_index]] if _ == 1.]),
                      len([_ for _ in data['y_tr'][train_ind[test_index]] if _ == -1.])),
            print(trial_i)
            if success:
                trial_i += 1
        if trial_i >= data['num_trials']:
            break
    pkl.dump(data, open(data_path + 'leukemia_data.pkl', 'wb'))

def cv_sht_am(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    all_results = dict()
    s_list = [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500]
    for para_s in s_list:
        tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
        te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
        x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
        y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
        results = dict()
        best_b, best_auc = None, None
        for para_b in range(1, 40, 1):
            wt, _, _, _ = c_algo_sht_auc(
                x_tr, __, __, __, y_tr, 0, data['p'], global_paras, 0, para_s, para_b, 1., 0.0)
            auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
            if best_b is None or best_auc is None or best_auc < auc_score:
                best_b, best_auc = para_b, auc_score
        tr_index = data['trial_%d' % trial_id]['tr_index']
        te_index = data['trial_%d' % trial_id]['te_index']
        x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
        y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
        wt, aucs, rts, epochs = c_algo_sht_auc(x_tr, __, __, __, y_tr, 0, data['p'], global_paras,
                                               0, para_s, best_b, 1., 0.0)
        results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, para_s, best_b],
                                        'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                                y_score=np.dot(data['x_tr'][te_index], wt)), 'wt': wt}
        print('best_b: %02d nonzero: %.4e test_auc: %.4f' %
              (best_b, float(np.count_nonzero(wt)), results[(trial_id, fold_id)]['auc_wt']))
        all_results[para_s] = results
    return trial_id, fold_id, all_results

def cv_sto_iht(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    s_list = [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500]
    all_results = dict()
    for para_s in s_list:
        tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
        te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
        x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
        y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
        results = dict()
        best_b, best_auc = None, None
        for para_b in range(1, 40, 1):
            wt, _, _, _ = c_algo_sto_iht(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, para_s, para_b, 1., 0.0)
            auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
            if best_b is None or best_auc is None or best_auc < auc_score:
                best_b, best_auc = para_b, auc_score
        tr_index = data['trial_%d' % trial_id]['tr_index']
        te_index = data['trial_%d' % trial_id]['te_index']
        x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
        y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
        wt, aucs, rts, epochs = c_algo_sto_iht(x_tr, __, __, __, y_tr, 0, data['p'],
                                               global_paras, para_s, best_b, 1., 0.0)
        results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, para_s, best_b],
                                        'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                                y_score=np.dot(data['x_tr'][te_index], wt)), 'wt': wt}
        print('best_b: %02d nonzero: %.4e test_auc: %.4f' %
              (best_b, float(np.count_nonzero(wt)), results[(trial_id, fold_id)]['auc_wt']))
        all_results[para_s] = results
    return trial_id, fold_id, all_results

def cv_hsg_ht(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    all_results = dict()
    s_list = [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500]
    for para_s in s_list:
        tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
        te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
        x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
        y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
        results = dict()
        best_c, best_auc = None, None
        for para_tau in [1., 10., 100., 1000.]:
            para_c, para_zeta = 3.0, 1.033
            wt, _, _, _ = c_algo_hsg_ht(x_tr, __, __, __, y_tr, 0, data['p'], global_paras,
                                        para_s, para_tau, para_zeta, para_c, 0.0)
            auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
            if best_auc is None or best_auc < auc_score:
                best_c, best_auc = para_c, auc_score
        tr_index = data['trial_%d' % trial_id]['tr_index']
        te_index = data['trial_%d' % trial_id]['te_index']
        x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
        y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
        wt, aucs, rts, epochs = c_algo_hsg_ht(x_tr, __, __, __, y_tr, 0, data['p'], global_paras,
                                              para_s, para_tau, para_zeta, best_c, 0.0)
        results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, best_c, para_s],
                                        'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                                y_score=np.dot(data['x_tr'][te_index], wt)), 'wt': wt}
        print('best_c: %02d nonzero: %.4e test_auc: %.4f' %
              (best_c, float(np.count_nonzero(wt)), results[(trial_id, fold_id)]['auc_wt']))
        all_results[para_s] = results
    return trial_id, fold_id, all_results

def cv_spam_l1(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    results = dict()
    tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
    te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    best_xi, best_l1, best_auc = None, None, None
    for para_xi, para_l1 in product(10. ** np.arange(-5, 3, 1, dtype=float),
                                    10. ** np.arange(-5, 3, 1, dtype=float)):
        wt, _, _, _ = c_algo_spam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, para_xi, para_l1, 0.0)
        auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
        if best_auc is None or best_auc < auc_score:
            best_xi, best_l1, best_auc = para_xi, para_l1, auc_score
    tr_index = data['trial_%d' % trial_id]['tr_index']
    te_index = data['trial_%d' % trial_id]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    wt, aucs, rts, epochs = c_algo_spam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, best_xi, best_l1, 0.0)
    results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, best_xi, best_l1],
                                    'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                            y_score=np.dot(data['x_tr'][te_index], wt)),
                                    'aucs': aucs, 'rts': rts, 'wt': wt}
    print('best_xi: %.1e best_l1: %.1e nonzero: %.4e test_auc: %.4f' %
          (best_xi, best_l1, float(np.count_nonzero(wt)), results[(trial_id, fold_id)]['auc_wt']))
    return trial_id, fold_id, results

def cv_spam_l2(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    results = dict()
    tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
    te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    best_xi, best_l2, best_auc = None, None, None
    for para_xi, para_l2, in product(10. ** np.arange(-5, 3, 1, dtype=float),
                                     10. ** np.arange(-5, 3, 1, dtype=float)):
        wt, _, _, _ = c_algo_spam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, para_xi, 0.0, para_l2)
        auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
        if best_auc is None or best_auc < auc_score:
            best_xi, best_l2, best_auc = para_xi, para_l2, auc_score
    tr_index = data['trial_%d' % trial_id]['tr_index']
    te_index = data['trial_%d' % trial_id]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    wt, aucs, rts, epochs = c_algo_spam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, best_xi, 0.0, best_l2)
    results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, best_xi, best_l2],
                                    'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                            y_score=np.dot(data['x_tr'][te_index], wt)),
                                    'aucs': aucs, 'rts': rts, 'wt': wt}
    print('best_xi: %.1e best_l2: %.1e nonzero: %.4e test_auc: %.4f' %
          (best_xi, best_l2, float(np.count_nonzero(wt)), results[(trial_id, fold_id)]['auc_wt']))
    return trial_id, fold_id, results

def cv_spam_l1l2(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    results = dict()
    tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
    te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    best_xi, best_l1, best_l2, best_auc = None, None, None, None
    for para_xi, para_l1, para_l2, in product(10. ** np.arange(-5, 3, 1, dtype=float),
                                              10. ** np.arange(-5, 3, 1, dtype=float),
                                              10. ** np.arange(-5, 3, 1, dtype=float)):
        wt, _, _, _ = c_algo_spam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, para_xi, para_l1, para_l2)
        auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
        if best_auc is None or best_auc < auc_score:
            best_xi, best_l1, best_l2, best_auc = para_xi, para_l1, para_l2, auc_score
    tr_index = data['trial_%d' % trial_id]['tr_index']
    te_index = data['trial_%d' % trial_id]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    wt, aucs, rts, epochs = c_algo_spam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, best_xi, best_l1, best_l2)
    results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, best_xi, best_l1, best_l2],
                                    'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                            y_score=np.dot(data['x_tr'][te_index], wt)),
                                    'aucs': aucs, 'rts': rts, 'wt': wt}
    print('best_xi: %.1e best_l1: %.1e best_l2: %.1e nonzero: %.4e test_auc: %.4f' %
          (best_xi, best_l1, best_l2, float(np.count_nonzero(wt)), results[(trial_id, fold_id)]['auc_wt']))
    return trial_id, fold_id, results

def cv_solam(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    results = dict()
    tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
    te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    best_xi, best_r, best_auc = None, None, None
    for para_xi, para_r, in product(np.arange(1, 101, 9, dtype=float), 10. ** np.arange(-1, 6, 1, dtype=float)):
        wt, _, _, _ = c_algo_solam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, para_xi, para_r)
        auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
        if best_auc is None or best_auc < auc_score:
            best_xi, best_r, best_auc = para_xi, para_r, auc_score
    tr_index = data['trial_%d' % trial_id]['tr_index']
    te_index = data['trial_%d' % trial_id]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    wt, aucs, rts, epochs = c_algo_solam(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, best_xi, best_r)
    results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, best_xi, best_r],
                                    'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                            y_score=np.dot(data['x_tr'][te_index], wt)),
                                    'aucs': aucs, 'rts': rts, 'wt': wt}
    print('best_xi: %.1e best_r: %.1e nonzero: %.4e test_auc: %.4f' %
          (best_xi, best_r, float(np.count_nonzero(wt)), results[(trial_id, fold_id)]['auc_wt']))
    return trial_id, fold_id, results

def cv_fsauc(para):
    data, trial_id, fold_id = para
    num_passes, step_len, verbose, record_aucs, stop_eps = 100, 1e2, 0, 1, 1e-6
    global_paras = np.asarray([num_passes, step_len, verbose, record_aucs, stop_eps], dtype=float)
    __ = np.empty(shape=(1,), dtype=float)
    results = dict()
    tr_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['tr_index']
    te_index = data['trial_%d_fold_%d' % (trial_id, fold_id)]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    best_g, best_r, best_auc = None, None, None
    aver_nonzero = []
    for para_g, para_r in product(2. ** np.arange(-10, 11, 1, dtype=float),
                                  10. ** np.arange(-1, 6, 1, dtype=float)):
        wt, _, _, _ = c_algo_fsauc(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, para_r, para_g)
        auc_score = roc_auc_score(y_true=data['y_tr'][te_index], y_score=np.dot(data['x_tr'][te_index], wt))
        if best_auc is None or best_auc < auc_score:
            best_g, best_r, best_auc = para_g, para_r, auc_score
    tr_index = data['trial_%d' % trial_id]['tr_index']
    te_index = data['trial_%d' % trial_id]['te_index']
    x_tr = np.asarray(data['x_tr'][tr_index], dtype=float)
    y_tr = np.asarray(data['y_tr'][tr_index], dtype=float)
    wt, aucs, rts, epochs = c_algo_fsauc(x_tr, __, __, __, y_tr, 0, data['p'], global_paras, best_r, best_g)
    results[(trial_id, fold_id)] = {'algo_para': [trial_id, fold_id, best_g, best_r],
                                    'auc_wt': roc_auc_score(y_true=data['y_tr'][te_index],
                                                            y_score=np.dot(data['x_tr'][te_index], wt)),
                                    'aucs': aucs, 'rts': rts, 'wt': wt}
    print('best_g: %.1e best_r: %.1e nonzero: %.4e test_auc: %.4f' %
          (best_g, best_r, float(np.mean(aver_nonzero)), results[(trial_id, fold_id)]['auc_wt']))
    return trial_id, fold_id, results

def run_methods(method):
    data_path = '/enter/your/directory/to/21_leukemia/'
    data = pkl.load(open(data_path + 'leukemia_data.pkl'))
    pool = multiprocessing.Pool(processes=int(sys.argv[2]))
    para_list = []
    for trial_id, fold_id in product(range(data['num_trials']), range(5)):
        para_list.append((data, trial_id, fold_id))
    if method == 'sht_am':
        ms_res = pool.map(cv_sht_am, para_list)
    elif method == 'sto_iht':
        ms_res = pool.map(cv_sto_iht, para_list)
    elif method == 'hsg_ht':
        ms_res = pool.map(cv_hsg_ht, para_list)
    elif method == 'spam_l1':
        ms_res = pool.map(cv_spam_l1, para_list)
    elif method == 'spam_l2':
        ms_res = pool.map(cv_spam_l2, para_list)
    elif method == 'spam_l1l2':
        ms_res = pool.map(cv_spam_l1l2, para_list)
    elif method == 'fsauc':
        ms_res = pool.map(cv_fsauc, para_list)
    elif method == 'solam':
        ms_res = pool.map(cv_solam, para_list)
    else:
        ms_res = None
    pool.close()
    pool.join()
    pkl.dump(ms_res, open(data_path + 're_%s.pkl' % method, 'wb'))

def summary_auc_results():
    data_path = '/enter/your/directory/to/21_leukemia/'
    all_aus = dict()
    for method_ind, method in enumerate(['solam', 'spam_l1', 'spam_l2', 'spam_l1l2', 'fsauc']):
        re_summary = pkl.load(open(data_path + 're_%s.pkl' % method, 'rb'))
        all_aus[method] = dict()
        for trial_id, fold_id, re in re_summary:
            all_aus[method][(trial_id, fold_id)] = re[(trial_id, fold_id)]['auc_wt']
        print(method, np.mean(np.asarray(all_aus[method].values())))
    for method_ind, method in enumerate(['sht_am', 'sto_iht', 'hsg_ht']):
        re_summary = pkl.load(open(data_path + 're_%s.pkl' % method, 'rb'))
        all_aus[method] = dict()
        for trial_id, fold_id, re in re_summary:
            for s in re:
                if s not in all_aus[method]:
                    all_aus[method][s] = dict()
                all_aus[method][s][(trial_id, fold_id)] = re[s][(trial_id, fold_id)]['auc_wt']
        for s in all_aus[method].keys():
            print(method, s, np.mean(np.asarray(all_aus[method][s].values())))
    pkl.dump(all_aus, open(data_path + 're_summary_all_aucs.pkl', 'wb'))

def summary_feature_results():
    data_path = '/enter/your/directory/to/21_leukemia/'
    all_features = dict()
    for method_ind, method in enumerate(['solam', 'spam_l1', 'spam_l2', 'spam_l1l2', 'fsauc']):
        re_summary = pkl.load(open(data_path + 're_%s.pkl' % method, 'rb'))
        all_features[method] = dict()
        for trial_id, fold_id, re in re_summary:
            re_features = set(np.nonzero(re[(trial_id, fold_id)]['wt'])[0])
            inter = set(related_genes.keys()).intersection(re_features)
            all_features[method][(trial_id, fold_id)] = float(len(inter)) / float(len(re_features))
        print(method, np.mean(np.asarray(all_features[method].values())))
    for method_ind, method in enumerate(['sht_am', 'sto_iht', 'hsg_ht']):
        re_summary = pkl.load(open(data_path + 're_%s.pkl' % method, 'rb'))
        all_features[method] = dict()
        for trial_id, fold_id, re in re_summary:
            for s in re:
                if s not in all_features[method]:
                    all_features[method][s] = dict()
                re_features = set(np.nonzero(re[s][(trial_id, fold_id)]['wt'])[0])
                inter = set(related_genes.keys()).intersection(re_features)
                if len(re_features) == 0:
                    all_features[method][s][(trial_id, fold_id)] = 0.0
                else:
                    all_features[method][s][(trial_id, fold_id)] = float(len(inter)) / float(len(re_features))
        for s in all_features[method].keys():
            print(method, s, np.mean(np.asarray(all_features[method][s].values())))
    pkl.dump(all_features, open(data_path + 're_summary_all_features.pkl', 'wb'))

def show_figure7():
    import matplotlib.pyplot as plt
    from matplotlib import rc
    from pylab import rcParams
    plt.rcParams["font.family"] = "serif"
    plt.rcParams["font.serif"] = "Times"
    plt.rcParams["font.size"] = 16
    rc('text', usetex=True)
    rcParams['figure.figsize'] = 5, 4
    fig, ax = plt.subplots(1, 1)
    ax.grid(color='lightgray', linestyle='--')
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    data_path = '/enter/your/directory/to/21_leukemia/'
    s_list = [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500]
    all_aucs = pkl.load(open(data_path + 're_summary_all_aucs.pkl'))
    method_list = ['sht_am', 'solam', 'spam_l1', 'spam_l2', 'fsauc', 'sto_iht', 'hsg_ht']
    method_label_list = ['SHT-AUC', 'SOLAM', 'SPAM-$\displaystyle \ell^1$',
                         'SPAM-$\displaystyle \ell^1$', 'FSAUC', 'StoIHT', 'HSG-HT']
    color_list = ['r', 'g', 'm', 'b', 'y', 'k', 'darkorange', 'olive', 'darkorange']
    marker_list = ['s', 'o', 'P', 'X', 'H', '*', 'x', 'v', '^', '+', '>']
    for method_ind, method in enumerate(method_list):
        if method in ['solam', 'spam_l1', 'spam_l2', 'fsauc']:
            print(method, '%.4f %.4f' % (np.mean(all_aucs[method].values()), np.std(all_aucs[method].values())))
            plt.plot(s_list, [np.mean(all_aucs[method].values())] * len(s_list), label=method_label_list[method_ind],
                     marker=marker_list[method_ind], markerfacecolor='w', color=color_list[method_ind], linewidth=1.5,
                     markersize=5.)
        if method in ['sht_am', 'sto_iht', 'hsg_ht']:
            aucs, aucs_std = [], []
            for s in s_list:
                aucs.append(np.mean(all_aucs[method][s].values()))
                aucs_std.append(np.std(all_aucs[method][s].values()))
            import operator
            index, value = max(enumerate(aucs), key=operator.itemgetter(1))
            print(method, '%.4f %.4f' % (value, aucs_std[index]))
            plt.plot(s_list, aucs, label=method_label_list[method_ind], marker=marker_list[method_ind],
                     markerfacecolor='w', color=color_list[method_ind], linewidth=1.5, markersize=5.)
    ax.legend(loc='lower right', framealpha=1., frameon=True, borderpad=0.1,
              labelspacing=.2, handletextpad=0.1, markerfirst=True)
    ax.set_xlabel('Sparsity $\displaystyle k$')
    ax.set_ylabel('AUC Score')
    ax.set_ylim([0.8, 1.01])
    ax.set_yticks([0.8, 0.85, 0.9, 0.95, 1.0])
    ax.set_yticklabels([0.8, 0.85, 0.9, 0.95, 1.0])
    root_path = '/enter/your/directory/to/save/'
    f_name = root_path + 'real_leukemia_auc.pdf'
    plt.savefig(f_name, dpi=600, bbox_inches='tight', pad_inches=0, format='pdf')
    plt.close()

def show_figure4_b():
    import matplotlib.pyplot as plt
    from matplotlib import rc
    from pylab import rcParams
    plt.rcParams["font.family"] = "serif"
    plt.rcParams["font.serif"] = "Times"
    plt.rcParams["font.size"] = 14
    rc('text', usetex=True)
    rcParams['figure.figsize'] = 5, 4
    fig, ax = plt.subplots(1, 1)
    ax.grid(color='lightgray', linestyle='--')
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    data_path = '/enter/your/directory/to/21_leukemia/'
    s_list = [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500]
    all_features = pkl.load(open(data_path + 're_summary_all_features.pkl'))
    method_list = ['sht_am', 'sto_iht', 'hsg_ht']
    method_label_list = ['SHT-AUC', 'StoIHT', 'HSG-HT']
    color_list = ['r', 'b', 'g']
    marker_list = ['D', 's', 'o']
    for method_ind, method in enumerate(method_list):
        ratio_features = []
        for s in s_list:
            ratio_features.append(np.mean(all_features[method][s].values()))
        plt.plot(s_list, ratio_features, label=method_label_list[method_ind], marker=marker_list[method_ind],
                 markerfacecolor='w', color=color_list[method_ind], linewidth=1.5, markersize=5.)
    ax.legend(loc='upper right', framealpha=1., frameon=True, borderpad=0.1,
              labelspacing=0.1, handletextpad=0.1, markerfirst=True)
    ax.set_xlabel('Sparsity $\displaystyle k$')
    ax.set_ylabel('Ratio of Selected Genes')
    ax.set_xticks([100, 200, 300, 400, 500])
    ax.set_xticklabels([100, 200, 300, 400, 500])
    root_path = '/enter/your/directory/to/save/'
    f_name = root_path + 'real_leukemia_features.pdf'
    plt.savefig(f_name, dpi=600, bbox_inches='tight', pad_inches=0, format='pdf')
    plt.close()

def main():
    if sys.argv[1] == 'run_sht_am':
        run_methods(method='sht_am')
    elif sys.argv[1] == 'run_sto_iht':
        run_methods(method='sto_iht')
    elif sys.argv[1] == 'run_spam_l1':
        run_methods(method='spam_l1')
    elif sys.argv[1] == 'run_spam_l2':
        run_methods(method='spam_l2')
    elif sys.argv[1] == 'run_fsauc':
        run_methods(method='fsauc')
    elif sys.argv[1] == 'run_hsg_ht':
        run_methods(method='hsg_ht')
    elif sys.argv[1] == 'run_solam':
        run_methods(method='solam')
    elif sys.argv[1] == 'show_figure7':
        # summary_auc_results()
        show_figure7()
    elif sys.argv[1] == 'show_figure4_b':
        # summary_feature_results()
        show_figure4_b()

if __name__ == '__main__':
    main()


import logging

import pkg_resources
from datetime import datetime

import six
from formencode import validators
from tg import request
from tg import tmpl_context as c, app_globals as g
from pytz import timezone
from tg import expose, redirect, validate, flash
from tg.decorators import without_trailing_slash
from decorator import decorator
from webob import exc

from allura import version
from allura.app import Application, SitemapEntry
from allura.controllers import BaseController
from allura.controllers.feed import FeedArgs, FeedController
from allura.controllers.rest import AppRestControllerMixin
from allura.lib import helpers as h
from allura.lib.decorators import require_post
from allura.lib.plugin import AuthenticationProvider
from allura.lib.security import require_access
from allura.lib.widgets.user_profile import SendMessageForm, SectionsUtil, SectionBase, ProjectsSectionBase
from allura.model import User, ACE, ProjectRole

log = logging.getLogger(__name__)

class F:
    send_message = SendMessageForm()

class UserProfileApp(Application):

    """
    This is the Profile tool, which is automatically installed as
    the default (first) tool on any user project.
    """
    __version__ = version.__version__
    tool_label = 'Profile'
    max_instances = 0
    has_notifications = False
    icons = {
        24: 'images/home_24.png',
        32: 'images/home_32.png',
        48: 'images/home_48.png'
    }

    def __init__(self, user, config):
        Application.__init__(self, user, config)
        self.root = UserProfileController()
        self.templates = pkg_resources.resource_filename(
            'allura.ext.user_profile', 'templates')
        self.api_root = UserProfileRestController()

    @property
    @h.exceptionless([], log)
    def sitemap(self):
        return [SitemapEntry('Profile', '.')]

    def admin_menu(self):
        return []

    def main_menu(self):
        return [SitemapEntry('Profile', '.')]

    def is_visible_to(self, user):
        # we don't work with user subprojects
        return c.project.is_root

    def install(self, project):
        pr = ProjectRole.by_user(c.user)
        if pr:
            self.config.acl = [
                ACE.allow(pr._id, perm)
                for perm in self.permissions]

    def uninstall(self, project):  # pragma no cover
        pass

    @property
    def profile_sections(self):
        """
        Loads and caches user profile sections from the entry-point
        group ``[allura.user_profile.sections]``.

        Profile sections are loaded unless disabled (see
        `allura.lib.helpers.iter_entry_points`) and are sorted according
        to the `user_profile_sections.order` config value.

        The config should contain a comma-separated list of entry point names
        in the order that they should appear in the profile.  Unknown or
        disabled sections are ignored, and available sections that are not
        specified in the order come after all explicitly ordered sections,
        sorted randomly.
        """
        if hasattr(UserProfileApp, '_sections'):
            return UserProfileApp._sections
        UserProfileApp._sections = SectionsUtil.load_sections('user_profile')
        return UserProfileApp._sections

class UserProfileController(BaseController, FeedController):

    def _check_security(self):
        require_access(c.project, 'read')

    def _check_can_message(self, from_user, to_user):
        if from_user.is_anonymous():
            flash('You must be logged in to send user messages.', 'info')
            redirect(six.ensure_text(request.referer or '/'))

        if not (from_user and from_user.get_pref('email_address')):
            flash('In order to send messages, you must have an email address '
                  'associated with your account.', 'info')
            redirect(six.ensure_text(request.referer or '/'))

        if not (to_user and to_user.get_pref('email_address')):
            flash('This user can not receive messages because they do not have '
                  'an email address associated with their account.', 'info')
            redirect(six.ensure_text(request.referer or '/'))

        if to_user.get_pref('disable_user_messages'):
            flash('This user has disabled direct email messages', 'info')
            redirect(six.ensure_text(request.referer or '/'))

    @expose('jinja:allura.ext.user_profile:templates/user_index.html')
    def index(self, **kw):
        """
        https://sf-11.xb.sf.net/u/admin1
        """
        user = c.project.user_project_of
        if not user:
            raise exc.HTTPNotFound()
        provider = AuthenticationProvider.get(request)
        sections = [section(user, c.project)
                    for section in c.app.profile_sections]

        noindex = True
        has_projects = False
        has_timeline = False
        for s in sections:
            s.setup_context()
            if s.context.get('projects'):
                has_projects = True
            if s.context.get('timeline'):
                has_timeline = True
        if has_projects and has_timeline:
            noindex = False
        return dict(
            user=user,
            reg_date=provider.user_registration_date(user),
            sections=sections,
            noindex=noindex,
        )

    def get_feed(self, project, app, user):
        """Return a :class:`allura.controllers.feed.FeedArgs` object describing
        the xml feed for this controller.

        Overrides :meth:`allura.controllers.feed.FeedController.get_feed`.

        """
        user = project.user_project_of
        return FeedArgs(
            {'author_link': user.url()},
            'Recent posts by %s' % user.display_name,
            project.url())

    @expose('jinja:allura.ext.user_profile:templates/send_message.html')
    def send_message(self):
        """Render form for sending a message to another user.

        """
        self._check_can_message(c.user, c.project.user_project_of)

        delay = c.user.time_to_next_user_message()
        expire_time = str(delay) if delay else None
        c.form = F.send_message
        if c.user.get_pref('message_reply_real_address'):
            c.form.fields.reply_to_real_address.attrs = {'checked': 'checked'}
        return dict(user=c.project.user_project_of, expire_time=expire_time)

    @require_post()
    @expose()
    @validate(dict(subject=validators.NotEmpty,
                   message=validators.NotEmpty))
    def send_user_message(self, subject='', message='', cc=None, reply_to_real_address=None):
        """Handle POST for sending a message to another user.

        """
        self._check_can_message(c.user, c.project.user_project_of)

        if cc:
            cc = c.user.get_pref('email_address')

        if c.user.can_send_user_message():
            if reply_to_real_address:
                c.user.set_pref('message_reply_real_address', True)
            else:
                c.user.set_pref('message_reply_real_address', False)
            c.user.send_user_message(
                c.project.user_project_of, subject, message, cc, reply_to_real_address, c.user.preferences.email_address)
            flash("Message sent.")
        else:
            flash("You can't send more than %i messages per %i seconds" % (
                g.user_message_max_messages,
                g.user_message_time_interval), 'error')
        return redirect(c.project.user_project_of.url())

    @without_trailing_slash
    @expose('jinja:allura.ext.user_profile:templates/user_card.html')
    def user_card(self):
        u = c.project.user_project_of
        locationData = u.get_pref('localization')
        webpages = u.get_pref('webpages')
        location = ''
        website = ''
        if locationData.city and locationData.country:
            location = locationData.city + ', ' + locationData.country
        elif locationData.country and not locationData.city:
            location = locationData.country
        elif locationData.city and not locationData.country:
            location = locationData.city

        if len(webpages) > 0:
            website = webpages[0]

        return dict(
            user=u,
            location=location,
            website=website)

class UserProfileRestController(AppRestControllerMixin):
    @expose('json:')
    def index(self, **kw):
        user = c.project.user_project_of
        if not user:
            raise exc.HTTPNotFound()
        sections = [section(user, c.project)
                    for section in c.app.profile_sections]
        json = {}
        for s in sections:
            if hasattr(s, '__json__'):
                json.update(s.__json__())
        return json

class ProfileSectionBase(SectionBase):

    """
    This is the base class for sections on the Profile tool.

    .. py:attribute:: template

       A resource string pointing to the template for this section.  E.g.::

           template = "allura.ext.user_profile:templates/projects.html"

    Sections must be pointed to by an entry-point in the group
    ``[allura.user_profile.sections]``.
    """

    def __init__(self, user, project):
        """
        Creates a section for the given :param:`user` and user
        :param:`project`.  Stores the values as attributes of
        the same name.
        """
        super().__init__(user)
        self.project = project

class PersonalDataSection(ProfileSectionBase):
    template = 'allura.ext.user_profile:templates/sections/personal-data.html'

    def prepare_context(self, context):
        context['timezone'] = self.user.get_pref('timezone')
        if context['timezone']:
            tz = timezone(context['timezone'])
            context['timezone'] = tz.tzname(datetime.utcnow())
        return context

    def __json__(self):
        auth_provider = AuthenticationProvider.get(request)
        return dict(
            username=self.user.username,
            name=self.user.display_name,
            joined=auth_provider.user_registration_date(self.user),
            localization=self.user.get_pref('localization')._deinstrument(),
            sex=self.user.get_pref('sex'),
            telnumbers=self.user.get_pref('telnumbers')._deinstrument(),
            skypeaccount=self.user.get_pref('skypeaccount'),
            webpages=self.user.get_pref('webpages')._deinstrument(),
            availability=self.user.get_pref('availability')._deinstrument())

class ProjectsSection(ProfileSectionBase, ProjectsSectionBase):
    template = 'allura.ext.user_profile:templates/sections/projects.html'

class SkillsSection(ProfileSectionBase):
    template = 'allura.ext.user_profile:templates/sections/skills.html'

    def __json__(self):
        return dict(skills=self.user.get_skills())

    def check_display(self):
        return bool(self.user.skills)

class ToolsSection(ProfileSectionBase):
    template = 'allura.ext.user_profile:templates/sections/tools.html'

class SocialSection(ProfileSectionBase):
    template = 'allura.ext.user_profile:templates/sections/social.html'

    def __json__(self):
        return dict(
            socialnetworks=self.user.get_pref('socialnetworks')._deinstrument())

    def check_display(self):
        return bool(self.user.get_pref('socialnetworks'))


from __future__ import print_function
import networkx as nx
import pandas as pd
from datetime import date
import requests
import json
import os

# Set globals
# Get source url from local `source_db.txt`
SOURCE_URL = open(
    os.path.join(
        os.path.dirname(
            os.path.abspath(__file__)),
        '..',
        'source_url.txt'),
    'r').read().strip()
# Set curation directory
NEIGHBOR_DIR='nbdb'
TEMPLATE_DIR='mvrs'

"""Import internal dependencies
"""
try:
    from analyze.prepare_data import __main__ as prepare_data
    from analyze.model import __template__
    from analyze.model import __model__
    from analyze.model import load_references
    from analyze.model import load_metabolite_synonym_dictionary
    from analyze.utils import remove_defective_reactions
    from utils import progress_feed, track_progress, read_network, \
                      get_metaboverse_cli_version, write_database, safestr, \
                      update_session_vars
except:
    import importlib.util
    spec = importlib.util.spec_from_file_location(
        "", os.path.abspath("./metaboverse_cli/analyze/prepare_data.py"))
    prepare_data = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(prepare_data)
    prepare_data = prepare_data.__main__

    spec = importlib.util.spec_from_file_location(
        "", os.path.abspath("./metaboverse_cli/analyze/model.py"))
    model = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(model)
    __template__ = model.__template__
    __model__ = model.__model__
    load_references = model.load_references
    load_metabolite_synonym_dictionary = model.load_metabolite_synonym_dictionary

    module_path = os.path.abspath(
        os.path.join(".", "metaboverse_cli", "analyze", "utils.py"
                     ))
    spec = importlib.util.spec_from_file_location("", module_path)
    analyze_utils = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(analyze_utils)
    remove_defective_reactions = analyze_utils.remove_defective_reactions

    spec = importlib.util.spec_from_file_location(
        "", os.path.abspath("./metaboverse_cli/utils.py"))
    utils = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(utils)
    progress_feed = utils.progress_feed
    track_progress = utils.track_progress
    read_network = utils.read_network
    get_metaboverse_cli_version = utils.get_metaboverse_cli_version
    write_database = utils.write_database
    safestr = utils.safestr
    update_session_vars = utils.update_session_vars

def process_data(
        network,
        args_dict):
    """
    """
    if str(args_dict['transcriptomics']).lower() != 'none' \
            or str(args_dict['proteomics']).lower() != 'none' \
            or str(args_dict['metabolomics']).lower() != 'none':

        data, stats, unmapped = prepare_data(
            network=network,
            transcriptomics_url=args_dict['transcriptomics'],
            proteomics_url=args_dict['proteomics'],
            metabolomics_url=args_dict['metabolomics'],
            database_source=args_dict['database_source'])
        flag_data = False

    else:
        data = pd.DataFrame()
        data['NoSample'] = [0, 0, 0]
        data.index = ['dummy_index1', 'dummy_index2', 'dummy_index3']

        stats = pd.DataFrame()
        stats['NoSample'] = [0, 0, 0]
        stats.index = ['dummy_index1', 'dummy_index2', 'dummy_index3']

        unmapped = {
            'transcriptomics_unmapped': [],
            'proteomics_unmapped': []
        }
        flag_data = True

    return data, stats, unmapped, flag_data

def read_template(
        args_dict,
        network,
        url,
        user_provided=False):
    """
    """
    def get_template_file(url, args_dict):
        file = os.path.join(
            args_dict['output'],
            args_dict['organism_id'] + '_template.mvrs')
        print('Downloading graph template database...', '\n\t', url)
        os.system('curl -kL ' + url + ' -o \"' + file + '\"')
        return file

    print('Downloading Metaboverse graph template for organism...')

    if user_provided == False:
        file = get_template_file(url, args_dict)
    else:
        file = url

    with open(file) as graph_template:
        graph_data = json.load(graph_template)

    graph = nx.readwrite.json_graph.node_link_graph(
        {
            'nodes': graph_data['nodes'],
            'links': graph_data['links']
        },
        directed=graph_data['directed'],
        multigraph=graph_data['multigraph'])
    network['reaction_database'] = graph_data['reaction_dictionary']
    network['pathway_database'] = graph_data['pathway_dictionary']
    super_pathways = graph_data['super_pathways']
    degree_dictionary = graph_data['degree_dictionary']

    reverse_genes, protein_dictionary, chebi_dictionary, \
    name_reference, uniprot_mapper = load_references(
        args_dict=args_dict,
        ensembl=network['ensembl_synonyms'],
        uniprot=network['uniprot_synonyms'],
        chebi=network['chebi_mapper'],
        uniprot_metabolites=network['uniprot_metabolites'])
    metabolite_mapper = load_metabolite_synonym_dictionary()

    args_dict["curation_version"] = network["metaboverse-curate_version"]
    args_dict["curation_date"] = network["curation_date"]
    args_dict["database_version"] = network["database_version"]
    args_dict['template_url'] = file
    args_dict['template_version'] = graph_data['metadata']['template_version']
    args_dict['template_date'] = graph_data['metadata']['template_date']

    progress_feed(args_dict, "graph", 9)

    return graph, args_dict, network, name_reference, \
        degree_dictionary, super_pathways, chebi_dictionary, \
        uniprot_mapper, metabolite_mapper

def download_neighbors_dictionary(
        args_dict,
        url,
        user_provided=False):
    """
    """
    print('Downloading Metaboverse neighbors dictionary for organism...')

    def get_neighbor_file(url, args_dict):
        file = os.path.join(
            args_dict['output'],
            args_dict['organism_id'] + '.nbdb')
        print('Downloading nearest neighbors database...', '\n\t', url)
        os.system('curl -kL ' + url + ' -o \"' + file + '\"')
        return file

    if user_provided == False:
        file = get_neighbor_file(url, args_dict)
    else:
        file = url

    neighbors_dictionary = read_network(
        file_path=args_dict['output'],
        network_url=file)

    neighbors_dictionary['nbdb-Metaboverse-url'] = file

    return neighbors_dictionary

def make_neighbors_dictionary(
        args_dict,
        graph,
        reaction_dictionary):
    """
    """
    reaction_ids = set(reaction_dictionary.keys())

    print('Generating Metaboverse neighbors dictionary for organism...')

    counter = 0
    edges = list(graph.edges)
    edge_len = len(edges)
    neighbors_dictionary = {}
    for e in edges:
        one = e[0]
        two = e[1]
        if one in neighbors_dictionary.keys():
            neighbors_dictionary[one].add(two)
        else:
            neighbors_dictionary[one] = set()
            neighbors_dictionary[one].add(two)
        if two in neighbors_dictionary.keys():
            neighbors_dictionary[two].add(one)
        else:
            neighbors_dictionary[two] = set()
            neighbors_dictionary[two].add(one)
        counter = track_progress(args_dict, counter, edge_len, 3)

    print('Tuning neighbors dictionary...')
    reaction_neighbors_dictionary = {}
    counter = 0
    neighbors = list(neighbors_dictionary.keys())
    neighbors_number = len(neighbors)
    for neighbor in neighbors:
        if neighbor in reaction_ids:
            components = neighbors_dictionary[neighbor]
            connected_reactions = set()
            for _c in components:
                for _c_ in neighbors_dictionary[_c]:
                    if _c_ in reaction_ids:
                        connected_reactions.add(_c_)
            connected_reactions = list(connected_reactions)
            reaction_neighbors_dictionary[neighbor] = connected_reactions
        counter = track_progress(args_dict, counter, neighbors_number, 3)

    reaction_neighbors_dictionary['nbdb-Metaboverse-version'] = get_metaboverse_cli_version()
    reaction_neighbors_dictionary['nbdb-Metaboverse-date'] = date.today().strftime('%Y-%m-%d')
    reaction_neighbors_dictionary['nbdb-Metaboverse-url'] = os.path.join(args_dict['output'], args_dict['organism_id'] + '.nbdb')

    print('Writing neighbors dictionary to database file...')
    write_database(
        output=args_dict['output'],
        file=args_dict['organism_id'] + '.nbdb',
        database=reaction_neighbors_dictionary)

    return reaction_neighbors_dictionary

def __main__(
        args_dict):
    """Analyze data on network model
    """

    # Get network curation info
    network = read_network(
        file_path=args_dict['output'],
        network_url=args_dict['curation'])
    progress_feed(args_dict, "graph", 1)

    if args_dict['organism_curation_file'] != 'None':
        args_dict['organism_id'] = network['organism_id']

    # Read in data (if any)
    data, stats, unmapped, flag_data = process_data(
        network=network,
        args_dict=args_dict)
    progress_feed(args_dict, "graph", 2)
    print("Data processed of dimensions: " + str(data.shape))

    # Generate graph template
    this_version = get_metaboverse_cli_version()
    test_url = (
        SOURCE_URL
        + 'v' + this_version + '/'
        + TEMPLATE_DIR + '/'
        + args_dict['organism_id'] + '_template.mvrs')

    # If unable to access pre-curated network, force new curation
    if args_dict['force_new_curation'] != True:
        try:
            url_response = requests.head(test_url)
        except:
            print("Unable to access source files from: " + str(test_url))
            print("Will force a new curation of source files instead...")
            args_dict['force_new_curation'] == True
            url_response = ''
    else:
        url_response = ''

    if (args_dict['force_new_curation'] == False \
    or args_dict['force_new_curation'] == "False") \
    and 'graph_template_file' in args_dict \
    and safestr(args_dict['graph_template_file']) != None \
    and safestr(args_dict['graph_template_file']) != 'None':
        try:
            graph, args_dict, network, name_reference, \
            degree_dictionary, super_pathways, chebi_dictionary, \
            uniprot_mapper, metabolite_mapper = read_template(
                args_dict=args_dict,
                network=network,
                url=args_dict['graph_template_file'],
                user_provided=True)
        except:
            graph, args_dict, network, name_reference, \
            degree_dictionary, super_pathways, chebi_dictionary, \
            uniprot_mapper, metabolite_mapper = __template__(
                args_dict=args_dict,
                network=network,
                species_id=args_dict['organism_id'],
                output_file=args_dict['output_file'])
    elif (args_dict['force_new_curation'] == False \
    or args_dict['force_new_curation'] == "False") \
    and url_response.status_code != 404:
        graph, args_dict, network, name_reference, \
        degree_dictionary, super_pathways, chebi_dictionary, \
        uniprot_mapper, metabolite_mapper = read_template(
            args_dict=args_dict,
            network=network,
            url=test_url)
    else:
        graph, args_dict, network, name_reference, \
        degree_dictionary, super_pathways, chebi_dictionary, \
        uniprot_mapper, metabolite_mapper = __template__(
            args_dict=args_dict,
            network=network,
            species_id=args_dict['organism_id'],
            output_file=args_dict['output_file'])

    if len(graph.nodes) == 0 or len(graph.edges) == 0:
        raise Exception("Unable to generate a reaction-based network based on the input organism template.")
    else:
        print("Successfully loaded network with " + str(len(graph.nodes)) + " nodes and " + str(len(graph.edges)) + " edges")

    # Generate graph template
    neighbors_url = (
        SOURCE_URL
        + 'v' + this_version + '/'
        + NEIGHBOR_DIR + '/'
        + args_dict['organism_id'] + '.nbdb')

    # If unable to access pre-curated network, force new curation
    if args_dict['force_new_curation'] != True:
        try:
            neighbor_response = requests.head(neighbors_url)
        except:
            print("Unable to access source files from: " + str(test_url))
            print("Will force a new curation of source files instead...")
            args_dict['force_new_curation'] == True
            neighbor_response = ''
    else:
        neighbor_response = ''

    force_neighbors = False
    if (args_dict['force_new_curation'] == False \
    or args_dict['force_new_curation'] == "False") \
    and 'neighbor_dictionary_file' in args_dict \
    and safestr(args_dict['neighbor_dictionary_file']) != None \
    and safestr(args_dict['neighbor_dictionary_file']) != 'None':
        try:
            neighbors_dictionary = download_neighbors_dictionary(
                args_dict=args_dict,
                url=args_dict['neighbor_dictionary_file'],
                user_provided=True)
        except:
            force_neighbors = True

    elif (args_dict['force_new_curation'] == False \
    or args_dict['force_new_curation'] == "False") \
    and neighbor_response.status_code != 404:
        try:
            neighbors_dictionary = download_neighbors_dictionary(
                args_dict=args_dict,
                url=neighbors_url)
        except:
            force_neighbors = True
    else:
        force_neighbors = True

    if force_neighbors == True:
        no_defective_reactions = remove_defective_reactions(
            network=network)
        neighbors_dictionary = make_neighbors_dictionary(
            args_dict=args_dict,
            graph=graph,
            reaction_dictionary=no_defective_reactions)
    else:
        progress_feed(args_dict, "graph", 6)

    # Overlay data on graph and collapse as able
    print("Modeling data onto network...")
    graph_name = __model__(
        graph=graph,
        args_dict=args_dict,
        network=network,
        data=data,
        stats=stats,
        species_id=args_dict['organism_id'],
        output_file=args_dict['output_file'],
        neighbors_dictionary=neighbors_dictionary,
        name_reference=name_reference,
        degree_dictionary=degree_dictionary,
        chebi_dictionary=chebi_dictionary,
        uniprot_mapper=uniprot_mapper,
        metabolite_mapper=metabolite_mapper,
        super_pathways=super_pathways,
        unmapped=unmapped,
        flag_data=flag_data)

    args_dict = update_session_vars(args_dict)

    return graph_name

def test():
    args_dict = {
        'output': "C:\\Users\\jorda\\Desktop",
        'curation': "MMU.mvdb",
        'organism_id': 'MMU',
        'output_file': "C:\\Users\\jorda\\Desktop\\MMU.mvrs",
        'labels': "0",
        'blocklist': "H+",
        'database_date': "",
        'curation_date': ""}
    args_dict = {
        'output': "C:\\Users\\jorda\\Desktop",
        'curation': "HSA.mvdb",
        'graph_template_file': "C:\\Users\\jorda\\Desktop\\HSA_template.mvrs",
        'organism_id': 'HSA'}

    network = read_network(
        file_path="C:\\Users\\jorda\\Desktop",
        network_url="MMU.mvdb")

    neighbors = read_network(
        file_path="C:\\Users\\jorda\\Desktop",
        network_url="MODEL1604210000.nbdb")

    neighbors_dictionary = download_neighbors_dictionary(
        args_dict=args_dict,
        url="C:\\Users\\jorda\\Desktop\\HSA.nbdb",
        user_provided=True)

    graph, args_dict, network, name_reference, \
    degree_dictionary, super_pathways, chebi_dictionary, \
        uniprot_mapper, metabolite_mapper = read_template(
        args_dict=args_dict,
        network=network,
        url=args_dict['graph_template_file'],
        user_provided=True)


import json
import logging
from collections import deque
from functools import wraps
from typing import Any, Deque, Dict, List, Optional, Set, Tuple, overload

from aiohttp import web

from gear import Database, maybe_parse_bearer_header
from hailtop.utils import secret_alnum_string

log = logging.getLogger('utils')

@web.middleware
async def unavailable_if_frozen(request: web.Request, handler):
    if request.method in ("POST", "PATCH") and request.app['frozen']:
        raise web.HTTPServiceUnavailable()
    return await handler(request)

def authorization_token(request):
    auth_header = request.headers.get('Authorization')
    if not auth_header:
        return None
    session_id = maybe_parse_bearer_header(auth_header)
    if not session_id:
        return None
    return session_id

def add_metadata_to_request(fun):
    @wraps(fun)
    async def wrapped(request, *args, **kwargs):
        request['batch_telemetry'] = {'operation': fun.__name__}

        batch_id = request.match_info.get('batch_id')
        if batch_id is not None:
            request['batch_telemetry']['batch_id'] = batch_id

        job_id = request.match_info.get('job_id')
        if job_id is not None:
            request['batch_telemetry']['job_id'] = job_id

        return await fun(request, *args, **kwargs)

    return wrapped

def coalesce(x, default):
    if x is not None:
        return x
    return default

class Box:
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return f'{self.value}'

class WindowFractionCounter:
    def __init__(self, window_size: int):
        self._window_size = window_size
        self._q: Deque[Tuple[str, bool]] = deque()
        self._n_true = 0
        self._seen: Set[str] = set()

    def clear(self):
        self._q.clear()
        self._n_true = 0
        self._seen = set()

    def push(self, key: str, x: bool):
        self.assert_valid()
        if key in self._seen:
            return
        while len(self._q) >= self._window_size:
            old_key, old = self._q.popleft()
            self._seen.remove(old_key)
            if old:
                self._n_true -= 1
        self._q.append((key, x))
        self._seen.add(key)
        if x:
            self._n_true += 1
        self.assert_valid()

    def fraction(self) -> float:
        self.assert_valid()
        # (1, 1) prior
        return (self._n_true + 1) / (len(self._q) + 2)

    def __repr__(self):
        self.assert_valid()
        return f'{self._n_true}/{len(self._q)}'

    def assert_valid(self):
        assert len(self._q) <= self._window_size
        assert 0 <= self._n_true <= self._window_size

class ExceededSharesCounter:
    def __init__(self):
        self._global_counter = WindowFractionCounter(10)

    def push(self, success: bool):
        token = secret_alnum_string(6)
        self._global_counter.push(token, success)

    def rate(self) -> float:
        return self._global_counter.fraction()

    def __repr__(self):
        return f'global {self._global_counter}'

async def query_billing_projects_with_cost(db, user=None, billing_project=None) -> List[Dict[str, Any]]:
    where_conditions = ["billing_projects.`status` != 'deleted'"]
    args = []

    if user:
        where_conditions.append("JSON_CONTAINS(users, JSON_QUOTE(%s))")
        args.append(user)

    if billing_project:
        where_conditions.append('billing_projects.name_cs = %s')
        args.append(billing_project)

    if where_conditions:
        where_condition = f'WHERE {" AND ".join(where_conditions)}'
    else:
        where_condition = ''

    sql = f'''
SELECT billing_projects.name as billing_project,
  billing_projects.`status` as `status`,
  users, `limit`, COALESCE(cost_t.cost, 0) AS accrued_cost
FROM billing_projects
LEFT JOIN LATERAL (
  SELECT billing_project, JSON_ARRAYAGG(`user_cs`) as users
  FROM billing_project_users
  WHERE billing_project_users.billing_project = billing_projects.name
  GROUP BY billing_project_users.billing_project
  LOCK IN SHARE MODE
) AS t ON TRUE
LEFT JOIN LATERAL (
  SELECT SUM(`usage` * rate) as cost
  FROM (
    SELECT billing_project, resource_id, CAST(COALESCE(SUM(`usage`), 0) AS SIGNED) AS `usage`
    FROM aggregated_billing_project_user_resources_v3
    WHERE billing_projects.name = aggregated_billing_project_user_resources_v3.billing_project
    GROUP BY billing_project, resource_id
    LOCK IN SHARE MODE
  ) AS usage_t
  LEFT JOIN resources ON resources.resource_id = usage_t.resource_id
  GROUP BY usage_t.billing_project
) AS cost_t ON TRUE
{where_condition}
LOCK IN SHARE MODE;
'''

    billing_projects = []
    async for record in db.execute_and_fetchall(sql, tuple(args)):
        record['users'] = json.loads(record['users']) if record['users'] is not None else []
        billing_projects.append(record)

    return billing_projects

async def query_billing_projects_without_cost(
    db: Database, user: Optional[str] = None, billing_project: Optional[str] = None
) -> List[Dict[str, Any]]:
    where_conditions = ["billing_projects.`status` != 'deleted'"]
    args = []

    if user:
        where_conditions.append("JSON_CONTAINS(users, JSON_QUOTE(%s))")
        args.append(user)

    if billing_project:
        where_conditions.append('billing_projects.name_cs = %s')
        args.append(billing_project)

    if where_conditions:
        where_condition = f'WHERE {" AND ".join(where_conditions)}'
    else:
        where_condition = ''

    sql = f'''
SELECT billing_projects.name as billing_project,
  billing_projects.`status` as `status`,
  users, `limit`
FROM billing_projects
LEFT JOIN LATERAL (
  SELECT billing_project, JSON_ARRAYAGG(`user_cs`) as users
  FROM billing_project_users
  WHERE billing_project_users.billing_project = billing_projects.name
  GROUP BY billing_project_users.billing_project
  LOCK IN SHARE MODE
) AS t ON TRUE
{where_condition}
LOCK IN SHARE MODE;
'''

    billing_projects = []
    async for record in db.execute_and_fetchall(sql, tuple(args)):
        record['users'] = json.loads(record['users']) if record['users'] is not None else []
        billing_projects.append(record)

    return billing_projects

def json_to_value(x):
    if x is None:
        return x
    return json.loads(x)

def regions_to_bits_rep(selected_regions, all_regions_mapping):
    result = 0
    for region in selected_regions:
        idx = all_regions_mapping[region]
        assert idx < 64, str(all_regions_mapping)
        result |= 1 << (idx - 1)
    return result

@overload
def regions_bits_rep_to_regions(regions_bits_rep: None, all_regions_mapping: Dict[str, int]) -> None:
    ...

@overload
def regions_bits_rep_to_regions(regions_bits_rep: int, all_regions_mapping: Dict[str, int]) -> List[str]:
    ...

def regions_bits_rep_to_regions(
    regions_bits_rep: Optional[int], all_regions_mapping: Dict[str, int]
) -> Optional[List[str]]:
    if regions_bits_rep is None:
        return None
    result = []
    for region, idx in all_regions_mapping.items():
        selected_region = bool((regions_bits_rep >> idx - 1) & 1)
        if selected_region:
            result.append(region)
    return result


class Solution(object):
    def exclusiveTime(self, N, logs):
        A = []
        START, END = 'start', 'end'
        for x in logs:
            a,b,c = x.split(':')
            A.append( (int(a), b, int(c)))

        ans = [0] * N
        stack = []
        start_time = A[0][2]
        cur_time = A[0][2]
        elapsed = 0
        for i, (fn1, typ1, time1) in enumerate(A):
            if typ1 == START:
                stack.append((fn1, time1))
            else:
                fn2, time2 = stack.pop()
                delta = time1 - time2 + 1
                ans[fn1] += delta
                stack = [(f, t+delta) for f,t in stack]
        return ans


import tensorflow as tf
import config as cfg

slim = tf.contrib.slim

class LeNet:
    def network_fn(self, input, is_training):
        reuse = False if is_training else True
        net = tf.image.resize_images(input, [cfg.LENET_INPUT_W, cfg.LENET_INPUT_H], method=tf.image.ResizeMethod.BILINEAR)
        with slim.arg_scope([slim.conv2d], stride=1, normalizer_fn=slim.batch_norm, padding='VALID'):
            with tf.variable_scope('lenet', reuse=reuse):
                with tf.variable_scope('conv1'):
                    net = slim.conv2d(net, 6, [5, 5])
                    net = slim.avg_pool2d(net, 2, 2)
                with tf.variable_scope('conv2'):
                    net = slim.conv2d(net, 16, [5, 5])
                    net = slim.avg_pool2d(net, 2, 2)
                with tf.variable_scope('flatten'):
                    net = slim.flatten(net, scope='flat6')
                with tf.variable_scope('full_conn'):
                    net = slim.fully_connected(net, 120)
                    net = slim.fully_connected(net, 84)
                    #net = slim.dropout(net, DROPOUT_RATIO, is_training=is_training, scope='dropout8')
                    net = slim.fully_connected(net, cfg.LABEL_SIZE)
                return net


import unittest
from unittest.mock import patch
from employee import Employee

class TestEmployee(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        """If you need to setup i.e data and access just to read it, this will be
        a good option"""
        print("setupClass")

    @classmethod
    def tearDownClass(cls):
        """You can clean your environment with this"""
        print("teardownClass")

    def setUp(self):
        # Run it's code before every single test
        print("setUp")
        self.emp_1 = Employee("Corey", "Schafer", 50000)
        self.emp_2 = Employee("Sue", "Smith", 60000)

    def tearDown(self):
        """For example you have some functions that added files to a directory or to a
        database, then in setUp you could create something to hold those files and in
        this method you could delete all of those, so you are clean for the next test"""
        print("tearDown")

    def test_email(self):
        print("test_email")
        self.emp_1.first = "John"
        self.emp_2.first = "Jane"

        self.assertEqual(self.emp_1.email, "John.Schafer@email.com")
        self.assertEqual(self.emp_2.email, "Jane.Smith@email.com")

    def test_fullname(self):
        print("test_fullname")
        self.assertEqual(self.emp_1.fullname, "Corey Schafer")
        self.assertEqual(self.emp_2.fullname, "Sue Smith")

        self.emp_1.first = "John"
        self.emp_2.first = "Jane"

        self.assertEqual(self.emp_1.fullname, "John Schafer")
        self.assertEqual(self.emp_2.fullname, "Jane Smith")

    def test_apply_raise(self):
        print("test_apply_raise")
        self.emp_1.apply_raise()
        self.emp_2.apply_raise()

        self.assertEqual(self.emp_1.pay, 52500)
        self.assertEqual(self.emp_2.pay, 63000)

    def test_monthly_schedule(self):
        print("test_monthly_schedule")
        with patch("employee.requests.get") as mocked_get:
            mocked_get.return_value.ok = True
            mocked_get.return_value.text = "Success"

            schedule = self.emp_1.monthly_schedule("May")
            mocked_get.assert_called_with("http://company.com/Schafer/May")
            self.assertEqual(schedule, "Success")

            #
            mocked_get.return_value.ok = False

            schedule = self.emp_2.monthly_schedule("June")
            mocked_get.assert_called_with("http://company.com/Smith/June")
            self.assertEqual(schedule, "Bad Response!")

if __name__ == "__main__":
    unittest.main()


### This is the code for Problem 2
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import ParameterGrid
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score

def generate_dateset(num, save_file_name, fig_title, fig_name):
    #------- for negative class
    nx1 = np.random.uniform(low=0.2, high=0.6, size=num)
    nx2 = ((nx1-0.4)*3)**2 + 0.225 + np.random.normal(loc=0.0, scale=0.025, size=num)
    ny = -1*np.ones(num)

    # ------- for positive class
    px1 = np.random.uniform(low=0.4, high=0.8, size=num)
    px2 = 1-((px1 - 0.6) * 3) ** 2 - 0.225 + np.random.normal(loc=0.0, scale=0.025, size=num)
    py = np.ones(num)

    ### visualize the dataset
    plt.figure()
    plt.scatter(nx1, nx2, marker = 'o', c='b', alpha=0.5, label='Negative')
    plt.scatter(px1, px2, marker='x', c='r', alpha=0.5, label='Positive')
    plt.xlabel('Dimension 1')
    plt.ylabel('Dimension 2')
    plt.title(fig_title)
    plt.legend()
    plt.savefig(fig_name, dpi=300)
    plt.close()

    #### now, we want to save our dataset into a csv file
    n_df = pd.DataFrame()
    n_df['x1'] = nx1
    n_df['x2'] = nx2
    n_df['y'] = ny

    p_df = pd.DataFrame()
    p_df['x1'] = px1
    p_df['x2'] = px2
    p_df['y'] = py

    df = pd.concat([n_df, p_df])
    df.to_csv(save_file_name, index=False)

if __name__ == '__main__':
    # #--- for training dataset
    num = 50
    save_file_name = 'Hyperbolas/hyperbolas_train.csv'
    fig_title = 'Training Dataset'
    fig_name = 'Hyperbolas/hyperbolas_train.png'
    generate_dateset(num, save_file_name, fig_title, fig_name)

    # --- for validation dataset
    num = 50
    save_file_name = 'Hyperbolas/hyperbolas_val.csv'
    fig_title = 'Validation Dataset'
    fig_name = 'Hyperbolas/hyperbolas_val.png'
    generate_dateset(num, save_file_name, fig_title, fig_name)

    # --- for test dataset
    num = 1000
    save_file_name = 'Hyperbolas/hyperbolas_test.csv'
    fig_title = 'Test Dataset'
    fig_name = 'Hyperbolas/hyperbolas_test.png'
    generate_dateset(num, save_file_name, fig_title, fig_name)


import numpy as np
import pandas as pd

CSV_FILE_TYPES = ['csv']
EXCEL_FILE_TYPES = ['xls, xlsx']
UNKNOWN_SONYNOMS = ["unknown", "n/a", "NaN", ""]

def get_headers(file_path):
    return pd.read_csv(file_path, nrows=1).columns

def handle_nans(iterable):
    return [np.nan if str(i).lower().strip() in UNKNOWN_SONYNOMS else i for i in iterable]

def dtype_share(iterable, dtype, threshold=0.9):
    success_count = 0
    for i in iterable:
        try:
            dtype(i)
        except ValueError:
            pass
        else:
            success_count += 1
    return success_count / len(iterable)

def evaluate_label_and_id(label_column, id_column, df,
                          problem_type):
    """
    Assess if there are any issues with the labelled data column
    """
    output = {}
    output["n_rows"] = len(df)
    for col in [label_column, id_column]:
        df[col] = handle_nans(df[col])
    output["label_nan_share"] = df[label_column].isnull().sum() / len(df[label_column])
    output["id_nan_share"] = df[id_column].isnull().sum() / len(df[id_column])
    if problem_type == "classification":
        output["number_of_categories"] = len(df[label_column].dropna().unique())
    if problem_type == "regression":
        output["not_a_number_share"] = dtype_share(df[label_column], float)

    return [
        {
            "id": "id_nan_share",
            "theme": "ID",
            "summary": "Many N/A's in ID column",
            "details": "The ID column contains many unknowns. We can still train the model but it's worth looking at the data set",
            "stat": output["id_nan_share"],
            "status": "yellow",
            "include": str(output["id_nan_share"] > 1000)
        },
        {
            "id": "label_nan_share",
            "theme": "Target",
            "summary": "Many N/A's in target column",
            "details": "The Target column contains many unknowns. We can still train the model but it's worth looking at the data set",
            "stat": output["label_nan_share"],
            "status": "yellow",
            "include": str(output["label_nan_share"] > 0.1)
        },
        {
            "id": "number_of_categories",
            "theme": "Target",
            "summary": "Many classes in target column",
            "details": "Your target data column contains too many different classes. It might be hard to generalize",
            "stat": output["number_of_categories"],
            "status": "red",
            "include": str(output["number_of_categories"] > 100)
        }
    ]

if __name__ == '__main__':
    df = pd.read_csv('/Users/mikkeld/ai-for-analysts/Backend/datasets/regression/titanic/data/train.csv')
    label_col = 'Survived'
    id_col = 'PassengerId'
    problem_type = 'classification'
    res = evaluate_label_and_id(
        label_column=label_col,
        id_column=id_col,
        df=df,
        problem_type='classification')
    print(res)


print('='*30)
print('BANCO CEV')
print('='*30)
cel50 = 0
while True:
    valor = int(input('Que valor você quer sacar? R$'))
    if valor >= 50:
        while valor // 50 == 0:
            valor -= 50
            cel50 += 1
            print(cel50)
    break
print(f'Total de {cel50} cédulas de R$50')


from django.shortcuts import render, HttpResponseRedirect
from . forms import PistesForm
from . import models

def formulaire(request):
    if request.method == "POST":
        form = PistesForm(request)
        return render(request, "basedonnees/pistesatterissage/formulaire.html", {"form": form})
    else:
        form = PistesForm()
        id = ""
        return render(request, "basedonnees/pistesatterissage/formulaire.html", {"form": form, "id": id})

def traitement(request):
    lform = PistesForm(request.POST)
    if lform.is_valid():
        lform.save()
        return HttpResponseRedirect('/indexpistes/')
    else:
        return render(request, "basedonnees/pistesatterissage/formulaire.html", {"form": lform})

def index(request):
    liste = list(models.Pistes.objects.all())
    return render(request, 'basedonnees/pistesatterissage/index.html', {'liste': liste})

def affiche(request, id):
    pistesatterissage = models.Pistes.objects.get(pk=id)
    return render(request, 'basedonnees/pistesatterissage/affiche.html', {"pistesatterissage": pistesatterissage})

def update(request, id):
    pistesatterissage = models.Pistes.objects.get(pk=id)
    form = PistesForm(pistesatterissage.dico())
    return render(request, 'basedonnees/pistesatterissage/update.html', {'form': form, 'id': id})

def updatetraitement(request, id):
    lform = PistesForm(request.POST)
    if lform.is_valid():
        pistesatterissage = lform.save(commit=False)
        pistesatterissage.id = id
        pistesatterissage.save()
        return HttpResponseRedirect('/indexpistes/')
    else:
        return render(request, "basedonnees/pistesatterissage/update.html", {"form": lform, "id": id})

def delete(request, id):
    pistesatterissage = models.Pistes.objects.get(pk=id)
    pistesatterissage.delete()
    return HttpResponseRedirect('/aeroports/indexpistes/')


"""Implementation of the rpki-rtr client function for EOM.

This module interfaces with a rpki-rtr server component to retrieve
validated RPKI information for use by routers. The received data is
saved through the aggregator module to an internal data store.
"""

import asyncore
import netaddr
from eom.rpki.rtr.client   import ClientChannel
from eom.rpki.rtr.channels import Timestamp
from eom.rpki.rtr.pdus     import ResetQueryPDU, SerialQueryPDU

from eom.generic_poller import EOMGenericPoller

ClientChannelClass = ClientChannel

class EOMRPKIRtrCli(EOMGenericPoller):
    """Poller for fetching RPKI prefix information from an RPKI-Rtr Manager instance"""
    def __init__(self, args, aggregator):
        """Instantiate the rpki-rtr client object.

        Args:
            conf(dict): Configuration parameters for the client
            aggregator(EOMAggregator): The data aggregator object
        """
        assert aggregator != None
        if args.reset_session:
            aggregator.reset_rpki_rtr_session(args.host, args.port)
        constructor = getattr(ClientChannelClass, args.protocol)
        self.client = constructor(args)
        self.sql = aggregator.get_sql_connection()
        self.client.set_sql_connection(self.sql)
        self.client.update_session()
        self.polled = self.client.updated
        self.pending = False

    def cleanup(self):
        """Clean up the client object associated with the poller.

        Args:
            None

        Returns:
            None
        """
        self.client.cleanup()

    def poll(self, now):
        """Poll data from configured RPKI-rtr server instances.

        Process the queue to identfy events that must be processed
        during this time interval.

        Args:
            now: current timestamp

        Returns:
            (Boolean, int): A tuple comprising of a pending status and
                            the timestamp for the next event in the queue.
        """
        # Check if we've either never polled or polled earlier than the last update
        if not self.polled or self.polled < self.client.updated:
            if self.client.serial is not None and now > self.client.updated + self.client.expire:
                logging.info("[Expiring client data: serial %s, last updated %s, expire %s]",
                             self.client.serial, self.client.updated, self.client.expire)
                self.client.cache_reset()

            if self.client.serial is None or self.client.nonce is None:
                self.polled = now
                self.client.push_pdu(ResetQueryPDU(version = self.client.version))

            elif now >= self.client.updated + self.client.refresh:
                self.polled = now
                self.client.push_pdu(SerialQueryPDU(version = self.client.version,
                                                    serial  = self.client.serial,
                                                    nonce   = self.client.nonce))

        # Find how long we must wait
        timer = self.client.retry if (now >= self.client.updated + self.client.refresh) else self.client.refresh
        wakeup = max(now, Timestamp(max(self.polled, self.client.updated) + timer))
        # Find if we're still waiting for data to arrive
        if self.client.updated < self.polled:
            pending = True
        else:
            pending = False
            if self.pending:
                # Fix the additional fields in the database
                self.update_rpki_rtr_minmax()
        self.pending = pending
        return (pending, wakeup)

    def update_rpki_rtr_minmax(self):
        """
        For each prefix corresponding to the client's cache id update
        the min and max prefix fields
        """
        cache_id = int(self.client.cache_id)
        cur = self.sql.cursor()
        cur.execute("SELECT asn, prefix, prefixlen, max_prefixlen "
                    "FROM prefix "
                    "WHERE cache_id = ?", (cache_id, ))
        rows = cur.fetchall()
        for r in rows:
            (asn, prefix, prefixlen, max_prefixlen) = r
            ipstr = unicode(str(prefix) + "/" + str(prefixlen), "utf-8")
            prefix_min = EOMGenericPoller.get_ip_str(netaddr.IPNetwork(ipstr).first)
            prefix_max = EOMGenericPoller.get_ip_str(netaddr.IPNetwork(ipstr).last)

            cur = self.sql.cursor()
            cur.execute("UPDATE prefix "
                        "SET prefix_min = ?, prefix_max = ? "
                        "WHERE cache_id = ? AND asn = ? AND prefix = ? AND prefixlen = ? AND max_prefixlen = ?",
                        (prefix_min, prefix_max, cache_id, asn, prefix, prefixlen, max_prefixlen))

        self.sql.commit()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from authorization.views import test_session, authorize, get_status, UserView,logout
from django.urls import path, include

urlpatterns = [
    path('test', test_session),
    path('authorize', authorize),
    path('status', get_status),
    path('user', UserView.as_view()),
    path('logout', logout)
]


"""
1650. Lowest Common Ancestor of a Binary Tree III
Medium

92

5

Add to List

Share
Given two nodes of a binary tree p and q, return their lowest common ancestor (LCA).

Each node will have a reference to its parent node. The definition for Node is below:

class Node {
    public int val;
    public Node left;
    public Node right;
    public Node parent;
}
According to the definition of LCA on Wikipedia: "The lowest common ancestor of two nodes p and q in a tree T is the lowest node that has both p and q as descendants (where we allow a node to be a descendant of itself)."

Example 1:

Input: root = [3,5,1,6,2,0,8,null,null,7,4], p = 5, q = 1
Output: 3
Explanation: The LCA of nodes 5 and 1 is 3.
Example 2:

Input: root = [3,5,1,6,2,0,8,null,null,7,4], p = 5, q = 4
Output: 5
Explanation: The LCA of nodes 5 and 4 is 5 since a node can be a descendant of itself according to the LCA definition.
Example 3:

Input: root = [1,2], p = 1, q = 2
Output: 1

Constraints:

The number of nodes in the tree is in the range [2, 105].
-109 <= Node.val <= 109
All Node.val are unique.
p != q
p and q exist in the tree.
"""

# Definition for a Node.
class Node:
    def __init__(self, val, left=None, right=None, parent=None):
        self.val = val
        self.left = left
        self.right = right
        self.parent = parent

"""

Tree traverse

traverse tree upwards to parent, and keep track of list of ancestors, and check if current parent node of current node is in the parent node list of the other node, if so, return the node as answer

"""

class Solution0:
    def lowestCommonAncestor(self, p: 'Node', q: 'Node') -> 'Node':
        if p.val == q.val:
            return p

        p_parents = set()
        q_parents = set()

        while p and q and p.val != q.val:
            p_parents.add(p.val)
            if p.val in q_parents:
                return p
            if p.parent:
                p = p.parent
            q_parents.add(q.val)
            if q.val in p_parents:
                return q
            if q.parent:
                q = q.parent

        return p

"""
Tree traverse - Two Pointers

Seen the parent inside each TreeNode, this is a Linked List question.

similar to intersection point of two linked lists (lc0160), traverse along parents list, when reaching root, set the pointer to start node of the other node's (path) starting point, when they eventually meet, it will be at the lowest common ancestor

time O(M+N)
space O(1)
"""

class Solution:
    def lowestCommonAncestor(self, p: 'Node', q: 'Node') -> 'Node':
        if p.val == q.val:
            return p
        p0, q0 = p, q  # keep starting node of p and q

        while p != q:
            # when p.parent is None p is at root, we restart p at q0
            p = p.parent if p.parent is not None else q0
            q = q.parent if q.parent is not None else p0

        # when exist loop, p and q would be pointing at lowest common ancestor
        return p