
from sklearn import *
import sklearn
import pandas as pd
import numpy as np
import xgboost as xgb
import lightgbm as lgb
from time import gmtime, strftime
import numpy.random as rng
from multiprocessing.dummy import Pool
import h5py
import concurrent.futures
import tensorflow as tf
import multiprocessing as mp

from sklearn.cross_validation import KFold
from keras.models import Sequential, Model
from keras.layers.core import Dense, Dropout, Flatten, Reshape
from keras.layers.normalization import BatchNormalization
from keras.layers.embeddings import Embedding
from keras.layers import Input, concatenate, merge
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D, AveragePooling2D
from keras.optimizers import SGD, RMSprop, Adam
from keras.callbacks import EarlyStopping
from keras.utils import np_utils
from keras import backend as K
from sklearn.metrics import log_loss
from keras import __version__ as keras_version

graph = tf.get_default_graph()

HIDDEN_UNITS = [64, 16, 8]
DNN_EPOCHS = 40
BATCH_SIZE = 5
DNN_BN = True
DROPOUT_RATE = 0.5
SIAMESE_PAIR_SIZE = 100000
MAX_WORKERS = 8
EMBEDDING_SIZE = 6

full_feature = True

data_folder = '../Data/'
train = pd.read_csv(data_folder + 'training_variants')
#print train.dtypes
test = pd.read_csv(data_folder + 'test_variants')
trainx = pd.read_csv(data_folder + 'training_text', sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"])
#print trainx.dtypes
testx = pd.read_csv(data_folder + 'test_text', sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"])

train = pd.merge(train, trainx, how='left', on='ID').fillna('')
#train = train.iloc[1:1000]
y = train['Class'].values
train = train.drop(['Class'], axis=1)

test = pd.merge(test, testx, how='left', on='ID').fillna('')
pid = test['ID'].values

#df_all = pd.concat((train, test), axis=0, ignore_index=True)
#df_all['Gene_Share'] = df_all.apply(lambda r: sum([1 for w in r['Gene'].split(' ') if w in r['Text'].split(' ')]), axis=1).astype(np.int8)
#df_all['Variation_Share'] = df_all.apply(lambda r: sum([1 for w in r['Variation'].split(' ') if w in r['Text'].split(' ')]), axis=1).astype(np.int8)
#
#print df_all[['Gene_Share', 'Variation_Share']].max()
## exit(0)
#if full_feature:
#    #commented for Kaggle Limits
#    for i in range(5):
#        df_all['Gene_'+str(i)] = df_all['Gene'].map(lambda x: str(x[i]) if len(x)>i else '')
#        df_all['Variation'+str(i)] = df_all['Variation'].map(lambda x: str(x[i]) if len(x)>i else '')
#    print df_all.dtypes
#
#    gen_var_lst = sorted(list(train.Gene.unique()) + list(train.Variation.unique()))
#    print(len(gen_var_lst))
#    gen_var_lst = [x for x in gen_var_lst if len(x.split(' '))==1]
#    print(len(gen_var_lst))
#    i_ = 0
#    #commented for Kaggle Limits
#    for gen_var_lst_itm in gen_var_lst:
#        if i_ % 100 == 0: print(i_)
#        df_all['GV_'+str(gen_var_lst_itm)] = df_all['Text'].map(lambda x: str(x).count(str(gen_var_lst_itm))).astype(np.int8)
#        i_ += 1
#        if i_ == 5:
#            break
#
#for c in df_all.columns:
#    if df_all[c].dtype == 'object':
#        if c in ['Gene','Variation']:
#            lbl = preprocessing.LabelEncoder()
#            df_all[c+'_lbl_enc'] = lbl.fit_transform(df_all[c].values)
#            df_all[c+'_len'] = df_all[c].map(lambda x: len(str(x)))
#            df_all[c+'_words'] = df_all[c].map(lambda x: len(str(x).split(' ')))
#        elif c != 'Text':
#            lbl = preprocessing.LabelEncoder()
#            df_all[c] = lbl.fit_transform(df_all[c].values)
#        if c=='Text':
#            df_all[c+'_len'] = df_all[c].map(lambda x: len(str(x)))
#            df_all[c+'_words'] = df_all[c].map(lambda x: len(str(x).split(' ')))
#
#train = df_all.iloc[:len(train)]
#print "... train dtypes before svd ..."
#print train.dtypes
#print train.head()
#exit(0)
#test = df_all.iloc[len(train):]
#
#class cust_regression_vals(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
#    def fit(self, x, y=None):
#        return self
#    def transform(self, x):
#        x = x.drop(['Gene', 'Variation','ID','Text'],axis=1).values
#        return x
#
#class cust_txt_col(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
#    def __init__(self, key):
#        self.key = key
#    def fit(self, x, y=None):
#        return self
#    def transform(self, x):
#        return x[self.key].apply(str)
#
#print('Pipeline...')
#fp = pipeline.Pipeline([
#    ('union', pipeline.FeatureUnion(
#        n_jobs = -1,
#        transformer_list = [
#            ('standard', cust_regression_vals()),
#            ('pi1', pipeline.Pipeline([('Gene', cust_txt_col('Gene')), ('count_Gene', feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 8))), ('tsvd1', decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12))])),
#            ('pi2', pipeline.Pipeline([('Variation', cust_txt_col('Variation')), ('count_Variation', feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 8))), ('tsvd2', decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12))])),
#            #commented for Kaggle Limits
#            ('pi3', pipeline.Pipeline([('Text', cust_txt_col('Text')), ('tfidf_Text', feature_extraction.text.TfidfVectorizer(ngram_range=(1, 2))), ('tsvd3', decomposition.TruncatedSVD(n_components=50, n_iter=25, random_state=12))]))
#        ])
#    )])
#
#train = fp.fit_transform(train);
#print type(train)
#print(train.shape)
#print (train.nbytes)
#np.save("train_array", train)
## print(df.dtypes)
## print(df.memory_usage())
#test = fp.transform(test); print(test.shape)
#np.save("test_array", test)
#exit(0)
train = np.load("./train_array.npy")
test = np.load("./test_array.npy")
# siamese_features_array = np.load("./siamese_features_array_2017_09_15_07_57_44.npy")
y = y - 1 #fix for zero bound array

CONTINUOUS_INDICES = []
SPARSE_INDICES = []

for i in range((train.shape)[1]):
    if (i >= 3205 and i <= 3212):
        pass
    elif (i >= 2 and i <= 113): # or (i >= 114 and i <= 3204):
        SPARSE_INDICES.append(i)
    else:
        CONTINUOUS_INDICES.append(i)
#train = train[:, CONTINUOUS_INDICES]
#test = test[:, CONTINUOUS_INDICES]

print('train shape after loading and selecting trainging columns: %s' % str(train.shape))

siamese_train_len = len(train) // 3
print('siamese_train_len is %d' % (siamese_train_len))
siamese_train_data = train[:siamese_train_len]
siamese_train_label = y[:siamese_train_len]

lgbm_train_data = train[siamese_train_len:]
lgbm_train_label = y[siamese_train_len:]
#train = train[:200]
#y = y[:200]
#test = test[:200]
#pid = pid[:200]

def xgbTrain(train_data, train_label, flod = 5):
    """
    """
    denom = 0
    fold = 5 #Change to 5, 1 for Kaggle Limits
    models = []
    for i in range(fold):
        params = {
            'eta': 0.03333,
            'max_depth': 4,
            'objective': 'multi:softprob',
            'eval_metric': 'mlogloss',
            'num_class': 9,
            'seed': i,
            'silent': True
        }
        x1, x2, y1, y2 = model_selection.train_test_split(train_data, train_label, test_size=0.18, random_state=i)
        watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]
        model = xgb.train(params, xgb.DMatrix(x1, y1), 1000,  watchlist, verbose_eval=50, early_stopping_rounds=100)
        score1 = metrics.log_loss(y2, model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit), labels = list(range(9)))
        #print(score1)

        models.append((model, 'x'))

    return models


def lgbm_train(train_data, train_label, fold = 5):
    """
    LGB Training
    """
   # print train.shape
   # print siamese_features_array.shape
   # train_merge = siamese_features_array #np.concatenate((train, siamese_features_array), axis = 1)
   # print train_merge.shape
   # # exit(0)
    print("Over all training size:")
    print(train_data.shape)
   # train_data = train_merge#[:train_len * 3 / 10]
   # train_label = lgbm_train_label#[:train_len * 3 / 10]
    #valide_data = train_merge[train_len * 9 / 10:]
    #valide_label = y[train_len * 9 / 10:]

    models = []
    for i in range(fold):
        d_train = lgb.Dataset(train_data, train_label) #, categorical_feature = SPARCE_INDICES)
        #d_valide = lgb.Dataset(valide_data, valide_label)

        params = {
            'task': 'train',
            'boosting_type': 'gbdt',
            'objective': 'multiclass',
            'metric': {'multi_logloss'},
            'num_class': 9,
          #  'num_leaves': 256,
          #  'max_depth': 12,
          #  'feature_fraction': 0.9,
          #  'bagging_fraction': 0.95,
          #  'bagging_freq': 5,
            'num_leaves': 60, # 60,
          #  'min_sum_hessian_in_leaf': 20,
            'max_depth': 10, # 10,
            'learning_rate': 0.02, # 0.02,
            'feature_fraction': 0.5,
            'verbose': 0,
          #   'valid_sets': [d_valide],
            'num_boost_round': 327,
            'feature_fraction_seed': i,
            # 'bagging_fraction': 0.9,
            # 'bagging_freq': 15,
            # 'bagging_seed': i,
            # 'early_stopping_round': 10
            # 'random_state': 10
            # 'verbose_eval': 20
            #'min_data_in_leaf': 665
        }

        # ROUNDS = 1
        print('fold: %d th light GBM train :-)' % (i))
        # params['feature_fraction_seed'] = i
        #bst = lgb.train(
        #                params ,
        #                d_train,
        #                verbose_eval = False
        #                # valid_sets = [d_valide]
        #                #num_boost_round = 1
        #                )
        cv_result = lgb.cv(params, d_train, nfold=10)
        pd.DataFrame(cv_result).to_csv('cv_result', index = False)
        exit(0)
        # pred = model_eval(bst, 'l', test)
        #print pred.shape
        #print pred[0, :]
        models.append((bst, 'l'))

    return models


def create_model(input_len):
    model = Sequential()
    model.add(Dense(HIDDEN_UNITS[0], activation='sigmoid', input_dim = input_len))
    if DNN_BN:
        model.add(BatchNormalization())
    if DROPOUT_RATE > 0:
        model.add(Dropout(DROPOUT_RATE))
    model.add(Dense(HIDDEN_UNITS[1], activation='sigmoid'))
    if DNN_BN:
        model.add(BatchNormalization())
    if DROPOUT_RATE > 0:
        model.add(Dropout(DROPOUT_RATE))
    # model.add(Dropout(0.1))
    #model.add(Dense(32, activation='relu'))
    #model.add(Dropout(0.2))
    model.add(Dense(9, activation='softmax'))

    # optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
    optimizer = RMSprop(lr=1e-3, rho = 0.9, epsilon = 1e-8)
    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics = ['accuracy'])

    return model


def create_embedding_model(CONTINUE_SIZE, SPARSE_SIZE):
    """
    """
    print('CONTINUOUS_SIZE = %d' % CONTINUE_SIZE)
    print('SPARSE_SIZE = %d' % SPARSE_SIZE)
    sparse_feature = Input(shape=(SPARSE_SIZE,))
    sparse_embedding = Embedding(55, EMBEDDING_SIZE, input_length = SPARSE_SIZE)(sparse_feature)
    sparse_embedding = Reshape((EMBEDDING_SIZE * SPARSE_SIZE,))(sparse_embedding)

    # print "model input size: %d" % CONTINUOUS_COLUMNS
    dense_input = Input(shape=(CONTINUE_SIZE,))
    merge_input = concatenate([dense_input, sparse_embedding], axis = 1)

    merge_len = CONTINUE_SIZE + EMBEDDING_SIZE * SPARSE_SIZE
    output = create_model(merge_len)(merge_input)

    model = Model([dense_input, sparse_feature], output)
    optimizer = RMSprop(lr=1e-3, rho = 0.9, epsilon = 1e-8)
    # optimizer = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(optimizer = Adam(),
            loss='categorical_crossentropy', metrics = ['accuracy'])

    return model


def keras_train(train_data, train_target, nfolds = 10):
    """
    Detect Fish or noFish
    """
    print("Start gen training data, shuffle and normalize!")

    #train_data = train
    train_target = np_utils.to_categorical(train_target)

    # train_data, train_target, siamese_data_loader = siamese_train(siamese_train_data, siamese_train_label)
    kf = KFold(len(train_target), n_folds=nfolds, shuffle=True)

    num_fold = 0
    models = []
    for train_index, test_index in kf:
        # model = create_model(classes = 2)
        model = create_embedding_model(len(CONTINUOUS_INDICES), len(SPARSE_INDICES))
        # model = create_siamese_net((train.shape)[1])

        X_train = train_data[train_index]
        Y_train = train_target[train_index]
        print('Positive samples in train: %d' % np.sum(Y_train))
        print('Negative samples in train: %d' % (len(Y_train) - np.sum(Y_train)))

        X_valid = train_data[test_index]
        Y_valid = train_target[test_index]
        print('Positive samples in valide: %d' % np.sum(Y_valid))
        print('Negative samples in valide: %d' % (len(Y_valid) - np.sum(Y_valid)))

        num_fold += 1
        print('Start KFold number {} from {}'.format(num_fold, nfolds))
        print('Split train: ', len(X_train), len(Y_train))
        print('Split valid: ', len(X_valid), len(Y_valid))

        callbacks = [
            EarlyStopping(monitor='val_loss', patience=5, verbose=0),
        ]
        model.fit([X_train[:, CONTINUOUS_INDICES], X_train[:, SPARSE_INDICES]],
                Y_train, batch_size=BATCH_SIZE, epochs=DNN_EPOCHS,
                shuffle=True, verbose=2,
                validation_data=([X_valid[:, CONTINUOUS_INDICES], X_valid[:, SPARSE_INDICES]], Y_valid)
                , callbacks=callbacks)

        model_name = 'keras' + strftime('_%Y_%m_%d_%H_%M_%S', gmtime())
        #model.save_weights(model_name)
        #siamese_features_array = gen_siamese_features(model, lgbm_train_data, siamese_train_data, siamese_train_label)
        models.append((model, 'k'))
        break

    return models #, siamese_features_array


def model_eval(model, model_type, data_frame):
    """
    """
    if model_type == 'l':
        preds = model.predict(data_frame)
    elif model_type == 'k':
        preds = model.predict(data_frame, batch_size=BATCH_SIZE, verbose=2)
    elif model_type == 't':
        print("ToDO")
    elif model_type == 'x':
        preds = model.predict(xgb.DMatrix(data_frame), ntree_limit=model.best_ntree_limit+80)

    return preds


def gen_sub(models, merge_features):
    """
    Evaluate single Type model
    """
    print('Start generate submission!')
    preds = None
    for (model, model_type) in models:
        pred = model_eval(model, model_type, merge_features)
        #print pred.shape
        #print pred[0, :]
        if preds is None:
            preds = pred.copy()
        else:
            preds += pred

    preds /= len(models)
    submission = pd.DataFrame(preds, columns=['class'+str(c+1) for c in range(9)])
    submission['ID'] = pid
    sub_name = "submission" + strftime('_%Y_%m_%d_%H_%M_%S', gmtime()) + ".csv"
    print('Output to ' + sub_name)
    submission.to_csv(sub_name, index=False)


def create_siamese_net(input_size):
    """
    """
    input_shape = (input_size, )
    left_input = Input(input_shape)
    right_input = Input(input_shape)
    #build model to use in each siamese 'leg'
    model = Sequential()
    model.add(Dense(HIDDEN_UNITS[0], activation='sigmoid', input_dim = input_size))
    if DNN_BN:
        model.add(BatchNormalization())
    if DROPOUT_RATE > 0:
        model.add(Dropout(DROPOUT_RATE))
    model.add(Dense(HIDDEN_UNITS[1], activation='sigmoid'))
    if DNN_BN:
        model.add(BatchNormalization())
    if DROPOUT_RATE > 0:
        model.add(Dropout(DROPOUT_RATE))
    #encode each of the two inputs into a vector with the convnet
    encoded_l = model(left_input)
    encoded_r = model(right_input)
    #merge two encoded inputs with the l1 distance between them
    L1_distance = lambda x: K.abs(x[0]-x[1])
    both = merge([encoded_l,encoded_r], mode = L1_distance, output_shape=lambda x: x[0])
    merge_layer = Dense(HIDDEN_UNITS[2],activation='sigmoid')(both)
    prediction = Dense(1,activation='sigmoid')(merge_layer)
    siamese_net = Model(input=[left_input,right_input],output=prediction)
    #optimizer = SGD(0.0004,momentum=0.6,nesterov=True,decay=0.0003)

    optimizer = Adam()
    #//TODO: get layerwise learning rates and momentum annealing scheme described in paperworking
    siamese_net.compile(loss="binary_crossentropy",optimizer=optimizer)

    # print siamese_net.count_params()
    return siamese_net


class Siamese_Loader:
    #For loading batches and testing tasks to a siamese net
    def __init__(self,Xtrain,Xval = None):
        self.Xval = Xval
        self.Xtrain = Xtrain
        self.n_classes = Xtrain.shape[0]
        self.feature_size = (Xtrain[0].shape)[1]
        self.n_examples = np.array([x.shape[0] for x in Xtrain])
        self.n_tot_examples = np.sum(self.n_examples)
        print('examples of different classes: %s' % str(self.n_examples))
        # self.n_val,self.n_ex_val,_,_ = Xval.shape

    def get_batch(self,n):
        #Create batch of pairs, half same class, half different class
        categories = rng.choice(self.n_classes,size=(n,),replace=True)
        pairs=np.zeros((2, n, self.feature_size))
        targets=np.zeros((n,))
        positive_begin_pos = n * 1 // 2
        targets[positive_begin_pos:] = 1
        categories_list = []
        for i in range(n):
            category = categories[i]
            idx_1 = rng.randint(0, self.n_examples[category])
            pairs[0][i] = self.Xtrain[category][idx_1] #.reshape(self.feature_size)
            #pick images of same class for 1st half, different for 2nd
            category_2 = category if i >= positive_begin_pos else (category + rng.randint(1,self.n_classes)) % self.n_classes
            idx_2 = rng.randint(0,self.n_examples[category_2])
            while i >= positive_begin_pos and idx_2 == idx_1:
                idx_2 = rng.randint(0,self.n_examples[category_2])
            pairs[1][i] = self.Xtrain[category_2][idx_2] #.reshape(self.w,self.h,1)
            categories_list.append((category, category_2))
        #pd.DataFrame(categories_list).to_csv('categories', index=False)
        #exit(0)
        # shuflle pairs to mix positive and negative
        rng.shuffle(pairs)
        return pairs, targets


def gen_test_on_support_data(Xsupport, Xtest):
    """
    """
    n_support, feature_size = Xsupport.shape
    pairs = np.zeros((2, n_support, feature_size))
    pairs[0] = Xtest
    pairs[1] = Xsupport
    return list(pairs)


def siamese_train(siamese_train_data, siamese_train_label):
    """
    """
    train_data = [[] for i in range(9)]
    label_ind = 0
    for feature in siamese_train_data:
        train_data[siamese_train_label[label_ind]].append(feature)
        label_ind += 1
    train_data = np.array([np.array(xi) for xi in train_data])
    print("train data shape before gen pair")
    print(train_data.shape)
    siamese_data_loader = Siamese_Loader(train_data, test)
    pairs, targets = siamese_data_loader.get_batch(SIAMESE_PAIR_SIZE)
    return pairs, targets, siamese_data_loader


def gen_siamese_features_meta(model, Xsupport_label, Xsupport, Xtest):
    """
    """
    siamese_pair = gen_test_on_support_data(Xsupport, Xtest)
    global graph
    with graph.as_default():
        preds = model.predict(siamese_pair, batch_size=BATCH_SIZE, verbose=2)
    preds = np.insert(preds, 1, Xsupport_label, axis = 1)
    preds = pd.DataFrame(preds, columns = ['sim', 'class'])
    siamese_features = preds.groupby('class', sort = False) \
            .agg({'sim': ['max', 'min', 'median', 'mean', 'std']})
    max_class = siamese_features['sim']['max'].idxmax()
    siamese_features = np.insert(siamese_features.values.flatten(), 0, max_class, axis = 0)
    return siamese_features

def gen_siamese_features(siamese_model, Xtest, Xsupport, Xsupport_label):
    """
    """
    if MAX_WORKERS <= 0:
        print("MAX_WORKERS should >= 1", file=sys.stderr)
        exit(1)
    siamese_features_array = list(range(len(Xtest)))
    test_begin = 0
    while test_begin < len(Xtest):
        test_end = min(test_begin + MAX_WORKERS, len(Xtest))
        with concurrent.futures.ThreadPoolExecutor(max_workers = MAX_WORKERS) as executor:
            future_predict = {executor.submit(gen_siamese_features_meta, siamese_model,
                        Xsupport_label,
                        Xsupport,
                        Xtest[ind]): ind for ind in range(test_begin, test_end)}
            for future in concurrent.futures.as_completed(future_predict):
                ind = future_predict[future]
                try:
                    siamese_features = future.result()
                    siamese_features_array[ind] = siamese_features
                except Exception as exc:
                    print('%dth feature generated an exception: %s' % (ind, exc))
        test_begin = test_end
        if test_begin % 100 == 0:
            print('Gen %d siamsese features' % test_begin)
    if test_begin != len(Xtest):
        print("Only gen %d siamese features" % test_begin, file=sys.stderr)
        exit(1)
    siamese_features_array = np.array(siamese_features_array)
    pd.DataFrame(siamese_features_array[:, 0]).astype(np.int8).to_csv('pred_label', index = False)
    return siamese_features_array


if __name__ == "__main__":
    model_k = keras_train(train, y, 10)
    #np.save("siamese_features_array" + \
    #        strftime('_%Y_%m_%d_%H_%M_%S', gmtime()) , siamese_features_array)
   #  gen_sub(model_k, 'k', th, F1)
    # ind = np.array([i * 5 for i in range(9)])
    # xgbTrain(siamese_features_array[:, ind], lgbm_train_label);
    #lgbm_features = siamese_features_array #np.concatenate((lgbm_train_data, siamese_features_array),
    # model_l = lgbm_train(train, y, 10) #lgbm_features, lgbm_train_label, 10)#model_k)
    # siamese_features_test_array = siamese_test(model_k[0][0], test)
    #np.save("siamese_features_test_array" + \
    #        strftime('_%Y_%m_%d_%H_%M_%S', gmtime()) , siamese_features_test_array)
    ##model_x = xgbTrain(5)#model_k)
    #gen_sub(model_l, siamese_features_test_array) #model_k)
