from __future__ import print_function

__docformat__ = 'restructedtext en'

import six.moves.cPickle as pickle
import gzip
import os

os.environ["THEANO_FLAGS"] = "floatX=float64"
import sys
import timeit
import numpy as np
from learning import getSeqEdvice
import theano
import theano.tensor as T
from theano.tensor.nlinalg import matrix_inverse
import matplotlib.animation as manimation
from mpl_toolkits import axes_grid1
from sklearn import datasets, linear_model
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import matplotlib
import time
import copy
import pickle
import os
import errno
import warnings

warnings.filterwarnings("ignore")

class HiddenLayer(object):
    def __init__(self, n_data, dim_in, dim_out, activation=T.tanh):
        self.n_data = n_data
        self.dim_in = dim_in
        self.dim_out = dim_out
        self.activation = activation

    def setParam(self, param_W, param_b):
        # beta=np.float32(param[0])
        # bias=np.float32(param[1])
        # DesignMatrix = np.concatenate((beta,bias.reshape((1,self.dim_out))),axis=0)
        # tmp_param = DesignMatrix.reshape(((self.dim_in+1)*self.dim_out,))
        # self.param = theano.shared(
        # value=tmp_param,
        # name='param',
        # borrow=True
        # )
        # tmp = self.param.reshape(((self.dim_in+1, self.dim_out)))
        # self.W = tmp[:self.dim_in,]
        # self.b = tmp[-1,]
        self.W = param_W
        self.b = param_b

    def setData(self, input, ifData):
        if (ifData):
            self.input = theano.shared(
                value=input,
                name='input',
                borrow=True
            )
        else:
            self.input = input
        # self.output = theano.shared(
        # value=np.zeros((self.n_data,self.dim_out),dtype=theano.config.floatX),
        # name='output',
        # borrow=True
        # )

    def setup(self, param_W, param_b, input, ifData):
        self.setParam(param_W, param_b)
        self.setData(input, ifData)
        lin_output = T.dot(self.input, self.W) + self.b
        # self.output.set_value(lin_output if activation is None else activation(lin_output))
        self.output = lin_output if self.activation is None else self.activation(lin_output)

class LogisticRegression(object):

    def __init__(self, n_data, dim_in, dim_out):
        self.dim_in = dim_in
        self.dim_out = dim_out
        self.n_data = n_data

    def setParam(self, param_W, param_b):
        # beta=np.float32(param[0])
        # bias=np.float32(param[1])
        # DesignMatrix = np.concatenate((beta,bias.reshape((1,self.dim_out))),axis=0)
        # tmp_param = DesignMatrix.reshape(((self.dim_in+1)*self.dim_out,))
        # self.param = theano.shared(
        # value=tmp_param,
        # name='param',
        # borrow=True
        # )
        # tmp = self.param.reshape(((self.dim_in+1, self.dim_out)))
        # self.W = tmp[:self.dim_in,]
        # self.b = tmp[-1,]
        self.W = param_W
        self.b = param_b

    def setData(self, input, dataY):
        self.input = input
        self.output = theano.shared(
            value=dataY,
            name='output',
            borrow=True
        )

    def setup(self, param_W, param_b, input, dataY):
        self.setParam(param_W, param_b)
        self.setData(input, dataY)
        self.p_y_given_x = T.nnet.sigmoid(T.dot(self.input, self.W) + self.b)
        # print(self.p_y_given_x.eval())

    def negative_log_likelihood_vector(self):
        if (self.dim_out == 1):
            tmp1 = T.concatenate([(1 - self.p_y_given_x), (self.p_y_given_x)], axis=1)[
                T.arange(self.n_data), self.output]
            # print(T.log(tmp1).eval())
            # tmp1[tmp1==0.0] = np.nextafter(0, 1)
            tmp = T.log(tmp1)
        else:
            tmp1 = self.p_y_given_x[T.arange(self.n_data), self.output]
            # print(tmp1.eval())
            # tmp1[tmp1==0.0] = np.nextafter(0, 1)
            tmp = T.log(tmp1)
        return -tmp

    def negative_log_likelihood_mean(self):
        if (self.dim_out == 1):
            tmp1 = T.concatenate([(1 - self.p_y_given_x), (self.p_y_given_x)], axis=1)[
                T.arange(self.n_data), self.output]
            # print(tmp1.eval())
            # tmp1[tmp1==0.0] = np.nextafter(0, 1)
            tmp = T.log(tmp1)
        else:
            tmp1 = self.p_y_given_x[T.arange(self.n_data), self.output]
            # print(tmp1.eval())
            # tmp1[tmp1==0.0] = np.nextafter(0, 1)
            tmp = T.log(tmp1)
            # print(tmp.shape.eval())
        return -T.mean(tmp)

class MLP(object):

    def __init__(self, n_data, dim_in, dim_hidden, dim_out):
        self.n_hidden = len(dim_hidden)
        self.n_data = n_data
        self.hiddenlayers = []
        self.numtotalparam = 0
        for i in range(self.n_hidden):
            if (i == 0):
                cur_dim_in = dim_in + 1
            else:
                cur_dim_in = dim_hidden[i - 1] + 1
            self.numtotalparam = self.numtotalparam + cur_dim_in * dim_hidden[i]
            self.hiddenlayers.append(HiddenLayer(
                n_data=n_data,
                dim_in=cur_dim_in,
                dim_out=dim_hidden[i],
                activation=T.tanh
            ))
        self.numtotalparam = self.numtotalparam + (dim_hidden[-1] + 1) * dim_out
        self.logRegressionLayer = LogisticRegression(
            n_data=n_data,
            dim_in=dim_hidden[-1] + 1,
            dim_out=dim_out
        )

    def setup(self, param, dataX, dataY):
        # print(dataY.shape)
        tmp_param = np.zeros(self.numtotalparam)
        pivot = 0
        tmp_DesignMatrixshape = []
        tmp_numparam = np.zeros(len(param), dtype=np.int)
        for i in range(len(param)):
            tmp_beta = param[0][i]
            tmp_bias = param[1][i]
            DesignMatrix = np.concatenate((tmp_beta, tmp_bias.reshape((1, tmp_bias.shape[0]))), axis=0)
            tmp_DesignMatrixshape.append(DesignMatrix.shape)
            tmp_numparam[i] = tmp_DesignMatrixshape[i][0] * tmp_DesignMatrixshape[i][1]
            tmp_param[pivot:(pivot + tmp_numparam[i])] = DesignMatrix.reshape((tmp_numparam[i],))
            pivot = pivot + tmp_numparam[i]

        # print(tmp_param)
        self.param = theano.shared(
            value=tmp_param,
            name='param',
            borrow=True
        )
        pivot = 0
        cur_input = dataX
        for i in range(len(param)):
            tmp = self.param[pivot:(pivot + tmp_numparam[i])].reshape(
                (tmp_DesignMatrixshape[i][0], tmp_DesignMatrixshape[i][1]))
            pivot = pivot + tmp_numparam[i]
            # print(tmp_DesignMatrixshape[i][0]-1)
            if (i == 0):
                self.hiddenlayers[i].setup(tmp[:tmp_DesignMatrixshape[i][0] - 1, ], tmp[-1,], cur_input, True)
                cur_input = self.hiddenlayers[i].output
            elif (i < (len(param) - 1)):
                self.hiddenlayers[i].setup(tmp[:tmp_DesignMatrixshape[i][0] - 1, ], tmp[-1,], cur_input, False)
                cur_input = self.hiddenlayers[i].output
            else:
                self.logRegressionLayer.setup(tmp[:tmp_DesignMatrixshape[i][0] - 1, ], tmp[-1,], cur_input, dataY)

    def negative_log_likelihood_vector(self):
        return (self.logRegressionLayer.negative_log_likelihood_vector())

    def negative_log_likelihood_mean(self):
        return (self.logRegressionLayer.negative_log_likelihood_mean())

    def GTIC_gradient(self):
        NLLV = self.negative_log_likelihood_vector()
        O = T.dmatrix('O')
        o = np.zeros((self.numtotalparam, self.numtotalparam))
        results, updates = theano.scan(lambda i, tmp: T.dot(T.grad(NLLV[i], self.param).reshape((-1, 1)),
                                                            T.grad(NLLV[i], self.param).reshape((-1, 1)).T) + tmp,
                                       sequences=[np.arange(self.n_data)],
                                       outputs_info=[O])
        result = results[-1]
        compute = theano.function([O], outputs=result)
        out = compute(o) / self.n_data
        return out

    def GTIC_hessian(self):
        NLLM = self.negative_log_likelihood_mean()
        # print(NLLM.eval())
        H_theta = theano.gradient.hessian(NLLM, self.param)
        # print(H_theta.eval())
        return H_theta

    def GTIC(self):
        # print(self.GTIC_hessian().eval())
        # your code that will (maybe) throw
        inverse = T.nlinalg.pinv(self.GTIC_hessian())
        tmp = T.nlinalg.trace(T.dot(inverse, self.GTIC_gradient())) / self.n_data
        return tmp

def sigmoid(t):  # Define the sigmoid function
    return (1 / (1 + np.exp(-t)))

def getLoss(X, y, mlp):
    P = mlp.predict_proba(X)
    n = X.shape[0]
    loglik = np.mean(np.log(P[np.arange(n), y]))
    return -loglik

def getAIC(n, k):
    return k / n

def getBIC(n, k):
    return (k * np.log(n)) / (2 * n)

def getDIC_1(n, k_AIC, Mn):
    adjust = 0
    for i in range(k_AIC):
        adjust = adjust + 1 / (i + 1)
    return Mn / n * adjust

def getDIC_2(n, k, kmax):
    adjust = 0
    for i in range(k):
        adjust = adjust + 1 / (i + 1)
    return kmax / n * adjust

def calculate_Loss(candModels, X_test, y_test):
    y_test = y_test[:, 0]
    N = len(candModels)
    loss = np.zeros((N, 1))
    dataSize = X_test.shape[0]
    for n in range(N):
        mlp = candModels[n]['mlp']
        # plot_decision_boundary(lambda x: mlp.predict(x), X_test, y_test)
        P = mlp.predict_proba(X_test)
        tmp = P[np.arange(dataSize), y_test]
        tmp[tmp == 0.0] = np.nextafter(0, 1)
        # plt.hist(tmp,bins=50)
        # plt.show()
        loss[n] = -np.mean(np.log(tmp))
    return loss

def generate_data(dataSize, seed):
    # np.random.seed(seed)
    # X, y = datasets.make_moons(dataSize, noise=0.20)
    X, y = datasets.make_circles(n_samples=dataSize, shuffle=True, noise=0.2, random_state=None, factor=0.8)
    y = y.reshape((-1, 1))
    return X[:, :2], y[:, :2]

def NeuralNetworkFit(candModels, Xt, yt, timing):
    GTICcompiletiming = 0
    N = len(candModels)
    kmax = 0
    Loss_AIC = np.inf
    for n in range(N):
        print(n)
        mlp_cv = MLPClassifier(hidden_layer_sizes=((n + 1),), activation='tanh', solver='lbfgs')

        if (n < np.floor((np.sqrt(Xt.shape[0]) / (Xt.shape[1] + 1)) - 1).astype(np.int) or n < 5):
            # Holdout
            start = time.time()
            X_train, X_test, y_train, y_test = crossvalition_p(Xt, yt, 0.7)
            mlp_cv.fit(X_train, y_train)
            Loss = getLoss(X_test, y_test, mlp_cv)
            candModels[n]['Holdout'] = Loss
            end = time.time()
            timing['Holdout'] = timing['Holdout'] + end - start
            # CV_10fold
            start = time.time()
            X_train, X_test, y_train, y_test = crossvalition_10fold(Xt, yt)
            Loss = np.zeros(len(X_train))
            for i in range(len(X_train)):
                mlp_cv.fit(X_train[i], y_train[i])
                Loss[i] = getLoss(X_test[i], y_test[i], mlp_cv)
            candModels[n]['CV_10fold'] = np.mean(Loss)
            end = time.time()
            timing['CV_10fold'] = timing['CV_10fold'] + end - start
            # CV_loo
            start = time.time()
            X_train, X_test, y_train, y_test = crossvalition_loo(Xt, yt)
            Loss = np.zeros(len(X_train))
            for i in range(len(X_train)):
                mlp_cv.fit(X_train[i], y_train[i])
                Loss[i] = getLoss(X_test[i], y_test[i], mlp_cv)
            candModels[n]['CV_loo'] = np.mean(Loss)
            end = time.time()
            timing['CV_loo'] = timing['CV_loo'] + end - start

        mlp = MLPClassifier(hidden_layer_sizes=((n + 1),), activation='tanh', solver='lbfgs')
        start = time.time()
        mlp.fit(Xt, yt)
        Loss = getLoss(Xt, yt, mlp)
        candModels[n]['mlp'] = mlp
        dataSize = Xt.shape[0]
        dim = (Xt.shape[1] + 1) * ((n + 1) + 1)
        end = time.time()
        time_loss = end - start

        if (n < np.floor((np.sqrt(Xt.shape[0]) / (Xt.shape[1] + 1)) - 1).astype(np.int) or n < 5):
            start_AIC = time.time()
            AIC = getAIC(dataSize, dim)
            candModels[n]['AIC'] = Loss + AIC
            end_AIC = time.time()
            timing['AIC'] = timing['AIC'] + time_loss + end_AIC - start_AIC

            start_BIC = time.time()
            BIC = getBIC(dataSize, dim)
            candModels[n]['BIC'] = Loss + BIC
            end_BIC = time.time()
            timing['BIC'] = timing['BIC'] + time_loss + end_BIC - start_BIC

            start_DIC_2 = time.time()
            DIC_2 = getDIC_2(dataSize, dim, dataSize ** (1 / 2))
            candModels[n]['DIC_2'] = Loss + DIC_2
            end_DIC_2 = time.time()
            timing['DIC_2'] = timing['DIC_2'] + time_loss + end_DIC_2 - start_DIC_2

            start_GTIC_compile = time.time()
            beta = mlp.coefs_
            bias = mlp.intercepts_
            dataX = Xt
            dataY = yt
            GTICKernel = MLP(n_data=dataX.shape[0], dim_in=dataX.shape[1], dim_hidden=[(n + 1)], dim_out=1)
            GTICKernel.setup([beta, bias], dataX, dataY)
            # loss = GTICKernel.negative_log_likelihood_mean()
            tic = GTICKernel.GTIC()
            GTIC_model = theano.function(
                inputs=[],
                outputs=tic,
            )
            end_GTIC_compile = time.time()
            GTICcompiletiming = GTICcompiletiming + end_GTIC_compile - start_GTIC_compile
            # print(GTIC_compile_timing)

            # print(Loss)
            # a=np.tanh(dataX.dot(beta[0])+bias[0]).dot(beta[1])+bias[1]
            # print(-(np.sum(a[dataY==1]) - np.sum(np.log(1+np.exp(a))))/dataX.shape[0])
            start_GTIC = time.time()
            GTIC = GTIC_model()
            # print(Loss)
            # print(GTIC)
            candModels[n]['GTIC'] = Loss + GTIC
            print(candModels[n]['GTIC'])
            end_GTIC = time.time()
            timing['GTIC'] = timing['GTIC'] + time_loss + end_GTIC - start_GTIC

            start_DIC_1 = time.time()
            if (candModels[n]['AIC'] < Loss_AIC):
                Loss_AIC = candModels[n]['AIC']
                kmax = n
            end_DIC_1 = time.time()
            timing['DIC_1'] = timing['DIC_1'] + time_loss + end_DIC_1 - start_DIC_1

    start_DIC_1 = time.time()
    for n in range(N):
        if (n < kmax):
            dataSize = Xt.shape[0]
            dim = Xt.shape[1] * ((n + 1) + 1)
            DIC_1 = getDIC_1(dataSize, dim, dataSize ** (1 / 3))
            candModels[n]['DIC_1'] = Loss + DIC_1
        else:
            candModels[n]['DIC_1'] = 0
    end_DIC_1 = time.time()
    timing['DIC_1'] = timing['DIC_1'] + end_DIC_1 - start_DIC_1

    return candModels, timing, GTICcompiletiming

def runLogisticExper():
    # %% pre-def the observation up to time T
    T = 301
    # dT = np.floor(T/2).astype(int) #largest candidate model considered
    # dT = np.sqrt(T).astype(int)
    dT = 20
    K = 3  # num of active models we maintain in sequential learning
    if dT < K:
        print('error in dT specification!')
    t_start = 101  # starting data size

    nJumpExpe = 1 * np.sqrt(T) / K  # expected #jump
    learningParams = {'eta': 1 * np.sqrt(8 * nJumpExpe / T), 'alpha': 2 * nJumpExpe / T,
                      'G': None}  # will use getSeqEdvice()
    actSet, move, thresh = range(K), False, 1 / K  # input to learning
    subW = np.zeros((K))
    subW[0] = 1
    N = 10000

    # pre-def the benchmark (testdata) to compute the expected loss
    X_test, y_test = generate_data(N, 100)  # testing data for loss computation
    X, y = generate_data(T, 0)  # all the training data
    # print(X.shape)

    # pre-def the fixed candidates
    candModels_fix = []
    for t in range(dT):
        candModels_fix.append({'mlp': None, 'AIC': None, 'BIC': None, 'DIC_1': None, 'DIC_2': None, 'GTIC': None, \
                               'Holdout': None, 'CV_10fold': None, 'CV_loo': None})
    L = -np.inf * np.ones((dT, T))  # store all the loss (each col) at each time
    candModels_time = []
    timing_image = {'AIC': [], 'BIC': [], 'DIC_1': [], 'DIC_2': [], 'GTIC': [], \
                    'Holdout': [], 'CV_10fold': [], 'CV_loo': []}
    # sequential procedure -- compute the loss for all candidates
    timing = {'AIC': 0, 'BIC': 0, 'DIC_1': 0, 'DIC_2': 0, 'GTIC': 0, \
              'Holdout': 0, 'CV_10fold': 0, 'CV_loo': 0}
    for t in range(t_start - 1, T):  # for each sample size
        print(t)
        Xt = X[range(t), :]
        yt = y[range(t), 0]
        candModels_fix, timing, GTICcompiletiming = NeuralNetworkFit(candModels_fix, Xt, yt, timing)
        # print(GTIC_compile_timing)
        for k in timing_image.keys():
            if (k == 'GTIC'):
                timing_image[k].append(timing[k] + GTICcompiletiming)
            else:
                timing_image[k].append(timing[k])
        # compute the loss matrix
        L[:, t] = np.squeeze(calculate_Loss(candModels_fix, X_test, y_test))
        print(L[:, t])
        for t in range(dT):
            candModels_time.append(copy.deepcopy(candModels_fix))
        # print(candModels_time[t-t_start+1][4]['GTIC'])
    print(L[:, 20])
    print(L[:, T - 20])
    L[L == -np.inf] = np.max(L)  # the none are set to be the max loss

    # sequential procedure -- compute the est loss and use graph-based learning
    actSet_start, actSet_end = np.zeros((T)), np.zeros((T))
    W_hy = np.zeros((dT, T))
    # seqPredLoss = np.zeros((T))
    # loss_ratio =  np.zeros((1,T))

    output_filename = './output/neuralnet/output.pickle'
    # mode = ['AIC','BIC','DIC_1','DIC_2','Holdout','CV_10fold','CV_loo','GTIC']
    # mode = ['AIC','BIC','DIC_1','DIC_2','GTIC','Holdout','CV_10fold','CV_loo']
    mode = ['GTIC', 'Holdout', 'CV_10fold', 'CV_loo']
    # mode = ['GTIC','AIC','BIC','DIC_1']
    # mode = ['GTIC','Holdout','AIC','BIC']
    # mode = ['Holdout','AIC','BIC']
    candModels_Sequential = []
    for i in range(len(mode)):
        candModels_Sequential.append(
            {'learningParams': learningParams, 'actSet': actSet, 'actSet_start': actSet_start, 'actSet_end': actSet_end,
             'move': move, 'nummove': 0, 'thresh': thresh, \
             'subW': subW, 'W_hy': W_hy, 'seqPredLoss': np.zeros((T)), 'loss_ratio': np.zeros((T)),
             'batch_opt_model': np.zeros((T), dtype=np.int), 'batch_opt_loss': np.zeros((T)) + np.inf,
             'batch_loss_ratio': np.zeros((T))})

    with open(output_filename, 'wb') as f:  # Python 3: open(..., 'wb')
        pickle.dump([candModels_Sequential, candModels_time, L, mode, t_start, T, timing_image], f)

    for t in range(t_start - 1, T):  # for each sample size
        # cur_dT = 10
        cur_dT = np.floor((np.sqrt(t) / (X.shape[1] + 1)) - 1).astype(np.int)
        seq_dT = K if (cur_dT < K) else cur_dT
        # print(t)
        # print(candModels_time[t-t_start+1][4]['GTIC'])
        for m in range(len(mode)):
            candModels_Sequential[m]['seqPredLoss'][t] = np.sum(candModels_Sequential[m]['subW'] * L[
                candModels_Sequential[m]['actSet'], t])  # masterE is wrong! should be real loss numerically computed
            if t % 10 == 0:
                print("At iteration t = ", t)
            if candModels_Sequential[m]['move']:
                candModels_Sequential[m]['actSet'] = [(x + 1) for x in candModels_Sequential[m]['actSet']]
                candModels_Sequential[m]['nummove'] = candModels_Sequential[m]['nummove'] + 1
                movedsample = t / candModels_Sequential[m]['nummove']
                tmp_nJumpExpe = movedsample / K
                candModels_Sequential[m]['learningParams']['alpha'] = 1 * tmp_nJumpExpe / T
                candModels_Sequential[m]['learningParams']['eta'] = 1 * np.sqrt(8 * tmp_nJumpExpe / T)
                print(candModels_Sequential[m]['learningParams']['alpha'])
                if max(candModels_Sequential[m]['actSet']) >= seq_dT:
                    candModels_Sequential[m]['actSet'] = range(seq_dT - K, seq_dT)
                    candModels_Sequential[m]['move'] = False
            candModels_Sequential[m]['actSet_start'][t], candModels_Sequential[m]['actSet_end'][t] = min(
                candModels_Sequential[m]['actSet']), max(candModels_Sequential[m]['actSet'])
            subE = np.array(
                [(candModels_time[t - t_start + 1][j][mode[m]]) for j in candModels_Sequential[m]['actSet']]).reshape(
                K, )
            subE = np.array([1 if x is None else x for x in subE])
            candModels_Sequential[m]['subW'], masterE, candModels_Sequential[m]['move'] = getSeqEdvice(subE,
                                                                                candModels_Sequential[
                                                                                m]['subW'],
                                                                                candModels_Sequential[
                                                                                m][
                                                                                'learningParams'], \
                                                                                candModels_Sequential[
                                                                                m]['move'],
                                                                                candModels_Sequential[
                                                                                m]['thresh'],
                                                                                t)
            candModels_Sequential[m]['W_hy'][candModels_Sequential[m]['actSet'], t] = candModels_Sequential[m]['subW']
            weight = candModels_Sequential[m]['subW']
            # candModels_Sequential[m]['loss_ratio'][t] = np.sum(L[candModels_Sequential[m]['actSet'],
            # t]*weight)/np.min(L[:,t])
            candModels_Sequential[m]['loss_ratio'][t] = L[np.argmax(candModels_Sequential[m]['W_hy'][:, t],
                                                                    axis=0), t] / np.min(L[:, t])
            # print(mode[m])
            for l in range(cur_dT):
                # print(candModels_time[t-t_start+1][l][mode[m]])
                if ((candModels_time[t - t_start + 1][l][mode[m]] is not None) and (
                        candModels_time[t - t_start + 1][l][mode[m]] < candModels_Sequential[m]['batch_opt_loss'][t])):
                    candModels_Sequential[m]['batch_opt_model'][t] = l
                    candModels_Sequential[m]['batch_opt_loss'][t] = candModels_time[t - t_start + 1][l][mode[m]]
            # print(candModels_Sequential[m]['batch_opt_loss'][t])
            candModels_Sequential[m]['batch_loss_ratio'][t] = L[candModels_Sequential[m]['batch_opt_model'][
                                                                    t], t] / np.min(L[:, t])

    # summarize results
    viewLoss(np.log(L), candModels_Sequential[mode.index('GTIC')]['actSet_start'],
             candModels_Sequential[mode.index('GTIC')]['actSet_end'])
    viewSeqWeight(candModels_Sequential[mode.index('GTIC')]['W_hy'], L)  # print subW
    # viewSeqLoss_all(mode, candModels_Sequential, L, t_start)
    # viewBatchLoss_all(mode, candModels_Sequential, L, t_start)
    # viewLossRatio_all(mode, candModels_Sequential, t_start, T)
    # viewBatchLossRatio_all(mode, candModels_Sequential, t_start, T)
    viewSeqBatchLossRatio(mode, candModels_Sequential, t_start, T)
    viewTiming(mode, timing_image, t_start, T)
    # Saving the objects:
    with open(output_filename, 'wb') as f:  # Python 3: open(..., 'wb')
        pickle.dump([candModels_Sequential, candModels_time, L, mode, t_start, T, timing_image], f)

# Getting back the objects:
# with open('objs.pickle') as f:  # Python 3: open(..., 'rb')
# obj0, obj1, obj2 = pickle.load(f)

#########################################################################################################################################################################################################
def crossvalition_p(X, y, p):
    dataSize = X.shape[0]
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=(1 - p), random_state=40)
    return (X_train, X_test, y_train, y_test)

def crossvalition_10fold(X, y):
    tenfold = KFold(n_splits=10)
    X_train = []
    y_train = []
    X_test = []
    y_test = []
    for train_index, test_index in tenfold.split(X, y):
        X_train.append(X[train_index, :])
        y_train.append(y[train_index])
        X_test.append(X[test_index, :])
        y_test.append(y[test_index])
    return (X_train, X_test, y_train, y_test)

def crossvalition_loo(X, y):
    loo = KFold(n_splits=X.shape[0])
    X_train = []
    y_train = []
    X_test = []
    y_test = []
    for train_index, test_index in loo.split(X, y):
        X_train.append(X[train_index, :])
        y_train.append(y[train_index])
        X_test.append(X[test_index])
        y_test.append(y[test_index])
    # print(X_train.shape)
    # print(X_test[0].shape)
    # print(X.shape)
    return (X_train, X_test, y_train, y_test)

#########################################################################################################################################################################################################
def add_colorbar(im, aspect=20, pad_fraction=0.5, **kwargs):
    """Add a vertical color bar to an image plot."""
    divider = axes_grid1.make_axes_locatable(im.axes)
    width = axes_grid1.axes_size.AxesY(im.axes, aspect=1 / aspect)
    pad = axes_grid1.axes_size.Fraction(pad_fraction, width)
    current_ax = plt.gca()
    cax = divider.append_axes("right", size=width, pad=pad)
    plt.sca(current_ax)
    return im.axes.figure.colorbar(im, cax=cax, **kwargs)

def viewLoss(L_transformed, actSet_start, actSet_end):
    nCandi, T = L_transformed.shape
    optModelIndex = np.argmin(L_transformed, axis=0)  # the first occurrence are returned.
    plt.figure(figsize=(10, 10), dpi=300, facecolor='w', edgecolor='k')
    bg = plt.imshow(L_transformed, cmap=plt.cm.Spectral)  # smaller the better
    # plt.colorbar()
    add_colorbar(bg)
    # plot along the active sets
    plt.scatter(range(T), optModelIndex, marker='o', color='k', s=1)
    #    plt.scatter(range(T), actSet_start, marker='x', color='b', s=30)
    #    plt.scatter(range(T), actSet_end, marker='x', color='b', s=30)
    plt.xlim(0, T - 1)
    plt.xlabel('Data Size', fontsize=10, color='black')
    plt.ylim(0, nCandi - 1)
    plt.ylabel('Model Complexity', fontsize=10, color='black')
    plt.title('Predictive Loss (in log)', fontsize=10)
    # plt.tight_layout()
    plt.savefig('./output/neuralnet/loss.png', bbox_inches='tight')
    # plt.show()

def videoLoss(L_transformed, actSet_start, actSet_end):
    FFMpegWriter = manimation.writers['ffmpeg']
    metadata = dict(title='Loss', artist='VGROUP', comment='Loss')
    writer = FFMpegWriter(fps=15, metadata=metadata)
    nCandi, T = L_transformed.shape
    tmp_L_transformed = np.zeros((nCandi, T)) * np.nan
    optModelIndex = np.argmin(L_transformed, axis=0)  # the first occurrence are returned.
    # tmp_optModelIndex = np.zeros(len(optModelIndex)))* np.nan
    # fig=plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    fig = plt.figure(facecolor='w', edgecolor='k')
    # plt.imshow(L_transformed, cmap=plt.cm.Spectral)  #smaller the better
    bg = plt.imshow(L_transformed, cmap=plt.cm.Spectral)
    # divider = make_axes_locatable(fig)
    # cax = divider.append_axes("right", size="5%", pad=0.05)
    add_colorbar(bg)

    # plot along the active sets
    # plt.scatter(range(T), optModelIndex, marker='o', color='k', s=30)
    dots = plt.scatter(range(T), optModelIndex, marker='o', color='k', s=10)
    #    plt.scatter(range(T), actSet_start, marker='x', color='b', s=30)
    #    plt.scatter(range(T), actSet_end, marker='x', color='b', s=30)
    plt.xlim(0, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    plt.ylim(0, nCandi)
    plt.ylabel('Model Complexity', fontsize=14, color='black')
    plt.title('Predictive Loss (in log)')
    with writer.saving(fig, "Loss.mp4", T):
        for i in range(T):
            tmp_L_transformed[:, i] = L_transformed[:, i]
            bg.set_array(tmp_L_transformed)
            x = range(i + 1)
            y = optModelIndex[:i + 1]
            offsets = [[x[j], y[j]] for j in range(i + 1)]
            dots.set_offsets(offsets)
            writer.grab_frame()

def viewSeqWeight(W_hy, L):
    dT, T = W_hy.shape
    plt.figure(figsize=(10, 10), dpi=300, facecolor='w', edgecolor='k')
    bg = plt.imshow(W_hy, cmap='hot')  # smaller the better
    # plt.colorbar()
    add_colorbar(bg)
    plt.xlim(0, T - 1)
    plt.xlabel('Data Size', fontsize=10, color='black')
    plt.ylim(0, dT - 1)
    plt.ylabel('Model Complexity', fontsize=10, color='black')
    plt.title('Learning Weight', fontsize=10)

    # plot along the true best model
    optModelIndex = np.argmin(L, axis=0)
    plt.scatter(range(T), optModelIndex, marker='o', color='b', s=5)
    # plt.tight_layout()
    plt.savefig('./output/neuralnet/weight.png', bbox_inches='tight')
    # plt.show()

def videoSeqWeight(W_hy, L):
    FFMpegWriter = manimation.writers['ffmpeg']
    metadata = dict(title='Loss', artist='VGROUP', comment='Loss')
    writer = FFMpegWriter(fps=15, metadata=metadata)
    dT, T = W_hy.shape
    tmp_W_hy = np.zeros((dT, T)) * np.nan
    optModelIndex = np.argmin(L, axis=0)
    fig = plt.figure(facecolor='w', edgecolor='k')
    bg = plt.imshow(W_hy, cmap='hot')  # smaller the better
    add_colorbar(bg)
    dots = plt.scatter(range(T), optModelIndex, marker='o', color='b', s=5)
    plt.xlim(0, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    plt.ylim(0, dT)
    plt.ylabel('Model Complexity', fontsize=14, color='black')
    plt.title('Learning Weight')
    # plot along the true best model
    with writer.saving(fig, "SeqWeight.mp4", T):
        for i in range(T):
            tmp_W_hy[:, i] = W_hy[:, i]
            bg.set_array(tmp_W_hy)
            x = range(i + 1)
            y = optModelIndex[:i + 1]
            offsets = [[x[j], y[j]] for j in range(i + 1)]
            dots.set_offsets(offsets)
            writer.grab_frame()

def viewSeqLoss(predL_transformed, L_transformed, t_start):
    dT, T = L_transformed.shape
    plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    plt.plot(range(t_start - 1, T), np.min(L_transformed[:, t_start - 1:T], axis=0), 'k-', label='Optimum',
             linewidth=3)  # smaller the better
    plt.plot(range(t_start - 1, T), predL_transformed[t_start - 1:T], 'b-', label='Predictor',
             linewidth=3)  # smaller the better
    plt.xlim(t_start - 1, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    #    plt.ylim(0,dT)
    plt.ylabel('Loss', fontsize=14, color='black')
    plt.title('Loss of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.show()

def viewSeqLoss_all(mode, candModels_Sequential, L_transformed, t_start):
    dT, T = L_transformed.shape
    plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    plt.plot(range(t_start - 1, T), np.min(L_transformed[:, t_start - 1:T], axis=0), 'k-', label='Optimum',
             linewidth=3)  # smaller the better
    colors = matplotlib.cm.rainbow(np.linspace(0, 1, len(mode)))
    linestyles = ['-', '--', '-.', ':']
    for i in range(len(mode)):
        plt.plot(range(t_start - 1, T), candModels_Sequential[i]['seqPredLoss'][t_start - 1:T], color=colors[i],
                 label=mode[i], linewidth=1, linestyle=linestyles[i])  # smaller the better
    plt.xlim(t_start - 1, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    #    plt.ylim(0,dT)
    plt.ylabel('Loss', fontsize=14, color='black')
    plt.title('Loss of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.show()

def viewBatchLoss_all(mode, candModels_Sequential, L_transformed, t_start):
    dT, T = L_transformed.shape
    plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    plt.plot(range(t_start - 1, T), np.min(L_transformed[:, t_start - 1:T], axis=0), 'k-', label='Optimum',
             linewidth=3)  # smaller the better
    colors = matplotlib.cm.rainbow(np.linspace(0, 1, len(mode)))
    linestyles = ['-', '--', '-.', ':']
    for i in range(len(mode)):
        rows = candModels_Sequential[i]['batch_opt_model'][t_start - 1:T].tolist()
        cols = range(t_start - 1, T)
        # print(L_transformed[rows,cols])
        plt.plot(range(t_start - 1, T), L_transformed[rows, cols], color=colors[i], label=mode[i], linewidth=1,
                 linestyle=linestyles[i])  # smaller the better
    plt.xlim(t_start - 1, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    plt.ylabel('Batch Loss', fontsize=14, color='black')
    plt.title('Batch Loss of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.show()

    plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    optModelIndex = np.argmin(L_transformed, axis=0)
    plt.scatter(range(T), optModelIndex, marker='o', color='k', s=30)
    colors = matplotlib.cm.rainbow(np.linspace(0, 1, len(mode)))
    markers = ['s', '*', '^', 'D']
    for i in range(len(mode)):
        plt.scatter(range(t_start - 1, T), candModels_Sequential[i]['batch_opt_model'][t_start - 1:T],
                    marker=markers[i], color=colors[i], label=mode[i], s=30)
    plt.xlim(0, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    plt.ylim(0, dT)
    plt.ylabel('Batch Model', fontsize=14, color='black')
    plt.title('Batch Model of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.show()

def videoSeqLoss(predL_transformed, L_transformed, t_start):
    FFMpegWriter = manimation.writers['ffmpeg']
    metadata = dict(title='Loss', artist='VGROUP', comment='Loss')
    writer = FFMpegWriter(fps=15, metadata=metadata)
    dT, T = L_transformed.shape
    fig = plt.figure(facecolor='w', edgecolor='k')
    # line1=plt.plot(range(t_start-1, T), np.min(L_transformed[:,t_start-1:T], axis=0), 'k-', label='Optimum',
    # linewidth=3)  #smaller the better
    # line2=plt.plot(range(t_start-1, T), predL_transformed[t_start-1:T], 'b-', label='Predictor', linewidth=3)
    # #smaller the better
    line1, = plt.plot(range(t_start - 1, T), np.min(L_transformed[:, t_start - 1:T], axis=0), 'k-', label='Optimum',
                      linewidth=3)  # smaller the better
    line2, = plt.plot(range(t_start - 1, T), predL_transformed[t_start - 1:T], 'b-', label='Predictor',
                      linewidth=3)  # smaller the better
    plt.xlim(t_start - 1, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    #    plt.ylim(0,dT)
    plt.ylabel('Loss', fontsize=14, color='black')
    plt.title('Loss of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    with writer.saving(fig, "SeqLoss.mp4", T):
        for i in range(t_start - 1, T):
            line1.set_data(range(t_start - 1, i + 1), np.min(L_transformed[:, t_start - 1:i + 1], axis=0))
            line2.set_data(range(t_start - 1, i + 1), predL_transformed[t_start - 1:i + 1])
            writer.grab_frame()

def viewSeqBatchLossRatio(mode, candModels_Sequential, t_start, T):
    colors = matplotlib.cm.rainbow(np.linspace(0, 1, len(mode)))
    plt.figure(figsize=(10, 10), dpi=300, facecolor='w', edgecolor='k')
    linestyles = ['--', '--', '-.', ':']
    for i in range(len(mode)):
        plt.plot(range(t_start - 1, T), candModels_Sequential[i]['batch_loss_ratio'][t_start - 1:T], color=colors[i],
                 label=mode[i], linewidth=1, linestyle=linestyles[i])
    plt.plot(range(t_start - 1, T), candModels_Sequential[0]['loss_ratio'][t_start - 1:T], color='k',
             label='GTIC_Sequential', linewidth=1, linestyle='--')
    plt.xlim(t_start - 1, T - 1)
    plt.xlabel('Data Size', fontsize=10, color='black')
    #    plt.ylim(0,dT)
    plt.ylabel('Loss Ratio', fontsize=10, color='black')
    plt.title('Loss ratio of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.tight_layout()
    plt.grid()
    plt.savefig('./output/neuralnet/lossratio.png', bbox_inches='tight')
    # plt.show()

def viewLossRatio(loss_ratio, t_start, T):
    plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    plt.plot(range(t_start - 1, T), loss_ratio[t_start - 1:T], 'k-', label='Optimum', linewidth=3)
    # plt.show()

def viewLossRatio_all(mode, candModels_Sequential, t_start, T):
    colors = matplotlib.cm.rainbow(np.linspace(0, 1, len(mode)))
    plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    linestyles = ['-', '--', '-.', ':']
    for i in range(len(mode)):
        plt.plot(range(t_start - 1, T), candModels_Sequential[i]['loss_ratio'][t_start - 1:T], color=colors[i],
                 label=mode[i], linewidth=1, linestyle=linestyles[i])
    plt.xlim(t_start - 1, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    #    plt.ylim(0,dT)
    plt.ylabel('Loss Ratio', fontsize=14, color='black')
    plt.title('Loss ratio of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.show()

def viewBatchLossRatio_all(mode, candModels_Sequential, t_start, T):
    colors = matplotlib.cm.rainbow(np.linspace(0, 1, len(mode)))
    plt.figure(figsize=(40, 40), dpi=150, facecolor='w', edgecolor='k')
    linestyles = ['-', '--', '-.', ':']
    for i in range(len(mode)):
        plt.plot(range(t_start - 1, T), candModels_Sequential[i]['batch_loss_ratio'][t_start - 1:T], color=colors[i],
                 label=mode[i], linewidth=1, linestyle=linestyles[i])
    plt.xlim(t_start - 1, T)
    plt.xlabel('Data Size', fontsize=14, color='black')
    #    plt.ylim(0,dT)
    plt.ylabel('Batch Loss Ratio', fontsize=14, color='black')
    plt.title('Batch Loss ratio of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.show()

def viewTiming(mode, timing_image, t_start, T):
    colors = matplotlib.cm.rainbow(np.linspace(0, 1, len(mode)))
    plt.figure(figsize=(10, 10), dpi=300, facecolor='w', edgecolor='k')
    linestyles = ['--', '--', '-.', ':']
    for i in range(len(mode)):
        plt.plot(range(t_start - 1, T), timing_image[mode[i]], color=colors[i], label=mode[i], linewidth=1,
                 linestyle=linestyles[i])
    plt.xlim(t_start - 1, T - 1)
    plt.xlabel('Data Size', fontsize=10, color='black')
    #    plt.ylim(0,dT)
    plt.ylabel('Elapsed Time', fontsize=10, color='black')
    plt.title('Elapsed Time of the optimal model and our predictor at each time')
    plt.legend(loc='upper right', prop={'size': 10})
    # plt.tight_layout()
    plt.grid()
    plt.savefig('./output/neuralnet/timing.png', bbox_inches='tight')
    # plt.show()

def main():
    try:
        os.makedirs('output/neuralnet')
    except OSError as e:
        if e.errno == errno.EEXIST:
            pass
        else:
            raise
    runLogisticExper()

if __name__ == "__main__":
    main()


from __future__ import print_function

import sys
import re
import os
import os.path
import subprocess
import argparse

import logging
logging.basicConfig(level=logging.DEBUG, format='[%(levelname)s] %(message)s')
logger = logging.getLogger(__name__)

import numpy as np

def check_qeb_args(x):
    if isinstance(x,tuple):
        return x
    m = re.match(r"^\((?P<f0>[0-9.e+-]+),(?P<Delta>[0-9.e+-]+),(?P<gamma>[0-9.e+-]+)\),(?P<Error>\d+)%$", x)
    if m is None:
        raise ValueError("check-qeb: expected argument in format \"(<f0>,<Delta>,<gamma>),<Error>%\"")
    return (float(m.group('f0')),float(m.group('Delta')),float(m.group('gamma')),float(m.group('Error'))/100.0)

def ftox_pair(ftoxstr):
    if isinstance(ftoxstr,tuple):
        return ftoxstr
    m = re.match(r"^\((?P<h>[0-9.e+-]+),(?P<s>[+-]?1)\)$", ftoxstr)
    if m is None:
        raise ValueError("ftox: expected \"(h,s)\" pair")
    return (float(m.group('h')), int(m.group('s')))

def float_ntuple_type(x, n):
    if isinstance(x,tuple):
        return x
    m = re.match(r"^\(" + ",".join( [r"([0-9.e+-]+)"] * n ) + r"\)", x)
    if m is None:
        raise ValueError("expected {}-tuple of values \"({})\"".format(n, ",".join(["?"]*n)))
    return tuple([float(m.group(k+1)) for k in range(n)])

def range_type(x): return float_ntuple_type(x,2)
def fitparams_type(x): return float_ntuple_type(x,4)

def percent_within_range_type(x):
    if isinstance(x, tuple):
        return x
    m = re.match(r"^(?P<percent>[0-9.e+-]+)%\s*\((?P<min>[0-9.e+-]+),\s*(?P<max>[0-9.e+-]+)\)$", x)
    if m is None:
        raise ValueError("percent_within_range: expected \"<percent>% (<min>,<max>)\" format")
    return (float(m.group('percent')), float(m.group('min')), float(m.group('max')))

def run_main():
    parser = argparse.ArgumentParser("test_tomorun_run")
    parser.add_argument("tomorun_argv", nargs='+')
    parser.add_argument("--setpath", action='store')
    parser.add_argument("--expect-exit-code", action='store', type=int, default=0)
    parser.add_argument("--no-further-checks", action='store_true', default=False)
    parser.add_argument("--check-histogram-file", action='store', default=None)
    parser.add_argument("--check-binning-vs-naive", action='store_true', default=None)
    parser.add_argument("--bin-rel-weight-threshold", action='store', type=float, default=0.01)
    parser.add_argument("--check-qeb", action='store', type=check_qeb_args)
    parser.add_argument("--ftox", action='store', type=ftox_pair, default=(0,1,))
    # can't do close tests on the chi2, because it's statistical!!! over 1000s of tests,
    # we'll have one or the other failure. So choose loose bounds
    parser.add_argument("--redchi2-range-ok", action='store', type=range_type, default=(0.01, 5))
    parser.add_argument("--check-fitparams", action='store', type=fitparams_type)
    parser.add_argument("--check-percent-within-range", action='append', type=percent_within_range_type)

    args = parser.parse_args()

    # setup sys.path, if needed
    if hasattr(args,'setpath') and args.setpath is not None:
        for p in reversed(args.setpath.split(':')):
            if p == '__TOMOGRAPHER_PYTHONPATH__':
                p = os.environ.get('TOMOGRAPHER_PYTHONPATH', None)
            if p:
                sys.path.insert(0, p)
        logger.debug("Set sys.path = {!r}".format(sys.path))

    # first, launch the tomorun process
    try:
        logger.debug("Launching tomorun process with argv={!r}".format(args.tomorun_argv))
        subprocess.check_call(args.tomorun_argv)
    except subprocess.CalledProcessError as e:
        if e.returncode != args.expect_exit_code:
            raise
        logger.debug("Got the expected return code %d"%(args.expect_exit_code))

    did_check = False

    # load the output histogram
    if args.check_histogram_file:

        if hasattr(args,'check_binning_vs_naive') and args.check_binning_vs_naive is not None:

            # check the binning analysis error bars versus the naive ones
            do_check_binning_vs_naive(args.check_histogram_file, args)
            did_check = True

        if hasattr(args,'check_qeb') and args.check_qeb is not None:

            # check the quantum error bars
            do_check_qeb(args.check_histogram_file, args.check_qeb, args)
            did_check = True

        if hasattr(args,'check_fitparams') and args.check_fitparams is not None:

            # check against known fit params
            do_check_fitparams(args.check_histogram_file, args.check_fitparams, args)
            did_check = True

        if hasattr(args,'check_percent_within_range') and args.check_percent_within_range is not None:

            # check that a certain percentatge of the total weight falls within the given
            # range (can be a list of ranges), if the option is specified multiple times
            for check_rng in args.check_percent_within_range:
                do_check_percent_within_range(args.check_histogram_file, check_rng)
            did_check = True

    if args.no_further_checks:
        return

    if not did_check:
        raise RuntimeError("Didn't check anything.")

def assert_approx_eq_rel(x, y, err):
    print("Testing {!r} =? {!r}  (to {!r} rel. precision) ...".format(x,y,err), end='')
    refom = np.maximum(np.absolute(x),np.absolute(y))
    assert np.absolute(x-y) < err*refom
    print(" OK")

def assert_in_range(x, a, b):
    print("Testing {!r} <= {!r} <= {!r} ... ".format(a,x,b), end='')
    assert a <= x and x <= b
    print(" OK")

def do_check_binning_vs_naive(hfile, args):

    hdata = np.loadtxt(hfile, skiprows=1)
    datalen = hdata.shape[0]

    # check third & fourth columns -- they should be similar for values above a certain threshold
    diffs = []
    for k in range(datalen):
        if hdata[k,1] > args.bin_rel_weight_threshold: # at least X% of points in this bin
            rldiff = (np.absolute(hdata[k,2] - hdata[k,3]) /
                      np.maximum(np.absolute(hdata[k,2]), np.absolute(hdata[k,3])))
            diffs.append(rldiff)

    #
    diffs.sort()

    # We require that 90% of the bins have less than 33% relative diff error between binning and naive
    fraction_in_range = 0.9
    maxreldiff = 0.33

    assert_in_range(diffs[int(fraction_in_range*len(diffs))], 0, maxreldiff)

def do_check_percent_within_range(hfile, percent_within_range):
    # only import tomographer here, after possibly having set sys.path
    import tomographer as t
    import tomographer.querrorbars as tq
    logger.debug("Loaded tomographer from {} and tomographer.querrorbars from {}".format(t.__path__, tq.__file__))

    h = tq.load_tomorun_csv_histogram_file(hfile)

    percent, rngmin, rngmax = percent_within_range

    if h.params.isWithinBounds(rngmin):
        idxmin = h.params.binIndex(rngmin)
    else:
        idxmin = 0
    if h.params.isWithinBounds(rngmax):
        idxmax = h.params.binIndex(rngmax)
    else:
        idxmax = h.numBins()-1

    w = np.sum(h.bins[idxmin:idxmax])

    logger.debug("Total weight within [{:.4g},{:.4g}] is {:.3g}%; required is {:.3g}%"
                 .format(rngmin,rngmax, w*100.0, percent))

    print("Testing {!r} > {!r} ... ".format(w, percent/100.0), end='')
    assert w > percent/100.0
    print(" OK")

def do_check_qeb(hfile, refqeb, args):
    # only import tomographer here, after possibly having set sys.path
    import tomographer as t
    import tomographer.querrorbars as tq
    logger.debug("Loaded tomographer from {} and tomographer.querrorbars from {}".format(t.__path__, tq.__file__))

    h = tq.load_tomorun_csv_histogram_file(hfile)

    kwargs = {}
    #if use_qeb_hint:
    ftox = args.ftox
    ftoxfn = lambda f, s=ftox[1], h=ftox[0]: s*(f - h)
    guessfitparams = tq.reskew_logmu_curve(*tq.qu_error_bars_to_deskewed_c(ftoxfn, *(refqeb[:3]),
                                                                           y0=np.log(np.amax(h.bins))))

    a = tq.HistogramAnalysis(h, ftox=ftox, threshold_fraction=0.01, p0=guessfitparams)

    a.printFitParameters(print_func=logger.debug)
    a.printQuantumErrorBars(print_func=logger.debug)
    qeb = a.quantumErrorBars()

    # a.plot() # DEBUG
    # a.plot(log_scale=True) # DEBUG

    # make sure the fit was ok
    redchi2 = a.fitRedChi2()
    logger.debug("redchi2 = {:.4g}".format(redchi2))
    assert_in_range(redchi2, args.redchi2_range_ok[0], args.redchi2_range_ok[1])

    assert_approx_eq_rel(qeb.f0, refqeb[0], refqeb[3])
    assert_approx_eq_rel(qeb.Delta, refqeb[1], refqeb[3])
    assert_approx_eq_rel(qeb.gamma, refqeb[2], refqeb[3])

def do_check_fitparams(hfile, fitparams, args):
    # only import tomographer here, after possibly having set sys.path
    import tomographer as t
    import tomographer.querrorbars as tq
    logger.debug("Loaded tomographer from {} and tomographer.querrorbars from {}".format(t.__path__, tq.__file__))

    h = tq.load_tomorun_csv_histogram_file(hfile).normalized()

    ftox = args.ftox
    ftoxfn = lambda f, s=ftox[1], h=ftox[0]: s*(f - h)
    xtoffn = lambda x, s=ftox[1], h=ftox[0]: h + s*x

    theo_fn = lambda x: tq.fit_fn_default(x, *fitparams)

    # get the indexes in the histogram where the points are selected (above threshold)
    idxok = np.where(h.bins > 0)
    xok = ftoxfn(h.values_center[idxok])
    logpok = np.log(h.bins[idxok])
    errlogpok = np.divide(h.delta[idxok], h.bins[idxok]);

    import matplotlib.pyplot as plt
    plt.errorbar(xok, logpok, errlogpok, c='b', fmt='.')
    plt.plot(xok, theo_fn(xok), c='r')
    plt.show()

    redchi2 = tq.calc_redchi2(xok, logpok, errlogpok, theo_fn, num_fit_params=4)

    # make sure the values align with theoretical curve (pre-calculated)

    logger.debug("redchi2 = {:.4g}".format(redchi2))
    assert_in_range(redchi2, args.redchi2_range_ok[0], args.redchi2_range_ok[1])

if __name__ == '__main__':
    run_main()


def read_file(filename):
    with open(filename, 'r') as f:
        text = f.read().split()
    return text

sid = read_file('siddhartha.txt')

with open('output.txt', 'w') as file:
    for word in sid:
        file.write(word.upper() + '\n')


"""
PAPADIAMANTIS Eval TP python
"""

import numpy as np
import matplotlib.pyplot as plt

def exo2(x,y):
    if x<y:
        return -1
    elif x == y:
        return 0
    else:
        return 1

def extermum_local(T):

    result = []

    # Check first
    print(np.abs(T[1]))
    if( np.abs(T[0]) > np.abs(T[1])):
        result.append((0,T[0]))

    # Check inbetween elements
    for i in range(1,T.shape[0]-1):
        if np.abs(T[i])>np.abs(T[i-1]) and np.abs(T[i+1])< np.abs(T[i]):
             result.append((i,T[i]))

    # Check last element
    if np.abs(T[-1])>np.abs(T[-2]):
        result.append((T.shape[0]-1,T[-1]))

    return result

def moyenne(T):

    # Calculate the sum of all elements of list
    somme = 0
    for i in range(0,T.shape[0]):
        somme += T[i]

    # Mean of elements
    m = somme / T.shape[0]
    return m

def Histogramme(T):
    # Allocating histogram
    H = np.zeros(256,dtype=int)

    # Get number of instances
    for k in range(0,T.size):
        H[T[k]] +=1

    return H

def exo8(H):
    somme = 0
    num = 0
    for i in range(0,H.shape[0]):
        somme += H[i]*i
        num += H[i]

    return  somme/num

def Coefficient(T, k):

    coeff = 0
    for i in range(0, T.size):
        coeff += T[i] * np.exp(-2j * np.pi * ((k * i) / T.size))

    return coeff

def TF(T):

    F = np.zeros(T.size, dtype=complex)

    for i in range(0, T.size):
        F[i] = Coefficient(T, i)

    return F

#Loadin text as ints
T = np.loadtxt("dataEvaluation.txt",dtype=int)

# Graph of Data in table t
plt.figure()
plt.plot(T)
plt.title("Data")
plt.show()

print("Extrema locale", extermum_local(T))

print("Moyenne de la fonction", moyenne(T))

hist = Histogramme(T.astype(int))

plt.figure()
plt.plot(hist)
plt.title("Histogramme")
plt.show()

print("Moyenne par hstogramme",exo8(hist))

plt.figure()
plt.plot(np.abs(TF(T)))
plt.title("Module Of Fourier transform")
plt.show()


import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
import torch.utils.data as data
import torchvision.datasets as datasets
import os
import glob
import random
from PIL import Image
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader
from sklearn.metrics import accuracy_score, confusion_matrix
import numpy as np
import matplotlib.pyplot as plt
from torchvision.datasets import ImageFolder

# set up class to get images
class RGBDepthDataset(data.Dataset):
    def __init__(self, image_paths, transform=None):
        self.image_paths = image_paths
        self.transform = transform

    def __getitem__(self, index):
        rgb_path, depth_path, label = self.image_paths[index]
        rgb_image = Image.open(rgb_path).convert('RGB')
        depth_image = Image.open(depth_path).convert('L')  # Convert depth image to grayscale

        if self.transform is not None:
            rgb_image = self.transform(rgb_image)
            depth_image = self.transform(depth_image)

        return rgb_image, depth_image, torch.tensor(label - 1, dtype=torch.long)  # Convert label to a scalar tensor. Model expects index starting at 0 so -1 included

    def __len__(self):
        return len(self.image_paths)

class PillClassifier(nn.Module):
    def __init__(self, num_classes=9):
        super(PillClassifier, self).__init__()

        self.conv1 = nn.Sequential(
            nn.Conv2d(3, 8, kernel_size=3, stride=2, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )

        self.conv2 = nn.Sequential(
            nn.Conv2d(8, 16, kernel_size=3, stride=2, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )

        self.conv3 = nn.Sequential(
            nn.Conv2d(16, 32, kernel_size=3, stride=2, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )

        self.fc1 = nn.Sequential(
            nn.Linear(32 * 10 * 10, 128),
            nn.ReLU()
        )

        self.fc2 = nn.Linear(128, num_classes)
        self.softmax = nn.Softmax(dim=1)

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)

        x = x.view(x.size(0), -1)
        x = self.fc1(x)
        x = self.fc2(x)
        x = self.softmax(x)

        return x

num_classes = 15

## UNCOMMENT TO USE RESNET18
#model = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)
#model.fc = nn.Linear(model.fc.in_features, num_classes)

## UNCOMMENT TO USE CUSTOM MODEL
model = PillClassifier()

# split data into test and train
data_root = './pills'
test_ratio = 0.2 # ratio for splitting into test and train
# Generate folder numbers as labels
pill_folders = [f"pill_{i}" for i in range(1, num_classes + 1)]
image_paths = [] # initialize with empty list
for pill_folder in pill_folders:
    # get files from each folder
    folder_path = os.path.join(data_root, pill_folder)
    # Use glob to find the image files matching the pattern
    rgb_files = sorted(glob.glob(os.path.join(folder_path, 'rgb_*.jpg')))
    depth_files = sorted(glob.glob(os.path.join(folder_path, 'depth_*.jpg')))

    assert len(rgb_files) == len(depth_files), f"Number of RGB and depth images do not match in {pill_folder} folder."

    # Collect the paths of RGB and depth images
    for i in range(len(rgb_files)):
        rgb_path = rgb_files[i]
        depth_path = depth_files[i]
        image_paths.append((rgb_path, depth_path))

# Shuffle the image paths within each pill folder
random.shuffle(image_paths)

# Create empty lists to store train and test image paths
train_image_paths = []
test_image_paths = []

# Split the image paths into train and test sets for each pill folder
for pill_folder in pill_folders:
    # Find the image paths corresponding to the current pill folder and assign labels
    folder_image_paths = [(rgb_path, depth_path, int(pill_folder.split('_')[1])) for (rgb_path, depth_path) in image_paths if pill_folder in rgb_path]

    # Split the current pill folder's image paths into train and test sets
    folder_train_image_paths, folder_test_image_paths = train_test_split(folder_image_paths, test_size=test_ratio, random_state=42, shuffle=True, stratify=[label for _, _, label in folder_image_paths])

    # Append the train and test image paths of the current pill folder to the overall lists
    train_image_paths.extend(folder_train_image_paths)
    test_image_paths.extend(folder_test_image_paths)

# Print the number of samples in train and test sets
print(f"Number of train samples: {len(train_image_paths)}")
print(f"Number of test samples: {len(test_image_paths)}")

# Set up train and test data
train_data = RGBDepthDataset(train_image_paths, transform=transforms.Compose([
    transforms.ToTensor(),
]))

test_data = RGBDepthDataset(test_image_paths, transform=transforms.Compose([
    transforms.ToTensor(),
]))

# Create Data Loaders
batch_size = 32
train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True, drop_last=True)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, drop_last=True)

# Loss Function and Optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# Train Model
num_epochs = 1 # update later to more

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)

for epoch in range(num_epochs):
    train_loss = 0.0

    model.train()
    for rgb_images, depth_images, labels in train_loader:
        rgb_images = rgb_images.to(device)
        depth_images = depth_images.to(device)
        labels = labels.to(device)

        optimizer.zero_grad()

        outputs = model(rgb_images)
        loss = criterion(outputs, labels)

        loss.backward()
        optimizer.step()

        train_loss += loss.item() * rgb_images.size(0)

    train_loss = train_loss / len(train_loader.dataset)
    print(f"Epoch {epoch+1}/{num_epochs}, Training Loss: {train_loss:.4f}")

# Evaluate Model
model.eval()

correct = 0
total = 0
predictions = []
true_labels = []

with torch.no_grad():
    for rgb_images, depth_images, labels in test_loader:
        rgb_images = rgb_images.to(device)
        depth_images = depth_images.to(device)
        labels = labels.to(device)

        outputs = model(rgb_images)
        _, predicted = torch.max(outputs.data, 1)

        total += labels.size(0)
        correct += (predicted == labels).sum().item()

        predictions.extend(predicted.cpu().numpy())
        true_labels.extend(labels.cpu().numpy())

accuracy = 100 * correct / total
print(f"Accuracy: {accuracy:.2f}%")

# Create a confusion matrix
confusion_mat = confusion_matrix(true_labels, predictions)

# Plot the confusion matrix
plt.figure(figsize=(10, 8))
plt.imshow(confusion_mat, cmap='Blues')
plt.title('Confusion Matrix')
plt.colorbar()
plt.xlabel('Predicted Labels')
plt.ylabel('True Labels')

# Display the class labels on the x and y axis
num_classes = len(pill_folders)
# Define the class labels based on the number of classes
class_labels = [f"pill_{i+1}" for i in range(num_classes)]
tick_marks = np.arange(num_classes)
plt.xticks(tick_marks, class_labels, rotation=45)
plt.yticks(tick_marks, class_labels)

# Fill the confusion matrix cells with the counts
thresh = confusion_mat.max() / 2
for i in range(num_classes):
    for j in range(num_classes):
        plt.text(j, i, f'{confusion_mat[i, j]}', ha='center', va='center', color='white' if confusion_mat[i, j] > thresh else 'black')

plt.tight_layout()
plt.show()


# Making candlestick chart
import datetime as dt
import matplotlib.pyplot as plt
from matplotlib import style
import pandas as pd
from pandas_datareader import data as pdr
import fix_yahoo_finance as yf
from mpl_finance import candlestick_ohlc
import matplotlib.dates as mdates

style.use('ggplot')
plt.rcParams.update({'font.size': 7})

# Data download from yahoo finance
symbol = ['^NSEI', '^BSESN', 'SBIN.NS', 'HDFCBANK.NS', 'ICICIBANK.NS', 'TATAMOTORS.NS', 'MARUTI.NS']
start_date = dt.datetime(2009,1,1)
end_date = dt.datetime(2019,1,31)
yf.pdr_override()
df = pdr.get_data_yahoo(symbol, start_date, end_date, group_by = 'ticker', auto_adjust = True, thread = 5)
df.head(5)
idx = pd.IndexSlice
HDFC = df.loc[idx[:],idx['HDFCBANK.NS',:]]
HDFC.columns = HDFC.columns.droplevel()
HDFC.to_csv('HDFCstockQuotes.csv')

# if we need date as a column, don't ask to parse dates in col 0 as index
# HDFCquotes = pd.read_csv('HDFCstockQuotes.csv')
HDFCquotes = pd.read_csv('HDFCstockQuotes.csv', parse_dates= True, index_col = 0)

# resample aggregates 10 day block data in 1 set; not rolling
hdfc_ohlc = HDFCquotes.resample('10D').agg({'Open': 'first', 'High': 'max', 'Low': 'min', 'Close': 'last'})
hdfc_ohlc.head(5)
# candlestick_ohlc wants from us dates in mdates format and Open, high, low and close
hdfc_ohlc.reset_index(inplace=True)
hdfc_ohlc.head(5)
# convert normal dates to matplot mdates
hdfc_ohlc['Date'] = hdfc_ohlc['Date'].map(mdates.date2num)

#Setting figures
fig, ax1 = plt.subplots(figsize=(16,9))
ax1.xaxis_date()

candlestick_ohlc(ax1, hdfc_ohlc.values, width=4, colorup='g', colordown='r')


class DataHandler:
    def __init__(self):
        self.handshake_table = dict()
        self.app_data_table = dict()
        self.locked_tracker = dict()

    def lock_field(self, field_name, *args):
        key = DataHandler.get_key(*args)

        if key not in self.locked_tracker:
            self.locked_tracker[key] = set()
        self.locked_tracker[key].add(field_name)

    def update_handshake_table(self, field_name, field_value, append, *args):
        key = DataHandler.get_key(*args)
        if key in self.locked_tracker and field_name in self.locked_tracker[key]:
            return
        else:
            if key not in self.handshake_table:
                self.handshake_table[key] = \
                    {
                        'client_random': '',
                        'server_random': '',
                        'cipher_suite': '',
                        'enc_p_master_secret': '',
                        'p_master_secret': '',
                        'master_secret': '',
                        'client_write_mac': '',
                        'server_write_mac': '',
                        'client_write_key': '',
                        'server_write_key': '',
                        'client_write_IV': '',
                        'server_write_IV': '',
                        'packets': b'',
                        'session_hash': ''
                    }
            if append:
                self.handshake_table[key][field_name] += field_value
            else:
                self.handshake_table[key][field_name] = field_value

    def add_to_app_data(self, packet_type, packet, *args):
        key = DataHandler.get_key(*args)

        if key not in self.app_data_table:
            self.app_data_table[key] = []
        self.app_data_table[key].append(
            {'packet_type': packet_type,
            'packet': packet
            })

    def get_next_from_app_data(self, *args):
        result = None
        key = DataHandler.get_key(*args)
        if key in self.app_data_table and self.app_data_table[key]:
            result = self.app_data_table[key].pop(0)
        return result

    def get_from_handshake_table(self, field_name, *args):
        result = None
        key = DataHandler.get_key(*args)
        if key in self.handshake_table and field_name in self.handshake_table[key]:
            result = self.handshake_table[key][field_name]
        return result

    @staticmethod
    def get_key(src_ip, dest_ip, src_port, dest_port):
        if src_ip + src_port < dest_ip + dest_port:
            return src_ip + src_port + dest_ip + dest_port
        else:
            return dest_ip + dest_port + src_ip + src_port


# train semi supervised
from func.load_dataset import airway_dataset
from func.loss_func import dice_loss_weights, dice_accuracy, dice_loss_power_weights, dice_loss, dice_loss_power
from func.ulti import load_obj, crop_one_3d_img, load_one_CT_img
from func.semi_supervise_learning import get_the_whole_img_set_dict, init_the_teacher_model,\
    use_teacher_model_to_process_one_img_generating_study_materials,\
        save_the_study_materials_for_one_img,\
            use_teacher_model_to_process_one_list_of_img_and_save_study_materials,\
                get_balanced_data_dict, get_data_dict_of_current_unlabled_image_and_pseudolabels
from func.model_arch import SegAirwayModel

import numpy as np
import torch
import os
import h5py
import shutil
import time
import torch
from torch.utils.data import DataLoader, RandomSampler

# settings
# ----------
load_path_teacher = "checkpoint/checkpoint.pkl"
load_path_student = "checkpoint/checkpoint.pkl"
save_path_student_model = "checkpoint/checkpoint_semi_supervise_learning_sample.pkl"
unlabel_data_path = "/data/Airway/LIDC-IDRI_3D/raw_data" #"/data/Airway/QMH_airway_data/Unlabeled_3d_data"
unlabel_data_indicator = "LIDC_IDRI_"
path_dataset_info_more_focus_on_airways_of_low_generation = \
    "dataset_info/train_dataset_info_EXACT09_LIDC_IDRI_crops_128_extended_more_low_generation_10"
path_dataset_info_more_focus_on_airways_of_high_generation = \
    "dataset_info/train_dataset_info_EXACT09_LIDC_IDRI_crops_128_extended_more_high_generation_1"
file_path_of_study_materials = "/data/Airway/QMH_airway_data/temp"

crop_cube_size_for_study_material_generation = [32, 128, 128]
stride_for_study_material_generation = [16, 64, 64]
crop_cube_size_for_study_material_saving = (128, 128, 128)
stride_for_study_material_saving = (64,64,64)

max_iteration_teacher_student_training = 5 # iteration of teacher-student training
num_of_imgs_used_for_each_tea_stu_iteration = 22
max_epoch = 2
num_samples_of_dataloader = 1000

learning_rate = 1e-5
batch_size=2
crop_size_for_train=(32, 128, 128)
windowMin_CT_img_HU = -1000 # min of CT image HU value
windowMax_CT_img_HU = 600 # max of CT image HU value
labeled_train_file_format = '.npy'
unlabeled_train_file_format = ".h5"
model_save_freq = 1
num_workers = 4

device_for_teacher = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
device_for_student = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
# ----------

# init the teacher model
teacher_model = init_the_teacher_model(SegAirwayModel,\
    in_channel=1, out_channel=2,\
        load_path = load_path_teacher, strict=False)
teacher_model.to(device_for_teacher)

# init the student model
student_model=SegAirwayModel(in_channels=1, out_channels=2)
checkpoint = torch.load(load_path_student)
student_model.load_state_dict(checkpoint['model_state_dict'], strict=False)
student_model.to(device_for_student)

# get the dict of unlabeled images
unlabeled_img_set_dict = get_the_whole_img_set_dict(unlabel_data_path, indicator = unlabel_data_indicator)

# optimizer setting
optimizer=torch.optim.Adam(student_model.parameters(), lr=learning_rate)

# datasets of labeled imgs
labeled_dataset_info_more_focus_on_airways_of_low_generation = load_obj(path_dataset_info_more_focus_on_airways_of_low_generation)
labeled_train_dataset_low_gen = airway_dataset(labeled_dataset_info_more_focus_on_airways_of_low_generation)
labeled_train_dataset_low_gen.set_para(file_format=labeled_train_file_format, crop_size=crop_size_for_train,\
    windowMin=windowMin_CT_img_HU, windowMax=windowMax_CT_img_HU, need_tensor_output=True, need_transform=True)

dataset_info_more_focus_on_airways_of_high_generation = load_obj(path_dataset_info_more_focus_on_airways_of_high_generation)
labeled_train_dataset_high_gen = airway_dataset(dataset_info_more_focus_on_airways_of_high_generation)
labeled_train_dataset_high_gen.set_para(file_format=labeled_train_file_format, crop_size=crop_size_for_train,\
    windowMin=windowMin_CT_img_HU, windowMax=windowMax_CT_img_HU, need_tensor_output=True, need_transform=True)

#deleting the file_path_of_study_materials
try:
    shutil.rmtree(file_path_of_study_materials)
except:
    print("no such filefolder")
    pass

start_time = time.time()
for ith_tea_stu_iteration in range(max_iteration_teacher_student_training):

    # under ith_iteration, devide the unlabeled images to smaller batches, and train on each batch
    pic_name_list = np.array(list(unlabeled_img_set_dict.keys()))
    np.random.shuffle(pic_name_list)
    pic_name_batch_list = []
    start_loc = 0
    while start_loc < len(pic_name_list):
        if start_loc+num_of_imgs_used_for_each_tea_stu_iteration<=len(pic_name_list):
            pic_name_batch_list.append(pic_name_list[start_loc:start_loc+num_of_imgs_used_for_each_tea_stu_iteration])
        else:
            pic_name_batch_list.append(pic_name_list[start_loc:len(pic_name_list)])
        start_loc+=num_of_imgs_used_for_each_tea_stu_iteration

    # so, we get pic_name_batch_list, let's do it one by one
    for ith_pic_name_batch, pic_name_batch in enumerate(pic_name_batch_list):
        # use teacher model generating pseudolabels for this batch, and getting the data dict
        data_dict_of_current_unlabled_image_and_pseudolabels = \
        use_teacher_model_to_process_one_list_of_img_and_save_study_materials(pic_name_batch, unlabeled_img_set_dict,
                                                                              teacher_model,
                                                                              device = device_for_teacher,
                                                                              file_path_of_study_materials = file_path_of_study_materials,
                                                                              crop_cube_size_for_study_material_generation = crop_cube_size_for_study_material_generation,
                                                                              stride_for_study_material_generation = stride_for_study_material_generation,
                                                                              crop_cube_size_for_study_material_saving = crop_cube_size_for_study_material_saving,
                                                                              stride_for_study_material_saving = stride_for_study_material_saving,
                                                                              min_crop_cube_size=crop_cube_size_for_study_material_generation)

        # extend the data dict
        # datasets of unlabeled imgs
        unlabeled_datset_info_more_focus_on_airways_of_low_generation = \
        get_balanced_data_dict(data_dict_of_current_unlabled_image_and_pseudolabels, is_more_big = True, copy_times_I = 10)
        unlabeled_train_dataset_low_gen = airway_dataset(unlabeled_datset_info_more_focus_on_airways_of_low_generation)
        unlabeled_train_dataset_low_gen.set_para(file_format=unlabeled_train_file_format, crop_size=crop_size_for_train,\
            windowMin=windowMin_CT_img_HU, windowMax=windowMax_CT_img_HU, need_tensor_output=True, need_transform=True)

        unlabeled_datset_info_more_focus_on_airways_of_high_generation = \
        get_balanced_data_dict(data_dict_of_current_unlabled_image_and_pseudolabels, is_more_big = False, copy_times_I = 1)
        unlabeled_train_dataset_high_gen = airway_dataset(unlabeled_datset_info_more_focus_on_airways_of_high_generation)
        unlabeled_train_dataset_high_gen.set_para(file_format=unlabeled_train_file_format, crop_size=crop_size_for_train,\
            windowMin=windowMin_CT_img_HU, windowMax=windowMax_CT_img_HU, need_tensor_output=True, need_transform=True)

        # labeled and unlabeled datasets
        dataset_list = [labeled_train_dataset_low_gen, unlabeled_train_dataset_low_gen,\
            labeled_train_dataset_high_gen, unlabeled_train_dataset_high_gen, labeled_train_dataset_low_gen]
        dataset_list_str = ["labeled, low gen",\
            "unlabeled, low gen",\
                "labeled images, high gen",\
                    "unlabeled, high gen",\
                        "labeled, low gen"]
        # dataset_list_str = ["on labeled images, focus more on airways of low generation",\
        #     "on unlabeled images, focus more on airways of low generation",\
        #         "on labeled images, focus more on airways of high generation",\
        #             "on unlabeled images, focus more on airways of high generation",\
        #                 "on labeled images, focus more on airways of low generation"]

        for ith_epoch in range(0, max_epoch):

            for ith_dataset, dataset in enumerate(dataset_list):
                sampler = RandomSampler(dataset, num_samples = num_samples_of_dataloader, replacement = True)
                dataset_loader = DataLoader(dataset, batch_size=batch_size, sampler = sampler,\
                    num_workers=num_workers, pin_memory=True, persistent_workers=(num_workers > 1))

                len_dataset_loader = len(dataset_loader)
                for ith_batch, batch in enumerate(dataset_loader):

                    # get the number of unconnected components in label
                    if torch.sum(batch['label'])>0:
                        unique_vals_of_label = torch.unique(batch['label'])
                        unique_vals_of_label = unique_vals_of_label[unique_vals_of_label>0]
                        num_of_unconnected_components_in_label= len(unique_vals_of_label)
                    else:
                        num_of_unconnected_components_in_label = 0
                    num_of_unconnected_components_in_label = max(num_of_unconnected_components_in_label, 1)

                    img_input=batch['image'].float().to(device_for_student)
                    groundtruth_foreground=batch['label'].float().to(device_for_student)
                    groundtruth_background=1-groundtruth_foreground

                    fore_pix_num = torch.sum(groundtruth_foreground)
                    back_pix_num = torch.sum(groundtruth_background)
                    fore_pix_per = fore_pix_num/(fore_pix_num+back_pix_num)
                    back_pix_per = back_pix_num/(fore_pix_num+back_pix_num)
                    weights = (torch.exp(back_pix_per)/(torch.exp(fore_pix_per)+torch.exp(back_pix_per))*torch.eq(groundtruth_foreground,1).float()+\
                        torch.exp(fore_pix_per)/(torch.exp(fore_pix_per)+torch.exp(back_pix_per))*torch.eq(groundtruth_foreground,0).float()).to(device_for_student)

                    #print("num_of_unconnected_components_in_label: "+str(num_of_unconnected_components_in_label))

                    img_output=student_model(img_input)

                    loss=1/num_of_unconnected_components_in_label * (dice_loss_weights(img_output[:,0,:,:,:], groundtruth_background, weights)+\
                        dice_loss_power_weights(img_output[:,1,:,:,:], groundtruth_foreground, weights, alpha=2))
                    accuracy=dice_accuracy(img_output[:,1,:,:,:], groundtruth_foreground)

                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()

                    time_consumption = time.time() - start_time
                    print(
                        "tea_stu_iteration [{0}/{1}]\t"
                        "unlabeled_img_batch [{2}/{3}]\t"
                        "epoch [{4}/{5}]\t"
                        "dataset [{6}]\t"
                        "train_batch [{7}/{8}]\t"
                        "time(sec) {time:.2f}\t"
                        "loss {loss:.4f}\t"
                        "acc {acc:.2f}%\t"
                        "fore pix {fore_pix_percentage:.2f}%\t"
                        "back pix {back_pix_percentage:.2f}%\t".format(
                            ith_tea_stu_iteration + 1,
                            max_iteration_teacher_student_training,
                            ith_pic_name_batch + 1,
                            len(pic_name_batch_list),
                            ith_epoch + 1,
                            max_epoch,
                            dataset_list_str[ith_dataset],
                            ith_batch,
                            len_dataset_loader,
                            time = time_consumption,
                            loss = loss.item(),
                            acc = accuracy.item()*100,
                            fore_pix_percentage = fore_pix_per*100,
                            back_pix_percentage = back_pix_per*100))

            if (ith_epoch+1)%model_save_freq==0:
                print('save stu model')
                student_model.to(torch.device('cpu'))
                torch.save({'model_state_dict': student_model.state_dict()}, save_path_student_model)
                student_model.to(device_for_student)

        #deleting the file_path_of_study_materials
        try:
            shutil.rmtree(file_path_of_study_materials)
        except:
            print("no such filefolder")
            pass

    print("update teacher model")
    teacher_model.to(torch.device('cpu'))
    checkpoint_of_stu_model = torch.load(save_path_student_model)
    teacher_model.load_state_dict(checkpoint_of_stu_model['model_state_dict'])
    teacher_model.to(device_for_teacher)


from django.db import connections

_FESM_LPC_PROJ_QUERY = """
SELECT s_date, e_date
FROM fesm_lpc_proj
WHERE LOWER(project_id)=LOWER(%s)
"""

def get_collection_years_for_project_id(project_id):
    """Queries the database for start/end years for the given project id.

    Returns a tuple of start/end year if found, otherwise returns Nones
    """
    with connections['gis_data'].cursor() as cursor:
        cursor.execute(_FESM_LPC_PROJ_QUERY, [project_id])
        row = cursor.fetchone()

        # Fallback if row not found
        if not row:
            return None, None
        else:
            s_date, e_date = row
            return s_date.year, e_date.year


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from django.http import JsonResponse
from django.conf import settings
from django.urls import reverse
from django.core.files import File
from django.core import serializers
from django.views .generic import ListView,DetailView
from django.contrib.auth.models import User
from .models import Project,ProjectFile
from posts.models import Post,PostFile,Comment
from posts.views import PostListView
from .AIalgo import generateAIseq
import time
import filetype
import uuid # for generating unique file name
import json
import ast
from PIL import Image
import cv2
import numpy as np
import random
from moviepy.editor import *

# global var for referencing the files user upload from frontend
# used for user customization of video
fileSrc = []   # the file path
custDur = []
custSpeed = []
# For saving and serving generated video during editing,
# and will also help with refering the video to store when creating post and project
VIDEO_PATH = settings.MEDIA_ROOT + '/generatedVideos/output_video.mp4'
VIDEO_URL = settings.MEDIA_URL + 'generatedVideos/output_video.mp4'

# For saving project and creating post purpose
AISeq = []
AIAlgo = ""
custSeq = []

# View functions which render a new template

@login_required
def createVideoHome(request):
    '''
    Render Create Video Home Page
    '''
    drafts = Project.objects.filter(author=request.user).order_by('-date_created')

    return render(request,'videos/create_video_home.html',{'draft':drafts})

@login_required
def editVideoMain(request):
    '''
    Render Edit Video Main Page starting a new project
    '''

    return render(request,'videos/edit_video_main.html')

@login_required
def editVideoMainFromDraft(request,pk_id,*args,**kwargs):
    '''
    Render Edit Video Main Page continuing with draft project
    '''
    global fileSrc,custDur,custSpeed
    # receive project object id from request
    project = Project.objects.get(pk=pk_id)
    project_files = ProjectFile.objects.filter(project=project)

    custDur = ast.literal_eval(project.cust_dur)
    custSpeed = ast.literal_eval(project.cust_speed)

    # 初始化fileSrc,custDur,custSpeed
    img_urls = []
    fileSrc = []
    org_dur = []
    for pjfile in project_files:
        url = pjfile.file.url
        img_urls.append(url)
        src = url.replace(settings.MEDIA_URL,settings.MEDIA_ROOT+'/')
        fileSrc.append(src)
    for src in fileSrc:
        if filetype.guess(src).mime.startswith("image"):
            org_dur.append([0,2])
        elif filetype.guess(src).mime.startswith("video"):
            org_dur.append([0,VideoFileClip(src).duration])

    msg = {'project':project,
            'project_files':project_files,
            'img_urls':img_urls,
            'org_dur':org_dur}

    return render(request,'videos/edit_video_main_continue_draft.html',msg)

@login_required
def createPost(request):
    '''
    Handle creating post functions
    redirect to system home Page
    '''
    if request.method == "POST":
        title = request.POST.get('post-title')
        content = request.POST.get('post-content')
        is_private = request.POST.get('post-visibility')
        author = request.user

        # 存储数据到数据库
        # create a new post record
        new_post = Post(title = title,
                        content = content,
                        author = author,
                        is_private=is_private,
                        AI_algo = AIAlgo,
                        AI_seq = AISeq,
                        cust_seq = custSeq,
                        cust_dur = custDur,
                        cust_speed = custSpeed)

        with open(VIDEO_PATH,'rb') as vf:
            # 生成随机文件名
            file_name = uuid.uuid4().hex
            new_post.video.save(file_name,File(vf))
        new_post.save()

        # save all the uploaded files
        for src in fileSrc:
            new_postfile = PostFile(post = new_post)
            # 生成随机文件名
            file_name = uuid.uuid4().hex
            with open(src,'rb') as f:
                new_postfile.file.save(file_name,File(f))

    # 返回到home页面
    return redirect('/')

@login_required
def saveProject(request,pj_id,*args,**kwargs):
    '''
    Handle saving project as draft function
    Redirect to create video home Page
    '''
    if request.method == "POST":
        title = request.POST.get('save-title')
        new_project = Project(title=title,
                              AI_algo=AIAlgo,
                              cust_seq=custSeq,
                              cust_dur=custDur,
                              cust_speed=custSpeed,
                              author=request.user)

        with open(VIDEO_PATH,'rb') as vf:
            # 生成随机文件名
            file_name = uuid.uuid4().hex
            new_project.thumbnail.save(file_name,File(vf))

        new_project.save()

        for src in fileSrc:
            new_projectfile = ProjectFile(project=new_project)
            # 生成随机文件名+原文件类型后缀
            file_name = uuid.uuid4().hex + '.' + src.split('.')[-1]
            with open(src,'rb') as f:
                new_projectfile.file.save(file_name,File(f))
        # delete the original project if any
        if pj_id != 9999:
            old_project = Project.objects.get(pk=pj_id)
            old_project.delete()

        # 返回到create video home 页面
        return redirect('/create-video')

# View functions which do not render new templates
def uploadFiles(request):

    global fileSrc,AIAlgo,AISeq,custSeq

    if request.method == "POST" and request.is_ajax():
        # 用户选择A的I模型
        model = request.POST.get('model')
        # 用户上传的图片/视频
        files = request.FILES.getlist('uploads[]')
        # 保存文件
        saveFiles(files)
        # 1. 获得AI模型输出的顺序
        ai_seq = getAIsequence(model,fileSrc)
        custSeq = ai_seq
        # 更新全局变量
        AISeq = ai_seq
        AIAlgo = model
        # 2. 生成视频
        video = generateVideo(ai_seq)
        # 3. 保存生成的视频
        save_path = VIDEO_PATH
        video.write_videofile(save_path,fps=24)
        # 4. 将生成的视频url返回前端
        video_url = VIDEO_URL
        backInfo = {
            'sequence': ai_seq,
            'initDuration': custDur,
            'speed': custSpeed,
            'url': video_url
        }

        return JsonResponse(backInfo,safe=False)

def updateVideo(request):
    global custSeq, custDur, custSpeed
    backInfo = []
    if request.method == "POST" and request.is_ajax():
        cust_seq = request.POST.getlist("seq")
        cust_dur = request.POST.getlist("duration")
        cust_speed = request.POST.getlist("speed")
        print("User customizing sequence: ",cust_seq)
        print("custom duartion: ",cust_dur) # ['0,2'.'0,2.5']
        print("custom speed: ",cust_speed) # ['1','1','1']

        cust_seq = [int(index) for index in cust_seq]
        # 更新custSeq
        custSeq = cust_seq
        # 更新custDur、custSpeed
        for i in range(len(cust_seq)):
            src_index = cust_seq[i]
            custDur[src_index] = [float(val) for val in cust_dur[i].split(',')]
            custSpeed[src_index] = float(cust_speed[i])

        # 生成视频
        video = generateVideo(cust_seq)
        # 保存生成的视频
        save_path = VIDEO_PATH
        video.write_videofile(save_path,fps=24)
        # 将生成的视频url返回前端
        video_url = VIDEO_URL
        backInfo = {
            'url': video_url
        }

        return JsonResponse(backInfo,safe=False)

def getAIsequence(algo, file_srcs):
    """
    function to get AI-empowered sequence of files
    arguments:
    algo: choosen AI algorithm
    file_srcs: an array of file paths
    """
    print('Processing file list:\n',file_srcs)
    new_seq = generateAIseq.main(algo, file_srcs)

    return new_seq

def generateVideo(new_seq):
    """
    argument: a list of file paths of image / short videos
    return: a composed video
    """

    clips = []
    for src_index in new_seq:
        src = fileSrc[src_index]
        if filetype.guess(src).mime.startswith("image"):
            video_clip = generateVideofromImage(src_index)

        elif filetype.guess(src).mime.startswith("video"):
            video_clip = generateVideofromVideo(src_index)

        else:
            print("Error: Uploaded file has wrong type!")

        clips.append(video_clip)
    concat_video = concatenate_videoclips(clips, method="compose")

    return concat_video

def generateVideofromImage(src_index):
    """
    generate video clip from an image (image/*)
    uisng moviepy
    arguments:
    src_index - image file path index from fileSrc
    """
    src = fileSrc[src_index]
    duration = custDur[src_index]
    speed = custSpeed[src_index]
    img = Image.open(src)
    # apply duration cut and speed up
    # for image the actual effect will vary in clip duration
    clip = ImageClip(np.array(img)).set_duration(duration[1]-duration[0]).fx(vfx.speedx,speed)
    # image clip default duration is 2s
    # clip.write_videofile("./output_test.mp4",fps=24)
    return clip

def generateVideofromVideo(src_index):
    """
    generate video clip from a video (video/*)
    using moviepy
    src_index - video file path index from fileSrc
    """
    # for moviepy VideoFileClip to create the video clip from python InMemoryUploadedFile object,
    # need to save the file object into a file first
    # haven't think of a better way to do this
    src = fileSrc[src_index]
    duration = custDur[src_index]
    speed = custSpeed[src_index]

    start = time.time()
    clip = VideoFileClip(src)
    print('Time-clip: ',time.time()-start)
    # apply duration cut and speed up
    clip = VideoFileClip(src).subclip(duration[0],duration[1]).fx(vfx.speedx,speed)

    return clip

def saveFiles(files):
    """
    function to save user uploaded images/videos on server for AI algorithm to process
    """
    global fileSrc,custDur, custSpeed,custSeq
    # 重置全局变量
    fileSrc = []
    custDur = []
    custSpeed = []
    custSeq = []

    src_list = []

    for i in range(len(files)):
        # 新文件src
        name = '{}.{}'.format(i,files[i].name.split('.')[-1])
        src = settings.MEDIA_ROOT + '/processing_files/'+ name
        # 保存图片到服务器
        if files[i].content_type.startswith("image"):
            img = Image.open(files[i])
            img.save(src)
            src_list.append(src)
            custDur.append([0,2])
            custSpeed.append(1)
        # 保存视频到服务器
        elif files[i].content_type.startswith("video"):
            start = time.time()
            with open(src,'wb+') as vidfile:
                for chunk in files[i].chunks():
                    vidfile.write(chunk)
            src_list.append(src)

            print('Time: ',time.time()-start)
            custDur.append([0,VideoFileClip(src).duration])
            custSpeed.append(1)
        else:
            print('File type not acceptable.')

    fileSrc = src_list

def deleteDraft(request,pj_id,*args,**kwargs):
    to_delete = Project.objects.get(pk=pj_id)
    to_delete.delete()
    return redirect('/create-video')


from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service
from subprocess import Popen, PIPE
from bs4 import BeautifulSoup
import fake_useragent
import os

URL_IP = 'https://2ip.ru/'
GECKO_PATH = 'path to geckodriver'
BINARY_PATH = 'path to firefox'
TOR_PATH = 'path to tor'

print('tor start')
tor = Popen(TOR_PATH, shell=True, stdout=PIPE, bufsize=-1)
if tor.poll() is not None:
    raise Exception('No connection to tor :^(')
result_code = str(tor.stdout.readlines()[-1]).lower()
if 'failed' in result_code:
    print('tor process is already running or port is in use')
if 'done' in result_code:
    print('tor process created')
print('tor works')

options = Options()
options.binary_location = BINARY_PATH
options.set_preference('network.proxy.type', 1)
options.set_preference('network.proxy.socks', '127.0.0.1')
options.set_preference('network.proxy.socks_port', 9050)
options.set_preference("network.proxy.socks_remote_dns", True)

fake_user = fake_useragent.UserAgent().random
options.set_preference('general.useragent.override', fake_user)

service = Service(executable_path=GECKO_PATH)

with Firefox(options=options, service=service) as driver:
    driver.get(url=URL_IP)
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    print(soup.find('div', {'class': 'ip'}).text.strip())
    print(soup.find_all('div', {'class': 'ip-icon-label'}))

tor.kill()
print('tor closed\ntor exit code = ' + str(tor.poll()))


class Solution:
    def canJump(self, nums):
        """
        :type nums: List[int]
        :rtype: bool
        """
        m = 0
        for i, num in enumerate(nums):
            if i > m:
                return False
            m = max(m, i + num)
        return True

if __name__ == "__main__":
    nums = [2,3,1,1,4]
    print(Solution().canJump(nums))


""" module for saving and loading dictionaries """
import os
import pickle

FILE_DIR = os.path.dirname(os.path.realpath('__file__'))

def save_dictionary(filename, obj):
    """save the dictionary into a file"""
    filename = os.path.join(FILE_DIR, 'assets/obj/' + filename)
    with open(filename, 'wb') as output:
        pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL)

def load_dictionary(filename):
    """load dictionary from the file"""
    filename = os.path.join(FILE_DIR, 'assets/obj/' + filename)
    try:
        with open(filename, 'rb') as input:
            return pickle.load(input)
    except Exception as e:
        print("exception", e)


#-*- coding: utf-8 -*-
import zipfile
    ##
    ##myzip = zipfile.ZipFile('D:\\2016-01-19\Zipfile.zip','w')
    ##myzip.write('C:\Users\Hojin\Desktop\\backup') #REQUIRE ISDIR
    ##
    ##myzip.close()

import os

## Need debugging!!

class zip():

    def zip_directory(self,path, drive, name, flag = 0):
	# path is the address[counter]
	#drive is different drive, need to be given.
	#name is also a different value.It requires input

        propeRdrive = self._drive_correction(drive)
        if flag ==0:
            new_directory = propeRdrive+str(date.today()) + "\\" + name
        else:
            new_directory = propeRdrive+str(datetime.now().strftime('%Y-%m-%d_%H')) + "\\" + name
            if not os.path.exists(new_directory +".zip"):
                zipname = zipfile.ZipFile(new_directory+".zip",'w')
            else:
                zipname = zipfile.ZipFile(new_directory+".zip",'a')
            paths = self.get_directory_address(path, drive, flag)
        for root, dirs, files in os.walk(paths):
            ## do something here
            if (files == []):
                zipname.write(root)
            else:
                for file in files:
                    zipname.write(os.path.join(root,file))

    def get_directory_address(self, path, drive, flag =0):

            #3) get_directory_address 등의 파라메터로 넘어오는 드라이브가 "D"  혹은 "D:" 와 같을때 어찌 되는지 테스트를 추가하고,  해당 함수를 처리하세요 ()
            #> 공통점 : 첫글자는 드라이브.

            ''' 저장되는 디렉토리는 LOG 의 주소를 따름.
            '''

            name = path[path.rfind("\\") +1:len(path)].replace(":","")
               #new_directory to toDirectory
            propeRdrive = self._drive_correction(drive)
            if flag ==0:
                new_directory = propeRdrive+str(date.today()) + "\\" + name
            else:
                new_directory = propeRdrive+str(datetime.now().strftime('%Y-%m-%d_%H')) + "\\" + name
            if not os.path.exists(new_directory):
                os.makedirs (new_directory)
            else: pass
            return new_directory

    def _drive_correction(self, drive):

            ''' drive:
               if drive is null : pass .\\ by copy_function >> Hence,  do nothing
               if drive is .\\  : pass
               if drive is given by D (the first letter) : append :\\ at the end of the string
               if drive is given by D:    :     append \\ at the end of the string.
            '''

            if drive == ".\\"   :   return ".\\"
            elif len(drive) == 1 :  return drive +":\\"
            elif drive.find(":") != -1 :
                if os.path.exists(drive+".\\") : return drive
                else : return ".\\"
            else : return ".\\"