from obspy.geodetics.base import gps2dist_azimuth, kilometer2degrees
from obspy.taup import TauPyModel
import pandas as pd
from tqdm import tqdm
from datetime import datetime, timedelta
import re
import os
import glob
import subprocess
from logger import logger

class SACProcess():

    def __init__(self, sac_path, instrument_path):
        os.putenv("SAC_DISPLAY_COPYRIGHT", "0")
        self.sac_path = sac_path
        self.instrument_path = instrument_path

    def getSACFile(self, get_all=False):
        if get_all == False:
            total_files = glob.glob(f"{self.sac_path}/*HLE*.SAC")
        else:
            total_files = glob.glob(f"{self.sac_path}/*.SAC")
        file_names = [os.path.basename(_) for _ in total_files]
        return file_names

    def getInstrumentFile(self):
        """
            Output: ['SAC_PZs_TW_A002_HLE_10_2019.051.00.00.00.0000_2599.365.23.59.59.99999',..]
        """
        total_files = glob.glob(f"{self.instrument_path}/*All*.99999")
        file_names = [os.path.basename(_) for _ in total_files]
        return file_names

    def removeInstrumentResponse(self, sac_file_names, instrument_file):
        """
            To remove instrument response by instrument file. Please don't remove twice !!!
            The steps are from GDMS website: https://gdms.cwa.gov.tw/help.php
            Note: check if remove instrument response by header-variables: https://seisman.github.io/SAC_Docs_zh/fileformat/header-variables/
        """
        for sac_file_name in sac_file_names:
            s = f"r {self.sac_path}/{sac_file_name}\n"
            s += "rmean; rtrend \n"
            s += "taper \n"
            s += f"trans from polezero s {self.instrument_path}/{instrument_file}\
                    to acc freq 0.02 0.1 1 10 \n"
            s += "w over \n"
            s += "q \n"
            subprocess.Popen(['sac'], stdin=subprocess.PIPE).communicate(
                s.encode())
        logger.info("remove finished.")

    def reName(self, path, file_name):
        """
            revised formated
            Input : TW.A002.10.HLE.D.2022.261.064200.SAC
            Output : TW.A002.10.HLE.D.20220918144400.SAC
        """
        timestamp = file_name.split('.')[-4:-1]  # date part
        year, day_of_year, time = timestamp
        formatted_timestamp = datetime.strptime(f"{year}{day_of_year}{time}",
                                                "%Y%j%H%M%S")
        time_difference = timedelta(hours=8, minutes=2)
        formatted_timestamp = formatted_timestamp + time_difference
        converted_timestamp = formatted_timestamp.strftime('%Y%m%d%H%M%S')
        pattern = r'\d{4}\.\d{3}\.\d+'
        new_file_name = re.sub(pattern, converted_timestamp, file_name)
        os.rename(f"{path}/{file_name}", f"{path}/{new_file_name}")
        return new_file_name

    def _getArrivalTime(self, records, sac_HLE):
        """
            To get arrival time from record
        """
        P_arrive = records[records["file_name"] ==
                           sac_HLE]["iasp91_P_arrival"].values[0]+120
        S_arrive = records[records["file_name"] ==
                           sac_HLE]["iasp91_S_arrival"].values[0]+120
        return P_arrive, S_arrive

    def autoPick(self, records, sac_file_names):
        """
            To pick the p and s wave by record data
        """
        for sac_file_name in sac_file_names:
            sac_HLE = sac_file_name
            sac_HLN = re.sub("HLE", "HLN", sac_file_name)
            sac_HLZ = re.sub("HLE", "HLZ", sac_file_name)
            P_arrive, S_arrive = self._getArrivalTime(records, sac_file_name)

            s = f"r {self.sac_path}/{sac_HLZ} \
                {self.sac_path}/{sac_HLE} \
                {self.sac_path}/{sac_HLN} \n"
            s += f"ch t1 {P_arrive} t2 {S_arrive} \n"
            s += "p1 \n"
            s += "w over \n"
            s += "q \n"
            subprocess.Popen(['sac'], stdin=subprocess.PIPE).communicate(
                s.encode())
        logger.info("auto pick finished.")

class recordProcess():

    def __init__(self, gdms_catalog, gcmt_catalog, stations):
        self.gdms_catalog = gdms_catalog
        self.stations = stations
        self.gcmt_catalog = gcmt_catalog

    def getDistance(self, records):
        """
            To calculate the distance between the station and earthquake
            Input : records = Dataframe records csv
            Output : the sta_dist column series
        """
        tmp = pd.merge(records, self.gdms_catalog, on='event_id', how='inner')
        result = pd.merge(tmp, self.stations, on='station', how='inner')
        result["sta_dist"] = (((result["lon_x"]-result["lon_y"])*101.7)**2 + (
            (result["lat_x"]-result["lat_y"])*110.9)**2 + (result["depth"]-result["height"])**2)**(1/2)
        return result["sta_dist"]

    def getArrivalTime(self, records):
        """
            To calculate the arrival time through iasp91 method
            Input : records = Dataframe records csv
            Output : list P_arrival and S_arrival
        """
        result = pd.merge(records, self.gdms_catalog,
                          on='event_id', how='inner')
        iasp91_P_arrival = []
        iasp91_S_arrival = []
        for i in tqdm(range(result.shape[0])):
            try:
                model = TauPyModel(model='iasp91')  # jb pwdk can test
                dist = kilometer2degrees(result["sta_dist"][i])
                depth = result["depth"][i]
                arrivals = model.get_travel_times(source_depth_in_km=depth, distance_in_degree=dist,
                                                  phase_list=["P", "S", 'p', 's'])
                iasp91_P_arrival.append(arrivals[0].time)
                iasp91_S_arrival.append(arrivals[-1].time)
            except:
                iasp91_P_arrival.append("NA")
                iasp91_S_arrival.append("NA")
        return iasp91_P_arrival, iasp91_S_arrival

    def getFocalMechanism(self, records):
        """
            To get the GCMT Focal mechanism from merged catalog
            Input : records = Dataframe records csv
            Output : the strike dip slip column series
        """
        result = pd.merge(records, self.gcmt_catalog,
                          on='event_id', how='inner')
        return result["strike1"], result["dip1"], result["slip1"], result["strike2"], result["dip2"], result["slip2"]

    def getFnmFrv(self, records):
        def Fnm(value):
            if value == "NULL":
                return "NA"
            elif value > -150 and value < -30:
                return 1
            else:
                return 0

        def Frv(value):
            if value == "NULL":
                return "NA"
            elif value > 30 and value < 150:
                return 1
            else:
                return 0
        records['Fnm_1'] = records['dip1'].apply(lambda x: Fnm(x))
        records['Frv_1'] = records['dip1'].apply(lambda x: Frv(x))
        records['Fnm_2'] = records['dip2'].apply(lambda x: Fnm(x))
        records['Frv_2'] = records['dip2'].apply(lambda x: Frv(x))

        return records['Fnm_1'], records['Frv_1'], records['Fnm_2'], records['Frv_2']

    def getMagnitudes(self, records):
        result = pd.merge(records, self.gdms_catalog,
                          on='event_id', how='inner')
        return result["Mw"], result["ML"]

    def getVs30(self, records):
        result = pd.merge(records, self.stations, on="station", how="inner")
        return result["Vs30"],result["Z1.0"]

    def getRecordDf(self, file_names):
        """
            To store the SAC file which matched with catalog
            Input : file_names = ['TW.C096.10.HLN.D.20220918145412.SAC',...]
        """
        record = {'event_id': [], 'file_name': [], 'station': [], 'year': [
        ], 'month': [], 'day': [], 'hour': [], 'minute': [], 'second': []}
        for file_name in file_names:
            for index, taiwan_time in enumerate(self.gdms_catalog['taiwan_time']):
                if str(taiwan_time) in file_name:
                    record['event_id'].append(
                        self.gdms_catalog['event_id'][index])
                    record['file_name'].append(file_name)
                    record['station'].append(file_name.split('.')[1])
                    record['year'].append(file_name.split('.')[5][0:4])
                    record['month'].append(file_name.split('.')[5][4:6])
                    record['day'].append(file_name.split('.')[5][6:8])
                    record['hour'].append(file_name.split('.')[5][8:10])
                    record['minute'].append(file_name.split('.')[5][10:12])
                    record['second'].append(file_name.split('.')[5][12:14])
        return record

    def buildRecordFile(self, record, record_path):
        df_record = pd.DataFrame(record)
        df_record.to_csv(f'{record_path}', index=False)

if __name__ == '__main__':

    #! SACProcess
    sac_path = "../TSMIP_Dataset/GuanshanChishangeq"
    instrument_path = f"../TSMIP_Dataset/InstrumentResponse/All"
    sac_process = SACProcess(sac_path, instrument_path)

    # ? step-2 remove instrument response
    # sac_files = sac_process.getSACFile(get_all=True)
    # instrument_file = sac_process.getInstrumentFile()
    # sac_process.removeInstrumentResponse(sac_files, instrument_file[0])

    # ? step-3 rename
    # file_names = [sac_process.reName(sac_path, os.path.basename(file)) for file in sac_files]

    #! recordProcess
    stations_path = "../TSMIP_Dataset/TSMIP_stations.csv"
    record_path = "../TSMIP_Dataset/GDMS_Record.csv"
    gdms_catalog_path = "../TSMIP_Dataset/GDMS_catalog.csv"
    gdms_catalog = pd.read_csv(gdms_catalog_path)
    records = pd.read_csv(record_path)
    stations = pd.read_csv(stations_path)
    gcmt_catalog = pd.read_csv("../TSMIP_Dataset/merged_catalog.csv")
    record_process = recordProcess(gdms_catalog, gcmt_catalog, stations)

    # #? step-4 build record csv
    # sac_files = sac_process.getSACFile(get_all=False)
    # record = record_process.getRecordDf(sac_files)
    # _ = record_process.buildRecordFile(record, record_path)

    # #? step-5 merge Magnitudes
    # Mw, ML = record_process.getMagnitudes(records)
    # records["Mw"] = Mw
    # records["ML"] = ML

    # #? step-6 merge Distance
    # result = record_process.getDistance(records)
    # records["sta_dist"] = result

    # #? step-7 merge P S arrival
    # iasp91_P_arrival, iasp91_S_arrival = record_process.getArrivalTime(records)
    # records["iasp91_P_arrival"] = iasp91_P_arrival
    # records["iasp91_S_arrival"] = iasp91_S_arrival

    # #? step-8 merge strike dip slip
    # strike1, dip1, slip1, strike2, dip2, slip2 = record_process.getFocalMechanism(records)
    # records["strike1"] = strike1
    # records["dip1"] = dip1
    # records["slip1"] = slip1
    # records["strike2"] = strike2
    # records["dip2"] = dip2
    # records["slip2"] = slip2

    # #? step-9 calculate Fnm Frv
    # Fnm_1, Frv_1, Fnm_2, Frv_2 = record_process.getFnmFrv(records)
    # records["Fnm_1"] = Fnm_1
    # records["Frv_1"] = Frv_1
    # records["Fnm_2"] = Fnm_2
    # records["Frv_2"] = Frv_2

    # #? step-10 merge Vs30
    vs30, z1_0 = record_process.getVs30(records)
    records["Vs30"] = vs30
    records["Z1.0"] = z1_0
    _ = record_process.buildRecordFile(records, record_path)

    # ? step-10 auto pick
    # records = pd.read_csv(record_path)
    # sac_files = sac_process.getSACFile(get_all=False)
    # sac_process.autoPick(records, sac_files)


import statsmodels.api as sm
import statsmodels.formula.api as smf
import numpy as np
import pandas as pd

def mult_reg_boot(formula, y, df, reps=2000):
    """Runs bootstrap reps of multiple linear regression to est outputs

           Args:
                formula (str): string identifying the formula as
                        input to stats models, typically of the form
                         'Y ~ X1 * X2' , or something like that.
                         See the Jupyter notebook for examples.
               y (string): string identifying the Pandas Series within
                        df that contains the response variable
               df (Pandas DataFrame): one of the dataframes prepared
                        using script finbeat_params_prep.py
               reps (int): number of bootstrap replicates

           Returns:
    """
    # Make a list to hold model objects
    # Method 1: Resampling residuals

    boot_outputs_method1 = []

    # Method 1
    reg_model1 = smf.ols(formula=formula, data=df).fit()
    predicted = reg_model1.fittedvalues
    residuals = reg_model1.resid
    stu_resid = reg_model1.wresid
    pearson_resid = reg_model1.resid_pearson

    for rep in np.arange(1, reps):
        df_copy = df.copy()
        # randomly resample residuals and add these random resids to y
        random_resid = np.random.choice(residuals, size=len(
            residuals), replace=True)
        df_copy[y] = df_copy[y] + random_resid
        # refit model using fake y and store output
        reg_model1_rep = smf.ols(formula=formula, data=df_copy).fit()
        boot_outputs_method1.append(reg_model1_rep)

    return boot_outputs_method1


# Basic data wranging and plotting
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot
# Creating the modeling dataset
from sklearn.datasets import make_classification
# Data processing
import pandas as pd
import numpy as np
# Data visualization
import matplotlib.pyplot as plt
import seaborn as sns
# Model and performance
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import precision_recall_curve, accuracy_score, classification_report, confusion_matrix, roc_auc_score, roc_curve, precision_recall_fscore_support
# Oversampling and under sampling
from imblearn.over_sampling import RandomOverSampler, SMOTE
from imblearn.under_sampling import RandomUnderSampler, NearMiss
from collections import Counter
# Processing time
import time
# Statistic
import statistics as s
# A little try for Reloading Utils when we update
from importlib import reload  # Python 3.4+ :: Just call the u = u.reload(u) on the main notebook
import sys

def model_run(models,resamplers,X,y,X_bbt,y_bbt,random_state,test_size,verbose,print_report):
    """

    INPUT
    models = Pandas List with model names
    resamples = Pandas List with resampler names
    X = DataFrame with independent variables
    y = Vector with dependent (response) variable
    random_state = INT random state number
    verbose = STR to switch execution log on or off
    print_report = STR to switch result printing on or off

    OUTPUT
    """

    start_time = time.time() #Start processing time
    print('Model training and evaluation routine started.')

    #Create list to hold firs run's trained models
    model_prediction = []

    #Collect dataframe to collect scores during first run
    Scores = ['Model',
            'Model Name',
            'Resampler Name',
            'TN',
            'FP',
            'FN',
            'TP',
            'Precision 0',
            'Precision 1',
            'Recall 0',
            'Recall 1',
            'F1-Score 0',
            'F1-Score 1',
            'Support 0',
            'Support 1',
            'FPR',
            'TPR',
            'Thresholds']
    model_scores_table = pd.DataFrame()
    model_scores_table['Scores'] = Scores

    #Create Dataframe to collect first run model's coeficient and feature importance measures
    model_coef_table = pd.DataFrame()
    model_coef_table['Features'] = X.columns

    for model_name in models:
        for resampler_name in resamplers:

            #Split, train and predict on models
            model,cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds =  model_predict(model_name,
                                                                                resampler_name,
                                                                                X,
                                                                                y,
                                                                                random_state,
                                                                                test_size,
                                                                                verbose,
                                                                                print_report)
            model_prediction.append((model,model_name,resampler_name))
            if verbose != 'off':print(model_name+' '+resampler_name+' Split, Training and Test Prediction ready.')

            #Collect Scores
            models_scores_append = [model,
                                    model_name,
                                    resampler_name,
                                    cm[0][0],
                                    cm[0][1],
                                    cm[1][0],
                                    cm[1][1],
                                    precision[0],
                                    precision[1],
                                    recall[0],
                                    recall[1],
                                    fbeta_score[0],
                                    fbeta_score[1],
                                    support[0],
                                    support[1],
                                    fpr,
                                    tpr,
                                    thresholds]

            model_scores_table[model_name+' '+resampler_name] = models_scores_append
            if verbose != 'off':print(model_name+' '+resampler_name+' Scores collected.')

            #Collect Coeficients and Feature Importance
            if model_name == 'Random Forest': model_coef_table_append = model.feature_importances_
            elif model_name == 'Logistic Regression': model_coef_table_append = model.coef_[0]

            model_coef_table[model_name+' '+resampler_name] = model_coef_table_append
            if verbose != 'off':print(model_name+' '+resampler_name+' Coeficients and Features Importance collected.')

            print('           '+model_name+' '+resampler_name+' routine finished')

    # Let's transpose the Model Scores Table to get a better look
    model_scores_table = transpose_model_scores(model_scores_table)

    print('Model training and evaluation finished.')
    print('---------------------------------------')

    # ---------------------------------------------------------------------------------
    #Run Big Blind Test routines:
    print('Starting Big Blind Test routine.')

    #Create dataframe to collect Big Blind Test performance scores
    BBT_model_scores_table = pd.DataFrame()
    BBT_model_scores_table['Scores'] = Scores

    #Create scaled version of X to use on Logistic Regression Model
    scaler = StandardScaler().fit(X_bbt)
    X_bbt_scaled = scaler.transform(X_bbt)
    X_bbt_scaled

    for model in model_prediction:
        BBT_model = model[0]
        BBT_model_name = model[1]
        BBT_resampler_name = model[2]

        #Predict on new Big Blind Test data
        if BBT_model_name == 'Logistic Regression':
            bbt_prediction = model[0].predict(X_bbt_scaled)
            bbt_model_proba = model[0].predict_proba(X_bbt_scaled)
        elif BBT_model_name == 'Random Forest':
            bbt_prediction = model[0].predict(X_bbt)
            bbt_model_proba = model[0].predict_proba(X_bbt)
        if verbose != 'off':print(BBT_model_name+' '+BBT_resampler_name+' Big Blind Data prediction ready.')

        #Evaluate perfomance of traind models on the new Big Blind Test data

        cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds = model_performance('Big Blind Test',
                                                                                BBT_model_name,
                                                                                BBT_resampler_name,
                                                                                y_bbt,
                                                                                bbt_prediction,
                                                                                bbt_model_proba,
                                                                                verbose,
                                                                                print_report)
        if verbose != 'off':print(model_name+' '+resampler_name+' Big Blind Test Score evaluation finished')

        #Collect Scores
        BBT_model_scores_table_append = [BBT_model,
                                         BBT_model_name,
                                         BBT_resampler_name,
                                         cm[0][0],
                                         cm[0][1],
                                         cm[1][0],
                                         cm[1][1],
                                         precision[0],
                                         precision[1],
                                         recall[0],
                                         recall[1],
                                         fbeta_score[0],
                                         fbeta_score[1],
                                         support[0],
                                         support[1],
                                         fpr,
                                         tpr,
                                         thresholds]

        BBT_model_scores_table[BBT_model_name+' '+BBT_resampler_name] = BBT_model_scores_table_append
        if verbose != 'off':print(BBT_model_name+' '+BBT_resampler_name+' Big Blind Data Scores collected.')

        print('           Big Blind Test '+BBT_model_name+' '+BBT_resampler_name+' routine finished')

    # Let's transpose the BBT Model Scores Table to get a better look
    BBT_model_scores_table = transpose_model_scores(BBT_model_scores_table)

    print("\nTotal Run processing time: --- %s seconds ---" % (time.time() - start_time))

    return model_prediction, model_scores_table, model_coef_table, BBT_model_scores_table

def model_predict(model_name,resampling_name,X,y,random_state,test_size,verbose,print_report):
    """
    Trains and Fits Models with different Resampling Techniques, than predicts, scores and measures coeficient and feature importance.

    INPUT
    model_name = Str with model name
    resampling_name = Str with resampling technique name
    X = DataFrame with independent variables
    y = Vector with dependent (response) variable
    random_state = INT random state number
    verbose = STR to switch execution log on or off
    print_report = STR to switch result printing on or off

    OUTPUT
    model_prediction = Vector with chosen model with resampling predictions on the test set
    """
    start_time = time.time() #Count processing time
    if verbose != 'off':print('\n--------------------------------------------------------------------------------\n')
    if verbose != 'off': print('\nStarting new sequence:',model_name,'with',resampling_name)
    model_prediction = []
    y_train_resampled = []

    #Define model
    model = define_model(model_name,random_state,verbose)
    #Split train and test sets + Apply scaling when need + Apply resampling
    X_train, X_test, y_train, y_test = split_resample_sets(model_name,
                                                           resampling_name,
                                                           X,
                                                           y,
                                                           test_size,
                                                           random_state,
                                                           verbose)
    #Train model
    if verbose != 'off': print('\nFitting model.')
    model = model.fit(X_train, y_train)
    if verbose != 'off': print("\nFitting model processing time: --- %s seconds ---" % (time.time() - start_time))
    #Predict on trained model
    if verbose != 'off': print('\nPredicting on model.')
    model_prediction = model.predict(X_test)
    if verbose != 'off': print("\nModel prediction processing time: --- %s seconds ---" % (time.time() - start_time))
    #Evaluate model performance
    model_proba = model.predict_proba(X_test)[:, 1]
    cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds = model_performance('First Run',
                                                                       model_name,
                                                                       resampling_name,
                                                                       y_test,
                                                                       model_prediction,
                                                                       model_proba,
                                                                       verbose,
                                                                       print_report)

    if verbose != 'off': print("\nTotal Model processing time: --- %s seconds ---" % (time.time() - start_time))

    #pyplot.show()

    return model,cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds

def define_model(model_name,random_state,verbose):
    """
    Instantiates and defines Classifier models.

    INPUT
    model_name = Str with model name
    random_state = INT random state number
    verbose = STR to switch execution log on or off

    OUTPUT
    Returns intantiated model according to users choice.
    """
    start_time = time.time() #Count processing time
    if verbose != 'off': print('\nInstantiating',model_name,'model.')

    if model_name == 'Random Forest':
        model = RandomForestClassifier(random_state = random_state,
#                                        class_weight='balanced_subsample',
#                                        max_depth = 2,
#                                        n_estimators=40,
#                                        max_leaf_nodes=10
                                       )
        if verbose != 'off': print('\nModel ready:',model)
        if verbose != 'off': print("Model Instatiating processing time: --- %s seconds ---" % (time.time() - start_time))

    elif model_name == 'Logistic Regression':
        model =  LogisticRegression(random_state = random_state,
#                                      class_weight='balanced',
#                                      penalty="l2", # l1_ratio = 0.5, only necessary with penalty="elasticnet"
#                                      tol=0.01,
#                                      solver="lbfgs",
#                                      C=0.01
                                    )
        if verbose != 'off': print('\nModel ready:',lr)
        if verbose != 'off': print("Model Instatiating processing time: --- %s seconds ---" % (time.time() - start_time))

    else:
        print('\nNo compatible model.')

    return model

def split_resample_sets(model_name,resampling_name,X,y,test_size,random_state,verbose):
    """
    Splits and resamples X dataset and y vector.

    INPUT
    model_name = Str with model name
    resampling_name = Str with resampling method
    X = DataFrame with independent variables
    y = Vector with dependent (response) variable
    test_size = Float (0 to 1) for test size porcentage of the train/test split
    random_state = INT random state number
    verbose = STR to switch execution log on or off

    OUTPUT
    X_train = DataFrame with independent variables splitted for train set
    X_test = DataFrame with independent variables splitted for test set
    y_train = Vector with dependent variable splitted for train set
    y_test = Vector with dependent variable splitted for test set
    """
    start_time = time.time() #Count processing time

    if verbose != 'off': print('\nIniatialing split for train and test sets. \nAnalyzing need for variable rescaling.')
    if model_name == 'Logistic Regression':
        if verbose != 'off': print('\nLogistic Regression requies scaling. \nVariable rescaling necessary.')
        scaler = StandardScaler().fit(X)
        X_scaled = scaler.transform(X)
        X_scaled

        # Split into train and test
        X_train, X_test, y_train, y_test = train_test_split(X_scaled,
                                                            y,test_size = test_size,
                                                            random_state = random_state)
    elif model_name == 'Random Forest':

        if verbose != 'off': print('\nRandom Forest does not require rescalling.')
        # Split into train and test
        X_train, X_test, y_train, y_test = train_test_split(X,
                                                            y,
                                                            test_size = test_size,
                                                            random_state = random_state)

    if verbose != 'off': print('\nApplying resampling technique choosen.')

    if resampling_name == 'Baseline':
        if verbose != 'off': print("\nBaseline doesn't require resampling.")

    elif resampling_name == 'Random Over Sampling':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = RandomOverSampler(random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    elif resampling_name == 'SMOTE':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = SMOTE(random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    elif resampling_name == 'NearMiss KNN':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = NearMiss(version=3,random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    elif resampling_name == 'Random Under Sampling':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = RandomUnderSampler(random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    if verbose != 'off': print("\nResampling processing time: --- %s seconds ---" % (time.time() - start_time))

    return X_train, X_test, y_train, y_test

def model_performance(run,model_name,resampling_name,y_test,model_prediction,model_proba,verbose,print_report):
    """
    Prints the performance reports: Classification report and Confusion Matrix

    Inputs
    run = Str indicating which run is being executed. Accepts only 'First Run' and 'Big Blind Test'
    model_name = Str with model name
    resampling_name = Str with resampling method
    y_test = Test vector
    model_prediction = Prediction vector
    verbose = STR to switch execution log on or off
    print_report = STR to switch resulting printing on or off

    Returns print with the reports
    """
    start_time = time.time() #Count processing time
    cr = classification_report(y_test, model_prediction,zero_division=0)
    cm = confusion_matrix(y_test, model_prediction)
    precision, recall, fbeta_score, support = precision_recall_fscore_support(y_test, model_prediction)
    if print_report != 'off': print('\n',model_name,'with',resampling_name,' Classification Report:')
    if print_report != 'off': print(cr)
    if print_report != 'off': print(cm)

    if verbose != 'off': print("\nModel Performane processing time: --- %s seconds ---" % (time.time() - start_time))

    # Plot the ROC Curve
    #Implementation derived from :
    ## https://scikit-learn.org/stable/modules/model_evaluation.html#roc-metrics
    ## https://stackoverflow.com/questions/33208897/how-to-interpret-this-triangular-shape-roc-auc-curve

    if run == 'First Run':
        fpr, tpr, thresholds = roc_curve(y_test, model_proba)
        pyplot.plot(fpr, tpr,label=run+' '+model_name+' '+resampling_name)
    elif run == 'Big Blind Test':
        fpr, tpr, thresholds = roc_curve(y_test, model_proba[:, 1])
        pyplot.plot(fpr, tpr,label=run+' '+model_name+' '+resampling_name,linestyle='--')
    pyplot.title('ROC and AUC Plot for tested models')
    pyplot.xlabel('False Positive Rate')
    pyplot.ylabel('True Positive Rate')
    pyplot.legend(bbox_to_anchor=(1,1), loc="upper left")

    return cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds

def Model_Coef_Table(models, X):
    """
    Function for analyzing the different importance coefients of both Logistic Regression and Random Forest models.
    Prints out a table and bar graph of the top 10 coeficients (simple summ)

    INPUT
    model_prediction = List containing the outputs of model_predict function:
                            model = Class Fitted Classfier model instances (Logistic Regression or Ranfom Forest)
                            cr = Str Classification Report
                            cm = NP.Array Confusion Matrix
                            precision =
                            recall =
                            fbeta-score =
                            support =
                            model_name = Str with model name
                            resampling_name = Str with resampling technique name
    X = pd.DataFrame with predictor (independent) variables

    OUTPUT
    Model_Coef_Table = Pandas Dataframe with Features and Coefients

    """
    Model_Coef_Table = pd.DataFrame()
    Model_Coef_Table['Features'] = X.columns

#     Model_Score_Table['Model'] = model_prediction[7] + model_prediction[8]

    for model in models:

        model_ = model[0] # Model
        model_name = model[1]# model_name
        resampler_name = model[2]# resampler_name

        print(model_)

        if model_name == 'Random Forest': Model_Coef_Table['Feature Importance '+model_name+resampler_name] = model_.feature_importances_
        elif model_name == 'Logistic Regression': Model_Coef_Table['Coeficient '+model_name+resampler_name] = model_.coef_[0]

    return Model_Coef_Table

def transpose_model_scores(model_scores_table):
    """
    Transposes Model Scores Table for easier analysis

    INPUT
    model_scores_table = pd.DataFrame() with model scores

    OUTPUT
    model_scores_table_T = pd.DataFrame() of transposed model scores table
    """
    # Let's transpose the Model Scores Table to get a better look
    model_scores_table_T = model_scores_table.T
    # Now let's promote the first row as header and drop the index
    model_scores_table_T = model_scores_table_T.rename(columns=model_scores_table_T.iloc[0]).drop(model_scores_table_T.index[0]).reset_index(drop=True)
    # Let's clean out the score related to predicting the majority class (1) and focus only on the minority
    #model_scores_table_T = model_scores_table_T.drop(columns=['Precision 1','Recall 1','F1-Score 1'])
    return model_scores_table_T


from hmm_bkt_models_final import *
import numpy as np

def dummy_hmm_test():
	'''
	the dummy example comes from russel and norvig's book
	'''
	em_umbrella = np.array([[0.9,0.2],
		[0.2,0.9]],order='F')

	# observation node: umbrella
	# 0 =
	umbrella_arr = np.array([[0],[0]])

	obs ={ "umbrella":umbrella_arr}
	emission_probs ={ "umbrella":em_umbrella }

	trans = np.array([[0.7,0.3],[0.3,0.7]])

	init_probs = np.array([0.5,0.5])

	test =build_hmm(["umbrella"] )

	test.set_model_parameters(obs,trans,init_probs, emission_probs )

	test.get_posterior_probs(obs)

def test_hmm():
	'''
	test the correctness of hmm model
	show how to construct a hmm model
	'''
	## load emission probs for each channel into a 2D array.
	## row = # of hidden states
	## 1: childless sing,with parents 2: childless single, left 3: xxx 4: xx
	## col = emission probs for channel states
	em_marr = np.array([[0.000997009, 0.003988036, 0.9950150],
		[0.001620140, 0.104561316, 0.8938185],
		[0.010343968, 0.413010967, 0.5766451],
		[0.031156530, 0.672357926, 0.2964855]],order='F')
	em_child = np.array([[0.9975050, 0.00249501],
		[0.9540918, 0.04590818],
		[0.8066367, 0.19336327],
		[0.5999251, 0.40007485]],order='F')
	em_left = np.array([[0.06524451, 0.9347555],
		[0.32372754, 0.6762725],
		[0.62761976, 0.3723802],
		[0.78393214, 0.2160679]],order='F')

	# observation data frame
	child_arr = np.loadtxt('example_data/Array2.csv',delimiter=',',dtype = np.uint32)

	left_arr = np.loadtxt('example_data/Array3.csv',delimiter=',',dtype = np.uint32)

	marr_arr = np.loadtxt('example_data/Array1.csv',delimiter=',',dtype = np.uint32)

	# first element is obs. sec element is emission probs
	obs ={ "marr":marr_arr,"child":child_arr,"left":left_arr}
	emission_probs ={ "marr":em_marr,"child":em_child,"left":em_left}

	# transition probs:
	# row = col = # hidden states
	trans = np.array([[0.90, 0.06, 0.03, 0.01],
       [0, 0.90, 0.07, 0.03],
       [0,    0, 0.90, 0.10],
       [0,    0,    0,    1]],order='F')

	# initial probs
	init_probs = np.array([0.9, 0.07, 0.02, 0.01],order='F')

	single_observation={ "marr":np.array([marr_arr[:,0]]).transpose(),"child":np.array([child_arr[:,0]]).transpose(),"left":np.array([left_arr[:,0]]).transpose()}

	test =build_hmm(["marr","child","left"] )

	test.set_model_parameters(obs,trans,init_probs, emission_probs )

	#test.set_parameters(trans,emission_probs,init_probs,single_observation)
	#test.parameter_estimation()

	test.get_posterior_probs(single_observation)

	# print('shape................child..')
	# print(child_arr[:,0])
	# print('marr')
	# print(marr_arr[:,0])
	# print('left')
	# print(left_arr[:,0])
	# print('......')
	test.get_posterior_probs()
	# viterbix=test.hidden_paths_viterbix()
	# # print("...................hidden paths viterbix .................")
	# with open('hidden_path.csv','wb') as out:
	# 	csv_out = csv.writer(out)
	# print(viterbix)
	# print(type(viterbix[0].shape))
	# print(viterbix[0][0,:])

def test_bkt():
	'''
	show how to construct a bkt model
	'''
	# observation node: answer correctness
	obs_task = np.array([[1,1,1,0],[1,1,0,0],[1,1,0,0],[0,0,1,0],[1,1,1,1],[1,1,0,1]],order='F')
	# observation node: affect node
	obs_aff = np.array([[0,0,1,0],[0,1,0,1],[0,1,1,0],[0,1,1,0],[1,1,1,1],[1,1,0,1]],order='F')

	single_task=[obs_task[0,:]]
	single_aff =[obs_aff[0,:]]

	test = build_bkt(['affect'])
	test.add_task_node_obs_data(obs_task)
	test.add_node_observation_data("affect",obs_aff)
	test.add_node_emission("affect",np.array([[0.23,0.44],[0.33,0.87]]))
	test.commit_bkt_model()

	# get the posterior probs without training the bkt model
	test.get_posterior_probs({'task_node':np.array(single_task).transpose(),'affect':np.array(single_aff).transpose()})

#test_bkt()
dummy_hmm_test()


import pandas as pd
import numpy as np
from sklearn import cluster
from sklearn import metrics
from scipy.spatial.distance import cdist
from tqdm.notebook import tqdm

# define function to kmeans_suggestion
# kmeans clustering with different k
# returns list of scores
def kmeans_suggestions(vectorized_suggestions, min_num_cluster, max_num_cluster):
    '''
    :params vectorized_suggestions: vectorized suggestions as list
    :params min_num_topics: min number of cluster
    :params max_num_topics: max number of cluster
    :return: dict of scores and num cluster
    '''

    scores = {'num_cluster':[], 'inertia':[], 'distortion':[], 'silhouette_score':[], 'calinski_harabasz_score':[]}

    # kmeans clustering for different num cluster and return coherence
    for i in tqdm(range(min_num_cluster, max_num_cluster+1)):
        kmeans = cluster.KMeans(n_clusters=i, random_state=1410)
        kmeans.fit(vectorized_suggestions)
        labels = kmeans.predict(vectorized_suggestions)

        # append to dict
        scores['num_cluster'].append(i)
        scores['inertia'].append(kmeans.inertia_)
        scores['distortion'].append(sum(np.min(cdist(vectorized_suggestions, kmeans.cluster_centers_,'euclidean'), axis=1)) / vectorized_suggestions.shape[0])
        scores['silhouette_score'].append(metrics.silhouette_score(vectorized_suggestions, labels))
        scores['calinski_harabasz_score'].append(metrics.calinski_harabasz_score(vectorized_suggestions, labels))

    return scores

# define function to dbscan_suggestions
# dbscan clustering with different parameters
# returns list of silhuoette scores
def dbscan_suggestions(vectorized_suggestions):
    '''
    :params vectorized_suggestions: vectorized suggestions as list
    :return: dict of scores and parameters
    '''

    scores = {'eps':[], 'min_samples':[], 'silhouette_score':[], 'num_cluster':[], 'num_noise':[]}
    # dbscan clustering for different eps and different min_samples and return silhuoette
    for i in tqdm(np.arange(0.05, 1, 0.05)):
        for j in np.arange(5, 16):
            dbscan = cluster.DBSCAN(eps=i, min_samples=j).fit(vectorized_suggestions)
            labels = dbscan.labels_

            # drop noise points from labels
            tmp = pd.DataFrame()
            tmp['labels'] = labels
            tmp['vector'] = vectorized_suggestions.tolist()
            tmp = tmp[tmp['labels']!=-1]
            labels_clean = tmp['labels'].tolist()
            vectors_clean = np.array(tmp['vector'].tolist())

            # append to dict
            scores['eps'].append(i)
            scores['min_samples'].append(j)
            try:
                scores['silhouette_score'].append(metrics.silhouette_score(vectors_clean, labels_clean))
            except:
                scores['silhouette_score'].append(1)
            scores['num_cluster'].append(len(set(labels)) - (1 if -1 in labels else 0))
            scores['num_noise'].append(list(labels).count(-1))
    return scores


#http://stackoverflow.com/questions/13979714/heap-sort-how-to-sort
import Util
import time
import Sort

class HeapSort():
    def __init__(self, intList):
        self.intList = intList

    def swap(self, i, j):
        sqc = self.intList
        sqc[i], sqc[j] = sqc[j], sqc[i]

    def heapify(self, end,i):
        sqc = self.intList
        l = 2 * i + 1
        r = 2 * (i + 1)
        max = i
        if l < end and sqc[i] < sqc[l]:
            max = l
        if r < end and sqc[max] < sqc[r]:
            max = r
        if max != i:
            self.swap(i, max)
            self.heapify(end, max)

    def sort(self, intList):
        self.intList = intList
        end = len(self.intList)
        start = end // 2 - 1 # use // instead of /
        for i in range(start, -1, -1):
            self.heapify(end, i)
        for i in range(end-1, 0, -1):
            self.swap(i, 0)
            self.heapify(i, 0)
        return self.intList

L = Util.generateList(10)

def forTime(L):
    A = HeapSort(L)
    Start = time.time()
    M = A.sort(L)
    runTime = time.time() - Start
    print(M)
    return runTime

#print(forTime(L))

"""

for i in range(10):
    L = Util.generateList();
    A = HeapSort(L);
    Start = time.time();
    M = A.sort(L)
    runTime = time.time() - Start;
    print(M)
    print(runTime);

The heapsort algorithm involves preparing the list by first turning it into a max heap. The algorithm then repeatedly swaps the first value of the list with the last value, decreasing the range of values considered in the heap operation by one, and sifting the new first value into its position in the heap. This repeats until the range of considered values is one value in length.

The steps are:

Call the buildMaxHeap() function on the list. Also referred to as heapify(), this builds a heap from a list in O(n) operations.
Swap the first element of the list with the final element. Decrease the considered range of the list by one.
Call the siftDown() function on the list to sift the new first element to its appropriate index in the heap.
Go to step (2) unless the considered range of the list is one element.
The buildMaxHeap() operation is run once, and is O(n) in performance. The siftDown() function is O(log(n)), and is called n times.
Therefore, the performance of this algorithm is O(n + n * log(n)) which evaluates to O(n log n).
"""


class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next
class Solution:
    def mergeTwoLists(self, l1: ListNode, l2: ListNode) -> ListNode:
        head = ListNode()
        last = head

        while (l1 != None and l2 != None):
            if (l1.val < l2.val):
                last.next = l1
                last = l1
                l1 = l1.next
            else:
                last.next = l2
                last = l2
                l2 = l2.next

        if (l1 != None):
            last.next = l1
        if (l2 != None):
            last.next = l2

        return head.next


import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import numpy as np
from tqdm.auto import tqdm
import random

class Coordinator(nn.Module):
    def __init__(self, distance_matrix: torch.FloatTensor, dim: int = 2,
        lr: float = 3e-2):
        super().__init__()
        self.distance_matrix = torch.FloatTensor(distance_matrix)
        self.coords = nn.Parameter(torch.rand(distance_matrix.shape[0], dim))
        self.optim = optim.AdamW([self.coords], lr = lr)
        self.patience = min(100, 10 * self.distance_matrix.shape[0])
        self.sched = optim.lr_scheduler.ReduceLROnPlateau(self.optim,
            mode = 'min', patience = self.patience, factor = 0.5)
        self.goal = (self.distance_matrix.mean() / 100) ** 2

    @staticmethod
    def get_distance_matrix(X: torch.FloatTensor) -> torch.FloatTensor:
        return torch.norm(torch.sub(X[:, None], X), dim = -1)

    def get_coords(self):
        with torch.no_grad():
            self.coords -= self.coords.mean()
            self.coords /= self.coords.norm()
            return self.coords.detach().numpy()

    def compile(self, nepochs: int = 200, batch_size: int = 256):
        N: int = self.coords.shape[0]
        ema: float = 0.
        ema_factor: float = min(N, batch_size) / (10 * N)
        idxs: np.array = np.arange(N)
        nbatches: int = N // batch_size
        nbatches = nbatches + 1 if N % batch_size else nbatches
        niter: int = 0
        bad_scores: int = 0
        best_score: float = float('inf')
        finished: bool = False

        with tqdm(desc = 'Computing coordinates', total = N * nepochs) as pbar:
            for epoch in range(nepochs):
                if finished: break
                self.optim.zero_grad()
                np.random.shuffle(idxs)

                for batch in np.array_split(idxs, nbatches):
                    niter += batch.shape[0]
                    pred = self.get_distance_matrix(self.coords[batch, :])
                    true = self.distance_matrix[np.ix_(batch, batch)]
                    loss = F.mse_loss(pred, true)

                    ema = (1 - ema_factor) * ema + ema_factor * float(loss)
                    ema /= 1 - (1 - ema_factor) ** niter

                    pbar.set_description(f'Computing coordinates - '\
                                         f'mean squared error {ema:.6f}')
                    pbar.update(batch.shape[0])

                    loss.backward()
                    self.optim.step()

                    if niter > batch_size / ema_factor:
                        if ema < best_score - self.goal:
                            best_score = ema
                            bad_scores = 0
                        else:
                            bad_scores += 1

                        if bad_scores > 3 * self.patience or ema < self.goal:
                            finished = True
                            break

                    self.sched.step(ema)

        return self

    def plot(self):
        if self.coords.shape[1] > 2:
            raise RuntimeError('Cannot plot coordinates in dimensions > 2')
        else:
            raise NotImplementedError('Cannot plot yet!')

    def score(self):
        with torch.no_grad():
            pred = self.get_distance_matrix(self.coords)
            diff = torch.mean(torch.abs(self.distance_matrix - pred))
            return -torch.log(diff).item()

if __name__ == '__main__':
    coords = torch.randn(2000, 2)
    distance_matrix = torch.norm(coords[:, None] - coords, dim = -1)

    coordinator = Coordinator(distance_matrix, dim = 2).compile()
    print(coordinator.score())


#!/usr/bin/env python

import ConfigParser
import logging
import threading
import subprocess
import time
import os

try:
    import rrb3
except:
    import rrb3mock as rrb3

logger = logging.getLogger(__name__)

class Rover(threading.Thread):

    def __init__(self, config):
        threading.Thread.__init__(self)

        self.config = config
        self.rate = float(self.config.get('rover', 'rate'))
        battery_voltage = self.config.get('hw', 'battery_voltage')
        motor_voltage = self.config.get('hw', 'motor_voltage')
        logging.info('Initializing rrb3...')
        self.rr = rrb3.RRB3(battery_voltage, motor_voltage)
        self.rr.set_led1(0)
        self.rr.set_led1(1)
        logger.info('Initialized rrb3')
        self.is_running = True

    def stop(self):
        self.rr.stop()
        self.rr.cleanup()
        self.is_running = False

    def shutdown(self):
        self.stop()
        self._run_shell_cmd('/sbin/shutdown -h now')

    def reboot(self):
        self.stop()
        self._run_shell_cmd('/sbin/reboot')

    def update(self):
        self._run_shell_cmd('git pull')

    def run(self):
        while (self.is_running):
            time.sleep(self.rate)

    def get_range(self):
        distance = self.rr.get_distance()
        logger.debug('Range: {}'.format(distance))
        return distance

    def set_motors(self, left, right):
        l = abs(left)
        r = abs(right)
        left_dir = 0 if left < 0 else 1
        right_dir = 0 if right < 0 else 1
        logger.debug('set_motors({}, {}, {}, {})'.format(l, left_dir, r, right_dir))
        self.rr.set_motors(l, left_dir, r, right_dir)

    def _run_shell_cmd(self, command):
        logger.info('Running shell command: {}'.format(command))
        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
        output = process.communicate()[0]
        logger.info(output)

def main():
    config = ConfigParser.ConfigParser()
    config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.ini')
    config.read(config_file)

    logger.setLevel(getattr(logging, config.get('logging', 'level')))
    logging.basicConfig(format='[%(levelname)-5s] %(asctime)-15s %(name)10s %(message)s')

    Rover(config).start()

if __name__ == '__main__':
    main()


import boto3
import json

def lambda_handler(event, context):
    # Create RDS and Config clients
    rds_client = boto3.client('rds')
    config_client = boto3.client('config')

    # Storing time
    orderingtime = json.loads(event['invokingEvent'])[
        'notificationCreationTime']

    # Get all RDS instances
    response = rds_client.describe_db_instances()
    rds_instances = response['DBInstances']

    # Iterate through all RDS instances
    for instance in rds_instances:
        # Get the engine type and instance identifier
        engine = instance['Engine']
        instance_id = instance['DBInstanceIdentifier']

        # Check if the engine type is MySQL
        if engine.lower() in ['oracle-ee', 'oracle-ee-cdb', 'oracle-se2', 'oracle-se2-cdb']:

            # Get the option groups name for the current instance
            db_option_groups = instance['OptionGroupMemberships'][0]['OptionGroupName']

            # Initialize the require_secure_transport variable
            ssl_configuration = None
            nne_configuration = None
            # Paginate results until "require_secure_transport" parameter is found
            paginator = rds_client.get_paginator('describe_option_groups')
            for page in paginator.paginate(OptionGroupName=db_option_groups):

                # Find the require_secure_transport option value
                for optionGroup in page['OptionGroupsList']:
                    if optionGroup['OptionGroupName'] == db_option_groups:
                        for option in optionGroup['Options']:
                            if option['OptionName'] == 'SSL':
                                for setting in option['OptionSettings']:
                                    if setting['Name'] == "SQLNET.SSL_VERSION":
                                        ssl_configuration = setting['Value'] in ['1.2']
                                        break
                                break
                            elif option['OptionName'] == 'NNE':
                                for setting in option['OptionSettings']:
                                    if setting['Name'] == "SQLNET.ENCRYPTION_CLIENT":
                                        nne_configuration = setting['Value'] in ['RC4_256', '3DES168', 'DES40']
                                        break
                                break
                        break

            # Determine compliance based on require_secure_transport value
            if ssl_configuration or nne_configuration:
                compliance_type='COMPLIANT'
            else:
                compliance_type='NON_COMPLIANT'
        else:
            # If the engine type is not Oracle, mark as non-applicable
            compliance_type='NON_APPLICABLE'

        # Submit evaluation to AWS Config
        config_client.put_evaluations(
            Evaluations=[
                {
                    'ComplianceResourceType': 'AWS::RDS::DBInstance',
                    'ComplianceResourceId': instance_id,
                    'ComplianceType': compliance_type,
                    'Annotation': f'RDS Instance is {compliance_type} due to SSL Configuration = {ssl_configuration} and/or NNE Configuration = {nne_configuration}',
                    'OrderingTimestamp': orderingtime
                },
            ],
            ResultToken=event['resultToken']
        )


""" Post-migration "fixers", which are executed after an object has been migrated
"""

import logging

from plone.app.multilingual.api import get_translation_manager

from .config import LANGUAGES, TOP_LEVEL, AST_PATHS, FULL_PAGE_PATHS, SECTOR_POLICIES
from .utils import make_uid

logger = logging.getLogger()

def onpath(path):

    def decorator_factory(func):
        def decorator(context):
            if not context.absolute_url(relative=True).endswith(path):
                return
            return func(context)

        return decorator

    return decorator_factory

def inpath(path):

    def decorator_factory(func):
        def decorator(context):
            if path not in context.absolute_url(relative=True):
                return
            return func(context)

        return decorator

    return decorator_factory

@onpath('/knowledge/adaptation-information/climate-services/climate-services')
def fix_climate_services_toc(context):
    # in first column block, replace the first paragraph with a horizontal navigation table of contents

    col_block_id = context.blocks_layout['items'][1]    # [0] is title block
    col = context.blocks[col_block_id]
    column_ids = col['data']['blocks_layout']['items']
    first_col_id = column_ids[0]
    first_col = col['data']['blocks'][first_col_id]

    first_block_id = first_col['blocks_layout']['items'][0]
    new_data = {"@type": 'toc',
                "variation": "horizontalMenu"}
    first_col['blocks'][first_block_id] = new_data

@onpath('/help/Webinars')
def fix_webinars(context):
    blocks = context.blocks
    layout = context.blocks_layout

    videos = {
        1:  {
            'text': {
                'id': layout['items'][-1],
                'block': blocks[layout['items'][-1]]
            },
            'video': {
                'id': layout['items'][-2],
                'block': blocks[layout['items'][-2]]
            }
        },
        2:  {
            'text': {
                'id': layout['items'][-3],
                'block': blocks[layout['items'][-3]]
            },
            'video': {
                'id': layout['items'][-4],
                'block': blocks[layout['items'][-4]]
            }
        },
        3:  {
            'text': {
                'id': layout['items'][-5],
                'block': blocks[layout['items'][-5]]
            },
            'video': {
                'id': layout['items'][-6],
                'block': blocks[layout['items'][-6]]
            }
        },
        4:  {
            'text': {
                'id': layout['items'][-7],
                'block': blocks[layout['items'][-7]]
            },
            'video': {
                'id': layout['items'][-8],
                'block': blocks[layout['items'][-8]]
            }
        },
        5:  {
            'text': {
                'id': layout['items'][-9],
                'block': blocks[layout['items'][-9]]
            },
            'video': {
                'id': layout['items'][-10],
                'block': blocks[layout['items'][-10]]
            }
        },
    }

    col_1_id = make_uid()
    col_1_item_1_id = make_uid()

    col_2_id = make_uid()
    col_2_item_1_id = make_uid()
    col_2_item_2_id = make_uid()

    col_3_id = make_uid()
    col_3_item_1_id = make_uid()
    col_3_item_2_id = make_uid()

    blocks[col_1_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_1_item_1_id: {
                    "blocks": {
                        videos[1]['video']['id']: videos[1]['video']['block'],
                        videos[1]['text']['id']: videos[1]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[1]['video']['id'],
                            videos[1]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_1_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    blocks[col_2_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_2_item_1_id: {
                    "blocks": {
                        videos[2]['video']['id']: videos[2]['video']['block'],
                        videos[2]['text']['id']: videos[2]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[2]['video']['id'],
                            videos[2]['text']['id'],
                        ]
                    }
                },
                col_2_item_2_id: {
                    "blocks": {
                        videos[3]['video']['id']: videos[3]['video']['block'],
                        videos[3]['text']['id']: videos[3]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[3]['video']['id'],
                            videos[3]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_2_item_2_id,
                    col_2_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    blocks[col_3_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_3_item_1_id: {
                    "blocks": {
                        videos[4]['video']['id']: videos[4]['video']['block'],
                        videos[4]['text']['id']: videos[4]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[4]['video']['id'],
                            videos[4]['text']['id'],
                        ]
                    }
                },
                col_3_item_2_id: {
                    "blocks": {
                        videos[5]['video']['id']: videos[5]['video']['block'],
                        videos[5]['text']['id']: videos[5]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[5]['video']['id'],
                            videos[5]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_3_item_2_id,
                    col_3_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    for index in range(0, 10):
        layout['items'].pop()

    layout['items'].append(col_3_id)
    layout['items'].append(col_2_id)
    layout['items'].append(col_1_id)

    context.blocks = blocks
    context.blocks_layout = layout
    context._p_changed = True

@onpath('/help/tutorial-videos/index_html')
def fix_tutorial_videos(context):
    blocks = context.blocks
    layout = context.blocks_layout

    videos = {
        1:  {
            'text': {
                'id': layout['items'][-1],
                'block': blocks[layout['items'][-1]]
            },
            'video': {
                'id': layout['items'][-2],
                'block': blocks[layout['items'][-2]]
            }
        },
        2:  {
            'text': {
                'id': layout['items'][-3],
                'block': blocks[layout['items'][-3]]
            },
            'video': {
                'id': layout['items'][-4],
                'block': blocks[layout['items'][-4]]
            }
        },
        3:  {
            'text': {
                'id': layout['items'][-5],
                'block': blocks[layout['items'][-5]]
            },
            'video': {
                'id': layout['items'][-6],
                'block': blocks[layout['items'][-6]]
            }
        },
        4:  {
            'text': {
                'id': layout['items'][-7],
                'block': blocks[layout['items'][-7]]
            },
            'video': {
                'id': layout['items'][-8],
                'block': blocks[layout['items'][-8]]
            }
        },

    }

    col_1_id = make_uid()
    col_1_item_1_id = make_uid()
    col_1_item_2_id = make_uid()

    col_2_id = make_uid()
    col_2_item_1_id = make_uid()
    col_2_item_2_id = make_uid()

    blocks[col_1_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_1_item_1_id: {
                    "blocks": {
                        videos[1]['video']['id']: videos[1]['video']['block'],
                        videos[1]['text']['id']: videos[1]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[1]['video']['id'],
                            videos[1]['text']['id'],
                        ]
                    }
                },
                col_1_item_2_id: {
                    "blocks": {
                        videos[2]['video']['id']: videos[2]['video']['block'],
                        videos[2]['text']['id']: videos[2]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[2]['video']['id'],
                            videos[2]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_1_item_2_id,
                    col_1_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    blocks[col_2_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_2_item_1_id: {
                    "blocks": {
                        videos[3]['video']['id']: videos[3]['video']['block'],
                        videos[3]['text']['id']: videos[3]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[3]['video']['id'],
                            videos[3]['text']['id'],
                        ]
                    }
                },
                col_2_item_2_id: {
                    "blocks": {
                        videos[4]['video']['id']: videos[4]['video']['block'],
                        videos[4]['text']['id']: videos[4]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[4]['video']['id'],
                            videos[4]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_2_item_2_id,
                    col_2_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    for index in range(0, 8):
        layout['items'].pop()

    layout['items'].append(col_2_id)
    layout['items'].append(col_1_id)

    context.blocks = blocks
    context.blocks_layout = layout
    context._p_changed = True

@onpath('/news-archive')
def fix_news_archive(context):

    current_lang = context.absolute_url(relative=True).split('/')[-2]

    listing_uid = make_uid()
    title_uid = make_uid()

    context.blocks = {
        title_uid: {
            "@type": "title",
            "hideContentType": True
        },
        listing_uid: {
            "@type": "listing",
            "headlineTag": "h2",
            "block": make_uid(),
            "itemModel": {
                "@type": "item",
                "hasDate": True,
                "hasDescription": True,
                "hasImage": False,
                "hasLink": True,
                "maxDescription": 2,
                "maxTitle": 2,
                "titleOnImage": False
            },
            "query": [],
            "querystring": {
                "query": [
                    {
                        "i": "portal_type",
                        "o": "plone.app.querystring.operation.selection.any",
                        "v": [
                            "News Item",
                            "Link"
                        ]
                    },
                    {
                        "i": "path",
                        "o": "plone.app.querystring.operation.string.absolutePath",
                        "v": "/" + current_lang + "/news-archive"
                    }
                ],
                "sort_on": "effective",
                "sort_order": "descending",
                "sort_order_boolean": True
            },
            "variation": "summary"
        }
    }

    context.blocks_layout = {"items": [title_uid, listing_uid]}
    context._p_changed = True

def are_on_path(url, paths):
    for path in paths:
        if url.endswith(path):
            return True

def are_in_path(url, paths):
    for path in paths:
        if path in url:
            return True

def fix_read_more(context):
    url = context.absolute_url(relative=True)

    PATHS = [
        '/knowledge/tools/urban-ast',
        '/knowledge/tools/adaptation-support-tool',
        '/knowledge/eu-vulnerability/eu-vulnerability-to-cc-impacts-occurring-outside',
        '/countries-regions/transnational-regions/baltic-sea-region/adaptation',
        '/countries-regions/transnational-regions/carpathian-mountains'
    ]

    def get_columns_block_id(blocks):
        columns_block = {k for k, v in blocks.items() if v['@type'] == 'columnsBlock'}
        col_id = list(columns_block)[0]
        return col_id

    def get_read_more_block_id(blocks):
        read_more_block = {k for k, v in blocks.items() if v['@type'] == 'readMoreBlock'}
        if read_more_block:
            read_more_block_id = list(read_more_block)[0]
            return read_more_block_id
        else:
            return None

    if are_in_path(url, SECTOR_POLICIES):
        col_id = get_columns_block_id(context.blocks)
        col = context.blocks[col_id]
        first_col_id = col['data']['blocks_layout']['items'][0]
        first_col = col['data']['blocks'][first_col_id]
        col_items = first_col['blocks_layout']['items']
        read_more_block_id = get_read_more_block_id(first_col['blocks'])
        tiles = {k for k, v in first_col['blocks'].items()
                if v['@type'] == 'relevantAceContent' or v['@type'] == 'filterAceContent'}
        if read_more_block_id:
            read_more_index = col_items.index(read_more_block_id)
            col_items.pop(read_more_index)
            col_items.insert(len(col_items)-len(tiles), read_more_block_id) # insert before acecontent blocks
            first_col['blocks_layout']['items'] = col_items

    elif are_in_path(url, PATHS):
        items = context.blocks_layout['items']
        read_more_block_id = get_read_more_block_id(context.blocks)
        if read_more_block_id:
            read_more_index = items.index(read_more_block_id)
            items.pop(read_more_index)
            items.insert(len(items) -1, read_more_block_id) # insert before last block
            context.blocks_layout['items'] = items

    else:
        items = context.blocks_layout['items']
        read_more_block_id = get_read_more_block_id(context.blocks)
        if read_more_block_id:
            read_more_index = items.index(read_more_block_id)
            items.pop(read_more_index)
            items.append(read_more_block_id) # insert as last one
            context.blocks_layout['items'] = items

    context._p_changed = True

def fix_images_in_slate(content):
    # for links like "resolveuid/231312"
    pass

def extract_first_column(context):
    # pull out the content from the column, we have a different layout for these pages
    title_block_id = context.blocks_layout['items'][0]
    title_block = context.blocks[title_block_id]

    if len(context.blocks_layout['items']) == 1:
        # no content
        return

    column_block_id = context.blocks_layout['items'][1]
    column_block = context.blocks[column_block_id]

    if not column_block['@type'] == 'columnsBlock':
        return

    second_column_id = column_block['data']['blocks_layout']['items'][1]
    second_column = column_block['data']['blocks'][second_column_id]

    blocks = second_column['blocks']
    blocks[title_block_id] = title_block

    context.blocks = second_column['blocks']
    block_ids = [title_block_id] + second_column['blocks_layout']['items']
    context.blocks_layout = {"items": block_ids}

@inpath('knowledge/tools/urban-ast/')
def fix_uast(context):

    if not context.blocks:  # unmigrated content?
        return

    return extract_first_column(context)

@inpath('knowledge/tools/adaptation-support-tool')
def fix_ast(context):
    return extract_first_column(context)

def fix_layout_size(context):
    url = context.absolute_url(relative=True)

    if are_on_path(url, FULL_PAGE_PATHS):
        return

    if are_in_path(url, AST_PATHS):
        return

    layout_uid = make_uid()
    page_blocks = context.blocks
    page_blocks_layout = context.blocks_layout
    uids = page_blocks_layout['items'] + [layout_uid]
    blocks_layout = {"items":  uids}
    _blocks = {}
    _blocks[layout_uid] = {"@type": "layoutSettings", "layout_size": "narrow_view"}

    for (uid, block) in page_blocks.items():
        _blocks[uid] = block

    context.blocks = _blocks
    context.blocks_layout = blocks_layout
    context._p_changed = True

def fix_field_encoding(context):

    title = context.title
    if isinstance(title, str):
        title = title.decode('utf-8')
        context.title = title

    description = context.description
    if isinstance(description, str):
        description = description.decode('utf-8')
        context.description = description

content_fixers = [fix_field_encoding, fix_images_in_slate,
                  fix_climate_services_toc, fix_tutorial_videos, fix_uast,
                  fix_ast, fix_webinars, fix_read_more]
folder_fixers = [fix_field_encoding, fix_news_archive]

def fix_content(content):
    for fixer in content_fixers:
        fixer(content)

def fix_folder(context):
    for fixer in folder_fixers:
        fixer(context)

languages = [lang for lang in LANGUAGES if lang != 'en']

def getpath(obj):
    return "/" + obj.absolute_url(relative=1)

def exclude(obj):
    obj.exclude_from_nav = True
    obj.reindexObject()     # update_metadata=True - only on p6

def include(obj):
    obj.exclude_from_nav = False
    obj.reindexObject()     # update_metadata=True - only on p6

def is_top_level_path(path):
    for test_path in TOP_LEVEL:
        if path == test_path or path == '/cca' + test_path:
            return True
    return False

def exclude_content_from_navigation(site):
    main = site.restrictedTraverse('en')
    for oid, obj in main.contentItems():
        path = getpath(obj)
        if not is_top_level_path(path):
            exclude(obj)

            logger.debug("Excluded from nav: %s", path)

            intl_mgr = get_translation_manager(obj)
            for lang in languages:
                trans = intl_mgr.get_translation(lang)

                if trans is None:
                    continue

                exclude(trans)
                logger.debug("Excluded from nav: %s", getpath(trans))
        else:
            include(obj)

            # children
            # for child_path in top_level[path]:
            #     child = obj[child_path]
            #     exclude(child)

site_fixers = [
    exclude_content_from_navigation
]

def fix_site(site):
    for fixer in site_fixers:
        fixer(site)


import time
import subprocess
import os
import webbrowser

os.environ["DISPLAY"] = ":0"

while (True):
    try:

        libcamera_process = subprocess.Popen(
            "libcamera-hello -t 0 -f --shutter 400 --framerate 2 -f --brightness 0.3", shell=True)

        # os.system("ps -C chromium-browser -o pid=|xargs kill -9")
        os.system("ps -C gpicview -o pid=|xargs kill -9")

        time.sleep(10)

        os.system("ps -C libcamera-hello -o pid=|xargs kill -9")

        time.sleep(3)

        image_process = subprocess.Popen(
            "libcamera-still -o test.jpg  --shutter 400 --brightness 0.3  -n --immediate", shell=True)

        time.sleep(3)

        # webbrowser.open('test.jpg', new=0, autoraise=True)

        gpicview_process = subprocess.Popen("gpicview test.jpg", shell=True)

        time.sleep(10)

    except:

        print("Error")
        time.sleep(5)
        continue


"""kryptomaniak URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""

from django.contrib import admin
from kryptomain import views
from django.contrib.auth import views as auth_views

from django.conf.urls import include, url

urlpatterns = [
	url(r'^username/(?P<slug>[-\w]+)/$', views.AktywacjaKonta, name='AktywacjaKonta'),
	url(r'^newprzelew/', views.nowyPrzelew, name='nowyPrzelew'),
	url(r'^rejestracja/',views.rejestracja, name="rejestracja"),
	url(r'^blogUser/(?P<slug>[-\w]+)/$', views.blogUser, name='blogUser'),
	url(r'^blogPost/(?P<cid>\d+)/$', views.blogPost, name='blogPost'),
	url(r'^api/', views.api, name='api'),
	url(r'^apiPost/', views.apiPost, name='apiPost'),
	url(r'^blogszukaj', views.blogszukaj, name='blogszukaj'),
	url(r'^blog/', views.blog, name='blog'),
	url(r'^taggit/', include('taggit_selectize.urls')),
    url(r'^admin/', include(admin.site.urls)),
	url(r'^apiprzelew/', views.apiPrzelew, name='apiprzelew'),
	url(r'^apikategoria/', views.apikategoria, name='apikategoria'),
	url(r'^newtemat/', views.newtemat, name='newtemat'),
	url(r'^tag/(?P<slug>[-\w]+)/$', views.TagIndexView, name='tagged'),
	url(r'^logowanie/',views.logowanie, name="logowanie"),
	url(r'^login/$', auth_views.login, {'template_name' : 'kryptomain/index.html'}, name='login'),
	url(r'^logout/$', auth_views.logout, {'next_page' : 'index'}, name='logout'),
	url(r'^pdfgenview/',views.pdfgenview, name="pdfgenview"),
	url(r'^edytujPrzelew/(?P<cid>\d+)/$', views.edytujPrzelew, name='edytujPrzelew'),
	url(r'^deletePrzlew/(?P<cid>\d+)/$', views.deletePrzelew, name='usunPrzelew'),
	url(r'^deletekryptowaluta/(?P<cid>\d+)/$', views.deleteKryptowaluta, name='usunKryptowaluta'),
	url(r'^deletewaluta/(?P<cid>\d+)/$', views.walutaUsun, name='usunwaluta'),
	url(r'^deletekategoria/(?P<cid>\d+)/$', views.deletekategoria, name='deletekategoria'),
	url(r'^edytujkryptowaluta/(?P<cid>\d+)/$', views.edytujKryptowaluta, name='edytujKryptowaluta'),
	url(r'^edytujwaluta/(?P<cid>\d+)/$', views.edytujwaluta, name='edytujwaluta'),
	url(r'^edytujkategoria/(?P<cid>\d+)/$', views.edytujkategoria, name='edytujkategoria'),
	url(r'^newcryptocurr/', views.nowaKryptoWaluta, name='nowaKryptoWaluta'),

	url(r'^newcurr/', views.walutaNowa, name='nowawal'),
	url(r'^kategoria/', views.kategoria, name='kategoria'),
	url(r'^curr/', views.waluty, name='currens'),
	url(r'^przelewy/', views.przelewy, name='Przelewy'),
	url(r'^cryptocurr/', views.Kryptowaluty, name='cryptocurrens'),
	url(r'^home/', views.home),
    url(r'^info/', views.info, name='informacja'),
	url(r'^kontakt/', views.Kontakt,  name='kontakt'),
    url(r'^admin/', admin.site.urls),
	url(r'', views.index, name='index' ),

]


from datetime import datetime

import sqlalchemy as sa

from ..db.base import Base

class Todo(Base):
    __tablename__ = 'todos'

    title = sa.Column(sa.String(255), nullable=False)
    description = sa.Column(sa.Text)
    created_at = sa.Column(sa.DateTime, default=datetime.now())

    def __init__(self, title: str, desciption: str, **kwargs) -> None:
        self.title = title
        self.description = desciption


import bpy
import bmesh
import mathutils
import os

if os.path.exists(bpy.path.abspath("//map.dump")):
    os.remove(bpy.path.abspath("//map.dump"))
dumpFile=open(bpy.path.abspath("//map.dump"),"w+")

mapCollection = bpy.data.collections["Map"]
props = mapCollection.children["Props"]

for prop in props.all_objects:
    name = prop.name.split(".")[0]
    line = 'PROP;'
    line += name + ';'
    for f in prop.location:
        line += str(f) + ';'
    for f in prop.rotation_euler:
        line += str(f) + ';'
    for f in prop.scale:
        line += str(f) + ';'

    line += '\n'
    dumpFile.write(line)


"""Script for exporting planes as card data, copy paste to console."""

import os
import bpy
import numpy

def looks_like_a_plane(obj):
    if obj.type != "MESH":
        return False
    if len(obj.data.vertices) != 4:
        return False
    norm = obj.data.vertices[0].normal
    return all(v.normal == norm for v in obj.data.vertices)

def conform_plane_data(plane_object):
    wp_vertices = [
        plane_object.matrix_world @ v.co
        for v in plane_object.data.vertices
    ]
    origin = (
        wp_vertices[0]
        + wp_vertices[1]
        + wp_vertices[2]
        + wp_vertices[3]
    ) * 0.25
    dx = (wp_vertices[1] - wp_vertices[0])
    dy = (wp_vertices[2] - wp_vertices[0])
    axis_x = dx.normalized()
    axis_y = dy.normalized()
    axis_z = axis_x.cross(axis_y)
    scale_x = dx.length
    scale_y = dy.length
    return [
        axis_x.x, axis_x.y, axis_x.z,
        axis_y.x, axis_y.y, axis_y.z,
        axis_z.x, axis_z.y, axis_z.z,
        origin.x, origin.y, origin.z,
        # Scale acts as the local extent,
        # we don't bother with scale.z in this case
        0.5 * scale_x, 0.5 * scale_y, 0,
    ]

def run_card_export():
    assert bpy.data.is_saved, ".blend needs to be saved first"
    # Collect all planes in the scene
    plane_object = (
        obj
        for obj in bpy.data.objects
        if looks_like_a_plane(obj)
    )
    # Compact them into a single stream
    for_export = numpy.array(
        [conform_plane_data(plane) for plane in plane_object],
        dtype = numpy.float32
    ).tobytes()
    # And finally write it
    out_file = "{0}.card_data".format(
        os.path.splitext(bpy.data.filepath)[0]
    )
    print("Exporting {0} bytes [{1} cards] to {2}".format(
        len(for_export), len(for_export) // 60, out_file
    ))
    with open(out_file, "wb") as out_fp:
        out_fp.write(for_export)

run_card_export()


"""Feature extraction transformers for dimensionality reduction."""
from __future__ import annotations

from typing import Generic, Sequence, TypeVar, Union

import numpy as np
from numpy import ndarray
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.utils.validation import check_is_fitted as sklearn_check_is_fitted

from ...._utils import _classifier_fit_depth_methods
from ....exploratory.depth import Depth, ModifiedBandDepth
from ....representation.grid import FData

T = TypeVar("T", bound=FData)

class DDGTransformer(
    BaseEstimator,  # type: ignore
    TransformerMixin,  # type: ignore
    Generic[T],
):
    r"""Generalized depth-versus-depth (DD) transformer for functional data.

    This transformer takes a list of k depths and performs the following map:

    .. math::
        \mathcal{X} &\rightarrow \mathbb{R}^G \\
        x &\rightarrow \textbf{d} = (D_1^1(x), D_1^2(x),...,D_g^k(x))

    Where :math:`D_i^j(x)` is the depth of the point :math:`x` with respect to
    the data in the :math:`i`-th group using the :math:`j`-th depth of the
    provided list.

    Note that :math:`\mathcal{X}` is possibly multivariate, that is,
    :math:`\mathcal{X} = \mathcal{X}_1 \times ... \times \mathcal{X}_p`.

    Parameters:
        depth_method:
            The depth class or sequence of depths to use when calculating
            the depth of a test sample in a class. See the documentation of
            the depths module for a list of available depths. By default it
            is ModifiedBandDepth.

    Examples:
        Firstly, we will import and split the Berkeley Growth Study dataset

        >>> from skfda.datasets import fetch_growth
        >>> from sklearn.model_selection import train_test_split
        >>> dataset = fetch_growth()
        >>> fd = dataset['data']
        >>> y = dataset['target']
        >>> X_train, X_test, y_train, y_test = train_test_split(
        ...     fd, y, test_size=0.25, stratify=y, random_state=0)

        >>> from skfda.preprocessing.dim_reduction.feature_extraction import \
        ... DDGTransformer
        >>> from sklearn.pipeline import make_pipeline
        >>> from sklearn.neighbors import KNeighborsClassifier

        We classify by first transforming our data using the defined map
        and then using KNN

        >>> pipe = make_pipeline(DDGTransformer(), KNeighborsClassifier())
        >>> pipe.fit(X_train, y_train)
        Pipeline(steps=[('ddgtransformer',
                         DDGTransformer(depth_method=[ModifiedBandDepth()])),
                        ('kneighborsclassifier', KNeighborsClassifier())])

        We can predict the class of new samples

        >>> pipe.predict(X_test)
        array([1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1,
               1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1])

        Finally, we calculate the mean accuracy for the test data

        >>> pipe.score(X_test, y_test)
        0.875

    References:
        Cuesta-Albertos, J. A., Febrero-Bande, M. and Oviedo de la Fuente, M.
        (2017). The DDG-classifier in the functional setting.
        TEST, 26. 119-142.
    """

    def __init__(  # noqa: WPS234
        self,
        depth_method: Union[Depth[T], Sequence[Depth[T]], None] = None,
    ) -> None:
        self.depth_method = depth_method

    def fit(self, X: T, y: ndarray) -> DDGTransformer[T]:
        """Fit the model using X as training data and y as target values.

        Args:
            X: FDataGrid with the training data.
            y: Target values of shape = (n_samples).

        Returns:
            self
        """
        if self.depth_method is None:
            self.depth_method = ModifiedBandDepth()

        if isinstance(self.depth_method, Depth):
            self.depth_method = [self.depth_method]

        classes, class_depth_methods = _classifier_fit_depth_methods(
            X, y, self.depth_method,
        )

        self._classes = classes
        self.class_depth_methods_ = class_depth_methods

        return self

    def transform(self, X: T) -> ndarray:
        """Transform the provided data using the defined map.

        Args:
            X: FDataGrid with the test samples.

        Returns:
            Array of shape (n_samples, G).
        """
        sklearn_check_is_fitted(self)

        return np.transpose([
            depth_method.transform(X)
            for depth_method in self.class_depth_methods_
        ])


# -*- coding: utf-8 -*-
from flask import request

from app.api.mod_org.validate import validate_for_user_modify_org
from app.api.models.base import Resource
from app.api.models.org import Org
from app.api.utils import request_util, response
from app.api.utils.decorators import (
    audit_modify,
)
from app.exc import GulDanException
from . import org_blueprint
from .utils import ensure_org

def parse_org_modification_info(request):
    op_info = request_util.parse_request(request)
    http_body = op_info.http_body

    private = request_util.item_check(http_body, "private")
    lower_private = str(private).lower()
    if lower_private == "true":
        op_info.visibility = Resource.Visibility.PRIVATE
    elif lower_private == "false":
        op_info.visibility = Resource.Visibility.PUBLIC
    else:
        raise GulDanException().with_code(400).with_message(
                u"非法的可见性类型:{}".format(private)
        )

    return op_info

@audit_modify(resource_type=Resource.Type.ORG)
def org_modify(resource_id=None, visibility=None):
    return Org.update_visibility(
        resource_id,
        visibility
    )

@org_blueprint.route("/<int:org_id>", methods=["POST"])
@response.dict_response_deco
def modify_org(org_id):
    ensure_org(org_id)
    org_op_info = parse_org_modification_info(request)
    validate_for_user_modify_org(
        org_op_info.user_hash,
        org_id
    )

    org_modify(
        resource_id=org_id,
        visibility=org_op_info.visibility
    )

    return {
        "msg": "OK"
    }


# -*- coding: utf-8 -*-
"""
Created on Wed Feb  5 10:29:04 2020

@author: tengjunwan
"""
import numpy as np

def adjust_anchors(anchors, locs):
    """这个函数是将anchors通过rpn得到的locs进行校正，注意，这个校正是对所有的anchor
    进行的，不论输入的input的图片的anchor是否为正样本或者其他。
        anchors(array):维度为(#anchors, 4),4为(xmin,ymin,xmax,ymax)
        locs(array):维度为(#anchors, 4),4为rpn网络的输出(tx,ty,tw,th)
    """
    if anchors.shape[0] == 0:  # 如果传进来的anchors数量为0
        return np.zeros((0, 4), dtype=locs.dtype)  # 返回一个(0,4)维的array

    # 转换为（w,h,ctr_x,ctr_y）的形式
    anchors_width = anchors[:, 2] - anchors[:, 0]
    anchors_height = anchors[:, 3] - anchors[:, 1]
    anchors_ctr_x = anchors[:, 0] + 0.5 * anchors_width
    anchors_ctr_y = anchors[:, 1] + 0.5 * anchors_height

    tx = locs[:, 0]
    ty = locs[:, 1]
    tw = locs[:, 2]
    th = locs[:, 3]

    ctr_x = tx * anchors_width + anchors_ctr_x
    ctr_y = ty * anchors_height + anchors_ctr_y
    width = np.exp(tw) * anchors_width
    height =  np.exp(th) * anchors_height

    # 再将adjust后的boxes改为(xmin,ymin,xmax,ymax)的形式,取名为roi,region of
    # interst
    roi = np.zeros(locs.shape, dtype=locs.dtype)  # 即(#anchors, 4)
    roi[:, 0] = ctr_x - 0.5 * width  # xmin
    roi[:, 2] = ctr_x + 0.5 * width  # xmax
    roi[:, 1] = ctr_y - 0.5 * height # ymin
    roi[:, 3] = ctr_y + 0.5 * height # ymax
    #print(roi.shape)
    return roi

def non_maximum_suppression(roi, thresh):
    """他原来的代码写的好鸡儿复杂，似乎涉及到了cuda代码(cupy)的编写，我这里的代码
    主要是网页上的代码的编写，不涉及到gpu计算
    参数:
        roi(array):维度(m,4),m表示在调节anchor之后进行loc后的剪切以及去除过小边的
            box的个数,m<=#anchors
        thresh(int):nms算法中去除"重叠一起"的roi的阈值
    """
    xmin = roi[:, 0]
    ymin = roi[:, 1]
    xmax = roi[:, 2]
    ymax = roi[:, 3]

    areas = (xmax - xmin) * (ymax - ymin)
    keep = []  # 初始化保留的roi的index的list
    order = np.arange(roi.shape[0])  # [0,1,2...,m-1]
    while order.size > 0:
        i = order[0]  # 取最大的score的roi的index,第一次遍历i=0
        keep.append(i)  # 收入keep中
        xx1 = np.maximum(xmin[i], xmin[order[1:]]) # (m-1,),m-1为剩下的roi
        xx2 = np.minimum(xmax[i], xmax[order[1:]])
        yy1 = np.maximum(ymin[i], ymin[order[1:]])
        yy2 = np.minimum(ymax[i], ymax[order[1:]])

        width = np.maximum(0.0, xx2 - xx1)
        height =  np.maximum(0.0, yy2 - yy1)
        inter = width * height  # (m-1,)挑出的roi和剩下的roi的重叠面积
        iou = inter / (areas[i] + areas[order[1:]] - inter)  # (m-1,)

        idx = np.where(iou <= thresh)[0]  # 表示剩下的roi(不满足一定重叠率)
        order = order[1 + idx]  # 剔除score最大以及iou重叠率的roi剩下roi的index

    roi_after_nms = roi[keep]
    return roi_after_nms, keep

class ProposalCreator():
    """
    通过调用内置函数__call__来通过rpn网络的locs值来校正anchors的位置,主要是完成
    rpn网络之后的nms工作(non maximum supression)
    参数:
        nms_thresh:
        n_train_pre_nums(int):在train阶段，进行nms之前保留的top anchors的数量/。
        n_train_post_nms(int):在test阶段，进行nms之后保留的top boxes的数量(基于
            anchor的校正)。
        n_test_pre_nms(int):同理，test阶段不同的设置。
        n_test_post_nms(int):同理，test阶段不同的设置。
        min_size(int):如果修正后的boxes的最小边小于该值,舍弃,防止roipool层切割得到
           某个维度为0.
    """
    def __init__(self,
                 parent_model,  # 传入rpn网络
                 nms_thresh=0.7,
                 n_train_pre_nms=12000,
                 n_train_post_nms=2000,
                 n_test_pre_nms=6000,
                 n_test_post_nms=300,
                 min_size=16
                 ):
        self.parent_model = parent_model
        self.nms_thresh = nms_thresh
        self.n_train_pre_nms = n_train_pre_nms
        self.n_train_post_nms = n_train_post_nms
        self.n_test_pre_nms = n_test_pre_nms
        self.n_test_post_nms = n_test_post_nms
        self.min_size = min_size

    def __call__(self, locs, scores, anchors, img_size):
        """
        参数:
            locs(array):维度(#anchors, 4),从rpn网络的输出rpn_locs传入
            scores(array):维度(#anchors, ),从rpn网络的输出rpn_scores传入
            anchors(array):维度(#anchors, 4),从数据中读取
            img_size(tuple of ints):(宽，高),从数据中读取
        返回:
            roi(array):维度为(#roi, 4),4为(xmin,ymin,xmax,ymax),如果是在training
                阶段,#roi=2000,如果是eval阶段,#roi=300
        """
        # self.parent_model是rpn,如果rpn.eval()会使得rpn.training = False
        if self.parent_model.training:
            n_pre_nms = self.n_train_pre_nms  # 12000
            n_post_nms = self.n_train_post_nms  # 2000
        else:
            n_pre_nms = self.n_test_pre_nms  # 6000
            n_post_nms = self.n_test_post_nms # 300

        # 先默认训练模式(v.10该回了上面的判断)
#        n_pre_nms = self.n_train_pre_nms  # 12000
#        n_post_nms = self.n_train_post_nms  # 2000

        roi = adjust_anchors(anchors, locs)  # (#anchors, 4),(xmin,ymin,xmax,ymax)

        # 剪切,因为校正的关系，会出现出界的情况
        roi[:, [0, 2]] = np.clip(roi[:, [0, 2]], 0, img_size[0])  # x轴剪切
        roi[:, [1, 3]] = np.clip(roi[:, [1, 3]], 0, img_size[1])  # y轴剪切
        #print("check1", roi.shape)
        # 去除 高 或 宽 < min_size的roi
        min_size = self.min_size
        roi_width = roi[:, 2] - roi[:, 0]
        roi_height = roi[:, 3] - roi[:, 1]
        keep = np.where((roi_width >= min_size) &
                        (roi_height >= min_size))[0]  # 满足条件的行index
        roi = roi[keep, :]

        #print("check2", roi.shape)

        # 对roi通过rpn的cls scores进行排序
        scores = scores[:, 1]  # 取第二列为“是object”的评分
        scores = scores[keep]  # (#keep,) 这里 #keep <= #anchors
        order = scores.argsort()[::-1]  # 得到scores的下降排列的坐标
        order = order[: n_pre_nms]   # 保留固定数量 训练 12000
        #print(order.shape)
        roi = roi[order]  # 从scores高到低的排序的roi
        #print("check3", roi.shape)  # 这里还是12000
        # nms
        roi_after_nms, _ = non_maximum_suppression(roi, thresh=self.nms_thresh)
        roi_after_nms = roi_after_nms[:n_post_nms]  # 保留固定数量 训练2000
        return roi_after_nms  # roi

if __name__ == "__main__":
    path = 'pretrained_model/checkpoints/vgg16-397923af.pth'
    from vgg16_to_rpn import Vgg16ToRPN
    vgg16torpn = Vgg16ToRPN(path)
    import torch
    x = torch.rand((1, 3, 600, 850))  # 第一张图片的大小

    rpn_locs, rpn_scores, anchors = vgg16torpn(x)  # tensor,tensor,numpy
    # 除去第一个batch维度
    # 因为我这里propoal的大部分操作是numpy操作
    rpn_locs = rpn_locs[0].detach().numpy()
    rpn_scores = rpn_scores[0].detach().numpy()

    proposalcreator = ProposalCreator()
    size = (600, 850)
    roi = proposalcreator(rpn_locs, rpn_scores, anchors, size)

    print(roi)  # (1653, 4)实际上不足2000，这种不足是在nms中导致的


import sys

def fact(n):

    if n > 1:
        return n*fact(n-1)
    else :
        return 1
a = int(sys.stdin.readline())
print(fact(a))

# fact (10) = 10*fact(9)
# fact(9) = 9*fact(8)
#
#
#
# fact(2) = 2*fact(1)
# fact(1) = 1

#%%


from django import template
from products.models import Color
register = template.Library()

additional_categories = {'Доп Аксессуары 1':'accessory_1_title', 'Доп Аксессуары 2':'accessory_2_title', 'Доп Аксессуары 3':'accessory_3_title', 'Доп Аксессуары 4':'accessory_4_title', 'Доп Аксессуары 5':'accessory_5_title', 'Доп Гаджеты 1':'gadget_1_title',
                         'Доп Гаджеты 2':'gadget_2_title', 'Доп Гаджеты 3':'gadget_3_title', 'Доп Гаджеты 4':'gadget_4_title', 'Доп Гаджеты 5':'gadget_5_title'}
@register.filter
def get_color_name(value):
    return value.name

@register.filter
def get_category_name(value, categories):
    if categories:
        categories = categories.__dict__
        if value in additional_categories:
            value = categories[additional_categories[value]]
    return value


import pandas as pd
import subprocess
import os

# skip_first = 30
# how_many = 68
#
# submit = True
# overwrite = False
# df = pd.read_csv('../CHO_train_compounds.csv')

setup = """mol_non_opt_fn='mol_non_opt.xyz'
mol_rad_non_opt_fn='mol_rad_non_opt.xyz'
opt_fn="opt_mol_rad.xyz"
nm_fn='mol_rad_nms.xyz'

scratch_path=/tmp/eg475
opt_workdir_root=orca_opt
orca_simple_input="UKS B3LYP def2-SV(P) def2/J D3BJ"
smearing=5000
opt_calc_kwargs="task=opt smearing=${smearing}"

n_run=1
n_hop=1
nm_calc_kwargs="orcasimpleinput='${orca_simple_input}'  scratch_path='$scratch_path' n_run='${n_run}' n_hop='${n_hop}' smearing='$smearing'"

mkdir -p $scratch_path
export  OMP_NUM_THREADS=${NSLOTS}
export AUTOPARA_NPOOL=${NSLOTS}
"""

generate_str = """
conda activate wo
wfl -v generate-configs smiles -o ${mol_non_opt_fn} -i "config_type=${config_type}" $smiles
wfl -v generate-configs remove-sp3-Hs -o ${mol_rad_non_opt_fn} ${mol_non_opt_fn}
"""

optimise_str = """
conda activate py3.8
echo 'optimising'
wfl -v ref-method orca-eval --output-prefix 'dft_' --calc-kwargs "${opt_calc_kwargs}" --scratch-path $scratch_path --base-rundir $opt_workdir_root  --output-file ${opt_fn}  ${mol_rad_non_opt_fn}
"""

modes = """
conda activate wo
echo 'generating normal modes'
wfl -v generate-configs derive-normal-modes --calc-kwargs "${nm_calc_kwargs}" --calc-name orca -p dft_ --parallel-hessian -o ${nm_fn} ${opt_fn}
"""

def sub_data(df_name, how_many, skip_first, submit, overwrite_sub,
             generate=False, optimise=False, modes=False,
             hours=48, no_cores=16, script_name='sub.sh'):

    df = pd.read_csv(df_name)

    homedir = os.getcwd()
    for idx, row in df.iterrows():

        if skip_first is not None and idx < skip_first:
            continue
        if idx == how_many:
            break

        name = row['Name']
        smi = row['SMILES']

        if not os.path.isdir(name):
            os.makedirs(name)

        if not overwrite_sub:
            if os.path.isdir(os.path.join(name, script_name)):
                continue

        os.chdir(name)

        print(f'making {name} with {smi}')

        with open(script_name, 'w') as f:
            f.write(f"""#!/bin/bash
#$ -pe smp {no_cores}        # number of cores requested
#$ -l h_rt={hours}:00:00  # time requested in HH:MM:SS format
#$ -S /bin/bash      # shell to run the job in
#$ -N j{name}
#$ -j yes            # combine error and output logs
#$ -cwd              # execute job in directory from which it was submitted

echo $(date)
""")
            f.write(f'smiles="{smi}"\n')
            f.write(f'config_type={name}\n')

            f.write(setup)

            if generate:
                f.write(generate_str)
            if optimise:
                f.write(optimise_str)
            if modes:
                f.write(modes_str)

        if submit:
            subprocess.run(f"qsub {script_name}", shell=True)

        os.chdir(homedir)

    print('Finished script')