import operator
import numpy as np
import pandas as pd
import os
import lightgbm as lgb
import optuna
from sklearn.model_selection import train_test_split
from copy import copy
from functools import partial
import plotly.express as px
import warnings
warnings.filterwarnings("ignore")
import matplotlib.pyplot as plt
from verstack.tools import timer, Printer
from verstack.lgbm_optuna_tuning.lgb_metrics import get_pruning_metric, get_optimization_metric_func, define_objective, classification_metrics, regression_metrics, get_eval_score, print_lower_greater_better, supported_metrics
from verstack.lgbm_optuna_tuning.args_validators import *
from verstack.lgbm_optuna_tuning.optuna_tools import Distribution, OPTUNA_DISTRIBUTIONS_MAP, SearchSpace
# BACKLOG: add option to pass different init params on class __init__
# BACKLOG: add option to pass different param_grid for optimization
# OPTIONAL TODO - add factorization to targets and inverse_transform predictions for classification (E.g. Ghosts, prudential)

supported_gridsearch_params = [
    'boosting_type', 'num_iterations', 'learning_rate', 'num_leaves', 'max_depth',
    'min_data_in_leaf', 'min_sum_hessian_in_leaf', 'bagging_fraction', 'feature_fraction',
    'max_delta_step', 'lambda_l1', 'lambda_l2', 'linear_lambda', 'min_gain_to_split',
    'drop_rate', 'top_rate', 'min_data_per_group', 'max_cat_threshold'
    ]

class LGBMTuner:

    __version__ = '1.3.1'

    def __init__(self, **kwargs):
        '''
        Class to automatically tune LGBM model with optuna.

        Model type (regressor/classifier) is inferred based on target variable & metric.
        Init parameters and search space are inferred by built in logic.

        Parameters
        ----------
        metric : str
            evaluation metric name.
        trials : int, optional
            number of hyperparameter search trials. The default is 200.
        refit : bool, optional
            Flag to refit the model with optimized parameters on whole dataset.
            The default is True.
        verbosity : int, optional
            console verbosity level: 0 - no output except for optuna CRITICAL errors and builtin exceptions;
            (1-5) based on optuna.logging options. The default is 1.
        visualization : bool, optional
            flag to print optimization & feature importance plots. The default is True.
        seed : int
            random_state.
        device_type : str
            cpu/gpu/cuda/cuda_exp.
        custom_lgbm_params : dict
            custom lgbm parameters to be passed to the model, please refer to LGBM documentation for available parameters.
        eval_results_callback : func
            callback function to be applied on the eval_results dictionary that is being populated
            with evaluation metric score upon completion of each training trial.
        grid: dict
            Parameters search space for optimization.

            A default search space is populated at fit, depending on number of trials:
            "feature_fraction" = (Distribution.UNIFORM, low=0.5, high=1.0)
            "num_leaves" = (Distribution.INTUNIFORM, low=16, high=255)
            if self.trials > 30:
                "bagging_fraction" = (Distribution.UNIFORM, low=0.5, high=1.0)
                "min_sum_hessian_in_leaf" = (Distribution.LOGUNIFORM, low=1e-3, high=10.0)
            if self.trials > 100:
                "lambda_l1" = (Distribution.LOGUNIFORM, low=1e-8, high=10.0)
                "lambda_l2" = (Distribution.LOGUNIFORM, low=1e-8, high=10.0)

            Grid can be modified with the following parameters to be included in the Search Space:
                boosting_type : 'gbdt', 'dart', 'rf'
                num_iterations : >= 0
                learning_rate : > 0.0
                num_leaves : 1 < num_leaves <= 131072
                max_depth :
                min_data_in_leaf : >= 0
                min_sum_hessian_in_leaf : >= 0.0
                bagging_fraction : 0.0 < bagging_fraction <= 1.0
                feature_fraction : 0.0 < feature_fraction <= 1.0
                max_delta_step :
                lambda_l1 : >=0
                lambda_l2 : >=0
                linear_lambda : >=0
                min_gain_to_split : >=0.0
                drop_rate : 0.0 <= drop_rate <= 1.0
                top_rate : 0.0 <= top_rate <= 1.0
                min_data_per_group : > 0
                max_cat_threshold : > 0

        Returns
        -------
        None.

        '''
        self.verbosity = kwargs.get('verbosity', 1)
        self.printer = Printer(verbose=True if self.verbosity > 0 else False)
        self.metric = kwargs.get('metric')
        self.trials = kwargs.get('trials', 100)
        self.refit = kwargs.get('refit', True)
        self.visualization = kwargs.get('visualization', True)
        self.seed = kwargs.get('seed', 42)
        self.device_type = kwargs.get('device_type', 'cpu')
        self.custom_lgbm_params = kwargs.get('custom_lgbm_params', {})
        self.target_minimum = None
        self._fitted_model = None
        self._feature_importances = None
        self._study = None # save optuna study for plotting
        self.target_classes = None
        self._init_params = None
        self._best_params = None
        self.eval_results = {} # evaluation metric results per each trial storage
        self.eval_results_callback = kwargs.get('eval_results_callback', None)
        self.search_space = self._get_default_search_space()
        self.grid = self._get_all_available_and_defined_grids()
        self.early_stopping_results = {} # stores early_stopping results per each trial

        # get user defined grid

    def _get_all_available_and_defined_grids(self):
        all_grids = {}
        for param in supported_gridsearch_params:
            if param in self.search_space:
                all_grids[param] = self.search_space[param].params
            else:
                all_grids[param] = None
        return all_grids

    # print init parameters when calling the class instance
    def __repr__(self):
        return f'LGBMTuner(Evaluation metric: {self._metric}\
            \n          trials: {self.trials}\
            \n          refit: {self.refit}\
            \n          verbosity: {self.verbosity}\
            \n          visualization: {self.visualization})\
            \n          device_type: {self.device_type})\
            \n          grid: {self.grid})'

    # Validate init arguments
    # =========================================================================
    # define cmmon functions for setters
    def _is_bool(self, val):
        return type(val) == bool

    def _is_int(self, val):
        return type(val) == int
    # -------------------------------------------------------------------------
    # metric
    @property
    def metric(self):
        return self._metric

    @metric.setter
    def metric(self, value):
        if type(value) != str : raise TypeError('metric must be a string')
        if value not in supported_metrics: raise KeyError(f'LGBMTuner supports the following evaluation metrics: {supported_metrics}')
        self._metric = value
    # -------------------------------------------------------------------------
    # trials
    @property
    def trials(self):
        return self._trials

    @trials.setter
    def trials(self, value):
        if not self._is_int(value) : raise TypeError('trials must be an integer')
        if value <= 0: raise ValueError('trials must be an integer > 0')
        self._trials = value
    # -------------------------------------------------------------------------
    # refit
    @property
    def refit(self):
        return self._refit

    @refit.setter
    def refit(self, value):
        if not self._is_bool(value) : raise TypeError('acceptable refit options are True/False')
        self._refit = value
    # -------------------------------------------------------------------------
    # verbosity
    @property
    def verbosity(self):
        return self._verbosity

    @verbosity.setter
    def verbosity(self, value):
        if not self._is_int(value) : raise TypeError('verbosity must be an integer')
        if value not in range(0,6) : raise ValueError('acceptable verbosity options are [0,1,2,3,4,5]')
        self._verbosity = value
    # -------------------------------------------------------------------------
    # visualization
    @property
    def visualization(self):
        return self._visualization

    @visualization.setter
    def visualization(self, value):
        if not self._is_bool(value) : raise TypeError('acceptable visualization options are True/False')
        self._visualization = value
    # -------------------------------------------------------------------------
    # device_type
    @property
    def device_type(self):
        return self._device_type

    @device_type.setter
    def device_type(self, value):
        if not value in ['cpu', 'gpu', 'cuda', 'cuda_exp'] : raise TypeError('acceptable device_type options are cpu/gpu/cuda/cuda_exp')
        self._device_type = value
    # -------------------------------------------------------------------------
    # seed
    @property
    def seed(self):
        return self._seed

    @seed.setter
    def seed(self, value):
        if not self._is_int(value) : raise TypeError('seed must be an integer')
        self._seed = value
    # -------------------------------------------------------------------------
    # only getters for the following arguments
    # -------------------------------------------------------------------------
    # init_params
    @property
    def init_params(self):
        return self._init_params
    # -------------------------------------------------------------------------
    # best_params
    @property
    def best_params(self):
        return self._best_params
    # -------------------------------------------------------------------------
    # feature_importances
    @property
    def feature_importances(self):
        if not self._fitted_model:
            raise AttributeError('LGBMTuner.fit(refit = True) must be applied before feature_importances can be displayed')
        return self._feature_importances
    # -------------------------------------------------------------------------
    # fitted_model
    @property
    def fitted_model(self):
        return self._fitted_model
    # -------------------------------------------------------------------------
    # study
    @property
    def study(self):
        return self._study
    # -------------------------------------------------------------------------
    # eval_results_callback
    @property
    def eval_results_callback(self):
        return self._eval_results_callback

    @eval_results_callback.setter
    def eval_results_callback(self, value):
        if not value is None:
            if not hasattr(value, '__call__') : raise TypeError('eval_results_callback must be a function')
        self._eval_results_callback = value
    # -------------------------------------------------------------------------
    def _init_params_on_input(self, rows_num, y):
        '''
        Get model parameters depending on dataset parameters.

        Parameters
        ----------
        rows_num : int
            number of rows in training data.
        y : pd.Series
            train labels

        Returns
        -------
        None.

        '''
        # TODO: use number of features

        # default lgbm parameters - basis for minor changes based on dataset length
        default_params_classification = {"task": "train",
                                        "learning_rate": 0.05,
                                        "num_leaves": 128,
                                        "feature_fraction": 0.7,
                                        "bagging_fraction": 0.7,
                                        "bagging_freq": 1,
                                        "max_depth": -1,
                                        "verbosity": -1,
                                        "lambda_l1": 1,
                                        "lambda_l2": 0.0,
                                        "min_split_gain": 0.0,
                                        "zero_as_missing": False,
                                        "max_bin": 255,
                                        "min_data_in_bin": 3,
                                        "num_iterations": 10000,
                                        "early_stopping_rounds": 100,
                                        "random_state": 42,
                                        "device_type": self.device_type}

        default_params_regression = {"learning_rate": 0.05,
                                    "num_leaves": 32,
                                    "feature_fraction": 0.9,
                                    "bagging_fraction": 0.9,
                                    "verbosity": -1,
                                    "num_iterations": 10000,
                                    "early_stopping_rounds": 100,
                                    "random_state": 42,
                                    "device_type": self.device_type}

        task = define_objective(self.metric, y)

        if task == 'regression':
            self._init_params = copy(default_params_regression)
        else:
            self._init_params = copy(default_params_classification)

        # define additional params based on task type
        if task == 'binary':
            self._init_params['num_classes'] = 1
        elif task == 'multiclass':
            self._init_params['num_classes'] = y.nunique()
        # -------------------------------------------------------------------------
        # populate objective and metric based
        self._init_params['objective'] = define_objective(self.metric, self.target_classes)
        self._init_params['metric'] = get_pruning_metric(self.metric, self.target_classes)
        # -------------------------------------------------------------------------
        # do not use all available threads and make sure cpu_count is not negative
        cpu_count = int(np.where(os.cpu_count()-2 < 0, 1, os.cpu_count()-2))

        self._init_params['num_threads'] = cpu_count

        if rows_num <= 10000:
            init_lr = 0.01
            ntrees = 3000
            es = 200
        elif rows_num <= 20000:
            init_lr = 0.02
            ntrees = 3000
            es = 200
        elif rows_num <= 100000:
            init_lr = 0.03
            ntrees = 1200
            es = 200
        elif rows_num <= 300000:
            init_lr = 0.04
            ntrees = 2000
            es = 100
        else:
            init_lr = 0.05
            ntrees = 2000
            es = 100

        if rows_num > 300000:
            self._init_params["num_leaves"] = 128 if task == "regression" else 244
        elif rows_num > 100000:
            self._init_params["num_leaves"] = 64 if task == "regression" else 128
        elif rows_num > 50000:
            self._init_params["num_leaves"] = 32 if task == "regression" else 64
            # params['lambda_l1'] = 1 if task == 'reg' else 0.5
        elif rows_num > 20000:
            self._init_params["num_leaves"] = 32 if task == "regression" else 32
            self._init_params["lambda_l1"] = 0.5 if task == "regression" else 0.0
        elif rows_num > 10000:
            self._init_params["num_leaves"] = 32 if task == "regression" else 64
            self._init_params["lambda_l1"] = 0.5 if task == "regression" else 0.2
        elif rows_num > 5000:
            self._init_params["num_leaves"] = 24 if task == "regression" else 32
            self._init_params["lambda_l1"] = 0.5 if task == "regression" else 0.5
        else:
            self._init_params["num_leaves"] = 16 if task == "regression" else 16
            self._init_params["lambda_l1"] = 1 if task == "regression" else 1

        self._init_params["learning_rate"] = init_lr
        self._init_params["num_iterations"] = ntrees
        self._init_params["early_stopping_rounds"] = es
    # -----------------------------------------------------------------------------

    def _get_default_search_space(self):
        '''Sample hyperparameters from suggested

        Parameters
        ----------
        estimated_n_trials : int
            number of optuna trails.

        Returns
        -------
        search_space : dict
            optimization search space grid and type of distribution for each parameter.

        '''
        # TODO: create addditional options based on bigger estimated_n_trials
        search_space = {}

        search_space["feature_fraction"] = SearchSpace(Distribution.UNIFORM, low=0.5, high=1.0)
        search_space["num_leaves"] = SearchSpace(Distribution.INTUNIFORM, low=16, high=255)

        if self.trials > 30:
            search_space["bagging_fraction"] = SearchSpace(Distribution.UNIFORM, low=0.5, high=1.0)
            search_space["min_sum_hessian_in_leaf"] = SearchSpace(Distribution.LOGUNIFORM, low=1e-3, high=10.0)

        if self.trials > 100:
            search_space["lambda_l1"] = SearchSpace(Distribution.LOGUNIFORM, low=1e-8, high=10.0)
            search_space["lambda_l2"] = SearchSpace(Distribution.LOGUNIFORM, low=1e-8, high=10.0)

        return search_space
    # -----------------------------------------------------------------------------

    def _sample_from_search_space(self, trial):
        '''
        Get params for a trial.

        Parameters
        ----------
        trial : optuna.Trial
            trial object.
        suggested_params : dict
            init params on input.

        Raises
        ------
        ValueError
            Optuna distribution error.

        Returns
        -------
        trial_values : dict
            trial parameters consisting from suggested_params modified by grid.

        '''
        trial_values = copy(self._init_params)
        for parameter, SearchSpace in self.search_space.items():
            if SearchSpace.distribution_type in OPTUNA_DISTRIBUTIONS_MAP:
                trial_values[parameter] = getattr(trial, OPTUNA_DISTRIBUTIONS_MAP[SearchSpace.distribution_type])(
                    name=parameter, **SearchSpace.params)
            else:
                for key, value in self.search_space.items():
                    print(key)
                    print(value.distribution_type)
                raise ValueError(f"Optuna does not support distribution {SearchSpace.distribution_type}")
        return trial_values
    # -----------------------------------------------------------------------------

    def _get_dtrain_dvalid_objects(self, X, y, metric, seed = None, return_raw_valid = False):
        '''
        Create lgbm.Datasets for training and validation.

        By default splits without defined random_state in order to replicate CV (sort of).
        Seed is used for optimize_num_iterations.

        Parameters
        ----------
        X : pd.DataFrame
            train features.
        y : pd.Series
            train labels.
        metric : str
            evaluation metric name.
        seed : int/None
            random state for split
        return_raw_valid : bool
            Flag to return valid_x, valid_y in addition to dvalid (for get_validation_score())

        Returns
        -------
        dtrain : lgbm.Dataset
            train data.
        dvalid : ldgb.Dataset
            valid data.

        '''
        if metric in classification_metrics:
            train_x, valid_x, train_y, valid_y = train_test_split(X, y, test_size=0.25, stratify = y)
        else:
            train_x, valid_x, train_y, valid_y = train_test_split(X, y, random_state = seed, test_size=0.25)
        dtrain = lgb.Dataset(train_x, label=train_y)
        dvalid = lgb.Dataset(valid_x, label=valid_y)
        if return_raw_valid:
            return dtrain, dvalid, valid_x, valid_y
        else:
            return dtrain, dvalid
    # -----------------------------------------------------------------------------
    def fit_optimized(self, X, y):
        '''
        Train model with tuned params on whole train data

        Parameters
        ----------
        X : np.array
            train features.
        y : np.array
            train target.

        Returns
        -------
        None.

        '''
        validate_numpy_ndarray_arguments(X)
        validate_numpy_ndarray_arguments(y)

        import lightgbm as lgb
        self.printer.print('Fitting optimized model with the follwing params:', order=2)
        for key, value in self._best_params.items():
            print(f'{key:<33}: {value}')
        self._fitted_model = lgb.train(self._best_params, lgb.Dataset(X,y))

    # ------------------------------------------------------------------------------------------
    def _get_target_minimum(self, y):
        '''Record target minimum value for replacing negative predictions in regression.'''
        if self.metric in regression_metrics:
            self.target_minimum = min(y)

    # ------------------------------------------------------------------------------------------
    def _get_validation_score(self, trial, dtrain, dvalid, valid_x, valid_y, optimization_metric_func, params, pruning_callback):
        '''
        Train model with trial params and validate on valid set against the defined metric.
        Print optimization result every iteration.
        If evaluation metric != optimization metric, print evaluation metric.

        Parameters
        ----------
        trial : optuna.Trial #1
            parameters tuning iteration.
        dtrain : lgb.Dataset
            train data.
        dvalid : lgb.Dataset
            valid data.
        valid_x : pd.DataFrame
            valid features.
        valid_y : pd.Series
            valid target.
        optimization_metric_func : func
            scorer function.
        params : dict #1
            model parameters.
        pruning_callback : func
            callback function.

        Returns
        -------
        result : float
            validation result.

        '''
        gbm = lgb.train(params, dtrain, valid_sets=[dvalid], callbacks=[pruning_callback])
        pred = gbm.predict(valid_x)

        result = optimization_metric_func(valid_y, pred)

        optimization_direction = 'lower-better'

        self.printer.print(f'Trial number: {trial.number} finished', order=3)
        self.printer.print(f'Optimization score ({optimization_direction:<4}): {optimization_metric_func.__name__}: {result}', order=4)
        # save early stopping results per each trial for further use in best_params
        self.early_stopping_results[trial.number] = gbm.best_iteration
        # save evaluation metric results per each trial
        self.eval_results[f'train_trial_{trial.number}'] = result
        # calculate & print eval_metric only if eval_metric != optimization_metric
        if self.metric != optimization_metric_func.__name__:
            eval_score = get_eval_score(valid_y, pred, self.metric, params['objective'])
            self.printer.print(f'Evaluation score ({print_lower_greater_better(self.metric):<4}): {self.metric}: {eval_score}', order=4)
            # save evaluation metric results per each trial
            self.eval_results[f'train_trial_{trial.number}'] = eval_score
        self.printer.print(breakline='.')

        if self.eval_results_callback:
            self.eval_results_callback(self.eval_results)

        return result

    # ------------------------------------------------------------------------------------------
    def _objective(self, trial, X, y):
        '''
        Define objective for optuna tiral training.

        - Create optimization metric based on evaluatin metric passed by user
        - Create train/valid splits
        - Define suggested initial parameters for LGB model based on data and eveluation metric
        - Define the search space for some params within suggested parameters
        - Create final params dict based on suggested initial params and the a step in optimization search space grid
        - Define a pruning callback to stop training of unpromissed trails
        - Train a given trail & validate by optimization_metric_func

        Parameters
        ----------
        trial : optuna.Trial
            training step.
        X : pd.DataFrame
            train features.
        y : pd.Series
            train labels.
        metric : str
            evaluation metric name.

        Returns
        -------
        result : float
            optimization metric validation result.

        '''
        optimization_metric_func = get_optimization_metric_func(self.metric)
        dtrain, dvalid, valid_x, valid_y = self._get_dtrain_dvalid_objects(X, y, self.metric, return_raw_valid = True)
        params = self._sample_from_search_space(trial)
        # Add a callback for pruning.
        pruning_callback = optuna.integration.LightGBMPruningCallback(trial, get_pruning_metric(self.metric, self.target_classes))
        result = self._get_validation_score(trial, dtrain, dvalid, valid_x, valid_y, optimization_metric_func, params, pruning_callback)

        return result
    # ------------------------------------------------------------------------------------------
    def _check_refit_status(self, method):
        if not self.refit:
            raise ModuleNotFoundError(f'LGBMTuner.fit(refit = True) must be applied before {method} execution')
    # ------------------------------------------------------------------------------------------

    def predict(self, test, threshold = 0.5):
        '''
        Predict on test set with fitted model.

        For regression replace negative preds by self.target_minimum (if train labels
        were not negative).

        Parameters
        ----------
        test : pd.DataFrame
            test features.
        threshold : float, optional
            binary classification probability threshold. The default is 0.5.

        Returns
        -------
        pred : array
            predicted values.

        '''
        self._check_refit_status('predict()')
        validate_features_argument(test)
        validate_threshold_argument(threshold)

        pred = self._fitted_model.predict(test.values, num_threads = self._init_params['num_threads'])

        # regression predict
        if self.metric in regression_metrics:
            if self.target_minimum >= 0:
                pred = np.where(pred < 0, self.target_minimum, pred)
            return pred
        else:
            # binary classification predict
            if self._fitted_model.params['objective'] == 'binary':
                pred = (pred > threshold).astype('int')
            # multiclass classification predict
            # TODO: SAVE CLASSES NAMES TO APPLY THEM BACK
            else:
                pred = np.argmax(pred, axis = 1)
            return pred
    # ------------------------------------------------------------------------------------------

    def predict_proba(self, test):
        '''
        Predict on test set (classification only) and return probabilities.

        Parameters
        ----------
        test : pd.DataFrame
            test features.

        Raises
        ------
        TypeError
            If self._fitted_model.params['objective'] == 'regression - notify
            that predict_proba() is for classification objectives only.

        Returns
        -------
        pred : array
            predicted values.

        '''
        self._check_refit_status('predict_proba()')
        validate_features_argument(test)

        if self.metric in regression_metrics:
            raise TypeError('predict_proba() is applicable for classification problems only')
        pred = self._fitted_model.predict(test.values, num_threads = self._init_params['num_threads'])
        return pred
    # ------------------------------------------------------------------------------------------

    def _populate_best_params_to_init_params(self, best_params):
        '''Populate the learned params into the suggested params'''
        # output params are temporary, because num_iterations tuning will follow
        temp_params = copy(self._init_params)
        for key, val in best_params.items():
            temp_params[key] = val
        # remove early_stopping & num_iterations from params (used during optuna optimization).
        # final early stopping will be trained during final_estimators_tuning
        del temp_params['early_stopping_rounds']
        del temp_params['num_iterations']

        return temp_params
    # ------------------------------------------------------------------------------------------

    def _save_feature_importances(self, train_features):
        '''Save feature importances in class instance as a pd.Series'''
        feat_importances = pd.Series(self._fitted_model.feature_importance(), index = train_features)
        normalized_importances = np.round((lambda x: x/sum(x))(feat_importances),5)
        self._feature_importances = normalized_importances
    # ------------------------------------------------------------------------------------------

    def __configure_matplotlib_style(self,
                                     dark = True):
        '''Configure matplotlib style for plots'''
        try:
            styles = plt.style.available
            if dark:
                styles_to_set = ['dark_background']
                for style in styles:
                    if 'deep' in style:
                        styles_to_set.append(style)
            else:
                styles_to_set = []
                for style in styles:
                    if 'pastel' in style:
                        styles_to_set.append(style)
            for style in styles_to_set:
                plt.style.use(style)
        except Exception as e:
            print(f'Error while configuring matplotlib style: {e}')
            print('Default style will be used')
            pass
    # ------------------------------------------------------------------------------------------

    def _plot_static_fim(self,
                        feat_imp,
                        figsize = (10,6),
                        dark=True,
                        save=False,
                        display=True):
        """
        Plot feature importance as a horizontal bar chart.

        Parameters
        ----------
        feat_imp : pd.Series
            feature importance values.
        figsize : tuple, optional
            figure size. The default is (10,6).
        dark : bool, optional
            dark theme. The default is True.
        save : bool, optional
            save figure. The default is False.
        display : bool, optional
            display figure. The default is True.

        Returns
        -------
        None.

        """
        fig, ax = plt.subplots(figsize=figsize)
        plt.tight_layout()
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_visible(False)
        ax.spines['bottom'].set_visible(False)
        self.__configure_matplotlib_style(dark)

        if dark:
            name = 'FIM_DARK'
            ax.barh(feat_imp.index, feat_imp, alpha=0.8, color='#F99245')
            fig.set_facecolor('#20253c')
            ax.set_facecolor('#20253c')
            ax.tick_params(axis='x', colors='white')
            ax.tick_params(axis='y', colors='white')
            ax.set_title('Feature importances (sum up to 1)', color='white')
        else:
            name = 'FIM_LIGHT'
            ax.barh(feat_imp.index, feat_imp, alpha=0.8, color='#007bff')
            fig.set_facecolor('#dee0eb')
            ax.set_facecolor('#dee0eb')
            ax.tick_params(axis='x', colors='#212529')
            ax.tick_params(axis='y', colors='#212529')
            ax.set_title('Feature importances (sum up to 1)', color='#212529')
        if display:
            plt.show()
        if save:
            plt.savefig(f'{name}.png', dpi=300, facecolor=fig.get_facecolor(),
                        edgecolor='none', bbox_inches='tight')
            print(f'Feature Importance Plot is saved to {os.path.join(os.getcwd(), f"{name}.png")}')
    # ------------------------------------------------------------------------------------------
    def _plot_interactive_fim(self, plot_df, dark=True, save=False, display=True, plotly_fig_update_layout_kwargs={}):
        '''
        Create and save interactive plotly plot.

        Parameters
        ----------
        plot_df : pd.Series
            Series with features names and importances.
        dark : bool, optional
            Dark theme. The default is True.
        save : bool, optional
            Save plot to working directory. The default is False.
        display : bool, optional
            Display plot in console or in browser (for interactive plots). The default is True.
        plotly_fig_update_layout_kwargs : dict, optional
            Dictionary with plotly layout parameters. The default is {}.

        Returns
        -------
        None.

        '''

        BAR_COLOR = '#F99245'
        fig = px.bar(plot_df,
                        x = 'importance',
                        y = 'feature',
                        orientation = 'h',
                        color_discrete_sequence = [BAR_COLOR]*len(plot_df),
                        text = 'importance_percent')
        default_plotly_dark_fig_update_layout_kwargs = {'plot_bgcolor':'#20253c', # plot color
                                                        'paper_bgcolor':'#2d3250', # html color
                                                        'font_color':'white',
                                                        'title_font_color':'lightgrey',
                                                        'xaxis':{'visible':False,
                                                                 'showticklabels':False,
                                                                 'showgrid':False},
                                                        'yaxis':{'showticklabels':True,
                                                                 'title':''}}
        default_plotly_light_fig_update_layout_kwargs = {'plot_bgcolor': '#dee0eb',
                                                         'paper_bgcolor': '#fff',
                                                         'font_color': '#34383d',
                                                         'title_font_color': 'lightgrey',
                                                         'xaxis': {'visible': False,
                                                                   'showticklabels': False,
                                                                   'showgrid': False},
                                                         'yaxis': {'showticklabels': True,
                                                                   'title': ''}}
        # set plotly update_layout kwargs
        if plotly_fig_update_layout_kwargs:
            kwargs = plotly_fig_update_layout_kwargs
        else:
            if dark:
                name = 'FIM_DARK.html'
                kwargs = default_plotly_dark_fig_update_layout_kwargs
            else:
                name = 'FIM_LIGHT.html'
                kwargs = default_plotly_light_fig_update_layout_kwargs
        fig.update_layout(**kwargs)
        fig.update_traces(textposition='inside',
                            marker_line_color=BAR_COLOR)
        if display:
            try:
                fig.write_html(name, config={'displaylogo': False})
                self._display_html(name)
                if not save:
                    os.remove(name)
            except Exception as e:
                print(f'Display html error: {e}')
        if save:
            fig.write_html(name, config={'displaylogo': False})
            print(f'Feature Importance Plot is saved to {os.path.join(os.getcwd(), name)}')
    # ------------------------------------------------------------------------------------------
    def _display_html(self, html_file):
        '''Run html plot in the default browser'''
        import webbrowser
        import time
        # 1st method how to open html files in chrome using
        filename = 'file:///'+os.getcwd()+'/' + html_file
        webbrowser.open_new_tab(filename)
        time.sleep(2)
        os.remove(html_file)
    # ------------------------------------------------------------------------------------------

    def plot_importances(self,
                         n_features=15,
                         figsize=(10,6),
                         interactive=False,
                         display=True,
                         dark=True,
                         save=True,
                         plotly_fig_update_layout_kwargs={}):
        '''
        Plot feature importances.

        Can plot interactive html plot or static png plot.

        Parameters
        ----------
        n_features : int, optional
            Number of features to plot. The default is 15.
        figsize : tuple, optional
            Figure size. The default is (10,6).
        interactive : bool, optional
            Create & save to current wd interactive html plot. The default is False.
        display: bool, optional
            Display plot in browser. If False, plot will be saved in cwd. The default is True.
        dark: bool, optional
            Display dark/light version of plot. The default is True.
        save: bool, optional
            Save plot to cwd. The default is True.
        plotly_fig_update_layout_kwargs: dict, optional
            kwargs for plotly.fig.update_layout() function. The default is empty dict and default_plotly_fig_update_layout_kwargs will be used

        Returns
        -------
        None.

        '''
        self._check_refit_status('plot_importances()')
        validate_plot_importances_n_features_argument(n_features)
        validate_plot_importances_figsize_argument(figsize)
        validate_plotting_interactive_argument(interactive)
        if interactive:
            importances_for_html_plot = pd.DataFrame(self._feature_importances.nlargest(n_features).sort_values(), columns = ['importance'])
            importances_for_html_plot['feature'] = importances_for_html_plot.index
            importances_for_html_plot.reset_index(drop = True, inplace = True)
            importances_for_html_plot['importance_percent'] = [str(np.round(val*100,3))+'%' for val in importances_for_html_plot['importance']]
            self._plot_interactive_fim(importances_for_html_plot,
                                       dark=dark,
                                       save=save,
                                       display=display,
                                       plotly_fig_update_layout_kwargs=plotly_fig_update_layout_kwargs)
        else:
            importances_for_png_plot = self._feature_importances.nlargest(n_features).sort_values()
            self._plot_static_fim(importances_for_png_plot,
                                  figsize = figsize,
                                  dark=dark,
                                  save=save,
                                  display=display)
    # ------------------------------------------------------------------------------------------

    def plot_optimization_history(self, interactive=False, display=True):
        '''
        Plot parameters optimization history.

        Parameters
        ----------
        interactive : bool, optional
            Create & save to current wd interactive html plot. The default is False.
        display: bool, optional
            Display plot in browser. If False, plot will be saved in cwd. The default is True.

        Returns
        -------
        None.

        '''
        self._check_refit_status('plot_optimization_history()')
        validate_plotting_interactive_argument(interactive)

        if interactive:
            from optuna.visualization import plot_optimization_history
            fig = plot_optimization_history(self._study)
            fig.write_html("optimization_history_plot.html")
            if display:
                try:
                    self._display_html("optimization_history_plot.html")
                except Exception as e:
                    print(f'Display html error: {e}')
                    print(f'Optimization History Plot is saved to {os.path.join(os.getcwd(), "optimization_history_plot.html")}')
        else:
            from optuna.visualization.matplotlib import plot_optimization_history
            import matplotlib.pyplot as plt
            plot_optimization_history(self._study)
            plt.show()
    # ------------------------------------------------------------------------------------------

    def plot_param_importances(self, interactive=False, display=True):
        '''
        Plot parameters importance.

        Parameters
        ----------
        interactive : bool, optional
            Create & save to current wd interactive html plot. The default is False.
        display: bool, optional
            Display plot in browser. If False, plot will be saved in cwd. The default is True.

        Returns
        -------
        None.

        '''
        self._check_refit_status('plot_param_importance()')
        validate_plotting_interactive_argument(interactive)

        if interactive:
            from optuna.visualization import plot_param_importances
            fig = plot_param_importances(self._study)
            fig.write_html("param_importances_plot.html")
            if display:
                try:
                    self._display_html('param_importances_plot.html')
                except Exception as e:
                    print(f'Display html error: {e}')
                    print(f'Param Importances Plot is saved to {os.path.join(os.getcwd(), "param_importances_plot.html")}')
        else:
            from optuna.visualization.matplotlib import plot_param_importances
            import matplotlib.pyplot as plt
            plot_param_importances(self._study)
            plt.show()
    # ------------------------------------------------------------------------------------------

    def plot_intermediate_values(self, interactive=False, legend=False, display=True):
        '''
        Plot optimization trials history. Shows successful and terminated trials.

        Parameters
        ----------
        interactive : bool, optional
            Create & show in default browsersave to current wd interactive html plot. The default is False.
        legend : bool, optional
            Flag to include legend in the static (not interactive) plot. The default is False.
        display: bool, optional
            Display plot in browser. If False, plot will be saved in cwd. The default is True.

        Returns
        -------
        None.

        '''
        self._check_refit_status('plot_intermediate_values()')
        validate_plotting_interactive_argument(interactive)
        validate_plotting_legend_argument(legend)

        if interactive:
            from optuna.visualization import plot_intermediate_values
            fig = plot_intermediate_values(self._study)
            fig.write_html("intermediate_values_plot.html")
            if display:
                try:
                    self._display_html('intermediate_values_plot.html')
                except Exception as e:
                    print(f'Display html error: {e}')
                    print(f'Intermediate Values Plot is saved to {os.path.join(os.getcwd(), "intermediate_values_plot.html")}')
        else:
            from optuna.visualization.matplotlib import plot_intermediate_values
            import matplotlib.pyplot as plt
            fig = plot_intermediate_values(self._study)
            if not legend:
                fig.get_legend().remove()
            plt.show()
    # ------------------------------------------------------------------------------------------

    def _set_optuna_verbosity(self, value):
        '''Set optimizaiton console output verbosity based on optuna.logging options'''
        value_to_optuna_verbosity_dict = {0 : 'CRITICAL',
                                          1 : 'CRITICAL',
                                          2 : 'ERROR',
                                          3 : 'WARNING',
                                          4 : 'INFO',
                                          5 : 'DEBUG'}
        optuna.logging.set_verbosity(getattr(optuna.logging, value_to_optuna_verbosity_dict[value]))
    # ------------------------------------------------------------------------------------------

    def _contains_float(self, iterable):
        '''Check if float any floats are in iterable'''
        return float in [type(x) for x in iterable]

    def _all_ints(self, iterable):
        '''Check if iterable contains only ints'''
        return np.all([type(x)==int for x in iterable])

    def _align_grid_and_search_space(self):
        '''Redefine self.search_space for optuna based on self.grid which could be amended by user'''
        unsupported_params = []
        for param_name, param_grid in self.grid.items():
            if param_name not in supported_gridsearch_params:
                unsupported_params.append(param_name)
                continue
            if isinstance(param_grid, list):
                self.search_space[param_name] = SearchSpace(Distribution.CHOICE, choices = param_grid)
            elif isinstance(param_grid, tuple):
                if self._contains_float(param_grid):
                    self.search_space[param_name] = SearchSpace(Distribution.UNIFORM, low = min(param_grid), high = max(param_grid))
                elif self._all_ints(param_grid):
                    self.search_space[param_name] = SearchSpace(Distribution.INTUNIFORM, low = min(param_grid), high = max(param_grid))
            elif isinstance(param_grid, dict):
                if 'choices' in param_grid:
                    self.search_space[param_name] = SearchSpace(Distribution.CHOICE, choices = param_grid['choices'])
                elif 'low' in param_grid:
                    if self._contains_float(param_grid.values()):
                        self.search_space[param_name] = SearchSpace(Distribution.UNIFORM, low = param_grid['low'], high = param_grid['high'])
                    elif self._all_ints(param_grid.values()):
                        self.search_space[param_name] = SearchSpace(Distribution.INTUNIFORM, low = param_grid['low'], high = param_grid['high'])
        if unsupported_params:
            self.printer.print(f'Following changed parameters are not supported for tuning: {unsupported_params}', order = 'error', trailing_blank_paragraph=True)
        self.grid = {key: value for key, value in self.grid.items() if key not in unsupported_params}

    @timer
    def fit(self, X, y, optuna_study_params = None):
        '''
        Find optimized parameters for LightGBM model based on training data and
        metric (defined by user at __init__).

        Considering the lower-better strategy:
            Regression optimization metric
                is selected based on eval_metric (passed by user at __init__), except for r2.
                If eval_metric == 'r2', then optimization metric is 'mean_squared_error'.
            Classification optimization metric
                Allways log_loss

        LGB Classifier/Regressor is inferred based on metric (defined by user at __init__)
        and target variable statistics.

        Initial LGBM parameters are inferred based on data stats & built in logic and are accesable
        by self._init_params

        Param_grid for hyperparameters search is inferred based on data stats & built in logic.

        Parameters
        ----------
        X : pd.DataFrame
            train features.
        y : pd.Series
            train labels.
        optuna_study_params : dict, optional
            Parameters for optuna study. The default is None. Custom optuna.study.create_study parameters
            https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html

        Returns
        -------
        None.

        '''
        validate_features_argument(X)
        validate_target_argument(y)

        optimization_metric_func = get_optimization_metric_func(self.metric)

        self.printer.print('Initiating LGBMTuner.fit', order=1)
        self.printer.print('Settings:', order=3)
        self.printer.print(f'Trying {self.trials} trials', order=4)
        self.printer.print(f'Evaluation metric: {self.metric} ', order=4)
        self.printer.print(f'Study direction: minimize {optimization_metric_func.__name__}', order=4)#{get_study_direction(metric)} {optimization_metric_func.__name__})
        print()

        self.target_classes = y.unique().tolist()
        self._get_target_minimum(y)
        self._init_params_on_input(len(X), y)
        # update the predefined params with custom params passed by user
        self._init_params.update(self.custom_lgbm_params)
        self._align_grid_and_search_space()
        self._set_optuna_verbosity(self.verbosity)

        sampler = optuna.samplers.TPESampler(seed=self.seed)

        optuna_params = {'pruner':optuna.pruners.MedianPruner(n_warmup_steps=5),
                         'sampler':sampler,
                         'direction':'minimize'}
        # incorporate user defined params if passed
        if optuna_study_params is not None:
            optuna_params.update(optuna_study_params)
        study = optuna.create_study(**optuna_params)#get_study_direction(metric))

        optimization_function = partial(self._objective, X = X.values, y = y.values)

        study.optimize(optimization_function, n_trials = self.trials)

        # populate the learned params into the suggested params
        temp_params = self._populate_best_params_to_init_params(study.best_params)
        # extract early stopping results from best trial

        best_trial_number = study.best_trial.number
        num_iterations_in_best_trial = self.early_stopping_results[best_trial_number]
        temp_params['num_iterations'] = num_iterations_in_best_trial

        # tune num_iterations
        # iteration, best_score = self.optimize_num_iterations(X.values, y.values, temp_params)
        # temp_params['num_iterations'] = iteration
        self._best_params = temp_params
        if self.refit:
            self.fit_optimized(X.values, y.values)
            self._save_feature_importances(X.columns)
        self._study = study

        if self.visualization:
            self.plot_optimization_history()
            self.plot_param_importances()
            self.plot_intermediate_values()
            if self.refit:
                self.plot_importances(dark=False, save=False)
        # clean up
        self.eval_results_callback = None
        # --------------------------------
        break_symbol = '|'
        print()
        self.printer.print(f"Optuna hyperparameters optimization finished", order=3)
        self.printer.print(f"Best trial number:{study.best_trial.number:>2}{break_symbol:>5}     {optimization_metric_func.__name__}:{study.best_trial.value:>29}", order=4, breakline='-')


import collections
import copy
import datetime
import os
from typing import Any, Dict, List, Mapping, Optional, Sequence, Union

from absl import logging
from tfx import types
from tfx import version
from tfx.dsl.io import fileio
from tfx.orchestration import data_types_utils
from tfx.orchestration import node_proto_view
from tfx.proto.orchestration import execution_result_pb2
from tfx.proto.orchestration import pipeline_pb2
from tfx.types import artifact as tfx_artifact
from tfx.types import artifact_utils
from tfx.types.value_artifact import ValueArtifact
from tfx.utils import proto_utils

from ml_metadata.proto import metadata_store_pb2

_SYSTEM = '.system'
_EXECUTOR_EXECUTION = 'executor_execution'
_DRIVER_EXECUTION = 'driver_execution'
_STATEFUL_WORKING_DIR = 'stateful_working_dir'
_DRIVER_OUTPUT_FILE = 'driver_output.pb'
_EXECUTOR_OUTPUT_FILE = 'executor_output.pb'
_VALUE_ARTIFACT_FILE_NAME = 'value'
# The fixed special value to indicate that the binary will set the output URI
# value during its execution.
# LINT.IfChange
RESOLVED_AT_RUNTIME = '{resolved_at_runtime}'
# LINT.ThenChange(<Internal source code>)
_ORCHESTRATOR_GENERATED_BCL_DIR = 'orchestrator_generated_bcl'

def make_output_dirs(
    output_dict: Mapping[str, Sequence[types.Artifact]]) -> None:
  """Make dirs for output artifacts' URI."""
  for _, artifact_list in output_dict.items():
    for artifact in artifact_list:
      # Omit lifecycle management for external artifacts.
      if artifact.is_external:
        continue
      if isinstance(artifact, ValueArtifact):
        # If this is a ValueArtifact, create the file if it does not exist.
        if not fileio.exists(artifact.uri):
          artifact_dir = os.path.dirname(artifact.uri)
          fileio.makedirs(artifact_dir)
          with fileio.open(artifact.uri, 'w') as f:
            # Because fileio.open won't create an empty file, we write an
            # empty string to it to force the creation.
            f.write('')
      else:
        # Otherwise create a dir.
        fileio.makedirs(artifact.uri)

def remove_output_dirs(
    output_dict: Mapping[str, Sequence[types.Artifact]]) -> None:
  """Remove dirs of output artifacts' URI."""
  for _, artifact_list in output_dict.items():
    for artifact in artifact_list:
      # Omit lifecycle management for external artifacts.
      if artifact.is_external:
        continue
      if fileio.isdir(artifact.uri):
        fileio.rmtree(artifact.uri)
      else:
        fileio.remove(artifact.uri)

def clear_output_dirs(
    output_dict: Mapping[str, Sequence[types.Artifact]]) -> None:
  """Clear dirs of output artifacts' URI."""
  for _, artifact_list in output_dict.items():
    for artifact in artifact_list:
      # Omit lifecycle management for external artifacts.
      if artifact.is_external:
        continue
      # Clear out the contents of the output directory while preserving the
      # output directory itself. Needed to preserve any storage attributes of
      # the output directory.
      if not fileio.isdir(artifact.uri):
        continue
      child_paths = [
          os.path.join(artifact.uri, filename)
          for filename in fileio.listdir(artifact.uri)
      ]
      for path in child_paths:
        if fileio.isdir(path):
          fileio.rmtree(path)
        else:
          fileio.remove(path)

def remove_stateful_working_dir(stateful_working_dir: str) -> None:
  """Remove stateful_working_dir."""
  # Clean up stateful working dir
  # Note that:
  # stateful_working_dir = os.path.join(
  #    self._node_dir,
  #    _SYSTEM,
  #    _STATEFUL_WORKING_DIR, <-- we want to clean from this level down.
  #    dir_suffix)
  stateful_working_dir = os.path.abspath(
      os.path.join(stateful_working_dir, os.pardir))
  try:
    fileio.rmtree(stateful_working_dir)
  except fileio.NotFoundError:
    logging.warning(
        'stateful_working_dir %s is not found, not going to delete it.',
        stateful_working_dir)

def _attach_artifact_properties(spec: pipeline_pb2.OutputSpec.ArtifactSpec,
                                artifact: types.Artifact):
  """Attaches properties of an artifact using ArtifactSpec."""
  for key, value in spec.additional_properties.items():
    if not value.HasField('field_value'):
      raise RuntimeError('Property value is not a field_value for %s' % key)
    if value.field_value.HasField('proto_value'):
      # Proto properties need to be unpacked from the google.protobuf.Any
      # message to its concrete message before setting the artifact property
      property_value = proto_utils.unpack_proto_any(
          value.field_value.proto_value)
    else:
      property_value = data_types_utils.get_metadata_value(value.field_value)
    setattr(artifact, key, property_value)

  for key, value in spec.additional_custom_properties.items():
    if not value.HasField('field_value'):
      raise RuntimeError('Property value is not a field_value for %s' % key)
    value_type = value.field_value.WhichOneof('value')
    if value_type == 'int_value':
      artifact.set_int_custom_property(key, value.field_value.int_value)
    elif value_type == 'string_value':
      artifact.set_string_custom_property(key, value.field_value.string_value)
    elif value_type == 'double_value':
      artifact.set_float_custom_property(key, value.field_value.double_value)
    elif value_type == 'proto_value':
      proto_value = proto_utils.unpack_proto_any(value.field_value.proto_value)
      artifact.set_proto_custom_property(key, proto_value)
    else:
      raise RuntimeError(f'Unexpected value_type: {value_type}')

def _get_node_dir(
    pipeline_runtime_spec: pipeline_pb2.PipelineRuntimeSpec, node_id: str
) -> str:
  """Gets node dir for the given pipeline node."""
  return os.path.join(
      pipeline_runtime_spec.pipeline_root.field_value.string_value, node_id
  )

class OutputsResolver:
  """This class has methods to handle launcher output related logic."""

  def __init__(self,
               pipeline_node: Union[pipeline_pb2.PipelineNode,
                                    node_proto_view.NodeProtoView],
               pipeline_info: pipeline_pb2.PipelineInfo,
               pipeline_runtime_spec: pipeline_pb2.PipelineRuntimeSpec,
               execution_mode: 'pipeline_pb2.Pipeline.ExecutionMode' = (
                   pipeline_pb2.Pipeline.SYNC)):
    self._pipeline_node = pipeline_node
    self._pipeline_info = pipeline_info
    self._pipeline_root = (
        pipeline_runtime_spec.pipeline_root.field_value.string_value)
    self._pipeline_run_id = (
        pipeline_runtime_spec.pipeline_run_id.field_value.string_value)
    self._execution_mode = execution_mode
    self._node_dir = _get_node_dir(
        pipeline_runtime_spec, pipeline_node.node_info.id
    )

  def generate_output_artifacts(
      self, execution_id: int) -> Dict[str, List[types.Artifact]]:
    """Generates output artifacts given execution_id."""
    return generate_output_artifacts(
        execution_id=execution_id,
        outputs=self._pipeline_node.outputs.outputs,
        node_dir=self._node_dir,
        pipeline_root=self._pipeline_root)

  def get_executor_output_uri(self, execution_id: int) -> str:
    """Generates executor output uri given execution_id."""
    return get_executor_output_uri(self._node_dir, execution_id)

  def get_driver_output_uri(self) -> str:
    driver_output_dir = os.path.join(
        self._node_dir, _SYSTEM, _DRIVER_EXECUTION,
        str(int(datetime.datetime.now().timestamp() * 1000000)))
    fileio.makedirs(driver_output_dir)
    return os.path.join(driver_output_dir, _DRIVER_OUTPUT_FILE)

  def get_stateful_working_directory(self,
                                     execution_id: Optional[int] = None) -> str:
    """Generates stateful working directory given (optional) execution id.

    Args:
      execution_id: An optional execution id which will be used as part of the
        stateful working dir path if provided. The stateful working dir path
        will be <node_dir>/.system/stateful_working_dir/<execution_id>. If
        execution_id is not provided, for backward compatibility purposes,
        <pipeline_run_id> is used instead of <execution_id> but an error is
        raised if the execution_mode is not SYNC (since ASYNC pipelines have no
        pipeline_run_id).

    Returns:
      Path to stateful working directory.

    Raises:
      ValueError: If execution_id is not provided and execution_mode of the
        pipeline is not SYNC.
    """
    return get_stateful_working_directory(self._node_dir, self._execution_mode,
                                          self._pipeline_run_id, execution_id)

  def make_tmp_dir(self, execution_id: int) -> str:
    """Generates a temporary directory."""
    return make_tmp_dir(self._node_dir, execution_id)

def _generate_output_artifact(
    output_spec: pipeline_pb2.OutputSpec) -> types.Artifact:
  """Generates each output artifact given output_spec."""
  artifact = artifact_utils.deserialize_artifact(output_spec.artifact_spec.type)
  _attach_artifact_properties(output_spec.artifact_spec, artifact)

  if output_spec.artifact_spec.is_async:
    # Mark the artifact state as REFERENCE to distinguish it from PUBLISHED
    # (LIVE in MLMD) intermediate artifacts emitted during a component's
    # execution. At the end  of the component's execution, its state will remain
    # REFERENCE instead of changing to PUBLISHED.
    artifact.state = tfx_artifact.ArtifactState.REFERENCE

  return artifact

def _validate_external_uri(external_uri: str,
                           pipeline_root: Optional[str]) -> None:
  """Validates a user-defined external artifact URI."""
  if external_uri == RESOLVED_AT_RUNTIME:
    return

  if pipeline_root and pipeline_root in external_uri:
    raise ValueError('External artifact URI %s is not allowed within the '
                     'pipeline base directory.' % external_uri)

def generate_output_artifacts(
    execution_id: int,
    outputs: Mapping[str, pipeline_pb2.OutputSpec],
    node_dir: str,
    pipeline_root: Optional[str] = None) -> Dict[str, List[types.Artifact]]:
  """Generates output artifacts.

  Args:
    execution_id: The id of the execution.
    outputs: Mapping from artifact key to its OutputSpec value in pipeline IR.
    node_dir: The root directory of the node.
    pipeline_root: Path to root directory of the pipeline.

  Returns:
    Mapping from artifact key to the list of TFX artifacts.

  Raises:
    ValueError: If any external artifact uri is inside the pipeline_root.
  """

  output_artifacts = collections.defaultdict(list)
  for key, output_spec in outputs.items():
    artifact = _generate_output_artifact(output_spec)
    if output_spec.artifact_spec.external_artifact_uris:
      for external_uri in output_spec.artifact_spec.external_artifact_uris:
        _validate_external_uri(external_uri, pipeline_root)
        external_artifact = copy.deepcopy(artifact)
        external_artifact.uri = external_uri
        external_artifact.is_external = True

        logging.debug('Creating external output artifact uri %s',
                      external_artifact.uri)
        output_artifacts[key].append(external_artifact)

    else:
      artifact.uri = os.path.join(node_dir, key, str(execution_id))
      if isinstance(artifact, ValueArtifact):
        artifact.uri = os.path.join(artifact.uri, _VALUE_ARTIFACT_FILE_NAME)

      logging.debug('Creating output artifact uri %s', artifact.uri)
      output_artifacts[key].append(artifact)

  return output_artifacts

def get_executor_output_uri(node_dir, execution_id: int) -> str:
  """Generates executor output uri given execution_id."""
  # LINT.IfChange(execution_output_uri)
  execution_dir = os.path.join(node_dir, _SYSTEM, _EXECUTOR_EXECUTION,
                               str(execution_id))
  fileio.makedirs(execution_dir)
  return os.path.join(execution_dir, _EXECUTOR_OUTPUT_FILE)
  # LINT.ThenChange(<Internal source code for executor_output_uri>)

def get_stateful_working_directory(node_dir: str,
                                   execution_mode: pipeline_pb2.Pipeline
                                   .ExecutionMode = pipeline_pb2.Pipeline.SYNC,
                                   pipeline_run_id: str = '',
                                   execution_id: Optional[int] = None) -> str:
  """Generates stateful working directory.

  Args:
    node_dir: The root directory of the node.
    execution_mode: Execution mode of the pipeline.
    pipeline_run_id: Optional pipeline_run_id, only available if execution mode
      is SYNC.
    execution_id: An optional execution id which will be used as part of the
      stateful working dir path if provided. The stateful working dir path will
      be <node_dir>/.system/stateful_working_dir/<execution_id>. If execution_id
      is not provided, for backward compatibility purposes, <pipeline_run_id> is
      used instead of <execution_id> but an error is raised if the
      execution_mode is not SYNC (since ASYNC pipelines have no
      pipeline_run_id).

  Returns:
    Path to stateful working directory.

  Raises:
    ValueError: If execution_id is not provided and execution_mode of the
      pipeline is not SYNC.
  """
  if (execution_id is None and execution_mode != pipeline_pb2.Pipeline.SYNC):
    raise ValueError(
        'Cannot create stateful working dir if execution id is `None` and '
        'the execution mode of the pipeline is not `SYNC`.')

  if execution_id is None:
    dir_suffix = pipeline_run_id
  else:
    dir_suffix = str(execution_id)

  # TODO(b/150979622): We should introduce an id that is not changed across
  # retries of the same component run to provide better isolation between
  # "retry" and "new execution". When it is available, introduce it into
  # stateful working directory.
  # NOTE: If this directory structure is changed, please update
  # the remove_stateful_working_dir function in this file accordingly.
  stateful_working_dir = os.path.join(node_dir, _SYSTEM, _STATEFUL_WORKING_DIR,
                                      dir_suffix)
  try:
    fileio.makedirs(stateful_working_dir)
  except Exception:  # pylint: disable=broad-except
    logging.exception('Failed to make stateful working dir: %s',
                      stateful_working_dir)
    raise
  return stateful_working_dir

def make_tmp_dir(node_dir: str, execution_id: int) -> str:
  """Generates a temporary directory."""
  result = os.path.join(node_dir, _SYSTEM, _EXECUTOR_EXECUTION,
                        str(execution_id), '.temp', '')
  fileio.makedirs(result)
  return result

def tag_output_artifacts_with_version(
    output_artifacts: Optional[Mapping[str, Sequence[types.Artifact]]] = None):
  """Tag output artifacts with the current TFX version."""
  if not output_artifacts:
    return
  for unused_key, artifact_list in output_artifacts.items():
    for artifact in artifact_list:
      if not artifact.has_custom_property(
          artifact_utils.ARTIFACT_TFX_VERSION_CUSTOM_PROPERTY_KEY):
        artifact.set_string_custom_property(
            artifact_utils.ARTIFACT_TFX_VERSION_CUSTOM_PROPERTY_KEY,
            version.__version__)

def populate_output_artifact(
    executor_output: execution_result_pb2.ExecutorOutput,
    output_dict: Mapping[str, Sequence[types.Artifact]]):
  """Populate output_dict to executor_output."""
  for key, artifact_list in output_dict.items():
    artifacts = execution_result_pb2.ExecutorOutput.ArtifactList()
    for artifact in artifact_list:
      artifacts.artifacts.append(artifact.mlmd_artifact)
    executor_output.output_artifacts[key].CopyFrom(artifacts)

def populate_exec_properties(
    executor_output: execution_result_pb2.ExecutorOutput,
    exec_properties: Mapping[str, Any]):
  """Populate exec_properties to executor_output."""
  for key, value in exec_properties.items():
    v = metadata_store_pb2.Value()
    if isinstance(value, str):
      v.string_value = value
    elif isinstance(value, int):
      v.int_value = value
    elif isinstance(value, float):
      v.double_value = value
    else:
      logging.info(
          'Value type %s of key %s in exec_properties is not '
          'supported, going to drop it', type(value), key)
      continue
    executor_output.execution_properties[key].CopyFrom(v)

def get_orchestrator_generated_bcl_dir(
    pipeline_runtime_spec: pipeline_pb2.PipelineRuntimeSpec, node_id: str
) -> str:
  """Generates a root directory to hold orchestrator generated BCLs for the given node.

  Args:
    pipeline_runtime_spec: pipeline runtime specifications.
    node_id: unique id of the node within the pipeline.

  Returns:
    Path to orchestrator generated bcl root dir, which has the format
    `<node_dir>/.system/orchestrator_generated_bcl`
  """
  node_dir = _get_node_dir(pipeline_runtime_spec, node_id)
  orchestrator_generated_bcl_dir = os.path.join(
      node_dir, _SYSTEM, _ORCHESTRATOR_GENERATED_BCL_DIR
  )
  if not fileio.exists(orchestrator_generated_bcl_dir):
    try:
      fileio.makedirs(orchestrator_generated_bcl_dir)
    except Exception:  # pylint: disable=broad-except
      logging.exception(
          'Failed to make orchestrator generated bcl dir: %s',
          orchestrator_generated_bcl_dir,
      )
      raise
  return orchestrator_generated_bcl_dir


""" Write a program to create a new string made of an input string’s first, middle, and last character.

Given:

str1 = "James"

Expected Output:

Jms
"""

#!/usr/bin python3
str1="James"
str2=str1[0]
str3=str1[2]
str4=str1[4]
print(f"{str2}{str3}{str4}")


# https://leetcode.com/problems/longest-palindromic-substring/

#%%
def is_palindrome(word: str) -> bool:
    return word == word[::-1]

class Solution:
    def longestPalindrome(self, s: str) -> str:
        list_palindromes = []
        for i, _ in enumerate(s):
            substring = s[i:]
            for j, _ in enumerate(substring):
                if j == 0:
                    substring_2 = substring
                else:
                    substring_2 = substring[:-j]

                if is_palindrome(substring_2):
                    list_palindromes.append(substring_2)
                    break

        list_len_palindrome = [len(x) for x in list_palindromes]
        max_size_palindrome = max(list_len_palindrome)
        idx = list_len_palindrome.index(max_size_palindrome)
        the_longest_palindrome = list_palindromes[idx]
        print(list_palindromes)
        return the_longest_palindrome

#%%
is_palindrome("bab")
is_palindrome("dassad")
is_palindrome("dassa")
#%%
s = "babad"
solution = Solution()
solution.longestPalindrome(s) in ["bab", "aba"]

#%%
s = "cbbd"
solution = Solution()
solution.longestPalindrome(s) in ["bb"]

#%%
s = "a"
solution = Solution()
solution.longestPalindrome(s) in ["a"]

#%%
s = "ac"
solution = Solution()
solution.longestPalindrome(s) in ["a", "c"]

# %%
s = "vnjwvalrbypfcbqnmopltjnoifmzwgvpzqzsdtvawndpjtpmpjbjionjifqtvvocpeaftvhpdgjjfafunfndztdjkcxyihtsyppendfzzjeyxlbwpdygiqmdqcdbmgyjigrmfkswcwryaydjilqqxvcnyvviesuncslvzikawwqykqwdfibggezufqihcjkebapmgkvwixywgdextafxycnipjglsndkyjoqfyfljfkkvoieksmavdlmlhhnstesibffiopqvlyuidvrawndbzonwzbsjmpeqoglmdbinkovqpzfkxihzitdopnomseqhmrrkcsvrzziphwpuhjngeotwcrebcmbtirkgeavojtmpakcewmexhxacngknokxsvtqobdgckutpexswgwqzbosjpxauyflnylfcxsucsehqvakbpvfmkelmkspsqxnutwfwacpqqvovdqafeylobneojdsgqowcbxfsvuqusdbylcgcvgrofgvzubakjmlbffjhrafvnqttwuyhokzpmhlludpbowuxzrebxsdusalljfjgjkucwzpmndqncykvfnbrxcrcaxwisjpstejjqbpwegpxyrtyafxklgralnkwxkmjpuqfixzkonznmguyizlancpxdzcfkgiotyelegprbaytdhbutbuihkxnbtuqrtezaskfqsmrznfohhlqp"
solution = Solution()
solution.longestPalindrome(s)

# %%


class solution:
    def topKfrequent(self,nums:list[int],k:int):
        # counting hashTable
        dCount = {}

        for i in range(len(nums)):
            dCount[nums[i]] = 1 + dCount.get(nums[i] , 0)

        # frequency array
        frequency = [[] for _ in range(len(nums) + 1)]

        #storing the count based element in frequency array
        for element,count in dCount.items():
            frequency[count].append(element)

        # iterating from reverse order and storing in resultant array
        res = []

        for idex in range(len(frequency)-1,-1,-1):
            for element in frequency[idex]:
                res.append(element)
                if k == len(res):
                    return res
x = solution()
print(x.topKfrequent([1,1,1,2,2,3], 2))

# Approach Bucket Sort
# time complexity O(N)
# space complexity O(2N)


import json
import matplotlib.pyplot as plt
import numpy as np

INDICES = [0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0]

data = {}
means = {}
for i in INDICES:
    data.update({str(i): [[] for _ in range(51)]})
    means.update({str(i): [[] for _ in range(51)]})

for i in range(1, 11):
    with open(f'vary_n/data_{i}.json', 'r') as f:
        obj = json.load(f)
        for k, v in obj.items():
            for i in range(1, 51):
                data[k][i] += v[i]

# find means
for k in data.keys():
    for i in range(1, 51):
        means[k] = [np.mean(row) if len(row) != 0 else 0 for row in data[k]]

lines = []
indices = []

# Uncomment to show one line
INDICES = [0.5,1.0,2.0,5.0]
bestfit = False

for i in INDICES:
    if not means[str(i)][50]: continue
    line, = plt.plot(list(range(500, 25001, 500)), means[str(i)][1:])
    lines.append(line)
    indices.append(f"β={i}")
    if bestfit:
        x = np.linspace(500, 25000, 5000)
        y = 40/((x * i) ** 0.55)
        curve, = plt.plot(x, y)
        lines.append(curve)
        indices.append(f"Estimate for β={i}")

plt.xlabel("n")
plt.ylabel("RMSE")
plt.legend(lines, indices)
plt.show()


from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from router import light
from config.database import db
from router.control import auto, manual
from router.model import Light

origins = ['*']

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"]
)

app.include_router(light.router)

@app.get("/")
def get_all_light():
    collection = db["smarthome_light"]
    data = collection.find({}, {"_id": False})
    result = list(data)

    return {"result": result}

@app.put("/")
def update_light(light: Light):
    if light.room_id not in range(0, 3):
        raise HTTPException(status_code=404, detail="Room Id not in range (0-2)")
    if light.brightness not in range(0, 256):
        raise HTTPException(status_code=404, detail="Brightness not in range (0-255)")

    res = list(db["smarthome_light"].find({"room_id": light.room_id}, {"_id": False}))
    if len(res) == 0:
        raise HTTPException(status_code=404, detail="Room not found")

    if res[0]["is_auto"]:
        auto("hardware", light)
    else:
        manual("hardware", light)

    return {"result": "Update success"}

# def read_root():
#     return {"msg": "Connect!"}


from string import ascii_lowercase as alphabet
from itertools import permutations

superscript = ['⁰', '¹', '²', '³', '⁴', '⁵', '⁶', '⁷', '⁸', '⁹']

def factorial(n):
	ans = 1
	for i in range(2, n + 1):
		ans *= i
	return ans

def multinomialCoefficient(n, *k):
	ans = factorial(n)

	for i in k:
		ans //= factorial(i)

	return ans

while True:
	try:
		m = int(input('Please enter an amount of nomials to include (: '))

		if m > len(alphabet):
			raise ValueError

		selectedLetters = alphabet[:m]
		break
	except ValueError:
		print('Please ensure that you choose a valid amount of nomials (less than or equal to 26 please [:)')
		continue

while True:
	try:
		n = int(input('Please enter a power to put (%s) to! ' % ' + '.join(selectedLetters)))
		break
	except ValueError:
		print('The power must be a valid integer!')
		continue

# [{ letter: power }]
nomials = []
currentPower = {}

# Filter the power combinations for only where they add up to n
powerCombinations = list(set(permutations([i for i in range(0, n + 1)] * m, m)))

for combo in powerCombinations:
	if sum(combo) != n:
		continue

	powers = {}

	for i in range(len(combo)):
		letter = selectedLetters[i]
		powers[letter] = combo[i]

	nomials.append(powers)

formattedTerms = []

for nomial in nomials:
	coefficient = multinomialCoefficient(n, *nomial.values())
	term = ''

	if coefficient == 0:
		continue
	elif coefficient != 1:
		term += str(coefficient)

	for letter in nomial:
		power = nomial[letter]

		if power == 1:
			term += letter
		elif power > 0:
			powerString = ''

			for digit in str(power):
				powerString += superscript[int(digit)]

			term += '%s%s' % (letter, powerString)

	formattedTerms.append(term)

print(' + '.join(formattedTerms))


# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None
        self.father = None

class Solution(object):
    def inorderSuccessor(self, q):
        """
        :type q: TreeNode
        :rtype: TreeNode
        """

        if q is None:
            return None

        # 分类讨论1：如果这个结点有右子树，返回这个右子树的最小者
        if q.right:
            node = q.right
            while node.left:
                node = node.left
            return node
        # 分类讨论2：如果这个结点没有右子树，向上追溯，追到父亲结点的左结点是自己
        while q.father:
            if q.father.left == q:
                return q.father
            q = q.father
        return None

8-二叉树的下一个结点.py


import cherrypy
from various import *
from mako.template import Template
import smtplib

from email.MIMEMultipart import MIMEMultipart
from email.MIMEBase import MIMEBase
from email.MIMEText import MIMEText
from email.Utils import COMMASPACE, formatdate, make_msgid
from email import Encoders
from email.Header import Header

from passwords import smtp_server, smtp_login, smtp_password

def getSMTPServer() :
    server = smtplib.SMTP(smtp_server)
    server.login(smtp_login, smtp_password)
    return server

def sendMail(to, mail) :
    """ Send a mail to a specified address. The mail comes from noreply@naphtaline.com and should never be responded to
    """
    try :
        server = getSMTPServer()
        server.sendmail('noreply@naphtaline.com', to, mail)
        server.quit()
    except :
        pass

def send_mail(mail):

    msg = MIMEMultipart()
    msg.set_charset('utf-8')
    msg['From'] = mail['From']
    msg['Bcc'] = COMMASPACE.join(mail['To'])
    msg['Date'] = formatdate(localtime=True)
    msg['Subject'] = mail['Subject']
    msg['Reply-To'] = mail['Reply-To']
    msg['Message-ID'] = mail['Message-ID']
    msg['In-Reply-To'] = mail['In-Reply-To']
    #msg['Content-Type'] = 'text/plain; charset=UTF-8'

    msg.attach( MIMEText(mail['text'], _charset="utf-8") )

    for f in mail['files']:
        part = MIMEBase('application', "octet-stream")
        part.set_payload( open('./files/%s' % f['id'],"rb").read() )
        Encoders.encode_base64(part)
        part.add_header('Content-Disposition', 'attachment; filename="%s"' % f['name'])
        msg.attach(part)

    try:
        smtp = getSMTPServer()
        content = msg.as_string()
        smtp.sendmail(mail['Reply-To'], mail['To'], content)
        smtp.close()
    except :
        pass

def sendNewEntryNotifications(con, lang, project, category, id, files) :

    cursor = con.cursor()

    # list of recipients
    query = """
    SELECT mail
    FROM Client
    WHERE
        subscription = True
        AND allowed = True
        AND project = %s
    """
    cursor.execute(query, (projectID(con, project), ))
    recipients = []
    for row in cursor.fetchall() :
        recipients.append(row[0])

    # content of the message
    query = """
    SELECT
        Entry.relativeID,
        Entry.name,
        Entry.description,
        Entry.status,
        Entry.severity,
        Client.login,
        Client.mail,
        extract( day FROM Entry.creationDate),
        extract( month FROM Entry.creationDate),
        extract( year FROM Entry.creationDate)
    FROM Entry
    INNER JOIN Client
    ON Entry.creator = Client.id
    WHERE Entry.id = %s"""
    cursor.execute(query, (id, ))
    row = cursor.fetchone()
    relativeID = int(row[0])
    title = row[1]
    description = unFormatText(row[2])
    status = int(row[3])
    severity = int(row[4])
    creator = row[5]
    creatorMail = row[6]
    day = row[7]
    month = row[8]
    year = row[9]

    # creating Message-ID for this entry, and therefore the entry-point for the thread
    messageID = make_msgid()
    query = """
    UPDATE Entry SET lastMessageID = %s WHERE id = %s"""
    cursor.execute(query, (messageID,id))
    con.commit()

    mytemplate = Template(filename='templates/'+lang+'/mails/newEntry.mail', output_encoding='utf-8', default_filters=['decode.utf8'], input_encoding='utf-8')
    text = mytemplate.render(
        creator = creator,
        title = title,
        description = description,
        files = files,
        status = status,
        severity = severity
    )
    category = category[0].upper()+category[1:].lower()

    h = Header()
    h.append(category, 'utf-8')
    h.append(u'#'+str(relativeID)+u':', 'utf-8')
    h.append(title, 'utf-8')

    mail = {
        'From' : creator + ' <' + creatorMail + '>',
        'To' : recipients,
        'Subject' : h,
        'Reply-To' : project + '@projects.naphtaline.net',
        'Message-ID' : messageID,
        'In-Reply-To' : '',
        'text' : text,
        'files' : files,
    }

    send_mail(mail)

def sendEditEntryNotifications(con, lang, project, category, id, files) :

    cursor = con.cursor()

    # list of recipients
    query = """
    SELECT mail
    FROM Client
    WHERE
        subscription = True
        AND allowed = True
        AND project = %s
    """
    cursor.execute(query, (projectID(con, project), ))
    recipients = []
    for row in cursor.fetchall() :
        recipients.append(row[0])

    # content of the message
    query = """
    SELECT
        Entry.relativeID,
        Entry.name,
        Entry.description,
        Entry.status,
        Entry.severity,
        Client.login,
        Client.mail,
        extract( day FROM Entry.creationDate),
        extract( month FROM Entry.creationDate),
        extract( year FROM Entry.creationDate),
        Entry.lastMessageID
    FROM Entry
    INNER JOIN Client
    ON Entry.maintainer = Client.id
    WHERE Entry.id = %s"""
    cursor.execute(query, (id, ))

    row = cursor.fetchone()
    relativeID = int(row[0])
    title = row[1]
    description = unFormatText(row[2])
    status = int(row[3])
    severity = int(row[4])
    maintainer = row[5]
    maintainerMail = row[6]
    day = row[7]
    month = row[8]
    year = row[9]
    inReplyTo = row[10]

    # updating Message-ID
    messageID = make_msgid()
    query = """
    UPDATE Entry SET lastMessageID = %s WHERE id = %s"""
    cursor.execute(query, (messageID,id))
    con.commit()

    mytemplate = Template(filename='templates/'+lang+'/mails/editEntry.mail', output_encoding='utf-8', default_filters=['decode.utf8'], input_encoding='utf-8')
    text = mytemplate.render(
        title = title,
        description = description,
        files = files,
        status = status,
        severity = severity,
        category = category,
        relativeID = relativeID,
        maintainer = maintainer
    )
    category = category[0].upper()+category[1:].lower()

    h = Header()
    h.append(u'Re:', 'utf-8')
    h.append(category, 'utf-8')
    h.append(u'#'+str(relativeID)+u':', 'utf-8')
    h.append(title, 'utf-8')

    mail = {
        'From' : maintainer + ' <' + maintainerMail + '>',
        'To' : recipients,
        'Subject' : h,
        'Reply-To' : project + '@projects.naphtaline.net',
        'Message-ID' : messageID,
        'In-Reply-To' : inReplyTo,
        'text' : text,
        'files' : files,
    }

    send_mail(mail)

def sendNewCommentNotifications(con, lang, project, category, id, content, files) :

    cursor = con.cursor()

    # creating the list of recipients
    query = """
    SELECT mail
    FROM Client
    WHERE
        subscription = True
        AND allowed = True
        AND project = %s
    """
    cursor.execute(query, (projectID(con, project), ))
    recipients = []
    for row in cursor.fetchall() :
        recipients.append(row[0])

    # creating the message
    query = """
    SELECT
        Client.login,
        Client.mail,
        Comment.entry,
        extract( day FROM Comment.creationDate),
        extract( month FROM Comment.creationDate),
        extract( year FROM Comment.creationDate),
        Entry.name,
        Entry.relativeID,
        Entry.lastMessageID
    FROM Comment
    INNER JOIN Entry
    ON Comment.entry = Entry.id
    INNER JOIN Client
    ON Comment.author = Client.id
    WHERE Comment.id = %s"""
    cursor.execute(query, (id, ))
    row = cursor.fetchone()
    author = row[0]
    authorMail = row[1]
    idEntry = int(row[2])
    day = int(row[3])
    month = int(row[4])
    year = int(row[5])
    title = row[6]
    relativeID = int(row[7])
    inReplyTo = row[8]

    # updating Message-ID
    messageID = make_msgid()
    query = """
    UPDATE Entry SET lastMessageID = %s WHERE id = %s"""
    cursor.execute(query, (messageID,id))
    con.commit()

    mytemplate = Template(filename='templates/'+lang+'/mails/newcomment.mail', output_encoding='utf-8', default_filters=['decode.utf8'], input_encoding='utf-8')
    text = mytemplate.render(
        author = author,
        comment = content,
        files = files
    )
    category = category[0].upper()+category[1:].lower()

    h = Header()
    h.append(u'Re:', 'utf-8')
    h.append(category, 'utf-8')
    h.append(u'#'+str(relativeID)+u':', 'utf-8')
    h.append(title, 'utf-8')

    mail = {
        'From' : author + ' <' + authorMail + '>',
        'To' : recipients,
        'Subject' : h,
        'Reply-To' : project + '@projects.naphtaline.net',
        'Message-ID' : messageID,
        'In-Reply-To' : inReplyTo,
        'text' : text,
        'files' : files,
    }

    send_mail(mail)

def sendDeleteNotifications(con, lang, project, category, relativeID, userID):
    """ Send a notification in case of delete """
    cursor = con.cursor()
    # creating the list of recipients
    query = """
    SELECT mail
    FROM Client
    WHERE
        subscription = True
        AND allowed = True
        AND project = %s"""
    cursor.execute(query, (projectID(con, project), ))
    recipients = []
    for row in cursor.fetchall() :
        recipients.append(row[0])
    # get message content
    query = """
    SELECT name, lastMessageID
    FROM Entry
    WHERE relativeID = %s AND category = %s"""
    categoryCode = {'bug' : 1, 'feature' : 2}
    cursor.execute(query, (relativeID, categoryCode[category]))
    row = cursor.fetchone()
    title = row[0]
    inReplyTo = row[1]
    # user caracteristics (the one who deleted the entry)
    query = """SELECT login, mail FROM Client WHERE id=%s"""
    cursor.execute(query, (userID, ))
    row = cursor.fetchone()
    author = row[0]
    authorMail = row[1]
    # load template
    mytemplate = Template(filename='templates/'+lang+'/mails/delete.mail', output_encoding='utf-8', default_filters=['decode.utf8'], input_encoding='utf-8')
    text = mytemplate.render(creator = author)
    category = category[0].upper()+category[1:].lower()

    h = Header()
    h.append(u'Re:', 'utf-8')
    h.append(category, 'utf-8')
    h.append(u'#'+str(relativeID)+u':', 'utf-8')
    h.append(title, 'utf-8')

    # make messageID
    # no need to save it to the database : the entry will be deleted
    messageID = make_msgid()

    mail = {'From' : author + ' <' + authorMail + '>',
            'To' : recipients,
            'Subject' : h,
            'Reply-To' : project + '@projects.naphtaline.net',
            'Message-ID' : messageID,
            'In-Reply-To' : inReplyTo,
            'text' : text,
            'files' : []}

    send_mail(mail)


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

###########################################
### Calculate dimension in pixel way ######
###########################################
df_tobii = pd.read_csv(r'C:\Users\lguo8\PycharmProjects\EyeTracking_baseline\Tobii\luyao_12_13_3_light_on\20221213T071219Z\cv2_pixels\pixel_coordinates.csv',header=None )
df_tobii.columns = ['tobii_pixel_x', 'tobii_pixel_y', 'baseline_x', 'baseline_y']

df_world_camera = pd.read_csv(r'C:\Users\lguo8\PycharmProjects\EyeTracking_baseline\Tobii\luyao_12_13_3_light_on\20221213T071219Z\cv2_pixels\pixel_world_camera.csv',header=None)
tobii_pixel_x = df_tobii.iloc[:]['tobii_pixel_x'].tolist()
tobii_pixel_y = df_tobii.iloc[:]['tobii_pixel_y'].tolist()
baseline_x = df_tobii.iloc[:]['baseline_x'].tolist()
baseline_y = df_tobii.iloc[:]['baseline_y'].tolist()
# plt.scatter(tobii_pixel_x, tobii_pixel_y)
# plt.show()

# define screen outlines pixels
# assume as rectangle

for i in range(len(baseline_x)):
    if baseline_x[i]==0 and baseline_y[i]== 8:
        index_left_top = i
    elif baseline_x[i]==0 and baseline_y[i]== 0:
        index_left_bottom = i
    elif baseline_x[i]==8 and baseline_y[i]== 0:
        index_right_bottom = i
    elif baseline_x[i]==8 and baseline_y[i]== 8:
        index_right_top = i
# print(index_left_top)
# due to the fact that pixel is defined by dot, a few space is saved for dot size
value_given = 6
# pixel in y axis is reverse to baseline to y axis

pixel_left_bottom = (tobii_pixel_x[index_left_bottom]-value_given, tobii_pixel_y[index_left_bottom]-value_given)
pixel_left_top = (tobii_pixel_x[index_left_top]-value_given, tobii_pixel_y[index_left_top]+value_given)
pixel_right_bottom = (tobii_pixel_x[index_right_bottom]+value_given, tobii_pixel_y[index_right_bottom]-value_given)
pixel_right_top = (tobii_pixel_x[index_right_top]+value_given, tobii_pixel_y[index_right_top]+value_given)

# use left_bottom, right_bottom and righttop to define screen outlines
x=[pixel_left_bottom[0],pixel_right_top[0],pixel_right_top[0],pixel_right_bottom[0]]
y=[pixel_left_bottom[1],pixel_right_top[1],pixel_right_top[1],pixel_right_bottom[1]]

# plt.plot(x,y, 'ro')

def connectpoints(x,y,p1,p2):
    x1, x2 = x[p1], x[p2]
    y1, y2 = y[p1], y[p2]
    plt.plot([x1,x2],[y1,y2],'k-')

# treat these 2 lines as screen outlines
plt.plot([x[0],x[3]],[y[0],y[3]],'k-')
plt.plot([x[2],x[3]],[y[2],y[3]],'k-')

# calculate slopes and intercepts
slope_bottom = (y[3]-y[0])/(x[3]-x[0])
intercept_bottom = y[3]-slope_bottom*x[3]

screen_left_top_x = x[0]+x[2]-x[3]
screen_left_top_y = y[0]+y[2]-y[3]
plt.plot([x[0],screen_left_top_x],[y[0],screen_left_top_y],'k-')
plt.plot([x[2],screen_left_top_x],[y[2],screen_left_top_y],'k-')

# define xlim and ylim by world_camera_pixels
world_x_min = df_world_camera.iloc[0][0]
world_x_max = df_world_camera.iloc[2][0]
world_y_min = df_world_camera.iloc[0][1]
world_y_max = df_world_camera.iloc[2][1]
plt.xlim([world_x_min, world_x_max])
plt.ylim([world_y_min, world_y_max])
# plt.axis('equal')
plt.show()

screen_x=[screen_left_top_x,pixel_left_bottom[0],pixel_right_top[0],pixel_right_bottom[0]]
screen_y=[screen_left_top_y-value_given,pixel_left_bottom[1],pixel_right_top[1],pixel_right_bottom[1]]

dot_gap_x = (screen_x[3]-screen_x[1])/8
dot_gap_y = (screen_y[2]-screen_y[3])/8
dot_scatter_x = []
dot_scatter_y = []
baseline_x_temp = []
baseline_y_temp = []

screen_bottom_line_y_segment = np.linspace(screen_y[1],screen_y[3],9)

for i in range(9):
    for j in range(8,-1,-2):
        dot_scatter_x.append(dot_gap_x*i+screen_x[1])
        dot_scatter_y.append(dot_gap_y * abs(j-8)+ screen_bottom_line_y_segment[i])
        plt.scatter(dot_scatter_x,dot_scatter_y, color='black')
        baseline_x_temp.append(i)
        baseline_y_temp.append(abs(j-8))
        # print(j)
print(dot_scatter_y)
print(baseline_y_temp)
# plt.show()
# print(dot_scatter_y)
# print(baseline_x)
# change order
order = [4,9,14,19,24,29,34,39,44]
order2 = []
for i in range(5):
    for j in order:
        order2.append((j-i))

# in real dimension cm
screen_width_cm = 154
screen_height_cm = 87
screen_width_pixel = x[2]-x[0]
screen_height_pixel = y[2]-y[3]
pixel_per_cm = 154/screen_width_pixel
with open('pixel_per_cm.txt','w') as f:
    f.write(str(pixel_per_cm))
dot_cm_x = []
dot_cm_y = []

for i in range(len(dot_scatter_x)):
    # dot_cm_x.append((dot_scatter_x[i]+x[1])*pixel_per_cm)
    # dot_cm_y.append((dot_scatter_y[i] + y[1]) * pixel_per_cm)
    dot_cm_x.append((dot_scatter_x[i]) * pixel_per_cm)
    dot_cm_y.append((dot_scatter_y[i]) * pixel_per_cm)
print(dot_cm_y)
# #change order
dot_cm_x = [dot_cm_x[i] for i in order2]
dot_cm_y = [dot_cm_y[i] for i in order2]
baseline_x_temp = [baseline_x_temp[i] for i in order2]
baseline_y_temp = [baseline_y_temp[i] for i in order2]
print(baseline_y_temp)
print(baseline_x_temp)

# Get world camera scene dimensions in cm
world_x_range_pixel = world_x_max-world_x_min
world_y_range_pixel = world_y_max-world_y_min
world_x_range_cm = world_x_range_pixel*pixel_per_cm
world_y_range_cm = world_y_range_pixel*pixel_per_cm

f = open('world_cm.txt','w')
f.write(str(world_x_range_cm)+"\n")
f.write(str(world_y_range_cm))
f.close()

# change coordinate: origin from left-top to left_bottom
# dot_cm_y = [(world_y_range_cm-dot_cm_y[i]) for i in range(len(dot_cm_y))]

dot_cm_x = np.tile(dot_cm_x,3)
dot_cm_y = np.tile(dot_cm_y,3)
baseline_x_temp = np.tile(baseline_x_temp,3)
baseline_y_temp = np.tile(baseline_y_temp,3)
# print(baseline_y_temp)

df_cm = pd.DataFrame(dot_cm_x,columns=['dot_x_cm'])
df_cm['dot_y_cm'] = pd.DataFrame(dot_cm_y)
df_cm['baseline_x_temp'] = pd.DataFrame(baseline_x_temp)
df_cm['baseline_y_temp'] = pd.DataFrame(baseline_y_temp)

df_dot = pd.read_csv('dot_data_0_8_8_8_2022_12_13_18_12_27.csv')
df_cm = pd.concat([df_dot,df_cm],axis=1)
df_cm.to_csv('cm.csv',index=False)

# Get world camera scene dimensions in cm
world_x_range_pixel = world_x_max-world_x_min
world_y_range_pixel = world_y_max-world_y_min
world_x_range_cm = world_x_range_pixel*pixel_per_cm
world_y_range_cm = world_y_range_pixel*pixel_per_cm
f = open('world_cm.txt','w')
f.write(str(world_x_range_cm)+"\n")
f.write(str(world_y_range_cm))
f.close()


import unittest

from cdr_cleaner.cleaning_rules.drop_ppi_duplicate_responses import (
    DropPpiDuplicateResponses, get_select_statement, get_delete_statement)
from constants.cdr_cleaner import clean_cdr as clean_consts

class DropPpiDuplicateResponsesTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        print('**************************************************************')
        print(cls.__name__)
        print('**************************************************************')

    def setUp(self):
        self.project_id = 'foo_project'
        self.dataset_id = 'foo_dataset'
        self.sandbox_id = 'foo_sandbox'
        self.client = None

        self.rule_instance = DropPpiDuplicateResponses(self.project_id,
                                                       self.dataset_id,
                                                       self.sandbox_id)

        self.assertEqual(self.rule_instance.project_id, self.project_id)
        self.assertEqual(self.rule_instance.dataset_id, self.dataset_id)
        self.assertEqual(self.rule_instance.sandbox_dataset_id, self.sandbox_id)

    def test_get_query_specs(self):
        # Pre conditions
        self.assertEqual(self.rule_instance.affected_datasets,
                         [clean_consts.RDR])

        # Test
        results_list = self.rule_instance.get_query_specs()
        # Post conditions
        sandbox_query = dict()
        sandbox_query[clean_consts.QUERY] = get_select_statement(
            self.project_id, self.dataset_id, self.sandbox_id,
            self.rule_instance.get_sandbox_tablenames()[0])

        update_query = dict()
        update_query[clean_consts.QUERY] = get_delete_statement(
            self.project_id, self.dataset_id)

        expected_list = [sandbox_query, update_query]

        for ex_dict, rs_dict in zip(expected_list, results_list):
            self.assertDictEqual(ex_dict, rs_dict)

    def test_log_queries(self):
        # Pre conditions
        self.assertEqual(self.rule_instance.affected_datasets,
                         [clean_consts.RDR])

        store_duplicate_rows = get_select_statement(
            self.project_id, self.dataset_id, self.sandbox_id,
            self.rule_instance.get_sandbox_tablenames()[0])

        delete_duplicate_rows = get_delete_statement(self.project_id,
                                                     self.dataset_id)

        # Test
        with self.assertLogs(level='INFO') as cm:
            self.rule_instance.log_queries()

            expected = [
                'INFO:cdr_cleaner.cleaning_rules.base_cleaning_rule:Generated SQL Query:\n'
                + store_duplicate_rows,
                'INFO:cdr_cleaner.cleaning_rules.base_cleaning_rule:Generated SQL Query:\n'
                + delete_duplicate_rows
            ]

            # Post condition
            self.assertEqual(cm.output, expected)


import json

import pytest
from bs4 import BeautifulSoup
from django.core.files.uploadedfile import SimpleUploadedFile

@pytest.fixture
def import_soup(client_superuser_loggedin):
    response = client_superuser_loggedin.get(
        "/__django_admin/organizerlist/organizerprofile/import/", follow=True
    )
    return BeautifulSoup(response.content, "html.parser")

@pytest.mark.django_db
def test_jsonインポート機能を使ってメイドプロフィールを登録できること(import_soup, client_superuser_loggedin):
    csv_id = {
        tag.text: tag.get("value")
        for tag in import_soup.select("select[name=input_format] option")
    }["json"]

    file = SimpleUploadedFile(
        "test.json",
        json.dumps(
            [
                {"code": "maidsan", "name": "メイドさん", "order": 10, "content": "おはよう"},
                {"code": "maidchan", "name": "メイドちゃん", "order": 20, "content": "おやすみ"},
            ]
        ).encode("utf-8"),
    )

    response = client_superuser_loggedin.submit_from_soup(
        "/__django_admin/maidlist/maidprofile/import/",
        import_soup,
        selector="#content form",
        data={
            "import_file": file,
            "input_format": csv_id,
            "_save": "確定",
        },
    )

    # 確認画面
    confirm_soup = BeautifulSoup(response.content, "html.parser")
    response = client_superuser_loggedin.submit_from_soup(
        "",
        confirm_soup,
        selector="#content form",
        data={"confirm": "インポート実行"},
    )

    # assert
    from apps.maidlist.models import MaidProfile

    data = list(MaidProfile.objects.values_list("code", "name", "order", "content"))
    assert data == [
        ("maidsan", "メイドさん", 10, "おはよう"),
        ("maidchan", "メイドちゃん", 20, "おやすみ"),
    ]


#=== Delete files from homeassistant service ===#
#=== exlab, Jan. 02, 2019 ===#
#==================#
# Code for Home Assistant
# Code in configuration.yaml
# hass_delete:
# Code in automation
# - alias: Example delete files
  # trigger:
    # platform: state
    # entity_id: domain.entity_id
    # to: 'on'
  # action:
    # service: hass_delete.files
    # data:
      # file_path: '/config/folder/file_name1.extension, /config/folder/file_name2.extension'
#==================#

domain = 'hass_delete'
import os

def setup(homeassistant, config):
  def delete_file(files):
    # Declare variable in Home Assistant: file_path
    file_path = files.data.get('file_path')
    #if list files by delimiter ","
    sDelimiter = ','
    file_path = file_path.replace(', ', sDelimiter)
    if file_path.find(sDelimiter) > 0:
      array_paths = file_path.split(sDelimiter)
      for iTem in array_paths:
        if os.path.exists(iTem):
          os.remove(iTem)
    else:
      if os.path.exists(file_path):
        os.remove(file_path)
  homeassistant.services.register(domain, 'files', delete_file)
  return True