# Pytorch Tabular
# Author: Manu Joseph <manujoseph@gmail.com>
# For license information, see LICENSE.TXT
"""Tabular Data Module."""
import re
from pathlib import Path
from typing import Iterable, List, Optional, Tuple, Union

import category_encoders as ce
import joblib
import numpy as np
import pandas as pd
import pytorch_lightning as pl
import torch
from omegaconf import DictConfig
from pandas.tseries import offsets
from pandas.tseries.frequencies import to_offset
from sklearn.base import copy, TransformerMixin
from sklearn.preprocessing import (
    FunctionTransformer,
    LabelEncoder,
    PowerTransformer,
    QuantileTransformer,
    StandardScaler,
)
from torch.utils.data import DataLoader, Dataset

from pytorch_tabular.config import InferredConfig
from pytorch_tabular.utils import get_logger

from .categorical_encoders import OrdinalEncoder

logger = get_logger(__name__)

class TabularDatamodule(pl.LightningDataModule):
    CONTINUOUS_TRANSFORMS = {
        "quantile_uniform": {
            "callable": QuantileTransformer,
            "params": {"output_distribution": "uniform", "random_state": None},
        },
        "quantile_normal": {
            "callable": QuantileTransformer,
            "params": {"output_distribution": "normal", "random_state": None},
        },
        "box-cox": {
            "callable": PowerTransformer,
            "params": {"method": "box-cox", "standardize": False},
        },
        "yeo-johnson": {
            "callable": PowerTransformer,
            "params": {"method": "yeo-johnson", "standardize": False},
        },
    }

    def __init__(
        self,
        train: pd.DataFrame,
        config: DictConfig,
        validation: pd.DataFrame = None,
        test: pd.DataFrame = None,
        target_transform: Optional[Union[TransformerMixin, Tuple]] = None,
        train_sampler: Optional[torch.utils.data.Sampler] = None,
        seed: Optional[int] = 42,
    ):
        """The Pytorch Lightning Datamodule for Tabular Data.

        Args:
            train (pd.DataFrame): The Training Dataframe

            config (DictConfig): Merged configuration object from ModelConfig, DataConfig,
                TrainerConfig, OptimizerConfig & ExperimentConfig

            validation (pd.DataFrame, optional): Validation Dataframe.
                If left empty, we use the validation split from DataConfig to split a random sample as validation.
                Defaults to None.

            test (pd.DataFrame, optional): Holdout DataFrame to check final performance on.
                Defaults to None.

            target_transform (Optional[Union[TransformerMixin, Tuple(Callable)]], optional):
                If provided, applies the transform to the target before modelling and inverse the transform during
                prediction. The parameter can either be a sklearn Transformer which has an inverse_transform method, or
                a tuple of callables (transform_func, inverse_transform_func)
        """
        super().__init__()
        self.train = train.copy()
        self.validation = validation
        self._set_target_transform(target_transform)
        self.test = test if test is None else test.copy()
        self.target = config.target
        self.batch_size = config.batch_size
        self.train_sampler = train_sampler
        self.config = config
        self.seed = seed
        self._fitted = False

    def _set_target_transform(self, target_transform: Union[TransformerMixin, Tuple]) -> None:
        if target_transform is not None:
            if isinstance(target_transform, Iterable):
                target_transform = FunctionTransformer(func=target_transform[0], inverse_func=target_transform[1])
            self.do_target_transform = True
        else:
            self.do_target_transform = False
        self.target_transform_template = target_transform

    def update_config(self, config) -> InferredConfig:
        """Calculates and updates a few key information to the config object.

        Args:
            config (DictConfig): The config object

        Returns:
            InferredConfig: The updated config object
        """
        categorical_dim = len(config.categorical_cols)
        continuous_dim = len(config.continuous_cols)
        if config.task == "regression":
            output_dim = len(config.target)
        elif config.task == "classification":
            output_dim = len(self.train[config.target[0]].unique())
        else:
            output_dim = None
        categorical_cardinality = None
        embedding_dims = None
        if not self.do_leave_one_out_encoder():
            categorical_cardinality = [
                int(self.train[col].fillna("NA").nunique()) + 1 for col in config.categorical_cols
            ]
            embedding_dims = [(x, min(50, (x + 1) // 2)) for x in categorical_cardinality]
            if hasattr(config, "embedding_dims"):
                if config.embedding_dims is not None:
                    embedding_dims = config.embedding_dims
        return InferredConfig(
            categorical_dim=categorical_dim,
            continuous_dim=continuous_dim,
            output_dim=output_dim,
            categorical_cardinality=categorical_cardinality,
            embedding_dims=embedding_dims,
        )

    def do_leave_one_out_encoder(self) -> bool:
        """Checks the special condition for NODE where we use a LeaveOneOutEncoder to encode categorical columns
        DEPRECATED: Automatically encoding categorical columns using LeaveOneOutEncoder is deprecated.

        Returns:
            bool: True if LeaveOneOutEncoder is used
        """
        if hasattr(self.config, "_model_name"):
            return (self.config._model_name == "NODEModel") and (not self.config.embed_categorical)
        else:
            return False

    def _encode_date_columns(self, data: pd.DataFrame) -> pd.DataFrame:
        added_features = []
        for field_name, freq in self.config.date_columns:
            data = self.make_date(data, field_name)
            data, _new_feats = self.add_datepart(data, field_name, frequency=freq, prefix=None, drop=True)
            added_features += _new_feats
        return data, added_features

    def _encode_categorical_columns(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if stage == "fit":
            if self.do_leave_one_out_encoder():
                logger.debug("Encoding Categorical Columns using LeavOneOutEncoder")
                self.categorical_encoder = ce.LeaveOneOutEncoder(
                    cols=self.config.categorical_cols, random_state=self.seed
                )
                # Multi-Target Regression uses the first target to encode the categorical columns
                if len(self.config.target) > 1:
                    logger.warning(
                        f"Multi-Target Regression: using the first target({self.config.target[0]})"
                        f" to encode the categorical columns"
                    )
                data = self.categorical_encoder.fit_transform(data, data[self.config.target[0]])
            else:
                logger.debug("Encoding Categorical Columns using OrdinalEncoder")
                self.categorical_encoder = OrdinalEncoder(
                    cols=self.config.categorical_cols,
                    handle_unseen="impute" if self.config.handle_unknown_categories else "error",
                    handle_missing="impute" if self.config.handle_missing_values else "error",
                )
                data = self.categorical_encoder.fit_transform(data)
        else:
            data = self.categorical_encoder.transform(data)
        return data

    def _transform_continuous_columns(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if stage == "fit":
            transform = self.CONTINUOUS_TRANSFORMS[self.config.continuous_feature_transform]
            if "random_state" in transform["params"] and self.seed is not None:
                transform["params"]["random_state"] = self.seed
            self.continuous_transform = transform["callable"](**transform["params"])
            # TODO implement quantile noise
            data.loc[:, self.config.continuous_cols] = self.continuous_transform.fit_transform(
                data.loc[:, self.config.continuous_cols]
            )
        else:
            data.loc[:, self.config.continuous_cols] = self.continuous_transform.transform(
                data.loc[:, self.config.continuous_cols]
            )
        return data

    def _normalize_continuous_columns(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if stage == "fit":
            self.scaler = StandardScaler()
            data.loc[:, self.config.continuous_cols] = self.scaler.fit_transform(
                data.loc[:, self.config.continuous_cols]
            )
        else:
            data.loc[:, self.config.continuous_cols] = self.scaler.transform(data.loc[:, self.config.continuous_cols])
        return data

    def _label_encode_target(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if self.config.task == "classification":
            if stage == "fit":
                self.label_encoder = LabelEncoder()
                data[self.config.target[0]] = self.label_encoder.fit_transform(data[self.config.target[0]])
            else:
                if self.config.target[0] in data.columns:
                    data[self.config.target[0]] = self.label_encoder.transform(data[self.config.target[0]])
        return data

    def _target_transform(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if self.config.task == "regression":
            # target transform only for regression
            if all(col in data.columns for col in self.config.target):
                if self.do_target_transform:
                    if stage == "fit":
                        target_transforms = []
                        for col in self.config.target:
                            _target_transform = copy.deepcopy(self.target_transform_template)
                            data[col] = _target_transform.fit_transform(data[col].values.reshape(-1, 1))
                            target_transforms.append(_target_transform)
                        self.target_transforms = target_transforms
                    else:
                        for col, _target_transform in zip(self.config.target, self.target_transforms):
                            data[col] = _target_transform.transform(data[col].values.reshape(-1, 1))
        return data

    def preprocess_data(self, data: pd.DataFrame, stage: str = "inference") -> Tuple[pd.DataFrame, list]:
        """The preprocessing, like Categorical Encoding, Normalization, etc. which any dataframe should undergo before
        feeding into the dataloder.

        Args:
            data (pd.DataFrame): A dataframe with the features and target
            stage (str, optional): Internal parameter. Used to distinguisj between fit and inference.
                Defaults to "inference".

        Returns:
            Returns the processed dataframe and the added features(list) as a tuple
        """
        added_features = None
        if self.config.encode_date_columns:
            data, added_features = self._encode_date_columns(data)
        # The only features that are added are the date features extracted
        # from the date which are categorical in nature
        if (added_features is not None) and (stage == "fit"):
            logger.debug(f"Added {added_features} features after encoding the date_columns")
            self.config.categorical_cols += added_features
            # Update the categorical dimension in config
            self.config.categorical_dim = (
                len(self.config.categorical_cols) if self.config.categorical_cols is not None else 0
            )
        # Encoding Categorical Columns
        if len(self.config.categorical_cols) > 0:
            data = self._encode_categorical_columns(data, stage)

        # Transforming Continuous Columns
        if (self.config.continuous_feature_transform is not None) and (len(self.config.continuous_cols) > 0):
            data = self._transform_continuous_columns(data, stage)
        # Normalizing Continuous Columns
        if (self.config.normalize_continuous_features) and (len(self.config.continuous_cols) > 0):
            data = self._normalize_continuous_columns(data, stage)
        # Converting target labels to a 0 indexed label
        data = self._label_encode_target(data, stage)
        # Target Transforms
        data = self._target_transform(data, stage)
        return data, added_features

    def setup(self, stage: Optional[str] = None) -> None:
        """Data Operations you want to perform on all GPUs, like train-test split, transformations, etc. This is called
        before accessing the dataloaders.

        Args:
            stage (Optional[str], optional):
                Internal parameter to distinguish between fit and inference. Defaults to None.
        """
        if stage == "fit" or stage is None:
            logger.info(f"Setting up the datamodule for {self.config.task} task")
            if self.validation is None:
                logger.debug(
                    f"No validation data provided."
                    f" Using {self.config.validation_split*100}% of train data as validation"
                )
                val_idx = self.train.sample(
                    int(self.config.validation_split * len(self.train)),
                    random_state=self.seed,
                ).index
                self.validation = self.train[self.train.index.isin(val_idx)]
                self.train = self.train[~self.train.index.isin(val_idx)]
            else:
                self.validation = self.validation.copy()
            # Preprocessing Train, Validation
            self.train, _ = self.preprocess_data(self.train, stage="fit")
            self.validation, _ = self.preprocess_data(self.validation, stage="inference")
            if self.test is not None:
                self.test, _ = self.preprocess_data(self.test, stage="inference")
            self._fitted = True

    # adapted from gluonts
    @classmethod
    def time_features_from_frequency_str(cls, freq_str: str) -> List[str]:
        """Returns a list of time features that will be appropriate for the given frequency string.

        Args:
            freq_str (str): Frequency string of the form `[multiple][granularity]` such as "12H", "5min", "1D" etc.

        Returns:
            List of added features
        """

        features_by_offsets = {
            offsets.YearBegin: [],
            offsets.YearEnd: [],
            offsets.MonthBegin: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
            ],
            offsets.MonthEnd: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
            ],
            offsets.Week: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week",
            ],
            offsets.Day: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
            ],
            offsets.BusinessDay: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
            ],
            offsets.Hour: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
                "Hour",
            ],
            offsets.Minute: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
                "Hour",
                "Minute",
            ],
        }

        offset = to_offset(freq_str)

        for offset_type, feature in features_by_offsets.items():
            if isinstance(offset, offset_type):
                return feature

        supported_freq_msg = f"""
        Unsupported frequency {freq_str}

        The following frequencies are supported:

            Y, YS   - yearly
                alias: A
            M, MS   - monthly
            W   - weekly
            D   - daily
            B   - business days
            H   - hourly
            T   - minutely
                alias: min
        """
        raise RuntimeError(supported_freq_msg)

    # adapted from fastai
    @classmethod
    def make_date(cls, df: pd.DataFrame, date_field: str) -> pd.DataFrame:
        """Make sure `df[date_field]` is of the right date type.

        Args:
            df (pd.DataFrame): Dataframe

            date_field (str): Date field name

        Returns:
            Dataframe with date field converted to datetime
        """
        field_dtype = df[date_field].dtype
        if isinstance(field_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
            field_dtype = np.datetime64
        if not np.issubdtype(field_dtype, np.datetime64):
            df[date_field] = pd.to_datetime(df[date_field], infer_datetime_format=True)
        return df

    # adapted from fastai
    @classmethod
    def add_datepart(
        cls,
        df: pd.DataFrame,
        field_name: str,
        frequency: str,
        prefix: str = None,
        drop: bool = True,
    ) -> Tuple[pd.DataFrame, List[str]]:
        """Helper function that adds columns relevant to a date in the column `field_name` of `df`.

        Args:
            df (pd.DataFrame): Dataframe

            field_name (str): Date field name

            frequency (str): Frequency string of the form `[multiple][granularity]` such as "12H", "5min", "1D" etc.

            prefix (str, optional): Prefix to add to the new columns. Defaults to None.

            drop (bool, optional): Drop the original column. Defaults to True.

        Returns:
            Dataframe with added columns and list of added columns
        """
        field = df[field_name]
        prefix = (re.sub("[Dd]ate$", "", field_name) if prefix is None else prefix) + "_"
        attr = cls.time_features_from_frequency_str(frequency)
        added_features = []
        for n in attr:
            if n == "Week":
                continue
            df[prefix + n] = getattr(field.dt, n.lower())
            added_features.append(prefix + n)
        # Pandas removed `dt.week` in v1.1.10
        if "Week" in attr:
            week = field.dt.isocalendar().week if hasattr(field.dt, "isocalendar") else field.dt.week
            df.insert(3, prefix + "Week", week)
            added_features.append(prefix + "Week")
        # TODO Not adding Elapsed by default. Need to route it through config
        # mask = ~field.isna()
        # df[prefix + "Elapsed"] = np.where(
        #     mask, field.values.astype(np.int64) // 10 ** 9, None
        # )
        # added_features.append(prefix + "Elapsed")
        if drop:
            df.drop(field_name, axis=1, inplace=True)

        # Removing features woth zero variations
        # for col in added_features:
        #     if len(df[col].unique()) == 1:
        #         df.drop(columns=col, inplace=True)
        #         added_features.remove(col)
        return df, added_features

    def train_dataloader(self, batch_size: Optional[int] = None) -> DataLoader:
        """Function that loads the train set.

        Args:
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: Train dataloader
        """
        dataset = TabularDataset(
            task=self.config.task,
            data=self.train,
            categorical_cols=self.config.categorical_cols,
            continuous_cols=self.config.continuous_cols,
            embed_categorical=(not self.do_leave_one_out_encoder()),
            target=self.target,
        )
        return DataLoader(
            dataset,
            batch_size if batch_size is not None else self.batch_size,
            shuffle=True if self.train_sampler is None else False,
            num_workers=self.config.num_workers,
            sampler=self.train_sampler,
            pin_memory=self.config.pin_memory,
        )

    def val_dataloader(self, batch_size: Optional[int] = None) -> DataLoader:
        """Function that loads the validation set.

        Args:
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: Validation dataloader
        """
        dataset = TabularDataset(
            task=self.config.task,
            data=self.validation,
            categorical_cols=self.config.categorical_cols,
            continuous_cols=self.config.continuous_cols,
            embed_categorical=(not self.do_leave_one_out_encoder()),
            target=self.target,
        )
        return DataLoader(
            dataset,
            batch_size if batch_size is not None else self.batch_size,
            shuffle=False,
            num_workers=self.config.num_workers,
            pin_memory=self.config.pin_memory,
        )

    def test_dataloader(self, batch_size: Optional[int] = None) -> DataLoader:
        """Function that loads the validation set.

        Args:
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: Test dataloader
        """
        if self.test is not None:
            dataset = TabularDataset(
                task=self.config.task,
                data=self.test,
                categorical_cols=self.config.categorical_cols,
                continuous_cols=self.config.continuous_cols,
                embed_categorical=(not self.do_leave_one_out_encoder()),
                target=self.target,
            )
            return DataLoader(
                dataset,
                batch_size if batch_size is not None else self.batch_size,
                shuffle=False,
                num_workers=self.config.num_workers,
                pin_memory=self.config.pin_memory,
            )

    def _prepare_inference_data(self, df: pd.DataFrame) -> pd.DataFrame:
        """Prepare data for inference."""
        # TODO Is the target encoding necessary?
        if len(set(self.target) - set(df.columns)) > 0:
            if self.config.task == "classification":
                df.loc[:, self.target] = np.array([self.label_encoder.classes_[0]] * len(df)).reshape(-1, 1)
            else:
                df.loc[:, self.target] = np.zeros((len(df), len(self.target)))
        df, _ = self.preprocess_data(df, stage="inference")
        return df

    def prepare_inference_dataloader(self, df: pd.DataFrame, batch_size: Optional[int] = None) -> DataLoader:
        """Function that prepares and loads the new data.

        Args:
            df (pd.DataFrame): Dataframe with the features and target
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: The dataloader for the passed in dataframe
        """
        df = df.copy()
        df = self._prepare_inference_data(df)
        dataset = TabularDataset(
            task=self.config.task,
            data=df,
            categorical_cols=self.config.categorical_cols,
            continuous_cols=self.config.continuous_cols,
            embed_categorical=(not self.do_leave_one_out_encoder()),
            target=self.target if all(col in df.columns for col in self.target) else None,
        )
        return DataLoader(
            dataset,
            batch_size if batch_size is not None else self.batch_size,
            shuffle=False,
            num_workers=self.config.num_workers,
        )

    def save_dataloader(self, path: Union[str, Path]) -> None:
        """Saves the dataloader to a path.

        Args:
            path (Union[str, Path]): Path to save the dataloader
        """
        if isinstance(path, str):
            path = Path(path)
        joblib.dump(self, path)

    @classmethod
    def load_datamodule(cls, path: Union[str, Path]):
        """Loads a datamodule from a path.

        Args:
            path (Union[str, Path]): Path to the datamodule

        Returns:
            TabularDatamodule (TabularDatamodule): The datamodule loaded from the path
        """
        if isinstance(path, str):
            path = Path(path)
        if not path.exists():
            raise FileNotFoundError(f"{path} does not exist.")
        datamodule = joblib.load(path)
        return datamodule

class TabularDataset(Dataset):
    def __init__(
        self,
        data: pd.DataFrame,
        task: str,
        continuous_cols: List[str] = None,
        categorical_cols: List[str] = None,
        embed_categorical: bool = True,
        target: List[str] = None,
    ):
        """Dataset to Load Tabular Data.

        Args:
            data (pd.DataFrame): Pandas DataFrame to load during training
            task (str):
                Whether it is a classification or regression task. If classification, it returns a LongTensor as target
            continuous_cols (List[str], optional): A list of names of continuous columns. Defaults to None.
            categorical_cols (List[str], optional): A list of names of categorical columns.
                These columns must be ordinal encoded beforehand. Defaults to None.
            embed_categorical (bool):
                Flag to tell the dataset whether to convert categorical columns to LongTensor or retain as float.
                If we are going to embed categorical cols with an embedding layer,
                we need to convert the columns to LongTensor
            target (List[str], optional): A list of strings with target column name(s). Defaults to None.
        """

        self.task = task
        self.n = data.shape[0]

        if target:
            self.y = data[target].astype(np.float32).values
            if isinstance(target, str):
                self.y = self.y.reshape(-1, 1)  # .astype(np.int64)
        else:
            self.y = np.zeros((self.n, 1))  # .astype(np.int64)

        if task == "classification":
            self.y = self.y.astype(np.int64)
        self.categorical_cols = categorical_cols if categorical_cols else []
        self.continuous_cols = continuous_cols if continuous_cols else []

        if self.continuous_cols:
            self.continuous_X = data[self.continuous_cols].astype(np.float32).values

        if self.categorical_cols:
            self.categorical_X = data[categorical_cols]
            if embed_categorical:
                self.categorical_X = self.categorical_X.astype(np.int64).values
            else:
                self.categorical_X = self.categorical_X.astype(np.float32).values

    def __len__(self):
        """Denotes the total number of samples."""
        return self.n

    def __getitem__(self, idx):
        """Generates one sample of data."""
        return {
            "target": self.y[idx],
            "continuous": self.continuous_X[idx] if self.continuous_cols else torch.Tensor(),
            "categorical": self.categorical_X[idx] if self.categorical_cols else torch.Tensor(),
        }


'''
 Main entry point for HelixMonitor Webapp
 '''
from optparse import OptionParser
from helix_monitor import app, setupHelixSpectator
import sys

def constructCommandLineOptions(parser):
    parser = OptionParser(usage="usage: %prog [options]")
    parser.add_option('--host', default='localhost', help='Host address to bind to. [default: %(default)s]')
    parser.add_option ('--port', default=8000, type=int, help='Port to listen on. [default: %(default)s]')
    parser.add_option("--zkSvr", action="store", dest="zkServer", default="eat1-app207.stg:12913", help="zookeeper address [default: %default]")
    parser.add_option("--cluster", action="store", dest="cluster", default="ESPRESSO_DEV_FT_4", help = "Cluster name [default: %default]")
#    parser.add_option("--zkSvr", action="store", dest="zkServer", default="localhost:2181", help="zookeeper address [default: %default]")
#    parser.add_option("--cluster", action="store", dest="cluster", default="storage-integration-cluster", help = "Cluster name [default: %default]")

    return parser

def main(args):
    parser = constructCommandLineOptions(args)
    (options, args) = parser.parse_args()
    setupHelixSpectator(options.zkServer, options.cluster)
    app.run(host=options.host, port=options.port, debug=True)

if __name__ == "__main__":
    main(sys.argv[1:])


from DatasetAugmentation import ImageAugmentator, LabelSorter

FLAG_WIDTH = 120
FLAG_HEIGHT = 80
FLAG_DIR = 'flags'

ls = LabelSorter.LabelSorter()
ls.sort_labels_in_dir(FLAG_DIR)

ia = ImageAugmentator.ImageAugemtator(FLAG_DIR, FLAG_WIDTH, FLAG_HEIGHT)
ia.create_image_variations()


import pandas as pd
import numpy as np
import string
import re
from nltk.stem.snowball import SnowballStemmer
from nltk.corpus import stopwords
import nltk
import gensim
from gensim.models import CoherenceModel
from gensim import corpora
import tqdm
import pyLDAvis.gensim_models
import pickle
import pyLDAvis
import pyLDAvis.sklearn
import tqdm
from gensim.matutils import hellinger,kullback_leibler,jensen_shannon
from scipy.spatial.distance import jensenshannon

def compute_coherence_values(corpus, dictionary, k, a, b):
    lda_model = gensim.models.LdaModel(corpus=corpus,
                                           id2word=dictionary,
                                           num_topics=k,
                                           random_state=100,
                                           chunksize=100,
                                           passes=10,
                                           alpha=a,
                                           eta=b)

    coherence_model_lda = CoherenceModel(model=lda_model, texts=final_data['paper'], dictionary=dictionary, coherence='c_v')

    return coherence_model_lda.get_coherence()

def parse_topic_string(topic):
    # takes the string returned by model.show_topics()
    # split on strings to get topics and the probabilities
    topic = topic.split('+')
    # list to store topic bows
    topic_bow = []
    for word in topic:
        # split probability and word
        prob, word = word.split('*')
        # get rid of spaces and quote marks
        word = word.replace(" ", "").replace('"', '')
        # convert to word_type
        word = lda_model.id2word.doc2bow([word])[0][0]
        topic_bow.append((word, float(prob)))
    return topic_bow

if __name__ == '__main__':

    list_of_terms = ['imprint','molecular','polym','mip','abstract','analysi','appli','develop','effect','function',
                     'g','good','high','linear','method','mmip','model','modifi','molecul','phase','rang','relat',
                     'respect','result','sampl','show','standard','studi','use','howev','system','differ','author',
                     'present','approach']
    data = pd.read_excel('Dataset4LDA.xlsx')
    exclude = set(string.punctuation)
    exclude.remove('-')
    joined_rows = []
    stemmer = SnowballStemmer("english")
    stopwords = stopwords.words('english')
    stopwords.append('the')
    stopwords.append('a')
    joined_questions = pd.DataFrame(columns=['paper'])
    final_patents = pd.DataFrame(columns=['paper'])
    final_data = pd.DataFrame(columns=['paper'])
    for index,row in data.iterrows():
        join = row['TI'] + '.' + row['AB']

        joined_questions = joined_questions.append({'paper':join},ignore_index=True)
    #

    joined_questions = joined_questions.apply(lambda x: x.astype(str).str.lower())
    joined_questions['cleaned'] = joined_questions['paper'].apply(lambda x:''.join([i for i in x if i not in exclude]))
    joined_questions['tokenized'] = joined_questions.apply(lambda row: nltk.word_tokenize(row['cleaned']), axis=1)
    joined_questions['no_stop'] = joined_questions['tokenized'].apply(lambda x: [item for item in x if item not in stopwords])
    joined_questions['stemmed'] = joined_questions['no_stop'].apply(lambda x: [stemmer.stem(y) for y in x])

    for index, row in joined_questions.iterrows():
        final_terms = []
        for term in row['stemmed']:
            if term not in list_of_terms:
                final_terms.append(term)
        if not final_terms:
            final_patents = final_patents.append({'paper': ['empty', 'paper']}, ignore_index=True)
        else:
            final_patents = final_patents.append({'paper': final_terms}, ignore_index=True)

    for index, row in final_patents.iterrows():
        final_terms = []
        for term in row['paper']:
            if not term.isnumeric():
                final_terms.append(term)
        if not final_terms:
            final_data = final_data.append({'paper': ['empty', 'paper']}, ignore_index=True)
        else:
            final_data = final_data.append({'paper': final_terms}, ignore_index=True)
    final_data.to_excel('trial.xlsx')
    dictionary = corpora.Dictionary(final_data['paper'])
    bow_corpus = [dictionary.doc2bow(row) for row in final_data['paper']]
    lda_model = gensim.models.LdaModel(corpus=bow_corpus,
                                       id2word=dictionary,
                                       num_topics=7,
                                       random_state=100,
                                       chunksize=100,
                                       passes=10,
                                       alpha=0.61,
                                       eta=0.61,per_word_topics=True,minimum_probability=0.0)

    coherence_model_lda = CoherenceModel(model=lda_model, texts=final_data['paper'], dictionary=dictionary, coherence='c_v')
    print(coherence_model_lda.get_coherence())
    # lda_visualization = pyLDAvis.gensim_models.prepare(lda_model, bow_corpus, dictionary, mds='mmds',
    #                                                    sort_topics=False)
    # pyLDAvis.save_html(lda_visualization, 'lda_7_topics_molecular_titles_abstracts_removed_terms.html')

    topic_distribution = []
    for i in range (0,len(bow_corpus)):
        get_document_topics = lda_model.get_document_topics(bow_corpus[i])
        topic_distribution.append(get_document_topics)

    share = {}
    dominance = {}
    for distribution in topic_distribution:
        max = -10
        indexx = 50
        for tuplee in distribution:
            if tuplee[1]>0.1:
                if tuplee[0] in share.keys():
                    share[tuplee[0]] +=1
                else:
                    share[tuplee[0]] = 1
            if tuplee[1] > max:
                max = tuplee[1]
                indexx = tuplee[0]
        if indexx in dominance.keys():
            dominance[indexx] +=1
        else:
            dominance[indexx] = 1

    for key in share.keys():
        share[key] = (share[key]/len(bow_corpus))*100
        dominance[key] = (dominance[key]/len(bow_corpus))*100

    share_df = pd.DataFrame(share.items(),columns=['Topic','Share']).to_excel('share_molecular.xlsx')
    dominance_df = pd.DataFrame(dominance.items(),columns=['Topic','Popularity']).to_excel('popularity_molecular.xlsx')

    topic_1, topic_2, topic_3, topic_4, topic_5, topic_6, topic_7, = lda_model.show_topics(num_topics=7, num_words=len(lda_model.id2word))
    topic_1_distribution = parse_topic_string(topic_1[1])
    topic_2_distribution = parse_topic_string(topic_2[1])
    topic_3_distribution = parse_topic_string(topic_3[1])
    topic_4_distribution = parse_topic_string(topic_4[1])
    topic_5_distribution = parse_topic_string(topic_5[1])
    topic_6_distribution = parse_topic_string(topic_6[1])
    topic_7_distribution = parse_topic_string(topic_7[1])

    distributions = [topic_1_distribution, topic_2_distribution, topic_3_distribution, topic_4_distribution,
                     topic_5_distribution,
                     topic_6_distribution, topic_7_distribution]
    new_dist = []
    for distribution in distributions:
        to_add = []
        for element in distribution:
            to_add.append(element[1])
        new_dist.append(to_add)

    index = 1
    distances = {}
    for distribution in new_dist:
        dst_list = []
        for distribution2 in new_dist:
            dst = jensenshannon(distribution, distribution2)
            dst_list.append(dst)
        distances['Topic {}'.format(index)] = dst_list
        index+=1

    distances_df = pd.DataFrame(distances)
    distances_df.to_excel('topic_distances_jensen_shannon_molecular.xlsx')

    all_memberships = {0:[],
                       1:[],
                       2:[],
                       3:[],
                       4:[],
                       5:[],
                       6:[],}
    for distro in topic_distribution:
        for tuplee in distro:
            all_memberships[tuplee[0]].append(tuplee[1])

    b = pd.DataFrame(all_memberships)
    b.to_excel('memberships.xlsx')
    # min_topics = 2
    # max_topics = 11
    # step_size = 1
    # topics_range = range(min_topics, max_topics, step_size)  # Alpha parameter
    # alpha = list(np.arange(0.01, 1, 0.3))
    # alpha.append('symmetric')
    # alpha.append('asymmetric')  # Beta parameter
    # beta = list(np.arange(0.01, 1, 0.3))
    # beta.append('symmetric')  # Validation sets
    # num_of_docs = len(bow_corpus)
    # model_results = {
    #                  'Topics': [],
    #                  'Alpha': [],
    #                  'Beta': [],
    #                  'Coherence': []
    #                  }  # Can take a long time to run
    #
    # pbar = tqdm.tqdm(total=270)
    #     # iterate through number of topics
    # for k in topics_range:
    #     # iterate through alpha values
    #     for a in alpha:
    #         # iterare through beta values
    #         for b in beta:
    #             # get the coherence score for the given parameters
    #             cv = compute_coherence_values(corpus=bow_corpus, dictionary=dictionary,
    #                                           k=k, a=a, b=b)
    #             print(cv)
    #             # Save the model results
    #             model_results['Topics'].append(k)
    #             model_results['Alpha'].append(a)
    #             model_results['Beta'].append(b)
    #             model_results['Coherence'].append(cv)
    #             pbar.update(1)
    # pd.DataFrame(model_results).to_excel('lda_tuning_results_molecular_titles_removed_terms.xlsx', index=False)
    # pbar.close()


#Flask-Mail

from flask import Flask
from flask_mail import Mail, Message

app = Flask(__name__)

def send(subject, msg, from_email, to_email):
    mail = Mail(app)
    msg = Message(subject, [to_email], body, sender=from_email)
    mail.send(msg) # Sensitive{code}


# Find Circles Example
#
# This example shows off how to find circles in the image using the Hough
# Transform. https://en.wikipedia.org/wiki/Circle_Hough_Transform
#
# Note that the find_circles() method will only find circles which are completely
# inside of the image. Circles which go outside of the image/roi are ignored...
#初始化
import sensor, image, time , lcd, math ,array
import gc
from modules import ybkey
KEY = ybkey()
from modules import ybrgb
RGB = ybrgb()
gc.enable()
lcd.init()
sensor.reset()
sensor.set_pixformat(sensor.GRAYSCALE) # grayscale is faster
sensor.set_framesize(sensor.QVGA)
sensor.skip_frames(time = 2000)
sensor.set_auto_gain(True) # must turn this off to prevent image washout...
clock = time.clock()

while(True):
    clock.tick()
    img = sensor.snapshot()
    i=0;
    mid_point_x=0.0;
    mid_point_y=0.0;
    point_list_x=array.array("I")
    point_list_y=array.array("I")
    # Circle objects have four values: x, y, r (radius), and magnitude. The
    # magnitude is the strength of the detection of the circle. Higher is
    # better...

    # `threshold` controls how many circles are found. Increase its value
    # to decrease the number of circles detected...

    # `x_margin`, `y_margin`, and `r_margin` control the merging of similar
    # circles in the x, y, and r (radius) directions.

    # r_min, r_max, and r_step control what radiuses of circles are tested.
    # Shrinking the number of tested circle radiuses yields a big performance boost.
    for c in img.find_circles(threshold = 2500, x_margin = 10, y_margin = 10, r_margin = 10,
            r_min = 4, r_max = 6, r_step = 1):
        img.draw_circle(c.x(), c.y(), c.r(), color = (255, 255, 255))
        mid_point_x=mid_point_x+c.x()
        mid_point_y=mid_point_y+c.y()
        print(c)
        i=i+1;
        point_list_x.append(c.x());
        point_list_y.append(c.y());
    img.draw_string(0, 0, "%i" %(i), color=(255, 255, 255), scale=2.0)
    if i==8:
        mid_point_x=mid_point_x/8.0
        mid_point_y=mid_point_y/8.0
        img.draw_circle(int(mid_point_x), int(mid_point_y), 6, color = (255, 255, 255))
        print(mid_point_x)
        print(mid_point_y)
        RGB.set(0,255,0)
    else:
        RGB.set(0,0,0)
    lcd.display(img)
    print("FPS %f" % clock.fps())
    print(i)
    if KEY.is_press() and i==8:
        RGB.set(0,0,0)
        break
#垃圾回收
gc.collect()
import  math
primary=[[217,62],[56,64],[171,86],[80,110],[240,131],[149,155],[104,178],[265,177]]

def change(data):
    x=0
    y=0
    for i in range(len(data)):
        x += data[i][0]
        y += data[i][1]
    x=x/8
    y=y/8
    second=[]
    distance=0
    max=7.7
    for i in range(len(data)):
        second.append([data[i][0]-x,data[i][1]-y])
    x = 0
    y = 0
    flag=0
    mid=second
    for i in range(len(second)):
        if math.sqrt((second[i][0]-x)**2+(second[i][1]-y)**2)>distance:
            distance=math.sqrt((second[i][0]-x)**2+(second[i][1]-y)**2)
            flag=i

    for j in range(1,20):
        if distance<max:
            break
        else:
            for i in range(len(second)):
                second[i][0] =mid[i][0]/j
                second[i][1] =mid[i][1]/j
            distance=math.sqrt((second[flag][0]-x)**2+(second[flag][1]-y)**2)

    for i in range(len(second)):
        second[i][0] = round(second[i][0]+5.5)
        second[i][1] = round(second[i][1]+5.5)
    for i in range(len(second)):
        second[i][1] = 10-second[i][1]
    print(second)
change(primary)


from __future__ import print_function

import logging
import os
import subprocess

import sassie.util.sasconfig as sasconfig
import sassie.util.module_utilities as module_utilities

import sassie.build.pdbscan.pdbscan as pdbscan
import sassie.build.pdbscan.pdbscan.report as report
import sassie.build.pdbscan.pdbscan.pdbscan_utils as pdbscan_utils

if sasconfig.__level__ == "DEBUG": DEBUG = True

app = 'pdbscan'

class module_variables():
    def __init__(self,parent = None):
        self.app = app

class PDBScan():

    def __init__(self, parent = None):
        pass

    def main(self, input_variables, txtOutput):

        self.mvars = module_variables()

        self.run_utils = module_utilities.run_utils(app,txtOutput)

        self.run_utils.setup_logging(self)

        self.log.debug('in main')

        self.unpack_variables(input_variables)

        self.run_utils.general_setup(self)

        self.run_scan()

        return

    def unpack_variables(self,variables):

        mvars = self.mvars

        self.log.debug('in unpack_variables')

        mvars.runname = variables['runname'][0]
        mvars.pdbfile = variables['pdbfile'][0]
        mvars.pdbname = os.path.splitext(os.path.basename(mvars.pdbfile))[0]

        return

    def generate_full_report(self, report_lines):
        """
        @type  report_lines:
        @param report_lines:
        @return:
        """

        mkd_file = self.mvars.pdbname + '.mkd'
        mkd_filepath = os.path.join(self.runpath, mkd_file)

        out_file = open(mkd_filepath,'w')

        for line in report_lines:
            print(line,file=out_file)

        return mkd_filepath

    def report_to_user(self, report_lines):

        for line in report_lines:
            self.run_utils.print_gui(line)

        return

    def run_scan(self):

        mvars = self.mvars
        pgui = self.run_utils.print_gui
        log = self.log

        pgui('STATUS\t0.1')

        log.debug('in run_scan')

        pgui('-'*50)
        pgui('PDB Scan')
        pgui('-'*50)

        mol = pdbscan.SasMolScan()
        mol.read_pdb(mvars.pdbfile)

        pgui('STATUS\t0.2')

        pgui('Initiating scan')
        mol.run_scan()
        mol.copy_biomt_segments()

        pgui('STATUS\t0.6')

        short_report, long_report = report.generate_reports(mol)

        self.report_to_user(short_report)

        pgui('STATUS\t0.9')

        mkd_filepath = self.generate_full_report(long_report)
        html_filepath = pdbscan_utils.convert_mkd_html(mkd_filepath)

        pgui('STATUS\t1.0')


import re
import unittest

from trac.tests.functional import FunctionalTestCaseSetup, \
                                  internal_error, tc

#TODO: split this into multiple smaller testcases
class TestPreferences(FunctionalTestCaseSetup):
    def runTest(self):
        """Set preferences for admin user"""
        self._tester.go_to_preferences()
        try:
            tc.notfind('Your preferences have been saved.')
            tc.formvalue('userprefs', 'name', ' System Administrator ')
            tc.formvalue('userprefs', 'email', ' admin@example.com ')
            tc.submit()
            tc.find('Your preferences have been saved.')
            self._tester.go_to_preferences("Localization")
            tc.formvalue('userprefs', 'tz', 'GMT -10:00')
            tc.submit()
            tc.find('Your preferences have been saved.')
            self._tester.go_to_preferences()
            tc.notfind('Your preferences have been saved.')
            tc.find('value="System Administrator"')
            tc.find(r'value="admin@example\.com"')
            self._tester.go_to_preferences("Localization")
            tc.find('GMT -10:00')
        finally:
            self._tester.go_to_preferences()
            tc.formvalue('userprefs', 'name', '')
            tc.formvalue('userprefs', 'email', '')
            tc.submit()
            tc.find('Your preferences have been saved.')

class TestDefaultHandler(FunctionalTestCaseSetup):
    def runTest(self):
        """Set default handler."""

        # Project default_handler is selected.
        self._tester.go_to_preferences("User Interface")
        tc.notfind(r'<option[^>]+selected="selected"')
        tc.find("Default: WikiModule")

        try:
            # Project default handler still selected after value is changed.
            hint = r"SearchModule is not a valid IRequestHandler or is not" \
                   r" enabled\."
            self._testenv.set_config('trac', 'default_handler',
                                     'SearchModule')
            self._tester.go_to_preferences("User Interface")
            tc.notfind('<option[^>]+selected="selected"')
            tc.find("Default: SearchModule")
            tc.notfind(hint)

            # Project default handler still selected after module is disabled.
            component = 'trac.search.web_ui.*'
            self._testenv.set_config('components', component, 'disabled')
            self._tester.go_to_preferences("User Interface")
            try:
                tc.notfind('<option[^>]+selected="selected"')
                tc.find(r"Default: SearchModule")
                tc.find(hint)
            finally:
                self._testenv.remove_config('components', component)
        finally:
            self._testenv.set_config('trac', 'default_handler', 'WikiModule')

        # Set session default handler and navigate to base URL.
        tc.formvalue('userprefs', 'default_handler', 'TimelineModule')
        tc.submit()
        tc.find(r"Your preferences have been saved\.")
        tc.find('<option[^>]+selected="selected"[^>]+TimelineModule')
        self._tester.go_to_front()
        tc.find("
Timeline
")

        # Clear session default handler.
        self._tester.go_to_preferences("User Interface")
        tc.formvalue('userprefs', 'default_handler', '')
        tc.submit()
        tc.find(r"Your preferences have been saved\.")
        tc.notfind(r'<option[^>]+selected="selected"')
        tc.find("Default: WikiModule")

class RegressionTestRev5785(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of the fix in r5785"""
        self._tester.go_to_preferences()
        tc.submit('logout', 'logout')
        tc.notfind(internal_error)  # See [5785]
        tc.follow('Login')

class RegressionTestTicket5765(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/5765
        Unable to turn off 'Enable access keys' in Preferences
        """
        self._tester.go_to_preferences("User Interface")
        tc.formvalue('userprefs', 'accesskeys', True)
        tc.submit(formname='userprefs')
        tc.find('name="accesskeys".*checked="checked"')
        tc.formvalue('userprefs', 'accesskeys', False)
        tc.submit(formname='userprefs')
        tc.notfind('name="accesskeys".*checked="checked"')

class RegressionTestTicket11319(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/11319
        Only alphanumeric characters can be used for session key in advanced
        panel.
        """
        try:
            self._tester.logout()
            self._tester.go_to_preferences('Advanced')
            tc.formvalue('userprefs', 'newsid', 'śeśśion_id')
            tc.submit('change')
            tc.notfind(internal_error)
            tc.find('Session ID must be alphanumeric')
            self._tester.go_to_preferences('Advanced')
            tc.formvalue('userprefs', 'loadsid', 'śeśśion_id')
            tc.submit('restore')
            tc.notfind(internal_error)
            tc.find('Session ID must be alphanumeric')
        finally:
            self._tester.login('admin')

class RegressionTestTicket11337(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/11337
        The language select will be disabled if Babel is not installed and a
        hint will be shown. The text of the hint is dependent on whether the
        user has TRAC_ADMIN and the message catalogs have been compiled.
        """
        from trac.util.translation import has_babel, get_available_locales

        babel_hint = "Install Babel for extended language support."
        catalog_hint = "Message catalogs have not been compiled."
        nonadmin_hint = r'\s+'.join("Please contact your Trac administrator "
                                    "to enable existing translations".split())
        language_select = '<select id="language" name="language">'
        disabled_language_select = \
            '<select id="language" name="language" disabled="disabled" ' \
            'title="Translations are currently unavailable">'

        self._tester.go_to_preferences("Localization")
        if has_babel:
            tc.notfind(babel_hint)
            if get_available_locales():
                tc.find(language_select)
                tc.notfind(babel_hint)
                tc.notfind(catalog_hint)
            else:
                tc.find(disabled_language_select)
                tc.find(catalog_hint)
                tc.notfind(babel_hint)
        else:
            tc.find(disabled_language_select)
            tc.find(babel_hint)
            tc.notfind(catalog_hint)
        tc.notfind(nonadmin_hint)

        # For users without TRAC_ADMIN, the Language tab should only be
        # present when Babel is installed
        self._tester.logout()
        self._tester.go_to_preferences("Localization")
        try:
            if has_babel and get_available_locales():
                tc.find(language_select)
                tc.notfind(nonadmin_hint)
            elif has_babel:
                tc.find(disabled_language_select)
                tc.find(nonadmin_hint)
            else:
                tc.find(disabled_language_select)
                tc.find(nonadmin_hint)
            tc.notfind(catalog_hint)
            tc.notfind(babel_hint)
        finally:
            self._tester.login('admin')

class RegressionTestTicket11515(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/11515
        Show a notice message with new language setting after it is changed.
        """
        from trac.util.translation import has_babel, get_available_locales
        from pkg_resources import resource_exists, resource_filename

        if not has_babel:
            print("SKIP: RegressionTestTicket11515 (Babel not installed)")
            return
        if not resource_exists('trac', 'locale'):
            return
        locale_dir = resource_filename('trac', 'locale')
        from babel.support import Translations
        string = 'Your preferences have been saved.'
        translated = None
        for second_locale_id in get_available_locales():
            tx = Translations.load(locale_dir, second_locale_id)
            translated = tx.dgettext('messages', string)
            if string != translated:
                break  # the locale has a translation
        else:
            print("SKIP: RegressionTestTicket11515 "
                  "(Message catalogs not compiled)")
            return

        try:
            self._tester.go_to_preferences('Localization')
            tc.formvalue('userprefs', 'language', second_locale_id)
            tc.submit()
            tc.find(re.escape(translated))
            tc.find('<option selected="selected" value="%s">'
                    % second_locale_id)
        finally:
            tc.formvalue('userprefs', 'language', '')  # revert to default
            tc.submit()
            tc.find('Your preferences have been saved')

class RegressionTestTicket11531(FunctionalTestCaseSetup):
    """Test for regression of https://trac.edgewall.org/ticket/11531
    PreferencesModule can be set as the default_handler."""
    def runTest(self):
        default_handler = self._testenv.get_config('trac', 'default_handler')
        self._testenv.set_config('trac', 'default_handler',
                                 'PreferencesModule')
        try:
            tc.go(self._tester.url)
            tc.notfind(internal_error)
            tc.find(r"\bPreferences\b")
        finally:
            self._testenv.set_config('trac', 'default_handler',
                                     default_handler)

def functionalSuite(suite=None):
    if not suite:
        import trac.tests.functional
        suite = trac.tests.functional.functionalSuite()
    suite.addTest(TestPreferences())
    suite.addTest(TestDefaultHandler())
    suite.addTest(RegressionTestRev5785())
    suite.addTest(RegressionTestTicket5765())
    suite.addTest(RegressionTestTicket11319())
    suite.addTest(RegressionTestTicket11337())
    suite.addTest(RegressionTestTicket11515())
    suite.addTest(RegressionTestTicket11531())
    return suite

test_suite = functionalSuite

if __name__ == '__main__':
    unittest.main(defaultTest='test_suite')


from django.urls import path
from .views import RegisterView, LoginView, logout_view

app_name = 'accounts'
urlpatterns = [
    path('register/', RegisterView.as_view(), name='register'),
    path('login/', LoginView.as_view(), name='login'),
    path('logout/', logout_view, name='logout'),

]


import boto3
import pdfplumber
from time import sleep
import requests
import os, sys, subprocess

AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
AWS_BUCKET_NAME = 'practisebucketaws'
TOPIC_ARN = 'arn:aws:sns:ap-southeast-1:524255913858:audiobook'
region = 'ap-southeast-1'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 '
                        '(KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'}

# Extract text from pdf
with pdfplumber.open('The Girl with the Dragon Tattoo.pdf') as pdf:
    all_pages = pdf.pages
    full_text = ''
    for page in all_pages[:20]:
        single_page_text = page.extract_text(x_tolerance=1)
        full_text = full_text + '\n' + single_page_text

# Create a boto3 session
polly_client = boto3.Session(aws_access_key_id=AWS_ACCESS_KEY_ID,
                             aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
                             region_name=region).client('polly')

# Request speech synthesis
response = polly_client.start_speech_synthesis_task(VoiceId='Joanna',
                                                    OutputS3BucketName=AWS_BUCKET_NAME,
                                                    OutputFormat='mp3',
                                                    Text=str(full_text),
                                                    SnsTopicArn=TOPIC_ARN,
                                                    Engine='neural')

# Get mp3 file uri from response
mp3_uri = response['SynthesisTask']['OutputUri']

# Wait for speech to be fully synthesised
retries = 0
max_retries = 29
still_try = True
while retries < max_retries and still_try == True:
    if requests.get(mp3_uri, headers=headers).status_code != 200:
        retries += 1
        print(f'mp3 file not ready..')
        sleep(60)
    else:
        print(f'mp3 file is ready..')
        audiobook = requests.get(mp3_uri, headers=headers)
        with open('audiobook.mp3', 'wb') as file:
            file.write(audiobook.content)
        still_try = False

# Play audiobook
if sys.platform == "win32":
    os.startfile('audiobook.mp3')
else:
    opener = "open" if sys.platform == "darwin" else "xdg-open"
    subprocess.call([opener, 'audiobook.mp3'])


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# author: tianhl

# using the new Workflow Engine for data analysis.

init_dict   = {
               'helloDaisy':{'class_name':'PyAlgorithms.AlgHelloDaisy',\
                          },\
              }

import Daisy

if __name__ == "__main__":
    wf = Daisy.Workflow.WorkflowHelloDaisy('HelloDaisyWorkflow')
    wf.initialize(workflow_engine='PyWorkflowEngine', workflow_environment = init_dict)
    wf.execute()
    alg =wf.algorithm_keys()
    print('algs in workflow')
    print(alg)

    wf.finalize()


#! /usr/bin/env python
# -*- coding:UTF-8 -*-

import unittest
from public.HTMLTestReportCN import HTMLTestRunner
import time

test_case_dir = '\\testcases'

def create_suite():
    test_suits = unittest.TestSuite()

    discover = unittest.defaultTestLoader.discover(
        test_case_dir,
        pattern='test*',
        top_level_dir=None
    )

    for test_suit in discover:
        for test_case in test_suit:
            test_suits.addTest(test_case)
            print(test_suits)

    return test_suits

all_test_cases = create_suite()

now = time.strftime('%Y-%m_%d-%H_%M_%S', time.localtime())

file_name = '\\report\\' + now + 'result.html'

with open(file_name, 'wb') as fp:
    runner = HTMLTestRunner(
        stream=fp,
        title=u'web自动化测试报告',
        description=u'用例具体执行情况如下：'
    )

runner.run(all_test_cases)


import PySimpleGUIQt as qt

class InsPyPongInstaller:

    def __init__(self, destination_dir=None):
        import platform
        if destination_dir is None:
            if platform.uname().system == 'Windows':
                print('You are on Windows!')
                self.destination = 'C:/Program\ Files/Inspyre\ Softworks/insPyPong'
                qt.PopupGetFolder('Is this where you\'d like to install insPyPong from Inspyre Softworks?',
                                  initial_folder=self.destination)

            if platform.uname().system == 'Linux':
                print('You are on Linux!')

InsPyPongInstaller()


class Config:
    bot_name = 'FeyiTechTradeBot'
    personal_website= 'https://feyitech.com'
    company_website= 'https://softbaker.com'
    personal_email= 'hello@feyitech.com'
    company_email= 'hello@softbaker.com'
    phone_number= '+234-9024-500-275'
    twitter= 'https://twitter.com/feyi_tech'
    telegram= 'https://t.me/feyitech'
    is_test = True
    update_messages = False
    fetch_interval_seconds = 10
    max_leverage = 100
    max_positions_per_chart = 1000
    timeframe = '1m'
    timeframe_in_seconds = 60
    market_info_update_interval_seconds = 86400 # 1 day
    base_assets_whitelist = []
    base_assets_blacklist = []
    quote_assets_whitelist = ['USDT', 'BUSD']
    quote_assets_blacklist = []
    time_format = '%b, %d %Y - %I:%M:%S %p'
    # the percentage of users profits the bot takes as profit when users make profits
    bot_fee_profit_percentage = 0
    # the ERC20 address the bot's fee should be withdrawn to when users make profits
    bot_fee_profit_destination = None
    class secrets:
        telegram_token = '' # enter your Telegram Bot token
        admin_chat_id = '' # enter your chat ID/user ID to prevent other users to use the bot
    class Binance:
        key = ""
        secret = ""


import turtle

def binary_tree(rank: int, length: float) -> None:
    turtle.forward(length)
    if rank > 0:
        turtle.left(45)
        binary_tree(rank - 1, length / 2)
        turtle.right(90)
        binary_tree(rank - 1, length / 2)
        turtle.left(45)
    turtle.back(length)

turtle.speed(0)
turtle.penup()
turtle.left(90)
turtle.back(350)
turtle.pendown()
turtle.pensize(3)
binary_tree(5, 400)
turtle.done()


#
# @lc app=leetcode.cn id=145 lang=python3
#
# [145] 二叉树的后序遍历
#

# @lc code=start
# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right

def postorderTraversal1(root):
    if not root:
        return []
    l,r=postorderTraversal1(root.left),postorderTraversal1(root.right)
    return [*l,*r,root.val]

def postorderTraversal2(root):
    s,res=[],[]
    if root:
        # store node and visited
        s.append((root, False))
    while s:
        node,visited=s.pop()
        if not node:
            continue
        if not visited:
            # hasn't visited, visit left,right ,then back to node itself
            s.append((node,True))
            s.append((node.right,False))
            s.append((node.left,False))
        else:
            # has visited left and right, visit current
            res.append(node.val)
    return res

class Solution:
    def postorderTraversal(self, root: Optional[TreeNode]) -> List[int]:
        return postorderTraversal1(root)
        #return postorderTraversal2(root)
# @lc code=end

145.二叉树的后序遍历.py


import argparse
from cerebralcortex.core.data_manager.raw.stream_handler import DataSet
from cerebralcortex.cerebralcortex import CerebralCortex
from cerebralcortex.core.datatypes.datastream import DataStream
from cerebralcortex.core.datatypes.datastream import DataPoint

import numpy
from datetime import datetime, timedelta, timezone, tzinfo

class SleepUnsupervisedPredictor():
    """
    Unsupervised sleep duration predictor from http://ieeexplore.ieee.org/document/6563918/
    """

    def find_longest_subvector(self, vector: object, val: object, thresh: object = 8, gap: object = 4) -> object:
        """

        :rtype: object
        :param vector:
        :param val:
        :param thresh:
        :param gap:
        :return:
        """
        cur_start_idx = -1

        candidates = []

        for i in range(vector.shape[0]):
            if vector[i] == val:
                if cur_start_idx == -1:
                    cur_start_idx = i

            else:
                if cur_start_idx == -1:
                    continue

                if i - cur_start_idx + 1 > thresh:
                    candidates.append([cur_start_idx, i])

                cur_start_idx = -1

        if cur_start_idx != -1:
            candidates.append([cur_start_idx, i - 1])

        max_item = [0, 0]
        current_item = None
        for item in candidates:
            if current_item is None:
                current_item = item
            elif item[0] - current_item[1] + 1 <= gap:
                current_item[1] = item[1]
            else:
                if current_item[1] - current_item[0] > max_item[1] - max_item[0]:
                    max_item = current_item

                current_item = item
        if current_item is not None and current_item[1] - current_item[0] > max_item[1] - max_item[0]:
            max_item = current_item

        return max_item[0], max_item[1]

    def rolling_window(self, a: object, size: object) -> object:
        """

        :rtype: object
        :param a:
        :param size:
        :return:
        """
        shape = a.shape[:-1] + (a.shape[-1] - size + 1, size)
        strides = a.strides + (a.strides[-1],)

        return numpy.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)

    def pyramid(self, input_vector: object, scale: object, threshold: object = 0.05) -> object:
        """

        :rtype: object
        :param input_vector:
        :param scale:
        :param threshold:
        :return:
        """
        input_matrix = numpy.reshape(input_vector, (-1, scale))

        input_mean = numpy.nanmean(input_matrix, axis=1)

        output = numpy.zeros(input_mean.shape)
        output[1:] = numpy.abs(numpy.diff(input_mean))

        output[numpy.isnan(output)] = 1000

        if (output.max() - output.min()) == 0:
            output = numpy.zeros(output.shape)
        else:
            output = (output - output.min()) / (output.max() - output.min())

        positive = output <= threshold
        output[positive] = 1
        output[~positive] = 0

        bool_indices = numpy.all(self.rolling_window(output, 2) == [0, 1], axis=1)
        bool_indices = numpy.concatenate((bool_indices, [False]))
        output[bool_indices] = 1

        return output

    def expand(self, input_vector: object, scale: object) -> object:
        """

        :rtype: object
        :param input_vector:
        :param scale:
        :return:
        """
        input_matrix = numpy.reshape(input_vector, (-1, 1))
        expand_matrix = numpy.tile(input_matrix, (1, int(scale)));

        return expand_matrix.flatten()

    def vote(self, vector: object, threshold: object = 0.05) -> object:
        """

        :rtype: object
        :param vector:
        :param threshold:
        :return:
        """
        vector_60m = self.pyramid(vector, 3600, threshold)
        vector_30m = self.pyramid(vector, 1800, threshold)
        vector_15m = self.pyramid(vector, 900, threshold)
        vector_7_5m = self.pyramid(vector, 450, threshold)

        vector_vote = self.expand(vector_60m, 60 / 7.5) * 0.3 + self.expand(vector_30m, 30 / 7.5) * 0.3 + self.expand(
            vector_15m, 15 / 7.5) * 0.2 + vector_7_5m * 0.1

        return vector_vote

    def predict(self, audio_vector: object, ligth_vector: object, act_vector: object, lock_vector: object) -> object:
        """

        :rtype: object
        :param audio_vector:
        :param ligth_vector:
        :param act_vector:
        :param lock_vector:
        :return:
        """
        act_vote = self.pyramid(act_vector, 900, threshold=0.03);
        lock_vote = self.pyramid(lock_vector, 900, threshold=0.1);

        audio_vote = self.vote(audio_vector, 0.05)
        light_vote = self.vote(ligth_vector, 0.008)

        sleep_vote = (light_vote + .5 * audio_vote)
        sleep_vote *= self.expand(act_vote, 15 / 7.5) * self.expand(lock_vote, 15 / 7.5)
        sleep_vote[sleep_vote < sleep_vote.max() * 0.6] = 0

        sleep_vote[sleep_vote > 0] = 1

        longest_start_idx, longest_end_idx = self.find_longest_subvector(sleep_vote, 1, 8, 1)

        return longest_start_idx, longest_end_idx, sleep_vote.shape


import telepot
import time
import datetime
# import logging

TOKEN = 'Your Telegram bot TOKEN'

# Job Q for reading message updates

class TGBot():
    def __init__(self, q):
        self.bot = telepot.Bot(TOKEN)
        self.chat_ids = []
        self.q = q

        def loop():
            print("loop")
            if not self.q.empty():
                msg = self.q.get()
                self.send_messages(msg)

        self.bot.message_loop(self.handle)
        while 1:
            time.sleep(3)
            loop()

    def send_messages(self, text):
        for i in self.chat_ids:
            self.bot.sendMessage(i, text)

    def follow(self, chat_id):
        if chat_id in self.chat_ids:
            return
        self.bot.sendMessage(chat_id, "Hey! Now you have started to monitor my creator's keyboard")
        self.chat_ids.append(chat_id)

    def handle(self, msg):
        chat_id = msg['chat']['id']
        command = msg['text']

        print('Got command: %s' % command)

        if command == '/follow':
            self.follow(chat_id)
        elif command == '/unfollow':
            self.unfollow(chat_id)

    def unfollow(self, chat_id):
        if chat_id in self.chat_ids:
            self.bot.sendMessage(chat_id, "See you later!")
            self.chat_ids.remove(chat_id)
        else:
            self.bot.sendMessage(chat_id, "At first You should use the command /follow to start to recieve messages")


from termcolor import colored
from utils import Point
import heapq, math

def d(a, b): return math.sqrt((a.real-b.real)**2+(a.imag-b.imag)**2)

def main(day_input):
    heightmap = {Point(x, y): h for y, row in enumerate(day_input) for x, h in enumerate(row)}

    for k, v in heightmap.items():
        if v == 'S':
            start = k
            heightmap[k] = 'a'
        if v == 'E':
            end = k
            heightmap[k] = 'z'

    xmax, ymax = int(max([m.real for m in heightmap.keys()])), int(max([m.imag for m in heightmap.keys()]))

    arrows = {-1: '<',1: '>',1j: 'v',-1j: '^',0j: 'X'}

    def print_map(path1, path2):
        print()
        def get_char(path, cpos):
            i = path.index(cpos)
            try: diff = path[i+1]-cpos
            except IndexError: diff = end-cpos
            try: return arrows[diff]
            except KeyError: return '#'
        for y in range(ymax+1):
            row=''
            for x in range(xmax+1):
                cpos = Point(x, y)
                if cpos in path1:
                    row += colored(get_char(path1, cpos), 'green')
                elif cpos in path2:
                    row += colored(get_char(path2, cpos), 'red')
                else:
                    row += heightmap[cpos]
            print(row)
        print()

    def walk(current_start):
        q = [(0, d(current_start, end), [current_start])]
        heapq.heapify(q)
        visited = set([])
        while q:
            _, _, path = heapq.heappop(q)

            pos = path[-1]

            if pos == end:
                return path

            for next_pos in [pos+off for off in (-1j, 1, 1j, -1)]:
                if next_pos in heightmap and next_pos not in visited and (ord(heightmap[next_pos])-ord(heightmap[pos])) <= 1:
                    heapq.heappush(q, (len(path)+1, d(next_pos, end), path + [next_pos]))
                    visited.add(next_pos)

    path = walk(start)
    min_path = min([walk(s) for s, h in heightmap.items() if h == 'a'], key=lambda p: len(p) if p else float('inf'))

    # print_map(min_path, path)

    return len(path)-1, len(min_path)-1


import unittest, sys
import testlib_data

use_private = True
try:
    from private.qrt.pango10 import Pango10PrivateTest
except ImportError:
    use_private = False
    print >>sys.stdout, "Skipping private tests"

class Pango10Test(testlib_data.DataCommon):
    '''Test pango1.0.'''

    def setUp(self):
        '''Set up prior to each test_* function'''
        testlib_data.DataCommon._setUp(self, dir='./pango1.0')
        self.exes = ['pango-view']

    def tearDown(self):
        '''Clean up after each test_* function'''
        pass

    def test_examples(self):
        '''Test example files'''
        for exe in self.exes:
            self._cmd([exe], "txt", url=False, expected_rc=1, dir='pango1.0')

if __name__ == '__main__':
    # more configurable
    suite = unittest.TestSuite()
    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(Pango10Test))

    # Pull in private tests
    if use_private:
        suite.addTest(unittest.TestLoader().loadTestsFromTestCase(Pango10PrivateTest))

    rc = unittest.TextTestRunner(verbosity=2).run(suite)
    if not rc.wasSuccessful():
        sys.exit(1)


from __future__ import absolute_import, print_function, division

import sys
import subprocess
import itertools
import ConfigParser
import fnmatch
from PyQt5.QtCore import QObject, pyqtSignal
from idc import *

from .index import Index

class Project(QObject):
    """Represents a continuum project and it's properties."""
    META_DIR_NAME = '.continuum'
    CFG_FILE_NAME = 'project.conf'

    def __init__(self):
        super(Project, self).__init__()
        self.conf = None
        self.index = None
        self.proj_dir = None
        self.meta_dir = None
        self.ignore_changes = False
        self.files = []

    def open(self, root, skip_analysis=False):
        """Opens an existing project by its root path."""
        if self.proj_dir:
            raise Exception("A project is already opened")

        # Find meta directory and read config.
        meta_dir = os.path.join(root, self.META_DIR_NAME)
        conf = ConfigParser.SafeConfigParser()
        if not conf.read(os.path.join(meta_dir, self.CFG_FILE_NAME)):
            raise Exception("Project is lacking its config file")

        # Determine project files.
        file_patterns = conf.get('project', 'file_patterns')
        if file_patterns is None:
            raise Exception("Project configuration lacks `file_patterns` directive")
        files = list(self.find_project_files(root, file_patterns))

        # Everything fine, put info into `self`.
        self.conf = conf
        self.proj_dir = root
        self.meta_dir = meta_dir
        self.files = files
        self.index = Index(self)

        if not skip_analysis:
            # Is index for *this* IDB built? If not, do so.
            if not self.index.is_idb_indexed(GetIdbPath()):
                self.index.index_types_for_this_idb()
                self.index.index_symbols_for_this_idb()

            # Analyze other files, if required.
            self._analyze_project_files()

    def _analyze_project_files(self):
        """Launches background analysis instances for binaries that aren't indexed yet."""
        plugin_root = os.path.dirname(os.path.realpath(__file__))
        procs = []

        for cur_file in self.files:
            # IDB already indexed? Skip.
            if self.index.is_idb_indexed(self.file_to_idb(cur_file)):
                continue

            procs.append(subprocess.Popen([
                sys.executable,
                '-A',
                '-S"{}"'.format(os.path.join(plugin_root, 'analyze.py')),
                '-L{}.log'.format(cur_file),
                cur_file,
            ]))

        return procs

    @staticmethod
    def file_to_idb(file):
        """Obtains the IDB path for a binary."""
        return os.path.splitext(file)[0] + '.idb'

    @classmethod
    def find_project_dir(cls, start_path):
        """
        Traverses up the directory tree, searching for a project root.
        If one is found, returns the path, else `None`.
        """
        tail = object()
        head = start_path
        while tail:
            head, tail = os.path.split(head)
            cur_meta_path = os.path.join(head, tail, cls.META_DIR_NAME)
            if os.path.exists(cur_meta_path):
                return os.path.join(head, tail)

    @staticmethod
    def find_project_files(root, file_patterns):
        """Locates all binaries that are part of a project."""
        file_patterns = [x.strip() for x in file_patterns.split(';')]
        for dirpath, _, filenames in os.walk(root):
            relevant_files = itertools.chain.from_iterable(
                fnmatch.filter(filenames, x) for x in file_patterns
            )

            # Py2 Y U NO SUPPORT "yield from"? :(
            for cur_file in relevant_files:
                yield os.path.join(dirpath, cur_file)

    @classmethod
    def create(cls, root, file_patterns):
        """Creates a new project."""
        # Create meta directory.
        cont_dir = os.path.join(root, cls.META_DIR_NAME)
        if os.path.exists(cont_dir):
            raise Exception("Directory is already a continuum project")
        os.mkdir(cont_dir)

        # Create config file.
        config = ConfigParser.SafeConfigParser()
        config.add_section('project')
        config.set('project', 'file_patterns', file_patterns)
        with open(os.path.join(cont_dir, cls.CFG_FILE_NAME), 'w') as f:
            config.write(f)

        # Create initial index.
        project = Project()
        project.open(root)

        return project


import random
sonuc = random.randint(1,100)

def tahmin():
    a = float(input("düş biraz: "))
    return a

def tahmin2():
    c = float(input("Çık: "))
    return c

a = float(input("1-100 arasında rastgele bir sayı giriniz:"))

if a >= 100:
    print("100'den büyük bir sayı girdiniz.")
elif a <= 0:
    print("1'den küçük bir sayı girdiniz.")
else:
    while True:
        if a == sonuc:
            print("Tam isabet")
            break
        elif a<sonuc:
            c = tahmin2()
            if c < a:
                print("çık !!!")
                break
            else:
                a = c
        else:
            b = tahmin()
            if b > a :
                print("düş !!!")
                break
            else:
                a = b

Temel Bölümler - OOP/Sayı Tahmin Oyunu.py


from django.views.generic import ListView, CreateView, UpdateView, DeleteView
from django.urls import reverse_lazy
from django.contrib.auth.mixins import LoginRequiredMixin
from helpers.mixins import PermissionMixin
from setting.models import OrganizationalStructure, ITEM_CHOICES
from django.shortcuts import redirect
from helpers.functions import get_organizational_structure

class OrganizationalStructureList(LoginRequiredMixin, PermissionMixin, ListView):
    permission_required = [
        'view_organizationalstructure',
        'add_organizationalstructure',
        'update_organizationalstructure',
        'delete_organizationalstructure'
    ]
    template_name = 'setting/organizational-structure/list.html'
    model = OrganizationalStructure

    def get_unused_options(self, existing_qs):
        items = ITEM_CHOICES
        return_items = list(ITEM_CHOICES)
        for item in items:
            counter = 0
            for existing_item in existing_qs:
                if item[0] == existing_item.item:
                    break
                counter += 1
            if counter != len(existing_qs):
                return_items.remove(item)
        if ('company', 'Company') in return_items:
            return_items.remove(('company', 'Company'))
        return return_items

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        org_structure_qs = OrganizationalStructure.objects.all().order_by('order')
        context['orgs'] = org_structure_qs
        context['unused_items'] = self.get_unused_options(existing_qs=org_structure_qs)
        return context

    def post(self, request, *args, **kwargs):
        data = request.POST['org-structure']
        if data == '':
            data = 'company'
        data_list = data.split(",")

        if '' in data_list:
            data_list.remove('')

        existing_structure_qs = OrganizationalStructure.objects.all()
        to_delete_items_list = []
        for obj_item in existing_structure_qs:
            if obj_item.item not in data_list:
                to_delete_items_list.append(obj_item.item)
        print(to_delete_items_list)
        for item in to_delete_items_list:
            try:
                OrganizationalStructure.objects.get(item=item).delete()
            except:
                pass

        i = 0
        parent = None
        while i < len(data_list):
            if i == 0:
                try:
                    parent, created = OrganizationalStructure.objects.update_or_create(
                        item=data_list[i], defaults={'parent_item': None, 'order': i+1}
                    )
                except:
                    break
            else:
                try:
                    parent, created = OrganizationalStructure.objects.update_or_create(
                        item=data_list[i], defaults={'parent_item': parent, 'order': i+1}
                    )
                except:
                    break
            i += 1

        return redirect(request.META['HTTP_REFERER'])

class OrganizationalStructureCreate(LoginRequiredMixin, PermissionMixin, CreateView):
    permission_required = 'add_organizationalstructure'
    template_name = 'setting/organizational-structure/create.html'
    model = OrganizationalStructure
    fields = '__all__'
    success_url = reverse_lazy('beehive_admin:setting:organizational_structure_list')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        return context

class OrganizationalStructureUpdate(LoginRequiredMixin, PermissionMixin, UpdateView):
    permission_required = 'update_organizationalstructure'
    template_name = 'setting/organizational-structure/update.html'
    model = OrganizationalStructure
    fields = '__all__'
    context_object_name = 'orgs'
    success_url = reverse_lazy('beehive_admin:setting:organizational_structure_list')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        return context

class OrganizationalStructureDelete(LoginRequiredMixin, PermissionMixin, DeleteView):
    permission_required = 'update_organizationalstructure'
    template_name = 'setting/organizational-structure/delete.html'
    model = OrganizationalStructure
    fields = '__all__'
    success_url = reverse_lazy('beehive_admin:setting:organizational_structure_list')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        return context


# coding: utf-8
# Author: gjwei
"""Find the left most node"""
def leftmost(root):
    """层序遍历，找到每一层的left node"""
    if root is None:
        return None
    queue = [root]
    result = []
    while queue:
        size = len(queue)
        for i in range(size):
            p = queue.pop(0)
            if i == 0:
                result.append(p.val)
            if p.left:
                queue.append(p.left)
            if p.right:
                queue.append(p.right)
    return result


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 28 14:47:06 2021

@author: xuziang
"""
import computation as C
from simulation_2 import simulation
import opyplus as op
import numpy as np
import pandas as pd
from numpy import genfromtxt
import matplotlib.pyplot as plt
import threading
import time

#%%
# =============================================================================
# Train
# =============================================================================
# errors = [error]

def optimize(S, dirs, indexs, actual, pre_error, worked_theta, gradients, season, dt, parametersRange, nostepsize):
    worked_errors = []
    iters = 1
    max_iter = 500
    # max_iter = 1
    error = pre_error
    while iters <= max_iter:
        threads = {}
        for theta in worked_theta:
            threads[theta] = C.training_thread(theta, dirs['S'][theta], indexs[season], error, gradients, dt, actual, parametersRange)
        for theta in threads:
            threads[theta].start()
        for theta in threads:
            threads[theta].join()

        """backtracking line search version3"""
        c = 0.5
        gradientnorm = 0
        for theta in gradients:
            gradientnorm += gradients[theta]**2
        beta = 0.5
        steps ={}
        threads = {}
        while True:
            for t in range(14): # the indexs for threads are from 1 to 14
                threads[t] = C.searching_thread(t, dirs['S'][t+1], indexs[season], beta, steps, actual, worked_theta, gradients)
            for t in range(14):
                threads[t].start()
            for t in range(14):
                threads[t].join()

            """line search stop criteria"""
            alphafound = False
            for t in range(14):
                alpha = beta*0.5**t
                if steps[t] <= error - c*alpha*gradientnorm:
                    alphafound = True
                    break
            if alphafound:
                break
            elif alpha <= 1e-30:
                nostepsize.append(None)
                break
            else:
                beta = alpha

        """update mainthread and synchronize all parameters"""
        print("alpha:", alpha)
        for theta in worked_theta:
            S.par_update(theta, gradients[theta], alpha)

        pars = S.par_print()
        for i in range(1, 15):
            dirs['S'][i].par_assign(pars)

        _, yc = S.simulate()
        error = S.MSE(actual[indexs[season]], yc[indexs[season]])
        worked_errors.append(error)

        """record errors for proof of no competing"""
        all_errors_g.append(S.MSE(actual[indexs['global']], yc[indexs['global']]))
        all_errors_s.append(S.MSE(actual[indexs['summer']], yc[indexs['summer']]))
        all_errors_w.append(S.MSE(actual[indexs['winter']], yc[indexs['winter']]))

        """stopping criteria"""
        if iters >= 2 and abs(worked_errors[-1] - worked_errors[-2])/worked_errors[-1] < 1e-4:
            break

        print(season + " error:", error)
        iters += 1
    return worked_errors

@C.my_timer
def BGD(S, dirs, actual, indexs, theta_g, theta_s, theta_w, dt, parametersRange, nostepsize):
    global all_errors_g
    global all_errors_s
    global all_errors_w
    global inner_iters
    all_errors_g = []
    all_errors_s = []
    all_errors_w = []
    pre_error_g = 0
    pre_error_s = 0
    pre_error_w = 0
    max_epoch = 100
    # max_epoch = 1
    global epoch
    global steps
    global delta_theta
    global error
    global gradients

    epoch = 0
    gradients = {}
    inner_iters = [0]

    _, yc = S.simulate()
    all_errors_g.append(S.MSE(actual[indexs['global']], yc[indexs['global']]))
    all_errors_s.append(S.MSE(actual[indexs['summer']], yc[indexs['summer']]))
    all_errors_w.append(S.MSE(actual[indexs['winter']], yc[indexs['winter']]))

    while epoch <= max_epoch:
        cur_error_g = all_errors_g[-1]
        cur_error_s = all_errors_s[-1]
        cur_error_w = all_errors_w[-1]

        gradients[epoch] = {}
        print("epoch:", epoch, " error_g:", cur_error_g)
        print("epoch:", epoch, " error_s:", cur_error_s)
        print("epoch:", epoch, " error_w:", cur_error_w)

        """stopping criteria"""
        if epoch >= 1 and abs(cur_error_g - pre_error_g)/cur_error_g < 1e-4 or abs(cur_error_s - pre_error_s)/cur_error_s < 1e-4 or abs(cur_error_w - pre_error_w)/cur_error_w < 1e-4:
            break

        pre_error_g = cur_error_g
        pre_error_s = cur_error_s
        pre_error_w = cur_error_w

        errors_g = optimize(S, dirs, indexs, actual, cur_error_g, theta_g, gradients[epoch], 'global', dt, parametersRange, nostepsize)
        inner_iters.append(inner_iters[-1] + len(errors_g))

        errors_s = optimize(S, dirs, indexs, actual, cur_error_s, theta_s, gradients[epoch], 'summer', dt, parametersRange, nostepsize)
        inner_iters.append(inner_iters[-1] + len(errors_s))

        errors_w = optimize(S, dirs, indexs, actual, cur_error_w, theta_w, gradients[epoch], 'winter', dt, parametersRange, nostepsize)
        inner_iters.append(inner_iters[-1] + len(errors_w))

        epoch += 1

    return inner_iters, all_errors_g, all_errors_s, all_errors_w

#%%
# =============================================================================
# Prepare paths and define subthreads
# =============================================================================
if __name__ == "__main__":
    all_dir = "C:/Users/xuziang/Downloads/Building Energy Consumption Analysis"
    idfname = '/545_update_part_sim.idf'
    epwfile = "/Austin_weather.epw"
    eplus = '/EnergyPlusV9-3-0'
    epm = op.Epm().load(all_dir + '/EplusModels' + idfname)

    S = simulation(epm, all_dir + '/EplusModels' + epwfile, all_dir + eplus)
    dirs = C.pre_dirs(all_dir, idfname, epwfile, eplus)
    useful_thetas = [1,3,4,5,6,7,9,10,11,13] # no parameters 2, 8, 12, 14
    all_thetas = [i for i in range(1,15)]
    actual = genfromtxt(all_dir + "/Optimizing Algs/Data/ActualEnergyConsumption.csv", delimiter=',')

    dt = 1e-4
    theta_s = [1,5,6,9]
    theta_w = [7,8,10]
    theta_g = [2,3,4,11,12,13,14]
    records_testerror = []
    nostepsize = []
    parametersRange = {3:400, 4:100, 5:3, 6:48000, 7:20000, 8:0.499, 11:40}
    for i in [1,2,9,10,12,13,14]:
        parametersRange[i] = 1
    # # parametersRange = {i:1 for i in range(1, 15)}
    train, test = C.split_dataset_2(1, 0.7)
    C.initialize(all_dir, S, dirs, 'init1', changeschedule=False)
    inner_iters, all_errors_g_dt1, all_errors_s_dt1, all_errors_w_dt1 = BGD(S, dirs, actual, train, theta_g, theta_s, theta_w, dt, parametersRange, nostepsize)
    # inner_iters, all_errors_g_dt1, all_errors_s_dt1 = BGD(actual, train)
    _, yc = S.simulate()
    test_error_g = S.MSE(actual[test["global"]], yc[test["global"]])
    test_error_s = S.MSE(actual[test["summer"]], yc[test["summer"]])
    test_error_w = S.MSE(actual[test["winter"]], yc[test["winter"]])
# records_testerror = []
# records_allerrorg = {}
# records_allerrors = {}
# records_allerrorw = {}
# records_inneriters = {}

# for i in [1,3,7]:
#     train, test = C.split_dataset(i, 0.7, season=True)
#     C.initialize(all_dir, S, dirs, 'init1', changeschedule=False)
#     inner_iters, all_errors_w, all_errors_s, all_errors_g = BGD(actual, train)
#     _, yc = S.simulate()
#     yc = np.array(yc)
#     train_error = S.MSE(actual[train['global']], yc[train['global']])
#     test_error = S.MSE(actual[test['global']], yc[test['global']])
#     test_error_summer = S.MSE(actual[test['summer']], yc[test['summer']])
#     test_error_winter = S.MSE(actual[test['winter']], yc[test['winter']])

#     print("train:", train_error)
#     print("test:", test_error)
#     record = [test_error, test_error_summer, test_error_winter]
#     records_testerror.append(record)

#     """used when we plot no competing proof for all ten seeds"""
#     records_allerrorg[i] = all_errors_g
#     records_allerrors[i] = all_errors_s
#     records_allerrorw[i] = all_errors_w
#     records_inneriters[i] = inner_iters

# records_testerror = np.array(records_testerror)
# print(np.mean(records_testerror, 0))
# print(np.std(records_testerror, 0))

#%%
# =============================================================================
# Plot erros for all seeds
# =============================================================================
# longest_record = 0
# for i in records_allerrorg:
#     record = records_allerrorg[i]
#     record = np.array(record)
#     inner_iters = records_inneriters[i]
#     plt.plot(record, label = "seed_" + str(i))
#     if len(record) > longest_record:
#         longest_record = len(record)
#     points = np.array(inner_iters)
#     plt.plot(points, record[points], 'r|')
#     plt.xticks(range(longest_record))
# plt.plot(all_errors_g_dt1, label="Lg")
# points = np.array(inner_iters)
# all_errors_g = np.array(all_errors_g_dt1)
# plt.plot(points, all_errors_g[points], 'r|')
# plt.legend()
# plt.title("BGD, global, LS1, dt="+str(dt))
# plt.show()

# longest_record = 0
# for i in records_allerrors:
#     record = records_allerrors[i]
#     record = np.array(record)
#     inner_iters = records_inneriters[i]
#     plt.plot(record, label = "seed_" + str(i))
#     if len(record) > longest_record:
#         longest_record = len(record)
#     points = np.array(inner_iters)
#     plt.plot(points, record[points], 'r|')
#     plt.xticks(range(longest_record))
# plt.plot(all_errors_s_dt1, label="Ls")
# points = np.array(inner_iters)
# all_errors_s = np.array(all_errors_s_dt1)
# plt.plot(points, all_errors_s[points], 'r|')
# plt.legend()
# plt.title("BGD, summer, LS1, dt="+str(dt))
# plt.show()

# longest_record = 0
# for i in records_allerrorw:
#     record = records_allerrorw[i]
#     record = np.array(record)
#     inner_iters = records_inneriters[i]
#     plt.plot(record, label = "seed_" + str(i))
#     if len(record) > longest_record:
#         longest_record = len(record)
#     plt.xticks(range(longest_record))
#     points = np.array(inner_iters)
#     plt.plot(points, record[points], 'r|')
# plt.plot(all_errors_w, label="Lw")
# points = np.array(inner_iters)
# all_errors_w = np.array(all_errors_w)
# plt.plot(points, all_errors_w[points], 'r|')
# plt.legend()
# plt.title("winter_unisch, dt=1e-8")
# plt.title("BGD, winter, LS1, dt="+str(dt))
# plt.show()

# # plt.title("BGD_ADAM, train, init_1, error_g")
# # plt.legend()
# # plt.show()
#%%