"""Dynamics model environment base class.

Use this class if you want to build an environment
based on a model of class `BaseDynamicsModel`.
"""
import os
from abc import ABC, abstractmethod
from typing import Dict, Union, List, Tuple, Optional, Callable, Any

import gym
import numpy as np

from agents import base_agent
from agents.base_agent import RL_MODEL_DIR
from dynamics.base_model import BaseDynamicsModel
from util.numerics import npf32
from util.util import Arr, create_dir, make_param_ext, str_to_np_dt, Num, day_offset_ts, ts_per_day, ProgWrap, \
    load_pickle, save_pickle
from util.visualize import rl_plot_path, plot_env_evaluation, plot_reward_details, OVERLEAF_IMG_DIR, MergeListT, \
    eval_env_evaluation

Agents = Union[List[base_agent.AgentBase], base_agent.AgentBase]

DEFAULT_ENV_SAMPLE_DATA: str = "train"

RejSamplerFun = Callable[[np.ndarray], bool]

class RejSampler:
    """Rejection sampling class.

    Can be used for unbalanced environment resets by
    rejecting unwanted states. Will impact performance if
    a lot are rejected.
    """

    name: str = "Rej"
    max_rej: int = 1

    def __init__(self, name: str, fun: RejSamplerFun = lambda x: True,
                 max_rej: int = 1):
        self.name = f"{name}{max_rej}"
        self.fun = fun
        self.max_rej = max_rej

    def __call__(self, hist: np.ndarray):
        return self.fun(hist)

class DynEnv(ABC, gym.Env):
    """The environment wrapper class for `BaseDynamicsModel`.

    Takes an instance of `BaseDynamicsModel` and adds
    all the functionality needed to turn it into an environment
    to be used for reinforcement learning.
    """
    # Fix data
    m: BaseDynamicsModel  #: Prediction model.
    act_dim: int  #: The dimension of the action space.
    state_dim: int  #: The dimension of the state space.
    n_ts_per_eps: int  #: The maximum number of timesteps per episode.
    n_ts_per_day: int  #: Number of time steps in a day.
    t_init_n: int  #: The index of the initial timestep in the dataset.

    short_name: str = None

    # State data, might change if `step` is called.
    n_ts: int = 0  #: The current number of timesteps.
    hist: np.ndarray  #: 2D array with current state.

    # The current data to sample initial conditions from.
    train_data: np.ndarray  #: The training data.
    train_indices: np.ndarray  #: The indices corresponding to `train_data`.
    n_start_data: int  #: The number of possible initializations using the training data.

    # Info about the current episode.
    use_noise: bool = False  #: Whether to add noise when simulating.
    curr_ind: int = None
    curr_n: int = None

    orig_actions: np.ndarray  #: Array with fallback actions

    # The one day data
    n_train_days: int  #: Number of training days
    train_days: List  #: The data of all days, where all data is available.
    day_inds: np.ndarray  #: Index vector storing the timestep offsets to the days
    day_ind: int = 0  #: The index of the day in `train_days`.

    # Scaling info
    info: Dict = None  #: A dict with info about the current agent.
    do_scaling: bool = False
    a_scaling_pars: Tuple[np.ndarray, np.ndarray] = None  #: Extra scaling parameters.

    # Plotting default values
    default_state_mask: np.ndarray = None
    default_series_merging: MergeListT = None

    _dummy_use: bool
    _plot_path: str
    _disturbance_is_modelled: bool = False

    def __init__(self, m: BaseDynamicsModel, name: str = None, max_eps: int = None,
                 disturb_fac: float = 1.0,
                 init_res: bool = True,
                 dummy_use: bool = False,
                 sample_from: str = DEFAULT_ENV_SAMPLE_DATA,
                 rejection_sampler: RejSampler = None,
                 verbose: int = 0):
        """Initialize the environment.

        Args:
            m: Full model predicting all the non-control features.
            max_eps: Number of continuous predictions in an episode.
        """
        self._dummy_use = dummy_use
        self.verbose = verbose
        if not self._dummy_use:
            # m.model_disturbance()
            pass
        self.m = m
        self.rej_sampler = rejection_sampler

        # Define name extensions
        sam_ext = f"_SAM_{sample_from}" if sample_from != DEFAULT_ENV_SAMPLE_DATA else ""
        h_sam_ext = f"_RejS_{rejection_sampler.name}" if rejection_sampler is not None else ""

        if name is not None:
            dist_ex = make_param_ext([("DF", disturb_fac)])
            self.name = f"{name}{dist_ex}{sam_ext}{h_sam_ext}_DATA_{m.name}"
        else:
            self.name = f"RLEnv_{m.name}"
        # self.plot_path = os.path.join(rl_plot_path, self.name)
        self._plot_path = os.path.join(rl_plot_path, self.name)

        # Set attributes.
        dat = m.data
        self.disturb_fac = disturb_fac
        self.act_dim = dat.n_c
        self.state_dim = dat.d

        # Time indices
        self.n_ts_per_eps = 100 if max_eps is None else max_eps
        self.t_init_n = day_offset_ts(dat.t_init, dat.dt,
                                      remaining=False)
        self.n_ts_per_day = ts_per_day(dat.dt)

        # Set data and initialize env.
        self._set_data(sample_from)
        if init_res:
            self.reset(use_noise=False)

    def __str__(self):
        """Generic string conversion."""
        n = self.short_name if self.short_name is not None else self.name
        return f"RL env of class {self.__class__.__name__} with name {n}"

    @property
    def plot_path(self) -> str:
        """Returns the path of the plot folder."""
        if self.short_name is not None:
            return os.path.join(rl_plot_path, self.short_name)
        return self._plot_path

    @property
    def model_path(self) -> str:
        """Returns the path of the model folder."""
        if self.short_name is not None:
            return os.path.join(RL_MODEL_DIR, self.short_name)
        else:
            raise ValueError("No short name specified!!!")

    def set_agent(self, a: base_agent.AgentBase) -> None:
        """Sets the given agent to the environment.

        Handles the extra scaling if it is required by the
        agent, the environment does not actually get access to the
        agent.

        Args:
            a: The agent to be used with the env.
        """
        # Get info about agent and check whether scaling is needed
        self.info = a.get_info()
        scaling = self.info.get('action_scaled_01')
        self.do_scaling = scaling is not None

        # Handle scaling
        if self.do_scaling:
            p1 = np.array([i[0] for i in scaling], dtype=np.float32)
            p2 = np.array([i[1] - i[0] for i in scaling], dtype=np.float32)
            self.a_scaling_pars = p1, p2

    def _set_data(self, part_str: str = "train") -> None:
        """Sets the data to use in the env.

        Args:
            part_str: The string specifying the part of the data.
        """
        self.m.data.check_part(part_str)
        self.train_data, _, self.train_indices = self.m.data.get_split(part_str)
        self.n_start_data = len(self.train_data)

    def get_plt_path(self, name: str) -> str:
        """Specifies the path of the plot with name 'name' where it should be saved.

        If there is not a directory
        for the current model, it is created.

        Args:
            name: Name of the plot.

        Returns:
            Full path of the plot file.
        """
        dir_name = self.plot_path
        create_dir(dir_name)
        return os.path.join(dir_name, name)

    def get_model_path(self, mod_name: str) -> str:
        """Returns the model path."""
        dir_name = self.model_path
        create_dir(dir_name)
        return os.path.join(dir_name, mod_name)

    reward_descs: List = []  #: The description of the detailed reward.

    @abstractmethod
    def detailed_reward(self, curr_pred: np.ndarray, action: Arr) -> np.ndarray:
        """Computes the different components of the reward and returns them all.

        Args:
            curr_pred: The current predictions.
            action: The most recent action taken.

        Returns:
            The reward components in an 1d array.
        """
        pass

    def compute_reward(self, curr_pred: np.ndarray, action: Arr) -> float:
        """Computes the reward to be maximized during the RL training.

        Args:
            curr_pred: The current predictions.
            action: The most recent action taken.

        Returns:
            Value of the reward of that outcome.
        """
        # The base implementation just sums up the different rewards.
        return np.sum(self.detailed_reward(curr_pred, action)).item()

    @abstractmethod
    def episode_over(self, curr_pred: np.ndarray) -> bool:
        """Defines criterion for episode to be over.

        Args:
            curr_pred: The next predicted state.

        Returns:
            True if episode is over, else false
        """
        return False

    def scale_action_for_step(self, action: Arr) -> Arr:
        """Scales the action to be used by the underlying ML model.

        This does nothing, override if needed.

        Args:
            action: The action to be scaled.
        Returns:
            The scaled action, same shape as the input.
        """
        return action

    @property
    def dummy_use(self):
        return self._dummy_use

    def step(self, action: Arr) -> Tuple[np.ndarray, float, bool, Dict]:
        """Evolve the model with the given control input `action`.

        This function should not be overridden, instead override
        `scale_action_for_step`! If it is, it should be called at some
        point in the overriding method.

        Args:
            action: The control input (action), will be scaled for the underlying model
                of the environment.

        Returns:
            The next state, the reward of having chosen that action and a bool
            determining if the episode is over. (And an empty dict)
        """
        if self._dummy_use:
            raise ValueError("Stepping not supported for dummy usage!")

        # Scale the action
        action = self.scale_action_for_step(action)

        # Add the control to the history
        self.hist[-1, -self.act_dim:] = action

        # Predict using the model
        hist_res = np.copy(self.hist).reshape((1, -1, self.state_dim))
        curr_pred = self.m.predict(hist_res)[0]

        # Add noise
        if self.use_noise:
            assert self._disturbance_is_modelled, "Need to model disturbance first!"
            curr_pred += self.disturb_fac * self.m.disturb()

        # Save the chosen action
        self.orig_actions[self.n_ts, :] = action

        # Update history
        self.hist[:-1, :] = self.hist[1:, :]
        self.hist[-1, :-self.act_dim] = curr_pred
        self.n_ts += 1

        # Compute reward and episode termination
        r = np.array(self.compute_reward(curr_pred, action)).item()
        ep_over = self.n_ts == self.n_ts_per_eps or self.episode_over(curr_pred)
        return curr_pred, r, ep_over, {}

    @property
    def curr_state(self) -> np.ndarray:
        """Returns the current state.

        Returns:
            Current state of environment, 1d array.
        """
        return self.hist[-1, :-self.act_dim]

    def scale_state(self, state: np.ndarray,
                    remove_mean: bool = False) -> np.ndarray:
        """Scales the state using the Dataset."""
        return self.m.data.scale(state, remove_mean=remove_mean, state_only=True)

    def get_curr_day_n(self) -> int:
        """Returns the current timestep index."""
        return (self.curr_n + self.n_ts) % self.n_ts_per_day

    def _model_disturbance(self):
        """Models the disturbance of the underlying model."""
        with ProgWrap("Modeling disturbance...", self.verbose > 0):
            self.m.model_disturbance()
            self._disturbance_is_modelled = True

    def reset(self, start_ind: int = None, use_noise: bool = True) -> np.ndarray:
        """Resets the environment.

        Always needs to be called if the episode is over.
        Initializes the environment with a new initial state.
        Uses rejection sampling to favor some initial conditions over
        others, as defined by `self.rej_sampler`.

        Args:
            start_ind: The index specifying the initial condition.
            use_noise: Whether noise should be added when simulating.

        Returns:
            A new initial state.
        """
        # Reset time step and disturbance
        self.use_noise = use_noise
        self.n_ts = 0
        self.m.reset_disturbance()

        # Model disturbance if not yet done
        if use_noise and not self._disturbance_is_modelled:
            if not self._dummy_use:
                self._model_disturbance()

        # Reset original actions (necessary?)
        self.orig_actions = np.empty((self.n_ts_per_eps, self.act_dim), dtype=np.float32)

        accepted, ret_val = False, None
        ct = 0
        while not accepted:
            # Select new start data
            if start_ind is None or self.rej_sampler is not None:
                start_ind = np.random.randint(self.n_start_data)
            else:
                if self.n_start_data <= start_ind:
                    raise ValueError("start_ind is too fucking large!")

            self.hist = np.copy(self.train_data[start_ind])
            self.curr_ind = self.train_indices[start_ind] + self.m.data.seq_len - 1
            self.curr_n = (self.t_init_n + self.curr_ind) % self.n_ts_per_day
            ret_val = np.copy(self.hist[-1, :-self.act_dim])

            if self.rej_sampler is not None:
                accepted = self.rej_sampler(ret_val)
                if ct >= self.rej_sampler.max_rej:
                    accepted = True
            else:
                accepted = True

            if not accepted:
                ct += 1

        # print(f"Chose start ind: {start_ind}, rejected: {ct}")
        return ret_val

    def get_scaled_init_state(self, init_ind: int, heat_inds) -> np.ndarray:
        """Returns the initial states when reset with `init_ind`.

        Args:
            init_ind: The reset index.
            heat_inds: The indices of the series to return.

        Returns:
            The scaled initial values of the series.
        """
        # Find the current heating water temperatures
        self.reset(init_ind, use_noise=False)
        return self.scale_state(self.curr_state, remove_mean=False)[heat_inds]

    def render(self, mode='human'):
        """Render method for compatibility with OpenAI gym."""
        print("Rendering not implemented!")

    def _to_scaled(self, action: Arr, to_original: bool = False,
                   extra_scaling: bool = False) -> np.ndarray:
        """Converts actions to the right range.

        Needs to be overridden by subclasses.
        """
        raise NotImplementedError("Implement this shit!")

    def analyze_agents_visually(self, agents: Agents,
                                fitted: bool = True,
                                use_noise: bool = False,
                                start_ind: int = None,
                                max_steps: int = None,
                                state_mask: np.ndarray = None,
                                plot_constrain_actions: bool = True,
                                show_rewards: bool = True,
                                title_ext: str = "",
                                put_on_ol: bool = False,
                                plot_rewards: bool = False,
                                bounds: List[Tuple[int, Tuple[Num, Num]]] = None,
                                series_merging_list: MergeListT = None,
                                overwrite: bool = False,
                                plot_all_rewards: bool = True) -> None:
        """Analyzes and compares a set of agents / control strategies.

        Uses the same initial condition of the environment and evaluates
        all the given agents in `agents` starting from these initial conditions.
        Then makes a plot comparing how the environment behaves under all the
        agents.

        Args:
            agents: A list of agents or a single agent.
            fitted: Whether the agents are already fitted.
            use_noise: Whether to use noise in the predictions.
            start_ind: Index of initial configuration, random if None.
            max_steps: The maximum number of steps of an episode.
            state_mask: Mask defining which series to plot.
            plot_constrain_actions: Whether to plot the actions constrained by the env.
            show_rewards: Use default!
            title_ext: The title to put in the plot.
            put_on_ol: Whether to save in Overleaf plot dir.
            plot_rewards: Whether to plot the reward bar plot for this scenario.
            bounds: Optional state bounds to put in plot.
            series_merging_list: Defines series that will be merged. Should be
                independent of agent's actions.
            overwrite: Whether to overwrite existing plot files.
            plot_all_rewards:

        Raises:
            ValueError: If `start_ind` is too large or if an agent is not suited
                for this environment.
        """
        print("This is deprecated!")

        # Make function compatible for single agent input
        if not isinstance(agents, list):
            agents = [agents]

        if max_steps is None:
            max_steps = 100000

        if not show_rewards:
            raise NotImplementedError("Do not do this!")

        # Use (possibly overridden) defaults
        if state_mask is None:
            state_mask = self.default_state_mask
        if series_merging_list is None:
            series_merging_list = self.default_series_merging

        # Choose same random start for all agents
        if start_ind is None:
            start_ind = np.random.randint(self.n_start_data)
        elif start_ind >= self.n_start_data:
            raise ValueError(f"start_ind: {start_ind} cannot be larger than: {self.n_start_data}!")

        # Check if file already exists
        analysis_plot_path = self._construct_plot_name("AgentAnalysis", start_ind, agents, put_on_ol)
        if os.path.isfile(analysis_plot_path + ".pdf") and not overwrite:
            return

        # Define arrays to save trajectories
        n_non_c_states = self.state_dim - self.act_dim
        n_agents = len(agents)
        action_sequences = npf32((n_agents, self.n_ts_per_eps, self.act_dim), fill=np.nan)
        clipped_action_sequences = npf32((n_agents, self.n_ts_per_eps, self.act_dim), fill=np.nan)
        trajectories = npf32((n_agents, self.n_ts_per_eps, n_non_c_states), fill=np.nan)
        rewards = npf32((n_agents, self.n_ts_per_eps), fill=np.nan)
        n_tot_rewards = len(self.reward_descs) + 1
        all_rewards = npf32((n_agents, self.n_ts_per_eps, n_tot_rewards), fill=np.nan)

        for a_id, a in enumerate(agents):
            # Check that agent references this environment
            if not a.env == self:
                raise ValueError(f"Agent {a_id} was not assigned to this env!")

            # Set agent, required for the ones that need special scaling
            self.set_agent(a)

            # Fit agent if not already fitted
            if not fitted:
                a.fit()

            # Reset env
            curr_state = self.reset(start_ind=start_ind, use_noise=use_noise)
            episode_over = False
            count = 0

            # Evaluate agent and save states, actions and reward
            while not episode_over and count < max_steps:
                curr_action = a.get_action(curr_state)
                scaled_a = self.scale_action_for_step(curr_action)

                curr_state, rew, episode_over, extra = self.step(curr_action)
                det_rew = self.detailed_reward(curr_state, scaled_a)
                action_sequences[a_id, count, :] = curr_action

                # Save trajectories and reward
                trajectories[a_id, count, :] = np.copy(curr_state)
                rewards[a_id, count] = rew
                all_rewards[a_id, count, 0] = rew
                all_rewards[a_id, count, 1:] = det_rew
                count += 1

            # Get original actions
            clipped_action_sequences[a_id, :self.n_ts, :] = self.orig_actions[:self.n_ts, :]

            # Scale actions
            if self.do_scaling:
                action_sequences[a_id] = self.a_scaling_pars[0] + action_sequences[a_id] * self.a_scaling_pars[1]

        # Scale the data to the right values
        trajectories = self.m.rescale_output(trajectories, out_put=True)
        s_ac = clipped_action_sequences.shape
        for k in range(s_ac[0]):
            for i in range(s_ac[1]):
                clipped_action_sequences[k, i] = self._to_scaled(clipped_action_sequences[k, i],
                                                                 to_original=True)
        if not plot_constrain_actions or np.allclose(clipped_action_sequences, action_sequences):
            clipped_action_sequences = None

        # Time stuff
        shifted_t_init = self.m.data.get_shifted_t_init(self.curr_ind)
        np_st_init = str_to_np_dt(shifted_t_init)

        # Plot all the things
        name_list = [a.get_short_name() for a in agents]
        add_pth = self._construct_plot_name("AgentAnalysisReward", start_ind, agents, put_on_ol)
        add_pth = add_pth if plot_rewards else None
        rewards_for_eval = all_rewards if plot_all_rewards else rewards
        plot_env_evaluation(action_sequences, trajectories, rewards_for_eval, self.m.data,
                            name_list, analysis_plot_path, clipped_action_sequences,
                            state_mask, show_rewards=show_rewards, title_ext=title_ext,
                            np_dt_init=np_st_init, rew_save_path=add_pth, bounds=bounds,
                            series_merging_list=series_merging_list,
                            reward_descs=self.reward_descs)

        # Plot the rewards for this episode
        add_pth = self._construct_plot_name("AgentAnalysisRewardDetailed", start_ind, agents, put_on_ol)
        names = [a.get_short_name() for a in agents]
        plot_reward_details(names, all_rewards, add_pth,
                            self.reward_descs, dt=self.m.data.dt, n_eval_steps=self.n_ts_per_eps,
                            title_ext=title_ext)

    def _construct_plot_name(self, base_name: str, start_ind: Optional[int], agent_list: List,
                             put_on_ol: bool = False):
        name_list = [a.get_short_name() for a in agent_list]
        agent_names = '_'.join(name_list)
        s_ext = "" if start_ind is None else f"_{start_ind}"
        base = f"{base_name}{s_ext}_{agent_names}"
        orig_name = self.get_plt_path(base)
        if put_on_ol:
            print(f"Original: {orig_name}")
            return os.path.join(OVERLEAF_IMG_DIR, base)
        return orig_name

    def _get_detail_eval_title_ext(self):
        # For the water temperature info in the title, see overriding method.
        # This is so fucking ugly!
        return None

    def detailed_eval_agents(self, agent_list: Agents,
                             n_steps: int = 100,
                             use_noise: bool = False,
                             put_on_ol: bool = False,
                             overwrite: bool = False,
                             verbose: int = 0,
                             plt_fun: Callable = plot_reward_details,
                             episode_marker: Callable = None,
                             visual_eval: bool = False,
                             bounds: List[Tuple[int, Tuple[Num, Num]]] = None,
                             agent_filter_ind: int = None,
                             filter_good_cases: bool = True,
                             plot_tot_reward_cases: bool = True,
                             plot_tot_eval: bool = True,
                             plot_constrained_actions: bool = False,
                             max_visual_evals: int = 4,
                             heating_title_ext: bool = False,
                             indicate_bad_case: bool = False,
                             disconnect_data: Any = None,
                             eval_quality: bool = False
                             ) -> Optional[np.ndarray]:
        """Evaluates the given agents for this environment.

        Let's the agents act in the environment and observes the
        rewards. These are then averaged, plotted and returned.
        If `overwrite` is False and the plot already exists, nothing
        is returned.

        Args:
            agent_list: List of agents that are based on this env.
            n_steps: Number of evaluation steps.
            use_noise: Whether to use noise in the evaluation.
            put_on_ol: Whether to put the resulting plot into the Overleaf plot folder.
            overwrite: Whether to overwrite pre-existing plot files.
            verbose: Verbosity.
            plt_fun: The plotting function.
            episode_marker: Marker for the case if `plt_fun`
                is :func:`util.visualize.plot_heat_cool_rew_det`
            visual_eval: Whether to do visual analysis with the evaluated data.
            bounds: The temperature bounds for the visual analysis plot.
                Ignored if `visual_eval` is False.
            agent_filter_ind: Index of agent whose rewards need to be
                among the best to be plotted. Ignored if `visual_eval` is False.
            filter_good_cases: Whether to filter out the bad cases and only plot
                the good ones in the time series analysis.
            plot_tot_reward_cases: Whether to plot the rewards corresponding to
                 the time series cases.
            plot_tot_eval: Whether to plot the total rewards of the whole evaluation.
            plot_constrained_actions: Whether to plot the constrained actions in the time series
                plot.
            max_visual_evals: Max. number of visual evaluations.
            heating_title_ext:
            indicate_bad_case:
            disconnect_data:
            eval_quality:

        Returns:
            The rewards seen by all the agents, or None if `overwrite` is False
            and the plot already exists.
        """

        if agent_filter_ind is None:
            agent_filter_ind = 0

        # Make function compatible for single agent input.
        if not isinstance(agent_list, list):
            agent_list = [agent_list]

        # Handle overwriting
        p_name = self._construct_plot_name("DetailAnalysis", n_steps, agent_list, put_on_ol)
        if os.path.isfile(p_name + ".pdf") and not overwrite:
            if verbose > 0:
                print("Agent evaluation plot already exists!")
            return
        p_1 = self._construct_plot_name(f"AgentAnalysis_E0", None,
                                        agent_list, put_on_ol)
        p_2 = self._construct_plot_name(f"AgentAnalysis_E0_n", None,
                                        agent_list, put_on_ol)
        if (os.path.isfile(p_1 + ".pdf") or os.path.isfile(p_2 + ".pdf")) and not overwrite:
            if visual_eval and not plot_tot_eval:
                print("Agent time series plot already exists!")
                return

        pickle_data_path = self._construct_plot_name(f"pickled_data_{n_steps}", None,
                                                     agent_list, False) + ".pkl"

        # Init scores.
        n_agents = len(agent_list)
        n_extra_rewards = len(self.reward_descs)
        n_tot_rewards = n_extra_rewards + 1

        # Load data from pickle if exists
        if os.path.isfile(pickle_data_path):
            if verbose > 0:
                print("Loading from previous evaluation...")
            all_rewards, all_rewards, all_states, all_marks, \
                all_actions, all_scaled_actions, ret_inds = load_pickle(pickle_data_path)
        else:
            all_rewards = npf32((n_agents, n_steps, n_tot_rewards))
            all_actions = npf32((n_agents, n_steps, self.act_dim))
            all_scaled_actions = npf32((n_agents, n_steps, self.act_dim))
            all_states = npf32((n_agents, n_steps, self.state_dim - self.act_dim))
            all_marks = np.empty((n_agents, n_steps), dtype=np.int32)
            ret_inds = None

            do_scaling = self.m.data.fully_scaled

            if verbose > 0:
                print("Evaluating agents...")
            for a_id, a in enumerate(agent_list):

                # Set agent
                self.set_agent(a)

                # Fit agent if not already fitted
                if verbose:
                    print(f"Fitting agent: {a}")
                a.fit(verbose=verbose, train_data=a.fit_data)

                # Check that agent references this environment
                if not a.env == self:
                    raise ValueError(f"Agent {a_id} was not assigned to this env!")

                # Evaluate agent.
                if verbose:
                    print(f"Evaluating agent: {a}")
                eval_res = a.eval(n_steps, reset_seed=True, detailed=True,
                                  use_noise=use_noise, scale_states=do_scaling,
                                  episode_marker=episode_marker,
                                  return_inds=True,
                                  verbose=verbose)
                rew, ex_rew, states, ep_marks, acs, s_acs, ret_inds = eval_res

                all_rewards[a_id, :, 0] = rew
                all_rewards[a_id, :, 1:] = ex_rew
                all_states[a_id] = states
                all_marks[a_id] = ep_marks
                all_actions[a_id] = acs
                all_scaled_actions[a_id] = s_acs

                if verbose:
                    print(f"Saving evaluation data...")
                all_data = (all_rewards, all_rewards, all_states, all_marks,
                            all_actions, all_scaled_actions, ret_inds)
                save_pickle(pickle_data_path, all_data)

        # Plot total rewards for all steps
        title_ext = ""  # self._get_detail_eval_title_ext()
        if plot_tot_eval:
            if verbose > 0:
                print("Plotting evaluation...")
            names = [a.get_plot_name() for a in agent_list]
            plt_fun(names, all_rewards, p_name,
                    self.reward_descs, dt=self.m.data.dt, n_eval_steps=n_steps,
                    title_ext=title_ext, all_states=all_states,
                    verbose=0, ep_marks=all_marks,
                    add_base_title=False)

        if eval_quality:
            quality_plot_path = self._construct_plot_name(f"Env_Quality_{n_steps}", None,
                                                          agent_list, put_on_ol)
            eval_env_evaluation(all_rewards, all_states,
                                all_marks, self.n_ts_per_eps,
                                plt_base_name=quality_plot_path)

        print(f"Mean rewards: {np.mean(all_rewards[:, :, 0], axis=1)}")

        # Plot time series
        if visual_eval:
            ct = 0
            n_vis_eval = n_steps // self.n_ts_per_eps
            for k in range(n_vis_eval):
                k0 = k * self.n_ts_per_eps
                k1 = (k + 1) * self.n_ts_per_eps

                # Extract stuff
                action_sequences = all_actions[:, k0:k1]
                clipped_action_sequences = all_scaled_actions[:, k0:k1]
                curr_states = all_states[:, k0:k1]
                curr_rew = all_rewards[:, k0:k1]
                ep_ind = ret_inds[k]

                mean_rew = np.mean(curr_rew[:, :, 0], axis=1)
                winner = np.argwhere(mean_rew == np.amax(mean_rew))

                ext_ = "" if [agent_filter_ind] not in winner else "_n"
                if filter_good_cases:
                    if [agent_filter_ind] not in winner or ct > max_visual_evals:
                        # if [agent_filter_ind] in winner:
                        #     print(f"Len: {len(winner)}")
                        # else:
                        #     print(f"Bad case!")
                        continue
                    else:
                        # print(f"Found time series!!")
                        ct += 1
                else:
                    ct += 1
                if ct > max_visual_evals:
                    continue

                # Time stuff
                shifted_t_init = self.m.data.get_shifted_t_init(ep_ind)
                np_st_init = str_to_np_dt(shifted_t_init)

                state_mask = self.default_state_mask
                name_list = [a.get_plot_name() for a in agent_list]
                add_pth = None
                series_merging_list = self.default_series_merging

                if heating_title_ext:
                    unscaled_state = curr_states[0, 0]
                    w_in = unscaled_state[2]
                    heating = w_in > unscaled_state[4]
                    ext = f"{'Heating' if heating else 'Cooling'}, inflow water temp.: {w_in:.1f} °C"
                    curr_title_ext = f"{title_ext} {ext}"
                    ext_ += "_h" if heating else "_c"
                    if indicate_bad_case:
                        assert n_agents == 4, "Fuck you!"
                        r_temp_closed = curr_states[1, -1, 4]
                        r_temp_open = curr_states[0, -1, 4]
                        h_case = r_temp_open > r_temp_closed
                        if h_case != heating:
                            print(f"Fuck! k = {k}, {heating}")
                            ext_ += "_b"
                        else:
                            print(f"Nice! k = {k}, {heating}")
                else:
                    curr_title_ext = title_ext

                analysis_plot_path = self._construct_plot_name(f"AgentAnalysis_E{k}{ext_}", None,
                                                               agent_list, put_on_ol)

                # Plot time series for this episode
                ex_actions = clipped_action_sequences if plot_constrained_actions else None
                plot_env_evaluation(action_sequences, curr_states, curr_rew, self.m.data,
                                    name_list, analysis_plot_path, extra_actions=ex_actions,
                                    series_mask=state_mask, show_rewards=True, title_ext=curr_title_ext,
                                    np_dt_init=np_st_init, rew_save_path=add_pth,
                                    bounds=bounds,
                                    series_merging_list=series_merging_list,
                                    reward_descs=self.reward_descs,
                                    disconnect_data=disconnect_data,
                                    tot_reward_only=True)

                # Plot the rewards for this episode
                if not plot_tot_reward_cases:
                    continue
                add_pth = self._construct_plot_name(f"AgentAnalysis_E{k}_RewardDetailed", None,
                                                    agent_list, put_on_ol)
                plot_reward_details(name_list, curr_rew, add_pth,
                                    self.reward_descs, dt=self.m.data.dt,
                                    n_eval_steps=self.n_ts_per_eps,
                                    title_ext=title_ext,
                                    add_base_title=False)

        return all_rewards


# Juan Ruiz Jiménez
# Ramón Arjona Quiñones
import numpy as np # Para los arrays
import pyaudio, wave # Para el audio
import kbhit # Para gestión de input
from scipy.io import wavfile # Para manejo de wavs
import time # Para esperar a la hebra
from Utilities import plotWave, Osc, ModulatorAmp, getFormat, getFormatFromString # Para pintar ondas y generar osciladores de varios tipos

CHUNK = 1024 # Tamaño de los bloques (en nº muestras)
numBloque = 0 # contador de bloques procesados (variable global)
SRATE = 44100 # frecuencia de muestreo para los osciladores
TYPE = np.float32 # generamos como floats en [0,1]

pianoKeys = {'z':0, 's':1, 'x':2, 'd':3, 'c':4, 'v':5, 'g':6, 'b':7, 'h':8, 'n':9, 'j':10, 'm':11, 'q':12, '2':13, 'w':14, '3':15, 'e':16, 'r':17, '5':18, 't':19, '6':20, 'y':21, '7':22, 'u':23}

def Ejercicios1y2(shape, frec, vol):
    '''
    Oscilador al que se le puede cambiar la frecuencia y volumen en ejecución
    '''
    print("Oscilador " + str(shape) + ": " + str(frec) + "Hz, Volumen: " + str(vol))

    # 1. Creamos instancias de PyAudio y de kbhit
    p = pyaudio.PyAudio()
    kb = kbhit.KBHit()

    # Miramos el formato de los samples (fmt)
    if TYPE == np.int16: fmt = 2
    elif TYPE == np.int32: fmt = 4
    elif TYPE == np.float32: fmt = 4
    elif TYPE == np.uint8: fmt = 1
    else: raise Exception('Not supported')

    # 3. Creamos el stream con los datos del oscilador
    osc = Osc(shape, frec, vol)
    stream = p.open(format=p.get_format_from_width(fmt), # formato de los samples
                    channels=1,                          # num canales
                    rate=osc.srate(),                          # frecuencia de muestreo
                    frames_per_buffer=osc.chunkSize(),             # tamanio buffer
                    output=True)                         # callback para leer los datos

    # Reproducimos el oscilador
    print("Press 'q' to quit")
    c= ' '
    volIncr = 0.05
    while c != 'q' and c != 'Q':
        samples = osc.next()
        stream.write(samples.astype(TYPE).tobytes())
        if kb.kbhit():
            vol = osc.getVol()
            frec = osc.getFrec()
            c = kb.getch()
            if(c == 'v'):
                osc.setVol(max(0,vol-volIncr))
            elif (c == 'V'):
                osc.setVol(min(1,vol+volIncr))
            if(c == 'f'):
                osc.setFrec(max(20,frec-10))
            elif (c == 'F'):
                osc.setFrec(min(20000,frec+10))
            print("Vol: ",vol)
            print("Frec: ",frec)

    # 5. Finalización
    # Del stream
    stream.stop_stream()
    stream.close()
    # De kbhit y PyAudio
    kb.set_normal_term()
    p.terminate()

def Ejercicio3(shape, frec, vol):
    '''
    Oscilador que se va a modular con un seno
    '''
    print("Oscilador " + str(shape) + ": " + str(frec) + "Hz, Volumen: " + str(vol))

    # 1. Creamos instancias de PyAudio y de kbhit
    p = pyaudio.PyAudio()
    kb = kbhit.KBHit()

    # Miramos el formato de los samples (fmt)
    fmt = getFormat(TYPE)

    # 2. Creamos el oscilador, el modulador de amplitud...
    osc = Osc(shape, frec, vol)
    modAmp = ModulatorAmp(1) #1Hz

    # ... y el stream con los datos del oscilador
    stream = p.open(format=p.get_format_from_width(fmt), # formato de los samples
                    channels=1,                          # num canales
                    rate=osc.srate(),                          # frecuencia de muestreo
                    frames_per_buffer=osc.chunkSize(),             # tamanio buffer
                    output=True)                         # callback para leer los datos

    # 3. Reproducimos el oscilador por chunks
    print("Press 'q' to quit")
    c= ' '
    volIncr = 0.1
    while c != 'q' and c != 'Q':
        samples = osc.next()
        modSamples = modAmp.next(samples)
        #plotWave(modSamples)
        stream.write(modSamples.astype(TYPE).tobytes())
        if kb.kbhit():
            vol = osc.getVol()
            frec = osc.getFrec()
            c = kb.getch()
            if(c == 'v'):
                osc.setVol(max(0,vol-0.05))
            elif (c == 'V'):
                osc.setVol(min(1,vol+0.05))
            if(c == 'f'):
                osc.setFrec(max(20,frec-10))
            elif (c == 'F'):
                osc.setFrec(min(20000,frec+10))
            print("Vol: ",vol)
            print("Frec: ",frec)

    # 4. Finalización
    # Del stream
    stream.stop_stream()
    stream.close()
    # De kbhit y PyAudio
    kb.set_normal_term()
    p.terminate()

def Ejercicio8():
    '''
    Piano
    '''
    # Nos definimos el callback aqui mismo para tener acceso a 'data' (y no hacerlo global)
    def callback(in_data, frame_count, time_info, status):
        '''
        Función que será llamada por la hebra de PyAudio
        para leer chinks
        -> frame_count: numero de frames a devolver
        -> frame_count = frames_per_buffer = CHUNK
        '''
        global numBloque # para incrementar aquí
        #print("Callback bloque ", numBloque, "fc ", frame_count)

        # Recogemos el bloque de array data
        bloque = data[ numBloque*CHUNK : numBloque*CHUNK+CHUNK ]
        numBloque += 1

        # Devolvemos el bloque
        return (bloque, pyaudio.paContinue)

    # 1. Creamos instancias de PyAudio y de kbhit
    p = pyaudio.PyAudio()
    kb = kbhit.KBHit()

    # 2. Abrimos el archivo de audio y recogemos la SRATE y el array de datos
    frate, data = wavfile.read("muestras/piano.wav")

    # Miramos el formato de los samples (fmt)
    fmt = getFormatFromString(data.dtype.name)

    # 3. Reproducimos el oscilador por chunks
    print("Press '0' to quit")
    c= ' '
    volIncr = 0.1
    stream = None
    while c != '0':
        # Reseteamos el nº bloque si ha terminado el stream
        if(stream != None and not stream.is_active()):
            global numBloque
            numBloque = 0
        # Pulsación de teclado
        if kb.kbhit():
            c = kb.getch()
            # Comprobamos que sea una tecla válida
            if(c in pianoKeys):
                # Se toca una nota que ya estaba sonando
                if(stream != None and stream.is_active()):
                    stream.stop_stream()
                    numBloque = 0
                # Creamos el stream con la nota dada
                noteRate = (int)(frate * (2 ** (pianoKeys[c] / 12.0)))
                stream = p.open(format=p.get_format_from_width(fmt), # formato de los samples
                        channels=len(data.shape),            # num canales (shape de data)
                        rate=noteRate,                           # frecuencia de muestreo
                        frames_per_buffer=CHUNK,             # tamanio buffer
                        output=True,                         # stream de salida
                        stream_callback=callback)            # callback para leer los datos
                stream.start_stream()

        else:
            time.sleep(0.1)

    # De kbhit y PyAudio
    kb.set_normal_term()
    p.terminate()

#Ejercicios1y2("triangle", 440, 0.7)
#Ejercicio3("triangle", 440, 0.7)
Ejercicio8()


import sys
from PyQt5.QtWidgets import QApplication, QWidget, QRadioButton, QPushButton, QButtonGroup, QGridLayout
from PyQt5.QtCore import Qt

class MyWindow(QWidget):
    def __init__(self):
        super().__init__()

    #     # Création des radio boutons
    #     self.radio1 = QRadioButton("Radio 1", self)
    #     self.radio2 = QRadioButton("Radio 2", self)
    #     self.radio3 = QRadioButton("Radio 3", self)

    #     # Création du bouton poussoir
    #     self.pushButton = QPushButton("Appuyer", self)

    #     # Ajout des radio boutons à un groupe
    #     self.group = QButtonGroup(self)
    #     self.group.addButton(self.radio1)
    #     self.group.addButton(self.radio2)
    #     self.group.addButton(self.radio3)

    #     # Connexion du signal toggled()
    #     self.radio1.toggled.connect(self.on_radio1_toggled)

    #     # Disposition des widgets
    #     layout = QGridLayout(self)
    #     layout.addWidget(self.radio1, 0, 0)
    #     layout.addWidget(self.radio2, 1, 0)
    #     layout.addWidget(self.radio3, 2, 0)

    #     # Masquage du bouton poussoir
    #     self.pushButton.hide()

    #     # Ajout du bouton poussoir à droite
    #     layout.addWidget(self.pushButton, 3, 0, alignment=Qt.AlignRight)

    #     # Définir la largeur de la première colonne à 1
    #     layout.setColumnStretch(0, 1)

    #     self.setLayout(layout)

    # def on_radio1_toggled(self):
    #     # Si le premier bouton est sélectionné, le bouton poussoir apparaît
    #     if self.radio1.isChecked():
    #         self.pushButton.show()
    #     # Sinon, le bouton poussoir disparaît
    #     else:
    #         self.pushButton.hide()

    # Création des boutons radio
        self.radio1 = QRadioButton("Substract by gradient", self)
        self.radio2 = QRadioButton("Simple Threshold", self)

        self.pushButton = QPushButton("Import image...", self)

        self.group = QButtonGroup(self)
        self.group.addButton(self.radio1)
        self.group.addButton(self.radio2)

if __name__ == "__main__":
    app = QApplication(sys.argv)
    window = MyWindow()
    window.show()
    sys.exit(app.exec_())


import ctypes
ctypes.windll.shcore.SetProcessDpiAwareness(1)   # 高分屏
import threading
import time

import psutil
from infi.systray import SysTrayIcon

import pynvml

pynvml.nvmlInit()
handle = pynvml.nvmlDeviceGetHandleByIndex(0)    # GPU id: 0

thread_flags = 1                                 # 多线程标记

monitor = 'cpu'
cpu_usage, mem_usage, gpu_usage = 0.1, 0.1, 0.1  # 初始化

icon_fav = 'favicon.ico'
icon_cpu = 'icons/cpu.ico'
icon_mem = 'icons/mem.ico'
icon_gpu = 'icons/gpu.ico'
icon_quit = 'icons/quit.ico'

icons_runcat = [f'icons/runcat/{i}.ico' for i in range(5)]
icons_mario = [f'icons/mario/{i}.ico' for i in range(3)]

icon_type = 'runcat'  # 设定默认图标，并加载
icon_list = icons_mario if icon_type == 'mario' else icons_runcat

hover_text = ''

# thread 1: get cpu usage
def thread_get_cpu_usage():
    global cpu_usage, mem_usage, gpu_usage

    while thread_flags:
        cpu_usage = psutil.cpu_percent(interval=1) / 100
        mem_usage = psutil.virtual_memory().percent / 100
        meminfo = pynvml.nvmlDeviceGetMemoryInfo(handle)
        gpu_usage = meminfo.used / meminfo.total
        # print(cpu_usage)
        time.sleep(0.5)

# thread 2: update icon
def thread_update_icon():
    while thread_flags:
        mon = cpu_usage
        if monitor == 'mem':
            mon = mem_usage
        elif monitor == 'gpu':
            mon = gpu_usage

        t = 0.2 - mon * 0.15
        # print(mon, t)
        for i in icon_list:
            tip = f'cpu: {cpu_usage:.2%} \nmem: {mem_usage:.2%} \ngpu: {gpu_usage:.2%}'
            systray.update(icon=i, hover_text=tip)
            # print(i, monitor_type, f'{mon}:.2%')
            time.sleep(t)

def changeIconType(new_icon_type):
    global icon_type, icon_list

    print(icon_type, new_icon_type)
    if new_icon_type != icon_type:
        icon_type = new_icon_type
        icon_list = icons_mario if icon_type == 'mario' else icons_runcat

# change monitor type
def changeMonitor(new_monitor):
    global monitor

    print(monitor, new_monitor)
    if monitor != new_monitor:
        monitor = new_monitor

def quit(systray):
    global thread_flags
    thread_flags = 0

    systray.shutdown()

menu_options = (('cpu', icon_cpu, lambda x: changeMonitor('cpu')),
                ('mem', icon_mem, lambda x: changeMonitor('mem')),
                ('gpu', icon_gpu, lambda x: changeMonitor('gpu')),
                ('runcat', icons_runcat[0], lambda x: changeIconType('runcat')),
                ('mario', icons_mario[0], lambda x: changeIconType('mario')))

systray = SysTrayIcon(icon_fav, hover_text, menu_options, on_quit=quit, default_menu_index=1)
systray.start()

threading.Timer(0.1, thread_get_cpu_usage, []).start()
threading.Timer(0.1, thread_update_icon, []).start()

## pyinstaller --hidden-import pkg_resources --hidden-import infi.systray -w -i favicon.ico runcat-pywin32.py --add-data "icons;icons"


import numpy as np
from scipy import signal
import matplotlib.pyplot as plt

A = [
 [0.5, 0.5, 0],
 [0.5, 0.25, 0.5],
 [0, 0.25, 0.5]
]

B = [[0], [0], [0]]
C = [0.8, 0.2, 0.2]
D = 0

# condizione iniziale (distribuzione di probabilità di un dato evento)
x0 = [1, 0, 0] # evento certo soleggiato

# modello a tempo discreto
previsioni = signal.dlti(A, B, C, D)

# tempi
t0 = 0
tf = 31
incr = 1 # anno
t = np.arange(t0, tf, incr, dtype=int)

# ingressi
u = np.zeros((len(t), 1)) # il sistema resta isolato anche con u != 0

# simulazione
_, y, x = signal.dlsim(previsioni, u, t, x0)

# graphs
## soleggiato
plt.subplot(3, 1, 1)
plt.stem(t, y[:,0])
plt.grid(True)
plt.title('Soleggiato')

## nuvoloso
plt.subplot(3, 1, 2)
plt.stem(t, x[:,1])
plt.grid(True)
plt.title('Nuvoloso')

## piovoso
plt.subplot(3, 1, 3)
plt.stem(t, x[:,2])
plt.grid(True)
plt.title('Piovoso')


#! /usr/bin/env python
# vim: expandtab ts=4 sw=4
### FOLOW_OPERATOR.PY ###

import rospy
import smach

from pal_smach_utils.utils.global_common import succeeded, preempted, aborted

from pal_smach_utils.navigation.listen_command_for_restaurant import ListenCommandForRestaurant
from pal_smach_utils.navigation.listen_command_for_follow_me import ListenCommandForFollowMe
from pal_smach_utils.navigation.listen_command_for_follow_me_again import ListenCommandForFollowMeAgain
from pal_smach_utils.navigation.listen_command_for_gpsr_follow import ListenCommandForGpsrFollow

class ListenCommands(smach.StateMachine):
    def __init__(self, state_machine_name="restaurant"):
        smach.StateMachine.__init__(self, [succeeded, preempted, aborted])
        with self:

            if (state_machine_name == "restaurant"):
                smach.StateMachine.add('LISTEN_COMMANDS_FOR_RESTAURANT',
                                       ListenCommandForRestaurant(),
                                       transitions={succeeded: succeeded})
            elif (state_machine_name == "follow_me"):
                smach.StateMachine.add('LISTEN_COMMANDS_FOR_FOLLOW_ME',
                                       ListenCommandForFollowMe(),
                                       transitions={succeeded: succeeded})
            elif (state_machine_name == "follow_me_again"):
                smach.StateMachine.add('LISTEN_COMMANDS_FOR_FOLLOW_ME_AGAIN',
                                       ListenCommandForFollowMeAgain(),
                                       transitions={succeeded: succeeded})
            elif (state_machine_name == "gpsr_follow"):
                smach.StateMachine.add('LISTEN_COMMANDS_FOR_GPSR_FOLLOW',
                                       ListenCommandForGpsrFollow(),
                                       transitions={succeeded: succeeded})
            else:
                rospy.logerr("NO STATE MACHINE NAME WAS PROVIDED TO START LISTENING COMMANDS")
                return aborted


f = open("weather.txt", 'r')
t = open("summary.txt", 'a')

def avTemp(dew):
    if dew <= 16.0:
        print("dry and comfortable")
    elif 16.0 < dew < 20.0:
        print("muggy")
    elif dew >= 20.0:
        print("very sticky and oppressive")
    return(avTemp)

# make a list for each value
daylist = []
datelist = []
dewlist = []
listTotal = []

x = 0

for Line in f:
    #strip new line character
    Line = Line.strip('\n')
    #turn line into a list split at each space
    Line = Line.split(' ')
    #make a list by popping the needed value from line and appending into a new list
    day = Line.pop(0)
    date = Line.pop(0)
    dew = Line.pop(0)
    daylist.append(day)
    datelist.append(date)
    dewlist.append(dew)
    #find max and min temps using the max and min function for the line
    maxTemp = max(Line)
    minTemp = min(Line)
    dailyTemps = Line

#print out the needed data per day by increasing the x value, which is the positions on the related list
while x != 100:
    #try each print, if the index becomes to high, exit while loop.
    try:
        print("daylist" , daylist[x])
        print("datelist" , datelist[x])
        print("dewlist" , dewlist[x])
        print("dailyTemps" , dailyTemps)
        print("maxtemp" , maxTemp)
        print("mintemp" , minTemp)
        av = avTemp(float(dew[0]))
        print(av)
        print("---")
        x = x + 1
    except (IndexError):
        x = 100

#change all the lists into strings as they were given, seperated by spaces
day = ' '.join(daylist)
date = ' '.join(datelist)
dew = ' '.join(dewlist)
max = ' '.join(maxTemp)
min = ' '.join(minTemp)

#write all the needed data in the summary.txt file
t.write(day)
t.write("\n")
t.write(date)
t.write("\n")
t.write(dew)
t.write("\n")
t.write(max)
t.write("\n")
t.write(min)
t.write("\n")
#t.write(av)
t.write("\n")

#close file
f.close()
t.close()


import logging

from sqlalchemy.engine import Engine
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import sessionmaker
from SQLAlchemy.ex_sa_00 import create_db_engine
from SQLAlchemy.ex_sa_09 import Base, Kyc

logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO)

def create_kyc_table(engine: Engine) -> bool:
    status = False

    if not engine.dialect.has_table(engine, 'kyc'):
        Base.metadata.create_all(db_engine)

        logging.info("Created the kyc table !!!")

        status = True
    else:
        logging.info("The kyc table already exists !!!")

    return status

def insert_kyc_recs(engine: Engine):
    if engine.dialect.has_table(engine, 'kyc'):
        Session = sessionmaker(bind=engine)

        session = Session()

        try:
            ad_kyc = Kyc(kyc_flag=True, cid=1, ano=1001)
            session.add(ad_kyc)
            session.commit()

            logging.info("Inserted kyc for Alice")
        except SQLAlchemyError as e:
            logging.error(e)

        try:
            bb_kyc = Kyc(cid=2, ano=1002)
            session.add(bb_kyc)
            session.commit()

            logging.info("Inserted kyc for Bob")
        except SQLAlchemyError as e:
            logging.error(e)

        try:
            cd_kyc = Kyc(cid=3, ano=1003)
            session.add(cd_kyc)
            session.commit()

            logging.info("Inserted kyc for Charlie")
        except SQLAlchemyError as e:
            logging.error(e)

        session.close()
    else:
        logging.info("The kyc table *DOES NOT* exist !!!")

if __name__ == "__main__":
    db_engine = create_db_engine()
    if create_kyc_table(db_engine):
        insert_kyc_recs(db_engine)


#!/usr/bin/python

import re
import glob
import time
import xml.etree.ElementTree as ET
from py2neo import authenticate, Graph
from pandas import DataFrame

def main():
    # Connect to neo4j
    authenticate("localhost:7474", "neo4j", "neo4jpw")
    graph = Graph("localhost:7474/db/data/")

    pathway_list = read_pathway_list("C:/Databases/KEGG/pathway.list")
    main_pathway = "Metabolism"
    use_pathways = [i[2] for i in pathway_list if i[0] in main_pathway]
    use_pathways.remove('01100')

    metabolic_reactions = read_kegg_xml("C:/Databases/KEGG/kgml/ko", use_pathways=use_pathways)

    # Read in files and create a new database
    reactions = kegg_reactions("C:/Databases/KEGG/reaction/reaction")
    enzymes = kegg_enzymes("C:/Databases/KEGG/enzyme/enzyme")
    rclass = kegg_rclass("C:/Databases/KEGG/rclass/rclass")
    compounds = kegg_compounds("C:/Databases/KEGG/compound/compound")
    triples = find_triples(rclass)

    # Create a new database using rclass triples
    # create_db_from_triples(triples, rclass, compounds, graph)

    # Create a new database using metabolic reactions parsed from xml files
    # create_db_from_xml(metabolic_reactions, graph, reactions, enzymes, compounds)

    # Create a new database using reactions
    create_db_from_reactions(reactions, graph, enzymes, compounds, rclass)

    # Test database
    test_database(graph)

    return 0

def read_pathway_list(filename):
    # Read in pathway list and return a hierarchy
    pathways = list()
    level_1_heading = ""
    level_2_heading = ""
    with open(filename) as f:
        for line in f:
            if line[0:1] == "#":
                if line[0:2] == "##":
                    level_2_heading = line[2:].strip()
                else:
                    level_1_heading = line[1:].strip()
            else:
                pathways.append([level_1_heading, level_2_heading, line[0:5], line[6:].strip()])
    f.close()
    return pathways

def read_kegg_xml(folder, use_pathways=None, ignore_pathways=None):
    # Read in and parse xml files
    xml_files = glob.glob(folder + '/*.xml')
    if type(use_pathways) == list:
        r = re.compile("|".join(use_pathways))
        l_keepme = map(lambda x: len(r.findall(x)) > 0, xml_files)
        keep_me = [i for i, x in enumerate(list(l_keepme)) if x]
        xml_files = [xml_files[i] for i in keep_me]
    elif type(ignore_pathways) == list:
        r = re.compile("|".join(ignore_pathways))
        l_keepme = map(lambda x: len(r.findall(x)) > 0, xml_files)
        keep_me = [i for i, x in enumerate(list(l_keepme)) if not x]
        xml_files = [xml_files[i] for i in keep_me]

    reactions = list()
    for filename in xml_files:
        tree = ET.parse(filename).getroot()
        pathway_number = tree.attrib['number']
        pathway_name = tree.attrib['title']
        reactions_xml = tree.findall("reaction")
        for r in reactions_xml:
            reaction = dict()
            reaction['id'] = r.attrib['id']
            reaction['name'] = [x[3:] for x in r.attrib['name'].split(" ")]
            reaction['type'] = r.attrib['type']
            reaction['pathway_id'] = pathway_number
            reaction['pathway_name'] = pathway_name
            for child in r:
                # Currently takes just a single substrate or product when multiple are present
                cpd = dict()
                cpd['id'] = child.attrib['id']
                if child.attrib['name'][0:2] == "g1":
                    cpd['name'] = child.attrib['name'][3:9]  # glycan
                else:
                    cpd['name'] = child.attrib['name'][4:10]  # limit attribute to 6 characters (only first cpd entry)
                reaction[child.tag] = cpd
            reactions.append(reaction)
    return reactions

def kegg_reactions(filename):
    # Read in reactions file
    file = open(filename, "r")
    print("Reading Reaction File")
    reactions = file.readlines()
    file.close()

    # Find end of each record
    end_list = []
    for line_i, line in enumerate(reactions, 1):
        if re.match("///", line):
            end_list.append(line_i)

    # Loop through each record extracting tokens
    print("Parsing Reaction Class Data")
    count2 = 0
    reaction_data = dict()
    for ref in range(0, len(end_list)-1):
        count1 = count2
        count2 = end_list[ref]
        r = reactions[count1:count2]
        token_list = ["ENTR", "NAME", "DEFI", "EQUA", "RCLA", "ENZY"]
        tokens = dict()

        for line_i, line in enumerate(r, 1):
            if line[0:4] in token_list:
                tokens[line[0:4]] = line_i - 1
        # Parse tokens, extracting information into a dictionary
        if "ENTR" in tokens.keys() and "RCLA" in tokens.keys():
            record = dict()
            # Entry token
            record["entry"] = r[tokens["ENTR"]][12:18]
            # RClass token
            found_end = False
            count = 0
            v_rclass = []
            while not found_end:
                current_line = r[tokens["RCLA"] + count]
                if current_line[0:4] in ["RCLA", "    "]:
                    v_rclass.append(re.findall("([RC0-9]+)", current_line[12:]))
                    count += 1
                else:
                    found_end = True
                record['rclass'] = v_rclass
            # Name token
            if "NAME" in tokens.keys():
                record['name'] = r[tokens["NAME"]][12:].strip()
            # Definition token
            if "DEFI" in tokens.keys():
                record['definition'] = r[tokens["DEFI"]][12:].strip()
            # Equation token
            if "EQUA" in tokens.keys():
                record['equation'] = r[tokens["EQUA"]][12:].strip()
            # Enzyme token
            if "ENZY" in tokens.keys():
                record['enzyme'] = r[tokens["ENZY"]][12:].strip()
            # Add record to list
            reaction_data[record['entry']] = record
    print(len(reaction_data), "reaction records created\n")
    return reaction_data

def kegg_enzymes(filename):
    # Read in enzyme file
    file = open(filename, "r")
    print("Reading Enzyme File")
    enzymes = file.readlines()
    file.close()

    # Find end of each record
    end_list = []
    for line_i, line in enumerate(enzymes, 1):
        if re.match("///", line):
            end_list.append(line_i)

    # Loop through each record extracting tokens
    print("Parsing Enzyme Data")
    count2 = 0
    enzyme_data = dict()
    for ref in range(0, len(end_list)-1):
        count1 = count2
        count2 = end_list[ref]
        e = enzymes[count1:count2]
        token_list = ["ENTR", "NAME", "PATH"]
        tokens = dict()

        for line_i, line in enumerate(e, 1):
            if line[0:4] in token_list:
                tokens[line[0:4]] = line_i - 1
        # Parse tokens, extracting information into a dictionary
        if "ENTR" in tokens.keys():
            record = dict()
            entry_id = re.search("EC \d+.\d+.\d+.\d+", e[tokens["ENTR"]])
            if entry_id:
                record["entry"] = entry_id.group()
            else:
                break
            # Name token
            if "NAME" in tokens.keys():
                name_text = e[tokens["NAME"]].rstrip()
                if name_text[len(name_text) - 1] == ";":
                    record['name'] = name_text[12:len(name_text) - 1]
                else:
                    record['name'] = name_text[12:]
            # Pathway token
            if "PATH" in tokens.keys():
                found_end = False
                count = 0
                v_pathway = []
                while not found_end:
                    current_line = e[tokens["PATH"] + count]
                    if current_line[0:4] in ["PATH", "    "]:
                        v_pathway.append(current_line[12:19])
                        count += 1
                    else:
                        found_end = True
                record['pathway'] = v_pathway
            # Add record to list
            enzyme_data[record['entry']] = record
    print(len(enzyme_data), "enzyme records created\n")
    return enzyme_data

def kegg_rclass(filename):
    # Read in rclass file
    file = open(filename, "r")
    print("Reading Reaction Class File")
    rclass = file.readlines()
    file.close()

    # Find end of each record
    end_list = []
    for line_i, line in enumerate(rclass, 1):
        if re.match("///", line):
            end_list.append(line_i)

    # Loop through each record extracting tokens
    print("Parsing Reaction Class Data")
    count2 = 0
    rclass_data = dict()
    for ref in range(0, len(end_list)-1):
        count1 = count2
        count2 = end_list[ref]
        r = rclass[count1:count2]
        token_list = ["ENTR", "DEFI", "RPAI", "PATH"]
        tokens = dict()
        for line_i, line in enumerate(r, 1):
            if line[0:4] in token_list:
                tokens[line[0:4]] = line_i - 1
        # Parse tokens, extracting information into a dictionary
        if "ENTR" in tokens.keys():
            record = dict()
            record["entry"] = r[tokens["ENTR"]][12:19]
            if "DEFI" in tokens.keys():
                found_end = False
                count = 0
                v_definition = []
                while not found_end:
                    current_line = r[tokens["DEFI"] + count]
                    if current_line[0:4] in ["DEFI", "    "]:
                        v_definition.append(current_line[12:].rstrip())
                        count += 1
                    else:
                        found_end = True
                record['definition'] = v_definition
            if "RPAI" in tokens.keys():
                found_end = False
                count = 0
                v_rpair = []
                while not found_end:
                    current_line = r[tokens["RPAI"] + count]
                    if current_line[0:4] in ["RPAI", "    "]:
                        v_rpair.extend(re.findall("([C0-9_]+)", current_line[12:]))
                        count += 1
                    else:
                        found_end = True
                record['rpairs'] = v_rpair
            if "PATH" in tokens.keys():
                found_end = False
                count = 0
                v_pathway = []
                while not found_end:
                    current_line = r[tokens["PATH"] + count]
                    if current_line[0:4] in ["PATH", "    "]:
                        v_pathway.append(current_line[12:20])
                        count += 1
                    else:
                        found_end = True
                record['pathway'] = v_pathway
            # Add record to list
            rclass_data[record['entry']] = record
    print(len(rclass_data), "reaction class records created\n")
    return rclass_data

def kegg_compounds(filename):
    # Read in compound file
    file = open(filename, "r")
    print("Reading Compound File")
    compound = file.readlines()
    file.close()

    # Find end of each record
    end_list = []
    for line_i, line in enumerate(compound, 1):
        if re.match("///", line):
            end_list.append(line_i)

    # Loop through each record extracting tokens
    print("Parsing Compound Data")
    count2 = 0
    compound_data = dict()
    for ref in range(0, len(end_list)-1):
        count1 = count2
        count2 = end_list[ref]
        c = compound[count1:count2]
        token_list = ["ENTR", "NAME", "FORM", "EXAC", "PATH"]
        tokens = dict()
        for line_i, line in enumerate(c, 1):
            if line[0:4] in token_list:
                tokens[line[0:4]] = line_i - 1
        # Parse tokens, extracting information into a dictionary
        if "ENTR" in tokens.keys():
            record = dict()
            record["entry"] = c[tokens["ENTR"]][12:18]
            # Name token
            if "NAME" in tokens.keys():
                name_text = c[tokens["NAME"]].rstrip()
                if name_text[len(name_text) - 1] == ";":
                    record['name'] = name_text[12:len(name_text) - 1]
                else:
                    record['name'] = name_text[12:]
            # Formula token
            if "FORM" in tokens.keys():
                formula_text = c[tokens["FORM"]].rstrip()
                record['formula'] = formula_text[12:]
            # Exact mass token
            if "EXAC" in tokens.keys():
                mass_text = re.search("[0-9.]+", c[tokens["EXAC"]])
                if mass_text:
                    record['mass'] = float(mass_text.group(0))
            # Pathways token
            if "PATH" in tokens.keys():
                found_end = False
                count = 0
                v_pathway = []
                while not found_end:
                    current_line = c[tokens["PATH"] + count]
                    if current_line[0:4] in ["PATH", "    "]:
                        v_pathway.append(current_line[12:20])
                        count += 1
                    else:
                        found_end = True
                record['pathway'] = v_pathway

            # Add record to list
            compound_data[record['entry']] = record
    print(len(compound_data), "compound records created\n")
    return compound_data

def find_triples(rclass):
    triples = []
    for r in rclass:
        for p in rclass[r]['rpairs']:
            rpair_triple = p.split("_")
            rpair_triple.append(rclass[r]["entry"])
            triples.append(rpair_triple)
    return triples

def flatten_dict(d):
    out = list()
    for k, v in d.items():
        if type(v) is str:
            out.append("{k}: \"{v}\"".format(k=k, v=v))
        else:
            out.append("{k}: {v}".format(k=k, v=v))
    return ", ".join(out)

# Create and populate a neo4j database using data from reactions
# Reaction data are cross-referenced with compounds, rclass and enzyme
# Two sets of relationships are generated - connections which specify a non-directional edge between two compound
# nodes and reactions which are directional and contain additional data
def create_db_from_reactions(reactions, graph, enzymes=None, compounds=None, rclass=None):
    # clear old data
    graph.delete_all()
    # begin Cypher transaction
    tx = graph.begin()
    cursors = []
    # iterate over each reaction and add to database
    # include optional data as properties if available
    print("Processing", len(reactions), "reactions")
    start_time = time.time()
    for index, reaction_ref in enumerate(reactions):
        react_data = dict()
        r = reactions[reaction_ref]
        react_data['entry'] = r['entry']
        if "definition" in r:
            react_data['definition'] = r['definition']
        if "equation" in r:
            react_data['equation'] = r['equation']
        # include enzyme data
        if "enzyme" in r:
            enz_id = "EC " + r['enzyme']
            if enz_id in enzymes.keys():
                if 'entry' in enzymes[enz_id]:
                    react_data['enzyme_entry'] = enzymes[enz_id]['entry']
                if 'name' in enzymes[enz_id]:
                    react_data['enzyme_name'] = enzymes[enz_id]['name']
                if 'pathway' in enzymes[enz_id]:
                    react_data['enzyme_pathway'] = enzymes[enz_id]['pathway']
        # include reaction class data
        if "rclass" in r:
            for rc in r['rclass']:
                # loop over reaction class and extract compound data
                if rc[0] in rclass.keys():
                    # RClass
                    if "entry" in rclass[rc[0]]:
                        react_data['rclass_entry'] = rclass[rc[0]]['entry']
                    if "definition" in rclass[rc[0]]:
                        react_data['rclass_definition'] = rclass[rc[0]]['definition']
                    if "pathway" in rclass[rc[0]]:
                        react_data['rclass_pathway'] = rclass[rc[0]]['pathway']
                    if "definition" in rclass[rc[0]]:
                        react_data['rclass_rpairs'] = rclass[rc[0]]['rpairs']
                    # Compound 1
                    if rc[1] in compounds.keys():
                        cpd1 = "MERGE (c1:Compound {{{id}}})".format(id=flatten_dict(compounds[rc[1]]))
                    else:
                        cpd1 = "MERGE (c1:Compound {{entry: \"{id}\"}})".format(id=rc[1])
                    # Compound 2
                    if rc[2] in compounds.keys():
                        cpd2 = "MERGE (c2:Compound {{{id}}})".format(id=flatten_dict(compounds[rc[2]]))
                    else:
                        cpd2 = "MERGE (c2:Compound {{entry: \"{id}\"}})".format(id=rc[2])
                    # Mass change
                    delta_mass = None
                    if all([rc[1] in compounds.keys(), rc[2] in compounds.keys()]):
                        if all(['mass' in compounds[rc[1]], 'mass' in compounds[rc[2]]]):
                            delta_mass = round(compounds[rc[1]]['mass'] - compounds[rc[2]]['mass'], 4)
                            react_data['delta_mass'] = delta_mass
                            react_data['abs_delta_mass'] = abs(delta_mass)
                    # generate MERGE for cypher transaction
                    react = "[:REACTION {{{r}}}]".format(r=flatten_dict(react_data))
                    merge_text = "{cpd1} {cpd2} MERGE (c1)-{react}-(c2)"\
                        .format(cpd1=cpd1, react=react, cpd2=cpd2)
                    tx.append(merge_text)
                    # Create simple connection between pairs of compounds
                    if delta_mass is not None:
                        merge_text = "{cp1} {cp2} MERGE (c1)-[:CONNECTION {{abs_delta_mass: {m1}}}]-(c2)" \
                            .format(cp1=cpd1, cp2=cpd2, m1=abs(delta_mass))
                    else:
                        merge_text = "{cp1} {cp2} MERGE (c1)-[:CONNECTION]-(c2)".format(cp1=cpd1, cp2=cpd2)
                    cursors.append(tx.run(merge_text))
    end_time = time.time()
    print("Time to create transaction =", int(end_time - start_time), "seconds")
    start_time = time.time()
    tx.commit()
    end_time = time.time()
    print("Time to upload transaction =", int(end_time - start_time), "seconds")
    return

# Create and populate a neo4j database using data read from kgml (xml) files
# Data are cross-referenced with compounds, reactions and enzyme
# The KEGG xml files are incomplete and not all network reactions are detailed using this approach
def create_db_from_xml(metabolic_reactions, graph, reactions=None, enzymes=None, compounds=None):
    # clear old data
    graph.delete_all()
    # begin Cypher transaction
    tx = graph.begin()
    cursors = []
    # iterate over each member of metabolic_reactions and add to database
    # include optional data as properties if available
    print("Processing", len(metabolic_reactions), "reactions")
    start_time = time.time()
    for index, m in enumerate(metabolic_reactions):
        # Compound 1
        if m["substrate"]["name"] in compounds.keys():
            cpd1 = "MERGE (c1:Compound {{{id}}})".format(id=flatten_dict(compounds[m["substrate"]["name"]]))
        else:
            cpd1 = "MERGE (c1:Compound {{entry: \"{id}\"}})".format(id=m["substrate"]["name"])
        # Compound 2
        if m["product"]["name"] in compounds.keys():
            cpd2 = "MERGE (c2:Compound {{{id}}})".format(id=flatten_dict(compounds[m["product"]["name"]]))
        else:
            cpd2 = "MERGE (c2:Compound {{entry: \"{id}\"}})".format(id=m["product"]["name"])
        # Mass change
        delta_mass = None
        if all([m["substrate"]["name"] in compounds.keys(), m["product"]["name"] in compounds.keys()]):
            if all(['mass' in compounds[m["substrate"]["name"]], 'mass' in compounds[m["product"]["name"]]]):
                delta_mass = round(compounds[m["product"]["name"]]['mass'] -
                                   compounds[m["substrate"]["name"]]['mass'], 4)
        # Reaction
        if m["type"] == "reversible":
            relation = "-"
        else:
            relation = "->"
        react_data = dict()
        react_data["type"] = m["type"]

        # loop through reactions
        for react_name in m["name"]:
            react_data["entry"] = react_name
            if delta_mass is not None:
                react_data["delta_mass"] = delta_mass
                react_data["abs_delta_mass"] = abs(delta_mass)
            if react_name in reactions.keys():
                if "definition" in reactions[react_name]:
                    react_data["definition"] = reactions[react_name]["definition"]
                if "equation" in reactions[react_name]:
                    react_data["equation"] = reactions[react_name]["equation"]
                if "name" in reactions[react_name]:
                    react_data["reaction_name"] = reactions[react_name]["name"]
                if "rclass" in reactions[react_name]:
                    react_data["reaction_class"] = [x[0] for x in reactions[react_name]["rclass"]]
                if "enzyme" in reactions[react_name]:
                    enz = reactions[react_name]["enzyme"]
                    react_data["enzyme"] = enz
                    if enz in enzymes.keys():
                        if "name" in enzymes[enz]:
                            react_data["enzyme_name"] = enzymes[enz]["name"]
                        if "pathway" in enzymes[enz]:
                            react_data["pathway"] = enzymes[enz]["pathway"]
            # generate MERGE for cypher transaction
            react = "[:REACTION {{{r}}}]".format(r=flatten_dict(react_data))
            merge_text = "{cpd1} {cpd2} MERGE (c1)-{react}{relation}(c2)"\
                .format(cpd1=cpd1, react=react, relation=relation, cpd2=cpd2)
            tx.append(merge_text)
        # Create simple connection between pairs of compounds
        if delta_mass is not None:
            merge_text = "{cp1} {cp2} MERGE (c1)-[:CONNECTION {{abs_delta_mass: \"{m1}\"}}]-(c2)"\
                .format(cp1=cpd1, cp2=cpd2, m1=abs(delta_mass))
        else:
            merge_text = "{cp1} {cp2} MERGE (c1)-[:CONNECTION]-(c2)" .format(cp1=cpd1, cp2=cpd2)
        cursors.append(tx.run(merge_text))

    end_time = time.time()
    print("Time to create transaction =", int(end_time - start_time), "seconds")
    start_time = time.time()
    tx.commit()
    end_time = time.time()
    print("Time to upload transaction =", int(end_time - start_time), "seconds")
    return

# Create and populate a neo4j database using rclass and compound data
# This is the simplest approach, however rclass data are non-directional
def create_db_from_triples(triples, rclass, compounds, graph):
    # clear old data
    graph.delete_all()
    # begin Cypher transaction
    tx = graph.begin()
    cursors = []
    # iterate over each triple and add to database
    print("Processing", len(triples), "relationships")
    start_time = time.time()
    for index, t in enumerate(triples):
        # Lookup each value in triple.  If does not exist then return a record with just the Entry id
        # Compound 1
        if t[0] in compounds.keys():
            cpd1 = "MERGE (c1:Compound {{{id}}})".format(id=flatten_dict(compounds[t[0]]))
        else:
            cpd1 = "MERGE (c1:Compound {{entry: \"{id}\"}})".format(id=t[0])
        # Compound 2
        if t[1] in compounds.keys():
            cpd2 = "MERGE (c2:Compound {{{id}}})".format(id=flatten_dict(compounds[t[1]]))
        else:
            cpd2 = "MERGE (c2:Compound {{entry: \"{id}\"}})".format(id=t[1])
        # Reaction
        if t[2] in rclass.keys():
            react_data = rclass[t[2]]
        else:
            react_data = {'entry': t[2]}
        # Mass change
        if all([t[0] in compounds.keys(), t[1] in compounds.keys()]):
            if all(['mass' in compounds[t[0]], 'mass' in compounds[t[1]]]):
                delta_mass = round(compounds[t[0]]['mass'] -
                                   compounds[t[1]]['mass'], 4)
                react_data['delta_mass'] = delta_mass
                react_data['abs_delta_mass'] = abs(delta_mass)
        react = "[:REACTION {{{r}}}]".format(r=flatten_dict(react_data))
        merge_text = "{cpd1} {cpd2} MERGE (c1)-{react}-(c2)" \
            .format(cpd1=cpd1, react=react, cpd2=cpd2)
        cursors.append(tx.run(merge_text))
    end_time = time.time()
    print("Time to create transaction =", int(end_time - start_time), "seconds")
    start_time = time.time()
    tx.commit()
    end_time = time.time()
    print("Time to upload transaction =", int(end_time - start_time), "seconds")
    return

# A few tests to make sure we can execute cypher queries
def test_database(graph):
    tests = list()
    ppm_limit = 100
    mass_search = 18.0105
    tests.append({"title": "Return first 10 compound IDs",
                  "query": "MATCH (c:Compound) RETURN c.entry LIMIT 10", "dataframe": False})
    for r in graph.relationship_types:
        tests.append({"title": "Return first 10 relationships, type = {r}".format(r=r),
                      "query": "MATCH p=()-[r:{r}]->() RETURN p LIMIT 10".format(r=r), "dataframe": False})
        tests.append({"title": "Using dataframes", "query": "MATCH p=(c1:Compound)-[r:{r}]->(c2:Compound) "
                                                            "RETURN c1.entry, r.entry, c2.entry LIMIT 25".format(r=r),
                      "dataframe": True})
        tests.append({"title": "Search for mass match - dehydration reactions, type = {r}".format(r=r),
                      "query": "MATCH p=(c1:Compound)-[r:{r}]->(c2:Compound) WHERE "
                               "1E6*abs({mass_search} - r.abs_delta_mass)/{mass_search} < {ppm_limit} RETURN "
                               "c1.entry, r.entry, c2.entry LIMIT 25"
                     .format(r=r, mass_search=mass_search, ppm_limit=ppm_limit), "dataframe": True})

    for t in tests:
        print("\n" + t['title'])
        print(t['query'])
        if t['dataframe']:
            out = DataFrame(graph.data(t['query']))
        else:
            out = graph.data(t['query'])
        print(out)
    return

if __name__ == "__main__":
    main()


from django.shortcuts import render
from .forms import GeeksForm
from .models import GeeksModel

def home_view(request):
    context = {}
    if request.method == "POST":
        form = GeeksForm(request.POST, request.FILES)
        if form.is_valid():
            name = form.cleaned_data.get("name")
            img = form.cleaned_data.get("geeks_field")
            obj = GeeksModel.objects.create(
                                 title = name,
                                 img = img
                                 )
            obj.save()
            print(obj)
    else:
        form = GeeksForm()
    context['form']= form
    return render(request, "home.html", context)


import numpy as np
from scipy.stats import poisson

REQUEST_RATE = (3., 4.)
RETURN_RATE  = (3., 2.)

GAMMA = 0.9
RENTAL_INCOME = 10
TRANSFER_COST = 2
TRANSFER_MAX  = 5
MAX_CAPACITY  = 20
#FREE_PARKING_CAP = 10
#PARKING_COST = 4

# location indicies
A = 0
B = 1

MAX_PMF = 30

action_space = np.arange(-TRANSFER_MAX, TRANSFER_MAX+1)

def get_most_probable_location(state):
     i = state.argmax()
     return (i//(MAX_CAPACITY+1), i%(MAX_CAPACITY+1))

def get_state_vector(a, b):
    s = np.zeros((MAX_CAPACITY+1)**2)
    s[a*(MAX_CAPACITY+1)+b] = 1
    return s

def get_request_transitions_for_one_location(loc):
    """
    Construct transition matrix P_{to, from} for one location only for requests.
    The matrix has form (21, 21).
  """
    assert(loc==A or loc==B)
    # transition matrix P_{to, from} for one location only requests
    transition_matrix = np.zeros([MAX_CAPACITY+1, MAX_CAPACITY+1])

    request_pmf = poisson.pmf(np.arange(MAX_PMF), REQUEST_RATE[loc])
    np.testing.assert_almost_equal(request_pmf[-1], 0., decimal=12)
    for i in range(MAX_CAPACITY+1):
        for j in range(MAX_CAPACITY+1):
            if j==0:
                transition_matrix[i,j] = request_pmf[i:].sum()
            elif j<=i:
                transition_matrix[i,j] = request_pmf[i-j]
    return transition_matrix.T

def full_transition_matrix_A(transition_one_loc):
    block_size = MAX_CAPACITY+1 # for convenience
    transition_matrix = np.zeros([block_size**2, block_size**2])
    for i in range(block_size):
        transition_matrix[i:block_size**2: block_size,
                          i:block_size**2: block_size] = transition_one_loc
    return transition_matrix

def full_transition_matrix_B(transition_one_loc):
    block_size = MAX_CAPACITY+1 # for convenience
    transition_matrix = np.zeros([block_size**2, block_size**2])
    for i in range(block_size):
        transition_matrix[i*block_size:(i*block_size)+block_size,
                          i*block_size:(i*block_size)+block_size] = transition_one_loc
    return transition_matrix

def get_return_transition_matrix_one_location(loc):
    """
    Construct transition matrix P_{to, from} for one location only for returns

    """
    assert(loc==0 or loc==1)
    transition_matrix = np.zeros([MAX_CAPACITY+1, MAX_CAPACITY+1])

    return_pmf = poisson.pmf(np.arange(MAX_PMF), RETURN_RATE[loc])
    np.testing.assert_almost_equal(return_pmf[-1], 0., decimal=12)
    for i in range(MAX_CAPACITY+1):
        for j in range(MAX_CAPACITY+1):
            if j==MAX_CAPACITY:
                transition_matrix[i,j] = return_pmf[j-i:].sum()
            elif j>=i and j<MAX_CAPACITY:
                transition_matrix[i,j] = return_pmf[j-i]
    return transition_matrix.T

def get_moves(a, b, action):
    if action > 0: # from A to B
        return min(a, action)
    else:
        return max(-b, action)

def get_nightly_moves():
    transition_matrix = np.zeros([(MAX_CAPACITY+1)**2, (MAX_CAPACITY+1)**2,
                                  action_space.shape[0]])
    for a in range(MAX_CAPACITY+1):
        for b in range(MAX_CAPACITY+1):
            for i, action in enumerate(action_space):
                old_state_index = a*(MAX_CAPACITY+1)+b
                moves = get_moves(a, b, action)
                new_a = min(a - moves, MAX_CAPACITY)
                new_b = min(b + moves, MAX_CAPACITY)
                new_state_index = new_a *(MAX_CAPACITY+1) + new_b
                transition_matrix[new_state_index, old_state_index, i] = 1.
    return transition_matrix

def create_P_matrix():
    P_request_A_one_loc = get_request_transitions_for_one_location(A)
    P_request_A = full_transition_matrix_A(P_request_A_one_loc)

    P_request_B_one_loc = get_request_transitions_for_one_location(B)
    P_request_B = full_transition_matrix_B(P_request_B_one_loc)

    P_request = np.dot(P_request_A, P_request_B)

    P_return_A_one_loc = get_return_transition_matrix_one_location(A)
    P_return_A = full_transition_matrix_A(P_return_A_one_loc)

    P_return_B_one_loc = get_return_transition_matrix_one_location(B)
    P_return_B = full_transition_matrix_B(P_return_B_one_loc)

    P_return = np.dot(P_return_B, P_return_A)

    P_return_request = np.dot(P_return, P_request)

    P_move = get_nightly_moves()
    # this is the transpose of the state transition probability kernel
    P = np.ndarray(((MAX_CAPACITY+1)**2, (MAX_CAPACITY+1)**2, action_space.shape[0]))
    for i in range(action_space.shape[0]): # TODO: without a loop?
        P[:,:,i] = np.dot(P_return_request, P_move[:,:,i])
    return P

def create_R_matrix():
    # rename from get_reward()
    poisson_mask = np.zeros((2, MAX_CAPACITY+1, MAX_CAPACITY+1))
    po = (poisson.pmf(np.arange(MAX_CAPACITY+1), REQUEST_RATE[A]),
          poisson.pmf(np.arange(MAX_CAPACITY+1), REQUEST_RATE[B]))
    for loc in (A,B):
        for i in range(MAX_CAPACITY+1):
            poisson_mask[loc, i, :i] = po[loc][:i]
            poisson_mask[loc, i, i] = po[loc][i:].sum()
    # the poisson mask contains the probability distribution for renting x cars (x column)
    # in each row j, with j the number of cars available at the location

    reward = np.zeros([MAX_CAPACITY+1, MAX_CAPACITY+1, 2*TRANSFER_MAX+1])
    for a in range(MAX_CAPACITY+1):
        for b in range(MAX_CAPACITY+1):
            for action in range(-TRANSFER_MAX, TRANSFER_MAX+1):
                moved_cars = min(action, a) if action>=0 else max(action, -b)
                a_ = a - moved_cars
                a_ = min(MAX_CAPACITY, max(0, a_))
                b_ = b + moved_cars
                b_ = min(MAX_CAPACITY, max(0, b_))
                reward_a = np.dot(poisson_mask[A, a_], np.arange(MAX_CAPACITY+1))
                reward_b = np.dot(poisson_mask[B, b_], np.arange(MAX_CAPACITY+1))
                reward[a, b, action+TRANSFER_MAX] = (
                            (reward_a + reward_b) * RENTAL_INCOME -
                            np.abs(action) * TRANSFER_COST )
                #if a==20 and b==20 and action==0:
                #    print (a_,b_, action)
                #    print (reward_a, reward_b)
                #    print (reward[a, b, action+TRANSFER_MAX])
    reward = reward.reshape(441,11)
    return reward

# Then we solve it with the modifications in Exercise 4.7:

# Write a program for policy iteration and re-solve Jack’s car rental problem
# with the following changes. One of Jack’s employees at the first location
# rides a bus home each night and lives near the second location. She is happy
# to shuttle one car to the second location for free. Each additional car still
# costs $2, as do all cars moved in the other direction. In addition, Jack has
# limited parking space at each location. If more than 10 cars are kept overnight
# at a location (after any moving of cars), then an additional cost of $4 must
# be incurred to use a second parking lot (independent of how many cars are kept
# there). These sorts of nonlinearities and arbitrary dynamics often occur in
# real problems and cannot easily be handled by optimization methods other than
# dynamic programming. To check your program, first replicate the results given
# for the original problem.


#%%

import time

import pandas
import numpy
import math

from typing import List, Sized

from IPython.display import Markdown
from pandas.api.types import CategoricalDtype

from .weighted_average import weighted_statistics

#%%

mtcars = pandas.read_stata("./mtcars.dta")

array = numpy.empty(101, dtype=numpy.int64)
start = time.time()
print("NumPy value:", array.mean())
end = time.time()
print("NumPy time:", end - start)

start = time.time()
python_list = [int(item) for item in array]
print("Vanilla Python value:", sum(python_list) / len(array))
end = time.time()
print("Vanilla Python time:", end - start)

#%%
numpy_mtcars = mtcars.to_numpy()
#%%
numpy_mtcars.ndim
#%%
numpy_mtcars.size
#%%
numpy_mtcars.shape
#%%
numpy.rot90(numpy_mtcars, k=-1)

#%%
ids = ["{:04d}".format(id) for id in range(1, 2001)]
years = range(1984, 2021)
income_table = pandas.DataFrame({"pid": [], "syear": []})

for year in years:
    income_table = income_table.append(
        pandas.DataFrame({"pid": ids, "syear": [year] * len(ids)})
    )

random_generator = numpy.random.default_rng(42)
income_table["income"] = random_generator.integers(
    low=1000,
    high=10000,
    size=(len(income_table["pid"]),),
)
income_table["weight"] = random_generator.uniform(
    low=1,
    high=2,
    size=(len(income_table["pid"]),),
)

#%%

income_grouped_by = income_table[["syear", "income", "weight"]].groupby(["syear"])
# Calculate weighted average
aggregated = income_grouped_by.apply(
    lambda data_frame: pandas.Series(
        weighted_statistics(
            data_frame[["income"]].to_numpy(), weights=data_frame["weight"]
        ),
        ["mean_income", "sd", "N", "error"],
    )
).reset_index()
# Calculate confidence interval boundaries
aggregated["lower"] = aggregated["mean_income"] - aggregated["error"]
aggregated["upper"] = aggregated["mean_income"] + aggregated["error"]


"""
# Runtime - 25 ms
# Beats 94.63%

# Memory - 13.9 MB
# Beats 9.74%

# Time Complexity - O(log(n))
# Space Complexity - O(1)
"""

def firstBadVersion(n):
    i = 1
    j = n

    while i <= j:
        mid = (i + j) // 2

        if isBadVersion(mid):
            if not isBadVersion(mid - 1):
                return mid

            j = mid - 1

        else:
            i = mid + 1


def backspace(probs, totlen):
    numbacks = 0
    bestev = 1000000000
    for numbacks in range(len(probs)):
        bestcase = (totlen - len(probs) + 2*numbacks + 1)
        worstcase = (totlen - len(probs) + 2*numbacks + 1) + totlen + 1
        bestprob = 1
        for i in range(len(probs) - numbacks):
            bestprob *= float(probs[i])
        worstprob = 1 - bestprob
        ev = bestprob * bestcase + worstprob * worstcase
        bestev = min(bestev, ev)

    return bestev

def giveup(probs, totlen):
    return totlen + 2

a_file = open('A-small-attempt0.in')
numcases = int(a_file.readline())

for i in range(numcases):

    param = a_file.readline()
    params = param.split()
    totchars = int(params[1])
    probstr = a_file.readline()
    strprobs = probstr.split()

    ev = min(giveup(strprobs, totchars), backspace(strprobs, totchars))

    print("Case #" + str(i + 1) + ": " + str(ev))


from weibopy.auth import OAuthHandler
from weibopy.api import API
import configparser

MAX_PIC_NUM = 3

def press_sina_weibo():
    '''''
    调用新浪微博Open Api实现通过命令行写博文，功能有待完善
    author: socrates
    date:2012-02-06
    新浪微博：@偶是正太
    '''
    sina_weibo_config = configparser.ConfigParser()
    #读取appkey相关配置文件
    try:
        sina_weibo_config.readfp(open('sina_weibo_config.ini'))
    except ConfigParser.Error:
        print ('read sina_weibo_config.ini failed.')

    #获取需要的信息
    consumer_key = sina_weibo_config.get("userinfo","CONSUMER_KEY")
    consumer_secret =sina_weibo_config.get("userinfo","CONSUMER_SECRET")
    token = sina_weibo_config.get("userinfo","TOKEN")
    token_sercet = sina_weibo_config.get("userinfo","TOKEN_SECRET")

    #调用新浪微博OpenApi(python版)
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.setAccessToken(token, token_sercet)
    api = API(auth)

    #通过命令行输入要发布的内容
#    weibo_content = raw_input('Please input content:')
#    status = api.update_status(status=weibo_content)
#    print "Press sina weibo successful, content is: %s" % status.text
    iNum = 2
    while True:
        #上传图片，名称和内容如果重复，open api会检查，内容采用了取当前时间的机制
        #图片名称从0-5循环遍历
        status = api.upload( str(iNum)+'.jpg', 'test '+str(iNum) )

        if iNum == MAX_PIC_NUM:
            break
        else:
            iNum += 1

if __name__ == '__main__':
    press_sina_weibo()


#-*- coding: utf-8 -*-
from trytond.model import ModelSQL, ModelView, fields

__all__ = ['Rango']

class Rango(ModelSQL, ModelView):
    "Rango"
    __name__ = 'sigcoop_usuario.rango'
    #minimo y maximo de rango de acciones asignadas
    minimo = fields.Integer('Minimo', required=True)
    maximo = fields.Integer('Maximo', required=True)
    #Referencia asociados
    asociado = fields.Many2One('party.party', 'Asociado', domain=[('asociado', '=', True)])


import pika, sys, os
import time

def main():
	connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
	channel = connection.channel()

	#Checkear si hay cola con tal nombre
	channel.queue_declare(queue="colaBasica")
	channel.basic_qos(prefetch_count=1)

	#recibir mensajes (sobreescribir una funcion de devolucion de llamada "callback")
	def callback(ch, method, properties, body):
		print(" [x] Recived %r" % body.decode())
		time.sleep(10)
		print(" [x] Done")
		ch.basic_ack(delivery_tag = method.delivery_tag)

	channel.basic_consume(queue='colaBasica', on_message_callback=callback)

	print(' [*] Waiting for messages. To exit press CTRL+C')
	channel.start_consuming()

	#Bucle infinito
if __name__ == '__main__':
	try:
		main()
	except KeyboardInterrupt:
		print('Interrupted')
		try:
			sys.exit(0)
		except SystemExit:
			os._exit(0)


from deeplab import DeeplabV3
import cv2
import numpy as np
import os

def checkOverlap(boxa, boxb):
    x1, y1, w1, h2 = boxa[0][0], boxa[0][1], boxa[1][0] - boxa[0][0], boxa[1][1] - boxa[0][1]
    x2, y2, w2, h3 = boxb[0][0], boxb[0][1], boxb[1][0] - boxb[0][0], boxb[1][1] - boxb[0][1]

    if (x1 > x2 + w2):
        return 0
    if (y1 > y2 + h3):
        return 0
    if (x1 + w1 < x2):
        return 0
    if (y1 + h2 < y2):
        return 0
    colInt = abs(min(x1 + w1, x2 + w2) - max(x1, x2))
    rowInt = abs(min(y1 + h2, y2 + h3) - max(y1, y2))
    overlap_area = colInt * rowInt
    area1 = w1 * h2
    area2 = w2 * h3
    return overlap_area / min( area1, area2 )

def unionBox(a, b):
    x = min( a[0][0], b[0][0] )
    y = min( a[0][1], b[0][1] )
    w = max( a[1][0], b[1][0] ) - x
    h = max( a[1][1], b[1][1] ) - y
    return [[x, y], [x+w, y+h]]

def rectMerge_sxf(rects: []):
    rectList = rects.copy()
    rectList.sort()
    new_array = []
    complete = 1
    i = 0
    while i < len(rectList):
        j = i + 1
        succees_once = 0
        while j < len(rectList):
            boxa, boxb = rectList[i], rectList[j]
            if checkOverlap(boxa, boxb) > 0.5:
                complete = 0
                new_array.append(unionBox(boxa, boxb))
                succees_once = 1
                rectList.remove(boxa)
                rectList.remove(boxb)
                break
            j += 1
        if succees_once:
            continue
        i += 1
    new_array.extend(rectList)

    if complete == 0:
        complete, new_array = rectMerge_sxf( new_array )
    return complete, new_array

def detect( ori_img, res_dir, modelPath ):
    plt_img = cv2.cvtColor(np.asarray(ori_img), cv2.COLOR_RGB2BGR)
    deeplab = DeeplabV3(model_path=modelPath)
    masked_img = np.asarray(deeplab.detect_image(ori_img))

    img = cv2.cvtColor(masked_img, cv2.COLOR_BGR2GRAY)
    img = cv2.blur(img, (23, 23))
    img[img != 0] = 255

    edge = cv2.Canny(img, 30, 150)

    _, cnts, _ = cv2.findContours(edge, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    keyArea = []
    for i, contour in enumerate(cnts):
        x, y, w, h = cv2.boundingRect(contour)
        if w * h < 20000:
            continue
        keyArea.append([[x, y], [x + w, y + h]])
        cv2.rectangle(plt_img, (x, y), (x + w, y + h), (0, 0, 255), 3)
    cv2.imwrite(os.path.join(res_dir,'detect.png'), plt_img)

    _, keyArea = rectMerge_sxf(keyArea)
    return keyArea, masked_img