import re
import sys
from typing import Any, Dict, List, Optional, Sequence, TextIO, Tuple, cast

import click
import koji
from bugzilla import Bugzilla
from bugzilla.bug import Bug
from jira import JIRA, Issue
from tenacity import retry, stop_after_attempt

from elliottlib import Runtime, brew
from elliottlib.assembly import AssemblyTypes
from elliottlib.cli.common import cli, click_coroutine
from elliottlib.config_model import KernelBugSweepConfig
from elliottlib.exceptions import ElliottFatalError
from elliottlib.util import green_print
from elliottlib.bzutil import JIRABugTracker

@retry(reraise=True, stop=stop_after_attempt(3))
def _search_issues(jira_client, *args, **kwargs):
    return jira_client.search_issues(*args, **kwargs)

class FindBugsKernelCli:
    def __init__(self, runtime: Runtime, trackers: Sequence[str],
                 clone: bool, reconcile: bool, comment: bool, dry_run: bool):
        self._runtime = runtime
        self._logger = runtime.logger
        self.trackers = list(trackers)
        self.clone = clone
        self.reconcile = reconcile
        self.comment = comment
        self.dry_run = dry_run
        self._id_bugs: Dict[int, Bug] = {}  # cache for kernel bug; key is bug_id, value is Bug object
        self._tracker_map: Dict[int, Issue] = {}  # bug_id -> KMAINT jira mapping

    async def run(self):
        logger = self._logger
        if self.reconcile and not self.clone:
            raise ElliottFatalError("--reconcile must be used with --clone")
        if self._runtime.assembly_type is not AssemblyTypes.STREAM:
            raise ElliottFatalError("This command only supports stream assembly")
        group_config = self._runtime.group_config
        raw_config = self._runtime.gitdata.load_data(key='bug', replace_vars=group_config.vars).data.get("kernel_bug_sweep")
        if not raw_config:
            logger.warning("kernel_bug_sweep is not defined in bug.yml")
            return
        config = KernelBugSweepConfig.parse_obj(raw_config)
        jira_tracker = self._runtime.get_bug_tracker("jira")
        jira_client: JIRA = jira_tracker._client
        bz_tracker = self._runtime.get_bug_tracker("bugzilla")
        bz_client: Bugzilla = bz_tracker._client
        koji_api = self._runtime.build_retrying_koji_client(caching=True)

        # Getting KMAINT trackers
        trackers_keys = self.trackers
        trackers: List[Issue] = []
        if not trackers_keys:
            logger.info("Searching for open trackers...")
            trackers = self._find_kmaint_trackers(jira_client, config.tracker_jira.project, config.tracker_jira.labels)
            trackers_keys = [t.key for t in trackers]
            logger.info("Found %s tracker(s): %s", len(trackers_keys), trackers_keys)
        else:
            logger.info("Find kernel bugs linked from KMAINT tracker(s): %s", trackers_keys)
            for key in trackers_keys:
                logger.info("Getting tracker JIRA %s...", key)
                tracker = jira_client.issue(key)
                trackers.append(tracker)

        # Get kernel bugs linked from KMAINT trackers
        report: Dict[str, Any] = {"kernel_bugs": []}
        for tracker in trackers:
            bugs = self._find_bugs(jira_client, tracker, bz_client, config.bugzilla.target_releases)
            bug_ids = {int(b.id) for b in bugs}
            logger.info("Found %s bug(s) from %s: %s", len(bugs), tracker, bug_ids)
            for bug_id, bug in zip(bug_ids, bugs):
                if bug_id in self._tracker_map and self._tracker_map[bug_id].key != tracker.key:
                    raise ValueError(f"Bug {bug_id} is linked in multiple KMAINT trackers: {tracker.key} {self._tracker_map[bug_id].key}")
                self._id_bugs[bug_id] = bug
                self._tracker_map[bug_id] = tracker
                report["kernel_bugs"].append({
                    "id": bug_id,
                    "status": bug.status,
                    "summary": bug.summary,
                    "tracker": tracker,
                })
            if self.comment:
                logger.info("Checking if making a comment on tracker %s is needed...", tracker.key)
                self._comment_on_tracker(jira_client, tracker, koji_api, config.target_jira)

        if self.clone and self._id_bugs:
            # Clone kernel bugs into OCP Jira
            logger.info("Cloning bugs...")
            cloned_issues = self._clone_bugs(jira_client, list(self._id_bugs.values()), config.target_jira)
            report["clones"] = {}
            for bug_id, issues in cloned_issues.items():
                report["clones"][bug_id] = sorted(issue.key for issue in issues)
            logger.info("Done.")

        # Print a report
        self._print_report(report, sys.stdout)

    @staticmethod
    def _find_kmaint_trackers(jira_client: JIRA, tracker_project: str, labels: List[str]):
        conditions = [
            f"project = {tracker_project}",
            "status != Closed",
        ]
        if labels:
            conditions.extend([f"labels = \"{label}\"" for label in labels])
        jql = f'{" AND ".join(conditions)} ORDER BY created DESC'
        # 50 most recently created KMAINT trackers should be more than enough
        matched_issues = _search_issues(jira_client, jql, maxResults=50)
        return cast(List[Issue], matched_issues)

    def _find_bugs(self, jira_client: JIRA, tracker: Issue, bz_client: Bugzilla, bz_target_releases: Sequence[str]):
        logger = self._logger
        logger.info("Searching bugs in JIRA %s...", tracker.key)
        links = jira_client.remote_links(tracker.key)
        # Search for kernel bugs in tracker content
        pattern = re.compile(r"(?:bugzilla.redhat.com/|bugzilla.redhat.com/show_bug.cgi\?id=|bz)(\d+)")
        content = f"{tracker.fields.summary}\n{tracker.fields.description}"
        for link in links:
            content += f"\n{link.object.title}\n{link.object.url}"
        m = pattern.findall(content)
        bug_ids = sorted(set(map(int, m)))
        if not bug_ids:
            logger.info("No bugs found from %s", tracker.key)
            return []
        filtered_bugs = self._get_and_filter_bugs(bz_client, bug_ids, bz_target_releases)
        return filtered_bugs

    def _get_and_filter_bugs(self, bz_client: Bugzilla, bug_ids: List[int], bz_target_releases: Sequence[str]):
        """ Get specified bugs from Bugzilla, then return those bugs that match the defined target release.
        """
        logger = self._logger
        filtered_bugs: List[Bug] = []
        logger.info("Getting bugs %s from Bugzilla...", bug_ids)
        bugs = cast(List[Optional[Bug]], bz_client.getbugs(bug_ids))
        target_releases = set(bz_target_releases)
        for bug_id, bug in zip(bug_ids, bugs):
            if not bug:
                raise IOError(f"Error getting bug {bug_id}")
            target_release = bug.cf_zstream_target_release
            if not target_release:
                logger.warning("Target release of bug %s is not set", bug.weburl)
                continue
            if target_release not in target_releases:
                logger.warning("Bug %s is skipped because target release \"%s\" is not listed", bug.weburl, target_release)
                continue
            logger.info("Found bug %s matching target release %s", bug_id, target_release)
            filtered_bugs.append(bug)
        return filtered_bugs

    def _clone_bugs(self, jira_client: JIRA, bugs: Sequence[Bug], conf: KernelBugSweepConfig.TargetJiraConfig):
        logger = self._logger
        ocp_target_release = conf.target_release
        result: Dict[int, List[Issue]] = {}  # key is bug_id, value is a list of cloned jiras
        for bug in bugs:
            bug_id = int(bug.id)
            kmaint_tracker = self._tracker_map.get(bug_id)
            kmaint_tracker_key = kmaint_tracker.key if kmaint_tracker else None
            logger.info("Checking if %s was already cloned to OCP %s...", bug_id, ocp_target_release)
            jql_str = f'project = {conf.project} and component = {conf.component} and labels = art:cloned-kernel-bug and labels = "art:bz#{bug_id}" and "Target Version" = "{ocp_target_release}" order by created DESC'
            found_issues = cast(List[Issue], _search_issues(jira_client, jql_str=jql_str))
            if not found_issues:  # this bug is not already cloned into OCP Jira
                logger.info("Creating JIRA for bug %s...", bug.weburl)
                fields = self._new_jira_fields_from_bug(bug, ocp_target_release, kmaint_tracker_key, conf)
                if not self.dry_run:
                    issue = jira_client.create_issue(fields)
                    jira_client.add_remote_link(issue.key, {"title": f"BZ{bug_id}", "url": bug.weburl})
                    if kmaint_tracker:
                        jira_client.create_issue_link("Blocks", issue.key, kmaint_tracker)
                    result[bug_id] = [issue]
                else:
                    logger.warning("[DRY RUN] Would have created Jira for bug %s", bug_id)
            else:  # this bug is already cloned into OCP Jira
                logger.info("Bug %s is already cloned into OCP: %s", bug_id, [issue.key for issue in found_issues])
                result[bug_id] = found_issues
                if not self.reconcile:
                    continue
                fields = self._new_jira_fields_from_bug(bug, ocp_target_release, kmaint_tracker_key, conf)
                for issue in found_issues:
                    if issue.fields.status.name.lower() == "closed":
                        logger.info("No need to reconcile %s because it is Closed.", issue.key)
                        continue
                    logger.info("Reconciling Jira %s (cloned from bug %s) for %s", issue.key, bug_id, ocp_target_release)
                    if not self.dry_run:
                        issue.update(fields)
                    else:
                        logger.warning("[DRY RUN] Would have updated Jira %s to match bug %s", issue.key, bug_id)

        return result

    @staticmethod
    def _print_report(report: Dict, out: TextIO):
        print_func = green_print if out.isatty() else print  # use green_print if out is a TTY
        bugs = sorted(report.get("kernel_bugs", []), key=lambda bug: bug["id"])
        clones = report.get("clones", {})
        for bug in bugs:
            cloned_issues = clones.get(bug['id'], [])
            text = f"{bug['tracker']}\t{bug['id']}\t{'N/A' if not cloned_issues else ','.join(cloned_issues)}\t{bug['status']}\t{bug['summary']}"
            print_func(text, file=out)

    def _comment_on_tracker(self, jira_client: JIRA, tracker: Issue, koji_api: koji.ClientSession,
                            conf: KernelBugSweepConfig.TargetJiraConfig):
        logger = self._runtime.logger
        # Determine which NVRs have the fix. e.g. ["kernel-5.14.0-284.14.1.el9_2"]
        nvrs = re.findall(r"(kernel(?:-rt)?-\S+-\S+)", tracker.fields.summary)
        if not nvrs:
            raise ValueError("Couldn't determine build NVRs for tracker %s", tracker.key)
        nvrs = sorted(nvrs)
        # Check if nvrs are already tagged into OCP
        logger.info("Getting Brew tags for build(s) %s...", nvrs)
        candidate_brew_tag = conf.candidate_brew_tag
        prod_brew_tag = conf.prod_brew_tag
        build_tags = brew.get_builds_tags(nvrs, koji_api)
        shipped = all([any(map(lambda t: t["name"] == prod_brew_tag, tags)) for tags in build_tags])
        modified = all([any(map(lambda t: t["name"] == candidate_brew_tag, tags)) for tags in build_tags])
        tracker_message = None
        if shipped:
            tracker_message = f"Build(s) {nvrs} was/were already shipped and tagged into {prod_brew_tag}."
        elif modified:
            tracker_message = f"Build(s) {nvrs} was/were already tagged into {candidate_brew_tag}."
        if not tracker_message:
            logger.info("No need to make a comment on %s", tracker.key)
            return
        comments = jira_client.comments(tracker.key)
        if any(map(lambda comment: comment.body == tracker_message, comments)):
            logger.info("A comment was already made on %s", tracker.key)
            return
        logger.info("Making a comment on tracker %s", tracker.key)
        if not self.dry_run:
            jira_client.add_comment(tracker.key, tracker_message)
            logger.info("Left a comment on tracker %s", tracker.key)
        else:
            logger.warning("[DRY RUN] Would have left a comment on tracker %s", tracker.key)

    @staticmethod
    def _new_jira_fields_from_bug(bug: Bug, ocp_target_version: str, kmaint_tracker: Optional[str], conf: KernelBugSweepConfig.TargetJiraConfig):
        summary = f"{bug.summary} [rhocp-{ocp_target_version}]"
        if not summary.startswith("kernel"):  # ensure bug summary start with "kernel"
            summary = "kernel[-rt]: " + summary
        description = f"Cloned from {bug.weburl} by OpenShift ART Team:\n----\n{bug.description}"
        priority_mapping = {
            "urgent": "Critical",
            "high": "Major",
            "medium": "Normal",
            "low": "Minor",
            "unspecified": "Undefined",
        }
        bug_groups = set(bug.groups)
        fields = {
            "project": {"key": conf.project},
            "components": [{"name": conf.component}],
            "security": {'name': 'Red Hat Employee'} if 'private' in bug_groups or 'redhat' in bug_groups else None,
            "priority": {'name': priority_mapping.get(bug.priority, "Undefined")},
            "summary": summary,
            "description": description,
            "issuetype": {"name": "Bug"},
            "versions": [{"name": ocp_target_version[:ocp_target_version.rindex(".")]}],
            f"{JIRABugTracker.FIELD_TARGET_VERSION}": [{
                "name": ocp_target_version,
            }],
            "labels": ["art:cloned-kernel-bug", f"art:bz#{bug.id}"],
        }
        if kmaint_tracker:
            fields["labels"].append(f"art:kmaint:{kmaint_tracker}")

        # TODO: The following lines are commented out because we haven't reached to agreement
        # on how to handle kernel CVEs in OCP at this moment.
        # Without the following lines, kernel CVEs will be copied as normal (non-CVE) bugs.

        # is_cve_tracker = set(constants.TRACKER_BUG_KEYWORDS).issubset(set(bug.keywords))
        # if is_cve_tracker:
        #     # Find flaw bugs associated with the CVE tracker
        #     cve_flaws = []
        #     for flaw_id, flaw_bug in zip(bug.blocks, bug.bugzilla.getbugs(bug.blocks)):
        #         if not flaw_bug:
        #             raise IOError(f"Error getting flaw bug {flaw_id}. Permission issue?")
        #         if not BugzillaBug(flaw_bug).is_flaw_bug():
        #             continue  # this is not a flaw bug
        #         cve_flaws.append(flaw_bug)
        #     labels = {"Security", "SecurityTracking"}
        #     cve_names = re.findall(r"(CVE-\d+-\d+)", bug.summary)
        #     labels |= set(cve_names)
        #     labels |= {f"pscomponent:{component}" for component in bug.components}
        #     labels |= {f"flaw:bz#{flaw.id}" for flaw in cve_flaws}
        #     fields["labels"] += sorted(labels)
        return fields

@cli.command("find-bugs:kernel", short_help="Find kernel bugs")
@click.option("--tracker", "trackers", metavar='JIRA_KEY', multiple=True,
              help="Find by the specified KMAINT tracker JIRA_KEY")
@click.option("--clone",
              is_flag=True,
              default=False,
              help="Clone kernel bugs into OCP Jira")
@click.option("--reconcile",
              is_flag=True,
              default=False,
              help="Update summary, description, etc for already cloned Jira bugs. Must be used with --clone")
@click.option("--comment",
              is_flag=True,
              default=False,
              help="Make comments on KMAINT trackers")
@click.option("--dry-run",
              is_flag=True,
              default=False,
              help="Don't change anything")
@click.pass_obj
@click_coroutine
async def find_bugs_kernel_cli(
        runtime: Runtime, trackers: Tuple[str, ...], clone: bool,
        reconcile: bool, comment: bool, dry_run: bool):
    """Find kernel bugs in Bugzilla for weekly kernel release through OCP.

    Example 1: Find kernel bugs and print them out
    \b
        $ elliott -g openshift-4.14 find-bugs:kernel

    Example 2: Find kernel bugs and clone them into OCP Jira
    \b
        $ elliott -g openshift-4.14 find-bugs:kernel --clone

    Example 3: Clone kernel bugs into OCP Jira and also update already cloned Jiras
    \b
        $ elliott -g openshift-4.14 find-bugs:kernel --clone --reconcile
    """
    runtime.initialize(mode="none")
    cli = FindBugsKernelCli(
        runtime=runtime,
        trackers=trackers,
        clone=clone,
        reconcile=reconcile,
        comment=comment,
        dry_run=dry_run
    )
    await cli.run()


import pandas as pd
import numpy as np
import cvxpy as cp
import scipy.optimize as opt

def mean_variance_opt(df_mean,df_cov):
    assets = df_mean.index
    ret = df_mean.values
    cov = df_cov.values
    n = len(assets)
    w = cp.Variable(n)
    objective = cp.Minimize(cp.quad_form(w,cov))
    constraints = [w>=0]
    constraints += [ret @ w >= 0.1]
    problem = cp.Problem(objective,constraints)
    problem.solve()
    results = w.value/sum(w.value)
    return pd.Series(results,index=assets)

def risk_parity_alternative(df_cov):
    assets = df_cov.index
    cov = df_cov.values
    n = len(assets)
    w = cp.Variable(n)
    target = 0
    p_variance = w.T @ cov @ w
    for i in range(n):
        denominator = cov[i,:] @ w * n
        target += (w[i] - p_variance/denominator)**2
    constraints = [w>=0,sum(w)== 1 ]
    objective = cp.Minimize(target)
    problem = cp.Problem(objective,constraints)
    problem.solve()
    return pd.Series(w.value, index=assets)

def risk_parity(df_cov):

    assets = df_cov.index
    cov = df_cov.values
    n = len(assets)
    w0 = np.repeat(1/n,n)
    A = np.repeat(1,n)[np.newaxis,:]
    lb = np.array([1])
    ub = np.array([1])
    constraints = opt.LinearConstraint(A=A,lb=lb,ub=ub)
    bounds = opt.Bounds(ub=np.repeat(np.inf,n),lb=np.repeat(0,n))
    def target(w):
        target = 0
        p_variance = w.T.dot(cov).dot(w)
        for i in range(n):
            denominator = cov[i, :].dot(w) * n
            target += (w[i] - p_variance / denominator) ** 2
        return target

    res = opt.minimize(target,w0,method='SLSQP',constraints=constraints,bounds=bounds)

    return pd.Series(res.x, index=assets)

df_meta = pd.read_excel(r'C:\Users\mrzha\OneDrive\Documents\AssetAllocation\Asset Allocation Low Risk.xlsx',
                        sheet_name='Candidate', index_col='Index')
df_meta['Fund ID'] = df_meta['Fund ID'].map(lambda x: '0' * (6 - len(str(x))) + str(x))
fund_list = df_meta['Fund ID']
fund_name = df_meta['Fund Name']
fund_mapping = dict(zip(fund_list, fund_name))

df = pd.read_csv(r'C:\Users\mrzha\OneDrive\Documents\AssetAllocation\Fund Return\Low Risk Return.csv',index_col=0,parse_dates=['Date'])
df.columns = df.columns.map(lambda x: '0'*(6-len(str(x)))+str(x))
candidates = ['003327','000171','000215','519062']
df = df[candidates]
df_month = df.resample('M').last()
df_month = df_month.pct_change()
date = '2020/01/31'

df_ewcorr = df_month.ewm(halflife=60,min_periods=36).corr().loc[date]
df_ewcov = df_month.ewm(halflife=60,min_periods=36).cov().loc[date]
df_ewcorr.index = df_ewcorr.index.get_level_values(1)
df_ewcov.index = df_ewcov.index.get_level_values(1)
df_mean = df_month.mean()
df_cov = df_ewcov.dropna(axis=1).dropna(axis=0)
df_mean = df_mean.reindex(df_cov.index)

results = pd.DataFrame(index=candidates)
results.loc[:,'fund_name'] = results.index.map(lambda x: fund_mapping[x])
results.loc[:,'return'] = 12*df_mean
results.loc[:,'vol'] = np.sqrt(12)*df_month.std()
results.loc[:,'ret over risk'] = results.loc[:,'return']/results.loc[:,'vol']
results.loc[:,'mean_variance_optimization'] = mean_variance_opt(df_mean,df_cov)
results.loc[:,'risk_parity'] = risk_parity(df_cov)

results.to_excel(r'C:\Users\mrzha\OneDrive\Documents\AssetAllocation\LowRiskAllocationResults.xlsx')


from __future__ import absolute_import, print_function
"""
SignFidata
"""
import torch
import torch.utils.data as data

import os
import sys
import numpy as np

def default_loader(root):   #root is the data storage path  eg. path = D:/CSI_Data/signfi_matlab2numpy/
    label_path = root + "/labels.npy"    #storage path : label of Sign
    amp_path = root + "/amp_datas.npy"   #storage path : the amplitude of CSI
    phase_path = root + "/phase_datas.npy" #storage path :  the phase value of CSI
    label_env_path = root + "/label_env.npy" #storage path : label of person or scenario
    label_all = np.load(label_path)     #label of Sign
    label_env_all = np.load(label_env_path) #lable of env(person or scenario)
    amp_all = np.load(amp_path)
    phase_all = np.load(phase_path)
    return label_all,label_env_all,amp_all,phase_all

class MyData(data.Dataset):
    def __init__(self, root, set_name, loader = default_loader):  #set_name = "test" or "train" ;  root is the path of data;

        self.root = root
        self.load = loader
        label_all,label_env_all,amp_all,phase_all = self.load(root)
        if set_name == "train":
            self.label = label_all[0:4500]
            self.label_env = label_env_all[0:4500]
            self.amp = amp_all[:,:,:,0:4500]
            self.phase = phase_all[:,:,:,0:4500]
        else :
            self.label = label_all[4500:7500]
            self.label_env = label_env_all[4500:7500]
            self.amp = amp_all[:,:,:,4500:7500]
            self.phase = phase_all[:,:,:,4500:7500]

    def __getitem__(self, index):
        label_index,label_env_index, amp_index, phase_index = self.label[index],self.label_env[index], self.amp[:,:,:,index], self.phase[:,:,:,index]
        # choosing amplitude or phase values of CSI for perception
        amp_index_change = np.empty([3,200,30],dtype= float) #the initial amp.shape is [200,30,3] change to [3,200,30]
        phase_index_change = np.empty([3,200,30],dtype= float)
        for i in range(0,3):
            amp_index_change[i,:,:] = amp_index[:,:,i]
            phase_index_change[i,:,:] = phase_index[:,:,i]
        amp_index_Tensor = torch.from_numpy(amp_index_change)    # change the type from numpy to tensor
        amp_index_Tensor = amp_index_Tensor.type(torch.FloatTensor) #change to Float
        phase_index_Tensor = torch.from_numpy(phase_index_change)
        phase_index_Tensor = phase_index_Tensor.type(torch.FloatTensor)  # change to Float

        label_index_Tensor = label_index.astype(np.int)
        label_index_Tensor = torch.tensor(label_index_Tensor)
        label_index_Tensor = label_index_Tensor.type(torch.FloatTensor)

        #return data
        return amp_index_Tensor,label_index_Tensor   #or output phase_index_Tensor   and   label_env

    def __len__(self):
        return len(self.label_env)

class SignFi:
    def __init__(self, root):
            self.train = MyData(root, set_name="train")
            self.test = MyData(root, set_name = "test")

def test_SignFi(path):
    print("hahahahhahahha")
    print(SignFi.__name__)
    data = SignFi(path)
    print("the length of training set:{}".format(len(data.train)))
    print("the length of test set:{}".format(len(data.test)))
    print("the data of train[0]:{}".format(data.train[0]))
    print("the data of train[3000]:{}".format(data.train[3000]))
    print("the data of test[1]:{}".format(data.test[1]))
    print("the data of test[780]:{}".format(data.test[780]))
    print("the type of test[780]:{}".format(type(data.test[780])))
    print("the shape of test[780][0]:{}".format(data.test[780][0].shape))

if __name__ == "__main__":
    test_SignFi("D:/CSI_Data/SignFi/")


import argparse
import itertools
import logging
from pprint import pprint
from typing import List, Dict

import PIL
from PIL import Image
import time
import sys

import mss
from mss import models
from mss.screenshot import ScreenShot

import espixelstick
from mapping import PixelAddress, PixelStrip
import img_proc

import signal
import sys

from region import ScreenRegion, CombineRegion

def signal_handler(sig, frame):
    print('Exiting...')

    # this doesn't turn off the leds for some reason
    # TODO: shut off the LEDs when we exit
    for i in range(1, 3):
        if sender[i] is not None:
            sender[i].dmx_data = (0, 0, 0, 0)

    sender.stop()  # do not forget to stop the sender
    sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)

def send_data(sender, pixel_strips: List[PixelStrip], save_image):
    # Turn off automatic flushing so we can send everything at the same time
    sender.manual_flush = True
    for pixel_strip in pixel_strips:
        pixel_data = []
        for region in pixel_strip.regions:
            img = region.capture_and_resize(pixel_strip.max_row_length, pixel_strip.rows, save_image)
            pixel_data += img_proc.create_color_data(img, pixel_strip, region)

        # print number of pixels
        # print(len(pixel_data)/4)

        # Set pixel data for the universe
        sender[pixel_strip.universe].dmx_data = pixel_data

    # Flush all data
    sender.flush()

    # Reenable automatic flushing
    sender.manual_flush = False

def flatten(l: List) -> List:
    return list(itertools.chain.from_iterable(l))

# Send a solid color that changes every 2s
# This doesn't use the pixel mapping structure, it just spams solid colors to the DMX
# universes defined in 'pixel_strips'.  Tests the physical LED wiring and controller setup
# ONLY (does not test the pixel mapping)
def test_strips(pixel_strips):
    sample_data = {
        "red": (255, 0, 0, 0) * 300,
        "green": (0, 255, 0, 0) * 300,
        "blue": (0, 0, 255, 0) * 300,
    }
    color_iter = itertools.cycle(sample_data.values())

    for i in range(20):
        color_data = next(color_iter)
        for pixel_strip in pixel_strips:
            sender[pixel_strip.universe].dmx_data = color_data

        time.sleep(1)

    time.sleep(10)
    sender.stop()

# Turn off the LEDs
def strips_off(pixel_strips):
    print("Turning strips off")
    for pixel_strip in pixel_strips:
        sender[pixel_strip.universe].dmx_data = (0, 0, 0, 0) * 1000
    sender.stop()
    print("Turned strips off.  This doesn't always exit, you might need to press ctrl+c again")

################################################################################################################################################

parser = argparse.ArgumentParser(description='esp-based bias lighting')
parser.add_argument('--debug', dest='debug', action='store_true', default=False, help='debug mode (print debug logs)')
parser.add_argument('--test', dest='test', action='store_true', default=False, help='test mode (spam solid colors)')
parser.add_argument('--slow', dest='slow', action='store_true', default=False, help='slow mode (3 fps)')
parser.add_argument('--save', dest='save', action='store_true', default=False, help='save png image files for debugging')
parser.add_argument('--off', dest='off', action='store_true', default=False, help='turn strips off')
parser.add_argument('--profile', dest='profile', action='store_true', default=False, help='profile (create cProfile profile for debugging performance)')
# parser.add_argument('--fps', dest='fps', action='store', default=30, help='fps')
args = parser.parse_args()

if args.debug:
    print("DEBUG MODE ENABLED")
    logging.basicConfig(level=logging.DEBUG)

frame_rate = 3 if args.debug or args.slow else 30

# I probably don't wanna do the sleep time thing, because this doesn't at all take
# into account how long it took to process the images.  probably responsible
# for some of the choppiness.  replace with asyncio or something like timeloop
# TODO: try out this and see if it helps: https://github.com/sankalpjonn/timeloop
sleep_time = round(1/frame_rate, 3)-.001

### A lot of the code below is very specific to my specific pixel setup.  The setup consists of:
# LED type: these are SK6812 LEDs which are RGBW.  This is what I already had in channels w/ connectors
# I would have preferred to use APA102 LEDs but I didn't have them in channels.  The RGBW LEDs require 4 DMX channels per light
# which reduces the number of LEDs we can have in a DMX Universe.  That's definitely a downside to these LEDs
# (DMX universes can have 512 lights, and each individual LED is RGBW=4x lights).
#
# 3 distinct LED fixtures
# - each fixture uses 1 esp8266 (dev board)
#
# - right monitor
#   - dmx universe 1
#   - 4 rows x 29 pixels in horizontal zigzag pattern starting at lower left
#
# - left monitor
#   - dmx universe 2
#   - 4 rows x 29 pixels in horizontal zigzag pattern starting at lower left
#
# - desk
#   - dmx universe 3+4
#   - 3 rows, horizontal zigzag, starting at upper right
#   - row #1 (dmx universe 3)
#     - back of desk, 75 pixels
#   - row #2 - first part - 53 pixels in universe #3
#   - row #2 - second part - 18 pixels in universe #4
#     - row 2 is split due to universe size limit
#   - row #3 - 71 pixels (dmx universe 4)
#     - row #2 and #3 are in the middle of the desk facing downwards

# function to determine if a specific pixel falls into a specific region
# used for the monitors which have 4x rows of 29 pixels each
# this is how we determine which LEDs should get the top of a screen vs the bottom of a screen
# this functionality should really be refactored to use the newer ScreenRegion class
def _region_fn_monitors(pixel: PixelAddress) -> str:
    if pixel.index < 29*2:
        return "bottom"
    else:
        return "top"

mss_instance = mss.mss()

regionTop3 = ScreenRegion("top", 3, mss_instance)
regionBottom3 = ScreenRegion("bottom", 3, mss_instance)
regionTop1 = ScreenRegion("top", 1, mss_instance)
regionBottom1 = ScreenRegion("bottom", 1, mss_instance)
allBottom = CombineRegion("bottom", 1, mss_instance)

# The list of pixels strips.  The PixelStrip has kind of grown into a catch-all for a bunch of functionality & data
pixel_strips = [
    # 192.168.1.237
    PixelStrip(
        strip_addr="192.168.1.237", universe=1, row_length=[29,29,29,29], rows=4, start_left=True,
        start_bottom=True, region_fn=_region_fn_monitors,
        regions=[regionBottom3, regionTop3],
    ),

    PixelStrip(
        strip_addr="192.168.1.240", universe=2, row_length=[29,29,29,29], rows=4, start_left=True,
        start_bottom=True, region_fn=_region_fn_monitors,
        regions=[regionBottom1, regionTop1],
    ),

    PixelStrip(
        strip_addr="192.168.1.243", universe=3, row_length=[75, 53], rows=2, start_left=False,
        start_bottom=False, region_fn=lambda _: "bottom",
        regions=[allBottom],
    ),

    # # 29 in first channel
    # # 42 in second in 2nd channel
    ## 24 in second channel that are part of universe 3
    # # 29+24 in universe 3, remainder (18) in universe 4
    # # 71 total
    # This one has the most complicated mapping due to the mismatched row lengths and that
    # it actually encompasses multiple DMX universes :/
    # the mapping doesn't work 100% here yet, there's something wrong with the offset calculations
    # in mapping.py.  its a few pixels off.  it could also be that we don't handle matrices with
    # multiple different row_lengths correctly when mapping screen pixels to led pixels
    PixelStrip(
        strip_addr="192.168.1.243", universe=4, row_length=[18, 71], rows=2, start_left=True,
        start_bottom=False, region_fn=lambda _: "bottom",
        regions=[allBottom], first_pixel_offset=53, max_row_length=71,
    ),
]

sender = espixelstick.create_sender(pixel_strips, fps=frame_rate)

if args.profile:
    for i in range(0, 100):
        send_data(sender, pixel_strips, save_image=args.save)
        time.sleep(sleep_time)

    sender.stop()
    sys.exit(0)

if args.test:
    test_strips(pixel_strips)
    sys.exit(0)

if args.off:
    strips_off(pixel_strips)
    sys.exit(0)

# for i in range(0, 1000):
while True:
    send_data(sender, pixel_strips, save_image=args.save)
    time.sleep(sleep_time)


import numpy as np
import math

class TreeNode:
    """
    A node in a decision tree, this class is used only to represent the tree structure (it does not contain any logic)
    """
    def __init__(self, feature: int, value: float, left, right):
        assert isinstance(left, bool) or isinstance(left, TreeNode), f"left must be a TreeNode object or a boolean value but is: {type(left)}"
        assert isinstance(right, bool) or isinstance(right, TreeNode), "right must be a TreeNode object or a boolean value"
        assert isinstance(feature, int), "feature must be a int"
        assert isinstance(value, float), "value must be a float"

        self.feature = feature
        self.value = value
        self.left = left
        self.right = right

def binary_entropy(p: float) -> float:
    """
    Return the entropy of a binary source with 2 symbols of probability "p" and "1-p"

    :param p: should be a value between 0 and 1
    """

    if p < 0 or p > 1:
        raise ValueError(f"p must be a value between 0 and 1, currently is: {p}")

    # confirmed by plot this function
    if p == 0 or p == 1:
        return 0

    return -p * math.log2(p) - (1 - p) * math.log2(1 - p)

def h_entropy(p: float, n: float) -> float:
    """
    Return the binary entropy of p / (p + n)

    :param p: number of positive sample
    :param n: number of negative sample
    """
    return binary_entropy(p / (p + n))

def plurality_value(y_train: list[bool]) -> bool:
    """
    return the most frequently repeated boolean element

    :param y_train: a list containing boolean value
    """
    assert isinstance(y_train, list), "boolean_list should be a list object"

    if len(y_train) == 0:
        raise ValueError("y_train should not be empty")

    positive = len([y for y in y_train if y])
    if positive >= len(y_train) / 2:
        return True

    # TODO: the pseudocode says to choose random if there is a tie
    return False

def find_median(x_train: list, feature: int):
    assert isinstance(x_train, list) or isinstance(x_train, np.ndarray), "x_train should be a list or a numpy array"
    assert isinstance(feature, int), "feature should be an int"

    x_train = sorted(x_train, key=lambda x: x[feature])
    if len(x_train) % 2 != 0:
        return x_train[len(x_train) // 2][feature]

    right = x_train[len(x_train) // 2][feature]
    left = x_train[len(x_train) // 2 - 1][feature]
    return (left + right) / 2

def make_partitions(x_train: list, y_train: list[bool], median: float, feature: int):
    assert isinstance(x_train, list) or isinstance(x_train, np.ndarray), "x_train should be a list or a numpy array"
    assert isinstance(y_train, list) or isinstance(y_train, np.ndarray), "y_train should be a list or a numpy array"
    assert isinstance(median, float), "median should be a float"
    assert isinstance(feature, int), "feature should be an int"

    x_train_left = []
    y_train_left = []
    x_train_right = []
    y_train_right = []
    for x, y in zip(x_train, y_train):
        if x[feature] <= median:
            x_train_left.append(x)
            y_train_left.append(y)
        else:
            x_train_right.append(x)
            y_train_right.append(y)
    return x_train_left, y_train_left, x_train_right, y_train_right

def remainder(x_train: list, y_train: list[bool], feature: int) -> float:
    """
    Return the remainder of the partition using the feature
    """
    assert isinstance(x_train, list) or isinstance(x_train, np.ndarray), "x_train should be a list or a numpy array"
    assert isinstance(y_train, list) or isinstance(y_train, np.ndarray), "y_train should be a list or a numpy array"
    assert isinstance(feature, int), "feature should be an int"

    median = find_median(x_train, feature)
    partitions = make_partitions(x_train, y_train, median, feature)
    x_train_left, y_train_left, x_train_right, y_train_right = partitions

    p1 = len([x for x in y_train_left if x])
    n1 = len(y_train_left) - p1

    p2 = len([x for x in y_train_right if x])
    n2 = len(y_train_right) - p2

    # if some partition is empty, the remainder is infinite because it means that the feature is useless
    if p1+n1 == 0 or p2+n2 == 0:
        return float("inf")

    h1 = h_entropy(p1, n1)
    r1 = h1 * (len(x_train_left) / len(x_train))

    h2 = h_entropy(p2, n2)
    r2 = h2 * (len(x_train_right) / len(x_train))

    return r1 + r2

def min_remainder(x_train: list, y_train: list, features: list[int]) -> int:
    """
    This function is used to find the feature that minimizes the remainder.\n
    In theory we should maximize the information gain, but since the first part
    of the gain formula it's the same for all the features, we can directly calculate
    the minimum reminder
    :param x_train: x train set
    :param y_train: y train set
    :param features: iterable list of features
    :return: the feature that minimizes the remainder
    """
    assert isinstance(x_train, list) or isinstance(x_train, np.ndarray), "x_train should be a list or a numpy array"
    assert isinstance(y_train, list) or isinstance(y_train, np.ndarray), "y_train should be a list or a numpy array"
    assert isinstance(features, list), "feature should be an int"

    if len(features) == 0:
        raise ValueError("features should have size > 0")

    return min([(f, remainder(x_train, y_train, f)) for f in features], key=lambda x: x[1])[0]

class MyDecisionTree:
    """
    This class is used to train and stores a binary decision tree classifier that splits the data of every feature
    using their median.\n
    Use the "fit" method to train the classifier and the "predict" method to predict the class of a new sample.
    """
    def __init__(self):
        self.tree = None

    def fit(self, x_train: list, y_train: list) -> None:
        """
        This method is used to train the classifier.

        :param x_train: a matrix where each row is a sample and each column is a feature
        :param y_train: a list of boolean values where each value is the class of the corresponding sample
        """
        if len(x_train) != len(y_train):
            raise ValueError("x_train e y_train should have the same size")
        if len(x_train) == 0:
            raise ValueError("x_train e y_train should not be empty")
        if len(x_train[0]) == 0:
            raise ValueError("x_train should contains non empty list")

        # TODO: maybe a set is better than a list for "features"
        features = list(range(len(x_train[0])))

        self.tree = self._make_tree(x_train, y_train, features, y_train)

    def _make_tree(self, x_train: list, y_train: list, features: list, y_train_parent: list) -> [TreeNode, bool]:
        if len(x_train) == 0:
            return plurality_value(y_train_parent)

        positive = len([y for y in y_train if y])
        if positive == len(y_train):
            return True
        if positive == 0:
            return False

        if len(features) == 0:
            return plurality_value(y_train)

        f = min_remainder(x_train, y_train, features)
        median = find_median(x_train, f)

        partitions = make_partitions(x_train, y_train, median, f)
        x_train_left, y_train_left, x_train_right, y_train_right = partitions

        features.remove(f)
        left = self._make_tree(x_train_left, y_train_left, features, y_train)

        right = self._make_tree(x_train_right, y_train_right, features, y_train)
        features.append(f)

        return TreeNode(f, median, left, right)

    def predict(self, x_test: list) -> list[bool]:
        assert isinstance(x_test, list) or isinstance(x_test, np.ndarray), "x_train_row should be a list or a numpy array"
        return [self.predict_single(x) for x in x_test]

    def predict_single(self, x_train_row: list) -> bool:
        assert isinstance(x_train_row, list) or isinstance(x_train_row, np.ndarray), "x_train_row should be a list or a numpy array"

        if self.tree is None:
            raise ValueError('You should train the model before using it; use the "fit" method')

        node = self.tree
        while not isinstance(node, bool):
            if x_train_row[node.feature] <= node.value:
                node = node.left
            else:
                node = node.right
        return node


def meanOrderFrequency(path_to_dataset):
    """
    Displays the mean order frequency by utilizing the orders table.
    :param path_to_dataset: this path should have all the .csv files for the dataset
    :type path_to_dataset: str
    """
    assert isinstance(path_to_dataset, str)
    import pandas as pd
    order_file_path = path_to_dataset + '/orders.csv'
    orders = pd.read_csv(order_file_path)
    print('On an average, people order once every ', orders['days_since_prior_order'].mean(), 'days')

def numOrdersVsDays(path_to_dataset):
    """
    Displays the number of orders and how this number varies with change in days since last order.
    :param path_to_dataset: this path should have all the .csv files for the dataset
    :type path_to_dataset: str
    """

    assert isinstance(path_to_dataset, str)

    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt
    import matplotlib

    order_file_path = path_to_dataset + '/orders.csv'

    orders = pd.read_csv(order_file_path)

    order_by_date = orders.groupby(by='days_since_prior_order').count()

    fig = plt.figure(figsize = [15, 7.5])
    ax = fig.add_subplot()
    order_by_date['order_id'].plot.bar(color = '0.75')
    ax.set_xticklabels(ax.get_xticklabels(), fontsize= 15)
    plt.yticks(fontsize=16)
    ax.get_xaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x))))
    ax.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x/1000))))
    ax.set_xlabel('Days since previous order', fontsize=16)
    ax.set_ylabel('Number of orders / 1000', fontsize=16)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.get_children()[7].set_color('0.1')
    ax.get_children()[14].set_color('0.1')
    ax.get_children()[21].set_color('0.1')
    ax.get_children()[30].set_color('0.1')
    my_yticks = ax.get_yticks()
    plt.yticks([my_yticks[-2]], visible=True)
    plt.xticks(rotation = 'horizontal');

def numOrderDaysSizeBubble(path_to_dataset):
    """
    Plots a bubble plot in which:
    x: Days since Previous Order
    y: Number of orders/1000
    size: Average Size of order given it was placed on x

    :param path_to_dataset: this path should have all the .csv files for the dataset
    :type path_to_dataset: str
    """
    import numpy as np
    import pandas as pd
    import matplotlib
    import matplotlib.pyplot as plt
    import seaborn as sns

    assert isinstance(path_to_dataset, str)

    order_file_path = path_to_dataset + '/orders.csv'
    order_product_prior_file_path = path_to_dataset + '/order_products__prior.csv'

    orders = pd.read_csv(order_file_path)
    order_products_prior = pd.read_csv(order_product_prior_file_path)

    order_id_count_products = order_products_prior.groupby(by='order_id').count()
    orders_with_count = order_id_count_products.merge(orders, on='order_id')
    order_by_date = orders.groupby(by='days_since_prior_order').count()
    # take above table and group by days_since_prior_order

    df_mean_order_size = orders_with_count.groupby(by='days_since_prior_order').mean()['product_id']
    df_mean_order_renamed = df_mean_order_size.rename('average_order_size')

    bubble_plot_dataframe = pd.concat([order_by_date['order_id'], df_mean_order_renamed], axis=1)

    bubble_plot_dataframe['average_order_size'].index.to_numpy()

    fig = plt.figure(figsize=[15,7.5])
    ax = fig.add_subplot()
    plt.scatter(bubble_plot_dataframe['average_order_size'].index.to_numpy(), bubble_plot_dataframe['order_id'].values, s=((bubble_plot_dataframe['average_order_size'].values/bubble_plot_dataframe['average_order_size'].values.mean())*10)**3.1, alpha=0.5, c = '0.5')

    plt.xticks(np.arange(0, 31, 1.0));
    ax.xaxis.grid(True)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.set_xlabel('Days since previous order', fontsize=16)
    ax.set_ylabel('Number of orders / 1000', fontsize=16)
    ax.get_xaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x))))
    ax.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x/1000))))
    my_yticks = ax.get_yticks()
    plt.yticks([my_yticks[-2], my_yticks[0]], visible=True);

    fig = plt.figure(figsize=[10,9])
    ax = fig.add_subplot()
    plt.scatter(bubble_plot_dataframe['average_order_size'].index.to_numpy()[:8], bubble_plot_dataframe['order_id'].values[:8], s=((bubble_plot_dataframe['average_order_size'].values[:8]/bubble_plot_dataframe['average_order_size'].values.mean())*10)**3.1, alpha=0.5, c = '0.5')

    plt.xticks(np.arange(0, 8, 1.0));
    ax.xaxis.grid(True)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.set_xlabel('Days since previous order', fontsize=16)
    ax.set_ylabel('Number of orders / 1000', fontsize=16)
    ax.get_xaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x))))
    ax.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x/1000))))
    my_yticks = ax.get_yticks()
    plt.yticks([my_yticks[-2], my_yticks[0]], visible=True);

def orderTimeHeatMaps(path_to_dataset):
    """
    Plots the distribution of order with respect to hour of day and day of the week.
    :param path_to_dataset: this path should have all the .csv files for the dataset
    :type path_to_dataset: str
    """

    assert isinstance(path_to_dataset, str)

    import pandas as pd
    import matplotlib.pyplot as plt
    import seaborn as sns
    import numpy as np

    order_file_path = path_to_dataset + '/orders.csv'

    orders = pd.read_csv(order_file_path)

    grouped_data = orders.groupby(["order_dow", "order_hour_of_day"])["order_number"].aggregate("count").reset_index()
    grouped_data = grouped_data.pivot('order_dow', 'order_hour_of_day', 'order_number')

    grouped_data.index = pd.CategoricalIndex(grouped_data.index, categories=[0,1,2,3,4,5,6])
    grouped_data.sort_index(level=0, inplace=True)

    plt.figure(figsize=(12,6))
    hour_of_day = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14','15','16', '17', '18', '19','20', '21', '22', '23']
    dow = [ 'SUN', 'MON', 'TUES', 'WED', 'THUR','FRI','SAT']

    ax = sns.heatmap(grouped_data, xticklabels=hour_of_day,yticklabels=dow,cbar_kws={'label': 'Number Of Orders Made/1000'})
    cbar = ax.collections[0].colorbar
    cbar.set_ticks([0, 10000, 20000, 30000, 40000, 50000])
    cbar.set_ticklabels(['0','10.0','20.0','30.0','40.0','50.0'])
    ax.figure.axes[-1].yaxis.label.set_size(15)
    ax.figure.axes[0].yaxis.label.set_size(15)
    ax.figure.axes[0].xaxis.label.set_size(15)

    ax.set(xlabel='Hour of Day', ylabel= "Day of the Week")
    ax.set_title("Number of orders made by Day of the Week vs Hour of Day", fontsize=15)
    plt.show()

    grouped_data = orders.groupby(["order_dow", "order_hour_of_day"])["order_number"].aggregate("count").reset_index()
    grouped_data = grouped_data.pivot('order_dow', 'order_hour_of_day', 'order_number')

    grouped_data.index = pd.CategoricalIndex(grouped_data.index, categories=[0,1,2,3,4,5,6])
    grouped_data.sort_index(level=0, inplace=True)

    plt.figure(figsize=(12,6))
    hour_of_day = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14','15','16', '17', '18', '19','20', '21', '22', '23']
    dow = [ 'SUN', 'MON', 'TUES', 'WED', 'THUR','FRI','SAT']

    ax = sns.heatmap(np.log(grouped_data), xticklabels=hour_of_day,yticklabels=dow,cbar=False)
    cbar = ax.collections[0].colorbar

    ax.figure.axes[-1].yaxis.label.set_size(15)
    ax.figure.axes[0].yaxis.label.set_size(15)
    ax.figure.axes[0].xaxis.label.set_size(15)

    ax.set(xlabel='Hour of Day', ylabel= "Day of the Week")
    ax.set_title("Number of orders made by Day of the Week vs Hour of Day (Log Scale)", fontsize=15)
    plt.show()

def generateWordCloud(path_to_dataset):
    """
    Generates word cloud.
    :param path_to_dataset: path to dataset
    :type path_to_dataset: str
    """

    assert isinstance(path_to_dataset, str)

    from wordcloud import WordCloud
    import pandas as pd
    import matplotlib.pyplot as plt

    product_path = path_to_dataset + "/products.csv"
    aisles_path = path_to_dataset + "/aisles.csv"
    departments_path = path_to_dataset + "/departments.csv"
    order_product_prior_path = path_to_dataset + "/order_products__prior.csv"

    df_products = pd.read_csv(product_path)
    df_aisles = pd.read_csv(aisles_path)
    df_departments = pd.read_csv(departments_path)
    df_order_products_prior = pd.read_csv(order_product_prior_path)

    # Merge Prior orders, Product, Aisle and Department
    df_order_products_prior_merged = pd.merge(
                                        pd.merge(pd.merge(df_order_products_prior, df_products, on="product_id", how="left"),
                                            df_aisles,
                                            on="aisle_id",
                                            how="left"),
                                        df_departments,
                                        on="department_id",
                                        how="left")
    # Top N products by frequency
    top_products = df_order_products_prior_merged["product_name"].value_counts()

    d = top_products.to_dict()

    wordcloud = WordCloud(background_color='white')
    wordcloud.generate_from_frequencies(frequencies=d)
    plt.figure(figsize = (8,8))
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis("off")
    plt.show()

def no_of_orders(path_to_data = './instacart-market-basket-analysis'):
    """
    pass path to orders.csv
    """

    bins = 10
    path = path_to_data + '/orders.csv'
    import numpy as np # linear algebra
    import matplotlib.pyplot as plt
    import seaborn as sns
    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt
    import matplotlib.mlab as mlab
    import seaborn as sns
    from scipy.optimize import curve_fit
    from IPython.display import display, HTML

    orders = pd.read_csv(path)
    sns.set_style('dark')
    customer_no = orders.groupby("user_id", as_index = False)["order_number"].max()

    n, bins, patches = plt.hist(customer_no["order_number"] , bins, color='blue', alpha=0.5)

    plt.xlabel("No. of Orders")
    plt.ylabel("Count")
    plt.title("Number of Orders per Customer")

def freq_product(path1 = "./instacart-market-basket-analysis/order_products__train.csv",path2 = "./instacart-market-basket-analysis/order_products__prior.csv" , path3 = "./instacart-market-basket-analysis/products.csv"):
    import numpy as np # linear algebra
    import matplotlib.pyplot as plt
    import seaborn as sns
    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt
    import matplotlib.mlab as mlab
    import seaborn as sns
    from scipy.optimize import curve_fit
    from IPython.display import display, HTML
    order_products_train = pd.read_csv(path1)
    order_products_prior = pd.read_csv(path2)
    products = pd.read_csv(path3)
    t_p = order_products_train.append(order_products_prior,ignore_index = True)
    prod = t_p.groupby("product_id",as_index = False)["order_id"].count()

    top = 20
    product_Count = prod.sort_values("order_id",ascending = False)
    df1 = product_Count.iloc[0:top,:]
    df1 = df1.merge(products,on = "product_id")
    display(df1.loc[:,["product_name"]])

def dept_prod(m = "./instacart-market-basket-analysis/products.csv" , n = "./instacart-market-basket-analysis/departments.csv" , p = "./instacart-market-basket-analysis/aisles.csv"):
    import numpy as np # linear algebra
    import matplotlib.pyplot as plt
    import seaborn as sns
    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt
    import matplotlib.mlab as mlab
    import seaborn as sns
    from scipy.optimize import curve_fit
    from IPython.display import display, HTML
    products = pd.read_csv(m)
    departments = pd.read_csv(n)
    aisles = pd.read_csv(p)
    x = pd.merge(left=products, right=departments, how='left')
    lists = pd.merge(left = x, right=aisles, how='left')

    group_list = lists.groupby("department")["product_id"].aggregate({'Total_products': 'count'})
    final = group_list.reset_index()
    final.sort_values(by='Total_products', ascending=False, inplace=True)

    sns.set_style('white')
    ax = sns.barplot(x="Total_products", y="department", data=final,color = 'gray' )

    r = ax.spines["right"].set_visible(False)
    t = ax.spines["top"].set_visible(False)

def dept_reorder(path = "order_products__prior.csv" , m = "products.csv" , n = "departments.csv" , p = "aisles.csv"):
    import numpy as np # linear algebra
    import matplotlib.pyplot as plt
    import seaborn as sns
    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt
    import matplotlib.mlab as mlab
    import seaborn as sns
    from scipy.optimize import curve_fit
    from IPython.display import display, HTML
    products = pd.read_csv(m)
    departments = pd.read_csv(n)
    aisles = pd.read_csv(p)

    order_products_prior = pd.read_csv(path)

    order_products_prior = pd.merge(order_products_prior, products, on='product_id', how='left')
    order_products_prior = pd.merge(order_products_prior, aisles, on='aisle_id', how='left')
    order_products_prior = pd.merge(order_products_prior, departments, on='department_id', how='left')

    df2 = order_products_prior.groupby(["department"])["reordered"].aggregate("mean").reset_index()

    plt.figure(figsize=(12,8))
    sns.set_style('white')

    ax1 = sns.scatterplot(df2['reordered'].values,df2['department'].values , color = 'gray')
    plt.ylabel('Department', fontsize=15)
    plt.xlabel('Reorder Ratio' , fontsize=15)
    plt.title("Department wise reorder ratio", fontsize=15)
    plt.xticks(rotation='horizontal')
    r = ax1.spines["right"].set_visible(False)
    t = ax1.spines["top"].set_visible(False)
    plt.show()


import datetime
from django.db import models
from django.utils import timezone  # 예약이 in_progress인지 아닌지 할때 이거 있어야함.
from core import models as core_models

# Create your models here.

class BookedDay(core_models.TimeStampedModel):

    day = models.DateField()
    reservation = models.ForeignKey("Reservation", on_delete=models.CASCADE)

    class Meta:
        verbose_name = "Booked Day"
        verbose_name_plural = "Booked Days"

    def __str__(self):
        return str(self.day)

class Reservation(core_models.TimeStampedModel):

    """Reservation Model Definition """

    STATUS_PENDING = "pending"
    STATUS_CONFIRMED = "confirmed"
    STATUS_CANCELED = "canceled"

    STATUS_CHOICES = (
        (STATUS_PENDING, "pending"),
        (STATUS_CONFIRMED, "confirmed"),
        (STATUS_CANCELED, "canceled"),
    )

    status = models.CharField(
        max_length=12, choices=STATUS_CHOICES, default=STATUS_PENDING
    )
    check_in = models.DateField()
    check_out = models.DateField()
    guest = models.ForeignKey(
        "users.User", related_name="reservation", on_delete=models.CASCADE
    )
    room = models.ForeignKey(
        "rooms.Room", related_name="reservation", on_delete=models.CASCADE
    )

    def __str__(self):
        return f"{self.room} - {self.check_in}"

    def in_progress(self):
        now = timezone.now().date()  # import 해야함.
        return now >= self.check_in and now <= self.check_out

    in_progress.boolean = True  # 이렇게 하면 true, false 이모티콘으로 바꿔줌.

    def is_finished(self):
        now = timezone.now().date()
        is_finished = now > self.check_out
        if is_finished:
            BookedDay.objects.filter(reservation=self).delete()
        return is_finished

    is_finished.boolean = True

    def save(self, *args, **kwargs):
        if self.pk is None:  # 우리가 만든 model이 new라는 뜻
            start = self.check_in
            end = self.check_out
            difference = end - start
            existing_booked_day = BookedDay.objects.filter(
                day__range=(start, end)
            ).exists()  # 사이에 예약이 있는지 확인

            if not existing_booked_day:
                super().save(*args, **kwargs)
                for i in range(difference.days + 1):
                    day = start + datetime.timedelta(days=i)
                    BookedDay.objects.create(day=day, reservation=self)  # bookedday 생성
                return

        return super().save(*args, **kwargs)


import subprocess
import re
import sys

path_to_rp = "/bin/rp-lin-x64"

mov_dict = {}
pop_list = []
ans_dict = {}
push_dict = {}
pop_dict = {}

def rop_load(filename):
    p = subprocess.run([path_to_rp, "-f", filename, "-r", "10", "--unique"],
                       stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    data = p.stdout.decode("utf-8").split("\n")
    return data

def load_plt_section(filename):
    p = subprocess.run(["/usr/bin/objdump", "-M", "intel", "-D", filename],
                       stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    data = p.stdout.decode("utf-8").split("\n")
    return data

def operand_parser(string):
    return string.replace(";", "").strip().split(",")

def create_movdict(data_list):
    for data in data_list:
        #addressing = re.findall(r"[0-9A-Fa-f]{8}.+mov [^;]+;.*ret", data)
        addressing = re.findall(r"[0-9A-Fa-f]{8}.+mov [^;]+;.*$", data)
        if addressing == []:
            continue
        address = re.findall(r"[0-9A-Fa-f]{8}", str(addressing))[0]
        operands = re.findall(r"(?<=mov )[^;]+;", data)
        for ope in operands:
            ope1, ope2 = operand_parser(ope)
            mov_dict[ope2.strip()] = (
                addressing[0], ope1.strip(), ope2.strip(), address)

def create_poplist(data_list):
    for data in data_list:
        addressing = re.findall(r"[0-9A-Fa-f]{8}.+pop [^;]+;.*$", data)
        if addressing == []:
            continue
        if("rsp" in data or "esp" in data):
             continue
        address = re.findall(r"[0-9A-Fa-f]{8}", addressing[0])[0]
        operands = re.findall(r"(?<=pop )[^;]+;", data)
        popnum = len(operands)
        for i, operand in enumerate(operands):
            ope = operand_parser(operand)[0].strip()
            pop_list.append((addressing[0], ope, popnum, i, address))
            if(ope in pop_dict):
                pop_dict[ope] += [(addressing[0], ope, i, address, popnum)]
            else:
                pop_dict[ope] = [(addressing[0], ope, i, address, popnum)]

def print_can_pop_to_mov():
    for pop in pop_list:
        if(pop[1] in mov_dict):
            mes = "\033[93m " + str(pop[2]) + "." + str(pop[3] + 1) + "X -> " + str((pop[1]) + " -> " + str(
                mov_dict[pop[1]][1]) + "\033[0m : " + "\033[91m0x" + pop[0]) + "\n" + " "*18 + "->  \033[91m0x" + str(mov_dict[pop[1]][0])
            if(mov_dict[pop[1]][3] in ans_dict):
                ans_dict[mov_dict[pop[1]][3]
                         ] += [(mes, pop[2], pop[3], pop[4])]
            else:
                ans_dict[mov_dict[pop[1]][3]] = [(mes, pop[2], pop[3], pop[4])]
    print("\033[92m-------------------------------------\n")
    for address in ans_dict.keys():
        for mes in list(set(ans_dict[address])):
            print(mes[0])
            print("[*]suggestion\n")
            buf = str("buf += p64(0x" + mes[3])
            buf += ")\n"
            popnum = mes[1]
            offset = mes[2]
            for i in range(popnum):
                if i == offset:
                    buf += str("buf += p64(value")
                    buf += ")\n"
                else:
                    buf += str("buf += p64(dummy")
                    buf += ")\n"
            buf += "buf += p64(0x" + address + ")"
            print(buf)
            print("\033[91m")
        if("call" in mes[0]):
            called = re.search(
                r"(?<=call )[^;]+;", mes[0]).group(0).replace(";", "")
            print("[+]to call next rop chain : " +
                  called + " = " + "pop x ret")
            registers = re.findall(r"r[0-9a-z]+", called.replace("word", ""))
            print("so you can use these gadget")
            for reg in registers:
                try:
                    for pop in pop_dict[reg]:
                        print(" "*30+"0x"+pop[0])
                        print("[*]suggestion")
                        print("\033[94m")
                        buf = str("buf += p64(0x" + pop[3])
                        buf += ")\n"
                        popnum = pop[4]
                        offset = pop[2]
                        for i in range(popnum):
                            if i == offset:
                                buf += str("buf += p64(&pop1_ret")
                                buf += ")\n"
                            else:
                                buf += str("buf += p64(dummy")
                                buf += ")\n"
                        buf += "your_gadget"
                        print(buf)
                        print("\033[91m")
                except:
                    print("I have no idea")
                    pass
        print("")
        print("\033[92m-------------------------------------")

def print_can_push_to_pop():
    for push in list(push_dict.keys()):
        print("push {} -> X".format(push))

def genarate_rop_code(code ,popnum, offset,address, operand):
        val = "_".join([i.replace(";", "").strip().replace(" ", "_") for i in re.findall(r"pop [^;]+;", code)])
        print(" \033[93m" + val + " = 0x" + address)
        print("\033[0m ")
        buf = str("buf += p64(" + val)
        buf += ")\n"
        for i in range(popnum):
            if i == offset:
                buf +=str("buf += p64(value_"+ operand)
                buf += ")\n"
            else:
                buf +=str("buf += p64(dummy")
                buf += ")\n"
        print("\n")
        return buf

argc = len(sys.argv)
if(argc <= 1):
    print("invalid argument")
    sys.exit()
filename = str(sys.argv[1])
data_list = rop_load(filename)
create_movdict(data_list)
create_poplist(data_list)
print("[+]poplist")
for pop in pop_list:                #pop code[0], ope, popnum, i, address
    print(pop[0])
    print("")
    buf = genarate_rop_code(pop[0],pop[2],pop[3],pop[4],pop[1])
print("")
print("pop chain")
print_can_pop_to_mov()
# for pop in pop_list:
#    print(pop)


'''
Start with two arrays of strings, A and B, each with its elements in alphabetical order and without duplicates. Return a new array containing the first N elements from the two arrays. The result array should be in alphabetical order and without duplicates. A and B will both have a length which is N or more. The best "linear" solution makes a single pass over A and B, taking advantage of the fact that they are in alphabetical order, copying elements directly to the new array.

mergeTwo(["a", "c", "z"], ["b", "f", "z"], 3) ---> ["a", "b", "c"]
mergeTwo(["a", "c", "z"], ["c", "f", "z"], 3) ---> ["a", "c", "f"]
mergeTwo(["f", "g", "z"], ["c", "f", "g"], 3) ---> ["c", "f", "g"]
'''
from collections import Counter

def mergeTwo(PuffDaddy01, PuffDaddy02, N):
    unique01counter = Counter(PuffDaddy01)
    unique01 = []
    for key in unique01counter:
        unique01.append(key)
    unique01 = sorted(unique01)
    unique02counter = Counter(PuffDaddy02)
    unique02 = []
    for key in unique02counter:
        unique02.append(key)
    unique02 = sorted(unique02)
    ParisJackson = []
    BlanketJackson = []
    for i in range(0, N-1, 1):
        ParisJackson.append(unique01[i])
    for i in range(0, N-1, 1):
        if unique02[i] in ParisJackson:
            pass
        else:
            ParisJackson.append(unique02[i])
    ParisJackson = sorted(ParisJackson)
    for i in range(0, N, 1):
        BlanketJackson.append(ParisJackson[i])
    print('{}, {}, {} {} {}'.format(unique01, unique02, N, ' --> ', BlanketJackson))

mergeTwo(["a", "c", "z"], ["b", "f", "z"], 3)
mergeTwo(["a", "c", "z"], ["c", "f", "z"], 3)
mergeTwo(["f", "g", "z"], ["c", "f", "g"], 3)
print(72*'-')
mergeTwo(['j','a','y','y','o','u','n','g','b','l','o','o','d'],['z','e','e','m','a','x'],5)


from datetime import datetime
from zipfile import ZipFile
from pymongo import MongoClient
import os
import logging
import pytz

from etl.extract.feed_service import fetch_geopackage_feed
from etl.extract.feed_service import download_zip
from etl.load.mongodb.mongo_helpers import initiliaze_source_document

log = logging.getLogger(__name__)

def unzip(path, to_path, to_filename):
    with ZipFile(path, allowZip64=True) as zf:
        zf.extract(to_filename, to_path)

def process_geopackage(reg_type):
    doc_type = reg_type.type_name
    log.info("Collecting for type: '%s'" % doc_type)

    try:
        mongoUrl = os.environ.get('mongodbUrl')
        client = MongoClient(mongoUrl)
        db = client.bro
    except Exception as e:
        log.error(str(e))
        return

    # Get geopackage ATOM feed based on source URL
    try:
        source_url = os.environ.get(reg_type.sourceKey)
        feed = fetch_geopackage_feed(source_url)
        db.source.update_one({"type": doc_type}, {
                             "$set": {"last_extracted": datetime.now()}}, upsert=True)
    except Exception as e:
        log.error(str(e))
        db.source.update_one({"type": doc_type}, {
                             "$set": {"status": "Failed extracting xml"}}, upsert=True)
        return

    # Download and process ZIP from source
    try:
        source = db.source.find_one({"type": doc_type})
        log.info("Datum in feed: %s en laatst update datum: %s." %
                 (feed.updated, source['updated']))

        download_data = feed.updated > pytz.utc.localize(source['updated'])

        if download_data:

            download_location = os.environ.get('downloadLocation', '/mnt/brodata/')
            path = download_location + feed.filename

            log.info("Start download van %s naar %s" % (feed.zip_url, path))
            if download_zip(feed.zip_url, path):
                unzip_filename = feed.filename.replace('zip', 'gpkg')
                log.info("Unzipping %s" % path)
                if doc_type == "bhr-gt":
                    unzip_filename = "brobhrgtvolledigeset.gpkg"
                unzip_location = os.environ.get('unzipLocation', '/mnt/brodata/')
                unzip(path, unzip_location, unzip_filename)

                db.source.update_one({"type": doc_type},
                                     {"$set": {"updated": feed.updated,
                                               "status": "OK", "zip_filename": feed.filename}},
                                     upsert=True)
                log.info("Extracted geopackage for type: %s" % doc_type)
    except Exception as e:
        log.error(str(e))
        db.source.update_one({"type": doc_type}, {
                             "$set": {"status": "Failed extracting gpkg"}}, upsert=True)
        return


from flask import jsonify
from flask.views import MethodView
from flask_jwt_extended import jwt_required
from flask_smorest import Blueprint
from flask_smorest import abort
from sqlalchemy.exc import SQLAlchemyError
from stores.extensions.database import ProductModel
from stores.extensions.database import StoreModel
from stores.extensions.database import TagModel
from stores.extensions.database import db
from stores.extensions.schemas import TagSchema

blp = Blueprint("tag", __name__, description="Department tag operations")

@blp.route("/tag/<int:tag_id>")
class Tags(MethodView):
    """Tags operations"""

    @blp.response(200, TagSchema)
    def get(self, tag_id):
        """Get tag by ID"""
        tag = TagModel.query.get_or_404(tag_id, description="Tag id not found")
        return tag

    # @blp.arguments(TagUpdateSchema)
    @blp.response(200, TagSchema)
    def put(self):
        """Update tag by ID"""
        raise NotImplementedError

    @jwt_required(fresh=True)
    @blp.response(204, description="Deletes the tag if not linked to a product")
    @blp.alt_response(400, description="Abort if the tag is associated to a product")
    def delete(self, tag_id):
        """Delete tag by ID"""
        tag = TagModel.query.get_or_404(tag_id, description="Tag id not found")

        if not tag.products:
            db.session.delete(tag)
            db.session.commit()
        else:
            abort(400, message="The given tag is associated to a product and could not be deleted")

@blp.route("/stores/<string:store_id>/tag")
class StoreTags(MethodView):
    """Tags operations based on stores"""

    @blp.response(200, TagSchema(many=True))
    def get(self, store_id):
        """Get all department tags from the given store"""
        store = StoreModel.query.get_or_404(store_id, description="Store id not found")
        return store.tags

    @jwt_required()
    @blp.arguments(TagSchema)
    @blp.response(201, TagSchema)
    def post(self, data, store_id):
        """Create a tag in a store"""
        if TagModel.query.filter(
            TagModel.store_id == store_id, TagModel.name == data["name"]
        ).first():
            abort(400, message="Tag name already exists in that store")

        tag = TagModel(**data, store_id=store_id)

        try:
            db.session.add(tag)
            db.session.commit()
        except SQLAlchemyError as error:
            abort(500, message=f"Error {error} while inserting tag to Store {id}")
        return tag

@blp.route("/products/<int:product_id>/tag/<int:tag_id>")
class ProductTags(MethodView):
    """Products and tags link operations"""

    @jwt_required()
    @blp.arguments(TagSchema)
    @blp.response(201, TagSchema)
    def post(self, product_id, tag_id):
        """Link a department tag to a product"""
        product = ProductModel.query.get_or_404(product_id, description="Product id not found")
        tag = TagModel.query.get_or_404(tag_id, description="Tag id not found")

        product.tags.append(tag)

        try:
            db.session.add(product)
            db.session.commit()
        except SQLAlchemyError as error:
            abort(
                500, message=f"Error {error} while linking Tag {tag_id} and Product {product_id}"
            )

        return tag

    @jwt_required()
    @blp.response(200)
    def delete(self, product_id, tag_id):
        """Unlink a tag from a product"""
        product = ProductModel.query.get_or_404(product_id, description="Product id not found")
        tag = TagModel.query.get_or_404(tag_id, description="Tag id not found")

        product.tags.remove(tag)

        try:
            db.session.add(product)
            db.session.commit()
        except SQLAlchemyError as error:
            abort(
                500, message=f"Error {error} while linking Tag {tag_id} and Product {product_id}"
            )

        return jsonify(message="Tag and product were unlinked", product=product, tag=tag)


import sys
import os
sys.path.append(os.getcwd())

import matplotlib.pyplot as plt
from datetime import datetime
from src.models.utils.logger import Logger

trainings = [
	("bc", "BC"),
	("pg", "PG"),
	("ppo", "PPO"),
	("ppo_lstm", "PPO w/ LSTM"),
	("ppo_attention", "PPO w/ AttentionNet"),
]

print("Alg", "Epochs", "Time", "Avg. Lifespan", "Max Lifespan")
for training in trainings:
	log_name = training[0]
	log_title = training[1]

	logs_path = os.path.join('logs', log_name, log_name + '0.log')
	if not os.path.exists(logs_path):
		print("Log file '{0}' not found.".format(logs_path))
		continue
	training_logs = Logger.read(logs_path)

	training_time = datetime.fromtimestamp(training_logs[-1]['timestamp']) - datetime.fromtimestamp(training_logs[0]['timestamp'])
	print(log_title, len(training_logs), training_time, training_logs[-1]['custom_metrics']['avg_lifespan_mean'], training_logs[-1]['custom_metrics']['max_lifespan_mean'])

	avg_lifespans = [training_log['custom_metrics']['avg_lifespan_mean'] for training_log in training_logs]
	max_lifespans = [training_log['custom_metrics']['max_lifespan_mean'] for training_log in training_logs]
	epochs = range(1,len(training_logs)+1)
	fig = plt.figure(dpi=1200)
	plt.plot(epochs, avg_lifespans, 'g', label='Avg. Lifespans')
	plt.plot(epochs, max_lifespans, 'b', label='Max Lifespans')
	plt.title(log_title + " Lifespans over Training")
	plt.xlabel('Epochs')
	plt.ylabel('Agent Iters Alive')
	plt.legend()
	# plt.show()
	plt.savefig(log_name + "_fig.png")
	plt.close(fig)


import os
import numpy as np
import matplotlib.pyplot as plt
from delaunay import delaunay_triangulation
import obj as OBJ
import corner_table

PATH = "./OBJ/"

def plot_tri(ax1, faces, vertex):
    verts = []
    idxs = []
    for tri in faces:
        tri_idxs = []
        for v in tri:
            v = tuple(v)
            if v not in verts:
                verts.append(v)
            tri_idxs.append(verts.index(v))
        idxs.append(tri_idxs)

    verts = np.asarray(verts)
    ax1.clear()
    ax1.scatter(vertex[:,0], vertex[:,1], color='r')
    ax1.triplot(verts[:,0], verts[:,1], triangles = idxs, color='k')
    plt.pause(0.05)

def main ():
    objs = OBJ.read_OBJ(PATH)
    fig1, ax1 = plt.subplots()
    ax1.set_aspect('equal')
    ax1.set_title('triplot of Delaunay triangulation')
    for obj in objs:
        corners = corner_table.build_corner_table(obj)

        delaunay = delaunay_triangulation(corners)

if __name__ == '__main__':
    main()


import random

coinList = []
numOfH = 0
numOfT = 0

numOfStreaks = 0

for x in range(10000):
    if (random.randint(0, 1) == 0):
        coinList.append("H")
    else:
        coinList.append("T")

for items in coinList:
    if (numOfH == 6 or numOfT == 6):
        numOfStreaks += 1
        numOfH = 0
        numOfT = 0

    if items == "H":
        numOfH += 1
        numOfT = 0
    else:
        numOfT += 1
        numOfH = 0

print(numOfStreaks)
# print(coinList)


"""Test optimization results."""
from pathlib import Path

import pytest

from sbmlsim.examples.experiments.midazolam.fitting_problems import op_mid1oh_iv
from sbmlsim.fit.analysis import OptimizationResult
from sbmlsim.fit.options import (
    OptimizationAlgorithmType,
    ResidualType,
    WeightingCurvesType,
    WeightingPointsType,
)
from sbmlsim.fit.runner import run_optimization

fit_kwargs_default = {
    "residual": ResidualType.ABSOLUTE,
    "weighting_curves": [WeightingCurvesType.POINTS],
    "weighting_points": WeightingPointsType.ERROR_WEIGHTING,
    "absolute_tolerance": 1e-6,
    "relative_tolerance": 1e-6,
}

@pytest.mark.skip(reason="no fit support")
def test_serialization(tmp_path: Path) -> None:
    """Test serialization of optimization result."""
    opt_res: OptimizationResult = run_optimization(
        problem=op_mid1oh_iv(),
        algorithm=OptimizationAlgorithmType.LEAST_SQUARE,
        size=1,
        n_cores=1,
        serial=True,
        **fit_kwargs_default
    )

    opt_res_path = tmp_path / "opt_res.json"
    opt_res.to_json(path=opt_res_path)
    opt_res2 = OptimizationResult.from_json(json_info=opt_res_path)

    assert opt_res.sid == opt_res2.sid
    assert [p.pid for p in opt_res.parameters] == [p.pid for p in opt_res2.parameters]

@pytest.mark.skip(reason="no fit support")
def test_combine(tmp_path: Path) -> None:
    """Test combination of optimization result."""
    opt_results = []
    for seed in [1234, 4567]:
        opt_res: OptimizationResult = run_optimization(
            problem=op_mid1oh_iv(),
            algorithm=OptimizationAlgorithmType.LEAST_SQUARE,
            size=1,
            n_cores=1,
            serial=True,
            seed=seed,
            **fit_kwargs_default
        )
        opt_results.append(opt_res)

    opt_result = OptimizationResult.combine(opt_results)
    assert len(opt_result.fits) == len(opt_results[0].fits) + len(opt_results[1].fits)


import cv2
import numpy as np
import os

def random_centers(K, dim):
    res = []
    for k in range(0,K):
        vec=[]
        for i in range(0,dim):
            vec.append(np.random.uniform(-1, 1))
        res.append(vec)
    return(res)

def cluster_points(X, mu):
    clusters  = {}
    for i in range(0, len(mu)):
        clusters[i]=[]
    for x in X:
        bestmukey = min([(i[0], np.linalg.norm(x[1]-mu[i[0]])) \
                    for i in enumerate(mu)], key=lambda t:t[1])[0]
        try:
            clusters[bestmukey].append(x)
        except KeyError:
            clusters[bestmukey] = [x]
    return clusters

def reevaluate_centers(mu, clusters):
    newmu = []
    keys = sorted(clusters.keys())
    for k in keys:
        newmu.append(vec_mean(clusters[k]))
    return newmu

def vec_mean(cluster, dim = 64):
    try:
        sum=[]
        for vector in cluster:
            if(len(sum)==0):
                sum = vector[1]
            else:
                for i in range(0, dim):
                    sum[i]=sum[i]+ vector[1][i]
        for i in range(0, dim):
            sum[i] = sum[i]/len(cluster)
    except IndexError:
        for i in range(0, dim):
            sum.append(0)
    return(sum)

def has_converged(mu, oldmu):
    return (set([tuple(a) for a in mu]) == set([tuple(a) for a in oldmu]))

def find_centers(X, K):
    # Initialize to K random centers
    dim = 64
    oldmu = random_centers(K, dim)
    mu = random_centers(K, dim)
    while not has_converged(mu, oldmu):
        oldmu = mu
        # Assign all points in X to clusters
        clusters = cluster_points(X, mu)
        # Reevaluate centers
        mu = reevaluate_centers(oldmu, clusters)
    return(mu, clusters)

trainingset=[]
direction = "/Users/Thomartin/mopsi/images/tour_eiffel"
surf = cv2.xfeatures2d.SURF_create()
trainingset = []
for file in os.listdir(direction):
    img = cv2.imread(direction+"/"+file)
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    (kps,descs) = surf.detectAndCompute(gray, None)
    for j in range(0,len(descs)):
        trainingset.append((kps[j],descs[j],direction+"/"+file))

print(type(trainingset[4][1].tolist()[1]))
cen = random_centers(4, 64)
clus = cluster_points(trainingset, cen)
# print(reevaluate_centers(cen, clus ))
# print(cen)
# print(clus.keys())
set


from importlib.util import set_loader

class BangBam:
    def __init__(self, kich_thuoc = 10):
        self.danh_sach = [None for _ in range(kich_thuoc)]
    #def

    def __str__(self):
        kq = '['
        stt1 = 0
        for x in self.danh_sach:
            stt1 += 1
            if stt1 != 1:
                kq = kq + ', '
            #if
            if x is None:
                kq = kq + '[None]'
            else:
                kq = kq + '['
                stt2 = 0
                for e in x:
                    stt2 += 1
                    if stt2 != 1:
                        kq = kq + ', '
                    #if
                    kq = kq + str(e[0])+ ': ' + str(e[1])
                #for
                kq = kq + ']'
            #if
        kq = kq + ']'
        return kq
    #def

    def bam(self, khoa):
        kich_thuoc = len(self.danh_sach)
        return hash(khoa) % kich_thuoc
    #def

    def them(self, khoa, gia_tri):
        chi_muc = self.bam(khoa)
        if self.danh_sach[chi_muc] is None:
            #them moi
            self.danh_sach[chi_muc] = list()
            self.danh_sach[chi_muc].append([khoa, gia_tri])
        else:
            #cap nhat
            cap_nhat = False
            for x in self.danh_sach[chi_muc]:
                if x[0] == khoa:
                    x[1] = gia_tri
                    cap_nhat = True
                    break
                #if
            #for
            if cap_nhat == False:
                self.danh_sach[chi_muc].append([khoa, gia_tri])
            #if
        #if
    #def

    def lay(self, khoa):
        chi_muc = self.bam(khoa)
        if self.danh_sach[chi_muc] is None:
            return None
        else:
            for x in self.danh_sach[chi_muc]:
                if x[0] == khoa:
                    return x[1]
                #if
            #for
        #if
    #def

    def __setitem__(self, khoa,gia_tri):
        self.them(khoa, gia_tri)
    #def

    def __getitem__(self, khoa):
        return self.lay(khoa)
    #def

#class
def main():
    bang_bam = BangBam(5)
    import random
    for _ in range(18):
        khoa = random.randint(0,10)
        gia_tri = random.randint(0, 100)
        print(f'* Them khoa = {khoa}, gia tri = {gia_tri}')
        #bang_bam.them(khoa, gia_tri)
        bang_bam[khoa] = gia_tri
        print(bang_bam)
        print()
    #for

    khoa = int(input('Nhap vao mot khoa: '))
    #gia_tri = bang_bam.lay(khoa)
    gia_tri = bang_bam[khoa]
    print(f'*Khoa {khoa} co gia tri la {gia_tri}')
#def

if __name__ == '__main__':
    main()
#if


import os
import requests
import json
from .exception import *

class HSRequest(object):
    ''' Object to handle HTTP requests

    Although we have greate requests package which can handle the HTTP request
    beautifully, we need this class to fit better our need like sending the
    requests with authentication information, download files, check HTTP
    errors...

    Attributes:
        DEFAULT_ENCODING (str): Default encoding for requests
        USER_AGENT (str): HTTP User agent used when sending requests
        parameters (dict): Some parameters for GET requests
        headers (dict): Custome headers for every requests
        http_status_code (int): HTTP status code returned of requests

    '''

    DEFAULT_ENCODING = "UTF-8"
    USER_AGENT = "HelloSign Python SDK"

    parameters = None
    headers = { 'User-Agent': USER_AGENT }
    http_status_code = 0
    verify_ssl = True

    def __init__(self, auth, env="production"):
        self.auth = auth
        self.env = env
        self.debug = (self.env != 'production')
        self.verify_ssl = (not self.debug)

    def _get_json_response(self, resp):
        ''' Parse a JSON response '''
        if resp is not None and resp.text is not None:
            return json.loads(resp.text)

    def get(self, url, headers=None, parameters=None, get_json=True):
        ''' Send a GET request with custome headers and parameters

        Args:
            url (str): URL to send the request to
            headers (str, optional): custom headers
            parameters (str, optional): optional parameters

        Returns:
            A JSON object of the returned response if `get_json` is True,
            Requests' response object otherwise

        '''

        if self.debug:
            print("GET: %s, headers=%s" % (url, headers))

        get_headers = self.headers
        get_parameters = self.parameters
        if get_parameters is None:
            # In case self.parameters is still empty
            get_parameters = {}
        if headers is not None:
            get_headers.update(headers)
        if parameters is not None:
            get_parameters.update(parameters)

        response = requests.get(url, headers=get_headers, params=get_parameters, auth=self.auth, verify=self.verify_ssl)
        self.http_status_code = response.status_code
        self._check_error(response)
        if get_json is True:
            return self._get_json_response(response)
        return response

    def get_file(self, url, filename, headers=None):
        ''' Get a file from a url and save it as `filename`

        Args:
            url (str): URL to send the request to

            filename (str): File name to save the file as, this can be either
                a full path or a relative path

            headers (str, optional): custom headers

        Returns:
            True if file is downloaded and written successfully, False
            otherwise.

        '''

        if self.debug:
            print("GET FILE: %s, headers=%s" % (url, headers))

        get_headers = self.headers
        if headers is not None:
            get_headers.update(headers)

        response = requests.get(url, headers=get_headers, auth=self.auth, verify=self.verify_ssl)

        self.http_status_code = response.status_code
        try:
            self._check_error(response)
            fd = os.open(filename, os.O_CREAT | os.O_RDWR)
            with os.fdopen(fd, "w+b") as f:
                f.write(response.content)
        except:
            return False

        return True

    def post(self, url, data=None, files=None, headers=None, get_json=True):
        ''' Make POST request to a url

        Args:
            url (str): URL to send the request to
            data (dict, optional): Data to send
            files (dict, optional): Files to send with the request
            headers (str, optional): custom headers

        Returns:
            A JSON object of the returned response if `get_json` is True,
            Requests' response object otherwise

        '''

        if self.debug:
            print("POST: %s, headers=%s" % (url, headers))

        post_headers = self.headers
        if headers is not None:
            post_headers.update(headers)
        response = requests.post(url, headers=post_headers, data=data, auth=self.auth, files=files, verify=self.verify_ssl)
        self.http_status_code = response.status_code
        self._check_error(response)
        if get_json is True:
            return self._get_json_response(response)
        return response

    # TODO: use a expected key in returned json, if the returned key does not match, return false...
    def _check_error(self, response):
        ''' Check for HTTP error code from the response, raise exception if there's any

        Args:
            response (object): Object returned by requests' `get` and `post`
                methods

        Raises:
            HTTPError: If the status code of response is either 4xx or 5xx

        Returns:
            True if status code is not error code

        '''

        # If status code is 4xx or 5xx, that should be an error
        if response.status_code >= 400:
            j = self._get_json_response(response)
            err_cls = self._check_http_error_code(response.status_code)
            # I intended to return False here but raising a meaningful exception
            # may make senses more.
            try:
                raise err_cls("%s error: %s" % (response.status_code, j["error"]["error_msg"]), response.status_code)
            # This is to catch error when we post get oath data
            except TypeError:
                raise err_cls("%s error: %s" % (response.status_code, j["error_description"]), response.status_code)
        # Return True if everything looks OK
        return True

    def _check_http_error_code(self, code):
        return {
            400: BadRequest,
            401: Unauthorized,
            402: PaymentRequired,
            403: Forbidden,
            404: NotFound,
            405: MethodNotAllowed,
            406: NotAcceptable,
            408: RequestTimeout,
            409: Conflict,
            410: Gone,
            414: RequestURITooLong,
            415: UnsupportedMediaType,
            416: RequestedRangeNotSatisfiable,
            500: InternalServerError,
            501: MethodNotImplemented,
            502: BadGateway,
            503: ServiceUnavailable,
            504: GatewayTimeout
        }.get(code, HTTPError)


import numpy as np
import pandas as pd
from sklearn.metrics import mean_absolute_error
import matplotlib.pyplot as plt
pd.options.mode.chained_assignment = None
from datetime import datetime

# Load test data
test = np.load("test2020.npy")
test = test.reshape(test.shape[0]*test.shape[1])

# Load NN univariate o multivariate
pred_cnn = np.load("mmcnn.npy")
pred_cnn = pred_cnn.reshape(pred_cnn.shape[0]*pred_cnn.shape[1])

pred_rnn = np.load("mmrnn.npy")
pred_rnn = pred_rnn.reshape(pred_rnn.shape[0]*pred_rnn.shape[1])

pred_crnn = np.load("mmcnnandrnn.npy")
pred_crnn = pred_crnn.reshape(pred_crnn.shape[0]*pred_crnn.shape[1])

# Load ARIMA or TFM
pred_arima = pd.read_csv("ARIMA_FINAL.csv")
pred_arima = pred_arima.to_numpy()
pred_arima = pred_arima.transpose()
pred_arima = pred_arima.reshape(pred_arima.shape[0]*pred_arima.shape[1],)

#### CREATE DATABASE FOR TABLES

#MAE
mae_cnn = mean_absolute_error(test, pred_cnn)
mae_rnn = mean_absolute_error(test, pred_rnn)
mae_arima = mean_absolute_error(test, pred_arima)
mae_crnn = mean_absolute_error(test, pred_crnn)

#RMSE
def rmse(predictions):
    rmse = np.sqrt(sum((test-predictions)**2)  / len(test))
    return rmse

rmse_cnn = rmse(pred_cnn)
rmse_rnn = rmse(pred_rnn)
rmse_arima = rmse(pred_arima)
rmse_crnn = rmse(pred_crnn)

#MAPE
def mape(predictions):
    mape = np.mean(np.abs((test - predictions)/test))*100
    return mape

mape_cnn = mape(pred_cnn)
mape_rnn = mape(pred_rnn)
mape_arima = mape(pred_arima)
mape_crnn = mape(pred_crnn)

# Create dataframe

df_error=pd.DataFrame(data={"RMSE": [rmse_arima, rmse_cnn, rmse_rnn, rmse_crnn],
                            "MAE": [mae_arima, mae_cnn, mae_rnn, mae_crnn],
                            "MAPE": [mape_arima, mape_cnn, mape_rnn, mape_crnn]}, index = ["ARIMA", "CNN", "RNN", "CNNandRNN"])

#### CREATE DATABASE FOR BARPLOTS

# Database. Columns: Error, day, hour, month
def data_base(error):
    df1=pd.read_csv("precios_14_20.txt", sep=",", index_col=0)
    df=df1[-8784:]
    error = error.tolist()
    df["error"]=error
    df=df.drop("price", axis = 1)
    index = df.index
    day = list()
    hour = list()
    month = list()
    for i in index:
        day.append(i[0:10])
        hour.append(i[11:13])
        month.append(i[5:7])
    day_dt = [datetime.strptime(x, '%Y-%m-%d') for x in day]
    weekday = [x.weekday() for x in day_dt]
    df["day"] = weekday
    df["hour"] = hour
    df["month"] = month
    return df

# MSE for each of the hours
def mse_hours(predictions):
    mse = (test- predictions)**2
    return mse

mse_hours_cnn =   mse_hours(pred_cnn)
mse_hours_rnn =   mse_hours(pred_rnn)
mse_hours_arima = mse_hours(pred_arima)
mse_hours_crnn =  mse_hours(pred_crnn)

df_cnn = data_base(mse_hours_cnn)
df_rnn = data_base(mse_hours_rnn)
df_arima = data_base(mse_hours_arima)
df_crnn = data_base(mse_hours_crnn)

# Group data by day, hour and month. Output: RMSE per day, hour and month
def group_data(df):
    day_error=df.groupby(["day"])["error"].mean()
    hour_error=df.groupby(["hour"])["error"].mean()
    month_error=df.groupby(["month"])["error"].mean()
    day_error = day_error.tolist()
    hour_error = hour_error.tolist()
    month_error = month_error.tolist()
    return np.sqrt(day_error), np.sqrt(hour_error), np.sqrt(month_error)

day_cnn, hour_cnn, month_cnn = group_data(df_cnn)
day_ARIMA, hour_ARIMA, month_ARIMA = group_data(df_arima)
day_rnn, hour_rnn, month_rnn = group_data(df_rnn)
day_crnn, hour_crnn, month_crnn = group_data(df_crnn)

# Hours NEURAL NETWORKS
x = np.arange(24)
width = 0.20
x_labels = ["00", "01", "02","03", "04", "05", "06","07", "08", "09","10","11","12","13","14","15","16","17","18","19",
      "20","21","22","23"]

plt.bar(x -0.2, hour_cnn, width, color = "SkyBlue" , edgecolor="black")
plt.bar(x , hour_rnn, width, edgecolor="black", color = "IndianRed")
plt.bar(x + 0.2 , hour_crnn, width, edgecolor="black", color = "forestgreen")
plt.title("RMSE per hours")
plt.ylabel("RMSE")
plt.xticks(x, x_labels, rotation=0)
plt.legend(["CNN", "RNN", "CNN+RNN"])
plt.show()

# Hours NN VS ARIMA
x = np.arange(24)
width = 0.40
x_labels = ["00", "01", "02","03", "04", "05", "06","07", "08", "09","10","11","12","13","14","15","16","17","18","19",
      "20","21","22","23"]

plt.bar(x - 0.2, hour_cnn, width, color = "SkyBlue" , edgecolor="black")
plt.bar(x + 0.2, hour_ARIMA, width, color = "IndianRed", edgecolor="black")
plt.title("RMSE per hours")
plt.ylabel("RMSE")
plt.xticks(x, x_labels, rotation=0)
plt.legend(["CNN", "ARIMA"])
plt.show()

### Days
x = np.arange(7)
width = 0.40
x_labels = ["Monday", "Tuesday", "Wednesday", "Thrusday", "Friday", "Saturday", "Sunday"]

plt.bar(x - 0.2, day_cnn, width, color = "SkyBlue" , edgecolor="black")
plt.bar(x + 0.2, day_ARIMA, width, color = "IndianRed", edgecolor="black")
plt.title("RMSE per days")
plt.ylabel("RMSE")
plt.xticks(x, x_labels, rotation=45)
plt.legend(["CNN", "ARIMA"])
plt.show()

### Months
x = np.arange(12)
width = 0.40
x_labels = ["January","February","March","April","May","June","July","August","September","October","November",
            "December"]
plt.bar(x - 0.2, month_cnn, width, color = "SkyBlue" , edgecolor="black")
plt.bar(x + 0.2, month_ARIMA, width, color = "IndianRed", edgecolor="black")
plt.title("RMSE per months")
plt.ylabel("RMSE")
plt.xticks(x, x_labels, rotation=45)
plt.legend(["CNN", "ARIMA"])
plt.show()


"""
Faça uma lista de comprar com listas
O usuário deve ter a possibilidade de
inserir, apagar e listar valores da sua lista
Não permita que o programa quebre com
erros de índices inexistentes na lista.
"""
import os

lista = []

while True:
    print('Selecione uma opção')
    opcao = input('[i]nserir [a]pagar [l]istar: ')

    if opcao == 'i':
        os.system('clear')
        valor = input('Valor: ')
        lista.append(valor)
    elif opcao == 'a':
        indice_str = input(
            'Escolha o índice para apagar: '
        )

        try:
            indice = int(indice_str)
            del lista[indice]
        except ValueError:
            print('Por favor digite número int.')
        except IndexError:
            print('Índice não existe na lista')
        except Exception:
            print('Erro desconhecido')
    elif opcao == 'l':
        os.system('clear')

        if len(lista) == 0:
            print('Nada para listar')

        for i, valor in enumerate(lista):
            print(i, valor)
    else:
        print('Por favor, escolha i, a ou l.')