from graphics import *
from gridutil import *

import random
import math
import numpy as np

class LocWorldEnv:
    actions = "turnleft turnright forward".split()

    def __init__(self, size, walls, start_loc, sigma_move, sigma_perc):
        self.size = size
        self.walls = walls
        self.action_sensors = []
        self.locations = {*locations(self.size)}.difference(self.walls)
        self.start_loc = start_loc
        self.sigma_move = sigma_move
        self.sigma_perc = sigma_perc
        self.lifes = 3
        self.reset()
        self.finished = False

    def reset(self):
        self.agentLoc = self.start_loc
        self.agentDir = 'N'

    def getPercept(self):
        percept = self.agentLoc[0] + random.gauss(0.0, self.sigma_perc)

        return percept

    def doAction(self, action):
        points = -1

        action += random.gauss(0.0, self.sigma_move)

        print('executed action %.3f' % action)
        self.agentLoc = ((self.agentLoc[0] + action) % self.size, self.agentLoc[1])

        return points  # cost/benefit of action

class LocView:
    # LocView shows a view of a LocWorldEnv. Just hand it an env, and
    #   a window will pop up.

    Size = .5
    Points = {'N': (0, -Size, 0, Size), 'E': (-Size, 0, Size, 0),
              'S': (0, Size, 0, -Size), 'W': (Size, 0, -Size, 0)}

    color = "black"

    def __init__(self, state, height=800, title="Loc World"):
        xySize = state.size
        win = self.win = GraphWin(title, 1.33 * height, height, autoflush=False)
        win.setBackground("gray99")
        win.setCoords(-.5, -.5, 1.33 * xySize - .5, xySize - .5)
        cells = self.cells = {}
        for x in range(xySize):
            for y in range(xySize):
                cells[(x, y)] = Rectangle(Point(x - .5, y - .5), Point(x + .5, y + .5))
                cells[(x, y)].setWidth(0)
                cells[(x, y)].draw(win)
        self.agt = None
        self.arrow = None
        self.prob_prim = None
        ccenter = 1.167 * (xySize - .5)
        # self.time = Text(Point(ccenter, (xySize - 1) * .75), "Time").draw(win)
        # self.time.setSize(36)
        # self.setTimeColor("black")

        self.agentName = Text(Point(ccenter, (xySize - 1) * .5), "").draw(win)
        self.agentName.setSize(20)
        self.agentName.setFill("Orange")

        self.info = Text(Point(ccenter, (xySize - 1) * .25), "").draw(win)
        self.info.setSize(20)
        self.info.setFace("courier")

        self.update(state)

    def setAgent(self, name):
        self.agentName.setText(name)

    # def setTime(self, seconds):
    #     self.time.setText(str(seconds))

    def setInfo(self, info):
        self.info.setText(info)

    def update(self, state, mu=None, sigma=None):
        # View state in exiting window
        for loc, cell in self.cells.items():
            if loc in state.walls:
                cell.setFill("black")
            else:
                cell.setFill("white")

        if self.prob_prim is not None:
            self.prob_prim.undraw()
        if mu is not None and sigma is not None:
            offset = state.agentLoc[1] + 2
            points = [Point(0.0, offset)]
            for x in np.arange(0.0, state.size, 0.2):
                y = 1.0 / (sigma * math.sqrt(2 * math.pi)) * math.exp(-(x - mu)**2 / (2 * sigma**2))
                points.append(Point(x, offset + 4 * y))
            points.append(Point(state.size, offset))
            self.prob_prim = Polygon(points)
            self.prob_prim.setWidth(1)
            self.prob_prim.setFill("blue")
            self.prob_prim.draw(self.win)

        if self.agt:
            self.agt.undraw()
        if state.agentLoc:
            self.agt = self.drawArrow(state.agentLoc, state.agentDir, 5, self.color)

    def drawRect(self, loc, height, color="blue"):
        x, y = loc
        a = Rectangle(Point(x - .5, y - .5), Point(x + .5, y - .5 + 4 * height))
        a.setWidth(0)
        a.setFill(color)
        a.draw(self.win)
        return a

    def drawDot(self, loc, color="blue"):
        x, y = loc
        a = Circle(Point(x, y), .1)
        a.setWidth(1)
        a.setFill(color)
        a.draw(self.win)
        return a

    def drawLine(self, loc1, loc2, color="blue"):
        x1, y1 = loc1
        x2, y2 = loc2
        p1 = Point(x1, y1)
        p2 = Point(x2, y2)
        a = Line(p1, p2)
        a.setWidth(2)
        a.setFill(color)
        a.draw(self.win)
        return a

    def drawArrow(self, loc, heading, width, color):
        x, y = loc
        dx0, dy0, dx1, dy1 = self.Points[heading]
        p1 = Point(x + dx0, y + dy0)
        p2 = Point(x + dx1, y + dy1)
        a = Line(p1, p2)
        a.setWidth(width)
        a.setArrow('last')
        a.setFill(color)
        a.draw(self.win)
        return a

    def pause(self):
        self.win.getMouse()

    # def setTimeColor(self, c):
    #     self.time.setTextColor(c)

    def close(self):
        self.win.close()

Przeszukiwanie i planowanie ścieżki/filtr_kalmana/env.py


from flask import Flask, render_template,request,make_response, request

from models import Links
import json

model_obj = Links()

app = Flask(__name__)

@app.route("/")
def index():
  return render_template("index.html")

@app.route("/links")
def links():
    global model_obj
    datas=model_obj.all()
    return json.dumps(datas)

@app.route("/search")
def search():
  global model_obj
  print(request.args)

  category = request.args['cat']
  content = request.args['content']

  searched_data = model_obj.search(cat=category, content=content)
  print(searched_data)
  return searched_data

if __name__ == "__main__":
  app.run(host="0.0.0.0",debug=True)


import argparse
import collections
import datetime
import json
import os
import sys

from simpleperf_report_lib import ReportLib
from utils import log_info, log_exit
from utils import Addr2Nearestline, get_script_dir, Objdump, open_report_in_browser
from utils import SourceFileSearcher

MAX_CALLSTACK_LENGTH = 750

class HtmlWriter(object):

    def __init__(self, output_path):
        self.fh = open(output_path, 'w')
        self.tag_stack = []

    def close(self):
        self.fh.close()

    def open_tag(self, tag, **attrs):
        attr_str = ''
        for key in attrs:
            attr_str += ' %s="%s"' % (key, attrs[key])
        self.fh.write('<%s%s>' % (tag, attr_str))
        self.tag_stack.append(tag)
        return self

    def close_tag(self, tag=None):
        if tag:
            assert tag == self.tag_stack[-1]
        self.fh.write('</%s>\n' % self.tag_stack.pop())

    def add(self, text):
        self.fh.write(text)
        return self

    def add_file(self, file_path):
        file_path = os.path.join(get_script_dir(), file_path)
        with open(file_path, 'r') as f:
            self.add(f.read())
        return self

def modify_text_for_html(text):
    return text.replace('>', '>').replace('<', '<')

class EventScope(object):

    def __init__(self, name):
        self.name = name
        self.processes = {}  # map from pid to ProcessScope
        self.sample_count = 0
        self.event_count = 0

    def get_process(self, pid):
        process = self.processes.get(pid)
        if not process:
            process = self.processes[pid] = ProcessScope(pid)
        return process

    def get_sample_info(self, gen_addr_hit_map):
        result = {}
        result['eventName'] = self.name
        result['eventCount'] = self.event_count
        processes = sorted(self.processes.values(), key=lambda a: a.event_count, reverse=True)
        result['processes'] = [process.get_sample_info(gen_addr_hit_map)
                               for process in processes]
        return result

    @property
    def threads(self):
        for process in self.processes.values():
            for thread in process.threads.values():
                yield thread

    @property
    def libraries(self):
        for process in self.processes.values():
            for thread in process.threads.values():
                for lib in thread.libs.values():
                    yield lib

class ProcessScope(object):

    def __init__(self, pid):
        self.pid = pid
        self.name = ''
        self.event_count = 0
        self.threads = {}  # map from tid to ThreadScope

    def get_thread(self, tid, thread_name):
        thread = self.threads.get(tid)
        if not thread:
            thread = self.threads[tid] = ThreadScope(tid)
        thread.name = thread_name
        if self.pid == tid:
            self.name = thread_name
        return thread

    def get_sample_info(self, gen_addr_hit_map):
        result = {}
        result['pid'] = self.pid
        result['eventCount'] = self.event_count
        threads = sorted(self.threads.values(), key=lambda a: a.event_count, reverse=True)
        result['threads'] = [thread.get_sample_info(gen_addr_hit_map)
                             for thread in threads]
        return result

class ThreadScope(object):

    def __init__(self, tid):
        self.tid = tid
        self.name = ''
        self.event_count = 0
        self.sample_count = 0
        self.libs = {}  # map from lib_id to LibScope
        self.call_graph = CallNode(-1)
        self.reverse_call_graph = CallNode(-1)

    def add_callstack(self, event_count, callstack, build_addr_hit_map):
        """ callstack is a list of tuple (lib_id, func_id, addr).
            For each i > 0, callstack[i] calls callstack[i-1]."""
        hit_func_ids = set()
        for i, (lib_id, func_id, addr) in enumerate(callstack):
            # When a callstack contains recursive function, only add for each function once.
            if func_id in hit_func_ids:
                continue
            hit_func_ids.add(func_id)

            lib = self.libs.get(lib_id)
            if not lib:
                lib = self.libs[lib_id] = LibScope(lib_id)
            function = lib.get_function(func_id)
            function.subtree_event_count += event_count
            if i == 0:
                lib.event_count += event_count
                function.event_count += event_count
                function.sample_count += 1
            if build_addr_hit_map:
                function.build_addr_hit_map(addr, event_count if i == 0 else 0, event_count)

        # build call graph and reverse call graph
        node = self.call_graph
        for item in reversed(callstack):
            node = node.get_child(item[1])
        node.event_count += event_count
        node = self.reverse_call_graph
        for item in callstack:
            node = node.get_child(item[1])
        node.event_count += event_count

    def update_subtree_event_count(self):
        self.call_graph.update_subtree_event_count()
        self.reverse_call_graph.update_subtree_event_count()

    def limit_percents(self, min_func_limit, min_callchain_percent, hit_func_ids):
        for lib in self.libs.values():
            to_del_funcs = []
            for function in lib.functions.values():
                if function.subtree_event_count < min_func_limit:
                    to_del_funcs.append(function.func_id)
                else:
                    hit_func_ids.add(function.func_id)
            for func_id in to_del_funcs:
                del lib.functions[func_id]
        min_limit = min_callchain_percent * 0.01 * self.call_graph.subtree_event_count
        self.call_graph.cut_edge(min_limit, hit_func_ids)
        self.reverse_call_graph.cut_edge(min_limit, hit_func_ids)

    def get_sample_info(self, gen_addr_hit_map):
        result = {}
        result['tid'] = self.tid
        result['eventCount'] = self.event_count
        result['sampleCount'] = self.sample_count
        result['libs'] = [lib.gen_sample_info(gen_addr_hit_map)
                          for lib in self.libs.values()]
        result['g'] = self.call_graph.gen_sample_info()
        result['rg'] = self.reverse_call_graph.gen_sample_info()
        return result

class LibScope(object):

    def __init__(self, lib_id):
        self.lib_id = lib_id
        self.event_count = 0
        self.functions = {}  # map from func_id to FunctionScope.

    def get_function(self, func_id):
        function = self.functions.get(func_id)
        if not function:
            function = self.functions[func_id] = FunctionScope(func_id)
        return function

    def gen_sample_info(self, gen_addr_hit_map):
        result = {}
        result['libId'] = self.lib_id
        result['eventCount'] = self.event_count
        result['functions'] = [func.gen_sample_info(gen_addr_hit_map)
                               for func in self.functions.values()]
        return result

class FunctionScope(object):

    def __init__(self, func_id):
        self.func_id = func_id
        self.sample_count = 0
        self.event_count = 0
        self.subtree_event_count = 0
        self.addr_hit_map = None  # map from addr to [event_count, subtree_event_count].
        # map from (source_file_id, line) to [event_count, subtree_event_count].
        self.line_hit_map = None

    def build_addr_hit_map(self, addr, event_count, subtree_event_count):
        if self.addr_hit_map is None:
            self.addr_hit_map = {}
        count_info = self.addr_hit_map.get(addr)
        if count_info is None:
            self.addr_hit_map[addr] = [event_count, subtree_event_count]
        else:
            count_info[0] += event_count
            count_info[1] += subtree_event_count

    def build_line_hit_map(self, source_file_id, line, event_count, subtree_event_count):
        if self.line_hit_map is None:
            self.line_hit_map = {}
        key = (source_file_id, line)
        count_info = self.line_hit_map.get(key)
        if count_info is None:
            self.line_hit_map[key] = [event_count, subtree_event_count]
        else:
            count_info[0] += event_count
            count_info[1] += subtree_event_count

    def gen_sample_info(self, gen_addr_hit_map):
        result = {}
        result['f'] = self.func_id
        result['c'] = [self.sample_count, self.event_count, self.subtree_event_count]
        if self.line_hit_map:
            items = []
            for key in self.line_hit_map:
                count_info = self.line_hit_map[key]
                item = {'f': key[0], 'l': key[1], 'e': count_info[0], 's': count_info[1]}
                items.append(item)
            result['s'] = items
        if gen_addr_hit_map and self.addr_hit_map:
            items = []
            for addr in sorted(self.addr_hit_map):
                count_info = self.addr_hit_map[addr]
                items.append({'a': addr, 'e': count_info[0], 's': count_info[1]})
            result['a'] = items
        return result

class CallNode(object):

    def __init__(self, func_id):
        self.event_count = 0
        self.subtree_event_count = 0
        self.func_id = func_id
        self.children = collections.OrderedDict()  # map from func_id to CallNode

    def get_child(self, func_id):
        child = self.children.get(func_id)
        if not child:
            child = self.children[func_id] = CallNode(func_id)
        return child

    def update_subtree_event_count(self):
        self.subtree_event_count = self.event_count
        for child in self.children.values():
            self.subtree_event_count += child.update_subtree_event_count()
        return self.subtree_event_count

    def cut_edge(self, min_limit, hit_func_ids):
        hit_func_ids.add(self.func_id)
        to_del_children = []
        for key in self.children:
            child = self.children[key]
            if child.subtree_event_count < min_limit:
                to_del_children.append(key)
            else:
                child.cut_edge(min_limit, hit_func_ids)
        for key in to_del_children:
            del self.children[key]

    def gen_sample_info(self):
        result = {}
        result['e'] = self.event_count
        result['s'] = self.subtree_event_count
        result['f'] = self.func_id
        result['c'] = [child.gen_sample_info() for child in self.children.values()]
        return result

class LibSet(object):
    """ Collection of shared libraries used in perf.data. """
    def __init__(self):
        self.lib_name_to_id = {}
        self.lib_id_to_name = []

    def get_lib_id(self, lib_name):
        lib_id = self.lib_name_to_id.get(lib_name)
        if lib_id is None:
            lib_id = len(self.lib_id_to_name)
            self.lib_name_to_id[lib_name] = lib_id
            self.lib_id_to_name.append(lib_name)
        return lib_id

    def get_lib_name(self, lib_id):
        return self.lib_id_to_name[lib_id]

class Function(object):
    """ Represent a function in a shared library. """
    def __init__(self, lib_id, func_name, func_id, start_addr, addr_len):
        self.lib_id = lib_id
        self.func_name = func_name
        self.func_id = func_id
        self.start_addr = start_addr
        self.addr_len = addr_len
        self.source_info = None
        self.disassembly = None

class FunctionSet(object):
    """ Collection of functions used in perf.data. """
    def __init__(self):
        self.name_to_func = {}
        self.id_to_func = {}

    def get_func_id(self, lib_id, symbol):
        key = (lib_id, symbol.symbol_name)
        function = self.name_to_func.get(key)
        if function is None:
            func_id = len(self.id_to_func)
            function = Function(lib_id, symbol.symbol_name, func_id, symbol.symbol_addr,
                                symbol.symbol_len)
            self.name_to_func[key] = function
            self.id_to_func[func_id] = function
        return function.func_id

    def trim_functions(self, left_func_ids):
        """ Remove functions excepts those in left_func_ids. """
        for function in self.name_to_func.values():
            if function.func_id not in left_func_ids:
                del self.id_to_func[function.func_id]
        # name_to_func will not be used.
        self.name_to_func = None

class SourceFile(object):
    """ A source file containing source code hit by samples. """
    def __init__(self, file_id, abstract_path):
        self.file_id = file_id
        self.abstract_path = abstract_path  # path reported by addr2line
        self.real_path = None  # file path in the file system
        self.requested_lines = set()
        self.line_to_code = {}  # map from line to code in that line.

    def request_lines(self, start_line, end_line):
        self.requested_lines |= set(range(start_line, end_line + 1))

    def add_source_code(self, real_path):
        self.real_path = real_path
        with open(real_path, 'r') as f:
            source_code = f.readlines()
        max_line = len(source_code)
        for line in self.requested_lines:
            if line > 0 and line <= max_line:
                self.line_to_code[line] = source_code[line - 1]
        # requested_lines is no longer used.
        self.requested_lines = None

class SourceFileSet(object):
    """ Collection of source files. """
    def __init__(self):
        self.path_to_source_files = {}  # map from file path to SourceFile.

    def get_source_file(self, file_path):
        source_file = self.path_to_source_files.get(file_path)
        if source_file is None:
            source_file = SourceFile(len(self.path_to_source_files), file_path)
            self.path_to_source_files[file_path] = source_file
        return source_file

    def load_source_code(self, source_dirs):
        file_searcher = SourceFileSearcher(source_dirs)
        for source_file in self.path_to_source_files.values():
            real_path = file_searcher.get_real_path(source_file.abstract_path)
            if real_path:
                source_file.add_source_code(real_path)

class RecordData(object):

    """RecordData reads perf.data, and generates data used by report.js in json format.
        All generated items are listed as below:
            1. recordTime: string
            2. machineType: string
            3. androidVersion: string
            4. recordCmdline: string
            5. totalSamples: int
            6. processNames: map from pid to processName.
            7. threadNames: map from tid to threadName.
            8. libList: an array of libNames, indexed by libId.
            9. functionMap: map from functionId to funcData.
                funcData = {
                    l: libId
                    f: functionName
                    s: [sourceFileId, startLine, endLine] [optional]
                    d: [(disassembly, addr)] [optional]
                }

            10.  sampleInfo = [eventInfo]
                eventInfo = {
                    eventName
                    eventCount
                    processes: [processInfo]
                }
                processInfo = {
                    pid
                    eventCount
                    threads: [threadInfo]
                }
                threadInfo = {
                    tid
                    eventCount
                    sampleCount
                    libs: [libInfo],
                    g: callGraph,
                    rg: reverseCallgraph
                }
                libInfo = {
                    libId,
                    eventCount,
                    functions: [funcInfo]
                }
                funcInfo = {
                    f: functionId
                    c: [sampleCount, eventCount, subTreeEventCount]
                    s: [sourceCodeInfo] [optional]
                    a: [addrInfo] (sorted by addrInfo.addr) [optional]
                }
                callGraph and reverseCallGraph are both of type CallNode.
                callGraph shows how a function calls other functions.
                reverseCallGraph shows how a function is called by other functions.
                CallNode {
                    e: selfEventCount
                    s: subTreeEventCount
                    f: functionId
                    c: [CallNode] # children
                }

                sourceCodeInfo {
                    f: sourceFileId
                    l: line
                    e: eventCount
                    s: subtreeEventCount
                }

                addrInfo {
                    a: addr
                    e: eventCount
                    s: subtreeEventCount
                }

            11. sourceFiles: an array of sourceFile, indexed by sourceFileId.
                sourceFile {
                    path
                    code:  # a map from line to code for that line.
                }
    """

    def __init__(self, binary_cache_path, ndk_path, build_addr_hit_map):
        self.binary_cache_path = binary_cache_path
        self.ndk_path = ndk_path
        self.build_addr_hit_map = build_addr_hit_map
        self.meta_info = None
        self.cmdline = None
        self.arch = None
        self.events = {}
        self.libs = LibSet()
        self.functions = FunctionSet()
        self.total_samples = 0
        self.source_files = SourceFileSet()
        self.gen_addr_hit_map_in_record_info = False

    def load_record_file(self, record_file, show_art_frames):
        lib = ReportLib()
        lib.SetRecordFile(record_file)
        # If not showing ip for unknown symbols, the percent of the unknown symbol may be
        # accumulated to very big, and ranks first in the sample table.
        lib.ShowIpForUnknownSymbol()
        if show_art_frames:
            lib.ShowArtFrames()
        if self.binary_cache_path:
            lib.SetSymfs(self.binary_cache_path)
        self.meta_info = lib.MetaInfo()
        self.cmdline = lib.GetRecordCmd()
        self.arch = lib.GetArch()
        while True:
            raw_sample = lib.GetNextSample()
            if not raw_sample:
                lib.Close()
                break
            raw_event = lib.GetEventOfCurrentSample()
            symbol = lib.GetSymbolOfCurrentSample()
            callchain = lib.GetCallChainOfCurrentSample()
            event = self._get_event(raw_event.name)
            self.total_samples += 1
            event.sample_count += 1
            event.event_count += raw_sample.period
            process = event.get_process(raw_sample.pid)
            process.event_count += raw_sample.period
            thread = process.get_thread(raw_sample.tid, raw_sample.thread_comm)
            thread.event_count += raw_sample.period
            thread.sample_count += 1

            lib_id = self.libs.get_lib_id(symbol.dso_name)
            func_id = self.functions.get_func_id(lib_id, symbol)
            callstack = [(lib_id, func_id, symbol.vaddr_in_file)]
            for i in range(callchain.nr):
                symbol = callchain.entries[i].symbol
                lib_id = self.libs.get_lib_id(symbol.dso_name)
                func_id = self.functions.get_func_id(lib_id, symbol)
                callstack.append((lib_id, func_id, symbol.vaddr_in_file))
            if len(callstack) > MAX_CALLSTACK_LENGTH:
                callstack = callstack[:MAX_CALLSTACK_LENGTH]
            thread.add_callstack(raw_sample.period, callstack, self.build_addr_hit_map)

        for event in self.events.values():
            for thread in event.threads:
                thread.update_subtree_event_count()

    def limit_percents(self, min_func_percent, min_callchain_percent):
        hit_func_ids = set()
        for event in self.events.values():
            min_limit = event.event_count * min_func_percent * 0.01
            for process in event.processes.values():
                to_del_threads = []
                for thread in process.threads.values():
                    if thread.call_graph.subtree_event_count < min_limit:
                        to_del_threads.append(thread.tid)
                    else:
                        thread.limit_percents(min_limit, min_callchain_percent, hit_func_ids)
                for thread in to_del_threads:
                    del process.threads[thread]
        self.functions.trim_functions(hit_func_ids)

    def _get_event(self, event_name):
        if event_name not in self.events:
            self.events[event_name] = EventScope(event_name)
        return self.events[event_name]

    def add_source_code(self, source_dirs, filter_lib):
        """ Collect source code information:
            1. Find line ranges for each function in FunctionSet.
            2. Find line for each addr in FunctionScope.addr_hit_map.
            3. Collect needed source code in SourceFileSet.
        """
        addr2line = Addr2Nearestline(self.ndk_path, self.binary_cache_path, False)
        # Request line range for each function.
        for function in self.functions.id_to_func.values():
            if function.func_name == 'unknown':
                continue
            lib_name = self.libs.get_lib_name(function.lib_id)
            if filter_lib(lib_name):
                addr2line.add_addr(lib_name, function.start_addr, function.start_addr)
                addr2line.add_addr(lib_name, function.start_addr,
                                   function.start_addr + function.addr_len - 1)
        # Request line for each addr in FunctionScope.addr_hit_map.
        for event in self.events.values():
            for lib in event.libraries:
                lib_name = self.libs.get_lib_name(lib.lib_id)
                if filter_lib(lib_name):
                    for function in lib.functions.values():
                        func_addr = self.functions.id_to_func[function.func_id].start_addr
                        for addr in function.addr_hit_map:
                            addr2line.add_addr(lib_name, func_addr, addr)
        addr2line.convert_addrs_to_lines()

        # Set line range for each function.
        for function in self.functions.id_to_func.values():
            if function.func_name == 'unknown':
                continue
            dso = addr2line.get_dso(self.libs.get_lib_name(function.lib_id))
            if not dso:
                continue
            start_source = addr2line.get_addr_source(dso, function.start_addr)
            end_source = addr2line.get_addr_source(dso, function.start_addr + function.addr_len - 1)
            if not start_source or not end_source:
                continue
            start_file_path, start_line = start_source[-1]
            end_file_path, end_line = end_source[-1]
            if start_file_path != end_file_path or start_line > end_line:
                continue
            source_file = self.source_files.get_source_file(start_file_path)
            source_file.request_lines(start_line, end_line)
            function.source_info = (source_file.file_id, start_line, end_line)

        # Build FunctionScope.line_hit_map.
        for event in self.events.values():
            for lib in event.libraries:
                dso = addr2line.get_dso(self.libs.get_lib_name(lib.lib_id))
                if not dso:
                    continue
                for function in lib.functions.values():
                    for addr in function.addr_hit_map:
                        source = addr2line.get_addr_source(dso, addr)
                        if not source:
                            continue
                        for file_path, line in source:
                            source_file = self.source_files.get_source_file(file_path)
                            # Show [line - 5, line + 5] of the line hit by a sample.
                            source_file.request_lines(line - 5, line + 5)
                            count_info = function.addr_hit_map[addr]
                            function.build_line_hit_map(source_file.file_id, line, count_info[0],
                                                        count_info[1])

        # Collect needed source code in SourceFileSet.
        self.source_files.load_source_code(source_dirs)

    def add_disassembly(self, filter_lib):
        """ Collect disassembly information:
            1. Use objdump to collect disassembly for each function in FunctionSet.
            2. Set flag to dump addr_hit_map when generating record info.
        """
        objdump = Objdump(self.ndk_path, self.binary_cache_path)
        cur_lib_name = None
        dso_info = None
        for function in sorted(self.functions.id_to_func.values(), key=lambda a: a.lib_id):
            if function.func_name == 'unknown':
                continue
            lib_name = self.libs.get_lib_name(function.lib_id)
            if lib_name != cur_lib_name:
                cur_lib_name = lib_name
                if filter_lib(lib_name):
                    dso_info = objdump.get_dso_info(lib_name)
                else:
                    dso_info = None
                if dso_info:
                    log_info('Disassemble %s' % dso_info[0])
            if dso_info:
                code = objdump.disassemble_code(dso_info, function.start_addr, function.addr_len)
                function.disassembly = code

        self.gen_addr_hit_map_in_record_info = True

    def gen_record_info(self):
        record_info = {}
        timestamp = self.meta_info.get('timestamp')
        if timestamp:
            t = datetime.datetime.fromtimestamp(int(timestamp))
        else:
            t = datetime.datetime.now()
        record_info['recordTime'] = t.strftime('%Y-%m-%d (%A) %H:%M:%S')

        product_props = self.meta_info.get('product_props')
        machine_type = self.arch
        if product_props:
            manufacturer, model, name = product_props.split(':')
            machine_type = '%s (%s) by %s, arch %s' % (model, name, manufacturer, self.arch)
        record_info['machineType'] = machine_type
        record_info['androidVersion'] = self.meta_info.get('android_version', '')
        record_info['recordCmdline'] = self.cmdline
        record_info['totalSamples'] = self.total_samples
        record_info['processNames'] = self._gen_process_names()
        record_info['threadNames'] = self._gen_thread_names()
        record_info['libList'] = self._gen_lib_list()
        record_info['functionMap'] = self._gen_function_map()
        record_info['sampleInfo'] = self._gen_sample_info()
        record_info['sourceFiles'] = self._gen_source_files()
        return record_info

    def _gen_process_names(self):
        process_names = {}
        for event in self.events.values():
            for process in event.processes.values():
                process_names[process.pid] = process.name
        return process_names

    def _gen_thread_names(self):
        thread_names = {}
        for event in self.events.values():
            for process in event.processes.values():
                for thread in process.threads.values():
                    thread_names[thread.tid] = thread.name
        return thread_names

    def _gen_lib_list(self):
        return [modify_text_for_html(x) for x in self.libs.lib_id_to_name]

    def _gen_function_map(self):
        func_map = {}
        for func_id in sorted(self.functions.id_to_func):
            function = self.functions.id_to_func[func_id]
            func_data = {}
            func_data['l'] = function.lib_id
            func_data['f'] = modify_text_for_html(function.func_name)
            if function.source_info:
                func_data['s'] = function.source_info
            if function.disassembly:
                disassembly_list = []
                for code, addr in function.disassembly:
                    disassembly_list.append([modify_text_for_html(code), addr])
                func_data['d'] = disassembly_list
            func_map[func_id] = func_data
        return func_map

    def _gen_sample_info(self):
        return [event.get_sample_info(self.gen_addr_hit_map_in_record_info)
                for event in self.events.values()]

    def _gen_source_files(self):
        source_files = sorted(self.source_files.path_to_source_files.values(),
                              key=lambda x: x.file_id)
        file_list = []
        for source_file in source_files:
            file_data = {}
            if not source_file.real_path:
                file_data['path'] = ''
                file_data['code'] = {}
            else:
                file_data['path'] = source_file.real_path
                code_map = {}
                for line in source_file.line_to_code:
                    code_map[line] = modify_text_for_html(source_file.line_to_code[line])
                file_data['code'] = code_map
            file_list.append(file_data)
        return file_list

URLS = {
    'jquery': 'https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js',
    'bootstrap4-css': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.2/css/bootstrap.min.css',
    'bootstrap4-popper':
        'https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js',
    'bootstrap4': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.2/js/bootstrap.min.js',
    'dataTable': 'https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js',
    'dataTable-bootstrap4': 'https://cdn.datatables.net/1.10.19/js/dataTables.bootstrap4.min.js',
    'dataTable-css': 'https://cdn.datatables.net/1.10.19/css/dataTables.bootstrap4.min.css',
    'gstatic-charts': 'https://www.gstatic.com/charts/loader.js',
}

class ReportGenerator(object):

    def __init__(self, html_path):
        self.hw = HtmlWriter(html_path)
        self.hw.open_tag('html')
        self.hw.open_tag('head')
        for css in ['bootstrap4-css', 'dataTable-css']:
            self.hw.open_tag('link', rel='stylesheet', type='text/css', href=URLS[css]).close_tag()
        for js in ['jquery', 'bootstrap4-popper', 'bootstrap4', 'dataTable', 'dataTable-bootstrap4',
                   'gstatic-charts']:
            self.hw.open_tag('script', src=URLS[js]).close_tag()

        self.hw.open_tag('script').add(
            "google.charts.load('current', {'packages': ['corechart', 'table']});").close_tag()
        self.hw.open_tag('style', type='text/css').add("""
            .colForLine { width: 50px; }
            .colForCount { width: 100px; }
            .tableCell { font-size: 17px; }
            .boldTableCell { font-weight: bold; font-size: 17px; }
            """).close_tag()
        self.hw.close_tag('head')
        self.hw.open_tag('body')
        self.record_info = {}

    def write_content_div(self):
        self.hw.open_tag('div', id='report_content').close_tag()

    def write_record_data(self, record_data):
        self.hw.open_tag('script', id='record_data', type='application/json')
        self.hw.add(json.dumps(record_data))
        self.hw.close_tag()

    def write_script(self):
        self.hw.open_tag('script').add_file('report_html.js').close_tag()

    def finish(self):
        self.hw.close_tag('body')
        self.hw.close_tag('html')
        self.hw.close()

def main():
    sys.setrecursionlimit(MAX_CALLSTACK_LENGTH * 2 + 50)
    parser = argparse.ArgumentParser(description='report profiling data')
    parser.add_argument('-i', '--record_file', nargs='+', default=['perf.data'], help="""
                        Set profiling data file to report. Default is perf.data.""")
    parser.add_argument('-o', '--report_path', default='report.html', help="""
                        Set output html file. Default is report.html.""")
    parser.add_argument('--min_func_percent', default=0.01, type=float, help="""
                        Set min percentage of functions shown in the report.
                        For example, when set to 0.01, only functions taking >= 0.01%% of total
                        event count are collected in the report. Default is 0.01.""")
    parser.add_argument('--min_callchain_percent', default=0.01, type=float, help="""
                        Set min percentage of callchains shown in the report.
                        It is used to limit nodes shown in the function flamegraph. For example,
                        when set to 0.01, only callchains taking >= 0.01%% of the event count of
                        the starting function are collected in the report. Default is 0.01.""")
    parser.add_argument('--add_source_code', action='store_true', help='Add source code.')
    parser.add_argument('--source_dirs', nargs='+', help='Source code directories.')
    parser.add_argument('--add_disassembly', action='store_true', help='Add disassembled code.')
    parser.add_argument('--binary_filter', nargs='+', help="""Annotate source code and disassembly
                        only for selected binaries.""")
    parser.add_argument('--ndk_path', nargs=1, help='Find tools in the ndk path.')
    parser.add_argument('--no_browser', action='store_true', help="Don't open report in browser.")
    parser.add_argument('--show_art_frames', action='store_true',
                        help='Show frames of internal methods in the ART Java interpreter.')
    args = parser.parse_args()

    # 1. Process args.
    binary_cache_path = 'binary_cache'
    if not os.path.isdir(binary_cache_path):
        if args.add_source_code or args.add_disassembly:
            log_exit("""binary_cache/ doesn't exist. Can't add source code or disassembled code
                        without collected binaries. Please run binary_cache_builder.py to
                        collect binaries for current profiling data, or run app_profiler.py
                        without -nb option.""")
        binary_cache_path = None

    if args.add_source_code and not args.source_dirs:
        log_exit('--source_dirs is needed to add source code.')
    build_addr_hit_map = args.add_source_code or args.add_disassembly
    ndk_path = None if not args.ndk_path else args.ndk_path[0]

    # 2. Produce record data.
    record_data = RecordData(binary_cache_path, ndk_path, build_addr_hit_map)
    for record_file in args.record_file:
        record_data.load_record_file(record_file, args.show_art_frames)
    record_data.limit_percents(args.min_func_percent, args.min_callchain_percent)

    def filter_lib(lib_name):
        if not args.binary_filter:
            return True
        for binary in args.binary_filter:
            if binary in lib_name:
                return True
        return False
    if args.add_source_code:
        record_data.add_source_code(args.source_dirs, filter_lib)
    if args.add_disassembly:
        record_data.add_disassembly(filter_lib)

    # 3. Generate report html.
    report_generator = ReportGenerator(args.report_path)
    report_generator.write_script()
    report_generator.write_content_div()
    report_generator.write_record_data(record_data.gen_record_info())
    report_generator.finish()

    if not args.no_browser:
        open_report_in_browser(args.report_path)
    log_info("Report generated at '%s'." % args.report_path)

if __name__ == '__main__':
    main()


from django.shortcuts import render_to_response, redirect
from django.template.loader import render_to_string
from deviceapp.models import *
from deviceapp.forms import *
from django.template import RequestContext, Context, loader
from django.contrib.admin.views.decorators import staff_member_required
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required
from django.utils.html import escape
from django.shortcuts import render
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
import views_email	as email_view
import commission as commission
from django.core.cache import cache
import json
import math
import difflib
import locale
import time

def index(request):
	items = Item.objects.filter(liststatus='active').order_by('savedcount')[:9]
	return render_to_response('general/index.html',{'featured':items},context_instance=RequestContext(request))

def buy(request):
	return HttpResponseRedirect('/productsearch/veterinary/all/all')

def tos(request):
	return render_to_response('general/tos.html',{'tos':True},context_instance=RequestContext(request))

def giveback(request):
	return render_to_response('general/giveback.html',{'giveback':True},context_instance=RequestContext(request))

def commissionpage(request):
	return render_to_response('general/commission.html',{'commission':True},context_instance=RequestContext(request))

def privacypolicy(request):
	return render_to_response('general/privacypolicy.html',{'privacypolicy':True},context_instance=RequestContext(request))

def categories(request):
	categories = Category.objects.all().order_by('name')
	return render_to_response('general/categories.html',{'categories':categories},context_instance=RequestContext(request))

def faq(request):
	return render_to_response('general/faqs.html',{'faq':True},context_instance=RequestContext(request))

def pvp(request):
	return render_to_response('general/pvp.html',{'pvp':True},context_instance=RequestContext(request))

def about(request):
	return render_to_response('general/about.html',{'about':True},context_instance=RequestContext(request))

def my_404_view(request):
	return render_to_response('404.html',context_instance=RequestContext(request))

def my_500_view(request):
	return render_to_response('500.html',context_instance=RequestContext(request))

def buyerprotect(request):
	return render_to_response('general/buyerprotect.html',{'buyerprotect':True,'PHONE_NUMBER':settings.CONTACT_PHONE_NUMBER},context_instance=RequestContext(request))

def contact(request):
	return render_to_response('general/contact.html',{'contact':True},context_instance=RequestContext(request))

def contactform(request):
	form = ContactForm(request.POST)
	if form.is_valid():
		# General user sign-up
		name = form.cleaned_data['name']
		email = form.cleaned_data['email']
		message = form.cleaned_data['message']
		user = None
		if request.user.is_authenticated():
			user = request.user.basicuser
		cf = Contact(user=user,name=name,email=email,message=message)
		cf.save()
		email_view.composeContactForm(cf)
		return render_to_response('general/contact.html',{'contact':True,'success':True},context_instance=RequestContext(request))
	return render_to_response('general/contact.html',{'contact':True,'failure':True},context_instance=RequestContext(request))

### Anytime there is an error, send the user here ####
def error(request,errorname):
	errormessage = ''
	if errorname == 'notpost':
		errormessage = 'There was an invalid POST request'
	if errorname == 'signup':
		errormessage = 'There was an error signing up'
	if errorname == 'itemdoesnotexist':
		errormessage = 'This item has been taken down or sold'
	return render_to_response('general/error.html',{'errormessage':errormessage},context_instance=RequestContext(request))

@login_required
def staffOverviewForward(request):
	return HttpResponseRedirect('/staff/overview/dashboard')

@staff_member_required
def staffOverview(request,type):
	if request.user.is_staff and request.user.is_authenticated():
		dict = {}
		if type == 'dashboard':
			# Get relevant objects
			pi = PurchasedItem.objects.all()
			users = BasicUser.objects.all().count()
			active_items = Item.objects.filter(liststatus="active").count()
			checkouts = Checkout.objects.filter(purchased=True).count()
			bp = BankPayout.objects.all()
			cp = CheckPayout.objects.all()

			total_amount_charged = 0
			number_items_sold = 0
			online_commission_revenue = 0
			for p in pi:
				total_amount_charged += p.quantity*p.unit_price
				number_items_sold += p.quantity
				if not p.cartitem.item.commission_paid:
					online_commission_revenue += commission.purchaseditemCommission(p)

			offline_commission_revenue = 0
			total_paidout = 0
			total_charity = 0
			total_ccfee_revenue = 0
			for b in bp:
				total_paidout += b.amount
				total_ccfee_revenue += b.cc_fee
				total_charity += b.total_charity
			for c in cp:
				total_paidout += c.amount
				total_ccfee_revenue += c.cc_fee
				total_charity += c.total_charity
			for comm in Commission.objects.all():
				offline_commission_revenue += comm.amount

			total_commission_revenue = online_commission_revenue + offline_commission_revenue
			tobepaidout = total_amount_charged-total_paidout-total_commission_revenue-total_ccfee_revenue-total_charity
			total_revenue = total_commission_revenue+total_ccfee_revenue
			dict = {'dashboard':True,'users':users,'checkouts':checkouts,'items_sold':number_items_sold,
			'offline_commission_revenue':offline_commission_revenue,'amount_sold':total_amount_charged,
			'total_paidout':total_paidout,'active_items':active_items, 'tobepaidout':tobepaidout,
			'online_commission_revenue':online_commission_revenue,
			'total_commission_revenue':total_commission_revenue,
			'total_revenue':total_revenue,
			'total_charity':total_charity}
		elif type == 'purchaseditem':
			dict['purchaseditems'] = PurchasedItem.objects.all()
		elif type == 'bankpayout':
			dict['bankpayout'] = BankPayout.objects.all()
		elif type == 'checkpayout':
			dict['checkpayout'] = CheckPayout.objects.all()
		elif type == 'commission':
			dict['commission'] = Commission.objects.all()
		elif type == 'checkpayment':
			checkpayment = []
			for order in Order.objects.all():
				if hasattr(order.payment,'checkpayment'):
					checkpayment.append(order)
			dict['checkpayment'] = checkpayment
		return render_to_response('general/adminoverview.html',dict,context_instance=RequestContext(request))
	return HttpResponseRedirect("/")

@staff_member_required
def staffMarkAsSent(request):
	if request.user.is_staff and request.user.is_authenticated() and request.method=="POST":
		cpid = request.POST.get('cp_id','')
		cpval = request.POST.get('cp_val','')
		cp = CheckPayout.objects.get(id=cpid)
		if cpval == 'sent':
			cp.sent = False
		else:
			cp.sent = True
		cp.save()
		return HttpResponse(json.dumps({'status':201,'sent':cp.sent}), content_type='application/json')
	else:
		return HttpResponse(json.dumps({'status':500}), content_type='application/json')

@staff_member_required
def staffMarkAsReceived(request):
	if request.user.is_staff and request.user.is_authenticated() and request.method=="POST":
		reid = request.POST.get('re_id','')
		reval = request.POST.get('re_val','')
		order = Order.objects.get(id=reid)
		if reval == 'received':
			order.payment.checkpayment.received = True
			for pi in order.purchaseditem_set.all():
				notification = SoldPaymentNotification(user=pi.seller,purchaseditem=pi)
				notification.save()
		else:
			order.payment.checkpayment.received = False
		order.payment.checkpayment.save()
		return HttpResponse(json.dumps({'status':201,'received':order.payment.checkpayment.received}), content_type='application/json')
	else:
		return HttpResponse(json.dumps({'status':500}), content_type='application/json')


"""Rectangle."""
import pygame

class Rectangle:
    """Rectangle."""

    def __init__(self, name, pos, width, height, color):
        """
        Initialize Rectangle Sprite.

        Args:
            name (string): The name of the sprite
            pos (tuple): Position of the sprite
            width (int): Width of the sprite
            height (int): Height of the sprite
            color (tuple): Color of the sprite
        """
        self.name = name

        self.x = pos[0]
        self.y = pos[1]

        self.width = width
        self.height = height
        self.color = color
        self.show = True

        self.image = pygame.Surface((width, height))
        self.rect = self.image.get_rect(center=(pos))

        self.image.fill(self.color)

    def changePos(self, x, y):
        """
        Change the position of the sprite.

        Args:
            x (int): x of the sprite.
            y (int): y of the sprite.
        """
        self.x = x
        self.y = y

        self.rect.center = (x, y)


from threading import Thread
from collections import namedtuple
try:
    from .event import ResponseEvent
except (SystemError, ValueError, ImportError):
    from event import ResponseEvent

class AbstractSubprocessReader(object):
    def input_reader(self, stream):
        return stream()

    def input_container(self, result):
        return result

    def input_exception(self, result, o):
        return False

    def stdout_reader(self, stream):
        return stream.readline()

    def stdout_container(self, result):
        jobno, offer, time_frame = result
        return ResponseEvent(
            int(jobno),  # ValueError
            offer,
            time_frame
        )
    def std_out_exception(self, result, o):
        if result[0] is 'E':
            print_console("{} Exception".format(o))
            return True
        elif result[0] is 'K':
            print_console("{} killed".format(o))
            return True

    def print_console(self, msg):
        if self.logging_on:
            print("SubprocessReader {}".format(msg))

class SubprocessReader(AbstractSubprocessReader):
    """
    A class representing a SubprocessReader.
    It contains an id for tracking with the ability
    to handle different types of streamed events.

    Parameters
    ----------
    identifier : int
        The human-readable identifier
    stream: input | stdout
        Stream to read
    expected : int, length of stream
        Length of message in stream
    log: bool
        Turns printing on/off
    option : 'stdout' or 'input'
        The SubprocessReader will read from the subprocess
        sys.stdout or input() stream and feed these back
        into the parent process events queue.
    """
    def __init__(
        self, identifer,
        stream, events_queue,
        expected, log=False, option='input'
    ):
        self.id = identifer
        self._s = stream
        self._q = events_queue
        self._e = expected
        self.logging_on = log
        self._l = self.print_console
        if option == 'input':
            self._r = self.input_reader
            self._c = self.input_container
            self._ex = self.input_exception
        elif option == 'stdout':
            self._r = self.stdout_reader
            self._c = self.stdout_container
            self._ex = self.std_out_exception

        def _stream_to_queue(
            id, stream, reader, container,
            events_queue, expected, logger,
            exception
        ):
            """
            Stream reading.
            """
            while True:
                try:
                    result = reader(stream).strip().split(', ')
                    if len(result) == expected:
                        try:
                            event = container(result)
                            events_queue.put(event)
                        except ValueError:
                            if exception(result, id):
                                break
                    else:
                        break
                except EOFError:
                    break
            logger("{} stopped".format(id))

        self.t = Thread(
            target=_stream_to_queue,
            args=(
                self.id,
                self._s,
                self._r,
                self._c,
                self._q,
                self._e,
                self._l,
                self._ex,
            )
        )
        self.t.daemon = True
        self.t.start()


# Databricks notebook source
## TEST PARAMS
# AS_OF_DT = 20230628
# STOFCK_SYMBOL = 0,1,2,3,4,5

# COMMAND ----------

## IMPORT LIBRARY
import requests
import json

# COMMAND ----------

## NOTEBOOK PARAMS
dbutils.widgets.text("PAGE", "")
dbutils.widgets.text("AS_OF_DT", "")

## ASSIGN VARAIBLES
in_page = dbutils.widgets.get("PAGE")
in_page_list = in_page.split(',')
as_of_dt = dbutils.widgets.get("AS_OF_DT")

global error_log

# COMMAND ----------

def extract_delisted_companies(page):
    r = requests.get('https://financialmodelingprep.com/api/v3/delisted-companies?page='+page+'&apikey='+dbutils.secrets.get(scope="keyvault_scope_01",key="apikey01"))
    if r.status_code == 200:
        try:
            return r.text
        except SomeError:
            return None
    else:
        error_log += f'Could not load data for page{page}, error with HTTP REQUEST ERROR '+str(r.status_code)

# COMMAND ----------

def write_on_storage_account(json_string, output_path):
    # create the path if it doesn't exist
    dbutils.fs.mkdirs(output_path.rsplit("/", 1)[0])
    with open('/dbfs' +output_path, 'w') as file:
        file.write(json_string)

# COMMAND ----------

## MAIN PROCESS
error_log = ''
for each_page in in_page_list:
    output_filepath = '/mnt/test-container-01/delisted_companies/'+as_of_dt+'/page'+each_page+'.json'
    r_json = extract_delisted_companies(each_page)
    write_on_storage_account(r_json, output_filepath)

# COMMAND ----------

## EXIT NOTEBOOK
if error_log == '':
    dbutils.notebook.exit(json.dumps({'status': 'SUCCEEDED'}))
else:
    dbutils.notebook.exit(json.dumps({'status': 'SUCCEEDED', 'message': error_log}))


from flask_wtf import FlaskForm
from wtforms import SubmitField, URLField, ValidationError
from wtforms.validators import URL, DataRequired, Length, Optional, Regexp

from settings import (DATA_REQUIRED_MESSAGE, INVALID_SHORT_MESSAGE,
                      ORIGINAL_MAX_LEN, PATTERN_FOR_SHORT, SHORT_MAX_LEN,
                      URL_VALIDATOR_MESSAGE)
from .models import URLMap

LABEL_ORIGINAL = 'Длинная ссылка'
LABEL_SHORT = 'Ваш вариант короткой ссылки'
EXIST_SHORT_MESSAGE = 'Имя {short_name} уже занято!'
SUBMIT_BUTTON = 'Создать'

class YacutForm(FlaskForm):
    original_link = URLField(
        LABEL_ORIGINAL,
        validators=[
            DataRequired(message=DATA_REQUIRED_MESSAGE),
            Length(max=ORIGINAL_MAX_LEN),
            URL(message=URL_VALIDATOR_MESSAGE)
        ]
    )
    custom_id = URLField(
        LABEL_SHORT,
        validators=[
            Optional(),
            Length(max=SHORT_MAX_LEN, message=INVALID_SHORT_MESSAGE),
            Regexp(regex=PATTERN_FOR_SHORT, message=INVALID_SHORT_MESSAGE)
        ]
    )
    submit = SubmitField(SUBMIT_BUTTON)

    def validate_custom_id(form, field):
        if URLMap.get(short=field.data):
            raise ValidationError(
                EXIST_SHORT_MESSAGE.format(short_name=field.data)
            )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 12 15:25:12 2018

@author: xgotda
"""

# import sys
from PeptideClass import Peptide
import staticVariables as sV

class Ions:

    def __init__(self):

        self.title = ""
        self.charge = 0
        self.RT = 0
        self.scanNo = 0
        self.MaxInts = 0
        self.pepmass = Peptide()
        self.Mass = 0  # Calculated using pepmass and charge.
        self.valid = False  # flag denoting if we have had at least one match
        self.npWorking = []
        self.fragments = {}
        self.fragmentCount = 0

    def calculateMass(self):
        ''' Calculate the mass of the peptide
            @return: Mass of original peptide
            @rtype: float '''
        self.Mass = self.charge * (self.pepmass.mz - sV._Hplus)

    def addFragment(self, key, intensity, slItem):
        ''' Adds the intensity value for the given key
            to the fragments dictionary.
            @return: amends fragments dictionary with new value added
            @rtype: dictionary '''
        if self.fragments.get(key, False):
            self.fragments[key][slItem.chtype] = intensity
        else:
            self.fragments[key] = { 0 : slItem.ptype,
                                    slItem.chtype : intensity}
        self.fragmentCount += 1
        self.valid = True


import frappe
from cre.investorexp import get_comments
def get_context(context):
    context.name = frappe.form_dict.name
    context.doctype = 'Property'
    context.doc = frappe.get_doc(context.doctype, context.name)
    context.no_breadcrumbs = False
    context.show_sidebar = True
    context.likes = frappe.db.get_value(context.doctype, context.name, '_liked_by')
    context.user = frappe.session.user
    if context.user in context.likes:
    	context.user_liked = 1
    else:
    	context.user_liked = 0

    cm_filters = {'comment_type': 'Comment',
    	'reference_doctype': context.doctype,
    	'reference_name': context.name
    }
    context.new_comments = frappe.db.get_all('Comment', filter=cm_filters)
    context.comments = []

    for comment in get_comments(context.doctype, context.name, 0, context.user):
    	comment_data = comment.get('data')
    	if comment_data.get('by'):
    		context.comments.append(comment)

    # context.comments = [c for c in get_comments(context.doctype, context.name, 0, context.user) if 'by' in c.keys() ]

    load_feedback(context)

def load_feedback(context):
		user = frappe.session.user

		feedback = frappe.get_all('CRE Feedback',
			fields=['like'],
			filters=dict(
				reference_doctype=context.doctype,
				reference_name=context.name,
				ip_address=frappe.local.request_ip,
				owner=user
			)
		)

		like_count = 0

		if frappe.db.count('CRE Feedback'):
			like_count = frappe.db.count('CRE Feedback',
				filters = dict(
					reference_doctype = context.doctype,
					reference_name = context.name,
					like = True
				)
			)

		context.user_feedback = feedback[0] if feedback else ''
		context.like_count = like_count


import asyncio, io, time

import aiohttp
import discord
from redbot.core import Config, checks, commands
from redbot.core.utils.chat_formatting import box, pagify

BaseCog = getattr(commands, "Cog", object)

class AutoKick(BaseCog):
    """Bye bye"""

    def __init__(self, bot):
        self.bot = bot
        self.config = Config.get_conf(self, identifier=974374573)
        default_guild = {}
        self.config.register_guild(**default_guild)

    @commands.Cog.listener()
    async def on_voice_state_update(self, member, before, after):
        if after.channel is not None:
            if (after.channel.id == 845055977182330901):
                time.sleep(1)
                await member.move_to(channel=None, reason="Get rekd nerd")


from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation
from tensorflow.keras import optimizers

def bulid_model():
    model = Sequential()

    model.add(Dense(16, input_shape=(8,)))  # 输入层，28*28=784
    model.add(Activation('softmax'))  # 激活函数是tanh
    # model.add(Dropout(0.5))  # 采用50%的dropout

    model.add(Dense(16))  # 隐藏层节点500个
    model.add(Activation('softmax'))
    # model.add(Dropout(0.5))

    # model.add(Dense(16))  # 隐藏层节点500个
    # model.add(Activation('softmax'))
    # model.add(Dropout(0.5))

    model.add(Dense(1))  # 输出结果是10个类别，所以维度是10

    model.compile(loss='mse', optimizer=optimizers.Adam(learning_rate=0.01), metrics=['mae'])
    return model


""" TMVA or Sklearn based Classifier to be trained in separating signals from backgrounds
"""
## stdl
import os, time, copy
from array import array
from collections import OrderedDict
import multiprocessing

## PyPI
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle
from root_numpy import root2array, tree2array
import numpy as np
import pandas as pd
import pickle, cPickle
from sklearn import utils

## local
from hpana import log
from hpana.categories import CLASSIFIER_CATEGORIES, TAU_IS_TRUE, ANTI_TAU
from hpana.mva import NN_HYPERPARAMS

## ROOT
import ROOT
from ROOT import TMVA

from hpana.mva import TRAINING_MASS_BINS

##----------------------------------------------------------------------------------
## Base classifier class
##----------------------------------------------------------------------------------
class Classifier(TMVA.Factory):
    def __init__(self,
                 method_type=TMVA.Types.kBDT,
                 method_name="BDT",
                 weight_file=None,
                 features=[],
                 output="TMVA.root",
                 channel="taujet",
                 outdir="./",
                 factory_name="TMVAClassification",
                 params="V:!Silent:Color:DrawProgressBar:Transformations=D,G:AnalysisType=Classification",):
        self.output = ROOT.TFile("%s/%s"%(outdir, output), "RECREATE")
        self.features = features
        self.factory_name = factory_name
        self.channel = channel
        self.outdir = outdir
        self.params = params
        self.dataloader = TMVA.DataLoader(self.outdir)
        self.method_type = method_type
        self.method_name = method_name

        ## instantiate Base
        super(Classifier, self).__init__(factory_name, self.output, self.params)

        ## add input features to the TMVA.DataLoader
        for var in self.features:
            self.dataloader.AddVariable(var.tformula, var.title,
                                        var.unit if hasattr(var, "unit") else "",)

        ## features dict in order to keep references for TMVA.Reader !
        self.features_dict = OrderedDict()

        ## retrieve trained model
        if weight_file:
            self.weight_file = weight_file
            ## instantiate Reader for model evaluation
            self.reader = TMVA.Reader()

            for feat in self.features:
                self.features_dict[feat.tformula] = array('f', [-1.])
                self.reader.AddVariable(feat.tformula, self.features_dict[feat.tformula]) #<! PASS REFRENCES HERE

            self.reader.BookMVA(self.method_name, self.weight_file)

    def book(self, params):
        """
        Book the Classifier method (set all the parameters)
        """
        pstring = ""
        if isinstance(params, dict):
            plist = []
            for p, val in params.iteritems():
                plist.append("%s=%s"%(p, val))
                pstring = ":".join(plist)
        elif isinstance(params, (list, tuple)):
            pstring = ":".join(params)

        elif isinstance(params, str):
            pstring = params
        else:
            raise TypeError("unsupported type, {}".format(params))

        self.BookMethod(self.dataloader, self.method_type, self.method_name, pstring)

    def train(self, backgrounds, signals,
              method_params={},
              signal_masses=[],
              fold_cut=None,
              selections=ROOT.TCut(""),
              bkg_weights=None,
              sig_weights=None,
              treename="NOMINAL"):
        """

        """
        if signal_masses:
            signals = filter(lambda s: s.mass in signal_masses, signals)

        ## prepare bkg and sig tree
        bkg_tchain = ROOT.TChain(treename)
        bkg_files = []
        for bkg in backgrounds:
            for ds in bkg.datasets:
                for ifile in ds.files:
                    bkg_tchain.Add(ifile)

        sig_tchain = ROOT.TChain(treename)
        for sig in signals:
            for ds in sig.datasets:
                for ifile in ds.files:
                    sig_tchain.Add(ifile)

        if not isinstance(selections, ROOT.TCut):
            selections = ROOT.TCut(selections)

        bkg_cuts = selections
        sig_cuts = selections
        if bkg_weights:
            bkg_cuts *= bkg_weights
        if sig_weights:
            sig_cuts *= sig_weights
        if fold_cut:
            bkg_cuts *= fold_cut
            sig_cuts *= fold_cut

        self.dataloader.AddSignalTree(sig_tchain, 1.0)
        self.dataloader.AddBackgroundTree(bkg_tchain, 1.0)

        self.book(method_params)
        self.dataloader.PrepareTrainingAndTestTree(
            sig_cuts, bkg_cuts,'SplitMode=Random:NormMode=NumEvents:!V')
        self.TrainAllMethods()
        self.EvaluateAllMethods()

    def predict(self, features_dict):
        """
        """
        assert os.path.isfile(self.weight_file), "weight file is not provided"
        ## update referenced features
        for feat, val in features_dict.iteritems():
            self.features_dict[feat] = val
        return self.reader.EvaluateMVA(self.method_name)

##----------------------------------------------------------------------------------
## Base classifier class
##----------------------------------------------------------------------------------
class SClassifier(GradientBoostingClassifier):
    """
    """
    # try:
    # MODEL_NAME_STR_FORMAT = "model_{0}_channel_{1}_mass_{2}_ntracks_{3}_nfolds_{4}_fold_{5}_nvars_{6}_learnRate_{7}_minSampsLeaf_{8}_nEst_{9}_minSampsSplit_{10}_maxDepth_{11}.pkl" #<! name, channel, ntracks, nfolds, fold, n_vars
    # except:
    MODEL_NAME_STR_FORMAT = "model_{0}_channel_{1}_mass_{2}_ntracks_{3}_nfolds_{4}_fold_{5}_nvars_{6}.pkl"

    def __init__(self, channel,
                    name="GB",
                    mass_range=[],
                    features=[],
                    train_df=None,
                    valid_df=None,
                    eval_df=None,
                    weight_sample=False,
                    is_trained=False,
                    kfolds=5,
                    fold_num=0,
                    ntracks=1,
                    optimize=False,
                    sigs=[],
                    bkgs=[],
                    **params):
        log.debug("Initializing SClassifier ...")
        self.kparams = params
        self.channel = channel
        self.name = name
        self.features = features
        self.mass_range = mass_range
        self.train_df = train_df
        self.valid_df = valid_df
        self.eval_df = eval_df
        self.weight_sample = weight_sample
        self.kfolds = kfolds
        self.fold_num = fold_num
        self.ntracks = ntracks
        self.is_trained = is_trained
        self.hyperparams = params
        self.optimize = optimize
        self.sigs = sigs
        self.bkgs = bkgs

        if self.optimize:
            MODEL_NAME_STR_FORMAT = "model_{0}_channel_{1}_mass_{2}_ntracks_{3}_nfolds_{4}_fold_{5}_nvars_{6}_learnRate_{7}_minSampsLeaf_{8}_nEst_{9}_minSampsSplit_{10}_maxDepth_{11}.pkl" #<! name, channel, ntracks, nfolds, fold, n_vars
        else:
            MODEL_NAME_STR_FORMAT = "model_{0}_channel_{1}_mass_{2}_ntracks_{3}_nfolds_{4}_fold_{5}_nvars_{6}.pkl" #<! name, channel, ntracks, nfolds, fold, n_vars

        ## instantiate the base
        super(SClassifier, self).__init__(**params)

    @staticmethod
    def prepare_data(backgrounds, signals, features,
                     branches=[],
                     category=None,
                     channel="taujet",
                     treename="NOMINAL",
                     train_data="CLF_DATA.pkl",
                     data_lumi=36200,
                     overwrite=False,
                     truth_match_tau=True):
        """Training input as pandas DataFrame
        """
        ## first check if all the samples are already available in the training Dataframe
        missing_bkgs = []
        missing_sigs = []
        cached_dframe = None
        if os.path.isfile(train_data) and not overwrite:
            log.info("Reading training data from %s"%train_data)
            with open(train_data, "r") as cache:
                cached_dframe = cPickle.load(cache)
            for b in backgrounds:
                if not (b.name in cached_dframe.index):
                    log.warning("missing %s in %s Dataframe"%(b.name, train_data))
                    missing_bkgs += [b]

            for s in signals:
                if not (s.name in cached_dframe.index):
                    log.warning("missing %s in %s Dataframe"%(s.name, train_data))
                    missing_sigs += [s]
            if not missing_sigs and not missing_bkgs:
                log.info("All requested samples are available in %s Dataframe"%train_data)
                return cached_dframe
        else:
            missing_bkgs = backgrounds
            missing_sigs = signals

        if category is None:
            category = CLASSIFIER_CATEGORIES[channel]
        cuts = category.cuts

        ## keep event number (is needed for kfold training)
        if not branches:
            branches = ["event_number"] + [ft.tformula for ft in features]

        ## feature name as column label
        columns = {}
        for feat in features:
            columns[feat.tformula] = feat.name

        ## backgrounds
        bkg_dfs = []
        for bkg in missing_bkgs:
            log.info("Adding %s bkg ..."%bkg.name)
            bfiles = []
            ## treat QCD fakes properly
            if "QCD" in bkg.name:

                ## @NOTE defensive copying to avoid clashing with cuts for other samples
                category_cp = copy.deepcopy(category)
                ## only FF weights are applicable here
                ws = bkg.ff_weights(categories=[category_cp])["NOMINAL"][category.name]
                ws = "*".join(ws)

                ## add antitau cut
                category_cp.tauid = ANTI_TAU
                category_cp.truth_tau = None #<! not applicable to DATA
                cuts = category_cp.cuts

                log.debug("Cut: %r\n"%cuts.GetTitle())
                log.debug("Weight: %r\n"%ws)

                ## pool of workers
                pool = multiprocessing.Pool()
                pool_res = []

                log.debug(
                    "**"*30 + " Parallel processing %i datasets(%r)  for %s "%(len(bkg.data.datasets), bkg.data.streams, bkg.name) + "**"*30)
                for ds in bkg.data.datasets:
                    ##@FIXME missing 2018 triggers in 2015-2017 v06 ntuples
                    if "DATA2018" in ds.name:
                        d_streams = ["2018"]
                    else:
                        d_streams = ["2015", "2016", "2017"]

                    trigger = bkg.triggers(categories=[category_cp], data_streams=d_streams, dtype="DATA")[category_cp.name]
                    selection = (trigger + cuts).GetTitle()

                    bfiles = ds.files
                    if not bfiles:
                        log.warning("No root file is found for %s"%ds.name)
                        continue
                    if empty_tree(bfiles, treename=treename):
                        log.warning("%s tree is empty for %s dataset; skipping!"%(treename, ds.name))
                        continue

                    kargs = {"treename": treename, "branches": branches+[ws], "selection": selection, "warn_missing_tree": True}
                    pool_res += [pool.apply_async(root2array, (bfiles,), dict(kargs))]

                ## close the pool and wait for the workers to finish
                pool.close()
                pool.join()

                ## retrive the pool output and create Dataframes
                b_dfs = []
                for res in pool_res:
                    r_arr = res.get(3600)
                    p_df = pd.DataFrame(r_arr.flatten())
                    columns[ws] = "weight"
                    p_df = p_df.rename(columns=columns)
                    log.debug("Weight sum: %r\n"%np.sum(p_df["weight"]))
                    log.debug("#events: %r\n"%p_df.shape[0])
                    b_dfs += [p_df]

                bkg_df = pd.concat(b_dfs, ignore_index=True, sort=False)
                ## class label for bkg
                bkg_df["class_label"] = pd.Series(np.zeros(bkg_df.size))
                bkg_dfs += [bkg_df]
                log.debug("--"*70)

            else:
                b_dfs = []
                pool = multiprocessing.Pool()
                pool_res = []
                pool_ws = []

                ## add trigger
                trigger = bkg.triggers(categories=[category], data_streams=bkg.config.data_streams, dtype="MC")[category.name]
                for ds in bkg.datasets:
                    log.debug("**"*30 + ds.name + "**"*30)
                    bfiles = ds.files
                    if not bfiles:
                        log.warning("No root file is found for %s"%ds.name)
                        continue
                    if empty_tree(bfiles, treename=treename):
                        log.warning("%s tree is empty for %s dataset; skipping!"%(treename, ds.name))
                        continue
                    if truth_match_tau:
                        cuts = category.cuts + TAU_IS_TRUE

                    ## common Scale Factors
                    ws = bkg.weights(categories=[category]).values()[0]
                    ws = "*".join(ws)

                    ## add luminosity weight
                    if ds.lumi_weight:
                        lumi_weight = bkg.data_lumi(ds.stream) * ds.lumi_weight
                    else:
                        lumi_weight = (bkg.data_lumi(ds.stream) * ds.xsec_kfact_effic) / ds.events
                    ws = ws + "*{}".format(lumi_weight)
                    pool_ws += [ws] #<! @NOTE each dataset has a different weight --> keep them to properly rename all to 'weight'
                    selection = (trigger + cuts).GetTitle()
                    log.debug("Cut: %r\n"%selection)
                    log.debug("Weight: %r\n"%ws)
                    kargs = {"treename": treename, "branches": branches+[ws], "selection": selection, "warn_missing_tree": True}
                    pool_res += [pool.apply_async(root2array, (bfiles,), dict(kargs))]

                ## close the pool and wait for the workers to finish
                pool.close()
                pool.join()

                ## retrive the pool output and create Dataframes
                for bds, pw, res in zip(bkg.datasets, pool_ws, pool_res):
                    r_arr = res.get(3600)
                    p_df = pd.DataFrame(r_arr.flatten())
                    ## @NOTE the weight string changes due to LUMI factor from dataset to dataset --> rename it to avoid concatenation confusion
                    columns[pw] = "weight"
                    p_df = p_df.rename(columns=columns)
                    log.debug("Weight sum: %r\n"%np.sum(p_df["weight"]))
                    log.debug("#events: %r\n"%p_df.shape[0])
                    b_dfs += [p_df]

                bkg_df = pd.concat(b_dfs, ignore_index=True, sort=False)
                ## class label for bkg
                bkg_df["class_label"] = pd.Series(np.zeros(bkg_df.size))
                bkg_dfs += [bkg_df]
                log.debug("--"*70)

        ## signals
        sig_dfs = []
        for sig in missing_sigs:
            log.info("Adding %s signal ..."%sig.name)
            ## add trigger
            trigger = bkg.triggers(categories=[category], data_streams=sig.config.data_streams, dtype="MC")[category.name]
            cuts = category.cuts
            if truth_match_tau:
                cuts = category.cuts + TAU_IS_TRUE

            s_dfs = []
            for ds in sig.datasets:
                log.debug("**"*30 + ds.name +"**"*30)
                sfiles = ds.files
                if not sfiles:
                    log.warning("No root file is found for %s"%sig.name)
                    continue

                ## common Scale Factors
                ws = sig.weights(categories=[category]).values()[0]
                ws = "*".join(ws)

                ## add luminosity weight
                if ds.lumi_weight:
                    lumi_weight = bkg.data_lumi(ds.stream) * ds.lumi_weight
                else:
                    lumi_weight = (bkg.data_lumi(ds.stream) * ds.xsec_kfact_effic) / ds.events
                ws = ws + "*{}".format(lumi_weight)
                selection = cuts.GetTitle()
                log.debug("Cut: %r\n"%selection)
                log.debug("Weight: %r\n"%ws)

                ## convert ROOT to numpy array
                s_arr = root2array(sfiles, treename=treename, branches=branches+[ws], selection=selection, warn_missing_tree=True)
                df = pd.DataFrame(s_arr.flatten())

                columns[ws] = "weight"
                df = df.rename(columns=columns)
                log.debug("Weight sum: %r\n"%np.sum(df["weight"]))
                log.debug("#events: %r\n"%df.shape[0])
                log.debug("--"*70)
                s_dfs += [df]

            s_df = pd.concat(s_dfs, ignore_index=True, sort=False)
            ## class label for bkg
            s_df["class_label"] = pd.Series(np.ones(s_df.size))
            sig_dfs += [s_df]

        ## concat sig & bkg and index based on the sample name
        m_keys = [bkg.name for bkg in missing_bkgs] + [sig.name for sig in missing_sigs]
        new_dframe = pd.concat(bkg_dfs+sig_dfs, keys=m_keys, sort=False)

        if cached_dframe is not None:
            keys = [sb.name for sb in backgrounds+signals]
            dframe = pd.concat([new_dframe, cached_dframe], sort=False)
        else:
            dframe = new_dframe

        if overwrite:
            log.warning("caching training data")
            os.system("rm %s"%train_data)

        with open(train_data, "a") as cache:
            cPickle.dump(dframe, cache, protocol=2)

        return dframe

##--------------------------------------------------------------------------
## util for parallel processing
##--------------------------------------------------------------------------
def train_model(model,
    train_df=None,
    features=[],
    positive_weights=True,
    balanced=True,
    outdir="",
    weight_sample=False,
    scale_features=True,
    save_model=True,
    overwrite=False,
    is_NN=False):
    """ Train a classifier and produce some validation plots.
    Parameters
    ----------
    model: DecisionTree;
        sklearn classifier
    train_df: pandas.DataFrame;
        training dataframe
    outdir: str;
        path to save the model and roc curve
    weight_sample: bool;
        whether to use the sample weight or go with the sklearn's default balanced weight procedure for the classes.
    save_model: bool;
        whether to save the trained model to disk or not

    Return
    ------
    trained model
    """
    if weight_sample:
        balanced = False
    if train_df is None:
        tr_df = model.train_df
    else:
        tr_df = train_df
    if not features :
        features = model.features
    # print 'tr_df.index.get_level_values(0).drop_duplicates():\n', tr_df.index.get_level_values(0).drop_duplicates()
    b_df = tr_df[tr_df["class_label"]==0]
    s_df = tr_df[tr_df["class_label"]==1]
    # print 'b_df.shape[0]: ', b_df.shape[0]
    # print 's_df.shape[0]: ', s_df.shape[0]

    # print 'b_df.index.get_level_values(0).drop_duplicates():\n', b_df.index.get_level_values(0).drop_duplicates()
    # print 's_df.index.get_level_values(0).drop_duplicates():\n', s_df.index.get_level_values(0).drop_duplicates()

    # log.info("is_NN=%s"%is_NN)
    # log.info("type(is_NN) %s" %type(is_NN))
    # log.info("is_NN==True %s"%(is_NN == True))
    # print 'balanced: ', balanced
    # print "is_NN:", is_NN
    # print 'weight_sample:', weight_sample

    oversample = False
    if is_NN == True:
        s_df["SampleWeight"] = 1.
        s_df["TruthMass"] = s_df.index.get_level_values(0)
        s_df["TruthMass"] = pd.to_numeric(s_df.TruthMass.replace({"Hplus": ""}, regex=True))
        MassPointWeight = [80./mass for mass in s_df["TruthMass"].values]
        s_df.insert(s_df.shape[1]-1, 'MassPointWeight', MassPointWeight)
        #b_df["TruthMass"] = np.random.choice( a=s_df["TruthMass"], size=b_df.shape[0] )
        train_masses = np.unique(s_df["TruthMass"].values)

        for i in train_masses:
            if weight_sample == True:
                b_df["SampleWeight"] = float(s_df.loc[s_df["TruthMass"]==i].shape[0])*b_df['weight'].abs() / b_df['weight'].abs().sum()
            else:
                b_df["SampleWeight"] = float(s_df.loc[s_df["TruthMass"]==i].shape[0])/b_df.shape[0]
            b_df["MassPointWeight"] = 80./i
            b_df["TruthMass"] = i
            if (i==80): b_df_masses = b_df.copy()
            else: b_df_masses = pd.concat([b_df_masses, b_df])
            if oversample:
                multiplication = b_df.shape[0] // s_df.loc[["Hplus"+str(i)]].shape[0]
                # print('multiplication: ', multiplication)
                s_df_tmp = s_df.loc["Hplus"+str(i)]
                s_df_tmp_2 = s_df.loc["Hplus"+str(i)]
                for _ in range(multiplication-1):
                    s_df_tmp_2 = pd.concat([s_df_tmp_2, s_df_tmp])
                s_df_tmp_2 = pd.concat([s_df_tmp_2, s_df_tmp[:(b_df.shape[0]-s_df_tmp_2.shape[0])]])
                # print 'concat ', i
                if (i==80): s_df_oversampled = s_df_tmp_2
                else: s_df_oversampled = pd.concat([s_df_oversampled, s_df_tmp_2])
            #     print 's_df_tmp_2.shape: ', s_df_tmp_2.shape
            #     print 's_df_oversampled.shape: ', s_df_oversampled.shape
            # print 'b_df_masses.shape: ', b_df_masses.shape

    if balanced and not is_NN:
        ## Set training weight of bkg events to 1. Signal events to N_bkg / N_sig.
        weight_sample = False
        b_df["BDT_Weight"] = 1
        s_df["BDT_Weight"] = float(b_df.shape[0])/float(s_df.shape[0])
        # X_weight = tr_df_ud["BDT_Weight"]
        log.info("Balancing training classes via weights in classifier. Setting signal weights to %s" %(float(b_df.shape[0])/float(s_df.shape[0])))

    else:
        log.info("Unbalanced training classes; signal events:{} | bkg events: {}".format(s_df.shape[0], b_df.shape[0]))

    background_weight = float(s_df.shape[0])/b_df.shape[0]
    train_weights = {0: background_weight, 1: 1}
    if is_NN == True:
        if oversample:
            tr_df_ud = pd.concat([b_df_masses, s_df_oversampled], axis=0)
        else:
            tr_df_ud = pd.concat([b_df_masses, s_df], axis=0)
    else:
        tr_df_ud = pd.concat([b_df, s_df], axis=0)

    # print('tr_df_ud.info()', tr_df_ud.info())
    tr_df_ud = shuffle(tr_df_ud, random_state=123)

    ## - - training arrays
    X_train = tr_df_ud[[ft.name for ft in features]]
    Y_train = tr_df_ud["class_label"]

    if weight_sample and not is_NN:
        log.info("Using event weight for training, events with negative weight are thrown away")
        X_weight = tr_df_ud["weight"]
        ## in order to avoid events with negative weight thrown away
        if positive_weights:
            log.info("using abs(weights)!")
            X_weight = np.absolute(X_weight.values)

    if scale_features:
        log.info("Scaling features using StandardScaler ...")
        scaler = StandardScaler()
        #X_train = scaler.fit_transform(X_train.values)

    mpath = os.path.join(outdir, model.name)
    is_trained = False
    if os.path.isfile(mpath):
        log.info("Loading %s model from disk ..."%mpath)
        with open(mpath, "r") as cache:
            model = cPickle.load(cache)
            if is_NN == True:
                try:
                    Keras_model = cPickle.load(cache)
                except:
                    pass
            else:
                Keras_model = None
            is_trained =  model.is_trained
            if is_trained:
                log.warning("The %s model is already trained! set overwrite=True, if you want to overwrite it"%mpath)
                if overwrite:
                    os.remove(mpath)
                    is_trained = False

    if not is_trained:
        ## train the model,

        try:
            if is_NN == True:
                import tensorflow as tf
                early_stopping = tf.keras.callbacks.EarlyStopping(monitor="loss", min_delta=0.00001, patience=10, mode="auto", restore_best_weights=True)
                # Keras_model.fit(X_train.values, Y_train.values, batch_size=NN_HYPERPARAMS["batch_size"], epochs=NN_HYPERPARAMS["epochs"], class_weight=train_weights, verbose=1)
                #Keras_model.fit(X_train.values, Y_train.values, batch_size=NN_HYPERPARAMS["batch_size"], epochs=NN_HYPERPARAMS["epochs"], sample_weight=tr_df_ud["SampleWeight"].values * tr_df_ud["MassPointWeight"].values, verbose=1)
                Keras_model.fit(X_train.values, Y_train.values, validation_split=0.2, batch_size=NN_HYPERPARAMS["batch_size"], epochs=NN_HYPERPARAMS["epochs"], callbacks=[early_stopping], sample_weight=tr_df_ud["SampleWeight"].values if not oversample else None, verbose=1)
            else:
                model = model.fit(X_train.values, Y_train.values, validation_split=0.2, sample_weight=X_weight if weight_sample else None)
        except:
            if is_NN == True:
                import tensorflow as tf
                early_stopping = tf.keras.callbacks.EarlyStopping(monitor="loss", min_delta=0.00001, patience=10, mode="auto", restore_best_weights=True)
                # Keras_model.fit(X_train, Y_train.values, batch_size=NN_HYPERPARAMS["batch_size"], epochs=NN_HYPERPARAMS["epochs"], class_weight=train_weights, verbose=1)
                Keras_model.fit(X_train.values, Y_train.values, batch_size=NN_HYPERPARAMS["batch_size"], epochs=NN_HYPERPARAMS["epochs"], callbacks=[early_stopping], sample_weight=tr_df_ud["SampleWeight"].values if not oversample else None, verbose=1)
            else:
                model = model.fit(X_train, Y_train.values, sample_weight=X_weight if weight_sample else None)

        model.is_trained = True
        if save_model:
            mpath = os.path.join(outdir, model.name)
            mpathh5 = mpath.replace("pkl", "h5")
            with open(mpath, "w") as cache:
                cPickle.dump(model, cache, protocol=2)
                if is_NN == True:
                    cPickle.dump(Keras_model, cache, protocol=2)
                    Keras_model.save(mpathh5)

    log.info("Trained %s model"%(model.name))
    return model

##--------------------------------------------------------------------------
##
##--------------------------------------------------------------------------
def empty_tree(files, treename="NOMINAL"):
    """ given a list of ROOT files see if at least one of them has treename
    """
    evts = 0
    log.debug(files)
    for fl in files:
        log.debug(fl)
        if not os.path.isfile(fl):
            raise Exception("%s does not exist" %(fl))
        tf = ROOT.TFile(fl, "READ")
        log.debug("line 584")
        if not tf.GetListOfKeys().Contains(treename):
            log.info("%s does not contain %s tree"%(fl,treename))
        if tf.IsZombie():
            log.info("%s is a Zombie." %(fl))
        log.debug("line 587")
        tr = tf.Get(treename)
        log.debug("line 589")
        if tr:
            evts += tr.GetEntries()
        log.debug("**"*100)
        tf.Close()
    return evts==0


import tweepy
import sys
import os, os.path
import _pickle
import re, string, unicodedata

#read twitter application credentials from file
with open('twitterCreds.txt','r') as f:
			myCreds = [line.strip() for line in f]

#choose application authentication to get the most requests possible
auth = tweepy.AppAuthHandler(myCreds[0],myCreds[1] )

api = tweepy.API(auth, wait_on_rate_limit=True,
				   wait_on_rate_limit_notify=True)

if (not api):
    print ("Can't Authenticate")
    sys.exit(-1)

#file that has list of geolocations
with open('locations.txt','r') as f:
	inputLocations = [line.strip() for line in f]
locations=[]

#split inputs into names and latitude/longitude
locations=[line.split("@")[0] for line in inputLocations]
geoLocations=[line.split("@")[1] for line in inputLocations]

tweets=[]
#if the program has been run before, load the existing values
max_id_fileName='maxId.txt'
if os.path.exists(max_id_fileName):
	print("\tReading existing id values from log")
	with open(max_id_fileName, 'rb') as f:
		inputData=_pickle.load(f)
		inputStrings=inputData.split("\t")
		sinceId=int(inputStrings[0])
		fileNumber=int(inputStrings[1])
else:
	print("\tmaking new id values")
	sinceId=None
	fileNumber=0

tweets_fileName='tweets.pkl'
if os.path.exists(tweets_fileName):
	print("\tAppending new tweets to existing list")
	with open(tweets_fileName, 'rb') as f:
		tweets=_pickle.load(f)

searchLocation=geoLocations[0]
#arbitrarily large, change to limit the amount of tweets to recieve
maxTweets=100000
#100 is the max possible tweets to return from get/search request
tweetsPerQry=100
max_id = -1
tweetCount=0

print("\tnew sinceID: {0}, new fileNumber {1}".format(sinceId,fileNumber))

print("Downloading max {0} tweets".format(maxTweets))

#f = open('tweets.txt','a+')

def saveInfo():
	with open('tweets.pkl', 'wb') as f:
		_pickle.dump(tweets,f)

	with open('maxId.txt','wb') as f:
		_pickle.dump(str(max_id)+"\t"+str(fileNumber),f)

while tweetCount < maxTweets:
	iterable = len(inputLocations)
	index=fileNumber % iterable

    #on every cycle of the loop, change the location
	geoLocation=geoLocations[index]
	location=locations[index]
	try:
		if (max_id <= 0):
			if (not sinceId):
				new_tweets = api.search(geocode=geoLocation, count=tweetsPerQry)
			else:
				new_tweets = api.search(geocode=geoLocation, count=tweetsPerQry,
				since_id=sinceId)
		else:
			if (not sinceId):
				new_tweets = api.search(geocode=geoLocation, count=tweetsPerQry,
				max_id=str(max_id - 1))
			else:
				new_tweets = api.search(geocode=geoLocation, count=tweetsPerQry,
				max_id=str(max_id - 1),
				since_id=sinceId)
		if not new_tweets:
			print("exiting")
			saveInfo()
			sys.exit(0)

		for tweet in new_tweets:
			#format outputs correctly
			temp = str(tweet.text)
			temp = re.sub(r"http\S+", "", temp)
			temp = "".join([x if ord(x) < 128 else '' for x in temp])
			temp = temp.replace("&"," ")
			temp = " ".join(temp.split())
			temp1 = location+"¦"+temp
			temp1=''.join(x for x in temp1 if unicodedata.category(x)!= 'Po')
			temp = temp1

			#f.write(temp)
			tweets.append(temp)

			fileNumber += 1

		tweetCount += len(new_tweets)
		print("Downloaded {0} tweets".format(tweetCount))
		max_id = new_tweets[-1].id
	except tweepy.TweepError as e:
		# Just exit if any errors
		print("some error : " + str(e))
		break

saveInfo()

print ("Downloaded {0} tweets".format(tweetCount))


import sys
import logging

log = logging.getLogger('slapi')

LOG_FORMAT = '[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s'

def configure_log(log, log_level, log_format=LOG_FORMAT):
    # Clear existing log handlers
    log.handlers = []

    # Configure Log Formatting
    formatter = logging.Formatter(log_format)

    # Configure Logging Level
    try:
        log.setLevel(getattr(logging, log_level))
    except AttributeError:
        log.warn("Unknown logging level: %s" % (log_level))
        log.setLevel(logging.INFO)

    # Configure Handler
    streamhandler = logging.StreamHandler(sys.stdout)
    streamhandler.setFormatter(formatter)
    log.addHandler(streamhandler)


#Donation address: xrb_33jc7cshocfi3pmx7jfnux4z16dqu1b59y9jp9u6ui3naipfzymysc8hxwst

#Python script to take mercatox csv data and format for upload to bitcoin.tax

#README:
#Go to mercatox Transactions History and click "Download CSV", make sure its saved as trade.csv in same folder as this script.
#Uses Python 2.7 or 3.5
#Tested on ubuntu but should work on Windows too.
#Output: mercatoxtrades_bitcointaxformat.csv

# INSTALL:
# python 2.7 https://www.python.org/download/releases/2.7/ (or apt-get python in ubuntu)

# RUN by typing in cmd/terminal: python mercatox_parsery.py

import csv
import sys
z=[]
#turns out mercatox reports timestamps in utc 0

def month_string_to_number(string):
    m = {
        'jan': 1,
        'feb': 2,
        'mar': 3,
        'apr':4,
         'may':5,
         'jun':6,
         'jul':7,
         'aug':8,
         'sep':9,
         'oct':10,
         'nov':11,
         'dec':12
        }
    s = string.strip()[:3].lower()

    try:
        out = m[s]
        return out
    except:
        raise ValueError('Not a month')

if (sys.version_info > (3, 0)):
    opentype = 'r'
else:
    opentype = 'rb'
with open('trade.csv',opentype) as csvfile:
    r = csv.reader(csvfile,delimiter=',')
    for row in r:
        if row[0] == 'txid':
            continue
        price = row[1]
        #price is always in currency of second coin
        amount = row[2]
        #amount always in first coin
        total = row[3]
        #total = price*amount + fee, always in second coin currency
        pair_id = row[4]
        first_coin = pair_id[0:pair_id.find('/')]
        second_coin = pair_id[pair_id.find('/')+1:]

        action = row[5]
        if action == 'buy':
            action = 'BUY'
        elif action == 'sell':
            action = 'SELL'
        else:
            print("WTF")
            break
        timestamp = row[6]

        #fix timestamp
        month = timestamp[0:timestamp.find(' ')]
        month = '%0.2i'%month_string_to_number(month)
        day = timestamp[timestamp.find(' ')+1:timestamp.find(',')]
        day = '%0.2i'%int(day)
        year = timestamp[timestamp.find(',')+2:timestamp.find(',')+6]
        tmp = timestamp[timestamp.find(',')+6:]
        hour = tmp[tmp.find(' ')+1:tmp.find(':')]
        if tmp.find('AM') > -1:
            minsec = tmp[tmp.find(':')+1:tmp.find(':')+6]
        else:
            minsec = tmp[tmp.find(':')+1:tmp.find(':')+6]
            hour = str(int(hour)+12)

        #new date
        date = year+'-'+month+'-'+day+' '+hour+':'+minsec
        if action == 'BUY':
            fee = str(float(amount)*float(price)*0.0025)
        else:
            fee = '0'
        #what currency are we buying or selling? always the first coin
        z.append([date,'Mercatox',action,first_coin,'%f'%float(amount),second_coin,'%f'%float(price),'%f'%float(fee)])

z.insert(0,['Date','Source','Action','Symbol','Volume','Currency','Price','Fee'])

if (sys.version_info > (3, 0)):
    with open("mercatoxtrades_bitcointaxformat.csv","w", newline='') as f:
        writer = csv.writer(f)
        writer.writerows(z)
else:
    with open("mercatoxtrades_bitcointaxformat.csv","wb") as f:
        writer = csv.writer(f)
        writer.writerows(z)


from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import json
import time
from aiogram import Bot, types
import os
import asyncio

config = json.load(open('config.json','r'))
API_TOKEN = config['bot_token']
CHANNEL_ID = config['channel_id']

# options = webdriver.ChromeOptions()
# options.add_argument("--headless")
print("start")

try:
    url = "https://www.bybit.com/en-US/nft/grabpic/"
    s = Service("chromedriver.exe")
    driver = webdriver.Chrome(service=s)#, options=options)
    driver.get(url=url)
    time.sleep(5)

    with open("grabpic.html", "w", encoding="utf-8") as file:
        file.write(driver.page_source)

    with open("grabpic.html", "r", encoding="utf-8") as file:
        src = file.read()

    soup = BeautifulSoup(src, "lxml")

    new_cards = soup.find_all(class_="card-wrapper")

    for item in new_cards:
        print("start loop")
        grabpic_link = "https://www.bybit.com" + item.find(class_="card-item").get("href")
        print(grabpic_link)

        with open("grabpic_link.txt", "r", encoding="utf-8") as file:
            all_links = file.read()

        if grabpic_link in all_links:
            print("already exist")
            print(grabpic_link)
        else:
            # url = grabpic_link
            # s = Service("chromedriver.exe")
            # driver = webdriver.Chrome(service=s)#, options=options)
            # driver.get(url=url)
            # time.sleep(5)

            # with open("grabpic.html", "w", encoding="utf-8") as file:
            #     file.write(driver.page_source)

            # with open("grabpic.html", "r", encoding="utf-8") as file:
            #     src = file.read()

            # card_data = BeautifulSoup(src, "lxml")
            #print(src)
            #print(item)
            img_link = item.find(class_="el-image__inner").get("src")
            print("Img_link " + img_link)
            title = item.find("h3").text
            print("Title " + title)
            #value = item.find(class_="right flex-row flex-items-center")
            #print(value.text)
            price = item.find(class_="right flex-row flex-items-center").find(class_="gray").text
            price = price.replace("(", "")
            price = price.replace(")", "")
            print("Price " + price)
            date1 = item.find(class_="countdown-time right flex-row flex-items-center")
            print(date1)
            date = date1.find_all("span")
            days = date[0].text
            days = days.replace("d", " days")
            hours = date[1].text
            hours = hours.replace("h", " hours")
            print("Date " + date)

            with open("grabpic_link.txt", "a", encoding="utf-8") as file:
                file.write(grabpic_link + "\n")
            print("save link in file")

            text_message_GP = hide_link(img_link) + '#GrabPic\n🔸' + title + '\n▪' + grabpic_link + '\n▪Price: ' + price + ' USDT\n▪Starts in: ' + days + hours
            #text_message_GP = (f'#GrabPic\n🔸<b>{title}</b>\n▪{grabpic_link}\n▪Price: {price} USDT\n▪Starts in: {date[0].text} day {date[1].text} hours {date[2].text} minutes')
            #print(text_message_GP)
            print("start TG")

            bot = Bot(token=API_TOKEN, parse_mode=types.ParseMode.HTML)

            async def send_message(channel_id: int, text: str):
                await bot.send_message(channel_id, text)

            async def main():
                await send_message(CHANNEL_ID, text_message_GP)

            if __name__ == '__main__':
                asyncio.run(main())
except Exception as ex:
    print(ex)
finally:
    driver.close()
    driver.quit()