from PyQt5.QtWidgets import *
from PyQt5 import uic
from PyQt5.QtCore import QThread, pyqtSignal, QDate
import os
import json
import datetime
import numpy as np
import pandas as pd
import sqlite3
import math
from StockListManager import StockListManager
from Database import Database

from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT as NavigationToolbar

import matplotlib.pyplot as plt
from mpl_finance import candlestick_ohlc

form_class = uic.loadUiType("dl_dc_sw.ui")[0]

class ViewController(QMainWindow, form_class):
    def __init__(self, parent=None):
        QMainWindow.__init__(self, parent)
        self.setupUi(self)
        with open('parameter.json') as f:
            self.user_param = json.load(f)

        self.slm = StockListManager(self.user_param['path']['root'])
        self.dateList = self.slm.load_dateList(format='datetime')

        self.connect_btn()
        self.symbolList = None
        self.current_page = 0
        self.current_code = ''

        today = datetime.date.today()
        self.dateEdit.setDate(QDate(today.year, today.month, today.day))
        self.current_date = datetime.date(*self.dateEdit.date().getDate())
        self.didx = len(self.dateList) - 1
        self.last_date = self.dateList[self.didx]
        self.limit = 100
        self.lineEdit.setText(str(self.limit))

        # for chart
        self.figure = plt.figure()
        gs = self.figure.add_gridspec(6, 1)

        self.canvas = FigureCanvas(self.figure)
        self.ax = [self.figure.add_subplot(gs[:2]),
                   self.figure.add_subplot(gs[2]),
                   self.figure.add_subplot(gs[3:5]),
                   self.figure.add_subplot(gs[5])]

        toolbar = NavigationToolbar(self.canvas, self)

        self.gridLayout_2.addWidget(self.canvas, 2, 0, 1, 2)
        self.gridLayout_2.addWidget(toolbar, 1, 0, 1, 2)
        self.figure.tight_layout()

        self.report = self.xaxis = self.df = self.df_int = self.xtickLabel = self.xaxis_int = self.xtickLabel_int = None
        self.clicked = False
        self.n = 0
        self.num_task = 0
        self.isSet = False

        self.radioButton.toggled.connect(self.onClicked)
        self.radioButton_2.toggled.connect(self.onClicked)
        self.lineEdit_2.returnPressed.connect(self.on_returned)

        self.type = '돌파'
        self.colors = {'돌파': 'r', '지지': 'g'}
        fpath = os.path.join(self.user_param['path']['root'], 'database', 'day_candle.db')
        self.con = sqlite3.connect(fpath)

        fpath = os.path.join(self.user_param['path']['root'], 'database', 'intra_day_3min_{}-{:02d}.db'.format(today.year, today.month))
        self.db_int = Database(self.user_param['path']['root'], 'intra_day_3min')
        self.con_int = sqlite3.connect(fpath)

    def connect_btn(self):
        btn_map = {self.pushButton: self.go_prev, self.pushButton_2: self.go_next, self.pushButton_3: self.save_task,
                   self.pushButton_4: self.load_task}

        for btn, func in btn_map.items():
            btn.clicked.connect(func)

    def load_task(self):
        self.current_date = datetime.date(*self.dateEdit.date().getDate())
        fpath = os.path.join(self.user_param['path']['root'], 'database/screen_tables', 'swing_table_{}.db'.format(self.current_date.year))
        con = sqlite3.connect(fpath)

        self.didx = np.where(self.dateList == self.current_date)[0][0]
        self.last_date = self.dateList[self.didx]

        rep_path = os.path.join(self.user_param['path']['root'], 'database/dl_data', self.type, str(self.current_date) + '.xlsx')

        df = pd.read_sql("SELECT * FROM '{}'".format(self.last_date), con, index_col='Symbol')
        self.n = df.shape[0]
        self.symbolList = list(df.index)
        if len(self.symbolList) > 0:
            if os.path.exists(rep_path):
                self.report = pd.read_excel(rep_path)
                if self.report.empty:
                    self.report = pd.DataFrame(index=self.symbolList, columns=['date', 'important_price'])
                    self.report.index.name = 'code'
                else:
                    self.report['code'] = self.report['code'].astype(str).str.zfill(6)
                    self.report = self.report.set_index('code', drop=True)
                    self.num_task = self.report.shape[0]
                    self.current_page = self.symbolList.index(self.report.index[-1]) + 1
            else:
                self.report = pd.DataFrame(index=self.symbolList, columns=['date', 'important_price'])
                self.report.index.name = 'code'
            self.current_code = self.symbolList[self.current_page]
            self.show_candle()
        else:
            print('해당 날짜에 해당 주식이 없음')

        self.label_2.setText('{} / 100'.format(self.num_task+1))
        # connect event handler for figures
        self.canvas.mpl_connect('motion_notify_event', self.on_motion)
        self.canvas.mpl_connect('button_press_event', self.on_click)

    # radio button
    def onClicked(self):
        radioButton = self.sender()
        if radioButton.text() == '돌파':
            self.type = '돌파'
        else:
            self.type = '지지'

    # for graph
    def on_click(self, event):
        if self.clicked is False and self.report is not None:
            x, y = event.xdata, event.ydata
            self.report.loc[self.current_code, 'date'] = self.current_date
            self.report.loc[self.current_code, 'important_price'] = y
            self.clicked = True
            self.isSet = True
        else:
            self.ax[0].clear()
            self.ax[2].clear()
            self.report.loc[self.current_code, 'important_price'] = float('nan')
            self.draw_candle([0, 2])

            self.canvas.draw()
            self.clicked = False

    # for stock name
    def on_returned(self):
        self.current_code = self.lineEdit_2.text()
        idx = self.symbolList.index(self.current_code)
        self.current_page = idx
        self.show_candle()

    def on_motion(self, event):
        if self.clicked is False and event.inaxes and self.xaxis is not None:
            if event.inaxes.rowNum == 0 or event.inaxes.rowNum == 3:
                x, y = event.xdata, event.ydata
                self.ax[0].clear()
                self.ax[2].clear()

                self.draw_candle([0, 2])
                self.ax[0].plot(self.xaxis, [y]*len(self.xaxis), color=self.colors[self.type])
                self.ax[2].plot(self.xaxis_int, [y]*len(self.xaxis_int), color=self.colors[self.type])
                self.canvas.draw()

    def show_candle(self):
        for ax in self.ax:
            ax.clear()
        rows = range(len(self.ax))

        self.df = pd.read_sql("SELECT * FROM (SELECT * FROM '{}' WHERE Date<='{}' ORDER BY Date DESC LIMIT {}) "
                    "ORDER BY Date".format(self.current_code, self.last_date, self.limit+20), self.con, index_col='Date')
        if self.df is None or self.df.empty:
            return
        self.df['MA_20'] = self.df['Close'].rolling(20).mean()
        self.df.dropna(how='any', inplace=True)

        pdate = self.dateList[self.didx-2]
        if Database.checkTableExists(self.con_int, self.current_code):
            try:
                self.df_int = self.db_int.get_data_by_datetime(self.current_code, pdate, self.last_date, '09:00:00', '15:20:00')
                if self.df_int is None or self.df_int.empty:
                    return
                self.df_int['MA_20'] = self.df_int['Close'].rolling(20).mean()
                self.df_int = self.df_int[self.df_int.index.date == self.last_date]
            except Exception as e:
                print(e)

        self.draw_candle(rows)
        self.canvas.draw()

    def draw_candle(self, rows):
        if self.df is None or self.df_int is None:
            return

        if 0 in rows:
            self.xaxis = np.arange(self.df.shape[0])
            candlestick_ohlc(self.ax[0], zip(self.xaxis, self.df['Open'], self.df['High'], self.df['Low'], self.df['Close']),
                             width=1, colorup='r', colordown='b')
            self.xtickLabel = [str(x) for x in self.df.index]
            self.ax[0].plot(self.xaxis, self.df['MA_20'], color='y')
            self.ax[0].set_xticks(self.xaxis[::4])
            self.ax[0].set_xticklabels(self.xtickLabel[::4], rotation=30, fontsize=6)
            self.ax[0].set_ylabel('Price')

        if 1 in rows:
            if self.df is None or self.df.empty:
                return
            self.ax[1].bar(self.xaxis, self.df['Volume'])
            self.ax[1].set_ylabel('Volume')

        if 2 in rows:
            if self.df_int is None or self.df_int.empty:
                return

            self.xaxis_int = np.arange(self.df_int.shape[0])
            candlestick_ohlc(self.ax[2], zip(self.xaxis_int, self.df_int['Open'], self.df_int['High'], self.df_int['Low'], self.df_int['Close']), width=1, colorup='r', colordown='b')

            self.xtickLabel_int = [str(x) for x in self.df_int.index.time]
            self.ax[2].plot(self.xaxis_int, self.df_int['MA_20'], color='y')
            self.ax[2].set_xticks(self.xaxis_int[::4])
            self.ax[2].set_xticklabels(self.xtickLabel_int[::4], rotation=30, fontsize=6)
            self.ax[2].set_ylabel('Price')

        if self.current_code in self.report.index and math.isnan(self.report.loc[self.current_code, 'important_price']) is False:
            self.ax[0].plot(self.xaxis, [self.report.loc[self.current_code, 'important_price']] * len(self.xaxis), color=self.colors[self.type])
            self.ax[2].plot(self.xaxis_int, [self.report.loc[self.current_code, 'important_price']] * len(self.xaxis_int), color=self.colors[self.type])
            self.clicked = True

        if 3 in rows:
            self.ax[3].bar(self.xaxis_int, self.df_int['Volume'])
            self.ax[3].set_ylabel('Volume')

        self.lineEdit_2.setText('{}'.format(self.current_code))
        self.figure.tight_layout()

    def save_task(self):
        self.report.dropna(how='any').to_excel(os.path.join(self.user_param['path']['root'], 'database/dl_data', self.type, str(self.current_date) + '.xlsx'))
        if self.isSet:
            self.num_task += 1

    def go_next(self):
        if len(self.symbolList) == 0:
            print('오늘은 작업은 없네요')
            return

        if self.current_page + 1 == self.n:
            return

        self.save_task()
        self.current_page += 1
        self.current_code = self.symbolList[self.current_page]
        self.show_candle()

        self.clicked = False
        self.isSet = False
        self.label_2.setText('{} / 100'.format(self.num_task + 1))

    def go_prev(self):
        if len(self.symbolList) == 0:
            print('오늘은 작업은 없네요')
            return
        if self.current_page == 0:
            return

        self.current_page -= 1
        self.current_code = self.symbolList[self.current_page]
        self.show_candle()

        self.clicked = False
        self.label_2.setText('{} / 100'.format(self.num_task+1))


'''
https://github.com/kuangliu/pytorch-cifar/blob/master/models/densenet.py
DenseNet in PyTorch
'''
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

class Bottleneck(nn.Module):
    def __init__(self, in_planes, growth_rate):
        super(Bottleneck, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.conv1 = nn.Conv2d(in_planes, 4*growth_rate, kernel_size=1, bias=False)
        self.bn2 = nn.BatchNorm2d(4*growth_rate)
        self.conv2 = nn.Conv2d(4*growth_rate, growth_rate, kernel_size=3, padding=1, bias=False)

    def forward(self, x):
        out = self.conv1(F.relu(self.bn1(x)))
        out = self.conv2(F.relu(self.bn2(out)))
        out = torch.cat([out, x], 1)
        return out

class Transition(nn.Module):
    def __init__(self, in_planes, out_planes):
        super(Transition, self).__init__()
        self.bn = nn.BatchNorm2d(in_planes)
        self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, bias=False)

    def forward(self, x):
        out = self.conv(F.relu(self.bn(x)))
        out = F.avg_pool2d(out, 2)
        return out

class RobustDenseNet(nn.Module):
    '''
    DenseNet121:
        depth_configs=[6, 12, 24, 16]
        channel_configs=[64, 128, 256, 512]
        growth_rate=32

    DenseNet169:
        depth_configs=[6, 12, 32, 32]
        channel_configs=[64, 128, 256, 640]
        growth_rate=32

    DenseNet201:
        depth_configs=[6, 12, 48, 32]
        channel_configs=[64, 128, 256, 896]
        growth_rate=32

    DenseNet161:
        depth_configs=[6, 12, 36, 24]
        channel_configs=[96, 192, 384, 1056]
        growth_rate=48
    '''
    def __init__(self, depth_configs=[6, 12, 24, 16], channel_configs=[64, 128, 256, 512],
                 growth_rate=32, reduction=0.5, num_classes=10):

        super(RobustDenseNet, self).__init__()
        self.growth_rate = growth_rate
        block = Bottleneck

        self.conv1 = nn.Conv2d(3, channel_configs[0], kernel_size=3, padding=1, bias=False)
        stages = []

        for i, nblocks in enumerate(depth_configs):
            block_out = nblocks * growth_rate + channel_configs[i]
            if i < len(depth_configs) - 1:
                out_planes = channel_configs[i+1]
                stage = nn.Sequential(self._make_dense_layers(block, channel_configs[i], nblocks),
                                      Transition(block_out, out_planes))
            else:
                stage = nn.Sequential(self._make_dense_layers(block, channel_configs[i], nblocks))
            stages.append(stage)

        self.stages = nn.Sequential(*stages)
        self.bn = nn.BatchNorm2d(block_out)
        self.linear = nn.Linear(block_out, num_classes)

    def _make_dense_layers(self, block, in_planes, nblock):
        layers = []
        for i in range(nblock):
            layers.append(block(in_planes, self.growth_rate))
            in_planes += self.growth_rate
        return nn.Sequential(*layers)

    def forward(self, x):
        out = self.conv1(x)
        out = self.stages(out)
        out = F.avg_pool2d(F.relu(self.bn(out)), 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


from typing import Dict, FrozenSet, List, Optional, Set, TypeVar, TypedDict

DigitSignal = FrozenSet[str]

class Entry(TypedDict):
    signal_patterns: List[str]
    output_digits: List[str]

def parse_entries(message: str) -> List[Entry]:
    entries: List[Entry] = []
    for line in message.splitlines():
        entry: Entry = {
            'signal_patterns': [],
            'output_digits': [],
        }
        parts = line.split(' | ')
        # TODO: These should be sets, not sorted strings, but how much cleanup do I bother with?
        entry['signal_patterns'] = list(map(lambda item: sort_string(item), parts[0].split(' ')))
        entry['output_digits'] = list(map(lambda item: sort_string(item), parts[1].split(' ')))
        entries.append(entry)
    return entries

# TODO: Remove (not needed)
def sort_string(s: str) -> str:
    return ''.join(sorted(list(s)))

T = TypeVar('T')

def get_single_item(s: FrozenSet[T]) -> T:
    assert len(s) == 1
    return list(s)[0]

def compute_signal_digit_mapping(entry: Entry) -> Dict[int, DigitSignal]:

    # Index signals by length
    signals_by_length: Dict[int,FrozenSet[DigitSignal]] = {}
    for signal_pattern_str in entry['signal_patterns']:
        # We use sets so they duplicates are filtered out
        signal_pattern: DigitSignal = frozenset(signal_pattern_str)
        signals_by_length.setdefault(len(signal_pattern), frozenset())
        signals_entry: Optional[FrozenSet[DigitSignal]] = signals_by_length.get(len(signal_pattern))
        assert signals_entry is not None
        signals_by_length[len(signal_pattern)] = signals_entry.union(frozenset({ signal_pattern }))

    known_numbers: Dict[int, DigitSignal] = {}
    known_segments: Dict[str, str] = {}

    # Breadown of Digits vs Segement Counts:
    #
    # (0,6,9): 6 segments
    # (1): 2 segements
    # (2,3,5): 5 segments
    # (4): 4 segements
    # (7): 3 segments
    # (8): 7 segments

    # 1.  We know signals "1", "4", "7", "8" (by segment counts)
    known_numbers[1] = list(signals_by_length[2])[0]
    known_numbers[4] = list(signals_by_length[4])[0]
    known_numbers[7] = list(signals_by_length[3])[0]
    known_numbers[8] = list(signals_by_length[7])[0]

    # 2.  We know segment "a" is "7" - "1"
    known_segments['a'] = get_single_item(known_numbers[7] - known_numbers[1])

    # 3.  We know signal "6" is any 6-segement signal that does not fully contain "1"
    for digit_signal in signals_by_length[6]:
        if len(known_numbers[1] - digit_signal) > 0:
            known_numbers[6] = digit_signal
            break
    assert 6 in known_numbers

    # 3b. We know segment "c" is the segment of "1" missing from "6"
    known_segments['c'] = get_single_item(known_numbers[1] - known_numbers[6])

    #  3c. We know segment "f" is the other segment from "1"
    known_segments['f'] = get_single_item(known_numbers[1] - set(known_segments['c']))

    # 4.  We know signal "5" is any 5-segement signal that does not contain "c"
    for digit_signal in signals_by_length[5]:
        if known_segments['c'] not in digit_signal:
            known_numbers[5] = digit_signal
            break
    assert 5 in known_numbers

    # 5.  We know signal "2" is any 5-segement signal that does not contain "f"
    for digit_signal in signals_by_length[5]:
        if known_segments['f'] not in digit_signal:
            known_numbers[2] = digit_signal
            break
    assert 2 in known_numbers

    # 6.  We know signal "3" is the remaining 5-segment signal
    for digit_signal in signals_by_length[5]:
        if digit_signal != known_numbers[2] and digit_signal != known_numbers[5]:
            known_numbers[3] = digit_signal
            break
    assert 3 in known_numbers

    # 7.  We know segment "b" is "5" - "2" - "1"
    known_segments['b'] = get_single_item(known_numbers[5] - known_numbers[2] - known_numbers[1])

    # 8.  We know segment "e" is "2" - "5" - "1"
    known_segments['e'] = get_single_item(known_numbers[2] - known_numbers[5] - known_numbers[1])

    # 9.  We know signal "9" is any 6-segement signal that does not contain "e"
    for digit_signal in signals_by_length[6]:
        if known_segments['e'] in digit_signal:
            continue
        known_numbers[9] = digit_signal
        break
    assert 9 in known_numbers

    # 10. We know signal "0" is the remaining 6-segment signal
    known_numbers[0] = get_single_item( \
        signals_by_length[6].difference({ known_numbers[6] }).difference({ known_numbers[9] }))

    # Not needed
    #
    # # 11. We know segment "d" is "8" - "0"
    # known_segments['d'] = get_single_item(known_numbers[8] - known_numbers[0])
    #
    # # 12. We know segment "g" is the remaining segment
    # all_segments = {'a','b','c','d','e','f','g'}
    # known_segments['d'] = get_single_item(all_segments - set(known_segments.values()))

    return known_numbers

def compute_output_value(entry: Entry) -> int:
    signal_digit_mapping: Dict[int, DigitSignal] = compute_signal_digit_mapping(entry)
    output_value = 0
    digit_multiplier = 1
    for output_digit_segments_str in reversed(entry['output_digits']):
        output_digit_segments = set(list(output_digit_segments_str))
        for digit,segments in signal_digit_mapping.items():
            if output_digit_segments != segments:
                continue
            output_value += digit * digit_multiplier
            digit_multiplier *= 10
            break
    return output_value

def compute_sum_of_output_values(entries: List[Entry]) -> int:
    sum: int = 0
    for entry in entries:
        sum += compute_output_value(entry)
    return sum


from flask import Blueprint

import form.struct
from service.dblist import DBList

forms = Blueprint("forms", __name__, url_prefix="/form")

dbs = Blueprint('dblist', __name__, url_prefix="/db")

@forms.route('/')
def index():
    print(form.struct)
    return form.struct.FormStruct().incorporate_field([{
        "text": "fdgs",
        "lang_type": "golang",
        "type": "int"
    }])

@dbs.route('/')
def dbindex():
    try:
        table_list = DBList().get_table_struct_for_mysql()
        return {
            "code": 100,
            "data": table_list
        }
    except Exception as e:
        print(e)
        return {
            "code": 99,
            "data": "error"
        }


import pygame
from constants import LENGTHWIN, WIDTHWIN, WHITE
from grid import Grid

pygame.init()

WIN = pygame.display.set_mode((LENGTHWIN, WIDTHWIN))
clock = pygame.time.Clock()
grid = Grid(WIN)
pygame.display.set_caption("Jeu de Sudoku")

def draw():
    WIN.fill(WHITE)
    grid.draw()
    pygame.display.update()

def main():
    while True:
        clock.tick(60)
        draw()

        for event in pygame.event.get():
            if event.type == pygame.QUIT or (
                event.type == pygame.KEYDOWN and event.key == pygame.K_q
            ):
                pygame.quit()
                return 0

            if event.type == pygame.MOUSEBUTTONDOWN:
                pos = pygame.mouse.get_pos()
                grid.caseClique(pos)

            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_s:
                    grid.solve()
                if event.key == pygame.K_c:
                    grid.clear()
                if event.key == pygame.K_f:
                    grid.solve_wait()

main()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import hashlib
import json
import os

import mozpack.path as mozpath
from mozpack.files import FileFinder

GENERATED_SOURCE_EXTS = (".rs", ".c", ".h", ".cc", ".cpp")

def sha512_digest(data):
    """
    Generate the SHA-512 digest of `data` and return it as a hex string.
    """
    return hashlib.sha512(data).hexdigest()

def get_filename_with_digest(name, contents):
    """
    Return the filename that will be used to store the generated file
    in the S3 bucket, consisting of the SHA-512 digest of `contents`
    joined with the relative path `name`.
    """
    digest = sha512_digest(contents)
    return mozpath.join(digest, name)

def get_generated_sources():
    """
    Yield tuples of `(objdir-rel-path, file)` for generated source files
    in this objdir, where `file` is either an absolute path to the file or
    a `mozpack.File` instance.
    """
    import buildconfig

    # First, get the list of generated sources produced by the build backend.
    gen_sources = os.path.join(buildconfig.topobjdir, "generated-sources.json")
    with open(gen_sources, "r") as f:
        data = json.load(f)
    for f in data["sources"]:
        # Exclute symverscript
        if mozpath.basename(f) != "symverscript":
            yield f, mozpath.join(buildconfig.topobjdir, f)
    # Next, return all the files in $objdir/ipc/ipdl/_ipdlheaders.
    base = "ipc/ipdl/_ipdlheaders"
    finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
    for p, f in finder.find("**/*.h"):
        yield mozpath.join(base, p), f
    # Next, return any source files that were generated into the Rust
    # object directory.
    rust_build_kind = "debug" if buildconfig.substs.get("MOZ_DEBUG_RUST") else "release"
    base = mozpath.join(buildconfig.substs["RUST_TARGET"], rust_build_kind, "build")
    finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
    for p, f in finder:
        if p.endswith(GENERATED_SOURCE_EXTS):
            yield mozpath.join(base, p), f

def get_s3_region_and_bucket():
    """
    Return a tuple of (region, bucket) giving the AWS region and S3
    bucket to which generated sources should be uploaded.
    """
    region = "us-west-2"
    level = os.environ.get("MOZ_SCM_LEVEL", "1")
    bucket = {
        "1": "gecko-generated-sources-l1",
        "2": "gecko-generated-sources-l2",
        "3": "gecko-generated-sources",
    }[level]
    return (region, bucket)


# Code to simulate the environment when trained
import os
#ignore tensorflow warnings
import warnings
warnings.filterwarnings("ignore")

import gym
import numpy as np
import matplotlib
matplotlib.use('pdf') # To avoid plt.show issues in virtualenv
import matplotlib.pyplot as plt

from stable_baselines.common.policies import MlpPolicy
from stable_baselines import PPO2
from Trajectory_gen import Navigation

env = Navigation(Random_reset=False, Process_perturbations=True)

tieme_steps_to_simulate = env.max_Episode_time_steps + 1 ## define the number of timesteps to simulate

######################################
##      POLICY LOADING SECTION      ##
######################################

# use of os.path.exists() to check and load the last policy evaluated by training
# function.
print("Policy loading...")

Policy_loading_mode = input("Insert loading mode\nlast: loads last policy saved\nbest: loads best policy saved\nsel: loads a specified policy\n-----> ")

if Policy_loading_mode == "last":
  for i in range(100, 0, -1): ## function look for the last policy evaluated.
    fileName_toFind = "/home/ghost/giorgio_diliberi/ReinforcementLearning/SBs_2_on_remote/Trajectory_gen/Policies/PPO_Quad_" + str(i) + ".zip"

    if os.path.exists(fileName_toFind):
      print("last policy found is PPO_Quad_" + str(i))

      Policy2Load = "Policies/PPO_Quad_" + str(i)

      break

elif Policy_loading_mode == "best":

  Policy2Load = "EvalClbkLogs/best_model.zip" # best policy name

else:

  Policy2Load = input("enter the number of policy to load (check before if exists): ")

model = PPO2.load(Policy2Load)
print("Policy ", Policy2Load, " loaded!")

######################################
##     SIMULATION SECTION           ##
######################################

#model = PPO2.load("Policies/PPO_Quad_1")  # uncomment this line to load a specific policy instead of the last one

obs = env.reset()

# info vectors initialization for simulation history
info_u = [env.state[0]]
info_v = [env.state[1]]
info_X = [env.state[2]]
info_Y = [env.state[3]]
action_memory = np.array([0., 0.]) ## vector to store actions during the simulation
#Throttle_memory = [env.dTt]
episode_reward = [env.getReward()]

time=0.
info_time=[time] # elapsed time vector

# SIMULATION

for i in range(tieme_steps_to_simulate): #last number is excluded

    action, _state = model.predict(obs, deterministic=True) # Add deterministic true for PPO to achieve better performane

    obs, reward, done, info = env.step(action)

    info_u.append(info["u"])
    info_v.append(info["v"])
    info_X.append(info["X"])
    info_Y.append(info["Y"])
    action_memory = np.vstack([action_memory, action])
    episode_reward.append(reward) # save the reward for all the episode

    time=time + env.timeStep # elapsed time since simulation start
    info_time.append(time)

    #env.render()
    if done:
      # obs = env.reset()
      break

## PLOT AND DISPLAY SECTION

plt.figure(1)
plt.plot(info_time, info_u)
plt.plot(info_time, info_v)
plt.xlabel('time')
plt.ylabel('Velocity [m/s]')
plt.title('u,v')
plt.legend(['u', 'v'])
plt.savefig('SimulationResults/Velocity.jpg')

plt.figure(2)
plt.plot(info_time, info_X)
plt.plot(info_time, info_Y)
plt.xlabel('time')
plt.ylabel('Position NE [m]')
plt.title('X,Y')
plt.legend(['X', 'Y'])
plt.savefig('SimulationResults/Position.jpg')

plt.figure(3)
plt.plot(info_time, episode_reward)
plt.xlabel('time')
plt.ylabel('Reward')
plt.title('Episode Reward')
plt.savefig('SimulationResults/reward.jpg')

plt.figure(4)
plt.plot(info_time, action_memory[:, 0])
plt.plot(info_time, action_memory[:, 1])
plt.xlabel('time')
plt.ylabel('Actions')
plt.title('Actions in episode [-1, 1]')
plt.legend(['tg_theta', 'tg_phi'])
plt.savefig('SimulationResults/action.jpg')

plt.figure(5)
plt.plot(info_X, info_Y)
plt.xlabel('X_Pos [m]')
plt.ylabel('Y_Pos [m]')
plt.title('Trajectory')
plt.savefig('SimulationResults/Trajectory.jpg')


# coding: utf-8

# In[2]:

def convert(s,numRows):
    if (numRows==1):return s
    a=[[]for i in range(numRows)]#产生一个有numRows单元的list，每个list可以往后面添加元素
    r=0
    direct=1
    for i in s:
        a[r].append(i)
        if r>=numRows-1:
            direct=-1
        elif r==0:
            direct=1
        r+=direct
    answer=''
    for row in a:
        for col in row:
            answer+=col
    return answer
if __name__=='__main__':
    print(convert('PAYPALISHIRING',4))


from PySide import QtCore, QtGui

import roslib

class ParameterDialog(QtGui.QDialog):
  '''
  This dialog creates an input mask for the given slots and their types.
  '''

  def __init__(self, slots, types, values=dict(), buttons=QtGui.QDialogButtonBox.Cancel|QtGui.QDialogButtonBox.Ok, parent=None):
    '''
    Creates an input dialog.
    @param slots: a list with  slots to enter
    @type slots: C{[str, ...]}
    @param types: a list with  types of the slots
    @type types: C{[str, ...]}
    @param values: a dictionary with slots and their values
    @type values: C{dict((str:[..]), ...)}
    '''
    QtGui.QDialog.__init__(self, parent)
    self.setObjectName("ParameterDialog")
    self.verticalLayout = QtGui.QVBoxLayout(self)
    self.verticalLayout.setObjectName("verticalLayout")

    scrollArea = QtGui.QScrollArea(self);
    scrollArea.setObjectName("scrollArea")
    scrollArea.setWidgetResizable(True)
    self.content = QtGui.QWidget()
    self.content.setObjectName("scrollAreaWidgetContents")
    self.contentLayout = QtGui.QFormLayout(self.content)
#    self.contentLayout.setVerticalSpacing(0)
#    self.contentLayout.setHorizontalSpacing(1)
    scrollArea.setWidget(self.content)

    self.verticalLayout.addWidget(scrollArea);

    self.params = list()
#    self.contentLayout.setFieldGrowthPolicy(QtGui.QFormLayout.ExpandingFieldsGrow)
    self._insertItems(self.content, self.contentLayout, slots, types, values)
    if self.params:
      self.params[0][2].setFocus(QtCore.Qt.OtherFocusReason)
#    self.content.setLayout(self.contentLayout)
    self.buttonBox = QtGui.QDialogButtonBox(self)
    self.buttonBox.setOrientation(QtCore.Qt.Horizontal)
    self.buttonBox.setStandardButtons(buttons)
    self.buttonBox.setObjectName("buttonBox")
    self.verticalLayout.addWidget(self.buttonBox)

    QtCore.QObject.connect(self.buttonBox, QtCore.SIGNAL("accepted()"), self.accept)
    QtCore.QObject.connect(self.buttonBox, QtCore.SIGNAL("rejected()"), self.reject)
    QtCore.QMetaObject.connectSlotsByName(self)

  def setText(self, text):
    '''
    Adds a label to the dialog's layout and shows the given text.
    @param text: the text to add to the dialog
    @type text: C{str}
    '''
    label = QtGui.QLabel(text, self.content)
    self.contentLayout.addRow(label)

  def _insertItems(self, parent, layout, names, types, values=dict()):
    '''
    Adds input fields to the layout of the dialog.
    @param values: a dictionary with slots and their values
    @type values: C{dict((str:[]), ...)}
    '''
    for name, type in zip(names, types):
      base_type = roslib.msgs.base_msg_type(type)
      if base_type in roslib.msgs.PRIMITIVE_TYPES:
        if base_type == 'bool':
          field = QtGui.QCheckBox(parent)
        else:
          if values.has_key(name):
            field = QtGui.QComboBox(parent)
            field.setSizePolicy(QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Fixed))
            field.setEditable(True)
            field.addItems(values[name])
          else:
            field = QtGui.QLineEdit(parent)
        field.setObjectName(name);
        label = QtGui.QLabel(''.join([name, ' (', type, ')']), parent)
        label.setObjectName(''.join([name, '_label']))
        label.setBuddy(field)
        layout.addRow(label, field)
        self.params.append((name, type, field))
      else:
        try:
          list_msg_class =roslib.message.get_message_class(base_type)
          box = QtGui.QGroupBox(type, parent)
          boxLayout = QtGui.QFormLayout(box)
#          boxLayout.setFieldGrowthPolicy(QtGui.QFormLayout.ExpandingFieldsGrow)
#          boxLayout.setVerticalSpacing(0)
#          boxLayout.setHorizontalSpacing(1)
#          boxLayout.addRow(box)
          self._insertItems(box, boxLayout, list_msg_class.__slots__, list_msg_class._slot_types)
          box.setLayout(boxLayout)
        except ValueError:
          pass

  def getKeywords(self):
    '''
    @returns: a directory with parameter and value for all entered fields.
    @rtype: C{dict(str(param) : str(value))}
    '''
    result = {}
    for key, type, value in self.params:
      if isinstance(value, QtGui.QCheckBox):
        result[key] = value.isChecked()
      else:
        test = ''
        if isinstance(value, QtGui.QLineEdit):
          text = value.text()
        elif isinstance(value, QtGui.QComboBox):
          text = value.currentText()
        if text:
          if 'int' in type:
            result[key] = int(text)
          elif 'float' in type:
            result[key] = float(text)
          else:
            result[key] = str(text)
    return result


# -*- coding: utf-8 -*-
"""
@author: Sanjay Kottapalli (svk4001@med.cornell.edu)
@date: 08/11/2022
Landau Lab, WCM

This script executes functions for the genotyping of cells,
starting from WT and MUT read counts.
"""
#imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
#import sklearn
import timeit

from sklearn.neighbors import KernelDensity
from numpy.polynomial.polynomial import Polynomial
from numpy.polynomial.polynomial import polyfit
from scipy.optimize import minimize
#from scipy.signal import savgol_filter
#from scipy.signal import find_peaks
from scipy.integrate import quad
from collections import Counter
#from sklearn.metrics.pairwise import euclidean_distances
from scipy.stats import gaussian_kde
#from sklearn.metrics import homogeneity_score
#from sklearn.cluster import SpectralClustering
from sklearn.semi_supervised import SelfTrainingClassifier
from sklearn.neighbors import KNeighborsClassifier

#from numpy.random import multivariate_normal
from sklearn.metrics import confusion_matrix
from scipy.stats import gmean

from joblib import Parallel, delayed

import numpy.matlib

import matplotlib as mpl
mpl.rcParams['figure.dpi'] = 500
sns.set(font_scale=1.2)
sns.set_style("ticks")
tab20=plt.get_cmap('tab20')
gen_cmap = {'MUT':tab20.colors[0],'WT':tab20.colors[2],
            'HET':tab20.colors[4],'NA':tab20.colors[6],'-1':tab20.colors[8]}

plt.clf()

def GotchaLabeling(path="", infile="", gene_id="", sample_id=""):
    time1 = timeit.default_timer()
    print("Reading in file.")
    typing = read_data(path+infile, gene_id, sample_id)
    sample_dir = path+sample_id+'/'
    try:
        os.mkdir(sample_dir)
    except:
        pass

    for i in ["WT", "MUT"]:
        print("Noise correcting {} read counts.".format(i))
        if i=="WT":
            typing, wt_min = noise_correct(typing,i,sample_dir,sample_id)
        else:
            typing, mut_min = noise_correct(typing,i,sample_dir,sample_id)

    print("Performing quadrant genotyping.")
    typing = quadrant_genotype(typing, wt_min, mut_min)

    print("Computing KNN-based clusters.")
    typing = KNN_cluster(typing, wt_min, mut_min,
                      0.05, sample_dir)

    typing.to_csv(sample_dir+sample_id+'_genotype_labels.csv')
    print("All analysis complete!")
    time2 = timeit.default_timer()
    print("Total time to execute: {}".format(time2-time1))

    return typing

def read_data(infile="", gene_id="", sample_id=""):
    '''
    This function reads in the sequencing reads from only real cells
    (based on ATAC calling). Cells without GoTChA reads are dropped.
    '''
    cell_line = pd.read_csv(infile, index_col=0, sep=",")
    #print(cell_line.head())
    try:
        genotyping = pd.DataFrame(index=cell_line.index)
        if sample_id in np.unique(cell_line['Sample'].values):
            cell_line = cell_line.loc[cell_line['Sample']==sample_id, :]
        genotyping['WTcount'] = cell_line[gene_id+'_WTcount']
        genotyping['MUTcount'] = cell_line[gene_id+'_MUTcount']
    except:
        cell_line = cell_line.set_index("WhiteListMatch")
        genotyping = pd.DataFrame(index=cell_line.index)
        genotyping['WTcount'] = cell_line['WTcount']
        genotyping['MUTcount'] = cell_line['MUTcount']

    #print(genotyping.head())
    index = genotyping[['WTcount', 'MUTcount']].dropna().index
    genotyping = genotyping.loc[index,:]
    print("Number of cells: "+str(genotyping.shape[0]))

    return genotyping

def noise_correct(typing, feature="", sample_dir="", sample_id=""):
    np.random.seed(0)
    pseudocount = 1
    X = typing[[feature+'count']]+pseudocount
    logged_counts = np.log(X) #change to log10 later, arcsinh? not log2

    #noise = np.random.normal(0,0.0,typing.shape[0]) #no noise
    #logged_counts = logged_counts + noise.reshape(-1,1)
    typing['transf_{}'.format(feature)] = logged_counts

    plt.hist(logged_counts, density=True, bins=50)
    plt.title(feature+' counts')
    plt.ylabel("Probability")
    plt.xlabel("Log(counts+1)")
    #plt.savefig(sample_dir+"log_{}_counts.pdf".format(feature),
    #            dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    bw = 0.1

    kde = KernelDensity(bandwidth=bw)#kernel='exponential')
    kde.fit(typing['transf_{}'.format(feature)].values.reshape(-1, 1))
    x_bin = np.histogram(typing['transf_{}'.format(feature)], bins=50)[1]
    kde_x = np.linspace(min(x_bin)-0.5,max(x_bin)+0.5,10000)
    kde_smooth = np.exp(kde.score_samples(kde_x.reshape(-1, 1)))

    plt.hist(typing['transf_{}'.format(feature)], density=True, bins=50)
    plt.plot(kde_x, kde_smooth, color='red')
    plt.title('Initial KDE')
    plt.ylabel("Probability")
    plt.xlabel("Log(counts+1)")
    plt.savefig(sample_dir+"{}_kde_initial.pdf".format(feature),
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    kde_test = kde_smooth

    def kde_func(x):
        return np.exp(kde.score_samples(x.reshape(1, -1))[0])

    plt.plot(kde_x, np.log(kde_test), color='red')
    plt.title("{} (all drops)".format(sample_id))
    plt.ylabel("Log-Probability")
    plt.xlabel("Log(WTcounts)")
    plt.show()
    plt.clf()

    kde_smooth2 = kde_test#np.log(kde_test)
    #kde_smooth2 = savgol_filter(kde_smooth2, 1001, 3)#, mode='constant', cval=-np.inf)
    plt.plot(kde_x, kde_smooth2, color='red')
    plt.show()
    plt.clf()

    fit = polyfit(kde_x, kde_smooth2, 12)
    poly = Polynomial(fit)
    poly_y = poly(kde_x)
    plt.plot(kde_x, poly_y, color='red')
    plt.show()
    plt.clf()

    mid = (max(kde_x)-min(kde_x))/2
    print(mid)
    result = minimize(poly, x0=np.array([mid]),
                        options={'disp':False}, bounds=((min(kde_x), max(kde_x)),),
                     method='Nelder-Mead')
    new_min = result.x[0]
    print(new_min)

    result = minimize(kde_func, x0=np.array([new_min]),
                        options={'disp':True}, bounds=((min(kde_x), max(kde_x)),),
                     method='Nelder-Mead')
    new_min = result.x[0]
    print(new_min)

    if feature=="WT":
        alt_feature="MUT"
    else:
        alt_feature="WT"

    if new_min==max(kde_x):
        zero_counts = typing['{}count'.format(alt_feature)]
        zero_counts = zero_counts[zero_counts==0].index

        new_min = np.percentile(typing.loc[zero_counts,'transf_{}'.format(feature)], 99.99)

    noise_values = logged_counts
    noise_values = noise_values[noise_values<new_min]
    noise_values = noise_values.dropna()
    prop_noise = len(noise_values)/len(logged_counts)

    kde_noise = KernelDensity(bandwidth=bw)#kernel='exponential')
    kde_noise.fit(noise_values.values.reshape(-1, 1))
    x_bin = np.histogram(typing['transf_{}'.format(feature)], bins=50)[1]
    #kde_x = np.linspace(min(x_bin),max(x_bin),1000)
    noise_smooth = np.exp(kde_noise.score_samples(kde_x.reshape(-1, 1)))

    signal_values = logged_counts
    signal_values = signal_values[signal_values>=new_min]
    signal_values = signal_values.dropna()
    prop_signal = 1-prop_noise

    kde_signal = KernelDensity(bandwidth=bw)#kernel='exponential')
    kde_signal.fit(signal_values.values.reshape(-1, 1))
    #x_bin = np.histogram(typing['transf_{}'.format(feature)], bins=50)[1]
    #kde_x = np.linspace(min(x_bin),max(x_bin),1000)
    signal_smooth = np.exp(kde_signal.score_samples(kde_x.reshape(-1, 1)))

    plt.hist(typing['transf_{}'.format(feature)], density=True, bins=50)
    plt.plot(kde_x, kde_test, color='red')
    plt.plot(kde_x, prop_noise*noise_smooth, color='pink')
    plt.plot(kde_x, prop_signal*signal_smooth, color='yellow')
    plt.title('Noise vs. Signal')
    plt.ylabel("Probability")
    plt.xlabel("Log(counts+1)")
    #plt.xticks(np.arange(0,6,1.0))
    plt.savefig(sample_dir+"{}_kde_mixture.pdf".format(feature),
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    def noise_func(x):
        return np.exp(kde_noise.score_samples(np.array(x).reshape(1, -1)))

    def mean_func(x):
        return x*noise_func(x)

    def var_func(x):
        return x*x*noise_func(x)

    noise_mean = quad(func=mean_func, a=-np.inf, b=np.inf, limit=100)[0]
    noise_var = quad(func=var_func, a=-np.inf,
                     b=np.inf, limit=100)[0] - noise_mean**2
    print("Noise mean, noise var:")
    print(noise_mean, noise_var)

    noise_std = np.sqrt(noise_var)
    #typing['transf_{}'.format(feature)] = (logged_counts-noise_mean)/noise_std

    added_var = 0.3**2
    new_var = noise_var - added_var
    new_std = np.sqrt(new_var)

    typing['transf_{}'.format(feature)] = (np.log(X)-noise_mean)/new_std

    plt.hist(typing['transf_{}'.format(feature)], bins=50, density=True)
    plt.title('{} Z-scores'.format(feature))
    plt.ylabel("Probability")
    plt.xlabel("Z-score")
    plt.savefig(sample_dir+"{}_zscores.pdf".format(feature),
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    feat_min = (new_min-noise_mean)/new_std
    print(feat_min)

    return typing, feat_min

def quadrant_genotype(typing, wt_min, mut_min):
    typing['quadrant_class'] = 'NA'
    for i in typing.index:
        if typing.loc[i,'transf_WT'] < wt_min:
            if typing.loc[i,'transf_MUT'] < mut_min:
                typing.loc[i,'quadrant_class'] = 'NA'
            else:
                typing.loc[i,'quadrant_class'] = 'MUT'
        else:
            if typing.loc[i,'transf_MUT'] < mut_min:
                typing.loc[i,'quadrant_class'] = 'WT'
            else:
                typing.loc[i,'quadrant_class'] = 'HET'

    return typing

def KNN_cluster(typing, wt_min, mut_min,
                      knn_window=0.05, sample_dir=""):

    typing['clusters'] = typing['quadrant_class']

    data = typing[['transf_WT', 'transf_MUT']].values
    range_wt = max(data[:,0])-min(data[:,0])
    range_mut = max(data[:,0])-min(data[:,0])

    indices1 = set(np.where(data[:,0]>wt_min-knn_window*range_wt)[0])
    indices1 = indices1.intersection(set(np.where(data[:,0]<=wt_min+knn_window*range_wt)[0]))
    indices2 = set(np.where(data[:,1]>mut_min-knn_window*range_mut)[0])
    indices2 = indices2.intersection(set(np.where(data[:,1]<=mut_min+knn_window*range_mut)[0]))
    indices = indices1.union(indices2)

    indices = np.array(list(indices))
    indices = typing.index[indices]

    typing.loc[indices,'clusters'] = -1

    n = list(dict(Counter(typing['quadrant_class'].values)).values())
    print(n)
    n_neighbors = round(np.sqrt(gmean(n)))
    print("Nearest neighbors: "+str(n_neighbors))

    print("Remove uncertain labels and re-label with self-training.")

    for label in np.unique(typing['clusters'].astype(str)):
        try:
            plt.scatter(data[np.where(typing['clusters'].astype(str)==label)[0],0],
                        data[np.where(typing['clusters'].astype(str)==label)[0],1], label=label,
                        s=7, color=gen_cmap[label])
        except:
            pass
    handles, labels = plt.gca().get_legend_handles_labels()
    by_label = dict(zip(labels, handles))
    plt.legend(by_label.values(), by_label.keys())
    #plt.legend()
    plt.title('Confident Calls')
    plt.xlabel('WT Z-scores')
    plt.ylabel('MUT Z-scores')
    plt.savefig(sample_dir+"confident_calls.pdf",
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    time1 = timeit.default_timer()
    STC = SelfTrainingClassifier(KNeighborsClassifier(n_neighbors=n_neighbors, weights='distance'),#gauss_func),
                            criterion='k_best', k_best=1, max_iter=None)
    STC.fit(data, typing['clusters'])
    time2 = timeit.default_timer()
    print("Label propagation completed in {} seconds.".format(
        time2-time1))

    cluster_pred = STC.transduction_
    typing['clusters'] = cluster_pred

    typing['genotype_pred'] = typing['clusters']
    del typing['clusters']

    x=data[:,0]#typing.loc[:,['transf_wt']]
    y=data[:,1]#typing.loc[:,['transf_mut']]

    for label in np.unique(typing['genotype_pred'].astype(str)):
        try:
            plt.scatter(data[np.where(typing['genotype_pred'].astype(str)==label)[0],0],
                        data[np.where(typing['genotype_pred'].astype(str)==label)[0],1], label=label,
                       s=7, color=gen_cmap[label])
        except:
            pass
    plt.legend()
    plt.title('Cluster Genotype')
    plt.xlabel('WT Z-scores')
    plt.ylabel('MUT Z-scores')

    plt.axhline(y=mut_min, color='r', linestyle='-')
    plt.axvline(x=wt_min, color='r', linestyle='-')

    #plt.colorbar()
    plt.savefig(sample_dir+"cluster_genotype.pdf",
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    for label in np.unique(typing['quadrant_class']):
        try:
            plt.scatter(data[np.where(typing['quadrant_class']==label)[0],0],
                        data[np.where(typing['quadrant_class']==label)[0],1], label=label,
                       s=7, color=gen_cmap[label])
        except:
            pass
    plt.title('Quadrant Genotype')
    plt.xlabel('WT Z-scores')
    plt.ylabel('MUT Z-scores')

    plt.axhline(y=mut_min, color='r', linestyle='-')
    plt.axvline(x=wt_min, color='r', linestyle='-')

    plt.legend()
    plt.savefig(sample_dir+"quadrant_genotype.pdf",
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    print('Proportion genotyped: (cluster method): ' +
          str(sum(typing['genotype_pred']!='NA')/typing.shape[0]))

    print('Proportion genotyped: (quadrant method): ' +
          str(sum(typing['quadrant_class']!='NA')/typing.shape[0]))

    print("Quadrant labels:")
    print(Counter(typing['quadrant_class']))

    print("Cluster labels:")
    print(Counter(typing['genotype_pred']))

    return typing


import torch
from torch import nn
import torch.nn.utils.prune as prune
import torch.nn.functional as F
import numpy as np
import torch.nn.utils.prune as prune
from matplotlib import pyplot as plt
import pickle
import scipy
from scipy.stats import entropy

class MagnitudeUnstructured(prune.BasePruningMethod):
    r"""Prune (currently unpruned) units in a tensor at by magnitude.
    Args:
        name (str): parameter name within ``module`` on which pruning
            will act.
        amount (int or float): quantity of parameters to prune.
            If ``float``, should be between 0.0 and 1.0 and represent the
            fraction of parameters to prune. If ``int``, it represents the
            absolute number of parameters to prune.
    """

    PRUNING_TYPE = 'unstructured'

    def __init__(self, amount):
        # Check range of validity of pruning amount
        prune._validate_pruning_amount_init(amount) # yes this is supposed to be an internal function, but why change what isn't broken?
        self.amount = amount

    def compute_mask(self, t, default_mask):
        mask = default_mask.clone(memory_format=torch.contiguous_format)
        # Check that the amount of units to prune is not > than the number of
        # parameters in t
        tensor_size = t.nelement()
        # Compute number of units to prune: amount if int,
        # else amount * tensor_size
        nparams_toprune = prune._compute_nparams_toprune(self.amount, tensor_size)
        # This should raise an error if the number of units to prune is larger
        # than the number of units in the tensor
        prune._validate_pruning_amount(nparams_toprune, tensor_size)

        if nparams_toprune != 0: # k=0 not supported by torch.kthvalue
            topk = torch.topk(torch.abs(t).view(-1), k=nparams_toprune, largest=False)
            mask.view(-1)[topk.indices] = 0

        return mask

    @classmethod
    def apply(cls, module, name, amount):
        r"""Adds the forward pre-hook that enables pruning on the fly and
        the reparametrization of a tensor in terms of the original tensor
        and the pruning mask.
        Args:
            module (nn.Module): module containing the tensor to prune
            name (str): parameter name within ``module`` on which pruning
                will act.
            amount (int or float): quantity of parameters to prune.
                If ``float``, should be between 0.0 and 1.0 and represent the
                fraction of parameters to prune. If ``int``, it represents the
                absolute number of parameters to prune.
        """
        return super(MagnitudeUnstructured, cls).apply(
            module, name, amount=amount
        )

def magnitude_unstructured(module, name, amount):
    MagnitudeUnstructured.apply(module, name, amount)

    return module

def l1_prune_network(model,amount):
    for m in model.modules():
        #print(type(m))
        if isinstance(m, torch.nn.Conv2d):
            prune.l1_unstructured(m, name="weight", amount=amount)
            #magnitude_unstructured(m, name="weight", amount=amount)
    measure_sparsity(model)

def magnitude_prune_network(model,amount):
    for m in model.modules():
        #print(type(m))
        if isinstance(m, torch.nn.Conv2d):
            magnitude_unstructured(m, name="weight", amount=amount)
            #magnitude_unstructured(m, name="weight", amount=amount)
    measure_sparsity(model)

def random_prune_network(model,amount):
    for m in model.modules():
        #print(type(m))
        if isinstance(m, torch.nn.Conv2d):
            prune.random_unstructured(m, name="weight", amount=amount)
            #magnitude_unstructured(m, name="weight", amount=amount)
    measure_sparsity(model)

def measure_sparsity(model):
    model_parameters = filter(lambda p: p.requires_grad, model.parameters())
    params = sum([np.prod(p.size()) for p in model_parameters])
    total = 0
    conv_total = 0
    zero = 0
    conv_zero = 0
    for m in model.modules():
        if not hasattr(m, 'weight'):
            continue
        #t =  m.weight.data.numel()
        t =  0
        for name, para in (m.named_parameters()):
            t+= len(para.data.cpu().numpy().flatten())
        pruned = int(torch.sum(m.weight.data.eq(0)))
        if isinstance(m, torch.nn.Conv2d):
            conv_total+= t
            conv_zero += pruned
        total+=t
        zero += pruned
    print("Total patams = %d, conv params = %d, total sparsity = %.4f, Conv sparsity %.4f "%(total, conv_total, float(zero)/total, float(conv_zero)/conv_total))
    return (total, conv_total, zero, conv_zero)


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import torch
from torch import nn
from torch import optim
from torchvision import datasets, transforms, models
import json
from collections import OrderedDict
from load import process_image
from load import imshow
import time

# Set up model

# In[ ]:

def setup_model(structure = 'vgg16', dropout = 0.5, lr=0.001, power = 'GPU', hidden_layer = 512):
    if structure == 'vgg16':
        model = models.vgg16(pretrained=True)
    if structure == 'alexnet':
        model = models.alexnet(pretrained=True)
    if structure == 'densenet121':
        model = models.densenet121(pretrained=True)

    for param in model.parameters():
        param.requires_grad = False

    num_features = model.classifier[0].in_features

    classifier = nn.Sequential(OrderedDict([
        ('dropout',nn.Dropout(p=dropout)),
        ('fc1', nn.Linear(num_features, hidden_layer)),
        ('relu', nn.ReLU()),
        ('fc2', nn.Linear(hidden_layer, 100)),
        ('relu2', nn.ReLU()),
        ('fc3', nn.Linear(100,2)),
        ('output', nn.LogSoftmax(dim=1))
    ]))

    model.classifier = classifier
    criterion = nn.NLLLoss()
    optimizer = optim.Adam(model.classifier.parameters(), lr=lr)

    if torch.cuda.is_available() and power == 'GPU':
        model.cuda()

    return model, criterion, optimizer

# train the model

# In[ ]:

def train_model(model, criterion, optimizer, train_dataloaders, vaild_dataloaders, power = 'GPU', epochs=1):
    accuracy_l = []
    training_l = []
    vaildation_l = []
    sensitivity_l = []
    specificity_l = []
    for e in range(epochs):
        running_loss = 0
        model.train()

        for images, labels in train_dataloaders:
            if torch.cuda.is_available() and power=='GPU':
                images,labels = images.to('cuda'), labels.to('cuda')
            log_ps = model(images)
            loss = criterion(log_ps, labels)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
        else:
            model.eval()
            accuracy = 0
            vaild_loss = 0

            accuracy = 0
            vaild_loss = 0

            TF_all = 0
            AF_all = 0
            TP_all = 0
            AP_all = 0
            AF = 0
            AP = 0

            with torch.no_grad():
                for images, labels in vaild_dataloaders:
                    if torch.cuda.is_available() and power=='GPU':
                        images,labels = images.to('cuda'), labels.to('cuda')

                    log_ps = model(images)
                    vaild_loss += criterion(log_ps, labels)
                    ps = torch.exp(log_ps)
                    top_p, top_class = ps.topk(1, dim=1)
                    labels = labels.view(*top_class.shape)

                    equals = top_class == labels
                    accuracy += torch.mean(equals.type(torch.FloatTensor))

                    AF = sum (labels == 0).numpy()[0]
                    if AF != 0:
                        TF_all += sum(top_class[labels == 0] == 0).numpy()
                        AF_all += AF

                    AP = sum(labels == 1).numpy()[0]
                    if AP != 0:
                        TP_all += sum(top_class[labels == 1] == 1).numpy()
                        AP_all += AP

            sensitivity = (TP_all / AP_all)
            specificity = (TF_all / AF_all)

            accuracy = (TP_all + TF_all) / (AP_all + AF_all)

            running_loss = running_loss / len(train_dataloaders)
            training_l += [running_loss]
            valid_loss = vaild_loss / len(vaild_dataloaders)
            valid_loss = vaild_loss
            vaildation_l += [valid_loss]

            accuracy_l += [accuracy]
            sensitivity_l += [sensitivity]
            specificity_l += [specificity]
            print("epoch {0}/{1} Training loss: {2:.4} ".format(e+1,epochs,running_loss),
                 "Vaildation loss: {0:.4}".format(valid_loss),
                 "Accurancy:{0:.4}".format(accuracy),
                 "Specificity:{0:.4}".format(specificity),
                 "Sensitivity:{0:.4}".format(sensitivity))
    else:
        print("Finished!")


"""
Test the basic backup and restore actions with api version >= 201
"""
import os
import unittest

import commontest as comtst
import fileset

class ActionBackupRestoreTest(unittest.TestCase):
    """
    Test that rdiff-backup really restores what has been backed-up
    """

    def setUp(self):
        self.base_dir = os.path.join(comtst.abs_test_dir,
                                     b"action_backuprestore")
        # Windows can't handle too long filenames
        long_multi = 10 if os.name == "nt" else 25
        self.from1_struct = {
            "from1": {"subs": {
                "fileA": {"content": "initial"},
                "fileB": {},
                "dirOld": {"type": "dir"},
                "itemX": {"type": "dir"},
                "itemY": {"type": "file"},
                "longdirnam" * long_multi: {"type": "dir"},
                "longfilnam" * long_multi: {"content": "not so long content"},
                # "somehardlink": {"link": "fileA"},
            }}
        }
        self.from1_path = os.path.join(self.base_dir, b"from1")
        self.from2_struct = {
            "from2": {"subs": {
                "fileA": {"content": "modified"},
                "fileC": {},
                "dirNew": {"type": "dir"},
                "itemX": {"type": "file"},
                "itemY": {"type": "dir"},
                "longdirnam" * long_multi: {"type": "dir"},
                "longfilnam" * long_multi: {"content": "differently long"},
                # "somehardlink": {"link": "fileA"},
            }}
        }
        self.from2_path = os.path.join(self.base_dir, b"from2")
        if os.name != "nt":
            # rdiff-backup can't handle (yet) hardlinks under Windows
            self.from1_struct["from1"]["subs"]["somehardlink"] = {
                "link": "fileA"}
            self.from2_struct["from2"]["subs"]["somehardlink"] = {
                "link": "fileA"}
        fileset.create_fileset(self.base_dir, self.from1_struct)
        fileset.create_fileset(self.base_dir, self.from2_struct)
        fileset.remove_fileset(self.base_dir, {"bak": {"type": "dir"}})
        fileset.remove_fileset(self.base_dir, {"to1": {"type": "dir"}})
        fileset.remove_fileset(self.base_dir, {"to2": {"type": "dir"}})
        self.bak_path = os.path.join(self.base_dir, b"bak")
        self.to1_path = os.path.join(self.base_dir, b"to1")
        self.to2_path = os.path.join(self.base_dir, b"to2")
        self.success = False

    def test_action_backuprestore(self):
        """test the "backup" and "restore" actions"""
        # we backup twice to the same backup repository at different times
        self.assertEqual(comtst.rdiff_backup_action(
            False, False, self.from1_path, self.bak_path,
            ("--api-version", "201", "--current-time", "10000"),
            b"backup", ()), 0)
        self.assertEqual(comtst.rdiff_backup_action(
            False, True, self.from2_path, self.bak_path,
            ("--api-version", "201", "--current-time", "20000"),
            b"backup", ()), 0)

        # then we restore the increment and the last mirror to two directories
        self.assertEqual(comtst.rdiff_backup_action(
            True, False, self.bak_path, self.to1_path,
            ("--api-version", "201"),
            b"restore", ("--at", "1B")), 0)
        self.assertEqual(comtst.rdiff_backup_action(
            True, True, self.bak_path, self.to2_path,
            ("--api-version", "201"),
            b"restore", ()), 0)

        self.assertFalse(fileset.compare_paths(self.from1_path, self.to1_path))
        self.assertFalse(fileset.compare_paths(self.from2_path, self.to2_path))

        # all tests were successful
        self.success = True

    def test_action_backuprestore_quoted(self):
        """
        test the backup and restore actions with quoted repository
        """
        # we backup using a specific chars-to-quote
        self.assertEqual(comtst.rdiff_backup_action(
            False, False, self.from1_path, self.bak_path,
            ("--api-version", "201", "--current-time", "10000",
             "--chars-to-quote", "A", "--use-compatible-timestamps"),
            b"backup", ()), 0)

        # then we restore once the full repo, once a sub-path
        self.assertEqual(comtst.rdiff_backup_action(
            True, False, os.path.join(self.bak_path, b"itemX"), self.to1_path,
            ("--api-version", "201"),
            b"restore", ()), 0)
        self.assertEqual(comtst.rdiff_backup_action(
            True, True, self.bak_path, self.to2_path,
            ("--api-version", "201"),
            b"restore", ()), 0)

    def tearDown(self):
        # we clean-up only if the test was successful
        if self.success:
            fileset.remove_fileset(self.base_dir, self.from1_struct)
            fileset.remove_fileset(self.base_dir, self.from2_struct)
            fileset.remove_fileset(self.base_dir, {"bak": {"type": "dir"}})
            fileset.remove_fileset(self.base_dir, {"to1": {"type": "dir"}})
            fileset.remove_fileset(self.base_dir, {"to2": {"type": "dir"}})

if __name__ == "__main__":
    unittest.main()


import os

def filter_data_by_classes(classes: list, image_path, label_path):
    labels = os.listdir(label_path)
    for label in labels:
        if label == ".DS_Store":
            continue
        with open(os.path.join(label_path, label), "r+") as f:
            append_content = []
            lines = f.readlines()
            lines = [line for line in lines if classes.__contains__(int(line.split(" ")[0]))]

            if len(lines) == 0:
                f.close()
                os.remove(os.path.join(label_path, label))
                os.remove(os.path.join(image_path, label.split(".")[0] + ".jpg"))
                continue

            lines = [str(classes.index(int(line.split(" ")[0]))) + " " + str.join(" ", line.split(" ")[1:])
                     for line in lines]

            f.seek(0)
            f.truncate(0)
            f.write(str.join("", lines))

if __name__ == '__main__':
    filter_data_by_classes([0, 24, 26, 28],
                           r".\datasets\coco128\images\train2017",
                           r".\datasets\coco128\labels\train2017")


# In this we are going to solve leetcode #119 problem
# https://leetcode.com/problems/pascals-triangle-ii/

# we are going to follow a specific process (D, D, P, A, B)
# Definition, Data, Pseudo code, Actual code, Big-O efficiency

## Definition -- Can be easily understood from leetcode problem statement

## Data (IOEA) -- Input, Output, Edge Cases, Assumptions
  # Input -- an integer represents the row of pascal triangle
  # Output -- return element of input rowIndex
  # Egde Cases -- first row of pascal array
  # Assumptions -- rowIndex should be 0 or greater than 0

# Actual code --
# Solution 1:

class Solution:
    def getRow(self, rowIndex: int) -> List[int]:
        if rowIndex == 0:
            return [1]
        elif rowIndex == 1:
            return [1,1]
        else:
            pascalArr = [1,1]
            counter = 1
            while rowIndex != counter:
                temp = []
                for i in range(len(pascalArr)-1):
                    temp.append(pascalArr[i] + pascalArr[i+1])
                temp = [1] + temp + [1]
                pascalArr = temp
                counter += 1
            return pascalArr

''' Big-O-efficiency
## time complexity - O(n) -- because we need to go to each row
## space complexity -- O(k) -- pascal array or temp array only holds elements k (row elements at any specific time)
'''


import telebot
import traceback

from async_bot import bot
from logger import logger, create_who_triggered_str

def _get_id_string(chat_id: int, user_id: int):
    text = f"chat_id = {chat_id}\n"
    text += f"user_id = {user_id}"
    return text

async def handle(message: telebot.types.Message):
    try:
        logger.info(f"[handle] /get_my_id: {create_who_triggered_str(message)}")
        if message.reply_to_message is None:
            text = "Твой id:\n"
            text += _get_id_string(message.chat.id, message.from_user.id)
        else:
            r = message.reply_to_message
            text = f"{r.from_user.first_name} id:\n"
            text += _get_id_string(r.chat.id, r.from_user.id)
        await bot.reply_to(message, text)
    except Exception as er:
        await bot.reply_to(message, f"Error: {er}")
        traceback.print_exc()


import json, requests

def get_news():
    import requests

    search_news = input("Enter the keyword for searching news or leave it in blank for default: ")
    limit = input("Introduce the number of news that you want to see (max = 100): ")
    main_url = f"https://newsapi.org/v2/top-headlines?q={search_news}&country=us&apiKey=xxxx"

    if(len(search_news) == 0):
        main_url = "https://newsapi.org/v2/top-headlines?country=us&apiKey=xxxx"
    else:
        pass

    if(len(limit) == 0):
        limit = 10
    else:
        limit = int(limit)

    # fetching data in json format
    open_page = requests.get(main_url).json()

    # getting all articles in a string article
    article = open_page["articles"]

    # empty list which will
    # contain all trending news
    results = []

    for ar in article:
        results.append(ar["title"])

    for i in range(0, limit):

        try:
            print(i + 1, results[i])
        except IndexError:
            print(f"Whoops, there isn't {limit} about {search_news}, but you have all of this ^^")
            break

    #to read the news out loud for us

# Driver Code
if __name__ == '__main__':

    # function call
    print(" ")
    print(get_news())


import os
import json
import requests
import pandas as pd
from datetime import timedelta
from datetime import datetime
## Formatted
def main():

    transactionResponse = {'status':200,'response':{
                           'data':{'COMPOUND':{"TOTAL_LIABILITIES":[],"TOTAL_ASSETS":[],"MARKET_CAP":[]}},
                           'result':{'COMPOUND':{'data':{'v2_ratio':None,}}}}}

    names ={"DAI"  : 'dai',
            "SAI"   : 'sai',
            "REP"   : 'augur',
            "TUSD"  : 'true-usd',
            "USDC"  : 'usd-coin',
            "USDT"  : "tether",
            "ETH"   : "ethereum",
            "LINK"  : "chainlink",
            "MKR"   : "maker",
            "RAI"   : "rai",
            "UNI"   : "uniswap",
            "WBTC"  : "wrapped-bitcoin",
            "SUSHI": "sushi",
            "YFI"   : "yearn-finance",
            "AAVE"  : "aave",
            "BAT"   : "basic-attention-token",
            "ZRX"   : "0x",
            # "COMP"  : "compound"
        }

    ind  = 0
    res  = requests.get("https://api.compound.finance/api/v2/ctoken")
    df   = pd.DataFrame(columns = list(res.json()['cToken'][0].keys()))

    for ctoken in res.json()['cToken']:

        if ctoken['symbol'][1:]=='COMP' or ctoken['symbol'][1:]=='cWBTC':
            continue

        for key in ctoken.keys():
            try:
                df.at[ind,key ] = ctoken[key]['value']
            except:
                df.at[ind,key ] = ctoken[key]

        supply_ctokens   = float(df.at[ind,'total_supply'])
        supply_tokens    = supply_ctokens * float(df.at[ind,'exchange_rate'])

        borrow_tokens   = float(df.at[ind,'total_borrows'])

        try:
            response         = requests.get("https://api.coingecko.com/api/v3/coins/"+ str(names[ctoken['symbol'][1:]])+ "?localization=false&tickers=false&market_data=true&community_data=false&developer_data=false&sparkline=false").json()
            current_price    = response['market_data']['current_price']['usd']
        except Exception as e:
            if 'BTC' in ctoken['symbol'][1:]:
                ctoken['symbol'] = 'cWBTC'
                response      = requests.get("https://api.coingecko.com/api/v3/coins/"+ str(names[ctoken['symbol'][1:]])+ "?localization=false&tickers=false&market_data=true&community_data=false&developer_data=false&sparkline=false").json()
                current_price = response['market_data']['current_price']['usd']
            else:
                current_price     = 0.02  # for compound

        total_supply     = supply_tokens * current_price
        total_borrow     = borrow_tokens * current_price
        position         = total_borrow - total_supply

        df.at[ind,'total_supply' ] = total_supply
        df.at[ind,'total_borrows' ] = total_borrow
        df.at[ind,'supply_tokens' ] = supply_tokens
        df.at[ind,'borrow_tokens' ] = borrow_tokens

        df.at[ind,'position($)' ]  = position
        df.at[ind,'position(B_$)'] = position / pow(10,9)

        # print(total_supply, total_borrow, position)
        # print(ctoken['symbol'][1:], names[ctoken['symbol'][1:]] , current_price)

        if ctoken['symbol'][1:]=='cETH':
            break

        ind += 1

    df['total_borrows'] = pd.to_numeric(df['total_borrows'])
    df['total_supply']  = pd.to_numeric(df['total_supply'])

    TOTAL_ASSETS      = df['total_borrows'].sum()
    TOTAL_LIABILITIES = df['total_supply'].sum()
    # response = requests.get('https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=100&page=1&sparkline=false').json()
    response = requests.get('https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=250&page=1&sparkline=false').json()
    for i in response:
        if i['symbol']=='comp':
            MARKET_CAP = i['market_cap']

    LCR = (MARKET_CAP * 0.7 ) / ( TOTAL_LIABILITIES* 0.2 + TOTAL_ASSETS* 0.1 )

    transactionResponse['response']['data']['COMPOUND']['TOTAL_LIABILITIES'].append(TOTAL_LIABILITIES)
    transactionResponse['response']['data']['COMPOUND']['TOTAL_ASSETS'].append(TOTAL_ASSETS)
    transactionResponse['response']['data']['COMPOUND']['MARKET_CAP'].append(MARKET_CAP)
    # transactionResponse['response']['data']['COMPOUND']['LCR'].append(LCR)
    transactionResponse['response']['result']['COMPOUND']['data']["total_liabilities"] = TOTAL_LIABILITIES
    transactionResponse['response']['result']['COMPOUND']['data']["total_assets"] = TOTAL_ASSETS
    transactionResponse['response']['result']['COMPOUND']['data']["market_cap"] = MARKET_CAP
    transactionResponse['response']['result']['COMPOUND']['data']['lcr'] = LCR

    return transactionResponse

def lambda_handler(event, context):
    print("Handler called")
    response = {}
    transactionResponse = {}
    try:
        # event           = json.loads(event)
        payload         = event["body"]
        transactionResponse  = main()

    except Exception as e:

        s = str(e)
        response["statusCode"] = 400
        transactionResponse["error"] = True
        transactionResponse["message"] = s
        response["headers"] = {}
        response["headers"]["Content-Type"] = "application/json"
        response["body"] = transactionResponse
        response_JSON = response
        return response_JSON

    response = transactionResponse
    return response

# if __name__ == "__main__":
#     input_data = {'body':{

#                         }
#                 }
#     # input_json = json.dumps(input_data)
#     result = lambda_handler(input_data,"Context")
#     print(result)


from tensorflow import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import RMSprop
from keras.layers.convolutional import Conv2D
from keras.layers.pooling import MaxPool2D
from keras.layers import GlobalAveragePooling2D
from keras.layers.core import Flatten, Activation
import matplotlib.pyplot as plt
# %matplotlib inline

plt.figure(figsize=(10,15))

batch_size = 128
num_classes = 10
epochs = 10

(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train = x_train.reshape(60000, 28,28,1)
x_test = x_test.reshape(10000, 28,28,1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)

"""
for i in range(100):
    plt.subplot(10, 10, i + 1)
    plt.title(y_train[i].argmax())
    plt.axis('off')
    plt.imshow(x_train[i].reshape((28, 28)), cmap='gray')
"""

model = Sequential()
model.add(Conv2D(5, (5, 5), padding='valid', input_shape=(28,28,1)))
model.add(Activation('relu')) # Conv2Dのところにまとめて書いても良い
model.add(MaxPool2D(strides=(2, 2)))
model.add(Conv2D(5, (3,3), padding='same', activation='relu'))
model.add(Conv2D(5, (3,3), padding='same', activation='relu'))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=['accuracy'])
#model.summary()

model.fit(
    x_train,
    y_train,
    batch_size=batch_size,
    epochs=epochs,
    verbose=1,
    validation_data=(x_test, y_test))

from google.colab import drive
drive.mount('/content/drive')

model.save('/content/drive/My Drive/model.h5')


class Solution:
    # @param {integer[]} nums
    # @return {integer}
    def majorityElement(self, nums):
        dict = collections.defaultdict(int)
        for num in nums:
            dict[num] += 1
        for num in dict:
            if dict[num] > len(nums) / 2:
                return num


# Create a Rectangle class
# That stores its size and its coordinates
# it should have a method called is_over, that takes an other Rectangle and
# returns True if the two rectangles are overlaping and returns False otherwise

class Rectangle():

    def __init__(self, x, y, width, height):
        self.x = x
        self.y = y
        self.width = width
        self.height = height

    def is_over(self, other):
        result = False
        print(other.x, other.y, other.width, other.height)
        if other.x >= self.x and other.x <= self.x + self.width and other.y >= self.y and other.y <= self.y + self.height:
            result = True

        elif other.x + other.width >= self.x and other.x + other.width <= self.x + self.width and other.y + other.height >= self.y and other.y + other.height <= self.y + self.height:
            result = True

        return result

rectangle1 = Rectangle(20, 20, 40, 40)
rectangle2 = Rectangle(10, 10, 30, 60)

print(rectangle1.is_over(rectangle2))


import re
import yaml
from shutil import copyfile
from loguru import logger

import revive.computation.operators as opt

OPERATORS_NAME = sorted(opt.__all__, reverse=False)

def get_nodes(yaml_file_path,):
    with open(yaml_file_path, 'r', encoding='UTF-8') as f:
        raw_config = yaml.load(f, Loader=yaml.FullLoader)

    data_config = raw_config['metadata']['columns']
    nodes_name = set([list(d.values())[0]['dim'] for d in data_config])
    nodes = { node_name : [list(d.keys())[0]  for d in data_config if list(d.values())[0]['dim'] == node_name] for node_name in nodes_name}

    for node in raw_config['metadata']['graph'].keys():
        if "next_" in node and node[5:] in nodes.keys():
            nodes[node] = nodes[node[5:]]

    return nodes

def matching_bracket(string, idx, brackets=[]):
    a = {
        "(": ")",
        "[": "]",
        "{": "}",
        "<": ">"
    }
    if brackets:
        a = { k:v for k in a.items() for k in brackets}

    close_bracket = a[string[idx]]

    if idx != string.rindex(string[idx]):
        b = string.rindex(string[idx])
        c = string[b:].index(close_bracket)
        str_list = list(string)
        str_list[b], str_list[b+c] = ".", "."
        return matching_bracket(''.join(str_list), idx)
    else:
        d = string[idx:].index(close_bracket)
        return idx + d

def strip(s):
    return re.sub("\s|\t|\n", "", s)

def find_all(s, sub_s):
    return [m.start() for m in re.finditer(sub_s, s)]

def find_node(line):
    nodes = {}
    for node_name in sorted(NODES.keys(), reverse=True):
        res = find_all(line, node_name)
        for index in res:
            bracket_start_index = index+len(node_name)
            if bracket_start_index==len(line):
                continue
            if line[bracket_start_index] != "[" or line[bracket_start_index+1:bracket_start_index+5] == "...,":
                continue
            bracket_stop_index = matching_bracket(line, bracket_start_index) + 1
            nodes[bracket_start_index] = bracket_stop_index

    nodes = [[index, nodes[index]] for index in sorted(nodes.keys())]
    return nodes

def find_column(line):
    columns = {}
    for node_name in NODES.keys():
        for column_name in NODES[node_name]:
            res = find_all(line, column_name)
            for index in res:
                if line[index-1:index+len(column_name)+1] == f'"{column_name}"' or \
                    line[index-1:index+len(column_name)+1] == f"'{column_name}'":
                    columns[index] = [column_name,node_name]

    columns = [[index, columns[index]] for index in sorted(columns.keys())]
    return columns

def find_operator(line):
    operators = {}
    for operator_name in OPERATORS_NAME:
        res = find_all(line, operator_name)
        for index in res:
            if line[index-1] in [" ", "=", "(", ","] and not bool(re.match('[a-zA-Z0-9_]+', line[index+len(operator_name)])):
                operators[index] = operator_name

    operators = [[index, operators[index]] for index in sorted(operators.keys())]

    return operators

def convert_operator(oral_operator_name):
    return "opt." + oral_operator_name

def convert_column(oral_column,flag):
    column_name, node_name = oral_column
    return f"{NODES[node_name].index(column_name)}"

def convert_node(bracket_start_index):
    return f"{NODES[node_name].index(column_name)}"

def convert_operators(line):
    """ Convert operator

    Example: add -> opt.add
    """
    operators = find_operator(line)
    if operators:
        first_operator_index, first_operator_name = operators[0]
        line = line[:first_operator_index] \
               + convert_operator(first_operator_name) \
               + line[first_operator_index+len(first_operator_name):]
        return convert_operators(line)
    else:
        return line

def convert_columns(line):
    """ Convert column name to column index

    Example: "obs_1" -> 1
    """
    columns = find_column(line)
    if columns:
        first_column_index, first_column_name = columns[0]
        if "[" == line[first_column_index-2]:
            flag = "L"
        elif "]" == line[first_column_index+len(first_column_name[0])+1]:
            flag = "R"
        else:
            flag = "C"

        line = line[:first_column_index-1] \
               + convert_column(first_column_name,flag) \
               + line[first_column_index+len(first_column_name[0])+1:]
        return convert_columns(line)
    else:
        return line

def convert_nodes(line):
    """ Convert node to tensor

    Example: obs[1,2] -> obs[...,[1,2]]
    """
    nodes = find_node(line)
    if nodes:
        bracket_start_index, bracket_stop_index = nodes[0]
        if ":" in line[bracket_start_index:bracket_stop_index]:
            line = line[:bracket_start_index] \
                   + "[...," \
                   + line[bracket_start_index+1:bracket_stop_index-1] \
                   + "]" \
                   + line[bracket_stop_index:]
        else:
            line = line[:bracket_start_index] \
                   + "[...," \
                   + line[bracket_start_index:bracket_stop_index] \
                   + "]" \
                   + line[bracket_stop_index:]
        return convert_nodes(line)
    else:
        return line

def convert_line(line):
    line = strip(line)
    line = convert_operators(line)
    line = convert_columns(line)
    line = convert_nodes(line)
    return line

def checkt_convert(line):
    comvert_line_copy = line
    for fn in [convert_operators,convert_columns,convert_nodes]:
        comvert_line_copy = fn(comvert_line_copy)
    if comvert_line_copy == line:
        return False
    return True

def convert_fn_def(origin_code : list):
    ''' find the intent of original code '''
    codes = []
    index = 0
    for i, code in enumerate(origin_code):
        if code.startswith('def '):
            bracket_start_index = code.index("(")
            codes.append(code[:bracket_start_index]+"(data: Dict[str, torch.Tensor]) -> torch.Tensor:\n")
            index = i
        if ":" in code:
            index = i
            break

    args = strip("".join(origin_code[:index+1]))
    bracket_start_index = args.index("(")
    bracket_stop_index = matching_bracket(args,bracket_start_index)
    args = args[bracket_start_index+1:bracket_stop_index].split(",")
    for arg in args:
        codes.append(f'    {arg}=data["{arg}"]\n')

    other_codes = origin_code[index+1:]

    return codes, other_codes

def get_fn_list(origin_code_list):
    fn_start_index = []
    fn_stop_index = []
    for i,code in enumerate(origin_code_list):
        if code.startswith('def '):
            if fn_start_index:
                fn_stop_index.append(i)
            fn_start_index.append(i)
    fn_stop_index.append(i+1)

    return [origin_code_list[i:j] for i,j in zip(fn_start_index,fn_stop_index)]

def parser(input_file : str,
           output_file : str,
           yaml_file : str):

    global NODES
    NODES = get_nodes(yaml_file)

    with open(input_file, 'r') as f:
        origin_code_list = f.readlines()

    for code in origin_code_list:
        if code.startswith("import torch"):
            logger.info(f'Not parser function in {input_file}')
            copyfile(input_file, output_file)
            return

    logger.info(f'Parser function in {input_file}')

    output_codes = []
    output_codes.append("import torch\n")
    output_codes.append("from typing import Dict\n")
    output_codes.append("\n")
    output_codes.append("import revive.computation.operators as opt\n")
    output_codes.append("\n")

    fn_list = get_fn_list(origin_code_list)

    for fn in fn_list:
        start_codes, other_codes = convert_fn_def(fn)
        output_codes += start_codes

        for code in other_codes:
            if not checkt_convert(code):
                output_codes.append(code)
                continue
            if " return " in code:
                if "(" not in code and "[" not in code:
                    output_codes.append(code)
                else:
                    sub_code = convert_line("return="+code[code.index("return")+len("return"):])
                    sub_code = sub_code.replace("return=", "return ")
                    output_codes.append(" "*(len(code) - len(code.lstrip()))+sub_code+"\n")
            else:
                output_codes.append(" "*(len(code) - len(code.lstrip()))+convert_line(code)+"\n")

    with open(output_file, 'w') as f:
        f.writelines(output_codes)


import re
import tensorflow as tf
from data_preprocessing import NMT_Data
from EncoderLSTM import EncoderLSTM
from DecoderLSTM import DecoderLSTM

data = NMT_Data("deu.txt")
EMBEDDING_DIM = 256
LATENT_DIM = 1024

test_data, training_data, \
max_len_input, max_len_output, \
encoder_padded_input, decoder_padded_input, decoder_padded_output, \
input_seq, target_sequences, \
word2idx_input, word2idx_output, \
vocab_size_encoder, vocab_size_decoder = data.preprocess()

sentence = ""

encoder = EncoderLSTM(vocab_size_encoder +1, max_len_input, EMBEDDING_DIM, LATENT_DIM)
decoder = DecoderLSTM(vocab_size_decoder+1, max_len_output, EMBEDDING_DIM, LATENT_DIM)

encoder.load_weights("encoder_lstm")
decoder.load_weights("decoder_lstm")
encoder_input = data.input4testing(sentence, word2idx_input, max_len_input)
final_sentence = data.compute_prediction_LSTM(encoder_input, encoder, decoder, word2idx_output
                            , max_len_output)
print("Sentence to translate: ", sentence)
print("Translation: ", final_sentence)


#!/usr/bin/python3
import sys

from datetime import datetime
from chapter2 import Node

class Stack:
    def __init__(self):
        self.top = None
        self.front = None

    def push(self, data):
        new_node = Node(data)
        if self.top:
            new_node.next = self.top
        else:
            self.front = new_node
        self.top = new_node

    def pop(self):
        if self.top:
            popped = self.top.data
            self.top = self.top.next
            return popped
        return None

    def peep(self):
        if self.top:
            return self.top.data

    def isEmpty(self):
        if not self.top:
            return True
        else:
            return False

    def popFront(self):
        node = self.top
        if not node.next:
            return self.pop()
        while node.next:
            next = node.next
            if not next.next:
                popped = next.data
                node.next = None
            else:
                node = next
        return popped

    def print_all(self, prepend):
        node = self.top
        s = ''
        while node:
            s += f'{node.data} -> '
            node = node.next
        print(f'{prepend} {s[:-4]}')

class StackWithMin(Stack):
    def __init__(self):
        super().__init__()
        self.internal_stack = Stack()

    def push(self, data):
        if data < self.min():
            self.internal_stack.push(data)
        return super().push(data)

    def pop(self):
        data = super().pop()
        if data == self.min():
            self.internal_stack.pop()
        return data

    def peek(self):
        return super().peep()

    def min(self):
        stack_min = self.internal_stack.peep()
        if not stack_min:
            return sys.maxsize
        return self.internal_stack.peep()

def problem1() -> Stack:
    """How would you design a stack which, in addition to push and pop, also has a min
    function which returns the minimum element? Push, pop, and min should all operate on O(1) time"""
    st = StackWithMin()
    st.push(2)
    st.push(4)
    st.push(1)
    st.push(6)
    print(st.peep())
    print(f'm: {st.min()}')
    print(st.pop())
    print(st.pop())
    print(f'm: {st.min()}')
    print(st.peep())
    return st

class SetOfStacks:
    def __init__(self, maximum):
        self.current_length = 0
        self.maximum = maximum
        self.stacks = [Stack()]

    def push(self, data):
        if self.current_length == self.maximum:
            print('adding stack!')
            self.current_length = 0
            self.stacks.append(Stack())
        else:
            self.current_length += 1
        self.current_stack().push(data)

    def pop(self):
        popped = self.current_stack().pop()
        if self.current_length == 0:
            print('removing stack!')
            self.stacks.pop()
            self.current_length = self.maximum
        else:
            self.current_length -= 1

        return popped

    def peep(self):
        return self.current_stack().peep()

    def current_stack(self):
        return self.stacks[-1]

    def popAt(self, substack):
        stack = self.stacks[substack]
        popped = stack.pop()
        for i in range(substack, len(self.stacks) - 1):
            cur_stack = self.stacks[i]
            next_stack = self.stacks[i+1]
            front = next_stack.popFront()
            print(f'front: {front}')
            cur_stack.push(front)
        if self.current_length == 0:
            print("shouldnt be here")
            self.stacks.pop()
            self.current_length = self.maximum
        self.current_length -= 1

        return popped

def problem2():
    """Implement a data structure SetOfStacks that is composed of several stacks and should
    create a new stack once the previosu one exceeds capacity."""
    sos = SetOfStacks(2)
    sos.push(2)
    print(f'peep: {sos.peep()}')
    sos.push(4)
    print(f'peep: {sos.peep()}')
    sos.push(6)
    print(f'peep: {sos.peep()}')
    sos.push(8)
    print(f'peep: {sos.peep()}')
    print(f'special: {sos.popAt(0)}')
    print(f'p: {sos.pop()}')
    print(f'p: {sos.pop()}')
    print(f'p: {sos.pop()}')

class TowerOfHanoi:
    def __init__(self, depth):
        self.depth = depth
        self.stacks = {
            'left': Stack(),
            'mid': Stack(),
            'right': Stack()
        }
        for i in reversed(range(depth)):
            self.stacks['left'].push(i + 1)

    def find_stack(self, n):
        for stack in self.stacks.values():
            if stack.peep() == n:
                return stack

    def get_dir_from_stack(self, match):
        for dir, stack in self.stacks.items():
            if stack == match:
                return dir

    def print_stacks(self):
        print('----------------------------')
        for t, stack in self.stacks.items():
            prepend = f'{t} stack:'
            stack.print_all(prepend)
        print('----------------------------')

    def solve_tower(self, depth, source, buffer, destination):
        dir1, dir2, dir3 = self.get_dir_from_stack(source), self.get_dir_from_stack(buffer), self.get_dir_from_stack(destination)
        print(f'solver! depth: {depth}, source = {dir1}, buffer = {dir2}, dest = {dir3}')

        if not depth:
            return
        if depth == 1:
            destination.push(source.pop())
            return

        self.solve_tower(depth-1, source, destination, buffer)
        self.print_stacks()

        destination.push(source.pop())
        self.print_stacks()

        self.solve_tower(depth-1, buffer, source, destination)
        self.print_stacks()

    def visualize_tower(self):
        list_of_values = []
        for stack in self.stacks.values():
            vals = []
            while stack.peep():
                vals.append(stack.pop())
            reversed(vals)

            list_of_values.append(vals)

        index = 0
        for stack in self.stacks.values():
            cur_list = list_of_values[index]
            for item in cur_list:
                stack.push(item)
            index += 1

        for l in list_of_values:
            for i in range(self.depth - len(l)):
                l.append('|')

        for i in range(len(list_of_values)):
            list_of_values[i] = list(reversed(list_of_values[i]))

        columns = []
        for i in range(self.depth):
            columns.append([])

        for l in list_of_values:
            for i in reversed(range(len(l))):
                columns[i].append(str(l[i]))

        for col in columns:
            print(' '.join(col))
        print("_ _ _")

        return list_of_values

def problem3(n: int):
    """towers of hanoi"""

    towers = TowerOfHanoi(n)
    towers.print_stacks()
    towers.solve_tower(n, towers.stacks['left'], towers.stacks['mid'], towers.stacks['right'])

class MyQueue:
    def __init__(self):
        self.older = Stack()
        self.newer = Stack()
        self.length = 0

    def enqueue(self, data):
        if self.older.peep():
            popped = self.older.pop()
            while popped:
                self.newer.push(popped)
                popped = self.older.pop()
        self.length += 1

        self.newer.push(data)

    def dequeue(self):
        popped = self.newer.pop()
        while popped:
            self.older.push(popped)
            popped = self.newer.pop()
        self.length -= 1
        return self.older.pop()

    def peek(self):
        popped = self.newer.pop()
        while popped:
            self.older.push(popped)
            popped = self.newer.pop()
        return self.older.peep()

    def empty(self):
        return not self.length

def problem4():
    """Implement a MyQueue class which implements a queue using two stacks"""
    q = MyQueue()
    q.enqueue(1)
    q.enqueue(2)
    q.enqueue(3)
    print(q.dequeue())
    print(q.dequeue())
    q.enqueue(11)
    q.enqueue(12)
    print(q.dequeue())
    print(q.dequeue())
    print(q.dequeue())

def problem5():
    """Write a program to sort a stack in ascending order (with biggest items on top). You may
    use at most one additional stack to hold items, but you may not copy the elements into any
    other data structure. The stack supports push, pop, peek, and isEmpty()"""
    stack = Stack()
    stack.push(8)
    stack.push(1)
    stack.push(6)
    stack.push(3)
    stack.push(4)
    stack.push(2)
    stack.push(5)

    sorted_stack = Stack()

    while not stack.isEmpty():
        stack.print_all('starting: ')
        sorted_stack.print_all('sorted: ')

        value_to_sort = stack.pop()
        print(f'value to sort: {value_to_sort}')
        if sorted_stack.isEmpty():
            sorted_stack.push(value_to_sort)
        else:
            while not sorted_stack.isEmpty():
                if value_to_sort < sorted_stack.peep():
                    print('here')
                    sorted_stack.push(value_to_sort)
                    break
                else:
                    stack.push(sorted_stack.pop())
    return sorted_stack

class Animal:
    def __init__(self, name):
        self.time_created = datetime.now()
        self.name = name

    def is_older(self, other: 'Animal'):
        return self.time_created < other.time_created

class Dog(Animal):
    def __init__(self, name):
        super().__init__(name)

class Cat(Animal):
    def __init__(self, name):
        super().__init__(name)
        self.name = name

    def get_name(self):
        return self.name

class AnimalQueue:
    def __init__(self):
        self.cat_queue = MyQueue()
        self.dog_queue = MyQueue()
        self.oldest_cat = None
        self.oldest_dog = None

    def isEmpty(self):
        return self.dog_queue.empty() and self.cat_queue.empty()

    def enqueue(self, animal: Animal):
        if isinstance(animal, Cat):
            self.cat_queue.enqueue(animal)
        else:
            self.dog_queue.enqueue(animal)

    def dequeue_dog(self):
        if self.dog_queue.empty():
            return None
        return self.dog_queue.dequeue()

    def dequeue_cat(self):
        if self.cat_queue.empty():
            return None
        return self.cat_queue.dequeue()

    def dequeue_any(self):
        dog = self.dog_queue.peek()
        cat = self.cat_queue.peek()
        if not dog or cat.is_older(dog):
            return self.dequeue_cat()
        if not cat or dog.is_older(cat):
            return self.dequeue_dog()

def problem6():
    """An animal shelter holds only dogs and cats, and operations on a strictly first in first out
    basis. People can select where they want a cat or a dog, or have no preference. Create a data
    structure to maintain this system and implement operations such as enqueue, dequeueAny, dequeueDog, dequeueCat"""

    q = AnimalQueue()
    q.enqueue(Dog('Fritz'))
    q.enqueue(Cat('Frida'))
    q.enqueue(Cat('Diego'))
    q.enqueue(Dog('Frankie'))

    print(q.dequeue_cat().name)
    print(q.dequeue_cat().name)
    print(q.dequeue_dog().name)
    print(q.dequeue_dog().name)
    q.enqueue(Dog('Chorizo'))
    q.enqueue(Cat('Kahlo'))
    print(q.dequeue_any().name)
    print(q.dequeue_any().name)

if __name__ == "__main__":
    problem1()
    problem2()
    problem3(5)
    problem4()
    problem5()
    problem6()


from aiogram import Dispatcher, executor

from commands.default import set_default_commands
from config import ADMINS
from loader import dp
from utils.misc.logging import logger

async def on_startup(dispatcher: Dispatcher):
    me = await dispatcher.bot.me

    logger.info(f'Bot startup {me.username}')
    dispatcher.bot.username = me.username

    for _id in ADMINS:
        await dispatcher.bot.send_message(_id, 'Bot started')

    await set_default_commands()

async def on_shutdown(dispatcher: Dispatcher):
    logger.warning('Shutting down..')

    for _id in ADMINS:
        await dispatcher.bot.send_message(_id, 'Bot stopped')

    await dp.storage.close()
    await dp.storage.wait_closed()

    logger.warning('Bye!')

if __name__ == '__main__':
    import filters, handlers  # noqa
    from middlewares import setup_middleware

    setup_middleware(dp)

    executor.start_polling(dp, on_startup=on_startup)