#!/usr/bin/python
# -*- coding: UTF-8 -*-

import wx,shutil,os,time,threading

def zipDir(dirPath,outPath):
    t = time.strftime('%Y%m%d%H%M%S',time.localtime())
    if os.path.isdir(dirPath) and os.path.isdir(outPath):
        name = os.path.join(outPath,t)
        shutil.make_archive(name,'zip',dirPath)

class Backup(wx.Frame):
    def __init__(self, parent, id, title):
        wx.Frame.__init__(self, parent, id, title, size=(500, 180))
        self.pnl = wx.Panel(self)

        self.makeMenu()
        self.makePannel()

        self.SetMaxSize((500,180))
        self.Center()
        self.Show()

    def makePannel(self):

        font = wx.Font(14, wx.DEFAULT, wx.NORMAL, wx.NORMAL)

        hboxBackPath = wx.BoxSizer(wx.HORIZONTAL)
        hboxTargetPath = wx.BoxSizer(wx.HORIZONTAL)
        hboxTime = wx.BoxSizer(wx.HORIZONTAL)
        hboxButton = wx.BoxSizer(wx.HORIZONTAL)
        vBox = wx.BoxSizer(wx.VERTICAL)

        backupPathTxt = wx.StaticText(self.pnl, label=u"备份路径:", size=(100, 30))
        backupPathTxt.SetFont(font)
        self.backupPathT = wx.TextCtrl(self.pnl, wx.ID_ANY, style=wx.TE_READONLY)
        button1 = wx.Button(self.pnl, -1, label="...", size=(30, 28))
        self.Bind(wx.EVT_BUTTON,self.chooseBackPath,button1)
        hboxBackPath.Add(backupPathTxt, proportion=2, flag=wx.LEFT, border=2)
        hboxBackPath.Add(self.backupPathT, proportion=8, flag=wx.LEFT, border=10)
        hboxBackPath.Add(button1, proportion=0, flag=wx.LEFT, border=2)

        targetPathTxt = wx.StaticText(self.pnl, label=u"目标路径:", size=(100, 30))
        targetPathTxt.SetFont(font)
        self.targetPathT = wx.TextCtrl(self.pnl, wx.ID_ANY, style=wx.TE_READONLY)
        button2 = wx.Button(self.pnl, -1, label="...", size=(30, 28))
        self.Bind(wx.EVT_BUTTON, self.chooseTargetPath, button2)
        hboxTargetPath.Add(targetPathTxt, proportion=2, flag=wx.LEFT, border=2)
        hboxTargetPath.Add(self.targetPathT, proportion=8, flag=wx.LEFT, border=10)
        hboxTargetPath.Add(button2, proportion=0, flag=wx.LEFT, border=2)

        timePathTxt = wx.StaticText(self.pnl, label=u"间隔时间(min):", size=(150, 30))
        timePathTxt.SetFont(font)
        self.timePathT = wx.TextCtrl(self.pnl, wx.ID_ANY)
        hboxTime.Add(timePathTxt, proportion=2, flag=wx.LEFT, border=2)
        hboxTime.Add(self.timePathT, proportion=4, flag=wx.LEFT, border=2)

        self.buttonStart = wx.Button(self.pnl, -1, label="开始备份", size=(70, 50))
        buttonExit = wx.Button(self.pnl, -1, label="结束退出", size=(70, 50))
        self.Bind(wx.EVT_BUTTON, self.OnQuit, buttonExit)
        self.Bind(wx.EVT_BUTTON, self.start, self.buttonStart)
        hboxButton.Add(self.buttonStart, proportion=0, flag=wx.RIGHT, border=2)
        hboxButton.Add(buttonExit, proportion=0, flag=wx.RIGHT, border=2)

        vBox.Add(hboxBackPath,proportion=1, flag=wx.LEFT, border=2)
        vBox.Add(hboxTargetPath,proportion=1, flag=wx.LEFT, border=2)
        vBox.Add(hboxTime,proportion=1, flag=wx.LEFT, border=2)
        vBox.Add(hboxButton, proportion=1, flag=wx.LEFT, border=2)

        self.SetSizer(vBox)

    def makeMenu(self):
        menuBar = wx.MenuBar()
        exitMenu = wx.Menu()
        fitem = exitMenu.Append(wx.ID_EXIT, "Quit", "Quit Applications")
        self.Bind(wx.EVT_MENU, self.OnQuit, fitem)
        menuBar.Append(exitMenu,"&quit")

        self.SetMenuBar(menuBar)

    def OnQuit(self, e):
        self.Close()

    def chooseBackPath(self,event):
        dlg = wx.DirDialog(self,u"选择文件夹",style=wx.DEFAULT_DIALOG_STYLE)
        if dlg.ShowModal() == wx.ID_OK:
            self.backupPathT.SetValue(dlg.GetPath())
        dlg.Destroy()

    def chooseTargetPath(self,event):
        dlg = wx.DirDialog(self,u"选择文件夹",style=wx.DEFAULT_DIALOG_STYLE)
        if dlg.ShowModal() == wx.ID_OK:
            self.targetPathT.SetValue(dlg.GetPath())
        dlg.Destroy()

    def vaildate(self):
        back = self.backupPathT.GetValue()
        target = self.targetPathT.GetValue()
        time:str = self.timePathT.GetValue()
        if len(back)==0 or len(target)==0 or len(time)==0:
            msg = wx.MessageDialog(None,u'填写信息不全，请重新填写！',u'警告',wx.OK)
            msg.ShowModal()
            return False
        if not time.isdigit():
            msg = wx.MessageDialog(None, u'时间格式不正确，请重新填写', u'警告', wx.OK)
            msg.ShowModal()
            return False
        return True

    def zipfile(self):
        if self.vaildate():
            zipDir(self.backupPathT.GetValue(),self.targetPathT.GetValue())
            t = threading.Timer(int(self.timePathT.GetValue())*60,self.zipfile)
            t.start()

    def start(self,event):
        if self.vaildate():
            self.buttonStart.Enable(False)
            self.zipfile()

if __name__ == '__main__':
    app = wx.App()
    Backup(None, -1, '备份文件')
    app.MainLoop()


from matplotlib import pyplot as plt
import numpy as np

def difference_example(color, dpi, image_path, extension):
    np.random.seed(42)
    samples = [np.random.normal(n/30) for n in range(100)]

    plt.plot(samples, label="y", color=color)
    plt.plot(np.diff(samples, 1), color="black", linestyle="--", label=r"$\nabla$y", linewidth=0.9)
    plt.xlabel("Tempo")
    plt.ylabel("Magnitude")
    plt.title("Exemplo de uso de operador de diferenças")
    plt.legend()

    path=f"{image_path}.{extension}"
    plt.savefig(path, dpi=dpi, format=extension)

    plt.close()


import csv
import logging

logging.basicConfig(
    filename=snakemake.log[0],
    level=logging.DEBUG,
    style='{',
    format='{asctime} - {message}'
)

def is_across_junction(data, junc_pos, dist):
    structure_start = int(data['T1'])
    structure_end = int(data['T4']) + int(data['TS'])

    if structure_start <= junc_pos - dist + 1:
        if structure_end >= junc_pos + dist:
            return True

    return False

with open(snakemake.input[0]) as f_in, open(snakemake.output[0], 'w') as out:
    title = next(f_in)
    title = title.rstrip('\n').split('\t')
    print(*title, sep='\t', file=out)
    logging.debug(title)

    reader = csv.DictReader(f_in, delimiter='\t', fieldnames=title)

    for data in reader:
        logging.debug(data)

        if is_across_junction(data, snakemake.params.L - 1, int(snakemake.wildcards.dist)):
            print(*data.values(), sep='\t', file=out)


import os
import logging

import pickle
import numpy as np
import torch

from torch.utils.data import Dataset, DataLoader

class CIFAR10Data(Dataset):
    def __init__(self, images, labels, transform=None):
        self.images = images
        self.labels = labels
        self.transform = transform

    def __len__(self):
        return len(self.images)

    def __getitem__(self, index):
        label = torch.tensor(self.labels[index], dtype=torch.long)
        image = self.images[index].astype('uint8')
        if self.transform:
            image = self.transform(image)
        return image, label

def load_CIFAR_batch(filename):
    with open(filename, "rb") as f:
        datadict = pickle.load(f, encoding="latin1")
        X = datadict["data"]
        Y = datadict["labels"]
        X = X.reshape(10000, 3, 32, 32).transpose(0, 2, 3, 1).astype("int")
        Y = np.array(Y)
        return X, Y

def load_CIFAR10(ROOT):
    """ load all of cifar batches (1-5 + test)"""
    xs = []
    ys = []
    for b in range(1, 6):
        f = os.path.join(ROOT, "data_batch_%d" % (b,))
        X, Y = load_CIFAR_batch(f)
        xs.append(X)
        ys.append(Y)
    Xtr = np.concatenate(xs)
    Ytr = np.concatenate(ys)
    del X, Y
    Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, "test_batch"))
    return {
        "X_train": Xtr,
        "y_train": Ytr,
        "X_test": Xte,
        "y_test": Yte,
    }

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)


import pickle
import os
import time
import numpy as np
import polychrom
from polychrom import polymerutils
from polychrom import forces
from polychrom import forcekits
from polychrom.simulation import Simulation
from polychrom.hdf5_format import HDF5Reporter, list_URIs, load_URI, load_hdf5_file
from polychrom.polymer_analyses import generate_bins
from polychrom.polymerutils import load_URI,  load
from polychrom import contactmaps
from polychrom.hdf5_format import HDF5Reporter, list_URIs, load_URI, load_hdf5_file
import openmm
import shutil
import pyximport
pyximport.install(setup_args={"include_dirs":np.get_include()},reload_support=True)
import warnings
import h5py as hp
import glob
from itertools import product
import re

from ..confinement_volumes.c_crescentus import *

def run_simulation(monomer_size,monomer_wig, N,\
        steps_per_sample, save_folder, saveEveryConfigs, \
        savedSamples, restartConfigurationEvery, GPU_choice = 0,\
        col_rate=0.1, trunc=0.5):
    """
    This function runs simulations of a linear, unconfined polymer.

    monomer_size : float
    The size of the monomer in nanometers.

    monomer_wig : float
    The monomer wiggle distance in simulation units.

    steps_per_sample : int
    The number of  polymer simulation integration steps to be carried out per block.

    save_folder : str
    Location where to dump data

    saveEveryConfigs : int
    Frequency of saving data.

    savedSamples : int
    Total number of blocks to save.

    restartConfigurationEvery : int
    Frequency of restarting the pre-allocation of bonds.

    GPU_choice : int
    The GPU number on which to run the simulation.

    col_rate : float
    Sets the friction for the polymer simulations

    trunc : float
    The excluded volume strength.

    """

    # assertions for easy managing code below
    assert restartConfigurationEvery % saveEveryConfigs == 0
    assert (savedSamples * saveEveryConfigs) % restartConfigurationEvery == 0

    savesPerInit = restartConfigurationEvery // saveEveryConfigs
    # InitsSkip = saveEveryConfigs * skipSamples  // restartConfigurationEvery
    InitsSave = savedSamples * saveEveryConfigs // restartConfigurationEvery
    InitsTotal  = InitsSave #+ InitsSkip
    print("Polymer will be initialized {0} times.".format(InitsTotal))

    block=0 #start count from zero
    # clean up the simulation directory
    folder = save_folder
    if os.path.exists(folder):
        shutil.rmtree(folder)

    # create the starting conformation
    start_data = start_point_linear_free(N) #random walk of length N, with step size = 1 simulation unit

    # create the reporter class
    reporter = HDF5Reporter(folder=folder, max_data_length=100)

    # Iterate over various BondUpdaterInitializations
    for BondUpdaterCount in range(InitsTotal):
        # simulation parameters are defined below
        a = Simulation(
                platform="cuda",
                integrator="variableLangevin",
                error_tol=0.001,
                GPU = "{}".format(GPU_choice),
                collision_rate=col_rate,
                N = len(start_data),
                max_Ek=100.,
                reporters=[reporter],
                PBCbox=False, #EDIT: no boundaries
                precision="mixed",
                verbose=True)  # timestep not necessary for variableLangevin

        a.set_data(start_data)  # loads polymer.

        # -----------Adding forces ---------------
        a.add_force(
            forcekits.polymer_chains(
                a,
                chains=[(0, N, False)], #LINEAR chain
                bond_force_func=forces.harmonic_bonds, # adds harmonic bonds for polymers
                bond_force_kwargs={
                    'bondLength':1.,
                    'bondWiggleDistance':monomer_wig, # Bond distance will fluctuate this much
                },

                angle_force_func=forces.angle_force,
                angle_force_kwargs={
                    'k':0.05 # we are making a very flexible polymer, basically not necessary here
                },
                nonbonded_force_func=forces.polynomial_repulsive, # this is the excluded volume potential
                nonbonded_force_kwargs={
                    #'trunc': trunc, # this will let chains cross sometimes (energy in kb*T units)
                    'trunc': trunc,
                    'radiusMult':1.05, # this is from old code
                    #'trunc':10.0, # this will resolve chain crossings and will not let chain cross anymore
                },
                except_bonds=True,
            )
        )

        print("\nNot adding tethers or confinement")
        a.step = block

        # Minimize energy for first bonds
        print("Polymer burn-in")
        a.local_energy_minimization()
        #Docs say first steps after energy minimization have large error; don't save
        a.integrator.step(100*steps_per_sample)

        a.step=block
        step_saved=0
        # Iterate over simulation time steps within each BondUpdater
        for i in range(restartConfigurationEvery-2):
            # BondUpdater updates bonds at each step.
            if i % saveEveryConfigs == 0: #save just before resetting configuration
                # save monomer positions
                step_saved+=1
                a.do_block(steps=steps_per_sample, save_extras={"time_step":step_saved})
            else:
                a.integrator.step(steps_per_sample)  #do steps without getting the positions from the GPU (faster)

        block = a.step
        del a

        time.sleep(0.2)  # wait 200ms for sanity (to let garbage collector do its magic)

    # dump data to output file
    reporter.dump_data()
    done_file = open(os.path.join(folder,'sim_done.txt'),"w+")
    done_file.close()
    del reporter


# try:
#     foobar
# except :
#     print("Error")

# print("After Error handling")

d = {"name" :"Kush"}
# d["Kush"]

def get(d, key):
    try:
        print(d[key])
    except KeyError:
        print("The given value is not found in the dictionary.")

get(d, "hello")
get(d, "name")


import re

with open('./input.txt') as f:
    lines = f.read().strip().split("\n")

# part 2

class Masker:

    def __init__(self):
        self.mask = None
        self.mask_len = 36
        self.mem = {}

    def set_mask(self, line):
        [_, mask_str] = line.split(' = ')
        self.mask = mask_str

    def set_mem(self, line):
        p = re.compile('mem\[(\d+)\] = (\d+)')
        m = p.match(line)
        key = m.group(1)
        value = m.group(2)
        # convert to binary and pad
        key_as_bin_str = bin(int(key))[2:].rjust(mask_len, '0')
        masked_keys = self.apply_mask(key_as_bin_str)
        for key in masked_keys:
            self.mem[key] = value

    # return list of possible keys
    def apply_mask(self, bin_string):
        if not self.mask:
            return bin_string
        # overwrite with 1s and Xs
        overwritten = ""
        for idx in range(len(bin_string)):
            if self.mask[idx] == '0':
                overwritten += bin_string[idx]
            else:
                overwritten += self.mask[idx]

        return self.generate_variants(overwritten)

    def generate_variants(self, bin_string):
        first_x_idx = bin_string.find('X')
        if first_x_idx == -1:
            return [bin_string]
        # split into 2 variants
        v1 = bin_string[0:first_x_idx] + '0' + bin_string[first_x_idx + 1:]
        v2 = bin_string[0:first_x_idx] + '1' + bin_string[first_x_idx + 1:]
        return self.generate_variants(v1) + self.generate_variants(v2)

    def run(self, steps):
        for step in steps:
            if "mask" in step:
                self.set_mask(step)
            if "mem" in step:
                self.set_mem(step)

    def current_mem_sum(self):
        mem_values = [int(v) for v in self.mem.values()]
        return sum(mem_values)


# Класс House
# 1. Создайте класс House
# 2. Создайте метод __init__() и определите внутри него два динамических
# свойства: _area и _price. Свои начальные значения они получают из
# параметров метода __init__()
# 3. Создайте метод final_price(), который принимает в качестве параметра
# размер скидки и возвращает цену с учетом данной скидки.
# Класс SmallHouse
# 1. Создайте класс SmallHouse, унаследовав его функционал от класса House
# 2. Внутри класса SmallHouse переопределите метод __init__() так, чтобы он
# создавал объект с площадью 40м2
# Класс Human
# 1. Реализуйте приватный метод make_deal(), который будет отвечать за
# техническую реализацию покупки дома: уменьшать количество денег на
# счету и присваивать ссылку на только что купленный дом. В качестве
# аргументов данный метод принимает объект дома и его цену.
# 2. Реализуйте метод buy_house(), который будет проверять, что у человека
# достаточно денег для покупки, и совершать сделку. Если денег слишком
# мало - нужно вывести предупреждение в консоль. Параметры метода: ссылка
# на дом и размер скидки
def isfloat(s):
    try:
        float(s)
        return True
    except(ValueError, TypeError):
        return False

class House:
    def __init__(self,  area, price):
        self._area = area
        self._price = price

    def final_price(self, prec):
        print(f"\nЦена с учетом скидки: {self._price - (self._price * prec) / 100}")
        return self._price - (self._price * prec) / 100

    def __del__(self):
        pass

class SmallHouse(House):
    def __init__(self, price):
        super().__init__(40, price)

    def __del__(self):
        pass

class Human:
    def __init__(self, dep):
        self.dep = dep

    def __del__(self):
        pass

    def __make_deal(self, obj_house, price_house):
        self.dep = self.dep - price_house
        self.obj_house_ref = obj_house

    def buy_house(self, obj_house, precent):
        price = obj_house.final_price(precent)
        if self.dep < price:
            return "Недостаточно денег на счету"
        self.__make_deal(obj_house, price)
        return f"\nДом успешно куплен\nНынешний счет пользователя: {self.dep}" \
               f"\nСсылка на объект дома: {self.obj_house_ref}"

def main():
    while True:
        price_house = input("Цена дома: ")
        if isfloat(price_house):
            if float(price_house) >= 0:
                price_house = float(price_house)
                obj_house = SmallHouse(price_house)
                while True:
                    deposit = input("Введите счет пользователя: ")
                    if isfloat(deposit):
                        if float(deposit) >= 0:
                            deposit = float(deposit)
                            new_human = Human(deposit)
                            while True:
                                pr = input("Введите скидку: ")
                                if isfloat(pr):
                                    if 100 >= float(pr) >= 0:
                                        pr = float(pr)
                                        print(new_human.buy_house(obj_house, pr))
                                        del new_human
                                        del obj_house
                                        exit(0)
                                    else:
                                        print("Некоректно введена скидка")
                                else:
                                    print("Введите корректную скидку")
                        else:
                            print("Введите положительное число")
                    else:
                        print("Введите корректный счет пользователя")
            else:
                print("Введите положительное число")
        else:
            print("Введите корректную цену дома")

if __name__ == "__main__":
    main()


import cPickle
import gzip
import zipfile
import os

import numpy as np

def load():
    """
    Returns digit data as <training data, validation data, test data>, where
    training data = <training_images, training_labels>
        training_images = numpy ndarray of 50,000 entries
            each entry is a numpy ndarray of 784 values, corresponding to pixels of images
        training_labels = numpy ndarray of 50,000 labels corresponding to above
            each entry is just a digit value labels
    validation data and test data are the same thing, except with only 10,000 each
    """
    f = gzip.open('mnist.pkl.gz', 'rb')
    training_data, validation_data, test_data = cPickle.load(f)
    f.close()
    return (training_data, validation_data, test_data)

def load_wrapper():
    """
    Makes the format of load() easier to work with neural networks.
    training_data = list of 50,000 2-tuples (x,y),
        where x is a 784d float numpy.ndarray that has the input image,
        and y is a 10d float numpy.ndarray unit vector for the correct digit for x that is returned by vectorized_result()
    validation_data and test_data = list of 10,000 2-tuples (x,y),
        where x is a 784d numpy.ndarray with the input image,
        and y = the actual digit values (integers) corresponding to x
    """
    tr_d, va_d, te_d = load()
    training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]] # tr_d[0] is the training images
    training_results = [vectorized_result(y) for y in tr_d[1]] # tr_d[1] is the training labels, in a unit vectorized form
    training_data = zip(training_inputs, training_results)
    validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
    validation_data = zip(validation_inputs, va_d[1])
    test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
    test_data = zip(test_inputs, te_d[1])
    return (training_data, validation_data, test_data)

def vectorized_result(j):
    """Return a 10-dimensional unit vector with a 1.0 in the jth
    position and zeroes elsewhere.  This is used to convert a digit
    (0...9) into a corresponding desired output from the neural
    network."""
    e = np.zeros((10, 1))
    e[j] = 1.0
    return e


from django.contrib import admin
from app.models import User, Club, Book, Chat, Message, FriendRequest, Post, Meeting

# Register your models here.

@admin.register(User)
class UserAdmin(admin.ModelAdmin):
    """Configuration of the admin interface for users."""

    list_display = [
        'username', 'first_name', 'last_name', 'email', 'location', 'id'
    ]

@admin.register(Club)
class ClubAdmin(admin.ModelAdmin):
    """Configuration of the admin interface for users."""

    list_display = [
        'name', 'description', 'visibility', 'public'
    ]

@admin.register(Book)
class BookAdmin(admin.ModelAdmin):
    """Configuration of the admin interface for users."""

    list_display = [
        'ISBN', 'title', 'author', 'publisher'
    ]

admin.site.register(Chat)
admin.site.register(Message)

@admin.register(FriendRequest)
class FriendRequest(admin.ModelAdmin):
    list_display = [
        'sender', 'receiver', 'created_at'
    ]

@admin.register(Post)
class Post(admin.ModelAdmin):
    list_display = [
        "title", "content", "created_at"
    ]
@admin.register(Meeting)
class Meeting(admin.ModelAdmin):
    list_display = [
        "name", "club", "book", "time"
    ]


from django.db import models
from django.contrib.auth.models import User
from cloudinary.models import CloudinaryField
from django.utils.text import slugify
from django.utils import timezone

class Post(models.Model):
    """
    Model for user posts
    """
    title = models.CharField(
        max_length=200,
        verbose_name="Title",
        )
    slug = models.SlugField(
        max_length=200,
        unique=True
        )
    author = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name="post_posts"
    )
    content = models.TextField(
        blank=True,
        null=True,
    )
    featured_image = CloudinaryField(
        'image',
        default='placeholder'
        )
    created_on = models.DateTimeField(
        auto_now_add=True
        )
    updated_on = models.DateTimeField(
        auto_now=True
        )
    likes = models.ManyToManyField(
        User,
        related_name='post_likes',
        blank=True,
        )

    #following code taken from Kim Bergstroem's PP4
    def save(self, *args, **kwargs):
            if not self.slug:
                # Generate a slug based on the username and post title
                base_slug = slugify(self.title)
                timestamp = timezone.now().strftime('%Y%m%d%H%M%S')
                unique_slug = f"{base_slug}-{timestamp}"
                self.slug = unique_slug

            return super().save(*args, **kwargs)

    class Meta:
        ordering = ["-created_on"]

    def __str__(self):
        return self.title

    def number_of_likes(self):
        return self.likes.count()

class Comment(models.Model):
    """
    Model for comments
    """

    post = models.ForeignKey(
        Post,
        on_delete=models.CASCADE,
        related_name="comments"
        )
    name = models.CharField(
        blank=True,
        null=True,
        max_length=80
        )
    author = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name="post_comment"
        )
    body = models.TextField()
    created_on = models.DateTimeField(
        auto_now_add=True
    )
    approved = models.BooleanField(default=True)

    class Meta:
        ordering = ["created_on"]

    def __str__(self):
        """
        Returns string for comment
        """
        return f"Comment {self.body} by {self.name}"


#!/usr/bin/env python3

#import commands
import sys, math
import os, shutil, signal
import subprocess as commands
import re
import random
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import time
import itertools
import timeit
from cosy import cosyrun, write_fox
import pygmo as pg
from problem import optimizeRes
import pandas as pd
from kmeans_cluster import run_kmeans
from view_db import is_pareto_efficient_simple

# specify Tex details for pretty plots
os.environ['PATH'] = os.environ['PATH'] + ':/mnt/misc/sw/indep/all/texlive/2013/bin/x86_64-linux/latex'
#os.environ['PATH'] = os.environ['PATH'] + ':/usr/bin/tex'
plt.rcParams.update({
    "text.usetex": True,
})

script, filename = sys.argv
optimized_params = 5
fNom = np.zeros(optimized_params)+1
fNames = [r"{FP1-res}${}^{-1}$",r"{FP2-res}${}^{-1}$",r"{FP3-res}${}^{-1}$",r"MaxBeamWidth",r"BeamSpotSize"]
fNames = fNames[:optimized_params]
#print(len(fNames))
magnet_dim = 19
objectives = ['FP1_res','FP2_res','FP3_res','MaxBeamWidth','DSSD_BeamSpot']
fNom1 = np.array([2384.9360856494263, 109.61548781662407, 510.8029152516118, 1.6251646888022029, 0.12574090408565933])

# read pop from h5 file (i.e. after running view_db.py)
def read_pop_df(filename, pop=None):
    df = pd.read_hdf(filename)
    magnet_dim = 19
    p_optimizeRes = pg.problem(optimizeRes(magnet_dim))
    nobj = p_optimizeRes.get_nobj()
    if pop == None:
        pop = pg.population(p_optimizeRes)
    nrow, ncol = df.shape
    for i in df.index:
        append=True
        xs = []
        for j in range(1,magnet_dim+1):
            xs.append(df["q"+str(j)][i])
        xs = np.asarray(xs)
        fs = []
        for j in range(magnet_dim,magnet_dim+nobj):
#            if i == 0:
#                print(df.iloc[i,magnet_dim:magnet_dim+p_optimizeRes.get_nobj()])
            fs.append(df.iloc[i,j])
        if append:
            pop.push_back(xs,f=fs)
    if 'PCA' in filename:
        costs = df[objectives]
        costs = np.array(costs)
        pareto = is_pareto_efficient_simple(costs)
        # add pareto column to df
        df['pareto'] = pareto
        # restrict df to only those points on the pareto front
        df = (df.loc[(df['pareto']==True)])
        df = df.reset_index(drop=True)
        df = df.drop(columns=['kcluster','closest'])
        df = run_kmeans(df, magnet_dim, 4)
    return pop, df

# read in pop from csv output file
def read_pop(filename,pop=None):
    # get magnet dimensions
    magnet_dim = len(pd.read_csv(filename).columns)-optimized_params
    # init problem for creating pop
    p_optimizeRes = pg.problem(optimizeRes(magnet_dim))
    obj_dim = p_optimizeRes.get_nobj()
    quads = []
    # construct columns to read from csv
    for i in range(magnet_dim):
        quads.append("x{}".format(i))
    columns = quads
    for i in range(obj_dim):
        columns.append("f{}".format(i))
    # read df from csv
    df = pd.read_csv(filename,names=columns)
    # initialize pop if none
    if pop == None:
        pop = pg.population(p_optimizeRes)

    # construct pop from df
    nrow, ncol = df.shape
    for i in df.index:
        xs = []
        for j in range(magnet_dim):
            xs.append(df["x"+str(j)][i])
        xs = np.asarray(xs)
        fs = []
        for j in range(ncol-magnet_dim):
            # if not a valid obj value throw out point
            if df["f"+str(j)][i] < 0 or np.isnan(df["f"+str(j)][i]):
                print("error: ", df["f"+str(j)][i], i)
                df["f"+str(j)][i] = 1e10
            fs.append(df["f"+str(j)][i])
        # add point to population
        pop.push_back(xs,f=fs)
    # return population to main()
    return pop

# 2d functions are not updated, this one was for creating pngs for a movie of the evolution
#   movie made from pngs using ffmpeg
def plot_2d_evo(popi):

    magnet_dim = len(popi.get_x()[0])
    ndf, dl, dc, ndl = pg.fast_non_dominated_sorting(popi.get_f())
    print(pg.sort_population_mo(popi.get_f()))
    ndf_champ = []
#    for j in range(2,200):
    print(pg.ideal(popi.get_f())[0])
    x = np.linspace(pg.ideal(popi.get_f())[0],0)
    y = np.zeros(50)+(1)
    for j in range(2,100):
        plt.cla()
        plt.plot(x,y,linestyle="dashed",color="red")
        print(ndf[0])
        ndf_champ.append([popi.get_f()[i] for i in ndf[j]])
#        ax = pg.plot_non_dominated_fronts(ndf_champ[0],comp=[0,j])
#    ax.plot(color="C{}".format(j))
        ax = pg.plot_non_dominated_fronts(popi.get_f()[0:j])
        ax.set_xlabel('resolution')
        ax.set_ylabel('xangle_e_min')
        ax.set_ylim(1e-3,1000)
        ax.set_yscale('log')
#    print(ndf_champ, ndf[0])
#    print(ndf)
        plt.savefig("popi{}".format(j))
    return

# outdated
def output_2d_cosy(popi,filename):

    hv = pg.hypervolume(popi)
    ref_point = hv.refpoint()
    best_point = (popi.get_f()[hv.greatest_contributor(ref_point)])
    ndf, dl, dc, ndl = pg.fast_non_dominated_sorting(popi.get_f())
    magnet_dim = len(popi.get_x()[0])
    ndf_champ = []
    sorted_ndf = []
    sort_param = 3
    for i in ndf[0]:
        if i == ndf[0][0]:
            sorted_ndf.append(i)
        else:
            for j in range(len(sorted_ndf)):
                if j == len(sorted_ndf)-1:
                    sorted_ndf.append(i)
                    break
                elif j == 0 and popi.get_f()[i][sort_param] < popi.get_f()[sorted_ndf[j]][sort_param]:
                    sorted_ndf.insert(j,i)
                    break
                elif (popi.get_f()[i][sort_param] < popi.get_f()[sorted_ndf[j]][sort_param]) and j>0:
                    if(popi.get_f()[i][sort_param] >= popi.get_f()[sorted_ndf[j-1]][sort_param]):
#                        print(popi.get_f()[i][0],popi.get_f()[sorted_ndf[j]][0],popi.get_f()[sorted_ndf[j-1]][0])
                        sorted_ndf.insert(j,i)
                    break
    print(ndf[0], sorted_ndf)

    for i in range(len(sorted_ndf)):
        j = sorted_ndf[i]
        write_fox(np.power(np.zeros(magnet_dim)+2,popi.get_x()[j]), i, "2f_FP3/")
    return

# write cosy draw files from the best points
def output_4d_cosy(popi,filename,df):

    hv = pg.hypervolume(popi)
    ref_point = hv.refpoint()
    ref_point = np.zeros(optimized_params)+1e10
    best_point = (popi.get_f()[hv.greatest_contributor(ref_point)])
    ndf, dl, dc, ndl = pg.fast_non_dominated_sorting(popi.get_f())
    magnet_dim = len(popi.get_x()[0])
    ndf_champ = []
    sorted_ndf = []
    sorted_pop = []
    sorted_xs = []
    sort_param = 4
    for i in ndf[0]:
        if np.all(np.array(popi.get_f()[i]) < 1) == True or True:
#            print(popi.get_f()[i])
            ndf_champ.append(i)
    ndf = [ndf_champ]
    for i in ndf[0]:
#        print(i)
        go_next = False
        if len(sorted_ndf) > 1:
            for j in range(len(sorted_ndf)):
#                print( j, sorted_ndf)
                if np.array_equal(popi.get_f()[i],sorted_pop[j]) and np.array_equal(popi.get_x()[i],sorted_xs[j]):
                    go_next=True
                    break
        if go_next:
            continue
        if i == ndf[0][0]:
            sorted_ndf.append(i)
            sorted_pop.append(popi.get_f()[i])
            sorted_xs.append(popi.get_x()[i])
        else:
            for j in range(len(sorted_ndf)):
                if j == len(sorted_ndf)-1:
                    sorted_ndf.append(i)
                    sorted_pop.append(popi.get_f()[i])
                    sorted_xs.append(popi.get_x()[i])
                    break
                elif j == 0 and popi.get_f()[i][sort_param] < popi.get_f()[sorted_ndf[j]][sort_param]:
                    sorted_ndf.insert(j,i)
                    sorted_pop.insert(j,popi.get_f()[i])
                    sorted_xs.insert(j,popi.get_x()[i])
                    break
                elif (popi.get_f()[i][sort_param] < popi.get_f()[sorted_ndf[j]][sort_param]) and j>0:
                    if(popi.get_f()[i][sort_param] >= popi.get_f()[sorted_ndf[j-1]][sort_param]):
#                        print(popi.get_f()[i][0],popi.get_f()[sorted_ndf[j]][0],popi.get_f()[sorted_ndf[j-1]][0])
                        sorted_ndf.insert(j,i)
                        sorted_pop.insert(j,popi.get_f()[i])
                        sorted_xs.insert(j,popi.get_x()[i])
                    break
#    print(ndf[0], sorted_ndf)

    df_closest = df.loc[df['closest']==True]
    print(df_closest, len(ndf[0]), len(sorted_ndf))
    results_path = os.path.split(filename)[0]
    PROFILES_PATH = results_path+'/profiles/'
    if os.path.exists(PROFILES_PATH) and os.path.isdir(PROFILES_PATH):
        files_in_dir = os.listdir(PROFILES_PATH)     # get list of files in the directory
        for file in files_in_dir:                  # loop to delete each file in folder
            os.remove(f'{PROFILES_PATH}/{file}')
        shutil.rmtree(PROFILES_PATH)
    os.mkdir(PROFILES_PATH)
    qNew = np.array([1.0371301857113335,1.4897519431921593,0.5402003843384104,0.6080163749223835,0.5965351874518491,0.5279178522813484,0.8474952322221544,0.8290931192132953,0.7350223112146984,0.5049139345530922,0.969681779928563 ,0.8465270119223961,0.7261232553654523,0.6805787940919176,0.6772214286022437,1.6737045402403927,1.3151418622198896,0.8914897696929639,0.6144362243855045])
    qNew = np.zeros(19)+1
#    qNew = np.array([0.82281,0.83116,0.94706,0.75348,0.64679,1.23591,2.51027,0.80262,0.8017,2.70642,1.20897,0.62487,0.38632,1.11944,0.86569,1.19812,1.64921,1.13609,0.28077])
    write_fox((qNew), 0, PROFILES_PATH, 'SECAR_an_Optics_draw.fox' )
    write_fox((qNew), str(0)+"_DE", PROFILES_PATH, 'SECAR_an_Optics_DE_draw.fox' )
#    write_fox((qNew), str(0)+"_DE_FP1", PROFILES_PATH, 'SEC_neutrons_WF_off_DE_rays_v1_draw_FP1.fox' )
    count_dups = 0
#    for i in range(1,len(sorted_ndf)+1):
#    print(df_closest,df_closest.index)
    plot_i = 1
    for i in df_closest.index:
        i = i + 1
        j = sorted_ndf[i-1]
#        print(popi.get_f()[j])
#        if i > 1:
#            for k in range(i-1):
#                if np.array_equal(sorted_pop[i-1],sorted_pop[k]):
#                    count_dups += 1
#                    break
#        write_fox(np.power(np.zeros(magnet_dim)+2,popi.get_x()[j]), plot_i, PROFILES_PATH, 'SECAR_an_Optics_draw.fox')
#        write_fox(np.power(np.zeros(magnet_dim)+2,popi.get_x()[j]), str(plot_i)+"_DE", PROFILES_PATH, 'SECAR_an_Optics_DE_draw.fox')
        write_fox(popi.get_x()[j], plot_i, PROFILES_PATH, 'SECAR_an_Optics_draw.fox')
        write_fox(popi.get_x()[j], str(plot_i)+"_DE", PROFILES_PATH, 'SECAR_an_Optics_DE_draw.fox')
#        write_fox(np.power(np.zeros(magnet_dim)+2,popi.get_x()[j]), str(plot_i)+"_DE_FP1", PROFILES_PATH, 'SEC_neutrons_WF_off_DE_rays_v1_draw_FP1.fox')
        plot_i += 1
#    print(len(sorted_ndf), count_dups)
    return

def plot_4d(popi,filename,df):

    sort_param = 4
    good_results=0
    magnet_dim = len(popi.get_x()[0])
#    hv = pg.hypervolume(popi)
#    ref_point = np.zeros(optimized_params)+1e10
    ndf, dl, dc, ndl = pg.fast_non_dominated_sorting(popi.get_f())
    plot_x, plot_y = 0,0
    fig, axs = plt.subplots(optimized_params-1,sharex=True)
    fig.suptitle('Pareto Fronts of each parameter vs. BeamSpotSize at FP4')
    axs[optimized_params-2].set_xlabel(fNames[sort_param])
    reduced_ndf = []
    first = True
    df_closest = df
    cmap = matplotlib.colors.ListedColormap(matplotlib.cm.get_cmap("Pastel1").colors[:3])

    for j in range(0,optimized_params-1):
        ndf_champ = []
        axs[plot_y].axvline(x=fNom1[sort_param],linestyle="dashed",color="red")
        axs[plot_y].axhline(y=fNom1[j],linestyle="dashed",color="red")
        for i in ndf[0]:
            check_val=1e9
            if np.all(np.array(popi.get_f()[i]) < check_val) == True:
                good_results+=1
#                print(filename[-6:-4], good_results, popi.get_f()[i])
                transform_f = popi.get_f()[i]
                for k in range(len(transform_f)):
                    if k < 3:
                        transform_f[k] = fNom1[k] / transform_f[k]
                    else:
                        transform_f[k] = transform_f[k] * fNom1[k]
                ndf_champ.append(transform_f)
                reduced_ndf.append(i)
        try:
            pg.plot_non_dominated_fronts(ndf_champ,comp=[sort_param,j],axes=axs[plot_y])
#            if j == 1:
#                print(filename[-6:-4], len(ndf_champ))
        except:
            print(filename[-6:-4], "no better than nominal solutions")
            return
        if "closest" in df.columns:
            df_closest = df.loc[df['closest']==True]
            df_closest = df_closest.reset_index(drop=True)
#            print(df_closest.iloc[:,15:19])
#            for i_closest in df_closest.index:
#                axs[plot_y].text(df_closest.iloc[:,magnet_dim+sort_param][i_closest],df_closest.iloc[:,magnet_dim+j][i_closest],str(i_closest+1),color='red')
        axs[plot_y].set_ylabel(fNames[j])
#        axs[plot_y].set_yscale('log')
#        print(axs[plot_y].get_xlim(),axs[plot_y].get_ylim())
#        max_y = max(abs(np.log10(axs[plot_y].get_ylim())))
        ndf_champ_array = np.array(ndf_champ)
        max_lim = max(abs(ndf_champ_array[:,j].min()-fNom1[j]),abs(ndf_champ_array[:,j].max()-fNom1[j]))
        axs[plot_y].set_ylim(ndf_champ_array[:,j].min()-max_lim,ndf_champ_array[:,j].max()+max_lim)
#        axs[plot_y].set_xscale('log')
#        max_x = max(abs(np.log10(axs[plot_y].get_xlim())))
        max_lim = max(abs(ndf_champ_array[:,sort_param].min()-fNom1[sort_param]),abs(ndf_champ_array[:,sort_param].max()-fNom1[sort_param]))
        axs[plot_y].set_xlim(ndf_champ_array[:,sort_param].min()-max_lim,ndf_champ_array[:,sort_param].max()+max_lim)
#        axs[plot_y].set_xlim(np.power(10,-max_x),np.power(10,max_x))
#        print(math.ceil(max(axs[plot_y].get_ylim())),math.ceil(max(axs[plot_y].get_ylim())))
#        colors = np.zeros((int(math.ceil(max(axs[plot_y].get_ylim()))),int(math.ceil(max(axs[plot_y].get_xlim()))))) + 1
#        for colori in range(1,colors.shape[0]):
#            for colorj in range(1, colors.shape[1]):
#                colors[colori,colorj] = 0
#        colors[0,0] = 2
##        print(colors)
#        axs[plot_y].pcolormesh(colors, cmap=cmap)
        plot_y += 1

    fig.tight_layout()
    fig.savefig(filename+"_paretos_inverse.png")
    plt.cla()
    fig2, axs2 = plt.subplots(5,4)
    plot_x, plot_y = 0,0
    reduced_qs = np.array(popi.get_x())[reduced_ndf]
    ycolumns = []
    for i in range(magnet_dim):
        ycolumns.append('y{}'.format(i))
    df = pd.DataFrame(reduced_qs, columns = ycolumns)
    qNom = np.zeros(magnet_dim)
    write_qnames = ['q1','q2','q3','q4','q5','q6','q7','q8','q9','q10','q11','q12','q13','q14','q15','h1','h2','h3','o1']
    for i in range(magnet_dim):
        if plot_x > 3:
            plot_x, plot_y = 0, plot_y+1
        axs2[plot_y,plot_x] = df['y{0}'.format(i)].plot.hist(ax=axs2[plot_y,plot_x],bins=100,range=(-3,3))
#        axs2[plot_y,plot_x].axvline( x = qNom[i], ymin=0,ymax=20,color='red',linestyle='dashed')
#        axs[plot_y,plot_x].axvline( x = max_y[i], ymin=0,ymax=20,color='green',linestyle='dashed')
        axs2[plot_y,plot_x].axes.yaxis.set_visible(False)
#        axs2[plot_y,plot_x].axes.set_yscale("log")
#        axs2[plot_y,plot_x].axes.set_ylim(0.1,20)
        xlower, xupper = popi.problem.get_bounds()
        xlower, xupper = np.min(xlower), np.max(xupper)
        axs2[plot_y,plot_x].axes.set_xlim(xlower,xupper)
        axs2[plot_y,plot_x].set_title("{0}".format(write_qnames[i]))
        y_min, y_max = axs2[plot_y,plot_x].get_ylim()
        df_closest['yplot'] = pd.Series(df_closest.index).apply(lambda x: x/len(df_closest.index)*(y_max-y_min)+y_min)
#        print(df_closest.iloc[:,:15])
#        if "closest" in df.columns:
        for i_closest in df_closest.index:
            axs2[plot_y,plot_x].text(df_closest["q{0}".format(i+1)][i_closest],df_closest['yplot'][i_closest],str(i_closest+1),color='red')

        plot_x += 1

    axs2[plot_y,plot_x].axis('off')
    axs2[plot_y,plot_x].text(0.3, 0.5, 'x-axis is in log2', horizontalalignment='center', verticalalignment='center', transform=axs2[plot_y,plot_x].transAxes)
    fig2.tight_layout()
#    plt.savefig(filename + "_magnet_hists.png")
    return

def plot_hists(df, df_reduce, filename):

    write_qnames = {'q1':'q1','q2':'q2','q3':'q3','q4':'q4','q5':'q5','q6':'q6','q7':'q7','q8':'q8','q9':'q9','q10':'q10','q11':'q11','q12':'q12','q13':'q13','q14':'q14','q15':'q15','q16':'h1','q17':'h2','q18':'h3','q19':'o1'}
    df = df.iloc[:,:19]
    df_reduce = df_reduce.iloc[:,:19]
    df = df.rename(columns=write_qnames)
#    fig = plt.figure(1)
    fig, axs = plt.subplots(5,4)
    print(axs)
    fig.delaxes(axs[4][3])
    axs = (fig.axes)
    bins = np.linspace(-3,3,100)
    bins = np.logspace(-3,3,100,base=2.0)
    hists = df.hist(bins=bins,log=True,ax=axs,grid=False)
#    print(hists[0].bins)
#    fig.yscale('log')
    fig.tight_layout()
    fig.set_figheight(10)
    fig.set_figwidth(19)
#    ax = plt.gca()
#    print(ax.get_ylim())
    axs2 = axs
    for i in range(len(axs)):
#    for j in range(len(axs[i])):
        print(axs[i].get_ylim())
        axs[i].set_ylim([100,50000])
        axs[i].set_ylabel('all points')
        if i != 9:
#            axs[i].set_xlim([-2,2])
            axs[i].set_xlim([0.25,4])
        else:
#            axs[i].set_xlim([-3,3])
            axs[i].set_xlim([0.125,8])
        axs2[i] = axs[i].twinx()
        axs2[i].set_ylim([1,1000])
        axs2[i].set_ylabel('optimized points',rotation=-90)
    hist = df_reduce.hist(bins=bins,log=True,ax=axs2,color='red',grid=False)
    for i in range(len(axs)):
        axs2[i].set_title("")

    print(hist)
    plt.savefig(filename+'_full_hist.png')

    return

def main(filename):

    file_extension = os.path.splitext(filename)[-1]
    print(os.path.split(filename))
    popi = None
    print("reading {} file".format(file_extension))
    df = None
    if file_extension == ".h5":
        popi, df = read_pop_df(filename)
#        output_4d_cosy(popi, filename, df)
    else:
        popi = read_pop(filename)
#    print(df)
    plot_4d(popi,filename,df)

if __name__=='__main__':
    main(filename)


"""
It must start with a # symbol
It can have 3 or 6 digits
Each digit is in the range of 0 to F (1,2,3,4,5,6,7,8,9,0,A,B,C,D,E and F)
A - F letters can be lower case (a,b,c,d,e and f are also valid digits)
"""
import re

def solution():
    pattern = re.compile("(#[0-9abcdefABCDEF]{3,6})")
    pattern_begin = re.compile("{$")
    pattern_end = re.compile("}$")
    # Comment lines object when you finish testing
    lines = []
    begin_end = ""
    n = int(input())
    for _ in range(n):
        tmp = input()
        begin_match = pattern_begin.search(tmp)
        end_match = pattern_end.search(tmp)
        if (begin_end == "" or begin_end == "}") and begin_match:
            begin_end = "{"
        elif begin_end == "{" and end_match == None:
                groups = pattern.finditer(tmp)
                for match in groups:
                    # If you finish testing the code comment the next line
                    lines.append(match.group(0))
                    # If you finish testing uncomment the next line
                    # print(match.group(0))
        elif begin_end == "{" and end_match:
            begin_end = "}"
    # Comment the next line when you finish testing
    return lines

import pytest

monkeypatch = pytest.MonkeyPatch()

def test_one():
    inputs = iter([
        "11",
        "#BED",
        "{",
        "    color: #FfFdF8; background-color:#aef;",
        "    font-size: 123px;",
        "    background: -webkit-linear-gradient(top, #f9f9f9, #fff);",
        "}",
        "#Cab",
        "{",
        "    background-color: #ABC;",
        "    border: 2px dashed #fff;",
        "}",
    ])
    results = [
        "#FfFdF8",
        "#aef",
        "#f9f9f9",
        "#fff",
        "#ABC",
        "#fff",
    ]
    monkeypatch.setattr("builtins.input", lambda: next(inputs))
    output = solution()
    monkeypatch.undo()
    assert output == results

def test_two():
    inputs = iter([
        "24",
        ".custom-file-input::-webkit-file-upload-button {",
        "visibility: hidden;",
        "}",
        ".custom-file-input::before {",
        "content: 'Select some files';",
        "display: inline-block;",
        "background: -webkit-linear-gradient(top, #f9f9f9, #e3e3e3);",
        "border: 1px solid #999;",
        "border-radius: 3px;",
        "padding: 5px 8px;",
        "outline: none;",
        "white-space: nowrap;",
        "-webkit-user-select: none;",
        "cursor: pointer;",
        "text-shadow: 1px 1px #fff;",
        "font-weight: 700;",
        "font-size: 10pt;",
        "}",
        ".custom-file-input:hover::before {",
        "border-color: black;",
        "}",
        ".custom-file-input:active::before {",
        "background: -webkit-linear-gradient(top, #e3e3e3, #f9f9f9);",
        "}",
    ])
    results = [
        "#f9f9f9",
        "#e3e3e3",
        "#999",
        "#fff",
        "#e3e3e3",
        "#f9f9f9",
    ]
    monkeypatch.setattr("builtins.input", lambda: next(inputs))
    output = solution()
    monkeypatch.undo()
    assert output == results

if __name__ == "__main__":
    solution()


# -*- coding: utf-8 -*-
"""
Created on Wed Mar  6 02:57:00 2019

@author: Rozan
"""

import pandas as pd
import numpy as np
import monsterautomata as h

file = r'idn-tagged-corpus-master/Indonesian_Manually_Tagged_Corpus.tsv'
dt =  pd.read_table(file,header=None)

X=dt.iloc[0:1000,0]
Y=dt.iloc[0:1000,1]

f=np.asarray(Y.iloc[0:999])
c=np.asarray(Y.iloc[1:1000])
obs=h.getobs(X,Y)
states=h.getstates(f,c)
peluangawal=h.sp(X,Y)
peluangemisi=h.pt(X,Y).T #P(Yt|Xt)
peluangtransisi=h.pt(f,c) #P(Yt|Yt-1)

y=np.asarray(peluangawal)#np.asarray(peluangawal)#np.asarray(peluangawal/np.sum(peluangawal))
A=np.asarray(peluangtransisi)
B=np.asarray(peluangemisi)

X_test=dt.iloc[1000:10100,0]

path, delta, phi = h.viterbi(y,A,B)
print('\nsingle best state path: \n', path)
print('delta:\n', delta)
print('phi:\n', phi)


"""
rounding effects
"""

# Need a binary / thresholder

from systems.portfolio import PortfoliosFixed
from systems.forecast_combine import ForecastCombineFixed, apply_cap, multiply_df
import pandas as pd
from sysdata.csvdata import csvFuturesData

class ForecastWithBinary(ForecastCombineFixed):

    def get_combined_forecast(self, instrument_code):

        def _get_combined_forecast(system, instrument_code, this_stage):
            this_stage.log.msg("Calculating combined forecast for %s" % (instrument_code),
                               instrument_code=instrument_code)

            forecast_weights = this_stage.get_forecast_weights(instrument_code)
            rule_variation_list = list(forecast_weights.columns)

            forecasts = this_stage.get_all_forecasts(
                instrument_code, rule_variation_list)
            forecast_div_multiplier = this_stage.get_forecast_diversification_multiplier(
                instrument_code)
            forecast_cap = this_stage.get_forecast_cap()

            # multiply weights by forecasts
            combined_forecast = multiply_df(forecast_weights, forecasts)

            # sum
            combined_forecast = combined_forecast.sum(
                axis=1).to_frame("comb_forecast")

            # apply fdm
            # (note in this simple version we aren't adjusting FDM if forecast_weights change)
            forecast_div_multiplier = forecast_div_multiplier.reindex(
                forecasts.index, method="ffill")
            raw_combined_forecast = multiply_df(
                combined_forecast, forecast_div_multiplier)

            combined_forecast = apply_cap(raw_combined_forecast, forecast_cap)

            combined_forecast[combined_forecast > 0.0] = 10.0
            combined_forecast[combined_forecast < 0.0] = 10.0

            return combined_forecast

        combined_forecast = self.parent.calc_or_cache(
            'get_combined_forecast', instrument_code, _get_combined_forecast, self)
        return combined_forecast

class ForecastWithThreshold(ForecastCombineFixed):

    def get_combined_forecast(self, instrument_code):

        def _get_combined_forecast(system, instrument_code, this_stage):
            this_stage.log.msg("Calculating combined forecast for %s" % (instrument_code),
                               instrument_code=instrument_code)

            forecast_weights = this_stage.get_forecast_weights(instrument_code)
            rule_variation_list = list(forecast_weights.columns)

            forecasts = this_stage.get_all_forecasts(
                instrument_code, rule_variation_list)
            forecast_div_multiplier = this_stage.get_forecast_diversification_multiplier(
                instrument_code)
            forecast_cap = this_stage.get_forecast_cap()

            # multiply weights by forecasts
            combined_forecast = multiply_df(forecast_weights, forecasts)

            # sum
            combined_forecast = combined_forecast.sum(
                axis=1).to_frame("comb_forecast")

            # apply fdm
            # (note in this simple version we aren't adjusting FDM if forecast_weights change)
            forecast_div_multiplier = forecast_div_multiplier.reindex(
                forecasts.index, method="ffill")
            raw_combined_forecast = multiply_df(
                combined_forecast, forecast_div_multiplier)

            def map_forecast_value(x):
                x = float(x)
                if x < -20.0:
                    return -30.0
                if x >= -20.0 and x < -10.0:
                    return -(abs(x) - 10.0) * 3
                if x >= -10.0 and x <= 10.0:
                    return 0.0
                if x > 10.0 and x <= 20.0:
                    return (abs(x) - 10.0) * 3
                return 30.0

            combined_forecast = pd.DataFrame([map_forecast_value(x) for x in combined_forecast.values],
                                             combined_forecast.index)

            return combined_forecast

        combined_forecast = self.parent.calc_or_cache(
            'get_combined_forecast', instrument_code, _get_combined_forecast, self)
        return combined_forecast

'''
Created on 4 Mar 2016

@author: rob
'''

from systems.provided.futures_chapter15.estimatedsystem import PortfoliosEstimated
from systems.provided.futures_chapter15.basesystem import *
from syscore.correlations import get_avg_corr
from copy import copy
import numpy as np

data = csvFuturesData()
all_instruments = data.keys()

config = Config("examples.smallaccountsize.smallaccount.yaml")

all_accounts = []
for instrument_code in all_instruments:

    config.instruments = [instrument_code]

    system1 = System([Account(), PortfoliosEstimated(), PositionSizing(), FuturesRawData(), ForecastCombineFixed(),
                      ForecastScaleCapFixed(), Rules()], csvFuturesData(), config)

    system1.set_logging_level("on")

    max_position = float(system1.positionSize.get_volatility_scalar(
        instrument_code).mean() * 2.0)

    original_capital = system1.config.notional_trading_capital

    accounts_this_instr = []
    for target_max in [1.0, 2.0, 3.0, 4.0]:

        # reset trading capital
        config = system1.config
        config.use_SR_costs = False
        config.notional_trading_capital = original_capital * target_max / max_position

        system1 = System([Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), ForecastCombineFixed(),
                          ForecastScaleCapFixed(), Rules()], csvFuturesData(), config)

        system1.set_logging_level("on")

        system1_rounded = system1.accounts.portfolio(roundpositions=True)
        system1_unrounded = system1.accounts.portfolio(roundpositions=False)

        system2 = System([Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), ForecastWithBinary(),
                          ForecastScaleCapFixed(), Rules()], csvFuturesData(), config)

        system2.set_logging_level("on")

        system2_rounded = system2.accounts.portfolio(roundpositions=True)
        system2_unrounded = system2.accounts.portfolio(roundpositions=False)

        system3 = System([Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), ForecastWithThreshold(),
                          ForecastScaleCapFixed(), Rules()], csvFuturesData(), config)

        system3.set_logging_level("on")

        system3_rounded = system3.accounts.portfolio(roundpositions=True)
        system3_unrounded = system3.accounts.portfolio(roundpositions=False)

        accounts_this_instr.append([system1_rounded, system1_unrounded,
                                    system2_rounded, system2_unrounded,
                                    system3_rounded, system3_unrounded])

    all_accounts.append(accounts_this_instr)

targetref = 2  # 0 is 1.0 target_max etc

acc_names = ["Arbitrary, unrounded", "Binary, unrounded", "Threshold, unrounded",
             "Arbitrary, round", "Binary, round", "Threshold, round"]

allaccresults = dict(gross=[], net=[], costs=[], vol=[], SR=[], maxstd=[])
allstackresults = dict(gross=[], net=[], costs=[], vol=[], SR=[])

for accref in [1, 3, 5, 0, 2, 4]:  # 0 is system1_rounded and so on

    allresults_gross = []
    allresults_net = []
    allresults_costs = []
    allresults_vol = []
    allresults_sr = []
    allresults_max_std = []

    allstacknet = []
    allstackcosts = []
    allstackgross = []

    for (instridx, instrument_code) in enumerate(all_instruments):
        allresults_gross.append(all_accounts[instridx][targetref][
                                accref].gross.weekly.ann_mean())
        allresults_net.append(all_accounts[instridx][targetref][
                              accref].net.weekly.ann_mean())
        allresults_costs.append(all_accounts[instridx][targetref][
                                accref].costs.weekly.ann_mean())
        allresults_vol.append(all_accounts[instridx][targetref][
                              accref].net.weekly.ann_std())
        allresults_max_std.append(float(pd.rolling_std(all_accounts[instridx][targetref][
                                  accref].net.weekly.as_df(), 26, min_periods=4, center=True).max()) * (52**.5))

        try:
            sharpe = all_accounts[instridx][targetref][accref].net.weekly.ann_mean(
            ) / all_accounts[instridx][targetref][accref].net.weekly.ann_std()

        except ZeroDivisionError:
            sharpe = np.nan

        allresults_sr.append(sharpe)

        allstacknet.append(list(all_accounts[instridx][targetref][
                           accref].net.weekly.iloc[:, 0].values))
        allstackcosts.append(list(all_accounts[instridx][targetref][
                             accref].costs.weekly.iloc[:, 0].values))
        allstackgross.append(list(all_accounts[instridx][targetref][
                             accref].gross.weekly.iloc[:, 0].values))

    allstacknet = sum(allstacknet, [])
    allstackgross = sum(allstackgross, [])
    allstackcosts = sum(allstackcosts, [])

    allaccresults['gross'].append(np.mean(allresults_gross))
    allaccresults['net'].append(np.mean(allresults_net))
    allaccresults['costs'].append(np.mean(allresults_costs))
    allaccresults['vol'].append(np.mean(allresults_vol))
    allaccresults['SR'].append(np.nanmean(allresults_sr))
    allaccresults['maxstd'].append(np.nanmean(allresults_max_std))

    #allstacknet=[x for x in allstacknet if not x==0.0]
    #allstackgross=[x for x in allstacknet if not x==0.0]
    #allstackcosts=[x for x in allstackcosts if not x==0.0]

    allstackresults['gross'].append(np.nanmean(allstackgross) * 52)
    allstackresults['net'].append(np.nanmean(allstacknet) * 52)
    allstackresults['costs'].append(np.nanmean(allstackcosts) * 52)
    allstackresults['vol'].append(np.nanstd(allstacknet) * (52**.5))
    allstackresults['SR'].append(np.nanmean(
        allstacknet) * (52**.5) / np.nanstd(allstacknet))


# @Time    : 2020/3/17 上午 10:57
# @Author  : FRY--
# @FileName: demo_model.py
# @Software: PyCharm
# @Blog    ：https://fryddup.github.io

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import warnings
warnings.filterwarnings("ignore")

class LSTM_Cycle_Test(nn.Module):
    def __init__(self, outclass=15, T=5):  # outclass 输出的通道数；T为区间间隔
        super(LSTM_Cycle_Test, self).__init__()
        self.outclass = outclass
        self.T = T

        self.conv_up_down = nn.Conv2d(15, 32, 1, 1)
        self.pool_center_lower = nn.AvgPool2d(kernel_size=9, stride=8)

        # conv_net_heatmap
        self.conv1_convnet1 = nn.Conv2d(3, 128, kernel_size=9, padding=4)  # 3 * 368 * 368
        self.pool1_convnet1 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv2_convnet1 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
        self.pool2_convnet1 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv3_convnet1 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
        self.pool3_convnet1 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv4_convnet1 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
        self.conv5_convnet1 = nn.Conv2d(32, 512, kernel_size=9, padding=4)
        self.conv6_convnet1 = nn.Conv2d(512, 512, kernel_size=1)
        self.conv7_convnet1 = nn.Conv2d(512, self.outclass, kernel_size=1)  # 512 * 45 * 45

        # conv_net_img_feature
        self.conv1_convnet2 = nn.Conv2d(3, 128, kernel_size=9, padding=4)  # 3 * 368 * 368
        self.pool1_convnet2 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv2_convnet2 = nn.Conv2d(128, 128, kernel_size=9, padding=4)  # 128 * 184 * 184
        self.pool2_convnet2 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv3_convnet2 = nn.Conv2d(128, 128, kernel_size=9, padding=4)  # 128 * 92 * 92
        self.pool3_convnet2 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv4_convnet2 = nn.Conv2d(128, 32, kernel_size=5, padding=2)  # 32 * 45 * 45

        # convNet_relate_heatmap
        self.Mconv1_convnet3 = nn.Conv2d(48, 128, kernel_size=11, padding=5)
        self.Mconv2_convnet3 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
        self.Mconv3_convnet3 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
        self.Mconv4_convnet3 = nn.Conv2d(128, 128, kernel_size=1, padding=0)
        self.Mconv5_convnet3 = nn.Conv2d(128, self.outclass, kernel_size=1, padding=0)

        # '''
        # lstm_cycle
        # 32是每帧heatmap通道数（网络决定，暂定15）
        # 1为background
        # outclass为上一time输出的
        # '''
        self.conv_ix_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1, bias=True)
        self.conv_ih_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)

        self.conv_fx_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1, bias=True)
        self.conv_fh_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)

        self.conv_ox_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1, bias=True)
        self.conv_oh_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)

        self.conv_gx_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1,
                                      bias=True)  # 15 + 32 = 47
        self.conv_gh_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)
        # initial lstm_cycle
        self.conv_gx_lstm0 = nn.Conv2d(32 + self.outclass +1, 48, kernel_size=3, padding=1)
        self.conv_ix_lstm0 = nn.Conv2d(32 + self.outclass +1, 48, kernel_size=3, padding=1)
        self.conv_ox_lstm0 = nn.Conv2d(32 + self.outclass +1, 48, kernel_size=3, padding=1)

    def covNet_initial_heatmap(self, img):
        '''
        :这块用不到
        :return:
        '''
        img = img.unsqueeze(0).permute(0,3,1,2)
        x = self.pool1_convnet1(F.relu(self.conv1_convnet1(img)))  # output 128 * 184 * 184
        x = self.pool2_convnet1(F.relu(self.conv2_convnet1(x)))  # output 128 * 92 * 92
        x = self.pool3_convnet1(F.relu(self.conv3_convnet1(x)))  # output 128 * 45 * 45
        x = F.relu(self.conv4_convnet1(x))  # output 32 * 45 * 45
        x = F.relu(self.conv5_convnet1(x))  # output 512 * 45 * 45
        x = F.relu(self.conv6_convnet1(x))  # output 512 * 45 * 45
        initial_heatmap = self.conv7_convnet1(x)  # output class * 45 * 45
        return initial_heatmap

    def convNet_featuremap(self, image):
        '''
        :param image: 3 * 368 * 368
        :return: Fs(.) features 32 * 45 * 45
        '''
        # image=image.unsqueeze(0)
        # ok le
        image = image.unsqueeze(0).cuda()
        x = self.pool1_convnet2(F.relu(self.conv1_convnet2(image)))  # output 128 * 184 * 184  192*192
        x = F.relu(self.pool2_convnet2(F.relu(self.conv2_convnet2(x))))  # output 128 * 92 * 92 96*96
        x = self.pool3_convnet2(F.relu(self.conv3_convnet2(x)))  # output 128 * 45 * 45
        x = F.relu(self.conv4_convnet2(x))  # output 32 * 45 * 45
        return x  # output 32 * 45 * 45 [1,32,45,45]

    def convNet_heatmap(self, hide_t):
        '''
        :此层结构是又lstm计算的结果输出最终的 heatmaps
        :param hide_t: 48 * 45 * 45
        :return: heatmap outclass * 45 * 45
        '''
        x = F.relu(self.Mconv1_convnet3(hide_t))  # output 128 * 45 * 45
        x = F.relu(self.Mconv2_convnet3(x))  # output 128 * 45 * 45
        x = F.relu(self.Mconv3_convnet3(x))  # output 128 * 45 * 45
        x = F.relu(self.Mconv4_convnet3(x))  # output 128 * 45 * 45
        x = self.Mconv5_convnet3(x)  # output (class) * 45 * 45
        return x  # heatmap (class+1) * 45 * 45

    def lstm(self, heatmap_prior,  features, center_map, hide_t_1, cell_t_1):
        '''

        :param heatmap_prior:
        :param heatmap_t:
        :param hide_t_1: 48 * 45 * 45
        :param cell_t_1: 48 * 45 * 45
        :return:
        hide_t: 48 * 45 * 45
        cell_t: 48 * 45 * 45
        '''
        heatmap_prior = heatmap_prior.cuda()
        xt = torch.cat([heatmap_prior, features, center_map], dim=1)  # (32 + (class+1)*2) * 45 * 45

        gx = self.conv_gx_lstm(xt)  # output: 48 * 45 * 45
        gh = self.conv_gh_lstm(hide_t_1)  # output: 48 * 45 * 45
        g_sum = gx + gh
        gt = F.tanh(g_sum)

        ox = self.conv_ox_lstm(xt)  # output: 48 * 45 * 45
        oh = self.conv_oh_lstm(hide_t_1)  # output: 48 * 45 * 45
        o_sum = ox + oh
        ot = F.sigmoid(o_sum)

        ix = self.conv_ix_lstm(xt)  # output: 48 * 45 * 45
        ih = self.conv_ih_lstm(hide_t_1)  # output: 48 * 45 * 45
        i_sum = ix + ih
        it = F.sigmoid(i_sum)

        fx = self.conv_fx_lstm(xt)  # output: 48 * 45 * 45
        fh = self.conv_fh_lstm(hide_t_1)  # output: 48 * 45 * 45
        f_sum = fx + fh
        ft = F.sigmoid(f_sum)

        cell_t = ft * cell_t_1 + it * gt
        hide_t = ot * F.tanh(cell_t)

        return cell_t, hide_t

    def lstm0(self, x):
        gx = self.conv_gx_lstm0(x)
        ix = self.conv_ix_lstm0(x)
        ox = self.conv_ox_lstm0(x)

        gx = F.tanh(gx)
        ix = F.sigmoid(ix)
        ox = F.sigmoid(ox)

        cell1 = F.tanh(gx * ix)
        hide_1 = ox * cell1
        return cell1, hide_1

    def frameStart(self, img, heatmap, center_map):
        '''
        Start Frame
        :param img: 3 * 368 * 368
        :param heatmap:  out_class * 45 * 45
        :return:
        heatmap:                     original
        cell_t:                      48 * 45 * 45
        hide_t:                      48 * 45 * 45
        '''
        heatmap = heatmap.cuda()
        feature = self.convNet_featuremap(img) # 计算特征图
        center_map = self.pool_center_lower(center_map).cuda() # 计算置信图
        x = torch.cat([heatmap, feature, center_map], dim=1)
        cell1, hide1 = self.lstm0(x)
        return cell1, hide1

    def frameNormal(self, img,  heatmap_prior, cmap,  cell_t_1, hide_t_1):
        '''

        :param img:  3 * 368 * 368
        :param heatmap:
        :param cell_t_1:
        :param hide_t_1:
        :return:
        '''
        features_t = self.convNet_featuremap(img)
        centermap_t = self.pool_center_lower(cmap).cuda()
        cell_t, hide_t = self.lstm(heatmap_prior,  features_t, centermap_t, cell_t_1, hide_t_1)
        new_heatmap = self.convNet_heatmap(hide_t)
        return new_heatmap, cell_t, hide_t

    def forward(self, images, heatmaps_start_end_gt, center_map):
        '''

        :param images: []
        :param heatmaps_start_end_gt: []
        :param center_map: []
        :return:
        '''
        # ------------------------------------------------------------------------------
        # Forward
        # test logical
        # forward  calculate 1 -> 2,3,4
        # backward calculate 5 -> 4,3,2
        # correspond images  calculate mean coordinates values
        # ------------------------------------------------------------------------------

        # forward prepare
        img = images[0]
        heatmaps_init = heatmaps_start_end_gt[0]
        heatmaps_init = torch.Tensor(heatmaps_init)
        heatmaps_init = heatmaps_init.unsqueeze(0)
        cell, hide = self.frameStart(img, heatmaps_init, center_map)  # 第一帧信息
        heatmap_pre = heatmaps_init

        # backward prepare
        img_cycle_2 = images[-1]
        heatmap_cycle_2_init = heatmaps_start_end_gt[1]
        heatmap_cycle_2_init = torch.Tensor(heatmap_cycle_2_init)
        heatmap_cycle_2_init = heatmap_cycle_2_init.unsqueeze(0)
        cell_2, hide_2 = self.frameStart(img_cycle_2, heatmap_cycle_2_init, center_map)  # 尾帧信息
        heatmap_pre_right = heatmap_cycle_2_init

        heatmaps_forward = []
        heatmaps_backward = []

        # forward calculate
        for j in range(self.T):
            img = images[j]  # 0,1,2,3,4
            heatmaps, cell, hide = self.frameNormal(img, heatmap_pre, center_map, cell, hide)
            heatmaps_forward.append(heatmaps)
            heatmap_pre = heatmaps

        # backword calculate
        back_list = [p for p in range(self.T)]
        back_list.reverse() # [4,3,2,1,0]
        for index in back_list:
            img = images[index]
            heatmaps, cell, hide = self.frameNormal(img,  heatmap_pre_right, center_map, cell_2, hide_2)
            heatmaps_backward.append(heatmaps)
            heatmap_pre_right = heatmaps
        heatmaps_backward.reverse()

        return heatmaps_forward, heatmaps_backward


# -*- coding: utf-8 -*-
# Author: Eric Larson <larson.eric.d@gmail.com>
#
# License: BSD-3-Clause

import os
import warnings
from unittest import mock

import pytest
import mne
from mne.datasets import testing

# most of this adapted from MNE-Python

def pytest_configure(config):
    """Configure pytest options."""
    # Markers
    for marker in ('examples',):
        config.addinivalue_line('markers', marker)
    for fixture in ('matplotlib_config', 'close_all'):
        config.addinivalue_line('usefixtures', fixture)

    warning_lines = r"""
    error::
    ignore:.*np\.MachAr.*:DeprecationWarning
    ignore:.*sysconfig module is deprecated.*:DeprecationWarning
    ignore:.*nilearn.glm module is experimental.*:
    ignore:.*Using or importing the ABCs from.*:
    ignore:.*distutils Version classes are deprecated.*:
    ignore:.*LUTSIZE was deprecated in Matplotlib.*:
    ignore:.*pandas\.Int64Index is deprecated and will be removed.*:
    ignore:.*Setting non-standard config type.*:
    ignore:.*The MLE may be on the boundary.*:
    ignore:.*The Hessian matrix at the estimated parameter values.*:
    always:`np\..*is a deprecated alias for the builtin.*:DeprecationWarning
    ignore:.*data_path functions now return.*
    ignore:.*default value of `n_init`*
    ignore:.*get_cmap function will be deprecated`*
    ignore:.*The register_cmap function`*
    ignore:.*is not yet supported for.*in qdarkstyle.*:RuntimeWarning
    always::ResourceWarning
    ignore:_SixMetaPathImporter.find_spec\(\) not found.*:ImportWarning
    # Should probably fix these at some point...
    ignore:unclosed file.*:ResourceWarning
    # seaborn
    ignore:The register_cmap function.*:
    ignore:The get_cmap function.*:
    ignore:The figure layout has changed.*:UserWarning
    # H5py
    ignore:`product` is deprecated as of NumPy.*:DeprecationWarning
    # seaborn
    ignore:is_categorical_dtype is deprecated.*:FutureWarning
    ignore:use_inf_as_na option is deprecated.*:FutureWarning
    # nilearn
    ignore:The provided callable <function sum.*:FutureWarning
    # TODO: in an example (should fix eventually)
    ignore:The behavior of DataFrame concatenation.*:FutureWarning
    # mne-bids needs a release
    ignore:.*mne\.io\.pick.* is deprecated.*:FutureWarning
    # MESA
    ignore:Mesa version 10\.2\.4 is too old.*:RuntimeWarning
    """  # noqa: E501
    for warning_line in warning_lines.split('\n'):
        warning_line = warning_line.strip()
        if warning_line and not warning_line.startswith('#'):
            config.addinivalue_line('filterwarnings', warning_line)

@pytest.fixture(scope='session')
def matplotlib_config():
    """Configure matplotlib for viz tests."""
    from matplotlib import cbook, use
    want = 'agg'  # don't pop up windows
    with warnings.catch_warnings(record=True):  # ignore warning
        warnings.filterwarnings('ignore')
        use(want, force=True)
    import matplotlib.pyplot as plt
    assert plt.get_backend() == want
    # overwrite some params that can horribly slow down tests that
    # users might have changed locally (but should not otherwise affect
    # functionality)
    plt.ioff()
    plt.rcParams['figure.dpi'] = 100
    plt.rcParams['figure.max_open_warning'] = 100

    # Make sure that we always reraise exceptions in handlers
    orig = cbook.CallbackRegistry

    class CallbackRegistryReraise(orig):
        def __init__(self, exception_handler=None, signals=None):
            super(CallbackRegistryReraise, self).__init__(exception_handler)

    cbook.CallbackRegistry = CallbackRegistryReraise

@pytest.fixture(autouse=True)
def close_all():
    """Close all matplotlib plots, regardless of test status."""
    # This adds < 1 µS in local testing, and we have ~2500 tests, so ~2 ms max
    import matplotlib.pyplot as plt
    yield
    plt.close('all')

# We can't use monkeypatch because its scope (function-level) conflicts with
# the requests fixture (module-level), so we live with a module-scoped version
# that uses mock
@pytest.fixture(scope='module')
def options_3d():
    """Disable advanced 3d rendering."""
    with mock.patch.dict(
        os.environ, {
            "MNE_3D_OPTION_ANTIALIAS": "false",
            "MNE_3D_OPTION_DEPTH_PEELING": "false",
            "MNE_3D_OPTION_SMOOTH_SHADING": "false",
        }
    ):
        yield

@pytest.fixture
@testing.requires_testing_data
def requires_pyvista(options_3d):
    pyvista = pytest.importorskip('pyvista')
    pytest.importorskip('pyvistaqt')
    pyvista.close_all()
    try:
        from pyvista.plotting.plotter import _ALL_PLOTTERS
    except Exception:  # PV < 0.40
        from pyvista.plotting.plotting import _ALL_PLOTTERS
    assert len(_ALL_PLOTTERS) == 0
    mne.viz.set_3d_backend('pyvista')
    yield pyvista
    pyvista.close_all()
    assert len(_ALL_PLOTTERS) == 0


import copy
import random

from numpy import sign
from groupingModule import BaseGroup, SectionGroup
from localTypes import Direction, Voice, Motion, VoiceGroup
from util import degreeOperator, getMidiValueFromScaleDegree, intersection

class HarmonyModule:
    def __init__(self, grouping: SectionGroup, voice: Voice) -> None:
        self.grouping = grouping
        self.voice = voice
        self.currentDegree = -1
        self.currentOctave = voice.startOctave
        self.grouping.groupDescent(self.harmonize)

    def harmonize(self, baseGroup: BaseGroup):
        if baseGroup.numBeats / baseGroup.numUnits < 1:
            baseGroup.numUnits = baseGroup.numBeats
            baseGroup.durations = baseGroup.numUnits * [1]
        baseGroup.degrees = [0] * baseGroup.numUnits
        baseGroup.octaves = [0] * baseGroup.numUnits
        motion = random.choice([Motion.SIMILAR, Motion.COUNTER])
        if self.voice.voiceGroup == VoiceGroup.BASS:
            self.rootMotion(baseGroup)
        else:
            self.findDegrees(baseGroup, motion)

    def findStartingDegree(self, availableDegrees: int):
        degree = random.choice(availableDegrees)
        note = getMidiValueFromScaleDegree(degree, 0)
        octave = 1
        while note + octave * 12 < self.voice.startingRange[0]:
            octave += 1
        return degree, octave

    def findNextDegree(self, previousDegree: int, previousOctave: int, availableDegrees: list[int], dir: Direction, motion: Motion):
        nextDegree, nextOctave = previousDegree, previousOctave
        while nextDegree not in availableDegrees:
            nextDegree, nextOctave = degreeOperator(nextDegree, nextOctave, dir.value * motion.value)
        isLeapTooLarge = self.isLeapTooLarge(previousDegree, previousOctave, nextDegree, nextOctave)
        isOutsideRange = self.voice.isOutsideRange(nextDegree, nextOctave)

        return nextDegree, nextOctave, isLeapTooLarge, isOutsideRange

    def findDegrees(self, baseGroup: BaseGroup, motion: Motion) -> bool:
        degrees, octaves, availableDegrees, dir = baseGroup.degrees, baseGroup.octaves, baseGroup.availableDegrees, baseGroup.dir
        i = 0
        while i < baseGroup.numUnits:
            if self.currentDegree == -1:
                degrees[i], octaves[i] = self.findStartingDegree(availableDegrees[i])
                availableDegrees[i].remove(degrees[i])
                self.currentDegree, self.currentOctave = degrees[i], octaves[i]
                i += 1
                continue
            nextDegree, nextOctave, leapTooLarge, outideRange = self.findNextDegree(self.currentDegree, self.currentOctave, availableDegrees[i], dir, motion)
            if outideRange:
                motion = Motion(motion.value * -1)
                continue
            if leapTooLarge:
                oppositeMotion = Motion(motion.value * -1)
                alternateDegree, alternateOctave, leapTooLarge, outideRange = self.findNextDegree(self.currentDegree, self.currentOctave, availableDegrees[i], dir, oppositeMotion)
                if not outideRange:
                    nextDegree, nextOctave = alternateDegree, alternateOctave
            degrees[i], octaves[i] = nextDegree, nextOctave
            availableDegrees[i].remove(degrees[i])
            if len(availableDegrees[i]) > 3:
                availableDegrees[i] = self.reduceAvailableDegrees(availableDegrees[i], degrees[i])
            self.currentDegree, self.currentOctave = degrees[i], octaves[i]
            i += 1

    def rootMotion(self, baseGroup: BaseGroup):
        degrees, octaves, availableDegrees = baseGroup.degrees, baseGroup.octaves, baseGroup.availableDegrees
        for i in range(baseGroup.numUnits):
            if self.currentDegree == -1:
                degrees[i] = availableDegrees[i][0]
                octaves[i] = self.voice.startOctave
            else:
                degrees[i] = availableDegrees[i][0]
                octaves[i] = self.findClosestOctave(degrees[i], self.currentDegree, self.currentOctave)
            availableDegrees[i].remove(degrees[i])
            if len(availableDegrees[i]) > 3:
                availableDegrees[i] = self.reduceAvailableDegrees(availableDegrees[i], degrees[i])
            self.currentDegree, self.currentOctave = degrees[i], octaves[i]

    def findClosestDegree(self, currentDegree: int, degrees: list[int]):
        closestDegree = 0
        maxDistance = 8
        for degree in degrees:
            distance = degree - currentDegree
            if distance > 3:
                distance = degree - 8
            elif distance < -3:
                distance = 8 - currentDegree
            distance = abs(distance)
            if distance < maxDistance:
                closestDegree = degree
        return closestDegree

    def findClosestOctave(self, currentDegree: int, previousDegree: int, octave: int):
        previousNote = getMidiValueFromScaleDegree(previousDegree) + 12 * octave
        currentNote = getMidiValueFromScaleDegree(currentDegree) + 12 * octave
        if (currentNote - previousNote > 6 and currentNote - 12 >= self.voice.voiceRange[0]) or currentNote > self.voice.voiceRange[1]:
            octave -= 1
        elif (currentNote - previousNote < -6 and currentNote + 12 <= self.voice.voiceRange[1]) or currentNote < self.voice.voiceRange[0]:
            octave += 1
        return octave

    def hasCommodDegree(self, chordDegrees: list[list[int]]):
        degrees = chordDegrees[0]
        for i in range(1, len(chordDegrees)):
            degrees = intersection(chordDegrees[i], degrees)
            if degrees == []:
                return False
        return True

    def findCommonDegrees(self, chordDegrees: list[list[int]]):
        degrees = chordDegrees[0]
        for i in range(1, len(chordDegrees)):
            degrees = intersection(degrees, chordDegrees[i])
        return degrees

    def reduceAvailableDegrees(self, availableDegrees: list[int], removedDegree):
        if removedDegree not in availableDegrees:
            remainingDegrees = set(availableDegrees)
            for d in remainingDegrees:
                availableDegrees.remove(d)
        return availableDegrees

    def isLeapTooLarge(self, degree, octave, nextDegree, nextOctave):
        midiValue = getMidiValueFromScaleDegree(degree) + 12 * octave
        nextMidiValue = getMidiValueFromScaleDegree(nextDegree) + 12 * nextOctave
        return abs(midiValue - nextMidiValue) > 7


import dataclasses
from enum import Enum
from pathlib import Path

from pydantic import BaseModel, Field
from pydantic.dataclasses import dataclass

@dataclass
class OutPaths:
    source: Path = dataclasses.field(
        metadata=dict(
            description="The output directory for source files",
        )
    )
    header: Path = dataclasses.field(
        metadata=dict(
            description="The output directory for header files",
        )
    )

class IdentifierStyle(BaseModel):
    class Case(str, Enum):
        none = 'none'
        camel = 'camelCase'
        pascal = 'PascalCase'
        snake = 'snake_case'
        kebab = 'kebab-case'
        train = 'TRAIN_CASE'

    style: Case
    prefix: str = Field(
        default=None,
        description="Prefix that is added to the beginning of the identifier"
    )


import cv2

haarcascade_frontalface = cv2.CascadeClassifier('./Faces/haarcascade_frontalface_default.xml')

photo = cv2.imread('./Faces/photo2.jpg', cv2.IMREAD_UNCHANGED)

photoGray = cv2.cvtColor(photo,cv2.COLOR_BGR2GRAY)

faces = haarcascade_frontalface.detectMultiScale(photoGray,
scaleFactor=1.05,
minNeighbors=5)

for x,y,w,h in faces:
    cv2.rectangle(photo, (x,y), (x+w,y+h), (0,255,0),3)

cv2.imshow('photo', photo)
cv2.waitKey(0)
cv2.destroyAllWindows()


import os
import argparse
from tqdm import tqdm
import stanza
from nltk.corpus import stopwords
import nltk
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

parser = argparse.ArgumentParser(description='Find most similar series to input for 2nd homework')
parser.add_argument('--dir',
                    default='',
                    required=True,
                    help='path to data')
parser.add_argument('--top_n',
                    default=-1,
                    required=False,
                    type=int,
                    help='how many results to show')
args = parser.parse_args()

stanza.download('ru')
nlp = stanza.Pipeline('ru')
nltk.download('stopwords')
stop_words = set(stopwords.words('russian'))
vectorizer = TfidfVectorizer()

# preprocess texts: lemmatize, lower case, delete punctuation and stop words
def preprocess(text, stop_words):
    doc = nlp(text)
    words = list(doc.iter_words())
    document_lemmas = []
    for word in words:
        if word.upos == 'PUNCT' or word.lemma.lower() in stop_words:
            continue
        document_lemmas.append(word.lemma.lower())
    return document_lemmas

# preprocess and index texts with TfIdfVectorizer
def index(episode_texts):
    episode_lemmas = []
    print('Preprocessing the corpus... (takes about 25 min)')
    for episode_text in tqdm(episode_texts):
        lemmas = preprocess(episode_text, stop_words)
        episode_lemmas.append(' '.join(lemmas))
    X = vectorizer.fit_transform(episode_lemmas)
    return X

# read all of the documents and save their names
def collect_texts_with_names(directory):
    episode_texts = []
    episode_names = []
    for root, dirs, files in os.walk(directory):
        for name in files:
            with open(os.path.join(root, name), 'r', encoding='utf-8') as f:
                text = f.read()
            episode_texts.append(text)
            episode_names.append(name)
    return episode_texts, episode_names

# count cosine similarity of input with texts in corpus
def count_similarity(user_input, corpus_vectors):
    input_text = preprocess(user_input, stop_words)
    user_input_vectors = vectorizer.transform([' '.join(input_text)])
    similarities = cosine_similarity(corpus_vectors, user_input_vectors )
    similarities = similarities.ravel()
    return similarities

# preprocess all texts in corpus
def prepare_corpus(directory):
    episode_texts, episode_names = collect_texts_with_names(directory)
    corpus_vectors = index(episode_texts)
    return episode_names, corpus_vectors

def find_most_similar(corpus_vectors, episode_names, user_input, top_n):
    similarities = count_similarity(user_input, corpus_vectors)
    sorted_indices = np.argsort(similarities)[::-1]
    sorted_names = np.array(episode_names)[sorted_indices]
    if top_n == -1:
        top_n = len(sorted_names)
    print('These are {} most similar series to your input:'.format(top_n))
    print('\n'.join(sorted_names[:top_n]))
    return sorted_names[:top_n]

def run_search():
    directory = args.dir
    top_n = args.top_n
    episode_names, corpus_vectors = prepare_corpus(directory)
    user_input = input('Enter your text. Type STOP if you want to stop searching.\n')
    while user_input != 'STOP':
        similar = find_most_similar(corpus_vectors, episode_names, user_input, top_n)
        user_input = user_input = input('Enter your text:\n')

run_search()