import numpy as np
from Ui_Files.Dialogs.line_dialog import Ui_Dialog as vhline_ui
from Ui_Files.Dialogs.axis_setup_dialog import Ui_Dialog as axis_setup_ui
from Ui_Files.Dialogs.Save_To_CSV import Ui_Dialog as STC_ui
from Ui_Files.Dialogs.app_settings import Ui_Dialog as app_settings
from Ui_Files.Dialogs.annotation_dialog import Ui_Dialog as annotation_ui
from Ui_Files.Dialogs.new_project_dialog import Ui_Dialog as new_project_dialog
from Ui_Files.Dialogs.simple_treeWidget_dialog import Ui_Dialog as simple_tw
from Ui_Files.Dialogs.bargraph_dialog import Ui_Dialog as bar_dialog
from Ui_Files.Dialogs.spine_color_dialog import Ui_Dialog as spine_color_dialog
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.pyplot import figure
import matplotlib
from matplotlib.widgets import SpanSelector
import pickle
from matplotlib.text import Text
import pandas as pd
from PySide2 import QtCore,QtWidgets
from shutil import copytree, copy2
import os
import seaborn as sns
from gui_elements.settings import ApplicationSettings
from scipy import sparse
from scipy.sparse.linalg import spsolve

def spine_color_fun(self):
    def finish():
        self.ax.spines['bottom'].set_color(ui.bspine_cb.currentText())
        self.ax.spines['top'].set_color(ui.tspine_cb.currentText())
        self.ax.spines['right'].set_color(ui.rspine_cb.currentText())
        self.ax.spines['left'].set_color(ui.lspine_cb.currentText())
        self.ax.xaxis.label.set_color(ui.bspine_cb.currentText())
        self.ax.yaxis.label.set_color(ui.lspine_cb.currentText())
        self.ax.tick_params(axis='x', colors=ui.bspine_cb.currentText())
        self.ax.tick_params(axis='y', colors=ui.lspine_cb.currentText())
        self.settings.setValue('top_spine_color',ui.tspine_cb.currentText())
        self.settings.setValue('bottom_spine_color', ui.bspine_cb.currentText())
        self.settings.setValue('left_spine_color', ui.lspine_cb.currentText())
        self.settings.setValue('right_spine_color', ui.rspine_cb.currentText())
        if self.ax_2 is not None:
            self.ax_2.tick_params(axis='y', colors=ui.rspine_cb.currentText())
            self.ax_2.yaxis.label.set_color(ui.rspine_cb.currentText())
        self.canvas.draw()
    dialog = QtWidgets.QDialog()
    ui = spine_color_dialog()
    ui.setupUi(dialog)
    colors = ['black','red','blue','green','purple','orange','yellow','white']
    ui.bspine_cb.addItems(colors)
    ui.bspine_cb.setCurrentText(self.settings.value('bottom_spine_color'))
    ui.tspine_cb.addItems(colors)
    ui.tspine_cb.setCurrentText(self.settings.value('top_spine_color'))
    ui.lspine_cb.addItems(colors)
    ui.lspine_cb.setCurrentText(self.settings.value('left_spine_color'))
    ui.rspine_cb.addItems(colors)
    ui.rspine_cb.setCurrentText(self.settings.value('right_spine_color'))

    ui.buttonBox.accepted.connect(lambda: finish())
    dialog.exec_()

def graph_test_fun(self):
    self.ax.set_xlim(0,200)
    print(self.ax_2.get_xlim())
    print(self.ax.get_ylim())
    self.ax.autoscale()
    self.n

def tight_figure(self):
    self.fig.tight_layout()
    self.canvas.draw()

def clearimage(self):
    from PIL import Image
    img = Image.open(r'C:\Users\austi\Desktop\image2.png')
    img = img.convert("RGBA")
    datas = img.getdata()

    newData = []
    for item in datas:
        if item[0] == 255 and item[1] == 255 and item[2] == 255:
            newData.append((255, 255, 255, 0))
        else:
            newData.append(item)

    img.putdata(newData)
    img.save(r"C:\Users\austi\Desktop\img2.png", "PNG")

def save_fig(self):
    if self.pickle_opened.text() == 'None':
        filename = QtWidgets.QFileDialog.getSaveFileName(
            None, 'Save Figure', self.settings.value('FIG_PATH'), "pickle (*.pkl);;All Files (*)")
        if filename[0] == '':
            print('passed')
        else:
            with open(filename[0], 'wb') as fid:
                try:
                    pickle.dump(self.fig, fid)
                except TypeError:
                    msg = QtWidgets.QMessageBox()
                    msg.setText("Can't save FigureCanvasQtAgg object")
            msg = QtWidgets.QMessageBox()
            msg.setText('Saved as: ' + filename[0])
            msg.exec_()
    else:
        filename = QtWidgets.QFileDialog.getSaveFileName(
            None, 'Save Figure', self.pickle_opened.text(), "pickle (*.pkl);;All Files (*)")
        if filename[0] == '':
            print('passed')
        else:
            with open(filename[0], 'wb') as fid:
                try:
                    pickle.dump(self.fig, fid)
                    self.pickle_opened.setText(filename[0])
                except TypeError:
                    msg = QtWidgets.QMessageBox()
                    msg.setText("Can't save FigureCanvasQtAgg object")
            msg = QtWidgets.QMessageBox()
            msg.setText('Saved as: ' + filename[0])
            msg.exec_()

def show_pickled_fig(self):
    self.cleargraph()
    if self.pickle_opened.text() == 'None':
        path,ext = QtWidgets.QFileDialog.getOpenFileName(self, 'Pickeled Figure', self.settings.value('FIG_PATH'))
        self.pickle_opened.setText(path)
    else:
        # print(self.pickle_opened.text())
        path, ext = QtWidgets.QFileDialog.getOpenFileName(self, 'Pickeled Figure', self.pickle_opened.text())
        self.pickle_opened.setText(path)
    if path == '':
        self.pickle_opened.setText('None')
    else:
        with open(path, 'rb') as fid:
            try:
                figx = pickle.load(fid)
            except EOFError:
                msg = QtWidgets.QMessageBox()
                msg.setText('There was a problem saving this so there is nothing to load')
                msg.exec_()
        try:
            ApplicationSettings.ALL_DATA_PLOTTED = {}
            self.canvas.draw()
            self.ui.verticalLayout.removeWidget(self.toolbar)
            self.toolbar.close()
            self.ui.verticalLayout.removeWidget(self.canvas)
            self.canvas.close()
            del self.fig
            del self.canvas
            del self.toolbar
            sns.set(context=self.context, style=self.style, palette=self.c_palette,
                    font=self.font, font_scale=self.fs, color_codes=True)
            self.fig = figx
            print(self.fig.axes)
            for num, ax in enumerate(self.fig.axes):
                if num == 0:
                    self.ax_1 = ax
                    self.ax = self.ax_1
                elif num == 1:
                    self.ax_2 = ax
                elif num == 2:
                    self.ax_3 = ax
                elif num == 3:
                    self.ax_4 = ax
                elif num == 4:
                    self.ax_5 = ax
                elif num == 5:
                    self.ax_6 = ax
                elif num == 6:
                    self.ax_7 = ax
                elif num == 7:
                    self.ax_8 = ax
                elif num == 8:
                    self.ax_9 = ax
                elif num == 9:
                    self.ax_10 = ax
                for i in ax.lines:
                    ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                for i in ax.texts:
                    ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                for i in ax.collections:
                    ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                # print(ax)
                # if ax is not self.ax_1 and ax is not self.ax_2:
                #     print('axes2')
                #     print(ax)
                #     self.ax_2 = ax
                #     for i in self.ax_2.lines:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                #     for i in self.ax_2.texts:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                #     for i in self.ax_2.collections:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                # elif ax is not self.ax_1 and ax is not self.ax_2 and ax is not self.ax_3:
                #     print('axes3')
                #     print(ax)
                #     self.ax_3 = ax
                #     for i in self.ax_3.lines:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                #     for i in self.ax_3.texts:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                #     for i in self.ax_3.collections:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                # elif ax is not self.ax_1 and ax is not self.ax_2 and ax is not self.ax_3:
                #     print('axes4')
                #     print(ax)
                #     self.ax_4 = ax
                #     for i in self.ax_4.lines:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                #     for i in self.ax_4.texts:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                #     for i in self.ax_4.collections:
                #         ApplicationSettings.ALL_DATA_PLOTTED[str(i)] = i
                # else:
                #     print('no home')
                #     print(ax)
            self.canvas = FigureCanvas(self.fig)
            self.toolbar = NavigationToolbar(self.canvas, self.canvas, coordinates=True)
            self.ui.verticalLayout.addWidget(self.toolbar)
            self.ui.verticalLayout.addWidget(self.canvas)
            self.canvas.installEventFilter(self)
            self.ax.callbacks.connect('xlim_changed', self.lims_change)
            self.ax.callbacks.connect('ylim_changed', self.lims_change)
            self.dragh = DragHandler(self, figure=self.fig)
            self.canvas.draw()

        except UnboundLocalError:
            msg = QtWidgets.QMessageBox()
            msg.setText('Press "Ctrl+1" for a new axes')
            msg.exec_()
# def Project_view_fun(self):
#     self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_ProjectView)
#     self.restoreDockWidget(self.dw_ProjectView)
#     self.dw_ProjectView.show()
#
#
# def DataBrowser_view_fun(self):
#     self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_Data_Broswer)
#     self.restoreDockWidget(self.dw_Data_Broswer)
#     self.dw_Data_Broswer.show()

def XPS_view_fun(self):
    self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_XPS)
    self.restoreDockWidget(self.dw_XPS)
    self.dw_XPS.show()

def FTIR_view_fun(self):
    self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_FTIR)
    self.restoreDockWidget(self.dw_FTIR)
    self.dw_FTIR.show()

def QCM_view_fun(self):
    self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_QCM)
    self.restoreDockWidget(self.dw_QCM)
    self.dw_QCM.show()

def XRD_view_fun(self):
    self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_XRD)
    self.restoreDockWidget(self.dw_XRD)
    self.dw_XRD.show()

def SE_view_fun(self):
    self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_SE)
    self.restoreDockWidget(self.dw_SE)
    self.dw_SE.show()

def CF_view_fun(self):
    self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_CF)
    self.restoreDockWidget(self.dw_CF)
    self.dw_CF.show()

def Console_view_fun(self):
    pass
    # import sys
    #
    # from qtpy.QtWidgets import QApplication
    # from pyqtconsole.console import PythonConsole
    #
    # def greet():
    #     print("hello world")
    #
    # if __name__ == '__main__':
    #     appl = QApplication([])
    #
    #     console = PythonConsole()
    #     console.push_local_ns('greet', greet)
    #     console.show()
    #     console.eval_in_thread()
    #     sys.exit(appl.exec_())
    # self.addDockWidget(QtCore.Qt.RightDockWidgetArea, self.dw_Console)
    # self.restoreDockWidget(self.dw_Console)
    # self.dw_Console.show()

def Calc_view_fun(self):
    self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, self.dw_calc)
    self.restoreDockWidget(self.dw_calc)
    self.dw_calc.show()

def toggle_legend(self):
    if self.ax.get_legend() is None:
        items = ('medium','xx-small','x-small','small','large','x-large','xx-large')
        size, ok = QtWidgets.QInputDialog.getItem(self, "Legend",
                                        "size", items, 0, False)
        if ok:
            items_2 = ('best', 'upper right', 'upper left', 'lower left', 'lower right', 'right', 'center left',
                       'center right', 'lower center', 'upper center', 'center')
            pos, ok_1 = QtWidgets.QInputDialog.getItem(self, "Legend",
                                                       "position", items_2, 0, False)
            if ok_1:
                all, ok_2 = QtWidgets.QInputDialog.getItem(self, "Legend",
                                                           "Combine all Axis Legends? ", ('No','Yes'), 0, False)
                if ok_2:
                    if all == 'Yes':
                        try:
                            lines_1, labels_1 = self.ax.get_legend_handles_labels()
                            lines_2, labels_2 = self.ax_2.get_legend_handles_labels()
                            print(labels_2)
                            lines = lines_1 + lines_2
                            labels = labels_1 + labels_2
                            self.ax.legend(lines, labels, loc=pos, fontsize = size)
                        except AttributeError:
                            self.ax.legend(loc=pos, fontsize=size)
                    else:
                        self.ax.legend(loc=pos, fontsize=size)
    else:
        self.ax.get_legend().remove()
    self.canvas.draw()

def Save_All_Plotted(self):

    def temp():
        temp = []
        for ix in ui.treeWidget.selectedIndexes():
            text = ix.data()
            temp.append(text)
            name = os.path.join(self.settings.value('SAVED_DATA_PATH'),ix.data())
            try:
                np.savetxt(str(name)+ui.save_as_LE.text()+ui.comboBox.currentText(),
                           ApplicationSettings.ALL_DATA_PLOTTED[ix.data()][0]._xy,delimiter=',')
            except TypeError:
                np.savetxt(str(name) + ui.save_as_LE.text() + ui.comboBox.currentText(),
                           ApplicationSettings.ALL_DATA_PLOTTED[ix.data()]._xy, delimiter=',')
    dialog = QtWidgets.QDialog()
    ui = STC_ui()
    ui.setupUi(dialog)
    dict = ApplicationSettings.ALL_DATA_PLOTTED
    Key_List = []
    for i in dict.keys():
        Key_List.append(QtWidgets.QTreeWidgetItem([i]))
    ui.treeWidget.addTopLevelItems(Key_List)
    ui.buttonBox.accepted.connect(lambda: temp())
    ui.treeWidget.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
    dialog.exec_()

def recoup_data(self):
    dict = ApplicationSettings.ALL_DATA_PLOTTED
    data = []
    x_label = self.ax.get_xlabel()
    y_label = self.ax.get_ylabel()
    for i in dict.keys():
        if type(dict[i]) == list:
            try:
                data.append(dict[i][0]._xy.T)
            except IndexError:
                pass
        else:
            try:
                data.append(dict[i]._xy.T)
            except IndexError:
                pass
    self.cleargraph()

def remove_line(self):
    def finish():
        for j in ui.treeWidget.selectedIndexes():
            line = ApplicationSettings.ALL_DATA_PLOTTED[j.data()]
            print(type(line))
            if isinstance(line, list):
                try:
                    self.ax.lines.remove(line[0])
                    ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                    del line
                    self.canvas.draw()
                except ValueError:
                    msg = QtWidgets.QMessageBox()
                    msg.setText('Removing Line From Wrong Axes!')
                    msg.exec_()
                except AttributeError:
                    pass
            elif isinstance(line,matplotlib.lines.Line2D):
                try:
                    self.ax.lines.remove(line)
                    ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                    del line
                except ValueError:
                    msg = QtWidgets.QMessageBox()
                    msg.setText('Removing Line From Wrong Axes!')
                    msg.exec_()
            elif isinstance(line, matplotlib.offsetbox.DraggableAnnotation):
                msg = QtWidgets.QMessageBox()
                msg.text("Can't Remove Annotation :(")
                msg.exec_()
            elif isinstance(line, matplotlib.collections.PolyCollection):
                line.remove()
                ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                del line
            elif isinstance(line, matplotlib.text.Text):
                line.remove()
                ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                del line
            elif isinstance(line, matplotlib.collections.LineCollection):
                line.remove()
                ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                del line
            else:
                print(line)
                print(type(line))
                line_0 = line.lines[0]
                self.ax.lines.remove(line_0)
                ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                del line_0

    all_lines = ApplicationSettings.ALL_DATA_PLOTTED
    dialog = QtWidgets.QDialog()
    ui = simple_tw()
    ui.setupUi(dialog)
    ui.treeWidget.setSelectionMode(QtWidgets.QAbstractItemView.MultiSelection)
    Key_List = []
    for i in all_lines.keys():
        Key_List.append(QtWidgets.QTreeWidgetItem([i]))
    ui.treeWidget.addTopLevelItems(Key_List)
    ui.buttonBox.accepted.connect(lambda:finish())
    dialog.exec_()
    self.canvas.draw()

def send_to_cf(self):
    fit_list = self.dw_SE.fitted_slopes
    self.dw_CF.ui.tableWidget.setRowCount(len(fit_list))
    for row in range(len(fit_list)):
        self.dw_CF.ui.tableWidget.setItem(row, 1, QtWidgets.QTableWidgetItem(str(fit_list[row])))

def random_c_plot(self):
    Z = np.random.rand(6, 10)
    c = self.ax.pcolor(Z)
    self.canvas.draw()
    self.fig.colorbar(c, ax=self.ax)

def app_settings_fun(self):
    def function():
        self.settings.setValue('app_style',ui.comboBox.currentText())
        # os.execl(sys.executable, sys.executable, *sys.argv)
    def change_path(settings_type):
        path = QtWidgets.QFileDialog.getExistingDirectory()
        self.settings.setValue(settings_type, path)
        if settings_type == 'FTIR_PATH':
            self.dw_FTIR.tree_view.setRootIndex(self.dw_FTIR.model.index(path))
        elif settings_type == 'QCM_PATH':
            self.dw_QCM.tree_view.setRootIndex(self.dw_QCM.model.index(path))
        elif settings_type == 'SE_PATH':
            self.dw_SE.tree_view.setRootIndex(self.dw_SE.model.index(path))
        elif settings_type == 'XPS_PATH':
            self.dw_XPS.tree_view.setRootIndex(self.dw_XPS.model.index(path))
        elif settings_type == 'XRD_PATH':
            self.dw_XRD.tree_view.setRootIndex(self.dw_XRD.model.index(path))
        update()
    def update():
        ui.datapath_le.setText(str(self.settings.value('DATA_PATH')))
        ui.projectpath_le.setText(str(self.settings.value('PROJECT_PATH')))
        ui.savepath_le.setText(str(self.settings.value('SAVED_DATA_PATH')))
        ui.fig_path_label.setText(str(self.settings.value('FIG_PATH')))
        ui.ftir_path_label.setText(str(self.settings.value('FTIR_PATH')))
        ui.se_path_label.setText(str(self.settings.value('SE_PATH')))
        ui.qcm_path_label.setText(str(self.settings.value('QCM_PATH')))
        ui.cf_path_label.setText(str(self.settings.value('CF_PATH')))
        ui.xps_path_label.setText(str(self.settings.value('XPS_PATH')))
        ui.xrd_path_label.setText(str(self.settings.value('XRD_PATH')))
        ui.comboBox.setCurrentText(self.settings.value('app_style'))

    d = QtWidgets.QDialog()
    ui = app_settings()
    ui.setupUi(d)
    ui.comboBox.addItems(QtWidgets.QStyleFactory.keys())
    ui.comboBox.addItems(['DarkStyle'])
    ui.comboBox.addItems(['DarkFusion'])
    ui.comboBox.addItems(['GrayFusion'])
    update()
    ui.buttonBox.accepted.connect(lambda: function())
    ui.comboBox.currentTextChanged.connect(lambda: function())
    ui.changedatapath_pb.clicked.connect(lambda: change_path('DATA_PATH'))
    ui.changesavepath_pb.clicked.connect(lambda: change_path('SAVED_DATA_PATH'))
    ui.changeprojectpath_pb.clicked.connect(lambda: change_path('PROJECT_PATH'))
    ui.change_figpath_pb.clicked.connect(lambda: change_path('FIG_PATH'))
    ui.change_ir_pb.clicked.connect(lambda: change_path('FTIR_PATH'))
    ui.change_qcm_pb.clicked.connect(lambda: change_path('QCM_PATH'))
    ui.change_se_pb.clicked.connect(lambda: change_path('SE_PATH'))
    ui.change_cf_pb.clicked.connect(lambda: change_path('CF_PATH'))
    ui.change_xps_pb.clicked.connect(lambda: change_path('XPS_PATH'))
    ui.changexrdpath_pb.clicked.connect(lambda: change_path('XRD_PATH'))
    d.exec_()

def send_to_custom_data(self):
    pass
    def temp():
        for ix in ui.treeWidget.selectedIndexes():
            text = ix.data()  # or ix.data()
            np.savetxt(ui.save_as_LE.text() + '.csv',
                       ApplicationSettings.ALL_DATA_PLOTTED[text][0]._xy, delimiter=',')
    dialog = QtWidgets.QDialog()
    ui = STC_ui()
    ui.setupUi(dialog)
    dict = ApplicationSettings.ALL_DATA_PLOTTED
    Key_List = []
    for i in dict.keys():
        Key_List.append(QtWidgets.QTreeWidgetItem([i]))
    ui.treeWidget.addTopLevelItems(Key_List)
    ui.buttonBox.accepted.connect(lambda: temp())
    dialog.exec_()

def new_project(self):
    def new_pro(project_name):
        os.makedirs(os.path.join(project_name,ui.project_le.text()))
        os.makedirs(os.path.join(project_name,ui.project_le.text(),'Data'))
        os.makedirs(os.path.join(project_name, ui.project_le.text(), 'Saved'))
        os.makedirs(os.path.join(project_name, ui.project_le.text(), 'Graphs'))
        project_path = os.path.join(project_name, ui.project_le.text())

        self.settings.setValue('PROJECT_PATH',project_path)
        self.settings.setValue('SAVED_DATA_PATH', os.path.join(project_path,'Saved'))
        self.settings.setValue('DATA_PATH', os.path.join(project_path, 'Data'))
        self.settings.setValue('FIG_PATH', os.path.join(project_path, 'Graphs'))
    dialog_path = QtWidgets.QFileDialog.getExistingDirectory()
    d = QtWidgets.QDialog()
    ui = new_project_dialog()
    ui.setupUi(d)
    ui.buttonBox.accepted.connect(lambda: new_pro(dialog_path))
    d.exec_()

def open_project(self):
    dialog = QtWidgets.QFileDialog.getExistingDirectory()
    self.settings.setValue('PROJECT_PATH', dialog)

def load_data(self):
    temp = self.settings.value()
    key_list = [i for i in temp.keys()]
    for i in key_list:
        data = temp[i][0]._xy.T
        self.ax.plot(data[0],data[1])
    self.canvas.draw()

def import_file(self):
    filepath = QtWidgets.QFileDialog.getOpenFileName()[0]
    try:
        filename = os.path.basename(filepath)
        datapath = self.settings.value('DATA_PATH')
        copy2(filepath, os.path.join(datapath,filename))
    except FileNotFoundError:
        pass

def import_directiory_function(self):
    src_directory = QtWidgets.QFileDialog.getExistingDirectory()
    dirname = src_directory.split('/')[-1]
    print(os.path.join(self.settings.value('DATA_PATH'),dirname))
    copytree(src_directory,os.path.join(self.settings.value('DATA_PATH'),dirname))

def plot_annotation(self):
    def finish():
        try:
            a_list = [ui.text.text(), ui.text_2.text(), ui.text_3.text()]
            while ("" in a_list):
                a_list.remove("")
            print(a_list)
            joined_string = "\n".join(a_list)
            print(joined_string)
            if ui.frame.currentText() == 'none':
                bbox = None
            else:
                bbox = dict(dict(boxstyle=ui.frame.currentText(), fc=ui.bgcolor.currentText(), ec="k", alpha=ui.alpha.value()))
            ApplicationSettings.ALL_DATA_PLOTTED[ui.text.text()] = \
                self.ax.text(np.average(self.ax.get_xlim()), np.average(self.ax.get_ylim()), joined_string,
                             color=ui.color.currentText(), fontsize=ui.size.value(), picker=True,
                             fontstyle=ui.fontstyle.currentText(), rotation=ui.rotation.value(), bbox=bbox,
                             ha=ui.ha_cb.currentText(), va='center')
            self.anno_alpha = ui.alpha.value()
            self.anno_text = ui.text.text()
            self.anno_text_2 = ui.text_2.text()
            self.anno_text_3 = ui.text_3.text()
            self.anno_bgcolor = ui.bgcolor.currentText()
            self.anno_fecolor = ui.framecolor.currentText()
            self.anno_rotation = ui.rotation.value()
            self.anno_color = ui.color.currentText()
            self.anno_frame_width = ui.framewidth_sb.value()
            self.anno_frame = ui.frame.currentText()
            self.anno_style = ui.fontstyle.currentText()
            self.anno_size = ui.size.value()
        except ValueError:
            msg = QtWidgets.QMessageBox()
            msg.setText('Annotation Syntax Error')
            msg.exec_()

        self.canvas.draw()
    d = QtWidgets.QDialog()
    ui = annotation_ui()
    ui.setupUi(d)
    ui.bgcolor.addItems(self.color_list)
    ui.framecolor.addItems(self.color_list)
    ui.color.addItems(self.color_list)

    ui.size.setValue(self.anno_size)
    ui.bgcolor.setCurrentText(self.anno_bgcolor)
    ui.color.setCurrentText(self.anno_color)
    ui.framecolor.setCurrentText(self.anno_fecolor)
    ui.frame.setCurrentText(self.anno_frame)
    ui.alpha.setValue(self.anno_alpha)
    ui.text.setText(self.anno_text)
    ui.text_2.setText(self.anno_text_2)
    ui.text_3.setText(self.anno_text_3)
    ui.rotation.setValue(self.anno_rotation)
    ui.framewidth_sb.setValue(self.anno_frame_width)
    ui.fontstyle.setCurrentText(self.anno_style)

    ui.buttonBox.accepted.connect(lambda: finish())
    d.exec_()

def bar_graph(self):
    d = QtWidgets.QDialog()
    ui = bar_dialog()
    ui.setupUi(d)
    l1 = [ui.x_list, ui.y1_list, ui.y2_list, ui.y3_list, ui.label1_le, ui.label2_le, ui.label3_le, ui.width_le, ui.num_sb]
    l2 = ['xlist', 'y1list', 'y2list', 'y3list', 'label1', 'label2', 'label3', 'width', 'num']
    def plot_bar():
        N = ui.num_sb.value()
        width = float(ui.width_le.text())
        labels = ui.x_list.text().split(' ')
        x = np.arange(len(labels))  # the label locations

        if N == 1:
            y1 = [float(i) for i in ui.y1_list.text().split(' ')]
            self.ax.bar(x, y1, width, label=ui.label1_le.text())
        elif N == 2:
            y1 = [float(i) for i in ui.y1_list.text().split(' ')]
            self.ax.bar(x - width / 2, y1, width, label=ui.label1_le.text())
            y2 = [float(i) for i in ui.y2_list.text().split(' ')]
            self.ax.bar(x + width / 2, y2, width, label=ui.label2_le.text())
        elif N == 3:
            y1 = [float(i) for i in ui.y1_list.text().split(' ')]
            self.ax.bar(x - 3*width / 3, y1, width, label=ui.label1_le.text())
            y2 = [float(i) for i in ui.y2_list.text().split(' ')]
            self.ax.bar(x + 3*width / 3, y2, width, label=ui.label2_le.text())
            y3 = [float(i) for i in ui.y3_list.text().split(' ')]
            self.ax.bar(x , y3, width, label=ui.label3_le.text())

        # Add some text for labels, title and custom x-axis tick labels, etc.
        self.ax.set_xticks(x)
        self.ax.set_xticklabels(labels)
        self.ax.legend()

        # self.fig.bar_label(rects1, padding=3)
        # self.fig.bar_label(rects2, padding=3)
        for i in range(len(l1)):
            self.bar[l2[i]] = l1[i].text()
        self.canvas.draw()
    for i in range(len(l1)):
        try:
            l1[i].setText(self.bar[l2[i]])
        except AttributeError:
            l1[i].setValue(int(self.bar[l2[i]]))
    ui.buttonBox.accepted.connect(lambda: plot_bar())
    d.exec_()

def baseline_als(y, lam, p, niter=10):
    L = len(y)
    D = sparse.diags([1,-2,1],[0,-1,-2], shape=(L,L-2))
    w = np.ones(L)
    for i in range(niter):
        W = sparse.spdiags(w, 0, L, L)
        Z = W + lam * D.dot(D.transpose())
        z = spsolve(Z, w*y)
        w = p * (y > z) + (1-p) * (y < z)
    return z

def fill_cols_fun(self):
    self.ui.tw_x.clear()
    self.ui.tw_y.clear()
    path = self.model.filePath(self.tree_view.currentIndex())
    filename, extension = os.path.splitext(self.model.filePath(self.tree_view.currentIndex()))
    try:
        skip_rows = self.ui.skip_rows_sb.value()
    except AttributeError:
        print('No Skipped Rows')
        skip_rows = 0
    if extension == '.CSV' or extension == '.csv':
        self.data = pd.read_csv(path, delimiter=',', skiprows=skip_rows)
    elif extension == '.xls' or extension == '.xlsx':
        excelfile = pd.ExcelFile(path)
        self.data = pd.read_excel(excelfile, "Sheet1", skiprows=skip_rows)
    elif extension == '.X01':
        self.data = pd.read_csv(path, delimiter='   ', skiprows=48, engine='python')
    elif extension =='.txt':
        self.data = pd.read_csv(path, sep='\t', skiprows=skip_rows)
    elif extension == '.dat':
        self.data = pd.read_csv(path, sep=' ', skiprows=skip_rows)
    else:
        print(extension)
        self.data = pd.read_csv(path, sep='\t')

    strings = [col for col in self.data.columns]
    column_list_x = []
    column_list_y = []
    for i in strings:
        column_list_x.append(QtWidgets.QTreeWidgetItem([i]))
        column_list_y.append(QtWidgets.QTreeWidgetItem([i]))
        self.ui.tw_x.addTopLevelItems(column_list_x)
        self.ui.tw_y.addTopLevelItems(column_list_y)

def move_line(self):
    def finish():
        for j in ui.treeWidget.selectedIndexes():
            line = ApplicationSettings.ALL_DATA_PLOTTED[j.data()]
            print(line)
            print(type(line))
            if isinstance(line, list):
                new_line = line[0]._xy.T[1] + float(ui.lineEdit.text())
                print(new_line)
                ApplicationSettings.ALL_DATA_PLOTTED[j.data() + '_' + str(ui.lineEdit.text())] = self.ax.plot(
                    line[0]._xy.T[0], new_line)
                self.ax.lines.remove(line[0])
                ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                del line
            elif isinstance(line, matplotlib.lines.Line2D):
                new_line = line._xy.T[1] + float(ui.lineEdit.text())
                print(new_line)
                print()
                ApplicationSettings.ALL_DATA_PLOTTED[j.data()+'_'+str(ui.lineEdit.text())] = self.ax.plot(
                    line._xy.T[0],new_line)
                self.ax.lines.remove(line)
                ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
                del line
            else:
                msg = QtWidgets.QMessageBox()
                msg.setText('Make sure its a line')
                msg.exec_()

    all_lines = ApplicationSettings.ALL_DATA_PLOTTED
    dialog = QtWidgets.QDialog()
    ui = simple_tw()
    ui.setupUi(dialog)
    ui.treeWidget.setSelectionMode(QtWidgets.QAbstractItemView.MultiSelection)
    Key_List = []
    for i in all_lines.keys():
        Key_List.append(QtWidgets.QTreeWidgetItem([i]))
    ui.treeWidget.addTopLevelItems(Key_List)
    ui.buttonBox.accepted.connect(lambda:finish())
    dialog.exec_()
    self.canvas.draw()

def change_axis(self,axis):
    if axis ==1:
        self.ax = self.ax_1
    elif axis ==2:
        self.ax = self.ax_2
    elif axis ==3:
        self.ax = self.ax_3
    elif axis ==4:
        self.ax = self.ax_4
    elif axis ==5:
        self.ax = self.ax_5
    elif axis ==6:
        self.ax = self.ax_6
    elif axis ==7:
        self.ax = self.ax_7
    elif axis ==8:
        self.ax = self.ax_8
    elif axis ==9:
        self.ax = self.ax_9
    try:
        self.ax.callbacks.connect('xlim_changed', self.lims_change)
        self.dragh = DragHandler(self, figure=self.fig)
    except AttributeError:
        print('No Axis made yet')

def axis_setup_fun(self):
    possible_axes = [self.ax_1, self.ax_2, self.ax_3, self.ax_4, self.ax_5,
                     self.ax_6, self.ax_7, self.ax_8, self.ax_9, self.ax_10]
    print(possible_axes)
    self.ax.clear()
    self.fig.clf()
    self.ui.verticalLayout.removeWidget(self.toolbar)
    self.ui.verticalLayout.removeWidget(self.canvas)
    self.toolbar.close()
    self.canvas.close()
    self.fig = figure(num=None, figsize=(8, 6), dpi=80)
    self.canvas = FigureCanvas(self.fig)
    self.toolbar = NavigationToolbar(self.canvas, self.canvas, coordinates=True)
    self.ui.verticalLayout.addWidget(self.toolbar)
    self.ui.verticalLayout.addWidget(self.canvas)
    self.canvas.installEventFilter(self)
    self.ax.callbacks.connect('xlim_changed', self.lims_change)
    vertical_axes = QtWidgets.QInputDialog.getInt(self, 'Axes Setup','Vertical?',1)
    horizontal_axes = QtWidgets.QInputDialog.getInt(self, 'Axes Setup', 'Horizontal?',1)
    axesstr = str(vertical_axes[0]) + str(horizontal_axes[0])
    axesint = [horizontal_axes[0], vertical_axes[0]]
    times = axesint[0]*axesint[1]

    pos_axes = [int(axesstr + str(i)) for i in range(1, times+1)]
    for i,num in enumerate(pos_axes):
        print(i)
        print(num)
        possible_axes[i] = self.fig.add_subplot(num)
        print(possible_axes)
    self.ax = self.ax_1
    self.fig.tight_layout()
    self.canvas.draw()

def add_line_to_graph(self):
    def finish():
        if ui.comboBox.currentText() == 'Verticle Line':
            ApplicationSettings.ALL_DATA_PLOTTED['vline'+str(ui.line_pos_sb.value())] = self.ax.axvline(x=ui.line_pos_sb.value(), linestyle="--", lw=ui.line_width_sb.value(),
                            color=ui.color_cb.currentText(), alpha=ui.alpha_sb.value())
        elif ui.comboBox.currentText() == 'Horizontal Line':
            ApplicationSettings.ALL_DATA_PLOTTED['hline'+str(ui.line_pos_sb.value())] = self.ax.axhline(y=ui.line_pos_sb.value(), linestyle="--", lw=ui.line_width_sb.value(),
                            color=ui.color_cb.currentText(), alpha=ui.alpha_sb.value())
        self.canvas.draw()
    d = QtWidgets.QDialog()
    ui = vhline_ui()
    ui.setupUi(d)
    ui.buttonBox.accepted.connect(lambda: finish())
    d.exec_()

def keycheck(dict, key):
    if key in dict.keys():
        return True
    else:
        return False

def addmpl(self,fig):
    self.main_window.canvas = FigureCanvas(fig)
    self.ui.verticalLayout.addWidget(self.main_window.canvas)
    self.main_window.canvas.draw()
    self.toolbar = NavigationToolbar(self.main_window.canvas,
                                     self, coordinates=True)
    self.ui.verticalLayout.addWidget(self.main_window.toolbar)

def rmmpl(self):
    self.ui.verticalLayout.removeWidget(self.main_window.canvas)
    self.main_window.canvas.close()
    self.ui.verticalLayout.removeWidget(self.main_window.toolbar)

def qcm_hc_mass(data, start_time=float, end_time=float, purge_time_a=float, purge_time_b=float, num_cycles=int):
    time = data[0]
    mass = data[2]
    exposure = []
    exposure_idx = []
    mass_hc_a = []
    mass_hc_b = []
    exp_length_idx = []
    # time_in_idx = find_nearest(time, end_time) - find_nearest(time, start_time)
    mass_process = mass[find_nearest(time, start_time): find_nearest(time, end_time)]
    time_process = time[find_nearest(time, start_time):find_nearest(time, end_time)]
    print(time_process)
    print(mass_process)
    for i in range(num_cycles):
        if i % 2 == 0:
            exposure.append(start_time + (purge_time_a * i))
            exposure_idx.append(find_nearest(time_process, start_time + (purge_time_a * i)))
        elif i % 2 == 1:
            exposure.append(start_time + (purge_time_b * i))
            exposure_idx.append(find_nearest(time_process, start_time + (purge_time_b * i)))
    for k in range(len(exposure_idx) - 1):
        exp_length_idx.append(exposure_idx[k + 1] - exposure_idx[k])
    for j in range(num_cycles - 1):
        if j % 2 == 0:
            mass_hc_a.append(
                mass_process[int((exposure_idx[j + 1] - exposure_idx[j]) * .9 + exposure_idx[j])] - mass_process[
                    exposure_idx[j]])
        elif j % 2 == 1:
            mass_hc_b.append(
                mass_process[int((exposure_idx[j + 1] - exposure_idx[j]) * .9 + exposure_idx[j])] - mass_process[
                    exposure_idx[j]])
    fc = np.zeros(len(mass_hc_b))
    for i in range(len(mass_hc_b)):
        fc[i] = mass_hc_a[i]+mass_hc_b[i]
    return mass_hc_a,mass_hc_b, fc

def plot_QCM(self, time,pressure,mass,a_exp=int,b_exp=int,ttp=float,threshold=float,from_time=int,to_time=int,
             wait_time=float, density = float):
    #  Few more assignments Exposures is the time of each Exposure, the Index is just what index it
    #  shows up as. M_Diff and P_Diff are the mass and pressure difference at different exposures
    #  Purge Time Index is the number of data points between one exposure and the next, does not
    #  differentiate for the exposure time.
    Exposures = []
    A_Exposures = a_exp
    Exposure_index = []
    QCM_M_Diff = []
    QCM_P_Diff = []
    MC_A = []
    MC_B = []
    MC_P = []
    MC_N = []
    MC_Cycle = []

    t = 0
    #  Time through purge is the time to "wait" after there is an exposure to not overcount exposures
    #  Time is just a temp variable that gives the time of the last exposure

    # This for loop is just for appending the exposures to Exposures and Exposure_index nee
    for num in range(len(pressure) - 10):
        QCM_M_Diff.append(mass[num + 3] - mass[num])
        QCM_P_Diff.append(pressure[num + 3] - pressure[num])
        if QCM_P_Diff[num] >= threshold and time[num] - t > wait_time and time[num] > from_time and time[num] < to_time:
            Exposures.append(time[num])
            Exposure_index.append(num)
            t = time[num]

            # self.main_window.ax.axvline(time[num], color='gray', lw=1, alpha=0.5)
            # above line will add a line for each exposure if someone is uncertain about if its right

    # This for loop is knowing how long each purge time is (or time from one exposure to next)
    Purge_Time_Index = np.zeros(len(Exposure_index))

    for num in range(len(Exposure_index) - 1):
        if (Exposure_index[num + 1] - Exposure_index[num]) < 2000:
            Purge_Time_Index[num] = Exposure_index[num + 1] - Exposure_index[num]

    Purge_Time_Index[-1] = Purge_Time_Index[0]

    # print(Purge_Time_Index)

    #  for loop finds the mass at the: Exposure time + Purge_time*(time_through_purge) which takes the time
    #  of the last exposure and then adds something like 0.8*Time of purge... Insert is to do the last one
    #  if else says if Mass_Middle was bigger or smaller than the last one and sorts it into one of two lists
    #  Last if else sorts based on if num is odd or even... For A-B experiemtns.. Nothing for A BBB or some
    #  Thing like that
    mass_middle = np.zeros(len(Exposure_index)+2)
    mass_middle[0] = mass[Exposure_index[0] - int(Purge_Time_Index[0] * ttp)]
    # mass_middle.insert(0, mass[Exposure_index[0] - int(Purge_Time_Index[0] * ttp)])

    for num in range(len(Exposure_index)):
        # mass_middle.append(mass[int(Exposure_index[num] + int(Purge_Time_Index[num] * ttp))])
        mass_middle[num+1] = mass[int(Exposure_index[num] + int(Purge_Time_Index[num] * ttp))]
        # self.main_window.ax.axvline(time[num], color='gray', lw=1, alpha=0.5)
        # above line will add a line for each time if someone is uncertain about if its right

    # mass_middle.append(mass[Exposure_index[-1] + int(Purge_Time_Index[0] * ttp)])
    mass_middle[-1] = mass[Exposure_index[-1] + int(Purge_Time_Index[0] * ttp)]

    # print('There was ' + str(len(Exposures)) + '  Precursor Exposures')
    self.main_window.dw_QCM.ui.num_doses.setText(str(len(Exposures)))

    for num in range(len(Exposure_index)):
        if mass_middle[num + 1] - mass_middle[num] > 0:
            MC_P.append(mass_middle[num + 1] - mass_middle[num])
        elif mass_middle[num + 1] - mass_middle[num] < 0:
            MC_N.append(mass_middle[num + 1] - mass_middle[num])
        if num % (b_exp + A_Exposures) == 0:
            MC_Cycle.append(mass_middle[num + b_exp + A_Exposures] - mass_middle[num])
        if num == a_exp - 1:
            MC_A.append(mass_middle[num + 1] - mass_middle[num + 1 - A_Exposures])
        elif num == b_exp + a_exp - 1:
            MC_B.append(mass_middle[num + 1] - mass_middle[num + 1 - b_exp])
        elif num == a_exp + b_exp:
            a_exp = a_exp + A_Exposures + b_exp
            if num == a_exp - 1:
                MC_A.append(mass_middle[num + 1] - mass_middle[
                    num + 1 - A_Exposures])

            # elif num == B_Doses + A_Doses:
            #     self.QCM_Dict['MC_B'].QCM_B.append(Mass_Middle[num] - Mass_Middle[num + 1 - int(B_Exposures.text())])
    half_cycle_density_A = np.zeros(len(MC_A))
    half_cycle_density_B = np.zeros(len(MC_B))
    full_cycle_density = np.zeros(len(MC_Cycle))
    for i in range(len(MC_A)):
        half_cycle_density_A[i] = MC_A[i] / (10.0 * density)
    for i in range(len(MC_B)):
        half_cycle_density_B[i] = MC_B[i] / (10.0 * density)
    for i in range(len(MC_Cycle)):
        full_cycle_density[i] = MC_Cycle[i] / (10 * density)

    return MC_A, MC_B, MC_Cycle, half_cycle_density_A, half_cycle_density_B, full_cycle_density

def find_nearest(array: object, value: object) -> object:
    array = np.asarray(array)
    idx = (np.abs(array - value)).argmin()
    return idx

def select_delete_range(self, enabled):
    if enabled:
        self.span = SpanSelector(self.main_window.ax, remove_data_0, 'horizontal', useblit=False,
                                 rectprops=dict(alpha=0.2, facecolor='blue'))
    else:
        del self.span

def remove_data_0(self, minimum, maximum):
    def finish():
        data = []
        for j in ui.treeWidget.selectedIndexes():
            line = ApplicationSettings.ALL_DATA_PLOTTED[j.data()]
            data.append(line._xy.T)
            temp = line._xy.T[0]
            print(find_nearest(temp,minimum))
            print(find_nearest(temp, maximum))
        print(data)
            # if isinstance(line, list):
            #     try:
            #         self.ax.lines.remove(line[0])
            #         ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
            #         del line
            #         self.canvas.draw()
            #     except ValueError:
            #         msg = QtWidgets.QMessageBox()
            #         msg.setText('Removing Line From Wrong Axes!')
            #         msg.exec_()
            #     except AttributeError:
            #         pass
            # elif isinstance(line, matplotlib.lines.Line2D):
            #     try:
            #         self.ax.lines.remove(line)
            #         ApplicationSettings.ALL_DATA_PLOTTED.pop(j.data())
            #         del line
            #     except ValueError:
            #         msg = QtWidgets.QMessageBox()
            #         msg.setText('Removing Line From Wrong Axes!')
            #         msg.exec_()
    all_lines = ApplicationSettings.ALL_DATA_PLOTTED
    dialog = QtWidgets.QDialog()
    ui = simple_tw()
    ui.setupUi(dialog)
    ui.treeWidget.setSelectionMode(QtWidgets.QAbstractItemView.MultiSelection)
    Key_List = []
    for i in all_lines.keys():
        Key_List.append(QtWidgets.QTreeWidgetItem([i]))
    ui.treeWidget.addTopLevelItems(Key_List)
    ui.buttonBox.accepted.connect(lambda: finish())
    dialog.exec_()
    self.canvas.draw()

def color_test(button):
    color = QtWidgets.QColorDialog()
    color.exec_()
    try:
        button.setStyleSheet("background-color: {}".format(color.currentColor().name()))
        button.setText(color.currentColor().name())
    except ValueError:
        print('err')

def gridspec(self):
    # self.ax.clear()
    # self.fig.clf()
    # self.ui.verticalLayout.removeWidget(self.toolbar)
    # self.ui.verticalLayout.removeWidget(self.canvas)
    # self.toolbar.close()
    # self.canvas.close()
    # self.fig = figure(num=None, figsize=(8, 6), dpi=80)
    # spec4 = self.fig.add_gridspec(ncols=1, nrows=2, height_ratios=[1, 3])
    # self.canvas = FigureCanvas(self.fig)
    # self.toolbar = NavigationToolbar(self.canvas, self.canvas, coordinates=True)
    # self.ui.verticalLayout.addWidget(self.toolbar)
    # self.ui.verticalLayout.addWidget(self.canvas)
    # self.canvas.installEventFilter(self)
    # self.ax1 = self.fig.add_subplot(spec4[0, 0])
    # self.ax2 = self.fig.add_subplot(spec4[1, 0])
    # self.canvas.draw
    rw, rh = 2, 5
    box1 = self.ax_1.get_position()
    box2 = self.ax_2.get_position()
    # current dimensions
    w1,h1 = box1.x1-box1.x0, box1.y1-box1.y0
    w2,h2 = box2.x1-box2.x0, box2.y1-box2.y0
    top1 = box1.y0+h1
    #top2 = box2.y0+h2
    full_h = h1+h2   #total height

    # compute new heights for each axes
    new_h1 = full_h*rw/(rw + rh)
    new_h2 = full_h*rh/(rw + rh)+.03

    #btm1,btm2 = box1.y0, box2.y0
    new_bottom1 = top1-new_h1

    # finally, set new location/dimensions of the axes
    self.ax_1.set_position([box1.x0, new_bottom1, w1, new_h1])
    self.ax_2.set_position([box2.x0, box2.y0, w2, new_h2])

    self.canvas.draw()

def label_size(self):
    size, ok = QtWidgets.QInputDialog.getDouble(None, 'Axis Label Size', 'Size: ')
    if ok:
        # sns.set(font_scale=size)
        ax_xlabel = self.ax_1.get_xlabel()
        self.ax_1.set_xlabel(ax_xlabel, fontsize=size)
        ax_ylabel = self.ax_1.get_ylabel()
        self.ax_1.set_ylabel(ax_ylabel, fontsize=size)
    #     self.ax.set_ylabel(self.ax.get_ylabel(), fontsize=size)
    #     self.ax.set_xticklabels(self.ax.get_xticklabels(), fontsize=size)
    #     self.ax.set_yticklabels(self.ax.get_yticklabels(), fontsize=size)
        try:
            ax2_xlabel = self.ax_2.get_xlabel()
            self.ax_2.set_xlabel(ax2_xlabel, fontsize=size)
            # self.ax_2.set_xticklabels(self.ax_2.get_xticklabels(), fontsize=size)
        except AttributeError:
            print('AttError')
        try:
            ax2_ylabel = self.ax_2.get_ylabel()
            self.ax_2.set_ylabel(ax2_ylabel, fontsize=size)
            # self.ax_2.set_ylabel(self.ax_2.get_ylabel(), fontsize=size)
            # self.ax_2.set_yticklabels(self.ax_2.get_yticklabels(), fontsize=size)
        except AttributeError:
            print('AttError')
        self.canvas.draw()

def print_something():
    print('something')

class DragHandler(object):
    """ A simple class to handle Drag n Drop.

    This is a simple example, which works for Text objects only
    """
    def __init__(self, main_window, figure=None):
        """ Create a new drag handler and connect it to the figure's event system.
        If the figure handler is not given, the current figure is used instead
        """
        if figure is None : figure = matplotlib.pyplot.gcf()
        # simple attibute to store the dragged text object
        self.dragged = None
        self.main_window = main_window
        # Connect events and callbacks
        figure.canvas.mpl_connect("pick_event", self.on_pick_event)
        figure.canvas.mpl_connect("button_release_event", self.on_release_event)
        # figure.canvas.mpl_connect("button_release_event", self.on_release)

    def on_pick_event(self, event):
        " Store which text object was picked and were the pick event occurs."
        if isinstance(event.artist, Text):
            self.dragged = event.artist
            self.pick_pos = (event.mouseevent.xdata, event.mouseevent.ydata)
        return True

    def on_release_event(self, event):
        " Update text position and redraw"
        if self.dragged is not None:
            old_pos = self.dragged.get_position()
            new_pos = (old_pos[0] + event.xdata - self.pick_pos[0],
                       old_pos[1] + event.ydata - self.pick_pos[1])
            self.dragged.set_position(new_pos)
            self.dragged = None
            self.main_window.canvas.draw()
        return True


import os
from datetime import timedelta

from tornado import gen

from tornadowebapi import exceptions
from tornadowebapi.resource import Resource
from tornadowebapi.resource_handler import ResourceHandler
from tornadowebapi.traitlets import Unicode, Dict, Absent

from remoteappmanager.netutils import wait_for_http_server_2xx
from remoteappmanager.webapi.decorators import authenticated

class Container(Resource):
    mapping_id = Unicode(allow_empty=False, strip=True)
    configurables = Dict(optional=True, scope="input")
    name = Unicode(scope="output", allow_empty=False, strip=True)
    image_name = Unicode(scope="output", allow_empty=False, strip=True)

class ContainerHandler(ResourceHandler):
    resource_class = Container

    @gen.coroutine
    @authenticated
    def create(self, resource, **kwargs):
        """Create the container."""

        mapping_id = resource.mapping_id

        webapp = self.application
        accountings = webapp.db.get_accounting_for_user(self.current_user)
        container_manager = webapp.container_manager

        choice = [(accounting.id,
                   accounting.application,
                   accounting.application_policy)
                  for accounting in accountings
                  if accounting.id == mapping_id]

        if not choice:
            self.log.warning("Could not find resource "
                             "for mapping id {}".format(mapping_id))
            raise exceptions.BadRepresentation(
                message="unrecognized mapping_id")

        _, app, policy = choice[0]

        image = yield container_manager.image(app.image)
        if image is None:
            raise exceptions.BadRepresentation(message="unrecognized image")

        try:
            environment = self._environment_from_configurables(image, resource)
        except Exception:
            self.log.exception("Invalid configurables")
            raise exceptions.BadRepresentation(message="invalid configurables")

        # Everything is fine. Start and wait for the container to come online.
        try:
            container = yield self._start_container(
                self.current_user.name,
                app,
                policy,
                mapping_id,
                self.application.command_line_config.base_urlpath,
                environment=environment
                )
        except Exception as e:
            raise exceptions.Unable(message=str(e))

        try:
            yield self._wait_for_container_ready(container)
        except Exception as e:
            self._remove_container_noexcept(container)
            raise exceptions.Unable(message=str(e))

        try:
            yield self.application.reverse_proxy.register(
                container.urlpath,
                container.host_url)
        except Exception as e:
            self._remove_container_noexcept(container)
            raise exceptions.Unable(message=str(e))

        resource.identifier = container.url_id

    @gen.coroutine
    @authenticated
    def retrieve(self, resource, **kwargs):
        """Return the representation of the running container."""
        container_manager = self.application.container_manager
        container = yield container_manager.find_container(
            url_id=resource.identifier,
            user_name=self.current_user.name)

        if container is None:
            self.log.warning("Could not find container for id {}".format(
                resource.identifier))
            raise exceptions.NotFound()

        return resource.fill(dict(
            name=container.name,
            image_name=container.image_name,
            mapping_id=container.mapping_id,
        ))

    @gen.coroutine
    @authenticated
    def delete(self, resource, **kwargs):
        """Stop the container."""
        container_manager = self.application.container_manager
        container = yield container_manager.find_container(
            url_id=resource.identifier,
            user_name=self.current_user.name
        )

        if not container:
            self.log.warning("Could not find container for id {}".format(
                             resource.identifier))
            raise exceptions.NotFound()

        try:
            yield self.application.reverse_proxy.unregister(
                container.urlpath
            )
        except Exception:
            # If we can't remove the reverse proxy, we cannot do much more
            # than log the problem and keep going, because we want to stop
            # the container regardless.
            self.log.exception("Could not remove reverse "
                               "proxy for id {}".format(resource.identifier))

        try:
            yield container_manager.stop_and_remove_container(
                container.docker_id)
        except Exception:
            self.log.exception("Could not stop and remove container "
                               "for id {}".format(resource.identifier))

    @gen.coroutine
    @authenticated
    def items(self, items_response, **kwargs):
        """"Return the list of containers we are currently running."""
        container_manager = self.application.container_manager

        accountings = self.application.db.get_accounting_for_user(
            self.current_user)

        running_containers = []

        for accounting in accountings:
            image = yield container_manager.image(accounting.application.image)

            if image is None:
                # The user has access to an application that is no longer
                # available in docker. We just move on.
                continue

            container = yield container_manager.find_container(
                user_name=self.current_user.name,
                mapping_id=accounting.id)

            if container is None:
                continue

            rest_container = Container(identifier=container.url_id)
            rest_container.fill(container)
            running_containers.append(rest_container)

        items_response.set(running_containers)

    ##################
    # Private

    @gen.coroutine
    def _remove_container_noexcept(self, container):
        """Removes container and silences (but logs) all exceptions
        during this circumstance."""

        # Note, can't use a context manager to perform this, because
        # context managers are only allowed to yield once
        container_manager = self.application.container_manager
        try:
            yield container_manager.stop_and_remove_container(
                container.docker_id)
        except Exception:
            self.log.exception(
                "Unable to stop container {} after "
                " failure to obtain a ready "
                "container".format(
                    container.docker_id))

    @gen.coroutine
    def _start_container(self,
                         user_name,
                         app,
                         policy,
                         mapping_id,
                         base_urlpath,
                         environment):
        """Start the container. This method is a helper method that
        works with low level data and helps in issuing the request to the
        data container.

        Parameters
        ----------
        user_name : str
            the username

        app : ABCApplication
            the application to start

        policy : ABCApplicationPolicy
            The startup policy for the application

        environment: Dict
            A dictionary of envvars to pass to the container.

        Returns
        -------
        remoteappmanager.docker.container.Container
        """

        image_name = app.image
        mount_home = policy.allow_home
        volume_spec = (policy.volume_source,
                       policy.volume_target,
                       policy.volume_mode)

        manager = self.application.container_manager
        volumes = {}

        if mount_home:
            home_path = os.environ.get('HOME')
            if home_path:
                volumes[home_path] = {'bind': '/workspace', 'mode': 'rw'}
            else:
                self.log.warning('HOME (%s) is not available for %s',
                                 home_path, user_name)
                pass

        if policy.app_license:
            environment['APP_LICENSE'] = policy.app_license
        if None not in volume_spec:
            volume_source, volume_target, volume_mode = volume_spec
            volumes[volume_source] = {'bind': volume_target,
                                      'mode': volume_mode}

        try:
            f = manager.start_container(user_name,
                                        image_name,
                                        mapping_id,
                                        base_urlpath,
                                        volumes,
                                        environment
                                        )
            container = yield gen.with_timeout(
                timedelta(
                    seconds=self.application.file_config.network_timeout
                ),
                f
            )
        except gen.TimeoutError as e:
            self.log.warning(
                "{user}'s container failed to start in a reasonable time. "
                "giving up".format(user=user_name)
            )
            e.reason = 'timeout'
            raise e
        except Exception as e:
            self.log.exception(
                "Unhandled error starting {user}'s "
                "container: {error}".format(user=user_name, error=e)
            )
            e.reason = 'error'
            raise e

        return container

    def _environment_from_configurables(self, image, resource):
        """Helper routine: extracts the configurables from the
        image, matches them to the appropriate configurables
        data in the representation, and returns the resulting environment
        """
        env = {}

        if resource.configurables is Absent:
            return env

        for img_conf in image.configurables:
            config_dict = resource.configurables.get(img_conf.tag)
            env.update(img_conf.config_dict_to_env(config_dict))

        return env

    @gen.coroutine
    def _wait_for_container_ready(self, container):
        """ Wait until the container is ready to be connected

        Parameters
        ----------
        container: docker.Container
           The container to be connected
        """
        # Note, we use the jupyterhub ORM server, but we don't use it for
        # any database activity.
        # Note: the end / is important. We want to get a 200 from the actual
        # websockify server, not the nginx (which presents the redirection
        # page).
        server_url = "http://{}:{}{}/".format(
            container.ip,
            container.port,
            container.urlpath)

        yield wait_for_http_server_2xx(
            server_url,
            self.application.file_config.network_timeout)


# -*- coding:utf-8 -*-
import tool
import pandas as pd
import d00_param
import imp

imp.reload(d00_param)
import pickle
import numpy as np
from modeling.stepwise import func_sort_col, func_stepwise_1
from modeling.cal_ks_auc_score import cal_ks_test, func_report, gb_add_woe, func_tt_vardescrible
import time
from sklearn.externals import joblib
from binning.best_ks_3 import apply_woe, filter_by_corr
from tool import get_bin
import os

def model(model_ex_corr=True):
    start_time = time.time()
    param_dic = d00_param.param_dic
    project_name = param_dic['project_name']
    path_ = tool.prepare_path(project_name, 'd04_')
    tag = param_dic['tag']
    key = param_dic['key']
    corr_limit = param_dic['corr_limit']
    Xpath_ = tool.prepare_path(project_name, 'd02_')
    info = pd.read_excel(Xpath_ + '_info.xlsx', sheet_name='特征IV汇总')
    new_info = info[[u'特征简称', u'特征中文名', 'IV', u'覆盖率']]
    new_info_dict = new_info.set_index(u'特征简称').to_dict('index')
    vars = info.sort_values('IV', ascending=False)[:3000][u'特征简称'].tolist()
    fin_vars = vars
    # fin_vars = ['register_channel_name']
    fin_vars.extend([key, tag])
    if model_ex_corr and u'是否因超过共线性阀值剔除' in list(info.columns):
        m_data_train = tool.read_file('./data/' + project_name + '_m_data_train.pkl')
        m_data_test = tool.read_file('./data/' + project_name + '_m_data_test.pkl')
        nd_vars_woe = (info[info['是否因超过共线性阀值剔除'] == 0]['特征简称'] + '_woe').tolist()
    elif u'是否因超过共线性阀值剔除' in list(info.columns):
        m_data_train = tool.read_file('./data/' + project_name + '_m_data_train.pkl')
        m_data_test = tool.read_file('./data/' + project_name + '_m_data_test.pkl')
        nd_vars_woe = (info['特征简称'] + '_woe').tolist()
    elif model_ex_corr:
        f = open(Xpath_ + '_vars_woe_bin_dic.pkl', "rb")
        dic_woe = pickle.load(f)
        df_train = tool.read_file('./data/' + project_name + '_df_train.pkl')
        df_test = tool.read_file('./data/' + project_name + '_df_test.pkl')
        print(10 * '*' + 'start apply woe.' + 10 * '*')
        m_data_train = apply_woe(df_train[fin_vars], dic_woe, str_vars, key, tag, path=path_)
        m_data_test = apply_woe(df_test[fin_vars], dic_woe, str_vars, key, tag, path=path_)
        print(10 * '*' + 'finish apply woe.' + 10 * '*')
        nd_vars_woe = filter_by_corr(m_data_train, fin_vars, nec_vars=[], corr_limit=corr_limit)
        for i in nd_vars_woe:
            m_data_train[i] = m_data_train[i].astype(float)
            m_data_test[i] = m_data_test[i].astype(float)
        m_data_train.to_pickle('./data/' + project_name + '_m_data_train_bin.pkl')
        m_data_train = m_data_train[[ii for ii in m_data_train if ii[-4:] != '_bin']]
        m_data_test = m_data_test[[ii for ii in m_data_train if ii[-4:] != '_bin']]
        m_data_train.to_pickle('./data/' + project_name + '_m_data_train.pkl')
        m_data_test.to_pickle('./data/' + project_name + '_m_data_test.pkl')
    print(10 * '*' + 'finish vars check corr')
    sort_vars = func_sort_col(nd_vars_woe, m_data_train, tag)[1]
    train_cols, result, vars_dic, ks_df, error_var, odds, A, B = func_stepwise_1(sort_vars, m_data_train, tag,
                                                                                pmml_btn=False, base_score=base_score,
                                                                                double_score=double_score)
    # 保存模型
    joblib.dump(result, path_ + '_fin.model')
    # 加载模型
    # RF = joblib.load('rf.model')
    # df_gp_uncut, df_gp_cut, df_gp_qcut, ks_df, p, df_rs_score = cal_ks_test(result, m_data_test, tag)
    df_score = func_report(m_data_train, result, tag, base_score=base_score, double_score=double_score, save_path=path_,
                           test=m_data_test, info=None, odds=odds, score_detail_btn=False, data_dic=new_info_dict)
    root_path = path_.replace('d04_' + project_name, '')
    error_file = [ii for ii in os.listdir(root_path) if ii.find('_var_error_value.csv') >= 0]
    if error_file:
        for ii in error_file:
            df = tool.read_file(root_path + ii, header=-1)
            df.columns = ['var_name', 'value', 'flag']
            df.drop_duplicates(inplace=True)
            df.to_csv(root_path + ii, index=False)
        print(10 * '*' + '存在特征未匹配到分箱请查看文件%s' % root_path + ii + 10 * '*')
    df_score = df_score[[key, tag, 'score']]
    df_score['score_bin'] = df_score['score'].apply(lambda x: get_bin(x, score_right_list))
    df_score.to_csv(path_ + '_modeling_score.csv', index=False)
    print(10 * '*' + 'finish logstic modeling')
    print('spend times:', time.time() - start_time)

def model_tune(tune_del_vars=[], tune_nec_vars=[], is_all_necess=False, score_right_list=[]):
    start_time = time.time()
    param_dic = d00_param.param_dic
    project_name = param_dic['project_name']
    path_ = tool.prepare_path(project_name, 'd04_')
    tag = param_dic['tag']
    key = param_dic['key']
    corr_limit = param_dic['corr_limit']
    Xpath_ = tool.prepare_path(project_name, 'd02_')
    info = pd.read_excel(Xpath_ + '_info.xlsx', sheet_name='特征IV汇总')
    new_info = info[[u'特征简称', u'特征中文名', 'IV', u'覆盖率']]
    new_info_dict = new_info.set_index(u'特征简称').to_dict('index')
    vars = info.sort_values('IV', ascending=False)[:3000][u'特征简称'].tolist()
    m_data_train = tool.read_file('./data/' + project_name + '_m_data_train.pkl')
    m_data_test = tool.read_file('./data/' + project_name + '_m_data_test.pkl')
    if is_all_necess:
        fin_vars = tune_nec_vars
        nd_vars_woe = [i + "_woe" for i in fin_vars
                       if i + "_woe" in m_data_train.columns]
    else:
        fin_vars = filter(lambda x: x not in tune_del_vars, vars)
        if model_ex_corr:
            nd_vars_woe = filter_by_corr(m_data_train, fin_vars, nec_vars=tune_nec_vars, corr_limit=corr_limit)
        else:
            nd_vars_woe = [ii + "_woe" for ii in fin_vars if ii + "_woe" in m_data_train.columns]
    for i in nd_vars_woe:
        m_data_train[i] = m_data_train[i].astype(float)
        m_data_test[i] = m_data_test[i].astype(float)
    print(10 * '*' + 'finish vars check corr')
    sort_vars = func_sort_col(nd_vars_woe, m_data_train, tag)[1]
    train_cols, result, vars_dic, ks_df, error_var, odds, A, B = func_stepwise_1(sort_vars, m_data_train, tag,
                                                                                pmml_btn=False, base_score=base_score,
                                                                                double_score=double_score,
                                                                                nec_vars=tune_nec_vars)
    # 保存模型
    joblib.dump(result, path_ + '_fin.model')
    df_score = func_report(m_data_train, result, tag, base_score=base_score, double_score=double_score, save_path=path_,
                           test=m_data_test, info=None, odds=odds, score_detail_btn=False, data_dic=new_info_dict)
    df_score = df_score[[key, tag, 'score']]
    df_score['score_bin'] = df_score['score'].apply(lambda x: get_bin(x, score_right_list))
    df_score.to_csv(path_ + '_modeling_score.csv', index=False)
    print(10 * '*' + 'odds:', odds)
    print(10 * '*' + 'finish logstic modeling')
    print('spend times:', time.time() - start_time)

def read_big_file(file_in):
    data = pd.read_csv(file_in, iterator=True)
    loop = True
    chunkSize = 100000
    chunks = []
    while loop:
        try:
            chunk = data.get_chunk(chunkSize)
            chunks.append(chunk)
        except StopIteration:
            loop = False
            print("Iteration is stopped.")
    df_train = pd.concat(chunks, ignore_index=True)
    return df_train

def output_score(file_in, model_vars, score_right_list=None):
    # encoding = tool.file_encoding(file_in)
    df = tool.read_file(file_in)
    param_dic = d00_param.param_dic
    project_name = param_dic['project_name']
    path_ = tool.prepare_path(project_name, 'd04_')
    tag = param_dic['tag']
    key = param_dic['key']
    fin_model_vars = model_vars
    fin_model_vars.extend([key, tag])
    Xpath_ = tool.prepare_path(project_name, 'd02_')
    f = open(Xpath_ + '_vars_woe_bin_dic.pkl', "rb")
    dic_woe = pickle.load(f)
    f = pd.read_excel(path_ + '_result.xlsx', sheet_name=u'评分卡刻度及公式')
    odds = float(f['odds'][0])
    print(10 * '*' + 'start to apply woe' + 10 * '*')
    m_data_cross = apply_woe(df[fin_model_vars], dic_woe, str_vars, key, tag, path=path_ + '_cross')
    print(10 * '*' + 'finish apply woe' + 10 * '*')
    m_data_train = tool.read_file('./data/' + project_name + '_m_data_train_bin.pkl')
    nd_vars_woe = [i + "_woe" for i in model_vars
                   if i + "_woe" in m_data_cross.columns]
    for i in nd_vars_woe:
        m_data_train[i] = m_data_train[i].astype(float)
        m_data_cross[i] = m_data_cross[i].astype(float)
    nd_vars_bin = [i[:-4] + '_bin' for i in nd_vars_woe]
    # 加载模型
    result = joblib.load(path_ + '_fin.model')
    df_gp_uncut, df_gp_cut, df_gp_qcut, ks_df, p, df_rs_score = cal_ks_test(result, m_data_cross, tag,
                                                                            base_score=base_score,
                                                                            double_score=double_score, odds=odds)
    writer = pd.ExcelWriter(path_ + '_cross_result.xlsx')
    title_Style = writer.book.add_format(tool.title_Style)
    cell_Style = writer.book.add_format(tool.cell_Style)
    # 跨时间集汇总指标
    ws_ks_df = writer.book.add_worksheet('跨时间集汇总指标')
    df_rs_score = df_rs_score[[key, tag, 'score']]
    df_rs_score['score_bin'] = df_rs_score['score'].apply(lambda x: get_bin(x, score_right_list))
    df_score = pd.read_csv(path_ + '_modeling_score.csv')
    model_ks, model_df = gb_add_woe(df_score, tag, 'artificial')
    cross_ks, cross_df = gb_add_woe(df_rs_score, tag, 'artificial')
    model_df = model_df[['bin', 'good', 'bad', 'bad_rate', 'bin_pct']]
    cross_df = cross_df[['bin', 'good', 'bad', 'bad_rate', 'bin_pct']]
    model_df.index = model_df['bin']
    cross_df.index = cross_df['bin']
    df_score_res = pd.concat([model_df, cross_df], axis=1, keys=['TRAIN', 'CROSS'], sort=False)
    df_score_res['PSI'] = (df_score_res[('TRAIN', 'bin_pct')] - df_score_res[('CROSS', 'bin_pct')]) * np.log(
        df_score_res[('TRAIN', 'bin_pct')] / df_score_res[('CROSS', 'bin_pct')])
    psi = sum([ii for ii in df_score_res['PSI'] if not pd.isnull(ii)])
    psi_df = pd.DataFrame({'describe': [psi]})
    psi_df.index = ['PSI']
    all_result = pd.concat([ks_df, psi_df])
    all_result = all_result.reset_index()
    all_result.columns = ['跨时间集', 'describe']
    print(all_result)
    df_score_res = df_score_res.reset_index(drop=True)
    base_bin = list(set(model_df['bin']) | set(cross_df['bin']))
    newbase = pd.DataFrame({'base_bin': base_bin})
    newbase = newbase.sort_values('base_bin')
    model_df = model_df.reset_index(drop=True)
    model_df_new = pd.merge(newbase, model_df, left_on='base_bin', right_on='bin', how='left')
    model_df_new.drop(columns=['bin'], inplace=True)
    cross_df = cross_df.reset_index(drop=True)
    cross_df_new = pd.merge(newbase, cross_df, left_on='base_bin', right_on='bin', how='left')
    cross_df_new.drop(columns=['bin'], inplace=True)
    row_num = all_result.shape[0]
    summary_df = all_result.fillna('')
    ws_ks_df.write_row(0, 0, summary_df.columns, title_Style)
    for ii in range(row_num):
        ws_ks_df.write_row(1 + ii, 0, summary_df.loc[ii,], cell_Style)
    ws_ks_df.set_column(0, summary_df.shape[1], 20)
    # 各分数段分布
    df_score_res.to_excel(writer, sheet_name='各分数段训练_跨时间集分布对比', startcol=0)
    df_rs_score.to_excel(writer, sheet_name='跨时间集打分明细', index=False)
    # 入模特征psi汇总指标及分箱明细对比
    func_tt_vardescrible(m_data_train, m_data_cross, writer, nd_vars_bin, tag)
    # 跨时间样本匹配不成功字段列表明细（如无匹配不成功则不输出文件）
    root_path = path_.replace('d04_' + project_name, '')
    error_file = [ii for ii in os.listdir(root_path) if ii.find('_var_error_value.csv') >= 0]
    if error_file:
        for ii in error_file:
            df = tool.read_file(root_path + ii, header=-1)
            df.columns = ['var_name', 'value', 'flag']
            df.drop_duplicates(inplace=True)
            df.to_csv(root_path + ii, index=False)
        print(10 * '*' + '存在特征未匹配到分箱请查看文件%s' % root_path + ii + 10 * '*')
    writer.save()

def output_score_psi(file_in, model_vars, score_right_list=None):
    df = tool.read_file(file_in)
    param_dic = d00_param.param_dic
    project_name = param_dic['project_name']
    path_ = tool.prepare_path(project_name, 'd04_')
    tag = param_dic['tag']
    key = param_dic['key']
    fin_model_vars = model_vars
    fin_model_vars.extend([key, tag])
    Xpath_ = tool.prepare_path(project_name, 'd02_')
    f = open(Xpath_ + '_vars_woe_bin_dic.pkl', "rb")
    dic_woe = pickle.load(f)
    print(10 * '*' + 'start to apply woe' + 10 * '*')
    m_data_cross = apply_woe(df[fin_model_vars], dic_woe, str_vars, key, tag, path=path_ + '_cross')
    print(10 * '*' + 'finish apply woe' + 10 * '*')
    m_data_train = tool.read_file('./data/' + project_name + '_m_data_train_bin.pkl')
    nd_vars_woe = [i + "_woe" for i in model_vars
                   if i + "_woe" in m_data_cross.columns]
    for i in nd_vars_woe:
        m_data_train[i] = m_data_train[i].astype(float)
        m_data_cross[i] = m_data_cross[i].astype(float)
    nd_vars_bin = [i[:-4] + '_bin' for i in nd_vars_woe]
    writer = pd.ExcelWriter(path_ + '_cross_result_vars_psi.xlsx')
    func_tt_vardescrible(m_data_train, m_data_cross, writer, nd_vars_bin, tag)
    root_path = path_.replace('d04_' + project_name, '')
    error_file = [ii for ii in os.listdir(root_path) if ii.find('_var_error_value.csv') >= 0]
    if error_file:
        for ii in error_file:
            df = tool.read_file(root_path + ii, header=-1)
            df.columns = ['var_name', 'value', 'flag']
            df.drop_duplicates(inplace=True)
            df.to_csv(root_path + ii, index=False)
        print(10 * '*' + '存在特征未匹配到分箱请查看文件%s' % root_path + ii + 10 * '*')
    writer.save()

if __name__ == "__main__":
    data_var_type = pd.read_excel('./data/data_source.xlsx')
    str_vars = set(data_var_type[data_var_type['var_type'] == 'str']['var_name'].tolist())
    param_dic = d00_param.param_dic
    project_name = param_dic['project_name']
    base_score = param_dic['base_score']
    double_score = param_dic['double_score']
    ex_cols = param_dic['ex_cols']
    nece_var = param_dic['nece_var']
    step = param_dic['step']
    is_all_necess = param_dic['is_all_necess']
    bin_ex_corr = param_dic['bin_ex_corr']
    model_ex_corr = param_dic['model_ex_corr']
    model_vars_psi = param_dic['model_vars_psi']
    path_ = tool.prepare_path(project_name, 'd04_')
    # 设定分数右边界list,默认左开右闭
    score_right_list = []
    for i in range(0, 31):
        score_right_list.append(10 * i + 400)
    # 弹出输入窗
    # try:
    #     model_step = int(input("model step:"))
    #     if model_step > 3 or model_step < 1:
    #         print ('input error ,please input int number in[1,2,3]')
    #         os._exit(0)
    # except:
    #     print('input error ,please input int number in[1,2,3]')
    #     os._exit(0)
    # # 首次训练模型
    if step == 1:
        print(10 * "*" + 'run model step %d' % step + 10 * "*")
        model(model_ex_corr=model_ex_corr)
        print(10 * "*" + 'finish model step %d' % step + 10 * "*")
    # # 调试模型
    if step == 2:
        print(10 * "*" + 'run model step %d' % step + 10 * "*")
        model_tune(tune_del_vars=ex_cols, tune_nec_vars=nece_var, is_all_necess=is_all_necess,
                   score_right_list=score_right_list)
        print(10 * "*" + 'finish model step %d' % step + 10 * "*")
    # # 跨时间样本打分
    if step == 3:
        print(10 * "*" + 'run model step %d' % step + 10 * "*")
        # model_vars = pd.read_csv(path_ + '_fin_model_vars.csv')
        # model_vars = model_vars['fin_vars'].tolist()
        f = pd.read_excel(path_ + '_result.xlsx', sheet_name=u'最终入模特征', skiprows=1)
        nan_index = list(f['特征简称']).index((np.nan))
        model_vars_list = [ii for ii in list(f['特征简称'])[:nan_index] if ii != 'intercept']
        file_in = './data/' + param_dic['cross_sample']
        output_score(file_in, model_vars_list, score_right_list)
        print(10 * "*" + 'finish model step %d' % step + 10 * "*")
    # # 跨时间样本变量psi
    if step == 4:
        print(10 * "*" + 'run model step %d' % step + 10 * "*")
        # model_vars_psi=[]
        file_in = './data/' + param_dic['cross_sample']
        output_score_psi(file_in, model_vars_psi)
        print(10 * "*" + 'finish model step %d' % step + 10 * "*")


from src.Environment import *
from src.Parser import *
from src.Interpreter import *
from src.Exceptions import *

class Cli:
    def main(self):
        """
        обеспечивает основной цикл ввода команд интерпретатора
        """
        env = Environment()
        while True:
            try:
                text = input('cli> ')
            except EOFError:
                break
            if not text:
                continue

            try:
                lexer = Lexer(text)
                parser = Parser(lexer)
                result = parser.parse()
                inter = Interpreter()
                result = inter.interpolation_commands_list(result, env)
                result = inter.from_abstract_to_concrete(result)
                inter.run_commands_list(result, env)
            except LexerException:
                print("Syntax error. Repeat input ")
            except ParserException as e:
                print("Parsing error: " + str(e))
            except CommandFileException as e:
                print("File error: " + str(e))
            except CommandExternalException as e:
                print("Command error: " + str(e))


from tqdm import tqdm
import os
import numpy as np
import librosa
from . import preprocess as pps
from .vad_utils import read_label_from_file, prediction_to_vad_label

def getmelFeature(frameData, sampleRate=16000,
                   frame_size: float=0.032, frame_shift: float=0.008,
                   NFFT=512, NMELS=40):
    '''
    Input:
        frameData(shape=(frameSize, frameNum))
    Return: melData(shape=(frameNum, NMELS))
    '''
    fftData = np.fft.rfft(frameData.T)
    periodogram = np.abs(fftData) ** 2 / NFFT
    melFBank = librosa.filters.mel(sr=sampleRate, n_fft=NFFT, n_mels=NMELS)
    melData = np.dot(periodogram, melFBank.T)
    return melData

def getMFCC(melData, n_mfcc=40):
    '''
    Input:
                            shape=(NMELS, frameNum)
    return: mfcc(np.ndarray[shape=(n_mfcc, frameNum)])
    '''
    mfcc = librosa.feature.mfcc(S=librosa.power_to_db(melData), n_mfcc=n_mfcc)
    return mfcc

# since the implementation in `preprocess.py` ocupying too much memory
# We use another way to read training data
def spectralData(waveDirPath: str, frame_size: float=0.032, frame_shift: float=0.008,
                 NFFT=512, NMELS=40, NMFCC=20, labelDirPath=None, featType='mfcc'):
    '''Generate data with spectral feature (and its coresponding label)

    Return:
        spectralData: np.ndarray[shape=(frameNum, NFFT/2 + 1)]
            or mfcc with shape=(N_MFCC=40, frameNum)
        label
    '''
    assert featType == 'mfcc' or featType =='mel', 'No such kind of feat{}'.format(featType)

    tmpsavePath = 'tmpData'
    if (featType == 'mfcc'):
        if os.path.exists(tmpsavePath + '/' + featType +'_' + str(NMFCC) +'_specX.npy'):
            trainX = np.load(tmpsavePath + '/' + featType + '_' + str(NMFCC) +'_specX.npy')
            trainY = np.load(tmpsavePath + '/' + 'specY.npy')
            return trainX, trainY
    else:
        if os.path.exists(tmpsavePath + '/' + featType +'_' + str(NMELS)+ '_specX.npy'):
            trainX = np.load(tmpsavePath + '/' + featType + '_' + str(NMELS) +'_specX.npy')
            trainY = np.load(tmpsavePath + '/' + 'specY.npy')
            return trainX, trainY

    wavfiles = os.listdir(waveDirPath)
    specData = np.zeros((1, NMELS ))

    if labelDirPath != None:
        wavelabel = read_label_from_file(labelDirPath, frame_size, frame_shift)
        labelist = []

    for b, fileName in enumerate(tqdm(wavfiles)):
        waveID = fileName.split('.')[0]
        frameData, sampleRate = pps.enframe(waveDirPath + '/' + fileName, frame_size, frame_shift)
        spectralData = getmelFeature(frameData, sampleRate, frame_size, frame_shift, NFFT, NMELS)
        specData = np.concatenate((specData, spectralData), axis=0)

        '''
        if( b % 600 == 599):
            c = int(b / 600)
            np.save("tmpData/{}_mel_".format(c) + str(NMELS) + "_specX.npy", specData[1:])
            specData = np.zeros((1, NMELS ))
        '''

        if labelDirPath != None:
            frameNum = frameData.shape[1]
            label = wavelabel[waveID]
            labelPad = np.pad(label, (0, np.maximum(frameNum - len(label), 0)))[:frameNum]
            labelist.append(labelPad)

    label = np.concatenate(labelist)
    np.save("tmpData/specY.npy", label)
    '''
    specData = np.zeros((1, NMELS ))
    for c in range(ceil(b/600)):
        arr = np.load("tmpdata/{}_mel_".format(c) + str(nmels) + "_specx.npy", specData[1:])
        specData = np.concatenate((specData, arr), axis=0)

    np.save("tmpdata/mel_" + str(nmels) + "_specx.npy", specData)
    '''
    if (featType == 'mel'):
        return specData[1:], label
    else:
        mfcc = getMFCC(specData[1:].T, n_mfcc=NMFCC)
        np.save("tmpData/mfcc_" + str(NMFCC) + "_specX.npy", mfcc)
        return mfcc, label

if __name__ == '__main__':
   specdata, label = spectralData('../data/dev', labelDirPath='../data/dev_label.txt')
   print(specdata.shape, label.shape)


from gurobipy import *
import networkx as nx
import random as rd

def solveMinCostFlow(G: nx.DiGraph, source: int, sink: int, demand: int) -> dict:
    """
    Solves the minimum cost flow problem.
    :param G: directed graph
    :param source: Source node in G
    :param sink: Sink node in G
    :param demand: Demand value
    :return: Dict of edges and flow values
    """

    minCostFlow = Model('MinCostFlow')

    # Variable
    X = dict()

    for a in G.edges():
        X[a] = minCostFlow.addVar(vtype=GRB.CONTINUOUS, lb=0, ub=G.get_edge_data(*a)['capacity'], name=f'X_{a}')

    # Objective function
    minCostFlow.setObjective(quicksum(X[a]*G.get_edge_data(*a)['cost'] for a in G.edges()), sense=GRB.MINIMIZE)

    # Constraints
    for v in G.nodes():
        if v == source:
            minCostFlow.addConstr(quicksum(X[a] for a in G.out_edges(v)) - quicksum(X[a] for a in G.in_edges(v)),
                              GRB.EQUAL, demand)
        elif v == sink:
            minCostFlow.addConstr(quicksum(X[a] for a in G.out_edges(v)) - quicksum(X[a] for a in G.in_edges(v)),
                              GRB.EQUAL, -demand)
        else:
            minCostFlow.addConstr(quicksum(X[a] for a in G.out_edges(v)) - quicksum(X[a] for a in G.in_edges(v)),
                              GRB.EQUAL, 0)

    # Solve model
    minCostFlow.update()
    minCostFlow.optimize()

    if minCostFlow.status == GRB.OPTIMAL:
        flows = dict()
        for a in G.edges():
            if minCostFlow.getVarByName(f'X_{a}').x > 0:
                flows[a] = minCostFlow.getVarByName(f'X_{a}').x
        return flows

    else:
        return dict()

if __name__ == '__main__':
    # Example: Complete graph on 100 vertices
    G = nx.complete_graph(100).to_directed()
    for a in G.edges():
        G[a[0]][a[1]]['capacity'] = rd.randint(1, 14)
        G[a[0]][a[1]]['cost'] = rd.randint(1, 14)

    flows = solveMinCostFlow(G, 0, 99, 5)

    print('Min cost flow edges: {}'.format(flows))


from controller.SerialController import SerialController
from controller.ViewController import ViewController
from view.MainView import Root

# init controllers and main window
serial_controller = SerialController()
view_controller = ViewController(serial_controller)
window = Root(view_controller)
view_controller.window = window

# set intervals for window
window.interval(1000, view_controller.check_devices)
window.interval(6000, view_controller.update_view)

# enter mainloop
window.mainloop()


from osgeo import ogr
import os
from osgeo import gdal
from osgeo.gdalconst import *
import subprocess
import math
import numpy as np

def read_shp(ds_path, map_path=None):
    ds = ogr.Open(ds_path)
    layer = ds.GetLayer(0)
    f = layer.GetNextFeature()

    count = 0
    lines = []
    while f:
        geom = f.GetGeometryRef()
        l = []
        if geom != None:
            # points = geom.GetPoints()
            points = geom.ExportToJson()
            points = eval(points)['coordinates']
            print(points)
            lines.append(points)
        count += 1
        f = layer.GetNextFeature()
    return lines

# if __name__ == '__main__':
#     read_shp('C:\Users\weiweiduan\Documents\Map_proj_data\CA\CA_Bray_100414_2001_24000_bag\data\Perfect_shp\CA_Bray_waterlines_2001_perfect.shp')


import json
from pathlib import Path
from typing import List, Optional
from zipfile import ZipFile

import liblzfse
import numpy as np
import open3d as o3d
import tqdm
from PIL import Image
from quaternion import as_rotation_matrix, quaternion
from torch.utils.data import Dataset

import sys
sys.path.append('..')

from dataloaders.scannet_200_classes import CLASS_LABELS_200

class MyDataset(Dataset):

    def __init__(self, num, custom_classes: Optional[List[str]] = CLASS_LABELS_200):
        self.image_lis, self.depth_lis, self.world_lis, self.conf_lis = [], [], [], []
        self.num = num
        image = np.load('../data/image000.npy')
        self.image_size = (image.shape[1], image.shape[0])
        if custom_classes:
            self._classes = custom_classes
        else:
            self._classes = CLASS_LABELS_200

        self._id_to_name = {i: x for (i, x) in enumerate(self._classes)}

        for i in range(self.num+1):
            num = '000' + str(self.num)
            num = num[-3:]
            image = np.load('../data/image'+num+'.npy')
            depth = (np.load('../data/depth'+num+'.npy')*0.001).astype(np.uint8)
            world = np.load('../data/world'+num+'.npy')

            self.image_lis.append(image)
            self.depth_lis.append(depth)
            self.world_lis.append(world)
            self.conf_lis.append(np.ones_like(depth))

    def __len__(self):
        return self.num

    def __getitem__(self, idx):
        result = {
            "xyz_position": self.world_lis[idx],
            "rgb": self.image_lis[idx],
            "depth": self.depth_lis[idx],
            "conf": self.conf_lis[idx],
        }

        return result

CUSTOM_LABELS = [
    "kitchen counter",
    "kitchen cabinet",
    "stove",
    "cabinet",
    "bathroom counter",
    "refrigerator",
    "microwave",
    "oven",
    "fireplace",
    "door",
    "sink",
    "furniture",
    "dish rack",
    "dining table",
    "shelf",
    "bar",
    "dishwasher",
    "toaster oven",
    "toaster",
    "mini fridge",
    "soap dish",
    "coffee maker",
    "table",
    "bowl",
    "rack",
    "bulletin board",
    "water cooler",
    "coffee kettle",
    "lamp",
    "plate",
    "window",
    "dustpan",
    "trash bin",
    "ceiling",
    "doorframe",
    "trash can",
    "basket",
    "wall",
    "bottle",
    "broom",
    "bin",
    "paper",
    "storage container",
    "box",
    "tray",
    "whiteboard",
    "decoration",
    "board",
    "cup",
    "windowsill",
    "potted plant",
    "light",
    "machine",
    "fire extinguisher",
    "bag",
    "paper towel roll",
    "chair",
    "book",
    "fire alarm",
    "blinds",
    "crate",
    "tissue box",
    "towel",
    "paper bag",
    "column",
    "fan",
    "object",
    "range hood",
    "plant",
    "structure",
    "poster",
    "mat",
    "water bottle",
    "power outlet",
    "storage bin",
    "radiator",
    "picture",
    "water pitcher",
    "pillar",
    "light switch",
    "bucket",
    "storage organizer",
    "vent",
    "counter",
    "ceiling light",
    "case of water bottles",
    "pipe",
    "scale",
    "recycling bin",
    "clock",
    "sign",
    "folded chair",
    "power strip",
]

dataset = MyDataset(48)

from dataloaders import DeticDenseLabelledDataset

labelled_dataset = DeticDenseLabelledDataset(
    dataset,
    use_extra_classes=False,
    exclude_gt_images=False,
    use_lseg=False,
    subsample_prob=0.01,
    visualize_results=True,
    detic_threshold=0.4,
    visualization_path="detic_labelled_results",
)


from peewee import Model, SqliteDatabase, TextField

class DataStorage:
    def __init__(self, db_path):
        self.db = SqliteDatabase(db_path)

        class BaseModel(Model):
            class Meta:
                database = self.db

        class Record(BaseModel):
            chat_id = TextField()
            name = TextField()
            sheet_id = TextField()
            range = TextField()
            old_value_hash = TextField()

        self.Record = Record

        self.db.connect()
        self.db.create_tables([self.Record])
        self.db.commit()
        self.db.close()

    def find_by_props(self, chat_id, sheet_id, cell_range):
        """
        :return: name of the notifier if found
                 None if not found
        """
        self.db.connect()
        same_record = self.Record.select().where(
            (self.Record.chat_id == chat_id)
            & (self.Record.sheet_id == sheet_id)
            & (self.Record.range == cell_range)
        )

        res = None if len(same_record) == 0 else same_record.get().name
        self.db.close()
        return res

    def find_by_name(self, chat_id, name):
        """
        :return: True or False
        """
        self.db.connect()
        record = self.Record.select().where(
            (self.Record.chat_id == chat_id) & (self.Record.name == name)
        )
        res = len(record) > 0
        self.db.close()
        return res

    def delete_with_name(self, chat_id, name):
        """
        :return: True   if deleted
                 False  if not found
        """
        self.db.connect()
        record = self.Record.select().where(
            (self.Record.chat_id == chat_id) & (self.Record.name == name)
        )
        if len(record) == 0:
            return False
        else:
            record.get().delete_instance()
            self.db.commit()
            self.db.close()
            return True

    def add_record(self, chat_id, name, sheet_id, cell_range, old_hash_value="---"):
        """
        :return: True if successfully set
                 False otherwise
        """
        self.db.connect()
        # print('old_hash:\t', old_hash_value)
        new_record = self.Record.create(
            chat_id=chat_id,
            name=name,
            sheet_id=sheet_id,
            range=cell_range,
            old_value_hash=old_hash_value,
        )
        new_record.save()
        self.db.commit()
        self.db.close()

    @staticmethod
    def transform_to_dict(records):
        return [
            {
                "chat_id": rec.chat_id,
                "name": rec.name,
                "sheet_id": rec.sheet_id,
                "range": rec.range,
                "old_value_hash": rec.old_value_hash,
            }
            for rec in records
        ]

    def records_by_user(self, chat_id):
        self.db.connect()

        records = self.transform_to_dict(
            self.Record.select().where(self.Record.chat_id == chat_id)
        )
        self.db.close()
        return records

    def get_all_records(self):
        self.db.connect()
        all_records = self.transform_to_dict(self.Record.select())
        self.db.close()
        return all_records


# GRADIENT BOOSTING (XGBOOST)

from base_model import BaseModel

import numpy as np
from sklearn import metrics
from xgboost import XGBClassifier

class Model (BaseModel):
	def __init__ (self):
		super().__init__('XGB')

	def train (self, Xt, Xv, Yt, Yv, verbose=False):
		model = XGBClassifier(verbose=verbose)
		model.fit(Xt, Yt)
		if verbose: print('fit model')

		pred = model.predict(Xv)
		acc = metrics.accuracy_score(Yv, pred)
		if verbose: print('Acc: ' + str(acc))

		self.src = model
		self.acc = acc

	def predict (self, X):
		return self.onehot_from_cat(self.src.predict(X))