from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import argparse
import datetime

import backtrader as bt

class St(bt.Strategy):
    params = dict(
        ma=bt.ind.SMA,
        p1=5,
        p2=15,
        limit=0.005,
        limdays=3,
        limdays2=1000,
        hold=10,
        usetarget=False,  # use order_target_size
        switchp1p2=False,  # switch prices of order1 and order2
        oco1oco2=False,  # False - use order1 as oco for order3, else order2
        do_oco=True,  # use oco or not
    )

    def notify_order(self, order):
        print('{}: Order ref: {} / Type {} / Status {}'.format(
            self.data.datetime.date(0),
            order.ref, 'Buy' * order.isbuy() or 'Sell',
            order.getstatusname()))

        if order.status == order.Completed:
            self.holdstart = len(self)

        if not order.alive() and order.ref in self.orefs:
            self.orefs.remove(order.ref)

    def __init__(self):
        ma1, ma2 = self.p.ma(period=self.p.p1), self.p.ma(period=self.p.p2)
        self.cross = bt.ind.CrossOver(ma1, ma2)

        self.orefs = list()

        if self.p.usetarget:
            print('-' * 5, 'Using order_target_size')
            self._dobuy = self.order_target_size
            self._doclose = self.order_target_size
        else:
            self._dobuy = self.buy
            self._doclose = self.close

    def next(self):
        if self.orefs:
            return  # pending orders do nothing

        if not self.position:
            if self.cross > 0.0:  # crossing up

                p1 = self.data.close[0] * (1.0 - self.p.limit)
                p2 = self.data.close[0] * (1.0 - 2 * 2 * self.p.limit)
                p3 = self.data.close[0] * (1.0 - 3 * 3 * self.p.limit)

                valid1 = datetime.timedelta(self.p.limdays)
                valid2 = valid3 = datetime.timedelta(self.p.limdays2)

                if self.p.switchp1p2:
                    p1, p2 = p2, p1
                    valid1, valid2 = valid2, valid1

                print('valid1 is:', valid1)

                kargs = dict(exectype=bt.Order.Limit)
                kargs[('target' * self.p.usetarget) or 'size'] = 1

                o1 = self._dobuy(price=p1, valid=valid1, **kargs)
                print('{}: Oref {} / Buy at {}'.format(
                    self.datetime.date(), o1.ref, p1))

                oco2 = o1 if self.p.do_oco else None
                o2 = self._dobuy(price=p2, valid=valid2, oco=oco2, **kargs)

                print('{}: Oref {} / Buy at {}'.format(
                    self.datetime.date(), o2.ref, p2))

                if self.p.do_oco:
                    oco3 = o1 if not self.p.oco1oco2 else oco2
                else:
                    oco3 = None

                o3 = self._dobuy(price=p3, valid=valid3, oco=oco3, **kargs)

                print('{}: Oref {} / Buy at {}'.format(
                    self.datetime.date(), o3.ref, p3))

                self.orefs = [o1.ref, o2.ref, o3.ref]

        else:  # in the market
            if (len(self) - self.holdstart) >= self.p.hold:
                self._doclose()

def runstrat(args=None):
    args = parse_args(args)

    cerebro = bt.Cerebro()

    # Data feed kwargs
    kwargs = dict()

    # Parse from/to-date
    dtfmt, tmfmt = '%Y-%m-%d', 'T%H:%M:%S'
    for a, d in ((getattr(args, x), x) for x in ['fromdate', 'todate']):
        if a:
            strpfmt = dtfmt + tmfmt * ('T' in a)
            kwargs[d] = datetime.datetime.strptime(a, strpfmt)

    # Data feed
    data0 = bt.feeds.BacktraderCSVData(dataname=args.data0, **kwargs)
    cerebro.adddata(data0)

    # Broker
    cerebro.broker = bt.brokers.BackBroker(**eval('dict(' + args.broker + ')'))

    # Sizer
    cerebro.addsizer(bt.sizers.FixedSize, **eval('dict(' + args.sizer + ')'))

    # Strategy
    cerebro.addstrategy(St, **eval('dict(' + args.strat + ')'))

    # Execute
    cerebro.run(**eval('dict(' + args.cerebro + ')'))

    if args.plot:  # Plot if requested to
        cerebro.plot(**eval('dict(' + args.plot + ')'))

def parse_args(pargs=None):
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description=(
            'Sample Skeleton'
        )
    )

    parser.add_argument('--data0', default='../../datas/2005-2006-day-001.txt',
                        required=False, help='Data to read in')

    # Defaults for dates
    parser.add_argument('--fromdate', required=False, default='',
                        help='Date[time] in YYYY-MM-DD[THH:MM:SS] format')

    parser.add_argument('--todate', required=False, default='',
                        help='Date[time] in YYYY-MM-DD[THH:MM:SS] format')

    parser.add_argument('--cerebro', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--broker', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--sizer', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--strat', required=False, default='',
                        metavar='kwargs', help='kwargs in key=value format')

    parser.add_argument('--plot', required=False, default='',
                        nargs='?', const='{}',
                        metavar='kwargs', help='kwargs in key=value format')

    return parser.parse_args(pargs)

if __name__ == '__main__':
    runstrat()


# This is the inward facing app, that does our process and runs the optimizer
import sys
from PyQt5.QtWidgets import (QApplication, QWidget, QHBoxLayout, QVBoxLayout, QLabel, QPushButton, QInputDialog,
                             QFileDialog, QMainWindow, QDialog, QTabWidget, QRadioButton, QProgressBar, QComboBox,
                             QLineEdit, QGridLayout)
from LookupReader import LookupBook
from GeneralMethods import InputPasser, qtime_to_time
# from PlateReader import HCRReader
from HCRPlateReaderV2 import HCRReader, CustomHCRReaderPopUp
from HTMLReader import HTMLReader
from OptimizerTable import (OptimizerView, PostalizersToPandas, PopUpView, PopUpToPandas, SchedulePopUpView,
                            SchedulePopUpToPandas, RoundTripView, RoundTripsToPandas, SingleRoundTripToPandas)
from ScheduleCompilation import Postalizer
from PostalizerInputTable import PostalizerInputsView, PostalizerInputsToLists
from OptimizedScheduleProcessor import (Process, OptimizedScheduleView, OptimizedSchedulesToPandas,
                                        OneOptScheduleView, OneOptScheduleToPandas)
from PostStaffingOptimization import PostStaffingProcess, StaffingOutputView, ReadInHuasOutput
from PostVehicleOptimization import VehicleSummarizer
from ExcelReader import JDAExcelReader
from StaffingManipulation import StaffingManipulationGUI

class App(QMainWindow):

    def __init__(self):
        super().__init__()
        self.setWindowTitle("Optimizing Mail Model")
        self.setGeometry(100, 100, 1000, 500)

        self.lb = LookupBook()

        self.tabs = QTabWidget()

        self.tab1 = PostalizerPrinterGUI(self.lb)
        self.tabs.addTab(self.tab1, "Home Page")

        self.tab2 = PostCPLEXGUI(self.lb, self.tab1.ip)
        self.tabs.addTab(self.tab2, "Post Scheduler")

        self.tab3 = PostStaffingGUI(self.tab1.ip)
        self.tabs.addTab(self.tab3, "Optimized Vehicles")

        self.tab4 = StaffingManipulationGUI(self.tab1.ip)
        self.tabs.addTab(self.tab4, "Staffing Manipulation")

        self.input_tab = PostalizerInputsGUI()
        self.tabs.addTab(self.input_tab, "Change Inputs")
        self.input_tab.submit_button.clicked[bool].connect(self.set_ip)

        self.setCentralWidget(self.tabs)
        self.show()

    def set_ip(self):

        self.tab1.ip = self.input_tab.ip
        self.tab2.ip = self.input_tab.ip
        self.tab3.ip = self.input_tab.ip
        self.tab4.ip = self.input_tab.ip
        self.tab1.deep_clear()

class PostalizerPrinterGUI(QWidget):

    def __init__(self, lb):
        super().__init__()

        self.ip = InputPasser(source="Optimizer")
        self.hcr_check_list = [self.ip.min_check_time, self.ip.max_check_time, self.ip.max_combined_time]
        self.strict_merge = False

        self.lb = lb

        self.load_plates_button = None
        self.custom_load_plates_button = None
        self.load_html_button = None
        self.load_excel_button = None
        self.print_cplex_input_button = None
        self.print_schedules_button = None
        self.clear_button = None
        self.main_table = None

        self.show_schedules = None
        self.show_trips = None
        self.case = 1

        self.file_names = []
        self.readers = []

        self.site_names = []
        self.postalizers = []
        self.initUI()

    def initUI(self):

        main_layout = QHBoxLayout()
        left_layout = QVBoxLayout()

        self.main_table = OptimizerView()
        self.main_table.doubleClicked.connect(self.pop_up_site)

        self.load_plates_button = QPushButton("Load HCR Plates", self)
        self.custom_load_plates_button = QPushButton("Custom load HCR Plates", self)
        self.load_html_button = QPushButton("Load HTML File", self)
        self.load_excel_button = QPushButton("Load JDA (Excel) File", self)
        self.print_cplex_input_button = QPushButton("Print CPLEX input for selected row", self)
        self.print_schedules_button = QPushButton("Print schedules from selected row", self)
        self.clear_button = QPushButton("Clear All", self)
        self.show_schedules = QRadioButton("Schedules")
        self.show_trips = QRadioButton("Round Trips")

        self.load_plates_button.clicked[bool].connect(self.load_plates)
        self.custom_load_plates_button.clicked[bool].connect(self.custom_load_plates)
        self.load_html_button.clicked[bool].connect(self.load_html_files)
        self.load_excel_button.clicked[bool].connect(self.load_excel)
        self.print_cplex_input_button.clicked[bool].connect(self.print_selected_round_trips)
        self.print_schedules_button.clicked[bool].connect(self.print_selected_schedules)
        self.clear_button.clicked[bool].connect(self.deep_clear)

        self.show_schedules.case = 1
        self.show_trips.case = 2

        self.show_schedules.toggled.connect(self.change_case)
        self.show_trips.toggled.connect(self.change_case)

        self.show_schedules.setChecked(True)

        left_layout.addWidget(self.load_plates_button)
        left_layout.addWidget(self.custom_load_plates_button)
        left_layout.addWidget(self.load_html_button)
        left_layout.addWidget(self.load_excel_button)
        left_layout.addWidget(self.print_cplex_input_button)
        left_layout.addWidget(self.print_schedules_button)
        left_layout.addWidget(self.clear_button)
        left_layout.addStretch(1)

        pop_up_label = QLabel("Double click to show: ")
        left_layout.addWidget(pop_up_label)
        left_layout.addWidget(self.show_schedules)
        left_layout.addWidget(self.show_trips)
        left_layout.addStretch(5)

        main_layout.addLayout(left_layout)
        main_layout.addWidget(self.main_table)

        self.setLayout(main_layout)

    def change_case(self):

        radiobutton = self.sender()

        if radiobutton.isChecked():
            self.case = radiobutton.case

    def load_plates(self):

        self.hcr_check_list = [self.ip.min_check_time, self.ip.max_check_time, self.ip.max_combined_time]

        files, _ = QFileDialog.getOpenFileNames(self, "Select HCR Plate(s)",
                                                "", "PDF Files (*.pdf);;PDF Files (*.pdf)")

        if files:

            progress_bar = ProgressBarWindow("Progress Reading Plates", len(files))
            QApplication.processEvents()

            for x, pdf in enumerate(files):

                file_name = pdf.split('/')[-1][:5]
                if file_name in self.file_names:
                    print("duplicate not added")
                    continue

                self.file_names.append(file_name)

                progress_bar.update(x)

                reader = HCRReader(pdf, self.lb, "Optimizer", self.hcr_check_list, self.strict_merge)
                if reader.is_readable:
                    self.readers.append(reader)
                else:
                    self.readers.append(None)
                    print("error reading " + pdf[:5])
        else:
            print("No files selected")
            return

        self.update_table()
        self.postalize_schedules()

    def custom_load_plates(self):

        self.hcr_check_list = [self.ip.min_check_time, self.ip.max_check_time, self.ip.max_combined_time]

        files, _ = QFileDialog.getOpenFileNames(self, "Select HCR Plate(s)",
                                                "", "PDF Files (*.pdf);;PDF Files (*.pdf)")

        if not files:
            print("Nothing selected")
            return

        for x, pdf in enumerate(files):

            file_name = pdf.split('/')[-1][:5]
            if file_name in self.file_names:
                print("duplicate not added")
                continue

            self.file_names.append(file_name)

            test_reader = HCRReader(pdf, self.lb, "Test", self.hcr_check_list, self.strict_merge)
            possible_names, possible_addresses = test_reader.get_potential_pdcs_and_addresses()

            # given_name, given_address = CustomHCRReaderPopUp(possible_names, possible_addresses, self.lb).exec_()
            given_name, given_address = CustomHCRReaderPopUp(possible_names, possible_addresses, self.lb).exec_()

            reader = HCRReader(pdf, self.lb, "Optimizer", self.hcr_check_list, self.strict_merge, custom_pdc_name=given_name,
                               custom_pdc_address=given_address)

            if reader.is_readable:
                self.readers.append(reader)
            else:
                self.readers.append(None)
                print("error reading " + pdf[:5])

        self.update_table()
        self.postalize_schedules()

    def load_html_files(self):

        files, _ = QFileDialog.getOpenFileNames(self, "Select HTML File(s)",
                                                "", "HTML Files (*.htm);;HTML Files (*.htm)")

        if files:
            progress_bar = ProgressBarWindow("Progress Reading HTMLs", len(files))
            QApplication.processEvents()

            for x, html in enumerate(files):

                file_name = str(html)
                if file_name in self.file_names:
                    print("duplicate not added")
                    continue

                self.file_names.append(file_name)

                progress_bar.update(x)

                reader = HTMLReader(html, self.lb)
                self.file_names.append(str(html))
                self.readers.append(reader)
        else:
            print("No files selected")
            return

        self.update_table()
        self.postalize_schedules()

    def load_excel(self):

        file, _ = QFileDialog.getOpenFileName(self, "Select JDA to SPS file", "",
                                              "Excel Files (*.xlsx);;Excel Files (*.xlsx)")

        if file:

            file_name = file.split("/")[-1]
            if file_name not in self.file_names:
                self.file_names.append(file_name)
                reader = JDAExcelReader(file, self.lb)
                self.readers.append(reader)
            else:
                print("Duplicate not added")
        else:
            print("No file selected")
            return

        self.update_table()
        self.postalize_schedules()

    def update_table(self):

        self.clear_all()
        self.site_names = []

        for reader in self.readers:
            if reader:
                site_name = reader.pvs_name
            else:
                site_name = "Unread plates"

            if site_name not in self.site_names:
                self.site_names.append(site_name)
                self.postalizers.append(Postalizer(self.ip, site_name))

            postalizer = next(x for x in self.postalizers if x.pvs_name == site_name)
            postalizer.add_reader(reader)

        compiled_df = PostalizersToPandas(self.postalizers).df
        self.main_table.set_model(compiled_df)
        self.main_table.update()

    def postalize_schedules(self):

        for postalizer in self.postalizers:
            postalizer.postalize_schedules()

        self.update_table()

    def clear_all(self):

        del self.file_names
        del self.postalizers
        del self.site_names

        self.site_names = []
        self.file_names = []
        self.postalizers = []
        compiled_df = PostalizersToPandas(self.postalizers).df
        self.main_table.set_model(compiled_df)
        self.main_table.update()

    def deep_clear(self):

        del self.readers
        self.readers = []

        self.clear_all()

    def pop_up_site(self):

        index = self.main_table.selectedIndexes()[0]
        index = self.main_table.model().mapToSource(index)

        site_name = self.site_names[index.row()]
        postalizer = self.postalizers[index.row()]

        if postalizer:
            round_trips = postalizer.round_trips

        if self.case == 1:
            PlatePopUpGUI(site_name, postalizer).exec_()
        elif self.case == 2:
            if round_trips:
                to_print = RoundTripsGUI(site_name, round_trips).exec_()
                if to_print:
                    self.print_selected_round_trips()
        else:
            print("WTF!")

    def print_selected_round_trips(self):

        if len(self.postalizers) == 0:
            return

        index = self.main_table.selectedIndexes()
        if not index:
            print("nothing selected")
            return

        row = self.main_table.model().mapToSource(index[0]).row()

        postalizer_to_print = self.postalizers[row]
        # print(postalizer_to_print.short_name)

        if not postalizer_to_print:
            print("nothing selected!")
            return

        postalizer_to_print.print_cplex_scheduler_input()

    def print_selected_schedules(self):

        if len(self.postalizers) == 0:
            return

        index = self.main_table.selectedIndexes()
        if not index:
            print("nothing selected")
            return

        row = self.main_table.model().mapToSource(index[0]).row()

        postalizer_to_print = self.postalizers[row]
        # print(postalizer_to_print.short_name)

        if not postalizer_to_print:
            print("nothing selected!")
            return

        postalizer_to_print.print_schedules()

class PlatePopUpGUI(QDialog):

    def __init__(self, site_name, postalizer, parent=None):
        super(PlatePopUpGUI, self).__init__(parent)

        self.site_name = site_name
        self.setWindowTitle(self.site_name)

        if site_name == "Unread plates" or not postalizer:
            self.no_reader()
            return

        self.schedules = postalizer.schedules
        self.close_button = None
        self.central_table = None
        self.initUI()

    def initUI(self):

        vbox = QVBoxLayout()
        button_box = QHBoxLayout()

        self.setGeometry(200, 200, 900, 500)
        self.central_table = PopUpView()
        self.central_table.doubleClicked.connect(self.pop_up_schedule)
        self.central_table.set_model(PopUpToPandas(self.schedules).df)

        self.close_button = QPushButton("Close", self)
        self.close_button.clicked[bool].connect(self.accept)

        vbox.addWidget(self.central_table)
        vbox.addWidget(self.close_button)

        self.setLayout(vbox)
        self.show()

    def no_reader(self):

        vbox = QVBoxLayout()
        vbox.addWidget(QLabel("Postalizer not read in correctly"))
        close_button = QPushButton("Close", self)
        close_button.clicked[bool].connect(self.accept)
        vbox.addWidget(close_button)
        self.setLayout(vbox)

    def pop_up_schedule(self):

        index = self.central_table.selectedIndexes()[0]
        index = self.central_table.model().mapToSource(index)
        schedule = self.schedules[index.row()]
        SchedulePopUpGUI(schedule).exec_()

class RoundTripsGUI(QDialog):

    def __init__(self, site_name, round_trips, parent=None):
        super(RoundTripsGUI, self).__init__(parent)

        self.trips = round_trips
        self.site_name = site_name
        self.setWindowTitle(self.site_name)
        self.setGeometry(150, 100, 950, 550)
        self.close_button = None
        self.round_trip_table = None
        self.vehicle_filter_combo = None
        self.select_all_button = None
        self.all_selected = True
        self.to_print = False
        self.trip_filter_combo = None
        self.print_selection_button = None

        self.all_trips = None
        self.real_trips = None
        self.spotter_trips = None

        self.vehicle_filter = ""
        self.trip_filter = ""

        self.initUI()

    def initUI(self):

        vbox = QVBoxLayout()
        top_hbox = QHBoxLayout()
        bottom_bar = QHBoxLayout()

        self.vehicle_filter_combo = QComboBox()
        self.vehicle_filter_combo.addItem("All Vehicles")
        self.vehicle_filter_combo.addItem("11-Ton")
        self.vehicle_filter_combo.addItem("Single")
        self.vehicle_filter_combo.setCurrentIndex(0)
        self.vehicle_filter_combo.currentIndexChanged.connect(self.vehicle_filter_change)

        self.trip_filter_combo = QComboBox()
        self.trip_filter_combo.addItem("All Trip Types")
        self.trip_filter_combo.addItem("Real trip")
        self.trip_filter_combo.addItem("Spotter trip")
        self.trip_filter_combo.addItem("Standby trip")
        self.trip_filter_combo.addItem("Lunch trip")
        self.trip_filter_combo.currentIndexChanged.connect(self.trip_filter_change)

        self.select_all_button = QPushButton("Select/Unselect All", self)
        self.select_all_button.clicked[bool].connect(self.select_all_click)

        self.all_trips = QRadioButton("All Trips")
        self.real_trips = QRadioButton("Real Trips")
        self.spotter_trips = QRadioButton("Spotter Trips")
        self.all_trips.setChecked(True)

        self.round_trip_table = RoundTripView()
        self.set_table()

        self.print_selection_button = QPushButton("Print CPlex Input for Selection", self)
        self.print_selection_button.clicked[bool].connect(self.print_selection)

        self.close_button = QPushButton("Close", self)
        self.close_button.clicked[bool].connect(self.accept)

        filter_label = QLabel("Filter Trips:")
        top_hbox.addWidget(filter_label)
        top_hbox.addStretch(2)
        top_hbox.addWidget(self.vehicle_filter_combo)
        top_hbox.addWidget(self.trip_filter_combo)
        top_hbox.addStretch(1)
        top_hbox.addWidget(self.select_all_button)

        vbox.addLayout(top_hbox)
        vbox.addWidget(self.round_trip_table)
        bottom_bar.addWidget(self.print_selection_button)
        bottom_bar.addWidget(self.close_button)
        vbox.addLayout(bottom_bar)

        self.setLayout(vbox)
        self.show()

    def vehicle_filter_change(self):

        self.vehicle_filter = self.vehicle_filter_combo.currentText()
        if self.vehicle_filter == "All Vehicles":
            self.vehicle_filter = ""

        self.filter_table()

    def trip_filter_change(self):

        self.trip_filter = self.trip_filter_combo.currentText()
        if self.trip_filter == "All Trip Types":
            self.trip_filter = ""

        self.filter_table()

    def filter_table(self):

        if self.round_trip_table.sorter:
            self.round_trip_table.sorter.setFilterByColumn(self.vehicle_filter, 8)
            self.round_trip_table.sorter.setFilterByColumn(self.trip_filter, 9)
        else:
            return

    def select_all_click(self):

        if self.all_selected:
            self.unselect_all_round_trips()
            self.all_selected = False
        else:
            self.select_all_round_trips()
            self.all_selected = True

    def select_round_trips(self, item):

        index = self.round_trip_table.model().mapToSource(item)
        if index.column() != 19:
            return

        row = index.row()
        col = index.column()

        new_val = self.round_trip_table.model().sourceModel()._data[row][col]

        if new_val:
            self.trips[row].is_selected = False
        else:
            self.trips[row].is_selected = True

        self.set_table()

    def select_all_round_trips(self):

        for trip in self.trips:
            trip.is_selected = True

        self.set_table()

    def unselect_all_round_trips(self):

        for trip in self.trips:
            trip.is_selected = False

        self.set_table()

    def set_table(self):

        self.round_trip_table.set_model(RoundTripsToPandas(self.trips).df)
        self.round_trip_table.setSortingEnabled(True)
        self.round_trip_table.doubleClicked.connect(self.pop_up_round_trip)
        self.round_trip_table.model().dataChanged.connect(self.select_round_trips)

    def print_selection(self):

        self.to_print = True
        self.accept()

    def pop_up_round_trip(self):

        try:
            index = self.round_trip_table.selectedIndexes()[0]
            index = self.round_trip_table.model().mapToSource(index)
            round_trip = self.trips[index.row()]
            RoundTripPopUpGUI(round_trip).exec_()
        except:
            return

    def exec_(self):

        super().exec_()
        self.close()

        return self.to_print

class SchedulePopUpGUI(QDialog):

    def __init__(self, schedule, parent=None):
        super(SchedulePopUpGUI, self).__init__(parent)

        self.schedule = schedule
        self.setWindowTitle(schedule.schedule_name)
        self.setGeometry(250, 100, 800, 550)
        self.close_button = None
        self.left_table = None
        self.right_table = None
        self.round_trip_table = None
        self.initUI()

    def initUI(self):

        vbox = QVBoxLayout()
        left_vbox = QVBoxLayout()
        right_vbox = QVBoxLayout()
        round_trip_vbox = QVBoxLayout()
        hbox = QHBoxLayout()

        left_label = QLabel("Schedule as read:")
        self.left_table = SchedulePopUpView()
        self.left_table.set_model(SchedulePopUpToPandas(self.schedule, 1).df)

        if self.schedule.is_postalized:
            right_label = "Postalized Schedule: Compliant"
        else:
            right_label = "Postalized Schedule: Non-Compliant"

        right_label = QLabel(right_label)
        self.right_table = SchedulePopUpView()
        self.right_table.set_model(SchedulePopUpToPandas(self.schedule, 2).df)

        round_trip_label = QLabel("Round Trips:")
        self.round_trip_table = RoundTripView()
        self.round_trip_table.set_model(RoundTripsToPandas(self.schedule.round_trips).df)
        self.round_trip_table.setSortingEnabled(False)
        self.round_trip_table.doubleClicked.connect(self.pop_up_round_trip)

        self.close_button = QPushButton("Close", self)
        self.close_button.clicked[bool].connect(self.accept)

        left_vbox.addWidget(left_label)
        left_vbox.addWidget(self.left_table)
        right_vbox.addWidget(right_label)
        right_vbox.addWidget(self.right_table)
        round_trip_vbox.addWidget(round_trip_label)
        round_trip_vbox.addWidget(self.round_trip_table)

        hbox.addLayout(left_vbox)
        hbox.addLayout(right_vbox)
        # hbox.addLayout(round_trip_vbox)
        vbox.addLayout(hbox)
        vbox.addLayout(round_trip_vbox)
        vbox.addWidget(self.close_button)

        self.setLayout(vbox)
        self.show()

    def pop_up_round_trip(self):

        index = self.round_trip_table.selectedIndexes()[0]
        index = self.round_trip_table.model().mapToSource(index)
        round_trip = self.schedule.round_trips[index.row()]
        RoundTripPopUpGUI(round_trip).exec_()

class RoundTripPopUpGUI(QDialog):

    def __init__(self, round_trip, parent=None):
        super(RoundTripPopUpGUI, self).__init__(parent)

        self.setWindowTitle(round_trip.schedule_name + " " + str(round_trip.trip_num))
        self.setGeometry(300, 300, 500, 300)
        self.round_trip = round_trip
        self.central_table = SchedulePopUpView()
        self.central_table.set_model(SingleRoundTripToPandas(round_trip).df)

        self.close_button = QPushButton("Close", self)
        self.close_button.clicked[bool].connect(self.accept)

        vbox = QVBoxLayout()
        label = QLabel("Round Trip: ")
        vbox.addWidget(label)
        vbox.addWidget(self.central_table)
        vbox.addWidget(self.close_button)
        self.setLayout(vbox)
        self.show()

class PostalizerInputsGUI(QWidget):

    def __init__(self):
        super().__init__()

        self.inputs_table = None
        self.ip = None

        # trip inputs
        self.pvs_to_pdc = None
        self.pvs_time = None
        self.pdc_time = None

        # lunch inputs
        self.lunch_duration = None
        self.hours_wo_lunch = None
        self.allow_non_postal = None

        # other requirements
        self.max_duration = None
        self.max_mileage = None
        self.tour_one = None
        self.nd_pm = None
        self.nd_am = None

        # u_part combination sensitivity
        self.min_check_minutes = None
        self.max_check_minutes = None
        self.max_combined_minutes = None
        self.use_check_codes = None

        self.submit_button = None
        self.restore_defaults_button = None
        self.initUI()

    def initUI(self):

        hbox = QHBoxLayout()
        button_hbox = QHBoxLayout()
        left_vbox = QVBoxLayout()

        self.restore_defaults_button = QPushButton("Restore Defaults", self)
        self.submit_button = QPushButton("Submit Changes", self)

        self.restore_defaults_button.clicked[bool].connect(self.restore_defaults)
        self.submit_button.clicked[bool].connect(self.save_changes)

        button_hbox.addWidget(self.restore_defaults_button)
        button_hbox.addWidget(self.submit_button)

        self.inputs_table = PostalizerInputsView()
        inputs, headers = PostalizerInputsToLists().get_lists()
        self.inputs_table.set_model(inputs, headers)
        left_vbox.addWidget(self.inputs_table)
        left_vbox.addLayout(button_hbox)

        hbox.addLayout(left_vbox, 1)
        self.setLayout(hbox)

    def restore_defaults(self):

        inputs, headers = PostalizerInputsToLists().get_lists()
        self.inputs_table.set_model(inputs, headers)
        self.inputs_table.update()

    def save_changes(self):

        data = self.inputs_table.model()._data
        pvs_time = data[1][1]
        pvs_to_pdc = data[2][1]
        pdc_time = data[3][1]
        lunch_minutes = data[5][1]
        hours_wo_lunch = data[6][1]
        allow_non_postal = data[7][1]
        wash_up_time = data[8][1]
        max_working_time = data[11][1]
        tour_one = qtime_to_time(data[12][1])
        nd_pm = qtime_to_time(data[13][1])
        nd_am = qtime_to_time(data[14][1])
        min_check_time = data[16][1]
        max_check_time = data[17][1]
        combined_max_time = data[18][1]
        new_ip = InputPasser(source="Optimizer", pvs_to_pdc=pvs_to_pdc, pvs_time=pvs_time, pdc_time=pdc_time,
                             lunch_duration=lunch_minutes, hours_wo_lunch=hours_wo_lunch,
                             allow_non_postal=allow_non_postal, max_duration=max_working_time,tour_one_time=tour_one,
                             nd_pm=nd_pm, nd_am=nd_am, min_check_time=min_check_time, max_check_time=max_check_time,
                             max_combined_time=combined_max_time, wash_up_time=wash_up_time)

        self.ip = new_ip

class ProgressBarWindow(QWidget):

    def __init__(self, title, length):
        super().__init__()
        self.setWindowTitle(title)
        vbox = QVBoxLayout()
        self.length = length
        self.pbar = QProgressBar(self)
        self.pbar.setMaximum(length)
        self.pbar.setGeometry(10, 10, 200, 50)
        vbox.addWidget(self.pbar)
        self.label = QLabel("Reading: x/" + str(length))
        vbox.addWidget(self.label)
        self.setLayout(vbox)
        self.setGeometry(300, 300, 300, 50)
        self.show()

    def update(self, value):

        self.pbar.setValue(value)
        self.label.setText("Reading Plate: " + str(value) + "/" + str(self.length))

class PostCPLEXGUI(QWidget):

    def __init__(self, lb, ip):
        super().__init__()

        self.process_button = None
        self.print_button = None
        self.processor = None
        self.schedules = None
        self.main_table = None
        self.lb = lb
        self.ip = ip

        self.initUI()

    def initUI(self):

        left_vbox = QVBoxLayout()
        hbox = QHBoxLayout()

        self.process_button = QPushButton("Process a Workbook", self)
        self.process_button.clicked[bool].connect(self.process_a_book)

        self.print_button = QPushButton("Print to Excel", self)
        self.print_button.clicked[bool].connect(self.print_processor)

        left_vbox.addWidget(self.process_button)
        left_vbox.addWidget(self.print_button)
        left_vbox.addStretch(1)

        self.main_table = OptimizedScheduleView()
        self.main_table.doubleClicked.connect(self.pop_up_a_schedule)

        hbox.addLayout(left_vbox)
        hbox.addWidget(self.main_table)

        self.setLayout(hbox)

    def process_a_book(self):

        file, _ = QFileDialog.getOpenFileName(self, "Select Post-Cplex File", "",
                                              "Excel Files (*.xlsx);;Excel Files (*.xlsx)")

        if file:
            self.processor = Process(file, self.lb, self.ip)
            self.schedules = self.processor.schedules
            self.update_table()

    def update_table(self):

        if not self.schedules:
            return

        else:
            self.main_table.set_model(OptimizedSchedulesToPandas(self.schedules).df)

    def pop_up_a_schedule(self):

        index = self.main_table.selectedIndexes()[0]
        index = self.main_table.model().mapToSource(index)

        schedule = self.schedules[index.row()]

        OptSchedulePopUp(schedule).exec_()

    def print_processor(self):

        if not self.processor:
            return

        self.processor.print_all()

class OptSchedulePopUp(QDialog):

    def __init__(self, schedule, parent=None):

        super(OptSchedulePopUp, self).__init__(parent)

        self.setWindowTitle(schedule.schedule_name)
        self.setGeometry(300, 200, 800, 400)

        vbox = QVBoxLayout()
        table_hbox = QHBoxLayout()

        self.left_table = OneOptScheduleView()
        self.left_table.set_model(OneOptScheduleToPandas(schedule, 2).df)

        self.right_table = OneOptScheduleView()
        self.right_table.set_model(OneOptScheduleToPandas(schedule, 1).df)

        self.close_button = QPushButton("Close", self)
        self.close_button.clicked[bool].connect(self.accept)

        table_hbox.addWidget(self.left_table)
        table_hbox.addWidget(self.right_table)

        vbox.addLayout(table_hbox)
        vbox.addWidget(self.close_button)

        self.setLayout(vbox)
        self.show()

class PostStaffingGUI(QWidget):

    def __init__(self, ip):
        super().__init__()

        self.ip = ip
        # self.process_button = None
        self.process_vehicles_button = None
        self.processor = None
        self.main_table = None

        self.initUI()

    def initUI(self):

        outer_layout = QHBoxLayout()
        left_vbox = QVBoxLayout()

        # self.process_button = QPushButton("Select Hua's output/Post-Scheduling File", self)
        # self.process_button.clicked[bool].connect(self.process_staffing)

        self.process_vehicles_button = QPushButton("Select 11-Ton and Single Solutions", self)
        self.process_vehicles_button.clicked[bool].connect(self.process_vehicles)

        self.main_table = StaffingOutputView()

        # left_vbox.addWidget(self.process_button)
        left_vbox.addWidget(self.process_vehicles_button)
        left_vbox.addStretch(1)
        outer_layout.addLayout(left_vbox)
        outer_layout.addWidget(self.main_table)

        self.setLayout(outer_layout)

    # def process_staffing(self):
    #
    #     hua_file, _ = QFileDialog.getOpenFileName(self, "Select Hua's File", "",
    #                                               "Excel Files (*.xls*);;Excel Files (*.xls*)")
    #
    #     file, _ = QFileDialog.getOpenFileName(self, "Select Staffing Input Data File", "",
    #                                           "Excel Files (*.xlsx);;Excel Files (*.xlsx)")
    #
    #     if not (hua_file and file):
    #         return
    #
    #     hua_processor = ReadInHuasOutput(hua_file, file)
    #     self.processor = PostStaffingProcess(file, self.ip)

    def process_vehicles(self):

        site_name, _ = QInputDialog.getText(self, "Input Site Name", "Site Name:", QLineEdit.Normal, "")

        if not site_name:
            site_name = "None Entered"

        eleven_ton_file, _ = QFileDialog.getOpenFileName(self, "Select Eleven-Ton File", "",
                                                         "Excel Files (*.xlsx);;Excel Files (*.xlsx)")

        single_file, _ = QFileDialog.getOpenFileName(self, "Select Single File", "",
                                                     "Excel Files (*.xlsx);;Excel Files (*.xlsx)")

        summaries = VehicleSummarizer(eleven_ton_file, single_file, site_name)

class CustomHCRReader(QDialog):

    def __init__(self, possible_names, possible_addresses, lb, parent=None):
        super(CustomHCRReader, self).__init__(parent)

        self.setWindowTitle("Select P&DC For Plate")

        self.read_names = possible_names
        self.read_addresses = possible_addresses

        self.lb = lb
        self.sorted = sorted(lb.hcr_pdcs)

        self.found_on_plate_selection = None
        self.found_in_lb_selection = None

        self.accept_button = None
        self.update_lookup_book_button = None

        self.line_edits = []
        for x in range(0, 8):
            self.line_edits.append(QLineEdit())

        self.labels = ["HCR P&DC Name", "PVS P&DC Name", "PVS Name", "Postal Facility Name", "Alt. PVS Name",
                       "Alt. P&DC Name", "Short Name", "Address"]

        self.initUI()

    def initUI(self):

        outer_layout = QVBoxLayout()
        top_grid = QGridLayout()
        bottom_grid = QGridLayout()

        self.found_on_plate_selection = QComboBox()
        self.found_on_plate_selection.addItem("None")
        for name in self.read_names:
            self.found_on_plate_selection.addItem(name)

        self.found_on_plate_selection.activated.connect(self.update_from_plate)

        self.found_in_lb_selection = QComboBox()
        self.found_in_lb_selection.addItem("None")
        for name in self.sorted:
            self.found_in_lb_selection.addItem(name)

        self.found_in_lb_selection.activated.connect(self.update_from_lb)

        top_grid.addWidget(QLabel("Found on Plate: "), 0, 0)
        top_grid.addWidget(QLabel("Found in Lookups: "), 1, 0)

        top_grid.addWidget(self.found_on_plate_selection, 0, 1)
        top_grid.addWidget(self.found_in_lb_selection, 1, 1)

        outer_layout.addWidget(QLabel("Select P&DC by Name: "))
        outer_layout.addLayout(top_grid)
        outer_layout.addStretch(1)

        for x in range(0, 8):
            bottom_grid.addWidget(QLabel(self.labels[x]), x, 0)
            bottom_grid.addWidget(self.line_edits[x], x, 1)

        outer_layout.addWidget(QLabel("Edit Information: "))
        outer_layout.addLayout(bottom_grid)
        outer_layout.addStretch(1)

        self.accept_button = QPushButton("Set Information", self)
        self.accept_button.clicked[bool].connect(self.accept)

        self.update_lookup_book_button = QPushButton("Update Lookups", self)
        self.update_lookup_book_button.clicked[bool].connect(self.update_lookups)

        outer_layout.addWidget(self.accept_button)
        outer_layout.addWidget(self.update_lookup_book_button)
        self.setLayout(outer_layout)

    def update_from_plate(self):

        name = self.found_on_plate_selection.currentText()

        if name in self.sorted:
            index = self.sorted.index(name)
        elif name in [x[:-4] for x in self.sorted]:
            index = [x[:-4] for x in self.sorted].index(name)
        else:
            index = -1

        self.found_in_lb_selection.setCurrentIndex(index + 1)

    def update_from_lb(self):

        self.found_on_plate_selection.setCurrentIndex(0)

    def update_lookups(self):

        pass

    def exec(self):

        super().exec_()
        self.close()

        return None

if __name__ == "__main__":
    app = QApplication(sys.argv)
    ex = App()
    sys.exit(app.exec_())


import paddle
import paddle.nn as nn
import numpy as np

class PSNRLoss(nn.Layer):

    def __init__(self, loss_weight=1.0, reduction='mean'):
        super(PSNRLoss, self).__init__()
        assert reduction == 'mean'
        self.loss_weight = loss_weight
        self.scale = 10 / np.log(10)

    def forward(self, pred, target):
        assert len(pred.shape) == 4

        return self.loss_weight * self.scale * paddle.log(((pred - target) ** 2).mean(axis=(1, 2, 3)) + 1e-8).mean()

class ShuiyinLoss(nn.Layer):

    def __init__(self,
                 mode='l1', ):
        super(ShuiyinLoss, self).__init__()
        assert mode in ['l1', 'psnr', ]
        if mode == 'l1':
            self.main_loss = nn.L1Loss()
        if mode == 'psnr':
            self.main_loss = PSNRLoss()

    def forward(self, pred, gt):
        loss = self.main_loss(pred, gt)
        return loss


import math
import scipy.stats as stats
import pandas as pd
import numpy as np
import backtrader as bt
import datetime
import psycopg2
import json
import os

class TemplateStrategy(bt.Strategy):

    params = {
        'trade_freq': None,
        'indicator': None,
    }

    def __init__(self):
        self.dataclose = self.datas[0].close
        self.order = None
        self.buyprice = None

        self.indicator = self.params.indicator()
        self.days = 0

    def log(self, txt, dt=None):
        dt = dt or self.datas[0].datetime.date(0)
        print('%s, %s' % (dt.isoformat(), txt))

    def notify_order(self, order):
        if order.status in [order.Submitted, order.Accepted]:
            return

        if order.status in [order.Completed]:
            if order.isbuy():
                self.log(
                    'BUY EXECUTED, PRICE: %.2f, Cost: %.2f, Comm %.2f' % (
                        order.executed.price,
                        order.executed.value,
                        order.executed.comm,
                    )
                )
            else:
                self.log(
                    'SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (
                        order.executed.price,
                        order.executed.value,
                        order.executed.comm,
                    )
                )
            self.bar_executed = len(self)

        self.order = None

    def notify_trade(self, trade):
        if not trade.isclosed:
            return

        self.log(
            'OPERATION PROFIT, GROSS %.2f, NET %.2f' % (
                trade.pnl,
                trade.pnlcomm,
            )
        )

    def next(self):
        self.days += 1

        if self.order:
            return

        if self.days > self.params.trade_freq - 1:
            self.days = 0
            if not self.position:
                if self.indicator > 0:
                    self.log('BUY CREATE, %.2f' % self.dataclose[0])
                    self.order = self.buy()
            elif self.indicator < 0:
                self.log('SELL CREATE, %.2f' % self.dataclose[0])
                self.order = self.sell()

class SimpleMovingAverageIndicator(bt.Indicator):
    lines = ('sma_trend',)
    params = {
        'long_ma': 30,
        'short_ma': 10,
    }

    def __init__(self):
        self.addminperiod(max(self.params.long_ma, self.params.short_ma))
        self.long_period = max(self.params.long_ma, self.params.short_ma)
        self.short_period = min(self.params.long_ma, self.params.short_ma)

    def next(self):
        sma_long = np.mean(self.data.get(size = self.long_period))
        sma_short = np.mean(self.data.get(size = self.short_period))
        self.lines.sma_trend[0] = sma_short - sma_long

if __name__ == '__main__':
    # Connect db
    conn = psycopg2.connect('dbname=securities user=postgres')
    cur = conn.cursor()

    # Query and format data
    cur.execute('SELECT * FROM prices WHERE symbol = \'SPY\' ORDER BY datetime ASC')
    data = pd.DataFrame(cur.fetchall(), columns = ['symbol', 'date', 'open', 'high', 'low', 'close'])
    data.loc[:, 'date'] = pd.to_datetime(data['date'])
    data = data.set_index('date')

    # Create backtest engine and add data/strategies
    cerebro = bt.Cerebro()
    cerebro_data = bt.feeds.PandasData(dataname = data, datetime = -1)
    cerebro.adddata(cerebro_data)
    cerebro.addstrategy(SimpleMovingAverageStrategy, trade_freq = 10, indicator = SimpleMovingAverageIndicator)
    cerebro.addsizer(bt.sizers.FixedSize, stake=10)

    # Run backtest and evaluate strategies
    print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    cerebro.run()
    print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())

    # Close database connections
    cur.close()
    conn.close()


import os
import nmap

# Define the subnet to scan
subnet = "192.168.0.0/24"

# Use nmap to scan the subnet for NFS shares
scanner = nmap.PortScanner()
scanner.scan(subnet, arguments="-p 2049")

# Loop through the hosts with open NFS ports
for host in scanner.all_hosts():
  # Check if the host has an open NFS port
  if scanner[host].has_tcp(2049):
    # Print the hostname and IP address of the NFS server
    print(f"Found NFS server: {host} ({scanner[host].hostname()})")

    # Get the list of NFS exports on the server
    exports = scanner[host]["tcp"][2049]["product"].split(",")

    # Loop through the exports
    for export in exports:
      # Mount the NFS export
      print(f"Mounting {export}")
      os.system(f"mount -t nfs {host}:{export} /mnt/{host}_{export.replace('/', '_')}")


#-*- coding:utf-8 -*-
"""joinlines.py work with cp1252 encoding files

"""
import sys,re,codecs

def join_plines(x):
 """ x is array of strings of length n
  if n = 0 or n=1 return x
  else:
   remove line-endings from x[0],...,x[n-2] (all but last line
  Then join the list of adjusted strings (so return a string)
 """
 n = len(x)
 if n == 0:
  return ''
 if n == 1:
  return x[0]
 y = []
 for i,s in enumerate(x):
  if i != (n-1):
   s1 = s.rstrip('\r\n')
  else:
   s1 = s
  y.append(s1)
 #
 return ''.join(y)

def joinlines(lines):
 """ A generator for array of new lines.
 """
 ans = [] # array of joined lines
 jlines = []

 for iline,line in enumerate(lines):
  #if iline > 10:break
  if line.startswith(('<H>','<p>')):
   if jlines != []:
    newline = join_plines(jlines)
    #print('yield:',iline+1,newline)
    yield newline
   jlines = [line]
  elif jlines == []:
   print("joinlines error at line number",iline+1)
   print('line=',line)
   exit(1)
  else:
   jlines.append(line)
 # last one
 if jlines != []:
  newline = join_plines(jlines)
  yield newline

if __name__=="__main__":
 filein = sys.argv[1] #  xxx.txt (path to digitization of xxx)
 fileout = sys.argv[2] # extended ascii

 with codecs.open(filein,"r","cp1252") as f:
  lines = [x for x in f]  # retain line-ending characters (\r\n)
 print(len(lines),"read from",filein)
 newlines = list(joinlines(lines))
 with codecs.open(fileout,"w","cp1252") as f:
  for line in newlines:
   f.write(line)
 print(len(newlines),"written to",fileout)


import json
import logging
import requests
from requests import exceptions

from django.core import serializers
from rest_framework.views import APIView
from django.http import JsonResponse
from agent.models import Agent
from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema

from logkit.utils.api import get_request_args

LOG = logging.getLogger('default')

# Create your views here.
class AgentView(APIView):
    """docstring"""

    token = openapi.Parameter("token", openapi.IN_HEADER, description="token",
                                   type=openapi.IN_HEADER)
    @swagger_auto_schema(operation_description="get agent info",
                         responses={404: 'args not found'},
                         manual_parameters=[token])
    def get(self, request):
        """get agent info"""
        response = {'status_code': 200, "message": "获取信息成功"}
        try:
            agents = Agent.objects.all()
            response['data'] = json.loads(serializers.serialize("json", agents))
        except Exception as e:
            response['status_code'] = "500"
            response['message'] = "获取agent信息失败. {}".format(e)

        return JsonResponse(response)

from django.conf import settings

class AgentHealthCheckView(APIView):
    """docsting"""

    token = openapi.Parameter("token", openapi.IN_HEADER, description="token",
                                   type=openapi.IN_HEADER)
    @swagger_auto_schema(operation_description="agent health check",
                         responses={404: 'args not found'},
                         manual_parameters=[token])
    def get(self, request):
        """docstring"""
        response = {'status_code': 200, "message": "操作成功"}

        try:
            agents_info = settings.OPSAGENT['agents_info']
            for agent_info in agents_info:
                agent_url = agent_info + "/agent/health_check"
                agent_ip = agent_url.split('/')[2]
                try:
                    r = requests.get(agent_url, timeout=5)
                    if r.status_code == 200:
                        Agent.objects.filter(agent_ip=agent_ip).update(agent_status='health')
                    else:
                        Agent.objects.filter(agent_ip=agent_ip).update(agent_status='abnormal')
                except exceptions.Timeout as e:
                    Agent.objects.filter(agent_ip=agent_ip).update(agent_status='abnormal')
                    LOG.error('agent status abnormal! error info: %s', e)
                LOG.info("check agent: %s", agent_ip)
        except Exception as e:
            response['status_code'] = "500"
            response['message'] = "检查agent信息失败. {}".format(e)
            LOG.error('check agent status failed! error info: %s', e)

        return JsonResponse(response)


# 119. Pascal's Triangle II

class Solution:
    def getRow(self, rowIndex: int) -> list[int]:
        if rowIndex == 0:
            return [1]
        elif rowIndex == 1:
            print([1, 1])
            return [1, 1]
        result = [1, 1]
        for i in range(2, rowIndex + 1):
            temp = [1]
            for x in range(1, len(result)):
                temp.append(result[x] + result[x - 1])

            temp.append(1)
            result = temp
        print(result)
        return result
###########################################################
"""         if rowIndex == 0:
            return [1]
        tmp = [0] + self.getRow(rowIndex-1) + [0]
        print(tmp)
        res = []
        for i in range(len(tmp)-1):
            res.append(tmp[i] + tmp[i+1])
        print('res ', res)
        return res """

if __name__ == "__main__":
    x = Solution()
    # t = 3
    z = 3
    x.getRow(z)


"""
-----find best historical casts-----

Written by: Breck Owens and Annie Wong
When: xx/12/2006
Converted to python by: Edward Small
When: 30/10/2019

Find ln_max_casts unique historical points that are most strongly correlated with the float profile

Rewritten in December 2006 to only use latitude, longitude, data, and water depth as arguments
and to return index of the station list

N.B. Change to code on the xx/06/2013: add age_large when computing correlation_large
- C Cabanes

N.B Change during conversion to python on the 31/10/2019: Potential Vorticity, Correlation, and
ellipse are calculated multiple times, so I moved them into their own function.
These functions can be vectorised using the numpy library (numpy.vectorize(function))
to use the functions on arrays.
- Edward Small

For information on how to use this file, check the README at either:

https://github.com/ArgoDMQC/matlab_owc
https://gitlab.noc.soton.ac.uk/edsmall/bodc-dmqc-python
"""

import math
import numpy as np
from .potential_vorticity.potential_vorticity import potential_vorticity
from .spatial_correlation.spatial_correlation import spatial_correlation
from .find_ellipse.find_ellipse import find_ellipse

# pylint: disable=fixme
# TODO: ARGODEV-155
# Refactor this code to take objects and dictionaries instead of a ludicrous amount of arguments
# In fact, this function still requires a serious refactor, because it is doing far too much
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
def find_besthist(
        grid_lat, grid_long, grid_dates, grid_z_value,
        lat, long, date, z_value,
        latitude_large, latitude_small, longitude_large, longitude_small,
        phi_large, phi_small, age_large, age_small, map_pv_use, max_casts
):
    """
    Finds ln_max_casts number of unique historical data points that are most strongly correlated
    with the float profile being processed
    :param grid_lat: array of latitudes of historical data
    :param grid_long: array of longitudes of historical data
    :param grid_dates: array of ages of the historical data
    :param grid_z_value: array of depths of the historical data
    :param lat: latitude of the float profile
    :param long: longitude of the float profile
    :param date: age of the float profile
    :param z_value: depth of the float profile
    :param latitude_large: latitude of large ellipse
    :param latitude_small: latitude of small ellipse
    :param longitude_large: longitude of large ellipse
    :param longitude_small: longitude of small ellipse
    :param phi_large: cross-isobath scale for large ellipse
    :param phi_small: cross-isobath scale for small ellipse
    :param age_large: age of data in large ellipse
    :param age_small: age of data in small ellipse
    :param map_pv_use: flag for whether to use potential vorticity (see load_configuration.py)
    :param max_casts: maximum number of data points wanted
    :return: indices of historical data to use
    """

    # make sure arrays are 1 dimensional
    grid_lat = grid_lat.flatten()
    grid_long = grid_long.flatten()
    grid_z_value = grid_z_value.flatten()
    grid_dates = grid_dates.flatten()

    # set up potential vorticity
    potential_vorticity_vec = np.vectorize(potential_vorticity)
    pv_float = 0
    pv_hist = 0

    # if we are using potential vorticity, calculate it
    if map_pv_use == 1:
        pv_float = potential_vorticity(lat, z_value)
        pv_hist = potential_vorticity_vec(grid_lat, grid_z_value)

    # calculate ellipse
    find_ellipse_vec = np.vectorize(find_ellipse)
    ellipse = find_ellipse_vec(grid_long, long, longitude_large, grid_lat, lat, latitude_large,
                               phi_large, pv_hist, pv_float)

    # find points that lie within the ellipse
    hist_long = []
    hist_lat = []
    hist_dates = []
    hist_z_value = []
    index_ellipse = np.argwhere(ellipse < 1)
    for i in index_ellipse:
        hist_long.append(grid_long[i])
        hist_lat.append(grid_lat[i])
        hist_dates.append(grid_dates[i])
        hist_z_value.append(grid_z_value[i])

    index = index_ellipse
    # check to see if too many data points were found
    if index.__len__() > max_casts:

        # uses pseudo-random numbers so the same random numbers are selected for each run
        np.random.seed(index_ellipse.__len__())

        # pick max_casts/3 random points
        rand_matrix = np.random.rand(math.ceil(max_casts / 3))
        index_rand = np.round(rand_matrix * (index_ellipse.__len__() - 1))

        # make sure the points are all unique
        index_rand = np.unique(index_rand)
        # sort them into ascending order
        index_rand.sort()

        # create an array containing the indices of the remaining reference data
        # (index_ellipse array without index_rand array)
        # Then create arrays containing the remaining reference lat, long, age, and z_value
        index_remain = []
        remain_hist_lat = []
        remain_hist_long = []
        remain_hist_z_value = []
        remain_hist_dates = []

        for i in index_ellipse.flatten():
            if i not in index_rand:
                index_remain.append(i)

        for i in index_remain:
            remain_hist_lat = np.append(remain_hist_lat, grid_lat[int(i)])
            remain_hist_long = np.append(remain_hist_long, grid_long[int(i)])
            remain_hist_z_value = np.append(remain_hist_z_value, grid_z_value[int(i)])
            remain_hist_dates = np.append(remain_hist_dates, grid_dates[int(i)])

        # sort remaining points by large spatial correlations
        # if using potential vorticity, calculate it for remaining data
        if map_pv_use == 1:
            pv_hist = potential_vorticity_vec(remain_hist_lat, remain_hist_z_value)

        # calculate the large spatial correlation for each point
        spatial_correlation_vec = np.vectorize(spatial_correlation)
        correlation_large = spatial_correlation_vec(remain_hist_long, long, longitude_large,
                                                    remain_hist_lat, lat, latitude_large,
                                                    remain_hist_dates, date, age_large,
                                                    pv_hist, pv_float, phi_large)

        # combine the large spatial correlation with their indices in a 2d matrix
        correlation_large_combined = np.stack((index_remain, correlation_large)).transpose()
        # sort the matrix in ascending order of correlation
        correlation_large_combined_sorted = correlation_large_combined[
            correlation_large_combined[:, 1].argsort()
        ]

        # work out how many large spatial data points needed
        lsegment2 = 2 * math.ceil(max_casts / 3) - index_rand.__len__()

        # select the best large spatial data points
        index_large_spatial = []
        for i in range(0, lsegment2):
            index_large_spatial = np.append(
                index_large_spatial, correlation_large_combined_sorted[i][0]
            )

        # create array to hold selected indices from random and spatial selection
        rand_large_index = np.concatenate((index_rand, index_large_spatial))
        # sort into ascending order
        rand_large_index.sort()

        # remove the currently selected indices again
        index_remain = []
        remain_hist_lat = []
        remain_hist_long = []
        remain_hist_z_value = []
        remain_hist_dates = []
        for i in index_ellipse.flatten():
            if i not in rand_large_index:
                index_remain.append(i)

        for i in index_remain:
            remain_hist_lat = np.append(remain_hist_lat, grid_lat[int(i)])
            remain_hist_long = np.append(remain_hist_long, grid_long[int(i)])
            remain_hist_z_value = np.append(remain_hist_z_value, grid_z_value[int(i)])
            remain_hist_dates = np.append(remain_hist_dates, grid_dates[int(i)])

        # sort the remaining points by short spatial and temporal correlations
        # if using potential vorticity, calculate it
        if map_pv_use == 1:
            pv_hist = potential_vorticity_vec(
                remain_hist_lat, remain_hist_z_value)

        # calculate the small spatial correlation for each point
        correlation_small = spatial_correlation_vec(remain_hist_long,
                                                    long, longitude_small,
                                                    remain_hist_lat,
                                                    lat, latitude_small,
                                                    remain_hist_dates,
                                                    date, age_small,
                                                    pv_hist, pv_float, phi_small)

        # combine the small spatial correlation with their indices in a 2d matrix
        correlation_small_combined = np.stack((index_remain, correlation_small)).transpose()
        # sort the matrix in ascending order of correlation
        correlation_small_combined_sorted = correlation_small_combined[
            correlation_small_combined[:, 1].argsort()
        ]

        # select the amount we need for short spatial/temporal correlation
        leftover = max_casts - index_rand.__len__() - lsegment2

        index_small_spatial = []
        for i in range(0, leftover):
            index_small_spatial = np.append(
                index_small_spatial, correlation_small_combined_sorted[i][0]
            )

        # put together the best random data, large spatial data, and small spatial data
        index = np.concatenate((index_rand, index_large_spatial, index_small_spatial))
        if index.__len__() != np.unique(index).__len__():
            print("WARNING: not all points are unique")
            rand_large = np.concatenate((index_rand, index_large_spatial))
            rand_small = np.concatenate((index_rand, index_small_spatial))
            large_small = np.concatenate((index_large_spatial, index_small_spatial))
            print("unique points: ", np.unique(index).__len__())
            print("unique rand large: ", rand_large.__len__(),
                  " : ", np.unique(rand_large).__len__())
            print("unique rand small: ", rand_small.__len__(),
                  " : ", np.unique(rand_small).__len__())
            print("unique large small: ", large_small.__len__(),
                  " : ", np.unique(large_small).__len__())

    # ensure that the index is integers
    index = index.flatten()
    index = index.astype(int)

    return index


import sys
import pandas as pd
import numpy as np
import sqlite3
from sqlalchemy import create_engine
import nltk
import re
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
import pickle
nltk.download(['punkt', 'wordnet'])
nltk.download('stopwords')

def load_data(database_filepath):
    """Load data from the database filepath
    & store them in X and y variable
    args:
        database_filepath: path to the database file
    returns:
        X, y & the list of categories
    """
    engine = create_engine('sqlite:///{}'.format(database_filepath))
    df = pd.read_sql_table('DisasterResponse', con=engine)
    X = df['message']
    Y = df.drop(['id', 'message', 'original', 'genre'], axis=1)
    category_names = Y.columns

    return X,Y,category_names

def tokenize(text):
    """
    args:
        Input value
    returns:
        returns the tokenize version of the input
    """

    url_re = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'

    # Detect and replace urls
    detected_urls = re.findall(url_re, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    # tokenize sentences
    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    # save cleaned tokens
    clean_tokens = [lemmatizer.lemmatize(tok).lower().strip() for tok in tokens]

    # remove stopwords
    STOPWORDS = list(set(stopwords.words('english')))
    clean_tokens = [token for token in clean_tokens if token not in STOPWORDS]

    return clean_tokens

def build_model():
    """
      build NLP pipeline - count words, tf-idf, multiple output classifier,
      grid search the best parameters
    Args:
        None
    Returns:
        cross validated classifier object
    """
    pipeline = Pipeline([
        ('vec', CountVectorizer(tokenizer=tokenize)),
        ('tfidf', TfidfTransformer()),
        ('clf', MultiOutputClassifier(RandomForestClassifier(n_estimators = 100, n_jobs = -1)))
    ])

    parameters = {
    'clf__estimator__n_estimators':[50,100]
    }

    cv = GridSearchCV(pipeline, param_grid=parameters, n_jobs= -1)

    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    """
        Evaluate the model performances, in terms of f1-score, precison and recall
    Args:
        model: the model to be evaluated
        X_test: X_test dataframe
        Y_test: Y_test dataframe
        category_names: category names list defined in load data
    Returns:
        perfomances (DataFrame)
    """
    Y_pred = model.predict(X_test)

    # build classification report on every column
    performances = []
    for i in range(len(category_names)):
        performances.append([f1_score(Y_test.iloc[:, i].values, Y_pred[:, i], average='micro'),
                             precision_score(Y_test.iloc[:, i].values, Y_pred[:, i], average='micro'),
                             recall_score(Y_test.iloc[:, i].values, Y_pred[:, i], average='micro')])
    # build dataframe
    performances = pd.DataFrame(performances, columns=['f1 score', 'precision', 'recall'],index = Y_test.columns)

    return performances

def save_model(model, model_filepath):
    """
        Save model to pickle
    """
    pickle.dump(model, open('model_filepath', 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


# O(n) time | O(1) space
def sort012(arr):
    i = 0
    left = 0
    right = len(arr) - 1

    while i <= right:

        if arr[i] == 1:
            i += 1
        elif arr[i] == 0:
            arr[i], arr[left] = arr[left], arr[i]
            left += 1
            i += 1
        else:
            arr[i], arr[right] = arr[right], arr[i]
            right -= 1

# Driver
arr = [2, 0, 1]
sort012(arr)
print(arr)


from .imports import *
from collections import defaultdict

class LinearDecoder(nn.Module):
    def __init__(self, encoder, output_size, dropout_p=0.1, weights=None):
        super(LinearDecoder, self).__init__()
        self.encoder = encoder
        self.dropout = nn.Dropout(dropout_p)
        self.hidden_to_tag = nn.Linear(ENCODER_OUTPUT_DIM, output_size)
        self.activation = nn.LogSoftmax(dim=2)
        self.loss_f = nn.NLLLoss(reduction="none", weight=weights)

    def forward(self, input_seq, output_seq, mask):
        """
        Returns predictions and loss
        """
        encoded_states = self.dropout(self.encoder(input_seq)[0][2])
        predictions = self.activation(self.hidden_to_tag(encoded_states))
        losses = list()
        for i, (pred, active, out) in enumerate(zip(predictions, mask, output_seq)):
            loss = self.loss_f(pred, out)
            masked_loss = loss * active
            losses.append(torch.sum(masked_loss))
        losses = torch.stack(losses)
        total_loss = torch.sum(losses)

        return predictions, total_loss

def get_encoder(itos):
    emb_sz=400
    lin_ftrs = [50]
    layers = [emb_sz*3] + lin_ftrs + [2]

    drops = np.array([0.4, 0.1])

    model = get_rnn_classifier(bptt=70,
                               max_seq=70*20,
                               vocab_sz=len(itos),
                               emb_sz=400,
                               n_hid=1150,
                               n_layers=3,
                               pad_token=1,
                               layers=layers,
                               drops=drops,
                               qrnn=False
                               )
    enc = model[0]
    enc.eval()
    return enc

def load_tag_model(path, device, itos):
    encoder = get_encoder(itos)
    model = LinearDecoder(encoder, DECODER_OUTPUT_DIM)
    state_dict = torch.load(path, map_location=device)
    model.load_state_dict(state_dict)
    model.eval()
    return model

def predict_binary(classifier):
    preds = classifier.get_preds(ordered=True, ds_type=DatasetType.Test)
    return [l.argmax().item() for l in preds[0]]

def predict_labels(model, data, device, thresh=0.0):
    flatten = lambda x: [item for sublist in x for item in sublist]
    model.eval()
    total_loss = 0

    all_labels = list()
    predicted_labels = list()
    predicted_proba = list()

    with torch.no_grad():
        for inputs, outputs, masks in data:
            inputs, outputs, masks = inputs.to(device), outputs.to(device), masks.to(device)
            predictions, loss = model(inputs, outputs, masks)
            total_loss += loss.item()

            outputs_ = outputs.cpu().detach().numpy()
            masks_ = masks.cpu().detach().numpy()
            predictions_ = predictions.cpu().detach().numpy()

            for output_, mask_, pred_ in zip(outputs_, masks_, predictions_):
                predicted = defaultdict(list)
                true_labels = output_[mask_.astype(bool)]
                preds = np.argmax(pred_, axis=1)[mask_.astype(bool)]
                preds_probas = np.exp(np.max(pred_, axis=1)[mask_.astype(bool)])
                thresholded = list()
                for i, (l, p) in enumerate(zip(preds, preds_probas)):
                    if p > thresh:
                        thresholded.append(l)
                        predicted[l].append((i, p))
                    else:
                        thresholded.append(out_stoi["O"])
                if len(predicted[out_stoi["A"]]) < len(predicted[out_stoi["P"]]):
                    less_probable = list(sorted(predicted[out_stoi["P"]], key=lambda x: x[1]))
                    for i, _ in less_probable[:-len(predicted[out_stoi["A"]])]:
                        thresholded[i] = out_stoi["O"]
                predicted_labels.append(thresholded)
                all_labels.extend(true_labels)
                predicted_proba.append(preds_probas)

    return total_loss, predicted_labels, predicted_proba