import sys
import os
import re
import platform
import zlib

from PyQt5.QtWidgets import QApplication, QMainWindow, QTableView, QFileDialog, QTableWidgetItem
from PyQt5.uic import loadUi
from PyQt5.QtCore import Qt, QObject, QRunnable, pyqtSignal, pyqtSlot, QThreadPool, QUrl
from PyQt5.QtGui import QColor, QDesktopServices

class WorkerData():
    """"""

    def __init__(self):
        self.row_id = None
        self.text = None
        self.background_color = None

class WorkerSignals(QObject):
    '''
    Defines the signals available from a running worker thread.

    progress
        int progress complete,from 0-100
    '''

    main_progression_signal = pyqtSignal(int, str, tuple) # row_index, crc32_from_file, color)
    current_file_progression = pyqtSignal(int) # pourcentage actuel

class Worker(QRunnable):
    '''
    Worker thread

    Inherits from QRunnable to handler worker thread setup, signals
    and wrap-up.
    '''

    def __init__(self, files_list):
        super(Worker, self).__init__()
        self.signals = WorkerSignals()
        self.files_list = files_list

    @pyqtSlot()
    def run(self):
        """"""

        for row_index, file in enumerate(self.files_list):
            crc32_from_name = file.crc32_from_name
            #crc32_from_file = file.get_crc_from_file()

            # region ----- Calcul du CRC -----
            # https://www.matteomattei.com/how-to-calculate-the-crc32-of-a-file-in-python/
            current_block = 0
            blocksize = 4096
            old_percentage = 0

            # TODO: if os.path.isfile(file.filepath):

            with open(file.filepath, 'rb') as f:
                crcvalue = 0
                data = f.read(blocksize)
                file_byte_size = os.path.getsize(file.filepath)
                self.signals.current_file_progression.emit(0)

                while len(data) > 0:
                    crcvalue = zlib.crc32(data, crcvalue)
                    data = f.read(blocksize)

                    # region ----- Progression of the current file -----
                    current_block += blocksize
                    percentage = int(current_block * (100 / file_byte_size))

                    # Percentage increase only if different (avoid too much refresh
                    if percentage != old_percentage:
                        self.signals.current_file_progression.emit(percentage)

                    old_percentage = percentage
                    # endregion

            crc32_from_file = format(crcvalue & 0xFFFFFFFF, '08x').upper()  # Exemple: a509ae4b
            # endregion

            if crc32_from_name:
                if crc32_from_file == crc32_from_name:
                    color = (0, 255, 0)
                else:
                    color = (255, 0, 0)
            else:
                color = (255, 255, 255)

            self.signals.main_progression_signal.emit(row_index, crc32_from_file, color)

class TableWidgetColumns:
    """"""

    name = 0
    size = 1
    folder = 2
    saved_crc = 3
    current_crc = 4

class MainWindow(QMainWindow):
    def __init__(self, folderpath):
        super().__init__()

        self.app_name = "CRC32 GUI"
        self.app_version = "0.3.3"

        self.init_ui()
        self.setup_events()

        self.threadpool = QThreadPool()

        self.root_folderpath = ""

        includes = self.allowed_extensions.text()
        self.includes = tuple(includes.replace(" ", "").split(","))
        self.files_list = []

        # Si on passe un dossier en argument
        if folderpath:
            self.root_folderpath = folderpath
            self.recursive_file_list()
            self.update_table()

    def init_ui(self):
        """"""

        loadUi(os.path.join(os.path.join(os.path.dirname(__file__)), 'gui.ui'), self)
        self.setWindowTitle("{} - {}".format(self.app_name, self.app_version))

        self.setAcceptDrops(True)

    def setup_events(self):
        """"""

        self.pushButton.clicked.connect(self.event_on_launch_button_clicked)
        self.pushButton_3.clicked.connect(self.event_on_select_folder_button_clicked)

        #
        self.pushButton_4.clicked.connect(self.on_open_folder_button_click)

    def dragEnterEvent(self, event):

        folderpath = QUrl(event.mimeData().text()).toLocalFile()

        self.root_folderpath = folderpath
        self.recursive_file_list()
        self.update_table()

    def on_open_folder_button_click(self):
        """"""

        current_row_index = self.tableWidget.currentRow()
        if(current_row_index != -1):
            print(current_row_index)
            folder_path = self.files_list[current_row_index].folder_path
            QDesktopServices.openUrl(QUrl.fromLocalFile(folder_path))

    def event_on_select_folder_button_clicked(self):
        """"""

        folder_path = QFileDialog.getExistingDirectory(self, "Choisir un dossier pour l'analyse")
        if folder_path and os.path.exists(folder_path):
            self.root_folderpath = folder_path

            if self.checkBox.checkState() != Qt.Checked:
                self.files_list.clear()

            # Et on liste les fichiers -> voir vue personalisée ?
            self.recursive_file_list()
            self.update_table()

    def update_table(self):
        """"""

        self.tableWidget.clearContents()
        self.tableWidget.setRowCount(len(self.files_list))

        for row_index, file in enumerate(self.files_list):
            self.tableWidget.setItem(row_index, TableWidgetColumns.name, QTableWidgetItem(file.filename))
            self.tableWidget.setItem(row_index, TableWidgetColumns.size, QTableWidgetItem(file.size))
            self.tableWidget.setItem(row_index, TableWidgetColumns.folder, QTableWidgetItem(file.folder_path))
            self.tableWidget.setItem(row_index, TableWidgetColumns.saved_crc, QTableWidgetItem(file.crc32_from_name))

        self.tableWidget.resizeColumnsToContents()

        self.label.setText("{} éléments".format(len(self.files_list)))

    def recursive_file_list(self):
        """"""

        print("Recursive folder:", self.root_folderpath)

        for root, directories, filenames in os.walk(self.root_folderpath):
            for filename in human_sort(filenames):
                file_path = os.path.join(root, filename)

                print(file_path)

                if filename.endswith(self.includes) and file_path not in self.files_list:
                    file_object = File(file_path)

                    # On affiche tout les fichier si coché
                    if self.checkBox_2.checkState() == Qt.Checked:
                        if file_object.crc32_from_name:
                            self.files_list.append(file_object)
                    else:
                        self.files_list.append(file_object)

    def event_on_launch_button_clicked(self):
        """"""

        if self.root_folderpath and os.path.exists(self.root_folderpath):
            # Init progressbar current value
            self.progressbar.setValue(self.progressbar.minimum())

            # Create thread
            worker = Worker(self.files_list)
            worker.signals.main_progression_signal.connect(self.when_file_is_checked)
            worker.signals.current_file_progression.connect(self.current_file_progression)

            # Execute
            self.threadpool.start(worker)

    def when_file_is_checked(self, row_index, crc32_from_file, color):
        """"""

        self.update_row_color(row_index, crc32_from_file, color)
        self.update_progress_bar(row_index)

    def current_file_progression(self, percentage):
        """"""

        self.progressbar_2.setValue(int(percentage))

    def update_row_color(self, row_index, crc32_from_file, color, row_id=TableWidgetColumns.current_crc):
        """"""

        item = QTableWidgetItem(crc32_from_file)
        item.setBackground(QColor(color[0], color[1], color[2]))
        self.tableWidget.setItem(row_index, row_id, item)

    def update_progress_bar(self, row_index):
        """"""

        percentage = (row_index + 1) * (100 / len(self.files_list))
        self.progressbar.setValue(int(percentage))

class File():
    def __init__(self, filepath):
        self.filepath = filepath
        self.folder_path = os.path.dirname(self.filepath)
        self.filename = os.path.basename(self.filepath)
        self.size = self.get_size()
        self.crc32_from_name = self.get_crc_from_name()
        self.crc32_from_data = None

    def __repr__(self):
        return self.filepath

    def __str__(self):
        return self.filepath

    def exists(self):
        """"""

        return os.path.exists(self.filepath)

    def get_crc_from_name(self):
        """"""

        # Regex trouvé dans AnimeCheck
        split_regex = re.split('([a-f0-9]{8})', self.filename, flags=re.IGNORECASE)
        return None if len(split_regex) < 2 else str(split_regex[-2]).upper()

    def get_size(self):
        """"""

        size_in_bytes = os.path.getsize(self.filepath)

        if size_in_bytes > 1024 * 1024 * 1024:
            size = int(size_in_bytes / (1024 * 1024 * 1024))
            label = "Go"
        elif size_in_bytes > 1024 * 1024:
            size = int(size_in_bytes / (1024 * 1024))
            label = "Mo"
        elif size_in_bytes > 1024:
            size = int(size_in_bytes / 1024)
            label = "Ko"
        else:
            size = size_in_bytes
            label = "o"

        return "{} {}".format(size, label)

    def get_crc_from_file(self):
        """"""

        # https://www.matteomattei.com/how-to-calculate-the-crc32-of-a-file-in-python/

        blocksize = 4096
        with open(self.filepath, 'rb') as f:
            data = f.read(blocksize)
            crcvalue = 0
            while len(data) > 0:
                crcvalue = zlib.crc32(data, crcvalue)
                data = f.read(blocksize)

        return format(crcvalue & 0xFFFFFFFF, '08x').upper() # Exemple: a509ae4b

    def is_file_ok(self):
        """"""

        return (self.crc32_from_name == self.crc32_from_data)

class CustomTableView(QTableView):
    """"""

    def __init__(self):
        super().__init__()

def human_sort(data: list):
    return data

def open_filebrowser(path):
    """Ouvre un explorateur de fichiers à l'adresse indiquée en argument"""

    try:
        if platform.system() == "Windows":
            from os import startfile
            startfile(path)

        elif platform.system() == "Darwin":
            from subprocess import Popen
            Popen(["open", path])

        else:
            from subprocess import Popen
            Popen(["xdg-open", path])

    except:
        return None

def main():
    application = QApplication(sys.argv)

    folderpath = None
    if len(sys.argv) == 2 and os.path.isdir(sys.argv[1]):
        folderpath = sys.argv[1]

    mainwindow = MainWindow(folderpath)
    mainwindow.show()
    application.exec_()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import math

AREA_MARGIN = 150  # actual search area in the table (apply margins)
DIST_MARGIN = 350  # threshold of detection

def coord_obstacle(sonarpos, distance):
    """
    Compute abs coords of obstacles and apply a mask on them.
    """
    x_obs = sonarpos[0] + distance * math.cos(sonarpos[2])
    y_obs = sonarpos[1] + distance * math.sin(sonarpos[2])

    if distance > DIST_MARGIN:
        return False
    if not AREA_MARGIN < y_obs < 3000-AREA_MARGIN:
        return False
    if not AREA_MARGIN < x_obs < 3000-AREA_MARGIN:
        return False
    return [x_obs, y_obs]


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="personal_commit_assistant",
    version="0.0.17",
    author="Chris Rao",
    author_email="chris.va.rao@gmail.com",
    description="Keeps your commits up to your standards",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/chrisvarao/personal_commit_assistant",
    install_requires=["pick>=0.6.4", "GitPython>=2.1.11"],
    packages=setuptools.find_packages(),
    console_scripts={
        "personal_commit_assistant=personal_commit_assistant"
    },
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
)


import plotly.graph_objects as go

from gd_analysis import helpers

from gd_analysis.analysis import (
    get_players_goal_differences,
    goal_difference_for_team,
)
from gd_analysis.data import (
    filter_competition,
    filter_season,
    filter_team_url,
    filter_player_url,
    filter_appearances,
)

TITLE_FONT = {"size": 34, "color": "white"}
LABEL_FONT = {"size": 28, "color": "white"}
TICK_FONT = {"size": 18, "color": "white"}
LEGEND_FONT = {"size": 20, "color": "white"}
PLAYER_TEXT_FONT = {"size": 18, "color": "white"}
ANNOTATION_TEXT_FONT = {"size": 18, "color": "white"}
DEFAULT_HEIGHT = 500
DEFAULT_WIDTH = 800
PLOT_BACKGROUND_COLOR = "rgba(35, 35, 35, 1)"

EMPTY_LAYOUT = go.Layout(
    height=DEFAULT_HEIGHT,
    paper_bgcolor=PLOT_BACKGROUND_COLOR,
    plot_bgcolor=PLOT_BACKGROUND_COLOR,
    xaxis={"showgrid": False},
    yaxis={"showgrid": False},
)

def get_default_layout(
    title,
    x_axis_title,
    y_axis_title,
    height=DEFAULT_HEIGHT,
    width=DEFAULT_WIDTH,
):
    layout = go.Layout(
        height=height,
        width=width,
        title={"text": f"{title}", "font": TITLE_FONT},
        paper_bgcolor=PLOT_BACKGROUND_COLOR,
        plot_bgcolor=PLOT_BACKGROUND_COLOR,
        xaxis={
            "title": f"{x_axis_title}",
            "showgrid": True,
            "zeroline": True,
            "ticks": "outside",
            "tickwidth": 1.5,
            "tickfont": TICK_FONT,
            "tickcolor": "white",
            "gridwidth": 0.5,
            "gridcolor": "grey",
            "title_font": LABEL_FONT,
        },
        yaxis={
            "title": f"{y_axis_title}",
            "showgrid": False,
            "zeroline": True,
            "ticks": "outside",
            "tickwidth": 1.5,
            "tickfont": TICK_FONT,
            "tickcolor": "white",
            "title_font": LABEL_FONT,
            "showline": True,
            "linewidth": 2,
            "linecolor": "white",
            "gridwidth": 0.5,
            "gridcolor": "grey",
        },
        legend=go.layout.Legend(
            x=0.1, y=1.07, font=LEGEND_FONT, orientation="h"
        ),
    )
    return layout

def get_scatter_for_df(
    df,
    x_column,
    y_column,
    name,
    marker_color="lightslategrey",
    line_color="white",
    marker_size=10,
    marker_symbol="circle",
):
    x_values = df[x_column].values
    y_values = df[y_column].values
    teams = df.index.get_level_values(0)
    players = df.index.get_level_values(2)

    return go.Scatter(
        x=x_values,
        y=y_values,
        name=name,
        mode="markers",
        marker={
            "color": marker_color,
            "size": marker_size,
            "line": {"width": 0, "color": line_color},
            "symbol": marker_symbol,
        },
        hovertext=[
            f"{team}
{player}
{x_column}={x:.1f}
{y_column}={y:.1f}"
            for team, player, x, y in zip(teams, players, x_values, y_values)
        ],
        hoverinfo="text",
    )

def scatter_players_for_season(
    df_players,
    df_matches,
    competition: str,
    season: str,
    x_column: str,
    y_column: str,
    min_appearances: int = 5,
    team: "str" = None,
):
    df_players = filter_competition(df_players, competition)
    df_players = filter_season(df_players, season)

    df_grouped = get_players_goal_differences(df_players)
    df_grouped = filter_appearances(df_grouped, min_appearances)

    teams = df_grouped.index.get_level_values(0)
    players = df_grouped.index.get_level_values(2)

    data = []

    if team is not None:
        team_name = helpers.get_map_from_url_to_name(df_players, "team")[team]
        trace_team = get_scatter_for_df(
            df_grouped.loc[[team]],
            x_column,
            y_column,
            name=team_name,
            marker_color="mediumvioletred",
            marker_symbol="diamond",
        )
        data.append(trace_team)
        df_grouped = df_grouped.drop(index=team)

    trace_all = get_scatter_for_df(
        df_grouped, x_column, y_column, name="All / other teams"
    )
    data.append(trace_all)

    layout = get_default_layout(f"{competition} {season}", x_column, y_column)
    layout.yaxis.update(rangemode="tozero")

    if team is not None:
        # get mean value for team
        df_matches = filter_competition(df_matches, competition)
        df_matches = filter_season(df_matches, season)
        team_goal_difference = goal_difference_for_team(df_matches, team)

        # for goal difference of team
        shapes = [
            go.layout.Shape(
                type="line",
                yref="paper",
                x0=team_goal_difference,
                y0=0,
                x1=team_goal_difference,
                y1=1,
                line=dict(color="mediumvioletred", width=4, dash="dash",),
            )
        ]
        layout.update(shapes=shapes)

    #  fig.update_xaxes(ticks="outside", tickwidth=2, tickcolor='crimson', ticklen=10)
    #  fig.update_yaxes(ticks="outside", tickwidth=2, tickcolor='crimson', ticklen=10, col=1)

    fig = go.Figure(data, layout=layout)

    return fig

def scatter_players_for_team(
    df_players,
    df_matches,
    competition,
    season,
    team,
    x_column="gd90",
    y_column="full_games",
    min_appearances: int = 5,
):
    column = "gd90"
    df_players = filter_competition(df_players, competition)
    df_players = filter_season(df_players, season)
    df_players = filter_team_url(df_players, team)
    team_name = helpers.get_map_from_url_to_name(df_players, "team")[team]

    df_players = get_players_goal_differences(df_players)
    df_players = filter_appearances(df_players, min_appearances)

    # display(df.head())

    # get mean value for team
    df_matches = filter_competition(df_matches, competition)
    df_matches = filter_season(df_matches, season)
    team_goal_difference = goal_difference_for_team(df_matches, team)

    # df_team = df_grouped[df_grouped.index.get_level_values(0) == team]

    player_names = df_players.index.get_level_values(2)

    trace = go.Scatter(
        x=df_players[x_column],
        y=df_players[y_column],
        mode="markers+text",
        name="Players",
        text=player_names,
        textposition="top center",
        marker={"color": "mediumvioletred", "size": 12, "symbol": "diamond",},
        textfont=PLAYER_TEXT_FONT,
    )

    # get mean value for team
    df_matches = filter_competition(df_matches, competition)
    df_matches = filter_season(df_matches, season)
    team_goal_difference = goal_difference_for_team(df_matches, team)

    # for goal difference of team
    shapes = [
        go.layout.Shape(
            type="line",
            yref="paper",
            x0=team_goal_difference,
            y0=0,
            x1=team_goal_difference,
            y1=1,
            line=dict(color="mediumvioletred", width=3, dash="dash",),
        )
    ]
    layout = get_default_layout(f"{team_name} {season}", x_column, y_column)
    layout.yaxis.update(rangemode="tozero")
    layout.update(shapes=shapes)

    fig = go.Figure([trace], layout=layout)

    return fig

def bar_players_for_team(
    df_players,
    df_matches,
    competition,
    season,
    team,
    column="gd90",
    weight_column="full_games",
    min_appearances: int = 5,
):
    df_players = filter_competition(df_players, competition)
    df_players = filter_season(df_players, season)
    df_players = filter_team_url(df_players, team)
    team_name = helpers.get_map_from_url_to_name(df_players, "team")[team]

    df_players = get_players_goal_differences(df_players)
    df_players = filter_appearances(df_players, min_appearances)

    df_players = df_players.sort_values(column)

    values = df_players[column]
    players = df_players.index.get_level_values(2)
    width_weights = df_players[weight_column]

    # get mean value for team
    df_matches = filter_competition(df_matches, competition)
    df_matches = filter_season(df_matches, season)
    team_goal_difference = goal_difference_for_team(df_matches, team)

    trace = go.Bar(
        x=players,
        y=df_players[column],
        width=0.05 + 0.7 * ((width_weights / width_weights.max())),
        orientation="v",
        hoverinfo="text",
        hovertext=[
            f"{player}
{column}={value:.1f}
{weight_column}={weight:.1f}"
            for player, value, weight in zip(players, values, width_weights)
        ],
        marker={"color": "mediumvioletred", "line": {"width": 0}},
    )

    layout = get_default_layout(f"{team_name} {season}", "Player", column)
    annotations = [
        go.layout.Annotation(
            x=-0.1,
            y=0.95,
            xref="paper",
            xanchor="left",
            yanchor="middle",
            yref="paper",
            text="Bar width scaled with minutes played",
            font=ANNOTATION_TEXT_FONT,
        )
    ]

    shapes = [
        go.layout.Shape(
            type="line",
            xref="paper",
            x0=0,
            y0=team_goal_difference,
            x1=1,
            y1=team_goal_difference,
            line=dict(color="mediumvioletred", width=4, dash="dash",),
        )
    ]

    layout.update(annotations=annotations, shapes=shapes)
    layout.xaxis.update(showgrid=False)
    layout.yaxis.update(showgrid=True)

    data = [trace]

    fig = go.Figure(data=data, layout=layout)

    return fig


# YOLOv5 YOLO-specific modules
# with binary
import argparse
import logging
import sys
from copy import deepcopy
import scipy.io as sio
from torch import mode

sys.path.append('./')  # to run '$ python *.py' files in subdirectories
logger = logging.getLogger(__name__)

from .common import *
# from models.swin_transformer import *
from .experimental import *
from .image_encoder import *
# from models.edsr import EDSR
from ..utils.autoanchor import check_anchor_order
from ..utils.general_samDet import make_divisible, check_file, set_logging, xywh2xyxy
from ..utils.torch_utils import time_synchronized, fuse_conv_and_bn, model_info, scale_img, initialize_weights, \
    select_device, copy_attr
#from codes.meta_SAM.segment_anything.segment_anything.build_sam import sam_model_registry
#from codes.meta_SAM.segment_anything.segment_anything.automatic_mask_generator import SamAutomaticMaskGenerator
import torchvision.ops as ops
import scipy.io as sio
import numpy
from torch.nn.utils.rnn import pad_sequence
# from models import build_model
try:
    import thop  # for FLOPS computation
except ImportError:
    thop = None
import torch.nn.functional as F

class Detect(nn.Module):
    stride = None  # strides computed during build
    export = False  # onnx export

    def __init__(self, nc=80, in_channels= 4610, representation_size=1024, anchors=(), ch=()):  # detection layer
        super(Detect, self).__init__()
        self.nc = nc  # number of classes
        self.no = nc + 5  # number of outputs per anchor
        # self.conv1 = Conv(256, 256)
        # self.c31 = C3(256, 128, 3)
        # self.conv2 = Conv(128, 64)
        # self.c32 = C3(64, 32, 3)

        #for samDet
        self.fc6 = nn.Linear(in_channels, representation_size)
        self.fc7 = nn.Linear(representation_size, representation_size)
            #prediction head
        self.cls_score = nn.Linear(representation_size, nc + 1)  # +1 should be added for background
        self.bbox_pred = nn.Linear(representation_size, 4) #num_classes * 4

    def forward(self, x, propos):
        # x = x.copy()  # for profiling
        z = []  # inference output

        self.training |= self.export

        #for samDet
        # x = self.conv1(x)
        # x = self.c31(x)
        # x = self.conv2(x)
        # x= self.c32(x)
        #x = x.flatten(start_dim=1)

        x = F.relu(self.fc6(x))
        x = F.relu(self.fc7(x))

        x = x.flatten(start_dim=1)
        scores = self.cls_score(x)
        bbox_deltas = self.bbox_pred(x)
        z.append(bbox_deltas)
        z.append(scores)
        x = torch.cat(z, 1)
        bs, _ = x.shape
        if not self.training:
                x[:,[0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12]] = x[:,[0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12]].sigmoid()
                y = x
                #y = y.view(propos.shape[0], -1, self.no)
                propos = propos.to(y.device)
                #y[..., 0:2] = y[..., 0:2] * 2. * 512. + propos[..., 0:2]#* 2. - 0.5 + self.grid[i]) * self.stride[i]  # xy
                #y[..., 2:4] = ((y[..., 2:4] * 2) ** 2) * 512. + propos[..., 2:4]#* 2) ** 2 * self.anchor_grid[i]

                y[..., 0] = (y[..., 0]  * (propos[..., 2] + 1e-6)) + propos[..., 0] * 1024
                y[..., 1] = ((y[..., 1] * (propos[..., 3] + 1e-6)) + propos[..., 1]) * 1024
                y[..., 2] = torch.exp(y[..., 2]) * (propos[..., 2] + 1e-6) *1024#*16#* 64
                y[..., 3] = torch.exp(y[..., 3]) * (propos[..., 3] + 1e-6) *1024#*16#* 64
                # x = x.view(propos.shape[0], -1, self.no)
                # x = x.unsqueeze(0)
                #y = y.unsqueeze(0)

        #x=  x.view(propos.shape[0], -1, self.no)
        # propos = propos.to(x.device)

        x = x.unsqueeze(0)
        return x if self.training else (y, x) #(torch.cat(z, 1), x)

    @staticmethod
    def _make_grid(nx=20, ny=20):
        yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])
        return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()

class Model(nn.Module):
    export = False  # onnx export
    def __init__(self, cfg='yolov5s.yaml',input_mode='RGB',ch_steam=3, ch=3, nc=None, anchors=None,config=None,sr=False,factor=2):  #att=False,sr_att=False model, input channels, number of classes
        super(Model, self).__init__()
        if isinstance(cfg, dict):
            self.yaml = cfg  # model dict
        else:  # is *.yaml
            import yaml  # for torch hub
            self.yaml_file = Path(cfg).name
            with open(cfg) as f:
                self.yaml = yaml.load(f, Loader=yaml.SafeLoader)  # model dict
        self.sr = sr
        ch = self.yaml['ch'] = self.yaml.get('ch', ch)  # input channels
        if nc and nc != self.yaml['nc']:
            logger.info(f"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}")
            self.yaml['nc'] = nc  # override yaml value
        if anchors:
            logger.info(f'Overriding model.yaml anchors with anchors={anchors}')
            self.yaml['anchors'] = round(anchors)  # override yaml value
        # self.input_mode = input_mode
        if input_mode == 'RGB+IR+fusion':
            self.steam, _ = parse_model(deepcopy(self.yaml),'steam', ch=[ch_steam],config=config)  # zjq model, savelist
        #self.model, self.save = parse_model(deepcopy(self.yaml),'backbone+head', ch=[ch],config=config)  # model, savelist   #* changed removed
        '''following part was for preparing the dataset
        model = "vit_b"
        weights_filepath = "/home/bbahaduri/sryolo/weights/sam_vit_b_01ec64.pth"
        model = sam_model_registry[model](checkpoint=weights_filepath)
        self.sam = model
        self.mask_generator = SamAutomaticMaskGenerator(
            model=self.sam,
            points_per_side=64,
            points_per_batch= 640,
            pred_iou_thresh=0.86,
            stability_score_thresh=0.92,
            crop_n_layers=0,
            crop_n_points_downscale_factor=1,
            #min_mask_region_area=100,  # Requires open-cv to run post-processing
            )
        '''
        self.conv1 = Conv(256, 256)
        self.c31 = C3(256, 128, 3)
        self.conv2 = Conv(128, 64)
        self.c32 = C3(64, 32, 3)
        self.pooler = ops.RoIPool((12, 12), 1.0)

        self.detect = Detect(nc=8)

        # self.f1=self.yaml['f1']  #蒸馏特征层层数
        # self.f2=self.yaml['f2']
        # self.f3=self.yaml['f3']

        # Build strides, anchors
         # only run once
        # m = self.model[-2]  # Detect()
        # if isinstance(m, Detect):
        #     s = 256  # 2x min stride
        #     #m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch_steam, s, s),torch.zeros(1, ch_steam, s, s),input_mode)[0]])  # forward
        #     m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch_steam, s, s),torch.zeros(1, ch_steam, s, s),input_mode)[0]])  # forward
        #     m.anchors /= m.stride.view(-1, 1, 1)
        #     check_anchor_order(m)
        #     self.stride = m.stride
        #     self._initialize_biases()  # only run once
        # Init weights, biases
        initialize_weights(self)
        self.info()
        logger.info('')

    def forward(self, x, ir=torch.randn(1,3,512,512), propos=None, input_mode='RGB+IR', augment=False, profile=False):

        # input_mode = 'RGB+IR' #IRRGB
        if input_mode=='RGB':
            ir=x
        if augment:
            img_size = x.shape[-2:]  # height, width
            s = [1, 0.83, 0.67]  # scales
            f = [None, 3, None]  # flips (2-ud, 3-lr)
            y = []  # outputs
            for si, fi in zip(s, f):
                xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max()))
                iri = scale_img(ir.flip(fi) if fi else ir, si, gs=int(self.stride.max()))
                if input_mode =='RGB+IR+fusion':
                    steam1 = self.forward_once(x,'steam',profile)
                    steam2 = self.forward_once(ir,'steam',profile)
                    steam = torch.cat([steam1,steam2],1)
                if input_mode == 'RGB+IR':
                    steam = torch.cat([xi,iri[:,0:1,:,:]],1)
                if input_mode == 'RGB':
                    steam = xi
                if input_mode == 'IR':
                    steam = iri#steam = iri[:,0:1,:,:]
                if input_mode == 'RGB+IR+MF':
                    steam = [x,ir[:,0:1,:,:]] #[:,0:1,:,:]
                yi = self.forward_once(steam, propos, 'yolo')[0]  # forward
                # yi = self.forward_once(xi)[0]  # forward
                # cv2.imwrite('img%g.jpg' % s, 255 * xi[0].numpy().transpose((1, 2, 0))[:, :, ::-1])  # save
                yi[..., :4] /= si  # de-scale
                if fi == 2:
                    yi[..., 1] = img_size[0] - yi[..., 1]  # de-flip ud
                elif fi == 3:
                    yi[..., 0] = img_size[1] - yi[..., 0]  # de-flip lr
                y.append(yi)
            return torch.cat(y, 1), None  # augmented inference, train
        else:
            if input_mode =='RGB+IR+fusion':
                steam1 = self.forward_once(x,'steam',profile)
                steam2 = self.forward_once(ir,'steam',profile)
                steam = torch.cat([steam1,steam2],1)
                # sio.savemat('features/output.mat', mdict={'data':steam.cpu().numpy()})
            if input_mode == 'RGB+IR':
                steam = torch.cat([x,ir[:,0:1,:,:]],1)
            if input_mode == 'RGB':
                steam = x
            if input_mode == 'IR':
                steam = ir#steam = ir[:,0:1,:,:]
            if input_mode == 'RGB+IR+MF':
                steam = [x,ir[:,0:1,:,:]] #[:,0:1,:,:]

            self.training |= self.export
            if self.training==True:
                if self.sr:
                    y,output_sr,features = self.forward_once(steam,'yolo', profile) #zjq
                    return y,output_sr,features
                else:
                    y,features = self.forward_once(steam, propos, 'yolo', profile) #zjq
                    return y,features
            else:
                y,features = self.forward_once(steam, propos, 'yolo', profile) #zjq

                return y,features[0], features[1]  #y[0],y[1]

    def forward_once(self, x, propos, string, profile=False):
        y, dt = [], []  # outputs
        if string == 'steam':
            for m in self.steam:
                if m.f != -1:  # if not from previous layer
                    x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f]  # from earlier layers

                if profile:
                    o = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 if thop else 0  # FLOPS
                    t = time_synchronized()
                    for _ in range(10):
                        _ = m(x)
                    dt.append((time_synchronized() - t) * 100)
                    print('%10.1f%10.0f%10.1fms %-40s' % (o, m.np, dt[-1], m.type))

                x = m(x)  # run
                #y.append(x if m.i in self.save_steam else None)  # save output
            return x
        elif string == 'yolo':
            # for samDet
            #TO DO SAM should be modified so it output image embedding as well
            # breakpoint()
            # feature_maps = self.sam.image_encoder(x)
            # breakpoint()
            # point_grids = build_all_layer_point_grids(64, 0, 1)
            '''
            *** for preparing dataset:
            _outputs, feature_maps = self.mask_generator.generate(x)
            '''
            _outputs = propos
            #feature_maps = x

            x = self.conv1(x)
            x = self.c31(x)
            x = self.conv2(x)
            x= self.c32(x)

            rois, proposals, batch_lenghts = prepare(_outputs, x.device)
            # rois_pool = deepcopy(rois)
            # for i, roi_pool in enumerate(rois_pool):

            #     roi_pool = torch.ceil(roi_pool[:, 0:4])
            #     rois_pool[i] = roi_pool.detach()
            #rois = rois.detach()
            x = self.pooler(x, rois)

            x = x.flatten(start_dim=1)
            proposals = proposals.detach()
            wh = deepcopy(proposals[:, -2:])
            wh_d = wh.detach()
            x = torch.cat([x, wh_d], dim=1)
            # rois_pool, rois, zero_mask, proposals = prepare(_outputs)
            # rois_pool = rois_pool.to(x.device)
            # zero_mask = zero_mask.to(x.device)colorstr(f'{task}: ')
            # rois = rois.to(x.device)
            # output_size = (12, 12)

            # rois_pool = rois_pool.type(torch.float32)
            # pooled_features = ops.roi_pool(x, rois_pool, output_size)

            # pooled_features = pooled_features.type(torch.float32)
            # zero_mask = zero_mask[:, None, None, None]
            # pooled_features = pooled_features * (~zero_mask).int()
            #proposals[:, 0:4] = proposals[:, 0:4] * 64
            y = self.detect(x, proposals)

            #feature_maps = feature_maps.squeeze(0)
            self.training |= self.export
            if self.training==True:
                if self.sr:
                    output_sr = self.model_up(y[self.l1],y[self.l2]) #在超分上加attention
                    return x,output_sr,y#(y[self.f1],y[self.f2],y[self.f3])#(y[4],y[8],y[18],y[21],y[24])#(y[7],y[15],y[-2])
                else:
                    return rois,y   #(y[self.f1],y[self.f2],y[self.f3])#(y[4],y[8],y[18],y[21],y[24])#(y[7],y[15],y[-2])
            else:
                y = list(y)
                y[0] = torch.split(y[0], batch_lenghts)
                y[0] = pad_sequence(y[0], batch_first=True, padding_value=0)
                y = tuple(y)

                return rois,y#(y[17],y[20],y[23])#(y[4],y[8],y[18],y[21],y[24])#(y[7],y[15],y[-2])(y[-4],y[-3],y[-2])

    def _initialize_biases(self, cf=None):  # initialize biases into Detect(), cf is class frequency
        # https://arxiv.org/abs/1708.02002 section 3.3
        # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.
        m = self.detect     #[-1]  # Detect() module   #*changed self.model[-1] to self.detect

        b_reg = m.bbox_pred.bias.view(1, -1)
        b_reg.data[:, :4] += math.log(0.1)  #adjusting the foreground/background prior

        b_cls = m.cls_score.bias.view(1, -1)  #1 is because we have 1 anchor and not 3 anchors
        b_cls.data += math.log((1 - 0.01) / 0.01)

        m.bboc_pred.bias = torch.nn.Parameter(b_reg.view(-1), requires_grad=True)
        m.cls_score.bias = torch.nn.Parameter(b_cls.view(-1), requires_grad=True)

    def _print_biases(self):
        m = self.model[-1]  # Detect() module
        for mi in m.m:  # from
            b = mi.bias.detach().view(m.na, -1).T  # conv.bias(255) to (3,85)
            print(('%6g Conv2d.bias:' + '%10.3g' * 6) % (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean()))

    # def _print_weights(self):
    #     for m in self.model.modules():
    #         if type(m) is Bottleneck:
    #             print('%10.3g' % (m.w.detach().sigmoid() * 2))  # shortcut weights

    def fuse(self):  # fuse model Conv2d() + BatchNorm2d() layers
        print('Fusing layers... ')
        for m in self.modules():   ##self.model  to self.modules
            if (type(m) is Conv) and hasattr(m, 'bn'):
                m.conv = fuse_conv_and_bn(m.conv, m.bn)  # update conv
                delattr(m, 'bn')  # remove batchnorm
                m.forward = m.fuseforward  # update forward
        self.info()
        return self

    def nms(self, mode=True):  # add or remove NMS module
        present = type(self.model[-1]) is NMS  # last layer is NMS
        if mode and not present:
            print('Adding NMS... ')
            m = NMS()  # module
            m.f = -1  # from
            m.i = self.model[-1].i + 1  # index
            self.model.add_module(name='%s' % m.i, module=m)  # add
            self.eval()
        elif not mode and present:
            print('Removing NMS... ')
            self.model = self.model[:-1]  # remove
        return self

    def autoshape(self):  # add autoShape module
        print('Adding autoShape... ')
        m = autoShape(self)  # wrap model
        copy_attr(m, self, include=('yaml', 'nc', 'hyp', 'names', 'stride'), exclude=())  # copy attributes
        return m

    def info(self, verbose=False, img_size=640):  # print model information
        model_info(self, verbose, img_size)

def parse_model(d, string, ch,config):  # model_dict, input_channels(3)
    logger.info('\n%3s%18s%3s%10s  %-40s%-30s' % ('', 'from', 'n', 'params', 'module', 'arguments'))
    anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']
    na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors  # number of anchors
    no = na * (nc + 5)  # number of outputs = anchors * (classes + 5)

    layers, save, c2 = [], [], ch[-1]  # layers, savelist, ch out
    i_shoud_add = 0

    stri = string.split('+')
    if len(stri)==2:
        string_1 = stri[0]
        string_2 = stri[1]
        d_ = d[string_1] + d[string_2]
        save.extend([2,4,5,6,8,9]) #save some layer of backbone
    else:
        d_ = d[stri[-1]]
    if string == 'head':
        ch.append(256)
    for i, (f, n, m, args) in enumerate(d_):  # from, number, module, args
        m = eval(m) if isinstance(m, str) else m  # eval strings
        for j, a in enumerate(args):
            try:
                args[j] = eval(a) if isinstance(a, str) else a  # eval strings
            except:
                pass

        n = max(round(n * gd), 1) if n > 1 else n  # depth gain
        if m in [Conv, ACmix, Bottleneck, SPP, DWConv, MixConv2d, Focus, CrossConv, BottleneckCSP, BottleneckCSP2, SPPCSP, C3, AttentionModel]:
            c1, c2 = ch[f], args[0]

            # Normal
            # if i > 0 and args[0] != no:  # channel expansion factor
            #     ex = 1.75  # exponential (default 2.0)
            #     e = math.log(c2 / ch[1]) / math.log(2)
            #     c2 = int(ch[1] * ex ** e)
            # if m != Focus:

            c2 = make_divisible(c2 * gw, 8) if c2 != no else c2

            # Experimental
            # if i > 0 and args[0] != no:  # channel expansion factor
            #     ex = 1 + gw  # exponential (default 2.0)
            #     ch1 = 32  # ch[1]
            #     e = math.log(c2 / ch1) / math.log(2)  # level 1-n
            #     c2 = int(ch1 * ex ** e)
            # if m != Focus:
            #     c2 = make_divisible(c2, 8) if c2 != no else c2

            args = [c1, c2, *args[1:]]
            if m in [BottleneckCSP, C3,BottleneckCSP2, SPPCSP]:
                args.insert(2, n)
                n = 1
        elif m is nn.BatchNorm2d:
            args = [ch[f]]
        elif m is Concat:# or m is SAM:
            c2 = sum([ch[x if x < 0 else x + 1] for x in f])
        elif m is Detect:
            args.append([ch[x + 1] for x in f])   #*changed removed + 1
            if isinstance(args[1], int):  # number of anchors
                args[1] = [list(range(args[1] * 2))] * len(f)
            print(*args)

        elif m is Contract:
            c2 = ch[f if f < 0 else f + 1] * args[0] ** 2
        elif m is Expand:
            c2 = ch[f if f < 0 else f + 1] // args[0] ** 2
        else:
            c2 = ch[f if f < 0 else f + 1]

        if string == 'backbone':
            m_ = m(img_size = args[0], patch_size=args[1], in_chans= args[2], out_chans=args[3], window_size= args[4], depth= 24, num_heads= 16, embed_dim= 1024,global_attn_indexes= [5, 11, 17, 23],)
        else:
            m_ = nn.Sequential(*[m(*args) for _ in range(n)]) if n > 1 else m(*args)  # module
        t = str(m)[8:-2].replace('__main__.', '')  # module type
        np = sum([x.numel() for x in m_.parameters()])  # number params
        m_.i, m_.f, m_.type, m_.np = i+i_shoud_add, f, t, np  # attach index, 'from' index, type, number params
        logger.info('%3s%18s%3s%10.0f  %-40s%-30s' % (i+i_shoud_add, f, n, np, t, args))  # print
        save.extend(x % (i+i_shoud_add+0.00001) for x in ([f] if isinstance(f, int) else f) if x != -1)  # append to savelist
        layers.append(m_)
        ch.append(c2)
    if string == 'backbone':
        return layers[0], sorted(save) #*changed  nn.Sequential(*layers) to layers[0]
    return nn.Sequential(*layers), sorted(save)

# if __name__ == '__main__':
#     parser = argparse.ArgumentParser()
#     #parser = argparse.ArgumentParser('Set transformer detector', add_help=False)    #newly added
#     parser.add_argument('--temp', default = 'something', type = str, help='it is nothing')
#     parser.add_argument('--cfg', default='yolov5s.yaml', type=str, help='model.yaml')
#     parser.add_argument('--device', default='cuda', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
#     #opt = parser.parse_args()
#     opt, _ = parser.parse_known_args()
#     opt.cfg = check_file(opt.cfg)  # check file
#     set_logging()
#     device = select_device(opt.device)

#     # Create model
#     model = Model(opt.cfg).to(device)
#     model.train()

    # Profile
    # img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 640, 640).to(device)
    # y = model(img, profile=True)

    # Tensorboard
    # from torch.utils.tensorboard import SummaryWriter
    # tb_writer = SummaryWriter()
    # print("Run 'tensorboard --logdir=models/runs' to view tensorboard at http://localhost:6006/")
    # tb_writer.add_graph(model.model, img)  # add model to tensorboard
    # tb_writer.add_image('test', img[0], dataformats='CWH')  # add model to tensorboard

def prepare(_outputs, device):
    max_num_boxes = 400
    if isinstance(_outputs, tuple):
        _outputs = list(_outputs)

    #new here
    proposals = deepcopy(_outputs)

    for i, rois in enumerate(_outputs):
        rois = rois * 64.0                         #changing from a base of 1 to base of 64 to match the feature maps
        rois[:, 0] = rois[:, 0] - rois[:, 2] / 2
        rois[:, 1] = rois[:, 1] - rois[:, 3] / 2
        rois[:, 2] = rois[:, 0] + rois[:, 2]
        rois[:, 3] = rois[:, 1] + rois[:, 3]
        rois = rois.to(device)
        _outputs[i] = rois

    for i, propos in enumerate(proposals):
        propos = propos.to(device)
        proposals[i] = propos

    batch_lenghts = [propos.shape[0] for propos in proposals]
    proposals = torch.cat(proposals)
    # for i  in range(len(_outputs)):

    #     if len(_outputs[i]) < 400:
    #         add_tensor = torch.zeros(max_num_boxes - len(_outputs[i]), 4)
    #         _outputs[i] = torch.cat((_outputs[i], add_tensor), dim=0)

#   proposals = torch.stack(_outputs)

#     rois = deepcopy(proposals)
#     rois = rois * 64.0                         #changing from a base of 1 to base of 64 to match the feature maps
#     rois[:, :, 0] = rois[:, :, 0] - rois[:, :, 2] / 2
#     rois[:, :, 1] = rois[:, :, 1] - rois[:, :, 3] / 2
#     rois[:, :, 2] = rois[:, :, 0] + rois[:, :, 2]
#     rois[:, :, 3] = rois[:, :, 1] + rois[:, :, 3]

    # batch_size = rois.shape[0]
    # batch_indices = torch.arange(rois.shape[0]).unsqueeze(1).unsqueeze(2).expand(rois.shape[0], 400, 1)

    # rois = torch.cat((batch_indices, rois), dim=2)

    # rois_pool = deepcopy(rois)
    # rois_pool = rois_pool.view(-1, 5)
    # output_size = (12, 12)

    # zero_mask = torch.all(rois_pool[:, 1:5] == 0, dim=1)
    # rois_pool[:, 1:5] = torch.ceil(rois_pool[:, 1:5])
    # return rois_pool, rois, zero_mask, proposals

    return _outputs, proposals, batch_lenghts


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
#%matplotlib inline

num_vectors = 1000
num_clusters = 3
num_steps = 1000
from sklearn import cluster, datasets
iris = datasets.load_iris()
X_iris = iris.data
y_iris = iris.target
X = pd.DataFrame(iris.data)
X.columns = ['Sepal_Length', 'Sepal_Width', 'Petal_Length', 'Petal_Width']

Y = pd.DataFrame(iris.target)
Y.columns = ['Targets']
plt.figure(figsize = (14,7))

colormap = np.array(['red', 'lime', 'black'])

plt.subplot(1, 2, 1)
plt.scatter(X.Sepal_Length, X.Sepal_Width, c=colormap[Y.Targets], s = 40)
plt.title('Sepal')

plt.subplot(1, 2, 2)
plt.scatter(X.Petal_Length, X.Petal_Width, c=colormap[Y.Targets], s = 40)
plt.title('Petal')

plt.show()

vectors = tf.constant(X_iris)
centroids = tf.Variable(tf.slice(tf.random_shuffle(vectors),[0,0],[num_clusters,-1]))

expanded_vectors = tf.expand_dims(vectors, 0)
expanded_centroids = tf.expand_dims(centroids, 1)

print (expanded_vectors.get_shape())
print (expanded_centroids.get_shape())

distances = tf.reduce_sum(tf.square(tf.sub(expanded_vectors, expanded_centroids)), 2)
assignments = tf.argmin(distances, 0)

means = tf.concat(0, [tf.reduce_mean(tf.gather(vectors, tf.reshape(tf.where(tf.equal(assignments, c)),[1,-1])),
	reduction_indices=[1])for c in range(num_clusters)])

update_centroids = tf.assign(centroids, means)
init_op = tf.initialize_all_variables()

with tf.Session() as sess:
    sess.run(init_op)
    for step in range(num_steps):
        _, centroid_values, assignment_values = sess.run([update_centroids, centroids, assignments])

print("centoroids")
print(centroid_values)
print(y_iris)
print(assignment_values)

data = {"x": [], "y": [], "z": [], "w": [], "cluster": []}
for i in range(len(assignment_values)):
    data["x"].append(X_iris[i][0])
    data["y"].append(X_iris[i][1])
    data["z"].append(X_iris[i][2])
    data["w"].append(X_iris[i][3])
    data["cluster"].append(assignment_values[i])
df = pd.DataFrame(data)

sns.lmplot("x", "y", data=df,
           fit_reg=False, size=7,
           hue="cluster", legend=False)
plt.show()

sns.lmplot("z", "w", data=df,
           fit_reg=False, size=7,
           hue="cluster", legend=False)
plt.show()

#raw data with t-sne and pca
X_tsne = TSNE(learning_rate=100).fit_transform(iris.data)
pca = PCA(n_components=2)
X_pca = pca.fit(iris.data).transform(iris.data)
#X_pca = PCA().fit_transform(iris.data)

#clustered data with t-sne and pca
X_tsne1 = TSNE(learning_rate=100).fit_transform(df)
pca1 = PCA(n_components=2)
X_pca1 = pca1.fit(df).transform(df)

plt.figure(figsize=(10, 5))
plt.subplot(1,2,1)
plt.scatter(X_tsne[:, 0], X_tsne[:, 1],c=colormap[Y.Targets])
plt.subplot(1,2,2)
plt.scatter(X_tsne1[:, 0], X_tsne1[:, 1],c=colormap[Y.Targets])
plt.show()

plt.figure(figsize=(10, 5))
plt.subplot(1,2,1)
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=colormap[Y.Targets])
plt.subplot(1,2,2)
plt.scatter(X_pca1[:, 0], X_pca1[:, 1], c=colormap[Y.Targets])
plt.show()

plt.figure(figsize=(10, 5))
plt.subplot(1,2,1)
plt.scatter(X_tsne1[:, 0], X_tsne1[:, 1],c=colormap[Y.Targets])
plt.subplot(1,2,2)
plt.scatter(X_pca1[:, 0], X_pca1[:, 1], c=colormap[Y.Targets])
plt.show()

plt.figure(figsize=(10, 5))
plt.subplot(1,2,1)
plt.scatter(X_tsne[:, 0], X_tsne[:, 1],c=colormap[Y.Targets])
plt.subplot(1,2,2)
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=colormap[Y.Targets])
plt.show()


# The contents of this file are subject to the Mozilla Public
# License Version 1.1 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of
# the License at http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS
# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
# implied. See the License for the specific language governing
# rights and limitations under the License.
#
# The Initial Owner of the Original Code is European Environment
# Agency (EEA).  Portions created by Finsiel Romania are
# Copyright (C) European Environment Agency.  All
# Rights Reserved.
#
# Authors:
#
# Alex Morega, Eau de Web
# Cornel Nitu, Eau de Web
# Valentin Dumitru, Eau de Web

from OFS.Folder import Folder
from Globals import InitializeClass
from AccessControl import ClassSecurityInfo
from AccessControl.Permissions import view
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from Products.ZCatalog.ZCatalog import manage_addZCatalog
#from Products.MailHost.MailHost import MailHost, manage_addMailHost
#from Products.ZTinyMCE.TinyMCE import manage_addZTinyMCE       --- no longer needed (served from zip)

from constants import *
from utilities import *
from Factsheet import manage_addFactsheet_html

#------------------------------------------------------------------------------

manage_addFactsheetFolder_html = PageTemplateFile('zpt/folder', globals())

def loadCatalogueMetadata(catalogue):
    # create columns
    catalogue.addColumn('id')
    catalogue.addColumn('title')
    catalogue.addColumn('meta_type')
    catalogue.addColumn('bobobase_modification_time')

    # create indexes
    catalogue.addIndex('id', 'FieldIndex')

    catalogue.manage_addIndex('model_keywords', 'TextIndexNG3', extra={'default_encoding': 'utf-8', 'splitter_casefolding': True})
    for field in searchable_lists:
        catalogue.addIndex(field, 'KeywordIndex')

#------------------------------------------------------------------------------

email_expr = re.compile(r'^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}$', re.IGNORECASE)

def manage_addFactsheetFolder(self, title, administrator_email, REQUEST=None):
    "Method for adding a Factsheet container"
    if not title:
        raise ValueError('Name of Folder is mandatory!')
    if not administrator_email:
        raise ValueError('The administrative email is mandatory!')
    if not email_expr.match(administrator_email):
        raise ValueError('Please enter a valid email address!')
    id = slugify(title)
    newFactsheetFolder = FactsheetFolder(id, title, administrator_email)
    self._setObject(id, newFactsheetFolder)
    folder = self._getOb(id)
    manage_addZCatalog(folder, id='catalogue', title='Models catalogue')
    catalogue = folder._getOb('catalogue')
    loadCatalogueMetadata(catalogue)

    #Check for and add a MailHost object
    #ob = getattr(folder, MAILHOST, None)
    #if not ob:
        #manage_addMailHost(self, id=MAILHOST, title=MAILHOST, smtp_host='localhost', smtp_port=25)

    # Alternative way to create a TinyMCE object and to set the configuration
    #manage_addZTinyMCE(folder, id='TinyMCE')
    #TinyMCE = folder._getOb('TinyMCE')
    #conf_file=getattr(TinyMCE, 'advanced.conf')
    #conf_file.manage_saveConfiguration(ZTinyMCE_CONFIGURATION, 'Example configuration')

    return self.manage_main(self, REQUEST)

#------------------------------------------------------------------------------

class ConfigurationError(Exception):
    pass

#------------------------------------------------------------------------------

class FactsheetFolder(Folder):
    def __init__(self, id, title, administrator_email,
                    introduction = 'Introduction text',
                    instructions = 'Instructions text',
                    sender_email = 'no-reply@eea.europa.eu',
                    smtp = 'localhost',
                    smtp_port = '25'):
        """ constructor """
        self.id = id
        self.title = title
        self.administrator_email = administrator_email
        self.introduction =introduction
        self.instructions = instructions
        self.sender_email = sender_email
        self.smtp = smtp
        self.smtp_port = smtp_port

    meta_type = "OMI Factsheet Folder"
    security = ClassSecurityInfo()

    index_html = PageTemplateFile('zpt/folder_index', globals())
    instructions_html = PageTemplateFile('zpt/folder_instructions', globals())
    base_html = PageTemplateFile('zpt/base_template', globals())
    _search_html = PageTemplateFile('zpt/search', globals())
    _edit_html = PageTemplateFile('zpt/folder_edit', globals())
    manage_addFactsheet_html = manage_addFactsheet_html

    security.declareProtected(MANAGE_FACTSHEET_FOLDER, 'edit_html')
    def edit_html(self, REQUEST):
        """ edit folder properties - for the moment just the administrator email"""
        submit = REQUEST.get('save', '')
        if submit:
            self.title = REQUEST.form.get('title')
            self.administrator_email = REQUEST.form.get('administrator_email')
            self.introduction = REQUEST.form.get('introduction')
            self.instructions = REQUEST.form.get('instructions')
            self.sender_email = REQUEST.form.get('sender_email')
            self.smtp = REQUEST.form.get('smtp')
            self.smtp_port = REQUEST.form.get('smtp_port')
            self.folder_add_edit_notification(self.administrator_email)
            REQUEST.RESPONSE.redirect('index_html')
        return self._edit_html(REQUEST)

    security.declareProtected(view, 'add_html')
    def add_html(self):
        """ """
        return self.manage_addFactsheet_html(self.REQUEST)

    def getFactsheetFolder(self):
        """ return self """
        return self

    def search_html(self, REQUEST=None):
        """ advanced search """
        results = []
        submit = REQUEST.get('search_models', '')
        mode = REQUEST.get('mode', '')
        if submit:
            catalogue = self.catalogue
            query = REQUEST.form.get('query', '')
            filters = {}
            if query:
                filters['model_keywords'] = query.translate(string.maketrans('*:\'"()[]{}', '__________'))
            if mode:
                # advanced search
                for k,v in REQUEST.form.items():
                    if k in searchable_lists:
                        filters[k] = v
            if filters:
                results = catalogue(filters)
                results = list(set(results))    # get brains
                results = list(get_objects(catalogue, results))   # get the actual objects
        return self._search_html(REQUEST, themes_covered = potential_themes, dominant_analytical_techniques = analytical_techniques, results=results)

    def add_message(self, message):
        self.REQUEST.SESSION.set('message', message)

    def show_message(self):
        message = self.REQUEST.SESSION.get('message', '')
        if message:
            self.REQUEST.SESSION.set('message', '')
        return message

    security.declareProtected(MANAGE_FACTSHEET, 'deleteFactsheet')
    def deleteFactsheet(self, REQUEST):
        """ Remove a factsheet"""
        id=REQUEST.form.get('factsheet_id')
        self.manage_delObjects(id)
        self.add_message('Model successfully deleted!')
        REQUEST.RESPONSE.redirect('index_html')

#    security.declarePrivate('get_mailhost')
#    def get_mailhost(self):
#        """ """
#        try:
#            ob = getattr(self, MAILHOST, None)
#            if isinstance(ob, MailHost):
#                return ob
#        except:
#            pass
#        raise ConfigurationError('Expected a MailHost object named '
#            '"MailHost". Please create one.')

    security.declarePrivate('send_mail')
    def send_mail(self, msg_to, msg_subject, msg_body, msg_body_text):
#        mailhost = self.get_mailhost()
#
#        from email.MIMEText import MIMEText
#        from email.MIMEMessage import MIMEMessage
#
#        msg = MIMEMessage(MIMEText(msg_body.encode('utf-8'), _charset='utf-8'))
#        msg['Subject'] = msg_subject
#        msg['From'] = self.administrator_email
#        msg['To'] = msg_to
#
#        mailhost.send(msg.as_string())
        import smtplib

        from email.MIMEMultipart import MIMEMultipart
        from email.MIMEText import MIMEText

        # Create message container - the correct MIME type is multipart/alternative.
        msg = MIMEMultipart('alternative')
        msg['Subject'] = msg_subject
        msg['From'] = self.sender_email
        msg['To'] = msg_to

        # Create the body of the message (a plain-text and an HTML version).
        text = msg_body_text
        html = msg_body

        # Record the MIME types of both parts - text/plain and text/html.
        part1 = MIMEText(text, 'plain')
        part2 = MIMEText(html, 'html')

        # Attach parts into message container.
        # According to RFC 2046, the last part of a multipart message, in this case
        # the HTML message, is best and preferred.
        msg.attach(part1)
        msg.attach(part2)

        # Send the message via local SMTP server.
        s = smtplib.SMTP(self.smtp, self.smtp_port)
        # sendmail function takes 3 arguments: sender's address, recipient's address
        # and message to send - here it is sent as one string.
        s.sendmail(self.sender_email, msg_to, msg.as_string())
        s.quit()

    security.declarePrivate('getCatalogedObjects')
    def getCatalogedObjects(self, meta_type=None, approved=0, howmany=-1, sort_on='releasedate', sort_order='reverse', has_local_role=0, **kwargs):
        results = []
        filter = {'submitted': 1} # only submitted items
        if approved == 1: l_filter['approved'] = 1
        if has_local_role == 1: filter['has_local_role'] = 1
        if sort_on != '':
            filter['sort_on'] = sort_on
            if sort_order != '':
                filter['sort_order'] = sort_order
        if meta_type: filter['meta_type'] = self.utConvertToList(meta_type)
        else: filter['meta_type'] = self.searchable_content
        # extra filters
        filter.update(kwargs)
        # perform the search
        results = self.__searchCatalog(filter)
        if howmany != -1:
            results = results[:howmany]
        results = self.__getObjects(results)
        return results

    security.declarePrivate('folder_add_edit_notification')
    def folder_add_edit_notification(self, email):
        """ send a notification when a folder is added / edited / commented"""
#        mailhost = self.get_mailhost()
#        if mailhost:
        values = {'folder_view_link': '%s' % self.absolute_url()}
        self.send_mail(msg_to=email,
                        msg_subject='%s - Folder added / edited' % self.title,
                        msg_body=FOLDER_ADD_EDIT_TEMPLATE % values,
                        msg_body_text=FOLDER_ADD_EDIT_TEMPLATE_TEXT % values
                        )

    def generate_password(self, email, passwordLength = 8):
        from random import Random

        # if the email is found in another model, the same password will be set
        for model in self.objectValues('OMI Factsheet'):
            if model.contact_email == email:
                return model.password

        rnd = Random()
        righthand = '23456qwertasdfgzxcvbQWERTASDFGZXCVB'
        lefthand = '789yuiophjknmYUIPHJKLNM'
        password = ''
        for i in range(passwordLength):
            if i%2:
                password += rnd.choice(lefthand)
            else:
                password += rnd.choice(righthand)
        return password

    def newlineToBr(self, text):
        """ """
        if text.find('\r') >= 0: text = ''.join(text.split('\r'))
        if text.find('\n') >= 0: text = '<br />'.join(text.split('\n'))
        return text

    def formatDateTime(self, date):
        """date is a DateTime object. This function returns a string 'dd month_name yyyy hh:mm:ss'"""
        try: return date.strftime('%d %B %Y %H:%M')
        except: return ''

    security.declarePublic('canManageFactsheetFolder')
    def canManageFactsheetFolder(self):
        """ Check the permissions to edit/delete factsheets """
        return checkPermission(MANAGE_FACTSHEET_FOLDER, self)

    security.declarePublic('get_models')
    def get_models(self):
        """ return the factsheet objects from the folder """
        return self.objectValues('OMI Factsheet')

InitializeClass(FactsheetFolder)


"""
sockets for cross-task communication: start threads to communicate over sockets;
independent programs can too, because sockets are system-wide, much like fifos;
see the GUI and Internet parts of the book for more realistic socket use cases;
some socket servers may also need to talk to clients in threads or processes;
sockets pass byte strings, but can be pickled objects or encoded Unicode text;
caveat: prints in threads may need to be synchronized if their output overlaps;
"""

from socket import socket, AF_INET, SOCK_STREAM                     # portable socket api

port = 50008                                                        # poer number identifies socket on machine
host = 'localhost'                                                  # machine name or IP address;  bind to port on this machine

def server():
    sock = socket(AF_INET, SOCK_STREAM)                              # create a socket; ip addresses tcp connection
    sock.bind((host, port))                                          # bind socket to port on this machine
    sock.listen(5)                                                   # listen for incoming connections
    while True:
        conn, addr = sock.accept()                                   # accept connection from client
        data = conn.recv(1024)                                       # receive bytes data from this client
        reply = 'server got: [%s]' % data                            # reply to client; conn is a new connected socket
        conn.send(reply.encode())                                    # send bytes reply to client

def client(name):
    sock = socket(AF_INET, SOCK_STREAM)                              # create a socket
    sock.connect((host, port))                                       # connect to server
    sock.send(('Hello, %s' % name).encode())                         # send bytes data to server
    reply = sock.recv(1024)                                          # receive bytes data from server
    sock.close()                                                     # close socket
    print('client got: [%s]' % reply.decode())                       # print decoded bytes data from server

if __name__ == '__main__':
    from threading import Thread
    sthread = Thread(target=server)
    sthread.daemon = True                                            # don't wait for server thread
    sthread.start()                                                  # do wait for children to exit
    for i in range(5):
        Thread(target=client, args=('client%s' % i,)).start()

"""
Once connected, the client and server transfer byte strings by
using send and receive calls instead of writes and reads.
"""


class Solution:
    def isPalindrome(self, x):
        """
        :type x: int
        :rtype: bool
        """
        n = []
        num = 0
        y = x
        if x < 0:
            return False
        if x == 0:
            return True
        while(y):
            n.append(y%10)
            y //= 10
        n.reverse()
        for i in range(len(n)):
            num += n[i]*pow(10,i)
        if num == x:
            return True
        else:
            return False
if __name__ == '__main__':
    palind = Solution()
    x = 12345654321
    print(palind.isPalindrome(x))


#-------------------------------------------------------------------------
# AUTHOR: Anthony Seward
# FILENAME: decision_tree.py
# SPECIFICATION: implementation of a decision tree
# FOR: CS 4210- Assignment #1
# TIME SPENT: how long it took you to complete the assignment
#-----------------------------------------------------------*/

#IMPORTANT NOTE: DO NOT USE ANY ADVANCED PYTHON LIBRARY TO COMPLETE THIS CODE SUCH AS numpy OR pandas. You have to work here only with standard
# dictionaries, lists, and arrays

#importing some Python libraries
from sklearn import tree
import matplotlib.pyplot as plt
import csv
db = []
X = []
Y = []

#reading the data in a csv file
with open('contact_lens.csv', 'r') as csvfile:
  reader = csv.reader(csvfile)
  for i, row in enumerate(reader):
      if i > 0: #skipping the header
         db.append (row)
         print(row)

#transform the original categorical training features into numbers and add to the 4D array X. For instance Young = 1, Prepresbyopic = 2, Presbyopic = 3
# so X = [[1, 1, 1, 1], [2, 2, 2, 2], ...]]
#--> add your Python code here
#i could automate this in the for loop, but it seemed quicker to do manually for this one
#define the numeric translations in a dictionary
translations = {'young': 0, 'prepresbyopic': 1, 'presbyopic': 2,
                'myope': 0, 'hypermetrope': 1,
                'yes': 0, 'no': 1,
                'reduced': 0, 'normal': 1}
rowtrans = [0, 0, 0, 0]
for i in range(len(db)): #all rows
  for j in range(len(db[i]) - 1): #all columns except the last
    rowtrans[j] = translations[db[i][j].lower()] #translate all the strings to numbers in each row
  X.append(tuple(rowtrans))

#transform the original categorical training classes into numbers and add to the vector Y. For instance Yes = 1, No = 2, so Y = [1, 1, 2, 2, ...]
#--> addd your Python code here
for i in range(len(db)):
  Y.append(translations[db[i][len(db[i])-1].lower()]) #add the translation of the last column in the row

#fitting the decision tree to the data
clf = tree.DecisionTreeClassifier(criterion = 'entropy')
clf = clf.fit(X, Y)

#plotting the decision tree
tree.plot_tree(clf, feature_names=['Age', 'Spectacle', 'Astigmatism', 'Tear'], class_names=['Yes','No'], filled=True, rounded=True)
plt.show()


from ..models import Score, User
from django.http import HttpResponse
from . import util
from django.views.decorators.csrf import csrf_exempt
from django.core.exceptions import ObjectDoesNotExist

@csrf_exempt
def score(request):
    if 'HTTP_ID' not in request.META or 'HTTP_KEY' not in request.META or 'HTTP_DATE' not in request.META:
        return HttpResponse(status=400)

    try:
        user = User.objects.get(id=request.META['HTTP_ID'], key=request.META['HTTP_KEY'])
    except ObjectDoesNotExist:
        return HttpResponse(status=401)

    if request.method == 'GET':
        scores = Score.objects.filter(user_id=user.id)
        print(scores)

        score_list = []
        for score_object in scores:
            item = {'type': score_object.type, 'score': score_object.score, 'must_title': score_object.must.title}
            score_list.append(item)
        print(score_list)

        return util.JSONResponse(score_list)

    else:
        return HttpResponse(status=404)


import datetime as dt

def exchange(configuration_filename, time, input_names,
              input_values, output_names, save_to_file=False,
              memory=None):
    """
    """
    # Process inputs
    result = 0
    P = 0
    Pmax = 2000
    Q = 0
    Qmax = 2000 * 1.05 * 0.44
    for name, value in zip(input_names, input_values):
        if 'KW' in name:
            P = float(value)
        if 'KVAR' in name:
            Q = float(value)
    result = 1.0 + 0.1 * P / float(Pmax) + 0.1 * Q / float(Qmax)
    outputs = [result]

    if memory is None:
        memory = 1
        header = ('sim_time,clock,' +
                  ','.join(input_names) + ',' +
                  ','.join(output_names) + '\n')
        with open("debug.csv", "w") as f:
            f.write(header)

    # [Optional debug]
    with open("debug.csv", "a") as f:
        row = (str(time) + ',' +
               str(dt.datetime.now()) + ',' +
               ','.join(map(str, input_values)) + ',' +
               ','.join(map(str, outputs)) + '\n')
        f.write(row)
    print(row)
    return [result, memory]


class Employe:
    no_of_leaves = 8

    def __init__(self, name, sallary, role):
        self.name = name
        self.sallary = sallary
        self.role = role

    def printDetails(self):
        return f"The name is {self.name}. His sallary is {self.sallary} and role is {self.role}"

    @classmethod
    def change_leaves(cls, newleaves):
        cls.no_of_leaves = newleaves

    @classmethod
    def from_dash(cls, string):
        return cls(*string.split(","))

anmol = Employe("Anmol", 24000, "Student")
harry = Employe.from_dash("Harry,12000000,Instructor")
anmol.change_leaves(10)
print(anmol.__dict__)
print(anmol.no_of_leaves)
print(harry.__dict__)
print(harry.no_of_leaves)


#!/usr/bin/env python3

import sys, os, re
import collections
import math
sys.setrecursionlimit(200)

def debug(x):
    print(x, file=sys.stderr)

def solve_small(K, C, S):
    ans = range(1, K+1)
    return " ".join([str(x) for x in ans])

def solve(K, C, S):
    ans = []
    k_seen = 0
    for s in range(S):
        x = 0
        for layer in range(C):
            if x + k_seen * K**layer < K**C:
                x += k_seen * K**layer
                k_seen += 1
        if str(x+1) not in ans:
            ans.append(str(x + 1))
    return " ".join(ans)

def main():
    inp = [x.strip() for x in sys.stdin.readlines()]
    T, D_raw = int(inp[0]), inp[1:]
    D = [[int(y) for y in x.split(" ")] for x in D_raw]

    debug(T)
    debug(D)

    for numinput, d in enumerate(D, 1):
        K, C, S = d
        debug("%d %d %d" % (K, C, S))
        #if S == K:
        #    ans = solve_small(K, C, S)
        if K > S * C:
            ans = "IMPOSSIBLE"
        else:
            ans = solve(K, C, S)
        print("Case #%d: %s" % (numinput, ans))

if __name__ == '__main__':
    main()


"""
Demo run of the service.

Loads a super basic animation studio schema with in-memory sqlite db.
"""
import click

from sqlalchemy import func, Column, DateTime, Index, Integer, String, Table

from dbsvc import api, service

class SQLiteIDManager(api.IDManager):
    def generate_id(self, table: Table) -> int:
        return super().generate_id(table) >> 32

class Schema(api.Schema):
    def build(self, metadata: api.MetaData):
        project_table = Table(
            "Project",
            metadata,
            Column("id", Integer, primary_key=True),
            Column("name", String, nullable=False, unique=True),
        )

        sequence_table = Table(
            "Sequence",
            metadata,
            Column("id", Integer, primary_key=True),
            Column("name", String, nullable=False),
            Column("project_id", Integer, nullable=False),
        )
        Index(
            "unique_sequence_name_project_id",
            sequence_table.c.name,
            sequence_table.c.project_id,
            unique=True,
        )

        shot_table = Table(
            "Shot",
            metadata,
            Column("id", Integer, primary_key=True),
            Column("name", String, nullable=False),
            Column("sequence_id", Integer, nullable=False),
            Column("frame_start", Integer),
            Column("frame_end", Integer),
        )
        Index(
            "unique_shot_name_sequence_id",
            shot_table.c.name,
            shot_table.c.sequence_id,
            unique=True,
        )

        asset_table = Table(
            "Asset",
            metadata,
            Column("id", Integer, primary_key=True),
            Column("type", String, nullable=False),
            Column("name", String, nullable=False),
            Column("project_id", Integer, nullable=False),
        )
        Index(
            "unique_asset_name_project_id",
            asset_table.c.name,
            asset_table.c.project_id,
            unique=True,
        )

        asset_shot_table = Table(
            "AssetXShot",
            metadata,
            Column("asset_id", Integer, nullable=False),
            Column("shot_id", Integer, nullable=False),
        )
        Index(
            "asset_shot",
            asset_shot_table.c.asset_id,
            asset_shot_table.c.shot_id,
            unique=True,
        )

        output_table = Table(
            "Output",
            metadata,
            Column("id", Integer, primary_key=True),
            Column("context", String, nullable=False),
            Column("datatype", String, nullable=False),
            Column("version", Integer, nullable=False),
            Column("template", String, nullable=False),
            Column("ext", String, nullable=False),
            Column("frame_range", String),  # eg, 1-10,19
            Column("variant", String),
            Column("status", Integer, default=0),
            Column("created_at", DateTime, nullable=False, default=func.now()),
            Column("created_by", String, nullable=False),
            Column("updated_at", DateTime, default=func.now()),
            Column("updated_by", String, nullable=False),
            Column("approved_at", DateTime),
            Column("approved_by", String),
        )
        Index(
            "unique_output",
            output_table.c.context,
            output_table.c.datatype,
            output_table.c.version,
            unique=True,
        )

def build_service(uri: str, ids: bool = False):
    id_manager = None
    if ids:
        id_manager = SQLiteIDManager() if uri.startswith("sqlite") else api.IDManager()
    return service.build(uri, Schema(), id_manager=id_manager)

@click.group()
@click.pass_context
@click.option(
    "--uri", default="sqlite://", help="Database URI to use, defaults to memory sqlite DB"
)
@click.option(
    "--ids", is_flag=True, flag_value=True, help="If enabled, generates IDs for the schema"
)
# Ideally needs some way to set the schema and/or IDManager
def cli(ctx, uri: str = None, ids: bool = False):
    ctx.obj = build_service(uri, ids=ids)

@cli.command()
@click.pass_context
def run(ctx):
    svc = ctx.obj
    svc.run(host="localhost", port=8080, debug=True, reloader=True)

@cli.command()
@click.pass_context
def routes(ctx):
    svc = ctx.obj
    for route in svc.routes:
        print(f"{route.method:7} {route.rule}")

if __name__ == "__main__":
    cli()


from typing import Sequence

import numpy as np
from sklearn.preprocessing import StandardScaler

from .pal_base import PALBase
from .validate_inputs import validate_nt_models, validate_optimizers, validate_positive_integer_list
from ..models.nt import JaxOptimizer, NTModel

# Again, the idea of having the core as pure functions outside of the class is that
# we could parallelize it easier in this way
def _ensemble_train_one_finite_width(  # pylint:disable=too-many-arguments, too-many-locals
    i: int,
    models: Sequence[NTModel],
    design_space: np.ndarray,
    objectives: np.ndarray,
    sampled: np.ndarray,
    optimizers: Sequence[JaxOptimizer],
    key: object,
    training_steps: Sequence[int],
    ensemble_size: Sequence[int],
):
    from jax import random  # pylint:disable=import-outside-toplevel
    from jax import grad, jit, vmap  # pylint:disable=import-outside-toplevel

    model = models[i]
    optimizer = optimizers[i]
    loss = jit(lambda params, x, y: 0.5 * np.mean((model.apply_fn(params, x) - y) ** 2))
    grad_loss = jit(lambda state, x, y: grad(loss)(optimizer.get_params(state), x, y))

    x_train = design_space[sampled[:, i]]

    scaler = StandardScaler()
    y_train = scaler.fit_transform(objectives[sampled[:, i], i].reshape(-1, 1))

    def train_network(key):
        _, params = model.init_fn(key, (-1, x_train.shape[1]))
        opt_state = optimizer.opt_init(params)

        for j in range(training_steps[i]):
            opt_state = optimizer.opt_update(j, grad_loss(opt_state, x_train, y_train), opt_state)

        return optimizer.get_params(opt_state)

    ensemble_key = random.split(key, ensemble_size[i])
    params = vmap(train_network)(ensemble_key)

    return params, scaler

def _ensemble_predict_one_finite_width(i: int, models: Sequence[NTModel], design_space):
    from jax.api import vmap  # pylint:disable=import-outside-toplevel

    model = models[i]

    ensemble_func = vmap(model.apply_fn, (0, None))(model.params, design_space)

    mean_func = np.reshape(np.mean(ensemble_func, axis=0), (-1,))
    std_func = np.reshape(np.std(ensemble_func, axis=0), (-1,))

    return mean_func, std_func

__all__ = ["PALJaxEnsemble", "NTModel", "JaxOptimizer"]

class PALJaxEnsemble(PALBase):  # pylint:disable=too-many-instance-attributes
    """Use PAL with and ensemble of finite-width neural networks.
    Note that we current assume that there is one model per output,
    i.e., we did not yet implement multihead support.
    """

    def __init__(self, *args, **kwargs):
        """Construct the PALJaxEnsemble instance

        Args:
            X_design (np.array): Design space (feature matrix)
            models (Sequence[NTModel]): You need to provide a sequence of
                 NTModel (`pyepal.models.nt.NTModel`).
                The elements of this dataclass are the `apply_fn`, `init_fn`,
                `kernel_fn` and `predict_fn` (for latter you can typically
                provide `None`).
                Can be constructed with
                :py:func:`pyepal.pal.models.nt.build_dense_network`.
            optimizer (Union[JaxOptimizer, Sequence[JaxOptimizer]]):
                Sequence of dataclasses with functions for a JAX optimizer,
                can be constructed with :py:func:`pyepal.pal.models.nt.get_optimizer`.
            ndim (int): Number of objectives
            epsilon (Union[list, float], optional): Epsilon hyperparameter.
                Defaults to 0.01.
            delta (float, optional): Delta hyperparameter. Defaults to 0.05.
            beta_scale (float, optional): Scaling parameter for beta.
                If not equal to 1, the theoretical guarantees do not necessarily hold.
                Also note that the parametrization depends on the kernel type.
                Defaults to 1/9.
            goals (List[str], optional): If a list, provide "min" for every objective
                that shall be minimized and "max" for every objective
                that shall be maximized. Defaults to None, which means
                that the code maximizes all objectives.
            coef_var_threshold (float, optional): Use only points with
                a coefficient of variation below this threshold
                in the classification step. Defaults to 3.
            key (int): Seed to generate the key for the JAX
                pseudo-random number generator. Defaults to 10.
            training_steps (Union[int, Sequence[int]]): Number of epochs,
                the networks are trained. Defaults to 500.
            ensemble_size (Union[int, Sequence[int]]): Size of the ensemble, i.e.,
                over how many randomly initialized neural networks we average
                to obtain estimates of mean and standard deviation.
                Automatically vectorized using `vmap`.
                Defaults to 100.
        """
        from jax import random  # pylint:disable=import-outside-toplevel

        self.optimizers = validate_optimizers(kwargs.pop("optimizers"), kwargs.get("ndim"))

        self.training_steps = validate_positive_integer_list(
            kwargs.pop("training_steps", 500), kwargs.get("ndim")
        )
        self.ensemble_size = validate_positive_integer_list(
            kwargs.pop("ensemble_size", 100), kwargs.get("ndim")
        )
        self.key = random.PRNGKey(kwargs.pop("key", 10))
        self.design_space_scaler = StandardScaler()
        super().__init__(*args, **kwargs)
        self.models = validate_nt_models(self.models, self.ndim)

    def _set_data(self):
        self.design_space = self.design_space_scaler.fit_transform(self.design_space)

    def _train(self):
        for i, _ in enumerate(self.models):
            params, scaler = _ensemble_train_one_finite_width(
                i,
                self.models,
                self.design_space,
                self.y,
                self.sampled,
                self.optimizers,
                self.key,
                self.training_steps,
                self.ensemble_size,
            )
            self.models[i].params = params
            self.models[i].scaler = scaler
            self.y[:, i] = scaler.transform(self.y[:, i].reshape(-1, 1)).flatten()

    def _predict(self):
        means, stds = [], []
        for i in range(len(self.models)):
            mean, std = _ensemble_predict_one_finite_width(i, self.models, self.design_space)
            means.append(mean.reshape(-1, 1))
            stds.append(std.reshape(-1, 1))

        self._means = np.hstack(means)
        self.std = np.hstack(stds)


from flask import render_template, request
from flask import jsonify
from app import app
import os
import requests
import json
import uuid
import copy
from ds import DataStorage

@app.route('/')
@app.route('/index')
def index():
    ds = DataStorage()
    all = ds.get_all()
    print(all)
    return render_template("index.html", title = "title", all = all)

@app.route('/saveRecipe', methods=['POST'])
def save_recipe():
    ds = DataStorage()
    new = ds.update(json.loads(request.json))
    ds.save()

    return jsonify({"was_new":new, "recipe": request.json})

@app.route('/getUUID')
def get_uuid():
    return str(uuid.uuid4())

@app.route('/getRecipe/<uuid>')
def get_recipe(uuid):
    ds = DataStorage()
    r, country = ds.find_recipe(uuid)
    r["country"] = country["country"]
    return json.dumps(r)

@app.route('/newCountry/<book_uuid>/<country>')
def new_country(book_uuid, country):
    ds = DataStorage()
    if ds.add_country(book_uuid, country):
        ds.save()
    else:
        print("Failed")

    return country

@app.route('/deleteRecipe/<uuid>')
def delete_recipe(uuid):
    ds = DataStorage()
    res = ds.delete_recipe(uuid)
    ds.save()

    return uuid

@app.context_processor
def utility_processor():
    def sc(country):
        return country.replace(" ", "-")
    return dict(sc=sc)


# *args和**kwargs主要用于函数定义，比如写函数装饰器时，也可以用来做猴子补丁（monkey patching，在程序运行时修改某些代码）

"""
*args 是用来发送一个非键值对的可变数量的参数列表给一个函数（预先不知道函数使用者会传递多少个参数给函数）
"""
def test_args(foo, *args):
    print('The first arg is ', foo)
    for arg in args:
        print('another arg: ', arg)

# test_args('yuefen', 'qiyue', 'bayue', 'jiuyue')

"""
**kwargs 允许将不定长度的键值对参数传递给函数
"""
def test_kwargs(**kwargs):
    for key, value in kwargs.items():
        print("{0} == {1}".format(key, value))

# test_kwargs(name='qiyue')

"""
使用*args和**kwargs调用一个函数
"""
def test_args_kwargs(arg1, arg2, arg3):
    print("arg1:", arg1)
    print("arg2:", arg2)
    print("arg3:", arg3)

args = ("two", 3, 5)
test_args_kwargs(*args)

kwargs = {"arg3": 3, "arg2": "two", "arg1": 5}
test_args_kwargs(**kwargs)


#!/usr/bin/python3
"""
    Script that prints the first State object from the database hbtn_0e_6_usa
"""
if __name__ == "__main__":
    from sqlalchemy import create_engine
    import sys
    from sqlalchemy.orm import sessionmaker, aliased
    from model_state import Base, State

    engine = create_engine("mysql://{}:{}@localhost/{}"
                           .format(sys.argv[1], sys.argv[2], sys.argv[3]))
    Session = sessionmaker(bind=engine)
    session = Session()

    newState = State(name='Louisiana')
    session.add(newState)
    session.commit()
    session.refresh(newState)
    print("{}".format(newState.id))


# -*- coding : utf-8 -*-
# Time      : 2021/4/2 22:14
# Author    : MurphyHou
# Proj_Name : cv_capture
# File_Name : keyboard.py
# Software  : PyCharm
# Here We Go!

from pynput.keyboard import Key, Listener
keyboard = Controller()  #实例化键盘对象

def on_press(key):  # 按键被按压调用这个函数
    # 输出按压的按键名字
    print('{0} pressed'.format(key))

def on_release(key):  # 按键被松开调用这个函数
    # 输出松开的按键名字
    print('{0} release'.format(key))
    if key == Key.esc:  # 如果按了Esc键就停止监听
        return False  # Stop listener

# 连接事件以及释放
with Listener(on_press=on_press, on_release=on_release) as listener:
    listener.join()


import pandas as pd
import numpy as np
import logging
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import StandardScaler, FunctionTransformer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.feature_extraction import DictVectorizer
from sklearn.decomposition import PCA
from scipy.stats import pearsonr
from sklearn.preprocessing import OneHotEncoder

logging.basicConfig(level=logging.INFO)

class DateTransformer(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        logging.info(f"Entered Date Columns: DateTransformer")
        X = X.copy()
        for col in X.columns:
            X[col] = (pd.Timestamp.now() - pd.to_datetime(X[col])).dt.days
        return X

    def get_feature_names_out(self, input_features=None):
        return input_features

class ExtremeValuesNumericalHandler(BaseEstimator, TransformerMixin):
    def __init__(self):
        self.lower_bound = None
        self.upper_bound = None

    def fit(self, X, y=None):
        logging.info(f"Entered Numerical Columns: ExtremeValuesNumericalHandler")
        X = pd.DataFrame(X)
        self.lower_bound = X.quantile(0.05)
        self.upper_bound = X.quantile(0.95)
        return self

    def transform(self, X):
        X = pd.DataFrame(X)
        for col in X.columns:
            X[col] = np.where(X[col] < self.lower_bound[col], self.lower_bound[col], X[col])
            X[col] = np.where(X[col] > self.upper_bound[col], self.upper_bound[col], X[col])
        return X.values

    def get_feature_names_out(self, input_features=None):
        return input_features

class DummyVariableCreator(BaseEstimator, TransformerMixin):
    def __init__(self, max_unique_values_cat=50):
        self.max_unique_values_cat = max_unique_values_cat

    def fit(self, X, y=None):
        X = pd.DataFrame(X)
        self.columns_to_encode = [col for col in X.columns if X[col].nunique() <= self.max_unique_values_cat]
        self.columns_to_drop = [col for col in X.columns if X[col].nunique() > self.max_unique_values_cat]
        return self

    def transform(self, X, y=None):
        logging.info(f"Entered Categorical Columns: DummyVariableCreator")
        X = pd.DataFrame(X)
        X = X.drop(columns=self.columns_to_drop)
        X = pd.get_dummies(X, columns=self.columns_to_encode, dummy_na=True, drop_first=True)
        self.feature_names_out_ = X.columns.tolist()  # get the output feature names after transformation
        return X

    def get_feature_names_out(self, input_features=None):
        return self.feature_names_out_

# class CleanFeatureNamesForXgboost(BaseEstimator, TransformerMixin):
#     def __init__(self):
#         self.lower_bound = None
#         self.upper_bound = None

#     def fit(self, X, y=None):
#         return self

#     def transform(self, X):
#         logging.info(f"Entered CleanFeatureNamesForXgboost")
#         X = pd.DataFrame(X)
#         X.columns = X.columns.astype(str)\
#                  .str.replace('[', '_replace_bracket_open_', regex=True)\
#                  .str.replace(']', '_replace_bracket_close_', regex=True)\
#                  .str.replace('<', '_smaller_than_', regex=True)
#         return X

#     def get_feature_names_out(self, input_features=None):
#         return input_features

# class LoggingPipeline(Pipeline):
#     def fit(self, X, y=None, **fit_params):
#         message = X.shape
#         for name, step in self.steps[:-1]:
#             print(f'entered {name}')
#             X = step[1].fit_transform(X, y, **fit_params)
#             logging.info(f"Finished {name}, Input shape: {message}, Output shape: {X.shape}")
#             message = X.shape
#         self.steps[-1][1].fit(X, y, **fit_params)
#         logging.info(f"Finished {self.steps[-1][0]}, Input shape: {message}, Output shape: {X.shape}")
#         return self

def preprocessing_pipeline(date_columns, numerical_columns, categorical_columns):

    # Define the pipeline steps
    date_pipeline = Pipeline([
        ('to_days', DateTransformer()),
        # ('clean_feature_names_for_date_cols', CleanFeatureNamesForXgboost())
])

    numerical_pipeline = Pipeline([
        ('imputer', SimpleImputer(strategy='median')),
        ('winsorizer', ExtremeValuesNumericalHandler()),
        ('scaler', StandardScaler()),
        # ('clean_feature_names_for_num_cols', CleanFeatureNamesForXgboost())
    ])

    categorical_pipeline = Pipeline([
        ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
        # ('onehot', OneHotEncoder(handle_unknown='ignore'))
        ('onehot_with_limit', DummyVariableCreator()),
        # ('clean_feature_names_for_cat_cols', CleanFeatureNamesForXgboost())
    ])

    # Combine the pipelines
    full_pipeline = ColumnTransformer(transformers=[
        ('date', date_pipeline, date_columns),
        ('num', numerical_pipeline, numerical_columns),
        ('cat', categorical_pipeline, categorical_columns)
    ])

    return full_pipeline


from typing import Dict
from azure.storage.blob import BlobClient, ContainerClient
import json
import os
import hashlib

class ModelStore(object):
    def __init__(
        self, connection_string: str, container_name: str, cache_dir: str
    ) -> None:
        self.connection_string = connection_string
        self.container_name = container_name
        self.cache_dir = cache_dir

    def _get_cache_path(self, blob_name: str, etag:str) -> str:
        return os.path.join(
            self.cache_dir, hashlib.md5((blob_name + etag).encode("utf-8")).hexdigest()
        )

    def _download_blob_to_cache(self, blob_name: str) -> str:
        blob_client = BlobClient.from_connection_string(
            conn_str=self.connection_string,
            container_name=self.container_name,
            blob_name=blob_name,
        )
        cached_path = self._get_cache_path(blob_name, blob_client.get_blob_properties().etag)
        if os.path.exists(cached_path):
            return cached_path
        if not os.path.exists(self.cache_dir):
            os.makedirs(self.cache_dir)
        blob_data = blob_client.download_blob()
        with open(cached_path, "wb") as cached_file:
            cached_file.write(blob_data.readall())
        return cached_path

    def publish(
        self,
        path_to_checkpoint: str,
        path_to_onnx: str,
        model_type: str,
        model_subtype: str,
        model_metadata: Dict[str, str],
    ) -> None:
        container_client = ContainerClient.from_connection_string(
            conn_str=self.connection_string, container_name=self.container_name
        )
        with open(path_to_checkpoint, "rb") as checkpoint_file:
            container_client.upload_blob(
                model_type + "/" + model_subtype + "/checkpoint.pt",
                checkpoint_file.read(),
                overwrite=True,
            )
        if path_to_onnx:
            with open(path_to_onnx, "rb") as checkpoint_file:
                container_client.upload_blob(
                    model_type + "/" + model_subtype + "/model.onnx",
                    checkpoint_file.read(),
                    overwrite=True,
                )
        container_client.upload_blob(
            model_type + "/" + model_subtype + "/metadata.json",
            json.dumps(model_metadata, indent=4),
            overwrite=True,
        )

    def get_checkpoint(self, model_type: str, model_subtype: str) -> str:
        blob_name = model_type + "/" + model_subtype + "/checkpoint.pt"
        return self._download_blob_to_cache(blob_name)


'''
Created on Nov 24, 2018

@author: joerg
'''
from interface import Corpus
import itertools

rhymes = []
# temp = '1231'
# with open('all_rhyme.pgold', 'r') as file:
#     for line in file:
#         if line == temp:
#             continue
#         else:
#             rhymes.append(line)
#             temp = line
count = 0
overall = 0
with open('all_rhyme.pgold', 'r') as file:
    for line in file:
        if not line:
            continue
        count+= 1
        overall += 1
        if not count == 4:
            if overall < 19000:
                rhymes.append(line.strip())
        else:
            count = 0

rhymes  = [line for line in rhymes if not line == '']

rhymes = [l.split() for l in rhymes]
alphas = []
digits = []

for i in range(len(rhymes)):
    if i %2 == 0:
        rhymes[i] = rhymes[i][1:]
        alphas.append(rhymes[i])
    else:
        digits.append(rhymes[i])

zips = []
for i in range(len(digits)):
    zips.append(list(zip(alphas[i], digits[i])))
train_data = []
for l in zips:
    for a,b in itertools.combinations(l,2):
        if a[1] == b[1]:
            train_data.append((a[0]+'\t'+b[0]+'\t'+'y'))
#         else:
#             train_data.append((a[0]+'\t'+b[0]+'\t'+'n'))

with open('/home/joerg/workspace/deep-siamese-text-similarity/train_data.tsv', 'w') as file:
    for line in train_data:
        file.write('%s\n' % line)

# print(alphas[:10])
# print(digits[:10])