from sklearn.metrics import (
    f1_score,
    log_loss,
    roc_auc_score,
    precision_score,
    recall_score,
    accuracy_score,
)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import os

def print_eval_metrics(y_true, y_pred, y_pred_prob=None):
    """
    print classification evaluation metrics
    :return: None
    """

    print("f1:", f1_score(y_true, y_pred))
    print("precision:", precision_score(y_true, y_pred))
    print("recall:", recall_score(y_true, y_pred))
    print("accuracy", accuracy_score(y_true, y_pred))
    if y_pred_prob:
        print("log loss:", log_loss(y_true, y_pred_prob))
        print("roc auc:", roc_auc_score(y_true, y_pred_prob))

def read_data(
    nrows=None,
    data_folder="data",
    read_products=False,
    read_orders=False,
    read_prior=False,
    read_train=False,
    read_test=False,
):
    if read_products:
        aisles = pd.read_csv("{}/aisles.csv".format(data_folder))
        departments = pd.read_csv("{}/departments.csv".format(data_folder))
        products = pd.read_csv("{}/products.csv".format(data_folder))

        # merge products with departments and aisles data
        products = products.merge(departments, how="left").merge(aisles, how="left")

        return products

    if read_orders:
        orders = pd.read_csv(
            "{}/orders.csv".format(data_folder),
            dtype={
                "order_id": np.int32,
                "user_id": np.int64,
                "eval_set": "category",
                "order_number": np.int16,
                "order_dow": np.int8,
                "order_hour_of_day": np.int8,
                "days_since_prior_order": np.float32,
            },
            nrows=nrows,
        )
        return orders

    if read_prior:
        prior = pd.read_csv(
            "{}/order_products__prior.csv".format(data_folder),
            dtype={
                "order_id": np.int32,
                "product_id": np.uint16,
                "add_to_cart_order": np.int16,
                "reordered": np.int8,
            },
            nrows=nrows,
        )
        return prior

    if read_train:
        train = pd.read_csv(
            "{}/order_products__train.csv".format(data_folder),
            dtype={
                "order_id": np.int32,
                "product_id": np.uint16,
                "add_to_cart_order": np.int16,
                "reordered": np.int8,
            },
            nrows=nrows,
        )

        return train

    if read_test:
        test = pd.read_csv(
            "{}/order_products__test.csv".format(data_folder),
            dtype={
                "order_id": np.int32,
                "product_id": np.uint16,
                "add_to_cart_order": np.int16,
                "reordered": np.int8,
            },
            nrows=nrows,
        )

        return test

def improve_data_type(data):
    import sys

    starting_size = sys.getsizeof(data)
    i = 0
    for c, dtype in zip(data.columns, data.dtypes):
        if "int" in str(dtype):
            print(c, dtype)
            if min(data[c]) >= 0:
                max_int = max(data[c])
                if max_int <= 255:
                    data[c] = data[c].astype(np.uint8)
                elif max_int <= 65535:
                    data[c] = data[c].astype(np.uint16)
                elif max_int <= 4294967295:
                    data[c] = data[c].astype(np.uint32)
                i += 1
    print("Number of columns adjusted: {}\n".format(i))
    ## Changing known reordered col to smaller int size
    if "reordered" in data.columns:
        data["reordered"] = np.nan_to_num(data["reordered"]).astype(np.uint8)

    # data['reordered'][data['reordered'] == 0] = np.nan
    print(
        "Reduced size {:.2%}".format(float(sys.getsizeof(data)) / float(starting_size))
    )
    return data

def split_data(data_full_features, test_size, data_folder="data", split_by=None):
    from sklearn.model_selection import train_test_split

    if split_by:
        unique_dimension = data_full_features[split_by].drop_duplicates()
        selected_test = unique_dimension.sample(frac=test_size, random_state=0)
        train, test = (
            data_full_features.loc[~data_full_features[split_by].isin(selected_test)],
            data_full_features.loc[data_full_features[split_by].isin(selected_test)],
        )
        X_train, y_train, X_test, y_test = (
            train.drop("reordered", axis=1),
            train["reordered"],
            test.drop("reordered", axis=1),
            test["reordered"],
        )
    else:
        y = data_full_features["reordered"]
        X = data_full_features.drop("reordered", axis=1)
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=test_size, random_state=0
        )

    return X_train, X_test, y_train, y_test

def plot_precision_recall_curve(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="precision")
    plt.plot(thresholds, recalls[:-1], "g--", label="recall")
    plt.title("precision_recall_curve")
    plt.show()

def save_fig_with_timestamp(
    file_name, fig_extension="png", resolution=300, folder="data"
):
    current_timestamp = time.strftime("%Y-%m-%d--%H-%M", time.localtime())
    path = os.path.join(
        folder, file_name + "--" + current_timestamp + "." + fig_extension
    )
    plt.savefig(path, format=fig_extension, dpi=resolution)

def p20(x):
    return x.quantile(0.2)

# 90th Percentile
def p80(x):
    return x.quantile(0.8)

def max_no_outliers(x):
    q3, q1 = x.quantile(0.75), x.quantile(0.25)
    return q3 + 1.5 * (q3 - q1)

def min_no_outliers(x):
    q3, q1 = x.quantile(0.75), x.quantile(0.25)
    return np.max([0, q1 - 1.5 * (q3 - q1)])

def trend_d1(y):
    from numpy import polyfit

    x = np.arange(1, len(y) + 1)
    if len(y) == 1:
        return np.nan
    # the coefficient
    else:
        return polyfit(x, y, 1)[0]

def skewness(y):
    from scipy.stats import skew

    if len(y) == 1:
        return np.nan
    else:
        return skew(y, axis=0, bias=False)

def compress(df, key):
    """
    compress df on key level
    """
    cols = [col for col in df.columns if "_id" not in col]
    grp = df.groupby(key)
    _df = df[[key]].drop_duplicates().set_index(key)
    for c in cols:
        tmp = grp.agg({c: ["mean", "median", "min", "max", "std"]})
        tmp.columns = [c + "-mean", c + "-median", c + "-min", c + "-max", c + "-std"]
        _df = pd.concat([_df, tmp], axis=1)

    # drop cols with 0 variance
    col_var = _df.var()
    col = col_var[col_var == 0].index
    _df.drop(col, axis=1, inplace=True)
    return _df.reset_index()

def download_user_order_history(uid, pid, label=""):
    from utils import read_data

    df = pd.read_pickle("data/prior_train_details.pickle")
    df = df.loc[df.user_id == uid]
    products = pd.read_pickle("data/products.pickle")
    df = df.merge(
        products[["product_id", "product_name", "aisle", "department"]], how="left"
    )
    df.to_csv(
        "data/{}_user_{}_order_products_history.csv".format(label, uid), index=False
    )
    if pid:
        product_name = products.loc[
            products["product_id"] == pid, "product_name"
        ].values[0]
        print(uid, product_name, " purchase history")
    df = read_data(read_orders=True)
    df = df.loc[df.user_id == uid]
    df.to_csv("data/{}_user_{}_order_history.csv".format(label, uid), index=False)

    print("user {} order history downloaded in data folder".format(uid))

def locate_by_uid_pid(df, uid=None, pid=None):
    if uid and pid:
        res = df.loc[(df.user_id == uid) & (df.product_id == pid)]
        print(res)
    if uid and not pid:
        res = df.loc[df.user_id == uid]
        print(res)

def feature_selection(df):
    # threshold corr >= 0.98
    high_corr_features = [
        "up_cart_order_median",
        "up_reorder_times",
        "up_purchase_ratio_r5",
        "up_purchase_interval_days_median",
        "up_order_interval_skewness_r5",
        "user_reorder_prod_total",
        "user_reordered_products_per_order",
        "p_sum_reordered",
        "p_sum_secondtime_purchase",
        "p_purchase_interval_days_max_woo",
        "up_overdue_days_diff_median_vs_others",
        "up_overdue_days_diff_p20_vs_others",
        "up_overdue_days_diff_p80_vs_others",
        "up_overdue_days_diff_min_vs_others",
        "up_overdue_orders_diff_mean_vs_others",
        "up_overdue_orders_diff_median_vs_others",
        "up_overdue_orders_diff_min_vs_others",
        "up_overdue_orders_percent_diff_range_vs_others",
    ]

    rfe_dropped = [
        "p_num_purchases_per_user_min",
        "up_purchases_pod_midnight",
        "p_num_purchases_per_user_p20",
        "p_is_gluten_free",
        "p_is_asian",
        "up_organic_substitute_num_purchases_r5",
        "p_is_organic",
        "p_num_products_sim_score_0.9_1.0",
        "up_norm_purchases_pod_midnight",
        "p_num_purchases_per_user_median",
        "up_organic_substitute_num_purchases",
        "up_purchases_dow_5",
        "up_purchases_dow_1",
        "p_purchase_interval_days_min_woo",
        "up_word2vec_substitute_num_purchases_r5",
        "users_purchases_dow_4",
        "up_purchases_dow_2",
        "users_asian_food_ratio",
        "up_purchases_dow_4",
        "up_purchases_pod_night",
        "up_norm_purchases_dow_1",
        "users_norm_purchases_dow_6",
        "up_purchases_dow_3",
        "up_purchases_pod_morning",
        "users_purchases_dow_5",
        "p_embedding_23",
        "users_asian_food_ratio_r5",
        "users_norm_purchases_pod_night",
        "up_norm_purchases_pod_morning",
        "users_norm_purchases_dow_5",
        "users_norm_purchases_dow_0",
        "up_norm_purchases_dow_6",
        "up_norm_purchases_dow_4",
        "up_purchases_pod_noon",
        "users_norm_purchases_pod_morning",
        "p_num_products_sim_score_0.8_0.9",
        "users_norm_purchases_dow_2",
        "p_embedding_38",
        "p_embedding_24",
        "p_num_products_sim_score_0.7_0.8",
        "user_product_total",
        "up_norm_purchases_dow_3",
        "up_norm_purchases_dow_2",
        "users_purchases_dow_0",
        "users_purchases_dow_2",
        "users_purchases_pod_night",
        "users_gluten_free_ratio_r5",
        "users_norm_purchases_dow_1",
        "p_embedding_7",
        "users_purchases_pod_midnight",
        "up_norm_purchases_dow_0",
        "p_embedding_30",
        "p_embedding_34",
        "p_embedding_32",
        "users_norm_purchases_pod_noon",
        "users_norm_purchases_dow_4",
        "up_norm_purchases_pod_noon",
        "p_embedding_10",
        "p_num_products_sim_score_0.6_0.7",
        "up_word2vec_substitute_num_purchases",
        "users_norm_purchases_dow_3",
        "users_norm_purchases_pod_midnight",
        "p_max_similarity_score",
        "p_embedding_15",
        "p_embedding_1",
        "p_embedding_29",
        "user_department_unique",
        "up_norm_purchases_dow_5",
        "users_purchases_dow_1",
        "p_embedding_4",
        "p_embedding_26",
        "user_std_order_dow",
        "p_embedding_11",
        "user_std_order_hour",
        "users_purchases_pod_noon",
        "user_mean_order_dow",
        "users_gluten_free_ratio",
        "p_embedding_37",
        "p_embedding_19",
        "users_purchases_dow_3",
        "p_embedding_33",
        "user_mean_order_hour",
        "users_purchases_dow_6",
        "user_basket_size_skew",
        "p_embedding_16",
        "p_embedding_28",
        "up_cart_order_std",
        "p_embedding_18",
        "p_embedding_5",
        "p_embedding_35",
        "p_embedding_39",
        "p_embedding_2",
        "p_embedding_36",
        "p_embedding_27",
        "uo_unique_aisle_std",
        "p_avg_first_order_num",
        "p_embedding_31",
    ]
    df.drop(high_corr_features, axis=1, inplace=True)
    df.drop(rfe_dropped, axis=1, inplace=True)
    return df


# -*- coding: utf-8 -*-

import math
import numpy
import seaborn as sns
from NormalDistribution.Distributions import Distribution

class Normal(Distribution):
    def __init__(self, mean, std):
        Distribution.__init__(self, mean, std)

    def calculate_mean(self):
        ## This function takes in the data list and return the mean of the data

        mean_list = sum(self.data)/len(self.data)
        self.mean = mean_list

        return self.mean

    def calculate_std(self, sample=True):
        ## This function takes in the data list and return the standard deviation of the data

        if sample:
            n = len(self.data)-1
        if not sample:
            n = len(self.data)
        sum_data = 0

        for data_point in self.data:
            sum_data += (data_point - (self.calculate_mean()))**2

        sum_data = sum_data/n
        sigma = math.sqrt(sum_data)
        self.stdev = sigma

        return self.stdev

    def distribution_plot(self):
        ## This function plot the normal distribution plot for the data

        ax = sns.distplot(self.data)
        ax.set(xlabel="data_list", ylabel="data_count")
        ax.set_title("Normal Distribution")

    def normal_distribution(self, x):
        ## This function takes in a data point and return the Gussian distribution value for it

        return ((1/math.sqrt(2*math.pi*(self.stdev **2))) * math.exp(-0.5*((x-self.mean) / self.stdev)**2))

    def add_distribution(self, other):
        ## Other: Normal instance
        ## This function combine the mean and standard deviation for two Normal(Gussian) distributions

        result = Normal()

        result.mean = self.mean + other.mean
        result.std = math.sqrt(self.stdev**2 + other.stdev**2)

        return result

    def total_distribution(self):
        ## This function return the mean and the standard deviation of the combined Gussian distribution

        return "mean {}, standard deviation {}".format(self.mean, self.stdev)


n = int(input())
t = []
x = []
y = []
for i in range(n):
    tt, xx, yy = map(int, input().split())
    t.append(tt)
    x.append(xx)
    y.append(yy)
t0 = 0
x0 = 0
y0 = 0
for i in range(n):
    tt = t[i] - t0
    xx = abs(x[i] - x0)
    yy = abs(y[i] - y0)
    temp = tt - (xx + yy)
    if temp >= 0 and temp % 2 == 0:
        t0 = t[i]
        x0 = x[i]
        y0 = y[i]
    else:
        print('No')
        exit(0)
print('Yes')


graph = [
    [],
    [2, 3, 8],
    [1, 7],
    [1, 4, 5],
    [3, 5],
    [3, 4],
    [7],
    [2, 6, 8],
    [1, 7]
]

visited = [False] * 9

def dfs(graph, node, visited):
    visited[node] = True
    print(node, end=' ')

    for adjacent_node in graph[node]:
        if not visited[adjacent_node]:
            dfs(graph, adjacent_node, visited)

dfs(graph, 1, visited)


import matplotlib
matplotlib.use('Qt5Agg')
from matplotlib.figure import Figure
import copy
import numpy as np
#import sys
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
import plotly.graph_objects as go
import scipy.ndimage as ndimage
#from afm_analyze import JPKAnalyze
#from plotly_viewer import PlotlyViewer
from plot2widget import PlotWidget

class AFMPlot:

    def __init__(self, jpk_anal=None, output_path=None):

        self.plotwin = None
        #mapping plot types (from ANALYSIS_MODE_DICT) to functions
        PLOT_DICT = {'2d': self.plot_2d,
                     '3d': self.plot_3d,
                     'line': self.plot_line}
        self.CLICK_STATUS = False #True when clicked first and plot doesn't exist

        if jpk_anal != None:
            self.jpk_anal = jpk_anal
            self.file_path = jpk_anal.file_path
            #plot data
            for mode in jpk_anal.df.keys():
                plot_params =  jpk_anal.anal_dict[mode]['plot_parameters']
                for plot_type in plot_params['type']:
                    fig = PLOT_DICT[plot_type](jpk_anal.df[mode], plot_params,
                                         file_path=output_path,
                                         points=plot_params['points_flag'],
                                        return_fig=True)
                    jpk_anal.anal_dict['Misc']['figure_list'].append(fig)

            #plt.show(block=False)
##        plt.pause(0.05)
##        if self.plotwin != None:
####            self.plotwin.show()
##            self.plotwin.app.exec_()

    def plot_2d(self, df, plot_params, file_path=None, points=False,return_fig=False):
        x = plot_params['x']
        y = plot_params['y']
        z = plot_params['z']
        title = plot_params['title']
        self.points_data = np.empty((0, 3), float)
        #organize data into matrix for heatmap plot
        df_data = df.pivot_table(values=z, index=y, columns=x,
                                 aggfunc='first')

        #plot
        plt.rcParams.update(plt.rcParamsDefault)
        fig2d = plt.figure(f'{title}')
        ax2d = fig2d.add_subplot(111)
        im2d = ax2d.pcolormesh(df_data.columns, df_data.index,
                               df_data, cmap='afmhot')
        ax2d.ticklabel_format(style='sci', scilimits=(0,0))
        ax2d.set_xlabel(x)
        ax2d.set_ylabel(y)
        fig2d.colorbar(im2d, ax=ax2d, label=z, format='%.1e')
        fig2d.suptitle(title)

        fig2d.savefig(f'{file_path}/{title}.png', bbox_inches = 'tight',
                      transparent = True)

        canvas = fig2d.canvas

##        rgb_data = np.fromstring(canvas.tostring_rgb(),
##                                 dtype=np.uint8, sep='')
##        rgb_data = rgb_data.reshape(canvas.get_width_height()[::-1] + (3,))

        if points == False:
            df_reference = df.pivot_table(values='Segment folder', index=y,
                                          columns=x, aggfunc='first')
            self.cid = fig2d.canvas.mpl_connect('button_press_event',
                                           lambda event: self.on_click(event,
                                                                       df_reference))
        else:
            self.cid = fig2d.canvas.mpl_connect('button_press_event',
                                           lambda event: self.on_click(event,
                                                                       df_data, points))

        #BUG: program doesn't end after callbacks
        fig2d.canvas.mpl_connect('close_event',
                                 lambda event: self.on_figclose(event, fig2d))
        plt.show(block=points)

        if return_fig == True:
            return fig2d

    def plot_3d(self, df, plot_params, file_path=None, points=False, return_fig=False):
        x = plot_params['x']
        y = plot_params['y']
        z = plot_params['z']
        #organize data into matrix for 3d plot
        df_data = df.pivot_table(values=z, index=y, columns=x,
                                 aggfunc='first')
        df_reference = df.pivot_table(values='Segment folder', index=y,
                                      columns=x, aggfunc='first')

        #plot
        fig = go.Figure(data=[go.Surface(z=df_data,
                                         x=df_data.columns,
                                         y=df_data.index)])
        fig.update_layout(title=plot_params['title'],
                          scene = dict(xaxis_title=x,
                                       yaxis_title=y,
                                       zaxis_title=z),
                          autosize=True)

        if return_fig == True:
            return fig
        #self.plotwin  = PlotlyViewer(fig)

##    def plot_3d(self, df, plot_params):
##        x = plot_params['x']
##        y = plot_params['y']
##        z = plot_params['z']
##
##        self.fig3d = plt.figure('3D Surface')
##        self.ax3d = self.fig3d.gca(projection='3d')
##        surf = self.ax3d.plot_trisurf(df[x], df[y], df[z])#, cmap='afmhot')
##        self.ax3d.ticklabel_format(style='sci', scilimits=(0,0))
##        self.ax3d.set_xlabel(x)
##        self.ax3d.set_ylabel(y)
##        self.ax3d.set_zlabel(z)
##        self.fig3d.suptitle(plot_params['title'])
##        ##fig3d.colorbar(surf, shrink=0.5, aspect=5)

    def plot_line(self, df, plot_params, label_text=None, file_path=None, points=False,
                  color=None, return_fig=False):
        x = plot_params['x']
        y = plot_params['y']
        style = plot_params['style']

        if self.CLICK_STATUS == False:
            self.init_fd_plot()
            self.CLICK_STATUS = True
            self.cursor_index = [0, -1]
        #functionality to make extend/retract colors same or different
        if color in df.columns:
            hue_param = color
            color_param = None
        elif color == None:
            hue_param = None
            color_param = None
        else:
            hue_param = None
            color_param = color

        sns.lineplot(x=x, y=y, style=style,hue=hue_param,
                     data=df, ax=self.ax_fd,
                     label = label_text, sort=False, ci=None,
                     color=color_param)
##        self.ax_fd.plot(df[x], df[y])
        self.ax_fd.ticklabel_format(style='sci', scilimits=(0,0))
        self.ax_fd.set_xlabel(x)
        self.ax_fd.set_ylabel(y)
        self.fig_fd.suptitle(plot_params['title'])

        if return_fig == True:
            return self.fig_fd

        #plt.show(block=False)
##        df.to_excel('test-fd-data.xlsx')

    def init_fd_plot(self): #initialize force-distance plot
        sns.set_theme(palette = 'husl')
        #self.sourceLabel = None
        #self.fig_fd = plt.figure('Line plot')
        self.fig_fd = Figure(figsize=(11, 5), dpi=100)
        self.ax_fd = self.fig_fd.add_subplot(111)
        self.fd_fit_line = self.ax_fd.plot([], [], 'k:') #fd fit line object
        #print('before')
##        self.fig_fd.canvas.mpl_connect('close_event',
##                                       lambda event: self.on_close(event))
        self.plotWidget = PlotWidget(fig = self.fig_fd,
                                         cursor1_init=0,
                                         cursor2_init=1,
                                         fixYLimits = False,
                                         method = self.updatePosition)
        self.plotWidget.wid.clicked_artist = None

    def updatePosition(self, trigger=False):

        # final_pos = tuple(self.plotWidget.wid.axes.transLimits.transform
        #                   ((self.plotWidget.wid.final_pos)))
##        if self.plotWidget.wid.clicked_artist == self.fitTextBox:
##            self.fit_pos = list(self.fitTextBox.get_position())
        # elif self.plotWidget.wid.clicked_artist == self.legend_main:
        #     pos = str(tuple(self.legend_main.get_window_extent()))
        #     self.configPlotWindow.plotDict['plot settings']['legend position'].setText(pos)
        if trigger == False:
            if self.plotWidget.wid.clicked_artist in [self.plotWidget.wid.cursor1,
                                                      self.plotWidget.wid.cursor2]:
                condition = True
            else:
                condition = False
        else:
            condition = True

        if condition == True:
##            if self.sourceLabel != None:
            xdata =  self.xAxisData
            ydata = self.yAxisData
            #print(xdata,ydata)
            x1 = self.plotWidget.wid.cursor1.get_xdata()[0]
            #x1_ind = np.searchsorted(xdata, x1)
            x1_ind = (np.abs(xdata-x1)).argmin()
            #print(xdata,x1,x1_ind,np.searchsorted(xdata, [x1]))
##            if len(self.sourceLabel.text().split(',')) == 2:
            if self.plotWidget.wid.cursor2 != None:
                x2 = self.plotWidget.wid.cursor2.get_xdata()[0]
                #x2_ind = np.searchsorted(xdata, [x2])[0]
                x2_ind = (np.abs(xdata-x2)).argmin()
                xstart = min(x1_ind, x2_ind)
                xend = max(x1_ind, x2_ind)
                xend = xend-1 if xend == len(xdata) else xend
##                    self.sourceLabel.setText(str(xstart) + ',' + str(xend))
            else:
                xstart = x1_ind-1 if x1_ind == len(xdata) else x1_ind
                xend = None
##                    self.sourceLabel.setText(str(xstart))
                # self.sourceLabel = None
            self.cursor_index = [xstart, xend]
            #print('cursors', xstart, xend)
            #print(x1,x1_ind,x2,x2_ind)
            #fitting
            fit_slice = slice(xstart,xend)
            #print("Fit slice", fit_slice)
            retract_fit = np.polyfit(xdata[fit_slice],
                                     ydata[fit_slice],self.fd_fit_order) #CHECK FIT ORDER
            fit_poly = np.poly1d(retract_fit)
            self.ax_fd.autoscale(enable=False)
            self.fd_fit_line[0].set_xdata(xdata)
            self.fd_fit_line[0].set_ydata(fit_poly(xdata))

#             x0 = min(xdata)
#             dydx = 2*retract_fit[0]*x0 + retract_fit[1]
#             y0 = fit_poly(x0)

#             force_sobel = ndimage.sobel(ydata) #sobel transform
#             idx_min = np.argmin(force_sobel)
#             snapin_distance = xdata[idx_min] - x0

#             intercept = (abs(y0-ydata[0])/dydx)

#             print(y0-ydata[0], dydx, intercept, snapin_distance, intercept/snapin_distance)
#             #print('polyfit x intercept',min(np.roots(retract_fit))-min(xdata))
#             print('cursor range', abs(xdata[xstart]-xdata[xend]))

    def on_close(self, event):
        self.CLICK_STATUS = False

    def on_figclose(self, event, fig):
        fig.canvas.mpl_disconnect(self.cid)
##        self.jpk_anal.data_zip.close() #CHECK THIS

    def on_click(self, event, df_ref, points=False):
        #print('click')
        x, y = event.xdata, event.ydata
        if x != None and y != None:
            #get segment path corresponding to clicked position
            col = sorted([[abs(a - x), a] for a in df_ref.columns],
                         key=lambda l:l[0])[0][1]
            ind = sorted([[abs(a - y), a] for a in df_ref.index],
                         key=lambda l:l[0])[0][1]

            if points == False:
                segment_path = df_ref[col][ind]
                print(segment_path, col, ind)
                #TODO: clean and organize this up
                mode = 'Force-distance'
                fd_data = self.jpk_anal
                segment_path_old = fd_data.segment_path
    ##            anal_dict_old = fd_data.anal_dict.copy()
                fd_data.clear_output(mode) #clear previous data
                fd_data.segment_path = segment_path
    ##            fd_data.anal_dict = fd_data.ANALYSIS_MODE_DICT[mode].copy()
    ##            print('old')
    ##            print(mode_dict_old)
                fd_data.get_data([mode])

                #fd_data = JPKAnalyze(self.file_path, mode, segment_path)
                plot_params = fd_data.ANALYSIS_MODE_DICT[mode]['plot_parameters']

                label_text = f'x={"{:.2e}".format(col)}, y={"{:.2e}".format(ind)}'
                self.plot_line(fd_data.df[mode], plot_params, label_text=label_text)

                #legend remove duplicates
                handles, labels = self.ax_fd.get_legend_handles_labels()
                leg_dict = dict(zip(labels[::-1],handles[::-1]))
                self.ax_fd.get_legend().remove()
                leg = self.ax_fd.legend(leg_dict.values(), leg_dict.keys())
                leg.set_draggable(True, use_blit=True)

                #CHECK
                fd_data.segment_path = None
    ##            fd_data.anal_dict = anal_dict_old

                self.fig_fd.show()
            else:
                z_value = df_ref[col][ind]
                self.points_data = np.append(self.points_data,
                                             np.array([[col,ind,z_value]]),
                                             axis=0)
                #print(col, ind, z_value)

    def plot_2dfit(self, fit_data, df_raw, plot_params, file_path=None):
        x = plot_params['x']
        y = plot_params['y']
        z = plot_params['z']
        z_raw = f'{z}_raw'
        z_fit = f'{z}_fit'

##        color_limits = [df[z_raw].min(), df[z_raw].max()]
##        title_text = ', '.join([f'{k}' + '={:.1e}'.format(v) \
##                                for k, v in fit_output.items()])
        title_text = 'Fitting result'

        #data reshape
##        df_raw = df.pivot_table(values=z_raw, index=y, columns=x,
##                                       aggfunc='first')
##        df_filtered = df.query(f'{z_fit}>={color_limits[0]}')
##        df_fit = fit_data.pivot_table(values=z_fit, index=y,
##                                           columns=x, aggfunc='first')

        #plot
        fig = go.Figure()
        fig.add_trace(go.Surface(legendgroup="Raw",
                                 name='Raw',
                                 z=df_raw,
                                 x=df_raw.columns,
                                 y=df_raw.index,
                                 opacity=0.7,
                                 colorscale ='Greens',
                                 reversescale=True,
                                 showlegend=True,
                                 showscale=False))
        #i = 0
        for key, val in fit_data.items():
##            print(key)
            #leg = True if i==0 else False
            fig.add_trace(go.Surface(#legendgroup="Fit",
                                     name=f'Fit-{key}',
                                     z=val[z],#df_fit,
                                     x=val[x],#df_fit.columns,
                                     y=val[y],#df_fit.index,
                                     opacity=0.7,
                                     colorscale ='Reds',
                                     reversescale=True,
                                     showlegend=True,#leg
                                     showscale=False))
            #i = 1
##        fig.update_traces(contours_z=dict(show=True, usecolormap=False,
##                                          highlightcolor="limegreen",
##                                          project_z=True))
        z_range = [df_raw.min().min(),1.2*df_raw.max().max()]
##        print(z_range)
        fig.update_layout(title=title_text,
                          scene = dict(xaxis_title=x,
                                       yaxis_title=y,
                                       zaxis_title=z,
                                       xaxis=dict(ticksuffix='m'),
                                       yaxis=dict(ticksuffix='m'),
                                       zaxis=dict(range=z_range,
                                                  ticksuffix='m'),
                                       aspectratio={"x": 1, "y": 1, "z": 0.4}
                                       ),
                          autosize=True)

        fig.write_html(f'{file_path}/3d_jumpin_distance.html')

        #self.plotwin  = PlotlyViewer(fig)

        return fig

def simul_plot1(simu_df):
    sns.set_context("talk")
    sns.set_style("ticks")
##    fig = plt.figure('Simulation data')
##
##    ax1 = fig.add_subplot(1,1,1)
    mk_num = len(simu_df['Top_Angle'].unique())
    g = sns.lmplot(x='Contact_Radius',y='Force_Calc',hue='Top_Angle',
                 #style='Top_Angle',
               data=simu_df,
                 #markers=['o']*mk_num,dashes=False,
                 legend='full',palette='flare',
               #ax=ax1,
               order=5, ci=None,
                   height=8, aspect=1.3)
    ax1 = g.ax
    ax1.axhline(y=0, color='0.8', dashes=(1, 1), zorder=0)

    ax1.set_title('Simulation data: Adhesion force')
    ax1.set_xlabel('Drop size, R/s')
    ax1.set_ylabel(r'$F/2\pi \gamma s$')
    leg = g.legend
    #leg = ax1.get_legend()
##    leg.remove()

##    ax2 = fig.add_subplot(1,2,2)
##    mk_num = len(simu_df['Top_Angle'].unique())
##    sns.lineplot(x='Contact_Radius',
##                 y='Average Wetted Height',hue='Top_Angle',
##                 style='Top_Angle',data=simu_df,
##                 markers=['o']*mk_num,dashes=False,
##                 legend='full',palette='flare', ax=ax2)
##
##    ax2.set_title('Wetted length')
##    ax2.set_xlabel('Drop size, R/s')
##    ax2.set_ylabel('w/s')
##    leg = ax2.get_legend()
    leg.set_title('Contact angle')
    fig = g.fig
    #plt.show(block=True)

    return fig

def simul_plot2(simu_df):
    sns.set_context("talk")
    sns.set_style("ticks")
    Rs = simu_df['Contact_Radius'].iloc[0]
##    fig = plt.figure(f'Simulation data FD Rs={Rs}')
##
##    ax1 = fig.add_subplot(1,1,1)
    mk_num = len(simu_df['Top_Angle'].unique())
    g = sns.lmplot(x='Height',y='Force_Calc',hue='Top_Angle',
                 #style='Top_Angle',
               data=simu_df,
                 #markers=['o']*mk_num,dashes=False,
                 legend='full',palette='flare',
               #ax=ax1,
               order=2, ci=None,
                   height=8, aspect=1.3)
    ax1 = g.ax
    ax1.axhline(y=0, color='0.8', dashes=(1, 1), zorder=0)

    ax1.set_title(f'Simulation data (FD): R/s={Rs}')
    ax1.set_xlabel('Height, h/s')
    ax1.set_ylabel(r'$F/2\pi \gamma s$')
    #leg = ax1.get_legend()
    leg = g.legend
##    leg.remove()
##
##    ax2 = fig.add_subplot(1,2,2)
##    mk_num = len(simu_df['Top_Angle'].unique())
##    sns.lineplot(x='Contact_Radius',
##                 y='Average Wetted Height',hue='Top_Angle',
##                 style='Top_Angle',data=simu_df,
##                 markers=['o']*mk_num,dashes=False,
##                 legend='full',palette='flare', ax=ax2)
##
##    ax2.set_title('Wetted length')
##    ax2.set_xlabel('Drop size, R/s')
##    ax2.set_ylabel('w/s')
##    leg = ax2.get_legend()
    leg.set_title('Contact angle')
    fig = g.fig
    #plt.show(block=True)

    return fig

def simul_plot3(simu_df):
    sns.set_context("talk")
    sns.set_style("ticks")
##    fig = plt.figure('Simulated contact angle')
##
##    ax1 = fig.add_subplot(1,1,1)
    mk_num = len(simu_df['Contact_Radius'].unique())
    g = sns.lmplot(x='Rupture_Distance',y='Top_Angle',hue='Contact_Radius',
                 #style='Contact_Radius',
                 data=simu_df,
                 #markers=['o']*mk_num,dashes=False,sort=False,
                 legend='full',palette='flare',
                 #ax=ax1,
                 order=3, ci=None)
    ax1 = g.ax
    ax1.set_title('Simulation data: rupture distance')
    ax1.set_xlabel('Rupture distance, r/s')
    ax1.set_ylabel('Contact angle')
    #leg = ax1.get_legend()
    leg = g.legend

    leg.set_title('Drop size, R/s')
    fig = g.fig
    #plt.show(block=True)

    return fig

##    def plot_2dfit(self, df, plot_params, fit_output):
##        x = plot_params['x']
##        y = plot_params['y']
##        z = plot_params['z']
##        z_raw = f'{z}_raw'
##        z_fit = f'{z}_fit'
##
##        color_limits = [df[z_raw].min(), df[z_raw].max()]
##
##        #fit data reshape
##        df_fit = df.pivot_table(values=z_fit, index=y, columns=x,
##                                aggfunc='first')
##        #plot
##        fig = plt.figure('Fit: 2D map')
##        ax_fit = fig.add_subplot(111)
##        im_fit = ax_fit.pcolormesh(df_fit.columns, df_fit.index,
##                                   df_fit, cmap='afmhot',
##                                   vmin=color_limits[0],
##                                   vmax=color_limits[1])
##        ax_fit.ticklabel_format(style='sci', scilimits=(0,0))
##        ax_fit.set_xlabel(x)
##        ax_fit.set_ylabel(y)
##        title_text = ', '.join([f'{k}' + '={:.1e}'.format(v) \
##                                for k, v in fit_output.items()])
##        ax_fit.set_title(title_text)
##        fig.colorbar(im_fit, ax=ax_fit, label=z)
##
##        fig.tight_layout()
##
##        #3d plot
##        z_lim = self.ax3d.get_zlim3d() #z limits of raw 3D
##        df_filtered = df.query(f'{z_fit}>={z_lim[0]}')
##        fig3d = plt.figure('Fit: 3D')
##        ax3d = fig3d.gca(projection='3d')
##        surf = ax3d.plot_trisurf(df_filtered[x],df_filtered[y],
##                                 df_filtered[z_fit])#, cmap='afmhot')
##        ax3d.ticklabel_format(style='sci', scilimits=(0,0))
##        ax3d.set_xlabel(x)
##        ax3d.set_ylabel(y)
##        ax3d.set_zlabel(z)
##        fig3d.suptitle(title_text)
##        ax3d.set_zlim3d(color_limits)
##        ax3d.set_xlim((df[x].min(), df[x].max()))
##        ax3d.set_ylim(df[y].min(), df[y].max())
##
##        plt.show(block=False)
##


import os,sys
# os.chdir(os.path.dirname(os.path.abspath(__file__)))
# sys.path.insert(1, os.path.join(sys.path[0], '..'))

import requests
import pandas as pd
import json
import random
import datetime,time
import logging
import re
import urllib.parse

mode ='local'

if(mode=='vpn'):
    def nsefetch(payload):
        if (("%26" in payload) or ("%20" in payload)):
            encoded_url = payload
        else:
            encoded_url = urllib.parse.quote(payload, safe=':/?&=')
        payload_var = 'curl -b cookies.txt "' + encoded_url + '"' + curl_headers + ''
        try:
            output = os.popen(payload_var).read()
            output=json.loads(output)
        except ValueError:  # includes simplejson.decoder.JSONDecodeError:
            payload2 = "https://www.nseindia.com"
            output2 = os.popen('curl -c cookies.txt "'+payload2+'"'+curl_headers+'').read()

            output = os.popen(payload_var).read()
            output=json.loads(output)
        return output
if(mode=='local'):
    def nsefetch(payload):
        try: #Try the normal way.
            output = requests.get(payload,headers=headers).json()
        except:
            try: # Follow Praty's hack as mentioned here at https://forum.unofficed.com/t/solution-equity-history-function-doesnt-work/1197
                output = requests.session().get(payload,headers=headers).json()

            except: #Mimick FrontPage Visit
                s=requests.session()

                payload2 = "https://www.nseindia.com"
                output2 = s.get(payload2,headers=headers).json()

                output = s.get(payload,headers=headers).json()
        return output

headers = {
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'DNT': '1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36',
    'Sec-Fetch-User': '?1',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-Mode': 'navigate',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en-US,en;q=0.9,hi;q=0.8',
}

#Curl headers
curl_headers = ''' -H "authority: beta.nseindia.com" -H "cache-control: max-age=0" -H "dnt: 1" -H "upgrade-insecure-requests: 1" -H "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" -H "sec-fetch-user: ?1" -H "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -H "sec-fetch-site: none" -H "sec-fetch-mode: navigate" -H "accept-encoding: gzip, deflate, br" -H "accept-language: en-US,en;q=0.9,hi;q=0.8" --compressed'''

run_time=datetime.datetime.now()

#Constants
indices = ['NIFTY','FINNIFTY','BANKNIFTY']

def running_status():
    start_now=datetime.datetime.now().replace(hour=9, minute=15, second=0, microsecond=0)
    end_now=datetime.datetime.now().replace(hour=15, minute=30, second=0, microsecond=0)
    return start_now<datetime.datetime.now()<end_now

#Getting FNO Symboles
def fnolist():
    # df = pd.read_csv("https://www1.nseindia.com/content/fo/fo_mktlots.csv")
    # return [x.strip(' ') for x in df.drop(df.index[3]).iloc[:,1].to_list()]

    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')

    nselist=['NIFTY','NIFTYIT','BANKNIFTY']

    i=0
    for x in range(i, len(positions['data'])):
        nselist=nselist+[positions['data'][x]['symbol']]

    return nselist

def nsesymbolpurify(symbol):
    symbol = symbol.replace('&','%26') #URL Parse for Stocks Like M&M Finance
    return symbol

def nse_optionchain_scrapper(symbol):
    symbol = nsesymbolpurify(symbol)
    if any(x in symbol for x in indices):
        payload = nsefetch('https://www.nseindia.com/api/option-chain-indices?symbol='+symbol)
    else:
        payload = nsefetch('https://www.nseindia.com/api/option-chain-equities?symbol='+symbol)
    return payload

def oi_chain_builder(symbol,expiry="latest",oi_mode="full"):

    payload = nse_optionchain_scrapper(symbol)

    if(oi_mode=='compact'):
        col_names = ['CALLS_OI','CALLS_Chng in OI','CALLS_Volume','CALLS_IV','CALLS_LTP','CALLS_Net Chng','Strike Price','PUTS_OI','PUTS_Chng in OI','PUTS_Volume','PUTS_IV','PUTS_LTP','PUTS_Net Chng']
    if(oi_mode=='full'):
        col_names = ['CALLS_Chart','CALLS_OI','CALLS_Chng in OI','CALLS_Volume','CALLS_IV','CALLS_LTP','CALLS_Net Chng','CALLS_Bid Qty','CALLS_Bid Price','CALLS_Ask Price','CALLS_Ask Qty','Strike Price','PUTS_Bid Qty','PUTS_Bid Price','PUTS_Ask Price','PUTS_Ask Qty','PUTS_Net Chng','PUTS_LTP','PUTS_IV','PUTS_Volume','PUTS_Chng in OI','PUTS_OI','PUTS_Chart']
    oi_data = pd.DataFrame(columns = col_names)

    #oi_row = {'CALLS_OI':0, 'CALLS_Chng in OI':0, 'CALLS_Volume':0, 'CALLS_IV':0, 'CALLS_LTP':0, 'CALLS_Net Chng':0, 'Strike Price':0, 'PUTS_OI':0, 'PUTS_Chng in OI':0, 'PUTS_Volume':0, 'PUTS_IV':0, 'PUTS_LTP':0, 'PUTS_Net Chng':0}
    oi_row = {'CALLS_OI':0, 'CALLS_Chng in OI':0, 'CALLS_Volume':0, 'CALLS_IV':0, 'CALLS_LTP':0, 'CALLS_Net Chng':0, 'CALLS_Bid Qty':0,'CALLS_Bid Price':0,'CALLS_Ask Price':0,'CALLS_Ask Qty':0,'Strike Price':0, 'PUTS_OI':0, 'PUTS_Chng in OI':0, 'PUTS_Volume':0, 'PUTS_IV':0, 'PUTS_LTP':0, 'PUTS_Net Chng':0,'PUTS_Bid Qty':0,'PUTS_Bid Price':0,'PUTS_Ask Price':0,'PUTS_Ask Qty':0}
    if(expiry=="latest"):
        expiry = payload['records']['expiryDates'][0]
    m=0
    for m in range(len(payload['records']['data'])):
        if(payload['records']['data'][m]['expiryDate']==expiry):
            if(1>0):
                try:
                    oi_row['CALLS_OI']=payload['records']['data'][m]['CE']['openInterest']
                    oi_row['CALLS_Chng in OI']=payload['records']['data'][m]['CE']['changeinOpenInterest']
                    oi_row['CALLS_Volume']=payload['records']['data'][m]['CE']['totalTradedVolume']
                    oi_row['CALLS_IV']=payload['records']['data'][m]['CE']['impliedVolatility']
                    oi_row['CALLS_LTP']=payload['records']['data'][m]['CE']['lastPrice']
                    oi_row['CALLS_Net Chng']=payload['records']['data'][m]['CE']['change']
                    if(oi_mode=='full'):
                        oi_row['CALLS_Bid Qty']=payload['records']['data'][m]['CE']['bidQty']
                        oi_row['CALLS_Bid Price']=payload['records']['data'][m]['CE']['bidprice']
                        oi_row['CALLS_Ask Price']=payload['records']['data'][m]['CE']['askPrice']
                        oi_row['CALLS_Ask Qty']=payload['records']['data'][m]['CE']['askQty']
                except KeyError:
                    oi_row['CALLS_OI'], oi_row['CALLS_Chng in OI'], oi_row['CALLS_Volume'], oi_row['CALLS_IV'], oi_row['CALLS_LTP'],oi_row['CALLS_Net Chng']=0,0,0,0,0,0
                    if(oi_mode=='full'):
                        oi_row['CALLS_Bid Qty'],oi_row['CALLS_Bid Price'],oi_row['CALLS_Ask Price'],oi_row['CALLS_Ask Qty']=0,0,0,0
                    pass

                oi_row['Strike Price']=payload['records']['data'][m]['strikePrice']

                try:
                    oi_row['PUTS_OI']=payload['records']['data'][m]['PE']['openInterest']
                    oi_row['PUTS_Chng in OI']=payload['records']['data'][m]['PE']['changeinOpenInterest']
                    oi_row['PUTS_Volume']=payload['records']['data'][m]['PE']['totalTradedVolume']
                    oi_row['PUTS_IV']=payload['records']['data'][m]['PE']['impliedVolatility']
                    oi_row['PUTS_LTP']=payload['records']['data'][m]['PE']['lastPrice']
                    oi_row['PUTS_Net Chng']=payload['records']['data'][m]['PE']['change']
                    if(oi_mode=='full'):
                        oi_row['PUTS_Bid Qty']=payload['records']['data'][m]['PE']['bidQty']
                        oi_row['PUTS_Bid Price']=payload['records']['data'][m]['PE']['bidprice']
                        oi_row['PUTS_Ask Price']=payload['records']['data'][m]['PE']['askPrice']
                        oi_row['PUTS_Ask Qty']=payload['records']['data'][m]['PE']['askQty']
                except KeyError:
                    oi_row['PUTS_OI'], oi_row['PUTS_Chng in OI'], oi_row['PUTS_Volume'], oi_row['PUTS_IV'], oi_row['PUTS_LTP'],oi_row['PUTS_Net Chng']=0,0,0,0,0,0
                    if(oi_mode=='full'):
                        oi_row['PUTS_Bid Qty'],oi_row['PUTS_Bid Price'],oi_row['PUTS_Ask Price'],oi_row['PUTS_Ask Qty']=0,0,0,0
            else:
                logging.info(m)

            if(oi_mode=='full'):
                oi_row['CALLS_Chart'],oi_row['PUTS_Chart']=0,0
            #oi_data = oi_data.append(oi_row, ignore_index=True)
            #oi_data = pd.concat([oi_data, oi_row], ignore_index=True)
            oi_data = pd.concat([oi_data, pd.DataFrame([oi_row])], ignore_index=True)

            oi_data['time_stamp']=payload['records']['timestamp']
    return oi_data,float(payload['records']['underlyingValue']),payload['records']['timestamp']

def nse_quote(symbol,section=""):
    #https://forum.unofficed.com/t/nsetools-get-quote-is-not-fetching-delivery-data-and-delivery-can-you-include-this-as-part-of-feature-request/1115/4
    symbol = nsesymbolpurify(symbol)

    if(section==""):
        if any(x in symbol for x in fnolist()):
            payload = nsefetch('https://www.nseindia.com/api/quote-derivative?symbol='+symbol)
        else:
            payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol)
        return payload

    if(section!=""):
        payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol+'&section='+section)
        return payload

def nse_expirydetails(payload,i=0): #Can make problem. Use nse_expirydetails_by_symbol()

    expiry_dates = payload['records']['expiryDates']
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]
    currentExpiry=expiry_dates[i]
    currentExpiry = datetime.datetime.strptime(currentExpiry,'%d-%b-%Y').date()  # converting json datetime to alice datetime
    date_today = run_time.strftime('%Y-%m-%d')  # required to remove hh:mm:ss
    date_today = datetime.datetime.strptime(date_today,'%Y-%m-%d').date()
    dte = (currentExpiry - date_today).days
    return currentExpiry,dte

def pcr(payload,inp='0'):
    ce_oi = 0
    pe_oi = 0
    for i in payload['records']['data']:
        if i['expiryDate'] == payload['records']['expiryDates'][inp]:
            try:
                ce_oi += i['CE']['openInterest']
                pe_oi += i['PE']['openInterest']
            except KeyError:
                pass
    return pe_oi / ce_oi

#forum.unofficed.com/t/unable-to-find-nse-quote-meta-api/702/4
#Refer https://forum.unofficed.com/t/changed-the-nse-quote-ltp-function/1276
def nse_quote_ltp(symbol,expiryDate="latest",optionType="-",strikePrice=0):
  payload = nse_quote(symbol)

  meta = "Options"
  if(optionType=="Fut"): meta = "Futures"
  if(optionType=="PE"):optionType="Put"
  if(optionType=="CE"):optionType="Call"

  if(expiryDate=="latest") or (expiryDate=="next"):

    if(meta=="Futures"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "futures" in key.lower()), None)
    if(meta=="Options"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "options" in key.lower()), None)

    expiry_dates=payload["expiryDatesByInstrument"][selected_key]
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]
    if(expiryDate=="latest"): expiryDate=expiry_dates[0]
    if(expiryDate=="next"): expiryDate=expiry_dates[1]

  if(optionType!="-"):
      for i in payload['stocks']:
        if meta in i['metadata']['instrumentType']:
          #print(i['metadata'])
          if(optionType=="Fut"):
              if(i['metadata']['expiryDate']==expiryDate):
                lastPrice = i['metadata']['lastPrice']

          if((optionType=="Put")or(optionType=="Call")):
              if (i['metadata']["expiryDate"]==expiryDate):
                if (i['metadata']["optionType"]==optionType):
                  if (i['metadata']["strikePrice"]==strikePrice):
                    #print(i['metadata'])
                    lastPrice = i['metadata']['lastPrice']

  if(optionType=="-"):
      lastPrice = payload['underlyingValue']

  return lastPrice

# print(nse_quote_ltp("RELIANCE"))
# print(nse_quote_ltp("RELIANCE","latest","Fut"))
# print(nse_quote_ltp("RELIANCE","next","Fut"))
# print(nse_quote_ltp("BANKNIFTY","latest","PE",32000))
# print(nse_quote_ltp("BANKNIFTY","next","PE",32000))
# print(nse_quote_ltp("BANKNIFTY","10-Jun-2021","PE",32000))
# print(nse_quote_ltp("BANKNIFTY","17-Jun-2021","PE",32000))
# print(nse_quote_ltp("RELIANCE","latest","PE",2300))
# print(nse_quote_ltp("RELIANCE","next","PE",2300))

def nse_quote_meta(symbol,expiryDate="latest",optionType="-",strikePrice=0):
  payload = nse_quote(symbol)
  #https://stackoverflow.com/questions/7961363/removing-duplicates-in-lists
  #https://stackoverflow.com/questions/19199984/sort-a-list-in-python

  #BankNIFTY and NIFTY has weekly options. Using this Jugaad which has primary base of assumption that Reliance will not step out of FNO.
  #forum.unofficed.com/t/unable-to-find-nse-quote-meta-api/702/4
  if((symbol in indices) and (optionType=="Fut")):
    dates = expiry_list("RELIANCE","list")
    if(expiryDate=="latest"): expiryDate=dates[0]
    if(expiryDate=="next"): expiryDate=dates[1]

  if(expiryDate=="latest") or (expiryDate=="next"):
    dates=list(set((payload["expiryDates"])))
    dates.sort(key = lambda date: datetime.datetime.strptime(date, '%d-%b-%Y'))
    if(expiryDate=="latest"): expiryDate=dates[0]
    if(expiryDate=="next"): expiryDate=dates[1]

  meta = "Options"
  if(optionType=="Fut"): meta = "Futures"
  if(optionType=="PE"):optionType="Put"
  if(optionType=="CE"):optionType="Call"

  if(optionType!="-"):
      for i in payload['stocks']:
        if meta in i['metadata']['instrumentType']:
          #print(i['metadata'])
          if(optionType=="Fut"):
              if(i['metadata']['expiryDate']==expiryDate):
                metadata = i['metadata']

          if((optionType=="Put")or(optionType=="Call")):
              if (i['metadata']["expiryDate"]==expiryDate):
                if (i['metadata']["optionType"]==optionType):
                  if (i['metadata']["strikePrice"]==strikePrice):
                    #print(i['metadata'])
                    metadata = i['metadata']

  if(optionType=="-"):
      metadata = i['metadata']

  return metadata

def nse_optionchain_ltp(payload,strikePrice,optionType,inp=0,intent=""):
    expiry_dates = payload['records']['expiryDates']
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]
    expiryDate=expiry_dates[inp]
    for x in range(len(payload['records']['data'])):
      if((payload['records']['data'][x]['strikePrice']==strikePrice) & (payload['records']['data'][x]['expiryDate']==expiryDate)):
          if(intent==""): return payload['records']['data'][x][optionType]['lastPrice']
          if(intent=="sell"): return payload['records']['data'][x][optionType]['bidprice']
          if(intent=="buy"): return payload['records']['data'][x][optionType]['askPrice']

def nse_eq(symbol):
    symbol = nsesymbolpurify(symbol)
    try:
        payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol)
        try:
            if(payload['error']=={}):
                print("Please use nse_fno() function to reduce latency.")
                payload = nsefetch('https://www.nseindia.com/api/quote-derivative?symbol='+symbol)
        except:
            pass
    except KeyError:
        print("Getting Error While Fetching.")
    return payload

def nse_fno(symbol):
    symbol = nsesymbolpurify(symbol)
    try:
        payload = nsefetch('https://www.nseindia.com/api/quote-derivative?symbol='+symbol)
        try:
            if(payload['error']=={}):
                print("Please use nse_eq() function to reduce latency.")
                payload = nsefetch('https://www.nseindia.com/api/quote-equity?symbol='+symbol)
        except KeyError:
            pass
    except KeyError:
        print("Getting Error While Fetching.")
    return payload

def quote_equity(symbol):
    return nse_eq(symbol)

def quote_derivative(symbol):
    return nse_fno(symbol)

def option_chain(symbol):
    return nse_optionchain_scrapper(symbol)

def nse_holidays(type="trading"):
    if(type=="clearing"):
        payload = nsefetch('https://www.nseindia.com/api/holiday-master?type=clearing')
    if(type=="trading"):
        payload = nsefetch('https://www.nseindia.com/api/holiday-master?type=trading')
    return payload

def holiday_master(type="trading"):
    return nse_holidays(type)

def nse_results(index="equities",period="Quarterly"):
    if(index=="equities") or (index=="debt") or (index=="sme"):
        if(period=="Quarterly") or (period=="Annual")or (period=="Half-Yearly")or (period=="Others"):
            payload = nsefetch('https://www.nseindia.com/api/corporates-financial-results?index='+index+'&period='+period)
            return pd.json_normalize(payload)
        else:
            print("Give Correct Period Input")
    else:
        print("Give Correct Index Input")

def nse_events():
    output = nsefetch('https://www.nseindia.com/api/event-calendar')
    return pd.json_normalize(output)

def nse_past_results(symbol):
    symbol = nsesymbolpurify(symbol)
    return nsefetch('https://www.nseindia.com/api/results-comparision?symbol='+symbol)

def expiry_list(symbol,type="list"):
    logging.info("Getting Expiry List of: "+ symbol)

    if(type!="list"):
        payload = nse_optionchain_scrapper(symbol)
        payload = pd.DataFrame({'Date':payload['records']['expiryDates']})
        return payload

    if(type=="list"):
        payload = nse_quote(symbol)
        dates=list(set((payload["expiryDates"])))
        dates.sort(key = lambda date: datetime.datetime.strptime(date, '%d-%b-%Y'))
        return dates

def nse_custom_function_secfno(symbol,attribute="lastPrice"):
    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    endp = len(positions['data'])
    for x in range(0, endp):
        if(positions['data'][x]['symbol']==symbol.upper()):
            return positions['data'][x][attribute]

def nse_blockdeal():
    payload = nsefetch('https://nseindia.com/api/block-deal')
    return payload

def nse_marketStatus():
    payload = nsefetch('https://nseindia.com/api/marketStatus')
    return payload

def nse_circular(mode="latest"):
    if(mode=="latest"):
        payload = nsefetch('https://nseindia.com/api/latest-circular')
    else:
        payload = nsefetch('https://www.nseindia.com/api/circulars')
    return payload

def nse_fiidii(mode="pandas"):
    try:
        if(mode=="pandas"):
            return pd.DataFrame(nsefetch('https://www.nseindia.com/api/fiidiiTradeReact'))
        else:
            return nsefetch('https://www.nseindia.com/api/fiidiiTradeReact')
    except:
        logger.info("Pandas is not working for some reason.")
        return nsefetch('https://www.nseindia.com/api/fiidiiTradeReact')

def nsetools_get_quote(symbol):
    payload = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    for m in range(len(payload['data'])):
        if(payload['data'][m]['symbol']==symbol.upper()):
            return payload['data'][m]

def nse_index():
    payload = nsefetch('https://iislliveblob.niftyindices.com/jsonfiles/LiveIndicesWatch.json')
    payload = pd.DataFrame(payload["data"])
    return payload

def nse_get_index_list():
    payload = nsefetch('https://iislliveblob.niftyindices.com/jsonfiles/LiveIndicesWatch.json')
    payload = pd.DataFrame(payload["data"])
    return payload["indexName"].tolist()

def nse_get_index_quote(index):
    payload = nsefetch('https://iislliveblob.niftyindices.com/jsonfiles/LiveIndicesWatch.json')
    for m in range(len(payload['data'])):
        if(payload['data'][m]["indexName"] == index.upper()):
            return payload['data'][m]

def nse_get_advances_declines(mode="pandas"):
    try:
        if(mode=="pandas"):
            positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
            return pd.DataFrame(positions['data'])
        else:
            return nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    except:
        logger.info("Pandas is not working for some reason.")
        return nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')

def nse_get_top_losers():
    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    df = pd.DataFrame(positions['data'])
    df = df.sort_values(by="pChange")
    return df.head(5)

def nse_get_top_gainers():
    positions = nsefetch('https://www.nseindia.com/api/equity-stockIndices?index=SECURITIES%20IN%20F%26O')
    df = pd.DataFrame(positions['data'])
    df = df.sort_values(by="pChange" , ascending = False)
    return df.head(5)

def nse_get_fno_lot_sizes(symbol="all",mode="list"):
    url="https://archives.nseindia.com/content/fo/fo_mktlots.csv"

    if(mode=="list"):
        s=requests.get(url).text
        res_dict = {}
        for line in s.split('\n'):
          if line != '' and re.search(',', line) and (line.casefold().find('symbol') == -1):
              (code, name) = [x.strip() for x in line.split(',')[1:3]]
              res_dict[code] = int(name)
        if(symbol=="all"):
            return res_dict
        if(symbol!=""):
            return res_dict[symbol.upper()]

    if(mode=="pandas"):
        payload = pd.read_csv(url)
        if(symbol=="all"):
            return payload
        else:
            payload = payload[(payload.iloc[:, 1] == symbol.upper())]
            return payload

def whoistheboss():
    return "subhash"

def indiavix():
    payload = nsefetch("https://www.nseindia.com/api/allIndices")
    for x in range(0, len(payload["data"])):
        if(payload["data"][x]["index"]=="INDIA VIX"):
            return payload["data"][x]["last"]

def index_info(index):
    payload = nsefetch("https://www.nseindia.com/api/allIndices")
    for x in range(0, len(payload["data"])):
        if(payload["data"][x]["index"]==index):
            return payload["data"][x]

import math
from scipy.stats import norm

def black_scholes_dexter(S0,X,t,σ="",r=10,q=0.0,td=365):

  if(σ==""):σ =indiavix()

  S0,X,σ,r,q,t = float(S0),float(X),float(σ/100),float(r/100),float(q/100),float(t/td)
  #https://unofficed.com/black-scholes-model-options-calculator-google-sheet/

  d1 = (math.log(S0/X)+(r-q+0.5*σ**2)*t)/(σ*math.sqrt(t))
  #stackoverflow.com/questions/34258537/python-typeerror-unsupported-operand-types-for-float-and-int

  #stackoverflow.com/questions/809362/how-to-calculate-cumulative-normal-distribution
  Nd1 = (math.exp((-d1**2)/2))/math.sqrt(2*math.pi)
  d2 = d1-σ*math.sqrt(t)
  Nd2 = norm.cdf(d2)
  call_theta =(-((S0*σ*math.exp(-q*t))/(2*math.sqrt(t))*(1/(math.sqrt(2*math.pi)))*math.exp(-(d1*d1)/2))-(r*X*math.exp(-r*t)*norm.cdf(d2))+(q*math.exp(-q*t)*S0*norm.cdf(d1)))/td
  put_theta =(-((S0*σ*math.exp(-q*t))/(2*math.sqrt(t))*(1/(math.sqrt(2*math.pi)))*math.exp(-(d1*d1)/2))+(r*X*math.exp(-r*t)*norm.cdf(-d2))-(q*math.exp(-q*t)*S0*norm.cdf(-d1)))/td
  call_premium =math.exp(-q*t)*S0*norm.cdf(d1)-X*math.exp(-r*t)*norm.cdf(d1-σ*math.sqrt(t))
  put_premium =X*math.exp(-r*t)*norm.cdf(-d2)-math.exp(-q*t)*S0*norm.cdf(-d1)
  call_delta =math.exp(-q*t)*norm.cdf(d1)
  put_delta =math.exp(-q*t)*(norm.cdf(d1)-1)
  gamma =(math.exp(-r*t)/(S0*σ*math.sqrt(t)))*(1/(math.sqrt(2*math.pi)))*math.exp(-(d1*d1)/2)
  vega = ((1/100)*S0*math.exp(-r*t)*math.sqrt(t))*(1/(math.sqrt(2*math.pi))*math.exp(-(d1*d1)/2))
  call_rho =(1/100)*X*t*math.exp(-r*t)*norm.cdf(d2)
  put_rho =(-1/100)*X*t*math.exp(-r*t)*norm.cdf(-d2)

  return call_theta,put_theta,call_premium,put_premium,call_delta,put_delta,gamma,vega,call_rho,put_rho

def equity_history_virgin(symbol,series,start_date,end_date):
    #url="https://www.nseindia.com/api/historical/cm/equity?symbol="+symbol+"&series=[%22"+series+"%22]&from="+str(start_date)+"&to="+str(end_date)+""
    url = 'https://www.nseindia.com/api/historical/cm/equity?symbol=' + symbol + '&series=["' + series + '"]&from=' + start_date + '&to=' + end_date

    payload = nsefetch(url)
    return pd.DataFrame.from_records(payload["data"])

# You shall see beautiful use the logger function.
def equity_history(symbol,series,start_date,end_date):
    #We are getting the input in text. So it is being converted to Datetime object from String.
    start_date = datetime.datetime.strptime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strptime(end_date, "%d-%m-%Y")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))

    #We are calculating the difference between the days
    diff = end_date-start_date
    logging.info("Total Number of Days: "+str(diff.days))
    logging.info("Total FOR Loops in the program: "+str(int(diff.days/40)))
    logging.info("Remainder Loop: " + str(diff.days-(int(diff.days/40)*40)))

    total=pd.DataFrame()
    for i in range (0,int(diff.days/40)):

        temp_date = (start_date+datetime.timedelta(days=(40))).strftime("%d-%m-%Y")
        start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")

        logging.info("Loop = "+str(i))
        logging.info("====")
        logging.info("Starting Date: "+str(start_date))
        logging.info("Ending Date: "+str(temp_date))
        logging.info("====")

        #total=total.append(equity_history_virgin(symbol,series,start_date,temp_date))
        #total=total.concat(equity_history_virgin(symbol,series,start_date,temp_date))
        total = pd.concat([total, equity_history_virgin(symbol, series, start_date, temp_date)])

        logging.info("Length of the Table: "+ str(len(total)))

        #Preparation for the next loop
        start_date = datetime.datetime.strptime(temp_date, "%d-%m-%Y")

    start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strftime(end_date, "%d-%m-%Y")

    logging.info("End Loop")
    logging.info("====")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))
    logging.info("====")

    #total=total.append(equity_history_virgin(symbol,series,start_date,end_date))
    #total=total.concat(equity_history_virgin(symbol,series,start_date,end_date))
    total = pd.concat([total, equity_history_virgin(symbol, series, start_date, end_date)])

    logging.info("Finale")
    logging.info("Length of the Total Dataset: "+ str(len(total)))
    payload = total.iloc[::-1].reset_index(drop=True)
    return payload

def derivative_history_virgin(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice="",optionType=""):

    instrumentType = instrumentType.lower()

    if(instrumentType=="options"):
        if("NIFTY" in symbol): instrumentType="FUTSTK"
        instrumentType="OPTSTK"
    if(instrumentType=="futures"):
        if("NIFTY" in symbol): instrumentType="OPTIDX"
        instrumentType="FUTIDX"

    if(((instrumentType=="OPTIDX")or (instrumentType=="OPTSTK")) and (expiry_date!="")):
        strikePrice = "%.2f" % strikePrice
        strikePrice = str(strikePrice)

    nsefetch_url = "https://www.nseindia.com/api/historical/fo/derivatives?&from="+str(start_date)+"&to="+str(end_date)+"&optionType="+optionType+"&strikePrice="+strikePrice+"&expiryDate="+expiry_date+"&instrumentType="+instrumentType+"&symbol="+symbol+""
    payload = nsefetch(nsefetch_url)
    logging.info(nsefetch_url)
    logging.info(payload)
    return pd.DataFrame.from_records(payload["data"])

def derivative_history(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice="",optionType=""):
    #We are getting the input in text. So it is being converted to Datetime object from String.
    start_date = datetime.datetime.strptime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strptime(end_date, "%d-%m-%Y")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))

    #We are calculating the difference between the days
    diff = end_date-start_date
    logging.info("Total Number of Days: "+str(diff.days))
    logging.info("Total FOR Loops in the program: "+str(int(diff.days/40)))
    logging.info("Remainder Loop: " + str(diff.days-(int(diff.days/40)*40)))

    total=pd.DataFrame()
    for i in range (0,int(diff.days/40)):

        temp_date = (start_date+datetime.timedelta(days=(40))).strftime("%d-%m-%Y")
        start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")

        logging.info("Loop = "+str(i))
        logging.info("====")
        logging.info("Starting Date: "+str(start_date))
        logging.info("Ending Date: "+str(temp_date))
        logging.info("====")

        #total=total.append(derivative_history_virgin(symbol,start_date,temp_date,instrumentType,expiry_date,strikePrice,optionType))
        #total=total.concat([total, derivative_history_virgin(symbol,start_date,temp_date,instrumentType,expiry_date,strikePrice,optionType)])
        total = pd.concat([total, derivative_history_virgin(symbol, start_date, temp_date, instrumentType, expiry_date, strikePrice, optionType)])

        logging.info("Length of the Table: "+ str(len(total)))

        #Preparation for the next loop
        start_date = datetime.datetime.strptime(temp_date, "%d-%m-%Y")

    start_date = datetime.datetime.strftime(start_date, "%d-%m-%Y")
    end_date = datetime.datetime.strftime(end_date, "%d-%m-%Y")

    logging.info("End Loop")
    logging.info("====")
    logging.info("Starting Date: "+str(start_date))
    logging.info("Ending Date: "+str(end_date))
    logging.info("====")

    #total=total.append(derivative_history_virgin(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice,optionType))
    #total = total.concat([total, derivative_history_virgin(symbol,start_date,end_date,instrumentType,expiry_date,strikePrice,optionType)])
    total = pd.concat([total, derivative_history_virgin(symbol, start_date, end_date, instrumentType, expiry_date, strikePrice, optionType)])

    logging.info("Finale")
    logging.info("Length of the Total Dataset: "+ str(len(total)))
    payload = total.iloc[::-1].reset_index(drop=True)
    return payload

def expiry_history(symbol,start_date="",end_date=""):
    if(end_date==""):end_date=end_date
    nsefetch_url = "https://www.nseindia.com/api/historical/fo/derivatives/meta?&from="+start_date+"&to="+end_date+"&symbol="+symbol+""
    payload = nsefetch(nsefetch_url)
    return payload['data'][2]

# # Nifty Indicies Site

niftyindices_headers = {
    'Connection': 'keep-alive',
    'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"',
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'DNT': '1',
    'X-Requested-With': 'XMLHttpRequest',
    'sec-ch-ua-mobile': '?0',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36',
    'Content-Type': 'application/json; charset=UTF-8',
    'Origin': 'https://niftyindices.com',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Dest': 'empty',
    'Referer': 'https://niftyindices.com/reports/historical-data',
    'Accept-Language': 'en-US,en;q=0.9,hi;q=0.8',
}

def index_history(symbol,start_date,end_date):
    data = "{'name':'"+symbol+"','startDate':'"+start_date+"','endDate':'"+end_date+"'}"
    payload = requests.post('https://niftyindices.com/Backpage.aspx/getHistoricaldatatabletoString', headers=niftyindices_headers,  data=data).json()
    payload = json.loads(payload["d"])
    payload=pd.DataFrame.from_records(payload)
    return payload

def index_pe_pb_div(symbol,start_date,end_date):
    data = "{'name':'"+symbol+"','startDate':'"+start_date+"','endDate':'"+end_date+"'}"
    payload = requests.post('https://niftyindices.com/Backpage.aspx/getpepbHistoricaldataDBtoString', headers=niftyindices_headers,  data=data).json()
    payload = json.loads(payload["d"])
    payload=pd.DataFrame.from_records(payload)
    return payload

def index_total_returns(symbol,start_date,end_date):
    data = "{'name':'"+symbol+"','startDate':'"+start_date+"','endDate':'"+end_date+"'}"
    payload = requests.post('https://niftyindices.com/Backpage.aspx/getTotalReturnIndexString', headers=niftyindices_headers,  data=data).json()
    payload = json.loads(payload["d"])
    payload=pd.DataFrame.from_records(payload)
    return payload

def get_bhavcopy(date):
    date = date.replace("-","")
    payload=pd.read_csv("https://archives.nseindia.com/products/content/sec_bhavdata_full_"+date+".csv")
    return payload

def get_bulkdeals():
    payload=pd.read_csv("https://archives.nseindia.com/content/equities/bulk.csv")
    return payload

def get_blockdeals():
    payload=pd.read_csv("https://archives.nseindia.com/content/equities/block.csv")
    return payload

#Request from subhash
## https://unofficed.com/how-to-find-the-beta-of-indian-stocks-using-python/
def get_beta_df_maker(symbol,days):
    if("NIFTY" in symbol):
        end_date = datetime.datetime.now().strftime("%d-%b-%Y")
        end_date = str(end_date)

        start_date = (datetime.datetime.now()- datetime.timedelta(days=days)).strftime("%d-%b-%Y")
        start_date = str(start_date)

        df2=index_history(symbol,start_date,end_date)
        df2["daily_change"]=df2["CLOSE"].astype(float).pct_change()
        df2=df2[['HistoricalDate','daily_change']]
        df2 = df2.iloc[1: , :]
        return df2
    else:
        end_date = datetime.datetime.now().strftime("%d-%m-%Y")
        end_date = str(end_date)

        start_date = (datetime.datetime.now()- datetime.timedelta(days=days)).strftime("%d-%m-%Y")
        start_date = str(start_date)

        df = equity_history(symbol,"EQ",start_date,end_date)

        df["daily_change"]=df["CH_CLOSING_PRICE"].pct_change()
        df=df[['CH_TIMESTAMP','daily_change']]
        df = df.iloc[1: , :] #thispointer.com/drop-first-row-of-pandas-dataframe-3-ways/
        return df

def getbeta(symbol,days=365,symbol2="NIFTY 50"):
    return get_beta(symbol,days,symbol2)

def get_beta(symbol,days=365,symbol2="NIFTY 50"):
    #Default is 248 days. (Input of Subhash)
    df = get_beta_df_maker(symbol,days)
    df2 = get_beta_df_maker(symbol2,days)

    x=df["daily_change"].tolist()
    y=df2["daily_change"].tolist()
    #stackoverflow.com/questions/42670055/is-there-any-better-way-to-calculate-the-covariance-of-two-lists-than-this
    mean_x = sum(x) / len(x)
    mean_y = sum(y) / len(y)
    covariance = sum((a - mean_x) * (b - mean_y) for (a,b) in zip(x,y)) / len(x)

    mean = sum(y) / len(y)
    variance = sum((i - mean) ** 2 for i in y) / len(y)

    beta = covariance/variance
    return round(beta,3)

def nse_preopen(key="NIFTY",type="pandas"):
    payload = nsefetch("https://www.nseindia.com/api/market-data-pre-open?key="+key+"")
    if(type=="pandas"):
        payload = pd.DataFrame(payload['data'])
        payload  = pd.json_normalize(payload['metadata'])
        return payload
    else:
        return payload

#By Avinash https://forum.unofficed.com/t/nsepython-documentation/376/102?u=dexter
def nse_preopen_movers(key="FO",filter=1.5):
    preOpen_gainer=nse_preopen(key)
    return preOpen_gainer[preOpen_gainer['pChange'] >1.5],preOpen_gainer[preOpen_gainer['pChange'] <-1.5]

# type = "securities"
# type = "etf"
# type = "sme"
#
# sort = "volume"
# sort = "value"

def nse_most_active(type="securities",sort="value"):
    payload = nsefetch("https://www.nseindia.com/api/live-analysis-most-active-"+type+"?index="+sort+"")
    payload = pd.DataFrame(payload["data"])
    return payload

def nse_eq_symbols():
    #https://forum.unofficed.com/t/feature-request-stocklist-api/1073/11
    eq_list_pd = pd.read_csv('https://archives.nseindia.com/content/equities/EQUITY_L.csv')
    return eq_list_pd['SYMBOL'].tolist()

def nse_price_band_hitters(bandtype="both",view="AllSec"):
  payload = nsefetch("https://www.nseindia.com/api/live-analysis-price-band-hitter")

  #bandtype can be upper, lower, both
  #view can be AllSec,SecGtr20,SecLwr20
  return pd.DataFrame(payload[bandtype][view]["data"])

def nse_largedeals(mode="bulk_deals"):
  payload = nsefetch('https://www.nseindia.com/api/snapshot-capital-market-largedeal')
  if(mode=="bulk_deals"):
    return pd.DataFrame(payload["BULK_DEALS_DATA"])
  if(mode=="short_deals"):
    return pd.DataFrame(payload["SHORT_DEALS_DATA"])
  if(mode=="block_deals"):
    return pd.DataFrame(payload["BLOCK_DEALS_DATA"])

def nse_largedeals_historical(from_date, to_date, mode="bulk_deals"):
    if mode == "bulk_deals":
        mode = "bulk-deals"
    elif mode == "short_deals":
        mode = "short-selling"
    elif mode == "block_deals":
        mode = "block-deals"

    url='https://www.nseindia.com/api/historical/' + mode + '?from=' + from_date + '&to=' + to_date
    logging.info("Fetching " + str(url))
    payload = nsefetch(url)
    return pd.DataFrame(payload["data"])

#https://forum.unofficed.com/t/feature-request-nse-fno-participant-wise-oi/1179/7
#print(get_fao_participant_oi("04-06-2021"))
def get_fao_participant_oi(date):
    date = date.replace("-","")
    payload=pd.read_csv("https://archives.nseindia.com/content/nsccl/fao_participant_oi_"+date+".csv")
    return payload

#https://forum.unofficed.com/t/how-to-check-if-the-market-is-open-today-or-not/1268/1
def is_market_open(segment = "FO"): #COM,CD,CB,CMOT,COM,FO,IRD,MF,NDM,NTRP,SLBS

    holiday_json = nse_holidays()[segment]

    # Get today's date in the format 'dd-Mon-yyyy'
    today_date = datetime.date.today().strftime('%d-%b-%Y')

    # Check if today's date is in the holiday_json
    for holiday in holiday_json:
        if holiday['tradingDate'] != today_date:
            print("FNO Market is open today. Have a Nice Trade!")
            return True
        if holiday['tradingDate'] == today_date:
            print(f"Market is closed today because of {holiday['description']}")
            return False

def nse_expirydetails_by_symbol(symbol,meta ="Futures",i=0):
    payload = nse_quote(symbol)

    if(meta=="Futures"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "futures" in key.lower()), None)
    if(meta=="Options"):
      selected_key = next((key for key in payload["expiryDatesByInstrument"] if "options" in key.lower()), None)

    expiry_dates=payload["expiryDatesByInstrument"][selected_key]
    expiry_dates = [datetime.datetime.strptime(date, "%d-%b-%Y").date() for date in expiry_dates]
    expiry_dates = [date.strftime("%d-%b-%Y") for date in expiry_dates if date >= datetime.datetime.now().date()]

    currentExpiry=expiry_dates[i]
    currentExpiry = datetime.datetime.strptime(currentExpiry,'%d-%b-%Y').date()
    dte = (currentExpiry - datetime.datetime.now().date()).days
    return currentExpiry,dte


import re
from functools import partial
from pathlib import Path
from typing import List, Optional

import sublime
import sublime_plugin

from ..core.git import git_status_porcelain
from ..core.parser import parse_git_status
from ..core.git_file import GitFile

class GitFilesGotoCommand(sublime_plugin.WindowCommand):
    def run(self):
        cwd = Path(self.window.extract_variables()["folder"])
        output = git_status_porcelain(cwd)
        git_files = parse_git_status(output)

        active_view = self.window.active_view()
        self._highlighted_view = None  # type: Optional[sublime.View]

        if len(git_files) > 0:
            self.window.show_quick_panel(
                items=self.to_quick_panel_item(git_files),
                on_select=partial(self.on_select, active_view, cwd, git_files),
                on_highlight=partial(self.on_highlight, cwd, git_files),
            )
        else:
            sublime.message_dialog("GitFiles: No changed files.")

    def to_quick_panel_item(
        self, git_files: List[GitFile]
    ) -> List[sublime.QuickPanelItem]:
        items = []
        for item in git_files:
            git_status = item.git_status
            items.append(
                sublime.QuickPanelItem(
                    item.file_name,
                    item.file_path,
                    git_status.description(),
                    git_status.kind(),
                )
            )
        return items

    def __open_file(
        self, cwd, items, index: int, on_highlight: bool
    ) -> Optional[sublime.View]:
        if index == -1:
            return None

        rename_match = re.search(r"(.+) -> (.*)", items[index].file_path)
        file_path = (
            rename_match.group(2)
            if rename_match is not None
            else items[index].file_path
        )
        full_path = Path(cwd, file_path)
        flag = sublime.TRANSIENT if on_highlight else sublime.NewFileFlags.NONE
        if full_path.is_file():
            return self.window.open_file(
                str(full_path), flag, self.window.active_group()
            )
        return None

    def on_select(self, active_view: Optional[sublime.View], cwd, items, index: int):
        if index == -1:
            if active_view is not None and active_view.is_valid():
                self.window.focus_view(active_view)
                active_view.show(active_view.sel()[0])

            if self._highlighted_view:
                sheet = self._highlighted_view.sheet()
                if sheet and sheet.is_transient():
                    self._highlighted_view.close()

        self.__open_file(cwd, items, index, False)

    def on_highlight(self, cwd, items, index: int):
        self._highlighted_view = self.__open_file(cwd, items, index, True)


#coding=utf-8
#此文件用于指定后台返回前台的协议

# 返回包封装类

import json
from bson import json_util

class DataProtocol(object):

    SUCCESS = 0
    ARGUMENT_ERROR = -1 #参数错误
    DB_ERROR = -2      #数据库访问错误
    DB_DUPLICATE_ERROR = -3 # 数据重复

    IS_NOT_AJAX_REQUEST = -4 #非 ajax请求却请求了ajax/xxx的文件
    DATAPROTOCOL_ERROR = -5  # -5 ：数据封装问题
    FILE_ERROR = -6 #文件读取失败
    AUTH_ERROR = -7
    USER_EXISTED_ERROR = -8 #用户已经存在，注册失败
    LOGIN_FAIL = -9
    SERVER_ERROR = -10

    PERMISSION_DENY = -11 #没有权限
    USER_EXISTED_ERROR = -12 #用户不存在

    AT_LEAST_ONE_TRUNK_ERROR = -20001 #只要要保存一辆货车
    BILL_REMOVED = -30001  #bill已经被删除
    BILL_NOT_OWN = -30002   #用户不用有该bill
    BILL_NOT_WAIT = -30003   #bill不再有效期
    USER_INVALID = -30004  #请求中用户不存在或者不是有效的对象(管理员)

    ALREADY_COMMENTED = -40001 #已经评论过了
    CANNOT_SELF_COMMENT = -40002 #不能给自己评论呢
    MESSAGE_DUPLICATE_ERROR = -50001 #重复添加message
    # code: 0 代表成功 ，负数代表失败，-1 ~ -100 通用失败， -1000 ~ -9999 自定义错误代码

    ALREADY_EXIST = -50001   #要添加的东西已经存在

    # datatype include "html","json","string","nothing"
    #"nothing " is default

    #MSG
    COMMON_MSG = "服务器繁忙，请稍后再试"
    ARGUMENT_ERROR_MSG ="请求失败，请重试"
    DB_ERROR = "服务器繁忙，请稍后再试"
    IS_NOT_AJAX_REQUEST_MSG="非法请求，请用正常方式请求"
    AT_LEAST_ONE_TRUNK_ERROR_MSG = "至少要保存一辆货车"
    ALREADY_COMMENTED_MSG = "已经评论过了"
    CANNOT_SELF_COMMENT_MSG = "不能给自己评论"
    BILL_REMOVED_MSG = "订单已经被删除"

    #什么都不判断直接输出return，不保证数据正确，只能被getJson()调用
    @staticmethod
    def _wrapJson(code, msg, data, datatype="nothing"):
        retDict = {
            "code":code,
            "msg":msg,
            "data":data,
            "datatype":datatype
            }
        return json.dumps(retDict,default=json_util.default)

    @staticmethod
    def _wrapError(msg="数据封装错误"):
        return DataProtocol._wrapJson(DataProtocol.DATAPROTOCOL_ERROR, msg,"","nothing")

    @staticmethod
    def getJson(code, msg="", data=None, datatype="nothing"):
        retJson = None
        if not isinstance(code,int):
            try:
                code = int(code)
            except:
                retJson = DataProtocol._wrapError("code is not int")

        if not isinstance(msg, str):
            try:
                msg = str(msg)
            except:
                retJson = DataProtocol._wrapError("msg is not string")

        if not isinstance(datatype, str):
            try:
                datatypeTmp = str(datatype)
            except:
                retJson = DataProtocol._wrapError("datatype is not string")
            else:
                if datatype == "string" or datatype == "json" or datatype == "html":
                    if not isinstance(data, str):
                        try:
                            dataTmpStr = str(data)
                        except:
                            retJson = DataProtocol._wrapError("datatype said is "+ datatype+ " but data is "+ str(type(data)))
                        else:
                            data = dataTmpStr
                elif datatype == "nothing":
                    pass
                else:
                    retJson = DataProtocol._wrapError("unknown datatype")

        if msg == "":
            if code == DataProtocol.ARGUMENT_ERROR:
                msg = DataProtocol.ARGUMENT_ERROR_MSG
            elif code ==DataProtocol.DB_ERROR:
                msg = DataProtocol.DB_ERROR_MSG
            elif code ==DataProtocol.IS_NOT_AJAX_REQUEST:
                msg = DataProtocol.IS_NOT_AJAX_REQUEST_MSG
            elif code ==DataProtocol.AT_LEAST_ONE_TRUNK_ERROR:
                msg = DataProtocol.AT_LEAST_ONE_TRUNK_ERROR_MSG
            elif code ==DataProtocol.ALREADY_COMMENTED:
                msg = DataProtocol.ALREADY_COMMENTED_MSG
            elif code ==DataProtocol.CANNOT_SELF_COMMENT:
                msg = DataProtocol.CANNOT_SELF_COMMENT_MSG
            else:
                msg = DataProtocol.COMMON_MSG

        # print "----------getJson",code,msg,data,datatype
        if not retJson is None:
            return retJson
        else:
            return DataProtocol._wrapJson(code,msg,data,datatype);

    @staticmethod
    def getSuccessJson(data=None,datatype="nothing"):
        return DataProtocol.getJson(0,"success",data, datatype)

#单元测试-_-
#if __name__ == "__main__":
#     print r"DataProtocol.getSuccessJson(\"hello, world\",\"string\")", DataProtocol.getSuccessJson("hello, world","string")
#     print r"DataProtocol.getSuccessJson(\"hello, world\",\"(123,123)\")", DataProtocol.getSuccessJson("hello, world","(123,123)")
#     print r"DataProtocol.getSuccessJson(1,2)", DataProtocol.getSuccessJson(1,2)
#     print DataProtocol.getJson("0","success")
#     print DataProtocol.getJson(0,"success")
#     print DataProtocol.getJson("o","success")
#     print DataProtocol.getJson(0,"success",data={"a":1},datatype="json")
#     print DataProtocol.getJson(0,"success",data=(1,2,3),datatype="json")
#     print DataProtocol.getJson("-4",(123,123),data=(1,2,3),datatype="json")
#     print DataProtocol.getJson(5,None,None,None)


import tkinter

# ウィンドウの作成
root = tkinter.Tk()

# タイトルを設定
root.title('Window Practice!')

# アイコンを設定
root.iconbitmap('icon.ico')

# ウィンドウのサイズを設定
root.geometry('300x800')
root.resizable(0, 1) # 横方向のサイズの変更を0とする（禁止）

# 色を設定
root.config(bg='red')

# サブウィンドウの作成
sub_window = tkinter.Toplevel()
sub_window.title('Second Window')
sub_window.config(bg='#123123')
sub_window.geometry('200x300+500+500') # +500で右に500、次の500が下に500移動させる

# ウィンドウのループ処理
# これが無いとウィンドウが消えてしまう
root.mainloop()


#!/usr/bin/env python
# -*- coding:utf-8 -*-
__author__ = 'MFC'
__time__ = '18/5/21 00:28'

"""
原始字符串
Python中字符串前面加上 r 表示原生字符串, 如: r'abc'
"""

import re

mm = "c:\\a\\b\\c"

print(mm)

# ret = re.match("c:\\a", mm).group()   # 未处理转义，所以匹配不到
# print(ret)

ret = re.match(r"c:\\a", mm).group()  # 防止字符转义
print(ret)


import datetime
import json
import urllib.request

from celery import shared_task
from django.core.mail import send_mail

from Task17.settings import API_KEY
from weather.models import City, Forecast

@shared_task
def send_email(interval):
    cities = City.objects.all()
    for city in cities:
        for sub in city.subscriptions.all():
            if sub.interval == interval:
                source = urllib.request.urlopen(
                    f'https://api.openweathermap.org/data/2.5/weather?q={city}&appid={API_KEY}&units=metric').read()
                list_of_data = json.loads(source)
                weather = str(list_of_data['weather'][0]['main'])
                country_code = str(list_of_data['sys']['country'])
                longitude = str(list_of_data['coord']['lon'])
                latidude = str(list_of_data['coord']['lat'])
                timezone = str(list_of_data['timezone'] // 3600)
                temperature = str(list_of_data['main']['temp'])
                temp_min = str(list_of_data['main']['temp_min'])
                temp_max = str(list_of_data['main']['temp_max'])
                pressure = str(list_of_data['main']['pressure'])
                humidity = str(list_of_data['main']['humidity'])
                feels_like = str(list_of_data['main']['feels_like'])
                visibility = str(list_of_data['visibility'])
                wind_speed = str(list_of_data['wind']['speed'])
                clouds = str(list_of_data['clouds']['all'])
                sunrise = datetime.datetime.fromtimestamp(list_of_data['sys']['sunrise']).strftime('%H:%M:%S')
                sunset = datetime.datetime.fromtimestamp(list_of_data['sys']['sunset']).strftime('%H:%M:%S')

                context = f"Thanks to our service, you get a weather forecast in the city {city} once every {interval}." \
                          f"
City name: {city}
" \
                          f"
country code: {country_code}, timezone: {timezone}
" \
                          f"
longitude: {longitude}, latidude: {latidude}
" \
                          f"
" \
                          f"
weather: {weather}
" \
                          f"
temperature: {temperature + 'C'}, temperature min: {temp_min + 'C'}, temperature max: {temp_max + 'C'}
" \
                          f"
feels_like: {feels_like}
" \
                          f"
" \
                          f"
pressure: {pressure}, humidity: {humidity}, visibility: {visibility}
" \
                          f"
wind speed: {wind_speed}, clouds: {clouds}
" \
                          f"
sunrise: {sunrise}, sunset: {sunset}
"
                f, created = Forecast.objects.get_or_create(
                    city=city,
                    weather=weather,
                    temp=temperature,
                    humidity=humidity,
                    pressure=pressure,
                    temp_min=temp_min,
                    temp_max=temp_max,
                    visibility=visibility,
                    wind_speed=wind_speed,
                    clouds=clouds,
                    sunrise=sunrise,
                    sunset=sunset,
                )
                send_mail(f'Forecast in {city}!',
                          f'The weather in {city}:',
                          'Weather Reminder Service',
                          [sub.user.email],
                          html_message=context,
                          fail_silently=False
                          )


import torch
from collections import Counter

def get_entities(seq,id2label,markup='bios'):
    """Gets entities from sequence.
    note: BIOS
    Args:
        seq (list): sequence of labels.
    Returns:
        list: list of (chunk_type, chunk_start, chunk_end).
    Example:
        # >>> seq = ['B-PER', 'I-PER', 'O', 'S-LOC']
        # >>> get_entity_bios(seq)
        [['PER', 0,1], ['LOC', 3, 3]]
    """
    chunks = []
    chunk = [-1, -1, -1]
    for indx, tag in enumerate(seq):
        if not isinstance(tag, str):
            tag = id2label[tag]
        if tag.startswith("S-"):
            if chunk[2] != -1:
                chunks.append(chunk)
            chunk = [-1, -1, -1]
            chunk[1] = indx
            chunk[2] = indx
            chunk[0] = tag.split('-')[1]
            chunks.append(chunk)
            chunk = (-1, -1, -1)
        if tag.startswith("B-"):
            if chunk[2] != -1:
                chunks.append(chunk)
            chunk = [-1, -1, -1]
            chunk[1] = indx
            chunk[0] = tag.split('-')[1]
        elif tag.startswith('I-') and chunk[1] != -1:
            _type = tag.split('-')[1]
            if _type == chunk[0]:
                chunk[2] = indx
            if indx == len(seq) - 1:
                chunks.append(chunk)
        else:
            if chunk[2] != -1:
                chunks.append(chunk)
            chunk = [-1, -1, -1]
    return chunks

class F1(object):
    def __init__(self):
        self.origins = []
        self.founds = []
        self.rights = []
    def compute(self, origin, found, right):
        recall = 0 if origin == 0 else (right / origin)
        precision = 0 if found == 0 else (right / found)
        f1 = 0. if recall + precision == 0 else (2 * precision * recall) / (precision + recall)
        return recall, precision, f1
    def result(self):
        num_examples = len(self.origins)
        origins = 0
        founds = 0
        rights = 0
        for i in range(num_examples):
            origin = self.origins[i]
            found = self.founds[i]
            right = self.rights[i]
            origins += len(origin)
            founds += len(found)
            rights += len(right)
        # print('origins: {}'.format(origins))
        # print('founds: {}'.format(founds))
        # print('rights: {}'.format(rights))
        recall, precision, f1 = self.compute(origins, founds, rights)
        return {'acc': precision, 'recall': recall, 'f1': f1}

    def update(self, pred, label):
        self.origins.append(label)
        self.founds.append(pred)
        self.rights.append(list(set(pred) & set(label)))

class FunctionAcc(object):
    def __init__(self, end_id):
        self.correct = 0
        self.tot = 0
        self.end_id = end_id
    def result(self):
        acc = self.correct / self.tot
        return acc
    def update(self, pred, label):
        match = True
        for i in range(min(len(pred), len(label))):
            if label[i] != pred[i]:
                match = False
                break
            if pred[i] == self.end_id and label[i] == self.end_id:
                break
        if match:
            self.correct += 1
        self.tot += 1

class SeqEntityScore(object):
    def __init__(self, id2label,markup='bios'):
        self.id2label = id2label
        self.markup = markup
        self.reset()

    def reset(self):
        self.origins = []
        self.founds = []
        self.rights = []

    def compute(self, origin, found, right):
        recall = 0 if origin == 0 else (right / origin)
        precision = 0 if found == 0 else (right / found)
        f1 = 0. if recall + precision == 0 else (2 * precision * recall) / (precision + recall)
        return recall, precision, f1

    def result(self):
        class_info = {}
        origin_counter = Counter([x[0] for x in self.origins])
        found_counter = Counter([x[0] for x in self.founds])
        right_counter = Counter([x[0] for x in self.rights])
        for type_, count in origin_counter.items():
            origin = count
            found = found_counter.get(type_, 0)
            right = right_counter.get(type_, 0)
            recall, precision, f1 = self.compute(origin, found, right)
            class_info[type_] = {"acc": round(precision, 4), 'recall': round(recall, 4), 'f1': round(f1, 4)}
        origin = len(self.origins)
        found = len(self.founds)
        right = len(self.rights)
        recall, precision, f1 = self.compute(origin, found, right)
        return {'acc': precision, 'recall': recall, 'f1': f1}, class_info

    def update(self, label_paths, pred_paths):
        '''
        labels_paths: [[],[],[],....]
        pred_paths: [[],[],[],.....]

        :param label_paths:
        :param pred_paths:
        :return:
        Example:
            >>> labels_paths = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
            >>> pred_paths = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
        '''
        for label_path, pre_path in zip(label_paths, pred_paths):
            label_entities = get_entities(label_path, self.id2label,self.markup)
            pre_entities = get_entities(pre_path, self.id2label,self.markup)
            self.origins.extend(label_entities)
            self.founds.extend(pre_entities)
            self.rights.extend([pre_entity for pre_entity in pre_entities if pre_entity in label_entities])