# !/usr/bin/env python
# -*- coding: utf-8 -*-
# Oliver J. Hall

import numpy as np
import pandas as pd
import corner as corner
import emcee

import glob
import sys
import ClosePlots as cp
from tqdm import tqdm

import matplotlib
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
matplotlib.rcParams['xtick.direction'] = 'out'
matplotlib.rcParams['ytick.direction'] = 'out'

import cMCMC
import cPrior
import cLLModels

'''<Unused in current build.>'''
def save_library(df, chain,labels_mc):
    results = pd.DataFrame(columns=labels_mc)
    stddevs = pd.DataFrame(columns=[l+'err' for l in labels_mc])

    npa = chain.shape[1]
    r = np.zeros(npa)
    s = np.zeros(npa)
    for idx in np.arange(npa):
        r[idx] = np.median(chain[:,idx])
        s[idx] = np.std(chain[:,idx])
    results.loc[0], stddevs.loc[0] = r, s

    output = pd.concat([results,stddevs],axis=1)
    output.to_csv('Output/2dmix_results.csv')
    df.to_csv('Output/2dmix_selected_data.csv')

    return results, stddevs

def tortoise(dfm):
    sfile = glob.glob('../data/TRILEGAL_sim/*all*.txt')[0]
    df = pd.read_csv(sfile, sep='\s+')
    labels = df['stage'].values
    Zish = df['[M/H]'].values
    logT = df['logTe'].values
    logL = df['logL'].values

    fig, ax = plt.subplots()
    fig2, ax2 = plt.subplots(3,3)
    c = ['r','b','c','g','y','k','m','darkorange','chartreuse']
    loc = [(0,0),(0,1),(0,2),(1,0),(1,1),(1,2),(2,0),(2,1),(2,2)]

    for i in range(int(np.nanmax(labels))+1):
        ax.scatter(logT[labels==i],logL[labels==i],s=5,c=c[i],label=str(i))
        im = ax2[loc[i]].scatter(logT[labels==i],logL[labels==i],s=10,c=Zish[labels==i],\
                            cmap='viridis',vmin=-4,vmax=0.7)#vmin=0.0,vmax=4.)
        ax2[loc[i]].scatter(dfm.logTe[dfm.stage==i],dfm.logL[dfm.stage==i],s=7,c='r',marker=',')
        ax2[loc[i]].set_title(str(i))
        ax2[loc[i]].invert_xaxis()

    ax.scatter(dfm['logTe'],dfm['logL'],s=5,c='r',marker=',',label='Mask')
    ax.legend(loc='best')
    ax.invert_xaxis()
    ax.set_xlabel('log(T)')
    ax.set_ylabel('log(L)')
    fig2.tight_layout()

    fig2.subplots_adjust(right=0.8)
    cbar_ax = fig2.add_axes([0.85,0.15,0.05,0.7])
    fig2.colorbar(im,cax=cbar_ax,label='[M/H]')

    fig.tight_layout()
    fig.savefig('../Output/HR_mask.png')
    fig2.savefig('../Output/HRsplit_mask.png')
    plt.show()
    return len(df[labels==4])

def get_errors(df, DR=2):
    if DR == 2:
        df['pi_err'] = np.abs(0.2 + np.random.normal(0,0.5,len(df)) * 0.1) #mas

    if DR == 'fin':
        df['pi_err'] = np.abs(10.e-3 + np.random.normal(0,0.5,len(df)) * 2.e-3) #mas

    df['d'] = 10**(1+ (df['m-M0']/5))       #Getting all distances (pc)
    df['pi'] = 1000/df['d']                 #Getting all parallax (mas)
    df['sig_d'] = (1000*df['pi_err']/df['pi']**2)  #Propagating the Gaia error

    df['sig_mu'] = np.sqrt( (5*np.log10(np.e)/df['d'])**2 * df['sig_d']**2 )
    df['sig_M'] = df['sig_mu']  #Until we incorporate errors on Aks and Ks

    plt.errorbar(df['pi'],df['Ks'],xerr=df['pi_err'],alpha=.1,fmt=".k",c='grey',zorder=999)
    plt.scatter(df['pi'],df['Ks'],s=5,zorder=1000)
    plt.show()

    plt.errorbar(df['d'],df['Ks'],xerr=df['sig_d'],alpha=.1,fmt=".k",c='grey',zorder=999)
    plt.scatter(df['d'],df['Ks'],s=5,zorder=1000)
    plt.show()
    return df
'''</unused in current build.>'''

def get_values():
    sfile = glob.glob('../Cuts_Data/cuts_MH_JKs_logg.txt')[0]
    try:
        df = pd.read_csv(sfile, sep='\s+')
    except IOError:
        print('Cuts file doesnt exist, run the slider first.')

    '''Errors not included in current run'''
    # df = get_errors(df, DR=2)
    df = df[(df.M_ks > -2.0) & (df.M_ks < -1.0)]
    df = df[0:10000]

    return df.M_ks.values, df.M_j.values, df.stage, df

class cLikelihood:
    '''A likelihood function that pulls in log likehoods from the LLModels class
    '''
    def __init__(self,_lnprior, _Model):
        self.lnprior = _lnprior
        self.Model = _Model

    #Likelihood for the 'foreground'
    def lnlike_fg(self, p):
        return self.Model.lorentzian(p[0:2]) + self.Model.lorentzian(p[4:6],dim='y')
        # return self.Model.lorentzian(p[0:2]) + self.Model.lorentzian(p[4:6],dim='y')

    def lnlike_bg(self, p):
        # return self.Model.lorentzian(p[2:4]) + self.Model.lorentzian(p[6:8],dim='y')
        return self.Model.lorentzian(p[2:4]) + self.Model.lorentzian(p[6:8],dim='y')

    def lnprob(self, p):
        Q = p[-1]

        # First check the prior.
        lp = self.lnprior(p)
        if not np.isfinite(lp):
            return -np.inf, None

        # Compute the vector of foreground likelihoods and include the q prior.
        ll_fg = self.lnlike_fg(p)
        arg1 = ll_fg + np.log(Q)

        # Compute the vector of background likelihoods and include the q prior.
        ll_bg = self.lnlike_bg(p)
        arg2 = ll_bg + np.log(1.0 - Q)

        # Combine these using log-add-exp for numerical stability.
        ll = np.nansum(np.logaddexp(arg1, arg2))

        return lp + ll

    def __call__(self, p):
        logL = self.lnprob(p)
        return logL

def probability_plot(x, y, df, bins, Ksx0, Jx0, lor_Ks_fg, lor_Ks_bg, lor_J_fg, lor_J_bg):
    #Plotting residuals with histograms
    left, bottom, width, height = 0.1, 0.35, 0.4, 0.60
    fig = plt.figure(1, figsize=(10,8))
    lax = fig.add_axes([left, bottom, width, height])           #Left-hand plot
    rax = fig.add_axes([left+width+0.05, bottom, width, height]) #Right-hand plot
    xax = fig.add_axes([left, 0.1, width, 0.22], sharex=lax)    #x histogram
    yax = fig.add_axes([left+width+0.05, 0.1, width, 0.22], sharex=rax) #y histogram
    #Clearing necessary axes
    lax.xaxis.set_visible(False)
    rax.xaxis.set_visible(False)
    yax.set_yticklabels([])
    yax2 = yax.twinx()
    yax2.set_yticklabels([])
    xax.set_yticklabels([])
    xax2 = xax.twinx()
    xax2.set_yticklabels([])
    #Turning on grid
    xax.grid()
    xax.set_axisbelow(True)
    yax.grid()
    yax.set_axisbelow(True)

    fig.suptitle('Probability functions to be applied to TRILEGAL data.')
    #Plotting left hand 2d hist
    lax.hist2d(x, df.Ks,bins=bins, cmap='Blues_r', zorder=1000)
    lax.axvline(Ksx0,c='r',zorder=1001)

    #Plotting right hand 2d hist
    rax.hist2d(y, df.J, bins=bins, cmap='Blues_r',zorder=1000)
    rax.axvline(Jx0, c='r', zorder=1001)

    #Plotting histogram with models for J
    yax.hist(y,bins=bins,histtype='step',color='r',normed=True)
    yax2.scatter(y,lor_J_fg,s=5,c='cornflowerblue',alpha=.5,label='Foreground Lorentzian in J')
    yax2.scatter(y,lor_J_bg,s=5,c='orange',alpha=.5,label='Background Lorentzian in J')
    yax2.scatter(y,lor_J_bg+lor_J_fg,s=1,c='green',alpha=.2,label='Combined')
    yax2.set_ylim(0.)
    yax.axvline(Jx0,c='r',label=r"$Jx0$")
    h1, l1 = yax.get_legend_handles_labels()
    h2, l2 = yax2.get_legend_handles_labels()
    yax.legend(h1+h2, l1+l2)

    #Plotting histograms with models for Ks
    xax.hist(x,bins=bins,histtype='step',color='r',normed=True)
    xax2.scatter(x,lor_Ks_fg,s=5,c='cornflowerblue',alpha=.5,label='Foreground Lorentzian in Ks')
    xax2.scatter(x,lor_Ks_bg,s=5,c='orange',alpha=.5,label='Background Lorentzian in Ks')
    xax2.scatter(x,lor_Ks_fg+lor_Ks_bg,s=1,c='green',alpha=.2,label='Combined')
    xax2.set_ylim(0.)
    xax.axvline(Ksx0,c='r',label=r"$Ksx0$")
    h1, l1 = xax.get_legend_handles_labels()
    h2, l2 = xax2.get_legend_handles_labels()
    xax.legend(h1+h2, l1+l2)

    #Set labels
    xax.set_xlabel(r"$M_{Ks}$")
    yax.set_xlabel(r"$M_{J}$")
    lax.set_ylabel(r"$m_{Ks}$")
    rax.set_ylabel(r"$m_{J}$")
    return fig

if __name__ == '__main__':
    plt.close('all')
####---SETTING UP DATA
    x, y, labels, df = get_values()

####---PLOTTING INITIAL DATA
    fig, ax = plt.subplots()
    c = ['r','b','c','g','y','k','m','darkorange','chartreuse']
    label = ['Pre-Main Sequence', 'Main Sequence', 'Subgiant Branch', 'Red Giant Branch', 'Core Helium Burning',\
                'RR Lyrae variables', 'Cepheid Variables', 'Asymptotic Giant Branch','Supergiants']
    ax.scatter(x[labels==3], df.Ks[labels==3], c=c[3], s=1,zorder=1000,label=label[3])
    ax.scatter(x[labels==4], df.Ks[labels==4], c=c[4], s=1,zorder=1001,label=label[4])
    sel = (labels!=3)&(labels!=4)
    ax.scatter(x[sel], df.Ks[sel], c=c[0], s=1, zorder=1002, label='Other types')
    ax.set_xlabel(r"$M_{Ks}$")
    ax.set_ylabel(r"$m_{Ks}$")
    ax.set_title("Labeled TRILEGAL simulated data cut in [M/H], logg & J-Ks")
    ax.legend(loc='best',fancybox=True)
    ax.grid()
    ax.set_axisbelow(True)
    fig.savefig('Output/investigate_TRILEGAL.png')
    plt.close('all')

####---EXAMINING & ESTIMATING PARAMETERS
    bins = int(np.sqrt(len(x)))

    n, b = np.histogram(x,bins=bins)
    Ksx0guess = b[np.argmax(n)]

    n, b = np.histogram(y, bins=bins)
    Jx0guess = b[np.argmax(n)]

####---SETTING UP MCMC
    style = 'lors'
    if style == 'lors':
        labels_mc = ["$x0 (Ks)$", r"$\gamma0$(Ks)",\
                    "$x1$(Ks)", r"$\gamma1$(Ks)",\
                    "$x0 (J)$", r"$\gamma0$(J)",\
                    "$x1$(J)", r"$\mu$(J)",\
                    "$Q$"]
        start_params = np.array([Ksx0guess, 0.02,   #Foreground params in Ks
                                -1.56, 0.1,         #Background params in Ks
                                Jx0guess, 0.05,     #Foreground params in J
                                -0.93, 0.1,         #Background params in J
                                0.5])              #Mixture model param
        bounds = [(-1.7,-1.6), (0.01,0.8),
                    (-1.6,-1.5), (0.08, 0.2),
                    (-1.1,-0.9), (0.01,0.8),
                    (-0.96,-0.9), (0.08, 1.2),
                    (0, 1)]
    if style == 'notlors':
        labels_mc = ["$x0 (Ks)$", r"$\gamma$ (Ks)",\
                    r"$\mu$(Ks)", r"$\sigma$(Ks)",\
                    "$x0 (J)$", r"$\gamma$ (J)",\
                    r"$\mu$(Ks)", r"$\sigma$(Ks)",\
                    "$Q$"]
        start_params = np.array([Ksx0guess, 0.02,   #Foreground params in Ks
                                -1.58, 0.14,         #Background params in Ks
                                Jx0guess, 0.05,     #Foreground params in J
                                -0.93, 0.1,         #Background params in J
                                0.5])              #Mixture model param
        bounds = [(-1.7,-1.63), (0.01,0.8),
                    (-1.63,-1.5), (0.08, 0.2),
                    (-1.1,-1.0), (0.01,0.8),
                    (-0.96,-0.9), (0.08, 1.2),
                    (0, 1)]

####---CHECKING MODELS BEFORE RUN
    #Getting other probability functions
    ModeLLs = cLLModels.LLModels(x, y, labels_mc)
    lor_Ks_fg = np.exp(ModeLLs.lorentzian(start_params[0:2]))
    lor_Ks_bg = np.exp(ModeLLs.lorentzian(start_params[2:4]))
    # lor_Ks_bg = np.exp(ModeLLs.gaussian(start_params[2:4]))
    lor_J_fg = np.exp(ModeLLs.lorentzian(start_params[4:6], dim='y'))
    lor_J_bg = np.exp(ModeLLs.lorentzian(start_params[6:8], dim='y'))
    # lor_J_bg = np.exp(ModeLLs.gaussian(start_params[6:8], dim='y'))

    #Visualising the probability distributions
    fig = probability_plot(x, y, df, bins, Ksx0guess, Jx0guess, lor_Ks_fg, lor_Ks_bg, lor_J_fg, lor_J_bg)
    fig.savefig('Output/visual_models.png')
    plt.show()
    plt.close('all')

####---RUNNING MCMC
    lnprior = cPrior.Prior(bounds)
    Like = cLikelihood(lnprior, ModeLLs)
    if np.isinf(lnprior(start_params)):
        print('Starting guesses out of bounds.')
        sys.exit()

    ntemps, nwalkers = 4, 32

    Fit = cMCMC.MCMC(start_params, Like, lnprior, 'none', ntemps, 1000, nwalkers)
    chain = Fit.run()

####---CONSOLIDATING RESULTS
    corner.corner(chain, bins=35, labels=labels_mc)
    plt.savefig('Output/corner.png')
    plt.show()
    plt.close()

    print('Plotting model results...')
    npa = chain.shape[1]
    res = np.zeros(npa)
    std = np.zeros(npa)
    for idx in np.arange(npa):
        res[idx] = np.median(chain[:,idx])
        std[idx] = np.std(chain[:,idx])

    #Calling probability functions with results
    lor_Ks_fg = np.exp(ModeLLs.lorentzian(res[0:2]))
    # lor_Ks_bg = np.exp(ModeLLs.lorentzian(res[2:4]))
    lor_Ks_bg = np.exp(ModeLLs.gaussian(res[2:4]))
    lor_J_fg = np.exp(ModeLLs.lorentzian(res[4:6], dim='y'))
    # lor_J_bg = np.exp(ModeLLs.lorentzian(res[6:8], dim='y'))
    lor_J_bg = np.exp(ModeLLs.gaussian(res[6:8], dim='y'))

    fig = probability_plot(x, y, df, bins, res[0], res[4], lor_Ks_fg, lor_Ks_bg, lor_J_fg, lor_J_bg)
    fig.savefig('Output/visual_result.png')

    print('Calculating posteriors...')
    lnK, fg_pp = Fit.log_bayes()
    sys.exit()

    #Calculate recall vs precision for various Kass+Raftery94 cut off scales
    recall, precision = [], []
    for lim in np.linspace(lnK.min(),lnK.max(),1000):
        mask = lnK > lim
        cheb_correct = len(x[mask][labels[mask]==4])
        cheb_total = len(x[labels==4])
        identified_total = len(x[mask])
        recall.append(float(cheb_correct)/float(cheb_total))
        try:
            precision.append(float(cheb_correct)/float(identified_total))
        except:
            precision.append(0.)
    f, a = plt.subplots()
    col = a.scatter(recall, precision, c=np.linspace(lnK.min(),lnK.max(),1000))
    a.axhline(0.9,c='r',label='0.9 precision')
    a.set_xlabel('Recall')
    a.set_ylabel('Precision')
    a.grid()
    a.set_axisbelow(True)
    f.colorbar(col, label='lnK cut-off point')
    f.savefig('Output/precisionvsrecall.png')
    plt.show()

    mask = lnK > 3

####---PLOTTING RESULTS

    # Plot mixture model results
    fig, ax = plt.subplots()
    ax.grid()
    ax.set_axisbelow(True)

    col = ax.scatter(x, df.Ks, c=fg_pp,s=5,cmap='viridis')
    ax.axvline(res[0],c='r',label=r"$\mu$")
    ax.set_title(r"Inlier Posterior Probabilities for TRILEGAL simulated data")
    ax.set_xlabel(r"$M_{Ks}$")
    ax.set_ylabel(r"$m_{Ks}$")
    fig.colorbar(col,label='Inlier Posterior Probability')
    ax.legend(loc='best',fancybox='True')
    fig.savefig('Output/posteriors.png')

    #Plotting identified results
    ffig, aax = plt.subplots()
    aax.scatter(x[~mask], df.Ks[~mask], c=c[3], s=1,zorder=1000,label='Outliers (RGB)')
    aax.scatter(x[mask], df.Ks[mask], c=c[4], s=1,zorder=1000,label='Inliers (RC)')
    aax.legend(loc='best',fancybox=True)

    cheb_correct = len(x[mask][labels[mask]==4])
    cheb_total = len(x[labels==4])
    identified_total = len(x[mask])
    recall = float(cheb_correct)/float(cheb_total)
    precision = float(cheb_correct)/float(identified_total)

    aax.set_xlabel(r"$M_{Ks}$")
    aax.set_ylabel(r"$m_{Ks}$")
    aax.set_title('Recall: '+str.format('{0:.2f}',recall) + '| Precision: '+str.format('{0:.2f}',precision))
    aax.legend(loc='best',fancybox=True)
    aax.grid()
    aax.set_axisbelow(True)

    ffig.savefig('Output/dataset.png')
    cp.show()
    plt.close('all')

    sys.exit()
    Fit.dump()

    '''Everything below here is redundant in current build.'''

####---SAVING STUFF INTO A PANDAS LIBRARY
    print('Saving output...')
    results, stddevs = save_library(df,chain,labels_mc)

    sys.exit()
#__________________GETTING CORRECTION____________________________
    print('Calculating corrections...')
    #Plot showing correction
    m_ks = df.Ks.values[mask]
    Aks = df.Aks.values[mask]
    pi_o = df['pi'].values[mask]
    pi_o_err = df['pi_err'].values[mask]

    #Calculating RC parallax from our fit value for magnitude
    Mrc = np.ones(y[mask].shape)*rc                 #Fit RC luminosity
    Mrcerr = np.ones(y[mask].shape)*err             #Fit RC error
    mu_rc = m_ks - Mrc - Aks                        #Calculating distance modulus

    d_rc = 10**(1+mu_rc/5)      #Calculating distance
    pi_rc = 1000/d_rc            #Calculating parallax

    d_err = np.sqrt( (2*np.log(10)*10**(mu_rc/5))**2 * Mrcerr**2 )
    pi_rc_err = 1000 * d_err / d_rc**2

    #Fit a straight line to the data
    fit = np.polyfit(pi_rc, pi_o, 1)
    fn = np.poly1d(fit)

    #Plotting results
    fig, ax = plt.subplots(2)
    ax[0].errorbar(pi_rc,pi_o, xerr=pi_rc_err, yerr=pi_o_err,fmt=",k",alpha=.1, ms=0, capsize=0, lw=1, zorder=999)
    im = ax[0].scatter(pi_rc,pi_o, marker="s", s=5, c=m_ks, cmap="viridis", zorder=1000)
    ax[0].plot(pi_rc,pi_rc,linestyle='--',c='r',alpha=.5, zorder=1001, label='One to One')
    ax[0].plot(pi_rc, fn(pi_rc),c='k',alpha=.9, zorder=1002, label='Straight line polyfit')
    ax[0].set_xlabel('RC parallax (mas)')
    ax[0].set_ylabel('TRILEGAL parallax (mas)')
    ax[0].set_title(r'Determined RC parallax compared to TRILEGAL parallax for classified RC stars')
    ax[0].legend(loc='best')

    yrr = np.sqrt( (pi_o_err / pi_rc)**2 + ( (pi_o/pi_rc**2)**2 * pi_rc_err**2))
    # yrr = np.sqrt(pi_rc_err**2 + pi_o_err**2)
    ax[1].errorbar(pi_rc,pi_o/pi_rc, xerr=pi_rc_err,yerr=yrr, fmt=",k",alpha=.2, ms=0, capsize=0, lw=1, zorder=999)
    ax[1].scatter(pi_rc,pi_o/pi_rc, s=1,c=m_ks,cmap='viridis',zorder=1000)
    ax[1].set_xlabel('RC parallax (mas)')
    ax[1].set_ylabel('TRILEGAL parallax/RC parallax')
    ax[1].set_title(r'Residuals to one-to-one for upper plot')
    ax[1].axhline(y=1.0,c='r',alpha=.5,linestyle='--')
    fig.tight_layout()

    fig.subplots_adjust(right=0.8)
    cbar_ax = fig.add_axes([0.85,0.15,0.05,0.7])
    norm = matplotlib.colors.Normalize(vmin=m_ks.min(),vmax=m_ks.max())
    col = matplotlib.colorbar.ColorbarBase(cbar_ax,cmap='viridis',norm=norm,orientation='vertical',label='TRILEGAL apaprent magnitude')

    ax[0].grid()
    ax[0].set_axisbelow(True)
    ax[1].grid()
    ax[1].set_axisbelow(True)

    plt.savefig('/home/oliver/Dropbox/Papers/Midterm/Images/C4_corrections.png')
    # plt.savefig('../Output/corrections.png')
    plt.show()

#__________________CALCULATING OUTLIER FRACTION____________________________
    '''Red clump label: 4 | RGB label: 3'''
    rctotal = tortoise(df[mask])
    rctotal_mod = len(df[labels==4])
    rcm = len(df[mask][labels==4])
    rgbm = len(df[mask][labels==3])
    starm = len(np.where(mask==True)[0])

    print('\nTotal number of RC stars in the reduced sample: '+str(rctotal_mod)+', '+str(rctotal_mod*100/rctotal)+ '% of the total sample.')
    print('Number of stars in mask: '+str(starm))
    print('Percentage of which are RC: '+str(rcm*100/starm)+'%')
    print('The remaining stars are all RGB stars.')
    print('\nFraction of all RC stars identified: ' +str(rcm*100/rctotal_mod)+'%')


class Solution:
    def isHappy(self, n):
        visited = [n]
        sq = {"1":1,"2":4,"3":9,"4":16,"5":25,"6":36,"7":49,"8":64,"9":81,"0":0}
        happy = False
        while(True):
            x = str(n)
            n=0
            for i in x:
                n += sq[i]
            if(n==1):
                happy=True
                break
            if(n in visited):
                break
            else:
                visited.append(n)
        return happy


import torch
from torch import nn
from torch.utils.data import DataLoader
import torchvision
from torchvision import transforms
from torchvision.datasets import CIFAR10
import numpy as np
import matplotlib.pyplot as plt

def imshow(img):
    img = torchvision.utils.make_grid(img)
    img = img / 2 + 0.5
    npimg = img.detach().numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

def train(net, criterion, optimizer, epochs, trainloader):
    losses = []
    output_and_label = []

    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    print('using device:', device)

    net = net.to(device)

    for epoch in range(1, epochs+1):
        print(f'epoch: {epoch}, ', end='')
        running_loss = 0.0
        for counter, (img, _) in enumerate(trainloader, 1):
            img = img.to(device)
            optimizer.zero_grad()
            output = net(img)
            loss = criterion(output, img)
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
        avg_loss = running_loss / counter
        losses.append(avg_loss)
        print('loss:', avg_loss)
        output_and_label.append((output, img))
    print('finished')
    return output_and_label, losses

transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
trainset = CIFAR10('./data', train=True, transform=transform, download=True)
testset = CIFAR10('./data', train=False, transform=transform, download=True)

batch_size = 50
trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True)
testloader = DataLoader(testset, batch_size=batch_size // 10, shuffle=False)

class AutoEncoder(torch.nn.Module):
    def __init__(self, enc, dec):
        super().__init__()
        self.enc = enc
        self.dec = dec
    def forward(self, x):
        x = self.enc(x)
        x = self.dec(x)
        return x

enc = torch.nn.Sequential(
    torch.nn.Conv2d(3, 16, kernel_size=4, padding=1, stride=2),
    torch.nn.ReLU(),
    torch.nn.Conv2d(16, 32, kernel_size=4, padding=1, stride=2),
    torch.nn.ReLU()
)

#For ConvTransposed http://makeyourownneuralnetwork.blogspot.com/2020/02/calculating-output-size-of-convolutions.html
dec = torch.nn.Sequential(
    torch.nn.ConvTranspose2d(32, 16, kernel_size=4, stride=2, padding=1),
    torch.nn.ReLU(),
    torch.nn.ConvTranspose2d(16, 3, kernel_size=4, stride=2, padding=1),
    torch.nn.Tanh()
)

net = AutoEncoder(enc, dec)
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(net.parameters(), lr=0.5)
EPOCHS = 100

output_and_label, losses = train(net, criterion, optimizer, EPOCHS, trainloader)


import numpy as np
import logging
import torch
import argparse
import os
import sys
import math
from collections import OrderedDict
from collections import defaultdict

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from lib import soe, ste, ckl, tste, gnmds, forte, landmark_oe
from preprocessing_utils.data_select_utils import select_dataset
from preprocessing_utils.TripletData import TripletBatchesDataset
from logging_utils import logging_util
from config_utils.config_eval import load_config
from preprocessing_utils.TripletData import procrustes_disparity
from preprocessing_utils.TripletData import knn_classification_error
from preprocessing_utils.TripletData import triplet_error_batches
import joblib
from lib.lsoe_utils.oracle import Oracle

from lib.lsoe_utils.lsoe_mproc import embedding_rest_indices as second_phase
from lib.lsoe_utils.lsoe_mproc import first_phase, first_phase_soe

from lib.oenn_utils import data_utils as data_utils_oenn
from lib.oenn_utils import training_routine_v3

def parse_args():
    """
    To run this file use "CUDA_VISIBLE_DEVICES=3 python train_soe.py -config configs/soe/soe_evaluation.json". See
    the config file in
    the path for an example of how to construct config files.
    """
    parser = argparse.ArgumentParser(description='Run SOE Experiments')
    parser.add_argument('-config', '--config_path', type=str, default='configs/soe/uniform_baseline.json',
                        required=True,
                        help='Input the Config File Path')
    parser.add_argument('-data', '--data_set', type=str, default='not_selected', required=False,
                        help='Manually change dataset')
    args = parser.parse_args()
    return args

def main(args):
    config = load_config(args.config_path)
    algorithm = config['algorithm']
    error_change_threshold = config['error_change_threshold']
    input_dim_range = config['tradeoff_set']['input_dimension']
    output_dimensions_range = config['tradeoff_set']['output_dimension']
    nmb_points_range = config['tradeoff_set']['number_of_points']
    batch_size_range = config['tradeoff_set']['batch_size']
    learning_rate_range = config['tradeoff_set']['learning_rate']
    triplet_multiplier_range = config['tradeoff_set']['triplets_multiplier']

    if args.data_set == 'not_selected':
        dataset_name = config['dataset_selected']
    else:
        dataset_name = args.data_set

    try:
        input_equals_output = config['tradeoff_set']['input_equals_output']
    except:
        input_equals_output = False

    epochs = config['nb_epochs']
    optimizer = config['optimizer']
    n_test_triplets = config['n_test_triplets']

    log_dir = config['log']['path']

    separator = '_'
    experiment_name = algorithm + \
                      '_data_' + dataset_name + \
                      '_input_dim_' + separator.join([str(i) for i in input_dim_range]) + \
                      '_output_dim_' + separator.join([str(i) for i in output_dimensions_range]) + \
                      '_n_pts_' + separator.join([str(i) for i in nmb_points_range]) + \
                      '_bs_' + separator.join([str(i) for i in batch_size_range]) + \
                      '_change_criterion_' + str(error_change_threshold) + \
                      '_lr_' + separator.join([str(i) for i in learning_rate_range]) + \
                      '_triplet_num_' + separator.join([str(i) for i in triplet_multiplier_range]) + \
                      '_ep_' + str(epochs)
        # create a log directory if it does not exist
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    logging_path = os.path.join(log_dir, experiment_name + '.log')
    logger = logging_util.my_custom_logger(logger_name=logging_path, level=logging.INFO)
    logger.info('Name of Experiments: ' + experiment_name)
    logger.info('Dataset Name: ' + dataset_name)
    logger.info('Error Change Threshold: ' + str(error_change_threshold))
    logger.info('Epochs: ' + str(epochs))

    tradeoff_results = defaultdict(dict)

    order = ['Input Dimension', 'Output Dimension', 'Number of Points',
             'Batch Size', 'Learning Rate', 'Triplet Multiplier',
             ['Train Error', 'Test Error', 'Procrustes Error', 'Knn Orig Error', 'Knn Ordinal Error', 'Time',
              'Embedding', 'Labels', 'Train Triplets']]
    experiment_range = OrderedDict({'input_dim': input_dim_range, 'output_dim': output_dimensions_range,
                                    'number_of_points': nmb_points_range, 'batch_size': batch_size_range,
                                    'learning_rate': learning_rate_range,
                                    'triplet_multiplier': triplet_multiplier_range})

    for input_dim_index, input_dim in enumerate(input_dim_range):
        for dimensions_index, embedding_dimension in enumerate(output_dimensions_range):
            for subset_index, nmb_points in enumerate(nmb_points_range):
                for batch_size_index, batch_size in enumerate(batch_size_range):
                    for lr_index, learning_rate in enumerate(learning_rate_range):
                        for trip_mindex, triplet_multiplier in enumerate(triplet_multiplier_range):

                            if (not input_equals_output) or (input_equals_output and input_dim == embedding_dimension):
                                logger.info('Learning Rate: ' + str(learning_rate))
                                logger.info('Number of Points: ' + str(nmb_points))
                                logger.info('Input Dimension: ' + str(input_dim))
                                logger.info('Output Dimension: ' + str(embedding_dimension))
                                logger.info('Number of Test Triplets: ' + str(n_test_triplets))
                                logger.info('Triplet Multiplier: ' + str(triplet_multiplier))
                                logger.info('Batch Size: ' + str(batch_size))

                                embedding, train_triplets, labels, train_error, test_error, \
                                procrustes_error, knn_orig, knn_embed, time_taken, \
                                loss_history, triplet_error_history, time_history = run_method(config, dataset_name,
                                                                                algorithm, nmb_points,
                                                                                input_dim,
                                                                                embedding_dimension,
                                                                                learning_rate,
                                                                                batch_size,
                                                                                triplet_multiplier,
                                                                                optimizer, epochs,
                                                                                n_test_triplets, logger,
                                                                                error_change_threshold)

                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 0] = train_error
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 1] = test_error
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 2] = procrustes_error
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 3] = knn_orig
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 4] = knn_embed
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 5] = time_taken
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 6] = embedding
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 7] = labels
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 8] = loss_history
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 9] = triplet_error_history
                                tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                                 batch_size_index, lr_index, trip_mindex, 10] = time_history
                                # tradeoff_results[input_dim_index, dimensions_index, subset_index,
                                #                  batch_size_index, lr_index, trip_mindex, 8] = train_triplets

    for input_dim_index, input_dim in enumerate(input_dim_range):
        for dimensions_index, embedding_dimension in enumerate(output_dimensions_range):
            for subset_index, nmb_points in enumerate(nmb_points_range):
                for batch_size_index, batch_size in enumerate(batch_size_range):
                    for lr_index, learning_rate in enumerate(learning_rate_range):
                        for trip_mindex, triplet_multiplier in enumerate(triplet_multiplier_range):
                            if (not input_equals_output) or (input_equals_output and input_dim == embedding_dimension):
                                logger.info('Input Dimension ' + str(input_dim) +
                                            ' Output Dimension ' + str(embedding_dimension) +
                                            ' Number of Points ' + str(nmb_points) +
                                            ' Batch Size ' + str(batch_size) +
                                            ' Learning Rate ' + str(learning_rate) +
                                            ' Triplet Multiplier ' + str(triplet_multiplier))
                                logger.info(' Train Error ' +
                                            str(tradeoff_results[input_dim_index, dimensions_index,
                                                                 subset_index, batch_size_index, lr_index, trip_mindex, 0])
                                            +
                                            ' Test Error ' +
                                            str(tradeoff_results[input_dim_index, dimensions_index,
                                                                 subset_index, batch_size_index, lr_index, trip_mindex, 1]))

                                logger.info(' Procrustes Error ' +
                                            str(tradeoff_results[input_dim_index, dimensions_index,
                                                                 subset_index, batch_size_index, lr_index, trip_mindex, 2]))

                                logger.info(' kNN original Emb Loss ' +
                                            str(tradeoff_results[input_dim_index, dimensions_index,
                                                                 subset_index, batch_size_index, lr_index, trip_mindex, 3])
                                            +
                                            ' kNN on Ordinal Emb Loss ' +
                                            str(tradeoff_results[input_dim_index, dimensions_index,
                                                                 subset_index, batch_size_index, lr_index, trip_mindex, 4]))

                                logger.info(' Time ' +
                                            str(tradeoff_results[input_dim_index, dimensions_index,
                                                                 subset_index, batch_size_index, lr_index, trip_mindex, 5]))
                                logger.info('-' * 20)

    data_dump = [order, experiment_range, tradeoff_results]
    joblib.dump(data_dump, os.path.join(log_dir, experiment_name + '.pkl'))

def run_method(config, dataset_name, algorithm, n_points, input_dim, embedding_dimension, learning_rate, batch_size,
               triplet_multiplier, optimizer, epochs, n_test_triplets, logger, error_change_threshold):
    vec_data, labels = select_dataset(dataset_name, n_samples=n_points, input_dim=input_dim)
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    n_points = vec_data.shape[0]

    triplet_num = np.int(np.ceil(triplet_multiplier * n_points * math.log2(n_points) * embedding_dimension))
    train_triplets = []
    loss_history, triplet_error_history, time_history = [], [], []

    batch_size = min(batch_size, triplet_num)

    logger.info('Computing Embedding...')
    logger.info('Number of Points: ' + str(n_points))
    logger.info('Number of Triplets: ' + str(triplet_num))
    logger.info('Input Dimension: ' + str(input_dim))
    logger.info('Output Dimension: ' + str(embedding_dimension))
    time_taken = 0
    train_error = -1  # active methods wont have a train error
    if optimizer == 'adam' and algorithm == 'soe':
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = soe.soe_adam(triplets=train_triplets,
                                                                                n=n_points,
                                                                                dim=embedding_dimension,
                                                                                epochs=epochs,
                                                                                batch_size=batch_size,
                                                                                learning_rate=learning_rate,
                                                                                device=device, logger=logger,
                                                                                error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)

    elif optimizer == 'sgd' and algorithm == 'soe':
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = soe.soe_sgd(triplets=train_triplets,
                                                                                n=n_points,
                                                                                dim=embedding_dimension,
                                                                                iterations=epochs, bs=batch_size,
                                                                                lr=learning_rate,
                                                                                device=device, logger=logger)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)
    elif algorithm == 'ste':
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = ste.ste_adam(triplets=train_triplets,
                                                                                n=n_points,
                                                                                dim=embedding_dimension,
                                                                                epochs=epochs,
                                                                                batch_size=batch_size,
                                                                                learning_rate=learning_rate,
                                                                                device=device, logger=logger,
                                                                                error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)
    elif algorithm == 'tste':
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = tste.t_ste_adam(triplets=train_triplets,
                                                                                n=n_points,
                                                                                emb_dim=embedding_dimension,
                                                                                epochs=epochs,
                                                                                batch_size=batch_size,
                                                                                learning_rate=learning_rate,
                                                                                device=device, logger=logger,
                                                                                error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)
    elif algorithm == 'triplet_loss':
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = soe.triplet_loss_adam(
            triplets=train_triplets,
            n=n_points, dim=embedding_dimension, iterations=epochs, batch_size=batch_size,
            lr=learning_rate, device=device, logger=logger, error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)
    elif algorithm == 'gnmds':
        regularizer = config['regularizer']
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = gnmds.gnmds(triplets=train_triplets,
                                                                                reg_lbda=regularizer,
                                                                                n=n_points,
                                                                                dim=embedding_dimension,
                                                                                epochs=epochs,
                                                                                batch_size=batch_size,
                                                                                learning_rate=learning_rate,
                                                                                device=device, logger=logger,
                                                                                error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)
    elif algorithm == 'forte':
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = forte.rank_d_pgd(triplets=train_triplets,
                                                                                n=n_points,
                                                                                dim=embedding_dimension,
                                                                                epochs=epochs,
                                                                                batch_size=batch_size,
                                                                                learning_rate=learning_rate,
                                                                                device=device,
                                                                                logger=logger,
                                                                                error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)
    elif algorithm == 'ckl':
        regularizer = config['regularizer']
        mu = config['mu']
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = ckl.ckl_k(triplets=train_triplets,
                                                                                reg_lbda=regularizer, mu=mu,
                                                                                n=n_points,
                                                                                dim=embedding_dimension,
                                                                                epochs=epochs,
                                                                                batch_size=batch_size,
                                                                                learning_rate=learning_rate,
                                                                                device=device, logger=logger,
                                                                                error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)

    elif algorithm == 'ckl_x':
        mu = config['mu']
        logger.info('Generating triplets...')

        train_triplets_dataset = TripletBatchesDataset(vec_data, labels, triplet_num, batch_size, device)
        train_triplets = train_triplets_dataset.trips_data_indices

        x, loss_history, triplet_error_history, time_taken, time_history = ckl.ckl_x(triplets=train_triplets, mu=mu,
                                                                                n=n_points,
                                                                                dim=embedding_dimension,
                                                                                epochs=epochs,
                                                                                batch_size=batch_size,
                                                                                learning_rate=learning_rate,
                                                                                device=device, logger=logger,
                                                                                error_change_threshold=error_change_threshold)
        # compute triplet error for train and test data
        train_error = triplet_error_batches(x, train_triplets)
    elif algorithm == 'loe':
        x, time_taken, train_error = landmark_oe.landmark_oe_with_data(data=vec_data, dim=embedding_dimension,
                                                                       trip_num=triplet_num,
                                                                       learning_rate=learning_rate, epochs=epochs,
                                                                       batch_size=batch_size,
                                                                       device=device, logger=logger)
    elif algorithm == 'oenn':

        number_of_neighbours = 50  # config['number_of_neighbours']
        metric = 'eu'  # config['metric']
        all_triplets, triplet_loaders = data_utils_oenn.prep_data_for_nn(vec_data=vec_data,
                                                                         labels=labels,
                                                                         triplet_num=triplet_num,
                                                                         batch_size=batch_size,
                                                                         metric=metric,
                                                                         number_of_neighbours=number_of_neighbours)

        hl_size = int(120 + (2 * embedding_dimension * math.log2(n_points)))
        x, loss_history, triplet_error_history, time_taken, time_history = training_routine_v3.create_and_train_triplet_network(
            dataset_name=dataset_name,
            ind_loaders=triplet_loaders,
            n=n_points,
            dim=embedding_dimension,
            layers=3,
            learning_rate=learning_rate,
            epochs=epochs,
            hl_size=hl_size,
            batch_size=batch_size,
            number_of_triplets=triplet_num,
            logger=logger,
            error_change_threshold=error_change_threshold)
        train_error = triplet_error_batches(x, all_triplets)

    elif algorithm == 'lloe':
        num_landmarks = config['optimizer_params']['num_landmarks']
        subset_size = config['optimizer_params']['subset_size']
        phase1_learning_rate = config['optimizer_params']['phase1_learning_rate']
        phase2_learning_rate = config['optimizer_params']['phase2_learning_rate']
        target_loss = config['optimizer_params']['target_loss']

        number_of_landmarks = min(int(num_landmarks * n_points), 100)
        subset_size = subset_size * number_of_landmarks
        landmarks, first_phase_indices, \
        first_phase_subset_size, first_phase_reconstruction, \
        first_phase_loss, first_phase_triplet_error, time_first_phase = first_phase_soe(
            num_landmarks=number_of_landmarks,
            subset_size=subset_size,
            data=vec_data, dataset_size=n_points,
            embedding_dim=embedding_dimension, epochs=epochs,
            first_phase_lr=phase1_learning_rate,
            device=device,
            target_loss=target_loss,
            batch_size=batch_size,
            logger=logger)
        embedded_indices = first_phase_indices
        embedded_points = first_phase_reconstruction
        non_embedded_indices = list(set(range(vec_data.shape[0])).difference(set(embedded_indices)))
        my_oracle = Oracle(data=vec_data)
        logger.info('Second Phase: ')
        logger.info('Oracle Created...')
        logger.info('Computing LLOE - Phase 2...')
        print(time_first_phase)
        # second phase for embedding point by point update
        second_phase_embeddings_index, \
        second_phase_embeddings, time_second_phase = second_phase(my_oracle=my_oracle,
                                                                  non_embedded_indices=non_embedded_indices,
                                                                  embedded_indices=embedded_indices,
                                                                  first_phase_embedded_points=embedded_points,
                                                                  dim=embedding_dimension,
                                                                  lr=phase2_learning_rate, logger=logger)
        # combine the first phase and second phase points and index
        x = np.zeros((vec_data.shape[0], embedding_dimension))
        # phase 1 points
        x[embedded_indices] = embedded_points
        # second phase points
        x[second_phase_embeddings_index] = second_phase_embeddings
        time_taken = time_first_phase + time_second_phase

    logger.info('Time Taken for experiment ' + str(time_taken) + ' seconds.')
    logger.info('Evaluating the computed embeddings...')

    test_triplets_dataset = TripletBatchesDataset(vec_data, labels, n_test_triplets, 1000, device)
    test_error = test_triplets_dataset.triplet_error(x)
    procrustes_error = procrustes_disparity(vec_data, x)
    knn_error_ord_emb, knn_error_true_emb = knn_classification_error(x, vec_data, labels)

    # log the errors
    logger.info('Train Error: ' + str(train_error))
    logger.info('Test Error: ' + str(test_error))
    logger.info('Procrustes Disparity: ' + str(procrustes_error))
    logger.info('kNN Classification Error on ground-truth: ' + str(knn_error_true_emb))
    logger.info('kNN Classification Error on embedding: ' + str(knn_error_ord_emb))
    return x, train_triplets, labels, train_error, test_error, procrustes_error, knn_error_true_emb, knn_error_ord_emb, time_taken, loss_history, triplet_error_history, time_history

if __name__ == "__main__":
    main(parse_args())


# Solving Kattis Magic Trick
# By Chance Parsons AKA Half-Qilin

import sys

# Debug
# from io import StringIO
# data = u"""\
# robust
# """
# sys.stdin = StringIO(data)
# End Debug

rawInput = sys.stdin.read()
string = rawInput.strip()
chars = []
for i in string:
    chars.append(i)
i = 0
b = True
while i < len(chars):
    j = i
    while j < len(chars) - 1:
        j += 1
        if chars[j] == chars[i]:
            print(0)
            b = False
            break
    if chars[j] == chars[i]:
        break
    i += 1
if b:
    print(1)


"""
Study Permutations and Combinations
where Repetition of outcomes is NOT allowed
"""

import math

# Lottery, 59 balls, draw 6, order is important

print(math.factorial(59) / math.factorial(59-6))

num_perms = 1
for index in range(59, 59-6, -1):
    num_perms *= index

print(num_perms)

Principles of Computing (Part 1)/Week 4 - Combinatorics, generators, and debugging/permutations_and_combinations.py


import math
num = math.tan((math.pi)*(1/4))
print(num)
list1=[]
list2=[]
def twodotsdeg2d(a, b, c, d):
    list = [a, b, c, d]
    num1 = math.degrees(math.atan(int(list[1])/int(list[0]))-math.atan(int(list[3])/int(list[2])))
    return abs(num1)
# thirddemsion for degree in two vectors
f=0
def magni(a, b):
    lista = a
    listb = b
    c = math.sqrt(lista[0]**2+lista[1]**2+lista[2]**2)
    d = math.sqrt(listb[0]**2 + listb[1]**2 + listb[2]**2)
    e = lista[0]**2 + lista[1]**2 + lista[2]**2
    f = listb[0]**2 + listb[1]**2 + listb[2]**2
    answer = {"len1": c, "len2": d, "nrlen1": e, "nrlen2": f}
    return answer
a = [6, 3, 2]
b = [4, 0, -1]
print("end")
answer = magni(a, b)
print(answer)

def twodotsdeg3d(a,b):
    answerf = magni(a, b)
    t1=(a[0]*b[0] + a[1]*b[1] + a[2]*b[2])/((math.sqrt(answerf["nrlen1"]*answerf["nrlen2"])))
    answerInt=math.degrees(math.acos(t1))
    return answerInt

a=[1,2,3] #17 of 13.3
b = [4,0,-1] #19
a = [0,1,1]
b = [1, 2, -3]
thd=twodotsdeg3d(a,b)
def checkdotsRTri(a,b):
    answerr = a[0]*b[0]+a[1]*b[1]+a[2]*b[2]
    return answerr
a1=checkdotsRTri([1,-3,-2],[2,0,-4])
print(a1)

print(thd)

def vectorMag(a,b,c):
    return (a**2+b**2+c**2)
print(vectorMag(3,4,5))
print(twodotsdeg3d([1,1,-1],[3,-4,5]))

#def crossProduct3d(a,b):
def vecCross3d(a,b):
    firstE = a[2]*b[3]-b[2]*a[3]
    secondE = a[1]*b[3]-b[1]*a[3]
    thirdE = a[1]*b[2]-b[1]*a[2]
    return "{}i + {}j + {}k".format(firstE, secondE, thirdE)
    print(vectCross3dd([2,-1,3],[3,2,1]))


def printTable(a):
    s = [0 for i in a[0] ]
    for b in a:
        s=list(map(max,map(len,b),s))
    for b in a:
        print(" ".join(map(lambda x,y:x.center(y),b,s)))

printTable([
    ['a','abcd','xy'],
    ['avfnkdfklk n','cd','']
    ])


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from PIL import Image
import PIL.ImageOps
import PIL
import cv2

# Now we have to load the npz image files
# This X variable will be used as our input(training)
X = np.load("image.npz")["arr_0"]
print(X)  # returns a 2-dimensonal array

# This Y Variable will be our result set
Y = pd.read_csv("data.csv")["labels"]
print(Y.head())  # prints the labels

# This value_counts() will show the number of occurances of each label in the data
print(pd.Series(Y).value_counts())

classes = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L",
           "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]
num_classes = len(classes)

# Now we have to split the data into train and test sets
train_X, test_X, train_Y, test_Y = train_test_split(
    X, Y, train_size=7500, test_size=2500)

# Now we have to scale the train_X and test_X
# so to scale the images we cannot use StandardScaler()
# so to scale the data we have to divide the train_X and test_X each by 255.0
train_X = train_X/255.0
test_X = test_X/255.0

# print(train_X)
# print(test_X)

# now its time to train our model
model = LogisticRegression(solver='saga', multi_class='multinomial')
model.fit(train_X, train_Y)

# now our model will predict the test_Y by taking the test_X(input factors) as input
prediction = model.predict(test_X)
# print(prediction)  # prints an array of predictions

# # accuracy
Accuracy = accuracy_score(test_Y, prediction)
print(f"Accuracy :- {Accuracy}")  # Good 99% accuracy

cameraObj = cv2.VideoCapture(0)

while(True):
    try:
        _ret, frame = cameraObj.read()

        # first we will convert the frame into a grayscale image
        grayscaled_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        # get the height and width of the whole video btw this will return a tuple
        height, width = grayscaled_image.shape()

        # we will draw a box on the center of the video that will be our region of interest
        upper_left = (int(width / 2 - 56), int(height / 2 - 56))
        bottom_right = (int(width / 2 + 56), int(height / 2 + 56))

        # now we will draw or plot the rectangle
        cv2.rectangle(grayscaled_image, upper_left,
                      bottom_right, (0, 255, 0), 2)

        #  we are creating a box in the video because we want the user to keep the digit inside the box
        #  our model will give more priority to the digit which is inside the box
        # it is also called roi = region of interest
        roi = grayscaled_image[upper_left[1]:bottom_right[1],
                               upper_left[0]:bottom_right[0]]

        # Converting cv2 image to pil format
        image_pil = Image.fromarray(roi)

        # converting to grayscale image - 'L' format means each pixel is
        # represented by a single value from 0 to 255
        image_bw = image_pil.convert('L')

        # resizing the image and also pixalating the image
        image_bw_resized = image_bw.resize((28, 28), Image.ANTIALIAS)

        # generally in Images and also in mirrors "the mirror efect" occurs
        # so we are inverting the resized image here
        image_bw_resized_inverted = PIL.ImageOps.invert(image_bw_resized)

        # applying a pixel filter to filter the pixalated image
        pixel_filter = 20
        min_pixel = np.percentile(image_bw_resized_inverted, pixel_filter)

        # clipping the image here (cutting or taking a particular part of the image)
        image_bw_resized_inverted_scaled = np.clip(
            image_bw_resized_inverted-min_pixel, 0, 255)

        # taking the maximum pixel from the inverted image
        max_pixel = np.max(image_bw_resized_inverted)

        image_bw_resized_inverted_scaled = np.asarray(
            image_bw_resized_inverted_scaled)/max_pixel

        # reshape() function returns the same array but with a new shape
        test_sample = np.array(
            image_bw_resized_inverted_scaled).reshape(1, 784)

        # our model is predicting by taking the test_sample
        test_pred = model.predict(test_sample)

        print(f"Predicted Class is :- {test_pred}")

        cv2.imshow('frame', grayscaled_image)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    except Exception as e:
        pass

cameraObj.release()
cv2.destroyAllWindows()

# !!! its not working , Its not giving any error message and I am fustrated
# !!!  the problem is that it is not showing the camera window in line 118


from typing import Iterable, TypeVar, Union

ItemType = TypeVar('ItemType')

class Sentinel:
    """Sentinel object."""

def sequentially_deduplicate(
    iterable: Iterable[ItemType]
) -> Iterable[ItemType]:
    """
    Remove sequential duplicates.

        ..pycon
        >>> list(sequentially_deduplicate([1, 3, 1, 1, 8, 1]))
        [1, 3, 8, 1]
    """
    sentinel = Sentinel()
    previous_item: Union[Sentinel, ItemType] = sentinel

    for item in iterable:   # noqa: WPS110 - this is an item of an iterable
        if item not in {previous_item, sentinel}:
            yield item

        previous_item = item


# Написать fizzbuzz для 20 троек чисел, которые записаны в файл.
# Читаете из файла первую строку, берете из нее числа, считаете для них fizzbuzz, выводите.

# for - переберает все элементы того что мы напсали после in
# и записывает нам в переменную, которую мы записали после for

# split - Разбивает строку по разделителю и создает список. По умолчанию разделитель - пробел
# int - целое число
# map - применяет переданную ей функцию к списку

f = open('numbers_fizz_buzz.txt')
f2 = open('file_hw.txt', 'w')

for s in f:
    numbers = list(s.split())
    fizz = int(numbers[0])
    buzz = int(numbers[1])
    number = int(numbers[2])

    counter = 1
    while counter <= number:
        if counter % fizz == 0 and counter % buzz == 0:
             f2.write("FB")
        elif counter % fizz == 0:
            f2.write("F")
        elif counter % buzz == 0:
            f2.write("B")
        else:
            f2.write(str(counter))
        counter += 1
    f2.write('\n')

f2.close()


from os import environ
from time import sleep

from tqdm import tqdm

from slack_progressing import SlackProgressing

def main():
    sp = SlackProgressing(
        token=environ.get("TOKEN"), channel=environ.get("CHANNEL")
    )

    acc = 0.0
    loss = 10.0

    for _epoch in sp(tqdm((range(10)))):
        acc += 1.0
        loss -= 1.0
        d = {"acc": acc, "loss": loss}
        sp.set_params(d)

        sleep(1)

if __name__ == "__main__":
    main()


import torch

from src.common import get_rays, raw2outputs_nerf_color, sample_pdf

class Renderer(object):
    def __init__(self, cfg, args, slam, points_batch_size=500000, ray_batch_size=50000):
        self.prefix_keys = ["", "2_", "3_"]
        self.num_sensors = cfg["num_sensors"]
        self.num_features = cfg["num_features"]

        self.patch_size = cfg["patch_size"]
        self.unused_list = ["error", "rays_o", "rays_d", "color", "local"]
        if self.num_features == 2:
            self.unused_list += ["dx", "dy", "normal"]
        elif self.num_features == 7:
            pass
        else:
            raise Exception("Number of features != 2 or 7")

        self.ray_batch_size = ray_batch_size
        self.points_batch_size = points_batch_size

        self.lindisp = cfg["rendering"]["lindisp"]
        self.perturb = cfg["rendering"]["perturb"]
        self.N_samples = cfg["rendering"]["N_samples"]
        self.N_surface = cfg["rendering"]["N_surface"]
        self.N_importance = cfg["rendering"]["N_importance"]

        self.scale = cfg["scale"]
        self.occupancy = cfg["occupancy"]
        self.bound = slam.bound

        self.H, self.W, self.fx, self.fy, self.cx, self.cy = (
            slam.H,
            slam.W,
            slam.fx,
            slam.fy,
            slam.cx,
            slam.cy,
        )

    def eval_points(self, p, d_feats, c_feats, decoders, c=None, stage="color", device="cuda:0"):
        """Evaluate the occupancy and/or color value for the points.

        Args:
            p (tensor, N*3): Point coordinates.
            d_feats (tensor, N*Y*2): Tensor of added features for uncertainty evaluation.
            c_feats (tensor, N*Z*2): Tensor of added features for color uncertainty evaluation.
            decoders (nn.module decoders): Decoders.
            c (dicts, optional): Feature grids. Defaults to None.
            stage (str, optional): Query stage, corresponds to different levels. Defaults to 'color'.
            device (str, optional): CUDA device. Defaults to 'cuda:0'.

        Returns:
            ret (tensor): occupancy (and color) value of input points.
        """

        p_split = torch.split(p, self.points_batch_size)

        bound = self.bound

        rets = []

        for i, pi in enumerate(p_split):
            # mask for points out of bound
            mask_x = (pi[:, 0] < bound[0][1]) & (pi[:, 0] > bound[0][0])
            mask_y = (pi[:, 1] < bound[1][1]) & (pi[:, 1] > bound[1][0])
            mask_z = (pi[:, 2] < bound[2][1]) & (pi[:, 2] > bound[2][0])
            mask = mask_x & mask_y & mask_z

            pi = pi.unsqueeze(0)
            ret, var = decoders(pi, c_grid=c, stage=stage, d_feats=d_feats, c_feats=c_feats)
            ret = ret.squeeze(0)
            if len(ret.shape) == 1 and ret.shape[0] == 4:
                ret = ret.unsqueeze(0)

            ret[~mask, 3] = 100
            rets.append(ret)

        ret = torch.cat(rets, dim=0)

        return ret, var

    def render_batch_ray(self, c, decoders, rays_d, rays_o, batch_dict, device, stage):
        """Render color, depth and uncertainty of a batch of rays.

        Args:
            c (dict): feature grids.
            decoders (nn.module): decoders.
            rays_d (tensor, N*3): rays direction.
            rays_o (tensor, N*3): rays origin.
            batch_dict (dict): Includes various features. sensor depth image, normal, color...
            device (str): device name to compute on.
            stage (str): query stage.

        Returns:
            depth (tensor): rendered depth.
            d_var (tensor): rendered sensor uncertainty.
            uncertainty (tensor): rendered model uncertainty.
            color (tensor): rendered color.

        """

        N_samples = self.N_samples
        N_surface = self.N_surface
        N_importance = self.N_importance

        N_rays = rays_o.shape[0]

        features = list(batch_dict.keys())

        c_features = [feat for feat in features if (feat[-1].isdigit() and feat.startswith("color"))]

        # only keeps expanded patch of features
        features = [
            feat for feat in features if (feat[-1].isdigit() and not any(feat.startswith(s) for s in self.unused_list))
        ]

        # filters out additional sensors
        features = [feat for feat in features if not feat.startswith("2")]

        gt_depth_samples = batch_dict["depth"].reshape(-1, 1).repeat(1, N_samples)
        for i in range(1, self.num_sensors):
            prefix = self.prefix_keys[i]
            gt_depth_samples += batch_dict[f"{prefix}depth"].reshape(-1, 1).repeat(1, N_samples)
        gt_depth_samples /= self.num_sensors
        near = gt_depth_samples * 0.01

        with torch.no_grad():
            det_rays_o = rays_o.clone().detach().unsqueeze(-1)  # (N, 3, 1)
            det_rays_d = rays_d.clone().detach().unsqueeze(-1)  # (N, 3, 1)
            t = (self.bound.unsqueeze(0).to(device) - det_rays_o) / det_rays_d  # (N, 3, 2)
            far_bb, _ = torch.min(torch.max(t, dim=2)[0], dim=1)
            far_bb = far_bb.unsqueeze(-1)
            far_bb += 0.01

        # in case the bound is too large
        max_depth = torch.max(batch_dict["depth"] * 1.2)
        for i in range(1, self.num_sensors):
            prefix = self.prefix_keys[i]
            max_depth = max((max_depth, torch.max(batch_dict[f"{prefix}depth"] * 1.2)))
        far = torch.clamp(far_bb, 0, max_depth)  # NOTE: may cause vis to be incomplete if all depth measurements are 0
        if far.mean() == 0:
            far = torch.ones_like(far) * far_bb

        # add points selected near surface
        if N_surface > 0:
            # since we want to colorize even on regions with no depth sensor readings,
            # meaning colorize on interpolated geometry region,
            # we sample all pixels (not using depth mask) for color loss.
            # Therefore, for pixels with non-zero depth value, we sample near the surface,
            # since it is not a good idea to sample 16 points near (half even behind) camera,
            # for pixels with zero depth value, we sample uniformly from camera to max_depth.

            z_val_surface_list = []

            for i in range(self.num_sensors):
                depth_key = f"{self.prefix_keys[i]}depth"
                gt_none_zero_mask = batch_dict[depth_key] > 0
                gt_none_zero = batch_dict[depth_key][gt_none_zero_mask]
                gt_none_zero = gt_none_zero.unsqueeze(-1)
                gt_depth_surface = gt_none_zero.repeat(1, N_surface)
                t_vals_surface = torch.linspace(0.0, 1.0, steps=N_surface).double().to(device)
                # emperical range 0.05*depth
                z_vals_surface_depth_none_zero = 0.95 * gt_depth_surface * (
                    1.0 - t_vals_surface
                ) + 1.05 * gt_depth_surface * (t_vals_surface)
                z_vals_surface = torch.zeros(batch_dict[depth_key].shape[0], N_surface).to(device).double()
                gt_none_zero_mask = gt_none_zero_mask.squeeze(-1)
                z_vals_surface[gt_none_zero_mask, :] = z_vals_surface_depth_none_zero
                near_surface = 0.001
                far_surface = torch.max(batch_dict[depth_key])
                z_vals_surface_depth_zero = near_surface * (1.0 - t_vals_surface) + far_surface * (t_vals_surface)

                z_vals_surface_depth_zero = z_vals_surface_depth_zero.unsqueeze(0).repeat((~gt_none_zero_mask).sum(), 1)

                z_vals_surface[~gt_none_zero_mask, :] = z_vals_surface_depth_zero

                z_val_surface_list.append(z_vals_surface)

            z_vals_surface = torch.cat(z_val_surface_list, -1)

        t_vals = torch.linspace(0.0, 1.0, steps=N_samples, device=device)

        if not self.lindisp:
            z_vals = near * (1.0 - t_vals) + far * (t_vals)
        else:
            z_vals = 1.0 / (1.0 / near * (1.0 - t_vals) + 1.0 / far * (t_vals))

        if self.perturb > 0.0:
            # get intervals between samples
            mids = 0.5 * (z_vals[..., 1:] + z_vals[..., :-1])
            upper = torch.cat([mids, z_vals[..., -1:]], -1)
            lower = torch.cat([z_vals[..., :1], mids], -1)
            # stratified samples in those intervals
            t_rand = torch.rand(z_vals.shape).to(device)
            z_vals = lower + (upper - lower) * t_rand

        if N_surface > 0:
            z_vals, _ = torch.sort(torch.cat([z_vals, z_vals_surface.double()], -1), -1)

        pts = rays_o[..., None, :] + rays_d[..., None, :] * z_vals[..., :, None]  # [N_rays, N_samples+N_surface, 3]

        pointsf = pts.reshape(-1, 3)

        features_list = []
        c_features_list = []
        c_num_features = 0

        for i in range(self.num_sensors):
            features_list.append([])
            prefix = self.prefix_keys[i]
            num_features = 0

            for feature in features:
                if batch_dict[feature].shape[-1] == 3:
                    num_features += 3
                    features_list[-1].append(batch_dict[f"{prefix}{feature}"].reshape(-1, 3))
                else:
                    num_features += 1
                    features_list[-1].append(batch_dict[f"{prefix}{feature}"].reshape(-1, 1))

        for feature in c_features:
            c_num_features += 3
            c_features_list.append(batch_dict[feature].reshape(-1, 3))

        d_feats_list = []

        for i in range(self.num_sensors):
            d_feats_list.append(torch.cat(features_list[i], 1).to(device).float())
            d_feats_list[-1] = d_feats_list[-1].reshape(-1, num_features)

        d_feats = torch.cat([feats.unsqueeze(2) for feats in d_feats_list], 2)
        c_feats = torch.cat(c_features_list, 1).to(device).float()
        c_feats = c_feats.reshape(-1, c_num_features)

        raw, var = self.eval_points(pointsf, d_feats, c_feats, decoders, c, stage, device)

        raw = raw.reshape(N_rays, N_samples + self.num_sensors * N_surface, -1)

        depth, var_out, uncertainty, color, weights = raw2outputs_nerf_color(
            raw,
            var,
            z_vals,
            rays_d,
            occupancy=self.occupancy,
            device=device,
        )

        if N_importance > 0:
            z_vals_mid = 0.5 * (z_vals[..., 1:] + z_vals[..., :-1])
            z_samples = sample_pdf(
                z_vals_mid,
                weights[..., 1:-1],
                N_importance,
                det=(self.perturb == 0.0),
                device=device,
            )
            z_samples = z_samples.detach()
            z_vals, _ = torch.sort(torch.cat([z_vals, z_samples], -1), -1)

            pts = rays_o[..., None, :] + rays_d[..., None, :] * z_vals[..., :, None]
            pts = pts.reshape(-1, 3)
            raw, var = self.eval_points(pts, decoders, c, stage, device)
            raw = raw.reshape(N_rays, N_samples + N_importance + N_surface, -1)

            depth, uncertainty, color, weights = raw2outputs_nerf_color(
                raw, var, z_vals, rays_d, occupancy=self.occupancy, device=device
            )

        return depth, var_out, uncertainty, color

    def render_img(self, c, decoders, c2w, frame_dict, feature_maps, device, stage):
        """Render out depth, uncertainty, and color images.

        Args:
            c (dict): feature grids.
            decoders (nn.module): decoders.
            c2w (tensor): camera to world matrix of current frame.
            device (str): device name to compute on.
            stage (str): query stage.
            gt_depth (tensor, optional): sensor depth image. Defaults to None.

        Returns:
            depth (tensor, H*W): rendered depth image.
            uncertainty (tensor, H*W): rendered uncertainty image.
            color (tensor, H*W*3): rendered color image.
        """

        img_dict = {}

        with torch.no_grad():
            H = self.H
            W = self.W

            rays_o, rays_d = get_rays(H, W, self.fx, self.fy, self.cx, self.cy, c2w, device)
            rays_o = rays_o.reshape(-1, 3)
            rays_d = rays_d.reshape(-1, 3)

            depth_list = []
            d_var_list = []
            uncertainty_list = []
            color_list = []

            ray_batch_size = self.ray_batch_size

            for feature in feature_maps:
                if frame_dict[feature].shape[-1] == 3:
                    img_dict[feature] = frame_dict[feature].reshape(-1, 3)
                else:
                    img_dict[feature] = frame_dict[feature].reshape(-1)
                for n in range(self.patch_size):
                    for m in range(self.patch_size):
                        mlp_index = n * self.patch_size + m
                        temp_feature_map = torch.roll(
                            frame_dict[feature],
                            shifts=(n - self.patch_size // 2, m - self.patch_size // 2),
                            dims=(0, 1),
                        )
                        if frame_dict[feature].shape[-1] == 3:
                            img_dict[f"{feature}_{mlp_index}"] = temp_feature_map.reshape(-1, 3)
                        else:
                            img_dict[f"{feature}_{mlp_index}"] = temp_feature_map.reshape(-1)

            img_dict["rays_o"] = rays_o.float()
            img_dict["rays_d"] = rays_d.float()

            batch_dict = {}

            for i in range(0, rays_d.shape[0], ray_batch_size):
                rays_d_batch = rays_d[i : i + ray_batch_size]
                rays_o_batch = rays_o[i : i + ray_batch_size]

                for feature in feature_maps + ["rays_o", "rays_d"]:
                    batch_dict[feature] = img_dict[feature][i : i + ray_batch_size]
                    for mlp_index in range(self.patch_size**2):
                        if feature not in ["rays_o", "rays_d"]:
                            batch_dict[f"{feature}_{mlp_index}"] = img_dict[f"{feature}_{mlp_index}"][
                                i : i + ray_batch_size
                            ]
                ret = self.render_batch_ray(c, decoders, rays_d_batch, rays_o_batch, batch_dict, device, stage)

                depth, d_var, uncertainty, color = ret
                depth_list.append(depth.double())
                d_var_list.append(d_var.double())
                uncertainty_list.append(uncertainty.double())
                color_list.append(color)

            depth = torch.cat(depth_list, dim=0)
            d_var = torch.cat(d_var_list, dim=0)
            uncertainty = torch.cat(uncertainty_list, dim=0)
            color = torch.cat(color_list, dim=0)

            depth = depth.reshape(H, W)
            d_var = d_var.reshape(H, W, -1)
            uncertainty = uncertainty.reshape(H, W)
            color = color.reshape(H, W, 3)
            return depth, d_var, uncertainty, color


#元组
"""
tup1=("google","baidu","bilibili","pixiv")
print(tup1[-1],tup1[1:3])
inter = len(tup1)
print(inter)
minn = min(tup1)
print(minn) #按字典序
add = id(tup1)
print(add)
tup1=(1,2,3,4,5,6)
add = id(tup1)
print(add)
"""

#集合
"""
basket = {'apple','orange','orange','pear','banana'}
print(basket)
boolen = 'apple' in basket
print (boolen)
basket.add('wanghao')
print(basket)
basket.remove('wanghao')#如果元素被移除之前不存在，会发生错误
basket.discard('wanghao')#如果元素被移除之前不存在，不会发生错误
print(basket)
thisset = set(("google","Runoob","Taobao"))
booln = "google" in thisset  #判断这个元素是否在集合中存在
print (booln)
thisset.clear()  #清除所有元素
print(thisset)
 while len(thisset) > 0 :
  x=thisset.pop()
  print(x)
"""

#条件控制
"""
var1 = 0
var2 = 1
while  var1+var2 < 100:
    if ((var1+var2)%3 ==0):
        print("var1=",var1,"var2=",var2)
    elif ((var1+var2)%3 ==1):
        print("sorry1")
    else :
        print("sorry2")
    var1+=1 注意和if else等对齐
    var2+=1

"""

#循环语句
"""
#python里面没有do while循环，有for 和 while
#1.0 while 的基础用法
inter1 = 0
while inter1 < 10:
  print(inter1)
  inter1+=1
#while (inter1) : print("hello") #写在一行也是可以的

#1.1 while-else的用法
count =0
while count < 5 :
  print (count,"小于5")
  count +=1
else :
  print (count,"大于或等于5")

#2.0 for in  -else (break)循环
languages= {"C","C++","Perl","Python"}
for x in languages :
  print(x)

sites = ["baidu","google","runoob","taobao"]
for site in sites:
   if site == "runoob" :
     print("菜鸟教程！")
     break
   print("循环数据"+ site)
else :
   print("没有循环数据")
#2.1 range 函数的使用
for i in range (5) :      #指定0-n
  print(i)
for i in range (5,9) :    #指定区间 5-8
  print(i)
for i in range (1,9,3) :  #指定步长 3 6 8
  print(i)
for i in range (-1,-100,-30) :#负数步长
  print(i)
pass
list(range(5))
[0,1,2,3,4]
print(list) #?这是啥意思呢
"""

#生成器和迭代器

"""
#迭代器 的基本方法 iter()和next(),用来访问 集合 元素
list =[1,2,3,4]
it =iter(list)
print(next(it)) #>>>1
print(next(it)) #>>>2
for x in it :
   print(x,end=" ") #>>>3 4

import sys         #引入sys模块
list2 = [1,2,3,4]
it = iter(list)
while True :
 try:              #T要大写
     print(next(it))
 except StopIteration :  #用于标识迭代的完成，防止死循环的出现。（else : raise StopIteration)
     #sys.exit(0)    #正常结束
     break
print("pause,now another one")

class MyNumbers:
    def __iter__(self):
        self.a = 1
        return self
    def __next__(self):
        x=self.a
        self.a+=1
        return x
myclass = MyNumbers()
myiter = iter(myclass)
for i in range(5) :
    print(next(myiter))
#StopIteration 的使用
class MyNumber :
    def __iter__(ass): #用def来定义一个函数
        ass.b = 1
        return ass
    def __next__(ass):
        if ass.b <= 20:
            x=ass.b
            ass.b += 1
            return x
        else :
            raise StopIteration
myclass = MyNumber()
myiter = iter(myclass)
for i in myiter :
    print(x)

#生成器
import sys
def fibonacci(n):
    a,b,counter=0,1,0
    while True :
         if(counter > n):
             return
         yield a
         a,b = b,a+b
         counter +=1
f=fibonacci (10) # f 是一个迭代器，由生成器返回生成
while True :
    try : #我们把可能发生错误的语句放在try里面，用except来处理异常
        print(next(f),end=" ")
    except StopIteration:
        sys.exit(0)
"""

"""
#函数的定义和使用
def max(a,b):
    if a>b:
        return a
    elif a<b :
        return b
    else :
        return a
x=max(2,3)
print(x)
print(" ")
def printme(str):#传入什么字符就在打印一次
    print(str)
    return
printme("ass")
printme("we")
printme(str="can")
print(" ")

#各种传参数
"传不可变对象实例"
def change (a):
    print(id(a)) #原有形参d的内存地址
    a=10    #内部形参被修改
    print(id(a))
a=1
print(id(a)) #与change函数第一个打印的值相同
change(a)    #10的地址
print(" ")
"传可变对象实例"
def changeme(mylist):
    mylist.append([1,2,3,4])
    print("函数内取值",mylist)
    return
mylist=[10,20,30]
changeme (mylist)
print("函数外取值",mylist)
print(" ")

"传入参数时可以不按指定顺序"
def printf(name,age):
    "打印任何传入的字符串"
    print("名字：",name)
    print("年龄",age)
    return
printf(age=50,name="runoob")
print(" ")
"有默认参数时"
def prints(name,age=50):
    print(name," ",age)
    return
x=input()
prints(x,50)
# prints(x,age)   这个会报错！
prints(x)
print(" ")

#不定长参数
'加了*的参数会以元组（tuple）的形式导入，存放所有未命名的变量参数'
def printinfo(arg1,*vartuple):
    "打印任何传入的参数"
    print("输出")
    print(arg1)
    print(vartuple)
printinfo(70,60,50)    #>>>70 (60,50)
printinfo(70,60,50,40) #>>>70 （60，50，40）
print(" ")
"若没有指定参数，他就是一个空元组。我们也可以向不同函数传递未命名的变量"
def print_s(arg1,*var):
    "打印任何传入的参数"
    print("输出:")
    print(arg1)
    for i in var:
        print(i)
    return
print_s(10)
print_s(70,60,50)
print(" ")
"若参数带两个星号**"
def print_f(arg1,**var):#加了两个星号的参数会以字典的形式导入
    print("输出：")
    print(arg1)
    print(var)
print_f(1,a=2,b=3)
print(" ")
"形参表中的星号也可以单独出现，*后的所有参数必须用关键字传入"
def f(a,b,*,c,d):
    return a+b+c+d
#f(1,2,3,4)会报错qwq
#f(1,2,c=3,4)也会报错qwq
x=f(1,2,c=3,d=4)
print(x)
print(" ")

#匿名函数lambda
"就是结构体很简单的函数，只有一句话捏，太简单了以至于不用命名"
x=lambda a:a+10
print(x(5))
sum =lambda var1,var2:var1+10+var2
print(sum(1,2))#冒号后应该为返回值
print(" ")
"我们也可以把匿名函数封装在一个函数内，使用同样的代码来创建多个匿名函数"
def myfunc(n):
    return lambda a:a*n
doubler =myfunc(2)#注意！doubler和tripler后面没有（）！
tripler =myfunc(3)
print(doubler(11)) #2*11
print(tripler(11)) #3*11
print(" ")
'''
#py3.8新增加的：强制位置参数(指定位置参数），msc2019好像运行不了qwq
def f(a,b,/,c,d,*,e,f):
    print(a,b,c,d,e,f)
'正确的写法：'f(10,20,30,d=40,e=50,f=60)
#错误的写法1： f(10,b=20bc=30,e=50,f=60),因为b不能使用关键字参数的方式
#错误的写法2： f(10,20,30,40,50,f=60),因为e必须使用关键字参数的方式
'''
"""
"""
#输入和输出
#输出美化的方式:两种输出方式：表达式语句和print（）函数
s = 'hello world'
print(str(s))#str(s)返回一个用户易读的表达形式
print(repr(s))#repr(s)返回一个解释器易读的表达形式
print(repr((1,2,('google\n','baidu'))))
"一个高级一点的输出方式"
for x in range(1,11):
    print(repr(x).rjust(2),repr(x*x).rjust(3),end=' ')
    print(repr(x*x*x).rjust(4))
"对于上面这种格式化（rjust右对齐填充空格，类似的还有ljust，center）的，我们更经常用print.format"
for x in range (1,11):
    print('{0:2d} {1:3d} {2:4d} '.format(x,x*x,x*x*x))
"还有不够位数则自动在数字左侧补0的.zfill（）"
print('12'.zfill(5))
print('-12.5'.zfill(7))#正负号和小数点都算位置
print('3.14159265359'.zfill(5))#超出就不补了，也不限制
"str.format的用法如下"
print('{}网址："{}!"'.format('b站','www.bilibili.com'))
print('{0}和{1}'.format('google','baidu'))
print('{1}和{0}'.format('google','baidu')) #里面的数字代表出现的次序，从0开始
print('{name}网址：{site}'.format(name='b站',site='www.bilibili.com'))
print('{0}和{1}和{other}'.format('ass','we',other='can'))
"一些格式化的转变,用冒号：实现"
table ={'google':1,'baidu':2,'taobao':3}
for name,number in table.items():
    print('{0:10}==>{1:10}'.format(name,number))
import math
print('常量PI的近似值为{0:.3f}'.format(math.pi))#0的意思是，冒号后的意思是至少有这么多宽度
"如果你有一个很长的格式化字符串, 而你不想将它们分开, 那么在格式化时通过变量名而非位置会是很好的事情。"
"最简单的就是传入一个字典, 然后使用方括号 [] 来访问键值"
print('baidu:{0[baidu]:d};google:{0[google]:d};taobao:{0[taobao]:d}'.format(table))

#旧式字符串格式化-用 % 操作符
import math
print('PI的值可以近似为：%5.3f'% math.pi)#注意，引号后没有逗号

#读取键盘输入-input()
str =input('请输入：')
print("你个憨憨输入了:",str)

#读写文件，文件处理：暂时略，详见runoob.com/python3/python3-inputoutput.html
"""

#面向对象编程（类class）
class myClass:
    i=12345
    def f(self):
     return 'hello world'
x = myClass()
print('这个类的数字是：',x.i)
print('这个类输出了：',x.f())
"类有一个名为__init__()的构造方法，在类实例化（即用类的定义来创建一个实例，如上面x=myClass())的时候会自动调用"
class Complex :
    def __init__(self,realpart,imagpart):#self 是惯用的参数名称，代表类的实例（不是类），但是它不是关键字，可以替换成其他的。
        self.r=realpart
        self.i=imagpart
x=Complex(3.0,-4.5)
print(x.r,x.i)
#类的方法（类里面的函数,形参表必须包含self，且作为第一个参数，代表类的实例）
class people :
    name=''
    age =0
    __weight =0
    def __init__(self,n,a,w):
        self.name =n
        self.age =a
        self.__weight =w
    def speak(self):
        print("%s 说：我%d 岁,%d 公斤。"%(self.name,self.age,self.__weight))
p=people('daonan',18,30)
p.speak()

#类的继承


"""
Magic:

Programa que recebe uma série de cartas, atributos e uma ordem de prioridade de atributos
e devolve as cartas ordenadas com base nessa ordem de prioridade utilizando o algoritmo de
insertion sort.
"""

def imprimir_insercoes(lista_ordenada, n_insercoes):
  for atributo in lista_ordenada:
    print('{:15s}'.format(atributo[0]), ''.join('{:>10}'.format(item) for item in atributo[1:]))
  print('Insercoes realizadas:', n_insercoes)

def criar_matriz_ordenada(entrada):
  return [list(_.values()) for _ in entrada]

def insertion_sort(lista, ordem):
  insercoes = 0
  for b in reversed(ordem):
    for i in range(1, len(lista)):
      chave = lista[i]
      t = i
      eh_insercao = False
      while (t > 0) and (chave[b] > lista[t-1][b]):
        eh_insercao = True
        lista[t] = lista[t - 1]
        t -= 1
      lista[t] = chave
      if eh_insercao:
        insercoes += 1

  return insercoes

def pedir_entrada():
    quantidade_de_cartas = int(input())
    atributos = input().split()
    lista_entradas = []
    for _ in range(quantidade_de_cartas):
        lista_entradas.append(dict(zip(atributos, input().split()))) # lista de dicionários

    ordem_de_prioridade = input().split()

    for i in (ordem_de_prioridade):
      for j in range(len(lista_entradas)):
        lista_entradas[j][i] = int(lista_entradas[j][i])

    return lista_entradas, ordem_de_prioridade

def main():
    entrada, ordem_de_prioridade = pedir_entrada()
    insercoes = insertion_sort(entrada, ordem_de_prioridade) # algoritmo pronto
    lista_ordenada = criar_matriz_ordenada(entrada) # cria uma matriz onde cada linha representa um animal
    imprimir_insercoes(lista_ordenada, insercoes)

main()


import  urllib2
def get_realtime(id):
    if id[0] == '6':
            cmd = "http://hq.sinajs.cn/list=sh%s"
    else:
        cmd = "http://hq.sinajs.cn/list=sz%s"
    if len(id) < 1:
        return
    id = id.zfill(6)
    cmd = cmd%id
    rs = urllib2.urlopen(cmd)
    rs = rs.read()
    items = rs.split(',')
    if len(items) < 3:
        return ''
    name = items[0].strip().split('"')[1]
    now = float(items[3])
    begin = float(items[2])
    rat = round((now - begin)/begin, 4)
    if rat > 0:
        rat = "+{:.2%}".format(rat)
    else:
        rat = "{:.2%}".format(rat)
    return id, name, rat


import contextlib
import re
from itertools import zip_longest
from typing import List

import nlpaug.augmenter.char as nac

from azimuth.config import PerturbationTestingConfig
from azimuth.types.perturbation_testing import PerturbationType, PerturbedUtteranceDetails
from azimuth.utils.ml.third_parties.contractions import ContractionExpansions
from azimuth.utils.validation import assert_not_none

"""
This file contains functions that perturb an utterance, and can be used by
PerturbationTestingModule.
"""

re_punctuation = re.compile(r"(\w+)")

def add_neutral_token(
    original: str, position: str, neutral_tokens: List[str]
) -> List[PerturbedUtteranceDetails]:
    """Add neutral tokens to an utterance at the position indicated.

    Args:
        original: Original utterance.
        position: prefix or suffix.
        neutral_tokens: list of neutral tokens to add.

    Returns:
        List of perturbed utterance details.

    Raises:
        ValueError: if position is invalid.

    """

    results = []
    for neutral_token in neutral_tokens:
        # Create a perturbed_utterance with a new token as prefix and get the predictions.
        if position == "prefix":
            utterance = f"{neutral_token} {original}"
            perturbation_type = PerturbationType.PreInsertion
        elif position == "suffix":
            utterance = f"{original} {neutral_token}"
            perturbation_type = PerturbationType.PostInsertion
        else:
            raise ValueError("Position not supported.")

        results.append(
            PerturbedUtteranceDetails(
                perturbed_utterance=utterance,
                perturbation_type=perturbation_type,
                perturbations=[neutral_token],
            )
        )
    return results

def neutral_prefix(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Add a neutral prefix to an utterance.

    Args:
        original: Original utterance.
        config: Azimuth Config.

    Returns:
         List of perturbed utterance details.

    """
    behavioral_testing_config = assert_not_none(config.behavioral_testing)
    prefix = behavioral_testing_config.neutral_token.prefix_list

    return add_neutral_token(original, "prefix", prefix)

def neutral_suffix(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Add a neutral suffix to an utterance.

    Args:
        original: Original utterance.
        config: Azimuth Config.

    Returns:
        List of perturbed utterance details.

    """
    behavioral_testing_config = assert_not_none(config.behavioral_testing)
    suffixes = behavioral_testing_config.neutral_token.suffix_list

    return add_neutral_token(original, "suffix", suffixes)

def add_all_neutral_tokens(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Add neutral tokens at the beginning and the end of the utterance.

    Args:
        original: Utterance to perturb.
        config: Azimuth Config.

    Returns:
         List of perturbed utterance details.

    """
    return neutral_prefix(original, config) + neutral_suffix(original, config)

def remove_or_add_final_punctuation(
    original: str, punctuation_sign: str
) -> List[PerturbedUtteranceDetails]:
    """Remove or add the specified punctuation sign at the end of an utterance.

    If an utterance's last character is the specified punctuation, it removes it. Otherwise,
    it either replaces the last character (if it's a period, exclamation mark, question mark or
    coma) or adds it.

    Args:
        original: Utterance to perturb.
        punctuation_sign: Punctuation sign to remove or add.

    Returns:
        List of perturbed utterance details.

    """
    last_char = original[-1]
    # Delete punctuation if last character is the specified punctuation sign.
    if last_char == punctuation_sign:
        perturb_utt = original[:-1]
        perturbation_type = PerturbationType.Deletion
    # Replace with specified punctuation if last character is another punctuation sign.
    elif last_char in [".", "!", "?", ","]:
        perturb_utt = original[:-1].rstrip() + punctuation_sign
        perturbation_type = PerturbationType.Replacement
    # Add specified punctuation otherwise.
    else:
        perturb_utt = original + punctuation_sign
        perturbation_type = PerturbationType.PostInsertion

    return [
        PerturbedUtteranceDetails(
            perturbed_utterance=perturb_utt,
            perturbation_type=perturbation_type,
            perturbations=[punctuation_sign],
        )
    ]

def remove_or_add_final_period(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Remove or add period at the end of an utterance.

    Args:
        original: Utterance to perturb.
        config: Azimuth Config.

    Returns:
        List of perturbed utterance details.

    """
    return remove_or_add_final_punctuation(original, ".")

def remove_or_add_final_question_mark(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Remove or add question mark at the end of an perturbed_utterance.

    Args:
        original: Utterance to perturb.
        config: Azimuth Config

    Returns:
         List of perturbed utterance details.

    """
    return remove_or_add_final_punctuation(original, "?")

def remove_or_add_inside_punctuation(
    original: str, punctuation_sign: str
) -> List[PerturbedUtteranceDetails]:
    """Remove or add a specified punctuation sign inside an utterance.

    It will not touch the punctuation at the end, except if there is just one word in the
    utterance.

    Args:
        original: Utterance to perturb.
        punctuation_sign:  Punctuation sign to add or remove.

    Returns:
        List of perturbed utterance details.

    """
    # If specified punctuation sign is in any position apart from the last, remove it.
    if punctuation_sign in original[:-1]:
        perturb_utt = (
            original[:-1].translate(str.maketrans("", "", punctuation_sign)) + original[-1]
        )
        perturbation_type = PerturbationType.Deletion
    #  Add the specified punctuation sign in the middle of the utterance (or after if there is
    #  only one word. This can produce odd results, such as 2 punctuation signs
    #  one beside the other.
    else:
        all_words = original.split()
        # Position of the added punctuation
        punctuation_position = (len(all_words) - 1) // 2
        perturb_utt = " ".join(
            word + punctuation_sign if pos == punctuation_position else word
            for pos, word in enumerate(all_words)
        )
        perturbation_type = PerturbationType.Insertion
    return [
        PerturbedUtteranceDetails(
            perturbed_utterance=perturb_utt,
            perturbation_type=perturbation_type,
            perturbations=[punctuation_sign],
        )
    ]

def remove_or_add_inside_comma(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Remove or add a comma inside an utterance (not at the end).

    Args:
        original: Utterance to perturb.
        config: Azimuth Config.

    Returns:
        List of perturbed utterance details.

    """
    return remove_or_add_inside_punctuation(original, ",")

def remove_or_add_inside_period(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Remove or add a period inside an utterance (not at the end).

    Args:
        original: Utterance to perturb.
        config: Azimuth Config.

    Returns:
        List of perturbed utterance details.

    """

    return remove_or_add_inside_punctuation(original, ".")

def get_utterances_diff(original_utterance: str, perturbed_utterance: str) -> List[str]:
    return list(set(perturbed_utterance.split()).difference(set(original_utterance.split())))

def typo(original: str, config: PerturbationTestingConfig) -> List[PerturbedUtteranceDetails]:
    """Create different types of typos in an utterance.

    It uses the nlpaug package for this test and introduce mistakes raised by OCR, keyboard typo,
    or common mistakes such as missing a char or swapping two chars. This test will generate
    several perturbed utterances per original utterance, but we limit the number of generated
    augmentations to one per type of augmentation.

    Args:
        original: Utterance to perturb.
        config: Azimuth Config.

    Returns:
       List of perturbed utterance details.

    """

    augmentations = []
    perturbation_types = []
    behavioral_testing_config = assert_not_none(config.behavioral_testing)
    # Special chars limited for all augs to facilitate cleaning up nac spacing/punctuation below
    for nb_typo in range(1, behavioral_testing_config.typo.nb_typos_per_utterance + 1):
        augmentations.extend(
            [
                nac.KeyboardAug(
                    min_char=4,
                    aug_word_max=nb_typo,
                    aug_char_p=0.1 * nb_typo,
                    include_special_char=False,
                ),
                nac.RandomCharAug(
                    action="swap",
                    min_char=4,
                    aug_word_max=nb_typo,
                    aug_char_p=0.1 * nb_typo,
                    spec_char="_",  # Breaks/includes others if min_char=1; nlpaug issue #315
                ),
                nac.RandomCharAug(
                    action="delete",
                    min_char=4,
                    aug_word_max=nb_typo,
                    aug_char_p=0.1 * nb_typo,
                    spec_char="_",  # Breaks/includes others if min_char=1; nlpaug issue #315
                ),
            ]
        )
        perturbation_types.extend(
            [PerturbationType.Replacement, PerturbationType.Swap, PerturbationType.Deletion]
        )

    results = []
    for aug, perturbation_type in zip(augmentations, perturbation_types):
        with contextlib.redirect_stdout(None):
            # While nlpaug fixes their useless print, we ignore it.
            perturbed_utterance = aug.augment(original, n=1)
        # nac alters spacing around punctuation (e.g., around '; after $; before ?) and replaces
        # apostrophes with quotes; revert to original. (See nlpaug issue #313)
        punctuations = re_punctuation.split(original)[::2]
        words = re_punctuation.split(perturbed_utterance)[1::2]
        perturbed_utterance = "".join(
            punctuation + word
            for punctuation, word in zip_longest(punctuations, words, fillvalue="")
        )
        perturbations = get_utterances_diff(original, perturbed_utterance)
        results.append(
            PerturbedUtteranceDetails(
                perturbed_utterance=perturbed_utterance,
                perturbation_type=perturbation_type,
                perturbations=perturbations,
            )
        )
    return results

def remove_or_add_contractions(
    original: str, config: PerturbationTestingConfig
) -> List[PerturbedUtteranceDetails]:
    """Expand or contract relevant expressions when present in the utterance.

    Args:
        original: Utterance to perturb.
        config: Azimuth Config.

    Returns:
       List of perturbed utterance details.

    """
    contract_exp = ContractionExpansions()
    results = []

    def check_original(new_utt, ori_utt, perturbation_type):
        if perturb_utt != ori_utt:
            perturbations = get_utterances_diff(ori_utt, new_utt)
            return PerturbedUtteranceDetails(
                perturbed_utterance=new_utt,
                perturbation_type=perturbation_type,
                perturbations=perturbations,
            )
        else:
            return None

    perturb_utt = contract_exp.expand_contractions(original)
    result = check_original(perturb_utt, original, PerturbationType.Expansion)
    if result:
        results.append(result)

    perturb_utt = contract_exp.contract(original)
    result = check_original(perturb_utt, original, PerturbationType.Contraction)
    if result:
        results.append(result)

    return results


'--------Author Arun------------'
'-------Created 03.04.2018-----'
from PyPDF2 import PdfFileReader, PdfFileWriter
import re

def pdf_splitter(wordpattern):
    pdfFileObj = PdfFileReader(open('filename.pdf', "rb")) # file path.
    pdf_writer = PdfFileWriter()
    pagenumber = pdfFileObj.getNumPages()
    startpage = [] # number of start pages for the subfile or newfile which is created by splitting the pdf base on some pattern/text/content.
    naming = []

    for i in range(1, pagenumber):
        page = pdfFileObj.getPage(i)
        content = (page.extractText())
        if (re.search((wordpattern), content)):
            startpage.append(i)
            name = re.search(wordpattern, content)
            naming.append(name.group())
    print(startpage)

    for k in range(0, (len(startpage))):
            try:
                x, y = startpage[k], startpage[k + 1]

                for m in range(x, y):
                    page = pdfFileObj.getPage(m)
                    pdf_writer.addPage(page)
            except:
                print("Reached Last page")
                for m in range(y, pagenumber):
                    page = pdfFileObj.getPage(m)
                    pdf_writer.addPage(page)
            newpdf = (open('newfilenames%s.pdf' % naming[k], "wb"))# The nefiles/subfiles are created here.
            pdf_writer.write(newpdf)
            newpdf.close
            pdf_writer = PdfFileWriter()

if __name__ == "__main__":
    pdf_splitter('/d{4}') #give some word pattern or word with which you want to split the file.
                          #--For example I have give /d{4},which means pdf file will split into new files  at places where 4 digit numbers are there.