__author__ = 'KATRINA'

'''

krakenDistributions.py [input directory] [kraken file]
- take in the kraken file (obtained from running ~/scripts/fullKraken.sh > filename) (on AbateLab server)
file contents should look like:
"
1000-143-DesulfotaleaPsychrophilaLSv54-1.fq

Processed 143 sequences (21450 bp) ...
143 sequences (0.02 Mbp) processed in 0.012s (695.7 Kseq/m, 104.35 Mbp/m).
  140 sequences classified (97.90%)
  3 sequences unclassified (2.10%) "

- assumes that there are *.fq-kr-ov files in the input directory
- will use the input file kraken file alongside the *.fq-kr-ov to determine the dominant purity

output:
kraken_purity.txt - output file containing filename and purity information

*note: the plotting functions should work, but we don't need a million pie charts,
 so they are commented out by default

'''

import sys
import glob
import os
import matplotlib.pyplot as plt
import operator

#this function is not in use currently
def createPieChart(labels, values):
    plt.pie(values, labels=labels)
    plt.show()

#open each .fq-kr-ov file and load the distribution of classified reads and counts
#append this distribution to the summary dictionary - used to generate the .tsv purity file later
def loadDistributions(array_of_filenames, summary_dictionary):
    massive_dictionary = {}
    total_of_all_reads = {}
    for file in array_of_filenames:
        print(file)
        f = open(file,'r').readlines()
        per_file_total = 0
        species_dict = {}
        for line in f:
            sp = line.split()
            if len(sp)<2: # this is the line corresponding to unclassified
                species_dict['low_classification'] = int(sp[0])
            else:
                species_dict[sp[1]] = int(sp[0])

        print(file.split('/')[-1].split('.')[0])
        summary_dictionary[file.split('/')[-1].split('.')[0]].append(species_dict)
        sorted_species_dict = sorted(species_dict)

        labels = []; values = []
        for v in sorted_species_dict:
            labels.append(v)
            values.append(species_dict[v])
            if v in total_of_all_reads.keys():
                total_of_all_reads[v] += species_dict[v]
            else:
                total_of_all_reads[v] = species_dict[v]

        massive_dictionary[file.split('\\')[-1].split('.')[0]] = [labels, values]
    return massive_dictionary, total_of_all_reads, summary_dictionary

#write the purity data obtained in logic above into a file
def writeOutputFile(summary_dictionary):
    f = open("purity_output.txt",'w')
    for i in summary_dictionary:
        output_string = [i]
        statistics = summary_dictionary[i]
        classified = statistics[0]
        unclassified = statistics[1]

        if (len(statistics) > 2):
            other = statistics[2]
            if classified > unclassified:
                max_species_value = max(other.values())
                total_values = sum(other.values())
                x = max_species_value*classified / total_values
                for k in other:
                    if other[k] == max_species_value:
                        max_species = k
                output_string.append(str(x)) ##TODO !! THIS SHOULD BE A NORMALIZED DECIMAL VALUE IDEALLY
                output_string.append(max_species)

            else:
                output_string.append(str(unclassified))
                output_string.append('unclassified')

            o_sorted = reversed(sorted(other.items(), key=operator.itemgetter(1)))
            for i in o_sorted:
                v = i[0]+":"+str(i[1])
                output_string.append(v)
            f.write('\t'.join(output_string)+"\n")

        else:
            if(unclassified > classified):
                output_string.append(str(unclassified))
                output_string.append('unclassified')
                f.write('\t'.join(output_string)+"\n")

#MAIN SCRIPT STARTS HERE
input_dir = sys.argv[1]
os.chdir(input_dir)
os.system('pwd')
array_of_filenames = glob.glob('./*-ov')

out_file_name=sys.argv[2]
out_file = os.path.join(input_dir,out_file_name)
f = open(out_file_name,'r').readlines()

summary_dictionary = {}
percentage_classified = []
percentage_unclassified = []
bargroups = []
for line in f:
    summary_line = []
    if '.fastq' in line: #note if file names are *.fq this will need to be updated
        bargroups.append(line.strip().split('.')[0])
    if 'sequences classified' in line:
        percent_c = float(line.split('(')[1].split('%')[0])
        percent_u = 100-percent_c
        percentage_classified.append(percent_c)
        percentage_unclassified.append(percent_u)

#add bargroup name, percent classified, and percent unclassified to the summary dictionary
for i in range(len(bargroups)):
    summary_dictionary[bargroups[i]] = [percentage_classified[i],percentage_unclassified[i]]

#create a histogram of percent unclassified reads
'''
fig1 = plt.figure()
plt.hist(percentage_unclassified,bins=40,color='#00bfff')
plt.title('distribution of the percent of unclassified reads')
plt.show()
'''

distributions = loadDistributions(array_of_filenames, summary_dictionary)
stats = distributions[2] ##this will be the summary array which I will want to post-process into .tsv
md = distributions[0] #for plotting the pie charts below

'''#create pie charts for all barcode groups
for x in md:
    print(x)
    print(md[x][0])
    print(md[x][1])
    createPieChart(md[x][0],md[x][1])
'''
totals = distributions[1]
#createPieChart(totals.keys(),list(totals.values()))

writeOutputFile(summary_dictionary)


import torch
import numpy as np
import scipy.io as io
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from mpl_toolkits.axes_grid1 import make_axes_locatable
from skimage.metrics import structural_similarity as ssim

import sys
import argparse
sys.path.append( "../../")
from nfCRT import *

def save_animation(image, time_int, filename):
    """Saves image to file with time labels

    Args:
        image (numpy array): space time image
        time_int (list): list of times
        filename (string): file name to be saved in
    """
    plt.clf()
    fig = plt.figure()
    ax = fig.add_subplot(111)
    div = make_axes_locatable(ax)
    cax = div.append_axes('right', '5%', '5%')
    im = ax.imshow(image[:,:,0], cmap = 'gray', origin = 'lower')
    cb = fig.colorbar(im,cax = cax)

    plt.tick_params(
        axis='both',          # changes apply to the x-axis
        which='both',      # both major and minor ticks are affected
        left = False,
        right = False,
        bottom=False,      # ticks along the bottom edge are off
        top=False,         # ticks along the top edge are off
        labelleft = False,
        labelbottom=False) # labels along the bottom edge are off

    tx = ax.set_title('Frame 0', size = 30)

    vmax = np.max(image)
    vmin = np.min(image)

    def animate(i):
        arr = image[:,:,i]
        im.set_data(arr)
        im.set_clim(vmin,vmax)
        t = int(100*(time_int[i] + 1/2) )/100
        tx.set_text('Time ' + str(t))
    ani = animation.FuncAnimation(fig, animate, save_count = image.shape[2])
    ani.save(filename)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Create animations')
    parser.add_argument('--out_folder',
                        default= 'VPT4/',
                        type = str)
    args = parser.parse_args()
    if torch.cuda.is_available():
        print("Cuda available, Implementing with GPU")
        dev = torch.device('cuda:0')
    else:
        print("Cuda not available, Implementing with CPU")
        dev = torch.device('cpu')

    image = io.loadmat('induced_pressure')['induced_pressure']
    image = (image[::2,::2,:] + image[1::2,::2,:] + image[1::2,::2,:] + image[1::2,1::2,:])/4
    s = image.shape[0]
    nA = image.shape[2]

    time_int = torch.linspace(-1/2, 1/2,nA)
    x_int = torch.linspace(-1/2,1/2,s)
    x1, x0, t =torch.meshgrid(x_int, x_int, time_int)
    x0 = x0.flatten()
    x1 = x1.flatten()
    t = t.flatten()
    x = torch.stack((x0, x1, t))
    x = torch.transpose(x,0,1).float().to(dev)

    save_animation(image, time_int, args.out_folder + 'Phantom.gif')
    image_vec = image.flatten()
    net = torch.load(args.out_folder + 'dynamic_net.pt')
    ave_net = torch.load(args.out_folder + 'static_net.pt')

    dpou = net['pou']
    dC = net['C']
    ave_pou = ave_net['pou']
    ave_C = ave_net['C']

    mx = 3
    mt = 3
    geodim = 3
    P = Polynomial(geodim, mx, mt = mt, st_separability = True )
    ave_P = Polynomial(geodim -1, mx, st_separability = False)

    out = torch.zeros(x.size()[0]).to(dev)
    diff = torch.zeros(x.size()[0]).to(dev)
    ave = torch.zeros(x.size()[0]).to(dev)
    r = int(x.size()[0]/nA)
    for i in range(nA):
        y = x[r*i:r*(i+1),:]
        out[r*i:r*(i+1)] = torch.einsum( 'ij,jk,ki -> i', dpou(y), dC, P(y)).detach()
        ave[r*i:r*(i+1)] = torch.einsum( 'ij,jk,ki -> i', ave_pou(y[:,0:-1]), ave_C, ave_P(y[:,0:-1])).detach()
    diff = out - ave

    Cs = ave_C.cpu().detach().numpy()
    plt.clf()
    plt.imshow(np.abs(Cs))
    plt.colorbar()
    plt.savefig(args.out_folder + "static_coeff_magnitudes.png")

    print("RMSE = ", np.linalg.norm(image_vec - out.cpu().detach().numpy()  )/np.linalg.norm(image_vec))
    print("Error = ", np.linalg.norm(image_vec - out.cpu().detach().numpy()  ))
    print("Norm of Image = ", np.linalg.norm(image_vec))
    out_image = out.detach().cpu().numpy().reshape((s,s,nA))

    SSIM = ssim(image, out_image, dynamic_range = out_image.max()-out_image.min())
    print("SSIM = ", SSIM)
    diff_image = diff.detach().cpu().numpy().reshape((s,s,nA))
    ave_image = ave.detach().cpu().numpy().reshape((s,s,nA))

    error = out_image - image

    save_animation(ave_image, time_int, args.out_folder + "Static_component.gif" )
    save_animation(out_image, time_int, args.out_folder + "Reconstructed.gif" )
    save_animation(diff_image, time_int, args.out_folder + "Dynamic_component.gif" )
    save_animation((image_vec - out.cpu().detach().numpy()).reshape((s,s,nA))  , time_int, args.out_folder + "Error.gif" )

    Cn = dC.cpu().detach().numpy()
    print(Cn.shape)
    plt.clf()
    plt.imshow(np.abs(Cn))
    plt.colorbar()
    plt.savefig(args.out_folder + "dynamic_coeff_magnitudes.png")


import random

def hangman(word=""):
    wrong = 0
    default = ["apple", "banana", "cat", "key", "ball"]
    stages = ["",
              "________           ",
              "|                  ",
              "|        |         ",
              "|        O         ",
              "|      / | \       ",
              "|       / \        ",
              "|                  "]
    if word == "":
        word = default[random.randint(0,len(default))]

    rletters = list(word)
    board = ["__"]*len(word)
    #print(board)
    win = False
    print("Welcome to Hangman")
    while(wrong < len(stages) - 1):
        print(" ")
        guess = input("Guess a letter " )
        if guess in rletters:
            for x in range(len(rletters)):
                if rletters[x] == guess:
                    board[x] = guess
        else:
            wrong += 1
        print(' '.join(board))
        e = wrong + 1
        print('\n'.join(stages[0:e]))
        if "__" not in board:
            print("You Win")
            #print("".join(board))
            win = True
            break
    if not win:
        print("\n".join(stages[0:wrong]))
        print("You lose! it was {}".format(word))

hangman()


from tkinter import *
from tkinter import ttk
from tkinter import filedialog
import pandas as pd

import backend # import backend class from backend.py file
import report_GUI # import report GUI class from the report_GUI.py file
import Edit_GUI # import edit GUI class from the Edit_GUI.py file
import delete_GUI # import delete GUI class from the delete_GUI.py file
import insert_GUI # import insert GUI class from the insert_GUI.py file
import tkinter.messagebox as msgBox

class GUI(backend.backend):

    """
    This constructor method to declare the GUI as soon as the object has been declared
    """
    def __init__(self):

        # Root declaration
        self.root = Tk()
        self.root.geometry("1080x720")
        self.root.title("LAB_CLASS")

        # Declare Label Frame
        self.Segment1 = LabelFrame(self.root, text='')
        self.Segment2 = LabelFrame(self.root, text='MENU')

        self.Segment1.pack(fill='both', expand='yes', padx= 10, pady=10)
        self.Segment2.pack(fill='both', expand='yes', padx= 10, pady=10)

        # Declare title
        self.title = Label(self.Segment1, text='Cafe Management', font=('bold', 30))
        self.title.pack(pady=20)

        # Declare Table
        self.orderBox = ttk.Treeview(self.Segment1, columns=(1,2,3,4), show='headings')
        self.orderBox.heading(1, text="ID")
        self.orderBox.heading(2, text="Name")
        self.orderBox.heading(3, text="Available")
        self.orderBox.heading(4, text="Price")
        self.orderBox.pack(side=BOTTOM, pady=100)

        # Declare Button
        self.add_btn = Button(self.Segment2, text='ADD', font=('regular', 18), bg="white", command=self.add)
        self.del_btn = Button(self.Segment2, text='DELETE', font=('regular', 18), bg="white", command=self.delete)
        self.edit_btn = Button(self.Segment2, text='EDIT', font=('regular', 18), bg="white", command=self.edit)
        self.rep_btn = Button(self.Segment2, text='REPORT', font=('regular', 18), bg="white", command=self.report)
        self.ref_btn = Button(self.Segment2, text='REFRESH', font=('regular', 18), bg="white", command=self.refresh_data)
        self.imp_btn = Button(self.Segment2, text='IMPORT', font=('regular', 18), bg="white", command=self.load)

        self.add_btn.pack(side=LEFT, padx=10)
        self.del_btn.pack(side=LEFT, padx=10)
        self.edit_btn.pack(side=RIGHT, padx=10)
        self.rep_btn.pack(side=RIGHT, padx=10)
        self.ref_btn.pack()
        self.imp_btn.pack()

        self.getData()

        self.root.mainloop()

    def load(self):
        filename = filedialog.askopenfilename(initialdir='/home/livevil/Documents/Software Engineering/Lab_Class/', title='Select File', filetypes=(('csv files','*.csv'), ('All files','*.*')))

        id_df = pd.read_csv(f'{filename}', usecols=[0])
        name_df = pd.read_csv(f'{filename}', usecols=[1])
        stock_df = pd.read_csv(f'{filename}', usecols=[2])
        price_df = pd.read_csv(f'{filename}', usecols=[3])

        id = id_df.values.tolist()
        name = name_df.values.tolist()
        stock = stock_df.values.tolist()
        price = price_df.values.tolist()

        for i in range(0, len(id)):
            self.action(f"INSERT INTO product VALUES ({id[i][0]}, '{name[i][0]}', {stock[i][0]}, {price[i][0]})")

        self.refresh_data()

        return True

    # Delete function for delete button command
    def delete(self):

        # Get the data that selected in the treeview
        selected = self.orderBox.focus()
        val = self.orderBox.item(selected, 'values')

        # Declare delete_GUI object to show the GUI
        delete_GUI.GUI(val)

        return True

    # Add function for add button command
    def add(self):

        # Declare insert_GUI object to show the GUI
        insert = insert_GUI.GUI()

        print('Add GUI is opened')

    # Edit function for edit button command
    def edit(self):

        # Get the data that selected in the treeview
        selected = self.orderBox.focus()
        val = self.orderBox.item(selected, 'values')

        # Declare Edit_GUI object to show the GUI
        Edit_GUI.GUI(val)

        print('Edit GUI is opened')

    # Refresh function for update the data in the treeview
    def refresh_data(self):
        x = self.orderBox.get_children()

        if x != '()':
            for child in x:
                self.orderBox.delete(child)

        self.getData()
        print("Record Updated")

    # GetData function to execute select query and gain all data to be returned
    def getData(self):
        value = self.select('SELECT * FROM product')

        for i in value:
            self.orderBox.insert('', 'end', values=i)

        print("Data successfully taken from database")

    # Edit function for edit button command
    def report(self):

        # Declare delete_GUI object to show the GUI
        rep = report_GUI.GUI()
        print("Generating Report...")

start = GUI()


from flask import Flask, render_template, request, flash

app = Flask(__name__) #to intialize our application, takes in our main module represented by "__name__". This creates a class for our app
app.config['SECRET_KEY'] = 'super secret key'

@app.route("/hello") #"/hello" represents the last part of the url, meaning www.myurl.com/hello . This is a decorator of a function, so it requires a function.
def index():
    flash("what's your name yowo?")
    return render_template("index.html")

@app.route("/greet", methods=["POST", "GET"])
def greet():
    flash("Hi " +str(request.form['name_input']) + ", great to see you")
    return render_template("index.html")


import pandas as pd

toxic = pd.read_csv("../../data/src_new/kaggle-toxic_train-V2.CSV")  # read kaggle_toxic dataset

""" Build toxic V2
toxic = pd.read_csv("data/src_new/kaggle-toxic_train-clean.CSV")  # read kaggle_toxic dataset
toxic["class"] = toxic[["toxic", "severe_toxic"]].max(axis=1)  # if it's toxic or severely toxic, class = 1

# (number of abusive examples, i.e. class=1) / (number of examples total)
percent_abusive = np.round(len(toxic[toxic["class"] == 1]) / len(toxic) * 100, 2)

# export dataset
toxic = toxic.drop(["toxic","severe_toxic","obscene","threat","insult","identity_hate"], axis=1)
toxic.to_csv("data/src_new/kaggle-toxic_train-V2.CSV", index=False)

print(f"percent abusive: {percent_abusive}")
"""

"""
# count # of trump comments in original
# toxic_trump = boost_data(toxic, "toxic", manual_boost=["trump"], verbose=True)

# Count most common words in kaggle_toxic abusive posts
stop = stopwords.words("english")
toxic["nostop"] = toxic["comment_text"].apply(lambda x: " ".join([word.replace("\"", "") for word in x.split() if word.lower() not in stop]))

abusive_only = toxic[toxic["class"] == 1]
toxic_count = Counter(" ".join(abusive_only["nostop"]).split()).most_common(200)
print(toxic_count)
"""

# count number of abusive entries per lexicon
base = pd.read_csv("../../data/lexicon_wiegand/baseLexicon.txt", delimiter="\t", names=["word", "class"])
expanded = pd.read_csv("../../data/lexicon_wiegand/expandedLexicon.txt", delimiter="\t", names=["word", "class"])
manual = pd.read_csv("../../data/lexicon_manual/lexicon.manual.all.csv", header=0)

base_abusive = base[base['class']]
exp_abusive = expanded[expanded['class'] > 0]
manual_abusive = manual[manual['class']]

print(f"manual: {len(manual_abusive)}")
print(f"base: {len(base_abusive)}")
print(f"expanded: {len(exp_abusive)}")

non_abusive = data[data["class"] == 0]
abusive_only = data[data["class"] == 1]


from flask import Blueprint, session, render_template, redirect, request, \
    url_for, current_app

topright_api = Blueprint('topright_api', __name__)

@topright_api.route('/overview')
def overview():
    if 'email' not in session:
        return redirect(url_for('authentication_api.login'))
    access_logger = current_app.cluster.get_access_logger(request,
                                                          session['email'],
                                                          init_log=False)

    return render_template('overview.jinja2')

@topright_api.route('/history')
def history():
    if 'email' not in session:
        return redirect(url_for('authentication_api.login'))
    access_logger = current_app.cluster.get_access_logger(request,
                                                          session['email'],
                                                          init_log=False)

    history = access_logger.get_user_access_log(session['email'])
    headings = ['method', 'endpoint',
                'level', 'message',
                'form_variables', 'get_variables',
    ]

    if 'advanced' in request.args:
        headings += ['request', 'request_update', ]

    description = '''Views:
        Simple |
        Advanced
        '''.format(request.url_rule)

    return render_template('history.jinja2',
                           history_log=True,
                           title='User History',
                           description=description,
                           headings=headings,
                           history=history)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Дана последовательность слов. Проверить, правильно ли в ней записаны буквосочетания
жи и ши."""

if __name__ == '__main__':
    s = input()
    s1 = s.replace('жы', 'жи')
    s1 = s1.replace('шы', 'ши')

    if s1 == s:
        print('Ok')
    else:
        print('Есть ошибки')

индивидуальные/2.py


#!/usr/bin/env python3

import argparse
import sys
import os
import numpy as np

sys.path.append('tools')
import plot

import torch
from torch.utils.data import DataLoader
import torch.optim as optim

sys.path.append('archs')

def get_args():
  parser = argparse.ArgumentParser(
    description="""This script trains a separation neural network""")

  parser.add_argument("arch_file", metavar="arch-file", type=str,
                      help="DNN architecture file")
  parser.add_argument("gpu_id", metavar="gpu-id", type=int,
                      help="GPU ID")
  parser.add_argument("data_dir", metavar="data-dir", type=str,
                      help="Training data directory")
  parser.add_argument("dirout", type=str,
                      help="Output directory")

  parser.add_argument("--cv-data-dir", type=str,
                      help="Cross validation data directory",
                      default="")
  parser.add_argument("--train-copy-location", type=str,
                      help="Copy training data here for I/O purposes",
                      default="")
  parser.add_argument("--model-config", type=str,
                      help="Config file for DNN",
                      default="")
  parser.add_argument("--batch-size", type=int,
                      help="Batch size",
                      default=100)
  parser.add_argument("--start-epoch", type=int,
                      help="Epoch to start from",
                      default=0)
  parser.add_argument("--num-epochs", type=int,
                      help="Total number of training epochs",
                      default=200)
  parser.add_argument("--learning-rate", type=float,
                      help="Learning rate",
                      default=0.001)

  args = parser.parse_args()
  return args

def load_losses(filename, loss_array):
  with open(filename, 'r') as lossF:
    for line in lossF:
      split = line.rstrip().split()
      loss_array[0].append(int(split[0]))
      loss_array[1].append(np.float(split[1]))

def main():
  args = get_args()
  global m
  print("Using "+args.arch_file+" DNN architecture")
  m = __import__(args.arch_file)

  print("Using GPU", args.gpu_id)
  torch.cuda.set_device(args.gpu_id)
  tmp = torch.ByteTensor([0])
  tmp.cuda()

  int_model_dir = args.dirout+'/intermediate_models/'
  plot_dir = args.dirout+'/train_stats/plots/'
  loss_file = args.dirout+'/train_stats/train_loss.txt'
  cv_loss_file = args.dirout+'/train_stats/cv_loss.txt'

  print("loading datset")
  dataset = m.TrainSet(args.data_dir, args.train_copy_location)
  dataloader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, collate_fn=dataset.collator, num_workers=1)
  if args.cv_data_dir:
    cv_dataset = m.TrainSet(args.cv_data_dir)
    cv_dataloader = DataLoader(cv_dataset, batch_size=args.batch_size, collate_fn=cv_dataset.collator)

  print("initializing model")
  if args.model_config:
    kwargs = dict()
    for line in open(args.model_config):
      kwargs[line.split('=')[0]] = line.rstrip().split('=')[1]
    model = m.SepDNN(args.gpu_id, **kwargs)
  else:
    model = m.SepDNN(args.gpu_id)
  model.cuda()
  optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
  print("using lr="+str(args.learning_rate))

  epoch_losses = [[], []]
  epoch_cv_losses = [[], []]
  lossF = open(loss_file, 'a')
  if args.cv_data_dir:
    cv_lossF = open(cv_loss_file, 'a')

  if args.start_epoch == 0:
    torch.save(model.state_dict(), int_model_dir+'init.mdl')
  else:
    model.load_state_dict(torch.load(int_model_dir+str(args.start_epoch).zfill(3)+'.mdl', map_location=lambda storage, loc: storage.cuda()))
    load_losses(loss_file, epoch_losses)
    if args.cv_data_dir:
      load_losses(cv_loss_file, epoch_cv_losses)

  print("training")
  for epoch in range(args.start_epoch, args.num_epochs):
    epoch_loss = 0.0
    epoch_norm = 0
    for i_batch, sample_batch in enumerate(dataloader):
      loss, norm = m.compute_loss(model, epoch, sample_batch)
      epoch_loss += loss.detach().cpu().numpy() * norm.detach().cpu().numpy()
      epoch_norm += norm.detach().cpu().numpy()
      loss.backward()
      torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25)
      optimizer.step()

    if args.cv_data_dir and epoch % 5 == 4:
      cv_epoch_loss = 0.0
      cv_epoch_norm = 0
      model.eval()
      with torch.no_grad():
        for i_batch_cv, sample_batch_cv in enumerate(cv_dataloader):
          if i_batch_cv == 0:
            cv_loss, cv_norm = m.compute_cv_loss(model, epoch, sample_batch_cv, plot_dir+'epoch'+str(epoch+1).zfill(3))
          else:
            cv_loss, cv_norm = m.compute_cv_loss(model, epoch, sample_batch_cv)
          cv_epoch_loss += cv_loss.detach().cpu().numpy() * cv_norm.detach().cpu().numpy()
          cv_epoch_norm += cv_norm.detach().cpu().numpy()
      model.train()
      print("For epoch: "+str(epoch+1).zfill(3)+" cv set loss is: "+str(cv_epoch_loss/cv_epoch_norm))
      cv_lossF.write(str(epoch+1).zfill(3)+' '+str(cv_epoch_loss/cv_epoch_norm)+'\n')
      cv_lossF.flush()
      epoch_cv_losses[0].append(epoch+1)
      epoch_cv_losses[1].append(cv_epoch_loss/cv_epoch_norm)

    print("For epoch: "+str(epoch+1).zfill(3)+" loss is: "+str(epoch_loss/epoch_norm))
    lossF.write(str(epoch+1).zfill(3)+' '+str(epoch_loss/epoch_norm)+'\n')
    lossF.flush()
    epoch_losses[0].append(epoch+1)
    epoch_losses[1].append(epoch_loss/epoch_norm)
    if epoch % 5 == 4:
      print("Saving model for epoch "+str(epoch+1).zfill(3))
      torch.save(model.state_dict(), int_model_dir+str(epoch+1).zfill(3)+'.mdl')
      os.system("mkdir -p "+plot_dir+'epoch'+str(epoch+1).zfill(3))
      plot.plot_loss(epoch_losses, epoch_cv_losses, plot_dir+'epoch'+str(epoch+1).zfill(3)+'/Loss_'+str(epoch_losses[0][0]).zfill(3)+'-'+str(epoch+1).zfill(3)+'.png')
    sys.stdout.flush()

  torch.save(model.state_dict(), args.dirout+'/final.mdl')
  plot.plot_loss(epoch_losses, epoch_cv_losses, plot_dir+'Loss_'+str(epoch_losses[0][0]).zfill(3)+'-'+str(args.num_epochs).zfill(3)+'.png')

if __name__ == '__main__':
  main()


import numpy as np
from vtk.util import numpy_support
import vtk
import cv2

def npMatrixToVtkMatrix(matrix):
    retMat = vtk.vtkMatrix4x4()
    shape = matrix.shape
    for r in range(0,shape[0]):
        for c in range(0,shape[1]):
            retMat.SetElement(r,c,matrix[r,c])
    return retMat

def vtkMatrixtoNpMatrix(matrix):
    return np.array(matrix.GetData(), (4,4))

# Sets up a three channel empty vtkImageData object
def makeVtkImage(imgDims):
    ''' Creates a blank vtkImageData object of specified dimensions
        Input: imgDims (int, int): desired width and height
        Output: imData (vtkImageData): empty VTK image with three
                                       channels for RGB
    '''
    imData = vtk.vtkImageData()
    imData.SetDimensions(imgDims[1],imgDims[0],1) #set dimensions as necessary
    imData.SetOrigin(0,0,0) #set origin as necessary
    imData.SetSpacing(1, 1, 1) #set spacing as necessary
    # Set number of channels (3)
    if vtk.VTK_MAJOR_VERSION <= 5:
        imData.SetNumberOfScalarComponents(3)
        imData.SetScalarTypeToUnsignedChar()
        imData.Update()
    else:
        imData.AllocateScalars(vtk.VTK_UNSIGNED_CHAR, 3)
    return imData

def numpyToVtkImage(numpyData,VTKImageData):
    ''' Copies a numpy array to a vtkImageData object in place.
        Used for updating existing vtkImageData with new pixel
        data.
        https://pyscience.wordpress.com/2014/09/06/numpy-to-vtk-converting-your-numpy-arrays-to-vtk-arrays-and-files/
        was extremely useful
    '''
    if len(numpyData.shape)==3 and numpyData.shape[2] > 2 and numpyData.shape[2] <= 4:
        # Data from CV comes as BGR and vtkImageData displays as RGB so we convert
        # rgb = cv2.cvtColor(numpyData, cv2.COLOR_BGR2RGB)
        rgb = numpyData.copy()
        rgb[:,:,0:3] = cv2.cvtColor(numpyData[:,:,0:3], cv2.COLOR_BGR2RGB)
        # Get a vtkDataArray object containing numpy pixel data
        VTK_data = numpy_support.numpy_to_vtk(num_array=rgb[::-1].ravel(),
                                              deep=True,
                                              array_type=vtk.VTK_UNSIGNED_CHAR)
        # Set number of channels in pixel data
        VTK_data.SetNumberOfComponents(numpyData.shape[2])
    else:
        #Get a vtkDataArray object containing pixel data
        VTK_data = numpy_support.numpy_to_vtk(num_array=numpyData[::-1].ravel(),
                                              deep=True,
                                              array_type=vtk.VTK_UNSIGNED_CHAR)
    VTKImageData.GetPointData().SetScalars(VTK_data)

def vtkImageToNumpy(VTKImageData):
    ''' Copies pixel data from a vtkImageData and returns a new numpy array
    '''
    size = VTKImageData.GetDimensions()
    shape = (size[1],size[0],VTKImageData.GetNumberOfScalarComponents())
    VTK_data = VTKImageData.GetPointData().GetScalars()
    numpyData = numpy_support.vtk_to_numpy(VTK_data)
    numpyData = numpyData.reshape(shape)
    numpyData = numpyData[::-1]
    return numpyData

def actorFromStl(stlPath):
    ''' Reads in an STL file and returns a vtkActor
        object that the vtkRenderer can use
    '''
    reader = vtk.vtkSTLReader()
    reader.SetFileName(stlPath)
    mapper = vtk.vtkPolyDataMapper()
    if vtk.VTK_MAJOR_VERSION <= 5:
        mapper.SetInput(reader.GetOutput())
    else:
        mapper.SetInputConnection(reader.GetOutputPort())
    # Create actor that we will position manually
    actor = vtk.vtkActor()
    actor.SetMapper(mapper)
    return actor

def setupRenWinForRegistration(renWin, bgImage, fgImage, camIntrinsic):
    ''' Sets up the render window for registration in place
        Input:
            renWin (vtkRenderWindow): render window to set up with
                                      actors and renderers
            bgImage (vtkImageData): image to put in the background
                                    that will be updated with video
                                    footage and masking feedback
            camIntrinsic (np.ndarray): 4x4 camera intrinsic matrix from
                                       OpenCV calibration
    '''

    renWin.SetNumberOfLayers(3)

    # Create main renderer for stl object
    ren = vtk.vtkRenderer()
    ren.SetLayer(1)
    renWin.AddRenderer(ren)
    imgDims = bgImage.GetDimensions()
    renWin.SetSize(imgDims[0],imgDims[1])
    camera = cameraFromMatrix(camIntrinsic,imgDims,imgDims)
    ren.SetActiveCamera(camera)

    # Create background renderer for static object
    backgroundRen = vtk.vtkRenderer()
    backgroundRen.SetLayer(0)
    backgroundRen.InteractiveOff()
    backgroundRen.SetBackground(1, 1, 1)
    # Create background object for showing video feed.
    bgCamera, backgroundActor = _makeImagePlane(bgImage)
    # Setup camera
    backgroundRen.AddActor(backgroundActor)
    backgroundRen.SetActiveCamera(bgCamera)
    # Add background renderer to window
    renWin.AddRenderer(backgroundRen)

    # Create background renderer for static object
    foregroundRen = vtk.vtkRenderer()
    foregroundRen.SetLayer(2)
    foregroundRen.InteractiveOff()
    # Create background object for showing video feed.
    fgCamera, foregroundActor = _makeImagePlane(fgImage)
    # Setup camera
    foregroundRen.AddActor(foregroundActor)
    foregroundRen.SetActiveCamera(fgCamera)
    # Add background renderer to window
    renWin.AddRenderer(foregroundRen)

    # Way of getting camera: renWin.GetRenderers().GetFirstRenderer()
    return ren, backgroundRen, foregroundRen

class vtkTimerCallback():
   # Callback that renders on a timer
    def __init__(self, renWin, parent=None):
        self.renderWindows = [renWin]
        self.parent = parent
    def execute(self,obj,event):
        # for renWin in self.renderWindows:
        #     if self.parent is None:
        #         renWin.Render()
        #     else:
        #         if self.parent.isVisible():
        #             renWin.Render()
        self.update()
    def addRenWin(self,renWin):
        self.renderWindows.append(renWin)
    def update(self):
        pass

def matrixFromCamInfo(camInfo):
    # Get intrinsic matrix of rectified image
    intrinsicMatrix = np.reshape(camInfo.P,(3,4))[0:3,0:3]
    # Get camera extrinsic matrix
    extrinsicMatrix = np.identity(4)
    # Get extrinsic rotation
    extrinsicMatrix [0:3,0:3] = np.reshape(camInfo.R,(3,3))
    # Get baseline translation (will usually be zero as this is for left camera)
    xBaseline = camInfo.P[3] / -camInfo.P[0]
    yBaseline = camInfo.P[7] / -camInfo.P[5]
    extrinsicMatrix [0:3,3] = [xBaseline, yBaseline, 0]

    return intrinsicMatrix, extrinsicMatrix

class zBuff:
    def __init__(self,renWin):
        self.renWin = renWin
        self.zBuffFilter = vtk.vtkWindowToImageFilter()
        self.zBuffFilter.SetInput(self.renWin)
        self.zBuffFilter.SetMagnification(1)
        self.zBuffFilter.SetInputBufferTypeToZBuffer()

        self.scaledZBuff = vtk.vtkImageShiftScale()
        self.scaledZBuff.SetOutputScalarTypeToUnsignedChar()
        self.scaledZBuff.SetInputConnection(self.zBuffFilter.GetOutputPort())
        self.scaledZBuff.SetShift(0)
        self.scaledZBuff.SetScale(-255)

    def GetOutput(self):
        # self.renWin.Render()
        self.zBuffFilter.Modified()
        self.scaledZBuff.Update()
        return self.scaledZBuff.GetOutput()

# Loads an image and returns a plane object and a camera pointed towards it
def _makeImagePlane(imageData) :
    ''' Takes a vtkImageData object and
        returns a plane textured with that image
        and a camera for the background renderer
        http://www.vtk.org/Wiki/VTK/Examples/Cxx/Images/BackgroundImage
        was the basis for this function
    '''
    imageActor = vtk.vtkImageActor()

    if vtk.VTK_MAJOR_VERSION <= 5:
        imageActor.SetInput(imageData)
    else:
        imageActor.SetInputData(imageData)

    origin = imageData.GetOrigin()
    spacing = imageData.GetSpacing()
    extent = imageData.GetExtent()

    camera = vtk.vtkCamera()
    camera.ParallelProjectionOn()

    xc = origin[0] + 0.5*(extent[0] + extent[1])*spacing[0]
    yc = origin[1] + 0.5*(extent[2] + extent[3])*spacing[1]
    yd = (extent[3] - extent[2] + 1)*spacing[1]
    d = camera.GetDistance()
    camera.SetParallelScale(0.5*yd)
    camera.SetFocalPoint(xc,yc,0.0)
    camera.SetPosition(xc,yc,d)

    return camera, imageActor

def setCameraExtrinsic(camera,matrix):
    R = matrix[0:3,0:3]
    up = np.dot(R,[0,-1,0])
    forward = np.dot(R,[0,0,1])
    camera.SetViewUp(up)
    camera.SetFocalPoint(forward)
    # eye = (-R'T), center = eye + R'(0,0,1)', up = -R[0][1]
    return

# Sets up a camera based on a camera intrinsics matrix from OpenCV
def cameraFromMatrix(camMatrix, imageDims, windowDims) :
    ''' Takes a camera matrix derived from OpenCV
        returns a camera for registration
        http://stackoverflow.com/questions/25539898/how-to-apply-the-camera-pose-transformation-computed-using-epnp-to-the-vtk-camer
        was a great help
        Input:
            camMatrix (np.ndarray) = 4x4 camera intrinsic matrix
            imageDims ((float, float))  = width and height of image to register
            imageDims ((float, float))  = width and height of render window
        Output:
            camera (vtk.vtkCamera) = camera to use for registration
    '''
    # Initialize camera
    camera = vtk.vtkCamera()

    # Set view angle
    focalLengthY = camMatrix[1,1]
    if( windowDims[1] != imageDims[1] ):
        factor = float(windowDims[1])/float(imageDims[1])
        focalLengthY = camMatrix[1,1] * factor
    viewAngle = np.arctan((windowDims[1]/2.0)/focalLengthY)*360/np.pi
    camera.SetViewAngle(viewAngle)

    #Set window center
    py = 0
    height = 0
    if imageDims[0] != windowDims[0] or imageDims[1] != windowDims[1] :
        factor = float(windowDims[1])/float(imageDims[1])
        px = factor * camMatrix[0,2]
        width = windowDims[0]
        expectedWindowSize = np.round(factor * float(imageDims[0]))
        if( expectedWindowSize != windowDims[0] ):
            diffX = (windowDims[0] - expectedWindowSize) / 2
            px = px + diffX
        py = factor * camMatrix[1,2]
        height = windowDims[1]
    else :
        px = camMatrix[0,2]
        width = imageDims[0]
        py = camMatrix[1,2]
        height = imageDims[1]
    cx = width - px
    cy = py
    windowCenter = [0,0]
    windowCenter[0] = cx / ( (width)/2 ) - 1
    windowCenter[1] = cy / ( (height)/2 ) - 1
    camera.SetWindowCenter(windowCenter[0],windowCenter[1])

    # Set camera to look forward from center
    camera.SetPosition(0,0,0)
    camera.SetViewUp([0, -1, 0])
    camera.SetFocalPoint(0,0,1)

    return camera


import pandas as pd
import numpy as np
import random
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, classification_report, confusion_matrix
from sklearn.model_selection import GridSearchCV

df = pd.read_table("dialog_acts.dat", header=None, names=['data'])
df['act'] = df['data'].str.split(' ').str[0]
df['sentence'] = df['data'].str.split(' ').str[1:]
df['sentence'] = df['sentence'].str.join(' ')
df.drop(labels='data', inplace=True, axis=1)

df.act.value_counts()

contraptions = {'doesnt': "does not",
               'im': 'i am',
               'dont': 'do not',
               'id': 'i would'}

df['sentence'] = df['sentence'].replace(contraptions, regex=True)
df['sentence_length'] = df.sentence.str.split().apply(len)
df['sentence_length'].describe()

x = df['sentence']
y = df['act']

train_categories = 0
test_categories = 0

while train_categories < 15 and test_categories < 15:
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.15)
    train_categories = len(y_train.value_counts())
    test_categories = len(y_test.value_counts())

print(f'\nSamples in train set: {len(x_train)}')
print(f'Samples in test set: {len(x_test)}')

i = 0
def create_cf_matrix(y_test, y_pred, save_fig=False):
    global i
    cf_matrix = confusion_matrix(y_test, y_pred, labels=list(y_test.unique()))
    cm_sum = np.sum(cf_matrix, axis=1, keepdims=True)
    f = plt.figure(figsize=(11, 11))
    ax = plt.subplot()
    sns.heatmap(cf_matrix / cm_sum, annot=True, ax=ax, fmt='0.1%', cmap='Blues')  # annot=True to annotate cells

    ax.set_xlabel('Predicted labels')
    ax.set_ylabel('True labels')
    ax.set_title('Confusion Matrix')
    ax.xaxis.set_ticklabels(list(y_test.unique()), rotation=45)
    ax.yaxis.set_ticklabels(list(y_test.unique()), rotation=45)

    titles = ['majority', 'rule_based', 'decision_tree', 'SVM', 'MLP']
    if save_fig:
        plt.savefig(f'confusion_matrix_{titles[i]}')
    i += 1

print('Accuracy of the majority class classifier on the test samples:')
y_pred = len(y_test) * ['inform']
print(f"accuracy: {accuracy_score(y_test, y_pred)}")
print(f"precision: {precision_score(y_test, y_pred, average='weighted')}")
print(f"recall: {recall_score(y_test, y_pred, average='weighted')}")
create_cf_matrix(y_test, y_pred)

print('Accuracy of the rulebased classifier on the test samples')
# dictionary with act as key and corresponding words as values
rules = {'ack': ['okay', 'okay um', 'alright'], 'affirm': ['yes right', 'right', 'yes'],
         'bye': ['see you', 'good bye', 'bye'], 'confirm': ['is it'], 'deny': ['i dont want'],
         'hello': ['hi', 'hello'],
         'inform': ['looking for'], 'negate': ['no'], 'repeat': ['can you repeat that', 'what did you say'],
         'reqalts': ['how about'], 'reqmore': ['more'], 'request': ['what is', 'where'], 'restart': ['start over'],
         'thankyou': ['thank you', 'thanks']}

y_pred = []

for x in x_test:
    acts = []
    for k, v in rules.items():
        if any(keywords in x.lower() for keywords in v):
            acts.append(k)
    if not acts:
        y_pred.append('null')
    else:
        y_pred.append(random.choice(acts))

print(f"accuracy: {accuracy_score(y_test, y_pred)}")
print(f"precision: {precision_score(y_test, y_pred, average='weighted')}")
print(f"recall: {recall_score(y_test, y_pred, average='weighted')}")
create_cf_matrix(y_test, y_pred)

max_sentences = 0.85
min_sentences = 1
BOW_vect = CountVectorizer(max_df=max_sentences, min_df=1, lowercase=True, strip_accents='ascii')
x_train_counts = BOW_vect.fit_transform(x_train)

### Decision Tree
print('Finding best hyperparameters for Decision tree using grid search and cross validation.')
params = {'max_depth': list(range(1,25)),
          'min_samples_split': [10, 50,100]}
grid_search_cv = GridSearchCV(DecisionTreeClassifier(), params, verbose=1, cv=3, n_jobs=-1)
grid_search_cv.fit(x_train_counts, y_train)
x_test_counts = BOW_vect.transform(x_test)
y_pred = grid_search_cv.predict(x_test_counts)
print('Evaluation of the best hyperparameters on the test set')
print(f"accuracy: {accuracy_score(y_test, y_pred)}")
print(f"precision: {precision_score(y_test, y_pred, average='weighted')}")
print(f"recall: {recall_score(y_test, y_pred, average='weighted')}")
create_cf_matrix(y_test, y_pred)

### SVM
print('Finding best parameters for SVM using grid search and cross validation.')
params = params = {
    "loss" : ["hinge", "log", "squared_hinge", "modified_huber"],
    "alpha" : [0.0001, 0.001, 0.01, 0.1],
    "penalty" : ["l2", "l1", "none", "elasticnet"],
}

grid_search_cv = GridSearchCV(SGDClassifier(), params, verbose=1, cv=3, n_jobs=-1)
grid_search_cv.fit(x_train_counts, y_train)

x_test_counts = BOW_vect.transform(x_test)
y_pred = grid_search_cv.predict(x_test_counts)
print('Evaluation of the best hyperparameters on the test set')
print(f"accuracy: {accuracy_score(y_test, y_pred)}")
print(f"precision: {precision_score(y_test, y_pred, average='weighted')}")
print(f"recall: {recall_score(y_test, y_pred, average='weighted')}")
create_cf_matrix(y_test, y_pred)

results = pd.DataFrame({'utterance': x_test, 'act': y_test, 'predicted_act':y_pred})
results['utterance_length'] = results.utterance.str.split().apply(len)
corrects = results[(results.act == results.predicted_act)]
mistakes = results[(results.act != results.predicted_act)]
print(f'Distribution True:\n{mistakes.act.value_counts()}')
print(f'Distribution predicted:\n{mistakes.predicted_act.value_counts()}')
print(f'Utterance length description:\n {results.utterance_length.describe()}')
print(f'Utterance length description:\n {mistakes.utterance_length.describe()}')
print(f'Mistakes: {mistakes}')

### MLP
print('Finding best parameters for MLP using grid search and cross validation.')
params = {
    'hidden_layer_sizes': [(10,30,10),(20, 20), (20)],
    'activation': ['tanh', 'relu'],
    'solver': ['sgd', 'adam'],
    'alpha': [0.0001, 0.05],
    'learning_rate': ['constant','adaptive'],
}

grid_search_cv = GridSearchCV(MLPClassifier(max_iter=100), params, n_jobs=-1, cv=3)
grid_search_cv.fit(x_train_counts, y_train)

x_test_counts = BOW_vect.transform(x_test)
y_pred = grid_search_cv.predict(x_test_counts)

print('Evaluation of the best hyperparameters on the test set')
print(f"accuracy: {accuracy_score(y_test, y_pred)}")
print(f"precision: {precision_score(y_test, y_pred, average='weighted')}")
print(f"recall: {recall_score(y_test, y_pred, average='weighted')}")

create_cf_matrix(y_test, y_pred)

results = pd.DataFrame({'utterance': x_test, 'act': y_test, 'predicted_act':y_pred})
results['utterance_length'] = results.utterance.str.split().apply(len)
corrects = results[(results.act == results.predicted_act)]
mistakes = results[(results.act != results.predicted_act)]
print(f'Distribution True:\n{mistakes.act.value_counts()}')
print(f'Distribution predicted:\n{mistakes.predicted_act.value_counts()}')
print(f'Utterance length description:\n {results.utterance_length.describe()}')
print(f'Utterance length description:\n {mistakes.utterance_length.describe()}')
print(f'Mistakes: {mistakes}')


import json
import asyncio
import base64

import sys
sys.path.append('..')
from uniswap_python_port.uniswap import *

import requests
from requests.auth import HTTPBasicAuth

import web3
from web3.auto.gethdev import w3
from web3.middleware import geth_poa_middleware

from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport

from pairs import Pair

# Pair abi info; to move later
pair_abi = json.load(open("files/abi/pairs.abi"))['abi']

# Setup GraphQL to query pairs; tidy
transport = AIOHTTPTransport(url="https://bsc.streamingfast.io/subgraphs/name/pancakeswap/exchange-v2")
client = Client(transport=transport, fetch_schema_from_transport=True)

# Bunch of config; again, tidy
BSC_NODE = "https://bsc-dataseed.binance.org/"
BSC_PROVIDER = web3.Web3.HTTPProvider(BSC_NODE)
w3 = web3.Web3(BSC_PROVIDER)
w3.middleware_onion.inject(geth_poa_middleware, layer=0)

# Create Uniswap instance; move to a config file
address = None
private_key = None
version = 2
web3 = w3
factory_contract_addr = w3.toChecksumAddress("0xcA143Ce32Fe78f1f7019d7d551a6402fC5350c73")
router_contract_addr = w3.toChecksumAddress("0x10ED43C718714eb63d5aA57B78B54704E256024E")

uniswap = Uniswap(address=address,
                  private_key=private_key,
                  version=version,
                  web3=web3,
                  factory_contract_addr=factory_contract_addr, router_contract_addr=router_contract_addr)

def buildPairs(w3):

    pairList = []

    cake_pairs = json.load(open("./files/50_pairs_pancake.json"))["data"]["pairs"]

    for i in range(len(cake_pairs)):
        pair_addr = cake_pairs[i]['id']
        pair_contract = util._load_pair_contract(w3, pair_abi, pair_addr)
        token0addr = pair_contract.functions.token0().call()
        token0sym = cake_pairs[i]['token0']['symbol']
        token1addr = pair_contract.functions.token1().call()
        token1sym = cake_pairs[i]['token1']['symbol']

        reserves = pair_contract.functions.getReserves().call()
        reserve0 = reserves[0]
        reserve1 = reserves[1]

        pair = Pair(w3,
                    pair_addr,
                    token0addr,
                    token0sym,
                    token1addr,
                    token1sym,
                    reserve0,
                    reserve1)

        pairList.append(pair)

    return pairList

def test():

    """
    # pair_address = uniswap.factory_contract.functions.allPairs(1).call()
    # pair_contract = util._load_pair_contract(w3, pair_abi, pair_address)

    # Info to build pair:
    token0addr = pair_contract.functions.token0().call()
    token0sym = "ABC"
    token1addr = pair_contract.functions.token1().call()
    token1sym = "XYZ"
    reserve0 = pair_contract.functions.getReserves().call()[0]
    reserve1 = pair_contract.functions.getReserves().call()[1]

    pair = Pair(w3,
                pair_address,
                token0addr,
                token0sym,
                token1addr,
                token1sym,
                reserve0,
                reserve1)

    pair.monitorPair()

    """
    pairList = buildPairs(w3)
    print(pairList[0].pairStruct)


import time
from datetime import datetime
import cryptocompare

while True:
    btc = cryptocompare.get_price('BTC',curr='USD')
    btcusd = btc['BTC']
    eth = cryptocompare.get_price('ETH',curr='USD')
    ethusd = eth['ETH']

    now = datetime.now()

    print("Updated at:", now)

    print("BTC in USD: ", btcusd['USD'])

    print("ETH in USD: ", ethusd['USD'])

    time.sleep(15)


from django.urls import path
from . import views

app_name = 'post'

urlpatterns = [
  path('',views.Home,name='home'),
  path('serach',views.Search,name='search'),
  path('detail/<int:id>/',views.PostDetail,name='detail'),
  path('likes/<int:pk>',views.likes,name="likes")

]


# -*- coding: UTF-8 -*-
"""
title: 值和下标之差都在给定的范围内
给你一个整数数组 nums 和两个整数 k 和 t 。请你判断是否存在 两个不同下标 i 和 j，使得 abs(nums[i] - nums[j]) <= t ，同时又满足 abs(i - j) <= k 。
如果存在则返回 true，不存在返回 false。

示例 1：
输入：nums = [1,2,3,1], k = 3, t = 0
输出：true

示例 2：
输入：nums = [1,0,1,1], k = 1, t = 2
输出：true

示例 3：
输入：nums = [1,5,9,1,5,9], k = 2, t = 3
输出：false

提示：
0 <= nums.length <= 2 * 10^4
-2^31 <= nums[i] <= 2^31 - 1
0 <= k <= 10^4
0 <= t <= 2^31 - 1
"""
from typing import List

class Solution:
    def containsNearbyAlmostDuplicate(self, nums: List[int], k: int, t: int) -> bool:
        """"
        利用桶排序的思想，对元素按大小进行分桶。
        桶ID为：nums[i] // (t + 1)
        假设t=9，则 0 ~ 9 会被分配到id为 0 的桶； -10 ~ -1 会被分配到id为 -1 的桶 …… 。abs(-10 - -1) == 9 <= t
        同一个桶中的元素，一定满足 abs(nums[i] - nums[j]) <= t 。若是相邻桶，则需比较大小。
        优化：一个桶中只需保存遇到的第一个元素，也就是说，value是一个值，而不是一个列表。
        可行原因：
        1、如果后续遍历，遇到同属一个桶的元素，则直接返回True，无需继续遍历了
        2、比较相邻桶时，也无需担心某些更相近的元素之前没加入到相邻桶，原因就是上面说的，如果存在属于相邻桶的更相近的元素，那么之前就已经return True了，
        根本不会遍历到现在，之所以能够遍历到现在，就是因为相邻桶自从加入了第一个元素后，一直也没遇到同属该桶的元素。
        """

        def get_id(num: int) -> int:
            """
            需要注意的是，Python中负数的整除、取余与Java/C++中不一样。Python在这块与日常理解不太一样！
            Python中的负数整除：-1 // 10 == -10 // 10 == -1        -11 // 10 == -2
            Java/C++中的负数整除：-1 // 10 == -9 // 10 == 0        -10 // 10 == -11 // 10 == -1
            所以，对于负数，Java/C++中需要写成：(num + 1) // (t + 1) - 1
            (num + 1)： -1 ~ -10 ——> 0 ~ -9
            // (t + 1)：0 ~ -9  ——> 0 // 10 == -9 // 10 == 0
             - 1：0 ——> -1
            最终实现：-1 // 10 == -10 // 10 == -1
            所以，Python中若想实现正常的负数整除：int(num / 10)     # 正负数整除都可以用这种方式，利用int()向0的方向取整，并不是单纯的向下取整

            Python中的负数取余：-1 % 10 == 9       -9 % 10 == 1        -10 % 10 == 0       -11 % 10 == 9
            Java/C++中的负数取余：-1 % 10 == -1    -9 % 10 == -9       -10  % 10 == 0      -11  % 10 == -1
            所以，Python中若想实现正常的负数取余：-(-num % 10)      # 负数取余用这种方式，正数取余还是用常规的 num % 10
            """
            return num // (t + 1)

        if len(nums) < 2 or k == 0:
            return False
        id2num = {}
        for idx, num in enumerate(nums):
            id = get_id(num)
            if id in id2num:
                return True
            if id - 1 in id2num and abs(id2num[id - 1] - num) <= t:
                return True
            if id + 1 in id2num and abs(id2num[id + 1] - num) <= t:
                return True
            id2num[id] = num
            if idx >= k:
                id2num.pop(get_id(nums[idx - k]))
        return False


#!/usr/bin/env python3
# Soubor:  kameny.py
# Datum:   06.11.2018 10:01
# Autor:   Marek Nožka, nozka <@t> spseol <d.t> cz
# Licence: GNU/GPL
############################################################################
import pyglet
import random
from math import sin, cos, radians, pi
import glob

from pyglet.window.key import LEFT, RIGHT, UP, DOWN, LCTRL

# from pyglet.window.mouse import LEFT as MouseLEFT

window = pyglet.window.Window(width=1200, height=950)
batch = pyglet.graphics.Batch()  # pro optimalizované vyreslování objektů
bg_batch = pyglet.graphics.Batch()  # pro optimalizované vyreslování objektů

class SpaceObject:
    def __init__(self, img_file, x=None, y=None):

        # načtu obrázek
        self.image = pyglet.image.load(img_file)
        # střed otáčení dám na střed obrázku
        self.image.anchor_x = self.image.width // 2
        self.image.anchor_y = self.image.height // 2
        # z obrázku vytvořím sprite
        self.sprite = pyglet.sprite.Sprite(self.image, batch=batch)

        # pokud není atribut zadán vytvořím ho náhodně
        self._x = x if x is not None else random.randint(0, window.width)
        self._y = y if y is not None else random.randint(0, window.height)
        # musím správně nastavit polohu sprite
        self.x = self._x
        self.y = self._y

        self.max = max(self.image.height, self.image.width)
        self.min = min(self.image.height, self.image.width)

    @property
    def x(self):
        return self._x

    @x.setter
    def x(self, new):
        self._x = self.sprite.x = new

    @property
    def y(self):
        return self._y

    @y.setter
    def y(self, new):
        self._y = self.sprite.y = new

class Meteor(SpaceObject):
    def __init__(
        self,
        img_file=None,
        x=None,
        y=None,
        direction=None,
        speed=None,
        rspeed=None,
    ):
        if img_file is None:
            img_file = random.choice(glob.glob("img/meteor*.png"))
        super().__init__(img_file, x, y=window.height + 20)

        self.direction = (
            direction if direction is not None else random.randint(150, 220)
        )
        self.speed = speed if speed is not None else random.randint(100, 300)
        self.rspeed = rspeed if rspeed is not None else random.randint(-50, 50)

    def tick(self, dt):
        # do promenne dt se uloží doba od posledního tiknutí
        self.x += dt * self.speed * cos(pi / 2 - radians(self.direction))
        self.sprite.x = self.x
        self.y += dt * self.speed * sin(pi / 2 - radians(self.direction))
        self.sprite.y = self.y
        self.sprite.rotation += dt * self.rspeed

    def __del__(self):
        print("Mem free!")

class Laser(SpaceObject):
    def __init__(self, img_file="img/laser.png", speed=1000):

        # načtu obrázek
        self.image = pyglet.image.load(img_file)
        # střed otáčení dám na střed obrázku
        self.image.anchor_x = self.image.width // 2
        self.image.anchor_y = self.image.height
        # z obrázku vytvořím sprite
        self.sprite = pyglet.sprite.Sprite(self.image, batch=batch)
        self.sprite.z = 0
        self.direction = 0
        self.speed = speed

        self.x = ship.x
        self.y = ship.y + 70

    def tick(self, dt):
        self.y += dt * self.speed

class SpaceShip(SpaceObject):
    def __init__(self, x=None, y=None, speed=300):

        self.keys = set()

        self.fire1 = pyglet.image.load("img/fire07.png")
        self.fire3 = pyglet.image.load("img/fire12.png")
        self.fire1.anchor_x = self.fire1.width // 2
        self.fire1.anchor_y = self.fire1.height
        self.fire3.anchor_x = self.fire3.width // 2
        self.fire3.anchor_y = 0
        self.fireL = pyglet.sprite.Sprite(self.fire1, batch=batch)
        self.fireR = pyglet.sprite.Sprite(self.fire1, batch=batch)
        self.fireFL = pyglet.sprite.Sprite(self.fire3, batch=batch)
        self.fireFR = pyglet.sprite.Sprite(self.fire3, batch=batch)

        # vzdálenost ohně motorů od lodi...
        self.fire_x_dist = 30
        self.fire_y_dist = 39

        super().__init__("img/ship.png", x=window.width / 2, y=77)

        self.speed = speed
        self.width = self.image.width
        self.height = self.image.height

    @property
    def x(self):
        return self._x

    @x.setter
    def x(self, new):
        self._x = self.sprite.x = new
        self.fireL.x = self.x - self.fire_x_dist
        self.fireR.x = self.x + self.fire_x_dist
        self.fireFL.x = self.x - 12
        self.fireFR.x = self.x + 12

    @property
    def y(self):
        return self._y

    @y.setter
    def y(self, new):
        self._y = self.sprite.y = new
        self.fireL.y = self.y - self.fire_y_dist
        self.fireR.y = self.y - self.fire_y_dist
        self.fireFL.y = self.y + 20
        self.fireFR.y = self.y + 20
        self.fireL.x = self.x - self.fire_x_dist
        self.fireR.x = self.x + self.fire_x_dist

    def tick(self, dt):
        if LEFT in self.keys and self.x > 0:
            self.fire_x_dist = 26
            self.fire_y_dist = 31
            self.x -= dt * self.speed
            self.sprite.rotation = -5
            self.sprite.scale_x = 0.85
        elif RIGHT in self.keys and self.x < window.width:
            self.fire_x_dist = 26
            self.fire_y_dist = 31
            self.x += dt * self.speed
            self.sprite.rotation = 5
            self.sprite.scale_x = 0.85
        else:
            self.sprite.rotation = 0
            self.sprite.scale_x = 1
            self.fire_x_dist = 30
            self.fire_y_dist = 39
        if DOWN in self.keys and self.y > self.height / 2:
            self.y -= dt * self.speed
            self.fireFL.visible = 1
            self.fireFR.visible = 1
        elif UP in self.keys and self.y < window.height - self.height:
            self.y += dt * self.speed
            self.fireL.visible = 1
            self.fireR.visible = 1
        else:
            self.fireL.visible = 0
            self.fireR.visible = 0
            self.fireFL.visible = 0
            self.fireFR.visible = 0

class Meet:
    meteors = list()
    lasers = list()

    def add_meteor(self, dt=None):
        self.meteors.append(Meteor())

    def add_laser(self, dt=None):
        self.lasers.append(Laser())

    def tick(self, dt):
        # pohnu kamenama
        for meteor in self.meteors:
            meteor.tick(dt)
            # vymažu ty, co nejsou vidět
            if (
                meteor.y < 0 - meteor.max
                or meteor.x < 0 - meteor.max
                or meteor.x > window.width + meteor.max
            ):
                meteor.sprite.delete()
                self.meteors.remove(meteor)
            distance = (
                (meteor.x - ship.x) ** 2 + (meteor.y - ship.y) ** 2
            ) ** 0.5
            if distance - meteor.min / 2 - 32 <= 0:
                self.remove = meteor  # schovám si ho abych ho pak vymazal
                self.hit()
                continue
            for laser in self.lasers:
                if (
                    laser.y >= meteor.y
                    and laser.x >= (meteor.x - meteor.min / 3)
                    and laser.x <= (meteor.x + meteor.min / 3)
                ):
                    self.lasers.remove(laser)
                    self.meteors.remove(meteor)
        # pohnu laserama
        for laser in self.lasers:
            laser.tick(dt)
            if laser.y > window.height + laser.sprite.height:
                laser.sprite.delete()
                self.lasers.remove(laser)

    def hit(self):
        pyglet.clock.unschedule(ticktack)
        pyglet.clock.unschedule(meet.add_meteor)
        pyglet.clock.schedule_once(self.renew, 3)

    def renew(self, dt):
        self.remove.sprite.delete()
        self.meteors.remove(self.remove)
        pyglet.clock.schedule_interval(ticktack, 1 / 30)
        pyglet.clock.schedule_interval(meet.add_meteor, 1 / 3)

@window.event
def on_draw():
    window.clear()
    bg_batch.draw()
    batch.draw()

@window.event
def on_key_press(sym, mod):
    # print(sym, mod)
    if sym == LCTRL:
        meet.add_laser()
    else:
        ship.keys |= {
            sym,
        }

@window.event
def on_key_release(sym, mod):
    ship.keys -= {
        sym,
    }

def ticktack(dt):
    meet.tick(dt)
    ship.tick(dt)

# pozadí...
bg = pyglet.image.load("img/bg.img")
x = 0
bg_sprites = ()
while x < window.width:
    y = 0
    while y < window.height:
        bg_sprites += (pyglet.sprite.Sprite(bg, x=x, y=y, batch=bg_batch),)
        y += bg.height
    x += bg.width

ship = SpaceShip()
# print(ship.width)
# print(ship.height)
meet = Meet()
meet.add_meteor()
meet.add_meteor()
meet.add_meteor()
meet.add_meteor()
meet.add_meteor()
pyglet.clock.schedule_interval(ticktack, 1 / 30)
pyglet.clock.schedule_interval(meet.add_meteor, 10 / 3)
pyglet.app.run()


import numpy as np
import matplotlib.pyplot as plt
import time
import import_ipynb

newparams = {'figure.figsize': (10, 5), 'axes.grid': True,
             'lines.linewidth': 1.5, 'lines.markersize': 5,
             'font.size': 14}
plt.rcParams.update(newparams)

def rhs(t,y):
    return np.cos(y*t**2)

def analytical(t, y):
    return 0.5*(1-np.exp(-t**2))

#steppers
def odesolver23(f,t,y,h):
    #Perform the next step for an ODE with rhs given by f, with step size h
    #from time t, to time t+h with expl. trapezoidal rule and a third order RK

    #f: rhs
    #t: current time
    #y: current solution value
    #h: current time step

    #Returns:
    #q: float. Order 2 approx.
    #w: float. Order 3 approx.

    s1 = f(t, y)
    s2 = f(t+h, y+h*s1)
    s3 = f(t+h/2.0, y+h*(s1+s2)/4.0)
    w = y + h*(s1+s2)/2.0
    q = y + h*(s1+4.0*s3+s2)/6.0
    return w, q

def odesolver12(f, t, y, h):
    s1 = f(t, y)
    s2 = f(t+h, y+h*s1)
    w = y + h*s1
    q = y + h/2.0*(s1+s2)
    return w, q

def rk_adaptive(ode,rhs,y0,t0,Tol,theta,tmax,iterMax=100):

    y=[]
    t=[]
    y.append(y0)
    t.append(t0)

    h=1 #step size selection: to improve (follow Hairer's book)
    i=0

    while (t[i]<tmax):
        w,q=ode(rhs,t[i],y[i],h)
        #local error estimate
        e=abs((w-q)/max(w,theta))
        if(e>Tol):
            h=0.8*(Tol*e)**(1/5)*h #change stepsize
            w,q=ode(rhs,t[i],y[i],h) #recompute the solution
            e=abs((w-q)/max(w,theta)) #check error again
            i2=0
            while (e>Tol) & (i2<iterMax):
                i2+=1
                h=h/2 #half the step size
                w,q=ode(rhs,t[i],y[i],h) #recompute the solution
                e=abs((w-q)/max(w,theta)) #check error
        y.append(q)
        if (t[i]+h>tmax): #hit the exact end-point
            h=tmax-t[i]

        t.append(t[i]+h)

        i+=1
        if(e<0.1*Tol):
            h=2*h

    return y,t,i

y,t,iterations=rk_adaptive(odesolver23,rhs,y0=3,t0=1,Tol=1e-4,theta=1e-3,tmax=3)

plt.plot(t,y,'-o',label='Numerical solution RK 23')
#ta=np.linspace(0,t[iterations])
#plt.plot(ta,analytical(ta,0),'-',label='Analytical solution')
plt.legend()
plt.show()


from tkinter import *

def create_table(root, data, colour = "white"):
    rows = len(data)
    columns = len(data[0])

    for i in range(rows):
        for j in range(columns):
            cell = Entry(root, width = 20, fg = colour)
            cell.grid(row=i, column=j)
            cell.insert(END, data[i][j])
    return


'''
Copyright (C) 2020 Yee Young Han <websearch@naver.com> (http://blog.naver.com/websearch)

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
'''

import os

class SdpAttributeCrypto():
  """ SDP 암호화 애트리뷰트 저장 클래스
  """

  def __init__( self ):
    self.strTag = ''
    self.strCryptoSuite = ''
    self.strKey = ''
    self.strText = ''

  def Parse( self, strText ):
    """ SDP 암호화 애트리뷰트 문자열을 파싱하여서 멤버 변수에 저장한다.

    Args:
        strText (string): SDP 암호화 애트리뷰트 문자열

    Returns:
        int: 성공하면 파싱 문자열 길이를 리턴하고 그렇지 않으면 -1 를 리턴한다.
    """
    self.Clear()

    iPos = 0
    iStartPos = 0
    iTextLen = len(strText)
    iType = 0

    if( iTextLen == 0 ):
      return -1

    while( iPos < iTextLen ):
      if( strText[iPos] == ' ' ):
        if( iType == 0 ):
          self.strTag = strText[iStartPos:iPos]
        elif( iType == 1 ):
          self.strCryptoSuite = strText[iStartPos:iPos]

        iStartPos = iPos + 1
        iType += 1
      elif( iType == 2 ):
        if( strText[iPos] == ':' ):
          iStartPos = iPos + 1
        elif( strText[iPos] == '|' ):
          self.strKey = strText[iStartPos:iPos]
          iStartPos = iPos + 1
          self.strText = strText[iStartPos:]

      iPos += 1

    if( iType == 2 and len(self.strKey) == 0 ):
      self.strKey = strText[iStartPos:]

    return iTextLen

  def __str__( self ):
    """ SDP 암호화 애트리뷰트 문자열를 리턴한다.

    Returns:
        string: SDP 암호화 애트리뷰트 문자열를 리턴한다.
    """
    if( self.Empty() ):
      return ''

    strText = self.strTag + " " + self.strCryptoSuite + " inline:" + self.strKey
    if( len(self.strText) > 0 ):
      strText += "|" + self.strText

    return strText

  def Clear( self ):
    """ 멤버 변수를 초기화시킨다.
    """
    self.strTag = ''
    self.strCryptoSuite = ''
    self.strKey = ''
    self.strText = ''

  def Empty( self ):
    """ SDP 암호화 애트리뷰트가 저장되어 있는지 확인한다.

    Returns:
        bool: SDP 암호화 애트리뷰트가 저장되어 있으면 False 를 리턴하고 그렇지 않으면 True 를 리턴한다.
    """
    if( len(self.strTag) == 0 or len(self.strCryptoSuite) == 0 or len(self.strKey) == 0 ):
      return True

    return False


import numpy as np
import cv2
import pickle

# mengekstraksi fitur citra

fitur = np.zeros((3040,4),dtype=np.float)
#kelas = np.zeros((3040,1),dtype=np.int)
for i in range(1,20):
    for j in range(1,21):
        nama_file='D:\VK_Python\Dataset Wajah\F'+str(i)+'.'+str(j)+'.jpg'
        img = cv2.imread(nama_file,1)
        for k in range(0, 3):
            fitur[(i-1)*20+j-1,k]=img[:,:,k].mean()
        #kelas[(i-1)*20+j-1]=i
        #fitur[(i-1)*20+j-1,3]=int(i)
        fitur[(i-1)*20+j-1,3]=int(i)

for i in range(1,114):
    for j in range(1,21):
        nama_file='D:\VK_Python\Dataset Wajah\M'+str(i)+'.'+str(j)+'.jpg'
        img = cv2.imread(nama_file,1)
        for k in range(0, 3):
            fitur[380+(i-1)*20+j-1,k]=img[:,:,k].mean()
        #kelas[380+(i-1)*20+j-1]=19+i
        #fitur[380+(i-1)*20+j-1,3]=int(i)
        fitur[380+(i-1)*20+j-1,3]=int(i+19)

for i in range(1,21):
    for j in range(1,21):
        nama_file='D:\VK_Python\Dataset Wajah\MS'+str(i)+'.'+str(j)+'.jpg'
        img = cv2.imread(nama_file,1)
        for k in range(0, 3):
            fitur[2640+(i-1)*20+j-1,k]=img[:,:,k].mean()
        #kelas[2640+(i-1)*20+j-1]=132+i
        #fitur[2640+(i-1)*20+j-1,3]=int(i)
        fitur[2640+(i-1)*20+j-1,3]=int(i+132)

# menyimpan fitur ke dalam file
outputFile = 'FiturWarnaWajah.data'
fw = open(outputFile, 'wb')
pickle.dump(fitur, fw)
fw.close()

#print(fitur)

#ret,th1 = cv2.threshold(img,127,255,cv2.THRESH_BINARY)
#th2 = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
#cv2.THRESH_BINARY,3,3)
#th3 = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
#cv2.THRESH_BINARY,3,3)
#titles = ['Original Image', 'Global Thresholding (v = 127)',
#'Adaptive Mean Thresholding', 'Adaptive Gaussian Thresholding']
#images = [img, th1, th2, th3]
#for i in xrange(4):
#    plt.subplot(2,2,i+1),plt.imshow(images[i],'gray')
#    plt.title(titles[i])
#    plt.xticks([]),plt.yticks([])
#plt.show()

#cv2.namedWindow('image', cv2.WINDOW_NORMAL)
#cv2.imshow('image',img)
#cv2.waitKey(0)
#cv2.destroyAllWindows()


import numpy as np
import openpyxl as pyxl

def test_loader(fname=None):
    fname = '/home/mattias/Dropbox/CommonCode/Lifetime_fitting/src/test_folder/500_1e2_1e8.dat'
    data = np.genfromtxt(fname, names=['nxc', 'tau'], usecols=(8, 9))
    return data

def sinton2014(fname):
    '''
    This grabs the calculated data from sinton
    Extracts columns A-G of the raw data page.
    Can't extract columns H-I as it is not complete data and this would take
    some more work
    This outputs the data as a structured array, with the names as the column
    '''

    wb = pyxl.load_workbook(fname, read_only=True, data_only=True)
    # the command page was only added in the later versions
    if 'Command' in wb.get_sheet_names():
        data = _openpyxl_Sinton2014_ExtractRawDatadata(wb)

    # remove all the nan values from the data
    data = data[~np.isnan(data['Tau (sec)'])]
    data.dtype.names = ['Time in s', 'Photovoltage', 'Reference Voltage', 'PCD with baseline shift.', 'Ref (suns)', 'Delta Suns', 'Generation (pairs/s)', 'Conductivity increase', 'Apparent CD', 'nxc',
                        'tau', 'Tau In Range', '1/Tau in Range', '1/Tau Corrected', '1/Tau corrected in Range', '1/Tau Fit (auger corrected)', 'ni', 'ni2/tau', 'Implied Voc', 'Implied Suns', 'MCD for Ref Cell>0.6V']

    return data

def _openpyxl_Sinton2014_ExtractRawDatadata(wb):
    '''
        reads the raw and caculated data from the 'Calc' sheet of a  sinton WCT-120 spreadsheet.

        inputs:
            wb:
             instance of a openpylx workbook.
    '''

    # make sure the sheet is in the book
    assert 'Calc' in wb.get_sheet_names()

    # get the worksheet
    ws = wb.get_sheet_by_name('Calc')

    # get first section of data
    values1 = np.array([[i.value for i in j] for j in ws['A9':'I133']],
                       dtype=np.float64)
    headers1 = tuple(
        [[j.value for j in i] for i in ws['A8':'I8']][0])

    # get second section of data
    values2 = np.array([[i.value for i in j] for j in ws['O9':'Z133']],
                       dtype=np.float64)
    headers2 = tuple(
        [[j.value for j in i] for i in ws['O8':'Z8']][0])

    # form into one array with names
    values = np.hstack((values1, values2))
    headers = headers1 + headers2

    Out = values.view(dtype=list(
        zip(headers, ["float64"] * len(headers)))).copy()

    return Out

loadersandext = {
    'test_loader': '*nothing',
    'sinton2014': '*.xlsm'
}

loaders = {
    'test_loader': test_loader,
    'sinton2014': sinton2014
}

if __name__ == '__main__':
    test_loader(
        '/home/mattias/Dropbox/CommonCode/Lifetime_fitting/src/test_folder/500_1e2_1e8.dat')


import cv2
import numpy as np
import argparse
import glob
import os
from tqdm import tqdm
import csv
import random

def main(args):
    if not os.path.exists(args.result_path):
        os.makedirs(args.result_path)

    codebook = np.load(args.codebook_path)
    channel, dim = codebook.shape
    print("book size:", channel, ",", dim)

    akaze = cv2.AKAZE_create()

    rows = []
    with open(args.bovw_path) as f:
        reader = csv.reader(f)
        for row in tqdm(reader):
            rows.append(row)
    rows_num = []
    for row in rows:
        rows_num.append([float(num) for num in row[1:]])
    bovw = np.array(rows_num)

    category = os.listdir(args.root_path)
    category = [f for f in category if os.path.isdir(os.path.join(args.root_path, f))]
    acc_dict = {}

    for c in tqdm(category):
        img_files = [*glob.glob(os.path.join(args.root_path, c, '*.jpg'), recursive=True),
                *glob.glob(os.path.join(args.root_path, c, '*.jpeg'), recursive=True),
                *glob.glob(os.path.join(args.root_path, c, '*.png'), recursive=True)]

        num_sample = 20
        if len(img_files) < num_sample*2:
            samples = random.sample(img_files, len(img_files))
            for i in range(num_sample*2 - len(img_files)):
                samples.append(random.choice(img_files))
        else:
            samples = random.sample(img_files, num_sample*2)

        count_per_category = 0
        denom = 0
        iteration = iter(samples)
        for s_1, s_2 in zip(iteration, iteration):

            query_img_1 = cv2.imread(s_1,cv2.IMREAD_GRAYSCALE)
            query_img_2 = cv2.imread(s_2,cv2.IMREAD_GRAYSCALE)
            _, query_des_1 = akaze.detectAndCompute(query_img_1, None)
            _, query_des_2 = akaze.detectAndCompute(query_img_2, None)
            if type(query_des_1) == type(None) or type(query_des_2) == type(None):
                continue

            denom += 1
            query_des_1 = np.array(query_des_1)
            query_des_2 = np.array(query_des_2)

            query_hist_1 = desToHist(query_des_1, codebook)
            query_hist_2 = desToHist(query_des_2, codebook)

            #query_hist = np.add(query_hist_1, query_hist_2)
            query_hist = np.amax([query_hist_1, query_hist_2], axis=0)
            scores = histIntersect(bovw, query_hist)
            #scores = cosSim(bovw, query_hist)

            y = np.argsort(scores)[::-1]

            correct_count = 0

            eval_count = 0
            index = 0
            while(eval_count<args.num):
                if rows[y[index]][0] == s_1 or rows[y[index]][0] == s_2:
                    index += 1
                    continue

                eval_count += 1
                if rows[y[index]][0].split('/')[-2] == c:
                    correct_count += 1
                index += 1
            #print(c, ":", correct_count/args.num)
            count_per_category += correct_count

        acc_dict[c] = count_per_category /args.num /denom
        print(acc_dict)

    return

def desToHist(descriptor: np.ndarray, book):
    K, D = book.shape
    hist = [0,] * K
    for n in range(descriptor.shape[0]):
        des = np.broadcast_to(descriptor[n],(K, D))
        tmp = np.power(des - book, 2).sum(axis=1)
        index = np.argmin(tmp)
        hist[index] += 1

    # normalize
    hist_norm = np.array([float(i)/sum(hist) for i in hist])

    return hist_norm

def histIntersect(bovw, query_hist):
    N, D = bovw.shape
    scores = np.zeros((N))
    for i in range(N):
        eval = np.vstack((bovw[i], query_hist))
        eval = np.amin(eval, axis=0)
        scores[i] = eval.sum()

    return scores

def cosSim(bovw, query_hist):
    N, D = bovw.shape
    #bovw_norm = bovw * bovw
    #bovw_norm = bovw.sum(axis=1)
    #print(bovw_norm)
    #scores = np.divide(np.dot(bovw, query_hist.T) / np.linalg.norm(query_hist, ord=2), np.broadcast_to(bovw_norm.reshape(1, N), (D, N)))
    scores = np.dot(bovw, query_hist.T)
    return scores

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument("--root_path", help="root dir path", required=True)
    parser.add_argument("--result_path", help="output dir path", required=True)
    parser.add_argument("--codebook_path", help="codebook path", required=True)
    parser.add_argument("--bovw_path", help="bovw path", required=True)
    parser.add_argument("--num", help="number of images to retrieve", type=int, default=10)

    args = parser.parse_args()

    main(args)


import pandas as pd
from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import fonctions

#Fonction pour récupérer la fiche individuelle d'un joueur à partir du lien de sa page
def Scraping_ficheindiv_from_lien(site_team):
    #l'url ou chercher les données : site_team

    #ouverture de la page html
    html = urlopen(site_team)
    #collecte du Code HTML de la page dans un object Soup :
    soup = BeautifulSoup(html, features="html.parser")
    #création d'une sous-partie de l'html de base
    divb = soup.find("div", {"itemtype":"https://schema.org/Person"})

    #récupération du nom du joueur
    #Pour cela on scrap la balise "span" dans la balise "h1"
    rowss = divb.findAll('h1')[0:]
    nom = [[h1.getText() for h1 in rowss[i].findAll('span')]
                    for i in range(len(rowss))]
    nom=nom[0][0]

    #création des noms de lignes
    headers=['Nom', 'Nom complet', 'Taille', 'Poids', 'Date de naissance', 'Lieu de Naissance', 'College' , 'Equipe de Draft', 'Date de Draft', 'Début en NBA']

    #récupération des données
    #On récupère l'ensemble du contenu des balises "p"
    rows = divb.findAll('p')[1:]

    #Conversion en string
    stri=str(rows)
    #Supression des balises et des espaces du texte
    stri=re.sub('<[^>]+>', '', stri)
    stri=stri.replace('  ','')
    stri = stri.replace('\n', ';')
    stri = stri.replace(';;;', ';;')
    stri = stri.replace(';;', ';')
    stri = stri.replace(';', '\n')
    stri = stri.replace('[', '')
    stri = stri.replace(']', '')
    stri = stri.replace(',', '  ')

    return stri

#Cette fonction permet de récupérer la fiche individuelle du ième joueur
#à partir du lien de la page de l'ensemble joueurs
def Scraping_ficheindiv_players(i,varurl):

    #définition de l'url ou chercher les données :
    html = urlopen(varurl)
    #collecte du Code HTML de la page dans un object Soup :
    soup = BeautifulSoup(html, features="html.parser")
    #création d'une sous-partie de l'html de base
    divb = soup.find("table", {"id":"totals_stats"})
    #récupération des liens des pages des jouerus dans le bon ordre
    liens=divb.findAll('a', attrs={'href': re.compile("^/players/")})
    #récupération de la balise du ième joueur
    balise = liens[i]
    #conversion en type string
    schaine=str(balise)
    #récuperation du lien parmi la balise
    y = schaine.find("html")
    lienprecis = schaine[9:y+4]
    #reconstruction du lien complet
    urlteam="https://www.basketball-reference.com"+lienprecis
    #récupération du sommaire du joueurs grâce à la fonction Scraping_sommaire_from_lien
    ficheindiv=Scraping_ficheindiv_from_lien(urlteam)
    return ficheindiv