from tkinter import *
from tkinter import ttk
from random import randint
import matplotlib.pyplot as plt
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg

root = Tk()
root.title("Operating System GUI")
#root.iconbitmap('python.ico')
root.geometry('500x550')

def hide(widget):
    # This will remove the widget from toplevel
    # widget.place_forget()
    widget.place(y=1000)
#############################################################################
f1 = ttk.Labelframe(root, text="Scheduale Data")
f1.place(x=10, y=5, width=480, height=105)
###########(######## FUNCTION #######################
NoProcess = 5
Quantum = 2
Num_Process = 5
def Scheduale(*args):
    begin()
    if value_inside.get() == options_list[1]:
        lblProcesses.place(x=70, y=59)
        entProcesses.place(width=80, x=200, y=59)
        btnProcesses.place(x=330, y=56)
    elif value_inside.get() == options_list[2] or value_inside.get() == options_list[4]:
        lblProcesses.place(x=70, y=59)
        entProcesses.place(width=80, x=200, y=59)
        btnProcesses.place(x=330, y=56)
        rb1.place(x=265, y=0)
        rb2.place(x=265, y=25)
    elif value_inside.get() == options_list[3]:
        lblProcesses.place(x=70, y=59)
        entProcesses.place(width=80, x=200, y=59)
        btnProcesses.place(x=330, y=56)
        lblQuantum.place(x=10, y=28)
        entQuantum.place(x=105, y=28, width=50)
def Deactivate():
    global NoProcess, Quantum
    if entProcesses.get() == "":
        NoProcess = randint(2, 5)
    if entQuantum.get() == "":
        Quantum = randint(2, 3)
    rb1.configure(state=['disabled'])
    rb2.configure(state=['disabled'])
    question_menu.configure(state=['disabled'])
    btnProcesses.configure(state=['disabled'])
    if pree.get() != 'Preemptive' and pree.get() != 'Non-Preemptive':
        pree.set('Non-Preemptive')
    if entProcesses.get() == "":
        entProcesses.insert(0, str(NoProcess))
    if entQuantum.get() == "":
        entQuantum.insert(0, str(Quantum))
    entQuantum.configure(state=['disabled'])
    entProcesses.configure(state=['disabled'])
def Submit():
    global NoProcess, Quantum, num, Num_Process
    if entProcesses.get() != "":
        NoProcess = int(entProcesses.get())
    if value_inside.get() == options_list[3] and entQuantum.get() != "":
        Quantum = int(entQuantum.get())
    Deactivate()
    Show_Processes_Data()
    if len(lstProcesses) == 0:
        lstProcesses.append(lstTime)
        lstProcesses.append(lstPriority)
        lstProcesses.append(lstArrival)
    num = 0
    string.set("Process no. " + str(num))
    lblProcess.configure(text=string.get())
    Num_Process = NoProcess
def Show_Processes_Data():
    lblProcess.place(x=30, y=0)
    lblTime.place(x=155, y=25)
    lblArrival.place(x=5, y=25)
    #(x=5, y=25) (x=75, y=25, width=50), (x=145, y=25) (x=203, y=25, width=50), (x=270, y=25) (x=350, y=25, width=50)
    entTime.place(x=227, y=25, width=50)
    entArrival.place(x=85, y=25, width=50)
    btnNext.place(x=300, y=52, width=60)
    if value_inside.get() == options_list[4]:
        lblPriority.place(x=297, y=25)
        entPriority.place(x=350, y=25, width=50)
####################################################
# Scheduale Option menu
options_list = ["Scheduale Type", "FCFS", "SJF", "Round-Robin", "Priority"]
value_inside = StringVar(f1)
value_inside.set(options_list[0])
question_menu = ttk.OptionMenu(f1, value_inside, *options_list)
value_inside.trace('w', Scheduale)
question_menu.place(width=120, height=25, x=10, y=0)

# Quantum time lbl & Entry
lblQuantum = ttk.Label(f1, text="Quantum Time :")
lblQuantum.place(x=10, y=28)
entQuantum = ttk.Entry(f1)
entQuantum.place(x=105, y=28, width=50)

# Preemptive or not
rb1 = ttk.Radiobutton(f1, text="Preemptive")
rb2 = ttk.Radiobutton(f1, text="Non-Preemptive")
pree = StringVar()
rb1.config(variable=pree, value="Preemptive")
rb2.config(variable=pree, value="Non-Preemptive")
rb1.place(x=265, y=0)
rb2.place(x=265, y=25)

# Number of Processes
lblProcesses = ttk.Label(f1, text="Number of Processes : ")
lblProcesses.place(x=70, y=59)
# Processes Entry
entProcesses = ttk.Entry(f1)
entProcesses.place(width=80, x=200, y=59)
# Processes btn
btnProcesses = ttk.Button(f1, text="Submit")
btnProcesses.config(command=lambda: Submit())
btnProcesses.place(x=330, y=56)
#################################################################################
f2 = ttk.Labelframe(root, text="Processes Data")
f2.place(x=10, y=115, width=480, height=100)
f3 = ttk.Labelframe(root, text="Gantt Chart")
f3.place(x=10, y=220, width=480, height=265)
################### FUNCTION #######################
avgWT = 0
# Declaring a figure "gnt"
fig, gnt = plt.subplots()
canvas = FigureCanvasTkAgg(fig, f3)
lstArrival = []
lstTime = []
lstPriority = []
lstProcesses = []
lstProcesses.append(lstTime)
lstProcesses.append(lstPriority)
lstProcesses.append(lstArrival)
lstStart = []
lstEnd = []
lstIndex = []
lstLegend = []
def Next():
    global Num_Process, num, string
    if int(Num_Process) == 1:
        hide(btnNext)
        btnOutput.place(x=380, y=52, width=80)
        entPriority.configure(state=["disabled"])
        entTime.configure(state=["disabled"])
        entArrival.configure(state=["disabled"])
    else:
        num = num + 1
        string.set("Process no. " + str(num))
        lblProcess.configure(text=string.get())
    Num_Process = int(Num_Process) - 1
    if entTime.get() != "":
        lstProcesses[0].append(int(entTime.get()))
    else:
        lstProcesses[0].append(randint(1, 5))
    if entArrival.get() != "":
        lstProcesses[2].append(int(entArrival.get()))
    else:
        lstProcesses[2].append(randint(1, 5))
    if value_inside.get() == options_list[4]:
        if entPriority.get() != "":
            lstProcesses[1].append(int(entPriority.get()))
        else:
            lstProcesses[1].append(randint(1, 5))
    entTime.delete(0, END)
    entArrival.delete(0, END)
    if value_inside.get() == options_list[4]:
        entPriority.delete(0, END)
def Calc():
    global lstStart, lstEnd, lstIndex, avgWT, lstProcesses
    done = 0
    time = 0
    first = 0
    q = []
    tmp = []
    time = min(lstProcesses[2])
    if value_inside.get() == options_list[1]:
        #FCFS
        ##FCFS(NoProcess, lstProcesses[0], lstProcesses[2])
        ##avgWT = waitingTimeFCFS()
        while done < NoProcess:
            for j in range(NoProcess):
                if lstProcesses[2][j] == time:
                    q.append([lstProcesses[0][j], j])
            if len(q) != 0:
                i = q[0][1]
                if done == 0:
                    lstStart.append(time)
                    lstEnd.append(lstStart[-1] + lstProcesses[0][i])
                else:
                    if lstProcesses[2][i] > lstEnd[-1]:
                        lstStart.append(time)
                    else:
                        lstStart.append(lstEnd[-1])
                    lstEnd.append(lstStart[-1] + lstProcesses[0][i])
                lstIndex.append(i)
                q.pop(0)
                done += 1
            time += 1

    elif value_inside.get() == options_list[2] and pree.get() == "Preemptive":
        # SJF #Preemptive
        while done < NoProcess:
            if len(q) == 0:
                first = 1
            else:
                first = 0
            for j in range(NoProcess):
                if lstProcesses[2][j] == time:
                    q.append([lstProcesses[0][j], j])
            q.sort()
            if len(q) != 0:
                i = q[0][1]
                if q[0][0] != 999:
                    if q[0][0] > 1:
                        if first == 1:
                            lstStart.append(time)
                        else:
                            if lstProcesses[2][i] > lstEnd[-1]:
                                lstStart.append(time)
                            else:
                                lstStart.append(lstEnd[-1])
                        q[0][0] -= 1
                        lstEnd.append(time + 1)
                    else:
                        if first == 1:
                            lstStart.append(time)
                        else:
                            if lstProcesses[2][i] > lstEnd[-1]:
                                lstStart.append(time)
                            else:
                                lstStart.append(lstEnd[-1])
                        lstEnd.append(time + 1)
                        q[0][0] = 999
                        done += 1
                    lstIndex.append(i)
            time += 1

    elif value_inside.get() == options_list[2] and pree.get() == "Non-Preemptive":
        #SJF #Non-Preemptive
        while done < NoProcess:
            for j in range(NoProcess):
                if lstProcesses[2][j] == time:
                    q.append([lstProcesses[0][j], j])
            q.sort()
            if len(q) != 0:
                i = q[0][1]
                if q[0][0] != 999:
                    if done == 0:
                        lstStart.append(time)
                        lstEnd.append(lstStart[-1] + lstProcesses[0][i])
                    else:
                        if lstProcesses[2][i] > lstEnd[-1]:
                            lstStart.append(time)
                        else:
                            lstStart.append(lstEnd[-1])
                        lstEnd.append(lstStart[-1] + lstProcesses[0][i])
                    lstIndex.append(i)
                    q[0][0] = 999
                    done += 1
            time += 1
    elif value_inside.get() == options_list[3]:
        #Round Robin
        #rr(NoProcess)
        for i in range(NoProcess):
            tmp.append(lstProcesses[2][i])
        for j in range(NoProcess):
            if tmp[j] <= time and tmp[j] != -1:
                q.append([lstProcesses[0][j], j])
                tmp[j] = -1
        while done < NoProcess:
            if len(q) != 0:
                i = q[0][1]
                if lstProcesses[0][i] > Quantum:
                    lstProcesses[0][i] = lstProcesses[0][i] - Quantum
                    lstStart.append(time)
                    lstEnd.append(lstStart[-1] + Quantum)
                    time += Quantum
                else:
                    lstStart.append(time)
                    lstEnd.append(lstStart[-1] + lstProcesses[0][i])
                    time += lstProcesses[0][i]
                    lstProcesses[0][i] -= lstProcesses[0][i]
                    done += 1
                lstIndex.append(i)
                for j in range(NoProcess):
                    if tmp[j] <= time and tmp[j] != -1:
                        q.append([lstProcesses[0][j], j])
                        tmp[j] = -1
                if lstProcesses[0][i] > 0:
                    q.append([lstProcesses[0][i], i])
                q.pop(0)
            else:
                for j in range(NoProcess):
                    if tmp[j] <= time and tmp[j] != -1:
                        q.append([lstProcesses[0][j], j])
                        tmp[j] = -1
                time += 1
    elif value_inside.get() == options_list[4] and pree.get() == "Preemptive":
        #Priority #Preemptive
        while done < NoProcess:
            if len(q) == 0:
                first = 1
            else:
                first = 0
            for j in range(NoProcess):
                if lstProcesses[2][j] == time:
                    q.append([lstProcesses[1][j], j])
            q.sort()
            if len(q) != 0:
                i = q[0][1]
                if q[0][0] != 999:
                    if lstProcesses[0][i] > 1:
                        if first == 1:
                            lstStart.append(time)
                        else:
                            if lstProcesses[2][i] > lstEnd[-1]:
                                lstStart.append(time)
                            else:
                                lstStart.append(lstEnd[-1])
                        lstProcesses[0][i] -= 1
                        lstEnd.append(time + 1)
                    else:
                        if first == 1:
                            lstStart.append(time)
                        else:
                            if lstProcesses[2][i] > lstEnd[-1]:
                                lstStart.append(time)
                            else:
                                lstStart.append(lstEnd[-1])
                        lstEnd.append(time + 1)
                        q[0][0] = 999
                        done += 1
                    lstIndex.append(i)
            time += 1

    elif value_inside.get() == options_list[4] and pree.get() == "Non-Preemptive":
        while done < NoProcess: #Priority #nonPreemptive
            for j in range(NoProcess):
                if lstProcesses[2][j] == time:
                    q.append([lstProcesses[1][j], j])
            q.sort()
            if len(q) != 0:
                i = q[0][1]
                if q[0][0] != 999:
                    if done == 0:
                        lstStart.append(time)
                        lstEnd.append(lstStart[-1] + lstProcesses[0][i])
                        lstIndex.append(i)
                        q[0][0] = 999
                        done += 1
                    else:
                        if time >= lstEnd[-1]:
                            if lstProcesses[2][i] > lstEnd[-1]:
                                lstStart.append(time)
                            else:
                                lstStart.append(lstEnd[-1])
                            lstEnd.append(lstStart[-1] + lstProcesses[0][i])
                            lstIndex.append(i)
                            q[0][0] = 999
                            done += 1
            time += 1
    ### When finish delete these lists ###
    #lstStart = [0, 5, 8, 10, 13, 17]
    #lstEnd = [5, 8, 10, 13, 17, 22]
    #lstIndex = [1, 2, 1, 3, 4, 3]
    ######################################
    lstProcesses.append(lstStart)
    lstProcesses.append(lstEnd)
    lstProcesses.append(lstIndex)
def Get_Color(Num):
    if Num % 10 == 0:
        return 'tab:green'
    elif Num % 10 == 1:
        return 'tab:blue'
    elif Num % 10 == 2:
        return 'tab:orange'
    elif Num % 10 == 3:
        return 'tab:red'
    elif Num % 10 == 4:
        return 'tab:purple'
    elif Num % 10 == 5:
        return 'tab:brown'
    elif Num % 10 == 6:
        return 'tab:pink'
    elif Num % 10 == 7:
        return 'tab:gray'
    elif Num % 10 == 8:
        return 'tab:cyan'
    elif Num % 10 == 9:
        return 'tab:olive'
def Unique_List():
    global lstProcesses, lstLegend, NoProcess
    for i in range(len(lstProcesses[5])):
        flag = False
        for j in range(len(lstLegend)):
            if lstLegend[j] == lstProcesses[5][i]:
                flag = True
                break
        if flag == False:
            lstLegend.append(lstProcesses[5][i])
def Gantt():
    global NoProcess, toolbar, canvas, fig, gnt, lstLegend
    Calc()
    #Draw The Chart
    # Setting X,Y-axis limits
    gnt.set_ylim(0, 20)
    Sum = lstProcesses[4][len(lstProcesses[5])-1]
    Sum += Sum / 10
    gnt.set_xlim(0, Sum)
    # Setting labels for x-axis and y-axis
    gnt.set_xlabel('Time in clks')
    gnt.set_ylabel('Tasks')
    # Setting ticks on y-axis
    lstYLines = [5, 20, 30]
    lstYNames = []
    lstXLines = []
    Unique_List()
    gnt.set_yticks(lstYLines)
    for i in range(1, NoProcess+1):
        if (value_inside.get() == options_list[4] and pree.get() == "Preemptive") or (value_inside.get() == options_list[2] and pree.get() == "Preemptive") or value_inside.get() == options_list[3]:
            Str = "P" + str(lstLegend[i - 1])
        else:
            Str = "P" + str(lstProcesses[5][i - 1])
        lstYNames.append(Str)
        gnt.set_yticklabels([])
    if (value_inside.get() == options_list[2] and pree.get() == "Preemptive") or (value_inside.get() == options_list[4] and pree.get() == "Preemptive"):
        new_start = []  # new start array for drawing
        new_end = []  # new end array for drawing
        new_index = []  # new index array for drawing
        for k in range(len(lstIndex)):
            if k == 0:
                new_start.append(lstStart[0])
                new_index.append(lstIndex[k])
            elif lstIndex[k] != lstIndex[k - 1]:
                new_end.append(lstEnd[k - 1])
                new_start.append(lstStart[k])
                new_index.append(lstIndex[k])
        new_end.append(lstEnd[-1])  # To add the last item in end array
        lstProcesses[3].clear()
        lstProcesses[4].clear()
        lstProcesses[5].clear()
        lstProcesses[3] = new_start
        lstProcesses[4] = new_end
        lstProcesses[5] = new_index
    lstXLines.append(0)
    if lstProcesses[3][0] != 0:
        lstXLines.append(lstProcesses[3][0])
    for i in range(len(lstProcesses[5])):
        if lstXLines[-1] != lstProcesses[3][i]:
            lstXLines.append(lstProcesses[3][i])
        lstXLines.append(lstProcesses[4][i])
        gnt.set_xticks(lstXLines)
        gnt.set_xticklabels(lstXLines)
    # Labelling tickes of y-axis
    # Setting graph attribute
    gnt.grid(True)
    # Declaring a bar in schedule
    for i in range(len(lstProcesses[3])):
        gnt.broken_barh([(lstProcesses[3][i], lstProcesses[4][i]-lstProcesses[3][i])], (5, 15), facecolors=(Get_Color(lstProcesses[5][i])))
    gnt.legend(lstYNames, bbox_to_anchor=(0.9, 1.15), loc='upper left')
    if (value_inside.get() == options_list[4] and pree.get() == "Preemptive") or (
            value_inside.get() == options_list[2] and pree.get() == "Preemptive") or value_inside.get() == options_list[3]:
        ax = plt.gca()
        leg = ax.get_legend()
        for i in range(len(lstLegend)):
            leg.legendHandles[i].set_color(Get_Color(lstLegend[i]))
    # Declaring multiple bars in at same level and same widthgnt.clear()
    canvas.draw()
    Show_AvgTime()
    btnOutput.config(state=["disabled"])
def avgWaitingTime (n,start,end,at,index):
    wt = []
    for i in range (n):
        count = 0
        wait = 0
        for j in range (len(index)):
            if i == index[j]:
                count += 1
        for j in range (len(index)):
            if i != index[j]:
                gap = 0
                if start[j] != end[j-1] and j != 0:
                    gap = start[j] - end[j-1]
                wait += end[j] - start[j] + gap
            else:
                count -= 1
                if count == 0:
                    break;
        wait -= (at[i] - start[0])
        if wait < 0:
            wait = 0
        wt.append(wait)
    s = 0   #sum
    for j in range (len(wt)):
        s += wt[j]
    avg = s / n     #average time
    return avg
def Show_AvgTime():
    global strAvg, avgWT, NoProcess
    lblAvgWT.place(x=15, y=5)
    avgWT = avgWaitingTime(NoProcess, lstProcesses[3], lstProcesses[4], lstProcesses[2], lstProcesses[5])
    strAvg.set(str(avgWT))
    lblAvgWTValue.config(text=strAvg.get())
    lblAvgWTValue.place(x=127, y=5)
####################################################
# label Process
num = 0
string = StringVar()
string.set("Process no. " + str(num))
lblProcess = ttk.Label(f2, text=string.get())
lblProcess.place(x=30, y=0)

# Burst Time lbl & Entry
lblTime = ttk.Label(f2, text="Burst Time :")
lblTime.place(x=5, y=25)
entTime = ttk.Entry(f2)
entTime.place(x=75, y=25, width=50)
# Priority lbl & Entry
lblPriority = ttk.Label(f2, text="Priority :")
lblPriority.place(x=145, y=25)
entPriority = ttk.Entry(f2)
entPriority.place(x=203, y=25, width=50)
# Arrival lbl & Entry
lblArrival = ttk.Label(f2, text="Arrival Time :")
lblArrival.place(x=242, y=25)
entArrival = ttk.Entry(f2)
entArrival.place(x=320, y=25, width=50)
# Next btn
btnNext = ttk.Button(f2, text="Next")
btnNext.config(command=lambda: Next())
btnNext.place(x=300, y=48, width=60)
# Get Output btn
btnOutput = ttk.Button(f2, text="Get Output")
btnOutput.config(command=lambda:Gantt())
btnOutput.place(x=380, y=48, width=80)
#################################################################################
f4 = ttk.Labelframe(root, text="Average Waiting Time")
f4.place(x=10, y=490, width=480, height=50)
################### FUNCTION #######################
strAvg = StringVar()
strAvg.set(avgWT)
def begin():
    global gnt, canvas
    hide(lblQuantum)
    hide(entQuantum)
    hide(rb1)
    hide(rb2)
    hide(entProcesses)
    hide(lblProcesses)
    hide(btnProcesses)
    hide(lblProcess)
    hide(lblTime)
    hide(entTime)
    hide(lblArrival)
    hide(entArrival)
    hide(lblPriority)
    hide(entPriority)
    hide(btnOutput)
    hide(btnNext)
    hide(lblAvgWT)
    hide(lblAvgWTValue)
    canvas.draw()
    canvas.get_tk_widget().place(x=5, y=10, width=468, height=225)
    gnt.clear()
def clean():
    entQuantum.configure(state=['!disabled'])
    entProcesses.configure(state=['!disabled'])
    entTime.configure(state=['!disabled'])
    entArrival.configure(state=['!disabled'])
    entPriority.configure(state=['!disabled'])
    rb1.configure(state=['!disabled'])
    rb2.configure(state=['!disabled'])
    question_menu.configure(state=['!disabled'])
    btnProcesses.configure(state=['!disabled'])
    btnOutput.configure(state=['!disabled'])
    entQuantum.delete(0, END)
    entProcesses.delete(0, END)
    entTime.delete(0, END)
    entArrival.delete(0, END)
    entPriority.delete(0, END)
    pree.set("")
def start():
    begin()
    value_inside.set(options_list[0])
    clean()
    for i in range(len(lstProcesses)):
        lstProcesses[i].clear()
    lstProcesses.clear()
    lstLegend.clear()
    lstStart.clear()
    lstEnd.clear()
    lstIndex.clear()
    lstTime.clear()
    lstArrival.clear()
    lstPriority.clear()
####################################################
# Average Waiting Time Show
lblAvgWT = ttk.Label(f4, text="Avg Waiting Time : ")
lblAvgWT.place(x=15, y=5)
# Deactivated Entry
lblAvgWTValue = ttk.Label(f4, text=strAvg.get())
lblAvgWTValue.place(x=127, y=5)
# Reset btn
btnReset = ttk.Button(f4, text="Reset")
btnReset.config(command=lambda: start())
btnReset.place(x=390, y=3, width=80)
btnReset.invoke()

root.mainloop()


import numpy as np
import matplotlib.pyplot as plt

vanilla_model = "200d_1x128LSTM_2x128out_50drop.txt"
pos_model = "200d_50pos_1x128LSTM_2x128out_50drop_2.txt"

def main():
    data_vanilla = load_data(vanilla_model)
    data_pos = load_data(pos_model)

    print(data_vanilla.shape)
    print(data_pos.shape)
    show_plot(data_vanilla, data_pos)

def load_data(path):
    # Return [None, 4]
    # 0:Train loss
    # 1:Train acc
    # 2:Valid loss
    # 3:Valid acc
    file = open(path)

    data = []
    for line in file:
        data.append(str_to_num(line))

    return np.array(data)

def str_to_num(str):
    # Gets a string containing numbers separated by space and returns a
    # list of floats.
    return [float(num_str) for num_str in str.split()]

def show_plot(data_v, data_p):
    # Accuracy plot
    plt.subplot(2,1,1)
    plt.plot(data_v[:,-1], label="Vanilla")
    plt.plot(data_p[:,-1], label="POS model")
    plt.ylabel("Validation accuracy")
    plt.legend(loc="lower right")

    plt.subplot(2,1,2)
    plt.plot(data_v[:,2], label="Vanilla")
    plt.plot(data_p[:,2], label="POS model")
    plt.ylabel("Average validation cross entropy")
    plt.xlabel("Iterations")
    plt.legend(loc="upper right")
    plt.show()
    return None

main()


from api.models import Garbage, Category
from rest_framework import serializers

class CategorySerializer(serializers.ModelSerializer):
    class Meta:
        model = Category
        fields = "__all__"

class GarbageSerializer(serializers.HyperlinkedModelSerializer):
    category = CategorySerializer()
    class Meta:
        model = Garbage
        fields = ('name', 'category')

class GarbageFuzzySearchSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Garbage
        fields = ['name']


import csv

import numpy as np

from interpolation import create_vandermonde, monomial_basis

if __name__ == '__main__':
    # set configuration
    NUM_INTERPOLATION_NODES = 20
    SAMPLE_POINTS = 100

    def f(x_i):
        return 1 / (1 + (10 * x_i) ** 2)

    # create the vandermonde
    max_polynomial_degree = 10
    vandermonde_matrix, _ = create_vandermonde(max_polynomial_degree)
    with open('vandermonde.csv', "w", newline="") as s:
        writer = csv.writer(s)
        writer.writerows(vandermonde_matrix.round(5))

    # running an interpolation
    coefficients = monomial_basis(f, NUM_INTERPOLATION_NODES, SAMPLE_POINTS, plot_save_show='save',
                                  file_prefix='prblm1-')

    # storing coefficients
    with open('coefficients.csv', "w", newline="") as s:
        writer = csv.writer(s)
        r = np.reshape(coefficients, (len(coefficients), 1))
        writer.writerows(r)


from __future__ import print_function
import sys, itertools

def combs(string, num):
    return sorted([ ''.join(sorted(x)) for x in itertools.combinations_with_replacement(string, num)])

def main():
    string, num = sys.stdin.read().strip().split()
    for line in combs(string, int(num)):
        print(line)

if __name__ == '__main__':
    main()


import logging
import sqlite3
from uuid import uuid4

log = logging.getLogger(__name__)

def create_connection():
    conn = sqlite3.connect("db/tournament_tool.db", check_same_thread=False)
    return conn

conn = create_connection()

class Tournament:
    def __init__(self):
        pass

    def check_active_tournament(self):
        cursor = conn.cursor()

        cursor.execute(
            "SELECT tournament_id FROM tournaments WHERE tournament_status = 'ONGOING'")

        tournament = cursor.fetchone()

        if not tournament:
            print("NO ACTIVE TOURNAMENT")
            return None

        return tournament[0]

    def init_players(self, tour_id):
        cursor = conn.cursor()

        with open("players.txt") as f:
            for conf in f.readlines():

                name = conf.split(" ")[0].replace("\n", "")
                cursor.execute(
                    """
                        INSERT INTO players (player_name, tournament_id) VALUES (?, ?)
                    """, (name, tour_id))
                conn.commit()

    def init_options(self, tournament_id):
        cursor = conn.cursor()

        cursor.execute("""
                INSERT INTO tournament_options (
                    tournament_id,
                    map,
                    time_limit
                ) VALUES (
                    ?,
                    1,
                    120
                )
                """, (tournament_id, ))
        conn.commit()

    def init_matches(self, tour_id):
        cursor = conn.cursor()

        cursor.execute(
            "SELECT player_id FROM players WHERE tournament_id = ?", (tour_id, ))

        players = cursor.fetchall()
        match_count = int(len(players) / 2)

        print(f"PLAYER COUNT: {len(players)}")
        print(f"MATCH COUNT: {match_count}")

        ct_side = [p[0] for p in players[:match_count]]
        t_side = [p[0] for p in players[match_count:]]

        for pair in range(match_count):

            match_name = f"{ct_side[pair]} VS {t_side[pair]}"
            match_id = str(uuid4())
            cursor.execute("""
                INSERT INTO matches (
                    match_id,
                    match_name,
                    map_id,
                    tournament_id
                ) VALUES (
                    ?,
                    ?,
                    1,
                    ?
                )
            """, (match_id, match_name, tour_id))
            conn.commit()

            params = [
                (match_id, ct_side[pair], "CT"),
                (match_id, t_side[pair], "T")
            ]

            cursor.executemany("""
                INSERT INTO players_in_matches (
                    match_id,
                    player_id,
                    side
                ) VALUES
                    (?, ?, ?)
            """, params)
            conn.commit()

    def activate_tournament(self, tour_id):
        cursor = conn.cursor()

        cursor.execute(
            "UPDATE tournaments SET tournament_status = 'ONGOING' WHERE tournament_id = ?", (tour_id, ))
        conn.commit()

    def init(self):
        cursor = conn.cursor()

        tour_id = str(uuid4())

        cursor.execute(
            "INSERT INTO tournaments (tournament_id, tournament_name) VALUES (?, 'DEMO TOURNAMENT')", (tour_id, ))
        conn.commit()

        self.init_options(tour_id)
        self.init_players(tour_id)
        self.init_matches(tour_id)
        self.activate_tournament(tour_id)

if __name__ == "__main__":
    tournament = Tournament()

    tournament.init()


from django.db import models

ZONA_MAX_LENGTH = 50

# Create your models here.
class Map(models.Model):
    Zona = models.CharField(max_length = ZONA_MAX_LENGTH)
    Latitud = models.FloatField()
    Longitud = models.FloatField()

    class Meta:
        verbose_name_plural = 'Mapa'

    def __str__(self):
        return self.title


import traceback
import inspect
import json
import simplejson

import importlib.util
import sys
import random

try:
	arg=json.loads(simplejson.dumps(args[0], use_decimal=True))
	DEBUG = plpy.execute('SELECT is_debug()')[0]['is_debug']
	assert isinstance(arg, dict),'Проверь формат JSON'
	assert 'parts' in arg,'Нужен параметр parts'
	assert 'previous_lay' in arg,'Нужен параметр previous_lay'
	assert 'pallet_length_x' in arg,'Нужен параметр pallet_length_x'
	assert 'pallet_length_y' in arg,'Нужен параметр pallet_length_y'
	assert 'part_indent' in arg,'Нужен параметр part_indent'

	def level_generator(start: int, max_: int) -> int:
		"""
		Генератор уровней
		:param start: начало отсчета
		:param max_: максимальный уровень
		:return: уровень
		"""
		while True:
				if start >= max_:
						start = 0
				yield start
				start += 1
	def sort_thickness_parts(parts_l: dict) -> dict:
		"""
		Сортировка по толщине
		:param parts_l: список деталей
		:return: детали сгруппированные по толщине
		"""
		parts_ = dict()
		for part_ in parts_l:
				parts_[part_['part_thickness_z']]=list()
				parts_[part_['part_thickness_z']].append(part_)
		return dict(parts_)

	sys.path.append('/home/kipia/opc_BHX')
	from py3dbp import Bin, Item, Packer, Painter

	parts, previous_lay, pallet_w, pallet_h, indent = (arg['parts'], arg['previous_lay'], arg['pallet_length_x'], arg['pallet_length_y'], arg['part_indent'])
	pallet_s = pallet_w * pallet_h  # площадь палеты
	sort_parts = sort_thickness_parts(parts)
	max_percent = 0  # наилучший результат заполнения
	result_parts = None  # список деталей при наилучшем заполнении
	# ищем лучший результат
	for thickness in sort_parts:
		# формируем паллету по толщине деталей
		pallet = Bin(partno='Bin', WHD=(pallet_w, pallet_h, thickness), max_weight=28080, corner=1, put_type=1)
		for i in range(len(parts)):
			# меняем приоритет деталей
			level = level_generator(i, len(sort_parts[thickness]))
			item_list = []
			for part in parts:
				item_list.append(Item(partno=part['part_id'],
									  name='part',
									  typeof='cube',
									  WHD=(part['part_length_x'] + indent,
										   part['part_length_y'] + indent,
										   part['part_thickness_z']),
									  level=next(level),
									  weight=1,
									  loadbear=0,
									  updown=False,
									  color="#" + ''.join([random.choice('0123456789ABCDEF') for j in range(6)])))
			packer = Packer()
			packer.addBin(pallet)
			for item in item_list:
				packer.addItem(item)
			# пакуем
			packer.pack(
				bigger_first=False,
				fix_point=True,
				binding=[('server', 'cabint')]
			)
			b = packer.bins[0]
			S = 0   # общая площадь занимаемая деталями
			layer_parts = []  # детали 0 слоя
			for item in b.items:
				if item.name != 'corner' and item.position[2] == 0:
					S += (item.width * item.height)
					item.center = (item.position[0] + int(item.width / 2), item.position[1] + int(item.height / 2))
					layer_parts.append({'part_pos_x': int(item.center[0]),
                                        'part_pos_y': int(item.center[1]),
                                        'part_id': item.partno,
                                        'part_angle_a': 0 if item.rotation_type == 0 else 90})
			percent = int((S / pallet_s) * 100)  # процент заполнения слоя
			if percent > max_percent:
				max_percent = percent
				result_parts = layer_parts.copy()
	res = {
		"filling": max_percent,
		"parts": result_parts
	}
	return res
except Exception as e:
	traceback_info=traceback.format_exc().splitlines()[1].split(', ')
	source=traceback_info[1]+' '+'_'.join(traceback_info[2].split('_')[slice(4,-1)])
	err_arg = [traceback_info[1]+': '+str(e),simplejson.dumps(args,ensure_ascii=False,use_decimal=True)]
	res = plpy.execute(plpy.prepare(f'SELECT write_error_log($1,$2)', ['text','text']), err_arg)
	return dict(source=source,error=str(e),args=arg)


# coding: utf-8

# In[ ]:

from scipy import sparse

train_tfidf=sparse.load_npz("./npy/data1_tfidf_train_x.npz")
train_count_fea=sparse.load_npz("./npy/data1_count_fea_train_x.npz")
train_x=sparse.hstack((train_tfidf,train_count_fea))

# In[ ]:

test_tfidf=sparse.load_npz("./npy/data1_tfidf_test_x.npz")
test_count_fea=sparse.load_npz("./npy/data1_count_fea_test_x.npz")
test_x=sparse.hstack((test_tfidf,test_count_fea))

# In[ ]:

import numpy as np
train_y=np.load("./npy/data1_train_y.npy")

# In[ ]:

from sklearn.model_selection import  StratifiedKFold
import lightgbm as lgb
from sklearn.metrics import  log_loss
nfold=5

kf=StratifiedKFold(nfold)
train_x=train_x.tocsr()
test_x=test_x.tocsr()

params = {
    'max_depth':-1,
    'boosting_type':'gbdt',
    'num_leaves': 31,
    'objective': 'multiclass',
    'min_data_in_leaf': 200,
    'learning_rate': 0.02,
    'max_bin':255,
    'feature_fraction': 0.8,
    'bagging_fraction': 0.7,
    'bagging_freq': 1,
    'metric': 'multi_logloss',
    'num_class':6,
    'num_threads': -1,
    'is_unbalance':True,
}
import numpy as np
oof_train=np.zeros((train_x.shape[0],6))
oof_test=np.zeros((test_x.shape[0],6))
oof_test_skf=np.zeros((nfold,test_x.shape[0],6))
logloss_csv=[]

from sklearn.linear_model import LogisticRegression

for i,(train,test) in  enumerate(kf.split(train_x,train_y)):
    print("*******第 %d 折********"%i)

    x_tr=train_x[train]
    x_te=train_x[test]
    y_tr=train_y[train]
    y_te=train_y[test]

    lr=LogisticRegression()
    lr.fit(x_tr,y_tr)
    eva_pre=lr.predict_proba(x_te)
    oof_train[test]=eva_pre
    logloss=log_loss(y_te,eva_pre)
    print("valid logloss %s"%logloss)
    logloss_csv.append(logloss)
    pre=lr.predict_proba(test_x)
    oof_test_skf[i,:]=pre

print("mean loss  %f"%np.mean(logloss_csv))

# In[ ]:

oof_test=oof_test_skf.mean(axis=0)
score=np.mean(logloss_csv)
print(score)

# In[ ]:

import numpy as np
def check_ans(result):
    new_result=[]
    for i in result:
        m=i.sum()
        for j in range(0,6):
            i[j]=i[j]/m
        if abs(i.sum()-1)>=1e-6:
            print(i.sum()-1)
        new_result.append(i)
    return new_result

# In[ ]:

import numpy as np

res=check_ans(oof_test)

# In[ ]:

#
test_prob=pd.DataFrame(oof_test)
test_prob.columns=['prob0','prob1','prob2','prob3','prob4','prob5']
test_prob['file_id']=test_data['file_id']

# In[ ]:

test_prob=test_prob[['file_id','prob0','prob1','prob2','prob3','prob4','prob5']]
test_prob.head(20)

# In[ ]:

test_prob.to_csv("lgb_submit_%f.csv"%np.mean(logloss_csv),index=False)

# In[ ]:

test_prob.info()

# In[ ]:

#保存train_prob
import pandas as pd
train_num=1
train_prob=pd.DataFrame(oof_train)
train_prob.columns=['class_%s'%i for i  in   range(0,6)]
test_prob=pd.DataFrame(oof_test)
test_prob.columns=['class_%s'%i for i in range(0,6)]
train_prob.to_csv("./stacking_csv/train%d_lr_train_prob_cv5_%f.csv"%(train_num,score),index=False)
test_prob.to_csv("./stacking_csv/train%d_lr_test_prob_cv5_%f.csv"%(train_num,score),index=False)

# In[ ]:

np.mean(logloss_csv)

# In[ ]:

from sklearn.metrics import  classification_report
y_pr=[np.argmax(i) for i in eva_pre]
print(classification_report(y_te,y_pr))


from magicbus.compat import ntob
import os
thismodule = os.path.abspath(__file__)
import sys

import pytest

from magicbus import bus
from magicbus.plugins import opsys
from magicbus.test import Process, WebAdapter, WebService
from magicbus.test import WebHandler

from magicbus.plugins import loggers
loggers.StdoutLogger(bus).subscribe()

pidfile = opsys.PIDFile(bus, os.path.join(thismodule + '.pid'))

class Handler(WebHandler):

    bus = bus

    def do_GET(self):
        if self.path == '/':
            self.respond('Hello World')
        elif self.path == '/pid':
            self.respond(str(os.getpid()))
        elif self.path == '/exit':
            self.respond('ok')
            self.bus.transition('EXITED')
        else:
            self.respond(status=404)
service = WebService(handler_class=Handler)

@pytest.mark.xfail(
    sys.version_info[0] > 2, reason="""
    Hangs under Python 3 because it loops too fast causing a race
    condition in `magicbus.plugins.opsys.PIDFile.join()`.

    Refs:
    * https://github.com/cherrypy/magicbus/issues/7
    * https://github.com/cherrypy/magicbus/pull/8
    """,
    run=False,
)
@pytest.mark.skipif(os.name != 'posix', reason='not on POSIX')
def test_daemonize():
    # Spawn the process and wait, when this returns, the original process
    # is finished.  If it daemonized properly, we should still be able
    # to access pages.
    p = Process([sys.executable, thismodule, 'daemonize'])
    p.start()
    pidfile.wait()
    try:
        # Just get the pid of the daemonization process.
        resp = service.do_GET('/pid')
        assert resp.status == 200
        page_pid = int(resp.read())
        with open(pidfile.pidfile, 'rb') as pid_file:
            actual_pid = pid_file.read()
        assert ntob(str(page_pid)) == actual_pid
    finally:
        # Shut down the spawned process
        service.do_GET('/exit')
    pidfile.join()

    # Wait until here to test the exit code because we want to ensure
    # that we wait for the daemon to finish running before we fail.
    p.process.wait()
    assert p.process.returncode == 0, (
        'Daemonized parent process returned exit code {rc!s}.'.
        format(rc=p.process.returncode)
    )

if __name__ == '__main__':
    mode = sys.argv[1]
    if mode == 'daemonize':
        opsys.Daemonizer(bus).subscribe()
    pidfile.subscribe()
    WebAdapter(bus, service).subscribe()
    bus.transition('RUN')
    bus.block()


from rsf.proj import *
from math import *
import math

## model (Minimum-phase wavelets with different frequencies)
Flow('spike', None, 'spike n1=1001 d1=0.001 o1=0 label1=Sample unit1=')

waves = []
for j in range(200):
    wave = 'wave%d' % j

    fre = 0.5+j*0.5

    Flow(wave, 'spike',
         '''
         math output="2*%g*(%g)*x1" |
         math output="exp(-input*input/25)*sin(input)"
         '''
         % (math.pi,fre) )
    waves.append(wave)

Flow('in', waves, 'cat ${SOURCES[1:%d]} axis=2 d=0.5 o=0.5' % len(waves))
Flow('max_pos','in','findmax1 verb=n|dd type=float')
Flow('fre',None,'math n1=200 d1=0.5 o1=0.5 output="1"')
Flow('nihe','fre',
     '''
     math output="0.232/x1"|math output="input/0.001"|dd type=int|
     dd type=float
     ''')
Result('gapdif', 'max_pos nihe',
       '''
       cat axis=2 ${SOURCES[1:2]} |
       graph title="" plotcol=5,6 dash=0,1 crowd1=0.6 crowd2=0.4
       label1=Frequency label2=Max_point_interval plotfat=8 unit1=Hz
       labelfat=3 font=2
       ''')

End()


# coding=utf-8
import re, sys
reload(sys)
sys.setdefaultencoding("utf-8")

import scrapy

class StackOverflowSpider(scrapy.Spider):
    name = 'priceza'
    start_urls = ['http://www.priceza.com.my/search', 'http://www.priceza.com.sg/search', 'http://www.priceza.com.ph/search', 'http://www.priceza.com.vn/search', 'http://www.priceza.com/search', 'http://www.priceza.co.id/search']

    def parse(self, response):
        for item in response.css('div.products div.item.group'):

            prices = item.css('small a::text').extract()[0]
            prices = re.sub("[^0-9]", "", prices)

            try:
                prices = int(prices)

                if prices > 1:
                    yield {
                        'name': item.css('h3::text').extract()[0],
                        'prices': prices,
                        'url' : response.urljoin(item.css('a::attr(href)').extract()[0])
                    }
            except:
                pass

        next_page = response.css('div.paging a.next::attr(href)')
        if len(next_page) > 0:
            full_url = response.urljoin(next_page[0].extract())
            yield scrapy.Request(full_url, callback=self.parse)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import scanpy.api as sc
import scanpy.external as sce
import os
import louvain
# import phate
import seaborn as sns
import anndata as ad
import time
import random
import tensorflow as tf
import statsmodels.api as sm
from sklearn.model_selection import train_test_split

from matplotlib import cm, colors

from palettable.scientific.sequential import Bilbao_20
from palettable.cartocolors.qualitative import Vivid_3
from palettable.cartocolors.qualitative import Vivid_4
from palettable.cartocolors.qualitative import Bold_4
from palettable.cartocolors.qualitative import Bold_6
from palettable.cartocolors.qualitative import Vivid_10
from palettable.scientific.diverging import Vik_20
from palettable.scientific.diverging import Berlin_10

random.seed(42)
np.random.seed(42)
tf.set_random_seed(42)

from model import SAUCIE
from loader import Loader

sc.settings.verbosity = 4  # verbosity: errors (0), warnings (1), info (2), hints (3)
sc.settings.set_figure_params(dpi=100, dpi_save=400)  # low dpi (dots per inch) yields small inline figures
sc.logging.print_versions()
sc.settings.figdir='./paper.figs/fig5/'

print(os.getcwd())
os.chdir("/home/UTHSCSA/iskra/CytOF_Datasets/10.08-10.10-Debarcoded/GG")

mapping = ['CD45', 'CD11b', 'FolR2', 'IAIE', 'TNFa', 'CD140a',
       'Vimentin', 'CD90', 'Ly6C', 'Ly6AE', 'Postn', 'IdU',
       'IL6', 'CD9', 'VEGF', 'CD146', 'CD200', 'Notch3',
       'aSMA', 'CD31', 'VEGFR2', 'ActCaspase3', 'P53', 'MTCO1',
       'HSP60', 'Ki67', 'pRb', 'CyclinB1', 'P21', 'bCatTotal',
       'bCatActive', 'pHistoneH3', 'Sample', 'Batch']

cytof_marks = ['CD45', 'CD11b', 'FolR2', 'IAIE', 'TNFa', 'CD140a',
       'Vimentin', 'CD90', 'Ly6C', 'Ly6AE', 'Postn', 'IdU',
       'IL6', 'CD9', 'VEGF', 'CD146', 'CD200', 'Notch3',
       'aSMA', 'CD31', 'VEGFR2', 'ActCaspase3', 'P53', 'MTCO1',
       'HSP60', 'Ki67', 'pRb', 'CyclinB1', 'P21', 'bCatTotal',
       'bCatActive', 'pHistoneH3']
expr3 = pd.read_csv("10.08.2019.gg.in.vitro.b4.csv", sep=',')

expr3_adata = ad.AnnData(expr3[mapping[:-2]])
expr3_adata.obs['Sample'] = expr3['Sample'].values
expr3_adata.obs['Batch'] = 1

vitro_adata1 = sc.pp.subsample(expr3_adata[expr3_adata.obs['Sample'] == np.unique(expr3_adata.obs['Sample'])[0]],
                              n_obs=11250,
                              random_state=42,
                              copy=True)

vitro_adata2 = sc.pp.subsample(expr3_adata[expr3_adata.obs['Sample'] == np.unique(expr3_adata.obs['Sample'])[1]],
                              n_obs=11250,
                              random_state=42,
                              copy=True)

vitro_adata3 = sc.pp.subsample(expr3_adata[expr3_adata.obs['Sample'] == np.unique(expr3_adata.obs['Sample'])[2]],
                              n_obs=11250,
                              random_state=42,
                              copy=True)

vitro_adata4 = sc.pp.subsample(expr3_adata[expr3_adata.obs['Sample'] == np.unique(expr3_adata.obs['Sample'])[3]],
                              n_obs=11250,
                              random_state=42,
                              copy=True)

expr3_adata = ad.AnnData.concatenate(vitro_adata1,
                                     vitro_adata2,
                                     vitro_adata3,
                                     vitro_adata4)

expr3_adata.var_names_make_unique()
sc.pp.filter_cells(expr3_adata, min_genes=1)

expr3_adata.obs['n_counts'] = expr3_adata.X.sum(axis=1)
sc.pl.violin(expr3_adata, ['n_counts', 'n_genes'],
             jitter=0.4, multi_panel=True, stripplot=False,
              save='vitro.pre.qc.plots.png'
             )

sc.pp.neighbors(expr3_adata, n_neighbors=30, random_state=42, method='umap')
sc.tl.louvain(expr3_adata, resolution=1, random_state=42)

sc.tl.paga(expr3_adata, groups='louvain')
sc.pl.paga(expr3_adata, plot=True, random_state=42)
sc.tl.umap(expr3_adata, init_pos='paga', random_state=42)

sc.pl.umap(expr3_adata, color='louvain', ncols=3, legend_loc='on data', save='.unsupervised.louvain.png')
sc.pl.umap(expr3_adata, color=['CD31', 'CD140a', 'CD45'], ncols=3, save='.key_markers.png')

sc.pl.dotplot(expr3_adata, groupby='louvain',
              var_names=cytof_marks, dendrogram=True,
              standard_scale='var', save='.invitro.louvain.png')

sum(expr3_adata.obs['louvain'].isin(['10', '13', '14', '15']))/45000 #Endo-like
sum(expr3_adata.obs['louvain'].isin(['0', '1', '9']))/45000 #Leuko
sum(expr3_adata.obs['louvain'].isin(['2', '3', '4', '5', '6', '7', '8', '11', '12']))/45000 #Fibro

major_clust = {'0':  'Leukocytes 30.5%',
               '1':  'Leukocytes 30.5%',
               '9':  'Leukocytes 30.5%',

               '10': 'Endothelial-like Cells 5.6%',
               '13': 'Endothelial-like Cells 5.6%',
               '14': 'Endothelial-like Cells 5.6%',
               '15': 'Endothelial-like Cells 5.6%',

               '2': 'Fibroblasts 63.9%',
               '3': 'Fibroblasts 63.9%',
               '4': 'Fibroblasts 63.9%',
               '5': 'Fibroblasts 63.9%',
               '6': 'Fibroblasts 63.9%',
               '7': 'Fibroblasts 63.9%',
               '8': 'Fibroblasts 63.9%',
               '11': 'Fibroblasts 63.9%',
               '12': 'Fibroblasts 63.9%',
              }

expr3_adata.obs['Major Clusters'] = expr3_adata.obs['louvain'].map(major_clust)

major_clust = {'0':  'FolR2 Hi Leukocytes 13.9%',
               '1':  'FolR2 Low Leukocytes 10.8%',
               '9':  'Replicating Leukocytes 5.78%',

               '10': 'CD140a Hi Endothelial-like Cells 3.04%',
               '13': 'ActCaspase3 Hi Endothelial-like Cells 1.56%',
               '14': 'CD90 Hi Endothelial-like Cells 0.511%',
               '15': 'Replicating Endothelial-like Cells 0.482%',

               '2': 'Ly6AE Low Fibroblasts 19.3%',
               '3': 'Ly6AE Hi Fibroblasts 37.9%',
               '4': 'Ly6AE Hi Fibroblasts 37.9%',
               '5': 'Ly6AE Hi Fibroblasts 37.9%',
               '6': 'Postn Hi Fibroblasts 7.98%',
               '7': 'Ly6AE Low Fibroblasts 19.3%',
               '8': 'Marker Low Fibroblasts 7.10%',
               '11': 'Ly6AE Low Fibroblasts 19.3%',
               '12': 'Replicating Fibroblasts 1.83%',
              }

expr3_adata.obs['Subclusters'] = expr3_adata.obs['louvain'].map(major_clust)
expr3_adata.obs['Condition'] = [i[:2] for i in np.asarray(expr3_adata.obs['Sample'])]

sc.pl.umap(expr3_adata, color='louvain', ncols=3, legend_loc='on data', save='.invitro.louvain.png')
sc.pl.umap(expr3_adata, color='Major Clusters', ncols=3, save='.invitro.mcs.png')
# sc.pl.paga(expr3_adata, plot=True, random_state=42, color='Major Clusters', save='.invitro.paga.mcs.png')
sc.pl.umap(expr3_adata, color='Subclusters', ncols=3, save='.invitro.subclusters.png')
sc.pl.dotplot(expr3_adata, groupby='Subclusters', var_names=cytof_marks, dendrogram=True, standard_scale='var', save='.invitro.subclusters.png')

expr3_adata.obs['Subclusters'] = [i[:-6] for i in expr3_adata.obs['Subclusters']]

prop_freq = pd.DataFrame(index=np.unique(expr3_adata.obs['Condition']), columns=np.unique(expr3_adata.obs['Subclusters']))

for i in np.unique(expr3_adata.obs['Condition']):
    for j in np.unique(expr3_adata.obs['Subclusters']):
        prop_freq[j].loc[i] = sum(expr3_adata[expr3_adata.obs['Condition']==i].obs['Subclusters']==j)/11250

prop_freq = prop_freq.T

cg = sns.clustermap(data=prop_freq.astype('float64'),
               cmap=Bilbao_20.mpl_colormap,
               figsize=(15, 15),
               method='ward',
               row_cluster=True,
               col_cluster=True,
               # annot=prop_freq.T.astype,
               # vmin=-1.0,
               # vmax=1.0,
               cbar_kws={"ticks":[0, 0.25, 0.5, 0.75, 1], "label":"Frequency of Cells"})

plt.setp(cg.ax_heatmap.xaxis.get_majorticklabels(), rotation=35)
plt.setp(cg.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)

cg.fig.subplots_adjust(right=.75, top=.90, bottom=.15)
#
# cg.ax_row_dendrogram.set_visible(False)
# dendro_box = cg.ax_row_dendrogram.get_position()
# dendro_box.x0 = 0.205
# dendro_box.x1 = 0.225
#cg.cax.set_position(dendro_box)
cg.cax.yaxis.set_ticks_position("left")
cg.cax.yaxis.set_label_position("left")

for a in cg.ax_row_dendrogram.collections:
    a.set_linewidth(2)

for a in cg.ax_col_dendrogram.collections:
    a.set_linewidth(2)

plt.savefig('./paper.figs/fig5/in.vitro.freq.png', dpi=400)
plt.close()

prop_freq = prop_freq.loc[prop_freq.index[cg.dendrogram_row.reordered_ind]]
prop_freq = prop_freq[prop_freq.columns[cg.dendrogram_col.reordered_ind]]

cg = sns.clustermap(data=prop_freq.astype('float64'),
               cmap=Bilbao_20.mpl_colormap,
               figsize=(15, 15),
               method='ward',
               row_cluster=True,
               col_cluster=True,
               annot=prop_freq.round(3),
               # vmin=-1.0,
               # vmax=1.0,
               cbar_kws={"ticks":[0, 0.25, 0.5, 0.75, 1], "label":"Frequency of Cells"})

plt.setp(cg.ax_heatmap.xaxis.get_majorticklabels(), rotation=35)
plt.setp(cg.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)

cg.fig.subplots_adjust(right=.75, top=.90, bottom=.15)
#
# cg.ax_row_dendrogram.set_visible(False)
# dendro_box = cg.ax_row_dendrogram.get_position()
# dendro_box.x0 = 0.205
# dendro_box.x1 = 0.225
#cg.cax.set_position(dendro_box)
cg.cax.yaxis.set_ticks_position("left")
cg.cax.yaxis.set_label_position("left")

for a in cg.ax_row_dendrogram.collections:
    a.set_linewidth(2)

for a in cg.ax_col_dendrogram.collections:
    a.set_linewidth(2)

plt.savefig('./paper.figs/fig5/in.vitro.freq.png', dpi=400)
plt.close()


import pygame
from City import City
from pygame.locals import KEYDOWN, QUIT, MOUSEBUTTONDOWN, K_RETURN, K_ESCAPE
import sys

# # globals
screen_x = 500
screen_y = 500

city_color = [10, 10, 200]  # blue
city_radius = 3

font_color = [255, 255, 255]  # white

def getCitiesPoints():
    cities = []
    draw(cities)

    collecting = True

    while collecting:
        for event in pygame.event.get():
            if event.type == QUIT:
                sys.exit(0)
            elif event.type == KEYDOWN and event.key == K_RETURN:
                collecting = False
            elif event.type == MOUSEBUTTONDOWN:
                pos = pygame.mouse.get_pos()
                city = City(pos[0], pos[1])
                cities.append(city)
                draw(cities)

    return cities

def draw(cities):
    screen.fill(0)
    for city in cities:
        pygame.draw.circle(screen, city_color, city.getPos(), city_radius)
    text = font.render("Nombre: %i" % len(cities), True, font_color)
    textRect = text.get_rect()
    screen.blit(text, textRect)
    pygame.display.flip()

def drawPath(cities, length):
    screen.fill(0)
    pygame.draw.lines(screen, city_color, True, cities)
    text = font.render("Meilleur chemin actuel %s" % length, True, font_color)
    textRect = text.get_rect()
    screen.blit(text, textRect)
    pygame.display.flip()

def showGUI():
    global window, screen, font
    pygame.init()
    window = pygame.display.set_mode((screen_x, screen_y))
    pygame.display.set_caption('Exemple')
    screen = pygame.display.get_surface()
    font = pygame.font.Font(None, 30)


from collections import deque
def DFS(n,graph,q):
    visited_DFS[q]=1
    print(q+1, end=' ')
    for i in range(0,n):
        if(visited_DFS[i]==0) and (graph[q][i]==1):
            DFS(n,graph,i)
def BFS(n,graph,q):
    queue=deque()
    queue.append(q)
    visited_BFS[q]=1
    while queue:
        tmp=queue.popleft()
        print(tmp+1, end=' ')
        for i in range(0,n):
            if(visited_BFS[i]==0) and (graph[tmp][i]==1):
                queue.append(i)
                visited_BFS[i]=1
n=4
q=0
graph=[[0]*(n+1) for _ in range(n+1)]
visited_DFS=[0]*n
visited_BFS=[0]*n
computers=[[1,2],[1,3],[1,4],[2,4],[3,4]]
for i in computers:
    graph[i[0]-1][i[1]-1]=graph[i[1]-1][i[0]-1]=1
print(graph)
DFS(n,graph,q)
print()
BFS(n,graph,q)


from rest_framework import serializers
from collections.abc import Mapping, Iterable
from .models import Genre, Book

class GenreOverviewSerializer(serializers.ModelSerializer):
    book_count = serializers.SerializerMethodField()

    class Meta:
        model = Genre
        fields = ['id', 'name', 'book_count']
        read_only_fields = ['id', 'book_count']

    def get_book_count(self, instance: Genre) -> int:

        list_of_genres = list()
        child_search = [instance.id, ]

        while len(child_search) > 0:
            next_to_search = child_search.pop()
            list_of_genres.append(next_to_search)

            for child in Genre.objects.filter(parent_id=next_to_search):
                child_search.append(child.id)

        book_count = Book.objects.filter(genre__in=list_of_genres).count()

        return book_count

class GenreForm(serializers.ModelSerializer):
    class Meta:
        model = Genre
        fields = ['id', 'name', 'parent']

class GenreDetailSerializer(serializers.ModelSerializer):
    parent = GenreForm(required=False, read_only=True)
    sub_genres = serializers.SerializerMethodField()
    book_count = serializers.SerializerMethodField()

    class Meta:
        model = Genre
        fields = ['id', 'name', 'parent', 'book_count', 'sub_genres']

    def get_sub_genres(self, instance: Genre) -> Iterable:
        subs = Genre.objects.filter(parent=instance)
        return GenreOverviewSerializer(subs, many=True).data

    def get_book_count(self, instance: Genre) -> int:
        return instance.book_count()

    @classmethod
    def get_root(cls) -> Mapping:
        return {
            'id': '0',
            'name': 'root',
            'parent': None,
            'book_count': Book.objects.all().count(),
            'sub_genres': GenreOverviewSerializer(Genre.objects.filter(parent__isnull=True), many=True).data,
        }


"""
Input	                    Output
1, 0, 1, 2, 0, 1, 3	        [1, 1, 2, 1, 3, 0, 0]
0, 5, 0, 4, 0, 0, 5	        [5, 4, 5, 0, 0, 0, 0]
"""

input_list = input().split(", ")
new_list = []
zeros = 0

for i in range(len(input_list)):
    if int(input_list[i]) > 0:
        new_list.append(int(input_list[i]))
    else:
        zeros += 1
for j in range(zeros):
    new_list.append(0)

print(new_list)

Python Programming Fundamentals/Ex_03_Lists Basics/Ex3_11 Zeroes to Back.py


import unittest
import os
import logging

from datetime import date

from decimal import Decimal

from ibrokersflex import parse_flex_accounts, parse_flex_positions, parse_flex_flows

class LoadFlexResultsTestCase(unittest.TestCase):
    def setUp(self):
        self.tree = None
        example_file_path = os.path.abspath(os.sep.join(['tests-data', 'example-result.xml']))
        logging.info('loading example result file: {}'.format(example_file_path))
        with open(example_file_path, 'r') as result_content:
            lines = result_content.readlines()
            content = ''.join(lines)
            self.content = content

    def test_load_accounts(self):
        accounts = parse_flex_accounts(self.content)
        self.assertEqual(accounts['U1812119']['nav_change'], 93262)
        self.assertEqual(accounts['U1812946']['account_alias'], 'Vol 946')
        self.assertEqual(accounts['U1812119']['cash'], 1520018)
        self.assertEqual(accounts['U1812946']['cash'], 1515189)
        total_cash = 0.
        total_nav = 0.
        for account_code in accounts:
            total_cash += accounts[account_code]['cash']
            total_nav += accounts[account_code]['nav_end']

        self.assertAlmostEqual(total_cash, 8245576, places=0)
        self.assertAlmostEqual(total_nav, 12176892, places=0)

    def test_load_flows(self):
        flows = parse_flex_flows(self.content)
        self.assertTrue(date(2017, 8, 7) in flows.index)
        self.assertTrue(date(2017, 8, 6) not in flows.index)
        result = flows.loc[date(2017, 8, 7)]
        expected = {
            'U1812119': Decimal('-650000'),
            'U2036485': Decimal('250000'),
        }
        self.assertDictEqual(result.to_dict(), expected)

    def test_load_positions(self):
        positions = parse_flex_positions(self.content)
        self.assertEqual(positions['U1760542']['132757576']['description'], 'EUR 18SEP17')
        self.assertEqual(positions['U1812038']['215313814']['description'], 'BAX 17DEC18')
        self.assertEqual(positions['U1812038']['119750055']['currency'], 'USD')
        self.assertEqual(positions['U1812038']['259910541']['assetCategory'], 'FUT')

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s:%(name)s:%(levelname)s:%(message)s')
    unittest.main()


from theBrain import *
from datetime import datetime
import shutil

SENSE_BUFFER = [np.zeros([TRUNC_BACKPROP_LENGTH, SENSOR_DATA_DIM]),
                    np.zeros([TRUNC_BACKPROP_LENGTH, OUTPUT_DIM])]

def truth_values(batch_size):
    # this is supposed to be high when I am here
    hour = datetime.now().hour/NUM_HOURS
    detection = int(sense.IAM_HERE_PIN.read())
    angle = 0

    return [[hour, detection, angle]]*batch_size

def create_buffer():
    global SENSE_BUFFER
    print("Creating sense buffer... please wait")
    for i in range(TRUNC_BACKPROP_LENGTH):
        updateBuffer()
        time.sleep(0.5)
    print("Buffer created.")

def updateBuffer():
    global SENSE_BUFFER
    truth = np.array(truth_values(1))
    sensed_data = np.array([sense.allSensors_read()])

    SENSE_BUFFER[0] = np.concatenate([SENSE_BUFFER[0][1:,:], sensed_data], axis=0)
    SENSE_BUFFER[1] = np.concatenate([SENSE_BUFFER[1][1:,:], truth], axis=0)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    # creating new summary writers and deleting old summary logs
    print("Preparing to log and serve the training summaries to tensorboard...")
    shutil.rmtree(log_dir, ignore_errors=True)
    train_writer, test_writer = getSummaryWriters(sess)

    # at the beginning previous time step data is set to zero
    cycles = 200000
    testStep = 500
    saveStep = 1000
    log_step = 5
    create_buffer()
    _cur_state = np.zeros([SENSOR_DATA_DIM, STATE_SIZE], dtype=np.float32)
    # print(type(_cur_state))
    for i in range(cycles):
        truth_val = truth_values(1)
        sensed_data = [sense.allSensors_read()]
        _, hour_pred, detection_pred, head_angle, _cur_state,loss_val,summary = sess.run([
            optim_t,
            hour_t,
            detection_t,
            motor_t,
            cur_state,
            loss_tensor,
            merged
            ],
            feed_dict={
                input_series: SENSE_BUFFER[0],
                truth_series: SENSE_BUFFER[1],
                init_state: _cur_state
            })

        updateBuffer()

        if i % log_step == 0: train_writer.add_summary(summary, i)
        if i % saveStep == 0: saveModel(sess, model_save_path[0])

        # print(hour_pred, detection_pred, head_angle, truth_val[0][:2], out[0].tolist())
        print('Time: %s, Detection: %s, Angle: %s'%(hour_pred,detection_pred,head_angle))
        print('Loss: %s'%loss_val)

        detection_LED.write(detection_pred)
        head.write(min(max(0,head_angle),180))
        time.sleep(0.5)


import tensorflow as tf
from tensorflow.keras.models import load_model
import pickle
import numpy as np
import argparse
import datetime
import glob
import time
import sys
import os
import re

import seaborn as sns
from matplotlib import pyplot as plt
from sklearn.metrics import (roc_auc_score, roc_curve,
  classification_report, confusion_matrix)

from milk.utilities import data_utils
from milk.utilities import model_utils
from milk.utilities import training_utils
from milk import MilkEncode, MilkPredict

# with open('../dataset/case_dict_obfuscated.pkl', 'rb') as f:
#with open('../dataset/cases_md5.pkl', 'rb') as f:
with open('case_dict_obfuscated.pkl', 'rb') as f:
  case_dict = pickle.load(f)

from milk.encoder_config import deep_args as encoder_args

def case_label_fn(data_path):
  case = os.path.splitext(os.path.basename(data_path))[0]
  y_ = case_dict[case]
  # print(data_path, y_)
  return y_

def read_test_list(test_file):
  test_list = []
  with open(test_file, 'r') as f:
    for l in f:
      test_list.append(l.replace('\n', ''))

  return test_list

def auc_curve(ytrue, yhat, savepath=None):
  plt.figure(figsize=(2,2), dpi=300)
  ytrue_c = ytrue[:,1]
  yhat_c = yhat[:,1]
  auc_c = roc_auc_score(y_true=ytrue_c, y_score=yhat_c)
  fpr, tpr, _ = roc_curve(y_true=ytrue_c, y_score=yhat_c)

  plt.plot(fpr, tpr,
      label='M1 AUC={:3.3f}'.format(auc_c))

  plt.legend(loc='lower right', frameon=True)
  plt.ylabel('Sensitivity')
  plt.xlabel('1 - specificity')

  if savepath is None:
    plt.show()
  else:
    plt.savefig(savepath, bbox_inches='tight')

""" Return a pretty string """
def pprint_cm(ytrue, yhat, labels=['0', '1']):
  n_classes = len(labels)
  cm = confusion_matrix(y_true=ytrue, y_pred=yhat)
  s = '\nConfusion Matrix:\n'
  s += '\t{}\t{}'.format(*labels)
  for k in range(n_classes):
    s += '\n'
    s += '{}'.format(labels[k])
    for j in range(n_classes):
      s += '\t{: 3d}'.format(cm[k, j])

  s += '\n'
  return s

def test_eval(ytrue, yhat, savepath=None):
  ytrue = ytrue[:,1]
  yhat_c = np.argmax(yhat, axis=-1)
  yhat = yhat[:,1]
  auc_c = 'AUC = {:3.5f}\n'.format(roc_auc_score(y_true=ytrue, y_score=yhat))
  cr = classification_report(y_true=ytrue, y_pred=yhat_c) + '\n'
  cm = pprint_cm(ytrue, yhat_c) + '\n'
  if savepath is not None:
    with open(savepath, 'w+') as f:
        f.write(auc_c)
        f.write(cr)
        f.write(cm)
  else:
    print(auc_c)
    print(cr)
    print(cm)

def run_sample(case_x, encode_model, predict_model, mcdropout=False,
             batch_size=64):
  if mcdropout:
    yhats = []
    for _ in range(25):
      zs = encode_model.predict(case_x, batch_size=batch_size)
      yhat = predict_model.predict_on_batch(zs)
      yhats.append(yhat)

    yhats = np.stack(yhats, axis=0)
    yhat = np.mean(yhats, axis=0)
  else:
    zs = encode_model.predict(case_x, batch_size=batch_size)
    yhat = predict_model.predict_on_batch(zs)

  return yhat

def main(args):
  transform_fn = data_utils.make_transform_fn(args.x_size, args.y_size,
                                              args.crop_size, args.scale)

  snapshot = 'save/{}.h5'.format(args.timestamp)
  # trained_model = load_model(snapshot)

  if args.mcdropout:
    encoder_args['mcdropout'] = True

  print('Model initializing')
  encode_model = MilkEncode(input_shape=(args.crop_size, args.crop_size, 3),
                            encoder_args=encoder_args, deep_classifier=args.deep_classifier)
  encode_shape = list(encode_model.output.shape)
  x_pl = tf.placeholder(shape=(None, args.input_dim, args.input_dim, 3), dtype=tf.float32)
  z_op = encode_model(x_pl)

  input_shape = z_op.shape[-1]
  predict_model = MilkPredict(input_shape=[input_shape], mode=args.mil)

  print('loading weights from {}'.format(snapshot))
  encode_model.load_weights(snapshot, by_name=True)
  predict_model.load_weights(snapshot, by_name=True)

  # models = model_utils.make_inference_functions(encode_model,
  #                                               predict_model,
  #                                               trained_model,
  #                                               attention_model=None)
  # encode_model, predict_model = models

  test_list = os.path.join(args.testdir, '{}.txt'.format(args.timestamp))
  test_list = read_test_list(test_list)

  ytrues = []
  yhats = []
  print('MC Dropout: {}'.format(args.mcdropout))
  for _ in range(args.n_repeat):
    for test_case in test_list:
      case_x, case_y = data_utils.load(
          data_path = test_case,
          transform_fn=transform_fn,
          case_label_fn=case_label_fn,
          all_tiles=True,
          )
      case_x = np.squeeze(case_x, axis=0)
      print('Running case x: ', case_x.shape)

      yhat = run_sample(case_x, encode_model, predict_model,
                        mcdropout=args.mcdropout,
                        batch_size=args.batch_size)
      ytrues.append(case_y)
      yhats.append(yhat)
      print(test_case, case_y, case_x.shape, yhat)

  ytrue = np.concatenate(ytrues, axis=0)
  yhat = np.concatenate(yhats, axis=0)

  ytrue_max = np.argmax(ytrue, axis=-1)
  yhat_max = np.argmax(yhat, axis=-1)
  accuracy = (ytrue_max == yhat_max).mean()
  print('Accuracy: {:3.3f}'.format(accuracy))

  if args.odir is not None:
    save_img = os.path.join(args.odir, '{}.png'.format(args.timestamp))
    save_metrics = os.path.join(args.odir, '{}.txt'.format(args.timestamp))
    save_yhat = os.path.join(args.odir, '{}.npy'.format(args.timestamp))
    save_ytrue = os.path.join(args.odir, '{}_ytrue.npy'.format(args.timestamp))
    np.save(save_yhat, yhat)
    np.save(save_ytrue, ytrue)
  else:
    save_img = None
    save_metrics = None
    save_yhat = None

  auc_curve(ytrue, yhat, savepath=save_img)
  test_eval(ytrue, yhat, savepath=save_metrics)

if __name__ == '__main__':
  # n_repeat controls re-sampling from the case
  # mcdropout is a flag for doing mcdropout to approximate posterior probability
  parser = argparse.ArgumentParser()
  parser.add_argument('--timestamp', default=None, type=str)
  parser.add_argument('--n_repeat', default=1, type=int)
  parser.add_argument('--mcdropout', default=False, action='store_true')
  parser.add_argument('--testdir', default='test_lists', type=str)
  parser.add_argument('--odir', default=None, type=str)

  parser.add_argument('--x_size', default=128, type=int)
  parser.add_argument('--y_size', default=128, type=int)
  parser.add_argument('--crop_size', default=96, type=int)
  parser.add_argument('--scale', default=1.0, type=float)
  parser.add_argument('--batch_size', default=64, type=int)

  parser.add_argument('--mil', default='attention', type=str)

  args = parser.parse_args()
  assert args.timestamp is not None
  main(args)


#!/usr/bin/env python3
# coding: utf-8

import argparse
from collections import defaultdict, Counter

# initial parameters
parser = argparse.ArgumentParser()
parser.add_argument('--semantic_concept', choices=('diminutives', 'social-gender'))
args = parser.parse_args()

# load data
google_data = defaultdict(lambda: defaultdict())
panlex_data = defaultdict(lambda: defaultdict())
corpora_data = defaultdict(lambda: defaultdict())
treq_data = defaultdict(lambda: defaultdict())
unsup_data = defaultdict(lambda: defaultdict())
for source, s_data in zip(
    ('google', 'panlex', 'parallel-corpora', 'treq', 'unsup-nn'),
    (google_data, panlex_data, corpora_data, treq_data, unsup_data)
):
    with open(f'../{source}/' + args.semantic_concept + '-cs-final.tsv', mode='r', encoding='U8') as cs_file, \
         open(f'../{source}/' + args.semantic_concept + '-en-final.tsv', mode='r', encoding='U8') as en_file, \
         open(f'../{source}/' + args.semantic_concept + '-de-final.tsv', mode='r', encoding='U8') as de_file, \
         open(f'../{source}/' + args.semantic_concept + '-nl-final.tsv', mode='r', encoding='U8') as nl_file, \
         open(f'../{source}/' + args.semantic_concept + '-ru-final.tsv', mode='r', encoding='U8') as ru_file, \
         open(f'../{source}/' + args.semantic_concept + '-fr-final.tsv', mode='r', encoding='U8') as fr_file, \
         open(f'../{source}/' + args.semantic_concept + '-es-final.tsv', mode='r', encoding='U8') as es_file:
        for cs_line in cs_file:
            en_line, de_line = next(en_file), next(de_file)
            nl_line, ru_line = next(nl_file), next(ru_file)
            fr_line, es_line = next(fr_file), next(es_file)

            s_data[cs_line.strip().split('\t')[0]]['xx'] = cs_line.strip().split('\t')[1]
            s_data[cs_line.strip().split('\t')[0]]['en'] = tuple(en_line.strip().split('\t'))
            s_data[cs_line.strip().split('\t')[0]]['de'] = tuple(de_line.strip().split('\t'))
            s_data[cs_line.strip().split('\t')[0]]['nl'] = tuple(nl_line.strip().split('\t'))
            s_data[cs_line.strip().split('\t')[0]]['ru'] = tuple(ru_line.strip().split('\t'))
            s_data[cs_line.strip().split('\t')[0]]['fr'] = tuple(fr_line.strip().split('\t'))
            s_data[cs_line.strip().split('\t')[0]]['es'] = tuple(es_line.strip().split('\t'))

# merge data
resulting_data = [list(), list(), list(), list(), list(), list(), list()]
for czech_word in set(list(google_data.keys()) + list(panlex_data.keys()) + list(corpora_data.keys()) + list(treq_data.keys()) + list(unsup_data.keys())):
    # Czech annotation
    cs_decisions = [
        google_data[czech_word].get('xx', None), panlex_data[czech_word].get('xx', None), corpora_data[czech_word].get('xx', None),
        treq_data[czech_word].get('xx', None), unsup_data[czech_word].get('xx', None)
    ]
    while None in cs_decisions:
        cs_decisions.remove(None)
    resulting_data[0].append((czech_word, cs_decisions[0]))

    # other languages annotations
    for idx, language in zip(range(1, 7), ('en', 'de', 'nl', 'ru', 'fr', 'es')):
        decisions = [
            google_data[czech_word].get(language, None), panlex_data[czech_word].get(language, None), corpora_data[czech_word].get(language, None),
            treq_data[czech_word].get(language, None), unsup_data[czech_word].get(language, None)
        ]
        while None in decisions:
            decisions.remove(None)

        # the most occurences across sources of translations
        dec_words = Counter([w for w, _ in decisions])
        dec_most = dec_words.most_common(1)
        without_most = list(dec_words.values())
        without_most.remove(dec_most[0][1])
        if dec_most[0][1] not in without_most:
            for word, annot in decisions:
                if word == dec_most[0][0]:
                    resulting_data[idx].append((word, annot))
                    break
            continue

        # hierarchy: panlex > treq > corpora > unsup > google
        dec_words = Counter([w for w, _ in decisions])
        max_count = max(dec_words.values())
        words = [w for w, c in dec_words.items() if c == max_count]

        found = False
        for source_data in (panlex_data, treq_data, corpora_data, unsup_data, google_data):
            for word in words:
                if source_data[czech_word].get(language, False):
                    resulting_data[idx].append(source_data[czech_word].get(language, False))
                    found = True
                    break
            if found:
                break

        if not found:
            print(czech_word)

# store data
with open(f'merged-data/{args.semantic_concept}-cs-final.tsv', mode='w', encoding='U8') as cs_file, \
     open(f'merged-data/{args.semantic_concept}-en-final.tsv', mode='w', encoding='U8') as en_file, \
     open(f'merged-data/{args.semantic_concept}-de-final.tsv', mode='w', encoding='U8') as de_file, \
     open(f'merged-data/{args.semantic_concept}-nl-final.tsv', mode='w', encoding='U8') as nl_file, \
     open(f'merged-data/{args.semantic_concept}-ru-final.tsv', mode='w', encoding='U8') as ru_file, \
     open(f'merged-data/{args.semantic_concept}-fr-final.tsv', mode='w', encoding='U8') as fr_file, \
     open(f'merged-data/{args.semantic_concept}-es-final.tsv', mode='w', encoding='U8') as es_file:
    for idx in range(len(resulting_data[0])):
        print('\t'.join(resulting_data[0][idx]), file=cs_file)
        print('\t'.join(resulting_data[1][idx]), file=en_file)
        print('\t'.join(resulting_data[2][idx]), file=de_file)
        print('\t'.join(resulting_data[3][idx]), file=nl_file)
        print('\t'.join(resulting_data[4][idx]), file=ru_file)
        print('\t'.join(resulting_data[5][idx]), file=fr_file)
        print('\t'.join(resulting_data[6][idx]), file=es_file)


from __future__ import with_statement
from fabric.api import *
from fabric.contrib.console import confirm

env.hosts = ['my_server']

def test():
    with settings(warn_only=True):
        result = local('./manage.py test my_app', capture=True)
    if result.failed and not confirm("Tests failed. Continue anyway?"):
        abort("Aborting at user request.")

def commit():
    local("git add -p && git commit")

def push():
    local("git push")

def prepare_deploy():
    test()
    commit()
    push()

def deploy():
    code_dir = '/srv/django/myproject'
    with settings(warn_only=True):
        if run("test -d %s" % code_dir).failed:
            run("git clone user@vcshost:/path/to/repo/.git %s" % code_dir)
    with cd(code_dir):
        run("git pull")
        run("touch app.wsgi")

"""
This fabfile makes use of a large portion of Fabric’s feature set:

1. defining fabfile tasks and running them with fab;
2. calling local shell commands with local;
3. modifying env vars with settings;
4. handling command failures, prompting the user, and manually aborting;
5. and defining host lists and run-ning remote commands.
However, there’s still a lot more we haven’t covered here! Please make sure you follow the various “see also” links, and check out the documentation table of contents on the main index page.
"""