# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
from Python.MAIN_FUNC import lib
from Python.MAIN_FUNC import main_actions
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.interval import IntervalTrigger
import atexit

app = lib.Flask(__name__, template_folder='web_service', static_folder='web_service')

@app.route("/", methods=['GET'])
def index():
    if lib.request.method == 'GET':
        return lib.render_template('index.html')
    elif lib.request.method == 'POST':
        if lib.request.form['submit_button'] == 'LOGIN':
            var_login = lib.request.form['login']
            var_pass = lib.request.form['password']
            # For debug, need change code below
            if var_login == 'YOUR_USER' and var_pass == 'YOUR_PASS':
                return lib.render_template('index.html')
            else:
                return 'Ошибка входа!'

@app.route("/", methods=['POST','GET'])
def button_run():
    print(lib.request.method)
    if lib.request.method == 'POST':
        if lib.request.form['button_run'] == 'run_full':
            var_stock = lib.request.form['Stock']
            var_batch = lib.request.form['Batch']
            main_class.execute(idProcessFunc = 'InsertData', var_batch=int(var_batch), list_tiket= str(var_stock))
            return lib.render_template('index.html')
        elif lib.request.form['button_run'] == 'run_delete':
            var_stock_del = lib.request.form['Stock_delete']
            if var_stock_del != '':
                main_class.delete_stock(idProcessFunc='DeleteData', stock_name=var_stock_del)
            elif 'Stock_delete' in lib.request.form:
                main_class.delete_stock(idProcessFunc='DeleteAllData', stock_name=var_stock_del, all_delete=True)
            return lib.redirect("/")
        elif lib.request.form['button_run'] == 'run_insertmain':
            main_class.executeNasdaq()
            return lib.redirect("/")

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    main_class = main_actions.ACTIONS()
    #main_class.executeNasdaq()
    scheduler = BackgroundScheduler()
    scheduler.start()
    scheduler.add_job(
        func=main_class.delta_loading,
        trigger=IntervalTrigger(hours=1),
        id='printing_time_job',
        name='Data loading',
        replace_existing=True)
    # Shut down the scheduler when exiting the app
    atexit.register(lambda: scheduler.shutdown())

    app.run(debug='True')

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


""" Thomas Parashos 2019
1. Take from the big list of things
2. remove unusabe and numeric (return to unusable later, potential links e.g. employee to id number)
3. make spreadsheet with that data
record_id, y_act, reason, col_name, unique_entry, times_entered, percent_of_total, list_of_rows

This now out to be a collection of methods
"""
import glob
import pandas as pd
from datetime import datetime

###Globals###

#Paths
main_folder = "./SH_data"
source_data_folder = "/datasets"
index_folder = "/meta_data"
prelim_labeled_data_folder = "/data_for_labeling"

data_folder = main_folder + source_data_folder + "/"

index_path = main_folder + index_folder +"/*.csv"
pre_labeled_path = main_folder + prelim_labeled_data_folder + "/reduced_labels.csv"

output_folder = main_folder + "/label_sets/"

log_file = "./log.txt"

#Open Files
log_file = open(log_file, "a+")

#Lists
encodings = [None, "cp1252", "ISO-8859-1"]

###Methods###

def log(log_string):
    log_file.write("[{}]\t".format(datetime.now()) +log_string + "\n")

def p_log(log_string):
    print(log_string)
    log(log_string)

def print_list(l):
    for e in l: print(e)

def multi_read(read_fun, file_list, give_list = False):
    #Returns a df of all the files in the list appended together
    dfs = []
    for path in file_list:
        try:
            dfs.append(read_fun(path))
        except:
            print("failed to append")
            print(path)
    if give_list: return dfs
    return pd.concat(dfs, sort=False)

def get_subset_eq(df, col_name, val):
    return df.loc[df[col_name] == val]

def read_tsv(file_path, encoding = None):
    return pd.read_csv(file_path, sep = "\t", encoding = encoding)

def glob1(file_path):
    #gives a string of the first file in a glob
    g = glob.glob(file_path)
    len_g = len(g)
    if len_g == 1: return g[0]
    if len_g == 2:
        print("Multiple files\n", g)
        return g[0]
    if len_g == 0:
        print("no files")
        return g

def col_replace(df, col_name, old_val, new_val):
    #inplace operation
    #replaces all specified values in a df's column with the given input
    df[col_name][df[col_name] == old_val] = new_val

def list_starting_from(tar_list, start_el, skip = 0):
    start = tar_list.index(start_el) + skip
    return tar_list[start:]

def read_csv_mult_encodings(file_path):
    #attempts to read file with each specified encoding in order
    #yells at you if it can't
    for encoding in encodings:
        try:
            return pd.read_csv(file_path, encoding = encoding)
        except: pass
    p_log("Opening Failed. File: "+file_path)
    return None

def write_list(file, string_list):
    for st in string_list:
        file.write(st)

def all_equal(my_list):
    return all(x == my_list[0] for x in my_list)

class Counter:

    def __init__(self, total, name):
        self.i = 0
        self.total = total
        self.name = name

    def inc(self, e_name, prefix):
        self.i += 1
        print("{}{}: {}, {} of {}, {:.2%}".format(prefix, self.name, e_name, self.i, self.total, self.i/self.total))

if __name__ == "__main__":
    # file_list = glob.glob("SH_data\label_sets\labeled\contains_d\*.csv")
    # print(len(file_list))

    # big_df = multi_read(read_csv_mult_encodings, file_list)
    # big_df.to_csv("cdfify.csv")

    # file_list = glob.glob("SH_data\label_sets\labeled\contains_d\*.csv")
    # print(len(file_list))

    # dfs = multi_read(read_csv_mult_encodings, file_list, give_list=True)
    # i_s = []
    # for i in range(0,len(dfs)):
    #     df = dfs[i]
    #     if len(df[df['reason'] == 'ahi']) > 0:
    #         df.to_csv(f"todays_data/{i}.csv")
    #     else:
    #         i_s.append(i)
    # dfs = [dfs[i] for i in i_s ]
    # pd.concat(dfs).to_csv('todays_data/good.csv')

    # reason_df = pd.read_csv("todays_data\\10.csv")
    # print(reason_df)
    # reason_df = reason_df[reason_df['reason'] == 'ahi']
    # reason_df.to_csv("todays_data\\10.csv", index=False)

    # import re
    # def alnum(my_str):
    #     return re.sub(r'\W+', '', my_str)

    # df = pd.read_csv("todays_data\\10.csv")
    # start = 1
    # for group in range(start, df["group"].max() + 1):
    #     curr_df = df[df["group"] == group]
    #     print(curr_df)
    #     words = curr_df[curr_df.columns[0]].tolist()

    #     if all_equal([w.lower() for w in words]): new_reason = "a"
    #     elif all_equal([w.replace(" ", "") for w in words]): new_reason = "h"
    #     elif all_equal([alnum(w) for w in words]): new_reason = "i"
    #     elif all_equal([w.lower().replace(" ", "") for w in words]): new_reason = "a h"
    #     elif all_equal([alnum(w).lower() for w in words]): new_reason = "a i"
    #     elif all_equal([alnum(w).replace(" ", "") for w in words]): new_reason = "h i"
    #     elif all_equal([alnum(w).replace(" ", "").lower() for w in words]): new_reason = "a h i"

    #     df.at[df["group"] == group, 'reason'] = new_reason
    #     df.to_csv("todays_data\\10.csv", index=False)

    # file_list = glob.glob("todays_data\\*.csv")
    # dfs = multi_read(read_csv_mult_encodings, file_list)
    # dfs = dfs[dfs['reason'] != "zzz"]
    # dfs = dfs['reason']
    # dfs.to_csv("todays_data\\data.csv")
    # import matplotlib.pyplot as plt
    # df = pd.read_csv("todays_data\\data.csv")
    # plot = df["reason"].value_counts().plot.pie()
    # plt.show()
    # print(df["reason"].value_counts())
    # df["reason"].value_counts().to_csv("todays_data\\counts.csv")

    import matplotlib.pyplot as plt
    df = pd.read_csv("todays_data\\counts.csv", index_col="reason")
    print(df)
    plot = df.plot.pie(y="Reason")
    plt.show()
    # print(df["reason"].value_counts())
    # df["reason"].value_counts().to_csv("todays_data\\counts.csv")


import pandas as pd
import numpy as np
import gensim
import jieba
from text_classification.predict import classification_predict
from text_similarity.predict import predict
from chitchat.interact import chitchat

df = pd.read_csv('data/qa_data.csv')
question = df['question'].values
answer = df['answer'].values

model = gensim.models.Word2Vec.load('word2vec/wiki.model')

def sen2vec(text):
    segment = list(jieba.cut(text))
    vec = np.zeros(100)
    for s in segment:
        # 假如我们的词不在词向量里，就会出现oov的问题
        try:
            vec += model.wv[s]
        except:
            pass
    vec = vec / len(segment)
    return vec

def cosine(a, b):
    return np.matmul(a, b.T) / np.linalg.norm(a) / np.linalg.norm(b, axis=-1)

question_vec = []
for q in question:
    question_vec.append(sen2vec(q))

def qa(text):
    # 先判断是否是闲聊
    prob = classification_predict(text)
    if prob > 0.5:
        print('闲聊')
        res = chitchat(text)
        print(res)
        return res
    vec = sen2vec(text)

    # 召回
    similarity = cosine(vec, np.array(question_vec))
    max_similarity = max(similarity)
    print('最大相似度', max_similarity)
    if max_similarity < 0.8:
        print('没有找到答案')
        return '没有找到答案'
    top_10 = np.argsort(-similarity)[0:10]
    candidate = question[top_10]

    # 精排
    esim_res = predict([q] * 10, candidate)

    index_dic = {}

    print('候选集：')
    for i, index in enumerate(top_10):
        print(candidate[i], ' ', similarity[index], ' ', esim_res[i])
        index_dic[i] = index

    esim_index = np.argsort(-esim_res)[0]
    print('最相似的问题: ', question[index_dic[esim_index]])
    print('答案: ', answer[index_dic[esim_index]])
    return answer[index_dic[esim_index]]

if __name__ == '__main__':
    while 1:
        q = input('请输入你的问题：')
        qa(q)


import struct
import pandas
import os

PrintFlag=0
gyro_scale_factor = 1000000
acc_scale_factor = 1000000
InertialSen_to_novatel_frame_transform_Flag =1
def IMR_header(szHeader=b'$IMURAW\0',  # 8B char[8]
                bIsIntelOrMotorola= 0,  # 1B int8_t
                dVersionNumber=8.80,    # 8B double
                bDeltaTheta= 1,         # 4B int32_t
                bDeltaVelocity=0,       # 4B int32_t
                dDataRateHz=100,        # 8B double
                dGyroScaleFactor=1,     # 8B double
                dAccelScaleFactor=1,    # 8B double
                iUtcOrGpsTime=2,        # 4B int32_t
                iRcvTimeOrCorrTime=0,   # 4B int32_t
                dTimeTagBias=0,         # 8B double
                szImuName=b'x'*32,       # 32B char[32]
                reserved1=b'xxxx',       # 4B char[4]
                szProgramName=b'x'*32,   # 32B char[32]
                tCreate=b'x'*12,              # 12B time_type
                bLeverArmValid=False,       # 1B bool
                lXoffset=0,             # 4B int32_t
                lYoffset=0,             # 4B int32_t
                lZoffset=0,             # 4B int32_t
                Reserved=b'x'*354):        # 354B int8_t[354]
                pass
                p0 = struct.pack('8s',b'$IMURAW\0')          # 8B char[8]
                p1 = struct.pack('b',bIsIntelOrMotorola)    # 1B int8_t
                p2 = struct.pack('<d', dVersionNumber)      # 8B double
                p3 = struct.pack( '<i',bDeltaTheta)         # 4B int32_t
                p4 = struct.pack('<i', bDeltaVelocity)      # 4B int32_t
                p5 = struct.pack('<d', dDataRateHz)         # 8B double
                p6 = struct.pack('<d', dGyroScaleFactor)    # 8B double
                p7 = struct.pack('<d', dAccelScaleFactor)   # 8B double
                p8 = struct.pack('<i',iUtcOrGpsTime)        # 4B int32_t
                p9 = struct.pack('<i',iRcvTimeOrCorrTime)   # 4B int32_t
                p10= struct.pack('<d', dTimeTagBias)        # 8B double
                p11= struct.pack('32s',szImuName)           # 32B char[32]
                p12= struct.pack('4s',reserved1)            # 4B char[4]
                p13= struct.pack('32s',szProgramName)       # 32B char[32]
                p14= struct.pack('12s',tCreate)             # 12B time_type
                p15= struct.pack('?',bLeverArmValid)        # 1B bool
                p16= struct.pack('<i',lXoffset)             # 4B int32_t
                p17= struct.pack('<i', lYoffset)            # 4B int32_t
                p18= struct.pack('<i',lZoffset)             # 4B int32_t
                p19= struct.pack('354s',Reserved)           # 354B int8_t[354]
                p= p0   +p1     +p2     +p3     +p4     +p5\
                        +p6     +p7     +p8     +p9     +p10\
                        +p11    +p12    +p13    +p14    +p15\
                        +p16    +p17    +p18    +p19
                # print(len(p))
                if (PrintFlag==1):
                    #op= struct.unpack('8scd',p)
                    # op= struct.unpack(''8s b d i i d d d i i d 32s 4s 32s 12s ? i i i 354s',p)
                    # print(struct.unpack_from('8s',p,0))
                #     print(op[0])
                    idx=0
                    print('header:',struct.unpack_from('8s',p,idx)[0])
                    idx+=8
                    print('bIsIntelOrMotorola:',struct.unpack_from('b',p,idx)[0])
                    idx+=1
                    print('dVersionNumber:',struct.unpack_from('<d',p,idx)[0])
                    idx+=8
                    print('bDeltaTheta',struct.unpack_from('<i',p,idx)[0])
                    idx+=4
                    print('bDeltaVelocity',struct.unpack_from('<d',p,idx)[0])
                    idx+=4
                    print('dDataRateHz',struct.unpack_from('<d',p,idx)[0])
                    idx+=8
                    print('dGyroScaleFactor',struct.unpack_from('<d',p,idx)[0])
                    idx+=8
                    print('dAccelScaleFactor',struct.unpack_from('<i',p,idx)[0])
                    idx+=8
                    print('iUtcOrGpsTime',struct.unpack_from('<i',p,idx)[0])
                    idx+=4
                    print('iRcvTimeOrCorrTime',struct.unpack_from('<d',p,idx)[0])
                    idx+=4+8
                    print('szImuName',struct.unpack_from('32s',p,idx)[0])
                    idx+=32+4
                    print('szProgramName',struct.unpack_from('32s',p,idx)[0])
                    idx+=32
                    print('tCreate',struct.unpack_from('12s',p,idx)[0])
                    idx+=12
                    print('bLeverArmValid',struct.unpack_from('b',p,idx)[0])
                    idx+= 1
                    print('lXoffset',struct.unpack_from('<i',p,idx)[0])
                    idx+=4
                    print('lYoffset',struct.unpack_from('<i',p,idx)[0])
                    idx+=4
                    print('lZoffset',struct.unpack_from('<i',p,idx)[0])
                    idx+=4
                    pass
                return p

def IMR_Record_struct(  time,           #8B double
                        gx,gy,gz,       #4B int32_t
                        ax,ay,az):      #
                    pt  = struct.pack('<d',time)
                    pgx = struct.pack('<i',gx)
                    pgy = struct.pack('<i',gy)
                    pgz = struct.pack('<i',gz)
                    pax = struct.pack('<i',ax)
                    pay = struct.pack('<i',ay)
                    paz = struct.pack('<i',az)
                    p =  pt   +pgx    +pgy    +pgz\
                            +pax    +pay    +paz

                    if(PrintFlag==1):
                        #print(struct.unpack('d i ',pt +pgx ))
                        upt,upgx,upgy,upgz,upax,upay,upaz= struct.unpack('d i i i i i i ',p)
                        print ('Time:',upt)
                        print ('gx:',upgx)
                        print ('gy:',upgy)
                        print ('gz:',upgz)
                        print ('ax:',upax)
                        print ('ay:',upay)
                        print ('az:',upaz)
                        pass
                    return p

def inertialsense_csv_to_imr(imu_file_path,gps_file_path,imr_file_path):
    #get gps time offset
    gpsfile = pandas.read_csv(gps_file_path)
    imufile = pandas.read_csv(imu_file_path)
    #get the time offset value from the last line of the csv file.
    gps_timeoffset=0
    #print(len(gpsfile))
    #print(gpsfile)
    if len(gpsfile)>0:
        gps_timeoffset = gpsfile.loc[len(gpsfile)-1].at['towOffset']

    #create header packet of imr file
    bDeltaTheta = 0 # data is read as scaled angular rates
    bDeltaVelocity = 0 # data is read as scaled accelerations
    dDataRateHz = 1
    dGyroScaleFactor = 1/gyro_scale_factor
    dAccelScaleFactor =1/ acc_scale_factor
    iUtcOrGpsTime = 2 # 1: UTC seconds of wee, 2: GPS seconds of week
    iRcvTimeOrCorrTime = 0 # 0: Unknown, will default to corrected time
    bLeverArmValid = False #True if lever arms from IMU to primary GNSS antenna are stored in this header

    header_packet_imr = IMR_header(bDeltaTheta=bDeltaTheta,
                                    bDeltaVelocity=bDeltaVelocity,
                                    dDataRateHz=dDataRateHz,
                                    dGyroScaleFactor=dGyroScaleFactor,
                                    dAccelScaleFactor=dAccelScaleFactor,
                                    iUtcOrGpsTime=iUtcOrGpsTime,
                                    iRcvTimeOrCorrTime=iRcvTimeOrCorrTime,
                                    bLeverArmValid=bLeverArmValid)

    #write 512B header
    opfile_name = ((os.path.basename(imu_file_path)).split('.'))[0] + '.imr'
    imrfile = open(imr_file_path+""+opfile_name,'wb')
    imrfile.write(header_packet_imr)

    for i in range(len(imufile)):
        line = imufile.loc[i]
        gps_sec = (line.at['time']) + gps_timeoffset
        gx = int(line.at['pqr1[0]']*gyro_scale_factor)
        gy = int(line.at['pqr1[1]']*gyro_scale_factor)
        gz = int(line.at['pqr1[2]']*gyro_scale_factor)
        ax= int(line.at['acc1[0]']*acc_scale_factor)
        ay= int(line.at['acc1[1]']*acc_scale_factor)
        az= int(line.at['acc1[2]']*acc_scale_factor)
        imr_data_packet = IMR_Record_struct(gps_sec,gx,gy,gz,ax,ay,az)
        imrfile.write(imr_data_packet)

    imrfile.close()

def inertialsense_csv_to_bin(imu_file_path,gps_file_path,bin_file_path):
    #get gps time offset
    gpsfile = pandas.read_csv(gps_file_path)
    imufile = pandas.read_csv(imu_file_path)
    #get the time offset value from the last line of the csv file.
    gps_timeoffset=0
    print('gps file len: ',len(gpsfile))
    print('imu file len: ',len(imufile))
    #print(gpsfile)
    if len(gpsfile)>0:
        gps_timeoffset = gpsfile.loc[len(gpsfile)-1].at['towOffset']

    print("gps time offset: ", gps_timeoffset)
    # #create header packet of imr file
    # bDeltaTheta = 0 # data is read as scaled angular rates
    # bDeltaVelocity = 0 # data is read as scaled accelerations
    # dDataRateHz = 1
    # dGyroScaleFactor = 1/gyro_scale_factor
    # dAccelScaleFactor =1/ acc_scale_factor
    # iUtcOrGpsTime = 2 # 1: UTC seconds of wee, 2: GPS seconds of week
    # iRcvTimeOrCorrTime = 0 # 0: Unknown, will default to corrected time
    # bLeverArmValid = False #True if lever arms from IMU to primary GNSS antenna are stored in this header

    # header_packet_imr = IMR_header(bDeltaTheta=bDeltaTheta,
    #                                 bDeltaVelocity=bDeltaVelocity,
    #                                 dDataRateHz=dDataRateHz,
    #                                 dGyroScaleFactor=dGyroScaleFactor,
    #                                 dAccelScaleFactor=dAccelScaleFactor,
    #                                 iUtcOrGpsTime=iUtcOrGpsTime,
    #                                 iRcvTimeOrCorrTime=iRcvTimeOrCorrTime,
    #                                 bLeverArmValid=bLeverArmValid)

    #write 512B header
    binfile_name = ((os.path.basename(imu_file_path)).split('.'))[0] + '.bin'
    # binfile_name = "raw_imu.bin"
    binfile = open(bin_file_path+"/"+binfile_name,'wb')
    # imrfile.write(header_packet_imr)

    for i in range(len(imufile)):
        try:

            line = imufile.loc[i]
            gps_sec = (line.at['time']) + gps_timeoffset
            if(InertialSen_to_novatel_frame_transform_Flag==1):
                gx = int(line.at['pqr1[1]']*gyro_scale_factor)
                gy = int(line.at['pqr1[0]']*gyro_scale_factor)
                gz = -1*int(line.at['pqr1[2]']*gyro_scale_factor)
                ax= int(line.at['acc1[1]']*acc_scale_factor)
                ay= int(line.at['acc1[0]']*acc_scale_factor)
                az= -1*int(line.at['acc1[2]']*acc_scale_factor)
            else:
                gx = int(line.at['pqr1[0]']*gyro_scale_factor)
                gy = int(line.at['pqr1[1]']*gyro_scale_factor)
                gz = int(line.at['pqr1[2]']*gyro_scale_factor)
                ax= int(line.at['acc1[0]']*acc_scale_factor)
                ay= int(line.at['acc1[1]']*acc_scale_factor)
                az= int(line.at['acc1[2]']*acc_scale_factor)
            imr_data_packet = IMR_Record_struct(gps_sec,gx,gy,gz,ax,ay,az)
            binfile.write(imr_data_packet)
        except:
            pass

    binfile.close()

#IMR_header(lZoffset=2)
#IMR_Record_struct(654321,10,11,12,20,21,22)
# f= pandas.read_csv('gps_file.csv')
# print(f)
# print(len(f))
# print(f.loc[len(f)-1].at['towOffset'])

# inertialsense_csv_to_imr('LOG_SN40336_20210523_224752_0001_imuDual_sdat.csv','gps_file.csv','')
# inertialsense_csv_to_bin('LOG_SN40336_20210523_224752_0001_imuDual_sdat.csv','gps_file.csv','')
imuFilepath = 'D:\\NSV\data\\20210601_sparrow_field_data\\car_runs\\run2\\20210531_182636\\LOG_SN40336_20210531_182636_0001_imuDualimuDual_merged_.csv'
gpsFilepath = 'D:\\NSV\data\\20210601_sparrow_field_data\\car_runs\\run2\\20210531_182636\\LOG_SN40336_20210531_182636_0001_gps1Pos.csv'
binFilePath = 'D:\\NSV\data\\20210601_sparrow_field_data\\car_runs\\run2\\20210531_182636\\'

# inertialsense_csv_to_bin(imu_file_path=imuFilepath,gps_file_path=gpsFilepath,bin_file_path=binFilePath)
# imu_file_path= os.path.basename(imuFilepath)
# print(((os.path.basename(imu_file_path)).split('.'))[0] + '.bin')


import socket

#creat a tcp/ip socket
sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)

#bind the socket to the port
server_address=('',8080)
print("starting up on %s poot %s" % server_address)
sock.bind(server_address)

#listen fir incomming connections
sock.listen(10)

while True:
    #wait for connection
    print("waiting for a connection.....")
    connection,client_address=sock.accept()

    try:
        print("connection from",client_address)
        while True:
            data=connection.recv(1024)
            print("received %s" % data.decode())
            if data:
                print("sending data back to the clien")
                connection.sendall(data)
            else:
                print("no more data from",client_address)
                break
    finally:
        connection.close()


import pdb
#
# config/yolov3.cfg
def parse_model_config(path):
    ''' Parses the yolo-v3 layer configuration file and returns module definitions'''
    file = open(path, 'r')
    lines = file.read().split('\n')
    lines = [x for x in lines if x and not x.startswith('#')]  # 값이 있는 줄이면서 #으로 시작하지 않는것만 가져옴
    lines = [x.rstrip().lstrip() for x in lines]  # 좌우 공백 제거
    module_defs = []
    for line in lines:
        if line.startswith('['):
            module_defs.append({})  # 다음 타입이 들어올때마다 새로운 딕셔너리 생성
            module_defs[-1]['type'] = line[1:-1].rstrip()  # 딕셔너리에 type 키값에 []제거한 값추가
            if module_defs[-1]['type'] == 'convolutional':
                module_defs[-1]['batch_normalize'] = 0  # batch_normalize가 써있지 않은 conv에서 0으로 따로 지정해놓음
        else:
            key, value = line.split('=')
            value = value.strip()
            module_defs[-1][key.rstrip()] = value.strip()  # 해당 type에 keys, values 추가

    return module_defs

#config/coco.data
def parse_data_config(path):
    '''Parses the data configuration file'''
    options = dict()
    options['gpus'] = '0,1,2,3'
    options['num_workers'] = '6'  # 내컴퓨터 6개임 github : '10'
    with open(path, 'r') as fp:
        lines = fp.readlines()
    for line in lines:
        line = line.strip()  # 양끝 공백제거 + \n 제거
        if line == '' or line.startswith('#'):
            continue
        key, value = line.split('=')
        options[key.strip()] = value.strip()
    return options

if __name__ == '__main__':
    a = parse_model_config('../config/yolov3.cfg')
    print(a)
    print(len(a))

    b = parse_data_config('../config/coco.data')
    print(b)


import json

from .plot_headline import PlotHeadline

class VisualizationSpecification:
    def __init__(self):
        self.plot_headline = PlotHeadline()
        self.target_id = -1
        self.gesture_target_id = -1
        self.horizontal_axis = None
        self.horizontal_group_axis = None
        self.vertical_axis = None
        self.data_query = None
        self.data_query_results = []
        self.dialogue_act = None
        self.response_text = None
        self.visualization_task = None
        self.plot_headline_history = []

    def get_json_obj(self):
        s = json.dumps(self, default=self._to_json, sort_keys=True, indent=4)
        obj = json.loads(s)
        return obj

    def get_json_str(self):
        return json.dumps(self, default=self._to_json, sort_keys=True, indent=4)

    def _to_json(self, o):
        if hasattr(o, '__dict__'):
            return o.__dict__
        elif isinstance(o, set):
            return list(o)


from torch.utils.data import DataLoader, Sampler, Dataset, RandomSampler, DistributedSampler
import json
# from lsp_model import GPT2Tokenizer
from transformers import GPT2Tokenizer
from tqdm import tqdm
import torch
from torch.nn.utils.rnn import pad_sequence
import random
import pickle
import os

def shuffle_persona(persona_list, persona_label):
    id_list = list(range(len(persona_list)))
    shuffled_id_list = list(range(len(persona_list)))
    random.shuffle(shuffled_id_list)

    n_list = [_ for _ in range(len(persona_list))]
    n_label = [-1] * len(persona_label)
    k = 0
    for i, j in zip(id_list, shuffled_id_list):
        n_list[j] = persona_list[i]
        if k < len(persona_label) and persona_label[k] == i:
            n_label[k] = j
            k += 1

    # for check
    # print ('==' * 20)
    # print (id_list)
    # print (shuffled_id_list)
    # print (persona_list, persona_label)
    # print (n_list, n_label)

    return n_list, n_label

def read_file(path, with_persona_label, shuffle=False, mode='label', all_seq_loss=False, single_turn=False, small_data=False, only_persona_response=False):
    END_OF_TEXT_TOKEN = '<|endoftext|>'
    tokenizer = GPT2Tokenizer.from_pretrained('gpt2') # the smallest version of GPT-2, with 124M parameters.
    # tokenizer = GPT2Tokenizer.from_pretrained("microsoft/DialoGPT-small")
    eos = tokenizer.encoder[END_OF_TEXT_TOKEN]

    examples = []

    pick_id = random.randint(1, 7)
    if not shuffle and os.path.exists(path.strip('output') + 'cached'):
        # return torch.load(path.strip('output') + 'cached.npy').tolist()
        return pickle.load(open(path.strip('output') + 'cached', "rb"))

    if shuffle and os.path.exists(path.strip('output') + f'shuffle{pick_id}'):
        # return torch.load(path.strip('output') + 'cached.npy').tolist()
        return pickle.load(open(path.strip('output') + f'shuffle{pick_id}', "rb"))

    with open(path, 'r', encoding='utf-8') as r:
        data = json.load(r)
    # [persona_list, history, response, persona_label] # response is not a list
    for i, (persona_list, history, response, persona_label) in tqdm(enumerate(data)):
    # for i, (persona_list, history, response, persona_label) in enumerate(data):
        if only_persona_response and persona_label[0] == -1:
            continue
        if shuffle:
            persona_list, persona_label = shuffle_persona(persona_list, persona_label)
        if single_turn:
            history = [history[-1]]
        response = [response]
        persona_label_entry = [persona_list[e] for e in persona_label if e > -1]
        persona_label_entry = [' '.join(persona_label_entry).strip()]
        persona_label = [str(e + 1) for e in persona_label] # idx + 1

        if with_persona_label: # add to response list
            if mode == 'entry' and persona_label_entry != ['']:
                # persona_label_entry[0] = persona_label_entry[0] + '\t'
                persona_label_entry[0] = persona_label_entry[0]
                response = [persona_label_entry[0], response[0]] # persona entries + '\t' + response
                assert len(response) == 2
            # print (persona_list, history, response, persona_label, persona_label_entry)
            else:
                # label_str = ' '.join(persona_label).strip() + '\t'
                # label_str = ' '.join(persona_label).strip() + END_OF_TEXT_TOKEN # 直接加<eos> tokenizer不识别。。。
                label_str = ' '.join(persona_label).strip()
                response = [label_str, response[0]]
                assert len(response) == 2

        # tokenize
        persona_list = [tokenizer.encode(s) for s in persona_list]
        history = [tokenizer.encode(s) for s in history]
        response = [tokenizer.encode(s) for s in response]
        persona_label_entry = [tokenizer.encode(s) for s in persona_label_entry]
        persona_label = [tokenizer.encode(str(s)) for s in ' '.join(persona_label).strip()]
        # if not with_persona_label:
        #     persona_label = None

        # making input_ids, position_ids, lm_ids...
        example = make_example_inputs(i, persona_list, history, response, persona_label, persona_label_entry, eos, all_seq_loss=all_seq_loss)
        examples.append(example)

        if small_data and i >= 200:
            break

    if not shuffle and not os.path.exists(path.strip('output') + 'cached'):
        # torch.save(path.strip('output') + 'cached.npy', np.array(examples))
        pickle.dump(examples, open(path.strip('output') + 'cached', "wb"))

    if shuffle and not os.path.exists(path.strip('output') + f'shuffle{pick_id}'):
        # torch.save(path.strip('output') + 'cached.npy', np.array(examples))
        pickle.dump(examples, open(path.strip('output') + f'shuffle{pick_id}', "wb"))
    return examples

def make_example_inputs(id, personas, context, response, persona_label, persona_label_entry, eos, all_seq_loss=False):
    # print (personas , context , persona_label , persona_label_entry, response)

    # sents = None
    # if persona_label:
    #     sents = personas + context + persona_label_entry + response # 0 + 1 + 2 + 2
    # else:
    #     sents = personas + context + response
    sents = personas + context + response

    # 1. input_ids: 每个uttr加了eos，去掉了最后一位
    # print (sents)
    input_ids = [i for s in sents for i in s+[eos]][:-1]
    token_type_ids = []  # this becomes round ids
    lm_labels = []

    # 2. lm_labels: input_ids[1:] + [eos]
    #    token_type_ids: 0 for persona, 1 for context, 2 for persona_label and response
    for i, s in enumerate(sents):
        if i == 0:
            token_type_ids += [0] * len(s)
            lm_labels += [-1] * len(s) if not all_seq_loss else s[1:] + [eos]
        elif i < len(personas): # persona: 0
            token_type_ids += [0] * (len(s) + 1)
            lm_labels += [-1] * (len(s) + 1) if not all_seq_loss else s + [eos]
        elif i < len(personas) + len(context): # context: 1
            token_type_ids += [1] * (len(s) + 1)
            lm_labels += [-1] * (len(s) + 1) if not all_seq_loss else s + [eos]
        else: # persona_label/entry + '\t' + response: 2
            token_type_ids += [2] * (len(s) + 1)
            lm_labels += (s + [eos])

    # handle trailing -1's
    i = len(lm_labels) - 1
    while i >= 0:
        if lm_labels[i] != -1:
            break
        i -= 1
    input_ids = input_ids[:i+1]
    lm_labels = lm_labels[:i+1]
    token_type_ids = token_type_ids[:i+1]

    # pad to multiples of 8
    while len(input_ids) % 8 != 0:
        input_ids.append(0)
        token_type_ids.append(0)
        lm_labels.append(-1)

    # 3. position_ids
    position_ids = list(range(len(input_ids)))
    assert (len(input_ids) == len(position_ids) == len(token_type_ids)
            == len(lm_labels))
    assert len(input_ids) % 8 == 0

    # example = [id, input_ids, position_ids, token_type_ids,
                            # lm_labels]
    # print (all_seq_loss, input_ids, lm_labels)

    example = {
        'id': id,
        'input_ids': input_ids,
        'position_ids': position_ids,
        'token_type_ids': token_type_ids,
        'lm_labels': lm_labels,
        'input_len': len(input_ids)
    }
    return example

class PersonaDataset(Dataset):
    """ pytorch dataset for GPT2 training """
    def __init__(self, path, max_len=None, with_persona_label=True, shuffle=False, all_seq_loss=False, single_turn=False, small_data=False, only_persona_response=False, **kwargs):
        self.example_ids = read_file(path, with_persona_label, shuffle, all_seq_loss=all_seq_loss, single_turn=single_turn, small_data=small_data, only_persona_response=only_persona_response)
        # print ('data_num = ', len(self.example_ids))
        self.max_len = max_len  # this max_len do truncate

    def __getitem__(self, i):
        return self.example_ids[i]

    def __len__(self):
        return len(self.example_ids)

    @staticmethod
    def collate(features):
        # print (features)
        input_ids = pad_sequence([torch.tensor(f['input_ids'], dtype=torch.long)
                                  for f in features],
                                 batch_first=True, padding_value=0)
        position_ids = pad_sequence([torch.tensor(f['position_ids'],
                                                  dtype=torch.long)
                                     for f in features],
                                    batch_first=True, padding_value=0)
        token_type_ids = pad_sequence([torch.tensor(f['token_type_ids'],
                                                    dtype=torch.long)
                                       for f in features],
                                      batch_first=True, padding_value=0)
        labels = pad_sequence([torch.tensor(f['lm_labels'], dtype=torch.long)
                               for f in features],
                              batch_first=True, padding_value=-1)
        return (input_ids, position_ids, token_type_ids, labels)

# test
# tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
# path = '/misc/kfdata01/kf_grp/lchen/D3/data_manipulation/data_distillation/predictions/test/output'
# path = '/misc/kfdata01/kf_grp/lchen/D3/data_manipulation/data_distillation/predictions/valid/output'
# path = '/misc/kfdata01/kf_grp/lchen/D3/data_manipulation/data_distillation/predictions/train/output'
# dataset = PersonaDataset(path, max_len=180, with_persona_label=False, shuffle=False)
# dataset = PersonaDataset(path, max_len=180, with_persona_label=False, shuffle=True)
# # dataset = PersonaDataset(path, max_len=256, with_persona_label=False, shuffle=False, single_turn=True)
# dataset = PersonaDataset(path, max_len=256, with_persona_label=False, shuffle=False, single_turn=True, only_persona_response=True)
# sampler = RandomSampler(dataset) if True else DistributedSampler(dataset)
# dataloader = DataLoader(dataset, sampler=sampler, batch_size=4, collate_fn=PersonaDataset.collate)

# for i, batch in enumerate(dataloader):
#     seq_len = batch[0].shape[1]
#     input_ids, position_ids, token_ids, label_ids, *_ = batch

#     if i > 5:
#         break

#     # visualize data
#     print ('=='*10 + ' visualize data ' + '=='*10)
#     print ('input_ids.shape, position_ids.shape, label_ids.shape = ', input_ids.shape, position_ids.shape, label_ids.shape) # torch.Size([4, 512]) torch.Size([4, 512])
#     print ('input_ids[0] = ', input_ids[0])
#     print ('position_ids[0] = ', position_ids[0])
#     print ('token_ids[0] = ', token_ids[0])
#     print ('label_ids[0] = ', label_ids[0])
#     # 'GPT2Tokenizer' object has no attribute 'batch_decode'???
#     # print (tokenizer.batch_decode(inputs[0], skip_special_tokens=True))
#     # print (tokenizer.batch_decode(labels[0], skip_special_tokens=True))

#     print ('input_ids = \n', tokenizer.decode(input_ids[0].tolist()))
#     # mask = token_ids.eq(2) | token_ids.eq(3)
#     mask = token_ids.eq(2)
#     print (mask.shape)
#     # print (mask[0])
#     print ('label_ids = \n', tokenizer.decode(label_ids[0][mask[0]].tolist()))
#     print ()
#     # break


# ----------------------------------------
# Created by Serhii Skliarov
#
# Version      Date     Info
# 1.0          2021     Init version
#
#-----------------------------------------

enemy = {
    "loc_x": 70,
    "loc_y": 50,
    "color": "green",
    "health": 100,
    "name": "zorg"
}

print("Loc X: " + str(enemy["loc_x"]) + " Loc Y: " + str(enemy["loc_y"]))
print("Name: " + enemy["name"])

""" enemy["rank"] = "admiral"
print(enemy)

del enemy["color"]
print(enemy) """

enemy["loc_x"] = enemy["loc_x"] + 40
enemy["health"] = enemy["health"] - 30
if enemy["health"] < 80:
    enemy["color"] = "red"

print(enemy)

print(enemy.keys())
print(enemy.values())


import os
import numpy as np
import cv2

def draw_bbox_results(image, results, input_path, save_dir=None):
    for result in results:
        [xmin, ymin, xmax, ymax] = result["bbox"]

        image = cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 255, 0),
                              2)

    image_name = os.path.basename(input_path)
    if save_dir is None:
        save_dir = "output"
    os.makedirs(save_dir, exist_ok=True)
    output_path = os.path.join(save_dir, image_name)
    cv2.imwrite(output_path, image)


def component_id(self):
    """Returns component_id of this GlobalStreamId

    Note that if HeronComponentSpec is specified as componentId and its name is not yet
    available (i.e. when ``name`` argument was not given in ``spec()`` method in Bolt or Spout),
    this property returns a message with uuid. However, this is provided only for safety
    with __eq__(), __str__(), and __hash__() methods, and not meant to be called explicitly
    before TopologyType class finally sets the name attribute of HeronComponentSpec.
    """
    if isinstance(self._component_id, HeronComponentSpec):
      if self._component_id.name is None:
        # HeronComponentSpec instance's name attribute might not be available until
        # TopologyType metaclass finally sets it. This statement is to support __eq__(),
        # __hash__() and __str__() methods with safety, as raising Exception is not
        # appropriate this case.
        return "<No name available for HeronComponentSpec yet, uuid: %s>" % self._component_id.uuid
      return self._component_id.name
    elif isinstance(self._component_id, str):
      return self._component_id
    else:
      raise ValueError("Component Id for this GlobalStreamId is not properly set: <%s:%s>"
                       % (str(type(self._component_id)), str(self._component_id)))


#goes through and marks any non number or character differences as dupes
#not always valed, eg 29 != 2,9
import pandas as pd
import glob
import re
import sys

if len(sys.argv) > 1:
    start_num = int(sys.argv[1])
else:
    start_num = 0

regex = re.compile('[^a-zA-Z0-9\*]')

def j_s(st):
    return regex.sub('', st.lower().replace("&", "and"))

look_at_folder = ".\SH_data\current_targets"

file_sub = ""

file_list = glob.glob(look_at_folder + "\\" + file_sub + "*")

edited = 0
group_num = 0

for f in file_list:
    changed = False
    df = pd.read_csv(f)
    uni = df[df.columns[0]].tolist()
    uni = [j_s(u) for u in uni]
    uni = [l for l in uni if uni.count(l) > 1]
    uni = list(set(uni))
    labels = []
    group_num = start_num
    for u in uni:
        group_num += 1
        labels.append([u, group_num, "ahi"])
    for label in labels:
        targets = ([ j_s(x) == label[0] for x in df[df.columns[0]] ])
        rows = df.index[targets]
        for row in rows:
            if (df.iloc[row]['group'] != -1) :
                continue
            df.iat[row, 4] = label[1]
            df.iat[row, 5] = label[2]
            if not changed : changed = (len(rows) > 0)

    if (changed):
        df.to_csv(f, index = False)
        edited += 1
        print(f)

print("Edited", edited, "files")
print("Created", group_num, "groups")


import sys

def isLeapYr(n):
    if (n%4 == 0 and n%100 != 0) or (n%400 == 0):
        return 1
    else: return 0

yr = 1900
mon = 12
day = 6
week = ["sun", "mon", "tue","wed", "thu", "fri", "sat" ]

li = [31,28,31,30,31,30,31,31,30,31,30,31]

if __name__ == "__main__":
    count = 0
    while (yr < 2001):
        if mon == 2:
            day += li[1] + isLeapYr(yr)
        else:
            day += li[mon-1]

        day = day%7

        print(week[day])

        if day == 0:
            count += 1

        mon +=1
        if mon == 13:
            print (yr)
            mon = 1
            yr +=1
            if yr == 2001 and day == 0:
                count = count-1

print(count)


from django_bootstrap import bootstrap
bootstrap()

import sys

from regis.models import Applicant
from appl.models import AdmissionProject, AdmissionResult, AdmissionRound, Major

def main():
    project_id = sys.argv[1]
    round_id = sys.argv[2]

    result_filename = sys.argv[3]
    natid_map_filename = sys.argv[4]
    major_map_filename = sys.argv[5]

    if natid_map_filename != '-':
        natid_map = dict([(f[0],f[1]) for f in
                          [l.strip().split(',') for l in
                           open(natid_map_filename).readlines()]])
    else:
        natid_map = None

    major_map = dict([(f[1],f[0]) for f in
                      [l.strip().split(',') for l in
                       open(major_map_filename).readlines()]])

    is_fake = len(sys.argv) <= 6 or (sys.argv[6] != 'real')

    admission_project = AdmissionProject.objects.get(pk=project_id)
    admission_round = AdmissionRound.objects.get(pk=round_id)
    project_round = admission_project.get_project_round_for(admission_round)

    majors = dict([(m.number, m) for m in
                   Major.objects.filter(admission_project=admission_project).all()])

    result_lines = open(result_filename).readlines()

    mcount = {}

    counter = 0
    for line in result_lines:
        items = line.strip().split(',')
        if len(items)!=3:
            continue

        nat_id = items[0].strip()
        major_id = items[1]
        decision = items[2]

        if natid_map != None:
            sys_nat_id = natid_map[nat_id]
        else:
            sys_nat_id = nat_id

        major_number = major_map[major_id]

        major = majors[int(major_number)]

        if decision == '1':
            if major.number not in mcount:
                mcount[major.number] = 0
            mcount[major.number] += 1

        try:
            applicant = Applicant.objects.get(national_id=sys_nat_id)
        except:
            print('ERROR', nat_id, sys_nat_id)
            continue

        admission_results = AdmissionResult.objects.filter(applicant=applicant,
                                                          major=major).all()
        if len(admission_results)!=1:
            print('ERROR',nat_id,major_id)
            continue

        admission_result = admission_results[0]

        if decision == '1':
            admission_result.is_tcas_confirmed = True
            if not is_fake:
                admission_result.save()
        else:
            admission_result.is_tcas_confirmed = False
            if not is_fake:
                admission_result.save()

        counter += 1
        #print(applicant)

        if counter % 1000 == 0:
            print(counter)

    for num in sorted(majors.keys()):
        if num not in mcount:
            c = 0
        else:
            c = mcount[num]

        print("{0},{1},{2}".format(num,majors[num].title,c))

if __name__ == '__main__':
    main()


# Gregory Rosenblatt
# 4/12/06

from weakref import WeakValueDictionary

def IsSingleton(obj, instances=WeakValueDictionary()):
	"""Assert that the class of obj has not already been instantiated."""
	if obj.__class__ in instances:
		return False
	instances[obj.__class__] = obj
	return True


from flask import Flask, render_template, send_from_directory,json,jsonify

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route("/example.json")
def data():
    f = open('static/js/example.json')
    data = json.load(f)
    return jsonify(data)


#pip install -q -U tensorflow-text

#import tensorflow_text as text

import seaborn as sns
import tensorflow as tf
import numpy as np
import pandas as pd
import os
import nltk
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import WhitespaceTokenizer
wn = WordNetLemmatizer()
stopwords = nltk.corpus.stopwords.words('english')
import re
import utils
from config import *

os.environ["OMP_NUM_THREADS"] = str(NUM_THREADS)
os.environ["TF_NUM_INTRAOP_THREADS"] = str(NUM_THREADS)
os.environ["TF_NUM_INTEROP_THREADS"] = str(NUM_THREADS)

tf.config.threading.set_inter_op_parallelism_threads(
    NUM_THREADS
)
tf.config.threading.set_intra_op_parallelism_threads(
    NUM_THREADS
)
tf.config.set_soft_device_placement(True)

df, unique_labels = utils.preprocess_data(PATH)
df = utils.clean_data(df)

def label_freq(df, unique_labels):
    freq = dict()
    for unique_label in unique_labels:
        freq[unique_label] = df[unique_label].mean() * df[unique_label].shape[0]
    print("Label frequency ", freq)
    sns.set(rc={'figure.figsize': (11.7, 8.27)})
    plot = sns.barplot(list(freq.keys()), list(freq.values()))
    plot.set_xticklabels(plot.get_xticklabels(),
                         rotation=45,
                         horizontalalignment='right')

def save_model(model, base_fp):
    # save the model: first the weights then the arch
    model.save_weights('{}-weights.{}'.format(base_fp, FILE_TYPE))
    with open('{}-architecture.json'.format(base_fp), 'w') as f:
        f.write(model.to_json())

with tf.device('/cpu:0'):
    from tensorflow.keras.preprocessing.text import Tokenizer
    from tensorflow.keras.preprocessing.sequence import pad_sequences
    tokenizer = Tokenizer(num_words=MAX_WORDS, lower=True)
    tokenizer.fit_on_texts(df['content'])
    sequences = tokenizer.texts_to_sequences(df['content'])
    x = pad_sequences(sequences, maxlen=MAX_LEN)

    train = df
    x_train = train["content"].str.lower()
    y_train = train[unique_labels].values

    tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=MAX_WORDS, lower=True)

    tokenizer.fit_on_texts(x_train)

    x_train = tokenizer.texts_to_sequences(x_train)
    x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=MAX_LEN)

    embeddings_index = {}

    with open(GLOVE_EMBEDDING, encoding='utf8') as f:
        for line in f:
            values = line.rstrip().rsplit(' ')
            word = values[0]
            embed = np.asarray(values[1:], dtype='float32')
            embeddings_index[word] = embed

    word_index = tokenizer.word_index

    embedding_matrix = np.zeros((MAX_WORDS, EMBEDDING_SIZE), dtype='float32')

    for word, i in word_index.items():

        if i >= MAX_WORDS:
            continue

        embedding_vector = embeddings_index.get(word)

        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector

    input = tf.keras.layers.Input(shape=(MAX_LEN,))
    x = tf.keras.layers.Embedding(MAX_WORDS, EMBEDDING_SIZE, weights=[embedding_matrix], trainable=False)(input)

    x = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(128, return_sequences=True, dropout=0.1,
                                                          recurrent_dropout=0.1))(x)

    x = tf.keras.layers.Conv1D(100, kernel_size=1)(x)

    avg_pool = tf.keras.layers.GlobalAveragePooling1D()(x)
    max_pool = tf.keras.layers.GlobalMaxPooling1D()(x)

    x = tf.keras.layers.concatenate([avg_pool, max_pool])
    # x = tf.keras.layers.Dense(10, activation='relu')(x)
    preds = tf.keras.layers.Dense(len(unique_labels), activation="sigmoid")(x)

    model = tf.keras.Model(input, preds)

    model.summary()

    model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(lr=1e-3), metrics=['accuracy'])

    batch_size = 128

    checkpoint_path = "./training_1/cp.ckpt"
    checkpoint_dir = os.path.dirname(checkpoint_path)

    cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,
                                                     save_weights_only=True,
                                                     verbose=1)

    callbacks = [
        tf.keras.callbacks.EarlyStopping(patience=5, monitor='val_loss'),
        tf.keras.callbacks.TensorBoard(log_dir='./logs'),
        cp_callback
    ]

    model.fit(x_train, y_train, validation_split=0.2, batch_size=batch_size,
              epochs=EPOCHS, callbacks=callbacks, verbose=1)

    save_model(model, BASE_FP)

    model.save(checkpoint_dir, save_format='tf')

    latest = tf.train.latest_checkpoint(checkpoint_dir)

    model.load_weights(latest)

    predictions = model.predict(np.expand_dims(x_train[43], 0))

    print(tokenizer.sequences_to_texts([x_train[43]]))
    print(y_train[43])
    print(predictions)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2019-03-12 14:42:47
# @Author  : Your Name (you@example.org)
# @Link    : http://example.org
# @Version : $Id$
# @Desc    : 用CNN做的模型  初始化权重 ReLU
#
import tensorflow as tf
from numpy import float32
import datetime
from tensorflow.examples.tutorials.mnist import input_data

# 初始化w、b
def  weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev = 0.1)
    return tf.Variable(initial)
def bias_variable(shape):
    initial = tf.constant(0.1, shape= shape)
    return tf.Variable(initial)

# 卷积和池化
def conv2d(x,W):
    return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding="SAME")
def max_pool_2x2(x):
    return tf.nn.max_pool(x,ksize=[1,2,2,1], strides=[1, 2, 2,1],padding="SAME")

start = datetime.datetime.now()
# 载入数据
mnist = input_data.read_data_sets('./data',one_hot=True)

# None表示可以是任何维度
x = tf.placeholder(tf.float32,[None,784])
# y:正确值(正确label)
y = tf.placeholder(tf.float32, [None, 10])

############ 第一层CNN ###############
# 28*28  --->   24*24  --->   12*12
W_conv1 = weight_variable([5,5,1,32])
b_conv1 = bias_variable([32])
# input
x_image = tf.reshape(x,[-1,28,28,1])
h_conv1 = tf.nn.relu(conv2d(x_image,W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)

############ 第二层CNN ###############
# 12*12   --->   8*8   --->  7*7

W_conv2 = weight_variable([5,5,32,64])
b_conv2 = bias_variable([64])

h_conv2 = tf.nn.relu(conv2d(h_pool1,W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)

############ 全连接层 ###############
W_fc1 = weight_variable([7*7*64,1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2,[-1,7*7*64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat,W_fc1)+b_fc1)

############ Dropout 减少过拟合 ###############
keep_prob = tf.placeholder("float")
h_fc1_drop = tf.nn.dropout(h_fc1,keep_prob)

############ 输出层 ###############
W_fc2 = weight_variable([1024,10])
b_fc2 = bias_variable([10])
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop,W_fc2) + b_fc2)

# 损失函数 loss function
cross_entropy = -tf.reduce_sum(y * tf.log(y_conv))

# ADAM优化器做梯度下降
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

# 找出预测正确的标签 tf.equal()返回的是true or false 数组
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y,1))
# 求正确率
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

#存储训练的模型
saver = tf.train.Saver(max_to_keep=1)

# 一批次大小
batch_size = 50
# 所有批次数量
n_batch = mnist.train.num_examples // batch_size
with tf.Session() as sess:
    saver_max_acc = 0
    sess.run(tf.initialize_all_variables())
    for epoch in range(1):
        for i in range(n_batch):
            batch = mnist.train.next_batch(50)
            sess.run(train_step,feed_dict={x:batch[0], y: batch[1], keep_prob: 0.5})
            if i%100 == 0:
                train_accuracy = accuracy.eval(feed_dict={x:batch[0], y: batch[1], keep_prob: 1.0})
                print("step %d acc : %g"%(i,train_accuracy))

                acc=accuracy.eval(feed_dict={x: mnist.test.images[0:2000], y: mnist.test.labels[0:2000], keep_prob: 1.0})
                print("test %d accuracy %g"%(epoch,acc))
                # 添加判断语句，选择保存精度最高的模型
                if acc > saver_max_acc:
                    saver.save(sess,'./model/mnist02_CNN/mnist02_CNN.ckpt',global_step=epoch+1)
                    saver_max_acc = acc

end = datetime.datetime.now()
print((end - start).seconds)


# author: Ziming Guo
# time: 2020/3/2
'''
    作业3：
    在list_helper.py中新增以下功能：
        (1)获取最小值
        (2)降序排列
        (3)根据指定条件删除元素
           （注意曾经的经验:倒着删除）
           案例:在敌人列表中，删除所有死人.
           案例:在敌人列表中，攻击力小于50的所有敌人.
           案例:在敌人列表中，防御力大于100的所有敌人.
'''

class Enemy:
    def __init__(self, name, atk, defense, hp):
        self.name = name
        self.atk = atk
        self.defense = defense
        self.hp = hp

    def __str__(self):
        return "名字是%s，攻击力是%d，防御力是%d，血量是%d" % (self.name, self.atk, self.defense, self.hp)
    # 再写一遍这个 __str__ 方法的作用：
    # 有了这个方法之后，再次打印 item 的时候就会按照这个方法里的返回值的形式来返回字符串了

list_enemy = [
    Enemy("威震天", 15, 10, 80),
    Enemy("霸天虎", 19, 13, 85),
    Enemy("灭霸", 30, 15, 69),
    Enemy("鲨鱼辣椒", 8, 6, 77),
    Enemy("大坏人", 6, 2, 0),
    Enemy("成昆", 55, 5, 0)
]

from common_my.list_helper_2 import *

# （1）

def smaller_condition(item):
    return item.atk

print(ListHelper.get_min(list_enemy, smaller_condition))

print(ListHelper.get_min(list_enemy, lambda item: item.atk))

# 注意：定义的新的条件函数的作用，只是，把要比较 or 提取的变量提取出来，不用做别的

print("------------------------------------------------")

# (2)
for item in ListHelper.down_order_ranking(list_enemy, lambda item: item.atk):
    print(item)

print("------------------------------------------------")

# (3)

# def delete_condition_01(item):
#     return item.atk == 0
#
#
# for item in ListHelper.delete_element(list_enemy, delete_condition_01):
#     print(item)

print("------------------------------------------------")

# def delete_condition_02(item):
#     return item.atk < 50
#
#
# for item in ListHelper.delete_element(list_enemy, delete_condition_02):
#     print(item)

print("------------------------------------------------")

def delete_condition_03(item):
    return item.atk > 100

for item in ListHelper.delete_element(list_enemy, delete_condition_03):
    print(item)


import time
from threading import Thread
from parser.tsnParser import TsnParser
from parser.ukrNetParser import UkrNetParser

class ParserThread(Thread):
    """
    A class used to represent a parser of news
    """

    def __init__(self, name, secondsUpdating):
        """
        Constructor

        Prameters
        ----------
        name : str
            the name of the thread

        secondsUpdating : int
            sleep time period before next parsing round
        """
        Thread.__init__(self)
        self.name = name
        self.secondsUpdating = secondsUpdating

    def run(self):
        ukrNetParser = UkrNetParser()
        tsnParser = TsnParser()
        while True:
            tsnParser.parseNews()
            ukrNetParser.parseNews()
            time.sleep(self.secondsUpdating)


"""
Simplemonitor logger for seq

Inspiration from
https://raw.githubusercontent.com/eifinger/appdaemon-scripts/master/seqSink/seqSink.py
"""

import datetime
import json
from typing import Optional, cast

import requests

from ..Monitors.monitor import Monitor
from .logger import Logger, register

@register
class SeqLogger(Logger):
    """Logging to seq"""

    logger_type = "seq"
    only_failures = False
    buffered = False
    dateformat = None

    def __init__(self, config_options: Optional[dict] = None) -> None:
        if config_options is None:
            config_options = {}
        super().__init__(config_options)

        # i.e. http://192.168.0.5:5341
        self.endpoint = cast(
            str, self.get_config_option("endpoint", required=True, allow_empty=False)
        )
        self.timeout = cast(
            int, self.get_config_option("timeout", required_type="int", default=5)
        )
        # Potentially, would need to add a header for ApiKey

        # Send message to indicate we have started logging
        self.log_to_seq(
            self.endpoint,
            "SeqLogger",
            "simpleMonitor",
            "__init__",
            None,
            "logging enabled for simpleMonitor",
            False,
        )

    def save_result2(self, name: str, monitor: Monitor) -> None:
        try:
            is_fail = monitor.test_success() is False

            self.log_to_seq(
                self.endpoint,
                name,
                monitor.name,
                monitor.monitor_type,
                str(monitor.get_params()),
                monitor.describe(),
                is_fail,
            )
        except Exception:
            self.logger_logger.exception("Error sending to seq in %s", monitor.name)

    def describe(self) -> str:
        return "Sends simple log to seq using raw endpoint"

    def log_to_seq(
        self, endpoint, name, app_name, monitor_type, params, description, is_fail
    ):
        """Send an event to seq"""
        event_data = {
            "Timestamp": str(datetime.datetime.now()),
            "Level": "Error" if is_fail is True else "Information",
            "MessageTemplate": str(description),
            "Properties": {
                "Type": "simpleMonitor",
                "Name": name,
                "Monitor": str(app_name),
                "MonitorType": monitor_type,
                # "Params": params
            },
        }
        if params is not None:
            event_data["Properties"]["Params"] = params

        request_body = {"Events": [event_data]}

        try:
            _ = json.dumps(request_body)  # This just checks it is valid...
        except TypeError:
            self.logger_logger.error("Could not serialise %s", request_body)
            return

        try:
            response = requests.post(
                self.endpoint, json=request_body, timeout=self.timeout
            )
            if not response.status_code == 200 and not response.status_code == 201:
                self.logger_logger.error(
                    "POST to seq failed with status code: %s", response
                )
        except requests.RequestException:
            self.logger_logger.exception("Failed to log to seq")


#!/usr/bin/env python

import os
# import shutil
import numpy as np
from math import pi, cos, sin, sqrt
import rospy
from std_msgs.msg import Float32
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from std_msgs.msg import Float32MultiArray
from geometry_msgs.msg import PoseStamped, Twist
from crazyflie_game.msg import Mocap
from player_recorder import DataRecorder

class GameRecorder(object):

	def __init__(self, Ds='', Is='',
				 max_size=1e4,
				 rate=10,
				 logger_dir='res1'):

		self._player_dict = dict()
		for i, D in enumerate(Ds):
			if D != '':
				self._player_dict['D'+str(i+1)] = D
		for i, I in enumerate(Is):
			if I != '':
				self._player_dict['I'+str(i+1)] = I

		script_dir = os.path.dirname(__file__)
		self._results_dir = os.path.join(script_dir, logger_dir + '/')
		# self._results_dir = os.path.join('RAgame/exp_results/', logger_dir + '/')
		self._a_dirc = os.path.join(self._results_dir, 'a.csv')
		if os.path.exists(self._a_dirc):
			os.remove(self._a_dirc)

		# if os.path.exists(self._results_dir):
		# 	shutil.rmtree(self._results_dir)
		if not os.path.isdir(self._results_dir):
			os.makedirs(self._results_dir)

		self._init_time = self._get_time()
		self._save_interval = 50
		self.rate = rospy.Rate(rate)

		self._locations = {'D1': DataRecorder(max_size=max_size),
						   'D2': DataRecorder(max_size=max_size),
						   'I1': DataRecorder(max_size=max_size)}
		self._headings = {'D1': DataRecorder(max_size=max_size),
						  'D2': DataRecorder(max_size=max_size),
						  'I1': DataRecorder(max_size=max_size)}
		self._location_sub_callback_dict = {'D1': self._getLocD1, 'D2': self._getLocD2, 'I1': self._getLocI}
		self._location_subs = dict()
		for p_id, cf_frame in self._player_dict.items():
			self._location_subs.update({p_id: rospy.Subscriber('/' + cf_frame + '/mocap', Mocap, self._location_sub_callback_dict[p_id])})
		self._a_sub = rospy.Subscriber('/a', Float32, self._update_a)
		# self._heading_sub_callback_dict = {'D1': self._getHeadingD1, 'D2': self._getHeadingD2, 'I1': self._getHeadingI}
		# self._heading_subs = dict()
		# for p_id, cf_frame in self._player_dict.items():
		#	 self._heading_subs.update(
		#		 {p_id: rospy.Subscriber('/' + cf_frame + '/heading_anl', Float32, self._heading_sub_callback_dict[p_id])})

		self._locs_plot = self._init_locs_plot()
		# last_res_id = 0
		# for _, dirc, file in os.walk(script_dir):
		#	 for d in dirc:
		#		 if 'res_' in d:
		#			 if int(d.split('_')[-1]) > last_res_id:
		#				 last_res_id = int(d.split('_')[-1])

	def _get_time(self):
		t = rospy.Time.now()
		return t.secs + t.nsecs * 1e-9

	def _update_a(self, a):
		t = self._get_time() - self._init_time
		with open(self._a_dirc, 'a') as f:
			f.write('%.3f, %.3f\n'%(t, a.data))

	def _rotate(self, data):
		return np.array([-data[1], data[0]])

	def _getLocD1(self, data):
		self._locations['D1'].record(self._get_time() - self._init_time, self._rotate(np.array([data.position[0], data.position[1]])))

	def _getLocD2(self, data):
		self._locations['D2'].record(self._get_time() - self._init_time, self._rotate(np.array([data.position[0], data.position[1]])))

	def _getLocI(self, data):
		self._locations['I1'].record(self._get_time() - self._init_time, self._rotate(np.array([data.position[0], data.position[1]])))

	def _getHeadingD1(self, data):
		self._headings['D1'].record(self._get_time() - self._init_time, data.data-pi/2)

	def _getHeadingD2(self, data):
		self._headings['D2'].record(self._get_time() - self._init_time, data.data-pi/2)

	def _getHeadingI(self, data):
		self._headings['I1'].record(self._get_time() - self._init_time, data.data-pi/2)

	def _init_locs_plot(self):
		fig, ax = plt.subplots(tight_layout=True)
		ax.set_xlabel('x(m)')
		ax.set_ylabel('y(m)')
		ax.grid()
		# gs = gridspec.GridSpec(2, 2)
		# axs = [fig.add_subplot(gs[2, 0]) for i in range(2)]
		# axs.append(fig.add_subplot(gs[:, 1]))
		# axs[0].set_xlabel('t(s)')
		# axs[0].set_ylabel('ID1(m)')
		# axs[1].set_xlabel('t(s)')
		# axs[1].set_ylabel('ID2(m)')
		# axs[2].set_xlabel('x(m)')
		# axs[2].set_ylabel('y(m)')

		plt.show(block=False)
		return {'fig': fig, 'axs': ax}

	def plot_locs(self):
		xD1 = np.asarray(self._locations['D1'].data)
		xD2 = np.asarray(self._locations['D2'].data)
		xI = np.asarray(self._locations['I1'].data)

		hD1 = np.asarray(self._headings['D1'].data)
		hD2 = np.asarray(self._headings['D2'].data)
		hI = np.asarray(self._headings['I1'].data)
		# print(hD1)

		def get_cap_ring(xd):
			rs = []
			for tht in np.linspace(0, 6.28, 50):
				x = xd[0] + .25*cos(tht)
				y = xd[1] + .25*sin(tht)
				rs.append((np.array([x, y])))
			return np.asarray(rs)

		def draw_anl_arrow(ax, x, y, heading):
			vx, vy = cos(heading), sin(heading)
			lenv = 10*sqrt(vx**2 + vy**2)
			vx, vy = vx/lenv, vy/lenv
			ax.arrow(x, y, vx, vy, fc='b', ec='b', head_width=.01, zorder=10)

		if len(xD1) > 100 and len(xD2) > 100 and len(xI) > 100:
			self._locs_plot['axs'].clear()

			self._locs_plot['axs'].plot([.5, .5], [-1.2, 1.2,], 'r')
			self._locs_plot['axs'].plot(xD1[100:-1, 0], xD1[100:-1, 1], 'b')
			self._locs_plot['axs'].plot(xD2[100:-1, 0], xD2[100:-1, 1], 'g')
			self._locs_plot['axs'].plot(xI[100:-1, 0], xI[100:-1, 1], 'r')
			ring1 = get_cap_ring(xD1[-1, :])
			ring2 = get_cap_ring(xD2[-1, :])
			self._locs_plot['axs'].plot(ring1[:, 0], ring1[:, 1], 'b')
			self._locs_plot['axs'].plot(ring2[:, 0], ring2[:, 1], 'g')
			if len(hD1) > 1 and len(hD2) > 1 and len(hI) > 1:
				draw_anl_arrow(self._locs_plot['axs'], xD1[-1, 0], xD1[-1, 1], hD1[-1])
				draw_anl_arrow(self._locs_plot['axs'], xD2[-1, 0], xD2[-1, 1], hD2[-1])
				draw_anl_arrow(self._locs_plot['axs'], xI[-1, 0], xI[-1, 1], hI[-1])
			self._locs_plot['fig'].canvas.draw()
			self._locs_plot['axs'].grid()
			self._locs_plot['axs'].axis('equal')
			if int(len(xI)) % self._save_interval == 0:
				self._locs_plot['fig'].savefig(self._results_dir + 'traj.png')

if __name__ == '__main__':

	rospy.init_node('game_recorder', anonymous=True)

	Ds = rospy.get_param("~Ds", '').split(',')
	Is = rospy.get_param("~Is", '').split(',')
	logger_dir = rospy.get_param("~logger_dir", '')
	# vd = rospy.get_param("~vd", 0.)
	# vi = rospy.get_param("~vi", 0.)
	# r = rospy.get_param("~a", 0.)
	# r_close = rospy.get_param("~r_close", 1.)
	# k_close = rospy.get_param("~k_close", .9)

	recorder = GameRecorder(Ds=Ds, Is=Is, logger_dir=logger_dir)

	while not rospy.is_shutdown():
		recorder.plot_locs()


import csv
import os

# File Path
file_path = file_path = os.path.join("Resources", "election_data.csv")

# Variables
total_votes = 0
candidates = {}
winner = None

# Opens CSV file
with open(file_path, 'r') as file:
    reader = csv.reader(file)
    next(reader)  # skips the header

    # Loops through each row in the CSV file
    for row in reader:
         # Extracting the candidate information
        _, _, candidate = row

        total_votes += 1

        # Tallys up the votes
        if candidate not in candidates:
            candidates[candidate] = 1
        else:
            candidates[candidate] += 1

# Determines the winner
winner = max(candidates, key=candidates.get)

# Displays the results
print("Election Results")
print("-------------------------")
print(f"Total Votes: {total_votes}")
print("-------------------------")
for candidate, votes in candidates.items():
    percentage = (votes / total_votes) * 100
    print(f"{candidate}: {percentage:.3f}% ({votes})")
print("-------------------------")
print(f"Winner: {winner}")
print("-------------------------")


from deck import Deck, values

class Hand():

	def __init__(self):
		self.cards = []
		self.value = 0
		self.aces = 0

	def add_card(self, card):
		self.cards.append(card)
		self.value += values[card.rank]

		# TRACK ACES
		if card.rank == 'Ace':
			self.aces += 1

	def adjust_for_ace(self):
		# IF TOTAL VALUE > 21 AND I STILL HAVE AN ACE
		# THAN CHANGE MY ACE TO BE A 1 INSTEAD OF AN 11

		while self.value > 21 and self.aces > 0:
			self.value -= 10
			self.aces  -= 1

test_player = Hand()


import pandas as pd
from IPython.display import display
import numpy as np

class netflixTopRecommenderSystem:
    # Constructor to initialize instance variables
    def __init__(self, datasetPath, genre, VotesThrs, TopN):
        self._datasetPath = datasetPath
        self._genre = genre
        self._voteThrs = VotesThrs
        self._topN = TopN
        self.df = None
        self._minVotes = None

        self.colDrop = ['year', 'certificate',
                        'duration', 'description', 'stars']
        self.colTitle = {"votes": "votes", "genre": "genre",
                         "title": "title", "rating": "rating"}

    # Method to read the csv file
    def readFile(self):
        self.df = pd.read_csv(self._datasetPath)

    # Method to clean the dataset
    def cleanDataset(self):

        # Dropping the unnecessary columns from the dataset
        for items in self.colDrop:
            self.df = self.df.drop([items], axis=1)

        # Dropping the duplicate rows based on the title column
        self.df = self.df.drop_duplicates(subset=[self.colTitle['title']])

        # Converting the votes column to float data type
        if type(self.df['votes'][0]) == str:
            self.df[self.colTitle['votes']] = self.df[self.colTitle['votes']].str.replace(
                ',', '').astype(float)

        # Filling the missing values with 0
        self.df = self.df.fillna(0)

    # Method to filter the dataset based on genre
    def filterGenre(self):
        self.df = self.df[self.df[self.colTitle['genre']].str.contains(
            self._genre, case=False).fillna(False)]

    # Method to filter the dataset based on the minimum number of votes threshold
    def threshVotes(self):
        self._minVotes = np.quantile(
            self.df[self.colTitle[self.colTitle['votes']]], self._voteThrs)
        self.df = self.df.drop(self.df[self.df.votes < self._minVotes].index)

    # Method to calculate the weighted average score for each title
    def weightedAvgScore(self):
        weightedAvg = []
        mean = self.df[self.colTitle['rating']].mean()
        self.df = self.df.reset_index(drop=True)

        for i in range(0, len(self.df[self.colTitle['rating']])):
            res = (self.df[self.colTitle['votes']][i]/(self.df[self.colTitle['votes']][i] + self._minVotes) *
                   self.df[self.colTitle['rating']][i]) + (self._minVotes/(self.df[self.colTitle['votes']][i]+self._minVotes))*mean
            weightedAvg.append(res)
        self.df["WeightedAvg"] = weightedAvg

    # Method to sort the titles based on the weighted average score and display top N titles
    def sortNscores(self):
        sort = self.df.sort_values(
            "WeightedAvg", ascending=False).head(self._topN)
        display(sort)

    # Method to execute all the steps of the recommender system
    def run(self):
        self.readFile()
        self.cleanDataset()
        self.filterGenre()
        self.threshVotes()
        self.weightedAvgScore()
        self.sortNscores()

path = "/home/hanzala/Development/Python_the_ultimate_course_2023/26. Portfolio Project Netflix Recommender Systems Introduction/NetflixDatasetMovies.csv"
best5Comedy = netflixTopRecommenderSystem(path, "Comedy", 0.8, 5)
best5Comedy.colDrop = ['year', 'certificate',
                       'duration', 'description', 'stars']
best5Comedy.colTitle = {"votes": "votes", "genre": "genre",
                        "title": "title", "rating": "rating"}
best5Comedy.run()

26. Portfolio Project Netflix Recommender Systems Introduction/Project.py


def solution(k):
    global curr

    if k == m:
        for i in range(m):
            print(temp[i], end=" ")
        print()
        return

    for i in range(orders[curr], n):
        num = numbers[i]
        temp[k] = num
        curr = num
        solution(k + 1)

n, m = map(int, input().split())
temp = [0] * (n + 1)
numbers = sorted(list(map(int, input().split())))

orders = {}
for i in range(len(numbers)):
    orders[numbers[i]] = i

curr = numbers[0]
solution(0)


from django import forms
from catalog.models import Products, Version

class MixinStyle:

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        for field_name, field in self.fields.items():
            field.widget.attrs['class'] = 'form-control'

class ProductsForms(MixinStyle, forms.ModelForm):
    class Meta:
        model = Products
        exclude = ('product_date_create', 'product_date_edit')

    def clean_product_name(self):
        cleaned_name = self.cleaned_data['product_name']
        forbidden_words = ["казино", "криптовалюта", "крипта", "биржа", "дешево", "бесплатно", "обман", "полиция",
                           "радар"]
        for word in forbidden_words:
            if word in cleaned_name:
                raise forms.ValidationError(f'Запрещено истольвозвать слово {word}.')
            if word.title() in cleaned_name:
                raise forms.ValidationError(f'Запрещено истольвозвать слово {word}.')
        return cleaned_name

    def clean_product_text(self):
        cleaned_text = self.cleaned_data['product_text']
        forbidden_words = ["казино", "криптовалюта", "крипта", "биржа", "дешево", "бесплатно", "обман", "полиция",
                           "радар"]
        for word in forbidden_words:
            if word.title() in cleaned_text:
                raise forms.ValidationError(f'Запрещено истольвозвать слово {word}.')
            if word in cleaned_text:
                raise forms.ValidationError(f'Запрещено истольвозвать слово {word}.')
        return cleaned_text

class VersionForms(MixinStyle, forms.ModelForm):
    class Meta:
        model = Version
        fields = '__all__'


import rospy
import warnings
import message_filters
from sensor_msgs.msg import Image, PointCloud2, CameraInfo
from geometry_msgs.msg import Transform
from std_msgs.msg import Bool
import tf2_ros

from benchmark_grasping.base.transformations import quaternion_to_matrix, matrix_to_quaternion

from benchmark_grasping_ros.srv import *
from benchmark_grasping_ros.msg import BenchmarkGrasp

from robot_grasp_srv.srv import RobotGrasp, RobotGraspRequest, RobotGraspResponse

import numpy as np

NEW_MSG = {
"new_data": False,
"data": {},
}

# TODO: find a way to define the grasp service depending on the type of grasp planner
GRASP_PLANNER_SRV = {
    'GraspPlanner': GraspPlanner,
    'GraspPlannerCloud': GraspPlannerCloud,
    }

class GraspingBenchmarksManager(object):
    def __init__(self, grasp_planner_service_name, grasp_planner_service, user_cmd_service_name, robot_service_name, verbose=False):

        self._verbose = verbose

        # --- new grasp command service --- #
        self._new_grasp_srv = rospy.Service(user_cmd_service_name, UserCmd, self.user_cmd)

        # --- grasp planner service --- #
        self._grasp_planner_srv = GRASP_PLANNER_SRV[grasp_planner_service]

        rospy.loginfo("GraspingBenchmarksManager: Waiting for grasp planner service...")
        rospy.wait_for_service(grasp_planner_service_name, timeout=30.0)
        self._grasp_planner = rospy.ServiceProxy(grasp_planner_service_name, self._grasp_planner_srv)
        rospy.loginfo("...Connected with service {}".format(grasp_planner_service_name))

        # --- robot service --- #
        robot_service_name =  "/robot_grasp"
        rospy.loginfo("GraspingBenchmarksManager: Waiting for robot control service...")
        rospy.wait_for_service(robot_service_name, timeout=60.0)
        self._robot = rospy.ServiceProxy(robot_service_name, RobotGrasp)
        rospy.loginfo("...Connected with service {}".format(robot_service_name))

        # --- subscribers to camera topics --- #
        self._cam_info_sub = message_filters.Subscriber('/camera/aligned_depth_to_color/camera_info', CameraInfo)
        self._rgb_sub = message_filters.Subscriber('/camera/color/image_raw', Image)
        self._depth_sub = message_filters.Subscriber('/camera/aligned_depth_to_color/image_raw', Image)
        self._pc_sub = message_filters.Subscriber('/camera/depth_registered/points', PointCloud2)
        self._seg_sub = rospy.Subscriber('rgb/image_seg', Image, self.seg_img_callback, queue_size=10)

        # --- camera data synchronizer --- #
        self._tss = message_filters.ApproximateTimeSynchronizer([self._cam_info_sub, self._rgb_sub, self._depth_sub, self._pc_sub],
                                                                queue_size=1, slop=0.5)
        self._tss.registerCallback(self._camera_data_callback)

        # --- robot/camera transform listener --- #
        self._tfBuffer = tf2_ros.buffer.Buffer()
        listener = tf2_ros.transform_listener.TransformListener(self._tfBuffer)

        # --- camera messages --- #
        self._cam_info_msg = None
        self._rgb_msg = None
        self._depth_msg = None
        self._pc_msg = None
        self._camera_pose = geometry_msgs.msg.Transform()

        self._seg_msg = NEW_MSG

        self._new_camera_data = False
        self._abort = False

    # ---------------------- #
    # Grasp planning handler #
    # ---------------------- #
    def user_cmd(self, req):
        """New grasp request handler

        Parameters
        ---------
        req: :obj:`ROS ServiceRequest`
            ROS `ServiceRequest` for grasp planner service.
        """
        if self._verbose:
            print("Received new command from user...")

        available_commands = "help : display available commands\n \
                              grasp: compute a new grasp and send it to the robot for execution\n \
                              abort: interrupt grasp computation / do not send computed pose to the robot"

        self._abort = False

        if req.cmd.data == "help":
            print("Available commands are:\n", available_commands)
            return Bool(True)

        elif req.cmd.data == "grasp":

            # --- get images --- #
            if self._verbose:
                print("... waiting for images ...")

            count = 0
            while not self._new_camera_data and count < 1000:
                count += 1

            if count >= 1000:
                print("...no images received")
                return Bool(False)

            self._new_camera_data = False

            # --- create srv request --- #
            # GraspPlanner
            if self._grasp_planner_srv is GraspPlanner:

                planner_req = GraspPlannerRequest()

                planner_req.color_image = self._rgb_msg
                planner_req.depth_image = self._depth_msg
                planner_req.camera_info = self._cam_info_msg

            # or GraspPlannerCloud
            elif self._grasp_planner_srv is GraspPlannerCloud:

                planner_req = GraspPlannerCloudRequest()

                # define cloud
                planner_req.cloud = self._pc_msg

                camera_pose_msg = geometry_msgs.msg.Pose()

                camera_pose_msg.position.x = self._camera_pose.translation.x
                camera_pose_msg.position.y = self._camera_pose.translation.y
                camera_pose_msg.position.z = self._camera_pose.translation.z
                camera_pose_msg.orientation = self._camera_pose.rotation

                planner_req.view_point = camera_pose_msg

            if self._verbose:
                print("... send request to server ...")

            if self._abort:
                rospy.loginfo("grasp computation was aborted by the user")
                return Bool(False)

            try:
                reply = self._grasp_planner(planner_req)

                print("Service {} reply is: \n{}" .format(self._grasp_planner.resolved_name, reply))

            except rospy.ServiceException as e:
                print("Service {} call failed: {}" .format(self._grasp_planner.resolved_name, e))

                return Bool(False)

            if self._abort:
                rospy.loginfo("grasp execution was aborted by the user")
                return Bool(False)

            return self.execute_grasp(reply.grasp, self._camera_pose)

        elif req.cmd.data == "abort":
            self._abort = True
            return Bool(True)

    def execute_grasp(self, grasp, cam_pose):
        """
        Parameters:
            - grasp (obj: BenchmarkGrasp msg)
            - cam_pose (obj: geometry_msgs.msg.Transform)
        """
        # Need to tranform the grasp pose from camera frame to world frame
        # w_T_grasp = w_T_cam * cam_T_grasp
        gp_quat = grasp.pose.pose.orientation
        gp_pose = grasp.pose.pose.position
        cam_T_grasp = np.eye(4)
        cam_T_grasp[:3,:3] = quaternion_to_matrix([gp_quat.x, gp_quat.y, gp_quat.z, gp_quat.w])
        cam_T_grasp[:3,3] = np.array([gp_pose.x, gp_pose.y, gp_pose.z])

        cam_quat = cam_pose.rotation
        cam_pose = cam_pose.translation
        w_T_cam = np.eye(4)
        w_T_cam[:3, :3] = quaternion_to_matrix([cam_quat.x, cam_quat.y, cam_quat.z, cam_quat.w])
        w_T_cam[:3, 3] = np.array([cam_pose.x, cam_pose.y, cam_pose.z])

        w_T_grasp = np.matmul(w_T_cam, cam_T_grasp)
        print("w_T_cam\n ", w_T_cam)
        print("cam_T_grasp\n ", cam_T_grasp)
        print("w_T_grasp\n ", w_T_grasp)

        # Create the ROS pose message to send to robot
        grasp_pose_msg = geometry_msgs.msg.PoseStamped()

        grasp_pose_msg.header.frame_id = 'robot_link0'

        grasp_pose_msg.pose.position.x = w_T_grasp[0,3]
        grasp_pose_msg.pose.position.y = w_T_grasp[1,3]
        grasp_pose_msg.pose.position.z = w_T_grasp[2,3]

        q = matrix_to_quaternion(w_T_grasp[:3,:3])
        grasp_pose_msg.pose.orientation.x = q[0]
        grasp_pose_msg.pose.orientation.y = q[1]
        grasp_pose_msg.pose.orientation.z = q[2]
        grasp_pose_msg.pose.orientation.w = q[3]

        robot_req = RobotGraspRequest()
        robot_req.grasp = grasp_pose_msg
        robot_req.width = grasp.width

        print("request to robot is: \n{}" .format(robot_req))

        try:
            reply = self._robot(robot_req)

            print("Service {} reply is: \n{}" .format(self._robot.resolved_name, reply))

            return Bool(True)

        except rospy.ServiceException as e:
            print("Service {} call failed: {}" .format(self._robot.resolved_name, e))

            return Bool(False)

    # ------------------- #
    # Camera data handler #
    # ------------------- #
    def _camera_data_callback(self, cam_info, rgb, depth, pc):
        # rospy.loginfo("New data from camera!")
        self._cam_info_msg = cam_info
        self._rgb_msg = rgb
        self._depth_msg = depth
        self._pc_msg = pc

        self._new_camera_data = True

        # define camera view point
        try:
            camera_pose_tf = self._tfBuffer.lookup_transform( 'panda_link0', self._cam_info_msg.header.frame_id, rospy.Time())
            self._camera_pose = camera_pose_tf.transform

        except (tf2_ros.LookupException, tf2_ros.ConnectivityException, tf2_ros.ExtrapolationException):
            warnings.warn("tf listener could not get camera pose. Are you publishing camera poses on tf?")

            self._camera_pose = geometry_msgs.msg.Transform()
            self._camera_pose.rotation.w = 1.0

    def seg_img_callback(self, data):
        if self._verbose:
            print("Got segmentation image...")

        self._seg_msg['data'] = data
        self._seg_msg['new_data'] = True

if __name__ == "__main__":
    # Init node
    rospy.init_node('grasping_benchmarks_manager')

    # Get rosparam config
    grasp_planner_service_name = rospy.get_param("~grasp_planner_service_name")
    grasp_planner_service = rospy.get_param("~grasp_planner_service")
    new_grasp_service_name = rospy.get_param("~user_cmd_service_name")
    robot_service_name = "/robot_grasp" # rospy.get_param("/robot_service_name")

    # Instantiate benchmark client class
    bench_manager = GraspingBenchmarksManager(grasp_planner_service_name, grasp_planner_service, new_grasp_service_name, robot_service_name, verbose=True)

    # Spin forever.
    rospy.spin()


import rclpy
from rclpy.node import Node
from std_msgs.msg import String
import socket
import threading
import json

class State:
    def __init__(self) -> None:
        self.pose = None
        self.gripper = None

class UrxNodePub(Node):
    def __init__(self, state):
        super().__init__('socket_robot')
        self.state = state
        self.robot_control_pub = self.create_publisher(String, 'robot_control', 1)
        self.robot_status = self.create_subscription(String, 'urx_status', self.robot_data_callback, 1)

    def robot_data_callback(self, data):
        payload = json.loads(data.data)
        self.state.pose = payload["position"]
        self.state.gripper = payload["gripper"]

    def send_command(self, data: str):
        send_data = String()
        send_data.data = data
        self.robot_control_pub.publish(send_data)

class SocketServer:
    def __init__(self, robot_control_pub, state, host: str = "0.0.0.0", port: int = 6666) -> None:
        self.robot_control_pub = robot_control_pub
        self.host = host
        self.port = port
        self.state = state

        self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server_socket.bind((self.host, self.port))
        self.server_socket.listen(5)

    def handle_client(self, client_socket):
        while True:
            try:
                data = client_socket.recv(1024)
                if data:
                    try:
                        data = json.loads(data)
                        if data["type"] == "movel":
                            self.robot_control_pub.send_command(json.dumps({
                                "type": "movel",
                                "data": data["pose"]
                            }))
                        elif data["type"] == "getl":
                            client_socket.sendall(json.dumps({"pose": self.state.pose}).encode("utf-8"))

                        elif data["type"] == "gripper_pose":
                            client_socket.sendall(json.dumps({"pose": self.state.gripper}).encode("utf-8"))

                        elif data["type"] == "gripper_action":
                            self.robot_control_pub.send_command(json.dumps({
                                "type": "gripper",
                                "data": data["pose"]
                            }))
                    except json.decoder.JSONDecodeError:
                        pass

            except ConnectionResetError: # If client disconnect
                client_socket.close()
                break

    def clients_joiner(self):
        while True:
            client_socket, addr = self.server_socket.accept()
            threading.Thread(target=lambda: self.handle_client(client_socket)).start()

def main(args=None):
    rclpy.init(args=args)

    state = State()
    robot_control_pub = UrxNodePub(state)
    socket_server = SocketServer(robot_control_pub, state)
    threading.Thread(target=socket_server.clients_joiner).start()

    rclpy.spin(robot_control_pub)
    robot_control_pub.destroy_node()
    rclpy.shutdown()

if __name__ == "__main__":
    main()


#!/usr/bin/python

import sqlite3
import os

def make_rsem_db(db_fname, samples_dir, filepath):
    conn = sqlite3.connect(db_fname)
    cur = conn.cursor()

    cur.execute(
        '''
        CREATE TABLE stringtie_rsem (
        sample TEXT NOT NULL,
        transcript_id TEXT NOT NULL,
        gene_id TEXT NOT NULL,
        length INT NOT NULL,
        eff_length REAL NOT NULL,
        expected_count REAL NOT NULL,
        tpm REAL NOT NULL,
        fpkm REAL NOT NULL,
        isopct REAL NOT NULL
        )
        '''
    )

    for sample in os.listdir(samples_dir):
        fname = os.path.join(samples_dir, sample, filepath)
        data = []
        with open(fname, 'r') as f:
            num_cols = len(f.readline().strip().split())+1
            for line in f:
                row = line.strip().split()
                row[2] = int(row[2])
                row[3:] = map(float, row[3:])
                row = [sample] + row
                data.append(row)
        cur.executemany(
        '''
        INSERT INTO stringtie_rsem VALUES ({})
        '''.format(",".join(["?"]*num_cols)),
        data
        )

    conn.commit()

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("db_fname")
    parser.add_argument("samples_dir")
    parser.add_argument("filepath")
    args = parser.parse_args()

    make_rsem_db(args.db_fname, args.samples_dir, args.filepath)