# -*- coding: utf-8 -*-
"""
@Date: Created on 2019/9/7
@Author: Haojun Gao
@Description:
"""

import os
import csv
import json
import numpy as np
import pandas as pd
from paras import load_init_params

def create_enti_feat_matrix(sentences, keywords_set, word_index, feature_set):
    """
    创建共现次数矩阵
    :param sentences: 数据集
    :param word_index:
    :param feature_set: 特征集合
    :return:
        coocurrence_matrix [np] 共现次数矩阵
        word_index [dictionary] 记录词的 index
        index_dict [dictionary] 记录节点 index 对应的词
    """
    enti_envi_matrix = np.zeros([len(word_index), len(feature_set)])

    feat_word_index = {}  # 记录特征词的index
    feat_index_dict = {}  # 记录特征节点index对应的词

    for i, v in enumerate(feature_set):
        feat_word_index[v] = i
        feat_index_dict[i] = v

    for sentence in sentences:
        word_list = sentence.split("/")
        intersec_enti = keywords_set.intersection(set(word_list))
        intersec_feat = feature_set.intersection(set(word_list))
        if len(intersec_enti) >= 1 and len(intersec_feat) >= 1:
            for key_1 in intersec_enti:
                for key_2 in intersec_feat:
                    enti_envi_matrix[word_index[key_1]][feat_word_index[key_2]] += 1

    return enti_envi_matrix

def KL_Divergence(text_environment_order_fre, event_environment):
    """
    计算两个分布之间的KL散度值
    :param text_environment_order_fre:
    :param event_environment:
    :return:
    """

    if len(text_environment_order_fre) == len(event_environment):
        # np.core.defchararray.strip(word_environment[:,1], '()').astype(int)解释数据，将‘22’，转化为int型数据
        fre_word = text_environment_order_fre
        fre_event = np.array(event_environment)
        nil = 1 / len(text_environment_order_fre)
        P = (fre_word / np.sum(fre_word)).reshape(len(text_environment_order_fre), 1)
        Q = (fre_event / np.sum(fre_event)).reshape(len(text_environment_order_fre), 1)
        KL_1 = np.sum(P * np.log((P + nil) / (Q + nil)))
        KL_2 = np.sum(Q * np.log((Q + nil) / (P + nil)))
        return KL_1, KL_2

    else:
        return "离散分布不能对应"

def create_kl_matrix(keywords_set, enti_index_dict, enti_feat_matrix):
    """
    计算实体之间特征词分布的KL散度矩阵
    :param keywords_set: 实体集合
    :param feature_set: 特征集合
    :param enti_feat_matrix: 实体-特征矩阵
    :return:
        mi_matrix [np] 实体互信息矩阵
        mi_matrix_norm [np] 实体互信息矩阵（标准化后）
    """

    kl_value = {}
    num = len(keywords_set)
    kl_matrix = np.zeros([num, num])
    for i in range(num):
        print("[处理进度] {} / {}".format(i + 1, num))
        for j in range(num - i - 1):
            k = i + j + 1
            kl_1, kl_2 = KL_Divergence(enti_feat_matrix[i], enti_feat_matrix[k])
            print("[KL散度矩阵] {} & {} : {} {}".format(enti_index_dict[i], enti_index_dict[k], kl_1, kl_2))
            kl_matrix[i][k] = kl_1
            kl_matrix[k][i] = kl_2
            kl_value[str((enti_index_dict[i], enti_index_dict[k]))] = str(kl_1) + str(kl_2)

    kl_value_sort = sorted(kl_value.items(), key=lambda x: x[1])

    return kl_matrix, kl_value, kl_value_sort


from pymongo import MongoClient
import pymongo
import scrapy
import datetime
import subprocess
import re
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dense
from keras.models import model_from_json
import numpy
import pandas
import time
from keras.layers import Masking

# to achiveve program requirement 5.1
def collectionDroppingandEmptyCollectionCreating(db):
    try:
        print("Dropping collection...")
        # Drop collection "course" if exist
        db.courses.drop()
        # Create empty collection "course"
        db.create_collection("courses")
        print("Collection dropping and empty collection creating are successful.")
    except pymongo.errors.ConnectionFailure as error:
        print("Collection Dropping and Collection Creating Failed! Error Message: \"{}\"".format(error))

# to achiveve program requirement 5.2
def dataCrawling(db):
    try:
        # address = input("Please enter the URL of the website you want to crawl: ")
        url = input('Please enter a URL for data crawling: ')
        if (url == "default"):
            url = 'http://comp4332.com/realistic/'
            with open('url.txt', 'w') as f:
                f.write(url)
            strCommand = "scrapy crawl mongo"
            subprocess.run(strCommand, shell=True)
            print("Data are crawled from project webpage.")
            print("Data Crawling is successful and all data are inserted into the database.")
        else:
            with open('url.txt', 'w') as f:
                f.write(url)
            strCommand = "scrapy crawl mongo"
            subprocess.run(strCommand, shell=True)
            print("Data are crawled from inputted URL.")
            print("Data Crawling is successful and all data are inserted into the database.")

    except pymongo.errors.ConnectionFailure as error:
        print("Course Insertion Failed! Error Message: \"{}\"".format(error))

# to achiveve program requirement 5.3.1
def courseSeachbyKeyword(db):
    keyword = input("Please enter the keyword(s) of the course you want to search: ")
    print("Searching...")
    regex_text = "\\b(" + "|".join(keyword.split()) + ")\\b"
    REGEX = re.compile(regex_text, re.IGNORECASE)
    listOfCourse = db.courses.aggregate(
        [
            {"$match": {'$or': [{'title': {'$regex': REGEX}}, {'description':{'$regex': REGEX}},{'remarks': {'$regex': REGEX}}]}},
            {"$project": {"code":1, "title":1, "credits":1,"sections":1, "_id": 0}},
            {"$sort":{"code":1}}

        ],allowDiskUse=True )

    # print(list(listOfCourse))
    for oneCourse in listOfCourse:
        code = str(oneCourse["code"])
        title = str(oneCourse["title"])
        credits = str(oneCourse["credits"])
        sectionstime = oneCourse["sections"][0]["recordTime"]
        print(code,title,credits)
        # result1 = db.courses.aggregate([
        #     {"$unwind": "$sections"},
        #     {"$project": {"sections": 1}},
        #     {"$match": {"title":title}},
        #     {"$project": {"sections": 1}},
        #     {"$group":{"_id": "$sectionId"}},
        #     {"$project": {"sections":1}}
        # ], allowDiskUse=True)

        result1 = db.courses.aggregate([
            {"$unwind": "$sections"},
            {"$match": {"$and": [{"sections.recordTime": sectionstime},
                                 {"code": code}]}},
            {"$project": {"sections": 1}},
            {"$sort": {"sections.sectionId": 1}}
        ], allowDiskUse=True)
        for suboneCourse in result1:
            sections1 = suboneCourse["sections"]
            sid = sections1["sectionId"]
            date_and_time = sections1["offerings"][0]["dateAndTime"]
            quota = str(sections1["quota"])
            enrol = str(sections1["enrol"])
            wait = str(sections1["wait"])
            avail=str(sections1["avail"])
            print(sid,date_and_time,quota,enrol,wait,avail)

    recordNumber=0
    # for oneCourse in listOfCourse:
    #     recordNumber = recordNumber + 1
    #     tempCourseCode = oneCourse["CourseCode"]
    #     tempCourseName = oneCourse["CourseName"]
    #     tempNumberofCredit = oneCourse["NumberofCredit"]
    #     tempSection = oneCourse["SectionID"]
    #     tempDateAndTime = oneCourse["DateAndTime"]
    #     tempQuota = oneCourse["Quota"]
    #     tempEnrol = oneCourse["Enrol"]
    #     tempWait = oneCourse["Wait"]
    #     tempAvail = oneCourse["Avail"]
    #
    #     print(tempCourseCode, tempCourseName, tempNumberofCredit, tempSection, tempDateAndTime, tempQuota, tempEnrol,
    #           tempWait, tempAvail)

# to achiveve program requirement 5.3.2
def courseSeachbyWaitingListSize(db):
    f = input(
        "Please enter the multiple of the waiting list size with enrolled student which is a non-negative number: ")
    while (float(f) < 0):
        f = input("The number is invalid. Please input again: ")

    start_ts = input("Please enter the starting time slot: ")
    end_ts = input("Please enter the ending time slot: ")
    # startTimeSlot = datetime.datetime.strptime(start_ts, "%Y-%m-%dT%H:%M:%S")
    # endTimeSlot = datetime.datetime.strptime(end_ts, "%Y-%m-%dT%H:%M:%S")
    start_ts = datetime.datetime.strptime(start_ts, "%Y-%m-%dT%H:%M:%S")
    end_ts = datetime.datetime.strptime(end_ts, "%Y-%m-%dT%H:%M:%S")
    print("Searching...")
    # listOfCourse = db.courses.aggregate(
    #     [
    #         {"$unwind": "$sections"},
    #         {"$project": {
    #             "comparedTimeSlotResult1": {"$gte": ["sections.recordTime", start_ts]},
    #             "comparedTimeSlotResult2": {"$lte": ["sections.recordTime", end_ts]},
    #             "comparedWaitListSizeResult": {"$gte": ["$sections.wait", {"$multiply": ["$sections.enrol", float(f)]}]}
    #                       }},
    #         {"$match": {"$and": [{"comparedTimeSlotResult1": True}, {"comparedTimeSlotResult2": True},{"comparedWaitListSizeResult":True}]}},
    #         {"$project": {"CourseCode": "$code","TimeSlot": "$recordTime", "Section": "$sections.sectionId"
    #                       }},
    #         {"$sort": {"TimeSlot":1,"CourseCode": 1, "Section": 1}},
    #         {"$project": {"CourseCode": 1, "TimeSlot": 1, "Section": 1}},
    #         {"$group": {"_id": "$sections.sectionId","MatchedTimeSlot": {"$last": "$recordTime"}}},
    #         {"$project": {"CourseCode": "$code", "CourseName": "$title", "NumberofCredit": "$credits",
    #                       "TimeSlot": "$MatchedTimeSlot", "Code": "$_id",
    #                       "DateAndTime": "$sections.offerings.dateAndTime", "Quota": "$sections.quota",
    #                       "Enrol": "$sections.enrol", "Wait": "$sections.wait","Satisfied":"Yes"}}
    #     ], allowDiskUse=True
    # )
    # print(list(listOfCourse))
    REGEX = re.compile('L\d+', re.IGNORECASE)
    result = db.courses.aggregate([
        {"$unwind": "$sections"},
        {"$project": {"fxenrol": {"$multiply": ["$sections.enrol", float(f)]}, "code": 1, "title": 1, "credits": 1,
                      "sections": 1}},
        {"$project": {"compareResult": {"$gte": ["$sections.wait", "$fxenrol"]}, "code": 1, "title": 1, "credits": 1,
                      "sections": 1}},
        {"$match": {"$and": [{"compareResult": True}, {"sections.recordTime": {"$gte": start_ts}},
                             {"sections.recordTime": {"$lte": end_ts}}]}},
        {"$project": {"code": 1, "title": 1, "credits": 1, "sections": 1, "_id": 0, "compareResult": 1}},
        {"$sort": {"sections.sectionId": 1}},
        {"$group": {"_id": {"code": "$code", "title": "$title", "credits": "$credits"},
                    "match_ts": {"$max": "$sections.recordTime"}, "sections": {
                "$push": {"recordTime": "$sections.recordTime", "sectionId": "$sections.sectionId",
                          "offerings": "$sections.offerings", "quota": "$sections.quota", "enrol": "$sections.enrol",
                          "wait": "$sections.wait", "avail": "$sections.avail", "Satisfied": "$compareResult"}}}},
        {"$unwind": "$sections"},
        # // delete sections that not in the lasest time slot
        {"$project": {"compareResult": {"$eq": ["$sections.recordTime", "$match_ts"]}, "sections": 1, "match_ts": 1}},
        {"$match": {"compareResult": True}},
        {"$match": {"sections.sectionId": {'$regex': REGEX}}},
        {"$group": {"_id": "$_id", "match_ts": {"$max": "$sections.recordTime"}, "sections": {
            "$push": {"recordTime": "$sections.recordTime", "sectionId": "$sections.sectionId",
                      "offerings": "$sections.offerings", "quota": "$sections.quota", "enrol": "$sections.enrol",
                      "wait": "$sections.wait", "avail": "$sections.avail", "Satisfied": "$sections.Satisfied"}}}},
        {"$project": {"code": "$_id.code", "title": "$_id.title", "credits": "$_id.credits", "sections": 1,
                      "match_ts": 1}},
        {"$sort": {"code": 1}}
        ], allowDiskUse=True)

    # a = 0
    for oneCourse in result:
        code = str(oneCourse["code"])
        title = str(oneCourse["title"])
        credits = str(oneCourse["credits"])
        sections = oneCourse["sections"]
        match_ts = str(oneCourse["match_ts"])
        print(code,title,credits,match_ts)

        result1 = db.courses.aggregate([
            {"$unwind": "$sections"},
            {"$match": {"$and": [{"sections.recordTime": datetime.datetime.strptime(match_ts, '%Y-%m-%d %H:%M:%S')},
                                 {"code": code}]}},
            {"$project": {"fxenrol": {"$multiply": ["$sections.enrol", float(f)]}, "code": 1, "title": 1, "credits": 1,
                          "sections": 1}},
            {"$project": {"Satisfied": {"$gte": ["$sections.wait", "$fxenrol"]}, "code": 1, "title": 1, "credits": 1,
                          "sections": 1}},
            {"$sort": {"code": 1, "sections.sectionId": 1}}
            ], allowDiskUse=True)
        for suboneCourse in result1:
            sections1 = suboneCourse["sections"]
            satisfied = str(suboneCourse["Satisfied"])
            sid = sections1["sectionId"]
            date_and_time = sections1["offerings"][0]["dateAndTime"]
            quota = str(sections1["quota"])
            enrol = str(sections1["enrol"])
            wait = str(sections1["wait"])
            avail = str(sections1["avail"])
            print(sid,date_and_time,quota,enrol,wait, avail,satisfied)

# to achiveve program requirement 5.4
def waitingListSizePrediction(db):
    cc = input("Please enter the course code: ")
    ln = input("Please enter the lecture number (ln): ")
    ts = input("Please enter the time slot: ")

    matched = db.courses.aggregate({'$project': {"comparedTimeSlotResult1": {'$eq': ["$sections.class_name", ln]}}})
    predictedResult = []
    if (matched):
        if (cc == "COMP1942"):
            dataframe1 = pandas.read_csv("COMP1942Result_1", usecols=[1])
            dataframe2 = pandas.read_csv("COMP1942Result_2", usecols=[1])
            dataframe3 = pandas.read_csv("COMP1942Result_3", usecols=[1])
            dataframe4 = pandas.read_csv("COMP1942Result_4", usecols=[1])
            dataframe5 = pandas.read_csv("COMP1942Result_5", usecols=[1])
            if (dataframe1 == ts):
                result = pandas.read_csv("COMP1942Result_1", usecols=[2])
                predictedResult[1] = result
            if (dataframe2 == ts):
                result = pandas.read_csv("COMP1942Result_2", usecols=[2])
                predictedResult[2] = result
            if (dataframe3 == ts):
                result = pandas.read_csv("COMP1942Result_3", usecols=[2])
                predictedResult[3] = result
            if (dataframe4 == ts):
                result = pandas.read_csv("COMP1942Result_4", usecols=[2])
                predictedResult[4] = result
            if (dataframe5 == ts):
                result = pandas.read_csv("COMP1942Result_5", usecols=[2])
                predictedResult[5] = result
            for i in range(predictedResult):
                print(predictedResult[i])
                if (i == range(predictedResult)):
                    break
                else:
                    print(",")
    else:
        print("There is no lecture section and thus there is no prediction result.")

# to achiveve program requirement 5.5
def waitingListSizeTraining(db,ln):
    # Write all the attributes to csv
    listOfWaitingList = db.courses.aggregate(
        [
            # {'$match': {{"code": ^COMP42, "class_name": ln}}},
            # {'$match': {{"code": ^COMP43, "class_name": ln}}},
            # {'$match': {{"code": ^RMBI, "class_name": ln}}},
            {'$match': {{"code": "COMP1942", "class_name": ln}}},
            {'$project': {"TimeSlot": "$recordTime", "Enrol": "$sections.enrol", "Wait": "$sections.wait", "_id": 0}},
            {'$sort': {"TimeSlot": 1}}
        ]
    )
    recordNumber = 0

    for oneCourse in listOfWaitingList:
        recordNumber = recordNumber + 1
        listOfWaitingList["Timeslot"] = oneCourse["Timeslot"]
        listOfWaitingList["Enrol"] = oneCourse["Enrol"]
        listOfWaitingList["Wait"] = oneCourse["Wait"]

    filename = "COMP1942Training_Timestamp.csv"
    csv = open(filename, "w")
    recordNumber = 0
    for recordNumber in listOfWaitingList:
        recordNumber = recordNumber + 1
        row = listOfWaitingList["Timeslot"] + "," + listOfWaitingList["Enrol"] + "," + listOfWaitingList["Wait"] + "\n"
        csv.write(row)

    filename2 = "COMP1942Training_Number.csv"
    csv = open(filename2, "w")
    recordNumber = 0
    for recordNumber in listOfWaitingList:
        row = recordNumber + "," + listOfWaitingList["Enrol"] + "," + listOfWaitingList["Wait"] + "\n"
        csv.write(row)

    # Model 1: Neural Network (Parameter set A)
    # Train the csv
    dataframe = pandas.read_csv(filename, usecols=[1])
    data_int_TwoDim = dataframe.values
    data_float_TwoDim = data_int_TwoDim.astype(float)

    look_back = 3
    dataX, dataY = [], []
    for i in range(len(data_float_TwoDim) - look_back):
        a = data_float_TwoDim[i:(i + look_back), 0]
        dataX.append(a)
        dataY.append(data_float_TwoDim[i + look_back, 0])
    numpyX = numpy.array(dataX)
    numpyY = numpy.array(dataY)

    # Generate random seed
    numpy.random.seed(time.time())

    # Define the 1st model
    model = Sequential()
    model.add(Dense(20, input_dim=look_back, activation='relu'))
    model.add(Dense(10, activation='relu'))
    model.add(Dense(1, activation='sigmoid'))

    # Compile the 1st model
    model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])

    # Fit the 1st model
    model.fit(numpyX, numpyY, epochs=2000, batch_size=20)

    # Evaluate the 1st model
    scores = model.evaluate(numpyX,numpyY)

    # Predict the 1st set of result using 1st model
    newY_TwoDim = model.predict(numpyX, batch_size=1)

    # Save the 1st set of predicted target attribute of the new data into a file
    numpy.savetxt("COMP1942Result_1", newY_TwoDim, delimiter=",", fmt="%.10f")

    print("Waiting list size training is successful")

    # Model 2: Neural Network (Parameter set B)
    # Train the csv
    dataframe = pandas.read_csv(filename, usecols=[1])
    data_int_TwoDim = dataframe.values
    data_float_TwoDim = data_int_TwoDim.astype(float)

    look_back = 3
    dataX, dataY = [], []
    for i in range(len(data_float_TwoDim) - look_back):
        a = data_float_TwoDim[i:(i + look_back), 0]
        dataX.append(a)
        dataY.append(data_float_TwoDim[i + look_back, 0])
    numpyX = numpy.array(dataX)
    numpyY = numpy.array(dataY)

    # Generate random seed
    numpy.random.seed(time.time())

    # Define the 2nd model
    model = Sequential()
    model.add(Dense(30, input_dim=look_back, activation='relu'))
    model.add(Dense(20, activation='relu'))
    model.add(Dense(10, activation='relu'))
    model.add(Dense(1, activation='sigmoid'))

    # Compile the 2nd model
    model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])

    # Fit the 2nd model
    model.fit(numpyX, numpyY, epochs=2000, batch_size=20)

    # Evaluate the 2nd model
    scores = model.evaluate(numpyX,numpyY)

    # Predict the 2nd set of result using 2nd model
    newY_TwoDim = model.predict(numpyX, batch_size=1)

    # Save the 2nd set of predicted target attribute of the new data into a file
    numpy.savetxt("COMP1942Result_2", newY_TwoDim, delimiter=",", fmt="%.10f")

    print("Waiting list size training is successful")

    # Model 3: Neural Network (Parameter set C)
    # Train the csv
    dataframe = pandas.read_csv(filename, usecols=[1])
    data_int_TwoDim = dataframe.values
    data_float_TwoDim = data_int_TwoDim.astype(float)

    look_back = 4
    dataX, dataY = [], []
    for i in range(len(data_float_TwoDim) - look_back):
        a = data_float_TwoDim[i:(i + look_back), 0]
        dataX.append(a)
        dataY.append(data_float_TwoDim[i + look_back, 0])
    numpyX = numpy.array(dataX)
    numpyY = numpy.array(dataY)

    # Generate random seed
    numpy.random.seed(time.time())

    # Define the 3rd model
    model.add(Dense(30, input_dim=look_back, activation='relu'))
    model.add(Dense(20, activation='relu'))
    model.add(Dense(10, activation='relu'))
    model.add(Dense(1, activation='sigmoid'))

    # Compile the 3rd model
    model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])

    # Fit the 3rd model
    model.fit(numpyX, numpyY, epochs=1500, batch_size=4)

    # Evaluate the 3rd model
    scores = model.evaluate(numpyX,numpyY)

    # Predict the 3rd set of result using 3rd model
    newY_TwoDim = model.predict(numpyX, batch_size=1)

    # Save the 3rd set of predicted target attribute of the new data into a file
    numpy.savetxt("COMP1942Result_3", newY_TwoDim, delimiter=",", fmt="%.10f")

    print("Waiting list size training is successful")

    # Model 4: LSTM (Parameter set A)
    # Train the csv
    dataframe = pandas.read_csv(filename, usecols=[1])
    data_int_TwoDim = dataframe.values
    data_float_TwoDim = data_int_TwoDim.astype(float)

    dataX, dataY = [], []
    for i in range(len(data_float_TwoDim)):
        a = data_float_TwoDim[i:(i + look_back), 0]
        dataX.append(a)
        dataY.append(data_float_TwoDim[i, 0])
    numpyX = numpy.array(dataX)
    numpyY = numpy.array(dataY)

    # Generate random seed
    numpy.random.seed(time.time())

    # Define the 4th model
    model = Sequential()
    model.add(Masking(mask_value=-1, input_shape=(4, 2)))
    model.add(LSTM(1))
    model.add(Dense(1, activation='relu'))

    # Compile the 4th model
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["mean_squared_error"])

    # Fit the 4th model
    model.fit(numpyX, numpyY, validation_split=0.2, epochs=3000, batch_size=1)

    # Evaluate the 4th model
    scores = model.evaluate(numpyX,numpyY)

    # Predict the 4th set of result using 4th model
    newY_TwoDim = model.predict(numpyX, batch_size=1)

    # Save the 4th set of predicted target attribute of the new data into a file
    numpy.savetxt("COMP1942Result_4", newY_TwoDim, delimiter=",", fmt="%.10f")

    print("Waiting list size training is successful")

    # Model 5: LSTM (Parameter set B)
    # Train the csv
    dataframe = pandas.read_csv(filename, usecols=[1])
    data_int_TwoDim = dataframe.values
    data_float_TwoDim = data_int_TwoDim.astype(float)

    dataX, dataY = [], []
    for i in range(len(data_float_TwoDim)):
        a = data_float_TwoDim[i:(i, 0)]
        dataX.append(a)
        dataY.append(data_float_TwoDim[i, 0])
    numpyX = numpy.array(dataX)
    numpyY = numpy.array(dataY)

    # Generate random seed
    numpy.random.seed(time.time())

    # Define the 5th model
    model = Sequential()
    model.add(Masking(mask_value=-1, input_shape=(4, 2)))
    model.add(LSTM(1))
    model.add(Dense(1, activation='relu'))

    # Compile the 5th model
    model.compile(loss="mean_squared_error", optimizer="adam", metrics=["mean_squared_error"])

    # Fit the 5th model
    model.fit(numpyX, numpyY, validation_split=0.2, epochs=4000, batch_size=1)

    # Evaluate the 5th model
    scores = model.evaluate(numpyX,numpyY)

    # Predict the 5th set of result using 5th model
    newY_TwoDim = model.predict(numpyX, batch_size=1)

    # Save the 5th set of predicted target attribute of the new data into a file
    numpy.savetxt("COMP1942Result_5", newY_TwoDim, delimiter=",", fmt="%.10f")

    print("Waiting list size training is successful")

# to display the menu
def displayMenu():
    print(" ")
    print("         Menu")
    print("=======================")
    print("1. Collection Dropping and Empty Collection Creating")
    print("2. Data Crawling")
    print("3. Course Search by Keyword")
    print("4. Course Search by Waiting List Size ")
    print("5. Waiting List Size Prediction")
    print("6. Waiting List Size Training")
    print("7. Exit")
    print("=======================")
    print(" ")

def main():
    try:
        # Making a DB connection
        print("Making a MongoDB connection...")
        client = MongoClient("mongodb://localhost:27017")

        # Getting a Database named "university"
        print("Getting a database named \"hkust\"")
        db = client["hkust"]

        while (True):
            displayMenu()
            # allow the user to choose one of the functions in the menu
            choice = input("Please input your choice (1-7): ")
            print("\n")
            # check the input and call the correspondence function
            if (choice == "1"):
                collectionDroppingandEmptyCollectionCreating(db)
            elif (choice == "2"):
                dataCrawling(db)
            elif (choice == "3"):
                courseSeachbyKeyword(db)
            elif (choice == "4"):
                courseSeachbyWaitingListSize(db)
            elif (choice == "5"):
                waitingListSizePrediction(db)
            elif (choice == "6"):
                waitingListSizeTraining(db)
            elif (choice == "7"):
                break
            else:
                print("Invalid Input!")

            # Closing connection with Database
            print("Closing connection with database...")
            client.close()
            exit()
    except pymongo.errors.ConnectionFailure as error:
        print("DB Connection Failed! Error Message: \"{}\"".format(error))

# def main():
#     url = input('Please enter a URL for data crawling: ')
#     if url == 'default':
#         url = 'http://comp4332.com/realistic'
#     with open('url.txt', 'w') as f:
# 	    f.write(url)
#     subprocess.run('scrapy crawl mongo', shell=True)
# 	# We use a file to pass URL, but you can use other methods.
# 	# A commonly used method is to pass the start_urls using the '-a' argument
# 	# If interested, you can refer to https://stackoverflow.com/a/15291961

if __name__ == '__main__':
    main()


# 1300 -> K번째 수
import sys

input = sys.stdin.readline

N = int(input())
K = int(input())

# 검색 범위 정의
first, second = 1, min(10 ** 9, N ** 2)
ans = 0

# 이분 탐색 시작
while first <= second:
    mid = (first + second) // 2 # 구하고자하는 숫자
    count = 0

    # 모든 행 순회
    for i in range(1, N + 1):
        temp = mid // i # 해당하는 i 행에 mid 보다 작거나 같은 수가 몇개 있는지?
        sum_factor = temp if temp < N else N # 만약에 temp가 N을 넘어서면 최대가 N개 이기 때문에 N을 반영함
        count += sum_factor

    # count 값 조사
    if count >= K:
        ans = mid
        second = mid - 1
    else:
        first = mid + 1

print(ans)


def encrypt(text,s):

    result = ""
    # transverse the plain text
    for i in range(len(text)):
        char = text[i]
        # Encrypt uppercase characters in plain text
        if (char.isupper()):
            result += chr((ord(char) + s-65) % 26 + 65)
        # Encrypt lowercase characters in plain text
        else:
            result += chr((ord(char) + s - 97) % 26 + 97)
    return result

def decrypt(ciphertext, s):
    result = ""
    for i in range(len(ciphertext)):
        char = ciphertext[i]
        if char.isupper():
            result += chr((ord(char) - s - 65) % 26 + 65)
        else:
            result += chr((ord(char) - s - 97) % 26 + 97)
    return result

#check the above function
text = input("Enter Plain Text: ")
s = int(input("Enter shift key: "))
print("==========================================")
print("Plain Text : " + text)
print("Shift pattern : " + str(s))
print("Cipher Text: " + encrypt(text,s))

print("==========================================")
# Check the decryption function
ciphertext = encrypt(text, s)  # Encrypt the text using the provided encryption function
print("Cipher Text : " + ciphertext)
print("Shift pattern : " + str(s))
print("Decrypted Text: " + decrypt(ciphertext, s))


'''
Created on June 5, 2018

@author: perej1
'''
import numpy as np
import pandas as pd
import argparse as arg
import matplotlib.pyplot as plt

def pplot(n1, n2, s, f1, f2, r, g, d):
    '''
    function for plotting Hill estimator convergence in probability.
    n1 = smallest sample size
    n2 = largest sample size
    s = step
    f1 = file path for data
    f2 = file path and name for the plot
    r = number of estimates in one csv
    g = true value of the extreme value index
    d = distribution
    '''
    q_1 = np.array([])
    q_2 = np.array([])
    q_3 = np.array([])
    n = np.arange(n1, n2+1, s)
    val = g * np.ones(n.shape)

    for i in n:
        k = int(np.floor(np.sqrt(i)))
        data_i = pd.read_csv("%s/%s-%s-%s-%s.csv" % (f1, i, r, k, d))
        q_1 = np.append(q_1, data_i[d].quantile(0.25))
        q_2 = np.append(q_2, data_i[d].median())
        q_3 = np.append(q_3, data_i[d].quantile(0.75))

    fig, ax = plt.subplots()
    plt.rcParams['font.sans-serif'] = 'Times New Roman'
    plt.rcParams["font.family"] = 'Times New Roman'
    ax.set_xlim(left=0, right=20000)
    ax.plot(n, q_1, '-r', lw=1, label='1st quartile')
    ax.plot(n, q_2, '-k', lw=1, label='median')
    ax.plot(n, q_3, '-b', lw=1, label='3rd quartile')
    ax.plot(n, val, '--g', alpha=0.75, lw=1, label=r'$ \gamma $')
    ax.set_ylabel(r'$\hat{\gamma}$')
    ax.set_xlabel('n')
    ax.legend()
    ax.set_title('Hill estimates with samples from %s distribution' % d.capitalize())
    plt.savefig(f2, format='pdf', dpi=1000)

def main():

    parser = arg.ArgumentParser(
        description="Plots graphs about Hill estimator"
    )
    parser.add_argument("-n1", help="lower bound", type=int)
    parser.add_argument("-n2", help="upper bound", type=int)
    parser.add_argument("-s", help="step", type=int)
    parser.add_argument("-f1", help="data file path", type=str)
    parser.add_argument("-f2", help="fig file path", type=str)
    parser.add_argument("-r", help="sample size", type=int)
    parser.add_argument("-p", help="plot type", type=str, choices=("pplot"))
    parser.add_argument("-g", help="extreme value index", type=float)
    parser.add_argument("-d", help="distribution", type=str,
                        choices=("pareto", "cauchy")
                        )

    args = parser.parse_args()

    if args.p == "pplot":
        pplot(args.n1, args.n2, args.s,
              args.f1, args.f2, args.r, args.g, args.d)

if __name__ == "__main__":
    main()


# Needs clear
import random
from hangman_art import logo
from hangman_art import stages
from hangman_words import word_list

print(logo)                      #Start game
input("Press any key (except space-bar) to play!")

lives = 6                  #6 Trials

current_stage = stages[lives]  # Start with the gallows only

chosen_word = random.choice(word_list) #PC-choice from word list

print(current_stage)
word_list = list(chosen_word) #make choice into a list
display = list(chosen_word)   #Display to player

for i in range(len(display)):
  display[i] = "_"            #Display should indicate "_" letters.
print(display)

game_ended = False

while word_list != display and game_ended == False:  #Play

  guess = input("Guess a letter: ").lower()
  Lose_life = True

  for position in range(len(chosen_word)):

    if chosen_word[position] == guess and display[position] == "_":
      display[position] = guess
      Lose_life = False

    elif chosen_word[position] != guess and display[position] != "_":
      display[position] = chosen_word[position]

    else:
      display[position] = "_"

  if Lose_life == True:                   #Getting hanged
    lives -=1
    current_stage = stages[lives]
    print(f"{guess} is not in the word, try again!")

  print(display)
  print(current_stage)

  if lives == 0:                     #Loss condition
    print("You Lost!")
    print(f"The word was: {chosen_word}")
    game_ended = True

  elif word_list == display:         #Win condition
    print("YOU WIN!")
    print(f"{chosen_word}")
    game_ended = True


import sys
import numpy as np
import joblib as jl
import pandas as pd
import matplotlib.pyplot as plt

# interpreter  python 3.7.2

# filename = 'model_NBC.sav'
# filename = '221105NBC.json'
filename = '221105NBC_2.pkl'
loaded_model = jl.load(filename)

# load models, test and return result
def pose_classification (angles) :
    angles_p = [angles] # input is 2d array!!
    # print(angles_p, type(angles_p))
    result = list(loaded_model.predict(angles_p))[0]
    return result

def main () :
    #get parameter
    ag = list(map(float, sys.argv[1:]))
    data = pose_classification(ag)
    # print("result : ", result, type(result))
    print(data)
    return data

if __name__ == "__main__" :
    main()


"""Support for collecting sensors from 4-noks devices."""

import async_timeout
import datetime
import logging
import time

from datetime import timedelta
from homeassistant.components.sensor import (SensorDeviceClass,
                                             SensorEntity,
                                             SensorStateClass)
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import (CONF_HOST, CONF_NAME)
from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers import event
from homeassistant.helpers.update_coordinator import (
    CoordinatorEntity,
    DataUpdateCoordinator,
    UpdateFailed,
)

from .common import get_dat
from .const import (CONF_SERIAL, DEFAULT_BRAND, DOMAIN)

_LOGGER = logging.getLogger(__name__)

async def async_setup_entry(hass: HomeAssistant, config: ConfigEntry, async_add_entities):
    """Set up energy sensors."""

    async def async_update_data():
        """Fetch data from socket endpoint."""

        async with async_timeout.timeout(10):
            dat = await hass.async_add_executor_job(get_dat, config.data, False)
            if dat is None:
                _LOGGER.error("No data available")

        return dat

    coordinator = DataUpdateCoordinator(
        hass,
        _LOGGER,
        # Name of the data. For logging purposes.
        name="4-noks sensor",
        update_method=async_update_data,
        # Polling interval. Will only be polled if there are subscribers.
        update_interval=timedelta(seconds=60),
    )

    # Get a list of sensors and units of measurement
    dat = await hass.async_add_executor_job(get_dat, config.data, True)

    entities = []
    for name, um in dat.items():
        entities.append(FourNOKSSensor(coordinator, config, name, um))

    async_add_entities(entities)

    # Fetch initial data so we have data when entities subscribe
    await coordinator.async_refresh()

class FourNOKSSensor(CoordinatorEntity, SensorEntity):
    """Representation of a 4-noks sensor."""

    def __init__(self, coordinator, config, sensor_name, sensor_um):
        """Initialize the sensor."""
        super().__init__(coordinator)
        self._device_name = config.data[CONF_NAME]
        self._serial = config.data[CONF_SERIAL]

        self._basename = sensor_name
        self._fullname = self._device_name + "_" + self._basename
        self._name = self._device_name + "_" + self._basename.lower().replace(' ', '_')
        self._unique_id = self._name
        self._um = sensor_um
        if (self._um.lower() == "kw"):
            self._device_class = SensorDeviceClass.POWER
            self._state_class = SensorStateClass.MEASUREMENT
        elif (self._um.lower() == "kwh"):
            self._device_class = SensorDeviceClass.ENERGY
            self._state_class = SensorStateClass.TOTAL_INCREASING
        elif (self._um.lower() == "dd/mm/yyyy hh:mm:ss"):
            self._um = ""
            self._device_class = SensorDeviceClass.DATE
#            self._device_class = None
            self._state_class = None
        else:
            self._device_class = None
            self._state_class = None

    @property
    def name(self):
        """Return the name of the sensor."""
        return self._name

    @property
    def native_value(self):
        """Return the value of the sensor."""
        if self.coordinator.data is not None:
            return self.coordinator.data[self._basename]

    @property
    def native_unit_of_measurement(self):
        """Return the unit of measurement of the sensor, if any."""
        return self._um

    @property
    def available(self) -> bool:
        """Return True if entity is available."""
        if not self.coordinator.last_update_success:
            return False

        return True

    @property
    def unique_id(self):
        """Return a unique ID."""
        return self._unique_id

    @property
    def device_class(self):
        """Return the icon to use in the frontend."""
        return self._device_class

    @property
    def state_class(self):
        """Return state class."""
        return self._state_class

    @property
    def device_info(self):
        """Return device specific attributes."""
        return {
            "name": self._device_name,
            "identifiers": {(DOMAIN, self._serial)},
            "manufacturer": DEFAULT_BRAND,
            "model": DOMAIN,
        }


import urllib.request,os
import urllib.parse
from bs4 import BeautifulSoup

url='https://rozetka.com.ua/ua/tablets/c130309/'

def Soup(url):
    try:
        resp = urllib.request.urlopen(url)
    except:
        pass
    else:
        respData = resp.read()
        return BeautifulSoup(respData, features="html.parser")

soup = Soup(url)
if not soup:
    os._exit(-1)

grid = soup.find(attrs={'class': 'catalog-grid'})
all_goods=grid.find_all_next('li',attrs={'class': 'catalog-grid__cell catalog-grid__cell_type_slim'})

for x in range(10):
    pr = all_goods[x].find(attrs={'class': 'goods-tile__price-value'})
    print(x+1,' price: ',pr.contents[0])
    img = all_goods[x].find(attrs={'class':'goods-tile__picture'})#attrs={'class':'ng-failed-lazyloaded', 'class':'ng-lazyloaded'}
    print(x+1,' product url',img.get('href'))
    soup = Soup(img.get('href'))
    print(x+1,' img src: ',soup.find(attrs={'class':"product-photo__picture"}).get('src'),'\n')


from PIL import Image, ImageDraw
from math import sin, cos, pi
import sys

def __strip_non_directives(inp):
	return [x for x in inp if x in 'F+-']

def make_image(raw_data, iters, growth, size):
	scale = pow(growth, iters)
	data =__strip_non_directives(raw_data)
	direction = 0

	move_dist = scale*size
	#loc = size*scale, size*scale
	loc = 0,0

	output = Image.new('RGBA', (size, size), (255,255,255))
	draw = ImageDraw.Draw(output)

	for rule in data:

		if rule == 'F':
			tmp_loc = loc[0] + move_dist*cos(direction), \
					  loc[1] + move_dist*sin(direction)

			draw.line((loc, tmp_loc), fill = (128, 128, 128))
			loc = tmp_loc
		elif rule == '+':
			direction -= pi/2
		elif rule == '-':
			direction += pi/2
		else:
			sys.stdout.write('This should never happen.\n')

	del draw
	return output


from drivegame_info import ginfo
from drivegame_map import create_map_line
from drivegame_draw import *
import tkinter.messagebox as msgbox

#ゲームの初期化
def init_game():
  global win
  #ウィンドウやキャンバスの生成と走行コースも生成
  win = create_window(ginfo)
  #キーボードイベントを設定
  win.bind("<Left>",key_event_left)
  win.bind("<Right>",key_event_right)

  #ゲームを開始する
  game_loop()
  win.mainloop()

  #カーソルキーを押したときの処理
def key_event_left(e):
  if ginfo["car"] > 0:
    ginfo["car"] -= 1

def key_event_right(e):
  if ginfo["car"] <= ginfo["cols"] - 1:
    ginfo["car"] += 1

#ゲームループ
def game_loop():
  #走行コースと自動車の描画
  draw_map(ginfo)
  win.title("ドライブゲーム スコア=" + str(ginfo["score"]))
  #ゲームオーバ−判定
  map_data = ginfo["map_data"]
  v = map_data[ginfo["rows"] - 2][ginfo["car"]]
  if v != 0:
    msgbox.showinfo(title = "ゲームオーバー",
                message = "コースアウトしました\n"
                "sスコア=" + str(ginfo["score"]))
    quit()
  #スコアを加算
  ginfo["score"] += 10
  #過ぎた道路を消す
  del(map_data[ginfo["rows"] - 1])
  line = create_map_line(ginfo)
  map_data.insert(0,line)
  #一定時間後にこの関数を実行
  win.after(100,game_loop)

if __name__ == "__main__":init_game()


import os
import json
import boto3
import logging
import argparse
import hashlib
import botocore
import time

from pprint import pprint

#############################################################################
#  Utils
#############################################################################

def get_config(parser):
    # FIXEM replace with fire (?)
    subparsers = parser.add_subparsers()

    # Copy Dashboard Sub command
    ####################################
    parser_copy  = subparsers.add_parser('copy')

    parser_copy_common = argparse.ArgumentParser(add_help=False)
    parser_copy_common.add_argument("--src-profile", action="store", dest="myAWSSrcProfile", required=True,
                        help="Your AWS source user Profile ")
    parser_copy_common.add_argument("--dst-profile", action="store", dest="myAWSDstProfile", required=True,
                        help="Your AWS destination user Profile ")
    parser_copy_common.add_argument("--src-name", action="store", dest="mySrcItemName",default="",
                        help="Copy command [iteam:version] ")
    parser_copy_common.add_argument("--data-set-name", action="store", dest="myDstDataSetName",default="",
                        help="Destination data set name ")
    parser_copy_common.add_argument("--dst-name", action="store", dest="myDstItemName",default=None,
                            help="Destination name")
    parser_copy_common.add_argument("--dst-perm-name", action="store", dest="myDstGroupName",default="",
                            help="Comment for Template creation")
    parser_copy_common.add_argument("--comment", action="store", dest="myComment",default="",
                            help="Comment for Template creation")

    parser_copy_sub = parser_copy.add_subparsers()
    parser_copy_dashboard = parser_copy_sub.add_parser('dashboard', parents=[parser_copy_common])

    parser_copy_dashboard.add_argument("--src-version", action="store", dest="mySrcItemVer",default=None,
                                help="Version [item:version] ")
    parser_copy_dashboard.set_defaults(func=copyItem,itemType='dashboard')

    parser_copy_analysis = parser_copy_sub.add_parser('analysis', parents=[parser_copy_common])
    parser_copy_analysis.add_argument("--src-version", action="store", dest="mySrcItemVer",default=None,
                                help="Version [item:version] ")
    parser_copy_analysis.set_defaults(func=copyItem,itemType='analysis')

    # ingest params by

    # List
    ####################################
    parser_list = subparsers.add_parser('list')
    parser_list_sub = parser_list.add_subparsers()
    #common vars
    parser_list_common = argparse.ArgumentParser(add_help=False)
    parser_list_common.add_argument("--profile", action="store", dest="myAWSProfile", required=True,
                                    help="Your AWS  user Profile ")

    parser_list_dashboard = parser_list_sub.add_parser('dashboard', parents=[parser_list_common])
    parser_list_dashboard.set_defaults(func=listDashboards)

    parser_list_dataset = parser_list_sub.add_parser('dataset', parents=[parser_list_common])
    parser_list_dataset.set_defaults(func=listDataSets)

    parser_list_template = parser_list_sub.add_parser('template', parents=[parser_list_common])
    parser_list_template.set_defaults(func=listTemplates)

    parser_list_analysis = parser_list_sub.add_parser('analysis', parents=[parser_list_common])
    parser_list_analysis.set_defaults(func=listAnalysis)

    # Desc
    ####################################
    parser_desc = subparsers.add_parser('desc')
    parser_desc_sub = parser_desc.add_subparsers()

    parser_desc_common = argparse.ArgumentParser(add_help=False)
    parser_desc_common.add_argument("--profile", action="store", dest="myAWSProfile", required=True,
                            help="Your AWS  user Profile ")
    parser_desc_common.add_argument("-id", action="store", dest="myItemId",default="",
                            help="ID ")

    parser_desc_dashboard = parser_desc_sub.add_parser('dashboard', parents=[parser_desc_common])
    parser_desc_dashboard.set_defaults(func=descDashboard)

    parser_desc_dataset = parser_desc_sub.add_parser('dataset', parents=[parser_desc_common])
    parser_desc_dataset.set_defaults(func=descDataSet)

    parser_desc_template = parser_desc_sub.add_parser('template', parents=[parser_desc_common])
    parser_desc_template.set_defaults(func=descTemplate)

    parser_desc_analysis = parser_desc_sub.add_parser('analysis', parents=[parser_desc_common])
    parser_desc_analysis.set_defaults(func=descAnalysis)

    # Global params
    ####################################
    #FIXME move to parents
    parser.add_argument("-v", "--verbosity", action="store",   dest="verbosity", default="INFO",
                        help="Debug level [ERROR|INFO|DEBUG]")

    args = parser.parse_args()
    return args

def init_logger( name, level, trace=False):
    #FIXME create config object
    log_level={'DEBUG':logging.DEBUG, 'INFO':logging.INFO, 'ERROR':logging.ERROR }
    logger = logging.getLogger(name)
    logger.setLevel(log_level[level])
    streamHandler = logging.StreamHandler()
    formatter     = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    streamHandler.setFormatter(formatter)
    logger.addHandler(streamHandler)

    return logger

def checkConfig(config,parser):
        pass

def check_status(response):
    if response['Status'] != 200:
        logger.error(f'Error : {response}')
        exit(1)

#############################################################################
#  Internal fucntions
#############################################################################
def copyDataSets():
        data_set_list = quicksight.list_data_sets(AwsAccountId=config.mySourceAccount)

        for data_set in data_set_list:
            data_set_details = quicksight.describe_data_sets(AwsAccountId=config.mySourceAccount,
                                                                DataSetId=data_set['DataSetId'])
            del data_set_details['Arn']
            del data_set_details['CreatedTime']
            del data_set_details['LastUpdatedTime']
            del data_set_details['ConsumedSpiceCapacityInBytes']

def getDashboardId(quicksight,awsAccountId,dashboardName):
    data = quicksight.list_dashboards(AwsAccountId=awsAccountId)
    #print(dashboards)

    for item in data['DashboardSummaryList']:
        if item['Name'] == dashboardName:
            return item

    return None

def getAnalysisId(quicksight, awsAccountId, analysisName):
    data = quicksight.list_analyses(AwsAccountId=awsAccountId)

    for item in data['AnalysisSummaryList']:
        if item['Name'] == analysisName:
            return item

    return None

def getDashboardById(quicksight, awsAccountId , dashboardId,VersionNumber=None):
    dashboard = quicksight.describe_dashboard(AwsAccountId=awsAccountId,
                                              DashboardId=dashboardId,
                                              VersionNumber=VersionNumber)['Dashboard']
    logger.debug('Describe dashboard '+ str(dashboard))
    return dashboard

def getAnalysisById(quicksight, awsAccountId , analisysId ):
    analysis = quicksight.describe_analysis(AwsAccountId=awsAccountId,
                                              AnalysisId=analisysId)['Analysis']
    logger.debug('Describe dashboard '+ str(analysis))
    return analysis

#cyclete over the dataset
def setTemplate(quicksight,awsAccountId,itemName,entityArn,dataSetArn,VersionNumber='0'):

    template_name = itemName+'-'+ VersionNumber+'-template'
    template_id = hashlib.sha224(template_name.encode()).hexdigest()

    #FIXME switch to use ALIAS
    entry = None
    logger.info(f'Searching template {template_id}')
    try :
        entry = quicksight.describe_template( AwsAccountId=awsAccountId,TemplateId=template_id)
        logger.info(f'Found template:  {template_id} ')
        logger.debug(f'Found template:  {entry} ')
    except botocore.exceptions.ClientError as error:
         if error.response['Error']['Code'] != 'ResourceNotFoundException':
            logger.info(f'Template {template_id} Not Found ')

    template_dict = {   'AwsAccountId':awsAccountId,
                        'TemplateId': template_id,
                        'Name': itemName+'-template',
                        'SourceEntity':{
                            'SourceAnalysis': {
                                'Arn': entityArn,
                                #'Arn': dashboard['Version']['SourceEntityArn'],
                                'DataSetReferences': [
                                    {
                                        'DataSetPlaceholder': 'dataset_placeholder',
                                        'DataSetArn': dataSetArn
                                        #'DataSetArn': dashboard['Version']['DataSetArns'][0]
                                    }
                                 ]
                              },
                         },
                         'VersionDescription':VersionNumber
                    }

    logger.debug(f'Template {template_dict}')

    if entry is None:
        entry = quicksight.create_template(**template_dict)
        logger.info(f'Template created with id {template_id} ')
    else :
        entry = quicksight.update_template(**template_dict)
        logger.info(f'Template updated with id {template_id}')

    #FIXME Check the Status and wait until resource is created
    return entry

def setAnalysis(quicksight, analysisName, templateArn, datasetArn, awsAccount, dstGroupName, ItemVer, comment):
    item = None
    analysisId = hashlib.sha224(analysisName.encode()).hexdigest()

    try:
        item = getAnalysisId(quicksight, awsAccount, analysisName)
        logger.info(f'Found analysis  : {item}')
        logger.debug(f'Found alaysis object : {item}')
    except botocore.exceptions.ClientError as error:
         if error.response['Error']['Code'] != 'ResourceNotFoundException':
            logger.info(f'Dashboard {analysisName} not Found')

    entry =  {
                    "AwsAccountId": awsAccount ,
                    "AnalysisId": analysisId,
                    "Name": analysisName,
                    "SourceEntity": {
                        "SourceTemplate": {
                            "DataSetReferences": [
                                {
                                    "DataSetPlaceholder": "dataset_placeholder",
                                    "DataSetArn": datasetArn
                                    #"DataSetArn": dataset['Arn']
                                }
                            ],
                            "Arn": templateArn
                            #"Arn": template['Arn']
                        }
                    }
                }

    logger.debug(f'analysis entry : {entry} ')

    if item is None:
        logger.info(f'analysis created in target account : {analysisName} ')
        response = quicksight.create_analysis(**entry)
    else:
        logger.info(f'analysis updated in target account : {analysisName} ')
        response = quicksight.update_analysis(**entry)

    logger.debug(f'Analysis  Creation result : {entry}')

    #Check status
    while True:

        item = getAnalysisById(quicksight, awsAccount , analysisId )

        logger.info(f"Status : {item['Status']} ")
        logger.debug(f'Status : {item} ')
        if 'Errors' in item and len(item['Errors']) > 0:
            logger.error(item['Errors'])
            exit(1)

        if item['Status'] == 'CREATION_SUCCESSFUL' or item['Status'] == 'UPDATE_SUCCESSFUL':
            break
        time.sleep(10)

    #update permission
    perm = getPermAnalisysTemplate(dstGroupName, awsAccount)

    logger.debug(f'Set Permission : {perm}')
    logger.info(f'Analysis  permission grant to: {dstGroupName}')
    response = quicksight.update_analysis_permissions( AwsAccountId=awsAccount,
                                                        AnalysisId=analysisId,
                                                        GrantPermissions=perm)
    # CHECK Result
    check_status(response)

def setDashboard(quicksight, dashboardName, templateArn, datasetArn, awsAccount , dstGroupName, dashboardVer, comment):
    item = None
    dashboardId = hashlib.sha224(dashboardName.encode()).hexdigest()
    #dashboardDstId =  dashboard['DashboardId']+"-"+target

    try:
        item = getDashboardId(quicksight, awsAccount, dashboardName)
        logger.info(f'Found dashboard in target account : {dashboardName}')
        logger.debug(f'Found Dashboard object : {item}')
    except botocore.exceptions.ClientError as error:
         if error.response['Error']['Code'] != 'ResourceNotFoundException':
            logger.info(f'Dashboard {dashboardName} not Found')

    # ADD some specific permissions?
    entry =  {
                    "AwsAccountId":awsAccount ,
                    "DashboardId": dashboardId,
                    "Name": dashboardName,
                    "SourceEntity": {
                        "SourceTemplate": {
                            "DataSetReferences": [
                                {
                                    "DataSetPlaceholder": "dataset_placeholder",
                                    "DataSetArn": datasetArn
                                    #"DataSetArn": dataset['Arn']
                                }
                            ],
                            "Arn": templateArn
                            #"Arn": template['Arn']
                        }
                    },
                    "VersionDescription": comment,
                    "DashboardPublishOptions": {
                        "AdHocFilteringOption": {
                            "AvailabilityStatus": "ENABLED"
                        },
                        "ExportToCSVOption": {
                            "AvailabilityStatus": "ENABLED"
                        },
                        "SheetControlsOption": {
                            "VisibilityState": "EXPANDED"
                        }
                    }
                }

    logger.debug(f'Dasboard entry : {entry} ')

    if item is None:
        logger.info(f'Dasboard created in target account : {dashboardName} ')
        entry = quicksight.create_dashboard(**entry)
    else:
        logger.info(f'Dasboard updated in target account : {dashboardName} ')
        entry = quicksight.update_dashboard(**entry)
        logger.info(f'Dasboard updated in target account : {dashboardName} ')

    logger.debug(f'Dashboard Creation result : {entry}')

    #FIXME check the response
    dstVer = int(entry['VersionArn'].split('/')[-1])

    #Check status
    while True:
        item =  getDashboardById(quicksight, awsAccount , dashboardId, int(dashboardVer) )
        logger.info(f"Status : {item['Version']['Status']} ")
        logger.debug(f'Status : {item} ')
        if 'Errors' in item['Version'] and len(item['Version']['Errors']) > 0:
            logger.error(item['Version']['Errors'])
            exit(1)

        if item['Version']['Status'] == 'CREATION_SUCCESSFUL' or item['Version']['Status'] == 'UPDATE_SUCCESSFUL':
            break
        time.sleep(10)

    response = quicksight.update_dashboard_published_version(AwsAccountId=awsAccount ,
                                                         DashboardId=dashboardId,
                                                         VersionNumber=int(dashboardVer))
    check_status(response)
    logger.info(f'Dasboard published in target account : {dashboardName} with version {dashboardVer} ')

    #update permission
    perm = getPermTemplate(dstGroupName, awsAccount)

    logger.debug(f'Set Permission : {perm}')
    logger.info(f'Dashboard permission grant to: {dstGroupName}')
    response = quicksight.update_dashboard_permissions( AwsAccountId=awsAccount,
                                                        DashboardId=dashboardId,
                                                        GrantPermissions=perm)
    # CHECK Result
    check_status(response)

def getPermTemplate(dstGroupName, awsAccountId):
    grpPerm = f"user/default/PA_DEVELOPER/{dstGroupName}"
    #update permission
    perm =  [
                {
                    "Principal": f"arn:aws:quicksight:eu-west-1:{awsAccountId}:{grpPerm}",
                    "Actions": [
                        "quicksight:DescribeDashboard",
                        "quicksight:ListDashboardVersions",
                        "quicksight:UpdateDashboardPermissions",
                        "quicksight:QueryDashboard",
                        "quicksight:UpdateDashboard",
                        "quicksight:DeleteDashboard",
                        "quicksight:DescribeDashboardPermissions",
                        "quicksight:UpdateDashboardPublishedVersion"
                    ]
                },
                {
                    "Principal": f"arn:aws:quicksight:eu-west-1:{awsAccountId}:namespace/default",
                    "Actions": [
                        "quicksight:DescribeDashboard",
                        "quicksight:ListDashboardVersions",
                        "quicksight:QueryDashboard"
                    ]
                }
            ]
    return perm

def getPermAnalisysTemplate(dstGroupName, awsAccountId):
    grpPerm = f"user/default/PA_DEVELOPER/{dstGroupName}"
    #update permission
    perm =  [
                {
                    "Principal": f"arn:aws:quicksight:eu-west-1:{awsAccountId}:{grpPerm}",
                    "Actions": [
                        "quicksight:RestoreAnalysis",
                        "quicksight:UpdateAnalysisPermissions",
                        "quicksight:DeleteAnalysis",
                        "quicksight:QueryAnalysis",
                        "quicksight:DescribeAnalysisPermissions",
                        "quicksight:DescribeAnalysis",
                        "quicksight:UpdateAnalysis"
                    ]
                },
                {
                    "Principal": f"arn:aws:quicksight:eu-west-1:{awsAccountId}:namespace/default",
                    "Actions": [
                        "quicksight:DescribeAnalysis",
                        "quicksight:QueryAnalysis"
                    ]
                }
            ]
    return perm

def getDataSetId(quicksight, awsAccountId, myDstDataSetName):
    data_set = quicksight.list_data_sets(AwsAccountId=awsAccountId)

    for item in data_set['DataSetSummaries']:
        if item['Name'] == myDstDataSetName:
            logger.info(f"Find Data set in target account : {item['Arn']}")
            logger.debug(f'Find Data set in target account : {item}')
            return item

    return None

def setTemplatePerm(quicksight,awsSrcAccount,awsDstAccount,templateId):

    if awsSrcAccount==awsDstAccount:
        logger.debug(f'Set permission skipped, source {awsSrcAccount} and destination {awsDstAccount} account are the same ')
        return

    entry = {'AwsAccountId':awsSrcAccount,
             'TemplateId':templateId,
             'GrantPermissions':[
                {
                    'Principal': f'arn:aws:iam::{awsDstAccount}:root',
                    #'Principal': f'arn:aws:iam::099769128691:root',
                    'Actions': [
                        "quicksight:DescribeTemplate",
                        "quicksight:UpdateTemplatePermissions"
                    ]
                 }
               ]
              }

    response = quicksight.update_template_permissions(**entry)

def getAwsAccount(profile):
    return profile.split('_')[0]

def getQskSession(myAWSProfile):
    session = boto3.session.Session(profile_name=myAWSProfile)
    return session.client('quicksight')

def printTable(title,data,fields):
    #FIXME moveto pretty table
    print(f'{title}')
    print('-------------------------------------------------------------------')
    for item in data:
        row = []
        for field in fields:
            row.append(f'{item[field]}\t')
        print(''.join(row))

#############################################################################
#  Commands
#############################################################################

def copyItem(config):
    global logger, quicksight

    #Open Conenctions
    logger.info(f'** {config.itemType} creation started ')
    awsSrcAccount = getAwsAccount(config.myAWSSrcProfile)
    quicksight[awsSrcAccount] = getQskSession(config.myAWSSrcProfile)
    awsDstAccount = getAwsAccount(config.myAWSDstProfile)
    quicksight[awsDstAccount] = getQskSession(config.myAWSDstProfile)

    if config.itemType == 'dashboard' :
        #Get Dashboard ID
        item = getDashboardId(quicksight[awsSrcAccount], awsSrcAccount,
                                        config.mySrcItemName)
        logger.info(f"Dashboard src ID : {item['DashboardId']}")
        #Get Dashboard description
        item = getDashboardById(quicksight[awsSrcAccount], awsSrcAccount,
                                        item['DashboardId'], int(config.mySrcItemVer) )

        dataSetSrcArn = item['Version']['DataSetArns'][0]
        entityArn  = item['Version']['SourceEntityArn']
    elif config.itemType == 'analysis':
        #Get Dashboard ID
        item = getAnalysisId(quicksight[awsSrcAccount], awsSrcAccount, config.mySrcItemName)
        logger.debug(f"Analysis  : {item}")
        item = getAnalysisById(quicksight[awsSrcAccount], awsSrcAccount,
                                        item['AnalysisId'])
        logger.debug(f"Analysis  : {item}")
        logger.info(f"Analysis src ID : {item['AnalysisId']}")
        dataSetSrcArn = item['DataSetArns'][0]
        entityArn  = item['Arn']
        config.mySrcItemVer = "1"

    #Create / Update template
    template = setTemplate(quicksight[awsSrcAccount],awsSrcAccount,config.mySrcItemName,
                                entityArn, dataSetSrcArn, config.mySrcItemVer)

    #set permissions to tempalte
    setTemplatePerm(quicksight[awsSrcAccount],awsSrcAccount,awsDstAccount,
                                template['TemplateId'])

    #Fetch dataset ARN
    dataSetDst = getDataSetId(quicksight[awsDstAccount], awsDstAccount,
                                config.myDstDataSetName)
    if dataSetDst is None:
        raise Exception(f'Destination dataset not found: {config.myDstDataSetName}')
    logger.info(f"Destination DataSet id : {dataSetDst['DataSetId']}")

    #Create / Update Item

    if config.myDstItemName is None:
        config.myDstItemName = config.mySrcItemName

    if config.myComment is None:
        config.myComment = '-'

    if config.itemType == 'dashboard' :
        setDashboard(quicksight[awsDstAccount], config.myDstItemName, template['Arn'], dataSetDst['Arn'], awsDstAccount,
                        config.myDstGroupName,config.mySrcItemVer,
                        config.myComment)
    elif config.itemType == 'analysis':
        setAnalysis(quicksight[awsDstAccount], config.myDstItemName, template['Arn'], dataSetDst['Arn'], awsDstAccount,
                        config.myDstGroupName, config.mySrcItemVer,
                        config.myComment)

    #FIXME movethe permission out of the dashboard Creation
    #setDashboardPerm()

    logger.info(f'**{config.itemType} creation Finished')

#
# LIST Section
###############################################################################
#FIXME build a decorator
def listDashboards(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    data = quicksight.list_dashboards(AwsAccountId=awsAccount)

    printTable(f'Dashboard List of the account {awsAccount}',
                                data['DashboardSummaryList'] ,
                                ["DashboardId","Name","PublishedVersionNumber"])

def listTemplates(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    data = quicksight.list_templates(AwsAccountId=awsAccount)

    printTable(f'Templates List of the account {awsAccount}',
                                data['TemplateSummaryList'] ,
                                ["TemplateId","Name","LatestVersionNumber"])

def listDataSets(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    data = quicksight.list_data_sets(AwsAccountId=awsAccount)

    printTable(f'DataSet List of the account {awsAccount}',
                                data['DataSetSummaries'] ,
                                ["DataSetId","Name"])

def listAnalysis(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    data = quicksight.list_analyses(AwsAccountId=awsAccount)

    printTable(f'Templates List of the account {awsAccount}',
                                    data['AnalysisSummaryList'] ,
                                    ["AnalysisId","Name","Status"])

#
# LIST Section
###############################################################################
def descDashboard(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    dashboard = quicksight.describe_dashboard(AwsAccountId=awsAccount,
                                                   DashboardId=config.myItemId)
                                                   #VersionNumber=int(config.myVersion))

    pprint(data)

def descDataSet(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    dataset = quicksight.describe_data_set(AwsAccountId=awsAccount,
                                                   DataSetId=config.myItemId,
                                                   )
    pprint(dataset)

def descTemplate(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    template = quicksight.describe_template(AwsAccountId=awsAccount ,
                                                   TemplateId=config.myItemId,
                                                   )
    pprint(template)

def descAnalysis(config):
    awsAccount = getAwsAccount(config.myAWSProfile)
    quicksight = getQskSession(config.myAWSProfile)
    template = quicksight.describe_analysis(AwsAccountId=awsAccount ,
                                                   AnalysisId=config.myItemId,
                                                   )
    pprint(template)

##############################################################################
# MAIN
##############################################################################
quicksight = dict()
config = None
parser = argparse.ArgumentParser()

config = get_config(parser)
    #checkConfig(config,parser)

logger = init_logger("QuickSight", config.verbosity)

#Execute command
config.func(config)
#main()


from django.shortcuts import render
from django.http import HttpResponse
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from category.models import Category
from category.serializers import CategorySerializer

@api_view(['GET'])
def category_collection(request):
    if request.method == 'GET':
        categories = Category.objects.all()
        serializer = CategorySerializer(categories, many=True)
        return Response(serializer.data)

@api_view(['GET'])
def category_element(request, pk):
    try:
        category = Category.objects.get(pk=pk)
    except Category.DoesNotExist:
        return HttpResponse(status=404)

    if request.method == 'GET':
        serializer = CategorySerializer(Category)
        return Response(serializer.data)

@api_view(['POST'])
def category_create(request):
    if request.method == 'POST':

        data = {'name': request.DATA.get('name'),
        }

        serializer = CategorySerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


import win32com.client
import datetime

def sendmail(today, finalresult2):
    outlook = win32com.client.Dispatch('outlook.application')
    mail = outlook.CreateItem(0)

    mail.To = "email"
    #mail.To = ""
    mail.Subject = "Inconsistencies discovered"
    mail.body = "Please note the following Inconsistencies detailed in the attachment."
    mail.Attachments.Add(finalresult2)
    #mail.CC =
    mail.Send()


import pickle
from typing import List, Tuple, Union
import logging

import numpy as np
import tensorflow as tf

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s - %(pathname)s: line %(lineno)d'))
logger.addHandler(handler)

def create_multiartifact_sample(artifacts: List[str],
                                normalization_value: float,
                                image_target_height: int,
                                image_target_width: int,
                                target_names: tf.Tensor,
                                n_artifacts: int,
                                ) -> Tuple[tf.Tensor, tf.Tensor]:
    """Open pickle files and load data.

    Args:
        artifacts: List of file paths to pickle files

    Returns:
        depthmaps of shape (IMAGE_TARGET_HEIGHT, IMAGE_TARGET_WIDTH, n_artifacts)
        targets of shape (1, )
    """
    targets_list = []
    depthmaps = np.zeros((image_target_height, image_target_width, n_artifacts))

    for i, artifact_path in enumerate(artifacts):
        depthmap, targets = _py_load_pickle(artifact_path, normalization_value,
                                            image_target_height, image_target_width, target_names)
        depthmap.set_shape((image_target_height, image_target_width, 1))
        depthmaps[:, :, i] = tf.squeeze(depthmap, axis=2)
        targets_list.append(targets)
    targets = targets_list[0]
    if not np.all(targets_list == targets):
        logger.info('Warning: Not all targets are the same!! \n target_list: %s \n artifacts: %s: ',
                    targets_list, artifacts)
    return depthmaps, targets

def _py_load_pickle(path: Union[tf.Tensor, str],
                    normalization_value: float,
                    image_target_height: int,
                    image_target_width: int,
                    target_names: tf.Tensor,
                    ) -> Tuple[Union[tf.Tensor, np.array], Union[tf.Tensor, np.array]]:
    path_ = path if isinstance(path, str) else path.numpy()
    try:
        depthmap, targets = pickle.load(open(path_, "rb"))
    except OSError as e:
        print(f"path: {path}, type(path) {str(type(path))}")
        print(e)
        raise e
    depthmap = _preprocess_depthmap(depthmap)
    depthmap = depthmap / normalization_value
    depthmap = tf.image.resize(depthmap, (image_target_height, image_target_width))
    targets = _preprocess_targets(targets, target_names)
    return depthmap, targets

def _preprocess_depthmap(depthmap: Union[tf.Tensor, np.array]) -> Union[tf.Tensor, np.array]:
    return depthmap.astype("float32")

def _preprocess_targets(targets: Union[tf.Tensor, np.array],
                        target_names_: tf.Tensor,
                        ) -> Union[tf.Tensor, np.array]:
    target_names = target_names_.numpy().tolist()
    targets = [targets[target_name.decode() if isinstance(target_name, bytes) else target_name]
               for target_name in target_names]
    return np.array(targets).astype("float32")


import os

def create_folders(paths):
    for path in paths:
        if not os.path.exists(path):
            os.makedirs(path)
            print(f"Utworzono folder: {path}")
        else:
            print(f"Folder już istnieje: {path}")

folders = ['zad1', 'zad1/zad11', 'zad1/zad11/zad111', 'zad1/zad12/zad121']
create_folders(folders)
os.remove('zad1')


#!/usr/bin/python

import sys
import pymongo

### Create location data

def list_locations(mydb):

    for doc in mydb.find():
        print(doc) # iterate the cursor

###############################################################################
# main
###############################################################################

def main(args):

    connection = pymongo.MongoClient('ds121861.mlab.com', 21861)
    db = connection['python_twitter']

    db.authenticate('test', 'nnnn1858')

    dont_delete = []
    dont_delete.append('system.indexes')
    print(dont_delete)
    for col in db.collection_names():

        #print(col not in dont_delete)
        if (col not in dont_delete):
            print('deleting', col)
            db.drop_collection(col)

    connection.close()

if __name__ == '__main__':
    main(sys.argv[1:])


import ctypes
from ctypes import wintypes, Structure, POINTER
from time import sleep
import kthread
import sys
from ctypes.wintypes import (
    HANDLE,
    DWORD,
    BOOL,
    ULONG,
    LPARAM,
    WPARAM,
    HINSTANCE,
    HHOOK,
    LPMSG,
    HWND,
)

LPDWORD = POINTER(DWORD)
LPHANDLE = POINTER(HANDLE)
ULONG_PTR = POINTER(ULONG)

kernel32 = ctypes.CDLL("kernel32")
user32 = ctypes.WinDLL("user32", use_last_error=True)

config = sys.modules[__name__]
config.done = False
config.keyboard_hook_id = 0
config.results = []

class KBDLLHOOKSTRUCT(Structure):
    _fields_ = [
        ("vkCode", wintypes.DWORD),
        ("scanCode", wintypes.DWORD),
        ("flags", wintypes.DWORD),
        ("time", wintypes.DWORD),
        ("dwExtraInfo", ULONG_PTR),
    ]

# dict from https://pypi.org/project/keyboard/

events = {
    0x100: "KEY_DOWN",
    0x101: "KEY_UP",
    0x104: "KEY_DOWN_SYS",
    0x105: "KEY_UP_SYS",
}

VK_CODELETTER = {
    0x03: ("control-break processing", False),
    0x08: ("backspace", False),
    0x09: ("tab", False),
    0x0C: ("clear", False),
    0x0D: ("enter", False),
    0x10: ("shift", False),
    0x11: ("ctrl", False),
    0x12: ("alt", False),
    0x13: ("pause", False),
    0x14: ("caps lock", False),
    0x15: ("ime kana mode / ime hanguel mode / ime hangul mode ", False),
    0x17: ("ime junja mode", False),
    0x18: ("ime final mode", False),
    0x19: ("ime hanja mode / ime kanji mode", False),
    0x1B: ("esc", False),
    0x1C: ("ime convert", False),
    0x1D: ("ime nonconvert", False),
    0x1E: ("ime accept", False),
    0x1F: ("ime mode change request", False),
    0x20: ("spacebar", False),
    0x21: ("page up", False),
    0x22: ("page down", False),
    0x23: ("end", False),
    0x24: ("home", False),
    0x25: ("left", False),
    0x26: ("up", False),
    0x27: ("right", False),
    0x28: ("down", False),
    0x29: ("select", False),
    0x2A: ("print", False),
    0x2B: ("execute", False),
    0x2C: ("print screen", False),
    0x2D: ("insert", False),
    0x2E: ("delete", False),
    0x2F: ("help", False),
    0x30: ("0", False),
    0x31: ("1", False),
    0x32: ("2", False),
    0x33: ("3", False),
    0x34: ("4", False),
    0x35: ("5", False),
    0x36: ("6", False),
    0x37: ("7", False),
    0x38: ("8", False),
    0x39: ("9", False),
    0x41: ("a", False),
    0x42: ("b", False),
    0x43: ("c", False),
    0x44: ("d", False),
    0x45: ("e", False),
    0x46: ("f", False),
    0x47: ("g", False),
    0x48: ("h", False),
    0x49: ("i", False),
    0x4A: ("j", False),
    0x4B: ("k", False),
    0x4C: ("l", False),
    0x4D: ("m", False),
    0x4E: ("n", False),
    0x4F: ("o", False),
    0x50: ("p", False),
    0x51: ("q", False),
    0x52: ("r", False),
    0x53: ("s", False),
    0x54: ("t", False),
    0x55: ("u", False),
    0x56: ("v", False),
    0x57: ("w", False),
    0x58: ("x", False),
    0x59: ("y", False),
    0x5A: ("z", False),
    0x5B: ("left windows", False),
    0x5C: ("right windows", False),
    0x5D: ("applications", False),
    0x5F: ("sleep", False),
    0x60: ("0", True),
    0x61: ("1", True),
    0x62: ("2", True),
    0x63: ("3", True),
    0x64: ("4", True),
    0x65: ("5", True),
    0x66: ("6", True),
    0x67: ("7", True),
    0x68: ("8", True),
    0x69: ("9", True),
    0x6A: ("*", True),
    0x6B: ("+", True),
    0x6C: ("separator", True),
    0x6D: ("-", True),
    0x6E: ("decimal", True),
    0x6F: ("/", True),
    0x70: ("f1", False),
    0x71: ("f2", False),
    0x72: ("f3", False),
    0x73: ("f4", False),
    0x74: ("f5", False),
    0x75: ("f6", False),
    0x76: ("f7", False),
    0x77: ("f8", False),
    0x78: ("f9", False),
    0x79: ("f10", False),
    0x7A: ("f11", False),
    0x7B: ("f12", False),
    0x7C: ("f13", False),
    0x7D: ("f14", False),
    0x7E: ("f15", False),
    0x7F: ("f16", False),
    0x80: ("f17", False),
    0x81: ("f18", False),
    0x82: ("f19", False),
    0x83: ("f20", False),
    0x84: ("f21", False),
    0x85: ("f22", False),
    0x86: ("f23", False),
    0x87: ("f24", False),
    0x90: ("num lock", False),
    0x91: ("scroll lock", False),
    0xA0: ("left shift", False),
    0xA1: ("right shift", False),
    0xA2: ("left ctrl", False),
    0xA3: ("right ctrl", False),
    0xA4: ("left menu", False),
    0xA5: ("right menu", False),
    0xA6: ("browser back", False),
    0xA7: ("browser forward", False),
    0xA8: ("browser refresh", False),
    0xA9: ("browser stop", False),
    0xAA: ("browser search key", False),
    0xAB: ("browser favorites", False),
    0xAC: ("browser start and home", False),
    0xAD: ("volume mute", False),
    0xAE: ("volume down", False),
    0xAF: ("volume up", False),
    0xB0: ("next track", False),
    0xB1: ("previous track", False),
    0xB2: ("stop media", False),
    0xB3: ("play/pause media", False),
    0xB4: ("start mail", False),
    0xB5: ("select media", False),
    0xB6: ("start application 1", False),
    0xB7: ("start application 2", False),
    0xBB: ("+", False),
    0xBC: (",", False),
    0xBD: ("-", False),
    0xBE: (".", False),
    0xE5: ("ime process", False),
    0xF6: ("attn", False),
    0xF7: ("crsel", False),
    0xF8: ("exsel", False),
    0xF9: ("erase eof", False),
    0xFA: ("play", False),
    0xFB: ("zoom", False),
    0xFC: ("reserved ", False),
    0xFD: ("pa1", False),
    0xFE: ("clear", False),
}

def keyboard_hook(nCode, wParam, lParam):
    if config.done:
        user32.UnhookWindowsHookEx(config.keyboard_hook_id)
        return 0
    if nCode >= 0:
        kb_struct = ctypes.cast(lParam, ctypes.POINTER(KBDLLHOOKSTRUCT)).contents

        vocodeletter = VK_CODELETTER.get(kb_struct.vkCode, "")
        if len(vocodeletter) <2:
            vocodeletter = (kb_struct.vkCode, None)

        config.results.append(
            (
                *vocodeletter,
                wParam,events.get(wParam, ""),
                kb_struct.scanCode,
                kb_struct.flags,
                kb_struct.time,
            )
        )
    return user32.CallNextHookEx(None, nCode, wParam, lParam)

def _start_hook():
    config.results.append(('letter', 'is_numpad', 'event_code', 'event', 'scan_code', 'flags', 'time'))
    keyboard_hook_cb = ctypes.WINFUNCTYPE(
        ctypes.c_int, WPARAM, LPARAM, ctypes.POINTER(ctypes.c_void_p)
    )
    keyboard_hook_ptr = keyboard_hook_cb(keyboard_hook)
    user32.SetWindowsHookExW.argtypes = [
        ctypes.c_int,
        keyboard_hook_cb,
        HINSTANCE,
        DWORD,
    ]
    user32.SetWindowsHookExW.restype = HHOOK
    user32.CallNextHookEx.restype = ctypes.c_int
    user32.UnhookWindowsHookEx.argtypes = [HHOOK]
    user32.UnhookWindowsHookEx.restype = BOOL

    config.keyboard_hook_id = user32.SetWindowsHookExW(13, keyboard_hook_ptr, None, 0)

    user32.GetMessageW.argtypes = [LPMSG, HWND, ctypes.c_uint, ctypes.c_uint]
    user32.GetMessageW.restype = BOOL

    user32.TranslateMessage.argtypes = [LPMSG]
    user32.TranslateMessage.restype = BOOL

    user32.DispatchMessageA.argtypes = [LPMSG]

    try:
        msg = ctypes.wintypes.MSG()
        while (
            user32.GetMessageW(ctypes.byref(msg), None, 0, 0) != 0
        ) and not config.done:
            user32.TranslateMessage(ctypes.byref(msg))
            user32.DispatchMessageA(ctypes.byref(msg))
        user32.UnhookWindowsHookEx(config.keyboard_hook_id)
    except Exception:
        try:
            user32.UnhookWindowsHookEx(config.keyboard_hook_id)
        except Exception:
            pass

    return

def killcheck(t):
    while t.is_alive():
        sleep(1)
        if config.done:
            sleep(2)
            try:
                if t.is_alive():
                    t.kill()
            except Exception:
                pass

def start_hook():
    t = kthread.KThread(target=_start_hook)
    t.start()
    t2 = kthread.KThread(target=killcheck, args=(t,))
    t2.start()


from flask import Blueprint, jsonify, request
from app.models import Category, db, Project, User

category_routes = Blueprint('categories', __name__)

@category_routes.route('/')
def categories():
    categories = Category.query.all()
    return {'categories': [category.to_dict() for category in categories]}

@category_routes.route('/<int:id>')
def get_category(id):
    categories = Category.query.filter(Project.category_id == id).all()
    categoryList = []
    for category in categories:
        categoryList.append(category.to_dict())
    return jsonify(categoryList)

@category_routes.route('/project/<int:id>')
def get_project_category(id):
    projects = Project.query.filter(Project.category_id == id).all()
    projectList = []
    for project in projects:
        projectList.append(project.to_dict())
    return jsonify(projectList)


from django.conf.urls import url

from . import views

urlpatterns = [
            url(r'^fort$', views.fort, name='fort'),
            url(r'^pokestop$', views.pokestop, name='pokestop'),
            url(r'^gym$', views.gym, name='gym'),
            url(r'^pokemon$', views.pokemon, name='pokemon'),
            ]


'''

Build the final annotation tables;

'''

import glob, sys, os, gzip
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['font.size'] = 6

sys.path.append('../../../')
import shared
from glbase3 import glload, utils, expression, genelist

all_genes = glload('../../../transcript_assembly/packed/all_genes.glb')
te = glload('../../te_transcripts/transcript_table_merged.mapped.glb')
not_te = te.map(genelist=all_genes, key='transcript_id', logic='notright')

#print(all_genes)
#print()
#print(te)

all_genes_ensg = set([g['ensg'] for g in all_genes if 'ENSG' in g['ensg']])
has_at_least_one_coding_isoform = set([g['ensg'] for g in all_genes if ';C;' in g['name'] and 'ENSG' in g['ensg']])

# coding and non-coding;
shared.pie('coding_genes_with_a_noncoding_transcript.png', [len(all_genes_ensg)-len(has_at_least_one_coding_isoform), len(has_at_least_one_coding_isoform)], ['no non-coding', 'has non-coding transcript'], 'Coding genes with a noncoding transcript')

#print(len(te), len(not_te), len(all_genes), len(te)+len(not_te))

# Piechart, percent of transcripts containing a TE:
shared.pie('pies/te_all.png', [len(not_te), len(te)], ['no-TE', 'TE'], 'All')

# collect stats:
res = {
    'pc': {'TE': 0, 'nonTE': 0},
    'pc_matching': {'TE': 0, 'nonTE': 0},
    'ncrna': {'TE': 0, 'nonTE': 0},
    'ncrna_matching': {'TE': 0, 'nonTE': 0},
    'non_coding_version_of_coding_gene': {'TE': 0, 'nonTE': 0},
    'novel': {'TE': 0, 'nonTE': 0},
    'novel_pc': {'TE': 0, 'nonTE': 0},
    'novel_ncrna': {'TE': 0, 'nonTE': 0},
    'all_variant': {'TE': 0, 'nonTE': 0},
    'pc_variant': {'TE': 0, 'nonTE': 0},
    'ncrna_variant': {'TE': 0, 'nonTE': 0},
    }

data = {'TE': te, 'nonTE': not_te}

for k in data:
    for g in data[k]:
        if ';!' in g['name']:
            res['novel'][k] += 1
        if ';~' in g['name']:
            res['all_variant'][k] += 1

        if ';C;' in g['name']:
            res['pc'][k] += 1
            if ';=' in g['name']:
                res['pc_matching'][k] += 1
            if ';~' in g['name']:
                res['pc_variant'][k] += 1
            if ';!' in g['name']:
                res['novel_pc'][k] += 1

        elif ';NC;' in g['name']:
            res['ncrna'][k] += 1
            if ';=' in g['name']:
                res['ncrna_matching'][k] += 1
            if ';~' in g['name']:
                res['ncrna_variant'][k] += 1
            if ';!' in g['name']:
                res['novel_ncrna'][k] += 1

        # non_coding_version_of_coding_gene
        if ';NC;' in g['name']:
            if 'ENST' in g['enst']:
                # filter out LINCs, etc:
                if g['ensg'] not in has_at_least_one_coding_isoform:
                    continue
                res['non_coding_version_of_coding_gene'][k] += 1

title_map = {'pc': 'protein-coding',
    'ncrna': 'non-coding RNA',
    'all_variant': 'Variant transcripts',
    'pc_variant': 'protein-coding (variant)',
    'ncrna_variant': 'non-coding RNA (variant)',
    'non_coding_version_of_coding_gene': 'non-coding version of coding gene',
    'novel': 'Novel',
    'novel_pc': 'Novel protein-coding',
    'novel_ncrna': 'Novel non-coding',
    'pc_matching': 'PC Matching',
    'ncrna_matching': 'ncRNA matching'
    }

for k in res:
    shared.pie('pies/te_%s.png' % k, [res[k]['nonTE'], res[k]['TE']], ['no-TE', 'TE'], title_map[k])

shared.split_bar('bar.png'.format(k), res, cols=['#ff7f0e', '#1f77b4'])

# pickle the results
import pickle
shared.pickle_it('results.pickle', res)


from django.urls import include, path
from rest_framework.authtoken.views import obtain_auth_token
from rest_framework.routers import DefaultRouter

from .views import CommentViewSet, PostViewSet

v1_router = DefaultRouter()
v1_router.register(r'posts', PostViewSet, basename='Post')
v1_router.register(r'posts/(?P<id>\d+)/comments', CommentViewSet,
                   basename='Comment')

api_patterns = [
    path('v1/api-token-auth/', obtain_auth_token),
    path('v1/', include(v1_router.urls))
]

urlpatterns = [
    path('api/', include(api_patterns)),
]


# encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http:# mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import, division, unicode_literals

from jx_base.expressions import SqlInstrOp as SqlInstrOp_, OrOp
from jx_sqlite.expressions._utils import check, SQLang, SqlScript
from jx_sqlite.sqlite import sql_call
from mo_json import JX_INTEGER

class SqlInstrOp(SqlInstrOp_):
    @check
    def to_sql(self, schema):
        value = self.value.to_sql(schema)
        find = self.find.to_sql(schema)

        return SqlScript(
            data_type=JX_INTEGER,
            expr=sql_call("INSTR", value.frum, find.frum),
            frum=self,
            miss=OrOp([self.value.missing(SQLang), self.find.missing(SQLang)]),
            schema=schema,
        )

    def partial_eval(self, lang):
        value = self.value.partial_eval(SQLang)
        find = self.find.partial_eval(SQLang)
        return SqlInstrOp([value, find])


import numpy as np
import matplotlib.pyplot as plt

def kernel_and_derivatives(x, y, a, mu, nu):
    """ From Starfield and Crouch, pages 49 and 82 """
    f = (
        -1
        / (4 * np.pi * (1 - nu))
        * (
            y * (np.arctan2(y, (x - a)) - np.arctan2(y, (x + a)))
            - (x - a) * np.log(np.sqrt((x - a) ** 2 + y ** 2))
            + (x + a) * np.log(np.sqrt((x + a) ** 2 + y ** 2))
        )
    )

    df_dx = (
        1
        / (4 * np.pi * (1 - nu))
        * (
            np.log(np.sqrt((x - a) ** 2 + y ** 2))
            - np.log(np.sqrt((x + a) ** 2 + y ** 2))
        )
    )

    df_dy = (
        -1
        / (4 * np.pi * (1 - nu))
        * ((np.arctan2(y, (x - a)) - np.arctan2(y, (x + a))))
    )

    df_dxy = (
        1
        / (4 * np.pi * (1 - nu))
        * (y / ((x - a) ** 2 + y ** 2) - y / ((x + a) ** 2 + y ** 2))
    )

    df_dxx = (
        1
        / (4 * np.pi * (1 - nu))
        * ((x - a) / ((x - a) ** 2 + y ** 2) - (x + a) / ((x + a) ** 2 + y ** 2))
    )

    df_dyy = df_dxx

    df_dxyy = (
        1
        / (4 * np.pi * (1 - nu))
        * (
            ((x - a) ** 2 - y ** 2) / ((x - a) ** 2 + y ** 2) ** 2
            - ((x + a) ** 2 - y ** 2) / ((x + a) ** 2 + y ** 2) ** 2
        )
    )

    df_dxxx = -df_dxyy

    df_dyyy = (
        2
        * y
        / (4 * np.pi * (1 - nu))
        * (
            (x - a) / ((x - a) ** 2 + y ** 2) ** 2
            - (x + a) / ((x + a) ** 2 + y ** 2) ** 2
        )
    )

    df_dxxy = -df_dyyy

    return f, df_dx, df_dy, df_dxy, df_dxx, df_dyy, df_dxyy, df_dxxx, df_dyyy, df_dxxy

def constant_traction_element(x, y, a, mu, nu, traction_x, traction_y):
    """ From Starfield and Crouch page 48 """
    f, df_dx, df_dy, df_dxy, df_dxx, df_dyy, df_dxyy, df_dxxx, df_dyyy, df_dxxy = kernel_and_derivatives(
        x, y, a, mu, nu
    )

    displacement_x = traction_x / (2 * mu) * (
        (3 - 4 * nu) * f + y * df_dy
    ) + traction_y / (2 * mu) * (-y * df_dx)

    displacement_y = traction_x / (2 * mu) * (-y * df_dx) + traction_y / (2 * mu) * (
        (3 - 4 * nu) * f - y * df_dy
    )

    stress_xx = traction_x * ((3 - 2 * nu) * df_dx + y * df_dxy) + traction_y * (
        2 * nu * df_dy + y * df_dyy
    )
    stress_yy = traction_x * (-1 * (1 - 2 * nu) * df_dx + y * df_dxy) + traction_y * (
        2 * (1 - nu) * df_dy - y * df_dyy
    )
    stress_xy = traction_x * (2 * (1 - nu) * df_dy + y * df_dyy) + traction_y * (
        (1 - 2 * nu) * df_dx - y * df_dxy
    )

    return displacement_x, displacement_y, stress_xx, stress_yy, stress_xy

def constant_slip_element(x, y, a, mu, nu, slip_x, slip_y):
    """ From Starfield and Crouch page 81
        with special cases from pages 82 and 83
    """

    f, df_dx, df_dy, df_dxy, df_dxx, df_dyy, df_dxyy, df_dxxx, df_dyyy, df_dxxy = kernel_and_derivatives(
        x, y, a, mu, nu
    )

    displacement_x = slip_x * (2 * (1 - nu) * df_dy - y * df_dxx) + slip_y * (
        -1 * (1 - 2 * nu) * df_dx - y * df_dxy
    )

    displacement_y = slip_x * (2 * (1 - 2 * nu) * df_dx - y * df_dxy) + slip_y * (
        2 * (1 - nu) * df_dy - y * df_dyy
    )

    stress_xx = 2 * slip_x * mu * (2 * df_dxy + y * df_dxyy) + 2 * slip_y * mu * (
        df_dyy + y * df_dyyy
    )

    stress_yy = 2 * slip_x * mu * (-y * df_dxyy) + 2 * slip_y * mu * (
        df_dyy + y * df_dyyy
    )

    stress_xy = 2 * slip_x * mu * (df_dyy + y * df_dyyy) + 2 * slip_y * mu * (
        -y * df_dxyy
    )

    return displacement_x, displacement_y, stress_xx, stress_yy, stress_xy

mu = 30e9
nu = 0.25
n_pts = 50
x = np.linspace(-5, 5, n_pts)
y = np.linspace(-5, 5, n_pts)
x, y = np.meshgrid(x, y)

# A single element
element = {}
element["x1"] = -1
element["y1"] = 0
element["x2"] = 1
element["y2"] = 0

# element["x1"] = -np.sqrt(2)/2
# element["y1"] = -np.sqrt(2)/2
# element["x2"] = np.sqrt(2)/2
# element["y2"] = np.sqrt(2)/2

element["angle"] = np.arctan2(element["y2"] - element["y1"], element["x2"] - element["x1"])
element["length"] = np.sqrt(
    (element["x2"] - element["x1"]) ** 2 + (element["y2"] - element["y1"]) ** 2
)
element["half_length"] = 0.5 * element["length"]
element["x_center"] = 0.5 * (element["x2"] + element["x1"])
element["y_center"] = 0.5 * (element["y2"] + element["y1"])
element["rotation_matrix"] = np.array(
    [
        [np.cos(element["angle"]), -np.sin(element["angle"])],
        [np.sin(element["angle"]), np.cos(element["angle"])],
    ]
)

x_calc = x - element["x_center"]
y_calc = y - element["y_center"]
rotated_coords = np.matmul(np.vstack((x_calc.flatten(), y_calc.flatten())).T, element["rotation_matrix"])
x_calc = rotated_coords[:, 0]
y_calc = rotated_coords[:, 1]
x_calc = np.reshape(x_calc, (n_pts, n_pts))
y_calc = np.reshape(y_calc, (n_pts, n_pts))

displacement_x, displacement_y, stress_xx, stress_yy, stress_xy = constant_traction_element(
    x_calc, y_calc, element["half_length"], mu, nu, 1, 0
)

_displacement_x, _displacement_y, _stress_xx, _stress_yy, _stress_xy = constant_slip_element(
    x_calc, y_calc, element["half_length"], mu, nu, 1, 0
)

def consistent_plots():
    plt.gca().set_aspect("equal")
    plt.xticks([-5, 0, 5])
    plt.yticks([-5, 0, 5])
    plt.colorbar(fraction=0.046, pad=0.04)

plt.close("all")
plt.figure(figsize=(16, 8))
n_contours = 10

plt.subplot(2, 6, 1)
plt.contourf(x, y, displacement_x, n_contours)
plt.title("displacement_x")
consistent_plots()

plt.subplot(2, 6, 2)
plt.contourf(x, y, displacement_y, n_contours)
plt.title("displacement_y")
consistent_plots()

plt.subplot(2, 6, 3)
plt.quiver(x, y, displacement_x, displacement_y, units='width')
plt.title("vectors")
plt.gca().set_aspect("equal")
plt.xticks([-5, 0, 5])
plt.yticks([-5, 0, 5])

plt.subplot(2, 6, 4)
plt.contourf(x, y, stress_xx, n_contours)
plt.title("stress_xx")
consistent_plots()

plt.subplot(2, 6, 5)
plt.contourf(x, y, stress_yy, n_contours)
plt.title("stress_yy")
consistent_plots()

plt.subplot(2, 6, 6)
plt.contourf(x, y, stress_xy, n_contours)
plt.title("stress_xy")
consistent_plots()

# Constant displacement
plt.subplot(2, 6, 7)
plt.contourf(x, y, _displacement_x, n_contours)
plt.title("displacement_x")
consistent_plots()

plt.subplot(2, 6, 8)
plt.contourf(x, y, _displacement_y, n_contours)
plt.title("displacement_y")
consistent_plots()

plt.subplot(2, 6, 9)
plt.quiver(x, y, _displacement_x, _displacement_y, units='width')
plt.title("vectors")
plt.gca().set_aspect("equal")
plt.xticks([-5, 0, 5])
plt.yticks([-5, 0, 5])

plt.subplot(2, 6, 10)
plt.contourf(x, y, _stress_xx, n_contours)
plt.title("stress_xx")
consistent_plots()

plt.subplot(2, 6, 11)
plt.contourf(x, y, _stress_yy, n_contours)
plt.title("stress_yy")
consistent_plots()

plt.subplot(2, 6, 12)
plt.contourf(x, y, _stress_xy, n_contours)
plt.title("stress_xy")
consistent_plots()

plt.tight_layout()

plt.figure(figsize=(10, 10))
plt.quiver(x, y, 10 * displacement_x, 10 * displacement_y, units="x")
plt.title("vectors")
plt.gca().set_aspect("equal")
plt.xticks([-5, 0, 5])
plt.yticks([-5, 0, 5])

plt.show(block=False)


import numpy as np
from matplotlib import pyplot as plt
from matplotlib import animation

fig = plt.figure()
fig.set_dpi(100)
fig.set_size_inches(7, 6.5)

ax = plt.axes(xlim=(-20, 20), ylim=(-20, 20))

size = 0.3
xcenter = 0
ycenter = 0
radius = 1

n = 10

j = 10

patch_list = []
for i in range(n):
    patch = plt.Circle((0,0), size, fc = 'r')
    patch_list.append(patch)
for p in patch_list:
    ax.add_patch(p)

def init():
    return patch_list

def animate(t):
    i = 1
#    xj = xcenter + j*radius * np.sin(np.radians(20*t/j))
#    yj = ycenter + j*radius * np.cos(np.radians(20*t/j))

    xj = 0
    yj = 0
    for p in patch_list:
        x, y = p.center
        r = i*radius
        x = -xj + xcenter + r * np.sin(np.radians(t/i))
        y = -yj + ycenter + r * np.cos(np.radians(t/i))
        # plt.plot(x,y,'o')
        p.center = (x, y)
        i=i+1
    return patch_list

anim = animation.FuncAnimation(fig, animate,
                               init_func=init,
                               frames=7200,
                               interval=20,
                               blit=True)
plt.show()


import copy

class Solution:
    def convert(self, s: str, numRows: int) -> str:
        if numRows == 1:
            return copy.copy(s)

        rows = [''] * numRows

        current_row = 0
        move_forward = True

        for c in s:
            rows[current_row] += c

            current_row += 1 if move_forward else -1
            # Change direction
            if current_row == 0:
                move_forward = True
            if current_row == numRows -1:
                move_forward = False

        return ''.join(rows)