from database import DataBase

PAGES_NUM = 3

class Page:

    @classmethod
    def get_one_page(cls, page_id: int, user_id):
        # DBと接続する
        db = DataBase()
        file_data = db.get_collection(collection_name="file_data")
        user_status = []

        # 指定したページ番号から情報を取得する
        search_result: dict = file_data.find_one({"page_id" : page_id}, {"_id" : False})
        # ユーザーがアカウント登録しているかを確認する
        for status in ["good", "went", "go"]:
            # ユーザーが既にいいねをしていたかを確認する
            for i in search_result[status]:
                if i == user_id:
                    user_status.append(status)
        search_result["user_status"] = user_status
        # ユーザーIDからユーザー数に変換
        for status in ["good", "went", "go"]:
            search_result[status] = len(search_result[status])

        return search_result

    def __init__(self):
        self.db = DataBase()
        self.file_data = self.db.get_collection(collection_name="file_data")

    # 複数ページを取得する
    def get_pages(self, tag, page_num):
        # リストの初期化
        kankou = list()
        gurume = list()
        tamasanpo = list()
        omiyage = list()

        # 取得したデータを分類する
        for doc in list(self.file_data.find({},{"_id":False})):
            # ユーザーIDからユーザー数に変換
            for status in ["good", "went", "go"]:
                doc[status] = len(doc[status])

            if (doc["tag"] == "kankou"):
                kankou.append(doc)
            elif (doc["tag"] == "gurume"):
                gurume.append(doc)
            elif (doc["tag"] == "tamasanpo"):
                tamasanpo.append(doc)
            else:
                omiyage.append(doc)

        # 数を数える
        max_page = {
            "kankou": len(kankou),
            "gurume": len(gurume),
            "tamasanpo": len(tamasanpo),
            "omiyage": len(omiyage),
        }

        # 取得する範囲の最初を決める
        start_index = PAGES_NUM * page_num

        # 結果を返却する
        if (tag == "kankou"):
            kankou.reverse()
            return {"result":kankou[start_index : start_index+PAGES_NUM], "max":max_page}
        elif (tag == "gurume"):
            gurume.reverse()
            return {"result":gurume[start_index : start_index+PAGES_NUM], "max":max_page}
        elif (tag == "tamasanpo"):
            tamasanpo.reverse()
            return {"result":tamasanpo[start_index : start_index+PAGES_NUM], "max":max_page}
        else:
            omiyage.reverse()
            return {"result":omiyage[start_index : start_index+PAGES_NUM], "max":max_page}

    # ページの投稿
    def post_page(self, page, user_id: int):
        # DBからファイル名の最大値を読み取る
        docs = list(self.file_data.find({}, {"_id":False}).sort("page_id", -1))

        # 受けとったjsonデータから新しいページを作成する
        new_page = {
            "page_id": docs[0]["page_id"] + 1,
            "title": page.title,
            "tag": page.tag,
            "text": page.text,
            "user": user_id,
            "location_name": page.location_name,
            "location": {
                "x": page.location.x,
                "y": page.location.y
            },
            "image" : page.image,
            "good": [],
            "went": [],
            "go": []
        }

        # 新しく作成したデータをDBに追加
        self.file_data.insert_one(new_page)

        return docs[0]["page_id"] + 1

    # 投稿したページの変更
    def put_page(self, page_id, page, user_id):
        update_page = {
            "page_id": page_id,
            "title": page.title,
            "tag": page.tag,
            "text": page.text,
            "user": user_id,
            "location": {
                "x": page.location.x,
                "y": page.location.y
            },
            "image": page.image
        }
        self.file_data.replace_one({"page_id":page_id}, update_page)
        return page_id

    # ページの削除
    def delete_page(self, page_id):
        self.file_data.delete_one({"page_id": page_id})
        return page_id


import zip_util
from geopy.distance import geodesic

zip_codes = zip_util.read_zip_all()

def write_loc():
    zip_code = input("Enter a ZIP Code to lookup: ")
    if isinstance(loc(zip_code), tuple):
        print("ZIP Code", zip_code, "is in", ", ".join(loc(zip_code)[1]), "\ncoordinates:",
              " ".join(map(str, (loc(zip_code)[0]))))
        return
    else:
        print(loc(zip_code))

def loc(zip_code):
    flag = False
    for i in range(len(zip_codes) - 1):
        if zip_code == zip_codes[i][0]:
            flag = True
            zip_code_location = zip_codes[i][1:3]
            loc_name = zip_codes[i][3:5]
            return zip_code_location, loc_name
    if not flag:
        return "Invalid or unknown ZIP Code"

def zip():
    flag = False
    city_name = input("Enter a city name to lookup: ").title()
    state_name = input("Enter the state name to lookup: ").upper()
    city_name_zips = []
    for i in range(len(zip_codes) - 1):
        if city_name == zip_codes[i][3] and state_name == zip_codes[i][4]:
            flag = True
            city_name_zips.append(zip_codes[i][0])
            print("The following ZIP Code(s) found for", city_name, state_name, ":", ", "
                  .join(city_name_zips))
            break
    if not flag:
        print("No ZIP Code found for: ", city_name, state_name)

def dist():
    zip_code_1 = input("Enter the first ZIP Code: ")
    zip_code_2 = input("Enter the second ZIP Code: ")
    if isinstance(loc(zip_code_1), tuple) \
            and isinstance(loc(zip_code_2), tuple):
        coordinate_zip_code_1 = loc(zip_code_1)[0]
        coordinate_zip_code_2 = loc(zip_code_2)[0]
        distance_miles = round(geodesic(coordinate_zip_code_1, coordinate_zip_code_2).miles, 2)
        print("The distance between", zip_code_1, "and", zip_code_2, "is", distance_miles, "miles")
        return
    else:
        print("The distance between", zip_code_1, "and", zip_code_2, "cannot be determined")


from logging.config import fileConfig
import logging
import os
import time

# Initialize logger
fileConfig('logging_config.ini')
log = logging.getLogger()

def pentaho_server_ce_logs(product_dir):
    log.info('Reading ---Pentaho Server CE logs---')
    exitcode_pentaho_log = 1
    exitcode_catalina_log = 1

    # Files to read:
    # - pentaho.log
    # - catalina.XXX.log
    # Reading pentaho.log
    pentaho_log_file = os.path.join(product_dir, 'pentaho-server', 'tomcat', 'logs', 'pentaho.log')
    log.debug('Reading Pentaho.log at [' + pentaho_log_file + ']')
    fo_pentaho_logs = open(pentaho_log_file, "r")
    read_pentaho_file = fo_pentaho_logs.read().lower()

    if read_pentaho_file.find('error') == -1 and read_pentaho_file.find('exception') == -1:
        log.info('[Pentaho] Server seems started successful.')
        log.debug('[Pentaho] No Error message found')
        exitcode_pentaho_log = 0
    else:
        log.error('[Pentaho] Server started with errors check them:')
        log.info(read_pentaho_file)
    fo_pentaho_logs.close()

    # Reading catalina.log
    today = time.localtime(time.time())
    today_str = time.strftime('%Y-%m-%d', today)
    log.debug('TODAY: ' + today_str)
    catalina_log_file = os.path.join(product_dir, 'pentaho-server', 'tomcat', 'logs', 'catalina.' + today_str + '.log')
    log.debug('Reading catalina.log at [' + catalina_log_file + ']')

    fo_catalina_logs = open(catalina_log_file, "r")
    read_catalina_file = fo_catalina_logs.read().lower()

    if read_catalina_file.find('error') == -1 and read_catalina_file.find('exception') == -1:
        log.info('[Catalina] Server seems started successful.')
        log.debug('[Catalina] No Error message found')
        exitcode_catalina_log = 0
    else:
        log.error('[Catalina] Server started with errors check them:')
        log.info(read_catalina_file)
    fo_catalina_logs.close()

    exit(exitcode_pentaho_log | exitcode_catalina_log)

# -------------------------------------------------------
#
#                READING LOGS of PSW
#
# -------------------------------------------------------
def psw_logs(product_dir, proc_output):
    log.info('Reading ---Pentaho Schema Workbench logs---')

    #read file 1
    psw_log_file = os.path.join(product_dir, 'schemaworkbench.log')
    log.debug('Reading schemaworkbench.log at [' + psw_log_file + ']')
    fo_psw_logs = open(psw_log_file, "r")
    read_psw_file = fo_psw_logs.read().lower()

    exitcode_psw_logs = 0
    if read_psw_file.find('error') == -1 and read_psw_file.find('exception') == -1:
        log.info('[PSW] Schema workbench started successfully.')
        log.debug('[PSW] No Error message found.')
    else:
        log.error('[PSW] Schema workbench started with errors check them:')
        exitcode_psw_logs = -1
    fo_psw_logs.close()

    log.debug('---- 1: BEGIN LOGS ----')
    log.debug(read_psw_file)
    log.debug('---- 1: END LOGS ----')

    # read file 2
    psw_log2_file = os.path.join(product_dir, 'schemaworkbench2.log')
    log.debug('Reading schemaworkbench.log at [' + psw_log2_file + ']')
    fo_psw_logs2 = open(psw_log2_file, "r")
    read_psw_file2 = fo_psw_logs2.read().lower()

    exitcode_psw_logs2 =0
    if read_psw_file2.find('error') == -1 and read_psw_file2.find('exception') == -1:
        log.info('[PSW] Schema workbench started successfully.')
        log.debug('[PSW] No Error message found.')
    else:
        log.error('[PSW] Schema workbench started with errors check them:')
        exitcode_psw_logs2 = -1
    fo_psw_logs2.close()

    log.debug('---- 2: BEGIN LOGS ----')
    log.debug(read_psw_file2)
    log.debug('---- 2: END LOGS ----')

    # read output
    exitcode_psw_logs3 = 0
    if proc_output.find('error') == -1 and proc_output.find('exception') == -1:
        log.info('[PSW] Schema workbench started successfully.')
        log.debug('[PSW] No Error message found.')
    else:
        log.error('[PSW-CE] Schema workbench started with errors check them:')
        exitcode_psw_logs3 = -1

    log.debug('---- 3:  BEGIN LOGS ----')
    log.debug(proc_output)
    log.debug('---- 3: END LOGS ----')

    exit(exitcode_psw_logs | exitcode_psw_logs2 | exitcode_psw_logs3)

# -------------------------------------------------------
#
#                READING LOGS of PME
#
# -------------------------------------------------------
def pme_logs(product_dir, proc_output):
    log.info('Reading ---Pentaho Metadata Editor logs---')

    #read file 1
    pme_log_file = os.path.join(product_dir, 'metadataeditor.log')
    log.debug('Reading metadataeditor.log at [' + pme_log_file + ']')

    pme_log_file_content = ''
    fo_pme_logs = open(pme_log_file, "r")
    # BEGIN - remove this code when we fix this: https://jira.pentaho.com/browse/PMD-932
    read_pme_file = fo_pme_logs.read().lower()
    with open(pme_log_file) as f:
        for line in f:
            myline = line.lower()
            if myline.find('unable to load query') == -1:
                pme_log_file_content += str(myline)
    # END - remove

    exitcode_pme_logs = 0
    if pme_log_file_content.find('error') == -1 and pme_log_file_content.find('exception') == -1:
        log.info('[PME] Metadata Editor started successfully.')
        log.debug('[PME] No Error message found.')
    else:
        log.error('[PME] Metadata Editor started with errors check them:')
        exitcode_pme_logs = -1
    fo_pme_logs.close()

    log.debug('---- 1: BEGIN LOGS ----')
    log.debug(read_pme_file)
    log.debug('---- 1: END LOGS ----')

    # read output
    exitcode_pme_logs2 = 0
    if proc_output.find('error') == -1 and proc_output.find('exception') == -1:
        log.info('[PME] Metadata Editor started successfully.')
        log.debug('[PME] No Error message found.')
    else:
        log.error('[PME] Metadata Editor started with errors check them:')
        exitcode_pme_logs2 = -1

    log.debug('---- 2: BEGIN LOGS ----')
    log.debug(proc_output)
    log.debug('---- 2: END LOGS ----')

    exit(exitcode_pme_logs | exitcode_pme_logs2)

# -------------------------------------------------------
#
#                READING LOGS of PRD
#
# -------------------------------------------------------
def prd_logs(product_dir, proc_output):
    log.info('Reading ---Pentaho Report Designer logs---')

    #read file 1
    prd_log_file = os.path.join(product_dir, 'reportdesigner.log')
    log.debug('Reading reportdesigner.log at [' + prd_log_file + ']')
    fo_prd_logs = open(prd_log_file, "r")
    read_prd_file = fo_prd_logs.read().lower()

    exitcode_prd_logs = 0
    if read_prd_file.find('error') == -1 and read_prd_file.find('exception') == -1:
        log.info('[PRD] Report Designer started successfully.')
        log.debug('[PRD] No Error message found.')
    else:
        log.error('[PRD] Report Designer started with errors check them:')
        exitcode_prd_logs = -1
    fo_prd_logs.close()

    log.debug('---- 1: BEGIN LOGS ----')
    log.debug(read_prd_file)
    log.debug('---- 1: END LOGS ----')

    # read output
    exitcode_prd_logs2 = 0
    if proc_output.find('error') == -1 and proc_output.find('exception') == -1:
        log.info('[PRD] Report Designer started successfully.')
        log.debug('[PRD] No Error message found.')
    else:
        log.error('[PRD] Report Designer started with errors check them:')
        exitcode_prd_logs2 = -1

    log.debug('---- 2: BEGIN LOGS ----')
    log.debug(proc_output)
    log.debug('---- 2: END LOGS ----')

    exit(exitcode_prd_logs | exitcode_prd_logs2)

# -------------------------------------------------------
#
#                READING LOGS of PDI
#
# -------------------------------------------------------
def pdi_logs(product_dir, proc_output):
    log.info('Reading ---Pentaho Data Integration logs---')

    # read file 1
    pdi_log_file = os.path.join(product_dir, 'dataintegration.log')
    log.debug('Reading dataintegration.log at [' + pdi_log_file + ']')
    fo_pdi_logs = open(pdi_log_file, "r")
    read_pdi_file = fo_pdi_logs.read().lower()

    exitcode_pdi_logs = 0
    if read_pdi_file.find('error') == -1 and read_pdi_file.find('exception') == -1:
        log.info('[PDI] Data Integration started successfully.')
        log.debug('[PDI] No Error message found.')
    else:
        log.error('[PDI] Data Integration started with errors check them:')
        exitcode_pdi_logs = -1
    fo_pdi_logs.close()

    log.debug('---- 1: BEGIN LOGS ----')
    log.debug(read_pdi_file)
    log.debug('---- 1: END LOGS ----')

    # read output
    exitcode_pdi_logs2 = 0
    if proc_output.find('error') == -1 and proc_output.find('exception') == -1:
        log.info('[PDI] Data Integration started successfully.')
        log.debug('[PDI] No Error message found.')
    else:
        log.error('[PDI] Data Integration started with errors check them:')
        exitcode_pdi_logs2 = -1

    log.debug('---- 2: BEGIN LOGS ----')
    log.debug(proc_output)
    log.debug('---- 2: END LOGS ----')

    exit(exitcode_pdi_logs | exitcode_pdi_logs2)

# -------------------------------------------------------
#
#                      read_logs
#
# -------------------------------------------------------
def read_logs(product_name, product_dir):
    log.info('Let\'s read logs')
    log.debug('The product name: [' + product_name + ']')
    log.debug('The product dir: [' + product_dir + ']')
    if product_name in 'pentaho-server-ce':
        pentaho_server_ce_logs(product_dir)
    else:
        log.error('NOT SUPPORTED')


# 1125. Smallest Sufficient Team
class Solution:
    def smallestSufficientTeam(self, req_skills: List[str], people: List[List[str]]) -> List[int]:
        '''
        Memoized DFS with state compression using bit operations.

        To clear the k-th bit on an int i, use i & ~(1 << k).
        Note that in Py, 'not' op works on signed int, it may not be clear on
        intermediate result, but works fine on bitwise operations.
        '''
        n = len(req_skills)
        skillsToIdx = { x: i for i, x in enumerate(req_skills) }
        people = [[skillsToIdx[s] for s in skills if s in skillsToIdx] for skills in people]
        skillToPeople = defaultdict(list)
        for pIdx, skillIdxs in enumerate(people):
            for sIdx in skillIdxs:
                skillToPeople[sIdx].append(pIdx)

        @lru_cache(None)
        def dfs(skillsNeeded):
            if skillsNeeded == 0:
                return []
            ans = None
            neededSkillId = next(sIdx for sIdx in range(n) if skillsNeeded & (1 << sIdx))
            for nextPeople in skillToPeople[neededSkillId]:
                tmp = skillsNeeded
                for sIdx in people[nextPeople]:
                    tmp &= ~(1 << sIdx)
                curList = dfs(tmp)
                if ans is None or len(curList) + 1 < len(ans):
                    ans = curList + [nextPeople]
            return ans
        return dfs((1 << n) - 1)


import decimal
import json
from time import sleep
from typing_extensions import assert_type
import allure
import pytest
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(
    os.path.dirname(os.path.abspath(__file__))))))
from Common import Http, Conf, Httpexplore
from Common.Loguru import logger

# 单签账户
@allure.feature("Transfers_Success!")
class Test_transfers_success_atom:
    test_data = [
        # 生产
        ("正常转账(自己转自己)!",["3f6c52cc8b317b9559cc13b3edcf7166eaadba1ca7d06d2848b7d2d530e0ec54"],["023b16c02f0b106c2bec8305d0e64593f7156da6a37dedbd2e85342b8ff500392f"],"ATOM","ATOM","cosmos1ku5klzmup3an5mxva6u9pr8jmhzrapa7lrtukh","cosmos1ku5klzmup3an5mxva6u9pr8jmhzrapa7lrtukh",Conf.Config.random_amount(4)),
        # ("正常转账maximum(自己转自己)!",["3f6c52cc8b317b9559cc13b3edcf7166eaadba1ca7d06d2848b7d2d530e0ec54"],["023b16c02f0b106c2bec8305d0e64593f7156da6a37dedbd2e85342b8ff500392f"],"ATOM","ATOM","cosmos1ku5klzmup3an5mxva6u9pr8jmhzrapa7lrtukh","cosmos1ku5klzmup3an5mxva6u9pr8jmhzrapa7lrtukh","maximum"),
        # ("正常转账!",["3f6c52cc8b317b9559cc13b3edcf7166eaadba1ca7d06d2848b7d2d530e0ec54"],["023b16c02f0b106c2bec8305d0e64593f7156da6a37dedbd2e85342b8ff500392f"],"ATOM","ATOM","cosmos1ku5klzmup3an5mxva6u9pr8jmhzrapa7lrtukh","cosmos1h5rgna0ca8tp9nv0alwx26sa5gdcszd62ln9y0",Conf.Config.random_amount(4)),
    ]

    @allure.story("Transfers_ATOM_Success!")
    @allure.title('单签账户转账-{test_title}')
    @pytest.mark.parametrize('test_title,privatekey,PublicKeys,networkCode,symbol,from_add,to_add,amount', test_data)
    def test_transfers_atom(self,test_title,privatekey,PublicKeys,networkCode,symbol,from_add,to_add,amount):

        with allure.step("查询From账户ATOM余额"):
            holder = Http.HttpUtils.get_holders(networkCode,symbol,from_add)
            assert holder.status_code == 200
            quantity = float(holder.json()[0]['quantity'])

        with allure.step("mintscan浏览器查询From账户ATOM余额"):
            holder_explore = Httpexplore.ATOM.balance(from_add)
            assert holder_explore.status_code == 200
            quantity_explore = int([b.get("amount") for b in holder_explore.json()["balances"] if b.get("denom") == "uatom"][0])/1000000

        with allure.step("余额对比"):
            assert quantity == quantity_explore

        # with allure.step("构建交易——instructions"):
        #     body = {
        #         "from":from_add,
        #         "to":to_add,
        #         "symbol":symbol,
        #         "amount":amount
        #     }
        #     transactionParams = {
        #         "memo":"hahahhahahhahahhaahhahhahhahahaha@@==这是一段描述！！====@@hahahhahahhahahhaahhahhahhahahaha"
        #     }
        #     transfer = Http.HttpUtils.post_instructions("Transfer",body,networkCode,PublicKeys,transactionParams)
        #     assert transfer.status_code == 200

        #     t_estimatedFee = transfer.json()['_embedded']['transactions'][0]['estimatedFee']
        #     t_hash = transfer.json()['_embedded']['transactions'][0]['hash']
        #     t_id = transfer.json()['_embedded']['transactions'][0]['id']
        #     t_networkCode = transfer.json()['_embedded']['transactions'][0]['networkCode']
        #     t_requiredSignings = transfer.json()['_embedded']['transactions'][0]['requiredSignings']
        #     t_serialized = transfer.json()['_embedded']['transactions'][0]['serialized']
        #     t_status = transfer.json()['_embedded']['transactions'][0]['status']
        #     ID = transfer.json()['id']
        #     body_amount = transfer.json()["body"]["amount"]

        # with allure.step("签名交易——sign"):
        #     signature = Conf.Config.sign(privatekey[0],t_requiredSignings[0]['hash'])
        #     signatures = [
        #         {
        #             "hash":t_requiredSignings[0]['hash'],
        #             "publickey":t_requiredSignings[0]['publicKeys'][0],
        #             "signature":signature
        #         }
        #     ]
        #     sig = Http.HttpUtils.post_sign_transfers(t_estimatedFee,t_hash,t_id,t_networkCode,t_requiredSignings,t_serialized,signatures)
        #     assert sig.status_code == 200

        # with allure.step("广播交易——send"):
        #     send = Http.HttpUtils.post_send_transfers(t_id)
        #     assert send.status_code == 200

        # with allure.step("查询交易记录——transfers by ID,交易状态变为:1"):
        #     # 循环查10次交易记录
        #     for i in range(1):
        #         sleep(20)
        #         logger.info("<----查询次数:第" + str(i+1) + "次---->")
        #         t = Http.HttpUtils.get_transactions_byid(ID)
        #         t_status = t.json()["_embedded"]["transactions"][0]["status"] #交易状态
        #         if (t_status == "PENDING" or t_status == "SENT"):
        #             assert t.json()["status"] == -1
        #         elif (t_status == "SETTLED"):
        #             assert t.json()["status"] == 1
        #             break

        # with allure.step("查询关联交易记录——balance-transactions by hash"):
        #     sleep(15)
        #     transcations = Http.HttpUtils.get_transactions_byhash(send.json()["hash"])

        #     if (from_add == to_add):
        #         assert len(transcations.json()) == 3 # 自己转自己3条交易记录
        #     else:
        #         assert len(transcations.json()) == 4 # 转其他地址4条交易记录

        # with allure.step("查询From账户holders信息——holders"):
        #     holders_after = Http.HttpUtils.get_holders(networkCode,symbol,from_add)
        #     assert holders_after.status_code == 200
        #     quantity_after = holders_after.json()[0]['quantity']

        # with allure.step("断言holders转出金额"):
        #     if from_add == to_add:
        #         assert quantity_before == quantity_after
        #     else:
        #         assert decimal.Decimal(body_amount) == decimal.Decimal(quantity_before) - decimal.Decimal(quantity_after)

if __name__ == '__main__':
    path = os.path.abspath(__file__) + ""
    pytest.main(["-vs", path,'--alluredir=Report/Allure'])
    # os.system(f'allure serve /Users/lilong/Documents/Test_Api/Report/Allure')


from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QApplication, QMainWindow
from PyQt5.QtGui import QPixmap
import sys
import requests
import urllib
import ctypes
import os

numpic = 0
genres = ["Action","Adventure","Animation","Comedy","Crime","Documentary","Drama","Family","Fantasy","History","Horror","Music","Mystery","Romance","Sci-Fi","Thriller","War", "Western"]
keys = ["28","12","16","35","80","99","18","10751","14","36","27","10402","9648","10749","878","53","10752", "37"]
ordertype = ["Popularity", "Year", "Rate"]
keys_2 = ["popularity.desc", "release_date.desc", "vote_average.desc"]
dicti = dict(zip(genres,keys))
dicti_2 = dict(zip(ordertype, keys_2))
stype = "title"
curdur = "forward"
curOrder = ""
maxcur = 10
class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(1036, 477)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(MainWindow.sizePolicy().hasHeightForWidth())
        MainWindow.setSizePolicy(sizePolicy)
        MainWindow.setWindowOpacity(1.0)
        MainWindow.setTabShape(QtWidgets.QTabWidget.Rounded)
        MainWindow.setFixedSize(1036, 477)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.lineEdit = QtWidgets.QLineEdit(self.centralwidget)
        self.lineEdit.setGeometry(QtCore.QRect(120, 20, 221, 31))
        self.lineEdit.setInputMask("")
        self.lineEdit.setText("")
        self.lineEdit.setObjectName("lineEdit")
        self.label = QtWidgets.QLabel(self.centralwidget)
        self.label.setGeometry(QtCore.QRect(20, 30, 101, 16))
        self.label.setObjectName("label")
        self.label_5 = QtWidgets.QLabel(self.centralwidget)
        self.label_5.setGeometry(QtCore.QRect(20, 440, 290, 16))
        self.label_5.setObjectName("label_5")
        self.pushButton = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton.setGeometry(QtCore.QRect(350, 20, 93, 28))
        self.pushButton.setFlat(False)
        self.pushButton.setObjectName("pushButton")
        self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_2.setGeometry(QtCore.QRect(10, 410, 841, 28))
        self.pushButton_2.setObjectName("pushButton_2")
        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        self.label_2.setGeometry(QtCore.QRect(20, 60, 811, 331))
        self.label_2.setText("")
        self.label_2.setScaledContents(True)
        self.label_2.setObjectName("label_2")
        self.label_3 = QtWidgets.QLabel(self.centralwidget)
        self.label_3.setGeometry(QtCore.QRect(340, 440, 200, 16))
        self.label_3.setObjectName("label_3")
        self.pushButton_3 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_3.setGeometry(QtCore.QRect(820, 440, 31, 28))
        self.pushButton_3.setObjectName("pushButton_3")
        self.pushButton_4 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_4.setGeometry(QtCore.QRect(790, 440, 31, 28))
        self.pushButton_4.setObjectName("pushButton_4")
        self.comboBox = QtWidgets.QComboBox(self.centralwidget)
        self.comboBox.setGeometry(QtCore.QRect(518, 20, 121, 31))
        self.comboBox.setObjectName("comboBox")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.comboBox.addItem("")
        self.pushButton_5 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_5.setGeometry(QtCore.QRect(750, 20, 93, 28))
        self.pushButton_5.setObjectName("pushButton_5")
        self.label_4 = QtWidgets.QLabel(self.centralwidget)
        self.label_4.setGeometry(QtCore.QRect(460, 30, 55, 16))
        self.label_4.setObjectName("label_4")
        self.line = QtWidgets.QFrame(self.centralwidget)
        self.line.setGeometry(QtCore.QRect(450, 10, 16, 41))
        self.line.setFrameShape(QtWidgets.QFrame.VLine)
        self.line.setFrameShadow(QtWidgets.QFrame.Sunken)
        self.line.setObjectName("line")
        self.listWidget = QtWidgets.QListWidget(self.centralwidget)
        self.listWidget.setGeometry(QtCore.QRect(860, 20, 171, 441))
        self.listWidget.setFrameShape(QtWidgets.QFrame.StyledPanel)
        self.listWidget.setObjectName("listWidget")
        self.comboBox_2 = QtWidgets.QComboBox(self.centralwidget)
        self.comboBox_2.setGeometry(QtCore.QRect(650, 20, 91, 31))
        self.comboBox_2.setObjectName("comboBox_2")
        self.comboBox_2.addItem("")
        self.comboBox_2.addItem("")
        self.comboBox_2.addItem("")
        MainWindow.setCentralWidget(self.centralwidget)

        self.retranslateUi(MainWindow)
        self.comboBox.setCurrentIndex(0)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "TY - Background Changer"))
        self.label.setText(_translate("MainWindow", "Name Of Movie :"))
        self.pushButton.setText(_translate("MainWindow", "Search"))
        self.pushButton.clicked.connect(self.b1clicked)
        self.pushButton_2.setText(_translate("MainWindow", "Set as Desktop Background Picture"))
        self.label_3.setText(_translate("MainWindow", ""))
        self.label_5.setText(_translate("MainWindow", ""))
        self.pushButton_2.clicked.connect(self.b2clicked)
        self.pushButton_2.setEnabled(False)
        self.pushButton_3.setEnabled(False)
        self.pushButton_4.setEnabled(False)
        self.pushButton_3.setText(_translate("MainWindow", ">"))
        self.pushButton_3.clicked.connect(self.b3clicked)
        self.pushButton_4.setText(_translate("MainWindow", "<"))
        self.pushButton_4.clicked.connect(self.b4clicked)
        self.comboBox.setCurrentText(_translate("MainWindow", "Action"))
        self.comboBox.setItemText(0, _translate("MainWindow", "Action"))
        self.comboBox.setItemText(1, _translate("MainWindow", "Adventure"))
        self.comboBox.setItemText(2, _translate("MainWindow", "Animation"))
        self.comboBox.setItemText(3, _translate("MainWindow", "Comedy"))
        self.comboBox.setItemText(4, _translate("MainWindow", "Crime"))
        self.comboBox.setItemText(5, _translate("MainWindow", "Documentary"))
        self.comboBox.setItemText(6, _translate("MainWindow", "Drama"))
        self.comboBox.setItemText(7, _translate("MainWindow", "Family"))
        self.comboBox.setItemText(8, _translate("MainWindow", "Fantasy"))
        self.comboBox.setItemText(9, _translate("MainWindow", "History"))
        self.comboBox.setItemText(10, _translate("MainWindow", "Horror"))
        self.comboBox.setItemText(11, _translate("MainWindow", "Music"))
        self.comboBox.setItemText(12, _translate("MainWindow", "Mystery"))
        self.comboBox.setItemText(13, _translate("MainWindow", "Romance"))
        self.comboBox.setItemText(14, _translate("MainWindow", "Sci-Fi"))
        self.comboBox.setItemText(15, _translate("MainWindow", "Thriller"))
        self.comboBox.setItemText(16, _translate("MainWindow", "War"))
        self.comboBox.setItemText(17, _translate("MainWindow", "Western"))
        self.pushButton_5.setText(_translate("MainWindow", "Search"))
        self.pushButton_5.clicked.connect(self.b5clicked)
        self.label_4.setText(_translate("MainWindow", "Genres :"))
        self.listWidget.itemClicked.connect(self.itemClicked)
        self.comboBox_2.setItemText(0, _translate("MainWindow", "Popularity"))
        self.comboBox_2.setItemText(1, _translate("MainWindow", "Year"))
        self.comboBox_2.setItemText(2, _translate("MainWindow", "Rate"))
        self.lineEdit.returnPressed.connect(self.b1clicked)

    def b1clicked(self):
        global stype, numpic
        numpic = 0
        stype = "title"
        self.listWidget.clear()
        self.pushButton_4.setEnabled(False)
        self.executee("https://api.themoviedb.org/3/search/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&query="+ self.lineEdit.text() +"&page=1&include_adult=false")

    def b5clicked(self):
        global dicti, stype, numpic, dicti_2, curOrder
        numpic = 0
        stype = "genres"
        self.listWidget.clear()
        curGenre = dicti[str(self.comboBox.currentText())]
        curOrder = dicti_2[str(self.comboBox_2.currentText())]
        self.pushButton_4.setEnabled(False)
        self.executee("https://api.themoviedb.org/3/discover/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&sort_by="+ curOrder +"&include_adult=false&include_video=false&page=1&with_genres=" + curGenre)

    def b2clicked(self):
        ctypes.windll.user32.SystemParametersInfoW(20, 0, os.getcwd() + '\\resim.jpg', 0)

    def b3clicked(self):
        global numpic,stype, curdur, dicti, curOrder, maxcur
        curdur = "forward"
        numpic += 1
        self.pushButton_4.setEnabled(True)
        if stype == "title":
            self.executee("https://api.themoviedb.org/3/search/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&query="+ self.lineEdit.text() +"&page=1&include_adult=false")
        else:
            curGenre = dicti[str(self.comboBox.currentText())]
            self.executee("https://api.themoviedb.org/3/discover/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&sort_by="+ curOrder +"&include_adult=false&include_video=false&page=1&with_genres=" + curGenre)

        if numpic == maxcur:
            self.pushButton_3.setEnabled(False)

    def b4clicked(self):
        global numpic,stype, curdur,dicti,curOrder
        curdur = "back"
        numpic -= 1
        if stype == "title":
            self.executee("https://api.themoviedb.org/3/search/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&query="+ self.lineEdit.text() +"&page=1&include_adult=false")
        else:
            curGenre = dicti[str(self.comboBox.currentText())]
            self.executee("https://api.themoviedb.org/3/discover/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&sort_by="+ curOrder +"&include_adult=false&include_video=false&page=1&with_genres=" + curGenre)

        self.pushButton_3.setEnabled(True)
        if numpic == 0:
            self.pushButton_4.setEnabled(False)

    def itemClicked(self):
        global numpic,dicti,curOrder
        numpic = self.listWidget.currentRow()
        if stype == "title":
            self.executee("https://api.themoviedb.org/3/search/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&query="+ self.lineEdit.text() +"&page=1&include_adult=false")
        else:
            curGenre = dicti[str(self.comboBox.currentText())]
            self.executee("https://api.themoviedb.org/3/discover/movie?api_key=f074077eac6b13f13cd586d06af0d486&language=en-US&sort_by="+ curOrder +"&include_adult=false&include_video=false&page=1&with_genres=" + curGenre)

    def executee(self, curUrl):
        try:
            self.listWidget.clear()
            response = requests.get(curUrl)
            getresults = response.json()['results']
            pictures = []
            titles = []
            global numpic, curdur, maxcur
            for d in getresults:
                try:
                    pic = d['backdrop_path']
                    pictures.append(pic)
                except :
                    pictures.append(None)

                try:
                    title = d['title']
                except :
                    try:
                        title = d['name']
                    except :
                        title = d['original_title']

                titles.append(title)
                try:
                    self.listWidget.addItem(d['title'])
                except :
                    try:
                        title = d['name']
                    except :
                        title = d['original_title']

            while pictures[numpic] is None:
                if curdur == "back":
                    numpic -= 1
                else:
                    if numpic < maxcur:
                        numpic += 1

            maxcur = len(pictures)
            url = 'https://image.tmdb.org/t/p/original' + pictures[numpic]
            data = urllib.request.urlopen(url).read()
            urllib.request.urlretrieve(url,"resim.jpg")
            global pixmap
            pixmap = QPixmap()
            pixmap.loadFromData(data)

            self.label_2.setPixmap(pixmap)
            self.label_5.setText(titles[numpic])
            self.pushButton_2.setEnabled(True)
            self.pushButton_3.setEnabled(True)
            self.label_3.setText("")
        except :
            self.label_3.setText("Error !. Please enter valid value")
            self.pushButton_2.setEnabled(False)
            self.pushButton_3.setEnabled(False)
            self.pushButton_4.setEnabled(False)
            self.label_2.clear()
            self.lineEdit.setFocus()

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    sys.exit(app.exec_())


"""Data processor for SMP-CAIL2020-Argmine.

Author: Yixu GAO (yxgao19@fudan.edu.cn)

In data file, each line contains 1 sc sentence and 5 bc sentences.
The data processor convert each line into 5 samples,
each sample with 1 sc sentence and 1 bc sentence.

Usage:
1. Tokenizer (used for RNN model):
    from data import Tokenizer
    vocab_file = 'vocab.txt'
    sentence = '我饿了，想吃东西了。'
    tokenizer = Tokenizer(vocab_file)
    tokens = tokenizer.tokenize(sentence)
    # ['我', '饿', '了', '，', '想', '吃', '东西', '了', '。']
    ids = tokenizer.convert_tokens_to_ids(tokens)
2. Data:
    from data import Data
    # For training, load train and valid set
    # For BERT model
    data = Data('model/bert/vocab.txt', model_type='bert')
    datasets = data.load_train_and_valid_files(
        'SMP-CAIL2020-train.csv', 'SMP-CAIL2020-valid.csv')
    train_set, valid_set_train, valid_set_valid = datasets
    # For RNN model
    data = Data('model/rnn/vocab.txt', model_type='rnn')
    datasets = data.load_all_files(
        'SMP-CAIL2020-train.csv', 'SMP-CAIL2020-valid.csv')
    train_set, valid_set_train, valid_set_valid = datasets
    # For testing, load test set
    data = Data('model/bert/vocab.txt', model_type='bert')
    test_set = data.load_file('SMP-CAIL2020-test.csv', train=False)
"""
import itertools
import os
# from itertools import islice
from typing import List, Dict, Tuple
# import jieba
import lawa
# import nltk
import torch
from dataclasses import dataclass

import pandas as pd
from gensim.models.word2vec import LineSentence
from torch.nn.utils.rnn import pad_sequence

from torch.utils.data import TensorDataset, Dataset
from transformers import BertTokenizer
from tqdm import tqdm
from difflib import SequenceMatcher
import logging
logger = logging.getLogger(__name__)

class Tokenizer:
    """Tokenizer for Chinese given vocab.txt.

    Attributes:
        dictionary: Dict[str, int], {<word>: <index>}
    """
    def __init__(self, language='en', vocab_file='vocab.txt'):
        """Initialize and build dictionary.

        Args:
            vocab_file: one word each line
        """
        self.language = language
        self.dictionary = {'[PAD]': 0, '[UNK]': 1}
        self._pad_token = '[PAD]'
        self.pad_token_id = 0
        count = 2
        with open(vocab_file, encoding='utf-8') as fin:
            for line in fin:
                word = line.strip()
                self.dictionary[word] = count
                count += 1

    def __len__(self):
        return len(self.dictionary)

    def tokenize(self, sentence: str) -> List[str]:
        """Cut words for a sentence.

        Args:
            sentence: sentence

        Returns:
            words list
        """
        if self.language == 'zh':
            words = lawa.lcut(sentence)
        else:  # 'en'
            words = nltk.word_tokenize(sentence)
        return words

    def convert_tokens_to_ids(
            self, tokens_list: List[str]) -> List[int]:
        """Convert tokens to ids.

        Args:
            tokens_list: word list

        Returns:
            index list
        """
        return [self.dictionary.get(w, 1) for w in tokens_list]

class LineByLineTextDataset(Dataset):
    """
    This will be superseded by a framework-agnostic approach
    soon.
    """

    def __init__(self, tokenizer, file_path: str, block_size: int):
        assert os.path.isfile(file_path)
        # Here, we do not cache the features, operating under the assumption
        # that we will soon use fast multithreaded tokenizers from the
        # `tokenizers` repo everywhere =)
        logger.info("Creating features from dataset file at %s", file_path)

        alines = LineSentence(file_path, limit=block_size)
        self.examples = [tokenizer.convert_tokens_to_ids(line) for line in alines]
        # with open(file_path, encoding="utf-8") as f:
        #     rawlines = []
        #     for line in islice(f, block_size):
        #         line = line.strip()
        #         if (len(line) > 0 and not line.isspace()):
        #             rawlines.append(line)
        #     # lines = [line for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]
        #     alines = [list(tokenizer.tokenize(line)) for line in rawlines]
        #     batch_encoding = [tokenizer.convert_tokens_to_ids(line) for line in alines]
        #     self.examples = batch_encoding

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, i) -> torch.Tensor:
        return torch.tensor(self.examples[i], dtype=torch.long)

@dataclass
class DataCollatorForLanguageModeling:
    """
    Data collator used for language modeling.
    - collates batches of tensors, honoring their tokenizer's pad_token
    - preprocesses batches for masked language modeling
    """

    tokenizer: Tokenizer
    mlm: bool = True
    mlm_probability: float = 0.15
    window_size = 5

    def __call__(self, examples: List[torch.Tensor]) -> Dict[str, torch.Tensor]:
        batch = self._tensorize_batch(examples)
        if self.mlm:
            inputs, labels = self.mask_tokens(batch)
            return {"input_ids": inputs, "labels": labels}
        else:
            labels = batch.clone().detach()
            labels[labels == self.tokenizer.pad_token_id] = -100
            return {"input_ids": batch, "labels": labels}

    def _tensorize_batch(self, examples: List[torch.Tensor]) -> torch.Tensor:
        length_of_first = examples[0].size(0)
        are_tensors_same_length = all(x.size(0) == length_of_first for x in examples)
        if are_tensors_same_length:
            return torch.stack(examples, dim=0)
        else:
            if self.tokenizer._pad_token is None:
                raise ValueError(
                    "You are attempting to pad samples but the tokenizer you are using"
                    f" ({self.tokenizer.__class__.__name__}) does not have one."
                )
            return pad_sequence(examples, batch_first=True, padding_value=self.tokenizer.pad_token_id)

    def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
        """
        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
        """

        # if self.tokenizer.mask_token is None:
        #     raise ValueError(
        #         "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the --mlm flag if you want to use this tokenizer."
        #     )

        context_l = []
        for w in range(self.window_size):
            labels = inputs.clone()

            prev_labels = torch.cat([labels[:,w:], labels[:,0:w]], dim=1)
            next_labels = torch.cat([labels[:,-w:], labels[:,0:-w]], dim=1)
            context_l.extend([prev_labels, next_labels])

        # randoms = inputs.clone()
        # indices_random = torch.bernoulli(torch.full(labels.shape)).bool()
        # random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
        # randoms[indices_random] = random_words[indices_random]

        # # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
        # probability_matrix = torch.full(labels.shape, self.mlm_probability)
        # # special_tokens_mask = [
        # #     self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()
        # # ]
        # # probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)
        # if self.tokenizer._pad_token is not None:
        #     padding_mask = labels.eq(self.tokenizer.pad_token_id)
        #     probability_matrix.masked_fill_(padding_mask, value=0.0)
        # masked_indices = torch.bernoulli(probability_matrix).bool()
        # labels[~masked_indices] = -100  # We only compute loss on masked tokens
        #
        # # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
        # indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices
        # inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
        #
        # # 10% of the time, we replace masked input tokens with random word
        # indices_random = torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices & ~indices_replaced
        # random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
        # inputs[indices_random] = random_words[indices_random]

        # The rest of the time (10% of the time) we keep the masked input tokens unchanged

        inputs = torch.cat([inputs] * 2 * self.window_size, dim=0)
        labels = torch.cat(context_l, dim=0)
        inputs = inputs.flatten()
        labels = labels.flatten()
        return inputs, labels

class Data:
    """Data processor for BERT and RNN model for SMP-CAIL2020-Argmine.

    Attributes:
        model_type: 'bert' or 'rnn'
        max_seq_len: int, default: 512
        tokenizer:  BertTokenizer for bert
                    Tokenizer for rnn
    """
    def __init__(self,
                 vocab_file='',
                 max_seq_len: int = 512,
                 model_type: str = 'bert', config=None):
        """Initialize data processor for SMP-CAIL2020-Argmine.

        Args:
            vocab_file: one word each line
            max_seq_len: max sequence length, default: 512
            model_type: 'bert' or 'rnn'
                If model_type == 'bert', use BertTokenizer as tokenizer
                Otherwise, use Tokenizer as tokenizer
        """
        self.model_type = model_type

        if self.model_type == 'bert':
            self.tokenizer = BertTokenizer.from_pretrained(config.bert_model_path)#BertTokenizer(vocab_file)
        else:  # rnn
            self.tokenizer = Tokenizer(vocab_file=vocab_file)
        self.max_seq_len = max_seq_len
        self.window_size = config.context_window
        self.max_corpus_line = config.max_corpus_line

    def load_file(self,
                  file_path='SMP-CAIL2020-train.csv',
                  train=True) -> TensorDataset:
        """Load SMP-CAIL2020-Argmine train file and construct TensorDataset.

        Args:
            file_path: train file with last column as label
            train:
                If True, train file with last column as label
                Otherwise, test file without last column as label

        Returns:
            BERT model:
            Train:
                torch.utils.data.TensorDataset
                    each record: (input_ids, input_mask, segment_ids, label)
            Test:
                torch.utils.data.TensorDataset
                    each record: (input_ids, input_mask, segment_ids)
            RNN model:
            Train:
                torch.utils.data.TensorDataset
                    each record: (s1_ids, s2_ids, s1_length, s2_length, label)
            Test:
                torch.utils.data.TensorDataset
                    each record: (s1_ids, s2_ids, s1_length, s2_length)
        """
        sc_list, bc_list, label_list = self._load_file(file_path, train)
        if self.model_type == 'bert':
            dataset = self._convert_sentence_pair_to_bert_dataset(
                sc_list, bc_list, label_list)
        else:  # rnn
            dataset = self._convert_sentence_pair_to_rnn_dataset(
                sc_list, bc_list, label_list)
        return dataset

    def load_train_and_valid_files(self, train_file):
        """Load all files for SMP-CAIL2020-Argmine.

        Args:
            train_file, valid_file: files for SMP-CAIL2020-Argmine

        Returns:
            train_set, valid_set_train, valid_set_valid
            all are torch.utils.data.TensorDataset
        """
        print('Loading train records for train...')
        train_set = LineByLineTextDataset(
            tokenizer=self.tokenizer,
            file_path=train_file,
            block_size=self.max_corpus_line,
        )
        collate_fn = DataCollatorForLanguageModeling(
            tokenizer=self.tokenizer, mlm=True, mlm_probability=0.15
        )
        # train_set = self.load_file(train_file, True)
        print(len(train_set), 'training records loaded.')
        # print('Loading train records for valid...')
        # valid_set_train = self.load_file(train_file, False)
        # print(len(valid_set_train), 'train records loaded.')
        # print('Loading valid records...')
        # valid_set_valid = self.load_file(valid_file, False)
        # print(len(valid_set_valid), 'valid records loaded.')
        return train_set, collate_fn

    def dynamic_fit_bert_size(self, sc_col, bc_col, max_seq_len):
        if len(sc_col) > max_seq_len // 2 + 1:
            sc_col = sc_col[:max_seq_len//2 + 1]
        if len(bc_col) > max_seq_len // 2:
            bc_col = bc_col[:max_seq_len//2]
        # if len(sc_col) + len(bc_col) > max_seq_len:
        #     sc_summary, bc_summary = self.summarizer.summarize([sc_col, bc_col])
        #     ratio = (len(sc_summary) + len(bc_summary)) * 1.0 / max_seq_len
        #     sc_dest = int(len(sc_summary) // ratio)
        #     bc_dest = int(len(bc_summary) // ratio)
        #     if len(sc_summary) > sc_dest:
        #         x = (len(sc_summary) - sc_dest//3 * 2)//2
        #         sc_summary = sc_summary[:sc_dest//3] +sc_summary[x:x+sc_dest//3] + sc_summary[-sc_dest//3:]
        #     if len(bc_summary) > bc_dest:
        #         x = (len(bc_summary) - sc_dest // 3 * 2) // 2
        #         bc_summary = bc_summary[:bc_dest//3] + bc_summary[x:x+bc_dest//3] +  bc_summary[-bc_dest//3:]
        #     return sc_summary, bc_summary
        return sc_col, bc_col

    def _load_file(self, filename, train: bool = True):
        """Load SMP-CAIL2020-Argmine train/test file.

        For train file,
        The ratio between positive samples and negative samples is 1:4
        Copy positive 3 times so that positive:negative = 1:1

        Args:
            filename: SMP-CAIL2020-Argmine file
            train:
                If True, train file with last column as label
                Otherwise, test file without last column as label

        Returns:
            sc_list, bc_list, label_list with the same length
            sc_list, bc_list: List[List[str]], list of word tokens list
            label_list: List[int], list of labels
        """
        data_frame = pd.read_csv(filename)
        sc_list, bc_list, label_list = [], [], []
        for row in data_frame.itertuples(index=False):
            candidates = row[3:8]
            answer = int(row[-1]) if train else None

            for i, _ in enumerate(candidates):
                sc_summary, bc_summary = self.dynamic_fit_bert_size(row[2], candidates[i], self.max_seq_len)
                sc_tokens = self.tokenizer.tokenize(sc_summary)
                bc_tokens = self.tokenizer.tokenize(bc_summary)

                if train:
                    answer_line = candidates[answer - 1]
                    if len(answer_line) > self.max_seq_len // 2:
                        answer_line = answer_line[:self.max_seq_len // 2]

                    if i + 1 == answer:
                        # Copy positive sample 4 times
                        for _ in range(len(candidates) - 1):
                            sc_list.append(sc_tokens)
                            bc_list.append(bc_tokens)
                            label_list.append(1)
                    else:
                        sc_list.append(sc_tokens)
                        bc_list.append(bc_tokens)

                        #答案相近则标记1
                        meter = SequenceMatcher(None, answer_line, bc_summary).ratio()
                        if meter >= 0.90:
                            label_list.append(1)
                        else:
                            label_list.append(0)

                else:  # test
                    sc_list.append(sc_tokens)
                    bc_list.append(bc_tokens)
        return sc_list, bc_list, label_list

    def _convert_sentence_pair_to_bert_dataset(
            self, s1_list, s2_list, label_list=None):
        """Convert sentence pairs to dataset for BERT model.

        Args:
            sc_list, bc_list: List[List[str]], list of word tokens list
            label_list: train: List[int], list of labels
                        test: []

        Returns:
            Train:
                torch.utils.data.TensorDataset
                    each record: (input_ids, input_mask, segment_ids, label)
            Test:
                torch.utils.data.TensorDataset
                    each record: (input_ids, input_mask, segment_ids)
        """
        all_input_ids, all_input_mask, all_segment_ids = [], [], []
        for i, _ in tqdm(enumerate(s1_list), ncols=80):
            CONST_LEN = 3 # CLS SEP SEP == 3
            tokens = ['[CLS]'] + s1_list[i] + ['[SEP]']
            segment_ids = [0] * len(tokens)
            tokens += s2_list[i] + ['[SEP]']
            segment_ids += [1] * (len(s2_list[i]) + 1)
            if len(tokens) > self.max_seq_len + CONST_LEN:
                tokens = tokens[:self.max_seq_len + CONST_LEN]
                assert len(tokens) == self.max_seq_len + CONST_LEN
                segment_ids = segment_ids[:self.max_seq_len + CONST_LEN]
            input_ids = self.tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)
            tokens_len = len(input_ids)
            input_ids += [0] * (self.max_seq_len + CONST_LEN - tokens_len)
            segment_ids += [0] * (self.max_seq_len + CONST_LEN - tokens_len)
            input_mask += [0] * (self.max_seq_len + CONST_LEN - tokens_len)
            all_input_ids.append(input_ids)
            all_input_mask.append(input_mask)
            all_segment_ids.append(segment_ids)
        all_input_ids = torch.tensor(all_input_ids, dtype=torch.long)
        all_input_mask = torch.tensor(all_input_mask, dtype=torch.long)
        all_segment_ids = torch.tensor(all_segment_ids, dtype=torch.long)
        if label_list:  # train
            all_label_ids = torch.tensor(label_list, dtype=torch.long)
            return TensorDataset(
                all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
        # test
        return TensorDataset(
            all_input_ids, all_input_mask, all_segment_ids)

    def _convert_sentence_pair_to_rnn_dataset(
            self, s1_list, s2_list, label_list=None):
        """Convert sentences pairs to dataset for RNN model.

        Args:
            sc_list, bc_list: List[List[str]], list of word tokens list
            label_list: train: List[int], list of labels
                        test: []

        Returns:
            Train:
            torch.utils.data.TensorDataset
                each record: (s1_ids, s2_ids, s1_length, s2_length, label)
            Test:
            torch.utils.data.TensorDataset
                each record: (s1_ids, s2_ids, s1_length, s2_length, label)
        """
        all_s1_ids, all_s2_ids = [], []
        all_s1_lengths, all_s2_lengths = [], []
        for i in tqdm(range(len(s1_list)), ncols=80):
            tokens_s1, tokens_s2 = s1_list[i], s2_list[i]
            all_s1_lengths.append(min(len(tokens_s1), self.max_seq_len))
            all_s2_lengths.append(min(len(tokens_s2), self.max_seq_len))
            if len(tokens_s1) > self.max_seq_len:
                tokens_s1 = tokens_s1[:self.max_seq_len]
            if len(tokens_s2) > self.max_seq_len:
                tokens_s2 = tokens_s2[:self.max_seq_len]
            s1_ids = self.tokenizer.convert_tokens_to_ids(tokens_s1)
            s2_ids = self.tokenizer.convert_tokens_to_ids(tokens_s2)
            if len(s1_ids) < self.max_seq_len:
                s1_ids += [0] * (self.max_seq_len - len(s1_ids))
            if len(s2_ids) < self.max_seq_len:
                s2_ids += [0] * (self.max_seq_len - len(s2_ids))
            all_s1_ids.append(s1_ids)
            all_s2_ids.append(s2_ids)
        all_s1_ids = torch.tensor(all_s1_ids, dtype=torch.long)
        all_s2_ids = torch.tensor(all_s2_ids, dtype=torch.long)
        all_s1_lengths = torch.tensor(all_s1_lengths, dtype=torch.long)
        all_s2_lengths = torch.tensor(all_s2_lengths, dtype=torch.long)
        if label_list:  # train
            all_label_ids = torch.tensor(label_list, dtype=torch.long)
            return TensorDataset(
                all_s1_ids, all_s2_ids, all_s1_lengths, all_s2_lengths,
                all_label_ids)
        # test
        return TensorDataset(
            all_s1_ids, all_s2_ids, all_s1_lengths, all_s2_lengths)

def test_data():
    """Test for data module."""
    # For BERT model
    data = Data('model/bert/vocab.txt', model_type='bert')
    _, _, _ = data.load_train_and_valid_files(
        'SMP-CAIL2020-train.csv',
        'SMP-CAIL2020-test1.csv')
    # For RNN model
    data = Data('model/rnn/vocab.txt', model_type='rnn')
    _, _, _ = data.load_train_and_valid_files(
        'SMP-CAIL2020-train.csv',
        'SMP-CAIL2020-test1.csv')

if __name__ == '__main__':
    test_data()


import socket

HOST = "" #Server address
PORT = 9001 #Server port

with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
	s.connect((HOST, PORT))
	s.sendall(bytes("Sé que no es el canal mas seguro, pero con todo el jaleo que llevamos he perdido la contraseña del servidor... ¿Puedes mandarmela?\n", "utf-8"))
	data = s.recv(1024)
	print(data.decode("utf-8"))

	s.sendall(bytes("La tenía guardada en otro ordenador. Mandamela por aquí mismo.\n", "utf-8"))
	data = s.recv(1024)
	print(data.decode("utf-8"))

	s.sendall(bytes("Gracias, ¿nos vemos el martes como siempre, no?\n", "utf-8"))
	data = s.recv(1024)
	print(data.decode("utf-8"))


from  datetime import datetime
from flask import Blueprint, request, jsonify, url_for, render_template
from app.database.db import get_db
# from apscheduler.schedulers.background import BackgroundScheduler
from pymongo.errors import PyMongoError

from app.form.reminder import run_scheduler

run_scheduler()

form = Blueprint('form', __name__)

# REST API
@form.route('/request_forms', methods=['POST'])
def request_forms():
	db = get_db()
	params = request.get_json()
	forms = list(db.forms.find({'username': params['username']}).sort('createdAt'))

	return jsonify(status=True, message='success', data=forms)

@form.route('/upsert_form', methods=['POST'])
def upsert_form():
	db = get_db()
	params = request.get_json()
	params['attrs']['createdAt'] = datetime.utcnow()
	try:
		result = db.forms.update_one({'_id': params['formId']}, {'$set':params['attrs']}, upsert=True)
		if result.upserted_id is not None:
			return jsonify(status=True, message='1 form created.')
		elif result.modified_count is 1:
			return jsonify(status=True, message='1 form updated.')
		else:
			return jsonify(status=False, message='Upsert with id = {0}  failed.'.format(params['formId']))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

@form.route('/delete_form', methods=['POST'])
def delete_form():
	db = get_db()
	params = request.get_json()
	try:
		result = db.forms.delete_one({'_id': params['formId']})
		return jsonify(status=result.deleted_count==1, message='{0} form deleted.'.format(result.deleted_count))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

@form.route('/request_form_content', methods=['POST'])
def request_form_content():
	db = get_db()
	params = request.get_json()
	try:
		result = db.forms.find_one({'_id':params['formId']})
		if result is None:
			return jsonify(status=False, message='Could not find the form with ID = {0}.'.format(params['formId']))

		# the user is not the author
		if ('public' not in result or result['public'] is False) \
			and params['username'] != result['username']:
			return jsonify(status=False, message='{0} do not have access to {1}.'.format(params['username'], params['formId']))

		# find questions
		questions = list(db.questions.find({'formId':params['formId']}).sort('createdAt'))

		# find all options
		options = []
		for question in questions:
			options += list(db.options.find({'questionId':question['id']}).sort('createdAt'))

		# find all drawings
		drawings = list(db.drawings.find({'parentId':params['formId']}).sort('createdAt'))
		for option in options:
			drawings += list(db.drawings.find({'parentId':option['id']}).sort('createdAt'))

		# find all responses
		responses = []
		if (params['viewMode']==False):
			responses = list(db.responses.find({'formId':params['formId']}).sort('createdAt'))

		data = {'form':result, 'questions':questions, 'options':options, \
			'drawings':drawings, 'responses':responses}
		return jsonify(status=True, message='Successfully fetched the form content.', data=data)
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

## ---------- question ------------ ##
@form.route('/upsert_question', methods=['POST'])
def upsert_question():
	db = get_db()
	params = request.get_json()
	params['attrs']['createdAt'] = datetime.utcnow()
	try:
		result = db.questions.update_one({'_id': params['questionId']}, {'$set':params['attrs']}, upsert=True)
		if result.upserted_id is not None:
			return jsonify(status=True, message='1 question created.')
		elif result.modified_count is 1:
			return jsonify(status=True, message='1 question updated.')
		else:
			return jsonify(status=False, message='Upsert with id = {0}  failed.'.format(params['questionId']))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

@form.route('/delete_question', methods=['POST'])
def delete_question():
	db = get_db()
	params = request.get_json()
	try:
		result = db.questions.delete_one({'_id': params['questionId']})
		return jsonify(status=result.deleted_count==1, message='{0} question deleted.'.format(result.deleted_count))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

## ---------- option ------------ ##
@form.route('/upsert_option', methods=['POST'])
def upsert_option():
	db = get_db()
	params = request.get_json()
	params['attrs']['createdAt'] = datetime.utcnow()
	try:
		result = db.options.update_one({'_id': params['optionId']}, {'$set':params['attrs']}, upsert=True)
		print(result.raw_result)
		if result.upserted_id is not None:
			return jsonify(status=True, message='1 option created.')
		elif result.modified_count is 1:
			return jsonify(status=True, message='1 option updated.')
		else:
			return jsonify(status=False, message='Upsert with id = {0}  failed.'.format(params['optionId']))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

@form.route('/delete_option', methods=['POST'])
def delete_option():
	db = get_db()
	params = request.get_json()
	try:
		result = db.options.delete_one({'_id': params['optionId']})
		return jsonify(status=result.deleted_count==1, message='{0} option deleted.'.format(result.deleted_count))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

## ---------- drawing ------------ ##
@form.route('/upsert_drawing', methods=['POST'])
def upsert_drawing():
	db = get_db()
	params = request.get_json()
	params['attrs']['createdAt'] = datetime.utcnow()
	try:
		result = db.drawings.update_one({'_id': params['drawingId']}, {'$set':params['attrs']}, upsert=True)
		if result.upserted_id is not None:
			return jsonify(status=True, message='1 drawing created.')
		elif result.modified_count is 1:
			return jsonify(status=True, message='1 drawing updated.')
		else:
			return jsonify(status=False, message='Upsert with id = {0}  failed.'.format(params['drawingId']))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

@form.route('/delete_drawing', methods=['POST'])
def delete_drawing():
	db = get_db()
	params = request.get_json()
	try:
		result = db.drawings.delete_one({'_id': params['drawingId']})
		return jsonify(status=result.deleted_count==1, message='{0} drawing deleted.'.format(result.deleted_count))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

## ---------- response ------------ ##
@form.route('/upsert_response', methods=['POST'])
def upsert_response():
	db = get_db()
	params = request.get_json()
	params['attrs']['createdAt'] = datetime.utcnow()
	try:
		result = db.responses.update_one({'_id': params['responseId']}, {'$set':params['attrs']}, upsert=True)
		if result.upserted_id is not None:
			return jsonify(status=True, message='1 response created.')
		elif result.modified_count is 1:
			return jsonify(status=True, message='1 response updated.')
		else:
			return jsonify(status=False, message='Upsert with id = {0}  failed.'.format(params['responseId']))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))

@form.route('/delete_response', methods=['POST'])
def delete_response():
	db = get_db()
	params = request.get_json()
	try:
		result = db.responses.delete_one({'_id': params['responseId']})
		return jsonify(status=result.deleted_count==1, message='{0} response deleted.'.format(result.deleted_count))
	except PyMongoError as e:
		return jsonify(status=False, message=str(e))


from email.mime import image

import pandas as pd
import numpy as np
import pickle

from django.shortcuts import render
from django.http import JsonResponse

# Create your views here.
from .forms import CAMFileForm,LimeTextFileForm
from django.views.decorators.csrf import csrf_exempt
from transparencytest.settings import BASE_DIR

folder_path = BASE_DIR/'unstructureddata//static//unstructureddata'

def transparency_image(request):
    if request.method == 'POST':
        pass
    else:
        cam_form = CAMFileForm()
    return render(request, 'unstructureddata/transparency_image.html', {'cam_form': cam_form})

def transparency_text(request):
    if request.method == 'POST':
        pass
    else:
        limetext_form = LimeTextFileForm()
    return render(request, 'unstructureddata/transparency_text.html', {'limetext_form':limetext_form})

def start_process(model_file_path, image_file_name, unique_id):
    import subprocess, os

    cmds = os.environ.get('_CONDA_ROOT', "C:/Users/deepak.upman/Miniconda3/") + f"Scripts\\activate\nconda activate app_tf2\npython {BASE_DIR}//unstructureddata//cam.py --model_file_path {model_file_path} --image_file_name {image_file_name} --folder_path {folder_path} --unique_id {unique_id}\n"
    cmd = subprocess.Popen("cmd.exe", universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    output, errors = cmd.communicate(cmds)
    cmd.wait()
    return output

@csrf_exempt
def camsubmit(request):
    import os, glob
    from datetime import timedelta, datetime as dt
    if request.method == 'POST':
        response = {}
        try:
            dir_to_clean = f"{folder_path}/cam_images/"
            list_of_files = filter(os.path.isfile, glob.glob(dir_to_clean + '*'))
            for filename in list_of_files:
                print(filename)
                datetime_obj = dt.strptime("_".join(filename.split("\\")[-1].split(".")[0].split("_")[:2]), "%m-%d-%y_%H-%M-%S")
                yesterday = dt.now() - timedelta(days = 1 )
                if (datetime_obj <= yesterday):
                    os.remove(filename)
        except:
            print("Unable to delete files....")
            pass
        try:
            model_file = request.FILES['mfile']
            image_file = request.FILES['file']

            # load keras model
            model_file_path = save_file(model_file)
            image_file_name = save_file(image_file)
            unique_id = dt.now().strftime("%m-%d-%y_%H-%M-%S")
            output = start_process(model_file_path, image_file_name, unique_id)
            if not output.__contains__("Heat Map generated successfully!!!"):
                status_code = 500
                response['status'] = 500
            else:
                filenames = [f'{unique_id}_original.png', f'{unique_id}_heat_map.png', f'{unique_id}_cam_result.png']
                response['filenames'] = filenames
                status_code = 200
                response['status'] = 200
        except Exception as ex:
            status_code = 500
            response['status'] = 500
        return JsonResponse(response, status=status_code)
    else:
        form = CAMFileForm()
    return render(request, 'unstructureddata/cam.html', {'form': form})

@csrf_exempt
def limetextsubmit(request):
    if request.method == 'POST':
        response = {}
        response['status'] = 200
        try:
            Pipeline_file = request.FILES['mfile']
            train_file = request.FILES['file']
            test_file = request.FILES['testfile']
            selected_features = request.POST['selected_feature']
            test_input_idx = request.POST['test_input_row']
            Pipeline_file_path = save_file(Pipeline_file)
            exp = lime_textExplainer(Pipeline_file_path,train_file, test_file,selected_features,test_input_idx)
            response['exp'] = exp['exp']
            response['test_row'] = exp['test_row']
        except Exception as ex:
            print(ex)
            response['status'] = 300
        return JsonResponse(response)
    else:
        form = LimeTextFileForm()
        return render(request, 'rough/lime.html', {'form': form})

@csrf_exempt
def get_features(request):
    response = {}
    response['dataset_columns'] = []
    status_code = 200
    try:
        data = pd.read_csv(request.FILES['file'])
        response['dataset_columns'] = list(data.columns)
        response['status'] = 200
    except Exception as exp:
        status_code = 500
        response['status'] = 500
    return JsonResponse(response, status = status_code)

def lime_textExplainer(Pipeline_file_path = None,
                       train_data_file_path=None, test_data_file_path=None,
                       target_variable_name=None, test_input_idx = None, num_features=None):

    from lime import lime_text
    #from sklearn.pipeline import make_pipeline
    if not train_data_file_path or not Pipeline_file_path or not test_data_file_path or not target_variable_name :
        return 0
    # read model from pickle file
    c = pickle.load(open(Pipeline_file_path, 'rb'))

    # read data from csv file
    X_train = pd.read_csv(train_data_file_path)
    X_test = pd.read_csv(test_data_file_path)

    #class_name = X_train[target_variable_name].unique()
    class_name = X_train[target_variable_name].unique()
    X_train = X_train.drop(target_variable_name, axis=1)
    print(X_test)

    #X_test= X_test.iloc[int(test_input_idx)]
    np_X_test = X_test.to_numpy()

    explainer = lime_text.LimeTextExplainer(
        class_names=class_name,
        )

    exp = explainer.explain_instance(np_X_test.item(int(test_input_idx)), c.predict_proba)
    predict_prob = exp.__dict__['predict_proba'][0]
    exp = exp.as_html()
    return {'exp':exp,'Document id:':test_input_idx ,'test_row':np_X_test.item(int(test_input_idx)), 'Probability':predict_prob}

def save_file(file_object):
    file_name = str(file_object)
    file_path = f'{folder_path}\\files\\{file_name}'
    with open(file_path, 'wb+') as f:
        for chunk in file_object.chunks():
            f.write(chunk)
    return file_path


def area(larg,comp):
    a = larg*comp
    print(f'A área de um terreno {larg} x {comp} é de {a}m²')

def linhas():
    print('-'*40)

linhas()
l=float(input('Largura (m): '))
c=float(input('Comprimento (m): '))
linhas()
area(l,c)
linhas()


import pickle
import numpy as np

fi = open("/data/wuning/NTLR/beijing/train_loc_set", "rb")
fs = open("sparse_spectral_labels", "rb")
train_cmt_set = []
train_loc_set = pickle.load(fi)
sp_label = pickle.load(fs)
for batch in train_loc_set:
  cmt_batch = []
  for tra in batch:
    cmt_tra = []
    last = None
    for item in tra:
      if not sp_label[item] == last:
        cmt_tra.append(sp_label[item])
        last = sp_label[item]
    cmt_batch.append(cmt_tra)
  train_cmt_set.append(cmt_batch)
print(np.array(train_cmt_set).shape)
pickle.dump(train_cmt_set, open("/data/wuning/RN-GNN/beijing/train_cmt_tra", "wb"))


import streamlit as st
from dotenv import load_dotenv
import os
from langchain.chat_models import ChatOpenAI
from langchain.chains import RetrievalQAWithSourcesChain
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import Chroma
import logging
import chromadb
load_dotenv()
logging.basicConfig(level=logging.DEBUG)

os.environ["OPENAI_API_KEY"] = os.getenv("OPEN_AI_KEY")
ABS_PATH = os.path.dirname(os.path.abspath(__file__))
DB_DIR = os.path.join(ABS_PATH, "db1")

def query_chromadb():
    client_settings = chromadb.config.Settings(
        chroma_db_impl="duckdb+parquet",
        persist_directory=DB_DIR,
        anonymized_telemetry=False
    )

    embeddings = OpenAIEmbeddings(openai_api_key=os.environ['OPENAI_API_KEY'])

    vectorstore = Chroma(
        collection_name="langchain_store",
        embedding_function=embeddings,
        client_settings=client_settings,
        persist_directory=DB_DIR,
    )
    return vectorstore

def load_chain():
    vectordb = query_chromadb()
    chain = RetrievalQAWithSourcesChain.from_chain_type(
        ChatOpenAI(temperature=0),
        chain_type="stuff",
        retriever=vectordb.as_retriever(),
        reduce_k_below_max_tokens=True
    )
    return chain

def get_prompt():
    prompt = st.text_input(label="",
                           placeholder="Enter the prompt", value="")
    return prompt

st.set_page_config(page_title="Hello World", page_icon=":robot:",
                   layout="centered", initial_sidebar_state="expanded")

with st.container():
    left, right = st.columns(2)
    with left:
        st.header("Product chat bot")
        st.write(
            "Hi, I am a chat bot that can answer questions about products from jcrew.com :wave:")
    with right:
        st.image("chatbot.png", width=300)

with st.container():
    st.write('---')
    st.markdown("## Please enter the your question below")
    input_text = get_prompt()
    st.write('---')

    if input_text:
        st.write(f'Your question: {input_text}')
        chain = load_chain()
        response=chain({"question": input_text}, return_only_outputs=True)
        st.write("chatbot's response:")
        st.write(response["answer"])
        st.write("chatbot's source for this answer:")
        st.write(response["sources"])


import glob
import json
import os
import sys

import django

sys.path.append("/var/projects/museum/")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "z2.settings")
django.setup()

from bencomic.models import Comic

def main():
    files = glob.glob(
        "/var/projects/museum/tools/bencomic/necocone.co/text/*.txt"
    )
    files.sort()

    for file in files:
        json_file = os.path.basename(file).split(".")[0]+".json"

        """
        title = models.CharField(max_length=100)
        stripcreator_id = models.IntegerField()
        stripcreator_account = models.CharField(
            max_length=8, choices=STRIPCREATOR_ACCOUNTS
        )
        date = models.DateField(null=True, blank=True, default=None)
        transcript = models.TextField()
        """

        # THE PROVIDED TRANSCRIPTS ARE MISSING THE FINAL COMIC FYI
        if json_file in ["300194.json", "315742.json", "339786.json", "343172.json", "343685.json", "343692.json", "344154.json", "379835.json", "422436.json", "433523.json"]:
            comic = Comic(title="!!FIX ME!!",
                          stripcreator_id=int(json_file.replace(".json", "")),
                          stripcreator_account="bencomic",
                          transcript="")

            print(comic)
            comic.save()
            continue

        with open("/var/projects/museum/tools/bencomic/json/" + json_file) as fh:
            comic = Comic()
            data = fh.read()
            data = json.loads(data)

            comic = Comic(title=data["title"],
                          stripcreator_id=data["sc_id"],
                          stripcreator_account=data["author"],
                          date=data["date"])

            # Transcript
            transcript = ""
            panel_idx = 1
            for panel in data["panels"]:
                transcript += "-- Panel{} --\n".format(panel_idx)
                if panel.get("narration"):
                    transcript += "<Narration> " + panel["narration"] + "\n"
                if panel.get("dialog"):
                    if panel["dialog"].get("left") and panel["dialog"]["left"].get("text"):
                        transcript += "<Left> " + panel["dialog"]["left"]["text"] + "\n"
                    if panel["dialog"].get("right") and panel["dialog"]["right"].get("text"):
                        transcript += "<Right> " + panel["dialog"]["right"]["text"] + "\n"
                panel_idx += 1
            comic.transcript = transcript
            comic.save()
            print(comic)
    return True

if __name__ == "__main__":
    main()


class TitleHandler:
    def __init__(self, title):
        self.title = title
        self.data = {} # dictionary
    def execute(self):
        self.extractCapitals()
        return self.data

    def extractCapitals(self):
        count = 0
        if isinstance(self.title, str):
            for char in self.title:
                if char.isupper():
                    count+=1
            self.data['title_capital_freq'] = count
        else:
            self.data['title_capital_freq'] = count


import sys
import numpy as np
import pandas as pd
# from matplotlib.pyplot import imread
from skimage.io import imread
from skimage.transform import resize
import pickle
import os
import matplotlib.pyplot as plt

import cv2
from time import time
import tensorflow as tf
from keras.models import Sequential, load_model
from keras.optimizers import Adam
from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate
from keras.models import Model

from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import Concatenate
from keras.layers.core import Lambda, Flatten, Dense
from keras.initializers import glorot_uniform

from keras.engine.topology import Layer
from keras.regularizers import l2
from keras import backend as K

from sklearn.utils import shuffle

import numpy.random as rng
from keras.utils import plot_model
from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask
from keras.callbacks import ModelCheckpoint, TensorBoard, LearningRateScheduler
from time import time
import glob
from IPython.display import Image

train_folder = "data/images_background/"
val_folder = 'data/images_evaluation/'
save_path = 'data/'
train_type = 'capsule' #CONVOLUTIONAL or CAPSULE

def loadimgs(path,n = 0):
    '''
    path => Path of train directory or test directory
    '''
    X=[]
    y = []
    cat_dict = {}
    lang_dict = {}
    curr_y = n
    # we load every alphabet seperately so we can isolate them later
    for alphabet in [x for x in os.listdir(path) if not x.startswith('.')]:
        print("loading alphabet: " + alphabet)
        lang_dict[alphabet] = [curr_y,None]
        alphabet_path = os.path.join(path,alphabet)
        # every letter/category has it's own column in the array, so  load seperately
        for letter in [x for x in os.listdir(alphabet_path) if not x.startswith('.')]:
            cat_dict[curr_y] = (alphabet, letter)
            category_images=[]
            letter_path = os.path.join(alphabet_path, letter)
            # read all the images in the current category
            for filename in [x for x in os.listdir(letter_path) if not x.startswith('.')]:
                image_path = os.path.join(letter_path, filename)
                image = imread(image_path)
#                 image = resize(image, (52, 52))
                category_images.append(image)
                y.append(curr_y)
            try:
                X.append(np.stack(category_images))
            # edge case  - last one
            except ValueError as e:
                print(e)
                print("error - category_images:", category_images)
            curr_y += 1
            lang_dict[alphabet][1] = curr_y - 1
    y = np.vstack(y)
    X = np.stack(X)
    return X,y,lang_dict

def initialize_weights(shape, name=None):
    """
        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf
        suggests to initialize CNN layer weights with mean as 0.0 and standard deviation of 0.01
    """
    return np.random.normal(loc = 0.0, scale = 1e-2, size = shape)

def initialize_bias(shape, name=None):
    """
        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf
        suggests to initialize CNN layer bias with mean as 0.5 and standard deviation of 0.01
    """
    return np.random.normal(loc = 0.5, scale = 1e-2, size = shape)

with open(os.path.join(save_path, "train.pickle"), "rb") as f:
    (Xtrain, train_classes) = pickle.load(f)

print("Training alphabets: \n")
print(list(train_classes.keys()))

with open(os.path.join(save_path, "val.pickle"), "rb") as f:
    (Xval, val_classes) = pickle.load(f)

print("Validation alphabets:", end="\n\n")
print(list(val_classes.keys()))

def get_batch(batch_size, s="train"):
    """Create batch of n pairs, half same class, half different class"""
    if s == 'train':
        X = Xtrain
        categories = train_classes
    else:
        X = Xval
        categories = val_classes
    n_classes, n_examples, w, h = X.shape

    # randomly sample several classes to use in the batch
    categories = rng.choice(n_classes, size=(batch_size,), replace=False)

    # initialize 2 empty arrays for the input image batch
    pairs = [np.zeros((batch_size, h, w, 1)) for i in range(2)]

    # initialize vector for the targets
    targets = np.zeros((batch_size,))

    # make one half of it '1's, so 2nd half of batch has same class
    targets[batch_size // 2:] = 1
    for i in range(batch_size):
        category = categories[i]
        idx_1 = rng.randint(0, n_examples)
        pairs[0][i, :, :, :] = X[category, idx_1].reshape(w, h, 1)
        idx_2 = rng.randint(0, n_examples)

        # pick images of same class for 1st half, different for 2nd
        if i >= batch_size // 2:
            category_2 = category
        else:
            # add a random number to the category modulo n classes to ensure 2nd image has a different category
            category_2 = (category + rng.randint(1, n_classes)) % n_classes

        pairs[1][i, :, :, :] = X[category_2, idx_2].reshape(w, h, 1)

    return pairs, targets

def generate(batch_size, s="train"):
    """a generator for batches, so model.fit_generator can be used. """
    while True:
        pairs, targets = get_batch(batch_size,s)
        yield (pairs, targets)

def make_oneshot_task(N, s="val", language=None):
    """Create pairs of test image, support set for testing N way one-shot learning. """
    if s == 'train':
        X = Xtrain
        categories = train_classes
    else:
        X = Xval
        categories = val_classes
    n_classes, n_examples, w, h = X.shape

    indices = rng.randint(0, n_examples, size=(N,))
    if language is not None:  # if language is specified, select characters for that language
        low, high = categories[language]
        if N > high - low:
            raise ValueError("This language ({}) has less than {} letters".format(language, N))
        categories = rng.choice(range(low, high), size=(N,), replace=False)

    else:  # if no language specified just pick a bunch of random letters
        categories = rng.choice(range(n_classes), size=(N,), replace=False)
    true_category = categories[0]
    ex1, ex2 = rng.choice(n_examples, replace=False, size=(2,))
    test_image = np.asarray([X[true_category, ex1, :, :]] * N).reshape(N, w, h, 1)
    support_set = X[categories, indices, :, :]
    support_set[0, :, :] = X[true_category, ex2]
    support_set = support_set.reshape(N, w, h, 1)
    targets = np.zeros((N,))
    targets[0] = 1
    targets, test_image, support_set = shuffle(targets, test_image, support_set)
    pairs = [test_image, support_set]

    return pairs, targets

def get_siamese_model(input_shape):
    """
        Model architecture based on the one provided in: http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf
    """

    # Define the tensors for the two input images
    left_input = Input(input_shape)
    right_input = Input(input_shape)

    # Convolutional Neural Network
    model = Sequential()
    model.add(Conv2D(64, (10, 10), activation='relu', input_shape=input_shape,
                     kernel_initializer=initialize_weights, kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(128, (7, 7), activation='relu',
                     kernel_initializer=initialize_weights,
                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(128, (4, 4), activation='relu', kernel_initializer=initialize_weights,
                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(256, (4, 4), activation='relu', kernel_initializer=initialize_weights,
                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))
    model.add(Flatten())
    model.add(Dense(4096, activation='sigmoid',
                    kernel_regularizer=l2(1e-3),
                    kernel_initializer=initialize_weights, bias_initializer=initialize_bias))

    # Generate the encodings (feature vectors) for the two images
    encoded_l = model(left_input)
    encoded_r = model(right_input)

    # Add a customized layer to compute the absolute difference between the encodings
    L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
    L1_distance = L1_layer([encoded_l, encoded_r])

    # Add a dense layer with a sigmoid unit to generate the similarity score
    prediction = Dense(1, activation='sigmoid', bias_initializer=initialize_bias)(L1_distance)

    # Connect the inputs with the outputs
    siamese_net = Model(inputs=[left_input, right_input], outputs=prediction)

    # return the model
    return siamese_net

# Hyper parameters
evaluate_every = 200 # interval for evaluating on one-shot tasks
batch_size = 32
n_iter = 20000 # No. of training iterations
N_way = 20 # how many classes for testing one-shot tasks
n_val = 250 # how many one-shot tasks to validate on
best = -1

model_path = "checkpoints_{0}".format(train_type)

os.makedirs("checkpoints_{0}".format(train_type), exist_ok=True)
file_path = "checkpoints_{0}".format(train_type)+"/siamese-epoch-{epoch:05d}-lr-" + "-train_loss-{loss:.4f}-val_loss-{val_loss:.4f}.hdf5"
checkpoint = ModelCheckpoint(file_path,
                             monitor=['loss', 'val_loss'],
                             verbose=1,
                             save_best_only=False,
                             save_weights_only=False,
                             mode='min',
                             period=2)
tensorboard = TensorBoard(log_dir="logs/{}".format(time()), histogram_freq=0)
lr_decay = LearningRateScheduler(schedule=lambda epoch: 0.001 * (0.9 ** epoch))

def get_capsnet_model(input_shape, n_class, routings):

    left_input = Input(shape=input_shape)
    right_input = Input(shape=input_shape)

    input = Input(shape=input_shape)
    # Layer 1: Just a conventional Conv2D layer
    conv1 = Conv2D(filters=128, kernel_size=36, strides=1, padding='valid', activation='relu', name='conv1',
                  kernel_initializer=initialize_weights, bias_initializer=initialize_bias,
                   kernel_regularizer=l2(2e-4),)(input)

    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=16, kernel_size=36, strides=2, padding='valid')

    # Layer 3: Capsule layer. Routing algorithm works here.
    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
                             name='digitcaps')(primarycaps)

    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
    # If using tensorflow, this will not be necessary. :)
    out_caps = Length(name='capsnet')(digitcaps)

    tunnel = Model(input, out_caps)
    tunnel.summary()

    encoded_l = tunnel(left_input)
    encoded_r = tunnel(right_input)

    L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
    L1_distance = L1_layer([encoded_l, encoded_r])

    prediction = Dense(1, activation='sigmoid', bias_initializer=initialize_bias)(L1_distance)

    train_model = Model(inputs=[left_input, right_input], outputs=prediction)

    return train_model

ckpts = glob.glob("checkpoints_{0}".format(train_type)+"/*.hdf5")
initial_epoch = 0
if len(ckpts) != 0 and False:
    latest_ckpt = max(ckpts, key=os.path.getctime)
    print("loading from checkpoint: ", latest_ckpt)
    initial_epoch = int(latest_ckpt[latest_ckpt.find("-epoch-") + len("-epoch-"):latest_ckpt.rfind("-lr-")])
    model = load_model(latest_ckpt, custom_objects={'CapsuleLayer': CapsuleLayer, 'Length': Length, 'PrimaryCap':PrimaryCap })
else:
    if train_type == 'capsule':
        print("***New model for capsule network loaded***")
        model = get_capsnet_model(input_shape=[105, 105, 1], n_class=10, routings=3)
    elif train_type == 'convolutional':
        print("***New model for convolutional network loaded***")
        model = get_siamese_model((105, 105, 1))
model.summary()
plot_model(model, to_file='model.png')
Image(retina=True, filename='model.png')

optimizer = Adam(lr = 0.001)
model.compile(loss="binary_crossentropy",optimizer=optimizer)

model.fit_generator(generator=generate(32),
                    steps_per_epoch=100,
                    epochs=1000,
                    initial_epoch=initial_epoch,
                    validation_data=generate(32, 'val'),
                    validation_steps=2,
                    callbacks=[checkpoint, tensorboard, lr_decay],
                    use_multiprocessing=True)


#Project imports
from em.base_classes.registrable import Registrable

#As things are right now, this is kind of overkill,
#but I wanted to make sure that the code was easily expandable
class DepsGraph:
    class Node:
        def __init__(self, _cls):
            self.cls = _cls
            self.edges = {
                #The ones which cls depends on
                'incoming': set(),
                #The ones that depend on cls
                'outgoing': set()
            }
            self.incoming = self.edges['incoming']
            self.outgoing = self.edges['outgoing']

        #Syntatic sugar for building a root node for classes without dependencies
        #Having said node allows for easy access to these classes
        @classmethod
        def RootNode(cls):
            return cls(None)

    #__init__ will only consider base classes that are derived from Registrable
    #No two classes should be named equally
    #Improper graphs (circular dependencies) will crash the code
    def __init__(self, classes):

        #Naming seems weird here, because you expect the root to be part of the graph,
        #but you should think of the instance of the DepsGraph class as the actual graph,
        #these are just variable names.
        self.root = self.Node.RootNode()
        self.graph = {}

        for _cls in classes:
            class_name = _cls.__name__
            node = self.try_set(class_name, _cls)
            bases = [base for base in _cls.__bases__ if Registrable in base.__bases__]
            node.incoming = set([base.__name__ for base in bases])

            #If there aren't any base classes, then we say our class 'depends'
            #on 'the root class' (which doesn't exist). We could add None to incoming
            #for consistency but it would be a waste of time.
            if not bases:
                self.root.outgoing.add(class_name)

            for base in bases:
                base_name = base.__name__
                base_node = self.try_set(base_name, base)
                base_node.outgoing.add(class_name)

    def try_set(self, class_name, _cls):
        node = None
        if not class_name in self.graph:
            node = self.Node(_cls)
            self.graph[class_name] = node
        if node:
            return node
        return self.graph[class_name]

    #Returns classes in an order suited for registration
    #They should be unregistered in reverse order
    @classmethod
    def Sorted(cls, classes):
        deps_graph = cls(classes)

        result = []
        no_dependencies = set(deps_graph.root.outgoing)
        while no_dependencies:
            class_name = no_dependencies.pop()
            class_node = deps_graph.graph[class_name]

            for out_name in class_node.outgoing:
                out_node = deps_graph.graph[out_name]
                out_node.incoming.remove(class_name)

                if not out_node.incoming:
                    no_dependencies.add(out_name)

            result.append(class_node.cls)

        return result


# Python 2.7
# Replaces contents to object in bucket

from google.cloud import storage

client = storage.Client()
bucket = client.get_bucket('ep-test-bucket')

blob = bucket.get_blob('test.txt')

blob.upload_from_string('New contents!')


#####punto 1 aplicacion de canciones ######

class ElementosDigitales ():
    def __init__(self,Nombreentrada,creadorentrada,fechaentrada):
        self.nombre = Nombreentrada
        self.creador = creadorentrada
        self.fecha = fechaentrada
    def atributos (self):
        print(f'hola,como estas?el nombre de la cancion es {self.nombre} esta cancion fue creada por {self.creador} en la fecha{self.fecha}')

elementos = ElementosDigitales('losing it', 'fisher', 18-10-2015)
elementos.atributos()
print('#'*30)

class Ususario ():
    def __init__(self,nombreentrada,edadentrada,sexoentrada,nacionalidadentrada):
        self.nombre = nombreentrada
        self.edad = edadentrada
        self.sexo = sexoentrada
        self.nacionalidad = nacionalidadentrada
    def atributos (self):
        print(f'hola mi nombre es {self.nombre} tengo {self.edad} años, soy de sexo {self.sexo} y soy del pais de {self.nacionalidad}')
    def cancion (self,cancionentrada):
        self.cancion = cancionentrada
        print(f'hola soy {self.nombre} y estoy esuchando esta cancion {self.cancion}')

usuario1 = Ususario('camilo', 18, 'masculino','peru')
usuario1.atributos()
usuario1.cancion('prende la fiesta')
print('#'*30)

class Pagina ():
    def __init__(self,entradacontenido,entradaformato,entradafecha):
        self.contenido = entradacontenido
        self.formato = entradaformato
        self.fecha = entradafecha
    def atributos (self):
        print(f'el contenido es {self.contenido} con formato {self.formato} y su fecha de publicacion es {self.fecha}')

pagina1 = Pagina('guaracha', 'wav', 'febrero 2019')
pagina1.atributos()
print('#'*30)

class Cancion (ElementosDigitales):
    def __init__(self, Nombreentrada, creadorentrada, fechaentrada,generoentrada,duracionentrada):
        ElementosDigitales.__init__(self, Nombreentrada, creadorentrada, fechaentrada)
        self.genero = generoentrada
        self.duracion = duracionentrada
    def cancion (self,cancionentrada):
        self.cancion = cancionentrada
        print(f'esta cancion nueva {self.cancion} fue creada en abril de este año')
    def numreproducciones (self,numreproduccionesentrada):
        for i in range(numreproduccionesentrada):
            print(f'{self.nombre} sonando {i+1} vez')

cancion1 = Cancion('periodico de ayer', 'hector lavoe', '1980', 'salsa', 560)
cancion1.cancion('me elevas')
cancion1.numreproducciones(3)

print('#'*30)

class Artista (Ususario):
    def __init__(self, nombreentrada, edadentrada, sexoentrada, nacionalidadentrada,generoentrada,cancionpublicadaentrada,numalbumentrada):
        Ususario.__init__(self, nombreentrada, edadentrada, sexoentrada, nacionalidadentrada)
        self.genero = generoentrada
        self.numcanciones = cancionpublicadaentrada
        self.numalbums = numalbumentrada
    def concierto (self,ciudadentrada):
        self.ciudad = ciudadentrada
        print(f'hola como estas?en esta ciudad {self.ciudad} dare un concierto el proximo viernes,te esperooo')

artista1 = Artista('ariana grande', 24, 'femenino', 'usa', 'pop', 200, 15)
artista1.concierto('medallo')

listafavs = ['hola bebe','te estoy llamando','ayayai','sobelo']
class Favoritos (Pagina):
    def __init__(self,entradacontenido, entradaformato, entradafecha,favoritoscomunidadentrada,ultimaactualizacionentrada):
        Pagina.__init__(self, entradacontenido, entradaformato, entradafecha)
        self.favoritos = favoritoscomunidadentrada
        self.actualizacion = ultimaactualizacionentrada
    def agregar (self,nombrecancionentrada,fechaactualizacionentrada):
        self.cancionnueva = nombrecancionentrada
        self.fechanueva = fechaactualizacionentrada
        listafavs.append(self.cancionnueva)
        print(f'la lista de las canciones favoritas de la comunidad quedo asi {listafavs} y esta es la fecha de la ultima actualizacion {self.fechanueva}')
    def eliminar (self,posicioneliminarentrada):
        print(listafavs)
        poseliminada=int(input('elige posicion que quieres eliminar'))
        self.posicioneliminada = posicioneliminarentrada
        listafavs.pop(self.posicioneliminada+1)

favorito1 = Favoritos('techhouse', 'mp3', '8 de junio', listafavs,'2 de febrero')
favorito1.agregar('aflojele', '28 de marzo')
favorito1.eliminar(poseliminada)