import ibapi
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
from ibapi.order import Order
from time import sleep
import threading
from datetime import datetime
from multiprocessing import Queue
from multiprocessing import queues
from ibapi.commission_report import CommissionReport
from ibapi.wrapper import TickerId
import pandas as pd

contract_dict = {"EURUSD" :("EUR",'USD'), 'EURGBP': ('EUR','GBP'), 'EURJPY': ('EUR','JPY'),'EURCHF':('EUR','CHF'), "EURCAD": ("EUR","CAD"), 'EURAUD': ('EUR','AUD'),'EURNZD':('EUR','NZD'),
                 "GBPUSD": ("GBP",'USD'),'USDJPY' :('USD','JPY'),'USDCHF':('USD','CHF'),'USDCAD':('USD','CAD'),'AUDUSD':('AUD','USD'),'NZDUSD':('NZD','USD'),
                 'GBPJPY' : ('GBP','JPY'),'GBPCHF':('GBP','CHF'),'GBPCAD':('GBP','CAD'),'GBPAUD':('GBP','AUD'),'GBPNZD':('GBP','NZD'),
                 "CHFJPY": ("CHF",'JPY'),'CADJPY':('CAD','JPY'),'AUDJPY':('AUD','JPY'), 'NZDJPY':('NZD','JPY'),
                 'CADCHF':('CAD','CHF'), 'AUDCHF':( 'AUD','CHF'),'NZDCHF':('NZD','CHF'),
                 'AUDCAD':('AUD','CAD'), 'AUDNZD':('AUD','NZD'),
                 'NZDCAD':('NZD','CAD'),
                 'USDSEK': ('USD','SEK'), 'USDRUB': ('USD','RUB'), 'USDNOK': ('USD','NOK'), 'USDMXN': ('USD','MXN'), 'USDDKK': ('USD','DKK'), 'USDCNH': ('USD','CNH'),
                 }
futures_dict = {'ZF':('ZF','USD','ECBOT','20190628','1000'),'ZB':('ZB','USD','ECBOT','20190619','1000'),'ZN':('ZN','USD','ECBOT','20190619','1000'),
           'BTP': ('BTP', 'EUR', 'DTB', '20190606', '1000'),'GBL':('GBL','EUR','DTB','20190606','1000'),'GBM':('GBM','EUR','DTB','20190606','1000'),
           'OAT': ('OAT', 'EUR', 'DTB', '20190606', '1000'),
                'ES':('ES','USD','GLOBEX_IND','20190621','50'),'NQ':('NQ','USD','GLOBEX_IND','20190621','20'),
                'DESX5': ('DESX5', 'EUR', 'DTB', '20191220', '100'),'YM': ('YM', 'USD', 'ECBOT', '20190621', '5'),
                'Z': ('Z', 'GBP', 'ICEEU', '20190621', '1000'),'SMI': ('SMI', 'CHF', 'SOFFEX', '20190621', '10'),
                'R': ('R', 'GBP', 'ICEEU', '20190626', '1000')}

KEYS = ['time','open','high','low','close']

class tickThread(threading.Thread):
    req_id = 0

    def __init__(self, threadID, conn):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.conn = conn

    def run(self):
        self.conn.run()

class IBInterface(EWrapper, EClient):
    wrapper_dict = {EWrapper.nextValidId.__name__: Queue(), EWrapper.openOrder.__name__: Queue(),
                    EWrapper.orderStatus.__name__: Queue(), EWrapper.updatePortfolio.__name__: Queue(),
                    EWrapper.execDetails.__name__: Queue(), EWrapper.commissionReport.__name__:Queue()}
    def __init__(self, host, port, client_id,account):
        self.host = host
        self.port = port
        self.open_orders = {}
        self.open_positions = {}
        self.req_id = 0
        self.account = account
        EWrapper.__init__(self)
        EClient.__init__(self, wrapper=self)
        self.connect(host, port, client_id)
        self.startApi()
        tickThread(101010, self).start()
        if self.isConnected():
            1
        else:
            while self.isConnected() == False:

                print('Sleeping 40 seconds')
                sleep(40)
                self.disconnect()
                self.done = False
                self.connect(host, port, client_id)
                self.startApi()
                tickThread(101010, self).start()
        self.reqAccountUpdates(True,account)

    #*********** Wrapper Functions *********************#
    def error(self, reqId:TickerId, errorCode:int, errorString:str):

        print('############## ' + str(reqId) + ' : ' + str(errorCode) + ' : ' + str(errorString) + ' ###################')
        if errorCode == 2107:
            contract = self.create_contract("EURJPY", 'FX')
            self.getHistoricalData(contract, '', '1 D', '30 mins', 'ASK', 1, 1)
    def nextValidId(self, orderId: int):

        while self.wrapper_dict[EWrapper.nextValidId.__name__].qsize()>0:
            self.wrapper_dict[EWrapper.nextValidId.__name__].get()
        self.wrapper_dict[EWrapper.nextValidId.__name__].put(orderId)
        print('Next valid ID ' + str(orderId))

    def realtimeBar(self, reqId: int, time: int, open: float, high: float, low: float, close: float, volume: int,
                    WAP: float, count: int):
        data_dict = {}
        data_dict['open'] = open
        data_dict['high'] = high
        data_dict['low'] = low
        data_dict['close'] = close
        data_dict['time'] = time
        data_dict['volume'] = volume
        self.wrapper_dict[EWrapper.realtimeBar.__name__+str(reqId)].put(data_dict)

    def connectionClosed(self):
        super().connectionClosed()
        print('Connection Closed')

    def historicalData(self, reqId:int , bars : object):

        data_dict = {}
        data_dict['open'] = bars.open
        data_dict['high'] = bars.high
        data_dict['low'] = bars.low
        data_dict['close'] = bars.close
        data_dict['time'] = bars.date
        data_dict['volume'] = bars.volume
        self.wrapper_dict[EWrapper.historicalData.__name__+str(reqId)].put(data_dict)

    def historicalDataEnd(self, reqId:int, start:str, end:str):
        super().historicalDataEnd(reqId,start,end)
        print('Historicla Data End')
        self.wrapper_dict[EWrapper.historicalData.__name__ + str(reqId)].put('End')

    # def historicalDataUpdate(self, reqId: int, bar: object):
    #     wrapper_dict[EWrapper.historicalDataUpdate.__name__ + str(reqId)].put(bar)

    def openOrder(self, orderId:int, contract:Contract, order:Order,
                  orderState:object):
        data_dict ={}
        data_dict['orderId'] = orderId
        print('Order open with orderID : '+str(orderId) + " for contract : "+str(contract.symbol)+str(contract.currency))
        data_dict['contract'] = contract
        data_dict['order'] = order
        data_dict['orderState']  =orderState
        print(f'Order ID : {order.orderId}, action : {order.action}, type : {order.orderType}, quantity : {order.totalQuantity}, price : {order.lmtPrice}')
        self.wrapper_dict[EWrapper.openOrder.__name__].put(data_dict)

    def orderStatus(self, orderId:int , status:str, filled:float,
                    remaining:float, avgFillPrice:float, permId:int,
                    parentId:int, lastFillPrice:float, clientId:int,
                    whyHeld:str,mktCapPrice:float):
        data_dict = {}
        data_dict['status'] = status
        data_dict['filled'] = filled
        data_dict['remaining'] = remaining
        data_dict['avgFillPrice'] = avgFillPrice
        data_dict['permId'] = permId
        data_dict['parentId'] = parentId
        data_dict['lastFillPrice'] = lastFillPrice
        data_dict['clientId'] = clientId
        data_dict['whyHeld'] = whyHeld
        data_dict['mktCapPrice'] = mktCapPrice

        self.wrapper_dict[EWrapper.orderStatus.__name__].put(data_dict)

    def updatePortfolio(self, contract:Contract, position:float,
                        marketPrice:float, marketValue:float,
                        averageCost:float, unrealizedPNL:float,
                        realizedPNL:float, accountName:str):
        data_dict = {}
        data_dict['contract'] = contract
        data_dict['position'] = position
        data_dict['marketPrice'] = marketPrice
        data_dict['marketValue'] = marketValue
        data_dict['averageCost'] = averageCost
        data_dict['unrealizedPNL'] = unrealizedPNL
        data_dict['realizedPNL'] = realizedPNL
        data_dict['accountName'] = accountName
        self.wrapper_dict[EWrapper.updatePortfolio.__name__].put(data_dict)

    def execDetails(self, reqId:int, contract:Contract, execution:object):
        data_dict={}
        data_dict['contract'] = contract
        data_dict['execution'] = execution
        print("To API mas dinei auta " + str(execution.side) + ' '+contract.symbol+contract.currency + ' stis  '+str(execution.time) + " me timh "+str(execution.price) + ' kai lot '+str(execution.shares) + ' ston account '+ str(execution.acctNumber))
        self.wrapper_dict[EWrapper.execDetails.__name__].put(data_dict)

    def commissionReport(self, commissionReport:CommissionReport):
        data_dict = {}
        data_dict['commission'] = commissionReport
        print(commissionReport.realizedPNL)
        self.wrapper_dict[EWrapper.commissionReport.__name__].put(data_dict)

    #**************** Client Methods ******************#

    def req_next_valid_req_id(self):
        self.req_id += 1
        return self.req_id

    def check_conn(self):
        if (self.isConnected() == False):
            while self.isConnected() == False:
                sleep(60)
                self.connect(self.host, self.port, self.client_id)
        return True

    def check_req_id(self):
        self.nextValidId()

    def reqIds(self):
        super().reqIds(-1)
        sleep(0.5)
        queue = self.wrapper_dict[EWrapper.nextValidId.__name__]
        print("Testing ReqID")
        try:
            if queue.qsize()>0:
                id = queue.get(block=True,timeout=5)
            else:
                id =0
        except queues.Empty:
            id = 0
        return id

    def reqRealTimeBars(self, contract, barSize,
                        whatToShow, useRTH):
        req_id = self.req_next_valid_req_id()
        super().reqRealTimeBars(reqId=req_id, contract=contract, barSize=barSize,
                                whatToShow=whatToShow, useRTH=useRTH, realTimeBarsOptions=[])
        queue = Queue()
        self.wrapper_dict[EWrapper.realtimeBar.__name__+str(req_id)] = queue

        return queue

    def getHistoricalData(self,contract,end_date_time,duration_string,bar_size_setting, what_to_show,userth=1,format_date=2,keep_up_to_date = False ):
        req_id = self.req_next_valid_req_id()
        queue = Queue()
        self.wrapper_dict[EWrapper.historicalData.__name__ + str(req_id)] = queue
        super().reqHistoricalData(reqId=req_id,contract= contract,endDateTime=end_date_time,durationStr=duration_string,barSizeSetting=bar_size_setting,whatToShow=what_to_show,useRTH=userth,formatDate=format_date,keepUpToDate=keep_up_to_date,chartOptions=[])
        data = []
        queue = self.wrapper_dict[EWrapper.historicalData.__name__+str(req_id)]
        while True:
            try:
                z = queue.get(block=True,timeout=100)
                if z != 'End':
                    data.append(z)
                else:
                    break

            except queues.Empty:
                break

        return data

    def getOpenOrders(self):
        super().reqOpenOrders()
        queue = self.wrapper_dict[EWrapper.openOrder.__name__]

        data = []
        orders = []
        print('At open Orders, queue size : '+str(queue.qsize()))
        try:

            for i in range(queue.qsize()):
            #TODO: Sometimes this will fail with queue.Empty
                data.append(queue.get(block=False))
            orders = self.set_open_orders(data)
        except queues.Empty:
            orders = []
            print("Queue Empty")

        return orders
    def getAllOpenOrders(self):
        super().reqAllOpenOrders()
        queue = self.wrapper_dict[EWrapper.openOrder.__name__]
        data = []
        sleep(1)
        print('At all open Orders, queue size : '+str(queue.qsize()))
        try:

            for i in range(queue.qsize()):
            #TODO: Sometimes this will fail with queue.Empty
                data.append(queue.get(block=False))
            orders = self.set_open_orders(data)
        except queues.Empty:
            orders = []
            print("Queue Empty")

        return orders
    def getRealTimeHistoricalData(self,contract,end_date_time,duration_string,bar_size_setting, what_to_show,userth,format_date,keep_up_to_date = True):
        req_id = self.req_next_valid_req_id()
        queue = Queue()
        self.wrapper_dict[EWrapper.historicalDataUpdate.__name__ + str(req_id)] = queue
        queue2 = Queue()
        self.wrapper_dict[EWrapper.historicalData.__name__ + str(req_id)] = queue2
        super().reqHistoricalData(reqId=req_id, contract=contract, endDateTime=end_date_time,
                                  durationStr=duration_string, barSizeSetting=bar_size_setting, whatToShow=what_to_show,
                                  useRTH=userth, formatDate=format_date,keepUpToDate=keep_up_to_date,chartOptions=[])
        queue2 = self.wrapper_dict[EWrapper.historicalData.__name__ + str(req_id)]
        data = []
        while True:
            try:
                z = queue2.get(block=True)
                if z != 'End':
                    data.append(z)
                else:
                    break
            except queues.Empty:
                break

        return queue,data

    def getAccountPositions(self,account):
        super().reqAccountUpdates(True,account)
        queue = self.wrapper_dict[EWrapper.updatePortfolio.__name__]

        data = []
        try:

            for i in range(queue.qsize()):
                data.append(queue.get(block=False))
            positions = self.set_open_positions(data)
        except queues.Empty:
            positions = {}
            print("Queue Empty")

        return positions

    def currentTime(self, time: int):
        super().currentTime(time)
        self.serverTime = time
        print(time)

    def create_contract(self,symbol, contract_type):
        if contract_type == "FX":
            contract = Contract()
            contract.symbol = contract_dict[symbol][0] #FOAT DEC 17
            contract.secType = "CASH" #FUT
            contract.currency = contract_dict[symbol][1] #EUR
            contract.exchange = "IDEALPRO" #DTP
            return contract
        elif contract_type=='FUTURE':
            contract = Contract()
            contract.symbol = futures_dict[symbol][0]
            contract.secType = "FUT"
            contract.currency = futures_dict[symbol][1]
            contract.exchange = futures_dict[symbol][2]
            contract.lastTradeDateOrContractMonth=futures_dict[symbol][3]
            contract.multiplier = futures_dict[symbol][4]
            return contract

    def create_order(self,action,quantity,type,price = 0):
        order = Order()
        order.orderType = type
        if type != 'MKT':
            order.lmtPrice = price
            order.tif = 'GTC'
        order.action = action
        order.totalQuantity = quantity
        order.account = self.account
        print('Parent Order Quantity '+str(quantity))
        return order

    def sell(self,quantity,contract,open_position,take_profit_price=0,stop_loss_price=0,limit_orders_lot=0,is_combined=False,reduce_lot = False,order_type = 'MKT',price =0):
        total_orders = []
        parent_id = self.reqIds()
        sleep(0.7)
        print('Apo to sell to ID einai : '+str(parent_id))
        other_id = parent_id

        order= self.create_order('SELL',quantity+abs(open_position),order_type,price=price)
        order.orderId = parent_id
        total_orders.append(order)
        if take_profit_price!= 0:

            take_profit_order = Order()

            other_id +=1
            take_profit_order.orderId = other_id
            if reduce_lot:
                take_profit_order.action = 'SELL'
            else:
                take_profit_order.action = 'BUY'
            take_profit_order.lmtPrice = take_profit_price
            take_profit_order.orderType = 'LMT'
            take_profit_order.totalQuantity = limit_orders_lot
            print('Take Profit Order Quantity :'+str(take_profit_order.totalQuantity))
            take_profit_order.tif = 'GTC'
            take_profit_order.ocaType = 1
            take_profit_order.account = self.account
            take_profit_order.ocaGroup = contract.symbol+contract.currency + str(parent_id)
            total_orders.append(take_profit_order)
        if stop_loss_price !=0:

            stop_loss_order = Order()

            other_id += 1
            stop_loss_order.orderId = other_id
            if reduce_lot:
                stop_loss_order.action = 'SELL'
            else:
                stop_loss_order.action = 'BUY'
            stop_loss_order.auxPrice = stop_loss_price
            stop_loss_order.orderType = 'STP'
            stop_loss_order.totalQuantity = limit_orders_lot
            print('Stop Loss Order Quantity :' + str(stop_loss_order.totalQuantity))
            stop_loss_order.tif = 'GTC'
            stop_loss_order.ocaType = 1
            stop_loss_order.account = self.account
            stop_loss_order.ocaGroup = contract.symbol+contract.currency + str(parent_id)
            total_orders.append(stop_loss_order)

        orders = self.getOpenOrders()
        if is_combined==False:
            for open_order in orders:
                if contract.symbol + contract.currency in orders[open_order][1].ocaGroup:
                    self.cancelOrder(open_order)
        for every_order in total_orders:
            self.placeOrder(every_order.orderId, contract,every_order)
            sleep(2)

        return self.wrapper_dict[EWrapper.openOrder.__name__], self.wrapper_dict[EWrapper.orderStatus.__name__]

    def buy(self,quantity,contract,open_position,take_profit_price=0,stop_loss_price=0,limit_orders_lot=0, is_combined=False,reduce_lot = False,order_type = 'MKT',price =0):
        total_orders = []
        parent_id = self.reqIds()
        sleep(0.7)
        print('Apo to buy to ID einai : ' + str(parent_id))
        other_id = parent_id
        order= self.create_order('BUY',quantity+abs(open_position),order_type,price=price)
        order.orderId = parent_id
        total_orders.append(order)
        if take_profit_price!= 0:

            take_profit_order = Order()

            other_id +=1
            take_profit_order.orderId = other_id
            if reduce_lot:
                take_profit_order.action = 'BUY'
            else:
                take_profit_order.action = 'SELL'
            take_profit_order.lmtPrice = take_profit_price
            take_profit_order.orderType = 'LMT'
            take_profit_order.totalQuantity = limit_orders_lot
            take_profit_order.tif = 'GTC'
            take_profit_order.ocaType = 1
            take_profit_order.account = self.account
            take_profit_order.ocaGroup = contract.symbol+contract.currency  + str(parent_id)
            total_orders.append(take_profit_order)
        if stop_loss_price !=0:

            stop_loss_order = Order()

            other_id += 1
            stop_loss_order.orderId = other_id
            if reduce_lot:
                stop_loss_order.action = 'BUY'
            else:
                stop_loss_order.action = 'SELL'
            stop_loss_order.auxPrice = stop_loss_price
            stop_loss_order.orderType = 'STP'
            stop_loss_order.totalQuantity = limit_orders_lot
            stop_loss_order.tif = 'GTC'
            stop_loss_order.ocaType = 1
            stop_loss_order.account = self.account
            stop_loss_order.ocaGroup = contract.symbol+contract.currency + str(parent_id)
            total_orders.append(stop_loss_order)
        orders = self.getOpenOrders()
        if is_combined == False:
            for open_order in orders:
                if contract.symbol+contract.currency in orders[open_order][1].ocaGroup:
                    self.cancelOrder(open_order)
        for every_order in total_orders:
            self.placeOrder(every_order.orderId, contract,every_order)

        return self.wrapper_dict[EWrapper.openOrder.__name__], self.wrapper_dict[EWrapper.orderStatus.__name__]

    def set_open_positions(self,queue_positions):
        for order in queue_positions:
            self.open_positions[order['contract'].symbol+ order['contract'].currency] = dict(position=order['position'],unrealizedPNL= order['unrealizedPNL'])

        return self.open_positions

    def set_open_orders(self,queue_orders):
        self.open_orders= {}
        for order in queue_orders:
            self.open_orders[order['orderId']] = (order['contract'],order['order'],order['orderState'])

        return self.open_orders

    def reqExecutionDetails(self,filter):

        self.reqExecutions(reqId=1,execFilter=filter)
        queue =  self.wrapper_dict[EWrapper.execDetails.__name__]
        queue_commision = self.wrapper_dict[EWrapper.commissionReport.__name__]
        return queue, queue_commision

    def getAccountPositionsQueue(self,account):
        super().reqAccountUpdates(True,account)
        queue = self.wrapper_dict[EWrapper.updatePortfolio.__name__]
        return queue

def store_historical_data_to_file(symbol, currency, data):
    data = pd.DataFrame(data)
    data = data.set_index(pd.to_datetime(data['time']))
    data = data.drop('time',axis =1)

    store = pd.HDFStore('tick_data' + symbol + currency + '.h5')
    store.open()

    if ('/'+symbol+currency) in store.keys():
        idx = store.select(symbol+currency,where="index in data.index",columns=['index']).index

        print('Store ' + symbol + currency + ' was written with length ' + str(len(data.query('index not in @idx'))))
        # print(dataframe)
        store.append(symbol + currency, data.query('index not in @idx'), format='t')
    else:
        store.append(symbol + currency, data, format='t')
        print(data)
    store.close()

# def store_queue_to_file(queue,symbol,currency):
#
#             msg_array = []
#             times = []
#             while True:
#                 try:
#                     t1 = time.time()
#                     now = datetime.now()
#                     msg = queue.get(block = True)
#                     msg_array.append(msg)
#                     times.append(time.time() - t1)
#                     if pd.to_datetime(msg['time'], unit='s').second == 55:
#                         break
#
#
#                 except queues.Empty:
#                     break
#             data_dict_array = []
#             if len(msg_array) > 1:
#                 for msg in msg_array:
#                     data_dict = {}
#                     for key in KEYS:
#                         data_dict[key] = msg[key]
#                     data_dict_array.append(data_dict)
#
#                 dataframe = pd.DataFrame(data=data_dict_array)
#                 dataframe['time'] = pd.to_datetime(dataframe['time'], unit='s')
#
#
#                 dataframe = dataframe.set_index(pd.to_datetime(dataframe['time']))
#                 dataframe = dataframe.drop('time', axis=1)
#                 print(dataframe.index)
#
#                 store = pd.HDFStore('tick_data' + symbol + currency + '.h5')
#                 store.open()
#                 print('Store ' + symbol + currency + ' was written with length ' + str(len(dataframe)))
#                 # print(dataframe)
#                 store.append(symbol + currency, dataframe, format='t')
#
#                 store.close()

# def set_open_orders(orders):
#     for order in orders:
#
#         if order['order'].ocaGroup != '':
#             open_orders[order['orderId']] = order['order'].ocaGroup

# if __name__ == '__main__':
#
#     test = IBInterface(host='127.0.0.1', port=7496, client_id=101011,account = 'DU517095')
#     test.startApi()
#     #test.reqAllOpenOrders()
#
#
#     import time
#
#     time.sleep(1)
#
#     tickThread(101010, test).start()
#     test.reqIds()
#     test.reqCurrentTime()
#     print("Order ID " + str(test.nextValidOrderId))
#     sleep(2)
#     orders = test.getOpenOrders()
#     open_positions = test.getAccountPositions('DU517095')
#     #set_open_orders(orders)
#     contract = test.create_contract("EURJPY", 'FX')
#
#     test.sell(10000,contract,open_positions['EURJPY'],127.5,128.4)
#     x = test.reqIds()
#     x = test.check_conn()
#     #queryTime = datetime.now().replace(month = -6).strftime("yyyyMMdd HH:mm:ss")
#     #y, data = test.getRealTimeHistoricalData(contract,'','1 D','30 secs','MIDPOINT',1,1)
#     real_time_bars = {}
#     # for contracts in contract_dict:
#     #     contract = test.create_contract(contracts, 'FX')
#     #
#     #     data = test.getHistoricalData(contract,'','2 D','1 min','ASK',1,1)
#     #     store_historical_data_to_file(contract_dict[contracts][0], contract_dict[contracts][1],data)
#     #
#     # for contracts in contract_dict:
#     #     contract = test.create_contract(contracts, 'FX')
#     #     real_time_bars[contracts] = test.reqRealTimeBars(contract, 5, 'ASK', True)
#     # sleep(2)
#     # 1
#     # # contract = test.create_contract('AUDCAD','FX')
#     # #
#     # # y = test.reqRealTimeBars(contract, 5, "ASK", True)
#     # # print(x)
#     # while True:
#     #     for contracts in contract_dict:
#     #         if test.isConnected():
#     #
#     #             store_queue_to_file(real_time_bars[contracts], contract_dict[contracts][0], contract_dict[contracts][1])


# -*- coding: utf-8 -*-
"""
Created on Mon Sep 28 11:41:36 2020

@author: LYL
"""

from pandas import read_csv
#from sklearn.preprocessing import Imputer
import numpy as np
import pandas as pd
import math

bank=read_csv('.../bank-full.csv',sep=';')

print(bank.head(5))
print(bank.shape)
print(bank.describe)
bank.isna().sum()

#Boxplot
import seaborn as sns
from matplotlib import pyplot as plt
plt.figure(figsize=(15,7))
sns.boxplot('age','y',data=bank,orient='h', sym=None)
plt.yticks(fontsize=15.0)
plt.show()

#It seems that there is no "null" value in the frame, however, we can see that there are many "unknown" values, so we still need to deal with them.
#Count the "unknown" value
for col in bank.columns:
    if bank[col].dtype == object:
        print("Percentage of \"unknown\" in %s：" %col ,bank[bank[col] == "unknown"][col].count(),"/",bank[col].count())
print(bank.info())
bank.replace('unknown',np.NaN, inplace=True)
bank.isnull().sum()

bank.drop(columns=["poutcome"])
#bank.drop(rows=[]) drop the outliners rows

#Count the 'Yes' and 'No' in variable 'y'
print("Yes:",bank['y'][bank['y']== 'yes'].count())
print("No:",bank['y'][bank['y']== 'no'].count())
#count the frequency of each value in each variable
for col in bank.columns:
    if bank[col].dtype == object:
        print(bank.groupby(bank[col]).apply(lambda x: x['y'][x['y']== 'yes'].count()/x['y'].count()))

#Change the English words of months into numbers

#Group the variables
month = bank['month']
string_features = bank.columns[bank.dtypes == "object"].to_series().values
int_features = bank.columns[bank.dtypes == "int64"].to_series().values
float_features = bank.columns[bank.dtypes == "float64"].to_series().values
numeric_features = np.append(int_features,float_features)

bin_features = ['default', 'housing', 'loan','y']  #binary features
order_features = ['education']
disorder_features = [ 'contact', 'job']

#Functions to deal with the missing value
def Missing_value_perprocessing_mean (bank_data_small_train,bank_data_small_test):
    col  = bank_data_small_train.columns
    #Train_copy = Train.copy()
    #Fill the missing value with average value
    from sklearn.preprocessing import Imputer
    imp = Imputer(missing_values=np.nan, strategy='mean', axis=0)
    imp.fit(bank_data_small_train)
    bank_data_small_train = imp.transform(bank_data_small_train)
    bank_data_small_test = imp.transform(bank_data_small_test)
    bank_data_small_train = pd.DataFrame(bank_data_small_train,columns = col)
    bank_data_small_test = pd.DataFrame(bank_data_small_test,columns = col)
    return bank_data_small_train,bank_data_small_test

def Missing_value_perprocessing_rf (bank_data_small_train,bank_data_small_test):
    Missing_features_dict = {}
    Missing_features_name = []
    #Find the columns with missing value
    for feature in bank_data_small_train.columns:
        Missing_count = bank_data_small_train[bank_data_small_train[feature].isnull()]['age'].count()
        if Missing_count > 0:
            Missing_features_dict.update({feature: Missing_count})
    #Sort the columns by amount of missing value
    Missing_features_name = sorted(Missing_features_dict.keys(),reverse=True)
    #print(Missing_features_name)
    for feature in Missing_features_name:
        train_miss_data = bank_data_small_train[bank_data_small_train[feature].isnull()]
        train_miss_data_X = train_miss_data.drop(Missing_features_name, axis=1)

        train_full_data = bank_data_small_train[bank_data_small_train[feature].notnull()]
        train_full_data_Y = train_full_data[feature]
        train_full_data_X = train_full_data.drop(Missing_features_name, axis=1)

        test_miss_data = bank_data_small_test[bank_data_small_test[feature].isnull()]
        test_miss_data_X = test_miss_data.drop(Missing_features_name, axis=1)

        test_full_data = bank_data_small_test[bank_data_small_test[feature].notnull()]
        test_full_data_Y = test_full_data[feature]
        test_full_data_X = test_full_data.drop(Missing_features_name, axis=1)
        from sklearn.ensemble import RandomForestClassifier
        #Random forest
        rf = RandomForestClassifier(n_estimators=100)
        rf.fit(train_full_data_X, train_full_data_Y)
        train_miss_data_Y = rf.predict(train_miss_data_X)
        test_miss_data_Y = rf.predict(test_miss_data_X)
        train_miss_data[feature] = train_miss_data_Y
        test_miss_data[feature] = test_miss_data_Y
        bank_data_small_train = pd.concat([train_full_data, train_miss_data])
        bank_data_small_test = pd.concat([test_full_data, test_miss_data])

    return bank_data_small_train,bank_data_small_test

def Missing_value_perprocessing_knn (bank_data_small_train,bank_data_small_test):
    Missing_features_dict = {}
    Missing_features_name = []
    for feature in bank_data_small_train.columns:
        Missing_count = bank_data_small_train[bank_data_small_train[feature].isnull()]['age'].count()
        if Missing_count > 0:
            Missing_features_dict.update({feature: Missing_count})
    Missing_features_name = sorted(Missing_features_dict.keys(),reverse=True)
    from sklearn.neighbors import KNeighborsClassifier
    for feature in Missing_features_name:
        train_miss_data = bank_data_small_train[bank_data_small_train[feature].isnull()]
        train_miss_data_X = train_miss_data.drop(Missing_features_name, axis=1)

        train_full_data = bank_data_small_train[bank_data_small_train[feature].notnull()]
        train_full_data_Y = train_full_data[feature]
        train_full_data_X = train_full_data.drop(Missing_features_name, axis=1)

        test_miss_data = bank_data_small_test[bank_data_small_test[feature].isnull()]
        test_miss_data_X = test_miss_data.drop(Missing_features_name, axis=1)

        test_full_data = bank_data_small_test[bank_data_small_test[feature].notnull()]
        test_full_data_Y = test_full_data[feature]
        test_full_data_X = test_full_data.drop(Missing_features_name, axis=1)

        #K-NN
        knn = KNeighborsClassifier()
        forest = knn.fit(train_full_data_X, train_full_data_Y)

        train_miss_data_Y = knn.predict(train_miss_data_X)
        test_miss_data_Y = knn.predict(test_miss_data_X)

        train_miss_data[feature] = train_miss_data_Y
        test_miss_data[feature] = test_miss_data_Y

        bank_data_small_train = pd.concat([train_full_data, train_miss_data])
        bank_data_small_test = pd.concat([test_full_data, test_miss_data])

    return bank_data_small_train,bank_data_small_test

#Normalization
#def Scale_perprocessing (Train):
#    col  = Train.columns
#    copy = Train.copy()
#    scaler = preprocessing.MinMaxScaler(feature_range=(0,1))
#    copy = scaler.fit_transform(copy)
#    Train = pd.DataFrame(copy,columns = col)
#    return Train

import calendar
list(calendar.month_abbr).index('jan')#input the English words of month

#Divide the age group
bank.loc[(bank.age < 35),  'AgeGroup'] = 'group 1'
bank.loc[(bank.age > 35 and bank.age<50),  'AgeGroup'] = 'group 2'
bank.loc[(bank.age >50 and bank.age <65),  'AgeGroup'] = 'group 3'
bank.loc[(bank.age > 65),  'AgeGroup'] = 'group 4'

#Deal with binary variables
def bin_features_perprocessing (bin_features, bank_data):
    for feature in bin_features:
        new = np.zeros(bank_data[feature].shape[0])
        for rol in range(bank_data[feature].shape[0]):
            if bank_data[feature][rol] == 'yes' :
                new[rol] = 1
            elif bank_data[feature][rol]  == 'no':
                new[rol] = 0
            else:
                new[rol] = None
        bank_data[feature] =  new
    return bank_data

#Sort the order variables
def order_features_perprocessing (order_features,bank_data):
    education_values = [ "primary", "secondary","tertiary",
    "unknown"]
    replace_values = list(range(1,5))    #len(education_values)
    bank_data[order_features] = bank_data[order_features].replace(education_values,replace_values)
    bank_data[order_features] = bank_data[order_features].astype("float")
    return bank_data

#Deal with the rest with onehot
#def disorder_features_perprocessing (disorder_features, bank_data):
#    for features in disorder_features:
#        features_onehot = pd.get_dummies(bank_data[features])
#        features_onehot = features_onehot.rename(columns=lambda x: features+'_'+str(x))
#        bank_data = pd.concat([bank_data,features_onehot],axis=1)
#        bank_data = bank_data.drop(features, axis=1)
#    return bank_data

bank = bin_features_perprocessing(bin_features, bank)
bank = order_features_perprocessing(order_features, bank)
bank = disorder_features_perprocessing(disorder_features, bank)

#Divide the bank data into train and test data
bank.shape[0]
round(bank.shape[0]*0.8)
bank = bank.sample(frac=1,random_state=12)

#import numpy as np
#from sklearn.impute import KNNImputer
#nan = np.nan
#imputer = KNNImputer(n_neighbors=2, weights="uniform")
#using the mean feature value of the two nearest neighbors of samples with missing values:
#imputer.fit_transform(bank)
#bank_train = Scale_perprocessing (bank_train)
#Missing_value_perprocessing
#Average
#bank_train,bank_test = Missing_value_perprocessing_mean(bank_train,bank_test)
#K-NN
#bank_train,bank_test=Missing_value_perprocessing_knn(bank_train,bank_test)
#Random forest
#bank_train,bank_test=Missing_value_perprocessing_rf(bank_train,bank_test)

#X_train = bank_train.drop(['y'], axis=1).copy()
#y_train = pd.DataFrame(bank_train['y'],columns = ['y'])

#X_test = bank_test.drop(['y'], axis=1).copy()
#y_test = pd.DataFrame(bank_test['y'],columns = ['y'])

#X_train = Scale_perprocessing(X_train)
#X_test = Scale_perprocessing(X_test)

#Output to documents
#outputpath1=r'D:\Study\Postgraduate\NTU-ANALYTICS\Data Mining\Project\Preprocess\bank_train.csv'
#bank_train.to_csv(outputpath1, sep=',',index=False,header=True)
#outputpath2=r'D:\Study\Postgraduate\NTU-ANALYTICS\Data Mining\Project\Preprocess\bank_test.csv'
#bank_test.to_csv(outputpath2,sep=',',index=False,header=True)
#outputpath3=r'D:\Study\Postgraduate\NTU-ANALYTICS\Data Mining\Project\Preprocess\X_train.csv'
#X_train.to_csv(outputpath3,sep=',',index=False,header=True)
#outputpath4=r'D:\Study\Postgraduate\NTU-ANALYTICS\Data Mining\Project\Preprocess\X_test.csv'
#X_test.to_csv(outputpath4,sep=',',index=False,header=True)
#outputpath5=r'D:\Study\Postgraduate\NTU-ANALYTICS\Data Mining\Project\Preprocess\y_train.csv'
#y_train.to_csv(outputpath5,sep=',',index=False,header=True)
#outputpath6=r'D:\Study\Postgraduate\NTU-ANALYTICS\Data Mining\Project\Preprocess\y_test.csv'
#y_test.to_csv(outputpath6,sep=',',index=False,header=True)

outputpath7=r'D:\Study\Postgraduate\NTU-ANALYTICS\Data Mining\Project\bank_new.csv'
bank.to_csv(outputpath7, sep=',',index=False,header=True)


import shutil
import numpy as np
import tensorflow as tf

tf.logging.set_verbosity(tf.logging.INFO)

BUCKET = None  # set from task.py
PATTERN = "of" # gets all files

# Determine CSV and label columns
CSV_COLUMNS = "weight_pounds,is_male,mother_age,plurality,gestation_weeks".split(',')
LABEL_COLUMN = "weight_pounds"

# Set default values for each CSV column
DEFAULTS = [[0.0], ["null"], [0.0], ["null"], [0.0]]

# Define some hyperparameters
TRAIN_STEPS = 10000
EVAL_STEPS = None
BATCH_SIZE = 512
NEMBEDS = 3
NNSIZE = [64, 16, 4]

# Create an input function reading a file using the Dataset API
# Then provide the results to the Estimator API
def read_dataset(filename_pattern, mode, batch_size):
    def _input_fn():
        def decode_csv(value_column):
            columns = tf.decode_csv(records = value_column, record_defaults = DEFAULTS)
            features = dict(zip(CSV_COLUMNS, columns))
            label = features.pop(LABEL_COLUMN)
            return features, label

        # Use filename_pattern to create file path
        file_path = "gs://{}/babyweight/preproc/{}*{}*".format(BUCKET, filename_pattern, PATTERN)

        # Create list of files that match pattern
        file_list = tf.gfile.Glob(filename = file_path)

        # Create dataset from file list
        dataset = (tf.data.TextLineDataset(filenames = file_list)  # Read text file
                    .map(map_func = decode_csv))  # Transform each elem by applying decode_csv fn

        # In training mode, shuffle the dataset and repeat indefinitely
        if mode == tf.estimator.ModeKeys.TRAIN:
            num_epochs = None # indefinitely
            dataset = dataset.shuffle(buffer_size = 10 * batch_size)
        else:
            num_epochs = 1 # end-of-input after this

        # This will now return batches of features, label
        dataset = dataset.repeat(count = num_epochs).batch(batch_size = batch_size)

        return dataset
    return _input_fn

# Define feature columns
def get_wide_deep():
    # Define column types
    fc_is_male,fc_plurality,fc_mother_age,fc_gestation_weeks = [\
        tf.feature_column.categorical_column_with_vocabulary_list(key = "is_male",
                                                                  vocabulary_list = ["True", "False", "Unknown"]),
        tf.feature_column.categorical_column_with_vocabulary_list(key = "plurality",
                                                                  vocabulary_list = ["Single(1)", "Twins(2)", "Triplets(3)", "Quadruplets(4)", "Quintuplets(5)", "Multiple(2+)"]),
        tf.feature_column.numeric_column(key = "mother_age"),
        tf.feature_column.numeric_column(key = "gestation_weeks")
    ]

    # Bucketized columns
    fc_age_buckets = tf.feature_column.bucketized_column(source_column = fc_mother_age, boundaries = np.arange(start = 15, stop = 45, step = 1).tolist())
    fc_gestation_buckets = tf.feature_column.bucketized_column(source_column = fc_gestation_weeks, boundaries = np.arange(start = 17, stop = 47, step = 1).tolist())

    # Sparse columns are wide, have a linear relationship with the output
    wide = [fc_is_male,
        fc_plurality,
        fc_age_buckets,
        fc_gestation_buckets]

    # Feature cross all the wide columns and embed into a lower dimension
    crossed = tf.feature_column.crossed_column(keys = wide, hash_bucket_size = 20000)
    fc_embed = tf.feature_column.embedding_column(categorical_column = crossed, dimension = 3)

    # Continuous columns are deep, have a complex relationship with the output
    deep = [fc_mother_age,
        fc_gestation_weeks,
        fc_embed]

    return wide, deep

# Create serving input function to be able to serve predictions later using provided inputs
def serving_input_fn():
    feature_placeholders = {
        "is_male": tf.placeholder(dtype = tf.string, shape = [None]),
        "mother_age": tf.placeholder(dtype = tf.float32, shape = [None]),
        "plurality": tf.placeholder(dtype = tf.string, shape = [None]),
        "gestation_weeks": tf.placeholder(dtype = tf.float32, shape = [None])
    }

    features = {
        key: tf.expand_dims(input = tensor, axis = -1)
        for key, tensor in feature_placeholders.items()
    }

    return tf.estimator.export.ServingInputReceiver(features = features, receiver_tensors = feature_placeholders)

# create metric for hyperparameter tuning
def my_rmse(labels, predictions):
    pred_values = predictions["predictions"]
    return {"rmse": tf.metrics.root_mean_squared_error(labels = labels, predictions = pred_values)}

# Create estimator to train and evaluate
def train_and_evaluate(output_dir):
    wide, deep = get_wide_deep()
    EVAL_INTERVAL = 300 # seconds

    run_config = tf.estimator.RunConfig(
        save_checkpoints_secs = EVAL_INTERVAL,
        keep_checkpoint_max = 3)

    estimator = tf.estimator.DNNLinearCombinedRegressor(
        model_dir = output_dir,
        linear_feature_columns = wide,
        dnn_feature_columns = deep,
        dnn_hidden_units = NNSIZE,
        config = run_config)

    # Illustrates how to add an extra metric
    estimator = tf.contrib.estimator.add_metrics(estimator, my_rmse)
    # For batch prediction, you need a key associated with each instance
    estimator = tf.contrib.estimator.forward_features(estimator, KEY_COLUMN)

    train_spec = tf.estimator.TrainSpec(
        input_fn = read_dataset("train", tf.estimator.ModeKeys.TRAIN, BATCH_SIZE),
        max_steps = TRAIN_STEPS)

    exporter = tf.estimator.LatestExporter(name = "exporter", serving_input_receiver_fn = serving_input_fn, exports_to_keep = None)

    eval_spec = tf.estimator.EvalSpec(
        input_fn = read_dataset("eval", tf.estimator.ModeKeys.EVAL, 2**15),  # no need to batch in eval
        steps = EVAL_STEPS,
        start_delay_secs = 60, # start evaluating after N seconds
        throttle_secs = EVAL_INTERVAL,  # evaluate every N seconds
        exporters = exporter)

    tf.estimator.train_and_evaluate(estimator = estimator, train_spec = train_spec, eval_spec = eval_spec)


import torch
import pytorch_lightning as pl
from misc.utils import compute_loss, decode_text, decode_graph, compute_scores
import torch.nn.functional as F
import torch.nn as nn
import os
from model.grapher import Grapher

def sorted_ls(path):
    mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime
    return list(sorted(os.listdir(path), key=mtime))

class FocalLoss(nn.modules.loss._WeightedLoss):

    def __init__(self, gamma, weight=None):
        super(FocalLoss, self).__init__(weight)
        self.gamma = gamma
        self.weight = weight

    def forward(self, input, target):

        # input: batch_size x vocab_size x d1 x ... x dn x seq_len
        # target: batch_size x d1 x ... x dn x seq_len
        ce_loss = F.cross_entropy(input, target, weight=self.weight, reduction='none')
        p = torch.exp(-ce_loss)
        focal_loss = ((1 - p) ** self.gamma * ce_loss)

        return focal_loss

class LitGrapher(pl.LightningModule):
    def __init__(self,
                 transformer_class,
                 transformer_name,
                 tokenizer,
                 cache_dir,
                 max_nodes,
                 max_edges,
                 edges_as_classes,
                 default_seq_len_edge,
                 num_classes,
                 dropout_rate,
                 num_layers,
                 vocab_size,
                 bos_token_id,
                 eos_token_id,
                 nonode_id,
                 noedge_id,
                 node_sep_id,
                 noedge_cl,
                 edge_classes,
                 focal_loss_gamma,
                 eval_dir,
                 lr,
                 ):
        super().__init__()
        self.save_hyperparameters()

        model = Grapher(transformer_class=transformer_class,
                        transformer_name=transformer_name,
                        cache_dir=cache_dir,
                        max_nodes=max_nodes,
                        edges_as_classes=edges_as_classes,
                        node_sep_id=node_sep_id,
                        default_seq_len_edge=default_seq_len_edge,
                        num_classes=num_classes,
                        dropout_rate=dropout_rate,
                        num_layers=num_layers,
                        vocab_size=vocab_size,
                        bos_token_id=bos_token_id)

        self.model = model
        self.criterion = {'ce': nn.CrossEntropyLoss(reduction='none'), 'focal': FocalLoss(focal_loss_gamma)}
        self.tokenizer = tokenizer
        self.cache_dir = cache_dir
        self.transformer_name = transformer_name
        self.edges_as_classes = edges_as_classes
        self.edge_classes = edge_classes
        self.focal_loss_gamma = focal_loss_gamma
        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id
        self.node_sep_id = node_sep_id
        self.max_nodes = max_nodes
        self.max_edges = max_edges
        self.noedge_cl = noedge_cl
        self.nonode_id = nonode_id
        self.noedge_id = noedge_id
        self.eval_dir=eval_dir
        self.lr = lr

    def training_step(self, batch, batch_idx):

        # target_nodes: batch_size X seq_len_node
        # target_edges: num_nodes X num_nodes X batch_size X seq_len_edge [FULL]
        # target_edges: batch_size X num_nodes X num_nodes [CLASSES]
        text_input_ids, text_input_attn_mask, target_nodes, target_nodes_mask, target_edges = batch

        # logits_nodes: batch_size X seq_len_node X vocab_size
        # logits_edges: num_nodes X num_nodes X batch_size X seq_len_edge X vocab_size [FULL]
        # logits_edges: num_nodes X num_nodes X batch_size X num_classes [CLASSES]
        logits_nodes, logits_edges= self.model(text_input_ids,
                                               text_input_attn_mask,
                                               target_nodes,
                                               target_nodes_mask,
                                               target_edges)

        loss = compute_loss(self.criterion, logits_nodes, logits_edges, target_nodes,
                            target_edges, self.edges_as_classes, self.focal_loss_gamma)

        self.log('train_loss', loss, on_step=True, on_epoch=True, logger=True, sync_dist=True, batch_size=text_input_ids.size(0))

        return loss

    def eval_step(self, batch, batch_idx, split):

        iteration = self.global_step
        rank = self.global_rank

        text_input_ids, text_input_attn_mask, target_nodes, target_nodes_mask, target_edges = batch

        logits_nodes, seq_nodes, logits_edges, seq_edges = self.model.sample(text_input_ids, text_input_attn_mask)

        text_dec = decode_text(self.tokenizer, text_input_ids, self.bos_token_id, self.eos_token_id)

        TB_str = []

        dec_target = decode_graph(self.tokenizer, self.edge_classes, target_nodes, target_edges, self.edges_as_classes,
                                  self.node_sep_id, self.max_nodes, self.noedge_cl, self.noedge_id,
                                  self.bos_token_id, self.eos_token_id)

        dec_pred = decode_graph(self.tokenizer, self.edge_classes, seq_nodes, seq_edges, self.edges_as_classes,
                                self.node_sep_id, self.max_nodes, self.noedge_cl, self.noedge_id,
                                self.bos_token_id, self.eos_token_id)

        if batch_idx == 0:
            for b_i in range(len(text_dec)):
                # ---- ground truth ----
                gt = '
'.join('-->'.join(tri) for tri in dec_target[b_i])

                # ---- predicted  -------
                pr = '
'.join('-->'.join(tri) for tri in dec_pred[b_i])

                strng = f'{b_i}
' + text_dec[b_i] + '
' \
                        + '-' * 40 + 'target' + '-' * 40 + '
' + gt + '
' \
                        + '-' * 40 + 'predicted' + '-' * 20 + '
' + pr + '
'
                TB_str.append(strng)

        decodes = {'text_dec': text_dec, 'dec_target': dec_target, 'dec_pred': dec_pred}

        for i, tb_str in enumerate(TB_str):
            self.logger.experiment.add_text(f'{split}_{rank}/{i}', tb_str, iteration)

        return decodes

    def eval_epoch_end(self, outputs, split):

        iteration = self.global_step
        rank = self.global_rank

        dec_target_all = []
        dec_pred_all = []

        for out in outputs:
            dec_target_all += out['dec_target']
            dec_pred_all += out['dec_pred']

        # make sure number of paths is smaller than 10
        dec_pred_all = [tr[:10] for tr in dec_pred_all]
        dec_target_all = [tr[:10] for tr in dec_target_all]

        # hack to avoid crashing the program if evaluation fails
        try:
            scores = compute_scores(dec_pred_all, dec_target_all, iteration, self.eval_dir, split, rank)
        except Exception:
            return

        for k, v in scores.items():
            self.logger.experiment.add_scalar(f'{split}_score/{k}', v, global_step=iteration)

        self.log_dict(scores)

    def validation_step(self, batch, batch_idx):
        return self.eval_step(batch, batch_idx, 'valid')

    def test_step(self, batch, batch_idx):
        return self.eval_step(batch, batch_idx, 'test')

    def validation_epoch_end(self, outputs):
        self.eval_epoch_end(outputs, 'valid')

    def test_epoch_end(self, outputs):
        self.eval_epoch_end(outputs, 'test')

    def configure_optimizers(self):
        optimizer = torch.optim.AdamW(self.parameters(), self.lr)
        return optimizer


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Write a function that will find all the anagrams of a word from a list. You
will be given two inputs a word and an array with words. You should return an
array of all the anagrams or an empty array if there are none.

Created on Mon Jun 18 20:53:45 2018

@author: shane
"""

def anagrams(word, words):

    output = []

    word = list(word)
    word.sort()

    for item in words:
        temp = list(item)
        temp.sort()
        if temp == word:
            output.append(item)

    return output

if __name__ == '__main__':
    print(anagrams('abba', ['aabb', 'abcd', 'bbaa', 'dada']))


import tensorflow as tf

def deconv2d_resize(inputs, filters, kernel_size, padding='SAME', strides=(1, 1), reuse=None, name=None, activation=None):
    """Resize input using nearest neighbor then apply convolution."""
    shape = inputs.get_shape().as_list()
    height = shape[1] * strides[0]
    width = shape[2] * strides[1]
    resized = tf.image.resize_nearest_neighbor(inputs, [height, width])

    return tf.layers.conv2d(inputs, filters,
                            kernel_size=kernel_size,
                            padding='SAME',
                            strides=strides,
                            reuse=reuse,
                            name=name,
                            activation=activation)


# https://deeplearningcourses.com/c/unsupervised-machine-learning-hidden-markov-models-in-python
# https://udemy.com/unsupervised-machine-learning-hidden-markov-models-in-python
# http://lazyprogrammer.me
# Discrete Hidden Markov Model (HMM) with scaling
from __future__ import print_function, division
from builtins import range
# Note: you may need to update your version of future
# sudo pip install -U future

import numpy as np
import matplotlib.pyplot as plt
from subprocess import run, PIPE

def random_normalized(d1, d2):
    x = np.random.random((d1, d2))
    return x / x.sum(axis=1, keepdims=True)

class HMM:
    def __init__(self, M):
        self.M = M # number of hidden states

    def fit(self, X, max_iter=30):
        np.random.seed(123)
        # train the HMM model using the Baum-Welch algorithm
        # a specific instance of the expectation-maximization algorithm

        # determine V, the vocabulary size
        # assume observables are already integers from 0..V-1
        # X is a jagged array of observed sequences
        V = max(max(x) for x in X) + 1
        N = len(X)

        self.pi = np.ones(self.M) / self.M # initial state distribution
        self.A = random_normalized(self.M, self.M) # state transition matrix
        self.B = random_normalized(self.M, V) # output distribution

        # print("initial A:", self.A)
        # print("initial B:", self.B)

        costs = []
        for it in range(max_iter):
            if it % 10 == 0:
                print("it:", it)
            # alpha1 = np.zeros((N, self.M))
            alphas = []
            betas = []
            scales = []
            logP = np.zeros(N)
            for n in range(N):
                x = X[n]
                T = len(x)
                scale = np.zeros(T)
                # alpha1[n] = self.pi*self.B[:,x[0]]
                alpha = np.zeros((T, self.M))
                alpha[0] = self.pi*self.B[:,x[0]]
                scale[0] = alpha[0].sum()
                alpha[0] /= scale[0]
                for t in range(1, T):
                    alpha_t_prime = alpha[t-1].dot(self.A) * self.B[:, x[t]]
                    scale[t] = alpha_t_prime.sum()
                    alpha[t] = alpha_t_prime / scale[t]
                logP[n] = np.log(scale).sum()
                alphas.append(alpha)
                scales.append(scale)

                beta = np.zeros((T, self.M))
                beta[-1] = 1
                for t in range(T - 2, -1, -1):
                    beta[t] = self.A.dot(self.B[:, x[t+1]] * beta[t+1]) / scale[t+1]
                betas.append(beta)

            cost = np.sum(logP)
            costs.append(cost)

            # now re-estimate pi, A, B
            self.pi = sum((alphas[n][0] * betas[n][0]) for n in range(N)) / N

            den1 = np.zeros((self.M, 1))
            den2 = np.zeros((self.M, 1))
            a_num = np.zeros((self.M, self.M))
            b_num = np.zeros((self.M, V))
            for n in range(N):
                x = X[n]
                T = len(x)
                den1 += (alphas[n][:-1] * betas[n][:-1]).sum(axis=0, keepdims=True).T
                den2 += (alphas[n] * betas[n]).sum(axis=0, keepdims=True).T

                # numerator for A
                # a_num_n = np.zeros((self.M, self.M))
                for i in range(self.M):
                    for j in range(self.M):
                        for t in range(T-1):
                            a_num[i,j] += alphas[n][t,i] * betas[n][t+1,j] * self.A[i,j] * self.B[j, x[t+1]] / scales[n][t+1]
                # a_num += a_num_n

                # numerator for B
                # for i in range(self.M):
                #     for j in range(V):
                #         for t in range(T):
                #             if x[t] == j:
                #                 b_num[i,j] += alphas[n][t][i] * betas[n][t][i]
                for i in range(self.M):
                    for t in range(T):
                        b_num[i,x[t]] += alphas[n][t,i] * betas[n][t,i]
            self.A = a_num / den1
            self.B = b_num / den2
        # print("A:", self.A)
        # print("B:", self.B)
        # print("pi:", self.pi)

        # plt.plot(costs)
        # plt.show()

    def log_likelihood(self, x):
        # returns log P(x | model)
        # using the forward part of the forward-backward algorithm
        T = len(x)
        scale = np.zeros(T)
        alpha = np.zeros((T, self.M))
        alpha[0] = self.pi*self.B[:,x[0]]
        scale[0] = alpha[0].sum()
        alpha[0] /= scale[0]
        for t in range(1, T):
            alpha_t_prime = alpha[t-1].dot(self.A) * self.B[:, x[t]]
            scale[t] = alpha_t_prime.sum()
            alpha[t] = alpha_t_prime / scale[t]
        return np.log(scale).sum()

    def log_likelihood_multi(self, X):
        return np.array([self.log_likelihood(x) for x in X])

    def get_state_sequence(self, x):
        # returns the most likely state sequence given observed sequence x
        # using the Viterbi algorithm
        T = len(x)
        delta = np.zeros((T, self.M))
        psi = np.zeros((T, self.M))
        delta[0] = np.log(self.pi) + np.log(self.B[:,x[0]])
        for t in range(1, T):
            for j in range(self.M):
                delta[t,j] = np.max(delta[t-1] + np.log(self.A[:,j])) + np.log(self.B[j, x[t]])
                psi[t,j] = np.argmax(delta[t-1] + np.log(self.A[:,j]))

        # backtrack
        states = np.zeros(T, dtype=np.int32)
        states[T-1] = np.argmax(delta[T-1])
        for t in range(T-2, -1, -1):
            states[t] = psi[t+1, states[t+1]]
        return states

def valid_conll_eval(fname):

    with open(fname, 'r') as file:
        data = file.read()

    pipe = run(["perl", "eval_conll2000_updated.pl"], stdout=PIPE, input=data, encoding='ascii')
    output = pipe.stdout

    tag_acc = float(output.split()[0])
    phrase_f1 = float(output.split()[1])

    print("tag_acc, phrase_f1", tag_acc, phrase_f1)
    return phrase_f1

def evaluate(predicted, target):
    filename = "temp.txt"
    f = open(filename, "w")
    for i in range(len(predicted)):
        if predicted[i] == 0:
            f.write("x y " + target[i] + " B\n")
        elif predicted[i] == 1:
            f.write("x y " + target[i] + " I\n")
        elif predicted[i] == 2:
            f.write("x y " + target[i] + " O\n")
        else:
            f.write("x y " + target[i] + " N\n")
    f.close()
    fscore = valid_conll_eval(filename)
    return fscore

def process_data():
    X = []
    x = []
    X_val = []
    x_val = []
    T_val = []
    t_val = []
    sequence_syms = dict()
    sequence = list()
    X_t = []
    x_t = []
    T = []
    t = []
    for line in open('../test.txt'):
        words = line.split()
        if len(words) == 0:
            X_t.append(x_t)
            x_t  = []
            T.append(t)
            t = []
            continue
        if words[0] not in sequence_syms:
            sequence_syms[words[0]] = len(sequence)
            sequence.append(words[0])
        x_t.append(sequence_syms[words[0]])
        t.append(words[2][0])

    for line in open('../validation.txt'):
        words = line.split()
        if len(words) == 0:
            X_val.append(x_val)
            x_val  = []
            T_val.append(t_val)
            t_val = []
            continue
        if words[0] not in sequence_syms:
            sequence_syms[words[0]] = len(sequence)
            sequence.append(words[0])
        x_val.append(sequence_syms[words[0]])
        t_val.append(words[2][0])

    for line in open('../train.txt'):
        words = line.split()
        if len(words) == 0:
            X.append(x)
            x  = []
            continue
        if words[0] not in sequence_syms:
            sequence_syms[words[0]] = len(sequence)
            sequence.append(words[0])
        x.append(sequence_syms[words[0]])
    return X, X_val, T_val, X_t, T

def main():
    X, X_val, T_val, X_t, T = process_data()

    num_states = [2, 3, 4]
    best_state_num = 0
    best_fscore = 0
    for n_state in num_states:
        hmm = HMM(n_state)
        hmm.fit(X)
        L = hmm.log_likelihood_multi(X).sum()
        print("LL with fitted params:", L)

        predicted = list()
        target = list()
        for i in range(len(X_val)):
            predicted.extend(hmm.get_state_sequence(X_val[i]))
            target.extend(T_val[i])

        fscore = evaluate(predicted, target)
        if fscore > best_fscore:
            best_fscore = fscore
            best_state_num = n_state

    print("Best number of hidden state:", best_state_num)
    hmm = HMM(best_state_num)
    hmm.fit(X + X_val)
    predicted = list()
    target = list()
    for i in range(len(X_t)):
        predicted.extend(hmm.get_state_sequence(X_t[i]))
        target.extend(T[i])

    evaluate(predicted, target)

if __name__ == '__main__':
    main()


from tinydb import TinyDB

player_database = TinyDB('models_players.json')

class Player:
    def __init__(self,first_name = None,
                       last_name = None,
                       birthdate = None,
                          gender = None,
                         ranking = None,
                           score = 0,
                       player_id = 0
                 ):
        self.first_name = first_name
        self.last_name = last_name
        self.birthdate = birthdate
        self.gender = gender
        self.ranking = ranking
        self.score = score
        self.player_id = player_id

class Game: #Non ici il faut une méthode game
    def __init__(self, game_wins=1,game_loses=0, game_draw=0.5):
        self.game_wins = game_wins
        self.game_loses = game_loses
        self.game_draw = game_draw

#Voici des instances de la classe Player
Jona = Player('Jonathan', 'BUISSON', '09.11.1985', 'Man' ,'700','90','1')
print(Jona.last_name)
Micha = Player('Michael','DWIGHT','01.01.1980','Man','700', '50','2')
print(Micha.score)
Maria = Player('Maria','PEREZ','12.08.1982','Woman','700','20','3')
print(Maria.gender)
Ivana = Player('Ivana','SIDOROV','15.12.1995','Woman','800','40','4')
print(Ivana.ranking)
Clark = Player('Clark','KENT','26.04.1977','Man','1000','100','5')
print(Clark.birthdate)
Zata = Player('Zatana','COOPER','20.03.1990','Woman','900','80','6')
print(Zata.first_name)
Martin = Player('Martin','DUPOND','22.09.1975','Man','990','90','7')
print(Martin.ranking)
Xena = Player('Xena','GABRIEL','29.03.1970','Woman','985','70','8')
print(Xena.player_id)

"""Lorsqu'on remplit un formulaire ou qu'on se présente formellement en anglais,
on donne toujours son prénom puis son nom à l'inverse du français"""
def serialized(self):
    player_infos = {}
    player_infos['Prenom'] = self.first_name
    player_infos['Nom'] = self.last_name
    player_infos['Date_de_naissance'] = self.birthdate
    player_infos['Sexe_ou_Genre'] = self.gender
    player_infos['Classement'] = self.ranking
    player_infos['Score'] = self.score
    player_infos['Id du joueur'] = self.player_id
    return player_infos

def unserialized(self, serialized_player):
    first_name = serialized_player['Prenom']
    last_name = serialized_player['Nom']
    birthdate = serialized_player['Date_de_naissance']
    gender = serialized_player['Sexe_ou_Genre']
    ranking = serialized_player['Classement']
    tournament_score = serialized_player['Score']
    player_id = serialized_player['Id_du_joueur']
    return Player(first_name,
                   last_name,
                   birthdate,
                     gender,
                    ranking,
                      score,
                  player_id
                 )

def __str__(self):
    return f"first_name:{self.first_name} last_name:{self.last_name}, classement: {self.ranking}"

#def __repr__(self):
   # return f"first_name:{self.first_name} last_name:{self.last_name}, classement: {self.ranking}"


import ImportMessage

def is_printable_char(character):
    """
    judge a character is whether a printable character
    :param character: a single character
    :return: True or False
    """
    if character >= 32 and character <= 127:
        return True
    if character == 10 or character == 13:
        return True
    return False

def text_binary_ngram(message,n):
    """
    if message is text or binary, use this function to complete n-gram
    :param message: single message(bytes type)
    :param n: the value of n in n-gram
    :return: the result of n-gram
    """
    if len(message)<n:
        return

    ngram_set = []
    j = 0

    for i in range(len(message)-n+1):
        ngram_set.append((message[i:i+n],j))
        j = j + 1

    return ngram_set

def mixed_ngram(message,n):
    """
    when message is mixed type, segmentation is used, and final n-gram used to the segmentation result
    :param message: mixed type message
    :param n: the value of n-gram
    :return: the result of n-gram
    """
    if len(message)<n:
        return

    ngram_set = []
    ngram_set_tmp = []
    seg_pos = []
    for i in range(len(message) - 1):
        if is_printable_char(message[i]) == True and is_printable_char(message[i + 1]) == False:
            seg_pos.append(i)
        if is_printable_char(message[i]) == False and is_printable_char(message[i + 1]) == True:
            seg_pos.append(i)
    ngram_set_tmp.append(message[0:seg_pos[0] + 1])
    for i in range(len(seg_pos) - 1):
        ngram_set_tmp.append(message[seg_pos[i] + 1:seg_pos[i + 1] + 1])
    ngram_set_tmp.append(message[seg_pos[i+1]+1:])

    k=0
    for i in range(len(ngram_set_tmp)):
        if len(ngram_set_tmp[i]) < n:
            continue
        for j in range(len(ngram_set_tmp[i])-n+1):
            ngram_set.append((ngram_set_tmp[i][j:j+n],k))
            k = k + 1
    return ngram_set

def get_type_ngram(message_type,type,n):
    """
    obtain one message type(text/bin/mixed)'s n-gram(with repetition)
    :param message_type: message list(the same type)
    :param type: text,bin,mixed
    :param n: the value of n-gram
    :return: the result of one message type's ngram
    """
    type_n_gram = []

    if type == 'text' or 'bin':
        for i in range(len(message_type)):
            type_n_gram.append(text_binary_ngram(message_type[i],n))

    if type == 'mixed':
        for i in range(len(message_type)):
            type_n_gram.append(mixed_ngram(message_type[i],n))

    return type_n_gram

def no_repetition_ngram(type_n_gram):
    """
    obatin the all ngram without repetition
    :param type_n_gram: the message's ngram list
    :return: no repetition ngram
    """
    no_repetition_n_gram = []

    for i in range(len(type_n_gram)):
        if type_n_gram[i] == None:
            continue
        for j in range(len(type_n_gram[i])):
            if type_n_gram[i][j] not in no_repetition_n_gram:
                no_repetition_n_gram.append(type_n_gram[i][j])

    return no_repetition_n_gram

def multi_N_ngram(message_type,type,minN,maxN):
    """
    under condition of the multiple n, obtain the ngram result of a messages list(the same type)
    :param message_type: a messages list
    :param type: text,bin or mixed
    :param minN: the min n in ngram
    :param maxN: the max n in ngram
    :return: the ngram result of a messages list
    """
    size = maxN - minN + 1
    some_type_ngram = [0] * size
    k = minN
    for i in range(size):
        some_type_ngram[i] = get_type_ngram(message_type,type,k)
        k = k + 1
    return some_type_ngram

def no_repetition_multi_N_ngram(some_type_ngram):
    """
    obtain the no repetition result of ngram of a messages list under different n
    :param some_type_ngram: the result of ngram of a message list under different n
    :return: no repetition multi N ngram
    """
    nrm_ngram = []
    for i in range(len(some_type_ngram)):
        nrm_ngram.append(no_repetition_ngram(some_type_ngram[i]))
    return nrm_ngram

if __name__ == '__main__':
    a = ImportMessage.import_file('S7_1.pcap')
    b = ImportMessage.import_file('http.pcap')
    c = a + b
    text_type, mixed_type, bin_type = ImportMessage.MessageClassifier(c)
    # mixed_type_ngram = get_type_ngram(mixed_type,'mixed',5)
    # nr_ngram = no_repetition_ngram(mixed_type_ngram)
    # for i in range(len(nr_ngram)):
    #     print(nr_ngram[i])
    # for i in range(len(mixed_type_ngram)):
    #     print(mixed_type_ngram[i])
    some_type_ngram = multi_N_ngram(mixed_type,'mixed',5,7)
    some_type_ngram = no_repetition_multi_N_ngram(some_type_ngram)
    for i in range(len(some_type_ngram)):
        print("")
        for j in range(len(some_type_ngram[i])):
            print(some_type_ngram[i][j])


# BOJ 3078 좋은친구
# 1. 문제조건
#   - 좋은 친구의 조건은 아래와 같다.
#   - 1) 등수 차이가 K를 넘지 않는다
#   - 2) 이름의 길이가 자신과 같아야 한다.
#
# 2. 아이디어
#   - 좋은 친구의 탐색범위는 자기 자신으로 부터 +K까지이다.
#   - 어차피 필요한건 이름 길이고 친구의 이름이 아니니까 input도 len() method로 저장해, 나중에 len() 연산을 하지 않는다. (그럼 그냥 길이 대조만 하면 되니까 -1 연산도 줄여도 되지않나?)
#   - 배열의 j는 i번째 학생의 다음 성적순 학생이다.
#       -> 반복문에서 j는 i+1부터 시작해 i+K까지 진행하고, i는 i+K가 학생수보다 작을 때 까지만 증가한다.

from collections import deque
import sys
input = sys.stdin.readline

N, K = map(int, input().split())
list = [len(input()) for i in range(N)]
cnt = 0

for i in range(N):
    for j in range(i+1, min(i+K+1,N)): #이때 index가 넘어가면 안됨
        if list[i] == list[j]:
            cnt +=1
print(cnt)


from itertools import combinations

T = int(input())
tmp = -1
result = 0
for tc in range(1,T+1):
    arr = list(map(int,input().split()))
    t = -1
    for a in list(combinations(arr,3)):
        if sum(a)%10>t:
            t = sum(a)%10
    if tmp<=t:
        tmp = t
        result = tc
print(result)


'''
Simulation of a selective sweep by a single beneficial mutation using the Out of Africa 2 population demographic
https://popsim-consortium.github.io/stdpopsim-docs/stable/catalog.html#sec_catalog_homsap_models_outofafrica_2t12
'''
# sloppy, but the 'warnings' module ignores a couple future warnings in stdpopsim
import warnings
import pandas as pd
warnings.filterwarnings("ignore")
import stdpopsim
import time

iterations = 5

start = time.time()

species = stdpopsim.get_species("HomSap")  # Homo sapiens
model = species.get_demographic_model("OutOfAfrica_2T12")
samples = {"AFR": 100, "EUR": 100}
# the contig is fully neutral, the sweeping mutation will be inserted later
# note, 'right' implies the length of chr22 to use. The full chromosome is 51 million SNPS in length, but
# we will just use a subset of this chromosome.
contig = species.get_contig("chr22", right=5e6)

'''
The next session will provide settings for the sweep. This includes:
 - randomly selecting a chromosome in a population of our choice
 - The position will be specific within the contig
'''

locus_id = "hard_sweep"
coordinate = round(contig.length / 2)  # this is where the sweep will be located
contig.add_single_site(
    id=locus_id,
    coordinate=coordinate
)

'''
Next the "extended events" are set up, which will modify the demography.
The stdpopsim.ext.selective_sweep() represents a general model for a mutation
that is beneficial within a single population.
'''
extended_events = stdpopsim.ext.selective_sweep(
    single_site_id=locus_id,
     # this is where the mutation starts
    population="AFR",
    selection_coeff=0.1,  # selection coefficient for the mutation [0.1 for humans]
    mutation_generation_ago=300,  # mutation originates 300 gens ago in AFR pop [5k-30k years, or 10k for one]
    min_freq_at_end=0.5  # mutation frequency at present day
)

'''
Now we run the simulation using SLiM. For comparison a neutral simulation will
also be run
'''
engine = stdpopsim.get_engine("slim")
for i in range(iterations):
    # sweep data simulations
    ts_sweep = engine.simulate(
        model,
        contig,
        samples,
        extended_events=extended_events,
        slim_scaling_factor=10,
        slim_burn_in=0.1
    )
    # neutral data simulations
    ts_neutral = engine.simulate(
        model,
        contig,
        samples,
        # no extended events
        slim_scaling_factor=10,
        slim_burn_in=0.1
    )

    '''
    export this data
    '''
    print("Saving VCF")
    # write the simulations in vcf format
    with open("output/sweep/ts_sweep_" + str(i) + ".vcf", "w") as vcf_file:
        ts_sweep.write_vcf(vcf_file)

    with open("output/neutral/ts_neutral_" + str(i) + ".vcf", "w") as vcf_file:
        ts_neutral.write_vcf(vcf_file)

    end = time.time()
    delta = round((end - start) / 60, 2)
    print("Simulation #: ", i, " out of ", iterations, " | Cumulative Time: ", delta, " minutes")


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020/2/15 12:00
# @Author  : Baimohan/PH
# @Site    : https://github.com/BaiMoHan
# @File    : self_in_constructor.py
# @Software: PyCharm

class InConstructor:
    def __init__(self):
        # 在构造方法中定义一个foo变量，局部变量
        foo = 0
        # 使用self代表该构造方法正在初始化的对象
        self.foo = 6
        # 所有使用InConstrutor创建的对象的foo实例变量将被设为6

if __name__ == '__main__':
    print(InConstructor().foo)