#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os
import pytz
import json, time
import  config, helpers
from etoro import Etoro
from analyzer import AnalyzeStocks
from apscheduler.schedulers.blocking import BlockingScheduler
from crawler_logging import logger
from datetime import datetime, timezone, timedelta

def get_etoro_instance():

	# Open Web Driver
	etoro_instance = Etoro()

	#Login To Site
	login_status = etoro_instance.login()

	if login_status is True:

		if config.AccountType == "VIRTUAL":
			logger.info(f"Switching to VIRTUAL Account ...")
			time.sleep(5)
			etoro_instance.switch_account(to_demo=True)
			time.sleep(3)

		if config.AccountType != etoro_instance.current_account_type:
			raise Exception(
				f"Switch Account Failed. should be <{config.AccountType}>"
				f" instead of <{etoro_instance.current_account_type}> !")

		return etoro_instance

	elif login_status is False and (
		etoro_instance.has_server_error or etoro_instance.site_not_reachable()
	):
		retries = 2
		while(login_status is False and retries < etoro_instance.MAX_RETRIES):
			logger.warning(
				f'Error Login :: {config.login_fail_msg}, Auto Retrying:'
				f'{retries}/{etoro_instance.MAX_RETRIES}'
			)
			# del etoro_instance
			# etoro_instance = Etoro()
			etoro_instance.login()
			login_status = etoro_instance.get_loggedin_username()
			retries+=1

		if login_status is False:
			raise Exception("Max retries done to login but failed")

	else:
		raise Exception("Login Failed. Check Username or Password")

def buy_trade(etoro_instance):
	try:
		detail = ""

		# etoro_instance.login()
		#checking  current balance
		clientCredit = helpers.clientCredit(login_data=etoro_instance.get_login_info)

		open_markets_only = config.analyze_open_markets_only
		logger.info(
			f"Analyzing Today Stocks For {'opened' if open_markets_only else 'all'} markets ..."
		)

		analyzer = AnalyzeStocks()

		top_markets = analyzer.today_price_analysis(
			stocks_sort_by=config.stocks_sort_by,
			time_slots_count=24,
			open_markets_only=open_markets_only,
			time_slots_pick=2)

		# top_markets = analyzer.trade_insights(
		# 	etoro_instance.get_insights(),
		#  	open_markets_only=open_markets_only, sort_by="growth")

		for t1, top_market in enumerate(top_markets):
			logger.info(f"\nGoing to open Buying Trade for :\n{top_market}\n")
			buy_trade, buy_trade_res  = etoro_instance.trade(
				ins=top_market.get("InstrumentId"), IsBuy=True)
			if buy_trade is False:
				detail = (
					f"Couldnot open buying position for: '{top_market.get('SymbolFull')}'"
					f" reason :\n{buy_trade_res}\n")
				logger.warning(detail)
			else:
				detail = (
					f"Opened buying position for: '{top_market.get('SymbolFull')}'"
					f" response :\n{buy_trade_res}\n")
				logger.info(detail)
				break

		logger.info(f"\nGoing to update User Date ...")
		user_data = etoro_instance.get_login_info
		logger.info(f"\nGot User Data :\n{user_data}\n")

		logger.info(f"\nGoing to update User Trade History ...")
		user_trade_history = etoro_instance.get_trade_history
		logger.info(f"\nGot User Trade History :\n{user_trade_history}\n")

		msg = '<buy_trade> finished ...'
		print('+'*len(msg))
		print(msg)
		print('+'*len(msg))

	except Exception as e:
		exc_type, exc_obj, exc_tb = sys.exc_info()
		fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
		err_detail = e, fname, exc_tb.tb_lineno
		detail = f"Following Error occured in buy_trade ::\n{err_detail}\n"
		logger.error(detail)

	#write trade logs
	helpers.write_csv("Buying", detail)

def sell_trade(etoro_instance):
	try:
		detail = ""
		# etoro_instance.login()

		logger.info(f"Going to get last Ordered Trades (For Opening Selling Position ) ...")

		lastOrderedTrades = helpers.lastOrderedTrade(isBuy=True)

		if not lastOrderedTrades:
			logger.info(f"No Last Ordered Trades found to open a selling position.")

		for t1, lastOrderedTrade in enumerate(lastOrderedTrades):

			instrumentID = lastOrderedTrade['InstrumentID']
			instrumentData = helpers.find_instrument_by_id(instrumentID)
			lastOrderedTrade.update({k:v for k,v in instrumentData.items()
				if k in ("InstrumentDisplayName", "SymbolFull",)})
			instrumentSymbol = instrumentData["SymbolFull"]
			instrumentDisplayName = instrumentData["InstrumentDisplayName"]
			instrumentTitle = f"{instrumentSymbol} - {instrumentDisplayName}"
			positionID = lastOrderedTrade['PositionID']

			logger.info(f"\n[{t1+1}/{len(lastOrderedTrades)}] :\n{lastOrderedTrade}\n")

			logger.info(f"Going to get User Trade History ...")
			tradeHistory = etoro_instance.get_trade_history

			closedOrder = helpers.isOrderClosed(
				positionID, data_list=tradeHistory,
				path=config.closed_trade_history_file)

			if not closedOrder:
				logger.info(
					f"<[{instrumentTitle}]: postionID->"
					f"{positionID} instrumentID->{instrumentID}>"
					" is not closed yet. skipping this...")
				continue

			# here opens a selling postion for this trade

			logger.info(f"Going to open Selling Trade for: {instrumentTitle}\n")
			sell_trade, sell_trade_res  = etoro_instance.trade(
				ins=instrumentID, IsBuy=False)

			if sell_trade is False:
				detail = (
					f"Couldnot open selling position for: '{instrumentTitle}'"
					f" reason :\n{sell_trade_res}\n")
				logger.warning(detail)
			else:
				detail = (
					f"Opened selling position for: '{instrumentTitle}'"
					f" response :\n{sell_trade_res}\n")
				logger.info(detail)

			logger.info(f"\nGoing to update User Date ...")
			user_data = etoro_instance.get_login_info
			logger.info(f"\nGot User Data :\n{user_data}\n")

			logger.info(f"\nGoing to update User Trade History ...")
			user_trade_history = etoro_instance.get_trade_history
			logger.info(f"\nGot User Trade History :\n{user_trade_history}\n")

		msg = "<sell_trade> finished ..."
		print('+'*len(msg))
		print(msg)
		print('+'*len(msg))

	except Exception as e:
		exc_type, exc_obj, exc_tb = sys.exc_info()
		fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
		err_detail = e, fname, exc_tb.tb_lineno
		detail = f"Following Error occured in sell_trade ::\n{err_detail}\n"
		logger.error(detail)

	#write trade logs
	helpers.write_csv("Selling", detail)

if __name__ == '__main__':
	exchange_to_follow = helpers.get_exchange_info(config.Default_Exchange)
	day_of_week_opens = exchange_to_follow['Market Opens'].split()[0].lower().replace("day","")
	day_of_week_closes = exchange_to_follow['Market Closes'].split()[0].lower().replace("day","")
	exchange_closing = datetime.strptime(exchange_to_follow['Market Closes'].split()[1], "%H:%M")
	hour_minute = exchange_closing - timedelta(minutes=config.Exchange_Schedular_Minutes)
	day_of_week = f"{day_of_week_opens}-{day_of_week_closes}"

	logger.info(f"\nScheduler started at: {datetime.now().astimezone(pytz.timezone('UTC'))}\n")
	logger.info(
		f"Trader For <{exchange_to_follow['Exchange']}> scheduled to run at:"
		f"\n[Day Weeks: {day_of_week}]\t[Exchange closing hour: "
		f"{exchange_closing.hour}:{exchange_closing.minute:02d} UTC]\t[Job starts at: "
		f"{hour_minute.hour}:{hour_minute.minute:02d} UTC]\n")

	#graps etoro instance with logged in driver and swticed to virtual/real account
	etoro_instance = get_etoro_instance()

	#scheduler = BlockingScheduler({'apscheduler.timezone': 'Europe/London'})
	scheduler = BlockingScheduler({'apscheduler.timezone': 'UTC'})

	# schedular to buy_trade
	scheduler.add_job(
		buy_trade, 'cron', args=[etoro_instance], day_of_week=day_of_week,
		minute=hour_minute.minute, hour=hour_minute.hour
	)

	# schedular to sell_trade
	scheduler.add_job(sell_trade,'interval',args=[etoro_instance],minutes=35)

	# scheduler.add_job(buy_trade, 'cron', args=[etoro_instance],
	# 	minute=9, hour=00)

	#scheduler to display scheduled jobs
	scheduler.add_job(lambda : scheduler.print_jobs(),'interval',minutes=10)
	scheduler.start()


#!/usr/bin/python
# -*- coding: utf-8 -*-
from flask import jsonify

class ErrorResponse:
    def __init__(self, message, details):
        self.msg = message
        self.details = details

    def as_json(self):
        return jsonify({
            "Error": True,
            "msg": self.msg,
            "details": self.details
        })

class SuccessResponse:
    def __init__(self, result, message, details):
        self.msg = message
        self.details = details
        self.result = result

    def as_json(self):
        return jsonify({
            "result": self.result,
            "msg": self.msg,
            "details": self.details
        })


from aoc.utils import get_input
from typing import List

def count_valid_permutations(jolts: List[int]) -> int:
    reachable_list = []
    for i, jolt in enumerate(jolts):
        reachable = [j for j in jolts[i + 1:] if j - jolt <= 3]
        reachable_list.append(reachable)

    arrangement_list = [1] + [0] * (len(jolts) - 1)
    for i, jolt in enumerate(jolts[1:]):
        index = i + 1
        arrangements = 0
        for i in range(1, 4):
            if jolt in reachable_list[index - i]:
                arrangements += arrangement_list[index - i]
        arrangement_list[index] = arrangements

    return arrangement_list[-1]

def solve_part_two(jolts_raw: str) -> int:
    jolts = [0] + [int(jolt) for jolt in jolts_raw.splitlines()]
    jolts.sort()
    jolts.append(max(jolts) + 3)
    return count_valid_permutations(jolts)

if __name__ == "__main__":
    jolts_raw = get_input(2020, 10)
    result = solve_part_two(jolts_raw)
    print(result)


import numpy as np
import torch
import torch.nn.functional as F
import os
import matplotlib.pyplot as plt
import inspect

def FDD_BSA(U_flex, x, W2_CX, W2_CY, W2_CZ, Eigens2, TrackDOFs, Nloc, weight, ToOneV, inputDim, outputDim, D, DOF_exc, DOF_flex, ComDevice, switch, g=9.8):
    batch_size = U_flex.size(0)
    nt = U_flex.size(1)
    nf = U_flex.size(2)
    # DOF_exc = 3
    # DOF_rigid = 9
    # DOF_flex = (1 / 3) * (u.shape[-1] - 3 * DOF_rigid)

    # Extracting and ToOne actions for all parameters
    Excitation, U_flex, dU_flex, ddU_flex = ToOne_Action_BSA(ToOneV, x, U_flex, D, DOF_exc, DOF_flex)

    # for i in range(0, ddU_rigid.size(-1)):
    #     plt.figure(i)
    #     plt.plot(ddU_rigid[0, :, i].to(torch.device('cpu')).numpy())
    #     plt.show()
    # Coupling Points Information
    K = 5e6
    C = 1e6
    ConnectNum = 3
    alpha_k = 0.001
    alpha_c = 0.001
    nDOF = TrackDOFs.size(0)
    TrackDOFs = TrackDOFs.long()

    # Computing General Mode Forces for flexible system (Compute the connection displacement and velocity first)
    ConnectDis = torch.cat((Excitation[:, :, 0].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(U_flex, W2_CX.transpose(0, 1)),
                            Excitation[:, :, 1].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(U_flex, W2_CY.transpose(0, 1)),
                            Excitation[:, :, 2].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(U_flex, W2_CZ.transpose(0, 1))), dim=-1)
    ConnectVel = torch.cat((Excitation[:, :, 3].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(dU_flex, W2_CX.transpose(0, 1)),
                            Excitation[:, :, 4].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(dU_flex, W2_CY.transpose(0, 1)),
                            Excitation[:, :, 5].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(dU_flex, W2_CZ.transpose(0, 1))), dim=-1)
    # print('The connection information dimension is {}'.format(ConnectDis.shape))

    Gvec = torch.matmul(K * ConnectDis + C * ConnectVel,
                        torch.cat((W2_CX, W2_CY, W2_CZ), dim=0))

    # Encoding physics laws for Flexible modes
    Du_flex = ddU_flex - Gvec + Eigens2.squeeze(-1) * U_flex + (Eigens2.squeeze(-1) * alpha_k + alpha_c) * dU_flex

    # Applying ODEs weights for all equations, the variable TargetMin is equal to r in Algorithm 1.
    TargetMin = 0.02

    Du_flex *= TargetMin / weight.unsqueeze(-2)[:, :, :U_flex.size(-1)].to(ComDevice)

    # Computing magnitude information for all DU
    Du_Magnitude = torch.cat((torch.std(Du_flex, [0, 1]), torch.mean(Du_flex, [0, 1])))

    # Regenerate the data tensor for derivatives output
    MeanV, StdV = ToOneV[0, 1:, 0], ToOneV[0, 1:, 3]
    DOF_de = DOF_exc + DOF_flex
    dU_flex = (dU_flex - MeanV[DOF_de: DOF_de + DOF_flex]) / StdV[DOF_de: DOF_de + DOF_flex]
    DOF_dde = DOF_de + DOF_flex
    ddU_flex = (ddU_flex - MeanV[DOF_dde: DOF_dde + DOF_flex]) / StdV[DOF_dde: DOF_dde + DOF_flex]

    if switch == 1:
        Derivative_Data = torch.cat([dU_flex, ddU_flex], dim=-1)
    else:
        Derivative_Data = torch.zeros(1, 1).to(ComDevice)
    return Du_flex, Du_Magnitude, Derivative_Data

def BSA_PINO_loss(U_flex, x, weight, ToOneV, inputDim, outputDim, D, DOF_exc, DOF_flex, W2_CX, W2_CY, W2_CZ, Eigens2, TrackDOFs, Nloc, ComDevice, switch):
    batchsize = U_flex.size(0)
    nt = U_flex.size(1)
    U_flex = U_flex.reshape(batchsize, nt, outputDim)

    Du_flex, Du_Magnitude, Derivative_Data = FDD_BSA(U_flex, x, W2_CX, W2_CY, W2_CZ, Eigens2, TrackDOFs, Nloc, weight, ToOneV, inputDim, outputDim, D, DOF_exc, DOF_flex, ComDevice, switch, g=9.8)
    # criterion = torch.nn.SmoothL1Loss()
    criterion = torch.nn.L1Loss()
    loss_f = criterion(Du_flex, torch.zeros_like(Du_flex))
    return loss_f, Du_Magnitude, Derivative_Data

def Var_Encoder_BSA(x):
    Excitation = x[:, 1:-1, 1:]   # Excitations
    return Excitation

def ToOne_Action_BSA(ToOneV, x, U_flex, D, DOF_exc, DOF_flex):
    Excitation = Var_Encoder_BSA(x)
    MeanV, StdV = ToOneV[0, 1:, 0], ToOneV[0, 1:, 3]
    # ReNorm for Excitations, Flexible modes and Rigid motions
    Excitation = Excitation * StdV[:DOF_exc] + MeanV[:DOF_exc]
    U_flex = U_flex * StdV[DOF_exc:DOF_exc + DOF_flex] + MeanV[DOF_exc:DOF_exc + DOF_flex]

    # Derivatives computation
    nt = U_flex.size(1)
    dt = D / nt

    dU_flex = (U_flex[:, 2:, :] - U_flex[:, :-2, :]) / (2 * dt)
    ddU_flex = (U_flex[:, 2:, :] - 2 * U_flex[:, 1:-1, :] + U_flex[:, :-2, :]) / (dt ** 2)

    # Reshape the solution of the system
    U_flex = U_flex[:, 1:-1, :]

    return Excitation, U_flex, dU_flex, ddU_flex


from dataclasses import dataclass
from typing import Any

import pytest

from xcept import Exception_
from tests.tests.exception.test_get_message import (
    BEHAVIOR_TEST_DATA,
    DEFAULT_TEMPLATE_TEST_DATA
)

@pytest.mark.parametrize(*BEHAVIOR_TEST_DATA)
def test_behavior(template: str, test_field_value: str, expected_result: str) -> None:
    @dataclass
    class TestException(Exception_):
        test_field: Any

    exception = TestException(template, test_field=test_field_value)

    assert str(exception) == expected_result

@pytest.mark.parametrize(*DEFAULT_TEMPLATE_TEST_DATA)
def test_default_template(template: str, test_field_value: str, expected_result: str) -> None:
    @dataclass
    class TestException(Exception_):
        DEFAULT_TEMPLATE = template
        test_field: Any

    exception = TestException(None, test_field=test_field_value)

    assert str(exception) == expected_result

def test_missing_template() -> None:
    @dataclass
    class TestException(Exception_):
        pass

    exception = TestException(None)

    assert str(exception) == str()


# %%
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import sklearn
import seaborn as sns
import math
plt.style.use('ggplot')
plt.rcParams['axes.edgecolor'] = 'black'
plt.rcParams['font.sans-serif'] = ['Arial Unicode MS']

# %%
shiduW=pd.read_excel('土壤湿度2012-2022各月份.xls')
youjiwuO=pd.read_excel('内蒙古自治区锡林郭勒盟典型草原不同放牧强度土壤碳氮监测数据集（2012年8月15日-2020年8月15日）.xlsx')

# %%
shiduW.columns

# %%
plt.figure(figsize=(6,5))
shidulen=len(shiduW)
xticklabelsname=[str(shiduW.iloc[i,0])+'年'+str(shiduW.iloc[i,1])+'月' for i in range(len(shiduW))]
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,4]])
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,5]])
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,6]])
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,7]])
plt.xticks(list(range(0,shidulen,11)),xticklabelsname[::11],rotation='vertical')
plt.legend(['10cm','40cm','100cm','200cm'])
plt.ylabel('湿度（kg/m2)')
plt.yticks(list(range(1,3)),['10mm','100mm'])

# %%
# 模拟生成一组实验数据
x = [1,3,5,7]
y = [0.41,0.38,0.43,0.46]
plt.figure(figsize=(8,8))
fig, ax = plt.subplots()
ax.plot(x, y, 'b--')
ax.set_xlabel('x')
ax.set_ylabel('y')

# 二次拟合
coef = np.polyfit(x, y, 2)
y_fit = np.polyval(coef, x)
ax.plot(x, y_fit, 'g')
# 找出其中的峰值/对称点
if coef[0] != 0:
    x0 = -0.5 * coef[1] / coef[0]
    x0 = round(x0, 2)
    ax.plot([x0]*5, np.linspace(min(y),max(y),5),'r--')
    print(x0)
else:
    raise ValueError('Fail to fit.')
plt.xlabel('放牧强度')
plt.ylabel('沙漠化指数SM')
plt.show()
coef

# %%
import math
def f(x):
    res=0
    res+=0.68/(0.04*math.pow(2.7,-x*x/128)+1.36)
    print(0.68/(0.04*math.pow(2.7,-x*x/128)+1.36))
    res+=0.00375*x*x-0.002*x+0.422
    return res

temp=list(range(50))
x=[i/100 for i in temp]
y=[f(i) for i in x]
plt.plot(x,y)
plt.xlabel('放牧强度')
plt.ylabel('SM+B')


import sys

def card2_2164() -> int:
    num = int(sys.stdin.readline())
    if num==1:
        print(1)
        return 0

    result = []
    cards = [i for i in range(1, num+1)]
    while(len(cards) > 1):
        if len(cards)%2:
            result += cards[0:-2:2]
            cards = [cards[-1]] + cards[1::2]
        else :
            result += cards[0::2]
            cards = cards[1::2]
    for re in result:
        print(re, end=" ")
    print(cards[0])

    return 0

if __name__ == "__main__":
    card2_2164()


from functools import reduce
from random import randint

class Packet():
	def __init__(self, stickers):
		self.stickers = stickers

	@classmethod
	def fromRandomSelection(cls, book):
		stickers = set()
		while len(stickers) < book.packetSize:
			s = randint(1, book.collectionSize)
			stickers.add(s)
		return cls(stickers)

	@classmethod
	def fromSpecifiedStickers(cls, book, stickers):
		return cls(stickers)

class StickerBook():
	def __init__(self, name, collectionSize, packetSize):
		self.name = name
		self.collectionSize = collectionSize
		self.packetSize = packetSize

	def __str__(self):
		return "Sticker Book {} with {} stickers in packets of {}".format(self.name, self.collectionSize, self.packetSize)

	def printPacket(self):
		return Packet.fromRandomSelection(self)

class StickerCollection():
	def __init__(self, book, collectionId=None, observer=None):
		self.collectionId = collectionId
		self.book = book
		self.observer=observer
		self.stickers = dict()
		self.packetsBought = 0

	def __str__(self):
		return "Packets Bought: {}, Stickers: {}".format(self.packetsBought, self.stickers)

	def receivePacket(self, packet):
		self.packetsBought += 1
		for s in packet.stickers:
			self.receiveSticker(s)

	def receiveSticker(self, whichSticker):
		if whichSticker in self.stickers:
			self.stickers[whichSticker] += 1
		else:
			self.stickers[whichSticker] = 1
		if self.observer:
			self.observer(self.collectionId, whichSticker, self.stickers[whichSticker])

	def giveAwaySticker(self, whichSticker, recipient):
		if (whichSticker in self.stickers):
			self.stickers[whichSticker] -= 1
			if (self.stickers[whichSticker] == 0):
				del(self.stickers[whichSticker])
			recipient.receiveSticker(whichSticker)
			return True
		return False

	def getSwaps(self):
		swaps = set(filter(lambda x: self.stickers[x] > 1, self.stickers.keys()))
		return swaps

	def isComplete(self):
		return len(self.stickers) == self.book.collectionSize

class GroupOfCollections():
	def __init__(self, book, collections, swapFunction):
		self.book = book
		self.collections = collections
		self.swapsMade = 0
		self.swapFunction = swapFunction

	@classmethod
	def withBookCountAndSwapFunction(cls, book, numberCollections, swapFunction):
		collections = []
		for x in range(numberCollections):
			c = StickerCollection(collectionId=x, book=book)
			collections.append(c)

		return cls(book, collections, swapFunction)

	@classmethod
	def withBookCollectionsAndSwapFunction(cls, book, collections, swapFunction):
		return cls(book, collections, swapFunction)

	def buyPackets(self):
		for c in self.collections:
			if not c.isComplete():
				packet = self.book.printPacket()
				c.receivePacket(packet)

	def runSwaps(self):
		for c in self.collections:
			for d in self.collections:
				if (c is not d):
					self.swapsMade += self.swapFunction(c, d)

	def getPacketsBought(self):
		packetsBought = map(lambda x: x.packetsBought, self.collections)
		total = reduce((lambda x, y: x + y), packetsBought)
		return total

	def isComplete(self):
		for c in self.collections:
			if not c.isComplete():
				return False
		return True

class SwapFunctions():
	# Returns all the swap functions in a dictionary, each function is given a name.
	@staticmethod
	def all():
		allFunctions = []
		allFunctions.append(SwapFunctions.noSwap)
		allFunctions.append(SwapFunctions.transactionalSwap)
		allFunctions.append(SwapFunctions.generousSwap)
		return allFunctions

	# Given two collectors, this function works out which ones each needs that the other has as a swap
	# It returns both sets, which can be unpacked by the caller
	@staticmethod
	def getRespectiveNeeds(first, second):
		# Get the list of swaps
		firstSwaps = first.getSwaps()
		secondSwaps = second.getSwaps()

		# Compose a set of stickers that can be swapped
		firstHasSecondNeeds = set(filter(lambda x: x not in second.stickers, firstSwaps))
		secondHasFirstNeeds = set(filter(lambda x: x not in first.stickers, secondSwaps))

		return firstHasSecondNeeds, secondHasFirstNeeds

	# A swap handler which simply doesn't do swapping, used as a 'control' in measurements
	@staticmethod
	def noSwap(first, second):
		return 0

	# This method works out what both collectors need, and what swaps are available.
	# It then swaps an equal number of stickers from first to second and vice versa
	# This is transactional because a collector only gives with the expectation of receiving
	@staticmethod
	def transactionalSwap(first, second):
		swapsMade = 0

		# Check that both people still need stickers, quit early
		if first.isComplete() or second.isComplete():
			return swapsMade

		# Work out what everyone needs
		firstHasSecondNeeds, secondHasFirstNeeds = SwapFunctions.getRespectiveNeeds(first, second)

		# While there are stickers left in both piles
		while (len(firstHasSecondNeeds) > 0) and (len(secondHasFirstNeeds) > 0):
			# Pick one of mine and theirs to swap
			fromFirst = firstHasSecondNeeds.pop()
			fromSecond = secondHasFirstNeeds.pop()
			swapsMade += 1

			first.giveAwaySticker(fromFirst, second)
			second.giveAwaySticker(fromSecond, first)
			swapsMade += 1

		return swapsMade

	# In this method, when two people are swapping, they pay no attention to how many they receive compared to how many they give.
	# This is 'generous' because each collector seeks to help the other with their swaps regardless of what they get back
	@staticmethod
	def generousSwap(first, second):
		swapsMade = 0

		# Check that at least one of the collectors needs more stickers
		if first.isComplete() and second.isComplete():
			return swapsMade

		# Work out what everyone needs
		firstHasSecondNeeds, secondHasFirstNeeds = SwapFunctions.getRespectiveNeeds(first, second)

		# First gives all required swaps to second
		for fromFirst in firstHasSecondNeeds:
			first.giveAwaySticker(fromFirst, second)
			swapsMade += 1

		# Second gives all required swaps to first
		for fromSecond in secondHasFirstNeeds:
			second.giveAwaySticker(fromSecond, first)
			swapsMade += 1

		return swapsMade


from inspect import signature, getargspec
from dash.dependencies import Input
from dash.exceptions import PreventUpdate
from dash import callback_context

from callback_structs import *
from visualiser import Visualiser
from abstract_dash import AbstractDash

assets_ignore = '.*bootstrap.*'
class DashBoard(AbstractDash):
    def __init__(self, name):
        super().__init__(Visualiser(), name, assets_ignore=assets_ignore)
        self._build_app()

    def run(self):
        self.app.run_server(debug=True)

    def _build_app(self):
        # Add Options
        form_elements, identifiers = self._create_form_elements(self.visualiser, id_prefix=id_prefix)
        update_i.update(identifiers)

        gns = [{"label": c, "value": c} for c in []]
        inp = (self.create_dropdown(load_s["gns"].component_id, gns, multi=True,placeholder="Load Design") +
               self.create_line_break(10) + self.create_button(load_i.component_id,"Submit"))
        acc_elements = [("Load Design", inp)]
        load_accordion = self.create_accordion("proj_accordion", acc_elements)

        form_div = self.create_div(graph_type_o["id"].component_id, form_elements)
        options = self.create_sidebar(not_modifier_identifiers["sidebar_id"], "Options", form_div, className="col sidebar")
        figure = self.visualiser.empty_graph(graph_id)
        graph = self.create_div(update_o["graph_id"].component_id, [figure], className="col")
        graph = self.create_div(load_o.component_id, graph)
        legend = self.create_div(update_o["legend_id"].component_id,[], className="col sidebar")
        elements = options+graph+legend
        container = self.create_div("row-main", elements, className="row flex-nowrap no-gutters")
        self.app.layout = self.create_div("main", load_accordion+container, className="container-fluid")[0]

        # Bind the callbacks
        def load_inner(click,gns):
            return self.load(click,gns)

        def update_graph_inner(*args):
            return self.update_graph(args)

        def export_img_inner(*args):
            return self.export_graph_img(*args)

        self.add_callback(load_inner, [load_i], [load_o],load_s.values())
        self.add_callback(update_graph_inner, update_i.values(), update_o.values())
        self.add_callback(export_img_inner,export_img_i, export_img_o)
        self.build()

    def load(self,click,gns):
        if not gns or not isinstance(self.visualiser, Visualiser):
            raise PreventUpdate()
        ctx = callback_context
        if ctx.triggered:
            self.visualiser.set_design_names(gns)
            figure = self.visualiser.build(graph_id=graph_id)
            d = self.create_div(update_o["graph_id"].component_id, figure, className="col")
            return d
        else:
            raise PreventUpdate()

    def update_graph(self, *args):
        if not isinstance(self.visualiser, Visualiser):
            raise PreventUpdate()
        args = args[0]
        for index, setter_str in enumerate(args):
            if setter_str is not None:
                try:
                    setter = getattr(self.visualiser, setter_str, None)
                    parameter = None
                except TypeError:
                    # Must be a input element rather than a checkbox.
                    # With annonymous implementation this is tough.
                    to_call = list(update_i.keys())[index]
                    parameter = setter_str
                    setter = getattr(self.visualiser, to_call, None)
                if setter is not None:
                    try:
                        if parameter is not None and len(getargspec(setter).args) > 1:
                            setter(parameter)
                        else:
                            setter()
                    except Exception as ex:
                        print(ex)
                        raise PreventUpdate()
        try:
            figure, legend = self.visualiser.build(graph_id=graph_id, legend=True)
            legend = self.create_legend(legend)
            return [figure], legend
        except Exception as ex:
            print(ex)
            raise PreventUpdate()

    def export_graph_img(self, get_jpg_clicks, get_png_clicks, get_svg_clicks):
        action = 'store'
        input_id = None
        ctx = callback_context
        if ctx.triggered:
            input_id = ctx.triggered[0]["prop_id"].split(".")[0]
            if input_id != "tabs":
                action = "download"
        else:
            raise PreventUpdate()
        return [{'type': input_id, 'action': action}]

    def _create_form_elements(self, visualiser, style={}, id_prefix=""):
        default_options = [visualiser.set_full_graph_view,
                           visualiser.set_concentric_layout,
                           visualiser.add_node_no_labels,
                           visualiser.add_edge_no_labels,
                           visualiser.add_standard_node_color,
                           visualiser.add_standard_edge_color]

        options = self._generate_options(visualiser)
        removal_words = ["Add", "Set"]
        elements = []
        identifiers = {}
        for k, v in options.items():
            name = self._beautify_name(k)
            identifier = id_prefix + "_" + k
            element = []

            if isinstance(v, (int, float)):
                min_v = int(v/1.7)
                max_v = int(v*1.7)
                default_val = int((min_v + max_v) / 2)
                step = 1

                element += (self.create_heading_6("", name) +
                            self.create_slider(identifier, min_v, max_v, default_val=default_val, step=step))
                identifiers[k] = Input(identifier, "value")

            elif isinstance(v, dict):
                removal_words = removal_words + \
                    [word for word in name.split(" ")]
                inputs = []
                default_button = None
                for k1, v1 in v.items():
                    label = self._beautify_name(k1)
                    label = "".join(
                        "" if i in removal_words else i + " " for i in label.split())
                    inputs.append({"label": label, "value": k1})
                    if v1 in default_options:
                        default_button = k1

                element = (self.create_heading_6(k, name) +
                           self.create_radio_item(identifier, inputs, value=default_button))
                identifiers[k] = Input(identifier, "value")

            breaker = self.create_horizontal_row(False)
            elements = elements + \
                self.create_div(identifier + "_contamanual_toolbariner",
                                element, style=style)
            elements = elements + breaker

        exports = self.create_heading_4("export_img_heading", "Image Export")
        for e_input in export_img_i:
            exports += self.create_button(e_input.component_id,
                                          className="export_img_button")
            exports += self.create_line_break()
        export_div = self.create_div("export_data_container", exports, style=style)

        return (elements + export_div, identifiers)

    def _beautify_name(self, name):
        name_parts = name.split("_")
        name = "".join([p.capitalize() + " " for p in name_parts])
        return name

    def _generate_options(self, visualiser):
        blacklist_functions = ["empty_graph",
                               "build",
                               "mode",
                               "view",
                               "node_size",
                               "edge_color",
                               "node_color",
                               "copy_settings",
                               "get_design_names",
                               "edge_shape",
                               "edge_text",
                               "node_shape",
                               "node_text",
                               "set_design_names",
                               "get_load_predicates",
                               "get_loaded_design_names"]

        options = {"view": {},
                   "layout": {}}

        for func_str in dir(visualiser):
            if func_str[0] == "_":
                continue
            func = getattr(visualiser, func_str, None)

            if func is None or func_str in blacklist_functions or not callable(func):
                continue

            if len(signature(func).parameters) > 0:
                # When there is parameters a slider will be used.
                # Some Paramterised setters will return there default val if one isnt provided.
                default_val = func()
                if default_val is None:
                    default_val = 1
                options[func_str] = default_val
            else:
                # When no params radiobox.
                if func_str.split("_")[-1] == "preset":
                    option_name = "preset"

                elif func_str.split("_")[-1] == "view":
                    option_name = "view"

                elif func_str.split("_")[-1] == "mode":
                    option_name = "mode"

                elif func_str.split("_")[-1] == "layout":
                    option_name = "layout"

                elif func_str.split("_")[-1] == "connect":
                    option_name = "connect"

                elif "node" in func_str:
                    option_name = "node" + "_" + func_str.split("_")[-1]

                elif "edge" in func_str:
                    option_name = "edge" + "_" + func_str.split("_")[-1]
                elif func_str.split("_")[-1] == "level":
                    option_name = "detail"
                else:
                    option_name = "misc"

                if option_name not in options.keys():
                    options[option_name] = {func_str: func}
                else:
                    options[option_name][func_str] = func
        return options


import numpy as np
import tensorflow as tf

from pdb import set_trace

log_2_e = 1.44269504089  # Constant to convert to binary entropies
epsilon = 1e-12

class MetricsTracker(object):

	def __init__(self):
		self.total_samples = 0
		self.b_ce = tf.keras.metrics.BinaryCrossentropy()
		self.b_acc = tf.keras.metrics.BinaryAccuracy()
		self.preds = np.array([])
		self.flush()

	def flush(self, flush_totals=False):
		if flush_totals:
			self.total_samples = 0
		self.preds = np.array([])
		self.b_acc.reset_states()
		self.b_ce.reset_states()

	def add_observation(self, targets, predictions, loss):
		# Compute overall statistics, gathering types and predictions accordingly
		num_samples = targets.shape[0]
		self.total_samples += int(num_samples)
		self.b_acc.update_state(targets, predictions)
		self.b_ce.update_state(targets, predictions)
		self.preds = np.append(self.preds, predictions.numpy())

	def get_stats(self):
		return self.total_samples, "{0:.3f}".format(self.b_ce.result() * log_2_e), "{0:.2%}".format(
			self.b_acc.result())

	def get_acc(self):
		return self.b_acc.result()

	def get_bce(self):
		return self.b_ce.result() * log_2_e

if __name__ == '__main__':
	import pickle
	import matplotlib.pyplot as plt
	import numpy as np
	from sklearn.metrics import auc, accuracy_score

	y_preds, y_true = [], []
	with open('../data/valid_shared_transformer.pkl', 'rb') as f:
		data = pickle.load(f)
		y_preds, y_true = data
	convert = lambda x: 1 if x > .50 else 0

	print(accuracy_score(y_true, [convert(x) for x in y_preds]))
	""" if aligned and say is aligned
		precision tp / (tp + fp)
		recall tp / (tp + fn)
	"""
	plt.subplots(2, 2)
	data = []
	roc = []
	ys = sorted(list(zip(y_true, y_preds)), key=lambda x: x[1])
	for threshold in np.arange(min(y_preds), max(y_preds), .01):
		fn, tp, fp, tn = 0, 0, 0, 0
		for t, p in ys:
			if t == 1.0:
				if p >= threshold:
					tp += 1
				else:
					fn += 1
			if t == 0.0:
				if p >= threshold:
					fp += 1
				else:
					tn += 1

		recall = tp / (tp + fn)
		precision = tp / (tp + fp)
		tpr = recall
		fpr = fp / (fp + tn)
		data.append((precision, recall, threshold))
		roc.append((tpr, fpr, threshold))

	recalls = [x[1] for x in data]
	precisions = [x[0] for x in data]

	plt.subplot(2, 2, 1)
	plt.plot(recalls, precisions, marker='.')
	plt.xlabel('Recall', fontsize='small')
	plt.ylabel('Precision', fontsize='small')
	plt.title('Aligned and Is Aligned', fontdict={'fontsize': 7})

	tprs = [x[0] for x in roc]
	fprs = [x[1] for x in roc]
	p = plt.subplot(2, 2, 2)
	plt.plot(fprs, tprs, marker='.')
	plt.xlabel('False Positive Rate ' + 'AUC {:.3f}'.format(auc(fprs, tprs)), fontsize='small')
	plt.ylabel('True Positive Rate', fontsize='small')
	plt.title('Aligned and Is Aligned', fontdict={'fontsize': 7})

	""" if not aligned and say is not aligned
		precision tp / (tp + fp)
		recall tp / (tp + fn)
	"""
	data = []
	roc = []
	ys = sorted(list(zip(y_true, y_preds)), key=lambda x: x[1])
	for threshold in np.arange(min(y_preds) + .01, max(y_preds), .01):
		fn, tp, fp, tn = 0, 0, 0, 0
		for t, p in ys:
			if t == 0.0:
				if p < threshold:
					tp += 1
				else:
					fn += 1
			if t == 1.0:
				if p < threshold:
					fp += 1
				else:
					tn += 1

		recall = tp / (tp + fn)
		precision = tp / (tp + fp)
		tpr = recall
		fpr = fp / (fp + tn)
		data.append((precision, recall, threshold))
		roc.append((tpr, fpr, threshold))

	recalls = [x[1] for x in data]
	precisions = [x[0] for x in data]
	plt.subplot(2, 2, 3)
	plt.plot(recalls, precisions, marker='.')
	plt.xlabel('Recall', fontsize='small')
	plt.ylabel('Precision', fontsize='small')
	plt.title('Not Aligned and Is Not Aligned', fontdict={'fontsize': 7})

	tprs = [x[0] for x in roc]
	fprs = [x[1] for x in roc]
	plt.subplot(2, 2, 4)
	plt.plot(fprs, tprs, marker='.')
	plt.xlabel('False Positive Rate ' + 'AUC {:.3f}'.format(auc(fprs, tprs)), fontsize='small')

	plt.ylabel('True Positive Rate', fontsize='small')

	plt.title('Not Aligned and Is Not Aligned', fontdict={'fontsize': 7})
	plt.tight_layout()
	plt.show()


from gnomad_hail import *
import pickle

public_root = 'gs://gnomad-public/papers/2019-high-ad-variants'
temp_root = 'gs://gnomad-tmp/high_ad_hets'
POPS_TO_USE = ['fin', 'nwe', 'onf', 'swe', 'est']

def build_frequency_case(mt):
    """
    Bin frequencies. Output refers to bin floor

    :param MatrixTable mt: Input MatrixTable with .freq annotation
    :return: Binned frequencies
    :rtype: FloatExpression
    """
    return (hl.case()
            .when(mt.freq[0].AF >= 0.2, 0.2)
            .when(mt.freq[0].AF >= 0.10, 0.10)
            .when(mt.freq[0].AF >= 0.05, 0.05)
            .when(mt.freq[0].AF >= 0.01, 0.01)
            .default(0))

def build_length_case(variant_length_expr):
    """
    Bin variant lengths: negative for deletions, positive for insertions, 0 for SNVs.
    Integer refers to bin floor

    :param IntExpression variant_length_expr: Input expression to bin
    :return: Binned variant lengths
    :rtype: IntExpression
    """
    return (hl.case()
            .when(variant_length_expr >= 10, 10)
            .when(variant_length_expr >= 5, 5)
            .when(variant_length_expr >= 1, 1)
            .when(variant_length_expr <= -10, -10)
            .when(variant_length_expr <= -5, -5)
            .when(variant_length_expr <= -1, -1)
            .default(0))

def format_hail_linreg(linreg_expr):
    """
    Format hail linear regression struct from arrays into slope and intercept

    :param StructExpression linreg_expr:
    :return: Formatted StructExpression
    :rtype: StructExpression
    """
    return linreg_expr.annotate(
        beta=linreg_expr.beta[1],
        standard_error=linreg_expr.standard_error[1],
        t_stat=linreg_expr.t_stat[1],
        p_value=linreg_expr.p_value[1],
        intercept_beta=linreg_expr.beta[0],
        intercept_standard_error=linreg_expr.standard_error[0],
        intercept_t_stat=linreg_expr.t_stat[0],
        intercept_p_value=linreg_expr.p_value[0]
    )

def main(args):
    hl.init(log='/high_ad_hets.log')

    for data_type in ('genomes', 'exomes'):
        if args.find_high_ad_hets:
            ht = get_gnomad_public_data(data_type)
            ht = ht.filter((ht.freq[0].AF >= 0.01) & (ht.freq[0].AF <= 0.5) & (hl.len(ht.filters) == 0))
            ht = ht.annotate(variant_length=hl.len(ht.alleles[1]) - hl.len(ht.alleles[0]))
            ht = ht.annotate(
                n_called_homs=ht.freq[0].homozygote_count,
                n_additional_homs=hl.sum(ht.ab_hist_alt.bin_freq[-2:]),  # corresponds to >= 0.9
                freq=build_frequency_case(ht),
                length_group=build_length_case(ht.variant_length)
            ).naive_coalesce(400)
            ht = ht.checkpoint(f'{public_root}/all_variants_{data_type}.ht', overwrite=args.overwrite, _read_if_exists=not args.overwrite)

            res = ht.group_by('freq', 'length_group').aggregate(
                called_homs=hl.agg.mean(ht.n_called_homs),
                added_homs=hl.agg.mean(ht.n_additional_homs),
                n_new_hom_variants=hl.agg.count_where((ht.n_called_homs == 0) & (ht.n_additional_homs > 0)),
                total_with_hom=hl.agg.count_where(ht.n_called_homs > 0),
                total_with_added_hom=hl.agg.count_where(ht.n_additional_homs > 0),
                total_with_1pct_added_hom=hl.agg.count_where(ht.n_additional_homs / ht.n_called_homs > 0.01),
                total_with_2pct_added_hom=hl.agg.count_where(ht.n_additional_homs / ht.n_called_homs > 0.02),
                total_with_5pct_added_hom=hl.agg.count_where(ht.n_additional_homs / ht.n_called_homs > 0.05),
                added_hom_hist=hl.agg.hist(ht.n_additional_homs, 0, 100, 100),
                prop_added_hom_hist=hl.agg.hist(hl.cond(ht.n_called_homs > 0, ht.n_additional_homs / ht.n_called_homs,
                                                        hl.int(ht.n_additional_homs > 0)), 0, 1, 100),
                total=hl.agg.count(),
                overall_ab_hist=hl.agg.array_agg(lambda ab: hl.agg.mean(ab), ht.ab_hist_alt.bin_freq)
            ).checkpoint(f'{public_root}/groupings_{data_type}.ht', overwrite=args.overwrite, _read_if_exists=not args.overwrite)
            res.flatten().export(f'{public_root}/groupings_{data_type}.txt')

        # Note: requires sample-level access to run
        if args.find_high_ad_hets_by_sample:
            mt = get_gnomad_data(data_type, non_refs_only=True, split=True, release_samples=True, release_annotations=True, adj=True, full_meta=True)
            mt = mt.filter_rows((hl.len(mt.filters) == 0) & (mt.freq[0].AF <= 0.5))
            mt = mt.annotate_rows(variant_length=hl.len(mt.alleles[1]) - hl.len(mt.alleles[0]),
                                  freq=build_frequency_case(mt))
            mt = mt.annotate_rows(length_group=build_length_case(mt.variant_length))
            mt = mt.group_rows_by('freq', 'length_group', 'segdup').partition_hint(400).aggregate_rows(
                n_variants=hl.agg.count()
            ).aggregate(
                number_reclassified=hl.agg.count_where(mt.GT.is_het() & (mt.AD[1] / mt.DP > 0.9)),
                number_homvars_with_ref_support=hl.agg.count_where(mt.GT.is_hom_var() & (mt.AD[0] > 0))
            )
            mt = mt.checkpoint(f'{temp_root}/num_reclassified_by_sample_{data_type}.mt', overwrite=args.overwrite, _read_if_exists=not args.overwrite)
            ht = mt.annotate_rows(mean_number_homvars_with_ref_support=hl.agg.mean(mt.number_homvars_with_ref_support),
                                  mean_number_reclassified=hl.agg.mean(mt.number_reclassified),
                                  linreg_number_reclassified=format_hail_linreg(hl.agg.linreg(mt.meta.freemix, [1.0, mt.number_reclassified])),
                                  linreg_number_homvars_with_ref_support=format_hail_linreg(hl.agg.linreg(mt.meta.freemix, [1.0, mt.number_homvars_with_ref_support]))
                                  ).rows()
            ht.flatten().export(f'{public_root}/contamination_summary_{data_type}.txt')

            ht = mt.select_cols(
                number_reclassified=hl.agg.sum(mt.number_reclassified),
                number_homvars_with_ref_support=hl.agg.sum(mt.number_homvars_with_ref_support),
                freemix=mt.meta.freemix
            ).cols().key_by().drop('s')
            ht.flatten().export(f'{temp_root}/num_reclassified_by_sample_all_variants_{data_type}.txt.bgz')
            data = ht.aggregate(linreg_number_reclassified=format_hail_linreg(hl.agg.linreg(ht.freemix, [1.0, ht.number_reclassified])),
                                linreg_number_homvars_with_ref_support=format_hail_linreg(hl.agg.linreg(ht.freemix, [1.0, ht.number_homvars_with_ref_support])))
            pprint(dict(data))
            with hl.hadoop_open(f'{public_root}/contamination_linregs_all_variants_{data_type}.pckl', 'wb') as f:
                pickle.dump(data, f)

    if args.recompute_ccr5_homozygosity:
        hts = []
        for data_type in ('genomes', 'exomes'):
            mt = get_gnomad_data(data_type, adj=True, release_samples=True, raw=True, split=False, full_meta=True)
            mt = mt.filter_rows(mt.locus == hl.parse_locus('3:46414943', 'GRCh37')).select_entries('GT', 'AD', 'DP', 'GQ', 'PL')
            mt = mt.checkpoint(f'{temp_root}/ccr5_{data_type}.mt', overwrite=args.overwrite, _read_if_exists=not args.overwrite)
            mt = hl.filter_alleles_hts(mt, lambda allele, _: hl.is_indel(mt.alleles[0], allele), subset=True)
            mt = annotate_adj(mt)
            if args.remove_contaminated:
                mt = mt.filter_cols(mt.meta.freemix < 0.01)
            mt = mt.select_cols(pop=hl.or_else(mt.meta.subpop, mt.meta.pop)).select_rows()
            hts.append(mt.entries())

        ht = hts[0].union(hts[1])
        ht = ht.filter(hl.array(POPS_TO_USE).contains(ht.pop))
        ht = ht.union(ht.annotate(pop='all'))

        annotations = {
            'reported': hl.agg.call_stats(ht.GT, ['', '']),
            'actual': hl.agg.call_stats(hl.cond(ht.GT.is_het() & (ht.AD[1] / ht.DP > 0.9), hl.call(1, 1), ht.GT), ['', ''])
        }
        ht = ht.group_by(ht.pop).aggregate(**annotations)
        ht = ht.annotate(
            **{f'{ann}_expected_homs': ht[ann].AN * (ht[ann].AF[1] ** 2) / 2 for ann in annotations.keys()}
        )
        ht = ht.annotate(
            **{f'{ann}_bi': ht[ann].homozygote_count[1] / ht[f'{ann}_expected_homs'] for ann in annotations.keys()}
        )
        fname = f'{temp_root}/ccr5_results{"_nocontam" if args.remove_contaminated else ""}.ht'
        ht = ht.checkpoint(fname, overwrite=args.overwrite, _read_if_exists=not args.overwrite)
        ht.select(reported_homs=ht.reported.homozygote_count[1], actual_homs=ht.actual.homozygote_count[1],
                  reported_AF=ht.reported.AF[1], actual_AF=ht.actual.AF[1],
                  reported_expected_homs=ht.reported_expected_homs, actual_expected_homs=ht.actual_expected_homs,
                  reported_bi=ht.reported_bi, actual_bi=ht.actual_bi).show(20)

    if args.recompute_background_distribution:
        POPS_TO_USE.append('all')
        pop_ht = hl.read_table(f'{temp_root}/ccr5_results.ht')
        ccr5_freqs = pop_ht.aggregate(hl.dict(hl.agg.collect((pop_ht.pop, pop_ht.row_value))), _localize=False)

        exome_mt = get_gnomad_data('exomes', adj=True, non_refs_only=True, release_samples=True, release_annotations=True)

        def format_pop_name(pop):
            if pop == 'all':
                return 'gnomad'
            elif pop == 'fin':
                return 'gnomad_fin'
            else:
                return 'gnomad_nfe_' + pop

        exome_mt = exome_mt.filter_rows(hl.any(lambda x: x >= 0.05,
            [exome_mt.freq[exome_mt.freq_index_dict[format_pop_name(pop)]].AF for pop in POPS_TO_USE]
        ))
        exome_mt = exome_mt.select_cols(pop=hl.or_else(exome_mt.meta.subpop, exome_mt.meta.pop))
        annotations = {
            'reported': hl.agg.call_stats(exome_mt.GT, 2),
            'actual': hl.agg.call_stats(hl.cond(exome_mt.GT.is_het() & (exome_mt.AD[1] / exome_mt.DP > 0.9),
                                                hl.call(1, 1), exome_mt.GT), 2)
        }
        exome_mt = exome_mt.group_cols_by(exome_mt.pop).aggregate(**annotations)
        exome_mt = exome_mt.checkpoint(f'{temp_root}/exome_common_freqs.mt', overwrite=args.overwrite, _read_if_exists=not args.overwrite)

        genome_mt = get_gnomad_data('genomes', adj=True, non_refs_only=True, release_samples=True, release_annotations=True)
        genome_mt = genome_mt.filter_rows(hl.is_defined(exome_mt.rows()[genome_mt.row_key]))
        genome_mt = genome_mt.select_cols(pop=hl.or_else(genome_mt.meta.subpop, genome_mt.meta.pop))
        annotations = {
            'reported': hl.agg.call_stats(genome_mt.GT, 2),
            'actual': hl.agg.call_stats(hl.cond(genome_mt.GT.is_het() & (genome_mt.AD[1] / genome_mt.DP > 0.9),
                                                hl.call(1, 1), genome_mt.GT), 2)
        }
        genome_mt = genome_mt.group_cols_by(genome_mt.pop).aggregate(**annotations)
        genome_mt = genome_mt.checkpoint(f'{temp_root}/genome_common_freqs.mt', overwrite=args.overwrite, _read_if_exists=not args.overwrite)

        genome_rows_join = genome_mt.rows()[exome_mt.row_key]
        exome_mt = exome_mt.annotate_rows(
            exome_pass=hl.len(exome_mt.filters) == 0,
            genome_pass=hl.len(genome_rows_join.filters) == 0,
            genome_freq=genome_rows_join.freq
        )

        def join_struct(exome_field, join_field, exome_an_field, genome_an_field):
            ac = exome_field.AC[1] + hl.or_else(join_field.AC[1], 0)
            an = exome_an_field + hl.or_else(genome_an_field, 0)
            hom_count = exome_field.homozygote_count[1] + hl.or_else(join_field.homozygote_count[1], 0)
            af = ac / an
            expected_homs = an * (af ** 2) / 2
            bi = hom_count / expected_homs
            return hl.struct(AC=ac, AN=an, AF=af, homozygote_count=hom_count, expected_homs=expected_homs, bi=bi)

        def format_pop_name_hail(pop):
            return (hl.case()
                    .when(pop == 'all', 'gnomad')
                    .when(pop == 'fin', 'gnomad_fin')
                    .default('gnomad_nfe_' + pop))

        genome_entries = genome_mt[exome_mt.row_key, exome_mt.col_key]
        genome_index = genome_mt.index_globals().freq_index_dict.get(format_pop_name_hail(exome_mt.pop))
        genome_an = hl.cond(hl.is_defined(genome_index), exome_mt.genome_freq[genome_index].AN, 0)
        mt = exome_mt.annotate_entries(
            **{ann: join_struct(
                exome_mt[ann], genome_entries[ann],
                exome_mt.freq[exome_mt.freq_index_dict[format_pop_name_hail(exome_mt.pop)]].AN, genome_an)
                for ann in annotations.keys()},
        )
        def agg_struct(ac, an, homozygote_count):
            ac = hl.int32(hl.agg.sum(ac))
            an = hl.int32(hl.agg.sum(an))
            hom_count = hl.int32(hl.agg.sum(homozygote_count))
            af = ac / an
            expected_homs = an * (af ** 2) / 2
            bi = hom_count / expected_homs
            return hl.struct(AC=ac, AN=an, AF=af, homozygote_count=hom_count, expected_homs=expected_homs, bi=bi)

        full_mt = mt.group_cols_by(pop='all').aggregate(
            **{ann: agg_struct(mt[ann].AC, mt[ann].AN, mt[ann].homozygote_count)
            for ann in annotations.keys()}
        )
        mt = mt.union_cols(full_mt)
        mt = mt.annotate_cols(ccr5_result=ccr5_freqs[mt.pop])
        mt = mt.filter_rows(mt.locus.in_autosome())

        def compute_p_against_background(mt, tol = 0.01):
            reported_criteria = ((mt.reported.AF >= mt.ccr5_result.reported.AF[1] - tol) &
                                               (mt.reported.AF <= mt.ccr5_result.reported.AF[1] + tol) &
                                               (mt.reported.AN > 0.8 * mt.ccr5_result.reported.AN))
            recomputed_criteria = ((mt.recomputed.AF >= mt.ccr5_result.recomputed.AF[1] - tol) &
                                             (mt.recomputed.AF <= mt.ccr5_result.recomputed.AF[1] + tol) &
                                             (mt.recomputed.AN > 0.8 * mt.ccr5_result.recomputed.AN))
            recomputed_criteria_pass = ((mt.recomputed.AF >= mt.ccr5_result.recomputed.AF[1] - tol) &
                                    (mt.recomputed.AF <= mt.ccr5_result.recomputed.AF[1] + tol) &
                                    mt.exome_pass & mt.genome_pass &
                                    (mt.recomputed.AN > 0.8 * mt.ccr5_result.recomputed.AN))

            return mt.annotate_cols(
                # Reported variant selection
                p_reported_callrate=hl.agg.filter(reported_criteria, hl.agg.fraction(mt.reported.bi < mt.ccr5_result.reported_bi)),
                total_reported_callrate=hl.agg.count_where(reported_criteria),
                mean_bi_reported_callrate=hl.agg.filter(reported_criteria, hl.agg.mean(mt.reported.bi)),
                median_bi_reported_callrate=hl.agg.filter(reported_criteria, hl.agg.approx_quantiles(mt.reported.bi, 0.5)),

                # Redone variant selection
                p_recomputed_callrate=hl.agg.filter(recomputed_criteria, hl.agg.fraction(mt.recomputed.bi < mt.ccr5_result.recomputed_bi)),
                total_recomputed_callrate=hl.agg.count_where(recomputed_criteria),
                mean_bi_recomputed_callrate=hl.agg.filter(recomputed_criteria, hl.agg.mean(mt.recomputed.bi)),
                median_bi_recomputed_callrate=hl.agg.filter(recomputed_criteria, hl.agg.approx_quantiles(mt.recomputed.bi, 0.5)),

                # Redone variants selection PASS
                p_recomputed_pass=hl.agg.filter(recomputed_criteria_pass, hl.agg.fraction(mt.recomputed.bi < mt.ccr5_result.recomputed_bi)),
                total_recomputed_pass=hl.agg.count_where(recomputed_criteria_pass),
                mean_bi_recomputed_pass=hl.agg.filter(recomputed_criteria_pass, hl.agg.mean(mt.recomputed.bi)),
                median_bi_recomputed_pass=hl.agg.filter(recomputed_criteria_pass, hl.agg.approx_quantiles(mt.recomputed.bi, 0.5)),

            ).cols()
        compute_p_against_background(mt).write(f'{temp_root}/ccr5_recalc.ht', overwrite=args.overwrite)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--find_high_ad_hets', help='Find high AD hets from public data', action='store_true')
    parser.add_argument('--find_high_ad_hets_by_sample', help='Find high AD hets from individual level data', action='store_true')
    parser.add_argument('--recompute_ccr5_homozygosity', help='Recompute frequency and HWE for CCR5-del32', action='store_true')
    parser.add_argument('--remove_contaminated', help='Remove contaminated samples from CCR5 computation', action='store_true')
    parser.add_argument('--recompute_background_distribution', help='Recompute frequency and HWE for CCR5-del32', action='store_true')
    parser.add_argument('--overwrite', help='Overwrite all data', action='store_true')
    args = parser.parse_args()

    main(args)


from typing import List
from collections import defaultdict

class Solution:
    def leastBricks(self, wall: List[List[int]]) -> int:

        h = defaultdict(int)
        m = len(wall)
        for i in range(m):
            s = 0
            for j in range(len(wall[i]) - 1):

                s += wall[i][j]
                h[s] += 1

        return m - max(h.values() or [0])


def _expand_key(key, n):
    temp_key, pos = [], 0
    while(pos < n):
        temp_key.append(key[pos % len(key)])
        pos += 1
    return "".join(temp_key)

def encrypt(text, key):
    key = _expand_key(key, len(text))
    encryption = []
    for x, k in zip(text, key):
        encryption.append(chr((ord(x)-ord('a') + ord(k)-ord('a')) % 26 + ord('a')))
    return "".join(encryption)

def decrypt(text, key):
    key = _expand_key(key, len(text))
    decryption = []
    for x, k in zip(text, key):
        decryption.append(chr(((ord(x)-ord('a')) - (ord(k)-ord('a')) + 26) % 26 + ord('a')))
    return "".join(decryption)

def main():
    print("Select Action:\n1. Encrypt a string.\n2. Decrypt a string.")

    while True:
        objective = int(input("Enter Option (1 or 2): "))
        if(objective==1 or objective==2):
            break
        else:
            print("Enter 1 or 2. Try again!")

    text = input("Enter string:\n")
    key = input("Enter key: ")

    words = []
    for c in text:
        if c.isalpha(): words.append(c)

    if(objective == 1):
        result = (encrypt("".join(words).lower(), key.lower()))
    elif(objective == 2):
        result = (decrypt("".join(words).lower(), key.lower()))

    print("\nResult :----")
    j = 0
    for c in text:
        if c.isalpha():
            print(result[j], end='')
            j += 1
        else: print(c, end='')

if __name__ == "__main__":
    main()


def bubble(a):
    l=len(a)
    for i in range(l):
        for j in range(l-i-1):
            if a[j]>a[j+1]:
                a[j+1],a[j]=a[j],a[j+1]
    return a

print ("Enter array elements")
a=input().split()
a=[int(x) for x in a]
print("Sorted array")
a=bubble(a)
for i in a:
    print(i)


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
from geopy.geocoders import Nominatim, Bing
from ordered_set import OrderedSet
import time
import os
import logging
import geopy

loger = logging.getLogger(__name__)

class AddKeywordMatchesPipeline(object):

    matches=list(OrderedSet([

        # List extracted with keyword analysis from InterventionsUrgenceEnvironnementQuebec.json:

        "déversement", "produits pétroliers", "coloration", "hydrocarbures", "résidus", "matières dangereuses",
        "odeur", "huile", "eaux usées", "gaz", "fuite", "moteur", "diesel", "pneus", "eaux de procédé", "matières résiduelles",
        "ciment", "distillat", "milieu humide", "hydrique", "eau", "rivière", "sol", "air", "infrastructure", "souterraine", "surface",
        "ministère des transports", "sûreté du québec", "suivi", "travaux", "réservoir",

        # English and French keywords for general green job findings

        "climate", "animal", "wildlife", "biomass", "pollution", "conservation", "biodiversity",
        "nature", "ecotourism", "sustainable", "renewable", "energy", "education", "food",
        "agriculture", "organic", "farming", "forest", "green", "social",  "business", "entrepreneurship",
        "leadership", "media", "journalism", "food security", "health", "ocean", "bike", "recycle", "waste",
        "intership",

        "climat", "animal", "faune", "biomasse", "pollution", "conservation", "biodiversité",
        "nature", "écotourisme", "durable", "renouvelable", "énergie", "éducation", "alimentation",
        "agriculture", "biologique", "agriculture", "forêt", "vert", "social", "entreprise", "esprit d'entreprise",
        "leadership", "médias", "journalisme", "sécurité alimentaire", "santé", "océan", "vélo", "recyclage", "déchets",
        "stage",

    ]))

    def process_item(self, item, spider):

        item['keywords_matched'] = [ m for m in self.matches if m in str(repr(item)).lower() ]

        return item

class AddGeocodePipeline(object):
    """
    Fill the 'latitude' , 'longitude' and other gocoder infos objects.

    Look for env variable 'BING_MAPS_KEYS' to init bing maps coder.
    Add Nominatim coder as a backup.
    """

    def __init__(self, *args, **kw):
        super().__init__(*args, **kw)
        self.coders = []

        bing_key = os.environ.get('BING_MAPS_KEYS', None)
        if bing_key:
            self.coders.append( Bing(api_key=bing_key, timeout=10) )
        self.coders.append( Nominatim(user_agent="InterventionsUrgenceEnvironnementQuebec",
            timeout=10) )

    def _geocode(self, lieu, municipalite, region, coder, retry=1):
        time.sleep(1)
        address_str = "{}, {} {}, Québec, Canada".format(lieu, municipalite, region)
        loger.info("Geocoding {} with coder {}".format(address_str, coder.__class__.__name__))

        try:

            resp = coder.geocode(address_str,
                exactly_one=True)
            if resp:
                return (resp.latitude, resp.longitude, resp.raw)
            else:
                loger.warning("Cannot find exact match for address {}".format(address_str))
                time.sleep(1)
                resp = coder.geocode(address_str,
                exactly_one=False)
                if resp:
                    return ('No exact match', 'No exact match', resp.raw)
                else:
                    loger.error("Geocoder {} cannot find any places for address {}".format(
                        coder.__class__.__name__, address_str))
                    return None

        except geopy.exc.GeocoderQuotaExceeded:
            return None
        except geopy.exc.GeopyError:
            if retry > 0:
                time.sleep(2)
                return self._geocode(lieu, municipalite, region, coder, retry=retry-1)
            else:
                return None

    def geocode(self, lieu, municipalite, region):
        """Resolve latitude, longitude and geocoder_infos or put 'Error'
        """
        result = None
        for _, coder in enumerate(self.coders):

            result = self._geocode(lieu, municipalite, region, coder=coder)
            if result:
                return result

    def process_item(self, item, spider):
        # Use geopy https://geopy.readthedocs.io/en/latest/

        resp = self.geocode(item['lieu'], item['municipalite'], item['region'])
        if resp:
            item['latitude'] = resp[0]
            item['longitude'] = resp[1]
            item['geocoder_infos'] = resp[2]
        else:
            item['latitude'] = 'Error'
            item['longitude'] = 'Error'
        return item


import pm4py

def execute_script():
    log = pm4py.read_xes("../tests/input_data/receipt.xes")
    fea_df = pm4py.extract_features_dataframe(log, include_case_id=True)
    # sets the case ID as index for the dataframe, so a row for a specific case
    # can be retrieved
    fea_df.set_index("case:concept:name", inplace=True)
    # identifies the features for the case with identifier "case-10017"
    features_per_case = fea_df.loc["case-10017"].to_dict()
    print(features_per_case)

if __name__ == "__main__":
    execute_script()


arr = [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 16]
item = 2
left = 0
right = len(arr) - 1

mid = (left + right)//2
print('Initial mid is: %d and item %d' %(mid, arr[mid]))
cycle = 1
while item != arr[mid] and left <= right:
    if item > arr[mid]:
        left = mid + 1 #which is currently item 4
    elif item < arr[mid]:
        right = mid - 1 #which is currently item 2

    mid = (left + right)//2
    print('%d cycles mid is: %d and item %d' %(cycle, mid, arr[mid]))
    cycle += 1

if item == arr[mid]:
    print('%d LOC in array is %d' %(item, mid+1))
    print('found in cycle number: %d' %cycle)
else:
    print('Item not found')


import os, re
import requests, json
import argparse

requests.packages.urllib3.disable_warnings()

header = "LeagueOfLegendsClient/12.12.448.6653"
def printJSON(data):
	print(json.dumps(data, ensure_ascii=False, indent=4))
def getAuth(folder, sucPrint=True, errPrint=True):
	if os.path.isdir(folder):
		for file in sorted(os.listdir(folder),reverse=True):
			reLog = re.match(r'.+?_LeagueClientUx\.log', file)
			if reLog:
				try:
					with open("{}\\{}".format(folder,file), 'r', encoding="utf-8") as data:
						reAuth = re.search(r'(https://riot:.+?@127\.0\.0\.1:\d+?)/(index|bootstrap)\.html\"', data.read())
				except:
					with open("{}\\{}".format(folder,file), 'r') as data:
						reAuth = re.search(r'(https://riot:.+?@127\.0\.0\.1:\d+?)/(index|bootstrap)\.html\"', data.read())
				if reAuth:
					auth = reAuth.group(1).strip()
					print("已獲取AuthInfo：{}\n".format(auth)) if sucPrint else print(end="")
					return auth
	print("錯誤：無法從LOL安裝路徑獲取授權資訊，請嘗試輸入安裝路徑。") if errPrint else print(end="")
	return False
def getParser():
	parser = argparse.ArgumentParser()
	parser.add_argument("-d", "--dir", "--folder", default="C:\\Garena\\Games\\32775\\Game\\Logs\\LeagueClient Logs")
	parser.add_argument("-t", "--token",  default="")
	return parser
def getChampDict(auth):
	# /lol-game-data/assets/v1/champion-icons/1.png
	# https://raw.communitydragon.org/latest/plugins/rcp-be-lol-game-data/global/default/v1/champion-icons/1.png
	url = auth + "/lol-summoner/v1/current-summoner" #url = auth + "/lol-login/v1/session"
	res = requests.get(url, verify=False).json()
	summonerId = res["summonerId"]

	url = f"{auth}/lol-champions/v1/inventories/{summonerId}/champions"
	res = requests.get(url, verify=False).json()[1:]

	'''url = auth + "/lol-champions/v1/owned-champions-minimal"
	res = requests.get(url, verify=False).json()'''
	champDict = { int(champ['id']): [ champ['name'], champ['alias'] ] for champ in res }

	#champDict[895] = ["淣菈" , "Nilah"]
	with open("champ.json", "w", encoding='utf-8') as f:
		json.dump(champDict, f, indent=4, ensure_ascii=False)
	roleDict = {'mage':[], 'fighter':[], 'tank':[], 'assassin':[], 'support':[], 'marksman':[]}
	for champ in res:
		for role in champ['roles']:
			roleDict[role].append(int(champ['id']))
	roleMap = {'mage':'法師', 'fighter':'鬥士', 'tank':'坦克', 'assassin':'刺客', 'support':'輔助', 'marksman':'射手'}
	for role in roleMap:
		roleDict[roleMap[role]] = roleDict[role]
		del roleDict[role]
	return champDict, roleDict
def getChallenges(auth):
	# /lol-game-data/assets/assets/challenges/config/303401/tokens/challenger.png
	# https://raw.communitydragon.org/latest/plugins/rcp-be-lol-game-data/global/default/assets/challenges/config/303401/tokens/challenger.png
	url = auth + "/lol-challenges/v1/challenges/local-player"
	res = requests.get(url, verify=False).json()
	chalList = []
	targetList = ["availableIds", "description", "descriptionShort", "levelToIconPath", "name", "thresholds"]
	for chal in res:
		if chal['category'] == "TEAMWORK":
			#if chal['availableIds'] or chal['name'] == "同業俱樂部":
			if chal['availableIds']:
				chalList += [ {key: value for key, value in chal.items() if key in targetList} ]
	return chalList

def output2CSV():
	jsonData = []
	with open("challenge.tsv", "w", encoding='utf8') as data:
		for chal in chalList:
			description = chal['descriptionShort'].replace("<em>", "").replace("</em>", "")
			byEng = sorted(chal['availableIds'], key = lambda x: (champDict[x][1], x)) # 照英文排序
			tmpListCH = [ champDict[champ][0] for champ in byEng ]
			data.write(f"{chal['name']}\t{description}\t{len(byEng)}\t{','.join(tmpListCH)}\n")
			jsonData.append({'name':chal['name'], 'description':description, 'idList':byEng, 'all5':(False if '3' in chal['description'] else True), 'icon':chal['levelToIconPath']['CHALLENGER'].lower()})
		for role in roleDict:
			description = "以 5 人組隊，使用 5 名同一職業（刺客、法師、射手、坦克、輔助或鬥士）的英雄贏得多場對戰"
			byEng = sorted(roleDict[role], key = lambda x: (champDict[x][1], x)) # 照英文排序
			tmpListCH = [ champDict[champ][0] for champ in byEng ]
			data.write(f"{role}\t\t{len(byEng)}\t{','.join(tmpListCH)}\n")
			jsonData.append({'name':f"{role}", 'description':description, 'idList':byEng, 'all5':True, 'icon':"/lol-game-data/assets/ASSETS/Challenges/Config/303408/Tokens/CHALLENGER.png".lower()})
	with open("data.json", "w", encoding='utf-8') as f:
		json.dump(jsonData, f, indent=4, ensure_ascii=False)

def main():
	global champDict, chalList, roleDict
	parser = getParser()
	args = parser.parse_args()
	auth = getAuth(args.dir)
	champDict, roleDict = getChampDict(auth)
	chalList = getChallenges(auth)
	output2CSV()

if __name__ == '__main__':
	main()


import os

from jupyter_server.base.handlers import JupyterHandler
from jupyter_server.extension.handler import (
    ExtensionHandlerJinjaMixin,
    ExtensionHandlerMixin,
)
from jupyter_server.utils import url_path_join as ujoin
from jupyterlab_server import LabServerApp

from main import AskemJupyterApp

HERE = os.path.join(os.path.dirname(__file__), "dev_ui")

def _jupyter_server_extension_points():
    return [{"module": __name__, "app": DevAskemJupyterApp}]

class DevHandler(ExtensionHandlerJinjaMixin, ExtensionHandlerMixin, JupyterHandler):
    """
    Serve a notebook file from the filesystem in the notebook interface
    """

    def get(self):
        """Get the main page for the application's interface."""
        # Options set here can be read with PageConfig.getOption
        mathjax_config = self.settings.get("mathjax_config", "TeX-AMS_HTML-full,Safe")
        mathjax_url = self.settings.get(
            "mathjax_url",
            "https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js",
        )

        config_data = {
            # Use camelCase here, since that's what the lab components expect
            "baseUrl": self.base_url,
            "token": self.settings["token"],
            "notebookPath": "test.ipynb",
            "fullStaticUrl": ujoin(self.base_url, "static", self.name),
            "frontendUrl": ujoin(self.base_url, "example/"),
            "mathjaxUrl": mathjax_url,
            "mathjaxConfig": mathjax_config,
        }
        return self.write(
            self.render_template(
                "index.html",
                static=self.static_url,
                base_url=self.base_url,
                token=self.settings["token"],
                page_config=config_data,
            )
        )

class DevAskemJupyterApp(AskemJupyterApp):
    name = __name__
    load_other_extensions = True

    extension_url = "/dev_ui"
    default_url = "/dev_ui"
    app_url = "/dev_ui"
    app_settings_dir = os.path.join(HERE, "build", "application_settings")
    schemas_dir = os.path.join(HERE, "build", "schemas")
    static_dir = os.path.join(HERE, "build")
    templates_dir = os.path.join(HERE, "templates")
    themes_dir = os.path.join(HERE, "build", "themes")
    user_settings_dir = os.path.join(HERE, "build", "user_settings")
    workspaces_dir = os.path.join(HERE, "build", "workspaces")
    log_level = 10

    def initialize_handlers(self):
        super().initialize_handlers()
        """Add dev handler"""
        self.handlers.append(("/dev_ui", DevHandler))

if __name__ == "__main__":
    DevAskemJupyterApp.launch_instance()


import pandas as pd
import numpy as np
import os
from sklearn.model_selection import StratifiedShuffleSplit
import numpy as np
import torch
from torch.utils.data import Dataset
import pickle
from collections import Counter

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset
from collections import Counter
import pickle as pkl
import random
import pdb
from torch.autograd import Variable
import string
import argparse

punctuations = string.punctuation

stop_words = ["i","me","my","myself","we","our","ours","ourselves","you","your","yours","yourself","yourselves","he","him","his","himself","she","her","hers","herself","it","its","itself","they","them","their","theirs","themselves","what","which","who","whom","this","that","these","those","am","is","are","was","were","be","been","being","have","has","had","having","do","does","did","doing","a","an","the","and","but","if","or","because","as","until","while","of","at","by","for","with","about","against","between","into","through","during","before","after","to","from","up","down","in","on","off","again","further","then","once","here","there","when","where","why","how","all","any","both","each","few","more","other","some","such","only","own","same","so","than","too","very","s","t","can","will","just","don","should","now"]

parser = argparse.ArgumentParser()

parser.add_argument('--source_path', type=str, default="./processed_data/",
                    help='path for the dataset. default=./processed_data/')

parser.add_argument('--dest_path', type=str, default='./processed_data/',
                    help='path where the processed dataset has to be stored, default=./processed_data')

parser.add_argument('--df_name', type=str, default="df_train.csv",
                    help='Name of the train dataset to process. default=df_train.csv')

parser.add_argument('--df_val', type=str, default="df_val.csv",
                    help='Name of the validation dataset to process. default=df_val.csv')

parser.add_argument('--df_save_name', type=str, default="idized_hollow_train.csv",
                    help='Name of the df to be saved. default=idized_hollow_train.csv')

parser.add_argument('--val_save_name', type=str, default="idized_hollow_val.csv",
                    help='Name of the validation df to be saved. default=idized_hollow_val.csv')

parser.add_argument('--n_vocab', type=int, default=50000,
                    help='Size of the vocabulary. default=50000')

parser.add_argument('--load_tok_id', type=str, default="None",
                    help='Token and ids of vocabulary. default=token_and_id.pk')

args = parser.parse_args()

def fun_apply(x):
    return [tok.lower() for tok in x.split(" ") if(tok not in punctuations and tok not in stop_words)]

def split(df):
    df['tokenized'] = df['extracted_content'].apply(fun_apply)
    return df

def build_vocab(all_tokens):
    token_counter = Counter(all_tokens)
    vocab, count = zip(*token_counter.most_common(MAX_VOCAB))
    id2token = list(vocab)
    token2id = dict(zip(vocab, range(2,2+len(vocab))))
    id2token = ['<pad>', '<unk>'] + id2token
    token2id['<pad>'] = PAD_IDX
    token2id['<unk>'] = UNK_IDX
    return token2id, id2token

def token2index_dataset(df,token2id):
    indices_data = []
    for tokens in df['tokenized']:
        index_list = [token2id[token] if token in token2id else UNK_IDX for token in tokens]
        indices_data.append(index_list)
    df['idized'] = indices_data
    return df

if __name__=='__main__':

    MAX_VOCAB = args.n_vocab
    PAD_IDX = 0
    UNK_IDX = 1
    print("reading train data")
    df_train = pd.read_csv(args.source_path+args.df_name)
    df_train = split(df_train)

    if args.load_tok_id=="None":
        all_toks = []
        for i,row in df_train.iterrows():
            all_toks+=row['tokenized']

        token_2id, id2_token = build_vocab(all_toks)

        with open(args.dest_path+"token_and_id.pk",'wb') as f:
            pickle.dump(token_2id,f)
            pickle.dump(id2_token,f)
            pickle.dump(all_toks,f)

    else:
        with open(args.load_tok_id,'rb') as f:
            token_2id = pickle.load(f)
            id2_token = pickle.load(f)
            all_toks = pickle.load(f)

    df_train = token2index_dataset(df_train,token_2id)
    print("reading val data")
    if args.df_val!="None":
        df_val = pd.read_csv(args.source_path+args.df_val)
        df_val = split(df_val)

        df_val = token2index_dataset(df_val,token_2id)
        df_val.to_csv(args.dest_path+args.val_save_name, index=None)

    df_train.to_csv(args.dest_path+args.df_save_name,index=None)


from testtools.matchers import Equals, Not

from snapcraft_legacy.extractors._metadata import ExtractedMetadata
from tests.legacy import unit

class ExtractedMetadataTestCase(unit.TestCase):
    def test_init(self):
        metadata = ExtractedMetadata(summary="summary")

        self.assertThat(metadata.get_summary(), Equals("summary"))
        self.assertThat(metadata.get_description(), Equals(None))

    def test_update_merge(self):
        metadata = ExtractedMetadata(summary="summary")
        metadata2 = ExtractedMetadata(description="description")
        metadata.update(metadata2)

        self.assertThat(metadata.get_summary(), Equals("summary"))
        self.assertThat(metadata.get_description(), Equals("description"))

    def test_update_overwrite(self):
        metadata = ExtractedMetadata(summary="summary", description="description")
        metadata2 = ExtractedMetadata(description="new description")
        metadata.update(metadata2)

        self.assertThat(metadata.get_summary(), Equals("summary"))
        self.assertThat(metadata.get_description(), Equals("new description"))

    def test_overlap(self):
        metadata = ExtractedMetadata(summary="summary", description="description")
        metadata2 = ExtractedMetadata(description="new description")

        self.assertThat(metadata.overlap(metadata2), Equals({"description"}))

    def test_eq(self):
        metadata1 = ExtractedMetadata(summary="summary")
        metadata2 = ExtractedMetadata(summary="summary")
        self.assertThat(metadata1, Equals(metadata2))

    def test_empty_eq(self):
        self.assertThat(ExtractedMetadata(), Equals(ExtractedMetadata()))

    def test_not_eq(self):
        metadata1 = ExtractedMetadata(summary="summary")
        metadata2 = ExtractedMetadata(description="description")
        self.assertThat(metadata1, Not(Equals(metadata2)))

    def test_len(self):
        metadata = ExtractedMetadata(version="version")
        self.assertThat(len(metadata), Equals(1))

        metadata = ExtractedMetadata(summary="summary", version="version")
        self.assertThat(len(metadata), Equals(2))

    def test_to_dict_partial(self):
        metadata = ExtractedMetadata(summary="summary")
        self.assertThat(metadata.to_dict(), Equals({"summary": "summary"}))

    def test_to_dict_complete(self):
        metadata = ExtractedMetadata(summary="summary", description="description")
        self.assertThat(
            metadata.to_dict(),
            Equals({"summary": "summary", "description": "description"}),
        )

    def test_to_dict_is_a_copy(self):
        metadata = ExtractedMetadata(summary="summary")
        metadata_dict = metadata.to_dict()
        metadata_dict["summary"] = "edited summary"

        # Ensure the metadata cannot be edited with its dict
        self.assertThat(metadata.get_summary(), Equals("summary"))

class TestExtractedMetadataGetters:
    scenarios = [
        ("common_id", {"prop": "common_id", "value": "test-value"}),
        ("summary", {"prop": "summary", "value": "test-value"}),
        ("description", {"prop": "description", "value": "test-value"}),
        ("version", {"prop": "version", "value": "test-value"}),
        ("grade", {"prop": "grade", "value": "test-value"}),
        ("icon", {"prop": "icon", "value": "test-value"}),
        ("desktop_file_paths", {"prop": "desktop_file_paths", "value": ["test-value"]}),
    ]

    properties = (
        "common_id",
        "summary",
        "description",
        "version",
        "grade",
        "icon",
        "desktop_file_paths",
    )

    def test_getters(self, tmp_work_path, prop, value):
        metadata = ExtractedMetadata(**{prop: value})
        for p in self.properties:
            gotten = getattr(metadata, "get_{}".format(p))()
            if p == prop:
                assert gotten == value
            else:
                assert gotten is None


import unittest

from poker.card import Card
from poker.validators import TwoPairValidator

class TwoPairValidatorTest(unittest.TestCase):

    def setUp(self):

        self.seven_of_hearts = Card(rank = "7", suit = "Hearts")
        self.seven_of_spades = Card(rank = "7", suit = "Spades")
        self.ten_of_hearts = Card(rank = "10", suit = "Hearts")
        self.ten_of_spades = Card(rank = "10", suit = "Spades")

        self.cards = [
            Card(rank = "5", suit = "Clubs"),
            Card(rank = "5", suit = "Diamonds"),
            self.seven_of_hearts,
            self.seven_of_spades,
            self.ten_of_hearts,
            self.ten_of_spades,
            Card(rank = "Jack", suit = "Spades")
        ]

    def test_validates_that_cards_have_two_pairs(self):
        validator = TwoPairValidator(cards = self.cards)
        self.assertEqual(
            validator.is_valid(),
            True
        )

    def test_returns_two_pairs_from_card_collection(self):
        validator = TwoPairValidator(cards = self.cards)
        self.assertEqual(
            validator.valid_cards(),
            [
                Card(rank = "Jack", suit = "Spades"),
                self.seven_of_hearts,
                self.seven_of_spades,
                self.ten_of_hearts,
                self.ten_of_spades
            ]
        )


from django.shortcuts import render,redirect
from .models import Post

def main(request):
    postes = Post.objects.all()
    context = {
        'postes':postes,
    }
    if request.method == 'POST':
        username = request.user.username
        description = request.POST['description']

        if len(description)>10:
                post = Post(fromWho = username,description=description,likeCount=0,commentCount=0)
                post.save()
                print('Paylaşım yapıldı.')
                return redirect('main')
        else:
            print('yetersiz karakter')
            return redirect('main')

    else:
        return render(request,'main/main.html',context)


##################################
### AUTHOR:
###     BENJAMIN WITTENBRINK
###     JOHN KOHLER
###     EVA BATELAAN
###     MICHELLE LAHRKAMP
###     THOMAS BRINK
###     MS&E 231 FINAL PROJECT
##################################

#%%
import tweepy
import argparse
import pandas as pd
import csv
import os
import time
import pickle
from datetime import datetime
import numpy as np

#%%
def load_politician_accounts(house, senate, prez):
    politicians_set = set()
    for filename in [house, senate, prez]:
        path = os.path.abspath(filename)
        with open(path) as f:
            account_list = csv.reader(f)
            for row in account_list:
                politicians_set.add(tuple(row))

    return politicians_set

#%%
def get_following_lists(client, users):
    for user in users:
        id = user[1]

        completed_files = os.listdir("data/following_lists")
        if np.any([str(id) in f for f in completed_files]):
            continue # we have already pulled this user's following list

        user = client.get_user(id = id, user_fields=["public_metrics"])
        if user.data is not None and user.data.public_metrics["following_count"] > 5000:
            continue # we will ignore this user's following

        paginator = tweepy.Paginator(
		    client.get_users_following,
		    id = id,
		    user_fields=['username'],
            max_results=1000
        )

        following = set()
        for rsp in paginator:
            time.sleep(60)
            if rsp.data is not None:
                for followee in rsp.data:
                    following.add((followee["username"],str(followee["id"]), followee["name"]))

        filename = "data/following_lists/" + id + "_following.pkl"
        with open(filename, "wb") as f:
            pickle.dump(following, f)

    return

#%%
def create_full_accounts_list(politicians):
    follower_lists_dir = os.listdir("data/following_lists")
    follower_lists = []
    for filename in follower_lists_dir:
        path = "data/following_lists/" + filename
        with open(path, "rb") as f:
            list = pickle.load(f)
        follower_lists.append(list)

    follower_counts = {}
    for list in follower_lists:
        for user in list:
            id = user[1]
            if id not in follower_counts.keys():
                follower_counts[id] = 1
            else:
                follower_counts[id] += 1

    politician_ids = set()
    for politician in politicians:
        id = politician[1]
        politician_ids.add(id)

    accounts_dict = {}
    for list in follower_lists:
        for user in list:
            id = user[1]
            if follower_counts[id] >= 15 or id in politician_ids:
                if id not in accounts_dict.keys():
                    accounts_dict[id] = user

    for politician in politicians:
        id = politician[1]
        if id not in accounts_dict.keys():
            print("lonely politician:" + politician[0])
            accounts_dict[id] = politician

    accounts_list = []
    for id in accounts_dict.keys():
        accounts_list.append(accounts_dict[id])

    filename = "data/accounts.pkl"
    with open(filename, "wb") as f:
        pickle.dump(accounts_list, f)

#%%
def pull_tweets(client, accounts):
    for account in accounts:
        id = account[1]

        completed_files = os.listdir("data/new_tweets")
        if np.any([str(id) in f for f in completed_files]):
            continue # we have already pulled this user's tweets

        start = datetime(year=2022,month=10,day=15,hour=0,minute=0,second=0)
        end = datetime(year=2022,month=12,day=1,hour=0,minute=0,second=0)

        paginator = tweepy.Paginator(
		    client.get_users_tweets,
		    id = id,
		    start_time = start,
            end_time = end,
            tweet_fields=[
			"author_id", "created_at", "public_metrics",
            "lang", "context_annotations"
            ],
            user_fields=["username"],
            max_results=100
        )

        tweets = []

        for rsp in paginator:
            time.sleep(1)
            if rsp.data is not None:
                for tweet in rsp.data:
                    tweets.append(tweet.data)

        # dump paginator
        filename = "data/new_tweets/" + str(id) + "_tweets.pkl"
        with open(filename, "wb") as f:
            pickle.dump(tweets, f)

    return

#%%
## MAIN
if __name__ == "__main__":
    parser = argparse.ArgumentParser(
		description="Get Following Matrix from Politician Twitter Accounts"
	)
    parser.add_argument("--keyfile", help="file with user credentials", required=True)

    flags = parser.parse_args()
    creds = {}
    for line in open(flags.keyfile, "r"):
        row = line.strip()
        if row:
            key, value = row.split()
            creds[key] = value

    politicians = load_politician_accounts("data/115th-Congress-House-seeds.csv", "data/115th-Congress-Senate-seeds.csv", "data/2016-presidential-candidates-seeds.csv")

    client = tweepy.Client(bearer_token=creds["bearer_token"])

    get_following_lists(client, politicians)

    create_full_accounts_list(politicians)
    with open("data/accounts.pkl", "rb") as f:
        full_accounts_list = pickle.load(f)

    get_following_lists(client, full_accounts_list)

# %%


import os, csv
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker

engine = create_engine(os.getenv("DATABASE_URL"))
db = scoped_session(sessionmaker(bind=engine))

file = open("/Dataset/books.csv")
reader = csv.reader(file)
for isbn, title, author, year in reader:

    db.execute("INSERT INTO books (isbn, title, author, year) VALUES (:isbn, :title, :author, :year)",
                {"isbn": isbn,
                 "title": title,
                 "author": author,
                 "year": year})

    print(f"Added book {title} to database.")
    db.commit()

return render_template("/HTML/BookMain.html")


# An assembly library for the VM code > Hack machine language translation. @DimitarYordanov17

class VirtualMachineLibrary:
    """
    Main class to map the Virtual Machine intermediate language to Hack machine language
    """

    def _get_primary(operation, a=None, b=None, treat_a_as_pointer=True, treat_b_as_pointer=True):
        """
        Define primary operations, which are going to be main building "blocks" of higher instructions
        """

        bytecode_dictionary = {
            "sp++": ["@SP", "M=M+1"],
            "sp--": ["@SP", "M=M-1"],
        }

        if operation == "*a=*b":
            load_b_into_d = [f"@{b}", "D=M"]

            if treat_b_as_pointer:
                load_b_into_d.insert(1, "A=M")

            load_d_into_a = [f"@{a}", "M=D"]

            if treat_a_as_pointer:
                load_d_into_a.insert(1, "A=M")

            load_b_into_a = load_b_into_d + load_d_into_a

            return load_b_into_a

        else:
            return bytecode_dictionary[operation]

    def get_arithmetic(instruction, function_block, file_name, total_instructions):
        """
        Returns bytecode for arithmetic instructions
        add | x + y
        sub | x - y
        neg | - y
        eq  | x == y
        gt  | x > y
        lt  | y < x
        and | x && y
        or  | x || y
        not | !y
        """

        direct_arithmetic_commands = {"add": "+", "sub": "-", "and": "&", "or": "|"}
        conditional_arithmetic_commands = {"eq": "JEQ", "gt": "JGT", "lt": "JLT"}  # I can just preppend "J" to the type and .upper(), because they match, but the symmetry would be ruined
        unary_commands = ["neg", "not"]

        final_bytecode = []

        if instruction in direct_arithmetic_commands:
            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp--"))  # sp--
            final_bytecode.extend(["@SP", "A=M", "D=M"])  # D=*sp

            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp--"))  # sp--
            final_bytecode.extend(
                ["@SP", "A=M", f"D=M{direct_arithmetic_commands[instruction]}D"])  # D=*sp (operand)  D

            final_bytecode.extend(["@SP", "A=M", "M=D"])  # *sp = D
            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp++"))  # sp++

        elif instruction in conditional_arithmetic_commands:
            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp--"))  # sp--
            final_bytecode.extend(["@SP", "A=M", "D=M"])  # D=*sp

            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp--"))  # sp--
            final_bytecode.extend(["@SP", "A=M", "D=M-D"])  # D=*sp -  D

            write_none_label = ":".join([file_name, function_block, str(total_instructions + 17), "WRITENONE"])
            increment_sp_label = ":".join([file_name, function_block, str(total_instructions + 20), "INCREMENTSP"])

            final_bytecode.extend([f"@{write_none_label}",
                                   f"D;{conditional_arithmetic_commands[instruction]}"])  # @WRITENONE, jump if the corresponding condition matches with D"s (x-y) value

            final_bytecode.extend(["@SP", "A=M", "M=0"])  # (WRITEZERO) block, *sp=0 (false)
            final_bytecode.extend(
                [f"@{increment_sp_label}", "0;JMP"])  # Jump instantly to sp++ part, skipping write -1

            final_bytecode.extend([f"({write_none_label})", "@SP", "A=M", "M=-1"])  # (WRITENONE) block, *sp=-1 (true)

            final_bytecode.extend([f"({increment_sp_label})"])
            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp++"))  # sp++

        else:  # unary command
            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp--"))  # sp--

            final_bytecode.extend(["@SP", "A=M", "D=M"])  # D=*sp

            if instruction == "not":
                final_bytecode.extend(["@SP", "A=M", "M=!D"])  # *sp = !D
            else:
                final_bytecode.extend(["@SP", "A=M", "M=-D"])

            final_bytecode.extend(VirtualMachineLibrary._get_primary("sp++"))  # sp++

        return final_bytecode

    def get_memory(instruction, file_name):
        """
        Returns the full memory access bytecode, which consists of:
        1. Loading address calculation in R13
        2. Decrementing SP, if pop else saving R13"s content into current SP available location
        3. Saving SP value in R13, if pop else incrementing SP
        """

        instruction_structure = instruction.split()
        instruction_type = instruction_structure[0]
        segment = instruction_structure[1]

        index = instruction_structure[2]

        calculated_address_bytecode = VirtualMachineLibrary._get_address_calculation(segment, index, file_name)

        if instruction_type == "push":
            treat_b_as_pointer = segment != "constant"  # If we don"t have a constant segment, then b (R13 in this case) must be treated as a pointer

            save_R13_into_stack_bytecode = VirtualMachineLibrary._get_primary("*a=*b", a="SP", b="R13",
                                                                              treat_b_as_pointer=treat_b_as_pointer)
            increment_sp = VirtualMachineLibrary._get_primary("sp++")

            return calculated_address_bytecode + save_R13_into_stack_bytecode + increment_sp

        else:
            decrement_sp = VirtualMachineLibrary._get_primary("sp--")
            save_stack_into_R13 = VirtualMachineLibrary._get_primary("*a=*b", a="R13", b="SP")

            return calculated_address_bytecode + decrement_sp + save_stack_into_R13

    def _get_address_calculation(segment, index, file_name):
        """
        Returns bytecode that loads address calculation (segment base address + index) in R13
        """

        if segment == "constant":  # Temp starts at 5
            load_bytecode = [f"@{index}", "D=A"]

        elif segment == "temp":
            load_bytecode = [f"@{int(index) + 5}", "D=A"]

        elif segment == "static":
            variable_name = file_name + "." + index
            load_bytecode = [f"@{variable_name}", "D=A"]

        elif segment == "pointer":
            if index == "0":
                register = "THIS"
            else:
                register = "THAT"

            load_bytecode = [f"@{register}", "D=A"]

        else:
            load_bytecode = [f"@{VirtualMachineLibrary._get_symbolic_symbol(segment)}", "D=M", f"@{index}", "D=D+A"]

        full_address_bytecode = load_bytecode + ["@R13", "M=D"]
        return full_address_bytecode

    def _get_symbolic_symbol(segment):
        """
        Returns Hack symbolic symbol equivalents
        """

        bytecode_dictionary = {
            "local": "LCL",
            "argument": "ARG",
            "this": "THIS",
            "that": "THAT",
        }

        try:
            return bytecode_dictionary[segment]
        except:  # If the segment is not available, it is most likely a variable, so just return it
            return segment

    def get_program_flow(instruction, label, function_block):
        """
        Returns full program flow instruction bytecode
        1. goto label
        2. label
        3. if-goto label
        """

        if instruction == "label":  # Set a label
            bytecode = [f"({function_block}{'$' if function_block else ''}{label})"]
        elif instruction == "goto": # Unconditional jumping
            bytecode = [f"@{label}", "0;JMP"]
        else: # Conditional jumping
            bytecode = []

            # Pop into D
            bytecode.extend(VirtualMachineLibrary._get_primary("sp--"))
            bytecode.extend(["@SP", "A=M", "D=M"])

            # Jump if D is not 0
            bytecode.extend([f"@{function_block}{('$' if function_block else '')}{label}", "D;JNE"])

        return bytecode

    def get_function(instruction_structure, total_instructions, file_name):
        """
        Returns full function instruction bytecode
        function function_name lVars
        call function_name nArgs
        return
        """

        state = ["LCL", "ARG", "THIS", "THAT"]
        instruction = instruction_structure[0]

        if instruction == "function":
            function_name = instruction_structure[1]
            vars_count = int(instruction_structure[2])

            bytecode = []

            # Start a function block
            bytecode.extend([f"({function_name})"])

            for _ in range(vars_count):
                bytecode.extend(VirtualMachineLibrary.get_memory("push constant 0", file_name))

        elif instruction == "call":
            function_name = instruction_structure[1]
            args_count = instruction_structure[2]

            bytecode = []

            return_label = ":".join([file_name, function_name, str(total_instructions), "RETURN"])

            # Push return address
            bytecode.extend([f"@{return_label}"])
            bytecode.extend(["D=A", "@SP", "A=M", "M=D"])
            bytecode.extend(VirtualMachineLibrary._get_primary("sp++"))

            # Save state
            for address in state:
                bytecode.extend([f"@{address}", "D=M", "@R13", "M=D"])
                bytecode.extend(VirtualMachineLibrary._get_primary("*a=*b", a="SP", b="R13", treat_b_as_pointer=False))
                bytecode.extend(VirtualMachineLibrary._get_primary("sp++"))

            # Set ARG to point to new base address (sp - 5 - args_count)
            bytecode.extend(["@SP", "D=M", "@5", "D=D-A", f"@{args_count}", "D=D-A", "@ARG", "M=D"])

            # Set LCL to point to current SP
            bytecode.extend(["@SP", "D=M", "@LCL", "M=D"])

            # Jump to function_name
            bytecode.extend([f"@{function_name}", "0;JMP"])

            # Set return label
            bytecode.extend([f"({return_label})"])

            bytecode = bytecode

        else:
            bytecode = []

            # Set R13 to point to callee"s LCL
            bytecode.extend(["@LCL", "D=M", "@R13", "M=D"])

            # Set R14 to return address
            bytecode.extend(["@R13", "D=M", "@5", "D=D-A", "A=D", "D=M", "@R14", "M=D"])

            # Set first callee"s argument to be return value
            bytecode.extend(VirtualMachineLibrary._get_primary("sp--"))
            bytecode.extend(VirtualMachineLibrary._get_primary("*a=*b", a="ARG", b="SP"))

            # Reposition SP to be after first callee"s argument
            bytecode.extend(["@ARG", "D=M+1", "@SP", "M=D"])

            # Restore registers
            for index, address in enumerate(reversed(state)):
                bytecode.extend(["@R13", "D=M", f"@{int(index) + 1}", "D=D-A", "A=D", "D=M", f"@{address}", "M=D"])

            # Return jump
            bytecode.extend(["@R14", "A=M", "0;JMP"])

        return bytecode