import abc
from typing import List, Dict

from lexer import Token, Block
from parser.arg_types import ArgumentType

class AbstractMacro(metaclass=abc.ABCMeta):
    pass

class MacroBuiltin(AbstractMacro, metaclass=abc.ABCMeta):
    @abc.abstractmethod
    def __init__(self):
        self.name = None
        self.arg_names = []
        self.arg_types = []

    def _check_args(self, namespace: "NameSpace", args: List[Token]):
        args_by_names = {}
        for tp, name, arg in zip(self.arg_types, self.arg_names, args):
            args_by_names[name] = tp(arg, namespace=namespace)
        return args_by_names

    def _compile(self, namespace: "NameSpace", args: Dict[str, ArgumentType]):
        raise NotImplementedError("Implement or not to call this function")

    def compile(self, namespace: "NameSpace", args: List[Token]):
        args_by_names = self._check_args(namespace, args)
        return self._compile(namespace, args_by_names)

class MacroBuiltinBlock(AbstractMacro, metaclass=abc.ABCMeta):
    @abc.abstractmethod
    def __init__(self):
        self.name = None
        self.arg_names = []
        self.arg_types = []

    @abc.abstractmethod
    def compile(self, namespace: "Name", args: List[Token], block: Block):
        pass

class MacroFunction(AbstractMacro):
    def compile(self, namespace: "NameSpace", args: List[Token]):
        raise NotImplementedError("This macro function, can't compile themself")

    def __init__(self, name: str,
                 arg_types: List[ArgumentType],
                 arg_names: List[str],
                 source: Block):
        self.name = name
        self.arg_types = arg_types
        self.arg_names = arg_names
        self.code = source.children
        self.source = source


import numpy as np
from utils import no_quotation
from sklearn.svm import SVC,SVR
from scipy.stats import ttest_rel,ttest_ind
from math import sqrt
from numpy.random import seed
from numpy.random import randn
from numpy import mean
from scipy.stats import sem
from scipy.stats import t
from scipy.stats import spearmanr,pearsonr,wilcoxon,mannwhitneyu
from scipy.stats import kstest,ks_2samp,linregress
import pandas as pd
df = pd.read_csv("../../new_tcga_data.csv")
df["log(TMB)"]=(df["TMB"].astype("int")/38).apply(np.log)
tmb = np.array(df["log(TMB)"])
tmb.reshape((tmb.shape[0],1))
apm = np.array(df["new.APM"])
apm.reshape((apm.shape[0],1))
tigs = np.array(df["new TIGs"])
tigs.reshape((tigs.shape[0],1))
genes = np.array(df.iloc[:,-25:-7])
print(tmb.shape,apm.shape,tigs.shape,genes.shape)
gene_apm_tigs = np.concatenate((genes,apm.reshape([apm.shape[0],1]),tigs.reshape([tigs.shape[0],1])),axis=1)
median_tmb = np.median(tmb)
gene_apm_tigs_high = gene_apm_tigs[np.where(tmb > median_tmb)[0], :]
gene_apm_tigs_low = gene_apm_tigs[np.where(tmb <= median_tmb)[0], :]

#ks 检验
ks = []
for i in range(gene_apm_tigs.shape[1]):
    ks_ret = ks_2samp(gene_apm_tigs_high[:,i],gene_apm_tigs_low[:,i])
    if ks_ret[1]<0.05:
        ks.append(ks_ret[0])
    else:
        ks.append(0)
ks = np.array(ks)
ks = ks.reshape(1,ks.shape[0])
#
#spearman检验
spr = []
for i in range(gene_apm_tigs.shape[1]):
    sp_ret = spearmanr(gene_apm_tigs_high[:min(gene_apm_tigs_high.shape[0],gene_apm_tigs_low.shape[0]),i],gene_apm_tigs_low[:min(gene_apm_tigs_high.shape[0],gene_apm_tigs_low.shape[0]),i])
    if sp_ret[1] < 0.05:
        spr.append(sp_ret[0])
    else:
        spr.append(0)
spr = np.array(spr)
spr = spr.reshape(1,spr.shape[0])
#
# #wilcoxon检验
# for i in range(gene_apm_tigs.shape[1]):
#     wc_ret = wilcoxon(np.sort(gene_apm_tigs_high[:min(gene_apm_tigs_high.shape[0],gene_apm_tigs_low.shape[0]),i]),np.sort(gene_apm_tigs_low[:min(gene_apm_tigs_high.shape[0],gene_apm_tigs_low.shape[0]),i]))
#     print(wc_ret)
#
# #mannwhitneyu检验
mwu = []
for i in range(gene_apm_tigs.shape[1]):
    mwu_ret = mannwhitneyu(gene_apm_tigs_high[:min(gene_apm_tigs_high.shape[0],gene_apm_tigs_low.shape[0],200),i],gene_apm_tigs_low[:min(gene_apm_tigs_high.shape[0],gene_apm_tigs_low.shape[0],200),i])
    if mwu_ret[1] < 0.05:
        mwu.append(mwu_ret[0])
        # mwu.append(-np.log10(mwu_ret[1]))
    else:
        mwu.append(0)
mwu = np.array(mwu)/np.max(mwu)
mwu = mwu.reshape(1,mwu.shape[0])
#Stats.linregress
lin = []
for i in range(gene_apm_tigs.shape[1]):
    # print(list(gene_apm_tigs[:,i].flatten()),list(tmb))
    lin_ret = linregress(list(gene_apm_tigs[:,i].flatten()),list(tmb.flatten()))
    if lin_ret[3] < 0.05:
        lin.append(lin_ret[2])
    else:
        lin.append(0)
lin = np.array(lin)
lin = lin.reshape(1,lin.shape[0])

deg = [-0.13952054, -0.20235395, -0.22468231,  0.20079217,  0.43271332,  0.10410169,  0.50875813,
0.26399990 ,-0.13704363, -0.18884181, -0.49473828, -0.16845841, -0.26478192,  0.06212003,
-0.24419899,  0.06607185,  0.04248935, -0.02402072,0,0]
deg = np.array(np.abs(deg))
deg = deg.reshape(1,deg.shape[0])
heat = np.concatenate((spr,ks,mwu,lin,deg),axis =0)
np.save("heat.npy",heat)
# model_svr = SVR(kernel='rbf')
# model_svr.fit(genes,apm)
# print(model_svr.score(genes,tmb))

# t-test p-value, person, spearman, regression t-test
# tt = np.concatenate((genes,tmb.reshape(tmb.shape[0],1)),axis=1)
# med = np.median(tt[:,-1])
# for i in range(tt.shape[0]):
#     if tt[i,-1]>med:
#         tt[i,-1]=1
#     else:
#         tt[i,-1]=0
# # print(ttest_ind([-0.78571512, -0.69826573,-0.66055641, 0.31116072, -0.94367846,
# #   -0.96233765],[-0.29325476, -0.05130731 ,-0.88141009,  -0.32023061, -0.96246204
# #   ,-1.0731199 ]))
# # exit()
# for i in range(tt.shape[1]-1):
#     a = tt[np.where(tt[:,-1]==1),i]
#     b = tt[np.where(tt[:, -1] == 0), i]
#     print(ttest_ind(a[0, :2000], b[0,:2000]))
#     # if a.shape[1]>b.shape[1]:
#     #     print(np.isnan(a), np.isnan(b))
#     #     print(ttest_ind(a[0:,b.shape[1]],b[0]))
#     # else:
#     #     print(np.isnan(a),np.isnan(b))
#     #     print(ttest_ind(a[0], b[0:,:a.shape[0]]))

# def KSTest(mat):
#     pass
# psr = np.concatenate((genes,tmb.reshape(tmb.shape[0],1)),axis=1)
# psr_v = []
# psr_p = []
# spr_v = []
# spr_p = []
# for i in range(psr.shape[1]-1):
#     psr_p.append(pearsonr(psr[:,i],psr[:,-1])[1])
#     spr_p.append(spearmanr(psr[:,i],psr[:,-1])[1])
#     psr_v.append(pearsonr(psr[:, i], psr[:, -1])[0])
#     spr_v.append(spearmanr(psr[:, i], psr[:, -1])[0])
# print(spr_p)


# 18. Subsets II
# 中文English
# Given a collection of integers that might contain duplicates, nums, return all possible subsets (the power set).
#
# Example
# Example 1:
#
# Input: [0]
# Output:
# [
#   [],
#   [0]
# ]
# Example 2:
#
# Input: [1,2,2]
# Output:
# [
#   [2],
#   [1],
#   [1,2,2],
#   [2,2],
#   [1,2],
#   []
# ]
# Challenge
# Can you do it in both recursively and iteratively?
#
# Notice
# Each element in a subset must be in non-descending order.
# The ordering between two subsets is free.
# The solution set must not contain duplicate subsets.

class Solution:
    """
    @param nums: A set of numbers.
    @return: A list of lists. All valid subsets.
    """

    def subsetsWithDup(self, nums):
        # write your code here
        nums.sort()
        rslt = []
        self.dfs_helper(nums, 0, [], rslt)
        return rslt

    def dfs_helper(self, nums, start_idx, curr_path, rslt):

        rslt.append(curr_path[:])
        for idx in range(start_idx, len(nums)):
            # remove duplicate
            # 去重逻辑，在某次dfs中，对于该次dfs可能访问到的node,
            # 如果该node不是第一个，且与之前的node.val相等，则continue.
            # for example. if we have [1,1,1,2,2,2,3,3] we label them as
            # [1_a, 1_b, 1_c, 2_a, 2_b, 2_c, 3_a, 3_b] to prevent duplicate result
            # we can only allow each time to select 1_a< 1_b< 1_c if we try to select 1_b before 1_a,
            # we should not allow that.
            if idx > 0 and idx != start_idx and nums[idx] == nums[idx - 1]:
                continue
            curr_path.append(nums[idx])
            self.dfs_helper(nums, idx + 1, curr_path, rslt)
            curr_path.pop()

sol = Solution()
sol.subsetsWithDup(nums=[1, 1, 1, 2, 2])
sol.subsetsWithDup(nums=[0])
sol.subsetsWithDup(nums=[4, 1, 0]) == [[], [0], [0, 1], [0, 1, 4], [0, 4], [1], [1, 4], [4]]


import os
from django.contrib.gis.utils import LayerMapping
from traffic.models import Philly_link

link_mapping = {
    'no' : 'NO',
    'fromnodeno' : 'FROMNODENO',
    'tonodeno' : 'TONODENO',
    'typeno' : 'TYPENO',
    'tsysset' : 'TSYSSET',
    'length' : 'LENGTH',
    'numlanes' : 'NUMLANES',
    'capprt' : 'CAPPRT',
    'v0prt' : 'V0PRT',
    'r_v0prt': 'R_V0PRT',
    'geom' : 'LINESTRING',
}

link_consolidate_mapping = {
    'no' : 'ID',
    'v0prt' : 'V0PRT',
    'length' : 'LENGTH',
    'geom' : 'LINESTRING',
}

link_with_attr_mapping = {
    'no' : 'ID',
    'v0prt' : 'V0PRT',
    'tsysset' : 'TSYSSET',
    'numlanes' : 'NUMLANES',
    'county_10' : 'COUNTY__10',
    'hwy_num' : 'HWY_NUM',
    'cap_1h' : 'CAP_1H',
    'length' : 'LENGTH',
    'geom' : 'LINESTRING',
}

philly_link_mapping = {
    'no' : 'ID',
    'v0prt' : 'V0PRT',
    'tsysset' : 'TSYSSET',
    'numlanes' : 'NUMLANES',
    'county_10' : 'COUNTY__10',
    'hwy_num' : 'HWY_NUM',
    'cap_1h' : 'CAP_1H',
    'direction' : 'DIRECTION',
    'length' : 'LENGTH',
    'geom' : 'LINESTRING',
}

link_shp = os.path.abspath("/home/hzn/projects/data_max/philly_link/with_direction/link_with_direction_4326.shp")

def run(verbose=True):
    lm = LayerMapping(Philly_link, link_shp, philly_link_mapping, transform=True)
    lm.save(strict=True, verbose=verbose)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 10 17:23:51 2021

@author: konam
"""

# Importing the necessary libraries for the project
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
import matplotlib.pyplot as plt
import streamlit as st

st.title("Predicting a Student Score")
url = "http://bit.ly/w-data"
# Function for getting the data
def get_data(url):
    data = pd.read_csv(url)
    return (data)

# Function for describing the data
def about_data(df):
    col1,col2,col3 = st.beta_columns(3)
    with col1:
        st.write("About the task")
        st.write("Predict the percentage of a student based on the no. of study hours.")
    with col2:
        st.write("Given Data")
        st.write(df)
    with col3:
        st.write("Data Statistics")
        st.write(df.describe())

# Function for data splitting
def data_split(df):
    x = np.array(df.Hours)
    y = np.array(df.Scores)
    x_train, x_test, y_train, y_test = train_test_split(x,y,test_size = 0.25, random_state=0)
    x_train = x_train.reshape(-1,1)
    x_test = x_test.reshape(-1,1)
    return (x,y,x_train,x_test,y_train,y_test)

# Function for building model
def model(x_train,y_train):
    regr = LinearRegression()
    regr.fit(x_train, y_train)
    return(regr)

#Reading the Functions
s_data = get_data(url)
about_data(s_data)
x,y,x_train,x_test,y_train,y_test = data_split(s_data)
regr = model(x_train,y_train)

#Comparing the Results and plotting the chart
st.write(f"Regression line: Scores = {regr.coef_[0].round(2)} * Hours + {regr.intercept_.round(2)}")
y_pred = regr.predict(x_test)

col1,col2 = st.beta_columns(2)
with col1:
    comp = pd.DataFrame({"Actual":y_test,"Prediction":y_pred.round(0)})
    st.write("Comparing the Actual Vs Predicted values",comp)
    st.write(f"R2 score of the regression model is: {metrics.r2_score(y_pred, y_test).round(2)}")
    st.write(f"Mean_absolute_error of the regression model is: {metrics.mean_absolute_error(y_pred,y_test).round(2)}")
with col2:
    st.write("Plotting the data points and the regression line")
    line = regr.coef_ * x + regr.intercept_
    fig,ax = plt.subplots()
    ax.scatter(x,y,color = "blue")
    ax.plot(x, line, color = 'green')
    ax.set_xlabel("Hours")
    ax.set_ylabel("Scores")
    st.pyplot(fig)

#Checking the target with user input and printing the score
st.sidebar.write("Enter study hours to predict the scores")
option = st.sidebar.text_input('Study Hours',value = 9.5)
max_study_hours = (100-regr.intercept_)/(regr.coef_)

def pred(hours):
    predicted_score = regr.predict(hours)
    st.sidebar.write(f"Predicted Score: {predicted_score[0].round(2)} for the study hours:{p_input}")

p_input = eval(option)
if(p_input >= max_study_hours):
    st.sidebar.write("Maximum Study hours reached")
elif(p_input<0):
    st.sidebar.write("Please enter a positive value")
else:
    hours = [[p_input]]
    pred(hours)

st.sidebar.write(f"Note1: The predicted score attains 100 when the study hours is {max_study_hours[0].round(2)}, so request to enter the hours less than the max hours.")
st.sidebar.write("Note2: The model predicts minimum score even when the hours is 0 ,since the model was build with less data and no scores with 0 hours on training data.")
#print(metrics.mean_absolute_error(y_test,y_pred))


import random

HANGMAN_ASCII_ART = """
_           _    _                              _            _   _               _
\ \   __   / /__| | ___ ___  _ __ ___   ___    | |_ ___     | |_| |__   ___     | |__   __ _ _ __   __ _ _ __ ___   __ _ _ __        __ _  __ _ _ __ ___   ___
 \ \ /  \ / / _ \ |/ __/ _ \| '_ ` _ \ / _ \   | __/ _ \    | __| '_ \ / _ \    | '_ \ / _' | '_ \ / _' | '_ ' _ \ / _' | '_ \      / _` |/ _` | '_ ` _ \ / _ \
  \ V /\ V /  __/ | (_| (_) | | | | | |  __/   | || (_) |   | |_| | | |  __/    | | | | (_| | | | | (_| | | | | | | (_| | | | |    | (_| | (_| | | | | | |  __/
   \_/  \_/ \___|_|\___\___/|_| |_| |_|\___|    \__\___/     \__|_| |_|\___|    |_| |_|\__,_|_| |_|\__, |_| |_| |_|\__,_|_| |_|     \__, |\__,_|_| |_| |_|\___|
                                                                                |___/                           |___/"""
win = """
                            _       ___
                 _        _(_)_ __  \ /
                 \ \ /\ / /| | '_ \ | |
                  \ V  V / | | | | ||_|
                   \_/\_/  |_|_| |_|(_)"""
lose = """
                 _
                | | ___  ___  ___
                | |/ _ \/ __|/ _ \
                | |_(_) \__ \  __/ _ _ _
                |____|_/|___/\___|(_)_)_)"""

MAX_TRIES = 7
print(HANGMAN_ASCII_ART, "\n", MAX_TRIES)

old_letter_guessed = []

def check_valid_input(letter_guessed, old_letters_guessed):
    letter_guessed = letter_guessed.lower()
    global old_letter_guessed
    if len(letter_guessed) > 1:
        print("X")
        print(" -> ".join(sorted(old_letter_guessed)))
        return False

    elif not(letter_guessed in "abcdefghijklmnopqrstuvwxyz"):
        print("X")
        print(" -> ".join(sorted(old_letter_guessed)))
        return False

    elif letter_guessed in old_letter_guessed:
        print("X")
        print(" -> ".join(sorted(old_letter_guessed)))
        return False

    else:
        old_letter_guessed += letter_guessed
        return True

def show_hidden_word(secret_word, old_letter_guessed):
    for char in secret_word:
        if char in old_letter_guessed:
            print(char, end =" ")
        else:
            print("_ ", end ="")

def check_win(secret_word, old_letter_guessed):
    for char in secret_word:
        if char not in old_letter_guessed:
            return False
    return True

HANGMAN_PHOTOS = {0: """   x-------x""", 1: """   x-------x
    |
    |
    |
    |
    |""", 2: """   x-------x
    |       |
    |       0
    |
    |
    |""", 3: """   x-------x
    |       |
    |       0
    |       |
    |
    |""", 4: """   x-------x
    |       |
    |       0
    |      /|
    |
    |""", 5: """   x-------x
    |       |
    |       0
    |      /|\
    |
    |""", 6: """   x-------x
    |       |
    |       0
    |      /|\
    |      /
    |""", 7: """   x-------x
    |       |
    |       0
    |      /|\
    |      / \
    |"""}

def print_hangman(num_of_tries):
    return HANGMAN_PHOTOS[num_of_tries]

num_of_tries = 0

file_path = "Anti Dark Lord Harry program point did begin to cover it Harry wondered what the Sorting Hat would say now if he could put it on his head\
No wonder Professor Quirrell had accused Harry of heading down the path of a Dark Lord Harry had been too slow on the uptake he should have seen the parallel right away\
Understand that the Dark Lord did not win that day His goal was to learn martial arts and yet he left without a single lesson\
Harry had entered the Potions class with the intent to learn Potions He left without a single lesson"
file_path = file_path.lower()

def choose_word(file_path, index):
    L = []
    L_file_path = file_path.split(" ")

    for i in L_file_path:
        if i not in L:
            L.append(i)
    count = len(L)
    return (count , L_file_path[(index-1) % len(L_file_path)])

index = random.randint(1, 100)
num_of_tries = 0
count, secret_word = choose_word(file_path, index)

print("\n")
print(print_hangman(num_of_tries))
num_of_tries += 1

print("\n")
show_hidden_word(secret_word, old_letter_guessed)

print("\n")
for i in range(1000):
    letter_guessed = input("Enter a char: ")
    booli = check_valid_input(letter_guessed, old_letter_guessed)
    if booli == True:
        if letter_guessed in secret_word:
            show_hidden_word(secret_word, old_letter_guessed)
            print("\n")
            if check_win(secret_word, old_letter_guessed) == True:
                print(win)
                break
        else:
            print(")-:")
            print(print_hangman(num_of_tries))
            print("You have just", 7-num_of_tries, "tries" )
            num_of_tries += 1
            show_hidden_word(secret_word, old_letter_guessed)
            print("\n")
            if num_of_tries == 8:
                print(lose)
                break


from typing import List
def primes_nums(numbers: List[int]) -> List[int]:
    # write your code here ^_^
    result = numbers.copy()
    for num in numbers:
        if num > 2:
            flag = True
            i = int((num ** .5) + 1)
            while flag and i > 1:
                if (num % i) == 0:
                    result.remove(num)
                    flag = False
                else:
                    i -= 1
        else:
            continue
    return result


# -*- coding:utf-8 -*-
import os

import xlrd

from my_wxpython.base import *

__author__ = 'you'
__create_time__: '2017/8/25 22:18'

def open_work_dir(evt, view):
    config_dir = r"G:\个人账户\账户单"
    config_config = view.year.GetValue()
    config_file = os.path.join(config_dir, config_config)

    os.system("start {}".format(config_file))

def query(evt, view):
    view.result.Clear()
    config_dir = r"G:\个人账户\账户单"
    config_config = view.year.GetValue()
    config_file = os.path.join(config_dir, config_config)
    month_name = view.month.GetValue()

    try:
        data = xlrd.open_workbook(config_file).sheet_by_name(month_name)
        value_1 = data.col_values(0)  # 时间
        value_2 = data.col_values(4)  # 支出原因
        value_3 = data.col_values(3)  # 支出
        value_4 = data.col_values(1)  # 收入
        pay_reason = list()  # 支出原因
        pay_money = list()  # 当天付款金额
        all_data = dict()  # 保存所有数据

        for index, reason in enumerate(value_2):
            if value_1[index] in ["个人账户支出收入情况", "时间"]:
                continue
            if value_1[index]:
                pay_reason = list()
                pay_money = list()
                all_data[value_1[index]] = (pay_reason, pay_money)
                if reason:
                    pay_reason.append(reason)
                if value_3[index]:
                    pay_money.append(value_3[index])
            else:
                if reason:
                    pay_reason.append(reason)
                if value_3[index]:
                    pay_money.append(value_3[index])

        for date, reason in all_data.items():
            print("{}".format(date))
            print("{} : {}".format("支出原因", '，'.join(reason[0])))
            print("{0} : {1:.2f}".format("当天支付金额总数", money_total(reason[1])))
            print()

        print("{0} : {1:.2f}".format("本月份总收入", value_4[2]))
        print("{0} : {1:.2f}".format("本月份总支出", money_all_total(value_3)))
        print("{0} : {1:.2f}".format("本月份剩余余额", value_4[2] - money_all_total(value_3)))

    except xlrd.biffh.XLRDError:
        print("你输入的月份不存在")

def money_total(num_list):
    money_sum = 0
    for i in num_list:
        money_sum += i
    return money_sum

def money_all_total(value):
    money_sum = 0
    for i in value:
        if i in ["支出"]:
            continue
        if i:
            money_sum += i
    return money_sum


#!/usr/bin/env python3
""" Contains definition of index_range helper function """

import csv
import math
from typing import Tuple, List

def index_range(page: int, page_size: int) -> Tuple[int, int]:
    """
    Calculate the start and end indices for pagination.

    Args:
        page (int): Page number (1-indexed).
        page_size (int): Number of items per page.

    Returns:
        tuple: A tuple containing start and end indices for the requested page.
    """
    start = 0
    end = 0

    for i in range(page):
        start = end
        end += page_size

    return (start, end)

class Server:
    """Server class to paginate a database of popular baby names.
    """
    DATA_FILE = "Popular_Baby_Names.csv"

    def __init__(self):
        self.__dataset = None

    def dataset(self) -> List[List]:
        """Cached dataset
        """
        if self.__dataset is None:
            with open(self.DATA_FILE) as f:
                reader = csv.reader(f)
                dataset = [row for row in reader]
            self.__dataset = dataset[1:]

        return self.__dataset

    def get_page(self, page: int = 1, page_size: int = 10) -> List[List]:
        """
        Get a specific page from the dataset.

        Args:
            page (int, optional): Page number (1-indexed). Defaults to 1.
            page_size (int, optional): No. of items per page. Defaults to 10.

        Returns:
            List[List]: The requested page of the dataset.
        """
        assert isinstance(page, int) and page > 0
        assert isinstance(page_size, int) and page_size > 0

        dataset = self.dataset()
        start, end = index_range(page, page_size)
        if start >= len(dataset):
            return []

        return dataset[start:end]


def count2(N):
  count = 0
  while N > 0:
    count += N // 2
    N //= 2
  return count

def count5(N):
  count = 0
  while N > 0:
    count += N // 5
    N //= 5
  return count

n, m = map(int, input().split())
two_count = count2(n) - count2(m) - count2(n-m)
five_count = count5(n) - count5(m) - count5(n-m)
print(min(two_count, five_count))


import config
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-c", "--config", dest="filename",
    help="config file", default="../config/sputnik.ini")
(options, args) = parser.parse_args()
if options.filename:
    config.reconfigure(options.filename)

__author__ = 'satosushi'

"""
A small fowarding service, every engine publishes their safe price to the frontend
and every safe price consumer subscribes to this device.
"""

from zmq_util import bind_subscriber, bind_publisher
from twisted.internet import reactor
from twisted.python import log
import json

if __name__ == "__main__":
    import sys
    log.startLogging(sys.stdout)
    subscriber = bind_subscriber(config.get("safe_price_forwarder", "zmq_frontend_address"))
    publisher = bind_publisher(config.get("safe_price_forwarder", "zmq_backend_address"))

    subscriber.subscribe("")

    safe_prices = {}
    def onPrice(*args):
        update = json.loads(args[0])
        log.msg("received update: %s" % update)
        safe_prices.update(update)
        publisher.publish(json.dumps(safe_prices), tag=b'')

    subscriber.gotMessage = onPrice
    reactor.run()


# -*- coding: utf-8 -*-
import datetime
import time
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from backend.model import Project
from django.db import connection

def timeline(params):
    data = {}
    try:
        # items = Project.objects.filter(task_id=params['task_id'], create_time__range=(
        #     params['timestamps'][0], params['timestamps'][params['number'] - 1]
        # )).values('id', 'create_time')
        items = Project.objects.filter(task_id=params['task_id'], create_time__gte=params['timestamps'][0]).values('id','create_time')

        # sql = 'select id, create_time from project where task_id=' + str(params['task_id']) + ' and create_time>="' + \
        #       params['timestamps'][0] + '" and create_time<="' + params['timestamps'][params['number'] - 1] + '"'
        # items = Project.objects.raw(sql)
        # print(params['timestamps'])

        # print(params)
        # 统计每个时间点抓取条数
        for item in items:
            create_time = item['create_time'].strftime("%H:%M:%S")
            item['create_time'] = item['create_time'].strftime("%Y-%m-%d %H:%M:%S")
            # print(item)

            if item['create_time'] in params['timestamps']:
                if data.get(create_time, None) is None:
                    data[create_time] = 1
                else:
                    data[create_time] += 1

    except Exception as e:
        print(e)
    return data

# 数据查询
def query(params):
    page = params['page']
    size = params['size']
    task_id = params['task_id']  # 哪个爬虫应用的
    # start = params['start']  # 起始时间
    # end = params['end'] if params['end'] else datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    result = {
        'data': [],
        'status': 'success',
        'totals': ''
    }

    try:
        lists = Project.objects.order_by('-create_time').filter(task_id=task_id).values(
            'id', 'name', 'company', 'date', 'price', 'architecter', 'url', 'create_time'
        )

        # 分页
        paginator = Paginator(lists, size)
        items = paginator.page(page)
        result['totals'] = paginator.count

        for item in items:
            result['data'].append({
                'id': str(item['id']),
                'name': item['name'],
                'company': item['company'],
                'project_date': item['date'].strftime("%Y-%m-%d"),
                'price': item['price'],
                'architecter': item['architecter'],
                'url': item['url'],
                'create_time': item['create_time'].strftime("%Y-%m-%d %H:%M:%S")
            })
    except Exception as e:
        print(e)

    return result

# 检测任务是否爬到数据并返回条数
def checkTaskData(task_id):
    number = 0
    try:
        number = Project.objects.filter(task_id=task_id).count()
    except Exception as e:
        print(e)

    return number


import os
import bpy
import blenderbim.tool as tool
import ifcopenshell.util.date
import ifcopenshell.util.cost
import ifcopenshell.util.unit
import blenderbim.bim.helper
import json

class Cost(blenderbim.core.tool.Cost):
    @classmethod
    def get_cost_schedule_attributes(cls):
        props = bpy.context.scene.BIMCostProperties
        return blenderbim.bim.helper.export_attributes(props.cost_schedule_attributes)

    @classmethod
    def disable_editing_cost_schedule(cls):
        bpy.context.scene.BIMCostProperties.active_cost_schedule_id = 0
        cls.disable_editing_cost_item()

    @classmethod
    def enable_editing_cost_schedule_attributes(cls, cost_schedule):
        bpy.context.scene.BIMCostProperties.active_cost_schedule_id = cost_schedule.id()
        bpy.context.scene.BIMCostProperties.is_editing = "COST_SCHEDULE_ATTRIBUTES"

    @classmethod
    def load_cost_schedule_attributes(cls, cost_schedule):
        def special_import(name, prop, data):
            if name in ["SubmittedOn", "UpdateDate"]:
                prop.string_value = "" if prop.is_null else ifcopenshell.util.date.ifc2datetime(data[name]).isoformat()
                return True

        props = bpy.context.scene.BIMCostProperties
        props.cost_schedule_attributes.clear()
        blenderbim.bim.helper.import_attributes2(cost_schedule, props.cost_schedule_attributes, callback=special_import)

    @classmethod
    def enable_editing_cost_items(cls, cost_schedule):
        props = bpy.context.scene.BIMCostProperties
        props.active_cost_schedule_id = cost_schedule.id()
        props.is_editing = "COST_ITEMS"

    @classmethod
    def play_sound(cls):
        if bpy.context.preferences.addons["blenderbim"].preferences.should_play_chaching_sound:
            cls.play_chaching_sound()  # lol

    @classmethod
    def play_chaching_sound(cls):
        # TODO: make pitch higher as costs rise
        try:
            import aud

            device = aud.Device()
            # chaching.mp3 is by Lucish_ CC-BY-3.0 https://freesound.org/people/Lucish_/sounds/554841/
            sound = aud.Sound(os.path.join(bpy.context.scene.BIMProperties.data_dir, "chaching.mp3"))
            handle = device.play(sound)
            sound_buffered = aud.Sound.buffer(sound)
            handle_buffered = device.play(sound_buffered)
            handle.stop()
            handle_buffered.stop()
        except:
            pass  # ah well

    @classmethod
    def load_cost_schedule_tree(cls):
        props = bpy.context.scene.BIMCostProperties
        props.is_cost_update_enabled = False
        cost_schedule = tool.Ifc.get().by_id(props.active_cost_schedule_id)
        props.cost_items.clear()
        cls.contracted_cost_items = json.loads(props.contracted_cost_items)
        [
            cls.create_new_cost_item_li(props.cost_items, cost_item, 0, type="cost")
            for rel in cost_schedule.Controls or []
            for cost_item in rel.RelatedObjects or []
        ]
        props.is_cost_update_enabled = True

    @classmethod
    def expand_cost_item(cls, cost_item):
        props = bpy.context.scene.BIMCostProperties
        if not hasattr(cls, "contracted_cost_items"):
            cls.contracted_cost_items = json.loads(props.contracted_cost_items)
        if cost_item.id() in cls.contracted_cost_items:
            cls.contracted_cost_items.remove(cost_item.id())
            props.contracted_cost_items = json.dumps(cls.contracted_cost_items)

    @classmethod
    def expand_cost_items(cls):
        props = bpy.context.scene.BIMCostProperties
        cls.contracted_cost_items = json.loads(props.contracted_cost_items)
        for cost_item in props.cost_items:
            if cost_item.ifc_definition_id in cls.contracted_cost_items:
                cls.contracted_cost_items.remove(cost_item.ifc_definition_id)
        props.contracted_cost_items = json.dumps(cls.contracted_cost_items)

    @classmethod
    def contract_cost_item(cls, cost_item):
        props = bpy.context.scene.BIMCostProperties
        if not hasattr(cls, "contracted_cost_items"):
            cls.contracted_cost_items = json.loads(props.contracted_cost_items)
        cls.contracted_cost_items.append(cost_item.id())
        props.contracted_cost_items = json.dumps(cls.contracted_cost_items)

    @classmethod
    def contract_cost_items(cls):
        props = bpy.context.scene.BIMCostProperties
        if not hasattr(cls, "contracted_cost_items"):
            cls.contracted_cost_items = json.loads(props.contracted_cost_items)
        for cost_item in props.cost_items:
            if cost_item.ifc_definition_id not in cls.contracted_cost_items:
                cls.contracted_cost_items.append(cost_item.ifc_definition_id)
        props.contracted_cost_items = json.dumps(cls.contracted_cost_items)

    @classmethod
    def clean_up_cost_item_tree(cls, cost_item_id):
        props = bpy.context.scene.BIMCostProperties
        if not hasattr(cls, "contracted_cost_items"):
            cls.contracted_cost_items = json.loads(props.contracted_cost_items)
        if props.active_cost_item_id == cost_item_id:
            props.active_cost_item_id = 0
        if props.active_cost_item_index in cls.contracted_cost_items:
            cls.contracted_cost_items.remove(props.active_cost_item_index)
        props.contracted_cost_items = json.dumps(cls.contracted_cost_items)
        cls.enable_editing_cost_items(cost_schedule=tool.Ifc.get().by_id(props.active_cost_schedule_id))

    @classmethod
    def enable_editing_cost_item_attributes(cls, cost_item):
        bpy.context.scene.BIMCostProperties.active_cost_item_id = cost_item.id()
        bpy.context.scene.BIMCostProperties.cost_item_editing_type = "ATTRIBUTES"

    @classmethod
    def load_cost_item_attributes(cls, cost_item):
        props = bpy.context.scene.BIMCostProperties
        props.cost_item_attributes.clear()
        blenderbim.bim.helper.import_attributes2(cost_item, props.cost_item_attributes)

    @classmethod
    def disable_editing_cost_item(cls):
        bpy.context.scene.BIMCostProperties.active_cost_item_id = 0
        bpy.context.scene.BIMCostProperties.change_cost_item_parent = False

    @classmethod
    def get_cost_item_attributes(cls):
        props = bpy.context.scene.BIMCostProperties
        return blenderbim.bim.helper.export_attributes(props.cost_item_attributes)

    @classmethod
    def get_active_cost_item(cls):
        props = bpy.context.scene.BIMCostProperties
        if not props.active_cost_item_id:
            return None
        return tool.Ifc.get().by_id(bpy.context.scene.BIMCostProperties.active_cost_item_id)

    @classmethod
    def get_highlighted_cost_item(cls):
        props = bpy.context.scene.BIMCostProperties
        if not props.active_cost_schedule_id:
            return
        if props.active_cost_item_index < len(props.cost_items):
            return tool.Ifc.get().by_id(props.cost_items[props.active_cost_item_index].ifc_definition_id)
        return

    @classmethod
    def load_cost_item_types(cls, cost_item=None):
        if not cost_item:
            cost_item = cls.get_highlighted_cost_item()
            if not cost_item:
                return
        props = bpy.context.scene.BIMCostProperties
        props.cost_item_type_products.clear()
        # TODO implement process and resource types
        # props.cost_item_processes.clear()
        # props.cost_item_resources.clear()
        for rel in cost_item.Controls or []:
            for related_object in rel.RelatedObjects:
                if related_object.is_a("IfcTypeProduct"):
                    new = props.cost_item_type_products.add()
                # TODO implement process and resource types
                # elif related_object.is_a("IfcProcess"):
                #    new = props.cost_item_processes.add()
                # elif related_object.is_a("IfcResource"):
                #    new = props.cost_item_resources.add()
                new.ifc_definition_id = related_object.id()
                new.name = related_object.Name or "Unnamed"

    @classmethod
    def load_cost_item_quantity_assignments(cls, cost_item, related_object_type):
        def create_list_items(collection, cost_item, is_deep):
            products = cls.get_cost_item_assignments(cost_item, filter_by_type=related_object_type, is_deep=False)
            for product in products:
                new = collection.add()
                new.ifc_definition_id = product.id()
                new.name = product.Name or "Unnamed"
                total_quantity, unit = cls.calculate_parametric_quantity(cost_item, product)
                new.total_quantity = total_quantity or 0
                new.unit_symbol = unit or ""
            if is_deep:
                for cost_item in ifcopenshell.util.cost.get_nested_cost_items(cost_item, is_deep):
                    create_list_items(collection, cost_item, is_deep=False)

        props = bpy.context.scene.BIMCostProperties
        if related_object_type == "PRODUCT":
            props.cost_item_products.clear()
            is_deep = bpy.context.scene.BIMCostProperties.show_nested_elements
            create_list_items(props.cost_item_products, cost_item, is_deep)
        elif related_object_type == "PROCESS":
            props.cost_item_processes.clear()
            is_deep = bpy.context.scene.BIMCostProperties.show_nested_tasks
            create_list_items(props.cost_item_processes, cost_item, is_deep)
        elif related_object_type == "RESOURCE":
            props.cost_item_resources.clear()
            is_deep = bpy.context.scene.BIMCostProperties.show_nested_resources
            create_list_items(props.cost_item_resources, cost_item, is_deep)

    @classmethod
    def calculate_parametric_quantity(cls, cost_item=None, product=None):
        quantities, unit = cls.get_assigned_quantities(cost_item, product)
        return sum(quantity[3] for quantity in quantities), unit

    @classmethod
    def get_assigned_quantities(cls, cost_item, product):
        selected_quantitites = []
        unit = ""
        for quantities in ifcopenshell.util.element.get_psets(product, qtos_only=True).values():
            for qto in tool.Ifc.get().by_id(quantities["id"]).Quantities or []:
                for quantity in cost_item.CostQuantities:
                    if quantity == qto:
                        selected_quantitites.append(quantity)
                        if not unit:
                            unit = cls.get_quantity_unit_symbol(quantity)
        return selected_quantitites, unit

    @classmethod
    def get_products(cls, related_object_type=None):
        if related_object_type == "PRODUCT":
            products = tool.Spatial.get_selected_products()
        elif related_object_type == "PROCESS":
            products = [tool.Sequence.get_highlighted_task()]
        elif related_object_type == "RESOURCE":
            products = [tool.Resource.get_highlighted_resource()]
        return products or []

    @classmethod
    def enable_editing_cost_item_quantities(cls, cost_item=None):
        props = bpy.context.scene.BIMCostProperties
        props.active_cost_item_id = cost_item.id()
        props.cost_item_editing_type = "QUANTITIES"

    @classmethod
    def enable_editing_cost_item_quantity(cls, physical_quantity=None):
        bpy.context.scene.BIMCostProperties.active_cost_item_quantity_id = physical_quantity.id()

    @classmethod
    def load_cost_item_quantity_attributes(cls, physical_quantity=None):
        props = bpy.context.scene.BIMCostProperties
        props.quantity_attributes.clear()
        blenderbim.bim.helper.import_attributes2(physical_quantity, props.quantity_attributes)

    @classmethod
    def enable_editing_cost_item_values(cls, cost_item=None):
        props = bpy.context.scene.BIMCostProperties
        props.active_cost_item_id = cost_item.id()
        props.cost_item_editing_type = "VALUES"

    @classmethod
    def disable_editing_cost_item_quantity(cls):
        bpy.context.scene.BIMCostProperties.active_cost_item_quantity_id = 0

    @classmethod
    def get_cost_item_quantity_attributes(cls):
        props = bpy.context.scene.BIMCostProperties
        return blenderbim.bim.helper.export_attributes(props.quantity_attributes)

    @classmethod
    def get_attributes_for_cost_value(cls, cost_type, cost_category):
        if cost_type == "FIXED":
            category = None
            attributes = {"AppliedValue": bpy.context.scene.BIMCostProperties.fixed_cost_value}
        elif cost_type == "SUM":
            category = "*"
            attributes = {"Category": category}
        elif cost_type == "CATEGORY":
            category = cost_category
            attributes = {"Category": category}
        return attributes

    @classmethod
    def load_cost_item_value_attributes(cls, cost_value=None):
        def import_attributes(name, prop, data, cost_value, is_rates, props_collection):
            if name == "AppliedValue":
                # TODO: for now, only support simple IfcValues (which are effectively IfcMonetaryMeasure)
                prop = props_collection.add()
                prop.data_type = "float"
                prop.name = "AppliedValue"
                prop.is_optional = True
                prop.float_value = (
                    0.0
                    if prop.is_null
                    else cls.calculate_applied_value(tool.Ifc.get().by_id(props.active_cost_item_id), cost_value)
                )
                return True
            if name == "UnitBasis" and is_rates:
                prop = props_collection.add()
                prop.name = "UnitBasisValue"
                prop.data_type = "float"
                prop.is_null = data["UnitBasis"] is None
                prop.is_optional = True
                if data["UnitBasis"] and data["UnitBasis"].ValueComponent:
                    prop.float_value = data["UnitBasis"].ValueComponent.wrappedValue or 0
                else:
                    prop.float_value = 0

                prop = props_collection.add()
                prop.name = "UnitBasisUnit"
                prop.data_type = "enum"
                prop.is_null = prop.is_optional = False
                units = cls.get_units()
                prop.enum_items = json.dumps(units)
                if data["UnitBasis"] and data["UnitBasis"].UnitComponent:
                    for key, value in json.loads(prop.enum_items).items():
                        if value == cls.format_unit(data["UnitBasis"].UnitComponent):
                            prop.enum_value = key
                            break
                return True

        props = bpy.context.scene.BIMCostProperties
        props.cost_value_attributes.clear()
        is_rates = cls.is_active_schedule_of_rates()
        callback = lambda name, prop, data: import_attributes(
            name, prop, data, cost_value, is_rates, props.cost_value_attributes
        )
        blenderbim.bim.helper.import_attributes2(cost_value, props.cost_value_attributes, callback=callback)

    @classmethod
    def calculate_applied_value(cls, cost_item, cost_value):
        return ifcopenshell.util.cost.calculate_applied_value(cost_item, cost_value)

    @classmethod
    def is_active_schedule_of_rates(cls):
        return (
            tool.Ifc.get().by_id(bpy.context.scene.BIMCostProperties.active_cost_schedule_id).PredefinedType
            == "SCHEDULEOFRATES"
        )

    @classmethod
    def enable_editing_cost_item_value(cls, cost_value=None):
        props = bpy.context.scene.BIMCostProperties
        props.active_cost_value_id = cost_value.id()
        props.cost_value_editing_type = "ATTRIBUTES"

    @classmethod
    def disable_editing_cost_item_value(cls):
        props = bpy.context.scene.BIMCostProperties
        props.active_cost_value_id = 0
        props.cost_value_editing_type = ""

    @classmethod
    def load_cost_item_value_formula_attributes(cls, cost_value=None):
        props = bpy.context.scene.BIMCostProperties
        props.cost_value_attributes.clear()
        bpy.context.scene.BIMCostProperties.cost_value_formula = ifcopenshell.util.cost.serialise_cost_value(cost_value)

    @classmethod
    def enable_editing_cost_item_value_formula(cls, cost_value=None):
        props = bpy.context.scene.BIMCostProperties
        props.active_cost_value_id = cost_value.id()
        props.cost_value_editing_type = "FORMULA"

    @classmethod
    def get_cost_item_value_formula(cls):
        return bpy.context.scene.BIMCostProperties.cost_value_formula

    @classmethod
    def get_cost_value_attributes(cls):
        def export_attributes(attributes, prop):
            if prop.name == "UnitBasisValue":
                if prop.is_null:
                    attributes["UnitBasis"] = None
                    return True
                attributes["UnitBasis"] = {
                    "ValueComponent": prop.float_value or 1,
                    "UnitComponent": cls.get_cost_value_unit_component(),
                }
                return True
            if prop.name == "UnitBasisUnit":
                return True

        props = bpy.context.scene.BIMCostProperties
        callback = lambda attributes, prop: export_attributes(attributes, prop)
        return blenderbim.bim.helper.export_attributes(props.cost_value_attributes, callback)

    @classmethod
    def get_cost_value_unit_component(cls):
        return tool.Ifc.get().by_id(
            int(bpy.context.scene.BIMCostProperties.cost_value_attributes.get("UnitBasisUnit").enum_value)
        )

    @classmethod
    def get_cost_item_assignments(cls, cost_item, filter_by_type=None, is_deep=False):
        return ifcopenshell.util.cost.get_cost_item_assignments(
            cost_item, filter_by_type=filter_by_type, is_deep=is_deep
        )

    @classmethod
    def show_nested_cost_item_elements(cls):
        return bpy.context.scene.BIMCostProperties.show_nested_elements

    @classmethod
    def get_cost_item_products(cls, cost_item, is_deep=False):
        return cls.get_cost_item_assignments(cost_item, filter_by_type="PRODUCT", is_deep=is_deep)

    @classmethod
    def get_cost_item_resources(cls, cost_item, is_deep=False):
        return cls.get_cost_item_assignments(cost_item, filter_by_type="RESOURCE", is_deep=is_deep)

    @classmethod
    def get_cost_item_processes(cls, cost_item, is_deep=False):
        return cls.get_cost_item_assignments(cost_item, filter_by_type="PROCESS", is_deep=is_deep)

    @classmethod
    def get_schedule_cost_items(cls, cost_schedule):
        return ifcopenshell.util.cost.get_schedule_cost_items(cost_schedule)

    @classmethod
    def get_cost_schedule_products(cls, cost_schedule):
        products = []
        for cost_item in ifcopenshell.util.cost.get_schedule_cost_items(cost_schedule):
            products.extend(cls.get_cost_item_products(cost_item))
        return products

    @classmethod
    def import_cost_schedule_csv(cls, file_path=None, is_schedule_of_rates=False):
        if not file_path:
            return
        from ifc5d.csv2ifc import Csv2Ifc
        import time

        start = time.time()
        csv2ifc = Csv2Ifc()
        csv2ifc.csv = file_path
        csv2ifc.file = tool.Ifc.get()
        csv2ifc.is_schedule_of_rates = is_schedule_of_rates
        csv2ifc.execute()
        print("Import finished in {:.2f} seconds".format(time.time() - start))

    @classmethod
    def add_cost_column(cls, name):
        props = bpy.context.scene.BIMCostProperties
        new = props.columns.add()
        new.name = name

    @classmethod
    def remove_cost_column(cls, name):
        props = bpy.context.scene.BIMCostProperties
        props.columns.remove(props.columns.find(name))

    @classmethod
    def expand_cost_item_rate(cls, cost_item):
        props = bpy.context.scene.BIMCostProperties
        contracted_cost_item_rates = json.loads(props.contracted_cost_item_rates)
        contracted_cost_item_rates.remove(cost_item)
        props.contracted_cost_item_rates = json.dumps(contracted_cost_item_rates)
        cls.load_schedule_of_rates_tree(schedule_of_rates=tool.Ifc.get().by_id(int(props.schedule_of_rates)))

    @classmethod
    def contract_cost_item_rate(cls, cost_item):
        props = bpy.context.scene.BIMCostProperties
        contracted_cost_item_rates = json.loads(props.contracted_cost_item_rates)
        contracted_cost_item_rates.append(cost_item)
        props.contracted_cost_item_rates = json.dumps(contracted_cost_item_rates)
        cls.load_schedule_of_rates_tree(schedule_of_rates=tool.Ifc.get().by_id(int(props.schedule_of_rates)))

    @classmethod
    def create_new_cost_item_li(cls, props_collection, cost_item, level_index, type="cost_rate"):
        new = props_collection.add()
        new.ifc_definition_id = cost_item.id()
        new.name = cost_item.Name or "Unnamed"
        new.identification = cost_item.Identification or "XXX"
        new.is_expanded = (
            cost_item.id() not in cls.contracted_cost_item_rates
            if type == "cost_rate"
            else cost_item.id() not in cls.contracted_cost_items
        )
        new.level_index = level_index
        if cost_item.IsNestedBy:
            new.has_children = True
            if new.is_expanded:
                [
                    cls.create_new_cost_item_li(props_collection, sub_cost_item, level_index + 1, type)
                    for rel in cost_item.IsNestedBy
                    for sub_cost_item in rel.RelatedObjects
                ]

    @classmethod
    def load_schedule_of_rates_tree(cls, schedule_of_rates):
        props = bpy.context.scene.BIMCostProperties
        props.is_cost_update_enabled = False
        props.cost_item_rates.clear()
        props.columns.clear()
        cls.contracted_cost_item_rates = json.loads(props.contracted_cost_item_rates)
        for rel in schedule_of_rates.Controls or []:
            [
                cls.create_new_cost_item_li(props.cost_item_rates, cost_item, 0, type="cost_rate")
                for cost_item in rel.RelatedObjects
            ]
        props.is_cost_update_enabled = True

    @classmethod
    def export_cost_schedules(cls, filepath, format=None, cost_schedule=None):
        import subprocess
        import os
        import sys

        if filepath:
            path = filepath
        else:
            path = os.path.join(bpy.context.scene.BIMProperties.data_dir, "build", "cost_schedules")

        if not os.path.exists(path):
            os.makedirs(path)
        if format == "CSV":
            from ifc5d.ifc5Dspreadsheet import Ifc5DCsvWriter

            writer = Ifc5DCsvWriter(file=tool.Ifc.get(), output=path, cost_schedule=cost_schedule)
            writer.write()
        elif format == "ODS":
            from ifc5d.ifc5Dspreadsheet import Ifc5DOdsWriter

            writer = Ifc5DOdsWriter(file=tool.Ifc.get(), output=path, cost_schedule=cost_schedule)
            writer.write()
        elif format == "XLSX":
            from ifc5d.ifc5Dspreadsheet import Ifc5DXlsxWriter

            writer = Ifc5DXlsxWriter(file=tool.Ifc.get(), output=path, cost_schedule=cost_schedule)
            writer.write()
        try:
            if path:
                if sys.platform == "win32":
                    os.startfile(path)
                elif sys.platform == "darwin":
                    subprocess.call(["open", path])
                elif sys.platform == "linux":
                    subprocess.call(["xdg-open", path])
        except:
            return "Could not open file location"

    @classmethod
    def get_units(cls):
        units = {}
        for unit in tool.Ifc.get().by_type("IfcNamedUnit"):
            if unit.get_info().get("UnitType", None) in [
                "AREAUNIT",
                "LENGTHUNIT",
                "TIMEUNIT",
                "VOLUMEUNIT",
                "MASSUNIT",
                "USERDEFINED",
            ]:
                units[unit.id()] = cls.format_unit(unit)
        return units

    @staticmethod
    def format_unit(unit):
        if unit.is_a("IfcContextDependentUnit"):
            return f"{unit.UnitType} / {unit.Name}"
        else:
            name = unit.Name
            if unit.get_info().get("Prefix", None):
                name = f"{unit.Prefix} {name}"
            return f"{unit.UnitType} / {name}"

    @classmethod
    def get_cost_schedule(cls, cost_item):
        for rel in cost_item.HasAssignments or []:
            if rel.is_a("IfcRelAssignsToControl") and rel.RelatingControl.is_a("IfcCostSchedule"):
                return rel.RelatingControl
        for rel in cost_item.Nests or []:
            return cls.get_cost_schedule(rel.RelatingObject)

    @classmethod
    def is_cost_schedule_active(cls, cost_schedule):
        return True if cost_schedule.id() == bpy.context.scene.BIMCostProperties.active_cost_schedule_id else False

    @classmethod
    def get_active_cost_schedule(cls):
        if not bpy.context.scene.BIMCostProperties.active_cost_schedule_id:
            return None
        return tool.Ifc.get().by_id(bpy.context.scene.BIMCostProperties.active_cost_schedule_id)

    @classmethod
    def highlight_cost_item(cls, cost_item):
        def expand_ancestors(cost_item):
            cls.expand_cost_item(cost_item)
            for rel in cost_item.Nests or []:
                parent_cost = rel.RelatingObject if rel.RelatingObject.is_a("IfcCostItem") else None
                if parent_cost:
                    expand_ancestors(parent_cost)
            cls.load_cost_schedule_tree()

        cost_props = bpy.context.scene.BIMCostProperties
        if not cost_item.id() in [item.ifc_definition_id for item in cost_props.cost_items]:
            expand_ancestors(cost_item)
        cost_item_index = [item.ifc_definition_id for item in bpy.context.scene.BIMCostProperties.cost_items].index(
            cost_item.id()
        ) or 0
        bpy.context.scene.BIMCostProperties.active_cost_item_index = cost_item_index

    @classmethod
    def get_cost_items_for_product(cls, product):
        return ifcopenshell.util.cost.get_cost_items_for_product(product)

    @classmethod
    def has_cost_assignments(cls, product, cost_schedule=None):
        cost_items = ifcopenshell.util.cost.get_cost_items_for_product(product)
        if cost_schedule:
            cost_items = [
                cost_item for cost_item in cost_items or [] if cls.get_cost_schedule(cost_item) == cost_schedule
            ]
        return bool(cost_items)

    @classmethod
    def load_product_cost_items(cls, product):
        props = bpy.context.scene.BIMCostProperties
        props.is_cost_update_enabled = False
        props.product_cost_items.clear()
        cost_items = ifcopenshell.util.cost.get_cost_items_for_product(product)
        if cost_items:
            for cost_item in cost_items:
                new = props.product_cost_items.add()
                new.name = cost_item.Name or "Unnamed"
                new.ifc_definition_id = cost_item.id()
                quantity, unit = cls.calculate_parametric_quantity(cost_item, product)
                new.total_quantity = quantity or 1
                new.unit_symbol = unit or ""
                new.total_cost_quantity = ifcopenshell.util.cost.get_total_quantity(cost_item)

    @classmethod
    def get_quantity_unit_symbol(cls, quantity):
        unit = ifcopenshell.util.unit.get_property_unit(quantity, tool.Ifc.get())
        if unit:
            return ifcopenshell.util.unit.get_unit_symbol(unit)
        else:
            return None

    @classmethod
    def is_root_cost_item(cls, cost_item):
        if cost_item.HasAssignments:
            for rel in cost_item.HasAssignments:
                if rel.is_a("IfcRelAssignsToControl") and rel.RelatingControl.is_a("IfcCostSchedule"):
                    return True

    @classmethod
    def toggle_cost_item_parent_change(cls, cost_item=None):
        if not cost_item:
            return
        props = bpy.context.scene.BIMCostProperties
        if props.change_cost_item_parent:
            props.active_cost_item_id = cost_item.id()
            props.cost_item_editing_type = "PARENT"
        else:
            cls.disable_editing_cost_item_parent()

    @classmethod
    def change_parent_cost_item(cls, cost_item, new_parent):
        ifcopenshell.api.run("nest.change_nest", tool.Ifc.get(), item=cost_item, new_parent=new_parent)

    @classmethod
    def disable_editing_cost_item_parent(cls):
        bpy.context.scene.BIMCostProperties.active_cost_item_id = 0
        bpy.context.scene.BIMCostProperties.change_cost_item_parent = False

    @classmethod
    def load_cost_item_quantities(cls, cost_item=None):
        if not cost_item:
            cost_item = cls.get_highlighted_cost_item()
        if not cost_item:
            return
        cls.load_cost_item_quantity_assignments(cost_item, related_object_type="PRODUCT")
        cls.load_cost_item_quantity_assignments(cost_item, related_object_type="PROCESS")
        cls.load_cost_item_quantity_assignments(cost_item, related_object_type="RESOURCE")

    @classmethod
    def update_cost_items(cls, product=None, pset=None):
        cost_items = []
        if product:
            cost_items = ifcopenshell.util.cost.get_cost_items_for_product(product)
        if pset:

            def get_products_from_pset(pset):
                products = []
                for rel in pset.DefinesOccurrence or []:
                    if rel.is_a("IfcRelDefinesByProperties"):
                        products.extend(rel.RelatedObjects)
                return products

            products = get_products_from_pset(pset)
            for product in products or []:
                cost_items.extend(ifcopenshell.util.cost.get_cost_items_for_product(product))
        for cost_item in cost_items:
            cls.load_cost_item_quantity_assignments(cost_item, related_object_type="PRODUCT")

    @classmethod
    def has_schedules(cls):
        return bool(tool.Ifc.get().by_type("IfcCostSchedule"))

    @classmethod
    def get_currency_attributes(cls):
        props = bpy.context.scene.BIMCostProperties
        currency = props.currency
        if currency == "CUSTOM":
            currency = props.custom_currency
        return {
            "Currency": currency,
        }


#!/usr/bin/env python
from __future__ import print_function
import sys,os,traceback
import argparse

import numpy as np
import scipy.stats as stats
import matplotlib.pyplot as plt

import swim_io
from nc_reader import NC_Reader
import stats_driver
import agg
from bunch import Bunch

def simple_load_data(files,varname):
    outputdata=[]
    for f in files:
        outputdata.append(swim_io.read_nc(f,varname).data)
    return np.concatenate(outputdata,axis=0)

def load_data(files,varname,extra):
    if extra[2]==None:
        return simple_load_data(files,varname)

    reader=NC_Reader(None,filelist=files,
                    geo_subset=extra[2],
                    readvars=[varname],ntimes=365,
                    latvar="lat",lonvar="lon")
    outputdata=[]
    nsamples=int(extra[3])
    hucfilename=extra[5]
    if hucfilename:
            hucreader=NC_Reader(None,filelist=[hucfilename],
                            geo_subset=extra[2],
                            readvars=["data"],ntimes=1,
                            latvar="lat",lonvar="lon")
            hucdata=hucreader.next()[0]
            hucreader.close()

    huclist=[]
    for data in reader:
        if nsamples==0:
            if hucfilename:
                aggdata=agg.data2hucs(data[0],hucdata,minvalue=0,maxvalue=500)
                data=np.array(aggdata.data).reshape((1,1,-1))
                huclist=aggdata.hucs
            outputdata.append(data)
        else:
            outputdata.append([agg.dataXn(data[0],nsamples)])

    reader.close()
    outputdata=np.concatenate(outputdata,axis=0)
    print(outputdata.shape)
    return Bunch(data=outputdata,hucs=huclist)

def calc_extreme_value(params,distribution,nyear,datafraction=1.0):
    """Given parameters for and a distribution,
    calculate the n-year return interval storm
    datafraction is the fraction of input data that were
        used when calculating the distribution"""
    probability=1.0/(nyear*365.0*datafraction)

    x=np.arange(0,10000,10)
    pdf=distribution.pdf(x,*params)
    pos=np.where((pdf>=probability) & (pdf>0))[0]

    if len(pos)==0:return 0
    pos=pos[-1]

    x=np.arange(x[pos]-10,x[pos]+10,0.1)
    pdf=distribution.pdf(x,*params)
    pos=np.where((pdf<=probability) &(pdf>0))[0]

    if x[pos[0]]<0:return 0
    return x[pos[0]]

def calc_extremes(files,v,output_base,info,extra,data=None,huclist=[]):
    print("Loading Data")
    sys.stdout.flush()
    if data==None:
        data=load_data(files,v,extra)
        huclist=data.hucs
        data=data.data
    nt,ny,nx=data.shape

    print(output_base)

    distribution=extra[0]
    outputdir=extra[4]
    outputname=outputdir+"extremes_"+output_base
    maxvalue=1000.0
    params=[]
    extreme_vals=[]

    nday_sum=1
    year_intervals=[2,10,50,100]
    nday_intervals=range(1,6)
    curparams=[0,0,0,0]
    curextremes=range(len(year_intervals))
    shortcut=1
    print("Calculating extreme values")
    for i in range(0,ny,shortcut):
        print("progress = "+str(float(i)/ny*100)+"%      ",end="\r")
        sys.stdout.flush()
        params.append([])
        extreme_vals.append([])
        for j in range(0,nx,shortcut):
            threshold=extra[1]*np.median(data[data[:,i,j]<maxvalue,i,j])
            usevals=np.where((data[:,i,j]>threshold)&(data[:,i,j]<(maxvalue*nday_sum)))[0]
            if len(usevals)>0:
                curparams=distribution.fit(data[usevals,i,j])
                curextremes=[calc_extreme_value(curparams,distribution,year,len(usevals)/float(nt)) for year in year_intervals]
                curparams=list(curparams)
                curparams.extend([len(usevals)/float(nt)])
                params[i/shortcut].append(curparams)
                extreme_vals[i/shortcut].append(curextremes)
            else:
                params[i/shortcut].append(list(np.array(curparams)*0))
                extreme_vals[i/shortcut].append(np.array(curextremes)*0)

    print("Writing Output")
    outputparams=np.array(params)
    extremes=np.array(extreme_vals)
    distname=str(distribution).split('.')[3].split()[0]

    if huclist!=[]:
        print(huclist.shape)
        print(extremes.shape)
        extravars=[Bunch(data=huclist,name="hucs",dims=('y',),dtype='d',attributes=None)]
    else:
        extravars=None

    swim_io.write(outputname+"_params"+"_"+distname,outputparams,extravars=extravars)
    swim_io.write(outputname+"_values"+"_"+distname,extremes,extravars=extravars)
    maxvals=[50,100,150,200]
    for i in range(extremes.shape[2]):
        plt.clf()
        if extremes.shape[0]==1:
            plt.imshow(extremes[:,:,i].repeat(extremes.shape[1],axis=0),
                       vmin=0,vmax=maxvals[i]*0.66,origin="lower")
        else:
            plt.imshow(extremes[:,:,i],vmin=0,vmax=maxvals[i]*0.66,origin="lower")
        plt.title(str(year_intervals[i])+"yr return interval")
        plt.colorbar()
        plt.savefig(outputname+"_yr{0:03d}.png".format(year_intervals[i]))

if __name__ == '__main__':
    try:
        parser= argparse.ArgumentParser(description='Calculate nday/nyear return period events. ')
        parser.add_argument('-method',dest="methods",nargs="?",action='store',
                    default=["SDmon","CAe0","SARe0","SDe0","CA","CAe1","SDe1","SARe1","SDe","CAe","SD"])
        parser.add_argument('-model',dest="models",nargs="?",action='store',
                    default=["ncep","narr"])
        parser.add_argument('-res',dest="resolution",nargs="?",action='store',
                    default=["12km","6km"])
        parser.add_argument('-var',dest="variable",nargs="?",action='store',
                    default=["pr"])
        parser.add_argument('-bc',dest="BC",nargs="?",action='store',
                    default=["BC",""])
        parser.add_argument('-agg',dest="aggfactor",nargs="?",action='store',
                    default=0)
        parser.add_argument('-huc',dest="huc",nargs="?",action='store',
                    default="")
        parser.add_argument('-out',dest="outputdir",nargs="?",action='store',
                    default="./")
        parser.add_argument('-yearsearch',dest="yearsearch",nargs="?",action='store',
                    default=["200[1-2]"])
        parser.add_argument ('--runobs', action='store_true',
                default=False, help='verbose output', dest='runobs')

        parser.add_argument ('-distribution', action='store_true',
                default="gamma", help='verbose output', dest='distribution')

        parser.add_argument('-v', '--version',action='version',
                version='extreme_pr v.1.0')
        parser.add_argument ('--verbose', action='store_true',
                default=False, help='verbose output', dest='verbose')
        args = parser.parse_args()

        if args.distribution=="gamma":
            distribution=stats.gamma
        elif args.distribution=="weibull":
            distribution=stats.weibull_min
        elif args.distribution=="exponential":
            distribution=stats.expon
        else:
            distribution=stats.gamma

        geosubset=[35,43,-113,-101]
        runobs=args.runobs
        runstat=not runobs
        aggfactor=args.aggfactor
        outputdir=args.outputdir
        hucfile=args.huc
        pr_threshold=0.5
        print(outputdir)
        # stats_driver.drive requires lists to iterate over, but CLI args will be individual
        #  elements.  However, default values are all lists, so we don't want to make them
        # lists of lists so we have to test to see if they are a list already first...
        for k in args.__dict__.keys():
            if type(args.__dict__[k])!=list:
                args.__dict__[k]=[args.__dict__[k]]

        if runobs:
            print("Running Observations Only")
            sys.stdout.flush()
        stats_driver.drive(calc_extremes,
                    yearsearch=args.yearsearch[0],
                    obs=runobs,stat=runstat,
                    extra=[distribution,pr_threshold,geosubset,aggfactor,outputdir,hucfile],
                    methods=args.methods,
                    BCs=args.BC,
                    models=args.models,
                    resolutions=args.resolution,
                    variables=args.variable,
                    extendedmethods=False)

    except Exception as e:
        print("Error Unexpected Exception")
        print(e)
        traceback.print_exc()
        os._exit(1)


import torch
from functools import reduce
from dataset import Dataset
import numpy as np
from numpy import genfromtxt
import time
from tqdm import tqdm
from torch.utils import data as torch_data
from utils import *
from models import *
from sklearn.utils import shuffle
from collections import defaultdict
import pickle
import logging
torch.manual_seed(7)
np.random.seed(7)

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class Tester:
    def __init__(self, dataset, args, model_path, valid_or_test):
        torch.manual_seed(7)
        np.random.seed(7)
        self.args = args
        self.fast_testing = args.fast_testing
        self.hierarchy = args.hierarchy
        self.dataset = dataset
        self.name = args.dataset
        self.setting = args.setting
        self.device = args.device
        self.model_name = args.model
        self.lmbda = args.reg
        self.dir = "datasets/" + self.dataset.name + "/"
        if self.model_name == 'TransE':
            self.model = TransE(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device)
        if self.model_name == 'TransH':
            self.model = TransH(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device)
        if self.model_name == 'TransR':
            self.model = TransR(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device)
        if self.model_name == 'TransD':
            self.model = TransD(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device)
        if self.model_name == 'HAKE':
            self.model = HAKE(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device,
                args.margin)
        if self.model_name == 'RotatE':
            self.model = RotatE2(
                dataset.num_ent(),
                dataset.num_rel(),
                args,
                self.device,
                args.margin)
        if self.model_name == 'DistMult':
            self.model = DistMult(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device,
                args)
        if self.model_name == 'ComplEx':
            self.model = ComplEx(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device,
                args)
        if self.model_name == 'SimplE':
            self.model = SimplE(
                dataset.num_ent(),
                dataset.num_rel(),
                args.dim,
                self.device,
                self.lmbda,
                args)
        if self.model_name == 'TuckER':
            self.model = TuckER(
                dataset.num_ent(),
                dataset.num_rel(),
                args,
                self.device)
        if self.model_name == 'ConvE':
            self.model = ConvE(
                dataset.num_ent(),
                dataset.num_rel(),
                args,
                self.device)
        if self.model_name == 'ConvKB1D':
            self.model = ConvKB1D(
                dataset.num_ent(),
                dataset.num_rel(),
                args,
                self.device)
        if self.model_name == 'ConvKB2D':
            self.model = ConvKB2D(
                dataset.num_ent(),
                dataset.num_rel(),
                args,
                self.device)
        if self.model_name == 'RGCN':
            self.model = RGCN(
                dataset.num_ent(),
                dataset.num_rel(),
                args,
                self.device)
        if self.model_name == 'CompGCN':
            self.model = CompGCN_DistMult(
                dataset.num_ent(), dataset.num_rel(), args, self.device)

        self.model.load_state_dict(torch.load(model_path))
        self.model.eval()
        self.valid_or_test = valid_or_test
        self.batch_size = args.batch_size
        self.neg_ratio = args.neg_ratio
        self.neg_sampler = args.neg_sampler
        self.metric = args.metrics
        self.sem = args.sem

        if self.args.lossfunc == 'bce':
            with open(self.dir + "pickle/observed_tails_inv.pkl", 'rb') as f:
                self.all_possible_ts = pickle.load(f)
        else:
            with open(self.dir + "pickle/observed_tails_original_kg.pkl", 'rb') as f:
                self.all_possible_ts = pickle.load(f)
            with open(self.dir + "pickle/observed_heads_original_kg.pkl", 'rb') as f:
                self.all_possible_hs = pickle.load(f)

        def read_pickle(file):
            try:
                with open(self.dir + "pickle/" + file + ".pkl", 'rb') as f:
                    pckl = pickle.load(f)
                    return pckl
            except BaseException:
                print(file + ".pkl not found.")

        def count_rel():
            rel_counts_h = {}
            rel_counts_t = {}
            for k, v in self.dataset.r2id2dom2id.items():

                rel_counts_h[k] = len(
                    self.dataset.class2id2ent2id[self.dataset.r2id2dom2id[k]])
                rel_counts_t[k] = len(
                    self.dataset.class2id2ent2id[self.dataset.r2id2range2id[k]])

            rel_counts_h = dict(
                sorted(
                    rel_counts_h.items(),
                    key=lambda x: x[1]))
            rel_counts_t = dict(
                sorted(
                    rel_counts_t.items(),
                    key=lambda x: x[1]))
            return rel_counts_h, rel_counts_t

        self.count_hs, self.count_ts = count_rel()

        def create_buckets():
            buckets_h, buckets_t = {}, {}
            if self.name == 'Yago14k':
                for r, hs in self.count_hs.items():
                    if hs < 2102:
                        buckets_h[r] = 1
                    elif hs <= 3624:
                        buckets_h[r] = 2
                    else:
                        buckets_h[r] = 3
                for r, ts in self.count_ts.items():
                    if ts < 2102:
                        buckets_t[r] = 1
                    elif ts <= 3624:
                        buckets_t[r] = 2
                    else:
                        buckets_t[r] = 3
            if self.name == 'FB15k187':
                for r, hs in self.count_hs.items():
                    if hs < 278:
                        buckets_h[r] = 1
                    elif hs <= 1391:
                        buckets_h[r] = 2
                    else:
                        buckets_h[r] = 3
                for r, ts in self.count_ts.items():
                    if ts < 278:
                        buckets_t[r] = 1
                    elif ts <= 1391:
                        buckets_t[r] = 2
                    else:
                        buckets_t[r] = 3
            if self.name == 'DBpedia77k':
                for r, hs in self.count_hs.items():
                    if hs < 1295:
                        buckets_h[r] = 1
                    elif hs <= 11586:
                        buckets_h[r] = 2
                    else:
                        buckets_h[r] = 3
                for r, ts in self.count_ts.items():
                    if ts < 1419:
                        buckets_t[r] = 1
                    elif ts <= 11586:
                        buckets_t[r] = 2
                    else:
                        buckets_t[r] = 3

            return buckets_h, buckets_t

        self.buckets_h, self.buckets_t = create_buckets()

    def get_observed_triples(self, train2id, valid2id, test2id):
        all_possible_hs = defaultdict(dict)
        all_possible_ts = defaultdict(dict)
        train2id = torch.as_tensor(train2id.to_numpy(), dtype=torch.int32)
        valid2id = torch.as_tensor(valid2id.to_numpy(), dtype=torch.int32)
        test2id = torch.as_tensor(test2id.to_numpy(), dtype=torch.int32)
        all_triples = torch.cat((train2id, valid2id, test2id))
        X = all_triples.detach().clone()
        for triple in range(X.shape[0]):
            h, r, t = X[triple][0].item(
            ), X[triple][1].item(), X[triple][2].item()
            try:
                all_possible_ts[h][r].append(t)
            except KeyError:
                all_possible_ts[h][r] = [t]

        for triple in range(X.shape[0]):
            h, r, t = X[triple][0].item(
            ), X[triple][1].item(), X[triple][2].item()
            try:
                all_possible_hs[t][r].append(h)
            except KeyError:
                all_possible_hs[t][r] = [h]

        all_possible_ts = dict(all_possible_ts)
        all_possible_hs = dict(all_possible_hs)
        return all_possible_hs, all_possible_ts

    def get_observed_h(self, h, r, t):
        return (
            list(set(self.all_possible_hs[t.item()][r.item()]) - set([h.item()])))

    def get_observed_t(self, h, r, t):
        try:
            return (
                list(set(self.all_possible_ts[h.item()][r.item()]) - set([t.item()])))
        except KeyError:
            return None

    def predictions(self, h, r, t, all_entities):
        heads = h.reshape(-1, 1).repeat(1, all_entities.size()
                                        [1]).to(self.device)
        rels = r.reshape(-1, 1).repeat(1, all_entities.size()
                                       [1]).to(self.device)
        tails = t.reshape(-1, 1).repeat(1, all_entities.size()
                                        [1]).to(self.device)
        triplets = torch.stack((heads, rels, all_entities),
                               dim=2).reshape(-1, 3).to(self.device)

        tails_predictions = self.model.forward2(
            (triplets[:, 0]), (triplets[:, 1]), (triplets[:, 2])).reshape(1, -1)
        triplets = torch.stack((all_entities, rels, tails),
                               dim=2).reshape(-1, 3).to(self.device)
        heads_predictions = self.model.forward2(
            (triplets[:, 0]), (triplets[:, 1]), (triplets[:, 2])).reshape(1, -1)
        return heads_predictions.squeeze(), tails_predictions.squeeze()

    def test(self):
        schema_CWA_h = {}
        schema_CWA_t = {}
        sem_h_triples_CWA = {}
        sem_t_triples_CWA = {}
        for b in range(1, self.args.buckets + 1):
            schema_CWA_h[b] = {'sem1_h': 0.0, 'sem3_h': 0.0, 'sem10_h': 0.0}
            schema_CWA_t[b] = {'sem1_t': 0.0, 'sem3_t': 0.0, 'sem10_t': 0.0}
            sem_h_triples_CWA[b] = 0
            sem_t_triples_CWA[b] = 0

        filt_hit_h, filt_hit_t = {}, {}
        for b in range(1, self.args.buckets + 1):
            filt_hit_h[b] = {'hit1_h': 0.0, 'hit3_h': 0.0, 'hit10_h': 0.0}
            filt_hit_t[b] = {'hit1_t': 0.0, 'hit3_t': 0.0, 'hit10_t': 0.0}

        filt_mrr, filt_mrr_h, filt_mrr_t = {}, {}, {}
        for b in range(1, self.args.buckets + 1):
            filt_mrr[b] = 0.0
            filt_mrr_h[b] = []
            filt_mrr_t[b] = []

        nb_buckets_h, nb_buckets_t = {}, {}
        for b in range(1, self.args.buckets + 1):
            nb_buckets_h[b] = 0
            nb_buckets_t[b] = 0

        zero_tensor = torch.tensor([0], device=self.device)
        one_tensor = torch.tensor([1], device=self.device)
        X_valid_or_test = torch.from_numpy(
            (self.dataset.data[self.valid_or_test]))
        num_ent = self.dataset.num_ent()
        all_entities = torch.arange(
            end=num_ent, device=self.device).unsqueeze(0)
        start = time.time()
        if self.args.lossfunc == 'bce':
            half_idx = int(X_valid_or_test.shape[0] / 2)
            X_tails = X_valid_or_test[:half_idx]
            X_inv = X_valid_or_test[half_idx:]
            for triple in tqdm(X_tails):
                h, r, t = triple[0], triple[1], triple[2]
                bucket = self.buckets_t[r.item()]
                nb_buckets_t[bucket] += 1
                rm_idx_t = self.get_observed_t(h, r, t)
                tails_predictions = self.model.get_score(
                    h.to(
                        self.device), r.to(
                        self.device), t.to(
                        self.device)).squeeze()
                tails_predictions[[rm_idx_t]] = - np.inf
                indices_tail = tails_predictions.argsort(descending=True)
                filt_rank_t = (
                    indices_tail == t).nonzero(
                    as_tuple=True)[0].item() + 1
                filt_mrr_t[bucket].append(1.0 / filt_rank_t)
                if self.metric == 'sem' or self.metric == 'all':
                    if self.sem == 'schema' or self.sem == 'both':
                        if r.item() in self.dataset.r2id2range2id.keys(
                        ) and self.dataset.r2id2range2id[r.item()] in self.dataset.class2id2ent2id.keys():
                            if len(
                                    self.dataset.class2id2ent2id[self.dataset.r2id2range2id[r.item()]]) >= 10:
                                if self.setting == 'both' or self.setting == 'CWA':
                                    s1, s3, s5, s10 = self.sem_at_k(
                                        indices_tail[:100], r.item(), side='tail', k=10, setting='CWA')
                                    schema_CWA_t[bucket]['sem1_t'], schema_CWA_t[bucket]['sem3_t'], schema_CWA_t[bucket]['sem10_t'] = schema_CWA_t[
                                        bucket]['sem1_t'] + s1, schema_CWA_t[bucket]['sem3_t'] + s3, schema_CWA_t[bucket]['sem10_t'] + s10
                                    sem_t_triples_CWA[bucket] += 1

                if self.metric == 'ranks' or self.metric == 'all':
                    filt_hit_t[bucket]['hit10_t'] += torch.where(
                        indices_tail[:10] == t.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_t[bucket]['hit3_t'] += torch.where(
                        indices_tail[:3] == t.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_t[bucket]['hit1_t'] += torch.where(
                        indices_tail[:1] == t.item(), one_tensor, zero_tensor).sum().item()

            # batch of reversed triples
            for triple in tqdm(X_inv):
                h, r, t = triple[0], triple[1], triple[2]
                bucket = self.buckets_h[r.item() - self.dataset.num_rel()]
                nb_buckets_h[bucket] += 1
                rm_idx_t = self.get_observed_t(h, r, t)
                tails_predictions = self.model.get_score(
                    h.to(
                        self.device), r.to(
                        self.device), t.to(
                        self.device)).squeeze()
                tails_predictions[[rm_idx_t]] = - np.inf
                indices_tail = tails_predictions.argsort(descending=True)
                filt_rank_h = (
                    indices_tail == t).nonzero(
                    as_tuple=True)[0].item() + 1
                filt_mrr_h[bucket].append(1.0 / filt_rank_h)
                if self.metric == 'sem' or self.metric == 'all':
                    if self.sem == 'schema' or self.sem == 'both':
                        if r.item() in self.dataset.r2id2range2id.keys(
                        ) and self.dataset.r2id2range2id[r.item()] in self.dataset.class2id2ent2id.keys():
                            if len(
                                    self.dataset.class2id2ent2id[self.dataset.r2id2range2id[r.item()]]) >= 10:
                                if self.setting == 'both' or self.setting == 'CWA':
                                    s1, s3, s5, s10 = self.sem_at_k(
                                        indices_tail[:100], r.item(), side='tail', k=10, setting='CWA')
                                    schema_CWA_h[bucket]['sem1_h'], schema_CWA_h[bucket]['sem3_h'], schema_CWA_h[bucket]['sem10_h'] = schema_CWA_h[
                                        bucket]['sem1_h'] + s1, schema_CWA_h[bucket]['sem3_h'] + s3, schema_CWA_h[bucket]['sem10_h'] + s10
                                    sem_h_triples_CWA[bucket] += 1

                if self.metric == 'ranks' or self.metric == 'all':
                    filt_hit_h[bucket]['hit10_h'] += torch.where(
                        indices_tail[:10] == t.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_h[bucket]['hit3_h'] += torch.where(
                        indices_tail[:3] == t.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_h[bucket]['hit1_h'] += torch.where(
                        indices_tail[:1] == t.item(), one_tensor, zero_tensor).sum().item()

        else:
            X_valid_or_test = torch.from_numpy(
                (self.dataset.data[self.valid_or_test]))
            for triple in tqdm(X_valid_or_test):
                h, r, t = triple[0], triple[1], triple[2]
                bucket_h = self.buckets_h[r.item()]
                bucket_t = self.buckets_t[r.item()]
                nb_buckets_h[bucket_h] += 1
                nb_buckets_t[bucket_t] += 1
                rm_idx_t = self.get_observed_t(h, r, t)
                rm_idx_h = self.get_observed_h(h, r, t)
                heads_predictions, tails_predictions = self.predictions(
                    h, r, t, all_entities)
                heads_predictions[[rm_idx_h]], tails_predictions[[
                    rm_idx_t]] = -np.inf, -np.inf
                indices_tail, indices_head = tails_predictions.argsort(
                    descending=True), heads_predictions.argsort(descending=True)
                filt_rank_h = (
                    indices_head == h).nonzero(
                    as_tuple=True)[0].item() + 1
                filt_rank_t = (
                    indices_tail == t).nonzero(
                    as_tuple=True)[0].item() + 1
                filt_mrr_h[bucket_h].append(1.0 / filt_rank_h)
                filt_mrr_t[bucket_t].append(1.0 / filt_rank_t)
                if self.metric == 'sem' or self.metric == 'all':
                    if self.sem == 'schema' or self.sem == 'both':
                        if r.item() in self.dataset.r2id2dom2id.keys(
                        ) and self.dataset.r2id2dom2id[r.item()] in self.dataset.class2id2ent2id.keys():
                            if len(
                                    self.dataset.class2id2ent2id[self.dataset.r2id2dom2id[r.item()]]) >= 10:
                                if self.setting == 'both' or self.setting == 'CWA':
                                    s1, s3, s5, s10 = self.sem_at_k(
                                        indices_head[:100], r.item(), side='head', k=10, setting='CWA')
                                    schema_CWA_h[bucket_h]['sem1_h'], schema_CWA_h[bucket_h]['sem3_h'], schema_CWA_h[bucket_h]['sem10_h'] = schema_CWA_h[
                                        bucket_h]['sem1_h'] + s1, schema_CWA_h[bucket_h]['sem3_h'] + s3, schema_CWA_h[bucket_h]['sem10_h'] + s10
                                    sem_h_triples_CWA[bucket_h] += 1

                        if r.item() in self.dataset.r2id2range2id.keys(
                        ) and self.dataset.r2id2range2id[r.item()] in self.dataset.class2id2ent2id.keys():
                            if len(
                                    self.dataset.class2id2ent2id[self.dataset.r2id2range2id[r.item()]]) >= 10:
                                if self.setting == 'both' or self.setting == 'CWA':
                                    s1, s3, s5, s10 = self.sem_at_k(
                                        indices_tail[:100], r.item(), side='tail', k=10, setting='CWA')
                                    schema_CWA_t[bucket_t]['sem1_t'], schema_CWA_t[bucket_t]['sem3_t'], schema_CWA_t[bucket_t]['sem10_t'] = schema_CWA_t[
                                        bucket_t]['sem1_t'] + s1, schema_CWA_t[bucket_t]['sem3_t'] + s3, schema_CWA_t[bucket_t]['sem10_t'] + s10
                                    sem_t_triples_CWA[bucket_t] += 1

                if self.metric == 'ranks' or self.metric == 'all':
                    filt_hit_h[bucket_h]['hit10_h'] += torch.where(
                        indices_head[:10] == h.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_h[bucket_h]['hit3_h'] += torch.where(
                        indices_head[:3] == h.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_h[bucket_h]['hit1_h'] += torch.where(
                        indices_head[:1] == h.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_t[bucket_t]['hit10_t'] += torch.where(
                        indices_tail[:10] == t.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_t[bucket_t]['hit3_t'] += torch.where(
                        indices_tail[:3] == t.item(), one_tensor, zero_tensor).sum().item()
                    filt_hit_t[bucket_t]['hit1_t'] += torch.where(
                        indices_tail[:1] == t.item(), one_tensor, zero_tensor).sum().item()

        print(time.time() - start)
        for b in range(1, self.args.buckets + 1):
            filt_mrr_h[b] = np.mean(filt_mrr_t[b])
            filt_mrr_t[b] = np.mean(filt_mrr_h[b])
            filt_mrr[b] = (filt_mrr_h[b] + filt_mrr_t[b]) / 2

        filtered_hits_at_1 = {}
        filtered_hits_at_3 = {}
        filtered_hits_at_10 = {}
        for b in range(1, self.args.buckets + 1):
            filt_hit_h[b]['hit1_h'] /= nb_buckets_h[b]
            filt_hit_h[b]['hit3_h'] /= nb_buckets_h[b]
            filt_hit_h[b]['hit10_h'] /= nb_buckets_h[b]
            filt_hit_t[b]['hit1_t'] /= nb_buckets_t[b]
            filt_hit_t[b]['hit3_t'] /= nb_buckets_t[b]
            filt_hit_t[b]['hit10_t'] /= nb_buckets_t[b]
            filtered_hits_at_1[b] = (
                filt_hit_h[b]['hit1_h'] + filt_hit_t[b]['hit1_t']) / 2
            filtered_hits_at_3[b] = (
                filt_hit_h[b]['hit3_h'] + filt_hit_t[b]['hit3_t']) / 2
            filtered_hits_at_10[b] = (
                filt_hit_h[b]['hit10_h'] + filt_hit_t[b]['hit10_t']) / 2

        if (self.sem == 'schema' or self.sem ==
                'both') and self.metric != 'ranks':
            schema_CWA = {}
            for b in range(1, self.args.buckets + 1):
                schema_CWA_h[b]['sem1_h'] /= sem_h_triples_CWA[b]
                schema_CWA_t[b]['sem1_t'] /= sem_t_triples_CWA[b]
                schema_CWA_h[b]['sem3_h'] /= sem_h_triples_CWA[b]
                schema_CWA_t[b]['sem3_t'] /= sem_t_triples_CWA[b]
                schema_CWA_h[b]['sem10_h'] /= sem_h_triples_CWA[b]
                schema_CWA_t[b]['sem10_t'] /= sem_t_triples_CWA[b]
                schema_CWA[b] = {}
                schema_CWA[b]['sem1'] = (
                    schema_CWA_h[b]['sem1_h'] + schema_CWA_t[b]['sem1_t']) / 2
                schema_CWA[b]['sem3'] = (
                    schema_CWA_h[b]['sem3_h'] + schema_CWA_t[b]['sem3_t']) / 2
                schema_CWA[b]['sem10'] = (
                    schema_CWA_h[b]['sem10_h'] + schema_CWA_t[b]['sem10_t']) / 2

        if self.metric == 'sem' or self.metric == 'all':
            return filt_mrr_h, filt_mrr_t, filt_mrr, filt_hit_h, filt_hit_t, filtered_hits_at_1, filtered_hits_at_3, filtered_hits_at_10, \
                schema_CWA_h, schema_CWA_t, schema_CWA

        else:
            return filt_mrr, filtered_hits_at_1, filtered_hits_at_3, filtered_hits_at_5, filtered_hits_at_10,\
                filt_mrr_h, filt_mrr_t, filt_h1_h, filt_h1_t, filt_h3_h, filt_h3_t, filt_h5_h, filt_h5_t, filt_h10_h, filt_h10_t

    def sem_at_k(self, preds, rel, side='head', k=10, setting='CWA'):
        valid_types = []
        for pred in preds.tolist():
            if len(valid_types) == 10:
                return ((valid_types[0]),
                        np.mean(valid_types[:3]),
                        np.mean(valid_types[:5]),
                        np.mean(valid_types[:10]))
            else:
                try:
                    classes = self.dataset.instype_all[pred]
                    if side == 'head':
                        dom = self.dataset.r2id2dom2id[rel]
                        valid_types.append(1 if dom in classes else 0)
                    elif side == 'tail':
                        rang = self.dataset.r2id2range2id[rel]
                        valid_types.append(1 if rang in classes else 0)
                except KeyError:
                    valid_types.append(0)


# Largest palindrome product

# A palindromic number reads the same both ways. The largest palindrome made from the product of
# two 2-digit numbers is 9009 = 91 × 99.

# Find the largest palindrome made from the product of two 3-digit numbers.

def palindrome(number):
    newNumber = int(str(number)[::-1])
    if number == newNumber:
        return True
    else:
        return False

limit = 1000
for i in range(limit):
    for j in range(limit):
        mult = i * j
        if palindrome(mult):
            firstValue = i
            secondValue = j

print(f'The largest numbers product a palindrome are {firstValue} x {secondValue} = {(firstValue * secondValue)}')


import numpy as np
import matplotlib.pylab as plt
import matplotlib
import os
from mpl_toolkits.axes_grid1 import make_axes_locatable
import matplotlib.colors as colors
import matplotlib.patches as patches
import csv

def read_data(file):
    with open(file, 'r') as txt_files:
        i = 0
        data = []
        for line in txt_files:
            data.append(line.split(","))
    return data

def heatmap(data, row_labels, col_labels, ax=None,
            cbar_kw=None, cbarlabel="", **kwargs):
    """
    Create a heatmap from a numpy array and two lists of labels.

    Parameters
    ----------
    data
        A 2D numpy array of shape (M, N).
    row_labels
        A list or array of length M with the labels for the rows.
    col_labels
        A list or array of length N with the labels for the columns.
    ax
        A `matplotlib.axes.Axes` instance to which the heatmap is plotted.  If
        not provided, use current axes or create a new one.  Optional.
    cbar_kw
        A dictionary with arguments to `matplotlib.Figure.colorbar`.  Optional.
    cbarlabel
        The label for the colorbar.  Optional.
    **kwargs
        All other arguments are forwarded to `imshow`.
    """

    if ax is None:
        ax = plt.gca()

    if cbar_kw is None:
        cbar_kw = {}

    # Plot the heatmap
#    im = ax.imshow(data, **kwargs, origin = 'lower')
    im = ax.imshow(data, **kwargs)

    # Show all ticks and label them with the respective list entries.
    ax.set_xticks(np.arange(data.shape[1]))
    ax.set_xticklabels(col_labels)
    ax.set_yticks(np.arange(data.shape[0]))
    ax.set_yticklabels(row_labels)

    # Let the horizontal axes labeling appear on top.
    ax.tick_params(top=False, bottom=True,
                   labeltop=False, labelbottom=True)

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=0,
             rotation_mode="anchor")

    # Turn spines off and create white grid.
    #ax.spines[:].set_visible(False)

    ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True)
    ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True)
    ax.set_axisbelow(True)
    ax.grid(which="minor", color="w", linestyle='-', linewidth=3)
    ax.add_patch(
    patches.Rectangle(xy=(2.5, -1.5), width=4, height=5, linewidth=3,  color='red', fill=False))
    ax.tick_params(which="minor", bottom=False, left=False)
    ax.set_xlabel("Average last_use (number of windows)")
    ax.set_ylabel("Accesses (number of windows)")

        # Create colorbar
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.2)

    cbar = ax.figure.colorbar(im, cax=cax)
    cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom")
    return im, cbar

def annotate_heatmap(im, data=None, valfmt="{x:.2f}",
                     textcolors=("black", "white"),
                     threshold=None,text=None, **textkw):
    """
    A function to annotate a heatmap.

    Parameters
    ----------
    im
        The AxesImage to be labeled.
    data
        Data used to annotate.  If None, the image's data is used.  Optional.
    valfmt
        The format of the annotations inside the heatmap.  This should either
        use the string format method, e.g. "$ {x:.2f}", or be a
        `matplotlib.ticker.Formatter`.  Optional.
    textcolors
        A pair of colors.  The first is used for values below a threshold,
        the second for those above.  Optional.
    threshold
        Value in data units according to which the colors from textcolors are
        applied.  If None (the default) uses the middle of the colormap as
        separation.  Optional.
    **kwargs
        All other arguments are forwarded to each call to `text` used to create
        the text labels.
    """

    if not isinstance(data, (list, np.ndarray)):
        data = im.get_array()

    # Normalize the threshold to the images color range.
    if threshold is not None:
        threshold = im.norm(threshold)
    else:
        threshold = im.norm(data.max())/2.

    # Set default alignment to center, but allow it to be
    # overwritten by textkw.
    kw = dict(horizontalalignment="center",
              verticalalignment="center")
    kw.update(textkw)

    # Get the formatter in case a string is supplied
    if isinstance(valfmt, str):
        valfmt = matplotlib.ticker.StrMethodFormatter(valfmt)

    # Loop over the data and create a `Text` for each "pixel".
    # Change the text's color depending on the data.
    texts = []
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])
            if text is not None:
                if i == data.shape[0]-1 and j>0:
                    textt = im.axes.text(j, i, "n.a", **kw)
                else:
                    textt = im.axes.text(j, i, text[i, j], **kw)
            else:
                if i == data.shape[0]-1 and j>0:
                    textt = im.axes.text(j, i, "n.a", **kw)
                else:
                    textt = im.axes.text(j, i, valfmt(data[i, j], None), **kw)
            texts.append(text)

    return texts

dir_path = os.path.dirname(os.path.realpath(__file__))
data_file = dir_path + "/../Data/ProcessedData/last_use_vector.txt"
plot_file = dir_path + "/../Data/PlotData/scatter_lastuse_access.pdf"
data = read_data(data_file)
matrix = np.zeros((7,6))
nb_data = 0
avg_last_use = 0
avg_transaction = 0
nb_unfriendly = 0
for x in data:
    coord = [0,0]
    nb_data += 1
    if float(x[1]) <= 1:
        coord[0] = 6
    elif float(x[1]) <= 2:
        coord[0] = 5
    elif float(x[1]) <= 3:
        coord[0] = 4
    elif float(x[1]) <= 5:
        coord[0] = 3
    elif float(x[1]) <= 10:
        coord[0] = 2
    elif float(x[1]) <= 20:
        coord[0] = 1
    else:
        coord[0] = 0

    if float(x[2]) <= 0.5:
        coord[1] = 0
    elif float(x[2]) <= 1:
        coord[1] = 1
    elif float(x[2]) <= 3:
        coord[1] = 2
    elif float(x[2]) <= 5:
        coord[1] = 3
    elif float(x[2]) <= 10:
        coord[1] = 4
    else:
        coord[1] = 5
    matrix[coord[0], coord[1]] = matrix[coord[0], coord[1]] + 1

    if coord[1] >= 3:
        nb_unfriendly += 1
        avg_last_use += float(x[2])
        avg_transaction += float(x[1])

print("nb unfriendly:", nb_unfriendly)
print("avg last use:", avg_last_use/nb_unfriendly)
print("avg tx:", avg_transaction/nb_unfriendly)

matrix = matrix / nb_data
matrix = matrix * 100
print(nb_data)
with open("data.csv",'w') as output_file:
    fieldnames = ['0','1','2','3','4','5']
    dict_writer = csv.DictWriter(output_file, fieldnames=fieldnames)
    dict_writer.writeheader()
    for x in range(7):
        record = {}
        for y in range(6):
            record[str(y)] = matrix[x,y]
        dict_writer.writerow(record)
print("done")

last_use = ["[0,0.5]\n ", "]0.5,1]\n ", "]1,3]\n ", "]3,5]\n ", "]5,10]\n", ">10\n "]
nb_access = [">20\n ", "[11,20]\n ", "[6,10]\n ", "[4,5]\n", "3\n ", "2\n ", "1\n "]

fig, ax = plt.subplots(figsize=(5, 5))

im, cbar = heatmap(matrix, nb_access, last_use, ax=ax,
                   cmap="Greys", norm=colors.PowerNorm(gamma = 0.4), cbarlabel="% of accounts")
texts = annotate_heatmap(im, valfmt="{x:.1f}", threshold=4)

plt.tight_layout()
plt.savefig(plot_file, format = "pdf")
#plt.show()