from models.resnet import ResNet50
from datasets.augLib import  class_map, class_num, _parse_example

import tensorflow as tf
import numpy as np
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

training_filenames = [r'F:\data\augLib\auglib_rain.tfrecord']
test_filenames = [r'F:\data\augLib\auglib_test.tfrecord']

learning_rate = 0.01
batch_size = 24
ckpt_dir = 'ckpt/lib_resnet50'

def _parse_function(example_proto):
    return _parse_example(example_proto,shape=[224,224,1])

def validate(filename_list, session, ops):
    if not isinstance(filename_list,list):
        filename_list = [filename_list]
    print('strat to validate ',filename_list)
    session.run(iterator.initializer, feed_dict={filenames: filename_list})
    step = 0
    loss_acc = [0,0]

    while True:
        try:
            logits_val, loss_val, acc_val = session.run(ops)
            if step % 50 == 0:
                print('step %d >>> loss:%f, acc:%f' % (step, loss_val, acc_val))

            loss_acc = [ r+t for (r, t) in zip(loss_acc,[loss_val,acc_val])]
            step += 1
        except tf.errors.OutOfRangeError:
            loss_acc = [ r/step for r in loss_acc]
            print('loss and acc:',loss_acc)
            break
    print('=========val end==========')

# 如果您有两组分别用于训练和验证的文件，
# 则可以使用 tf.placeholder(tf.string) 来表示文件名，
# 并使用适当的文件名初始化迭代器：
filenames = tf.placeholder(tf.string, shape=[None])
dataset = tf.data.TFRecordDataset(filenames)
dataset = dataset.map(_parse_function)  # Parse the record into tensors.
dataset = dataset.shuffle(buffer_size=10000)
# dataset = dataset.repeat()  # Repeat the input indefinitely.
dataset = dataset.batch(batch_size)
iterator = dataset.make_initializable_iterator()

next_img, next_label = iterator.get_next()
print(next_img.get_shape())
print(next_label.get_shape())

model = ResNet50(next_img, class_num)
logits = model.logits
print(logits)

with tf.name_scope('loss'):
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=next_label,
                                                                   logits=logits,
                                                                   name='xentropy')
    loss = tf.reduce_mean(cross_entropy, name='train_loss')
    l2 = tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()])

with tf.name_scope('train'):
    train_op = tf.train.MomentumOptimizer(
        learning_rate=learning_rate,momentum=0.9).minimize(loss=loss + l2 * 1e-5)

with tf.name_scope('accuracy'):
    # correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(next_label, 1))
    # accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
    correct = tf.equal(tf.argmax(logits, 1), tf.cast(next_label,tf.int64))
    # correct = tf.nn.in_top_k(output, next_label, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())  # 又尼玛忘了
    saver = tf.train.Saver(max_to_keep=3)
    # ckpt_path = tf.train.latest_checkpoint(ckpt_dir)
    # if ckpt_path and tf.train.checkpoint_exists(ckpt_path):
    #     saver.restore(sess, ckpt_path)
    #     print('restore model from ', ckpt_path)

    # Compute for n epochs.
    print('开始训练')
    # for epoch in range(3):
    sess.run(iterator.initializer, feed_dict={filenames: training_filenames})
    print('初始化迭代器成功')

    img,label = sess.run([next_img,next_label])

    """
    bug bug bug
    Process finished with exit code -1073741819 (0xC0000005)
    """

    # print(img)
    # print(label)

        # step = 0
        # while True:
        #     try:
        #         _, logits_train, loss_train,  acc_train = sess.run([train_op,logits,loss,accuracy])
        #         if step % 50 == 0:
        #             print('epoch %d, step %d >>> loss:%f, acc:%f' % (epoch,step,loss_train,acc_train))
        #         step += 1
        #     except tf.errors.OutOfRangeError:
        #         break
        # save_path = ckpt_dir + '/' + 'lib-11-100_resnet_%d'
        # saver.save(sess, save_path % epoch)
        #
        # ops = [logits,loss,train_op,accuracy]
        # validate(test_filenames,sess,ops=ops)


import sys
from PyQt5.QtWidgets import *

class MyWindow(QMainWindow):
    def __init__(self):
        super().__init__()

        btn1 = QPushButton("Button1")
        btn2 = QPushButton("Button2")

        # QMainWindow 상속 받은 window를 사용하면서
        # QWidget을 QMainWindow 안쪽으로 하나 만들고
        # 그 안으로 QVBoxLayout 을 놓고 그 안에 원하는 위젯 배치하기

        widget = QWidget()
        layout = QVBoxLayout(widget)
        layout.addWidget(btn1)
        layout.addWidget(btn2)
        self.setCentralWidget(widget)

        # QWidgte->QVBoxLayout->QPushButton

if __name__ == "__main__":
    app = QApplication(sys.argv)
    win = MyWindow()
    win.show()
    app.exec_()


# https://www.acmicpc.net/problem/14502
import sys
from collections import deque
import copy

input = sys.stdin.readline

def dfs(depth, si, sj):
    global answer
    if depth == 3:
        arr = copy.deepcopy(board)
        q = copy.deepcopy(dq)
        d = [(1, 0), (-1, 0), (0, 1), (0, -1)]
        while q:
            x, y = q.popleft()
            for dx, dy in d:
                nx = x + dx
                ny = y + dy
                if -1 < nx < N and -1 < ny < M:
                    if arr[nx][ny] == 0:
                        q.append((nx, ny))
                        arr[nx][ny] = 2
        tmp = 0
        for a in arr:
            tmp += a.count(0)
        answer = max(answer, tmp)
        return
    for i in range(si, N):
        if i > si:
            sj = 0
        for j in range(sj, M):
            if board[i][j] == 0:
                board[i][j] = 1
                dfs(depth + 1, i, j)
                board[i][j] = 0

N, M = map(int, input().split())
board = [list(map(int, input().split())) for _ in range(N)]
dq = deque()
for i in range(N):
    for j in range(M):
        if board[i][j] == 2:
            dq.append((i, j))
answer = 0
dfs(0, 0, 0)
print(answer)


import csv, os

file_to_load = os.path.join("Resources", "budget_data.csv")
file_to_output = os.path.join("Analysis", "budget_analysis")

total_months = 0
preview_revenue = 0
total_revenue = 0
months_of_change = []
revenue_change_list = []
greatest_increase = ["", 0]
greatest_decrease = ["", 9999999999999]

with open(file_to_load) as revenue_data:
    reader = csv.DictReader(revenue_data)

    for row in reader:
        total_months = total_months + 1
        total_revenue = int(row["Profit/Losses"])


from django.urls import path

from users.views import *

app_name = 'users'
urlpatterns = [
    path('register/', RegisterView.as_view(), name='register'),  # 注册
    path('active/<slug:active_code>', ActiveView.as_view(), name='active'),  # 激活
    path('login/', LoginView.as_view(), name='login'),  # 登陆
    path('logout/', LogoutView.as_view(), name='logout'),  # 登出
    path('forget_pwd/', ForgetPwdView.as_view(), name='forget_pwd'),  # 忘记密码
    path('find/<slug:find_code>', ResetView.as_view(), name='find'),  # 从邮箱点击到找回密码页面
    path('new_pwd/', NewPwdView.as_view(), name='new_pwd'),  # 修改密码
    path('userinfo/', UserInfoView.as_view(), name='userinfo'),  # 个人中心
    path('userinfo/update', UpdateUserInfoView.as_view(), name='update_userinfo'),  # 修改个人信息
    path('userinfo/update_pwd', UpdateUserPwdView.as_view(), name='update_user_pwd'),  # 修改个人信息
]


import sublime
import sublime_plugin
import xml.etree.ElementTree as ET
import xml.sax.saxutils as SAX

settings = sublime.load_settings("FileMaker.sublime-settings")

def get_selection(view, select_all=False):
    """ Returns regions of view to operate on
    """
    if select_all and not view.has_non_empty_selection_region():
        selection = [sublime.Region(0, view.size())]
    else:
        selection = view.sel()
    return selection

def syntax_to_filemaker(self):
    """Changes syntax to FileMaker if its in plain text
    """
    if "Plain text" in self.view.settings().get('syntax'):
        self.view.set_syntax_file("Packages/FileMaker/FileMaker.tmLanguage")

def syntax_to_clipboard(self):
    """Changes syntax to FileMaker Clipboard
    """
    self.view.set_syntax_file("Packages/FileMaker/FileMaker Clipboard.tmLanguage")

def quote(text):
    text = text.replace('\\', '\\\\')
    text = text.replace('"', '\\"')
    text = text.replace('¶', '\\¶')
    return '"' + text + '"'

def quote_and_append(text):
    textNew = ""
    for line in text.splitlines():
        textNew += quote(line) + ' & ¶ &' + "\n"
    return textNew + "\n" + '""'

def extract_table_field_names(text):
    root = ET.fromstring(text)
    textNew = ""
    for elem in root.iterfind('Field'):
      textNew += elem.get('name') + "\n"
    return textNew

def convert_to_script_comments(text):
    template = '<Step enable="True" id="" name="Comment"><Text>{0}</Text></Step>'
    textNew = '<fmxmlsnippet type="FMObjectList">' + "\n"
    for line in text.splitlines():
        lineNew = SAX.escape(line)
        textNew += template.format(' ' + lineNew + "\n")
    textNew += '</fmxmlsnippet>'
    return textNew

class FilemakerQuoteCommand(sublime_plugin.TextCommand):
    """ Quotes calculation as FileMaker string, escaping as necessary
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, quote(view.substr(sel)))

        if select_all:
            self.syntax_to_filemaker()

class FilemakerQuoteAndAppendCommand(sublime_plugin.TextCommand):
    """ Quotes each line in calculation as FileMaker string, appending each line of text with carriage return
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, quote_and_append(view.substr(sel)))

        if select_all:
            self.syntax_to_filemaker()

class FilemakerExtractTableFieldNamesCommand(sublime_plugin.TextCommand):
    """ Extracts list of field names from FileMaker clipboard object
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, extract_table_field_names(view.substr(sel)))

        if select_all:
            extract_table_field_names(view.substr(selection))

class FilemakerConvertToScriptComments(sublime_plugin.TextCommand):
    """ Converts each line of text to a script comment to be pasted into FileMaker
    """
    def run(self, edit):
        view = self.view
        select_all = settings.get("use_entire_file_if_no_selection", True)
        selection = get_selection(view, select_all)
        for sel in selection:
            view.replace(edit, sel, convert_to_script_comments(view.substr(sel)))

        if select_all:
            self.syntax_to_clipboard()


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.5
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
from astropy.table import Table
import matplotlib.pyplot as plt
from matplotlib import rc
rc('font', **{'size':18})
# %matplotlib inline
from multiprocessing import Pool
# from profilestats import profile
from scipy.stats import binned_statistic
from module.classes.analysis import *
from os import listdir
import os
import time
from module.preprocessing.binning import bin_data
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

# +
wdir = cfg.W_DIR
band = 'r'

# t_max w/o SSA = 6751 for qsos
# t_max w/o SSA = 7772 for stars
# USE cfg.MAX_DT_REST_FRAME
# config = {'obj':'qsos'       ,'ID':'uid'  ,'t_max':23576,'n_bins_t':200,'n_bins_m':200,'n_t_chunk':19, 'width':2, 'steepness':0.005}
config = {'obj':'calibStars','ID':'uid_s','t_max':25600,'n_bins_t':200,'n_bins_m':200,'n_t_chunk':19, 'width':1, 'steepness':0.005}

width   = config['width']
steepness = config['steepness']
obj = config['obj']
ID  = config['ID']
t_max = config['t_max']
n_bins_t = config['n_bins_t']
n_bins_m = config['n_bins_m']
n_t_chunk = config['n_t_chunk']

data_path = cfg.D_DIR + 'computed/{}/dtdm/raw/{}/'.format(obj,band)

# sort based on filesize, then do ordered shuffle so that each core recieves the same number of large files
fnames = [a for a in listdir(data_path) if (len(a)>27)]
size=[]
for file in fnames:
    size.append(os.path.getsize(data_path+file))

fnames = [name for i in [0,1,2,3] for sizename, name in sorted(zip(size, fnames))[i::4]]

# +
import pandas as pd
pd.options.mode.chained_assignment = None
import numpy as np
from astropy.table import Table
import matplotlib.pyplot as plt
import matplotlib
from multiprocessing import Pool
# from profilestats import profile
from scipy.stats import binned_statistic
from module.classes.analysis import *
# %matplotlib inline

def reader(n_subarray):
    return pd.read_csv('../data/merged/{}/r_band/lc_{}.csv'.format(obj,n_subarray), comment='#', nrows=None, index_col = ID, dtype = {'catalogue': np.uint8, 'mag_ps': np.float32, 'magerr': np.float32, 'mjd': np.float64, ID: np.uint32})

# -

obj = 'calibStars'
ID  = 'uid_s'
band = 'r'
redshift_bool = False
dr = analysis(ID, obj)

# dr.read_in(reader, redshift=redshift_bool)
dr.band = band
dr.group(keys = ['uid_s'],read_in=True, redshift=redshift_bool)

fpath = data_path + fnames[5]
df = pd.read_csv(fpath, index_col = ID, dtype = {ID: np.uint32, 'dt': np.float32, 'dm': np.float32, 'de': np.float32, 'cat': np.uint8});

# +
# # combining several dtdm files into one. Takes a little while to run.
# # We have already run and saved this
# df_list = []
# for fname in fnames:
#     df = pd.read_csv(fpath, index_col = ID, dtype = {ID: np.uint32, 'dt': np.float32, 'dm': np.float32, 'de': np.float32, 'cat': np.uint8});
#     df = df[df['cat']==2]
#     df_list.append(df)
# df = pd.concat(df_list)
# -

# load in dtdm ps ssa pairwise observations
df = pd.read_csv('transformations/ssa_error_calculation/dtdm_ps_ssa.csv', index_col='uid_s')
df = df.join(dr.df_grouped['mag_mean'], on='uid_s')

# plot the dm with with increasing mag slices
var = []
n=21
x = np.linspace(18.5, 21, n)
fig, ax = plt.subplots(n-1,1, figsize=(10,30), sharex=True)
intervals = [(x[i], x[i+1]) for i in range(len(x)-1)]
for i, interval in enumerate(intervals):
    lower, upper = interval
    subdf = df[(lower < df['mag_mean']) & (df['mag_mean'] < upper)]

    mean = subdf['dm'].mean()
    std  = subdf['dm'].std()

    z = (subdf['dm'] - mean)/std
    ax[i].axvline(x=mean + std*3)
    ax[i].axvline(x=mean - std*3)
    ax[i].axvline(x=0, color='k', ls='--')

    ax[i].hist(subdf['dm'], bins=100, label=str(interval))
    subdf = subdf[np.abs(z) < 3]
    var.append(subdf['dm'].std())
    ax[i].legend()

# plot the widths vs mag slices and save result
centres = (x[1:] + x[:-1])/2
plt.plot(centres, var)
np.savetxt('transformations/ssa_error_calculation/ps_ssa_mag_magerr.csv',np.array([centres,var]).T, delimiter = ',', fmt='%.5f')

# +
# df = df.join(dr.df_grouped['mag_mean'], on='uid_s')

# +
# df2=pd.read_csv('../data/surveys/ps/calibStars/gb.csv')
# df2 = df2[df2['filtercode'] == 'r']
# result = dr.df.groupby('uid_s').agg({'mag':'median','magerr':'median'})

# +
# var = []
# n=21
# intervals = [(x[i], x[i+1]) for i in range(len(x)-1)]
# for i, interval in enumerate(intervals):
#     lower, upper = interval
#     subdf = df[(lower < df['mag']) & (df['mag'] < upper)]
#     var.append(subdf['magerr'].mean())
# -

intervals = [(x[i], x[i+1]) for i in range(len(x)-1)]
centres = (x[1:] + x[:-1])/2

a = np.loadtxt('transformations/ssa_error_calculation/ps_mag_magerr.csv', delimiter=',')
b = np.loadtxt('transformations/ssa_error_calculation/ps_ssa_mag_magerr.csv', delimiter=',')

a

# +
ssa_magerr = np.sqrt(b[:,1]**2 - a[:,1]**2)
m, c = np.polyfit(centres, ssa_magerr, 1)
plt.plot(centres, ssa_magerr)
plt.plot(centres, m*centres+c)
m,c

# maybe start from 15th mag?


import requests
import re
import time
from bs4 import BeautifulSoup as bs
import execjs

PRE_SCRIPT_1 = '''

function abc111(){
var document = Object();
document.createElement = function () {
    return 0
}
document.getElementsByTagName = function (a) {
    return [document];
}
document.parentNode = Object();
document.parentNode.insertBefore = function (a,b) {
    return 0;
}'''

PRE_SCRIPT_2 = '''}'''

SCRIPT_PROCESS = '''
function _process(s) {
	var result = s.substring(4, 6).concat('m').concat(s.substring(13, 14)).concat('p').concat(s.substring(8, 12)).concat('u').concat(s.substring(4)).concat('z').concat(s.substring(10, 18));
	return result.substr(0, 32);
};
'''
# type in the SESSION after you login
# 写上你登陆之后的cookie里面的SESSION=的值
SESSION = 'MWQ4OWJiYWQtNTA3Mi00OWNjLWJjYWEtNWFhOGU0NTA3YjI0'
class openLaw():
    def __init__(self):
        self.s = requests.Session()
        self.Header = {'Cookie':'SESSION=MjQxODc0MWItZWRjNC00NWFhLWE4MmQtMDJkODJiYzE3ZDY3',
                       'User-Agent':"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36"}
        self._process = execjs.compile(SCRIPT_PROCESS)
        self.main()

    def main(self):
        for i in range(1,2):
            i = str(i)
            res1 = self.s.get('http://openlaw.cn/',headers=self.Header)
            print('res1 cookie = ',res1.cookies.get_dict())
            # 在这里修改你想要的日期，修改代码加循环等等...
            res = self.s.get('http://openlaw.cn/search/judgement/advanced?judgeDateBegin=2019-06-01&judgeDateEnd=2019-06-03&page='+i,headers=self.Header)
            content =res.content.decode('utf-8')
            print(content)
            print()
            if "如果你能看到这个" in content:
                self.jsProcess(content)
                content = self.s.get('http://openlaw.cn/search/judgement/advanced?judgeDateBegin=2019-06-01&judgeDateEnd=2019-06-03&page='+i,headers=self.Header).content.decode('utf-8')
            # 返回搜索页面，自己提取url再访问，解析即可
            print(content)
    def jsProcess(self,content):
        # handle the script and get anomynous function
        script = bs(content,'lxml').script.text
        flag = script.find("$")
        script1 = script[:flag]
        script2 = script[flag:]
        flag2 = script2.find('$.$($.$($.$')
        script2 = script2[:flag2] + 'return '+script2[flag2:][:-4]+'.toString();'
        finalScript = PRE_SCRIPT_1 + script1+script2 + PRE_SCRIPT_2
        ctx = execjs.compile(finalScript)
        result =ctx.call("abc111")
        print(result)
        #get Keys
        OPEN_S_INDEX = re.findall('OPEN_S=(.*?);',result)[0][-3:].replace('0','')
        print('OPEN_S_INDEX=',OPEN_S_INDEX)
        temp = re.findall('[a-z0-9]{32}',result)
        SIGNIN_ID = temp[0]
        SIGNIN_UC = temp[1]
        UNDEFINED = temp[2]
        print(temp)
        process = result[result.find('function _process'):result.find('var _switch')]

        OPEN_S = re.findall('[a-z0-9]{32}',script1)[eval(OPEN_S_INDEX)-1]
        OPEN_C = execjs.compile(process).call('_process',re.findall('[a-z0-9]{32}',script1)[eval(OPEN_S_INDEX)-1])
        print('OPEN_S = ',OPEN_S)
        print('OPEN_C = ',OPEN_C)
        # set Cookie
        self.Header['Cookie'] = 'SESSION=%s; OPEN_S=%s; OPEN_C=%s; SIGNIN_ID=%s; SIGNIN_UC=%s;UNDEFINED=%s; Hm_lvt_a105f5952beb915ade56722dc85adf05=1560915411,1560943092,1560944621,1561191343; Hm_lpvt_a105f5952beb915ade56722dc85adf05=1561192080' % (SESSION,OPEN_S,OPEN_C,SIGNIN_ID,SIGNIN_UC,UNDEFINED)
        self.Header['Referer'] = 'http://openlaw.cn/search/judgement/default?type=&typeValue=&lawyerId=&lawFirmId=&courtId=&keyword='
        self.Header['Host'] = 'openlaw.cn'
        print("更新后Header=",self.Header)

if __name__ == '__main__':
    e  = openLaw()


import os
import numpy as np
import glob
import cv2, json
import argparse
from sklearn.utils import shuffle
from keras.optimizers import SGD, Adam
from keras.models import Sequential
from keras.layers import Dropout, Dense
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator
from keras import applications
from keras.applications.inception_v3 import InceptionV3
from keras.applications.inception_v3 import preprocess_input
from keras.layers import LSTM
from keras.models import load_model, Model
from sequence_data_generator import FramesSeqGenerator, FeaturesSeqGenerator
from data_generator import DataGenerator
from keras import backend as K

# BATCH_SIZE = 32
# SEED = 42

def extract_cnn_features(raw_data_dir, features_dir):
    # base_model = InceptionV3(include_top=False, weights='imagenet', input_shape=(224, 224, 3))
    # print("Base model shape")
    # print(base_model.input_shape)

    model = InceptionV3(weights='imagenet', include_top=True)
    base_model = Model(inputs=model.input, outputs=model.get_layer('avg_pool').output)
    _, height, width, channels = base_model.input_shape

    frames = FramesSeqGenerator(raw_data_dir, BATCH_SIZE, 20, height, width, channels, False)
    for i, video in frames.videos_list.iterrows():
        video_name = video.frames_dir.split('/')[-1]
        label = video.label
        print(f'Video label = {label}')
        features_path = f'{features_dir}/{label}/{video_name}.npy'
        X, y = frames.generate(video)
        print(f'X shape = {X.shape}')
        features = base_model.predict(X)
        print(f'features shape = {features.shape}')

        if not os.path.exists(f'{features_dir}/{label}'):
            os.makedirs(f'{features_dir}/{label}')
        np.save(features_path, features)
    return

def extract_cnn_features_all(raw_data_dir, features_dir):
    model = InceptionV3(weights='imagenet', include_top=True)
    base_model = Model(inputs=model.input, outputs=model.get_layer('avg_pool').output)
    _, height, width, channels = base_model.input_shape

    print(base_model.input_shape)

    signs = sorted([name for name in os.listdir(raw_data_dir) if os.path.isdir(os.path.join(raw_data_dir, name))])
    for sign in signs:
        print("\nSign = {0}\n-------------------".format(sign))
        videos = sorted([name for name in os.listdir(os.path.join(raw_data_dir, sign)) if os.path.isdir(os.path.join(raw_data_dir, sign, name))])

        # For each video we will save the cnn features computed on each image
        for video in videos:
            print("Video = {0}".format(video))
            features_path = f'{features_dir}/{sign}/{video}.npy'
            if not os.path.exists(f'{features_dir}/{sign}'):
                os.makedirs(f'{features_dir}/{sign}')

            pathname = os.path.join(raw_data_dir, sign, video, '*.jpg')
            imagenames = sorted(glob.glob(pathname))
            featuresList = []
            for imagename in imagenames:
                img = image.load_img(imagename, target_size=(height, width))
                img_data = image.img_to_array(img)
                img_data = np.expand_dims(img_data, axis=0)
                img_data = preprocess_input(img_data)
                inceptionv3_feature = base_model.predict(img_data)
                featuresList.append(inceptionv3_feature)
            stackedFeatures = np.vstack(featuresList)
            np.save(features_path, stackedFeatures)
            print("{0} images processed\n".format(len(featuresList)))

def train_lstm(features_path, epochs, num_features, num_classess):
    model = Sequential()
    model.add(LSTM(num_features, dropout=0.2, input_shape=(20, num_features), return_sequences=True))
    model.add(LSTM(num_features * 1, return_sequences=False))
    model.add(Dense(num_classess, activation='softmax'))
    optimizer = Adam(lr=1e-4)
    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
    # callbacks = [EarlyStopping(monitor='val_loss', patience=10, verbose=0), ModelCheckpoint('video_1_LSTM_1_1024.h5', monitor='val_loss', save_best_only=True, verbose=0)]
    callbacks = [ModelCheckpoint('video_1_LSTM_1_1024.h5', monitor='val_loss', save_best_only=True, verbose=0)]

    train_features = FeaturesSeqGenerator(features_path + '/train', BATCH_SIZE, model.input_shape[1:])
    val_features = FeaturesSeqGenerator(features_path + '/test', BATCH_SIZE, model.input_shape[1:])
    model.fit_generator(generator=train_features,
            validation_data=val_features,
            epochs=int(epochs),
            workers=1,
            use_multiprocessing=False,
            verbose=1,
            callbacks=callbacks)
    return model

def train_lstm_jittered(features_path, sequencelength, epochs, num_features, num_classess):
    model = Sequential()
    model.add(LSTM(2048, return_sequences=False,input_shape=(sequencelength, num_features),dropout=0.5))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classess, activation='softmax'))
    optimizer = Adam(lr=1e-4)
    metrics = ['accuracy']
    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=metrics)

    callbacks = [ModelCheckpoint('video_LSTM.h5', monitor='val_loss', save_best_only=True, verbose=1)]

    data_path, batch_size, featureLength, shuffle = 'cnn_features/train', 32, 2048, True
    train_gen = DataGenerator(data_path,
                            batch_size=batch_size,
                            featureLength=featureLength,
                            seqLength=sequencelength,
                            shuffle=shuffle)

    data_path, batch_size, featureLength, shuffle = 'cnn_features/test', 16, 2048, False
    val_gen = DataGenerator(data_path,
                            batch_size=batch_size,
                            featureLength=featureLength,
                            seqLength=sequencelength,
                            shuffle=shuffle)

    model.fit_generator(generator=train_gen,
            validation_data=val_gen,
            epochs=int(epochs),
            workers=1,
            use_multiprocessing=False,
            verbose=1,
            callbacks=callbacks)
    return model

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Train CNN+LSTM model for short video snippets')

    parser.add_argument('--data-path', default='processed_data', required=False, help='path to the train and test directories')
    parser.add_argument('--lstm-epochs', default=100, required=False, help='number of LSTM epochs')
    parser.add_argument('--reload-cnn-features', default=True, required=False, help='re-extract CNN features from frames')
    parser.add_argument('--features-path', default='cnn_features', required=False, help='extracted CNN features path')
    args = vars(parser.parse_args())
    print(args)
    if args.get('data_path') is None:
        print('Please provide path to video frames')
        exit()

    # GPU configuration
    # K.tensorflow_backend._get_available_gpus()
    # config = tf.ConfigProto( device_count = {'GPU': 200, 'CPU': 4} )
    # sess = tf.Session(config=config)
    # keras.backend.set_session(sess)

    # extract_cnn_features(f'{args["data_path"]}/train', f'{args["features_path"]}/train')
    # extract_cnn_features(f'{args["data_path"]}/test', f'{args["features_path"]}/test')

    # extract_cnn_features_all(f'{args["data_path"]}/train', f'{args["features_path"]}/train')
    # extract_cnn_features_all(f'{args["data_path"]}/test', f'{args["features_path"]}/test')

    # model = train_lstm(args['features_path'], args['lstm_epochs'], num_features, num_classes)

    num_features = 2048
    num_classes = len(os.listdir(f'{args["features_path"]}/train'))
    seqLength = 10
    model = train_lstm_jittered(args['features_path'], seqLength, args['lstm_epochs'], num_features, num_classes)

    # model.save("final_model.h5")


"Storing Records Persistently"
'Using Shelves'
"""
pickling one record per file also obviates the need to load and store the entire database for each
update. If we really want keyed access to records, though, the Python standard library
offers an even higher-level tool: shelves.
"""

'''
Shelves automatically pickle objects to and from a keyed-access filesystem. They behave
much like dictionaries that must be opened, and they persist after each program exits.
Because they give us key-based access to stored records, there is no need to manually
manage one flat file per record—the shelve system automatically splits up stored records
and fetches and updates only those records that are accessed and changed. In
this way, shelves provide utility similar to per-record pickle files, but they are usually
easier to code.
'''
from initdata import bob, sue
import shelve

db = shelve.open('people-shelve')
db['bob'] = bob
db['sue'] = sue
db.close()

'''
This script creates one or more files in the current directory with the name peopleshelve
as a prefix (in Python 3.1 on Windows, people-shelve.bak, people-shelve.dat, and
people-shelve.dir). You shouldn’t delete these files (they are your database!), and you
should be sure to use the same base name in other scripts that access the shelve.
'''


# -------------------------------------------------------------------------------------------------------------------------------------
# Following code curated for GCPNet-EMA (https://github.com/BioinfoMachineLearning/GCPNet-EMA):
# -------------------------------------------------------------------------------------------------------------------------------------

import torch
from beartype import beartype
from beartype.typing import Union
from jaxtyping import Float
from torch.nn import functional as F

@beartype
def _normalize(tensor: torch.Tensor, dim: int = -1) -> torch.Tensor:
    """Normalizes a `torch.Tensor` along a given dimension `dim` by its L2 norm.

    From: https://github.com/drorlab/gvp-pytorch

    :param tensor: `torch.Tensor` to normalize.
    :param dim: Dimension along which to normalize.
    :return: Normalized `torch.Tensor`.
    """
    return torch.nan_to_num(torch.div(tensor, torch.norm(tensor, dim=dim, keepdim=True)))

@beartype
def _rbf(
    D: torch.Tensor,
    D_min: float = 0.0,
    D_max: float = 20.0,
    D_count: int = 16,
    device: Union[torch.device, str] = "cpu",
) -> torch.Tensor:
    """Returns an RBF embedding of `torch.Tensor` `D` along a new axis=-1. That is, if `D` has
    shape [...dims], then the returned tensor will have shape [...dims, D_count].

    From https://github.com/jingraham/neurips19-graph-protein-design

    :param D: `torch.Tensor` of pairwise distances.
    :param D_min: Minimum distance to consider.
    :param D_max: Maximum distance to consider.
    :param D_count: Number of RBFs to use.
    :param device: Device to use.
    :return: RBF embedding of `D`.
    """
    D_mu = torch.linspace(D_min, D_max, D_count, device=device)
    D_mu = D_mu.view([1, -1])
    D_sigma = (D_max - D_min) / D_count
    D_expand = torch.unsqueeze(D, -1)

    RBF = torch.exp(-(((D_expand - D_mu) / D_sigma) ** 2))
    return RBF

def _orientations(
    X: Float[torch.Tensor, "num_nodes 3"]  # noqa: F722
) -> Float[torch.Tensor, "num_nodes 2 3"]:  # noqa: F722
    """Orientations of edges in a graph.

    :param X: `torch.Tensor` of node coordinates.
    :return: `torch.Tensor` of edge orientations.
    """
    forward = _normalize(X[1:] - X[:-1])
    backward = _normalize(X[:-1] - X[1:])
    forward = F.pad(forward, [0, 0, 0, 1])
    backward = F.pad(backward, [0, 0, 1, 0])
    return torch.cat((forward.unsqueeze(-2), backward.unsqueeze(-2)), dim=-2)


#!/usr/bin/env python3.9
# -*- coding: utf-8 -*-
"""
Programme ou module pour ...

Créé le Fri Nov 26 15:36:57 2021

@author: ejetzer
"""

import datetime

import sqlalchemy as sqla

from sqlalchemy import Column, MetaData, Table
from sqlalchemy.orm import declarative_base

from ..outils.database.dtypes import get_type, column

metadata = MetaData()

def colonnes_communes():
    return (column('index', int, primary_key=True),)

cols = colonnes_communes() + (column('Technicien', str),
                              column('Payeur', str),
                              column('Date', datetime.datetime),
                              column('Description des travaux effectués', str),
                              column('Demandeur', str),
                              column('Heures', float),
                              column('Atelier', bool),
                              column('Précision si pour département', str),
                              column('Autres', str))
cols_admin = cols + (column('Facturé', bool),)

heures = Table('heures', metadata, *cols)
#admin = Table('admin', metadata, *cols_admin)


import numpy as np
import os
from cil.framework import AcquisitionData, AcquisitionGeometry, ImageData, ImageGeometry

h5pyAvailable = True
try:
    import h5py
except:
    h5pyAvailable = False

class NEXUSDataReader(object):

    """
    Create a reader for NeXus files.

    Parameters
    ----------
    file_name: str
        the full path to the NeXus file to read.
    """

    def __init__(self, file_name=None):

        self.file_name = file_name

        if self.file_name is not None:
            self.set_up(file_name = self.file_name)

    def set_up(self,
               file_name = None):
        """
        Initialise reader.

        Parameters
        ----------
        file_name : str
            Full path to NeXus file
        """

        self.file_name = os.path.abspath(file_name)

        # check that h5py library is installed
        if (h5pyAvailable == False):
            raise Exception('h5py is not available, cannot load NEXUS files.')

        if self.file_name == None:
            raise Exception('Path to nexus file is required.')

        # check if nexus file exists
        if not(os.path.isfile(self.file_name)):
            raise Exception('File\n {}\n does not exist.'.format(self.file_name))

        self._geometry = None

    def read_dimension_labels(self, attrs):
        dimension_labels = [None] * 4
        for k,v in attrs.items():
            if k in ['dim0', 'dim1', 'dim2' , 'dim3']:
                dimension_labels[int(k[3:])] = v

        # remove Nones
        dimension_labels = [i for i in dimension_labels if i]

        if len(dimension_labels) == 0:
            dimension_labels = None

        return dimension_labels

    def get_geometry(self):
        """
        Parse NEXUS file and return acquisition or reconstructed volume
        parameters, depending on file type.

        Returns
        -------
        AcquisitionGeometry or ImageGeometry
            Acquisition or reconstructed volume parameters. Exact type
            depends on file content.
        """

        with h5py.File(self.file_name,'r') as dfile:

            if np.string_(dfile.attrs['creator']) != np.string_('NEXUSDataWriter.py'):
                raise Exception('We can parse only files created by NEXUSDataWriter.py')

            ds_data = dfile['entry1/tomo_entry/data/data']

            if ds_data.attrs['data_type'] == 'ImageData':

                self._geometry = ImageGeometry(voxel_num_x = int(ds_data.attrs['voxel_num_x']),
                                                voxel_num_y = int(ds_data.attrs['voxel_num_y']),
                                                voxel_num_z = int(ds_data.attrs['voxel_num_z']),
                                                voxel_size_x = ds_data.attrs['voxel_size_x'],
                                                voxel_size_y = ds_data.attrs['voxel_size_y'],
                                                voxel_size_z = ds_data.attrs['voxel_size_z'],
                                                center_x = ds_data.attrs['center_x'],
                                                center_y = ds_data.attrs['center_y'],
                                                center_z = ds_data.attrs['center_z'],
                                                channels = ds_data.attrs['channels'])

                if ds_data.attrs.__contains__('channel_spacing') == True:
                    self._geometry.channel_spacing = ds_data.attrs['channel_spacing']

                # read the dimension_labels from dim{}
                dimension_labels = self.read_dimension_labels(ds_data.attrs)

            else:   # AcquisitionData
                if ds_data.attrs.__contains__('dist_source_center') or dfile['entry1/tomo_entry'].__contains__('config/source/position'):
                    geom_type = 'cone'
                else:
                    geom_type = 'parallel'

                if ds_data.attrs.__contains__('num_pixels_v'):
                    num_pixels_v = ds_data.attrs.get('num_pixels_v')
                elif ds_data.attrs.__contains__('pixel_num_v'):
                    num_pixels_v = ds_data.attrs.get('pixel_num_v')
                else:
                    num_pixels_v = 1

                if num_pixels_v > 1:
                    dim = 3
                else:
                    dim = 2

                if self.is_old_file_version():
                    num_pixels_h = ds_data.attrs.get('pixel_num_h', 1)
                    num_channels = ds_data.attrs['channels']
                    ds_angles = dfile['entry1/tomo_entry/data/rotation_angle']

                    if geom_type == 'cone' and dim == 3:
                        self._geometry = AcquisitionGeometry.create_Cone3D(source_position=[0, -ds_data.attrs['dist_source_center'], 0],
                                                                               detector_position=[0, ds_data.attrs['dist_center_detector'],0])
                    elif geom_type == 'cone' and dim == 2:
                        self._geometry = AcquisitionGeometry.create_Cone2D(source_position=[0, -ds_data.attrs['dist_source_center']],
                                                        detector_position=[0, ds_data.attrs['dist_center_detector']])
                    elif geom_type == 'parallel' and dim == 3:
                        self._geometry = AcquisitionGeometry.create_Parallel3D()
                    elif geom_type == 'parallel' and dim == 2:
                        self._geometry = AcquisitionGeometry.create_Parallel2D()

                else:
                    num_pixels_h = ds_data.attrs.get('num_pixels_h', 1)
                    num_channels = ds_data.attrs['num_channels']
                    ds_angles = dfile['entry1/tomo_entry/config/angles']

                    rotation_axis_position = list(dfile['entry1/tomo_entry/config/rotation_axis/position'])
                    detector_position = list(dfile['entry1/tomo_entry/config/detector/position'])

                    ds_detector = dfile['entry1/tomo_entry/config/detector']
                    if ds_detector.__contains__('direction_x'):
                        detector_direction_x = list(dfile['entry1/tomo_entry/config/detector/direction_x'])
                    else:
                        detector_direction_x = list(dfile['entry1/tomo_entry/config/detector/direction_row'])

                    if ds_detector.__contains__('direction_y'):
                        detector_direction_y = list(dfile['entry1/tomo_entry/config/detector/direction_y'])
                    elif ds_detector.__contains__('direction_col'):
                        detector_direction_y = list(dfile['entry1/tomo_entry/config/detector/direction_col'])

                    ds_rotate = dfile['entry1/tomo_entry/config/rotation_axis']
                    if ds_rotate.__contains__('direction'):
                        rotation_axis_direction = list(dfile['entry1/tomo_entry/config/rotation_axis/direction'])

                    if geom_type == 'cone':
                        source_position = list(dfile['entry1/tomo_entry/config/source/position'])

                        if dim == 2:
                            self._geometry = AcquisitionGeometry.create_Cone2D(source_position, detector_position, detector_direction_x, rotation_axis_position)
                        else:
                            self._geometry = AcquisitionGeometry.create_Cone3D(source_position,\
                                                detector_position, detector_direction_x, detector_direction_y,\
                                                rotation_axis_position, rotation_axis_direction)
                    else:
                        ray_direction = list(dfile['entry1/tomo_entry/config/ray/direction'])

                        if dim == 2:
                            self._geometry = AcquisitionGeometry.create_Parallel2D(ray_direction, detector_position, detector_direction_x, rotation_axis_position)
                        else:
                            self._geometry = AcquisitionGeometry.create_Parallel3D(ray_direction,\
                                                detector_position, detector_direction_x, detector_direction_y,\
                                                rotation_axis_position, rotation_axis_direction)

                # for all Aquisition data
                #set angles
                angles = list(ds_angles)
                angle_unit = ds_angles.attrs.get('angle_unit','degree')
                initial_angle = ds_angles.attrs.get('initial_angle',0)
                self._geometry.set_angles(angles, initial_angle=initial_angle, angle_unit=angle_unit)

                #set panel
                pixel_size_v = ds_data.attrs.get('pixel_size_v', ds_data.attrs['pixel_size_h'])
                origin = ds_data.attrs.get('panel_origin','bottom-left')
                self._geometry.set_panel((num_pixels_h, num_pixels_v),\
                                        pixel_size=(ds_data.attrs['pixel_size_h'], pixel_size_v),\
                                        origin=origin)

                # set channels
                self._geometry.set_channels(num_channels)

                dimension_labels = []
                dimension_labels = self.read_dimension_labels(ds_data.attrs)

        #set labels
        self._geometry.set_labels(dimension_labels)

        return self._geometry

    def get_data_scale(self):
        """
        Parse NEXUS file and return the scale factor applied to compress
        the dataset.

        Returns
        -------
        scale : float
            The scale factor applied to compress the dataset
        """

        with h5py.File(self.file_name,'r') as dfile:
            ds_data = dfile['entry1/tomo_entry/data/data']
            try:
                scale = ds_data.attrs['scale']
            except:
                scale = 1.0

        return scale

    def get_data_offset(self):
        """
        Parse NEXUS file and return the offset factor applied to compress
        the dataset.

        Returns
        -------
        offset : float
            The offset factor applied to compress the dataset
        """

        with h5py.File(self.file_name,'r') as dfile:
            ds_data = dfile['entry1/tomo_entry/data/data']
            try:
                offset = ds_data.attrs['offset']
            except:
                offset = 0.0

        return offset

    def __read_as(self, dtype=np.float32):
        """
        Parse NEXUS file and return raw file content.

        Parameters
        ----------
        dtype : data-type
            The data type used for storing the parsed data.

        Returns
        -------
        output : ImageData or AcquisitionData
            The parsed raw data. Exact type depends on file content.
        """

        if self._geometry is None:
            self.get_geometry()

        #allocate data container as requested type
        output = self._geometry.allocate(None, dtype=dtype)

        with h5py.File(self.file_name,'r') as dfile:

            ds_data = dfile['entry1/tomo_entry/data/data']
            ds_data.read_direct(output.array)

        return output

    def read_as_original(self):
        """
        Returns the compressed data from the file.

        Returns
        -------
        output : ImageData or AcquisitionData
            The raw, compressed data. Exact type depends on file content.
        """

        with h5py.File(self.file_name,'r') as dfile:
            ds_data = dfile['entry1/tomo_entry/data/data']
            dtype = ds_data.dtype

        return self.__read_as(dtype)

    def read(self):
        """
        Returns the uncompressed data as numpy.float32.

        Returns
        -------
        output : ImageData or AcquisitionData
            The uncompressed data. Exact type depends on file content.
        """

        output = self.__read_as(np.float32)
        scale = self.get_data_scale()
        offset = self.get_data_offset()

        if offset != 0:
            output -= offset
        if scale != 1:
            output /= scale

        return output

    def load_data(self):
        """
        Alias of `read`.

        See Also
        --------
        read
        """

        return self.read()

    def is_old_file_version(self):
        #return ds_data.attrs.__contains__('geom_type')
        with h5py.File(self.file_name,'r') as dfile:

            if np.string_(dfile.attrs['creator']) != np.string_('NEXUSDataWriter.py'):
                raise Exception('We can parse only files created by NEXUSDataWriter.py')

            ds_data = dfile['entry1/tomo_entry/data/data']

            return 'geom_type' in ds_data.attrs.keys()
            # return True


import numpy as np
from scipy.special import erf as ERF
from scipy.stats import norm as NORM
try:
    from surrogate import MultiSurrogates
except:
    from .surrogate import MultiSurrogates
import _csupport as CS
from evoalgos.performance import FonsecaHyperVolume as FH
try:
    from BO_base import BayesianOptBase
except:
    from .BO_base import BayesianOptBase

class MultiSurrogate(BayesianOptBase):
    '''
    Base class for multi-surrogate approaches.
    '''

    def __init__(self, func, \
                    n_dim, n_obj, lower_bounds, upper_bounds, \
                    obj_sense=[-1, -1], args = (), kwargs={}, X=None, Y=None, \
                    kern=None, ref_vector=None):
        '''
        This constructor creates the multi-surrogate base class.

        Parameters.
        -----------
        func (method): the method to call for actual evaluations
        n_dim (int): number of decision space dimensions
        n_obj (int): number of obejctives
        lower_bounds (np.array): lower bounds per dimension in the decision space
        upper_boudns (np.array): upper bounds per dimension in the decision space
        obj_sense (np.array or list): whether to maximise or minimise each
                objective (ignore this as it was not tested).
                keys.
                    -1: minimisation
                     1: maximisation
        args (tuple): the required arguments for the objective function
        kwargs (dictionary): dictionary of keyword argumets to pass to the
                            objective function.
        X (np.array): initial set of decision vectors.
        Y (np.array): initial set of objective values (associated with X).
        kern (GPy kernel): kernel to be used with Gaussian process.
        ref_vector (np.array): reference vector in the objective space.
        '''
        super().__init__(func, n_dim, n_obj, lower_bounds, upper_bounds, \
                                obj_sense, args, kwargs, X, Y,\
                                ref_vector=ref_vector)
        self.kernels = kern
        self.hpv = FH(self.ref_vector)

    def get_toolbox(self, xtr, skwargs, cfunc=None, \
                        cargs=(), ckwargs={}, verbose=True):
        """
        Generate a DEAP toolbox for the infill criterion optimiser.

        Parameters.
        -----------
        xtr (np.array): traing decision vectors.
        skwargs (dict): options for infill criterion calculation; varies with
                        technique.
        cfunc (function): cheap constraint function.
        cargs (tuple): argumetns for constraint function.
        ckwargs (dict): keyword arguments for constraint function.
        verbose (bool): whether to print more comments.

        Returns a DEAP toolbox.
        """
        self.budget = skwargs.get('budget', 250)
        self.update(xtr) # update various parameters of multi-surrogates
        return self.init_deap(self.scalarise, cfunc=cfunc, cargs=cargs, \
                        ckwargs=ckwargs)

    def build_models(self, xtr, ytr, kernels):
        '''
        Build multiple surrogate models.

        Parameters.
        -----------
        xtr (np.array): training decision vectors.
        ytr (np.array): training objective vectors.
        kernels (list): GPy kernels to be used with multiple Gaussian processes.

        Returns multiple regression models.
        '''
        self.xtr = xtr
        self.ytr = ytr
        models = MultiSurrogates(xtr, ytr, kernels)
        return models

    def update(self, x_new):
        '''
        Update a range of setting for multi-surrogate.

        Parameters.
        -----------
        x_new (np.array): decision vector.

        Updates attributes, but returns nothing.
        '''
        # set/update xtr and ytr
        self.xtr = x_new
        self.ytr = self.m_obj_eval(x_new)
        assert self.xtr.shape[0] == self.ytr.shape[0] # check for smae amount of
                                                      # data
        yt, comp_mat = self.get_dom_matrix(self.ytr)
        self.comp_mat = comp_mat
        # update budget count
        self.b_count = self.budget - len(self.xtr) - 1
        # update + optimise models
        self.models = self.build_models(self.xtr, self.ytr, \
                            [kern.copy() for kern in self.kernels])
        # current pf
        self.pfr_inds = self.get_front(self.ytr, self.comp_mat)
        # current hv
        self.current_hv = self.current_hpv()
        # epsilon for sms-ego
        n_pfr = len(self.pfr_inds)
        c = 1 - (1/ 2**self.n_obj)
        self.epsilon = (np.max(self.ytr, axis=0) - np.min(self.ytr, axis=0))\
                        /(n_pfr + (c * self.b_count))

class MPoI(MultiSurrogate):
    '''
    Multi-Surrogate Minimum Probability of Improvement infill Criterion.
    '''

    def __init__(self, func, \
                    n_dim, n_obj, lower_bounds, upper_bounds, \
                    obj_sense=[-1, -1], args = (), kwargs={}, X=None, Y=None, \
                    kern=None, ref_vector=None):
        '''
        Simple constructor for invoking parent class.

        Parameters.
        -----------
        func (method): the method to call for actual evaluations
        n_dim (int): number of decision space dimensions
        n_obj (int): number of obejctives
        lower_bounds (np.array): lower bounds per dimension in the decision space
        upper_boudns (np.array): upper bounds per dimension in the decision space
        obj_sense (np.array or list): whether to maximise or minimise each
                objective (ignore this as it was not tested).
                keys.
                    -1: minimisation
                     1: maximisation
        args (tuple): the required arguments for the objective function
        kwargs (dictionary): dictionary of keyword argumets to pass to the
                            objective function.
        X (np.array): initial set of decision vectors.
        Y (np.array): initial set of objective values (associated with X).
        kern (GPy kernel): kernels to be used with Gaussian process.
        ref_vector (np.array): reference vector in the objective space.
        '''
        super().__init__(func, n_dim, n_obj, lower_bounds, upper_bounds, \
                                obj_sense, args, kwargs, X, Y,\
                                kern=kern, ref_vector=ref_vector)

    def scalarise(self, x, cfunc=None, cargs=(), ckwargs={}):
        '''
        Calculate the minimum probability of improvement compared to current
        Pareto front. Refer to the paper for full details.

        parameters:
        -----------
        x (np.array): decision vectors.
        cfunc (function): cheap constraint function.
        cargs (tuple): argument for constraint function.
        ckwargs (dict): keyword arguments for constraint function.

        Returns scalarised cost.
        '''
        n_sols = x.shape[0]
        if cfunc is not None:
            if not cfunc(x, *cargs, **ckwargs):
                return np.zeros((1,1))  # penalise for constraint violation
        yp, stdp = self.models.predict(x)
        y = self.ytr[self.pfr_inds]
        res = np.zeros((yp.shape[0], 1))
        sqrt2 = np.sqrt(2)
        for i in range(yp.shape[0]):
            m = (yp[i] - y)/(sqrt2 * stdp[i])
            pdom = 1 - np.prod(0.5 * (1 + ERF(m)), axis=1)
            res[i] = np.min(pdom)
        return res

class SMSEGO(MultiSurrogate):
    '''
    Multi-surrogate SMS-EGO. Note that this is inspired from GPareto package in
    R.
    '''

    def __init__(self, func, \
                    n_dim, n_obj, lower_bounds, upper_bounds, \
                    obj_sense=[-1, -1], args = (), kwargs={}, X=None, Y=None, \
                    kern=None, ref_vector=None):
        '''
        Simple constructor for invoking parent class.

        Parameters.
        -----------
        func (method): the method to call for actual evaluations
        n_dim (int): number of decision space dimensions
        n_obj (int): number of obejctives
        lower_bounds (np.array): lower bounds per dimension in the decision space
        upper_boudns (np.array): upper bounds per dimension in the decision space
        obj_sense (np.array or list): whether to maximise or minimise each
                objective (ignore this as it was not tested).
                keys.
                    -1: minimisation
                     1: maximisation
        args (tuple): the required arguments for the objective function
        kwargs (dictionary): dictionary of keyword argumets to pass to the
                            objective function.
        X (np.array): initial set of decision vectors.
        Y (np.array): initial set of objective values (associated with X).
        kern (GPy kernel): kernels to be used with Gaussian process.
        ref_vector (np.array): reference vector in the objective space.
        '''
        super().__init__(func, n_dim, n_obj, lower_bounds, upper_bounds, \
                                obj_sense, args, kwargs, X, Y,\
                                kern=kern, ref_vector=ref_vector)
        self.gain = - NORM.ppf(0.5 * (0.5**(1/self.n_obj)))

    def compare_add_solution(self, y, ytest, obj_sense):
        '''
        Compare and add a solution to the data set given its not dominated.

        Parameters.
        -----------
        y (np.array): current Pareto front objective vectors.
        y_test (np.array): candidate for adding to the archive.

        Returns latest Pareto front.
        '''
        result = np.ones(y.shape[0])
        for i in range(y.shape[0]):
            result[i] = CS.compare_solutions(y[i], ytest, self.obj_sense)
            if result[i] == 0:
                return y
        inds = np.where(result==3)[0]
        try:
            return np.concatenate([y[inds], ytest])
        except ValueError:
            print("Likely error in y: ", y[inds])
            return ytest

    def penalty(self, y, y_test):
        '''
        Penalty mechanism in the infill criterion. Penalise if dominated by the
        current front.

        Parameters.
        -----------
        y (np.array): current Pareto front elements.
        y_test (np.array): tentative solution.

        Returns a penalty value.
        '''
        yt = y_test - (self.epsilon * self.obj_sense)
        l = [-1 + np.prod(1 + y_test - y[i]) \
                if CS.compare_solutions(y[i], yt, self.obj_sense) == 0\
                else 0 for i in range(y.shape[0])]
        return (max([0, max(l)]))

    def scalarise(self, x, cfunc=None, cargs=(), ckwargs={}):
        '''
        S-metric infill criterion.

        parameters:
        -----------
        x (np.array): decision vectors.
        cfunc (function): cheap constraint function.
        cargs (tuple): argument for constraint function.
        ckwargs (dict): keyword arguments for constraint function.

        Returns scalarised cost.
        '''
        n_sols = len(x)
        if cfunc is not None:
            if not cfunc(x, *cargs, **ckwargs):
                return np.ones((1,1))*-100 # heavily penalise for constraint
                                           # violation
        # predictions
        yp, stdp = self.models.predict(x)
        # lower confidence bounds
        yl = yp - (self.gain * np.multiply(self.obj_sense, stdp))
        pen = self.penalty(self.ytr[self.pfr_inds], yl)
        if pen > 0:
            return np.array([-pen])
        # new front
        yn = self.compare_add_solution(self.ytr[self.pfr_inds], yl, self.obj_sense)
        return np.array([self.hpv.assess_non_dom_front(yn) - self.current_hv])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep  4 11:13:06 2019

@author: wanglab
"""

import os, numpy as np, pandas as pd, matplotlib.pyplot as plt
from tools.conv_net.utils.io import pairwise_distance_metrics, read_roi_zip

pth = "/jukebox/wang/zahra/kelly_cell_detection_analysis/comparison_to_clearmap/annotated_volumes"
dst = "/jukebox/wang/zahra/kelly_cell_detection_analysis/comparison_to_clearmap/results"

flds = [os.path.join(pth, "sweep/"+xx) for xx in os.listdir(os.path.join(pth, "sweep")) if "max" in xx] #only using the ones from the new sweep
roi_pths = [os.path.join(pth, xx) for xx in os.listdir(pth) if "RoiSet.zip" in xx]; roi_pths.sort() #the sorts here are important, for order of volumes

#these will be zyx
#note that importing it this way, the z dimension does not start from 0, but neither does clearmaps, so probably ok???
annotated_cells = np.array([np.array([[int(yy) for yy in xx[0].replace(".roi", "").split("-")] for xx in
                            read_roi_zip(roi_pth, include_roi_name=True)]) for roi_pth in roi_pths])

#voxel pair cutoff
cutoffs = [5,7,10]

for cutoff in cutoffs:
    #init dataframe
    print("cutoff: %s \n\n" % cutoff)
    df = pd.DataFrame()
    df["parameters"] = [os.path.basename(xx) for xx in flds]

    df["tp_f37104_demonstrator"] = np.zeros(len(df))
    df["tp_m37079_mouse2"] = np.zeros(len(df))
    df["tp_f37077_observer"] = np.zeros(len(df))

    df["tp_f37104_demonstrator"] = np.zeros(len(df))
    df["fp_m37079_mouse2"] = np.zeros(len(df))
    df["fp_f37077_observer"] = np.zeros(len(df))

    df["fn_f37104_demonstrator"] = np.zeros(len(df))
    df["fn_m37079_mouse2"] = np.zeros(len(df))
    df["fn_f37077_observer"] = np.zeros(len(df))

    df["f1_f37104_demonstrator"] = np.zeros(len(df))
    df["f1_m37079_mouse2"] = np.zeros(len(df))
    df["f1_f37077_observer"] = np.zeros(len(df))

    for n, fld in enumerate(flds):
        if n%100 == 0: print(n)
        arr_pths = [os.path.join(fld, xx) for xx in os.listdir(fld) if not os.path.isdir(os.path.join(fld, xx))]; arr_pths.sort() #the sorts here are important, for order of volumes
        detected_cells = np.asarray([np.load(arr_pth) for arr_pth in arr_pths])
        measures = [pairwise_distance_metrics(annotated_cells[i], detected_cells[i], cutoff = cutoff, verbose = False) for i
                    in range(len(arr_pths))]

        for m, measure in enumerate(measures):
            paired, tp, fp, fn = measure
            try:
                precision = tp/(tp+fp)
                recall = tp/(tp+fn) #calculating precision and recall
                f1 = 2*( (precision*recall)/(precision+recall) ) #calculating f1 score
            except Exception as e:
                print(e)
                f1 = np.nan #if tp, fn, etc. are 0
            if m == 0: #these will always be in this order, hence saving this way, easier than doin this auto with the long filenames
                df.loc[df.parameters == os.path.basename(fld), "f1_f37104_demonstrator"] = f1
                df.loc[df.parameters == os.path.basename(fld), "tp_f37104_demonstrator"] = tp
                df.loc[df.parameters == os.path.basename(fld), "fp_f37104_demonstrator"] = fp
                df.loc[df.parameters == os.path.basename(fld), "fn_f37104_demonstrator"] = fn
            if m == 1:
                df.loc[df.parameters == os.path.basename(fld), "f1_m37079_mouse2"] = f1
                df.loc[df.parameters == os.path.basename(fld), "tp_m37079_mouse2"] = tp
                df.loc[df.parameters == os.path.basename(fld), "fp_m37079_mouse2"] = fp
                df.loc[df.parameters == os.path.basename(fld), "fn_m37079_mouse2"] = fn
            if m == 2:
                df.loc[df.parameters == os.path.basename(fld), "f1_f37077_observer"] = f1
                df.loc[df.parameters == os.path.basename(fld), "tp_f37077_observer"] = tp
                df.loc[df.parameters == os.path.basename(fld), "fp_f37077_observer"] = fp
                df.loc[df.parameters == os.path.basename(fld), "fn_f37077_observer"] = fn

    df.to_csv(os.path.join(dst, "clearmap_performance_measures_kelly_cfos_voxelcutoff_%02d_v2.csv" % cutoff))


import Scraper
import csv

games=[]
tags = []

with open('SteamGameLinks.csv') as file:
    reader = csv.reader(file, delimiter=',')

    counter = 0
    for row in reader:
        games.append(Scraper.Game(row[0], row[1]))

    for game in games:
        game.printVals()
        if(game.intrested):
            tags.append(game.tags)
        print('- - - - - - - - - - - - - - - - - - - - - - - - ')

    topTags = Scraper.CombineListsToDict(tags)
    print('\nYour favorite tags: ', topTags)


import os.path
import random

class DB():

	def __init__(self):
		self.doc_name_path = None
		self.root = os.path.abspath('./')
		self.word = []
		self.trans = []
		self.random_number = 0

	def set_path(self,path):
		self.doc_name_path= path
		return 0

	def init_words(self):
		# current_doc_name = self.root + "\\data\\" + str(self.doc_pointer) + ".txt"
		current_doc_name = self.doc_name_path

		f = open(current_doc_name,"r",encoding='utf-8')
		lines = f.readlines()
		f.close()

		for line in lines:
			line = line.strip("\n")
			line = line.split("-")
			self.word.append(line[0])
			self.trans.append(line[1])
		return self.word,self.trans

	def write_in(self,in_word,in_trans):
		current_doc_name = self.doc_name_path
		# current_doc_name = self.root + "\\data\\" + str(self.doc_pointer) + ".txt"

		f = open(current_doc_name,"a+",encoding='utf-8')
		f.write("\n"+in_word + '-'+in_trans)
		f.close()

		return 0

	def random_pick(self):
		r_index = random.randint(0,len(self.word)-1)
		self.random_number = r_index
		return self.word[r_index],self.trans[r_index]


def permutate1(l, s='', cand=[]):
    ll = len(l) - 1
    if l != []:
        for n in range(len(l)):
            ns = s
            ul = l[ : ]
            ns += ul[ll - n]
            ul.remove(ul[ll - n])
            cand.append(ns)
            permutate1(ul, ns)
            if ul == []:
                cand.append(ns)
    return cand
dict = open(r'/Users/pel/Documents/python projects/dict.en', 'r')
stripped_dict = []
poss_plays = []
words = dict.readlines()
for word in words:
    stripped_dict.append(word.strip('\n'))
print(stripped_dict)

#INSERT YOUR LETTERS HERE!
letters = ['d', 'e', 'm', 'o', 'n', 'a']

candidates = permutate1(letters)

for word in candidates:
    if word in stripped_dict:
        poss_plays.append(word)
print(poss_plays)


"""
Collect a set of tweets based on keywords and save them to a specified file
"""

import sys
import tweepy
import json
import signal

from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream

consumer_key = ''
consumer_secret = ''
access_token = ''
access_token_secret = ''

# This is global so it can be access in the ctrl-c interrupt
tweet_writer = None

class TweetStreamWriter:
    def __init__(self, output_file):
        self.data = []
        self.output_file = output_file

    def add_tweet(self, tweet_data):
        self.data.append(tweet_data)

    def store_twitter_data(self):
        with open(self.output_file, 'w') as data_file:
            first_item = True
            data_file.write('[')

            for item in self.data:
                if first_item:
                    first_item = False
                else:
                    data_file.write(',')
                data_file.write(item)

            data_file.write(']')

        self.data = []

    def cleanup(self):
        # Write the extra data if there is any
        if self.data:
            self.store_twitter_data()

class TwitterStreamReader(StreamListener):
    def __init__(self, tweet_aggregator, max_tweets):
        self.tweet_aggregator = tweet_aggregator
        self.count = 0
        self.max_tweets = max_tweets

    def on_data(self, data):
        self.tweet_aggregator.add_tweet(data)
        self.count += 1
        return (self.count < self.max_tweets)

    def on_error(self, status):
        print(status)

def collect_tweets(output_file, max_tweets, keywords):
    def signal_handler(signal, frame):
        global tweet_writer
        tweet_writer.cleanup()
        sys.exit(0)

    # Register a signal handler for SIGINT
    signal.signal(signal.SIGINT, signal_handler)

    global tweet_writer
    tweet_writer = TweetStreamWriter(output_file)
    stream_reader = TwitterStreamReader(tweet_writer, max_tweets)

    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, stream_reader)
    stream.filter(track=keywords)
    tweet_writer.cleanup()

if __name__ == '__main__':
    if len(sys.argv) < 4:
        print('Usage: python stream.py output_file max_tweets keywords...')
        exit()
    collect_tweets(sys.argv[1], int(sys.argv[2]), sys.argv[3:])


import sys
from heapq import heappop, heappush

def dijkstra(start):
    dist = [inf for _ in range(n + 1)]
    dist[start] = 0
    heap = []
    heappush(heap, [0, start])
    while heap:
        we, nu = heappop(heap)
        for n_n, weight in adj[nu]:
            n_w = weight + we
            if n_w < dist[n_n]:
                dist[n_n] = n_w
                heappush(heap, [n_w, n_n])

    return dist

inf = 2000 * 50000 + 1
T = int(sys.stdin.readline())
for ___ in range(T):
    n, m, t = map(int, sys.stdin.readline().split())
    s, g, h = map(int, sys.stdin.readline().split())
    adj = [[] for _ in range(n + 1)]
    for i in range(m):
        a, b, d = map(int, sys.stdin.readline().split())
        adj[a].append([b, d])
        adj[b].append([a, d])
        if a == g and b == h:
            leng = d
        if a == h and b == g:
            leng = d
    destination = []
    for i in range(t):
        destination.append(int(sys.stdin.readline()))
    answer = []

    for i in destination:
        ans_ = dijkstra(s)
        g_ = dijkstra(g)
        h_ = dijkstra(h)
        if ans_[g] + leng + h_[i] == ans_[i] or ans_[h] + leng + g_[i] == ans_[i]:
            answer.append(i)

    answer.sort()
    for i in range(len(answer)):
        answer[i] = str(answer[i])
    print(' '.join(answer))


from typing import Dict, Tuple

import tensorflow as tf

from mlptts import MLPTextToSpeech

class TrainWrapper:
    """MLP-TTS train wrapper.
    """
    def __init__(self, model: MLPTextToSpeech):
        """Initializer.
        Args:
            model: target model.
        """
        self.model = model

    def forward(self, *args, **kwargs):
        """Forward to the model.
        """
        return self.model(*args, **kwargs)

    def apply_gradient(self,
                       optim: tf.keras.optimizers.Optimizer,
                       text: tf.Tensor,
                       textlen: tf.Tensor,
                       mel: tf.Tensor,
                       mellen: tf.Tensor) -> Tuple[
            tf.Tensor, Dict[str, tf.Tensor], Dict[str, tf.Tensor]]:
        """Apply gradients.
        Args:
            optim: optimizer.
            text: [tf.int32; [B, S]], text tokens.
            textlen: [tf.int32; [B]], length of the text sequence.
            mel: [tf.float32; [B, T, mel]], ground-truth mel-spectrogram.
            mellen: [tf.int32; [B]], length of the mel-spectrogram.
        Returns:
            loss: [tf.float32; []], loss values.
            losses: {key: [tf.float32; []]}, individual loss values.
            aux: {key: tf.Tensor}, auxiliary outputs.
                  attn: [tf.float32; [B, T, S]], attention alignment.
                  mel: [tf.float32; [B, T, mel]], generated mel.
                  mellen: [tf.float32; [B]], length of the mel-spectrogram
                  gradnorm: gradient norm.
                  paramnorm: parameter norm.
        """
        with tf.GradientTape() as tape:
            loss, losses, aux = self.compute_loss(text, textlen, mel, mellen)
        # compute gradient
        grad = tape.gradient(loss, self.model.trainable_variables)
        # update
        optim.apply_gradients(zip(grad, self.model.trainable_variables))
        # norms
        gradnorm = tf.reduce_mean([
            tf.norm(g) for g in grad if g is not None])
        paramnorm = tf.reduce_mean([
            tf.norm(p) for p in self.model.trainable_variables])
        return loss, losses, {
            'gradnorm': gradnorm.numpy().item(),
            'paramnorm': paramnorm.numpy().item(), **aux}

    def compute_loss(self,
                     text: tf.Tensor,
                     textlen: tf.Tensor,
                     mel: tf.Tensor,
                     mellen: tf.Tensor) -> Tuple[
            tf.Tensor, Dict[str, tf.Tensor], Dict[str, tf.Tensor]]:
        """Compute loss of mlp-tts.
        Args:
            text: [tf.int32; [B, S]], text tokens.
            textlen: [tf.int32; [B]], length of the text sequence.
            mel: [tf.float32; [B, T, mel]], ground-truth mel-spectrogram.
            mellen: [tf.int32; [B]], length of the mel-spectrogram.
        Returns:
            loss: [tf.float32; []], loss values.
            losses: {key: [tf.float32; []]}, individual loss values.
            aux: {key: tf.Tensor}, auxiliary outputs.
                  attn: [tf.float32; [B, T, S]], attention alignment.
                  mel: [tf.float32; [B, T, mel]], generated mel.
                  mellen: [tf.float32; [B]], length of the mel-spectrogram.
        """
        pred, _, aux = self.model(text, textlen, mel=mel, mellen=mellen)
        # [B, S]
        text_mask = aux['mask'][:, 0]

        ## 1. Mel-spectrogram prediction loss
        # [B], [B]
        mellen, textlen = tf.cast(mellen, tf.float32), tf.cast(textlen, tf.float32)
        # [], l1-loss
        melloss = tf.reduce_mean(
            tf.reduce_sum(tf.abs(pred - mel), axis=1) / mellen[:, None])

        ## 2. Duration loss
        # [B, S]
        logdur = tf.math.log(tf.maximum(tf.reduce_sum(aux['attn'], axis=1), 1.))
        # [B, S]
        predur = tf.math.log(tf.maximum(aux['durations'], 1.))
        # []
        durloss = tf.reduce_mean(
            tf.reduce_sum(tf.square(logdur - predur), axis=1) / textlen)
        # []
        ctcloss = -tf.reduce_mean(aux['ctc'])

        ## 3. Residual latent
        # [B, S, C]
        dkl = self.nll(aux['latent']) - self.nll(aux['latent'], aux['mu'], aux['sigma'])
        # []
        dkl = tf.reduce_mean(
            tf.reduce_sum(dkl * text_mask[..., None], axis=1) / textlen[:, None])
        # []
        loss = melloss + self.model.config.ctc_lambda * ctcloss + durloss + dkl
        losses = {'melloss': melloss, 'ctcloss': ctcloss, 'durloss': durloss, 'dkl': dkl}
        return loss, losses, {
            'attn': aux['attn'], 'mel': pred, 'mellen': tf.cast(mellen, tf.int32)}

    def nll(self, inputs: tf.Tensor, mean: tf.Tensor = 0., stddev: tf.Tensor = 1.) -> tf.Tensor:
        """Gaussian negative log-likelihood.
        Args:
            inputs: [tf.float32; [...]], input tensor.
            mean: [tf.float32; [...]], mean.
            stddev: [tf.float32; [...]], standard deviation.
        Returns:
            [tf.float32; [...]], likelihood.
        """
        logstd = tf.math.log(tf.maximum(stddev, 1e-5))
        return 2 * logstd + tf.exp(-2 * logstd) * tf.square(inputs - mean)


import numpy as np

def getCurvilinearCoordinates(state, pt_ref):

    x, y, psi = state[0], state[1], state[2]
    x_ref, y_ref, psi_ref, distance_along = pt_ref[0], pt_ref[1], pt_ref[2], pt_ref[3]

    cross_track_error = -(x-x_ref)*np.sin(psi_ref) + (y-y_ref)*np.cos(psi_ref)
    heading_error = psi-psi_ref

    return np.array([cross_track_error, heading_error])

def getHMatrix(state, pt_ref):

    x, y, psi = state[0], state[1], state[2]
    xc , yc = 5, 0

    R = 5

    return np.array([[-np.sin(pt_ref[2]), np.cos(pt_ref[2]), 0], [0, 0, 1]])


#!/usr/bin/env python

import argparse
from Bio import SeqIO

parser = argparse.ArgumentParser(
    description=('Find motifs in sequences in a FASTA file.'))

parser.add_argument(
    'filename',
    help='The alignment file.')

parser.add_argument(
    '--verbose', default=False, action='store_true',
    help='If given, print extra info.')

parser.add_argument(
    '--target', required=True,
    help='The target to look for in the sequences.')

parser.add_argument(
    '--limit', type=int,
    help='The maximum number of sequences to process.')

parser.add_argument(
    '--start', type=int, default=0,
    help='The maximum number of sequences to process.')

parser.add_argument(
    '--stop', type=int,
    help='The maximum number of sequences to process.')

parser.add_argument(
    '--mismatch', type=int, default=0,
    help='The maximum number of sequences to process.')

parser.add_argument(
    '--step', type=int, default=1,
    help='The maximum number of sequences to process.')

args = parser.parse_args()

if args.start < 0:
    print('Error: start negative!')
    exit()

if args.stop is not None:
    if args.stop < 0:
        print('Error: stop negative!')
        exit()

    if args.start > args.stop:
        print('Error: start bigger than stop!')
        exit()

def countMismatches_v1(string1, string2):
    mismatches = 0

    for char1, char2 in zip(string1, string2):
        if char1 != char2:
            mismatches += 1

    return mismatches

def countMismatches(string1, string2):
    return sum((char1 != char2) for char1, char2 in zip(string1, string2))

def findMatches(subseq, target, targetLen, maxMismatches=0):
    for index in range(len(subseq) - targetLen):
        subsubseq = subseq[index:index+targetLen]
        distance = countMismatches(target, subsubseq)
        if distance <= maxMismatches:
            yield (index, subsubseq, distance)

recordCount = 0
target = str(args.target)
targetLen = len(target)

for record in SeqIO.parse(args.filename, 'fasta'):
    subseq = str(record.seq[args.start:args.stop])

    print(record.id)

    for (position, subsubseq, distance) in findMatches(
            subseq, target, targetLen, args.mismatch):
        print('  ', position, subsubseq, distance)

    recordCount += 1
    if recordCount == args.limit:
        break


#!/usr/bin/env python
# coding: utf-8

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import Model, layers
from einops import rearrange, repeat
from einops.layers.tensorflow import Rearrange, Reduce

class MLPBlock(layers.Layer):
    def __init__(self, mixing, d_ff):
        super(MLPBlock, self).__init__()
        if mixing != 'token' and mixing != 'channel':
            raise ValueError("undefiend mixing")
        self.mixing = mixing
        self.d_ff = d_ff

    def build(self, input_shape):
        d_out = input_shape[-1] if self.mixing=='channel' else input_shape[-2]
        self.dense_1 = layers.Dense(self.d_ff, use_bias=False)
        self.dense_2 = layers.Dense(d_out, use_bias=False)
        self.norm = layers.LayerNormalization(epsilon=1e-6)

    def call(self, inputs):
        '''
        inputsは(batches, tokens, channels)
        Denseは最後の軸に作用
        '''
        x = self.norm(inputs)
        x = x if self.mixing!='token' else tf.transpose(x, perm=[0, 2, 1])
        x =self.dense_1(x)
        x = tf.keras.activations.gelu(x, approximate=True)
        x = self.dense_2(x)
        x = x if self.mixing!='token' else tf.transpose(x, perm=[0, 2, 1])
        return x + inputs

class MixerBlock(layers.Layer):
    def __init__(self, d_s, d_c):
        super(MixerBlock, self).__init__()
        self.mlp_1 = MLPBlock(mixing='token', d_ff=d_s)
        self.mlp_2 = MLPBlock(mixing='channel', d_ff=d_c)

    def call(self, inputs):
        x = self.mlp_1(inputs)
        x = self.mlp_2(x)
        return x

#StemがDenseModel
def MLP_Mixer1_build(**config):
    shapes = (config['image_size'], config['image_size'], config['channels'])
    hidden_dim  = config['hidden_dim']
    patch_size = config['patch_size']
    num_blocks = config['depth']
    tokens_mlp_dim = config['tokens_mlp_dim']
    channels_mlp_dim = config['channels_mlp_dim']
    num_classes = config['num_classes']
    size = (config['image_size']//config['patch_size'])  #4

    inputs = layers.Input(shape=shapes)
    x = Rearrange('b (h x) (w y) c -> b (h w) (x y c)', x=size, y=size)(inputs) #(B, 64, 48)
    x = layers.Dense(hidden_dim, use_bias=False)(x)

    for i in range(num_blocks):
      x = MixerBlock(tokens_mlp_dim, channels_mlp_dim)(x)

    x = layers.GlobalMaxPooling1D()(x)
    x = layers.LayerNormalization(epsilon=1e-6)(x)
    outputs = layers.Dense(num_classes, activation="softmax")(x)

    return Model(inputs, outputs)

#StemがConvModel
def MLP_Mixer2_build(**config):
    shapes = (config['image_size'], config['image_size'], config['channels'])
    hidden_dim  = config['hidden_dim']
    patch_size = config['patch_size']
    num_blocks = config['depth']
    tokens_mlp_dim = config['tokens_mlp_dim']
    channels_mlp_dim = config['channels_mlp_dim']
    num_classes = config['num_classes']
    size = config['image_size']//config['patch_size']

    inputs = layers.Input(shape=shapes)
    x = layers.Conv2D(hidden_dim, (size, size), strides=(size, size))(inputs)
    x = Rearrange('n h w c -> n (h w) c')(x)

    for _ in range(num_blocks):
      x = MixerBlock(tokens_mlp_dim, channels_mlp_dim)(x)

    x = layers.GlobalMaxPooling1D()(x)
    x = layers.LayerNormalization(epsilon=1e-6)(x)
    outputs = layers.Dense(num_classes, activation="softmax")(x)

    return Model(inputs, outputs)


"""
모든 연도 재무제표 연산 필요 (연도별로 12시간, 총 72시간 이상 소요 예상)
기업재무 제표를 조회(?), 조건에 따라 기업명 가져오기
조건 : 안정성, 성장성, 수익성.
판단 기준:
이슈, 화제성, 주도업종에서(업종코드 활용) 종목 가져오기
각 비율지표 시계열 분석, 연도별 시총증가율 등등. 사용자가 입력한 기준으로 쿼리(연도 지정, 각 비율 커트라인 수치). 기준을 더 세세하게 추가할 필요가 있음

가져올 기업명 개수 설정
가져온 기업명을 토대로 해당 기업들의 주가에 현대 포트폴리오 이론 적용(?)

아니면 무작위의 모든 종목을 대입해서 리스크가 적고 가장큰 이익을 내는 알고리즘은 어떨까?
포트폴리오 이론은 과거 데이터를 기반으로 계산한건데 이익을 내는데 효과가 있는지 의문 스럽지만 지금까지 계산 해본 결과 손해는 없었다. 다만 수익률은 조금 미미했다.(100만원 투자 기준)
아니면 단타 전략을 짜는게 더 낫지 않을까? 그렇다면 지금까지 얻은 데이터로 어떻게 단타 전략을 짤 수 있을까
현대 포트폴리오 이론은 수익률 보다 안정성에 더 중점을 둔 이론이다.(손실을 보지 않기 위한 종목별 비중을 달리둔 포트폴리오) 그렇기 때문에 어떤 종목이 더 오를지에 대한 분석이 아니다.

과거의 데이터로 포트폴리오를 시뮬레이션해보는 정도다. 미래가치는 알 수 없음.
"""

"""
샤프 지수 = (포트폴리오 기대 수익률 - 무위자산 수익률) / (포트폴리오 수익률의 표준편차)
*계산의 편의를 고려해 무위험률을 0으로 가정했으며, 샤프 지수는 포트폴리오의 예상 수익률을 수익률의 표준편차로 나누어서 구했다. *
예를 들어 예상 수익률이 7%이고 수익률의 표준편차가 5%인 경우, 샤프 지수는 7 / 5 = 1.4 가 된다. 샤프 지수가 높을수록 위험에 대한 보상이 더 크다.
"""
"""
효율적 투자선 이론을 개량한것으로, 위험 단위당 수익률이 가장 높은 포트폴리오는 Sharpe값이 가장 큰 행을 찾으면 되고,
가장 안전한 포트폴리오는 Risk값이 가장 작은 행을 찾으면 된다.
"""

# 이걸 어떻게 웹에서 사용자와 상호작용 시키지?
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from REST_API.DB.StockPriceSearcher import Market as mkdb

class PortfolioSelector:
    def __init__(self):
        self.mk = mkdb()
        self.stocks_name_list = ['NICE평가정보', '미스터블루', '고려제약']  # ex) '삼성전자', 'SK하이닉스', '현대자동차', 'NAVER'
        self.start_date = '2020-01-02'  # 당연한거지만 언제 사서 언제 파는냐에 따라 수익률이 다르다
        self.end_date = '2021-07-02'
        # 사용자가 입력해야할 예상 투자금
        self.investment = 1000000  # 투자금액이 너무 적으면 투자분산이 되지 않는다. (종목 선별 함수에 투자금액에 맞는 종목을 필터링 하는 조건을 추가시켜볼까?)
        # 랜덤하게 생성할 포트폴리오 개수
        self.portfolio_count = 30000

        # 비중 계산용
        self.df = pd.DataFrame()
        # 주가 계산용
        self.df_price = pd.DataFrame()

        # 총 예상 수익을 담을 리스트
        self.total_revenue = []
        # 총 투자금액을 담을 리스트
        self.total_investment = []

        # 실행 함수
        self.portfolio_revenue()

    # 종목 선별 함수 : 그나마 중요한건 이건데...
    def portfolio_select(self):
        try:
            #
            stocks_name_list = []
            return stocks_name_list
        except:
            return

    # 선별된 종목으로 포트폴리오 비중 계산 : 과거 데이터로 시뮬레이션 돌려보는 정도 밖에 안된다.
    def portfolio_revenue(self):
        try:
            for s in self.stocks_name_list:
                # 액면분할,유/무상 증자의 전후 주가가 뒤섞이면 수익률 계산이 어렵다. 따라서 액면분할의 영향을 피해서 검색해야한다.
                # ex) s, '2016-01-04', '2018-04-27')['close']
                self.df[s] = self.mk.get_daily_price(s, self.start_date, self.end_date)['close']

            # 종목의 수익률을 비교하려면 종가 대신 일간 변동률로 비교를 해야 하기 때문에
            # 데이터프레임에서 제공하는 pct_change() 함수를 사용해 종목들의 일간 변동률을 구한다.
            daily_ret = self.df.pct_change()  # 일간 수익률 daily_ret

            # 일간 변동률의 평균값에 252를 곱해서 연간 수익률을 구한다. 252는 미국의 1년 평균 개장일로, 우리나라 실정에 맞게 다른 숫자로 바꾸어도 무방하다.
            annual_ret = daily_ret.mean() * 248  # 연간 수익률 annual_ret (2020년 기준 248일 개장)

            # 일간 리스크는 cov() 함수를 사용해 일간 변동률의 공분산으로 구한다.
            daily_cov = daily_ret.cov()  # 일간 리스크 daily_cov

            # 연간 공분산은 일간 공분상에 252를 곱해 계산한다.
            annual_cov = daily_cov * 248  # 연간 리스크 annual_cov

            # 종목 비중을 다르게 해 포트폴리오 20,000개를 생성한다.
            # 포트폴리오 수익률, 리스크, 종목 비중을 저장할 각 리스트를 생성한다.
            port_ret = []  # 포트폴리오 수익률,
            port_risk = []  # 포트폴리오 리스크,
            port_weights = []  # 포트폴리오 목 비중
            sharpe_ratio = []  # 샤프지수 리스트

            #  매우 많은 난수를 이용해 함수의 값을 확률적으로 계산하는 것을 몬테카를로 시뮬레이션이라고 한다.
            #  다음은 몬테카를로 시뮬레이션을 이용해 포트폴리오 20,000개를 생성한 후 각각의 포트폴리오별로 수익률, 리스크, 종목 비중을 데이터프레임으로 구하는 코드다.
            for _ in range(self.portfolio_count):  # 포트폴리오 20,000개를 생성하는데 range() 함수와 for in 구문을 사용했다. for in 구문에서 반복횟수를 사용할 일이 없으면 관습적으로 _ 변수에 할당한다.
                weights = np.random.random(len(self.stocks_name_list))  # 4개의 랜덤 숫자로 구성된 배열을 생성한다.
                weights /= np.sum(weights)  # 위에서 구한 4개의 랜덤 숫자를 랜덤 숫자의 총합으로 나눠 4종목 비중의 합이 1이 되도록 조정한다.

                returns = np.dot(weights,
                                 annual_ret)  # 랜덤하게 생성한 <종목별 비중 배열>과 <종목별 연간 수익률>을 곱해 해당 포트폴리오 전체 수익률(returns)을 구한다.

                # 종목별 연간 공분산과 종목별 비중 배열을 곱한 뒤 이를 다시 종목별 비중의 전치로 곱한다.
                # 이렇게 구한 결괏값의 제곱근을 sqrt() 함수로 구하면 해당 포트폴리오 전체 리스크를 구할 수 있다. (포트폴리오 리스크 공식 참고)
                risk = np.sqrt(np.dot(weights.T, np.dot(annual_cov, weights)))

                # 포트폴리오 20,000개 수익률, 리스크, 종목별 비중을 각각 리스트에 추가한다.
                port_ret.append(returns)  # 수익률
                port_risk.append(risk)  # 리스크
                port_weights.append(weights)  # 종목별 비중
                sharpe_ratio.append(
                    returns / risk)  # *포트폴리오의 수익률을 리스크로 나눈 값을 샤프 지수 리스트에 추가한다.* 제대로된 계산을 하기 위해선 공부를 좀 더 해야함

            portfolio = {'Returns': port_ret, 'Risk': port_risk, 'Sharpe': sharpe_ratio}
            for i, s in enumerate(
                    self.stocks_name_list):  # i값은 0,1,2,3 순으로 변한다. 이 때 s값은 '삼성전자', 'SK하이닉스', '현대자동차', 'NAVER'순으로 변한다.
                portfolio[s] = [weights[i] for weights in
                                port_weights]  # portfolio 딕셔너리에 '삼성전자', 'SK하이닉스', '현대자동차', 'NAVER' 키 순서로 비중값을 추가한다.
            df = pd.DataFrame(portfolio)

            # 최종 생성된 df 데이터프레임을 출력하면,
            # 시총 상위 4 종목의 보유 비율에 따라 포트폴리오 20,000개가 각기 다른 리스크와 예상 수익률을 가지는 것을 확인할 수 있다.
            df = df[['Returns', 'Risk', 'Sharpe'] + [s for s in self.stocks_name_list]]  # 샤프 지수 칼럼을 데이터프레임에 추가한다.
            max_sharpe = df.loc[df['Sharpe'] == df['Sharpe'].max()]  # 샤프 지수 칼럼에서 샤프 지수값이 제일 큰 행을 max_sharpe로 정한다.
            min_risk = df.loc[df['Risk'] == df['Risk'].min()]  # 샤프 지수 칼럼에서 리스크랎이 제일 작은 행을 min_risk로 정한다.

            # max_sharpe와 min_sharpe의 포트폴리오 비중 수치표시
            # print('max_sharpe : *')
            # print(max_sharpe)
            # # print(max_sharpe['삼성전자'])
            # print('')
            #
            # print('min_sharpe : X')
            # print(min_risk)
            # print('')

            for s in self.stocks_name_list:
                # 종목별 투자금 비율계산 : min_risk(손실최소화)보다 max_sharpe(이익극대화)가 수익률과 안정성이 더 좋을수도 있다.
                stock_investment = int(self.investment * max_sharpe[s].values[0])
                # 투자금 할당률
                investment_ratio = int(round(max_sharpe[s].values[0], 2)* 100)
                self.df_price[s] = self.mk.get_daily_price(s, self.start_date, self.end_date)['close']

                # 종목별 투자금으로 매입할수 있는 종목갯수
                amount = int(stock_investment / self.df_price[s].values[0])
                # print(f"{df_price[s].name} 매입가: {df_price[s].values[0]} 할당된 투자금 : {stock_investment} 매입수: {amount}")
                buy = self.df_price[s].values[0] * amount
                sale = self.df_price[s].values[-1] * amount
                revenue = sale - buy

                print(
                    f"[{self.df_price[s].name}] 할당된 투자금: {stock_investment} ({investment_ratio}%), 매수 종가:{self.df_price[s].values[0]}, 매도 종가: {self.df_price[s].values[-1]}, 매입개수:{amount}, 매수 : {buy}, 매도 : {sale}, "
                    f"수익 : {revenue}")

                self.total_investment.append(buy)
                self.total_revenue.append(revenue)

            print('')
            print(f"투자기간 : {self.start_date} ~ {self.end_date}")
            print(f"투자 전 예치금 : {self.investment}")
            print(f"총 투자금 : {sum(self.total_investment)}")
            print(f"예상 수익금 : {sum(self.total_revenue)}")
            print(f"투자 후 예상 예치금 : {self.investment + sum(self.total_revenue)}")

            """
            df 데이터프레임을 산점도로 출력하면, 몬테카를로 시뮬레이션으로 생성한 효율적 투자선을 눈으로 확인할 수 있다.
            scatter() 함수를 호출할 때 x축 값으로 해당 포트폴리오의 리스크, y축 값으로 예상 수익률을 넘겨주면 된다.
            """

            # # 포트폴리오의 샤프 지수에 따라 컬러맵을 'viridis' 로 표시하고 테두리는 검정(k)으로 표시한다.
            # df.plot.scatter(x='Risk', y='Returns', c='Sharpe', cmap='viridis', edgecolors='k', figsize=(11, 7),
            #                 grid=True)
            # plt.scatter(x=max_sharpe['Risk'], y=max_sharpe['Returns'], c='r', marker='*',
            #             s=300)  # 샤프 지수가 가장 큰 포트폴리오를 300 크기의 붉은 별로 표시한다.
            # plt.scatter(x=min_risk['Risk'], y=min_risk['Returns'], c='r', marker='X',
            #             s=200)  # 리스크가 제일 작은 포트폴리오를 200 크기의 붉은 엑스표로 표시한다.
            # plt.title('Portfolio Optimization')
            # plt.xlabel('Risk')
            # plt.ylabel('Expected Returns')
            # plt.show()
            return
        except:
            pass

if __name__ == '__main__':
    execution = PortfolioSelector()