"""
Created on 9 Oct 2015

@author: Filip Lindau
"""

import numpy as np
import Raytracer.Ray as Ray
import matplotlib.pyplot as mpl
import logging

class RaySource(object):
    def __init__(self, num_rays=1):
        """ Defines a ray source with a set of rays that can be traced.
        Contains a list of rays.

        Input
        numRays: Number of rays in the list
        """
        self.num_rays = num_rays
        self.rays = None
        self.ray_store_list = list()
        self.logger = logging.getLogger("RaySource.")
        self.logger.setLevel(logging.INFO)

    def generate_rays(self):
        """ Reimplement in subclass
        """
        pass

    def draw_rays(self, fig):
        for ray in self.rays:
            ray.drawRay(fig)

    def get_ray_points(self):
        """ Get list of points including color to draw
        """
        points = list()
        for ray_store in self.ray_store_list:
            points.append((ray_store.getArrayPoints(), ray_store.color))
        return points

    def get_rays_pos_array(self):
        """ Get array with ray coordinates. Used for computing.
        """
        return np.array([rl.x for rl in self.ray_store_list])

    def get_rays_time_on_surface(self, surface_number):
        """ Get array of group arrival time for the selected surface.
        Use -1 for last surface.
        """
        return np.array([rl.time[surface_number] for rl in self.ray_store_list])

class Collimated1DSource(RaySource):
    def __init__(self, num_rays=1000, x_dim=1e-3, l=567e-9, W=1, color=(0, 0, 0.85)):
        """ Defines an 1D ray source with a set of rays that can be traced. The rays
        are collimated in the z direction and initialized along the x axis with
        an extent of xDim.
        Contains a list of rays.

        Input
        numRays: Number of rays in the list
        xDim: The rays are evenly spaced around -xDim/2, +xDim/2
        l: Wavelength
        W: Energy
        color: Drawing color of rays
        """
        super(Collimated1DSource, self).__init__(num_rays)
        self.color = color
        self.generate_rays(x_dim, l, W, color)

    def generate_rays(self, x_dim=5e-3, l=567e-9, W=1, color=(0, 0, 0.85)):
        """ Generate rays matrix and a list of RayStores. A RayStore keeps track
        of a ray as it moves through the optical system.

        rays is 3d matrix:
        index0: ray number
        index1: data type (0=x, 1=xp, 2=optical data)
        index2: 4 element data vector

        The optical data is encoded as (wavelength, n, ng, W)

        Examples:
        rays[10, 0, :]... position vector of ray 10
        rays[ :, 1, :]... direction vectors of all rays
        rays[15, 2, 1]... current refractive index for ray 15
        """
        xx = np.linspace(-x_dim / 2, x_dim / 2, self.num_rays)
        x = np.column_stack((xx, np.zeros(self.num_rays), np.zeros(self.num_rays), np.ones(self.num_rays)))
        xp = np.column_stack((np.zeros(self.num_rays), np.zeros(self.num_rays), np.ones(self.num_rays), np.zeros(self.num_rays)))
        data = np.column_stack((l * np.ones(self.num_rays), 1.0 * np.ones(self.num_rays), 1.0 * np.ones(self.num_rays), W * np.ones(self.num_rays)))
        self.rays = np.dstack((x, xp, data)).swapaxes(1, 2)
        self.ray_store_list = list()
        for rn in range(self.num_rays):
            x = self.rays[rn, 0, :]
            xp = self.rays[rn, 1, :]
            l = self.rays[rn, 2, 0]
            n = self.rays[rn, 2, 1]
            W = self.rays[rn, 2, 3]
            r = Ray.RayStore(x=x, xp=xp, n=n, l=l, W=W, color=color)
            self.ray_store_list.append(r)

    def update_rays(self, new_rays):
        """ Update ray data with new rays (in global coordinates)
        Sets rays matrix to newRays matrix and updates the RayStoresList
        """
        d = np.sqrt(np.sum((new_rays[:, 0, 0:3] - self.rays[:, 0, 0:3]) ** 2, 1))
        self.rays = new_rays
        for rn in range(self.num_rays):
            x = self.rays[rn, 0, :]
            xp = self.rays[rn, 1, :]
            l = self.rays[rn, 2, 0]
            n = self.rays[rn, 2, 1]
            ng = self.rays[rn, 2, 2]
            W = self.rays[rn, 2, 3]
            self.ray_store_list[rn].addPos(newX=x, newXp=xp, newN=n, newNg=ng, W=W, distance=d[rn])


import torch
from model import Net
from load_data import (
    data,
    train_dl,
    valid_dl,
    auc_dl,
    all_dl,
    test_dl
)
from utils import (
    display_data,
    init_weights,
    train,
    plot_history,
    plot_auc_curve,
    save_results
)
from parameters import (
    LABELS_MAP,
    DEVICE,
    OPTIMIZER,
    LEARNING_RATE,
    N_EPOCHS,
    LOSS_FN,
    RESULTS_PATH
)

display_data(data, classes=LABELS_MAP)

model = Net().to(DEVICE)
model.apply(init_weights)

print('\nTraining... (with validation)\n')

optimizer = OPTIMIZER(model.parameters(),
                      lr=LEARNING_RATE)
history = train(model,
                train_dl,
                valid_dl,
                optimizer=optimizer,
                lr=LEARNING_RATE,
                epochs=N_EPOCHS,
                loss_fn=LOSS_FN)

plot_history(history, validation=True)

plot_auc_curve(model, auc_dl)

model_ = Net().to(DEVICE)
model_.apply(init_weights)

print('\nTraining... (without validation)\n')

optimizer = OPTIMIZER(model_.parameters(),
                      lr=LEARNING_RATE)
history = train(model_,
                all_dl,
                optimizer=optimizer,
                lr=LEARNING_RATE,
                epochs=N_EPOCHS,
                loss_fn=LOSS_FN)

plot_history(history)

torch.save(model_, './model.pth')

print('\nSaving results...')

save_results(RESULTS_PATH, test_dl, model_)


# Import packages
import numpy as np

import json
import torch
from torch import nn, optim
from torchvision import datasets, transforms, models

from workspace_utils import active_session
from helper_functions import train_parser, load_dataset, define_transforms
from model_functions import DeepNetworkClassifier, train, test

# Parse arguments from command line
arg_dict = train_parser()

# Load the data
data_directory = arg_dict['data_dir']
train_dir, valid_dir, test_dir = load_dataset(data_directory)

# Define transforms
train_transforms = define_transforms(train=True)
validate_transforms = define_transforms(train=False)
test_transforms = define_transforms(train=False)

# Load the datasets with ImageFolder
train_dataset = datasets.ImageFolder(train_dir, transform=train_transforms)
validate_dataset = datasets.ImageFolder(valid_dir, transform=validate_transforms)
test_dataset = datasets.ImageFolder(test_dir, transform=test_transforms)

# Using the image datasets and the transforms, define the dataloaders
trainloader = torch.utils.data.DataLoader(train_dataset, batch_size = 32, shuffle=True)
validloader = torch.utils.data.DataLoader(validate_dataset, batch_size = 32, shuffle=True)
testloader = torch.utils.data.DataLoader(test_dataset, batch_size = 32, shuffle=True)

# Create a dictionary of torchvision models to choose from
print('Downloading models...')
arch_dict = {'vgg19_bn': models.vgg19_bn(pretrained=True),
             'alexnet': models.alexnet(pretrained=True)}

# Create a dictionary of input sizes for each of the available torchvision models
arch_input_dict = {'vgg19_bn': 25088,
                   'alexnet': 9216}

arch=arg_dict['arch']

# Create an instance of the user-specified pretrained network
model = arch_dict[arch]

# Identify the device on which the model will be trained
GPU = arg_dict['gpu']
if GPU:
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

# Turn off gradient calculations for model parameters
for param in model.parameters():
    param.requires_grad = False

# Build the model classifier
hidden_units = arg_dict['hidden_units']
classifier = DeepNetworkClassifier(arch_input_dict[arch], 102, hidden_units)

model.classifier = classifier

# Add the NLLLoss criterion (since LogSoftmax is called in the classifier)
criterion = nn.NLLLoss()

# Define the optimizer
learning_rate = arg_dict['learning_rate']
optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate)

# Send the model to the device
model.to(device);

# Train the network
epochs = arg_dict['epochs']

if device == torch.device('cuda'):
    # Keep the workspace active
    with active_session():
        print('Begin training...')
        train(model, trainloader, validloader, criterion, optimizer, epochs, device)
else:
    print('Begin training...')
    train(model, trainloader, validloader, criterion, optimizer, epochs, device)

# Test the network
test(model, testloader, device)

# Save the checkpoint
checkpoint = {'arch':arch,
              'input_size': arch_input_dict[arch],
              'output_size': 102,
              'hidden_layers': hidden_units,
              'state_dict': model.state_dict(),
              'epochs': epochs,
              'optimizer_state_dict': optimizer.state_dict(),
              'class_to_idx':train_dataset.class_to_idx}

save_dir=arg_dict['save_dir']
torch.save(checkpoint, save_dir)
print('Model saved to', save_dir)


import numpy as np
import tensorflow as tf
import time
import sys
import os
from dialog_model import get_model_fn

sys.path.append('../vae')
from load_hparams import hparams
import easyfs

# Set up logging
tensors_to_log = {'loss': 'loss'}
logging_hook = tf.train.LoggingTensorHook(
        tensors=tensors_to_log, every_n_iter=10)

# Create the Estimator
model_fn = get_model_fn(hparams)
chkp_config = tf.estimator.RunConfig(keep_checkpoint_max=hparams['keep_checkpoint_max'])
estimator = tf.estimator.Estimator(
        model_fn=model_fn, model_dir=hparams['checkpoints_path'],
        config=chkp_config)

# so: source, a: answer
chunk_so_files = sorted([fname for fname in easyfs.onlyFiles(hparams['data_path']) if 'so_train' in fname])
chunk_a_files = sorted([fname for fname in easyfs.onlyFiles(hparams['data_path']) if 'a_train' in fname])

so_eval = np.load(hparams['data_path'] + 'so_dev0.npy')
a_eval = np.load(hparams['data_path'] + 'a_dev0.npy')

for epoch_i in range(hparams['num_epochs']):
    for so_fname, a_fname in zip(chunk_so_files, chunk_a_files):
        # Train one chunk
        start = time.time()
        so_data = np.load(hparams['data_path'] + so_fname)
        a_data = np.load(hparams['data_path'] + a_fname)
        print('Chunk:', so_fname, so_data.shape, a_fname)
        train_input_fn = tf.estimator.inputs.numpy_input_fn(
                x={'mu_sigma': so_data},
                y=a_data,
                batch_size=hparams['batch_size_train'],
                num_epochs=10,
                shuffle=True)
        estimator.train(
                input_fn=train_input_fn,
                hooks=[logging_hook])
        print('Chunk training took', time.time() - start, 'sec')

        # Evaluate the model and print results
        eval_input_fn = tf.estimator.inputs.numpy_input_fn(
                x={"mu_sigma": so_eval},
                y=a_eval,
                num_epochs=1,
                shuffle=False)
        eval_results = estimator.evaluate(input_fn=eval_input_fn)
        print(eval_results)


# -*- coding:UTF-8  -*-
"""
http://www.94xmn.com/
@author: hikaru
email: hikaru870806@hotmail.com
如有问题或建议请联系
"""
import os
import re
import time
import traceback
from pyquery import PyQuery as pq
from common import *

SUB_PATH_LIST = {
    "xiaoqingxin": "1",
    "xingganmeinv": "2",
    "siwameitui": "3",
    "ribenmeinv": "4",
    "hanguomeinv": "5",
    "xioumeinv": "6",
    "jiepai": "8",
}

# 获取指定一页的图集
def get_one_page_album(sub_path, page_count):
    album_pagination_url = "http://www.94xmn.com/%s/list_%s_%s.html" % (sub_path, SUB_PATH_LIST[sub_path], page_count)
    album_pagination_response = net.http_request(album_pagination_url, method="GET", header_list={"Host": "www.94xmn.com"})
    result = {
        "album_info_list": {},  # 全部图集信息
        "is_over": False,  # 是否最后一页图集
    }
    if album_pagination_response.status == 409:
        log.step("分页图集 %s 访问异常，重试" % album_pagination_url)
        time.sleep(5)
        return get_one_page_album(sub_path, page_count)
    elif album_pagination_response.status != net.HTTP_RETURN_CODE_SUCCEED:
        raise crawler.CrawlerException(crawler.request_failre(album_pagination_response.status))
    # 页面编码
    album_pagination_response_content = album_pagination_response.data.decode(errors="ignore")
    # 获取图集信息，存在两种页面样式
    album_list_selector = pq(album_pagination_response_content).find(".wf-main .wf-cld a")
    if album_list_selector.length == 0:
        raise crawler.CrawlerException("页面截取图集列表失败\n%s" % album_pagination_response_content)
    for album_index in range(0, album_list_selector.length):
        result_album_info = {
            "album_id": None,  # 图集id
            "album_title": "",  # 图集id
            "album_url": None,  # 图集地址
        }
        album_selector = album_list_selector.eq(album_index)
        # 获取图集地址
        album_url = album_selector.attr("href")
        if not album_url:
            raise crawler.CrawlerException("图集列表截取图集地址失败\n%s" % album_selector.html())
        result_album_info["album_url"] = album_url
        # 获取图集id
        # http://www.94xmn.com/plus/view.php?aid=25
        if album_url.find("/view.php?") > 0:
            album_id = album_url.split("aid=")[-1]
        else:
            album_id = album_url.split("/")[-1].split(".")[0]
        if not crawler.is_integer(album_id):
            raise crawler.CrawlerException("图集地址截取图集id失败\n%s" % album_url)
        result_album_info["album_id"] = int(album_id)
        # 获取图集标题
        album_title = album_selector.attr("title")
        if not album_title:
            raise crawler.CrawlerException("图集列表截取图集标题失败\n%s" % album_url)
        result_album_info["album_title"] = album_title
        result["album_info_list"][result_album_info["album_id"]] = result_album_info
    # 判断是不是最后一页
    max_page_count = pq(album_pagination_response_content).find("div.page li:last span.pageinfo strong:first").html()
    if not crawler.is_integer(max_page_count):
        raise crawler.CrawlerException("页面截取总页数失败\n%s" % album_pagination_response_content)
    result["is_over"] = page_count >= int(max_page_count)
    return result

# 获取指定图集
def get_album_page(album_url):
    page_count = max_page_count = 1
    result = {
        "photo_url_list": [],  # 全部图片地址
    }
    while page_count <= max_page_count:
        if page_count == 1:
            pagination_album_url = album_url
        else:
            pagination_album_url = album_url.replace(".html", "_%s.html" % page_count)
        album_response = net.http_request(pagination_album_url, method="GET", header_list={"Host": "www.94xmn.com"})
        if album_response.status == 409:
            log.step("图集分页 %s 访问异常，重试" % pagination_album_url)
            time.sleep(5)
            continue
        elif album_response.status != net.HTTP_RETURN_CODE_SUCCEED:
            raise crawler.CrawlerException("第%s页" % page_count + crawler.request_failre(album_response.status))
        album_response_content = album_response.data.decode(errors="ignore")
        # 获取图片地址
        photo_list_selector = pq(album_response_content).find("div.bbox a img")
        if photo_list_selector.length == 0:
            raise crawler.CrawlerException("第%s页页面匹配图片地址失败\n%s" % (page_count, album_response_content))
        for photo_index in range(0, photo_list_selector.length):
            result["photo_url_list"].append("http://www.94xmn.com" + photo_list_selector.eq(photo_index).attr("src"))
        # 获取总页数
        if page_count == 1:
            max_page_html = pq(album_response_content).find("#fenye li:first a").html()
            if not max_page_html:
                raise crawler.CrawlerException("页面匹配总页数信息失败\n%s" % album_response_content)
            max_page_count_find = re.findall("共(\d*)页: ", max_page_html)
            if len(max_page_count_find) == 0:
                raise crawler.CrawlerException("总页数信息匹配总页数失败\n%s" % album_response_content)
            max_page_count = int(max_page_count_find[0])
        page_count += 1
    return result

class Album(crawler.Crawler):
    def __init__(self, **kwargs):
        # 设置APP目录
        crawler.PROJECT_APP_PATH = os.path.abspath(os.path.dirname(__file__))

        # 初始化参数
        sys_config = {
            crawler.SYS_DOWNLOAD_PHOTO: True,
            crawler.SYS_NOT_CHECK_SAVE_DATA: True,
        }
        crawler.Crawler.__init__(self, sys_config, **kwargs)

        # 解析存档文件
        # sub_path  last_album_id
        self.account_list = crawler.read_save_data(self.save_data_path, 0, ["", "0"])
        for sub_path in SUB_PATH_LIST:
            if sub_path not in self.account_list:
                self.account_list[sub_path] = [sub_path, "0"]

    def main(self):
        # 循环下载每个id
        thread_list = []
        for sub_path in sorted(self.account_list.keys()):
            # 提前结束
            if not self.is_running():
                break

            # 开始下载
            thread = Download(self.account_list[sub_path], self)
            thread.start()
            thread_list.append(thread)

            time.sleep(1)

        # 等待子线程全部完成
        while len(thread_list) > 0:
            thread_list.pop().join()

        # 未完成的数据保存
        if len(self.account_list) > 0:
            file.write_file(tool.list_to_string(list(self.account_list.values())), self.temp_save_data_path)

        # 重新排序保存存档文件
        crawler.rewrite_save_file(self.temp_save_data_path, self.save_data_path)

        log.step("全部下载完毕，耗时%s秒，共计图片%s张" % (self.get_run_time(), self.total_photo_count))

class Download(crawler.DownloadThread):
    def __init__(self, account_info, main_thread):
        crawler.DownloadThread.__init__(self, account_info, main_thread)
        self.sub_path = self.account_info[0]
        self.display_name = self.sub_path
        self.step("开始")

    # 获取所有可下载图集
    def get_crawl_list(self):
        page_count = 1
        album_info_list = []
        temp_album_info_list = {}
        is_over = False
        # 获取全部图集
        while not is_over:
            self.main_thread_check()  # 检测主线程运行状态
            self.step("开始解析第%s页图集" % page_count)

            # 获取一页图集
            try:
                album_response = get_one_page_album(self.sub_path, page_count)
            except crawler.CrawlerException as e:
                self.error("第%s页图集解析失败，原因：%s" % (page_count, e.message))
                raise

            self.trace("第%s页解析的全部图集：%s" % (page_count, album_response["album_info_list"]))
            self.step("第%s页解析获取%s个图集" % (page_count, len(album_response["album_info_list"])))

            temp_album_info_list.update(album_response["album_info_list"])

            if album_response["is_over"]:
                is_over = True
            else:
                page_count += 1

        # 获取全部还未下载过需要解析的图集
        for album_id in sorted(temp_album_info_list.keys(), reverse=True):
            # 检查是否达到存档记录
            if album_id > int(self.account_info[1]):
                album_info_list.append(temp_album_info_list[album_id])
            else:
                break

        return album_info_list

    # 解析单个图集
    def crawl_album(self, album_info):
        self.step("开始解析图集%s %s" % (album_info["album_id"], album_info["album_url"]))

        # 获取图集全部图片
        try:
            album_response = get_album_page(album_info["album_url"])
        except crawler.CrawlerException as e:
            self.error("图集%s解析失败，原因：%s" % (album_info["album_id"], e.message))
            raise

        self.trace("图集%s《%s》 %s 解析的全部图片：%s" % (album_info["album_id"], album_info["album_title"], album_info["album_url"], album_response["photo_url_list"]))
        self.step("图集%s《%s》 %s 解析获取%s张图片" % (album_info["album_id"], album_info["album_title"], album_info["album_url"], len(album_response["photo_url_list"])))

        photo_index = 1
        # 过滤标题中不支持的字符
        album_title = path.filter_text(album_info["album_title"])
        if album_title:
            album_path = os.path.join(self.main_thread.photo_download_path, "%04d %s" % (album_info["album_id"], album_title))
        else:
            album_path = os.path.join(self.main_thread.photo_download_path, "%04d" % album_info["album_id"])
        # 设置临时目录
        self.temp_path_list.append(album_path)
        for photo_url in album_response["photo_url_list"]:
            self.main_thread_check()  # 检测主线程运行状态
            self.step("图集%s《%s》开始下载第%s张图片 %s" % (album_info["album_id"], album_info["album_title"], photo_index, photo_url))

            file_path = os.path.join(album_path, "%03d.%s" % (photo_index, net.get_file_type(photo_url)))
            save_file_return = net.save_net_file(photo_url, file_path)
            if save_file_return["status"] == 1:
                self.step("图集%s《%s》第%s张图片下载成功" % (album_info["album_id"], album_info["album_title"], photo_index))
                photo_index += 1
            else:
                self.error("图集%s《%s》 %s 第%s张图片 %s 下载失败，原因：%s" % (album_info["album_id"], album_info["album_title"], album_info["album_url"], photo_index, photo_url, crawler.download_failre(save_file_return["code"])))

        # 图集内图片全部下载完毕
        self.temp_path_list = []  # 临时目录设置清除
        self.total_photo_count += photo_index - 1  # 计数累加
        self.account_info[1] = str(album_info["album_id"])  # 设置存档记录

    def run(self):
        try:
            # 获取所有可下载图集
            album_info_list = self.get_crawl_list()
            self.step("需要下载的全部图集解析完毕，共%s个" % len(album_info_list))

            # 从最早的图集开始下载
            while len(album_info_list) > 0:
                self.crawl_album(album_info_list.pop())
                self.main_thread_check()  # 检测主线程运行状态
        except SystemExit as se:
            if se.code == 0:
                self.step("提前退出")
            else:
                self.error("异常退出")
            # 如果临时目录变量不为空，表示某个图集正在下载中，需要把下载了部分的内容给清理掉
            self.clean_temp_path()
        except Exception as e:
            self.error("未知异常")
            self.error(str(e) + "\n" + traceback.format_exc(), False)

        # 保存最后的信息
        with self.thread_lock:
            file.write_file("\t".join(self.account_info), self.main_thread.temp_save_data_path)
            self.main_thread.total_photo_count += self.total_photo_count
            self.main_thread.account_list.pop(self.sub_path)
        self.step("下载完毕，总共获得%s张图片" % self.total_photo_count)
        self.notify_main_thread()

if __name__ == "__main__":
    Album().main()


#importing necessary libraries
import pandas as pd
import numpy as np

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier

# visualization
import seaborn as sns
import matplotlib.pyplot as plt

#dataset
train_df = pd.read_csv('train.csv')
test_df = pd.read_csv('test.csv')
combine = [train_df, test_df]

print(train_df.columns.values)
train_df.head()
train_df.tail()

train_df.info()
train_df.describe()

train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False).mean().sort_values(by='Survived', ascending=False)

train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False).mean().sort_values(by='Survived', ascending=False)

train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False).mean().sort_values(by='Survived', ascending=False)

train_df[["Parch", "Survived"]].groupby(['Parch'], as_index=False).mean().sort_values(by='Survived', ascending=False)
#SibSp and Parch These features have zero correlation for certain values. It is best to derive a feature or a set of features from these individual features

#Correlating numerical features
g = sns.FacetGrid(train_df, col='Survived')
g.map(plt.hist, 'Age', bins=20)

#Correlating numerical and ordinal features
grid = sns.FacetGrid(train_df, col='Survived', row='Pclass', size=2.2, aspect=1.6)
grid.map(plt.hist, 'Age', alpha=.5, bins=20)
grid.add_legend();

#Correlating categorical features
grid = sns.FacetGrid(train_df, row='Embarked', size=2.2, aspect=1.6)
grid.map(sns.pointplot, 'Pclass', 'Survived', 'Sex', palette='deep')
grid.add_legend()

#Correlating categorical and numerical features
grid = sns.FacetGrid(train_df, row='Embarked', col='Survived', size=2.2, aspect=1.6)
grid.map(sns.barplot, 'Sex', 'Fare', alpha=.5, ci=None)
grid.add_legend()

#dropping features
print("Before", train_df.shape, test_df.shape, combine[0].shape, combine[1].shape)
train_df = train_df.drop(['Ticket', 'Cabin'], axis=1)
test_df = test_df.drop(['Ticket', 'Cabin'], axis=1)
combine = [train_df, test_df]
print("After", train_df.shape, test_df.shape, combine[0].shape, combine[1].shape)

for dataset in combine:
    dataset['Title'] = dataset.Name.str.extract(' ([A-Za-z]+)\.', expand=False)

pd.crosstab(train_df['Title'], train_df['Sex'])

#We can replace many titles with a more common name or classify them as Rare.
for dataset in combine:
    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',
 	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer'], 'Rare')

    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')  #Mlle french for miss
    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')    #Mme equivalent for mrs

train_df[['Title', 'Survived']].groupby(['Title'], as_index=False).mean()

#converting the categorical titles to ordinal.
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
    dataset['Title'] = dataset['Title'].map(title_mapping)
    dataset['Title'] = dataset['Title'].fillna(0)

train_df.head()

#Now we can safely drop the Name feature from training and testing datasets. We also do not need the PassengerId feature in the training dataset.
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
test_df = test_df.drop(['Name'], axis=1)
combine = [train_df, test_df]
train_df.shape, test_df.shape

#Converting a categorical feature
for dataset in combine:
    dataset['Sex'] = dataset['Sex'].map( {'female': 1, 'male': 0} ).astype(int)

train_df.head()

#Completing a numerical continuous feature
grid = sns.FacetGrid(train_df, row='Pclass', col='Sex', size=2.2, aspect=1.6)
grid.map(plt.hist, 'Age', alpha=.5, bins=20)
grid.add_legend()

#Let we start by preparing an empty array to contain guessed Age values based on Pclass x Gender combinations.
guess_ages = np.zeros((2,3))
guess_ages

#Now we iterate over Sex (0 or 1) and Pclass (1, 2, 3) to calculate guessed values of Age for the six combinations.
for dataset in combine:
    for i in range(0, 2):
        for j in range(0, 3):
            guess_df = dataset[(dataset['Sex'] == i) &
                                  (dataset['Pclass'] == j+1)]['Age'].dropna()

            age_guess = guess_df.median()

            # Convert random age float to nearest .5 age
            guess_ages[i,j] = int( age_guess/0.5 + 0.5 ) * 0.5

    for i in range(0, 2):
        for j in range(0, 3):
            dataset.loc[ (dataset.Age.isnull()) & (dataset.Sex == i) & (dataset.Pclass == j+1),'Age'] = guess_ages[i,j]
    dataset['Age'] = dataset['Age'].astype(int)

train_df.head()

#creating Age bands and determine correlations with Survived.
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False).mean().sort_values(by='AgeBand', ascending=True)

#Replacing Age with ordinals based on these bands.
for dataset in combine:
    dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
    dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32), 'Age'] = 1
    dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2
    dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3
    dataset.loc[ dataset['Age'] > 64, 'Age'] =4
train_df.head()

#We can now remove the AgeBand feature.
train_df = train_df.drop(['AgeBand'], axis=1)
combine = [train_df, test_df]
train_df.head()

#Creating new feature combining existing features
for dataset in combine:
    dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1

train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False).mean().sort_values(by='Survived', ascending=False)

#We can create another feature called IsAlone.
for dataset in combine:
    dataset['IsAlone'] = 0
    dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1

train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False).mean()

#droping Parch, SibSp, and FamilySize features in favor of IsAlone.
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
combine = [train_df, test_df]
train_df.head()

#We can also create a feature combining Pclass and Age.
for dataset in combine:
    dataset['Age*Class'] = dataset.Age * dataset.Pclass

train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10)

#Completing a categorical feature
#Embarked feature takes S, Q, C values based on port of embarkation.
freq_port = train_df.Embarked.dropna().mode()[0]
freq_port
for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)

train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False).mean().sort_values(by='Survived', ascending=False)

#Converting categorical feature to numeric
for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].map( {'S': 0, 'C': 1, 'Q': 2} ).astype(int)
train_df.head()

#We can now complete the Fare feature for single missing value in test dataset using median to get the value for this feature
test_df['Fare'].fillna(test_df['Fare'].dropna().median(), inplace=True)
test_df.head()

#We can now create FareBand.
train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False).mean().sort_values(by='FareBand', ascending=True)

#Converting the Fare feature to ordinal values based on the FareBand.
for dataset in combine:
    dataset.loc[(dataset['Fare'] > -0.001) &(dataset['Fare'] <= 7.91), 'Fare'] = 0
    dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1
    dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare']   = 2
    dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
    dataset['Fare'] = dataset['Fare'].astype(int)

train_df = train_df.drop(['FareBand'], axis=1)
combine = [train_df, test_df]
train_df.head(10)

test_df.head(10)

#model building
X_train = train_df.drop("Survived", axis=1)
Y_train = train_df["Survived"]
X_test  = test_df.drop("PassengerId", axis=1).copy()
X_train.shape, Y_train.shape, X_test.shape

# 1.Logistic Regression
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
Y_pred = logreg.predict(X_test)
acc_log = round(logreg.score(X_train, Y_train) * 100, 2)
acc_log

# =============================================================================
# Sex is highest positivie coefficient, implying as the Sex value increases (male: 0 to female: 1), the probability of Survived=1 increases the most.
# Inversely as Pclass increases, probability of Survived=1 decreases the most.
# This way Age*Class is a good feature to model as it has second highest negative correlation with Survived.
# So is Title as second highest positive correlation.
# =============================================================================
coeff_df = pd.DataFrame(train_df.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df["Correlation"] = pd.Series(logreg.coef_[0])
coeff_df.sort_values(by='Correlation', ascending=False)

#---------------------------------------------------------
#   2. Support Vector Machines
#---------------------------------------------------------
svc = SVC()
svc.fit(X_train, Y_train)
Y_pred = svc.predict(X_test)
acc_svc = round(svc.score(X_train, Y_train) * 100, 2)
acc_svc

#---------------------------------------------------------
#   3. K-nearest neighbour
#---------------------------------------------------------
knn = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X_train, Y_train)
Y_pred = knn.predict(X_test)
acc_knn = round(knn.score(X_train, Y_train) * 100, 2)
acc_knn

#---------------------------------------------------------
#   4. Gaussian Naive Bayes(least score)
#---------------------------------------------------------
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
Y_pred = gaussian.predict(X_test)
acc_gaussian = round(gaussian.score(X_train, Y_train) * 100, 2)
acc_gaussian

#---------------------------------------------------------
#   5. Decision Tree
#---------------------------------------------------------
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
Y_pred = decision_tree.predict(X_test)
acc_decision_tree = round(decision_tree.score(X_train, Y_train) * 100, 2)
acc_decision_tree

#---------------------------------------------------------
#   6. Random Forest(max score alongside decision tree)
#---------------------------------------------------------
random_forest = RandomForestClassifier(n_estimators=100)
random_forest.fit(X_train, Y_train)
Y_pred = random_forest.predict(X_test)
random_forest.score(X_train, Y_train)
acc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)
acc_random_forest

#MODEL EVALUATION
models = pd.DataFrame({
    'Model': ['Logistic Regression','Support Vector Machines', 'KNN',
               'Naive Bayes','Decision Tree','Random Forest',],
    'Score': [acc_log, acc_svc, acc_knn, acc_gaussian,
              acc_decision_tree,acc_random_forest]})
models.sort_values(by='Score', ascending=False)

#test data predictions
results = pd.DataFrame({
        "PassengerId": test_df["PassengerId"],
        "Survived": Y_pred
    })

print(results)


import pytest
import sqlalchemy as sa
from fastapi.testclient import TestClient
from sqlalchemy.orm import Session, sessionmaker

from app import crud, models, schemas
from app.api.deps import get_db
from app.db.database import Base
from app.main import app

# TODO fix this to use database.py stuff
SQLALCHEMY_DATABASE_URL = "sqlite:///./test.db"

engine = sa.create_engine(
    SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}, future=True
)
TestingSessionLocal = sessionmaker(
    autocommit=False, autoflush=False, bind=engine, future=True
)

# Set up the database once
Base.metadata.drop_all(bind=engine)
Base.metadata.create_all(bind=engine)

_db = TestingSessionLocal()

class TestData:
    USER: models.User = crud.user.create(
        _db, obj_in=schemas.UserCreate(email="gert@bert.nl")
    )
    EXER_PARA_1: models.ExerciseParameter = crud.exercise_parameter.create(
        _db, obj_in=schemas.ExerciseParameterCreate(name="mm")
    )
    EXER_PARA_2: models.ExerciseParameter = crud.exercise_parameter.create(
        _db, obj_in=schemas.ExerciseParameterCreate(name="kg")
    )
    EXERCISE_1: models.Exercise = crud.exercise.save(
        _db,
        db_obj=models.Exercise(
            name="Moeilijke Oefening", parameters=[EXER_PARA_1, EXER_PARA_2]
        ),
    )
    EXERCISE_2: models.Exercise = crud.exercise.save(
        _db,
        db_obj=models.Exercise(name="Makkelijke Oefening", parameters=[EXER_PARA_1]),
    )

# These two event listeners are only needed for sqlite for proper
# SAVEPOINT / nested transaction support. Other databases like postgres
# don't need them.
# From: https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl # noqa
@sa.event.listens_for(engine, "connect")
def do_connect(dbapi_connection, connection_record):
    # disable pysqlite's emitting of the BEGIN statement entirely.
    # also stops it from emitting COMMIT before any DDL.
    dbapi_connection.isolation_level = None
    # dbapi_connection.execute("PRAGMA foreign_keys=ON;")

@sa.event.listens_for(engine, "begin")
def do_begin(conn):
    # emit our own BEGIN
    conn.exec_driver_sql("BEGIN")

# This fixture is the main difference to before. It creates a nested
# transaction, recreates it when the application code calls session.commit
# and rolls it back at the end.
# Based on: https://docs.sqlalchemy.org/en/14/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites # noqa
@pytest.fixture()
def db() -> Session:
    connection = engine.connect()
    transaction = connection.begin()
    session = TestingSessionLocal(bind=connection)

    # Begin a nested transaction (using SAVEPOINT).
    nested = connection.begin_nested()

    # If the application code calls session.commit, it will end the nested
    # transaction. Need to start a new one when that happens.
    @sa.event.listens_for(session, "after_transaction_end")
    def end_savepoint(session, transaction):
        nonlocal nested
        if not nested.is_active:
            nested = connection.begin_nested()

    yield session

    # Rollback the overall transaction, restoring the state before the test ran.
    session.close()
    transaction.rollback()
    connection.close()

# A fixture for the fastapi test client which depends on the
# previous session fixture. Instead of creating a new session in the
# dependency override as before, it uses the one provided by the
# session fixture.
@pytest.fixture()
def client(db) -> TestClient:
    def override_get_db():
        yield db

    app.dependency_overrides[get_db] = override_get_db
    yield TestClient(app)
    del app.dependency_overrides[get_db]


audio='AudioClip.wav' #location

import librosa
import librosa.display
import matplotlib.pyplot as plt

# Load an audio file
y, sr = librosa.load(audio)

# Compute harmonic component of the audio signal
y_harmonic = librosa.effects.harmonic(y)

# Compute tonnetz
tonnetz = librosa.feature.tonnetz(y=y_harmonic, sr=sr)

# Display the tonnetz
plt.figure(figsize=(10, 4))
librosa.display.specshow(tonnetz, x_axis='time', y_axis='tonnetz')
plt.colorbar()
plt.title('Tonal Centroids (Tonnetz)')
plt.tight_layout()
plt.show()


import json
import logging

from tg import config
from webob.exc import HTTPRequestEntityTooLarge
from paste.deploy.converters import asbool
import pysolr
from pysolr import SolrError
import six

from allura.lib.helpers import shlex_split

log = logging.getLogger(__name__)

escape_rules = {'+': r'\+',
                '-': r'\-',
                '&': r'\&',
                '|': r'\|',
                '!': r'\!',
                '(': r'\(',
                ')': r'\)',
                '{': r'\{',
                '}': r'\}',
                '[': r'\[',
                ']': r'\]',
                '^': r'\^',
                '~': r'\~',
                '*': r'\*',
                '?': r'\?',
                ':': r'\:',
                '"': r'\"',
                ';': r'\;'}

def escape_solr_arg(term):
    """ Apply escaping to the passed in query terms
        escaping special characters like : , etc"""
    term = term.replace('\\', r'\\')   # escape \ first
    for char, escaped_char in escape_rules.items():
        term = term.replace(char, escaped_char)

    return term

def make_solr_from_config(push_servers, query_server=None, **kwargs):
    """
    Make a :class:`Solr <Solr>` instance from config defaults.  Use
    `**kwargs` to override any value
    """
    solr_kwargs = dict(
        commit=asbool(config.get('solr.commit', True)),
        commitWithin=config.get('solr.commitWithin'),
        timeout=int(config.get('solr.long_timeout', 60)),
    )
    solr_kwargs.update(kwargs)
    return Solr(push_servers, query_server, **solr_kwargs)

class Solr:

    """Solr interface that pushes updates to multiple solr instances.

    `push_servers`: list of servers to push to.
    `query_server`: server to read from. Uses `push_servers[0]` if not specified.

    Also, accepts default values for `commit` and `commitWithin`
    and passes those values through to each `add` and `delete` call,
    unless explicitly overridden.
    """

    def __init__(self, push_servers, query_server=None,
                 commit=True, commitWithin=None, **kw):
        self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers]
        if query_server:
            self.query_server = pysolr.Solr(query_server, **kw)
        else:
            self.query_server = self.push_pool[0]
        self._commit = commit
        self.commitWithin = commitWithin

    def add(self, *args, **kw):
        if 'commit' not in kw:
            kw['commit'] = self._commit
        if self.commitWithin and 'commitWithin' not in kw:
            kw['commitWithin'] = self.commitWithin
        responses = []
        for solr in self.push_pool:
            try:
                responses.append(solr.add(*args, **kw))
            except SolrError as e:
                if '(HTTP 413)' in str(e):
                    raise HTTPRequestEntityTooLarge() from e
                else:
                    raise
        return responses

    def delete(self, *args, **kw):
        if 'commit' not in kw:
            kw['commit'] = self._commit
        responses = []
        for solr in self.push_pool:
            responses.append(solr.delete(*args, **kw))
        return responses

    def commit(self, *args, **kw):
        responses = []
        for solr in self.push_pool:
            responses.append(solr.commit(*args, **kw))
        return responses

    def search(self, *args, **kw):
        return self.query_server.search(*args, **kw)

class MockSOLR:

    class MockHits(list):

        @property
        def hits(self):
            return len(self)

        @property
        def docs(self):
            return self

        @property
        def facets(self):
            return {'facet_fields': {}}

    def __init__(self):
        self.db = {}

    def add(self, objects):
        for o in objects:
            o['text'] = ''.join(o['text'])
            json.dumps(o)  # ensure no errors (since pysolr 3.9+ uses json API to solr)
            self.db[o['id']] = o

    def commit(self):
        pass

    def search(self, q, fq=None, **kw):
        if q is None:
            q = ''  # shlex will hang on None
        # Parse query
        preds = []
        q_parts = shlex_split(q)
        if fq:
            q_parts += fq
        for part in q_parts:
            if part in ('&&', 'AND'):
                continue
            if part in ('||', 'OR'):
                log.warning(f"MockSOLR doesn't implement OR yet; treating as AND. q={q} fq={fq}")
                continue
            if ':' in part:
                field, value = part.split(':', 1)
                if value.startswith('(') and value.endswith(')'):
                    value = value[1:-1]
                preds.append((field, value))
            else:
                preds.append(('text', part))
        result = self.MockHits()
        for obj in self.db.values():
            for field, value in preds:
                neg = False
                if field[0] in ('!', '-'):
                    neg = True
                    field = field[1:]
                if field == 'text' or field.endswith('_t'):
                    if (value not in str(obj.get(field, ''))) ^ neg:
                        break
                elif field.endswith('_b'):
                    if (asbool(value) != obj.get(field, False)) ^ neg:
                        break
                else:
                    if (value != str(obj.get(field, ''))) ^ neg:
                        break
            else:
                result.append(obj)

        if asbool(kw.get('hl')):
            result.highlighting = {}
        return result

    def delete(self, *args, **kwargs):
        if kwargs.get('q', None) == '*:*':
            self.db = {}
        elif kwargs.get('id', None):
            del self.db[kwargs['id']]
        elif kwargs.get('q', None):
            for doc in self.search(kwargs['q']):
                self.delete(id=doc['id'])


import time
from decimal import Decimal
import pandas as pd
import numpy as np

from threading import Thread

from kucoin.client import Trade, Market, User
from app.bot.crud import insert_trade, insert_tp, update_coin, backtest_insert_tp
from app.bot.helper import clean_trade_dict

class MarketAction(Thread):
    user = None
    market = None
    client = None
    api_key = ''
    api_secret = ''
    api_passphrase = ''

    start_balance = None
    allow_trade = True
    config = None

    backtest_profit = 0  # TODO: remove

    def __init__(self, stream, config, coin):
        Thread.__init__(self)
        MarketAction.init_class_variables(config)

        self.coin = coin
        self.symbol = coin.kucoin_name
        self.funds = config['funds']
        self.stream = stream
        self.coef = config['coef']
        self.window = config['window']

        self.quoteIncrement = self.calculate_increment()  # TODO: correct precision
        self.order_ids = []
        self.buy_order_id = ''
        self.stop_loss_order_id = ''
        self.trade_amount = ''
        self.live_trade = config['mode'] == 'live'
        self.backtest = config['mode'] == 'backtest'
        self.stop = False
        self.indicator = ''

    def run(self):
        self.indicator = Indicator(self.stream.data, self.buy_order_id)

        if self.live_trade:
            print(f'Trading in Live for {self.symbol}')
            self.initialize_trade_info(self.coin)

            while not self.stop:
                if self.stream.check_for_action:
                    self.indicator.is_bought = self.buy_order_id  # update buy status
                    self.indicator.data = self.stream.data

                    # if self.buy_order_id:
                    #     print(self.symbol, ': MarketAction buy_order_id', self.buy_order_id, 'Indicator:', self.indicator.is_bought)

                    # print(f'Action -> Checking for action! {self.symbol}')
                    if self.indicator.buy() and self.allow_trade:

                        self.create_order('buy', funds=self.funds)

                    elif self.indicator.sell():
                        self.create_order('sell', size=self.trade_amount)

                    self.stream.check_for_action = False
                time.sleep(0.5)

        elif self.backtest:
            print('Back testing bot')
            while not self.stop:
                if self.stream.check_for_action:
                    self.indicator.data = self.stream.data
                    self.indicator.is_bought = self.buy_order_id  # update buy status

                    # print(f'Action -> Checking for action! {self.symbol}')
                    if self.indicator.buy() and self.allow_trade:
                        self.backtest_create_order('buy', funds=self.funds)

                    elif self.indicator.sell():
                        self.backtest_create_order('sell', funds=self.funds)
                        print('backtest_profit:', MarketAction.backtest_profit)

                    self.stream.check_for_action = False

        print('Action Thread Stopped!')

    def create_order(self, side, **kwargs):
        try:
            if side == 'buy':
                order_data = self.client.create_market_order(self.symbol, side, **kwargs)
                order_id = order_data.get('orderId')
                print(f'{side} {self.symbol}, order_id: {order_id}')

            if side == 'sell':
                stop_executed = False
                if self.stop_loss_order_id:  # Check if stop loss was executed
                    stop_loss_data = self.client.get_order_details(self.stop_loss_order_id)
                    if not stop_loss_data.get('isActive'):  # Check if stop loss was executed
                        print(f'Stop Loss Executed for {self.symbol}')
                        stop_executed = True
                        order_id = stop_loss_data.get('id')  # Set order Id for saving correct trade in database

                if not stop_executed:  # Create sell order and cancel stop loss
                    print(f'Stop Loss DID NOT Execute for {self.symbol}')
                    order_data = self.client.create_market_order(self.symbol, side, **kwargs)
                    order_id = order_data.get('orderId')
                    print(f'{side} {self.symbol}, order_id: {order_id}')
                    self.client.cancel_order(self.stop_loss_order_id)
                    self.stop_loss_order_id = ''

        except Exception as e:
            message = f'Exception (in create_order): {side} {self.symbol} {e}'
            print(message)

        else:
            if side == 'buy':
                while True:  # wait for order to fill
                    order_data = self.client.get_order_details(order_id)
                    if not order_data.get('isActive'):
                        trade_amount = order_data.get('dealSize')
                        self.trade_amount = self.round_trade_amount(trade_amount, self.quoteIncrement)
                        break
                    time.sleep(0.5)

                self.buy_order_id = order_id
                self.stop_loss_order_id = self.create_stop_loss(order_data)  # Create Stop Loss order

                # TODO: Monitor
                update_coin(kucoin_name=self.symbol, trade_id=self.buy_order_id)
                insert_trade(**clean_trade_dict(order_data))

            elif side == 'sell':
                while True:  # wait for order to fill
                    order_data = self.client.get_order_details(order_id)
                    if not order_data.get('isActive'):
                        break
                    time.sleep(0.5)

                # TODO: Monitor
                update_coin(kucoin_name=self.symbol, trade_id='')
                insert_trade(**clean_trade_dict(order_data))

                if self.buy_order_id:  # if sell trade has pair buy save in pairs
                    buy_id = self.buy_order_id
                    sell_id = order_id
                    profit = self.calculate_pair_profit(buy_id=buy_id, sell_id=sell_id)
                    insert_tp(buy_id=buy_id, sell_id=sell_id, profit=profit)

                self.buy_order_id = ''
                self.trade_amount = '0'
                self.update_trade_allowance()

    def create_stop_loss(self, data, coef=0.96):
        stop_price = float(data.get('dealFunds')) / float(data.get('dealSize')) * coef
        kwargs = {'symbol': self.symbol,
                  'side': 'sell',
                  'funds': self.trade_amount,
                  'stopPrice': stop_price,
                  }

        order_data = self.client.create_market_stop_order(**kwargs)
        stop_order_id = order_data.get('orderId')
        self.stop_loss_order_id = stop_order_id
        return stop_order_id

    def calculate_pair_profit(self, buy_id, sell_id):
        """ Calculates and returns relative profit in percentage"""
        sell_trade = self.client.get_order_details(sell_id)
        buy_trade = self.client.get_order_details(buy_id)

        total_profit = float(sell_trade.get('dealFunds')) - \
                       float(sell_trade.get('fee')) - \
                       float(buy_trade.get('dealFunds')) - \
                       float(buy_trade.get('fee'))

        relative_profit_perc = total_profit / float(buy_trade.get('dealFunds')) * 100
        return relative_profit_perc

    def calculate_increment(self):
        increments = {data.get('symbol'): data.get('quoteIncrement') for data in self.market.get_symbol_list()}
        return increments.get(self.symbol)

    def initialize_trade_info(self, coin):
        """If coin is bought get information about trade when initialising"""
        try:
            if coin.bought_id:
                order_data = self.client.get_order_details(coin.bought_id)
                if not order_data.get('isActive'):
                    trade_amount = order_data.get('dealSize')
                    trade_amount = self.round_trade_amount(trade_amount, self.quoteIncrement)
            else:  # if is not bought make trade amount empty string
                trade_amount = ''
        except Exception as e:
            self.buy_order_id = ''
            self.trade_amount = '0'
            message = f'Exception in loading Trade info from CSV for {self.symbol}: {e}'
            if '400100' not in str(e):  # if different error than: 'order not exist'
                print(message)
        else:
            if trade_amount:
                self.buy_order_id = coin.bought_id
                self.trade_amount = trade_amount

    def backtest_create_order(self, side, **kwargs):
        print('backtest_create_order')
        if side == 'buy':
            print('--------------- buy')
            self.buy_order_id = 'buy_id'
            self.backtest_buy_price = self.stream.data.Close.iloc[-1]
            self.backtest_buy_time = self.stream.data.index[-1]

        if side == 'sell':
            print('--------------- sell')

            sell_price = self.stream.data.Close.iloc[-1]
            relative_profit_perc = (sell_price / self.backtest_buy_price - 1.002) * 100
            MarketAction.backtest_profit += relative_profit_perc
            # print(self.symbol, relative_profit_perc)

            data_kwargs = {'symbol': self.symbol,
                           'buy_time': self.backtest_buy_time,
                           'sell_time':self.stream.data.index[-1],
                           'profit': relative_profit_perc,
                           }
            backtest_insert_tp(**data_kwargs)

            self.buy_order_id = ''
            self.backtest_buy_time = ''
            self.backtest_buy_price = 0

    @staticmethod
    def round_trade_amount(funds, increment):
        # use maximum of '0.0001' and value returned from Kucoin as values got from Kucoin are incorrect
        increment = max(increment, '0.0001')
        return str(Decimal(funds).quantize(Decimal(increment), rounding='ROUND_DOWN'))

    @classmethod
    def sell_everything(cls, quote='USDT', account_type='trade'):
        price_dict = dict()
        for data in cls.market.get_all_tickers().get('ticker'):
            symbol = data.get('symbol')
            price_dict[symbol] = float(data.get('sell'))

        account_balance = cls.user.get_account_list(account_type=account_type)
        for data in account_balance:
            currency = data.get('currency')
            if currency != quote:
                if f'{currency}-{quote}' in price_dict.keys():
                    symbol = f'{currency}-{quote}'
                    price = price_dict[symbol]
                    balance = float(data.get('balance')) * price
                    if balance > 5:
                        cls.client.create_market_order(symbol, 'sell', funds=cls.round_trade_amount(balance, '0.001'))
                        print(f'Sold {symbol}')
                elif f'{quote}-{currency}' in price_dict.keys():
                    symbol = f'{quote}-{currency}'
                    price = price_dict[symbol]
                    balance = float(data.get('balance')) / price
                    if balance > 5:
                        cls.client.create_market_order(symbol, 'buy', size=cls.round_trade_amount(balance, '0.001'))
                        print(f'Sold {symbol}')

        print('Done Selling!')

    @classmethod
    def calculate_total_balance(cls, quote='USDT', account_type='trade'):
        price_dict = dict()
        for data in cls.market.get_all_tickers().get('ticker'):
            symbol = data.get('symbol')
            price_dict[symbol] = float(data.get('sell'))

        account_balance = cls.user.get_account_list(account_type=account_type)
        balance = 0
        for data in account_balance:
            currency = data.get('currency')
            if currency != quote:
                if f'{currency}-{quote}' in price_dict.keys():
                    price = price_dict[f'{currency}-{quote}']
                    balance += float(data.get('balance')) * price
                elif f'{quote}-{currency}' in price_dict.keys():
                    price = price_dict[f'{quote}-{currency}']
                    balance += float(data.get('balance')) / price
            else:
                balance += float(data.get('balance'))
        message = f'Total Balance is {balance}'
        return balance

    @classmethod
    def update_trade_allowance(cls):
        """check if balance is not very low to allow further trading"""
        cls.allow_trade = cls.calculate_total_balance() > cls.start_balance * 0.8
        # TODO: change allow_trade coefficient to be included in config file

    @classmethod
    def init_class_variables(cls, config):
        """Initialize class variables on the first time"""
        if not MarketAction.user:
            cls.api_key = config['api_key']
            cls.api_secret = config['api_secret']
            cls.api_passphrase = config['api_passphrase']
            cls.user = User(cls.api_key, cls.api_secret, cls.api_passphrase)
            cls.market = Market(cls.api_key, cls.api_secret, cls.api_passphrase)
            cls.client = Trade(cls.api_key, cls.api_secret, cls.api_passphrase)

            cls.start_balance = cls.calculate_total_balance()
            cls.config = config
            print('MarketAction Class Variables Successfully Initialized!')

class Indicator:
    def __init__(self, data, is_bought):
        self.atr_period = 14
        self.ma_len = 5
        self.ma_source = 'Close'
        self.atr_coef = 3

        self.is_bought = is_bought
        self.stop_price = -np.inf
        self.high_val = -np.inf
        self.data = data

    def atr_ema(self):
        diff1 = self.data['High'] - self.data['Low']
        diff2 = self.data['High'] - self.data['Low'].shift(periods=1)
        diff3 = self.data['Close'] - self.data['Low'].shift(periods=1)

        ATR = pd.DataFrame({'diff1': diff1, 'diff2': diff2, "diff3": diff3})
        ATR['max'] = ATR.max(axis=1)
        ema = ATR.ta.ema(close='max', length=self.atr_period)
        return ema

    def buy(self):
        self.data.loc[:, 'MA'] = self.data.ta.sma(close=self.ma_source, length=self.ma_len)
        self.data.loc[:, 'ATR'] = self.atr_ema()
        self.data.loc[:, 'buy_price'] = self.data['MA'].shift(periods=1) + self.data['ATR']

        if self.is_bought:  # TODO: Move 'if' outside buy function
            self.high_val = max(self.high_val, self.data['Close'].iloc[-1])
            self.stop_price = self.high_val - self.atr_coef * self.data['ATR'].iloc[-1]
        else:
            self.stop_price = -np.inf
            self.high_val = -np.inf

        ######## Buy ############
        self.data.loc[:, 'ind_buy'] = self.data['Close'] > self.data['buy_price']
        # print((self.data['Close'] > self.data['buy_price']).iloc[0], self.data['Close'].iloc[0], self.data['buy_price'].iloc[0])

        ##### EMA ###########
        self.data.loc[:, 'EMA20'] = self.data.ta.sma(close='Close', length=20)
        self.data.loc[:, 'EMA50'] = self.data.ta.sma(close='Close', length=50)
        ind_buy_ema = self.data['EMA50'] < self.data['EMA20'] + self.data['ATR']
        self.data.loc[:, 'ind_buy'] *= ind_buy_ema

        ######### VOLUME ##########
        self.data.loc[:, 'Vol_SMA24'] = self.data.ta.sma(close='Volume', length=24)
        ind_buy_vol = self.data['Volume'] > 1.5 * self.data['Vol_SMA24']
        self.data.loc[:, 'ind_buy'] *= ind_buy_vol

        buy = self.data.loc[:, 'ind_buy'].iloc[-1]

        # b_t1 = (self.data['Close'] > self.data['buy_price']).iloc[-1]
        # if buy:
        #     print(f"""Ping, is_bought -{self.is_bought}-, Close>Buy -{b_t1}-, buy {buy},
        #         Buy Price {self.data['buy_price'].iloc[-1]},
        #         Close {self.data['Close'].iloc[-1]}""")

        if buy and not self.is_bought:
            return True
        return False

    def sell(self):
        buy_price = self.data['buy_price'].iloc[-1]
        sell = self.stop_price > self.data['Low'].iloc[-1] and buy_price > self.data['Low'].iloc[-1]
        if sell and self.is_bought:
            return True
        return False

# def indicator_buy(self):
#     good_trend = self.trend_v1()  # check if trend is up or flat
#     up_candles = self.stream.data['HA_Close'] > self.stream.data['HA_Open']
#
#     try:  # see if Trades are available
#         indicator = (self.stream.data['Volume'] > self.coef * self.stream.data['EMA60_Volume']) * (
#                 self.stream.data['Trades'] > self.coef * self.stream.data['EMA60_Trades'])
#     except Exception:
#         indicator = self.stream.data['Volume'] > self.coef * self.stream.data['EMA60_Volume']
#
#     indicator *= good_trend * up_candles
#     buy = indicator.iloc[-self.window:].sum() == self.window
#     #         indicator = indicator.rolling(window=window).apply(lambda x: np.sum(x) >= window).fillna(0)
#     return buy
#
# def trend_v1(self):
#     good_trend = self.stream.data['EMA50'].rolling(window=60).apply(
#         lambda x: (x[-1] - x[0]) / x[0] > -10e-3).fillna(0)
#     return good_trend
#
# def trend_v2(self):
#     EMA5 = self.stream.data['EMA5']
#     EMA9 = self.stream.data['EMA9']
#     EMA12 = self.stream.data['EMA12']
#     good_trend = EMA12 < EMA9 < EMA5
#     return good_trend
#
# def indicator_sell(self):
#     lose_momentum = self.lose_momentum_v2()
#     if lose_momentum:
#         return True
#     return False
#
# def lose_momentum_v1(self):
#     lose_momentum = self.stream.data['HA_Close_slow'].iloc[-1] < self.stream.data['HA_Open_slow'].iloc[-1]
#     return lose_momentum
#
# def lose_momentum_v2(self):
#         EMA5 = self.stream.data['EMA5'].iloc[-1]
#         EMA9 = self.stream.data['EMA9'].iloc[-1]
#         EMA12 = self.stream.data['EMA12'].iloc[-1]
#         lose_momentum = EMA12 > EMA9 and EMA12 > EMA5
#         return lose_momentum


from PIL import Image

# difference of getcolors, getpixel, getdata

def main():
    filename = "d:/YID_git_repos/DLCoding/naruto1.jpg"
    image = Image.open(filename).crop((0,0,3,3))
    size = width, height = image.size

    print(image.getcolors( width*height))
    print(image.getpixel((2,2))         )
    print(list(image.getdata())         )

if __name__ == '__main__':
    main()


from typing import Tuple
import random
import pygame
from IslandPy.Render.ARenderObject import ARenderObject
from IslandPy.Scenes.AScene import AScene

class Obstacle(ARenderObject):

    speed = 8

    def __init__(self, scene: AScene, size: Tuple[int, int] = (0, 0)):
        super().__init__(scene, size)
        self.images = []
        for i in range(1, 6):
            self.images.append(pygame.image.load(f"res/cactus{i}.png").convert_alpha())

        self._body = self.images[0]
        self.rect = self._body.get_rect()
        self.__start_pos_y = 0
        self.is_started = False
        self.hide()

    def update(self, dt) -> None:
        if self.is_started:
            self.rect.x -= Obstacle.speed

    def change_image(self) -> None:
        b = self.rect.bottom
        self._body = random.choice(self.images)
        self.rect.w, self.rect.h = self._body.get_rect().w, self._body.get_rect().h
        self.rect.bottom = b

    def start(self) -> None:
        self.show()
        self.is_started = True

    def draw(self, surface: pygame.Surface) -> None:
        if self.is_draw:
            surface.blit(self._body, self.rect)


from random import randint
from time import sleep

def sorteado(sorteados):  # Inicializando a função com parâmetro
    print('Serão sorteados \033[97m10\033[m valores entre \033[37m-10\033[m e \033[30m10\033[m: ', end='')
    for i in range(0, 10):  # Alcance de 0 a 9 (O último número não entra no loop)
        num = randint(-10, 10)  # Sorteando valores entre -10 e 10
        sorteados.append(num)  # Adicionando na lista sorteados o número sorteado
        # (variável valores no final do código inicializada, podendo assim adicionar os números na variável
        # passada como parâmetro)
        print(f'\033[35m{num}\033[m ', end='')
        # end='' Serve para mostrar os valores na mesma linha
        sleep(0.4)
    print('  \033[36mFinalizado\033[m!!!')

def verificar(sorteados):  # Inicializando a função com parâmetro e declarando as variáveis
    qntd_neg = 0
    qntd_pos = 0
    par = 0
    impar = 0
    for valor in sorteados:  # Looping para cada valor na lista dos números sorteados
        if valor < 0:
            qntd_neg += 1
        if valor >= 0:
            qntd_pos += 1
        if valor % 2 == 0:
            par += 1
        if valor % 2 == 1:
            impar += 1
    print(f'\nA quantidade de números negativos foram: \033[31m{qntd_neg}\033[m')
    print(f'\nA quantidade de números positivos foram: \033[32m{qntd_pos}\033[m')
    print(f'\nA quantidade de pares é: \033[33m{par}\033[m')
    print(f'\nA quantidade de ímpares é: \033[34m{impar}\033[m')

valores = list()  # Criando uma lista

sorteado(valores)  # Passando a lista vazia como parâmetro
verificar(valores)  # Passando a lista vazia como parâmetro


from secondaires.navigation.equipage.signaux import *

from ..ordre import *

class ChargerBoulet(Ordre):

    """Ordre charger_boulet.

    Cet ordre demande au matelot de charger en boulet un canon du
    navire. Cet ordre est également responsable de collecter un boulet
    de canon de la cale si besoin est.

    """

    cle = "charger_boulet"
    etats_autorises = ("charger_canon", "")

    def __init__(self, matelot, navire, canon=None, bruyant=False):
        Ordre.__init__(self, matelot, navire, canon, bruyant)
        self.canon = canon
        self.bruyant = bruyant

    def executer(self):
        """Exécute l'ordre : colmate."""
        navire = self.navire
        matelot = self.matelot
        personnage = matelot.personnage
        canon = self.canon
        salle = canon.parent
        yield 0.3
        boulet = self.prendre_boulet_canon(personnage)
        if boulet is None:
            yield SignalAbandonne("Il n'y a pas de boulet de canon en " \
                    "cale !", self.bruyant)

        yield canon.pre_charger(personnage)
        canon.post_charger(personnage, boulet)
        yield SignalTermine()

    def prendre_boulet_canon(self, personnage):
        """Prend un boulet de canon depuis la cale si besoin."""
        self.jeter_ou_entreposer("boulet de canon")
        for objet in list(personnage.equipement.tenus):
            if objet.nom_type == "boulet de canon":
                personnage.equipement.tenus.retirer(objet)
                return objet

        # On cherche à en récupérer dans la cale
        cale = personnage.salle.navire.cale
        boulets = cale.boulets
        if not boulets:
            return None

        boulets = [importeur.objet.prototypes[cle] for cle in \
                boulets.keys()]
        boulets.sort(key=lambda prototype: prototype.degats)
        boulet = boulets[0]
        return cale.recuperer(personnage, boulet.cle, donner=False)


# 处理raw_data文件，得到可以用于后续训练的文件
# 目前是第一阶段，只预测罪名
# 导入使用的库
from helper import *
import os, json, time, jieba, re
import matplotlib.pyplot as plt
import multiprocessing
import numpy as np
from sklearn.datasets.base import Bunch
from sklearn.externals import joblib
from sklearn.preprocessing import MultiLabelBinarizer

def process(raw_data_file, trainbunch_file, testbunch_file, split_value):
    '''
    此函数为控制整个处理过程的函数
    输入为原始文件，以及处理后要储存的bunch文件，还有训练集和测试集的分割比例
    函数没有返回值，最终将处理后的bunch写入文件为止
    '''
    databunch = Bunch(contents=[], accu=[])  # databunch 储存对raw_data_file处理后的数据，然后再shuffle成训练集和测试集。
    # contents储存的是处理后的fact， accu储存的是汉字的犯罪标签，new_accu储存的二值化的犯罪标签。
    print("正在写入数据》》》》"); st = time.time()
    with open(raw_data_file, "r", encoding="utf-8") as rawdata_file:
        file_lenth = len(rawdata_file.readlines()); cnt = 0    #  用于记录数据处理进度
    rawdata_file = open(raw_data_file, "r", encoding="utf-8")
    for line_json in rawdata_file.readlines():   # line_json中包含着我们所需的所有信息
        line = json.loads(line_json)             # line是一个字典
        fact = line["fact"]
        accusation = line["meta"]["accusation"]
        new_fact = process_fact(fact)   # new_fact是处理后的犯罪事实，保留了中文, 然后用jieba分词得到最终的结果。
        databunch.contents.append(new_fact)
        databunch.accu.append(accusation)
        cnt += 1
        if cnt%1000 == 0:
            print("写入数据进度为：{:.3f}%".format(cnt/file_lenth*100))
    rawdata_file.close(); et = time.time()
    print("写入数据完毕！！！！用时：{:.3f}s".format(et-st))
    print("正在处理数据》》》》"); st = time.time()
    # new_fact和accu已经写入bunch, 将databunch打乱顺序, 分为trainbunch和testbunch
    random_seed = np.random.randint(0,100)  # 保证databunch中各个内容的shuffle次序是一样的。
    for i in databunch.keys():
        np.random.seed(random_seed)
        np.random.shuffle(databunch[i])
    trainbunch, testbunch = split_train_test(databunch, split_value)
    # 对accu二值化
    multilabelbinarizer = MultiLabelBinarizer(accu_list)
    trainbunch.new_accu = multilabelbinarizer.fit_transform(trainbunch.accu)
    testbunch.new_accu = multilabelbinarizer.transform(testbunch.accu)
    # 将trainbunch, testbunch写入trainbunch_file, testbunch_file
    joblib.dump(trainbunch, trainbunch_file)
    joblib.dump(testbunch, testbunch_file)
    et = time.time()
    print("处理数据完毕！！！！用时：{:.3f}s".format(et - st))

def process_fact(fact):
    new_fact = []
    pattern = re.compile(u"[\u4e00-\u9fa5]+")
    seg_fact = re.findall(pattern=pattern, string=fact)  # 此时的fact为去掉非汉字的列表，['昌宁县人民检察院指控', '年', '月', '日下午', '时许', '被告人段某驾拖车经过鸡飞乡澡塘街子', '时逢堵车', '段某将车停在', '冰凉一夏', '冷饮店门口', '被害人王某的侄子王', '某示意段某靠边未果', '后上前敲打车门让段某离开', '段某遂驾车离开', '但对此心生怨愤', '同年', '月', '日', '时许', '被告人段某酒后与其妻子王', '某一起准备回家', '走到鸡飞乡澡塘街富达通讯手机店门口时停下', '段某进入手机店内对被害人王某进行吼骂', '紧接着从手机店出来拿得一个石头又冲进手机店内朝王某头部打去', '致王某右额部粉碎性骨折', '右眼眶骨骨折', '经鉴定', '被害人王某此次损伤程度为轻伤一级']
    for sub_sentence in seg_fact:
        new_fact.extend([word for word in jieba.cut(sub_sentence)])
    return new_fact

def shuffle_bunch(databunch):
    new_databunch = Bunch()
    for i in databunch.keys():
        np.random.shuffle(databunch[i])
    return new_databunch

def split_train_test(new_databunch, split_value):
    lenth = len(new_databunch.contents) # 数据总量
    split_point = int((1-split_value)*lenth)
    trainbunch = Bunch(); testbunch = Bunch()
    trainbunch.contents = new_databunch.contents[:split_point]
    testbunch.contents = new_databunch.contents[split_point:]
    trainbunch.accu = new_databunch.accu[:split_point]
    testbunch.accu = new_databunch.accu[split_point:]
    return trainbunch, testbunch

if __name__ == '__main__':
    process(RAW_SMALL_DATA, SMALL_TRAINBUNCH_FILE, SMALL_TESTBUNCH_FILE, split_value=0.2)

'''
        for word in line:
            if word not in stopwordslist and len(word)>1 and "县" not in word and "市" not in word and "区" not in word \
                    and "省" not in word and "自治区" not in word and "村" not in word and "镇" not in word \
                    and "营" not in word:
                 newline += word + " "
        if len(newline) > 10:
            newline = newline + "\n"
            o.writelines(newline)
    # 生成相关参数
    accudict, stopwordslist, train_data_list, test_data_list = gen_parameters()

    # 读取JSON文件
    st = time.time()
    writeData(TRAIN_DATA_JSON, TRAIN_DATA, accudict)
    writeData(TEST_DATA_JSON, TEST_DATA, accudict)
    et = time.time()
    time0 = et - st
    print("写入文件用时： {}s".format(str(time0)))

    # 生成分词后文件
    st = time.time()
    pool = multiprocessing.Pool(processes=12)
    for i in train_data_list:
        pool.apply_async(segData, (i,))
    pool.close()
    pool.join()
    pool = multiprocessing.Pool(processes=12)
    for i in test_data_list:
        pool.apply_async(segData, (i,))
    pool.close()
    pool.join()
    et = time.time()
    time0 = et - st
    print("分词用时： {}s".format(str(time0)))

    # 生成process文件,去掉停用词并且过滤掉长度为1的词, 去掉句子长度小于10的句子
    st = time.time()
    pool = multiprocessing.Pool(processes=12)
    for i in train_data_list:
        pool.apply_async(processData, (i,))
    pool.close()
    pool.join()
    pool = multiprocessing.Pool(processes=12)
    for i in test_data_list:
        pool.apply_async(processData, (i,))
    pool.close()
    pool.join()
    et = time.time()
    time0 = et - st
    print("去停用词用时： {}s".format(str(time0)))
    '''


from __future__ import unicode_literals

import json
import yaml
from os.path import basename
from functools import partial
from collections import OrderedDict

from .fs import Node, DocumentIndexer
from .engine import Markment
from .views import TemplateContext
from .events import before, after

class Project(object):
    metadata_filename = '.markment.yml'

    def __init__(self, path):
        self.path = path

        self.node = Node(path)
        self.tree = DocumentIndexer(path)
        self.name = basename(path)
        self.version = ''
        self.description = ''

        documentation_index_fallback = 'README.md'

        found = self.node.grep(r'\.(md|markdown)')

        if found:
            documentation_index_fallback = found[0].basename

        self.meta = {
            'project': {
                'name': self.name,
            },
            'documentation': {
                'index': documentation_index_fallback
            }
        }
        self._found_files = []
        self.load(path)

    def load(self, path):
        metadata = self.parse_metadata(path)

        self.meta.update(metadata)

        p = self.meta['project']
        self.name = p.get('name', self.name)
        self.version = p.get('version', self.version)
        self.description = p.get('description', self.description)
        self.github_url = p.get('github_url', '')

        tarball_fallback = zipball_fallback = ''

        if self.github_url:
            tarball_fallback = '{0}/archive/master.tar.gz'.format(self.github_url)
            zipball_fallback = '{0}/archive/master.zip'.format(self.github_url)

        self.zipball_download_url = p.get('zipball_download_url', zipball_fallback)
        self.tarball_download_url = p.get('tarball_download_url', tarball_fallback)

    def parse_metadata(self, path):
        if not self.node.contains(self.metadata_filename):
            return {}

        with self.node.open(self.metadata_filename) as f:
            data = f.read()

        return yaml.load(data.decode('utf-8'))

    def find_markdown_files(self):
        if self._found_files:
            return self._found_files

        for_blobs = lambda info: info['type'] == 'blob'
        blobs = filter(for_blobs, self.tree.find_all_markdown_files())

        for info in blobs:
            name = info['relative_path']
            self._found_files.append(info)

        return self._found_files

    def get_master_index(self):
        """returns the master index, respecting any existing toc"""
        original_master_index = self.find_markdown_files()
        master_index = []
        table_of_contents = self.meta.get('toc', [])
        for toc_item in table_of_contents:
            for item in original_master_index:
                if toc_item == item['relative_path']:
                    master_index.append(item)

        for item in original_master_index:
            if item not in master_index:
                master_index.append(item)

        return master_index

    def pre_render(self, theme, static_url_cb, link_cb, **kw):
        master_index = self.get_master_index()
        total_indexes = len(master_index)
        for position, info in enumerate(master_index, start=1):
            partial_link_cb = partial(link_cb, current_document_info=info)
            before.rendering_markdown.shout(
                info, theme, kw, position, total_indexes)

            info['extra'] = kw
            info['markment'] = md = self.load_markment(
                info, partial_link_cb, **kw)
            info['markdown'] = md.raw
            info['indexes'] = md.index()
            info['documentation'] = md.rendered

            info.update(self.render_html_from_markdown_info(
                        md, info, theme,
                partial(static_url_cb, current_document_info=info),
                partial_link_cb, [], **kw))

            after.partially_rendering_markdown.shout(
                info, theme, kw, position, total_indexes, master_index)

    def post_render(self, theme, static_url_cb, link_cb, **kw):
        master_index = self.get_master_index()
        total_indexes = len(master_index)
        for position, info in enumerate(master_index, start=1):
            info.update(self.render_html_from_markdown_info(
                        info['markment'], info, theme,
                partial(static_url_cb, current_document_info=info),
                partial(link_cb, current_document_info=info), master_index, **kw))
            after.rendering_markdown.shout(
                info, theme, kw, position, total_indexes, master_index)

        return master_index
    def generate(self, theme, static_url_cb=None, link_cb=None, **kw):
        if not static_url_cb:
            static_url_cb = lambda link, *a, **kw: link

        if not link_cb:
            link_cb = lambda link, *a, **kw: link

        self.pre_render(theme, static_url_cb, link_cb, **kw)

        # rendering again, now with the full master_index
        return self.post_render(theme, static_url_cb, link_cb, **kw)

    def load_markment(self, info, link_cb, **kw):
        with self.node.open(info['path']) as f:
            data = f.read()

        try:
            decoded = data.decode('utf-8')
        except UnicodeEncodeError:
            decoded = data

        return Markment(decoded, url_prefix=link_cb)

    def render_html_from_markdown_info(
            self, md, info, theme, static_url_cb, link_cb, master_index, **kw):

        Context = TemplateContext(
            project=self.meta['project'],
            meta=self.meta,
            documentation=md.rendered,
            index=md.index(),
            master_index=list(master_index),
            json=json,
            static_url_cb=static_url_cb,
            link_cb=link_cb,
            info=info,
            current=info,
            **kw)

        ctx = Context.ready_to_render()
        before.rendering_html.shout(info, ctx)
        info['html'] = theme.render(**ctx).encode('utf-8')
        after.rendering_html.shout(info, ctx)
        return info

    @classmethod
    def discover(cls, path, *args, **kw):
        return cls(path, *args, **kw)

    def __repr__(self):
        return '<Project({0})>'.format(repr(self.path))


import firebase_admin
import hashlib
from firebase_admin import credentials
from firebase_admin import firestore

# Use the application default credentials
cred = credentials.Certificate("airflow/dags/cred.json")
firebase_admin.initialize_app(cred)

db = firestore.client()

from random import randint
from datetime import datetime

from airflow import DAG
from airflow.operators.python import PythonOperator, BranchPythonOperator
from airflow.operators.bash import BashOperator

def _parseHttpGet():
    import datetime
    from os import remove

    from sqlalchemy import null

    numberOfLines = 0
    numberOfGet = 0

    f = open("airflow/dags/http.log", "r")
    w = open("airflow/dags/resultGet.txt", "w")

    def returnMonthNumber(arg):
        switcher = {
            "Jan": 1,
            "Feb": 2,
            "Mar": 3,
            "Apr": 4,
            "May": 5,
            "Jun": 6,
            "Jul": 7,
            "Aug": 8,
            "Sep": 9,
            "Oct": 10,
            "Nov": 11,
            "Dec": 12
        }
        return switcher.get(arg)

    getRequests=[]

    for x in f:
        for index, word in enumerate(x.split()):
            if word == '200':
                # print(x)
                if(x.split()[index-3]) == '"GET':

                    tarih = x.split()[index-5]
                    buyukluk = x.split()[index+1]
                    year = 1000*int(tarih[8])+100*int(tarih[9]) + 10*int(tarih[10])+int(tarih[11])
                    month = returnMonthNumber(tarih[4] + tarih[5] + tarih[6])
                    day = 10*int(tarih[1])+int(tarih[2])
                    hour = 10*int(tarih[13])+int(tarih[14])
                    minute = 10*int(tarih[16])+int(tarih[17])
                    second = 10*int(tarih[19])+int(tarih[20])
                    date = int(datetime.datetime(year, month, day, hour, minute, second).timestamp())
                    getRequests.append((date,'GET',int(buyukluk)))
                    numberOfGet += 1

                numberOfLines += 1

    # sortedGet = sorted(getRequests, key=lambda x: x[0])

    Gets = {}

    seenGet = set()

    for x in getRequests:
        if x[0] not in seenGet:
            seenGet.add(x[0])

    for x in seenGet:
        Gets[x]=[]

    for x in seenGet:
        for y in getRequests:
            if y[0] == x:
                Gets[x].append(y[2])

    w.write('Total 200 requests={}\n'.format(numberOfLines))
    w.write('GET={}\n'.format(numberOfGet))

    for x in Gets:
        w.write(str(x)+':'+str(sum(Gets[x]))+'\n')

    f.close()
    w.close()
    return Gets

def _parseHttpPost():
    import datetime
    from os import remove

    from sqlalchemy import null

    numberOfLines = 0
    numberOfPost = 0
    f = open("airflow/dags/http.log", "r")
    w = open("airflow/dags/resultPost.txt", "w")

    def returnMonthNumber(arg):
        switcher = {
            "Jan": 1,
            "Feb": 2,
            "Mar": 3,
            "Apr": 4,
            "May": 5,
            "Jun": 6,
            "Jul": 7,
            "Aug": 8,
            "Sep": 9,
            "Oct": 10,
            "Nov": 11,
            "Dec": 12
        }
        return switcher.get(arg)

    postRequests=[]

    for x in f:
        for index, word in enumerate(x.split()):
            if word == '200':
                if(x.split()[index-3]) == '"POST':
                    tarih = x.split()[index-5]
                    buyukluk = x.split()[index+1]
                    year = 1000*int(tarih[8])+100*int(tarih[9]) + 10*int(tarih[10])+int(tarih[11])
                    month = returnMonthNumber(tarih[4] + tarih[5] + tarih[6])
                    day = 10*int(tarih[1])+int(tarih[2])
                    hour = 10*int(tarih[13])+int(tarih[14])
                    minute = 10*int(tarih[16])+int(tarih[17])
                    second = 10*int(tarih[19])+int(tarih[20])
                    date = int(datetime.datetime(year, month, day, hour, minute, second).timestamp())# print(date)
                    postRequests.append((date,'POST',int(buyukluk)))
                    numberOfPost += 1
                numberOfLines += 1

    # sortedPost = sorted(postRequests, key=lambda x: x[0])

    Posts = {}

    seenPost = set()

    for x in postRequests:
        if x[0] not in seenPost:
            seenPost.add(x[0])

    for x in seenPost:
        Posts[x]=[]

    for x in seenPost:
        for y in postRequests:
            if y[0] == x:
                Posts[x].append(y[2])

    w.write('Total 200 requests={}\n'.format(numberOfLines))
    w.write('POST={}\n'.format(numberOfPost))

    for x in Posts:
        w.write(str(x)+':'+str(sum(Posts[x]))+'\n')

    f.close()
    w.close()
    return Posts

def _addToDatabase(ti):
    Requests = ti.xcom_pull(task_ids=[
        'parseHttpGet',
        'parseHttpPost',
    ])
    for x in Requests[0]:
        data = {
            u'timeStamp':x,
            u'size':sum(Requests[0][x]),
            u'type':'GET'
        }
        db.collection(u'GetRequests').document(hashlib.sha256(str(x).encode('utf-8')).hexdigest()).set(data)

    for x in Requests[1]:
        data = {
            u'timeStamp':x,
            u'size':sum(Requests[1][x]),
            u'type':'POST'
        }
        db.collection(u'PostRequests').document(hashlib.sha256(str(x).encode('utf-8')).hexdigest()).set(data)

with DAG("httpChartDag", start_date=datetime(2022, 1, 1), schedule_interval='@daily', catchup=False) as dag:
    parseHttpGet = PythonOperator(
        task_id="parseHttpGet",
        python_callable=_parseHttpGet
    )
    parseHttpPost = PythonOperator(
        task_id="parseHttpPost",
        python_callable=_parseHttpPost
    )
    addToDatabase = PythonOperator(
        task_id="addToDatabase",
        python_callable=_addToDatabase
    )

[parseHttpGet,parseHttpPost] >> addToDatabase


# gestion des exceptions sur la fonction Fibonacci

def fibonacci():
    ''' fonction suite de Fibonacci '''
    try:
        print('Saisir un nombre entier supérieur à 1:')
        x=int(input())
        # l'utilisateur doit saisir un nombre supérieur à 1
        if x<=1:
            raise Exception("Vous devez saisir un nombre supérieur à 1.")
        # l'utilisateur doit saisir un nombre inférieur ou égal à 1000
        if x>1000:
            raise Exception("Vous avez saisi un nombre trop grand.")
        fibo=[0,1]
        for i in range(2,x):
            fibo.append(fibo[-1]+fibo[-2])

        print('La suite de Fibonacci pour '+str(x)+' est '+str(fibo))
    except ValueError:
        #  l'utilisateur doit saisir un nombre entier, les chaînes ne pouvant être converties sont refusées
        print("Vous devez saisir un nombre entier")

# test de la fonction
fibonacci()


from collections import defaultdict

class Solution:
    """
    This is a variation of longest common substring, we can solve it similar to LCS but space used will
    be massive.

    Also can solve this using Trie but is not most efficient

    1. Binary search on (1, len(s)) to find the longest repeating substring. If match found then try to find longer else find shorter. This will be our window

    2. Using Rabin-Karp hashing algo, find pattern of length = window

    3. Keep going until longest is found
    """

    def pattern_match(self, s: str, length: int, computed_array: "list[int]") -> bool:
        """
        use rabin karp algo to find patterns of len length and return true / false depending on
        whether there was a match or not

        'baa' = ( ( ( 2 * 26**2 ) % p + ( 1 * 26**1 ) ) % p + ( 1 * 26 ** 0 ) ) % p
        """

        hash_map = defaultdict(list)

        mod = 10000001

        hash_ = 0

        # calculate hash for the first window
        for i in range(length):
            hash_ = (hash_ * 26 + (ord(s[i]) - ord("a"))) % mod

        hash_map[hash_].append(0)

        # computing rolling hash here, so no need to keep adding hash
        for i in range(length, len(s)):
            hash_ = (
                (hash_ - computed_array[length - 1] * (ord(s[i - length]) - ord("a")))
                % mod
                + mod
            ) % mod

            # left shifting the hash
            hash_ = (hash_ * 26 + (ord(s[i]) - ord("a")) * 26 ** 0) % mod

            if hash_ in hash_map:
                for index in hash_map[hash_]:
                    string1 = s[index : index + length]
                    string2 = s[i - length + 1 : i + 1]

                    if string1 == string2:
                        return (True, string1)

            hash_map[hash_].append(i - length + 1)

        print(hash_map)

        return (False, "")

    def longestDupSubstring(self, s: str) -> str:
        if len(s) == 0:
            return ""

        low = 1
        high = len(s)
        longest_str = ""

        # {pattern of length n: [ indicies where the patterns start ]}
        computed_array = [26 ** i for i in range(len(s))]

        """
        use binary search so that if string is of len 10, then we try to find longest repeating substring of length
        5, and if that exists then we try to find LRS of length > 5 and < 10 and so on
        """

        while low <= high:  # needs to be low <= high
            middle = (low + high) // 2

            match, string = self.pattern_match(s, middle, computed_array)

            if match:
                low = middle + 1
                longest_str = max(longest_str, string, key=lambda x: len(x))
            else:
                high = middle - 1

        return longest_str

def rabin_karp():
    s = "aaaaaa"

    h = 0

    for (index, char) in enumerate(s[:3]):
        h += 26 ** (2 - index) + (ord(s[index]) - ord("a") + 1)
        print(h)

    print(h)

print(
    Solution().longestDupSubstring(
        "nyvzwttxsshphczjjklqniaztccdrawueylaelkqtjtxdvutsewhghcmoxlvqjktgawwgpytuvoepnyfbdywpmmfukoslqvdrkuokxcexwugogcwvsuhcziwuwzfktjlhbiuhkxcreqrdbj"
    )
)

# rabin_karp()


import torch
from torch import Tensor
import torch.nn.functional as F
from _utils_ import trg_mask

def first_prediction(image, encoder, decoder, conf, start=0):
    encoder.eval()
    decoder.eval()
    start_token = start
    feat = encoder(image)
    encoded, weights = decoder.encoder_att(feat=feat)

    outputs = Tensor([start_token]).long()

    trg_mask_ = trg_mask(outputs.unsqueeze(0))
    out = decoder.transformer(memory=encoded, trg=outputs.unsqueeze(0), trg_mask=trg_mask_)
    out = F.softmax(out, dim=-1)
    probs, ix = out[:, -1].data.topk(conf.k)
    log_scores = torch.log(probs)
    # log_scores = probs
    # print("log: ", log_scores)
    outputs = torch.zeros(conf.k, conf.max_dec_seq).long()
    outputs[:, 0] = start_token
    outputs[:, 1] = ix[0]
    encoded_outputs = torch.zeros(conf.k, encoded.size(-2), encoded.size(-1))
    encoded_outputs[:, :] = encoded[0]

    return outputs, encoded_outputs, log_scores, weights

def k_best_outputs(outputs, out, log_scores, i, k):
    probs, ix = out[:, -1].data.topk(k)
    # print("Top-K", out[:, :].data.topk(10))
    # E.g., out -> [[0.9, 0.8, 0.7], [0.7, 0.91, 0.89], [0.4, 0.98, 0.77]]
    log_probs = torch.log(probs) + log_scores
    # log_probs = probs + log_scores
    k_probs, k_ix = log_probs.view(-1).topk(k)
    # print("log prob: ", log_probs)
    row = k_ix // k
    col = k_ix % k
    # print("row, col: ", row, col)
    outputs[:, :i] = outputs[row, :i]
    outputs[:, i] = ix[row, col]

    log_scores = k_probs.unsqueeze(0)

    return outputs, log_scores

def beam_search(image, encoder, decoder, conf, end=1):
    outputs, en_outputs, log_scores, weights = first_prediction(image,
                                                       encoder,
                                                       decoder,
                                                       conf)
    end_token = end
    ind = None
    # print("Out::", outputs.size(), en_outputs.size(), log_scores.size())
    for i in range(2, conf.max_dec_seq):
        trg_mask_ = trg_mask(torch.LongTensor(i).unsqueeze(0))
        out = decoder.transformer(memory=en_outputs,
                                  trg=outputs[:, :i],
                                  trg_mask=trg_mask_)
        out = F.softmax(out, dim=-1)
        outputs, log_scores = k_best_outputs(outputs,
                                             out,
                                             log_scores,
                                             i,
                                             conf.k)
        none_end = torch.nonzero(outputs == end_token)
        sentence_lengths = torch.zeros(len(outputs)).long()

        for vec in none_end:
            i = vec[0]
            if sentence_lengths[i] == 0:
                sentence_lengths[i] = vec[1]

        num_finished_sentences = len([s for s in sentence_lengths if s > 0])
        if num_finished_sentences == conf.k:
            alpha = 0.7
            div = 1/(sentence_lengths.type_as(log_scores)**alpha)
            # print("div: ", log_scores)
            _, ind = torch.max(log_scores*div, 1)
            # print("Ind: ", ind)
            ind = ind.data[0]
            # ind = torch.argmax(log_scores, dim=-1).data[0]
            break

    if ind is None:
        length = torch.nonzero(outputs[0] == end_token)[0]
        sentence = [tok.item() for tok in outputs[0][1:length]]
        return sentence, weights
    else:
        length = torch.nonzero(outputs[ind] == end_token)[0]
        sentence = [tok.item() for tok in outputs[ind][1:length]]
        return sentence, weights


# #重点:
# 导入PANDAS, 利用PANDAS 的数据绘制关系图

# 如何自行输入经纬度绘制所在区域的地图

# 学习如何读取气象资讯档: netCDF4

# 作业
# 读入 etopo5 地形/降水量资料集: "etopo5.cdf". 绘制分布图.
# 延续使用者资料的导入使用, 除了, "文字档", "网络资讯", 还有图像档;
# 所以尝试使用一张"earth_lights_lrg.jpg" 搭配不同的投影方式绘制地图

from mpl_toolkits.basemap import Basemap, shiftgrid, cm
import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset

# 读入 etopo5 地形/降水量。.
#url = 'https://github.com/NOAA-PMEL/FerretDatasets/blob/master/data/etopo5.cdf'
# 要确认资料集所在路径
etopodata = Dataset('etopo5.cdf')
# 从这里开始写code
# 取得资料集里的资料定义, 使用variables.keys()
etopodata .variables.keys()
# dict_keys(['ETOPO05_X', 'ETOPO05_Y', 'ROSE'])

# # 汇入资料 使用variables['keys'][:]
lons = etopodata.variables['ETOPO05_X'][:]
topoin = etopodata.variables['ROSE'][:]
lats =  etopodata.variables['ETOPO05_Y'][:]

# # 移位资料, 使 lon 从 - 180 到 180, 而不是 20 到 380
topoin, lons = shiftgrid(180.,topoin ,lons ,start=False)

# # 绘制地形/降水量
# # 创建图形和轴实例
fig = plt.figure()
ax = fig.add_axes([0.1,0.1,0.8,0.8])

# #设定地理资讯图('lcc' = lambert conformal conic).
# # 使用 Wgs84 椭圆体的主要和次要球体半径, 做基础的底图
# # 请填入 resolution='??',area_thresh=???.,projection='??'
# # area_thresh 为一数字, 低于这个数字不呈现

m = Basemap(llcrnrlon=-145.5,llcrnrlat=1.,urcrnrlon=-2.566,urcrnrlat=46.352,\
             rsphere=(6378137.00,6356752.3142),\
             resolution='l',area_thresh=10000.,projection='lcc',\
             lat_1=50.,lon_0=-107.,ax=ax )

# # 转换为 nx x ny 定期分空 5km 本机投影网格
nx = int((m.xmax-m.xmin)/5000.)+1 ; ny = int((m.ymax-m.ymin)/5000.)+1
topodat = m.transform_scalar(topoin,lons,lats,nx,ny)
# # plot image over map with imshow.
im = m.imshow(topodat,cm.GMT_haxby)
# #划定海岸线和边境界限, 总共需要3个参数
m.drawcountries()
m.drawmapboundary()   # 绘制边界
m.drawcoastlines()    # 绘制海岸线
m.drawstates()        # 绘制州

# # 绘制经纬线
# # 设定地图左侧和底部的标签
parallels = np.arange(-90.,120.,30.)
m.drawparallels(parallels, labels = [1,0,0,1])
meridians = np.arange(0.,420.,60.)
m.drawmeridians(meridians,labels = [1,0,0,1] )

# # 加入 colorbar
cb = m.colorbar(im,"right", size="5%", pad='2%')
ax.set_title('ETOPO5 Topography - Lambert Conformal Conic')
plt.show()

#from __future__ import (absolute_import, division, print_function)

from mpl_toolkits.basemap import Basemap
import numpy as np
import matplotlib.pyplot as plt

# 说明使用warpimage method 方法显示图像背景。
# 在地图投影区域。 预设背景是‘蓝色。
#来自美国宇航局的大理石图像 (http://visibleearth.nasa.gov)

# 建立新图像
fig=plt.figure()
# 定义以北美为中心的正交投影.
m = Basemap(projection='ortho',lat_0=40,lon_0=-100,resolution='l')
# 显示非预设影像 - 汇入要使用的IMAGE
m.warpimage(image='earth_lights_lrg.jpg')
# 绘制海岸线。
m.drawcoastlines(linewidth=0.5,color='0.5')
# 每 30 度绘制一组 lat/lon 网格线。
m.drawmeridians(np.arange(0,360,30),color='0.5')
m.drawparallels(np.arange(-90,90,30),color='0.5')
#添加图示标题
plt.title("Lights at Night image warped from 'cyl' to 'ortho' projection",fontsize=12)
plt.show()
print('warp to orthographic map ...')

#从这里开始写code

# 建立新图像
fig = plt.figure()

#新定义圆柱形等距投影。
m = Basemap(projection='cyl',llcrnrlon=-180,llcrnrlat=-90,urcrnrlon=180,urcrnrlat=90,resolution='l')

# 绘图 (未扭曲) rgba 图像
im = m.bluemarble(scale=0.5)
m.warpimage(image='earth_lights_lrg.jpg')
# 绘制海岸线。

# 绘制经纬度网格
m.drawmeridians(np.arange(-180,180,60),labels=[0,0,0,1],color='0.5')

parallels = np.arange(-90.,120.,30.)
m.drawparallels(parallels, labels = [1,0,0,1])

# 输出图像标题
plt.title("Blue Marble image - native 'cyl' projection",fontsize=12)
print('plot native cylindrical map (no warping needed) ...')

plt.show()