#  Module that contains the implementation of the populations used in the algorithms
#
# Author: Fernando García <ga.gu.fernando@gmail.com>
#
# External dependencies
import numpy as np
from random import randrange
from copy import deepcopy
# Module dependencies
from .representation import Individual
from ..error.exceptions import *

class Population:
    """
    Class that represent the basic population implementation.
    """

    def __init__(self, size: int, min_feat_init: int = 2, max_feat_init: int = None):
        """
            __init__(size: int, min_feat_init: int = 2, max_feat_init: int = None)
        """
        self.min_feat_init = min_feat_init
        self.max_feat_init = max_feat_init
        self.size = size
        self._individuals = None
        self._features = None

    def __repr__(self):
        return f"Population(size={self.size}, min_feat_init={self.min_feat_init}, max_feat_init={self.max_feat_init})"

    def __str__(self):
        return self.__repr__()

    @property
    def length(self):
        """
        Returns the current size of the population (note that it is not the same as the size).

        Returns
        --------
        :return int
        """
        if self._individuals is None:
            return 0

        return len(self._individuals)

    @property
    def individuals(self):
        """
        Returns the individuals list consists on pywin.representation.Individual

        Returns
        ---------
        :return 1d-array
        """
        return self._individuals

    @property
    def fitness(self):
        """
        Returns the fitness value of all individuals in population

        Returns
        --------
        :return 1d-array
        """
        return [individual.fitness for individual in self._individuals]

    @property
    def features(self):
        """
        Return population features

        Returns
        --------
        :return dict
        """
        return self._features

    @features.setter
    def features(self, new_features: dict):
        """
        Select new population features.

        Parameters
        -----------
        :param new_features: list
        """
        if self.max_feat_init is None:
            self.max_feat_init = len(new_features)

        self._features = new_features

    @property
    def features_num(self):
        """
        Return features indices

        Returns
        --------
        :return list
        """
        return list(self._features.keys())

    def init(self, include_all: bool = True):
        """
        Initializes a random population.

        Parameters
        -----------
        :param include_all: bool
            Indicates if all features will be included in the population initialization.
        """
        self._individuals = self._population_init(self.size, include_all=include_all)

    def set_new_individuals(self, new_individuals: list):
        """
        Update population individuals.

        Parameters
        -----------
        :param new_individuals: list
        """
        self._individuals = new_individuals

    def init_features(self, features: list):
        """
        Function that assigns to the numbers with which the genes (features) in the algorithm are encoded
        the real name of the feature. If no features are provided, the default names will be the column number.

        Parameters
        -------------
        :param features: list
            Features in the same order as the columns of the predictor variables.
        """
        # Encode features in integers and save the names
        self._features = {n: col for n, col in enumerate(features)}

        if len(self._features) <= self.min_feat_init:
            warning = "Minimum number of features in the random initialization of the population" \
                      " incompatible with the total number of features."
            raise InconsistentParameters(warning)

        if self.max_feat_init is None:
            self.max_feat_init = len(self._features)

        if len(self._features) < self.max_feat_init:
            warning = "Maximum number of features in the random initialization of the population" \
                      " incompatible with the total number of features."
            raise InconsistentParameters(warning)

    def copy(self):
        """
        Return a deep copy of the current population.

        Returns
        --------
        :return Population
        """
        return deepcopy(self)

    def pop_individual(self, idx: int):
        """
        Take and remove an individual from population by index.

        Parameters
        -----------
        :param idx: int

        Returns
        --------
        :return pywin.population.representation.Individual
        """
        return self._individuals.pop(idx)

    def get_individual(self, idx: int):
        """
        Takes an individual from population by index.

        Parameters
        -----------
        :param idx: int

        Returns
        --------
        :return pywin.population.representation.Individual
        """
        return self._individuals[idx]

    def pop_random_individual(self):
        """
        Take and remove an individual from population randomly.

        Returns
        --------
        :return pywin.population.representation.Individual
        """
        idx = randrange(len(self._individuals))

        return self.pop_individual(idx)

    def get_random_individual(self):
        """
        Take an individual from population randomly without remove it.

        Returns
        --------
        :return pywin.population.representation.Individual
        """
        idx = randrange(len(self._individuals))

        return self.get_individual(idx)

    def generate_random_population(self, size: int):
        """
        This method generates a new random population with a given size.

        Parameters
        -----------
        :param size: int

        Returns
        --------
        :return Population
        """
        if size == 0:
            return None

        random_individuals = self._population_init(size=size, include_all=False)

        # Create new population
        random_population = Population(size=len(random_individuals))
        random_population.set_new_individuals(random_individuals)
        random_population.features = self.features

        return random_population

    def create_new_population(self, size: int, individuals: list):
        """
        Method that creates a new population using a given size and individuals.

        Parameters
        -----------
        :param size: int
        :param individuals: list
            List of :return pywin.population.representation.Individual

        Returns
        --------
        :return Population
        """
        new_population = Population(size=size, max_feat_init=self.max_feat_init, min_feat_init=self.min_feat_init)
        new_population.set_new_individuals(individuals)
        new_population.features = self.features

        return new_population

    @staticmethod
    def merge_population(population_1, population_2):
        """
        Method that takes two populations and mixes them.

        Parameters
        -----------
        :param population_1: Population
        :param population_2: Population

        Returns
        ---------
        :return Population
        """
        # Merge individuals and their fitness
        individuals = population_1.individuals + population_2.individuals

        # Sum population sizes
        size = population_1.size + population_2.size

        # Merge population features
        features = {**population_1.features, **population_2.features}

        # Create new population
        mixed_population = Population(size=size)

        # Assign individuals and their fitness
        mixed_population.set_new_individuals(individuals)

        # Assign features
        mixed_population.features = features

        return mixed_population

    def _population_init(self, size: int, include_all: bool = True, num_feats: int = None):
        """
        Function that generates a random population based on the size received as argument. By default, all features
        are included in the population unless the include_all variable is specified as false.

        Parameters
        -------------
        :param size: int
            Population size to be generated.
        :param include_all: bool
            True to include all features.
        :param num_feats: int
            Number of features that the returned population will have.

        Returns
        ----------
        :return: list
            [pywin.population.representation.Individual, ...]
        """
        initial_population = []

        all_feats = np.array(self.features_num)

        # It's possible to balance the number of features eliminated from the population providing a num_feats
        if num_feats is not None:
            num_genes = int(num_feats / size)

        for n in range(size):
            if num_feats is None:
                # A minimum individual size can be selected by user
                num_genes = randrange(self.min_feat_init, self.max_feat_init)

            # Individual initialization
            individual_feat = np.random.choice(all_feats, size=num_genes, replace=False)
            # Append to population
            initial_population.append(individual_feat)

        # For first initialization
        if include_all:
            # Check the features that are not in the population and add them
            features_used = np.concatenate(initial_population, axis=0)
            not_used_features = np.setdiff1d(all_feats, features_used)
            for feature in not_used_features:
                r_num = randrange(size)
                initial_population[r_num] = np.append(initial_population[r_num], [feature])

        return [Individual(individual_feat) for individual_feat in initial_population]

class Block:
    """
    A block defines a set of features that are grouped together and can be treated as a single feature in
    the population.
    """

    def __init__(self, structure: dict, features: list):
        """
            __init__(structure, features)

        Parameters
        -------------
        :param structure : dict
            Dictionary where the keys define the name of the block and the values correspond to the features that
            will be treated as a block.
        :param features: 1d-array
            Name of the columns of the original data. All must be present in the values of structure param.
        """
        self.block = {}
        for idx, feat in enumerate(features):
            # Get the block to which a certain feature belongs
            key, idx_feat = Block._get_key(structure, feat)

            # Annotate feature inside a block maintaining its original position in the data.
            if self.block.get(key, None) is None:
                self.block[key] = [(idx, structure[key][idx_feat])]
            else:
                self.block[key].append((idx, structure[key][idx_feat]))

    def __repr__(self):
        return f"Block(num_of_blocks:{len(self.block.keys())})"

    def __str__(self):
        return self.__repr__()

    def __eq__(self, other):
        """
        Comparision between two blocks.

        Parameters
        ------------
        :param other: object

        Returns
        --------
        :return bool
        """
        if isinstance(other, __class__):
            # Blocks content comparision
            if other.block == self.block:
                return True

        return False

    @property
    def blocks(self):
        """
        Return the blocks defined as keys in structure.

        Returns
        ---------
        :return list
            Block names (keys from structure)
        """
        return list(self.block.keys())

    def index(self, features: list):
        """
        Returns the indices where the features of the selected block are found in the original data.

        Parameters
        -------------
        :param features: 1d-array
            List of features (name of blocks = keys from structure).

        Returns
        ---------
        :return list
            Original position of the elements that form a block in the data.
        """
        return self._get_from_block(n=0, features=features)

    def values(self, features: list):
        """
        Returns the values (column names) where the features of the selected blocks are found in the original data.

        Parameters
        -------------
        :param features: 1d-array
            List of features (name of blocks = keys from structure).

        Returns
        ---------
        :return list
            Block feature names in the original data
        """
        return self._get_from_block(n=1, features=features)

    def _get_from_block(self, n, features):
        """
        Returns the names of the features that make up a block (n=0) or their index (m=1).

        Parameters
        ------------
        :param n: int
            n=1 to get the name or n=0 to get the indices.
        :param features: 1d-array

        Returns
        ----------
        :return elements: list
            Block members names or indices.
        """
        return [value[n] for feat in features for value in self.block[feat]]

    @staticmethod
    def _get_key(dic: dict, target_val: str):
        """
        Returns the name of the block to which a feature defined in structure belongs. If it cannot find the
        name, that is, not all the features defined in structure are present, it will throw an error.

        Parameters
        -----------
        :param dic: dict
            User-defined structure passed to the constructor
        :param target_val: str
            Target value

        Returns
        ---------
        :return str
            Block name.
        """
        target_key, target_idx = None, None
        for key, value in dic.items():
            for idx, feat in enumerate(value):
                # A feature is found inside a block
                if feat == target_val:
                    # If a feature is duplicated it will raise an error
                    if target_key is not None:
                        raise Exception("Duplicated value: %s and %s values "
                                        "must be unique." % (feat, target_key))
                    # Annotate feature and feature position
                    target_key, target_idx = key, idx

        # If feature hasn't been found
        if target_key is None:
            raise Exception("A feature from features is not present in dictionary (%s). "
                            "Please check that all values are present" % target_val)

        return target_key, target_idx

    @classmethod
    def merge(cls, block_1, block_2):
        """
        Merge two block.

        Parameters
        ------------
        :param block_1: Block
        :param block_2: Block

        Returns
        --------
        :return Block
        """
        new_block = {**block_1.block, **block_2.block}

        return new_block

class BlockPopulation(Population):
    """
    Class that uses a Block as features (see Block documentation).
    """

    def __init__(self, size: int, min_feat_init: int = 2, max_feat_init: int = None,
                 structure: dict = None, features: list = None, block: Block = None):
        """
        __init__(size: int, min_feat_init: int = 2, max_feat_init: int = None, structure: dict = None,
                    features: list = None, block: Block = None)

        Notes
        ------
        This population can be initialized from an instance of Block or directly from a structure (dictionary)
        and a list of features.
        """
        # Select maximum number of features in initialization
        if max_feat_init is None:
            max_feat_init = len(structure.keys())

        # Initialize population
        super().__init__(size=size, min_feat_init=min_feat_init, max_feat_init=max_feat_init)

        # Select features using a Block or a structure + list of features
        if not (structure is None and features is None):
            self._features = Block(structure=structure, features=features)
        else:
            self._features = block

    @property
    def features(self):
        """
        Return the features of an entire block

        Returns
        --------
        :return dict
        """
        return {n: block for n, block in enumerate(self._features.blocks)}

    @features.setter
    def features(self, new_features):
        """
        Select new features using a dictionary or a Block.

        Parameters
        -----------
        :param new_features: dict or Block
        """

        # Update max_feat_init (for security)
        if self.max_feat_init is None:
            self.max_feat_init = len(new_features)

        # If new features come as a dictionary, transform it in a Block
        # (implemented for pywin.wrapper.IslandModel compatibility)
        if isinstance(new_features, dict):
            self._features.block = new_features
        else:
            # If the new features come as a Block assign directly to features
            self._features = new_features

    @property
    def features_num(self):
        """
        Return features numerical indices.

        Returns
        ---------
        :return list
        """
        return [n for n in range(len(self._features.block))]

    @property
    def block(self):
        """
        Return a Block.

        Returns
        ---------
        :return Block
        """
        return self._features

    def init_features(self, features: list):
        """
        Method not implemented for this class.
        """
        pass

    def generate_random_population(self, size: int):
        """
        This method generates a new random population with a given size.

        Parameters
        -----------
        :param size: int

        Returns
        --------
        :return BlockPopulation
        """
        if size == 0:
            return None

        # Create new population
        random_population = BlockPopulation(size=size, block=self._features,
                                            min_feat_init=self.min_feat_init,
                                            max_feat_init=self.max_feat_init)

        # Initializes the population individuals
        random_population.init(include_all=False)

        return random_population

    def create_new_population(self, size: int, individuals: list):
        """
        Method that creates a new population using a given size and individuals.

        Parameters
        -----------
        :param size: int
        :param individuals tuple
            ([features], fitness)

        Returns
        --------
        :return BlockPopulation
        """
        # Create population
        new_population = BlockPopulation(size=size, block=self._features,
                                         max_feat_init=self.max_feat_init,
                                         min_feat_init=self.min_feat_init)
        # Set individuals
        new_population.set_new_individuals(individuals)

        return new_population

    @staticmethod
    def merge_population(population_1, population_2):
        """
    Method that takes two populations and mixes them.

        Parameters
        -----------
        :param population_1: BlockPopulation
        :param population_2: BlockPopulation

        Returns
        ---------
        :return BlockPopulation
        """
        # Merge individuals and their fitness
        individuals = population_1.individuals + population_2.individuals

        # Sum population sizes
        size = population_1.size + population_2.size

        # Merge population features
        if not population_1.block == population_2.block:
            new_block = Block.merge(population_1.block, population_2.block)
        else:
            new_block = population_1.block

        # Create a new population
        mixed_population = BlockPopulation(size=size, block=new_block,
                                           min_feat_init=population_1.min_feat_init,
                                           max_feat_init=population_1.max_feat_init)

        # Assign individuals and their fitness
        mixed_population.set_new_individuals(individuals)

        return mixed_population


from flask import request, render_template, abort, send_from_directory
import flask  # For creating a web application interface
from Crypto.Hash import SHA256
import sys
sys.path.append("/root/projs/APathThroughDust")
from util.DBCommand import getDBConnection, RegUser2DB, getPubKey
import os
import util.rsaSignVerify as rsautil

app = flask.Flask(__name__)

# Welcome page
@app.route('/', methods=['GET'])
def welcome():
    return render_template("welcome.html")

# User register
@app.route('/register', methods=['GET', 'POST'])
def register():
    if request.method == 'POST':
        username = request.form['username']
        usertype = request.form['usertype']
        duration = request.form['duration']
        cost = 0
        '''
        print(f"Username: {username}")
        print(f"User Type: {usertype}")
        print(f"Duration: {duration}")
        '''
        if not username or not usertype or not duration:
            abort(400)  # 触发400错误，跳转到错误页面
        elif username.__len__() > 15:
            render_template("用户名过长，请重试")
            return render_template("register.heml")
        if usertype == 'personal':
            cost = int(duration)
        else:
            cost = int(duration) * 1000
        return render_template('ConfirmRegister.html', username=username, usertype=usertype, duration=duration,
                               cost=cost)
    return render_template('register.html')

'''
确定注册（缴费）只能跳转，不能直接访问
生成密钥对
向数据库写入用户信息
    昵称--VARCHAR(50)
    注册时间--date
    公钥--BINARY(1500)
    用户类型--INT
        1：个人用户
        2：企业用户
    有效时间--INT

提示用户下载私钥（privatekeyURL）
'''

@app.route('/confirmReg', methods=['POST'])
def confirm():
    keypem, pubkeypem = rsautil.rsaKeyPairGen()
    username = request.form['username']
    usertype = request.form['usertype']
    duration = request.form['duration']
    if (not RegUser2DB(username, usertype, duration, pubkeypem)):
        return "注册出现错误"

    # 将私钥写入临时文件
    filePath = 'Resources/TmpKeys/' + username + 'privatekey.pom'  # 私钥临时文件的相对地址
    keyfile = open(filePath, 'wb')
    keyfile.write(keypem)
    keyfile.close()

    temp_url = 'localhost:5000/tempkeys/' + username + 'privatekey.pom'
    return render_template("postRegister.html", privatekeyURL=temp_url)

'''
私钥下载页面
'''
@app.route('/tempkeys/<filename>', methods=['GET'])
def serve_temp_file(filename):
    # 临时目录
    #print('进入私钥下载流程')
    relativeFilePathtoProj = 'Resources/TmpKeys/' + filename
    abPath=os.path.abspath("/root/projs/APathThroughDust")
    # 发送文件
    return send_from_directory(abPath+'/', relativeFilePathtoProj, as_attachment=True)

''''
实名认证页面
向保存实名信息的数据库写入信息
    昵称--VARCHAR(50),
    真实姓名 VARCHAR(50),
    真实姓名+身份证的SHA256哈希值 BINARY(32),

'''
@app.route('/RealNameAuth',methods=['GET', 'POST'])
def realNameAuthentication():
    if request.method=='GET':
        return render_template('realNameAuth.html')

    #从表单中提取信息
    print('收到实名认证POST请求')
    nickName = request.form['nickname']
    realName = request.form['real_name']
    idNum    = request.form['id_number']
    sig      = request.form['signature']

    #print(nickName)
    #print(realName)
    #print(idNum)

    #确认昵称已经注册
    if(not is_nickname_registered(nickName)):
        return "还未注册，请返回主界面注册"
    pubKey = getPubKey(nickName)
    if(rsautil.verify_sign(realName+"+"+idNum,sig,pubKey)):
        __insert_user_real_name(nickName,realName,idNum)
        return "注册成功"
    return "签名错误，请确认私钥无误后重新提交实名认证申请"

# 确认昵称是否被注册
def is_nickname_registered(nickname):
    # 连接到MySQL数据库
    connection = getDBConnection()

    try:
        with connection.cursor() as cursor:
            # 查询昵称是否存在
            sql = "SELECT COUNT(*) as count FROM ChainUser WHERE nickname = %s"
            cursor.execute(sql, (nickname,))
            result = cursor.fetchone()

            count = result['count']
            if count > 0:
                return True
            else:
                return False

    finally:
        # 关闭数据库连接
        connection.close()

# 向实名信息数据库中新增行
def __insert_user_real_name(nickname, real_name, id_number):
    # 计算真实姓名和身份证号的SHA256哈希值
    h = SHA256.new()
    h.update(bytearray(real_name + id_number))
    hash_value = h.digest()

    # 连接到MySQL数据库
    connection = getDBConnection()

    try:
        with connection.cursor() as cursor:
            # 插入用户实名信息
            sql = "INSERT INTO UserRealName (nickname, real_name, hash_value) VALUES (%s, %s, %s)"
            cursor.execute(sql, (nickname, real_name, hash_value))

        # 提交事务
        connection.commit()
        print("用户实名信息已成功插入数据库！")

    finally:
        # 关闭数据库连接
        connection.close()

# Running the Web App
app.run(host='0.0.0.0', port=5000)


from bs4 import BeautifulSoup as bS
import requests
import pandas as pd

s = requests.Session()
BASE_URL = "https://www.fifaindex.com"

def get_basic_data(player_html):
    nat = player_html.find("td", {"data-title": "Nationality"}).find("a").get("title")
    ovr_span, pot_span = player_html.find("td", {"data-title": "OVR / POT"}).find_all("span")
    ovr, pot = int(ovr_span.text), int(pot_span.text)
    name = player_html.find("td", {"data-title": "Name"}).find("a").get("title")[:-8]
    positions = '/'.join([position.text for position in player_html.find("td", {"data-title": "Preferred Positions"}).find_all("a")])
    age = int(player_html.find("td", {"data-title": "Age"}).text)
    team = player_html.find("td", {"data-title": "Team"}).find("a").get("title")[:-8]

    return [name, nat, ovr, pot, positions, age, team]

def get_additional_data(profile):
    row = []

    first_card_body = profile.find("div", {"class": "col-lg-8"}).find("div", {"class": "card mb-5"}).find("div", {"class": "card-body"})
    for j, p in enumerate(first_card_body.find_all("p")):
        if j in [3, 4, 5, 10, 11, 13, 14]:
            continue
        if j in [0, 1]:
            row.append(int(p.find("span", {"class": "data-units-metric"}).text[:-3]))
        if j == 2:
            row.append(p.find("span").text)
        if j == 6:
            row.append(p.find("span").text.replace(" ", ""))
        if j in [7, 8]:
            row.append(len(p.find_all("i", {"class": "fas"})))
        if j in [9, 12]:
            row.append(int(p.find("span", {"class": "float-right"}).text[1:].replace(".", "")))

    other_cards = profile.find("div", {"class": "row grid"}).find_all("p")
    for p in other_cards:
        if p.text.startswith("Composure"):
            continue
        num = p.find("span", {"class": "float-right"})
        if num:
            row.append(int(num.text))

    return row

def scrape_data():
    players = []
    clubs = set()
    for fifa in range(21, 22):
        i = 1
        print(fifa)
        while True:
            print(i)

            url = f"{BASE_URL}/players/fifa{fifa}/{i}/?league=19&order=desc"
            page_html = s.get(url).text
            page_parsed = bS(page_html, 'html.parser')
            table = page_parsed.find("table", {"class": "table table-striped table-players"})

            if table is None:
                break

            for player in table.find("tbody").find_all("tr"):
                if not player.has_attr("data-playerid"):
                    continue

                player_data = [fifa] + get_basic_data(player)
                clubs.add(player.find("td", {"data-title": "Team"}).find("a").get("href"))

                profile_url = f'{BASE_URL}{player.find("td", {"data-title": "Name"}).find("a").get("href")}'
                profile_parsed = bS(s.get(profile_url).text, 'html.parser')

                player_data.extend(get_additional_data(profile_parsed))

                players.append(player_data)

            i += 1

    return players, clubs

def scrape_clubs(clubs):
    clubs_rows = []
    for club in clubs:
        club_row = [club.split("/")[-2][-2:]]
        club_url = f'{BASE_URL}{club}'
        club_parsed = bS(s.get(club_url).text, 'html.parser')

        pl = club_parsed.find("div", {"class": "pl-3"})
        club_row.append(pl.find("h1").text[:-8])
        stars = len(pl.find_all("i", {"class": "fas fa-star fa-lg"})) + 0.5 * len(pl.find_all("i", {"class": "fas fa-star-half-alt fa-lg"}))
        club_row.append(stars)

        main_card = club_parsed.find("div", {"class": "col-lg-8"}).find("div", {"class": "card mb-5"})
        for j, li in enumerate(main_card.find_all("li")[0:5]):
            if j == 4:
                club_row.append(li.find("span").text[1:].replace(".", ""))
            else:
                club_row.append(li.find("span").text)
        clubs_rows.append(club_row)

    return clubs_rows

def create_players_df(players):
    df = pd.DataFrame(players, columns=["FIFA", "Name", "Nationality", "OVR", "POT", "Positions", "Age", "Team",
                                        "Height", "Weight", "Foot", "WorkRate", "WeakFoot", "SkillMoves", "Value", "Wage",
                                        'Ball_Control', 'Dribbling', 'Marking', 'Slide_Tackle', 'Stand_Tackle', 'Aggression',
                                        'Reactions', 'Att._Position', 'Interceptions', 'Vision', 'Crossing',
                                        'Short_Pass', 'Long_Pass', 'Acceleration', 'Stamina', 'Strength', 'Balance', 'Sprint_Speed',
                                        'Agility', 'Jumping', 'Heading', 'Shot_Power', 'Finishing', 'Long_Shots', 'Curve', 'FK_Acc.',
                                        'Penalties', 'Volleys', 'GK_Positioning', 'GK_Diving', 'GK_Handlin', 'GK_Kicking', 'GK_Reflexes'])

    df.to_csv("data/players_fifa.csv")

def create_clubs_df(clubs):
    df = pd.DataFrame(clubs, columns=["Fifa", "Name", "Rating", "RivalTeam", "Attack", "Midfield", "Defence", "TransferBudget"])
    df.to_csv("data/clubs_fifa_2021.csv")

if __name__ == "__main__":

    players_data, clubs_list = scrape_data()
    clubs_data = scrape_clubs(clubs_list)

    create_players_df(players_data)
    create_clubs_df(clubs_data)


#-*- coding:utf-8 -*-
#! /usr/bin/python
import time
from datetime import date, timedelta

class Time():

    def localtime(self, timeformat='%m/%d/%Y'):
        # 日期格式月日年
        localtime = time.localtime()
        localtime = time.strftime(timeformat, localtime)
        return localtime

    def get_day_of_day(self, n=0, timeformat='%m/%d/%Y'):
        if (n<0):
            n = abs(n)
            a = str(date.today()-timedelta(days=n))
            timea = time.strptime(a, '%Y-%m-%d')
            timea = time.strftime(timeformat, timea)
            return timea
        else:
            a = str(date.today()+timedelta(days=n))
            timea = time.strptime(a, '%Y-%m-%d')
            timea = time.strftime(timeformat, timea)
            return timea

if __name__ == '__main__':
    Time = Time()
    print(Time.localtime())
    print(Time.get_day_of_day(90))


"""
This file implements all classes needed to parse boolean expression.
Here is the definition of our parser:
    + Our alphabet is composed with:
        - [a-zA-Z] letters
        - &, |, , !, ( and ) symbols
        - white space

    + Our language is composed by 3 types of words:
        - '!', '&&' and '||' operator
        - '(' and ')' parenthesis
        - [a-zA-Z]+ words

    + Our grammar is defined here:
        S::= Expression
        Expression::= SubExpression
                      | SubExpression '||' Expression  Operator: OR
                      | SubExpression '&&' Expression  Operator: AND
        SubExpression::= '('Expression')'
                         | '!'SubExpression            Operator: NOT
                         | RoleName
        RoleName::= [a-zA-Z]+
"""

class ParsingError(Exception):
    """This is a custom exception raised when a parsing error is detected."""

class Node(object):
    """This class defines a node contains in our tree built from parsing."""

    def eval(self, index):
        """
        This function to implement defines how we should eval the current node.
        + params:
            - index: ReverseIndex object where we look for results.
        + returns:
            a set containing all posting_ids corresponding to the boolean expression given:
            {posting_ids ...}
        """
        raise NotImplementedError

class UnaryNode(Node):
    """This class defines a node which has one child."""

    def __init__(self, expression):
        """self.expression contains the child node."""
        self.expression = expression

class BinaryNode(Node):
    """This class defines a node which has two children."""

    def __init__(self, left_expression, right_expression):
        """self.left_expression and self.right_expression contains the two children node."""
        self.left_expression = left_expression
        self.right_expression = right_expression

class AndNode(BinaryNode):
    """This class defines the AND operator logic in our tree."""

    def eval(self, index):
        """
        Implements the and eval function.
        This is the intersection between the result of eval function of the two children nodes.
        """
        return self.left_expression.eval(index).intersection(self.right_expression.eval(index))

    def __str__(self):
        return '({} && {})'.format(self.left_expression, self.right_expression)

class OrNode(BinaryNode):
    """This class defines the OR operator logic in our tree."""

    def eval(self, index):
        """
        Implements the or eval function.
        This is the union between the result of eval function of the two children nodes.
        """
        return self.left_expression.eval(index).union(self.right_expression.eval(index))

    def __str__(self):
        return '({} || {})'.format(self.left_expression, self.right_expression)

class NotNode(UnaryNode):
    """This class defines the NOT operator logic in our tree."""

    def eval(self, index):
        """
        Implements the not eval function.
        This is the difference between:
            - all posting_ids in index
            - the result of eval function of the child node.
        """
        all_posting_ids = {
            posting_id
            for term_id in index.keys()
            for posting_id, _ in index[term_id][1]
        }
        return all_posting_ids.difference(self.expression.eval(index))

    def __str__(self):
        return '!({})'.format(self.expression)

class RoleNode(Node):
    """This class defines the logic for leaves in our tree."""

    def __init__(self, role_name):
        """self.role_name is the value of the leaf (a word)"""
        self.role_name = role_name

    def eval(self, index):
        """
        Implements the eval function for RoleNode.
        Return all posting_ids associated to self.role_name.
        """
        return {posting_id for posting_id, _, in index[self.role_name][1]}

    def __str__(self):
        return str(self.role_name)

class Tokenizer(object):
    """This class defines all the logic to tokenize a boolean expression."""

    @staticmethod
    def tokenize(text):
        """
        This methods tokenizes the given text input.
        + params:
            - text: string to tokenize
        + returns:
            a list of tokens
        """
        tokenized_text = []
        current_token = ''

        for char in text:
            if char == ' ':
                continue

            if char in '!()':
                if len(current_token) > 0:
                    tokenized_text.append(current_token)
                    current_token = ''
                tokenized_text.append(char)
                continue

            if char in '|&':
                if len(current_token) > 0:
                    if char == current_token[-1]:
                        tokenized_text.append(current_token[:-1]) if current_token[:-1] else None
                        current_token = ''
                        tokenized_text.append(char * 2)
                        continue

            current_token += char
        tokenized_text.append(current_token) if current_token else None

        return tokenized_text

class BooleanParser(object):
    """This class contains the logic to build a tree given a tokenized expression."""
    index = 0

    @staticmethod
    def parse(tokens):
        """
        Method to call to build tree from tokenized expression.
        Reset BooleanParser.index to 0 and build the tree for the given token list.
        + params:
            - tokens: tokenized expression [token ...]
        + returns:
            root Node of our tokenized expression
        """
        BooleanParser.index = 0
        return BooleanParser.parse_exp(tokens)

    @staticmethod
    def parse_exp(tokens):
        """This methods implements Expression rule."""
        left_expression = BooleanParser.parse_subexp(tokens)

        if BooleanParser.index >= len(tokens):
            return left_expression

        elif tokens[BooleanParser.index] == ')':
            return left_expression

        current_token = tokens[BooleanParser.index]

        if current_token == '&&':
            BooleanParser.index += 1
            right_expression = BooleanParser.parse_exp(tokens)
            return AndNode(left_expression, right_expression)

        elif current_token == '||':
            BooleanParser.index += 1
            right_expression = BooleanParser.parse_exp(tokens)
            return OrNode(left_expression, right_expression)

        raise ParsingError('Expected \'&&\' or \'||\' or EOF')

    @staticmethod
    def parse_subexp(tokens):
        """This method implements SubExpression rule."""
        current_token = tokens[BooleanParser.index]

        if current_token == '(':
            BooleanParser.index += 1
            expression = BooleanParser.parse_exp(tokens)

            if tokens[BooleanParser.index] != ')':
                raise ParsingError('Expected \')\'')

            BooleanParser.index += 1
            return expression

        elif current_token == '!':
            BooleanParser.index += 1
            expression = BooleanParser.parse_subexp(tokens)
            return NotNode(expression)

        BooleanParser.index += 1
        return RoleNode(current_token)


import os
import responses
from functools import partial

from smtp2go.core import Smtp2goClient
from smtp2go.settings import API_ROOT, ENDPOINT_SEND

SEND_ENDPOINT = API_ROOT + ENDPOINT_SEND
TEST_API_KEY = 'testapikey'
HEADERS = {
    'X-Ratelimit-Remaining': '250',
    'X-Ratelimit-Limit': '250',
    'X-Ratelimit-Reset': '37'
}

PAYLOAD = {
    'sender': 'dave@example.com',
    'recipients': ['matt@example.com'],
    'subject': 'Trying out smtp2go',
    'text': 'Test message',
    'html': '
Test Message
'
}
SUCCESSFUL_RESPONSE_BODY = {
    "request_id": "aa253464-0bd0-467a-b24b-6159dcd7be60",
    "data": {
        "succeeded": 1,
        "failed": 0,
        "failures": []
    }
}
FAILED_RESPONSE_BODY = {
    "request_id": "aa253464-0bd0-467a-b24b-6159dcd7be60",
    "data": {
        "succeeded": 0,
        "failed": 1,
        "failures": [
            'The API Key passed was not in the correct format, Please '
            'check the key is correct and try again, The full API key can '
            'be found in the API Keys section in the admin console.'
        ]
    }
}

class EnvironmentVariableContextManager():
    """
    Context manager for creating a temporary environment variable.
    """
    def __init__(self, key, value):
        if not value:
            value = ''
        self.key = key
        self.new_value = value

    def __enter__(self):
        # sets the environment variable and saves the old value:
        self.old_value = os.environ.get(self.key)
        os.environ[self.key] = self.new_value

    def __exit__(self, *args):
        # resets environment variable or deletes it:
        if self.old_value:
            os.environ[self.key] = self.old_value
        else:
            del os.environ[self.key]

@responses.activate
def get_response(endpoint, successful=True, status_code=200,
                 headers=None, payload=None):
    with EnvironmentVariableContextManager('SMTP2GO_API_KEY', TEST_API_KEY):
        if not payload:
            payload = PAYLOAD
        # Mock out API Endpoint:
        body = SUCCESSFUL_RESPONSE_BODY if successful else FAILED_RESPONSE_BODY
        responses.add(responses.POST, endpoint, json=body, status=status_code,
                      content_type='application/json',
                      adding_headers=headers)
        client = Smtp2goClient()
        response = client.send(**payload)
        return response

get_successful_response = partial(
    get_response, SEND_ENDPOINT, successful=True,
    status_code=200, headers=HEADERS)
get_failed_response = partial(
    get_response, SEND_ENDPOINT, successful=False,
    status_code=400, headers=HEADERS)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Filmes',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('TituloFilme', models.CharField(max_length=100, null=True, verbose_name=b'Titulo do Filme')),
                ('Genero', models.CharField(max_length=100, null=True, verbose_name=b'Genero do Filme')),
                ('Categoria', models.CharField(max_length=100, null=True, verbose_name=b'Categoria do Filme')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]


import argparse
import os
import torch
import numpy as np

from exp.exp_informer import Exp_Informer

parser = argparse.ArgumentParser(description='[Informer] Long Sequences Forecasting')

parser.add_argument('--model', type=str, required=True, default='informer',help='model of the experiment')

parser.add_argument('--data', type=str, required=True, default='ETTh1', help='data')
parser.add_argument('--root_path', type=str, default='./data/ETT/', help='root path of the data file')
parser.add_argument('--data_path', type=str, default='ETTh1.csv', help='location of the data file')
parser.add_argument('--features', type=str, default='M', help='features [S, M]')
parser.add_argument('--target', type=str, default='OT', help='target feature')

parser.add_argument('--seq_len', type=int, default=96, help='input series length')
parser.add_argument('--label_len', type=int, default=48, help='help series length')
parser.add_argument('--pred_len', type=int, default=24, help='predict series length')
parser.add_argument('--enc_in', type=int, default=7, help='encoder input size')
parser.add_argument('--dec_in', type=int, default=7, help='decoder input size')
parser.add_argument('--c_out', type=int, default=7, help='output size')
parser.add_argument('--d_model', type=int, default=512, help='dimension of model')
parser.add_argument('--n_heads', type=int, default=8, help='num of heads')
parser.add_argument('--e_layers', type=int, default=3, help='num of encoder layers')
parser.add_argument('--d_layers', type=int, default=2, help='num of decoder layers')
parser.add_argument('--d_ff', type=int, default=1024, help='dimension of fcn')
parser.add_argument('--factor', type=int, default=5, help='prob sparse factor')

parser.add_argument('--dropout', type=float, default=0.05, help='dropout')
parser.add_argument('--attn', type=str, default='prob', help='attention [prob, full]')
parser.add_argument('--embed', type=str, default='fixed', help='embedding type [fixed, learned]')
parser.add_argument('--activation', type=str, default='gelu',help='activation')
parser.add_argument('--output_attention', action='store_true', help='control attention output')
parser.add_argument('--num_workers', type=int, default=0, help='data loader num workers')

parser.add_argument('--itr', type=int, default=1, help='each params run iteration')
parser.add_argument('--train_epochs', type=int, default=6, help='train epochs')
parser.add_argument('--batch_size', type=int, default=32, help='input data batch size')
parser.add_argument('--patience', type=int, default=3, help='early stopping patience')
parser.add_argument('--learning_rate', type=float, default=0.0001, help='optimizer learning rate')
parser.add_argument('--des', type=str, default='test',help='exp description')
parser.add_argument('--loss', type=str, default='mse',help='loss function')
parser.add_argument('--lradj', type=str, default='type1',help='adjust learning rate')

parser.add_argument('--use_gpu', type=bool, default=True, help='use gpu')
parser.add_argument('--gpu', type=int, default=0, help='gpu')

parser.add_argument('--seq_diff', type=int, default=1)
parser.add_argument('--test_seq_diff', type=int, default=48)
parser.add_argument('--keep_ratio', type=float, default=0.8)
parser.add_argument('--target_res', type=str, default='30min', help='30min/6-hour/day/all')
parser.add_argument('--with_ts_delta', action='store_true')
parser.add_argument('--no_input_mask', dest='with_input_mask', action='store_false', help='set if include mask in input')
parser.add_argument('--single_input_res', action='store_true', help='only the target resolution in input')
parser.add_argument('--resolution_type', type=str, default='agg')

parser.add_argument('--seed', type=int, default=-1)
parser.add_argument('--expname_suffix', type=str, default='')

parser.add_argument('--load_ckpt', type=str, default='')
parser.add_argument('--notrain', action='store_true')

parser.add_argument('--chrono_arr_cats', type=str, default='mdwh')

args = parser.parse_args()

# initialize seeds
if args.seed > 0:
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)

data_parser = {
    'ETTh1':{'data':'ETTh1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]},
    'ETTh2':{'data':'ETTh2.csv','T':'OT','M':[7,7,7],'S':[1,1,1]},
    'ETTm1':{'data':'ETTm1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]},
    'NYCTaxi_30d-7d-1stride':{'data':'sampled_dense_multires_seqs_30d-7d_diff1d.npz','T':None,'M':[2,2,2],'S':[1,1,1]},
    'NYCTaxi_48h-24h-1stride':{'data':'sampled_dense_multires_seqs_48h-24h_diff1d.npz','T':None,'M':[2,2,2],'S':[1,1,1]},
    'NYCTaxi_30d-7d':{'data':'sampled_dense_multires_seqs_30d-7d_diff1_node2.npz','T':None,'M':[2,2,2],'S':[1,1,1]},
    'NYCTaxi_48h-24h':{'data':'sampled_dense_multires_seqs_48h-24h_diff1_node2.npz','T':None,'M':[2,2,2],'S':[1,1,1]},
    'NYCTaxi_7d-7d-1stride':{'data':'sampled_dense_multires_seqs.npz','T':None,'M':[2,2,2],'S':[1,1,1]},
    'NYCTaxi_Multires_6h-1d':{'data':None,'T':None,'M':[1404,156,156],'S':None},
    'NYCTaxiGreen_Multires_6h-1d':{'data':None,'T':None,'M':[88*9*2,88*2,88*2],'S':None},
    'NYCTaxiFHV_Multires_6h-1d':{'data':None,'T':None,'M':[1404,156,156],'S':None},
    'SolarEnergy_Multires_6h-1d':{'data':None,'T':None,'M':[1323,147,147],'S':None},
    'SolarEnergy10min_Multires_1h-6h':{'data':None,'T':None,'M':[1323,147,147],'S':None},
    'SolarEnergyAr_Multires_6h-1d':{'data':None,'T':None,'M':[96*9,96,96],'S':None},
    'METR-LA_Multires_1h-4h':{'data':None,'T':None,'M':[9*228,228,228],'S':None},
    'PEMS-BAY_Multires_1h-4h':{'data':None,'T':None,'M':[9*357,357,357],'S':None},
    'ECL_Multires_6h-1d':{'data':None,'T':None,'M':[9*321,321,321],'S':None}
}

input_res_num = 3
if args.single_input_res:
    input_res_num = 1
output_res_num = 1
if args.target_res == 'all':
    output_res_num = 3
input_dim_each_f_n = (1 + int(args.with_ts_delta) + int(args.with_input_mask)) * input_res_num
output_dim_each_f_n = output_res_num

for dataname in ['NYCTaxi_Multires_6h-1d', 'NYCTaxiGreen_Multires_6h-1d', 'NYCTaxiFHV_Multires_6h-1d', 'SolarEnergy_Multires_6h-1d', 'SolarEnergy10min_Multires_1h-6h', 'SolarEnergyAr_Multires_6h-1d', 'METR-LA_Multires_1h-4h', 'PEMS-BAY_Multires_1h-4h', 'ECL_Multires_6h-1d']:
    f_n_num = data_parser[dataname]['M'][2]
    data_parser[dataname]['M'] = [f_n_num * input_dim_each_f_n, f_n_num * output_dim_each_f_n, f_n_num * output_dim_each_f_n]

# if not args.with_ts_delta:
#     data_parser['NYCTaxi_Multires_6h-1d']['M'][0] = 936
#     data_parser['NYCTaxiGreen_Multires_6h-1d']['M'][0] = 88*6*2
#     data_parser['NYCTaxiFHV_Multires_6h-1d']['M'][0] = 936
#     data_parser['SolarEnergy_Multires_6h-1d']['M'][0] = 882
#     data_parser['SolarEnergy10min_Multires_1h-6h']['M'][0] = 882
#     data_parser['SolarEnergyAr_Multires_6h-1d']['M'][0] = 6*96
#     data_parser['METR-LA_Multires_1h-4h']['M'][0] = 6*228
#     data_parser['PEMS-BAY_Multires_1h-4h']['M'][0] = 6*357
#     data_parser['ECL_Multires_6h-1d']['M'][0] = 6*321

# if args.target_res == 'all':
#     data_parser['NYCTaxi_Multires_6h-1d']['M'][1:3] = [468,468]
#     data_parser['NYCTaxiGreen_Multires_6h-1d']['M'][1:3] = [88*3*2,88*3*2]
#     data_parser['NYCTaxiFHV_Multires_6h-1d']['M'][1:3] = [468,468]
#     data_parser['SolarEnergy_Multires_6h-1d']['M'][1:3] = [441,441]
#     data_parser['SolarEnergy10min_Multires_1h-6h']['M'][1:3] = [441,441]
#     data_parser['SolarEnergyAr_Multires_6h-1d']['M'][1:3] = [3*96,3*96]
#     data_parser['METR-LA_Multires_1h-4h']['M'][1:3] = [3*228,3*228]
#     data_parser['PEMS-BAY_Multires_1h-4h']['M'][1:3] = [3*357,3*357]
#     data_parser['ECL_Multires_6h-1d']['M'][1:3] = [3*321,3*321]
#     if (args.seq_len < 48) or (args.label_len < 48):
#         data_parser['PEMS-BAY_Multires_1h-4h']['M'][1:3] = [2*357,2*357]
#         data_parser['PEMS-BAY_Multires_1h-4h']['M'][0] = 4*357

# if args.resolution_type == 'sample':
#     data_parser['NYCTaxi_Multires_6h-1d']['M'][0] = 2*156
#     data_parser['SolarEnergy_Multires_6h-1d']['M'][0] = 2*147
#     data_parser['SolarEnergyAr_Multires_6h-1d']['M'][0] = 2*96
#     data_parser['METR-LA_Multires_1h-4h']['M'][0] = 2*228
#     data_parser['PEMS-BAY_Multires_1h-4h']['M'][0] = 2*357

if args.data in data_parser.keys():
    data_info = data_parser[args.data]
    args.data_path = data_info['data']
    args.target = data_info['T']
    print(data_info[args.features])
    args.enc_in, args.dec_in, args.c_out = data_info[args.features]

Exp = Exp_Informer

for ii in range(args.itr):
    setting = '{}_{}_ft{}_sl{}_ll{}_pl{}_dm{}_nh{}_el{}_dl{}_df{}_at{}_eb{}_{}_{}'.format(args.model, args.data, args.features,
                args.seq_len, args.label_len, args.pred_len,
                args.d_model, args.n_heads, args.e_layers, args.d_layers, args.d_ff, args.attn, args.embed, args.des, ii)

    exp = Exp(args)

    if not args.notrain:
        print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting))
        exp.train(setting)

    print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting))
    exp.test(setting)


from collections import Counter

import spacy
from nltk import word_tokenize, SnowballStemmer
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer
from textblob import TextBlob
from wordcloud import WordCloud
import many_stop_words
import languages
import re
from spacy import displacy
import xx_ent_wiki_sm
import en_core_web_sm
import en_core_web_lg

import topics

def get_stopwords(language_code: str, extra_stopwords: {str}) -> {str}:
    available_languages = set(many_stop_words.available_languages)
    if language_code in available_languages:
        my_stopwords = many_stop_words.get_stop_words(language_code)
        return my_stopwords.union(extra_stopwords)

def stem_text(text: str, lang_code: str) -> [str]:
    if lang_code in languages.languages.keys():
        tokens = word_tokenize(text)
        stemmer = SnowballStemmer(languages.languages[lang_code])
        stems = [stemmer.stem(token) for token in tokens]
        return stems
    return []

def get_cloud(text: str, page_name: str, language_code: str, personalized_stopwords: {str}):
    available_languages = set(many_stop_words.available_languages)
    if language_code in available_languages:
        # my_stopwords = many_stop_words.get_stop_words(language_code)
        wordcloud = WordCloud(stopwords=personalized_stopwords, include_numbers=True, width=600, height=400).generate(
            text)
    else:
        wordcloud = WordCloud(stopwords=personalized_stopwords, include_numbers=True).generate(text)

    png_title = re.sub(r'[\W\s]+', '_', page_name)
    if png_title[-1] == '_':
        png_title = png_title[:-1]

    path = f'static/images/{png_title}.png'
    wordcloud.to_file(path)
    return path

def get_sentiment_scores(text: str) -> (float, float):
    polarity, subjectivity = TextBlob(text).sentiment
    return polarity, subjectivity

def analyze_sentiment(polarity: float) -> str:
    if polarity <= -0.5:
        return 'negative'
    elif -0.5 < polarity < -0.15:
        return 'slightly negative'
    elif -0.15 <= polarity <= 0.15:
        return 'neutral'
    elif 0.15 < polarity < 0.5:
        return 'slightly positive'
    return 'positive'

def analyze_subjectivity(subjectivity: float) -> str:
    if subjectivity >= 0.66:
        return 'personal opinion'
    elif 0.66 > subjectivity > 0.33:
        return 'balance between personal opinion and factual information'
    return 'factual information'

def get_most_frequent_words(text: str, lang_code: str, extra_stopwords: {str}, most_common_n: int) -> {str: int}:
    my_stopwords = list(get_stopwords(lang_code, extra_stopwords))
    words = text.split(' ')
    meaningful_words = [word for word in words if word not in my_stopwords]
    bow = Counter(meaningful_words)
    return bow.most_common(most_common_n)

def count_words(clear_text: str, lang_code: str, extra_stopwords: {str}) -> {str: int}:
    my_stopwords = get_stopwords(lang_code, extra_stopwords)
    stemmed_stopwords = stem_text(' '.join(my_stopwords), lang_code)
    all_words = clear_text.split(' ')
    all_unique_words = set(all_words)
    words_no_stopwords = [True for word in all_words if word not in my_stopwords]
    unique_words_no_stopwords = [True for word in all_unique_words if word not in my_stopwords]
    all_stems = stem_text(clear_text, lang_code)
    all_unique_stems = set(stem_text(clear_text, lang_code))
    stems_no_stopwords = [True for word in all_stems if word not in stemmed_stopwords]
    unique_stems_no_stopwords = [True for word in all_unique_stems if word not in stemmed_stopwords]
    counts = {'all_words': len(all_words),
              'all_unique_words': len(all_unique_words),
              'words_no_stopwords': len(words_no_stopwords),
              'unique_words_no_stopwords': len(unique_words_no_stopwords),
              'all_stems': len(all_stems),
              'all_unique_stems': len(all_unique_stems),
              'stems_no_stopwords': len(stems_no_stopwords),
              'unique_stems_no_stopwords': len(unique_stems_no_stopwords)}
    return counts

def get_and_save_ner(unprocessed_text: str, lang_code: str, doc_title: str):
    if lang_code == 'en':
        nlp = en_core_web_sm.load()
    else:
        nlp = xx_ent_wiki_sm.load()
    doc = nlp(unprocessed_text)
    entity_label_pairs = [(x.text, x.label_) for x in doc.ents]
    label_counts = Counter([x.label_ for x in doc.ents])

    html_title = re.sub(r'[\W\s]', '_', doc_title)
    if html_title[-1] == '_':
        html_title = html_title[:-1]
    html_title = f'static/labelled_texts/html_{html_title}.html'
    # displacy.serve(doc, style='ent')
    ner_text = displacy.render(doc, style='ent')
    with open(html_title, 'w', encoding='utf-8') as html_file:
        html_file.write(ner_text)

    return entity_label_pairs, label_counts, ner_text, html_title

def discover_topics(n: int, lang_code: str, extra_stopwords: {}, text: str):
    # TODO change the algorithm as now it takes very long to obtain the results
    """
        Function that calculates similarity of words used in the given text with the provided topics
    :param n: the number of topics
    :param lang_code:
    :param extra_stopwords:
    :param text: text without noise
    :return: list of tuples:
        1st element - topic, cumulative score of top 5 words
        2nd element - 5 most relevant words for a given topic
    """
    if lang_code == 'en':
        nlp = spacy.load('en_core_web_lg')
        tokens = set(nlp(text))

        similarity_results = {}
        similarity_pairs = {}
        for topic in topics.topics:
            similarity_results[topic] = {}
            nlp_topic = nlp(topic)
            for token in tokens:
                similarity_results[topic][token] = nlp_topic.similarity(token)
            most_similar_tokens = sorted(similarity_results[topic].items(), key=lambda item: item[1], reverse=True)[:5]
            top_5_similarity_sum = sum([similarity[1] for similarity in most_similar_tokens])
            similarity_pairs[(topic, top_5_similarity_sum)] = [similarity[0] for similarity in most_similar_tokens]

        most_similar = sorted(similarity_pairs.items(), key=lambda item: item[0][1], reverse=True)[:n]
        return most_similar
    return 'supported only for English'


"""
Common internal functions used by Splitgraph commands.
"""
from datetime import date, datetime, time
from decimal import Decimal
from functools import wraps
from random import getrandbits
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    List,
    Optional,
    Set,
    Tuple,
    Union,
    cast,
)

from psycopg2.sql import SQL, Identifier

from splitgraph.config import SPLITGRAPH_API_SCHEMA
from splitgraph.core.output import parse_date, parse_dt
from splitgraph.core.sql.queries import select
from splitgraph.exceptions import ImageNotFoundError, RepositoryNotFoundError

if TYPE_CHECKING:
    from splitgraph.core.repository import Repository
    from splitgraph.engine.postgres.engine import PostgresEngine

OBJECT_MANAGER_TABLES = ["object_cache_status", "object_cache_occupancy"]

def set_tag(repository: "Repository", image_hash: Optional[str], tag: str) -> None:
    """Internal function -- add a tag to an image."""
    engine = repository.engine
    engine.run_sql(
        SQL("SELECT {}.tag_image(%s,%s,%s,%s)").format(Identifier(SPLITGRAPH_API_SCHEMA)),
        (repository.namespace, repository.repository, image_hash, tag),
    )

def set_tags_batch(repository: "Repository", hashes_tags: List[Tuple[str, str]]) -> None:
    engine = repository.engine

    args = [
        (repository.namespace, repository.repository, image_hash, tag)
        for image_hash, tag in hashes_tags
    ]

    engine.run_sql_batch(
        SQL("SELECT {}.tag_image(%s,%s,%s,%s)").format(Identifier(SPLITGRAPH_API_SCHEMA)),
        args,
    )

def set_head(repository: "Repository", image: Optional[str]) -> None:
    """Sets the HEAD pointer of a given repository to a given image. Shouldn't be used directly."""
    set_tag(repository, image, "HEAD")

def manage_audit_triggers(
    engine: "PostgresEngine", object_engine: Optional["PostgresEngine"] = None
) -> None:
    """Does bookkeeping on audit triggers / audit table:

        * Detect tables that are being audited that don't need to be any more
          (e.g. they've been unmounted)
        * Drop audit triggers for those and delete all audit info for them
        * Set up audit triggers for new tables

    If the metadata engine isn't the same as the object engine, this does nothing.

    :param engine: Metadata engine with information about images and their checkout state
    :param object_engine: Object engine where the checked-out table and the audit triggers are located.
    """

    object_engine = object_engine or engine

    if object_engine != engine:
        return

    from splitgraph.core.engine import get_current_repositories

    repos_tables = [
        (r.to_schema(), t)
        for r, head in get_current_repositories(engine)
        if head is not None
        for t in set(object_engine.get_all_tables(r.to_schema())) & set(head.get_tables())
        if not r.is_overlay_view(t)
    ]
    tracked_tables = object_engine.get_tracked_tables()

    to_untrack = [t for t in tracked_tables if t not in repos_tables]
    to_track = [t for t in repos_tables if t not in tracked_tables]

    if to_untrack:
        object_engine.untrack_tables(to_untrack)

    if to_track:
        object_engine.track_tables(to_track)

def manage_audit(func: Callable) -> Callable:
    """A decorator to be put around various Splitgraph commands
    that adds/removes audit triggers for new/committed/deleted tables.
    """

    # Make sure docstrings are passed through
    @wraps(func)
    def wrapped(self, *args, **kwargs):
        from .image import Image

        if isinstance(self, Image):
            repository = self.repository
        else:
            repository = self
        try:
            manage_audit_triggers(repository.engine, repository.object_engine)
            return func(self, *args, **kwargs)
        finally:
            repository.object_engine.commit()
            manage_audit_triggers(repository.engine, repository.object_engine)

    return wrapped

def aggregate_changes(
    query_result: List[Tuple[int, int]], initial: Optional[Tuple[int, int, int]] = None
) -> Tuple[int, int, int]:
    """Add a changeset to the aggregated diff result"""
    result = list(initial) if initial else [0, 0, 0]
    for kind, kind_count in query_result:
        assert kind in (0, 1, 2)
        result[kind] += kind_count
    return result[0], result[1], result[2]

def slow_diff(
    repository: "Repository",
    table_name: str,
    image_1: Optional[str],
    image_2: Optional[str],
    aggregate: bool,
) -> Union[Tuple[int, int, int], List[Tuple[bool, Tuple]]]:
    """Materialize both tables and manually diff them"""
    with repository.materialized_table(table_name, image_1) as (mp_1, table_1):
        with repository.materialized_table(table_name, image_2) as (mp_2, table_2):
            # Check both tables out at the same time since then table_2 calculation can be based
            # on table_1's snapshot.
            left = repository.object_engine.run_sql(
                SQL("SELECT * FROM {}.{}").format(Identifier(mp_1), Identifier(table_1))
            )
            right = repository.object_engine.run_sql(
                SQL("SELECT * FROM {}.{}").format(Identifier(mp_2), Identifier(table_2))
            )

    if aggregate:
        return sum(1 for r in right if r not in left), sum(1 for r in left if r not in right), 0

    # Return format: list of [(False for deleted/True for inserted, full row)]
    return [(False, r) for r in left if r not in right] + [
        (True, r) for r in right if r not in left
    ]

def gather_sync_metadata(
    target: "Repository",
    source: "Repository",
    overwrite_objects=False,
    overwrite_tags=False,
    single_image: Optional[str] = None,
) -> Any:
    """
    Inspects two Splitgraph repositories and gathers metadata that is required to bring target up to
    date with source.

    :param target: Target Repository object
    :param source: Source repository object
    :param overwrite_objects: If True, will return metadata for all objects
        belonging to new images (or existing image if single_image=True)
    :param single_image: If set, only grab a single image with this hash/tag from the source.
    :param overwrite_tags: If True and single_image is set, will return all tags for that image.
        If single_image is not set, will return all tags in the source repository.
        If False, will only return tags in the source that don't exist on the target.

    :returns: Tuple of metadata for  new_images, new_tables, object_locations, object_meta, tags
    """

    # Transaction handling: this is run as a single read-only transaction to the source and the
    # target engine. This is so that:
    #   1) we can in the future route these to read-only replicas
    #   2) we don't hold an open transaction to the registry while we're
    #       downloading/uploading objects.
    target.engine.run_sql("SET TRANSACTION READ ONLY")
    if target.engine != source.engine:
        source.engine.run_sql("SET TRANSACTION READ ONLY")

    try:
        result = _gather_sync_metadata(
            target, source, overwrite_objects, overwrite_tags, single_image
        )
        target.commit_engines()
        source.commit_engines()
        return result
    except Exception:
        target.rollback_engines()
        source.rollback_engines()
        raise

def _gather_sync_metadata(target, source, overwrite_objects, overwrite_tags, single_image) -> Any:
    # Currently, images can't be altered once pushed out. We intend to relax this:
    # same image hash means same contents and same tables but the composition of an image
    # can change (if we refragment a table so that querying it is faster). But it's frowned
    # upon.
    single_image_hash: Optional[str] = None
    if single_image:
        image = source.images[single_image]
        single_image_hash = image.image_hash
        try:
            # If the image already exists on the target, we shouldn't overwrite it.
            # The user can get around this by deleting the image manually.
            _ = target.images[single_image_hash]
            new_images = []
            new_image_hashes = []
        except ImageNotFoundError:
            new_images = [image]
            new_image_hashes = [single_image_hash]
        except RepositoryNotFoundError:
            new_images = [image]
            new_image_hashes = [single_image_hash]
    else:
        # If an image hasn't been specified, get/push all non-existing images.
        target_images = {i.image_hash: i for i in target.images()}
        source_images = {i.image_hash: i for i in source.images()}
        new_image_hashes = [i for i in source_images if i not in target_images]
        new_images = [source_images[i] for i in new_image_hashes]

    # Get the meta for all tables we'll need to fetch.
    table_meta = []

    # Also grab the list of all objects in this repository in case overwrite_objects=True
    all_objects: Set[str] = set()
    for t in source.engine.run_sql(
        select(
            "get_all_tables",
            "image_hash, table_name, table_schema, object_ids",
            schema=SPLITGRAPH_API_SCHEMA,
            table_args="(%s,%s)",
        ),
        (source.namespace, source.repository),
    ):
        if t[0] in new_image_hashes:
            all_objects = all_objects.union(t[-1])
            table_meta.append(t)
        elif t[0] == single_image_hash:
            # Overwrite objects from existing image if it was
            # passed as single_image (similar behaviour to tags).
            all_objects = all_objects.union(t[-1])

    # Get the tags
    existing_tags = [t for s, t in target.get_all_hashes_tags()]
    tags = {
        t: s
        for s, t in source.get_all_hashes_tags()
        if (
            # Only get new tags (unless we're overwriting them)
            t not in existing_tags
            or overwrite_tags
        )
        and (
            # Only get tags for the new image (unless we're pulling the whole repo)
            not single_image
            or s == single_image_hash
        )
    }

    # Get objects that don't exist on the target
    table_objects = list({o for table in table_meta for o in table[3]})
    new_objects = list(set(target.objects.get_new_objects(table_objects)))

    if overwrite_objects:
        new_objects = list(all_objects)

    object_locations = source.objects.get_external_object_locations(new_objects)

    if new_objects:
        object_meta = source.objects.get_object_meta(new_objects)
    else:
        object_meta = {}
    return new_images, table_meta, object_locations, object_meta, tags

_TYPE_MAP: Dict[str, Callable] = {
    k: cast(Callable, v)
    for ks, v in [
        (["integer", "bigint", "smallint"], int),
        (["numeric", "real", "double precision"], float),
        (["timestamp", "timestamp without time zone"], parse_dt),
        (["date"], parse_date),
    ]
    for k in ks
}

def adapt(value: Any, pg_type: str) -> Any:
    """
    Coerces a value with a PG type into its Python equivalent.

    :param value: Value
    :param pg_type: Postgres datatype
    :return: Coerced value.
    """
    if value is None:
        return None
    if not isinstance(value, str):
        return value
    if pg_type in _TYPE_MAP:
        return _TYPE_MAP[pg_type](value)
    return value

class Tracer:
    """
    Accumulates events and returns the times between them.
    """

    def __init__(self) -> None:
        self.start_time = datetime.now()
        self.events: List[Tuple[datetime, str]] = []

    def log(self, event: str) -> None:
        """
        Log an event at the current time
        :param event: Event name
        """
        self.events.append((datetime.now(), event))

    def get_durations(self) -> List[Tuple[str, float]]:
        """
        Return all events and durations between them.
        :return: List of (event name, time to this event from the previous event (or start))
        """
        result = []
        prev = self.start_time
        for event_time, event in self.events:
            result.append((event, (event_time - prev).total_seconds()))
            prev = event_time
        return result

    def get_total_time(self) -> float:
        """
        :return: Time from start to the final logged event.
        """
        return (self.events[-1][0] - self.start_time).total_seconds()

    def __str__(self) -> str:
        result = ""
        for event, duration in self.get_durations():
            result += "\n%s: %.3f" % (event, duration)
        result += "\nTotal: %.3f" % self.get_total_time()
        return result[1:]

def coerce_val_to_json(val: Any) -> Any:
    """
    Turn a Python value to a string/float that can be stored as JSON.
    """
    if isinstance(val, list):
        val = [coerce_val_to_json(v) for v in val]
    elif isinstance(val, tuple):
        val = tuple(coerce_val_to_json(v) for v in val)
    elif isinstance(val, dict):
        val = {k: coerce_val_to_json(v) for k, v in val.items()}
    elif isinstance(val, (Decimal, date, time, datetime)):
        # See https://www.postgresql.org/docs/11/datatype-datetime.html
        # "ISO 8601 specifies the use of uppercase letter T to separate the date and time.
        # PostgreSQL accepts that format on input, but on output it uses a space rather
        # than T, as shown above. "
        #
        # This also matches python's str().
        return str(val)
    return val

class CallbackList(list):
    """
    Used to pass around and call multiple callbacks at once.
    """

    def __call__(self, *args, **kwargs) -> None:
        for listener in self:
            listener(*args, **kwargs)

def get_temporary_table_id() -> str:
    """Generate a random ID for temporary/staging objects that haven't had their ID calculated yet."""
    return str.format("sg_tmp_{:032x}", getrandbits(128))


import os
import random
import uuid
from typing import Callable, Optional
from unittest.mock import Mock

import pytest

from pytest_testrail.controller import _TestRailController
from pytest_testrail.plugin import PyTestRailPlugin
from pytest_testrail.result_item import ResultItem
from pytest_testrail.store import Store

from .mock_response import MockResponse

pytest_plugins = "pytester"

@pytest.fixture()
def api_client():
    """Get a mock for the TestRailAPI client."""
    client = Mock()

    client.request_kwargs = {}
    client.get_run().get.return_value = MockResponse({'is_completed': False})
    client.get_plan().get.return_value = MockResponse({'is_completed': False})
    return client

@pytest.fixture
def tr_controller(api_client):
    return _TestRailController(
        api_client,
        version='1.0.0.0',
        custom_comment="This is custom comment",
        testrun_name=None,
    )

@pytest.fixture
def tr_plugin(api_client, request):
    assign_user_id = 3
    milestone_id = 5
    project_id = 4
    suite_id = 1

    testrail_controller = _TestRailController(
        api_client,
        version='1.0.0.0',
        custom_comment="This is custom comment",
        testrun_name=None,
    )

    return PyTestRailPlugin(
        request.config,
        testrail_controller,
        api_client,
        assign_user_id=assign_user_id,
        project_id=project_id,
        suite_id=suite_id,
        tr_name=None,
        tr_description='This is a test description',
        version='1.0.0.0',
        milestone_id=milestone_id,
    )

@pytest.fixture()
def new_resultitem() -> Callable:
    def _get_result(
        test_name=None,
        case_id=None,
        status_id=None,
        defects=None,
        timestamp=None,
        duration=None,
        comment=None,
        test_parametrize=None,
    ) -> ResultItem:
        test_name = test_name or f'test_foo_{uuid.uuid4()}'
        case_id = case_id or random.randint(1, 1000)
        status_id = status_id
        comment = comment or ''
        timestamp = timestamp or random.random()

        rv = ResultItem(
            test_name=test_name,
            case_id=case_id,
            status_id=status_id,
            duration=duration,
            comment=comment,
            defects=defects,
            test_parametrize=test_parametrize,
            timestamp=timestamp,
        )

        return rv

    return _get_result

@pytest.fixture()
def dummy_test_file(pytester):
    # Register marks
    conftest_source = """
    def pytest_configure(config) -> None:
        # Register marks
        config.addinivalue_line(
            "markers", "case_id(str): Mark tests with a testrail case_id"
        )
        config.addinivalue_line(
            "markers",
            (
                "defect_ids(*str): Mark tests with an issue tracker's IDs."
                "Sent to testrail testrun results."
            )
        )
    """
    pytester.makeconftest(conftest_source)

    file_data = """
        import pytest

        @pytest.mark.case_id('C1234')
        def test_func():
            pass

        @pytest.mark.case_id('C8765')
        @pytest.mark.defect_ids('PF-418', 'PF-517')
        def test_other_func():
            pass
    """
    pytester.makepyfile(file_data)
    return file_data

@pytest.fixture()
def test_items(pytester, dummy_test_file):
    return [item for item in pytester.getitems(dummy_test_file) if item.name != 'testrail']

def pytest_sessionfinish(session, exitstatus):
    """Remove the store file after the tests are finished.

    The tests will invoke the plugin class and create a store file.
    However, the tests do not actually install the plugin.
    As a result, the store file is not cleared when the test session ends.

    This function is a hack to manually remove the store file.
    """
    xdist_worker = os.getenv('PYTEST_XDIST_WORKER')
    xdist_worker_count: Optional[str] = os.getenv('PYTEST_XDIST_WORKER_COUNT')

    if not xdist_worker and not xdist_worker_count:
        Store(session.config).clear()


# Author:Zhang Yuan
from MyPackage import *
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import seaborn as sns
import statsmodels.api as sm
from scipy import stats

# ------------------------------------------------------------
__mypath__ = MyPath.MyClass_Path("")  # 路径类
mylogging = MyDefault.MyClass_Default_Logging(activate=False)  # 日志记录类，需要放在上面才行
myfile = MyFile.MyClass_File()  # 文件操作类
myword = MyFile.MyClass_Word()  # word生成类
myexcel = MyFile.MyClass_Excel()  # excel生成类
myini = MyFile.MyClass_INI()  # ini文件操作类
mytime = MyTime.MyClass_Time()  # 时间类
myparallel = MyTools.MyClass_ParallelCal()  # 并行运算类
myplt = MyPlot.MyClass_Plot()  # 直接绘图类(单个图窗)
mypltpro = MyPlot.MyClass_PlotPro()  # Plot高级图系列
myfig = MyPlot.MyClass_Figure(AddFigure=False)  # 对象式绘图类(可多个图窗)
myfigpro = MyPlot.MyClass_FigurePro(AddFigure=False)  # Figure高级图系列
mynp = MyArray.MyClass_NumPy()  # 多维数组类(整合Numpy)
mypd = MyArray.MyClass_Pandas()  # 矩阵数组类(整合Pandas)
mypdpro = MyArray.MyClass_PandasPro()  # 高级矩阵数组类
myDA = MyDataAnalysis.MyClass_DataAnalysis()  # 数据分析类
myDefault = MyDefault.MyClass_Default_Matplotlib()  # 画图恢复默认设置类
# myMql = MyMql.MyClass_MqlBackups() # Mql备份类
# myBaidu = MyWebCrawler.MyClass_BaiduPan() # Baidu网盘交互类
# myImage = MyImage.MyClass_ImageProcess()  # 图片处理类
myBT = MyBackTest.MyClass_BackTestEvent()  # 事件驱动型回测类
myBTV = MyBackTest.MyClass_BackTestVector()  # 向量型回测类
myML = MyMachineLearning.MyClass_MachineLearning()  # 机器学习综合类
mySQL = MyDataBase.MyClass_MySQL(connect=False)  # MySQL类
mySQLAPP = MyDataBase.MyClass_SQL_APPIntegration()  # 数据库应用整合
myWebQD = MyWebCrawler.MyClass_QuotesDownload(tushare=False)  # 金融行情下载类
myWebR = MyWebCrawler.MyClass_Requests()  # Requests爬虫类
myWebS = MyWebCrawler.MyClass_Selenium(openChrome=False)  # Selenium模拟浏览器类
myWebAPP = MyWebCrawler.MyClass_Web_APPIntegration()  # 爬虫整合应用类
myEmail = MyWebCrawler.MyClass_Email()  # 邮箱交互类
myReportA = MyQuant.MyClass_ReportAnalysis()  # 研报分析类
myFactorD = MyQuant.MyClass_Factor_Detection()  # 因子检测类
myKeras = MyDeepLearning.MyClass_tfKeras()  # tfKeras综合类
myTensor = MyDeepLearning.MyClass_TensorFlow()  # Tensorflow综合类
myMT5 = MyMql.MyClass_ConnectMT5(connect=False)  # Python链接MetaTrader5客户端类
myMT5Pro = MyMql.MyClass_ConnectMT5Pro(connect=False)  # Python链接MT5高级类
myMT5Indi = MyMql.MyClass_MT5Indicator()  # MT5指标Python版
myMT5Report = MyMT5Report.MyClass_StratTestReport(AddFigure=False)  # MT5策略报告类
myMT5Lots_Fix = MyMql.MyClass_Lots_FixedLever(connect=False)  # 固定杠杆仓位类
myMT5Lots_Dy = MyMql.MyClass_Lots_DyLever(connect=False)  # 浮动杠杆仓位类
myMT5run = MyMql.MyClass_RunningMT5()  # Python运行MT5
myMT5code = MyMql.MyClass_CodeMql5()  # Python生成MT5代码
myMoneyM = MyTrade.MyClass_MoneyManage()  # 资金管理类
myDefault.set_backend_default("Pycharm")  # Pycharm下需要plt.show()才显示图
# ------------------------------------------------------------

'''
根据MT5文章：利用箱形图（BOXPLOT）探索金融时间序列的季节性形态
https://www.mql5.com/zh/articles/7038
'''
#%%
myMT5Pro.__init__(connect=True)

rates_original = myMT5Pro.getsymboldata("EURUSD", "TIMEFRAME_D1", [2010,1,1], [2020,1,1], index_time=False)
# leave only 'time' and 'close' columns
rates = rates_original[["time","close"]]

# get percent change (price returns)
returns = pd.DataFrame(rates['close'].pct_change(1))
returns = returns.set_index(rates['time'])
returns = returns[1:]
returns.head(5)

# 把数据按照 年-月 group
indexyear = returns.index.year.rename('year')
indexmonth = returns.index.month.rename('month')
Monthly_Returns = returns.groupby([indexyear, indexmonth]).mean()
# 按月来画箱型图，每个箱型图是统计每年的指定月数据
Monthly_Returns.boxplot(column='close', by='month', figsize=(15, 8))
plt.show()
'''
我们可以看到，第五个月（五月份）的增幅中位数偏移到零轴下方，且其异常值显见高于零轴。 通常，从十年的统计数据里可以看出，五月份的市场相对于三月份有所下跌。 只有一个年份，五月份市场上涨。 这是一个有趣的思路，很符合交易者的格言“在五月份卖掉，并离开！”。
我们看一下五月份之后的六月份。 相对于五月份，六月份市场几乎总是（排除一年以外）在增长，这种情况每年都在重复。 六月份的波动范围很小，没有异常值（与五月份不同），这表明良好的季节性稳定。
请注意第 11 个月（十一月份）。 在此期间市场下跌的概率很高。 之后，在十二月份，市场通常会再度上行。 一月份（第一个月）的波动性很高，且相对于十二月份有所下跌。
所获得的数据可为交易决策提供很有用的基础条件概览。 而且，概率可以集成到交易系统当中。 例如，可以在某些月份执行更多的买卖操作。
'''

# 我们利用相同的 10 年度观察一周中每个单独交易日的价格增幅分布：
indexweek = returns.index.week.rename('week')
indexday = returns.index.dayofweek.rename('day')
Daily_Returns = returns.groupby([indexweek, indexday]).mean()
# 按 星期几 来统计
Daily_Returns.boxplot(column='close', by='day', figsize=(15, 8))
plt.show()
'''
此处零对应于星期一，四则对应于星期五。 根据价格范围，按日波动率几乎保持不变。 然而不能据此得出结论，即在一周中的某个特定日期交易更为密集。 平均来说，市场在星期一和星期五时更倾向于下行非上行。
'''
# 也许在一些单独的月份中，按日分布的样子会有所不同。
# leave only one month "returns.index[~returns.index.month.isin([1])"
for i in range(1,13): # i=1
    returns_solo = returns.drop(returns.index[~returns.index.month.isin([i])])
    indexweek_solo = returns_solo.index.week.rename('week')
    indexday_solo = returns_solo.index.dayofweek.rename('day')
    Daily_Returns_solo = returns_solo.groupby([indexweek_solo, indexday_solo]).mean()
    # 按 星期几 来统计
    Daily_Returns_solo.boxplot(column='close', by='day', figsize=(15, 8))
    plt.suptitle("Boxplot grouped by day month_%s"%i)
    plt.show()

#%%
### 分析日内形态
# 在创建交易系统时，通常要考虑日内的分布，例如，除了日线和月线的分布，还要用到小时线数据。 这轻易就可做到。
rates_original = myMT5Pro.getsymboldata("EURUSD", "TIMEFRAME_M15", [2010,1,1], [2019, 11, 25], index_time=False)
# leave only 'time' and 'close' columns
rates = rates_original[["time","close"]]
# get percent change (price returns)
returns = pd.DataFrame(rates['close'].pct_change(1))
returns = returns.set_index(rates['time'])
returns = returns[1:]

# 将数据按日和小时分组
indexday = returns.index.day.rename('day')
indexhour = returns.index.hour.rename('hour')
Hourly_Returns = returns.groupby([indexday, indexhour]).median()
Hourly_Returns.boxplot(column='close', by='hour', figsize=(10, 5))
plt.show()

#%%
### 按均线（MA）搜索去趋势化形态
# 正确检测趋势分量非常棘手。 有时，时间序列可能会太平滑。 在这种情况下，交易信号极少。 如果缩短平滑周期，那么高频成交可能无法负担点差和佣金。 我们编辑代码，以便利用移动平均线进行去趋势化：
rates_original = myMT5Pro.getsymboldata("EURUSD", "TIMEFRAME_M15", [2010,1,1], [2019, 11, 25], index_time=False)
# leave only 'time' and 'close' columns
rates = rates_original[["time","close"]]
rates = rates.set_index('time')
# set the moving average period
window = 25
# detrend tome series by MA
ratesM = rates.rolling(window).mean()
plt.figure(figsize=(10, 5))
plt.plot(rates)
plt.plot(ratesM)
plt.show()
# 我们得到了小时收盘价与 15 分钟移动平均线之间的平均偏差
ratesD = rates[window:] - ratesM[window:]
# 从收盘价中减去移动平均线值，得到一个去趋势化的时间序列
plt.plot(ratesD)
plt.show()

# 现在，我们得到每个交易小时的偏差分布小时统计信息：
Hourly_Returns = ratesD.groupby([ratesD.index.day.rename('day'), ratesD.index.hour.rename('hour')]).median()
Hourly_Returns.boxplot(column='close', by='hour', figsize=(15, 8))
plt.show()
# 下一个合乎逻辑的步骤是仔细检查分布矩，以便获得更准确的统计评估。 例如，以箱形图的形式计算得到的去趋势化序列的标准差：
Hourly_std = ratesD.groupby([ratesD.index.day.rename('day'), ratesD.index.hour.rename('hour')]).std()
Hourly_std.boxplot(column='close', by='hour', figsize=(15, 8))
plt.show()
# 另一个有趣的点是偏度。 我们计算一下：
Hourly_skew = ratesD.groupby([ratesD.index.day.rename('day'), ratesD.index.hour.rename('hour')]).skew()
Hourly_skew.boxplot(column='close', by='hour', figsize=(15, 8))
plt.show()
# 峰度的统计数据显示类似的结果：
Hourly_kurt = ratesD.groupby([ratesD.index.day.rename('day'), ratesD.index.hour.rename('hour')]).apply(pd.DataFrame.kurt)
Hourly_kurt.boxplot(column='close', by='hour', figsize=(15, 8))
plt.show()

#%%
### 搜索特定月份或一周中某天的周期形态，按均线去趋势化
rates_original = myMT5Pro.getsymboldata("EURUSD", "TIMEFRAME_M15", [2010,1,1], [2019, 11, 25], index_time=False)
# leave only 'time' and 'close' columns
rates = rates_original[["time","close"]]
rates = rates.set_index('time')
# set the moving average period
window = 25
# detrend tome series by MA
ratesM = rates.rolling(window).mean()
ratesD = rates[window:] - ratesM[window:]
# 仅分析指定月份
ratesD = ratesD.drop(ratesD.index[~ratesD.index.month.isin([11])])
Hourly_Returns = ratesD.groupby([ratesD.index.day.rename('day'), ratesD.index.hour.rename('hour')]).median()
Hourly_Returns.boxplot(column='close', by='hour', figsize=(11, 6))
plt.show()

# 仅分析指定周几
ratesD = ratesD.drop(ratesD.index[~ratesD.index.dayofweek.isin([0])])
Hourly_Returns = ratesD.groupby([ratesD.index.day.rename('day'), ratesD.index.hour.rename('hour')]).median()
Hourly_Returns.boxplot(column='close', by='hour', figsize=(11, 6))
plt.show()

Project_MT5文章调试/1.利用箱形图(BOXPLOT)探索金融时间序列的周期形态.py

1.利用箱形图(BOXPLOT)探索金融时间序列的周期形态.py