import json
import os
import copy

def main():
    #for which in [0, 1, 2]:
    for which in range(24):
        fix_season(which)

def fix_season(which_season0):
    print(f"\naccumulations script is now repairing season0={which_season0}...")

    nteams = 4

    f_rainbows = lambda x: 11*x[0] + 7*x[1] + 3*x[2]
    rainbow_values = [11, 7, 3, 0]

    postseason_json_file = os.path.join(f'season{which_season0}', 'postseason.json')
    if not os.path.exists(postseason_json_file):
        raise Exception(f"Error: could not find json file: {postseason_json_file}")
    with open(postseason_json_file, 'r') as f:
        post = json.load(f)

    # Initialize new postseason data structure
    new_post = {}

    # Initialize new W23L/points accumulators
    lcs_sw23l = {}
    rcs_sw23l = {}
    lcs_points = {}
    rcs_points = {}
    # Loop over all teams involved
    last_day = post['LCS'][-1]
    for game in last_day:
        for i in range(nteams):
            abbr_key = f"team{i+1}Abbr"
            abbr_val = game[abbr_key]
            lcs_sw23l[abbr_val] = [0, 0, 0, 0]
            rcs_sw23l[abbr_val] = [0, 0, 0, 0]
            lcs_points[abbr_val] = 0
            rcs_points[abbr_val] = 0

    # Iterate through LCS, filtering games as we go
    miniseason = []
    for day in post['LCS']:
        new_day = []
        for game in day:
            # Scores and ranks are ok,
            # SeriesW23L and SeriesTotalPoints are not
            new_game = copy.deepcopy(game)

            # First, fix the incorrect fields
            for i in range(nteams):
                abbr_key = f"team{i+1}Abbr"
                abbr_val = team_abbr = game[abbr_key]

                this_sw23l = copy.deepcopy(lcs_sw23l[team_abbr])

                sw23l_key = f"team{i+1}SeriesW23L"
                #del new_game[sw23l_key]
                new_game[sw23l_key] = []

                stp_key = f"team{i+1}SeriesTotalPoints"
                #del new_game[stp_key]
                new_game[stp_key] = 0

                new_game[sw23l_key] = this_sw23l
                new_game[stp_key] = lcs_points[team_abbr]

            new_day.append(new_game)

            # Now accumulate W23L and points (correctly)
            for i in range(nteams):
                abbr_key = f"team{i+1}Abbr"
                abbr_val = game[abbr_key]

                rank_key = f"team{i+1}Rank"
                rank_val = game[rank_key]

                score_key = f"team{i+1}Score"
                score_val = game[score_key]

                tmp = lcs_sw23l[abbr_val]
                tmp[rank_val] += 1
                lcs_sw23l[abbr_val] = tmp
                lcs_points[abbr_val] += score_val

        miniseason.append(new_day)
    new_post['LCS'] = miniseason

    # Iterate through RCS, filtering games as we go
    miniseason = []
    for day in post['RCS']:
        new_day = []
        for game in day:
            # Scores and ranks are ok,
            # SeriesW23L and SeriesTotalPoints are not
            new_game = copy.deepcopy(game)

            # First, fix the incorrect fields
            for i in range(nteams):
                abbr_key = f"team{i+1}Abbr"
                abbr_val = team_abbr = game[abbr_key]

                this_sw23l = copy.deepcopy(rcs_sw23l[team_abbr])

                sw23l_key = f"team{i+1}SeriesW23L"
                #del new_game[sw23l_key]
                new_game[sw23l_key] = []

                stp_key = f"team{i+1}SeriesTotalPoints"
                #del new_game[stp_key]
                new_game[stp_key] = 0

                new_game[sw23l_key] = this_sw23l
                new_game[stp_key] = rcs_points[team_abbr]

            new_day.append(new_game)

            # Now accumulate
            for i in range(nteams):
                abbr_key = f"team{i+1}Abbr"
                abbr_val = game[abbr_key]

                rank_key = f"team{i+1}Rank"
                rank_val = game[rank_key]

                score_key = f"team{i+1}Score"
                score_val = game[score_key]

                tmp = rcs_sw23l[abbr_val]
                tmp[rank_val] += 1
                rcs_sw23l[abbr_val] = tmp
                rcs_points[abbr_val] += score_val

        miniseason.append(new_day)
    new_post['RCS'] = miniseason

    with open(postseason_json_file, 'w') as f:
        json.dump(new_post, f, indent=4)

    print(f"Done repairing {postseason_json_file}.")

if __name__=="__main__":
    main()


import argparse
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import sys

from statsmodels.tsa.seasonal import seasonal_decompose

def parse_opt() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument('--house', type=str, default='3', help='house id')
    parser.add_argument('--plot_type', type=str, default='normal',
                        help='possible values: normal, histogram, box-plot, pearson, violin, decompose')
    parser.add_argument('--interval', type=str, default='month', help='show the seasonality on specific interval')
    parser.add_argument('--show_plot', default=True, action='store_true', help='flag for showing the plot or not')
    parser.add_argument('--save_plot', default=True, action='store_true', help='flag of storing the plot or not')
    return parser.parse_args()

def box_plot_seasonality(df: pd.DataFrame, args: argparse.Namespace) -> None:
    interval = args.interval
    if interval == 'month':
        df['month'] = df.index.month
    elif interval == 'week':
        df['week'] = df.index.day_of_week + 1
    else:
        df['hour'] = df.index.hour + 1

    fig, ax = plt.subplots(figsize=(7, 3.5))
    df.boxplot(column='IMPORT_KW', by=interval, ax=ax, )
    df.groupby(interval)['IMPORT_KW'].median().plot(style='o-', linewidth=0.8, ax=ax)
    ax.set_ylabel('Consumption')
    ax.set_title('Consumption distribution by {}'.format(interval))
    fig.suptitle('')
    if args.show_plot:
        plt.show()
    if args.save_plot:
        fig.savefig(r'../plots/time_series_analysis/consumption_distribution_by_{}.png'.format(interval))
        plt.close()

def consumption_on_holidays(df: pd.DataFrame, args: argparse.Namespace) -> None:
    df = df.assign(HOLIDAY=df.HOLIDAY.astype(str))
    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7, 3.5))
    sns.violinplot(
        x='IMPORT_KW',
        y='HOLIDAY',
        data=df,
        palette='tab10',
        ax=ax
    )
    ax.set_title('Distribution of consumption between holidays and non-holidays')
    ax.set_xlabel('Demand')
    ax.set_ylabel('Holiday')
    if args.show_plot:
        plt.show()
    if args.save_plot:
        fig.savefig(r'../plots/time_series_analysis/consumption_on_holidays.png')
        plt.close()

def correlation_check(df: pd.DataFrame, args: argparse.Namespace) -> None:
    """
    Plots a Pearson Correlation Heatmap.
    ---
    Args:
        df (pd.DataFrame): dataframe to plot
        args (argparse.Namespace): resolving input arguments

    Returns: None
    """
    # Figure
    fig, ax = plt.subplots(figsize=(16, 12), facecolor='w')
    correlations_df = df.corr(method='pearson', min_periods=1)
    sns.heatmap(correlations_df, cmap="Blues", annot=True, linewidth=.1)

    # Labels
    ax.set_title("Pearson Correlation Heatmap", fontsize=15, pad=10)
    ax.set_facecolor(color='white')
    if args.show_plot:
        plt.show()
    if args.save_plot:
        fig.savefig(r'../plots/time_series_analysis/pearson_correlation_heatmap.png')
        plt.close()

def decompose_series(df: pd.DataFrame) -> None:
    """
    This function decomposes the time series
    into trend, seasonality and residuals.
    ---
    Args:
        df (pd.DataFrame): Dataframe that contains the timeseries data

    Returns: None
    """
    df = df.resample('D').sum()
    # Decomposition
    decomposition = seasonal_decompose(df['IMPORT_KW'])
    decomposition.plot()

    plt.tight_layout()
    plt.show()

def histogram(df: pd.DataFrame):
    df.hist(bins=50, column='IMPORT_KW')
    plt.show()

def plot_house_data(df: pd.DataFrame, args: argparse.Namespace) -> None:
    house_df = df.resample('D').sum()
    house_df.reset_index(inplace=True)
    house_df.plot.line(x='timestamp', y=['IMPORT_KW'], title='Consumption')
    # Add legend
    plt.legend()
    # Auto space
    plt.tight_layout()
    # Display plot
    if args.show_plot:
        plt.show()
    # if args.save_plot:
    #     plt.savefig(r'plots/aggregated_train.png')
    #     plt.close()

def main():
    # Load input arguments
    args = parse_opt()
    df = pd.read_csv(r'../../data/house_data_cleaned/house{}.csv'.format(args.house), parse_dates=['timestamp'],
                     index_col='timestamp')
    df.drop(columns=['EXPORT_KW', 'PV_KW', 'BATTERY_KW', 'IX'], inplace=True)
    plot_type = args.plot_type
    if plot_type == 'normal':
        plot_house_data(df, args)
    elif plot_type == 'histogram':
        histogram(df)
    elif plot_type == 'box-plot':
        box_plot_seasonality(df, args)
    elif plot_type == 'pearson':
        correlation_check(df, args)
    elif plot_type == 'violin':
        consumption_on_holidays(df, args)
    elif plot_type == 'decompose':
        decompose_series(df)
    else:
        print('Unknown plot type')

if __name__ == '__main__':
    sys.exit(main())


from itertools import groupby
from typing import List, Tuple

from hwt.hdl.statements.assignmentContainer import HdlAssignmentContainer
from hwt.hdl.operator import Operator, isConst
from hwt.hdl.operatorDefs import AllOps
from hwt.serializer.utils import RtlSignal_sort_key
from hwtGraph.elk.containers.constants import PortType, PortSide
from hwtGraph.elk.containers.lNode import LNode

class InterfaceSplitInfo(tuple):
    pass

def extractSplitsAsSingleNode(root: LNode, sliceParts: List[Tuple[slice, HdlAssignmentContainer, LNode]], toL: dict):
        # replace multiple index nodes with a larger slice node
    sliceParts.sort(key=lambda x: x[0].start)
    n = toL[sliceParts[0][1]]
    p = n.west[0]
    if not p.incomingEdges:
        # was already replaced
        return

    srcPorts = p.incomingEdges[0].srcs
    assert len(srcPorts) == 1
    srcPort = srcPorts[0]

    dstPortsOnInputNet = list(p.incomingEdges[0].dsts)
    sliceNode = root.addNode(
        name="SLICE", cls="Operator",
        originObj=InterfaceSplitInfo(x[1] for x in sliceParts))
    inputPort = sliceNode.addPort(
        "", PortType.INPUT, PortSide.WEST)

    # create new sliceNode
    for sliceRange, _, oldAssigNode in sliceParts:
        # create a new port on main slice node which will be used instead of this operator
        if sliceRange.start - sliceRange.stop == 1:
            portName = f"[{sliceRange.stop:d}]"
        else:
            portName = f"[{sliceRange.start:d}:{sliceRange.stop:d}]"
        outPort = sliceNode.addPort(
            portName, PortType.OUTPUT, PortSide.EAST)

        # disconnect and replace the index and assignment node itself
        assert root is oldAssigNode.parent
        dstPortsOnInputNet.remove(oldAssigNode.west[0])
        for e in list(oldAssigNode.west[0].incomingEdges):
            e.remove()

        dstPorts = []
        # this assignment of slice is directly in the root
        for e in list(oldAssigNode.east[0].outgoingEdges):
            for _dst in e.dsts:
                dstPorts.append((_dst, e.originObj))
            e.remove()

        oldAssigNode.parent.children.remove(oldAssigNode)
        # remove index value node (we know that it is constant,
        # from original select)
        _e = oldAssigNode.west[1].incomingEdges[0]
        _e.removeTarget(oldAssigNode.west[1])
        assert len(_e.srcs) == 1
        indexValNodeP = _e.srcs[0]
        if not _e.dsts:
            _e.remove()

        if not indexValNodeP.outgoingEdges:
            root.children.remove(indexValNodeP.parentNode)

        root.addHyperEdge([outPort],
                          [dst[0] for dst in dstPorts],
                          originObj=dstPorts[0][1])

    dstPortsOnInputNet.append(inputPort)
    root.addHyperEdge([srcPort, ], dstPortsOnInputNet,
                      name=e.name, originObj=e.originObj)

def extractSplits(root: LNode):
    """
    convert group of indexed assignments witch are splitting signal to Split node

    a = sig[0]
    b = sig[1]
    to
    a, b = sig

    :param toL: dictionary {hdl object: layout object}
    """
    toL = root._node2lnode
    signals = sorted(root.originObj._ctx.signals, key=RtlSignal_sort_key)

    # search from "sig" side (look at doc string)
    for s in signals:
        if len(s.drivers) == 1 and len(s.endpoints) > 1:
            sliceParts = []
            for ep in s.endpoints:
                if isinstance(ep, HdlAssignmentContainer) and not ep.indexes and ep.src.hidden:
                    op = ep.src.origin
                else:
                    op = ep

                if isinstance(op, Operator)\
                        and op.operator == AllOps.INDEX\
                        and op.operands[0] is s:
                    index = op.operands[1]
                    if isConst(index):
                        sliceRange = index.staticEval().to_py()
                        if isinstance(sliceRange, int):
                            sliceRange = slice(sliceRange + 1, sliceRange)

                        if ep not in toL:
                            continue
                        sliceParts.append((sliceRange, ep, toL[ep]))

            for _root, _sliceParts in groupby(sliceParts, lambda x: x[2].parent):
                # the slices may be spoted in some sub node, in that case we need to extract it only in this sub node
                _sliceParts = list(_sliceParts)
                if _sliceParts:
                    extractSplitsAsSingleNode(_root, _sliceParts, toL)


"""
This module handles settings of system.

Platform-dependent, currently works on Unix only (uses '/proc' filesystem).

Part of 'Adaptor' framework.

Author: Michael Pankov, 2012-2013.

Please do not redistribute.
"""

import os
import recordtype as rt

from data_types import *

SettingsBase = rt.recordtype('SettingsBase',
    'program_name framework_root_dir benchmark_root_dir benchmark_bin_dir '
    'build_settings run_settings')

class Settings(PrintableStructure, SettingsBase):
    def __init__(self,
                 program_name,
                 benchmark_root_dir='data/sources/polybench-c-3.2/',
                 benchmark_bin_dir='data/bin/'):
        """ Initialize framework settings and return Settings object.

        program_name is name of directory to be found in benchmark_root_dir.
        benchmark_root_dir is directory, containing all sources of given
            benchmark.
        benchmark_bin_dir is directory, where binaries will be put.
        """
        self.program_name = program_name
        self.framework_root_dir = os.path.join(os.path.dirname(
            os.path.abspath(__file__)), '..')
        self.benchmark_root_dir = os.path.realpath(
            os.path.join(self.framework_root_dir, benchmark_root_dir))
        self.benchmark_bin_dir = os.path.realpath(
            os.path.join(self.framework_root_dir, benchmark_bin_dir))
        self.define_build_settings('src', '')
        self.define_run_settings()

    def define_build_settings(self, sources_path, other_flags):
        self.build_settings = BuildSettings(
            benchmark_source_dir=os.path.join(
                self.benchmark_root_dir, '', sources_path),
            program_source="{0}.c".format(self.program_name),
            compiler=None,
            base_opt=None,
            optimization_flags=None,
            other_flags=other_flags,
            linker_options='-lm')

    def define_run_settings(self):
        self.run_settings = RunSettings()


"""Define the Separable Covariance Function

Author:
    Ilias Bilionis

Date:
    11/20/2012

"""

__all__ = ['SeparableCovarianceFunction']

import numpy as np
import itertools as iter
from . import CovarianceFunction

class SeparableCovarianceFunction(CovarianceFunction):

    """A Separable Covariance Function."""

    # A tuple of covariance functions
    _cov = None

    @property
    def cov(self):
        """Return covariance."""
        return self._cov

    @cov.setter
    def cov(self, value):
        """Set the covariance functions."""
        if not isinstance(value, tuple):
            raise TypeError('value must be a tuple.')
        for c in value:
            if not isinstance(c, CovarianceFunction):
                raise TypeError(
                    'value is not a tuple of covariance functions.')
        self._cov = value

    def __init__(self, cov=None, name='SeparableCovarianceFunction'):
        """Initialize the object.

        Keyword Arguments:
        cov     ---     A tuple of covariance functions

        """
        super(SeparableCovarianceFunction, self).__init__(name=name)
        if cov is not None:
            self.cov = cov

    def _check_if_tuple(self, x, var_name):
        """Check if x is a tuple of the right length"""
        if not isinstance(x, tuple):
            raise TypeError(var_name +
                    ' must be a tuple.')
        if not len(x) == len(self.cov):
            raise TypeError(var_name +
                    ' must have the same number of components as the number'
                    + ' of covariance functions.')

    def __call__(self, hyp, x1, x2=None, A=None):
        """Evaluate the covariance function."""
        self._check_if_tuple(hyp, 'hyp')
        self._check_if_tuple(x1, 'x1')
        if x2 is None:
            x2 = ()
            for _ in xrange(len(self.cov)):
                x2 += (None, )
        else:
            self._check_if_tuple(x2, 'x2')
        if A is None:
            A = ()
            for c, chyp, cx1, cx2 in iter.izip(self.cov, hyp, x1, x2):
                A += (c(chyp, cx1, cx2), )
            return A
        else:
            self._check_if_tuple(A, 'A')
            for c, chyp, cx1, cx2, cA in iter.izip(self.cov, hyp, x1, x2, A):
                c(chyp, cx1, cx2, cA)

    def __str__(self):
        """Return a string representation of the object."""
        s = super(SeparableCovarianceFunction, self).__str__()
        for c in self.cov:
            s += '\n' + str(c)
        return s


from openvino.inference_engine import IENetwork, IECore
import numpy as np
import sys
import os
import cv2

class InferenceModel:
    def __init__(self, device='MYRIAD'):
        #self.ie = IECore()
        self.device = device

    def create_exec_infer_model(self, model_dir, labels, num_requests=2):

        model_xml = os.path.join(model_dir, 'frozen_inference_graph.xml')
        model_bin = os.path.join(model_dir, 'frozen_inference_graph.bin')
        exported_model = os.path.join(model_dir, 'exported_model')

        assert os.path.isfile(model_bin)
        assert os.path.isfile(model_xml)

        ie = IECore()
        net = IENetwork(model=model_xml, weights=model_bin)

        # return ExecInferModel()

        img_info_input_blob = None
        feed_dict = {}
        for blob_name in net.inputs:
            if len(net.inputs[blob_name].shape) == 4:
                input_blob = blob_name
            elif len(net.inputs[blob_name].shape) == 2:
                img_info_input_blob = blob_name
            else:
                raise RuntimeError("Unsupported {}D input layer '{}'. Only 2D and 4D input layers are supported"
                                   .format(len(net.inputs[blob_name].shape), blob_name))

        assert len(
            net.outputs) == 1, "Demo supports only single output topologies"
        out_blob = next(iter(net.outputs))

        if os.path.isfile(exported_model):  # found exported mode
            print('found model to import')
            exec_net = ie.import_network(
                model_file=exported_model, device_name=self.device)
        else:
            print('creating exec model')
            exec_net = ie.load_network(
                network=net, num_requests=num_requests, device_name=self.device)
            exec_net.export(exported_model)

        n, c, h, w = net.inputs[input_blob].shape
        if img_info_input_blob:
            feed_dict[img_info_input_blob] = [h, w, 1]

        del net
        del ie

        return ExecInferModel(exec_net, input_blob, out_blob, feed_dict, n, c, h, w, num_requests, labels)

class ExecInferModel:
    def __init__(self, exec_net, input_blob, out_blob, feed_dict, n, c, h, w, num_requests, labels):
        self.exec_net = exec_net
        self.labels = labels
        self.input_blob = input_blob
        self.out_blob = out_blob
        self.feed_dict = feed_dict
        self.n = n
        self.c = c
        self.h = h
        self.w = w
        self.num_requests = num_requests
        self.current_frames = {}
        self.detected_objects = {}

    def infer_image(self, image, threshhold=0.7):
        height, width = image.shape[:2]
        in_frame = cv2.resize(image, (self.w, self.h))
        in_frame = in_frame.transpose((2, 0, 1))
        in_frame = in_frame.reshape(
            (self.n, self.c, self.h, self.w))

        self.feed_dict[self.input_blob] = in_frame
        self.exec_net.start_async(
            request_id=0, inputs=self.feed_dict)

        if self.exec_net.requests[0].wait(-1) == 0:
            res = self.exec_net.requests[0].outputs[self.out_blob]

            for obj in res[0][0]:
                if obj[2] > threshhold:
                    xmin = int(obj[3] * width)
                    ymin = int(obj[4] * height)
                    xmax = int(obj[5] * width)
                    ymax = int(obj[6] * height)

                    class_id = int(obj[1])
                    color = (0, 255, 255)
                    cv2.rectangle(image, (xmin, ymin),
                                  (xmax, ymax), color, 2)

                    if self.labels:
                        det_label = self.labels[class_id-1]
                    else:
                        det_label = 'class id ' + str(class_id)
                    cv2.putText(image, det_label + ' ' + str(round(obj[2] * 100, 1)) + ' %', (xmin, ymin - 7),
                                cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1)
        return image


import os
import unittest

from DataBase.DBConstants import DBConstants
from DataBase.DataBaseDataTypes.data_base_data_batch import DBBatch
from DataBase.DataBaseDataTypes.data_base_range import DBRange
from DataBase.data_base import DataBase
from support.configuration_values import ConfigurationValues

class TestDataBase(unittest.TestCase):
    config_path = "config.ini"
    db_path = "db.nc"

    def test_create_new_file(self):
        # we first make sure the file really is missing
        if os.path.isfile(TestDataBase.db_path):
            os.remove(TestDataBase.db_path)

        _ = DataBase(ConfigurationValues(TestDataBase.config_path))
        self.assertTrue(os.path.isfile(TestDataBase.db_path), "the file is missing")

    def test_insert_and_load_temp(self):
        db = DataBase(ConfigurationValues(TestDataBase.config_path))

        data_range = DBRange(time_range=(0, 1, 1),
                             lat_range=(0, 30, 0.01),
                             lon_range=(0, 30, 0.01)
                             )
        batch = DBBatch(DBConstants.VAR_TEMP, data_range)
        for i in range(30):
            batch.insert(1, i, i, i*10)
        db.insert(batch)

        data_loaded = db.load(data_range, DBConstants.VAR_TEMP)
        for i in range(30):
            self.assertEqual(data_loaded[(1, i, i)], i*10,
                             "loaded data is missing coordinates")

    def test_res_validation(self):
        """
        check if the db allow the user to insert data with the wrong resolution
        :return:
        """
        pass


from enum import Enum
from graphviz import Digraph
from datetime import datetime

class ErrType(Enum):
    LEXIC = 1
    SINTACTIC = 2

class ErrNode():
    def __init__(self, description, row):
        self.description = str(description)
        self.row = str(row)

lexicArgs = []
sintacticArgs =  []
#this variable needs to be updtaed every time
#the parser is called

def addErr(type, description,row):
    '''this function adds errors to the error report'''
    #the line number needs to be fixed first
    if type == ErrType.LEXIC:
        lexicArgs.append(ErrNode(str(description), row))
    elif type == ErrType.SINTACTIC:
        sintacticArgs.append(ErrNode(str(description), row))

def createReport():
    '''this function create the three error reports'''
    addContentToDot(Digraph(name="Lexic_errors",node_attr={'shape':'plaintext', 'color':'green'}),lexicArgs,"Lexical errors")
    addContentToDot(Digraph(name="Sintactic_errors",node_attr={'shape':'plaintext','color':'blue'}),sintacticArgs,"Sintactic errors")
    lexicArgs.clear()
    sintacticArgs.clear()

def addContentToDot(dotObj,content,reportName):
    '''add nodes to the graphviz object dotObj'''
    #string to store de html value
    lblStr = '<
'
    lblStr += "
" + reportName +"
"
    lblStr += "
Description	 Line
"
    for i in content:
        lblStr += "
"+i.description+"	 "+str(i.row)+"
"
    now = datetime.now()
    fstr = now.strftime("%d%m%y-%H%M%S")
    lblStr += "
"+fstr+"
"
    lblStr += "
>"
    dotObj.node("d1",lblStr)
    dotObj.render(reportName,'',False,True,'pdf')


def number_compare(a, b):
    """Report on whether a>b, b>a, or b==a

        >>> number_compare(1, 1)
        'Numbers are equal'

        >>> number_compare(-1, 1)
        'Second is greater'

        >>> number_compare(1, -2)
        'First is greater'
    """
    first = a
    second = b

    result = None

    if first > second:
        result = "First number is greater"
    elif first < second:
        result = "Second number is greater"
    else:
        result = "Numbers are equal"

    return result


""" File Checkpoint """

from xcputils.checkpoints import Checkpoint
from xcputils.ingestion.file import FileIngestor
from xcputils.ingestion.string import StringIngestor
from xcputils.streaming.file import FileStreamWriter

class FileCheckpoint(Checkpoint):
    """ File Checkpoint """

    def __init__(
        self,
        name: str,
        directory: str,
        ):
        """ Constructor """
        self.file_path=f"{directory}/{name}.json"

    def get(self, default: str = None) -> str:
        """ Get checkpoint """
        try:
            checkpoint = FileIngestor(self.file_path).write_to_string()
        except:
            checkpoint = ""

        if checkpoint == "":
            checkpoint = default

        return checkpoint

    def set(self, value: str):
        """ Set checkpoint """
        StringIngestor(value, FileStreamWriter(self.file_path)).ingest()


# pylint: disable=missing-docstring
from unittest.mock import Mock

import pytest

from peltak.core.hooks import HooksRegister

def test_is_called_when_subscribed():
    fake_handler = Mock()
    register = HooksRegister()

    register('fake-hook')(fake_handler)
    register.call('fake-hook')

    fake_handler.assert_called_once_with()

def test_is_not_called_after_unsubscribed():
    conf = {}
    fake_handler = Mock()
    register = HooksRegister()

    register('fake-hook')(fake_handler)
    register.remove('fake-hook', fake_handler)
    register.call('fake-hook', conf)

    fake_handler.assert_not_called()

def test_passes_arguments_to_the_hooks():
    param = {'name': 'hello'}
    fake_handler = Mock()
    register = HooksRegister()

    register('fake-hook')(fake_handler)
    register.call('fake-hook', param, 1)

    fake_handler.assert_called_once_with(param, 1)

def test_can_register_hooks_with_decorator():
    fake_fn = Mock()
    register = HooksRegister()

    @register('fake-hook')
    def post_conf_load():       # pylint: disable=unused-variable
        fake_fn()

    register.call('fake-hook')

    fake_fn.assert_called_once_with()

def test_registering_with_empty_name_raises_ValueError():
    fake_handler = Mock()
    register = HooksRegister()

    with pytest.raises(ValueError):
        register('')(fake_handler)

def test_calling_a_hook_with_empty_name_raises_ValueError():
    register = HooksRegister()

    with pytest.raises(ValueError):
        register.call('')

def test_trying_to_remove_unregistered_hook_raises_ValueError():
    fake_handler1 = Mock()
    fake_handler2 = Mock()
    register = HooksRegister()

    with pytest.raises(ValueError):
        register.remove('fake-hook', fake_handler1)

    register('fake-hook')(fake_handler2)

    with pytest.raises(ValueError):
        register.remove('fake-hook', fake_handler1)

def test_registering_hook_twice_raises_ValueError():
    fake_handler = Mock()
    register = HooksRegister()

    register('fake-hook')(fake_handler)

    with pytest.raises(ValueError):
        register('fake-hook')(fake_handler)


# import gdkeji
import caizhengbu
# import fagaiwei
# import gdshangwu
# import guowuyuan
# import kejibu
import xlwt
import os

#创建文件夹
def mkdir(path):
    folder = os.path.exists(path)
    if not folder:  # 判断是否存在文件夹如果不存在则创建为文件夹
        os.makedirs(path)  # makedirs 创建文件时如果路径不存在会创建这个路径
        print("---  new folder...  ---")
        print("---  OK  ---")

    else:
        print("---  There is this folder!  ---")

if __name__ == '__main__':
    #创建文件夹调用
    file = "/home/260199/政府政策公告信息/超链接"
    mkdir(file)  # 调用函数

    workbook = xlwt.Workbook()
    caizhengbu.main(workbook)
    workbook.save("/home/260199/政府政策公告信息/政府政策公告.xlsx")


"""
This module contains the class TedTalkDatabase, which provides an interface to
an SQLite database containing information about Ted Talks. It is designed to be
used for the HTML web pages in tedtalks_html.py and
REST API implementation in tedtalks_api.py.
"""

import os
import sqlite3
import csv

def row_to_dict_or_none(cur):
    """
    Given a cursor that has just been used to execute a query, try to fetch one
    row. If the there is no row to fetch, return None, otherwise return a
    dictionary representation of the row.

    :param cur: a cursor that has just been used to execute a query
    :return: a dict representation of the next row, or None
    """
    row = cur.fetchone()

    if row is None:
        return None
    else:
        return dict(row)

class TedTalkDatabase:
    """
    This class provides methods for getting and inserting information about
    speeches, speakers, topics, and reviews into an SQLite database.
    """

    def __init__(self, sqlite_filename):
        """
        Creates a connection to the database, and creates tables if the
        database file did not exist prior to object creation.

        :param sqlite_filename: the name of the SQLite database file
        """
        if os.path.isfile(sqlite_filename):
            create_tables = False
        else:
            create_tables = True

        self.conn = sqlite3.connect(sqlite_filename)
        self.conn.row_factory = sqlite3.Row

        cur = self.conn.cursor()
        cur.execute('PRAGMA foreign_keys = 1')
        cur.execute('PRAGMA journal_mode = WAL')
        cur.execute('PRAGMA synchronous = NORMAL')

        if create_tables:
            self.create_tables()

    def create_tables(self):
        """
        Create the tables speaker, topic, speech, and review.
        """
        cur = self.conn.cursor()
        cur.execute('CREATE TABLE speaker(speaker_id INTEGER PRIMARY KEY, '
                    '    speaker TEXT UNIQUE)')
        cur.execute('CREATE TABLE topic(topic_id INTEGER PRIMARY KEY, '
                    '    topic TEXT UNIQUE)')
        cur.execute('CREATE TABLE speech(speech_id INTEGER PRIMARY KEY, '
                    '    title TEXT, duration INTEGER, '
                    '    views INTEGER, date DATE, '
                    '    topic_id INTEGER, speaker_id INTEGER, '
                    '    FOREIGN KEY (topic_id) REFERENCES topic(topic_id), '
                    '    FOREIGN KEY (speaker_id) '
                    '       REFERENCES speaker(speaker_id))')
        cur.execute('CREATE TABLE review(review_id INTEGER PRIMARY KEY, '
                    '    content TEXT, speech_id INTEGER, '
                    '    FOREIGN KEY (speech_id) REFERENCES speech(speech_id))')
        self.conn.commit()

    def insert_review(self, content, speech_id):
        """
        Inserts a review into the database. The review's speech_id
        must already exist in the database.

        Returns a dictionary representation of the review.

        :param content: content of the review
        :param speech_id: id of the speech that the review refers to
        :return: a dict representing the review
        """
        cur = self.conn.cursor()
        speech_dict = self.get_speech_by_id(speech_id)
        try:
            speech_id = speech_dict['speech_id']

            query = ('INSERT INTO review(content, speech_id) '
                     'VALUES(?, ?)')

            cur.execute(query, (content, speech_id))

            self.conn.commit()

            return self.get_review_by_id(cur.lastrowid)

        except TypeError:
            print("Speech with the given ID does not exist.")
            return 1

    def get_review_by_id(self, review_id):
        """
        Given a review's primary key, return a dictionary representation of the
        review, or None if there is no review with that primary key.

        :param review_id: the primary key of the review
        :return: a dict representing the review
        """
        cur = self.conn.cursor()

        query = ('SELECT review.review_id as review_id, '
                 'review.content as content, '
                 'speech.speech_id as speech_id, '
                 'speech.title as speech_title '
                 'FROM speech, review '
                 'WHERE review.speech_id = speech.speech_id '
                 'AND review.review_id = ?')

        cur.execute(query, (review_id,))
        return row_to_dict_or_none(cur)

    def get_all_reviews(self):
        """
        Return a list dictionaries representing all of the reviews in the
        database.

        :return: a list of dict objects representing reviews
        """

        cur = self.conn.cursor()

        query = ('SELECT review.review_id as review_id, '
                 'review.content as content, '
                 'speech.speech_id as speech_id, '
                 'speech.title as speech_title '
                 'FROM speech, review '
                 'WHERE review.speech_id = speech.speech_id')

        reviews = []
        cur.execute(query)

        for row in cur.fetchall():
            reviews.append(dict(row))

        return reviews

    def get_reviews_by_speech(self, speech_id):
        """
        Given a speech's primary key, return a list of
        all reviews for that speech.

        :param speech_id: the primary key of the speech
        :return: a list dict representing the reviews for the given speech
        """
        cur = self.conn.cursor()

        query = ('SELECT review.review_id as review_id, '
                 'review.content as content, '
                 'speech.title as speech_title, '
                 'speech.speech_id as speech_id '
                 'FROM review, speech '
                 'WHERE speech.speech_id = review.speech_id '
                 'AND speech.speech_id = ?')

        reviews = []
        cur.execute(query, (speech_id,))
        for row in cur.fetchall():
            reviews.append(dict(row))

        return reviews

    def delete_review(self, review_id):
        """
        Delete the review with the given primary key.

        :param review_id: primary key of the review
        """

        cur = self.conn.cursor()

        query = 'DELETE FROM review WHERE review_id = ?'
        cur.execute(query, (review_id,))

        self.conn.commit()

    def insert_speech(self, title, duration, views, date, topic, speaker):
        """
        Inserts a speech into the database. If the speech's topic/speaker
        is not already in the database, inserts topic/speaker too.

        Returns a dictionary representation of the speech.

        :param title: title of the speech
        :param duration: duration of the speech, in seconds
        :param views: number of views of the speech
        :param date: the date of the speech
        :param topic: the topic of the speaker
        :param speaker: name of the speaker
        :return: a dict representing the speech
        """
        cur = self.conn.cursor()
        self.insert_topic(topic)
        self.insert_speaker(speaker)
        topic_dict = self.get_topic_by_name(topic)
        topic_id = topic_dict['topic_id']
        speaker_dict = self.get_speaker_by_name(speaker)
        speaker_id = speaker_dict['speaker_id']

        query = ('INSERT INTO speech(title, duration, views, '
                 '                   date, topic_id, speaker_id) '
                 'VALUES(?, ?, ?, ?, ?, ?)')

        cur.execute(query, (title, duration, views,
                            date, topic_id, speaker_id))
        self.conn.commit()

        return self.get_speech_by_id(cur.lastrowid)

    def get_speech_by_id(self, speech_id):
        """
        Given a speech's primary key, return a dictionary representation of the
        speech, or None if there is no speech with that primary key.

        :param speech_id: the primary key of the speech
        :return: a dict representing the speech
        """
        cur = self.conn.cursor()

        query = ('SELECT speech.speech_id as speech_id, speech.title as title, '
                 'speech.duration as duration, speech.views as views, '
                 'speech.date as date, topic.topic as topic, '
                 'topic.topic_id as topic_id, '
                 'speaker.speaker as speaker, '
                 'speaker.speaker_id as speaker_id '
                 'FROM speech, topic, speaker '
                 'WHERE speech.topic_id = topic.topic_id '
                 'AND speech.speaker_id = speaker.speaker_id '
                 'AND speech.speech_id = ?')

        cur.execute(query, (speech_id,))
        return row_to_dict_or_none(cur)

    def get_all_speeches(self):
        """
        Return a list dictionaries representing all of the speeches in the
        database.

        :return: a list of dict objects representing speeches
        """

        cur = self.conn.cursor()

        query = ('SELECT speech.speech_id as speech_id, speech.title as title, '
                 'speech.duration as duration, speech.views as views, '
                 'speech.date as date, topic.topic as topic, '
                 'topic.topic_id as topic_id, '
                 'speaker.speaker as speaker, '
                 'speaker.speaker_id as speaker_id '
                 'FROM speech, topic, speaker '
                 'WHERE speech.topic_id = topic.topic_id '
                 'AND speech.speaker_id = speaker.speaker_id')

        speeches = []
        cur.execute(query)

        for row in cur.fetchall():
            speeches.append(dict(row))

        return speeches

    def get_speeches_by_speaker(self, speaker_id):
        """
        Given a speaker's primary key, return a list of
        all speeches by that speaker.

        :param speaker_id: the primary key of the speaker
        :return: a list dict representing the speeches by that speaker
        """
        cur = self.conn.cursor()

        query = ('SELECT speech.speech_id as speech_id, speech.title as title, '
                 'speech.duration as duration, speech.views as views, '
                 'speech.date as date, topic.topic as topic, '
                 'topic.topic_id as topic_id, '
                 'speaker.speaker as speaker, '
                 'speaker.speaker_id as speaker_id '
                 'FROM speech, topic, speaker '
                 'WHERE speech.topic_id = topic.topic_id '
                 'AND speech.speaker_id = speaker.speaker_id '
                 'AND speaker.speaker_id = ?')

        speeches = []
        cur.execute(query, (speaker_id,))
        for row in cur.fetchall():
            speeches.append(dict(row))

        return speeches

    def get_speeches_by_topic(self, topic_id):
        """
        Given a topic's primary key, return a list of
        all speeches with that topic.

        :param topic_id: the primary key of the topic
        :return: a list dict representing the speeches with the given topic
        """
        cur = self.conn.cursor()

        query = ('SELECT speech.speech_id as speech_id, speech.title as title, '
                 'speech.duration as duration, speech.views as views, '
                 'speech.date as date, topic.topic as topic, '
                 'topic.topic_id as topic_id, '
                 'speaker.speaker as speaker, '
                 'speaker.speaker_id as speaker_id '
                 'FROM speech, topic, speaker '
                 'WHERE speech.topic_id = topic.topic_id '
                 'AND speech.speaker_id = speaker.speaker_id '
                 'AND topic.topic_id = ?')

        speeches = []
        cur.execute(query, (topic_id,))
        for row in cur.fetchall():
            speeches.append(dict(row))

        return speeches

    def delete_speech(self, speech_id):
        """
        Delete the speech with the given primary key.

        :param speech_id: primary key of the speech
        """

        cur = self.conn.cursor()

        query1 = 'DELETE FROM review WHERE speech_id = ?'
        cur.execute(query1, (speech_id,))
        query2 = 'DELETE FROM speech WHERE speech_id = ?'
        cur.execute(query2, (speech_id,))

        self.conn.commit()

    def insert_topic(self, topic):
        """
        Insert a topic into the database if it does not exist. Do nothing if
        there is already a topic with the given name in the database.

        Return a dict representation of the topic.

        :param topic: the topic of the speech
        :return: dict representing the topic
        """
        cur = self.conn.cursor()
        query = 'INSERT OR IGNORE INTO topic(topic) VALUES(?)'
        cur.execute(query, (topic,))
        self.conn.commit()
        return self.get_topic_by_name(topic)

    def get_all_topics(self):
        """
        Get a list of dictionary representations of all the topics in the
        database.

        :return: list of dicts representing all topics
        """
        cur = self.conn.cursor()

        query = 'SELECT * FROM topic'

        topics = []
        cur.execute(query)

        for row in cur.fetchall():
            topics.append(dict(row))

        return topics

    def get_topic_by_id(self, topic_id):
        """
        Get a dictionary representation of the topic with the given primary
        key. Return None if the topic does not exist.

        :param topic_id: primary key of the topic
        :return: a dict representing the topic, or None
        """
        cur = self.conn.cursor()
        query = 'SELECT topic_id, topic FROM topic WHERE topic_id = ?'
        cur.execute(query, (topic_id,))
        return row_to_dict_or_none(cur)

    def get_topic_by_name(self, topic):
        """
        Get a dictionary representation of the topic with the given name.
        Return None if there is no such topic.

        :param topic: the name of the topic
        :return: a dict representing the topic, or None
        """
        cur = self.conn.cursor()
        query = 'SELECT topic_id, topic FROM topic WHERE topic = ?'
        cur.execute(query, (topic,))
        return row_to_dict_or_none(cur)

    def delete_topic(self, topic_id):
        """
        Delete the topic with the given id, and
        delete relevant speeches and reviews for that topic.

        :param topic_id: primary key of the topic
        """

        cur = self.conn.cursor()

        speeches = []
        speeches = self.get_speeches_by_topic(topic_id)
        for speech in speeches:
            speech_id = speech['speech_id']
            query = 'DELETE FROM review WHERE speech_id = ?'
            cur.execute(query, (speech_id,))

        query1 = 'DELETE FROM speech WHERE topic_id = ?'
        cur.execute(query1, (topic_id,))
        query2 = 'DELETE FROM topic WHERE topic_id = ?'
        cur.execute(query2, (topic_id,))

        self.conn.commit()

    def insert_speaker(self, speaker):
        """
        Insert a speaker into the database if he/she/they does not exist.
        Do nothing if there is already a speaker
        with the given name in the database.

        Return a dict representation of the speaker.

        :param speaker: name of the speaker
        :return: dict representing the speaker
        """
        cur = self.conn.cursor()
        query = 'INSERT OR IGNORE INTO speaker(speaker) VALUES(?)'
        cur.execute(query, (speaker,))
        self.conn.commit()
        return self.get_speaker_by_name(speaker)

    def get_all_speakers(self):
        """
        Get a list of dictionary representations of all the speakers in the
        database.

        :return: list of dicts representing all speakers
        """
        cur = self.conn.cursor()

        query = 'SELECT * FROM speaker'

        speakers = []
        cur.execute(query)

        for row in cur.fetchall():
            speakers.append(dict(row))

        return speakers

    def get_speaker_by_id(self, speaker_id):
        """
        Get a dictionary representation of the speaker with the given primary
        key. Return None if the speaker does not exist.

        :param speaker_id: primary key of the speaker
        :return: a dict representing the speaker, or None
        """
        cur = self.conn.cursor()
        query = 'SELECT speaker_id, speaker FROM speaker WHERE speaker_id = ?'
        cur.execute(query, (speaker_id,))
        return row_to_dict_or_none(cur)

    def get_speaker_by_name(self, speaker):
        """
        Get a dictionary representation of the speaker with the given name.
        Return None if there is no such speaker.

        :param speaker: name of the speaker
        :return: a dict representing the speaker, or None
        """
        cur = self.conn.cursor()
        query = 'SELECT speaker_id, speaker FROM speaker WHERE speaker = ?'
        cur.execute(query, (speaker,))
        return row_to_dict_or_none(cur)

    def delete_speaker(self, speaker_id):
        """
        Delete the speaker and the speeches with the given speaker id.

        :param speaker_id: primary key of the speaker
        """

        cur = self.conn.cursor()

        speeches = []
        speeches = self.get_speeches_by_speaker(speaker_id)
        for speech in speeches:
            speech_id = speech['speech_id']
            query = 'DELETE FROM review WHERE speech_id = ?'
            cur.execute(query, (speech_id,))

        query1 = 'DELETE FROM speech WHERE speaker_id = ?'
        cur.execute(query1, (speaker_id,))
        query2 = 'DELETE FROM speaker WHERE speaker_id = ?'
        cur.execute(query2, (speaker_id,))

        self.conn.commit()

if __name__ == '__main__':

    db = TedTalkDatabase('tedtalks.sqlite')
    with open('sample.csv', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        for row in reader:
            db.insert_speech(row['title'], row['duration'], row['views'],
                             row['date'], row['topic'], row['speaker'])


import os
import sys
import nltk
import json
import re
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import math
import yfinance as yf

from nltk.stem.wordnet import WordNetLemmatizer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import jaccard_score
from sklearn.metrics.pairwise import cosine_similarity as cs

def main():
    sentiment_scores_test('fb', 'tokenized_words.json')
    pass

def sentiment_scores_test(ticker, input_filename):
    sentiments = pd.read_csv('supporting_data/sentiment_dataframe.csv')

    with open(input_filename, 'r') as f:
        tokens = json.loads(f.read())

    documents = reformat_documents(ticker, tokens)
    sbow = sentiment_bag_of_words(documents, sentiments)
    scores = sentiment_scores(sbow)
    print(scores)
    plot_sentiment_scores(ticker, scores)

def sentiment_scores(sbow):
    scores = {}

    for year in sbow['positive']:
        scores[year] = 0
        scores[year] += sum(sbow['positive'][year]) / len(sbow['positive'][year])

    for year in sbow['superfluous']:
        scores[year] += sum(sbow['superfluous'][year]) / len(sbow['superfluous'][year])

    for year in sbow['interesting']:
        scores[year] += sum(sbow['interesting'][year]) / len(sbow['interesting'][year])

    for year in sbow['negative']:
        scores[year] -= sum(sbow['negative'][year]) / len(sbow['negative'][year])

    for year in sbow['litigious']:
        scores[year] -= sum(sbow['litigious'][year]) / len(sbow['litigious'][year])

    for year in sbow['uncertainty']:
        scores[year] -= sum(sbow['uncertainty'][year]) / len(sbow['uncertainty'][year])

    for year in sbow['constraining']:
        scores[year] -= sum(sbow['constraining'][year]) / len(sbow['constraining'][year])

    for item in scores:
        scores[item] = math.tanh(scores[item])

    return dict(sorted(scores.items()))

def plot_sentiment_scores(ticker, scores):
    m = min(scores.keys())
    M = max(scores.keys())

    df = yf.download(ticker, start = f'20{m}-01-01', end = f'20{M}-03-01', interval = '3mo')
    df.to_csv(f'price_time_series/{ticker}.csv')
    df = pd.read_csv(f'price_time_series/{ticker}.csv')
    price = {}

    for year in scores.keys():
        price[year] = 0

        for index, row in df.iterrows():
            if row['Date'][2:4] == year and row['High'] > price[year]:
                price[year] = row['High']

    try:
        price.pop('99')
        scores.pop('99')
    except:
        pass

    figure, (ax1, ax2) = plt.subplots(2)#, sharex = True

    ax1.plot(price.keys(), price.values())
    ax1.set(title = f'{ticker}', xlabel = 'years', ylabel = 'stock price')
    ax2.plot(scores.keys(), scores.values())
    ax2.set(xlabel = 'years', ylabel = 'sentiment score')

    #plt.savefig('sample', dpi=300)

    plt.show()

def similarity_test(input_filename, ticker):

    sentiments = pd.read_csv('supporting_data/sentiment_dataframe.csv')

    with open(input_filename, 'r') as f:
        tokens = json.loads(f.read())

    documents = reformat_documents(ticker, tokens)

    sbow = sentiment_bag_of_words(documents, sentiments)

    similarities = jaccard_similarity(sbow)

    with open('supporting_data/jaccard_similarities.json', 'w') as f:
        json.dump(similarities, f)

    stfidf = sentiment_tfidf(documents, sentiments)

    similarities = cosine_similarity(stfidf)

    with open('supporting_data/cosine_similarities.json', 'w') as f:
        json.dump(similarities, f)

def create_sentiment_dataframe():

    df = pd.read_csv('supporting_data/LoughranMcDonald_MasterDictionary_2018.csv')

    # Set column names and words to lower case
    df.columns = df.columns.str.lower()
    df['word'] = [str(word).lower() for word in df['word']]

    # Select sentiment word and word columns
    sentiment_words = list(df.columns[7:14])
    df = df[['word'] + sentiment_words]

    # Remove words with 0 occurences
    df[sentiment_words] = df[sentiment_words].astype(bool)
    df = df[(df[sentiment_words]).any(1)]

    # Stem words and remove duplicates
    wnl = WordNetLemmatizer()
    #df['word'] = WordNetLemmatizer().lemmatize(df['word'])
    df['word'] = [wnl.lemmatize(str(word)) for word in df['word']]
    df = df.drop_duplicates('word')

    return df

def reformat_documents(ticker, tokens):
    documents = {}

    for year in tokens[ticker]:
        documents[year] = ' '.join([item for sublist in tokens[ticker][year] for item in sublist])

    return documents

# Analysis
def sentiment_bag_of_words(documents, sentiments):
    sentiment_words = list(sentiments.columns[2:9])
    sbow = {}

    for word in sentiment_words:
        sbow[word] = {}
        vectorizer = CountVectorizer(vocabulary = sentiments[sentiments[word]]['word'],
                analyzer = 'word',
                lowercase = False,
                dtype = np.int8)

        model = vectorizer.fit(documents.values())

        for year in documents.keys():
            sbow[word][year] = model.transform([documents[year]]).toarray()[0]

    return sbow

def sentiment_tfidf(documents, sentiments):
    sentiment_words = list(sentiments.columns[2:9])

    stfidf = {}

    for word in sentiment_words:
        stfidf[word] = {}
        vectorizer = TfidfVectorizer(vocabulary = sentiments[sentiments[word]]['word'],
                analyzer = 'word',
                lowercase = False,
                dtype = np.int8)

        model = vectorizer.fit(documents.values())

        for year in documents.keys():
            stfidf[word][year] = vectorizer.transform([documents[year]]).toarray()[0]

    return stfidf

def jaccard_similarity(sbow):

    similarities = {}

    for word in sbow:
        similarities[word] = {}

        years = sorted(sbow[word].keys())
        for i in range(len(years)-1):
            x = sbow[word][years[i]].astype(bool)
            y = sbow[word][years[i + 1]].astype(bool)
            similarities[word][years[i]] = jaccard_score(x, y)

    return similarities

def cosine_similarity(stfidf):

    similarities = {}

    for word in stfidf:
        similarities[word] = {}

        years = sorted(stfidf[word].keys())
        for i in range(len(years)-1):
            x = stfidf[word][years[i]].reshape(1, -1)
            y = stfidf[word][years[i+1]].reshape(1, -1)
            sim = cs(x, y)[0,0]
            similarities[word][years[i]] = cs(x, y)[0,0]

    return similarities

if __name__ == '__main__':
    main()


import json
import pickle
from pathlib import Path

from flask import Flask, jsonify, request, render_template
import pandas as pd

from forms import PredictionParametersForm
from DS.train_and_export import generate_model

app = Flask('Futurice')
app.config['SECRET_KEY'] = 'secret_key'
model_path = Path('.') / 'model_sklearn.pkl'

if not model_path.exists():
    generate_model(output_dir=Path(__file__).parent)

with open(model_path, 'rb') as f:
    model = pickle.load(f)

def single_prediction(X):
    if isinstance(model['model'], str):
        coeffs = model['coeffs']
        intercept = model['intercept']
        col_order = model['col_order']
        std_err = model['std_err']
        data = pd.DataFrame.from_dict(X, orient='index').T
        data_point = data[col_order]
        prediction = intercept+data_point.dot(coeffs)
    else:
        regression_model = model['model']
        std_err = model['std_err']
        col_order = model['col_order']
        data = pd.DataFrame.from_dict(X, orient='index').T
        data_point = data[col_order]
        prediction = regression_model.predict(data_point)[0]
    return prediction, std_err

@app.route('/predict', methods=['POST'])
def predict():
    if request.headers['Content-Type'] == 'application/json':
        json_data = request.json
    else:
        json_data = json.loads(request.data)
    form = PredictionParametersForm.from_json(json_data)
    if form.validate():
        prediction, std_err = single_prediction(json_data)
        return jsonify({'house_value': prediction, 'stddev': std_err})
    return render_template('400.html', errors=form.errors), 400

@app.errorhandler(400)
def bad_request(e):
    return render_template('400.html'), 400

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

if __name__ == '__main__':
    app.run(debug=True)


"""buggy URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from pushers import views

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^$', views.rolls, name='rolls'),
    url(r'^login$', views.login_view, name='login'),
    # Roll day urls
    url(r'^rolls/create_roll_day$', views.create_roll_day, name='create_roll_day'),
    url(r'^rolls/(?P<r_id>[0-9]+)/edit_active_pushers$', views.edit_active_pushers, name='edit_active_pushers'),
    url(r'^rolls/(?P<r_id>[0-9]+)/activate_pusher/(?P<p_id>[0-9]+)$', views.activate_pusher, name='activate_pusher'),
    url(r'^rolls/(?P<r_id>[0-9]+)/deactivate_pusher/(?P<p_id>[0-9]+)$', views.deactivate_pusher, name='deactivate_pusher'),

    # Roll urls
    url(r'^rolls/(?P<r_id>[0-9]+)/create_roll$', views.create_roll, name='create_roll'),
    url(r'^rolls/edit_roll/(?P<r_id>[0-9]+)$', views.edit_roll, name='edit_roll'),

    # Pusher urls
    url(r'^pushers$', views.pushers, name='pushers'),
    url(r'^create_pusher$', views.create_pusher, name='create_pusher'),
    url(r'^delete_pusher/(?P<p_id>[0-9]+)$', views.delete_pusher, name='delete_pusher'),

    # Buggy urls
    url(r'^buggies$', views.buggies, name='buggies'),
    url(r'^create_buggy$', views.create_buggy, name='create_buggy'),
    url(r'^delete_buggy/(?P<b_id>[0-9]+)$', views.delete_buggy, name='delete_buggy'),
]


from app import app, database_storage
from flask_login import LoginManager, login_required, current_user, login_user
from flask import render_template, Flask, request, redirect, url_for, Response
import json

@app.route("/create_assignment", methods = ["POST"])
@login_required
def create_assignment():
    assignment_name = request.form["assignment_name"]
    assignment_points = request.form["assignment_points"]
    klass_id = request.form["klass_id"]
    if assignment_name == "":
        return "assignment_name parameter required."
    if assignment_points == "":
        return "assignment_points parameter required."
    if klass_id == "":
        return "klass_id parameter required."
    assignment_id = database_storage.createAssignment(current_user.get_id(), klass_id, assignment_name, assignment_points)
    return str(assignment_id)


from flask import Flask, render_template, Response
import cv2

app = Flask(__name__)

# Load the pre-trained Haar Cascade face detection model
face_cascade = cv2.CascadeClassifier(
    cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')

# OpenCV video capture object
cap = cv2.VideoCapture(0)

# Define route for homepage
@app.route('/')
def index():
    return render_template('index.html')

# Define route for video feed
@app.route('/video_feed')
def video_feed():
    def gen():
        while True:
            # Read a frame from the video stream
            success, frame = cap.read()

            # If a frame was successfully read, detect faces in the frame and draw rectangles around them
            if success:
                gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
                faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1,
                                                      minNeighbors=5)
                for (x, y, w, h) in faces:
                    cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)

                # Encode the frame as a jpeg image and yield it
                ret, jpeg = cv2.imencode('.jpg', frame)
                yield (b'--frame\r\n'
                       b'Content-Type: image/jpeg\r\n\r\n' + jpeg.tobytes() + b'\r\n')
            else:
                break

    # Use the Response class to stream video frames back to the web app
    return Response(gen(), mimetype='multipart/x-mixed-replace; boundary=frame')

if __name__ == '__main__':
    app.run(debug=True)


from bs4 import BeautifulSoup
import requests

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def get_movies_titles(website_link, count_tables):
    """
    Getting Titles of all the
    """
    r = requests.get(website_link)
    soup = BeautifulSoup(r.content, features="html.parser")

    # finding the tables for orbital and suborbital flights
    tables = soup.findAll('table', attrs={
        'class': ['wikitable', 'collapsible', 'mw-collapsible',
                    'mw-made-collapsible']})

    all_movies = []
    for table in tables[0:count_tables]:
        title_cells = table.find_all("th", attrs={'scope':"row"})
        for cell in title_cells:
            cell_link = cell.findAll('a')[0]
            all_movies.append(cell_link.get('title'))

    return all_movies

def get_imdb_link(movie_title, driver):
    """
    Scraping DuckDuckGo to get imdb page of a movie by its title
    movie_title (str) - title of the movie
    driver - Chrome driver for selenium

    """
    # searching the query
    query = movie_title + " imdb"
    # html_page = requests.get("https://duckduckgo.com/?q="+query)
    driver.get("https://duckduckgo.com/?q="+query)

    soup = BeautifulSoup(driver.page_source, 'html.parser')
    link_class = "eVNpHGjtxRBq_gLOfGDr"
    link_div_elems = soup.find_all("a", {"class": link_class})

    return link_div_elems[0]['href']

def get_movies_locations(all_movies, movie_locations):
    """
    Getting Filming Locations of the list of movies
    all_movies (list: str) - list of titles of movies to search for
    movie_locations (dict) - list of locations (str) for each movie title (str), store here
    """
    driver_location = '/home/sayyor/Desktop/selfStudy/race-charts/drivers/chromedriver'

    # setting up the driver
    ser = Service(driver_location)
    op = webdriver.ChromeOptions()
    driver = webdriver.Chrome(service=ser, options=op)

    for movie_title in all_movies:
        # search for the movie title
        page_link = get_imdb_link(movie_title, driver)

        # go to the locations page
        page_link = page_link.split("/")
        page_link = page_link[:-1] # removing unnecessary part
        page_link = "/".join(page_link)+'/locations' # going to the filming locations page
        html_page = requests.get(page_link)
        soup = BeautifulSoup(html_page.content, 'html.parser')

        url_links = soup.find_all("a", {"itemprop": "url"})
        print(url_links[0])
        movie_locations[movie_title] = [url_link.encode_contents().decode("utf-8").replace("\n", "") for url_link in url_links]

    driver.quit()
    return movie_locations


lst=[1,2,3]
lst1=["pune","mumbai","nashik","nagpur"]

#it will help us to navigate through multiple list parallely
for v,p in zip(lst,lst1):  #[(1,"pune"),(2,"mumbai)
    print(v,"--->",p)

lst=[12,1,3,14]

#search value in the list
s=3
cnt=0
for v in lst:
    if v==s:
        print(v,cnt)
        break
    cnt=cnt+1

#enumerate numbers the list, numbering starts with 0
for idx,v in enumerate(lst): #[(0,12),(1,1),(2,3)
    if v==s:
        print(v,idx)
        break
#enumerate numbers the list,it will start with 1
for idx,v in enumerate(lst,1):
    print(idx,".",v)

#it will pass data in the reverse order in the for loop
#it helps to print data in reverse order without changing the original list
for v in reversed(lst):
    print(v)
print(lst)

#sorted function will pass the data in the for loop in ascending order
#it display data in sorted order without changing original list
#sorted(lst,reverse=True) it will sort the data in descending order
for v in sorted(lst):
    print(v)

print(lst)


'''
clic.db.book: Fetch/store book dicts to DB
******************************************
'''
import psycopg2
import psycopg2.extras

from clic.db.lookup import rclass_id_lookup
from clic.tokenizer import types_from_string

def put_book(cur, book):
    """
    Import a book object:

    * name: The shortname of the book
    * content: The full book string, as per instructions in the corpora repository
    * An entry for each rclass, e.g. "chapter.text": See /schema/10-rclass.sql. A list of...
      - off_start: Character offset for start of this region
      - off_end: Character offset for end of this region, non-inclusive
      - rvalue: rvalue, e.g. chapter number (see /schema/10-rclass.sql)

    The book contents / regions will be imported into the database, and any
    "chapter.text" region will be tokenised.
    """
    rclass = rclass_id_lookup(cur)

    # Insert book / update content, get ID for other updates
    cur.execute("""
        SELECT book_id, token_tbl, region_tbl FROM book_import_init(%(name)s, %(content)s)
    """, dict(
        name=book['name'],
        content=book['content'],
    ))
    (book_id, token_tbl, region_tbl) = cur.fetchone()

    # Replace regions with new values
    for rclass_name in book.keys():
        if '.' not in rclass_name:
            continue  # Not an rclass
        rclass_id = rclass[rclass_name]
        psycopg2.extras.execute_values(cur, """
            INSERT INTO """ + region_tbl + """ (book_id, crange, rclass_id, rvalue) VALUES %s
        """, ((
            book_id,
            psycopg2.extras.NumericRange(off_start, off_end),
            rclass_id,
            rvalues[0] if len(rvalues) > 0 else None,
        ) for off_start, off_end, *rvalues in book[rclass_name]))

    # Tokenise each chapter text region and add it to the database
    psycopg2.extras.execute_values(cur, """
        INSERT INTO """ + token_tbl + """ (book_id, crange, ttype) VALUES %s
    """, ((
        book_id,
        psycopg2.extras.NumericRange(off_start, off_end),
        ttype,
    ) for ttype, off_start, off_end in types_from_string(
        book['content'],
        offset=0
    )))

    # Finalise token import, let DB update metadata, indexes
    cur.execute("""
        SELECT * FROM book_import_finalise(%(book_id)s)
    """, dict(
        book_id=book_id,
    ))

def get_book(cur, book_id_name, content=False, regions=False):
    """Get book from DB, specifying what details are required"""
    out = dict()

    # Initial fetch from main table
    cur.execute("".join([
        "SELECT book_id, name",
        ", content" if content else ", NULL AS content",
        " FROM book WHERE",
        " book_id = %s" if isinstance(book_id_name, int) else " name = %s",
    ]), (book_id_name,))
    (out['id'], out['name'], out['content'],) = cur.fetchone()
    if not content:
        del out['content']

    if regions:
        cur.execute("""
            SELECT (SELECT name FROM rclass WHERE rclass_id = r.rclass_id) rclass_name
                 , r.crange
                 , r.rvalue
              FROM region r
             WHERE r.book_id = %(book_id)s
          ORDER BY r.crange
        """, dict(
            book_id=out['id'],
        ))

        for rclass_name, crange, rvalue in cur:
            if rclass_name not in out:
                out[rclass_name] = []
            out[rclass_name].append((crange.lower, crange.upper, rvalue))

    return out


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import pandas as pd
import numpy as np
import time
import os
from operator import itemgetter

import sys
sys.path.append(".")   #设置自定义包的搜索路径
import Params_determining_CLPSO as cp

import train_Iter
import train_Dir
import train_MIMO

import test_Iter
import test_Dir
import test_MIMO

from sklearn.svm import SVR
from sklearn.neural_network import MLPRegressor
from sklearn.neighbors import KNeighborsRegressor
#sys.path.append("C:/D/HUST/research_flu_forecast/experiment/code/MSVR_python")
from MSVR import MSVR

# In[ ]:
F = 20          #feature dimension
TIMES = 20       #repeat times                                                   #change

# data_sourse_list = ["201001-202012_train2test1", "201001-202012_train8test2"]

series_list = ["N_ILI_ori", "N_ILI_sadj", "S_ILI_ori", "S_ILI_sadj"]
model_key = ['SVR', 'MLP', 'KNN']
model_dict = {'SVR': SVR, 'MLP':MLPRegressor, 'KNN':KNeighborsRegressor}
strategy_key = ['Iter', 'Dir', 'MIMO']
train_dict = {'Iter': train_Iter, 'Dir': train_Dir, 'MIMO':train_MIMO}
test_dict = {'Iter':test_Iter, 'Dir':test_Dir, 'MIMO':test_MIMO}

save_path = "../result_new/"                                                   #change

data_sourse_label = "201001-202012_train2test1"
print(data_sourse_label)

total_start_time = time.time()
for series_label in series_list[2:3]:                                           #change:Sori
    type = series_label[:5]

    series_save_name = series_label[0] + series_label[-3:]
    if os.path.exists(save_path + series_save_name) == False:
        os.makedirs(save_path + series_save_name)

    for model_name in model_key[1:2]:                                            #change:MLP
        ML_model = model_dict[model_name]

        for strategy_name in strategy_key[1:2]:                                   #change:Dir
            if model_name == 'SVR' and strategy_name == 'MIMO':
                ML_model = MSVR

            train = train_dict[strategy_name]
            test = test_dict[strategy_name]

            model_folder_path = save_path + series_save_name + "/"
            model_folder_name = model_name + "_"  + strategy_name
            if os.path.exists(model_folder_path + model_folder_name) == False:
                os.makedirs(model_folder_path + model_folder_name)

            file_path = model_folder_path + model_folder_name + "/"
            if os.path.exists(file_path + "aggregate_metric") == False:
                os.makedirs(file_path + "aggregate_metric")
            if os.path.exists(file_path + "TOP10") == False:
                os.makedirs(file_path + "TOP10")

            total_metric_name = "Total_Metrics_" + series_save_name + "_" + model_name + "_"  + strategy_name + ".xlsx"
            mape_step_name = "MAPE_Step_" + series_save_name + "_" + model_name + "_"  + strategy_name + ".xlsx"
            rmse_step_name = "RMSE_Step_" + series_save_name + "_" + model_name + "_"  + strategy_name + ".xlsx"
            running_time_name = "Running_Time_" + series_save_name + "_" + model_name + "_"  + strategy_name + ".xlsx"

            writer_tm = pd.ExcelWriter(file_path + "aggregate_metric/" + total_metric_name)
            writer_ms = pd.ExcelWriter(file_path + "aggregate_metric/" + mape_step_name)
            writer_rs = pd.ExcelWriter(file_path + "aggregate_metric/" + rmse_step_name)
            writer_time = pd.ExcelWriter(file_path + "aggregate_metric/" + running_time_name)

            for H in range(2,11):       #forecast horizon 2-10：range(2,11)       #change:H = 2-10

                train_data_file = "../" + data_sourse_label + "/Xy/" + series_label + "/train_" + series_label + "_X" + str(F) + "_y" + str(H) + ".xlsx"
                test_data_file = "../" + data_sourse_label + "/Xy/" + series_label + "/test_" + series_label + "_X" + str(F) + "_y" + str(H) + ".xlsx"
                sea_train_data_file = "../" + data_sourse_label + "/Xy/" + type + "_s/train_" + type + "_s_X" + str(F) + "_y" + str(H) + ".xlsx"
                sea_test_data_file = "../" + data_sourse_label + "/Xy/" + type + "_s/test_" + type + "_s_X" + str(F) + "_y" + str(H) + ".xlsx"

                results_list = []

                for t in range(TIMES):
                    print(series_save_name + "_" + model_name + "_" + strategy_name + "_H" + str(H) + "_time" + str(t) + "_START")

                    time_start = time.time()

                    if strategy_name == 'Dir':
                        model = []
                        selected_feature_index = []
                        best_params = []
                        for step in range(H):
                            train_score, single_model, single_feature_index, single_best_params = cp.run_clpso(model_name, strategy_name, train_data_file, F, H, train, ML_model, step)

                            best_params.append(single_best_params)
                            selected_feature_index.append(single_feature_index)
                            model.append(single_model)
                    else:
                        train_score, model, selected_feature_index, best_params = cp.run_clpso(model_name, strategy_name, train_data_file, F, H, train, ML_model, step = None)

                    #get RMSE on training set as weight of each model and each step for BMA
                    y_train, y_train_pred, y_train_series, y_train_pred_series = test.test_model(train_data_file, model, F, H, series_label, sea_train_data_file, selected_feature_index)
                    mape_train, rmse_train, mape_train_step, mape_train_step_std, rmse_train_step, rmse_train_step_std = test.evaluate(y_train,y_train_pred, H)
#                    test.graph(y_train_series, y_train_pred_series, H, F)
#                    print("train_total_MAPE:", mape_train)
#                    print("train_total_RMSE:", rmse_train)

                    #result of test set
                    y_test, y_test_pred, y_test_series, y_test_pred_series = test.test_model(test_data_file, model, F, H, series_label, sea_test_data_file, selected_feature_index)
#                    test.graph(y_test_series, y_test_pred_series, H, F)
                    mape, rmse, mape_step, mape_step_std, rmse_step, rmse_step_std = test.evaluate(y_test, y_test_pred, H)

                    print('----------test part----------')
                    print("total_MAPE", mape)
                    print("total_RMSE", rmse)
                    print("best params:", best_params)
                    print("best selected feature index:", selected_feature_index)

                    time_end = time.time()
                    print('run time(min):',(time_end - time_start)/60)

                    result = {}
                    result['metric_total_train'] = np.hstack((mape_train, rmse_train)).reshape(1,-1)
                    result['metric_step_train'] = np.vstack((mape_train_step, rmse_train_step))

                    result['y_test'] = y_test
                    result['y_test_pred'] = y_test_pred
                    result['y_series'] = np.concatenate((y_test_series.reshape(-1,1), y_test_pred_series.reshape(-1,1)),axis = 1)

                    result['mape'] = mape
                    result['rmse'] = rmse
                    result['metric_step'] = np.vstack((mape_step, mape_step_std, rmse_step, rmse_step_std))

                    if model_name == 'MLP':
                        result['best_params'] = np.array(best_params).reshape(1,-1)
                    else:
                        result['best_params'] = np.array(best_params)

                    result['selected_feature_index'] = np.array(selected_feature_index)
                    result['running_time'] = (time_end - time_start)/60
                    results_list.append(result)

                    print(series_save_name + "_" + model_name + "_" + strategy_name + "_H" + str(H) + "_time" + str(t) + "_END")

                #根据MAPE和RMSE和对重复实验的结果排序
                #sorted():python自带的排序函数，可结合operator模块，根据key进行复杂排序（此处：对所有result先按mape排序，若mape相同，再按rmse排序）
                sorted_results_list = sorted(results_list, key = itemgetter("mape", "rmse"))

                #save detailed results
                column_step = ['step'+str(i+1) for i in range(H)]
                index_rank = ['rank' + str(i+1) for i in range(TIMES)]

                for rank, s_r in enumerate(sorted_results_list):
                    if rank < 10:                                                #change; final experiment: rank < 10 (0-9)
                        result_path = file_path + "TOP10/"
                    else:
                        result_path = file_path

                    df1 = pd.DataFrame(s_r['metric_total_train'], columns = ['mape_train','rmse_train'])
                    df2 = pd.DataFrame(s_r['metric_step_train'], index = ['MPAE_train_step', 'RMSE_train_step'], columns = column_step)
                    df3 = pd.DataFrame(s_r['y_test'], columns = column_step)
                    df4 = pd.DataFrame(s_r['y_test_pred'], columns = column_step)
                    df5 = pd.DataFrame(s_r['y_series'], columns = ["y_test_series","y_test_series_pred"])
                    df6 = pd.DataFrame({'MAPE':[s_r['mape']],'RMSE':[s_r['rmse']]})
                    df7 = pd.DataFrame(s_r['metric_step'], index = ['MAPE_step','stdAPE_stpe','RMSE_step','stdSE_step'], columns = column_step)
                    df8 = pd.DataFrame(s_r['best_params'])
                    df9 = pd.DataFrame(s_r['selected_feature_index'])

                    result_name = series_save_name + "_" + model_name + "_"  + strategy_name + "_" + "y" + str(H) + "_rank" + str(rank + 1) + ".xlsx"

                    writer1 = pd.ExcelWriter(result_path + result_name)
                    df1.to_excel(writer1,'metric_total_train',index = False)
                    df2.to_excel(writer1,'metric_step_train')
                    df3.to_excel(writer1,'y_test',index = False)
                    df4.to_excel(writer1,'y_test_pred',index = False)
                    df5.to_excel(writer1,'y_series', index = False)
                    df6.to_excel(writer1,'metric_total', index = False)
                    df7.to_excel(writer1,'metric_step')
                    df8.to_excel(writer1,'best_params', index = False)
                    df9.to_excel(writer1,'selected_feature_index', index = False)
                    writer1.save()

                    print(rank)
                    print(sorted_results_list[rank]["mape"])
                    print(sorted_results_list[rank]["rmse"])

                #summarization of results
                MAPE_set = np.array([s_r["mape"] for s_r in sorted_results_list]).reshape(-1,1)
                RMSE_set = np.array([s_r["rmse"] for s_r in sorted_results_list]).reshape(-1,1)
                metric_set = np.concatenate((MAPE_set, RMSE_set), axis = 1)
                df_tm = pd.DataFrame(metric_set, index = index_rank, columns = ["MAPE","RMSE"])
                df_tm.to_excel(writer_tm,'H'+str(H))

                mape_step_set = np.array([s_r['metric_step'][0] for s_r in sorted_results_list])
                df_ms = pd.DataFrame(mape_step_set, index = index_rank, columns = column_step)
                df_ms.to_excel(writer_ms, 'H'+str(H))

                rmse_step_set = np.array([s_r['metric_step'][2] for s_r in sorted_results_list])
                df_rs = pd.DataFrame(rmse_step_set, index = index_rank, columns = column_step)
                df_rs.to_excel(writer_rs, 'H'+str(H))

                running_time_set = np.array([s_r['running_time'] for s_r in sorted_results_list]).reshape(-1,1)
                df_time = pd.DataFrame(running_time_set, index = index_rank, columns = ['running_time (min)'])
                df_time.to_excel(writer_time, 'H'+str(H))

            writer_tm.save()
            writer_ms.save()
            writer_rs.save()
            writer_time.save()

total_end_time = time.time()
print('totally run time(min):',(total_end_time - total_start_time)/60)


from eth_utils import ValidationError
import pytest

from eth2._utils.hash import hash_eth2
from eth2._utils.merkle.normal import (
    calc_merkle_tree,
    get_merkle_proof,
    get_merkle_root,
    get_merkle_root_from_items,
    get_root,
    verify_merkle_proof,
)

@pytest.mark.parametrize(
    "leaves,tree",
    [
        ((b"single leaf",), ((hash_eth2(b"single leaf"),),)),
        (
            (b"left", b"right"),
            (
                (hash_eth2(hash_eth2(b"left") + hash_eth2(b"right")),),
                (hash_eth2(b"left"), hash_eth2(b"right")),
            ),
        ),
        (
            (b"1", b"2", b"3", b"4"),
            (
                (
                    hash_eth2(
                        hash_eth2(hash_eth2(b"1") + hash_eth2(b"2"))
                        + hash_eth2(hash_eth2(b"3") + hash_eth2(b"4"))
                    ),
                ),
                (
                    hash_eth2(hash_eth2(b"1") + hash_eth2(b"2")),
                    hash_eth2(hash_eth2(b"3") + hash_eth2(b"4")),
                ),
                (hash_eth2(b"1"), hash_eth2(b"2"), hash_eth2(b"3"), hash_eth2(b"4")),
            ),
        ),
    ],
)
def test_merkle_tree_calculation(leaves, tree):
    calculated_tree = calc_merkle_tree(leaves)
    assert calculated_tree == tree
    assert get_root(tree) == tree[0][0]
    assert get_merkle_root_from_items(leaves) == get_root(tree)

@pytest.mark.parametrize("leave_number", [0, 3, 5, 6, 7, 9])
def test_invalid_merkle_root_calculation(leave_number):
    with pytest.raises(ValueError):
        get_merkle_root_from_items((b"",) * leave_number)

@pytest.mark.parametrize(
    "leaves,index,proof",
    [
        ((b"1", b"2"), 0, (hash_eth2(b"2"),)),
        ((b"1", b"2"), 1, (hash_eth2(b"1"),)),
        (
            (b"1", b"2", b"3", b"4"),
            0,
            (hash_eth2(b"2"), hash_eth2(hash_eth2(b"3") + hash_eth2(b"4"))),
        ),
        (
            (b"1", b"2", b"3", b"4"),
            1,
            (hash_eth2(b"1"), hash_eth2(hash_eth2(b"3") + hash_eth2(b"4"))),
        ),
        (
            (b"1", b"2", b"3", b"4"),
            2,
            (hash_eth2(b"4"), hash_eth2(hash_eth2(b"1") + hash_eth2(b"2"))),
        ),
        (
            (b"1", b"2", b"3", b"4"),
            3,
            (hash_eth2(b"3"), hash_eth2(hash_eth2(b"1") + hash_eth2(b"2"))),
        ),
    ],
)
def test_merkle_proofs(leaves, index, proof):
    tree = calc_merkle_tree(leaves)
    root = get_root(tree)
    item = leaves[index]
    calculated_proof = get_merkle_proof(tree, index)
    assert calculated_proof == proof
    assert verify_merkle_proof(root, item, index, calculated_proof)

    assert not verify_merkle_proof(b"\x00" * 32, item, index, proof)
    assert not verify_merkle_proof(root, b"\x00" * 32, index, proof)
    assert not verify_merkle_proof(root, item, (index + 1) % len(leaves), proof)
    for replaced_index in range(len(proof)):
        altered_proof = (
            proof[:replaced_index] + (b"\x00" * 32,) + proof[replaced_index + 1 :]
        )
        assert not verify_merkle_proof(root, item, index, altered_proof)

def test_single_element_merkle_proof():
    leaves = (b"1",)
    tree = calc_merkle_tree(leaves)
    root = get_root(tree)
    assert get_merkle_proof(tree, 0) == ()
    assert verify_merkle_proof(root, b"1", 0, ())
    assert not verify_merkle_proof(b"\x00" * 32, b"1", 0, ())
    assert not verify_merkle_proof(root, b"2", 0, ())
    assert not verify_merkle_proof(root, b"1", 0, (b"\x00" * 32,))

@pytest.mark.parametrize("leaves", [(b"1",), (b"1", b"2"), (b"1", b"2", b"3", b"4")])
def test_proof_generation_index_validation(leaves):
    tree = calc_merkle_tree(leaves)
    for invalid_index in [-1, len(leaves)]:
        with pytest.raises(ValidationError):
            get_merkle_proof(tree, invalid_index)

def test_get_merkle_root():
    hash_0 = b"0" * 32
    leaves = (hash_0,)
    root = get_merkle_root(leaves)
    assert root == hash_0

    hash_1 = b"1" * 32
    leaves = (hash_0, hash_1)
    root = get_merkle_root(leaves)
    assert root == hash_eth2(hash_0 + hash_1)


import unittest
import logging

try:
    from gi.repository import Grl
except:
    logging.warning("Unable to import Grilo's introspection bindings")
    exit()
try:
    from gi.repository import GLib
except TypeError:
    # there seems to be a bug with GVariants these days that triggers an
    # exception when importing GLib. Seems to be harmless though. We import
    # explicitly GLib in a try/except here so that we can handle that, else the
    # exception would be raised the first time it is imported by other code
    # (e.g. by using a GDateTime)
    from gi.repository import GLib

class TestMedia(unittest.TestCase):
    def test_creation(self):
        media = Grl.Media.new()
        self.assertFalse(media is None)

    def test_publication_date(self):
        media = Grl.Media.new()
        date_string = "2001-02-03T04:05:06Z"
        date = Grl.date_time_from_iso8601 (date_string)

        self.assertTrue(media.get_publication_date() is None)
        media.set_publication_date(date)
        d = media.get_publication_date()
        self.assertTrue(d is not None)
        self.assertEqual(d.get_year(), 2001)
        self.assertEqual(d.get_month(), 2)
        self.assertEqual(d.get_day_of_month(), 3)
        self.assertEqual(d.get_hour(), 4)
        self.assertEqual(d.get_minute(), 5)
        self.assertEqual(d.get_second(), 6)
        self.assertEqual(d.format("%FT%H:%M:%S%Z"), date.format("%FT%H:%M:%S%Z"))

        # date only
        date_string = "2001-02-03"
        date = Grl.date_time_from_iso8601 (date_string)
        media.set_publication_date(date)
        d = media.get_publication_date()
        self.assertTrue(d is not None)
        self.assertEqual(d.get_year(), 2001)
        self.assertEqual(d.get_month(), 2)
        self.assertEqual(d.get_day_of_month(), 3)
        self.assertEqual(d.get_hour(), 12)
        self.assertEqual(d.get_minute(), 0)
        self.assertEqual(d.get_second(), 0)
        self.assertEqual(d.format("%FT%H:%M:%S%Z"), date.format("%FT%H:%M:%S%Z"))
        self.assertEqual(d.format("%FT%H:%M:%S%Z"), date_string + "T12:00:00UTC")

# who said "this is ugly" ?
Grl.init([])
registry = Grl.Registry.get_default()
try:
    registry.load_plugin_by_id("grl-filesystem")
except GLib.Error:
    pass


import os
import season

if wiz.request.uri() == "/pwa/sw.js":
    swjs = "self.addEventListener('fetch', function (e) {});"
    wiz.response.send(swjs, content_type="text/javascript")

if wiz.request.uri() == "/pwa/manifest.json":
    config = wiz.model("dizest").package()

    manifest = dict()
    manifest['name'] = config['title']
    manifest['short_name'] = config['title']
    manifest['start_url'] = '/'
    manifest['icons'] = [
        {
            "src": "/pwa/icon.png",
            "sizes": "192x192",
            "type": "image/png"
        },
        {
            "src": "/pwa/icon.png",
            "sizes": "512x512",
            "type": "image/png"
        }
    ]

    manifest['theme_color'] = '#3843d0'
    manifest['background_color'] = '#3843d0'
    manifest['display'] = 'standalone'
    manifest['orientation'] = 'any'

    wiz.response.json(manifest)

if wiz.request.uri() == "/pwa/icon.png":
    BASEPATH = os.path.realpath(season.path.project + "/..")
    fs = season.util.os.FileSystem(BASEPATH)
    if fs.exists('icon.png') == False:
        respath = os.path.join(wiz.branchpath(), 'resources')
        iconpath = season.util.os.FileSystem(respath).abspath('images/brand/icon.png')
        wiz.response.download(iconpath)

    iconpath = fs.abspath('icon.png')
    wiz.response.download(iconpath)

wiz.response.status(404)


# pylint: disable=missing-docstring
from unittest.mock import Mock, patch

import pytest

from peltak.core import util

class TestCachedResults:
    def test_caching_works(self):
        @util.cached_result()
        def foo():
            call_count = getattr(foo, 'call_count', 0)
            call_count += 1
            setattr(foo, 'call_count', call_count)

        foo()
        assert foo.call_count == 1

        foo()
        assert foo.call_count == 1

        util.cached_result.clear(foo)

        foo()
        assert foo.call_count == 2

    def test_clear_deletes_the_cached_attr(self):
        @util.cached_result()
        def foo():
            pass

        foo()
        assert hasattr(foo, util.cached_result.CACHE_VAR)

        util.cached_result.clear(foo)
        assert not hasattr(foo, util.cached_result.CACHE_VAR)

    def test_can_call_clear_on_uncalled_function(self):
        @util.cached_result()
        def foo():
            pass

        util.cached_result.clear(foo)

class TestTimedBlock:
    # TODO: Use freezegun here
    @patch('time.time', Mock(side_effect=[1, 2]))
    def test_returns_proper_results(self):

        with util.timed_block() as t:
            pass

        assert t.elapsed == 1
        assert t.elapsed_ms == 1000
        assert t.elapsed_s == 1

class TestGetFromDict:
    @pytest.fixture
    def test_dict(self):
        yield {
            'root': 'root value',
            'test1': {
                'sub': 'nested value'
            }
        }

    @pytest.mark.parametrize('query,expected', [
        ('root', 'root value'),
        ('test1.sub', 'nested value'),
        ('test1', {'sub': 'nested value'}),
    ])
    def test_returns_correct_values(self, test_dict, query, expected):
        assert util.get_from_dict(test_dict, query) == expected

    @pytest.mark.parametrize('query', [
        'test1.invalid',
        'invalid.sub'
    ])
    def test_return_default_when_provided_and_value_not_found(self, test_dict, query):
        assert util.get_from_dict(test_dict, query, -1) == -1

    @pytest.mark.parametrize('query', [
        'test1.invalid',
        'invalid',
        'invalid.sub',
    ])
    def test_raises_KeyError_if_default_not_given(self, query, test_dict):
        with pytest.raises(KeyError):
            util.get_from_dict(test_dict, query)

class TestSetInDict:
    def test_can_set_root_value(ctx):
        d = {}
        util.set_in_dict(d, 'test', 'value')

        assert d == {'test': 'value'}

    def test_can_set_sub_value(ctx):
        d = {}
        util.set_in_dict(d, 'test.sub', 'value')

        assert d == {'test': {'sub': 'value'}}

    def test_raises_KeyError_if_part_of_the_path_is_not_a_dict(ctx):
        d = {'test': 'value'}

        with pytest.raises(KeyError):
            util.set_in_dict(d, 'test.sub', 'value')

        with pytest.raises(KeyError):
            util.set_in_dict(d, 'test.sub.sub', 'value')


from django.shortcuts import render, redirect
from phones.models import Phone

def index(request):
    return redirect('catalog')

def show_catalog(request):
    sort = request.GET.get('sort')
    template = 'catalog.html'
    phones = Phone.objects.all()
    if sort == 'min_price':
        phones = Phone.objects.order_by('price')
    elif sort == 'name':
        phones = Phone.objects.order_by('name')
    elif sort == 'max_price':
        phones = Phone.objects.order_by('-price')
    else:
        phones = Phone.objects.all()
    print(phones)
    print(phones[1].name)
    context = {'phones': phones}
    return render(request, template, context)

def show_product(request, slug):
    template = 'product.html'
    phone = Phone.objects.filter(slug=slug)
    context = {'phone': phone[0]}
    return render(request, template, context)


from django.http.response import HttpResponse
from django.shortcuts import render
from intervenciones.models import Intervencion, Rig

# Create your views here.

def bienvenido(request):
#    mensajes = {'msg1' : 'Valor mensaje 1', 'msg2':'Valor mensaje 2'}
    no_intervenciones = Intervencion.objects.count()
    #personas = Persona.objects.all()
    intervenciones = Intervencion.objects.order_by('id')
    return render(request, 'bienvenido.html', {'no_intervenciones': no_intervenciones, 'intervenciones':intervenciones})
    #return render(request, 'bienvenido.html', mensajes)

def rigs(request):
#    mensajes = {'msg1' : 'Valor mensaje 1', 'msg2':'Valor mensaje 2'}
    no_rigs = Rig.objects.count()
    #domicilios = Domicilio.objects.all()
    rigs = Rig.objects.order_by('id')
    return render(request, 'rigs.html', {'no_rigs': no_rigs, 'rigs':rigs})

# def despedirse(request):
#     return HttpResponse('Despedida desde Django')


import socket

HOST = '127.0.0.1'
PORT = 8006
user_input = []
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((HOST, PORT))
s.listen()
conn, addr = s.accept()
print(f'Waiting for connection on {PORT} port')
print(f'Connected by {addr}')
while True:
    data = conn.recv(1024)
    if data != b'--END--\r\n':
        user_input.append(data)
    else:
        line_counter = len(user_input)
        final_string = b''
        for i in range(len(user_input)):
            final_string += user_input[i]
        final_string.decode()
        length_counter = len(final_string)
        conn.sendall(f"Hello, Client!\n".encode())
        conn.sendall(f"Total lines: {line_counter}\n".encode())
        conn.sendall((f"Total length: {length_counter}\n".encode()))
        conn.close()
        s.close()
        break