# This script checks that distribution strategies work

# ##############################################################################################
# Importations
from datetime import datetime

from lib.DefaultNatures.DefaultNatures import *

from src.common.Agent import Agent
from src.common.Aggregator import Aggregator
from src.common.Strategy import *
from src.common.World import World

from src.tools.GraphAndTex import GraphOptions
from src.tools.SubclassesDictionary import get_subclasses

# ##############################################################################################
# Minimum
# the following objects are necessary for the simulation to be performed
# you need exactly one object of each type
# ##############################################################################################

# ##############################################################################################
# Importation of subclasses
# all the subclasses are imported in the following dictionary
subclasses_dictionary = get_subclasses()

# ##############################################################################################
# Creation of the world
# a world contains all the other elements of the model
# a world needs just a name
name_world = "validation"
world = World(name_world)  # creation

# ##############################################################################################
# Definition of the path to the files
world.set_directory("cases/ValidationCases/Results/DistributionStrategy")  # here, you have to put the path to your results directory

# ##############################################################################################
# Definition of the random seed
# The default seed is the current time (the value returned by datetime.now())
world.set_random_seed("seed")

# ##############################################################################################
# Time parameters
# it needs a start date, the value of an iteration in hours and the total number of iterations
start_date = datetime(year=1, month=1, day=1, hour=0, minute=0, second=0, microsecond=0)  # a start date in the datetime format
world.set_time(start_date,  # time management: start date
               1,  # value of a time step (in hours)
               24)  # number of time steps simulated

# ##############################################################################################
# Model
# ##############################################################################################

# ##############################################################################################
# Creation of nature
# low voltage electricity
LVE = load_low_voltage_electricity()

# ##############################################################################################
# Creation of daemons
price_manager_elec = subclasses_dictionary["Daemon"]["PriceManagerDaemon"]("prices", {"nature": LVE.name, "buying_price": 0, "selling_price": 0})  # sets prices for flat rate
price_manager_elec_expensive = subclasses_dictionary["Daemon"]["PriceManagerDaemon"]("expensive_prices", {"nature": LVE.name, "buying_price": 1, "selling_price": 0})  # sets prices for flat rate

subclasses_dictionary["Daemon"]["LimitPricesDaemon"]({"nature": LVE.name, "limit_buying_price": 1, "limit_selling_price": 1})  # sets prices for the system operator

# Water temperature
# this daemon is responsible for the value of the water temperature in the catalog
water_temperature_daemon = subclasses_dictionary["Daemon"]["ColdWaterTemperatureDaemon"]({"location": "Pau"}, "cases/ValidationCases/AdditionalData/Meteo/ColdWaterTemperatureProfiles.json")

# ##############################################################################################
# Creation of strategies
# the different distribution strategies
strategy_emergency = subclasses_dictionary["Strategy"]["AutarkyFullButFew"](get_emergency)
strategy_price = subclasses_dictionary["Strategy"]["AutarkyFullButFew"](get_price)
strategy_quantity = subclasses_dictionary["Strategy"]["AutarkyFullButFew"](get_quantity)
strategy_partial = subclasses_dictionary["Strategy"]["AutarkyPartialButAll"]()

# strategy grid, which always proposes an infinite quantity to sell and to buy
grid_strategy = subclasses_dictionary["Strategy"]["Grid"]()

# ##############################################################################################
# Manual creation of agents
early_emergency_owner = Agent("early_emergency_owner")
expensive_emergency_owner = Agent("expensive_emergency_owner")
consuming_emergency_owner = Agent("consuming_emergency_owner")

early_price_owner = Agent("early_price_owner")
expensive_price_owner = Agent("expensive_price_owner")
consuming_price_owner = Agent("consuming_price_owner")

early_quantity_owner = Agent("early_quantity_owner")
expensive_quantity_owner = Agent("expensive_quantity_owner")
consuming_quantity_owner = Agent("consuming_quantity_owner")

early_partial_owner = Agent("early_partial_owner")
expensive_partial_owner = Agent("expensive_partial_owner")
consuming_partial_owner = Agent("consuming_partial_owner")

aggregators_manager = Agent("aggregators_manager")

# ##############################################################################################
# Manual creation of contracts
elec_contract = subclasses_dictionary["Contract"]["CooperativeContract"]("elec_contract", LVE, price_manager_elec)

elec_expensive_contract = subclasses_dictionary["Contract"]["CooperativeContract"]("elec_expensive_contract", LVE, price_manager_elec_expensive)

producer_elec_contract = subclasses_dictionary["Contract"]["CurtailmentContract"]("producer_elec_contract", LVE, price_manager_elec)

# ##############################################################################################
# Creation of aggregators
aggregator_grid = Aggregator("national_grid", LVE, grid_strategy, aggregators_manager)
aggregator_elec_emergency = Aggregator("local_grid_emergency", LVE, strategy_emergency, aggregators_manager, aggregator_grid, elec_contract)
aggregator_elec_price = Aggregator("local_grid_price", LVE, strategy_price, aggregators_manager, aggregator_grid, elec_contract)
aggregator_elec_quantity = Aggregator("local_grid_quantity", LVE, strategy_quantity, aggregators_manager, aggregator_grid, elec_contract)
aggregator_elec_partial = Aggregator("local_grid_partial", LVE, strategy_partial, aggregators_manager, aggregator_grid, elec_contract)

# ##############################################################################################
# Manual creation of devices

# Each device is created 4 times
# emergency strategy
device_early_emergency = subclasses_dictionary["Device"]["HotWaterTank"]("device_early_emergency", elec_contract, early_emergency_owner, aggregator_elec_emergency, {"user": "dummy_user_early", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_expensive_emergency = subclasses_dictionary["Device"]["HotWaterTank"]("device_expensive_emergency", elec_expensive_contract, expensive_emergency_owner, aggregator_elec_emergency, {"user": "dummy_user", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_consuming_emergency = subclasses_dictionary["Device"]["HotWaterTank"]("device_consuming_emergency", elec_contract, consuming_emergency_owner, aggregator_elec_emergency, {"user": "dummy_user", "device": "dummy_usage_consuming"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")

production_emergency = subclasses_dictionary["Device"]["Background"]("production_emergency", producer_elec_contract, aggregators_manager, aggregator_elec_emergency, {"user": "dummy_user", "device": "dummy_producer_distribution_test"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")

# price strategy
device_early_price = subclasses_dictionary["Device"]["HotWaterTank"]("device_early_price", elec_contract, early_price_owner, aggregator_elec_price, {"user": "dummy_user_early", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_expensive_price = subclasses_dictionary["Device"]["HotWaterTank"]("device_expensive_price", elec_expensive_contract, expensive_price_owner, aggregator_elec_price, {"user": "dummy_user", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_consuming_price = subclasses_dictionary["Device"]["HotWaterTank"]("device_consuming_price", elec_contract, consuming_price_owner, aggregator_elec_price, {"user": "dummy_user", "device": "dummy_usage_consuming"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")

production_price = subclasses_dictionary["Device"]["Background"]("production_price", producer_elec_contract, aggregators_manager, aggregator_elec_price, {"user": "dummy_user", "device": "dummy_producer_distribution_test"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")

# quantity strategy
device_early_quantity = subclasses_dictionary["Device"]["HotWaterTank"]("device_early_quantity", elec_contract, early_quantity_owner, aggregator_elec_quantity, {"user": "dummy_user_early", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_expensive_quantity = subclasses_dictionary["Device"]["HotWaterTank"]("device_expensive_quantity", elec_expensive_contract, expensive_quantity_owner, aggregator_elec_quantity, {"user": "dummy_user", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_consuming_quantity = subclasses_dictionary["Device"]["HotWaterTank"]("device_consuming_quantity", elec_contract, consuming_quantity_owner, aggregator_elec_quantity, {"user": "dummy_user", "device": "dummy_usage_consuming"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")

production_quantity = subclasses_dictionary["Device"]["Background"]("production_quantity", producer_elec_contract, aggregators_manager, aggregator_elec_quantity, {"user": "dummy_user", "device": "dummy_producer_distribution_test"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")

# partial strategy
device_early_partial = subclasses_dictionary["Device"]["HotWaterTank"]("device_early_partial", elec_contract, early_partial_owner, aggregator_elec_partial, {"user": "dummy_user_early", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_expensive_partial = subclasses_dictionary["Device"]["HotWaterTank"]("device_expensive_partial", elec_expensive_contract, expensive_partial_owner, aggregator_elec_partial, {"user": "dummy_user", "device": "dummy_usage"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")
device_consuming_partial = subclasses_dictionary["Device"]["HotWaterTank"]("device_consuming_partial", elec_contract, consuming_partial_owner, aggregator_elec_partial, {"user": "dummy_user", "device": "dummy_usage_consuming"}, {"cold_water_temperature_daemon": water_temperature_daemon.name}, "cases/ValidationCases/AdditionalData/DevicesProfiles/HotWaterTank.json")

production_partial = subclasses_dictionary["Device"]["Background"]("production_partial", producer_elec_contract, aggregators_manager, aggregator_elec_partial, {"user": "dummy_user", "device": "dummy_producer_distribution_test"}, filename="cases/ValidationCases/AdditionalData/DevicesProfiles/Background.json")

# ##############################################################################################
# Creation of the validation daemon
description = "This script checks that distribution strategies work"

filename = "distribution_strategy_validation"

reference_values = {"early_emergency_owner.LVE.energy_bought": [418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "expensive_emergency_owner.LVE.energy_bought": [0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "consuming_emergency_owner.LVE.energy_bought": [0, 418/3.6/10000*2, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000*2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],

                    "early_price_owner.LVE.energy_bought": [0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "expensive_price_owner.LVE.energy_bought": [418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "consuming_price_owner.LVE.energy_bought": [0, 418/3.6/10000*2, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000*2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],

                    "early_quantity_owner.LVE.energy_bought": [0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "expensive_quantity_owner.LVE.energy_bought": [0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "consuming_quantity_owner.LVE.energy_bought": [418/3.6/10000, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 418/3.6/10000, 418/3.6/10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],

                    "early_partial_owner.LVE.energy_bought": [418/3.6/10000/4, 418/3.6/10000*3/4, 0, 0, 0, 0, 0, 0, 418/3.6/10000/4, 418/3.6/10000*3/4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "expensive_partial_owner.LVE.energy_bought": [418/3.6/10000/4, 418/3.6/10000*3/4, 0, 0, 0, 0, 0, 0, 418/3.6/10000/4, 418/3.6/10000*3/4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                    "consuming_partial_owner.LVE.energy_bought": [418/3.6/10000/2, 418/3.6/10000*3/2, 0, 0, 0, 0, 0, 0, 418/3.6/10000/2, 418/3.6/10000*3/2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                    }

name = "LVE_emergency_energy_bought"
export_plot_emergency = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{C}_{ref.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_emergency_owner.LVE.energy_bought_reference", "style": "points", "legend": r"early"},
                      {"catalog_name_entry": "expensive_emergency_owner.LVE.energy_bought_reference", "style": "points", "legend": r"expens."},
                      {"catalog_name_entry": "consuming_emergency_owner.LVE.energy_bought_reference", "style": "points", "legend": r"consum."}
                    ]
          },
    "Y2": {"label": r"$\mathcal{C}_{num.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_emergency_owner.LVE.energy_bought_simulation", "style": "lines",  "legend": r""},
                      {"catalog_name_entry": "expensive_emergency_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""},
                      {"catalog_name_entry": "consuming_emergency_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""}
                      ]
          }
}

name = "LVE_price_energy_bought"
export_plot_price = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathsf{P}_{ref.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_price_owner.LVE.energy_bought_reference", "style": "points", "legend": r"early"},
                      {"catalog_name_entry": "expensive_price_owner.LVE.energy_bought_reference", "style": "points", "legend": r"expens."},
                      {"catalog_name_entry": "consuming_price_owner.LVE.energy_bought_reference", "style": "points", "legend": r"consum."}
                    ]
          },
    "Y2": {"label": r"$\mathsf{P}_{num.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_price_owner.LVE.energy_bought_simulation", "style": "lines",  "legend": r""},
                      {"catalog_name_entry": "expensive_price_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""},
                      {"catalog_name_entry": "consuming_price_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""}
                      ]
          }
}

name = "LVE_quantity_energy_bought"
export_plot_quantity = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{C}_{ref.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_quantity_owner.LVE.energy_bought_reference", "style": "points",  "legend": r"early"},
                      {"catalog_name_entry": "expensive_quantity_owner.LVE.energy_bought_reference", "style": "points", "legend": r"expens."},
                      {"catalog_name_entry": "consuming_quantity_owner.LVE.energy_bought_reference", "style": "points", "legend": r"consum."}
                    ]
          },
    "Y2": {"label": r"$\mathcal{C}_{num.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_quantity_owner.LVE.energy_bought_simulation", "style": "lines",  "legend": r""},
                      {"catalog_name_entry": "expensive_quantity_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""},
                      {"catalog_name_entry": "consuming_quantity_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""}
                    ]
           }
}

name = "LVE_partial_energy_bought"
export_plot_partial = {
    "name": name,
    "filename": "export_"+name,
    "options": GraphOptions(f"{name}_graph_options", ["csv", "LaTeX"], "multiple_series"),
    "X": {"catalog_name_entry": "iteration", "label": r"$t \, [\si{\hour}]$"},
    "Y": {"label": r"$\mathcal{C}_{ref.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_partial_owner.LVE.energy_bought_reference", "style": "points",  "legend": r"early"},
                      {"catalog_name_entry": "expensive_partial_owner.LVE.energy_bought_reference", "style": "points", "legend": r"expens."},
                      {"catalog_name_entry": "consuming_partial_owner.LVE.energy_bought_reference", "style": "points", "legend": r"consum."}
                    ]
          },
    "Y2": {"label": r"$\mathcal{C}_{num.} \, [$\euro{}$]$",
          "graphs": [ {"catalog_name_entry": "early_partial_owner.LVE.energy_bought_simulation", "style": "lines",  "legend": r""},
                      {"catalog_name_entry": "expensive_partial_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""},
                      {"catalog_name_entry": "consuming_partial_owner.LVE.energy_bought_simulation", "style": "lines", "legend": r""}
                    ]
           }
}

parameters = {"description": description, "filename": filename, "reference_values": reference_values, "tolerance": 1E-6, "export_plots": [export_plot_emergency, export_plot_price, export_plot_quantity, export_plot_partial]}

validation_daemon = subclasses_dictionary["Daemon"]["ValidationDaemon"]("distribution_strategy_test", parameters)

# ##############################################################################################
# Simulation start
world.start()


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    # recursive solution
    def maxDepth(self, root: TreeNode) -> int:
        # base case
        if not root: return 0
        # this root is valid so its +1 depth with whatever the max of both subtrees is
        return 1 + max(self.maxDepth(root.left), self.maxDepth(root.right))

    # iterative solution
    def maxDepth(self, root: TreeNode) -> int:
        if not root: return 0
        # stack for DFS (add tuple of node val, and depth)
        stack = [(root, 1)]
        # max depth
        res = 1

        # go until stack empty
        while stack:
            # pop top
            cur, depth = stack.pop()
            # set new max depth
            res = max(res, depth)
            # add left and right if not null
            if cur.left: stack.append((cur.left, depth + 1))
            if cur.right: stack.append((cur.right, depth + 1))

        return res


import re
import pandas as pd
import nltk
import string
from plotly import graph_objects as go
import plotly.express as px
import numpy as np

nltk.download("stopwords")
stopword_list = nltk.corpus.stopwords.words("english")

def read_file(filename):
    """
    Converts given text file into dataframe with cols of label and
    (positive, negative, neutral)
    """
    tweet_df = pd.read_csv(filename, sep="\t", lineterminator="\n")
    tweet_df.columns = ["label", "tweet"]
    return tweet_df

def clean_tweet(tweet):
    """
    tweet: string
    returns: cleaned tweet (string)
    """
    tweet = tweet.lower()
    # remove punctuation
    tweet = "".join([c for c in tweet if c not in string.punctuation])
    # remove stop words
    tweet = "".join([(c + " ") for c in tweet.split(" ") if c not in stopword_list])
    # remove all @mentions
    tweet = re.sub(r"(@.*?)[\s]", " ", tweet)
    # remove twitter URL at end of tweet (https://t.co/...)
    tweet = re.sub("http[s]?://\S+", "", tweet)
    return tweet

def visualize_tweets(tweet_df):
    """
    tweet_df: dataframe
    """
    index = ["negative", "neutral", "positive"]

    tweet_u, tweet_counts = np.unique(tweet_df["label"], return_counts=True)
    text = [round(x / len(tweet_df), 3) for x in tweet_counts]
    fig = go.Figure(
        data=[
            go.Bar(
                name="Tweets",
                x=index,
                y=tweet_counts / len(tweet_df),
                text=text,
            ),
        ],
        layout=go.Layout(
            title="Proportion of Tweets by Sentiment",
            yaxis_title="Proportion of Total Tweets",
        ),
    )
    fig.show()

def preprocess_tweets(filename):
    tweet_df = read_file(filename)
    tweet_df["tweet"] = tweet_df["tweet"].apply(clean_tweet)
    return tweet_df


from imdb import Cinemagoer
from csv import writer
import pandas as pd
import os

ia = Cinemagoer()

def addToList(name: str) -> None:
    movies = ia.search_movie(name)
    correct = None

    for movie in movies:
        os.system("clear")
        print("Assert correct found [y/n]")
        response = input(f"{movie['long imdb title']}\n")

        if response == "y":
            correct = movie
            break

    if correct is None:
        print("Series was not found")

    df = pd.read_csv("serier.csv", dtype={"title": "str", "ID": "str"})
    if correct.getID() in df.values:
        return

    with open("serier.csv", "a") as f_object:
        writer_object = writer(f_object)

        title = correct["title"]
        ID = correct.getID()
        writer_object.writerow([title, ID])

        f_object.close()


#!/usr/bin/env python
import logging
import time
from proxy5 import (MitmProxy, utils)

if __name__ == "__main__":
    import optparse
    try:
        import __pypy__
        __pypy__
    except ImportError:
        __pypy__ = False

    parser = optparse.OptionParser()
    parser.add_option(
        '-p', '--performance', help="Disable Logger for performance",
        default=__pypy__, action="store_true")
    parser.add_option(
        '-t', '--timeout', help="disable after n seconds",
        default=float('inf'), type='int')
    parser.add_option(
        '-a', '--all',
        help='show all logger messages (enables logging on PyPy at a cost!)',
        action="store_true", default=False)
    parser.add_option(
        '--port', default=0,
        help='set a port or let it be dynamic', type='int')
    options, args = parser.parse_args()

    if options.performance and not options.all:
        logging  # shut up the linter
        logging = \
            utils.FakeLogging()

    logger = logging.getLogger('trollius')
    logger.addHandler(logging.StreamHandler())
    logger.handlers[0].setLevel(logging.DEBUG)
    logger.setLevel(logging.DEBUG)

    logger = logging.getLogger('proxy5')
    handler = logging.StreamHandler()
    handler.setFormatter(
        logging.Formatter(
            '[%(asctime)s] [%(levelname)s] [%(name)s]  %(message)s'))
    logger.addHandler(handler)
    handler.setLevel(logging.INFO)
    if options.all:
        handler.setLevel(logging.DEBUG)
    logger.setLevel(logging.DEBUG)

    proxy = MitmProxy(logger, port=options.port)
    logger.info("Port {0!r}".format(proxy.port))
    proxy.start()
    try:
        ts = time.time()
        while time.time() - ts < options.timeout:
            time.sleep(0.1)
        proxy.stop()
    except KeyboardInterrupt:
        pass
    proxy.stop()
    ts = time.time()
    while proxy.is_alive():
        time.sleep(0.1)
    print("Took {0:.2f} s".format(time.time() - ts))


from datetime import datetime
from typing import List, Optional
import sqlalchemy as sa
from sqlalchemy.engine import Row
from db.model_client import Client
from .baseRepo import BaseRepo, LOGGER

class ClientRepo(BaseRepo):
    """ `Row` filds: `id`, `work_id`, `status_id`, `api_id`, `api_hash`, `phone`, `created_at`"""

    async def get_all(self, limit: int = 100) -> List[Row]:
        try:
            query = sa.select(Client).limit(limit)
            return await self.database.fetch_all(query)
        except Exception as err:
            await self.database.disconnect()
            await self.database.connect()
            LOGGER.critical(err)

    async def get_by_id(self, id: str) -> Optional[Row]:
        query = sa.select(Client).where(Client.id == id)
        return await self.database.fetch_one(query)

    async def get_by_phone(self, phone: str) -> Optional[Row]:
        query = sa.select(Client).where(Client.phone == phone)
        return await self.database.fetch_one(query)

    async def get_by_work_id(self, work_id: int,
                             limit: int = 100,
                             skip: int = 0
                             ) -> List[Row]:
        query = sa.select(Client).limit(limit).offset(skip).where(Client.work_id == work_id)
        return await self.database.fetch_all(query)

    async def get_by_status_id(self, status_id: int,
                               limit: int = 100,
                               skip: int = 0
                               ) -> List[Row]:
        query = sa.select(Client).limit(limit).offset(skip).where(Client.status_id == status_id)
        return await self.database.fetch_all(query)

    async def get_by_user_id(self, user_id):
        query = sa.select(Client).where(Client.user_id == str(user_id))
        return await self.database.fetch_all(query)

    async def get_by_user_id_and_chat_id(self, user_id, chat_id):
        query = sa.select(Client).where(Client.proxy_status_id==1, Client.status_id==1, Client.user_id == str(user_id), Client.channel_id==int(chat_id))
        return await self.database.fetch_all(query)

    async def get_reserve(self, status_id: int = 5,
                               limit: int = 10, #TODO Лимит аккаунтов на 1го пользователя
                               skip: int = 0
                               ) -> List[Row]:
        query = sa.select(Client).limit(limit).offset(skip).where(Client.status_id == status_id)
        return await self.database.fetch_all(query)
    async def get_next_reserve(self,status_id: int = 5):
        query = sa.select(Client).where(Client.status_id == status_id)
        return await self.database.fetch_one(query)

    async def get_by_status_and_work_id(self, status_id: int, work_id: int,
                               limit: int = 100,
                               skip: int = 0
                               ) -> List[Row]:
        query = sa.select(Client).limit(limit).offset(skip).where(Client.status_id == status_id, Client.work_id == work_id)
        return await self.database.fetch_all(query)

    async def get_status_id(self, id:str):
        query = sa.select(Client).where(Client.id == id)
        res = await self.database.fetch_one(query)
        return res[2]

    async def set_user_and_channel_id(self, phone, user_id, channel_id):
        print(phone, type(phone))
        print(user_id, type(user_id))
        print(channel_id, type(channel_id))
        query = sa.update(Client).where(Client.phone==phone).values(user_id=str(user_id), channel_id=str(channel_id), status_id=1)
        return await self.database.execute(query)

    async def get_all_channel(self, user_id, channel_id):
        query = sa.select(Client).where(Client.user_id==user_id, Client.channel_id==channel_id)
        res = await self.database.fetch_all(query)
        return [i[11] for i in res]

    async def get_id_by_api_id(self, api_id):
        query = sa.select(Client).where(Client.api_id == api_id)
        res = await self.database.fetch_one(query)
        return res[0]

# Нужно будет изменить, если добавим к каждому пользователю ограниченное ко-во аккаунтов
    async def check_valid_all_acc(self):
        list_status = [status[2] for status in await self.get_all()]
        if not 1 in list_status:
            raise Exception('Все аккаунты неактивны!')

    async def check_valid_acc_client(self, mes, user_id):
            await mes.answer('Инвайтинг был завершен с ошибкой!')
            raise Exception('Все аккаунты клиента неактивны, а других нет!')

    async def create(self, work_id: int,
                     status_id: int,
                     proxy_status_id: int,
                     api_id: int,
                     api_hash: str,
                     phone: str,
                     ) -> int:
        """`return`  id: `int`"""
        client_acc = {
            'work_id': work_id,
            'status_id': status_id,
            'proxy_status_id': proxy_status_id,
            'api_id': api_id,
            'api_hash': api_hash,
            'phone': phone,
            'count_invite': 0,
            'created_at': datetime.now(),
        }

        query = sa.insert(Client).values(**client_acc)
        return await self.database.execute(query)

    async def update(self, id: str, **kwargs) -> int:
        """`return`  id: `int`"""
        query = sa.update(Client).where(Client.id == id).values(**kwargs)
        return await self.database.execute(query)

    async def delete(self, id: str):
        query = sa.delete(Client).where(Client.id == id)
        return await self.database.execute(query)

    async def checkBan(self, id:str):
        query = sa.select(Client).where(Client.id == id)
        res = await self.database.fetch_one(query)
        if res[2] == 3:
            return True
        else:
            return False

    async def banned(self, id: str):
        query = sa.update(Client).where(Client.id == id).values(status_id=3)
        return await self.database.execute(query)

    async def bannedByPhone(self, phone: str):
        query = sa.update(Client).where(Client.phone == phone).values(status_id=3)
        return await self.database.execute(query)

    async def bannedByApiId(self, api_id):
        query = sa.update(Client).where(Client.api_id == api_id).values(status_id=3)
        return await self.database.execute(query)

    async def get_count_invite(self, id: str):
        query = sa.select(Client).where(Client.id == id)
        res = await self.database.fetch_one(query)
        if res[8] == None:
            await self.set_count_invite(id, 0)
            return 0
        else:
            return res[8]

    async def set_count_invite(self, id: str, val: int):
        query = sa.update(Client).where(Client.id == id).values(count_invite=val)
        return await self.database.execute(query)

    async def plus_count_invite(self, id:str, val:int):
        count = await self.get_count_invite(id)
        query = sa.update(Client).where(Client.id == id).values(count_invite=count+1)
        return await self.database.execute(query)

    async def set_data_paused_invite(self, id:str):
        query = sa.update(Client).where(Client.id == id).values(data_paused=datetime.now())
        return await self.database.execute(query)

    async def get_data_paused_invite(self, id:str):
        query = sa.select(Client).where(Client.id == id)
        res = await self.database.fetch_one(query)
        return res[9]

    async def set_none_date_paused_invite(self, id: str):
        query = sa.update(Client).where(Client.id == id).values(data_paused=None)
        return await self.database.execute(query)

    async def set_unPaused_invite(self, id: str):
        query = sa.update(Client).where(Client.id == id).values(status_id=1)
        return await self.database.execute(query)

    async def check_end_pause(self, id:str):
        try:
            if (datetime.now() - datetime.strptime(str(await self.get_data_paused_invite(id)),
                                                   "%Y-%m-%d %H:%M:%S")).total_seconds() // 60 > 1439:
                await self.set_none_date_paused_invite(id)
                await self.set_count_invite(id, 0)
                await self.set_unPaused_invite(id)
                return True
            else:
                return False
        except:
            return False

    async def get_channel_id(self, id:int):
        query = sa.select(Client).where(Client.id == id)
        res = await self.database.fetch_one(query)
        if res[10]:
            return res[10]
        else:
            return None

# class Client(Base):
#     __tablename__ = 'client_accounts'
#     __table_args__ = {"comment": "Аккаунты парсеры"}

#     id = Column(Integer, primary_key=True, autoincrement=True)
#     work_id = Column(Integer, ForeignKey('client_workes.id'),nullable=False, default=CWorkes.UNWORKING.value['id'])
#     status_id = Column(Integer, ForeignKey('client_statuses.id'),nullable=False, default=CStatuses.WAITING_AUTHORIZATION.value['id'])
#     api_id = Column(Integer, nullable=False)
#     api_hash = Column(String(50), nullable=False)
#     phone = Column(String(16), nullable=False)
#     created_at = Column(DateTime, nullable=False, default=datetime.datetime.now)


from abc import ABC
from abc import abstractmethod
import time

import torch

class TimerBase(ABC):

    def __init__(self, name):
        self.name = name

    @abstractmethod
    def start(self, barrier=False):
        pass

    @abstractmethod
    def stop(self, barrier=False):
        pass

    @abstractmethod
    def reset(self):
        pass

    @abstractmethod
    def elapsed(self, reset=True, barrier=False):
        pass

class DummyTimer(TimerBase):

    def __init__(self):
        super().__init__('dummy timer')

    def start(self, barrier=False):
        return

    def stop(self, barrier=False):
        return

    def reset(self):
        return

    def elapsed(self, reset=True, barrier=False):
        raise Exception('dummy timer should not be used to '
                        'calculate elapsed time')

class Timer(TimerBase):
    """
    Comment on using `barrier`: If this flag is passed, then all
    the caller processes will wait till all reach the timing routine.
    It is up to the user to make sure all the ranks in `barrier_group`
    call it otherwise, it will result in a hang.
    Comment on `barrier_group`: By default it is set to None which
    in torch distributed land, it will result in the global communicator.
    """

    def __init__(self, name):
        super().__init__(name)
        self._elapsed = 0.0
        self._started = False
        # Note that None will default to the global process group
        self._barrier_group = None
        self._start_time = time.time()

    def set_barrier_group(self, barrier_group):
        self._barrier_group = barrier_group

    def start(self, barrier=False):
        """Start the timer."""
        assert not self._started, 'timer has already been started'
        if barrier:
            torch.distributed.barrier(group=self._barrier_group)
        torch.cuda.synchronize()
        self._start_time = time.time()
        self._started = True

    def stop(self, barrier=False):
        """Stop the timer."""
        assert self._started, 'timer is not started'
        if barrier:
            torch.distributed.barrier(group=self._barrier_group)
        torch.cuda.synchronize()
        self._elapsed += (time.time() - self._start_time)
        self._started = False

    def reset(self):
        """Reset timer."""
        self._elapsed = 0.0
        self._started = False

    def elapsed(self, reset=True, barrier=False):
        """Calculate the elapsed time."""
        _started = self._started
        # If the timing in progress, end it first.
        if self._started:
            self.stop(barrier=barrier)
        # Get the elapsed time.
        _elapsed = self._elapsed
        # Reset the elapsed time
        if reset:
            self.reset()
        # If timing was in progress, set it back.
        if _started:
            self.start(barrier=barrier)
        return _elapsed

class Timers:
    """Group of timers."""

    def __init__(self, log_level, log_option):
        self._log_level = log_level
        self._log_option = log_option
        self._timers = {}
        self._log_levels = {}
        self._dummy_timer = DummyTimer()
        self._max_log_level = 2

    def __call__(self, name, log_level=None):
        # If the timer has already been set, then check if the log-level
        # is provided, it matches the one that the timer was created with.
        if name in self._timers:
            if log_level is not None:
                assert log_level == self._log_levels[name], \
                    'input log level {} does not match already existing '\
                    'log level {} for {} timer'.format(
                        log_level, self._log_levels[name], name)
            return self._timers[name]
        # If timer does not exist and no log level is provided,
        # set it to the max log level which is 2.
        if log_level is None:
            log_level = self._max_log_level
        assert log_level <= self._max_log_level, \
            'log level {} is larger than max supported log level {}'.format(
                log_level, self._max_log_level)
        # Now if the input log level is larger than the one set for
        # the timers class, just ignore it and return a dummy timer.
        if log_level > self._log_level:
            return self._dummy_timer
        # Otherwise, initalize the timer and set the level.
        self._timers[name] = Timer(name)
        self._log_levels[name] = log_level
        return self._timers[name]

    def _get_elapsed_time_all_ranks(self, names, reset, barrier):
        """
        Assumptions:
            - All the ranks call this function.
            - `names` are identical on all ranks.
        If the above assumptions are not met, calling this function will
        result in hang.
        Arguments:
            - names: list of timer names
            - reset: reset the timer after recording the elapsed time
            - barrier: if set, do a global barrier before time measurments
        """

        # First make sure all the callers are in sync.
        if barrier:
            torch.distributed.barrier()

        world_size = torch.distributed.get_world_size()
        rank = torch.distributed.get_rank()

        # Here we can use gather on the rank we want to print the
        # timing, however, there is no gather_base support in
        # pytorch yet. It is simpler to deal with a single tensor
        # and since we are only gathering a small amount of data,
        # it should be ok to use all-gather instead of gather.
        rank_name_to_time = torch.zeros((world_size, len(names)),
                                        dtype=torch.float,
                                        device=torch.cuda.current_device())
        for i, name in enumerate(names):
            if name in self._timers:
                # Here we don't need to pass the barrier flag as all
                # the processes are already in sync. This avoids the
                # issue of different timers having different barrier
                # groups inside their class.
                rank_name_to_time[rank, i] = self._timers[name].elapsed(
                    reset=reset)

        # See the note above for why we are not using gather.
        torch.distributed._all_gather_base(rank_name_to_time.view(-1),
                                           rank_name_to_time[rank, :].view(-1))

        return rank_name_to_time

    def _get_global_min_max_time(self, names, reset, barrier, normalizer):
        """Report only min and max times across all ranks."""

        rank_name_to_time = self._get_elapsed_time_all_ranks(names, reset,
                                                             barrier)
        name_to_min_max_time = {}
        for i, name in enumerate(names):
            rank_to_time = rank_name_to_time[:, i]
            # filter out the ones we did not have any timings for
            rank_to_time = rank_to_time[rank_to_time > 0.0]
            # If the timer exists:
            if rank_to_time.numel() > 0:
                name_to_min_max_time[name] = (
                    rank_to_time.min().item() / normalizer,
                    rank_to_time.max().item() / normalizer)
        return name_to_min_max_time

    def _get_global_min_max_time_string(self, names, reset, barrier,
                                        normalizer, max_only):
        name_to_min_max_time = self._get_global_min_max_time(
            names, reset, barrier, normalizer)
        if not name_to_min_max_time:
            return None
        output_string = '(min, max) time across ranks (ms):'
        for name in name_to_min_max_time:
            min_time, max_time = name_to_min_max_time[name]
            if max_only:
                output_string += '\n    {}: {:.2f}'.format(
                    (name+' ').ljust(48, '.'), max_time)
            else:
                output_string += '\n    {}: ({:.2f}, {:.2f})'.format(
                    (name+' ').ljust(48, '.'), min_time, max_time)
        return output_string

    def _get_all_ranks_time_string(self, names, reset, barrier, normalizer):
        """Report times across all ranks."""
        rank_name_to_time = self._get_elapsed_time_all_ranks(names, reset,
                                                             barrier)

        output_string = 'times across ranks (ms):'
        no_reported_timing = True
        for i, name in enumerate(names):
            not_yet_found = True
            for rank in range(torch.distributed.get_world_size()):
                if rank_name_to_time[rank, i] > 0:
                    no_reported_timing = False
                    if not_yet_found:
                        not_yet_found = False
                        output_string += '\n  {}:'.format(name)
                    output_string += '\n     rank {:2d}: {:.2f}'.format(
                        rank, rank_name_to_time[rank, i] / normalizer)
        if no_reported_timing:
            return None
        return output_string

    def log(self, names, rank=None, normalizer=1.0, reset=True, barrier=False):
        """Log a group of timers."""

        # Print.
        assert normalizer > 0.0
        if self._log_option in ['max', 'minmax']:
            max_only = False
            if self._log_option == 'max':
                max_only = True
            output_string = self._get_global_min_max_time_string(
                names, reset, barrier, normalizer/1000.0, max_only)
        elif self._log_option == 'all':
            output_string = self._get_all_ranks_time_string(names,
                                                            reset, barrier,
                                                            normalizer/1000.0)
        else:
            raise Exception('unknown timing log option {}'.format(
                self._log_option))

        # If no input rank is provided, log on last rank.
        if rank is None:
            rank = torch.distributed.get_world_size() - 1
        if rank == torch.distributed.get_rank() and output_string is not None:
            print(output_string, flush=True)

    def write(self, names, writer, iteration, normalizer=1.0,
              reset=False, barrier=False):
        """Write timers to a tensorboard writer
        Note that we only report maximum time across ranks to tensorboard.
        """
        # currently when using add_scalars,
        # torch.utils.add_scalars makes each timer its own run, which
        # polutes the runs list, so we just add each as a scalar
        assert normalizer > 0.0
        name_to_min_max_time = self._get_global_min_max_time(
            names, reset, barrier, normalizer)
        if writer is not None:
            for name in name_to_min_max_time:
                _, max_time = name_to_min_max_time[name]
                writer.add_scalar(name + '-time', max_time, iteration)


MAX_STATE_NAMES = 2

def open_file(filename):
    ''' Returns a file stream if filename found, otherwise None '''
    try:
        file_stream = open(filename, "r")
        return file_stream
    except FileNotFoundError:
        return None

def find_year_column(year_list, year):
    ''' Finds the column in the year_list corresponding to the given year '''
    for col in range(0, len(year_list)):
        if year == year_list[col]:
            return col
    return None # should not happen

def get_valid_years(file_stream):
    ''' Get the years that are in the header row of the file stream
        Return the years as a list '''
    valid_years = []
    for line in file_stream:
        line_list = line.strip().split()
        # Only interested in the years and need to convert to ints
        valid_years = [int(i) for i in line_list[1:]]
        break
    return valid_years

def get_population_state_tuple(line_list, valid_years, year_col):
    ''' Returns a tuples consisting of population and state by extracting it
        from the given list '''

    if len(line_list) == len(valid_years) + MAX_STATE_NAMES: # then the state comprises more than one names
            state = line_list[0] + " " + line_list[1]
            population = line_list[year_col + 1]
    else:
        state = line_list[0]
        population = line_list[year_col]

    population = int(population)
    return (population,state)

def get_tuple_list(file_stream, valid_years, year_col):
    ''' Reads a file stream and returns a list of tuples for the given year
        The header row in the file stream has already been read '''
    result_list = []
    for line in file_stream:
        line_list = line.strip().split()
        pop_state_tuple = get_population_state_tuple(line_list, valid_years, year_col)
        result_list.append(pop_state_tuple)
    return result_list

def get_year(valid_years):
    ''' Returns a valid year input by the user '''
    error_str = 'Invalid year!'

    while True:
        try:
            year = int(input("Enter year: "))
            if year in valid_years:
                return year
            print(error_str)
        except ValueError:
            print(error_str)

def print_min_max(population_tuples):
    ''' Prints the min and max population from the given tuple.
        Each tuple is of the form (population, state)
        Assumes that the list of tuples is sorted on the first item in each tuple '''

    print("Minimum: {}".format(min(population_tuples)))
    print("Maximum: {}".format(max(population_tuples)))

# Main program starts here
filename = input("Enter filename: ")
file_stream = open_file(filename)
if file_stream:
    valid_years = get_valid_years(file_stream)
    year = get_year(valid_years)
    year_column = find_year_column(valid_years, year)
    # Add 1 to year_column because the state info is in the first column
    population_tuples = get_tuple_list(file_stream, valid_years, year_column + 1)
    print_min_max(population_tuples)
    file_stream.close()
else:
    print("Filename {} not found!".format(filename))


import pandas as pd
import numpy as np
from scipy.stats import norm # Normal Distribution

# Variables
r = 0.01  # Risk-Free Rate
S = 30   # Underline asset price
K = 40   # Strike Price
T = 240/365 # Time to Expiration
sigma = 0.30 # Volatility aka Risk
Type = input("Type 'Call' or 'Put' for European Option Type: ")

# cdf is the cumulative distribution function
# pdf is the probability distribution function

def blackScholes(r, S, K, T, sigma, Type):
    "Calculate Black Scholes option price for a call or put european option"

    d1 = (np.log(S/K) + (r + sigma**2/2)*T)/ (sigma*np.sqrt(T))
    d2 = d1 - sigma*np.sqrt(T)
    try:
        if Type == "Call":
            price = S*norm.cdf(d1, 0, 1) - K*np.exp(-r*T)*norm.cdf(d2, 0, 1)
        elif Type == "Put":
            price = K*np.exp(-r*T)*norm.cdf(-d2, 0, 1) - S*norm.cdf(-d1, 0, 1)
        return price
    except:
        print("Make sure you type in 'Call' or 'Put' (Case Sensitive) ")

def delta(r, S, K, T, sigma, Type): # Rate of change of the option value with respect to the underline asset price.
    "Calculate Delta for a call or put european option"

    d1 = (np.log(S/K) + (r + sigma**2/2)*T)/ (sigma*np.sqrt(T))
    try:
        if Type == "Call":
            delta = norm.cdf(d1, 0, 1)
        elif Type == "Put":
            delta = -norm.cdf(-d1, 0, 1)
        return delta
    except:
        print("Make sure you type in 'Call' or 'Put' (Case Sensitive) ")

def gamma(r, S, K, T, sigma, Type): # Rate of change of Delta value with respect to the underline asset price.
    "Calculate Gamma for a call or put european option"

    d1 = (np.log(S/K) + (r + sigma**2/2)*T)/ (sigma*np.sqrt(T))
    d2 = d1 - sigma*np.sqrt(T)
    try:
        gamma = S*norm.pdf(d1, 0, 1)/(S*sigma*np.sqrt(T))
        return gamma
    except:
        print("Make sure you type in 'Call' or 'Put' (Case Sensitive) ")

def vegas(r, S, K, T, sigma, Type): # Vega measures the sensitivity to volatility. Derivative of the option value to derivative to option volatility
    "Calculate Gamma for a call or put european option"

    d1 = (np.log(S/K) + (r + sigma**2/2)*T)/ (sigma*np.sqrt(T))
    d2 = d1 - sigma*np.sqrt(T)
    try:
        vegas = S*norm.pdf(d1, 0, 1)*np.sqrt(T)
        return vegas*0.01
    except:
        print("Make sure you type in 'Call' or 'Put' (Case Sensitive) ")

def theta(r, S, K, T, sigma, Type): # Sensitivity of the value of the derivative to the passage of time per day
    "Calculate Delta for a call or put european option"
    d1 = (np.log(S/K) + (r + sigma**2/2)*T)/ (sigma*np.sqrt(T))
    d2 = d1 - sigma*np.sqrt(T)
    try:
        if Type == "Call":
            theta = -S*norm.pdf(d1, 0, 1)*sigma/(2*np.sqrt(T)) - r*K*np.exp(-r*T)*norm.cdf(d2, 0, 1)
        elif Type == "Put":
            theta = -S*norm.pdf(d1, 0, 1)*sigma/(2*np.sqrt(T)) + r*K*np.exp(-r*T)*norm.cdf(-d2, 0, 1)
        return theta/365
    except:
        print("Make sure you type in 'Call' or 'Put' (Case Sensitive)")

def rho(r, S, K, T, sigma, Type): # Sensitivity to the interest rate
    "Calculate Delta for a call or put european option"
    d1 = (np.log(S/K) + (r + sigma**2/2)*T)/ (sigma*np.sqrt(T))
    d2 = d1 - sigma*np.sqrt(T)
    try:
        if Type == "Call":
            rho = K*T*np.exp(-r*T)*norm.cdf(d2, 0, 1)
        elif Type == "Put":
            rho = -K*T*np.exp(-r*T)*norm.cdf(-d2, 0, 1)
        return rho*0.01
    except:
        print("Make sure you type in 'Call' or 'Put' (Case Sensitive)")

optionPrice = blackScholes(r, S, K, T, sigma, Type).round(2)
delta_calc = delta(r, S, K, T, sigma, Type).round(3)
gamma_calc = gamma(r, S, K, T, sigma, Type).round(3)
vega_calc = vegas(r, S, K, T, sigma, Type).round(3)
theta_calc = theta(r, S, K, T, sigma, Type).round(3)
rho_calc = rho(r, S, K, T, sigma, Type).round(3)
print(f"Option Price is: {optionPrice} ")
print(f"Delta: {delta_calc} ")
print(f"Gamma: {gamma_calc} ")
print(f"Vega: {vega_calc} ")
print(f"Theta: {theta_calc} ")
print(f"Rho: {rho_calc} ")


import pygame, json, pygame_gui, sys
import lib.button, lib.color, lib.save_manager, lib.options

#from lib.paint import Paint
#from name import *
#from menu import *
# from winlose import WinLose

# # Import DATA
# setting = json.load(open('data/setting.json'))
# game_data = json.load(open('data/game_data.json'))

# # LOAD DATA
# SCREEN_WIDTH  = setting['screen']['width']
# SCREEN_HEIGHT = setting['screen']['height']
# PLAYER_NAME = game_data["PlayerName"]

class SettingWindow(pygame_gui.elements.UIWindow):
    """class SettingWindow là class kế thừa (Inheritance) từ class pygame_gui.elements.UIWindow

    Args:
        pygame_gui (_type_): _description_
    """
    def __init__(self, rect, ui_manager,SCREEN_WIDTH, SCREEN_HEIGHT, ingame_sound, win_sound, click_sound, menu_sound):
        """Hàm khởi tạo (constructor)

        Args:
            rect (_type_): _description_
            ui_manager (_type_): _description_
            SCREEN_WIDTH (int): chiều dài (ngang) của cửa sổ
            SCREEN_HEIGHT (int): chiều cao (dọc) của cửa sổ
        """
        self.ingame_sound = ingame_sound
        self.win_sound = win_sound
        self.click_sound = click_sound
        self.menu_sound = menu_sound

        super().__init__(rect, ui_manager,
                         window_display_title='Setting',
                         object_id='#setting_window',
                         resizable=True)

        # Import DATA
        self.setting = json.load(open('data/setting.json'))
        self.game_data = json.load(open('data/game_data.json'))

        self.is_blocking = True  # blocks all clicking events from interacting beyond this window

        # LOAD DATA
        self.SCREEN_WIDTH  = self.setting['screen']['width']
        self.SCREEN_HEIGHT = self.setting['screen']['height']
        PLAYER_NAME = self.game_data["PlayerName"]

        # list chứa các giá trị hiện trong menu drop down
        self.pieces_mode = ['3', '4', '5', '6']
        self.pieces_mode_pos = -1
        self.board_size = [
            ['3x3'],
            ['10x10', '12x12'],
            ['16x16', '20x20'],
            ['20x20', '24x24']
        ]
        # self.resolution = ['640x480', '800x600', '1024x768', '1280x960']
        self.resolution = ['1280x960']

        self.options = lib.options.Options(SCREEN_WIDTH, SCREEN_HEIGHT)
        self.btn_size = ( int(self.rect.width *0.6), 30 )

        # Số quân liên tiếp cần để thắng
        self.pieces_mode_label = pygame_gui.elements.UILabel(pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 - 250)),
                                                            self.btn_size),
                                                            "Số quân liên tiếp để thắng: ",
                                                            self.ui_manager,
                                                            container=self)
        # Lấy giá trị của số quân liên tiếp để thắng
        self.current_pieces_mode = (str(self.setting["game"]["win_cnt"]))

        # Dòng này hiện ra menu xổ xuống của số quân liên tiếp cần để thắng
        self.pieces_mode_drop_down = pygame_gui.elements.UIDropDownMenu(self.pieces_mode,
                                                  self.current_pieces_mode,
                                                  pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 - 200)),
                                                            self.btn_size),
                                                  self.ui_manager,
                                                  container=self)

        # Kích cỡ bàn cờ
        self.pieces_mode_label = pygame_gui.elements.UILabel(pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 - 150)),
                                                            self.btn_size),
                                                            "Board size: ",
                                                            self.ui_manager,
                                                            container=self)
        # Lấy giá trị của kích cỡ bàn cờ hiện tại
        self.current_board_size = (str(self.setting["grid"]["size_x"])
                                   + 'x'
                                   + str(self.setting["grid"]["size_y"]))

        self.pieces_mode_pos = self.pieces_mode.index(self.pieces_mode_drop_down.selected_option)
        # self.update_pieces_mode_index()
        # Dòng này hiện ra menu xổ xuống của kích cỡ bàn cờ
        self.board_size_drop_down = pygame_gui.elements.UIDropDownMenu(self.board_size[self.pieces_mode_pos],
                                                  self.current_board_size,
                                                  pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 - 100)),
                                                            self.btn_size),
                                                  self.ui_manager,
                                                  container=self)

        # Độ phân giải
        self.resolution_label = pygame_gui.elements.UILabel(pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 - 50)),
                                                            self.btn_size),
                                                            "Resolution: ",
                                                            self.ui_manager,
                                                            container=self)
        # Lấy giá trị của độ phân giải màn hình hiện tại
        self.current_resolution_string = (str(self.options.resolution[0]) +
                                     'x' +
                                     str(self.options.resolution[1]))

        # Dòng này hiện ra menu xổ xuống
        self.resolution_drop_down = pygame_gui.elements.UIDropDownMenu(self.resolution,
                                                  self.current_resolution_string,
                                                  pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 + 0)),
                                                            self.btn_size),
                                                  self.ui_manager,
                                                  container=self)

        # Setting Volume
        # Dòng chữ "Volume"
        self.volume_label = pygame_gui.elements.UILabel(pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 + 50)),
                                                            self.btn_size),
                                                            "Volume: ",
                                                            self.ui_manager,
                                                            container=self)
        # Thanh trượt qua lại
        self.volume_settings = pygame_gui.elements.UIHorizontalSlider(pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2) + 100),
                                                            self.btn_size),
                                                            self.ingame_sound.get_volume() * 100.0,
                                                            (0.0, 100.0),
                                                            self.ui_manager,
                                                            container=self,
                                                            click_increment=5)
        #self.volume_settings.left_button.kill()
        #self.volume_settings.right_button.kill()
            # Âm thanh
        self.current_volume = pygame_gui.elements.UILabel(pygame.Rect((int(self.rect.width / 2 + self.btn_size[0] /2),
                                                int(self.rect.height / 2 + 100)),
                                                (50, 50)),
                                    str(int(self.volume_settings.get_current_value())),
                                    self.ui_manager,
                                    container=self)

    # def update_pieces_mode_index(self) -> None:
    #     self.pieces_mode_pos = self.pieces_mode.index(self.pieces_mode_drop_down.selected_option)

    def update_board_size_drop_down(self) -> None:
        self.pieces_mode_pos = self.pieces_mode.index(self.pieces_mode_drop_down.selected_option)

        # kill thanh cũ để tạo thanh mới
        self.board_size_drop_down.kill()

        self.board_size_drop_down = pygame_gui.elements.UIDropDownMenu(self.board_size[self.pieces_mode_pos],
                                                  self.board_size[self.pieces_mode_pos][0],
                                                  pygame.Rect((int(self.rect.width / 2 - self.btn_size[0] / 2),
                                                            int(self.rect.height / 2 - 100)),
                                                            self.btn_size),
                                                  self.ui_manager,
                                                  container=self)
        # print(self.board_size_drop_down.selected_option)
        self.board_size_drop_down.selected_option = self.board_size[self.pieces_mode_pos][0]

    def on_close_window_button_pressed(self):
        """
        Override this method to call 'hide()' instead if you want to hide a window when the
        close button is pressed.
        """
        # self.kill()
        self.hide()

        # self.health_bar = UIScreenSpaceHealthBar(pygame.Rect((int(self.rect.width / 9),
        #                                                       int(self.rect.height * 0.7)),
        #                                                      (200, 30)),
        #                                          self.ui_manager,
        #                                          container=self)

    def update(self, time_delta):
        super().update(time_delta)
        if self.alive() and self.volume_settings.has_moved_recently:
            cur_volume = int(self.volume_settings.get_current_value())
            self.current_volume.set_text(str(cur_volume))
            self.ingame_sound.set_volume(cur_volume / 100.0)
            self.win_sound.set_volume(cur_volume / 100.0)
            self.click_sound.set_volume(cur_volume / 100.0)
            self.menu_sound.set_volume(cur_volume / 100.0)


import os
import requests
from collections import Counter
from bs4 import BeautifulSoup

url = 'http://pokemondb.net/pokedex/all'
poke_folder = os.makedirs('poke_images', exist_ok = True)
soup = BeautifulSoup(requests.get(url).text)

poke_links = soup.select('.ent-name')
nicknames = {}
top_words = {}

prev_url = None
for link in poke_links[-10:]:
	poke_url = link['href']
	number = link['title'][18:21]
	if prev_url == poke_url: continue
	name = link.text
	poke_soup = BeautifulSoup(requests.get(url[:-12] + poke_url).text)
	nicknames[name] = poke_soup.select('q')[0].text
	descriptions_html = poke_soup.select('table.vitals-table')[-2]
	description_list = []
	for description in descriptions_html:
		description_list.extend(description.text.split())
	dl = []
	for poke, count in dict(Counter(description_list)).items():
		dl.append((count,poke))
	dl.sort(reverse = True)
	top_words[name] = dl[:5]
	img_urls = poke_soup.select('div > ul > li > div > div > img')
	for index, img_url in enumerate(img_urls):
		print('Downloading image of', name)
		res = requests.get(img_url['src'])
		img_file = open(os.path.join('poke_images', number+'_'+name+'_'+str(index)+'.jpg'), 'wb')
		for chunk in res.iter_content(100000):
			img_file.write(chunk)
		img_file.close()
	prev_url = poke_url


# -*- coding: utf-8 -*-

import upemtk as tk
from fonctions import *

def dessinerQuadrillage(largeurFenetre, hauteurFenetre, largeurCase, carte):
    #Dessine le quadrillage
    for i in range(len(carte)): #i les lignes
        for j in range(len(carte[i])): #j les colonnes
            if j != 0 and i != 0:#Condition pour qu'il n'y ai pas de quadrillage sur les côtés
                tk.ligne(j*largeurCase,0,j*largeurCase,hauteurFenetre)
                tk.ligne(0,i*largeurCase,largeurFenetre, i*largeurCase)

def dessinerCroix(x, y, largeur, couleur = 'black', epaisseur = 1):
    #Dessine une croix
    tk.ligne(x, y, x+largeur, y+largeur, couleur, epaisseur)
    tk.ligne(x, y+largeur, x+largeur, y, couleur, epaisseur)

def dessinerBoite(x, y, largeurCase, couleurCroix = "brown"):
    #Dessine un petit carré
    distanceAuBord = largeurCase/4
    tk.rectangle(x+ distanceAuBord, y+ distanceAuBord, x+largeurCase-distanceAuBord, y+largeurCase-distanceAuBord, "brown", "#ffa724",2, tag = "Carre-Boite")
    dessinerCroix(x+distanceAuBord, y+ distanceAuBord, largeurCase-distanceAuBord*2, couleurCroix, 2)

def dessinerMur(x, y,largeurCase):
    #Dessine le mur
    largeurBrique = largeurCase / 2
    hauteurBrique = largeurCase / 4
    X,Y = 0,0
    for i in range(4):
        for j in range(2):
            if i % 2 == 0:
                tk.rectangle(x + X,y + Y,x + largeurBrique + X,y + hauteurBrique + Y,"white","brown")
            else:
                tk.rectangle(x + X + largeurBrique / 2,y + Y,x + largeurBrique * 1.5 + X,y + hauteurBrique + Y,"white","brown")
                tk.rectangle(x + X,y + Y,x + X + largeurBrique / 2,y + hauteurBrique + Y,"white","brown")
                tk.rectangle(x + largeurBrique * 1.5 + X,y + Y,x + 2 * largeurBrique,y + hauteurBrique + Y,"white","brown")
                break
            X += largeurBrique
        X = 0
        Y += hauteurBrique
    tk.rectangle(x, y, x+largeurCase, y+largeurCase, 'black', '')# Le contour de la case

def dessinerJoueur(x, y,largeurCase):
    #Dessine le joueur
    #Dessine un emplacement pour une caisse/ une cible
    rayon = largeurCase / 2.5
    cstCentrage = largeurCase / 2
    tk.cercle(x + cstCentrage,y + cstCentrage,rayon,"blue","blue", 2, tag = "Joueur")

def dessinerClef(x, y, largeurCase):
    #Dessine une clef
    rayon = largeurCase / 4
    espacementX = largeurCase / 3 - rayon
    espacementY = largeurCase / 2 - rayon
    #Cercle:
    positionX_cercle = x + espacementX + rayon
    positionY_cercle = y + espacementY + rayon
    tk.cercle(positionX_cercle,positionY_cercle,rayon, "#339933", "#339933")
    #Rectangle principal
    hauteurRectangle = rayon / 2.8
    tk.rectangle(positionX_cercle,positionY_cercle - hauteurRectangle,x + largeurCase - espacementX,positionY_cercle + hauteurRectangle, "#339933", "#339933")
    #Rectangles secondaires:
    PositionBDX_Rectangle2 = x + largeurCase - espacementX
    PositionBDY_Rectangle2 = y + largeurCase - espacementY - hauteurRectangle
    PositionHGX_Rectangle2 = PositionBDX_Rectangle2 - hauteurRectangle
    PositionHGY_Rectangle2 = y + espacementY + rayon + hauteurRectangle
    tk.rectangle(PositionHGX_Rectangle2, PositionHGY_Rectangle2,PositionBDX_Rectangle2,PositionBDY_Rectangle2, "#339933", "#339933")
    tk.rectangle(PositionHGX_Rectangle2 - 1.5*espacementX, PositionHGY_Rectangle2,PositionBDX_Rectangle2 - 1.5*espacementX,PositionBDY_Rectangle2, "#339933", "#339933")

def dessinerPorte(x, y,largeurCase):
    #Dessine une porte
    centreDuCercleX = x + largeurCase/2
    centreDuCercleY = y + largeurCase/2.5
    rayon = (largeurCase/3)//2
    tk.rectangle(x, y, x+largeurCase, y+largeurCase, 'black', '#339933')# Le reste de la case
    tk.cercle(centreDuCercleX, centreDuCercleY, rayon, 'black','black', 2)#Le rond de la serrure
    largeurRectangle1 = rayon/3
    #la serrure
    tk.rectangle(centreDuCercleX - largeurRectangle1, centreDuCercleY -10 + rayon, centreDuCercleX + largeurRectangle1, centreDuCercleY -10 + rayon+ largeurCase//3, 'black', 'black')

def dessinerInventaire(x, y, largeur, hauteur):
    tk.rectangle(x, y, x+largeur, y+hauteur, 'black', 'black')

def dessinerCible(x, y,largeurCase):
    #Dessine un emplacement pour une caisse/ une cible
    rayon = largeurCase/3
    cstCentrage = largeurCase / 2
    tk.cercle(x + cstCentrage,y + cstCentrage,rayon,"red","#ff5050", 2)
    tk.cercle(x + cstCentrage,y + cstCentrage,rayon/1.2,"red","#ff5050", 2)

def dessinerVictoire(largeurFenetre,hauteurFenetre):
    tk.texte(largeurFenetre // 4, hauteurFenetre // 3, "Victory !", 'black', police = "Times New Roman", taille = 60)

if __name__ == '__main__':
    """ Ce programme sert à tester les différentes fonctions de dessin """
    tk.cree_fenetre(300, 300)
    dessinerQuadrillage(300, 300, 100, [['.', '.','.'],
                                ['.','.','.'],
                                ['.','.','.']])
    dessinerMur(0,0,100)
    dessinerClef(100,0,100)

    dessinerCible(0, 100, 100)
    dessinerBoite(100, 100, 100)

    dessinerJoueur(0, 200, 100)
    dessinerPorte(100, 200, 100)

    tk.attente_clic()
    tk.ferme_fenetre()


# -*- coding: utf-8 -*-

import logging
import unittest

import shlex
import subprocess

from .base import log_info

__title__ = 'tld.tests.test_commands'
__author__ = 'Artur Barseghyan'
__copyright__ = '2013-2019 Artur Barseghyan'
__license__ = 'GPL 2.0/LGPL 2.1'
__all__ = ('TestCommands',)

LOGGER = logging.getLogger(__name__)

class TestCommands(unittest.TestCase):
    """Tld commands tests."""

    def setUp(self):
        """Set up."""

    @log_info
    def test_1_update_tld_names_command(self):
        """Test updating the tld names (re-fetch mozilla source)."""
        res = subprocess.check_output(shlex.split('update-tld-names')).strip()
        self.assertEqual(res, b'True')
        return res

    @log_info
    def test_2_update_tld_names_module(self):
        """Test updating the tld names (re-fetch mozilla source)."""
        res = subprocess.check_output(
            shlex.split('python src/tld/commands/update_tld_names.py')
        ).strip()
        self.assertEqual(res, b'True')
        return res

if __name__ == '__main__':
    unittest.main()


import os, pickle, logging, numpy as np
from tqdm import tqdm

from .. import utils as U
from .transformer import pre_normalization

class NTU_Reader():
    def __init__(self, args, root_folder, transform, ntu60_path, ntu120_path, **kwargs):
        self.max_channel = 3
        self.max_frame = 300
        self.max_joint = 25
        self.max_person = 4
        self.select_person_num = 2
        self.dataset = args.dataset
        self.split = args.dataset_args['split']
        self.progress_bar = not args.no_progress_bar
        self.transform = transform
        self.mutual_only = ('mutual' in self.dataset)
        self.mutual_action_map = {50: 1, 51: 2, 52: 3, 53: 4, 54: 5, 55: 6, 56: 7, 57: 8, 58: 9, 59: 10, 60: 11, 106: 12, 107: 13, 108: 14, 109: 15, 110: 16, 111: 17, 112: 18, 113: 19, 114: 20, 115: 21, 116: 22, 117: 23, 118: 24, 119: 25, 120: 26}

        # Set paths
        ntu_ignored = '{}/ntu_ignore.txt'.format(os.path.dirname(os.path.realpath(__file__)))
        if self.transform:
            self.out_path = '{}/transformed/{}'.format(root_folder, self.dataset+'-'+self.split)
        else:
            self.out_path = '{}/{}'.format(root_folder, self.dataset+'-'+self.split)
        U.create_folder(self.out_path)

        # Divide train and eval samples
        training_samples = dict()

        training_samples['xview'] = [2, 3]
        training_samples['xsub'] = [
            1, 2, 4, 5, 8, 9, 13, 14, 15, 16, 17, 18, 19, 25, 27, 28, 31, 34, 35,
            38, 45, 46, 47, 49, 50, 52, 53, 54, 55, 56, 57, 58, 59, 70, 74, 78,
            80, 81, 82, 83, 84, 85, 86, 89, 91, 92, 93, 94, 95, 97, 98, 100, 103
        ]
        training_samples['xset'] = set(range(2, 33, 2))
        self.training_sample = training_samples[self.split]

        # Get ignore samples
        try:
            with open(ntu_ignored, 'r') as f:
                self.ignored_samples = [line.strip() + '.skeleton' for line in f.readlines()]
        except:
            logging.info('')
            logging.error('Error: Wrong in loading ignored sample file {}'.format(ntu_ignored))
            raise ValueError()

        # Get skeleton file list
        self.file_list = []
        for folder in [ntu60_path, ntu120_path]:
            for filename in os.listdir(folder):
                self.file_list.append((folder, filename))
            if '120' not in self.dataset:  # for NTU 60, only one folder
                break

    def read_file(self, file_path):
        skeleton = np.zeros((self.max_person, self.max_frame, self.max_joint, self.max_channel), dtype=np.float32)
        with open(file_path, 'r') as fr:
            frame_num = int(fr.readline())
            for frame in range(frame_num):
                person_num = int(fr.readline())
                for person in range(person_num):
                    person_info = fr.readline().strip().split()
                    joint_num = int(fr.readline())
                    for joint in range(joint_num):
                        joint_info = fr.readline().strip().split()
                        skeleton[person,frame,joint,:] = np.array(joint_info[:self.max_channel], dtype=np.float32)
        return skeleton[:,:frame_num,:,:], frame_num

    def get_nonzero_std(self, s):  # (T,V,C)
        index = s.sum(-1).sum(-1) != 0  # select valid frames
        s = s[index]
        if len(s) != 0:
            s = s[:, :, 0].std() + s[:, :, 1].std() + s[:, :, 2].std()  # three channels
        else:
            s = 0
        return s

    def gendata(self, phase):
        sample_data = []
        sample_label = []
        sample_path = []
        sample_length = []
        iterizer = tqdm(sorted(self.file_list), dynamic_ncols=True) if self.progress_bar else sorted(self.file_list)
        for folder, filename in iterizer:
            if filename in self.ignored_samples:
                continue

            # Get sample information
            file_path = os.path.join(folder, filename)
            setup_loc = filename.find('S')
            camera_loc = filename.find('C')
            subject_loc = filename.find('P')
            action_loc = filename.find('A')
            setup_id = int(filename[(setup_loc+1):(setup_loc+4)])
            camera_id = int(filename[(camera_loc+1):(camera_loc+4)])
            subject_id = int(filename[(subject_loc+1):(subject_loc+4)])
            action_class = int(filename[(action_loc+1):(action_loc+4)])

            # filter out single-person samples
            if self.mutual_only and action_class not in self.mutual_action_map:
                continue

            # Distinguish train or eval sample
            if self.split == 'xview':
                is_training_sample = (camera_id in self.training_sample)
            elif self.split == 'xsub':
                is_training_sample = (subject_id in self.training_sample)
            elif self.split == 'xset':
                is_training_sample = (setup_id in self.training_sample)
            else:
                logging.info('')
                logging.error('Error: Do NOT exist this dataset {}'.format(self.dataset))
                raise ValueError()
            if (phase == 'train' and not is_training_sample) or (phase == 'eval' and is_training_sample):
                continue

            # Read one sample
            data = np.zeros((self.max_channel, self.max_frame, self.max_joint, self.select_person_num), dtype=np.float32)
            skeleton, frame_num = self.read_file(file_path)

            # Select person by max energy
            energy = np.array([self.get_nonzero_std(skeleton[m]) for m in range(self.max_person)])
            index = energy.argsort()[::-1][:self.select_person_num]
            skeleton = skeleton[index]
            data[:,:frame_num,:,:] = skeleton.transpose(3, 1, 2, 0)

            sample_data.append(data)
            sample_path.append(file_path)
            if self.mutual_only:
                action_class = self.mutual_action_map[action_class]
            sample_label.append(action_class - 1)  # to 0-indexed
            sample_length.append(frame_num)

        # Save label
        with open('{}/{}_label.pkl'.format(self.out_path, phase), 'wb') as f:
            pickle.dump((sample_path, list(sample_label), list(sample_length)), f)

        # Transform data
        sample_data = np.array(sample_data)
        if self.transform:
            sample_data = pre_normalization(sample_data, progress_bar=self.progress_bar)

        # Save data
        np.save('{}/{}_data.npy'.format(self.out_path, phase), sample_data)

    def start(self):
        for phase in ['train', 'eval']:
            logging.info('Phase: {}'.format(phase))
            self.gendata(phase)


def rotate(text, key):

    if text is None or len(text) == 0:
        raise Exception('Invalid text')

    if key < 0 or key > 26:
        raise Exception('Invalid key')

    cipher = ''

    for char in list(text):

        if not char.isalpha():
            cipher += char
            continue

        if char.istitle():
            base = 65
        else:
            base = 97

        val = ord(char) - base
        val += key
        val %= 26
        char = chr(val + base)
        cipher += char

    return cipher


import os
import re
import textract

from pyspark import SparkContext, SQLContext
from pyspark.rdd import RDD
from pyspark.sql import SparkSession, DataFrame, Row, functions as F
from pyspark.sql.types import StringType, StructField, StructType

START_DIR = "/home/jeff/git/BigData/026-ProfilesPySpark/profiles/"
STOP_WORDS_FILENAME: str = "terrier-stop.txt"

FULL_WEIGHTED_OUTPUT_FILENAME:str  = "FullWeighted.out"
JOINED_OUTPUT_FILENAME:str = "Joined.out"

WORD_COLUMN_NAME:  str    = "word"
COUNT_COLUMN_NAME: str    = "count"
WEIGHTED_COLUMN_NAME: str = "weight"
FREQUENCY_COUNT_FN: str   = "freq_count"

# FOUR PROFILES TO GATHER DATA ON
THE_CHOSEN_FOUR:list = [ "JeffreyReiherProfile.doc",\
                         "VicenteGuevaraProfile.doc",\
                         "KislayJhaProfile.docx",\
                         "ChrisPeng-Profile.docx"\
                       ]

TOOL_LIST: list =      [ "kafka",  "spark",   "emr",     "mr",        "mapreduce",
                         "flume",  "aws",     "hadoop",  "spark",     "data bricks",
                         "hive",   "presto",  "airflow", "ec2",       "amazon",
                         "python", "pyspark", "redshift" "snowflake", "googleads"
                       ]
SC: SparkContext = SparkContext()
SPARK: SparkSession = SparkSession(SC)

total_counts_for_the_four: list = []

def read_all_profiles() -> str:
    """
    Open each profile in the directory and extract the text
    :return: a string of all the profiles concatenated
    """
    text = ""

    # READ EACH OF THE PROFILE DOCUMENTS AND ADD TO TEXT VAR
    for dir, subdirs, files in os.walk(START_DIR):
        for fname in files:
            full_path = START_DIR + fname
            try:
                text += str(textract.process(full_path).decode("utf-8") ).lower()
            except:
                print("Exception on " + fname)
    return text.lower()

def read_one_profile( fname: str ) -> str:
    # READ EACH OF THE PROFILE DOCUMENTS AND ADD TO TEXT VAR
    full_path = START_DIR + fname
    return str(textract.process(full_path).decode("utf-8") ).lower()

#def into_lines( text:str ) -> list:
def into_unique_lines(text: str) -> RDD:
    """
    Split a string into a list of strings by separating the supplied string at each carriage return
    :param text: The text to split
    :return: a list of strings - one for each line in the text
    """
    result = []
    split_lines: list = re.split("\n", text )
    for i in split_lines:
        result.append( i.strip() )

    return SC.parallelize(result).distinct()

def into_words( rdd_in: RDD ) -> RDD:
    words: RDD = rdd_in.flatMap( lambda x: x.split(' ') ).filter( lambda x: len(x) > 1 )
    return words.map( lambda x: re.sub( ',()',"", x ))

def remove_stop_words( rdd_in: RDD ) -> RDD:
    file = open(STOP_WORDS_FILENAME, "r")
    sw_list: list = file.read().split("\n")
    return rdd_in.filter( lambda x: x not in sw_list )

def compute_sum_of_values( rdd_in: RDD ) ->  int:
    """
    Given an RDD, compute sum of all the counts (the 2nd item in the tuple)
    :param rdd_in: the rdd to process
    :return: the total number of words counted in this rdd
    """
    sum: int = 0
    for (i,j) in rdd_in:
        sum+=j

    return sum

def write_weighted_rdd( weighted_rdd: RDD, file_name: str ):
    sql_ctx: SQLContext = SQLContext(SC)
    df:DataFrame = sql_ctx.createDataFrame(weighted_rdd, [ WORD_COLUMN_NAME,COUNT_COLUMN_NAME,
                                                           WEIGHTED_COLUMN_NAME
                                                         ]).orderBy( F.col(COUNT_COLUMN_NAME).desc() )

    df.coalesce(1).write.format("com.databricks.spark.csv")\
                .options(header="true")\
                .save("file:///home/jeff/git/BigData/026-ProfilesPySpark/results/" + file_name )

def clean_row( r: Row ) -> Row:
    for i in [1,3,5,7]:
        if r[i] == "":
            r[i] = 0
    r[9] = r[1] + r[3] + r[5] + r[7]
    return r

def write_joined_df( df: DataFrame, file_name: str ) -> DataFrame:
    #df=df.fillna(0, subset= [ "count_0",  "count_1",  "count_2",  "count_3"  ] )
    #df=df.fillna(0, subset= [ "weight_0", "weight_1", "weight_2", "weight_3" ] )
    df=df.fillna( 0 )
    df=df.withColumn("total", df[1] + df[3] + df[5] + df[7])

    sum_df: DataFrame = df.groupBy().sum()
    total_words:int = sum_df.collect()[0][4]

    #freq_db_fn = F.udf(lambda cnt: freq_db(cnt, total_words) )
    df=df.withColumn("total-weight", F.round(df[9]/total_words*100,3) )

    df.coalesce(1).write.format("com.databricks.spark.csv")\
                .options(header="true")\
                .save("file:///home/jeff/git/BigData/026-ProfilesPySpark/results/" + file_name )
    return df

def calculate_weights( rdd_in: RDD ) -> RDD:
    rdd_in = rdd_in.collect()
    total: int = compute_sum_of_values( rdd_in )
    total_counts_for_the_four.append( total )

    return SC.parallelize(rdd_in).map(lambda x: (x[0], x[1], round(100*x[1]/total)))

def total_weight_of_list( words: list, joined_weights: DataFrame ) -> list:
    # WORD LIST TO DF
    words_df = SPARK.createDataFrame(words, StringType()).toDF( WORD_COLUMN_NAME )
    #joined_weights = joined_weights.fillna(0)

    # INNER JOIN DFs
    words_df = words_df.join( joined_weights, on=WORD_COLUMN_NAME, how="inner")

    # COMPUTE TOTAL COUNTS AND WEIGHTS FOR EACH "PROFILE"
    words_df = words_df.agg(F.sum("count_0"), F.sum("count_1"), F.sum("count_2"), F.sum("count_3")).collect()

    return [ 100*words_df[0][i]/total_counts_for_the_four[i] for i in range(4) ]

def convert_to_words( lines: list ) -> RDD:
    words: RDD = into_words( u_lines )
    return remove_stop_words( words )

if __name__ == '__main__':
    # READ TEXT FROM DOC FILES AND CONVERT TO TEXT
    text: str    = read_all_profiles()

    # CONVERT TEXT INTO A LIST OF UNIQUE LINES
    u_lines: RDD = into_unique_lines( text )

    '''
    with open( "profile-lines.txt", "w" ) as file:
        for i in u_lines.collect():
            file.write(i)
            file.write('\n')
    quit()
    '''

    # CONVERT TEXT INTO A LIST OF WORDS (WITH STOP WORDS REMOVED)
    word_rdd: RDD = convert_to_words( u_lines )

    # MAP AND REDUCE THE DATA
    reduced: RDD = word_rdd.map(lambda x: (x, 1)).reduceByKey(lambda i, j: i + j)

    # CALCULATE THE WEIGHT
    weighted_rdd: RDD = calculate_weights( reduced )

    # WRITE CSV TO DISK
    write_weighted_rdd( weighted_rdd, FULL_WEIGHTED_OUTPUT_FILENAME )

    join_df: DataFrame = None
    filenum:int = 0
    for f in THE_CHOSEN_FOUR:
        # READ TEXT FROM DOC FILES AND CONVERT TO TEXT
        text = read_one_profile( f )

        # CONVERT TEXT INTO A LIST OF LINES
        u_lines = into_unique_lines( text )

        # CONVERT TEXT INTO A LIST OF WORDS (WITH STOP WORDS REMOVED)
        word_rdd = convert_to_words( u_lines )

        # MAP AND REDUCE THE DATA
        reduced = word_rdd.map(lambda x: (x, 1)).reduceByKey(lambda i, j: i + j)

        # CALCULATE THE WEIGHT
        weighted_rdd = calculate_weights( reduced )

        # WRITE CSV TO DISK
        write_weighted_rdd( weighted_rdd, f )

        # sql_ctx: SQLContext = SQLContext(SC)
        sql_ctx: SQLContext = SQLContext(SC)
        new_df: DataFrame\
            = sql_ctx.createDataFrame( weighted_rdd,
                                       [ WORD_COLUMN_NAME,
                                         COUNT_COLUMN_NAME + "_" + str(filenum),
                                         WEIGHTED_COLUMN_NAME + "_" + str(filenum)
                                       ]
                                      )

        if join_df == None:
            join_df = new_df
        else:
            join_df = join_df.join( new_df, on=WORD_COLUMN_NAME, how="outer" )

        filenum += 1

    join_df = write_joined_df( join_df, JOINED_OUTPUT_FILENAME )

    for i in total_weight_of_list(TOOL_LIST, join_df ):
        print(i)


"""
Implement a basic calculator to evaluate a simple expression string.

The expression string may contain open ( and closing parentheses ), the plus + or minus sign -, non-negative integers and empty spaces .

Example 1:
Input: "1 + 1"
Output: 2

Example 2:
Input: " 2-1 + 2 "
Output: 3

Example 3:
Input: "(1+(4+5+2)-3)+(6+8)"
Output: 23

Note:
You may assume that the given expression is always valid.
Do not use the eval built-in library function.
"""

class Solution(object):
    """
    类似于词法分析 + 语法分析过程结合
    从左到右扫描整个串，遇到括号产生递归，递归调用返回括号内表达式的值
    """

    def calculator(self, string):
        self.expr = string.replace(' ', '')
        self.idx = 0
        return self.expression()

    def expression(self):
        expr = 0
        sign = True
        while self.idx < len(self.expr):
            if self.expr[self.idx] in "1234567890":
                num = ""
                while self.idx < len(self.expr) and self.expr[self.idx] in "1234567890":
                    num += self.expr[self.idx]
                    self.idx += 1
                expr += int(num) if sign else -int(num)
            elif self.expr[self.idx] in '+-':
                sign = self.expr[self.idx] == '+'
                self.idx += 1
            elif self.expr[self.idx] == '(':
                self.idx += 1
                num = self.expression()
                expr += int(num) if sign else -int(num)
            elif self.expr[self.idx] == ')':
                self.idx += 1
                break
        return expr

    def test(self):
        string = " ( 1+ ( 4 +5+ 2)- 3)+( 6  + 8) "
        print(self.calculator(string))

class Solution2(object):
    """
    通用解法（加减，乘除需要额外添加功能，但很容易实现）
    先词法分析得到infix tokens
    再将 infix tokens 转换为 postfix tokens （调度场算法）
    再对 postfix 表达式求值
    由于相当于跑了三遍，略慢
    """

    def calculator(self, infix_expr):
        infix_tokens = self.tokenize(infix_expr)
        postfix_tokens = self.infix_to_postfix(infix_tokens)
        return self.postfix_calc(postfix_tokens)

    def tokenize(self, infix_expr):
        tokens = []
        i = 0
        while i < len(infix_expr):
            if infix_expr[i] == ' ':
                i += 1
            elif infix_expr[i] in "+-()":
                tokens.append(infix_expr[i])
                i += 1
            elif infix_expr[i] in "1234567890":
                num = ""
                while i < len(infix_expr) and infix_expr[i] in "1234567890":
                    num += infix_expr[i]
                    i += 1
                tokens.append(int(num))
        return tokens

    def infix_to_postfix(self, infix_tokens):
        postfix_tokens = []
        ops_stack = []
        for token in infix_tokens:
            if type(token) == int:
                postfix_tokens.append(token)
            elif token in "+-":
                while ops_stack and ops_stack[-1] != '(':
                    postfix_tokens.append(ops_stack.pop())
                ops_stack.append(token)
            elif token == '(':
                ops_stack.append(token)
            elif token == ')':
                while ops_stack and ops_stack[-1] != '(':
                    postfix_tokens.append(ops_stack.pop())
                ops_stack.pop()
        while ops_stack:
            postfix_tokens.append(ops_stack.pop())
        return postfix_tokens

    def postfix_calc(self, postfix_tokens):

        def _calc(op, a, b):
            if op == '+':
                return a + b
            if op == '-':
                return a - b

        stack = []
        for token in postfix_tokens:
            if type(token) == int:
                stack.append(token)
            else:
                b = stack.pop()
                a = stack.pop()
                stack.append(_calc(token, a, b))

        return stack[-1]

class Solution3(object):
    """
    添加乘除法（不同优先级）运算
    """

    def calculate(self, infix_expr):
        return self.postfix_calc(self.infix_to_postfix(self.tokenize(infix_expr)))

    def tokenize(self, infix_expr):
        """tokennize 过程不变"""
        tokens = []
        i = 0
        while i < len(infix_expr):
            if infix_expr[i] == ' ':
                i += 1
            elif infix_expr[i] in "+-*/()":
                tokens.append(infix_expr[i])
                i += 1
            elif infix_expr[i] in "1234567890":
                num = ""
                while i < len(infix_expr) and infix_expr[i] in "1234567890":
                    num += infix_expr[i]
                    i += 1
                tokens.append(int(num))
        return tokens

    def postfix_calc(self, postfix_tokens):
        """ 后缀式计算过程不变 """
        def _calc(op, a, b):
            if op == '+':
                return a + b
            if op == '-':
                return a - b
            if op == '*':
                return a * b
            if op == '/':
                return a // b

        stack = []
        for token in postfix_tokens:
            print(stack)
            if type(token) == int:
                stack.append(token)
            else:
                b = stack.pop()
                a = stack.pop()
                stack.append(_calc(token, a, b))

        return stack[-1]

print(Solution3().calculate("(2+6* 3+5- (3*14/    7 +2)* 5)+ 3"))