import pygame
import sys
import os
import pathlib

pygame.mixer.init()
pygame.init()

# this use for build .exe file
def resource_path(relative_path):
    try:
        # PyInstaller creates a temp folder and stores path in _MEIPASS
        base_path = sys._MEIPASS
    except Exception:
        base_path = pathlib.Path(__file__).parent.resolve()
    return os.path.join(base_path, relative_path)

gameDefaultSettings = {
    "WINDOW_WIDTH": 1000,
    "WINDOW_HEIGHT": 600,
    "WINDOW_TITLE": "Flappy Peter Teo (Made by Thuongton999)",
    "BIRD_WIDTH": 100,
    "BIRD_HEIGHT": 80,
    "SPAWN_POSITION": (200, 255),
    "BIRD_JUMP_SPEED": -8,
    "WINDOW_SPEED": 3,
    "FPS": 60,
    "GRAVITY": 0.5,
    "MIN_COLUMN_HEIGHT": 100,
    "MAX_COLUMN_HEIGHT": 300,
    "COLUMN_SPACING": 250,
    "COLUMNS": 6,
    "DEFAULT_TEXT": pygame.font.Font(resource_path("font/FlappyBirdRegular.ttf"), 48),
    "DEFAULT_HEADER": pygame.font.Font(resource_path("font/FlappyBirdRegular.ttf"), 78),
    "DEFAULT_TITLE": pygame.font.Font(resource_path("font/FlappyBirdRegular.ttf"), 128),
    "COPYRIGHT": pygame.font.Font(resource_path("font/FlappyBirdRegular.ttf"), 42),
    "BIRD_IMAGE": pygame.image.load(resource_path("images/peter_teo.png")),
    "COLUMN_IMAGE": pygame.image.load(resource_path("images/dieucay.png")),
    "ICON": pygame.image.load(resource_path("images/favicon.ico")),
    "BACKGROUND": pygame.image.load(resource_path("images/background.png")),
    "SCOREBOARD": pygame.image.load(resource_path("images/scoreboard.png")),
    "START_BUTTON": pygame.image.load(resource_path("images/start_button.png")),
    "BRONZE_MEDAL": pygame.image.load(resource_path("images/medal_bronze.png")),
    "SILVER_MEDAL": pygame.image.load(resource_path("images/medal_silver.png")),
    "GOLD_MEDAL": pygame.image.load(resource_path("images/medal_gold.png")),
    "PlATINUM_MEDAL": pygame.image.load(resource_path("images/medal_platinum.png")),
    "MEDAL_HOVER": pygame.image.load(resource_path("images/medal_hover.png")),
    "TAP_SOUND": pygame.mixer.Sound(resource_path("sounds/pop.wav")),
    "GAME_OVER_SOUND": pygame.mixer.Sound(resource_path("sounds/game_over.wav")),
}

colors = {
    "white": (255, 255, 255),
    "black": (0, 0, 0),
    "clotting": (82,58,74),
    "sun": (230, 193, 32),
    "grey": (143, 143, 143),
    "lemon_chiffon": (255, 250, 205),
}

def setKwargsToProps(props, kwargs):
    settings = kwargs.items()
    for setting, value in settings:
        try:
            props[setting] = value
        except KeyError:
            raise "Key setting %s not found" % (setting)

class Environment:
    def __init__(self, **kwargs):
        properties = {
            "gravity": gameDefaultSettings["GRAVITY"],
            "tap_sound": gameDefaultSettings["TAP_SOUND"],
            "game_over_sound": gameDefaultSettings["GAME_OVER_SOUND"]
        }
        setKwargsToProps(properties, kwargs)
        self.gravity = properties["gravity"]
        self.tapSound = properties["tap_sound"]
        self.gameOverSound = properties["game_over_sound"]


# Faça um programa que leia um numero inteiro e defina se ele é primo ou não
num = int(input("Digite um número inteiro: "))
tot=0
for i in range(1,num+1):#expressões de multiplicação e divisão começe o for por um, zero não é divisivel nem multiplicavel
    if num % i == 0:
        print('\033[33m',end=' ')
        tot+=1
    else:
        print('\033[31m', end=' ')
    print('{}'.format(i), end=' ')

if tot==2:
    print('\nÉ primo')
else:
    print("\nNão é primo")


import logging
import os
import time
from copy import deepcopy
from typing import Dict, List, Optional, Union

from .common import (input_artifact_pattern, input_parameter_pattern,
                     step_output_artifact_pattern,
                     step_output_parameter_pattern)
from .config import config, s3_config
from .context_syntax import GLOBAL_CONTEXT
from .io import Inputs, Outputs
from .op_template import OPTemplate
from .step import Step, add_slices

try:
    from argo.workflows.client import V1alpha1Metadata, V1alpha1Template
except Exception:
    pass

class Steps(OPTemplate):
    """
    Steps

    Args:
        name: the name of the steps
        inputs: inputs in the template
        outputs: outputs in the template
        steps: a sequential list of steps
        memoize_key: memoized key of the steps
        annotations: annotations for the OP template
        parallelism: maximum number of running pods for the OP template
        """

    def __init__(
            self,
            name: Optional[str] = None,
            inputs: Optional[Inputs] = None,
            outputs: Optional[Outputs] = None,
            steps: List[Union[Step, List[Step]]] = None,
            memoize_key: Optional[str] = None,
            annotations: Dict[str, str] = None,
            parallelism: Optional[int] = None,
    ) -> None:
        super().__init__(name=name, inputs=inputs, outputs=outputs,
                         memoize_key=memoize_key, annotations=annotations)
        self.parallelism = parallelism
        self.steps = []
        if steps is not None:
            for step in steps:
                self.add(step)

    @classmethod
    def from_dict(cls, d, templates):
        kwargs = {
            "name": d.get("name", None),
            "inputs": Inputs.from_dict(d.get("inputs", {})),
            "outputs": Outputs.from_dict(d.get("outputs", {})),
            "memoize_key": d.get("memoize", {}).get("key", None),
            "annotations": d.get("metadata", {}).get("annotations", None),
            "parallelism": d.get("parallelism", None),
        }
        kwargs["steps"] = [[Step.from_dict(ps, templates) for ps in step]
                           for step in d.get("steps", [[]])]
        return cls(**kwargs)

    def __iter__(self):
        return iter(self.steps)

    def add(
            self,
            step: Union[Step, List[Step]],
    ) -> None:
        """
        Add a step or a list of parallel steps to the steps

        Args:
            step: a step or a list of parallel steps to be added to the
                entrypoint of the workflow
        """
        assert isinstance(step, (Step, list))
        if isinstance(step, Step):
            if step.prepare_step is not None:
                self.steps.append(step.prepare_step)
        elif isinstance(step, list):
            prepare_steps = [ps.prepare_step for ps in step
                             if ps.prepare_step is not None]
            if prepare_steps:
                self.steps.append(prepare_steps)
        for ps in [step] if isinstance(step, Step) else step:
            if ps.parallelism is not None:
                if self.parallelism is None:
                    self.parallelism = ps.parallelism
                else:
                    self.parallelism = min(self.parallelism, ps.parallelism)
                    logging.warn("Multiple steps specified parallelism, small"
                                 "er one %s will be used" % self.parallelism)
        self.steps.append(step)
        if isinstance(step, Step):
            if step.check_step is not None:
                self.steps.append(step.check_step)
        elif isinstance(step, list):
            check_steps = [ps.check_step for ps in step
                           if ps.check_step is not None]
            if check_steps:
                self.steps.append(check_steps)

    def convert_to_argo(self, memoize_prefix=None,
                        memoize_configmap="dflow", context=None):
        argo_steps = []
        templates = []
        assert len(self.steps) > 0, "Steps %s is empty" % self.name
        for step in self.steps:
            # each step of steps should be a list of parallel steps, if not,
            # create a sigleton
            if not isinstance(step, list):
                step = [step]
            argo_parallel_steps = []
            for ps in step:
                argo_parallel_steps.append(ps.convert_to_argo(context))
                # template may change after conversion
                templates.append(ps.template)
            argo_steps.append(argo_parallel_steps)

        self.handle_key(memoize_prefix, memoize_configmap)
        argo_template = \
            V1alpha1Template(name=self.name,
                             metadata=V1alpha1Metadata(
                                 annotations=self.annotations),
                             steps=argo_steps,
                             inputs=self.inputs.convert_to_argo(),
                             outputs=self.outputs.convert_to_argo(),
                             memoize=self.memoize,
                             parallelism=self.parallelism)
        return argo_template, templates

    def convert_to_graph(self):
        graph_steps = []
        templates = []
        for step in self.steps:
            if not isinstance(step, list):
                step = [step]
            graph_parallel_steps = []
            for ps in step:
                if not ps.name.endswith("-init-artifact") and \
                    not ps.name.endswith("-check-num-success") and \
                        not ps.name.endswith("-check-success-ratio"):
                    graph_parallel_steps.append(ps.convert_to_graph())
                    templates.append(ps.template)
            if len(graph_parallel_steps) > 0:
                graph_steps.append(graph_parallel_steps)

        return {
            "type": "Steps",
            "name": self.name,
            "annotations": self.annotations,
            "steps": graph_steps,
            "inputs": self.inputs.convert_to_graph(),
            "outputs": self.outputs.convert_to_graph(),
            "parallelism": self.parallelism,
        }, templates

    @classmethod
    def from_graph(cls, graph, templates):
        assert graph.pop("type") == "Steps"
        graph["inputs"] = Inputs.from_graph(graph.get("inputs", {}))
        steps = graph.pop("steps")
        outputs = graph.pop("outputs", {})

        obj = cls(**graph)
        templates[graph["name"]] = obj

        step_dict = {}
        for step in steps:
            parallel_steps = []
            for ps in step:
                s = Step.from_graph(ps, templates)
                parallel_steps.append(s)
                step_dict[s.name] = s
            obj.add(parallel_steps)

        # replace variable references with pointers
        for step in obj.steps:
            for ps in step:
                for name, par in ps.inputs.parameters.items():
                    value = getattr(par, "value", None)
                    if isinstance(value, str):
                        match = input_parameter_pattern.match(value)
                        if match:
                            ps.set_parameters({
                                name: obj.inputs.parameters[match.group(1)]})
                        match = step_output_parameter_pattern.match(value)
                        if match:
                            ps.set_parameters({name: step_dict[match.group(
                                1)].outputs.parameters[match.group(2)]})
                for name, art in ps.inputs.artifacts.items():
                    source = art.source
                    if isinstance(source, str):
                        match = input_artifact_pattern.match(source)
                        if match:
                            ps.set_artifacts({
                                name: obj.inputs.artifacts[match.group(1)]})
                        match = step_output_artifact_pattern.match(source)
                        if match:
                            ps.set_artifacts({name: step_dict[match.group(
                                1)].outputs.artifacts[match.group(2)]})

        obj.outputs = Outputs.from_graph(outputs)
        # replace variable references with pointers
        for par in obj.outputs.parameters.values():
            _from = par.value_from_parameter
            if isinstance(_from, str):
                match = input_parameter_pattern.match(_from)
                if match:
                    par.value_from_parameter = obj.inputs.parameters[
                        match.group(1)]
                match = step_output_parameter_pattern.match(_from)
                if match:
                    par.value_from_parameter = step_dict[match.group(1)].\
                        outputs.parameters[match.group(2)]
        for art in obj.outputs.artifacts.values():
            _from = art._from
            if isinstance(_from, str):
                match = input_artifact_pattern.match(_from)
                if match:
                    art._from = obj.inputs.artifacts[match.group(1)]
                match = step_output_artifact_pattern.match(_from)
                if match:
                    art._from = step_dict[match.group(1)].outputs.\
                        artifacts[match.group(2)]
        return obj

    def run(self, workflow_id=None, context=None):
        self.workflow_id = workflow_id
        for step in self:
            if isinstance(step, list):
                import concurrent.futures
                cwd = os.getcwd()
                max_workers = config["debug_pool_workers"]
                if max_workers == -1:
                    max_workers = len(step)
                with concurrent.futures.ProcessPoolExecutor(
                        max_workers) as pool:
                    futures = []
                    for i, ps in enumerate(step):
                        ps.phase = "Pending"
                        try:
                            future = pool.submit(
                                ps.run_with_config, self, context, config,
                                s3_config, cwd)
                        except concurrent.futures.process.BrokenProcessPool \
                                as e:
                            # retrieve exception of subprocess before exit
                            for future in concurrent.futures.as_completed(
                                    futures):
                                future.result()
                            raise e
                        futures.append(future)
                        if config["debug_batch_size"] and i != len(step) - 1 \
                                and (i+1) % config["debug_batch_size"] == 0:
                            logging.info(
                                "Wait %s seconds before submitting next "
                                "batch" % config["debug_batch_interval"])
                            time.sleep(config["debug_batch_interval"])

                    for future in concurrent.futures.as_completed(futures):
                        j = futures.index(future)
                        try:
                            ps = future.result()
                        except Exception:
                            import traceback
                            traceback.print_exc()
                            step[j].phase = "Failed"
                            if not step[j].continue_on_failed:
                                raise RuntimeError("Step %s failed" % step[j])
                        else:
                            step[j].outputs = deepcopy(ps.outputs)
            else:
                step.run(self, context)

    def add_slices(self, slices, layer=0):
        add_slices(self, slices, layer=layer)

    def copy(self):
        new_template = self.deepcopy()
        for step, new_step in zip(self.steps, new_template.steps):
            if isinstance(new_step, list):
                for ps, new_ps in zip(step, new_step):
                    new_ps.template = ps.template
            else:
                new_step.template = step.template
        return new_template

    def __enter__(self) -> 'Steps':
        GLOBAL_CONTEXT.in_context = True
        GLOBAL_CONTEXT.current_workflow = self
        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        GLOBAL_CONTEXT.in_context = False


import numpy as np
import cv2
import math

# Structuring element for morphology
kernel = np.ones((12,12),np.uint8) # 12by12  square

# VideoCapture using the only available camera
cap = cv2.VideoCapture(0)

# Will wait until black is seen (Code Reset) and used the location to only focus on panel while detecting the code
focused = False
# Hold set of seen colors
res = []
# define range of black in HSV
black_upper_hsv = np.array([179, 60, 95], dtype = "uint8")
black_lower_hsv = np.array([0, 0, 0], dtype = "uint8")

# Set up the detector with default parameters.
# Setup SimpleBlobDetector parameters.
params = cv2.SimpleBlobDetector_Params()
# Filter by Area.
params.filterByArea = True
params.minArea = 100
# Create detector
detector = cv2.SimpleBlobDetector(params)

# define the list of boundaries
boundaries = [
    (['R'], [17, 15, 100], [50, 56, 200]),      # R
    (['B'], [110, 100, 140], [160, 185, 255]),  # B
    (['Y'], [25, 146, 190], [62, 174, 250]),    # Y
    (['G'], [103, 86, 65], [145, 133, 128])     # G
]

imgFile = cv2.imread('/Users/ianstankosh/Desktop/HeadShot.jpg')
cv2.imshow('Original Black', imgFile)

hsv = cv2.cvtColor(imgFile, cv2.COLOR_BGR2HSV)
cv2.imshow('HSV_BLK', hsv)

# Threshold the HSV image to get only input colors
blk = cv2.inRange(hsv, black_lower_hsv, black_upper_hsv)
#blk = cv2.bitwise_not(blk)
cv2.imshow('blk', blk)
closing = cv2.morphologyEx(blk, cv2.MORPH_CLOSE, kernel)
cv2.imshow('Closing_blk', closing)

# Detect blobs.
keypoints = detector.detect(closing)

#Draw detected blobs as red circles.
#cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS ensures the size of the circle corresponds to the size of blob
im_with_keypoints = cv2.drawKeypoints(closing, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
cv2.imshow('Panel', im_with_keypoints)

panel = closing

imgFile2 = cv2.imread('BluePic.jpg')
cv2.imshow('Original Blue', imgFile2)
# Mask the frame to only focus on light panel
masked_data = cv2.bitwise_and(imgFile2, imgFile2, mask=panel)
cv2.imshow('Masked Data', masked_data)

# Covert to HSV to detect color
hsv2 = cv2.cvtColor(masked_data, cv2.COLOR_BGR2HSV)
cv2.imshow('HSV_BLU', hsv2)

# cv2.imwrite('hue.png', hsv2[:,:,0])
# cv2.imwrite('sat.png', hsv2[:,:,1])
# cv2.imwrite('val.png', hsv2[:,:,2])

# loop over the boundaries
for (color, lower, upper) in boundaries:
    # create NumPy arrays from the boundaries
    lower = np.array(lower, dtype = "uint8")
    upper = np.array(upper, dtype = "uint8")

    # find the colors within the specified boundaries and apply
    # the mask
    mask = cv2.inRange(hsv2, lower, upper)
    cv2.imshow('mask', mask)
    closing = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
    cv2.imshow('Closing', closing)
    #output = cv2.bitwise_and(image, image, mask = mask)
    #closing = cv2.bitwise_not(closing)

    # Detect blobs.
    keypoints = detector.detect(closing)

    #Draw detected blobs as red circles.
    #cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS ensures the size of the circle corresponds to the size of blob
    im_with_keypoints = cv2.drawKeypoints(closing, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
    cv2.imshow('Found', im_with_keypoints)

    if keypoints:
        res.append(color)

print ("Colors seen: ", res)

cv2.waitKey(0)
cv2.destroyAllWindows()


from bilibili import bilibili
from login import login
import time
import traceback
import datetime
import asyncio
from printer import Printer

had_gotted_guard = []

def CurrentTime():
    currenttime = int(time.mktime(datetime.datetime.now().timetuple()))
    return str(currenttime)

class OnlineHeart:

    async def apppost_heartbeat(self):
        await bilibili().apppost_heartbeat()

    async def pcpost_heartbeat(self):
        response = await bilibili().pcpost_heartbeat()
        return response

    async def heart_gift(self):
        await bilibili().heart_gift()

    async def guard_lottery(self):
        response = await bilibili().guard_list()
        json_response = response.json()
        for i in range(0, len(json_response)):
            if json_response[i]['Status']:
                GuardId = json_response[i]['GuardId']
                if GuardId not in had_gotted_guard:
                    had_gotted_guard.append(GuardId)
                    OriginRoomId = json_response[i]['OriginRoomId']
                    response2 = await bilibili().get_gift_of_captain(OriginRoomId, GuardId)
                    json_response2 = await response2.json(content_type=None)
                    if json_response2['code'] == 0:
                        Printer().printer(f"获取到房间[{OriginRoomId}]编号[{GuardId}]的上船亲密度:{json_response2['data']['message']}",
                                          "Lottery", "cyan")
                    elif json_response2['code'] == 400 and json_response2['msg'] == "你已经领取过啦":
                        Printer().printer(
                            f"房间[{OriginRoomId}]编号[{GuardId}]的上船亲密度已领过",
                            "Info", "green")
                    else:
                        Printer().printer(
                            f"房间[{OriginRoomId}]编号[{GuardId}] 的上船亲密度领取出错,{json_response2}",
                            "Error", "red")
            else:
                pass

    async def draw_lottery(self):
        black_list = ["测试", "test", "12345"]
        for i in range(212, 300):
            response = await bilibili().get_lotterylist(i)
            json_response = await response.json()
            if json_response['code'] == 0:
                temp = json_response['data']['title']
                for k in black_list:
                    if k in temp:
                        Printer().printer(f"检测到疑似钓鱼类测试抽奖，默认不参与，请自行判断抽奖可参与性","Warning","red")
                else:
                    check = len(json_response['data']['typeB'])
                    for g in range(0, check):
                        join_end_time = json_response['data']['typeB'][g]['join_end_time']
                        join_start_time = json_response['data']['typeB'][g]['join_start_time']
                        ts = CurrentTime()
                        if int(join_end_time) > int(ts) > int(join_start_time):
                            response1 = await bilibili().get_gift_of_lottery(i, g)
                            json_response1 = await response1.json()
                            Printer().printer(f"参与实物抽奖回显:{json_response1}", "Lottery","cyan")
                        else:
                            pass
            else:
                break

    async def run(self):
        while 1:
            try:
                Printer().printer("心跳", "Info","green")
                response = await self.pcpost_heartbeat()
                json_response = await response.json()
                if json_response['code'] == 3:
                    Printer().printer(f"cookie过期,将重新登录","Error","red")
                    login().login()
                await self.apppost_heartbeat()
                await self.heart_gift()
                await self.guard_lottery()
                await self.draw_lottery()
                await asyncio.sleep(300)
            except:
                await asyncio.sleep(10)
                traceback.print_exc()


s = """
def vvv():
    person = ['张三', '李四']

    for p in person:
        print('name=', p)

    return 1

va = vvv()
"""

try:
    c = compile(s, '<string>', 'exec')

    if c is not None:
        print('compile succeed!')

    exec(c, globals(), locals())

    a = globals()

    b = locals()

    d = a['va']

    print(str(d))

except IndentationError:
    print('compile error')


#!/usr/bin/env python3
# -*- coding:utf-8 -*-

# Importation des librairies
from code_project.data_science.utils_project import load_saved_file, process_nlp, apply_vectorization
from code_project.config_project import *

import pandas as pd

def load_test_set(path_test_set):
    """
    Charge le dataset de test
    """
    if not os.path.exists(path_test_set):
        raise FileNotFoundError("file {} doesn't exist".format(path_test_set))
    data_test = pd.read_excel(path_test_set)
    data_test = data_test[['phrase', 'emotion']]

    return data_test

def evaluate_model(display=True):
    """
    Evalue un modèle
    """
    from sklearn.metrics import accuracy_score, confusion_matrix

    # Chargement du modèle et de la vectorisation enregistrés
    model = load_saved_file(MODEL_PATH)
    vectorizer = load_saved_file(VECTOR_PATH)

    # Chargement du dataset de test
    data_test = load_test_set(TEST_SET_PATH)

    # Traitement des données textuelles
    data_test['nlp_sentence'] = process_nlp(data_test['phrase'])

    # Séparation des features et du label
    y_test = data_test.emotion
    x_test = data_test.nlp_sentence

    # Vectorisation des features
    features = apply_vectorization(x_test, vectorizer, type_vector=TYPE_VECTOR)

    # Prédictions à partir du modèle
    predictions = model.predict(features)

    # calcul des performances
    accuracy = accuracy_score(y_test, predictions)
    conf_mat = confusion_matrix(y_test, predictions)

    if display:
        print('EVALUATION FOR MODEL {} with vectorization {}: '.format(MODEL_NAME, VECTOR_NAME))
        print('accuracy : {}'.format(round(accuracy, 3)))
        print('confusion matrix : \n{}'.format(conf_mat))

    return accuracy

if __name__ == '__main__':
    evaluate_model(display=True)


import json
import typing
import uuid
from hashlib import md5
from random import choice
from string import digits, ascii_letters
from typing import Dict

import requests

class BusinessApi:
    """
        При отправке запроса к API приложение должно выполнить ряд действий.

        К массиву параметров запроса добавляются параметры: nonce, app_id и token, где
        -- nonce - случайно генерируемая, уникальная для каждого запроса строка,
        -- app_id - уникальный идентификатор, выданный при подключении интеграции на аккаунте,
        -- token - полученный токен интеграции.
        Полученный массив параметров сортируется в алфавитном порядке и преобразуется в json строку - params_string.
        К запросу добавляется заголовок sign. Подпись запроса вычисляется по формуле: sign = MD5( params_string + secret ),
        где параметром функции MD5 служит конкатенация компонентов:
        -- params_string - url-кодированная строка параметров
        -- secret - секретный ключ, полученный при подключении интеграции на аккаунте.
    """

    def __init__(self, secret_key, app_id):
        """
        Init - метод конструктор собирает библиотеку для работы.
        :param secret_key: секретный ключ, полученный при подключении интеграции на аккаунте.
        :param app_id: уникальный идентификатор, выданный при подключении интеграции на аккаунте.
        :param nonce: случайно генерируемая, уникальная для каждого запроса строка.
        """
        self.secret_key = secret_key
        self.app_id = app_id
        self.nonce = "".join(choice(digits + ascii_letters) for _ in range(10))

    @staticmethod
    def get_sign(secret: str, param: dict) -> str:
        """
        Создание уникальной подписи для каждого запроса.
            Вернет сгенерированную подпись из параметров запроса.
        :param secret:
        :param param:
        :return:
        """
        params_string = json.dumps(param).replace(' ', '')
        string = params_string + secret
        return md5(string.encode("utf-8")).hexdigest()

    @staticmethod
    def get_unique_hashcat():
        hashcat_one = uuid.uuid4()
        hashcat_two = uuid.uuid4()
        type_to_string = str(hashcat_one) + str(hashcat_two)
        hashcat = md5(type_to_string.encode("utf-8")).hexdigest()
        return hashcat

    def get_token(self):
        """
        Получение и восстановление токена.
            Вернёт новый токен если его нет, если есть то текущий токен.
        """
        try:
            param: Dict = {"app_id": self.app_id, "nonce": self.get_unique_hashcat()}
            sign = self.get_sign(self.secret_key, param)
            headers = {'Content-type': 'application/json; charset=utf-8',
                       'Accept': 'application/json', 'sign': sign}
            response = requests.get(url="https://check.business.ru/open-api/v1/Token/", params=param, headers=headers)
            data = response.json()
            token = json.dumps(data["token"])
            return token.strip('""')
        except Exception as error:
            print(error)

    def get_user(self):
        """
        Работа с пользователем / аккаунтом
            Вернет информацию о текущем пользователе / аккаунте
        """
        param: Dict = {"app_id": self.app_id, "nonce": self.nonce, "token": self.get_token()}
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/User/", params=param, headers=headers)
        data = response.json()
        print(data)

    def get_command_id(self, **kwargs):
        """
        Получение данных о команде в ФР по ID пользователя и ID команды
            Вернёт информацию о команде ФР
                Подробнее можно узнать на:

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_check

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_shift

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_cash

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_report

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_correction

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/Shifts

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/Checks

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/Goods
        """
        param: Dict = {"app_id": self.app_id, "nonce": self.nonce, "token": self.get_token()}
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/Command/{}".format(kwargs), params=param,
                                headers=headers)
        data = response.json()
        print(data)

    def get_command(self, filter_date_create_from, filter_date_create_to, filter_date_update_from,
                    filter_date_update_to, filter_date_result_from, filter_date_result_to, page, c_num):
        """
        Получение данных о команде в ФР по ID пользователя и ID команды
            Вернёт информацию о команде ФР с учётом фильтров
        """
        param: Dict = {"app_id": self.app_id,
                       "filter_date_create_from": filter_date_create_from,
                       "filter_date_create_to": filter_date_create_to,
                       "filter_date_update_from": filter_date_update_from,
                       "filter_date_update_to": filter_date_update_to,
                       "filter_date_result_from": filter_date_result_from,
                       "filter_date_result_to": filter_date_result_to,
                       "page": page,
                       "c_num": c_num,
                       "nonce": self.nonce,
                       "token": self.get_token(),
                       }

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': self.get_unique_sign()}
        response = requests.get(url="https://check.business.ru/open-api/v1/Command", params=param,
                                headers=headers)
        data = response.json()
        print(data)

    def get_state_system(self):
        """
        Получение данных о состоянии системы.
            Вернёт информацию о текущем состоянии системы.
        """
        param: Dict = {"app_id": self.app_id, "nonce": self.nonce, "token": self.get_token()}
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/StateSystem/", params=param, headers=headers)
        data = response.json()
        return data

    def get_shift(self, filter_date_create_from, filter_date_create_to, filter_date_update_from,
                  filter_date_update_to, filter_date_result_from, filter_date_result_to, page, c_num):
        """
        Получение данных о сменах.
            Вернёт информацию о сменах с учётом фильтров.
        """
        param: Dict = {"app_id": self.app_id,
                       "filter_date_create_from": filter_date_create_from,
                       "filter_date_create_to": filter_date_create_to,
                       "filter_date_update_from": filter_date_update_from,
                       "filter_date_update_to": filter_date_update_to,
                       "filter_date_result_from": filter_date_result_from,
                       "filter_date_result_to": filter_date_result_to,
                       "page": page,
                       "c_num": c_num,
                       "nonce": self.nonce,
                       "token": self.get_token(),
                       }
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/Shift/", params=param, headers=headers)
        data = response.json()
        return data


from src.neural_net import NeuralNet

keys = {
    'Iris-virginica': [1.0, 0.0, 0.0],
    'Iris-versicolor': [0.0, 1.0, 0.0],
    'Iris-setosa': [0.0, 0.0, 1.0]
}

def get_training_data():
    training_inputs = []
    training_expected = []
    test_inputs = []
    test_expected = []

    f = open('iris.data')
    lines = f.readlines()
    f.close()

    index = 0
    min_val = 1000.0
    max_val = 0.0
    input_data = []
    output_data = []

    for line in lines:
        parts = line.split(',')
        sepal_length = float(parts[0])
        sepal_width = float(parts[1])
        petal_length = float(parts[2])
        petal_width = float(parts[3])
        cls = parts[4].strip()

        min_val = min(min_val, sepal_length)
        min_val = min(min_val, sepal_width)
        min_val = min(min_val, petal_length)
        min_val = min(min_val, petal_width)

        max_val = max(max_val, sepal_length)
        max_val = max(max_val, sepal_width)
        max_val = max(max_val, petal_length)
        max_val = max(max_val, petal_width)

        input_data.append([sepal_length,
                           sepal_width,
                           petal_length,
                           petal_width])
        output_data.append(keys[cls])

    for datum in input_data:
        for i in range(len(datum)):
            datum[i] = (datum[i] - min_val) / (max_val - min_val)

    for inputs, output in zip(input_data, output_data):
        if index % 51 == 0:
            test_inputs.append(inputs)
            test_expected.append(output)
        else:
            training_inputs.append(inputs)
            training_expected.append(output)
        index += 1
    return training_inputs, training_expected, test_inputs, test_expected

def main():
    training_data = get_training_data()

    nn = NeuralNet([4, 4, 3])

    nn.train(training_data[0], training_data[1], 5000)

    for i, o in zip(training_data[2], training_data[3]):
        print(i)
        print(nn.predict(i), end='::')
        print(o)
        print()

if __name__ == '__main__':
    main()


from ast import Lambda

def sol(P, T):
    T.sort(reverse=True)
    P.sort()
    T_final = []
    k = 0
    for i in range(0,len(T),4):
        for j in T[i:i+4]:
            T_final.append(j+P[k])
        k+=1
    return max(T_final)

print(sol([8,10],[3,1,8,7,4,2,5,2]))

print(sol([5,11],[3,1,8,7,4,2,5,2]))


class Solution:
    def topKFrequent(self, nums: List[int], k: int) -> List[int]:
        #freq = defaultdict(int)
        #for num in nums:
            #freq[num] += 1
        freq = Counter(nums)
        arr = []
        for key in freq.keys():
            arr.append([key,freq[key]])

        arr.sort(key = lambda x : x[1])
        arr.reverse()
        ans = []
        for i in range(k):
            ans.append(arr[i][0])

        return ans


"""
81. RNNによる予測
ID番号で表現された単語列x=(x1,x2,…,xT)がある．ただし，Tは単語列の長さ，xt∈ℝVは単語のID番号のone-hot表記である（Vは単語の総数である）．
再帰型ニューラルネットワーク（RNN: Recurrent Neural Network）を用い，単語列xからカテゴリyを予測するモデルとして，次式を実装せよ．
"""
import tensorflow as tf

from tf_models import RNNModel
from utils import load_data, preprocess, build_vocabulary, text2sequence, filter_embeddings, seed_everything

if __name__ == '__main__':

    # データのロード
    dfs = load_data()

    # 前処理
    dfs['train']['clean_title'] = dfs['train'][['title']].apply(preprocess)
    dfs['valid']['clean_title'] = dfs['valid'][['title']].apply(preprocess)
    dfs['test']['clean_title'] = dfs['test'][['title']].apply(preprocess)

    # ボキャブラリの生成
    # 「出現頻度が2回未満の単語のID番号はすべて0とせよ．」は無視しています.（時間があったらstopword除外を入れます）
    vocab = build_vocabulary(dfs['train']['clean_title'])
    # 単語IDを確認する
    result = {k: vocab.word_index[k] for k in list(vocab.word_index)[:15]}
    print(result)
    """
    >>
    {'<UNK>': 1, 'update': 2, 'us': 3, 'new': 4, 'says': 5, 'stocks': 6, 'china': 7, 'kardashian': 8, 'euro': 9,
    'kim': 10, 'first': 11, 'may': 12, 'ecb': 13, 'shares': 14, 'fed': 15}
    """

    # 単語IDの列を取得（学習データ）
    X_train = text2sequence(dfs['train']['clean_title'], vocab)
    X_valid = text2sequence(dfs['valid']['clean_title'], vocab)
    X_test = text2sequence(dfs['test']['clean_title'], vocab)
    print(X_train.shape)
    print(X_valid.shape)
    print(X_test.shape)

    # 目的変数の生成
    category_dict = {'b': 0, 't': 1, 'e': 2, 'm': 3}
    y_train = dfs['train']['category'].map(category_dict)
    y_valid = dfs['valid']['category'].map(category_dict)
    y_test = dfs['test']['category'].map(category_dict)

    # モデルの生成
    seed_everything(42)
    tf.keras.backend.clear_session()
    model = RNNModel(len(vocab.word_index)+1, len(y_train.unique())).build()
    model.summary()
    """
    >>
    Model: "model_1"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    input (InputLayer)           [(None, None)]            0
    _________________________________________________________________
    embedding (Embedding)        (None, None, 300)         3733200
    _________________________________________________________________
    rnn (SimpleRNN)              (None, 100)               40100
    _________________________________________________________________
    dense_2 (Dense)              (None, 4)                 404
    =================================================================
    Total params: 3,773,704
    Trainable params: 3,773,704
    Non-trainable params: 0
    _________________________________________________________________
    """

    print(model(X_train[:4]))
    """
    >>
    tf.Tensor(
    [[0.26211333 0.20876467 0.18698843 0.34213355]
    [0.2443882  0.23337469 0.29404563 0.22819154]
    [0.31824696 0.18265769 0.19796237 0.30113298]
    [0.20688418 0.2832237  0.2216745  0.28821763]], shape=(4, 4), dtype=float32)
    """


import os
from googleapiclient.discovery import build
from dotenv import load_dotenv

load_dotenv()
api_key = os.environ.get('YOUTUBE_API_KEY')
print(api_key)
youtube = build('youtube', 'v3', developerKey=api_key)

request = youtube.channels().list(
    part='statistics',
    forUsername='schafer5'
)

response = request.execute()

print(response)


import pygame

# Inicializa Pygame
pygame.init()

# Inicializa el módulo joystick
pygame.joystick.init()

# Obtiene el número de joysticks
joystick_count = pygame.joystick.get_count()

# Si hay al menos un joystick
if joystick_count > 0:
    # Obtiene el primer joystick
    joystick = pygame.joystick.Joystick(0)
    joystick.init()

    # Bucle principal
    while True:
        # Procesa los eventos
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()

        # Obtiene los valores de la palanca
        axis_x = joystick.get_axis(0)  # Cambia el índice según la palanca que quieras leer
        axis_y = joystick.get_axis(1)  # Cambia el índice según la palanca que quieras leer

        # print(f"Valor de la palanca en X: {axis_x}, en Y: {axis_y}")
        print(f"Valor de la palanca en X: {axis_x}")


from __future__ import absolute_import

import logging
from pandas.tseries.offsets import BDay
from itertools import chain
from apache_beam.io.gcp.internal.clients import bigquery

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from datetime import datetime, date
from  .marketstats_utils import is_above_52wk,get_prices,MarketBreadthCombineFn, get_all_stocks, is_below_52wk,\
                            combine_movers,get_prices2, get_vix, ParseNonManufacturingPMI, get_all_us_stocks2,\
                            get_all_prices_for_date, InnerJoinerFn, create_bigquery_ppln,\
                            ParseManufacturingPMI,get_economic_calendar, get_equity_putcall_ratio,\
                            get_cftc_spfutures, create_bigquery_ppln_cftc, get_market_momentum, \
                            get_senate_disclosures, create_bigquery_manufpmi_bq, create_bigquery_nonmanuf_pmi_bq,\
                            get_sector_rotation_indicator, get_latest_fed_fund_rates

from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail, Email, Personalization
from functools import reduce

class MarketStatsCombineFn(beam.CombineFn):
    def create_accumulator(self):
        return []

    def add_input(self, sum_count, input_data):
        holder = sum_count
        logging.info('Adding:{}'.format(input_data))
        holder.append(input_data)
        return holder

    def merge_accumulators(self, accumulators):
        return chain(*accumulators)

    def extract_output(self, sum_count):
        all_data = sum_count
        sorted_els = sorted(all_data, key=lambda t: t[0])
        mapped = list(map (lambda tpl: tpl[1], sorted_els))

        stringified = list(map(lambda x: '{}-{}-{}'.format(x['AS_OF_DATE'],
                                                           x['LABEL'],
                                                           x['VALUE']), mapped))

        logging.info('MAPPED IS :{}'.format(mapped))
        return stringified

class MarketStatsSinkCombineFn(beam.CombineFn):
    def create_accumulator(self):
        return []

    def add_input(self, sum_count, input_data):
        holder = sum_count
        holder.append(input_data)
        return holder

    def merge_accumulators(self, accumulators):
        return chain(*accumulators)

    def extract_output(self, all_data):
        return [i for i in all_data]

class EmailSender(beam.DoFn):
    def __init__(self, recipients, key):
        self.recipients = recipients.split(',')
        self.key = key

    def _build_personalization(self, recipients):
        personalizations = []
        for recipient in recipients:
            logging.info('Adding personalization for {}'.format(recipient))
            person1 = Personalization()
            person1.add_to(Email(recipient))
            personalizations.append(person1)
        return personalizations

    def process(self, element):
        logging.info('Attepmting to send emamil to:{}, using key:{}'.format(self.recipients, self.key))
        template = "{}"
        content = template.format(element)
        print('Sending \n {}'.format(content))
        message = Mail(
            from_email='gcp_cloud_mm@outlook.com',
            to_emails=self.recipients,
            subject='Market Stats',
            html_content=content)

        personalizations = self._build_personalization(self.recipients)
        for pers in personalizations:
            message.add_personalization(pers)

        sg = SendGridAPIClient(self.key)

        response = sg.send(message)
        print(response.status_code, response.body, response.headers)

class XyzOptions(PipelineOptions):

    @classmethod
    def _add_argparse_args(cls, parser):
        parser.add_argument('--key')
        parser.add_argument('--sendgridkey')
        parser.add_argument('--recipients', default='mmistroni@gmail.com')

def run_pmi(p):
    return (p | 'startstart' >> beam.Create(['20210101'])
                    | 'pmi' >>   beam.ParDo(ParseNonManufacturingPMI())
                    | 'remap  pmi' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'NON-MANUFACTURING-PMI', 'VALUE' : d['Last']})
            )

def run_putcall_ratio(p):
    return (p | 'start putcall ratio' >> beam.Create(['20210101'])
            | 'putcall' >> beam.Map(lambda d: get_equity_putcall_ratio())
            | 'remap pcratio' >> beam.Map(
                lambda d: {'AS_OF_DATE': date.today().strftime('%Y-%m-%d'), 'LABEL': 'EQUITY_PUTCALL_RATIO', 'VALUE': str(d)})
            )

def run_manufacturing_pmi(p):
    return (p | 'startstartnpmi' >> beam.Create(['20210101'])
                    | 'manifpmi' >>   beam.ParDo(ParseManufacturingPMI())
                    | 'manufremap  pmi' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'MANUFACTURING-PMI', 'VALUE' : d['Last']})
            )

def run_economic_calendar(p, key):
    return (p | 'startcal' >> beam.Create(['20210101'])
                    | 'econcalendar' >>   beam.FlatMap(lambda d: get_economic_calendar(key))
                    | 'reMapping' >> beam.Map(lambda d: {'AS_OF_DATE' : d['date'],
                                                         'LABEL' : d['event'],
                                                         'VALUE' : f"Previous:{d['previous']},Estimate:{d['estimate']},Actual:{d.get('actual') or ''}"
                                                         }
                                              )

            )

def run_vix(p, key):
    return (p | 'start run_vix' >> beam.Create(['20210101'])
                    | 'vix' >>   beam.Map(lambda d:  get_vix(key))
                    | 'remap vix' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'VIX', 'VALUE' : str(d)})
            )

def run_senate_disclosures(p, key):
    return (p | 'start run_sd' >> beam.Create(['20210101'])
              | 'run sendisclos' >> beam.FlatMap(lambda d : get_senate_disclosures(key))
            )

def run_fed_fund_rates(p):
    return (p | 'start run_ffr' >> beam.Create(['20210101'])
              | 'run ffrates' >> beam.Map(lambda d : get_latest_fed_fund_rates())
              | 'remap fr' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'FED_FUND_RATES', 'VALUE' : str(d)})
            )

def run_market_momentum(p, key):
    return (p | 'start run_mm' >> beam.Create(['20210101'])
                    | 'mm' >>   beam.Map(lambda d:  get_market_momentum(key))
                    | 'remap mm' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'MARKET_MOMENTUM', 'VALUE' : str(d)})
            )

def run_growth_vs_value(p, key):
    return (p | 'start run_gv' >> beam.Create(['20210101'])
                    | 'gv' >>   beam.Map(lambda d: get_sector_rotation_indicator (key))
                    | 'remap mmgv' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'SECTOR ROTATION(GROWTH/VALUE)', 'VALUE' : str(d)})
            )

def run_cftc_spfutures(p, key):
    return (p | 'start_cftc' >> beam.Create(['20210101'])
                    | 'sptufutres' >>   beam.Map(lambda d:  get_cftc_spfutures(key))
                    | 'remap cftcspfutures' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'CFTC-SPFUTURES', 'VALUE' : str(d)})
            )

def run_exchange_pipeline(p, key, exchange):
    all_us_stocks = list(map(lambda t: (t, {}), get_all_us_stocks2(key, exchange)))
    asOfDate = (date.today() - BDay(1)).date()
    prevDate = (asOfDate - BDay(1)).date()

    dt = get_all_prices_for_date(key, asOfDate.strftime('%Y-%m-%d'))
    ydt = get_all_prices_for_date(key, prevDate.strftime('%Y-%m-%d'))

    filtered = [(d['symbol'], d)  for d in dt]
    y_filtered = [(d['symbol'], {'prevClose': d['close']}) for d in ydt]

    tmp = [tpl[0] for tpl in all_us_stocks]
    fallus = [tpl for tpl in filtered if tpl[0] in tmp]
    yfallus = [tpl for tpl in y_filtered if tpl[0] in tmp]

    pcoll1 = p | f'Create coll1={exchange}' >> beam.Create(all_us_stocks)
    pcoll2 = p | f'Create coll2={exchange}' >> beam.Create(fallus)
    pcoll3 = p | f'Crete ydaycoll={exchange}' >> beam.Create(yfallus)

    pcollStocks = pcoll2 | f'Joining y{exchange}' >> beam.ParDo(InnerJoinerFn(),
                                                     right_list=beam.pvalue.AsIter(pcoll3))

    return  (
                    pcoll1
                    | f'InnerJoiner: JoinValues {exchange}' >> beam.ParDo(InnerJoinerFn(),
                                                              right_list=beam.pvalue.AsIter(pcollStocks))
                    | f'Map to flat tpl {exchange}' >> beam.Map(lambda tpl: (tpl[0], tpl[1]['close'], tpl[1]['close'] - tpl[1]['prevClose']))
                    | f'Combine MarketBreadth Statistics {exchange}' >> beam.CombineGlobally(MarketBreadthCombineFn())
                    | f'mapping {exchange}' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'),
                                                        'LABEL' : '{}_{}'.format(exchange.upper(), d[0:d.find(':')]),
                                                       'VALUE' : d[d.rfind(':')+1:]})

            )

def run_prev_dates_statistics(p) :
    # Need to amend the query to order by asofdate sc
    nysebqp = ( create_bigquery_ppln(p, 'NEW YORK STOCK EXCHANGE_MARKET BREADTH')
               #| 'map to tpl2' >> beam.Map(lambda d: ( d['AS_OF_DATE'], d['LABEL'], d['VALUE'] ))
    )

    return nysebqp

def run_prev_dates_statistics_cftc(p):
    # Need to amend the query to order by asofdate sc
    cftc_ppln = (create_bigquery_ppln_cftc(p)
               )

    return cftc_ppln

def run_prev_dates_statistics_manuf_pmi(p):
    # Need to amend the query to order by asofdate sc
    pmi_ppln = (create_bigquery_manufpmi_bq(p)
               )

    return pmi_ppln

def run_prev_dates_statistics_non_manuf_pmi(p):
    # Need to amend the query to order by asofdate sc
    npmi_ppln = (create_bigquery_nonmanuf_pmi_bq(p)
               )

    return npmi_ppln

def run_prev_dates_statistics_cftc(p):
    # Need to amend the query to order by asofdate sc
    cftc_ppln = (create_bigquery_ppln_cftc(p)
               )

    return cftc_ppln

def write_all_to_sink(results_to_write, sink):
    to_tpl = tuple(results_to_write)
    return (
            to_tpl
            | 'FlattenCombine all sink' >> beam.Flatten()
            | 'write all to sink' >> sink

    )

def run(argv=None, save_main_session=True):
    """Main entry point; defines and runs the wordcount pipeline."""

    # We use the save_main_session option because one or more DoFn's in this
    # workflow rely on global context (e.g., a module imported at module level).

    # Check  this https://medium.datadriveninvestor.com/markets-is-a-correction-coming-aa609fba3e34

    pipeline_options = XyzOptions()
    pipeline_options.view_as(SetupOptions).save_main_session = save_main_session
    with beam.Pipeline(options=pipeline_options) as p:

        iexapi_key = pipeline_options.key
        logging.info(pipeline_options.get_all_options())
        current_dt = datetime.now().strftime('%Y%m%d-%H%M')

        destination = 'gs://mm_dataflow_bucket/outputs/shareloader/{}_run_{}.csv'

        logging.info('====== Destination is :{}'.format(destination))
        logging.info('SendgridKey=={}'.format(pipeline_options.sendgridkey))

        bq_sink = beam.io.WriteToBigQuery(
            bigquery.TableReference(
                projectId="datascience-projects",
                datasetId='gcp_shareloader',
                tableId='market_stats'),
            schema='AS_OF_DATE:STRING,LABEL:STRING,VALUE:STRING',
            write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
            create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED)

        run_weekday = date.today().weekday()

        logging.info('Run pmi')

        pmi_res = run_pmi(p)
        manuf_pmi_res = run_manufacturing_pmi(p)

        if date.today().weekday() == 2 and date.today().day() < 15:
            # We need to store it only once a month
            logging.info(*'Running Market Stats')
            pmi_res | 'WritinNG PMI TO SINK' >> bq_sink
            manuf_pmi_res | 'writing non manuf pmi to sink' >> bq_sink

        if run_weekday == 5:
            logging.info(f'Weekday for rundate is {run_weekday}')
            cftc = run_cftc_spfutures(p, iexapi_key)
            cftc | 'cftc to sink' >> bq_sink

        vix_res = run_vix(p, iexapi_key)

        mmomentum_res = run_market_momentum(p, iexapi_key)

        growth_vs_val_res = run_growth_vs_value(p, iexapi_key)

        senate_disc = run_senate_disclosures(p, iexapi_key)

        logging.info('Run NYSE..')
        nyse = run_exchange_pipeline(p, iexapi_key, "New York Stock Exchange")
        logging.info('Run Nasdaq..')
        nasdaq = run_exchange_pipeline(p, iexapi_key, "NASDAQ Global Select")

        equity_pcratio = run_putcall_ratio(p)

        fed_funds = run_fed_fund_rates(p)

        econ_calendar = run_economic_calendar(p, iexapi_key)

        staticStart = (p | 'Create static start' >> beam.Create(
            [dict(AS_OF_DATE='------- ', LABEL=' THIS WEEK ECONOMIC CALENDAR', VALUE='--------')])
                   )

        static1 = (p |'Create static1' >>  beam.Create([dict(AS_OF_DATE='------- ', LABEL=' TODAYS PERFORMANCE', VALUE='--------')])
                 )

        static = (p | 'Create static 2' >> beam.Create([dict(AS_OF_DATE='------- ', LABEL=' LAST 5 DAYS PERFORMANCE', VALUE='--------')])
                 )
        statistics = run_prev_dates_statistics(p)

        cftc_historical = run_prev_dates_statistics_cftc(p)

        pmi_hist = run_prev_dates_statistics_manuf_pmi(p)

        non_pmi_hist = run_prev_dates_statistics_non_manuf_pmi(p)

        staticStart_key = staticStart | 'Add -2' >> beam.Map(lambda d: (-2, d))
        econCalendarKey = econ_calendar | 'Add -1' >> beam.Map(lambda d: (-1, d))
        static1_key = static1 | 'Add 0' >> beam.Map(lambda d: (0, d))
        pmi_key = pmi_res | 'Add 1' >> beam.Map(lambda d: (1, d))
        manuf_pmi_key = manuf_pmi_res | 'Add 2' >> beam.Map(lambda d: (2, d))

        vix_key = vix_res | 'Add 3' >> beam.Map(lambda d: (3, d))
        nyse_key = nyse | 'Add 4' >> beam.Map(lambda d: (4, d))
        nasdaq_key = nasdaq | 'Add 5' >> beam.Map(lambda d: (5, d))
        epcratio_key = equity_pcratio | 'Add 6' >> beam.Map(lambda d: (6, d))
        mm_key = mmomentum_res | 'Add mm' >> beam.Map(lambda d: (7, d))
        sd_key = senate_disc | 'Add sd' >> beam.Map(lambda d: (8, d))
        growth_vs_val_key = growth_vs_val_res | 'Add 14' >> beam.Map(lambda d: (9, d))
        fed_funds_key = fed_funds | 'Add ff' >> beam.Map(lambda d: (10, d))

        static_key = static | 'Add 10' >> beam.Map(lambda d: (15, d))
        stats_key = statistics | 'Add 11' >> beam.Map(lambda d: (16, d))
        cftc_key = cftc_historical | 'Add 12' >> beam.Map(lambda d: (17, d))

        # we need a global combiner to write to sink

        pmi_hist_key = pmi_hist | 'Add 20' >> beam.Map(lambda d: (20, d))

        non_manuf_pmi_hist_key = non_pmi_hist | 'Add 30' >> beam.Map(lambda d: (30, d))

        final = (
                (staticStart_key, econCalendarKey, static1_key, pmi_key,
                    manuf_pmi_key, nyse_key, nasdaq_key,  epcratio_key, mm_key, cftc_key,  vix_key, sd_key, growth_vs_val_key,
                        fed_funds_key,
                        static_key, stats_key,
                        pmi_hist_key, non_manuf_pmi_hist_key,
                        )
                | 'FlattenCombine all' >> beam.Flatten()
                | ' do A PARDO combner:' >> beam.CombineGlobally(MarketStatsCombineFn())
                | ' FlatMapping' >> beam.FlatMap(lambda x: x)
                | 'Combine' >> beam.CombineGlobally(lambda x: '

'.join(x))
                | 'SendEmail' >> beam.ParDo(EmailSender(pipeline_options.recipients, pipeline_options.sendgridkey))

        )

        final_sink_results = [
                 vix_res, mmomentum_res, growth_vs_val_res, nyse,
                 nasdaq, equity_pcratio, fed_funds,
                ]

        # Writing everything to sink
        logging.info('Writing all to sink')

        destination = 'gs://mm_dataflow_bucket/outputs/marketstats_{}'.format(
            date.today().strftime('%Y-%m-%d %H:%M'))

        tmp_sink = beam.io.WriteToText(destination, num_shards=1)

        write_all_to_sink(final_sink_results, bq_sink)

if __name__ == '__main__':
    logging.getLogger().setLevel(logging.INFO)
    run()


import turtle as t
# sigil generator.
# input the phrase you are making the sigil for and it will give you a sigil

# initialize variables
STRING_SET = 'swkcgzpmqjfxldrnthvb'
COORDINATES = [(-150, 200), (-50, 200), (50, 200), (150, 200),
               (-150, 100), (-50, 100), (50, 100), (150, 100),
               (-150, 00), (-50, 00), (50, 00), (150, 00),
               (-150, -100), (-50, -100), (50, -100), (150, -100),
               (-150, -200), (-50, -200), (50, -200), (150, -200)]
win = t.Screen()
sheldon = t.Turtle()
sheldon.penup()

# make a turtle draw the list of coordinates and display the image
def draw_sigil():
    string = ''
    coords = []

    # collect input
    print("Please type what you would like to turn into a sigil.")
    sigil = input('>>> ')
    sigil.lower()

    # process the input
    for c in sigil:
        if (c in STRING_SET) and (c not in string):
            string += c

    # switch the characters to designated coordinates
    for c in string:
        coords.append(COORDINATES[STRING_SET.find(c)])

    # start drawing
    sheldon.goto(coords[0])
    sheldon.pendown()
    for i in range(1, len(coords)):
        sheldon.goto(coords[i])
    t.done()

# main loop
def main():
    draw_sigil()

if __name__ == '__main__':
    main()


# importing all the necessary libraries
import numpy as np
import scipy as sp

# Data handling
import pandas as pd

# Date handling
import datetime as dt

# Graphics
import matplotlib.pyplot as plt

# Processing the data so that we can get annual traces
def getYearAndDoY( dateStr ):
    # Convert the date string to a Timestamp object
    ts = pd.to_datetime( dateStr, dayfirst=True )

    # Extract the desired data
    return( [ts.year, ts.day_of_year] )

# Test the function: result should be [1960, 31+16=47]
getYearAndDoY( "16/2/1960" )

list_of_pairs = medlock_df["Date"].apply(getYearAndDoY).to_list()

# Convert the list of pairs into a pair of lists.
# See: https://stackoverflow.com/questions/21444338/transpose-nested-list-in-python
pair_of_lists = np.array( list_of_pairs ).transpose().tolist()

# Add the new columns
medlock_df = medlock_df.assign( Year=pair_of_lists[0], DayOfYear=pair_of_lists[1] )

# Look at the result
medlock_df.tail() # shows bottom 5 rows

# Making a numpy array each of which hold's a years worth of data
# Work out how many complete years we have
firstYear = 1974
lastYear = 2019 # data for 2020 stops at the end of September
nYears = lastYear - firstYear + 1

# Build the matrix and fill it
annualFlowSeriesMat = np.zeros( (365, nYears) )
for j in range(nYears):
    crntYear = firstYear + j
    annualFlowSeriesMat[:,j] = medlock_df[(medlock_df["Year"] == crntYear) & (medlock_df["DayOfYear"] < 366)]["Flow"]

# Look at the result
annualFlowSeriesMat.shape


# Given a string, return a new string made of 3 copies of the last 2 chars of the original string.
# The string length will be at least 2.

def extra_end(word):
    last_chars = word if len(word) < 2 else word[-2:]
    print(f"extra_end('{word}') -> {last_chars * 3}")

extra_end("Hello")
extra_end("ab")
extra_end("Hi")


'''
Pands project for Programming and Scripting
analysis.py
Author: Nur Bujang

To write a program called analysis.py that:
    1. Outputs a summary of each variable to a single text file,
    2. Saves a histogram of each variable to png files,
    3. Outputs a scatter plot of each pair of variables.
    4. Perform any other analysis I think is appropriate

Save iris.data file from UCI website and upload into pands-project folder
General Process: Load data, Analyze/visualize dataset, Model training, Model Evaluation, Model Testing,
'''

'''
IMPORT MODULES
'''
import numpy as np # for computational operations
import pandas as pd # for data loading from other sources and processing
import seaborn as sns # for data visualization
import matplotlib.pyplot as plt # for data visualization
from sklearn.model_selection import train_test_split # to split data
from sklearn.neighbors import KNeighborsClassifier # import kNN Classifier from sklearn
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, classification_report, f1_score, accuracy_score # import sklearn module to print confusion matrix and report
from sklearn.linear_model import LogisticRegression # import Logistic Regression from sklearn
from sklearn.tree import DecisionTreeClassifier # import Decision Tree Classifier from sklearn
from sklearn.svm import SVC # import Support Vector Machine Classification from sklearn
from sklearn.ensemble import RandomForestClassifier # import Random Forest Classifier from sklearn
from sklearn.naive_bayes import GaussianNB # import Gaussian Naïve Bayes from sklearn

'''
LOAD DATA AND ADD COLUMN HEADER
'''
columns = ['Sepal length (cm)', 'Sepal width (cm)', 'Petal length (cm)', 'Petal width (cm)', 'Iris species'] # instantiate a list of column names
# instantiate a pandas dataframe object called df, read the csv file with the read_csv funtion and assign a name to each column
df = pd.read_csv('iris.data', names=columns) # df contains iris.data and add column names to the dataframe
print('\nThe first 2 lines of the dataset\n',df.head(2))  # print out first 2 lines to see if the column names were added properly
# Output: Column names were added properly

'''
PRE-PROCESSING - Quick lookover on the dataset, check for missing values, duplicates
'''
# QUICK LOOKOVER OF THE DATASET
df.value_counts("Iris species")  # value_counts funtion to see how many lines for each species
print('\nThe number of rows for each Iris species\n',df.value_counts("Iris species"))  # print how many lines for each species
# Output: 50 lines for each species

# BASIC INFO ABOUT THE DATASET, COLUMN NUMBERS, DATA TYPES, NON-NULL VALUES
# can also be used to see of there are missing values from the number of non-null values
df.info()  # method to print out in terminal output the basic information about the dataframe
# print(df.info()) # commented out because I don't want to print it twice
# Output:
# It is a dataframe
# 150 rows, 0 to 149, 5 data columns and column names
# number of non-null values (data that is not missing) in each column
# 4 float datatypes and 1 object

# GET NUMBER OF MISSING VALUES IN EACH COLUMN
df.isnull().sum() # method to get sum of null values in the dataset
print ('\nThe number of missing value\n',df.isnull().sum()) # print out number of missing values
# OR
# df.isna().sum() # sum of na values
# print(df.isna().sum()) # print out number of missing values

# REMOVE DUPLICATES
df.drop_duplicates() # method to remove duplicates from the dataset
print('\nData shape after duplicate removal is',df.drop_duplicates().shape)  # print out the shape of the dataset after duplicate removal
# output: 147 rows, 5 columns, meaning 3 rows were removed

'''
Question 1. SUMMARY into text file, containing basic statistical analysis
df. describe to get count, mean, standard deviation, min and max values, lower, mid and upper percentile
'''
df.describe()  # describe method to get basic statistical analysis data
# print (df.describe()) # to print out description of data in terminal output, but I do not want this
# What I want is to export data into a .txt file called summary.txt
# use the built-in open() function
text_file = open("summary.txt", "wt") # with open function, open a new file called summary, mode is w, and t is text mode and call it text_file
# then, export into text file called summary, convert pandas dataframe to string
n = text_file.write(df.describe().to_string()) # with write method, write the converted df.describe() string into a text file, and call it n
text_file.close()  # ALWAYS close file when done

'''
Question 2. DATA VISUALISATION: Histogram of each variable into png file
'''
sns.set(style="whitegrid")  # set background of seaborn histogram to whitegrid
fig, axs = plt.subplots(2, 2, figsize=(7, 8)) # with subplots function, create a figure containing subplots with multiple axes
# set subplot arrangement/grid position in 2 directions (2D grid) (2 down by 2 across)
# set size of whole figure (7 width, 8 height))
sns.histplot(data=df, x="Sepal length (cm)", # x-axis is sepal length
             kde=True, color="olive", ax=axs[0, 0]) # ax lists the subplot location at first row, first column
# Where:
# seaborn histplot function was used to plot the histogram
# dataset used was df
# determine x-axis label
# display KDE line : Kernel density estimation (KDE) shows the data using a continuous curve
# KDE creates data 'smoothing' of the density distribution with a Gaussian kernel
# set the histogram bar color for this plot to olive
# ax list contains the coordinate/grid location of each subplot on the figure
sns.histplot(data=df, x="Sepal width (cm)", kde=True, color="green", # x-axis is sepal width, histogram bar color is green
             ax=axs[0, 1])  # list of subplot location at first row, second column
sns.histplot(data=df, x="Petal length (cm)", # x-axis is petal length, histogram bar color is blue
             kde=True, color="blue", ax=axs[1, 0]) # list of subplot location at second row, first column
sns.histplot(data=df, x="Petal width (cm)", # x-axis is petal width, histogram bar color is purple
             kde=True, color="purple", ax=axs[1, 1]) # list of subplot location at second row, second column
fig.suptitle('Histogram of petal and sepal dimensions of three Iris species',
             fontweight='bold', size=14)  # function to customize figure's super title, default font color is black, fontweight is bold, fontsize is 14
fig.tight_layout()  # method to fit all subplots into one figure nicely automatically
plt.savefig('iris.png')  # function to save output into png file
plt.show()  # function to show plot

'''
Question 3. DATA VISUALISATION: Scatter plot of each pair of variables
'''
# df contains 3 classes (setosa, virginicus, versicolor) and 50 replicates each
# within that, the variables are Petal length, Petal width, Sepal length and Sepal width
# Perform a Scatter Plot matrix (aka Pair Plot) to see the relationship between a pair of variables within a combination of multiple variables
# to visualize the whole dataset, using 'Iris species' variable to assign different color to different species
pp = sns.pairplot(df, hue='Iris species', markers=[ # instantiate a seaborn pairplot function called pp
                  "o", "s", "D"], palette='brg', kind='reg', plot_kws={'line_kws': {'color': 'blue'}})
# Where:
# dataset used was df
# seaborn pairplot was used to plot the Scatter Plot
# The diagonal plots represent each of that column variable's data distribution
# hue colors the plot based on the different Iris species value
# marker o is circle, s is square, D is diamond
# palette is set to brg palette
# kind is a chart type, kind='reg' applies a linear regression line to identify the relationship within the scatter plot
# plot_kws and pass in a dictionary object to customize the regression fit and line to blue
plt.suptitle('Scatter Plot for sepal and petal attributes of three Iris species', color='#3D3F38',
             fontweight='bold', size=17) # function to customize figure's super title, font color is hex number 3D3F38, fontweight is bold, fontsize 17
pp.fig.subplots_adjust(top=0.92, bottom=0.08)
# fig.subplots_adjust function shifts the pairplot position, 0.92 of the default 0.9 for the top edge and 0.08 of the default 0.1 for the bottom edge
# this had to be done to show the super title properly
plt.show()  # function to show plot
# Results:
# I. setosa is distinctly different and forms a separate cluster from I. virginica and I. versicolor, which shows some pairwise relationship between these two
# The petal length and width of I. setosa have much narrower distribution compared to the other two species.
# there are overlaps in sepal length and width of all three species.

'''
Question 4. OTHER ANALYSIS:
Data visualization to detect outliers, trends/pattern and Basic Machine Learning analysis
'''

'''
4.1 Pearson correlation analysis by attributes
to determine the degree/strength of linear relationship between 2 continuous variables
correlation efficient closer to 1 indicates a strong +ve relationship, closer to -1 indicates a strong -ve relationship
'''
corr = df.corr(method="pearson")  # instantiate a corr method and call it corr
# default is already Pearson Correlation (for linear), but can be changed to Kendall and Spearman for non-parametric. eg: method="Spearman"
bool_upper_matrix = np.tril(np.ones(corr.shape)).astype(bool)  # eliminate upper triangle for better readibility
# Numpy tril function to extract lower triangle or triu to extract upper triangle
# np.ones returns an array of 1 to create a boolean matrix with the same size as the correlation matrix
# astype converts the upper triangle values to False, while the lower triangle will have the True values
corr = corr.where(bool_upper_matrix) # Pandas where() returns same-sized dataframe, but False is converted to NaN on the upper triangle
print('\nPearson Correlation by attributes\n',corr)  # print as terminal output
# Results:
# High positive correlation between Petal length & Petal width, Petal length & Sepal length and Sepal length and petal width

# OR,

# build a Correlation matrix to visualize the parameters which best correlate with each other easier
# set size of whole figure (9 width, 6 height)
fig, ax = plt.subplots(figsize=(9, 6)) # with subplots function, create 1 figure and a single axes and set the figure size to be width=9 and height=6
fig.suptitle('Correlation matrix for petal and sepal attributes of three Iris species',
             color='#3D3F38', fontweight='bold')  # customize figure's main/super title, color is hex number 3D3F38, fontweight bold
hm = sns.heatmap(corr, annot=True, ax=ax, cmap='coolwarm', square=True, linewidths=0.1, # using heatmap function, create a seaborn heatmap called hm
                linecolor='yellow', cbar_kws={'label': 'Range', 'shrink': 0.9})  # customize heatmap
# Where:
# annot=True: if True, the data value in each cell will be displayed
# coolwarm color palette for the heatmap
# square = True: cell is square-shaped
# linewidth is the width of lines separating each cell and the color is yellow
# I passed arguments into the color bar to show Range as label and shrank it to 0.9 times the original size
# default color bar is vertical, but to move it to the bottom, just add 'orientation': 'horizontal' to cbar argument
hm.set_xticklabels(hm.get_xticklabels(), rotation=0, fontsize=10)
# set the label for x-axis to follow each column name
# rotation sets the xticks "upright" as opposed to sideways in any figure size, just to read easier
hm.set_yticklabels(hm.get_yticklabels(), rotation=0, fontsize=10)
# set the label for y axis to follow each column name
# rotation sets the yticks "upright" in any figure size
plt.show()  # function to show plot
# Results:
# Strong positive correlation between Petal length & Petal width, Petal length & Sepal length, Sepal length & Petal width (same as above)

'''
4.2 Pearson correlation analysis by species
If I group them by species, I will get more insights on which attributes are highly correlated for each Iris species:
'''
df.groupby("Iris species").corr(method="pearson") # Pearson Correlation, but grouped by class
print('\nPearson Correlation by species\n',df.groupby("Iris species").corr(method="pearson")) # print as terminal output
# Results:
# Iris setosa: high correlation between Sepal length & Sepal width
# Iris versicolor: strong correlation between Petal length & Petal width, Petal length & Sepal length
# Iris virginica: high correlation between Petal length & Sepal length

'''
4.3 DATA VISUALIZATION: Box plot
to display data point distribution/spread, skewness, variance and outliers
shows the minimum, first quartile, median, third quartile and maximum
'''
def graph(y):  # define a graph function of y axis
    sns.boxplot(x="Iris species", y=y, data=df) # seaborn boxplot, with Iris species on x-axis
# on x-axis is Iris species, on y-axis is y (attributes) and the data used is the iris dataframe
    sns.stripplot(x="Iris species", y=y, data=df, # function to add a seaborn stripplot/jitter plot over the boxplot
                  jitter=True, linewidth=1, edgecolor="red", alpha=0.35)
# Where:
# on x-axis is Iris species, on y-axis is y (attributes) and the data used is the iris dataframe
# jitter=True will display the dots (jitter)
# linewidth is width of line around the dots, it has to be set to >0 to be visible
# the edgecolor of the jitter/dots is red to make the dots look clearer
# set transparency (alpha) to 0.35 so it is not too opaque
plt.figure(figsize=(10, 10))
plt.subplot(221)  # function to set grid position to top left (2 rows, 2 columns, first top)
graph('Sepal length (cm)') # y-axis is Sepal length (cm)
plt.subplot(222)  # function to set grid position to top right (2 rows, 2 columns, second top)
graph('Sepal width (cm)') # y-axis is Sepal width (cm)
plt.subplot(223)  # function to set grid position to bottom left (2 rows, 2 columns, first bottom)
graph('Petal length (cm)') # y-axis is Petal length (cm)
plt.subplot(224) # function to set grid position to bottom right (2 rows, 2 columns, second bottom)
graph('Petal width (cm)') # y-axis is Petal width (cm)
plt.suptitle('Box Plot for sepal and petal attributes of three Iris species', color='#3D3F38',
             fontweight='bold', size=17) # function to customize the figure's super title, color is hex number 3D3F38, fontweight bold, font size 17
plt.show()  # function to show plot
# Results:
# Iris setosa has the least distributed and smallest petal size
# Iris virginica has the biggest petal size, and Iris versicolor's petal size is between Iris setosa and virginica
# Sepal size may not be a good variable to differentiate species

'''
4.4 DATA VISUALIZATION: Violin Plot
to give more insight into data distribution and density on the y-axis
it contains all data points, unlike the box plot which shows minimum, first quartile, median, third quartile and maximum and error bars
'''
fig, axs = plt.subplots(1, len(columns)-1, figsize=(20,5)) # using subplot function, create a figure containing subplots with multiple axes
# plot the subplots in 1 row of 4 subplots (stacked in 1 direction only, but still use axs because each has their own axes)
# length -1 means all columns (5) -1 = 4 columns
# set the figure size to 20 width and 5 height
for i in range(0,len(columns)-1):
    sns.violinplot(x='Iris species', y=df[columns[i]], data=df, ax=axs[i])
    axs[i].set_ylabel(columns[i])
# Where:
# for i in range can be translated to: for item in columns 1 to 4
# seaborn violinplot, x-axis is Iris species, y is each column name, data is iris dataframe
# ax is the object to draw the plot into, in this case, the columns
# axs[i].set_ylabel(columns[i]) means each column list i is set as the y-axis label
plt.suptitle('Violin Plot for sepal and petal attributes of three Iris species', color='#3D3F38',
             fontweight='bold', size=15) # function to customize the figure's super title, color is hex number 3D3F38, fontweight bold, font size 15
plt.show() # function to show plot

'''
*************DATA PREPARATION FOR BASIC MACHINE LEARNING***************

SPLITTING THE DATA FOR TRAINING AND TESTING
First, split the data into training (80%) and testing (20%) to detect overfitting (=model learned the training data very well but fails on testing)
Later, the testing dataset will be used to check the accuracy of the model.

While most ML examples available uses only 2 variables (sepal length and width only or petal length and width only) to simplify analysis:
X = df.iloc[:,:2] # take everything until the second column (columns 0 and 1) and store the first two columns (Sepal length and Sepal width) into attributes (X)
y = df.iloc[:,4] # store the target variable (Iris species) into labels (y)
I used all 4 variables because the best determinants are still unknown at this point (whether sepal or petal attribute is better than the other).
'''

X = df.iloc[:, :-1].values # X dataset is everything up until the last column but not including the last column (Iris species)
y = df.iloc[:, 4].values # = [:, -1],  y dataset gets all the rows in the last/5th column (target variable (Iris species) into labels (y))
print('\nDataset shape before Train-Test split is',X.shape, y.shape)
# print out the shape (number of rows and columns) AFTER the data split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42) # split the dataset into training (80%) and testing (20%)
# test size is 20%, meaning training size is 80%
# random_state is the seed of randomness to help get the same results everytime. It can be any number, really.
print('\nDataset shape after Train-test split is',X_train.shape, X_test.shape, y_train.shape, y_test.shape)
# print out the shape (number of rows and columns) of test and train data AFTER split in terminal output

'''
4.5 kNN Classifier
used for regression and classification
kNN calculates the distance between the data points and predict the correct species class for the datapoint
where k = number of neighbors/points closest to the test data
'''
for i in np.arange(7, 10): # for i in numpy arange starting at 7 and stopping at 10
# keep k small because there are only 3 species, to prevent overfitting
    knn = KNeighborsClassifier(n_neighbors=i) # instantiate a class and name it knn
    # number of neighbors range from 7-10
    knn.fit(X_train, y_train) # fit method fits the model to the training set
    y_pred_knn = knn.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_knn
    print("\nk-Nearest Neighbor model accuracy for k = %d accuracy is" % i, knn.score(X_test,
          y_test)*100)  # Calculate and print the accuracy of the model using testing dataset ranging (from 7-10)*100
# %d is a placeholder for a number, %s is for string
# % on its own means the values of each i in 7-10 range are then passed in through a tuple using the % operator
    print("k-Nearest Neighbor F1 score for k = %d F1 score is" % i, f1_score(y_test, y_pred_knn, average='micro'))
# Calculate and print the F1 score the model using testing dataset ranging (from 7-10)*100
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive
# Other averaging methods are macro (meaning it is calculated using regular/unweighted mean) and weighted (takes into account each class's support)
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# F1 combines precision (% of correct predictions) and recall (proportion of correct predictions over total occurences)

'''
4.6 Logistic Regression
To estimate the relationship between 1 dependent variable and 1 or more independent variables
Used for classification and prediction analysis
uses Sigmoid function (logistic function) instead of linear function in Linear regression
'''
lr = LogisticRegression(C=0.5, random_state=42)  # instantiate a class and name it lr
# C value is a model hyperparameter, which is a model criteria outside of the model and its value cannot be estimated from the data
# default C is 1.0
# High C value means training data is more reliable (reflects real world data), low C value means training data may not reflect real world data
# I picked 0.5 to be somewhere in the middle
# random_state is the seed of randomness to help get the same results everytime. It can be any number
lr.fit(X_train, y_train)  # fit method fits the model to the training set
# compare model’s output (y_pred) with target values that we already have (y_test)
y_pred_lr = lr.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_lr
print('\nLogistic Regression model accuracy is',
      accuracy_score(y_test, y_pred_lr) * 100) # print accuracy score * 100
# Model accuracy score is how many times the model makes correct predictions over the total number of predictions
print('Logistic Regression model F1 score is',
      f1_score(y_test, y_pred_lr, average='micro')) # print out F1 score
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive
'''
4.7 Decision Tree Classifier
to build a classification or regression
branches bifurcate based on Y/N or T/F and breaks the dataset smaller everytime
to eventually form a tree structure with decision nodes and leaf nodes
'''
dtclassifier = DecisionTreeClassifier(random_state=42) # instantiate a class and name it dtclassifier
dtclassifier.fit(X_train, y_train)  # fit method fits the model to the training set
# Predict the response for test dataset
y_pred_dt = dtclassifier.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_dt
# Evaluate the model using testing dataset
dispdt = ConfusionMatrixDisplay.from_estimator(dtclassifier, X_test, y_test) # instantiate a ConfusionMatrixDisplay class and from_estimator method and call it dispdt
# Confusion matrix contains Actual values of Positive(1) and Negative(0) on the x-axis and Predicted values of Positive(1) and Negative(0) on the y-axis
# CM for iris contains Actual setosa, versicolor, virginica on the x-axis and Predicted setosa, versicolor, virginica on the y-axis
# True Positive = the actual and predicted value should be the same = 10
# True Negative = 9+0+0+11 = 20
# False Positive = 0+0 (across) = 0
# False Negative = 0+0 (down) = 0
plt.suptitle('Decision Tree Confusion Matrix for sepal and petal \nattributes of three Iris species',
             color='#3D3F38', fontweight='bold', size=13) # function to customize super title, font color is hex number 3D3F38, fontweight bold, font size 13
plt.grid(False) # function to eliminate white grid within the confusion matrix
plt.show() # function to display Confusion Matrix
print('\nDecision Tree Classification model accuracy is',
      accuracy_score(y_test, y_pred_dt)*100)  # print out accuracy score *100
print('Decision Tree Classification model F1 score is',
      f1_score(y_test, y_pred_dt, average='micro')) # print out F1 score
# Precision = True Positive : (True Positive + False Positive) = 10/(10+0+0 (across)) = 1
# Recall = True positive : (True Positive + False Negative) = 10/(10+0+0 (down)) = 1
# F1 score = 2*((precision*recall)/(precision+recall)) = 2(1/2) = 1
# that is, F1 = how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive

'''
4.8 Support Vector Machine Classifier
for regression and classification
to map data points into a high dimensional space
and then create the best boundary to separate data into classes by creating a hyperplane line with the most margin from the data point
'''
svclassifier = SVC() # instantiate a class and name it svclassifier
svclassifier.fit(X_train, y_train) # fit method fits the model to the training set
y_pred_svc = svclassifier.predict(X_test)  # instantiate a predict method using the test dataset and call it y_pred_svc
print('\nClassification report for Support Vector Classifier\n',classification_report(y_test, y_pred_svc)) # print out classification report in terminal output
# Classification report results:
# Precision = Ratio of True Positive : (True Positive + False Positive. So, setosa (10/10=1), versicolor (9/9=1), virginica (11/11=1)
# Recall = Ratio of True positive : (True Positive + False Negative)
# F1 score = 2*((precision*recall)/(precision+recall)) = 2(1/2) = 1
# Support = number of actual occurences of that class
# Accuracy = correct predictions for all classes / total number of predictions = 10/10 = 1
print('Confusion matrix for Support Vector Classifier\n',confusion_matrix(y_test, y_pred_svc)) # print out Confusion matrix in terminal output
print('\nSupport Vector Classifier model accuracy is', accuracy_score(
    y_test, y_pred_svc)*100)  # print out accuracy score *100
print('Support Vector Classifier model F1 score is',
      f1_score(y_test, y_pred_svc, average='micro')) # print out F1 score
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive

'''
4.9 Random Forest Classifier
used to perform both regression and classification
to create a cluster of decision trees containing different sub-features from the features, thus forming a 'forest'
each bunch is trained on random subsets from training group (drawn with replacement and can be used again) and features (drawn without replacement and cannot be reused)
each tree picks the features randomly, making it possible to find which features are more important than others
'''
rf = RandomForestClassifier(n_estimators=10, n_jobs=-1) # instantiate a class and name it rf
rf.fit(X_train, y_train)  # fit method fits the model to the training set
# compare model’s output (y_pred) with the target values that we already have (y_test)
y_pred_rf = rf.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_rf
# Random Forest visualization using confusion matrix
disprf= ConfusionMatrixDisplay.from_estimator(rf, X_test, y_test) # instantiate a ConfusionMatrixDisplay class and from_estimator method and call it disprf
plt.suptitle('Random Forest Confusion Matrix for sepal and petal \nattributes of three Iris species',
             color='#3D3F38', fontweight='bold', size=13) # function to customize super title, font color is hex number 3D3F38, fontweight bold, font size 13
plt.grid(False) # function to eliminate white grid within the confusion matrix
plt.show() # function to show plot
print('\nRandom Forest model accuracy score is',
      accuracy_score(y_test, y_pred_rf)*100) # print accuracy score * 100
print('Random Forest model F1 score is', f1_score(
    y_test, y_pred_rf, average='micro')) # print out F1 score
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive

'''
4.10 Gaussian Naïve Bayes Classifier
'Naive' because it assumes that each variable are independent of each other
it predicts the probability of different species based on different attributes
I used Gaussian because data is continuous, and assumed to be normally-distributed
'''
gaussian = GaussianNB() # instantiate a class and name it gaussian
gaussian.fit(X_train, y_train) # fit method fits the model to the training set
y_pred_gs = gaussian.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_gs
cm = confusion_matrix(y_test, y_pred_gs) # instantiate confusion matrix function and call it cm
accuracy = accuracy_score(y_test,y_pred_gs)*100 # instantiate accuracy score function and call it accuracy
# multiply by 100 here because it is too complicated to do so in the print format
f1 = f1_score(y_test,y_pred_gs,average='micro') # instantiate an f1 score function and name it f1
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive
print('\nClassification report for Naive-Bayes Classifier\n',classification_report(y_test, y_pred_gs)) # print Classification Report in terminal output
print('\nConfusion matrix for Naive Bayes\n',cm) # print confusion matrix in terminal output
print('\nNaive-Bayes model accuracy score is %.1f' %accuracy) # print out accuracy score, .1f is float with 1 decimal point of the accuracy value
print('Naive-Bayes model F1 score is %.3f' %f1) # # print out F1 score, .3f is float with 3 decimal points of the f1 value
# %.1f and %.3f are format specifiers. They begin with %, then followed by a character that represents the data type, which is a float
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset


from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from flask import Flask, request, render_template, redirect
import os
from Google import Create_Service
import base64

app = Flask(__name__)

usernames = ['Wesley'] # Triggers

# Send e-mail with GMAIL API
def send_email(username, service):

    # Get email to send from page request
    TO = request.form["TO"]

    # Create message container - the correct MIME type is multipart/alternative.
    msg = MIMEMultipart('alternative')
    msg['Subject'] = request.form["Title"].format(username)
    msg['To'] = request.form["TO"]
    send = [TO]

    # Get HTML from page request
    html = request.form["html"].format(username)       # + str(request.files['file'].read(), 'utf-8')

    # Record the MIME types with html format
    html_code = MIMEText(html.encode('utf-8'), 'html', 'utf-8')
    msg.attach(html_code)

    # Record MSG message in an RFC 2822 formatted and base64url encoded string
    raw_string = base64.urlsafe_b64encode(msg.as_bytes()).decode()

    return service.users().messages().send(userId='me', body={'raw': raw_string}).execute()

# Get user e-mail from GMAIL API
def get_user_email(service):
    get_user_email = service.users().getProfile(userId='me').execute()
    user_email = get_user_email['emailAddress''']
    return user_email

# Auth GMAIL API with OAUTH2
def oauth2_gmail_login():
    CLIENT_SECRET_FILE = 'client_secret.json'
    API_NAME = 'gmail'
    API_VERSION = 'v1'
    SCOPES = ['https://www.googleapis.com/auth/gmail.send','https://www.googleapis.com/auth/gmail.readonly']

    service = Create_Service(CLIENT_SECRET_FILE, API_NAME, API_VERSION, SCOPES)

    return service

# Home route
@app.route('/', methods=["POST", "GET"])
def home():
    if request.method == "POST":
        html = request.form["html"]
        print(html)
        for username in usernames:
            send_email(username, service=oauth2_gmail_login())
        return render_template("home.html", user_email=get_user_email(service=oauth2_gmail_login()))
    elif request.method == "GET":
        return render_template("home.html", user_email=get_user_email(service=oauth2_gmail_login()))

#Login route
@app.route('/login', methods=["POST","GET"])
def login():
    if request.method == "GET":
        return render_template("login.html")
    elif request.method == "POST":
        oauth2_gmail_login()
        return redirect('http://127.0.0.1:5000/')

if __name__ == '__main__':
    app.run(debug=True)


# local

from .base import FactorizedTensor
import ivy

# global
import warnings

class TRTensor(FactorizedTensor):
    def __init__(self, factors):
        super().__init__()
        shape, rank = TRTensor.validate_tr_tensor(factors)
        self.shape = tuple(shape)
        self.rank = tuple(rank)
        self.factors = factors

    # Built-ins #
    # ----------#
    def __getitem__(self, index):
        return self.factors[index]

    def __setitem__(self, index, value):
        self.factors[index] = value

    def __iter__(self):
        for index in range(len(self)):
            yield self[index]

    def __len__(self):
        return len(self.factors)

    def __repr__(self):
        message = (
            f"factors list : rank-{self.rank} tensor ring tensor of shape {self.shape}"
        )
        return message

    # Public Methods #
    # ---------------#

    def to_tensor(self):
        return TRTensor.tr_to_tensor(self.factors)

    def to_unfolded(self, mode):
        return TRTensor.tr_to_unfolded(self.factors, mode)

    def to_vec(self):
        return TRTensor.tr_to_vec(self.factors)

    # Properties #
    # ---------------#
    @property
    def n_param(self):
        factors = self.factors
        total_params = sum(int(ivy.prod(tensor.shape)) for tensor in factors)
        return total_params

    # Class Methods #
    # ---------------#
    @staticmethod
    def validate_tr_tensor(factors):
        n_factors = len(factors)

        if n_factors < 2:
            raise ValueError(
                "A Tensor Ring tensor should be composed of at least two factors."
                f"However, {n_factors} factor was given."
            )

        rank = []
        shape = []
        next_rank = None
        for index, factor in enumerate(factors):
            current_rank, current_shape, next_rank = ivy.shape(factor)

            # Check that factors are third order tensors
            if len(factor.shape) != 3:
                raise ValueError(
                    "TR expresses a tensor as third order factors (tr-cores).\n"
                    f"However, ivy.ndim(factors[{index}]) = {len(factor.shape)}"
                )

            # Consecutive factors should have matching ranks
            if ivy.shape(factors[index - 1])[2] != current_rank:
                raise ValueError(
                    "Consecutive factors should have matching ranks\n -- e.g."
                    " ivy.shape(factors[0])[2]) == ivy.shape(factors[1])[0])\nHowever,"
                    f" ivy.shape(factor[{index-1}])[2] =="
                    f" {ivy.shape(factors[index-1])[2]} but"
                    f" ivy.shape(factor[{index}])[0] == {current_rank}"
                )

            shape.append(current_shape)
            rank.append(current_rank)

        # Add last rank (boundary condition)
        rank.append(next_rank)

        return tuple(shape), tuple(rank)

    @staticmethod
    def tr_to_tensor(factors):
        full_shape = [f.shape[1] for f in factors]
        full_tensor = ivy.reshape(factors[0], (-1, factors[0].shape[2]))

        for factor in factors[1:-1]:
            rank_prev, _, rank_next = factor.shape
            factor = ivy.reshape(factor, (rank_prev, -1))
            full_tensor = ivy.dot(full_tensor, factor)
            full_tensor = ivy.reshape(full_tensor, (-1, rank_next))

        full_tensor = ivy.reshape(
            full_tensor, (factors[-1].shape[2], -1, factors[-1].shape[0])
        )
        full_tensor = ivy.moveaxis(full_tensor, 0, -1)
        full_tensor = ivy.reshape(
            full_tensor, (-1, factors[-1].shape[0] * factors[-1].shape[2])
        )
        factor = ivy.moveaxis(factors[-1], -1, 1)
        factor = ivy.reshape(factor, (-1, full_shape[-1]))
        full_tensor = ivy.dot(full_tensor, factor)
        return ivy.reshape(full_tensor, full_shape)

    @staticmethod
    def tr_to_unfolded(factors, mode):
        return ivy.unfold(TRTensor.tr_to_tensor(factors), mode)

    @staticmethod
    def tr_to_vec(factors):
        return ivy.reshape(
            TRTensor.tr_to_tensor(factors),
            (-1,),
        )

    @staticmethod
    def validate_tr_rank(tensor_shape, rank="same", rounding="round"):
        if rounding == "ceil":
            rounding_fun = ivy.ceil
        elif rounding == "floor":
            rounding_fun = ivy.floor
        elif rounding == "round":
            rounding_fun = ivy.round
        else:
            raise ValueError(
                f"Rounding should be round, floor or ceil, but got {rounding}"
            )

        if rank == "same":
            rank = float(1)

        n_dim = len(tensor_shape)
        if n_dim == 2:
            warnings.warn(
                "Determining the TR-rank for the trivial case of a matrix"
                f" (order 2 tensor) of shape {tensor_shape}, not a higher-order tensor."
            )

        if isinstance(rank, float):
            # Choose the *same* rank for each mode
            n_param_tensor = ivy.prod(tensor_shape) * rank

            # R_k I_k R_{k+1} = R^2 I_k
            solution = int(
                rounding_fun(ivy.sqrt(n_param_tensor / ivy.sum(tensor_shape)))
            )
            rank = (solution,) * (n_dim + 1)

        else:
            # Check user input for potential errors
            n_dim = len(tensor_shape)
            if isinstance(rank, int):
                rank = (rank,) * (n_dim + 1)
            elif n_dim + 1 != len(rank):
                message = (
                    "Provided incorrect number of ranks. Should verify len(rank) =="
                    f" len(tensor.shape)+1, but len(rank) = {len(rank)} while"
                    f" len(tensor.shape)+1 = {n_dim + 1}"
                )
                raise ValueError(message)

            # Check first and last rank
            if rank[0] != rank[-1]:
                message = (
                    f"Provided rank[0] == {rank[0]} and rank[-1] == {rank[-1]}"
                    " but boundary conditions dictate rank[0] == rank[-1]"
                )
                raise ValueError(message)

        return list(rank)

    @staticmethod
    def tr_n_param(tensor_shape, rank):
        factor_params = []
        for i, s in enumerate(tensor_shape):
            factor_params.append(rank[i] * s * rank[i + 1])
        return ivy.sum(factor_params)