import binascii
import sys

class SM3:
    MAX = 2 ** 32
    IV = "7380166f4914b2b9172442d7da8a0600a96f30bc163138aae38dee4db0fb0e4e"

    def __init__(self, string):
        self.input = string
        self.b_input = bytes(self.input, "utf-8")
        self.hex_input = hex(int.from_bytes(self.b_input, "big"))[2:]  # 按照大端计算

    @property
    def hash(self):
        """
        获取结果Hash值
        :return: 256位16进制hash值
        """
        return self.iterationCourse(self.fill(self.hexToBin(self.hex_input)))[-1]

    def getT(self, j):
        """
        常量
        :param j:
        :return: 79cc4519 (0 <= j <= 15), 7a879d8a (16 <= j <= 63)
        """
        if 0 <= j <= 15:
            return int("79cc4519", 16)
        elif 16 <= j <= 63:
            return int("7a879d8a", 16)

    def FF(self, X, Y, Z, j):
        """
        布尔函数
        :param X: 16进制string
        :param Y: 16进制string
        :param Z: 16进制string
        :return: X ^ Y ^ Z (0 <= j <= 15), (X & Y) | (X & Z) | (Y & Z) (16 <= j <= 63)
        """
        if 0 <= j <= 15:
            return X ^ Y ^ Z
        elif 16 <= j <= 63:
            return (X & Y) | (X & Z) | (Y & Z)
        else:
            return 0

    def GG(self, X, Y, Z, j):
        """
        布尔函数
        :param X: 16进制string
        :param Y: 16进制string
        :param Z: 16进制string
        :return: X ^ Y ^ Z (0 <= j <= 15), (X & Y) | (～X & Z) (16 <= j <= 63)
        """
        if 0 <= j <= 15:
            return X ^ Y ^ Z
        elif 16 <= j <= 63:
            return (X & Y) | (self.non(X) & Z)
        else:
            return 0

    def P0(self, X):
        """
        置换函数
        :param X: 整数值
        :return: X ^ (X <<< 9) ^ (X <<< 17)
        """
        return X ^ self.leftRotate(X, 9) ^ self.leftRotate(X, 17)

    def P1(self, X):
        """
        置换函数
        :param X: 整数值
        :return: X ^ (X <<< 15) ^ (X <<< 23)
        """
        return X ^ self.leftRotate(X, 15) ^ self.leftRotate(X, 23)

    def leftRotate(self, X, k):
        """
        循环左移
        :param X: 整数值
        :param k: 位移的位数
        :param bit: 整数对应二进制的位数
        :return: 二进制左移k位的整数值
        """
        res = list(self.intToBin(X))
        for i in range(k):
            temp = res.pop(0)
            res.append(temp)
        return int("".join(res), 2)

    def fill(self, bin_string):
        """
        填充二进制消息string
        :param bin_string: 对任意消息使用self.msgToBin()得到的结果
        :return: 填充后的二进制消息m'，其比特长度为512的倍数
        """
        tail = "{:064b}".format(len(bin_string))
        bin_string += "1"
        div, mod = divmod(len(bin_string), 512)
        if mod >= 448:
            bin_string += "0" * (512 - mod + 448)
        else:
            bin_string += "0" * (448 - mod)

        return bin_string + tail

    def iterationCourse(self, msg):
        """
        迭代压缩消息
        :param msg: 填充后的二进制消息m', self.fill(msg)
        :return: Hash值(杂凑值)
        """
        # 将填充消息m'按512比特进行分组
        n = len(msg) // 512
        m = [msg[i * 512:(i + 1) * 512] for i in range(n)]
        # 对m[i]进行压缩迭代, msg = self.bigLittleEndianConvert(Vi, "big")  # 小端数据转换为大端
        V = [self.IV]
        for i in range(n):
            V.append(hex(int(self.intToBin(self.CF(V[-1], m[i]), 256), 2))[2:])

        return V

    def CF(self, Vi, mi):
        """
        压缩函数
        :param Vi: 512比特16进制数据
        :param mi: 512比特二进制数据
        :return: 512比特16进制数据
        """
        # 将Vi存储到字寄存器
        msg = Vi
        A = [int(msg[i * 8: (i + 1) * 8], 16) for i in range(len(msg) // 8)]

        # 消息扩展，得到W和W'
        W1, W2 = self.informationExtend(mi)

        # 压缩消息
        for j in range(64):
            factor1 = self.leftRotate(A[0], 12)
            factor2 = self.leftRotate(self.getT(j), j % 32)
            SS1 = self.leftRotate((factor1 + A[4] + factor2) % self.MAX, 7)
            factor3 = self.leftRotate(A[0], 12)
            SS2 = SS1 ^ factor3
            TT1 = (self.FF(A[0], A[1], A[2], j) + A[3] + SS2 + W2[j]) % self.MAX
            TT2 = (self.GG(A[4], A[5], A[6], j) + A[7] + SS1 + W1[j]) % self.MAX
            A[3] = A[2]
            A[2] = self.leftRotate(A[1], 9)
            A[1] = A[0]
            A[0] = TT1
            A[7] = A[6]
            A[6] = self.leftRotate(A[5], 19)
            A[5] = A[4]
            A[4] = self.P0(TT2)
        temp = self.intToBin(A[0], 32) + self.intToBin(A[1], 32) + self.intToBin(A[2], 32) + \
               self.intToBin(A[3], 32) + self.intToBin(A[4], 32) + self.intToBin(A[5], 32) + \
               self.intToBin(A[6], 32) + self.intToBin(A[7], 32)
        temp = int(temp, 2)
        return temp ^ int(Vi, 16)

    def informationExtend(self, mi):
        """
        消息扩展, 将512比特二进制消息扩展为132个字
        :param mi: Bi，512比特二进制数据
        :return: W -> 68个16进制消息字, W' -> 64个16进制消息字
        """
        # 第一步，将消息Bi划分为16个字W0~W15
        mi = self.binToHex(mi)
        W1 = [int(mi[i * 8: (i + 1) * 8], 16) for i in range(len(mi) // 8)]
        # 第二步
        for j in range(16, 68):
            p = self.P1(W1[j - 16] ^ W1[j - 9] ^ self.leftRotate(W1[j - 3], 15))
            W1.append(p ^ self.leftRotate(W1[j - 13], 7) ^ W1[j - 6])
        # 第三步
        W2 = [W1[j] ^ W1[j + 4] for j in range(64)]

        return W1, W2

    def bigLittleEndianConvert(self, data, need="big"):
        """
        大小端16进制数据转换
        :param data: 16进制小端string
        :param need: 转换为大端还是小端
        :return: 转换后16进制string
        """
        if sys.byteorder != need:
            return binascii.hexlify(binascii.unhexlify(data)[::-1])
        return data

    def hexToBin(self, hex_string):
        """
        十六进制string转换为二进制string
        :param hex: 十六进制string
        :return: 二进制string
        """
        res = ""
        for h in hex_string:
            res += '{:04b}'.format(int(h, 16))
        return res

    def binToHex(self, bin_string):
        """
        二进制string转换为十六进制string
        :param bin_string: 二进制string
        :return: 十六进制string
        """
        res = ""
        for i in range(len(bin_string) // 4):
            s = bin_string[i * 4: (i + 1) * 4]
            res += '{:x}'.format(int(s, 2))
        return res

    def msgToHex(self, msg):
        """
        字符串转换为16进制字符串
        :param msg: string
        :return: msg.encode("utf-8").hex()
        """
        return msg.encode("utf-8").hex()

    def msgToBin(self, msg):
        """
        字符串转换为二进制字符串
        :param msg: string
        :return: self.hexToBin(self.msgToHex(msg))
        """
        return self.hexToBin(self.msgToHex(msg))

    def intToBin(self, X, bits=32):
        """
        整数值转二进制
        :param X: 整数值
        :return: 32位二进制字符串
        """
        return ('{:0%db}' % bits).format(X)

    def non(self, X):
        """
        按位非
        :param X: 整数值
        :return: 按位非后的整数值
        """
        X = self.intToBin(X)
        Y = ""
        for i in X:
            if i == "0":
                Y += "1"
            else:
                Y += "0"
        return int(Y, 2)

if __name__ == "__main__":
    print(SM3("abc").hash)
    print(SM3("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd").hash)
    print(SM3("你好").hash)


from itertools import product
from flask import request,  jsonify
from app.config.database import db
from app.models.categories import CategorieModel
from app.models.products import ProductModel
from sqlalchemy.orm.session import Session
from sqlalchemy.orm import Query
from datetime import datetime, timedelta
from http import HTTPStatus
from sqlalchemy.exc import IntegrityError
from dotenv import load_dotenv
import os
from app.config.auth import auth_user, auth_partner

load_dotenv()

@auth_partner.login_required
def register_product():
    from app.tasks import close_auction, open_auction

    session: Session = db.session()

    data:dict = request.get_json()

    data["partner_id"] = auth_user.current_user().id
    open_time = datetime.strptime(data["auction_start"], "%Y-%m-%d %H:%M") - datetime.now()
    close_time = datetime.strptime(data["auction_end"], "%Y-%m-%d %H:%M") - datetime.now()

    try:
        new_list = []
        if data.get("categories"):
            new_list = data["categories"]
            data.pop("categories")
        product_info = ProductModel(**data)

        if new_list:
            for i in new_list:
                product_category = session.query(CategorieModel).filter_by(name = i).first()
                if product_category:
                    product_info.categories.append(product_category)

        session.add(product_info)
        session.commit()

        open_auction.delay(product_info.id, open_time.seconds)
        task = close_auction.delay(product_info.id, close_time.seconds)

        setattr(product_info, "task_id", task.task_id)

        # session.add(product_info)
        session.commit()
        return jsonify(product_info), HTTPStatus.CREATED
    except:
        #tratar possíveis erros no registro do produto
        {"erro":"Verifique sua requisição"}, HTTPStatus.BAD_REQUEST

@auth_partner.login_required
def update_product(product_id):
    session: Session = db.session()

    data = request.get_json()

    product = ProductModel.query.get(product_id)
    print(product)
    for key, value in data.items():
        setattr(product, key, value)

    session.add(product)
    session.commit()

    return jsonify(product), HTTPStatus.ACCEPTED

def get_products():
    products_list_query: Query = db.session.query(ProductModel)
    products_list = products_list_query.all()

    return jsonify(products_list), HTTPStatus.OK

def get_product_by_id(product_id):

    product: ProductModel = ProductModel.query.filter_by(id = product_id).first()

    return jsonify(product), HTTPStatus.OK


'''Capítulo 6: Funciones
Ejercicio 4:

Desarrollar un programa para calcular el factorial de un
número con ayuda de una función recursiva.'''

def factorial(num):
    if num>0:
        resultado= num* factorial(num-1)
    else:
        resultado=1
    return resultado

valor=factorial(0)
print(valor)


from contextlib import nullcontext
from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union
from uuid import uuid4

import torch
from fairscale.nn.data_parallel import FullyShardedDataParallel as FSDP
from fairscale.nn.data_parallel import ShardedDataParallel as SDDP
from torch.nn.parallel import DataParallel as DP
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader as DL
from torch.utils.data import Dataset
from torch.utils.data.distributed import DistributedSampler, Sampler

from stoke.configs import (
    AMPConfig,
    ApexConfig,
    ClipGradConfig,
    ClipGradNormConfig,
    DDPConfig,
    DeepspeedConfig,
    FairscaleFSDPConfig,
    FairscaleOSSConfig,
    FairscaleSDDPConfig,
    HorovodConfig,
    StokeOptimizer,
)
from stoke.data import StokeDataLoader
from stoke.distributed import RunnerDistEnum
from stoke.extensions import RunnerOptimizerEnum
from stoke.fp16 import RunnerFP16Enum
from stoke.io_ops import RunnerIOEnum
from stoke.status import DistributedOptions, FP16Options, StokeStatus
from stoke.utils import (
    ParamNormalize,
    T_co,
    _collate_fn_t,
    _worker_init_fn_t,
    zero_optimizer_grads,
)

class Stoke:
    """High level stoke object that manages all necessary configs and provides a unified interface to ops

    This is the main class within Stoke. Functionally it manages all interfaces to the necessary wrapped ops (model,
    loss, backward, step), provides helper functions, and dynamically constructs the runtime that handles the
    combinatorics problem of underlying frameworks (DDP, Horovod, Deepspeed, Fairscale),
    mixed-precision (AMP or APEX) and devices (CPU or GPU)

    Attributes
    ----------
    amp_config
    apex_config
    batch_size
    cuda
    ddp_config
    deepspeed_config
    distributed
    effective_batch_size
    ema_loss
    fp16
    fsdp_config
    fully_sharded
    gpu
    grad_accum
    grad_clip
    horovod_config
    is_amp
    is_apex
    is_ddp
    is_deepspeed
    is_horovod
    loss_access
    model_access
    nccl
    num_model_parameters
    optimizer
    oss
    oss_config
    rank
    scaler
    sddp_config
    sharded
    status
    world_size
    _agg_loss: Union[float, List[float], Tuple[float]]
        aggregated loss for grad accumulation (single or multiple losses)
    _backward_steps: int
        Number of times gradients have been calculated on a batch of samples (calls to backward)
    _grad_accum_counter: int
        counter for grad accumulation steps
    _loss: Union[Callable, List[Callable], Tuple[Callable]]
        callable function that calculates a loss from the model outputs
    _last_step_loss: list, tuple, or float
        last loss step calculation aggregated over device(s)
    _model: torch.nn.Module
        instance of torch.nn.Module for Stoke to handle
    _optimizer: StokeOptimizer
        StokeOptimizer config object that describes the torch.optim.Optimizer and it's kwargs
    _optimizer_steps: int
        Number of times step has been called on the optimizer
    _runner: StokeRunner
        the dynamically created runtime object that handles all ops
    _status: StokeStatus
        StokeStatus object that sets and maintains the current configuration
    _verbose: bool
        print verbosity
    _rolling_loss_steps: int
        number of steps that have been called for the rolling loss
    _rolling_mean_loss: list, tuple, or float
        current ema loss
    _ema_weight: float
        weight used for any ema calculation on metrics

    """

    def __init__(
        self,
        model: torch.nn.Module,
        optimizer: StokeOptimizer,
        loss: Union[Callable, List[Callable], Tuple[Callable]],
        batch_size_per_device: int,
        grad_accum_steps: Optional[int] = 1,
        grad_clip: Optional[Union[ClipGradConfig, ClipGradNormConfig]] = None,
        gpu: bool = False,
        fp16: Optional[FP16Options] = None,
        distributed: Optional[DistributedOptions] = None,
        fairscale_oss: bool = False,
        fairscale_sddp: bool = False,
        fairscale_fsdp: bool = False,
        configs: Optional[
            List[
                Union[
                    AMPConfig,
                    ApexConfig,
                    DDPConfig,
                    DeepspeedConfig,
                    FairscaleOSSConfig,
                    FairscaleSDDPConfig,
                    FairscaleFSDPConfig,
                    HorovodConfig,
                ]
            ]
        ] = None,
        info_rank: Optional[Union[int, List[int]]] = 0,
        verbose: bool = True,
        ema_weight: float = 0.1,
    ):
        """Init for Stoke class object

        Parameters
        ----------
        model: torch.nn.Module
            PyTorch model
        optimizer: StokeOptimizer
            Optimizer configuration
        loss: Union[Callable, List[Callable], Tuple[Callable]]
            Callable loss function or functions
        batch_size_per_device: int
            Batch size at the single device level
        grad_accum_steps: Optional[int], default: 1
            Number of gradient accumulation steps
        grad_clip: Optional[Union[ClipGradConfig, ClipGradNormConfig]], default: None
            Gradient clipping configuration
        gpu: bool, default: False
            flag to use GPU device(s)
        fp16: Optional[FP16Options], default: None
            Choice of mixed-precision backend
        distributed: Optional[DistributedOptions], default: None
            Choice of distributed backend
        fairscale_oss: bool, default: False
            Flag to activate optimizer state sharding using Fairscale
        fairscale_sddp: bool, default: False
            Flag to activate sharded DDP using Fairscale
        fairscale_fsdp: bool, default: False
            Flag to activate fully sharded DDP using Fairscale
        configs: Optional[List[Union[AMPConfig, ApexConfig, DDPConfig, DeepspeedConfig, FairscaleOSSConfig, FairscaleSDDPConfig, FairscaleFSDPConfig, HorovodConfig]], default: None
            Configuration objects for runtimes
        info_rank: Optional[Union[int, List[int]]], default = 0
            Constrain prints to specific devices
        verbose: bool, default: True
            Flag for verbosity
        ema_weight: float, default: 0.5
            weight used for any ema calculation on metrics

        """
        # Verbosity
        self._verbose = verbose
        # Info rank
        self._info_rank = info_rank
        # EMA
        self._ema_weight = ema_weight
        # Setup the StokeState
        self._status = StokeStatus(
            batch_size_per_device=batch_size_per_device,
            grad_accum=grad_accum_steps,
            grad_clip=grad_clip,
            gpu=gpu,
            fp16=fp16,
            distributed=distributed,
            fairscale_oss=fairscale_oss,
            fairscale_sddp=fairscale_sddp,
            fairscale_fsdp=fairscale_fsdp,
            configs=configs,
        )
        # Run some checks
        self._model = self._check_model(model)
        self._optimizer = self._check_optimizer(optimizer)
        self._loss = self._check_loss(loss)
        # Dynamically construct the StokeRunner from the StokeStatus
        self._runner, class_info = self._build_runner()
        # Setup distributed backend
        self._runner.setup_distributed()
        # Post here the runner will have the print_device function that is mapped to the self.print here
        # as it needs rank to be accessible before working
        if self._verbose:
            dev_id = (
                self.rank
                if (self.rank == "cpu" or self.rank == "gpu")
                else self._info_rank
            )
            self.print(f"Printing verbose information on rank(s): {dev_id}")
            # Print the runner class info from the mixins
            self.print(class_info)
        # Possibly place model on GPU depending on StokeStatus -- before wrap calls
        self._place_model_on_gpu()
        # Handle the wrap ops in the correct order
        self._handle_ordered_wrap_ops(optimizer=optimizer)
        # Create some tracking vars
        self._grad_accum_counter = 0
        self._optimizer_steps = 0
        self._backward_steps = 0
        self._last_step_loss = self._set_loss_to_zero()
        self._agg_loss = self._set_loss_to_zero()
        self._rolling_mean_loss = self._set_loss_to_zero()
        self._rolling_loss_steps = 0
        # Set post-init status variables
        self._status.set_post_init_values(world_size=self.world_size)
        # Print the final configuration
        if self._verbose:
            self.print(msg=self._status)

    def _wrap_optimizer_then_model(self, optimizer: StokeOptimizer):
        """Handles wrapping of optimizer then the model

        This holds only for SDDP, Horovod, and APEX as these need to use an instantiated optimizer before wrapped
        methods are called

        Parameters
        ----------
        optimizer: StokeOptimizer
            Optimizer configuration

        Returns
        -------
        None

        """
        # Build the optimizer
        self._optimizer = self._runner.build_optimizer(
            optimizer=optimizer["optimizer"],
            optimizer_kwargs=optimizer["optimizer_kwargs"],
            model=self._model,
        )
        # Setup/Initialize FP16 backend -- in this case the optimizer is passed through
        self._runner.wrap_fp16(model=self._model, optimizer=self._optimizer)
        # Wrap with distributed backend -- in this case the optimizer is passed through
        self._model, self._optimizer = self._runner.wrap_distributed(
            model=self._model, grad_accum=self.grad_accum, optimizer=self._optimizer
        )

    def _wrap_model_then_optimizer(self, optimizer: StokeOptimizer):
        """Handles wrapping of model then optimizer

        Parameters
        ----------
        optimizer: StokeOptimizer
            Optimizer configuration

        Returns
        -------
        None

        """
        # Wrap with distributed backend -- in this case the optimizer is passed as None since it doesn't exist yet
        # don't use the return for the optimizer in this case
        self._model, _ = self._runner.wrap_distributed(
            model=self._model, grad_accum=self.grad_accum, optimizer=None
        )
        # Setup/Initialize FP16 backend -- in this case the optimizer is passed as None since it doesn't exist yet
        self._runner.wrap_fp16(model=self._model, optimizer=None)
        # Build the optimizer
        self._optimizer = self._runner.build_optimizer(
            optimizer=optimizer["optimizer"],
            optimizer_kwargs=optimizer["optimizer_kwargs"],
            model=self._model,
        )

    def _handle_ordered_wrap_ops(self, optimizer: StokeOptimizer):
        """Handles wrapping model, using FP16, and wrapping optimizer in the correct order depending on Stoke Status

        Parameters
        ----------
        optimizer: StokeOptimizer
            Optimizer configuration

        Returns
        -------
        None

        """
        # if SDDP + OSS, Horovod, and APEX then we need to make sure that the optimizer gets wrapped before the model
        # gets wrapped, all other models follow standard DDP paradigm (or their own DeepSpeed)
        if (self.sharded and self.oss) or self.is_apex or self.is_horovod:
            self._wrap_optimizer_then_model(optimizer=optimizer)
        else:
            self._wrap_model_then_optimizer(optimizer=optimizer)

    def _check_accum(self):
        """Checks if the current step is the last accumulation step

        Returns
        -------
        bool

        """
        return (self._grad_accum_counter + 1) % (self.grad_accum + 1) == 0

    def _check_pre_accum(self):
        """Checks if we are at the pre-accumulate step

        Returns
        -------
        bool

        """
        return (self._grad_accum_counter + 1) % (self.grad_accum + 1) == self.grad_accum

    def _set_loss_to_zero(self):
        """Used to set a loss tracker to zero depending on the type

        Returns
        -------
        float or list or tuple of reset loss

        """
        return (
            type(self._loss)([0.0] * len(self._loss))
            if isinstance(self._loss, (list, tuple))
            else 0.0
        )

    def reset_ema(self):
        """Used to reset the current state of the rolling mean loss

        Returns
        -------
        None

        """
        self._rolling_mean_loss = self._set_loss_to_zero()
        self._rolling_loss_steps = 0

    def print_ema_loss(
        self, prepend_msg: str = "Current EMA Loss", single_line: bool = False
    ):
        """Prints the current ema loss synced across all devices

        Handles single or multiple losses. Prints only on devices specified by self._info_rank

        Parameters
        ----------
        prepend_msg: str, default: "Current EMA Loss"
            message prepend to print
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        if isinstance(self._rolling_mean_loss, (list, tuple)):
            print_vals = [
                f"{prepend_msg} {idx}: {val:.3f}"
                for idx, val in enumerate(self._rolling_mean_loss)
            ]
            self.print(print_vals, single_line=single_line)
        else:
            self.print(f"{prepend_msg}: {self._rolling_mean_loss:.3f}")

    def print_mean_accumulated_synced_loss(
        self,
        prepend_msg: str = "Mean Accumulated & Synced Loss",
        pre_backwards: bool = True,
        single_line: bool = False,
    ):
        """Prints the mean accumulated and device synced loss only after the grad accumulation step

        Handles single or multiple losses. Prints only on devices specified by self._info_rank

        Parameters
        ----------
        prepend_msg: str, default: "Mean Accumulated & Synced Loss"
            message prepend to print
        pre_backwards: bool, default: True
            if being called pre backward step
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        check_fn = self._check_pre_accum if pre_backwards else self._check_accum
        if check_fn():
            if isinstance(self._agg_loss, (list, tuple)):
                print_vals = self._scale_agg_loss()
                self.print(print_vals, single_line=single_line)
            else:
                self.print(f"{prepend_msg}: {self._scale_agg_loss():.3f}")

    def _scale_agg_loss(self):
        """Scales the mean aggregated loss by  grad accum

        Returns
        -------
        scale_vals: list or float of mean aggregated loss

        """
        if isinstance(self._agg_loss, (list, tuple)):
            scale_vals = [
                val / self.grad_accum for idx, val in enumerate(self._agg_loss)
            ]
        else:
            scale_vals = self._agg_loss / self.grad_accum
        return scale_vals

    def print_synced_loss(
        self,
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]],
        prepend_msg: str = "Step Synced Loss",
        device=None,
        single_line: bool = False,
    ):
        """Prints a device synced loss at a single step

        Handles single or multiple losses. Prints only on devices specified by self._info_rank

        Parameters
        ----------
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
            current loss(es) on the device
        prepend_msg: str, default: "Step Synced Loss"
            message prepend to print
        device: default: None
            specify the device to place the synced loss on (defaults to same device)
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        printable_loss = self.detach_and_sync_loss(loss, device)
        if isinstance(printable_loss, (list, tuple)):
            print_vals = [
                f"{prepend_msg} {idx}: {val * self.grad_accum:.3f}"
                for idx, val in enumerate(printable_loss)
            ]
            self.print(print_vals, single_line=single_line)
        else:
            self.print(msg=f"{prepend_msg}: {printable_loss * self.grad_accum:.3f}")

    def print_on_devices(
        self, msg: Union[str, List[str]], rank: Optional[Union[int, List[int]]] = 0
    ):
        """Wraps runner print interface for shorter semantics

        Parameters
        ----------
        msg: str
            message to print
        rank: Union[int, List[int]], default: 0
            which ranks to print on

        Returns
        -------
        None

        """
        self._runner.print_device(msg=msg, rank=rank)

    def print(self, msg: Union[str, List[str]], single_line: bool = False):
        """Wraps the runners print device and forces print on the _info_rank attribute(s)

        Parameters
        ----------
        msg: str
            message to print
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        self._runner.print_device(
            msg=msg, rank=self._info_rank, single_line=single_line
        )

    @staticmethod
    def _check_model(model: torch.nn.Module):
        """Verifies the type of the model

        Parameters
        ----------
        model: torch.nn.Module
            current torch model

        Returns
        -------
        None

        """
        # Check if the model is an nn.Module such that it has a forward method
        if not isinstance(model, torch.nn.Module):
            raise TypeError(
                f"Stoke -- Model is not of type torch.nn.Module, currently {type(model)}"
            )
        return model

    @staticmethod
    def _check_optimizer(optimizer: StokeOptimizer):
        """Verifies the type of the optimizer

        Parameters
        ----------
        optimizer: StokeOptimizer
            Current optimizer configuration TypedDict (aka dict)

        Returns
        -------
        None

        """
        if not isinstance(optimizer, dict):
            raise TypeError(
                f"Stoke -- Optimizer is not of type torch.optim.Optimizer, currently {type(optimizer)}"
            )
        return optimizer

    def _check_loss(self, loss: Union[Callable, List[Callable], Tuple[Callable]]):
        """Checks to make sure the loss function(s) is/are callable

        Parameters
        ----------
        loss: Union[Callable, List[Callable], Tuple[Callable]]
            Current callable loss(es)

        Returns
        -------
        None

        """
        if isinstance(loss, (list, tuple)):
            loss = [self._check_loss(val) for val in loss]
            return loss
        elif isinstance(loss, Callable):
            return loss
        else:
            raise TypeError(
                f"Stoke -- Loss is not of type Callable, currently {type(loss)}"
            )

    def _place_model_on_gpu(self):
        """Automatically moves the model to GPU device(s)

        Returns
        -------
        None

        """
        if self.gpu and not self.is_deepspeed:
            if self._verbose:
                self.print(f"Automatically handling moving model to GPU(s)...")
            self._model.cuda()

    def _build_runner(self):
        """Builds the runtime object from the mixin style classes

        Mixes the distributed class, fp16 class, and optimizer class into a single object such that all can be called
        from the same interface. Prevents verbose calls to multiple objects and unifies all functionality under a
        a single interface. Might prevent some IDE type-hinting as it's dynamic

        Returns
        -------
        StokeRunner
            runtime runner object

        """
        # Get the classes
        dist_class = self._get_distributed_mixin()
        fp16_class = self._get_fp16_mixin()
        optimizer_class = self._get_optimizer_mixin()
        io_class = self._get_io_mixin()

        # Python MRO hack to make sure the inits of all the Mixin classes get called
        def __multiple_mixin_init__(*args, **kwargs):
            dist_class.__init__(*args, **kwargs)
            fp16_class.__init__(*args, **kwargs)
            optimizer_class.__init__(*args, **kwargs)
            io_class.__init__(*args, **kwargs)

        # Configs pass through
        kwargs_dict = {
            "amp_config": self.amp_config,
            "apex_config": self.apex_config,
            "ddp_config": self.ddp_config,
            "deepspeed_config": self.deepspeed_config,
            "horovod_config": self.horovod_config,
            "oss_config": self.oss_config,
            "sharded_config": self.sddp_config,
            "fully_sharded_config": self.fsdp_config,
        }
        # Generate the runner class from the mixins based on the StokeStatus
        runner_class = type(
            "StokeRunner",
            (dist_class, fp16_class, optimizer_class, io_class),
            {"__init__": __multiple_mixin_init__},
        )(
            verbose=self._verbose,
            batch_size_per_device=self.batch_size,
            grad_accum_steps=self.grad_accum,
            grad_clip=self.grad_clip,
            info_rank=self._info_rank,
            loss=self._loss,
            **kwargs_dict,
        )
        # Make a list of class info for print later
        class_info = [
            f"Distributed Mixin: {dist_class.__name__}",
            f"Optimizer Mixin: {dist_class.__name__}",
            f"FP16 Mixin: {fp16_class.__name__}",
            f"IO Mixin: {io_class.__name__}",
        ]
        return runner_class, class_info

    def _get_io_mixin(self):
        """Determines which IO class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated ioclass

        """
        if self.is_deepspeed:
            return_class = RunnerIOEnum.deepspeed.value
        elif self.is_horovod:
            return_class = RunnerIOEnum.horovod.value
        elif self.is_ddp:
            return_class = RunnerIOEnum.ddp.value
        else:
            return_class = RunnerIOEnum.base.value
        return return_class

    def _get_optimizer_mixin(self):
        """Determines which optimizer class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated optimizer class

        """
        if self.oss:
            return_class = RunnerOptimizerEnum.oss.value
        else:
            return_class = RunnerOptimizerEnum.base.value
        return return_class

    def _get_distributed_mixin(self):
        """Determines which distributed class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated distributed class

        """
        # if not gpu then fall to cpu single
        if not self.gpu:
            return_class = RunnerDistEnum.cpu.value
        # if gpu but no distributed then fall to single gpu
        elif self.gpu and (self.distributed is None):
            return_class = RunnerDistEnum.gpu.value
        elif self.gpu and (self.distributed is not None):
            return_class = RunnerDistEnum[self.distributed].value
        else:
            raise ValueError("Stoke -- Cannot map to a valid distributed class")
        return return_class

    def _get_fp16_mixin(self):
        """Determines which fp16 class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated fp16 class

        """
        if self.fp16 is not None:
            return_class = RunnerFP16Enum[self.fp16].value
        else:
            return_class = RunnerFP16Enum.full.value
        return return_class

    def DataLoader(
        self,
        dataset: Dataset[T_co],
        shuffle: bool = False,
        sampler: Optional[Sampler[int]] = None,
        batch_sampler: Optional[Sampler[Sequence[int]]] = None,
        num_workers: int = 0,
        collate_fn: _collate_fn_t = None,
        pin_memory: bool = False,
        drop_last: bool = False,
        timeout: float = 0,
        worker_init_fn: Optional[_worker_init_fn_t] = None,
        multiprocessing_context=None,
        generator=None,
        *,
        prefetch_factor: int = 2,
        persistent_workers: bool = False,
    ):
        """Provides a shim interface to torch.utils.data.DataLoader with mapped kwargs.

        Shim is necessary for two reasons... to inject some horovod runtime configs (make sure forkserver is called)
        and to automatically handle device placement since the gpu/fp16 flags can't be determined until the StokeStatus
        object is available which is post init. This could be disconnected from this class but it would require the
        user to forward on device or fp16 configs which breaks the paradigm that the flags only need to be set and
        never handled

        Parameters
        ----------
        dataset: Dataset
            dataset from which to load the data.
        shuffle: bool, default: False
            set to ``True`` to have the data reshuffled at every epoch.
        sampler: Sampler or Iterable, default: None
            defines the strategy to draw samples from the dataset. Can be any ``Iterable`` with ``__len__``
            implemented. If specified, :attr:`shuffle` must not be specified.
        batch_sampler: Sampler or Iterable, default: None:
            like :attr:`sampler`, but returns a batch of indices at a time. Mutually exclusive with
            :attr:`batch_size`, :attr:`shuffle`, :attr:`sampler`, and :attr:`drop_last`.
        num_workers: int, default: 0
            how many subprocesses to use for data loading. ``0`` means that the data will be loaded in the main process.
        collate_fn: callable, optional:
            merges a list of samples to form a mini-batch of Tensor(s).  Used when using batched loading from a
            map-style dataset.
        pin_memory: bool, default: False:
            If ``True``, the data loader will copy Tensors into CUDA pinned memory before returning them. If your
            data elements are a custom type, or your :attr:`collate_fn` returns a batch that is a custom type,
            see the example below.
        drop_last: bool, default: False
            set to ``True`` to drop the last incomplete batch, if the dataset size is not divisible by the batch size.
            If ``False`` and the size of dataset is not divisible by the batch size, then the last batch
            will be smaller.
        timeout: numeric, default: 0
            if positive, the timeout value for collecting a batch from workers. Should always be non-negative.
        worker_init_fn: callable, default: None
            If not ``None``, this will be called on each worker subprocess with the worker id
            (an int in ``[0, num_workers - 1]``) as input, after seeding and before data loading.
        generator: torch.Generator: None
            If not ``None``, this RNG will be used by RandomSampler to generate random indexes and multiprocessing
            to generate `base_seed` for workers.
        prefetch_factor: int, default: 2
            Number of samples loaded in advance by each worker. ``2`` means there will be a total of 2 * num_workers
            samples prefetched across all workers.
        persistent_workers: bool, default: False
            If ``True``, the data loader will not shutdown the worker processes after a dataset has been
            consumed once. This allows to maintain the workers `Dataset` instances alive.

        Returns
        -------
        StokeDataLoader
            wrapped torch.utils.data.DataLoader object

        """
        # Check if forkserver is available for horovod and use
        if (
            num_workers > 0
            and "forkserver" in torch.multiprocessing.get_all_start_methods()
            and self.is_horovod
            and self.horovod_config.use_fork_server
        ):
            torch.multiprocessing.set_start_method("forkserver")
            if self._verbose:
                print(
                    f"Stoke -- Attempting to use forkserver as multiprocessing_context"
                )

        if self.distributed is not None and not isinstance(sampler, DistributedSampler):
            raise TypeError(
                f"Stoke -- Using a distributed backend requires passing an instance of a "
                f"DistributedSampler to the sampler argument"
            )
        if self._verbose and self.gpu:
            self.print(
                f"Stoke -- Automatically handling moving model input data to GPU(s)..."
            )
        # Assemble a kwargs dict as the super call with direct named args can cause un-traceable behavior (#23)
        # Insight from PyTorch Lightning that has to handle their DataLoader shims with all kwargs
        # https://github.com/PyTorchLightning/pytorch-lightning/blob/6609b2e46f5eb2cde6c42aedf5b843d050a4bb8d/pytorch_lightning/trainer/data_loading.py#L215
        kwargs = {
            "batch_size": self.batch_size,
            "shuffle": shuffle,
            "sampler": sampler,
            "batch_sampler": batch_sampler,
            "num_workers": num_workers,
            "collate_fn": collate_fn,
            "pin_memory": pin_memory,
            "drop_last": drop_last,
            "timeout": timeout,
            "worker_init_fn": worker_init_fn,
            "multiprocessing_context": multiprocessing_context,
            "generator": generator,
            "prefetch_factor": prefetch_factor,
            "persistent_workers": persistent_workers,
        }
        # Forward the already known options from the Stoke status
        return StokeDataLoader(dataset, gpu=self.gpu, fp16=self.fp16, **kwargs)

    def model(self, *args, **kwargs):
        """Wrapped model forward call

        Parameters
        ----------
        *args: list or tuple
            Additional arguments should be passed as keyword arguments
        **kwargs: dict, optional
            Extra arguments passed to the model forward call

        Returns
        -------
        model forward output

        """
        with self._runner.model_context:
            return self._model(*args, **kwargs)
            # return self.model_access(*args, **kwargs)

    def loss(self, *args, **kwargs):
        """Wrapped callable loss function call

        Handles internal logic of aggregating up the losses for single and multiple losses

        Parameters
        ----------
        *args: list or tuple
            Additional arguments should be passed as keyword arguments
        **kwargs: dict, optional
            Extra arguments passed to the loss function call(s)

        Returns
        -------
        outputs of callable loss function(s)

        """
        # TODO: WIP Handle multiple losses. Should support list/tuple of losses. Check non base PyTorch
        with self._runner.loss_context:
            if isinstance(self._loss, (list, tuple)):
                loss = type(self._loss)(val(*args, **kwargs) for val in self._loss)
                sync_loss = [self.detach_and_sync_loss(val) for val in loss]
                self._last_step_loss = type(self._loss)(
                    val for idx, val in enumerate(sync_loss)
                )
                self._agg_loss = type(self._loss)(
                    self._agg_loss[idx] + val for idx, val in enumerate(sync_loss)
                )
                self._handle_ema_loss(loss=sync_loss)
                if self.grad_accum > 1 and self.model_access.training:
                    loss = type(loss)(val / self.grad_accum for val in loss)
            else:
                loss = self._loss(*args, **kwargs)
                sync_loss = self.detach_and_sync_loss(loss)
                self._last_step_loss = sync_loss
                self._agg_loss += sync_loss
                self._handle_ema_loss(loss=sync_loss)
                # Handle grad accumulation by dividing by the accumulation steps
                if self.grad_accum > 1 and self.model_access.training:
                    loss = loss / self.grad_accum
            return loss

    def _handle_ema_loss(self, loss: Union[float, List[float], Tuple[float]]):
        """Handles calculating the ema loss

        Parameters
        ----------
        loss: Union[float, List[float], Tuple[float]]
            current calculated loss list, tuple or float

        Returns
        -------
        None

        """
        self._rolling_loss_steps += 1
        if isinstance(loss, (list, tuple)):
            self._rolling_mean_loss = type(self._rolling_mean_loss)(
                self._ema_loss(value=val, current_mean=self._rolling_mean_loss[idx])
                for idx, val in enumerate(loss)
            )
        else:
            self._rolling_mean_loss = self._ema_loss(
                value=loss, current_mean=self._rolling_mean_loss
            )

    def _ema_loss(self, value: float, current_mean: float):
        """Calculate the ema of the loss

        Parameters
        ----------
        value: float
            current loss value
        current_mean: float
            current mean value

        Returns
        -------
        current ema value: float

        """
        if self._rolling_loss_steps == 1:
            return value
        else:
            return (self._ema_weight * value) + (
                (1.0 - self._ema_weight) * current_mean
            )

    def backward(
        self, loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
    ):
        """Wrapped backwards call

        Parameters
        ----------
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
            Callable loss function(s)

        Returns
        -------
        None

        """
        # Increment the grad counter
        self._grad_accum_counter += 1
        # Set the context based on the counter
        dist_cm = (
            nullcontext()
            if self._check_accum()
            else self._runner.grad_accum_context(self._model)
        )
        with dist_cm:
            self._runner.backward_call(
                loss=loss, model=self.model_access, optimizer=self._optimizer
            )
        # Increment the number of total calls to backward (each backward to a loss is only considered 1)
        self._backward_steps += 1

    def step(self):
        """Wrapped step call

        Handles grad clipping internally

        Returns
        -------
        None

        """
        # Step the optimizer only if the modulo is zero
        if self._check_accum():
            if self._verbose and self.grad_accum > 0:
                self.print(f"Gradient Accumulation Steps: {self.grad_accum}")
            # Clip if needed
            if self.grad_clip is not None:
                self._runner.clip_grad(
                    self.grad_clip,
                    self._model if self.fully_sharded else self.model_access,
                    self._optimizer,
                    oss=self.oss,
                    horovod=self.is_horovod,
                    deepspeed=self.is_deepspeed,
                    fsdp=self.fully_sharded,
                )
            # Handle the optimizer step
            step_cm = (
                self._runner.step_context(self._optimizer)
                if self.grad_clip is not None
                else nullcontext()
            )
            with step_cm:
                self._runner.step_call(
                    model=self.model_access, optimizer=self._optimizer
                )
            # Reset for the accumulated step
            self._reset()
            # Increment the number of step calls to the optimizer
            self._optimizer_steps += 1
        # if deepspeed we need to step everytime as it handles the grad accumulation internally
        elif self.is_deepspeed:
            # Handle the optimizer step
            step_cm = (
                self._runner.step_context(self._optimizer)
                if self.grad_clip is not None
                else nullcontext()
            )
            with step_cm:
                self._runner.step_call(
                    model=self.model_access, optimizer=self._optimizer
                )

    def _reset(self):
        """Resets the state post optimizer step call

        Returns
        -------
        None

        """
        if self._verbose:
            self.print("Resetting all grad/variables for next optimizer step")
        # Zero the grads if not deepspeed
        if not self.is_deepspeed:
            self.zero_grads()
        # Reset counter
        self._grad_accum_counter = 0
        # Reset agg loss -- single or mutiple losses
        self._agg_loss = self._set_loss_to_zero()

    def save(
        self,
        path: str,
        name: str = uuid4(),
        extension: str = "pt",
        create_directory: bool = True,
        extras: Optional[dict] = None,
    ):
        """Saves a model checkpoint using the correct backend interface

        Parameters
        ----------
        path: str
            path to directory to save the model checkpoint (prefer absolute paths over relative paths)
        name: str, default: uuid4()
            name used to save checkpoint file
        extension: str, default: '.pt'
            extension used to save PyTorch model checkpoint
        create_directory: bool, default: True
            flag to create the directory path if it doesn't exist
        extras: dict, default: None
            a dictionary of any extra things to save

        Returns
        -------
        path: str
            path to directory that the model checkpoint was saved
        tag: str
            full tag name the model checkpoint was saved as

        """
        out_path, tag = self._runner.save(
            model=self._model if self.fully_sharded else self.model_access,
            optimizer=self.optimizer,
            path=path,
            backward_step=self._backward_steps,
            grad_accum_step=self._grad_accum_counter,
            optimizer_step=self._optimizer_steps,
            name=name,
            scaler_dict=self.fp16_state_dict,
            extension=extension,
            create_directory=create_directory,
            extras=extras,
            status=self.status.status,
        )
        self.print(f"Successfully saved model checkpoint to {out_path}/{tag}")
        return out_path, tag

    def load(self, path: str, tag: str, strict: bool = True):
        """Loads a model checkpoint using the correct backend interface

        Parameters
        ----------
        path: str
            path to directory that the model checkpoint was saved (prefer absolute paths over relative paths)
        tag: str
            full tag name the model checkpoint was saved as
        strict: bool
            ignore non-matching keys

        Returns
        -------
        extras: dict, default: None
            a dictionary of any custom fields the user passed to the save function

        """
        # TODO: How to deal with mapping between backends? e.g. FP16 model back to FP32? Or multi-gpu to CPU?
        backward_step, grad_accum_step, optimizer_step, extras = self._runner.load(
            model=self._model if self.fully_sharded else self.model_access,
            optimizer=self.optimizer,
            gpu=self.gpu,
            path=path,
            tag=tag,
            scaler_dict_fn=self._load_fp16_state_dict_fn(),
            strict=strict,
        )
        # Reset values based on what was in the load dict
        self._backward_steps = backward_step
        self._grad_accum_counter = grad_accum_step
        self._optimizer_steps = optimizer_step
        self.print(f"Successfully loaded model checkpoint from {path}/{tag}")
        # Return the extras dict
        return extras

    def print_num_model_parameters(
        self, normalize: ParamNormalize = ParamNormalize.MILLION
    ):
        """

        Parameters
        ----------
        normalize: ParamNormalize, default: ParamNormalize.MILLION
            ParamNormalize choice for pretty print normalizing

        Returns
        -------
        None

        """
        self.print(
            f"Total Trainable Model Parameters: "
            f"{(self.num_model_parameters / normalize.value):.3f} {normalize.name}"
        )

    def detach_and_sync_loss(
        self,
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]],
        device=None,
    ):
        """Shorthand method to detach and sync loss

        Maps to the runner function of the same name

        Parameters
        ----------
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
            current loss(es)
        device: default: None
            device to sync across

        Returns
        -------
        loss that is synced across devices and all_reduced w/ SUM

        """
        return self._runner.detach_and_sync_loss(loss=loss, device=device)

    def zero_grads(self):
        """Zeros the optimizer grads depending on the optimizer type

        Returns
        -------
        None

        """
        zero_optimizer_grads(
            optimizer=self._optimizer, apex=self.is_apex, horovod=self.is_horovod
        )

    def reset(self):
        """Public method for resetting the underlying stoke state

        Returns
        -------
        None

        """
        self._reset()

    def reset_tracking(self):
        """Public method for resetting all underlying stoke tracked variables

        Returns
        -------
        None

        """
        # Create some tracking vars
        self._grad_accum_counter = 0
        self._optimizer_steps = 0
        self._backward_steps = 0
        self._last_step_loss = self._set_loss_to_zero()
        self._agg_loss = self._set_loss_to_zero()
        self._rolling_mean_loss = self._set_loss_to_zero()
        self._rolling_loss_steps = 0

    def dump_model_parameter_info(self):
        """Dumps all parameter information for named parameters (shape, device, dtype)

        Returns
        -------
        None

        """
        self.print("Dumping all model parameter information to stdout....")
        for name, param in self.model_access.named_parameters():
            if param.requires_grad:
                self.print(
                    f"Name: {name}, Shape: {param.shape}, "
                    f"Device: {param.device}, dtype: {param.dtype}"
                )

    def _load_fp16_state_dict_fn(self):
        """Returns the function to load the sacler state dict

        Returns
        -------
        mp_state_dict_fn: Callable, default: None
            callable function to load the scaler state dict

        """
        mp_state_dict_fn = None
        if self.scaler is not None:
            if self.is_apex:
                try:
                    from apex import amp

                    mp_state_dict_fn = amp.load_state_dict
                except ImportError as e:
                    print(
                        e,
                        ": Stoke -- apex cannot be imported -- please install (https://github.com/NVIDIA/apex)",
                    )
            else:
                mp_state_dict_fn = self.scaler.load_state_dict
        return mp_state_dict_fn

    def barrier(self):
        """Calls the underlying distributed barrier if available"""
        self._runner.barrier()

    @property
    def step_loss(self):
        """Gets the last step loss synced across device(s) (unscaled)"""
        return self._last_step_loss

    @property
    def model_access(self):
        """Interface for model access due to the different types between the DP, DDP, and SDDP implementations"""
        if isinstance(self._model, (DDP, DP, SDDP, FSDP)):
            return self._model.module
        else:
            return self._model

    @property
    def loss_access(self):
        """Gets loss tensor(s)"""
        return self._loss

    @property
    def optimizer(self):
        """Gets the optimizer"""
        return self._optimizer

    @property
    def scaler(self):
        """Gets the current scaler object"""
        return self._runner.scaler

    @property
    def fp16_state_dict(self):
        """Gets the fp16 state dict from various methods"""
        mp_state_dict = None
        if self.scaler is not None:
            if self.is_apex:
                try:
                    from apex import amp

                    mp_state_dict = amp.state_dict()
                except ImportError as e:
                    print(
                        e,
                        ": Stoke -- apex cannot be imported -- please install (https://github.com/NVIDIA/apex)",
                    )
            elif self.is_amp:
                mp_state_dict = self.scaler.state_dict()
        return mp_state_dict

    @property
    def status(self):
        """Gets the StokeStatus object"""
        return self._status

    @property
    def batch_size(self):
        """Shortcut to batch size"""
        return self._status.batch_size

    @property
    def effective_batch_size(self):
        """Shortcut to effective batch size"""
        return self._status.effective_batch_size

    @property
    def grad_clip(self):
        """Shortcut to get grad clip"""
        return self._status.grad_clip

    @property
    def grad_accum(self):
        """Shortcut to get grad accumulation"""
        return self._status.grad_accum

    @property
    def gpu(self):
        """Shortcut to get GPU status"""
        return self._status.gpu

    @property
    def cuda(self):
        """Shortcut to get cuda status"""
        return self._status.cuda

    @property
    def nccl(self):
        """Shortcut to get nccl status"""
        return self._status.nccl

    @property
    def fp16(self):
        """Shortcut to get FP16 status"""
        return self._status.fp16

    @property
    def is_apex(self):
        """Returns if APEX is activated"""
        return self._status.is_fp16_apex

    @property
    def is_amp(self):
        """Returns if AMP is activated"""
        return self._status.is_fp16_amp

    @property
    def distributed(self):
        """Shortcut to distributed status"""
        return self._status.distributed

    @property
    def is_ddp(self):
        """Returns if DDP is activated"""
        return self._status.is_distributed_ddp

    @property
    def is_horovod(self):
        """Returns if Horovod is activated"""
        return self._status.is_distributed_horovod

    @property
    def is_deepspeed(self):
        """Returns if Deepspeed is acticated"""
        return self._status.is_distributed_deepspeed

    @property
    def oss(self):
        """Returns if Fairscale optimizer state sharding status"""
        return self._status.oss

    @property
    def sharded(self):
        """Returns if Fairscale sharded DDP status"""
        return self._status.sharded

    @property
    def fully_sharded(self):
        """Returns if Fairscale fully sharded DDP status"""
        return self._status.fully_sharded

    @property
    def world_size(self):
        """Shortcut to get world size"""
        return self._runner.world_size

    @property
    def rank(self):
        """Shortcut to get rank"""
        return self._runner.rank

    @property
    def amp_config(self):
        """Returns amp config or None based on amp state"""
        return self._status.amp_config if self.is_amp else None

    @property
    def apex_config(self):
        """Returns apex config or None based on apex state"""
        return self._status.apex_config if self.is_apex else None

    @property
    def ddp_config(self):
        """Returns ddp config or None based on ddp state"""
        return self._status.ddp_config if self.is_ddp else None

    @property
    def deepspeed_config(self):
        """Returns deepspeed config or None based on deepspeed state"""
        return self._status.deepspeed_config if self.is_deepspeed else None

    @property
    def oss_config(self):
        """Returns oss config or None based on ossstate"""
        return self._status.oss_config if self.oss else None

    @property
    def sddp_config(self):
        """Returns sddp config or None based on sddp state"""
        return self._status.sddp_config if self.sharded else None

    @property
    def fsdp_config(self):
        """Returns fsdp config or None based on fsdp state"""
        return self._status.fsdp_config if self.fully_sharded else None

    @property
    def horovod_config(self):
        """Returns horovod config or None based on horovod state"""
        return self._status.horovod_config if self.is_horovod else None

    @property
    def num_model_parameters(self):
        """Returns number of parameters that require gradients"""
        return sum(p.numel() for p in self.model_access.parameters() if p.requires_grad)

    @property
    def ema_loss(self):
        """Returns the current rolling mean loss"""
        return self._rolling_mean_loss


import warnings
import sys

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sqlalchemy import create_engine

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords, wordnet
from nltk.stem.wordnet import WordNetLemmatizer
import re

from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.metrics import classification_report

from sklearn.externals import joblib
from flask import Flask
from flask import render_template, request, jsonify
from plotly.graph_objs import Bar

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('wordnet')
nltk.download('words')

app = Flask(__name__)

def tokenize_fct(text):

    '''
    Tokenize by word, remove stop words, lemmatize all phrases in the dataframe

    INPUT:
    text

    OUTPUT:
    df - df with three new columns: one normalised, one tokenized, another with stop words removed in addition, and the
    last one lemmatized in addition to previous steps
    '''

    new_text = re.sub(r"[^a-zA-Z]", " ", text.lower())
    new_text = word_tokenize(new_text)

    sw_list = stopwords.words("english")
    eng_words = set(nltk.corpus.words.words())
    new_text = [w for w in new_text if w not in sw_list]
    new_text = [w for w in new_text if w in eng_words]

    for postag in ['a','s','r','n','v']:
        new_text = [WordNetLemmatizer().lemmatize(w, pos = postag) for w in new_text]

    return new_text

class TfidfEmbeddingVectorizer(BaseEstimator, TransformerMixin):

    def fit(self, X, y = None, col_name = ['message'], vect_size = 50):
        '''
        Tokenize sentences, then fit with w2v model (for word vectors) and tfidf
        for weighting
        '''
        self.vect_size = vect_size

        self.tok_X = []
        for line in X[col_name].values:
            self.tok_X.append(tokenize_fct(line[0]))

        # Fit w2v model
        model = Word2Vec(self.tok_X, size=vect_size)
        self.word_vector = dict(zip(model.wv.index2word, model.wv.vectors))

        # Fit tfidf for weighting
        tfidf = TfidfVectorizer(analyzer=lambda x: x, lowercase = False)
        tfidf.fit(self.tok_X)
        self.word_list = list(tfidf.vocabulary_.keys())
        self.idf_list = list(tfidf.idf_[list(tfidf.vocabulary_.values())])
        self.word_weight = dict(zip(word_list, idf_list))

        print('Embedding fitted!')
        return self

    def transform(self, X):
        '''
        return weighted mean vector of vect size as specified above
        '''
        self.weighted_vector = np.empty((0, self.vect_size), float)
        for line in self.tok_X:
            if len(line)>0:
                new_weight = [self.word_vector[w] * self.word_weight[w] for w in line \
                              if (w in self.word_vector) and (w in self.word_weight)]\
                            + [np.zeros(self.vect_size) for w in line \
                               if (w not in self.word_vector) or (w not in self.word_weight)]
                new_weight_mean = np.mean(new_weight, axis = 0).reshape(1, -1)
                self.weighted_vector = np.concatenate((self.weighted_vector, new_weight_mean), axis = 0)
            else:
                self.weighted_vector = np.concatenate((self.weighted_vector,
                                                       np.zeros(self.vect_size).reshape(1,-1)), axis = 0)

        return pd.DataFrame(self.weighted_vector)

class tfidfExtractor(BaseEstimator, TransformerMixin):

    def fit(self, X, y=None, col_name = 'message'):
        self.col_name = col_name
        self.countvect = CountVectorizer(tokenizer = tokenize_fct)
        X_cvect = self.countvect.fit_transform(X[col_name].values.ravel())
        self.tfidftrans = TfidfTransformer(smooth_idf=False)
        self.tfidftrans.fit(X_cvect)

        print('tfidf fitted!')
        return self

    def transform(self, X):
        # apply tfidf
        X_cvect = self.countvect.transform(X[self.col_name].values.ravel())
        X_tfidf = self.tfidftrans.transform(X_cvect)
        return X_tfidf

class GenreExtractor(BaseEstimator, TransformerMixin):

    def fit(self, X, y=None, col_name = 'genre'):
        self.col_name = col_name
        print('Genre fitted!')
        return self

    def transform(self, X):
        # apply one hot encoding
        X_one_hot = pd.get_dummies(X[self.col_name], drop_first = True)
        return X_one_hot

class LengthExtractor(BaseEstimator, TransformerMixin):

    def fit(self, X, y=None, col_name = 'original'):
        self.col_name = col_name
        return self

    def transform(self, X):
        new_col = pd.DataFrame(X[self.col_name].fillna('').apply(lambda x:len(x)))
        return new_col.values

# load data
def load_data(data_filepath = '../data/cleaned_data.db'):
    '''
    Load in data from designated filepath
    '''
    engine = create_engine('sqlite:///' + data_filepath)
    df = pd.read_sql_table('cleaned_data', engine)
    return df

df = load_data()

# load model
model = joblib.load("../models/trained_model.pkl")

# index webpage displays cool visuals and receives user input text for model
@app.route('/')
@app.route('/index')
def index():

    # extract data needed for visuals
    genre_counts = df.groupby('genre').count()['message']
    genre_names = list(genre_counts.index)

    Y = df.drop(['id', 'genre', 'message', 'original'], axis = 1)
    top5_cat = (Y.sum()/len(Y)).sort_values(ascending = False)[:5]
    top5_cat_val = top5_cat.values * 100
    top5_cat_names = [i.replace('_', ' ') for i in list(top5_cat.index)]

    # create visuals
    graphs = [
        {
            'data': [
                Bar(
                    x=genre_names,
                    y=genre_counts
                )
            ],

            'layout': {
                'title': 'Distribution of Message Genres',
                'yaxis': {
                    'title': "Count"
                },
                'xaxis': {
                    'title': "Genre"
                }
            }
        },

        {
            'data': [
                Bar(
                    x=top5_cat_names,
                    y=top5_cat_val
                )
            ],

            'layout': {
                'title': 'Top 5 message categories',
                'yaxis': {
                    'title': "Proportion of messages in the category (%)"
                },
                'xaxis': {
                    'title': "Category"
                }
            }
        }
    ]

    # encode plotly graphs in JSON
    ids = ["graph-{}".format(i) for i, _ in enumerate(graphs)]
    graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)

    # render web page with plotly graphs
    return render_template('master.html', ids=ids, graphJSON=graphJSON)

# web page that handles user query and displays model results
@app.route('/go')
def go():
    # save user input in query
    query = request.args.get('query', '')

    # use model to predict classification for query
    classification_labels = model.predict([query])[0]
    classification_results = dict(zip(df.columns[4:], classification_labels))

    # This will render the go.html Please see that file.
    return render_template(
        'go.html',
        query=query,
        classification_result=classification_results
    )

def main():
    app.run(host='0.0.0.0', port=3001, debug=True)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# encoding: utf-8

import logging
import os

import numpy as np

from fairseq.data import (
    data_utils,
    Dictionary,
    IdDataset,
    OffsetTokensDataset,
    NestedDictionaryDataset,
    NumelDataset,
    NumSamplesDataset,
    RightPadDataset,
    PrependTokenDataset,
    SortDataset,
    StripTokenDataset,
    TruncateDataset,
    RawLabelDataset,
    RawNumberDataset,
)
from fairseq.tasks import register_task, LegacyFairseqTask
from fairseq.data.shorten_dataset import maybe_shorten_dataset
from fairseq import utils
try:
    from command import params
except ImportError:
    from . import params

logger = logging.getLogger(__name__)

@register_task('func_name_pred')
class FuncNamePred(LegacyFairseqTask):
    """Task for training function name prediction model"""

    @staticmethod
    def add_args(parser):
        """Add task-specific arguments to the parser."""
        parser.add_argument('data', help='colon separated path to data directories list, \
                                                will be iterated upon during epochs in round-robin manner')
        parser.add_argument('--num-classes', type=int, default=-1,
                            help='size of vocabulary of the internal function name words')
        parser.add_argument('--num-external', type=int, default=-1,
                            help='size of vocabulary of the external function names')
        parser.add_argument('--num-calls', type=int, default=2,
                            help='number of function calls (callers, internal calees and external callees) to be considered')
        parser.add_argument('--external-emb', type=str, default='one_hot',
                            help='external callee embedding method, options: {one_hot, embedding}')
        parser.add_argument('--no-shuffle', action='store_true', default=False)

    def __init__(self, args, data_dictionary_dict, label_dictionary, dictionary_cf, external_dictionary):
        super().__init__(args)
        self.dictionary_dict = data_dictionary_dict
        self.dictionary_cf = dictionary_cf
        self._label_dictionary = label_dictionary
        self.external_dict = external_dictionary
        self._call_dictionary = Dictionary()

        if not hasattr(args, 'max_positions'):
            self._max_positions = 512
        else:
            self._max_positions = args.max_positions

        args.tokens_per_sample = self._max_positions
        self.seed = args.seed
        self.fields = params.fields

    @classmethod
    def load_dictionary(cls, args, filename, source=True, with_mask=True):
        """Load the dictionary from the filename

        Args:
            filename (str): the filename
        """
        dictionary = Dictionary.load(filename)
        if with_mask:
            dictionary.add_symbol('<mask>')
        return dictionary

    @classmethod
    def setup_task(cls, args, **kwargs):

        assert args.num_classes > 0, 'Must set --num-classes'
        assert args.num_external > 0, 'Must set --num-external'
        data_dictionary_dict = {}
        for field in params.fields:
            data_dictionary_dict[field] = cls.load_dictionary(
                args,
                os.path.join(args.data, 'self', field, 'dict.txt'),
                source=True
            )
            logger.info(f'| [input] {field} dictionary: {len(data_dictionary_dict[field])} types')

        # load vocabulary for internal functions
        label_dict = cls.load_dictionary(
            args,
            os.path.join(args.data, 'self', 'label', 'dict.txt'),
            source=False,
            with_mask=False,
        )
        print('| [internal function] dict: {} types'.format(len(label_dict)))

        # control flow label
        dictionary_cf = Dictionary.load(os.path.join(args.data, 'self', params.field_cf, 'dict.txt'))
        logger.info(f'{params.field_cf} dictionary: {len(dictionary_cf)} types')

        # load vocabulary of external callees
        external_dict = cls.load_dictionary(
            args,
            os.path.join(args.data, 'external_callee1', 'label', 'dict.txt'),
            source=False,
            with_mask=False,
        )
        print('| [external function] dictionary: {} types'.format(len(external_dict)))
        print('| [external function] embedding method: {}'.format(args.external_emb))

        return cls(args, data_dictionary_dict, label_dict, dictionary_cf, external_dict)

    def load_dataset_fields(self, split, target, combine=False):
        src_tokens = {}

        for field in self.fields:
            split_path = os.path.join(self.args.data, target, field, split)

            src_dataset = data_utils.load_indexed_dataset(
                split_path,
                self.source_dictionary[field],
                self.args.dataset_impl,
                combine=combine,
            )

            if src_dataset is None:
                raise FileNotFoundError('Dataset not found: {} ({})'.format(split, split_path))

            src_tokens[field] = PrependTokenDataset(
                RightPadDataset(
                    StripTokenDataset(
                        TruncateDataset(
                            src_dataset, self.max_positions()),
                        id_to_strip=self.source_dictionary[field].eos()
                    ),
                    pad_idx=self.source_dictionary[field].pad()
                ),
                self.source_dictionary[field].bos()
            )

        return src_tokens, src_dataset

    def load_dataset(self, split, epoch=1, combine=False, **kwargs):
        """Load a given dataset split.

        Args:
            split (str): name of the split (e.g., train, valid, test)
        """
        paths = utils.split_paths(self.args.data)
        assert len(paths) > 0

        target = {}

        src_tokens, src_dataset = self.load_dataset_fields(split, target='self')

        with data_utils.numpy_seed(self.args.seed + epoch):
            shuffle = np.random.permutation(len(src_dataset))

        net_input = dict()
        net_input['src_tokens'] = src_tokens
        net_input['src_lengths'] = NumelDataset(src_dataset, reduce=False)

        for i in range(self.args.num_calls):

            # load internal callees and callers
            callee_tokens, callee_dataset = self.load_dataset_fields(split, target=f'internal_callee{i+1}')
            caller_tokens, caller_dataset = self.load_dataset_fields(split, target=f'caller{i+1}')
            net_input[f'callee_tokens{i+1}'] = callee_tokens
            net_input[f'callee_lengths{i+1}'] = NumelDataset(callee_dataset, reduce=False)
            net_input[f'caller_tokens{i+1}'] = caller_tokens
            net_input[f'caller_lengths{i+1}'] = NumelDataset(caller_dataset, reduce=False)

            # load external callees
            external_path = os.path.join(self.args.data, f'external_callee{i+1}', 'label', split)
            external_dataset = data_utils.load_indexed_dataset(
                external_path,
                self.external_dict,
                self.args.dataset_impl,
                combine=combine,
            )

            if external_dataset is None:
                raise FileNotFoundError('Dataset not found: {} ({})'.format(split, external_path))

            net_input[f'external{i+1}'] = RightPadDataset(
                StripTokenDataset(
                    TruncateDataset(
                        external_dataset,
                        self.max_positions(),
                    ), id_to_strip=self.label_dictionary.eos()),
                pad_idx=self.label_dictionary.pad()
            )

        # Net input has multiple fields
        dataset = {
            'id': IdDataset(),
            'net_input': net_input,
            'target': target,
            'nsentences': NumSamplesDataset(),
            'ntokens': NumelDataset(src_dataset, reduce=True),
        }

        label_path = os.path.join(self.args.data, 'self', 'label', split)
        label_dataset = data_utils.load_indexed_dataset(
            label_path,
            self.label_dictionary,
            self.args.dataset_impl,
            combine=combine,
        )

        if label_dataset is None:
            raise FileNotFoundError('Dataset not found: {} ({})'.format(split, label_path))

        dataset.update(
            target=RightPadDataset(
                # OffsetTokensDataset(
                StripTokenDataset(
                    TruncateDataset(
                        label_dataset,
                        self.max_positions(),
                    ), id_to_strip=self.label_dictionary.eos()),

                pad_idx=self.label_dictionary.pad()
            )
        )

        nested_dataset = NestedDictionaryDataset(
            dataset,
            sizes=[src_dataset.sizes],
        )

        if self.args.no_shuffle:
            self.datasets[split] = nested_dataset
        else:
            # shuffle_sorted = np.sort(shuffle)
            self.datasets[split] = SortDataset(
                nested_dataset,
                # shuffle
                sort_order=[shuffle],
            )

        logger.info("Loaded {0} with #samples: {1}".format(split, len(self.datasets[split])))
        return self.datasets[split]

    def build_model(self, args):
        from fairseq import models
        model = models.build_model(args, self)

        model.register_classification_list(
            getattr(args, 'classification_head_name', 'func_name_pred'),
            num_classes=self.args.num_classes,
            num_external=self.args.num_external,
            num_calls=self.args.num_calls,
            external_emb=self.args.external_emb,
        )

        return model

    def max_positions(self):
        return self._max_positions

    @property
    def source_dictionary(self):
        return self.dictionary_dict

    @property
    def target_dictionary(self):
        return self.dictionary_dict

    @property
    def label_dictionary(self):
        return self._label_dictionary

    @property
    def target_cf_dictionary(self):
        return self.dictionary_cf

    @property
    def call_dictionary(self):
        return self._call_dictionary


from svgwrite import Drawing
from svgwrite.shapes import Line, Rect
from svgwrite.text import Text
from svgwrite.container import Group
from svgwrite.image import Image

from .logo_lookup import get_logo
from .draw_runners import DrawRunners
from .constants import (ORIGIN_X, ORIGIN_Y, ATBAT_W, ATBAT_HT, NAME_W,
                        TEXT_HOP, SCORE_W, SEPARATION,
                        AWAY_NAME_X, AWAY_SCORING_X,
                        HOME_NAME_X, HOME_SCORING_X, HASH_SEP, HASH_LEN, flip)

class DrawScorecard:
    def __init__(self):
        self.runner_drawer = DrawRunners()

    def draw(self, game):
        self.game = game
        self.players = game.players

        self.dwg = Drawing(debug=True, profile='full')

        self.draw_inning_stripes()
        self.draw_inning_separators()
        self.draw_inning_numbers()
        self.draw_team_boxes()
        self.draw_game()
        if self.game.innings and len(self.game.innings[0].halves) == 2:
            self.draw_pitcher_hash_marks()
            self.draw_pitcher_names()
        self.draw_score()
        self.draw_logos()

        return self.dwg

    def draw_logos(self):
        self.dwg.add(Image(get_logo(self.game.away),
                           (ORIGIN_X, ORIGIN_Y - 50), (40, 40)))
        self.dwg.add(Image(get_logo(self.game.home),
                           (ORIGIN_X + ATBAT_W + SEPARATION + ATBAT_W - 40,
                            ORIGIN_Y - 50), (40, 40)))

    def draw_score(self):
        away_score = 0
        home_score = 0
        for inning in self.game:
            for half_inning in inning:
                for atbat in half_inning:
                    away_score = atbat.away_score
                    home_score = atbat.home_score
        self.dwg.add(Text(away_score,
                          x=[ORIGIN_X + ATBAT_W], y=[ORIGIN_Y - ATBAT_HT],
                          class_='score', text_anchor='end'))
        self.dwg.add(Text(home_score,
                          x=[ORIGIN_X + ATBAT_W + SEPARATION],
                          y=[ORIGIN_Y - ATBAT_HT], class_='score',
                          text_anchor='start'))

    def get_team_box(self, id, ht):
        box = Group()
        box['id'] = id
        box['class'] = 'team-box'
        box.add(Rect((ORIGIN_X, ORIGIN_Y), (ATBAT_W, ht)))
        box.add(Line((ORIGIN_X + NAME_W, ORIGIN_Y),
                     (ORIGIN_X + NAME_W, ORIGIN_Y + ht)))
        box.add(Line((ORIGIN_X + NAME_W + SCORE_W, ORIGIN_Y),
                     (ORIGIN_X + NAME_W + SCORE_W, ORIGIN_Y + ht)))
        return box

    def draw_team_boxes(self):
        away_ht = sum(
            self.get_inning_height(inning) for inning in
            self.game.innings)
        away_team = self.get_team_box('away_team', away_ht)

        home_ht = away_ht
        if self.is_no_final_bottom():
            final_inning = self.game.innings[-1]
            home_ht = away_ht - len(final_inning.halves[0].atbats) * ATBAT_HT
        home_team = self.get_team_box('home_team', home_ht)
        flip(home_team)

        self.dwg.add(away_team)
        self.dwg.add(home_team)

    def draw_inning_stripes(self):
        y = ORIGIN_Y
        for i, inning in enumerate(self.game.innings):
            inning_ht = self.get_inning_height(inning)
            y += inning_ht
            if not i % 2:
                self.dwg.add(Rect((ORIGIN_X, y - inning_ht),
                                  (ATBAT_W, inning_ht),
                             class_='inning-fill'))

            if (not (i == len(self.game.innings) - 1 and
                     self.is_no_final_bottom())):
                if not i % 2:
                    self.dwg.add(Rect((ORIGIN_X + ATBAT_W + SEPARATION,
                                       y - inning_ht), (ATBAT_W, inning_ht),
                                 class_='inning-fill'))

    def draw_inning_separators(self):
        y = ORIGIN_Y
        for i, inning in enumerate(self.game.innings[:-1]):
            inning_ht = self.get_inning_height(inning)
            y += inning_ht
            self.dwg.add(Line((ORIGIN_X, y), (ORIGIN_X + ATBAT_W, y),
                              class_='team-box'))

            if (not (i == len(self.game.innings) - 2 and
                     self.is_no_final_bottom())):
                self.dwg.add(Line((ORIGIN_X + ATBAT_W + SEPARATION, y),
                                  (ORIGIN_X + 2 * ATBAT_W + SEPARATION, y),
                                  class_='team-box'))

    def draw_inning_numbers(self):
        y = ORIGIN_Y
        for i, inning in enumerate(self.game.innings):
            inning_ht = self.get_inning_height(inning)
            y += inning_ht
            self.draw_inning_number(i + 1, y - inning_ht / 2)

    def draw_inning_number(self, num, y):
        self.dwg.add(Text(num, x=[ORIGIN_X + ATBAT_W + SEPARATION / 2], y=[y],
                          class_='inning-num', text_anchor='middle',
                          dominant_baseline='middle'))

    def draw_game(self):
        self.y = ORIGIN_Y + ATBAT_HT
        for inning in self.game.innings:
            self.draw_inning(inning)

    def draw_inning(self, inning):
        inning_start_y = self.y
        inning_end_y = 0
        for half_inning in inning.halves:
            self.y = inning_start_y
            self.draw_half_inning(half_inning)
            inning_end_y = max(inning_end_y, self.y)
        self.y = inning_end_y

    def draw_half_inning(self, half_inning):
        for atbat in half_inning.atbats:
            self.set_x_and_anchor(half_inning.num)
            is_home_team_batting = half_inning.num % 1.0
            self.draw_atbat(atbat, is_home_team_batting)
            self.y += ATBAT_HT

    def draw_atbat(self, atbat, is_home_team_batting):
        atbat_group = Group()
        atbat_group.set_desc(atbat.get_description())
        atbat_group.add(self.get_batter_name_text(atbat))
        atbat_group.add(self.get_scoring_text(atbat))
        self.runner_drawer.execute(self.y, atbat, atbat_group,
                                   is_home_team_batting)
        self.dwg.add(atbat_group)

    def set_x_and_anchor(self, inning):
        if self.is_home_team_batting(inning):
            self.name_x = HOME_NAME_X
            self.scoring_x = HOME_SCORING_X
            self.name_anchor = 'start'
        else:
            self.name_x = AWAY_NAME_X
            self.scoring_x = AWAY_SCORING_X
            self.name_anchor = 'end'

    def get_batter_name_text(self, atbat):
        return Text(self.players.get(atbat.batter),
                    x=[self.name_x], y=[self.y - TEXT_HOP],
                    class_='batter-name',
                    text_anchor=self.name_anchor)

    def get_scoring_text(self, atbat):
        text = Text(atbat.scoring.code,
                    x=[self.scoring_x], y=[self.y - TEXT_HOP],
                    class_=atbat.scoring.result,
                    text_anchor='middle')
        if atbat.scoring.code == 'Kl':
            text.text = 'K'
            text.translate(2 * self.scoring_x, 0)
            text.scale(-1, 1)
        return text

    def draw_pitcher_hash_marks(self):
        self.away_hash_ys = []
        self.home_hash_ys = []

        # Starting pitchers
        self.home_pitchers = [self.game.innings[0].halves[0].atbats[0].pitcher]
        self.away_pitchers = [self.game.innings[0].halves[1].atbats[0].pitcher]

        self.y = ORIGIN_Y
        self.draw_both_hashes()
        for inning in self.game.innings:
            inning_start = self.y
            for half_inning in inning.halves:
                for atbat in half_inning.atbats:
                    if (atbat.pitcher != self.home_pitchers[-1] and
                            atbat.pitcher != self.away_pitchers[-1]):
                        self.draw_hash(half_inning.num)
                        self.swap_pitcher(atbat.pitcher, half_inning.num)
                    self.y += ATBAT_HT
                self.y = inning_start
            self.y = (inning_start + self.get_inning_height(inning))

        self.draw_hash(1.0)
        if self.is_no_final_bottom():
            self.y = self.y - ATBAT_HT * len(inning.halves[0].atbats)
        self.draw_hash(1.5)

    def get_inning_height(self, inning):
        return ATBAT_HT * max(len(h.atbats) for h in inning.halves)

    def draw_both_hashes(self):
        self.draw_hash(1.0)
        self.draw_hash(1.5)

    def draw_hash(self, inning):
        line = Line((ORIGIN_X + ATBAT_W + HASH_SEP, self.y),
                    (ORIGIN_X + ATBAT_W + HASH_SEP + HASH_LEN, self.y))
        if self.is_home_team_batting(inning):
            flip(line)
            self.home_hash_ys.append(self.y)
            line['class'] = 'away-pitcher-hash'
        else:
            self.away_hash_ys.append(self.y)
            line['class'] = 'home-pitcher-hash'
        self.dwg.add(line)

    def swap_pitcher(self, pitcher, inning_num):
        if self.is_home_team_batting(inning_num):
            self.away_pitchers.append(pitcher)
        else:
            self.home_pitchers.append(pitcher)

    def is_no_final_bottom(self):
        return self.game.innings and not len(self.game.innings[-1].halves) == 2

    def draw_pitcher_names(self):
        self._draw_pitcher_names(self.away_hash_ys, False)
        self._draw_pitcher_names(self.home_hash_ys, True)

    def _draw_pitcher_names(self, y_array, flip):
        for i, y in enumerate(y_array[:-1]):
            y_t = (y + y_array[i + 1]) / 2
            if flip:
                x = ORIGIN_X + ATBAT_W + SEPARATION - HASH_SEP - HASH_LEN / 2
                pitcher_name = self.players[self.away_pitchers[i]]
            else:
                x = ORIGIN_X + ATBAT_W + HASH_SEP + HASH_LEN / 2
                pitcher_name = self.players[self.home_pitchers[i]]
            txt = Text(pitcher_name, x=[x], y=[y_t], text_anchor='middle',
                       dominant_baseline='middle')
            if flip:
                txt['id'] = 'away-pitcher-%02d' % i
                txt['class'] = 'away-pitcher-name'
                txt.rotate(-90, (x, y_t))
            else:
                txt['id'] = 'home-pitcher-%02d' % i
                txt['class'] = 'home-pitcher-name'
                txt.rotate(90, (x, y_t))

            self.dwg.add(txt)

    def is_home_team_batting(self, inning):
        return inning % 1.0


import cv2
import numpy as np

net = cv2.dnn.readNet("yolov3.weights", "yolov3.cfg")
classes = []
with open ("coco.names","r") as f:
    classes = [line.strip() for line in f.readlines()]
layer_names = net.getLayerNames()
outputlayers = [layer_names[i[0]-1] for i in net.getUnconnectedOutLayers()]
# print(layer_names)

# img = cv2.imread("images/sample.jpg")
img = cv2.imread("images/sample2.jpg")
# img = cv2.imread("images/sample.jpg")
# img = cv2.resize(img, None,fx=0.4, fy=0.4)
height, width, channels  = img.shape

blob = cv2.dnn.blobFromImage(img,0.00392,(416,416),(0,0,0),True,crop=False)
# for b in blob:
#     for n, img_blob in enumerate(b):
#         cv2.imshow(str(n),img_blob)

net.setInput(blob)
outs = net.forward(outputlayers)

# print(outs)
for out in outs:
    for detection in out:
        scores = detection[5:]
        class_id = np.argmax(scores)
        confidence = scores[class_id]
        if confidence > 0.5:
            # Object detected
            center_x = int(detection[0] * width)
            center_y = int(detection[1] * height)
            w = int(detection[2] * width)
            h = int(detection[3] * height)
            # Rectangle coordinates
            x = int(center_x - w / 2)
            y = int(center_y - h / 2)
            # cv2.circle(img,(center_x,center_y),10,(255,0,0),2)
            cv2.rectangle(img,(x,y),(x+w,y+h),(0,255,0),2)

cv2.imshow("Image",img)
cv2.waitKey(0)
cv2.destroyAllWindows()


from customtkinter import CTk,CTkTabview
from paginas import inicio,saques,depositos,rendimento,relatorios,bens

screenMaster = CTk()
screenMaster.title("Crowler Fin")
screenMaster.geometry("900x600")
screenMaster.resizable(False,False)
screenMaster.iconbitmap("utils/01.ico")

menu = CTkTabview(master=screenMaster,width=890,height=595)
tab_inicio = menu.add("Inicio")
tab_depositos = menu.add("Depositos")
tab_saques = menu.add("Saques")
tab_bens = menu.add("Bens")
tab_relatorios = menu.add("Relatórios")
tab_rendimento = menu.add("Rendimento Financeiro")
menu.pack()

inicio.start_Inicio(tab_inicio)
depositos.start_Depositos(tab_depositos)
saques.start_Saques(tab_saques)
bens.start_Bens(tab_bens)
relatorios.start_Relatorios(tab_relatorios)
rendimento.start_Rendimento(tab_rendimento)

screenMaster.mainloop()


import numpy as np, pyemma as py
# from msmbuilder.decomposition.tica import tICA
from sklearn.kernel_approximation import Nystroem

"""modified from https://github.com/msmbuilder/msmbuilder/blob/master/msmbuilder/decomposition/ktica.py"""
"""reference: [1] Schwantes, Christian R., and Vijay S. Pande. J. Chem Theory Comput. 11.2 (2015): 600--608."""

class Kernel_tica(object):
    def __init__(self, n_components, lag_time,
                 gamma,             # gamma value for rbf kernel
                 n_components_nystroem=100,  # number of components for Nystroem kernel approximation
                 landmarks = None,
                 shrinkage = None,
                 weights='empirical'    # if 'koopman', use Koopman reweighting for tICA (see Wu, Hao, et al. "Variational Koopman models: slow collective variables and molecular kinetics from short off-equilibrium simulations." The Journal of Chemical Physics 146.15 (2017): 154104.)
                 ):
        self._n_components = n_components
        self._lag_time = lag_time
        self._n_components_nystroem = n_components_nystroem
        self._landmarks = landmarks
        self._gamma = gamma
        self._nystroem = Nystroem(gamma=gamma, n_components=n_components_nystroem)
        self._weights = weights
        # self._tica = tICA(n_components=n_components, lag_time=lag_time, shrinkage=shrinkage)
        self._shrinkage = shrinkage
        return

    def fit(self, sequence_list):
        if self._landmarks is None:
            self._nystroem.fit(np.concatenate(sequence_list))
        else:
            print("using landmarks")
            self._nystroem.fit(self._landmarks)
        sequence_transformed = [self._nystroem.transform(item) for item in sequence_list]
        # define tica object at fit() with sequence_list supplied for initialization, as it is required by
        # Koopman reweighting
        self._tica = py.coordinates.tica(sequence_transformed, lag=self._lag_time,
                                         dim=self._n_components, kinetic_map=True,
                                         weights=self._weights)
        return

    def transform(self, sequence_list):
        return self._tica.transform(
            [self._nystroem.transform(item) for item in sequence_list])

    def fit_transform(self, sequence_list):
        self.fit(sequence_list)
        return self.transform(sequence_list)

    def score(self, sequence_list):
        model = self.__class__(n_components = self._n_components, lag_time=self._lag_time, gamma=self._gamma,
                               n_components_nystroem=self._n_components_nystroem, landmarks=self._landmarks,
                               shrinkage=self._shrinkage)
        model.fit(sequence_list)
        return np.sum(model._tica.eigenvalues)


# Get user input for the first number
num1 = float(input("Please enter a number: "))

# Get user input for the second number
num2 = float(input("Please enter a second number: "))

# Calculate the addition and subtraction
addition_result = num1 + num2
subtraction_result = num1 - num2

# Display the results
print(f"Addition result: {num1} + {num2} = {addition_result}")
print(f"Subtraction result: {num1} - {num2} = {subtraction_result}")


import logging
from spaceone.core.service import *
from spaceone.monitoring.manager.data_source_manager import DataSourceManager

_LOGGER = logging.getLogger(__name__)

@authentication_handler
@authorization_handler
@event_handler
class DataSourceService(BaseService):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.data_source_mgr: DataSourceManager = self.locator.get_manager('DataSourceManager')

    @check_required(['options'])
    def init(self, params):
        """ init plugin by options
        """
        return self.data_source_mgr.init(params)

    @transaction
    @check_required(['options', 'secret_data'])
    def verify(self, params):
        """ Verifying data source plugin

        Args:
            params (dict): {
                'schema': 'str',
                'options': 'dict',
                'secret_data': 'dict'
            }

        Returns:
            plugin_verify_response (dict)
        """
        self.data_source_mgr.verify(params)


# import the necessary packages
from skimage import feature
import numpy as np
from sklearn.svm import LinearSVC
from imutils import paths
import os
import cv2
import csv

class LocalBinaryPatterns:
	def __init__(self, numPoints, radius):
		# store the number of points and radius
		self.numPoints = numPoints
		self.radius = radius

	def describe(self, image, eps=1e-7):
		# compute the Local Binary Pattern representation
		# of the image, and then use the LBP representation
		# to build the histogram of patterns
		lbp = feature.local_binary_pattern(image, self.numPoints,
			self.radius, method="uniform")
		(hist, _) = np.histogram(lbp.ravel(),
			bins=np.arange(0, self.numPoints + 3),
			range=(0, self.numPoints + 2))

		# normalize the histogram
		hist = hist.astype("float")
		hist /= (hist.sum() + eps)

		# return the histogram of Local Binary Patterns
		return hist

# initialize the local binary patterns descriptor along with
# the data and label lists
desc = LocalBinaryPatterns(48,16)
data = []
labels = []

path = "/Users/ahmetsina/PycharmProjects/GenderAnalyzer/FeatureExtraction/images/aligned_cropped_test_images"
images_path = os.listdir(path)
images_lbp = []
for i in images_path:
    imagePath = path + "/" + i
    print(imagePath)
    image = cv2.imread(imagePath)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    hist = desc.describe(gray)
    images_lbp.append(hist)

with open("FeatureExtraction/lbp_aligned_test.csv","w") as f:
    writer = csv.writer(f)
    writer.writerows(images_lbp)
# train a Linear SVM on the data

model = LinearSVC(C=100.0, random_state=42)
model.fit(data, labels)


# -*- coding: utf-8 -*-

from odoo import api, models

class Session(models.Model):
    _inherit = "pos.session"

    @api.multi
    def action_pos_session_closing_control(self):
        super(Session, self).action_pos_session_closing_control()
        return self.env.ref("evap_pos.action_report_pos_session").\
            report_action(self)

    @api.multi
    def get_discount_data(self):
        res = {}
        total_discount = 0.0

        orders = self.order_ids.filtered(
            lambda order: order.lines.filtered('discount')
        )

        for order in orders:
            customer = order.partner_id
            for line in order.lines:
                discount = (line.price_unit * line.qty) - line.price_subtotal
                if customer.id not in res:
                    res[customer.id] = \
                        [customer.name, discount, line.currency_id]
                else:
                    res[customer.id][1] += discount

                total_discount += discount

        return {
            'vals': list(res.values()),
            'total_discount': total_discount,
            'currency': self.env.user.company_id.currency_id
        }


from tsp.ga.crossover import CrossoverStrategy
from tsp.ga.chromosome import Chromosome

class MCrossoverOperator(CrossoverStrategy):
  def __init__(self):
    self.splits = 3

  def crossover(self, p1, p2):
    split1 = len(p1.genes) / self.splits
    split2 = len(p1.genes) - split1

    p1Splits = self.get_splits(p1.genes, split1, split2)
    p2Splits = self.get_splits(p2.genes, split1, split2)

    children = []
    for i in xrange(self.splits):
      children += self.do_crossover(p1Splits[i], p2Splits, p1.graph)
      children += self.do_crossover(p2Splits[i], p1Splits, p2.graph)
    assert len(children) == 2 * self.splits * self.splits
    return sorted(children, key=lambda c: c.fitness())[:2]

  def get_splits(self, p, split1, split2):
    return [p[:split1], p[split1:split2], p[split2:]]

  def do_crossover(self, addition, splits, graph):
      children = [self.make_gene(addition, splits[:i], splits[i:])
                  for i in xrange(len(splits))]
      return [Chromosome(child, graph) for child in children]

  def make_gene(self, addition, start, end):
      genes = []
      for sec in start:
        genes += [i for i in sec if i not in addition]
      genes += addition
      for sec in end:
        genes += [i for i in sec if i not in addition]
      return genes