from easydict import EasyDict as edict

from mindspore.train.model import Model
from mindspore.train import amp
from mindspore.train.amp import _config_level
from mindspore.common import dtype as mstype
from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell
from mindspore.parallel._utils import _get_parallel_mode
from mindspore.train.model import ParallelMode
from mindspore.train.amp import _do_keep_batchnorm_fp32
from mindspore.train.amp import _add_loss_network
from mindspore import context
from mindspore import nn
from mindspore import Tensor
from mindspore.ops import composite as C
from mindspore.ops import operations as P
from mindspore.ops import functional as F
from mindspore.ops.operations import NPUGetFloatStatus
from mindspore.ops.operations import NPUAllocFloatStatus
from mindspore.ops.operations import NPUClearFloatStatus
from mindspore.ops.operations import ReduceSum
from mindspore.ops.operations import LessEqual
from mindspore.parallel._utils import _get_gradients_mean
from mindspore.parallel._utils import _get_device_num
from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
from mindspore.common.parameter import Parameter
from mindspore.nn.wrap.loss_scale import _grad_overflow
from mindspore.nn import Cell
from mindspore import ParameterTuple

from mindarmour.utils.logger import LogUtil
from mindarmour.utils import _check_param as validator
from mindarmour.utils._check_param import check_value_positive, check_param_type
from mindarmour.utils._check_param import check_int_positive, check, check_value_type
from ..mechanisms.mechanisms import _MechanismsParamsUpdater

LOGGER = LogUtil.get_instance()
TAG = 'DP model'

_grad_scale = C.MultitypeFuncGraph("grad_scale")
_reciprocal = P.Reciprocal()

@_grad_scale.register("Tensor", "Tensor")
def tensor_grad_scale(scale, grad):
    """ grad scaling """
    return grad * F.cast(_reciprocal(scale), F.dtype(grad))

class DPModel(Model):
    """
    DPModel is used for constructing a model for differential privacy training.
    This class is overload mindspore.train.Model.

    For details, please check `Protecting User Privacy with Differential Privacy Mechanism
    <https://mindspore.cn/mindarmour/docs/en/master/protect_user_privacy_with_differential_privacy.html#%E5%B7%AE%E5%88%86%E9%9A%90%E7%A7%81>`_.

    Args:
        micro_batches (int): The number of small batches split from an original
            batch. Default: ``2``.
        norm_bound (float): The norm bound that is used to clip the gradient of
         each sample. Default: ``1.0``.
        noise_mech (Mechanisms): The object can generate the different type of
            noise. Default: ``None``.
        clip_mech (Mechanisms): The object is used to update the adaptive clip.
            Default: ``None``.
        optimizer (Cell): Optimizer used for differential privacy training, which can be original mindspore
            optimizers (for example, Momentum optimizer) or optimizers generated by DPOptimizerClassFactory.
            Default: ``nn.Momentum``.

    Raises:
        ValueError: If `optimizer` is ``None``.
        ValueError: If `optimizer` is not DPOptimizer and `noise_mech` is ``None``.
        ValueError: If `optimizer` is DPOptimizer and `noise_mech` is not ``None``.
        ValueError: If `noise_mech` or DPOptimizer's mech method is adaptive while `clip_mech` is not ``None``.

    """

    def __init__(self, micro_batches=2, norm_bound=1.0, noise_mech=None,
                 clip_mech=None, optimizer=nn.Momentum, **kwargs):
        if micro_batches:
            self._micro_batches = check_int_positive('micro_batches',
                                                     micro_batches)
        else:
            self._micro_batches = None
        norm_bound = check_param_type('norm_bound', norm_bound, float)
        norm_bound = check_value_positive('norm_bound', norm_bound)
        norm_bound = Tensor(norm_bound, mstype.float32)
        self._norm_bound = Parameter(norm_bound, 'norm_bound')
        if optimizer is None:
            msg = 'Optimizer need to be set, but got None.'
            LOGGER.error(TAG, msg)
            raise ValueError(msg)
        opt_name = optimizer.__class__.__name__
        # Check whether noise_mech and DPOptimizer are both None or not None, if so, raise ValueError.
        # And check whether noise_mech or DPOtimizer's mech method is adaptive while clip_mech is not None,
        # if so, raise ValuerError too.
        if noise_mech is not None and "DPOptimizer" in opt_name:
            msg = 'DPOptimizer is not supported while noise_mech is not None'
            LOGGER.error(TAG, msg)
            raise ValueError(msg)
        if noise_mech is None:
            if "DPOptimizer" in opt_name:
                if 'Ada' in optimizer._mech.__class__.__name__ and clip_mech is not None:
                    msg = "When DPOptimizer's mech method is adaptive, clip_mech must be None."
                    LOGGER.error(TAG, msg)
                    raise ValueError(msg)
            else:
                msg = 'DPModel should set noise_mech or DPOptimizer configure, ' \
                      'please refer to example.'
                LOGGER.error(TAG, msg)
                raise ValueError(msg)
        self._noise_mech = noise_mech
        if noise_mech is not None:
            if 'Ada' in noise_mech.__class__.__name__ and clip_mech is not None:
                msg = 'When noise_mech is Adaptive, clip_mech must be None.'
                LOGGER.error(TAG, msg)
                raise ValueError(msg)

        if clip_mech is None or isinstance(clip_mech, Cell):
            self._clip_mech = clip_mech
        super(DPModel, self).__init__(optimizer=optimizer, **kwargs)

        # judge device_target, only GPU or Ascend is supported until now
        device = context.get_context("device_target")
        if device not in ["GPU", "Ascend"]:
            msg = "'device_target' for DP training should be 'GPU' or 'Ascend', but got {}.".format(device)
            LOGGER.error(TAG, msg)
            raise ValueError(msg)

    def _amp_build_train_network(self, network, optimizer, loss_fn=None,
                                 level='O0', **kwargs):
        """
        Build the mixed precision training cell automatically.

        Args:
            network (Cell): Definition of the network.
            loss_fn (Union[None, Cell]): Definition of the loss_fn. If None,
                the `network` should have the loss inside. Default: ``None``.
            optimizer (Optimizer): Optimizer to update the Parameter.
            level (str): Supports [O0, O2]. Default: "O0".

                - O0: Do not change.
                - O2: Cast network to float16, keep batchnorm and `loss_fn`
                  (if set) run in float32, using dynamic loss scale.

            cast_model_type (:class:`mindspore.dtype`): Supports `mstype.float16`
                or `mstype.float32`. If set to `mstype.float16`, use `float16`
                mode to train. If set, overwrite the level setting.
            keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set,
                overwrite the level setting.
            loss_scale_manager (Union[None, LossScaleManager]): If None, not
                scale the loss, or else scale the loss by LossScaleManager.
                If set, overwrite the level setting.
        """
        check_value_type('network', network, nn.Cell, None)
        check_value_type('optimizer', optimizer, nn.Optimizer, None)
        check('level', level, "", ['O0', 'O2'], validator.IN, None)
        self._check_kwargs(kwargs)
        config = dict(_config_level[level], **kwargs)
        config = edict(config)

        if config.cast_model_type == mstype.float16:
            network.to_float(mstype.float16)

            if config.keep_batchnorm_fp32:
                _do_keep_batchnorm_fp32(network)

        if loss_fn:
            network = _add_loss_network(network, loss_fn,
                                        config.cast_model_type)

        if _get_parallel_mode() in (
                ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL):
            network = _VirtualDatasetCell(network)

        loss_scale = 1.0
        if config.loss_scale_manager is not None:
            loss_scale_manager = config.loss_scale_manager
            loss_scale = loss_scale_manager.get_loss_scale()
            update_cell = loss_scale_manager.get_update_cell()
            if update_cell is not None:
                # only cpu not support `TrainOneStepWithLossScaleCell` for control flow.
                if not context.get_context("enable_ge") and context.get_context(
                        "device_target") == "CPU":
                    msg = "Only `loss_scale_manager=None` and " \
                          "`loss_scale_manager=FixedLossScaleManager(drop_overflow" \
                          "_update=False)` are supported in current version. " \
                          "If you use `O2` option, please use " \
                          "`loss_scale_manager=None` or `FixedLossScaleManager`"
                    LOGGER.error(TAG, msg)
                    raise ValueError(msg)
                network = _TrainOneStepWithLossScaleCell(network,
                                                         optimizer,
                                                         scale_update_cell=update_cell,
                                                         micro_batches=self._micro_batches,
                                                         norm_bound=self._norm_bound,
                                                         clip_mech=self._clip_mech,
                                                         noise_mech=self._noise_mech).set_train()
                return network

        network = _TrainOneStepCell(network,
                                    optimizer,
                                    self._norm_bound,
                                    loss_scale,
                                    micro_batches=self._micro_batches,
                                    clip_mech=self._clip_mech,
                                    noise_mech=self._noise_mech).set_train()
        return network

    def _build_train_network(self):
        """Build train network"""
        network = self._network
        if self._micro_batches:
            if self._optimizer:
                if self._loss_scale_manager_set:
                    network = self._amp_build_train_network(network,
                                                            self._optimizer,
                                                            self._loss_fn,
                                                            level=self._amp_level,
                                                            loss_scale_manager=self._loss_scale_manager,
                                                            keep_batchnorm_fp32=self._keep_bn_fp32)
                else:
                    network = self._amp_build_train_network(network,
                                                            self._optimizer,
                                                            self._loss_fn,
                                                            level=self._amp_level,
                                                            keep_batchnorm_fp32=self._keep_bn_fp32)
            elif self._loss_fn:
                network = nn.WithLossCell(network, self._loss_fn)
        else:
            if self._optimizer:
                if self._loss_scale_manager_set:
                    network = amp.build_train_network(network,
                                                      self._optimizer,
                                                      self._loss_fn,
                                                      level=self._amp_level,
                                                      loss_scale_manager=self._loss_scale_manager,
                                                      keep_batchnorm_fp32=self._keep_bn_fp32)
                else:
                    network = amp.build_train_network(network,
                                                      self._optimizer,
                                                      self._loss_fn,
                                                      level=self._amp_level,
                                                      keep_batchnorm_fp32=self._keep_bn_fp32)
            elif self._loss_fn:
                network = nn.WithLossCell(network, self._loss_fn)

        if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL,
                                   ParallelMode.AUTO_PARALLEL):
            network.set_auto_parallel()
        return network

class _ClipGradients(nn.Cell):
    """
    Clip gradients.

    Inputs:
        grads (tuple[Tensor]): Gradients to clip.
        clip_norm (float): The l2-norm bound used to clip the gradients.
        cur_norm (float): The l2-norm of grads. If None, the norm will be
            calculated in this function. Default: ``None``.

    Outputs:
        tuple[Tensor], clipped gradients.
    """

    def __init__(self):
        super(_ClipGradients, self).__init__()
        self._add = P.Add()
        self._reduce_sum = P.ReduceSum()
        self._square_all = P.Square()
        self._sqrt = P.Sqrt()

    def construct(self, grads, clip_norm, cur_norm=None):
        """
        construct a compute flow.
        """
        if cur_norm is None:
            # calculate current l2-norm of grads
            square_sum = Tensor(0, mstype.float32)
            for grad in grads:
                square_sum = self._add(square_sum, self._reduce_sum(self._square_all(grad)))
            cur_norm = self._sqrt(square_sum)

        if cur_norm <= clip_norm:
            return grads

        new_grads = ()
        for grad in grads:
            clipped_grad = grad * (clip_norm / cur_norm)
            new_grads = new_grads + (clipped_grad,)
        return new_grads

class _TupleAdd(nn.Cell):
    def __init__(self):
        super(_TupleAdd, self).__init__()
        self.add = P.Add()
        self.hyper_map = C.HyperMap()

    def construct(self, input1, input2):
        """Add two tuple of data."""
        out = self.hyper_map(self.add, input1, input2)
        return out

class _TrainOneStepWithLossScaleCell(Cell):
    r"""
    Network training with loss scaling.

    This is a training step with loss scaling. It takes a network, an optimizer
    and possibly a scale update Cell as args. The loss scale value can be
    updated in both host side or device side. The TrainOneStepWithLossScaleCell
    will be compiled to be graph which takes `data`, `label`, `sens` as input
    data. The `sens` is acting as loss scaling value. If you want to update it
    on host side, the value should be provided. If `sens` is not given, the loss
    scale update logic should be provied by `scale_update_cell`. If
    `scale_update_cell` is not None and `sens` is provided, the
    `scale_update_cell` will be ignored.

    Args:
        network (Cell): The training network.
        optimizer (Cell): Optimizer for updating the weights.
        scale_update_cell(Cell): The loss scaling update logic cell.
            Default: ``None``.
        micro_batches (int): The number of small batches split from an original
            batch. Default: ``None``.
        norm_bound (Tensor): The norm bound that is used to clip the gradient of
         each sample. Default: 1.0.
        noise_mech (Mechanisms): The object can generate the different type of
            noise. Default: ``None``.

    Inputs:
        - **inputs** (Tensor) - Tensor of shape :math:`(N, \ldots)`.
        - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`.
        - **scaling_sens** (Tensor) - Tensor of shape :math:`()`.

    Outputs:
        Tuple of 3 Tensor, the loss, overflow flag and current loss scaling value.

        - **loss** (Tensor) -  Tensor with shape :math:`()`.
        - **overflow** (Tensor) -  Tensor with shape :math:`()`, type is bool.
        - **loss_scale** (Tensor) -  Tensor with shape :math:`()`.
    """

    def __init__(self, network, optimizer, scale_update_cell=None,
                 micro_batches=None, norm_bound=1.0, noise_mech=None,
                 clip_mech=None):
        super(_TrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False)
        self.network = network
        self.network.set_grad()
        self.network.add_flags(defer_inline=True)
        self.weights = ParameterTuple(network.trainable_params())
        self.optimizer = optimizer
        self.grad = C.GradOperation(get_by_list=True, sens_param=True)
        self.hyper_map = C.HyperMap()
        if context.get_context("device_target") == "GPU":
            self.gpu_target = True
            self.float_status = P.FloatStatus()
            self.addn = P.AddN()
            self.reshape = P.Reshape()
        else:
            self.gpu_target = False
            self.alloc_status = NPUAllocFloatStatus()
            self.get_status = NPUGetFloatStatus()
            self.clear_status = NPUClearFloatStatus()
        self.reduce_sum = ReduceSum(keep_dims=False)
        self.base = Tensor(1, mstype.float32)
        self.less_equal = LessEqual()
        self.allreduce = P.AllReduce()
        self.parallel_mode = _get_parallel_mode()
        self.grad_reducer = F.identity
        self.reducer_flag = self.parallel_mode in [ParallelMode.DATA_PARALLEL,
                                                   ParallelMode.HYBRID_PARALLEL]
        if self.reducer_flag:
            mean = _get_gradients_mean()
            degree = _get_device_num()
            self.grad_reducer = DistributedGradReducer(optimizer.parameters,
                                                       mean, degree)
        self.is_distributed = self.parallel_mode != ParallelMode.STAND_ALONE

        self.loss_scale = None
        self.loss_scaling_manager = scale_update_cell
        if scale_update_cell:
            self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32),
                                        name="loss_scale")
        self.add_flags(has_effect=True)

        # dp params
        self._micro_batches = micro_batches
        self._norm_bound = norm_bound
        self._split = P.Split(0, self._micro_batches)
        self._clip_by_global_norm = _ClipGradients()
        self._noise_mech = noise_mech
        self._clip_mech = clip_mech
        self._add = P.Add()
        self._norm = nn.Norm()
        self._tuple_add = _TupleAdd()
        self._hyper_map = C.HyperMap()
        self._micro_float = Tensor(micro_batches, mstype.float32)
        self._zero = Tensor(0, mstype.float32)
        self._assign = P.Assign()
        self._div = P.Div()
        self._sqrt = P.Sqrt()
        self._reduce_sum = P.ReduceSum()
        self._square_all = P.Square()
        self._less = P.Less()
        self._cast = P.Cast()

        self._noise_mech_param_updater = None
        if self._noise_mech is not None and self._noise_mech._decay_policy is not None:
            self._noise_mech_param_updater = _MechanismsParamsUpdater(
                decay_policy=self._noise_mech._decay_policy,
                decay_rate=self._noise_mech._noise_decay_rate,
                cur_noise_multiplier=self._noise_mech._noise_multiplier,
                init_noise_multiplier=self._noise_mech._initial_noise_multiplier)

    def construct(self, data, label, sens=None):
        """
        construct a compute flow.
        """
        init = False
        if not self.gpu_target:
            # init overflow buffer
            init = self.alloc_status()
            # clear overflow buffer
            self.clear_status(init)

        if sens is None:
            scaling_sens = self.loss_scale
        else:
            scaling_sens = sens

        # DP clip
        weights = self.weights
        record_datas = self._split(data)
        record_labels = self._split(label)
        # first index
        loss = self.network(record_datas[0], record_labels[0])
        scaling_sens_filled = C.ones_like(loss) * F.cast(scaling_sens,
                                                         F.dtype(loss))
        record_grad = self.grad(self.network, weights)(record_datas[0],
                                                       record_labels[0],
                                                       scaling_sens_filled)

        beta = self._zero
        square_sum = self._zero
        for grad in record_grad:
            square_sum = self._add(square_sum,
                                   self._reduce_sum(self._square_all(grad)))
        norm_grad = self._sqrt(square_sum)
        beta = self._add(beta,
                         self._cast(self._less(norm_grad, self._norm_bound),
                                    mstype.float32))
        record_grad = self._clip_by_global_norm(record_grad,
                                                self._norm_bound, norm_grad)
        grads = record_grad
        total_loss = loss
        for i in range(1, self._micro_batches):
            loss = self.network(record_datas[i], record_labels[i])
            scaling_sens_filled = C.ones_like(loss) * F.cast(scaling_sens,
                                                             F.dtype(loss))
            record_grad = self.grad(self.network, weights)(record_datas[i],
                                                           record_labels[i],
                                                           scaling_sens_filled)

            square_sum = self._zero
            for grad in record_grad:
                square_sum = self._add(square_sum,
                                       self._reduce_sum(self._square_all(grad)))
            norm_grad = self._sqrt(square_sum)
            beta = self._add(beta,
                             self._cast(self._less(norm_grad, self._norm_bound),
                                        mstype.float32))

            record_grad = self._clip_by_global_norm(record_grad,
                                                    self._norm_bound, norm_grad)
            grads = self._tuple_add(grads, record_grad)
            total_loss = P.Add()(total_loss, loss)
        loss = P.Div()(total_loss, self._micro_float)
        beta = self._div(beta, self._micro_batches)

        if self._noise_mech is not None:
            grad_noise_tuple = ()
            for grad_item in grads:
                grad_noise = self._noise_mech(grad_item)
                grad_noise_tuple = grad_noise_tuple + (grad_noise,)
            grads = self._tuple_add(grads, grad_noise_tuple)
            grads = self._hyper_map(F.partial(_grad_scale, self._micro_float),
                                    grads)
            # update mech parameters

            if self._noise_mech_param_updater is not None:
                multiplier = self._noise_mech_param_updater()
                loss = F.depend(loss, multiplier)

        grads = self.hyper_map(F.partial(_grad_scale, scaling_sens), grads)
        # apply grad reducer on grads
        grads = self.grad_reducer(grads)
        # get the overflow buffer
        if not self.gpu_target:
            self.get_status(init)
            # sum overflow buffer elements, 0:not overflow , >0:overflow
            flag_sum = self.reduce_sum(init, (0,))
        else:
            flag_sum = self.hyper_map(F.partial(_grad_overflow), grads)
            flag_sum = self.addn(flag_sum)
            # convert flag_sum to scalar
            flag_sum = self.reshape(flag_sum, (()))
        if self.is_distributed:
            # sum overflow flag over devices
            flag_reduce = self.allreduce(flag_sum)
            cond = self.less_equal(self.base, flag_reduce)
        else:
            cond = self.less_equal(self.base, flag_sum)
        overflow = cond
        if sens is None:
            overflow = self.loss_scaling_manager(self.loss_scale, cond)
        # if there is no overflow, do optimize
        if overflow:
            opt = False
        else:
            opt = self.optimizer(grads)
        ret = (loss, cond, scaling_sens)

        if self._clip_mech is not None:
            next_norm_bound = self._clip_mech(beta, self._norm_bound)
            P.assign(self._norm_bound, next_norm_bound)
        return F.depend(ret, opt)

class _TrainOneStepCell(Cell):
    r"""
    Network training package class.

    Wraps the network with an optimizer. The resulting Cell be trained with
    input data and label. Backward graph will be created in the construct
    function to do parameter updating. Different parallel modes are available
    to run the training.

    Args:
        network (Cell): The training network.
        optimizer (Cell): Optimizer for updating the weights.
        sens (Number): The scaling number to be filled as the input of back
            propagation. Default value is 1.0.
        micro_batches (int): The number of small batches split from an original
            batch. Default: ``None``.
        norm_bound (Tensor): The norm bound that is used to clip the gradient of
         each sample. Default: 1.0.
        noise_mech (Mechanisms): The object can generate the different type
            of noise. Default: ``None``.
        clip_mech (Mechanisms): The object is used to update the adaptive clip.
            Default: ``None``.

    Inputs:
        - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`.
        - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`.

    Outputs:
        Tensor, a scalar Tensor with shape :math:`()`.
    """

    def __init__(self, network, optimizer, norm_bound=1.0, sens=1.0,
                 micro_batches=None,
                 noise_mech=None, clip_mech=None):
        super(_TrainOneStepCell, self).__init__(auto_prefix=False)
        self.network = network
        self.network.set_grad()
        self.network.add_flags(defer_inline=True)
        self.weights = optimizer.parameters
        self.optimizer = optimizer
        self.grad = C.GradOperation(get_by_list=True, sens_param=True)
        self.sens = sens
        self.reducer_flag = False
        self.grad_reducer = None
        parallel_mode = _get_parallel_mode()
        if parallel_mode in (
                ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL):
            self.reducer_flag = True
        if self.reducer_flag:
            mean = _get_gradients_mean()
            degree = _get_device_num()
            self.grad_reducer = DistributedGradReducer(optimizer.parameters,
                                                       mean, degree)

        # dp params
        if micro_batches is None:
            msg = 'micro_batches must give in differential privacy, but got value: {}'.format(
                micro_batches)
            LOGGER.error(TAG, msg)
            raise ValueError(msg)
        self._micro_batches = micro_batches
        self._norm_bound = norm_bound
        self._split = P.Split(0, self._micro_batches)
        self._clip_by_global_norm = _ClipGradients()
        self._noise_mech = noise_mech
        self._clip_mech = clip_mech
        self._tuple_add = _TupleAdd()
        self._add = P.Add()
        self._norm = nn.Norm()
        self._hyper_map = C.HyperMap()
        self._zero = Tensor(0, mstype.float32)
        self._assign = P.Assign()
        self._div = P.Div()
        self._sqrt = P.Sqrt()
        self._reduce_sum = P.ReduceSum()
        self._square_all = P.Square()
        self._less = P.Less()
        self._cast = P.Cast()

        self._micro_float = Tensor(micro_batches, mstype.float32)

        self._noise_mech_param_updater = None
        if self._noise_mech is not None and self._noise_mech._decay_policy is not None:
            self._noise_mech_param_updater = _MechanismsParamsUpdater(
                decay_policy=self._noise_mech._decay_policy,
                decay_rate=self._noise_mech._noise_decay_rate,
                cur_noise_multiplier=self._noise_mech._noise_multiplier,
                init_noise_multiplier=self._noise_mech._initial_noise_multiplier)

    def construct(self, data, label):
        """
        construct a compute flow.
        """
        weights = self.weights
        record_datas = self._split(data)
        record_labels = self._split(label)
        loss = self.network(record_datas[0], record_labels[0])
        sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens)
        record_grad = self.grad(self.network, weights)(record_datas[0],
                                                       record_labels[0], sens)

        # calcu norm_grad
        square_sum = self._zero
        for grad in record_grad:
            square_sum = self._add(square_sum, self._reduce_sum(self._square_all(grad)))
        norm_grad = self._sqrt(square_sum)

        # calcu beta
        beta = self._zero
        if self._clip_mech is not None:
            beta = self._add(beta,
                             self._cast(self._less(norm_grad, self._norm_bound),
                                        mstype.float32))

        record_grad = self._clip_by_global_norm(record_grad,
                                                self._norm_bound, norm_grad)
        grads = record_grad
        total_loss = loss
        for i in range(1, self._micro_batches):
            loss = self.network(record_datas[i], record_labels[i])
            sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens)
            record_grad = self.grad(self.network, weights)(record_datas[i],
                                                           record_labels[i],
                                                           sens)

            # calcu norm_grad
            square_sum = self._zero
            for grad in record_grad:
                square_sum = self._add(square_sum,
                                       self._reduce_sum(self._square_all(grad)))
            norm_grad = self._sqrt(square_sum)

            # calcu beta
            if self._clip_mech is not None:
                beta = self._add(beta,
                                 self._cast(self._less(norm_grad, self._norm_bound),
                                            mstype.float32))

            record_grad = self._clip_by_global_norm(record_grad,
                                                    self._norm_bound, norm_grad)
            grads = self._tuple_add(grads, record_grad)
            total_loss = P.Add()(total_loss, loss)
        loss = self._div(total_loss, self._micro_float)

        if self._noise_mech is not None:
            grad_noise_tuple = ()
            for grad_item in grads:
                grad_noise = self._noise_mech(grad_item)
                grad_noise_tuple = grad_noise_tuple + (grad_noise,)
            grads = self._tuple_add(grads, grad_noise_tuple)
            grads = self._hyper_map(F.partial(_grad_scale, self._micro_float),
                                    grads)
            # update mech parameters
            if self._noise_mech_param_updater is not None:
                multiplier = self._noise_mech_param_updater()
                loss = F.depend(loss, multiplier)

        if self.reducer_flag:
            # apply grad reducer on grads
            grads = self.grad_reducer(grads)

        if self._clip_mech is not None:
            beta = self._div(beta, self._micro_batches)
            next_norm_bound = self._clip_mech(beta, self._norm_bound)
            self._norm_bound = self._assign(self._norm_bound, next_norm_bound)
            loss = F.depend(loss, self._norm_bound)

        return F.depend(loss, self.optimizer(grads))


class Solution:
    def isCovered(self, ranges: List[List[int]], left: int, right: int) -> bool:

        given_inter = set(range(left, right + 1))
        for r in ranges:
            curr_inter = set(range(r[0], r[1] + 1))
            given_inter.difference_update(curr_inter)

            if not given_inter:
                return True

        return False


from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask_bower import Bower
import os, sqlite3

db = SQLAlchemy()
bower = Bower()

def create_app(config=None):
    app = Flask(__name__)

    if config is not None:
        app.config.from_object(config)

    db.init_app(app)
    app.engine = db.create_engine(app.config['SQLALCHEMY_DATABASE_URI'])
    Bower(app)

    from .features_bp.views import features_bp
    app.register_blueprint(features_bp, url_prefix='')

    @app.cli.command('initdb')
    def initdb_command():
        """Initializes the database."""
        db.create_all()
        print('Initialized the database.')

    @app.cli.command('secretkey')
    def secretkey_command():
        """Print the secret key."""
        print(app.config['SECRET_KEY'])

    return app

app = create_app('config.DevelopmentConfig')


import utils
import config
import pandas as pd
import re
import os
import sys
import argparse
import logging
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader
import torch
import esm

def get_embeds(df, dataset):
    '''

    input: dataframe (e.g. ../example/dataset/train.csv)
    output: .pt (e.g. ../example/embeds/train.pt)

    '''

    model, alphabet = esm.pretrained.esm1b_t33_650M_UR50S()
    batch_converter = alphabet.get_batch_converter()

    # truncate the long sequence into 1022
    df['Length'] = df['wt_seq'].apply(lambda x: len(x))

    if max(set(df.Length)) > 1022:
        df = utils.df_process(df)
    else:
        df['new_index'] = df['aa_index']

    print('data length:', df.shape[0])

    df['record_id'] = df['target_id']

    utils.generate_embeds_and_save(df, save_path = config.esm_storage_path, data_class=dataset, model = model, batch_converter = batch_converter, alphabet = alphabet)

def train_VariPred(train_ds, test_ds, valid_ds=None,train=True):

    '''

    input:
        train_ds: path of embeddings of training set (e.g. ../example/embeds/train.pkl)
        test_ds: path of embeddings of test set
        valid_ds: path of embeddings of valid set

    output:
        model weights (./model/model.ckpt)
        model predictions (e.g. ../example/output_results/model_eval_result.txt)
        model performance (MCC, AUC) (e.g. ../example/output_results/model_performance.txt)

    '''

    X_train, y_train, _ = utils.unpickler(ds_name=train_ds)
    X_test, y_test, record_id = utils.unpickler(ds_name=test_ds)

    if valid_ds is not None:
        X_valid, y_valid = utils.unpickler(ds_name=valid_ds)
        print('valid set name: ', valid_ds)
    else:
        X_train, X_valid, y_train, y_valid = train_test_split(X_train,y_train,
                                                            test_size=0.3,
                                                            shuffle=True,
                                                            stratify=y_train,
                                                            random_state=config.SEED)

    print('X_train shape: ', X_train.shape)
    print('X_test shape: ', X_test.shape)
    print('X_valid shape: ', X_valid.shape)

    train_dataset = utils.VariPredDataset(X_train, y_train)
    val_dataset = utils.VariPredDataset(X_valid, y_valid)
    test_dataset = utils.VariPredDataset(X_test, y_test)

    #> Feed dataset to dataloader
    train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=config.batch_size)
    test_loader = DataLoader(test_dataset, batch_size=config.batch_size)

    print('\n\n\n\n')
    print('=============== VariPred model training start ===============')

    model_size = X_train.shape[1]
    num_hidden = int(model_size/2)

    print('model_size: ', model_size)
    print('num_hidden: ', num_hidden)

    model = utils.MLPClassifier_LeakyReLu(num_input = model_size, num_hidden = num_hidden, num_output = config.label_num).to(config.device)

    total_params = sum(p.numel() for p in model.parameters())
    print(f'{total_params:,} total parameters.')
    total_trainable_params = sum(
        p.numel() for p in model.parameters() if p.requires_grad)
    print(f'{total_trainable_params:,} training parameters.')

    utils.trainer(train_loader, val_loader, model)

    print('=============== Predicting & Evaluating the trained model ===============')
    storage_path = f'./model'
    checkpoint=torch.load(f'{storage_path}/model.ckpt')
    model.load_state_dict(checkpoint['model_state_dict'])

    preds, y_true = utils.predict(test_loader, model, config.device)

    utils.predict_results(y_true, preds,
                          record_id = record_id,
                          train = train
                          )

def run_VariPred(target_ds,output):

    '''

    input:
        target_ds: path of embeddings of target set (without a true label)

    output:
        model predictions (e.g. ../example/output_results/VariPred_output.txt)

    '''

    X_target, y_target, record_id = utils.unpickler(ds_name=target_ds)
    print('X_target shape: ', X_target.shape)

    target_dataset = utils.VariPredDataset(X_target, y_target)
    target_loader = DataLoader(target_dataset, batch_size=config.batch_size)

    model_size = X_target.shape[1]
    num_hidden = int(model_size/2)
    model = utils.MLPClassifier_LeakyReLu(num_input = model_size, num_hidden = num_hidden, num_output = config.label_num).to(config.device)

    storage_path = f'./model'

    if not os.path.exists(storage_path):
        print('Please train the model first')

    checkpoint=torch.load(f'{storage_path}/model.ckpt')
    model.load_state_dict(checkpoint['model_state_dict'])

    preds, y_true = utils.predict(target_loader, model, config.device)

    utils.predict_results(y_true, preds,
                          record_id = record_id,
                          output_name=output
                          )
    print()
    print(f"Your prediction results are saved in ../example/output_results/{output}.txt")

parser = argparse.ArgumentParser(description='add args for training the VariPred model')
parser.add_argument('--df_path', '-p', default='../example/dataset', type=str)
parser.add_argument('--train_ds', '-tr', default='train', type=str)
parser.add_argument('--test_ds', '-ts', default='test', type=str)
parser.add_argument('--pred', '-i', default='target', type=str)
parser.add_argument('--output', '-o', default='VariPred_output', type=str)
parser.add_argument('--train', '-t', action="store_true")

args = parser.parse_args()

if __name__ == '__main__':

    print('=============== Loading data... ===============')

    storage_path = args.df_path

    print(f'=============== Start getting embeddings ... ===============')

    # if necessary, update the parameters of the last MLP layer.
    if args.train:
        train_df = pd.read_csv(f'{storage_path}/{args.train_ds}.csv')
        test_df = pd.read_csv(f'{storage_path}/{args.test_ds}.csv')

        print(f'getting embeds for {args.train_ds}.csv')
        get_embeds(train_df, dataset = args.train_ds)

        print(f'getting embeds for {args.test_ds}.csv')
        get_embeds(test_df, dataset = args.test_ds)
        train_VariPred(args.train_ds, args.test_ds)

    else:
        # predict the target df with VariPred
        target_df = pd.read_csv(f'{storage_path}/{args.pred}.csv')
        target_df['label'] = -1 # it doesn't matter what the true label is. It's just to ensure the programme can run properly.
        if not os.path.exists(f'{config.esm_storage_path}/{args.pred}.pkl'):
            print(f'getting embeds for {args.pred}.csv')
            get_embeds(target_df, dataset = args.pred)
        run_VariPred(target_ds=args.pred, output=args.output)

    print('\n\n')
    print('=============== No Bug No Error, Finished!!! ===============')

    print('\n\n')


"""Filters collection for gleitzsch."""
import sys
import random
import numpy as np
from skimage import exposure
from skimage import transform as tf
from skimage import filters
from skimage import color
from skimage.draw import polygon
# from modules.blur_detection import detect_blur

def eprint(line, end="\n"):
    """Like print but for stdout."""
    sys.stderr.write(line + end)

def parts(lst, n=25):
    """Split an iterable into list of iterables of size n."""
    return [lst[i:i + n] for i in iter(range(0, len(lst), n))]

def adjust_contrast(im, l_p, r_p):
    """Contrast correction."""
    perc_left, perc_right = np.percentile(im, (l_p, r_p))
    im = exposure.rescale_intensity(im, in_range=(perc_left, perc_right))
    return im

def xyz_shift(img, kt):
    """Apply chromatic aberration."""
    img = color.rgb2xyz(img)
    shp = img.shape
    red = img[:, :, 0]
    green = img[:, :, 1]
    blue = img[:, :, 2]
    # split channels, make shift
    red = tf.resize(red, output_shape=(shp[0], shp[1]))
    green = tf.resize(green, output_shape=(shp[0] - kt, shp[1] - kt))
    blue = tf.resize(blue, output_shape=(shp[0] - 2 * kt, shp[1] - 2 * kt))

    w, h = blue.shape
    ktd2 = int(kt / 2)
    red_n = np.reshape(red[kt: -kt, kt: -kt], (w, h, 1))
    green_n = np.reshape(green[ktd2: -1 * ktd2, ktd2: -1 * ktd2], (w, h, 1))
    blue_n = np.reshape(blue[:, :], (w, h, 1))

    new_im = np.concatenate((red_n, green_n, blue_n), axis=2)
    new_im = tf.resize(new_im, (shp[0], shp[1]))
    new_im = color.xyz2rgb(new_im)
    return new_im

def rgb_shift(img, kt):
    """Apply chromatic aberration."""
    shp = img.shape
    red = img[:, :, 0]
    green = img[:, :, 1]
    blue = img[:, :, 2]
    # split channels, make shift
    red = tf.resize(red, output_shape=(shp[0], shp[1]))
    green = tf.resize(green, output_shape=(shp[0] - kt, shp[1] - kt))
    blue = tf.resize(blue, output_shape=(shp[0] - 2 * kt, shp[1] - 2 * kt))

    w, h = blue.shape
    ktd2 = int(kt / 2)
    red_n = np.reshape(red[kt: -kt, kt: -kt], (w, h, 1))
    green_n = np.reshape(green[ktd2: -1 * ktd2, ktd2: -1 * ktd2], (w, h, 1))
    blue_n = np.reshape(blue[:, :], (w, h, 1))

    new_im = np.concatenate((red_n, green_n, blue_n), axis=2)
    new_im = tf.resize(new_im, (shp[0], shp[1]))
    return new_im

def __bayer(im):
    """Make monochrome pixels."""
    h, w, d = im.shape
    for i in range(h):
        for j in range(w):
            channels_to_rm = np.random.choice(range(3), size=2, replace=False)
            im[i][j][channels_to_rm[0]] = 0.0
            im[i][j][channels_to_rm[1]] = 0.0
    return im

def bayer(im):
    """It is not a bayer filter anymore actually."""
    w, h, d = im.shape
    imlen = w * h * d
    pix_num = w * h
    chan_numbers = np.random.choice([0, 1, 2], pix_num)
    im_flat, new_im_flat = np.reshape(im, imlen), []
    # new_im_flat = [im_flat[i] if yes_not[i] == 1 else 0.0 for i in range(imlen)]
    for num, pixel in enumerate(parts(im_flat, n=3)):
        color = [0.0, 0.0, 0.0]
        chan_num = chan_numbers[num]
        color[chan_num] = pixel[chan_num]
        new_im_flat.extend(color)
    print(len(new_im_flat))
    new_im = np.reshape(np.array(new_im_flat), (w, h, d))
    return new_im

def _bayer(im):
    """It is not a bayer filter anymore actually."""
    w, h, d = im.shape
    imlen = w * h * d
    # yes_not = np.random.choice([0, 1], imlen, p=[0.1, 0.9])
    chan_numbers = np.random.choice([0, 1, 2], imlen)
    im_flat, new_im_flat = np.reshape(im, imlen), []
    # new_im_flat = [im_flat[i] if yes_not[i] == 1 else 0.0 for i in range(imlen)]
    for i in range(imlen):
        chan = chan_numbers[i]
        if chan == 0:
            color = [im_flat[0], 0, 0]
        elif chan == 1:
            color = [0, im_flat[1], 0]
        else:
            color = [0, 0, im_flat[2]]
        new_im_flat.extend(color)
    new_im = np.reshape(np.array(new_im_flat), (w, h, d))
    return new_im

def amplify(im):
    """Self-overlap."""
    REPEATS = 0
    shift = int(np.random.uniform(low=30, high=130))
    sign = np.random.choice([-1, 1], 1)[0]
    shift *= sign
    # print(shift)
    delim, kt = 1, 3
    layer_sh = np.roll(a=im, axis=1, shift=shift) / kt
    im += layer_sh
    delim += 1 / kt
    kt /= 3

    for _ in range(REPEATS):
        layer_sh = np.roll(a=layer_sh, axis=1, shift=shift) / kt
        im += layer_sh
        delim += 1 / kt
        kt /= 3

    im /= delim
    # im[im > 1] = 1.0
    return im

def glitter(im, alen=300):
    """Make glitter."""
    dots = []
    w, h, _ = im.shape
    for _ in range(alen):
        dx = random.choice(range(w))
        dy = random.choice(range(h))
        dots.append((dx, dy))
    for dot in dots:
        try:
            im[dot[0] - 1: dot[0], dot[1] - 3: dot[1] + 3, :] = 1
        except IndexError:
            pass
    return im

def color_shift(im, val):
    """Change color throw HSV space."""
    im_hsv = color.rgb2hsv(im)
    hue = im_hsv[:, :, 0]
    hue += val
    hue[hue > 1] = hue[hue > 1] - 1
    hue[hue < 0] = hue[hue < 0] + 1
    im_hsv[:, :, 0] = hue
    im_rgb = color.hsv2rgb(im_hsv)
    return im_rgb

def black(im, thr):
    """Return two-colored bw image."""
    # get pixel sum (one number) instead of pixel (3 nums)
    col_sum = np.sum(im, axis=2)
    black_im = np.zeros((col_sum.shape[0], col_sum.shape[1]))
    # 0 - less than thr, 1 - more than
    black_im[col_sum > thr] = 1
    black_im = np.reshape(black_im, (black_im.shape[0], black_im.shape[1], 1))
    return np.concatenate((black_im, black_im, black_im), axis=2)

def remove_whites(im):
    """Diminish colors that are brighter than a threshold set. For an image."""
    col_sum = np.sum(im, axis=2)
    over_thr = np.reshape((col_sum - 2), (im.shape[0], im.shape[1], 1))
    over_thr = np.concatenate((over_thr, over_thr, over_thr), axis=2)
    new_im = np.where(over_thr > 0, im - over_thr, im)
    new_im[new_im < 0.0] = 0.0
    new_im[new_im > 1.0] = 1.0
    return new_im

def rainbow_layer(im):
    new_im = rgb_shift(im, kt=88)
    new_im = filters.gaussian(new_im, sigma=30, multichannel=True, mode='reflect', cval=0.6)
    img_hsv = color.rgb2hsv(new_im)
    img_hsv[..., 1] *= 3
    img_hsv[..., 2] *= 1.4
    img_hsv[img_hsv >= 1.0] = 1.0
    new_im = color.hsv2rgb(img_hsv)
    new_im = remove_whites(new_im)
    return new_im

def make_rainbow(im):
    """Apply rainbow filter."""
    black_instance = black(im, thr=2.1)
    rainbow_pic = rainbow_layer(black_instance)
    rainbow_pic /= 2
    new_arr = rainbow_pic + im
    new_arr[new_arr > 1.0] = 1.0
    return new_arr

def interlace(im, zero_prob=0.9):
    """Add interlacing fields."""
    w, h, d = im.shape
    coeff, processed = 3, []
    shift_wide = 2

    half_prob = (1 - zero_prob) / 2
    half_probs = [half_prob / shift_wide for _ in range(shift_wide)]
    shift_probs = half_probs + [zero_prob] + half_probs
    shif_pos = list(range(- shift_wide, shift_wide + 1))
    shift = 0

    for num, i in enumerate(range(0, w, coeff)):
        row = im[i: i + coeff + 1, :, :]
        row = row / 1.35 if num % 2 == 0 else row
        shift_p = np.random.choice(shif_pos, 1, p=shift_probs)[0]
        shift += shift_p
        row = np.roll(a=row, axis=0, shift=shift)  # small part
        row = np.roll(a=row, axis=1, shift=shift)  # long size
        processed.append(row)

    merge = np.concatenate(processed, axis=0)
    merge = tf.resize(merge, (w, h))
    return merge

def add_vertical(image):
    """Hard to say."""
    first_row = np.reshape(image[0, :, :], newshape=(1, image.shape[1], image.shape[2]))
    stretch = np.repeat(first_row, image.shape[0], axis=0)
    add = image + stretch / 10
    add[add > 1] = 1.0
    return add

def vert_streaks(im):
    """Add some vertical streaks on image."""
    w, h, d = im.shape
    processed = []
    streaks_borders_num = random.choice(range(6, 16, 2))
    streaks_borders = [0] + list(sorted(np.random.choice(range(h), streaks_borders_num, replace=False))) + [h]
    for num, border in enumerate(streaks_borders[1:]):
        prev_border = streaks_borders[num]
        pic_piece = im[:, prev_border: border, :]
        if num % 2 != 0:  # don't touch this part
            processed.append(pic_piece)
            continue
        piece_h, piece_w, _ = pic_piece.shape
        piece_rearranged = []
        shifts_raw = sorted([i if i > 0 else -i for i in map(int, np.random.normal(5, 10, piece_w))])
        shifts_add = np.random.choice(range(-5, 2), piece_w)
        shifts_mod = [shifts_raw[i] + shifts_add[i] for i in range(piece_w)]
        shifts_left = [shifts_mod[i] for i in range(0, piece_w, 2)]
        shifts_right = sorted([shifts_mod[i] for i in range(1, piece_w, 2)], reverse=True)
        shifts = shifts_left + shifts_right
        for col_num, col_ind in enumerate(range(piece_w)):
            col = pic_piece[:, col_ind: col_ind + 1, :]
            col = np.roll(col, axis=0, shift=shifts[col_num])
            piece_rearranged.append(col)
        piece_shifted = np.concatenate(piece_rearranged, axis=1)
        processed.append(piece_shifted)
    new_im = np.concatenate(processed, axis=1)
    new_im = tf.resize(new_im, (w, h))
    return new_im

def horizonal_shifts(im, colorized=True):
    """Add random horizontal shifts."""
    w, h, d = im.shape
    processed = []
    shifts_borders_num = random.choice(range(6, 16, 2))
    shifts_borders = [0] + list(sorted(np.random.choice(range(w), shifts_borders_num, replace=False))) + [h]
    for num, border in enumerate(shifts_borders[1:]):
        prev_border = shifts_borders[num]
        pic_piece = im[prev_border: border, :, :]
        shift = 0 if num % 2 != 0 else random.choice(range(40))
        shifted = np.roll(pic_piece, shift=shift, axis=1)
        shifted = shifted if not colorized else np.roll(shifted, shift=shift, axis=2)
        processed.append(shifted)
    new_im = np.concatenate(processed, axis=0)
    new_im = tf.resize(new_im, (w, h))
    return new_im

def _vert_streaks(im, kernel=1, partial=False):
    """Add some vertical streaks on image."""
    w, h, d = im.shape
    processed = []
    shifts = np.random.choice([-2, -1, 0, 1, 2], h)
    current_shift = 0
    change_points = np.random.choice([False, True], h, p=[0.97, 0.03]) if partial else [0 for _ in range(h)]
    point = True
    for num, i in enumerate(range(0, h, kernel)):
        change_ = change_points[num]
        point = not point if change_ else point
        col = im[:, i: i + kernel + 1, :]
        if not point:
            processed.append(col)
            current_shift = 0
            continue
        current_shift += shifts[num]
        col = np.roll(col, axis=0, shift=current_shift)
        processed.append(col)
    merge = np.concatenate(processed, axis=1)
    merge = tf.resize(merge, (w, h))
    return merge

def add_magic(im):
    """Add some magic."""
    eprint("detecting blur...")
    w, h, d = im.shape
    blur_map_layer = np.reshape(detect_blur(im, kernel_size=8), (w, h, 1))
    blur_map_layer = filters.gaussian(blur_map_layer, sigma=5)
    blur_map = np.concatenate((blur_map_layer, blur_map_layer, blur_map_layer), axis=2)
    blur_map[blur_map < 0] = 0.0
    blur_map[blur_map > 1] = 1.0
    eprint("blur detected")
    magic_layer = im
    magic_layer = rgb_shift(magic_layer, kt=28)
    magic_layer = vert_streaks(magic_layer)
    magic_layer = filters.gaussian(magic_layer, sigma=3)
    magic_layer -= blur_map
    magic_layer[magic_layer < 0] = 0.0
    im += magic_layer
    im[im > 1] = 1.0
    return im

def add_figs(im):
    """Add random shapes."""
    figs = np.zeros((im.shape[0], im.shape[1], 3))
    wide = np.random.choice(range(10, 200), 1)[0]
    poly = np.array((
        (0, 0),
        (0, wide),
        (1000, wide),
        (1000, 0),
    ))
    rr, cc = polygon(poly[:, 0], poly[:, 1], figs.shape)
    figs[rr, cc, 0] = 0.7
    figs[rr, cc, 1] = 0.1
    figs[rr, cc, 2] = 0.2
    figs = np.roll(figs, shift=np.random.choice(range(1000), 1)[0], axis=1)
    figs = filters.gaussian(figs, sigma=5, multichannel=True, mode='reflect', cval=0.6)
    im += figs
    im[im > 1.0] = 1.0
    return im

def reconstruct(im, kHz):
    """Reconstruct shifted rows."""
    if kHz == 16.0:
        return im
    w, h, d = im.shape
    processed = []
    for num, i in enumerate(range(0, w, 1)):
        row = im[i: i + 1, :, :]
        if kHz == 16.01:
            shift = int(num * 0.5)
        elif kHz == 15.99:
            shift = int(num * -0.5)
        elif kHz == 15.98:
            shift = int(num * -1)
        else:
            shift = 0
        row = np.roll(a=row, axis=1, shift=shift)
        processed.append(row)
    merge = np.concatenate(processed, axis=0)
    merge = tf.resize(merge, (w, h))
    return merge

def increase_saturation(im):
    """Increase saturation."""
    hsl_im = color.rgb2hsv(im)
    hsl_im[:, :, 1] *= 1.45
    hsl_im[hsl_im > 1] = 1.0
    im = color.hsv2rgb(hsl_im)
    return im


class Solution:
    def findWords(self, words: List[str]) -> List[str]:
        # Create letter-row directory
        rows = {}
        for i, characters in enumerate(['qwertyuiop', 'asdfghjkl', 'zxcvbnm']):
            for char in characters:
                rows[char] = i

        # Test each word by weather the row numbers for each letter are the same
        ## Find the row number from the letter-row dict
        ## Use set to check whether all row numbers are the same
        res = []
        for word in words:
            rownum = set([rows[i.lower()] for i in word])
            if len(rownum) == 1:
                res.append(word)
        return res

Algorithms - Python/Arrays/500. Keyboard Row.py


from ducho.runner.Runner import MultimodalFeatureExtractor
from datasets import load_dataset
import numpy as np
import os

np.random.seed(42)

def main():
    dataset = load_dataset('ashraq/fashion-product-images-small')
    dataset = np.array(dataset['train'])[np.random.choice(len(dataset['train']), 100, replace=False)].tolist()
    if not os.path.exists('./local/data/demo1/images/'):
        os.makedirs('./local/data/demo1/images/')
    with open(f'./local/data/demo1/descriptions.tsv', 'a') as f:
        f.write('PRODUCT_ID\tdescription\n')
        for file in dataset:
            img = file['image']
            description = f'{file["gender"]} {file["masterCategory"]} ' \
                          f'{file["subCategory"]} {file["articleType"]} ' \
                          f'{file["baseColour"]} {file["season"]} ' \
                          f'{file["year"]} {file["usage"]} ' \
                          f'{file["productDisplayName"]}'
            img.save(f'./local/data/demo1/images/{file["id"]}.jpg')
            f.write(f'{file["id"]}\t{description}\n')
    extractor_obj = MultimodalFeatureExtractor(config_file_path='./demos/demo1/config.yml')
    extractor_obj.execute_extractions()

if __name__ == '__main__':
    main()


#Para usar graficos, precisei baixar a biblioteca matplotlib. Para isso basta executar no CMD
#pip install -U matplotlib

import pandas as pd
import matplotlib.pyplot as plt

import os
for f in os.listdir("C:\workspace\Analise de dados com Python e Pandas\Arquivos"):
	print(f)

df1 = pd.read_excel("C:\workspace\Analise de dados com Python e Pandas\Arquivos\_Aracaju.xlsx")
df2 = pd.read_excel("C:\workspace\Analise de dados com Python e Pandas\Arquivos\Fortaleza.xlsx")
df3 = pd.read_excel("C:\workspace\Analise de dados com Python e Pandas\Arquivos\_Natal.xlsx")
df4 = pd.read_excel("C:\workspace\Analise de dados com Python e Pandas\Arquivos\Recife.xlsx")
df5 = pd.read_excel("C:\workspace\Analise de dados com Python e Pandas\Arquivos\Salvador.xlsx")

print(df1.head())
print(df2.head())
print(df3.head())
print(df4.head())
print(df5.head())

#Para fazar a junção de todos os aruivos. Usar o concat

df = pd.concat([df1,df2,df3,df4,df5])

#Verificando o tipo de data de cada coluna

print(df.dtypes)

#Alterando o tipo de dado da coluna Data para tipo inteiro
df["Data"] = df["Data"].astype("int64")

print(df.dtypes)

#Alterando o tipo de dado da coluna Data para tipo Datatime

df["Data"] = pd.to_datetime(df["Data"])

print(df.dtypes)

#Criando novas colunas:
#Neste caso vamos criar uma coluna de receitas (vendas x Qtd). Mul (de multiplicação)

df["Receita"] = df["Vendas"].mul(df["Qtde"])

print(df.head())

print(df.dtypes)

#Agrupamento por ano
print(df.groupby(df["Data"].dt.year)["Receita"].sum())

#Criando uma nova coluna com o ano

df["Ano_Venda"] = df["Data"].dt.year

print(df.sample(5))

#Extraindo o mês e o dia da venda. Criando as colunas pos mês e dia

df["Mes_Venda"], df["Dia_Venda"] = (df["Data"].dt.month, df["Data"].dt.day)

print(df.sample(5))

#Retornando a Data mais antiga

print(df["Data"].min())

#Calculando a diferença de dias (na mesma coluna) e criando uma nova coluna para essa informação

df["Diferenca_Dias"] = df["Data"] - df["Data"].min()
print(df.sample(5))

#Criando uma coluna de trimestre

df["Trimestre_Venda"] = df["Data"].dt.quarter
print(df.sample(5))

#Filtrando as vendas de 2019 e do mes de março. (Buscando um Subconjunto)

vendas_marco_2019 = df.loc[(df["Data"].dt.year == 2019) & (df["Data"].dt.month == 3)]

print(vendas_marco_2019)

#value_counts Faz uma contagem, no caso abaixo esta pegando do conjunto de dados e informando quantas linhas
#temos para cada LojaID.
#ascending=False ira trazer do maior para o menor

print(df["LojaID"].value_counts(ascending=False))

#Grafico de Barras na vertical
df["LojaID"].value_counts(ascending=False).plot.bar()
plt.show()

#Grafico de Barras na horizontaç
df["LojaID"].value_counts(ascending=True).plot.barh();
plt.show()

#Grafico de Pizza
df.groupby(df["Data"].dt.year)["Receita"].sum().plot.pie()
plt.show()

#Total de vendas por cidade
print(df["Cidade"].value_counts())

#Adicionando um titulo e alterando o nome dos eixos

df["Cidade"].value_counts().plot.bar(title= "Total de vendas por Cidade")
plt.xlabel("Cidade")
plt.ylabel("Total Vendas")
plt.show()

#Alterando a cor

df["Cidade"].value_counts().plot.bar(title= "Total de vendas por Cidade", color= "pink")
plt.xlabel("Cidade")
plt.ylabel("Total Vendas")
plt.show()

#Alterando o estilo

plt.style.use("ggplot")

df["Cidade"].value_counts().plot.bar(title= "Total de vendas por Cidade", color= "pink")
plt.xlabel("Cidade")
plt.ylabel("Total Vendas")
plt.show()

#plot é um grafico com linhas
df.groupby(df["Mes_Venda"])["Qtde"].sum().plot(title = "Total Produtos Vendidos por mês")
plt.xlabel("Mês")
plt.ylabel("Total Produtos Vendidos")
plt.legend()
plt.show()


from typing import Dict, List, Tuple

from pyramid.authorization import Allow
from pyramid.view import view_config
from snovault import collection, load_schema
from snovault.types.access_key import (
    AccessKey as SnovaultAccessKey,
    access_key_add as sno_access_key_add,
    access_key_reset_secret as sno_access_key_reset_secret,
    access_key_view_raw as sno_access_key_view_raw
)
from snovault.util import debug_log
from snovault.validators import validate_item_content_post

from .acl import ALLOW_AUTHENTICATED_CREATE_ACL, ONLY_ADMIN_VIEW_ACL
from .base import DELETED_ACL
from .base import Item as SMAHTItem

@collection(
    name='access-keys',
    unique_key="access_key:access_key_id",  # Required for GET via /access-keys/{access_key_id}/
    acl=ALLOW_AUTHENTICATED_CREATE_ACL,
    properties={
        "title": "Access keys",
        "description": "Programmatic access keys",
    },
)
class AccessKey(SMAHTItem, SnovaultAccessKey):
    """AccessKey class."""
    ACCESS_KEY_EXPIRATION_TIME = 90  # days
    item_type = "access_key"
    schema = load_schema("encoded:schemas/access_key.json")
    embedded_list = []

    STATUS_ACL = {
        "current": [(Allow, "role.owner", ["view", "edit"])] + ONLY_ADMIN_VIEW_ACL,
        "deleted": DELETED_ACL,
    }

    class Collection(SMAHTItem.Collection):
        pass

    def __acl__(self) -> List[Tuple[str, str, List[str]]]:
        return SnovaultAccessKey.__acl__(self)

    def __ac_local_roles__(self) -> Dict[str, str]:
        return SnovaultAccessKey.__ac_local_roles__(self)

@view_config(context=AccessKey.Collection, request_method="POST",
             permission="add",
             validators=[validate_item_content_post])
@debug_log
def access_key_add(context, request):
    return sno_access_key_add(context, request)

@view_config(name="reset-secret", context=AccessKey,
             permission="add",
             request_method="POST", subpath_segments=0)
@debug_log
def access_key_reset_secret(context, request):
    return sno_access_key_reset_secret(context, request)

@view_config(context=AccessKey, permission="view_raw", request_method="GET",
             name="raw")
@debug_log
def access_key_view_raw(context, request):
    return sno_access_key_view_raw(context, request)


texto = input('Qual arquivo quer consultar o número de vogais?')
vogais = ['a', 'á', 'à', 'â', 'ã', 'e', 'é', 'ê', 'i', 'í', 'o', 'ó', 'ô', 'õ', 'u', 'ú', 'ü']
qtd = 0

with open(texto) as arquivo:
    text = (arquivo.read()).lower()
    for i in text:
        if i in vogais:
            qtd += 1
print(f'Este texo possui {qtd} vogais')


from textwrap import dedent

from docx import Document
from docx.oxml import OxmlElement, parse_xml
from docx.table import _Cell
from docx.enum.table import WD_TABLE_ALIGNMENT, WD_CELL_VERTICAL_ALIGNMENT, WD_ROW_HEIGHT_RULE
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT, WD_TAB_ALIGNMENT
from docx.shared import Pt, Cm, Inches, RGBColor
from docx.oxml.ns import nsdecls, qn

CHECK_BOX = u" □ "
# font functions
def set_cell_border(cell, **kwargs):
    """
    Set cell`s border
    Usage:

    set_cell_border(
        cell,
        top={"sz": 12, "val": "single", "color": "#FF0000", "space": "0"},
        bottom={"sz": 12, "color": "#00FF00", "val": "single"},
        start={"sz": 24, "val": "dashed", "shadow": "true"},
        end={"sz": 12, "val": "dashed"},
    )
    """
    tc = cell._tc
    tcPr = tc.get_or_add_tcPr()

    # check for tag existnace, if none found, then create one
    tcBorders = tcPr.first_child_found_in("w:tcBorders")
    if tcBorders is None:
        tcBorders = OxmlElement('w:tcBorders')
        tcPr.append(tcBorders)

    # list over all available tags
    for edge in ('start', 'top', 'end', 'bottom', 'insideH', 'insideV'):
        edge_data = kwargs.get(edge)
        if edge_data:
            tag = 'w:{}'.format(edge)

            # check for tag existnace, if none found, then create one
            element = tcBorders.find(qn(tag))
            if element is None:
                element = OxmlElement(tag)
                tcBorders.append(element)

            # looks like order of attributes is important
            for key in ["sz", "val", "color", "space", "shadow"]:
                if key in edge_data:
                    element.set(qn('w:{}'.format(key)), str(edge_data[key]))

def set_cell_margins(cell: _Cell, **kwargs):
    """
    cell:  actual cell instance you want to modify

    usage:

        set_cell_margins(cell, top=50, start=50, bottom=50, end=50)

    provided values are in twentieths of a point (1/1440 of an inch).
    read more here: http://officeopenxml.com/WPtableCellMargins.php
    """
    tc = cell._tc
    tcPr = tc.get_or_add_tcPr()
    tcMar = OxmlElement('w:tcMar')

    for m in [
        "top",
        "start",
        "bottom",
        "end",
    ]:
        if m in kwargs:
            node = OxmlElement("w:{}".format(m))
            node.set(qn('w:w'), str(kwargs.get(m)))
            node.set(qn('w:type'), 'dxa')
            tcMar.append(node)

    tcPr.append(tcMar)

def get_merge_cells(table,row,start,end):
    start_cell = table.cell(row, start)
    end_cell = table.cell(row, end)
    new_cell = start_cell.merge(end_cell)
    return new_cell

def set_table_alignment(table):
    for row in table.rows:
        row.height = Inches(0.25)
        for cell in row.cells:
            cell.paragraphs[0].paragraph_format.space_after = Pt(0)
            cell.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.CENTER

# text functions
def get_header(header_text='RENTAL APPLICATION'):
    header = document.add_heading(header_text, 1)
    header.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
    header_font = header.style.font
    header_font.color.rgb = RGBColor(0, 0, 0)
    header_font.name = 'Calibri'
    header_font.size = Pt(20)

def get_header_paragraph(text):
    header_paragraph = document.add_paragraph()
    header_paragraph_text = f'''
    Property Address: {text}
    Unit #: {text}
    City, State, ZIP: {text}
    Date of Application: {text}
    '''
    header_paragraph_font = header_paragraph.style.font
    header_paragraph_font.name = 'Calibri'
    header_paragraph_font.size = Pt(12)
    header_paragraph.add_run(dedent(header_paragraph_text)).bold = True

def get_table_applicant_information():

    initials = ['First Name','','Middle Name','','Last Name','']
    contact_informations = ['Email','','Phone #1','','Phone #2','']
    document_informations = ['Date of Birth','_ _/ _ _/ _ _ _ _','Social Security #','','Driver’s License #','']

    table_applicant_information = document.add_table(rows=4,cols=6)

    table_applicant_information.autofit = False

    header_cell = get_merge_cells(table_applicant_information,0,0,5)
    header_cell.text = 'APPLICANT INFORMATION'
    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    header_cell._tc.get_or_add_tcPr().append(table_header_color)

    for row in range(4):
        for cell in range(6):
            if cell % 2 != 0:
                set_cell_border(table_applicant_information.cell(row, cell), bottom={"sz": 6, "color": "#000000", "val": "single"})

    row_second_cells = table_applicant_information.rows[1].cells
    for cell, initial in enumerate(initials):
        row_second_cells[cell].text = initial

    row_third_cells = table_applicant_information.rows[2].cells
    for cell, contact_information in enumerate(contact_informations):
        row_third_cells[cell].text = contact_information

    row_fourth_cells = table_applicant_information.rows[3].cells

    for cell, document_information in enumerate(document_informations):
        row_fourth_cells[cell].text = document_information

    header_paragraph = document.add_paragraph()
    header_paragraph.aligmnet = WD_PARAGRAPH_ALIGNMENT.CENTER
    header_paragraph_font = header_paragraph.style.font
    header_paragraph_font.name = 'Calibri'
    header_paragraph_font.size = Pt(12)

def get_table_additional_occupant(rows=4):
    table_additional_occupant = document.add_table(rows=rows, cols=7)

    table_additional_occupant.autofit = False

    paragraph = document.add_paragraph()
    paragraph.aligmnet = WD_TAB_ALIGNMENT
    paragraph_font = paragraph.style.font
    paragraph_font.name = 'Calibri'
    paragraph_font.size = Pt(10)

    header_cell = get_merge_cells(table_additional_occupant,0,0,6)
    header_cell.text = 'ADDITIONAL OCCUPANT(S)'
    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    header_cell._tc.get_or_add_tcPr().append(table_header_color)

    for row in range(1, rows):
        column = table_additional_occupant.rows[row].cells
        column[0].text = 'Name'
        column[0].width = Inches(0.5)
        column_1_2 = get_merge_cells(table_additional_occupant, row, 1, 2)
        set_cell_border(column_1_2, bottom={"sz": 6, "color": "#000000", "val": "single"})

        column[3].text = 'Relationship'
        column[3].width = Inches(0.9)
        set_cell_border(column[4], bottom={"sz": 6, "color": "#000000", "val": "single"})

        column[5].text = 'Age'
        column[5].width = Inches(0.5)
        set_cell_border(column[6], bottom={"sz": 6, "color": "#000000", "val": "single"})

def get_table_residence_history():
    table_residence_history = document.add_table(rows=5, cols=14)

    for index, column in enumerate(table_residence_history.columns):
        for cell in column.cells:
            if index in [0, 1]:
                cell.width = Inches(0.75)
            elif index in [6, 7, 10, 11]:
                cell.width = Inches(0.6)
            else:
                cell.width = Inches(0.475)

    i = 0
    get_merge_cells(table_residence_history, i, 0, len(table_residence_history.columns) - 1)

    current_row = table_residence_history.rows[i]

    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    current_row.cells[0]._tc.get_or_add_tcPr().append(table_header_color)

    current_row.cells[0].text = 'residence history'.upper()

    i = 1
    get_merge_cells(table_residence_history, i, 0, 1)
    get_merge_cells(table_residence_history, i, 2, 5)
    get_merge_cells(table_residence_history, i, 6, 7)
    get_merge_cells(table_residence_history, i, 8, 9)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'Current Address'
    current_row.cells[6].text = 'Unit #'
    current_row.cells[10].text = 'Rent'
    current_row.cells[11].text = u"  □  "
    current_row.cells[12].text = 'Own'
    current_row.cells[13].text = u"  □  "

    set_cell_border(current_row.cells[2], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[8], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[11], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[13], bottom={"sz": 6, "color": "#000000", "val": "single"})

    i = 2
    get_merge_cells(table_residence_history, i, 6, 9)
    get_merge_cells(table_residence_history, i, 10, 13)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'City'
    current_row.cells[2].text = 'State'
    current_row.cells[4].text = 'ZIP'
    current_row.cells[6].text = 'Monthly Payment or Rent $'

    set_cell_border(current_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[3], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[5], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[10], bottom={"sz": 6, "color": "#000000", "val": "single"})

    i = 3
    get_merge_cells(table_residence_history, i, 0, 1)
    get_merge_cells(table_residence_history, i, 2, 5)
    get_merge_cells(table_residence_history, i, 6, 7)
    get_merge_cells(table_residence_history, i, 8, 9)
    get_merge_cells(table_residence_history, i, 10, 11)
    get_merge_cells(table_residence_history, i, 12, 13)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'Dates of Residence'
    current_row.cells[2].text = f'__/__/____ to __/__/____'
    current_row.cells[6].text = 'Present Landlord'
    current_row.cells[10].text = 'Landlord Phone#'

    set_cell_border(current_row.cells[2], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[8], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[12], bottom={"sz": 6, "color": "#000000", "val": "single"})

    i = 4
    get_merge_cells(table_residence_history, i, 0, 1)
    get_merge_cells(table_residence_history, i, 2, len(table_residence_history.columns) - 1)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'Reason for moving out'

    set_cell_border(current_row.cells[2], bottom={"sz": 6, "color": "#000000", "val": "single"})

def get_table_residence_history_no_header():
    table_residence_history = document.add_table(rows=4, cols=14)

    for index, column in enumerate(table_residence_history.columns):
        for cell in column.cells:
            if index in [0, 1]:
                cell.width = Inches(0.75)
            elif index in [6, 7, 10, 11]:
                cell.width = Inches(0.6)
            else:
                cell.width = Inches(0.475)

    i = 0
    get_merge_cells(table_residence_history, i, 0, 1)
    get_merge_cells(table_residence_history, i, 2, 5)
    get_merge_cells(table_residence_history, i, 6, 7)
    get_merge_cells(table_residence_history, i, 8, 9)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'Previous Address'
    current_row.cells[6].text = 'Unit #'
    current_row.cells[10].text = 'Rent'
    current_row.cells[11].text = u"  □  "
    current_row.cells[12].text = 'Own'
    current_row.cells[13].text = u"  □  "

    set_cell_border(current_row.cells[2], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[8], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[11], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[13], bottom={"sz": 6, "color": "#000000", "val": "single"})

    i = 1
    get_merge_cells(table_residence_history, i, 6, 9)
    get_merge_cells(table_residence_history, i, 10, 13)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'City'
    current_row.cells[2].text = 'State'
    current_row.cells[4].text = 'ZIP'
    current_row.cells[6].text = 'Monthly Payment or Rent $'

    set_cell_border(current_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[3], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[5], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[10], bottom={"sz": 6, "color": "#000000", "val": "single"})

    i = 2
    get_merge_cells(table_residence_history, i, 0, 1)
    get_merge_cells(table_residence_history, i, 2, 5)
    get_merge_cells(table_residence_history, i, 6, 7)
    get_merge_cells(table_residence_history, i, 8, 9)
    get_merge_cells(table_residence_history, i, 10, 11)
    get_merge_cells(table_residence_history, i, 12, 13)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'Dates of Residence'
    current_row.cells[2].text = f'__/__/____ to __/__/____'
    current_row.cells[6].text = 'Present Landlord'
    current_row.cells[10].text = 'Landlord Phone#'

    set_cell_border(current_row.cells[2], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[8], bottom={"sz": 6, "color": "#000000", "val": "single"})
    set_cell_border(current_row.cells[12], bottom={"sz": 6, "color": "#000000", "val": "single"})

    i = 3
    get_merge_cells(table_residence_history, i, 0, 1)
    get_merge_cells(table_residence_history, i, 2, len(table_residence_history.columns) - 1)

    current_row = table_residence_history.rows[i]

    current_row.cells[0].text = 'Reason for moving out'

    set_cell_border(current_row.cells[2], bottom={"sz": 6, "color": "#000000", "val": "single"})

def get_table_employment_information():
    table_employment_information = document.add_table(rows=5, cols=8)

    for row in table_employment_information.rows:
        for cell in row.cells:
            cell.width = Inches(0.7)

    for row in table_employment_information.rows:
        row.cells[0].width = Inches(1.4)
        row.cells[1].width = Inches(2.1)

    header_cell = get_merge_cells(table_employment_information, 0, 0, 7)
    header_cell.text = 'employment information'.upper()
    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    header_cell._tc.get_or_add_tcPr().append(table_header_color)

    second_row = table_employment_information.rows[1]

    second_row.cells[0].text = 'Current Employer'

    set_cell_border(second_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    get_merge_cells(table_employment_information, 1, 2, 3)
    second_row.cells[2].text = 'Position/Title'

    get_merge_cells(table_employment_information, 1, 4, 7)
    set_cell_border(second_row.cells[4], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    third_row = table_employment_information.rows[2]

    third_row.cells[0].text = 'Supervisor'

    set_cell_border(third_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    get_merge_cells(table_employment_information, 2, 2, 3)
    third_row.cells[2].text = 'Phone #'

    get_merge_cells(table_employment_information, 2, 4, 7)
    set_cell_border(third_row.cells[4], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    fourth_row = table_employment_information.rows[3]

    fourth_row.cells[0].text = 'Address'
    set_cell_border(fourth_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )
    fourth_row.cells[2].text = 'City'
    set_cell_border(fourth_row.cells[3], bottom={"sz": 6, "color": "#000000", "val": "single"}, )
    fourth_row.cells[4].text = 'State'
    set_cell_border(fourth_row.cells[5], bottom={"sz": 6, "color": "#000000", "val": "single"}, )
    fourth_row.cells[6].text = 'ZIP'
    set_cell_border(fourth_row.cells[7], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    fifth_row = table_employment_information.rows[4]

    fifth_row.cells[0].text = 'Dates of Employment'

    set_cell_border(fifth_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    get_merge_cells(table_employment_information, 4, 2, 3)
    fifth_row.cells[2].text = 'Monthly Income $'

    get_merge_cells(table_employment_information, 4, 4, 7)
    set_cell_border(fifth_row.cells[4], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

def get_table_employment_information_no_header():
    table_employment_information = document.add_table(rows=4, cols=8)

    for row in table_employment_information.rows:
        for cell in row.cells:
            cell.width = Inches(0.7)

    for row in table_employment_information.rows:
        row.cells[0].width = Inches(1.4)
        row.cells[1].width = Inches(2.1)

    second_row = table_employment_information.rows[0]

    second_row.cells[0].text = 'Previous Employer'

    set_cell_border(second_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    get_merge_cells(table_employment_information, 0, 2, 3)
    second_row.cells[2].text = 'Position/Title'

    get_merge_cells(table_employment_information, 0, 4, 7)
    set_cell_border(second_row.cells[4], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    third_row = table_employment_information.rows[1]

    third_row.cells[0].text = 'Supervisor'

    set_cell_border(third_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    get_merge_cells(table_employment_information, 1, 2, 3)
    third_row.cells[2].text = 'Phone #'

    get_merge_cells(table_employment_information, 1, 4, 7)
    set_cell_border(third_row.cells[4], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    fourth_row = table_employment_information.rows[2]

    fourth_row.cells[0].text = 'Address'
    set_cell_border(fourth_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )
    fourth_row.cells[2].text = 'City'
    set_cell_border(fourth_row.cells[3], bottom={"sz": 6, "color": "#000000", "val": "single"}, )
    fourth_row.cells[4].text = 'State'
    set_cell_border(fourth_row.cells[5], bottom={"sz": 6, "color": "#000000", "val": "single"}, )
    fourth_row.cells[6].text = 'ZIP'
    set_cell_border(fourth_row.cells[7], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    fifth_row = table_employment_information.rows[3]

    fifth_row.cells[0].text = 'Dates of Employment'

    set_cell_border(fifth_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

    get_merge_cells(table_employment_information, 3, 2, 3)
    fifth_row.cells[2].text = 'Monthly Income $'

    get_merge_cells(table_employment_information, 3, 4, 7)
    set_cell_border(fifth_row.cells[4], bottom={"sz": 6, "color": "#000000", "val": "single"}, )

def get_table_additional_income():
    table_additional_income = document.add_table(rows=3, cols=6)

    for index, column in enumerate(table_additional_income.columns):
        for cell in column.cells:
            if index in [0, 4]:
                cell.width = Inches(1.25)
            elif index in [1]:
                cell.width = Inches(2.5)
            else:
                cell.width = Inches(0.9)

    i = 0
    get_merge_cells(table_additional_income, i, 0, len(table_additional_income.columns) - 1)

    current_row = table_additional_income.rows[i]

    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    current_row.cells[0]._tc.get_or_add_tcPr().append(table_header_color)

    current_row.cells[0].text = 'additional income'.upper()

    for i in range(1,3):

        current_row = table_additional_income.rows[i]

        current_row.cells[0].text = 'Source of Income'
        current_row.cells[2].text = 'Amount $'
        current_row.cells[4].text = 'Proof of Income'
        current_row.cells[5].text = f'YES {CHECK_BOX} NO {CHECK_BOX}'

        set_cell_border(current_row.cells[1], bottom={"sz": 6, "color": "#000000", "val": "single"})
        set_cell_border(current_row.cells[3], bottom={"sz": 6, "color": "#000000", "val": "single"})

def get_table_vehicle_information():
    vehicle_information_text = ['Make','','Model','','Year','','State','','Plate #']
    table_vehicle_information = document.add_table(rows=3, cols=10)
    header_cell = get_merge_cells(table_vehicle_information, 0, 0, 9)
    header_cell.text = 'VEHICLE INFORMATION'
    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    header_cell._tc.get_or_add_tcPr().append(table_header_color)

    table_vehicle_information.autofit = False

    for row_number in range(1,3):
        row = table_vehicle_information.rows[row_number].cells
        for cell in range(10):
            if cell%2 == 0:
                row[cell].text = vehicle_information_text[cell]
                row[cell].width = Inches(0.55)
            else:
                row[cell].width = Inches(1)
                set_cell_border(row[cell], bottom={"sz": 6, "color": "#000000", "val": "single"})

    paragraph = document.add_paragraph()
    paragraph.aligmnet = WD_TAB_ALIGNMENT
    paragraph_font = paragraph.style.font
    paragraph_font.name = 'Calibri'
    paragraph_font.size = Pt(10)

def get_table_pets_information():
    table_pets_information = document.add_table(rows=2, cols=13)
    header_cell = get_merge_cells(table_pets_information, 0, 0, 12)
    header_cell.text = 'pets information'.upper()
    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    header_cell._tc.get_or_add_tcPr().append(table_header_color)

    table_pets_information.autofit = False

    row = table_pets_information.rows[1].cells
    row[0].text = 'Pet(s)'
    row_1_2 = get_merge_cells(table_pets_information, 1, 1, 2)
    row_1_2.text = f'YES {CHECK_BOX} NO {CHECK_BOX}'
    row_3_4 = get_merge_cells(table_pets_information, 1, 3, 4)
    row_3_4.text = 'Numbers of Pets'
    set_cell_border(row[5], bottom={"sz": 6, "color": "#000000", "val": "single"})
    row[6].text = 'Type'
    row_7_8 = get_merge_cells(table_pets_information, 1, 7, 8)
    set_cell_border(row_7_8, bottom={"sz": 6, "color": "#000000", "val": "single"})
    row[9].text = 'Breed'
    row_10_12 = get_merge_cells(table_pets_information, 1, 10, 12)
    set_cell_border(row_10_12, bottom={"sz": 6, "color": "#000000", "val": "single"})

    paragraph = document.add_paragraph()
    paragraph.aligmnet = WD_TAB_ALIGNMENT
    paragraph_font = paragraph.style.font
    paragraph_font.name = 'Calibri'
    paragraph_font.size = Pt(10)

def get_table_personal_references():
    table_personal_references = document.add_table(rows=3, cols=8)
    header_cell = get_merge_cells(table_personal_references, 0, 0, 7)
    header_cell.text = 'PERSONAL REFERENCES'
    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    header_cell._tc.get_or_add_tcPr().append(table_header_color)

    table_personal_references.autofit = False

    second_row = table_personal_references.rows[1].cells
    second_row[0].text = 'Full Name'
    second_1_3_row = get_merge_cells(table_personal_references, 1, 1, 3)
    set_cell_border(second_1_3_row, bottom={"sz": 6, "color": "#000000", "val": "single"})
    second_row[4].text = 'Address'
    second_5_7_row = get_merge_cells(table_personal_references, 1, 5, 7)
    set_cell_border(second_5_7_row, bottom={"sz": 6, "color": "#000000", "val": "single"})

    third_row = table_personal_references.rows[2].cells
    third_row[0].text = 'Relationship'
    third_1_3_row = get_merge_cells(table_personal_references, 2, 1, 3)
    set_cell_border(third_1_3_row, bottom={"sz": 6, "color": "#000000", "val": "single"})
    third_row[4].text = 'Phone #'
    third_5_7_row = get_merge_cells(table_personal_references, 2, 5, 7)
    set_cell_border(third_5_7_row, bottom={"sz": 6, "color": "#000000", "val": "single"})

    paragraph = document.add_paragraph()
    paragraph.aligmnet = WD_TAB_ALIGNMENT
    paragraph_font = paragraph.style.font
    paragraph_font.name = 'Calibri'
    paragraph_font.size = Pt(10)

def get_extra_table_personal_references():
    table_personal_references = document.add_table(rows=2, cols=8)
    table_personal_references.autofit = False

    second_row = table_personal_references.rows[0].cells
    second_row[0].text = 'Full Name'
    second_1_3_row = get_merge_cells(table_personal_references, 0, 1, 3)
    set_cell_border(second_1_3_row, bottom={"sz": 6, "color": "#000000", "val": "single"})
    second_row[4].text = 'Address'
    second_5_7_row = get_merge_cells(table_personal_references, 0, 5, 7)
    set_cell_border(second_5_7_row, bottom={"sz": 6, "color": "#000000", "val": "single"})

    third_row = table_personal_references.rows[1].cells
    third_row[0].text = 'Relationship'
    third_1_3_row = get_merge_cells(table_personal_references, 1, 1, 3)
    set_cell_border(third_1_3_row, bottom={"sz": 6, "color": "#000000", "val": "single"})
    third_row[4].text = 'Phone #'
    third_5_7_row = get_merge_cells(table_personal_references, 1, 5, 7)
    set_cell_border(third_5_7_row, bottom={"sz": 6, "color": "#000000", "val": "single"})

    paragraph = document.add_paragraph()
    paragraph.aligmnet = WD_TAB_ALIGNMENT
    paragraph_font = paragraph.style.font
    paragraph_font.name = 'Calibri'
    paragraph_font.size = Pt(10)

def get_table_additional_questions():
    questions = [
        'Have you ever broken a rental agreement?',
        'Have you ever been evicted or asked to move?',
        'Have you ever refused to pay the rent?',
        'Have you ever filed for bankruptcy?',
        'Have you ever been convicted of a crime?',
        'Are any of the occupants smokers?'
    ]
    table_additional_questions = document.add_table(rows=12, cols=12)
    header_cell = get_merge_cells(table_additional_questions, 0, 0, 11)
    header_cell.text = 'ADDITIONAL QUESTIONS'
    table_header_color = parse_xml(r'<w:shd {} w:fill="#1f3864"/>'.format(nsdecls('w')))
    header_cell._tc.get_or_add_tcPr().append(table_header_color)

    table_additional_questions.autofit = False

    for row_number in range(1,12):
        if row_number%2 != 0:
            row_0_4 = get_merge_cells(table_additional_questions, row_number, 0, 4)
            row_0_4.text = questions[row_number//2]
            row_5_6 = get_merge_cells(table_additional_questions, row_number, 5, 6)
            row_5_6.text = f'YES {CHECK_BOX} NO {CHECK_BOX}'
            row_7_11 = get_merge_cells(table_additional_questions, row_number, 7, 11)
            row_7_11.text = ''
        else:
            row_0_2 = get_merge_cells(table_additional_questions, row_number, 0, 2)
            row_0_2.text = 'If yes, provide explanation'
            row_3_11 = get_merge_cells(table_additional_questions, row_number, 3, 11)
            set_cell_border(row_3_11, bottom={"sz": 6, "color": "#000000", "val": "single"})

def get_additionals_questions_text():
    header_paragraph = document.add_paragraph()

    text = '''
    It is against a law to discriminate against any person in the terms, conditions, or privileges of sale or rental of
    a dwelling, or in the provision of services or facilities in connection therewith, because of race, color, religion,
     sex, familial status, or national origin.

    By filling this form, applicant authorizes the verification of all information provided in this application,
     including Social Security Number, employment and income, credit history, previous and current rental history and
     any other relevant information necessary for the Landlord to evaluate the application.

    Applicant agrees that false or incomplete information filled in this application may result in a rejection
    of this application and/or termination of a rental agreement.
    Non-refundable application fee: $20.00
    '''
    header_paragraph.aligmnet = WD_TAB_ALIGNMENT
    header_paragraph_font = header_paragraph.style.font
    header_paragraph_font.name = 'Calibri'
    header_paragraph_font.size = Pt(12)
    header_paragraph.add_run(dedent(text)).bold = True

def get_table_sign():
    table_sign = document.add_table(4,3)

    first_row = table_sign.rows[0].cells
    first_row[0].text = ''
    set_cell_border(first_row[0], bottom={"sz": 6, "color": "#000000", "val": "single"})
    first_row[1].text = ''
    first_row[2].text = ''

    set_cell_border(first_row[2], bottom={"sz": 6, "color": "#000000", "val": "single"})

    second_row = table_sign.rows[1].cells
    aplicant_parg = second_row[0].add_paragraph('Applicant Signature')
    aplicant_parg.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
    second_row[1].text = ''
    landlord_parg = second_row[2].add_paragraph('Landlord Signature')
    landlord_parg.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER

    third_row = table_sign.rows[2].cells
    third_row[0].text = ''
    set_cell_border(third_row [0], bottom={"sz": 6, "color": "#000000", "val": "single"})
    third_row[1].text = ''
    set_cell_border(third_row [2], bottom={"sz": 6, "color": "#000000", "val": "single"})

    fourth_row = table_sign.rows[3].cells
    first_date_parg = fourth_row[0].add_paragraph('Date')
    first_date_parg.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
    fourth_row[1].text = ''
    first_second_date = fourth_row[2].add_paragraph('Date')
    first_second_date.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER

    paragraph = document.add_paragraph()
    paragraph.aligmnet = WD_TAB_ALIGNMENT
    paragraph_font = paragraph.style.font
    paragraph_font.name = 'Calibri'
    paragraph_font.size = Pt(10)

    # for row in range(4):
    #     table_sign.rows[row].cells[0].width = Inches(2.5)
    # table_sign.rows[0].cells[0].text = ''
    # signature = table_sign.rows[1].cells[0].add_paragraph('Landlord Signature')
    # signature.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
    # # table_sign.rows[1].cells[0].text = 'Landlord Signature'
    # table_sign.rows[2].cells[0].text = ''
    # # set_cell_border(table_sign.rows[2].cells[0], bottom={"sz": 6, "color": "#000000", "val": "single"}, )
    # table_sign.rows[3].cells[0].text = 'Date'

if __name__=='__main__':

    document = Document()

    sections = document.sections
    for section in sections:
        section.top_margin = Cm(0.5)
        section.bottom_margin = Cm(1)
        section.left_margin = Cm(1)
        section.right_margin = Cm(1)

    get_header()
    get_header_paragraph('test')
    get_table_applicant_information()
    get_table_additional_occupant()
    get_table_residence_history()
    document.add_paragraph()
    get_table_residence_history_no_header()
    document.add_paragraph()
    get_table_residence_history_no_header()
    document.add_page_break()
    get_table_employment_information()
    document.add_paragraph()
    get_table_employment_information_no_header()
    document.add_paragraph()
    get_table_employment_information_no_header()
    document.add_paragraph()
    get_table_additional_income()
    document.add_paragraph()
    get_table_vehicle_information()
    document.add_paragraph()
    get_table_pets_information()
    document.add_paragraph()
    get_table_personal_references()
    get_extra_table_personal_references()
    get_extra_table_personal_references()
    document.add_paragraph()
    get_table_additional_questions()
    get_additionals_questions_text()
    get_table_sign()

    for table in document.tables:
        set_table_alignment(table)

    document.save('test_doc.docx')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import webapp2

class MainPage(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'text/html; charset=UTF-8'
        self.response.write('こんにちは！')

app = webapp2.WSGIApplication([
    ('/', MainPage),
], debug=True)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Read raw text dumps from http://data.statmt.org/ngrams/raw/
and prints url and number of bytes
"""

import sys
from collections import defaultdict

buf = defaultdict(int)
domain = None

for line in sys.stdin:
    d, lang, n = line.split()
    n = int(n)
    if domain != d:
        for l in buf:
            sys.stdout.write("%s %s %d\n" % (domain, l, buf[l]))
        buf = defaultdict(int)
        domain = d
    buf[lang] += n

for l in buf:
    sys.stdout.write("%s %s %d\n" % (domain, l, buf[l]))


from rich.console import Group
from rich.panel import Panel
from rich.table import Table

from app.model.schema import TxDataDetail

class TxDetailInfo:
    def __init__(self, tx_detail: TxDataDetail) -> None:
        self.tx_detail = tx_detail

    def __str__(self) -> str:
        return str(self.tx_detail)

    def __rich__(self) -> Group:
        common_table = Table(box=None, expand=False, show_header=False, show_edge=False)
        common_table.add_column(style="deep_pink2 bold")
        common_table.add_column()

        common_table.add_row("Transaction Hash:", self.tx_detail.hash)
        common_table.add_row("Block:", str(self.tx_detail.block_number))
        common_table.add_row("From:", self.tx_detail.from_address)
        common_table.add_row("Nonce:", str(self.tx_detail.nonce))
        common_table.add_row("To:", self.tx_detail.to_address)

        common_table.add_row("Value:", str(self.tx_detail.value))
        common_table.add_row("Gas Price:", str(self.tx_detail.gas_price))
        common_table.add_row("Gas:", str(self.tx_detail.gas))

        contract_table = Table(
            box=None, expand=False, show_header=False, show_edge=False
        )
        contract_table.add_column(style="deep_pink2 bold")
        contract_table.add_row("Contract Name:", self.tx_detail.contract_name)
        contract_table.add_row("Contract Function:", self.tx_detail.contract_function)
        if self.tx_detail.contract_parameters is not None:
            function_arguments_table = Table(
                box=None, expand=False, show_header=False, show_edge=False
            )
            for k, v in self.tx_detail.contract_parameters.items():
                function_arguments_table.add_row(f"{k}: ", str(v))
            contract_table.add_row(
                "Contract Function Arguments:", Panel(function_arguments_table)
            )

        return Group(
            Panel(common_table, expand=True, title="Common", title_align="left"),
            Panel(contract_table, expand=True, title="Contract", title_align="left"),
        )


import sys
from io import StringIO

test_input1 = '''3, 6
7, 1, 3, 3, 2, 1
1, 3, 9, 8, 5, 6
4, 6, 7, 9, 1, 0
'''
test_input2 = '''2, 4
10, 11, 12, 13
14, 15, 16, 17
'''

# sys.stdin = StringIO(test_input2)

def read_matrix():
    n, m = [int(x) for x in input().split(', ')]

    matrix = []

    for _ in range(n):
        row = [int(x) for x in input().split(', ')]
        matrix.append(row)
    return matrix

matrix = read_matrix()

sums = []
n = len(matrix)
m = len(matrix[0])
max_sum = 0
position = 0
for r in range(n - 1):
    for c in range(m - 1):
        current_sum = matrix[r][c] + \
            matrix[r][c + 1] +\
            matrix[r+1][c] +\
            matrix[r+1][c+1]
        sums.append((
            current_sum,
            r,
            c
        ))
        if current_sum > max_sum:
            max_sum = current_sum
            position = (r, c)

r, c = position
print(f"{matrix[r][c]} {matrix[r][c+1]}")
print(f"{matrix[r+1][c]} {matrix[r+1][c+1]}")

print(max_sum)

03-Multidimensional Lists/07. Square with Maximum Sum.py


import time

from web_app.netcat.netcat import Netcat
from web_app.utils.recipes import read_all

from pandas import read_csv

# path = '/Users/joaoneves/Documents/demo-siemens/data/g_force_journeys/*.csv'
path = '/Users/joaoneves/Documents/demo-siemens/data/journeys/*.csv'
files = read_all(path)

serialize = lambda x: [','.join([str(i) for i in row[1:-1]]) for row in x.values.tolist()]
# serialize = lambda x: [','.join([str(i) for i in row]) for row in x.values.tolist()]

journeys = [serialize(read_csv(file)) for file in files]

nc = Netcat(ip='localhost', port=5900)
# nc = Netcat(ip='localhost', port=5901)

for journey in journeys:

    for row in journey:
        time.sleep(0.5)
        nc.write('{}\n'.format(row))

nc.close()


from spyre import server

import pickle

class StockExample(server.App):

    title = "Historical Stock Prices"

    inputs = [{
        "input_type": 'dropdown',
        "label": 'Company',
        "options": [
            {"label": "Google", "value": "GOOG"},
            {"label": "Yahoo", "value": "YHOO"},
            {"label": "Apple", "value": "AAPL"}],
        "variable_name": 'ticker',
        "action_id": "plot"
    }]

    outputs = [
        {"output_type": "plot", "output_id": "plot"},
        {"output_type": "table", "output_id": "table"}
    ]

    def getData(self, params):
        dfs = pickle.load((open("stocks.pkl", "rb")))
        ticker = params['ticker']
        return dfs[ticker]

    def getPlot(self, params):
        df = self.getData(params)
        plt_obj = df.set_index('Date').drop(['volume'], axis=1).plot()
        plt_obj.set_ylabel("Price")
        fig = plt_obj.get_figure()
        return fig

if __name__ == '__main__':
    app = StockExample()
    app.launch(port=9093)


import argparse
import re

parser = argparse.ArgumentParser()

## Required parameters
parser.add_argument("--raw",
                    default='metrics/ref-raw.txt',
                    type=str,
                    help="The raw corpus.")

parser.add_argument("--clean",
                    default='metrics/tst-clean.txt',
                    type=str,
                    help="The clean corpus.")

args = parser.parse_args()

raw_file = '{}'.format(args.raw)
clean_file = '{}'.format(args.clean)

sents = []
with open(raw_file, 'r') as r:
    for line in r:
        line = re.sub(pattern="\[CLS\]|\[SEP\]|\[PAD\]|\[unused[0-9]+\]|\\n", repl="", string=line)
        line = re.sub(pattern=" +", repl=" ", string=line)
        line = re.sub(pattern="^ +| +$", repl="", string=line)
        sents.append(line)

with open(clean_file, 'w') as fp:
    for s in sents:
        fp.write(s)
        fp.write('\n')