"""
# Pyqtree

Pyqtree is a pure Python spatial index for GIS or rendering usage.
It stores and quickly retrieves items from a 2x2 rectangular grid area,
and grows in depth and detail as more items are added.
The actual quad tree implementation is adapted from
[Matt Rasmussen's compbio library](https://github.com/mdrasmus/compbio/blob/master/rasmus/quadtree.py)
and extended for geospatial use.

## Platforms

Python 2 and 3.

## Dependencies

Pyqtree is written in pure Python and has no dependencies.

## Installing It

Installing Pyqtree can be done by opening your terminal or commandline and typing:

    pip install pyqtree

Alternatively, you can simply download the "pyqtree.py" file and place
it anywhere Python can import it, such as the Python site-packages folder.

## Example Usage

Start your script by importing the module.

    import pyqtree

Setup the spatial index, giving it a bounding box area to keep track of.
The bounding box being in a four-tuple: (xmin,ymin,xmax,ymax).

    spindex = pyqtree.Index(bbox=[0,0,100,100])

Populate the index with items that you want to be retrieved at a later point,
along with each item's geographic bbox.

    # this example assumes you have a list of items with bbox attribute
    for item in items:
        spindex.insert(item=item, bbox=item.bbox)

Then when you have a region of interest and you wish to retrieve items from that region,
just use the index's intersect method. This quickly gives you a list of the stored items
whose bboxes intersects your region of interests.

    overlapbbox = (51,51,86,86)
    matches = spindex.intersect(overlapbbox)

There are other things that can be done as well, but that's it for the main usage!

## More Information:

- [Home Page](http://github.com/karimbahgat/Pyqtree)
- [API Documentation](http://pythonhosted.org/Pyqtree)

## License:

This code is free to share, use, reuse, and modify according to the MIT license, see LICENSE.txt.

## Credits:

Karim Bahgat (2015)

"""

__version__ = "0.24"

#PYTHON VERSION CHECK
import sys
PYTHON3 = int(sys.version[0]) == 3
if PYTHON3:
    xrange = range

#INTERNAL USE ONLY
def _normalize_rect(rect):
    x1, y1, x2, y2 = rect
    if x1 > x2:
        x1, x2 = x2, x1
    if y1 > y2:
        y1, y2 = y2, y1
    return (x1, y1, x2, y2)

class _QuadNode:
    def __init__(self, item, rect):
        self.item = item
        self.rect = rect

class _QuadTree:
    """
    Internal backend version of the index.

    The index being used behind the scenes. Has all the same methods as the user
    index, but requires more technical arguments when initiating it than the
    user-friendly version.

    Args:

    - **x**:
        The x center coordinate of the area that the quadtree should keep track of.
    - **y**
        The y center coordinate of the area that the quadtree should keep track of.
    - **width**:
        How far from the xcenter that the quadtree should look when keeping track.
    - **height**:
        How far from the ycenter that the quadtree should look when keeping track
    """

    def __init__(self, x, y, width, height, depth=0, maxitems=10, maxdepth=20):
        self.nodes = []
        self.children = []
        self.center = [x, y]
        self.width,self.height = width,height
        self.depth = depth
        self.maxitems = maxitems
        self.maxdepth = maxdepth

    def __iter__(self):
        def loopallchildren(parent):
            for child in parent.children:
                if child.children:
                    for subchild in loopallchildren(parent=child):
                        yield subchild
                yield child
        for child in loopallchildren(self):
            yield child

    def _insert(self, item, bbox):
        rect = _normalize_rect(bbox)
        if len(self.children) == 0:
            node = _QuadNode(item, rect)
            self.nodes.append(node)

            if len(self.nodes) > self.maxitems and self.depth < self.maxdepth:
                self._split()
        else:
            self._insert_into_children(item, rect)

    def _intersect(self, bbox, results=None):
        rect = bbox
        if results is None:
            rect = _normalize_rect(rect)
            results = set()
        # search children
        if len(self.children) > 0:
            if rect[0] <= self.center[0]:
                if rect[1] <= self.center[1]:
                    self.children[0]._intersect(rect, results)
                if rect[3] > self.center[1]:
                    self.children[1]._intersect(rect, results)
            if rect[2] > self.center[0]:
                if rect[1] <= self.center[1]:
                    self.children[2]._intersect(rect, results)
                if rect[3] > self.center[1]:
                    self.children[3]._intersect(rect, results)
        # search node at this level
        for node in self.nodes:
            if (node.rect[2] > rect[0] and node.rect[0] <= rect[2] and
                node.rect[3] > rect[1] and node.rect[1] <= rect[3]):
                results.add(node.item)
        return results

    #INTERNAL USE ONLY

    def _insert_into_children(self, item, rect):
        # if rect spans center then insert here
        if ((rect[0] <= self.center[0] and rect[2] > self.center[0]) and
            (rect[1] <= self.center[1] and rect[3] > self.center[1])):
            node = _QuadNode(item, rect)
            self.nodes.append(node)
        else:
            # try to insert into children
            if rect[0] <= self.center[0]:
                if rect[1] <= self.center[1]:
                    self.children[0]._insert(item, rect)
                if rect[3] > self.center[1]:
                    self.children[1]._insert(item, rect)
            if rect[2] > self.center[0]:
                if rect[1] <= self.center[1]:
                    self.children[2]._insert(item, rect)
                if rect[3] > self.center[1]:
                    self.children[3]._insert(item, rect)

    def _split(self):
        quartwidth = self.width/4.0
        quartheight = self.height/4.0
        halfwidth = self.width/2.0
        halfheight = self.height/2.0
        self.children = [_QuadTree(self.center[0] - quartwidth,
                                  self.center[1] - quartheight,
                                  width=halfwidth, height=halfheight,
                                  depth=self.depth + 1,
                                  maxitems=self.maxitems,
                                  maxdepth=self.maxdepth),
                         _QuadTree(self.center[0] - quartwidth,
                                  self.center[1] + quartheight,
                                  width=halfwidth, height=halfheight,
                                  depth=self.depth + 1,
                                  maxitems=self.maxitems,
                                  maxdepth=self.maxdepth),
                         _QuadTree(self.center[0] + quartwidth,
                                  self.center[1] - quartheight,
                                  width=halfwidth, height=halfheight,
                                  depth=self.depth + 1,
                                  maxitems=self.maxitems,
                                  maxdepth=self.maxdepth),
                         _QuadTree(self.center[0] + quartwidth,
                                  self.center[1] + quartheight,
                                  width=halfwidth, height=halfheight,
                                  depth=self.depth + 1,
                                  maxitems=self.maxitems,
                                  maxdepth=self.maxdepth)]
        nodes = self.nodes
        self.nodes = []
        for node in nodes:
            self._insert_into_children(node.item, node.rect)

#USER CLASSES AND FUNCTIONS

class Index(_QuadTree):
    """
    The top spatial index to be created by the user. Once created it can be
    populated with geographically placed members that can later be tested for
    intersection with a user inputted geographic bounding box. Note that the
    index can be iterated through in a for-statement, which loops through all
    all the quad instances and lets you access their properties.
    """
    def __init__(self, bbox, maxitems=10, maxdepth=20):
        """
        Parameters:

        - **bbox**: The coordinate system bounding box of the area that the quadtree should
            keep track of, as a 4-length sequence (xmin,ymin,xmax,ymax)
        - **maxmembers** (optional): The maximum number of items allowed per quad before splitting
            up into four new subquads. Default is 10.
        - **maxdepth** (optional): The maximum levels of nested subquads, after which no more splitting
            occurs and the bottommost quad nodes may grow indefinately. Default is 20.
        """
        x1,y1,x2,y2 = bbox
        width,height = x2-x1,y2-y1
        midx,midy = x1+width/2.0, y1+height/2.0
        self.nodes = []
        self.children = []
        self.center = [midx, midy]
        self.width,self.height = width,height
        self.depth = 0
        self.maxitems = maxitems
        self.maxdepth = maxdepth

    def insert(self, item, bbox):
        """
        Inserts an item into the quadtree along with its bounding box.

        Parameters:

        - **item**: The item to insert into the index, which will be returned by the intersection method
        - **bbox**: The spatial bounding box tuple of the item, with four members (xmin,ymin,xmax,ymax)
        """
        self._insert(item, bbox)

    def intersect(self, bbox):
        """
        Intersects an input boundingbox rectangle with all of the items
        contained in the quadtree.

        Parameters:

        - **bbox**: A spatial bounding box tuple with four members (xmin,ymin,xmax,ymax)

        Returns:

        - A list of inserted items whose bounding boxes intersect with the input rectangle.
        """
        return self._intersect(bbox)

    def countmembers(self):
        """
        Returns:

        - A count of the total number of members/items/nodes inserted into
            this quadtree and all of its child trees.
        """
        size = 0
        for child in self.children:
            size += child.countmembers()
        size += len(self.nodes)
        return size


import abc
from typing import (
    TYPE_CHECKING,
    Any,
    Iterable,
    MutableSequence,
    Optional,
    Sequence,
    TypeVar,
    Union,
    cast,
    no_type_check,
    overload,
)

import numpy as np

from docarray.utils._internal.misc import import_library

T_item = TypeVar('T_item')
T = TypeVar('T', bound='IndexingSequenceMixin')

IndexIterType = Union[slice, Iterable[int], Iterable[bool], None]

def _is_np_int(item: Any) -> bool:
    dtype = getattr(item, 'dtype', None)
    ndim = getattr(item, 'ndim', None)
    if dtype is not None and ndim is not None:
        try:
            return ndim == 0 and np.issubdtype(dtype, np.integer)
        except TypeError:
            return False
    return False  # this is unreachable, but mypy wants it

class IndexingSequenceMixin(Iterable[T_item]):
    """
    This mixin allow sto extend a list into an object that can be indexed
    a la numpy/pytorch.

    You can index into, delete from, and set items in a IndexingSequenceMixin like a numpy doc_list or torch tensor:

    .. code-block:: python
        docs[0]  # index by position
        docs[0:5:2]  # index by slice
        docs[[0, 2, 3]]  # index by list of indices
        docs[True, False, True, True, ...]  # index by boolean mask

    """

    _data: MutableSequence[T_item]

    @abc.abstractmethod
    def __init__(
        self,
        docs: Optional[Iterable[T_item]] = None,
    ):
        ...

    @abc.abstractmethod
    def __len__(self) -> int:
        ...

    @staticmethod
    def _normalize_index_item(
        item: Any,
    ) -> Union[int, slice, Iterable[int], Iterable[bool], None]:
        # basic index types
        if item is None or isinstance(item, (int, slice, tuple, list)):
            return item

        # numpy index types
        if _is_np_int(item):
            return item.item()

        index_has_getitem = hasattr(item, '__getitem__')
        is_valid_bulk_index = index_has_getitem and isinstance(item, Iterable)
        if not is_valid_bulk_index:
            raise ValueError(f'Invalid index type {type(item)}')

        if isinstance(item, np.ndarray) and (
            item.dtype == np.bool_ or np.issubdtype(item.dtype, np.integer)
        ):
            return item.tolist()

        # torch index types
        if TYPE_CHECKING:
            import torch
        else:
            torch = import_library('torch', raise_error=True)

        allowed_torch_dtypes = [
            torch.bool,
            torch.int64,
        ]
        if isinstance(item, torch.Tensor) and (item.dtype in allowed_torch_dtypes):
            return item.tolist()

        return item

    def _get_from_indices(self: T, item: Iterable[int]) -> T:
        results = []
        for ix in item:
            results.append(self._data[ix])
        return self.__class__(results)

    def _set_by_indices(self: T, item: Iterable[int], value: Iterable[T_item]):
        for ix, doc_to_set in zip(item, value):
            try:
                self._data[ix] = doc_to_set
            except KeyError:
                raise IndexError(f'Index {ix} is out of range')

    def _get_from_mask(self: T, item: Iterable[bool]) -> T:
        return self.__class__(
            [doc for doc, mask_value in zip(self, item) if mask_value]
        )

    def _set_by_mask(self: T, item: Iterable[bool], value: Sequence[T_item]):
        i_value = 0
        for i, mask_value in zip(range(len(self)), item):
            if mask_value:
                self._data[i] = value[i_value]
                i_value += 1

    def _del_from_mask(self: T, item: Iterable[bool]) -> None:
        idx_to_delete = [i for i, val in enumerate(item) if val]
        self._del_from_indices(idx_to_delete)

    def _del_from_indices(self: T, item: Iterable[int]) -> None:
        for ix in sorted(item, reverse=True):
            # reversed is needed here otherwise some the indices are not up to date after
            # each delete
            del self._data[ix]

    def __delitem__(self, key: Union[int, IndexIterType]) -> None:
        item = self._normalize_index_item(key)

        if item is None:
            return
        elif isinstance(item, (int, slice)):
            del self._data[item]
        else:
            head = item[0]  # type: ignore
            if isinstance(head, bool):
                item_ = cast(Iterable[bool], item)
                return self._del_from_mask(item_)
            elif isinstance(head, int):
                return self._del_from_indices(item)
            else:
                raise TypeError(f'Invalid type {type(head)} for indexing')

    @overload
    def __getitem__(self: T, item: int) -> T_item:
        ...

    @overload
    def __getitem__(self: T, item: IndexIterType) -> T:
        ...

    def __getitem__(self, item):
        item = self._normalize_index_item(item)

        if type(item) == slice:
            return self.__class__(self._data[item])

        if isinstance(item, int):
            return self._data[item]

        if item is None:
            return self

        # _normalize_index_item() guarantees the line below is correct
        head = item[0]  # type: ignore
        if isinstance(head, bool):
            return self._get_from_mask(item)
        elif isinstance(head, int):
            return self._get_from_indices(item)
        else:
            raise TypeError(f'Invalid type {type(head)} for indexing')

    @overload
    def __setitem__(self: T, key: IndexIterType, value: Sequence[T_item]):
        ...

    @overload
    def __setitem__(self: T, key: int, value: T_item):
        ...

    @no_type_check
    def __setitem__(self: T, key, value):
        key_norm = self._normalize_index_item(key)

        if isinstance(key_norm, int):
            self._data[key_norm] = value
        elif isinstance(key_norm, slice):
            self._data[key_norm] = value
        else:
            # _normalize_index_item() guarantees the line below is correct
            head = key_norm[0]
            if isinstance(head, bool):
                return self._set_by_mask(key_norm, value)
            elif isinstance(head, int):
                return self._set_by_indices(key_norm, value)
            else:
                raise TypeError(f'Invalid type {type(head)} for indexing')


import logging
from datetime import datetime

import redis

from rlq_scheduler.common.config_helper import GlobalConfigHelper
from rlq_scheduler.common.redis_connection import RedisConnectionFactory
from rlq_scheduler.common.system_events.event import *
from rlq_scheduler.common.utils.decorators import class_fetch_exceptions

class SystemEventConsumer:

    def __init__(self,
                 global_config: GlobalConfigHelper,
                 logger: logging.Logger,
                 host=None,
                 generic_event_callback=None,
                 system_status_changed_event_callback=None,
                 resource_status_changed_event_callback=None,
                 prepare_new_run_event_callback=None,
                 start_run_event_callback=None,
                 stats_updated_event_callback=None,
                 scheduling_started_event_callback=None,
                 trajectory_completed_event_callback=None,
                 scheduling_completed_event_callback=None,
                 execution_completed_event_callback=None,
                 run_completed_event_callback=None,
                 callbacks_context=None):
        self.name = 'SystemEventConsumer'
        self.global_config: GlobalConfigHelper = global_config
        self.logger: logging.Logger = logger
        redis_config = self.global_config.redis_config()
        if host is not None:
            redis_config['host'] = host
        self.redis_connection_factory = RedisConnectionFactory(redis_config)
        self.redis_connection: redis.Redis = self.redis_connection_factory.get_connection()
        self.system_info_topic = self.global_config.redis_system_info_topic()
        self.pubsub = None
        self.consumer_thread = None
        self.is_started = False
        self.callbacks_context = callbacks_context
        self.event_switcher = {
            SystemEvent.GENERIC: {
                'wrapper_callback': self._generic_event_callback_wrapper,
                'callback': generic_event_callback
            },
            SystemEvent.SYSTEM_STATUS_CHANGED: {
                'wrapper_callback': self._system_status_changed_event_callback_wrapper,
                'callback': system_status_changed_event_callback
            },
            SystemEvent.RESOURCE_STATUS_CHANGED: {
                'wrapper_callback': self._resource_status_changed_event_callback_wrapper,
                'callback': resource_status_changed_event_callback
            },
            SystemEvent.PREPARE_NEW_RUN: {
                'wrapper_callback': self._prepare_new_run_event_callback_wrapper,
                'callback': prepare_new_run_event_callback
            },
            SystemEvent.START_RUN: {
                'wrapper_callback': self._start_run_event_callback_wrapper,
                'callback': start_run_event_callback
            },
            SystemEvent.STATS_UPDATED: {
                'wrapper_callback': self._stats_updated_event_callback_wrapper,
                'callback': stats_updated_event_callback
            },
            SystemEvent.SCHEDULING_STARTED: {
                'wrapper_callback': self._scheduling_started_event_callback_wrapper,
                'callback': scheduling_started_event_callback
            },
            SystemEvent.TRAJECTORY_COMPLETED: {
                'wrapper_callback': self._trajectory_completed_event_callback_wrapper,
                'callback': trajectory_completed_event_callback
            },
            SystemEvent.SCHEDULING_COMPLETED: {
                'wrapper_callback': self._scheduling_completed_event_callback_wrapper,
                'callback': scheduling_completed_event_callback
            },
            SystemEvent.EXECUTION_COMPLETED: {
                'wrapper_callback': self._execution_completed_event_callback_wrapper,
                'callback': execution_completed_event_callback
            },
            SystemEvent.RUN_COMPLETED: {
                'wrapper_callback': self._run_completed_event_callback_wrapper,
                'callback': run_completed_event_callback
            }
        }

    def consume(self):
        self.logger.info('{} is going to subscribe for {} events'.format(self.name, self.system_info_topic),
                         resource='SystemConsumer')
        self.pubsub: redis.client.PubSub = self.redis_connection.pubsub()
        self.pubsub.subscribe(**{self.system_info_topic: self.event_callback})
        self.consumer_thread = self.pubsub.run_in_thread(sleep_time=0.001)
        self.is_started = True

    def stop(self):
        if self.consumer_thread is not None:
            self.consumer_thread.stop()
        if self.pubsub is not None:
            self.pubsub.close()
            self.pubsub.connection_pool.disconnect()
        self.logger.info('{} stopped'.format(self.name), resource='SystemConsumer')

    def event_callback(self, message):
        try:
            if message['type'] == 'message':
                event_json_data = message['data']
                event: Event = Event.from_json(event_json_data)
                if event.type in self.event_switcher:
                    wrapper_callback = self.event_switcher[event.type]['wrapper_callback']
                    wrapper_callback(event)
                else:
                    self.logger.error('No callback for event of type {}'.format(event.type), resource='SystemConsumer')
            if message['type'] == 'subscribe':
                self.logger.info('{} correctly subscribed to {}'.format(self.name, self.system_info_topic),
                                 resource='SystemConsumer')
        except Exception as e:
            self.logger.exception(e)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _generic_event_callback_wrapper(self, event: Event):
        self.logger.info('{} received {} event with payload: {} at time: {}'
                         .format(self.name, event, event.payload,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling generic callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _system_status_changed_event_callback_wrapper(self, event: SystemStatusChangedEvent):
        self.logger.info('SystemStatusChangedEvent received with status: {} at time: {}'
                         .format(event.system_status,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling system_status_changed callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _resource_status_changed_event_callback_wrapper(self, event: ResourceStatusChangedEvent):
        self.logger.info('ResourceStatusChangedEvent received from resource: {} with status: {} at time: {}'
                         .format(event.resource_name, event.resource_status,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling resource_status_changed callback with context: {}'
                              .format(self.callbacks_context), resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _prepare_new_run_event_callback_wrapper(self, event: PrepareNewRunEvent):
        self.logger.info('PrepareNewRunEvent received for resource {} at time: {}'
                         .format(event.new_run_config,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling prepare_new_run callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _start_run_event_callback_wrapper(self, event: StartRunEvent):
        self.logger.info('StartRunEvent received with run_code {} at time: {}'
                         .format(event.run_code,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling start_run callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _stats_updated_event_callback_wrapper(self, event: StatsUpdateEvent):
        self.logger.info('StatsUpdatedEvent received with run_code: {} stat_group: {} and stat_property: {} at time: {}'
                         .format(event.run_code, event.stat_group, event.stat_property,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling stats_updated callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _scheduling_started_event_callback_wrapper(self, event: SchedulingStartedEvent):
        self.logger.info('SchedulingStartedEvent received for phase: {} with task_to_generate: {} and'
                         ' run_config: {} at time: {}'
                         .format(event.phase, event.task_to_generate, event.task_to_schedule_config,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling scheduling_started callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _trajectory_completed_event_callback_wrapper(self, event: TrajectoryCompletedEvent):
        self.logger.info('TrajectoryCompletedEvent received with trajectory_id: {} at time: {}'
                         .format(event.trajectory_id,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling trajectory_completed callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _scheduling_completed_event_callback_wrapper(self, event: SchedulingCompletedEvent):
        self.logger.info('SchedulingCompletedEvent received for phase: {} with task_scheduled: {} at time: {}'
                         .format(event.phase, event.task_scheduled,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling scheduling_completed callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _execution_completed_event_callback_wrapper(self, event: ExecutionCompletedEvent):
        self.logger.info('ExecutionCompletedEvent received for phase: {} with task_executed: {} at time: {}'
                         .format(event.phase, event.task_executed,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling execution_completed callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)

    @class_fetch_exceptions(publish_error=False, is_context=False)
    def _run_completed_event_callback_wrapper(self, event: RunCompletedEvent):
        self.logger.info('RunCompletedEvent received with run_code: {} at time: {}'
                         .format(event.run_code,
                                 datetime.fromtimestamp(event.timestamp).strftime('%Y-%m-%d_%H:%M:S')),
                         resource='SystemConsumer')
        callback = self.event_switcher[event.type]['callback']
        if callback is not None:
            self.logger.debug('Calling run_completed callback with context: {}'.format(self.callbacks_context),
                              resource='SystemConsumer')
            callback(event, context=self.callbacks_context)


# This script is used to rescue sequenca output
# This is a WIP

# Filter name: "/home/rdmorin/agilent_sureselect_all_exons_v5_and_utr.sort.merge.bed"
import sys
import subprocess
import atexit

# Obtains arguments from the command line, stores them in an array, and returns the array.
# If there are no arguments, prints out a list of all availible options
def GetStartingVariables():
	print("")
	# If there are no arguments, print out the help message
	if len(sys.argv) == 1 :

		print("This script is designed to rescue variants that would normally be missed in the sequqnza cleanup")
		print("The following commands are availible\n")
		print("     -f <filterSeq.fa>    Reference file used in mpileup")
		print("     -d <int>             Minimum filter depth [Default:10]")
		print("     -D <int>             Maximum filter depth [Default:100]")
		print("     -l <filterFile.bed>  Only look for variants sites")
		print("     -o <amendment>       Ammends file name with this string [Default:_filt]")
		print("     -p <path>            Specify an output path for the files [Default is inputpath]")
		print("     -debug               Prints out all the variables used for mpileup,call, and filter, as well as input and outut file names")
		print("")
		exit()
	else:
		cmdArgs=[]

		for i in range(1, len(sys.argv)):

			cmdArgs.append(sys.argv[i])

	return cmdArgs

# Prints the output file names and parameters listed in the dictionary to the console
def printRawParameters(parameters):

	print("\nOutputFileNames")
	for names in parameters["output"]:
		print(names)

	print("\nDictionary parameters")
	for key, value in parameters.items():
		print(key + ": " + str(value))

def setOutputFileNames(inputSequences, ammendedPath, ammendedName="_filt"):
	"This sets an output file name for an input file using defaults or user variables (if supplied)"

	# If no input files were provided, quit the program
	outputNames=[]
	if len(inputSequences) == 0:
		print("No input file was specified")
		exit()

	# If there are input files,
	else:

		fileNumber=1
		# For each file provided
		for inputName in inputSequences:

			# Obtains the start of the file name in the string (Removed directory bit)
			inputFileStart=len(inputName)
			for char in reversed(inputName):

				if char == "/":

					break
				else:

					inputFileStart-=1

			# Obtains the end of the file name in the string (remoives file extensions)
			inputFileEnd=inputFileStart
			for char in inputName[inputFileStart:]:
				if char == ".":
					break
				else:
					inputFileEnd+=1

			# If the user has specified a file path, uses that instead of the default
			if not ammendedPath == "":

				outputFileName=ammendedPath
			else:

				outputFileName=(inputName[:inputFileStart])

			outputFileName +=  inputName[inputFileStart:inputFileEnd] + ammendedName + ".vcf.gz"
			outputNames.append(outputFileName)

	return outputNames

# Sorts arguments provided by the command line, and suplies defaults if no arguments are provided for some options. A complete set of parameters is returned as a dictionary
def SetParam(userInput):

	# Creates lists which contain the input out output file names
	inputSequences=[]
	# Creates two dictionaries, one which loads default parameters, while the other tracks if parameters were modified
	userParameters={"-d":10, "-D":100, "-f": "/morinlab/reference/igenomes/Homo_sapiens/GSC/GRCh38/Sequence/WholeGenomeFasta/GRCh38_no_alt.fa",
	"-l": False, "-o": "_filt", "input": inputSequences, "debuggingMode":False, "-p":""}
	modifiedParameters={"-d":False, "-D":False, "-f":False, "-l":False, "-o":False, "-p":False}

	# Cycles through each argument
	currentCom=-1
	optionAdded=False
	for args in userInput:

		currentCom +=1
		if optionAdded:
			optionAdded=False
			continue

		# If an input one of these option
		if (args  == "-f" or args == "-l" or args == "-d" or args == "-D" or args == "-o" or args == "-p"):

			# If there is is a paramter for the specified option
			if (currentCom+1) < len(userInput):

				# Modifies Dictionary values if they input types match
				if ((args == "-f" or args == "-l" or args == "-o" or args == "-p") and isinstance(userInput[currentCom+1], str)):

					userParameters[args]=userInput[currentCom+1]
					modifiedParameters[args]=True
					optionAdded=True

				#
				elif ((args == "-d" or args == "-D") and userInput[currentCom+1].isdigit()):

					userParameters[args]=userInput[currentCom+1]
					modifiedParameters[args]=True
					optionAdded=True

				else:
					#Replace this with an expection
					print("The specified paramer for " + args + " is invlid")
					exit()

			else:
				print("No parameter was specified for " + args)
				exit()

		# Activates debugging mode
		elif args == "-debug":

			userParameters["debuggingMode"]=True

		elif isinstance(args, str):

			inputSequences.append(args)
			modifiedParameters["seqInput"]=True

		else:

			print("Unknown input: " + args + ", ignoring...")

	userParameters["output"]=setOutputFileNames(inputSequences, userParameters["-p"], userParameters["-o"])

	print("")

	# Informs the user which options are being left at default
	for key, modified in modifiedParameters.items():
		if not modified:

			if key == "input" or key == "-l":
				continue

			else:
				print("[" + key + " was not supplied, using " + str(userParameters[key]) + " ]")

	return userParameters

def killChildProcesses(*children):

	print("Ending process")
	# for process in children:
		# process.terminate()

def runProcesses(parameters):

	# If using debugging mode, allow the stderr to print to the terminal
	if  parameters["debuggingMode"]:

		#If a region to be analyzed is defined, run the process using that region
		if (parameters["-l"]== False):
			currentFileNum=0
			for file in parameters["input"]:

				print(["bcftools", "filter", "-e", "\'DP<" + str(parameters["-d"]) + " || DP>" + str(parameters["-D"]) + "\'", "-o", parameters["output"][currentFileNum], "-O", "z"])
				mpileup=subprocess.Popen(["samtools", "mpileup", "-ugf", parameters["-f"], file], stdout=subprocess.PIPE)

				bcfCall=subprocess.Popen(["bcftools", "call", "-vmO", "z", "-o", parameters["output"][currentFileNum]], stdin=mpileup.stdout)

				# bcfFilter=subprocess.Popen(["bcftools", "filter", "-e", "\'DP<" + str(parameters["-d"]) + " || DP>" + str(parameters["-D"]) + "\'", "-o", parameters["output"][currentFileNum], "-O", "z", "-"], stdin=bcfCall.stdout)

				atexit.register(killChildProcesses, mpileup, bcfCall)
				currentFileNum += 1

		else:
			currentFileNum=0
			for file in parameters["input"]:

				mpileup=subprocess.Popen(["samtools", "mpileup", "-ugf", parameters["-f"], "-l", parameters["-l"], file], stdout=subprocess.PIPE)
				bcfCall=subprocess.Popen(["bcftools", "call", "-vmO", "z", "-o", parameters["output"][currentFileNum]], stdin=mpileup.stdout)
				# bcfFilter=subprocess.Popen(["bcftools", "filter", "-e", "DP<" + str(parameters["-d"]), "-o", parameters["output"][currentFileNum], "-O", "z", "-"], stdin=bcfCall.stdout)

				currentFileNum += 1

	else:
		if (parameters["-l"] == False):
			currentFileNum=0
			for file in parameters["input"]:

				print(["bcftools", "filter", "-e", "\'DP<" + str(parameters["-d"]) + " || DP>" + str(parameters["-D"]) + "\'", "-o", parameters["output"][currentFileNum], "-O", "z"])
				mpileup=subprocess.Popen(["samtools", "mpileup", "-ugf", parameters["-f"], file], stdout=subprocess.PIPE, stderr=None)

				bcfCall=subprocess.Popen(["bcftools", "call", "-vmO", "z", "-o", parameters["output"][currentFileNum]], stdin=mpileup.stdout, stderr=None)
				# bcfFilter=subprocess.Popen(["bcftools", "filter", "-e", "\'DP<" + str(parameters["-d"]) + " || DP>" + str(parameters["-D"]) + "\'", "-o", parameters["output"][currentFileNum], "-O", "z", "-"], stdin=bcfCall.stdout)

				currentFileNum += 1

		else:
			currentFileNum = 0
			for file in parameters["input"]:

				mpileup=subprocess.Popen(["samtools", "mpileup", "-ugf", parameters["-f"], "-l", parameters["-l"], file], stdout=subprocess.PIPE, stderr=None)
				bcfCall=subprocess.Popen(["bcftools", "call", "-vmO", "z", "-o", parameters["output"][currentFileNum]], stdin=mpileup.stdout, stderr=None)
				# bcfFilter=subprocess.Popen(["bcftools", "filter", "-e", "DP<" + str(parameters["-d"]), "-o", parameters["output"][currentFileNum], "-O", "z", "-"], stdin=bcfCall.stdout)

				currentFileNum += 1

cmdArgs=GetStartingVariables()

parameters=SetParam(cmdArgs)

# If debuggind mode is enabled, print out the options and filenames listed in the dictionary
if parameters["debuggingMode"]: printRawParameters(parameters)

print("\nSequencaRescuer: Starting mpileup\n")

runProcesses(parameters)

atexit.register(killChildProcesses, mpileup, bcfCall)


import json
import boto3

client = boto3.client('connect')

def lambda_handler(event, context):

    parameters = event['Details']['Parameters']['keep'].split(",")
    attributes = dict(event['Details']['ContactData']['Attributes'])
    contactID = str(event['Details']['ContactData']['ContactId'])
    newattrib = {}
    instanceID = event['Details']['ContactData']['InstanceARN'].split('/')[1]

    for key,value in attributes.items():
        if key in parameters:
            newattrib[key]=value
        else:
            newattrib[key]=''

    response = client.update_contact_attributes(
            InitialContactId = contactID,
            InstanceId = instanceID,
            Attributes=newattrib
        )

    return {
        'statusCode': 200,
    }


# -*- coding: utf-8 -*-
# UTF-8 encoding when using korean

#55/100point

T = int(input())
answer = []

for i in range(T):
	sum1 = 0

	N, M, K = map(int, input().split())
	L = list(map(int, input().split()))

	max_L = 0
	for num in range(N):
		if L[num] > L[max_L]:
			max_L = num
	m = L[max_L]

	for i in range(N):
		sum1 = sum1 + L[i]

	req = 3 * K + sum1
	sup = N * M
	count = req // sup + 1

	if m > count * M:
		count = m // M + 1

	answer.append(count)

for j in range(0, T):
	print(answer[j])


from .node import Node

class BinaryTree(object):
    def __init__(self, iterable=None):
        """ constructor method
        """
        self.root = None

        if iterable is not None:
            for i in range(len(iterable)):
                self.insert(iterable[i])

    def __str__(self):
        """ note for users
        """
        return f'{self.root}'

    def __repr__(self):
        """ note for devs
        """
        return f'<BST | Root: {self.root}>'

    def insert(self, value):
        """ insert a value into bt after validating its value
        """
        node = Node(value)
        if self.root is None:
            self.root = node
        current = self.root

        while current is not None:
            if current.val > node.val:
                if current.left is not None:
                    current = current.left
                else:
                    current.left = node
                    return
            elif current.val < node.val:
                if current.right is not None:
                    current = current.right
                else:
                    current.right = node
                    return
            else:
                return

    def pre_order(self, callable=lambda node: print(node)):
        """ Go visit, then go left and go right
        """
        def _walk(node=None):
            if node is None:
                return

            # Go Visit
            callable(node)

            # Go Left
            if node.left is not None:
                _walk(node.left)

            # Go Right
            if node.right is not None:
                _walk(node.right)

        _walk(self.root)

    def in_order(self, callable=lambda node: print(node)):
        """ Go left, visit, then go right
        """
        def _walk(node=None):
            if node is None:
                return

            # Go Left
            if node.left is not None:
                _walk(node.left)

            # Go Visit
            callable(node)

            # Go Right
            if node.right is not None:
                _walk(node.right)

        _walk(self.root)

    def post_order(self, callable=lambda node: print(node)):
        """ Go left, go right, then visit
        """
        def _walk(node=None):
            if node is None:
                return

            # Go Left
            if node.left is not None:
                _walk(node.left)

            # Go Right
            if node.right is not None:
                _walk(node.right)

            # Go Visit
            callable(node)

        _walk(self.root)

    def breadth_first(self, callable=lambda front: print(front)):
        """ Breadth first approach traversing through a bt
        """
        if self.root is None:
            raise ValueError

        queue = []
        queue.append(self.root)

        try:
            while queue[0] is not None:
                front = queue.pop(0)
                callable(front)
                if front.left is not None:
                    queue.append(front.left)
                if front.right is not None:
                    queue.append(front.right)
        except IndexError:
            return


class Solution:
    def maxProfit(self, prices: List[int]) -> int:

        lowest_buy = float("inf")
        best_profit = 0

        # rank sell times in order of preference
        sell_times = iter(reversed(sorted(
            list(range(0, len(prices))),
            key=lambda i: prices[i]
        )))

        buy = 0
        sell = next(sell_times)

        while buy < len(prices):
            if sell < buy:
                try:
                    sell = next(sell_times)
                except StopIteration:
                    break
            elif prices[buy] >= lowest_buy:
                buy += 1
            else:
                profit = prices[sell] - prices[buy]
                best_profit = max(best_profit, profit)
                lowest_buy = min(lowest_buy, prices[buy])

                buy += 1

        return best_profit


# -*- coding: utf-8 -*-
""" spfs/utils/datasets/handlers """

import json
import os

import numpy as np
from gutils.numpy_.numpy_ import LabelMatrixManager
from sklearn.model_selection import train_test_split

import settings
from spfs.utils.datasets.items import InMemoryDatasetItems, LazyDatasetItems
from spfs.utils.datasets.mixins import BaseDBHandlerMixin, DataTransformsMixin
from spfs.utils.utils import using_quick_tests

class BaseDBHandler(BaseDBHandlerMixin):
    """
    Base database handler providing only general intialization
    functionality. DO NOT USE IT DIRECTLY, you must use this class to
    create a Dataset handler that defines how to load and return the features
    properly

    Usage:
        train_feats, train_labels, test_feats, test_labels = DBhandler()()

    Inspired on: https://github.com/giussepi/INCREMENTAL-LC-KSVD/blob/master/utils/datasets/patchcamelyon.py

    Returns:
        train/test feats (list of lists)
        train/test labels (np.ndarray) with shape [num labels, num samples]
    """

    def __init__(self, verbose=False):
        """ Loads the training and testing datasets """
        for db_split, db_path, attr_name in (
                ('training', settings.TRAINING_DATA_DIRECTORY_DATASET_PATH, 'training_data'),
                ('testing', settings.TESTING_DATA_DIRECTORY_DATASET_PATH, 'testing_data')
        ):
            if verbose:
                print("Loading {} dataset".format(db_split))

            with open(db_path, 'r') as file_:
                data = json.load(file_)

            setattr(self, attr_name, dict())

            # if it's not a 2D label matrix coded as JSON, then turn it into a 2D matrix
            if not isinstance(data['labels'][0], list):
                getattr(self, attr_name)['labels'] = LabelMatrixManager.get_2d_matrix_from_1d_array(
                    np.array(data['labels']))
            else:
                getattr(self, attr_name)['labels'] = np.array(data['labels'])

            # QuickTests
            if using_quick_tests():
                getattr(self, attr_name)['labels'] = \
                    getattr(self, attr_name)['labels'][:, :settings.QUICK_TESTS]

            getattr(self, attr_name)['codes'] = data['codes']

class InMemoryDBHandler(BaseDBHandler):
    """
    Memory-intensive dataset handler

    Handles the codes by loading them into memory using a single 2D matrix

    Usage:
        from core.feature_extractors import SpatialPyramidFeatures
        from utils.datasets.handlers import InMemoryDBHandler

        train_feats, train_labels, test_feats, test_labels = InMemoryDBHandler()()

        spf = SpatialPyramidFeatures(LazyDBHandler)
        spf.create_codebook()
        spf.create_spatial_pyramid_features()

    Returns:
        training_data_codes (InMemoryDatasetItems): see class definiton
        training_data_labels          (np.ndarray): array with shape [feats, samples]
        testing_data_codes  (InMemoryDatasetItems): see class definiton
        testing_data_labels           (np.ndarray): array with shape [feats, samples]
    """

    def __init__(self, verbose=False):
        super().__init__(verbose)
        self.training_data['codes'] = InMemoryDatasetItems(self.training_data['codes'])
        self.testing_data['codes'] = InMemoryDatasetItems(self.testing_data['codes'])

class LazyDBHandler(BaseDBHandler):
    """
    Memory-efficent dataset handler

    Handles the codes, in a memory efficent way, by loading one at a time into memory only when
    it is required

    Usage:
        from core.feature_extractors import SpatialPyramidFeatures
        from utils.datasets.handlers import LazyDBHandler

        train_feats, train_labels, test_feats, test_labels = LazyDBHandler()()

        spf = SpatialPyramidFeatures(LazyDBHandler)
        spf.create_codebook()
        spf.create_spatial_pyramid_features()

    Returns:
        training_data_codes (LazyDatasetItems): see class definiton
        training_data_labels      (np.ndarray): array with shape [feats, samples]
        testing_data_codes  (LazyDatasetItems): see class definiton
        testing_data_labels       (np.ndarray): array with shape [feats, samples]
    """

    def __init__(self, verbose=False):
        super().__init__(verbose)
        self.training_data['codes'] = LazyDatasetItems(self.training_data['codes'])
        self.testing_data['codes'] = LazyDatasetItems(self.testing_data['codes'])

class FeatsHandler(DataTransformsMixin, BaseDBHandlerMixin):
    """
    Handler for generated spatial pyramid features

    The format of the arrays are [features, samples]

    Usage:
        train_feats, train_labels, test_feats, test_labels = FeatsHandler()()
        # create a subdataset using a percentage and load it
        FeatsHandler().create_subsets(percentage=20, verbose=True)
        train_feats, train_labels, test_feats, test_labels = FeatsHandler(percentage=20, verbose=True)()

    Inspired on:
        https://github.com/giussepi/INCREMENTAL-LC-KSVD/blob/master/utils/datasets/patchcamelyon.py
    """

    def __init__(self, percentage=0, verbose=False):
        """ Loads the training and testing datasets """
        assert isinstance(percentage, int)

        if 0 < percentage < 100:
            train_path = settings.GENERATED_FEATS_FILENAME_TEMPLATE.format(
                '{}%_train'.format(percentage))
            test_path = settings.GENERATED_FEATS_FILENAME_TEMPLATE.format(
                '{}%_test'.format(percentage))
        else:
            train_path = settings.GENERATED_FEATS_FILENAME_TEMPLATE.format('train')
            test_path = settings.GENERATED_FEATS_FILENAME_TEMPLATE.format('test')

        filepath = os.path.join(settings.GENERATED_DATA_DIRECTORY, train_path)

        if verbose:
            print("Loading training dataset")
        with open(filepath, 'r') as file_:
            self.training_data = json.load(file_)
            self.to_numpy(self.training_data)

        filepath = os.path.join(settings.GENERATED_DATA_DIRECTORY, test_path)

        if verbose:
            print("Loading testing dataset")
        with open(filepath, 'r') as file_:
            self.testing_data = json.load(file_)
            self.to_numpy(self.testing_data)

    def create_subsets(self, percentage=20, verbose=False):
        """
        Creates subsets of the spatial pyramid features training dataset considering
        the provided percentage as the percentage covered by the subset training dataset.
        Thus, the features training dataset is splitted into 'percentage'% for training
        and '100-percentage' % for testing.

        Args:
            percentage (int): percentage of the featues dataset to be used for the training subset
            verbose   (bool): whether to pring messages or not
        """
        assert 0 < percentage < 100

        train_feats, train_labels, test_feats, test_labels = self()

        filepath = os.path.join(
            settings.GENERATED_DATA_DIRECTORY,
            settings.GENERATED_FEATS_FILENAME_TEMPLATE.format('{}%_train'.format(percentage))
        )

        if verbose:
            print("Saving training dataset subset at {}".format(filepath))

        with open(filepath, 'w') as file_:
            xtrain, _, ytrain, _ = train_test_split(
                train_feats.T, train_labels[1], train_size=percentage/100,
                random_state=settings.RANDOM_STATE, shuffle=True,
                stratify=train_labels[1]
            )
            formatted_data = dict(
                codes=xtrain.T.tolist(),
                labels=LabelMatrixManager.get_2d_matrix_from_1d_array(ytrain).tolist()
            )
            json.dump(formatted_data, file_)

        filepath = os.path.join(
            settings.GENERATED_DATA_DIRECTORY,
            settings.GENERATED_FEATS_FILENAME_TEMPLATE.format('{}%_test'.format(percentage))
        )

        if verbose:
            print("Saving testing dataset subset at {}".format(filepath))

        with open(filepath, 'w') as file_:
            xtrain, _, ytrain, _ = train_test_split(
                test_feats.T, test_labels[1], train_size=percentage/100,
                random_state=settings.RANDOM_STATE, shuffle=True,
                stratify=test_labels[1]
            )
            formatted_data = dict(
                codes=xtrain.T.tolist(),
                labels=LabelMatrixManager.get_2d_matrix_from_1d_array(ytrain).tolist()
            )
            json.dump(formatted_data, file_)


name = "  hamzaARAIN is"
char = '4'
num = 1, 2, 3, 4
n = 4

variable = name.capitalize()                       #  capitalize() -> Hamzaarain
variable = name.title()                                #  title()         -> Hamzaarain
variable = name.lower()                             #  lower()      -> hamzaarain
variable = name.upper()                            #  upper()      -> HAMZAARAIN

variable = name.index("h")                           # index() -> 2
variable = name.endswith("is")                  # True
variable = name.startswith("hamza")             # False

variable = name.join("."+char)    # join() -> .  hamzaARAIN is4  -> to concatinate
variable = name.replace("is", "are")            # replace()  -> hamzaARAIN are
variable = name.find("a", 2)                        # find()  -> ('item', index to start find ) 3th index after 2nd index

variable = name.strip()                                # strip()  -> (move white spaces) amzaARAIN is
variable = name.rstrip()                                # strip()  -> (remove white spaces of R) amzaARAIN is
variable = name.lstrip()                                # strip()  -> (remove white spaces of l) amzaARAIN is
variable = name.rjust(10, '*')                      # rjust() -> return white spaces with string on right
variable = name.center(10, '=')                     # center() -> return white spaces with string on center
variable = name.ljust(10)                           # ljust() -> return white spaces with string on left

variable = name.count('a')                          # count()  -> total items
variable = len(name)                                    # len() -> (total char) 15
variable = range(1, 5, 2)                                   # range() -> 1, 3 -> range(start, end+2, jump)

variable = type(name)                                   # type() ->  <class 'str'> used as T/F
variable = list(num)                                      # list()  -> [1, 2, 3, 4]
variable = tuple(num)                                   # tuple()  -> (1, 2, 3, 4)
variable = dict()                                             # empty dictinoary
variable = set(num)                                        # Converting into set {1, 2, 3, 4}
variable = float(n)                                          # float()  -> 4.0
variable = str(n)                                             # str() ->  '4'
variable = int(n)                                              # int() -> 4

variable = input("enter name: ")                   # input()   -> to input

name1, name2 = ("Hamza, Arain").split(',')    # split() -> to split by char
#   |               |
# Hamza     Arain

variable = name.isupper()               # Checj string is upper
variable = name.lower()                 # Checj string is lower
variable = name.isascii()              # isascii() -> True -> T:if all ASCII in string
variable = char.isalpha()              # isascii() -> False  -> T: if all char in sring are aplhabatic not numeric
variable = char.isdecimal()          # isdecimal() -> True  -> T: if all char in sring are deciaml not alphabatic
variable = char.isalnum()              # isalnum() -> check all is number
variable = char.isspace()               # isspace() -> check white spaces
variable = char.istitle()               # istitle() -> check string is title

variable = max(5,7)                     # max() -> 7
variable = min(5,7)                      # min() -> 5

# print(variable)
# print(name1, name2)


import requests
from bs4 import BeautifulSoup
print("University No: SJC22MCA-2021 \nName: Christin Benny \nBatch: S3 MCA \n______________________________________\n")
def getdata(url):
    r=requests.get(url)
    return r.content
htmldata = getdata("https://www.toppr.com/guides/essays/globalization-essay/")
soup = BeautifulSoup(htmldata,'html.parser')
data =''
pr = len(soup.find_all('p'))
print("P tag:",pr)
for data in soup.find_all('p'):
    print(data.get_text())


import ipdb
import numpy

class ScribeParameters:
	def __init__(self, seed = 0):
		numpy.random.seed(seed = seed)
		self.batch_size = 20
		self.frame_size = 3
		self.k = 20
		self.target_size = self.frame_size * self.k
		self.hidden_size_recurrent = 400
		self.lr = 10 ** (7*numpy.random.rand() - 8)
		self.max_size = 1200
		self.seq_size = 100
		self.tbptt_flag = True
		self.exp_name = 'baseline_{}'.format(seed)
		self.n_batches = 2000
		self.tbptt_flag = True

if __name__ == "__main__":
	scribe_params = ScribeParameters()
	ipdb.set_trace()


"""
A more flexible VLOOKUP using Natural Language Processing
from scikit-learn.

To use in Excel add the folder this file is in to your pythonpath
in the pyxll.cfg file and add 'nlp_vlookup' to the modules list
in the same file.
"""
from sklearn import feature_extraction, metrics
from pyxll import xl_func, xl_return
from typing import List, Any
import pandas as pd

@xl_func(name="NLP.VLOOKUP")
@xl_return("dataframe<index=False, columns=False>")
def nlp_vlookup(value: str,
                table: List[List[Any]],
                col_index: int = None,
                include_score: bool = False,
                all_matches: bool = False,
                threshold: float = 0.5) -> pd.DataFrame:
    """
    Do a 'VLOOKUP' for value in table and return the value at col_index
    of the matching row.

    :param value: Value to match.
    :param table: Table of input values with the candidates in the left most column.
    :param col_index: Index of column in the table to return.
    :param include_score: Include the score in the return value.
    :param all_matches: If True return a table of all matches.
    :param threshold: Exclude matches below this threshold (0-1).
    """
    # Get the first column in the table as the list of words to match.
    words = [str(x[0]) for x in table]

    # Vectorize all the strings by creating a Bag-of-Words matrix, which extracts
    # the vocabulary from the corpus and counts how many times the words appear in each string.
    vectorizer = feature_extraction.text.CountVectorizer()
    vectors = vectorizer.fit_transform([value]+words).toarray()

    # Then, we calculate the cosine similarity, a measure based on the angle between two
    # non-zero vectors, which equals the inner product of the same vectors normalized to both
    # have length 1.
    cosine_sim = metrics.pairwise.cosine_similarity(list(vectors))

    # Get the scores for each word and put them into a DataFrame.
    scores = cosine_sim[0][1:]
    scores_df = pd.DataFrame({"score": scores}, index=words)

    # Join the input table and the scores into a single DataFrame.
    table_df = pd.DataFrame(table, index=words)
    df = table_df.join(scores_df)

    # Filter out any with a score below the threshold.
    df = df[df["score"] >= threshold]

    # If there are no matches then raise an exception.
    if not len(df.index):
        raise ValueError("No matches found")

    # Sort by score.
    df = df.sort_values(by="score", ascending=False)

    # Get the top result if not returning all matches.
    if not all_matches:
        df = df.head(1)

    # Reindex to get only the columns we're interested in and put the score first.
    columns = table_df.columns.to_list() if col_index is None else [col_index-1]
    if include_score:
        columns = ["score"] + columns
    df = df.reindex(columns=columns)

    return df

if __name__ == "__main__":
    words = [
        ["Bridgewater Associates", "United States Westport, CT", "$98,918"],
        ["Renaissance Technologies", "United States East Setauket, NY", "$70,000"],
        ["Man Group", "United Kingdom London", "$62,300"],
        ["Millennium Management", "United States New York City, NY", "$43,912"],
        ["Elliott Management", "United States New York City, NY", "$42,000"],
        ["BlackRock", "United States New York City, NY", "$39,907"],
        ["Two Sigma Investments", "United States New York City, NY", "$38,842"],
        ["The Children's Investment Fund Management", "United Kingdom London", "$35,000"],
        ["Citadel LLC", "United States Chicago, IL", "$34,340"],
        ["D.E. Shaw & Co.", "United States New York City, NY", "$34,264"],
        ["AQR Capital Management", "United States Greenwich, CT", "$32,100"],
        ["Davidson Kempner Capital Management", "United States New York City, NY", "$31,850"],
        ["Farallon Capital", "United States San Francisco, CA", "$30,000"],
        ["Baupost Group", "United States Boston, MA", "$29,100"],
        ["Marshall Wace", "United Kingdom London", "$27,800"],
        ["Capula Investment Management", "United Kingdom London", "$23,000"],
        ["Canyon Capital Advisors", "United States Los Angeles, CA", "$22,800"],
        ["Wellington Management Company", "United States Boston, MA", "$21,000"],
        ["Viking Global Investors", "United States Greenwich, CT", "$19,950"],
        ["PIMCO", "United States Newport Beach, CA", "$17,453"],
    ]

    match = nlp_vlookup("Capital", words, include_score=True, all_matches=True, threshold=0.5)
    print(match)


import os
import numpy as np

import var

from shutil import copyfile
from gensim.models.keyedvectors import KeyedVectors
from keras.callbacks import ModelCheckpoint
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer
from keras.utils.np_utils import to_categorical

from parse_data import clean_credit, get_glove_vectors, get_data, \
    get_mapping, get_one_hot_vectors, \
    normalize_vectors, get_pos_freqs, \
    remove_stop_words
from definitions_of_models import cnn_model, bi_lstm_model, \
    bi_lstm_cnn_model, cnn_bi_lstm_model, parallel_cnn_bi_lstm_model

def train_model():
    '''
    Trains a model using the parameters specified in var.py. The
    model training checkpoints are saved to var.FOLDER_NAME. A
    copy of the var.py used is also saved in that folder. Only
    the lowest val loss models for each trained model is saved.
    '''
    copyfile('var.py', os.path.join(var.FOLDER_NAME, 'var.py'))

    print("Reading word vectors... ")

    embeddings = None
    if var.USE_WORD2VEC:
        embeddings = KeyedVectors.load_word2vec_format(
            var.WORD2VEC_BIN_PATH, binary=True)
    else:
        embeddings = get_glove_vectors(var.GLOVE_VECTOR_PATH)

    print("--- DONE ---")

    print("Getting input data... ")

    train_labels, train_sentences, train_subjects, train_party, train_credit = get_data(
        var.TRAINING_DATA_PATH)
    val_labels, val_sentences, val_subjects, val_party, val_credit = get_data(
        var.VALIDATION_DATA_PATH)

    print("--- DONE ---")

    if var.NO_STOP_WORDS:
        train_sentences = remove_stop_words(train_sentences)
        val_sentences = remove_stop_words(val_sentences)

    print("Preparing data for model... ")

    # Convert the input sentences into sequences of integers
    # length MAX_SEQUENCE_LENGTH where each integer maps to a
    # word and the sequence only considers the first
    # MAX_SEQUENCE_LENGTH words in the statement being evaluated
    tokenizer = Tokenizer(num_words=var.MAX_NUM_WORDS)
    tokenizer.fit_on_texts(train_sentences)
    train_sequences = tokenizer.texts_to_sequences(train_sentences)
    val_sequences = tokenizer.texts_to_sequences(val_sentences)
    word_index = tokenizer.word_index

    x_train = pad_sequences(train_sequences, maxlen=var.MAX_SEQUENCE_LENGTH)
    x_val = pad_sequences(val_sequences, maxlen=var.MAX_SEQUENCE_LENGTH)
    y_train = to_categorical(np.asarray(
        [var.LABEL_MAPPING[label] for label in train_labels]))
    y_val = to_categorical(np.asarray(
        [var.LABEL_MAPPING[label] for label in val_labels]))

    # Get the Part of Speech frequencies
    x_train_pos = np.asarray(get_pos_freqs(train_sentences))
    x_val_pos = np.asarray(get_pos_freqs(val_sentences))

    # Populate SUBJECT_MAPPING with freq information from training data
    var.SUBJECT_MAPPING = get_mapping(train_subjects)

    # Create one hot vectors for the subject
    x_train_subject = np.asarray(
        get_one_hot_vectors(
            train_subjects,
            var.NUM_SUBJECTS,
            var.SUBJECT_MAPPING))
    x_val_subject = np.asarray(
        get_one_hot_vectors(
            val_subjects,
            var.NUM_SUBJECTS,
            var.SUBJECT_MAPPING))

    # Convert party to list of list format
    train_party = [[party] for party in train_party]
    val_party = [[party] for party in val_party]

    # Populate PARTY_MAPPING with training data
    var.PARTY_MAPPING = get_mapping(train_party)

    # Get One Hot Vectors representing Party
    x_train_party = np.asarray(
        get_one_hot_vectors(
            train_party,
            var.NUM_PARTIES,
            var.PARTY_MAPPING))
    x_val_party = np.asarray(
        get_one_hot_vectors(
            val_party,
            var.NUM_PARTIES,
            var.PARTY_MAPPING))

    # Remove current label from credit vector
    train_credit = clean_credit(train_labels, train_credit)
    val_credit = clean_credit(val_labels, val_credit)

    # Normalize Credit Vector
    x_train_credit = np.asarray(normalize_vectors(train_credit))
    x_val_credit = np.asarray(normalize_vectors(val_credit))

    # Create embedding matrix for embedding layer. Matrix will be
    # (num_words + 1) x EMBEDDING_DIM since word_index starts at 1
    num_words = min(var.MAX_NUM_WORDS, len(word_index))
    embedding_matrix = np.zeros((num_words + 1, var.EMBEDDING_DIM))
    for word, rank in word_index.items():
        if rank <= var.MAX_NUM_WORDS:
            embedding = None
            if var.USE_WORD2VEC:
                if word in embeddings.vocab:
                    embedding = embeddings[word]
            else:
                embedding = embeddings.get(word, None)
            if embedding is not None:
                embedding_matrix[rank] = embedding

    print("--- DONE ---")

    for i in range(var.NUM_MODELS):
        print("Creating model " + str(i + 1) +
              " out of " + str(var.NUM_MODELS) + " ...")
        if var.MODEL_TYPE == "CNN":
            model = cnn_model(embedding_matrix, num_words, pooling=var.POOLING,
                              subject=var.USE_SUBJECTS, party=var.USE_PARTY,
                              credit=var.USE_CREDIT, pos=var.USE_POS)
        elif var.MODEL_TYPE == "BI_LSTM":
            model = bi_lstm_model(
                embedding_matrix,
                num_words,
                subject=var.USE_SUBJECTS,
                party=var.USE_PARTY,
                credit=var.USE_CREDIT,
                pos=var.USE_POS)
        elif var.MODEL_TYPE == "BI_LSTM_CNN":
            model = bi_lstm_cnn_model(
                embedding_matrix,
                num_words,
                pooling=var.POOLING,
                subject=var.USE_SUBJECTS,
                party=var.USE_PARTY,
                credit=var.USE_CREDIT,
                pos=var.USE_POS)
        elif var.MODEL_TYPE == "CNN_BI_LSTM":
            model = cnn_bi_lstm_model(
                embedding_matrix,
                num_words,
                subject=var.USE_SUBJECTS,
                party=var.USE_PARTY,
                credit=var.USE_CREDIT,
                pos=var.USE_POS)
        elif var.MODEL_TYPE == "PARALLEL":
            model = parallel_cnn_bi_lstm_model(
                embedding_matrix,
                num_words,
                pooling=var.POOLING,
                subject=var.USE_SUBJECTS,
                party=var.USE_PARTY,
                credit=var.USE_CREDIT,
                pos=var.USE_POS)
        else:
            raise Exception("Invalid MODEL_TYPE")

        print("--- DONE ---")

        print("Training model... ")

        # Save trained model after each epoch

        checkpoint_file = os.path.join(
            var.FOLDER_NAME, str(i).zfill(2) + '_' + var.FILE_NAME)
        checkpoint = ModelCheckpoint(
            checkpoint_file,
            monitor='val_loss',
            verbose=1,
            save_best_only=True)
        callbacks = [checkpoint]

        train_input = [x_train]
        val_input = [x_val]
        if var.USE_SUBJECTS:
            train_input.append(x_train_subject)
            val_input.append(x_val_subject)
        if var.USE_PARTY:
            train_input.append(x_train_party)
            val_input.append(x_val_party)
        if var.USE_CREDIT:
            train_input.append(x_train_credit)
            val_input.append(x_val_credit)
        if var.USE_POS:
            train_input.append(x_train_pos)
            val_input.append(x_val_pos)

        print("train_input_size", len(train_input))

        model.fit(train_input, y_train,
                  validation_data=(val_input, y_val),
                  epochs=var.NUM_EPOCHS,
                  batch_size=var.BATCH_SIZE,
                  callbacks=callbacks)

        print("--- DONE ---")

        model.summary()
        del model

if __name__ == '__main__':
    train_model()